From 281b63c20573a51627ba11447ad9caeb04f9d775 Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Tue, 25 Nov 2025 05:34:35 +0000 Subject: [PATCH 01/19] Cleanup Script: Phase 1 --- tools/cloud-build/project-cleanup.yaml | 203 +++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 tools/cloud-build/project-cleanup.yaml diff --git a/tools/cloud-build/project-cleanup.yaml b/tools/cloud-build/project-cleanup.yaml new file mode 100644 index 0000000000..17260a282a --- /dev/null +++ b/tools/cloud-build/project-cleanup.yaml @@ -0,0 +1,203 @@ +# cloudbuild.yaml +substitutions: + _DRY_RUN: "true" # Set to "false" to enable actual deletion + _PROJECT_ID: "hpc-toolkit-dev" + _EXCLUSION_BUCKET: "hpc-ctk1357" # CHANGE: Your bucket name + _EXCLUSION_FILE: "cleanup/exclusions.txt" # CHANGE: Path to the exclusion file in the bucket + +steps: +- name: gcr.io/cloud-builders/gcloud + entrypoint: /bin/bash + args: + - '-c' + - | + set -e # Exit immediately if a command exits with a non-zero status. + set -u # Treat unset variables as an error. + set -o pipefail # Return value of a pipeline is the status of the last command to exit with a non-zero status. + + PROJECT_ID="${_PROJECT_ID}" + DRY_RUN="${_DRY_RUN}" + EXCLUSION_GCS_PATH="gs://${_EXCLUSION_BUCKET}/${_EXCLUSION_FILE}" + + echo "--- STARTING RESOURCE CLEANUP in project $$PROJECT_ID ---" + echo "DRY_RUN mode: $$DRY_RUN" + echo "Fetching exclusion list from: $$EXCLUSION_GCS_PATH" + + # --- Safety Mechanism: Load Exclusion List from GCS --- + EXCLUDE_LIST=() + while IFS= read -r line || [[ -n "$$line" ]]; do + # Trim whitespace and ignore empty lines or comments + trimmed_line=$(echo "$$line" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | grep -v '^#' | grep -v '^$$') + if [[ -n "$$trimmed_line" ]]; then + EXCLUDE_LIST+=("$$trimmed_line") + fi + done < <(gcloud storage cat "$$EXCLUSION_GCS_PATH") + + if [[ $${#EXCLUDE_LIST[@]} -eq 0 ]]; then + echo "WARNING: Exclusion list from $$EXCLUSION_GCS_PATH is empty or the file was not found." + # exit 1 # Consider exiting if the list is mandatory + fi + + echo "Exclusion list loaded with $${#EXCLUDE_LIST[@]} entries:" + printf " - %s\n" "$${EXCLUDE_LIST[@]}" + + is_excluded() { + local resource_name="$$1" + # Always exclude the default network + for excluded in "$${EXCLUDE_LIST[@]}"; do + if [[ "$$resource_name" == "$$excluded" ]]; then + return 0 # True - is excluded + fi + done + return 1 # False - not excluded + } + + run_command() { + if [[ "$$DRY_RUN" == "true" ]]; then + echo "[DRY RUN] Would run: gcloud $$@" + else + echo "[EXECUTE] No running" + fi + } + + echo "--- Deletion Phase 1: GKE Clusters ---" + gcloud container clusters list --project="$$PROJECT_ID" --format="value(name,zone)" | while read -r NAME LOCATION; do + if ! is_excluded "$$NAME"; then + run_command container clusters delete "$$NAME" --project="$$PROJECT_ID" --zone="$$LOCATION" --quiet + else + echo "Skipping GKE Cluster: $$NAME (In exclusion list)" + fi + done + + echo "--- Deletion Phase 1: Compute Instances ---" + gcloud compute instances list --project="$$PROJECT_ID" --format="value(name,zone)" | while read -r NAME ZONE; do + if ! is_excluded "$$NAME"; then + run_command compute instances delete "$$NAME" --project="$$PROJECT_ID" --zone="$$ZONE" --quiet + else + echo "Skipping Instance: $$NAME (In exclusion list)" + fi + done + + echo "--- Deletion Phase 1: Filestore Instances ---" + gcloud filestore instances list --project="$$PROJECT_ID" --format="value(name,location)" | while read -r NAME LOCATION; do + if ! is_excluded "$$NAME"; then + run_command filestore instances delete "$$NAME" --project="$$PROJECT_ID" --location="$$LOCATION" --quiet + else + echo "Skipping Filestore: $$NAME (In exclusion list)" + fi + done + + REGIONS=$(gcloud compute regions list --project="$$PROJECT_ID" --format="value(name)") + + echo "--- Deletion Phase 2: Routers ---" + for REGION in $$REGIONS; do + gcloud compute routers list --project="$$PROJECT_ID" --filter="region:($$REGION)" --format="value(name)" | while read -r NAME; do + if ! is_excluded "$$NAME"; then + run_command compute routers delete "$$NAME" --project="$$PROJECT_ID" --region="$$REGION" --quiet + else + echo "Skipping Router: $$NAME in $$REGION (In exclusion list)" + fi + done + done + + echo "--- Deletion Phase 2: Firewall Rules ---" + gcloud compute firewall-rules list --project="$$PROJECT_ID" --format="value(name,network)" | while read -r NAME NETWORK; do + NETWORK_NAME=$(basename "$$NETWORK") + if ! is_excluded "$$NAME"; then + if [[ "$$NETWORK_NAME" == "default" ]]; then + echo "Skipping Firewall: $$NAME (On default network, not typically managed by tests)" + continue + fi + run_command compute firewall-rules delete "$$NAME" --project="$$PROJECT_ID" --quiet + else + echo "Skipping Firewall: $$NAME (In exclusion list)" + fi + done + + echo "--- Deletion Phase 3: Service Networking Connections ---" + gcloud compute networks list --project="$$PROJECT_ID" --format="value(name)" | while read -r NETWORK_NAME; do + if is_excluded "$$NETWORK_NAME"; then + echo "Skipping Service Networking deletion for excluded network: $$NETWORK_NAME" + continue + fi + + echo "Attempting to remove Service Networking connection from network: $$NETWORK_NAME" + # The --service defaults to servicenetworking.googleapis.com + # This command will not fail the script if the connection doesn't exist. + run_command services vpc-peerings delete --network="$$NETWORK_NAME" --project="$$PROJECT_ID" --quiet + done + + echo "--- Deletion Phase 4: Subnetworks ---" + for REGION in $$REGIONS; do + gcloud compute networks subnets list --project="$$PROJECT_ID" --filter="region:($$REGION)" --format="value(name,network)" | while read -r NAME NETWORK; do + NETWORK_NAME=$(basename "$$NETWORK") + if ! is_excluded "$$NAME"; then + run_command compute networks subnets delete "$$NAME" --project="$$PROJECT_ID" --region="$$REGION" --quiet + else + echo "Skipping Subnet: $$NAME in $$REGION (In exclusion list)" + fi + done + done + + echo "--- Deletion Phase 5: Networks ---" + gcloud compute networks list --project="$$PROJECT_ID" --format="value(name)" | while read -r NAME; do + if ! is_excluded "$$NAME"; then + run_command compute networks delete "$$NAME" --project="$$PROJECT_ID" --quiet + else + echo "Skipping Network: $$NAME (In exclusion list or is default)" + fi + done + + echo "--- Deletion Phase 6: Instance Templates ---" + gcloud compute instance-templates list --project="$$PROJECT_ID" --format="value(name)" | while read -r NAME; do + if ! is_excluded "$$NAME"; then + run_command compute instance-templates delete "$$NAME" --project="$$PROJECT_ID" --quiet + else + echo "Skipping Instance Template: $$NAME (In exclusion list)" + fi + done + + echo "--- Deletion Phase 7: Disks ---" + gcloud compute disks list --project="$$PROJECT_ID" --format="value(name,zone)" | while read -r NAME ZONE; do + if ! is_excluded "$$NAME"; then + run_command compute disks delete "$$NAME" --project="$$PROJECT_ID" --zone="$$ZONE" --quiet + else + echo "Skipping Disk: $$NAME (In exclusion list)" + fi + done + + echo "--- Deletion Phase 8: Addresses ---" + for REGION in $$REGIONS; do + gcloud compute addresses list --project="$$PROJECT_ID" --filter="region:($$REGION)" --format="value(name)" | while read -r NAME; do + if ! is_excluded "$$NAME"; then + run_command compute addresses delete "$$NAME" --project="$$PROJECT_ID" --region="$$REGION" --quiet + else + echo "Skipping Address: $$NAME in $$REGION (In exclusion list)" + fi + done + done + gcloud compute addresses list --project="$$PROJECT_ID" --global --format="value(name)" | while read -r NAME; do + if ! is_excluded "$$NAME"; then + run_command compute addresses delete "$$NAME" --project="$$PROJECT_ID" --global --quiet + else + echo "Skipping Global Address: $$NAME (In exclusion list)" + fi + done + + echo "--- Deletion Phase 9: GCS Buckets (CAUTION) ---" + gcloud storage ls --project="$$PROJECT_ID" | while read -r BUCKET; do + BUCKET_NAME=$(echo "$$BUCKET" | sed 's|gs://||; s|/||') + if ! is_excluded "$$BUCKET_NAME"; then + if [[ "$$DRY_RUN" == "true" ]]; then + echo "[DRY RUN - BUCKET] Would run: gcloud storage rm -r $$BUCKET" + else + echo "[EXECUTE - BUCKET] Running: gcloud storage rm -r $$BUCKET" + gcloud storage rm -r "$$BUCKET" + fi + else + echo "Skipping Bucket: $$BUCKET_NAME (In exclusion list)" + fi + done + + echo "--- CLEANUP SCRIPT FINISHED ---" + From 0af3b921e16929253359bb1291e085df9483c04e Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Tue, 2 Dec 2025 17:54:49 +0000 Subject: [PATCH 02/19] Cleanup Phase 2 --- changescript.txt | 3492 +++++++++++++++++++++ checking.txt | 3090 ++++++++++++++++++ cleanup.sh | 711 +++++ disk.txt | 3015 ++++++++++++++++++ dockerimages.txt | 1352 ++++++++ exclusions.txt | 72 + filestores.txt | 179 ++ firewalls.txt | 2155 +++++++++++++ iam.txt | 4836 ++++++++++++++++++++++++++++ images.txt | 7315 +++++++++++++++++++++++++++++++++++++++++++ instances.txt | 2847 +++++++++++++++++ network.txt | 2708 ++++++++++++++++ networks.txt | 2086 ++++++++++++ peer.txt | 566 ++++ policy-bindings.txt | 2834 +++++++++++++++++ rdisk.txt | 373 +++ routers.txt | 1350 ++++++++ subnetworks.txt | 3453 ++++++++++++++++++++ template.txt | 4227 +++++++++++++++++++++++++ 19 files changed, 46661 insertions(+) create mode 100644 changescript.txt create mode 100644 checking.txt create mode 100755 cleanup.sh create mode 100644 disk.txt create mode 100644 dockerimages.txt create mode 100644 exclusions.txt create mode 100644 filestores.txt create mode 100644 firewalls.txt create mode 100644 iam.txt create mode 100644 images.txt create mode 100644 instances.txt create mode 100644 network.txt create mode 100644 networks.txt create mode 100644 peer.txt create mode 100644 policy-bindings.txt create mode 100644 rdisk.txt create mode 100644 routers.txt create mode 100644 subnetworks.txt create mode 100644 template.txt diff --git a/changescript.txt b/changescript.txt new file mode 100644 index 0000000000..f198636a4d --- /dev/null +++ b/changescript.txt @@ -0,0 +1,3492 @@ +[2025-11-28 13:44:49] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 13:44:49] [INFO] Time Cutoff (General): 2025-11-28T12:44:49+0000 +[2025-11-28 13:44:49] [INFO] Time Cutoff (Images): 2025-09-29T13:44:49+0000 +[2025-11-28 13:44:49] [INFO] Delete Limit per Type: 10 +[2025-11-28 13:44:49] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 13:44:50] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 13:44:51] [INFO] No Service Accounts found matching prefix. +[2025-11-28 13:44:51] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 13:44:53] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 13:44:53] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 13:44:54] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 13:44:54] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 13:44:54] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 13:44:54] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 13:44:54] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 13:44:54] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 13:44:54] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 13:44:57] [INFO] No Filestore found matching criteria. +[2025-11-28 13:44:57] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 13:44:58] [DRY-RUN] Would delete VM Image: a3lavtest-u22-20250825t071123z +[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3lavtest-u22-20250825t121357z +[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-ctkhar-20250610t181958z +[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-2c1a21-u22-20250828t204934z +[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-45dc0b-u22-20250819t184231z +[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-5a8205-u22-20250822t221818z +[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-96bae0-u22-20250822t232353z +[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-986401-u22-20250819t055812z +[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-c6360d-u22-20250819t173658z +[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3mergesc-slurm-u22-20250929t131528z +[2025-11-28 13:44:59] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 13:44:59] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 13:45:11] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 13:45:13] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 13:45:13] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 13:45:13] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 13:45:13] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 13:45:13] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 13:45:13] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 13:45:15] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 13:45:15] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 13:45:16] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:45:17] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:45:17] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 13:45:17] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 13:45:17] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 13:45:17] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 13:45:17] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 13:45:18] [INFO] No Global Address found matching criteria. +[2025-11-28 13:45:18] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 13:45:35] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 13:45:37] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 13:45:37] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 13:45:37] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 13:45:37] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 13:45:37] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 13:45:37] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 13:45:37] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 13:45:37] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 13:45:39] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:39] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 13:45:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:45:41] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 13:45:42] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 13:45:42] [INFO] CLEANUP RUN FINISHED +[2025-11-28 13:47:53] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 13:47:53] [INFO] Time Cutoff (General): 2025-11-28T12:47:53+0000 +[2025-11-28 13:47:53] [INFO] Time Cutoff (Images): 2025-09-29T13:47:53+0000 +[2025-11-28 13:47:53] [INFO] Delete Limit per Type: 10 +[2025-11-28 13:47:53] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 13:47:54] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 13:47:55] [INFO] No Service Accounts found matching prefix. +[2025-11-28 13:47:55] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 13:47:57] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 13:47:57] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 13:47:59] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 13:47:59] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 13:47:59] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 13:47:59] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 13:47:59] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 13:47:59] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 13:47:59] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 13:48:01] [INFO] No Filestore found matching criteria. +[2025-11-28 13:48:01] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 13:48:03] [EXECUTE] Deleting VM Image: a3lavtest-u22-20250825t071123z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3lavtest-u22-20250825t071123z]. +[2025-11-28 13:48:11] [SUCCESS] Deleted a3lavtest-u22-20250825t071123z +[2025-11-28 13:48:11] [EXECUTE] Deleting VM Image: a3lavtest-u22-20250825t121357z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3lavtest-u22-20250825t121357z]. +[2025-11-28 13:48:17] [SUCCESS] Deleted a3lavtest-u22-20250825t121357z +[2025-11-28 13:48:17] [EXECUTE] Deleting VM Image: a3m-ctkhar-20250610t181958z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-ctkhar-20250610t181958z]. +[2025-11-28 13:48:25] [SUCCESS] Deleted a3m-ctkhar-20250610t181958z +[2025-11-28 13:48:25] [EXECUTE] Deleting VM Image: a3m-slurm-2c1a21-u22-20250828t204934z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-2c1a21-u22-20250828t204934z]. +[2025-11-28 13:48:32] [SUCCESS] Deleted a3m-slurm-2c1a21-u22-20250828t204934z +[2025-11-28 13:48:32] [EXECUTE] Deleting VM Image: a3m-slurm-45dc0b-u22-20250819t184231z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-45dc0b-u22-20250819t184231z]. +[2025-11-28 13:48:41] [SUCCESS] Deleted a3m-slurm-45dc0b-u22-20250819t184231z +[2025-11-28 13:48:41] [EXECUTE] Deleting VM Image: a3m-slurm-5a8205-u22-20250822t221818z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-5a8205-u22-20250822t221818z]. +[2025-11-28 13:48:48] [SUCCESS] Deleted a3m-slurm-5a8205-u22-20250822t221818z +[2025-11-28 13:48:49] [EXECUTE] Deleting VM Image: a3m-slurm-96bae0-u22-20250822t232353z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-96bae0-u22-20250822t232353z]. +[2025-11-28 13:48:56] [SUCCESS] Deleted a3m-slurm-96bae0-u22-20250822t232353z +[2025-11-28 13:48:56] [EXECUTE] Deleting VM Image: a3m-slurm-986401-u22-20250819t055812z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-986401-u22-20250819t055812z]. +[2025-11-28 13:49:03] [SUCCESS] Deleted a3m-slurm-986401-u22-20250819t055812z +[2025-11-28 13:49:03] [EXECUTE] Deleting VM Image: a3m-slurm-c6360d-u22-20250819t173658z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-c6360d-u22-20250819t173658z]. +[2025-11-28 13:49:11] [SUCCESS] Deleted a3m-slurm-c6360d-u22-20250819t173658z +[2025-11-28 13:49:11] [EXECUTE] Deleting VM Image: a3mergesc-slurm-u22-20250929t131528z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3mergesc-slurm-u22-20250929t131528z]. +[2025-11-28 13:49:19] [SUCCESS] Deleted a3mergesc-slurm-u22-20250929t131528z +[2025-11-28 13:49:19] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 13:49:19] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 13:49:32] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 13:49:33] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 13:49:33] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 13:49:33] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 13:49:33] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 13:49:33] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 13:49:33] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 13:49:35] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 13:49:35] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 13:49:37] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 13:49:37] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 13:49:39] [INFO] No Global Address found matching criteria. +[2025-11-28 13:49:39] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 13:49:56] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 13:49:58] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 13:49:58] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 13:49:58] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 13:49:58] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 13:49:58] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 13:49:58] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 13:49:58] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 13:49:58] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 13:50:00] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:01] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:01] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:01] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:01] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:01] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:01] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 13:50:02] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:50:02] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 13:50:04] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 13:50:04] [INFO] CLEANUP RUN FINISHED +[2025-11-28 13:51:01] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 13:51:01] [INFO] Time Cutoff (General): 2025-11-28T12:51:01+0000 +[2025-11-28 13:51:01] [INFO] Time Cutoff (Images): 2025-09-29T13:51:01+0000 +[2025-11-28 13:51:01] [INFO] Delete Limit per Type: 50 +[2025-11-28 13:51:01] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 13:51:01] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 13:51:03] [INFO] No Service Accounts found matching prefix. +[2025-11-28 13:51:03] [INFO] --- Processing: GKE Cluster (Limit: 50) --- +[2025-11-28 13:51:04] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 13:51:04] [INFO] --- Processing: Compute Instance (Limit: 50) --- +[2025-11-28 13:51:06] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 13:51:06] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 13:51:06] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 13:51:06] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 13:51:06] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 13:51:06] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 13:51:06] [INFO] --- Processing: Filestore (Limit: 50) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 13:51:08] [INFO] No Filestore found matching criteria. +[2025-11-28 13:51:08] [INFO] --- Processing: VM Images (Limit: 50) --- +[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a3u-image-u22-20250325t162635z +[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a3u-pp-u22-20250324t225357z +[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a3u-slurm-561cbc-u22-20250819t055542z +[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a3u-slurm-e26388-u22-20250822t220551z +[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-0a7d41-u22-20250819t061447z +[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-90f096-u22-20250806t160005z +[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-a69479-u22-20250731t113616z +[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-122222-u22-20250825t101805z +[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-1e376d-u22-20250829t132242z +[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-2c330f-u22-20250919t101939z +[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-3c7a77-u22-20250903t145654z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-5535cc-u22-20250828t101953z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-570431-u22-20250822t204218z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-63819f-u22-20250905t101930z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-6dfb8d-u22-20250912t144538z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-76c357-u22-20250908t085635z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-869494-u22-20250825t042131z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-8b9201-u22-20250917t101932z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-946daf-u22-20250721t172352z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-a6d7f5-u22-20250825t142931z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-aa5bfc-u22-20250916t101953z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ac8613-u22-20250826t060423z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ad3d05-u22-20250918t101923z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-add7ca-u22-20250902t164931z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-b00d3e-u22-20250926t172512z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-bb93b4-u22-20250926t113807z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-bc10ee-u22-20250922t101931z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-c74ec8-u22-20250915t065803z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-d60a9e-u22-20250819t055629z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-e2a125-u22-20250901t101947z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-e3f6d6-u22-20250912t002144z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ec0d72-u22-20250904t101939z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-edeb3b-u22-20250912t074741z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ee0684-u22-20250814t130214z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-f0afe0-u22-20250923t110504z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4high-image-builder-20250214t220935z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: arpit-a3-toolkitest-u22-20250825t164840z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250904t073016z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250910t104131z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250910t144314z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: arpit-toolkit-u22-20250826t034632z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: chs-dcgmi-metric-u22-20250925t121709z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: common-slurm-image-20250725t234825z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: cx-a3u-u22-20250701t080501z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: harsh-a4-image +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t085040z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t101254z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t163709z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t214154z +[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: pbspro0 +[2025-11-28 13:51:11] [INFO] Hit delete limit (50) for VM Images. +[2025-11-28 13:51:11] [INFO] --- Processing: Docker Images (Limit: 50) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 13:51:23] [INFO] --- Processing: Cloud Router (Limit: 50) --- +[2025-11-28 13:51:25] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 13:51:25] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 13:51:25] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 13:51:25] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 13:51:25] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 13:51:25] [INFO] --- Processing: Firewall Rules (Limit: 50) --- +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 13:51:27] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 13:51:27] [INFO] --- Processing: Regional Address (Limit: 50) --- +[2025-11-28 13:51:29] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 13:51:29] [INFO] --- Processing: Global Address (Limit: 50) --- +[2025-11-28 13:51:31] [INFO] No Global Address found matching criteria. +[2025-11-28 13:51:31] [INFO] --- Processing: Service Networking Connections (Limit: 50) --- +[2025-11-28 13:51:47] [INFO] --- Processing: Zonal Disk (Limit: 50) --- +[2025-11-28 13:51:49] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 13:51:49] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 13:51:49] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 13:51:49] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 13:51:49] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 13:51:49] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 13:51:49] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 13:51:49] [INFO] --- Processing: Subnetworks (Limit: 50) --- +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 13:51:51] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:51] [INFO] --- Processing: VPC Networks (Limit: 50) --- +[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 13:51:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:51:53] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 50) --- +[2025-11-28 13:51:54] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 13:51:54] [INFO] CLEANUP RUN FINISHED +[2025-11-28 13:58:41] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 13:58:41] [INFO] Time Cutoff (General): 2025-11-28T12:58:41+0000 +[2025-11-28 13:58:41] [INFO] Time Cutoff (Images): 2025-09-29T13:58:41+0000 +[2025-11-28 13:58:41] [INFO] Delete Limit per Type: 10 +[2025-11-28 13:58:41] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 13:58:41] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 13:58:43] [INFO] No Service Accounts found matching prefix. +[2025-11-28 13:58:43] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 13:58:44] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 13:58:44] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 13:58:46] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 13:58:46] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 13:58:46] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 13:58:46] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 13:58:46] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 13:58:46] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 13:58:46] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 13:58:48] [INFO] No Filestore found matching criteria. +[2025-11-28 13:58:48] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 13:58:50] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a3u-pp-u22-20250324t225357z +[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a3u-slurm-561cbc-u22-20250819t055542z +[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a3u-slurm-e26388-u22-20250822t220551z +[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-0a7d41-u22-20250819t061447z +[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-90f096-u22-20250806t160005z +[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-a69479-u22-20250731t113616z +[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-122222-u22-20250825t101805z +[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-1e376d-u22-20250829t132242z +[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-2c330f-u22-20250919t101939z +[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-3c7a77-u22-20250903t145654z +[2025-11-28 13:58:51] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 13:58:51] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 13:59:03] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 13:59:05] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 13:59:05] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 13:59:05] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 13:59:05] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 13:59:05] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 13:59:05] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 13:59:06] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:59:06] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:59:06] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:59:06] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:59:06] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 13:59:07] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 13:59:07] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 13:59:08] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 13:59:08] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 13:59:10] [INFO] No Global Address found matching criteria. +[2025-11-28 13:59:10] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 13:59:27] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 13:59:28] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 13:59:28] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 13:59:28] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 13:59:28] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 13:59:28] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 13:59:28] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 13:59:28] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 13:59:28] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 13:59:30] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:31] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 13:59:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 13:59:32] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 13:59:34] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 13:59:34] [INFO] CLEANUP RUN FINISHED +./cleanup.sh: line 521: n: command not found +[2025-11-28 13:59:40] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 13:59:40] [INFO] Time Cutoff (General): 2025-11-28T12:59:40+0000 +[2025-11-28 13:59:40] [INFO] Time Cutoff (Images): 2025-09-29T13:59:40+0000 +[2025-11-28 13:59:40] [INFO] Delete Limit per Type: 10 +[2025-11-28 13:59:40] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 13:59:40] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 13:59:42] [INFO] No Service Accounts found matching prefix. +[2025-11-28 13:59:42] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 13:59:44] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 13:59:44] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 13:59:46] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 13:59:46] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 13:59:46] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 13:59:46] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 13:59:46] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 13:59:46] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 13:59:46] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 13:59:48] [INFO] No Filestore found matching criteria. +[2025-11-28 13:59:48] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 13:59:50] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 13:59:50] [EXECUTE] Deleting VM Image: a3u-pp-u22-20250324t225357z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3u-pp-u22-20250324t225357z]. +[2025-11-28 13:59:57] [SUCCESS] Deleted a3u-pp-u22-20250324t225357z +[2025-11-28 13:59:57] [EXECUTE] Deleting VM Image: a3u-slurm-561cbc-u22-20250819t055542z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3u-slurm-561cbc-u22-20250819t055542z]. +[2025-11-28 14:00:04] [SUCCESS] Deleted a3u-slurm-561cbc-u22-20250819t055542z +[2025-11-28 14:00:04] [EXECUTE] Deleting VM Image: a3u-slurm-e26388-u22-20250822t220551z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3u-slurm-e26388-u22-20250822t220551z]. +[2025-11-28 14:00:11] [SUCCESS] Deleted a3u-slurm-e26388-u22-20250822t220551z +[2025-11-28 14:00:11] [EXECUTE] Deleting VM Image: a4h-slurm-0a7d41-u22-20250819t061447z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-0a7d41-u22-20250819t061447z]. +[2025-11-28 14:00:18] [SUCCESS] Deleted a4h-slurm-0a7d41-u22-20250819t061447z +[2025-11-28 14:00:18] [EXECUTE] Deleting VM Image: a4h-slurm-90f096-u22-20250806t160005z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-90f096-u22-20250806t160005z]. +[2025-11-28 14:00:25] [SUCCESS] Deleted a4h-slurm-90f096-u22-20250806t160005z +[2025-11-28 14:00:25] [EXECUTE] Deleting VM Image: a4h-slurm-a69479-u22-20250731t113616z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-a69479-u22-20250731t113616z]. +[2025-11-28 14:00:32] [SUCCESS] Deleted a4h-slurm-a69479-u22-20250731t113616z +[2025-11-28 14:00:32] [EXECUTE] Deleting VM Image: a4h-slurm-flex-122222-u22-20250825t101805z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-122222-u22-20250825t101805z]. +[2025-11-28 14:00:39] [SUCCESS] Deleted a4h-slurm-flex-122222-u22-20250825t101805z +[2025-11-28 14:00:39] [EXECUTE] Deleting VM Image: a4h-slurm-flex-1e376d-u22-20250829t132242z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-1e376d-u22-20250829t132242z]. +[2025-11-28 14:00:46] [SUCCESS] Deleted a4h-slurm-flex-1e376d-u22-20250829t132242z +[2025-11-28 14:00:46] [EXECUTE] Deleting VM Image: a4h-slurm-flex-2c330f-u22-20250919t101939z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-2c330f-u22-20250919t101939z]. +[2025-11-28 14:00:54] [SUCCESS] Deleted a4h-slurm-flex-2c330f-u22-20250919t101939z +[2025-11-28 14:00:54] [EXECUTE] Deleting VM Image: a4h-slurm-flex-3c7a77-u22-20250903t145654z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-3c7a77-u22-20250903t145654z]. +[2025-11-28 14:01:01] [SUCCESS] Deleted a4h-slurm-flex-3c7a77-u22-20250903t145654z +[2025-11-28 14:01:01] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 14:01:01] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 14:01:14] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 14:01:15] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 14:01:15] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 14:01:15] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 14:01:15] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 14:01:15] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 14:01:15] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 14:01:17] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 14:01:17] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 14:01:19] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 14:01:19] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 14:01:21] [INFO] No Global Address found matching criteria. +[2025-11-28 14:01:21] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 14:01:37] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 14:01:39] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 14:01:39] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 14:01:39] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 14:01:39] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 14:01:39] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 14:01:39] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 14:01:39] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 14:01:39] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 14:01:41] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:41] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 14:01:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:01:43] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 14:01:44] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 14:01:44] [INFO] CLEANUP RUN FINISHED +[2025-11-28 14:02:39] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 14:02:39] [INFO] Time Cutoff (General): 2025-11-28T13:02:39+0000 +[2025-11-28 14:02:39] [INFO] Time Cutoff (Images): 2025-09-29T14:02:39+0000 +[2025-11-28 14:02:39] [INFO] Delete Limit per Type: 10 +[2025-11-28 14:02:39] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 14:02:39] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 14:02:41] [INFO] No Service Accounts found matching prefix. +[2025-11-28 14:02:41] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 14:02:43] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 14:02:43] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 14:02:44] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 14:02:44] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 14:02:44] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 14:02:44] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 14:02:44] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 14:02:44] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 14:02:44] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 14:02:47] [INFO] No Filestore found matching criteria. +[2025-11-28 14:02:47] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 14:02:49] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-5535cc-u22-20250828t101953z +[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-570431-u22-20250822t204218z +[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-63819f-u22-20250905t101930z +[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-6dfb8d-u22-20250912t144538z +[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-76c357-u22-20250908t085635z +[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-869494-u22-20250825t042131z +[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-8b9201-u22-20250917t101932z +[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-946daf-u22-20250721t172352z +[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-a6d7f5-u22-20250825t142931z +[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-aa5bfc-u22-20250916t101953z +[2025-11-28 14:02:49] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 14:02:49] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 14:03:01] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 14:03:03] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 14:03:03] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 14:03:03] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 14:03:03] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 14:03:03] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 14:03:03] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 14:03:05] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 14:03:05] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 14:03:07] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 14:03:07] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 14:03:08] [INFO] No Global Address found matching criteria. +[2025-11-28 14:03:08] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 14:03:25] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 14:03:27] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 14:03:27] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 14:03:27] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 14:03:27] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 14:03:27] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 14:03:27] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 14:03:27] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 14:03:27] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 14:03:29] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:29] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 14:03:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 14:03:31] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 14:03:32] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 14:03:32] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:17:44] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:17:45] [INFO] Time Cutoff (General): 2025-11-28T14:17:44+0000 +[2025-11-28 15:17:45] [INFO] Time Cutoff (Images): 2025-09-29T15:17:44+0000 +[2025-11-28 15:17:45] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:17:45] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:17:45] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 15:17:47] [INFO] No Service Accounts found matching prefix. +[2025-11-28 15:17:47] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 15:17:48] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 15:17:48] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 15:17:50] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:17:50] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:17:50] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:17:50] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:17:50] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:17:50] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 15:17:50] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 15:17:52] [INFO] No Filestore found matching criteria. +[2025-11-28 15:17:52] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:17:54] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-5535cc-u22-20250828t101953z +[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-570431-u22-20250822t204218z +[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-63819f-u22-20250905t101930z +[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-6dfb8d-u22-20250912t144538z +[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-76c357-u22-20250908t085635z +[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-869494-u22-20250825t042131z +[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-8b9201-u22-20250917t101932z +[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-946daf-u22-20250721t172352z +[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-a6d7f5-u22-20250825t142931z +[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-aa5bfc-u22-20250916t101953z +[2025-11-28 15:17:54] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:17:54] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 15:18:07] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 15:18:09] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 15:18:09] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 15:18:09] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 15:18:09] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 15:18:09] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 15:18:09] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 15:18:11] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 15:18:11] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 15:18:13] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 15:18:13] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 15:18:15] [INFO] No Global Address found matching criteria. +[2025-11-28 15:18:15] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 15:18:31] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 15:18:33] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:18:33] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:18:33] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:18:33] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:18:33] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:18:33] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 15:18:33] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 15:18:33] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 15:18:35] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:35] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 15:18:37] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:18:37] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 15:18:38] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 15:18:38] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:19:39] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:19:39] [INFO] Time Cutoff (General): 2025-11-28T14:19:39+0000 +[2025-11-28 15:19:39] [INFO] Time Cutoff (Images): 2025-09-29T15:19:39+0000 +[2025-11-28 15:19:39] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:19:39] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:19:39] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 15:19:41] [INFO] No Service Accounts found matching prefix. +[2025-11-28 15:19:41] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 15:19:42] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 15:19:42] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 15:19:44] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:19:44] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:19:44] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:19:44] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:19:44] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:19:44] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 15:19:44] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 15:19:46] [INFO] No Filestore found matching criteria. +[2025-11-28 15:19:46] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:19:48] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:19:48] [EXECUTE] Deleting VM Image: a4h-slurm-flex-5535cc-u22-20250828t101953z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-5535cc-u22-20250828t101953z]. +[2025-11-28 15:19:55] [SUCCESS] Deleted a4h-slurm-flex-5535cc-u22-20250828t101953z +[2025-11-28 15:19:55] [EXECUTE] Deleting VM Image: a4h-slurm-flex-570431-u22-20250822t204218z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-570431-u22-20250822t204218z]. +[2025-11-28 15:20:03] [SUCCESS] Deleted a4h-slurm-flex-570431-u22-20250822t204218z +[2025-11-28 15:20:03] [EXECUTE] Deleting VM Image: a4h-slurm-flex-63819f-u22-20250905t101930z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-63819f-u22-20250905t101930z]. +[2025-11-28 15:20:10] [SUCCESS] Deleted a4h-slurm-flex-63819f-u22-20250905t101930z +[2025-11-28 15:20:10] [EXECUTE] Deleting VM Image: a4h-slurm-flex-6dfb8d-u22-20250912t144538z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-6dfb8d-u22-20250912t144538z]. +[2025-11-28 15:20:18] [SUCCESS] Deleted a4h-slurm-flex-6dfb8d-u22-20250912t144538z +[2025-11-28 15:20:18] [EXECUTE] Deleting VM Image: a4h-slurm-flex-76c357-u22-20250908t085635z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-76c357-u22-20250908t085635z]. +[2025-11-28 15:20:26] [SUCCESS] Deleted a4h-slurm-flex-76c357-u22-20250908t085635z +[2025-11-28 15:20:26] [EXECUTE] Deleting VM Image: a4h-slurm-flex-869494-u22-20250825t042131z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-869494-u22-20250825t042131z]. +[2025-11-28 15:20:33] [SUCCESS] Deleted a4h-slurm-flex-869494-u22-20250825t042131z +[2025-11-28 15:20:33] [EXECUTE] Deleting VM Image: a4h-slurm-flex-8b9201-u22-20250917t101932z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-8b9201-u22-20250917t101932z]. +[2025-11-28 15:20:40] [SUCCESS] Deleted a4h-slurm-flex-8b9201-u22-20250917t101932z +[2025-11-28 15:20:40] [EXECUTE] Deleting VM Image: a4h-slurm-flex-946daf-u22-20250721t172352z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-946daf-u22-20250721t172352z]. +[2025-11-28 15:20:48] [SUCCESS] Deleted a4h-slurm-flex-946daf-u22-20250721t172352z +[2025-11-28 15:20:48] [EXECUTE] Deleting VM Image: a4h-slurm-flex-a6d7f5-u22-20250825t142931z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-a6d7f5-u22-20250825t142931z]. +[2025-11-28 15:20:55] [SUCCESS] Deleted a4h-slurm-flex-a6d7f5-u22-20250825t142931z +[2025-11-28 15:20:55] [EXECUTE] Deleting VM Image: a4h-slurm-flex-aa5bfc-u22-20250916t101953z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-aa5bfc-u22-20250916t101953z]. +[2025-11-28 15:21:03] [SUCCESS] Deleted a4h-slurm-flex-aa5bfc-u22-20250916t101953z +[2025-11-28 15:21:03] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:21:03] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 15:21:15] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 15:21:17] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 15:21:17] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 15:21:17] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 15:21:17] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 15:21:17] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 15:21:17] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 15:21:19] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 15:21:19] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 15:21:21] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 15:21:21] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 15:21:23] [INFO] No Global Address found matching criteria. +[2025-11-28 15:21:23] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 15:21:39] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 15:21:41] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:21:41] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:21:41] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:21:41] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:21:41] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:21:41] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 15:21:41] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 15:21:41] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 15:21:43] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:43] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 15:21:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:21:45] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 15:21:46] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 15:21:46] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:22:19] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:22:19] [INFO] Time Cutoff (General): 2025-11-28T14:22:19+0000 +[2025-11-28 15:22:19] [INFO] Time Cutoff (Images): 2025-09-29T15:22:19+0000 +[2025-11-28 15:22:19] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:22:19] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:22:19] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 15:22:21] [INFO] No Service Accounts found matching prefix. +[2025-11-28 15:22:21] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +WARNING: The following zones did not respond: europe-west3. List results may be incomplete. +[2025-11-28 15:22:27] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 15:22:27] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 15:22:29] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:22:29] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:22:29] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:22:29] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:22:29] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:22:29] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 15:22:29] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 15:22:31] [INFO] No Filestore found matching criteria. +[2025-11-28 15:22:31] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:22:33] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ac8613-u22-20250826t060423z +[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ad3d05-u22-20250918t101923z +[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-add7ca-u22-20250902t164931z +[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-b00d3e-u22-20250926t172512z +[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-bb93b4-u22-20250926t113807z +[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-bc10ee-u22-20250922t101931z +[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-c74ec8-u22-20250915t065803z +[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-d60a9e-u22-20250819t055629z +[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-e2a125-u22-20250901t101947z +[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-e3f6d6-u22-20250912t002144z +[2025-11-28 15:22:33] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:22:33] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 15:22:46] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 15:22:47] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 15:22:47] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 15:22:47] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 15:22:47] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 15:22:47] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 15:22:48] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 15:22:49] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 15:22:49] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 15:22:51] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 15:22:51] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 15:22:53] [INFO] No Global Address found matching criteria. +[2025-11-28 15:22:53] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 15:23:09] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 15:23:11] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:23:11] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:23:11] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:23:11] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:23:11] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:23:11] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 15:23:11] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 15:23:11] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 15:23:13] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:13] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 15:23:15] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:23:15] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 15:23:16] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 15:23:16] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:23:34] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:23:34] [INFO] Time Cutoff (General): 2025-11-28T14:23:34+0000 +[2025-11-28 15:23:34] [INFO] Time Cutoff (Images): 2025-09-29T15:23:34+0000 +[2025-11-28 15:23:34] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:23:34] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:23:34] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 15:23:36] [INFO] No Service Accounts found matching prefix. +[2025-11-28 15:23:36] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 15:23:37] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 15:23:37] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 15:23:39] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:23:39] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:23:39] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:23:39] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:23:39] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:23:39] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 15:23:39] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 15:23:41] [INFO] No Filestore found matching criteria. +[2025-11-28 15:23:41] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:23:43] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:23:43] [EXECUTE] Deleting VM Image: a4h-slurm-flex-ac8613-u22-20250826t060423z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-ac8613-u22-20250826t060423z]. +[2025-11-28 15:23:51] [SUCCESS] Deleted a4h-slurm-flex-ac8613-u22-20250826t060423z +[2025-11-28 15:23:51] [EXECUTE] Deleting VM Image: a4h-slurm-flex-ad3d05-u22-20250918t101923z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-ad3d05-u22-20250918t101923z]. +[2025-11-28 15:23:59] [SUCCESS] Deleted a4h-slurm-flex-ad3d05-u22-20250918t101923z +[2025-11-28 15:23:59] [EXECUTE] Deleting VM Image: a4h-slurm-flex-add7ca-u22-20250902t164931z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-add7ca-u22-20250902t164931z]. +[2025-11-28 15:24:06] [SUCCESS] Deleted a4h-slurm-flex-add7ca-u22-20250902t164931z +[2025-11-28 15:24:06] [EXECUTE] Deleting VM Image: a4h-slurm-flex-b00d3e-u22-20250926t172512z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-b00d3e-u22-20250926t172512z]. +[2025-11-28 15:24:14] [SUCCESS] Deleted a4h-slurm-flex-b00d3e-u22-20250926t172512z +[2025-11-28 15:24:14] [EXECUTE] Deleting VM Image: a4h-slurm-flex-bb93b4-u22-20250926t113807z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-bb93b4-u22-20250926t113807z]. +[2025-11-28 15:24:21] [SUCCESS] Deleted a4h-slurm-flex-bb93b4-u22-20250926t113807z +[2025-11-28 15:24:21] [EXECUTE] Deleting VM Image: a4h-slurm-flex-bc10ee-u22-20250922t101931z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-bc10ee-u22-20250922t101931z]. +[2025-11-28 15:24:28] [SUCCESS] Deleted a4h-slurm-flex-bc10ee-u22-20250922t101931z +[2025-11-28 15:24:28] [EXECUTE] Deleting VM Image: a4h-slurm-flex-c74ec8-u22-20250915t065803z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-c74ec8-u22-20250915t065803z]. +[2025-11-28 15:24:35] [SUCCESS] Deleted a4h-slurm-flex-c74ec8-u22-20250915t065803z +[2025-11-28 15:24:35] [EXECUTE] Deleting VM Image: a4h-slurm-flex-d60a9e-u22-20250819t055629z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-d60a9e-u22-20250819t055629z]. +[2025-11-28 15:24:42] [SUCCESS] Deleted a4h-slurm-flex-d60a9e-u22-20250819t055629z +[2025-11-28 15:24:42] [EXECUTE] Deleting VM Image: a4h-slurm-flex-e2a125-u22-20250901t101947z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-e2a125-u22-20250901t101947z]. +[2025-11-28 15:24:50] [SUCCESS] Deleted a4h-slurm-flex-e2a125-u22-20250901t101947z +[2025-11-28 15:24:50] [EXECUTE] Deleting VM Image: a4h-slurm-flex-e3f6d6-u22-20250912t002144z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-e3f6d6-u22-20250912t002144z]. +[2025-11-28 15:24:57] [SUCCESS] Deleted a4h-slurm-flex-e3f6d6-u22-20250912t002144z +[2025-11-28 15:24:57] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:24:57] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 15:25:10] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 15:25:12] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 15:25:12] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 15:25:12] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 15:25:12] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 15:25:12] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 15:25:12] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 15:25:13] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 15:25:13] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 15:25:15] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 15:25:15] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 15:25:17] [INFO] No Global Address found matching criteria. +[2025-11-28 15:25:17] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 15:25:34] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 15:25:36] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:25:36] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:25:36] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:25:36] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:25:36] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:25:36] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 15:25:36] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 15:25:36] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 15:25:38] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:38] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 15:25:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:25:40] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 15:25:41] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 15:25:41] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:26:08] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:26:08] [INFO] Time Cutoff (General): 2025-11-28T14:26:08+0000 +[2025-11-28 15:26:08] [INFO] Time Cutoff (Images): 2025-09-29T15:26:08+0000 +[2025-11-28 15:26:08] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:26:08] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:26:09] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 15:26:10] [INFO] No Service Accounts found matching prefix. +[2025-11-28 15:26:10] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 15:26:11] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 15:26:11] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 15:26:13] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:26:13] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:26:13] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:26:13] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:26:13] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:26:13] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 15:26:13] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 15:26:15] [INFO] No Filestore found matching criteria. +[2025-11-28 15:26:15] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:26:17] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ec0d72-u22-20250904t101939z +[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-edeb3b-u22-20250912t074741z +[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ee0684-u22-20250814t130214z +[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-f0afe0-u22-20250923t110504z +[2025-11-28 15:26:18] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: arpit-a3-toolkitest-u22-20250825t164840z +[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250904t073016z +[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250910t104131z +[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250910t144314z +[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: arpit-toolkit-u22-20250826t034632z +[2025-11-28 15:26:18] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:26:18] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: cx-a3u-u22-20250701t080501z +[2025-11-28 15:26:18] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:26:18] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 15:26:30] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 15:26:31] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 15:26:31] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 15:26:31] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 15:26:31] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 15:26:31] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 15:26:31] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 15:26:33] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 15:26:33] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 15:26:35] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 15:26:35] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 15:26:37] [INFO] No Global Address found matching criteria. +[2025-11-28 15:26:37] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 15:26:53] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 15:26:55] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:26:55] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:26:55] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:26:55] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:26:55] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:26:55] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 15:26:55] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 15:26:55] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 15:26:57] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:57] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 15:26:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:26:58] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 15:27:00] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 15:27:00] [INFO] CLEANUP RUN FINISHED +./cleanup.sh: line 521: n: command not found +[2025-11-28 15:27:10] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:27:10] [INFO] Time Cutoff (General): 2025-11-28T14:27:10+0000 +[2025-11-28 15:27:10] [INFO] Time Cutoff (Images): 2025-09-29T15:27:10+0000 +[2025-11-28 15:27:10] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:27:10] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:27:11] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 15:27:12] [INFO] No Service Accounts found matching prefix. +[2025-11-28 15:27:12] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 15:27:14] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 15:27:14] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 15:27:16] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:27:16] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:27:16] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:27:16] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:27:16] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:27:16] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 15:27:16] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 15:27:18] [INFO] No Filestore found matching criteria. +[2025-11-28 15:27:18] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:27:20] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:27:21] [EXECUTE] Deleting VM Image: a4h-slurm-flex-ec0d72-u22-20250904t101939z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-ec0d72-u22-20250904t101939z]. +[2025-11-28 15:27:27] [SUCCESS] Deleted a4h-slurm-flex-ec0d72-u22-20250904t101939z +[2025-11-28 15:27:27] [EXECUTE] Deleting VM Image: a4h-slurm-flex-edeb3b-u22-20250912t074741z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-edeb3b-u22-20250912t074741z]. +[2025-11-28 15:27:35] [SUCCESS] Deleted a4h-slurm-flex-edeb3b-u22-20250912t074741z +[2025-11-28 15:27:35] [EXECUTE] Deleting VM Image: a4h-slurm-flex-ee0684-u22-20250814t130214z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-ee0684-u22-20250814t130214z]. +[2025-11-28 15:27:42] [SUCCESS] Deleted a4h-slurm-flex-ee0684-u22-20250814t130214z +[2025-11-28 15:27:42] [EXECUTE] Deleting VM Image: a4h-slurm-flex-f0afe0-u22-20250923t110504z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-f0afe0-u22-20250923t110504z]. +[2025-11-28 15:27:50] [SUCCESS] Deleted a4h-slurm-flex-f0afe0-u22-20250923t110504z +[2025-11-28 15:27:50] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:27:50] [EXECUTE] Deleting VM Image: arpit-a3-toolkitest-u22-20250825t164840z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/arpit-a3-toolkitest-u22-20250825t164840z]. +[2025-11-28 15:27:58] [SUCCESS] Deleted arpit-a3-toolkitest-u22-20250825t164840z +[2025-11-28 15:27:58] [EXECUTE] Deleting VM Image: arpit-a3slurm-toolkit-u22-20250904t073016z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/arpit-a3slurm-toolkit-u22-20250904t073016z]. +[2025-11-28 15:28:05] [SUCCESS] Deleted arpit-a3slurm-toolkit-u22-20250904t073016z +[2025-11-28 15:28:05] [EXECUTE] Deleting VM Image: arpit-a3slurm-toolkit-u22-20250910t104131z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/arpit-a3slurm-toolkit-u22-20250910t104131z]. +[2025-11-28 15:28:12] [SUCCESS] Deleted arpit-a3slurm-toolkit-u22-20250910t104131z +[2025-11-28 15:28:12] [EXECUTE] Deleting VM Image: arpit-a3slurm-toolkit-u22-20250910t144314z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/arpit-a3slurm-toolkit-u22-20250910t144314z]. +[2025-11-28 15:28:20] [SUCCESS] Deleted arpit-a3slurm-toolkit-u22-20250910t144314z +[2025-11-28 15:28:20] [EXECUTE] Deleting VM Image: arpit-toolkit-u22-20250826t034632z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/arpit-toolkit-u22-20250826t034632z]. +[2025-11-28 15:28:28] [SUCCESS] Deleted arpit-toolkit-u22-20250826t034632z +[2025-11-28 15:28:28] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:28:28] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:28:28] [EXECUTE] Deleting VM Image: cx-a3u-u22-20250701t080501z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/cx-a3u-u22-20250701t080501z]. +[2025-11-28 15:28:35] [SUCCESS] Deleted cx-a3u-u22-20250701t080501z +[2025-11-28 15:28:35] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:28:35] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 15:28:47] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 15:28:48] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 15:28:48] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 15:28:48] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 15:28:48] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 15:28:48] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 15:28:48] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 15:28:50] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 15:28:50] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 15:28:52] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 15:28:52] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 15:28:54] [INFO] No Global Address found matching criteria. +[2025-11-28 15:28:54] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 15:29:10] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 15:29:12] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:29:12] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:29:12] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:29:12] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:29:12] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:29:12] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 15:29:12] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 15:29:12] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 15:29:14] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:14] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 15:29:16] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:29:16] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 15:29:17] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 15:29:17] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:29:25] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:29:25] [INFO] Time Cutoff (General): 2025-11-28T14:29:25+0000 +[2025-11-28 15:29:25] [INFO] Time Cutoff (Images): 2025-09-29T15:29:25+0000 +[2025-11-28 15:29:25] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:29:25] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:29:26] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 15:29:27] [INFO] No Service Accounts found matching prefix. +[2025-11-28 15:29:27] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 15:29:29] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 15:29:29] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 15:29:31] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:29:31] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:29:31] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:29:31] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:29:31] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:29:31] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 15:29:31] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 15:29:34] [INFO] No Filestore found matching criteria. +[2025-11-28 15:29:34] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:29:36] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:29:36] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:29:36] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:29:36] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: harsh-a4-image +[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t085040z +[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t101254z +[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t163709z +[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t214154z +[2025-11-28 15:29:36] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: raasa-a3h-slurm-u20-20250916t084718z +[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250919t102219z +[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t002502z +[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t024123z +[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t034057z +[2025-11-28 15:29:36] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:29:36] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 15:29:49] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 15:29:51] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 15:29:51] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 15:29:51] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 15:29:51] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 15:29:51] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 15:29:51] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 15:29:53] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 15:29:53] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 15:29:54] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 15:29:54] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 15:29:56] [INFO] No Global Address found matching criteria. +[2025-11-28 15:29:56] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 15:30:16] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 15:30:18] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:30:18] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:30:18] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:30:18] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:30:18] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:30:18] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 15:30:18] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 15:30:18] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 15:30:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 15:30:20] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:20] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 15:30:21] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:30:21] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 15:30:23] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 15:30:23] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:30:44] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:30:44] [INFO] Time Cutoff (General): 2025-11-28T14:30:44+0000 +[2025-11-28 15:30:44] [INFO] Time Cutoff (Images): 2025-09-29T15:30:44+0000 +[2025-11-28 15:30:44] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:30:45] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:30:45] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 15:30:46] [INFO] No Service Accounts found matching prefix. +[2025-11-28 15:30:46] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 15:30:48] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 15:30:48] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 15:30:50] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:30:50] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:30:50] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:30:50] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:30:50] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:30:50] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 15:30:50] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 15:30:51] [INFO] No Filestore found matching criteria. +[2025-11-28 15:30:51] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:30:53] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:30:54] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:30:54] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:30:54] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:30:54] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t085040z +[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t101254z +[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t163709z +[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t214154z +[2025-11-28 15:30:54] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: raasa-a3h-slurm-u20-20250916t084718z +[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250919t102219z +[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t002502z +[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t024123z +[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t034057z +[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: rac-a4h-nccl-u22-20250919t085137z +[2025-11-28 15:30:54] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:30:54] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 15:31:06] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 15:31:08] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 15:31:08] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 15:31:08] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 15:31:08] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 15:31:08] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 15:31:08] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 15:31:10] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 15:31:10] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 15:31:11] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 15:31:12] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 15:31:12] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 15:31:13] [INFO] No Global Address found matching criteria. +[2025-11-28 15:31:13] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 15:31:30] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 15:31:32] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:31:32] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:31:32] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:31:32] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:31:32] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:31:32] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 15:31:32] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 15:31:32] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 15:31:34] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:34] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 15:31:36] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:31:36] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 15:31:37] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 15:31:37] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:32:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:32:47] [INFO] Time Cutoff (General): 2025-11-28T14:32:47+0000 +[2025-11-28 15:32:47] [INFO] Time Cutoff (Images): 2025-09-29T15:32:47+0000 +[2025-11-28 15:32:47] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:32:47] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:32:47] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 15:32:48] [INFO] No Service Accounts found matching prefix. +[2025-11-28 15:32:48] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 15:32:50] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 15:32:50] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 15:32:52] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:32:52] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:32:52] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:32:52] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:32:52] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:32:52] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 15:32:52] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 15:32:54] [INFO] No Filestore found matching criteria. +[2025-11-28 15:32:54] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:32:56] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:32:56] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:32:56] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:32:56] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:32:56] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:32:56] [EXECUTE] Deleting VM Image: hpc-exr-2-u22-20250912t085040z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/hpc-exr-2-u22-20250912t085040z]. +[2025-11-28 15:33:04] [SUCCESS] Deleted hpc-exr-2-u22-20250912t085040z +[2025-11-28 15:33:04] [EXECUTE] Deleting VM Image: hpc-exr-2-u22-20250912t101254z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/hpc-exr-2-u22-20250912t101254z]. +[2025-11-28 15:33:12] [SUCCESS] Deleted hpc-exr-2-u22-20250912t101254z +[2025-11-28 15:33:12] [EXECUTE] Deleting VM Image: htcondor-10x-20250901t163709z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/htcondor-10x-20250901t163709z]. +[2025-11-28 15:33:19] [SUCCESS] Deleted htcondor-10x-20250901t163709z +[2025-11-28 15:33:19] [EXECUTE] Deleting VM Image: htcondor-10x-20250901t214154z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/htcondor-10x-20250901t214154z]. +[2025-11-28 15:33:27] [SUCCESS] Deleted htcondor-10x-20250901t214154z +[2025-11-28 15:33:27] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:33:27] [EXECUTE] Deleting VM Image: raasa-a3h-slurm-u20-20250916t084718z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/raasa-a3h-slurm-u20-20250916t084718z]. +[2025-11-28 15:33:35] [SUCCESS] Deleted raasa-a3h-slurm-u20-20250916t084718z +[2025-11-28 15:33:35] [EXECUTE] Deleting VM Image: rac-a3ul-nccl-u22-20250919t102219z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a3ul-nccl-u22-20250919t102219z]. +[2025-11-28 15:33:42] [SUCCESS] Deleted rac-a3ul-nccl-u22-20250919t102219z +[2025-11-28 15:33:42] [EXECUTE] Deleting VM Image: rac-a3ul-nccl-u22-20250920t002502z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a3ul-nccl-u22-20250920t002502z]. +[2025-11-28 15:33:49] [SUCCESS] Deleted rac-a3ul-nccl-u22-20250920t002502z +[2025-11-28 15:33:49] [EXECUTE] Deleting VM Image: rac-a3ul-nccl-u22-20250920t024123z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a3ul-nccl-u22-20250920t024123z]. +[2025-11-28 15:33:57] [SUCCESS] Deleted rac-a3ul-nccl-u22-20250920t024123z +[2025-11-28 15:33:57] [EXECUTE] Deleting VM Image: rac-a3ul-nccl-u22-20250920t034057z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a3ul-nccl-u22-20250920t034057z]. +[2025-11-28 15:34:05] [SUCCESS] Deleted rac-a3ul-nccl-u22-20250920t034057z +[2025-11-28 15:34:05] [EXECUTE] Deleting VM Image: rac-a4h-nccl-u22-20250919t085137z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a4h-nccl-u22-20250919t085137z]. +[2025-11-28 15:34:12] [SUCCESS] Deleted rac-a4h-nccl-u22-20250919t085137z +[2025-11-28 15:34:12] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:34:12] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 15:34:25] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 15:34:27] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 15:34:27] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 15:34:27] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 15:34:27] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 15:34:27] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 15:34:27] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 15:34:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 15:34:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 15:34:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 15:34:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 15:34:29] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 15:34:29] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 15:34:30] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 15:34:30] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 15:34:32] [INFO] No Global Address found matching criteria. +[2025-11-28 15:34:32] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 15:34:49] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 15:34:51] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:34:51] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:34:51] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:34:51] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:34:51] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:34:51] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 15:34:51] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 15:34:51] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 15:34:53] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:53] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 15:34:55] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:34:55] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 15:34:56] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 15:34:56] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:36:15] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:36:15] [INFO] Time Cutoff (General): 2025-11-28T14:36:15+0000 +[2025-11-28 15:36:15] [INFO] Time Cutoff (Images): 2025-09-29T15:36:15+0000 +[2025-11-28 15:36:15] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:36:15] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:36:15] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 15:36:17] [INFO] No Service Accounts found matching prefix. +[2025-11-28 15:36:17] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 15:36:18] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 15:36:18] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 15:36:20] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:36:20] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:36:20] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:36:20] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:36:20] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:36:20] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 15:36:20] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 15:36:23] [INFO] No Filestore found matching criteria. +[2025-11-28 15:36:23] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:36:25] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:36:25] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:36:25] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:36:25] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:36:25] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:36:25] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rac-a4h-nccl-u22-20250919t101657z +[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rac-a4h-nccl-u22-20250920t002518z +[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rac-a4h-u22-20250917t065059z +[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t015604z +[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t032350z +[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t040319z +[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t054432z +[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t064626z +[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t155746z +[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t164626z +[2025-11-28 15:36:25] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:36:25] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 15:36:37] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 15:36:39] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 15:36:39] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 15:36:39] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 15:36:39] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 15:36:39] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 15:36:39] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 15:36:41] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 15:36:41] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 15:36:43] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 15:36:43] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 15:36:44] [INFO] No Global Address found matching criteria. +[2025-11-28 15:36:44] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 15:37:01] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 15:37:03] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:37:03] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:37:03] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:37:03] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:37:03] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:37:03] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 15:37:03] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 15:37:03] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 15:37:05] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:05] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 15:37:07] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:37:07] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 15:37:08] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 15:37:08] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:37:48] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:37:48] [INFO] Time Cutoff (General): 2025-11-28T14:37:48+0000 +[2025-11-28 15:37:48] [INFO] Time Cutoff (Images): 2025-09-29T15:37:48+0000 +[2025-11-28 15:37:48] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:37:48] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:37:48] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-28 15:37:50] [INFO] No Service Accounts found matching prefix. +[2025-11-28 15:37:50] [INFO] --- Processing: GKE Cluster (Limit: 10) --- +[2025-11-28 15:37:51] [SKIP] gke-a3-nccl-test (Protected Substring) +[2025-11-28 15:37:51] [INFO] --- Processing: Compute Instance (Limit: 10) --- +[2025-11-28 15:37:53] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:37:53] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:37:53] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:37:53] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:37:53] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:37:53] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-28 15:37:53] [INFO] --- Processing: Filestore (Limit: 10) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-28 15:37:56] [INFO] No Filestore found matching criteria. +[2025-11-28 15:37:56] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:37:58] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:37:58] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:37:58] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:37:58] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:37:58] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:37:58] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:37:58] [EXECUTE] Deleting VM Image: rac-a4h-nccl-u22-20250919t101657z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a4h-nccl-u22-20250919t101657z]. +[2025-11-28 15:38:05] [SUCCESS] Deleted rac-a4h-nccl-u22-20250919t101657z +[2025-11-28 15:38:05] [EXECUTE] Deleting VM Image: rac-a4h-nccl-u22-20250920t002518z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a4h-nccl-u22-20250920t002518z]. +[2025-11-28 15:38:12] [SUCCESS] Deleted rac-a4h-nccl-u22-20250920t002518z +[2025-11-28 15:38:12] [EXECUTE] Deleting VM Image: rac-a4h-u22-20250917t065059z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a4h-u22-20250917t065059z]. +[2025-11-28 15:38:20] [SUCCESS] Deleted rac-a4h-u22-20250917t065059z +[2025-11-28 15:38:20] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t015604z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t015604z]. +[2025-11-28 15:38:28] [SUCCESS] Deleted rach-a3ul-u22-20250911t015604z +[2025-11-28 15:38:28] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t032350z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t032350z]. +[2025-11-28 15:38:36] [SUCCESS] Deleted rach-a3ul-u22-20250911t032350z +[2025-11-28 15:38:36] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t040319z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t040319z]. +[2025-11-28 15:38:43] [SUCCESS] Deleted rach-a3ul-u22-20250911t040319z +[2025-11-28 15:38:43] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t054432z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t054432z]. +[2025-11-28 15:38:51] [SUCCESS] Deleted rach-a3ul-u22-20250911t054432z +[2025-11-28 15:38:51] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t064626z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t064626z]. +[2025-11-28 15:38:58] [SUCCESS] Deleted rach-a3ul-u22-20250911t064626z +[2025-11-28 15:38:58] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t155746z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t155746z]. +[2025-11-28 15:39:06] [SUCCESS] Deleted rach-a3ul-u22-20250911t155746z +[2025-11-28 15:39:06] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t164626z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t164626z]. +[2025-11-28 15:39:13] [SUCCESS] Deleted rach-a3ul-u22-20250911t164626z +[2025-11-28 15:39:13] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:39:13] [INFO] --- Processing: Docker Images (Limit: 10) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 15:39:26] [INFO] --- Processing: Cloud Router (Limit: 10) --- +[2025-11-28 15:39:27] [SKIP] default-net-router (In Exclusion List) +[2025-11-28 15:39:27] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-28 15:39:27] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-28 15:39:27] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-28 15:39:27] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) +[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) +[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) +[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) +[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) +[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) +[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) +[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) +[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) +[2025-11-28 15:39:27] [INFO] --- Processing: Firewall Rules (Limit: 10) --- +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) +[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) +[2025-11-28 15:39:29] [INFO] --- Processing: Compute Addresses --- +[2025-11-28 15:39:29] [INFO] --- Processing: Regional Address (Limit: 10) --- +[2025-11-28 15:39:31] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) +[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) +[2025-11-28 15:39:31] [INFO] --- Processing: Global Address (Limit: 10) --- +[2025-11-28 15:39:33] [INFO] No Global Address found matching criteria. +[2025-11-28 15:39:33] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- +[2025-11-28 15:39:49] [INFO] --- Processing: Zonal Disk (Limit: 10) --- +[2025-11-28 15:39:51] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) +[2025-11-28 15:39:51] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) +[2025-11-28 15:39:51] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) +[2025-11-28 15:39:51] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-28 15:39:51] [SKIP] image-inspector (In Exclusion List) +[2025-11-28 15:39:51] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-28 15:39:51] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-28 15:39:51] [INFO] --- Processing: Subnetworks (Limit: 10) --- +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) +[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) +[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) +[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) +[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) +[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) +[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) +[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) +[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) +[2025-11-28 15:39:53] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:53] [INFO] --- Processing: VPC Networks (Limit: 10) --- +[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) +[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) +[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) +[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) +[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) +[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) +[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) +[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) +[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-net (Protected Substring) +[2025-11-28 15:39:55] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-28 15:39:55] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- +[2025-11-28 15:39:57] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-28 15:39:57] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:41:13] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:41:13] [INFO] Time Cutoff (General): 2025-11-28T14:41:13+0000 +[2025-11-28 15:41:13] [INFO] Time Cutoff (Images): 2025-09-29T15:41:13+0000 +[2025-11-28 15:41:13] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:41:13] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:41:14] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:41:16] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:41:16] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:41:16] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:41:16] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:41:16] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:41:16] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t180026z +[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t191956z +[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t210446z +[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t223305z +[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250912t030352z +[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250912t043440z +[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250912t054729z +[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ult-u22-20250916t052511z +[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rasa-a3h-slurm-u20-20250915t114049z +[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rasa-a3h-slurm-u20-20250915t131548z +[2025-11-28 15:41:16] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:41:16] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:41:31] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:41:31] [INFO] Time Cutoff (General): 2025-11-28T14:41:31+0000 +[2025-11-28 15:41:31] [INFO] Time Cutoff (Images): 2025-09-29T15:41:31+0000 +[2025-11-28 15:41:31] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:41:31] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:41:31] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:41:33] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:41:34] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:41:34] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:41:34] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:41:34] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:41:34] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:41:34] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t180026z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t180026z]. +[2025-11-28 15:41:42] [SUCCESS] Deleted rach-a3ul-u22-20250911t180026z +[2025-11-28 15:41:42] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t191956z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t191956z]. +[2025-11-28 15:41:48] [SUCCESS] Deleted rach-a3ul-u22-20250911t191956z +[2025-11-28 15:41:48] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t210446z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t210446z]. +[2025-11-28 15:41:56] [SUCCESS] Deleted rach-a3ul-u22-20250911t210446z +[2025-11-28 15:41:56] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t223305z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t223305z]. +[2025-11-28 15:42:03] [SUCCESS] Deleted rach-a3ul-u22-20250911t223305z +[2025-11-28 15:42:03] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250912t030352z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250912t030352z]. +[2025-11-28 15:42:10] [SUCCESS] Deleted rach-a3ul-u22-20250912t030352z +[2025-11-28 15:42:10] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250912t043440z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250912t043440z]. +[2025-11-28 15:42:17] [SUCCESS] Deleted rach-a3ul-u22-20250912t043440z +[2025-11-28 15:42:17] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250912t054729z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250912t054729z]. +[2025-11-28 15:42:25] [SUCCESS] Deleted rach-a3ul-u22-20250912t054729z +[2025-11-28 15:42:25] [EXECUTE] Deleting VM Image: rach-a3ult-u22-20250916t052511z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ult-u22-20250916t052511z]. +[2025-11-28 15:42:32] [SUCCESS] Deleted rach-a3ult-u22-20250916t052511z +[2025-11-28 15:42:32] [EXECUTE] Deleting VM Image: rasa-a3h-slurm-u20-20250915t114049z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rasa-a3h-slurm-u20-20250915t114049z]. +[2025-11-28 15:42:40] [SUCCESS] Deleted rasa-a3h-slurm-u20-20250915t114049z +[2025-11-28 15:42:40] [EXECUTE] Deleting VM Image: rasa-a3h-slurm-u20-20250915t131548z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rasa-a3h-slurm-u20-20250915t131548z]. +[2025-11-28 15:42:47] [SUCCESS] Deleted rasa-a3h-slurm-u20-20250915t131548z +[2025-11-28 15:42:47] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:42:47] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:43:01] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:43:01] [INFO] Time Cutoff (General): 2025-11-28T14:43:01+0000 +[2025-11-28 15:43:01] [INFO] Time Cutoff (Images): 2025-09-29T15:43:01+0000 +[2025-11-28 15:43:01] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:43:01] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:43:02] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:43:03] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:43:04] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:43:04] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:43:04] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:43:04] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:43:04] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: rocka4h-rocky9-20250908t175724z +[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: rocka4hf-rocky9-20250910t040750z +[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sa-chs-ops-u22-20250925t092448z +[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: salsa-a3h-slurm-u22-20250919t083351z +[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sar-a3h-slurm-u20-20250912t102555z +[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t094000z +[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t104421z +[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: saral-saara-a3h-slurm-u22-20250926t054333z +[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sarasa-a3h-slurm-u22-20250919t125328z +[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sasa-a3h-slurm-u22-20250919t064159z +[2025-11-28 15:43:04] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:43:04] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:48:50] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:48:50] [INFO] Time Cutoff (General): 2025-11-28T14:48:50+0000 +[2025-11-28 15:48:50] [INFO] Time Cutoff (Images): 2025-09-29T15:48:50+0000 +[2025-11-28 15:48:50] [INFO] Delete Limit per Type: 10 +[2025-11-28 15:48:50] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:48:51] [INFO] --- Processing: VM Images (Limit: 10) --- +[2025-11-28 15:48:53] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:48:53] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:48:53] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:48:53] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:48:53] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:48:53] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:48:53] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-28 15:48:53] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sa-chs-ops-u22-20250925t092448z +[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: salsa-a3h-slurm-u22-20250919t083351z +[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sar-a3h-slurm-u20-20250912t102555z +[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t094000z +[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t104421z +[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: saral-saara-a3h-slurm-u22-20250926t054333z +[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sarasa-a3h-slurm-u22-20250919t125328z +[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sasa-a3h-slurm-u22-20250919t064159z +[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sl-saara-a3hm-slurm-u22-20250925t103201z +[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: slara-saara-a3h-slurm-u22-20250926t035224z +[2025-11-28 15:48:53] [INFO] Hit delete limit (10) for VM Images. +[2025-11-28 15:48:53] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:49:16] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:49:16] [INFO] Time Cutoff (General): 2025-11-28T14:49:16+0000 +[2025-11-28 15:49:16] [INFO] Time Cutoff (Images): 2025-09-29T15:49:16+0000 +[2025-11-28 15:49:16] [INFO] Delete Limit per Type: 20 +[2025-11-28 15:49:16] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:49:16] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-28 15:49:18] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:49:18] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:49:19] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:49:19] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:49:19] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:49:19] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:49:19] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-28 15:49:19] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sa-chs-ops-u22-20250925t092448z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: salsa-a3h-slurm-u22-20250919t083351z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sar-a3h-slurm-u20-20250912t102555z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t094000z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t104421z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: saral-saara-a3h-slurm-u22-20250926t054333z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sarasa-a3h-slurm-u22-20250919t125328z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sasa-a3h-slurm-u22-20250919t064159z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sl-saara-a3hm-slurm-u22-20250925t103201z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slara-saara-a3h-slurm-u22-20250926t035224z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slm-saara-a3h-slurm-u22-20250925t114209z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slr-saara-a3h-slurm-u22-20250925t151755z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slr-saara-a3h-slurm-u22-20250925t160703z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slra-saara-a3h-slurm-u22-20250925t163630z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slsa-a3h-slurm-u20-20250918t115814z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slsa-a3h-slurm-u20-20250918t124720z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250825t103536z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250825t113315z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250825t120950z +[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250825t135812z +[2025-11-28 15:49:19] [INFO] Hit delete limit (20) for VM Images. +[2025-11-28 15:49:19] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:50:02] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:50:02] [INFO] Time Cutoff (General): 2025-11-28T14:50:02+0000 +[2025-11-28 15:50:02] [INFO] Time Cutoff (Images): 2025-09-29T15:50:02+0000 +[2025-11-28 15:50:02] [INFO] Delete Limit per Type: 20 +[2025-11-28 15:50:02] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:50:03] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-28 15:50:05] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:50:05] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:50:05] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:50:05] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:50:05] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:50:05] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:50:05] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-28 15:50:05] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-28 15:50:05] [EXECUTE] Deleting VM Image: sa-chs-ops-u22-20250925t092448z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sa-chs-ops-u22-20250925t092448z]. +[2025-11-28 15:50:13] [SUCCESS] Deleted sa-chs-ops-u22-20250925t092448z +[2025-11-28 15:50:13] [EXECUTE] Deleting VM Image: salsa-a3h-slurm-u22-20250919t083351z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/salsa-a3h-slurm-u22-20250919t083351z]. +[2025-11-28 15:50:20] [SUCCESS] Deleted salsa-a3h-slurm-u22-20250919t083351z +[2025-11-28 15:50:20] [EXECUTE] Deleting VM Image: sar-a3h-slurm-u20-20250912t102555z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sar-a3h-slurm-u20-20250912t102555z]. +[2025-11-28 15:50:27] [SUCCESS] Deleted sar-a3h-slurm-u20-20250912t102555z +[2025-11-28 15:50:27] [EXECUTE] Deleting VM Image: sara-a3h-slurm-u20-20250915t094000z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sara-a3h-slurm-u20-20250915t094000z]. +[2025-11-28 15:50:35] [SUCCESS] Deleted sara-a3h-slurm-u20-20250915t094000z +[2025-11-28 15:50:35] [EXECUTE] Deleting VM Image: sara-a3h-slurm-u20-20250915t104421z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sara-a3h-slurm-u20-20250915t104421z]. +[2025-11-28 15:50:42] [SUCCESS] Deleted sara-a3h-slurm-u20-20250915t104421z +[2025-11-28 15:50:42] [EXECUTE] Deleting VM Image: saral-saara-a3h-slurm-u22-20250926t054333z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/saral-saara-a3h-slurm-u22-20250926t054333z]. +[2025-11-28 15:50:50] [SUCCESS] Deleted saral-saara-a3h-slurm-u22-20250926t054333z +[2025-11-28 15:50:50] [EXECUTE] Deleting VM Image: sarasa-a3h-slurm-u22-20250919t125328z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sarasa-a3h-slurm-u22-20250919t125328z]. +[2025-11-28 15:50:57] [SUCCESS] Deleted sarasa-a3h-slurm-u22-20250919t125328z +[2025-11-28 15:50:57] [EXECUTE] Deleting VM Image: sasa-a3h-slurm-u22-20250919t064159z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sasa-a3h-slurm-u22-20250919t064159z]. +[2025-11-28 15:51:05] [SUCCESS] Deleted sasa-a3h-slurm-u22-20250919t064159z +[2025-11-28 15:51:05] [EXECUTE] Deleting VM Image: sl-saara-a3hm-slurm-u22-20250925t103201z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sl-saara-a3hm-slurm-u22-20250925t103201z]. +[2025-11-28 15:51:12] [SUCCESS] Deleted sl-saara-a3hm-slurm-u22-20250925t103201z +[2025-11-28 15:51:12] [EXECUTE] Deleting VM Image: slara-saara-a3h-slurm-u22-20250926t035224z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slara-saara-a3h-slurm-u22-20250926t035224z]. +[2025-11-28 15:51:20] [SUCCESS] Deleted slara-saara-a3h-slurm-u22-20250926t035224z +[2025-11-28 15:51:20] [EXECUTE] Deleting VM Image: slm-saara-a3h-slurm-u22-20250925t114209z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slm-saara-a3h-slurm-u22-20250925t114209z]. +[2025-11-28 15:51:28] [SUCCESS] Deleted slm-saara-a3h-slurm-u22-20250925t114209z +[2025-11-28 15:51:28] [EXECUTE] Deleting VM Image: slr-saara-a3h-slurm-u22-20250925t151755z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slr-saara-a3h-slurm-u22-20250925t151755z]. +[2025-11-28 15:51:35] [SUCCESS] Deleted slr-saara-a3h-slurm-u22-20250925t151755z +[2025-11-28 15:51:35] [EXECUTE] Deleting VM Image: slr-saara-a3h-slurm-u22-20250925t160703z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slr-saara-a3h-slurm-u22-20250925t160703z]. +[2025-11-28 15:51:43] [SUCCESS] Deleted slr-saara-a3h-slurm-u22-20250925t160703z +[2025-11-28 15:51:43] [EXECUTE] Deleting VM Image: slra-saara-a3h-slurm-u22-20250925t163630z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slra-saara-a3h-slurm-u22-20250925t163630z]. +[2025-11-28 15:51:51] [SUCCESS] Deleted slra-saara-a3h-slurm-u22-20250925t163630z +[2025-11-28 15:51:51] [EXECUTE] Deleting VM Image: slsa-a3h-slurm-u20-20250918t115814z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slsa-a3h-slurm-u20-20250918t115814z]. +[2025-11-28 15:51:59] [SUCCESS] Deleted slsa-a3h-slurm-u20-20250918t115814z +[2025-11-28 15:51:59] [EXECUTE] Deleting VM Image: slsa-a3h-slurm-u20-20250918t124720z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slsa-a3h-slurm-u20-20250918t124720z]. +[2025-11-28 15:52:06] [SUCCESS] Deleted slsa-a3h-slurm-u20-20250918t124720z +[2025-11-28 15:52:06] [EXECUTE] Deleting VM Image: slurm-a3mega-20250825t103536z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250825t103536z]. +[2025-11-28 15:52:13] [SUCCESS] Deleted slurm-a3mega-20250825t103536z +[2025-11-28 15:52:13] [EXECUTE] Deleting VM Image: slurm-a3mega-20250825t113315z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250825t113315z]. +[2025-11-28 15:52:21] [SUCCESS] Deleted slurm-a3mega-20250825t113315z +[2025-11-28 15:52:21] [EXECUTE] Deleting VM Image: slurm-a3mega-20250825t120950z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250825t120950z]. +[2025-11-28 15:52:28] [SUCCESS] Deleted slurm-a3mega-20250825t120950z +[2025-11-28 15:52:28] [EXECUTE] Deleting VM Image: slurm-a3mega-20250825t135812z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250825t135812z]. +[2025-11-28 15:52:36] [SUCCESS] Deleted slurm-a3mega-20250825t135812z +[2025-11-28 15:52:36] [INFO] Hit delete limit (20) for VM Images. +[2025-11-28 15:52:36] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:53:05] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:53:05] [INFO] Time Cutoff (General): 2025-11-28T14:53:05+0000 +[2025-11-28 15:53:05] [INFO] Time Cutoff (Images): 2025-09-29T15:53:05+0000 +[2025-11-28 15:53:05] [INFO] Delete Limit per Type: 20 +[2025-11-28 15:53:05] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:53:05] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-28 15:53:07] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:53:07] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:53:07] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:53:07] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:53:07] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:53:07] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:53:07] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-28 15:53:07] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250828t190928z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250902t123613z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250902t150129z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250903t032910z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250917t065710z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250912t071120z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250916t093232z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250917t061957z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-gcp-next-hpc-rocky-linux-8-1739990978 +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-gcp-next-hpc-rocky-linux-8-1740100297 +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t145301z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t153428z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t200206z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t210655z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250901t115638z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250902t024825z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250903t141250z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250904t070453z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t022348z +[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t064250z +[2025-11-28 15:53:07] [INFO] Hit delete limit (20) for VM Images. +[2025-11-28 15:53:07] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:53:56] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:53:56] [INFO] Time Cutoff (General): 2025-11-28T14:53:56+0000 +[2025-11-28 15:53:56] [INFO] Time Cutoff (Images): 2025-09-29T15:53:56+0000 +[2025-11-28 15:53:56] [INFO] Delete Limit per Type: 20 +[2025-11-28 15:53:56] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:53:57] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-28 15:53:59] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:53:59] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:53:59] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:53:59] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:53:59] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:53:59] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:53:59] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-28 15:53:59] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-28 15:53:59] [EXECUTE] Deleting VM Image: slurm-a3mega-20250828t190928z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250828t190928z]. +[2025-11-28 15:54:06] [SUCCESS] Deleted slurm-a3mega-20250828t190928z +[2025-11-28 15:54:06] [EXECUTE] Deleting VM Image: slurm-a3mega-20250902t123613z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250902t123613z]. +[2025-11-28 15:54:14] [SUCCESS] Deleted slurm-a3mega-20250902t123613z +[2025-11-28 15:54:14] [EXECUTE] Deleting VM Image: slurm-a3mega-20250902t150129z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250902t150129z]. +[2025-11-28 15:54:21] [SUCCESS] Deleted slurm-a3mega-20250902t150129z +[2025-11-28 15:54:21] [EXECUTE] Deleting VM Image: slurm-a3mega-20250903t032910z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250903t032910z]. +[2025-11-28 15:54:27] [SUCCESS] Deleted slurm-a3mega-20250903t032910z +[2025-11-28 15:54:27] [EXECUTE] Deleting VM Image: slurm-a3mega-20250917t065710z + + +Command killed by keyboard interrupt + +[2025-11-28 15:55:20] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:55:20] [INFO] Time Cutoff (General): 2025-11-28T14:55:19+0000 +[2025-11-28 15:55:20] [INFO] Time Cutoff (Images): 2025-09-29T15:55:19+0000 +[2025-11-28 15:55:20] [INFO] Delete Limit per Type: 20 +[2025-11-28 15:55:20] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:55:20] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-28 15:55:22] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:55:22] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:55:22] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:55:22] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:55:22] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:55:22] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:55:22] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-28 15:55:22] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-28 15:55:22] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250917t065710z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250912t071120z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250916t093232z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250917t061957z +[2025-11-28 15:55:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-28 15:55:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t145301z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t153428z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t200206z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t210655z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250901t115638z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250902t024825z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250903t141250z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250904t070453z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t022348z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t064250z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250921t061944z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: test-a4-nccl-01-u22-20250409t085350z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: test-a4-nccl-u22-20250409t065232z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: test-a4-nccl-u22-20250409t074128z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: try-a3hh-slurm-u22-20250925t075018z +[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: try-a3hhi-slurm-u22-20250925t091429z +[2025-11-28 15:55:23] [INFO] Hit delete limit (20) for VM Images. +[2025-11-28 15:55:23] [INFO] CLEANUP RUN FINISHED +[2025-11-28 15:55:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 15:55:47] [INFO] Time Cutoff (General): 2025-11-28T14:55:47+0000 +[2025-11-28 15:55:47] [INFO] Time Cutoff (Images): 2025-09-29T15:55:47+0000 +[2025-11-28 15:55:47] [INFO] Delete Limit per Type: 20 +[2025-11-28 15:55:47] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 15:55:48] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-28 15:55:49] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 15:55:50] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 15:55:50] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 15:55:50] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 15:55:50] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 15:55:50] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 15:55:50] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-28 15:55:50] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-28 15:55:50] [EXECUTE] Deleting VM Image: slurm-a3mega-20250917t065710z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250917t065710z]. +[2025-11-28 15:55:57] [SUCCESS] Deleted slurm-a3mega-20250917t065710z +[2025-11-28 15:55:57] [EXECUTE] Deleting VM Image: slurm-dlvm-20250912t071120z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-dlvm-20250912t071120z]. +[2025-11-28 15:56:05] [SUCCESS] Deleted slurm-dlvm-20250912t071120z +[2025-11-28 15:56:05] [EXECUTE] Deleting VM Image: slurm-dlvm-20250916t093232z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-dlvm-20250916t093232z]. +[2025-11-28 15:56:12] [SUCCESS] Deleted slurm-dlvm-20250916t093232z +[2025-11-28 15:56:12] [EXECUTE] Deleting VM Image: slurm-dlvm-20250917t061957z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-dlvm-20250917t061957z]. +[2025-11-28 15:56:19] [SUCCESS] Deleted slurm-dlvm-20250917t061957z +[2025-11-28 15:56:19] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-28 15:56:19] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-28 15:56:19] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t145301z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250828t145301z]. +[2025-11-28 15:56:26] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250828t145301z +[2025-11-28 15:56:26] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t153428z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250828t153428z]. +[2025-11-28 15:56:33] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250828t153428z +[2025-11-28 15:56:33] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t200206z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250830t200206z]. +[2025-11-28 15:56:40] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250830t200206z +[2025-11-28 15:56:40] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t210655z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250830t210655z]. +[2025-11-28 15:56:48] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250830t210655z +[2025-11-28 15:56:48] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250901t115638z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250901t115638z]. +[2025-11-28 15:56:56] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250901t115638z +[2025-11-28 15:56:56] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250902t024825z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250902t024825z]. +[2025-11-28 15:57:03] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250902t024825z +[2025-11-28 15:57:03] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250903t141250z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250903t141250z]. +[2025-11-28 15:57:09] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250903t141250z +[2025-11-28 15:57:09] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250904t070453z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250904t070453z]. +[2025-11-28 15:57:16] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250904t070453z +[2025-11-28 15:57:16] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t022348z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250919t022348z]. +[2025-11-28 15:57:23] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250919t022348z +[2025-11-28 15:57:23] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t064250z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250919t064250z]. +[2025-11-28 15:57:29] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250919t064250z +[2025-11-28 15:57:29] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250921t061944z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250921t061944z]. +[2025-11-28 15:57:36] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250921t061944z +[2025-11-28 15:57:36] [EXECUTE] Deleting VM Image: test-a4-nccl-01-u22-20250409t085350z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/test-a4-nccl-01-u22-20250409t085350z]. +[2025-11-28 15:57:43] [SUCCESS] Deleted test-a4-nccl-01-u22-20250409t085350z +[2025-11-28 15:57:43] [EXECUTE] Deleting VM Image: test-a4-nccl-u22-20250409t065232z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/test-a4-nccl-u22-20250409t065232z]. +[2025-11-28 15:57:50] [SUCCESS] Deleted test-a4-nccl-u22-20250409t065232z +[2025-11-28 15:57:50] [EXECUTE] Deleting VM Image: test-a4-nccl-u22-20250409t074128z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/test-a4-nccl-u22-20250409t074128z]. +[2025-11-28 15:57:58] [SUCCESS] Deleted test-a4-nccl-u22-20250409t074128z +[2025-11-28 15:57:58] [EXECUTE] Deleting VM Image: try-a3hh-slurm-u22-20250925t075018z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/try-a3hh-slurm-u22-20250925t075018z]. +[2025-11-28 15:58:05] [SUCCESS] Deleted try-a3hh-slurm-u22-20250925t075018z +[2025-11-28 15:58:05] [EXECUTE] Deleting VM Image: try-a3hhi-slurm-u22-20250925t091429z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/try-a3hhi-slurm-u22-20250925t091429z]. +[2025-11-28 15:58:12] [SUCCESS] Deleted try-a3hhi-slurm-u22-20250925t091429z +[2025-11-28 15:58:13] [INFO] Hit delete limit (20) for VM Images. +[2025-11-28 15:58:13] [INFO] CLEANUP RUN FINISHED +[2025-11-28 16:00:00] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:00:00] [INFO] Time Cutoff (General): 2025-11-28T15:00:00+0000 +[2025-11-28 16:00:00] [INFO] Time Cutoff (Images): 2025-09-29T16:00:00+0000 +[2025-11-28 16:00:00] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:00:00] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:00:01] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-28 16:00:03] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 16:00:03] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 16:00:03] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 16:00:03] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 16:00:03] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 16:00:03] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 16:00:03] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-28 16:00:03] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-28 16:00:03] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-28 16:00:03] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-28 16:00:03] [DRY-RUN] Would delete VM Image: ysaarj-a3h-slurm-u22-20250925t053055z +[2025-11-28 16:00:03] [DRY-RUN] Would delete VM Image: ysaarj-a3h-slurm-u22-20250925t062142z +[2025-11-28 16:00:04] [DRY-RUN] Would delete VM Image: ysaj-a3h-slurm-u22-20250924t145223z +[2025-11-28 16:00:04] [INFO] CLEANUP RUN FINISHED +[2025-11-28 16:00:40] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:00:40] [INFO] Time Cutoff (General): 2025-11-28T15:00:40+0000 +[2025-11-28 16:00:40] [INFO] Time Cutoff (Images): 2025-09-29T16:00:40+0000 +[2025-11-28 16:00:40] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:00:40] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:00:40] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-28 16:00:42] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-28 16:00:42] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-28 16:00:42] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-28 16:00:42] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-28 16:00:42] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-28 16:00:42] [SKIP] pbspro0 (In Exclusion List) +[2025-11-28 16:00:42] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-28 16:00:42] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-28 16:00:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-28 16:00:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-28 16:00:43] [EXECUTE] Deleting VM Image: ysaarj-a3h-slurm-u22-20250925t053055z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/ysaarj-a3h-slurm-u22-20250925t053055z]. +[2025-11-28 16:00:50] [SUCCESS] Deleted ysaarj-a3h-slurm-u22-20250925t053055z +[2025-11-28 16:00:50] [EXECUTE] Deleting VM Image: ysaarj-a3h-slurm-u22-20250925t062142z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/ysaarj-a3h-slurm-u22-20250925t062142z]. +[2025-11-28 16:00:57] [SUCCESS] Deleted ysaarj-a3h-slurm-u22-20250925t062142z +[2025-11-28 16:00:57] [EXECUTE] Deleting VM Image: ysaj-a3h-slurm-u22-20250924t145223z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/ysaj-a3h-slurm-u22-20250924t145223z]. +[2025-11-28 16:01:05] [SUCCESS] Deleted ysaj-a3h-slurm-u22-20250924t145223z +[2025-11-28 16:01:05] [INFO] CLEANUP RUN FINISHED +[2025-11-28 16:01:17] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:01:17] [INFO] Time Cutoff (General): 2025-11-28T15:01:17+0000 +[2025-11-28 16:01:17] [INFO] Time Cutoff (Images): 2025-09-29T16:01:17+0000 +[2025-11-28 16:01:17] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:01:17] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:01:18] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 16:01:28] [INFO] CLEANUP RUN FINISHED +[2025-11-28 16:02:00] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:02:00] [INFO] Time Cutoff (General): 2025-11-28T15:02:00+0000 +[2025-11-28 16:02:00] [INFO] Time Cutoff (Images): 2025-09-29T16:02:00+0000 +[2025-11-28 16:02:00] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:02:00] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:02:01] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 16:02:13] [INFO] CLEANUP RUN FINISHED diff --git a/checking.txt b/checking.txt new file mode 100644 index 0000000000..869b83ae88 --- /dev/null +++ b/checking.txt @@ -0,0 +1,3090 @@ +[2025-11-29 11:10:50] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 11:10:50] [INFO] Time Cutoff (General): 2025-11-29T10:10:50+0000 +[2025-11-29 11:10:50] [INFO] Time Cutoff (Images): 2025-09-30T11:10:50+0000 +[2025-11-29 11:10:50] [INFO] Delete Limit per Type: 20 +[2025-11-29 11:10:50] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 11:10:50] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 11:10:52] [INFO] No Service Accounts found matching prefix. +[2025-11-29 11:10:52] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 11:10:53] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 11:10:53] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 11:10:55] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 11:10:55] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 11:10:55] [EXECUTE] Deleting Compute Instance: simtestnet-controller us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/simtestnet-controller]. +[2025-11-29 11:11:49] [SUCCESS] Deleted simtestnet-controller +[2025-11-29 11:11:49] [EXECUTE] Deleting Compute Instance: simtestnet-slurm-login-001 us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/simtestnet-slurm-login-001]. +[2025-11-29 11:12:42] [SUCCESS] Deleted simtestnet-slurm-login-001 +[2025-11-29 11:12:42] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 11:12:42] [INFO] --- Processing: Filestore (Limit: 20) --- +[2025-11-29 11:12:44] [EXECUTE] Deleting Filestore: simtestnet-5a54389c (Global) +ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. +The [instance] resource is not properly specified. +Failed to find attribute [zone]. The attribute can be set in the following ways: +- provide the argument `instance` on the command line with a fully specified name +- provide the argument `--zone` on the command line +- provide the argument `region` on the command line +- provide the argument `location` on the command line +- set the property `filestore/zone` +- set the property `filestore/region` +- set the property `filestore/location` +[2025-11-29 11:12:46] [ERROR] Failed to delete simtestnet-5a54389c +[2025-11-29 11:12:46] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 11:12:47] [EXECUTE] Deleting VM Image: a3mergesca-slurm-u22-20250929t173859z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3mergesca-slurm-u22-20250929t173859z]. +[2025-11-29 11:12:55] [SUCCESS] Deleted a3mergesca-slurm-u22-20250929t173859z +[2025-11-29 11:12:55] [EXECUTE] Deleting VM Image: a3mergesla-slurm-u22-20250930t043239z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3mergesla-slurm-u22-20250930t043239z]. +[2025-11-29 11:13:03] [SUCCESS] Deleted a3mergesla-slurm-u22-20250930t043239z +[2025-11-29 11:13:03] [EXECUTE] Deleting VM Image: a3qclavek-u22-20250930t085450z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3qclavek-u22-20250930t085450z]. +[2025-11-29 11:13:10] [SUCCESS] Deleted a3qclavek-u22-20250930t085450z +[2025-11-29 11:13:10] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 11:13:10] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 11:13:10] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 11:13:10] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 11:13:10] [EXECUTE] Deleting VM Image: ctk-swarnabm-sep29-03-u22-20250929t161201z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/ctk-swarnabm-sep29-03-u22-20250929t161201z]. +[2025-11-29 11:13:18] [SUCCESS] Deleted ctk-swarnabm-sep29-03-u22-20250929t161201z +[2025-11-29 11:13:18] [EXECUTE] Deleting VM Image: ctk-swarnabm-sep29-04-u22-20250930t022418z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/ctk-swarnabm-sep29-04-u22-20250930t022418z]. +[2025-11-29 11:13:26] [SUCCESS] Deleted ctk-swarnabm-sep29-04-u22-20250930t022418z +[2025-11-29 11:13:26] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 11:13:26] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 11:13:26] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 11:13:26] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 11:13:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 11:13:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 11:13:27] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 11:13:28] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 11:13:28] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 11:13:28] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 11:13:28] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 11:13:28] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-29 11:13:28] [EXECUTE] Deleting Cloud Router: simtestnet-net-0-router us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/simtestnet-net-0-router]. +[2025-11-29 11:13:32] [SUCCESS] Deleted simtestnet-net-0-router +[2025-11-29 11:13:32] [EXECUTE] Deleting Cloud Router: simtestnet-net-1-router us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/simtestnet-net-1-router]. +[2025-11-29 11:13:35] [SUCCESS] Deleted simtestnet-net-1-router +[2025-11-29 11:13:35] [EXECUTE] Deleting Cloud Router: simtestnet-net-router us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/simtestnet-net-router]. +[2025-11-29 11:13:39] [SUCCESS] Deleted simtestnet-net-router +[2025-11-29 11:13:39] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-29 11:13:41] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 11:13:41] [INFO] --- Processing: Regional Address (Limit: 20) --- +[2025-11-29 11:13:42] [EXECUTE] Deleting Regional Address: simtestnet-net-0-nat-ips-us-central1-0 us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-0-nat-ips-us-central1-0]. +[2025-11-29 11:13:45] [SUCCESS] Deleted simtestnet-net-0-nat-ips-us-central1-0 +[2025-11-29 11:13:45] [EXECUTE] Deleting Regional Address: simtestnet-net-0-nat-ips-us-central1-1 us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-0-nat-ips-us-central1-1]. +[2025-11-29 11:13:47] [SUCCESS] Deleted simtestnet-net-0-nat-ips-us-central1-1 +[2025-11-29 11:13:47] [EXECUTE] Deleting Regional Address: simtestnet-net-1-nat-ips-us-central1-0 us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-1-nat-ips-us-central1-0]. +[2025-11-29 11:13:49] [SUCCESS] Deleted simtestnet-net-1-nat-ips-us-central1-0 +[2025-11-29 11:13:49] [EXECUTE] Deleting Regional Address: simtestnet-net-1-nat-ips-us-central1-1 us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-1-nat-ips-us-central1-1]. +[2025-11-29 11:13:51] [SUCCESS] Deleted simtestnet-net-1-nat-ips-us-central1-1 +[2025-11-29 11:13:51] [EXECUTE] Deleting Regional Address: simtestnet-net-nat-ips-us-central1-0 us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-nat-ips-us-central1-0]. +[2025-11-29 11:13:54] [SUCCESS] Deleted simtestnet-net-nat-ips-us-central1-0 +[2025-11-29 11:13:54] [EXECUTE] Deleting Regional Address: simtestnet-net-nat-ips-us-central1-1 us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-nat-ips-us-central1-1]. +[2025-11-29 11:13:56] [SUCCESS] Deleted simtestnet-net-nat-ips-us-central1-1 +[2025-11-29 11:13:56] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 11:13:58] [INFO] No Global Address found matching criteria. +[2025-11-29 11:13:58] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 11:14:08] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 11:14:10] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 11:14:10] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 11:14:10] [EXECUTE] Deleting Zonal Disk: simtestnet-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/simtestnet-controller-save]. +[2025-11-29 11:14:12] [SUCCESS] Deleted simtestnet-controller-save +[2025-11-29 11:14:12] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 11:14:12] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 11:14:12] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 11:14:13] [SKIP] hpc-vpc (In Exclusion List) +WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork +[2025-11-29 11:14:15] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-0]. +[2025-11-29 11:14:27] [SUCCESS] Deleted simtestnet-mrdma-sub-0 +WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork +[2025-11-29 11:14:28] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-1]. +[2025-11-29 11:14:38] [SUCCESS] Deleted simtestnet-mrdma-sub-1 +WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork +[2025-11-29 11:14:40] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-2 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-2]. +[2025-11-29 11:14:51] [SUCCESS] Deleted simtestnet-mrdma-sub-2 +WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork +[2025-11-29 11:14:52] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-3 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-3]. +[2025-11-29 11:15:03] [SUCCESS] Deleted simtestnet-mrdma-sub-3 +WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork +[2025-11-29 11:15:05] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-4]. +[2025-11-29 11:15:16] [SUCCESS] Deleted simtestnet-mrdma-sub-4 +WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork +[2025-11-29 11:15:18] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-5 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-5]. +[2025-11-29 11:15:29] [SUCCESS] Deleted simtestnet-mrdma-sub-5 +WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork +[2025-11-29 11:15:31] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-6 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-6]. +[2025-11-29 11:15:42] [SUCCESS] Deleted simtestnet-mrdma-sub-6 +WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork +[2025-11-29 11:15:44] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-7 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-7]. +[2025-11-29 11:15:55] [SUCCESS] Deleted simtestnet-mrdma-sub-7 +WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork +[2025-11-29 11:15:57] [EXECUTE] Deleting Subnetwork: simtestnet-primary-subnet +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-primary-subnet]. +[2025-11-29 11:16:09] [SUCCESS] Deleted simtestnet-primary-subnet +WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork +[2025-11-29 11:16:11] [EXECUTE] Deleting Subnetwork: simtestnet-sub-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-sub-0]. +[2025-11-29 11:16:38] [SUCCESS] Deleted simtestnet-sub-0 +WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork +[2025-11-29 11:16:40] [EXECUTE] Deleting Subnetwork: simtestnet-sub-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-sub-1]. +[2025-11-29 11:16:50] [SUCCESS] Deleted simtestnet-sub-1 +[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:51] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 11:16:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:16:54] [EXECUTE] Deleting Dep. Route: default-route-2106bb9792a0cd0a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-2106bb9792a0cd0a]. +[2025-11-29 11:17:04] [SUCCESS] Deleted default-route-2106bb9792a0cd0a +[2025-11-29 11:17:04] [EXECUTE] Deleting Dep. Route: default-route-4853b5b69eb20ef0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-4853b5b69eb20ef0]. +[2025-11-29 11:17:26] [SUCCESS] Deleted default-route-4853b5b69eb20ef0 +[2025-11-29 11:17:26] [EXECUTE] Deleting Dep. Route: default-route-c4bfdfb86628f2ec +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-c4bfdfb86628f2ec]. +[2025-11-29 11:17:37] [SUCCESS] Deleted default-route-c4bfdfb86628f2ec +[2025-11-29 11:17:37] [EXECUTE] Deleting Dep. Route: peering-route-7922d039802e0f43 + + +Command killed by keyboard interrupt + +[2025-11-29 11:18:30] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 11:18:30] [INFO] Time Cutoff (General): 2025-11-29T10:18:30+0000 +[2025-11-29 11:18:30] [INFO] Time Cutoff (Images): 2025-09-30T11:18:30+0000 +[2025-11-29 11:18:30] [INFO] Delete Limit per Type: 20 +[2025-11-29 11:18:30] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 11:18:31] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 11:18:33] [INFO] No Service Accounts found matching prefix. +[2025-11-29 11:18:33] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 11:18:34] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 11:18:34] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 11:18:36] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 11:18:36] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 11:18:36] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 11:18:36] [INFO] --- Processing: Filestore (Limit: 20) --- +[2025-11-29 11:18:38] [EXECUTE] Deleting Filestore: simtestnet-5a54389c (Global) +ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. +The [instance] resource is not properly specified. +Failed to find attribute [zone]. The attribute can be set in the following ways: +- provide the argument `instance` on the command line with a fully specified name +- provide the argument `--zone` on the command line +- provide the argument `region` on the command line +- provide the argument `location` on the command line +- set the property `filestore/zone` +- set the property `filestore/region` +- set the property `filestore/location` +[2025-11-29 11:18:39] [ERROR] Failed to delete simtestnet-5a54389c +[2025-11-29 11:18:39] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 11:18:41] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 11:18:41] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 11:18:42] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 11:18:42] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 11:18:42] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 11:18:42] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 11:18:42] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 11:18:42] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 11:18:42] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 11:18:42] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 11:18:42] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 11:18:44] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 11:18:44] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 11:18:44] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 11:18:44] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 11:18:44] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-29 11:18:44] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-29 11:18:46] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 11:18:46] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 11:18:48] [INFO] No Regional Address found matching criteria. +[2025-11-29 11:18:48] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 11:18:49] [INFO] No Global Address found matching criteria. +[2025-11-29 11:18:49] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 11:18:59] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 11:19:01] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 11:19:01] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 11:19:01] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 11:19:01] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 11:19:01] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:19:03] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 11:19:05] [SKIP] hpc-vpc (In Exclusion List) +simtestnet-net https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net +[2025-11-29 11:19:07] [EXECUTE] Deleting Dep. Route: peering-route-7922d039802e0f43 +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +[2025-11-29 11:19:08] [ERROR] Failed to delete peering-route-7922d039802e0f43 +[2025-11-29 11:19:10] [EXECUTE] Deleting Network: simtestnet-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net]. +[2025-11-29 11:19:42] [SUCCESS] Deleted simtestnet-net +simtestnet-net-0 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net-0 +[2025-11-29 11:19:43] [EXECUTE] Deleting Dep. Route: peering-route-7922d039802e0f43 +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +[2025-11-29 11:19:45] [ERROR] Failed to delete peering-route-7922d039802e0f43 +[2025-11-29 11:19:47] [EXECUTE] Deleting Network: simtestnet-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net-0]. +[2025-11-29 11:20:39] [SUCCESS] Deleted simtestnet-net-0 +simtestnet-net-1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net-1 +[2025-11-29 11:20:42] [EXECUTE] Deleting Network: simtestnet-net-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net-1]. +[2025-11-29 11:21:21] [SUCCESS] Deleted simtestnet-net-1 +simtestnet-rdma-net https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-rdma-net +[2025-11-29 11:21:24] [EXECUTE] Deleting Network: simtestnet-rdma-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-rdma-net]. +[2025-11-29 11:21:53] [SUCCESS] Deleted simtestnet-rdma-net +[2025-11-29 11:21:53] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 11:21:54] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 11:21:54] [INFO] CLEANUP RUN FINISHED +[2025-11-29 11:24:15] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 11:24:15] [INFO] Time Cutoff (General): 2025-11-29T10:24:15+0000 +[2025-11-29 11:24:15] [INFO] Time Cutoff (Images): 2025-09-30T11:24:15+0000 +[2025-11-29 11:24:15] [INFO] Delete Limit per Type: 20 +[2025-11-29 11:24:15] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 11:24:15] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 11:24:17] [INFO] No Service Accounts found matching prefix. +[2025-11-29 11:24:17] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 11:24:19] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 11:24:19] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 11:24:20] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 11:24:21] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 11:24:21] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 11:24:21] [INFO] --- Processing: Filestore (Limit: 20) --- +[2025-11-29 11:24:23] [EXECUTE] Deleting Filestore: simtestnet-5a54389c (Global) +ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. +The [instance] resource is not properly specified. +Failed to find attribute [zone]. The attribute can be set in the following ways: +- provide the argument `instance` on the command line with a fully specified name +- provide the argument `--zone` on the command line +- provide the argument `region` on the command line +- provide the argument `location` on the command line +- set the property `filestore/zone` +- set the property `filestore/region` +- set the property `filestore/location` +[2025-11-29 11:24:24] [ERROR] Failed to delete simtestnet-5a54389c +[2025-11-29 11:24:24] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 11:24:26] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 11:24:26] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 11:24:27] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 11:24:27] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 11:24:27] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 11:24:27] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 11:24:27] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 11:24:27] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 11:24:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 11:24:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 11:24:27] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 11:24:29] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 11:24:29] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 11:24:29] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 11:24:29] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 11:24:29] [SKIP] default-router-us-west4 (In Exclusion List) +./cleanup.sh: line 544: process_firewalls: command not found +[2025-11-29 11:24:29] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 11:24:29] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 11:24:30] [INFO] No Regional Address found matching criteria. +[2025-11-29 11:24:30] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 11:24:32] [INFO] No Global Address found matching criteria. +[2025-11-29 11:24:32] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 11:24:36] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 11:24:37] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 11:24:37] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 11:24:37] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 11:24:37] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 11:24:37] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:40] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 11:24:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 11:24:41] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 11:24:43] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 11:24:43] [INFO] CLEANUP RUN FINISHED +[2025-11-29 11:26:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 11:26:47] [INFO] Time Cutoff (General): 2025-11-29T10:26:47+0000 +[2025-11-29 11:26:47] [INFO] Time Cutoff (Images): 2025-09-30T11:26:47+0000 +[2025-11-29 11:26:47] [INFO] Delete Limit per Type: 20 +[2025-11-29 11:26:47] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 11:26:48] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 11:26:49] [INFO] No Service Accounts found matching prefix. +[2025-11-29 11:26:49] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 11:26:51] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 11:26:51] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 11:26:53] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 11:26:53] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 11:26:53] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 11:26:53] [INFO] --- Processing: Filestore (Limit: 20) --- +[2025-11-29 11:26:55] [EXECUTE] Deleting Filestore: simtestnet-5a54389c (Global) +ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. +The [instance] resource is not properly specified. +Failed to find attribute [zone]. The attribute can be set in the following ways: +- provide the argument `instance` on the command line with a fully specified name +- provide the argument `--zone` on the command line +- provide the argument `region` on the command line +- provide the argument `location` on the command line +- set the property `filestore/zone` +- set the property `filestore/region` +- set the property `filestore/location` +[2025-11-29 11:26:56] [ERROR] Failed to delete simtestnet-5a54389c +[2025-11-29 11:26:56] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 11:26:58] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 11:26:58] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 11:26:58] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 11:26:58] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 11:26:58] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 11:26:58] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 11:26:58] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 11:26:58] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 11:26:59] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 11:26:59] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 11:26:59] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 11:27:01] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 11:27:01] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 11:27:01] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 11:27:01] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 11:27:01] [SKIP] default-router-us-west4 (In Exclusion List) +./cleanup.sh: line 545: process_firewalls: command not found +[2025-11-29 11:27:01] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 11:27:01] [INFO] --- Processing: Regional Address (Limit: 20) --- +Traceback (most recent call last): + File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 193, in + main() + File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 187, in main + gcloud_main = _import_gcloud_main() + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 90, in _import_gcloud_main + import googlecloudsdk.gcloud_main + File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/gcloud_main.py", line 42, in + from googlecloudsdk.core.credentials import creds_context_managers + File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/core/credentials/creds_context_managers.py", line 28, in + from googlecloudsdk.api_lib.iamcredentials import util as iamcred_util + File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/api_lib/iamcredentials/util.py", line 26, in + from apitools.base.py import http_wrapper + File "/usr/bin/../lib/google-cloud-sdk/lib/third_party/apitools/base/py/http_wrapper.py", line 39, in + from oauth2client.client import HttpAccessTokenRefreshError as TokenRefreshError # noqa + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/bin/../lib/google-cloud-sdk/lib/third_party/oauth2client/client.py", line 52, in + from oauth2client import crypt + File "/usr/bin/../lib/google-cloud-sdk/lib/third_party/oauth2client/crypt.py", line 41, in + from oauth2client import _openssl_crypt + File "/usr/bin/../lib/google-cloud-sdk/lib/third_party/oauth2client/_openssl_crypt.py", line 16, in + from OpenSSL import crypto + File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/OpenSSL/__init__.py", line 8, in + from OpenSSL import SSL, crypto + File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/OpenSSL/SSL.py", line 35, in + from OpenSSL.crypto import ( + File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/OpenSSL/crypto.py", line 22, in + from cryptography import utils, x509 + File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/cryptography/x509/__init__.py", line 8, in + from cryptography.x509.base import ( + File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/cryptography/x509/base.py", line 15, in + from cryptography.hazmat.primitives import hashes, serialization + File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/cryptography/hazmat/primitives/serialization/__init__.py", line 25, in + from cryptography.hazmat.primitives.serialization.ssh import ( + File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/cryptography/hazmat/primitives/serialization/ssh.py", line 19, in + from cryptography.hazmat.primitives.asymmetric import ( + File "", line 1360, in _find_and_load + File "", line 1331, in _find_and_load_unlocked + File "", line 935, in _load_unlocked + File "", line 995, in exec_module + File "", line 1128, in get_code + File "", line 757, in _compile_bytecode +KeyboardInterrupt +[2025-11-29 11:28:45] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 11:28:45] [INFO] Time Cutoff (General): 2025-11-29T10:28:45+0000 +[2025-11-29 11:28:45] [INFO] Time Cutoff (Images): 2025-09-30T11:28:45+0000 +[2025-11-29 11:28:45] [INFO] Delete Limit per Type: 20 +[2025-11-29 11:28:45] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 11:28:45] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 11:28:47] [INFO] No Service Accounts found matching prefix. +[2025-11-29 11:28:47] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 11:28:48] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 11:28:48] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 11:28:50] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 11:28:50] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 11:28:50] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 11:28:50] [INFO] --- Processing: Filestore (Limit: 20) --- +[2025-11-29 11:28:52] [EXECUTE] Deleting Filestore: simtestnet-5a54389c (Global) +ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. +The [instance] resource is not properly specified. +Failed to find attribute [zone]. The attribute can be set in the following ways: +- provide the argument `instance` on the command line with a fully specified name +- provide the argument `--zone` on the command line +- provide the argument `region` on the command line +- provide the argument `location` on the command line +- set the property `filestore/zone` +- set the property `filestore/region` +- set the property `filestore/location` +[2025-11-29 11:28:53] [ERROR] Failed to delete simtestnet-5a54389c +[2025-11-29 11:28:53] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 11:28:55] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 11:28:55] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 11:28:55] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 11:28:55] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 11:28:56] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 11:28:56] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 11:28:56] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 11:28:56] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 11:28:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 11:28:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 11:28:56] [INFO] --- Processing: Cloud Router (Limit: 20) --- + + +Command killed by keyboard interrupt + +[2025-11-29 19:02:55] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 19:02:55] [INFO] Time Cutoff (General): 2025-11-29T18:02:55+0000 +[2025-11-29 19:02:55] [INFO] Time Cutoff (Images): 2025-09-30T19:02:55+0000 +[2025-11-29 19:02:55] [INFO] Delete Limit per Type: 20 +[2025-11-29 19:02:55] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 19:02:55] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 19:02:58] [INFO] No Service Accounts found matching prefix. +[2025-11-29 19:02:58] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 19:03:00] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 19:03:00] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 19:03:02] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:03:02] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:03:02] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 19:03:02] [INFO] --- Processing: Filestore (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 19:03:04] [INFO] No Filestore found matching criteria. +[2025-11-29 19:03:04] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 19:03:07] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 19:03:07] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 19:03:07] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 19:03:07] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 19:03:07] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 19:03:07] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 19:03:07] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 19:03:07] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 19:03:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 19:03:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 19:03:08] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 19:03:10] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 19:03:10] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 19:03:10] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 19:03:10] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 19:03:10] [SKIP] default-router-us-west4 (In Exclusion List) +./cleanup.sh: line 570: process_firewalls: command not found +[2025-11-29 19:03:10] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 19:03:10] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:03:12] [INFO] No Regional Address found matching criteria. +[2025-11-29 19:03:12] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:03:14] [INFO] No Global Address found matching criteria. +[2025-11-29 19:03:14] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 19:03:17] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 19:03:19] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:03:19] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:03:19] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 19:03:19] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 19:03:19] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 19:03:21] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:22] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 19:03:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:03:24] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 19:03:25] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 19:03:25] [INFO] CLEANUP RUN FINISHED +[2025-11-29 19:14:16] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 19:14:16] [INFO] Time Cutoff (General): 2025-11-29T19:14:16+0000 +[2025-11-29 19:14:16] [INFO] Time Cutoff (Images): 2025-09-30T19:14:16+0000 +[2025-11-29 19:14:16] [INFO] Delete Limit per Type: 20 +[2025-11-29 19:14:16] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 19:14:16] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 19:14:18] [INFO] No Service Accounts found matching prefix. +[2025-11-29 19:14:18] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 19:14:19] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 19:14:20] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 19:14:23] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:14:23] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:14:23] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 19:14:23] [INFO] --- Processing: Filestore (Limit: 20) --- +[2025-11-29 19:14:25] [DRY-RUN] Would delete Filestore: simk (Global) +[2025-11-29 19:14:25] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 19:14:27] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 19:14:28] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 19:14:28] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 19:14:28] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 19:14:28] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 19:14:28] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 19:14:28] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 19:14:28] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 19:14:28] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 19:14:28] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 19:14:28] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 19:14:30] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 19:14:30] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 19:14:30] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 19:14:30] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 19:14:30] [SKIP] default-router-us-west4 (In Exclusion List) +./cleanup.sh: line 570: process_firewalls: command not found +[2025-11-29 19:14:30] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 19:14:30] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:14:32] [INFO] No Regional Address found matching criteria. +[2025-11-29 19:14:32] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:14:34] [INFO] No Global Address found matching criteria. +[2025-11-29 19:14:34] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 19:14:37] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 19:14:39] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:14:39] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:14:39] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 19:14:39] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 19:14:39] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 19:14:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:42] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 19:14:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:14:44] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 19:14:45] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 19:14:45] [INFO] CLEANUP RUN FINISHED +[2025-11-29 19:14:54] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 19:14:54] [INFO] Time Cutoff (General): 2025-11-29T19:14:54+0000 +[2025-11-29 19:14:54] [INFO] Time Cutoff (Images): 2025-09-30T19:14:54+0000 +[2025-11-29 19:14:54] [INFO] Delete Limit per Type: 20 +[2025-11-29 19:14:54] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 19:14:55] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 19:14:57] [INFO] No Service Accounts found matching prefix. +[2025-11-29 19:14:57] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 19:14:58] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 19:14:58] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 19:15:02] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:15:02] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:15:02] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 19:15:02] [INFO] --- Processing: Filestore (Limit: 20) --- +[2025-11-29 19:15:04] [EXECUTE] Deleting Filestore: simk (Global) +ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. +The [instance] resource is not properly specified. +Failed to find attribute [zone]. The attribute can be set in the following ways: +- provide the argument `instance` on the command line with a fully specified name +- provide the argument `--zone` on the command line +- provide the argument `region` on the command line +- provide the argument `location` on the command line +- set the property `filestore/zone` +- set the property `filestore/region` +- set the property `filestore/location` +[2025-11-29 19:15:05] [ERROR] Failed to delete simk +[2025-11-29 19:15:05] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 19:15:07] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 19:15:07] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 19:15:08] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 19:15:08] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 19:15:08] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 19:15:08] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 19:15:08] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 19:15:08] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 19:15:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 19:15:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 19:15:08] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 19:15:10] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 19:15:10] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 19:15:10] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 19:15:10] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 19:15:10] [SKIP] default-router-us-west4 (In Exclusion List) +./cleanup.sh: line 570: process_firewalls: command not found +[2025-11-29 19:15:10] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 19:15:10] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:15:12] [INFO] No Regional Address found matching criteria. +[2025-11-29 19:15:12] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:15:14] [INFO] No Global Address found matching criteria. +[2025-11-29 19:15:14] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 19:15:18] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 19:15:20] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:15:20] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:15:20] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 19:15:20] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 19:15:20] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:22] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 19:15:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:15:23] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 19:15:25] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 19:15:25] [INFO] CLEANUP RUN FINISHED +[2025-11-29 19:16:44] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 19:16:44] [INFO] Time Cutoff (General): 2025-11-29T19:16:44+0000 +[2025-11-29 19:16:44] [INFO] Time Cutoff (Images): 2025-09-30T19:16:44+0000 +[2025-11-29 19:16:44] [INFO] Delete Limit per Type: 20 +[2025-11-29 19:16:44] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 19:16:44] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 19:16:46] [INFO] No Service Accounts found matching prefix. +[2025-11-29 19:16:46] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 19:16:47] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 19:16:47] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 19:16:50] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:16:50] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:16:50] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 19:16:50] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +[2025-11-29 19:16:52] [DEBUG] Filestore JSON Output: [ + { + "createTime": "2025-11-29T19:05:02.285881388Z", + "customPerformanceSupported": true, + "fileShares": [ + { + "capacityGb": "1024", + "name": "hhh" + } + ], + "name": "projects/hpc-toolkit-dev/locations/us-central1/instances/simk", + "networks": [ + { + "connectMode": "DIRECT_PEERING", + "ipAddresses": [ + "10.203.98.194" + ], + "modes": [ + "MODE_IPV4" + ], + "network": "hpc-vpc", + "reservedIpRange": "10.203.98.192/26" + } + ], + "performanceLimits": { + "maxIops": "12000", + "maxReadIops": "12000", + "maxReadThroughputBps": "125829120", + "maxWriteIops": "4000", + "maxWriteThroughputBps": "104857600" + }, + "protocol": "NFS_V3", + "state": "READY", + "tier": "REGIONAL" + } +] +[2025-11-29 19:16:52] [DEBUG] Parsed Filestore List (Location\tName): +[2025-11-29 19:16:52] [DEBUG] us-central1 simk +[2025-11-29 19:16:52] [DEBUG] Processing Instance: Name='simk', Location='us-central1' +[2025-11-29 19:16:52] [DRY-RUN] Would delete Filestore: simk (us-central1) +[2025-11-29 19:16:52] [INFO] Finished processing Filestore instances. Attempted to delete 1. +[2025-11-29 19:16:52] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 19:16:55] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 19:16:55] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 19:16:55] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 19:16:55] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 19:16:55] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 19:16:55] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 19:16:55] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 19:16:55] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 19:16:55] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 19:16:55] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 19:16:55] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 19:16:57] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 19:16:57] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 19:16:57] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 19:16:57] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 19:16:57] [SKIP] default-router-us-west4 (In Exclusion List) +./cleanup.sh: line 567: process_firewalls: command not found +[2025-11-29 19:16:57] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 19:16:57] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:16:59] [INFO] No Regional Address found matching criteria. +[2025-11-29 19:16:59] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:17:01] [INFO] No Global Address found matching criteria. +[2025-11-29 19:17:01] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 19:17:05] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 19:17:07] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:17:07] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:17:07] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 19:17:07] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 19:17:07] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:09] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 19:17:11] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:11] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 19:17:12] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 19:17:12] [INFO] CLEANUP RUN FINISHED +[2025-11-29 19:17:20] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 19:17:20] [INFO] Time Cutoff (General): 2025-11-29T19:17:20+0000 +[2025-11-29 19:17:20] [INFO] Time Cutoff (Images): 2025-09-30T19:17:20+0000 +[2025-11-29 19:17:20] [INFO] Delete Limit per Type: 20 +[2025-11-29 19:17:20] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 19:17:21] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 19:17:22] [INFO] No Service Accounts found matching prefix. +[2025-11-29 19:17:22] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 19:17:24] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 19:17:24] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 19:17:26] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:17:26] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:17:26] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 19:17:26] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +[2025-11-29 19:17:28] [DEBUG] Filestore JSON Output: [ + { + "createTime": "2025-11-29T19:05:02.285881388Z", + "customPerformanceSupported": true, + "fileShares": [ + { + "capacityGb": "1024", + "name": "hhh" + } + ], + "name": "projects/hpc-toolkit-dev/locations/us-central1/instances/simk", + "networks": [ + { + "connectMode": "DIRECT_PEERING", + "ipAddresses": [ + "10.203.98.194" + ], + "modes": [ + "MODE_IPV4" + ], + "network": "hpc-vpc", + "reservedIpRange": "10.203.98.192/26" + } + ], + "performanceLimits": { + "maxIops": "12000", + "maxReadIops": "12000", + "maxReadThroughputBps": "125829120", + "maxWriteIops": "4000", + "maxWriteThroughputBps": "104857600" + }, + "protocol": "NFS_V3", + "state": "READY", + "tier": "REGIONAL" + } +] +[2025-11-29 19:17:28] [DEBUG] Parsed Filestore List (Location\tName): +[2025-11-29 19:17:28] [DEBUG] us-central1 simk +[2025-11-29 19:17:28] [DEBUG] Processing Instance: Name='simk', Location='us-central1' +[2025-11-29 19:17:28] [DRY-RUN] Would delete Filestore: simk (us-central1) +[2025-11-29 19:17:28] [INFO] Finished processing Filestore instances. Attempted to delete 1. +[2025-11-29 19:17:28] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 19:17:30] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 19:17:30] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 19:17:30] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 19:17:30] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 19:17:30] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 19:17:30] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 19:17:30] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 19:17:30] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 19:17:31] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 19:17:31] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 19:17:31] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 19:17:32] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 19:17:32] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 19:17:32] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 19:17:32] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 19:17:32] [SKIP] default-router-us-west4 (In Exclusion List) +./cleanup.sh: line 567: process_firewalls: command not found +[2025-11-29 19:17:32] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 19:17:32] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:17:34] [INFO] No Regional Address found matching criteria. +[2025-11-29 19:17:34] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:17:36] [INFO] No Global Address found matching criteria. +[2025-11-29 19:17:36] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 19:17:40] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 19:17:42] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:17:42] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:17:42] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 19:17:42] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 19:17:42] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:44] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 19:17:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:17:46] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 19:17:47] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 19:17:47] [INFO] CLEANUP RUN FINISHED +[2025-11-29 19:17:50] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 19:17:50] [INFO] Time Cutoff (General): 2025-11-29T19:17:50+0000 +[2025-11-29 19:17:50] [INFO] Time Cutoff (Images): 2025-09-30T19:17:50+0000 +[2025-11-29 19:17:50] [INFO] Delete Limit per Type: 20 +[2025-11-29 19:17:50] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 19:17:50] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 19:17:52] [INFO] No Service Accounts found matching prefix. +[2025-11-29 19:17:52] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 19:17:53] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 19:17:53] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 19:17:56] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:17:56] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:17:56] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 19:17:56] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +[2025-11-29 19:17:58] [DEBUG] Filestore JSON Output: [ + { + "createTime": "2025-11-29T19:05:02.285881388Z", + "customPerformanceSupported": true, + "fileShares": [ + { + "capacityGb": "1024", + "name": "hhh" + } + ], + "name": "projects/hpc-toolkit-dev/locations/us-central1/instances/simk", + "networks": [ + { + "connectMode": "DIRECT_PEERING", + "ipAddresses": [ + "10.203.98.194" + ], + "modes": [ + "MODE_IPV4" + ], + "network": "hpc-vpc", + "reservedIpRange": "10.203.98.192/26" + } + ], + "performanceLimits": { + "maxIops": "12000", + "maxReadIops": "12000", + "maxReadThroughputBps": "125829120", + "maxWriteIops": "4000", + "maxWriteThroughputBps": "104857600" + }, + "protocol": "NFS_V3", + "state": "READY", + "tier": "REGIONAL" + } +] +[2025-11-29 19:17:58] [DEBUG] Parsed Filestore List (Location\tName): +[2025-11-29 19:17:58] [DEBUG] us-central1 simk +[2025-11-29 19:17:58] [DEBUG] Processing Instance: Name='simk', Location='us-central1' +[2025-11-29 19:17:58] [EXECUTE] Deleting Filestore: simk (us-central1) +Waiting for [operation-1764443880298-644c09ab6193d-0d3292e2-0685018e] to finish... +..............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................done. +[2025-11-29 19:22:49] [SUCCESS] Deleted simk +[2025-11-29 19:22:49] [INFO] Finished processing Filestore instances. Attempted to delete 1. +[2025-11-29 19:22:49] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 19:22:51] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 19:22:51] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 19:22:52] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 19:22:52] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 19:22:52] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 19:22:52] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 19:22:52] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 19:22:52] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 19:22:52] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 19:22:52] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 19:22:52] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 19:22:54] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 19:22:54] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 19:22:54] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 19:22:54] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 19:22:54] [SKIP] default-router-us-west4 (In Exclusion List) +./cleanup.sh: line 567: process_firewalls: command not found +[2025-11-29 19:22:54] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 19:22:54] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:22:56] [INFO] No Regional Address found matching criteria. +[2025-11-29 19:22:56] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:22:58] [INFO] No Global Address found matching criteria. +[2025-11-29 19:22:58] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 19:23:01] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 19:23:03] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:23:03] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:23:03] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 19:23:03] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 19:23:03] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:06] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 19:23:07] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:23:07] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 19:23:09] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 19:23:09] [INFO] CLEANUP RUN FINISHED +[2025-11-29 19:26:33] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 19:26:33] [INFO] Time Cutoff (General): 2025-11-29T19:26:33+0000 +[2025-11-29 19:26:33] [INFO] Time Cutoff (Images): 2025-09-30T19:26:33+0000 +[2025-11-29 19:26:33] [INFO] Delete Limit per Type: 20 +[2025-11-29 19:26:33] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 19:26:33] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 19:26:35] [INFO] No Service Accounts found matching prefix. +[2025-11-29 19:26:35] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 19:26:37] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 19:26:37] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 19:26:38] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:26:38] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:26:38] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 19:26:38] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 19:26:41] [INFO] No Filestore instances found matching criteria. +[2025-11-29 19:26:41] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 19:26:43] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 19:26:43] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 19:26:43] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 19:26:43] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 19:26:44] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 19:26:44] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 19:26:44] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 19:26:44] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 19:26:44] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 19:26:44] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 19:26:44] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 19:26:46] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 19:26:46] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 19:26:46] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 19:26:46] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 19:26:46] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-29 19:26:46] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-29 19:26:48] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 19:26:48] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:26:49] [INFO] No Regional Address found matching criteria. +[2025-11-29 19:26:49] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 19:26:51] [INFO] No Global Address found matching criteria. +[2025-11-29 19:26:51] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 19:26:55] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 19:26:57] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 19:26:57] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 19:26:57] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 19:26:57] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 19:26:57] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:26:59] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 19:27:01] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 19:27:01] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 19:27:02] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 19:27:02] [INFO] CLEANUP RUN FINISHED + +[2025-11-29 20:50:13] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 20:50:13] [INFO] Time Cutoff (General): 2025-11-29T20:50:13+0000 +[2025-11-29 20:50:13] [INFO] Time Cutoff (Images): 2025-09-30T20:50:13+0000 +[2025-11-29 20:50:13] [INFO] Delete Limit per Type: 20 +[2025-11-29 20:50:13] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 20:50:13] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 20:50:15] [INFO] No Service Accounts found matching prefix. +[2025-11-29 20:50:15] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 20:50:17] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 20:50:17] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 20:50:19] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 20:50:19] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 20:50:19] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 20:50:19] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 20:50:21] [INFO] No Filestore instances found matching criteria. +[2025-11-29 20:50:21] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 20:50:24] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 20:50:24] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 20:50:24] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 20:50:24] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 20:50:24] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 20:50:24] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 20:50:24] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 20:50:24] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 20:50:24] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 20:50:24] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 20:50:24] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 20:50:26] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 20:50:26] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 20:50:26] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 20:50:26] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 20:50:26] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-29 20:50:26] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-29 20:50:28] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 20:50:28] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 20:50:30] [INFO] No Regional Address found matching criteria. +[2025-11-29 20:50:30] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 20:50:32] [INFO] No Global Address found matching criteria. +[2025-11-29 20:50:32] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 20:50:36] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 20:50:38] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 20:50:38] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 20:50:38] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 20:50:38] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 20:50:38] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:41] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 20:50:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:50:43] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 20:50:44] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 20:50:44] [INFO] CLEANUP RUN FINISHED +[2025-11-29 20:51:05] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 20:51:06] [INFO] Time Cutoff (General): 2025-11-29T20:51:05+0000 +[2025-11-29 20:51:06] [INFO] Time Cutoff (Images): 2025-09-30T20:51:05+0000 +[2025-11-29 20:51:06] [INFO] Delete Limit per Type: 20 +[2025-11-29 20:51:06] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 20:51:06] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 20:51:08] [INFO] No Service Accounts found matching prefix. +[2025-11-29 20:51:08] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 20:51:09] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 20:51:09] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 20:51:11] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 20:51:11] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 20:51:11] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 20:51:11] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 20:51:14] [INFO] No Filestore instances found matching criteria. +[2025-11-29 20:51:14] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 20:51:16] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 20:51:16] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 20:51:16] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 20:51:16] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 20:51:16] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 20:51:16] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 20:51:16] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 20:51:16] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 20:51:17] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 20:51:17] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 20:51:17] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[2025-11-29 20:51:17] [INFO] Policy: Delete images updated before Sat Nov 15 08:51:17 PM UTC 2025 +[2025-11-29 20:51:20] [INFO] Scanning Target Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) +[2025-11-29 20:51:40] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 20:51:42] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 20:51:42] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 20:51:42] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 20:51:42] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 20:51:42] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-29 20:51:42] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-29 20:51:44] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 20:51:44] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 20:51:46] [INFO] No Regional Address found matching criteria. +[2025-11-29 20:51:46] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 20:51:48] [INFO] No Global Address found matching criteria. +[2025-11-29 20:51:48] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 20:51:52] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 20:51:54] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 20:51:54] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 20:51:54] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 20:51:54] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 20:51:54] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:56] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 20:51:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 20:51:58] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 20:52:00] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 20:52:00] [INFO] CLEANUP RUN FINISHED +[2025-11-29 20:59:28] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-29 20:59:28] [INFO] Time Cutoff (General): 2025-11-29T20:59:28+0000 +[2025-11-29 20:59:28] [INFO] Time Cutoff (Images): 2025-09-30T20:59:28+0000 +[2025-11-29 20:59:28] [INFO] Delete Limit per Type: 20 +[2025-11-29 20:59:28] [INFO] Loading exclusions from exclusions.txt... +[2025-11-29 20:59:29] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-29 20:59:30] [INFO] No Service Accounts found matching prefix. +[2025-11-29 20:59:30] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 20:59:32] [INFO] No GKE Cluster found matching criteria. +[2025-11-29 20:59:32] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-29 20:59:34] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 20:59:34] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 20:59:34] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-29 20:59:34] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-29 20:59:36] [INFO] No Filestore instances found matching criteria. +[2025-11-29 20:59:36] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-29 20:59:38] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-29 20:59:39] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-29 20:59:39] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-29 20:59:39] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-29 20:59:39] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-29 20:59:39] [SKIP] pbspro0 (In Exclusion List) +[2025-11-29 20:59:39] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-29 20:59:39] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-29 20:59:39] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-29 20:59:39] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-29 20:59:39] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[2025-11-29 20:59:39] [INFO] Policy: Delete images updated before Sat Nov 15 08:59:39 PM UTC 2025 (Timestamp: 1763240379) +[2025-11-29 20:59:43] [INFO] Scanning Target Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new +[2025-11-29 21:00:04] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-29 21:00:06] [SKIP] default-net-router (In Exclusion List) +[2025-11-29 21:00:06] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-29 21:00:06] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-29 21:00:06] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-29 21:00:06] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-29 21:00:06] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-29 21:00:08] [INFO] --- Processing: Compute Addresses --- +[2025-11-29 21:00:08] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 21:00:10] [INFO] No Regional Address found matching criteria. +[2025-11-29 21:00:10] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-29 21:00:12] [INFO] No Global Address found matching criteria. +[2025-11-29 21:00:12] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-29 21:00:16] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-29 21:00:18] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-29 21:00:18] [SKIP] image-inspector (In Exclusion List) +[2025-11-29 21:00:18] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-29 21:00:18] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-29 21:00:18] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:20] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-29 21:00:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-29 21:00:22] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-29 21:00:24] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-29 21:00:24] [INFO] CLEANUP RUN FINISHED diff --git a/cleanup.sh b/cleanup.sh new file mode 100755 index 0000000000..efdca4e398 --- /dev/null +++ b/cleanup.sh @@ -0,0 +1,711 @@ +#!/bin/bash + +# ============================================================================== +# CONFIGURATION & GLOBAL VARIABLES +# ============================================================================== + +set -u +set -o pipefail + +# Output Redirection +LOG_FILE="template.txt" +exec >> "$LOG_FILE" 2>&1 + +PROJECT_ID="hpc-toolkit-dev" +DRY_RUN="false" # Set to "false" to actually delete +EXCLUSION_FILE="exclusions.txt" +DELETE_LIMIT=200 +PROTECTED_SUBSTRING="topology" + +# Service Account Config +SA_DELETE_PREFIX="test-sa-" + +# VM Image Config +IMAGE_AGE_DAYS=60 + +# Time Calculations +# Standard Resource Cutoff (1 hour buffer) +CUTOFF_TIME=$(date -d "5 hours ago" -u +%Y-%m-%dT%H:%M:%S%z) +# Image Cutoff (60 days ago) +CUTOFF_TIME_IMAGES=$(date -d "$IMAGE_AGE_DAYS days ago" -u +%Y-%m-%dT%H:%M:%S%z) + +# Associative array for exclusions +declare -A EXCLUSION_MAP + +# ============================================================================== +# HELPER FUNCTIONS +# ============================================================================== + +log() { + local level="$1" + local message="$2" + echo "[$(date +'%Y-%m-%d %H:%M:%S')] [$level] $message" +} + +check_dependencies() { + local dependencies=("gcloud" "awk" "grep" "sort" "jq" "date") + for cmd in "${dependencies[@]}"; do + if ! command -v "$cmd" &> /dev/null; then + log "ERROR" "Missing required dependency: $cmd" + exit 1 + fi + done +} + +load_exclusions() { + if [[ ! -f "$EXCLUSION_FILE" ]]; then + log "WARNING" "Exclusion file not found: $EXCLUSION_FILE. Proceeding without exclusions." + return + fi + + log "INFO" "Loading exclusions from $EXCLUSION_FILE..." + while IFS= read -r line || [[ -n "$line" ]]; do + local trimmed_line + trimmed_line=$(echo "$line" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + if [[ -n "$trimmed_line" ]] && [[ "$trimmed_line" != \#* ]]; then + EXCLUSION_MAP["$trimmed_line"]=1 + fi + done < "$EXCLUSION_FILE" +} + +is_excluded() { + local resource_name="$1" + if [[ "$resource_name" == *"$PROTECTED_SUBSTRING"* ]]; then + log "SKIP" "$resource_name (Protected Substring)" + return 0 + fi + if [[ -n "${EXCLUSION_MAP[$resource_name]:-}" ]]; then + log "SKIP" "$resource_name (In Exclusion List)" + return 0 + fi + return 1 +} + +execute_delete() { + local resource_type="$1" + local resource_name="$2" + local cmd_str="$3" + local extra_info="${4:-}" + + if [[ "$DRY_RUN" == "true" ]]; then + log "DRY-RUN" "Would delete $resource_type: $resource_name $extra_info" + else + log "EXECUTE" "Deleting $resource_type: $resource_name $extra_info" + if eval "$cmd_str"; then + log "SUCCESS" "Deleted $resource_name" + else + log "ERROR" "Failed to delete $resource_name" + fi + fi +} + +# ============================================================================== +# STANDARD PROCESSOR +# ============================================================================== + +process_resources() { + local label="$1" + local list_command="$2" + local delete_command_base="$3" + local scope_type="$4" # location, zone, region, or none + + log "INFO" "--- Processing: $label (Limit: $DELETE_LIMIT) ---" + + local resources + if ! resources=$(eval "$list_command"); then + log "ERROR" "Failed to list $label" + return + fi + + if [[ -z "$resources" ]]; then + log "INFO" "No $label found matching criteria." + return + fi + + local count=0 + while read -r line; do + [[ -z "$line" ]] && continue + local name scope + read -r name scope <<< "$line" + + if [[ -z "$name" ]]; then continue; fi + + # --- LIMIT CHECK --- + if [[ $count -ge $DELETE_LIMIT ]]; then + log "INFO" "Hit delete limit ($DELETE_LIMIT) for $label." + break + fi + + if is_excluded "$name"; then continue; fi + + local final_cmd="$delete_command_base \"$name\" --quiet" + if [[ "$scope_type" != "none" && -n "$scope" ]]; then + final_cmd="$final_cmd --$scope_type=\"$scope\"" + fi + + execute_delete "$label" "$name" "$final_cmd" "${scope:-(Global)}" + ((count++)) + done <<< "$resources" +} + +# ============================================================================== +# SPECIFIC HANDLERS +# ============================================================================== + +process_instance_templates() { + log "INFO" "--- Processing: Instance Templates (Limit: $DELETE_LIMIT) ---" + + # List templates created before CUTOFF_TIME + local templates + if ! templates=$(gcloud compute instance-templates list \ + --project="$PROJECT_ID" \ + --filter="creationTimestamp < '$CUTOFF_TIME'" \ + --format="value(name)" | sort); then + log "ERROR" "Failed to list instance templates." + return 1 + fi + + if [[ -z "$templates" ]]; then + log "INFO" "No instance templates found matching criteria." + return + fi + + local count=0 + while read -r name; do + if [[ -z "$name" ]]; then continue; fi + + # --- LIMIT CHECK --- + if [[ $count -ge $DELETE_LIMIT ]]; then + log "INFO" "Hit delete limit ($DELETE_LIMIT) for Instance Templates." + break + fi + + if is_excluded "$name"; then continue; fi + + execute_delete "Instance Template" "$name" \ + "gcloud compute instance-templates delete \"$name\" --project=\"$PROJECT_ID\" --quiet" \ + "(Global)" + + ((count++)) + done <<< "$templates" +} + +process_addresses() { + log "INFO" "--- Processing: Compute Addresses ---" + + # These use the Standard Processor, so the limit is handled inside process_resources + process_resources "Regional Address" \ + "gcloud compute addresses list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND region:*\" --format=\"value(name,region)\" | sort" \ + "gcloud compute addresses delete --project=\"$PROJECT_ID\"" \ + "region" + + process_resources "Global Address" \ + "gcloud compute addresses list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND NOT region:*\" --format=\"value(name)\" | sort" \ + "gcloud compute addresses delete --project=\"$PROJECT_ID\" --global" \ + "none" +} + + +process_vpc_peerings() { + log "INFO" "--- Processing: VPC Peerings (Limit: $DELETE_LIMIT) ---" + + local networks_json + if ! networks_json=$(gcloud compute networks list --project="$PROJECT_ID" --format="json"); then + log "ERROR" "Failed to list networks." + return 1 + fi + + if [[ -z "$networks_json" || "$networks_json" == "[]" ]]; then + log "INFO" "No networks found in project." + return + fi + + local count=0 + + # Use process substitution <(...) to avoid subshell issues so 'count' updates correctly + while IFS= read -r net_obj; do + if [[ $count -ge $DELETE_LIMIT ]]; then break; fi + + local net_name + net_name=$(echo "$net_obj" | jq -r '.name') + + if [[ -z "$net_name" || "$net_name" == "null" ]]; then continue; fi + + # log "DEBUG" "Checking network: $net_name" + + # Check for peerings array inside the network object + local peerings_json + peerings_json=$(echo "$net_obj" | jq -c '.peerings // []') + + # If empty array or null, skip + if [[ "$peerings_json" == "[]" || "$peerings_json" == "null" ]]; then + continue + fi + + # Inner loop for peerings + while IFS= read -r peering_obj; do + if [[ $count -ge $DELETE_LIMIT ]]; then break; fi + + local peering_name + peering_name=$(echo "$peering_obj" | jq -r '.name') + + if [[ -z "$peering_name" || "$peering_name" == "null" ]]; then + log "DEBUG" " Skipping peering with no name" + continue + fi + + local peer_network + peer_network=$(echo "$peering_obj" | jq -r '.network // ""') + local state + state=$(echo "$peering_obj" | jq -r '.state // ""') + + if is_excluded "$peering_name" || is_excluded "$net_name"; then + continue + fi + + if [[ "$peering_name" == "servicenetworking-googleapis-com" ]]; then + log "INFO" " [ACTION] Deleting Service Networking peering on: $net_name" + execute_delete "Service Peering" "$peering_name" \ + "gcloud services vpc-peerings delete --service=servicenetworking.googleapis.com --network=\"$net_name\" --project=\"$PROJECT_ID\" --quiet" \ + "(Network: $net_name)" + ((count++)) + elif [[ "$peering_name" == filestore-peer-* ]]; then + log "INFO" " [SKIP] Managed Filestore peering: $peering_name on $net_name. This is tied to a Filestore instance lifecycle." + continue + elif [[ "$peer_network" == *"/global/networks/servicenetworking" ]]; then + log "INFO" " [SKIP] Reverse Service Networking peering: $peering_name on $net_name" + continue + else + # Standard VPC Peering + log "INFO" " [ACTION] Deleting Standard VPC peering: $peering_name on: $net_name (State: $state)" + execute_delete "VPC Peering" "$peering_name" \ + "gcloud compute networks peerings delete \"$peering_name\" --network=\"$net_name\" --project=\"$PROJECT_ID\" --quiet" \ + "(Network: $net_name, State: $state)" + ((count++)) + fi + done < <(echo "$peerings_json" | jq -c '.[]') + + done < <(echo "$networks_json" | jq -c '.[]') + + if [[ $count -ge $DELETE_LIMIT ]]; then + log "INFO" "Hit delete limit ($DELETE_LIMIT) for VPC Peerings." + fi + log "INFO" "Finished processing VPC Peerings. $count peerings actioned." +} + +process_service_accounts() { + log "INFO" "--- Processing: Service Accounts (Prefix: $SA_DELETE_PREFIX) ---" + + # We use 'head -n' here to enforce the limit at the list level + local sas + sas=$(gcloud iam service-accounts list --project="$PROJECT_ID" \ + --filter="email ~ ^$SA_DELETE_PREFIX" \ + --format="value(email)" | head -n "$DELETE_LIMIT") + + if [[ -z "$sas" ]]; then + log "INFO" "No Service Accounts found matching prefix." + return + fi + + for email in $sas; do + if is_excluded "$email"; then continue; fi + execute_delete "Service Account" "$email" \ + "gcloud iam service-accounts delete \"$email\" --project=\"$PROJECT_ID\" --quiet" + done +} + +process_iam_deleted_members() { + log "INFO" "--- Processing: IAM Role Bindings for Deleted SAs (Limit: $DELETE_LIMIT) ---" + + local policy_json + if ! policy_json=$(gcloud projects get-iam-policy "$PROJECT_ID" --format=json); then + log "ERROR" "Failed to get IAM policy." + return + fi + + local deleted_bindings + deleted_bindings=$(echo "$policy_json" | jq -r '.bindings[] | .role as $r | .members[] | select(startswith("deleted:serviceAccount:")) | "\($r)\t\(.)"') + + if [[ -z "$deleted_bindings" ]]; then + log "INFO" "No 'deleted:serviceAccount' bindings found." + return + fi + + local count=0 + + while IFS=$'\t' read -r role member; do + if [[ -z "$role" || -z "$member" ]]; then continue; fi + + # --- LIMIT CHECK --- + if [[ $count -ge $DELETE_LIMIT ]]; then + log "INFO" "Hit delete limit ($DELETE_LIMIT) for IAM Bindings." + break + fi + + if [[ "$DRY_RUN" == "true" ]]; then + log "DRY-RUN" "Would remove IAM binding: $member from role $role" + else + log "EXECUTE" "Removing IAM binding: $member from role $role" + gcloud projects remove-iam-policy-binding "$PROJECT_ID" \ + --member="$member" --role="$role" --condition=None --quiet >/dev/null || log "ERROR" "Failed to remove binding" + fi + + ((count++)) + done <<< "$deleted_bindings" +} + +process_vm_images() { + log "INFO" "--- Processing: VM Images (Limit: $DELETE_LIMIT) ---" + + local images + images=$(gcloud compute images list --project="$PROJECT_ID" --no-standard-images \ + --format="value(name,creationTimestamp)") + + if [[ -z "$images" ]]; then + log "INFO" "No custom VM images found." + return + fi + + local cutoff_seconds + cutoff_seconds=$(date -d "$CUTOFF_TIME_IMAGES" +%s) + local count=0 + + while read -r name timestamp; do + [[ -z "$name" ]] && continue + + # --- LIMIT CHECK --- + if [[ $count -ge $DELETE_LIMIT ]]; then + log "INFO" "Hit delete limit ($DELETE_LIMIT) for VM Images." + break + fi + + if is_excluded "$name"; then continue; fi + + local ts_seconds + if ! ts_seconds=$(date -d "$timestamp" +%s 2>/dev/null); then + log "WARNING" "Could not parse timestamp '$timestamp' for image $name. Skipping." + continue + fi + + if [[ $ts_seconds -lt $cutoff_seconds ]]; then + execute_delete "VM Image" "$name" \ + "gcloud compute images delete \"$name\" --project=\"$PROJECT_ID\" --quiet" + ((count++)) + fi + + done <<< "$images" +} + +process_docker_images() { + log "INFO" "--- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: $DELETE_LIMIT) ---" + + # 1. Calculate Cutoff (14 Days Ago) using UTC + local cutoff_date + cutoff_date=$(date -u -d "14 days ago" '+%Y-%m-%dT%H:%M:%SZ') + local cutoff_seconds + if ! cutoff_seconds=$(date -u -d "$cutoff_date" +%s); then + log "ERROR" "Failed to calculate cutoff_seconds." + return 1 + fi + log "INFO" "Policy: Delete 'test-runner' images updated before $cutoff_date (Unix: $cutoff_seconds)" + + local location="us-central1" + local repo_name="hpc-toolkit-repo" + local package_name="test-runner" + local full_package_url="${location}-docker.pkg.dev/${PROJECT_ID}/${repo_name}/${package_name}" + + log "INFO" "Scanning Target Package: $full_package_url" + + # 5. List image versions for "test-runner" using CSV format + local images_output + if ! images_output=$(gcloud artifacts docker images list "$full_package_url" \ + --format="csv[no-heading](uri,updateTime)" \ + --sort-by="updateTime"); then + log "WARNING" "Failed to list images for $full_package_url" + return 1 + fi + + if [[ -z "$images_output" ]]; then + log "INFO" " > No image versions found for $package_name." + return + fi + + local count=0 + # 6. Iterate Images Line by Line (using CSV parsing) + while IFS=, read -r full_image_ref update_time; do + if [[ -z "$full_image_ref" ]]; then continue; fi + + # Sanity check: Ensure we have a valid timestamp + if [[ -z "$update_time" ]]; then + log "DEBUG" " Skipping line, missing timestamp for: $full_image_ref" + continue + fi + + # We only care about images with a digest in the URI (@sha256:...) + if [[ "$full_image_ref" != *"@sha256:"* ]]; then + # log "DEBUG" " Skipping URI without digest: $full_image_ref" + continue + fi + + # --- TIME CHECK --- + local image_seconds + if ! image_seconds=$(date -u -d "$update_time" +%s 2>/dev/null); then + log "WARNING" " Could not parse date '$update_time' for $full_image_ref. Skipping." + continue + fi + + if [[ $image_seconds -ge $cutoff_seconds ]]; then + log "INFO" " [KEEP] .../test-runner... (Updated: $update_time) - Too new" + else + # --- DELETE LOGIC --- + if [[ $count -ge $DELETE_LIMIT ]]; then + log "INFO" "Hit delete limit ($DELETE_LIMIT) for Docker Images." + break # Break the loop, don't exit script + fi + + if is_excluded "$package_name"; then + log "INFO" " [SKIP] $package_name is excluded" + continue + fi + + log "INFO" " [DELETE] $full_image_ref (Updated: $update_time)" + + execute_delete "Docker Image Version" "$full_image_ref" \ + "gcloud artifacts docker images delete \"$full_image_ref\" --project=\"$PROJECT_ID\" --delete-tags --quiet" \ + "(Updated: $update_time)" + ((count++)) + fi + done <<< "$images_output" + + log "INFO" "Finished Docker Image processing for $package_name. $count images marked for deletion." +} + + +process_firewalls() { + log "INFO" "--- Processing: Firewall Rules (Limit: $DELETE_LIMIT) ---" + + local fws + fws=$(gcloud compute firewall-rules list --project="$PROJECT_ID" \ + --filter="creationTimestamp < '$CUTOFF_TIME'" \ + --format="value(name,network)" | sort) + + if [[ -z "$fws" ]]; then + log "INFO" "No Firewall Rules found matching criteria." + return + fi + + local count=0 + while read -r name network_uri; do + [[ -z "$name" ]] && continue + + # --- PROTECT DEFAULT NETWORK --- + local network_name + network_name=$(basename "$network_uri") + if [[ "$network_name" == "default" ]]; then continue; fi + + # --- LIMIT CHECK --- + if [[ $count -ge $DELETE_LIMIT ]]; then + log "INFO" "Hit delete limit ($DELETE_LIMIT) for Firewall Rules." + break + fi + + if is_excluded "$name"; then continue; fi + + execute_delete "Firewall Rule" "$name" \ + "gcloud compute firewall-rules delete \"$name\" --project=\"$PROJECT_ID\" --quiet" + + ((count++)) + + done <<< "$fws" +} + +process_filestore() { + log "INFO" "--- Processing: Filestore Instances (Limit: $DELETE_LIMIT) ---" + + # 1. Fetch JSON output for robust parsing + local fs_json + if ! fs_json=$(gcloud filestore instances list \ + --project="$PROJECT_ID" \ + --filter="createTime < '$CUTOFF_TIME'" \ + --format="json"); then # Removed 2>/dev/null to see gcloud errors + log "ERROR" "Failed to list Filestore instances." + return 1 + fi + + if [[ -z "$fs_json" || "$fs_json" == "[]" ]]; then + log "INFO" "No Filestore instances found matching criteria." + return + fi + + + # 2. Extract Location and Name using jq + local fs_list + if ! fs_list=$(echo "$fs_json" | jq -r '.[] | select(.name) | "\(.name | split("/")[3])\t\(.name | split("/")[-1])"'); then + log "ERROR" "Failed to parse Filestore JSON with jq." + return 1 + fi + + if [[ -z "$fs_list" ]]; then + log "INFO" "No instances found after jq parsing." + return + fi + + local count=0 + + # 3. Iterate + while IFS=$'\t' read -r location name; do + # Trim potential whitespace + location=$(echo "$location" | awk '{$1=$1};1') + name=$(echo "$name" | awk '{$1=$1};1') + + if [[ -z "$location" || -z "$name" ]]; then + log "DEBUG" "Skipping line with empty fields: location='${location}', name='${name}'" + continue + fi + + log "DEBUG" "Processing Instance: Name='${name}', Location='${location}'" + + # --- LIMIT CHECK --- + if [[ $count -ge $DELETE_LIMIT ]]; then + log "INFO" "Hit delete limit ($DELETE_LIMIT) for Filestore." + break + fi + + if is_excluded "$name"; then + log "INFO" "Skipping excluded Filestore: $name" + continue + fi + + # Construct delete command with explicit location + local delete_cmd="gcloud filestore instances delete \"$name\" --project=\"$PROJECT_ID\" --location=\"$location\" --quiet --force" + + execute_delete "Filestore" "$name" "$delete_cmd" "($location)" + + ((count++)) + + done <<< "$fs_list" + + log "INFO" "Finished processing Filestore instances. Attempted to delete $count." +} + +process_subnetworks() { + log "INFO" "--- Processing: Subnetworks (Limit: $DELETE_LIMIT) ---" + + local subnets + subnets=$(gcloud compute networks subnets list --project="$PROJECT_ID" --filter="creationTimestamp < '$CUTOFF_TIME'" --format="value(name,region,network,selfLink)") + + local count=0 + while IFS=$'\t' read -r name region network_uri self_link; do + [[ -z "$name" ]] && continue + + # --- PROTECT DEFAULT NETWORK --- + local network_name=$(basename "$network_uri") + if [[ "$network_name" == "default" ]]; then continue; fi + + # --- LIMIT CHECK --- + if [[ $count -ge $DELETE_LIMIT ]]; then + log "INFO" "Hit delete limit ($DELETE_LIMIT) for Subnetworks." + break + fi + + if is_excluded "$name"; then continue; fi + + local dependents=$(gcloud compute addresses list --project="$PROJECT_ID" --filter="purpose=GCE_ENDPOINT AND region=(\"$region\") AND subnetwork=(\"$self_link\")" --format="value(name)") + for addr in $dependents; do + execute_delete "Dependent Address" "$addr" "gcloud compute addresses delete \"$addr\" --project=\"$PROJECT_ID\" --region=\"$region\" --quiet" + done + + execute_delete "Subnetwork" "$name" "gcloud compute networks subnets delete \"$name\" --project=\"$PROJECT_ID\" --region=\"$region\" --quiet" + ((count++)) + done <<< "$subnets" +} + +process_networks() { + log "INFO" "--- Processing: VPC Networks (Limit: $DELETE_LIMIT) ---" + + local networks + networks=$(gcloud compute networks list --project="$PROJECT_ID" --filter="creationTimestamp < '$CUTOFF_TIME'" --format="value(name,selfLink)") + + local count=0 + while IFS=$'\t' read -r name self_link; do + [[ -z "$name" ]] && continue + + # --- PROTECT DEFAULT NETWORK --- + if [[ "$name" == "default" ]]; then continue; fi + + # --- LIMIT CHECK --- + if [[ $count -ge $DELETE_LIMIT ]]; then + log "INFO" "Hit delete limit ($DELETE_LIMIT) for Networks." + break + fi + + if is_excluded "$name"; then continue; fi + + echo "$name $self_link" + local routes=$(gcloud compute routes list --project="$PROJECT_ID" --filter="network=\"$self_link\"" --format="value(name)") + for r in $routes; do execute_delete "Dep. Route" "$r" "gcloud compute routes delete \"$r\" --project=\"$PROJECT_ID\" --quiet"; done + + local fws=$(gcloud compute firewall-rules list --project="$PROJECT_ID" --filter="network=\"$self_link\"" --format="value(name)") + for fw in $fws; do execute_delete "Dep. FW" "$fw" "gcloud compute firewall-rules delete \"$fw\" --project=\"$PROJECT_ID\" --quiet"; done + + execute_delete "Network" "$name" "gcloud compute networks delete \"$name\" --project=\"$PROJECT_ID\" --quiet" + ((count++)) + done <<< "$networks" +} + +# ============================================================================== +# MAIN EXECUTION +# ============================================================================== + +main() { + log "INFO" "STARTING RESOURCE CLEANUP: $PROJECT_ID" + log "INFO" "Time Cutoff (General): $CUTOFF_TIME" + log "INFO" "Time Cutoff (Images): $CUTOFF_TIME_IMAGES" + log "INFO" "Delete Limit per Type: $DELETE_LIMIT" + + check_dependencies + load_exclusions + + # --- Phase 1: High Level Resources --- + # process_service_accounts + + # process_resources "GKE Cluster" \ + # "gcloud container clusters list --project=\"$PROJECT_ID\" --filter=\"createTime < '$CUTOFF_TIME'\" --format=\"value(name,location)\" | sort" \ + # "gcloud container clusters delete --project=\"$PROJECT_ID\"" "location" + + process_instance_templates + + # process_resources "Compute Instance" \ + # "gcloud compute instances list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,zone)\" | sort" \ + # "gcloud compute instances delete --project=\"$PROJECT_ID\"" "zone" + + # process_filestore + # # --- Phase 2: Images & Artifacts --- + # process_vm_images + # process_docker_images + + # --- Phase 3: Network Infrastructure --- + # process_resources "Cloud Router" \ + # "gcloud compute routers list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,region)\" | sort" \ + # "gcloud compute routers delete --project=\"$PROJECT_ID\"" "region" + + # process_firewalls + + # process_addresses + # process_vpc_peerings + + # process_resources "Zonal Disk" \ + # "gcloud compute disks list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND zone:*\" --format=\"value(name,zone)\" | sort" \ + # "gcloud compute disks delete --project=\"$PROJECT_ID\"" "zone" + + # # --- Phase 4: Networking Hierarchies --- + # process_subnetworks + # process_networks + + # # --- Phase 5: IAM Cleanup --- + # process_iam_deleted_members + + log "INFO" "CLEANUP RUN FINISHED" +} + +main \ No newline at end of file diff --git a/disk.txt b/disk.txt new file mode 100644 index 0000000000..c4d91c3c2d --- /dev/null +++ b/disk.txt @@ -0,0 +1,3015 @@ +--- Thu Nov 27 12:05:43 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T08:05:43+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +lustredev0-controller us-central1-a +lustredev0-slurm-login-001 us-central1-a + gcloud compute instances delete "lustredev0-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "lustredev0-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +lustre-dev-06-net-router us-central1 +[DRY RUN] Cloud Router: Would delete lustre-dev-06-net-router in us-central1 + Command: gcloud compute routers delete "lustre-dev-06-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +lustre-dev-06-net-fw-allow-iap-ingress +lustre-dev-06-net-fw-allow-internal-traffic +[DRY RUN] Firewall Rule: Would delete lustre-dev-06-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "lustre-dev-06-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete lustre-dev-06-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "lustre-dev-06-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +lustre-dev-06-primary-subnet in us-central1 +--- Processing Subnet: lustre-dev-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-dev-06-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete lustre-dev-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-dev-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-dev-06-net +lustre-qa-05-net +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-9a732912a95e848c for network lustre-dev-06-net +[DRY RUN] Route: Would delete default-route-r-fff77e2697290c10 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete lustre-dev-06-net-fw-allow-iap-ingress for network lustre-dev-06-net +[DRY RUN] Firewall Rule: Would delete lustre-dev-06-net-fw-allow-internal-traffic for network lustre-dev-06-net +[DRY RUN] Network: Would delete lustre-dev-06-net + Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +The following Zonal Disks are targeted for deletion: +a3h23d4-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hca628-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hcb15f-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hcdf00-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsa628c1-ff9f3704-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsa628c1-ff9f3704-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsb15fa8-57b77541-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsb15fa8-57b77541-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsdf0061-6327333f-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsdf0061-6327333f-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3lavnew-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3m0280-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m1312-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +a3m14b2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m1b32-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m1bc7-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m26af-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3m2fb0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +a3m3500-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m3d4c-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m4246-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3m49e4-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3m5577-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m6293-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3m7038-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m72ec-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m816d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +a3m9051-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m9518-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3mbc98-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3h23d4-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hca628-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hcb15f-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hcdf00-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsa628c1-ff9f3704-boot-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsa628c1-ff9f3704-nfs-instance-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsb15fa8-57b77541-boot-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsb15fa8-57b77541-nfs-instance-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsdf0061-6327333f-boot-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsdf0061-6327333f-nfs-instance-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3lavnew-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m0280-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m1312-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m14b2-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m1b32-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m1bc7-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m26af-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m2fb0-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m3500-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m3d4c-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m4246-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m49e4-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m5577-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m6293-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m7038-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m72ec-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m816d-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m9051-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m9518-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3mbc98-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +--- Thu Nov 27 12:06:11 PM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 12:06:33 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T08:06:33+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +lustredev0-controller us-central1-a +lustredev0-slurm-login-001 us-central1-a +[EXECUTE] Instance: Deleting lustredev0-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustredev0-controller]. +[EXECUTE] Instance: Deleting lustredev0-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustredev0-slurm-login-001]. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +lustre-dev-06-net-router us-central1 +[EXECUTE] Cloud Router: Deleting lustre-dev-06-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/lustre-dev-06-net-router]. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +lustre-dev-06-net-fw-allow-iap-ingress +lustre-dev-06-net-fw-allow-internal-traffic +[EXECUTE] Firewall Rule: Deleting lustre-dev-06-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-dev-06-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting lustre-dev-06-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-dev-06-net-fw-allow-internal-traffic]. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +lustre-dev-06-primary-subnet in us-central1 +--- Processing Subnet: lustre-dev-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-dev-06-primary-subnet in us-central1. +[EXECUTE] Subnetwork: Deleting lustre-dev-06-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-dev-06-primary-subnet]. +Successfully deleted Subnetwork lustre-dev-06-primary-subnet in us-central1. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-dev-06-net +lustre-qa-05-net +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-9a732912a95e848c for network lustre-dev-06-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-9a732912a95e848c]. +[EXECUTE] Route: Deleting peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b99c802ac7b2e10 +[EXECUTE] Route: Deleting peering-route-7869e60dfba46542 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-7869e60dfba46542 +[EXECUTE] Route: Deleting peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-87752b9a8f2ebae2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-dev-06-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-dev-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-8b558489' + +ERROR: Failed to delete Network lustre-dev-06-net. Check for remaining dependencies. +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b91a4552351d170 +[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-6f7c1d8537c80540 +[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-qa-05-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' + +ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +The following Zonal Disks are targeted for deletion: +a3h23d4-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hca628-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hcb15f-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hcdf00-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsa628c1-ff9f3704-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsa628c1-ff9f3704-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsb15fa8-57b77541-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsb15fa8-57b77541-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsdf0061-6327333f-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3hnfsdf0061-6327333f-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3lavnew-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +a3m0280-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m1312-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +a3m14b2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m1b32-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m1bc7-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m26af-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3m2fb0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +a3m3500-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m3d4c-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m4246-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3m49e4-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3m5577-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m6293-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3m7038-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m72ec-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m816d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +a3m9051-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3m9518-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3mbc98-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +[EXECUTE] Zonal Disk: Deleting a3h23d4-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3h23d4-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3hca628-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hca628-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3hcb15f-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hcb15f-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3hcdf00-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hcdf00-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3hnfsa628c1-ff9f3704-boot-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsa628c1-ff9f3704-boot-disk]. +[EXECUTE] Zonal Disk: Deleting a3hnfsa628c1-ff9f3704-nfs-instance-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsa628c1-ff9f3704-nfs-instance-disk]. +[EXECUTE] Zonal Disk: Deleting a3hnfsb15fa8-57b77541-boot-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsb15fa8-57b77541-boot-disk]. +[EXECUTE] Zonal Disk: Deleting a3hnfsb15fa8-57b77541-nfs-instance-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsb15fa8-57b77541-nfs-instance-disk]. +[EXECUTE] Zonal Disk: Deleting a3hnfsdf0061-6327333f-boot-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsdf0061-6327333f-boot-disk]. +[EXECUTE] Zonal Disk: Deleting a3hnfsdf0061-6327333f-nfs-instance-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsdf0061-6327333f-nfs-instance-disk]. +[EXECUTE] Zonal Disk: Deleting a3lavnew-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3lavnew-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m0280-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m0280-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m1312-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/a3m1312-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m14b2-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m14b2-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m1b32-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m1b32-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m1bc7-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m1bc7-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m26af-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3m26af-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m2fb0-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/a3m2fb0-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m3500-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m3500-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m3d4c-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m3d4c-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m4246-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3m4246-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m49e4-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3m49e4-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m5577-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m5577-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m6293-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3m6293-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m7038-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m7038-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m72ec-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m72ec-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m816d-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/a3m816d-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m9051-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m9051-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3m9518-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m9518-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3mbc98-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3mbc98-controller-save]. +--- Thu Nov 27 12:10:18 PM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 12:14:16 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T08:14:16+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-dev-06-net +lustre-qa-05-net +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-dev-06-net + Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +The following Zonal Disks are targeted for deletion: +a3mc60d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +a3me1d2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3me62f-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3me777-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3med3e-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3mega-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3mfe07-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a4h333e-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4h5c04-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4h639c-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4h68f2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4h6c3b-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hc0e2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hcf79-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4he340-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hee35-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hfa418-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hrrsarth-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4newimgek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b +a4oldimgek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b +a7f7bcslur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +bfa462slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +c2dtest7-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +c379slurms-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +c52cb1fa4h-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +ce64slurms-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +d3eslurmsi-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +d3slurmsim-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +d72c8slurm-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +de3580slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3mc60d-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3me1d2-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3me62f-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3me777-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3med3e-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3mega-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a3mfe07-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4h333e-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4h5c04-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4h639c-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4h68f2-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4h6c3b-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc0e2-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hcf79-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4he340-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hee35-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hfa418-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hrrsarth-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4newimgek-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4oldimgek-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a7f7bcslur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "bfa462slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "c2dtest7-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "c379slurms-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "c52cb1fa4h-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "ce64slurms-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "d3eslurmsi-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "d3slurmsim-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "d72c8slurm-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "de3580slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +--- Thu Nov 27 12:14:42 PM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 12:15:16 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T08:15:16+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-dev-06-net +lustre-qa-05-net +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b99c802ac7b2e10 +[EXECUTE] Route: Deleting peering-route-7869e60dfba46542 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-7869e60dfba46542 +[EXECUTE] Route: Deleting peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-87752b9a8f2ebae2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-dev-06-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-dev-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-8b558489' + +ERROR: Failed to delete Network lustre-dev-06-net. Check for remaining dependencies. +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b91a4552351d170 +[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-6f7c1d8537c80540 +[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-qa-05-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' + +ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +The following Zonal Disks are targeted for deletion: +a3mc60d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +a3me1d2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3me62f-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3me777-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3med3e-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +a3mega-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a3mfe07-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +a4h333e-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4h5c04-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4h639c-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4h68f2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4h6c3b-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hc0e2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hcf79-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4he340-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hee35-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hfa418-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hrrsarth-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4newimgek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b +a4oldimgek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b +a7f7bcslur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +bfa462slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +c2dtest7-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +c379slurms-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +c52cb1fa4h-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +ce64slurms-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +d3eslurmsi-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +d3slurmsim-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +d72c8slurm-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +de3580slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[EXECUTE] Zonal Disk: Deleting a3mc60d-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/a3mc60d-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3me1d2-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3me1d2-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3me62f-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3me62f-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3me777-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3me777-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3med3e-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3med3e-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3mega-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3mega-controller-save]. +[EXECUTE] Zonal Disk: Deleting a3mfe07-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3mfe07-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4h333e-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4h333e-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4h5c04-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4h5c04-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4h639c-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4h639c-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4h68f2-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4h68f2-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4h6c3b-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4h6c3b-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4hc0e2-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hc0e2-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4hcf79-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hcf79-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4he340-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4he340-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4hee35-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hee35-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4hfa418-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hfa418-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4hrrsarth-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hrrsarth-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4newimgek-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b/disks/a4newimgek-controller-save]. +[EXECUTE] Zonal Disk: Deleting a4oldimgek-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b/disks/a4oldimgek-controller-save]. +[EXECUTE] Zonal Disk: Deleting a7f7bcslur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/a7f7bcslur-controller-save]. +[EXECUTE] Zonal Disk: Deleting bfa462slur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/bfa462slur-controller-save]. +[EXECUTE] Zonal Disk: Deleting c2dtest7-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/c2dtest7-controller-save]. +[EXECUTE] Zonal Disk: Deleting c379slurms-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/c379slurms-controller-save]. +[EXECUTE] Zonal Disk: Deleting c52cb1fa4h-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/c52cb1fa4h-controller-save]. +[EXECUTE] Zonal Disk: Deleting ce64slurms-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/ce64slurms-controller-save]. +[EXECUTE] Zonal Disk: Deleting d3eslurmsi-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/d3eslurmsi-controller-save]. +[EXECUTE] Zonal Disk: Deleting d3slurmsim-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/d3slurmsim-controller-save]. +[EXECUTE] Zonal Disk: Deleting d72c8slurm-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/d72c8slurm-controller-save]. +[EXECUTE] Zonal Disk: Deleting de3580slur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/de3580slur-controller-save]. +--- Thu Nov 27 12:17:20 PM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 12:18:27 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T08:18:27+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +a4h-slurm-net-router us-central1 +[DRY RUN] Cloud Router: Would delete a4h-slurm-net-router in us-central1 + Command: gcloud compute routers delete "a4h-slurm-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +a4h-slurm-net-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-internal-traffic +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +a4h-slurm-c1a329-primary-subnet in us-central1 +--- Processing Subnet: a4h-slurm-c1a329-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-c1a329-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-c1a329-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-c1a329-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +a4h-slurm-net +lustre-dev-06-net +lustre-qa-05-net +--- Processing Network: a4h-slurm-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-c4140f28d05b01fa for network a4h-slurm-net +[DRY RUN] Route: Would delete default-route-c9eb018ca4302458 for network a4h-slurm-net +[DRY RUN] Route: Would delete default-route-e0f1d431390409f8 for network a4h-slurm-net +[DRY RUN] Route: Would delete default-route-r-563d93643dd6c2ce for network a4h-slurm-net +[DRY RUN] Route: Would delete default-route-r-6fc73fd854e13923 for network a4h-slurm-net +[DRY RUN] Route: Would delete default-route-r-d9145f71426839a1 for network a4h-slurm-net +[DRY RUN] Route: Would delete peering-route-6bf11df1af8e59ac for network a4h-slurm-net +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete a4h-slurm-c1a329 for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic for network a4h-slurm-net +[DRY RUN] Network: Would delete a4h-slurm-net + Command: gcloud compute networks delete "a4h-slurm-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-dev-06-net + Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +The following Zonal Disks are targeted for deletion: +dynpoc-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +ebf828slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4a36-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +f4e324slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +f88073slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +fa4slurmsi-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +g4qclav-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +hpcdy-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +hpcdydis-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +hpcimg-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +hpcslurm-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +laveeek29-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b +lustre06-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[DRY RUN] Zonal Disk: gcloud compute disks delete "dynpoc-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "ebf828slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "f4e324slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "f88073slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "fa4slurmsi-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "g4qclav-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "hpcdy-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "hpcdydis-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "hpcimg-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "hpcslurm-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "laveeek29-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "lustre06-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +--- Thu Nov 27 12:18:59 PM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 01:21:16 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T09:21:16+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +a4hc1a3-a4highnodeset-0 us-central1-b +a4hc1a3-a4highnodeset-1 us-central1-b +a4hc1a3-controller us-central1-b +a4hc1a3-slurm-login-001 us-central1-b +lustreprod-controller us-central1-a +lustreprod-slurm-login-001 us-central1-a + gcloud compute instances delete "a4hc1a3-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4hc1a3-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4hc1a3-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4hc1a3-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "lustreprod-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "lustreprod-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +a4h-slurm-net-0-router us-central1 +a4h-slurm-net-1-router us-central1 +a4h-slurm-net-router us-central1 +lustre-prod-06-net-router us-central1 +[DRY RUN] Cloud Router: Would delete a4h-slurm-net-0-router in us-central1 + Command: gcloud compute routers delete "a4h-slurm-net-0-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Cloud Router: Would delete a4h-slurm-net-1-router in us-central1 + Command: gcloud compute routers delete "a4h-slurm-net-1-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Cloud Router: Would delete a4h-slurm-net-router in us-central1 + Command: gcloud compute routers delete "a4h-slurm-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Cloud Router: Would delete lustre-prod-06-net-router in us-central1 + Command: gcloud compute routers delete "lustre-prod-06-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +a4h-slurm-c1a329 +a4h-slurm-internal-0 +a4h-slurm-internal-1 +a4h-slurm-net-0-fw-allow-iap-ingress +a4h-slurm-net-1-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-internal-traffic +lustre-prod-06-net-fw-allow-iap-ingress +lustre-prod-06-net-fw-allow-internal-traffic +[DRY RUN] Firewall Rule: Would delete a4h-slurm-c1a329 + Command: gcloud compute firewall-rules delete "a4h-slurm-c1a329" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 + Command: gcloud compute firewall-rules delete "a4h-slurm-internal-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 + Command: gcloud compute firewall-rules delete "a4h-slurm-internal-1" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete lustre-prod-06-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "lustre-prod-06-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete lustre-prod-06-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "lustre-prod-06-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +a4h-slurm-c1a329-primary-subnet in us-central1 +a4h-slurm-mrdma-sub-0 in us-central1 +a4h-slurm-mrdma-sub-1 in us-central1 +a4h-slurm-mrdma-sub-2 in us-central1 +a4h-slurm-mrdma-sub-3 in us-central1 +a4h-slurm-mrdma-sub-4 in us-central1 +a4h-slurm-mrdma-sub-5 in us-central1 +a4h-slurm-mrdma-sub-6 in us-central1 +a4h-slurm-mrdma-sub-7 in us-central1 +a4h-slurm-sub-0 in us-central1 +a4h-slurm-sub-1 in us-central1 +lustre-prod-06-primary-subnet in us-central1 +--- Processing Subnet: a4h-slurm-c1a329-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-c1a329-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-c1a329-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-c1a329-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: a4h-slurm-mrdma-sub-0 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-0 in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: a4h-slurm-mrdma-sub-1 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-1 in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-1 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-1" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: a4h-slurm-mrdma-sub-2 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-2 in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-2 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-2" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: a4h-slurm-mrdma-sub-3 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-3 in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-3 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-3" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: a4h-slurm-mrdma-sub-4 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-4 in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-4 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-4" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: a4h-slurm-mrdma-sub-5 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-5 in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-5 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-5" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: a4h-slurm-mrdma-sub-6 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-6 in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-6 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-6" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: a4h-slurm-mrdma-sub-7 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-7 in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-7 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-7" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: a4h-slurm-sub-0 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-sub-0 in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: a4h-slurm-sub-1 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-sub-1 in us-central1. +[DRY RUN] Subnetwork: Would delete a4h-slurm-sub-1 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-sub-1" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: lustre-prod-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-prod-06-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete lustre-prod-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-prod-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +a4h-slurm-net-0 +a4h-slurm-net-1 +a4h-slurm-net +a4h-slurm-rdma-net +lustre-dev-06-net +lustre-prod-06-net +lustre-qa-05-net +--- Processing Network: a4h-slurm-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-c9eb018ca4302458 for network a4h-slurm-net-0 +[DRY RUN] Route: Would delete default-route-r-6fc73fd854e13923 for network a4h-slurm-net-0 +[DRY RUN] Route: Would delete peering-route-6bf11df1af8e59ac for network a4h-slurm-net-0 +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete a4h-slurm-c1a329 for network a4h-slurm-net-0 +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 for network a4h-slurm-net-0 +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress for network a4h-slurm-net-0 +[DRY RUN] Network: Would delete a4h-slurm-net-0 + Command: gcloud compute networks delete "a4h-slurm-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: a4h-slurm-net-1 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-c4140f28d05b01fa for network a4h-slurm-net-1 +[DRY RUN] Route: Would delete default-route-r-563d93643dd6c2ce for network a4h-slurm-net-1 +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 for network a4h-slurm-net-1 +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress for network a4h-slurm-net-1 +[DRY RUN] Network: Would delete a4h-slurm-net-1 + Command: gcloud compute networks delete "a4h-slurm-net-1" --project="hpc-toolkit-dev" --quiet +--- Processing Network: a4h-slurm-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-c4140f28d05b01fa for network a4h-slurm-net +[DRY RUN] Route: Would delete default-route-c9eb018ca4302458 for network a4h-slurm-net +[DRY RUN] Route: Would delete default-route-e0f1d431390409f8 for network a4h-slurm-net +[DRY RUN] Route: Would delete default-route-r-563d93643dd6c2ce for network a4h-slurm-net +[DRY RUN] Route: Would delete default-route-r-6fc73fd854e13923 for network a4h-slurm-net +[DRY RUN] Route: Would delete default-route-r-d9145f71426839a1 for network a4h-slurm-net +[DRY RUN] Route: Would delete peering-route-6bf11df1af8e59ac for network a4h-slurm-net +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete a4h-slurm-c1a329 for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress for network a4h-slurm-net +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic for network a4h-slurm-net +[DRY RUN] Network: Would delete a4h-slurm-net + Command: gcloud compute networks delete "a4h-slurm-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: a4h-slurm-rdma-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-r-19088df9372a47db for network a4h-slurm-rdma-net +[DRY RUN] Route: Would delete default-route-r-2bee294d1f7bbb6f for network a4h-slurm-rdma-net +[DRY RUN] Route: Would delete default-route-r-338b3187b72833ab for network a4h-slurm-rdma-net +[DRY RUN] Route: Would delete default-route-r-96dc060624d84425 for network a4h-slurm-rdma-net +[DRY RUN] Route: Would delete default-route-r-b7660fe09523cc85 for network a4h-slurm-rdma-net +[DRY RUN] Route: Would delete default-route-r-c9124f49ccbd9cd1 for network a4h-slurm-rdma-net +[DRY RUN] Route: Would delete default-route-r-ce12262b3077e680 for network a4h-slurm-rdma-net +[DRY RUN] Route: Would delete default-route-r-f87d27e61d42aad8 for network a4h-slurm-rdma-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete a4h-slurm-rdma-net + Command: gcloud compute networks delete "a4h-slurm-rdma-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-dev-06-net + Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-prod-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-efa6510eac5f44f0 for network lustre-prod-06-net +[DRY RUN] Route: Would delete default-route-r-c93d4441b25056d3 for network lustre-prod-06-net +[DRY RUN] Route: Would delete peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net +[DRY RUN] Route: Would delete peering-route-eebb81463c1f952f for network lustre-prod-06-net +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete lustre-prod-06-net-fw-allow-iap-ingress for network lustre-prod-06-net +[DRY RUN] Firewall Rule: Would delete lustre-prod-06-net-fw-allow-internal-traffic for network lustre-prod-06-net +[DRY RUN] Network: Would delete lustre-prod-06-net + Command: gcloud compute networks delete "lustre-prod-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +The following Zonal Disks are targeted for deletion: +a4hc1a3-a4highnodeset-0 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hc1a3-a4highnodeset-1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hc1a3-controller https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hc1a3-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +a4hc1a3-slurm-login-001 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +dynpoc-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +ebf828slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4a36-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +f4e324slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +f88073slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +fa4slurmsi-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +g4qclav-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +hpcdy-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc1a3-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc1a3-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc1a3-controller" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc1a3-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc1a3-slurm-login-001" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "dynpoc-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "ebf828slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "f4e324slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "f88073slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "fa4slurmsi-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "g4qclav-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "hpcdy-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet +--- Thu Nov 27 01:22:18 PM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 01:22:39 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T09:22:39+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +a4hc1a3-a4highnodeset-0 us-central1-b +a4hc1a3-a4highnodeset-1 us-central1-b +a4hc1a3-controller us-central1-b +a4hc1a3-slurm-login-001 us-central1-b +lustreprod-controller us-central1-a +lustreprod-slurm-login-001 us-central1-a +[EXECUTE] Instance: Deleting a4hc1a3-a4highnodeset-0 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc1a3-a4highnodeset-0]. +[EXECUTE] Instance: Deleting a4hc1a3-a4highnodeset-1 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc1a3-a4highnodeset-1]. +[EXECUTE] Instance: Deleting a4hc1a3-controller in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc1a3-controller]. +[EXECUTE] Instance: Deleting a4hc1a3-slurm-login-001 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc1a3-slurm-login-001]. +[EXECUTE] Instance: Deleting lustreprod-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustreprod-controller]. +[EXECUTE] Instance: Deleting lustreprod-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustreprod-slurm-login-001]. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +a4h-slurm-net-0-router us-central1 +a4h-slurm-net-1-router us-central1 +a4h-slurm-net-router us-central1 +lustre-prod-06-net-router us-central1 +[EXECUTE] Cloud Router: Deleting a4h-slurm-net-0-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-0-router]. +[EXECUTE] Cloud Router: Deleting a4h-slurm-net-1-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-1-router]. +[EXECUTE] Cloud Router: Deleting a4h-slurm-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-router]. +[EXECUTE] Cloud Router: Deleting lustre-prod-06-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/lustre-prod-06-net-router]. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +a4h-slurm-c1a329 +a4h-slurm-internal-0 +a4h-slurm-internal-1 +a4h-slurm-net-0-fw-allow-iap-ingress +a4h-slurm-net-1-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-internal-traffic +lustre-prod-06-net-fw-allow-iap-ingress +lustre-prod-06-net-fw-allow-internal-traffic +[EXECUTE] Firewall Rule: Deleting a4h-slurm-c1a329 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-c1a329]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-internal-0]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-internal-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-internal-1]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-1-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-1-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting lustre-prod-06-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-prod-06-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting lustre-prod-06-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-prod-06-net-fw-allow-internal-traffic]. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +a4h-slurm-c1a329-primary-subnet in us-central1 +a4h-slurm-mrdma-sub-0 in us-central1 +a4h-slurm-mrdma-sub-1 in us-central1 +a4h-slurm-mrdma-sub-2 in us-central1 +a4h-slurm-mrdma-sub-3 in us-central1 +a4h-slurm-mrdma-sub-4 in us-central1 +a4h-slurm-mrdma-sub-5 in us-central1 +a4h-slurm-mrdma-sub-6 in us-central1 +a4h-slurm-mrdma-sub-7 in us-central1 +a4h-slurm-sub-0 in us-central1 +a4h-slurm-sub-1 in us-central1 +lustre-prod-06-primary-subnet in us-central1 +--- Processing Subnet: a4h-slurm-c1a329-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-c1a329-primary-subnet in us-central1. +[EXECUTE] Subnetwork: Deleting a4h-slurm-c1a329-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-c1a329-primary-subnet]. +Successfully deleted Subnetwork a4h-slurm-c1a329-primary-subnet in us-central1. +--- Processing Subnet: a4h-slurm-mrdma-sub-0 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-0 in us-central1. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-0 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-0]. +Successfully deleted Subnetwork a4h-slurm-mrdma-sub-0 in us-central1. +--- Processing Subnet: a4h-slurm-mrdma-sub-1 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-1 in us-central1. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-1 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-1]. +Successfully deleted Subnetwork a4h-slurm-mrdma-sub-1 in us-central1. +--- Processing Subnet: a4h-slurm-mrdma-sub-2 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-2 in us-central1. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-2 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-2]. +Successfully deleted Subnetwork a4h-slurm-mrdma-sub-2 in us-central1. +--- Processing Subnet: a4h-slurm-mrdma-sub-3 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-3 in us-central1. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-3 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-3]. +Successfully deleted Subnetwork a4h-slurm-mrdma-sub-3 in us-central1. +--- Processing Subnet: a4h-slurm-mrdma-sub-4 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-4 in us-central1. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-4 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-4]. +Successfully deleted Subnetwork a4h-slurm-mrdma-sub-4 in us-central1. +--- Processing Subnet: a4h-slurm-mrdma-sub-5 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-5 in us-central1. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-5 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-5]. +Successfully deleted Subnetwork a4h-slurm-mrdma-sub-5 in us-central1. +--- Processing Subnet: a4h-slurm-mrdma-sub-6 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-6 in us-central1. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-6 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-6]. +Successfully deleted Subnetwork a4h-slurm-mrdma-sub-6 in us-central1. +--- Processing Subnet: a4h-slurm-mrdma-sub-7 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-mrdma-sub-7 in us-central1. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-7 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-7]. +Successfully deleted Subnetwork a4h-slurm-mrdma-sub-7 in us-central1. +--- Processing Subnet: a4h-slurm-sub-0 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-sub-0 in us-central1. +[EXECUTE] Subnetwork: Deleting a4h-slurm-sub-0 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-sub-0]. +Successfully deleted Subnetwork a4h-slurm-sub-0 in us-central1. +--- Processing Subnet: a4h-slurm-sub-1 in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for a4h-slurm-sub-1 in us-central1. +[EXECUTE] Subnetwork: Deleting a4h-slurm-sub-1 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-sub-1]. +Successfully deleted Subnetwork a4h-slurm-sub-1 in us-central1. +--- Processing Subnet: lustre-prod-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-prod-06-primary-subnet in us-central1. +[EXECUTE] Subnetwork: Deleting lustre-prod-06-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-prod-06-primary-subnet]. +Successfully deleted Subnetwork lustre-prod-06-primary-subnet in us-central1. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +a4h-slurm-net-0 +a4h-slurm-net-1 +a4h-slurm-net +a4h-slurm-rdma-net +lustre-dev-06-net +lustre-prod-06-net +lustre-qa-05-net +--- Processing Network: a4h-slurm-net-0 --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-c9eb018ca4302458 for network a4h-slurm-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-c9eb018ca4302458]. +[EXECUTE] Route: Deleting peering-route-6bf11df1af8e59ac for network a4h-slurm-net-0 +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-6bf11df1af8e59ac +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting a4h-slurm-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4h-slurm-net-0]. +Successfully deleted Network a4h-slurm-net-0. +--- Processing Network: a4h-slurm-net-1 --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-c4140f28d05b01fa for network a4h-slurm-net-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-c4140f28d05b01fa]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting a4h-slurm-net-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4h-slurm-net-1]. +Successfully deleted Network a4h-slurm-net-1. +--- Processing Network: a4h-slurm-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-e0f1d431390409f8 for network a4h-slurm-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-e0f1d431390409f8]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting a4h-slurm-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4h-slurm-net]. +Successfully deleted Network a4h-slurm-net. +--- Processing Network: a4h-slurm-rdma-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting a4h-slurm-rdma-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4h-slurm-rdma-net]. +Successfully deleted Network a4h-slurm-rdma-net. +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b99c802ac7b2e10 +[EXECUTE] Route: Deleting peering-route-7869e60dfba46542 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-7869e60dfba46542 +[EXECUTE] Route: Deleting peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-87752b9a8f2ebae2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-dev-06-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-dev-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-8b558489' + +ERROR: Failed to delete Network lustre-dev-06-net. Check for remaining dependencies. +--- Processing Network: lustre-prod-06-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-efa6510eac5f44f0 for network lustre-prod-06-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-efa6510eac5f44f0]. +[EXECUTE] Route: Deleting peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-0a55a53b5c4fdb82 +[EXECUTE] Route: Deleting peering-route-eebb81463c1f952f for network lustre-prod-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-eebb81463c1f952f +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-prod-06-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-prod-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-297f4f1a' + +ERROR: Failed to delete Network lustre-prod-06-net. Check for remaining dependencies. +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b91a4552351d170 +[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-6f7c1d8537c80540 +[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-qa-05-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' + +ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +The following Zonal Disks are targeted for deletion: +a4hc1a3-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +dynpoc-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +ebf828slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4691-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +exascaler-cloud-4a36-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-4a36-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +exascaler-cloud-a2a0-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +f4e324slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +f88073slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +fa4slurmsi-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +g4qclav-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +hpcdy-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +hpcdydis-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +hpcimg-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +hpcslurm-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +laveeek29-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b +[EXECUTE] Zonal Disk: Deleting a4hc1a3-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hc1a3-controller-save]. +[EXECUTE] Zonal Disk: Deleting dynpoc-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/dynpoc-controller-save]. +[EXECUTE] Zonal Disk: Deleting ebf828slur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/ebf828slur-controller-save]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-mds0-mdt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-mds0-mdt0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-mgs0-mgt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-mgs0-mgt0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-mgs0-mnt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-mgs0-mnt0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-oss0-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-oss0-ost0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-oss1-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-oss1-ost0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-oss2-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-oss2-ost0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-mds0-mdt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-mds0-mdt0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-mgs0-mgt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-mgs0-mgt0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-mgs0-mnt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-mgs0-mnt0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-oss0-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-oss0-ost0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-oss1-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-oss1-ost0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-oss2-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-oss2-ost0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-mds0-mdt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-mds0-mdt0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-mgs0-mgt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-mgs0-mgt0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-mgs0-mnt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-mgs0-mnt0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-oss0-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-oss0-ost0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-oss1-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-oss1-ost0-disk]. +[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-oss2-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-oss2-ost0-disk]. +[EXECUTE] Zonal Disk: Deleting f4e324slur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/f4e324slur-controller-save]. +[EXECUTE] Zonal Disk: Deleting f88073slur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/f88073slur-controller-save]. +[EXECUTE] Zonal Disk: Deleting fa4slurmsi-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/fa4slurmsi-controller-save]. +[EXECUTE] Zonal Disk: Deleting g4qclav-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/g4qclav-controller-save]. +[EXECUTE] Zonal Disk: Deleting hpcdy-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/hpcdy-controller-save]. +[EXECUTE] Zonal Disk: Deleting hpcdydis-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/hpcdydis-controller-save]. +[EXECUTE] Zonal Disk: Deleting hpcimg-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/hpcimg-controller-save]. +[EXECUTE] Zonal Disk: Deleting hpcslurm-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/hpcslurm-controller-save]. +[EXECUTE] Zonal Disk: Deleting laveeek29-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b/disks/laveeek29-controller-save]. +--- Thu Nov 27 01:41:20 PM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 01:42:00 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T09:42:00+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Resource: gke-gke-a3-nccl-test-system-5a732ef0-nglr (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-5a732ef0-nglr in us-west4-b (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-bbd87395-all (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-bbd87395-all (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-bbd87395-exkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-bbd87395-exkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-bbd87395-inkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-bbd87395-inkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-bbd87395-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-bbd87395-vms (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +Skip Resource: gke-a3-nccl-test-gpunet-0-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-1-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-2-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-3-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-4-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-5-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-6-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-7-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-gke-a3-nccl-test-93689389-pe-subnet (Contains protected substring: gke-a3-nccl-test) +No Subnetworks found to delete in this run after filtering. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Resource: gke-a3-nccl-test-gpunet-0 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-0 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-1 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-2 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-3 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-5 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-6 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-7 (In exclusion list) +Skip Resource: gke-a3-nccl-test-net (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-net (In exclusion list) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-dev-06-net +lustre-prod-06-net +lustre-qa-05-net +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-dev-06-net + Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-prod-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net +[DRY RUN] Route: Would delete peering-route-eebb81463c1f952f for network lustre-prod-06-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-prod-06-net + Command: gcloud compute networks delete "lustre-prod-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +The following Zonal Disks are targeted for deletion: +lustre06-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +lustredev0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +lustreprod-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +lustreqa05-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +lustretest-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +mainek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b +monitoring-8323fe-fb7b1106-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c +monitoring-8323fe-fb7b1106-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c +packer-07ae14 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-119ae1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-29c351 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-2c0c79 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b +packer-2f2917 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-3e3b45 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-3f3cd1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +packer-40dd5c https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +packer-469685 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-4dd90f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +packer-536b09 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +packer-5458a5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-578a64 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-58b950 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-59a068 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-5a390a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-5c41cc https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b +packer-6b4d6d https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-74179b https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +packer-825add https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-84a235 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b +packer-90b969 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +[DRY RUN] Zonal Disk: gcloud compute disks delete "lustre06-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "lustredev0-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "lustreprod-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "lustreqa05-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "lustretest-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "mainek-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "monitoring-8323fe-fb7b1106-boot-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "monitoring-8323fe-fb7b1106-nfs-instance-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-07ae14" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-119ae1" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-29c351" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-2c0c79" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-2f2917" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-3e3b45" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-3f3cd1" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-40dd5c" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-469685" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-4dd90f" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-536b09" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-5458a5" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-578a64" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-58b950" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-59a068" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-5a390a" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-5c41cc" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-6b4d6d" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-74179b" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-825add" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-84a235" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-90b969" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +--- Thu Nov 27 01:42:31 PM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 01:42:55 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T09:42:55+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-bbd87395-all (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-bbd87395-all (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-bbd87395-exkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-bbd87395-exkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-bbd87395-inkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-bbd87395-inkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-bbd87395-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-bbd87395-vms (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +Skip Resource: gke-a3-nccl-test-gpunet-0-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-1-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-2-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-3-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-4-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-5-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-6-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-gpunet-7-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-a3-nccl-test-subnet (Contains protected substring: gke-a3-nccl-test) +Skip Resource: gke-gke-a3-nccl-test-93689389-pe-subnet (Contains protected substring: gke-a3-nccl-test) +No Subnetworks found to delete in this run after filtering. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Resource: gke-a3-nccl-test-gpunet-0 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-0 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-1 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-2 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-3 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-5 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-6 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7 (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-gpunet-7 (In exclusion list) +Skip Resource: gke-a3-nccl-test-net (Contains protected substring: gke-a3-nccl-test) +Skip Network: gke-a3-nccl-test-net (In exclusion list) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-dev-06-net +lustre-prod-06-net +lustre-qa-05-net +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b99c802ac7b2e10 +[EXECUTE] Route: Deleting peering-route-7869e60dfba46542 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-7869e60dfba46542 +[EXECUTE] Route: Deleting peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-87752b9a8f2ebae2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-dev-06-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-dev-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-8b558489' + +ERROR: Failed to delete Network lustre-dev-06-net. Check for remaining dependencies. +--- Processing Network: lustre-prod-06-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-0a55a53b5c4fdb82 +[EXECUTE] Route: Deleting peering-route-eebb81463c1f952f for network lustre-prod-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-eebb81463c1f952f +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-prod-06-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-prod-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-297f4f1a' + +ERROR: Failed to delete Network lustre-prod-06-net. Check for remaining dependencies. +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b91a4552351d170 +[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-6f7c1d8537c80540 +[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-qa-05-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' + +ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +The following Zonal Disks are targeted for deletion: +lustre06-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +lustredev0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +lustreprod-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +lustreqa05-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +lustretest-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +mainek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b +monitoring-8323fe-fb7b1106-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c +monitoring-8323fe-fb7b1106-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c +packer-07ae14 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-119ae1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-29c351 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-2c0c79 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b +packer-2f2917 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-3e3b45 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-3f3cd1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +packer-40dd5c https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +packer-469685 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-4dd90f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +packer-536b09 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +packer-5458a5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-578a64 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-58b950 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-59a068 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-5a390a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-5c41cc https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b +packer-6b4d6d https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-74179b https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +packer-825add https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-84a235 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b +packer-90b969 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +[EXECUTE] Zonal Disk: Deleting lustre06-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/lustre06-controller-save]. +[EXECUTE] Zonal Disk: Deleting lustredev0-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/lustredev0-controller-save]. +[EXECUTE] Zonal Disk: Deleting lustreprod-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/lustreprod-controller-save]. +[EXECUTE] Zonal Disk: Deleting lustreqa05-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/lustreqa05-controller-save]. +[EXECUTE] Zonal Disk: Deleting lustretest-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/lustretest-controller-save]. +[EXECUTE] Zonal Disk: Deleting mainek-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b/disks/mainek-controller-save]. +[EXECUTE] Zonal Disk: Deleting monitoring-8323fe-fb7b1106-boot-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/disks/monitoring-8323fe-fb7b1106-boot-disk]. +[EXECUTE] Zonal Disk: Deleting monitoring-8323fe-fb7b1106-nfs-instance-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/disks/monitoring-8323fe-fb7b1106-nfs-instance-disk]. +[EXECUTE] Zonal Disk: Deleting packer-07ae14 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-07ae14]. +[EXECUTE] Zonal Disk: Deleting packer-119ae1 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-119ae1]. +[EXECUTE] Zonal Disk: Deleting packer-29c351 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-29c351]. +[EXECUTE] Zonal Disk: Deleting packer-2c0c79 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b/disks/packer-2c0c79]. +[EXECUTE] Zonal Disk: Deleting packer-2f2917 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-2f2917]. +[EXECUTE] Zonal Disk: Deleting packer-3e3b45 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-3e3b45]. +[EXECUTE] Zonal Disk: Deleting packer-3f3cd1 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/packer-3f3cd1]. +[EXECUTE] Zonal Disk: Deleting packer-40dd5c in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/packer-40dd5c]. +[EXECUTE] Zonal Disk: Deleting packer-469685 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-469685]. +[EXECUTE] Zonal Disk: Deleting packer-4dd90f in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/packer-4dd90f]. +[EXECUTE] Zonal Disk: Deleting packer-536b09 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/packer-536b09]. +[EXECUTE] Zonal Disk: Deleting packer-5458a5 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-5458a5]. +[EXECUTE] Zonal Disk: Deleting packer-578a64 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-578a64]. +[EXECUTE] Zonal Disk: Deleting packer-58b950 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-58b950]. +[EXECUTE] Zonal Disk: Deleting packer-59a068 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-59a068]. +[EXECUTE] Zonal Disk: Deleting packer-5a390a in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-5a390a]. +[EXECUTE] Zonal Disk: Deleting packer-5c41cc in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b/disks/packer-5c41cc]. +[EXECUTE] Zonal Disk: Deleting packer-6b4d6d in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-6b4d6d]. +[EXECUTE] Zonal Disk: Deleting packer-74179b in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/packer-74179b]. +[EXECUTE] Zonal Disk: Deleting packer-825add in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-825add]. +[EXECUTE] Zonal Disk: Deleting packer-84a235 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b/disks/packer-84a235]. +[EXECUTE] Zonal Disk: Deleting packer-90b969 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-90b969]. +--- Thu Nov 27 01:45:25 PM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 02:10:35 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T10:10:35+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-dev-06-net +lustre-prod-06-net +lustre-qa-05-net +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-dev-06-net + Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-prod-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net +[DRY RUN] Route: Would delete peering-route-eebb81463c1f952f for network lustre-prod-06-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-prod-06-net + Command: gcloud compute networks delete "lustre-prod-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: vertexui-do-not-kill-boot in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) +Skip Zonal Disk: vertexui-do-not-kill-data in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) +The following Zonal Disks are targeted for deletion: +packer-92ca3a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-9e2891 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-a392b5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +packer-a575bb https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-a947ca https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-aa9b5a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-b0173f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-b20310 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +packer-b70fdc https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-b9da76 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-bafb89 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-c44786 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-ce39a2 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-d02e9e https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-d0b2dd https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-d7777f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-e11a23 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b +packer-e861ff https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +packer-ea78e4 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-ec1ba5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-f7d004 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-f9879b https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-ffda40 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +pvc-c70cba5c-a091-449a-9659-b84bd4f11316 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central2-b +ractesh4d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +ractesth4d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +slurm0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-92ca3a" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-9e2891" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-a392b5" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-a575bb" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-a947ca" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-aa9b5a" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-b0173f" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-b20310" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-b70fdc" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-b9da76" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-bafb89" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-c44786" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-ce39a2" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-d02e9e" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-d0b2dd" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-d7777f" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-e11a23" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-e861ff" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-ea78e4" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-ec1ba5" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-f7d004" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-f9879b" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-ffda40" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "pvc-c70cba5c-a091-449a-9659-b84bd4f11316" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central2-b" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "ractesh4d-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "ractesth4d-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet +[DRY RUN] Zonal Disk: gcloud compute disks delete "slurm0-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet +--- Thu Nov 27 02:11:06 PM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 02:13:44 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T10:13:44+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-dev-06-net +lustre-prod-06-net +lustre-qa-05-net +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b99c802ac7b2e10 +[EXECUTE] Route: Deleting peering-route-7869e60dfba46542 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-7869e60dfba46542 +[EXECUTE] Route: Deleting peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-87752b9a8f2ebae2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-dev-06-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-dev-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-8b558489' + +ERROR: Failed to delete Network lustre-dev-06-net. Check for remaining dependencies. +--- Processing Network: lustre-prod-06-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-0a55a53b5c4fdb82 +[EXECUTE] Route: Deleting peering-route-eebb81463c1f952f for network lustre-prod-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-eebb81463c1f952f +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-prod-06-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-prod-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-297f4f1a' + +ERROR: Failed to delete Network lustre-prod-06-net. Check for remaining dependencies. +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b91a4552351d170 +[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-6f7c1d8537c80540 +[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-qa-05-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' + +ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: vertexui-do-not-kill-boot in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) +Skip Zonal Disk: vertexui-do-not-kill-data in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) +The following Zonal Disks are targeted for deletion: +packer-92ca3a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-9e2891 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-a392b5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +packer-a575bb https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-a947ca https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-aa9b5a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-b0173f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-b20310 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +packer-b70fdc https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +packer-b9da76 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-bafb89 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-c44786 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-ce39a2 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-d02e9e https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-d0b2dd https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-d7777f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-e11a23 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b +packer-e861ff https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +packer-ea78e4 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-ec1ba5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-f7d004 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-f9879b https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +packer-ffda40 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +pvc-c70cba5c-a091-449a-9659-b84bd4f11316 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central2-b +ractesh4d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +ractesth4d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +slurm0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +[EXECUTE] Zonal Disk: Deleting packer-92ca3a in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-92ca3a]. +[EXECUTE] Zonal Disk: Deleting packer-9e2891 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-9e2891]. +[EXECUTE] Zonal Disk: Deleting packer-a392b5 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/packer-a392b5]. +[EXECUTE] Zonal Disk: Deleting packer-a575bb in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-a575bb]. +[EXECUTE] Zonal Disk: Deleting packer-a947ca in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-a947ca]. +[EXECUTE] Zonal Disk: Deleting packer-aa9b5a in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-aa9b5a]. +[EXECUTE] Zonal Disk: Deleting packer-b0173f in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-b0173f]. +[EXECUTE] Zonal Disk: Deleting packer-b20310 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/packer-b20310]. +[EXECUTE] Zonal Disk: Deleting packer-b70fdc in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-b70fdc]. +[EXECUTE] Zonal Disk: Deleting packer-b9da76 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-b9da76]. +[EXECUTE] Zonal Disk: Deleting packer-bafb89 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-bafb89]. +[EXECUTE] Zonal Disk: Deleting packer-c44786 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-c44786]. +[EXECUTE] Zonal Disk: Deleting packer-ce39a2 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-ce39a2]. +[EXECUTE] Zonal Disk: Deleting packer-d02e9e in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-d02e9e]. +[EXECUTE] Zonal Disk: Deleting packer-d0b2dd in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-d0b2dd]. +[EXECUTE] Zonal Disk: Deleting packer-d7777f in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-d7777f]. +[EXECUTE] Zonal Disk: Deleting packer-e11a23 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b/disks/packer-e11a23]. +[EXECUTE] Zonal Disk: Deleting packer-e861ff in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/packer-e861ff]. +[EXECUTE] Zonal Disk: Deleting packer-ea78e4 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-ea78e4]. +[EXECUTE] Zonal Disk: Deleting packer-ec1ba5 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-ec1ba5]. +[EXECUTE] Zonal Disk: Deleting packer-f7d004 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-f7d004]. +[EXECUTE] Zonal Disk: Deleting packer-f9879b in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-f9879b]. +[EXECUTE] Zonal Disk: Deleting packer-ffda40 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/packer-ffda40]. +[EXECUTE] Zonal Disk: Deleting pvc-c70cba5c-a091-449a-9659-b84bd4f11316 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central2-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central2-b/disks/pvc-c70cba5c-a091-449a-9659-b84bd4f11316]. +[EXECUTE] Zonal Disk: Deleting ractesh4d-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/ractesh4d-controller-save]. +[EXECUTE] Zonal Disk: Deleting ractesth4d-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/ractesth4d-controller-save]. +[EXECUTE] Zonal Disk: Deleting slurm0-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/slurm0-controller-save]. +--- Deletion Phase 4b: Regional Persistent Disks (Top 30) --- +WARNING: The following filter keys were not present in any resource : region +No Regional Disks found matching criteria or list command failed. +--- Thu Nov 27 02:16:09 PM UTC 2025 --- Cleanup Script Run Finished --- + diff --git a/dockerimages.txt b/dockerimages.txt new file mode 100644 index 0000000000..e1286f2154 --- /dev/null +++ b/dockerimages.txt @@ -0,0 +1,1352 @@ +[2025-11-28 16:04:25] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:04:25] [INFO] Time Cutoff (General): 2025-11-28T15:04:25+0000 +[2025-11-28 16:04:25] [INFO] Time Cutoff (Images): 2025-09-29T16:04:25+0000 +[2025-11-28 16:04:25] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:04:25] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:04:25] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 16:04:29] [INFO] CLEANUP RUN FINISHED +[2025-11-28 16:06:10] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:06:10] [INFO] Time Cutoff (General): 2025-11-28T15:06:10+0000 +[2025-11-28 16:06:10] [INFO] Time Cutoff (Images): 2025-09-29T16:06:10+0000 +[2025-11-28 16:06:10] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:06:10] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:06:11] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 16:39:05] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:39:05] [INFO] Time Cutoff (General): 2025-11-28T15:39:05+0000 +[2025-11-28 16:39:05] [INFO] Time Cutoff (Images): 2025-09-29T16:39:05+0000 +[2025-11-28 16:39:05] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:39:05] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:39:05] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 16:39:09] [DEBUG] Found repositories: +[2025-11-28 16:39:09] [DEBUG] gcr.io + release + cleanup-repo + gcf-artifacts + h4d + hpc-toolkit-repo + slurm +[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. +[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. +[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. +[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. +[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. +[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. +[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. +[2025-11-28 16:39:09] [INFO] Finished processing Docker Images. Deleted 0 images. +[2025-11-28 16:41:41] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:41:41] [INFO] Time Cutoff (General): 2025-11-28T15:41:41+0000 +[2025-11-28 16:41:41] [INFO] Time Cutoff (Images): 2025-09-29T16:41:41+0000 +[2025-11-28 16:41:41] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:41:41] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:41:42] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 16:41:46] [DEBUG] Found repositories output: +[2025-11-28 16:41:46] [DEBUG] gcr.io + release + cleanup-repo + gcf-artifacts + h4d + hpc-toolkit-repo + slurm +[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: gcr.io +[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: release +[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: cleanup-repo +[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: gcf-artifacts +[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: h4d +[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: hpc-toolkit-repo +[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: slurm +[2025-11-28 16:41:46] [INFO] Finished processing Docker Images. Deleted 0 images. +[2025-11-28 16:44:38] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:44:38] [INFO] Time Cutoff (General): 2025-11-28T15:44:38+0000 +[2025-11-28 16:44:38] [INFO] Time Cutoff (Images): 2025-09-29T16:44:38+0000 +[2025-11-28 16:44:38] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:44:38] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:44:38] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: gcr.io. Skipping. +[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: release. Skipping. +[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: cleanup-repo. Skipping. +[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: gcf-artifacts. Skipping. +[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: h4d. Skipping. +[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: hpc-toolkit-repo. Skipping. +[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: slurm. Skipping. +[2025-11-28 16:54:35] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:54:35] [INFO] Time Cutoff (General): 2025-11-28T15:54:35+0000 +[2025-11-28 16:54:35] [INFO] Time Cutoff (Images): 2025-09-29T16:54:35+0000 +[2025-11-28 16:54:35] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:54:35] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:54:36] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 16:55:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:55:29] [INFO] Time Cutoff (General): 2025-11-28T15:55:29+0000 +[2025-11-28 16:55:29] [INFO] Time Cutoff (Images): 2025-09-29T16:55:29+0000 +[2025-11-28 16:55:29] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:55:29] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:55:30] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 16:58:03] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 16:58:03] [INFO] Time Cutoff (General): 2025-11-28T15:58:03+0000 +[2025-11-28 16:58:03] [INFO] Time Cutoff (Images): 2025-09-29T16:58:03+0000 +[2025-11-28 16:58:03] [INFO] Delete Limit per Type: 20 +[2025-11-28 16:58:03] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 16:58:03] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[2025-11-28 16:58:03] [INFO] Listing repositories... +[2025-11-28 16:58:07] [INFO] Found 8 repositories. Scanning... +[2025-11-28 16:58:07] [INFO] Scanning Repository: Listing items under project hpc-toolkit-dev-docker.pkg.dev/hpc-toolkit-dev/across all locations. +[2025-11-28 16:58:08] [WARNING] Failed to list images in Listing items under project hpc-toolkit-dev-docker.pkg.dev/hpc-toolkit-dev/across all locations. (Check permissions?) +[2025-11-28 17:00:39] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:00:39] [INFO] Time Cutoff (General): 2025-11-28T16:00:39+0000 +[2025-11-28 17:00:39] [INFO] Time Cutoff (Images): 2025-09-29T17:00:39+0000 +[2025-11-28 17:00:39] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:00:39] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:00:39] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[2025-11-28 17:00:39] [INFO] Listing repositories... +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 17:00:43] [INFO] Found 7 repositories. Scanning... +[2025-11-28 17:03:35] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:03:35] [INFO] Time Cutoff (General): 2025-11-28T16:03:35+0000 +[2025-11-28 17:03:35] [INFO] Time Cutoff (Images): 2025-09-29T17:03:35+0000 +[2025-11-28 17:03:35] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:03:35] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:03:36] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[2025-11-28 17:03:36] [INFO] Listing repositories... +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 17:06:38] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:06:38] [INFO] Time Cutoff (General): 2025-11-28T16:06:38+0000 +[2025-11-28 17:06:38] [INFO] Time Cutoff (Images): 2025-09-29T17:06:38+0000 +[2025-11-28 17:06:38] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:06:38] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:06:38] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 17:08:27] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:08:27] [INFO] Time Cutoff (General): 2025-11-28T16:08:27+0000 +[2025-11-28 17:08:27] [INFO] Time Cutoff (Images): 2025-09-29T17:08:27+0000 +[2025-11-28 17:08:27] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:08:27] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:08:27] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +gcr.io +[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: gcr.io +release +[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: release +cleanup-repo +[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: cleanup-repo +gcf-artifacts +[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: gcf-artifacts +h4d +[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: h4d +hpc-toolkit-repo +[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: hpc-toolkit-repo +slurm +[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: slurm +[2025-11-28 17:14:25] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:14:25] [INFO] Time Cutoff (General): 2025-11-28T16:14:25+0000 +[2025-11-28 17:14:25] [INFO] Time Cutoff (Images): 2025-09-29T17:14:25+0000 +[2025-11-28 17:14:25] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:14:25] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:14:26] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + +[2025-11-28 17:16:22] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:16:22] [INFO] Time Cutoff (General): 2025-11-28T16:16:22+0000 +[2025-11-28 17:16:22] [INFO] Time Cutoff (Images): 2025-09-29T17:16:22+0000 +[2025-11-28 17:16:22] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:16:22] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:16:22] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +Listing items under project hpc-toolkit-dev, across all locations. + + gcr.io + release + cleanup-repo + gcf-artifacts + h4d + hpc-toolkit-repo + slurm +./cleanup.sh: line 339: location: unbound variable +[2025-11-28 17:21:04] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:21:04] [INFO] Time Cutoff (General): 2025-11-28T16:21:04+0000 +[2025-11-28 17:21:04] [INFO] Time Cutoff (Images): 2025-09-29T17:21:04+0000 +[2025-11-28 17:21:05] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:21:05] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:21:05] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[2025-11-28 17:21:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:21:47] [INFO] Time Cutoff (General): 2025-11-28T16:21:47+0000 +[2025-11-28 17:21:47] [INFO] Time Cutoff (Images): 2025-09-29T17:21:47+0000 +[2025-11-28 17:21:47] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:21:47] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:21:47] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[ + { + "createTime": "2025-10-13T08:03:59.173812Z", + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us/repositories/gcr.io", + "sizeBytes": "39598014", + "updateTime": "2025-10-13T12:09:09.310894Z", + "vulnerabilityScanningConfig": { + "enablementState": "SCANNING_ACTIVE" + } + }, + { + "cleanupPolicyDryRun": true, + "createTime": "2023-11-01T17:59:10.830905Z", + "dockerConfig": {}, + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us/repositories/release", + "updateTime": "2023-11-01T17:59:10.830905Z", + "vulnerabilityScanningConfig": { + "enablementState": "SCANNING_ACTIVE", + "lastEnableTime": "2023-11-01T17:59:09.991552887Z" + } + }, + { + "createTime": "2025-10-22T06:35:13.473857Z", + "description": "cleanup Docker images", + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/cleanup-repo", + "satisfiesPzi": true, + "sizeBytes": "245560553", + "updateTime": "2025-10-22T06:46:30.327071Z", + "vulnerabilityScanningConfig": { + "enablementState": "SCANNING_ACTIVE", + "lastEnableTime": "2025-10-22T06:35:12.884195815Z" + } + }, + { + "createTime": "2025-10-03T10:00:14.065755Z", + "description": "This repository is created and used by Cloud Functions for storing function docker images.", + "format": "DOCKER", + "labels": { + "goog-managed-by": "cloudfunctions" + }, + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/gcf-artifacts", + "satisfiesPzi": true, + "updateTime": "2025-10-03T10:30:13.100631Z", + "vulnerabilityScanningConfig": { + "enablementState": "SCANNING_ACTIVE", + "lastEnableTime": "2025-10-03T10:00:13.385328838Z" + } + }, + { + "cleanupPolicyDryRun": true, + "createTime": "2025-04-24T06:52:17.227166Z", + "dockerConfig": {}, + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/h4d", + "satisfiesPzi": true, + "sizeBytes": "280691926", + "updateTime": "2025-11-14T10:23:27.244749Z", + "vulnerabilityScanningConfig": { + "enablementConfig": "INHERITED", + "enablementState": "SCANNING_ACTIVE", + "lastEnableTime": "2025-04-24T06:52:16.563653918Z" + } + }, + { + "cleanupPolicyDryRun": true, + "createTime": "2021-11-09T08:19:59.809355Z", + "description": "Repo for HPC Toolkit build artifacts", + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/hpc-toolkit-repo", + "satisfiesPzi": true, + "sizeBytes": "721938296284", + "updateTime": "2025-11-27T18:50:53.202608Z", + "vulnerabilityScanningConfig": { + "enablementState": "SCANNING_ACTIVE" + } + }, + { + "cleanupPolicyDryRun": true, + "createTime": "2025-09-26T04:16:29.290075Z", + "dockerConfig": {}, + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us-west4/repositories/slurm", + "satisfiesPzi": true, + "sizeBytes": "1127322284", + "updateTime": "2025-09-26T21:27:32.377895Z", + "vulnerabilityScanningConfig": { + "enablementConfig": "INHERITED", + "enablementState": "SCANNING_ACTIVE", + "lastEnableTime": "2025-09-26T04:16:21.552547084Z" + } + } +] +[2025-11-28 17:24:49] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:24:49] [INFO] Time Cutoff (General): 2025-11-28T16:24:49+0000 +[2025-11-28 17:24:49] [INFO] Time Cutoff (Images): 2025-09-29T17:24:49+0000 +[2025-11-28 17:24:49] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:24:49] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:24:49] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[2025-11-28 17:26:16] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:26:16] [INFO] Time Cutoff (General): 2025-11-28T16:26:16+0000 +[2025-11-28 17:26:16] [INFO] Time Cutoff (Images): 2025-09-29T17:26:16+0000 +[2025-11-28 17:26:16] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:26:16] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:26:17] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[ + { + "createTime": "2025-10-13T08:03:59.173812Z", + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us/repositories/gcr.io", + "sizeBytes": "39598014", + "updateTime": "2025-10-13T12:09:09.310894Z", + "vulnerabilityScanningConfig": { + "enablementState": "SCANNING_ACTIVE" + } + }, + { + "cleanupPolicyDryRun": true, + "createTime": "2023-11-01T17:59:10.830905Z", + "dockerConfig": {}, + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us/repositories/release", + "updateTime": "2023-11-01T17:59:10.830905Z", + "vulnerabilityScanningConfig": { + "enablementState": "SCANNING_ACTIVE", + "lastEnableTime": "2023-11-01T17:59:09.991552887Z" + } + }, + { + "createTime": "2025-10-22T06:35:13.473857Z", + "description": "cleanup Docker images", + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/cleanup-repo", + "satisfiesPzi": true, + "sizeBytes": "245560553", + "updateTime": "2025-10-22T06:46:30.327071Z", + "vulnerabilityScanningConfig": { + "enablementState": "SCANNING_ACTIVE", + "lastEnableTime": "2025-10-22T06:35:12.884195815Z" + } + }, + { + "createTime": "2025-10-03T10:00:14.065755Z", + "description": "This repository is created and used by Cloud Functions for storing function docker images.", + "format": "DOCKER", + "labels": { + "goog-managed-by": "cloudfunctions" + }, + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/gcf-artifacts", + "satisfiesPzi": true, + "updateTime": "2025-10-03T10:30:13.100631Z", + "vulnerabilityScanningConfig": { + "enablementState": "SCANNING_ACTIVE", + "lastEnableTime": "2025-10-03T10:00:13.385328838Z" + } + }, + { + "cleanupPolicyDryRun": true, + "createTime": "2025-04-24T06:52:17.227166Z", + "dockerConfig": {}, + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/h4d", + "satisfiesPzi": true, + "sizeBytes": "280691926", + "updateTime": "2025-11-14T10:23:27.244749Z", + "vulnerabilityScanningConfig": { + "enablementConfig": "INHERITED", + "enablementState": "SCANNING_ACTIVE", + "lastEnableTime": "2025-04-24T06:52:16.563653918Z" + } + }, + { + "cleanupPolicyDryRun": true, + "createTime": "2021-11-09T08:19:59.809355Z", + "description": "Repo for HPC Toolkit build artifacts", + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/hpc-toolkit-repo", + "satisfiesPzi": true, + "sizeBytes": "721938296284", + "updateTime": "2025-11-27T18:50:53.202608Z", + "vulnerabilityScanningConfig": { + "enablementState": "SCANNING_ACTIVE" + } + }, + { + "cleanupPolicyDryRun": true, + "createTime": "2025-09-26T04:16:29.290075Z", + "dockerConfig": {}, + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY", + "name": "projects/hpc-toolkit-dev/locations/us-west4/repositories/slurm", + "satisfiesPzi": true, + "sizeBytes": "1127322284", + "updateTime": "2025-09-26T21:27:32.377895Z", + "vulnerabilityScanningConfig": { + "enablementConfig": "INHERITED", + "enablementState": "SCANNING_ACTIVE", + "lastEnableTime": "2025-09-26T04:16:21.552547084Z" + } + } +] +null gcr.io +null release +null cleanup-repo +null gcf-artifacts +null h4d +null hpc-toolkit-repo +null slurm +Here: null gcr.io +Here: null release +Here: null cleanup-repo +Here: null gcf-artifacts +Here: null h4d +Here: null hpc-toolkit-repo +Here: null slurm +[2025-11-28 17:29:00] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:29:00] [INFO] Time Cutoff (General): 2025-11-28T16:29:00+0000 +[2025-11-28 17:29:00] [INFO] Time Cutoff (Images): 2025-09-29T17:29:00+0000 +[2025-11-28 17:29:00] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:29:00] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:29:00] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[2025-11-28 17:29:03] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io +[2025-11-28 17:29:05] [DRY-RUN] Would delete Docker Image: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go +[2025-11-28 17:29:05] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/release +[2025-11-28 17:29:07] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo +[2025-11-28 17:29:08] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo/scriptimage +[2025-11-28 17:29:08] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/gcf-artifacts +[2025-11-28 17:29:09] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d +[2025-11-28 17:29:11] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-gke-irdma-health-check +[2025-11-28 17:29:11] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-irdma-webhook-server +[2025-11-28 17:29:11] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo +[2025-11-28 17:29:17] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler +[2025-11-28 17:29:17] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim +[2025-11-28 17:29:17] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +[2025-11-28 17:29:17] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +[2025-11-28 17:29:17] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +[2025-11-28 17:29:17] [INFO] Scanning Repository: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm +[2025-11-28 17:29:19] [DRY-RUN] Would delete Docker Image: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm/slurmd-pyxis +[2025-11-28 17:32:51] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:32:51] [INFO] Time Cutoff (General): 2025-11-28T16:32:51+0000 +[2025-11-28 17:32:51] [INFO] Time Cutoff (Images): 2025-09-29T17:32:51+0000 +[2025-11-28 17:32:51] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:32:51] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:32:51] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[2025-11-28 17:32:55] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io +[2025-11-28 17:32:57] [DRY-RUN] Would delete Docker Image: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go +[2025-11-28 17:32:57] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/release +[2025-11-28 17:32:58] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo +[2025-11-28 17:33:00] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo/scriptimage +[2025-11-28 17:33:00] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/gcf-artifacts +[2025-11-28 17:33:01] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d +[2025-11-28 17:33:03] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-gke-irdma-health-check +[2025-11-28 17:33:03] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-irdma-webhook-server +[2025-11-28 17:33:03] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo +[2025-11-28 17:33:13] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler +[2025-11-28 17:33:13] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim +[2025-11-28 17:33:13] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +[2025-11-28 17:33:13] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +[2025-11-28 17:33:13] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +[2025-11-28 17:33:13] [INFO] Scanning Repository: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm +[2025-11-28 17:33:15] [DRY-RUN] Would delete Docker Image: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm/slurmd-pyxis +[2025-11-28 17:37:11] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:37:11] [INFO] Time Cutoff (General): 2025-11-28T16:37:11+0000 +[2025-11-28 17:37:11] [INFO] Time Cutoff (Images): 2025-09-29T17:37:11+0000 +[2025-11-28 17:37:11] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:37:11] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:37:11] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[2025-11-28 17:37:15] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io +[2025-11-28 17:37:16] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/release +[2025-11-28 17:37:18] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo +[2025-11-28 17:37:19] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/gcf-artifacts +[2025-11-28 17:37:21] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d +[2025-11-28 17:37:22] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo +[2025-11-28 17:37:29] [INFO] Scanning Repository: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm +[2025-11-28 17:38:40] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-28 17:38:40] [INFO] Time Cutoff (General): 2025-11-28T16:38:40+0000 +[2025-11-28 17:38:40] [INFO] Time Cutoff (Images): 2025-09-29T17:38:40+0000 +[2025-11-28 17:38:41] [INFO] Delete Limit per Type: 20 +[2025-11-28 17:38:41] [INFO] Loading exclusions from exclusions.txt... +[2025-11-28 17:38:41] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- +[2025-11-28 17:38:45] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io +us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go +us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go +us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go +us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go +us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go +[2025-11-28 17:38:46] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/release + + +[2025-11-28 17:38:48] [INFO] > No images found. +[2025-11-28 17:38:48] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo +us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo/scriptimage +us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo/scriptimage +[2025-11-28 17:38:49] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/gcf-artifacts + + +[2025-11-28 17:38:51] [INFO] > No images found. +[2025-11-28 17:38:51] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d +us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-gke-irdma-health-check +us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-irdma-webhook-server +us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-gke-irdma-health-check +us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-irdma-webhook-server +[2025-11-28 17:38:52] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http +us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +[2025-11-28 17:38:59] [INFO] Scanning Repository: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm +us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm/slurmd-pyxis +us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm/slurmd-pyxis +us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm/slurmd-pyxis diff --git a/exclusions.txt b/exclusions.txt new file mode 100644 index 0000000000..110d6b442f --- /dev/null +++ b/exclusions.txt @@ -0,0 +1,72 @@ +vertexui-do-not-kill +hpc-ctk1357 +hpc-toolkit-dev@appspot.gserviceaccount.com +build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com +cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com +cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com +cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com +508417052821-compute@developer.gserviceaccount.com +hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com +hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com +htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com +htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com +htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com +pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com +test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com +telemetry@hpc-toolkit-dev.iam.gserviceaccount.com +telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com +telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com +vertexui-do-not-kill-boot +vertexui-do-not-kill-data +default +default-router-us-west1 +default-router-us-west4 +default-net-router +default-router-australia-southeast1 +default-router-us-east4 +image-inspector-550 +image-inspector +gke-managed-lustre-basic-net-fw-allow-iap-ingress +gke-managed-lustre-basic-net-fw-allow-internal-traffic +a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +hpc-vpc +allow-internal +allow-ssh +a4high-image-builder-20250214t220935z +chs-dcgmi-metric-u22-20250925t121709z +common-slurm-image-20250725t234825z +pbspro0 +a3u-image-u22-20250325t162635z +harsh-a4-image +rocka4hf-rocky9-20250910t040750z +rocka4h-rocky9-20250908t175724z +slurm-gcp-next-hpc-rocky-linux-8-1739990978 +slurm-gcp-next-hpc-rocky-linux-8-1740100297 +a3mega-compute-a3meganodeset-20251118080924120000000004 +a3mega-compute-debugnodeset-20251118080924115100000003 +a3mega-controller-default-20251118080901356800000001 +a3mega-login-login-20251118080901434600000002 +a3slurmsy-compute-a3nodeset-20251016115123978200000002 +a3slurmsy-controller-default-20251016115129563200000003 +a3slurmsy-login-slurm-login-20251016115120300800000001 +batch-job-instance-template-20250901212237920900000001 +batch-job-instance-template-20250912070019961500000001 +buildslurm-compute-debugnodeset-20251030080636567600000001 +buildslurm-controller-default-20251030080646114800000002 +welp-insta-temp +testing2 \ No newline at end of file diff --git a/filestores.txt b/filestores.txt new file mode 100644 index 0000000000..022161d854 --- /dev/null +++ b/filestores.txt @@ -0,0 +1,179 @@ +--- Thu Nov 27 03:41:05 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Targeting resources created before: 2025-11-26T23:41:05+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +./cleanup.sh: line 77: ---: command not found +--- Deletion Phase 1: GKE Clusters (Top 20) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +The following Filestore instances are targeted for deletion in this run: +a4h-slurm-c0e262-f5260d85 +./cleanup.sh: line 182: NAME: unbound variable +--- Thu Nov 27 03:42:30 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Targeting resources created before: 2025-11-26T23:42:30+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +./cleanup.sh: line 77: ---: command not found +--- Deletion Phase 1: GKE Clusters (Top 20) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Thu Nov 27 03:42:37 AM UTC 2025 --- Cleanup Script Run Finished --- + diff --git a/firewalls.txt b/firewalls.txt new file mode 100644 index 0000000000..cbc2696d86 --- /dev/null +++ b/firewalls.txt @@ -0,0 +1,2155 @@ +--- Thu Nov 27 03:47:02 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-26T23:47:02+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-all (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-all (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-vms (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +a4h-slurm-c0e262 +a4h-slurm-internal-0 +a4h-slurm-internal-1 +a4h-slurm-net-0-fw-allow-iap-ingress +a4h-slurm-net-1-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-internal-traffic +mainek-net-fw-allow-iap-ingress +mainek-net-fw-allow-internal-traffic +managed-lustre-03-net-fw-allow-iap-ingress +[DRY RUN] Firewall Rule: Would delete a4h-slurm-c0e262 + Command: gcloud compute firewall-rules delete "a4h-slurm-c0e262" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 + Command: gcloud compute firewall-rules delete "a4h-slurm-internal-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 + Command: gcloud compute firewall-rules delete "a4h-slurm-internal-1" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete managed-lustre-03-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "managed-lustre-03-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +--- Thu Nov 27 03:47:13 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 03:49:36 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-26T23:49:36+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - managed-lustre-03-net-fw-allow-iap-ingress +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-all (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-all (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-vms (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: managed-lustre-03-net-fw-allow-iap-ingress (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +a4h-slurm-c0e262 +a4h-slurm-internal-0 +a4h-slurm-internal-1 +a4h-slurm-net-0-fw-allow-iap-ingress +a4h-slurm-net-1-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-internal-traffic +mainek-net-fw-allow-iap-ingress +mainek-net-fw-allow-internal-traffic +managed-lustre-03-net-fw-allow-internal-traffic +[DRY RUN] Firewall Rule: Would delete a4h-slurm-c0e262 + Command: gcloud compute firewall-rules delete "a4h-slurm-c0e262" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 + Command: gcloud compute firewall-rules delete "a4h-slurm-internal-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 + Command: gcloud compute firewall-rules delete "a4h-slurm-internal-1" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete managed-lustre-03-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "managed-lustre-03-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +--- Thu Nov 27 03:49:46 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 03:50:25 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-26T23:50:25+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-all (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-all (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-vms (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +a4h-slurm-c0e262 +a4h-slurm-internal-0 +a4h-slurm-internal-1 +a4h-slurm-net-0-fw-allow-iap-ingress +a4h-slurm-net-1-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-internal-traffic +mainek-net-fw-allow-iap-ingress +mainek-net-fw-allow-internal-traffic +managed-lustre-03-net-fw-allow-iap-ingress +[DRY RUN] Firewall Rule: Would delete a4h-slurm-c0e262 + Command: gcloud compute firewall-rules delete "a4h-slurm-c0e262" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 + Command: gcloud compute firewall-rules delete "a4h-slurm-internal-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 + Command: gcloud compute firewall-rules delete "a4h-slurm-internal-1" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete managed-lustre-03-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "managed-lustre-03-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +--- Thu Nov 27 03:50:37 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 03:51:13 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-26T23:51:13+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-all (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-all (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-vms (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +a4h-slurm-c0e262 +a4h-slurm-internal-0 +a4h-slurm-internal-1 +a4h-slurm-net-0-fw-allow-iap-ingress +a4h-slurm-net-1-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-iap-ingress +a4h-slurm-net-fw-allow-internal-traffic +mainek-net-fw-allow-iap-ingress +mainek-net-fw-allow-internal-traffic +managed-lustre-03-net-fw-allow-iap-ingress +[EXECUTE] Firewall Rule: Deleting a4h-slurm-c0e262 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-c0e262]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-internal-0]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-internal-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-internal-1]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-1-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-1-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting mainek-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting mainek-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting managed-lustre-03-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/managed-lustre-03-net-fw-allow-iap-ingress]. +--- Thu Nov 27 03:52:42 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 04:01:50 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T00:01:50+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-all (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-all (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-vms (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-vms (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +managed-lustre-03-net-fw-allow-internal-traffic +mglsa-net-fw-allow-iap-ingress +mglsa-net-fw-allow-internal-traffic +mglsard-net-fw-allow-iap-ingress +mglsard-net-fw-allow-internal-traffic +ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress +ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic +ml-gke-net-fw-allow-iap-ingress +ml-gke-net-fw-allow-internal-traffic +monitoring-8323fe-net-fw-allow-iap-ingress +[DRY RUN] Firewall Rule: Would delete managed-lustre-03-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "managed-lustre-03-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mglsa-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "mglsa-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mglsa-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "mglsa-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mglsard-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "mglsard-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mglsard-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "mglsard-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete ml-gke-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "ml-gke-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete ml-gke-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "ml-gke-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete monitoring-8323fe-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "monitoring-8323fe-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +--- Thu Nov 27 04:02:00 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 05:08:31 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:08:31+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +managed-lustre-03-net-fw-allow-internal-traffic +mglsa-net-fw-allow-iap-ingress +mglsa-net-fw-allow-internal-traffic +mglsard-net-fw-allow-iap-ingress +mglsard-net-fw-allow-internal-traffic +ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress +ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic +ml-gke-net-fw-allow-iap-ingress +ml-gke-net-fw-allow-internal-traffic +monitoring-8323fe-net-fw-allow-iap-ingress +[DRY RUN] Firewall Rule: Would delete managed-lustre-03-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "managed-lustre-03-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mglsa-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "mglsa-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mglsa-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "mglsa-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mglsard-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "mglsard-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete mglsard-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "mglsard-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete ml-gke-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "ml-gke-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete ml-gke-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "ml-gke-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete monitoring-8323fe-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "monitoring-8323fe-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +--- Thu Nov 27 05:08:43 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 05:09:35 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:09:35+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +managed-lustre-03-net-fw-allow-internal-traffic +mglsa-net-fw-allow-iap-ingress +mglsa-net-fw-allow-internal-traffic +mglsard-net-fw-allow-iap-ingress +mglsard-net-fw-allow-internal-traffic +ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress +ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic +ml-gke-net-fw-allow-iap-ingress +ml-gke-net-fw-allow-internal-traffic +monitoring-8323fe-net-fw-allow-iap-ingress +[EXECUTE] Firewall Rule: Deleting managed-lustre-03-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/managed-lustre-03-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting mglsa-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mglsa-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting mglsa-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mglsa-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting mglsard-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mglsard-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting mglsard-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mglsard-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting ml-gke-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/ml-gke-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting ml-gke-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/ml-gke-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting monitoring-8323fe-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/monitoring-8323fe-net-fw-allow-iap-ingress]. +--- Thu Nov 27 05:10:55 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 05:11:11 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:11:11+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +monitoring-8323fe-net-fw-allow-internal-traffic +sa-chs-ops-internal-0 +sa-chs-ops-net-0-fw-allow-iap-ingress +sispot3u-internal-0 +sispot3u-net-0-fw-allow-iap-ingress +slurm-a3-base-sysnet-fw-allow-iap-ingress +slurm-a3-base-sysnet-fw-allow-internal-traffic +sp-helmtest1-net-fw-allow-iap-ingress +sp-helmtest1-net-fw-allow-internal-traffic +static-sarthakag-net-fw-allow-iap-ingress +[DRY RUN] Firewall Rule: Would delete monitoring-8323fe-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "monitoring-8323fe-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete sa-chs-ops-internal-0 + Command: gcloud compute firewall-rules delete "sa-chs-ops-internal-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete sa-chs-ops-net-0-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "sa-chs-ops-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete sispot3u-internal-0 + Command: gcloud compute firewall-rules delete "sispot3u-internal-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete sispot3u-net-0-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "sispot3u-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete slurm-a3-base-sysnet-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "slurm-a3-base-sysnet-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete slurm-a3-base-sysnet-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "slurm-a3-base-sysnet-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete sp-helmtest1-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "sp-helmtest1-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete sp-helmtest1-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "sp-helmtest1-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete static-sarthakag-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "static-sarthakag-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +--- Thu Nov 27 05:11:21 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 05:11:39 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:11:39+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 1: GKE Clusters (Top 10) --- +WARNING: The following filter keys were not present in any resource : createTime +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) +Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +monitoring-8323fe-net-fw-allow-internal-traffic +sa-chs-ops-internal-0 +sa-chs-ops-net-0-fw-allow-iap-ingress +sispot3u-internal-0 +sispot3u-net-0-fw-allow-iap-ingress +slurm-a3-base-sysnet-fw-allow-iap-ingress +slurm-a3-base-sysnet-fw-allow-internal-traffic +sp-helmtest1-net-fw-allow-iap-ingress +sp-helmtest1-net-fw-allow-internal-traffic +static-sarthakag-net-fw-allow-iap-ingress +[EXECUTE] Firewall Rule: Deleting monitoring-8323fe-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/monitoring-8323fe-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting sa-chs-ops-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sa-chs-ops-internal-0]. +[EXECUTE] Firewall Rule: Deleting sa-chs-ops-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sa-chs-ops-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting sispot3u-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sispot3u-internal-0]. +[EXECUTE] Firewall Rule: Deleting sispot3u-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sispot3u-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting slurm-a3-base-sysnet-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/slurm-a3-base-sysnet-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting slurm-a3-base-sysnet-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/slurm-a3-base-sysnet-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting sp-helmtest1-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sp-helmtest1-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting sp-helmtest1-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sp-helmtest1-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting static-sarthakag-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/static-sarthakag-net-fw-allow-iap-ingress]. +--- Thu Nov 27 05:12:55 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 05:14:50 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:14:50+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 1: GKE Clusters (Top 10) --- +WARNING: The following filter keys were not present in any resource : createTime +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +static-sarthakag-net-fw-allow-internal-traffic +[DRY RUN] Firewall Rule: Would delete static-sarthakag-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "static-sarthakag-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +--- Thu Nov 27 05:15:00 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 05:15:13 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:15:13+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 1: GKE Clusters (Top 10) --- +WARNING: The following filter keys were not present in any resource : createTime +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +static-sarthakag-net-fw-allow-internal-traffic +[EXECUTE] Firewall Rule: Deleting static-sarthakag-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/static-sarthakag-net-fw-allow-internal-traffic]. +--- Thu Nov 27 05:15:30 AM UTC 2025 --- Cleanup Script Run Finished --- + diff --git a/iam.txt b/iam.txt new file mode 100644 index 0000000000..db3313fece --- /dev/null +++ b/iam.txt @@ -0,0 +1,4836 @@ +--- Wed Nov 26 03:34:42 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 43 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic +--- Deletion Phase 2: Service Accounts (Prefix: a3, Top 20) --- +The following Service Accounts are targeted for deletion in this run: +a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[DRY RUN] Service Account: Would delete a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Service Account: Would delete a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:34:44 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:35:42 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 43 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic +--- Deletion Phase 2: Service Accounts (Prefix: a3, Top 20) --- +The following Service Accounts are targeted for deletion in this run: +a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:35:45 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:43:06 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 50 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- +The following Service Accounts are targeted for deletion in this run: +a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:43:09 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:43:24 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 50 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- +The following Service Accounts are targeted for deletion in this run: +a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com +--- Wed Nov 26 03:43:26 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:43:44 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 50 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- +The following Service Accounts are targeted for deletion in this run: +a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 03:44:00 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:44:15 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 50 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- +Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +The following Service Accounts are targeted for deletion in this run: +a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:44:18 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:44:43 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- +Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +The following Service Accounts are targeted for deletion in this run: +a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-test-01-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "a3u-test-01-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:44:46 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:45:11 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- +Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +The following Service Accounts are targeted for deletion in this run: +a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +a3u-test-01-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting a3u-test-01-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3u-test-01-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 03:45:26 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:45:36 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- +Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +The following Service Accounts are targeted for deletion in this run: +a3u-test-01-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "a3u-test-01-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:45:38 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:45:52 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- +Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +The following Service Accounts are targeted for deletion in this run: +a3u-test-01-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting a3u-test-01-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [a3u-test-01-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 03:45:55 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:46:46 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: ag, Top 10) --- +The following Service Accounts are targeted for deletion in this run: +ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +agkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +agkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +agrkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +agrkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "agkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "agkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "agrkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "agrkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:46:48 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:47:15 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: ag, Top 10) --- +The following Service Accounts are targeted for deletion in this run: +ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +agkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +agkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +agrkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +agrkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting agkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [agkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting agkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [agkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting agrkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [agrkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting agrkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [agrkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 03:47:29 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:47:44 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: kh, Top 10) --- +The following Service Accounts are targeted for deletion in this run: +kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:47:46 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:48:03 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: kh, Top 10) --- +The following Service Accounts are targeted for deletion in this run: +kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 03:48:13 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:48:40 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: enter, Top 10) --- +The following Service Accounts are targeted for deletion in this run: +enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:48:42 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:48:53 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: enter, Top 30) --- +The following Service Accounts are targeted for deletion in this run: +enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:48:55 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:49:07 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: enter, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-c8c582-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e12bc9-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e12bc9-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e12bc9-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e34500-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e34500-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e34500-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-c8c582-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-e12bc9-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-e12bc9-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-e12bc9-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-e34500-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-e34500-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-e34500-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:49:09 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:49:20 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: enter, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-c8c582-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e12bc9-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e12bc9-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e12bc9-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e34500-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e34500-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-e34500-login@hpc-toolkit-dev.iam.gserviceaccount.com +enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com +enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com +enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-c8c582-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-c8c582-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-e12bc9-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-e12bc9-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-e12bc9-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-e12bc9-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-e12bc9-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-e12bc9-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-e34500-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-e34500-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-e34500-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-e34500-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-e34500-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-e34500-login@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 03:50:20 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:50:31 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: enter, Top 50) --- +No Service Accounts matching prefix "enter" found to delete in this run. +--- Wed Nov 26 03:50:33 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:59:00 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a2high, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-a2high-0d9dc0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-0d9dc0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-23b433-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-23b433-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-2b0192-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-3c4787-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-697aa6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-697aa6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-b8013c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-b8013c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-d0dde5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-d0dde5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-eb5c24-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-eb5c24-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-ed0848-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-ed0848-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "gke-a2high-0d9dc0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-0d9dc0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-23b433-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-23b433-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-2b0192-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-3c4787-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-697aa6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-697aa6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-b8013c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-b8013c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-d0dde5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-d0dde5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-eb5c24-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-eb5c24-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-ed0848-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a2high-ed0848-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:59:02 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:59:15 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a2high, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-a2high-0d9dc0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-0d9dc0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-23b433-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-23b433-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-2b0192-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-3c4787-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-697aa6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-697aa6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-b8013c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-b8013c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-d0dde5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-d0dde5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-eb5c24-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-eb5c24-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-ed0848-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a2high-ed0848-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting gke-a2high-0d9dc0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-0d9dc0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-0d9dc0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-0d9dc0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-23b433-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-23b433-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-23b433-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-23b433-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-2b0192-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-2b0192-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-3c4787-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-3c4787-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-697aa6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-697aa6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-697aa6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-697aa6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-b8013c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-b8013c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-b8013c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-b8013c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-d0dde5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-d0dde5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-d0dde5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-d0dde5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-eb5c24-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-eb5c24-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-eb5c24-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-eb5c24-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-ed0848-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-ed0848-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a2high-ed0848-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a2high-ed0848-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 03:59:41 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:00:18 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 51 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a3high-, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:00:20 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:00:49 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 57 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a3high-, Top 50) --- +Skip Service Account: gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +The following Service Accounts are targeted for deletion in this run: +gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:00:51 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:01:04 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 57 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a3high-, Top 50) --- +Skip Service Account: gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +The following Service Accounts are targeted for deletion in this run: +gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 04:01:25 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:01:59 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 57 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a3ultra-, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-a3ultra-38a428-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-38a428-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-3dae81-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-3dae81-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-4c44e7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-4c44e7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-6a0980-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-6a0980-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-7274d4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-7274d4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-75f21e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-75f21e-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-7c2246-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-7c2246-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-80a51f-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-80a51f-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-a5f9fe-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-a5f9fe-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-ba0d18-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-ba0d18-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-bd974d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-bd974d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-c1cc47-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-c1cc47-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-d90ea2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-d90ea2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-e06534-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-e06534-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-e1e6b2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-e1e6b2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "gke-a3ultra-38a428-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-38a428-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-3dae81-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-3dae81-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-4c44e7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-4c44e7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-6a0980-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-6a0980-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-7274d4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-7274d4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-75f21e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-75f21e-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-7c2246-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-7c2246-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-80a51f-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-80a51f-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-a5f9fe-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-a5f9fe-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-ba0d18-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-ba0d18-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-bd974d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-bd974d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-c1cc47-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-c1cc47-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-d90ea2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-d90ea2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-e06534-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-e06534-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-e1e6b2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a3ultra-e1e6b2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:02:01 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:02:20 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 57 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a3ultra-, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-a3ultra-38a428-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-38a428-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-3dae81-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-3dae81-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-4c44e7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-4c44e7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-6a0980-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-6a0980-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-7274d4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-7274d4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-75f21e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-75f21e-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-7c2246-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-7c2246-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-80a51f-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-80a51f-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-a5f9fe-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-a5f9fe-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-ba0d18-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-ba0d18-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-bd974d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-bd974d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-c1cc47-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-c1cc47-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-d90ea2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-d90ea2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-e06534-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-e06534-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-e1e6b2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a3ultra-e1e6b2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting gke-a3ultra-38a428-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-38a428-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-38a428-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-38a428-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-3dae81-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-3dae81-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-3dae81-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-3dae81-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-4c44e7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-4c44e7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-4c44e7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-4c44e7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-6a0980-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-6a0980-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-6a0980-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-6a0980-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-7274d4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-7274d4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-7274d4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-7274d4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-75f21e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-75f21e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-75f21e-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-75f21e-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-7c2246-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-7c2246-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-7c2246-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-7c2246-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-80a51f-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-80a51f-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-80a51f-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-80a51f-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-a5f9fe-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-a5f9fe-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-a5f9fe-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-a5f9fe-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-ba0d18-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-ba0d18-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-ba0d18-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-ba0d18-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-bd974d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-bd974d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-bd974d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-bd974d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-c1cc47-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-c1cc47-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-c1cc47-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-c1cc47-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-d90ea2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-d90ea2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-d90ea2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-d90ea2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-e06534-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-e06534-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-e06534-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-e06534-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-e1e6b2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-e1e6b2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a3ultra-e1e6b2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a3ultra-e1e6b2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 04:03:08 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:03:28 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 57 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a4-, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:03:30 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:03:59 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a4-, Top 50) --- +Skip Service Account: gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +The following Service Accounts are targeted for deletion in this run: +gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:04:01 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:04:14 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a4-, Top 50) --- +Skip Service Account: gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +The following Service Accounts are targeted for deletion in this run: +gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 04:04:31 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:05:01 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-dws, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-dws-fs-5ac270-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dws-fs-899431-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dws-fs-c9bf69-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dwsfs-cba5fb-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dwsfs-cba5fb-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dws-fs-fe875c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dws-fs-fe875c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dws-fs-fffe70-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "gke-dws-fs-5ac270-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-dws-fs-899431-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-dws-fs-c9bf69-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-dwsfs-cba5fb-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-dwsfs-cba5fb-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-dws-fs-fe875c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-dws-fs-fe875c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-dws-fs-fffe70-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:05:03 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:05:15 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-dws, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-dws-fs-5ac270-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dws-fs-899431-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dws-fs-c9bf69-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dwsfs-cba5fb-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dwsfs-cba5fb-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dws-fs-fe875c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dws-fs-fe875c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-dws-fs-fffe70-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting gke-dws-fs-5ac270-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-dws-fs-5ac270-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-dws-fs-899431-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-dws-fs-899431-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-dws-fs-c9bf69-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-dws-fs-c9bf69-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-dwsfs-cba5fb-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-dwsfs-cba5fb-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-dwsfs-cba5fb-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-dwsfs-cba5fb-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-dws-fs-fe875c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-dws-fs-fe875c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-dws-fs-fe875c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-dws-fs-fe875c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-dws-fs-fffe70-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-dws-fs-fffe70-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 04:05:28 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:06:06 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-ml, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-ml-136c4c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-ml-136c4c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-ml-6e40a9-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-ml-6e40a9-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-ml-7535ac-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-ml-7535ac-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "gke-ml-136c4c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-ml-136c4c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-ml-6e40a9-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-ml-6e40a9-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-ml-7535ac-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-ml-7535ac-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:06:09 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:06:20 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-ml, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-ml-136c4c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-ml-136c4c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-ml-6e40a9-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-ml-6e40a9-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-ml-7535ac-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-ml-7535ac-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting gke-ml-136c4c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-ml-136c4c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-ml-136c4c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-ml-136c4c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-ml-6e40a9-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-ml-6e40a9-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-ml-6e40a9-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-ml-6e40a9-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-ml-7535ac-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-ml-7535ac-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-ml-7535ac-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-ml-7535ac-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 04:06:31 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:08:29 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: ml-gke, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +ml-gke-65f28f-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-c60a00-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-c60a00-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-cc0c06-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-61611c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-61611c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-7df443-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-7df443-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-a8fae6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-a8fae6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-aa06c5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-aa06c5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-b3b573-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-b3b573-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-b8a492-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-b8a492-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-ceae83-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-ceae83-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-fb384b-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-fb384b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "ml-gke-65f28f-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-c60a00-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-c60a00-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-cc0c06-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-61611c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-61611c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-7df443-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-7df443-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-a8fae6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-a8fae6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-aa06c5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-aa06c5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-b3b573-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-b3b573-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-b8a492-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-b8a492-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-ceae83-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-ceae83-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-fb384b-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "ml-gke-e2e-fb384b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:08:31 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:09:15 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: ml-gke, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +ml-gke-65f28f-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-c60a00-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-c60a00-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-cc0c06-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-61611c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-61611c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-7df443-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-7df443-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-a8fae6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-a8fae6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-aa06c5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-aa06c5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-b3b573-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-b3b573-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-b8a492-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-b8a492-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-ceae83-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-ceae83-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-fb384b-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +ml-gke-e2e-fb384b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting ml-gke-65f28f-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-65f28f-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-c60a00-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-c60a00-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-c60a00-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-c60a00-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-cc0c06-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-cc0c06-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-61611c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-61611c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-61611c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-61611c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-7df443-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-7df443-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-7df443-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-7df443-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-a8fae6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-a8fae6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-a8fae6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-a8fae6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-aa06c5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-aa06c5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-aa06c5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-aa06c5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-b3b573-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-b3b573-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-b3b573-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-b3b573-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-b8a492-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-b8a492-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-b8a492-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-b8a492-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-ceae83-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-ceae83-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-ceae83-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-ceae83-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-fb384b-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-fb384b-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting ml-gke-e2e-fb384b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [ml-gke-e2e-fb384b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 04:09:47 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:10:30 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: poornima, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "poornima-gke-h4d-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "poornima-gke-h4d-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "poornima-gke-h4d-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "poornima-gke-h4d-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "poornima-gke-h4d-4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "poornima-gke-h4d-4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "poornima-gke-h4d-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "poornima-gke-h4d-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:10:32 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:10:44 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: poornima, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +poornima-gke-h4d-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting poornima-gke-h4d-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [poornima-gke-h4d-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting poornima-gke-h4d-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [poornima-gke-h4d-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting poornima-gke-h4d-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [poornima-gke-h4d-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting poornima-gke-h4d-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [poornima-gke-h4d-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting poornima-gke-h4d-4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [poornima-gke-h4d-4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting poornima-gke-h4d-4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [poornima-gke-h4d-4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting poornima-gke-h4d-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [poornima-gke-h4d-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting poornima-gke-h4d-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [poornima-gke-h4d-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 04:11:01 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:12:27 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a4x, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4x-pb1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4x-pb1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4x-pb1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-a4x-pb1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:12:29 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:12:41 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-a4x, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4x-pb1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-a4x-pb1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4x-pb1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4x-pb1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-a4x-pb1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-a4x-pb1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 04:12:49 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:13:28 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-hyperdisk, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-hyperdisk-4491a7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-hyperdisk-4491a7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-hyperdisk-450674-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-hyperdisk-450674-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "gke-hyperdisk-4491a7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-hyperdisk-4491a7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-hyperdisk-450674-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-hyperdisk-450674-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:13:30 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:13:38 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-hyperdisk, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-hyperdisk-4491a7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-hyperdisk-4491a7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-hyperdisk-450674-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-hyperdisk-450674-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting gke-hyperdisk-4491a7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-hyperdisk-4491a7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-hyperdisk-4491a7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-hyperdisk-4491a7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-hyperdisk-450674-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-hyperdisk-450674-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-hyperdisk-450674-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-hyperdisk-450674-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 04:13:46 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:15:49 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-h4d, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-h4d-f6a672-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + gcloud iam service-accounts delete "gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet + gcloud iam service-accounts delete "gke-h4d-f6a672-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 04:15:52 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:16:04 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: gke-h4d, Top 50) --- +The following Service Accounts are targeted for deletion in this run: +gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +gke-h4d-f6a672-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +[EXECUTE] Service Account: Deleting gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +[EXECUTE] Service Account: Deleting gke-h4d-f6a672-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com +deleted service account [gke-h4d-f6a672-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] +--- Wed Nov 26 04:16:10 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:17:26 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 50 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 2: Service Accounts (Prefix: a3m-, Top 50) --- +Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) +No Service Accounts matching prefix "a3m-" found to delete in this run. +--- Wed Nov 26 04:17:28 PM UTC 2025 --- Cleanup Script Run Finished --- +[2025-11-30 16:42:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 16:42:47] [INFO] Time Cutoff (General): 2025-11-30T16:42:47+0000 +[2025-11-30 16:42:47] [INFO] Time Cutoff (Images): 2025-10-01T16:42:47+0000 +[2025-11-30 16:42:47] [INFO] Delete Limit per Type: 200 +[2025-11-30 16:42:47] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 16:42:47] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 16:42:49] [INFO] No Service Accounts found matching prefix. +[2025-11-30 16:42:49] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 16:42:51] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 16:42:51] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 16:42:54] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 16:42:54] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 16:42:54] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 16:42:54] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 16:42:56] [INFO] No Filestore instances found matching criteria. +[2025-11-30 16:42:56] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 16:42:59] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 16:42:59] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 16:42:59] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 16:42:59] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 16:43:00] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 16:43:00] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 16:43:00] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 16:43:00] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 16:43:00] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 16:43:00] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 16:43:00] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 16:43:00] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T16:43:00Z (Unix: 1763311380) +[2025-11-30 16:43:00] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763311380]) +[2025-11-30 16:43:03] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 16:43:03] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 16:43:05] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 16:43:05] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 16:43:05] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 16:43:05] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 16:43:05] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 16:43:05] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 16:43:08] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 16:43:08] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 16:43:10] [INFO] No Regional Address found matching criteria. +[2025-11-30 16:43:10] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 16:43:13] [INFO] No Global Address found matching criteria. +[2025-11-30 16:43:13] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- +[2025-11-30 16:43:17] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 16:43:20] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 16:43:20] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 16:43:20] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 16:43:20] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 16:43:20] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:25] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 16:43:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:43:27] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/artifactregistry.reader +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/artifactregistry.reader +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/artifactregistry.reader +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/artifactregistry.reader +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/artifactregistry.reader +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/artifactregistry.reader +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/artifactregistry.reader +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/artifactregistry.reader +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/artifactregistry.reader +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/artifactregistry.reader +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/artifactregistry.reader +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/compute.instanceAdmin.v1 +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/compute.instanceAdmin.v1 +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/compute.instanceAdmin.v1 +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/compute.instanceAdmin.v1 +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/compute.instanceAdmin.v1 +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/compute.instanceAdmin.v1 +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/container.admin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/container.admin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/iam.serviceAccountUser +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/iam.serviceAccountUser +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/iam.serviceAccountUser +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/iam.serviceAccountUser +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/iam.serviceAccountUser +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/iam.serviceAccountUser +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/logging.logWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/monitoring.metricWriter +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/monitoring.viewer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/monitoring.viewer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/monitoring.viewer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/monitoring.viewer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/monitoring.viewer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/monitoring.viewer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/monitoring.viewer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/monitoring.viewer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/monitoring.viewer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/monitoring.viewer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/monitoring.viewer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/pubsub.admin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/pubsub.admin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/pubsub.admin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/pubsub.admin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/pubsub.admin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/pubsub.admin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/stackdriver.resourceMetadata.writer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/stackdriver.resourceMetadata.writer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/stackdriver.resourceMetadata.writer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/stackdriver.resourceMetadata.writer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/stackdriver.resourceMetadata.writer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/stackdriver.resourceMetadata.writer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/stackdriver.resourceMetadata.writer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/stackdriver.resourceMetadata.writer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/stackdriver.resourceMetadata.writer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/stackdriver.resourceMetadata.writer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/stackdriver.resourceMetadata.writer +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/storage.objectAdmin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/storage.objectAdmin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/storage.objectAdmin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/storage.objectAdmin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/storage.objectAdmin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/storage.objectAdmin +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/storage.objectCreator +[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/storage.objectCreator +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/storage.objectCreator +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/storage.objectCreator +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/storage.objectCreator +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/storage.objectCreator +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/storage.objectViewer +[2025-11-30 16:43:30] [INFO] CLEANUP RUN FINISHED +[2025-11-30 16:44:09] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 16:44:09] [INFO] Time Cutoff (General): 2025-11-30T16:44:09+0000 +[2025-11-30 16:44:09] [INFO] Time Cutoff (Images): 2025-10-01T16:44:09+0000 +[2025-11-30 16:44:09] [INFO] Delete Limit per Type: 200 +[2025-11-30 16:44:09] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 16:44:09] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 16:44:11] [INFO] No Service Accounts found matching prefix. +[2025-11-30 16:44:11] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 16:44:13] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 16:44:13] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 16:44:16] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 16:44:16] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 16:44:16] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 16:44:16] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 16:44:19] [INFO] No Filestore instances found matching criteria. +[2025-11-30 16:44:19] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 16:44:22] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 16:44:22] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 16:44:22] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 16:44:22] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 16:44:22] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 16:44:22] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 16:44:22] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 16:44:22] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 16:44:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 16:44:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 16:44:23] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 16:44:23] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T16:44:23Z (Unix: 1763311463) +[2025-11-30 16:44:23] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763311463]) +[2025-11-30 16:44:25] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 16:44:25] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 16:44:28] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 16:44:28] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 16:44:28] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 16:44:28] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 16:44:28] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 16:44:28] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 16:44:30] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 16:44:30] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 16:44:33] [INFO] No Regional Address found matching criteria. +[2025-11-30 16:44:33] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 16:44:35] [INFO] No Global Address found matching criteria. +[2025-11-30 16:44:35] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- +[2025-11-30 16:44:40] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 16:44:43] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 16:44:43] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 16:44:43] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 16:44:43] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 16:44:43] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:46] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 16:44:48] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 16:44:48] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 16:44:50] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:44:53] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:44:57] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:00] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:03] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:06] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:09] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:13] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:16] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:20] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:23] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:26] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:29] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:32] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:35] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:39] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:42] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:45] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/container.admin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:48] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/container.admin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:52] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:55] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:45:58] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:01] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:04] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:08] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:10] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:14] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:17] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:20] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:23] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:26] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:30] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:33] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:37] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:40] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:43] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:46] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:50] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:53] [EXECUTE] Removing IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:56] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:46:59] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:02] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:05] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:09] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:12] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:16] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:18] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:21] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:24] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:28] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:31] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:34] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:37] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:40] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:44] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:47] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:50] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:53] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:56] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:47:59] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:02] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:05] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:08] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:11] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:14] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:17] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:20] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:23] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:27] [EXECUTE] Removing IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:30] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:33] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:36] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:40] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:43] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:46] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:49] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:52] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:55] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:48:58] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:01] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:04] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:08] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:11] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:14] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:17] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:20] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:23] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:26] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:29] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:32] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:35] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:38] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:41] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:45] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:48] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:51] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:54] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:49:57] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:00] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:04] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:07] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:10] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:13] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:16] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:19] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:22] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:26] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:29] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:32] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:35] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:38] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:41] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:44] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:48] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:51] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:54] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:50:57] [EXECUTE] Removing IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:00] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:03] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:06] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:09] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:12] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:15] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:18] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:21] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:24] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:28] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:31] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:34] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:37] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:40] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:43] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:47] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:50] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:53] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:56] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:51:59] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:52:03] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:52:06] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:52:09] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:52:12] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:52:16] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:52:19] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. +[2025-11-30 16:52:22] [INFO] CLEANUP RUN FINISHED +[2025-11-30 17:31:37] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 17:31:37] [INFO] Time Cutoff (General): 2025-11-30T17:31:37+0000 +[2025-11-30 17:31:37] [INFO] Time Cutoff (Images): 2025-10-01T17:31:37+0000 +[2025-11-30 17:31:37] [INFO] Delete Limit per Type: 200 +[2025-11-30 17:31:37] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 17:31:37] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 17:31:40] [INFO] No Service Accounts found matching prefix. +[2025-11-30 17:31:40] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 17:31:42] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 17:31:42] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 17:31:45] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 17:31:45] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 17:31:45] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 17:31:45] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 17:31:48] [INFO] No Filestore instances found matching criteria. +[2025-11-30 17:31:48] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 17:31:51] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 17:31:51] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 17:31:51] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 17:31:51] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 17:31:51] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 17:31:51] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 17:31:51] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 17:31:51] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 17:31:51] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 17:31:51] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 17:31:52] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 17:31:52] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T17:31:52Z (Unix: 1763314312) +[2025-11-30 17:31:52] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763314312]) +[2025-11-30 17:31:54] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 17:31:54] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 17:31:57] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 17:31:57] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 17:31:57] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 17:31:57] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 17:31:57] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 17:31:57] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 17:31:59] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 17:31:59] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 17:32:01] [INFO] No Regional Address found matching criteria. +[2025-11-30 17:32:01] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 17:32:04] [INFO] No Global Address found matching criteria. +[2025-11-30 17:32:04] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- +[2025-11-30 17:32:08] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 17:32:11] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 17:32:11] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 17:32:11] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 17:32:11] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 17:32:11] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:14] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 17:32:16] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:32:16] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 17:32:18] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 17:32:18] [INFO] CLEANUP RUN FINISHED diff --git a/images.txt b/images.txt new file mode 100644 index 0000000000..d0d7ac0ba1 --- /dev/null +++ b/images.txt @@ -0,0 +1,7315 @@ +[2025-11-30 14:49:34] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 14:49:34] [INFO] Time Cutoff (General): 2025-11-30T14:49:34+0000 +[2025-11-30 14:49:34] [INFO] Time Cutoff (Images): 2025-10-01T14:49:34+0000 +[2025-11-30 14:49:34] [INFO] Delete Limit per Type: 20 +[2025-11-30 14:49:34] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 14:49:34] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 14:49:37] [INFO] No Service Accounts found matching prefix. +[2025-11-30 14:49:37] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 14:49:39] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 14:49:39] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 14:49:41] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 14:49:41] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 14:49:41] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 14:49:41] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 14:49:44] [INFO] No Filestore instances found matching criteria. +[2025-11-30 14:49:44] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 14:49:47] [DRY-RUN] Would delete VM Image: a3qc-u22-20251001t070936z +[2025-11-30 14:49:47] [DRY-RUN] Would delete VM Image: a3qch-u22-20251001t093852z +[2025-11-30 14:49:47] [DRY-RUN] Would delete VM Image: a3qclavek-u22-20251001t025923z +[2025-11-30 14:49:47] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 14:49:48] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 14:49:48] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 14:49:48] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 14:49:48] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 14:49:48] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 14:49:48] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 14:49:48] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 14:49:48] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 14:49:48] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 14:49:48] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 14:49:48] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T14:49:48Z (Unix: 1763304588) +[2025-11-30 14:49:48] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 (Updated: 2024-03-27T23:09:45 [TS: 1711580985] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 (Updated: 2024-03-27T23:09:45) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 (Updated: 2024-03-29T01:38:38 [TS: 1711676318] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 (Updated: 2024-03-29T01:38:38) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d (Updated: 2024-03-29T07:18:04 [TS: 1711696684] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d (Updated: 2024-03-29T07:18:04) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 (Updated: 2024-03-30T07:18:27 [TS: 1711783107] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 (Updated: 2024-03-30T07:18:27) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 (Updated: 2024-03-31T07:18:48 [TS: 1711869528] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 (Updated: 2024-03-31T07:18:48) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 (Updated: 2024-04-01T07:18:16 [TS: 1711955896] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 (Updated: 2024-04-01T07:18:16) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 (Updated: 2024-04-02T07:18:23 [TS: 1712042303] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 (Updated: 2024-04-02T07:18:23) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c (Updated: 2024-04-03T07:19:36 [TS: 1712128776] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c (Updated: 2024-04-03T07:19:36) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb (Updated: 2024-04-04T00:38:49 [TS: 1712191129] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb (Updated: 2024-04-04T00:38:49) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 (Updated: 2024-04-04T07:18:02 [TS: 1712215082] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 (Updated: 2024-04-04T07:18:02) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 (Updated: 2024-04-05T07:18:24 [TS: 1712301504] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 (Updated: 2024-04-05T07:18:24) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba (Updated: 2024-04-06T07:18:49 [TS: 1712387929] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba (Updated: 2024-04-06T07:18:49) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 (Updated: 2024-04-07T07:19:00 [TS: 1712474340] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 (Updated: 2024-04-07T07:19:00) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 (Updated: 2024-04-08T07:18:15 [TS: 1712560695] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 (Updated: 2024-04-08T07:18:15) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 (Updated: 2024-04-09T07:18:35 [TS: 1712647115] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 (Updated: 2024-04-09T07:18:35) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb (Updated: 2024-04-10T07:18:20 [TS: 1712733500] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb (Updated: 2024-04-10T07:18:20) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d (Updated: 2024-04-11T07:18:36 [TS: 1712819916] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d (Updated: 2024-04-11T07:18:36) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 (Updated: 2024-04-12T07:18:04 [TS: 1712906284] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 (Updated: 2024-04-12T07:18:04) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d (Updated: 2024-04-13T07:18:53 [TS: 1712992733] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d (Updated: 2024-04-13T07:18:53) +[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae (Updated: 2024-04-14T07:18:44 [TS: 1713079124] < Cutoff: [TS: 1763304588]) +[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae (Updated: 2024-04-14T07:18:44) +[2025-11-30 14:49:54] [INFO] Hit delete limit (20) for Docker Images. +[2025-11-30 14:49:54] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 14:49:55] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 14:49:57] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 14:49:57] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 14:49:57] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 14:49:57] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 14:49:57] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 14:49:57] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 14:49:59] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 14:49:59] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 14:50:02] [INFO] No Regional Address found matching criteria. +[2025-11-30 14:50:02] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 14:50:04] [INFO] No Global Address found matching criteria. +[2025-11-30 14:50:04] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-30 14:50:09] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 14:50:11] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 14:50:11] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 14:50:11] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 14:50:11] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 14:50:11] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:14] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 14:50:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:50:17] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 14:50:18] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 14:50:18] [INFO] CLEANUP RUN FINISHED +[2025-11-30 14:50:31] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 14:50:31] [INFO] Time Cutoff (General): 2025-11-30T14:50:31+0000 +[2025-11-30 14:50:31] [INFO] Time Cutoff (Images): 2025-10-01T14:50:31+0000 +[2025-11-30 14:50:31] [INFO] Delete Limit per Type: 20 +[2025-11-30 14:50:31] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 14:50:32] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 14:50:34] [INFO] No Service Accounts found matching prefix. +[2025-11-30 14:50:34] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 14:50:36] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 14:50:36] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 14:50:39] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 14:50:39] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 14:50:39] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 14:50:39] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 14:50:41] [INFO] No Filestore instances found matching criteria. +[2025-11-30 14:50:41] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 14:50:44] [EXECUTE] Deleting VM Image: a3qc-u22-20251001t070936z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3qc-u22-20251001t070936z]. +[2025-11-30 14:50:53] [SUCCESS] Deleted a3qc-u22-20251001t070936z +[2025-11-30 14:50:53] [EXECUTE] Deleting VM Image: a3qch-u22-20251001t093852z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3qch-u22-20251001t093852z]. +[2025-11-30 14:51:00] [SUCCESS] Deleted a3qch-u22-20251001t093852z +[2025-11-30 14:51:00] [EXECUTE] Deleting VM Image: a3qclavek-u22-20251001t025923z +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3qclavek-u22-20251001t025923z]. +[2025-11-30 14:51:09] [SUCCESS] Deleted a3qclavek-u22-20251001t025923z +[2025-11-30 14:51:09] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 14:51:10] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 14:51:10] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 14:51:10] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 14:51:10] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 14:51:10] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 14:51:10] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 14:51:10] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 14:51:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 14:51:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 14:51:11] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 14:51:11] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T14:51:11Z (Unix: 1763304671) +[2025-11-30 14:51:11] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 14:51:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 (Updated: 2024-03-27T23:09:45 [TS: 1711580985] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 (Updated: 2024-03-27T23:09:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/280fef8f-40e8-4723-b732-f1bc3ba2e3cc] to complete... +.....done. +[2025-11-30 14:51:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 +[2025-11-30 14:51:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 (Updated: 2024-03-29T01:38:38 [TS: 1711676318] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 (Updated: 2024-03-29T01:38:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4c40cc8e-d225-4ae7-aff7-a8f08dd0755e] to complete... +.....done. +[2025-11-30 14:51:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 +[2025-11-30 14:51:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d (Updated: 2024-03-29T07:18:04 [TS: 1711696684] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d (Updated: 2024-03-29T07:18:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1b2fe78c-9f92-40c4-a748-ff5ac25d719d] to complete... +.....done. +[2025-11-30 14:51:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d +[2025-11-30 14:51:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 (Updated: 2024-03-30T07:18:27 [TS: 1711783107] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 (Updated: 2024-03-30T07:18:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/03c29cd7-c320-4a30-80a8-9c99fe572afe] to complete... +.....done. +[2025-11-30 14:51:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 +[2025-11-30 14:51:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 (Updated: 2024-03-31T07:18:48 [TS: 1711869528] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 (Updated: 2024-03-31T07:18:48) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8d5277a8-a066-460d-b549-ad4684e28755] to complete... +.....done. +[2025-11-30 14:51:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 +[2025-11-30 14:51:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 (Updated: 2024-04-01T07:18:16 [TS: 1711955896] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 (Updated: 2024-04-01T07:18:16) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d7d21773-b0af-411a-b105-505261372cf9] to complete... +.....done. +[2025-11-30 14:51:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 +[2025-11-30 14:51:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 (Updated: 2024-04-02T07:18:23 [TS: 1712042303] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 (Updated: 2024-04-02T07:18:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/79833c81-b677-4423-ae90-3a766aad2a9d] to complete... +.....done. +[2025-11-30 14:51:41] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 +[2025-11-30 14:51:41] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c (Updated: 2024-04-03T07:19:36 [TS: 1712128776] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:41] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c (Updated: 2024-04-03T07:19:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ca10467d-ccb0-4def-abff-5304e4326b54] to complete... +.....done. +[2025-11-30 14:51:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c +[2025-11-30 14:51:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb (Updated: 2024-04-04T00:38:49 [TS: 1712191129] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb (Updated: 2024-04-04T00:38:49) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/36123b73-ce5b-4237-943d-8d34e5ebcf85] to complete... +.....done. +[2025-11-30 14:51:48] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb +[2025-11-30 14:51:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 (Updated: 2024-04-04T07:18:02 [TS: 1712215082] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 (Updated: 2024-04-04T07:18:02) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c7e2f6f2-9966-4630-833d-2c67cfa3092b] to complete... +.....done. +[2025-11-30 14:51:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 +[2025-11-30 14:51:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 (Updated: 2024-04-05T07:18:24 [TS: 1712301504] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 (Updated: 2024-04-05T07:18:24) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/95e449b6-bdff-4186-8803-a226776d77e1] to complete... +......done. +[2025-11-30 14:51:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 +[2025-11-30 14:51:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba (Updated: 2024-04-06T07:18:49 [TS: 1712387929] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba (Updated: 2024-04-06T07:18:49) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1e2e3b9f-c8d2-43f8-a2d9-b598ef6c1758] to complete... +.....done. +[2025-11-30 14:51:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba +[2025-11-30 14:51:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 (Updated: 2024-04-07T07:19:00 [TS: 1712474340] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:51:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 (Updated: 2024-04-07T07:19:00) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d300e354-55a2-4133-b8d9-e60a88c2ea4d] to complete... +......done. +[2025-11-30 14:52:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 +[2025-11-30 14:52:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 (Updated: 2024-04-08T07:18:15 [TS: 1712560695] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:52:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 (Updated: 2024-04-08T07:18:15) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/35066762-f7bd-4245-be58-c0d2e050e758] to complete... +......done. +[2025-11-30 14:52:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 +[2025-11-30 14:52:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 (Updated: 2024-04-09T07:18:35 [TS: 1712647115] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:52:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 (Updated: 2024-04-09T07:18:35) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c37cbc0a-5992-4a07-98c9-9436ee23b8bb] to complete... +.....done. +[2025-11-30 14:52:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 +[2025-11-30 14:52:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb (Updated: 2024-04-10T07:18:20 [TS: 1712733500] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:52:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb (Updated: 2024-04-10T07:18:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a5ce4155-a5d5-4544-8205-dde4a26392b8] to complete... +.....done. +[2025-11-30 14:52:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb +[2025-11-30 14:52:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d (Updated: 2024-04-11T07:18:36 [TS: 1712819916] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:52:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d (Updated: 2024-04-11T07:18:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5cb25c3a-8273-42d7-9622-14f2868c1f81] to complete... +.....done. +[2025-11-30 14:52:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d +[2025-11-30 14:52:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 (Updated: 2024-04-12T07:18:04 [TS: 1712906284] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:52:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 (Updated: 2024-04-12T07:18:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f8f63255-a4e6-458b-bc25-f6e870909f77] to complete... +.....done. +[2025-11-30 14:52:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 +[2025-11-30 14:52:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d (Updated: 2024-04-13T07:18:53 [TS: 1712992733] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:52:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d (Updated: 2024-04-13T07:18:53) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1f21e30e-b6dc-4dc2-ba8f-caf8aa4ff7af] to complete... +.....done. +[2025-11-30 14:52:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d +[2025-11-30 14:52:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae (Updated: 2024-04-14T07:18:44 [TS: 1713079124] < Cutoff: [TS: 1763304671]) +[2025-11-30 14:52:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae (Updated: 2024-04-14T07:18:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a8f32c5f-d036-4f58-ac01-b2489872c85e] to complete... +.....done. +[2025-11-30 14:52:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae +[2025-11-30 14:52:27] [INFO] Hit delete limit (20) for Docker Images. +[2025-11-30 14:52:27] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 14:52:27] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 14:52:29] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 14:52:29] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 14:52:29] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 14:52:29] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 14:52:29] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 14:52:29] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 14:52:31] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 14:52:32] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 14:52:34] [INFO] No Regional Address found matching criteria. +[2025-11-30 14:52:34] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 14:52:36] [INFO] No Global Address found matching criteria. +[2025-11-30 14:52:36] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-30 14:52:41] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 14:52:44] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 14:52:44] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 14:52:44] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 14:52:44] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 14:52:44] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:47] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 14:52:50] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:52:50] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 14:52:52] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 14:52:52] [INFO] CLEANUP RUN FINISHED +[2025-11-30 14:53:18] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 14:53:18] [INFO] Time Cutoff (General): 2025-11-30T14:53:18+0000 +[2025-11-30 14:53:18] [INFO] Time Cutoff (Images): 2025-10-01T14:53:18+0000 +[2025-11-30 14:53:18] [INFO] Delete Limit per Type: 20 +[2025-11-30 14:53:18] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 14:53:19] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 14:53:21] [INFO] No Service Accounts found matching prefix. +[2025-11-30 14:53:21] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 14:53:23] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 14:53:23] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 14:53:25] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 14:53:25] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 14:53:25] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 14:53:25] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 14:53:28] [INFO] No Filestore instances found matching criteria. +[2025-11-30 14:53:28] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 14:53:31] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 14:53:31] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 14:53:31] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 14:53:31] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 14:53:31] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 14:53:31] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 14:53:31] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 14:53:31] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 14:53:32] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 14:53:32] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 14:53:32] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 14:53:32] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T14:53:32Z (Unix: 1763304812) +[2025-11-30 14:53:32] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19 [TS: 1713165559] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17 [TS: 1713251897] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57 [TS: 1713338277] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08 [TS: 1713424748] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26 [TS: 1713511166] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36 [TS: 1713597516] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57 [TS: 1713683937] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38 [TS: 1713770258] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51 [TS: 1713856671] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09 [TS: 1713943149] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19 [TS: 1714029499] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26 [TS: 1714115966] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09 [TS: 1714202349] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58 [TS: 1714288738] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09 [TS: 1714375269] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18 [TS: 1714461498] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48 [TS: 1714547868] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44 [TS: 1714634264] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43 [TS: 1714720723] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43) +[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00 [TS: 1714807080] < Cutoff: [TS: 1763304812]) +[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00) +[2025-11-30 14:53:38] [INFO] Hit delete limit (20) for Docker Images. +[2025-11-30 14:53:38] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 14:53:38] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 14:53:41] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 14:53:41] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 14:53:41] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 14:53:41] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 14:53:41] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 14:53:41] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 14:53:43] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 14:53:44] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 14:53:46] [INFO] No Regional Address found matching criteria. +[2025-11-30 14:53:46] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 14:53:48] [INFO] No Global Address found matching criteria. +[2025-11-30 14:53:48] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- +[2025-11-30 14:53:53] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 14:53:55] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 14:53:55] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 14:53:55] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 14:53:55] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 14:53:55] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:53:58] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 14:54:01] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:01] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 14:54:02] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 14:54:02] [INFO] CLEANUP RUN FINISHED +[2025-11-30 14:54:08] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 14:54:08] [INFO] Time Cutoff (General): 2025-11-30T14:54:08+0000 +[2025-11-30 14:54:08] [INFO] Time Cutoff (Images): 2025-10-01T14:54:08+0000 +[2025-11-30 14:54:08] [INFO] Delete Limit per Type: 200 +[2025-11-30 14:54:08] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 14:54:09] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 14:54:11] [INFO] No Service Accounts found matching prefix. +[2025-11-30 14:54:11] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 14:54:13] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 14:54:13] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 14:54:15] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 14:54:15] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 14:54:16] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 14:54:16] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 14:54:18] [INFO] No Filestore instances found matching criteria. +[2025-11-30 14:54:18] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 14:54:21] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 14:54:21] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 14:54:21] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 14:54:21] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 14:54:22] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 14:54:22] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 14:54:22] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 14:54:22] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 14:54:22] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 14:54:22] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 14:54:22] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 14:54:22] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T14:54:22Z (Unix: 1763304862) +[2025-11-30 14:54:22] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19 [TS: 1713165559] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17 [TS: 1713251897] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57 [TS: 1713338277] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08 [TS: 1713424748] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26 [TS: 1713511166] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36 [TS: 1713597516] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57 [TS: 1713683937] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38 [TS: 1713770258] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51 [TS: 1713856671] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09 [TS: 1713943149] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19 [TS: 1714029499] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26 [TS: 1714115966] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09 [TS: 1714202349] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58 [TS: 1714288738] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09 [TS: 1714375269] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18 [TS: 1714461498] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48 [TS: 1714547868] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44 [TS: 1714634264] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43 [TS: 1714720723] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00 [TS: 1714807080] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 (Updated: 2024-05-05T07:19:28 [TS: 1714893568] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 (Updated: 2024-05-05T07:19:28) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 (Updated: 2024-05-06T07:18:08 [TS: 1714979888] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 (Updated: 2024-05-06T07:18:08) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 (Updated: 2024-05-07T07:20:44 [TS: 1715066444] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 (Updated: 2024-05-07T07:20:44) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 (Updated: 2024-05-08T07:18:32 [TS: 1715152712] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 (Updated: 2024-05-08T07:18:32) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b (Updated: 2024-05-09T07:19:05 [TS: 1715239145] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b (Updated: 2024-05-09T07:19:05) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd (Updated: 2024-05-10T07:18:32 [TS: 1715325512] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd (Updated: 2024-05-10T07:18:32) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 (Updated: 2024-05-11T07:18:53 [TS: 1715411933] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 (Updated: 2024-05-11T07:18:53) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 (Updated: 2024-05-12T07:18:59 [TS: 1715498339] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 (Updated: 2024-05-12T07:18:59) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 (Updated: 2024-05-13T07:18:49 [TS: 1715584729] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 (Updated: 2024-05-13T07:18:49) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc (Updated: 2024-05-14T07:16:18 [TS: 1715670978] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc (Updated: 2024-05-14T07:16:18) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 (Updated: 2024-05-15T07:18:59 [TS: 1715757539] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 (Updated: 2024-05-15T07:18:59) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 (Updated: 2024-05-16T07:18:34 [TS: 1715843914] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 (Updated: 2024-05-16T07:18:34) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 (Updated: 2024-05-17T07:18:04 [TS: 1715930284] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 (Updated: 2024-05-17T07:18:04) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 (Updated: 2024-05-18T07:18:46 [TS: 1716016726] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 (Updated: 2024-05-18T07:18:46) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd (Updated: 2024-05-19T07:19:20 [TS: 1716103160] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd (Updated: 2024-05-19T07:19:20) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c (Updated: 2024-05-20T07:18:54 [TS: 1716189534] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c (Updated: 2024-05-20T07:18:54) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce (Updated: 2024-05-21T07:18:14 [TS: 1716275894] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce (Updated: 2024-05-21T07:18:14) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e (Updated: 2024-05-22T07:17:56 [TS: 1716362276] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e (Updated: 2024-05-22T07:17:56) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 (Updated: 2024-05-23T07:17:57 [TS: 1716448677] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 (Updated: 2024-05-23T07:17:57) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 (Updated: 2024-05-24T07:17:47 [TS: 1716535067] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 (Updated: 2024-05-24T07:17:47) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 (Updated: 2024-05-25T07:18:10 [TS: 1716621490] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 (Updated: 2024-05-25T07:18:10) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 (Updated: 2024-05-26T07:18:34 [TS: 1716707914] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 (Updated: 2024-05-26T07:18:34) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 (Updated: 2024-05-27T07:18:35 [TS: 1716794315] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 (Updated: 2024-05-27T07:18:35) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf (Updated: 2024-05-28T07:18:07 [TS: 1716880687] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf (Updated: 2024-05-28T07:18:07) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 (Updated: 2024-05-29T07:17:33 [TS: 1716967053] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 (Updated: 2024-05-29T07:17:33) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d (Updated: 2024-05-30T07:18:38 [TS: 1717053518] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d (Updated: 2024-05-30T07:18:38) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 (Updated: 2024-05-31T07:18:40 [TS: 1717139920] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 (Updated: 2024-05-31T07:18:40) +[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de (Updated: 2024-06-01T07:18:20 [TS: 1717226300] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de (Updated: 2024-06-01T07:18:20) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb (Updated: 2024-06-02T07:19:05 [TS: 1717312745] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb (Updated: 2024-06-02T07:19:05) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 (Updated: 2024-06-03T07:18:42 [TS: 1717399122] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 (Updated: 2024-06-03T07:18:42) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 (Updated: 2024-06-04T07:18:28 [TS: 1717485508] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 (Updated: 2024-06-04T07:18:28) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 (Updated: 2024-06-05T07:18:24 [TS: 1717571904] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 (Updated: 2024-06-05T07:18:24) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 (Updated: 2024-06-06T07:20:38 [TS: 1717658438] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 (Updated: 2024-06-06T07:20:38) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d (Updated: 2024-06-07T07:18:27 [TS: 1717744707] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d (Updated: 2024-06-07T07:18:27) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 (Updated: 2024-06-08T07:18:28 [TS: 1717831108] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 (Updated: 2024-06-08T07:18:28) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e (Updated: 2024-06-09T07:18:23 [TS: 1717917503] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e (Updated: 2024-06-09T07:18:23) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 (Updated: 2024-06-10T07:18:11 [TS: 1718003891] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 (Updated: 2024-06-10T07:18:11) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 (Updated: 2024-06-11T07:18:43 [TS: 1718090323] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 (Updated: 2024-06-11T07:18:43) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 (Updated: 2024-06-12T07:18:47 [TS: 1718176727] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 (Updated: 2024-06-12T07:18:47) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a (Updated: 2024-06-13T07:19:10 [TS: 1718263150] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a (Updated: 2024-06-13T07:19:10) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c (Updated: 2024-06-14T07:18:33 [TS: 1718349513] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c (Updated: 2024-06-14T07:18:33) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 (Updated: 2024-06-15T07:19:01 [TS: 1718435941] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 (Updated: 2024-06-15T07:19:01) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 (Updated: 2024-06-16T07:18:57 [TS: 1718522337] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 (Updated: 2024-06-16T07:18:57) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 (Updated: 2024-06-17T07:17:57 [TS: 1718608677] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 (Updated: 2024-06-17T07:17:57) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 (Updated: 2024-06-18T07:18:50 [TS: 1718695130] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 (Updated: 2024-06-18T07:18:50) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 (Updated: 2024-06-19T07:18:24 [TS: 1718781504] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 (Updated: 2024-06-19T07:18:24) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c (Updated: 2024-06-20T07:18:11 [TS: 1718867891] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c (Updated: 2024-06-20T07:18:11) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 (Updated: 2024-06-20T17:43:06 [TS: 1718905386] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 (Updated: 2024-06-20T17:43:06) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a (Updated: 2024-06-21T07:20:29 [TS: 1718954429] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a (Updated: 2024-06-21T07:20:29) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 (Updated: 2024-06-21T18:43:44 [TS: 1718995424] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 (Updated: 2024-06-21T18:43:44) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 (Updated: 2024-06-21T20:07:06 [TS: 1719000426] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 (Updated: 2024-06-21T20:07:06) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db (Updated: 2024-06-21T22:45:30 [TS: 1719009930] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db (Updated: 2024-06-21T22:45:30) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 (Updated: 2024-06-21T23:36:46 [TS: 1719013006] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 (Updated: 2024-06-21T23:36:46) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e (Updated: 2024-06-22T07:18:01 [TS: 1719040681] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e (Updated: 2024-06-22T07:18:01) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 (Updated: 2024-06-23T07:18:54 [TS: 1719127134] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 (Updated: 2024-06-23T07:18:54) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 (Updated: 2024-06-24T07:18:00 [TS: 1719213480] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 (Updated: 2024-06-24T07:18:00) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e (Updated: 2024-06-24T16:40:01 [TS: 1719247201] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e (Updated: 2024-06-24T16:40:01) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 (Updated: 2024-06-24T17:31:54 [TS: 1719250314] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 (Updated: 2024-06-24T17:31:54) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 (Updated: 2024-06-25T07:19:35 [TS: 1719299975] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 (Updated: 2024-06-25T07:19:35) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 (Updated: 2024-06-26T07:19:18 [TS: 1719386358] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 (Updated: 2024-06-26T07:19:18) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf (Updated: 2024-06-27T07:18:30 [TS: 1719472710] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf (Updated: 2024-06-27T07:18:30) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b (Updated: 2024-06-28T07:18:41 [TS: 1719559121] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b (Updated: 2024-06-28T07:18:41) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de (Updated: 2024-06-29T07:18:57 [TS: 1719645537] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de (Updated: 2024-06-29T07:18:57) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 (Updated: 2024-06-30T07:20:51 [TS: 1719732051] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 (Updated: 2024-06-30T07:20:51) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 (Updated: 2024-07-01T07:18:08 [TS: 1719818288] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 (Updated: 2024-07-01T07:18:08) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 (Updated: 2024-07-02T07:18:47 [TS: 1719904727] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 (Updated: 2024-07-02T07:18:47) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 (Updated: 2024-07-03T07:20:11 [TS: 1719991211] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 (Updated: 2024-07-03T07:20:11) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f (Updated: 2024-07-04T07:18:39 [TS: 1720077519] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f (Updated: 2024-07-04T07:18:39) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 (Updated: 2024-07-05T07:18:56 [TS: 1720163936] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 (Updated: 2024-07-05T07:18:56) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 (Updated: 2024-07-06T07:16:34 [TS: 1720250194] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 (Updated: 2024-07-06T07:16:34) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 (Updated: 2024-07-07T07:19:21 [TS: 1720336761] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 (Updated: 2024-07-07T07:19:21) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 (Updated: 2024-07-08T07:19:37 [TS: 1720423177] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 (Updated: 2024-07-08T07:19:37) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 (Updated: 2024-07-09T07:19:28 [TS: 1720509568] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 (Updated: 2024-07-09T07:19:28) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f (Updated: 2024-07-10T07:19:10 [TS: 1720595950] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f (Updated: 2024-07-10T07:19:10) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 (Updated: 2024-07-11T07:18:45 [TS: 1720682325] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 (Updated: 2024-07-11T07:18:45) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 (Updated: 2024-07-12T07:19:53 [TS: 1720768793] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 (Updated: 2024-07-12T07:19:53) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d (Updated: 2024-07-13T07:19:14 [TS: 1720855154] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d (Updated: 2024-07-13T07:19:14) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 (Updated: 2024-07-14T07:19:26 [TS: 1720941566] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 (Updated: 2024-07-14T07:19:26) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 (Updated: 2024-07-15T07:19:03 [TS: 1721027943] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 (Updated: 2024-07-15T07:19:03) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 (Updated: 2024-07-16T07:20:23 [TS: 1721114423] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 (Updated: 2024-07-16T07:20:23) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 (Updated: 2024-07-17T07:17:53 [TS: 1721200673] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 (Updated: 2024-07-17T07:17:53) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 (Updated: 2024-07-18T07:19:07 [TS: 1721287147] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 (Updated: 2024-07-18T07:19:07) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 (Updated: 2024-07-19T07:18:50 [TS: 1721373530] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 (Updated: 2024-07-19T07:18:50) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 (Updated: 2024-07-20T07:18:11 [TS: 1721459891] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 (Updated: 2024-07-20T07:18:11) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 (Updated: 2024-07-21T07:19:01 [TS: 1721546341] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 (Updated: 2024-07-21T07:19:01) +[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a (Updated: 2024-07-22T07:19:06 [TS: 1721632746] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a (Updated: 2024-07-22T07:19:06) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 (Updated: 2024-07-23T07:19:03 [TS: 1721719143] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 (Updated: 2024-07-23T07:19:03) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 (Updated: 2024-07-24T07:18:45 [TS: 1721805525] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 (Updated: 2024-07-24T07:18:45) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd (Updated: 2024-07-25T07:18:14 [TS: 1721891894] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd (Updated: 2024-07-25T07:18:14) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 (Updated: 2024-07-26T07:18:21 [TS: 1721978301] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 (Updated: 2024-07-26T07:18:21) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 (Updated: 2024-07-27T07:20:10 [TS: 1722064810] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 (Updated: 2024-07-27T07:20:10) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 (Updated: 2024-07-28T07:18:54 [TS: 1722151134] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 (Updated: 2024-07-28T07:18:54) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 (Updated: 2024-07-29T07:17:00 [TS: 1722237420] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 (Updated: 2024-07-29T07:17:00) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 (Updated: 2024-07-30T07:18:35 [TS: 1722323915] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 (Updated: 2024-07-30T07:18:35) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 (Updated: 2024-07-31T07:19:04 [TS: 1722410344] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 (Updated: 2024-07-31T07:19:04) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb (Updated: 2024-08-01T07:18:18 [TS: 1722496698] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb (Updated: 2024-08-01T07:18:18) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 (Updated: 2024-08-02T07:18:39 [TS: 1722583119] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 (Updated: 2024-08-02T07:18:39) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd (Updated: 2024-08-03T07:18:31 [TS: 1722669511] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd (Updated: 2024-08-03T07:18:31) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 (Updated: 2024-08-04T07:19:22 [TS: 1722755962] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 (Updated: 2024-08-04T07:19:22) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a (Updated: 2024-08-05T07:19:47 [TS: 1722842387] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a (Updated: 2024-08-05T07:19:47) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e (Updated: 2024-08-06T07:18:59 [TS: 1722928739] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e (Updated: 2024-08-06T07:18:59) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 (Updated: 2024-08-07T07:19:22 [TS: 1723015162] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 (Updated: 2024-08-07T07:19:22) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 (Updated: 2024-08-08T07:18:44 [TS: 1723101524] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 (Updated: 2024-08-08T07:18:44) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 (Updated: 2024-08-09T07:19:24 [TS: 1723187964] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 (Updated: 2024-08-09T07:19:24) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a (Updated: 2024-08-10T07:18:12 [TS: 1723274292] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a (Updated: 2024-08-10T07:18:12) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f (Updated: 2024-08-11T07:18:52 [TS: 1723360732] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f (Updated: 2024-08-11T07:18:52) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff (Updated: 2024-08-12T07:18:33 [TS: 1723447113] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff (Updated: 2024-08-12T07:18:33) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff (Updated: 2024-08-13T07:19:14 [TS: 1723533554] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff (Updated: 2024-08-13T07:19:14) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a (Updated: 2024-08-14T07:20:03 [TS: 1723620003] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a (Updated: 2024-08-14T07:20:03) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 (Updated: 2024-08-15T07:18:25 [TS: 1723706305] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 (Updated: 2024-08-15T07:18:25) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b (Updated: 2024-08-16T07:18:57 [TS: 1723792737] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b (Updated: 2024-08-16T07:18:57) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b (Updated: 2024-08-17T07:19:21 [TS: 1723879161] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b (Updated: 2024-08-17T07:19:21) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e (Updated: 2024-08-18T07:19:19 [TS: 1723965559] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e (Updated: 2024-08-18T07:19:19) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 (Updated: 2024-08-19T07:19:35 [TS: 1724051975] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 (Updated: 2024-08-19T07:19:35) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 (Updated: 2024-08-20T07:19:04 [TS: 1724138344] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 (Updated: 2024-08-20T07:19:04) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 (Updated: 2024-08-21T07:19:04 [TS: 1724224744] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 (Updated: 2024-08-21T07:19:04) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3b91af27e525f56f3ea343a5b051fc87a0f446e614700ba93b62d0eaebaa270 (Updated: 2024-08-22T07:18:34 [TS: 1724311114] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3b91af27e525f56f3ea343a5b051fc87a0f446e614700ba93b62d0eaebaa270 (Updated: 2024-08-22T07:18:34) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:701c37147c8af20d38046af61d1b92ffa9f17043b68ac37b20fed22e3b3f8e79 (Updated: 2024-08-23T07:19:29 [TS: 1724397569] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:701c37147c8af20d38046af61d1b92ffa9f17043b68ac37b20fed22e3b3f8e79 (Updated: 2024-08-23T07:19:29) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bf48f10d95c8d54302ad4f8867103021165ecc19863e553ca631e80257563c9 (Updated: 2024-08-24T07:18:56 [TS: 1724483936] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bf48f10d95c8d54302ad4f8867103021165ecc19863e553ca631e80257563c9 (Updated: 2024-08-24T07:18:56) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df1f07f0794958aa141b2333b7e891c76069a030a4df49df26ed55f685487e05 (Updated: 2024-08-25T07:18:09 [TS: 1724570289] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df1f07f0794958aa141b2333b7e891c76069a030a4df49df26ed55f685487e05 (Updated: 2024-08-25T07:18:09) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:984af1e0f55f42155e6ec839142c7c38f04ca1c1a25e19574de1d76c7a94758c (Updated: 2024-08-26T07:19:22 [TS: 1724656762] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:984af1e0f55f42155e6ec839142c7c38f04ca1c1a25e19574de1d76c7a94758c (Updated: 2024-08-26T07:19:22) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e85e777472660994c5d895d3ba68f320a661307ea61591b315086e52460586a (Updated: 2024-08-27T07:19:37 [TS: 1724743177] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e85e777472660994c5d895d3ba68f320a661307ea61591b315086e52460586a (Updated: 2024-08-27T07:19:37) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e33a34d6aef4605164895bba151e8a8ddaaa850e4feeece12c8328c477cbf94 (Updated: 2024-08-28T07:19:10 [TS: 1724829550] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e33a34d6aef4605164895bba151e8a8ddaaa850e4feeece12c8328c477cbf94 (Updated: 2024-08-28T07:19:10) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19aefe718d2632c0206b274ffee14c8cce3139254c73c13a8c85ce08e193f724 (Updated: 2024-08-29T07:21:40 [TS: 1724916100] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19aefe718d2632c0206b274ffee14c8cce3139254c73c13a8c85ce08e193f724 (Updated: 2024-08-29T07:21:40) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:256f6245d97633997bd4c3cc2573f97f39b348629a527a5b3fb8e22b7b857d15 (Updated: 2024-08-30T07:19:17 [TS: 1725002357] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:256f6245d97633997bd4c3cc2573f97f39b348629a527a5b3fb8e22b7b857d15 (Updated: 2024-08-30T07:19:17) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 (Updated: 2024-08-31T07:19:15 [TS: 1725088755] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 (Updated: 2024-08-31T07:19:15) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 (Updated: 2024-09-01T07:18:35 [TS: 1725175115] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 (Updated: 2024-09-01T07:18:35) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 (Updated: 2024-09-02T07:19:08 [TS: 1725261548] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 (Updated: 2024-09-02T07:19:08) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 (Updated: 2024-09-03T07:18:34 [TS: 1725347914] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 (Updated: 2024-09-03T07:18:34) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 (Updated: 2024-09-04T07:19:12 [TS: 1725434352] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 (Updated: 2024-09-04T07:19:12) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd (Updated: 2024-09-04T15:00:21 [TS: 1725462021] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd (Updated: 2024-09-04T15:00:21) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 (Updated: 2024-09-05T07:19:39 [TS: 1725520779] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 (Updated: 2024-09-05T07:19:39) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 (Updated: 2024-09-06T07:19:00 [TS: 1725607140] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 (Updated: 2024-09-06T07:19:00) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 (Updated: 2024-09-07T07:20:06 [TS: 1725693606] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 (Updated: 2024-09-07T07:20:06) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 (Updated: 2024-09-08T07:20:00 [TS: 1725780000] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 (Updated: 2024-09-08T07:20:00) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f (Updated: 2024-09-09T07:18:55 [TS: 1725866335] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f (Updated: 2024-09-09T07:18:55) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 (Updated: 2024-09-10T07:18:23 [TS: 1725952703] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 (Updated: 2024-09-10T07:18:23) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 (Updated: 2024-09-11T07:18:27 [TS: 1726039107] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 (Updated: 2024-09-11T07:18:27) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b (Updated: 2024-09-12T07:19:28 [TS: 1726125568] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b (Updated: 2024-09-12T07:19:28) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 (Updated: 2024-09-13T07:18:59 [TS: 1726211939] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 (Updated: 2024-09-13T07:18:59) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 (Updated: 2024-09-14T07:21:55 [TS: 1726298515] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 (Updated: 2024-09-14T07:21:55) +[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 (Updated: 2024-09-15T07:18:46 [TS: 1726384726] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 (Updated: 2024-09-15T07:18:46) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 (Updated: 2024-09-16T07:19:43 [TS: 1726471183] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 (Updated: 2024-09-16T07:19:43) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 (Updated: 2024-09-17T07:18:21 [TS: 1726557501] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 (Updated: 2024-09-17T07:18:21) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af (Updated: 2024-09-18T07:18:46 [TS: 1726643926] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af (Updated: 2024-09-18T07:18:46) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 (Updated: 2024-09-19T07:19:58 [TS: 1726730398] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 (Updated: 2024-09-19T07:19:58) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c (Updated: 2024-09-20T07:19:34 [TS: 1726816774] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c (Updated: 2024-09-20T07:19:34) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 (Updated: 2024-09-21T07:18:42 [TS: 1726903122] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 (Updated: 2024-09-21T07:18:42) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f (Updated: 2024-09-22T07:19:13 [TS: 1726989553] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f (Updated: 2024-09-22T07:19:13) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 (Updated: 2024-09-23T07:19:27 [TS: 1727075967] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 (Updated: 2024-09-23T07:19:27) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a (Updated: 2024-09-24T07:19:03 [TS: 1727162343] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a (Updated: 2024-09-24T07:19:03) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 (Updated: 2024-09-25T07:19:26 [TS: 1727248766] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 (Updated: 2024-09-25T07:19:26) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 (Updated: 2024-09-26T07:18:34 [TS: 1727335114] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 (Updated: 2024-09-26T07:18:34) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 (Updated: 2024-09-27T07:18:51 [TS: 1727421531] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 (Updated: 2024-09-27T07:18:51) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 (Updated: 2024-09-28T07:19:08 [TS: 1727507948] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 (Updated: 2024-09-28T07:19:08) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b (Updated: 2024-09-29T07:19:59 [TS: 1727594399] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b (Updated: 2024-09-29T07:19:59) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 (Updated: 2024-09-30T07:18:47 [TS: 1727680727] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 (Updated: 2024-09-30T07:18:47) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 (Updated: 2024-10-01T07:18:16 [TS: 1727767096] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 (Updated: 2024-10-01T07:18:16) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 (Updated: 2024-10-02T07:19:22 [TS: 1727853562] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 (Updated: 2024-10-02T07:19:22) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d (Updated: 2024-10-03T07:19:40 [TS: 1727939980] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d (Updated: 2024-10-03T07:19:40) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c (Updated: 2024-10-04T07:20:00 [TS: 1728026400] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c (Updated: 2024-10-04T07:20:00) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 (Updated: 2024-10-05T07:20:18 [TS: 1728112818] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 (Updated: 2024-10-05T07:20:18) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d (Updated: 2024-10-06T07:19:45 [TS: 1728199185] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d (Updated: 2024-10-06T07:19:45) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 (Updated: 2024-10-07T07:20:12 [TS: 1728285612] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 (Updated: 2024-10-07T07:20:12) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 (Updated: 2024-10-08T07:19:06 [TS: 1728371946] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 (Updated: 2024-10-08T07:19:06) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 (Updated: 2024-10-09T07:19:20 [TS: 1728458360] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 (Updated: 2024-10-09T07:19:20) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e (Updated: 2024-10-10T07:19:58 [TS: 1728544798] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e (Updated: 2024-10-10T07:19:58) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf (Updated: 2024-10-11T07:19:31 [TS: 1728631171] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf (Updated: 2024-10-11T07:19:31) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 (Updated: 2024-10-12T07:19:51 [TS: 1728717591] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 (Updated: 2024-10-12T07:19:51) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 (Updated: 2024-10-13T07:20:18 [TS: 1728804018] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 (Updated: 2024-10-13T07:20:18) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 (Updated: 2024-10-14T07:18:56 [TS: 1728890336] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 (Updated: 2024-10-14T07:18:56) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 (Updated: 2024-10-15T07:18:42 [TS: 1728976722] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 (Updated: 2024-10-15T07:18:42) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 (Updated: 2024-10-16T07:18:53 [TS: 1729063133] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 (Updated: 2024-10-16T07:18:53) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 (Updated: 2024-10-17T07:19:38 [TS: 1729149578] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 (Updated: 2024-10-17T07:19:38) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 (Updated: 2024-10-18T07:21:27 [TS: 1729236087] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 (Updated: 2024-10-18T07:21:27) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 (Updated: 2024-10-19T07:21:08 [TS: 1729322468] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 (Updated: 2024-10-19T07:21:08) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 (Updated: 2024-10-20T07:20:13 [TS: 1729408813] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 (Updated: 2024-10-20T07:20:13) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 (Updated: 2024-10-21T07:19:27 [TS: 1729495167] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 (Updated: 2024-10-21T07:19:27) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 (Updated: 2024-10-22T07:19:36 [TS: 1729581576] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 (Updated: 2024-10-22T07:19:36) +[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 (Updated: 2024-10-23T07:18:47 [TS: 1729667927] < Cutoff: [TS: 1763304862]) +[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 (Updated: 2024-10-23T07:18:47) +[2025-11-30 14:54:31] [INFO] Hit delete limit (200) for Docker Images. +[2025-11-30 14:54:31] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 14:54:31] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 14:54:34] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 14:54:34] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 14:54:34] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 14:54:34] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 14:54:34] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 14:54:34] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 14:54:36] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 14:54:36] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 14:54:38] [INFO] No Regional Address found matching criteria. +[2025-11-30 14:54:38] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 14:54:41] [INFO] No Global Address found matching criteria. +[2025-11-30 14:54:41] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- +[2025-11-30 14:54:46] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 14:54:48] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 14:54:48] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 14:54:48] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 14:54:48] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 14:54:48] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:51] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 14:54:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 14:54:53] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 14:54:55] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 14:54:55] [INFO] CLEANUP RUN FINISHED +[2025-11-30 14:54:59] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 14:54:59] [INFO] Time Cutoff (General): 2025-11-30T14:54:59+0000 +[2025-11-30 14:54:59] [INFO] Time Cutoff (Images): 2025-10-01T14:54:59+0000 +[2025-11-30 14:54:59] [INFO] Delete Limit per Type: 200 +[2025-11-30 14:54:59] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 14:54:59] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 14:55:01] [INFO] No Service Accounts found matching prefix. +[2025-11-30 14:55:01] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 14:55:03] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 14:55:03] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 14:55:05] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 14:55:05] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 14:55:05] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 14:55:05] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 14:55:08] [INFO] No Filestore instances found matching criteria. +[2025-11-30 14:55:08] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 14:55:11] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 14:55:11] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 14:55:11] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 14:55:11] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 14:55:11] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 14:55:11] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 14:55:11] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 14:55:11] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 14:55:12] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 14:55:12] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 14:55:12] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 14:55:12] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T14:55:12Z (Unix: 1763304912) +[2025-11-30 14:55:12] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 14:55:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19 [TS: 1713165559] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fdbb17db-84a0-49ed-9453-cadd0b3aaf04] to complete... +.....done. +[2025-11-30 14:55:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 +[2025-11-30 14:55:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17 [TS: 1713251897] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2d2b4fd8-6c71-407a-829c-7edb3611ef06] to complete... +.....done. +[2025-11-30 14:55:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe +[2025-11-30 14:55:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57 [TS: 1713338277] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7ee525c9-c9b8-4493-b95d-f1f1e5c7c2bb] to complete... +.....done. +[2025-11-30 14:55:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 +[2025-11-30 14:55:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08 [TS: 1713424748] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b0c80bb2-8d1e-4a80-bb56-1215d9f124d3] to complete... +......done. +[2025-11-30 14:55:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 +[2025-11-30 14:55:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26 [TS: 1713511166] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b441399d-cdd1-4b57-8bf2-0bca58eecec8] to complete... +.....done. +[2025-11-30 14:55:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a +[2025-11-30 14:55:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36 [TS: 1713597516] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0bdff779-3814-4f6a-8490-2e949eb47612] to complete... +.....done. +[2025-11-30 14:55:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c +[2025-11-30 14:55:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57 [TS: 1713683937] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/449226a3-aa7f-4d94-a7de-ea3f23785646] to complete... +.....done. +[2025-11-30 14:55:41] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 +[2025-11-30 14:55:41] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38 [TS: 1713770258] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:41] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/14fe9a67-bc2e-41b6-a946-818a8e3531ab] to complete... +.....done. +[2025-11-30 14:55:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 +[2025-11-30 14:55:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51 [TS: 1713856671] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/efa195f6-a784-4348-baa2-98a69a2c09df] to complete... +.....done. +[2025-11-30 14:55:48] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 +[2025-11-30 14:55:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09 [TS: 1713943149] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7d8906eb-4fb1-4577-ae4d-0dafce979c00] to complete... +......done. +[2025-11-30 14:55:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa +[2025-11-30 14:55:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19 [TS: 1714029499] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d11827a6-b7c1-4553-ab33-46c4272da5ff] to complete... +.....done. +[2025-11-30 14:55:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 +[2025-11-30 14:55:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26 [TS: 1714115966] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e6dda756-3406-4e4d-8492-317c2770f9db] to complete... +.....done. +[2025-11-30 14:55:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d +[2025-11-30 14:55:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09 [TS: 1714202349] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:55:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f72967bd-f7ee-4098-b988-c9f3d6333b6b] to complete... +.....done. +[2025-11-30 14:56:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 +[2025-11-30 14:56:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58 [TS: 1714288738] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ea5a78e6-38b7-470e-b38c-5610eaf6520f] to complete... +.....done. +[2025-11-30 14:56:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 +[2025-11-30 14:56:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09 [TS: 1714375269] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/24d36492-952c-42d8-9810-0f7f88cf7d86] to complete... +.....done. +[2025-11-30 14:56:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 +[2025-11-30 14:56:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18 [TS: 1714461498] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/84a8eedc-4865-4f31-8f62-70a3c990ed19] to complete... +.....done. +[2025-11-30 14:56:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f +[2025-11-30 14:56:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48 [TS: 1714547868] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1b9dd06a-d5d4-4624-bc92-a99b2a1af57f] to complete... +......done. +[2025-11-30 14:56:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 +[2025-11-30 14:56:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44 [TS: 1714634264] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/98f7868d-eab0-4b02-b940-4b93d19051d3] to complete... +.....done. +[2025-11-30 14:56:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 +[2025-11-30 14:56:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43 [TS: 1714720723] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/21632ad0-cd26-42a5-a3e9-6fba7fcbfbfd] to complete... +.....done. +[2025-11-30 14:56:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc +[2025-11-30 14:56:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00 [TS: 1714807080] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0ae5fb9e-54ad-4286-a6de-e6c460711756] to complete... +.....done. +[2025-11-30 14:56:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 +[2025-11-30 14:56:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 (Updated: 2024-05-05T07:19:28 [TS: 1714893568] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 (Updated: 2024-05-05T07:19:28) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d1f73bc7-1c11-4029-bf03-65bbeeddefd3] to complete... +......done. +[2025-11-30 14:56:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 +[2025-11-30 14:56:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 (Updated: 2024-05-06T07:18:08 [TS: 1714979888] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 (Updated: 2024-05-06T07:18:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e0a71f50-88e2-4fc4-a6fb-ae06270f00bd] to complete... +.....done. +[2025-11-30 14:56:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 +[2025-11-30 14:56:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 (Updated: 2024-05-07T07:20:44 [TS: 1715066444] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 (Updated: 2024-05-07T07:20:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0ad96f72-aaa4-43bc-9902-fb836fd35d1c] to complete... +.....done. +[2025-11-30 14:56:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 +[2025-11-30 14:56:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 (Updated: 2024-05-08T07:18:32 [TS: 1715152712] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 (Updated: 2024-05-08T07:18:32) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f7e45e9f-22d9-47a9-80f6-1583f80508c7] to complete... +.....done. +[2025-11-30 14:56:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 +[2025-11-30 14:56:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b (Updated: 2024-05-09T07:19:05 [TS: 1715239145] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b (Updated: 2024-05-09T07:19:05) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/693edaf6-e9bd-4e9b-9145-d1c04e5e617d] to complete... +.....done. +[2025-11-30 14:56:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b +[2025-11-30 14:56:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd (Updated: 2024-05-10T07:18:32 [TS: 1715325512] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd (Updated: 2024-05-10T07:18:32) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/738386a2-1ca0-453e-9e37-8f155596602f] to complete... +.....done. +[2025-11-30 14:56:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd +[2025-11-30 14:56:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 (Updated: 2024-05-11T07:18:53 [TS: 1715411933] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 (Updated: 2024-05-11T07:18:53) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8fb12ac3-485d-4e13-a57e-5af50568d261] to complete... +.....done. +[2025-11-30 14:56:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 +[2025-11-30 14:56:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 (Updated: 2024-05-12T07:18:59 [TS: 1715498339] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 (Updated: 2024-05-12T07:18:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cd0a0897-12d1-4736-85ff-f55e141335ad] to complete... +.....done. +[2025-11-30 14:56:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 +[2025-11-30 14:56:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 (Updated: 2024-05-13T07:18:49 [TS: 1715584729] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 (Updated: 2024-05-13T07:18:49) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1a3c05ea-c11f-40a3-b4f5-1c1e7258e925] to complete... +......done. +[2025-11-30 14:56:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 +[2025-11-30 14:56:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc (Updated: 2024-05-14T07:16:18 [TS: 1715670978] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:56:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc (Updated: 2024-05-14T07:16:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/64a36cbc-6e34-4ae1-bc80-36778391f068] to complete... +.....done. +[2025-11-30 14:57:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc +[2025-11-30 14:57:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 (Updated: 2024-05-15T07:18:59 [TS: 1715757539] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 (Updated: 2024-05-15T07:18:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4bcae04e-a220-4eb3-a02a-5222ac476c4c] to complete... +......done. +[2025-11-30 14:57:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 +[2025-11-30 14:57:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 (Updated: 2024-05-16T07:18:34 [TS: 1715843914] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 (Updated: 2024-05-16T07:18:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c5a5a145-44ff-478a-9e94-661c9b038c00] to complete... +.....done. +[2025-11-30 14:57:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 +[2025-11-30 14:57:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 (Updated: 2024-05-17T07:18:04 [TS: 1715930284] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 (Updated: 2024-05-17T07:18:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bd291f25-af70-41ab-9459-01c5aea175d0] to complete... +......done. +[2025-11-30 14:57:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 +[2025-11-30 14:57:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 (Updated: 2024-05-18T07:18:46 [TS: 1716016726] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 (Updated: 2024-05-18T07:18:46) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/47bcc8e7-de77-4438-8731-dcf6d5c3c0ad] to complete... +.....done. +[2025-11-30 14:57:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 +[2025-11-30 14:57:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd (Updated: 2024-05-19T07:19:20 [TS: 1716103160] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd (Updated: 2024-05-19T07:19:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c304f58d-7edd-46ee-b8f0-ad1e0d4913e3] to complete... +.....done. +[2025-11-30 14:57:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd +[2025-11-30 14:57:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c (Updated: 2024-05-20T07:18:54 [TS: 1716189534] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c (Updated: 2024-05-20T07:18:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b5cb8a87-1cc4-4ce3-ad79-aecd60172fc3] to complete... +.....done. +[2025-11-30 14:57:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c +[2025-11-30 14:57:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce (Updated: 2024-05-21T07:18:14 [TS: 1716275894] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce (Updated: 2024-05-21T07:18:14) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4d34d297-f956-4914-ac69-ea66b91af928] to complete... +......done. +[2025-11-30 14:57:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce +[2025-11-30 14:57:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e (Updated: 2024-05-22T07:17:56 [TS: 1716362276] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e (Updated: 2024-05-22T07:17:56) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2bf45745-42cf-4fe6-843c-f8f703bb57d3] to complete... +.....done. +[2025-11-30 14:57:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e +[2025-11-30 14:57:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 (Updated: 2024-05-23T07:17:57 [TS: 1716448677] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 (Updated: 2024-05-23T07:17:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b4f7a340-1e92-4b07-a3c9-5a5ea0d7945b] to complete... +.....done. +[2025-11-30 14:57:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 +[2025-11-30 14:57:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 (Updated: 2024-05-24T07:17:47 [TS: 1716535067] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 (Updated: 2024-05-24T07:17:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6d7d8367-1dbc-4864-82d7-56b8ba4e1b10] to complete... +.....done. +[2025-11-30 14:57:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 +[2025-11-30 14:57:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 (Updated: 2024-05-25T07:18:10 [TS: 1716621490] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 (Updated: 2024-05-25T07:18:10) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e6251c19-2355-4c03-9f53-8d44e2ace8ec] to complete... +.....done. +[2025-11-30 14:57:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 +[2025-11-30 14:57:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 (Updated: 2024-05-26T07:18:34 [TS: 1716707914] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 (Updated: 2024-05-26T07:18:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/337a0709-ddfd-4099-bea4-b0486f84ec10] to complete... +.....done. +[2025-11-30 14:57:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 +[2025-11-30 14:57:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 (Updated: 2024-05-27T07:18:35 [TS: 1716794315] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 (Updated: 2024-05-27T07:18:35) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/059e6c66-2752-4bd9-b345-fe672fd0684a] to complete... +.....done. +[2025-11-30 14:57:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 +[2025-11-30 14:57:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf (Updated: 2024-05-28T07:18:07 [TS: 1716880687] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf (Updated: 2024-05-28T07:18:07) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d828038b-88dd-443e-bc61-89b320887c02] to complete... +.....done. +[2025-11-30 14:57:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf +[2025-11-30 14:57:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 (Updated: 2024-05-29T07:17:33 [TS: 1716967053] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 (Updated: 2024-05-29T07:17:33) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a836a188-4f4b-4e0f-9ee6-ca2942059814] to complete... +.....done. +[2025-11-30 14:57:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 +[2025-11-30 14:57:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d (Updated: 2024-05-30T07:18:38 [TS: 1717053518] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d (Updated: 2024-05-30T07:18:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/26e66eb0-0e4d-4fe6-aa1f-a029ef6de4f6] to complete... +.....done. +[2025-11-30 14:57:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d +[2025-11-30 14:57:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 (Updated: 2024-05-31T07:18:40 [TS: 1717139920] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:57:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 (Updated: 2024-05-31T07:18:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5dd2f14a-2266-4d15-88d6-d7f5c655d534] to complete... +.....done. +[2025-11-30 14:58:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 +[2025-11-30 14:58:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de (Updated: 2024-06-01T07:18:20 [TS: 1717226300] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de (Updated: 2024-06-01T07:18:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d2eca114-25ee-431b-a6d9-1d98a0f30c67] to complete... +.....done. +[2025-11-30 14:58:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de +[2025-11-30 14:58:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb (Updated: 2024-06-02T07:19:05 [TS: 1717312745] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb (Updated: 2024-06-02T07:19:05) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9531cd8d-4b21-4ba8-941a-387f1e9bc71b] to complete... +......done. +[2025-11-30 14:58:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb +[2025-11-30 14:58:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 (Updated: 2024-06-03T07:18:42 [TS: 1717399122] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 (Updated: 2024-06-03T07:18:42) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7b029aeb-523b-48ee-bda2-a01354e1587c] to complete... +.....done. +[2025-11-30 14:58:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 +[2025-11-30 14:58:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 (Updated: 2024-06-04T07:18:28 [TS: 1717485508] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 (Updated: 2024-06-04T07:18:28) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/68b265d3-002a-4cfa-ba1f-1fe40ff6372d] to complete... +.....done. +[2025-11-30 14:58:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 +[2025-11-30 14:58:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 (Updated: 2024-06-05T07:18:24 [TS: 1717571904] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 (Updated: 2024-06-05T07:18:24) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/594858ff-d5ee-4e5f-91b5-7477b9a84612] to complete... +......done. +[2025-11-30 14:58:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 +[2025-11-30 14:58:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 (Updated: 2024-06-06T07:20:38 [TS: 1717658438] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 (Updated: 2024-06-06T07:20:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e59d0392-8c2f-47ef-bbe1-4d4058bc13c1] to complete... +......done. +[2025-11-30 14:58:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 +[2025-11-30 14:58:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d (Updated: 2024-06-07T07:18:27 [TS: 1717744707] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d (Updated: 2024-06-07T07:18:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4dbdd5be-aa1a-43d3-ad6d-6fac43de8a74] to complete... +.....done. +[2025-11-30 14:58:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d +[2025-11-30 14:58:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 (Updated: 2024-06-08T07:18:28 [TS: 1717831108] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 (Updated: 2024-06-08T07:18:28) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c6a10fb9-3a55-483c-9ab7-556491107ad7] to complete... +.....done. +[2025-11-30 14:58:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 +[2025-11-30 14:58:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e (Updated: 2024-06-09T07:18:23 [TS: 1717917503] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e (Updated: 2024-06-09T07:18:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/14253a47-a556-4848-8219-50d3a4d484bc] to complete... +.....done. +[2025-11-30 14:58:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e +[2025-11-30 14:58:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 (Updated: 2024-06-10T07:18:11 [TS: 1718003891] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 (Updated: 2024-06-10T07:18:11) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/270588a0-8ce3-4ae6-abc6-8e725f199342] to complete... +.....done. +[2025-11-30 14:58:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 +[2025-11-30 14:58:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 (Updated: 2024-06-11T07:18:43 [TS: 1718090323] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 (Updated: 2024-06-11T07:18:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2501e5d5-5251-4ce3-8e93-6ffb13530929] to complete... +.....done. +[2025-11-30 14:58:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 +[2025-11-30 14:58:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 (Updated: 2024-06-12T07:18:47 [TS: 1718176727] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 (Updated: 2024-06-12T07:18:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/470e51e9-0439-424c-9883-a754214b5e31] to complete... +.....done. +[2025-11-30 14:58:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 +[2025-11-30 14:58:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a (Updated: 2024-06-13T07:19:10 [TS: 1718263150] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a (Updated: 2024-06-13T07:19:10) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/27e6ec1a-1642-48ee-b681-ab4ac5703c7b] to complete... +......done. +[2025-11-30 14:58:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a +[2025-11-30 14:58:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c (Updated: 2024-06-14T07:18:33 [TS: 1718349513] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c (Updated: 2024-06-14T07:18:33) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4a7234b8-39c8-482e-bedf-fd22ebeed203] to complete... +.....done. +[2025-11-30 14:58:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c +[2025-11-30 14:58:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 (Updated: 2024-06-15T07:19:01 [TS: 1718435941] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 (Updated: 2024-06-15T07:19:01) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/93ad0cc5-8044-4645-81c2-006f70eebd8f] to complete... +.....done. +[2025-11-30 14:58:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 +[2025-11-30 14:58:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 (Updated: 2024-06-16T07:18:57 [TS: 1718522337] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 (Updated: 2024-06-16T07:18:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f7d00588-e25e-447d-812b-1c84db008dfc] to complete... +.....done. +[2025-11-30 14:58:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 +[2025-11-30 14:58:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 (Updated: 2024-06-17T07:17:57 [TS: 1718608677] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:58:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 (Updated: 2024-06-17T07:17:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a45744ad-e6a2-4598-994c-00a5c8d3256d] to complete... +.....done. +[2025-11-30 14:59:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 +[2025-11-30 14:59:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 (Updated: 2024-06-18T07:18:50 [TS: 1718695130] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 (Updated: 2024-06-18T07:18:50) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/84311460-80c7-498f-90af-2f124687686f] to complete... +.....done. +[2025-11-30 14:59:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 +[2025-11-30 14:59:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 (Updated: 2024-06-19T07:18:24 [TS: 1718781504] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 (Updated: 2024-06-19T07:18:24) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1f08f0cb-11c0-4bd6-9228-348bbc230dfd] to complete... +......done. +[2025-11-30 14:59:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 +[2025-11-30 14:59:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c (Updated: 2024-06-20T07:18:11 [TS: 1718867891] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c (Updated: 2024-06-20T07:18:11) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/da006e24-8cb1-4d03-a9ca-ffc3930feccc] to complete... +.....done. +[2025-11-30 14:59:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c +[2025-11-30 14:59:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 (Updated: 2024-06-20T17:43:06 [TS: 1718905386] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 (Updated: 2024-06-20T17:43:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f09b92f2-2256-43bc-843a-1e945ee40e30] to complete... +.....done. +[2025-11-30 14:59:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 +[2025-11-30 14:59:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a (Updated: 2024-06-21T07:20:29 [TS: 1718954429] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a (Updated: 2024-06-21T07:20:29) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7520ce0d-9cfc-4cf8-ad86-a36ad8558993] to complete... +.....done. +[2025-11-30 14:59:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a +[2025-11-30 14:59:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 (Updated: 2024-06-21T18:43:44 [TS: 1718995424] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 (Updated: 2024-06-21T18:43:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7fadf1ca-eb50-4d40-a00e-d950f4107705] to complete... +.....done. +[2025-11-30 14:59:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 +[2025-11-30 14:59:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 (Updated: 2024-06-21T20:07:06 [TS: 1719000426] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 (Updated: 2024-06-21T20:07:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f017fafc-2960-4fd1-91e1-ff5cbad072a8] to complete... +.....done. +[2025-11-30 14:59:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 +[2025-11-30 14:59:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db (Updated: 2024-06-21T22:45:30 [TS: 1719009930] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db (Updated: 2024-06-21T22:45:30) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e6190cac-b89a-4ef9-8135-735f882af9fa] to complete... +.....done. +[2025-11-30 14:59:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db +[2025-11-30 14:59:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 (Updated: 2024-06-21T23:36:46 [TS: 1719013006] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 (Updated: 2024-06-21T23:36:46) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8133e7f1-63fe-4775-96c3-e0206cd87636] to complete... +.....done. +[2025-11-30 14:59:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 +[2025-11-30 14:59:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e (Updated: 2024-06-22T07:18:01 [TS: 1719040681] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e (Updated: 2024-06-22T07:18:01) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/597bc3da-70bb-4739-93e1-c95cec863e5a] to complete... +.....done. +[2025-11-30 14:59:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e +[2025-11-30 14:59:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 (Updated: 2024-06-23T07:18:54 [TS: 1719127134] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 (Updated: 2024-06-23T07:18:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/02545905-5c63-4994-9eb7-d5107a6e954d] to complete... +.....done. +[2025-11-30 14:59:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 +[2025-11-30 14:59:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 (Updated: 2024-06-24T07:18:00 [TS: 1719213480] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 (Updated: 2024-06-24T07:18:00) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/190c0e84-9d75-4665-9aa2-51dda9cd1c18] to complete... +.....done. +[2025-11-30 14:59:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 +[2025-11-30 14:59:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e (Updated: 2024-06-24T16:40:01 [TS: 1719247201] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e (Updated: 2024-06-24T16:40:01) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/43247d19-97ea-490d-8c24-83fdacbbc804] to complete... +......done. +[2025-11-30 14:59:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e +[2025-11-30 14:59:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 (Updated: 2024-06-24T17:31:54 [TS: 1719250314] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 (Updated: 2024-06-24T17:31:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 + +Tags: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner:test-kubectl +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8b9bc5ef-47de-4afd-ac6e-7e7bcfdcacd8] to complete... +.....done. +[2025-11-30 14:59:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 +[2025-11-30 14:59:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 (Updated: 2024-06-25T07:19:35 [TS: 1719299975] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 (Updated: 2024-06-25T07:19:35) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/01c9ac72-0340-4e85-b263-9f7fff326a4a] to complete... +.....done. +[2025-11-30 14:59:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 +[2025-11-30 14:59:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 (Updated: 2024-06-26T07:19:18 [TS: 1719386358] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 (Updated: 2024-06-26T07:19:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b0cd5be1-27f8-4aec-8bff-aefbef5e5305] to complete... +......done. +[2025-11-30 14:59:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 +[2025-11-30 14:59:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf (Updated: 2024-06-27T07:18:30 [TS: 1719472710] < Cutoff: [TS: 1763304912]) +[2025-11-30 14:59:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf (Updated: 2024-06-27T07:18:30) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f8da73d3-2020-4533-b249-87309c114809] to complete... +.....done. +[2025-11-30 15:00:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf +[2025-11-30 15:00:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b (Updated: 2024-06-28T07:18:41 [TS: 1719559121] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b (Updated: 2024-06-28T07:18:41) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/69217e24-d9de-4c5a-94e3-13893feed995] to complete... +.....done. +[2025-11-30 15:00:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b +[2025-11-30 15:00:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de (Updated: 2024-06-29T07:18:57 [TS: 1719645537] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de (Updated: 2024-06-29T07:18:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f61199da-077b-4461-ad81-b2e2962129fb] to complete... +.....done. +[2025-11-30 15:00:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de +[2025-11-30 15:00:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 (Updated: 2024-06-30T07:20:51 [TS: 1719732051] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 (Updated: 2024-06-30T07:20:51) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4f95586a-e372-4383-96c9-387dd4383757] to complete... +......done. +[2025-11-30 15:00:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 +[2025-11-30 15:00:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 (Updated: 2024-07-01T07:18:08 [TS: 1719818288] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 (Updated: 2024-07-01T07:18:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bf9bd343-1029-416c-8c26-8abe239bb73d] to complete... +.....done. +[2025-11-30 15:00:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 +[2025-11-30 15:00:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 (Updated: 2024-07-02T07:18:47 [TS: 1719904727] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 (Updated: 2024-07-02T07:18:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2ec2483e-a897-4b85-8144-32d8c84c52fe] to complete... +.....done. +[2025-11-30 15:00:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 +[2025-11-30 15:00:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 (Updated: 2024-07-03T07:20:11 [TS: 1719991211] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 (Updated: 2024-07-03T07:20:11) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/35bf6fa9-e13f-4af1-b9a8-5b5128a99114] to complete... +.....done. +[2025-11-30 15:00:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 +[2025-11-30 15:00:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f (Updated: 2024-07-04T07:18:39 [TS: 1720077519] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f (Updated: 2024-07-04T07:18:39) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/811fbec7-dc28-4309-a838-8c07ef3cf73b] to complete... +.....done. +[2025-11-30 15:00:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f +[2025-11-30 15:00:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 (Updated: 2024-07-05T07:18:56 [TS: 1720163936] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 (Updated: 2024-07-05T07:18:56) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/53fe5206-b951-44de-80b5-476383806bad] to complete... +.....done. +[2025-11-30 15:00:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 +[2025-11-30 15:00:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 (Updated: 2024-07-06T07:16:34 [TS: 1720250194] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 (Updated: 2024-07-06T07:16:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/17b471f7-20d3-423e-a7de-8a657f304861] to complete... +.....done. +[2025-11-30 15:00:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 +[2025-11-30 15:00:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 (Updated: 2024-07-07T07:19:21 [TS: 1720336761] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 (Updated: 2024-07-07T07:19:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/144e1b99-f7a9-4853-aeab-43045cced4d0] to complete... +.....done. +[2025-11-30 15:00:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 +[2025-11-30 15:00:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 (Updated: 2024-07-08T07:19:37 [TS: 1720423177] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 (Updated: 2024-07-08T07:19:37) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/beac3e5a-36b6-4f55-9df7-c40cb2b373de] to complete... +......done. +[2025-11-30 15:00:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 +[2025-11-30 15:00:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 (Updated: 2024-07-09T07:19:28 [TS: 1720509568] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 (Updated: 2024-07-09T07:19:28) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d2c043f4-256b-4aa7-a3ac-7fb37fcfb5b6] to complete... +......done. +[2025-11-30 15:00:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 +[2025-11-30 15:00:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f (Updated: 2024-07-10T07:19:10 [TS: 1720595950] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f (Updated: 2024-07-10T07:19:10) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bcacade1-5023-4579-a83c-f4b3aa837745] to complete... +......done. +[2025-11-30 15:00:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f +[2025-11-30 15:00:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 (Updated: 2024-07-11T07:18:45 [TS: 1720682325] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 (Updated: 2024-07-11T07:18:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a57c96c8-4799-4d19-8f5f-2bb0cc0dc89b] to complete... +......done. +[2025-11-30 15:00:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 +[2025-11-30 15:00:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 (Updated: 2024-07-12T07:19:53 [TS: 1720768793] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 (Updated: 2024-07-12T07:19:53) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/22e31337-4ed9-4515-8361-c6f34e74c543] to complete... +......done. +[2025-11-30 15:00:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 +[2025-11-30 15:00:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d (Updated: 2024-07-13T07:19:14 [TS: 1720855154] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d (Updated: 2024-07-13T07:19:14) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4882351c-2a43-4e08-a348-a0c4a61f6e3c] to complete... +.....done. +[2025-11-30 15:00:59] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d +[2025-11-30 15:00:59] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 (Updated: 2024-07-14T07:19:26 [TS: 1720941566] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:00:59] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 (Updated: 2024-07-14T07:19:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4a19e534-ebb8-4850-8d79-9584d50893f4] to complete... +.....done. +[2025-11-30 15:01:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 +[2025-11-30 15:01:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 (Updated: 2024-07-15T07:19:03 [TS: 1721027943] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 (Updated: 2024-07-15T07:19:03) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/af5c3390-8ab8-4f03-9b4c-1a368ae70025] to complete... +.....done. +[2025-11-30 15:01:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 +[2025-11-30 15:01:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 (Updated: 2024-07-16T07:20:23 [TS: 1721114423] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 (Updated: 2024-07-16T07:20:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/acd58f1e-8f64-4252-826b-3fb0e90788fc] to complete... +.....done. +[2025-11-30 15:01:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 +[2025-11-30 15:01:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 (Updated: 2024-07-17T07:17:53 [TS: 1721200673] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 (Updated: 2024-07-17T07:17:53) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5151d5ea-0890-4293-8526-7e1e06e5c7e1] to complete... +.....done. +[2025-11-30 15:01:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 +[2025-11-30 15:01:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 (Updated: 2024-07-18T07:19:07 [TS: 1721287147] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 (Updated: 2024-07-18T07:19:07) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4c19ac17-5502-46fb-97a7-99d7df10e881] to complete... +.....done. +[2025-11-30 15:01:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 +[2025-11-30 15:01:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 (Updated: 2024-07-19T07:18:50 [TS: 1721373530] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 (Updated: 2024-07-19T07:18:50) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e267c491-ee80-4fde-b3ea-5f2253ec01d1] to complete... +.....done. +[2025-11-30 15:01:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 +[2025-11-30 15:01:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 (Updated: 2024-07-20T07:18:11 [TS: 1721459891] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 (Updated: 2024-07-20T07:18:11) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4989c3f1-a41b-49df-8225-33347580711b] to complete... +......done. +[2025-11-30 15:01:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 +[2025-11-30 15:01:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 (Updated: 2024-07-21T07:19:01 [TS: 1721546341] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 (Updated: 2024-07-21T07:19:01) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cff1ddc4-13ec-41e4-b1f4-d7afb9034fff] to complete... +......done. +[2025-11-30 15:01:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 +[2025-11-30 15:01:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a (Updated: 2024-07-22T07:19:06 [TS: 1721632746] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a (Updated: 2024-07-22T07:19:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/94ca1c60-87cd-49de-af02-d1118255a792] to complete... +.....done. +[2025-11-30 15:01:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a +[2025-11-30 15:01:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 (Updated: 2024-07-23T07:19:03 [TS: 1721719143] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 (Updated: 2024-07-23T07:19:03) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8bb8dda1-cc97-4f29-a0f7-00576771588b] to complete... +.....done. +[2025-11-30 15:01:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 +[2025-11-30 15:01:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 (Updated: 2024-07-24T07:18:45 [TS: 1721805525] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 (Updated: 2024-07-24T07:18:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/15563df9-b44d-45aa-845b-26cee7b5bfc9] to complete... +.....done. +[2025-11-30 15:01:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 +[2025-11-30 15:01:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd (Updated: 2024-07-25T07:18:14 [TS: 1721891894] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd (Updated: 2024-07-25T07:18:14) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/590f2894-9273-42f1-9801-5e8aa72de232] to complete... +.....done. +[2025-11-30 15:01:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd +[2025-11-30 15:01:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 (Updated: 2024-07-26T07:18:21 [TS: 1721978301] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 (Updated: 2024-07-26T07:18:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7a1d588e-12d7-40c1-ba87-142c4bf32bf6] to complete... +.....done. +[2025-11-30 15:01:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 +[2025-11-30 15:01:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 (Updated: 2024-07-27T07:20:10 [TS: 1722064810] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 (Updated: 2024-07-27T07:20:10) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/11ef9a6d-f165-4b6a-9922-2151438673e1] to complete... +.....done. +[2025-11-30 15:01:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 +[2025-11-30 15:01:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 (Updated: 2024-07-28T07:18:54 [TS: 1722151134] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 (Updated: 2024-07-28T07:18:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/384b1d85-e8e7-4046-8175-934d10c07ae0] to complete... +.....done. +[2025-11-30 15:01:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 +[2025-11-30 15:01:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 (Updated: 2024-07-29T07:17:00 [TS: 1722237420] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 (Updated: 2024-07-29T07:17:00) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/10d5b400-c0cf-46ca-81fb-e1a9799bff33] to complete... +.....done. +[2025-11-30 15:01:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 +[2025-11-30 15:01:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 (Updated: 2024-07-30T07:18:35 [TS: 1722323915] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:01:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 (Updated: 2024-07-30T07:18:35) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7ec300f0-24cd-4069-8c4c-5d10a5c69188] to complete... +.....done. +[2025-11-30 15:02:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 +[2025-11-30 15:02:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 (Updated: 2024-07-31T07:19:04 [TS: 1722410344] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 (Updated: 2024-07-31T07:19:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f8d86a90-aad2-4815-8e62-e291ca253f26] to complete... +.....done. +[2025-11-30 15:02:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 +[2025-11-30 15:02:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb (Updated: 2024-08-01T07:18:18 [TS: 1722496698] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb (Updated: 2024-08-01T07:18:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/43caa55c-bfd0-4f69-b78a-cff5a9f173b8] to complete... +......done. +[2025-11-30 15:02:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb +[2025-11-30 15:02:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 (Updated: 2024-08-02T07:18:39 [TS: 1722583119] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 (Updated: 2024-08-02T07:18:39) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0cc5a41c-aeec-497f-869a-d3c2eacb9c77] to complete... +.....done. +[2025-11-30 15:02:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 +[2025-11-30 15:02:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd (Updated: 2024-08-03T07:18:31 [TS: 1722669511] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd (Updated: 2024-08-03T07:18:31) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6586a7d0-5d6d-4a81-8450-39ac6933aeee] to complete... +.....done. +[2025-11-30 15:02:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd +[2025-11-30 15:02:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 (Updated: 2024-08-04T07:19:22 [TS: 1722755962] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 (Updated: 2024-08-04T07:19:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d6e4435f-2da2-40f2-95b0-0edf462bc84d] to complete... +.....done. +[2025-11-30 15:02:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 +[2025-11-30 15:02:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a (Updated: 2024-08-05T07:19:47 [TS: 1722842387] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a (Updated: 2024-08-05T07:19:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c0897b36-bff0-4e48-a8a8-c579403bc638] to complete... +......done. +[2025-11-30 15:02:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a +[2025-11-30 15:02:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e (Updated: 2024-08-06T07:18:59 [TS: 1722928739] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e (Updated: 2024-08-06T07:18:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c21af908-daaf-4e08-a6a0-9ce470ba8adf] to complete... +......done. +[2025-11-30 15:02:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e +[2025-11-30 15:02:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 (Updated: 2024-08-07T07:19:22 [TS: 1723015162] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 (Updated: 2024-08-07T07:19:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/30db16db-915a-497f-aab4-64d7543e1bb6] to complete... +......done. +[2025-11-30 15:02:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 +[2025-11-30 15:02:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 (Updated: 2024-08-08T07:18:44 [TS: 1723101524] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 (Updated: 2024-08-08T07:18:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4e8ff82e-f27d-4433-bbdd-378399d8b8bc] to complete... +......done. +[2025-11-30 15:02:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 +[2025-11-30 15:02:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 (Updated: 2024-08-09T07:19:24 [TS: 1723187964] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 (Updated: 2024-08-09T07:19:24) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7a0c4f06-3ba0-49c5-a870-ce582d6410de] to complete... +.....done. +[2025-11-30 15:02:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 +[2025-11-30 15:02:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a (Updated: 2024-08-10T07:18:12 [TS: 1723274292] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a (Updated: 2024-08-10T07:18:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/63ab4150-940d-42ea-961f-bd03d6bdff7f] to complete... +......done. +[2025-11-30 15:02:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a +[2025-11-30 15:02:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f (Updated: 2024-08-11T07:18:52 [TS: 1723360732] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f (Updated: 2024-08-11T07:18:52) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7a64fc06-22a9-475f-8330-791bf800ee75] to complete... +......done. +[2025-11-30 15:02:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f +[2025-11-30 15:02:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff (Updated: 2024-08-12T07:18:33 [TS: 1723447113] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff (Updated: 2024-08-12T07:18:33) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8ad78b33-39e5-4070-a5d7-809c4dbb9328] to complete... +.....done. +[2025-11-30 15:02:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff +[2025-11-30 15:02:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff (Updated: 2024-08-13T07:19:14 [TS: 1723533554] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff (Updated: 2024-08-13T07:19:14) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/75c535fb-8c98-4097-b953-8cfb50d1f24a] to complete... +......done. +[2025-11-30 15:02:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff +[2025-11-30 15:02:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a (Updated: 2024-08-14T07:20:03 [TS: 1723620003] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a (Updated: 2024-08-14T07:20:03) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bfe3661a-25d5-4ab8-8fe6-5c92b95449fb] to complete... +......done. +[2025-11-30 15:02:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a +[2025-11-30 15:02:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 (Updated: 2024-08-15T07:18:25 [TS: 1723706305] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:02:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 (Updated: 2024-08-15T07:18:25) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0f2bc4c7-6030-4138-9247-4721f4fd5f53] to complete... +.....done. +[2025-11-30 15:03:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 +[2025-11-30 15:03:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b (Updated: 2024-08-16T07:18:57 [TS: 1723792737] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:03:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b (Updated: 2024-08-16T07:18:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/06e7f6f6-8a4e-4dff-9609-c7cb5b17e3fe] to complete... +.....done. +[2025-11-30 15:03:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b +[2025-11-30 15:03:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b (Updated: 2024-08-17T07:19:21 [TS: 1723879161] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:03:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b (Updated: 2024-08-17T07:19:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b9a596d0-5655-4608-9c72-ee787bb2b756] to complete... +......done. +[2025-11-30 15:03:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b +[2025-11-30 15:03:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e (Updated: 2024-08-18T07:19:19 [TS: 1723965559] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:03:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e (Updated: 2024-08-18T07:19:19) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fb060761-f36f-4033-8e38-653eec90e4c5] to complete... +......done. +[2025-11-30 15:03:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e +[2025-11-30 15:03:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 (Updated: 2024-08-19T07:19:35 [TS: 1724051975] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:03:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 (Updated: 2024-08-19T07:19:35) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/60fe7d1f-8e4a-4219-b9e1-7edc3b604dd9] to complete... +......done. +[2025-11-30 15:03:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 +[2025-11-30 15:03:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 (Updated: 2024-08-20T07:19:04 [TS: 1724138344] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:03:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 (Updated: 2024-08-20T07:19:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/950d1860-c53c-428d-8742-830d14ec46a8] to complete... +.....done. +[2025-11-30 15:03:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 +[2025-11-30 15:03:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 (Updated: 2024-08-21T07:19:04 [TS: 1724224744] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:03:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 (Updated: 2024-08-21T07:19:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9afca90e-6d1a-4b2d-9501-41bb35689475] to complete... +.....done. +[2025-11-30 15:03:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 +[2025-11-30 15:03:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3b91af27e525f56f3ea343a5b051fc87a0f446e614700ba93b62d0eaebaa270 (Updated: 2024-08-22T07:18:34 [TS: 1724311114] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:03:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3b91af27e525f56f3ea343a5b051fc87a0f446e614700ba93b62d0eaebaa270 (Updated: 2024-08-22T07:18:34) +[2025-11-30 15:03:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:256f6245d97633997bd4c3cc2573f97f39b348629a527a5b3fb8e22b7b857d15 +[2025-11-30 15:03:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 (Updated: 2024-08-31T07:19:15 [TS: 1725088755] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:03:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 (Updated: 2024-08-31T07:19:15) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/734c08b3-3eb7-466c-b798-f2abc3e40093] to complete... +.....done. +[2025-11-30 15:04:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 +[2025-11-30 15:04:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 (Updated: 2024-09-01T07:18:35 [TS: 1725175115] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 (Updated: 2024-09-01T07:18:35) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9b63ff5b-6d50-486b-9410-559ccb5d4442] to complete... +......done. +[2025-11-30 15:04:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 +[2025-11-30 15:04:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 (Updated: 2024-09-02T07:19:08 [TS: 1725261548] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 (Updated: 2024-09-02T07:19:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c4033f09-ab33-40cd-b001-4b6ecc533c12] to complete... +.....done. +[2025-11-30 15:04:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 +[2025-11-30 15:04:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 (Updated: 2024-09-03T07:18:34 [TS: 1725347914] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 (Updated: 2024-09-03T07:18:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d0f1d9ff-de22-4ecb-a9ff-b651bdea0f3e] to complete... +.....done. +[2025-11-30 15:04:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 +[2025-11-30 15:04:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 (Updated: 2024-09-04T07:19:12 [TS: 1725434352] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 (Updated: 2024-09-04T07:19:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cc2caa33-7952-40a1-99aa-4069ee500e72] to complete... +......done. +[2025-11-30 15:04:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 +[2025-11-30 15:04:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd (Updated: 2024-09-04T15:00:21 [TS: 1725462021] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd (Updated: 2024-09-04T15:00:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd + +Tags: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner:test +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8b70d4d9-d4e2-4b31-9692-95f8a2001225] to complete... +......done. +[2025-11-30 15:04:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd +[2025-11-30 15:04:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 (Updated: 2024-09-05T07:19:39 [TS: 1725520779] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 (Updated: 2024-09-05T07:19:39) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/dc9682fe-5606-4b37-8b3b-1b842374e9cc] to complete... +......done. +[2025-11-30 15:04:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 +[2025-11-30 15:04:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 (Updated: 2024-09-06T07:19:00 [TS: 1725607140] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 (Updated: 2024-09-06T07:19:00) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/33b7771b-d4c0-4894-9781-af3b33c4dbf6] to complete... +......done. +[2025-11-30 15:04:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 +[2025-11-30 15:04:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 (Updated: 2024-09-07T07:20:06 [TS: 1725693606] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 (Updated: 2024-09-07T07:20:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/22525bf9-d25a-448b-add5-1e49908e3a53] to complete... +......done. +[2025-11-30 15:04:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 +[2025-11-30 15:04:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 (Updated: 2024-09-08T07:20:00 [TS: 1725780000] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 (Updated: 2024-09-08T07:20:00) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/348f2339-6895-4af7-8bbe-a1c0b9f1c7d3] to complete... +......done. +[2025-11-30 15:04:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 +[2025-11-30 15:04:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f (Updated: 2024-09-09T07:18:55 [TS: 1725866335] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f (Updated: 2024-09-09T07:18:55) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fe4435b9-2be0-4d26-b096-8e95902d1747] to complete... +.....done. +[2025-11-30 15:04:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f +[2025-11-30 15:04:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 (Updated: 2024-09-10T07:18:23 [TS: 1725952703] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 (Updated: 2024-09-10T07:18:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a869b618-95c7-47f1-8a02-21f14b99a29c] to complete... +......done. +[2025-11-30 15:04:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 +[2025-11-30 15:04:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 (Updated: 2024-09-11T07:18:27 [TS: 1726039107] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 (Updated: 2024-09-11T07:18:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ebcb23b0-25dd-4534-a2bf-4ab0a104c2d6] to complete... +......done. +[2025-11-30 15:04:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 +[2025-11-30 15:04:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b (Updated: 2024-09-12T07:19:28 [TS: 1726125568] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b (Updated: 2024-09-12T07:19:28) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/350ced73-0482-4755-93fa-8dd81b2f8158] to complete... +......done. +[2025-11-30 15:04:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b +[2025-11-30 15:04:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 (Updated: 2024-09-13T07:18:59 [TS: 1726211939] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 (Updated: 2024-09-13T07:18:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/46daa7e5-4637-46b7-8161-bd40ddc9ee73] to complete... +.....done. +[2025-11-30 15:04:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 +[2025-11-30 15:04:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 (Updated: 2024-09-14T07:21:55 [TS: 1726298515] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 (Updated: 2024-09-14T07:21:55) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/be551cdd-1642-47ee-828d-4e05d48c3a86] to complete... +......done. +[2025-11-30 15:04:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 +[2025-11-30 15:04:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 (Updated: 2024-09-15T07:18:46 [TS: 1726384726] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:04:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 (Updated: 2024-09-15T07:18:46) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b1e095bd-a455-419b-91e4-ec520392eb19] to complete... +......done. +[2025-11-30 15:05:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 +[2025-11-30 15:05:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 (Updated: 2024-09-16T07:19:43 [TS: 1726471183] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 (Updated: 2024-09-16T07:19:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8153c94c-fe6a-48e7-bd26-ed027f4f30a5] to complete... +......done. +[2025-11-30 15:05:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 +[2025-11-30 15:05:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 (Updated: 2024-09-17T07:18:21 [TS: 1726557501] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 (Updated: 2024-09-17T07:18:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0f68dbbb-7eb7-4e49-8f39-c91e72dbc51e] to complete... +......done. +[2025-11-30 15:05:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 +[2025-11-30 15:05:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af (Updated: 2024-09-18T07:18:46 [TS: 1726643926] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af (Updated: 2024-09-18T07:18:46) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bf6a3e87-d241-4c40-a2b6-de65f1f75dc8] to complete... +......done. +[2025-11-30 15:05:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af +[2025-11-30 15:05:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 (Updated: 2024-09-19T07:19:58 [TS: 1726730398] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 (Updated: 2024-09-19T07:19:58) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0b5f4818-9c32-49ea-91ea-ce0c2d1f4d67] to complete... +......done. +[2025-11-30 15:05:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 +[2025-11-30 15:05:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c (Updated: 2024-09-20T07:19:34 [TS: 1726816774] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c (Updated: 2024-09-20T07:19:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7adaecf8-a513-4c69-acd2-1a6c1c7a85a6] to complete... +......done. +[2025-11-30 15:05:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c +[2025-11-30 15:05:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 (Updated: 2024-09-21T07:18:42 [TS: 1726903122] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 (Updated: 2024-09-21T07:18:42) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/23d8ef73-80a8-4b44-88dd-d33552703102] to complete... +......done. +[2025-11-30 15:05:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 +[2025-11-30 15:05:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f (Updated: 2024-09-22T07:19:13 [TS: 1726989553] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f (Updated: 2024-09-22T07:19:13) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8c6974e5-5298-4d03-a125-d96f6aecbece] to complete... +.....done. +[2025-11-30 15:05:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f +[2025-11-30 15:05:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 (Updated: 2024-09-23T07:19:27 [TS: 1727075967] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 (Updated: 2024-09-23T07:19:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0578c740-b239-46c4-b7af-8e33399d0ccd] to complete... +.....done. +[2025-11-30 15:05:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 +[2025-11-30 15:05:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a (Updated: 2024-09-24T07:19:03 [TS: 1727162343] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a (Updated: 2024-09-24T07:19:03) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/416c0a72-39ba-47f4-8533-f16b7bbed847] to complete... +.....done. +[2025-11-30 15:05:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a +[2025-11-30 15:05:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 (Updated: 2024-09-25T07:19:26 [TS: 1727248766] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 (Updated: 2024-09-25T07:19:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9cf6e646-0df7-41e4-b0d6-adf0332b6383] to complete... +......done. +[2025-11-30 15:05:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 +[2025-11-30 15:05:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 (Updated: 2024-09-26T07:18:34 [TS: 1727335114] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 (Updated: 2024-09-26T07:18:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b895d6a2-77e0-4f1d-afaf-e901dc15cf5f] to complete... +......done. +[2025-11-30 15:05:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 +[2025-11-30 15:05:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 (Updated: 2024-09-27T07:18:51 [TS: 1727421531] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 (Updated: 2024-09-27T07:18:51) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fd999bbe-bfbf-4fe9-bdaa-603e77e64416] to complete... +......done. +[2025-11-30 15:05:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 +[2025-11-30 15:05:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 (Updated: 2024-09-28T07:19:08 [TS: 1727507948] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 (Updated: 2024-09-28T07:19:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/25c4c2f6-9b35-42c7-9e6e-8496c4250948] to complete... +......done. +[2025-11-30 15:05:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 +[2025-11-30 15:05:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b (Updated: 2024-09-29T07:19:59 [TS: 1727594399] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b (Updated: 2024-09-29T07:19:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ac70b5e7-42ec-43ca-9914-ef9545e43871] to complete... +......done. +[2025-11-30 15:05:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b +[2025-11-30 15:05:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 (Updated: 2024-09-30T07:18:47 [TS: 1727680727] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:05:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 (Updated: 2024-09-30T07:18:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bc5e27f5-4c04-4dc6-8d9f-b17c77a25130] to complete... +......done. +[2025-11-30 15:06:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 +[2025-11-30 15:06:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 (Updated: 2024-10-01T07:18:16 [TS: 1727767096] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 (Updated: 2024-10-01T07:18:16) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4285abe6-3421-477f-85e4-351c69bd5d11] to complete... +......done. +[2025-11-30 15:06:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 +[2025-11-30 15:06:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 (Updated: 2024-10-02T07:19:22 [TS: 1727853562] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 (Updated: 2024-10-02T07:19:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8f297a13-1fe9-4430-bde6-320b14ef9c69] to complete... +......done. +[2025-11-30 15:06:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 +[2025-11-30 15:06:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d (Updated: 2024-10-03T07:19:40 [TS: 1727939980] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d (Updated: 2024-10-03T07:19:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/06953728-a6b9-4ef7-9784-4d439aebb515] to complete... +......done. +[2025-11-30 15:06:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d +[2025-11-30 15:06:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c (Updated: 2024-10-04T07:20:00 [TS: 1728026400] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c (Updated: 2024-10-04T07:20:00) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7a74c22f-f6dc-44b4-9292-6d863e87e1b0] to complete... +.....done. +[2025-11-30 15:06:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c +[2025-11-30 15:06:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 (Updated: 2024-10-05T07:20:18 [TS: 1728112818] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 (Updated: 2024-10-05T07:20:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8099fc49-39f3-48e2-b54e-860e6544c1ad] to complete... +.....done. +[2025-11-30 15:06:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 +[2025-11-30 15:06:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d (Updated: 2024-10-06T07:19:45 [TS: 1728199185] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d (Updated: 2024-10-06T07:19:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/280d8a74-209d-4233-9329-02ea48f1216a] to complete... +......done. +[2025-11-30 15:06:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d +[2025-11-30 15:06:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 (Updated: 2024-10-07T07:20:12 [TS: 1728285612] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 (Updated: 2024-10-07T07:20:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6487728b-299a-4ea6-9f78-cce63d6c6000] to complete... +......done. +[2025-11-30 15:06:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 +[2025-11-30 15:06:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 (Updated: 2024-10-08T07:19:06 [TS: 1728371946] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 (Updated: 2024-10-08T07:19:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c2163135-6630-4a48-b988-d5f7baca107c] to complete... +......done. +[2025-11-30 15:06:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 +[2025-11-30 15:06:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 (Updated: 2024-10-09T07:19:20 [TS: 1728458360] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 (Updated: 2024-10-09T07:19:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f6f4eb44-712c-414f-8ee0-f304bc03e78f] to complete... +......done. +[2025-11-30 15:06:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 +[2025-11-30 15:06:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e (Updated: 2024-10-10T07:19:58 [TS: 1728544798] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e (Updated: 2024-10-10T07:19:58) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4b684cf0-cb2c-41ff-90b8-d98b32098c61] to complete... +......done. +[2025-11-30 15:06:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e +[2025-11-30 15:06:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf (Updated: 2024-10-11T07:19:31 [TS: 1728631171] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf (Updated: 2024-10-11T07:19:31) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d25e04d1-8231-42dc-b502-8dccb1c76093] to complete... +......done. +[2025-11-30 15:06:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf +[2025-11-30 15:06:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 (Updated: 2024-10-12T07:19:51 [TS: 1728717591] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 (Updated: 2024-10-12T07:19:51) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d34c18a6-5aab-4631-b690-1c265f3353b7] to complete... +......done. +[2025-11-30 15:06:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 +[2025-11-30 15:06:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 (Updated: 2024-10-13T07:20:18 [TS: 1728804018] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 (Updated: 2024-10-13T07:20:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/313c36a8-d2a2-4e4e-bb1b-3ffa8e83c52e] to complete... +......done. +[2025-11-30 15:06:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 +[2025-11-30 15:06:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 (Updated: 2024-10-14T07:18:56 [TS: 1728890336] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 (Updated: 2024-10-14T07:18:56) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3ccecd5a-5289-446f-81ee-bb8f956cf425] to complete... +......done. +[2025-11-30 15:06:59] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 +[2025-11-30 15:06:59] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 (Updated: 2024-10-15T07:18:42 [TS: 1728976722] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:06:59] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 (Updated: 2024-10-15T07:18:42) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/78be52f7-724a-414f-8cea-df672f17544d] to complete... +.....done. +[2025-11-30 15:07:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 +[2025-11-30 15:07:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 (Updated: 2024-10-16T07:18:53 [TS: 1729063133] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:07:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 (Updated: 2024-10-16T07:18:53) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b21da8ff-d534-43df-a503-926aa026fc41] to complete... +.....done. +[2025-11-30 15:07:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 +[2025-11-30 15:07:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 (Updated: 2024-10-17T07:19:38 [TS: 1729149578] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:07:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 (Updated: 2024-10-17T07:19:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/724f27ab-0add-42e9-9ce5-d154fe885be8] to complete... +......done. +[2025-11-30 15:07:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 +[2025-11-30 15:07:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 (Updated: 2024-10-18T07:21:27 [TS: 1729236087] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:07:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 (Updated: 2024-10-18T07:21:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/039a7dfa-8125-4176-90de-6b5b4f91a9d9] to complete... +......done. +[2025-11-30 15:07:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 +[2025-11-30 15:07:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 (Updated: 2024-10-19T07:21:08 [TS: 1729322468] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:07:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 (Updated: 2024-10-19T07:21:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d7357cb4-57cf-4433-8b94-ae599dfec4ee] to complete... +......done. +[2025-11-30 15:07:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 +[2025-11-30 15:07:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 (Updated: 2024-10-20T07:20:13 [TS: 1729408813] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:07:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 (Updated: 2024-10-20T07:20:13) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4c0b6f9b-18db-4b22-93aa-7556feb36b52] to complete... +.....done. +[2025-11-30 15:07:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 +[2025-11-30 15:07:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 (Updated: 2024-10-21T07:19:27 [TS: 1729495167] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:07:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 (Updated: 2024-10-21T07:19:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b86cf1b3-407a-43ee-9797-79132a3565e4] to complete... +......done. +[2025-11-30 15:07:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 +[2025-11-30 15:07:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 (Updated: 2024-10-22T07:19:36 [TS: 1729581576] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:07:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 (Updated: 2024-10-22T07:19:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3db0b93f-b2a7-4e36-b9bb-f793d3d0fbdf] to complete... +......done. +[2025-11-30 15:07:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 +[2025-11-30 15:07:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 (Updated: 2024-10-23T07:18:47 [TS: 1729667927] < Cutoff: [TS: 1763304912]) +[2025-11-30 15:07:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 (Updated: 2024-10-23T07:18:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ed80c11c-a015-4dc2-9251-fafe5894b861] to complete... +.....done. +[2025-11-30 15:07:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 +[2025-11-30 15:07:34] [INFO] Hit delete limit (200) for Docker Images. +[2025-11-30 15:07:34] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 15:07:34] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 15:07:37] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 15:07:37] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 15:07:37] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 15:07:37] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 15:07:37] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 15:07:37] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 15:07:39] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 15:07:39] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:07:42] [INFO] No Regional Address found matching criteria. +[2025-11-30 15:07:42] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:07:44] [INFO] No Global Address found matching criteria. +[2025-11-30 15:07:44] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- +[2025-11-30 15:07:48] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 15:07:51] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:07:51] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:07:51] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 15:07:51] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 15:07:51] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 15:07:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:54] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 15:07:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:07:56] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 15:07:58] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 15:07:58] [INFO] CLEANUP RUN FINISHED +[2025-11-30 15:08:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 15:08:29] [INFO] Time Cutoff (General): 2025-11-30T15:08:29+0000 +[2025-11-30 15:08:29] [INFO] Time Cutoff (Images): 2025-10-01T15:08:29+0000 +[2025-11-30 15:08:29] [INFO] Delete Limit per Type: 200 +[2025-11-30 15:08:29] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 15:08:29] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 15:08:32] [INFO] No Service Accounts found matching prefix. +[2025-11-30 15:08:32] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:08:34] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 15:08:34] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 15:08:37] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:08:37] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:08:37] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 15:08:37] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:08:40] [INFO] No Filestore instances found matching criteria. +[2025-11-30 15:08:40] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 15:08:43] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 15:08:43] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 15:08:43] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 15:08:43] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 15:08:43] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 15:08:43] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 15:08:43] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 15:08:43] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 15:08:44] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 15:08:44] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 15:08:44] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 15:08:44] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:08:44Z (Unix: 1763305724) +[2025-11-30 15:08:44] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 (Updated: 2024-10-24T07:21:12 [TS: 1729754472] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 (Updated: 2024-10-24T07:21:12) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 (Updated: 2024-10-25T07:19:11 [TS: 1729840751] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 (Updated: 2024-10-25T07:19:11) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea (Updated: 2024-10-26T07:19:04 [TS: 1729927144] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea (Updated: 2024-10-26T07:19:04) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 (Updated: 2024-10-27T07:18:59 [TS: 1730013539] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 (Updated: 2024-10-27T07:18:59) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f (Updated: 2024-10-28T07:19:53 [TS: 1730099993] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f (Updated: 2024-10-28T07:19:53) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa (Updated: 2024-10-29T07:21:32 [TS: 1730186492] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa (Updated: 2024-10-29T07:21:32) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c (Updated: 2024-10-30T07:20:09 [TS: 1730272809] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c (Updated: 2024-10-30T07:20:09) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca (Updated: 2024-10-31T07:19:50 [TS: 1730359190] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca (Updated: 2024-10-31T07:19:50) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb (Updated: 2024-11-01T07:19:35 [TS: 1730445575] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb (Updated: 2024-11-01T07:19:35) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 (Updated: 2024-11-02T07:19:20 [TS: 1730531960] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 (Updated: 2024-11-02T07:19:20) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b (Updated: 2024-11-03T07:19:34 [TS: 1730618374] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b (Updated: 2024-11-03T07:19:34) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc (Updated: 2024-11-04T08:20:03 [TS: 1730708403] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc (Updated: 2024-11-04T08:20:03) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed (Updated: 2024-11-05T08:19:36 [TS: 1730794776] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed (Updated: 2024-11-05T08:19:36) +[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e (Updated: 2024-11-06T08:19:42 [TS: 1730881182] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e (Updated: 2024-11-06T08:19:42) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 (Updated: 2024-11-07T08:22:39 [TS: 1730967759] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 (Updated: 2024-11-07T08:22:39) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 (Updated: 2024-11-08T08:19:05 [TS: 1731053945] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 (Updated: 2024-11-08T08:19:05) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 (Updated: 2024-11-09T08:19:52 [TS: 1731140392] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 (Updated: 2024-11-09T08:19:52) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 (Updated: 2024-11-10T08:19:21 [TS: 1731226761] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 (Updated: 2024-11-10T08:19:21) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e (Updated: 2024-11-11T08:18:44 [TS: 1731313124] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e (Updated: 2024-11-11T08:18:44) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c (Updated: 2024-11-12T08:20:12 [TS: 1731399612] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c (Updated: 2024-11-12T08:20:12) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d (Updated: 2024-11-13T08:18:46 [TS: 1731485926] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d (Updated: 2024-11-13T08:18:46) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 (Updated: 2024-11-14T08:19:32 [TS: 1731572372] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 (Updated: 2024-11-14T08:19:32) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa (Updated: 2024-11-15T08:18:37 [TS: 1731658717] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa (Updated: 2024-11-15T08:18:37) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 (Updated: 2024-11-16T08:19:38 [TS: 1731745178] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 (Updated: 2024-11-16T08:19:38) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 (Updated: 2024-11-17T08:19:08 [TS: 1731831548] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 (Updated: 2024-11-17T08:19:08) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c (Updated: 2024-11-18T08:19:40 [TS: 1731917980] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c (Updated: 2024-11-18T08:19:40) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e (Updated: 2024-11-19T08:19:29 [TS: 1732004369] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e (Updated: 2024-11-19T08:19:29) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 (Updated: 2024-11-20T08:19:24 [TS: 1732090764] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 (Updated: 2024-11-20T08:19:24) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf (Updated: 2024-11-21T08:19:48 [TS: 1732177188] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf (Updated: 2024-11-21T08:19:48) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 (Updated: 2024-11-22T08:19:50 [TS: 1732263590] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 (Updated: 2024-11-22T08:19:50) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb (Updated: 2024-11-23T08:18:52 [TS: 1732349932] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb (Updated: 2024-11-23T08:18:52) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 (Updated: 2024-11-24T08:19:55 [TS: 1732436395] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 (Updated: 2024-11-24T08:19:55) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 (Updated: 2024-11-25T08:18:26 [TS: 1732522706] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 (Updated: 2024-11-25T08:18:26) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e (Updated: 2024-11-26T08:19:19 [TS: 1732609159] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e (Updated: 2024-11-26T08:19:19) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f (Updated: 2024-11-27T08:19:05 [TS: 1732695545] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f (Updated: 2024-11-27T08:19:05) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af (Updated: 2024-11-28T08:18:54 [TS: 1732781934] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af (Updated: 2024-11-28T08:18:54) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 (Updated: 2024-11-29T08:19:25 [TS: 1732868365] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 (Updated: 2024-11-29T08:19:25) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db (Updated: 2024-11-30T08:19:26 [TS: 1732954766] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db (Updated: 2024-11-30T08:19:26) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b (Updated: 2024-12-01T08:20:11 [TS: 1733041211] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b (Updated: 2024-12-01T08:20:11) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c (Updated: 2024-12-02T08:18:21 [TS: 1733127501] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c (Updated: 2024-12-02T08:18:21) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff (Updated: 2024-12-03T08:19:17 [TS: 1733213957] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff (Updated: 2024-12-03T08:19:17) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 (Updated: 2024-12-04T08:20:29 [TS: 1733300429] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 (Updated: 2024-12-04T08:20:29) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 (Updated: 2024-12-04T23:35:41 [TS: 1733355341] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 (Updated: 2024-12-04T23:35:41) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e (Updated: 2024-12-05T08:19:43 [TS: 1733386783] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e (Updated: 2024-12-05T08:19:43) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 (Updated: 2024-12-06T08:19:51 [TS: 1733473191] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 (Updated: 2024-12-06T08:19:51) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da (Updated: 2024-12-07T08:19:02 [TS: 1733559542] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da (Updated: 2024-12-07T08:19:02) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d (Updated: 2024-12-08T08:18:49 [TS: 1733645929] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d (Updated: 2024-12-08T08:18:49) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e (Updated: 2024-12-09T08:18:56 [TS: 1733732336] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e (Updated: 2024-12-09T08:18:56) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f (Updated: 2024-12-10T08:20:41 [TS: 1733818841] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f (Updated: 2024-12-10T08:20:41) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba (Updated: 2024-12-11T08:19:22 [TS: 1733905162] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba (Updated: 2024-12-11T08:19:22) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 (Updated: 2024-12-12T08:19:21 [TS: 1733991561] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 (Updated: 2024-12-12T08:19:21) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb (Updated: 2024-12-13T08:19:59 [TS: 1734077999] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb (Updated: 2024-12-13T08:19:59) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 (Updated: 2024-12-14T08:19:12 [TS: 1734164352] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 (Updated: 2024-12-14T08:19:12) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b (Updated: 2024-12-15T08:19:07 [TS: 1734250747] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b (Updated: 2024-12-15T08:19:07) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 (Updated: 2024-12-16T08:18:40 [TS: 1734337120] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 (Updated: 2024-12-16T08:18:40) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 (Updated: 2024-12-17T08:19:36 [TS: 1734423576] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 (Updated: 2024-12-17T08:19:36) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 (Updated: 2024-12-18T08:19:28 [TS: 1734509968] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 (Updated: 2024-12-18T08:19:28) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 (Updated: 2024-12-19T08:19:45 [TS: 1734596385] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 (Updated: 2024-12-19T08:19:45) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 (Updated: 2024-12-20T08:20:22 [TS: 1734682822] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 (Updated: 2024-12-20T08:20:22) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b (Updated: 2024-12-21T08:18:43 [TS: 1734769123] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b (Updated: 2024-12-21T08:18:43) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 (Updated: 2024-12-22T08:19:03 [TS: 1734855543] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 (Updated: 2024-12-22T08:19:03) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 (Updated: 2024-12-23T08:19:14 [TS: 1734941954] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 (Updated: 2024-12-23T08:19:14) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 (Updated: 2024-12-24T08:19:47 [TS: 1735028387] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 (Updated: 2024-12-24T08:19:47) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c (Updated: 2024-12-25T08:19:25 [TS: 1735114765] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c (Updated: 2024-12-25T08:19:25) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d (Updated: 2024-12-26T08:19:08 [TS: 1735201148] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d (Updated: 2024-12-26T08:19:08) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b (Updated: 2024-12-27T08:19:18 [TS: 1735287558] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b (Updated: 2024-12-27T08:19:18) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 (Updated: 2024-12-28T08:20:48 [TS: 1735374048] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 (Updated: 2024-12-28T08:20:48) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 (Updated: 2024-12-29T08:19:41 [TS: 1735460381] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 (Updated: 2024-12-29T08:19:41) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a (Updated: 2024-12-30T08:19:44 [TS: 1735546784] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a (Updated: 2024-12-30T08:19:44) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 (Updated: 2024-12-31T08:19:20 [TS: 1735633160] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 (Updated: 2024-12-31T08:19:20) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a (Updated: 2025-01-01T08:20:09 [TS: 1735719609] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a (Updated: 2025-01-01T08:20:09) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 (Updated: 2025-01-02T08:19:36 [TS: 1735805976] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 (Updated: 2025-01-02T08:19:36) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 (Updated: 2025-01-03T08:19:51 [TS: 1735892391] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 (Updated: 2025-01-03T08:19:51) +[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee (Updated: 2025-01-04T08:19:50 [TS: 1735978790] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee (Updated: 2025-01-04T08:19:50) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 (Updated: 2025-01-05T08:19:40 [TS: 1736065180] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 (Updated: 2025-01-05T08:19:40) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 (Updated: 2025-01-06T08:19:23 [TS: 1736151563] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 (Updated: 2025-01-06T08:19:23) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 (Updated: 2025-01-07T08:19:20 [TS: 1736237960] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 (Updated: 2025-01-07T08:19:20) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 (Updated: 2025-01-08T08:19:34 [TS: 1736324374] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 (Updated: 2025-01-08T08:19:34) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 (Updated: 2025-01-09T08:19:09 [TS: 1736410749] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 (Updated: 2025-01-09T08:19:09) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf (Updated: 2025-01-10T08:18:36 [TS: 1736497116] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf (Updated: 2025-01-10T08:18:36) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe (Updated: 2025-01-11T08:19:12 [TS: 1736583552] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe (Updated: 2025-01-11T08:19:12) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 (Updated: 2025-01-12T08:19:47 [TS: 1736669987] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 (Updated: 2025-01-12T08:19:47) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 (Updated: 2025-01-13T08:19:33 [TS: 1736756373] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 (Updated: 2025-01-13T08:19:33) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 (Updated: 2025-01-15T08:19:08 [TS: 1736929148] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 (Updated: 2025-01-15T08:19:08) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e (Updated: 2025-01-16T08:19:33 [TS: 1737015573] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e (Updated: 2025-01-16T08:19:33) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 (Updated: 2025-01-17T08:22:38 [TS: 1737102158] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 (Updated: 2025-01-17T08:22:38) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f (Updated: 2025-01-18T08:20:38 [TS: 1737188438] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f (Updated: 2025-01-18T08:20:38) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 (Updated: 2025-01-19T08:23:33 [TS: 1737275013] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 (Updated: 2025-01-19T08:23:33) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c (Updated: 2025-01-20T08:20:58 [TS: 1737361258] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c (Updated: 2025-01-20T08:20:58) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 (Updated: 2025-01-21T08:21:03 [TS: 1737447663] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 (Updated: 2025-01-21T08:21:03) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a (Updated: 2025-01-22T08:20:37 [TS: 1737534037] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a (Updated: 2025-01-22T08:20:37) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 (Updated: 2025-01-23T08:20:21 [TS: 1737620421] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 (Updated: 2025-01-23T08:20:21) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 (Updated: 2025-01-24T08:19:44 [TS: 1737706784] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 (Updated: 2025-01-24T08:19:44) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 (Updated: 2025-01-25T08:20:25 [TS: 1737793225] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 (Updated: 2025-01-25T08:20:25) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c (Updated: 2025-01-26T08:19:43 [TS: 1737879583] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c (Updated: 2025-01-26T08:19:43) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d (Updated: 2025-01-27T08:21:26 [TS: 1737966086] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d (Updated: 2025-01-27T08:21:26) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e (Updated: 2025-01-28T08:22:47 [TS: 1738052567] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e (Updated: 2025-01-28T08:22:47) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 (Updated: 2025-01-29T08:20:27 [TS: 1738138827] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 (Updated: 2025-01-29T08:20:27) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 (Updated: 2025-01-30T08:20:39 [TS: 1738225239] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 (Updated: 2025-01-30T08:20:39) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 (Updated: 2025-01-31T08:20:22 [TS: 1738311622] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 (Updated: 2025-01-31T08:20:22) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 (Updated: 2025-02-01T08:20:50 [TS: 1738398050] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 (Updated: 2025-02-01T08:20:50) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 (Updated: 2025-02-02T08:20:43 [TS: 1738484443] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 (Updated: 2025-02-02T08:20:43) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 (Updated: 2025-02-03T08:20:14 [TS: 1738570814] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 (Updated: 2025-02-03T08:20:14) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae (Updated: 2025-02-04T08:20:52 [TS: 1738657252] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae (Updated: 2025-02-04T08:20:52) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b (Updated: 2025-02-05T08:20:48 [TS: 1738743648] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b (Updated: 2025-02-05T08:20:48) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d (Updated: 2025-02-06T08:20:11 [TS: 1738830011] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d (Updated: 2025-02-06T08:20:11) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c (Updated: 2025-02-07T08:21:04 [TS: 1738916464] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c (Updated: 2025-02-07T08:21:04) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef (Updated: 2025-02-08T08:20:54 [TS: 1739002854] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef (Updated: 2025-02-08T08:20:54) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 (Updated: 2025-02-09T08:20:35 [TS: 1739089235] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 (Updated: 2025-02-09T08:20:35) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 (Updated: 2025-02-10T08:20:55 [TS: 1739175655] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 (Updated: 2025-02-10T08:20:55) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af (Updated: 2025-02-11T08:19:45 [TS: 1739261985] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af (Updated: 2025-02-11T08:19:45) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 (Updated: 2025-02-12T08:23:12 [TS: 1739348592] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 (Updated: 2025-02-12T08:23:12) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 (Updated: 2025-02-13T08:21:40 [TS: 1739434900] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 (Updated: 2025-02-13T08:21:40) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 (Updated: 2025-02-14T08:20:29 [TS: 1739521229] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 (Updated: 2025-02-14T08:20:29) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 (Updated: 2025-02-15T08:20:08 [TS: 1739607608] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 (Updated: 2025-02-15T08:20:08) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f (Updated: 2025-02-16T08:20:40 [TS: 1739694040] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f (Updated: 2025-02-16T08:20:40) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 (Updated: 2025-02-17T08:24:07 [TS: 1739780647] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 (Updated: 2025-02-17T08:24:07) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 (Updated: 2025-02-18T08:17:40 [TS: 1739866660] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 (Updated: 2025-02-18T08:17:40) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 (Updated: 2025-02-19T08:21:02 [TS: 1739953262] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 (Updated: 2025-02-19T08:21:02) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c (Updated: 2025-02-20T08:18:01 [TS: 1740039481] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c (Updated: 2025-02-20T08:18:01) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 (Updated: 2025-02-21T08:20:07 [TS: 1740126007] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 (Updated: 2025-02-21T08:20:07) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 (Updated: 2025-02-25T08:20:52 [TS: 1740471652] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 (Updated: 2025-02-25T08:20:52) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 (Updated: 2025-02-26T08:21:06 [TS: 1740558066] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 (Updated: 2025-02-26T08:21:06) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 (Updated: 2025-02-27T08:21:06 [TS: 1740644466] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 (Updated: 2025-02-27T08:21:06) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 (Updated: 2025-02-28T08:21:16 [TS: 1740730876] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 (Updated: 2025-02-28T08:21:16) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c (Updated: 2025-03-01T08:20:02 [TS: 1740817202] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c (Updated: 2025-03-01T08:20:02) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 (Updated: 2025-03-02T08:21:22 [TS: 1740903682] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 (Updated: 2025-03-02T08:21:22) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 (Updated: 2025-03-03T08:17:41 [TS: 1740989861] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 (Updated: 2025-03-03T08:17:41) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b (Updated: 2025-03-04T08:22:52 [TS: 1741076572] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b (Updated: 2025-03-04T08:22:52) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 (Updated: 2025-03-05T08:21:41 [TS: 1741162901] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 (Updated: 2025-03-05T08:21:41) +[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 (Updated: 2025-03-06T08:21:32 [TS: 1741249292] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 (Updated: 2025-03-06T08:21:32) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc (Updated: 2025-03-07T08:19:43 [TS: 1741335583] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc (Updated: 2025-03-07T08:19:43) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c (Updated: 2025-03-08T08:19:23 [TS: 1741421963] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c (Updated: 2025-03-08T08:19:23) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a (Updated: 2025-03-09T08:18:08 [TS: 1741508288] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a (Updated: 2025-03-09T08:18:08) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b (Updated: 2025-03-10T07:21:20 [TS: 1741591280] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b (Updated: 2025-03-10T07:21:20) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e (Updated: 2025-03-11T07:20:43 [TS: 1741677643] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e (Updated: 2025-03-11T07:20:43) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e (Updated: 2025-03-12T07:19:34 [TS: 1741763974] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e (Updated: 2025-03-12T07:19:34) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 (Updated: 2025-03-13T07:20:51 [TS: 1741850451] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 (Updated: 2025-03-13T07:20:51) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 (Updated: 2025-03-14T07:20:37 [TS: 1741936837] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 (Updated: 2025-03-14T07:20:37) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 (Updated: 2025-03-15T07:21:23 [TS: 1742023283] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 (Updated: 2025-03-15T07:21:23) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 (Updated: 2025-03-16T07:20:44 [TS: 1742109644] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 (Updated: 2025-03-16T07:20:44) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 (Updated: 2025-03-17T07:21:09 [TS: 1742196069] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 (Updated: 2025-03-17T07:21:09) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 (Updated: 2025-03-18T07:22:07 [TS: 1742282527] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 (Updated: 2025-03-18T07:22:07) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac (Updated: 2025-03-19T07:21:36 [TS: 1742368896] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac (Updated: 2025-03-19T07:21:36) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a (Updated: 2025-03-20T07:21:15 [TS: 1742455275] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a (Updated: 2025-03-20T07:21:15) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 (Updated: 2025-03-21T07:20:54 [TS: 1742541654] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 (Updated: 2025-03-21T07:20:54) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 (Updated: 2025-03-22T07:21:14 [TS: 1742628074] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 (Updated: 2025-03-22T07:21:14) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f (Updated: 2025-03-23T07:20:50 [TS: 1742714450] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f (Updated: 2025-03-23T07:20:50) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca (Updated: 2025-03-24T07:21:04 [TS: 1742800864] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca (Updated: 2025-03-24T07:21:04) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f (Updated: 2025-03-25T07:21:17 [TS: 1742887277] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f (Updated: 2025-03-25T07:21:17) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 (Updated: 2025-03-26T07:20:59 [TS: 1742973659] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 (Updated: 2025-03-26T07:20:59) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a (Updated: 2025-03-27T07:20:48 [TS: 1743060048] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a (Updated: 2025-03-27T07:20:48) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 (Updated: 2025-03-28T07:20:36 [TS: 1743146436] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 (Updated: 2025-03-28T07:20:36) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 (Updated: 2025-03-29T07:21:17 [TS: 1743232877] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 (Updated: 2025-03-29T07:21:17) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 (Updated: 2025-03-30T07:21:38 [TS: 1743319298] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 (Updated: 2025-03-30T07:21:38) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 (Updated: 2025-03-31T07:20:18 [TS: 1743405618] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 (Updated: 2025-03-31T07:20:18) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 (Updated: 2025-04-01T07:21:20 [TS: 1743492080] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 (Updated: 2025-04-01T07:21:20) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d (Updated: 2025-04-02T07:20:23 [TS: 1743578423] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d (Updated: 2025-04-02T07:20:23) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 (Updated: 2025-04-03T07:20:49 [TS: 1743664849] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 (Updated: 2025-04-03T07:20:49) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b (Updated: 2025-04-04T07:20:41 [TS: 1743751241] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b (Updated: 2025-04-04T07:20:41) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a (Updated: 2025-04-05T07:20:21 [TS: 1743837621] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a (Updated: 2025-04-05T07:20:21) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 (Updated: 2025-04-06T07:23:45 [TS: 1743924225] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 (Updated: 2025-04-06T07:23:45) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 (Updated: 2025-04-07T07:20:24 [TS: 1744010424] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 (Updated: 2025-04-07T07:20:24) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee (Updated: 2025-04-08T07:22:43 [TS: 1744096963] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee (Updated: 2025-04-08T07:22:43) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 (Updated: 2025-04-09T07:20:20 [TS: 1744183220] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 (Updated: 2025-04-09T07:20:20) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 (Updated: 2025-04-10T07:21:00 [TS: 1744269660] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 (Updated: 2025-04-10T07:21:00) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 (Updated: 2025-04-11T07:20:54 [TS: 1744356054] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 (Updated: 2025-04-11T07:20:54) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 (Updated: 2025-04-12T07:21:40 [TS: 1744442500] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 (Updated: 2025-04-12T07:21:40) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 (Updated: 2025-04-13T07:20:59 [TS: 1744528859] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 (Updated: 2025-04-13T07:20:59) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c (Updated: 2025-04-14T07:20:24 [TS: 1744615224] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c (Updated: 2025-04-14T07:20:24) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 (Updated: 2025-04-15T07:20:46 [TS: 1744701646] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 (Updated: 2025-04-15T07:20:46) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 (Updated: 2025-04-16T07:20:12 [TS: 1744788012] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 (Updated: 2025-04-16T07:20:12) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 (Updated: 2025-04-17T07:19:55 [TS: 1744874395] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 (Updated: 2025-04-17T07:19:55) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb (Updated: 2025-04-18T07:20:41 [TS: 1744960841] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb (Updated: 2025-04-18T07:20:41) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa (Updated: 2025-04-19T07:21:15 [TS: 1745047275] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa (Updated: 2025-04-19T07:21:15) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 (Updated: 2025-04-20T07:21:25 [TS: 1745133685] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 (Updated: 2025-04-20T07:21:25) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d (Updated: 2025-04-21T07:21:59 [TS: 1745220119] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d (Updated: 2025-04-21T07:21:59) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a (Updated: 2025-04-22T07:21:34 [TS: 1745306494] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a (Updated: 2025-04-22T07:21:34) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 (Updated: 2025-04-23T07:21:06 [TS: 1745392866] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 (Updated: 2025-04-23T07:21:06) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 (Updated: 2025-04-24T07:21:59 [TS: 1745479319] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 (Updated: 2025-04-24T07:21:59) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 (Updated: 2025-04-25T07:20:22 [TS: 1745565622] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 (Updated: 2025-04-25T07:20:22) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d (Updated: 2025-04-26T07:20:43 [TS: 1745652043] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d (Updated: 2025-04-26T07:20:43) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da (Updated: 2025-04-27T07:21:38 [TS: 1745738498] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da (Updated: 2025-04-27T07:21:38) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 (Updated: 2025-04-28T07:21:01 [TS: 1745824861] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 (Updated: 2025-04-28T07:21:01) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 (Updated: 2025-04-29T07:21:25 [TS: 1745911285] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 (Updated: 2025-04-29T07:21:25) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 (Updated: 2025-04-30T07:20:20 [TS: 1745997620] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 (Updated: 2025-04-30T07:20:20) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 (Updated: 2025-05-01T07:21:24 [TS: 1746084084] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 (Updated: 2025-05-01T07:21:24) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 (Updated: 2025-05-02T07:21:04 [TS: 1746170464] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 (Updated: 2025-05-02T07:21:04) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 (Updated: 2025-05-03T07:21:59 [TS: 1746256919] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 (Updated: 2025-05-03T07:21:59) +[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 (Updated: 2025-05-04T07:21:07 [TS: 1746343267] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 (Updated: 2025-05-04T07:21:07) +[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 (Updated: 2025-05-05T07:21:39 [TS: 1746429699] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 (Updated: 2025-05-05T07:21:39) +[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 (Updated: 2025-05-06T07:21:39 [TS: 1746516099] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 (Updated: 2025-05-06T07:21:39) +[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb (Updated: 2025-05-07T07:22:18 [TS: 1746602538] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb (Updated: 2025-05-07T07:22:18) +[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d (Updated: 2025-05-08T07:22:24 [TS: 1746688944] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d (Updated: 2025-05-08T07:22:24) +[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b (Updated: 2025-05-09T07:21:16 [TS: 1746775276] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b (Updated: 2025-05-09T07:21:16) +[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 (Updated: 2025-05-10T07:21:20 [TS: 1746861680] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 (Updated: 2025-05-10T07:21:20) +[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 (Updated: 2025-05-11T07:20:26 [TS: 1746948026] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 (Updated: 2025-05-11T07:20:26) +[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 (Updated: 2025-05-12T07:21:06 [TS: 1747034466] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 (Updated: 2025-05-12T07:21:06) +[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 (Updated: 2025-05-13T07:21:27 [TS: 1747120887] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 (Updated: 2025-05-13T07:21:27) +[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 (Updated: 2025-05-14T07:21:24 [TS: 1747207284] < Cutoff: [TS: 1763305724]) +[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 (Updated: 2025-05-14T07:21:24) +[2025-11-30 15:08:54] [INFO] Hit delete limit (200) for Docker Images. +[2025-11-30 15:08:54] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 15:08:54] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 15:08:56] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 15:08:56] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 15:08:56] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 15:08:56] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 15:08:56] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 15:08:56] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 15:08:59] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 15:08:59] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:09:01] [INFO] No Regional Address found matching criteria. +[2025-11-30 15:09:01] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:09:03] [INFO] No Global Address found matching criteria. +[2025-11-30 15:09:03] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- +[2025-11-30 15:09:08] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 15:09:10] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:09:10] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:09:10] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 15:09:10] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 15:09:10] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:13] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 15:09:16] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:09:16] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 15:09:17] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 15:09:17] [INFO] CLEANUP RUN FINISHED +[2025-11-30 15:10:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 15:10:29] [INFO] Time Cutoff (General): 2025-11-30T15:10:29+0000 +[2025-11-30 15:10:29] [INFO] Time Cutoff (Images): 2025-10-01T15:10:29+0000 +[2025-11-30 15:10:29] [INFO] Delete Limit per Type: 200 +[2025-11-30 15:10:29] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 15:10:30] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 15:10:32] [INFO] No Service Accounts found matching prefix. +[2025-11-30 15:10:32] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:10:34] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 15:10:34] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 15:10:37] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:10:37] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:10:37] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 15:10:37] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:10:40] [INFO] No Filestore instances found matching criteria. +[2025-11-30 15:10:40] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 15:10:43] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 15:10:43] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 15:10:43] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 15:10:43] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 15:10:43] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 15:10:43] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 15:10:43] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 15:10:43] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 15:10:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 15:10:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 15:10:44] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 15:10:44] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:10:44Z (Unix: 1763305844) +[2025-11-30 15:10:44] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 15:10:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 (Updated: 2024-10-24T07:21:12 [TS: 1729754472] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:10:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 (Updated: 2024-10-24T07:21:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1d72fb41-5046-4044-a316-d2b78c8f8cb3] to complete... +.....done. +[2025-11-30 15:10:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 +[2025-11-30 15:10:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 (Updated: 2024-10-25T07:19:11 [TS: 1729840751] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:10:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 (Updated: 2024-10-25T07:19:11) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3de8a708-6e9a-4a23-b990-68d5f9007748] to complete... +.....done. +[2025-11-30 15:10:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 +[2025-11-30 15:10:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea (Updated: 2024-10-26T07:19:04 [TS: 1729927144] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:10:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea (Updated: 2024-10-26T07:19:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/395c03bc-1131-4bff-b308-a7b41251ba7b] to complete... +......done. +[2025-11-30 15:11:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea +[2025-11-30 15:11:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 (Updated: 2024-10-27T07:18:59 [TS: 1730013539] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 (Updated: 2024-10-27T07:18:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/02096d61-a872-4a5e-a333-bd255c67dd7b] to complete... +......done. +[2025-11-30 15:11:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 +[2025-11-30 15:11:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f (Updated: 2024-10-28T07:19:53 [TS: 1730099993] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f (Updated: 2024-10-28T07:19:53) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fb615cd5-97ab-4b29-bc51-c993addcff73] to complete... +......done. +[2025-11-30 15:11:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f +[2025-11-30 15:11:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa (Updated: 2024-10-29T07:21:32 [TS: 1730186492] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa (Updated: 2024-10-29T07:21:32) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0246878c-20aa-45a0-9613-9dbb21d0568f] to complete... +......done. +[2025-11-30 15:11:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa +[2025-11-30 15:11:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c (Updated: 2024-10-30T07:20:09 [TS: 1730272809] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c (Updated: 2024-10-30T07:20:09) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cfa4b060-0d32-4142-94b6-b0e97028c7d8] to complete... +.....done. +[2025-11-30 15:11:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c +[2025-11-30 15:11:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca (Updated: 2024-10-31T07:19:50 [TS: 1730359190] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca (Updated: 2024-10-31T07:19:50) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1ba620f4-2f1f-4c24-9b6a-f313785b452a] to complete... +.....done. +[2025-11-30 15:11:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca +[2025-11-30 15:11:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb (Updated: 2024-11-01T07:19:35 [TS: 1730445575] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb (Updated: 2024-11-01T07:19:35) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2f84b137-270b-4543-945d-d9ae70c2410e] to complete... +......done. +[2025-11-30 15:11:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb +[2025-11-30 15:11:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 (Updated: 2024-11-02T07:19:20 [TS: 1730531960] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 (Updated: 2024-11-02T07:19:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b5ecf31d-9a9c-474e-b87d-5c2d174b1de7] to complete... +......done. +[2025-11-30 15:11:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 +[2025-11-30 15:11:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b (Updated: 2024-11-03T07:19:34 [TS: 1730618374] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b (Updated: 2024-11-03T07:19:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5f3e8277-dd77-4433-8d65-a183fc08a534] to complete... +......done. +[2025-11-30 15:11:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b +[2025-11-30 15:11:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc (Updated: 2024-11-04T08:20:03 [TS: 1730708403] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc (Updated: 2024-11-04T08:20:03) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/27069aa2-141e-45fc-8d80-c7e7be7b5fe7] to complete... +......done. +[2025-11-30 15:11:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc +[2025-11-30 15:11:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed (Updated: 2024-11-05T08:19:36 [TS: 1730794776] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed (Updated: 2024-11-05T08:19:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/908020ef-fac3-4535-a381-3d44e555daa4] to complete... +......done. +[2025-11-30 15:11:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed +[2025-11-30 15:11:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e (Updated: 2024-11-06T08:19:42 [TS: 1730881182] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e (Updated: 2024-11-06T08:19:42) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/74ab1756-6c67-499c-8b7b-e058f065f6bb] to complete... +.....done. +[2025-11-30 15:11:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e +[2025-11-30 15:11:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 (Updated: 2024-11-07T08:22:39 [TS: 1730967759] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 (Updated: 2024-11-07T08:22:39) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/90d71dca-2654-4cdb-8293-79c6ee79a908] to complete... +.....done. +[2025-11-30 15:11:48] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 +[2025-11-30 15:11:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 (Updated: 2024-11-08T08:19:05 [TS: 1731053945] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 (Updated: 2024-11-08T08:19:05) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f3003f49-8aa2-4f19-91ee-e9bce6e44c19] to complete... +......done. +[2025-11-30 15:11:52] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 +[2025-11-30 15:11:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 (Updated: 2024-11-09T08:19:52 [TS: 1731140392] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:52] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 (Updated: 2024-11-09T08:19:52) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9c7788c1-18ab-4b6b-927f-2a3412c69f8e] to complete... +......done. +[2025-11-30 15:11:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 +[2025-11-30 15:11:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 (Updated: 2024-11-10T08:19:21 [TS: 1731226761] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 (Updated: 2024-11-10T08:19:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/95594b98-7af2-4441-93a4-e449d360acda] to complete... +.....done. +[2025-11-30 15:11:59] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 +[2025-11-30 15:11:59] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e (Updated: 2024-11-11T08:18:44 [TS: 1731313124] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:11:59] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e (Updated: 2024-11-11T08:18:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f678e910-b8be-4213-a8c3-3030be9a422d] to complete... +.....done. +[2025-11-30 15:12:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e +[2025-11-30 15:12:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c (Updated: 2024-11-12T08:20:12 [TS: 1731399612] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c (Updated: 2024-11-12T08:20:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bf899df0-0928-4103-ae7d-3474e906632a] to complete... +......done. +[2025-11-30 15:12:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c +[2025-11-30 15:12:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d (Updated: 2024-11-13T08:18:46 [TS: 1731485926] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d (Updated: 2024-11-13T08:18:46) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/80a2932b-60b5-430e-9c03-e061fac2c639] to complete... +......done. +[2025-11-30 15:12:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d +[2025-11-30 15:12:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 (Updated: 2024-11-14T08:19:32 [TS: 1731572372] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 (Updated: 2024-11-14T08:19:32) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d61ebc54-ae66-4ed6-bb92-db4795d2fba4] to complete... +.....done. +[2025-11-30 15:12:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 +[2025-11-30 15:12:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa (Updated: 2024-11-15T08:18:37 [TS: 1731658717] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa (Updated: 2024-11-15T08:18:37) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2e06c71b-5735-49d1-919f-a44390d1b65c] to complete... +.....done. +[2025-11-30 15:12:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa +[2025-11-30 15:12:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 (Updated: 2024-11-16T08:19:38 [TS: 1731745178] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 (Updated: 2024-11-16T08:19:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/20d51519-7b64-44de-85b8-5ea71ac24823] to complete... +.....done. +[2025-11-30 15:12:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 +[2025-11-30 15:12:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 (Updated: 2024-11-17T08:19:08 [TS: 1731831548] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 (Updated: 2024-11-17T08:19:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/98cffb83-76f1-4e75-9f90-f551e1c139a1] to complete... +......done. +[2025-11-30 15:12:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 +[2025-11-30 15:12:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c (Updated: 2024-11-18T08:19:40 [TS: 1731917980] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c (Updated: 2024-11-18T08:19:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/89f233e7-7dc3-4216-9ab9-a79aa856975d] to complete... +......done. +[2025-11-30 15:12:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c +[2025-11-30 15:12:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e (Updated: 2024-11-19T08:19:29 [TS: 1732004369] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e (Updated: 2024-11-19T08:19:29) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6e7a3baf-1ec2-427b-95ce-1203f0150f7a] to complete... +......done. +[2025-11-30 15:12:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e +[2025-11-30 15:12:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 (Updated: 2024-11-20T08:19:24 [TS: 1732090764] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 (Updated: 2024-11-20T08:19:24) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/059d29e8-cce5-4970-9b02-52b04188f041] to complete... +.....done. +[2025-11-30 15:12:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 +[2025-11-30 15:12:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf (Updated: 2024-11-21T08:19:48 [TS: 1732177188] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf (Updated: 2024-11-21T08:19:48) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d4aa1482-d133-41e4-8a83-bb0cd334acbe] to complete... +......done. +[2025-11-30 15:12:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf +[2025-11-30 15:12:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 (Updated: 2024-11-22T08:19:50 [TS: 1732263590] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 (Updated: 2024-11-22T08:19:50) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/21c13c9b-fc83-4fe9-b780-cc92769af79f] to complete... +......done. +[2025-11-30 15:12:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 +[2025-11-30 15:12:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb (Updated: 2024-11-23T08:18:52 [TS: 1732349932] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb (Updated: 2024-11-23T08:18:52) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c3493829-3493-4cb8-babe-e8733460d3d8] to complete... +......done. +[2025-11-30 15:12:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb +[2025-11-30 15:12:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 (Updated: 2024-11-24T08:19:55 [TS: 1732436395] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 (Updated: 2024-11-24T08:19:55) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bfd83599-ec9c-4944-80d8-6643b6c82c5c] to complete... +.....done. +[2025-11-30 15:12:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 +[2025-11-30 15:12:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 (Updated: 2024-11-25T08:18:26 [TS: 1732522706] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 (Updated: 2024-11-25T08:18:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8b465f77-040c-492c-97ca-8c55a29ab780] to complete... +......done. +[2025-11-30 15:12:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 +[2025-11-30 15:12:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e (Updated: 2024-11-26T08:19:19 [TS: 1732609159] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:12:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e (Updated: 2024-11-26T08:19:19) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2e49a45b-963a-4ed5-8fda-de2386c86f39] to complete... +.....done. +[2025-11-30 15:13:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e +[2025-11-30 15:13:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f (Updated: 2024-11-27T08:19:05 [TS: 1732695545] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f (Updated: 2024-11-27T08:19:05) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a425b84f-6836-420a-8987-8bc4b50a95f7] to complete... +......done. +[2025-11-30 15:13:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f +[2025-11-30 15:13:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af (Updated: 2024-11-28T08:18:54 [TS: 1732781934] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af (Updated: 2024-11-28T08:18:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e2d546a4-f58d-484f-9d11-268ecde1dbd4] to complete... +.....done. +[2025-11-30 15:13:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af +[2025-11-30 15:13:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 (Updated: 2024-11-29T08:19:25 [TS: 1732868365] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 (Updated: 2024-11-29T08:19:25) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/200ea5c8-0603-4866-a491-152d7866ba66] to complete... +......done. +[2025-11-30 15:13:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 +[2025-11-30 15:13:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db (Updated: 2024-11-30T08:19:26 [TS: 1732954766] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db (Updated: 2024-11-30T08:19:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2b506b8f-3309-4a37-96c8-d14015a38604] to complete... +.....done. +[2025-11-30 15:13:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db +[2025-11-30 15:13:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b (Updated: 2024-12-01T08:20:11 [TS: 1733041211] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b (Updated: 2024-12-01T08:20:11) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7cc6888a-c2d4-4176-82ba-aea96989929a] to complete... +.....done. +[2025-11-30 15:13:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b +[2025-11-30 15:13:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c (Updated: 2024-12-02T08:18:21 [TS: 1733127501] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c (Updated: 2024-12-02T08:18:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/28f95d0f-4dbf-4043-b45d-b2086a837b18] to complete... +......done. +[2025-11-30 15:13:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c +[2025-11-30 15:13:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff (Updated: 2024-12-03T08:19:17 [TS: 1733213957] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff (Updated: 2024-12-03T08:19:17) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ff423dc4-8ff3-4d19-9545-a06179659408] to complete... +.....done. +[2025-11-30 15:13:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff +[2025-11-30 15:13:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 (Updated: 2024-12-04T08:20:29 [TS: 1733300429] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 (Updated: 2024-12-04T08:20:29) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f6b39aae-8d7d-4df0-94a1-cb934d1235f4] to complete... +.....done. +[2025-11-30 15:13:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 +[2025-11-30 15:13:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 (Updated: 2024-12-04T23:35:41 [TS: 1733355341] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 (Updated: 2024-12-04T23:35:41) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/915fc1b0-cc0c-462d-bf79-30c623e4111c] to complete... +......done. +[2025-11-30 15:13:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 +[2025-11-30 15:13:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e (Updated: 2024-12-05T08:19:43 [TS: 1733386783] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e (Updated: 2024-12-05T08:19:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8499967c-7465-4379-a907-2a830827bf91] to complete... +.....done. +[2025-11-30 15:13:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e +[2025-11-30 15:13:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 (Updated: 2024-12-06T08:19:51 [TS: 1733473191] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 (Updated: 2024-12-06T08:19:51) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9a8bd1d5-f131-4f70-8ee9-a598c447836c] to complete... +......done. +[2025-11-30 15:13:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 +[2025-11-30 15:13:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da (Updated: 2024-12-07T08:19:02 [TS: 1733559542] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da (Updated: 2024-12-07T08:19:02) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bcb835c6-30b7-4591-aca6-dd7f853680ee] to complete... +......done. +[2025-11-30 15:13:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da +[2025-11-30 15:13:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d (Updated: 2024-12-08T08:18:49 [TS: 1733645929] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d (Updated: 2024-12-08T08:18:49) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6a0a6bed-2f06-48d0-ac50-861d46964648] to complete... +......done. +[2025-11-30 15:13:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d +[2025-11-30 15:13:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e (Updated: 2024-12-09T08:18:56 [TS: 1733732336] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e (Updated: 2024-12-09T08:18:56) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d7b8649b-d5ea-4308-815d-5aff95ce3ba4] to complete... +......done. +[2025-11-30 15:13:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e +[2025-11-30 15:13:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f (Updated: 2024-12-10T08:20:41 [TS: 1733818841] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f (Updated: 2024-12-10T08:20:41) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e075644b-a6b1-48a7-88bd-4fd8a60ff043] to complete... +......done. +[2025-11-30 15:13:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f +[2025-11-30 15:13:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba (Updated: 2024-12-11T08:19:22 [TS: 1733905162] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:13:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba (Updated: 2024-12-11T08:19:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eead26ba-ee59-4a46-b1e2-542e761cc320] to complete... +......done. +[2025-11-30 15:14:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba +[2025-11-30 15:14:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 (Updated: 2024-12-12T08:19:21 [TS: 1733991561] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 (Updated: 2024-12-12T08:19:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/aa386bf8-39d6-46c7-b09f-153a0c4263a5] to complete... +.....done. +[2025-11-30 15:14:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 +[2025-11-30 15:14:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb (Updated: 2024-12-13T08:19:59 [TS: 1734077999] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb (Updated: 2024-12-13T08:19:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/28e66f6f-4acb-420e-b798-02cd16857a99] to complete... +.....done. +[2025-11-30 15:14:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb +[2025-11-30 15:14:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 (Updated: 2024-12-14T08:19:12 [TS: 1734164352] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 (Updated: 2024-12-14T08:19:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/14499298-53ca-484d-a5f4-107826a5fc45] to complete... +......done. +[2025-11-30 15:14:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 +[2025-11-30 15:14:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b (Updated: 2024-12-15T08:19:07 [TS: 1734250747] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b (Updated: 2024-12-15T08:19:07) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/db3a4ba6-eaa6-4625-bab1-c2349190db13] to complete... +.....done. +[2025-11-30 15:14:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b +[2025-11-30 15:14:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 (Updated: 2024-12-16T08:18:40 [TS: 1734337120] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 (Updated: 2024-12-16T08:18:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cf627d8f-b714-4538-af41-c9b9e125f514] to complete... +......done. +[2025-11-30 15:14:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 +[2025-11-30 15:14:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 (Updated: 2024-12-17T08:19:36 [TS: 1734423576] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 (Updated: 2024-12-17T08:19:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2a3c7787-ae3a-477a-bcea-f55635ae87a5] to complete... +.....done. +[2025-11-30 15:14:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 +[2025-11-30 15:14:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 (Updated: 2024-12-18T08:19:28 [TS: 1734509968] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 (Updated: 2024-12-18T08:19:28) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/111726e2-7009-4e6b-9292-a34da8ae7774] to complete... +......done. +[2025-11-30 15:14:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 +[2025-11-30 15:14:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 (Updated: 2024-12-19T08:19:45 [TS: 1734596385] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 (Updated: 2024-12-19T08:19:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9ddb08d5-d5c1-4efb-9072-1971d959bda5] to complete... +......done. +[2025-11-30 15:14:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 +[2025-11-30 15:14:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 (Updated: 2024-12-20T08:20:22 [TS: 1734682822] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 (Updated: 2024-12-20T08:20:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f7254158-8896-49ce-85d7-ebbc79126364] to complete... +.....done. +[2025-11-30 15:14:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 +[2025-11-30 15:14:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b (Updated: 2024-12-21T08:18:43 [TS: 1734769123] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b (Updated: 2024-12-21T08:18:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/17b0a4e9-2e49-4273-a483-0edeb749430b] to complete... +......done. +[2025-11-30 15:14:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b +[2025-11-30 15:14:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 (Updated: 2024-12-22T08:19:03 [TS: 1734855543] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 (Updated: 2024-12-22T08:19:03) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c49e96e4-faae-490a-99cc-9ad29ade3d0f] to complete... +.....done. +[2025-11-30 15:14:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 +[2025-11-30 15:14:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 (Updated: 2024-12-23T08:19:14 [TS: 1734941954] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 (Updated: 2024-12-23T08:19:14) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a712c0a0-3b1c-48a5-85fb-7083d9189258] to complete... +.....done. +[2025-11-30 15:14:48] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 +[2025-11-30 15:14:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 (Updated: 2024-12-24T08:19:47 [TS: 1735028387] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 (Updated: 2024-12-24T08:19:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/31470aaf-d18e-4e4f-bd14-8b6cc6cbceaa] to complete... +......done. +[2025-11-30 15:14:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 +[2025-11-30 15:14:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c (Updated: 2024-12-25T08:19:25 [TS: 1735114765] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c (Updated: 2024-12-25T08:19:25) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/237f0e19-5c0b-4405-9581-ad1fdd1de6ef] to complete... +.....done. +[2025-11-30 15:14:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c +[2025-11-30 15:14:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d (Updated: 2024-12-26T08:19:08 [TS: 1735201148] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d (Updated: 2024-12-26T08:19:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cd8e148f-f667-47aa-815f-9ea7f38d86b1] to complete... +......done. +[2025-11-30 15:14:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d +[2025-11-30 15:14:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b (Updated: 2024-12-27T08:19:18 [TS: 1735287558] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:14:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b (Updated: 2024-12-27T08:19:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/17f120df-8744-4a79-b668-69fe6875d33d] to complete... +.....done. +[2025-11-30 15:15:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b +[2025-11-30 15:15:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 (Updated: 2024-12-28T08:20:48 [TS: 1735374048] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 (Updated: 2024-12-28T08:20:48) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f89727ee-b535-4afa-adb2-71e2103a930e] to complete... +.....done. +[2025-11-30 15:15:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 +[2025-11-30 15:15:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 (Updated: 2024-12-29T08:19:41 [TS: 1735460381] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 (Updated: 2024-12-29T08:19:41) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c8d94839-5258-4a80-889e-483f2a303027] to complete... +.....done. +[2025-11-30 15:15:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 +[2025-11-30 15:15:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a (Updated: 2024-12-30T08:19:44 [TS: 1735546784] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a (Updated: 2024-12-30T08:19:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/40b2ad56-f54c-4960-bec4-4933a34dacf6] to complete... +......done. +[2025-11-30 15:15:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a +[2025-11-30 15:15:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 (Updated: 2024-12-31T08:19:20 [TS: 1735633160] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 (Updated: 2024-12-31T08:19:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/64223c45-2771-4835-bb21-cf21cea7ef4f] to complete... +.....done. +[2025-11-30 15:15:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 +[2025-11-30 15:15:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a (Updated: 2025-01-01T08:20:09 [TS: 1735719609] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a (Updated: 2025-01-01T08:20:09) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8c0e6484-e4ea-4f83-8694-da8db62b099d] to complete... +......done. +[2025-11-30 15:15:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a +[2025-11-30 15:15:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 (Updated: 2025-01-02T08:19:36 [TS: 1735805976] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 (Updated: 2025-01-02T08:19:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f9d381f8-594d-43fa-a3b4-13ead53891b9] to complete... +.....done. +[2025-11-30 15:15:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 +[2025-11-30 15:15:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 (Updated: 2025-01-03T08:19:51 [TS: 1735892391] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 (Updated: 2025-01-03T08:19:51) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/765cfec0-1291-48ca-89dc-682ac1223e11] to complete... +.....done. +[2025-11-30 15:15:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 +[2025-11-30 15:15:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee (Updated: 2025-01-04T08:19:50 [TS: 1735978790] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee (Updated: 2025-01-04T08:19:50) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/221b1a8f-874c-480a-9ae2-24ce17f5ea03] to complete... +......done. +[2025-11-30 15:15:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee +[2025-11-30 15:15:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 (Updated: 2025-01-05T08:19:40 [TS: 1736065180] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 (Updated: 2025-01-05T08:19:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/65d93cad-f11b-4269-9cec-7b1a501c1f62] to complete... +......done. +[2025-11-30 15:15:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 +[2025-11-30 15:15:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 (Updated: 2025-01-06T08:19:23 [TS: 1736151563] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 (Updated: 2025-01-06T08:19:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7260811b-0cbb-460e-8123-b47d96de6a03] to complete... +.....done. +[2025-11-30 15:15:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 +[2025-11-30 15:15:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 (Updated: 2025-01-07T08:19:20 [TS: 1736237960] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 (Updated: 2025-01-07T08:19:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/38cc0369-3b20-4814-9f0c-3831a4189aa8] to complete... +......done. +[2025-11-30 15:15:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 +[2025-11-30 15:15:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 (Updated: 2025-01-08T08:19:34 [TS: 1736324374] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 (Updated: 2025-01-08T08:19:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bdbdaae3-7daf-438e-b877-6d7a2d90886c] to complete... +.....done. +[2025-11-30 15:15:45] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 +[2025-11-30 15:15:45] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 (Updated: 2025-01-09T08:19:09 [TS: 1736410749] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:45] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 (Updated: 2025-01-09T08:19:09) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/26ff91e3-1c3a-4324-a0e3-86f81b58deb7] to complete... +.....done. +[2025-11-30 15:15:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 +[2025-11-30 15:15:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf (Updated: 2025-01-10T08:18:36 [TS: 1736497116] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf (Updated: 2025-01-10T08:18:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7db598b0-d8ba-4320-b345-3683a9324247] to complete... +......done. +[2025-11-30 15:15:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf +[2025-11-30 15:15:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe (Updated: 2025-01-11T08:19:12 [TS: 1736583552] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe (Updated: 2025-01-11T08:19:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a1e90c63-f71d-484d-97b8-ad5244cd13e2] to complete... +.....done. +[2025-11-30 15:15:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe +[2025-11-30 15:15:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 (Updated: 2025-01-12T08:19:47 [TS: 1736669987] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:15:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 (Updated: 2025-01-12T08:19:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5e353a84-2938-44a4-a72e-82c72994bad0] to complete... +.....done. +[2025-11-30 15:16:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 +[2025-11-30 15:16:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 (Updated: 2025-01-13T08:19:33 [TS: 1736756373] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 (Updated: 2025-01-13T08:19:33) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b8ee05d9-a556-433e-83fe-2147254e5edd] to complete... +.....done. +[2025-11-30 15:16:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 +[2025-11-30 15:16:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 (Updated: 2025-01-15T08:19:08 [TS: 1736929148] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 (Updated: 2025-01-15T08:19:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0f13fce1-f4e5-429b-9359-a23c118bcf62] to complete... +.....done. +[2025-11-30 15:16:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 +[2025-11-30 15:16:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e (Updated: 2025-01-16T08:19:33 [TS: 1737015573] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e (Updated: 2025-01-16T08:19:33) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c0458cd0-bc7b-4219-97a0-b4d1f5ed2239] to complete... +......done. +[2025-11-30 15:16:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e +[2025-11-30 15:16:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 (Updated: 2025-01-17T08:22:38 [TS: 1737102158] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 (Updated: 2025-01-17T08:22:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/deb08cdd-51a3-44b9-933d-69b96681a2df] to complete... +......done. +[2025-11-30 15:16:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 +[2025-11-30 15:16:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f (Updated: 2025-01-18T08:20:38 [TS: 1737188438] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f (Updated: 2025-01-18T08:20:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cbc4a5fe-2eab-44c2-a441-c8ec91e4cf73] to complete... +.....done. +[2025-11-30 15:16:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f +[2025-11-30 15:16:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 (Updated: 2025-01-19T08:23:33 [TS: 1737275013] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 (Updated: 2025-01-19T08:23:33) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/62210de8-e0fc-4921-9f98-ed1641bb85cb] to complete... +......done. +[2025-11-30 15:16:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 +[2025-11-30 15:16:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c (Updated: 2025-01-20T08:20:58 [TS: 1737361258] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c (Updated: 2025-01-20T08:20:58) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ed167d2f-4201-41c7-9ad7-38386eb648a5] to complete... +.....done. +[2025-11-30 15:16:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c +[2025-11-30 15:16:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 (Updated: 2025-01-21T08:21:03 [TS: 1737447663] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 (Updated: 2025-01-21T08:21:03) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/26015732-c98a-4ad4-b672-179bec651261] to complete... +.....done. +[2025-11-30 15:16:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 +[2025-11-30 15:16:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a (Updated: 2025-01-22T08:20:37 [TS: 1737534037] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a (Updated: 2025-01-22T08:20:37) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d23e711a-f007-4574-9e61-26cbf6202d60] to complete... +.....done. +[2025-11-30 15:16:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a +[2025-11-30 15:16:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 (Updated: 2025-01-23T08:20:21 [TS: 1737620421] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 (Updated: 2025-01-23T08:20:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/547f8e39-d72d-4d97-bce0-81a2d5343055] to complete... +.....done. +[2025-11-30 15:16:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 +[2025-11-30 15:16:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 (Updated: 2025-01-24T08:19:44 [TS: 1737706784] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 (Updated: 2025-01-24T08:19:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5304569a-fd90-44dc-be73-f54d5ec73fdb] to complete... +......done. +[2025-11-30 15:16:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 +[2025-11-30 15:16:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 (Updated: 2025-01-25T08:20:25 [TS: 1737793225] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 (Updated: 2025-01-25T08:20:25) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/83d47863-f0ee-41e0-ad15-f7de3aa0de23] to complete... +.....done. +[2025-11-30 15:16:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 +[2025-11-30 15:16:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c (Updated: 2025-01-26T08:19:43 [TS: 1737879583] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c (Updated: 2025-01-26T08:19:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5ffeb95e-44d1-4a34-9e49-badaeac69d8d] to complete... +.....done. +[2025-11-30 15:16:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c +[2025-11-30 15:16:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d (Updated: 2025-01-27T08:21:26 [TS: 1737966086] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d (Updated: 2025-01-27T08:21:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bc62280e-3850-4eec-8251-446e5dfc3339] to complete... +......done. +[2025-11-30 15:16:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d +[2025-11-30 15:16:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e (Updated: 2025-01-28T08:22:47 [TS: 1738052567] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e (Updated: 2025-01-28T08:22:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f378ff13-619d-459e-9f64-b84ee0f80c2d] to complete... +......done. +[2025-11-30 15:16:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e +[2025-11-30 15:16:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 (Updated: 2025-01-29T08:20:27 [TS: 1738138827] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 (Updated: 2025-01-29T08:20:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b1891a54-223e-4b32-b5b9-20f94b6731ad] to complete... +......done. +[2025-11-30 15:16:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 +[2025-11-30 15:16:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 (Updated: 2025-01-30T08:20:39 [TS: 1738225239] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:16:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 (Updated: 2025-01-30T08:20:39) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/622a93e2-9461-4fd9-b0fc-f2aa880cda6e] to complete... +......done. +[2025-11-30 15:17:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 +[2025-11-30 15:17:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 (Updated: 2025-01-31T08:20:22 [TS: 1738311622] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 (Updated: 2025-01-31T08:20:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eb173b80-f08e-4868-9bf4-a6bda5fbfad8] to complete... +.....done. +[2025-11-30 15:17:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 +[2025-11-30 15:17:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 (Updated: 2025-02-01T08:20:50 [TS: 1738398050] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 (Updated: 2025-02-01T08:20:50) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7145a20c-038f-4c5f-aca2-724b90678ef9] to complete... +.....done. +[2025-11-30 15:17:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 +[2025-11-30 15:17:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 (Updated: 2025-02-02T08:20:43 [TS: 1738484443] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 (Updated: 2025-02-02T08:20:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7800581e-1c48-4191-970d-85cc24b3651b] to complete... +.....done. +[2025-11-30 15:17:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 +[2025-11-30 15:17:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 (Updated: 2025-02-03T08:20:14 [TS: 1738570814] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 (Updated: 2025-02-03T08:20:14) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5261cc68-2b99-4d4b-8c49-fb058c2740fa] to complete... +.....done. +[2025-11-30 15:17:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 +[2025-11-30 15:17:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae (Updated: 2025-02-04T08:20:52 [TS: 1738657252] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae (Updated: 2025-02-04T08:20:52) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9a8871e3-8495-410d-b607-46056e6b5d44] to complete... +......done. +[2025-11-30 15:17:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae +[2025-11-30 15:17:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b (Updated: 2025-02-05T08:20:48 [TS: 1738743648] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b (Updated: 2025-02-05T08:20:48) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0f46ff97-65af-4586-a2af-fffa12f72335] to complete... +.....done. +[2025-11-30 15:17:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b +[2025-11-30 15:17:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d (Updated: 2025-02-06T08:20:11 [TS: 1738830011] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d (Updated: 2025-02-06T08:20:11) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b2cb5f5a-d555-488e-a7aa-3956f9fda431] to complete... +......done. +[2025-11-30 15:17:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d +[2025-11-30 15:17:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c (Updated: 2025-02-07T08:21:04 [TS: 1738916464] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c (Updated: 2025-02-07T08:21:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/79744cbb-3db1-4bd8-880c-994827fc13d6] to complete... +.....done. +[2025-11-30 15:17:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c +[2025-11-30 15:17:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef (Updated: 2025-02-08T08:20:54 [TS: 1739002854] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef (Updated: 2025-02-08T08:20:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9e5c0385-97d1-4374-b00f-21f90f461ea3] to complete... +.....done. +[2025-11-30 15:17:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef +[2025-11-30 15:17:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 (Updated: 2025-02-09T08:20:35 [TS: 1739089235] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 (Updated: 2025-02-09T08:20:35) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/79dd60a5-24eb-4cf8-8c7f-1cfdb402d2eb] to complete... +.....done. +[2025-11-30 15:17:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 +[2025-11-30 15:17:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 (Updated: 2025-02-10T08:20:55 [TS: 1739175655] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 (Updated: 2025-02-10T08:20:55) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/99c9df64-cbeb-44de-97f3-c404f68fc8cc] to complete... +.....done. +[2025-11-30 15:17:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 +[2025-11-30 15:17:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af (Updated: 2025-02-11T08:19:45 [TS: 1739261985] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af (Updated: 2025-02-11T08:19:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a2fba2a7-4223-4fd8-8c32-f20651144da0] to complete... +.....done. +[2025-11-30 15:17:45] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af +[2025-11-30 15:17:45] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 (Updated: 2025-02-12T08:23:12 [TS: 1739348592] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:45] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 (Updated: 2025-02-12T08:23:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7fc977b9-ea13-48f5-aaef-0d1c23469bb2] to complete... +......done. +[2025-11-30 15:17:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 +[2025-11-30 15:17:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 (Updated: 2025-02-13T08:21:40 [TS: 1739434900] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 (Updated: 2025-02-13T08:21:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/608946ac-8589-4c71-b607-13c3eb211554] to complete... +.....done. +[2025-11-30 15:17:52] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 +[2025-11-30 15:17:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 (Updated: 2025-02-14T08:20:29 [TS: 1739521229] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:52] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 (Updated: 2025-02-14T08:20:29) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0d8b5f70-5ab9-40b6-b1b5-2a8868e07e2c] to complete... +.....done. +[2025-11-30 15:17:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 +[2025-11-30 15:17:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 (Updated: 2025-02-15T08:20:08 [TS: 1739607608] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 (Updated: 2025-02-15T08:20:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e4ab18ec-53a8-4c10-bdd2-3f6af00ab576] to complete... +......done. +[2025-11-30 15:17:59] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 +[2025-11-30 15:17:59] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f (Updated: 2025-02-16T08:20:40 [TS: 1739694040] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:17:59] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f (Updated: 2025-02-16T08:20:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e42c1bfd-6a84-49a2-b663-1f630339e84d] to complete... +.....done. +[2025-11-30 15:18:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f +[2025-11-30 15:18:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 (Updated: 2025-02-17T08:24:07 [TS: 1739780647] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 (Updated: 2025-02-17T08:24:07) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eb8098e3-5eab-41b3-8565-e664c8968c4a] to complete... +......done. +[2025-11-30 15:18:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 +[2025-11-30 15:18:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 (Updated: 2025-02-18T08:17:40 [TS: 1739866660] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 (Updated: 2025-02-18T08:17:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a6f79ba4-fcfa-4462-9e52-319bc4bdbcb9] to complete... +.....done. +[2025-11-30 15:18:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 +[2025-11-30 15:18:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 (Updated: 2025-02-19T08:21:02 [TS: 1739953262] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 (Updated: 2025-02-19T08:21:02) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d96b73f9-f27a-4923-b7eb-cbfa23e8f19e] to complete... +.....done. +[2025-11-30 15:18:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 +[2025-11-30 15:18:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c (Updated: 2025-02-20T08:18:01 [TS: 1740039481] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c (Updated: 2025-02-20T08:18:01) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c8d0615b-8179-46a8-9bde-cba2db5e9c79] to complete... +.....done. +[2025-11-30 15:18:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c +[2025-11-30 15:18:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 (Updated: 2025-02-21T08:20:07 [TS: 1740126007] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 (Updated: 2025-02-21T08:20:07) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/00e479ec-5af7-4db9-a9fb-b4e9389fc354] to complete... +......done. +[2025-11-30 15:18:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 +[2025-11-30 15:18:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 (Updated: 2025-02-25T08:20:52 [TS: 1740471652] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 (Updated: 2025-02-25T08:20:52) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5c092c38-328b-483a-b2dd-83548f53aeaa] to complete... +......done. +[2025-11-30 15:18:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 +[2025-11-30 15:18:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 (Updated: 2025-02-26T08:21:06 [TS: 1740558066] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 (Updated: 2025-02-26T08:21:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4bacb1cc-25e7-4e19-94f1-56d4466611b3] to complete... +......done. +[2025-11-30 15:18:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 +[2025-11-30 15:18:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 (Updated: 2025-02-27T08:21:06 [TS: 1740644466] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 (Updated: 2025-02-27T08:21:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d583fd59-9f57-40e8-9e86-a0fe35d28054] to complete... +......done. +[2025-11-30 15:18:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 +[2025-11-30 15:18:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 (Updated: 2025-02-28T08:21:16 [TS: 1740730876] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 (Updated: 2025-02-28T08:21:16) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/dc2af5ad-a0b6-4097-a032-9cd47a6735dd] to complete... +......done. +[2025-11-30 15:18:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 +[2025-11-30 15:18:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c (Updated: 2025-03-01T08:20:02 [TS: 1740817202] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c (Updated: 2025-03-01T08:20:02) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9f9a369c-3175-4395-a6da-c7f19cb7323e] to complete... +.....done. +[2025-11-30 15:18:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c +[2025-11-30 15:18:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 (Updated: 2025-03-02T08:21:22 [TS: 1740903682] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 (Updated: 2025-03-02T08:21:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0282f1d3-de95-4b5e-9789-84b2b3b42ff6] to complete... +.....done. +[2025-11-30 15:18:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 +[2025-11-30 15:18:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 (Updated: 2025-03-03T08:17:41 [TS: 1740989861] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 (Updated: 2025-03-03T08:17:41) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5b6b6571-406a-459c-b9b9-fd1bd52d8994] to complete... +.....done. +[2025-11-30 15:18:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 +[2025-11-30 15:18:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b (Updated: 2025-03-04T08:22:52 [TS: 1741076572] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b (Updated: 2025-03-04T08:22:52) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/baceb358-fe46-43a3-bc91-44c3022f8042] to complete... +.....done. +[2025-11-30 15:18:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b +[2025-11-30 15:18:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 (Updated: 2025-03-05T08:21:41 [TS: 1741162901] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 (Updated: 2025-03-05T08:21:41) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0afb0c25-ec0b-43b9-b557-b3bd15d2c159] to complete... +.....done. +[2025-11-30 15:18:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 +[2025-11-30 15:18:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 (Updated: 2025-03-06T08:21:32 [TS: 1741249292] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 (Updated: 2025-03-06T08:21:32) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/97e5379d-a50e-4fae-882a-c00b46bc1d55] to complete... +.....done. +[2025-11-30 15:18:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 +[2025-11-30 15:18:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc (Updated: 2025-03-07T08:19:43 [TS: 1741335583] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:18:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc (Updated: 2025-03-07T08:19:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/de48f5de-06b9-428b-8caa-51a58ee2a3cb] to complete... +.....done. +[2025-11-30 15:19:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc +[2025-11-30 15:19:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c (Updated: 2025-03-08T08:19:23 [TS: 1741421963] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c (Updated: 2025-03-08T08:19:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/21aed277-ebaa-4d63-b317-e971a6296fca] to complete... +.....done. +[2025-11-30 15:19:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c +[2025-11-30 15:19:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a (Updated: 2025-03-09T08:18:08 [TS: 1741508288] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a (Updated: 2025-03-09T08:18:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/90e81efe-d447-4f64-8568-ab0296805023] to complete... +.....done. +[2025-11-30 15:19:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a +[2025-11-30 15:19:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b (Updated: 2025-03-10T07:21:20 [TS: 1741591280] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b (Updated: 2025-03-10T07:21:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e36f1439-3fc6-49b4-be48-b63c8ca04886] to complete... +.....done. +[2025-11-30 15:19:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b +[2025-11-30 15:19:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e (Updated: 2025-03-11T07:20:43 [TS: 1741677643] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e (Updated: 2025-03-11T07:20:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5914289e-6d0f-4c69-b39b-4134ca8793ec] to complete... +.....done. +[2025-11-30 15:19:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e +[2025-11-30 15:19:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e (Updated: 2025-03-12T07:19:34 [TS: 1741763974] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e (Updated: 2025-03-12T07:19:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1577694f-3577-4746-8dce-cbbc499cc475] to complete... +.....done. +[2025-11-30 15:19:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e +[2025-11-30 15:19:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 (Updated: 2025-03-13T07:20:51 [TS: 1741850451] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 (Updated: 2025-03-13T07:20:51) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e6313482-221e-4869-8816-ad627694ef62] to complete... +.....done. +[2025-11-30 15:19:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 +[2025-11-30 15:19:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 (Updated: 2025-03-14T07:20:37 [TS: 1741936837] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 (Updated: 2025-03-14T07:20:37) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bc09cee9-0217-419a-9eab-df4dfb090e68] to complete... +.....done. +[2025-11-30 15:19:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 +[2025-11-30 15:19:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 (Updated: 2025-03-15T07:21:23 [TS: 1742023283] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 (Updated: 2025-03-15T07:21:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8e44511e-9b39-4dc6-8f92-96065bc5056f] to complete... +.....done. +[2025-11-30 15:19:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 +[2025-11-30 15:19:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 (Updated: 2025-03-16T07:20:44 [TS: 1742109644] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 (Updated: 2025-03-16T07:20:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b4da2193-082e-4d6a-b642-4f3b9556b6fc] to complete... +......done. +[2025-11-30 15:19:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 +[2025-11-30 15:19:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 (Updated: 2025-03-17T07:21:09 [TS: 1742196069] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 (Updated: 2025-03-17T07:21:09) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e09b9c22-8238-448b-8608-560a33bd5efc] to complete... +.....done. +[2025-11-30 15:19:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 +[2025-11-30 15:19:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 (Updated: 2025-03-18T07:22:07 [TS: 1742282527] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 (Updated: 2025-03-18T07:22:07) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/82286eac-44ec-4fcd-8311-f321e6c8368f] to complete... +......done. +[2025-11-30 15:19:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 +[2025-11-30 15:19:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac (Updated: 2025-03-19T07:21:36 [TS: 1742368896] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac (Updated: 2025-03-19T07:21:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c9f2c33f-f619-4b4d-8b85-da3ed41357e0] to complete... +.....done. +[2025-11-30 15:19:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac +[2025-11-30 15:19:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a (Updated: 2025-03-20T07:21:15 [TS: 1742455275] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a (Updated: 2025-03-20T07:21:15) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a2e13152-c6a2-4e19-b5b1-28f3fdda7790] to complete... +.....done. +[2025-11-30 15:19:45] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a +[2025-11-30 15:19:45] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 (Updated: 2025-03-21T07:20:54 [TS: 1742541654] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:45] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 (Updated: 2025-03-21T07:20:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7bf6c370-194d-495b-93c3-00d64f8bb812] to complete... +......done. +[2025-11-30 15:19:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 +[2025-11-30 15:19:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 (Updated: 2025-03-22T07:21:14 [TS: 1742628074] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 (Updated: 2025-03-22T07:21:14) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f54e03ee-6a6d-44ce-a2e5-3f2f7f8b4403] to complete... +......done. +[2025-11-30 15:19:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 +[2025-11-30 15:19:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f (Updated: 2025-03-23T07:20:50 [TS: 1742714450] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f (Updated: 2025-03-23T07:20:50) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3a19055e-a37b-4a94-adcc-fb03de837b06] to complete... +.....done. +[2025-11-30 15:19:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f +[2025-11-30 15:19:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca (Updated: 2025-03-24T07:21:04 [TS: 1742800864] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:19:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca (Updated: 2025-03-24T07:21:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/af12c77c-cc82-4f1e-8c4f-e7ca397e7cd0] to complete... +.....done. +[2025-11-30 15:20:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca +[2025-11-30 15:20:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f (Updated: 2025-03-25T07:21:17 [TS: 1742887277] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f (Updated: 2025-03-25T07:21:17) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3d7ac828-bdb0-4f70-b21e-47b6ab26e3f4] to complete... +.....done. +[2025-11-30 15:20:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f +[2025-11-30 15:20:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 (Updated: 2025-03-26T07:20:59 [TS: 1742973659] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 (Updated: 2025-03-26T07:20:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d703d169-8470-4090-bdf5-53e3c6823ff2] to complete... +.....done. +[2025-11-30 15:20:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 +[2025-11-30 15:20:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a (Updated: 2025-03-27T07:20:48 [TS: 1743060048] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a (Updated: 2025-03-27T07:20:48) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/919d2591-0773-4aa8-bbae-fcf200a69249] to complete... +.....done. +[2025-11-30 15:20:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a +[2025-11-30 15:20:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 (Updated: 2025-03-28T07:20:36 [TS: 1743146436] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 (Updated: 2025-03-28T07:20:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/07ccbd80-5be0-42d1-8b78-0e96d180b9c9] to complete... +.....done. +[2025-11-30 15:20:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 +[2025-11-30 15:20:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 (Updated: 2025-03-29T07:21:17 [TS: 1743232877] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 (Updated: 2025-03-29T07:21:17) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7de707fe-b1e7-453a-8801-538125db413a] to complete... +.....done. +[2025-11-30 15:20:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 +[2025-11-30 15:20:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 (Updated: 2025-03-30T07:21:38 [TS: 1743319298] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 (Updated: 2025-03-30T07:21:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/418cb23a-e880-4352-9ca2-54711dbb2314] to complete... +.....done. +[2025-11-30 15:20:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 +[2025-11-30 15:20:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 (Updated: 2025-03-31T07:20:18 [TS: 1743405618] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 (Updated: 2025-03-31T07:20:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0131a587-8a3f-458b-b07c-6eaf57c9cac3] to complete... +.....done. +[2025-11-30 15:20:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 +[2025-11-30 15:20:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 (Updated: 2025-04-01T07:21:20 [TS: 1743492080] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 (Updated: 2025-04-01T07:21:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/600c2c2d-43f9-4615-bf48-b3d96f573afe] to complete... +.....done. +[2025-11-30 15:20:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 +[2025-11-30 15:20:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d (Updated: 2025-04-02T07:20:23 [TS: 1743578423] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d (Updated: 2025-04-02T07:20:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/93615534-b7e1-40a9-a54d-ddd0c024b455] to complete... +.....done. +[2025-11-30 15:20:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d +[2025-11-30 15:20:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 (Updated: 2025-04-03T07:20:49 [TS: 1743664849] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 (Updated: 2025-04-03T07:20:49) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/571447de-1879-4fde-9954-b35e6ce99df5] to complete... +.....done. +[2025-11-30 15:20:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 +[2025-11-30 15:20:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b (Updated: 2025-04-04T07:20:41 [TS: 1743751241] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b (Updated: 2025-04-04T07:20:41) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2041092c-d1eb-4a0d-855d-77c828884b4c] to complete... +.....done. +[2025-11-30 15:20:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b +[2025-11-30 15:20:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a (Updated: 2025-04-05T07:20:21 [TS: 1743837621] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a (Updated: 2025-04-05T07:20:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/06c2386d-3789-4acd-ba0a-dd57aaa7e7df] to complete... +.....done. +[2025-11-30 15:20:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a +[2025-11-30 15:20:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 (Updated: 2025-04-06T07:23:45 [TS: 1743924225] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 (Updated: 2025-04-06T07:23:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/67832465-d1cf-4e5a-b8f9-c5663f221380] to complete... +.....done. +[2025-11-30 15:20:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 +[2025-11-30 15:20:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 (Updated: 2025-04-07T07:20:24 [TS: 1744010424] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 (Updated: 2025-04-07T07:20:24) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eae5c461-0c20-4556-befd-b08ed91074ea] to complete... +.....done. +[2025-11-30 15:20:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 +[2025-11-30 15:20:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee (Updated: 2025-04-08T07:22:43 [TS: 1744096963] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee (Updated: 2025-04-08T07:22:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b48c5f0f-a002-4f2e-a99d-2fcc8871b980] to complete... +.....done. +[2025-11-30 15:20:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee +[2025-11-30 15:20:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 (Updated: 2025-04-09T07:20:20 [TS: 1744183220] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 (Updated: 2025-04-09T07:20:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/93704621-002e-4175-83d5-dc6584a66c40] to complete... +.....done. +[2025-11-30 15:20:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 +[2025-11-30 15:20:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 (Updated: 2025-04-10T07:21:00 [TS: 1744269660] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:20:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 (Updated: 2025-04-10T07:21:00) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e2d9d0f6-edc0-4d57-9c08-44b2ef70d700] to complete... +.....done. +[2025-11-30 15:21:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 +[2025-11-30 15:21:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 (Updated: 2025-04-11T07:20:54 [TS: 1744356054] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 (Updated: 2025-04-11T07:20:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/56fb8f24-0081-4c12-92b2-4ee5361d756f] to complete... +.....done. +[2025-11-30 15:21:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 +[2025-11-30 15:21:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 (Updated: 2025-04-12T07:21:40 [TS: 1744442500] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 (Updated: 2025-04-12T07:21:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bb71c750-5d2e-4996-9aef-f573502c91d6] to complete... +.....done. +[2025-11-30 15:21:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 +[2025-11-30 15:21:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 (Updated: 2025-04-13T07:20:59 [TS: 1744528859] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 (Updated: 2025-04-13T07:20:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c0f699b8-7456-4990-bba7-fd1f407096f5] to complete... +.....done. +[2025-11-30 15:21:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 +[2025-11-30 15:21:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c (Updated: 2025-04-14T07:20:24 [TS: 1744615224] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c (Updated: 2025-04-14T07:20:24) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cccafaa5-65e5-48e3-bb51-27c8a2063873] to complete... +.....done. +[2025-11-30 15:21:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c +[2025-11-30 15:21:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 (Updated: 2025-04-15T07:20:46 [TS: 1744701646] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 (Updated: 2025-04-15T07:20:46) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/457bb2a4-234f-4ed5-817a-d2107b62119a] to complete... +......done. +[2025-11-30 15:21:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 +[2025-11-30 15:21:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 (Updated: 2025-04-16T07:20:12 [TS: 1744788012] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 (Updated: 2025-04-16T07:20:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1782a945-717e-4ac9-b1b0-0a6c256570bf] to complete... +.....done. +[2025-11-30 15:21:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 +[2025-11-30 15:21:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 (Updated: 2025-04-17T07:19:55 [TS: 1744874395] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 (Updated: 2025-04-17T07:19:55) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3fe2f825-3f75-429b-bc4e-66c791c70d95] to complete... +.....done. +[2025-11-30 15:21:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 +[2025-11-30 15:21:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb (Updated: 2025-04-18T07:20:41 [TS: 1744960841] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb (Updated: 2025-04-18T07:20:41) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c65df95f-34e6-404b-a0ca-879a87f6693b] to complete... +......done. +[2025-11-30 15:21:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb +[2025-11-30 15:21:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa (Updated: 2025-04-19T07:21:15 [TS: 1745047275] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa (Updated: 2025-04-19T07:21:15) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e9741419-65bb-46c4-af17-b68bf77e3a68] to complete... +.....done. +[2025-11-30 15:21:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa +[2025-11-30 15:21:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 (Updated: 2025-04-20T07:21:25 [TS: 1745133685] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 (Updated: 2025-04-20T07:21:25) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2f4c8846-5322-4309-b16b-33ceab778ae0] to complete... +.....done. +[2025-11-30 15:21:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 +[2025-11-30 15:21:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d (Updated: 2025-04-21T07:21:59 [TS: 1745220119] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d (Updated: 2025-04-21T07:21:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6f7abf11-ac26-437a-ac79-0ae1d9cd9fb2] to complete... +.....done. +[2025-11-30 15:21:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d +[2025-11-30 15:21:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a (Updated: 2025-04-22T07:21:34 [TS: 1745306494] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a (Updated: 2025-04-22T07:21:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/10934be6-c3ce-4ebe-9736-e1ad8a0c3f5c] to complete... +.....done. +[2025-11-30 15:21:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a +[2025-11-30 15:21:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 (Updated: 2025-04-23T07:21:06 [TS: 1745392866] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 (Updated: 2025-04-23T07:21:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2638cda2-663d-4626-9983-258237333034] to complete... +.....done. +[2025-11-30 15:21:45] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 +[2025-11-30 15:21:45] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 (Updated: 2025-04-24T07:21:59 [TS: 1745479319] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:45] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 (Updated: 2025-04-24T07:21:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b00b6823-40ba-4f9c-9081-cc928b6057df] to complete... +.....done. +[2025-11-30 15:21:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 +[2025-11-30 15:21:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 (Updated: 2025-04-25T07:20:22 [TS: 1745565622] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 (Updated: 2025-04-25T07:20:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e788c620-84fe-47dd-8283-a74aac3c3e80] to complete... +.....done. +[2025-11-30 15:21:52] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 +[2025-11-30 15:21:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d (Updated: 2025-04-26T07:20:43 [TS: 1745652043] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:52] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d (Updated: 2025-04-26T07:20:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/355147be-d407-41af-9358-e2387e5f1739] to complete... +......done. +[2025-11-30 15:21:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d +[2025-11-30 15:21:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da (Updated: 2025-04-27T07:21:38 [TS: 1745738498] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:21:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da (Updated: 2025-04-27T07:21:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7fef88a4-1a39-4d36-9e80-74ff9aa51bbe] to complete... +.....done. +[2025-11-30 15:22:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da +[2025-11-30 15:22:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 (Updated: 2025-04-28T07:21:01 [TS: 1745824861] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 (Updated: 2025-04-28T07:21:01) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d0495266-18e8-4d24-9407-657fb84a2579] to complete... +......done. +[2025-11-30 15:22:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 +[2025-11-30 15:22:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 (Updated: 2025-04-29T07:21:25 [TS: 1745911285] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 (Updated: 2025-04-29T07:21:25) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2238de96-f54c-42de-846d-18ef52ff5dcd] to complete... +......done. +[2025-11-30 15:22:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 +[2025-11-30 15:22:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 (Updated: 2025-04-30T07:20:20 [TS: 1745997620] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 (Updated: 2025-04-30T07:20:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/786e4953-e669-4ba3-acb9-afa982faa47e] to complete... +.....done. +[2025-11-30 15:22:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 +[2025-11-30 15:22:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 (Updated: 2025-05-01T07:21:24 [TS: 1746084084] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 (Updated: 2025-05-01T07:21:24) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eec93fa2-f697-4ac4-9bbe-07468d2549c9] to complete... +......done. +[2025-11-30 15:22:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 +[2025-11-30 15:22:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 (Updated: 2025-05-02T07:21:04 [TS: 1746170464] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 (Updated: 2025-05-02T07:21:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/17ecec7a-5ed2-4c81-ac4c-a16be4fa08b0] to complete... +......done. +[2025-11-30 15:22:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 +[2025-11-30 15:22:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 (Updated: 2025-05-03T07:21:59 [TS: 1746256919] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 (Updated: 2025-05-03T07:21:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/254afc4b-74eb-4e4d-a47d-fa44d2c04b1f] to complete... +.....done. +[2025-11-30 15:22:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 +[2025-11-30 15:22:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 (Updated: 2025-05-04T07:21:07 [TS: 1746343267] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 (Updated: 2025-05-04T07:21:07) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6474c9d5-ffe8-464d-8c75-2a9a05658f9b] to complete... +......done. +[2025-11-30 15:22:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 +[2025-11-30 15:22:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 (Updated: 2025-05-05T07:21:39 [TS: 1746429699] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 (Updated: 2025-05-05T07:21:39) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/69c2a80b-8b08-4b1a-ab85-7d2f57c3e1f1] to complete... +.....done. +[2025-11-30 15:22:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 +[2025-11-30 15:22:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 (Updated: 2025-05-06T07:21:39 [TS: 1746516099] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 (Updated: 2025-05-06T07:21:39) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0d725914-bd53-466c-8c22-66eab151706e] to complete... +.....done. +[2025-11-30 15:22:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 +[2025-11-30 15:22:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb (Updated: 2025-05-07T07:22:18 [TS: 1746602538] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb (Updated: 2025-05-07T07:22:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/deb3bbe0-c114-474a-aa04-be50573dce26] to complete... +.....done. +[2025-11-30 15:22:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb +[2025-11-30 15:22:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d (Updated: 2025-05-08T07:22:24 [TS: 1746688944] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d (Updated: 2025-05-08T07:22:24) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b71fe70e-f9a4-4a67-a093-76b4f6a72291] to complete... +.....done. +[2025-11-30 15:22:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d +[2025-11-30 15:22:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b (Updated: 2025-05-09T07:21:16 [TS: 1746775276] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b (Updated: 2025-05-09T07:21:16) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cfd9113f-abbe-42c6-8c4e-7850bbce9e8e] to complete... +......done. +[2025-11-30 15:22:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b +[2025-11-30 15:22:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 (Updated: 2025-05-10T07:21:20 [TS: 1746861680] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 (Updated: 2025-05-10T07:21:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8db16416-b432-45e9-b1fe-7e9a3e31d0bf] to complete... +.....done. +[2025-11-30 15:22:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 +[2025-11-30 15:22:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 (Updated: 2025-05-11T07:20:26 [TS: 1746948026] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 (Updated: 2025-05-11T07:20:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5b865889-906d-4a53-9ad7-a71d95eb46d3] to complete... +......done. +[2025-11-30 15:22:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 +[2025-11-30 15:22:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 (Updated: 2025-05-12T07:21:06 [TS: 1747034466] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 (Updated: 2025-05-12T07:21:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d8b68b09-df98-4a1d-8cec-95e1cf67bc5f] to complete... +......done. +[2025-11-30 15:22:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 +[2025-11-30 15:22:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 (Updated: 2025-05-13T07:21:27 [TS: 1747120887] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 (Updated: 2025-05-13T07:21:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/403db624-0675-4396-813e-186f87098604] to complete... +......done. +[2025-11-30 15:22:59] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 +[2025-11-30 15:22:59] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 (Updated: 2025-05-14T07:21:24 [TS: 1747207284] < Cutoff: [TS: 1763305844]) +[2025-11-30 15:22:59] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 (Updated: 2025-05-14T07:21:24) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5684dced-0dd8-40e1-95d0-0634f6d6daa9] to complete... +......done. +[2025-11-30 15:23:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 +[2025-11-30 15:23:03] [INFO] Hit delete limit (200) for Docker Images. +[2025-11-30 15:23:03] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 15:23:03] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 15:23:06] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 15:23:06] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 15:23:06] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 15:23:06] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 15:23:06] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 15:23:06] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 15:23:08] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 15:23:08] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:23:11] [INFO] No Regional Address found matching criteria. +[2025-11-30 15:23:11] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:23:13] [INFO] No Global Address found matching criteria. +[2025-11-30 15:23:13] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- +[2025-11-30 15:23:17] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 15:23:20] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:23:20] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:23:20] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 15:23:20] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 15:23:20] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:23] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 15:23:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:23:25] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 15:23:27] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 15:23:27] [INFO] CLEANUP RUN FINISHED +[2025-11-30 15:24:51] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 15:24:51] [INFO] Time Cutoff (General): 2025-11-30T15:24:51+0000 +[2025-11-30 15:24:51] [INFO] Time Cutoff (Images): 2025-10-01T15:24:51+0000 +[2025-11-30 15:24:51] [INFO] Delete Limit per Type: 200 +[2025-11-30 15:24:51] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 15:24:51] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 15:24:54] [INFO] No Service Accounts found matching prefix. +[2025-11-30 15:24:54] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:24:56] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 15:24:56] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 15:24:58] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:24:58] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:24:58] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 15:24:58] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:25:01] [INFO] No Filestore instances found matching criteria. +[2025-11-30 15:25:01] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 15:25:04] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 15:25:04] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 15:25:05] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 15:25:05] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 15:25:05] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 15:25:05] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 15:25:05] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 15:25:05] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 15:25:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 15:25:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 15:25:05] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 15:25:05] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:25:05Z (Unix: 1763306705) +[2025-11-30 15:25:05] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 (Updated: 2025-05-15T07:21:31 [TS: 1747293691] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 (Updated: 2025-05-15T07:21:31) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 (Updated: 2025-05-16T07:21:17 [TS: 1747380077] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 (Updated: 2025-05-16T07:21:17) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da (Updated: 2025-05-17T07:21:15 [TS: 1747466475] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da (Updated: 2025-05-17T07:21:15) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d (Updated: 2025-05-18T07:21:43 [TS: 1747552903] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d (Updated: 2025-05-18T07:21:43) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de (Updated: 2025-05-19T07:21:57 [TS: 1747639317] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de (Updated: 2025-05-19T07:21:57) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 (Updated: 2025-05-20T07:22:12 [TS: 1747725732] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 (Updated: 2025-05-20T07:22:12) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 (Updated: 2025-05-21T07:19:51 [TS: 1747811991] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 (Updated: 2025-05-21T07:19:51) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f (Updated: 2025-05-22T07:21:21 [TS: 1747898481] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f (Updated: 2025-05-22T07:21:21) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 (Updated: 2025-05-23T07:22:28 [TS: 1747984948] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 (Updated: 2025-05-23T07:22:28) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c (Updated: 2025-05-24T07:21:57 [TS: 1748071317] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c (Updated: 2025-05-24T07:21:57) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be (Updated: 2025-05-25T07:21:23 [TS: 1748157683] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be (Updated: 2025-05-25T07:21:23) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 (Updated: 2025-05-26T07:28:04 [TS: 1748244484] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 (Updated: 2025-05-26T07:28:04) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 (Updated: 2025-05-27T07:19:54 [TS: 1748330394] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 (Updated: 2025-05-27T07:19:54) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a (Updated: 2025-05-28T07:20:28 [TS: 1748416828] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a (Updated: 2025-05-28T07:20:28) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d (Updated: 2025-05-29T07:21:22 [TS: 1748503282] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d (Updated: 2025-05-29T07:21:22) +[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 (Updated: 2025-05-30T07:21:07 [TS: 1748589667] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 (Updated: 2025-05-30T07:21:07) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 (Updated: 2025-05-31T07:20:46 [TS: 1748676046] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 (Updated: 2025-05-31T07:20:46) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 (Updated: 2025-06-01T07:20:52 [TS: 1748762452] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 (Updated: 2025-06-01T07:20:52) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a (Updated: 2025-06-02T07:23:55 [TS: 1748849035] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a (Updated: 2025-06-02T07:23:55) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 (Updated: 2025-06-03T07:21:05 [TS: 1748935265] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 (Updated: 2025-06-03T07:21:05) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd (Updated: 2025-06-04T07:22:23 [TS: 1749021743] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd (Updated: 2025-06-04T07:22:23) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 (Updated: 2025-06-05T07:21:27 [TS: 1749108087] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 (Updated: 2025-06-05T07:21:27) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf (Updated: 2025-06-06T07:22:27 [TS: 1749194547] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf (Updated: 2025-06-06T07:22:27) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c (Updated: 2025-06-07T07:22:15 [TS: 1749280935] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c (Updated: 2025-06-07T07:22:15) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 (Updated: 2025-06-08T07:21:06 [TS: 1749367266] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 (Updated: 2025-06-08T07:21:06) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f (Updated: 2025-06-09T07:20:04 [TS: 1749453604] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f (Updated: 2025-06-09T07:20:04) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f (Updated: 2025-06-10T07:20:54 [TS: 1749540054] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f (Updated: 2025-06-10T07:20:54) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 (Updated: 2025-06-11T07:20:59 [TS: 1749626459] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 (Updated: 2025-06-11T07:20:59) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 (Updated: 2025-06-12T07:19:36 [TS: 1749712776] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 (Updated: 2025-06-12T07:19:36) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 (Updated: 2025-06-13T07:21:52 [TS: 1749799312] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 (Updated: 2025-06-13T07:21:52) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 (Updated: 2025-06-14T07:21:53 [TS: 1749885713] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 (Updated: 2025-06-14T07:21:53) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 (Updated: 2025-06-15T07:20:57 [TS: 1749972057] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 (Updated: 2025-06-15T07:20:57) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 (Updated: 2025-06-16T07:20:18 [TS: 1750058418] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 (Updated: 2025-06-16T07:20:18) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe (Updated: 2025-06-17T07:21:49 [TS: 1750144909] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe (Updated: 2025-06-17T07:21:49) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 (Updated: 2025-06-18T07:20:13 [TS: 1750231213] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 (Updated: 2025-06-18T07:20:13) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 (Updated: 2025-06-19T07:20:59 [TS: 1750317659] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 (Updated: 2025-06-19T07:20:59) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b (Updated: 2025-06-20T07:21:01 [TS: 1750404061] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b (Updated: 2025-06-20T07:21:01) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f (Updated: 2025-06-21T07:22:20 [TS: 1750490540] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f (Updated: 2025-06-21T07:22:20) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb (Updated: 2025-06-22T07:21:21 [TS: 1750576881] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb (Updated: 2025-06-22T07:21:21) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 (Updated: 2025-06-23T07:21:18 [TS: 1750663278] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 (Updated: 2025-06-23T07:21:18) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 (Updated: 2025-06-24T07:21:15 [TS: 1750749675] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 (Updated: 2025-06-24T07:21:15) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 (Updated: 2025-06-25T07:21:11 [TS: 1750836071] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 (Updated: 2025-06-25T07:21:11) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba (Updated: 2025-06-26T07:20:29 [TS: 1750922429] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba (Updated: 2025-06-26T07:20:29) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c (Updated: 2025-06-27T07:20:18 [TS: 1751008818] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c (Updated: 2025-06-27T07:20:18) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb (Updated: 2025-06-28T07:20:26 [TS: 1751095226] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb (Updated: 2025-06-28T07:20:26) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c (Updated: 2025-06-29T07:21:11 [TS: 1751181671] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c (Updated: 2025-06-29T07:21:11) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 (Updated: 2025-06-30T07:22:37 [TS: 1751268157] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 (Updated: 2025-06-30T07:22:37) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f (Updated: 2025-07-01T07:20:54 [TS: 1751354454] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f (Updated: 2025-07-01T07:20:54) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 (Updated: 2025-07-02T07:21:52 [TS: 1751440912] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 (Updated: 2025-07-02T07:21:52) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb (Updated: 2025-07-03T07:20:06 [TS: 1751527206] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb (Updated: 2025-07-03T07:20:06) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e (Updated: 2025-07-04T07:21:34 [TS: 1751613694] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e (Updated: 2025-07-04T07:21:34) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 (Updated: 2025-07-05T07:21:32 [TS: 1751700092] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 (Updated: 2025-07-05T07:21:32) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf (Updated: 2025-07-06T07:21:21 [TS: 1751786481] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf (Updated: 2025-07-06T07:21:21) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 (Updated: 2025-07-07T07:22:25 [TS: 1751872945] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 (Updated: 2025-07-07T07:22:25) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 (Updated: 2025-07-08T07:21:26 [TS: 1751959286] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 (Updated: 2025-07-08T07:21:26) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 (Updated: 2025-07-09T07:22:45 [TS: 1752045765] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 (Updated: 2025-07-09T07:22:45) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 (Updated: 2025-07-10T07:21:44 [TS: 1752132104] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 (Updated: 2025-07-10T07:21:44) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b (Updated: 2025-07-11T07:21:43 [TS: 1752218503] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b (Updated: 2025-07-11T07:21:43) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 (Updated: 2025-07-12T07:20:14 [TS: 1752304814] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 (Updated: 2025-07-12T07:20:14) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 (Updated: 2025-07-13T07:22:16 [TS: 1752391336] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 (Updated: 2025-07-13T07:22:16) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 (Updated: 2025-07-14T07:20:21 [TS: 1752477621] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 (Updated: 2025-07-14T07:20:21) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a (Updated: 2025-07-15T07:21:57 [TS: 1752564117] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a (Updated: 2025-07-15T07:21:57) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 (Updated: 2025-07-16T07:20:47 [TS: 1752650447] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 (Updated: 2025-07-16T07:20:47) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d (Updated: 2025-07-17T07:20:43 [TS: 1752736843] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d (Updated: 2025-07-17T07:20:43) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 (Updated: 2025-07-18T07:22:46 [TS: 1752823366] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 (Updated: 2025-07-18T07:22:46) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 (Updated: 2025-07-19T07:21:08 [TS: 1752909668] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 (Updated: 2025-07-19T07:21:08) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 (Updated: 2025-07-20T07:20:44 [TS: 1752996044] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 (Updated: 2025-07-20T07:20:44) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 (Updated: 2025-07-21T07:20:42 [TS: 1753082442] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 (Updated: 2025-07-21T07:20:42) +[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 (Updated: 2025-07-22T07:21:21 [TS: 1753168881] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 (Updated: 2025-07-22T07:21:21) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 (Updated: 2025-07-23T07:19:58 [TS: 1753255198] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 (Updated: 2025-07-23T07:19:58) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 (Updated: 2025-07-24T07:20:48 [TS: 1753341648] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 (Updated: 2025-07-24T07:20:48) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 (Updated: 2025-07-25T07:22:43 [TS: 1753428163] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 (Updated: 2025-07-25T07:22:43) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c (Updated: 2025-07-26T07:20:47 [TS: 1753514447] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c (Updated: 2025-07-26T07:20:47) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a (Updated: 2025-07-27T07:19:47 [TS: 1753600787] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a (Updated: 2025-07-27T07:19:47) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c (Updated: 2025-07-28T07:22:46 [TS: 1753687366] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c (Updated: 2025-07-28T07:22:46) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 (Updated: 2025-07-29T07:23:16 [TS: 1753773796] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 (Updated: 2025-07-29T07:23:16) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e (Updated: 2025-07-30T07:22:14 [TS: 1753860134] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e (Updated: 2025-07-30T07:22:14) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 (Updated: 2025-07-31T07:21:38 [TS: 1753946498] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 (Updated: 2025-07-31T07:21:38) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 (Updated: 2025-08-01T07:20:39 [TS: 1754032839] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 (Updated: 2025-08-01T07:20:39) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 (Updated: 2025-08-02T07:20:51 [TS: 1754119251] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 (Updated: 2025-08-02T07:20:51) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d (Updated: 2025-08-03T07:20:40 [TS: 1754205640] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d (Updated: 2025-08-03T07:20:40) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 (Updated: 2025-08-04T07:21:29 [TS: 1754292089] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 (Updated: 2025-08-04T07:21:29) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 (Updated: 2025-08-05T07:20:11 [TS: 1754378411] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 (Updated: 2025-08-05T07:20:11) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 (Updated: 2025-08-06T07:20:49 [TS: 1754464849] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 (Updated: 2025-08-06T07:20:49) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 (Updated: 2025-08-07T07:21:48 [TS: 1754551308] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 (Updated: 2025-08-07T07:21:48) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 (Updated: 2025-08-08T07:20:18 [TS: 1754637618] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 (Updated: 2025-08-08T07:20:18) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c (Updated: 2025-08-09T07:21:20 [TS: 1754724080] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c (Updated: 2025-08-09T07:21:20) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c (Updated: 2025-08-10T07:21:33 [TS: 1754810493] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c (Updated: 2025-08-10T07:21:33) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e (Updated: 2025-08-11T07:21:44 [TS: 1754896904] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e (Updated: 2025-08-11T07:21:44) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd (Updated: 2025-08-12T07:21:45 [TS: 1754983305] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd (Updated: 2025-08-12T07:21:45) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 (Updated: 2025-08-13T07:20:21 [TS: 1755069621] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 (Updated: 2025-08-13T07:20:21) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 (Updated: 2025-08-14T07:21:25 [TS: 1755156085] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 (Updated: 2025-08-14T07:21:25) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd (Updated: 2025-08-15T07:19:57 [TS: 1755242397] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd (Updated: 2025-08-15T07:19:57) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a (Updated: 2025-08-16T07:20:46 [TS: 1755328846] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a (Updated: 2025-08-16T07:20:46) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 (Updated: 2025-08-17T07:21:09 [TS: 1755415269] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 (Updated: 2025-08-17T07:21:09) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 (Updated: 2025-08-18T07:21:01 [TS: 1755501661] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 (Updated: 2025-08-18T07:21:01) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 (Updated: 2025-08-19T07:20:59 [TS: 1755588059] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 (Updated: 2025-08-19T07:20:59) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 (Updated: 2025-08-20T07:21:18 [TS: 1755674478] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 (Updated: 2025-08-20T07:21:18) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 (Updated: 2025-08-21T07:21:02 [TS: 1755760862] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 (Updated: 2025-08-21T07:21:02) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 (Updated: 2025-08-22T07:20:17 [TS: 1755847217] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 (Updated: 2025-08-22T07:20:17) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e (Updated: 2025-08-23T07:22:26 [TS: 1755933746] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e (Updated: 2025-08-23T07:22:26) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b (Updated: 2025-08-24T07:21:13 [TS: 1756020073] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b (Updated: 2025-08-24T07:21:13) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 (Updated: 2025-08-25T07:23:27 [TS: 1756106607] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 (Updated: 2025-08-25T07:23:27) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 (Updated: 2025-08-26T07:21:52 [TS: 1756192912] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 (Updated: 2025-08-26T07:21:52) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 (Updated: 2025-08-27T07:22:57 [TS: 1756279377] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 (Updated: 2025-08-27T07:22:57) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f (Updated: 2025-08-28T07:23:00 [TS: 1756365780] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f (Updated: 2025-08-28T07:23:00) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c (Updated: 2025-08-29T07:21:43 [TS: 1756452103] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c (Updated: 2025-08-29T07:21:43) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 (Updated: 2025-08-30T07:22:13 [TS: 1756538533] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 (Updated: 2025-08-30T07:22:13) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c (Updated: 2025-08-31T07:21:37 [TS: 1756624897] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c (Updated: 2025-08-31T07:21:37) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e (Updated: 2025-09-01T07:22:45 [TS: 1756711365] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e (Updated: 2025-09-01T07:22:45) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd (Updated: 2025-09-02T07:20:23 [TS: 1756797623] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd (Updated: 2025-09-02T07:20:23) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 (Updated: 2025-09-03T07:22:13 [TS: 1756884133] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 (Updated: 2025-09-03T07:22:13) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 (Updated: 2025-09-04T07:19:51 [TS: 1756970391] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 (Updated: 2025-09-04T07:19:51) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc (Updated: 2025-09-05T07:20:48 [TS: 1757056848] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc (Updated: 2025-09-05T07:20:48) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 (Updated: 2025-09-06T07:21:41 [TS: 1757143301] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 (Updated: 2025-09-06T07:21:41) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 (Updated: 2025-09-07T07:22:23 [TS: 1757229743] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 (Updated: 2025-09-07T07:22:23) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 (Updated: 2025-09-08T07:22:09 [TS: 1757316129] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 (Updated: 2025-09-08T07:22:09) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 (Updated: 2025-09-09T07:22:21 [TS: 1757402541] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 (Updated: 2025-09-09T07:22:21) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 (Updated: 2025-09-10T07:22:45 [TS: 1757488965] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 (Updated: 2025-09-10T07:22:45) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b (Updated: 2025-09-11T07:21:36 [TS: 1757575296] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b (Updated: 2025-09-11T07:21:36) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 (Updated: 2025-09-12T07:21:47 [TS: 1757661707] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 (Updated: 2025-09-12T07:21:47) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df (Updated: 2025-09-13T07:22:23 [TS: 1757748143] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df (Updated: 2025-09-13T07:22:23) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 (Updated: 2025-09-14T07:21:44 [TS: 1757834504] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 (Updated: 2025-09-14T07:21:44) +[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 (Updated: 2025-09-15T07:21:21 [TS: 1757920881] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 (Updated: 2025-09-15T07:21:21) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa (Updated: 2025-09-16T07:18:40 [TS: 1758007120] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa (Updated: 2025-09-16T07:18:40) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 (Updated: 2025-09-17T07:22:13 [TS: 1758093733] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 (Updated: 2025-09-17T07:22:13) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 (Updated: 2025-09-18T07:22:40 [TS: 1758180160] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 (Updated: 2025-09-18T07:22:40) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 (Updated: 2025-09-19T07:20:23 [TS: 1758266423] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 (Updated: 2025-09-19T07:20:23) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 (Updated: 2025-09-20T07:21:10 [TS: 1758352870] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 (Updated: 2025-09-20T07:21:10) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 (Updated: 2025-09-21T07:22:13 [TS: 1758439333] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 (Updated: 2025-09-21T07:22:13) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e (Updated: 2025-09-22T07:21:54 [TS: 1758525714] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e (Updated: 2025-09-22T07:21:54) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 (Updated: 2025-09-23T07:23:02 [TS: 1758612182] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 (Updated: 2025-09-23T07:23:02) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 (Updated: 2025-09-23T07:23:05 [TS: 1758612185] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 (Updated: 2025-09-23T07:23:05) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 (Updated: 2025-09-24T07:20:14 [TS: 1758698414] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 (Updated: 2025-09-24T07:20:14) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 (Updated: 2025-09-24T07:20:17 [TS: 1758698417] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 (Updated: 2025-09-24T07:20:17) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 (Updated: 2025-09-25T07:21:15 [TS: 1758784875] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 (Updated: 2025-09-25T07:21:15) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae (Updated: 2025-09-25T07:21:18 [TS: 1758784878] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae (Updated: 2025-09-25T07:21:18) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 (Updated: 2025-10-09T07:22:47 [TS: 1759994567] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 (Updated: 2025-10-09T07:22:47) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda (Updated: 2025-10-09T07:22:51 [TS: 1759994571] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda (Updated: 2025-10-09T07:22:51) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef (Updated: 2025-10-10T07:21:41 [TS: 1760080901] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef (Updated: 2025-10-10T07:21:41) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c (Updated: 2025-10-10T07:21:44 [TS: 1760080904] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c (Updated: 2025-10-10T07:21:44) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 (Updated: 2025-10-11T07:21:16 [TS: 1760167276] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 (Updated: 2025-10-11T07:21:16) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e (Updated: 2025-10-11T07:21:19 [TS: 1760167279] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e (Updated: 2025-10-11T07:21:19) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 (Updated: 2025-10-12T07:20:46 [TS: 1760253646] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 (Updated: 2025-10-12T07:20:46) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be (Updated: 2025-10-12T07:20:50 [TS: 1760253650] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be (Updated: 2025-10-12T07:20:50) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca (Updated: 2025-10-13T07:22:50 [TS: 1760340170] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca (Updated: 2025-10-13T07:22:50) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef (Updated: 2025-10-13T07:22:53 [TS: 1760340173] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef (Updated: 2025-10-13T07:22:53) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 (Updated: 2025-10-14T07:21:23 [TS: 1760426483] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 (Updated: 2025-10-14T07:21:23) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 (Updated: 2025-10-14T07:21:26 [TS: 1760426486] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 (Updated: 2025-10-14T07:21:26) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 (Updated: 2025-10-15T07:21:13 [TS: 1760512873] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 (Updated: 2025-10-15T07:21:13) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c (Updated: 2025-10-15T07:21:17 [TS: 1760512877] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c (Updated: 2025-10-15T07:21:17) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 (Updated: 2025-10-16T07:20:31 [TS: 1760599231] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 (Updated: 2025-10-16T07:20:31) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 (Updated: 2025-10-16T07:20:37 [TS: 1760599237] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 (Updated: 2025-10-16T07:20:37) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be (Updated: 2025-10-17T07:22:22 [TS: 1760685742] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be (Updated: 2025-10-17T07:22:22) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 (Updated: 2025-10-17T07:22:28 [TS: 1760685748] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 (Updated: 2025-10-17T07:22:28) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 (Updated: 2025-10-18T07:19:49 [TS: 1760771989] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 (Updated: 2025-10-18T07:19:49) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 (Updated: 2025-10-18T07:19:55 [TS: 1760771995] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 (Updated: 2025-10-18T07:19:55) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 (Updated: 2025-10-19T07:20:50 [TS: 1760858450] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 (Updated: 2025-10-19T07:20:50) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 (Updated: 2025-10-19T07:20:57 [TS: 1760858457] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 (Updated: 2025-10-19T07:20:57) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a (Updated: 2025-10-20T07:22:34 [TS: 1760944954] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a (Updated: 2025-10-20T07:22:34) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 (Updated: 2025-10-20T07:22:40 [TS: 1760944960] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 (Updated: 2025-10-20T07:22:40) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 (Updated: 2025-10-21T07:23:22 [TS: 1761031402] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 (Updated: 2025-10-21T07:23:22) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 (Updated: 2025-10-21T07:23:25 [TS: 1761031405] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 (Updated: 2025-10-21T07:23:25) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 (Updated: 2025-10-22T07:21:05 [TS: 1761117665] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 (Updated: 2025-10-22T07:21:05) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 (Updated: 2025-10-22T07:21:08 [TS: 1761117668] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 (Updated: 2025-10-22T07:21:08) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 (Updated: 2025-10-23T07:21:12 [TS: 1761204072] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 (Updated: 2025-10-23T07:21:12) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 (Updated: 2025-10-23T07:21:15 [TS: 1761204075] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 (Updated: 2025-10-23T07:21:15) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 (Updated: 2025-10-24T07:21:00 [TS: 1761290460] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 (Updated: 2025-10-24T07:21:00) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f (Updated: 2025-10-24T07:21:03 [TS: 1761290463] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f (Updated: 2025-10-24T07:21:03) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 (Updated: 2025-10-25T07:22:28 [TS: 1761376948] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 (Updated: 2025-10-25T07:22:28) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 (Updated: 2025-10-25T07:22:32 [TS: 1761376952] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 (Updated: 2025-10-25T07:22:32) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 (Updated: 2025-10-26T07:17:54 [TS: 1761463074] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 (Updated: 2025-10-26T07:17:54) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 (Updated: 2025-10-26T07:18:06 [TS: 1761463086] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 (Updated: 2025-10-26T07:18:06) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e (Updated: 2025-10-27T07:20:57 [TS: 1761549657] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e (Updated: 2025-10-27T07:20:57) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e (Updated: 2025-10-27T07:21:00 [TS: 1761549660] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e (Updated: 2025-10-27T07:21:00) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 (Updated: 2025-10-28T07:21:03 [TS: 1761636063] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 (Updated: 2025-10-28T07:21:03) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c (Updated: 2025-10-28T07:21:09 [TS: 1761636069] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c (Updated: 2025-10-28T07:21:09) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a (Updated: 2025-10-29T07:21:55 [TS: 1761722515] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a (Updated: 2025-10-29T07:21:55) +[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 (Updated: 2025-10-29T07:22:01 [TS: 1761722521] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 (Updated: 2025-10-29T07:22:01) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 (Updated: 2025-10-30T07:21:11 [TS: 1761808871] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 (Updated: 2025-10-30T07:21:11) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf (Updated: 2025-10-30T07:21:17 [TS: 1761808877] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf (Updated: 2025-10-30T07:21:17) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 (Updated: 2025-10-31T07:22:12 [TS: 1761895332] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 (Updated: 2025-10-31T07:22:12) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 (Updated: 2025-10-31T07:22:18 [TS: 1761895338] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 (Updated: 2025-10-31T07:22:18) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa (Updated: 2025-11-01T07:22:50 [TS: 1761981770] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa (Updated: 2025-11-01T07:22:50) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 (Updated: 2025-11-01T07:22:54 [TS: 1761981774] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 (Updated: 2025-11-01T07:22:54) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 (Updated: 2025-11-02T07:21:16 [TS: 1762068076] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 (Updated: 2025-11-02T07:21:16) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 (Updated: 2025-11-02T07:21:20 [TS: 1762068080] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 (Updated: 2025-11-02T07:21:20) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 (Updated: 2025-11-03T08:23:08 [TS: 1762158188] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 (Updated: 2025-11-03T08:23:08) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a (Updated: 2025-11-03T08:23:12 [TS: 1762158192] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a (Updated: 2025-11-03T08:23:12) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 (Updated: 2025-11-04T08:17:45 [TS: 1762244265] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 (Updated: 2025-11-04T08:17:45) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 (Updated: 2025-11-04T08:17:48 [TS: 1762244268] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 (Updated: 2025-11-04T08:17:48) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 (Updated: 2025-11-05T08:20:13 [TS: 1762330813] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 (Updated: 2025-11-05T08:20:13) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 (Updated: 2025-11-05T08:20:17 [TS: 1762330817] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 (Updated: 2025-11-05T08:20:17) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a (Updated: 2025-11-06T08:20:55 [TS: 1762417255] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a (Updated: 2025-11-06T08:20:55) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 (Updated: 2025-11-06T08:20:58 [TS: 1762417258] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 (Updated: 2025-11-06T08:20:58) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd (Updated: 2025-11-07T08:18:30 [TS: 1762503510] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd (Updated: 2025-11-07T08:18:30) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a (Updated: 2025-11-07T08:18:33 [TS: 1762503513] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a (Updated: 2025-11-07T08:18:33) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af (Updated: 2025-11-08T08:18:22 [TS: 1762589902] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af (Updated: 2025-11-08T08:18:22) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 (Updated: 2025-11-08T08:18:26 [TS: 1762589906] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 (Updated: 2025-11-08T08:18:26) +[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd (Updated: 2025-11-09T08:21:42 [TS: 1762676502] < Cutoff: [TS: 1763306705]) +[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd (Updated: 2025-11-09T08:21:42) +[2025-11-30 15:25:13] [INFO] Hit delete limit (200) for Docker Images. +[2025-11-30 15:25:13] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 15:25:13] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 15:25:15] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 15:25:15] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 15:25:15] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 15:25:15] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 15:25:15] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 15:25:15] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 15:25:18] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 15:25:18] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:25:20] [INFO] No Regional Address found matching criteria. +[2025-11-30 15:25:20] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:25:23] [INFO] No Global Address found matching criteria. +[2025-11-30 15:25:23] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- +[2025-11-30 15:25:27] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 15:25:30] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:25:30] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:25:30] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 15:25:30] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 15:25:30] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:33] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 15:25:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:25:35] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 15:25:37] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 15:25:37] [INFO] CLEANUP RUN FINISHED +[2025-11-30 15:26:40] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 15:26:40] [INFO] Time Cutoff (General): 2025-11-30T15:26:40+0000 +[2025-11-30 15:26:40] [INFO] Time Cutoff (Images): 2025-10-01T15:26:40+0000 +[2025-11-30 15:26:40] [INFO] Delete Limit per Type: 200 +[2025-11-30 15:26:40] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 15:26:41] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 15:26:44] [INFO] No Service Accounts found matching prefix. +[2025-11-30 15:26:44] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:26:46] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 15:26:46] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 15:26:49] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:26:49] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:26:49] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 15:26:49] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:26:52] [INFO] No Filestore instances found matching criteria. +[2025-11-30 15:26:52] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 15:26:55] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 15:26:55] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 15:26:55] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 15:26:55] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 15:26:55] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 15:26:55] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 15:26:55] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 15:26:55] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 15:26:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 15:26:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 15:26:56] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 15:26:56] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:26:56Z (Unix: 1763306816) +[2025-11-30 15:26:56] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 15:27:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 (Updated: 2025-05-15T07:21:31 [TS: 1747293691] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 (Updated: 2025-05-15T07:21:31) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/dd64c01e-245f-411d-af04-01a07018e5fd] to complete... +......done. +[2025-11-30 15:27:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 +[2025-11-30 15:27:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 (Updated: 2025-05-16T07:21:17 [TS: 1747380077] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 (Updated: 2025-05-16T07:21:17) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0e68f557-1e12-444c-af99-5d75e4759d05] to complete... +.....done. +[2025-11-30 15:27:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 +[2025-11-30 15:27:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da (Updated: 2025-05-17T07:21:15 [TS: 1747466475] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da (Updated: 2025-05-17T07:21:15) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f8eff88c-80b2-4d55-92c2-76868b25fa98] to complete... +.....done. +[2025-11-30 15:27:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da +[2025-11-30 15:27:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d (Updated: 2025-05-18T07:21:43 [TS: 1747552903] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d (Updated: 2025-05-18T07:21:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6909be50-62d0-438d-9fdc-743de9d1ff34] to complete... +.....done. +[2025-11-30 15:27:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d +[2025-11-30 15:27:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de (Updated: 2025-05-19T07:21:57 [TS: 1747639317] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de (Updated: 2025-05-19T07:21:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/88c0a1a5-03ba-45a4-bdc7-6cc526928dd3] to complete... +.....done. +[2025-11-30 15:27:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de +[2025-11-30 15:27:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 (Updated: 2025-05-20T07:22:12 [TS: 1747725732] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 (Updated: 2025-05-20T07:22:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/77bb28e8-ba96-4a90-9896-0ffac7d5b846] to complete... +.....done. +[2025-11-30 15:27:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 +[2025-11-30 15:27:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 (Updated: 2025-05-21T07:19:51 [TS: 1747811991] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 (Updated: 2025-05-21T07:19:51) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7ab4dc78-adba-489a-adea-5afd2f7d628d] to complete... +.....done. +[2025-11-30 15:27:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 +[2025-11-30 15:27:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f (Updated: 2025-05-22T07:21:21 [TS: 1747898481] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f (Updated: 2025-05-22T07:21:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b033446a-9573-424a-b53e-7f6731ff4926] to complete... +.....done. +[2025-11-30 15:27:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f +[2025-11-30 15:27:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 (Updated: 2025-05-23T07:22:28 [TS: 1747984948] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 (Updated: 2025-05-23T07:22:28) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/12347116-b964-4e06-b05b-8f7444f3d1cd] to complete... +.....done. +[2025-11-30 15:27:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 +[2025-11-30 15:27:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c (Updated: 2025-05-24T07:21:57 [TS: 1748071317] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c (Updated: 2025-05-24T07:21:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/596db79a-b74d-45bd-aa19-e790ea172040] to complete... +......done. +[2025-11-30 15:27:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c +[2025-11-30 15:27:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be (Updated: 2025-05-25T07:21:23 [TS: 1748157683] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be (Updated: 2025-05-25T07:21:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/50afc1a9-eb5c-4f4f-ad2a-e577c45983c7] to complete... +.....done. +[2025-11-30 15:27:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be +[2025-11-30 15:27:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 (Updated: 2025-05-26T07:28:04 [TS: 1748244484] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 (Updated: 2025-05-26T07:28:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5caef63e-0db0-4f6d-b9f6-83be2350dc0a] to complete... +.....done. +[2025-11-30 15:27:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 +[2025-11-30 15:27:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 (Updated: 2025-05-27T07:19:54 [TS: 1748330394] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 (Updated: 2025-05-27T07:19:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/82deb512-b38d-4efa-9826-ef5a78bcbc10] to complete... +.....done. +[2025-11-30 15:27:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 +[2025-11-30 15:27:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a (Updated: 2025-05-28T07:20:28 [TS: 1748416828] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a (Updated: 2025-05-28T07:20:28) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/51755698-a4a5-4c79-9f6f-1918e2c8b793] to complete... +.....done. +[2025-11-30 15:27:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a +[2025-11-30 15:27:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d (Updated: 2025-05-29T07:21:22 [TS: 1748503282] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d (Updated: 2025-05-29T07:21:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2335d05b-70b7-4404-8dc7-7e9451ba2ac4] to complete... +.....done. +[2025-11-30 15:27:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d +[2025-11-30 15:27:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 (Updated: 2025-05-30T07:21:07 [TS: 1748589667] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 (Updated: 2025-05-30T07:21:07) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/061ee211-0be7-4971-99c4-17e0a30bed95] to complete... +.....done. +[2025-11-30 15:27:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 +[2025-11-30 15:27:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 (Updated: 2025-05-31T07:20:46 [TS: 1748676046] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:27:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 (Updated: 2025-05-31T07:20:46) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ab03e21b-4029-419f-abd5-b1e2ec1c8b65] to complete... +.....done. +[2025-11-30 15:28:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 +[2025-11-30 15:28:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 (Updated: 2025-06-01T07:20:52 [TS: 1748762452] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 (Updated: 2025-06-01T07:20:52) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/362825fc-af7b-4921-8dc0-728be3e5d609] to complete... +.....done. +[2025-11-30 15:28:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 +[2025-11-30 15:28:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a (Updated: 2025-06-02T07:23:55 [TS: 1748849035] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a (Updated: 2025-06-02T07:23:55) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f5065687-1a44-4b5a-b165-d44e5e2681a0] to complete... +......done. +[2025-11-30 15:28:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a +[2025-11-30 15:28:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 (Updated: 2025-06-03T07:21:05 [TS: 1748935265] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 (Updated: 2025-06-03T07:21:05) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3e5cb436-cd37-44bc-b0af-317f1f5abc60] to complete... +.....done. +[2025-11-30 15:28:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 +[2025-11-30 15:28:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd (Updated: 2025-06-04T07:22:23 [TS: 1749021743] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd (Updated: 2025-06-04T07:22:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/80c687e8-979b-4b48-a497-748e018f7170] to complete... +......done. +[2025-11-30 15:28:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd +[2025-11-30 15:28:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 (Updated: 2025-06-05T07:21:27 [TS: 1749108087] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 (Updated: 2025-06-05T07:21:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/266d57b5-0a51-4fea-a982-d8518e68319a] to complete... +.....done. +[2025-11-30 15:28:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 +[2025-11-30 15:28:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf (Updated: 2025-06-06T07:22:27 [TS: 1749194547] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf (Updated: 2025-06-06T07:22:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4d27c469-2f39-4c46-9db2-1e17880f017e] to complete... +.....done. +[2025-11-30 15:28:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf +[2025-11-30 15:28:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c (Updated: 2025-06-07T07:22:15 [TS: 1749280935] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c (Updated: 2025-06-07T07:22:15) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bacc7e17-11de-440e-9fc0-d431e5dfe8a6] to complete... +.....done. +[2025-11-30 15:28:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c +[2025-11-30 15:28:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 (Updated: 2025-06-08T07:21:06 [TS: 1749367266] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 (Updated: 2025-06-08T07:21:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a6f0d1c7-96dc-438c-ae33-0d1df8014aee] to complete... +.....done. +[2025-11-30 15:28:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 +[2025-11-30 15:28:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f (Updated: 2025-06-09T07:20:04 [TS: 1749453604] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f (Updated: 2025-06-09T07:20:04) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b5467a09-5203-4b85-b82f-5f99b515eb44] to complete... +.....done. +[2025-11-30 15:28:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f +[2025-11-30 15:28:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f (Updated: 2025-06-10T07:20:54 [TS: 1749540054] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f (Updated: 2025-06-10T07:20:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2a09dca9-84b8-4153-a3cb-5b2aca6140fb] to complete... +.....done. +[2025-11-30 15:28:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f +[2025-11-30 15:28:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 (Updated: 2025-06-11T07:20:59 [TS: 1749626459] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 (Updated: 2025-06-11T07:20:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/547c8e26-dbe6-49f2-808e-f0f67f883784] to complete... +......done. +[2025-11-30 15:28:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 +[2025-11-30 15:28:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 (Updated: 2025-06-12T07:19:36 [TS: 1749712776] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 (Updated: 2025-06-12T07:19:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9a3cc358-a13e-4216-b6cd-d3077eed7c2e] to complete... +.....done. +[2025-11-30 15:28:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 +[2025-11-30 15:28:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 (Updated: 2025-06-13T07:21:52 [TS: 1749799312] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 (Updated: 2025-06-13T07:21:52) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/81e696f9-e0ec-4326-89c7-623f85f250eb] to complete... +.....done. +[2025-11-30 15:28:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 +[2025-11-30 15:28:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 (Updated: 2025-06-14T07:21:53 [TS: 1749885713] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 (Updated: 2025-06-14T07:21:53) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/616e1f3d-51f5-4ebe-851d-46266a5723c4] to complete... +.....done. +[2025-11-30 15:28:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 +[2025-11-30 15:28:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 (Updated: 2025-06-15T07:20:57 [TS: 1749972057] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 (Updated: 2025-06-15T07:20:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3f478b30-d35a-47b1-a9cd-1e391ff0ee30] to complete... +.....done. +[2025-11-30 15:28:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 +[2025-11-30 15:28:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 (Updated: 2025-06-16T07:20:18 [TS: 1750058418] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 (Updated: 2025-06-16T07:20:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1c76ab08-3161-4401-895b-3ade123d1890] to complete... +......done. +[2025-11-30 15:28:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 +[2025-11-30 15:28:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe (Updated: 2025-06-17T07:21:49 [TS: 1750144909] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:28:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe (Updated: 2025-06-17T07:21:49) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/05a89e17-f248-4c3e-8e9b-03d1044cfcdc] to complete... +.....done. +[2025-11-30 15:29:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe +[2025-11-30 15:29:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 (Updated: 2025-06-18T07:20:13 [TS: 1750231213] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 (Updated: 2025-06-18T07:20:13) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/25101083-fdb8-4b84-85bd-67cabd30fd9a] to complete... +.....done. +[2025-11-30 15:29:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 +[2025-11-30 15:29:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 (Updated: 2025-06-19T07:20:59 [TS: 1750317659] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 (Updated: 2025-06-19T07:20:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b645df05-937c-4faa-8fd2-45fd27356faa] to complete... +......done. +[2025-11-30 15:29:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 +[2025-11-30 15:29:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b (Updated: 2025-06-20T07:21:01 [TS: 1750404061] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b (Updated: 2025-06-20T07:21:01) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4ac9df5b-c155-4ab4-af44-cbe3a2a21984] to complete... +.....done. +[2025-11-30 15:29:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b +[2025-11-30 15:29:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f (Updated: 2025-06-21T07:22:20 [TS: 1750490540] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f (Updated: 2025-06-21T07:22:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d3a9e7c9-7c07-476a-8fb8-46fd7e86b495] to complete... +.....done. +[2025-11-30 15:29:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f +[2025-11-30 15:29:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb (Updated: 2025-06-22T07:21:21 [TS: 1750576881] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb (Updated: 2025-06-22T07:21:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eb89a0c0-0956-4658-af00-ef87c5d94c85] to complete... +.....done. +[2025-11-30 15:29:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb +[2025-11-30 15:29:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 (Updated: 2025-06-23T07:21:18 [TS: 1750663278] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 (Updated: 2025-06-23T07:21:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/315d43a0-bcf1-416c-add0-6ce5a883034f] to complete... +......done. +[2025-11-30 15:29:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 +[2025-11-30 15:29:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 (Updated: 2025-06-24T07:21:15 [TS: 1750749675] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 (Updated: 2025-06-24T07:21:15) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6773eb36-884d-45a7-8739-c4e5290b12ba] to complete... +.....done. +[2025-11-30 15:29:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 +[2025-11-30 15:29:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 (Updated: 2025-06-25T07:21:11 [TS: 1750836071] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 (Updated: 2025-06-25T07:21:11) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eb6b2f8d-57de-4d56-a554-7905f7a31f6b] to complete... +.....done. +[2025-11-30 15:29:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 +[2025-11-30 15:29:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba (Updated: 2025-06-26T07:20:29 [TS: 1750922429] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba (Updated: 2025-06-26T07:20:29) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/db1c55b0-9f43-40ab-b8f0-f2677771e2ab] to complete... +.....done. +[2025-11-30 15:29:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba +[2025-11-30 15:29:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c (Updated: 2025-06-27T07:20:18 [TS: 1751008818] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c (Updated: 2025-06-27T07:20:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ee175c44-cc0f-464f-8ef7-36d20de2c9a9] to complete... +.....done. +[2025-11-30 15:29:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c +[2025-11-30 15:29:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb (Updated: 2025-06-28T07:20:26 [TS: 1751095226] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb (Updated: 2025-06-28T07:20:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a55b3ca4-eda6-4e25-b245-e3bf9e6e4ab9] to complete... +.....done. +[2025-11-30 15:29:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb +[2025-11-30 15:29:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c (Updated: 2025-06-29T07:21:11 [TS: 1751181671] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c (Updated: 2025-06-29T07:21:11) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/72c895ea-3cd7-4ffd-ad23-ac104f0b998d] to complete... +.....done. +[2025-11-30 15:29:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c +[2025-11-30 15:29:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 (Updated: 2025-06-30T07:22:37 [TS: 1751268157] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 (Updated: 2025-06-30T07:22:37) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/413fa74f-8f29-44d9-921f-97f856b30267] to complete... +.....done. +[2025-11-30 15:29:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 +[2025-11-30 15:29:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f (Updated: 2025-07-01T07:20:54 [TS: 1751354454] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f (Updated: 2025-07-01T07:20:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/89a43850-b016-42ed-82f5-6d68726e2a18] to complete... +.....done. +[2025-11-30 15:29:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f +[2025-11-30 15:29:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 (Updated: 2025-07-02T07:21:52 [TS: 1751440912] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 (Updated: 2025-07-02T07:21:52) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0e040ef4-b2df-421e-aced-9f20b59fe47f] to complete... +.....done. +[2025-11-30 15:29:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 +[2025-11-30 15:29:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb (Updated: 2025-07-03T07:20:06 [TS: 1751527206] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb (Updated: 2025-07-03T07:20:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/269b3e69-07cd-4636-ba32-48dc62e4d61f] to complete... +.....done. +[2025-11-30 15:29:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb +[2025-11-30 15:29:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e (Updated: 2025-07-04T07:21:34 [TS: 1751613694] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:29:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e (Updated: 2025-07-04T07:21:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/367e73c5-2fa3-489e-affe-5c182ef1f215] to complete... +.....done. +[2025-11-30 15:30:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e +[2025-11-30 15:30:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 (Updated: 2025-07-05T07:21:32 [TS: 1751700092] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 (Updated: 2025-07-05T07:21:32) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6616a7bd-fb9f-4a49-903a-d1e16c032f3b] to complete... +......done. +[2025-11-30 15:30:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 +[2025-11-30 15:30:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf (Updated: 2025-07-06T07:21:21 [TS: 1751786481] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf (Updated: 2025-07-06T07:21:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/416917dd-c2cf-4c24-aa04-a960b5f37eee] to complete... +......done. +[2025-11-30 15:30:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf +[2025-11-30 15:30:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 (Updated: 2025-07-07T07:22:25 [TS: 1751872945] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 (Updated: 2025-07-07T07:22:25) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9814478d-649b-44b9-b160-8db3a4c81c7d] to complete... +.....done. +[2025-11-30 15:30:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 +[2025-11-30 15:30:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 (Updated: 2025-07-08T07:21:26 [TS: 1751959286] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 (Updated: 2025-07-08T07:21:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/46bd48bd-826d-466c-932c-7a9c52c1adef] to complete... +.....done. +[2025-11-30 15:30:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 +[2025-11-30 15:30:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 (Updated: 2025-07-09T07:22:45 [TS: 1752045765] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 (Updated: 2025-07-09T07:22:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f1eb9bb1-e3f7-4c90-b5f5-aeb47edb2c17] to complete... +.....done. +[2025-11-30 15:30:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 +[2025-11-30 15:30:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 (Updated: 2025-07-10T07:21:44 [TS: 1752132104] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 (Updated: 2025-07-10T07:21:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c4d28673-55fc-4d6e-825d-b7b0d4b5c401] to complete... +......done. +[2025-11-30 15:30:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 +[2025-11-30 15:30:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b (Updated: 2025-07-11T07:21:43 [TS: 1752218503] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b (Updated: 2025-07-11T07:21:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7241bee7-7f5c-4441-bd29-9fa3de2687b1] to complete... +.....done. +[2025-11-30 15:30:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b +[2025-11-30 15:30:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 (Updated: 2025-07-12T07:20:14 [TS: 1752304814] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 (Updated: 2025-07-12T07:20:14) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d81fc468-5e43-4e2c-92dd-18fe2bdbaec1] to complete... +.....done. +[2025-11-30 15:30:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 +[2025-11-30 15:30:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 (Updated: 2025-07-13T07:22:16 [TS: 1752391336] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 (Updated: 2025-07-13T07:22:16) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ebe9fd6c-3195-4c45-b7ca-cea46b9a181a] to complete... +.....done. +[2025-11-30 15:30:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 +[2025-11-30 15:30:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 (Updated: 2025-07-14T07:20:21 [TS: 1752477621] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 (Updated: 2025-07-14T07:20:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6d0f1a80-677b-474d-a79a-fb2ab8bcb6fb] to complete... +.....done. +[2025-11-30 15:30:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 +[2025-11-30 15:30:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a (Updated: 2025-07-15T07:21:57 [TS: 1752564117] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a (Updated: 2025-07-15T07:21:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/068cb159-da1a-4671-8865-308cffb998ae] to complete... +......done. +[2025-11-30 15:30:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a +[2025-11-30 15:30:41] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 (Updated: 2025-07-16T07:20:47 [TS: 1752650447] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:41] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 (Updated: 2025-07-16T07:20:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d5e9cf1c-eff4-457e-8291-4e32780485a9] to complete... +.....done. +[2025-11-30 15:30:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 +[2025-11-30 15:30:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d (Updated: 2025-07-17T07:20:43 [TS: 1752736843] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d (Updated: 2025-07-17T07:20:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ed86b53e-e238-4add-b8e4-0b1c6302f8c1] to complete... +.....done. +[2025-11-30 15:30:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d +[2025-11-30 15:30:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 (Updated: 2025-07-18T07:22:46 [TS: 1752823366] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 (Updated: 2025-07-18T07:22:46) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/362e6ff0-7f50-4f9d-8b9d-c9a0f7e8b7a0] to complete... +.....done. +[2025-11-30 15:30:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 +[2025-11-30 15:30:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 (Updated: 2025-07-19T07:21:08 [TS: 1752909668] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 (Updated: 2025-07-19T07:21:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c9676628-863d-4941-bd0c-68e07fed481e] to complete... +.....done. +[2025-11-30 15:30:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 +[2025-11-30 15:30:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 (Updated: 2025-07-20T07:20:44 [TS: 1752996044] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 (Updated: 2025-07-20T07:20:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/32b5f987-3fb9-4714-9976-3e9e46fb4af5] to complete... +.....done. +[2025-11-30 15:30:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 +[2025-11-30 15:30:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 (Updated: 2025-07-21T07:20:42 [TS: 1753082442] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:30:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 (Updated: 2025-07-21T07:20:42) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/aafd4a78-d13e-4d74-ade5-45a598f08473] to complete... +.....done. +[2025-11-30 15:31:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 +[2025-11-30 15:31:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 (Updated: 2025-07-22T07:21:21 [TS: 1753168881] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 (Updated: 2025-07-22T07:21:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f59135cf-c887-4cb6-8744-82a38010d581] to complete... +.....done. +[2025-11-30 15:31:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 +[2025-11-30 15:31:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 (Updated: 2025-07-23T07:19:58 [TS: 1753255198] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 (Updated: 2025-07-23T07:19:58) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/18bbb4a6-db80-4b3e-8749-370e1f4122f2] to complete... +.....done. +[2025-11-30 15:31:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 +[2025-11-30 15:31:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 (Updated: 2025-07-24T07:20:48 [TS: 1753341648] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 (Updated: 2025-07-24T07:20:48) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/908b321c-53c9-4d5c-83b9-038a3aaf0169] to complete... +......done. +[2025-11-30 15:31:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 +[2025-11-30 15:31:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 (Updated: 2025-07-25T07:22:43 [TS: 1753428163] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 (Updated: 2025-07-25T07:22:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/07b26725-df87-4f42-8823-19e510a41785] to complete... +......done. +[2025-11-30 15:31:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 +[2025-11-30 15:31:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c (Updated: 2025-07-26T07:20:47 [TS: 1753514447] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c (Updated: 2025-07-26T07:20:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/07c7a440-7803-4e55-8b67-a28f6c892542] to complete... +......done. +[2025-11-30 15:31:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c +[2025-11-30 15:31:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a (Updated: 2025-07-27T07:19:47 [TS: 1753600787] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a (Updated: 2025-07-27T07:19:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6840aad2-7da6-4f1e-bf40-c37bb52e136c] to complete... +......done. +[2025-11-30 15:31:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a +[2025-11-30 15:31:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c (Updated: 2025-07-28T07:22:46 [TS: 1753687366] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c (Updated: 2025-07-28T07:22:46) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/53fff091-b58d-46fa-af18-4288c48e4249] to complete... +.....done. +[2025-11-30 15:31:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c +[2025-11-30 15:31:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 (Updated: 2025-07-29T07:23:16 [TS: 1753773796] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 (Updated: 2025-07-29T07:23:16) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b6f40155-01ee-48a3-bb59-cee8eb1f30be] to complete... +.....done. +[2025-11-30 15:31:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 +[2025-11-30 15:31:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e (Updated: 2025-07-30T07:22:14 [TS: 1753860134] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e (Updated: 2025-07-30T07:22:14) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bc77c010-ea51-4243-9e09-2b2ab5bc79ae] to complete... +.....done. +[2025-11-30 15:31:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e +[2025-11-30 15:31:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 (Updated: 2025-07-31T07:21:38 [TS: 1753946498] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 (Updated: 2025-07-31T07:21:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9579e56a-1e06-45ab-86bb-de534dfe3784] to complete... +.....done. +[2025-11-30 15:31:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 +[2025-11-30 15:31:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 (Updated: 2025-08-01T07:20:39 [TS: 1754032839] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 (Updated: 2025-08-01T07:20:39) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0c2dd2a7-6980-4238-9c4e-ae5d6a35e450] to complete... +.....done. +[2025-11-30 15:31:41] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 +[2025-11-30 15:31:41] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 (Updated: 2025-08-02T07:20:51 [TS: 1754119251] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:41] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 (Updated: 2025-08-02T07:20:51) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d82adf63-3c14-4780-abc7-bd6f9b56f23a] to complete... +.....done. +[2025-11-30 15:31:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 +[2025-11-30 15:31:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d (Updated: 2025-08-03T07:20:40 [TS: 1754205640] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d (Updated: 2025-08-03T07:20:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/08fc3f03-8a5e-4b84-9eca-91ebab75c6d7] to complete... +.....done. +[2025-11-30 15:31:48] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d +[2025-11-30 15:31:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 (Updated: 2025-08-04T07:21:29 [TS: 1754292089] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 (Updated: 2025-08-04T07:21:29) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7db1396c-60c1-4c5c-a9b9-3577fd4b6e6c] to complete... +.....done. +[2025-11-30 15:31:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 +[2025-11-30 15:31:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 (Updated: 2025-08-05T07:20:11 [TS: 1754378411] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 (Updated: 2025-08-05T07:20:11) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/389b3e2e-8e63-4ddd-86ff-e383674cbaf5] to complete... +.....done. +[2025-11-30 15:31:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 +[2025-11-30 15:31:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 (Updated: 2025-08-06T07:20:49 [TS: 1754464849] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 (Updated: 2025-08-06T07:20:49) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fa0cde11-ddff-4fe9-bbcd-267538765da7] to complete... +.....done. +[2025-11-30 15:31:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 +[2025-11-30 15:31:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 (Updated: 2025-08-07T07:21:48 [TS: 1754551308] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:31:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 (Updated: 2025-08-07T07:21:48) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6d02ffa5-5b49-4082-a5e0-d0581fcf7268] to complete... +.....done. +[2025-11-30 15:32:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 +[2025-11-30 15:32:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 (Updated: 2025-08-08T07:20:18 [TS: 1754637618] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 (Updated: 2025-08-08T07:20:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ce44895f-5e13-46fd-9725-f6d5cd9c89bf] to complete... +.....done. +[2025-11-30 15:32:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 +[2025-11-30 15:32:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c (Updated: 2025-08-09T07:21:20 [TS: 1754724080] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c (Updated: 2025-08-09T07:21:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8b0f84e4-46d1-471a-be65-635ad0e8e5da] to complete... +.....done. +[2025-11-30 15:32:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c +[2025-11-30 15:32:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c (Updated: 2025-08-10T07:21:33 [TS: 1754810493] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c (Updated: 2025-08-10T07:21:33) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/24b1e3ce-7c1e-4802-ba5b-125e90b666d1] to complete... +.....done. +[2025-11-30 15:32:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c +[2025-11-30 15:32:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e (Updated: 2025-08-11T07:21:44 [TS: 1754896904] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e (Updated: 2025-08-11T07:21:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c9663d3b-71b5-45e5-9a11-0d5b6d922317] to complete... +.....done. +[2025-11-30 15:32:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e +[2025-11-30 15:32:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd (Updated: 2025-08-12T07:21:45 [TS: 1754983305] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd (Updated: 2025-08-12T07:21:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2ced74b3-aec1-4f03-896b-00b28e03569a] to complete... +.....done. +[2025-11-30 15:32:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd +[2025-11-30 15:32:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 (Updated: 2025-08-13T07:20:21 [TS: 1755069621] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 (Updated: 2025-08-13T07:20:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/360b70f4-793e-468f-9ca7-bbc6f45eb1a3] to complete... +.....done. +[2025-11-30 15:32:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 +[2025-11-30 15:32:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 (Updated: 2025-08-14T07:21:25 [TS: 1755156085] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 (Updated: 2025-08-14T07:21:25) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1570def0-11ff-406b-8f6e-efb44a28d8be] to complete... +.....done. +[2025-11-30 15:32:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 +[2025-11-30 15:32:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd (Updated: 2025-08-15T07:19:57 [TS: 1755242397] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd (Updated: 2025-08-15T07:19:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/29a50ecc-61d5-4d37-9476-fc586d97e5ae] to complete... +.....done. +[2025-11-30 15:32:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd +[2025-11-30 15:32:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a (Updated: 2025-08-16T07:20:46 [TS: 1755328846] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a (Updated: 2025-08-16T07:20:46) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7a965f06-ea39-46ea-8347-9b5c1eb57fd2] to complete... +.....done. +[2025-11-30 15:32:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a +[2025-11-30 15:32:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 (Updated: 2025-08-17T07:21:09 [TS: 1755415269] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 (Updated: 2025-08-17T07:21:09) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d4b577a7-82ee-4bb0-b9e9-e9527200ac99] to complete... +......done. +[2025-11-30 15:32:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 +[2025-11-30 15:32:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 (Updated: 2025-08-18T07:21:01 [TS: 1755501661] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 (Updated: 2025-08-18T07:21:01) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6317acb1-3206-4377-9df7-46e41e8081b4] to complete... +.....done. +[2025-11-30 15:32:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 +[2025-11-30 15:32:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 (Updated: 2025-08-19T07:20:59 [TS: 1755588059] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 (Updated: 2025-08-19T07:20:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8994b351-13c9-4ad5-9151-af0e006ae554] to complete... +.....done. +[2025-11-30 15:32:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 +[2025-11-30 15:32:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 (Updated: 2025-08-20T07:21:18 [TS: 1755674478] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 (Updated: 2025-08-20T07:21:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0cac5019-7308-4402-a8f8-e838744d8e7c] to complete... +.....done. +[2025-11-30 15:32:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 +[2025-11-30 15:32:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 (Updated: 2025-08-21T07:21:02 [TS: 1755760862] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 (Updated: 2025-08-21T07:21:02) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/14c46048-89ae-436b-806c-23151d3aeca6] to complete... +.....done. +[2025-11-30 15:32:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 +[2025-11-30 15:32:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 (Updated: 2025-08-22T07:20:17 [TS: 1755847217] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 (Updated: 2025-08-22T07:20:17) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6832d74a-d10c-4ed3-a29b-35ce7dcc5372] to complete... +.....done. +[2025-11-30 15:32:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 +[2025-11-30 15:32:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e (Updated: 2025-08-23T07:22:26 [TS: 1755933746] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e (Updated: 2025-08-23T07:22:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a3ef0c15-ea36-4aae-bfdc-2e4a670a82e9] to complete... +.....done. +[2025-11-30 15:32:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e +[2025-11-30 15:32:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b (Updated: 2025-08-24T07:21:13 [TS: 1756020073] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:32:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b (Updated: 2025-08-24T07:21:13) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/33c3810a-8d07-43c4-980f-fcb0032c3296] to complete... +.....done. +[2025-11-30 15:33:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b +[2025-11-30 15:33:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 (Updated: 2025-08-25T07:23:27 [TS: 1756106607] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 (Updated: 2025-08-25T07:23:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0d66e745-d8f4-4d00-a1ee-799f1cc0e42a] to complete... +.....done. +[2025-11-30 15:33:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 +[2025-11-30 15:33:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 (Updated: 2025-08-26T07:21:52 [TS: 1756192912] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 (Updated: 2025-08-26T07:21:52) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/dab1b37a-647b-45eb-8ef9-981fa52bb02f] to complete... +.....done. +[2025-11-30 15:33:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 +[2025-11-30 15:33:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 (Updated: 2025-08-27T07:22:57 [TS: 1756279377] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 (Updated: 2025-08-27T07:22:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/284394f5-ea90-49e6-a514-f0b1482e0e64] to complete... +.....done. +[2025-11-30 15:33:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 +[2025-11-30 15:33:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f (Updated: 2025-08-28T07:23:00 [TS: 1756365780] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f (Updated: 2025-08-28T07:23:00) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/467a04fc-cfcf-467b-bb3e-a486435137f8] to complete... +.....done. +[2025-11-30 15:33:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f +[2025-11-30 15:33:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c (Updated: 2025-08-29T07:21:43 [TS: 1756452103] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c (Updated: 2025-08-29T07:21:43) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/113e4b72-8058-4d4c-9b56-7d23d9680cc8] to complete... +.....done. +[2025-11-30 15:33:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c +[2025-11-30 15:33:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 (Updated: 2025-08-30T07:22:13 [TS: 1756538533] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 (Updated: 2025-08-30T07:22:13) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a16715e0-98ae-4a7d-b761-4388aa7b8d5f] to complete... +.....done. +[2025-11-30 15:33:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 +[2025-11-30 15:33:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c (Updated: 2025-08-31T07:21:37 [TS: 1756624897] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c (Updated: 2025-08-31T07:21:37) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e96b6202-6eeb-459d-aaad-333304f8fdc9] to complete... +.....done. +[2025-11-30 15:33:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c +[2025-11-30 15:33:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e (Updated: 2025-09-01T07:22:45 [TS: 1756711365] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e (Updated: 2025-09-01T07:22:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0ab8537d-2963-4f4a-a5e5-a03b811a0697] to complete... +......done. +[2025-11-30 15:33:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e +[2025-11-30 15:33:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd (Updated: 2025-09-02T07:20:23 [TS: 1756797623] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd (Updated: 2025-09-02T07:20:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8ad31bea-933e-4d49-a2be-b5566c8b2a95] to complete... +.....done. +[2025-11-30 15:33:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd +[2025-11-30 15:33:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 (Updated: 2025-09-03T07:22:13 [TS: 1756884133] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 (Updated: 2025-09-03T07:22:13) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/34a407d5-8f28-4f54-aa4d-ea7952fd263f] to complete... +.....done. +[2025-11-30 15:33:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 +[2025-11-30 15:33:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 (Updated: 2025-09-04T07:19:51 [TS: 1756970391] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 (Updated: 2025-09-04T07:19:51) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/10f4099c-c3b2-4158-8b43-e2037db31a9a] to complete... +.....done. +[2025-11-30 15:33:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 +[2025-11-30 15:33:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc (Updated: 2025-09-05T07:20:48 [TS: 1757056848] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc (Updated: 2025-09-05T07:20:48) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8b213ef9-bcfe-437f-88cc-a5cc43ae7c93] to complete... +.....done. +[2025-11-30 15:33:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc +[2025-11-30 15:33:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 (Updated: 2025-09-06T07:21:41 [TS: 1757143301] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 (Updated: 2025-09-06T07:21:41) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/40789e58-fcf5-4021-a523-63c2e22aa78c] to complete... +.....done. +[2025-11-30 15:33:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 +[2025-11-30 15:33:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 (Updated: 2025-09-07T07:22:23 [TS: 1757229743] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 (Updated: 2025-09-07T07:22:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9b860017-bb21-4b4d-bdcb-3d8d8bf72fb0] to complete... +.....done. +[2025-11-30 15:33:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 +[2025-11-30 15:33:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 (Updated: 2025-09-08T07:22:09 [TS: 1757316129] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 (Updated: 2025-09-08T07:22:09) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b92b7c80-5fa0-40ad-9b3d-c8f8c848095e] to complete... +.....done. +[2025-11-30 15:33:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 +[2025-11-30 15:33:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 (Updated: 2025-09-09T07:22:21 [TS: 1757402541] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 (Updated: 2025-09-09T07:22:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c2072c55-61b9-4337-9f33-79fb0ffd2b4b] to complete... +......done. +[2025-11-30 15:33:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 +[2025-11-30 15:33:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 (Updated: 2025-09-10T07:22:45 [TS: 1757488965] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:33:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 (Updated: 2025-09-10T07:22:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e956c9ae-7b6b-4b04-b077-3fe3804e24a2] to complete... +......done. +[2025-11-30 15:34:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 +[2025-11-30 15:34:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b (Updated: 2025-09-11T07:21:36 [TS: 1757575296] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b (Updated: 2025-09-11T07:21:36) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/de596c8f-d4f5-46b0-a49a-0ae0e296a61c] to complete... +.....done. +[2025-11-30 15:34:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b +[2025-11-30 15:34:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 (Updated: 2025-09-12T07:21:47 [TS: 1757661707] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 (Updated: 2025-09-12T07:21:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a1e0675e-f7ad-4c10-8492-85238658a17b] to complete... +.....done. +[2025-11-30 15:34:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 +[2025-11-30 15:34:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df (Updated: 2025-09-13T07:22:23 [TS: 1757748143] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df (Updated: 2025-09-13T07:22:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cbd9d8ca-e848-490a-883a-886e79395928] to complete... +.....done. +[2025-11-30 15:34:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df +[2025-11-30 15:34:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 (Updated: 2025-09-14T07:21:44 [TS: 1757834504] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 (Updated: 2025-09-14T07:21:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4480d01e-290f-4327-9347-2292b8d499f5] to complete... +.....done. +[2025-11-30 15:34:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 +[2025-11-30 15:34:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 (Updated: 2025-09-15T07:21:21 [TS: 1757920881] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 (Updated: 2025-09-15T07:21:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/021ae34b-955a-4726-b8f4-136adfa25c5c] to complete... +.....done. +[2025-11-30 15:34:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 +[2025-11-30 15:34:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa (Updated: 2025-09-16T07:18:40 [TS: 1758007120] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa (Updated: 2025-09-16T07:18:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8238eb6e-28f7-4182-8bc4-2818ed6773c8] to complete... +.....done. +[2025-11-30 15:34:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa +[2025-11-30 15:34:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 (Updated: 2025-09-17T07:22:13 [TS: 1758093733] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 (Updated: 2025-09-17T07:22:13) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ecb140c8-9b11-497d-aa3d-6a12f1a1ed1d] to complete... +.....done. +[2025-11-30 15:34:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 +[2025-11-30 15:34:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 (Updated: 2025-09-18T07:22:40 [TS: 1758180160] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 (Updated: 2025-09-18T07:22:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8f85b98b-c9b4-437a-b89c-d046a71a2ae0] to complete... +.....done. +[2025-11-30 15:34:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 +[2025-11-30 15:34:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 (Updated: 2025-09-19T07:20:23 [TS: 1758266423] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 (Updated: 2025-09-19T07:20:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f582295f-dea6-4365-bb39-4ccd442a8e0f] to complete... +.....done. +[2025-11-30 15:34:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 +[2025-11-30 15:34:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 (Updated: 2025-09-20T07:21:10 [TS: 1758352870] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 (Updated: 2025-09-20T07:21:10) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0d9bfebf-924e-4e57-9443-45575ed5cded] to complete... +.....done. +[2025-11-30 15:34:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 +[2025-11-30 15:34:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 (Updated: 2025-09-21T07:22:13 [TS: 1758439333] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 (Updated: 2025-09-21T07:22:13) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2b8c84d1-8989-4a3a-9abe-c0df99d441dc] to complete... +.....done. +[2025-11-30 15:34:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 +[2025-11-30 15:34:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e (Updated: 2025-09-22T07:21:54 [TS: 1758525714] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e (Updated: 2025-09-22T07:21:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/24f2581b-b2c6-4328-991b-9f29b88bc76e] to complete... +.....done. +[2025-11-30 15:34:41] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e +[2025-11-30 15:34:41] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 (Updated: 2025-09-23T07:23:02 [TS: 1758612182] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:41] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 (Updated: 2025-09-23T07:23:02) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9554246b-7236-4704-90f3-73d5db54fbf3] to complete... +......done. +[2025-11-30 15:34:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 +[2025-11-30 15:34:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 (Updated: 2025-09-23T07:23:05 [TS: 1758612185] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 (Updated: 2025-09-23T07:23:05) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/488522e0-3b0d-4771-8108-e4fd4b1cc685] to complete... +.....done. +[2025-11-30 15:34:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 +[2025-11-30 15:34:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 (Updated: 2025-09-24T07:20:14 [TS: 1758698414] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 (Updated: 2025-09-24T07:20:14) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8339acfb-a632-49be-aaf0-31ef24649f2a] to complete... +......done. +[2025-11-30 15:34:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 +[2025-11-30 15:34:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 (Updated: 2025-09-24T07:20:17 [TS: 1758698417] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 (Updated: 2025-09-24T07:20:17) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c629ddbf-76ee-47db-bee2-4a75791abbeb] to complete... +.....done. +[2025-11-30 15:34:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 +[2025-11-30 15:34:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 (Updated: 2025-09-25T07:21:15 [TS: 1758784875] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 (Updated: 2025-09-25T07:21:15) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/41c2d152-bfba-4da9-9722-750ba0d6ea95] to complete... +.....done. +[2025-11-30 15:34:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 +[2025-11-30 15:34:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae (Updated: 2025-09-25T07:21:18 [TS: 1758784878] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:34:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae (Updated: 2025-09-25T07:21:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/73551225-09c7-45e2-a837-51a3ec093851] to complete... +.....done. +[2025-11-30 15:35:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae +[2025-11-30 15:35:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 (Updated: 2025-10-09T07:22:47 [TS: 1759994567] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 (Updated: 2025-10-09T07:22:47) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b44f9112-ba3b-4268-9984-ed6b1ec615e8] to complete... +.....done. +[2025-11-30 15:35:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 +[2025-11-30 15:35:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda (Updated: 2025-10-09T07:22:51 [TS: 1759994571] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda (Updated: 2025-10-09T07:22:51) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a76448cb-aba6-43eb-832b-3a21c9bcdb5c] to complete... +.....done. +[2025-11-30 15:35:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda +[2025-11-30 15:35:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef (Updated: 2025-10-10T07:21:41 [TS: 1760080901] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef (Updated: 2025-10-10T07:21:41) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1790af12-f417-4d99-9b88-d4ac2fd8941a] to complete... +.....done. +[2025-11-30 15:35:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef +[2025-11-30 15:35:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c (Updated: 2025-10-10T07:21:44 [TS: 1760080904] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c (Updated: 2025-10-10T07:21:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4a681845-39f9-4baa-a2df-300fdd6d0bf4] to complete... +.....done. +[2025-11-30 15:35:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c +[2025-11-30 15:35:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 (Updated: 2025-10-11T07:21:16 [TS: 1760167276] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 (Updated: 2025-10-11T07:21:16) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b61f91de-55d6-49a9-a167-e6048ec92cd5] to complete... +......done. +[2025-11-30 15:35:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 +[2025-11-30 15:35:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e (Updated: 2025-10-11T07:21:19 [TS: 1760167279] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e (Updated: 2025-10-11T07:21:19) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/107d8573-fcef-4ad9-a1a7-9adb4fd461e0] to complete... +.....done. +[2025-11-30 15:35:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e +[2025-11-30 15:35:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 (Updated: 2025-10-12T07:20:46 [TS: 1760253646] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 (Updated: 2025-10-12T07:20:46) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b7a80d1a-ff03-45d4-8cb3-0769bf6b99f2] to complete... +.....done. +[2025-11-30 15:35:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 +[2025-11-30 15:35:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be (Updated: 2025-10-12T07:20:50 [TS: 1760253650] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be (Updated: 2025-10-12T07:20:50) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c188c146-8f54-44d5-9f3a-5511a0ab2abd] to complete... +.....done. +[2025-11-30 15:35:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be +[2025-11-30 15:35:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca (Updated: 2025-10-13T07:22:50 [TS: 1760340170] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca (Updated: 2025-10-13T07:22:50) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/11cb77c7-2861-40ff-86e9-dff1ac7ca81d] to complete... +.....done. +[2025-11-30 15:35:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca +[2025-11-30 15:35:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef (Updated: 2025-10-13T07:22:53 [TS: 1760340173] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef (Updated: 2025-10-13T07:22:53) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8d869620-afee-4902-9af3-2551965874f6] to complete... +.....done. +[2025-11-30 15:35:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef +[2025-11-30 15:35:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 (Updated: 2025-10-14T07:21:23 [TS: 1760426483] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 (Updated: 2025-10-14T07:21:23) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/914bac45-2917-472f-93e1-ff5f262def91] to complete... +.....done. +[2025-11-30 15:35:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 +[2025-11-30 15:35:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 (Updated: 2025-10-14T07:21:26 [TS: 1760426486] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 (Updated: 2025-10-14T07:21:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/43f5fd27-f421-412e-8a59-131396132b31] to complete... +.....done. +[2025-11-30 15:35:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 +[2025-11-30 15:35:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 (Updated: 2025-10-15T07:21:13 [TS: 1760512873] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 (Updated: 2025-10-15T07:21:13) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a6521bfa-bb27-492b-9eb5-8456b5f4c774] to complete... +.....done. +[2025-11-30 15:35:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 +[2025-11-30 15:35:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c (Updated: 2025-10-15T07:21:17 [TS: 1760512877] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c (Updated: 2025-10-15T07:21:17) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b47c2435-066f-44a6-aea4-55a1502f303c] to complete... +.....done. +[2025-11-30 15:35:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c +[2025-11-30 15:35:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 (Updated: 2025-10-16T07:20:31 [TS: 1760599231] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 (Updated: 2025-10-16T07:20:31) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7830fe9c-6486-4fe2-9ec3-4253557283c0] to complete... +......done. +[2025-11-30 15:35:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 +[2025-11-30 15:35:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 (Updated: 2025-10-16T07:20:37 [TS: 1760599237] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 (Updated: 2025-10-16T07:20:37) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3f282dd8-c17a-4e37-98df-a33162f9d5bd] to complete... +......done. +[2025-11-30 15:35:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 +[2025-11-30 15:35:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be (Updated: 2025-10-17T07:22:22 [TS: 1760685742] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:35:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be (Updated: 2025-10-17T07:22:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/30e5ad30-e8bb-4a26-9dd6-afd1e0acc4b7] to complete... +......done. +[2025-11-30 15:36:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be +[2025-11-30 15:36:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 (Updated: 2025-10-17T07:22:28 [TS: 1760685748] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 (Updated: 2025-10-17T07:22:28) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b7d654f9-8e5f-42cb-9d1b-b0aa2ea2d611] to complete... +.....done. +[2025-11-30 15:36:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 +[2025-11-30 15:36:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 (Updated: 2025-10-18T07:19:49 [TS: 1760771989] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 (Updated: 2025-10-18T07:19:49) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2fd81796-dbe4-4440-9f17-a0d3accd0acb] to complete... +.....done. +[2025-11-30 15:36:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 +[2025-11-30 15:36:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 (Updated: 2025-10-18T07:19:55 [TS: 1760771995] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 (Updated: 2025-10-18T07:19:55) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e06aa0db-0336-467c-834a-46b0a17e3302] to complete... +.....done. +[2025-11-30 15:36:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 +[2025-11-30 15:36:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 (Updated: 2025-10-19T07:20:50 [TS: 1760858450] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 (Updated: 2025-10-19T07:20:50) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/22088a1b-2ef3-4325-98d0-3f98e8c4fc93] to complete... +.....done. +[2025-11-30 15:36:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 +[2025-11-30 15:36:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 (Updated: 2025-10-19T07:20:57 [TS: 1760858457] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 (Updated: 2025-10-19T07:20:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/437fb55a-b67b-4a79-b6df-63decf026a6a] to complete... +.....done. +[2025-11-30 15:36:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 +[2025-11-30 15:36:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a (Updated: 2025-10-20T07:22:34 [TS: 1760944954] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a (Updated: 2025-10-20T07:22:34) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/627712dc-e249-410b-81e6-0678437e64d7] to complete... +.....done. +[2025-11-30 15:36:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a +[2025-11-30 15:36:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 (Updated: 2025-10-20T07:22:40 [TS: 1760944960] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 (Updated: 2025-10-20T07:22:40) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b55c9456-44eb-4777-b40f-701bc9142890] to complete... +.....done. +[2025-11-30 15:36:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 +[2025-11-30 15:36:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 (Updated: 2025-10-21T07:23:22 [TS: 1761031402] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 (Updated: 2025-10-21T07:23:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/020036f2-1648-44b0-bcf4-b4ebd02465d9] to complete... +.....done. +[2025-11-30 15:36:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 +[2025-11-30 15:36:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 (Updated: 2025-10-21T07:23:25 [TS: 1761031405] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 (Updated: 2025-10-21T07:23:25) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a09c56a3-4402-4d13-b800-14bf968621bf] to complete... +.....done. +[2025-11-30 15:36:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 +[2025-11-30 15:36:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 (Updated: 2025-10-22T07:21:05 [TS: 1761117665] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 (Updated: 2025-10-22T07:21:05) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1e96b748-9cc1-4b52-9770-5de09d0be1cf] to complete... +.....done. +[2025-11-30 15:36:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 +[2025-11-30 15:36:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 (Updated: 2025-10-22T07:21:08 [TS: 1761117668] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 (Updated: 2025-10-22T07:21:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5e8d9721-5872-4330-b819-2f8a11e01b98] to complete... +.....done. +[2025-11-30 15:36:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 +[2025-11-30 15:36:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 (Updated: 2025-10-23T07:21:12 [TS: 1761204072] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 (Updated: 2025-10-23T07:21:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7475419b-cccf-400d-a776-37a5ddc9742c] to complete... +.....done. +[2025-11-30 15:36:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 +[2025-11-30 15:36:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 (Updated: 2025-10-23T07:21:15 [TS: 1761204075] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 (Updated: 2025-10-23T07:21:15) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/80d05e30-872b-488e-b383-853368e1cf90] to complete... +.....done. +[2025-11-30 15:36:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 +[2025-11-30 15:36:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 (Updated: 2025-10-24T07:21:00 [TS: 1761290460] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 (Updated: 2025-10-24T07:21:00) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a51d3e66-2907-421f-80f1-65a74131908e] to complete... +.....done. +[2025-11-30 15:36:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 +[2025-11-30 15:36:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f (Updated: 2025-10-24T07:21:03 [TS: 1761290463] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f (Updated: 2025-10-24T07:21:03) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8607ef22-a786-466a-8c21-cdbf06430939] to complete... +.....done. +[2025-11-30 15:36:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f +[2025-11-30 15:36:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 (Updated: 2025-10-25T07:22:28 [TS: 1761376948] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 (Updated: 2025-10-25T07:22:28) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/df43520c-3876-43c7-8d49-71bb0ef36d69] to complete... +.....done. +[2025-11-30 15:36:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 +[2025-11-30 15:36:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 (Updated: 2025-10-25T07:22:32 [TS: 1761376952] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:36:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 (Updated: 2025-10-25T07:22:32) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7f02acad-2731-4cd3-ad0f-65a8c22f98e8] to complete... +......done. +[2025-11-30 15:37:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 +[2025-11-30 15:37:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 (Updated: 2025-10-26T07:17:54 [TS: 1761463074] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 (Updated: 2025-10-26T07:17:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/933c44e0-ea39-46d2-beab-4b2b7e988f3c] to complete... +.....done. +[2025-11-30 15:37:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 +[2025-11-30 15:37:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 (Updated: 2025-10-26T07:18:06 [TS: 1761463086] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 (Updated: 2025-10-26T07:18:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/46877bba-ca7b-4f9e-adff-3c0e3fb5bc32] to complete... +.....done. +[2025-11-30 15:37:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 +[2025-11-30 15:37:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e (Updated: 2025-10-27T07:20:57 [TS: 1761549657] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e (Updated: 2025-10-27T07:20:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b91950e5-fc0a-4c3a-92d3-28b14aed5009] to complete... +.....done. +[2025-11-30 15:37:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e +[2025-11-30 15:37:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e (Updated: 2025-10-27T07:21:00 [TS: 1761549660] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e (Updated: 2025-10-27T07:21:00) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bc70b6de-6624-4017-9a97-cacad3ebb674] to complete... +.....done. +[2025-11-30 15:37:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e +[2025-11-30 15:37:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 (Updated: 2025-10-28T07:21:03 [TS: 1761636063] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 (Updated: 2025-10-28T07:21:03) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/771f2787-6a8d-401c-ab51-fa8faa47c5f5] to complete... +.....done. +[2025-11-30 15:37:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 +[2025-11-30 15:37:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c (Updated: 2025-10-28T07:21:09 [TS: 1761636069] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c (Updated: 2025-10-28T07:21:09) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2a204351-0a43-4e4a-93ee-cfa5923db1bf] to complete... +.....done. +[2025-11-30 15:37:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c +[2025-11-30 15:37:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a (Updated: 2025-10-29T07:21:55 [TS: 1761722515] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a (Updated: 2025-10-29T07:21:55) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ced28ac0-eda7-451e-a0e9-02dc79e97c32] to complete... +.....done. +[2025-11-30 15:37:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a +[2025-11-30 15:37:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 (Updated: 2025-10-29T07:22:01 [TS: 1761722521] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 (Updated: 2025-10-29T07:22:01) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/361023c0-dd12-4bc4-ba07-ff55f92c9d25] to complete... +.....done. +[2025-11-30 15:37:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 +[2025-11-30 15:37:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 (Updated: 2025-10-30T07:21:11 [TS: 1761808871] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 (Updated: 2025-10-30T07:21:11) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/20a7fd0f-33d1-46da-b931-544c62bd11fa] to complete... +.....done. +[2025-11-30 15:37:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 +[2025-11-30 15:37:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf (Updated: 2025-10-30T07:21:17 [TS: 1761808877] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf (Updated: 2025-10-30T07:21:17) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/70a44954-781d-49af-b475-5b4715260b3f] to complete... +.....done. +[2025-11-30 15:37:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf +[2025-11-30 15:37:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 (Updated: 2025-10-31T07:22:12 [TS: 1761895332] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 (Updated: 2025-10-31T07:22:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/586252f7-a883-4c13-808a-52b9c1537c6b] to complete... +.....done. +[2025-11-30 15:37:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 +[2025-11-30 15:37:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 (Updated: 2025-10-31T07:22:18 [TS: 1761895338] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 (Updated: 2025-10-31T07:22:18) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6fcb9c7b-2fdb-4d77-abdb-23b977d34225] to complete... +.....done. +[2025-11-30 15:37:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 +[2025-11-30 15:37:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa (Updated: 2025-11-01T07:22:50 [TS: 1761981770] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa (Updated: 2025-11-01T07:22:50) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e114b7ab-034f-464c-833a-af8d5ba58fea] to complete... +......done. +[2025-11-30 15:37:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa +[2025-11-30 15:37:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 (Updated: 2025-11-01T07:22:54 [TS: 1761981774] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 (Updated: 2025-11-01T07:22:54) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/14ff1dc4-9560-45aa-8194-fc430049705f] to complete... +.....done. +[2025-11-30 15:37:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 +[2025-11-30 15:37:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 (Updated: 2025-11-02T07:21:16 [TS: 1762068076] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 (Updated: 2025-11-02T07:21:16) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7548581d-b8c6-41b2-8cf4-59be0475de00] to complete... +.....done. +[2025-11-30 15:37:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 +[2025-11-30 15:37:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 (Updated: 2025-11-02T07:21:20 [TS: 1762068080] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 (Updated: 2025-11-02T07:21:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d1e0b52f-131b-4d3e-9918-05014f1f1ea7] to complete... +.....done. +[2025-11-30 15:37:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 +[2025-11-30 15:37:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 (Updated: 2025-11-03T08:23:08 [TS: 1762158188] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:37:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 (Updated: 2025-11-03T08:23:08) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0046fdef-0839-4104-8f6b-fcd70567270f] to complete... +.....done. +[2025-11-30 15:38:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 +[2025-11-30 15:38:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a (Updated: 2025-11-03T08:23:12 [TS: 1762158192] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a (Updated: 2025-11-03T08:23:12) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/dc38a112-2d49-4dcc-b535-492d5203d99a] to complete... +.....done. +[2025-11-30 15:38:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a +[2025-11-30 15:38:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 (Updated: 2025-11-04T08:17:45 [TS: 1762244265] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 (Updated: 2025-11-04T08:17:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/82e158ab-89a5-4923-83c0-b85517487154] to complete... +.....done. +[2025-11-30 15:38:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 +[2025-11-30 15:38:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 (Updated: 2025-11-04T08:17:48 [TS: 1762244268] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 (Updated: 2025-11-04T08:17:48) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b0243605-3924-484a-8718-77c7798c7b6d] to complete... +......done. +[2025-11-30 15:38:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 +[2025-11-30 15:38:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 (Updated: 2025-11-05T08:20:13 [TS: 1762330813] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 (Updated: 2025-11-05T08:20:13) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a9ccfb7d-1285-47c9-9e67-76c95e7e2411] to complete... +.....done. +[2025-11-30 15:38:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 +[2025-11-30 15:38:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 (Updated: 2025-11-05T08:20:17 [TS: 1762330817] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 (Updated: 2025-11-05T08:20:17) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9385310f-c275-4271-a36f-34cf67b3b027] to complete... +.....done. +[2025-11-30 15:38:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 +[2025-11-30 15:38:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a (Updated: 2025-11-06T08:20:55 [TS: 1762417255] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a (Updated: 2025-11-06T08:20:55) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f7fbaf5a-f17a-4c83-8944-191641c21331] to complete... +......done. +[2025-11-30 15:38:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a +[2025-11-30 15:38:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 (Updated: 2025-11-06T08:20:58 [TS: 1762417258] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 (Updated: 2025-11-06T08:20:58) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/001b962b-2941-427a-99bb-44465efae926] to complete... +.....done. +[2025-11-30 15:38:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 +[2025-11-30 15:38:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd (Updated: 2025-11-07T08:18:30 [TS: 1762503510] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd (Updated: 2025-11-07T08:18:30) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fbf6a82f-fc89-428d-99ba-e14693841c4f] to complete... +......done. +[2025-11-30 15:38:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd +[2025-11-30 15:38:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a (Updated: 2025-11-07T08:18:33 [TS: 1762503513] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a (Updated: 2025-11-07T08:18:33) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/13102179-62f5-455e-b23a-00ecf0b80516] to complete... +.....done. +[2025-11-30 15:38:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a +[2025-11-30 15:38:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af (Updated: 2025-11-08T08:18:22 [TS: 1762589902] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af (Updated: 2025-11-08T08:18:22) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7b3344f4-d0f1-4217-b8bb-475a45bbc53c] to complete... +.....done. +[2025-11-30 15:38:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af +[2025-11-30 15:38:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 (Updated: 2025-11-08T08:18:26 [TS: 1762589906] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 (Updated: 2025-11-08T08:18:26) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1b1f81c0-34b4-4f2c-a58b-d8272761af3d] to complete... +.....done. +[2025-11-30 15:38:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 +[2025-11-30 15:38:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd (Updated: 2025-11-09T08:21:42 [TS: 1762676502] < Cutoff: [TS: 1763306816]) +[2025-11-30 15:38:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd (Updated: 2025-11-09T08:21:42) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5b819612-8b72-4af6-a6c8-c00242f777ae] to complete... +.....done. +[2025-11-30 15:38:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd +[2025-11-30 15:38:44] [INFO] Hit delete limit (200) for Docker Images. +[2025-11-30 15:38:44] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 15:38:44] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 15:38:46] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 15:38:46] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 15:38:46] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 15:38:46] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 15:38:46] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 15:38:46] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 15:38:49] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 15:38:49] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:38:51] [INFO] No Regional Address found matching criteria. +[2025-11-30 15:38:51] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:38:53] [INFO] No Global Address found matching criteria. +[2025-11-30 15:38:53] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- +[2025-11-30 15:38:58] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 15:39:01] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:39:01] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:39:01] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 15:39:01] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 15:39:01] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:04] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 15:39:07] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:07] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 15:39:09] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 15:39:09] [INFO] CLEANUP RUN FINISHED +[2025-11-30 15:39:14] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 15:39:14] [INFO] Time Cutoff (General): 2025-11-30T15:39:14+0000 +[2025-11-30 15:39:14] [INFO] Time Cutoff (Images): 2025-10-01T15:39:14+0000 +[2025-11-30 15:39:14] [INFO] Delete Limit per Type: 200 +[2025-11-30 15:39:14] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 15:39:15] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 15:39:17] [INFO] No Service Accounts found matching prefix. +[2025-11-30 15:39:17] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:39:19] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 15:39:19] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 15:39:21] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:39:21] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:39:21] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 15:39:21] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:39:24] [INFO] No Filestore instances found matching criteria. +[2025-11-30 15:39:24] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 15:39:27] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 15:39:27] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 15:39:27] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 15:39:28] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 15:39:28] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 15:39:28] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 15:39:28] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 15:39:28] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 15:39:28] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 15:39:28] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 15:39:28] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 15:39:28] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:39:28Z (Unix: 1763307568) +[2025-11-30 15:39:28] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 (Updated: 2025-11-09T08:21:45 [TS: 1762676505] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 (Updated: 2025-11-09T08:21:45) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb (Updated: 2025-11-10T08:20:16 [TS: 1762762816] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb (Updated: 2025-11-10T08:20:16) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 (Updated: 2025-11-10T08:20:19 [TS: 1762762819] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 (Updated: 2025-11-10T08:20:19) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e (Updated: 2025-11-11T08:19:38 [TS: 1762849178] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e (Updated: 2025-11-11T08:19:38) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a (Updated: 2025-11-11T08:19:44 [TS: 1762849184] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a (Updated: 2025-11-11T08:19:44) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe (Updated: 2025-11-12T08:20:59 [TS: 1762935659] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe (Updated: 2025-11-12T08:20:59) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 (Updated: 2025-11-12T08:21:06 [TS: 1762935666] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 (Updated: 2025-11-12T08:21:06) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 (Updated: 2025-11-13T08:20:14 [TS: 1763022014] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 (Updated: 2025-11-13T08:20:14) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 (Updated: 2025-11-13T08:20:21 [TS: 1763022021] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 (Updated: 2025-11-13T08:20:21) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 (Updated: 2025-11-14T08:20:20 [TS: 1763108420] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 (Updated: 2025-11-14T08:20:20) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d (Updated: 2025-11-14T08:20:27 [TS: 1763108427] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d (Updated: 2025-11-14T08:20:27) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 (Updated: 2025-11-15T08:21:45 [TS: 1763194905] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 (Updated: 2025-11-15T08:21:45) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 (Updated: 2025-11-15T08:21:52 [TS: 1763194912] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 (Updated: 2025-11-15T08:21:52) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 (Updated: 2025-11-16T08:21:57 [TS: 1763281317] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 (Updated: 2025-11-16T08:21:57) +[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae (Updated: 2025-11-16T08:22:05 [TS: 1763281325] < Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae (Updated: 2025-11-16T08:22:05) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763307568]) +[2025-11-30 15:39:31] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 15:39:31] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 15:39:34] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 15:39:34] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 15:39:34] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 15:39:34] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 15:39:34] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 15:39:34] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 15:39:36] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 15:39:36] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:39:39] [INFO] No Regional Address found matching criteria. +[2025-11-30 15:39:39] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:39:41] [INFO] No Global Address found matching criteria. +[2025-11-30 15:39:41] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- +[2025-11-30 15:39:46] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 15:39:49] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:39:49] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:39:49] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 15:39:49] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 15:39:49] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:51] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 15:39:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:39:54] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 15:39:55] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 15:39:55] [INFO] CLEANUP RUN FINISHED +[2025-11-30 15:39:59] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 15:39:59] [INFO] Time Cutoff (General): 2025-11-30T15:39:59+0000 +[2025-11-30 15:39:59] [INFO] Time Cutoff (Images): 2025-10-01T15:39:59+0000 +[2025-11-30 15:39:59] [INFO] Delete Limit per Type: 200 +[2025-11-30 15:39:59] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 15:40:00] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 15:40:02] [INFO] No Service Accounts found matching prefix. +[2025-11-30 15:40:02] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:40:04] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 15:40:04] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 15:40:08] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:40:08] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:40:08] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 15:40:08] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 15:40:11] [INFO] No Filestore instances found matching criteria. +[2025-11-30 15:40:11] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 15:40:14] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 15:40:14] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 15:40:14] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 15:40:14] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 15:40:14] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 15:40:15] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 15:40:15] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 15:40:15] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 15:40:15] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 15:40:15] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 15:40:15] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 15:40:15] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:40:15Z (Unix: 1763307615) +[2025-11-30 15:40:15] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 15:40:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 (Updated: 2025-11-09T08:21:45 [TS: 1762676505] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 (Updated: 2025-11-09T08:21:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6faafbb1-7528-404f-90d3-f46225e56bf0] to complete... +.....done. +[2025-11-30 15:40:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 +[2025-11-30 15:40:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb (Updated: 2025-11-10T08:20:16 [TS: 1762762816] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb (Updated: 2025-11-10T08:20:16) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/25eb1ae8-2d01-4e1a-bb6e-a9e19e437f93] to complete... +.....done. +[2025-11-30 15:40:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb +[2025-11-30 15:40:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 (Updated: 2025-11-10T08:20:19 [TS: 1762762819] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 (Updated: 2025-11-10T08:20:19) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6ad2fe8f-2df3-494b-b021-b159c6e6abae] to complete... +.....done. +[2025-11-30 15:40:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 +[2025-11-30 15:40:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e (Updated: 2025-11-11T08:19:38 [TS: 1762849178] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e (Updated: 2025-11-11T08:19:38) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bf01f721-a319-4e11-9287-fcca3bc7f36b] to complete... +.....done. +[2025-11-30 15:40:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e +[2025-11-30 15:40:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a (Updated: 2025-11-11T08:19:44 [TS: 1762849184] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a (Updated: 2025-11-11T08:19:44) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cb532182-020d-4250-a0cc-35d6b848f5cc] to complete... +.....done. +[2025-11-30 15:40:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a +[2025-11-30 15:40:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe (Updated: 2025-11-12T08:20:59 [TS: 1762935659] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe (Updated: 2025-11-12T08:20:59) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d2267cfc-f79e-4baa-ad94-3522995188fe] to complete... +......done. +[2025-11-30 15:40:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe +[2025-11-30 15:40:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 (Updated: 2025-11-12T08:21:06 [TS: 1762935666] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 (Updated: 2025-11-12T08:21:06) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9944c4ff-eb4f-442d-b56a-db74e63546e7] to complete... +.....done. +[2025-11-30 15:40:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 +[2025-11-30 15:40:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 (Updated: 2025-11-13T08:20:14 [TS: 1763022014] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 (Updated: 2025-11-13T08:20:14) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/99699167-777a-4f47-8315-b0e11d5af629] to complete... +......done. +[2025-11-30 15:40:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 +[2025-11-30 15:40:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 (Updated: 2025-11-13T08:20:21 [TS: 1763022021] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 (Updated: 2025-11-13T08:20:21) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/07a1793a-fde4-4285-a39e-517fb0ea81ab] to complete... +.....done. +[2025-11-30 15:40:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 +[2025-11-30 15:40:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 (Updated: 2025-11-14T08:20:20 [TS: 1763108420] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 (Updated: 2025-11-14T08:20:20) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c2d3adb2-c23e-436b-b639-f212b0c9dc45] to complete... +......done. +[2025-11-30 15:40:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 +[2025-11-30 15:40:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d (Updated: 2025-11-14T08:20:27 [TS: 1763108427] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d (Updated: 2025-11-14T08:20:27) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/455a2d68-2594-4371-b0ad-7fd6a67fb03f] to complete... +.....done. +[2025-11-30 15:40:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d +[2025-11-30 15:40:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 (Updated: 2025-11-15T08:21:45 [TS: 1763194905] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:40:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 (Updated: 2025-11-15T08:21:45) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d5d4fc28-dd0d-4dde-93b5-d897b729f785] to complete... +.....done. +[2025-11-30 15:41:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 +[2025-11-30 15:41:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 (Updated: 2025-11-15T08:21:52 [TS: 1763194912] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 (Updated: 2025-11-15T08:21:52) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d291f5b5-c122-4a88-96c1-e505c9e36400] to complete... +.....done. +[2025-11-30 15:41:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 +[2025-11-30 15:41:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 (Updated: 2025-11-16T08:21:57 [TS: 1763281317] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 (Updated: 2025-11-16T08:21:57) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0c4ddb0c-6cff-4106-beef-ed8e183d6a15] to complete... +......done. +[2025-11-30 15:41:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 +[2025-11-30 15:41:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae (Updated: 2025-11-16T08:22:05 [TS: 1763281325] < Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae (Updated: 2025-11-16T08:22:05) +Digests: +- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae +Delete request issued. +Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6820bcba-dc10-40e0-829c-f29db84577e0] to complete... +.....done. +[2025-11-30 15:41:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763307615]) +[2025-11-30 15:41:13] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 15:41:13] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 15:41:16] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 15:41:16] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 15:41:16] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 15:41:16] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 15:41:16] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 15:41:16] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 15:41:18] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 15:41:18] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:41:21] [INFO] No Regional Address found matching criteria. +[2025-11-30 15:41:21] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 15:41:23] [INFO] No Global Address found matching criteria. +[2025-11-30 15:41:23] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- +[2025-11-30 15:41:27] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 15:41:30] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 15:41:30] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 15:41:30] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 15:41:30] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 15:41:30] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:33] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 15:41:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 15:41:35] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 15:41:37] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 15:41:37] [INFO] CLEANUP RUN FINISHED diff --git a/instances.txt b/instances.txt new file mode 100644 index 0000000000..48a10b7c0c --- /dev/null +++ b/instances.txt @@ -0,0 +1,2847 @@ +--- Wed Nov 26 09:53:14 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 26 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +The following GKE clusters are targeted for deletion in this run: +h4d-res-swarnabm4-3 us-central1 +ml-gke-e2e-a8fae6 asia-southeast1 + gcloud container clusters delete "h4d-res-swarnabm4-3" --project="hpc-toolkit-dev" --location="us-central1" --quiet + gcloud container clusters delete "ml-gke-e2e-a8fae6" --project="hpc-toolkit-dev" --location="asia-southeast1" --quiet +--- Wed Nov 26 09:53:16 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 09:53:42 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 26 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +The following GKE clusters are targeted for deletion in this run: +h4d-res-swarnabm4-3 us-central1 +ml-gke-e2e-a8fae6 asia-southeast1 +[EXECUTE] GKE Cluster: Deleting h4d-res-swarnabm4-3 in us-central1 +Deleting cluster h4d-res-swarnabm4-3... +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................done. +Deleted [https://container.googleapis.com/v1/projects/hpc-toolkit-dev/zones/us-central1/clusters/h4d-res-swarnabm4-3]. +[EXECUTE] GKE Cluster: Deleting ml-gke-e2e-a8fae6 in asia-southeast1 +Deleting cluster ml-gke-e2e-a8fae6... +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................done. +Deleted [https://container.googleapis.com/v1/projects/hpc-toolkit-dev/zones/asia-southeast1/clusters/ml-gke-e2e-a8fae6]. +--- Wed Nov 26 10:02:05 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 10:03:24 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 26 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +a1c0c3topo-nodeset-2 us-central1-a +a3hca628-controller us-west1-a +a3hca628-login-001 us-west1-a +a3hcb15f-controller us-west1-a +a3hcb15f-login-001 us-west1-a +a3hcdf00-controller us-west1-a +a3hcdf00-login-001 us-west1-a +a3hnfsa628c1-ff9f3704-nfs-instance us-west1-a +a3hnfsb15fa8-57b77541-nfs-instance us-west1-a +a3hnfsdf0061-6327333f-nfs-instance us-west1-a +[DRY RUN] Instance: Would delete a1c0c3topo-nodeset-2 in us-central1-a + Command: gcloud compute instances delete "a1c0c3topo-nodeset-2" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +[DRY RUN] Instance: Would delete a3hca628-controller in us-west1-a + Command: gcloud compute instances delete "a3hca628-controller" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet +[DRY RUN] Instance: Would delete a3hca628-login-001 in us-west1-a + Command: gcloud compute instances delete "a3hca628-login-001" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet +[DRY RUN] Instance: Would delete a3hcb15f-controller in us-west1-a + Command: gcloud compute instances delete "a3hcb15f-controller" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet +[DRY RUN] Instance: Would delete a3hcb15f-login-001 in us-west1-a + Command: gcloud compute instances delete "a3hcb15f-login-001" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet +[DRY RUN] Instance: Would delete a3hcdf00-controller in us-west1-a + Command: gcloud compute instances delete "a3hcdf00-controller" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet +[DRY RUN] Instance: Would delete a3hcdf00-login-001 in us-west1-a + Command: gcloud compute instances delete "a3hcdf00-login-001" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet +[DRY RUN] Instance: Would delete a3hnfsa628c1-ff9f3704-nfs-instance in us-west1-a + Command: gcloud compute instances delete "a3hnfsa628c1-ff9f3704-nfs-instance" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet +[DRY RUN] Instance: Would delete a3hnfsb15fa8-57b77541-nfs-instance in us-west1-a + Command: gcloud compute instances delete "a3hnfsb15fa8-57b77541-nfs-instance" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet +[DRY RUN] Instance: Would delete a3hnfsdf0061-6327333f-nfs-instance in us-west1-a + Command: gcloud compute instances delete "a3hnfsdf0061-6327333f-nfs-instance" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet +--- Wed Nov 26 10:03:28 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 10:03:57 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 26 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +a1c0c3topo-nodeset-2 us-central1-a +a3hca628-controller us-west1-a +a3hca628-login-001 us-west1-a +a3hcb15f-controller us-west1-a +a3hcb15f-login-001 us-west1-a +a3hcdf00-controller us-west1-a +a3hcdf00-login-001 us-west1-a +a3hnfsa628c1-ff9f3704-nfs-instance us-west1-a +a3hnfsb15fa8-57b77541-nfs-instance us-west1-a +a3hnfsdf0061-6327333f-nfs-instance us-west1-a +[EXECUTE] Instance: Deleting a1c0c3topo-nodeset-2 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/a1c0c3topo-nodeset-2]. +[EXECUTE] Instance: Deleting a3hca628-controller in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hca628-controller]. +[EXECUTE] Instance: Deleting a3hca628-login-001 in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hca628-login-001]. +[EXECUTE] Instance: Deleting a3hcb15f-controller in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hcb15f-controller]. +[EXECUTE] Instance: Deleting a3hcb15f-login-001 in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hcb15f-login-001]. +[EXECUTE] Instance: Deleting a3hcdf00-controller in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hcdf00-controller]. +[EXECUTE] Instance: Deleting a3hcdf00-login-001 in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hcdf00-login-001]. +[EXECUTE] Instance: Deleting a3hnfsa628c1-ff9f3704-nfs-instance in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hnfsa628c1-ff9f3704-nfs-instance]. +[EXECUTE] Instance: Deleting a3hnfsb15fa8-57b77541-nfs-instance in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hnfsb15fa8-57b77541-nfs-instance]. +[EXECUTE] Instance: Deleting a3hnfsdf0061-6327333f-nfs-instance in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hnfsdf0061-6327333f-nfs-instance]. +--- Wed Nov 26 10:12:55 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 10:13:53 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 26 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +a3mega-controller us-west4-a +a3mega-login-001 us-west4-a +a4h639c-a4highnodeset-0 us-central1-b +a4h639c-a4highnodeset-1 us-central1-b +a4h639c-controller us-central1-b +a4h639c-slurm-login-001 us-central1-b +a7f7bcslur-controller us-central1-a +a7f7bcslur-slurm-login-001 us-central1-a +c379slurms-controller us-central1-a +ce64slurms-controller us-central1-a + gcloud compute instances delete "a3mega-controller" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet + gcloud compute instances delete "a3mega-login-001" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet + gcloud compute instances delete "a4h639c-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4h639c-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4h639c-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4h639c-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a7f7bcslur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "a7f7bcslur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "c379slurms-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "ce64slurms-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +--- Wed Nov 26 10:13:57 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 10:14:29 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 26 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +a3mega-controller us-west4-a +a3mega-login-001 us-west4-a +a4h639c-a4highnodeset-0 us-central1-b +a4h639c-a4highnodeset-1 us-central1-b +a4h639c-controller us-central1-b +a4h639c-slurm-login-001 us-central1-b +a7f7bcslur-controller us-central1-a +a7f7bcslur-slurm-login-001 us-central1-a +c379slurms-controller us-central1-a +ce64slurms-controller us-central1-a +[EXECUTE] Instance: Deleting a3mega-controller in us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/instances/a3mega-controller]. +[EXECUTE] Instance: Deleting a3mega-login-001 in us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/instances/a3mega-login-001]. +[EXECUTE] Instance: Deleting a4h639c-a4highnodeset-0 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4h639c-a4highnodeset-0]. +[EXECUTE] Instance: Deleting a4h639c-a4highnodeset-1 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4h639c-a4highnodeset-1]. +[EXECUTE] Instance: Deleting a4h639c-controller in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4h639c-controller]. +[EXECUTE] Instance: Deleting a4h639c-slurm-login-001 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4h639c-slurm-login-001]. +[EXECUTE] Instance: Deleting a7f7bcslur-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/a7f7bcslur-controller]. +[EXECUTE] Instance: Deleting a7f7bcslur-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/a7f7bcslur-slurm-login-001]. +[EXECUTE] Instance: Deleting c379slurms-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/c379slurms-controller]. +[EXECUTE] Instance: Deleting ce64slurms-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/ce64slurms-controller]. +--- Wed Nov 26 10:28:03 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 10:28:21 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 26 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +ce64slurms-slurm-login-001 us-central1-a +cluster0vk-nodeset1-0 us-central1-c +cluster0vk-nodeset1-1 us-central1-c +cluster8ix-nodeset1-0 us-east4-b +cluster8ix-nodeset1-1 us-east4-b +clustermce-nodeset1-0 us-east5-a +clustermce-nodeset1-1 us-east5-a +clustermce-nodeset1-2 us-east5-a +clusterum7-nodeset1-1 us-central1-c +clusterum7-nodeset1-2 us-central1-c + gcloud compute instances delete "ce64slurms-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "clustermce-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clustermce-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet +--- Wed Nov 26 10:28:25 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 10:31:34 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 26 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +ce64slurms-slurm-login-001 us-central1-a +cluster0vk-nodeset1-0 us-central1-c +cluster0vk-nodeset1-1 us-central1-c +cluster8ix-nodeset1-0 us-east4-b +cluster8ix-nodeset1-1 us-east4-b +clustermce-nodeset1-0 us-east5-a +clustermce-nodeset1-2 us-east5-a +clusterum7-nodeset1-0 us-central1-c +clusterum7-nodeset1-1 us-central1-c +clusterum7-nodeset1-2 us-central1-c + gcloud compute instances delete "ce64slurms-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "clustermce-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clusterum7-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet +--- Wed Nov 26 10:31:38 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 10:31:55 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 26 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +ce64slurms-slurm-login-001 us-central1-a +cluster0vk-nodeset1-0 us-central1-c +cluster0vk-nodeset1-1 us-central1-c +cluster8ix-nodeset1-0 us-east4-b +cluster8ix-nodeset1-1 us-east4-b +clustermce-nodeset1-2 us-east5-a +clusterum7-nodeset1-0 us-central1-c +clusterum7-nodeset1-1 us-central1-c +clusterum7-nodeset1-2 us-central1-c +clusterum7-nodeset1-3 us-central1-c +[EXECUTE] Instance: Deleting ce64slurms-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/ce64slurms-slurm-login-001]. +[EXECUTE] Instance: Deleting cluster0vk-nodeset1-0 in us-central1-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-0]. +[EXECUTE] Instance: Deleting cluster0vk-nodeset1-1 in us-central1-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-1]. +[EXECUTE] Instance: Deleting cluster8ix-nodeset1-0 in us-east4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/instances/cluster8ix-nodeset1-0]. +[EXECUTE] Instance: Deleting cluster8ix-nodeset1-1 in us-east4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/instances/cluster8ix-nodeset1-1]. +[EXECUTE] Instance: Deleting clustermce-nodeset1-2 in us-east5-a +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-east5-a/instances/clustermce-nodeset1-2' was not found + +--- Wed Nov 26 10:43:27 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 28 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +cluster0vk-nodeset1-0 us-central1-c +cluster0vk-nodeset1-1 us-central1-c +cluster8ix-nodeset1-0 us-east4-b +cluster8ix-nodeset1-1 us-east4-b +clustermce-nodeset1-0 us-east5-a +clustermce-nodeset1-1 us-east5-a +clustermce-nodeset1-2 us-east5-a +clusterum7-nodeset1-0 us-central1-c +clusterum7-nodeset1-1 us-central1-c +clusterum7-nodeset1-2 us-central1-c + gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "clustermce-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clustermce-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clusterum7-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet +--- Wed Nov 26 10:43:32 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 10:44:42 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 28 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +cluster0vk-nodeset1-0 us-central1-c +cluster0vk-nodeset1-1 us-central1-c +cluster8ix-nodeset1-0 us-east4-b +cluster8ix-nodeset1-1 us-east4-b +clustermce-nodeset1-0 us-east5-a +clustermce-nodeset1-1 us-east5-a +clustermce-nodeset1-2 us-east5-a +clusterum7-nodeset1-0 us-central1-c +clusterum7-nodeset1-1 us-central1-c +clusterum7-nodeset1-2 us-central1-c + gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "clustermce-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clustermce-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clusterum7-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet +--- Wed Nov 26 10:44:47 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 10:45:01 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 28 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +cluster0vk-nodeset1-0 us-central1-c +cluster0vk-nodeset1-1 us-central1-c +cluster8ix-nodeset1-0 us-east4-b +cluster8ix-nodeset1-1 us-east4-b +clustermce-nodeset1-0 us-east5-a +clustermce-nodeset1-1 us-east5-a +clustermce-nodeset1-2 us-east5-a +clusterum7-nodeset1-0 us-central1-c +clusterum7-nodeset1-1 us-central1-c +clusterum7-nodeset1-2 us-central1-c +[EXECUTE] Instance: Deleting cluster0vk-nodeset1-0 in us-central1-c +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-0' was not found + +--- Wed Nov 26 10:47:32 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 28 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +cluster0vk-nodeset1-0 us-central1-c +cluster0vk-nodeset1-1 us-central1-c +cluster8ix-nodeset1-0 us-east4-b +cluster8ix-nodeset1-1 us-east4-b +clustermce-nodeset1-1 us-east5-a +clustermce-nodeset1-2 us-east5-a +clusterum7-nodeset1-0 us-central1-c +clusterum7-nodeset1-1 us-central1-c +clusterum7-nodeset1-2 us-central1-c +clusterum7-nodeset1-3 us-central1-c + gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "clustermce-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clusterum7-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-3" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet +--- Wed Nov 26 10:47:36 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 10:47:49 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 28 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +cluster0vk-nodeset1-0 us-central1-c +cluster0vk-nodeset1-1 us-central1-c +cluster8ix-nodeset1-0 us-east4-b +cluster8ix-nodeset1-1 us-east4-b +clustermce-nodeset1-1 us-east5-a +clustermce-nodeset1-2 us-east5-a +clusterum7-nodeset1-0 us-central1-c +clusterum7-nodeset1-1 us-central1-c +clusterum7-nodeset1-2 us-central1-c +clusterum7-nodeset1-3 us-central1-c +[EXECUTE] Instance: Deleting cluster0vk-nodeset1-0 in us-central1-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-0]. +[EXECUTE] Instance: Deleting cluster0vk-nodeset1-1 in us-central1-c +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-1' was not found + +[EXECUTE] Instance: Deleting cluster8ix-nodeset1-0 in us-east4-b +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-east4-b/instances/cluster8ix-nodeset1-0' was not found + +[EXECUTE] Instance: Deleting cluster8ix-nodeset1-1 in us-east4-b +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-east4-b/instances/cluster8ix-nodeset1-1' was not found + +[EXECUTE] Instance: Deleting clustermce-nodeset1-1 in us-east5-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east5-a/instances/clustermce-nodeset1-1]. +[EXECUTE] Instance: Deleting clustermce-nodeset1-2 in us-east5-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east5-a/instances/clustermce-nodeset1-2]. +[EXECUTE] Instance: Deleting clusterum7-nodeset1-0 in us-central1-c +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-0' was not found + +[EXECUTE] Instance: Deleting clusterum7-nodeset1-1 in us-central1-c +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-1' was not found + +[EXECUTE] Instance: Deleting clusterum7-nodeset1-2 in us-central1-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-2]. +[EXECUTE] Instance: Deleting clusterum7-nodeset1-3 in us-central1-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-3]. +--- Wed Nov 26 10:58:13 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 10:58:47 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 28 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +cluster0vk-nodeset1-0 us-central1-c +cluster8ix-nodeset1-0 us-east4-b +cluster8ix-nodeset1-1 us-east4-b +clustermce-nodeset1-0 us-east5-a +clustermce-nodeset1-1 us-east5-a +clustermce-nodeset1-2 us-east5-a +clusterum7-nodeset1-0 us-central1-c +clusterum7-nodeset1-1 us-central1-c +clusterum7-nodeset1-2 us-central1-c +clusterum7-nodeset1-4 us-central1-c + gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet + gcloud compute instances delete "clustermce-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clustermce-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet + gcloud compute instances delete "clusterum7-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-4" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet +--- Wed Nov 26 10:58:51 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 11:00:14 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 38 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - cluster0vk-nodeset1-0 + - cluster8ix-nodeset1-0 + - cluster8ix-nodeset1-1 + - clustermce-nodeset1-0 + - clustermce-nodeset1-1 + - clustermce-nodeset1-2 + - clusterum7-nodeset1-0 + - clusterum7-nodeset1-1 + - clusterum7-nodeset1-2 + - clusterum7-nodeset1-4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: cluster0vk-nodeset1-0 in us-central1-c (In exclusion list) +Skip Instance: cluster8ix-nodeset1-0 in us-east4-b (In exclusion list) +Skip Instance: clustermce-nodeset1-0 in us-east5-a (In exclusion list) +Skip Instance: clustermce-nodeset1-1 in us-east5-a (In exclusion list) +Skip Instance: clustermce-nodeset1-2 in us-east5-a (In exclusion list) +Skip Instance: clusterum7-nodeset1-0 in us-central1-c (In exclusion list) +Skip Instance: clusterum7-nodeset1-1 in us-central1-c (In exclusion list) +Skip Instance: clusterum7-nodeset1-2 in us-central1-c (In exclusion list) +The following Instances are targeted for deletion in this run: +cluster0vk-nodeset1-1 us-central1-c +clusterum7-nodeset1-5 us-central1-c +clusteurop-nodeset1-0 europe-west1-b +d3eslurmsi-controller us-central1-a +d3eslurmsi-slurm-login-001 us-central1-a +d3slurmsim-controller us-central1-a +d3slurmsim-slurm-login-001 us-central1-a +d72c8slurm-controller us-central1-a +d72c8slurm-slurm-login-001 us-central1-a +de3580slur-controller us-central1-a + gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusterum7-nodeset1-5" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet + gcloud compute instances delete "clusteurop-nodeset1-0" --project="hpc-toolkit-dev" --zone="europe-west1-b" --quiet + gcloud compute instances delete "d3eslurmsi-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "d3eslurmsi-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "d3slurmsim-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "d3slurmsim-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "d72c8slurm-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "d72c8slurm-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "de3580slur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +--- Wed Nov 26 11:00:18 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 11:00:45 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 38 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - cluster0vk-nodeset1-0 + - cluster8ix-nodeset1-0 + - cluster8ix-nodeset1-1 + - clustermce-nodeset1-0 + - clustermce-nodeset1-1 + - clustermce-nodeset1-2 + - clusterum7-nodeset1-0 + - clusterum7-nodeset1-1 + - clusterum7-nodeset1-2 + - clusterum7-nodeset1-4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: cluster0vk-nodeset1-0 in us-central1-c (In exclusion list) +Skip Instance: cluster8ix-nodeset1-0 in us-east4-b (In exclusion list) +Skip Instance: cluster8ix-nodeset1-1 in us-east4-b (In exclusion list) +Skip Instance: clustermce-nodeset1-0 in us-east5-a (In exclusion list) +Skip Instance: clustermce-nodeset1-1 in us-east5-a (In exclusion list) +Skip Instance: clustermce-nodeset1-2 in us-east5-a (In exclusion list) +Skip Instance: clusterum7-nodeset1-0 in us-central1-c (In exclusion list) +Skip Instance: clusterum7-nodeset1-1 in us-central1-c (In exclusion list) +Skip Instance: clusterum7-nodeset1-2 in us-central1-c (In exclusion list) +The following Instances are targeted for deletion in this run: +cluster0vk-nodeset1-1 us-central1-c +clusterum7-nodeset1-3 us-central1-c +clusterum7-nodeset1-5 us-central1-c +clusteurop-nodeset1-0 europe-west1-b +clusteurop-nodeset1-1 europe-west1-b +d3eslurmsi-controller us-central1-a +d3eslurmsi-slurm-login-001 us-central1-a +d3slurmsim-controller us-central1-a +d3slurmsim-slurm-login-001 us-central1-a +d72c8slurm-controller us-central1-a +[EXECUTE] Instance: Deleting cluster0vk-nodeset1-1 in us-central1-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-1]. +[EXECUTE] Instance: Deleting clusterum7-nodeset1-3 in us-central1-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-3]. +[EXECUTE] Instance: Deleting clusterum7-nodeset1-5 in us-central1-c +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-5' was not found + +[EXECUTE] Instance: Deleting clusteurop-nodeset1-0 in europe-west1-b +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/europe-west1-b/instances/clusteurop-nodeset1-0' was not found + +[EXECUTE] Instance: Deleting clusteurop-nodeset1-1 in europe-west1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b/instances/clusteurop-nodeset1-1]. +[EXECUTE] Instance: Deleting d3eslurmsi-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d3eslurmsi-controller]. +[EXECUTE] Instance: Deleting d3eslurmsi-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d3eslurmsi-slurm-login-001]. +[EXECUTE] Instance: Deleting d3slurmsim-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d3slurmsim-controller]. +[EXECUTE] Instance: Deleting d3slurmsim-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d3slurmsim-slurm-login-001]. +[EXECUTE] Instance: Deleting d72c8slurm-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d72c8slurm-controller]. +--- Wed Nov 26 11:12:55 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 11:27:49 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 38 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - cluster0vk-nodeset1-0 + - cluster8ix-nodeset1-0 + - cluster8ix-nodeset1-1 + - clustermce-nodeset1-0 + - clustermce-nodeset1-1 + - clustermce-nodeset1-2 + - clusterum7-nodeset1-0 + - clusterum7-nodeset1-1 + - clusterum7-nodeset1-2 + - clusterum7-nodeset1-4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +d72c8slurm-slurm-login-001 us-central1-a +de3580slur-controller us-central1-a +de3580slur-nodeset-0 us-central1-a +de3580slur-nodeset-1 us-central1-a +de3580slur-nodeset-2 us-central1-a +de3580slur-nodeset-3 us-central1-a +de3580slur-nodeset-4 us-central1-a +de3580slur-slurm-login-001 us-central1-a +dynpoc-controller us-central1-a +dynpoc-slurm-login-001 us-central1-a + gcloud compute instances delete "d72c8slurm-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "de3580slur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "de3580slur-nodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "de3580slur-nodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "de3580slur-nodeset-2" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "de3580slur-nodeset-3" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "de3580slur-nodeset-4" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "de3580slur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "dynpoc-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "dynpoc-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +--- Wed Nov 26 11:27:53 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 11:28:14 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 38 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - cluster0vk-nodeset1-0 + - cluster8ix-nodeset1-0 + - cluster8ix-nodeset1-1 + - clustermce-nodeset1-0 + - clustermce-nodeset1-1 + - clustermce-nodeset1-2 + - clusterum7-nodeset1-0 + - clusterum7-nodeset1-1 + - clusterum7-nodeset1-2 + - clusterum7-nodeset1-4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip GKE Cluster: mglsard (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +d72c8slurm-slurm-login-001 us-central1-a +de3580slur-controller us-central1-a +de3580slur-nodeset-0 us-central1-a +de3580slur-nodeset-1 us-central1-a +de3580slur-nodeset-2 us-central1-a +de3580slur-nodeset-3 us-central1-a +de3580slur-nodeset-4 us-central1-a +de3580slur-slurm-login-001 us-central1-a +dynpoc-controller us-central1-a +dynpoc-slurm-login-001 us-central1-a +[EXECUTE] Instance: Deleting d72c8slurm-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d72c8slurm-slurm-login-001]. +[EXECUTE] Instance: Deleting de3580slur-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-controller]. +[EXECUTE] Instance: Deleting de3580slur-nodeset-0 in us-central1-a +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-nodeset-0' was not found + +[EXECUTE] Instance: Deleting de3580slur-nodeset-1 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-nodeset-1]. +[EXECUTE] Instance: Deleting de3580slur-nodeset-2 in us-central1-a +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-nodeset-2' was not found + +[EXECUTE] Instance: Deleting de3580slur-nodeset-3 in us-central1-a +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-nodeset-3' was not found + +[EXECUTE] Instance: Deleting de3580slur-nodeset-4 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-nodeset-4]. +[EXECUTE] Instance: Deleting de3580slur-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-slurm-login-001]. +[EXECUTE] Instance: Deleting dynpoc-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/dynpoc-controller]. +[EXECUTE] Instance: Deleting dynpoc-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/dynpoc-slurm-login-001]. +--- Wed Nov 26 11:38:44 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 11:44:26 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 38 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - cluster0vk-nodeset1-0 + - cluster8ix-nodeset1-0 + - cluster8ix-nodeset1-1 + - clustermce-nodeset1-0 + - clustermce-nodeset1-1 + - clustermce-nodeset1-2 + - clusterum7-nodeset1-0 + - clusterum7-nodeset1-1 + - clusterum7-nodeset1-2 + - clusterum7-nodeset1-4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +ebf828slur-controller us-central1-a +ebf828slur-slurm-login-001 us-central1-a +exascaler-cloud-4691-mds0 us-central1-a +exascaler-cloud-4691-mgs0 us-central1-a +exascaler-cloud-4691-oss0 us-central1-a +exascaler-cloud-4691-oss1 us-central1-a +exascaler-cloud-4691-oss2 us-central1-a +exascaler-cloud-4a36-mds0 europe-west4-c +exascaler-cloud-4a36-mgs0 europe-west4-c +exascaler-cloud-4a36-oss0 europe-west4-c + gcloud compute instances delete "ebf828slur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "ebf828slur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "exascaler-cloud-4691-mds0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "exascaler-cloud-4691-mgs0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "exascaler-cloud-4691-oss0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "exascaler-cloud-4691-oss1" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "exascaler-cloud-4691-oss2" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "exascaler-cloud-4a36-mds0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "exascaler-cloud-4a36-mgs0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "exascaler-cloud-4a36-oss0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet +--- Wed Nov 26 11:44:30 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 11:45:08 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 38 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - cluster0vk-nodeset1-0 + - cluster8ix-nodeset1-0 + - cluster8ix-nodeset1-1 + - clustermce-nodeset1-0 + - clustermce-nodeset1-1 + - clustermce-nodeset1-2 + - clusterum7-nodeset1-0 + - clusterum7-nodeset1-1 + - clusterum7-nodeset1-2 + - clusterum7-nodeset1-4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +ebf828slur-controller us-central1-a +ebf828slur-slurm-login-001 us-central1-a +exascaler-cloud-4691-mds0 us-central1-a +exascaler-cloud-4691-mgs0 us-central1-a +exascaler-cloud-4691-oss0 us-central1-a +exascaler-cloud-4691-oss1 us-central1-a +exascaler-cloud-4691-oss2 us-central1-a +exascaler-cloud-4a36-mds0 europe-west4-c +exascaler-cloud-4a36-mgs0 europe-west4-c +exascaler-cloud-4a36-oss0 europe-west4-c +[EXECUTE] Instance: Deleting ebf828slur-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/ebf828slur-controller]. +[EXECUTE] Instance: Deleting ebf828slur-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/ebf828slur-slurm-login-001]. +[EXECUTE] Instance: Deleting exascaler-cloud-4691-mds0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/exascaler-cloud-4691-mds0]. +[EXECUTE] Instance: Deleting exascaler-cloud-4691-mgs0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/exascaler-cloud-4691-mgs0]. +[EXECUTE] Instance: Deleting exascaler-cloud-4691-oss0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/exascaler-cloud-4691-oss0]. +[EXECUTE] Instance: Deleting exascaler-cloud-4691-oss1 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/exascaler-cloud-4691-oss1]. +[EXECUTE] Instance: Deleting exascaler-cloud-4691-oss2 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/exascaler-cloud-4691-oss2]. +[EXECUTE] Instance: Deleting exascaler-cloud-4a36-mds0 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-4a36-mds0]. +[EXECUTE] Instance: Deleting exascaler-cloud-4a36-mgs0 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-4a36-mgs0]. +[EXECUTE] Instance: Deleting exascaler-cloud-4a36-oss0 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-4a36-oss0]. +--- Wed Nov 26 11:50:39 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 11:51:04 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 38 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - cluster0vk-nodeset1-0 + - cluster8ix-nodeset1-0 + - cluster8ix-nodeset1-1 + - clustermce-nodeset1-0 + - clustermce-nodeset1-1 + - clustermce-nodeset1-2 + - clusterum7-nodeset1-0 + - clusterum7-nodeset1-1 + - clusterum7-nodeset1-2 + - clusterum7-nodeset1-4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +exascaler-cloud-4a36-oss1 europe-west4-c +exascaler-cloud-4a36-oss2 europe-west4-c +exascaler-cloud-a2a0-mds0 europe-west4-c +exascaler-cloud-a2a0-mgs0 europe-west4-c +exascaler-cloud-a2a0-oss0 europe-west4-c +exascaler-cloud-a2a0-oss1 europe-west4-c +exascaler-cloud-a2a0-oss2 europe-west4-c +f4e324slur-controller us-central1-a +f4e324slur-slurm-login-001 us-central1-a +f88073slur-controller us-central1-a + gcloud compute instances delete "exascaler-cloud-4a36-oss1" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "exascaler-cloud-4a36-oss2" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "exascaler-cloud-a2a0-mds0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "exascaler-cloud-a2a0-mgs0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "exascaler-cloud-a2a0-oss0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "exascaler-cloud-a2a0-oss1" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "exascaler-cloud-a2a0-oss2" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "f4e324slur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "f4e324slur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "f88073slur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +--- Wed Nov 26 11:51:08 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 11:51:22 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 38 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - cluster0vk-nodeset1-0 + - cluster8ix-nodeset1-0 + - cluster8ix-nodeset1-1 + - clustermce-nodeset1-0 + - clustermce-nodeset1-1 + - clustermce-nodeset1-2 + - clusterum7-nodeset1-0 + - clusterum7-nodeset1-1 + - clusterum7-nodeset1-2 + - clusterum7-nodeset1-4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +exascaler-cloud-4a36-oss1 europe-west4-c +exascaler-cloud-4a36-oss2 europe-west4-c +exascaler-cloud-a2a0-mds0 europe-west4-c +exascaler-cloud-a2a0-mgs0 europe-west4-c +exascaler-cloud-a2a0-oss0 europe-west4-c +exascaler-cloud-a2a0-oss1 europe-west4-c +exascaler-cloud-a2a0-oss2 europe-west4-c +f4e324slur-controller us-central1-a +f4e324slur-slurm-login-001 us-central1-a +f88073slur-controller us-central1-a +[EXECUTE] Instance: Deleting exascaler-cloud-4a36-oss1 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-4a36-oss1]. +[EXECUTE] Instance: Deleting exascaler-cloud-4a36-oss2 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-4a36-oss2]. +[EXECUTE] Instance: Deleting exascaler-cloud-a2a0-mds0 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-a2a0-mds0]. +[EXECUTE] Instance: Deleting exascaler-cloud-a2a0-mgs0 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-a2a0-mgs0]. +[EXECUTE] Instance: Deleting exascaler-cloud-a2a0-oss0 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-a2a0-oss0]. +[EXECUTE] Instance: Deleting exascaler-cloud-a2a0-oss1 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-a2a0-oss1]. +[EXECUTE] Instance: Deleting exascaler-cloud-a2a0-oss2 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-a2a0-oss2]. +[EXECUTE] Instance: Deleting f4e324slur-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/f4e324slur-controller]. +[EXECUTE] Instance: Deleting f4e324slur-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/f4e324slur-slurm-login-001]. +[EXECUTE] Instance: Deleting f88073slur-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/f88073slur-controller]. +--- Wed Nov 26 11:57:54 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 11:59:13 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 28 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +f88073slur-slurm-login-001 us-central1-a +fa4slurmsi-controller us-central1-a +fa4slurmsi-slurm-login-001 us-central1-a +g4qclav-controller us-central1-b +g4qclav-g4nodeset-0 us-central1-b +g4qclav-slurm-login-001 us-central1-b +gke-1395b4-0 us-central1-a +hpcdy-controller europe-west4-c +hpcdydis-controller europe-west4-c +hpcdydis-slurm-login-001 europe-west4-c + gcloud compute instances delete "f88073slur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "fa4slurmsi-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "fa4slurmsi-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "g4qclav-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "g4qclav-g4nodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "g4qclav-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "gke-1395b4-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "hpcdy-controller" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "hpcdydis-controller" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "hpcdydis-slurm-login-001" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet +--- Wed Nov 26 11:59:18 AM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 12:00:36 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 28 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +f88073slur-slurm-login-001 us-central1-a +fa4slurmsi-controller us-central1-a +fa4slurmsi-slurm-login-001 us-central1-a +g4qclav-controller us-central1-b +g4qclav-g4nodeset-0 us-central1-b +g4qclav-slurm-login-001 us-central1-b +gke-1395b4-0 us-central1-a +hpcdy-controller europe-west4-c +hpcdydis-controller europe-west4-c +hpcdydis-slurm-login-001 europe-west4-c +[EXECUTE] Instance: Deleting f88073slur-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/f88073slur-slurm-login-001]. +[EXECUTE] Instance: Deleting fa4slurmsi-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/fa4slurmsi-controller]. +[EXECUTE] Instance: Deleting fa4slurmsi-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/fa4slurmsi-slurm-login-001]. +[EXECUTE] Instance: Deleting g4qclav-controller in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/g4qclav-controller]. +[EXECUTE] Instance: Deleting g4qclav-g4nodeset-0 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/g4qclav-g4nodeset-0]. +[EXECUTE] Instance: Deleting g4qclav-slurm-login-001 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/g4qclav-slurm-login-001]. +[EXECUTE] Instance: Deleting gke-1395b4-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/gke-1395b4-0]. +[EXECUTE] Instance: Deleting hpcdy-controller in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/hpcdy-controller]. +[EXECUTE] Instance: Deleting hpcdydis-controller in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/hpcdydis-controller]. +[EXECUTE] Instance: Deleting hpcdydis-slurm-login-001 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/hpcdydis-slurm-login-001]. +--- Wed Nov 26 12:17:01 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 12:50:16 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 28 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +The following Instances are targeted for deletion in this run: +a4he340-a4highnodeset-0 us-central1-b +a4he340-a4highnodeset-1 us-central1-b +a4he340-controller us-central1-b +a4he340-slurm-login-001 us-central1-b +hpcdy-slurm-login-001 europe-west4-c +hpcimg-controller us-central1-a +hpcimg-slurm-login-001 us-central1-a +image-inspector-550 us-west1-a +image-inspector us-west1-a +instance-20250625-210718 australia-southeast1-c + gcloud compute instances delete "a4he340-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4he340-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4he340-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4he340-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "hpcdy-slurm-login-001" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "hpcimg-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "hpcimg-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "image-inspector-550" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet + gcloud compute instances delete "image-inspector" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet + gcloud compute instances delete "instance-20250625-210718" --project="hpc-toolkit-dev" --zone="australia-southeast1-c" --quiet +--- Wed Nov 26 12:50:20 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 12:51:11 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +The following Instances are targeted for deletion in this run: +a4he340-a4highnodeset-0 us-central1-b +a4he340-a4highnodeset-1 us-central1-b +a4he340-controller us-central1-b +a4he340-slurm-login-001 us-central1-b +hpcdy-slurm-login-001 europe-west4-c +hpcimg-controller us-central1-a +hpcimg-slurm-login-001 us-central1-a +instance-20250625-210718 australia-southeast1-c +instance-20250918-073640 us-central1-b +instance-20251124-055307 us-central1-b + gcloud compute instances delete "a4he340-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4he340-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4he340-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4he340-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "hpcdy-slurm-login-001" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet + gcloud compute instances delete "hpcimg-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "hpcimg-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "instance-20250625-210718" --project="hpc-toolkit-dev" --zone="australia-southeast1-c" --quiet + gcloud compute instances delete "instance-20250918-073640" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "instance-20251124-055307" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet +--- Wed Nov 26 12:51:15 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 12:51:45 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +The following Instances are targeted for deletion in this run: +a4he340-a4highnodeset-0 us-central1-b +a4he340-a4highnodeset-1 us-central1-b +a4he340-controller us-central1-b +a4he340-slurm-login-001 us-central1-b +hpcdy-slurm-login-001 europe-west4-c +hpcimg-controller us-central1-a +hpcimg-slurm-login-001 us-central1-a +instance-20250625-210718 australia-southeast1-c +instance-20250918-073640 us-central1-b +instance-20251124-055307 us-central1-b +[EXECUTE] Instance: Deleting a4he340-a4highnodeset-0 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4he340-a4highnodeset-0]. +[EXECUTE] Instance: Deleting a4he340-a4highnodeset-1 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4he340-a4highnodeset-1]. +[EXECUTE] Instance: Deleting a4he340-controller in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4he340-controller]. +[EXECUTE] Instance: Deleting a4he340-slurm-login-001 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4he340-slurm-login-001]. +[EXECUTE] Instance: Deleting hpcdy-slurm-login-001 in europe-west4-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/hpcdy-slurm-login-001]. +[EXECUTE] Instance: Deleting hpcimg-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/hpcimg-controller]. +[EXECUTE] Instance: Deleting hpcimg-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/hpcimg-slurm-login-001]. +[EXECUTE] Instance: Deleting instance-20250625-210718 in australia-southeast1-c +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/australia-southeast1-c/instances/instance-20250625-210718]. +[EXECUTE] Instance: Deleting instance-20250918-073640 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/instance-20250918-073640]. +[EXECUTE] Instance: Deleting instance-20251124-055307 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/instance-20251124-055307]. +--- Wed Nov 26 01:04:17 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:05:06 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +The following Instances are targeted for deletion in this run: +khu-h4d-cluster-test-0 us-central1-a +khu-h4d-cluster-test-1 us-central1-a +khushi-ansible-deb11-0 us-central1-a +khushi-ansible-deb12-0 us-central1-a +khushi-ansible-failed-deb11-0 us-central1-a +khushi-ansible-failed-deb12-0 us-central1-a +khushi-ansible-failed-rhel8-0 us-central1-a +khushi-ansible-failed-rhel9-0 us-central1-a +khushi-ansible-failed-rocky8-0 us-central1-a +khushi-ansible-failed-rocky9-0 us-central1-a + gcloud compute instances delete "khu-h4d-cluster-test-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khu-h4d-cluster-test-1" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-deb11-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-deb12-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-failed-deb11-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-failed-deb12-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-failed-rhel8-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-failed-rhel9-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-failed-rocky8-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-failed-rocky9-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +--- Wed Nov 26 01:05:11 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:05:28 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +The following Instances are targeted for deletion in this run: +khu-h4d-cluster-test-0 us-central1-a +khu-h4d-cluster-test-1 us-central1-a +khushi-ansible-deb11-0 us-central1-a +khushi-ansible-deb12-0 us-central1-a +khushi-ansible-failed-deb11-0 us-central1-a +khushi-ansible-failed-deb12-0 us-central1-a +khushi-ansible-failed-rhel8-0 us-central1-a +khushi-ansible-failed-rhel9-0 us-central1-a +khushi-ansible-failed-rocky8-0 us-central1-a +khushi-ansible-failed-rocky9-0 us-central1-a +[EXECUTE] Instance: Deleting khu-h4d-cluster-test-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-h4d-cluster-test-0]. +[EXECUTE] Instance: Deleting khu-h4d-cluster-test-1 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-h4d-cluster-test-1]. +[EXECUTE] Instance: Deleting khushi-ansible-deb11-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-deb11-0]. +[EXECUTE] Instance: Deleting khushi-ansible-deb12-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-deb12-0]. +[EXECUTE] Instance: Deleting khushi-ansible-failed-deb11-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-deb11-0]. +[EXECUTE] Instance: Deleting khushi-ansible-failed-deb12-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-deb12-0]. +[EXECUTE] Instance: Deleting khushi-ansible-failed-rhel8-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-rhel8-0]. +[EXECUTE] Instance: Deleting khushi-ansible-failed-rhel9-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-rhel9-0]. +[EXECUTE] Instance: Deleting khushi-ansible-failed-rocky8-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-rocky8-0]. +[EXECUTE] Instance: Deleting khushi-ansible-failed-rocky9-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-rocky9-0]. +--- Wed Nov 26 01:16:48 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:17:01 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +The following Instances are targeted for deletion in this run: +khushi-ansible-failed-ubuntu2204-0 us-central1-a +khushi-ansible-failed-ubuntu2404-0 us-central1-a +khushi-ansible-failed-ubuntu2404arm-0 us-central1-a +khushi-ansible-rhel8-0 us-central1-a +khushi-ansible-rhel9-0 us-central1-a +khushi-ansible-rocky8-0 us-central1-a +khushi-ansible-rocky9-0 us-central1-a +khushi-ansible-ubuntu2204-0 us-central1-a +khushi-ansible-ubuntu2404-0 us-central1-a +khushi-ansible-ubuntu2404arm-0 us-central1-a +khu-test-ansible-sleep-ubuntu2204-0 us-central1-a +khu-test-ansible-sleep-ubuntu2404-0 us-central1-a +khu-test-ansible-sleep-ubuntu2404arm-0 us-central1-a +khu-test-ansible-ubuntu2204-0 us-central1-a +khu-test-ansible-ubuntu2404-0 us-central1-a +khu-test-ansible-ubuntu2404arm-0 us-central1-a +ml-gke-e2e-a8fae6-0 asia-southeast1-b +my-a3-spot-vm us-central1-a +packer-119ae1 us-west1-a +packer-2c0c79 asia-southeast1-b +packer-3e3b45 us-west1-a +packer-59a068 us-west1-a +packer-5c41cc us-south1-b +packer-84a235 europe-west4-b +packer-90b969 us-west4-a +packer-a947ca us-west1-a +temp-image-check us-west1-a +test us-central1-a +testvm01 us-central1-a +testvm727 us-central1-b + gcloud compute instances delete "khushi-ansible-failed-ubuntu2204-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-failed-ubuntu2404-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-failed-ubuntu2404arm-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-rhel8-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-rhel9-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-rocky8-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-rocky9-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-ubuntu2204-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-ubuntu2404-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khushi-ansible-ubuntu2404arm-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khu-test-ansible-sleep-ubuntu2204-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khu-test-ansible-sleep-ubuntu2404-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khu-test-ansible-sleep-ubuntu2404arm-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khu-test-ansible-ubuntu2204-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khu-test-ansible-ubuntu2404-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "khu-test-ansible-ubuntu2404arm-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "ml-gke-e2e-a8fae6-0" --project="hpc-toolkit-dev" --zone="asia-southeast1-b" --quiet + gcloud compute instances delete "my-a3-spot-vm" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "packer-119ae1" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet + gcloud compute instances delete "packer-2c0c79" --project="hpc-toolkit-dev" --zone="asia-southeast1-b" --quiet + gcloud compute instances delete "packer-3e3b45" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet + gcloud compute instances delete "packer-59a068" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet + gcloud compute instances delete "packer-5c41cc" --project="hpc-toolkit-dev" --zone="us-south1-b" --quiet + gcloud compute instances delete "packer-84a235" --project="hpc-toolkit-dev" --zone="europe-west4-b" --quiet + gcloud compute instances delete "packer-90b969" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet + gcloud compute instances delete "packer-a947ca" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet + gcloud compute instances delete "temp-image-check" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet + gcloud compute instances delete "test" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "testvm01" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "testvm727" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet +--- Wed Nov 26 01:17:06 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:17:30 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +The following Instances are targeted for deletion in this run: +khushi-ansible-failed-ubuntu2204-0 us-central1-a +khushi-ansible-failed-ubuntu2404-0 us-central1-a +khushi-ansible-failed-ubuntu2404arm-0 us-central1-a +khushi-ansible-rhel8-0 us-central1-a +khushi-ansible-rhel9-0 us-central1-a +khushi-ansible-rocky8-0 us-central1-a +khushi-ansible-rocky9-0 us-central1-a +khushi-ansible-ubuntu2204-0 us-central1-a +khushi-ansible-ubuntu2404-0 us-central1-a +khushi-ansible-ubuntu2404arm-0 us-central1-a +khu-test-ansible-sleep-ubuntu2204-0 us-central1-a +khu-test-ansible-sleep-ubuntu2404-0 us-central1-a +khu-test-ansible-sleep-ubuntu2404arm-0 us-central1-a +khu-test-ansible-ubuntu2204-0 us-central1-a +khu-test-ansible-ubuntu2404-0 us-central1-a +khu-test-ansible-ubuntu2404arm-0 us-central1-a +ml-gke-e2e-a8fae6-0 asia-southeast1-b +my-a3-spot-vm us-central1-a +packer-119ae1 us-west1-a +packer-2c0c79 asia-southeast1-b +packer-3e3b45 us-west1-a +packer-59a068 us-west1-a +packer-5c41cc us-south1-b +packer-84a235 europe-west4-b +packer-90b969 us-west4-a +packer-a947ca us-west1-a +temp-image-check us-west1-a +test us-central1-a +testvm01 us-central1-a +testvm727 us-central1-b +[EXECUTE] Instance: Deleting khushi-ansible-failed-ubuntu2204-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-ubuntu2204-0]. +[EXECUTE] Instance: Deleting khushi-ansible-failed-ubuntu2404-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-ubuntu2404-0]. +[EXECUTE] Instance: Deleting khushi-ansible-failed-ubuntu2404arm-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-ubuntu2404arm-0]. +[EXECUTE] Instance: Deleting khushi-ansible-rhel8-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-rhel8-0]. +[EXECUTE] Instance: Deleting khushi-ansible-rhel9-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-rhel9-0]. +[EXECUTE] Instance: Deleting khushi-ansible-rocky8-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-rocky8-0]. +[EXECUTE] Instance: Deleting khushi-ansible-rocky9-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-rocky9-0]. +[EXECUTE] Instance: Deleting khushi-ansible-ubuntu2204-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-ubuntu2204-0]. +[EXECUTE] Instance: Deleting khushi-ansible-ubuntu2404-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-ubuntu2404-0]. +[EXECUTE] Instance: Deleting khushi-ansible-ubuntu2404arm-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-ubuntu2404arm-0]. +[EXECUTE] Instance: Deleting khu-test-ansible-sleep-ubuntu2204-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-sleep-ubuntu2204-0]. +[EXECUTE] Instance: Deleting khu-test-ansible-sleep-ubuntu2404-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-sleep-ubuntu2404-0]. +[EXECUTE] Instance: Deleting khu-test-ansible-sleep-ubuntu2404arm-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-sleep-ubuntu2404arm-0]. +[EXECUTE] Instance: Deleting khu-test-ansible-ubuntu2204-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-ubuntu2204-0]. +[EXECUTE] Instance: Deleting khu-test-ansible-ubuntu2404-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-ubuntu2404-0]. +[EXECUTE] Instance: Deleting khu-test-ansible-ubuntu2404arm-0 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-ubuntu2404arm-0]. +[EXECUTE] Instance: Deleting ml-gke-e2e-a8fae6-0 in asia-southeast1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b/instances/ml-gke-e2e-a8fae6-0]. +[EXECUTE] Instance: Deleting my-a3-spot-vm in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/my-a3-spot-vm]. +[EXECUTE] Instance: Deleting packer-119ae1 in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/packer-119ae1]. +[EXECUTE] Instance: Deleting packer-2c0c79 in asia-southeast1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b/instances/packer-2c0c79]. +[EXECUTE] Instance: Deleting packer-3e3b45 in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/packer-3e3b45]. +[EXECUTE] Instance: Deleting packer-59a068 in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/packer-59a068]. +[EXECUTE] Instance: Deleting packer-5c41cc in us-south1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b/instances/packer-5c41cc]. +[EXECUTE] Instance: Deleting packer-84a235 in europe-west4-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b/instances/packer-84a235]. +[EXECUTE] Instance: Deleting packer-90b969 in us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/instances/packer-90b969]. +[EXECUTE] Instance: Deleting packer-a947ca in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/packer-a947ca]. +[EXECUTE] Instance: Deleting temp-image-check in us-west1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/temp-image-check]. +[EXECUTE] Instance: Deleting test in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/test]. +[EXECUTE] Instance: Deleting testvm01 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/testvm01]. +[EXECUTE] Instance: Deleting testvm727 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/testvm727]. +--- Wed Nov 26 01:41:23 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:41:38 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +today24 us-central1-b + gcloud compute instances delete "today24" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet +--- Wed Nov 26 01:41:42 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:41:56 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +today24 us-central1-b +[EXECUTE] Instance: Deleting today24 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/today24]. +--- Wed Nov 26 01:42:48 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Thu Nov 27 03:21:06 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Targeting resources created before: 2025-11-26T23:21:06+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +./cleanup.sh: line 72: ---: command not found +--- Deletion Phase 1: GKE Clusters (Top 20) --- +The following GKE clusters are targeted for deletion in this run: +gke-a3-nccl-test us-west4 + gcloud container clusters delete "gke-a3-nccl-test" --project="hpc-toolkit-dev" --location="us-west4" --quiet +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +a4hc0e2-a4highnodeset-0 us-central1-b +a4hc0e2-a4highnodeset-1 us-central1-b +a4hc0e2-controller us-central1-b +a4hc0e2-slurm-login-001 us-central1-b +gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk us-west4-a +gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 us-west4-a +gke-gke-a3-nccl-test-system-17f71453-fns8 us-west4-c + gcloud compute instances delete "a4hc0e2-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4hc0e2-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4hc0e2-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4hc0e2-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet + gcloud compute instances delete "gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet + gcloud compute instances delete "gke-gke-a3-nccl-test-system-17f71453-fns8" --project="hpc-toolkit-dev" --zone="us-west4-c" --quiet +--- Thu Nov 27 03:21:10 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 03:27:17 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Targeting resources created before: 2025-11-26T23:27:17+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +./cleanup.sh: line 77: ---: command not found +--- Deletion Phase 1: GKE Clusters (Top 20) --- +./cleanup.sh: line 50: resource_name: unbound variable +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +./cleanup.sh: line 50: resource_name: unbound variable +No Instances found to delete in this run. +--- Thu Nov 27 03:27:22 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 03:27:52 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Targeting resources created before: 2025-11-26T23:27:52+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +./cleanup.sh: line 77: ---: command not found +--- Deletion Phase 1: GKE Clusters (Top 20) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +a4hc0e2-a4highnodeset-0 us-central1-b +a4hc0e2-a4highnodeset-1 us-central1-b +a4hc0e2-controller us-central1-b +a4hc0e2-slurm-login-001 us-central1-b + gcloud compute instances delete "a4hc0e2-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4hc0e2-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4hc0e2-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet + gcloud compute instances delete "a4hc0e2-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet +--- Thu Nov 27 03:27:57 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 03:28:56 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 20 resources of each type per run. +Targeting resources created before: 2025-11-26T23:28:56+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +./cleanup.sh: line 77: ---: command not found +--- Deletion Phase 1: GKE Clusters (Top 20) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +a4hc0e2-a4highnodeset-0 us-central1-b +a4hc0e2-a4highnodeset-1 us-central1-b +a4hc0e2-controller us-central1-b +a4hc0e2-slurm-login-001 us-central1-b +[EXECUTE] Instance: Deleting a4hc0e2-a4highnodeset-0 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc0e2-a4highnodeset-0]. +[EXECUTE] Instance: Deleting a4hc0e2-a4highnodeset-1 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc0e2-a4highnodeset-1]. +[EXECUTE] Instance: Deleting a4hc0e2-controller in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc0e2-controller]. +[EXECUTE] Instance: Deleting a4hc0e2-slurm-login-001 in us-central1-b +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc0e2-slurm-login-001]. +--- Thu Nov 27 03:35:47 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 03:36:13 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Targeting resources created before: 2025-11-26T23:36:13+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +./cleanup.sh: line 77: ---: command not found +--- Deletion Phase 1: GKE Clusters (Top 20) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +The following Filestore instances are targeted for deletion in this run: +a4h-slurm-c0e262-f5260d85 +[DRY RUN] Filestore Instance: Would delete a4h-slurm-c0e262-f5260d85 in + Command: gcloud filestore instances delete "a4h-slurm-c0e262-f5260d85" --project="hpc-toolkit-dev" --location="" --quiet --force +--- Thu Nov 27 03:36:19 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 03:36:45 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 20 resources of each type per run. +Targeting resources created before: 2025-11-26T23:36:45+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +./cleanup.sh: line 77: ---: command not found +--- Deletion Phase 1: GKE Clusters (Top 20) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +The following Filestore instances are targeted for deletion in this run: +a4h-slurm-c0e262-f5260d85 +[EXECUTE] Filestore Instance: Deleting a4h-slurm-c0e262-f5260d85 in +ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. +The [instance] resource is not properly specified. +Failed to find attribute [zone]. The attribute can be set in the following ways: +- provide the argument `instance` on the command line with a fully specified name +- provide the argument `--zone` on the command line +- provide the argument `region` on the command line +- provide the argument `location` on the command line +- set the property `filestore/zone` +- set the property `filestore/region` +- set the property `filestore/location` +--- Thu Nov 27 03:36:52 AM UTC 2025 --- Cleanup Script Run Finished --- + diff --git a/network.txt b/network.txt new file mode 100644 index 0000000000..03333e56ef --- /dev/null +++ b/network.txt @@ -0,0 +1,2708 @@ +--- Thu Nov 27 09:55:54 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T05:55:54+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +lustre-06-primary-subnet in us-central1 +lustre-test-06-primary-subnet in us-central1 +--- Processing Subnet: lustre-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-06-primary-subnet in us-central1. +[EXECUTE] Subnetwork: Deleting lustre-06-primary-subnet in us-central1 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustre06-slurm-login-001' + +ERROR: Failed to delete Subnetwork lustre-06-primary-subnet in us-central1. Check for other dependencies. +--- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-test-06-primary-subnet in us-central1. +[EXECUTE] Subnetwork: Deleting lustre-test-06-primary-subnet in us-central1 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-test-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustretest-controller' + +ERROR: Failed to delete Subnetwork lustre-test-06-primary-subnet in us-central1. Check for other dependencies. +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +The following Networks are targeted for deletion in this run: +a3u-onspot-slurm-829610-net-0 +a3u-slurm-net +a4hsarthakag-net +a4htest-net-0 +a4newimgek-net +a4newimgek-net-0 +a4newimgek-net-1 +a4newimgek-rdma-net +a4oldimgek-net +a4oldimgek-net-0 +a4oldimgek-net-1 +a4oldimgek-rdma-net +a4oldimg-net +a4xlavhpcnew-a4x-net-0 +a4xlavhpcnew-a4x-net-1 +a4xlavhpcnew-a4x-rdma-net +a4xslurm-net +cx-a3u-net-0 +db451c7-ml-slurm-v6-net +dynpoc-net +g4-dwsq-1-net-1 +g4qclav-net +gke-1395b4-net +gke-managed-lustre-basic-net +h4d-cluster-rdma-net-0 +h4dqc-net +h4dqc-rdma-net-0 +h4d-res-swarnabm4-3-net +h4d-res-swarnabm4-3-rdma-net +hanu-a3u-net +[EXECUTE] Network: Deleting a3u-onspot-slurm-829610-net-0 +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/a3u-onspot-slurm-829610-net-0' is already being used by 'projects/hpc-toolkit-dev/global/routes/default-route-ea3b20e196a82d45' + +[EXECUTE] Network: Deleting a3u-slurm-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/global/networks/a3u-slurm-net' was not found + +[EXECUTE] Network: Deleting a4hsarthakag-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/a4hsarthakag-net' is already being used by 'projects/hpc-toolkit-dev/global/routes/default-route-6884ebdc9d4edd99' + +[EXECUTE] Network: Deleting a4htest-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4htest-net-0]. +[EXECUTE] Network: Deleting a4newimgek-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4newimgek-net]. +[EXECUTE] Network: Deleting a4newimgek-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4newimgek-net-0]. +[EXECUTE] Network: Deleting a4newimgek-net-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4newimgek-net-1]. +[EXECUTE] Network: Deleting a4newimgek-rdma-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4newimgek-rdma-net]. +[EXECUTE] Network: Deleting a4oldimgek-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4oldimgek-net]. +[EXECUTE] Network: Deleting a4oldimgek-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4oldimgek-net-0]. +[EXECUTE] Network: Deleting a4oldimgek-net-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4oldimgek-net-1]. +[EXECUTE] Network: Deleting a4oldimgek-rdma-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4oldimgek-rdma-net]. +[EXECUTE] Network: Deleting a4oldimg-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4oldimg-net]. +[EXECUTE] Network: Deleting a4xlavhpcnew-a4x-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4xlavhpcnew-a4x-net-0]. +[EXECUTE] Network: Deleting a4xlavhpcnew-a4x-net-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4xlavhpcnew-a4x-net-1]. +[EXECUTE] Network: Deleting a4xlavhpcnew-a4x-rdma-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4xlavhpcnew-a4x-rdma-net]. +[EXECUTE] Network: Deleting a4xslurm-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4xslurm-net]. +[EXECUTE] Network: Deleting cx-a3u-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/cx-a3u-net-0]. +[EXECUTE] Network: Deleting db451c7-ml-slurm-v6-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/db451c7-ml-slurm-v6-net]. +[EXECUTE] Network: Deleting dynpoc-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/dynpoc-net]. +[EXECUTE] Network: Deleting g4-dwsq-1-net-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/g4-dwsq-1-net-1]. +[EXECUTE] Network: Deleting g4qclav-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/g4qclav-net]. +[EXECUTE] Network: Deleting gke-1395b4-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/gke-1395b4-net]. +[EXECUTE] Network: Deleting gke-managed-lustre-basic-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/gke-managed-lustre-basic-net' is already being used by 'projects/hpc-toolkit-dev/global/firewalls/gke-managed-lustre-basic-net-fw-allow-iap-ingress' + +[EXECUTE] Network: Deleting h4d-cluster-rdma-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/h4d-cluster-rdma-net-0]. +[EXECUTE] Network: Deleting h4dqc-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/h4dqc-net]. +[EXECUTE] Network: Deleting h4dqc-rdma-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/h4dqc-rdma-net-0]. +[EXECUTE] Network: Deleting h4d-res-swarnabm4-3-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/h4d-res-swarnabm4-3-net]. +[EXECUTE] Network: Deleting h4d-res-swarnabm4-3-rdma-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/h4d-res-swarnabm4-3-rdma-net]. +[EXECUTE] Network: Deleting hanu-a3u-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hanu-a3u-net]. +./cleanup.sh: line 470: syntax error near unexpected token `then' +./cleanup.sh: line 470: ` then' +--- Thu Nov 27 10:14:54 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T06:14:54+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +lustre-06-primary-subnet in us-central1 +lustre-test-06-primary-subnet in us-central1 +--- Processing Subnet: lustre-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-06-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-test-06-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +a3u-onspot-slurm-829610-net-0 +a4hsarthakag-net +gke-managed-lustre-basic-net +hanu-test-net +hpc-01-net +hpcdydis-net +hpcdy-net +hpc-exr-2-net-0 +hpcimg-net +hpc-lustre-test-02-net +khu-h4d-cluster-test-net +khu-h4d-cluster-test-rdma-net-0 +laveeek29-net-0 +laveeek29-net-1 +laveeek29-net +laveeek29-rdma-net +lavoldchk-net +lavold-net +lavrohek29-net-0 +lustre-06-net +lustre-test-06-net +mainek-net-0 +mainek-net-1 +mainek-net +mainek-rdma-net +managed-lustre-03-net +mglsa-net +mglsard-net +ml-gke-e2e-a8fae6-net +ml-gke-net +--- Processing Network: a3u-onspot-slurm-829610-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-ea3b20e196a82d45 for network a3u-onspot-slurm-829610-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete a3u-onspot-slurm-829610-net-0 + Command: gcloud compute networks delete "a3u-onspot-slurm-829610-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: a4hsarthakag-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-6884ebdc9d4edd99 for network a4hsarthakag-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete a4hsarthakag-net + Command: gcloud compute networks delete "a4hsarthakag-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: gke-managed-lustre-basic-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-98972e97de7c239b for network gke-managed-lustre-basic-net +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete gke-managed-lustre-basic-net-fw-allow-iap-ingress for network gke-managed-lustre-basic-net +[DRY RUN] Firewall Rule: Would delete gke-managed-lustre-basic-net-fw-allow-internal-traffic for network gke-managed-lustre-basic-net +[DRY RUN] Network: Would delete gke-managed-lustre-basic-net + Command: gcloud compute networks delete "gke-managed-lustre-basic-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hanu-test-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-d487416773e0d26a for network hanu-test-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hanu-test-net + Command: gcloud compute networks delete "hanu-test-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpc-01-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-bff8e31ecb82ca1f for network hpc-01-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpc-01-net + Command: gcloud compute networks delete "hpc-01-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpcdydis-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-83226775efe0e36f for network hpcdydis-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpcdydis-net + Command: gcloud compute networks delete "hpcdydis-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpcdy-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-11ccedffb21834f9 for network hpcdy-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpcdy-net + Command: gcloud compute networks delete "hpcdy-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpc-exr-2-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-ca20f8f43f32a6ad for network hpc-exr-2-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpc-exr-2-net-0 + Command: gcloud compute networks delete "hpc-exr-2-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpcimg-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-78cfceb7232d782e for network hpcimg-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpcimg-net + Command: gcloud compute networks delete "hpcimg-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpc-lustre-test-02-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-0d79bd2bca25d91e for network hpc-lustre-test-02-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpc-lustre-test-02-net + Command: gcloud compute networks delete "hpc-lustre-test-02-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: khu-h4d-cluster-test-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-87a3ee47eaab9ab6 for network khu-h4d-cluster-test-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete khu-h4d-cluster-test-net + Command: gcloud compute networks delete "khu-h4d-cluster-test-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: khu-h4d-cluster-test-rdma-net-0 --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete khu-h4d-cluster-test-rdma-net-0 + Command: gcloud compute networks delete "khu-h4d-cluster-test-rdma-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: laveeek29-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-1a384a0ae7610067 for network laveeek29-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete laveeek29-net-0 + Command: gcloud compute networks delete "laveeek29-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: laveeek29-net-1 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-70d6a0878f6e1100 for network laveeek29-net-1 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete laveeek29-net-1 + Command: gcloud compute networks delete "laveeek29-net-1" --project="hpc-toolkit-dev" --quiet +--- Processing Network: laveeek29-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-1a384a0ae7610067 for network laveeek29-net +[DRY RUN] Route: Would delete default-route-3f3b00742a1eca0f for network laveeek29-net +[DRY RUN] Route: Would delete default-route-70d6a0878f6e1100 for network laveeek29-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete laveeek29-net + Command: gcloud compute networks delete "laveeek29-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: laveeek29-rdma-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete laveeek29-rdma-net + Command: gcloud compute networks delete "laveeek29-rdma-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lavoldchk-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-5df8aadab663ab36 for network lavoldchk-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lavoldchk-net + Command: gcloud compute networks delete "lavoldchk-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lavold-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-6e021fe7f303c1fe for network lavold-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lavold-net + Command: gcloud compute networks delete "lavold-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lavrohek29-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-3af10cb6b3f7804e for network lavrohek29-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lavrohek29-net-0 + Command: gcloud compute networks delete "lavrohek29-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-a798499d8bf40405 for network lustre-06-net +[DRY RUN] Route: Would delete default-route-r-d1138650da68a40d for network lustre-06-net +[DRY RUN] Route: Would delete peering-route-ce292d9a6fb17bbc for network lustre-06-net +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete lustre-06-net-fw-allow-iap-ingress for network lustre-06-net +[DRY RUN] Firewall Rule: Would delete lustre-06-net-fw-allow-internal-traffic for network lustre-06-net +[DRY RUN] Network: Would delete lustre-06-net + Command: gcloud compute networks delete "lustre-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-test-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-799f13a785b25782 for network lustre-test-06-net +[DRY RUN] Route: Would delete default-route-r-802e2cad48aae840 for network lustre-test-06-net +[DRY RUN] Route: Would delete peering-route-2273ac2f3dccd077 for network lustre-test-06-net +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete lustre-test-06-net-fw-allow-iap-ingress for network lustre-test-06-net +[DRY RUN] Firewall Rule: Would delete lustre-test-06-net-fw-allow-internal-traffic for network lustre-test-06-net +[DRY RUN] Network: Would delete lustre-test-06-net + Command: gcloud compute networks delete "lustre-test-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mainek-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-4dce75664c4c0cf2 for network mainek-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mainek-net-0 + Command: gcloud compute networks delete "mainek-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mainek-net-1 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-ceb2655d4bb6b336 for network mainek-net-1 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mainek-net-1 + Command: gcloud compute networks delete "mainek-net-1" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mainek-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-4dce75664c4c0cf2 for network mainek-net +[DRY RUN] Route: Would delete default-route-936046db1078bf03 for network mainek-net +[DRY RUN] Route: Would delete default-route-ceb2655d4bb6b336 for network mainek-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mainek-net + Command: gcloud compute networks delete "mainek-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mainek-rdma-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mainek-rdma-net + Command: gcloud compute networks delete "mainek-rdma-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: managed-lustre-03-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-912e02dbc2c43267 for network managed-lustre-03-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete managed-lustre-03-net + Command: gcloud compute networks delete "managed-lustre-03-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mglsa-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-111ca8b8a3346fb4 for network mglsa-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mglsa-net + Command: gcloud compute networks delete "mglsa-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mglsard-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-4afd256f7ddc3be2 for network mglsard-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mglsard-net + Command: gcloud compute networks delete "mglsard-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: ml-gke-e2e-a8fae6-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-2c3a152cc75c7587 for network ml-gke-e2e-a8fae6-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete ml-gke-e2e-a8fae6-net + Command: gcloud compute networks delete "ml-gke-e2e-a8fae6-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: ml-gke-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-1dadbcd4c82ffb80 for network ml-gke-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete ml-gke-net + Command: gcloud compute networks delete "ml-gke-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Thu Nov 27 10:17:13 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 10:17:29 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T06:17:29+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 56 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +lustre06-controller us-central1-a +lustre06-slurm-login-001 us-central1-a +lustretest-controller us-central1-a +lustretest-slurm-login-001 us-central1-a + gcloud compute instances delete "lustre06-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "lustre06-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "lustretest-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "lustretest-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +--- Deletion Phase 1: Filestore Instances (Top 30) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +lustre-06-primary-subnet in us-central1 +lustre-test-06-primary-subnet in us-central1 +--- Processing Subnet: lustre-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-06-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-test-06-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +a3u-onspot-slurm-829610-net-0 +a4hsarthakag-net +gke-managed-lustre-basic-net +hanu-test-net +hpc-01-net +hpcdydis-net +hpcdy-net +hpc-exr-2-net-0 +hpcimg-net +hpc-lustre-test-02-net +khu-h4d-cluster-test-net +khu-h4d-cluster-test-rdma-net-0 +laveeek29-net-0 +laveeek29-net-1 +laveeek29-net +laveeek29-rdma-net +lavoldchk-net +lavold-net +lavrohek29-net-0 +lustre-06-net +lustre-test-06-net +mainek-net-0 +mainek-net-1 +mainek-net +mainek-rdma-net +managed-lustre-03-net +mglsa-net +mglsard-net +ml-gke-e2e-a8fae6-net +ml-gke-net +--- Processing Network: a3u-onspot-slurm-829610-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-ea3b20e196a82d45 for network a3u-onspot-slurm-829610-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete a3u-onspot-slurm-829610-net-0 + Command: gcloud compute networks delete "a3u-onspot-slurm-829610-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: a4hsarthakag-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-6884ebdc9d4edd99 for network a4hsarthakag-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete a4hsarthakag-net + Command: gcloud compute networks delete "a4hsarthakag-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: gke-managed-lustre-basic-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-98972e97de7c239b for network gke-managed-lustre-basic-net +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete gke-managed-lustre-basic-net-fw-allow-iap-ingress for network gke-managed-lustre-basic-net +[DRY RUN] Firewall Rule: Would delete gke-managed-lustre-basic-net-fw-allow-internal-traffic for network gke-managed-lustre-basic-net +[DRY RUN] Network: Would delete gke-managed-lustre-basic-net + Command: gcloud compute networks delete "gke-managed-lustre-basic-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hanu-test-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-d487416773e0d26a for network hanu-test-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hanu-test-net + Command: gcloud compute networks delete "hanu-test-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpc-01-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-bff8e31ecb82ca1f for network hpc-01-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpc-01-net + Command: gcloud compute networks delete "hpc-01-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpcdydis-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-83226775efe0e36f for network hpcdydis-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpcdydis-net + Command: gcloud compute networks delete "hpcdydis-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpcdy-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-11ccedffb21834f9 for network hpcdy-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpcdy-net + Command: gcloud compute networks delete "hpcdy-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpc-exr-2-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-ca20f8f43f32a6ad for network hpc-exr-2-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpc-exr-2-net-0 + Command: gcloud compute networks delete "hpc-exr-2-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpcimg-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-78cfceb7232d782e for network hpcimg-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpcimg-net + Command: gcloud compute networks delete "hpcimg-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: hpc-lustre-test-02-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-0d79bd2bca25d91e for network hpc-lustre-test-02-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete hpc-lustre-test-02-net + Command: gcloud compute networks delete "hpc-lustre-test-02-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: khu-h4d-cluster-test-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-87a3ee47eaab9ab6 for network khu-h4d-cluster-test-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete khu-h4d-cluster-test-net + Command: gcloud compute networks delete "khu-h4d-cluster-test-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: khu-h4d-cluster-test-rdma-net-0 --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete khu-h4d-cluster-test-rdma-net-0 + Command: gcloud compute networks delete "khu-h4d-cluster-test-rdma-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: laveeek29-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-1a384a0ae7610067 for network laveeek29-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete laveeek29-net-0 + Command: gcloud compute networks delete "laveeek29-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: laveeek29-net-1 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-70d6a0878f6e1100 for network laveeek29-net-1 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete laveeek29-net-1 + Command: gcloud compute networks delete "laveeek29-net-1" --project="hpc-toolkit-dev" --quiet +--- Processing Network: laveeek29-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-1a384a0ae7610067 for network laveeek29-net +[DRY RUN] Route: Would delete default-route-3f3b00742a1eca0f for network laveeek29-net +[DRY RUN] Route: Would delete default-route-70d6a0878f6e1100 for network laveeek29-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete laveeek29-net + Command: gcloud compute networks delete "laveeek29-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: laveeek29-rdma-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete laveeek29-rdma-net + Command: gcloud compute networks delete "laveeek29-rdma-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lavoldchk-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-5df8aadab663ab36 for network lavoldchk-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lavoldchk-net + Command: gcloud compute networks delete "lavoldchk-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lavold-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-6e021fe7f303c1fe for network lavold-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lavold-net + Command: gcloud compute networks delete "lavold-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lavrohek29-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-3af10cb6b3f7804e for network lavrohek29-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lavrohek29-net-0 + Command: gcloud compute networks delete "lavrohek29-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-a798499d8bf40405 for network lustre-06-net +[DRY RUN] Route: Would delete default-route-r-d1138650da68a40d for network lustre-06-net +[DRY RUN] Route: Would delete peering-route-ce292d9a6fb17bbc for network lustre-06-net +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete lustre-06-net-fw-allow-iap-ingress for network lustre-06-net +[DRY RUN] Firewall Rule: Would delete lustre-06-net-fw-allow-internal-traffic for network lustre-06-net +[DRY RUN] Network: Would delete lustre-06-net + Command: gcloud compute networks delete "lustre-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-test-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-799f13a785b25782 for network lustre-test-06-net +[DRY RUN] Route: Would delete default-route-r-802e2cad48aae840 for network lustre-test-06-net +[DRY RUN] Route: Would delete peering-route-2273ac2f3dccd077 for network lustre-test-06-net +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete lustre-test-06-net-fw-allow-iap-ingress for network lustre-test-06-net +[DRY RUN] Firewall Rule: Would delete lustre-test-06-net-fw-allow-internal-traffic for network lustre-test-06-net +[DRY RUN] Network: Would delete lustre-test-06-net + Command: gcloud compute networks delete "lustre-test-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mainek-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-4dce75664c4c0cf2 for network mainek-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mainek-net-0 + Command: gcloud compute networks delete "mainek-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mainek-net-1 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-ceb2655d4bb6b336 for network mainek-net-1 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mainek-net-1 + Command: gcloud compute networks delete "mainek-net-1" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mainek-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-4dce75664c4c0cf2 for network mainek-net +[DRY RUN] Route: Would delete default-route-936046db1078bf03 for network mainek-net +[DRY RUN] Route: Would delete default-route-ceb2655d4bb6b336 for network mainek-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mainek-net + Command: gcloud compute networks delete "mainek-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mainek-rdma-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mainek-rdma-net + Command: gcloud compute networks delete "mainek-rdma-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: managed-lustre-03-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-912e02dbc2c43267 for network managed-lustre-03-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete managed-lustre-03-net + Command: gcloud compute networks delete "managed-lustre-03-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mglsa-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-111ca8b8a3346fb4 for network mglsa-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mglsa-net + Command: gcloud compute networks delete "mglsa-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mglsard-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-4afd256f7ddc3be2 for network mglsard-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mglsard-net + Command: gcloud compute networks delete "mglsard-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: ml-gke-e2e-a8fae6-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-2c3a152cc75c7587 for network ml-gke-e2e-a8fae6-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete ml-gke-e2e-a8fae6-net + Command: gcloud compute networks delete "ml-gke-e2e-a8fae6-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: ml-gke-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-1dadbcd4c82ffb80 for network ml-gke-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete ml-gke-net + Command: gcloud compute networks delete "ml-gke-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Thu Nov 27 10:19:45 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 10:19:57 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T06:19:57+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 50 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +lustre06-controller us-central1-a +lustre06-slurm-login-001 us-central1-a +lustretest-controller us-central1-a +lustretest-slurm-login-001 us-central1-a +[EXECUTE] Instance: Deleting lustre06-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustre06-controller]. +[EXECUTE] Instance: Deleting lustre06-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustre06-slurm-login-001]. +[EXECUTE] Instance: Deleting lustretest-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustretest-controller]. +[EXECUTE] Instance: Deleting lustretest-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustretest-slurm-login-001]. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +lustre-06-net-router us-central1 +lustre-test-06-net-router us-central1 +[EXECUTE] Cloud Router: Deleting lustre-06-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/lustre-06-net-router]. +[EXECUTE] Cloud Router: Deleting lustre-test-06-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/lustre-test-06-net-router]. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +lustre-06-net-fw-allow-iap-ingress +lustre-06-net-fw-allow-internal-traffic +lustre-test-06-net-fw-allow-iap-ingress +lustre-test-06-net-fw-allow-internal-traffic +[EXECUTE] Firewall Rule: Deleting lustre-06-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-06-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting lustre-06-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-06-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting lustre-test-06-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-test-06-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting lustre-test-06-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-test-06-net-fw-allow-internal-traffic]. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +lustre-06-primary-subnet in us-central1 +lustre-test-06-primary-subnet in us-central1 +--- Processing Subnet: lustre-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-06-primary-subnet in us-central1. +[EXECUTE] Subnetwork: Deleting lustre-06-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-06-primary-subnet]. +Successfully deleted Subnetwork lustre-06-primary-subnet in us-central1. +--- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-test-06-primary-subnet in us-central1. +[EXECUTE] Subnetwork: Deleting lustre-test-06-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-test-06-primary-subnet]. +Successfully deleted Subnetwork lustre-test-06-primary-subnet in us-central1. +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +a3u-onspot-slurm-829610-net-0 +a4hsarthakag-net +gke-managed-lustre-basic-net +hanu-test-net +hpc-01-net +hpcdydis-net +hpcdy-net +hpc-exr-2-net-0 +hpcimg-net +hpc-lustre-test-02-net +khu-h4d-cluster-test-net +khu-h4d-cluster-test-rdma-net-0 +laveeek29-net-0 +laveeek29-net-1 +laveeek29-net +laveeek29-rdma-net +lavoldchk-net +lavold-net +lavrohek29-net-0 +lustre-06-net +lustre-test-06-net +mainek-net-0 +mainek-net-1 +mainek-net +mainek-rdma-net +managed-lustre-03-net +mglsa-net +mglsard-net +ml-gke-e2e-a8fae6-net +ml-gke-net +--- Processing Network: a3u-onspot-slurm-829610-net-0 --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-ea3b20e196a82d45 for network a3u-onspot-slurm-829610-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-ea3b20e196a82d45]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting a3u-onspot-slurm-829610-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a3u-onspot-slurm-829610-net-0]. +Successfully deleted Network a3u-onspot-slurm-829610-net-0. +--- Processing Network: a4hsarthakag-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-6884ebdc9d4edd99 for network a4hsarthakag-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-6884ebdc9d4edd99]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting a4hsarthakag-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4hsarthakag-net]. +Successfully deleted Network a4hsarthakag-net. +--- Processing Network: gke-managed-lustre-basic-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-98972e97de7c239b for network gke-managed-lustre-basic-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-98972e97de7c239b]. +Checking for dependent firewall rules... +[EXECUTE] Firewall Rule: Deleting gke-managed-lustre-basic-net-fw-allow-iap-ingress for network gke-managed-lustre-basic-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/gke-managed-lustre-basic-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting gke-managed-lustre-basic-net-fw-allow-internal-traffic for network gke-managed-lustre-basic-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/gke-managed-lustre-basic-net-fw-allow-internal-traffic]. +[EXECUTE] Network: Deleting gke-managed-lustre-basic-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/gke-managed-lustre-basic-net]. +Successfully deleted Network gke-managed-lustre-basic-net. +--- Processing Network: hanu-test-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-d487416773e0d26a for network hanu-test-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-d487416773e0d26a]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting hanu-test-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hanu-test-net]. +Successfully deleted Network hanu-test-net. +--- Processing Network: hpc-01-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-bff8e31ecb82ca1f for network hpc-01-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-bff8e31ecb82ca1f]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting hpc-01-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpc-01-net]. +Successfully deleted Network hpc-01-net. +--- Processing Network: hpcdydis-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-83226775efe0e36f for network hpcdydis-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-83226775efe0e36f]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting hpcdydis-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpcdydis-net]. +Successfully deleted Network hpcdydis-net. +--- Processing Network: hpcdy-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-11ccedffb21834f9 for network hpcdy-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-11ccedffb21834f9]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting hpcdy-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpcdy-net]. +Successfully deleted Network hpcdy-net. +--- Processing Network: hpc-exr-2-net-0 --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-ca20f8f43f32a6ad for network hpc-exr-2-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-ca20f8f43f32a6ad]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting hpc-exr-2-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpc-exr-2-net-0]. +Successfully deleted Network hpc-exr-2-net-0. +--- Processing Network: hpcimg-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-78cfceb7232d782e for network hpcimg-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-78cfceb7232d782e]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting hpcimg-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpcimg-net]. +Successfully deleted Network hpcimg-net. +--- Processing Network: hpc-lustre-test-02-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-0d79bd2bca25d91e for network hpc-lustre-test-02-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-0d79bd2bca25d91e]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting hpc-lustre-test-02-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpc-lustre-test-02-net]. +Successfully deleted Network hpc-lustre-test-02-net. +--- Processing Network: khu-h4d-cluster-test-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-87a3ee47eaab9ab6 for network khu-h4d-cluster-test-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-87a3ee47eaab9ab6]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting khu-h4d-cluster-test-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/khu-h4d-cluster-test-net]. +Successfully deleted Network khu-h4d-cluster-test-net. +--- Processing Network: khu-h4d-cluster-test-rdma-net-0 --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting khu-h4d-cluster-test-rdma-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/khu-h4d-cluster-test-rdma-net-0]. +Successfully deleted Network khu-h4d-cluster-test-rdma-net-0. +--- Processing Network: laveeek29-net-0 --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-1a384a0ae7610067 for network laveeek29-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-1a384a0ae7610067]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting laveeek29-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/laveeek29-net-0]. +Successfully deleted Network laveeek29-net-0. +--- Processing Network: laveeek29-net-1 --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-70d6a0878f6e1100 for network laveeek29-net-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-70d6a0878f6e1100]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting laveeek29-net-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/laveeek29-net-1]. +Successfully deleted Network laveeek29-net-1. +--- Processing Network: laveeek29-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-3f3b00742a1eca0f for network laveeek29-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-3f3b00742a1eca0f]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting laveeek29-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/laveeek29-net]. +Successfully deleted Network laveeek29-net. +--- Processing Network: laveeek29-rdma-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting laveeek29-rdma-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/laveeek29-rdma-net]. +Successfully deleted Network laveeek29-rdma-net. +--- Processing Network: lavoldchk-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-5df8aadab663ab36 for network lavoldchk-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-5df8aadab663ab36]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lavoldchk-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/lavoldchk-net]. +Successfully deleted Network lavoldchk-net. +--- Processing Network: lavold-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-6e021fe7f303c1fe for network lavold-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-6e021fe7f303c1fe]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lavold-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/lavold-net]. +Successfully deleted Network lavold-net. +--- Processing Network: lavrohek29-net-0 --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-3af10cb6b3f7804e for network lavrohek29-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-3af10cb6b3f7804e]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lavrohek29-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/lavrohek29-net-0]. +Successfully deleted Network lavrohek29-net-0. +--- Processing Network: lustre-06-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-a798499d8bf40405 for network lustre-06-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-a798499d8bf40405]. +[EXECUTE] Route: Deleting peering-route-ce292d9a6fb17bbc for network lustre-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-ce292d9a6fb17bbc +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-06-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/lustre-06-net]. +Successfully deleted Network lustre-06-net. +--- Processing Network: lustre-test-06-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-799f13a785b25782 for network lustre-test-06-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-799f13a785b25782]. +[EXECUTE] Route: Deleting peering-route-2273ac2f3dccd077 for network lustre-test-06-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-2273ac2f3dccd077 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-test-06-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/lustre-test-06-net]. +Successfully deleted Network lustre-test-06-net. +--- Processing Network: mainek-net-0 --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-4dce75664c4c0cf2 for network mainek-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-4dce75664c4c0cf2]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting mainek-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mainek-net-0]. +Successfully deleted Network mainek-net-0. +--- Processing Network: mainek-net-1 --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-ceb2655d4bb6b336 for network mainek-net-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-ceb2655d4bb6b336]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting mainek-net-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mainek-net-1]. +Successfully deleted Network mainek-net-1. +--- Processing Network: mainek-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-936046db1078bf03 for network mainek-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-936046db1078bf03]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting mainek-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mainek-net]. +Successfully deleted Network mainek-net. +--- Processing Network: mainek-rdma-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting mainek-rdma-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mainek-rdma-net]. +Successfully deleted Network mainek-rdma-net. +--- Processing Network: managed-lustre-03-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-912e02dbc2c43267 for network managed-lustre-03-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-912e02dbc2c43267]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting managed-lustre-03-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/managed-lustre-03-net]. +Successfully deleted Network managed-lustre-03-net. +--- Processing Network: mglsa-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-111ca8b8a3346fb4 for network mglsa-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-111ca8b8a3346fb4]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting mglsa-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/mglsa-net' is already being used by 'projects/hpc-toolkit-dev/regions/us-central1/routers/mglsa-net-router' + +ERROR: Failed to delete Network mglsa-net. Check for remaining dependencies. +--- Processing Network: mglsard-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-4afd256f7ddc3be2 for network mglsard-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-4afd256f7ddc3be2]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting mglsard-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/mglsard-net' is already being used by 'projects/hpc-toolkit-dev/regions/us-west4/routers/mglsard-net-router' + +ERROR: Failed to delete Network mglsard-net. Check for remaining dependencies. +--- Processing Network: ml-gke-e2e-a8fae6-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-2c3a152cc75c7587 for network ml-gke-e2e-a8fae6-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-2c3a152cc75c7587]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting ml-gke-e2e-a8fae6-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/ml-gke-e2e-a8fae6-net]. +Successfully deleted Network ml-gke-e2e-a8fae6-net. +--- Processing Network: ml-gke-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-1dadbcd4c82ffb80 for network ml-gke-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-1dadbcd4c82ffb80]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting ml-gke-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/ml-gke-net]. +Successfully deleted Network ml-gke-net. +--- Network Deletion Process Complete --- +--- Thu Nov 27 10:50:43 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 11:01:01 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T07:01:01+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 50 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +lustreqa05-controller us-central1-a +lustreqa05-slurm-login-001 us-central1-a + gcloud compute instances delete "lustreqa05-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "lustreqa05-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +lustre-qa-05-net-router us-central1 +[DRY RUN] Cloud Router: Would delete lustre-qa-05-net-router in us-central1 + Command: gcloud compute routers delete "lustre-qa-05-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +lustre-qa-05-net-fw-allow-iap-ingress +lustre-qa-05-net-fw-allow-internal-traffic +[DRY RUN] Firewall Rule: Would delete lustre-qa-05-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "lustre-qa-05-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete lustre-qa-05-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "lustre-qa-05-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +lustre-qa-05-primary-subnet in us-central1 +--- Processing Subnet: lustre-qa-05-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-qa-05-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete lustre-qa-05-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-qa-05-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-qa-05-net +mglsa-net +mglsard-net +monitoring-8323fe-net +pkrv6ff952b-net +sa-chs-ops-net-0 +sarthakagrr-net +sispot3u-net-0 +slurm-a3-base-sysnet +sp-helmtest1-net +static-sarthakag-net +test-ssd-psc +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-659a1f3173aa28ed for network lustre-qa-05-net +[DRY RUN] Route: Would delete default-route-r-58923476b12e6284 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +[DRY RUN] Firewall Rule: Would delete lustre-qa-05-net-fw-allow-iap-ingress for network lustre-qa-05-net +[DRY RUN] Firewall Rule: Would delete lustre-qa-05-net-fw-allow-internal-traffic for network lustre-qa-05-net +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mglsa-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mglsa-net + Command: gcloud compute networks delete "mglsa-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mglsard-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mglsard-net + Command: gcloud compute networks delete "mglsard-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: monitoring-8323fe-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-b3d726b99719ebcd for network monitoring-8323fe-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete monitoring-8323fe-net + Command: gcloud compute networks delete "monitoring-8323fe-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: pkrv6ff952b-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-e6359607fabd1429 for network pkrv6ff952b-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete pkrv6ff952b-net + Command: gcloud compute networks delete "pkrv6ff952b-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: sa-chs-ops-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-e4841ee375dabbb7 for network sa-chs-ops-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete sa-chs-ops-net-0 + Command: gcloud compute networks delete "sa-chs-ops-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: sarthakagrr-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-60f6e9638a61ada1 for network sarthakagrr-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete sarthakagrr-net + Command: gcloud compute networks delete "sarthakagrr-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: sispot3u-net-0 --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-5330704133972804 for network sispot3u-net-0 +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete sispot3u-net-0 + Command: gcloud compute networks delete "sispot3u-net-0" --project="hpc-toolkit-dev" --quiet +--- Processing Network: slurm-a3-base-sysnet --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-6345135f78abc28d for network slurm-a3-base-sysnet +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete slurm-a3-base-sysnet + Command: gcloud compute networks delete "slurm-a3-base-sysnet" --project="hpc-toolkit-dev" --quiet +--- Processing Network: sp-helmtest1-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-b4f451a5b7595629 for network sp-helmtest1-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete sp-helmtest1-net + Command: gcloud compute networks delete "sp-helmtest1-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: static-sarthakag-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-31f39ec480a9048d for network static-sarthakag-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete static-sarthakag-net + Command: gcloud compute networks delete "static-sarthakag-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: test-ssd-psc --- +Checking for dependent routes... +[DRY RUN] Route: Would delete default-route-6484ee065c661d7a for network test-ssd-psc +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete test-ssd-psc + Command: gcloud compute networks delete "test-ssd-psc" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Thu Nov 27 11:02:06 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 11:02:47 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T07:02:47+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 50 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +lustreqa05-controller us-central1-a +lustreqa05-slurm-login-001 us-central1-a +[EXECUTE] Instance: Deleting lustreqa05-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustreqa05-controller]. +[EXECUTE] Instance: Deleting lustreqa05-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustreqa05-slurm-login-001]. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +lustre-qa-05-net-router us-central1 +[EXECUTE] Cloud Router: Deleting lustre-qa-05-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/lustre-qa-05-net-router]. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +lustre-qa-05-net-fw-allow-iap-ingress +lustre-qa-05-net-fw-allow-internal-traffic +[EXECUTE] Firewall Rule: Deleting lustre-qa-05-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-qa-05-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting lustre-qa-05-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-qa-05-net-fw-allow-internal-traffic]. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +lustre-qa-05-primary-subnet in us-central1 +--- Processing Subnet: lustre-qa-05-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-qa-05-primary-subnet in us-central1. +[EXECUTE] Subnetwork: Deleting lustre-qa-05-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-qa-05-primary-subnet]. +Successfully deleted Subnetwork lustre-qa-05-primary-subnet in us-central1. +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-qa-05-net +mglsa-net +mglsard-net +monitoring-8323fe-net +pkrv6ff952b-net +sa-chs-ops-net-0 +sarthakagrr-net +sispot3u-net-0 +slurm-a3-base-sysnet +sp-helmtest1-net +static-sarthakag-net +test-ssd-psc +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-659a1f3173aa28ed for network lustre-qa-05-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-659a1f3173aa28ed]. +[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b91a4552351d170 +[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-6f7c1d8537c80540 +[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-qa-05-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' + +ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. +--- Processing Network: mglsa-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting mglsa-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/mglsa-net' is already being used by 'projects/hpc-toolkit-dev/regions/us-central1/routers/mglsa-net-router' + +ERROR: Failed to delete Network mglsa-net. Check for remaining dependencies. +--- Processing Network: mglsard-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting mglsard-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/mglsard-net' is already being used by 'projects/hpc-toolkit-dev/regions/us-west4/routers/mglsard-net-router' + +ERROR: Failed to delete Network mglsard-net. Check for remaining dependencies. +--- Processing Network: monitoring-8323fe-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-b3d726b99719ebcd for network monitoring-8323fe-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-b3d726b99719ebcd]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting monitoring-8323fe-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/monitoring-8323fe-net]. +Successfully deleted Network monitoring-8323fe-net. +--- Processing Network: pkrv6ff952b-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-e6359607fabd1429 for network pkrv6ff952b-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-e6359607fabd1429]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting pkrv6ff952b-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/pkrv6ff952b-net]. +Successfully deleted Network pkrv6ff952b-net. +--- Processing Network: sa-chs-ops-net-0 --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-e4841ee375dabbb7 for network sa-chs-ops-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-e4841ee375dabbb7]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting sa-chs-ops-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/sa-chs-ops-net-0]. +Successfully deleted Network sa-chs-ops-net-0. +--- Processing Network: sarthakagrr-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-60f6e9638a61ada1 for network sarthakagrr-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-60f6e9638a61ada1]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting sarthakagrr-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/sarthakagrr-net]. +Successfully deleted Network sarthakagrr-net. +--- Processing Network: sispot3u-net-0 --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-5330704133972804 for network sispot3u-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-5330704133972804]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting sispot3u-net-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/sispot3u-net-0]. +Successfully deleted Network sispot3u-net-0. +--- Processing Network: slurm-a3-base-sysnet --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-6345135f78abc28d for network slurm-a3-base-sysnet +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-6345135f78abc28d]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting slurm-a3-base-sysnet +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/slurm-a3-base-sysnet]. +Successfully deleted Network slurm-a3-base-sysnet. +--- Processing Network: sp-helmtest1-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-b4f451a5b7595629 for network sp-helmtest1-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-b4f451a5b7595629]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting sp-helmtest1-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/sp-helmtest1-net]. +Successfully deleted Network sp-helmtest1-net. +--- Processing Network: static-sarthakag-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-31f39ec480a9048d for network static-sarthakag-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-31f39ec480a9048d]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting static-sarthakag-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/static-sarthakag-net]. +Successfully deleted Network static-sarthakag-net. +--- Processing Network: test-ssd-psc --- +Checking for dependent routes... +[EXECUTE] Route: Deleting default-route-6484ee065c661d7a for network test-ssd-psc +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-6484ee065c661d7a]. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting test-ssd-psc +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/test-ssd-psc]. +Successfully deleted Network test-ssd-psc. +--- Network Deletion Process Complete --- +--- Thu Nov 27 11:13:01 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 11:13:51 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T07:13:51+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +mglsa-net-router us-central1 +mglsard-net-router us-west4 +[DRY RUN] Cloud Router: Would delete mglsa-net-router in us-central1 + Command: gcloud compute routers delete "mglsa-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Cloud Router: Would delete mglsard-net-router in us-west4 + Command: gcloud compute routers delete "mglsard-net-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +--- Thu Nov 27 11:17:54 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T07:17:54+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +mglsa-net-router us-central1 +mglsard-net-router us-west4 +[EXECUTE] Cloud Router: Deleting mglsa-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/mglsa-net-router]. +[EXECUTE] Cloud Router: Deleting mglsard-net-router in us-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west4/routers/mglsard-net-router]. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +--- Thu Nov 27 11:19:04 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T07:19:04+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +The following Subnetworks (and their dependent addresses) are targeted for deletion: + in +--- Processing Subnet: in --- +ERROR: (gcloud.compute.addresses.list) could not parse resource [] +ERROR: Failed to list dependent addresses for in . +WARNING: Skipping deletion of Subnet in . +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-qa-05-net +mglsa-net +mglsard-net +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mglsa-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mglsa-net + Command: gcloud compute networks delete "mglsa-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: mglsard-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete mglsard-net + Command: gcloud compute networks delete "mglsard-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Thu Nov 27 11:19:31 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 11:19:45 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T07:19:45+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +The following Subnetworks (and their dependent addresses) are targeted for deletion: + in +--- Processing Subnet: in --- +ERROR: (gcloud.compute.addresses.list) could not parse resource [] +ERROR: Failed to list dependent addresses for in . +WARNING: Skipping deletion of Subnet in . +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-qa-05-net +mglsa-net +mglsard-net +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-3b91a4552351d170 +[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-6f7c1d8537c80540 +[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +ERROR: (gcloud.compute.routes.delete) Could not fetch resource: + - The auto-generated peering route cannot be deleted. + +ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting lustre-qa-05-net +ERROR: (gcloud.compute.networks.delete) Could not fetch resource: + - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' + +ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. +--- Processing Network: mglsa-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting mglsa-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mglsa-net]. +Successfully deleted Network mglsa-net. +--- Processing Network: mglsard-net --- +Checking for dependent routes... +No dependent routes found. +Checking for dependent firewall rules... +No dependent firewall rules found. +[EXECUTE] Network: Deleting mglsard-net +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mglsard-net]. +Successfully deleted Network mglsard-net. +--- Network Deletion Process Complete --- +--- Thu Nov 27 11:21:42 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 11:35:00 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T07:35:00+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +The following Subnetworks (and their dependent addresses) are targeted for deletion: + in +--- Processing Subnet: in --- +ERROR: (gcloud.compute.addresses.list) could not parse resource [] +ERROR: Failed to list dependent addresses for in . +WARNING: Skipping deletion of Subnet in . +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-qa-05-net +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Thu Nov 27 11:35:24 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 11:54:19 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T07:54:19+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-qa-05-net +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Thu Nov 27 11:54:37 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 11:55:13 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T07:55:13+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-qa-05-net +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Thu Nov 27 11:55:29 AM UTC 2025 --- Cleanup Script Run Finished --- + diff --git a/networks.txt b/networks.txt new file mode 100644 index 0000000000..d93f7d4dc6 --- /dev/null +++ b/networks.txt @@ -0,0 +1,2086 @@ +--- Wed Nov 26 02:16:54 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 37 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +The following Firewall Rules are targeted for deletion in this run: +a3mega-sys-net-shu7-fw-allow-iap-ingress +a3mega-sys-net-shu7-fw-allow-internal-traffic +a3u-slurm-net-fw-allow-iap-ingress +a3u-slurm-net-fw-allow-internal-traffic +a4hsarthakag-net-fw-allow-iap-ingress +a4hsarthakag-net-fw-allow-internal-traffic +a4htest-internal-0 +a4htest-net-0-fw-allow-iap-ingress +a4newimgek-internal-0 +a4newimgek-internal-1 +a4newimgek-net-0-fw-allow-iap-ingress +a4newimgek-net-1-fw-allow-iap-ingress +a4newimgek-net-fw-allow-iap-ingress +a4newimgek-net-fw-allow-internal-traffic +a4oldimgek-internal-0 +a4oldimgek-internal-1 +a4oldimgek-net-0-fw-allow-iap-ingress +a4oldimgek-net-1-fw-allow-iap-ingress +a4oldimgek-net-fw-allow-iap-ingress +a4oldimgek-net-fw-allow-internal-traffic +[DRY RUN] Firewall Rule: Would delete a3mega-sys-net-shu7-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a3mega-sys-net-shu7-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a3mega-sys-net-shu7-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "a3mega-sys-net-shu7-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a3u-slurm-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a3u-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a3u-slurm-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "a3u-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4hsarthakag-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4hsarthakag-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4hsarthakag-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "a4hsarthakag-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4htest-internal-0 + Command: gcloud compute firewall-rules delete "a4htest-internal-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4htest-net-0-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4htest-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4newimgek-internal-0 + Command: gcloud compute firewall-rules delete "a4newimgek-internal-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4newimgek-internal-1 + Command: gcloud compute firewall-rules delete "a4newimgek-internal-1" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4newimgek-net-0-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4newimgek-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4newimgek-net-1-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4newimgek-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4newimgek-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4newimgek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4newimgek-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "a4newimgek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4oldimgek-internal-0 + Command: gcloud compute firewall-rules delete "a4oldimgek-internal-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4oldimgek-internal-1 + Command: gcloud compute firewall-rules delete "a4oldimgek-internal-1" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4oldimgek-net-0-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4oldimgek-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4oldimgek-net-1-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4oldimgek-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4oldimgek-net-fw-allow-iap-ingress + Command: gcloud compute firewall-rules delete "a4oldimgek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Firewall Rule: Would delete a4oldimgek-net-fw-allow-internal-traffic + Command: gcloud compute firewall-rules delete "a4oldimgek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 02:17:04 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:17:44 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 37 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +The following Firewall Rules are targeted for deletion in this run: +a3mega-sys-net-shu7-fw-allow-iap-ingress +a3mega-sys-net-shu7-fw-allow-internal-traffic +a3u-slurm-net-fw-allow-iap-ingress +a3u-slurm-net-fw-allow-internal-traffic +a4hsarthakag-net-fw-allow-iap-ingress +a4hsarthakag-net-fw-allow-internal-traffic +a4htest-internal-0 +a4htest-net-0-fw-allow-iap-ingress +a4newimgek-internal-0 +a4newimgek-internal-1 +a4newimgek-net-0-fw-allow-iap-ingress +a4newimgek-net-1-fw-allow-iap-ingress +a4newimgek-net-fw-allow-iap-ingress +a4newimgek-net-fw-allow-internal-traffic +a4oldimgek-internal-0 +a4oldimgek-internal-1 +a4oldimgek-net-0-fw-allow-iap-ingress +a4oldimgek-net-1-fw-allow-iap-ingress +a4oldimgek-net-fw-allow-iap-ingress +a4oldimgek-net-fw-allow-internal-traffic +[EXECUTE] Firewall Rule: Deleting a3mega-sys-net-shu7-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a3mega-sys-net-shu7-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a3mega-sys-net-shu7-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a3mega-sys-net-shu7-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting a3u-slurm-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a3u-slurm-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a3u-slurm-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a3u-slurm-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting a4hsarthakag-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4hsarthakag-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4hsarthakag-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4hsarthakag-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting a4htest-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4htest-internal-0]. +[EXECUTE] Firewall Rule: Deleting a4htest-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4htest-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4newimgek-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-internal-0]. +[EXECUTE] Firewall Rule: Deleting a4newimgek-internal-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-internal-1]. +[EXECUTE] Firewall Rule: Deleting a4newimgek-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4newimgek-net-1-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-net-1-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4newimgek-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4newimgek-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting a4oldimgek-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-internal-0]. +[EXECUTE] Firewall Rule: Deleting a4oldimgek-internal-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-internal-1]. +[EXECUTE] Firewall Rule: Deleting a4oldimgek-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4oldimgek-net-1-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-net-1-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4oldimgek-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4oldimgek-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-net-fw-allow-internal-traffic]. +--- Wed Nov 26 02:20:54 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:21:04 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 37 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +a4oldimg-net-fw-allow-iap-ingress +a4oldimg-net-fw-allow-internal-traffic +a4xlavhpcnew-a4x-internal-0 +a4xlavhpcnew-a4x-internal-1 +a4xlavhpcnew-a4x-net-0-fw-allow-iap-ingress +a4xlavhpcnew-a4x-net-0-fw-allow-internal-traffic +a4xlavhpcnew-a4x-net-0-fw-allow-ssh-ingress +a4xlavhpcnew-a4x-net-1-fw-allow-iap-ingress +a4xlavhpcnew-a4x-net-1-fw-allow-internal-traffic +a4xslurm-net-fw-allow-iap-ingress +a4xslurm-net-fw-allow-internal-traffic +cx-a3u-internal-0 +cx-a3u-net-0-fw-allow-iap-ingress +db451c7-ml-slurm-v6-net-fw-allow-iap-ingress +db451c7-ml-slurm-v6-net-fw-allow-internal-traffic +dynpoc-net-fw-allow-iap-ingress +dynpoc-net-fw-allow-internal-traffic +g4qclav-net-fw-allow-iap-ingress +g4qclav-net-fw-allow-internal-traffic +gke-1395b4-net-fw-allow-iap-ingress + gcloud compute firewall-rules delete "a4oldimg-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "a4oldimg-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-internal-0" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-internal-1" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-net-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-net-0-fw-allow-ssh-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-net-1-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "a4xslurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "a4xslurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "cx-a3u-internal-0" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "cx-a3u-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "db451c7-ml-slurm-v6-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "db451c7-ml-slurm-v6-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "dynpoc-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "dynpoc-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "g4qclav-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "g4qclav-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-1395b4-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 02:21:13 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:21:47 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 37 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +a4oldimg-net-fw-allow-iap-ingress +a4oldimg-net-fw-allow-internal-traffic +a4xlavhpcnew-a4x-internal-0 +a4xlavhpcnew-a4x-internal-1 +a4xlavhpcnew-a4x-net-0-fw-allow-iap-ingress +a4xlavhpcnew-a4x-net-0-fw-allow-internal-traffic +a4xlavhpcnew-a4x-net-0-fw-allow-ssh-ingress +a4xlavhpcnew-a4x-net-1-fw-allow-iap-ingress +a4xlavhpcnew-a4x-net-1-fw-allow-internal-traffic +a4xslurm-net-fw-allow-iap-ingress +a4xslurm-net-fw-allow-internal-traffic +cx-a3u-internal-0 +cx-a3u-net-0-fw-allow-iap-ingress +db451c7-ml-slurm-v6-net-fw-allow-iap-ingress +db451c7-ml-slurm-v6-net-fw-allow-internal-traffic +dynpoc-net-fw-allow-iap-ingress +dynpoc-net-fw-allow-internal-traffic +g4qclav-net-fw-allow-iap-ingress +g4qclav-net-fw-allow-internal-traffic +gke-1395b4-net-fw-allow-iap-ingress +[EXECUTE] Firewall Rule: Deleting a4oldimg-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimg-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4oldimg-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimg-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-internal-0]. +[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-internal-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-internal-1]. +[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-net-0-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-net-0-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-net-0-fw-allow-ssh-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-net-0-fw-allow-ssh-ingress]. +[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-net-1-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-net-1-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-net-1-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-net-1-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting a4xslurm-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xslurm-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting a4xslurm-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xslurm-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting cx-a3u-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/cx-a3u-internal-0]. +[EXECUTE] Firewall Rule: Deleting cx-a3u-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/cx-a3u-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting db451c7-ml-slurm-v6-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/db451c7-ml-slurm-v6-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting db451c7-ml-slurm-v6-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/db451c7-ml-slurm-v6-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting dynpoc-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/dynpoc-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting dynpoc-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/dynpoc-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting g4qclav-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/g4qclav-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting g4qclav-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/g4qclav-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting gke-1395b4-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/gke-1395b4-net-fw-allow-iap-ingress]. +--- Wed Nov 26 02:24:23 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:24:39 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 37 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +gke-1395b4-net-fw-allow-internal-traffic +gke-managed-lustre-basic-net-fw-allow-iap-ingress +gke-managed-lustre-basic-net-fw-allow-internal-traffic +h4dqc-net-fw-allow-iap-ingress +h4dqc-net-fw-allow-internal-traffic +h4dqc-rdma-0 +h4dqc-rdma-net-0-fw-allow-iap-ingress +h4dqc-rdma-net-0-fw-allow-internal-traffic +h4d-res-swarnabm4-3-internal +h4d-res-swarnabm4-3-net-fw-allow-iap-ingress +h4d-res-swarnabm4-3-net-fw-allow-internal-traffic +h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress +h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic +hpc-01-net-fw-allow-iap-ingress +hpc-01-net-fw-allow-internal-traffic +hpcdydis-net-fw-allow-iap-ingress +hpcdydis-net-fw-allow-internal-traffic +hpcdy-net-fw-allow-iap-ingress +hpcdy-net-fw-allow-internal-traffic +hpc-exr-2-internal-0 + gcloud compute firewall-rules delete "gke-1395b4-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-managed-lustre-basic-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-managed-lustre-basic-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-rdma-0" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-internal" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-01-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-01-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdy-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdy-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-exr-2-internal-0" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 02:24:48 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:25:53 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 37 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +gke-1395b4-net-fw-allow-internal-traffic +gke-managed-lustre-basic-net-fw-allow-iap-ingress +gke-managed-lustre-basic-net-fw-allow-internal-traffic +h4dqc-net-fw-allow-iap-ingress +h4dqc-net-fw-allow-internal-traffic +h4dqc-rdma-0 +h4dqc-rdma-net-0-fw-allow-iap-ingress +h4dqc-rdma-net-0-fw-allow-internal-traffic +h4d-res-swarnabm4-3-internal +h4d-res-swarnabm4-3-net-fw-allow-iap-ingress +h4d-res-swarnabm4-3-net-fw-allow-internal-traffic +h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress +h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic +hpc-01-net-fw-allow-iap-ingress +hpc-01-net-fw-allow-internal-traffic +hpcdydis-net-fw-allow-iap-ingress +hpcdydis-net-fw-allow-internal-traffic +hpcdy-net-fw-allow-iap-ingress +hpcdy-net-fw-allow-internal-traffic +hpc-exr-2-internal-0 + gcloud compute firewall-rules delete "gke-1395b4-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-managed-lustre-basic-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-managed-lustre-basic-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-rdma-0" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-internal" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-01-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-01-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdy-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdy-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-exr-2-internal-0" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 02:26:03 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:26:46 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 39 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +gke-1395b4-net-fw-allow-internal-traffic +h4dqc-net-fw-allow-iap-ingress +h4dqc-net-fw-allow-internal-traffic +h4dqc-rdma-0 +h4dqc-rdma-net-0-fw-allow-iap-ingress +h4dqc-rdma-net-0-fw-allow-internal-traffic +h4d-res-swarnabm4-3-internal +h4d-res-swarnabm4-3-net-fw-allow-iap-ingress +h4d-res-swarnabm4-3-net-fw-allow-internal-traffic +h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress +h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic +hpc-01-net-fw-allow-iap-ingress +hpc-01-net-fw-allow-internal-traffic +hpcdydis-net-fw-allow-iap-ingress +hpcdydis-net-fw-allow-internal-traffic +hpcdy-net-fw-allow-iap-ingress +hpcdy-net-fw-allow-internal-traffic +hpc-exr-2-internal-0 +hpc-exr-2-net-0-fw-allow-iap-ingress +hpcimg-net-fw-allow-iap-ingress + gcloud compute firewall-rules delete "gke-1395b4-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-rdma-0" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-internal" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-01-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-01-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdy-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcdy-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-exr-2-internal-0" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-exr-2-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpcimg-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 02:26:56 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:27:17 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 39 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +gke-1395b4-net-fw-allow-internal-traffic +h4dqc-net-fw-allow-iap-ingress +h4dqc-net-fw-allow-internal-traffic +h4dqc-rdma-0 +h4dqc-rdma-net-0-fw-allow-iap-ingress +h4dqc-rdma-net-0-fw-allow-internal-traffic +h4d-res-swarnabm4-3-internal +h4d-res-swarnabm4-3-net-fw-allow-iap-ingress +h4d-res-swarnabm4-3-net-fw-allow-internal-traffic +h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress +h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic +hpc-01-net-fw-allow-iap-ingress +hpc-01-net-fw-allow-internal-traffic +hpcdydis-net-fw-allow-iap-ingress +hpcdydis-net-fw-allow-internal-traffic +hpcdy-net-fw-allow-iap-ingress +hpcdy-net-fw-allow-internal-traffic +hpc-exr-2-internal-0 +hpc-exr-2-net-0-fw-allow-iap-ingress +hpcimg-net-fw-allow-iap-ingress +[EXECUTE] Firewall Rule: Deleting gke-1395b4-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/gke-1395b4-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting h4dqc-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4dqc-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting h4dqc-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4dqc-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting h4dqc-rdma-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4dqc-rdma-0]. +[EXECUTE] Firewall Rule: Deleting h4dqc-rdma-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4dqc-rdma-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting h4dqc-rdma-net-0-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4dqc-rdma-net-0-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting h4d-res-swarnabm4-3-internal +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4d-res-swarnabm4-3-internal]. +[EXECUTE] Firewall Rule: Deleting h4d-res-swarnabm4-3-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4d-res-swarnabm4-3-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting h4d-res-swarnabm4-3-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4d-res-swarnabm4-3-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting hpc-01-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-01-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting hpc-01-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-01-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting hpcdydis-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcdydis-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting hpcdydis-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcdydis-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting hpcdy-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcdy-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting hpcdy-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcdy-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting hpc-exr-2-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-exr-2-internal-0]. +[EXECUTE] Firewall Rule: Deleting hpc-exr-2-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-exr-2-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting hpcimg-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcimg-net-fw-allow-iap-ingress]. +--- Wed Nov 26 02:29:58 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:11:18 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 39 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +a8a55slurm-nodeset-0 us-central1-a + gcloud compute instances delete "a8a55slurm-nodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +hpcimg-net-fw-allow-internal-traffic +hpc-lustre-test-02-net-fw-allow-iap-ingress +hpc-lustre-test-02-net-fw-allow-internal-traffic +khu-h4d-cluster-test-net-fw-allow-iap-ingress +khu-h4d-cluster-test-net-fw-allow-internal-traffic +khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress +laveeek29-internal-0 +laveeek29-internal-1 +laveeek29-net-0-fw-allow-iap-ingress +laveeek29-net-1-fw-allow-iap-ingress +laveeek29-net-fw-allow-iap-ingress +laveeek29-net-fw-allow-internal-traffic +lavoldchk-net-fw-allow-iap-ingress +lavoldchk-net-fw-allow-internal-traffic +lavrohek29-internal-0 +lavrohek29-net-0-fw-allow-iap-ingress +lustre-06-net-fw-allow-iap-ingress +lustre-06-net-fw-allow-internal-traffic +lustre-test-06-net-fw-allow-iap-ingress +lustre-test-06-net-fw-allow-internal-traffic + gcloud compute firewall-rules delete "hpcimg-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-lustre-test-02-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-lustre-test-02-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "khu-h4d-cluster-test-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "khu-h4d-cluster-test-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-internal-0" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-internal-1" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lavoldchk-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lavoldchk-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lavrohek29-internal-0" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lavrohek29-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lustre-06-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lustre-06-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lustre-test-06-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lustre-test-06-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:11:28 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:12:06 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 43 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +a8a55slurm-nodeset-0 us-central1-a + gcloud compute instances delete "a8a55slurm-nodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +hpcimg-net-fw-allow-internal-traffic +hpc-lustre-test-02-net-fw-allow-iap-ingress +hpc-lustre-test-02-net-fw-allow-internal-traffic +khu-h4d-cluster-test-net-fw-allow-iap-ingress +khu-h4d-cluster-test-net-fw-allow-internal-traffic +khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress +laveeek29-internal-0 +laveeek29-internal-1 +laveeek29-net-0-fw-allow-iap-ingress +laveeek29-net-1-fw-allow-iap-ingress +laveeek29-net-fw-allow-iap-ingress +laveeek29-net-fw-allow-internal-traffic +lavoldchk-net-fw-allow-iap-ingress +lavoldchk-net-fw-allow-internal-traffic +lavrohek29-internal-0 +lavrohek29-net-0-fw-allow-iap-ingress +mainek-internal-0 +mainek-internal-1 +mainek-net-0-fw-allow-iap-ingress +mainek-net-1-fw-allow-iap-ingress + gcloud compute firewall-rules delete "hpcimg-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-lustre-test-02-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "hpc-lustre-test-02-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "khu-h4d-cluster-test-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "khu-h4d-cluster-test-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-internal-0" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-internal-1" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "laveeek29-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lavoldchk-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lavoldchk-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lavrohek29-internal-0" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "lavrohek29-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "mainek-internal-0" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "mainek-internal-1" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "mainek-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "mainek-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:12:15 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:12:45 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 43 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +hpcimg-net-fw-allow-internal-traffic +hpc-lustre-test-02-net-fw-allow-iap-ingress +hpc-lustre-test-02-net-fw-allow-internal-traffic +khu-h4d-cluster-test-net-fw-allow-iap-ingress +khu-h4d-cluster-test-net-fw-allow-internal-traffic +khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress +laveeek29-internal-0 +laveeek29-internal-1 +laveeek29-net-0-fw-allow-iap-ingress +laveeek29-net-1-fw-allow-iap-ingress +laveeek29-net-fw-allow-iap-ingress +laveeek29-net-fw-allow-internal-traffic +lavoldchk-net-fw-allow-iap-ingress +lavoldchk-net-fw-allow-internal-traffic +lavrohek29-internal-0 +lavrohek29-net-0-fw-allow-iap-ingress +mainek-internal-0 +mainek-internal-1 +mainek-net-0-fw-allow-iap-ingress +mainek-net-1-fw-allow-iap-ingress +[EXECUTE] Firewall Rule: Deleting hpcimg-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcimg-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting hpc-lustre-test-02-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-lustre-test-02-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting hpc-lustre-test-02-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-lustre-test-02-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting khu-h4d-cluster-test-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/khu-h4d-cluster-test-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting khu-h4d-cluster-test-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/khu-h4d-cluster-test-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting laveeek29-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-internal-0]. +[EXECUTE] Firewall Rule: Deleting laveeek29-internal-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-internal-1]. +[EXECUTE] Firewall Rule: Deleting laveeek29-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting laveeek29-net-1-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-net-1-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting laveeek29-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting laveeek29-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting lavoldchk-net-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lavoldchk-net-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting lavoldchk-net-fw-allow-internal-traffic +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lavoldchk-net-fw-allow-internal-traffic]. +[EXECUTE] Firewall Rule: Deleting lavrohek29-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lavrohek29-internal-0]. +[EXECUTE] Firewall Rule: Deleting lavrohek29-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lavrohek29-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting mainek-internal-0 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-internal-0]. +[EXECUTE] Firewall Rule: Deleting mainek-internal-1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-internal-1]. +[EXECUTE] Firewall Rule: Deleting mainek-net-0-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-net-0-fw-allow-iap-ingress]. +[EXECUTE] Firewall Rule: Deleting mainek-net-1-fw-allow-iap-ingress +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-net-1-fw-allow-iap-ingress]. +--- Wed Nov 26 03:15:43 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:19:25 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 43 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +gke-a3mega-7b5154-remote-node-0 us-west4-a + gcloud compute instances delete "gke-a3mega-7b5154-remote-node-0" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +gke-a3mega-7b5154-gpunet-0-router us-west4 +gke-a3mega-7b5154-gpunet-1-router us-west4 +gke-a3mega-7b5154-gpunet-2-router us-west4 +gke-a3mega-7b5154-gpunet-3-router us-west4 +gke-a3mega-7b5154-gpunet-4-router us-west4 +gke-a3mega-7b5154-gpunet-5-router us-west4 +gke-a3mega-7b5154-gpunet-6-router us-west4 +gke-a3mega-7b5154-gpunet-7-router us-west4 +gke-a3mega-7b5154-net-router us-west4 + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-0-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-1-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-2-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-3-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-4-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-5-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-6-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-7-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-net-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +The following Firewall Rules are targeted for deletion in this run: +gke-a3mega-7b5154-gpunet-0-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-0-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-1-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-1-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-2-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-2-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-3-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-3-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-4-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-4-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-5-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-5-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-6-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-6-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-7-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-7-fw-allow-internal-traffic +gke-a3mega-7b5154-net-fw-allow-iap-ingress +gke-a3mega-7b5154-net-fw-allow-internal-traffic +mainek-net-fw-allow-iap-ingress +mainek-net-fw-allow-internal-traffic + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-1-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-2-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-2-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-3-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-3-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-4-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-4-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-5-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-5-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-6-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-6-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-7-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-7-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "mainek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "mainek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:19:35 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:23:12 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 43 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic +--- Deletion Phase 1: GKE Clusters (Top 20) --- +The following GKE clusters are targeted for deletion in this run: +gke-a3mega-7b5154 us-west4 + gcloud container clusters delete "gke-a3mega-7b5154" --project="hpc-toolkit-dev" --location="us-west4" +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +d87d8bslur-controller us-central1-a +d87d8bslur-slurm-login-001 us-central1-a +gke-a3mega-7b5154-remote-node-0 us-west4-a +gke-gke-a3mega-7b5154-default-pool-92973a4c-zdc6 us-west4-a +gke-gke-a3mega-7b5154-default-pool-b8fa115c-f11p us-west4-b +gke-gke-a3mega-7b5154-default-pool-cab11a55-856w us-west4-c + gcloud compute instances delete "d87d8bslur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "d87d8bslur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet + gcloud compute instances delete "gke-a3mega-7b5154-remote-node-0" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet + gcloud compute instances delete "gke-gke-a3mega-7b5154-default-pool-92973a4c-zdc6" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet + gcloud compute instances delete "gke-gke-a3mega-7b5154-default-pool-b8fa115c-f11p" --project="hpc-toolkit-dev" --zone="us-west4-b" --quiet + gcloud compute instances delete "gke-gke-a3mega-7b5154-default-pool-cab11a55-856w" --project="hpc-toolkit-dev" --zone="us-west4-c" --quiet +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +gke-a3mega-7b5154-gpunet-0-router us-west4 +gke-a3mega-7b5154-gpunet-1-router us-west4 +gke-a3mega-7b5154-gpunet-2-router us-west4 +gke-a3mega-7b5154-gpunet-3-router us-west4 +gke-a3mega-7b5154-gpunet-4-router us-west4 +gke-a3mega-7b5154-gpunet-5-router us-west4 +gke-a3mega-7b5154-gpunet-6-router us-west4 +gke-a3mega-7b5154-gpunet-7-router us-west4 +gke-a3mega-7b5154-net-router us-west4 + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-0-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-1-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-2-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-3-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-4-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-5-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-6-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-gpunet-7-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "gke-a3mega-7b5154-net-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet +--- Deletion Phase 3: Firewall Rules (Top 20) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +The following Firewall Rules are targeted for deletion in this run: +gke-a3mega-7b5154-gpunet-0-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-0-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-1-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-1-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-2-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-2-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-3-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-3-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-4-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-4-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-5-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-5-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-6-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-6-fw-allow-internal-traffic +gke-a3mega-7b5154-gpunet-7-fw-allow-iap-ingress +gke-a3mega-7b5154-gpunet-7-fw-allow-internal-traffic +gke-a3mega-7b5154-net-fw-allow-iap-ingress +gke-a3mega-7b5154-net-fw-allow-internal-traffic +gke-gke-a3mega-7b5154-2358afec-all +gke-gke-a3mega-7b5154-2358afec-exkubelet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-1-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-2-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-2-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-3-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-3-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-4-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-4-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-5-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-5-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-6-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-6-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-7-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-7-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-a3mega-7b5154-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-gke-a3mega-7b5154-2358afec-all" --project="hpc-toolkit-dev" --quiet + gcloud compute firewall-rules delete "gke-gke-a3mega-7b5154-2358afec-exkubelet" --project="hpc-toolkit-dev" --quiet +--- Wed Nov 26 03:23:21 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 03:23:41 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 43 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic +--- Deletion Phase 1: GKE Clusters (Top 20) --- +The following GKE clusters are targeted for deletion in this run: +gke-a3mega-7b5154 us-west4 +[EXECUTE] GKE Cluster: Deleting gke-a3mega-7b5154 in us-west4 +The following clusters will be deleted. + - [gke-a3mega-7b5154] in [us-west4] + +Do you want to continue (Y/n)? +ERROR: (gcloud.container.clusters.delete) This prompt could not be answered because you are not in an interactive session. You can re-run the command with the --quiet flag to accept default answers for all prompts. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +The following Instances are targeted for deletion in this run: +d87d8bslur-controller us-central1-a +d87d8bslur-slurm-login-001 us-central1-a +gke-a3mega-7b5154-remote-node-0 us-west4-a +gke-gke-a3mega-7b5154-default-pool-92973a4c-zdc6 us-west4-a +gke-gke-a3mega-7b5154-default-pool-b8fa115c-f11p us-west4-b +gke-gke-a3mega-7b5154-default-pool-cab11a55-856w us-west4-c +[EXECUTE] Instance: Deleting d87d8bslur-controller in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d87d8bslur-controller]. +[EXECUTE] Instance: Deleting d87d8bslur-slurm-login-001 in us-central1-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d87d8bslur-slurm-login-001]. +[EXECUTE] Instance: Deleting gke-a3mega-7b5154-remote-node-0 in us-west4-a +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/instances/gke-a3mega-7b5154-remote-node-0]. +[EXECUTE] Instance: Deleting gke-gke-a3mega-7b5154-default-pool-92973a4c-zdc6 in us-west4-a +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-west4-a/instances/gke-gke-a3mega-7b5154-default-pool-92973a4c-zdc6' was not found + +[EXECUTE] Instance: Deleting gke-gke-a3mega-7b5154-default-pool-b8fa115c-f11p in us-west4-b +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-west4-b/instances/gke-gke-a3mega-7b5154-default-pool-b8fa115c-f11p' was not found + +[EXECUTE] Instance: Deleting gke-gke-a3mega-7b5154-default-pool-cab11a55-856w in us-west4-c +ERROR: (gcloud.compute.instances.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/zones/us-west4-c/instances/gke-gke-a3mega-7b5154-default-pool-cab11a55-856w' was not found + +./cleanup.sh: line 134: syntax error near unexpected token `(' +./cleanup.sh: line 134: `echo "--- Deletion Phase 1: Filestore Instances (Top $DELETE_LIMIT) ---"' +--- Thu Nov 27 09:50:25 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T05:50:25+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +lustre-06-primary-subnet in us-central1 +lustre-test-06-primary-subnet in us-central1 +--- Processing Subnet: lustre-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-06-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-test-06-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +The following Networks are targeted for deletion in this run: +a3mega-sys-net-shu7 +a3u-onspot-slurm-829610-net-0 +a3u-slurm-net +a4hsarthakag-net +a4htest-net-0 +a4newimgek-net +a4newimgek-net-0 +a4newimgek-net-1 +a4newimgek-rdma-net +a4oldimgek-net +a4oldimgek-net-0 +a4oldimgek-net-1 +a4oldimgek-rdma-net +a4oldimg-net +a4xlavhpcnew-a4x-net-0 +a4xlavhpcnew-a4x-net-1 +a4xlavhpcnew-a4x-rdma-net +a4xslurm-net +cx-a3u-net-0 +db451c7-ml-slurm-v6-net +dynpoc-net +g4-dwsq-1-net-1 +g4qclav-net +gke-1395b4-net +gke-managed-lustre-basic-net +h4d-cluster-rdma-net-0 +h4dqc-net +h4dqc-rdma-net-0 +h4d-res-swarnabm4-3-net +h4d-res-swarnabm4-3-rdma-net +[DRY RUN] Network: Would delete a3mega-sys-net-shu7 + Command: gcloud compute networks delete "a3mega-sys-net-shu7" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a3u-onspot-slurm-829610-net-0 + Command: gcloud compute networks delete "a3u-onspot-slurm-829610-net-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a3u-slurm-net + Command: gcloud compute networks delete "a3u-slurm-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4hsarthakag-net + Command: gcloud compute networks delete "a4hsarthakag-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4htest-net-0 + Command: gcloud compute networks delete "a4htest-net-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4newimgek-net + Command: gcloud compute networks delete "a4newimgek-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4newimgek-net-0 + Command: gcloud compute networks delete "a4newimgek-net-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4newimgek-net-1 + Command: gcloud compute networks delete "a4newimgek-net-1" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4newimgek-rdma-net + Command: gcloud compute networks delete "a4newimgek-rdma-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4oldimgek-net + Command: gcloud compute networks delete "a4oldimgek-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4oldimgek-net-0 + Command: gcloud compute networks delete "a4oldimgek-net-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4oldimgek-net-1 + Command: gcloud compute networks delete "a4oldimgek-net-1" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4oldimgek-rdma-net + Command: gcloud compute networks delete "a4oldimgek-rdma-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4oldimg-net + Command: gcloud compute networks delete "a4oldimg-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4xlavhpcnew-a4x-net-0 + Command: gcloud compute networks delete "a4xlavhpcnew-a4x-net-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4xlavhpcnew-a4x-net-1 + Command: gcloud compute networks delete "a4xlavhpcnew-a4x-net-1" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4xlavhpcnew-a4x-rdma-net + Command: gcloud compute networks delete "a4xlavhpcnew-a4x-rdma-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete a4xslurm-net + Command: gcloud compute networks delete "a4xslurm-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete cx-a3u-net-0 + Command: gcloud compute networks delete "cx-a3u-net-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete db451c7-ml-slurm-v6-net + Command: gcloud compute networks delete "db451c7-ml-slurm-v6-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete dynpoc-net + Command: gcloud compute networks delete "dynpoc-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete g4-dwsq-1-net-1 + Command: gcloud compute networks delete "g4-dwsq-1-net-1" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete g4qclav-net + Command: gcloud compute networks delete "g4qclav-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete gke-1395b4-net + Command: gcloud compute networks delete "gke-1395b4-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete gke-managed-lustre-basic-net + Command: gcloud compute networks delete "gke-managed-lustre-basic-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete h4d-cluster-rdma-net-0 + Command: gcloud compute networks delete "h4d-cluster-rdma-net-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete h4dqc-net + Command: gcloud compute networks delete "h4dqc-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete h4dqc-rdma-net-0 + Command: gcloud compute networks delete "h4dqc-rdma-net-0" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete h4d-res-swarnabm4-3-net + Command: gcloud compute networks delete "h4d-res-swarnabm4-3-net" --project="hpc-toolkit-dev" --quiet +[DRY RUN] Network: Would delete h4d-res-swarnabm4-3-rdma-net + Command: gcloud compute networks delete "h4d-res-swarnabm4-3-rdma-net" --project="hpc-toolkit-dev" --quiet +--- Thu Nov 27 09:50:44 AM UTC 2025 --- Cleanup Script Run Finished --- + diff --git a/peer.txt b/peer.txt new file mode 100644 index 0000000000..6ae514f4e8 --- /dev/null +++ b/peer.txt @@ -0,0 +1,566 @@ +[2025-11-30 17:48:42] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 17:48:42] [INFO] Time Cutoff (General): 2025-11-30T17:48:42+0000 +[2025-11-30 17:48:42] [INFO] Time Cutoff (Images): 2025-10-01T17:48:42+0000 +[2025-11-30 17:48:42] [INFO] Delete Limit per Type: 200 +[2025-11-30 17:48:42] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 17:48:43] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 17:48:45] [INFO] No Service Accounts found matching prefix. +[2025-11-30 17:48:45] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 17:48:47] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 17:48:47] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 17:48:49] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 17:48:49] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 17:48:49] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 17:48:49] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 17:48:52] [INFO] No Filestore instances found matching criteria. +[2025-11-30 17:48:52] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 17:48:55] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 17:48:55] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 17:48:55] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 17:48:55] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 17:48:55] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 17:48:56] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 17:48:56] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 17:48:56] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 17:48:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 17:48:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 17:48:56] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 17:48:56] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T17:48:56Z (Unix: 1763315336) +[2025-11-30 17:48:56] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763315336]) +[2025-11-30 17:48:59] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 17:48:59] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 17:49:01] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 17:49:01] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 17:49:01] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 17:49:01] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 17:49:01] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 17:49:01] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 17:49:03] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 17:49:03] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 17:49:05] [INFO] No Regional Address found matching criteria. +[2025-11-30 17:49:05] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 17:49:08] [INFO] No Global Address found matching criteria. +[2025-11-30 17:49:08] [INFO] --- Processing: VPC Peerings (Limit: 200) --- +[2025-11-30 17:49:12] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:12] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 17:49:15] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 17:49:15] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 17:49:15] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 17:49:15] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 17:49:15] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:18] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 17:49:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:49:20] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 17:49:22] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 17:49:22] [INFO] CLEANUP RUN FINISHED +./cleanup.sh: line 636: syntax error near unexpected token `in' +./cleanup.sh: line 636: `in' +[2025-11-30 17:58:51] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 17:58:51] [INFO] Time Cutoff (General): 2025-11-30T17:58:51+0000 +[2025-11-30 17:58:51] [INFO] Time Cutoff (Images): 2025-10-01T17:58:51+0000 +[2025-11-30 17:58:51] [INFO] Delete Limit per Type: 200 +[2025-11-30 17:58:51] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 17:58:52] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 17:58:54] [INFO] No Service Accounts found matching prefix. +[2025-11-30 17:58:54] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 17:58:56] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 17:58:56] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 17:58:58] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 17:58:58] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 17:58:58] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 17:58:58] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 17:59:01] [INFO] No Filestore instances found matching criteria. +[2025-11-30 17:59:01] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 17:59:04] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 17:59:04] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 17:59:04] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 17:59:04] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 17:59:04] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 17:59:04] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 17:59:04] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 17:59:04] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 17:59:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 17:59:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 17:59:05] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 17:59:05] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T17:59:05Z (Unix: 1763315945) +[2025-11-30 17:59:05] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763315945]) +[2025-11-30 17:59:07] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 17:59:07] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 17:59:10] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 17:59:10] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 17:59:10] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 17:59:10] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 17:59:10] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 17:59:10] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 17:59:12] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 17:59:12] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 17:59:15] [INFO] No Regional Address found matching criteria. +[2025-11-30 17:59:15] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 17:59:17] [INFO] No Global Address found matching criteria. +[2025-11-30 17:59:17] [INFO] --- Processing: VPC Peerings (Limit: 200) --- +[2025-11-30 17:59:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:24] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 17:59:26] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 17:59:26] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 17:59:26] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 17:59:26] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 17:59:26] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:29] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 17:59:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 17:59:31] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 17:59:33] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 17:59:33] [INFO] CLEANUP RUN FINISHED +[2025-11-30 18:05:23] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:05:23] [INFO] Time Cutoff (General): 2025-11-30T18:05:23+0000 +[2025-11-30 18:05:23] [INFO] Time Cutoff (Images): 2025-10-01T18:05:23+0000 +[2025-11-30 18:05:23] [INFO] Delete Limit per Type: 200 +[2025-11-30 18:05:23] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:05:23] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:05:26] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:05:26] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:05:27] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:05:27] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 18:05:30] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:05:30] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:05:30] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:05:30] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:05:33] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:05:33] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 18:05:35] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:05:36] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:05:36] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:05:36] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:05:36] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:05:36] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:05:36] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:05:36] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:05:36] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:05:36] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:05:36] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 18:05:36] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:05:36Z (Unix: 1763316336) +[2025-11-30 18:05:36] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763316336]) +[2025-11-30 18:05:39] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:05:39] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 18:05:41] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:05:41] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:05:41] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:05:41] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:05:41] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:05:41] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 18:05:44] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:05:44] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:05:46] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:05:46] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:05:48] [INFO] No Global Address found matching criteria. +[2025-11-30 18:05:48] [INFO] --- Processing: VPC Peerings (Limit: 200) --- +[2025-11-30 18:05:55] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:05:55] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 18:05:57] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:05:57] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:05:57] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:05:57] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:05:57] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:00] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 18:06:02] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:06:02] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 18:06:04] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:06:04] [INFO] CLEANUP RUN FINISHED +[2025-11-30 18:06:34] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:06:34] [INFO] Time Cutoff (General): 2025-11-30T18:06:34+0000 +[2025-11-30 18:06:34] [INFO] Time Cutoff (Images): 2025-10-01T18:06:34+0000 +[2025-11-30 18:06:34] [INFO] Delete Limit per Type: 200 +[2025-11-30 18:06:34] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:06:35] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:06:37] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:06:37] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:06:39] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:06:39] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 18:06:42] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:06:42] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:06:42] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:06:42] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:06:44] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:06:44] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 18:06:47] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:06:47] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:06:47] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:06:47] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:06:47] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:06:48] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:06:48] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:06:48] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:06:48] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:06:48] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:06:48] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 18:06:48] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:06:48Z (Unix: 1763316408) +[2025-11-30 18:06:48] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763316408]) +[2025-11-30 18:06:51] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:06:51] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 18:06:53] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:06:53] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:06:53] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:06:53] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:06:53] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:06:53] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 18:06:55] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:06:55] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:06:58] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:06:58] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:07:00] [INFO] No Global Address found matching criteria. +[2025-11-30 18:07:00] [INFO] --- Processing: VPC Peerings (Limit: 200) --- +[2025-11-30 18:07:02] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:02] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:07:02] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 18:07:05] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:07:05] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:07:05] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:07:05] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:07:05] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 18:07:07] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:07] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:07] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:08] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 18:07:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:07:10] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 18:07:12] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:07:12] [INFO] CLEANUP RUN FINISHED +./cleanup.sh: line 235: syntax error near unexpected token `done' +./cleanup.sh: line 235: ` done' +./cleanup.sh: line 235: syntax error near unexpected token `done' +./cleanup.sh: line 235: ` done' +./cleanup.sh: line 235: syntax error near unexpected token `done' +./cleanup.sh: line 235: ` done' +./cleanup.sh: line 235: syntax error near unexpected token `done' +./cleanup.sh: line 235: ` done' +[2025-11-30 18:29:28] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:29:28] [INFO] Time Cutoff (General): 2025-11-30T18:29:28+0000 +[2025-11-30 18:29:28] [INFO] Time Cutoff (Images): 2025-10-01T18:29:28+0000 +[2025-11-30 18:29:28] [INFO] Delete Limit per Type: 200 +[2025-11-30 18:29:28] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:29:28] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:29:31] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:29:31] [INFO] --- Processing: GKE Cluster (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:29:33] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:29:33] [INFO] --- Processing: Compute Instance (Limit: 200) --- +[2025-11-30 18:29:36] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:29:36] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:29:36] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:29:36] [INFO] --- Processing: Filestore Instances (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:29:39] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:29:39] [INFO] --- Processing: VM Images (Limit: 200) --- +[2025-11-30 18:29:42] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:29:43] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:29:43] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:29:43] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:29:43] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:29:43] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:29:43] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:29:43] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:29:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:29:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:29:43] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- +[2025-11-30 18:29:43] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:29:43Z (Unix: 1763317783) +[2025-11-30 18:29:43] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763317783]) +[2025-11-30 18:29:46] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:29:46] [INFO] --- Processing: Cloud Router (Limit: 200) --- +[2025-11-30 18:29:48] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:29:48] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:29:48] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:29:48] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:29:48] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:29:48] [INFO] --- Processing: Firewall Rules (Limit: 200) --- +[2025-11-30 18:29:51] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:29:51] [INFO] --- Processing: Regional Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:29:53] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:29:53] [INFO] --- Processing: Global Address (Limit: 200) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:29:56] [INFO] No Global Address found matching criteria. +[2025-11-30 18:29:56] [INFO] --- Processing: VPC Peerings (Limit: 200) --- +[2025-11-30 18:29:58] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:29:58] [INFO] --- Processing: Zonal Disk (Limit: 200) --- +[2025-11-30 18:30:01] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:30:01] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:30:01] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:30:01] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:30:01] [INFO] --- Processing: Subnetworks (Limit: 200) --- +[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:04] [INFO] --- Processing: VPC Networks (Limit: 200) --- +[2025-11-30 18:30:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:30:06] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- +[2025-11-30 18:30:08] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:30:08] [INFO] CLEANUP RUN FINISHED diff --git a/policy-bindings.txt b/policy-bindings.txt new file mode 100644 index 0000000000..1d98b7f105 --- /dev/null +++ b/policy-bindings.txt @@ -0,0 +1,2834 @@ +--- Wed Nov 26 04:43:43 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 7: Clean up IAM Policy Bindings for Deleted Service Accounts --- +IAM Binding Cleanup: Total members before: 820 +Found the following deleted service account members in IAM policy: +deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 +deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 +deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 +deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 +deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 +deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 +deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 +deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 +deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 +deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 +deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 +deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 +deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 +deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 +deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 +deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 +deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 +deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 +deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 +deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 +deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 +deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 +deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 +deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 +deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 +deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 +deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 +deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 +deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 +deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 +deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 +deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 +deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 +deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 +deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 +deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 +deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 +deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 +deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 +deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 +deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 +deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 +deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 +deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 +deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 +deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 +deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 +deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 +deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 +deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 +deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 +deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 +deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 +deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 +deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 +deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 +deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 +deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 +deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 +deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 +deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 +deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 +deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 +deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 +deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 +deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 +deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 +deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 +deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 +deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 +deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 +deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 +deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 +deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 +deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 +deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 +deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 +--- Processing member: deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/container.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/container.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/container.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/storage.objectCreator + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009" --role="roles/storage.objectCreator" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/compute.instanceAdmin.v1 + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/compute.instanceAdmin.v1" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/compute.instanceAdmin.v1. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/iam.serviceAccountUser + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/iam.serviceAccountUser" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/iam.serviceAccountUser. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/pubsub.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/pubsub.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/pubsub.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/container.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/container.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/container.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/container.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/container.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/container.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/container.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/container.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/container.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/container.admin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/container.admin" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/container.admin. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/storage.objectViewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/storage.objectViewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 --- +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/artifactregistry.reader + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/artifactregistry.reader" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/artifactregistry.reader. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/logging.logWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/logging.logWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/logging.logWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.metricWriter + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/monitoring.metricWriter" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.metricWriter. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.viewer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/monitoring.viewer" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.viewer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/stackdriver.resourceMetadata.writer + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/stackdriver.resourceMetadata.writer. +[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/storage.objectAdmin + gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/storage.objectAdmin" --all --quiet + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/storage.objectAdmin. +IAM Binding Cleanup: Total members after: 820 +--- Wed Nov 26 04:43:48 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 04:44:21 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 7: Clean up IAM Policy Bindings for Deleted Service Accounts --- +IAM Binding Cleanup: Total members before: 820 +Found the following deleted service account members in IAM policy: +deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 +deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 +deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 +deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 +deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 +deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 +deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 +deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 +deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 +deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 +deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 +deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 +deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 +deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 +deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 +deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 +deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 +deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 +deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 +deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 +deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 +deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 +deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 +deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 +deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 +deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 +deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 +deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 +deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 +deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 +deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 +deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 +deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 +deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 +deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 +deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 +deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 +deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 +deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 +deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 +deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 +deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 +deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 +deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 +deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 +deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 +deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 +deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 +deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 +deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 +deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 +deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 +deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 +deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 +deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 +deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 +deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 +deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 +deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 +deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 +deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 +deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 +deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 +deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 +deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 +deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 +deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 +deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 +deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 +deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 +deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 +deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 +deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 +deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 +deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 +deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 +deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 +--- Processing member: deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/container.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/container.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/storage.objectCreator +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/storage.objectCreator. +--- Processing member: deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/compute.instanceAdmin.v1 +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/compute.instanceAdmin.v1. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/iam.serviceAccountUser +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/iam.serviceAccountUser. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/pubsub.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/pubsub.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/container.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/container.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/container.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/container.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/container.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/container.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/container.admin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/container.admin. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 from roles/storage.objectAdmin. +--- Processing member: deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/storage.objectViewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/storage.objectViewer. +--- Processing member: deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 --- +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/artifactregistry.reader +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/artifactregistry.reader. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/logging.logWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/logging.logWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.metricWriter +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.metricWriter. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.viewer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.viewer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/stackdriver.resourceMetadata.writer +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/stackdriver.resourceMetadata.writer. +[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/storage.objectAdmin +Updated IAM policy for project [hpc-toolkit-dev]. + Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/storage.objectAdmin. +IAM Binding Cleanup: Total members after: 445 +--- Wed Nov 26 05:01:01 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 05:15:36 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 7: Clean up IAM Policy Bindings for Deleted Service Accounts --- +IAM Binding Cleanup: Total members before: 432 +IAM Binding Cleanup: No deleted service accounts found in IAM policy. +IAM Binding Cleanup: Total members after: 432 +--- Wed Nov 26 05:15:40 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 05:15:59 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 7: Clean up IAM Policy Bindings for Deleted Service Accounts --- +IAM Binding Cleanup: Total members before: 432 +IAM Binding Cleanup: No deleted service accounts found in IAM policy. +IAM Binding Cleanup: Total members after: 432 +--- Wed Nov 26 05:16:03 PM UTC 2025 --- Cleanup Script Run Finished --- diff --git a/rdisk.txt b/rdisk.txt new file mode 100644 index 0000000000..8107f7ef42 --- /dev/null +++ b/rdisk.txt @@ -0,0 +1,373 @@ +--- Thu Nov 27 02:16:12 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T10:16:12+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Traceback (most recent call last): + File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 193, in + main() + File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 187, in main + gcloud_main = _import_gcloud_main() + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 90, in _import_gcloud_main + import googlecloudsdk.gcloud_main + File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/gcloud_main.py", line 42, in + from googlecloudsdk.core.credentials import creds_context_managers + File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/core/credentials/creds_context_managers.py", line 29, in + from googlecloudsdk.core.credentials import store + File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/core/credentials/store.py", line 34, in + from googlecloudsdk.api_lib.auth import external_account as auth_external_account + File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/api_lib/auth/external_account.py", line 24, in + from googlecloudsdk.core.credentials import creds as c_creds + File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/core/credentials/creds.py", line 33, in + from google.auth import compute_engine as google_auth_compute_engine + File "/usr/bin/../lib/google-cloud-sdk/lib/third_party/google/auth/compute_engine/__init__.py", line 18, in + from google.auth.compute_engine.credentials import Credentials + File "", line 1360, in _find_and_load + File "", line 1331, in _find_and_load_unlocked + File "", line 935, in _load_unlocked + File "", line 995, in exec_module + File "", line 1091, in get_code + File "", line 1190, in get_data +KeyboardInterrupt +--- Thu Nov 27 02:16:41 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T10:16:41+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-dev-06-net +lustre-prod-06-net +lustre-qa-05-net +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-dev-06-net + Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-prod-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net +[DRY RUN] Route: Would delete peering-route-eebb81463c1f952f for network lustre-prod-06-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-prod-06-net + Command: gcloud compute networks delete "lustre-prod-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: vertexui-do-not-kill-boot in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) +Skip Zonal Disk: vertexui-do-not-kill-data in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) +No Zonal Disks to delete in this run. +--- Deletion Phase 4b: Regional Persistent Disks (Top 30) --- +WARNING: The following filter keys were not present in any resource : region +No Regional Disks found matching criteria or list command failed. +--- Thu Nov 27 02:17:13 PM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 02:18:53 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T13:18:53+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 47 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - default + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) +Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) +Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) +Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +No Subnetworks found to delete in this run after filtering. +--- Subnetwork Deletion Phase Complete --- +--- Deletion Phase 6: Networks (Top 30) --- +Skip Network: default (Is default) +Skip Network: hpc-vpc (In exclusion list) +The following Networks and their dependencies are targeted for deletion in this run: +lustre-dev-06-net +lustre-prod-06-net +lustre-qa-05-net +--- Processing Network: lustre-dev-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net +[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-dev-06-net + Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-prod-06-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net +[DRY RUN] Route: Would delete peering-route-eebb81463c1f952f for network lustre-prod-06-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-prod-06-net + Command: gcloud compute networks delete "lustre-prod-06-net" --project="hpc-toolkit-dev" --quiet +--- Processing Network: lustre-qa-05-net --- +Checking for dependent routes... +[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net +[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net +Checking for dependent firewall rules... +No dependent firewall rules found. +[DRY RUN] Network: Would delete lustre-qa-05-net + Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet +--- Network Deletion Process Complete --- +--- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- +Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) +Skip Zonal Disk: vertexui-do-not-kill-boot in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) +Skip Zonal Disk: vertexui-do-not-kill-data in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) +No Zonal Disks to delete in this run. +--- Deletion Phase 4b: Regional Persistent Disks (Top 30) --- +WARNING: The following filter keys were not present in any resource : region +No Regional Disks found matching criteria or list command failed. +--- Thu Nov 27 02:19:25 PM UTC 2025 --- Cleanup Script Run Finished --- + diff --git a/routers.txt b/routers.txt new file mode 100644 index 0000000000..74b7af4e48 --- /dev/null +++ b/routers.txt @@ -0,0 +1,1350 @@ +--- Wed Nov 26 01:51:04 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 10) --- +The following Cloud Routers are targeted for deletion in this run: +a3mega-sys-net-shu7-router us-west4 +a3u-slurm-net-router us-south1 +a4hsarthakag-net-router us-central1 +a4htest-net-0-router us-central1 +a4newimgek-net-0-router us-south1 +a4newimgek-net-1-router us-south1 +a4newimgek-net-router europe-west4 +a4oldimgek-net-0-router europe-west4 +a4oldimgek-net-1-router europe-west4 +a4oldimgek-net-router europe-west4 + gcloud compute routers delete "a3mega-sys-net-shu7-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "a3u-slurm-net-router" --project="hpc-toolkit-dev" --region="us-south1" --quiet + gcloud compute routers delete "a4hsarthakag-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "a4htest-net-0-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "a4newimgek-net-0-router" --project="hpc-toolkit-dev" --region="us-south1" --quiet + gcloud compute routers delete "a4newimgek-net-1-router" --project="hpc-toolkit-dev" --region="us-south1" --quiet + gcloud compute routers delete "a4newimgek-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet + gcloud compute routers delete "a4oldimgek-net-0-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet + gcloud compute routers delete "a4oldimgek-net-1-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet + gcloud compute routers delete "a4oldimgek-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +--- Wed Nov 26 01:51:11 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:51:59 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 10) --- +The following Cloud Routers are targeted for deletion in this run: +a3mega-sys-net-shu7-router us-west4 +a3u-slurm-net-router us-south1 +a4hsarthakag-net-router us-central1 +a4htest-net-0-router us-central1 +a4newimgek-net-0-router us-south1 +a4newimgek-net-1-router us-south1 +a4newimgek-net-router europe-west4 +a4oldimgek-net-0-router europe-west4 +a4oldimgek-net-1-router europe-west4 +a4oldimgek-net-router europe-west4 +[EXECUTE] Cloud Router: Deleting a3mega-sys-net-shu7-router in us-west4 +[EXECUTE] Cloud Router: Deleting a3u-slurm-net-router in us-south1 +[EXECUTE] Cloud Router: Deleting a4hsarthakag-net-router in us-central1 +[EXECUTE] Cloud Router: Deleting a4htest-net-0-router in us-central1 +[EXECUTE] Cloud Router: Deleting a4newimgek-net-0-router in us-south1 +[EXECUTE] Cloud Router: Deleting a4newimgek-net-1-router in us-south1 +[EXECUTE] Cloud Router: Deleting a4newimgek-net-router in europe-west4 +[EXECUTE] Cloud Router: Deleting a4oldimgek-net-0-router in europe-west4 +[EXECUTE] Cloud Router: Deleting a4oldimgek-net-1-router in europe-west4 +[EXECUTE] Cloud Router: Deleting a4oldimgek-net-router in europe-west4 +--- Wed Nov 26 01:52:07 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:52:24 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 10) --- +The following Cloud Routers are targeted for deletion in this run: +a3mega-sys-net-shu7-router us-west4 +a3u-slurm-net-router us-south1 +a4hsarthakag-net-router us-central1 +a4htest-net-0-router us-central1 +a4newimgek-net-0-router us-south1 +a4newimgek-net-1-router us-south1 +a4newimgek-net-router europe-west4 +a4oldimgek-net-0-router europe-west4 +a4oldimgek-net-1-router europe-west4 +a4oldimgek-net-router europe-west4 +[EXECUTE] Cloud Router: Deleting a3mega-sys-net-shu7-router in us-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west4/routers/a3mega-sys-net-shu7-router]. +[EXECUTE] Cloud Router: Deleting a3u-slurm-net-router in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/routers/a3u-slurm-net-router]. +[EXECUTE] Cloud Router: Deleting a4hsarthakag-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4hsarthakag-net-router]. +[EXECUTE] Cloud Router: Deleting a4htest-net-0-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4htest-net-0-router]. +[EXECUTE] Cloud Router: Deleting a4newimgek-net-0-router in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/routers/a4newimgek-net-0-router]. +[EXECUTE] Cloud Router: Deleting a4newimgek-net-1-router in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/routers/a4newimgek-net-1-router]. +[EXECUTE] Cloud Router: Deleting a4newimgek-net-router in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/a4newimgek-net-router]. +[EXECUTE] Cloud Router: Deleting a4oldimgek-net-0-router in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/a4oldimgek-net-0-router]. +[EXECUTE] Cloud Router: Deleting a4oldimgek-net-1-router in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/a4oldimgek-net-1-router]. +[EXECUTE] Cloud Router: Deleting a4oldimgek-net-router in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/a4oldimgek-net-router]. +--- Wed Nov 26 01:53:20 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:56:35 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 30 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 10) --- +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +a4oldimg-net-router europe-west4 +a4xlavhpcnew-a4x-net-0-router us-west8 +a4xlavhpcnew-a4x-net-1-router us-west8 +a4xslurm-net-router us-west8 +cx-a3u-net-0-router europe-west1 +db451c7-ml-slurm-v6-net-router asia-southeast1 +default-net-router us-central1 +default-router-australia-southeast1 australia-southeast1 +default-router-us-east4 us-east4 +dynpoc-net-router us-central1 + gcloud compute routers delete "a4oldimg-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet + gcloud compute routers delete "a4xlavhpcnew-a4x-net-0-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet + gcloud compute routers delete "a4xlavhpcnew-a4x-net-1-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet + gcloud compute routers delete "a4xslurm-net-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet + gcloud compute routers delete "cx-a3u-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "db451c7-ml-slurm-v6-net-router" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet + gcloud compute routers delete "default-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "default-router-australia-southeast1" --project="hpc-toolkit-dev" --region="australia-southeast1" --quiet + gcloud compute routers delete "default-router-us-east4" --project="hpc-toolkit-dev" --region="us-east4" --quiet + gcloud compute routers delete "dynpoc-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Wed Nov 26 01:56:43 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:57:33 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 33 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +a4oldimg-net-router europe-west4 +a4xlavhpcnew-a4x-net-0-router us-west8 +a4xlavhpcnew-a4x-net-1-router us-west8 +a4xslurm-net-router us-west8 +cx-a3u-net-0-router europe-west1 +db451c7-ml-slurm-v6-net-router asia-southeast1 +dynpoc-net-router us-central1 +g4-dwsq-1-net-1-router us-central1 +g4qclav-net-router us-central1 +gke-1395b4-net-router us-central1 + gcloud compute routers delete "a4oldimg-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet + gcloud compute routers delete "a4xlavhpcnew-a4x-net-0-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet + gcloud compute routers delete "a4xlavhpcnew-a4x-net-1-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet + gcloud compute routers delete "a4xslurm-net-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet + gcloud compute routers delete "cx-a3u-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "db451c7-ml-slurm-v6-net-router" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet + gcloud compute routers delete "dynpoc-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "g4-dwsq-1-net-1-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "g4qclav-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "gke-1395b4-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Wed Nov 26 01:57:40 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:57:54 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 33 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +a4oldimg-net-router europe-west4 +a4xlavhpcnew-a4x-net-0-router us-west8 +a4xlavhpcnew-a4x-net-1-router us-west8 +a4xslurm-net-router us-west8 +cx-a3u-net-0-router europe-west1 +db451c7-ml-slurm-v6-net-router asia-southeast1 +dynpoc-net-router us-central1 +g4-dwsq-1-net-1-router us-central1 +g4qclav-net-router us-central1 +gke-1395b4-net-router us-central1 +[EXECUTE] Cloud Router: Deleting a4oldimg-net-router in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/a4oldimg-net-router]. +[EXECUTE] Cloud Router: Deleting a4xlavhpcnew-a4x-net-0-router in us-west8 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/routers/a4xlavhpcnew-a4x-net-0-router]. +[EXECUTE] Cloud Router: Deleting a4xlavhpcnew-a4x-net-1-router in us-west8 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/routers/a4xlavhpcnew-a4x-net-1-router]. +[EXECUTE] Cloud Router: Deleting a4xslurm-net-router in us-west8 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/routers/a4xslurm-net-router]. +[EXECUTE] Cloud Router: Deleting cx-a3u-net-0-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/cx-a3u-net-0-router]. +[EXECUTE] Cloud Router: Deleting db451c7-ml-slurm-v6-net-router in asia-southeast1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/asia-southeast1/routers/db451c7-ml-slurm-v6-net-router]. +[EXECUTE] Cloud Router: Deleting dynpoc-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/dynpoc-net-router]. +[EXECUTE] Cloud Router: Deleting g4-dwsq-1-net-1-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/g4-dwsq-1-net-1-router]. +[EXECUTE] Cloud Router: Deleting g4qclav-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/g4qclav-net-router]. +[EXECUTE] Cloud Router: Deleting gke-1395b4-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/gke-1395b4-net-router]. +--- Wed Nov 26 01:58:47 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 01:59:14 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 33 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +gke-managed-lustre-basic-net-router us-central1 +h4dqc-net-router us-central1 +h4d-res-swarnabm4-3-net-router us-central1 +hpc-01-net-router us-central1 +hpcdydis-net-router europe-west4 +hpcdy-net-router europe-west4 +hpc-exr-2-net-0-router europe-west1 +hpcimg-net-router us-central1 +hpc-lustre-test-02-net-router us-central1 +khu-h4d-cluster-test-net-router us-central1 + gcloud compute routers delete "gke-managed-lustre-basic-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "h4dqc-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "h4d-res-swarnabm4-3-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "hpc-01-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "hpcdydis-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet + gcloud compute routers delete "hpcdy-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet + gcloud compute routers delete "hpc-exr-2-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "hpcimg-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "hpc-lustre-test-02-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "khu-h4d-cluster-test-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Wed Nov 26 01:59:23 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:02:27 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 33 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +gke-managed-lustre-basic-net-router us-central1 +h4dqc-net-router us-central1 +h4d-res-swarnabm4-3-net-router us-central1 +hpc-01-net-router us-central1 +hpcdydis-net-router europe-west4 +hpcdy-net-router europe-west4 +hpc-exr-2-net-0-router europe-west1 +hpcimg-net-router us-central1 +hpc-lustre-test-02-net-router us-central1 +khu-h4d-cluster-test-net-router us-central1 + gcloud compute routers delete "gke-managed-lustre-basic-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "h4dqc-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "h4d-res-swarnabm4-3-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "hpc-01-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "hpcdydis-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet + gcloud compute routers delete "hpcdy-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet + gcloud compute routers delete "hpc-exr-2-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "hpcimg-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "hpc-lustre-test-02-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "khu-h4d-cluster-test-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Wed Nov 26 02:02:35 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:03:11 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 33 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +gke-managed-lustre-basic-net-router us-central1 +h4dqc-net-router us-central1 +h4d-res-swarnabm4-3-net-router us-central1 +hpc-01-net-router us-central1 +hpcdydis-net-router europe-west4 +hpcdy-net-router europe-west4 +hpc-exr-2-net-0-router europe-west1 +hpcimg-net-router us-central1 +hpc-lustre-test-02-net-router us-central1 +khu-h4d-cluster-test-net-router us-central1 +[EXECUTE] Cloud Router: Deleting gke-managed-lustre-basic-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/gke-managed-lustre-basic-net-router]. +[EXECUTE] Cloud Router: Deleting h4dqc-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/h4dqc-net-router]. +[EXECUTE] Cloud Router: Deleting h4d-res-swarnabm4-3-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/h4d-res-swarnabm4-3-net-router]. +[EXECUTE] Cloud Router: Deleting hpc-01-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/hpc-01-net-router]. +[EXECUTE] Cloud Router: Deleting hpcdydis-net-router in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/hpcdydis-net-router]. +[EXECUTE] Cloud Router: Deleting hpcdy-net-router in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/hpcdy-net-router]. +[EXECUTE] Cloud Router: Deleting hpc-exr-2-net-0-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/hpc-exr-2-net-0-router]. +[EXECUTE] Cloud Router: Deleting hpcimg-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/hpcimg-net-router]. +[EXECUTE] Cloud Router: Deleting hpc-lustre-test-02-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/hpc-lustre-test-02-net-router]. +[EXECUTE] Cloud Router: Deleting khu-h4d-cluster-test-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/khu-h4d-cluster-test-net-router]. +--- Wed Nov 26 02:04:01 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:04:14 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 33 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +laveeek29-net-0-router europe-west1 +laveeek29-net-1-router europe-west1 +laveeek29-net-router europe-west1 +lavoldchk-net-router europe-west1 +lavrohek29-net-0-router europe-west1 +lustre-06-net-router us-central1 +lustre-test-06-net-router us-central1 +mainek-net-0-router europe-west1 +mainek-net-1-router europe-west1 +mainek-net-router europe-west1 +managed-lustre-03-net-router us-central1 +mglsa-net-router us-central1 +mglsard-net-router us-west4 +ml-gke-e2e-a8fae6-net-router asia-southeast1 +ml-gke-net-router us-central1 +monitoring-8323fe-net-router us-central1 +sa-chs-ops-net-0-router europe-west1 +sispot3u-net-0-router europe-west1 +slurm-a3-base-sysnet-router us-west1 +sp-helmtest1-net-router us-central1 + gcloud compute routers delete "laveeek29-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "laveeek29-net-1-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "laveeek29-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "lavoldchk-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "lavrohek29-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "lustre-06-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "lustre-test-06-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "mainek-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "mainek-net-1-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "mainek-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "managed-lustre-03-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "mglsa-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "mglsard-net-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet + gcloud compute routers delete "ml-gke-e2e-a8fae6-net-router" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet + gcloud compute routers delete "ml-gke-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "monitoring-8323fe-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "sa-chs-ops-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "sispot3u-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "slurm-a3-base-sysnet-router" --project="hpc-toolkit-dev" --region="us-west1" --quiet + gcloud compute routers delete "sp-helmtest1-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Wed Nov 26 02:04:22 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:05:46 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 37 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +laveeek29-net-0-router europe-west1 +laveeek29-net-1-router europe-west1 +laveeek29-net-router europe-west1 +lavoldchk-net-router europe-west1 +lavrohek29-net-0-router europe-west1 +mainek-net-0-router europe-west1 +mainek-net-1-router europe-west1 +mainek-net-router europe-west1 +managed-lustre-03-net-router us-central1 +ml-gke-e2e-a8fae6-net-router asia-southeast1 +ml-gke-net-router us-central1 +monitoring-8323fe-net-router us-central1 +sa-chs-ops-net-0-router europe-west1 +sispot3u-net-0-router europe-west1 +slurm-a3-base-sysnet-router us-west1 +sp-helmtest1-net-router us-central1 +static-sarthakag-net-router us-central1 + gcloud compute routers delete "laveeek29-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "laveeek29-net-1-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "laveeek29-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "lavoldchk-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "lavrohek29-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "mainek-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "mainek-net-1-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "mainek-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "managed-lustre-03-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "ml-gke-e2e-a8fae6-net-router" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet + gcloud compute routers delete "ml-gke-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "monitoring-8323fe-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "sa-chs-ops-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "sispot3u-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet + gcloud compute routers delete "slurm-a3-base-sysnet-router" --project="hpc-toolkit-dev" --region="us-west1" --quiet + gcloud compute routers delete "sp-helmtest1-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet + gcloud compute routers delete "static-sarthakag-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Wed Nov 26 02:05:53 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:06:46 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 37 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +laveeek29-net-0-router europe-west1 +laveeek29-net-1-router europe-west1 +laveeek29-net-router europe-west1 +lavoldchk-net-router europe-west1 +lavrohek29-net-0-router europe-west1 +mainek-net-0-router europe-west1 +mainek-net-1-router europe-west1 +mainek-net-router europe-west1 +managed-lustre-03-net-router us-central1 +ml-gke-e2e-a8fae6-net-router asia-southeast1 +ml-gke-net-router us-central1 +monitoring-8323fe-net-router us-central1 +sa-chs-ops-net-0-router europe-west1 +sispot3u-net-0-router europe-west1 +slurm-a3-base-sysnet-router us-west1 +sp-helmtest1-net-router us-central1 +static-sarthakag-net-router us-central1 +[EXECUTE] Cloud Router: Deleting laveeek29-net-0-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/laveeek29-net-0-router]. +[EXECUTE] Cloud Router: Deleting laveeek29-net-1-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/laveeek29-net-1-router]. +[EXECUTE] Cloud Router: Deleting laveeek29-net-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/laveeek29-net-router]. +[EXECUTE] Cloud Router: Deleting lavoldchk-net-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/lavoldchk-net-router]. +[EXECUTE] Cloud Router: Deleting lavrohek29-net-0-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/lavrohek29-net-0-router]. +[EXECUTE] Cloud Router: Deleting mainek-net-0-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/mainek-net-0-router]. +[EXECUTE] Cloud Router: Deleting mainek-net-1-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/mainek-net-1-router]. +[EXECUTE] Cloud Router: Deleting mainek-net-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/mainek-net-router]. +[EXECUTE] Cloud Router: Deleting managed-lustre-03-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/managed-lustre-03-net-router]. +[EXECUTE] Cloud Router: Deleting ml-gke-e2e-a8fae6-net-router in asia-southeast1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/asia-southeast1/routers/ml-gke-e2e-a8fae6-net-router]. +[EXECUTE] Cloud Router: Deleting ml-gke-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/ml-gke-net-router]. +[EXECUTE] Cloud Router: Deleting monitoring-8323fe-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/monitoring-8323fe-net-router]. +[EXECUTE] Cloud Router: Deleting sa-chs-ops-net-0-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/sa-chs-ops-net-0-router]. +[EXECUTE] Cloud Router: Deleting sispot3u-net-0-router in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/sispot3u-net-0-router]. +[EXECUTE] Cloud Router: Deleting slurm-a3-base-sysnet-router in us-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west1/routers/slurm-a3-base-sysnet-router]. +[EXECUTE] Cloud Router: Deleting sp-helmtest1-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/sp-helmtest1-net-router]. +[EXECUTE] Cloud Router: Deleting static-sarthakag-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/static-sarthakag-net-router]. +--- Wed Nov 26 02:08:16 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Wed Nov 26 02:08:27 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 20 resources of each type per run. +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 37 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router +--- Deletion Phase 1: GKE Clusters (Top 20) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 20) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 20) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 2: Cloud Routers (Top 20) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Wed Nov 26 02:08:34 PM UTC 2025 --- Cleanup Script Run Finished --- +--- Thu Nov 27 03:43:35 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-26T23:43:35+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +./cleanup.sh: line 77: ---: command not found +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +a4h-slurm-net-0-router us-central1 +a4h-slurm-net-1-router us-central1 +a4h-slurm-net-router us-central1 +[DRY RUN] Cloud Router: Would delete a4h-slurm-net-0-router in us-central1 + Command: gcloud compute routers delete "a4h-slurm-net-0-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Cloud Router: Would delete a4h-slurm-net-1-router in us-central1 + Command: gcloud compute routers delete "a4h-slurm-net-1-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Cloud Router: Would delete a4h-slurm-net-router in us-central1 + Command: gcloud compute routers delete "a4h-slurm-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Thu Nov 27 03:43:45 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 03:44:29 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-26T23:44:29+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 59 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com +--- Deletion Phase 1: GKE Clusters (Top 10) --- +Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) +Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) +Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) +Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) +Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) +Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +The following Cloud Routers are targeted for deletion in this run: +a4h-slurm-net-0-router us-central1 +a4h-slurm-net-1-router us-central1 +a4h-slurm-net-router us-central1 +[EXECUTE] Cloud Router: Deleting a4h-slurm-net-0-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-0-router]. +[EXECUTE] Cloud Router: Deleting a4h-slurm-net-1-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-1-router]. +[EXECUTE] Cloud Router: Deleting a4h-slurm-net-router in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-router]. +--- Thu Nov 27 03:44:48 AM UTC 2025 --- Cleanup Script Run Finished --- diff --git a/subnetworks.txt b/subnetworks.txt new file mode 100644 index 0000000000..a8bae75926 --- /dev/null +++ b/subnetworks.txt @@ -0,0 +1,3453 @@ +--- Thu Nov 27 05:20:28 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:20:28+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 10) --- +WARNING: The following filter keys were not present in any resource : createTime +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 10 per region) --- +Processing Subnetworks in region: africa-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in africa-south1 in this run. +Processing Subnetworks in region: asia-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-east1 in this run. +Processing Subnetworks in region: asia-east2 +No Subnetworks found to delete in asia-east2 in this run. +Processing Subnetworks in region: asia-northeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-northeast1 in this run. +Processing Subnetworks in region: asia-northeast2 +No Subnetworks found to delete in asia-northeast2 in this run. +Processing Subnetworks in region: asia-northeast3 +No Subnetworks found to delete in asia-northeast3 in this run. +Processing Subnetworks in region: asia-south1 +No Subnetworks found to delete in asia-south1 in this run. +Processing Subnetworks in region: asia-south2 +No Subnetworks found to delete in asia-south2 in this run. +Processing Subnetworks in region: asia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in asia-southeast1 are targeted for deletion in this run: +db451c7-ml-slurm-v6-primary-subnet asia-southeast1 +ml-gke-e2e-a8fae6-subnet asia-southeast1 +[DRY RUN] Subnetwork: Would delete db451c7-ml-slurm-v6-primary-subnet in asia-southeast1 + Command: gcloud compute networks subnets delete "db451c7-ml-slurm-v6-primary-subnet" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet +[DRY RUN] Subnetwork: Would delete ml-gke-e2e-a8fae6-subnet in asia-southeast1 + Command: gcloud compute networks subnets delete "ml-gke-e2e-a8fae6-subnet" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet +Processing Subnetworks in region: asia-southeast2 +No Subnetworks found to delete in asia-southeast2 in this run. +Processing Subnetworks in region: asia-southeast3 +No Subnetworks found to delete in asia-southeast3 in this run. +Processing Subnetworks in region: australia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in australia-southeast1 in this run. +Processing Subnetworks in region: australia-southeast2 +No Subnetworks found to delete in australia-southeast2 in this run. +Processing Subnetworks in region: europe-central2 +No Subnetworks found to delete in europe-central2 in this run. +Processing Subnetworks in region: europe-north1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north1 in this run. +Processing Subnetworks in region: europe-north2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north2 in this run. +Processing Subnetworks in region: europe-southwest1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-southwest1 in this run. +Processing Subnetworks in region: europe-west1 +WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in europe-west1 are targeted for deletion in this run: +cx-a3u-sub-0 europe-west1 +hanu-a3u-primary-subnet europe-west1 +hpc-exr-2-sub-0 europe-west1 +laveeek29-mrdma-sub-0 europe-west1 +laveeek29-mrdma-sub-1 europe-west1 +laveeek29-mrdma-sub-2 europe-west1 +laveeek29-mrdma-sub-3 europe-west1 +laveeek29-mrdma-sub-4 europe-west1 +laveeek29-mrdma-sub-5 europe-west1 +laveeek29-mrdma-sub-6 europe-west1 +[DRY RUN] Subnetwork: Would delete cx-a3u-sub-0 in europe-west1 + Command: gcloud compute networks subnets delete "cx-a3u-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete hanu-a3u-primary-subnet in europe-west1 + Command: gcloud compute networks subnets delete "hanu-a3u-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete hpc-exr-2-sub-0 in europe-west1 + Command: gcloud compute networks subnets delete "hpc-exr-2-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-0 in europe-west1 + Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-1 in europe-west1 + Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-1" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-2 in europe-west1 + Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-2" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-3 in europe-west1 + Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-3" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-4 in europe-west1 + Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-4" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-5 in europe-west1 + Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-5" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-6 in europe-west1 + Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-6" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +Processing Subnetworks in region: europe-west10 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west10 in this run. +Processing Subnetworks in region: europe-west12 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west12 in this run. +Processing Subnetworks in region: europe-west15 +No Subnetworks found to delete in europe-west15 in this run. +Processing Subnetworks in region: europe-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west2 in this run. +Processing Subnetworks in region: europe-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west3 in this run. +Processing Subnetworks in region: europe-west4 +The following Subnetworks in europe-west4 are targeted for deletion in this run: +a4newimgek-primary-subnet europe-west4 +a4oldimgek-mrdma-sub-0 europe-west4 +a4oldimgek-mrdma-sub-1 europe-west4 +a4oldimgek-mrdma-sub-2 europe-west4 +a4oldimgek-mrdma-sub-3 europe-west4 +a4oldimgek-mrdma-sub-4 europe-west4 +a4oldimgek-mrdma-sub-5 europe-west4 +a4oldimgek-mrdma-sub-6 europe-west4 +a4oldimgek-mrdma-sub-7 europe-west4 +a4oldimgek-primary-subnet europe-west4 +[DRY RUN] Subnetwork: Would delete a4newimgek-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "a4newimgek-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-0 in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-0" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-1 in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-1" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-2 in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-2" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-3 in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-3" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-4 in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-4" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-5 in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-5" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-6 in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-6" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-7 in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-7" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete a4oldimgek-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimgek-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +Processing Subnetworks in region: europe-west6 +No Subnetworks found to delete in europe-west6 in this run. +Processing Subnetworks in region: europe-west8 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west8 in this run. +Processing Subnetworks in region: europe-west9 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west9 in this run. +Processing Subnetworks in region: me-central1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central1 in this run. +Processing Subnetworks in region: me-central2 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central2 in this run. +Processing Subnetworks in region: me-west1 +No Subnetworks found to delete in me-west1 in this run. +Processing Subnetworks in region: northamerica-northeast1 +No Subnetworks found to delete in northamerica-northeast1 in this run. +Processing Subnetworks in region: northamerica-northeast2 +No Subnetworks found to delete in northamerica-northeast2 in this run. +Processing Subnetworks in region: northamerica-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in northamerica-south1 in this run. +Processing Subnetworks in region: southamerica-east1 +No Subnetworks found to delete in southamerica-east1 in this run. +Processing Subnetworks in region: southamerica-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in southamerica-west1 in this run. +Processing Subnetworks in region: us-central1 +The following Subnetworks in us-central1 are targeted for deletion in this run: +a4h-slurm-c0e262-primary-subnet us-central1 +a4h-slurm-mrdma-sub-0 us-central1 +a4h-slurm-mrdma-sub-1 us-central1 +a4h-slurm-mrdma-sub-2 us-central1 +a4h-slurm-mrdma-sub-3 us-central1 +a4h-slurm-mrdma-sub-4 us-central1 +a4h-slurm-mrdma-sub-5 us-central1 +a4h-slurm-mrdma-sub-6 us-central1 +a4h-slurm-mrdma-sub-7 us-central1 +a4h-slurm-sub-0 us-central1 +[DRY RUN] Subnetwork: Would delete a4h-slurm-c0e262-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-c0e262-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-1 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-1" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-2 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-2" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-3 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-3" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-4 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-4" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-5 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-5" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-6 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-6" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-7 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-7" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete a4h-slurm-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +Processing Subnetworks in region: us-central2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-central2 in this run. +Processing Subnetworks in region: us-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east1 in this run. +Processing Subnetworks in region: us-east4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east4 in this run. +Processing Subnetworks in region: us-east5 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east5 in this run. +Processing Subnetworks in region: us-east7 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east7 in this run. +Processing Subnetworks in region: us-south1 +The following Subnetworks in us-south1 are targeted for deletion in this run: +a3u-slurm-3224ec-primary-subnet us-south1 +a4newimgek-mrdma-sub-0 us-south1 +a4newimgek-mrdma-sub-1 us-south1 +a4newimgek-mrdma-sub-2 us-south1 +a4newimgek-mrdma-sub-3 us-south1 +a4newimgek-mrdma-sub-4 us-south1 +a4newimgek-mrdma-sub-5 us-south1 +a4newimgek-mrdma-sub-6 us-south1 +a4newimgek-mrdma-sub-7 us-south1 +a4newimgek-sub-0 us-south1 +[DRY RUN] Subnetwork: Would delete a3u-slurm-3224ec-primary-subnet in us-south1 + Command: gcloud compute networks subnets delete "a3u-slurm-3224ec-primary-subnet" --project="hpc-toolkit-dev" --region="us-south1" --quiet +[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-0 in us-south1 + Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-0" --project="hpc-toolkit-dev" --region="us-south1" --quiet +[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-1 in us-south1 + Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-1" --project="hpc-toolkit-dev" --region="us-south1" --quiet +[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-2 in us-south1 + Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-2" --project="hpc-toolkit-dev" --region="us-south1" --quiet +[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-3 in us-south1 + Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-3" --project="hpc-toolkit-dev" --region="us-south1" --quiet +[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-4 in us-south1 + Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-4" --project="hpc-toolkit-dev" --region="us-south1" --quiet +[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-5 in us-south1 + Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-5" --project="hpc-toolkit-dev" --region="us-south1" --quiet +[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-6 in us-south1 + Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-6" --project="hpc-toolkit-dev" --region="us-south1" --quiet +[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-7 in us-south1 + Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-7" --project="hpc-toolkit-dev" --region="us-south1" --quiet +[DRY RUN] Subnetwork: Would delete a4newimgek-sub-0 in us-south1 + Command: gcloud compute networks subnets delete "a4newimgek-sub-0" --project="hpc-toolkit-dev" --region="us-south1" --quiet +Processing Subnetworks in region: us-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in us-west1 are targeted for deletion in this run: +slurm-a3-base-sysnet-subnet us-west1 +[DRY RUN] Subnetwork: Would delete slurm-a3-base-sysnet-subnet in us-west1 + Command: gcloud compute networks subnets delete "slurm-a3-base-sysnet-subnet" --project="hpc-toolkit-dev" --region="us-west1" --quiet +Processing Subnetworks in region: us-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west2 in this run. +Processing Subnetworks in region: us-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west3 in this run. +Processing Subnetworks in region: us-west4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in us-west4 are targeted for deletion in this run: +a3mega-sys-subnet us-west4 +mglsard-subnet us-west4 +[DRY RUN] Subnetwork: Would delete a3mega-sys-subnet in us-west4 + Command: gcloud compute networks subnets delete "a3mega-sys-subnet" --project="hpc-toolkit-dev" --region="us-west4" --quiet +[DRY RUN] Subnetwork: Would delete mglsard-subnet in us-west4 + Command: gcloud compute networks subnets delete "mglsard-subnet" --project="hpc-toolkit-dev" --region="us-west4" --quiet +Processing Subnetworks in region: us-west8 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in us-west8 are targeted for deletion in this run: +a4xlavhpcnew-a4x-sub-0 us-west8 +a4xlavhpcnew-a4x-sub-1 us-west8 +a4x-mrdma-sub-0 us-west8 +a4x-mrdma-sub-1 us-west8 +a4x-mrdma-sub-2 us-west8 +a4x-mrdma-sub-3 us-west8 +a4xslurm-primary-subnet us-west8 +[DRY RUN] Subnetwork: Would delete a4xlavhpcnew-a4x-sub-0 in us-west8 + Command: gcloud compute networks subnets delete "a4xlavhpcnew-a4x-sub-0" --project="hpc-toolkit-dev" --region="us-west8" --quiet +[DRY RUN] Subnetwork: Would delete a4xlavhpcnew-a4x-sub-1 in us-west8 + Command: gcloud compute networks subnets delete "a4xlavhpcnew-a4x-sub-1" --project="hpc-toolkit-dev" --region="us-west8" --quiet +[DRY RUN] Subnetwork: Would delete a4x-mrdma-sub-0 in us-west8 + Command: gcloud compute networks subnets delete "a4x-mrdma-sub-0" --project="hpc-toolkit-dev" --region="us-west8" --quiet +[DRY RUN] Subnetwork: Would delete a4x-mrdma-sub-1 in us-west8 + Command: gcloud compute networks subnets delete "a4x-mrdma-sub-1" --project="hpc-toolkit-dev" --region="us-west8" --quiet +[DRY RUN] Subnetwork: Would delete a4x-mrdma-sub-2 in us-west8 + Command: gcloud compute networks subnets delete "a4x-mrdma-sub-2" --project="hpc-toolkit-dev" --region="us-west8" --quiet +[DRY RUN] Subnetwork: Would delete a4x-mrdma-sub-3 in us-west8 + Command: gcloud compute networks subnets delete "a4x-mrdma-sub-3" --project="hpc-toolkit-dev" --region="us-west8" --quiet +[DRY RUN] Subnetwork: Would delete a4xslurm-primary-subnet in us-west8 + Command: gcloud compute networks subnets delete "a4xslurm-primary-subnet" --project="hpc-toolkit-dev" --region="us-west8" --quiet +--- Thu Nov 27 05:22:15 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 05:24:37 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:24:37+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 10 per region) --- +Processing Subnetworks in region: africa-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in africa-south1 in this run. +Processing Subnetworks in region: asia-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-east1 in this run. +Processing Subnetworks in region: asia-east2 +No Subnetworks found to delete in asia-east2 in this run. +Processing Subnetworks in region: asia-northeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-northeast1 in this run. +Processing Subnetworks in region: asia-northeast2 +No Subnetworks found to delete in asia-northeast2 in this run. +Processing Subnetworks in region: asia-northeast3 +No Subnetworks found to delete in asia-northeast3 in this run. +Processing Subnetworks in region: asia-south1 +No Subnetworks found to delete in asia-south1 in this run. +Processing Subnetworks in region: asia-south2 +No Subnetworks found to delete in asia-south2 in this run. +Processing Subnetworks in region: asia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in asia-southeast1 are targeted for deletion in this run: +db451c7-ml-slurm-v6-primary-subnet asia-southeast1 +ml-gke-e2e-a8fae6-subnet asia-southeast1 +[EXECUTE] Subnetwork: Deleting db451c7-ml-slurm-v6-primary-subnet in asia-southeast1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/asia-southeast1/subnetworks/db451c7-ml-slurm-v6-primary-subnet]. +[EXECUTE] Subnetwork: Deleting ml-gke-e2e-a8fae6-subnet in asia-southeast1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/asia-southeast1/subnetworks/ml-gke-e2e-a8fae6-subnet]. +Processing Subnetworks in region: asia-southeast2 +No Subnetworks found to delete in asia-southeast2 in this run. +Processing Subnetworks in region: asia-southeast3 +No Subnetworks found to delete in asia-southeast3 in this run. +Processing Subnetworks in region: australia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in australia-southeast1 in this run. +Processing Subnetworks in region: australia-southeast2 +No Subnetworks found to delete in australia-southeast2 in this run. +Processing Subnetworks in region: europe-central2 +No Subnetworks found to delete in europe-central2 in this run. +Processing Subnetworks in region: europe-north1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north1 in this run. +Processing Subnetworks in region: europe-north2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north2 in this run. +Processing Subnetworks in region: europe-southwest1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-southwest1 in this run. +Processing Subnetworks in region: europe-west1 +WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in europe-west1 are targeted for deletion in this run: +cx-a3u-sub-0 europe-west1 +hanu-a3u-primary-subnet europe-west1 +hpc-exr-2-sub-0 europe-west1 +laveeek29-mrdma-sub-0 europe-west1 +laveeek29-mrdma-sub-1 europe-west1 +laveeek29-mrdma-sub-2 europe-west1 +laveeek29-mrdma-sub-3 europe-west1 +laveeek29-mrdma-sub-4 europe-west1 +laveeek29-mrdma-sub-5 europe-west1 +laveeek29-mrdma-sub-6 europe-west1 +[EXECUTE] Subnetwork: Deleting cx-a3u-sub-0 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/cx-a3u-sub-0]. +[EXECUTE] Subnetwork: Deleting hanu-a3u-primary-subnet in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/hanu-a3u-primary-subnet]. +[EXECUTE] Subnetwork: Deleting hpc-exr-2-sub-0 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/hpc-exr-2-sub-0]. +[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-0 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-0]. +[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-1 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-1]. +[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-2 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-2]. +[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-3 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-3]. +[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-4 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-4]. +[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-5 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-5]. +[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-6 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-6]. +Processing Subnetworks in region: europe-west10 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west10 in this run. +Processing Subnetworks in region: europe-west12 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west12 in this run. +Processing Subnetworks in region: europe-west15 +No Subnetworks found to delete in europe-west15 in this run. +Processing Subnetworks in region: europe-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west2 in this run. +Processing Subnetworks in region: europe-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west3 in this run. +Processing Subnetworks in region: europe-west4 +The following Subnetworks in europe-west4 are targeted for deletion in this run: +a4newimgek-primary-subnet europe-west4 +a4oldimgek-mrdma-sub-0 europe-west4 +a4oldimgek-mrdma-sub-1 europe-west4 +a4oldimgek-mrdma-sub-2 europe-west4 +a4oldimgek-mrdma-sub-3 europe-west4 +a4oldimgek-mrdma-sub-4 europe-west4 +a4oldimgek-mrdma-sub-5 europe-west4 +a4oldimgek-mrdma-sub-6 europe-west4 +a4oldimgek-mrdma-sub-7 europe-west4 +a4oldimgek-primary-subnet europe-west4 +[EXECUTE] Subnetwork: Deleting a4newimgek-primary-subnet in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4newimgek-primary-subnet]. +[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-0 in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-0]. +[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-1 in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-1]. +[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-2 in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-2]. +[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-3 in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-3]. +[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-4 in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-4]. +[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-5 in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-5]. +[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-6 in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-6]. +[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-7 in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-7]. +[EXECUTE] Subnetwork: Deleting a4oldimgek-primary-subnet in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-primary-subnet]. +Processing Subnetworks in region: europe-west6 +No Subnetworks found to delete in europe-west6 in this run. +Processing Subnetworks in region: europe-west8 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west8 in this run. +Processing Subnetworks in region: europe-west9 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west9 in this run. +Processing Subnetworks in region: me-central1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central1 in this run. +Processing Subnetworks in region: me-central2 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central2 in this run. +Processing Subnetworks in region: me-west1 +No Subnetworks found to delete in me-west1 in this run. +Processing Subnetworks in region: northamerica-northeast1 +No Subnetworks found to delete in northamerica-northeast1 in this run. +Processing Subnetworks in region: northamerica-northeast2 +No Subnetworks found to delete in northamerica-northeast2 in this run. +Processing Subnetworks in region: northamerica-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in northamerica-south1 in this run. +Processing Subnetworks in region: southamerica-east1 +No Subnetworks found to delete in southamerica-east1 in this run. +Processing Subnetworks in region: southamerica-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in southamerica-west1 in this run. +Processing Subnetworks in region: us-central1 +The following Subnetworks in us-central1 are targeted for deletion in this run: +a4h-slurm-c0e262-primary-subnet us-central1 +a4h-slurm-mrdma-sub-0 us-central1 +a4h-slurm-mrdma-sub-1 us-central1 +a4h-slurm-mrdma-sub-2 us-central1 +a4h-slurm-mrdma-sub-3 us-central1 +a4h-slurm-mrdma-sub-4 us-central1 +a4h-slurm-mrdma-sub-5 us-central1 +a4h-slurm-mrdma-sub-6 us-central1 +a4h-slurm-mrdma-sub-7 us-central1 +a4h-slurm-sub-0 us-central1 +[EXECUTE] Subnetwork: Deleting a4h-slurm-c0e262-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-c0e262-primary-subnet]. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-0 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-0]. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-1 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-1]. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-2 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-2]. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-3 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-3]. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-4 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-4]. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-5 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-5]. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-6 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-6]. +[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-7 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-7]. +[EXECUTE] Subnetwork: Deleting a4h-slurm-sub-0 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-sub-0]. +Processing Subnetworks in region: us-central2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-central2 in this run. +Processing Subnetworks in region: us-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east1 in this run. +Processing Subnetworks in region: us-east4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east4 in this run. +Processing Subnetworks in region: us-east5 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east5 in this run. +Processing Subnetworks in region: us-east7 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east7 in this run. +Processing Subnetworks in region: us-south1 +The following Subnetworks in us-south1 are targeted for deletion in this run: +a3u-slurm-3224ec-primary-subnet us-south1 +a4newimgek-mrdma-sub-0 us-south1 +a4newimgek-mrdma-sub-1 us-south1 +a4newimgek-mrdma-sub-2 us-south1 +a4newimgek-mrdma-sub-3 us-south1 +a4newimgek-mrdma-sub-4 us-south1 +a4newimgek-mrdma-sub-5 us-south1 +a4newimgek-mrdma-sub-6 us-south1 +a4newimgek-mrdma-sub-7 us-south1 +a4newimgek-sub-0 us-south1 +[EXECUTE] Subnetwork: Deleting a3u-slurm-3224ec-primary-subnet in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a3u-slurm-3224ec-primary-subnet]. +[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-0 in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-0]. +[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-1 in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-1]. +[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-2 in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-2]. +[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-3 in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-3]. +[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-4 in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-4]. +[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-5 in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-5]. +[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-6 in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-6]. +[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-7 in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-7]. +[EXECUTE] Subnetwork: Deleting a4newimgek-sub-0 in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-sub-0]. +Processing Subnetworks in region: us-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in us-west1 are targeted for deletion in this run: +slurm-a3-base-sysnet-subnet us-west1 +[EXECUTE] Subnetwork: Deleting slurm-a3-base-sysnet-subnet in us-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west1/subnetworks/slurm-a3-base-sysnet-subnet]. +Processing Subnetworks in region: us-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west2 in this run. +Processing Subnetworks in region: us-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west3 in this run. +Processing Subnetworks in region: us-west4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in us-west4 are targeted for deletion in this run: +a3mega-sys-subnet us-west4 +mglsard-subnet us-west4 +[EXECUTE] Subnetwork: Deleting a3mega-sys-subnet in us-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west4/subnetworks/a3mega-sys-subnet]. +[EXECUTE] Subnetwork: Deleting mglsard-subnet in us-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west4/subnetworks/mglsard-subnet]. +Processing Subnetworks in region: us-west8 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in us-west8 are targeted for deletion in this run: +a4xlavhpcnew-a4x-sub-0 us-west8 +a4xlavhpcnew-a4x-sub-1 us-west8 +a4x-mrdma-sub-0 us-west8 +a4x-mrdma-sub-1 us-west8 +a4x-mrdma-sub-2 us-west8 +a4x-mrdma-sub-3 us-west8 +a4xslurm-primary-subnet us-west8 +[EXECUTE] Subnetwork: Deleting a4xlavhpcnew-a4x-sub-0 in us-west8 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4xlavhpcnew-a4x-sub-0]. +[EXECUTE] Subnetwork: Deleting a4xlavhpcnew-a4x-sub-1 in us-west8 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4xlavhpcnew-a4x-sub-1]. +[EXECUTE] Subnetwork: Deleting a4x-mrdma-sub-0 in us-west8 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4x-mrdma-sub-0]. +[EXECUTE] Subnetwork: Deleting a4x-mrdma-sub-1 in us-west8 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4x-mrdma-sub-1]. +[EXECUTE] Subnetwork: Deleting a4x-mrdma-sub-2 in us-west8 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4x-mrdma-sub-2]. +[EXECUTE] Subnetwork: Deleting a4x-mrdma-sub-3 in us-west8 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4x-mrdma-sub-3]. +[EXECUTE] Subnetwork: Deleting a4xslurm-primary-subnet in us-west8 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4xslurm-primary-subnet]. +--- Thu Nov 27 05:38:32 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 05:38:42 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:38:42+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 10 per region) --- +Processing Subnetworks in region: africa-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in africa-south1 in this run. +Processing Subnetworks in region: asia-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-east1 in this run. +Processing Subnetworks in region: asia-east2 +No Subnetworks found to delete in asia-east2 in this run. +Processing Subnetworks in region: asia-northeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-northeast1 in this run. +Processing Subnetworks in region: asia-northeast2 +No Subnetworks found to delete in asia-northeast2 in this run. +Processing Subnetworks in region: asia-northeast3 +No Subnetworks found to delete in asia-northeast3 in this run. +Processing Subnetworks in region: asia-south1 +No Subnetworks found to delete in asia-south1 in this run. +Processing Subnetworks in region: asia-south2 +No Subnetworks found to delete in asia-south2 in this run. +Processing Subnetworks in region: asia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-southeast1 in this run. +Processing Subnetworks in region: asia-southeast2 +No Subnetworks found to delete in asia-southeast2 in this run. +Processing Subnetworks in region: asia-southeast3 +No Subnetworks found to delete in asia-southeast3 in this run. +Processing Subnetworks in region: australia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in australia-southeast1 in this run. +Processing Subnetworks in region: australia-southeast2 +No Subnetworks found to delete in australia-southeast2 in this run. +Processing Subnetworks in region: europe-central2 +No Subnetworks found to delete in europe-central2 in this run. +Processing Subnetworks in region: europe-north1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north1 in this run. +Processing Subnetworks in region: europe-north2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north2 in this run. +Processing Subnetworks in region: europe-southwest1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-southwest1 in this run. +Processing Subnetworks in region: europe-west1 +WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in europe-west1 are targeted for deletion in this run: +laveeek29-mrdma-sub-7 europe-west1 +laveeek29-primary-subnet europe-west1 +laveeek29-sub-0 europe-west1 +laveeek29-sub-1 europe-west1 +lavoldchk-primary-subnet europe-west1 +lavrohek29-sub-0 europe-west1 +mainek-mrdma-sub-0 europe-west1 +mainek-mrdma-sub-1 europe-west1 +mainek-mrdma-sub-2 europe-west1 +mainek-mrdma-sub-3 europe-west1 +[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-7 in europe-west1 + Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-7" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete laveeek29-primary-subnet in europe-west1 + Command: gcloud compute networks subnets delete "laveeek29-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete laveeek29-sub-0 in europe-west1 + Command: gcloud compute networks subnets delete "laveeek29-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete laveeek29-sub-1 in europe-west1 + Command: gcloud compute networks subnets delete "laveeek29-sub-1" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete lavoldchk-primary-subnet in europe-west1 + Command: gcloud compute networks subnets delete "lavoldchk-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete lavrohek29-sub-0 in europe-west1 + Command: gcloud compute networks subnets delete "lavrohek29-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-0 in europe-west1 + Command: gcloud compute networks subnets delete "mainek-mrdma-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-1 in europe-west1 + Command: gcloud compute networks subnets delete "mainek-mrdma-sub-1" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-2 in europe-west1 + Command: gcloud compute networks subnets delete "mainek-mrdma-sub-2" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-3 in europe-west1 + Command: gcloud compute networks subnets delete "mainek-mrdma-sub-3" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +Processing Subnetworks in region: europe-west10 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west10 in this run. +Processing Subnetworks in region: europe-west12 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west12 in this run. +Processing Subnetworks in region: europe-west15 +No Subnetworks found to delete in europe-west15 in this run. +Processing Subnetworks in region: europe-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west2 in this run. +Processing Subnetworks in region: europe-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west3 in this run. +Processing Subnetworks in region: europe-west4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in europe-west4 are targeted for deletion in this run: +a4oldimgek-sub-0 europe-west4 +a4oldimgek-sub-1 europe-west4 +a4oldimg-primary-subnet europe-west4 +hpcdydis-primary-subnet europe-west4 +hpcdy-primary-subnet europe-west4 +[DRY RUN] Subnetwork: Would delete a4oldimgek-sub-0 in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimgek-sub-0" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete a4oldimgek-sub-1 in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimgek-sub-1" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete a4oldimg-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "a4oldimg-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +Processing Subnetworks in region: europe-west6 +No Subnetworks found to delete in europe-west6 in this run. +Processing Subnetworks in region: europe-west8 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west8 in this run. +Processing Subnetworks in region: europe-west9 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west9 in this run. +Processing Subnetworks in region: me-central1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central1 in this run. +Processing Subnetworks in region: me-central2 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central2 in this run. +Processing Subnetworks in region: me-west1 +No Subnetworks found to delete in me-west1 in this run. +Processing Subnetworks in region: northamerica-northeast1 +No Subnetworks found to delete in northamerica-northeast1 in this run. +Processing Subnetworks in region: northamerica-northeast2 +No Subnetworks found to delete in northamerica-northeast2 in this run. +Processing Subnetworks in region: northamerica-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in northamerica-south1 in this run. +Processing Subnetworks in region: southamerica-east1 +No Subnetworks found to delete in southamerica-east1 in this run. +Processing Subnetworks in region: southamerica-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in southamerica-west1 in this run. +Processing Subnetworks in region: us-central1 +Skip Subnet: default (On default network) +The following Subnetworks in us-central1 are targeted for deletion in this run: +a4h-slurm-sub-1 us-central1 +a4htest-sub-0 us-central1 +dynpoc-primary-subnet us-central1 +g4qclav-primary-subnet us-central1 +gke-1395b4-subnet us-central1 +h4d-cluster-rdma-sub-0 us-central1 +h4dqc-primary-subnet us-central1 +h4dqc-rdma-sub-0 us-central1 +h4d-res-swarnabm4-3-rdma-sub us-central1 +h4d-res-swarnabm4-3-sub us-central1 +[DRY RUN] Subnetwork: Would delete a4h-slurm-sub-1 in us-central1 + Command: gcloud compute networks subnets delete "a4h-slurm-sub-1" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete a4htest-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "a4htest-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete dynpoc-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "dynpoc-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete g4qclav-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "g4qclav-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete gke-1395b4-subnet in us-central1 + Command: gcloud compute networks subnets delete "gke-1395b4-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete h4d-cluster-rdma-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "h4d-cluster-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete h4dqc-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "h4dqc-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete h4dqc-rdma-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "h4dqc-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete h4d-res-swarnabm4-3-rdma-sub in us-central1 + Command: gcloud compute networks subnets delete "h4d-res-swarnabm4-3-rdma-sub" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete h4d-res-swarnabm4-3-sub in us-central1 + Command: gcloud compute networks subnets delete "h4d-res-swarnabm4-3-sub" --project="hpc-toolkit-dev" --region="us-central1" --quiet +Processing Subnetworks in region: us-central2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-central2 in this run. +Processing Subnetworks in region: us-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east1 in this run. +Processing Subnetworks in region: us-east4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east4 in this run. +Processing Subnetworks in region: us-east5 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east5 in this run. +Processing Subnetworks in region: us-east7 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east7 in this run. +Processing Subnetworks in region: us-south1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in us-south1 are targeted for deletion in this run: +a4newimgek-sub-1 us-south1 +hanu-test-subnet us-south1 +[DRY RUN] Subnetwork: Would delete a4newimgek-sub-1 in us-south1 + Command: gcloud compute networks subnets delete "a4newimgek-sub-1" --project="hpc-toolkit-dev" --region="us-south1" --quiet +[DRY RUN] Subnetwork: Would delete hanu-test-subnet in us-south1 + Command: gcloud compute networks subnets delete "hanu-test-subnet" --project="hpc-toolkit-dev" --region="us-south1" --quiet +Processing Subnetworks in region: us-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west1 in this run. +Processing Subnetworks in region: us-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west2 in this run. +Processing Subnetworks in region: us-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west3 in this run. +Processing Subnetworks in region: us-west4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west4 in this run. +Processing Subnetworks in region: us-west8 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west8 in this run. +--- Thu Nov 27 05:40:32 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 05:41:03 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:41:03+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 10 per region) --- +Processing Subnetworks in region: africa-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in africa-south1 in this run. +Processing Subnetworks in region: asia-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-east1 in this run. +Processing Subnetworks in region: asia-east2 +No Subnetworks found to delete in asia-east2 in this run. +Processing Subnetworks in region: asia-northeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-northeast1 in this run. +Processing Subnetworks in region: asia-northeast2 +No Subnetworks found to delete in asia-northeast2 in this run. +Processing Subnetworks in region: asia-northeast3 +No Subnetworks found to delete in asia-northeast3 in this run. +Processing Subnetworks in region: asia-south1 +No Subnetworks found to delete in asia-south1 in this run. +Processing Subnetworks in region: asia-south2 +No Subnetworks found to delete in asia-south2 in this run. +Processing Subnetworks in region: asia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-southeast1 in this run. +Processing Subnetworks in region: asia-southeast2 +No Subnetworks found to delete in asia-southeast2 in this run. +Processing Subnetworks in region: asia-southeast3 +No Subnetworks found to delete in asia-southeast3 in this run. +Processing Subnetworks in region: australia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in australia-southeast1 in this run. +Processing Subnetworks in region: australia-southeast2 +No Subnetworks found to delete in australia-southeast2 in this run. +Processing Subnetworks in region: europe-central2 +No Subnetworks found to delete in europe-central2 in this run. +Processing Subnetworks in region: europe-north1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north1 in this run. +Processing Subnetworks in region: europe-north2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north2 in this run. +Processing Subnetworks in region: europe-southwest1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-southwest1 in this run. +Processing Subnetworks in region: europe-west1 +WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in europe-west1 are targeted for deletion in this run: +laveeek29-mrdma-sub-7 europe-west1 +laveeek29-primary-subnet europe-west1 +laveeek29-sub-0 europe-west1 +laveeek29-sub-1 europe-west1 +lavoldchk-primary-subnet europe-west1 +lavrohek29-sub-0 europe-west1 +mainek-mrdma-sub-0 europe-west1 +mainek-mrdma-sub-1 europe-west1 +mainek-mrdma-sub-2 europe-west1 +mainek-mrdma-sub-3 europe-west1 +[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-7 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-7]. +[EXECUTE] Subnetwork: Deleting laveeek29-primary-subnet in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-primary-subnet]. +[EXECUTE] Subnetwork: Deleting laveeek29-sub-0 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-sub-0]. +[EXECUTE] Subnetwork: Deleting laveeek29-sub-1 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-sub-1]. +[EXECUTE] Subnetwork: Deleting lavoldchk-primary-subnet in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/lavoldchk-primary-subnet]. +[EXECUTE] Subnetwork: Deleting lavrohek29-sub-0 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/lavrohek29-sub-0]. +[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-0 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-0]. +[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-1 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-1]. +[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-2 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-2]. +[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-3 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-3]. +Processing Subnetworks in region: europe-west10 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west10 in this run. +Processing Subnetworks in region: europe-west12 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west12 in this run. +Processing Subnetworks in region: europe-west15 +No Subnetworks found to delete in europe-west15 in this run. +Processing Subnetworks in region: europe-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west2 in this run. +Processing Subnetworks in region: europe-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west3 in this run. +Processing Subnetworks in region: europe-west4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in europe-west4 are targeted for deletion in this run: +a4oldimgek-sub-0 europe-west4 +a4oldimgek-sub-1 europe-west4 +a4oldimg-primary-subnet europe-west4 +hpcdydis-primary-subnet europe-west4 +hpcdy-primary-subnet europe-west4 +[EXECUTE] Subnetwork: Deleting a4oldimgek-sub-0 in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-sub-0]. +[EXECUTE] Subnetwork: Deleting a4oldimgek-sub-1 in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-sub-1]. +[EXECUTE] Subnetwork: Deleting a4oldimg-primary-subnet in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimg-primary-subnet]. +[EXECUTE] Subnetwork: Deleting hpcdydis-primary-subnet in europe-west4 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdydis-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-mds0-internal-address' + +[EXECUTE] Subnetwork: Deleting hpcdy-primary-subnet in europe-west4 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdy-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss2-internal-address' + +Processing Subnetworks in region: europe-west6 +No Subnetworks found to delete in europe-west6 in this run. +Processing Subnetworks in region: europe-west8 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west8 in this run. +Processing Subnetworks in region: europe-west9 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west9 in this run. +Processing Subnetworks in region: me-central1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central1 in this run. +Processing Subnetworks in region: me-central2 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central2 in this run. +Processing Subnetworks in region: me-west1 +No Subnetworks found to delete in me-west1 in this run. +Processing Subnetworks in region: northamerica-northeast1 +No Subnetworks found to delete in northamerica-northeast1 in this run. +Processing Subnetworks in region: northamerica-northeast2 +No Subnetworks found to delete in northamerica-northeast2 in this run. +Processing Subnetworks in region: northamerica-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in northamerica-south1 in this run. +Processing Subnetworks in region: southamerica-east1 +No Subnetworks found to delete in southamerica-east1 in this run. +Processing Subnetworks in region: southamerica-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in southamerica-west1 in this run. +Processing Subnetworks in region: us-central1 +Skip Subnet: default (On default network) +The following Subnetworks in us-central1 are targeted for deletion in this run: +a4h-slurm-sub-1 us-central1 +a4htest-sub-0 us-central1 +dynpoc-primary-subnet us-central1 +g4qclav-primary-subnet us-central1 +gke-1395b4-subnet us-central1 +h4d-cluster-rdma-sub-0 us-central1 +h4dqc-primary-subnet us-central1 +h4dqc-rdma-sub-0 us-central1 +h4d-res-swarnabm4-3-rdma-sub us-central1 +h4d-res-swarnabm4-3-sub us-central1 +[EXECUTE] Subnetwork: Deleting a4h-slurm-sub-1 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-sub-1]. +[EXECUTE] Subnetwork: Deleting a4htest-sub-0 in us-central1 +--- Thu Nov 27 05:50:14 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:50:14+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 10 per region) --- +Processing Subnetworks in region: africa-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in africa-south1 in this run. +Processing Subnetworks in region: asia-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-east1 in this run. +Processing Subnetworks in region: asia-east2 +No Subnetworks found to delete in asia-east2 in this run. +Processing Subnetworks in region: asia-northeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-northeast1 in this run. +Processing Subnetworks in region: asia-northeast2 +No Subnetworks found to delete in asia-northeast2 in this run. +Processing Subnetworks in region: asia-northeast3 +No Subnetworks found to delete in asia-northeast3 in this run. +Processing Subnetworks in region: asia-south1 +No Subnetworks found to delete in asia-south1 in this run. +Processing Subnetworks in region: asia-south2 +No Subnetworks found to delete in asia-south2 in this run. +Processing Subnetworks in region: asia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-southeast1 in this run. +Processing Subnetworks in region: asia-southeast2 +No Subnetworks found to delete in asia-southeast2 in this run. +Processing Subnetworks in region: asia-southeast3 +No Subnetworks found to delete in asia-southeast3 in this run. +Processing Subnetworks in region: australia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in australia-southeast1 in this run. +Processing Subnetworks in region: australia-southeast2 +No Subnetworks found to delete in australia-southeast2 in this run. +Processing Subnetworks in region: europe-central2 +No Subnetworks found to delete in europe-central2 in this run. +Processing Subnetworks in region: europe-north1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north1 in this run. +Processing Subnetworks in region: europe-north2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north2 in this run. +Processing Subnetworks in region: europe-southwest1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-southwest1 in this run. +Processing Subnetworks in region: europe-west1 +WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in europe-west1 are targeted for deletion in this run: +mainek-mrdma-sub-4 europe-west1 +mainek-mrdma-sub-5 europe-west1 +mainek-mrdma-sub-6 europe-west1 +mainek-mrdma-sub-7 europe-west1 +mainek-primary-subnet europe-west1 +mainek-sub-0 europe-west1 +mainek-sub-1 europe-west1 +sa-chs-ops-sub-0 europe-west1 +sispot3u-sub-0 europe-west1 +[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-4 in europe-west1 + Command: gcloud compute networks subnets delete "mainek-mrdma-sub-4" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-5 in europe-west1 + Command: gcloud compute networks subnets delete "mainek-mrdma-sub-5" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-6 in europe-west1 + Command: gcloud compute networks subnets delete "mainek-mrdma-sub-6" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-7 in europe-west1 + Command: gcloud compute networks subnets delete "mainek-mrdma-sub-7" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete mainek-primary-subnet in europe-west1 + Command: gcloud compute networks subnets delete "mainek-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete mainek-sub-0 in europe-west1 + Command: gcloud compute networks subnets delete "mainek-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete mainek-sub-1 in europe-west1 + Command: gcloud compute networks subnets delete "mainek-sub-1" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete sa-chs-ops-sub-0 in europe-west1 + Command: gcloud compute networks subnets delete "sa-chs-ops-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +[DRY RUN] Subnetwork: Would delete sispot3u-sub-0 in europe-west1 + Command: gcloud compute networks subnets delete "sispot3u-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet +Processing Subnetworks in region: europe-west10 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west10 in this run. +Processing Subnetworks in region: europe-west12 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west12 in this run. +Processing Subnetworks in region: europe-west15 +No Subnetworks found to delete in europe-west15 in this run. +Processing Subnetworks in region: europe-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west2 in this run. +Processing Subnetworks in region: europe-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west3 in this run. +Processing Subnetworks in region: europe-west4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in europe-west4 are targeted for deletion in this run: +hpcdydis-primary-subnet europe-west4 +hpcdy-primary-subnet europe-west4 +[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +Processing Subnetworks in region: europe-west6 +No Subnetworks found to delete in europe-west6 in this run. +Processing Subnetworks in region: europe-west8 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west8 in this run. +Processing Subnetworks in region: europe-west9 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west9 in this run. +Processing Subnetworks in region: me-central1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central1 in this run. +Processing Subnetworks in region: me-central2 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central2 in this run. +Processing Subnetworks in region: me-west1 +No Subnetworks found to delete in me-west1 in this run. +Processing Subnetworks in region: northamerica-northeast1 +No Subnetworks found to delete in northamerica-northeast1 in this run. +Processing Subnetworks in region: northamerica-northeast2 +No Subnetworks found to delete in northamerica-northeast2 in this run. +Processing Subnetworks in region: northamerica-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in northamerica-south1 in this run. +Processing Subnetworks in region: southamerica-east1 +No Subnetworks found to delete in southamerica-east1 in this run. +Processing Subnetworks in region: southamerica-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in southamerica-west1 in this run. +Processing Subnetworks in region: us-central1 +Skip Subnet: default (On default network) +The following Subnetworks in us-central1 are targeted for deletion in this run: +dynpoc-primary-subnet us-central1 +g4qclav-primary-subnet us-central1 +gke-1395b4-subnet us-central1 +h4d-cluster-rdma-sub-0 us-central1 +h4dqc-primary-subnet us-central1 +h4dqc-rdma-sub-0 us-central1 +h4d-res-swarnabm4-3-rdma-sub us-central1 +h4d-res-swarnabm4-3-sub us-central1 +hpc-01-primary-subnet us-central1 +hpcimg-primary-subnet us-central1 +[DRY RUN] Subnetwork: Would delete dynpoc-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "dynpoc-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete g4qclav-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "g4qclav-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete gke-1395b4-subnet in us-central1 + Command: gcloud compute networks subnets delete "gke-1395b4-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete h4d-cluster-rdma-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "h4d-cluster-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete h4dqc-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "h4dqc-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete h4dqc-rdma-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "h4dqc-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete h4d-res-swarnabm4-3-rdma-sub in us-central1 + Command: gcloud compute networks subnets delete "h4d-res-swarnabm4-3-rdma-sub" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete h4d-res-swarnabm4-3-sub in us-central1 + Command: gcloud compute networks subnets delete "h4d-res-swarnabm4-3-sub" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete hpc-01-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "hpc-01-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete hpcimg-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "hpcimg-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +Processing Subnetworks in region: us-central2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-central2 in this run. +Processing Subnetworks in region: us-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east1 in this run. +Processing Subnetworks in region: us-east4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east4 in this run. +Processing Subnetworks in region: us-east5 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east5 in this run. +Processing Subnetworks in region: us-east7 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east7 in this run. +Processing Subnetworks in region: us-south1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in us-south1 are targeted for deletion in this run: +a4newimgek-sub-1 us-south1 +hanu-test-subnet us-south1 +[DRY RUN] Subnetwork: Would delete a4newimgek-sub-1 in us-south1 + Command: gcloud compute networks subnets delete "a4newimgek-sub-1" --project="hpc-toolkit-dev" --region="us-south1" --quiet +[DRY RUN] Subnetwork: Would delete hanu-test-subnet in us-south1 + Command: gcloud compute networks subnets delete "hanu-test-subnet" --project="hpc-toolkit-dev" --region="us-south1" --quiet +Processing Subnetworks in region: us-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west1 in this run. +Processing Subnetworks in region: us-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west2 in this run. +Processing Subnetworks in region: us-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west3 in this run. +Processing Subnetworks in region: us-west4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west4 in this run. +Processing Subnetworks in region: us-west8 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west8 in this run. +--- Thu Nov 27 05:52:26 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 05:52:57 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T01:52:57+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 10 per region) --- +Processing Subnetworks in region: africa-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in africa-south1 in this run. +Processing Subnetworks in region: asia-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-east1 in this run. +Processing Subnetworks in region: asia-east2 +No Subnetworks found to delete in asia-east2 in this run. +Processing Subnetworks in region: asia-northeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-northeast1 in this run. +Processing Subnetworks in region: asia-northeast2 +No Subnetworks found to delete in asia-northeast2 in this run. +Processing Subnetworks in region: asia-northeast3 +No Subnetworks found to delete in asia-northeast3 in this run. +Processing Subnetworks in region: asia-south1 +No Subnetworks found to delete in asia-south1 in this run. +Processing Subnetworks in region: asia-south2 +No Subnetworks found to delete in asia-south2 in this run. +Processing Subnetworks in region: asia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-southeast1 in this run. +Processing Subnetworks in region: asia-southeast2 +No Subnetworks found to delete in asia-southeast2 in this run. +Processing Subnetworks in region: asia-southeast3 +No Subnetworks found to delete in asia-southeast3 in this run. +Processing Subnetworks in region: australia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in australia-southeast1 in this run. +Processing Subnetworks in region: australia-southeast2 +No Subnetworks found to delete in australia-southeast2 in this run. +Processing Subnetworks in region: europe-central2 +No Subnetworks found to delete in europe-central2 in this run. +Processing Subnetworks in region: europe-north1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north1 in this run. +Processing Subnetworks in region: europe-north2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north2 in this run. +Processing Subnetworks in region: europe-southwest1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-southwest1 in this run. +Processing Subnetworks in region: europe-west1 +WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in europe-west1 are targeted for deletion in this run: +mainek-mrdma-sub-4 europe-west1 +mainek-mrdma-sub-5 europe-west1 +mainek-mrdma-sub-6 europe-west1 +mainek-mrdma-sub-7 europe-west1 +mainek-primary-subnet europe-west1 +mainek-sub-0 europe-west1 +mainek-sub-1 europe-west1 +sa-chs-ops-sub-0 europe-west1 +sispot3u-sub-0 europe-west1 +[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-4 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-4]. +[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-5 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-5]. +[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-6 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-6]. +[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-7 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-7]. +[EXECUTE] Subnetwork: Deleting mainek-primary-subnet in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-primary-subnet]. +[EXECUTE] Subnetwork: Deleting mainek-sub-0 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-sub-0]. +[EXECUTE] Subnetwork: Deleting mainek-sub-1 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-sub-1]. +[EXECUTE] Subnetwork: Deleting sa-chs-ops-sub-0 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/sa-chs-ops-sub-0]. +[EXECUTE] Subnetwork: Deleting sispot3u-sub-0 in europe-west1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/sispot3u-sub-0]. +Processing Subnetworks in region: europe-west10 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west10 in this run. +Processing Subnetworks in region: europe-west12 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west12 in this run. +Processing Subnetworks in region: europe-west15 +No Subnetworks found to delete in europe-west15 in this run. +Processing Subnetworks in region: europe-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west2 in this run. +Processing Subnetworks in region: europe-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west3 in this run. +Processing Subnetworks in region: europe-west4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in europe-west4 are targeted for deletion in this run: +hpcdydis-primary-subnet europe-west4 +hpcdy-primary-subnet europe-west4 +[EXECUTE] Subnetwork: Deleting hpcdydis-primary-subnet in europe-west4 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdydis-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-mds0-internal-address' + +[EXECUTE] Subnetwork: Deleting hpcdy-primary-subnet in europe-west4 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdy-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss2-internal-address' + +Processing Subnetworks in region: europe-west6 +No Subnetworks found to delete in europe-west6 in this run. +Processing Subnetworks in region: europe-west8 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west8 in this run. +Processing Subnetworks in region: europe-west9 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west9 in this run. +Processing Subnetworks in region: me-central1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central1 in this run. +Processing Subnetworks in region: me-central2 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central2 in this run. +Processing Subnetworks in region: me-west1 +No Subnetworks found to delete in me-west1 in this run. +Processing Subnetworks in region: northamerica-northeast1 +No Subnetworks found to delete in northamerica-northeast1 in this run. +Processing Subnetworks in region: northamerica-northeast2 +No Subnetworks found to delete in northamerica-northeast2 in this run. +Processing Subnetworks in region: northamerica-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in northamerica-south1 in this run. +Processing Subnetworks in region: southamerica-east1 +No Subnetworks found to delete in southamerica-east1 in this run. +Processing Subnetworks in region: southamerica-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in southamerica-west1 in this run. +Processing Subnetworks in region: us-central1 +Skip Subnet: default (On default network) +The following Subnetworks in us-central1 are targeted for deletion in this run: +dynpoc-primary-subnet us-central1 +g4qclav-primary-subnet us-central1 +gke-1395b4-subnet us-central1 +h4d-cluster-rdma-sub-0 us-central1 +h4dqc-primary-subnet us-central1 +h4dqc-rdma-sub-0 us-central1 +h4d-res-swarnabm4-3-rdma-sub us-central1 +h4d-res-swarnabm4-3-sub us-central1 +hpc-01-primary-subnet us-central1 +hpcimg-primary-subnet us-central1 +[EXECUTE] Subnetwork: Deleting dynpoc-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/dynpoc-primary-subnet]. +[EXECUTE] Subnetwork: Deleting g4qclav-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/g4qclav-primary-subnet]. +[EXECUTE] Subnetwork: Deleting gke-1395b4-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/gke-1395b4-subnet]. +[EXECUTE] Subnetwork: Deleting h4d-cluster-rdma-sub-0 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/h4d-cluster-rdma-sub-0]. +[EXECUTE] Subnetwork: Deleting h4dqc-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/h4dqc-primary-subnet]. +[EXECUTE] Subnetwork: Deleting h4dqc-rdma-sub-0 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/h4dqc-rdma-sub-0]. +[EXECUTE] Subnetwork: Deleting h4d-res-swarnabm4-3-rdma-sub in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/h4d-res-swarnabm4-3-rdma-sub]. +[EXECUTE] Subnetwork: Deleting h4d-res-swarnabm4-3-sub in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/h4d-res-swarnabm4-3-sub]. +[EXECUTE] Subnetwork: Deleting hpc-01-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/hpc-01-primary-subnet]. +[EXECUTE] Subnetwork: Deleting hpcimg-primary-subnet in us-central1 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/hpcimg-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-oss1-internal-address' + +Processing Subnetworks in region: us-central2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-central2 in this run. +Processing Subnetworks in region: us-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east1 in this run. +Processing Subnetworks in region: us-east4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east4 in this run. +Processing Subnetworks in region: us-east5 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east5 in this run. +Processing Subnetworks in region: us-east7 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east7 in this run. +Processing Subnetworks in region: us-south1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in us-south1 are targeted for deletion in this run: +a4newimgek-sub-1 us-south1 +hanu-test-subnet us-south1 +[EXECUTE] Subnetwork: Deleting a4newimgek-sub-1 in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-sub-1]. +[EXECUTE] Subnetwork: Deleting hanu-test-subnet in us-south1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/hanu-test-subnet]. +Processing Subnetworks in region: us-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west1 in this run. +Processing Subnetworks in region: us-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west2 in this run. +Processing Subnetworks in region: us-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west3 in this run. +Processing Subnetworks in region: us-west4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west4 in this run. +Processing Subnetworks in region: us-west8 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west8 in this run. +--- Thu Nov 27 06:00:45 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 09:31:39 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T05:31:39+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 10 per region) --- +Processing Subnetworks in region: africa-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in africa-south1 in this run. +Processing Subnetworks in region: asia-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-east1 in this run. +Processing Subnetworks in region: asia-east2 +No Subnetworks found to delete in asia-east2 in this run. +Processing Subnetworks in region: asia-northeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-northeast1 in this run. +Processing Subnetworks in region: asia-northeast2 +No Subnetworks found to delete in asia-northeast2 in this run. +Processing Subnetworks in region: asia-northeast3 +No Subnetworks found to delete in asia-northeast3 in this run. +Processing Subnetworks in region: asia-south1 +No Subnetworks found to delete in asia-south1 in this run. +Processing Subnetworks in region: asia-south2 +No Subnetworks found to delete in asia-south2 in this run. +Processing Subnetworks in region: asia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in asia-southeast1 in this run. +Processing Subnetworks in region: asia-southeast2 +No Subnetworks found to delete in asia-southeast2 in this run. +Processing Subnetworks in region: asia-southeast3 +No Subnetworks found to delete in asia-southeast3 in this run. +Processing Subnetworks in region: australia-southeast1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in australia-southeast1 in this run. +Processing Subnetworks in region: australia-southeast2 +No Subnetworks found to delete in australia-southeast2 in this run. +Processing Subnetworks in region: europe-central2 +No Subnetworks found to delete in europe-central2 in this run. +Processing Subnetworks in region: europe-north1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north1 in this run. +Processing Subnetworks in region: europe-north2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-north2 in this run. +Processing Subnetworks in region: europe-southwest1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-southwest1 in this run. +Processing Subnetworks in region: europe-west1 +WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west1 in this run. +Processing Subnetworks in region: europe-west10 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west10 in this run. +Processing Subnetworks in region: europe-west12 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west12 in this run. +Processing Subnetworks in region: europe-west15 +No Subnetworks found to delete in europe-west15 in this run. +Processing Subnetworks in region: europe-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west2 in this run. +Processing Subnetworks in region: europe-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in europe-west3 in this run. +Processing Subnetworks in region: europe-west4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in europe-west4 are targeted for deletion in this run: +hpcdydis-primary-subnet europe-west4 +hpcdy-primary-subnet europe-west4 +[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +Processing Subnetworks in region: europe-west6 +No Subnetworks found to delete in europe-west6 in this run. +Processing Subnetworks in region: europe-west8 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west8 in this run. +Processing Subnetworks in region: europe-west9 +Skip Subnet: default (On default network) +No Subnetworks found to delete in europe-west9 in this run. +Processing Subnetworks in region: me-central1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central1 in this run. +Processing Subnetworks in region: me-central2 +Skip Subnet: default (On default network) +No Subnetworks found to delete in me-central2 in this run. +Processing Subnetworks in region: me-west1 +No Subnetworks found to delete in me-west1 in this run. +Processing Subnetworks in region: northamerica-northeast1 +No Subnetworks found to delete in northamerica-northeast1 in this run. +Processing Subnetworks in region: northamerica-northeast2 +No Subnetworks found to delete in northamerica-northeast2 in this run. +Processing Subnetworks in region: northamerica-south1 +Skip Subnet: default (On default network) +No Subnetworks found to delete in northamerica-south1 in this run. +Processing Subnetworks in region: southamerica-east1 +No Subnetworks found to delete in southamerica-east1 in this run. +Processing Subnetworks in region: southamerica-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in southamerica-west1 in this run. +Processing Subnetworks in region: us-central1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +The following Subnetworks in us-central1 are targeted for deletion in this run: +hpcimg-primary-subnet us-central1 +hpc-lustre-test-02-primary-subnet us-central1 +khu-h4d-cluster-test-primary-subnet us-central1 +khu-h4d-cluster-test-rdma-sub-0 us-central1 +lustre-06-primary-subnet us-central1 +lustre-test-06-primary-subnet us-central1 +managed-lustre-03-primary-subnet us-central1 +ml-gke-subnet us-central1 +monitoring-8323fe-primary-subnet us-central1 +sarthakagrr-primary-subnet us-central1 +[DRY RUN] Subnetwork: Would delete hpcimg-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "hpcimg-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete hpc-lustre-test-02-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "hpc-lustre-test-02-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-rdma-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete managed-lustre-03-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "managed-lustre-03-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete ml-gke-subnet in us-central1 + Command: gcloud compute networks subnets delete "ml-gke-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete monitoring-8323fe-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "monitoring-8323fe-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete sarthakagrr-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "sarthakagrr-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +Processing Subnetworks in region: us-central2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-central2 in this run. +Processing Subnetworks in region: us-east1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east1 in this run. +Processing Subnetworks in region: us-east4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east4 in this run. +Processing Subnetworks in region: us-east5 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east5 in this run. +Processing Subnetworks in region: us-east7 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-east7 in this run. +Processing Subnetworks in region: us-south1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-south1 in this run. +Processing Subnetworks in region: us-west1 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west1 in this run. +Processing Subnetworks in region: us-west2 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west2 in this run. +Processing Subnetworks in region: us-west3 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west3 in this run. +Processing Subnetworks in region: us-west4 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west4 in this run. +Processing Subnetworks in region: us-west8 +Skip Subnet: default (On default network) +Skip Subnet: hpc-vpc (In exclusion list) +No Subnetworks found to delete in us-west8 in this run. +--- Thu Nov 27 09:33:27 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 09:37:08 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 10 resources of each type per run. +Targeting resources created before: 2025-11-27T05:37:08+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 10) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 10) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 10) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 10) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 10) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 10 across all regions) --- +Skip Subnet: default in africa-south1 (On default network) +Skip Subnet: default in asia-east1 (On default network) +Skip Subnet: default in asia-northeast1 (On default network) +Skip Subnet: default in asia-southeast1 (On default network) +Skip Subnet: default in australia-southeast1 (On default network) +Skip Subnet: default in europe-north1 (On default network) +Skip Subnet: default in europe-north2 (On default network) +Skip Subnet: default in europe-southwest1 (On default network) +Skip Subnet: default in europe-west10 (On default network) +Skip Subnet: default in europe-west12 (On default network) +Skip Subnet: default in europe-west1 (On default network) +Skip Subnet: default in europe-west2 (On default network) +Skip Subnet: default in europe-west3 (On default network) +Skip Subnet: default in europe-west4 (On default network) +Skip Subnet: default in europe-west8 (On default network) +Skip Subnet: default in europe-west9 (On default network) +Skip Subnet: default in me-central1 (On default network) +Skip Subnet: default in me-central2 (On default network) +Skip Subnet: default in northamerica-south1 (On default network) +Skip Subnet: default in southamerica-west1 (On default network) +Skip Subnet: default in us-central1 (On default network) +Skip Subnet: default in us-central2 (On default network) +Skip Subnet: default in us-east1 (On default network) +Skip Subnet: default in us-east4 (On default network) +Skip Subnet: default in us-east5 (On default network) +Skip Subnet: default in us-east7 (On default network) +Skip Subnet: default in us-south1 (On default network) +Skip Subnet: default in us-west1 (On default network) +Skip Subnet: default in us-west2 (On default network) +Skip Subnet: default in us-west3 (On default network) +Skip Subnet: default in us-west4 (On default network) +Skip Subnet: default in us-west8 (On default network) +Skip Subnet: hpc-vpc in asia-east1 (In exclusion list) +Skip Subnet: hpc-vpc in asia-northeast1 (In exclusion list) +Skip Subnet: hpc-vpc in asia-southeast1 (In exclusion list) +Skip Subnet: hpc-vpc in australia-southeast1 (In exclusion list) +Skip Subnet: hpc-vpc in europe-north1 (In exclusion list) +Skip Subnet: hpc-vpc in europe-north2 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west1 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west2 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west3 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west4 (In exclusion list) +Skip Subnet: hpc-vpc in southamerica-west1 (In exclusion list) +Skip Subnet: hpc-vpc in us-central1 (In exclusion list) +Skip Subnet: hpc-vpc in us-central2 (In exclusion list) +Skip Subnet: hpc-vpc in us-east1 (In exclusion list) +Skip Subnet: hpc-vpc in us-east4 (In exclusion list) +Skip Subnet: hpc-vpc in us-east5 (In exclusion list) +Skip Subnet: hpc-vpc in us-east7 (In exclusion list) +Skip Subnet: hpc-vpc in us-south1 (In exclusion list) +Skip Subnet: hpc-vpc in us-west1 (In exclusion list) +Skip Subnet: hpc-vpc in us-west2 (In exclusion list) +Skip Subnet: hpc-vpc in us-west3 (In exclusion list) +Skip Subnet: hpc-vpc in us-west4 (In exclusion list) +Skip Subnet: hpc-vpc in us-west8 (In exclusion list) +The following Subnetworks are targeted for deletion in this run: +hpcdydis-primary-subnet europe-west4 +hpcdy-primary-subnet europe-west4 +hpcimg-primary-subnet us-central1 +hpc-lustre-test-02-primary-subnet us-central1 +khu-h4d-cluster-test-primary-subnet us-central1 +khu-h4d-cluster-test-rdma-sub-0 us-central1 +lustre-06-primary-subnet us-central1 +lustre-test-06-primary-subnet us-central1 +managed-lustre-03-primary-subnet us-central1 +ml-gke-subnet us-central1 +[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete hpcimg-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "hpcimg-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete hpc-lustre-test-02-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "hpc-lustre-test-02-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-rdma-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete managed-lustre-03-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "managed-lustre-03-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete ml-gke-subnet in us-central1 + Command: gcloud compute networks subnets delete "ml-gke-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Subnetwork Deletion Process Complete --- +--- Thu Nov 27 09:37:20 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 09:37:41 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T05:37:41+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +Skip Subnet: default in africa-south1 (On default network) +Skip Subnet: default in asia-east1 (On default network) +Skip Subnet: default in asia-northeast1 (On default network) +Skip Subnet: default in asia-southeast1 (On default network) +Skip Subnet: default in australia-southeast1 (On default network) +Skip Subnet: default in europe-north1 (On default network) +Skip Subnet: default in europe-north2 (On default network) +Skip Subnet: default in europe-southwest1 (On default network) +Skip Subnet: default in europe-west10 (On default network) +Skip Subnet: default in europe-west12 (On default network) +Skip Subnet: default in europe-west1 (On default network) +Skip Subnet: default in europe-west2 (On default network) +Skip Subnet: default in europe-west3 (On default network) +Skip Subnet: default in europe-west4 (On default network) +Skip Subnet: default in europe-west8 (On default network) +Skip Subnet: default in europe-west9 (On default network) +Skip Subnet: default in me-central1 (On default network) +Skip Subnet: default in me-central2 (On default network) +Skip Subnet: default in northamerica-south1 (On default network) +Skip Subnet: default in southamerica-west1 (On default network) +Skip Subnet: default in us-central1 (On default network) +Skip Subnet: default in us-central2 (On default network) +Skip Subnet: default in us-east1 (On default network) +Skip Subnet: default in us-east4 (On default network) +Skip Subnet: default in us-east5 (On default network) +Skip Subnet: default in us-east7 (On default network) +Skip Subnet: default in us-south1 (On default network) +Skip Subnet: default in us-west1 (On default network) +Skip Subnet: default in us-west2 (On default network) +Skip Subnet: default in us-west3 (On default network) +Skip Subnet: default in us-west4 (On default network) +Skip Subnet: default in us-west8 (On default network) +Skip Subnet: hpc-vpc in asia-east1 (In exclusion list) +Skip Subnet: hpc-vpc in asia-northeast1 (In exclusion list) +Skip Subnet: hpc-vpc in asia-southeast1 (In exclusion list) +Skip Subnet: hpc-vpc in australia-southeast1 (In exclusion list) +Skip Subnet: hpc-vpc in europe-north1 (In exclusion list) +Skip Subnet: hpc-vpc in europe-north2 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west1 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west2 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west3 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west4 (In exclusion list) +Skip Subnet: hpc-vpc in southamerica-west1 (In exclusion list) +Skip Subnet: hpc-vpc in us-central1 (In exclusion list) +Skip Subnet: hpc-vpc in us-central2 (In exclusion list) +Skip Subnet: hpc-vpc in us-east1 (In exclusion list) +Skip Subnet: hpc-vpc in us-east4 (In exclusion list) +Skip Subnet: hpc-vpc in us-east5 (In exclusion list) +Skip Subnet: hpc-vpc in us-east7 (In exclusion list) +Skip Subnet: hpc-vpc in us-south1 (In exclusion list) +Skip Subnet: hpc-vpc in us-west1 (In exclusion list) +Skip Subnet: hpc-vpc in us-west2 (In exclusion list) +Skip Subnet: hpc-vpc in us-west3 (In exclusion list) +Skip Subnet: hpc-vpc in us-west4 (In exclusion list) +Skip Subnet: hpc-vpc in us-west8 (In exclusion list) +The following Subnetworks are targeted for deletion in this run: +hpcdydis-primary-subnet europe-west4 +hpcdy-primary-subnet europe-west4 +hpcimg-primary-subnet us-central1 +hpc-lustre-test-02-primary-subnet us-central1 +khu-h4d-cluster-test-primary-subnet us-central1 +khu-h4d-cluster-test-rdma-sub-0 us-central1 +lustre-06-primary-subnet us-central1 +lustre-test-06-primary-subnet us-central1 +managed-lustre-03-primary-subnet us-central1 +ml-gke-subnet us-central1 +monitoring-8323fe-primary-subnet us-central1 +sarthakagrr-primary-subnet us-central1 +sp-helmtest1-subnet us-central1 +static-sarthakag-primary-subnet us-central1 +[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete hpcimg-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "hpcimg-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete hpc-lustre-test-02-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "hpc-lustre-test-02-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-rdma-sub-0 in us-central1 + Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete managed-lustre-03-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "managed-lustre-03-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete ml-gke-subnet in us-central1 + Command: gcloud compute networks subnets delete "ml-gke-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete monitoring-8323fe-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "monitoring-8323fe-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete sarthakagrr-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "sarthakagrr-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete sp-helmtest1-subnet in us-central1 + Command: gcloud compute networks subnets delete "sp-helmtest1-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete static-sarthakag-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "static-sarthakag-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Subnetwork Deletion Process Complete --- +--- Thu Nov 27 09:37:53 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 09:38:17 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T05:38:17+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +Skip Subnet: default in africa-south1 (On default network) +Skip Subnet: default in asia-east1 (On default network) +Skip Subnet: default in asia-northeast1 (On default network) +Skip Subnet: default in asia-southeast1 (On default network) +Skip Subnet: default in australia-southeast1 (On default network) +Skip Subnet: default in europe-north1 (On default network) +Skip Subnet: default in europe-north2 (On default network) +Skip Subnet: default in europe-southwest1 (On default network) +Skip Subnet: default in europe-west10 (On default network) +Skip Subnet: default in europe-west12 (On default network) +Skip Subnet: default in europe-west1 (On default network) +Skip Subnet: default in europe-west2 (On default network) +Skip Subnet: default in europe-west3 (On default network) +Skip Subnet: default in europe-west4 (On default network) +Skip Subnet: default in europe-west8 (On default network) +Skip Subnet: default in europe-west9 (On default network) +Skip Subnet: default in me-central1 (On default network) +Skip Subnet: default in me-central2 (On default network) +Skip Subnet: default in northamerica-south1 (On default network) +Skip Subnet: default in southamerica-west1 (On default network) +Skip Subnet: default in us-central1 (On default network) +Skip Subnet: default in us-central2 (On default network) +Skip Subnet: default in us-east1 (On default network) +Skip Subnet: default in us-east4 (On default network) +Skip Subnet: default in us-east5 (On default network) +Skip Subnet: default in us-east7 (On default network) +Skip Subnet: default in us-south1 (On default network) +Skip Subnet: default in us-west1 (On default network) +Skip Subnet: default in us-west2 (On default network) +Skip Subnet: default in us-west3 (On default network) +Skip Subnet: default in us-west4 (On default network) +Skip Subnet: default in us-west8 (On default network) +Skip Subnet: hpc-vpc in asia-east1 (In exclusion list) +Skip Subnet: hpc-vpc in asia-northeast1 (In exclusion list) +Skip Subnet: hpc-vpc in asia-southeast1 (In exclusion list) +Skip Subnet: hpc-vpc in australia-southeast1 (In exclusion list) +Skip Subnet: hpc-vpc in europe-north1 (In exclusion list) +Skip Subnet: hpc-vpc in europe-north2 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west1 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west2 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west3 (In exclusion list) +Skip Subnet: hpc-vpc in europe-west4 (In exclusion list) +Skip Subnet: hpc-vpc in southamerica-west1 (In exclusion list) +Skip Subnet: hpc-vpc in us-central1 (In exclusion list) +Skip Subnet: hpc-vpc in us-central2 (In exclusion list) +Skip Subnet: hpc-vpc in us-east1 (In exclusion list) +Skip Subnet: hpc-vpc in us-east4 (In exclusion list) +Skip Subnet: hpc-vpc in us-east5 (In exclusion list) +Skip Subnet: hpc-vpc in us-east7 (In exclusion list) +Skip Subnet: hpc-vpc in us-south1 (In exclusion list) +Skip Subnet: hpc-vpc in us-west1 (In exclusion list) +Skip Subnet: hpc-vpc in us-west2 (In exclusion list) +Skip Subnet: hpc-vpc in us-west3 (In exclusion list) +Skip Subnet: hpc-vpc in us-west4 (In exclusion list) +Skip Subnet: hpc-vpc in us-west8 (In exclusion list) +The following Subnetworks are targeted for deletion in this run: +hpcdydis-primary-subnet europe-west4 +hpcdy-primary-subnet europe-west4 +hpcimg-primary-subnet us-central1 +hpc-lustre-test-02-primary-subnet us-central1 +khu-h4d-cluster-test-primary-subnet us-central1 +khu-h4d-cluster-test-rdma-sub-0 us-central1 +lustre-06-primary-subnet us-central1 +lustre-test-06-primary-subnet us-central1 +managed-lustre-03-primary-subnet us-central1 +ml-gke-subnet us-central1 +monitoring-8323fe-primary-subnet us-central1 +sarthakagrr-primary-subnet us-central1 +sp-helmtest1-subnet us-central1 +static-sarthakag-primary-subnet us-central1 +[EXECUTE] Subnetwork: Deleting hpcdydis-primary-subnet in europe-west4 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdydis-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-mds0-internal-address' + +ERROR: Failed to delete Subnetwork hpcdydis-primary-subnet in europe-west4 +[EXECUTE] Subnetwork: Deleting hpcdy-primary-subnet in europe-west4 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdy-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss2-internal-address' + +ERROR: Failed to delete Subnetwork hpcdy-primary-subnet in europe-west4 +[EXECUTE] Subnetwork: Deleting hpcimg-primary-subnet in us-central1 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/hpcimg-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-oss1-internal-address' + +ERROR: Failed to delete Subnetwork hpcimg-primary-subnet in us-central1 +[EXECUTE] Subnetwork: Deleting hpc-lustre-test-02-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/hpc-lustre-test-02-primary-subnet]. +[EXECUTE] Subnetwork: Deleting khu-h4d-cluster-test-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/khu-h4d-cluster-test-primary-subnet]. +[EXECUTE] Subnetwork: Deleting khu-h4d-cluster-test-rdma-sub-0 in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/khu-h4d-cluster-test-rdma-sub-0]. +[EXECUTE] Subnetwork: Deleting lustre-06-primary-subnet in us-central1 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustre06-slurm-login-001' + +ERROR: Failed to delete Subnetwork lustre-06-primary-subnet in us-central1 +[EXECUTE] Subnetwork: Deleting lustre-test-06-primary-subnet in us-central1 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-test-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustretest-controller' + +ERROR: Failed to delete Subnetwork lustre-test-06-primary-subnet in us-central1 +[EXECUTE] Subnetwork: Deleting managed-lustre-03-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/managed-lustre-03-primary-subnet]. +[EXECUTE] Subnetwork: Deleting ml-gke-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/ml-gke-subnet]. +[EXECUTE] Subnetwork: Deleting monitoring-8323fe-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/monitoring-8323fe-primary-subnet]. +[EXECUTE] Subnetwork: Deleting sarthakagrr-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/sarthakagrr-primary-subnet]. +[EXECUTE] Subnetwork: Deleting sp-helmtest1-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/sp-helmtest1-subnet]. +[EXECUTE] Subnetwork: Deleting static-sarthakag-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/static-sarthakag-primary-subnet]. +--- Subnetwork Deletion Process Complete --- +--- Thu Nov 27 09:41:11 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 09:42:33 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: true +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T05:42:33+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +hpcdydis-primary-subnet in europe-west4 +hpcdy-primary-subnet in europe-west4 +hpcimg-primary-subnet in us-central1 +lustre-06-primary-subnet in us-central1 +lustre-test-06-primary-subnet in us-central1 +--- Processing Subnet: hpcdydis-primary-subnet in europe-west4 --- +Found dependent addresses for hpcdydis-primary-subnet in europe-west4: +exascaler-cloud-a2a0-mds0-internal-address +exascaler-cloud-a2a0-mgs0-internal-address +exascaler-cloud-a2a0-oss0-internal-address +exascaler-cloud-a2a0-oss1-internal-address +exascaler-cloud-a2a0-oss2-internal-address +[DRY RUN] Address: Would delete exascaler-cloud-a2a0-mds0-internal-address in europe-west4 + Command: gcloud compute addresses delete "exascaler-cloud-a2a0-mds0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-a2a0-mgs0-internal-address in europe-west4 + Command: gcloud compute addresses delete "exascaler-cloud-a2a0-mgs0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-a2a0-oss0-internal-address in europe-west4 + Command: gcloud compute addresses delete "exascaler-cloud-a2a0-oss0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-a2a0-oss1-internal-address in europe-west4 + Command: gcloud compute addresses delete "exascaler-cloud-a2a0-oss1-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-a2a0-oss2-internal-address in europe-west4 + Command: gcloud compute addresses delete "exascaler-cloud-a2a0-oss2-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +--- Processing Subnet: hpcdy-primary-subnet in europe-west4 --- +Found dependent addresses for hpcdy-primary-subnet in europe-west4: +exascaler-cloud-4a36-mds0-internal-address +exascaler-cloud-4a36-mgs0-internal-address +exascaler-cloud-4a36-oss0-internal-address +exascaler-cloud-4a36-oss1-internal-address +exascaler-cloud-4a36-oss2-internal-address +[DRY RUN] Address: Would delete exascaler-cloud-4a36-mds0-internal-address in europe-west4 + Command: gcloud compute addresses delete "exascaler-cloud-4a36-mds0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-4a36-mgs0-internal-address in europe-west4 + Command: gcloud compute addresses delete "exascaler-cloud-4a36-mgs0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-4a36-oss0-internal-address in europe-west4 + Command: gcloud compute addresses delete "exascaler-cloud-4a36-oss0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-4a36-oss1-internal-address in europe-west4 + Command: gcloud compute addresses delete "exascaler-cloud-4a36-oss1-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-4a36-oss2-internal-address in europe-west4 + Command: gcloud compute addresses delete "exascaler-cloud-4a36-oss2-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 + Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet +--- Processing Subnet: hpcimg-primary-subnet in us-central1 --- +Found dependent addresses for hpcimg-primary-subnet in us-central1: +exascaler-cloud-4691-mds0-internal-address +exascaler-cloud-4691-mgs0-internal-address +exascaler-cloud-4691-oss0-internal-address +exascaler-cloud-4691-oss1-internal-address +exascaler-cloud-4691-oss2-internal-address +[DRY RUN] Address: Would delete exascaler-cloud-4691-mds0-internal-address in us-central1 + Command: gcloud compute addresses delete "exascaler-cloud-4691-mds0-internal-address" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-4691-mgs0-internal-address in us-central1 + Command: gcloud compute addresses delete "exascaler-cloud-4691-mgs0-internal-address" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-4691-oss0-internal-address in us-central1 + Command: gcloud compute addresses delete "exascaler-cloud-4691-oss0-internal-address" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-4691-oss1-internal-address in us-central1 + Command: gcloud compute addresses delete "exascaler-cloud-4691-oss1-internal-address" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Address: Would delete exascaler-cloud-4691-oss2-internal-address in us-central1 + Command: gcloud compute addresses delete "exascaler-cloud-4691-oss2-internal-address" --project="hpc-toolkit-dev" --region="us-central1" --quiet +[DRY RUN] Subnetwork: Would delete hpcimg-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "hpcimg-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: lustre-06-primary-subnet in us-central1 --- +No dependent addresses found for lustre-06-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- +No dependent addresses found for lustre-test-06-primary-subnet in us-central1. +[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 + Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet +--- Thu Nov 27 09:42:54 AM UTC 2025 --- Cleanup Script Run Finished --- + +--- Thu Nov 27 09:44:01 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- +DRY_RUN mode: false +Will attempt to delete up to 30 resources of each type per run. +Targeting resources created before: 2025-11-27T05:44:01+0000 +Reading exclusion list from local file: exclusions.txt +Exclusion list loaded with 60 entries: + - vertexui-do-not-kill + - hpc-ctk1357 + - hpc-toolkit-dev@appspot.gserviceaccount.com + - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com + - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com + - 508417052821-compute@developer.gserviceaccount.com + - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com + - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com + - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com + - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com + - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - vertexui-do-not-kill-boot + - vertexui-do-not-kill-data + - lustretest-controller + - lustretest-slurm-login-001 + - lustre06-controller + - lustre06-slurm-login-001 + - default + - mglsard + - default-router-us-west1 + - default-router-us-west4 + - default-net-router + - default-router-australia-southeast1 + - default-router-us-east4 + - image-inspector-550 + - image-inspector + - lustre-06-net-router + - lustre-test-06-net-router + - mglsa-net-router + - mglsard-net-router + - gke-managed-lustre-basic-net-fw-allow-iap-ingress + - gke-managed-lustre-basic-net-fw-allow-internal-traffic + - lustre-06-net-fw-allow-iap-ingress + - lustre-06-net-fw-allow-internal-traffic + - lustre-test-06-net-fw-allow-iap-ingress + - lustre-test-06-net-fw-allow-internal-traffic + - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com + - hpc-vpc +--- Deletion Phase 1: GKE Clusters (Top 30) --- +No GKE clusters found to delete in this run. +--- Deletion Phase 1: Compute Instances (Top 30) --- +Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) +Skip Instance: image-inspector in us-west1-a (In exclusion list) +Skip Instance: lustre06-controller in us-central1-a (In exclusion list) +Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: lustretest-controller in us-central1-a (In exclusion list) +Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) +Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) +No Instances found to delete in this run. +--- Deletion Phase 1: Filestore Instances (Top 30) --- +No Filestore instances found to delete in this run. +--- Deletion Phase 3: Cloud Routers (Top 30) --- +Skip Cloud Router: default-net-router (In exclusion list) +Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) +Skip Cloud Router: default-router-us-east4 (In exclusion list) +Skip Cloud Router: default-router-us-west1 (In exclusion list) +Skip Cloud Router: default-router-us-west4 (In exclusion list) +Skip Cloud Router: lustre-06-net-router (In exclusion list) +Skip Cloud Router: lustre-test-06-net-router (In exclusion list) +Skip Cloud Router: mglsa-net-router (In exclusion list) +Skip Cloud Router: mglsard-net-router (In exclusion list) +No Cloud Routers found to delete in this run. +--- Deletion Phase 4: Firewall Rules (Top 30) --- +Skip Firewall: a3hc-308e7e (On default network) +Skip Firewall: a3hc-754952 (On default network) +Skip Firewall: a3hc-a628c1 (On default network) +Skip Firewall: a3hc-b15fa8 (On default network) +Skip Firewall: a3hc-c27fd0 (On default network) +Skip Firewall: a3hc-df0061 (On default network) +Skip Firewall: a3mc-c9f69e (On default network) +Skip Firewall: a3mc-f7e2b2 (On default network) +Skip Firewall: allow-internal (On default network) +Skip Firewall: allow-ssh (On default network) +Skip Firewall: default-allow-http (On default network) +Skip Firewall: default-allow-https (On default network) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) +Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) +No Firewall Rules found to delete in this run. +--- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- +The following Subnetworks (and their dependent addresses) are targeted for deletion: +hpcdydis-primary-subnet in europe-west4 +hpcdy-primary-subnet in europe-west4 +hpcimg-primary-subnet in us-central1 +lustre-06-primary-subnet in us-central1 +lustre-test-06-primary-subnet in us-central1 +--- Processing Subnet: hpcdydis-primary-subnet in europe-west4 --- +Found dependent addresses for hpcdydis-primary-subnet in europe-west4: +exascaler-cloud-a2a0-mds0-internal-address +exascaler-cloud-a2a0-mgs0-internal-address +exascaler-cloud-a2a0-oss0-internal-address +exascaler-cloud-a2a0-oss1-internal-address +exascaler-cloud-a2a0-oss2-internal-address +[EXECUTE] Address: Deleting exascaler-cloud-a2a0-mds0-internal-address in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-mds0-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-a2a0-mgs0-internal-address in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-mgs0-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-a2a0-oss0-internal-address in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-oss0-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-a2a0-oss1-internal-address in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-oss1-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-a2a0-oss2-internal-address in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-oss2-internal-address]. +[EXECUTE] Subnetwork: Deleting hpcdydis-primary-subnet in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdydis-primary-subnet]. +Successfully deleted Subnetwork hpcdydis-primary-subnet in europe-west4. +--- Processing Subnet: hpcdy-primary-subnet in europe-west4 --- +Found dependent addresses for hpcdy-primary-subnet in europe-west4: +exascaler-cloud-4a36-mds0-internal-address +exascaler-cloud-4a36-mgs0-internal-address +exascaler-cloud-4a36-oss0-internal-address +exascaler-cloud-4a36-oss1-internal-address +exascaler-cloud-4a36-oss2-internal-address +[EXECUTE] Address: Deleting exascaler-cloud-4a36-mds0-internal-address in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-mds0-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-4a36-mgs0-internal-address in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-mgs0-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-4a36-oss0-internal-address in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss0-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-4a36-oss1-internal-address in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss1-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-4a36-oss2-internal-address in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss2-internal-address]. +[EXECUTE] Subnetwork: Deleting hpcdy-primary-subnet in europe-west4 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdy-primary-subnet]. +Successfully deleted Subnetwork hpcdy-primary-subnet in europe-west4. +--- Processing Subnet: hpcimg-primary-subnet in us-central1 --- +Found dependent addresses for hpcimg-primary-subnet in us-central1: +exascaler-cloud-4691-mds0-internal-address +exascaler-cloud-4691-mgs0-internal-address +exascaler-cloud-4691-oss0-internal-address +exascaler-cloud-4691-oss1-internal-address +exascaler-cloud-4691-oss2-internal-address +[EXECUTE] Address: Deleting exascaler-cloud-4691-mds0-internal-address in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-mds0-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-4691-mgs0-internal-address in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-mgs0-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-4691-oss0-internal-address in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-oss0-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-4691-oss1-internal-address in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-oss1-internal-address]. +[EXECUTE] Address: Deleting exascaler-cloud-4691-oss2-internal-address in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-oss2-internal-address]. +[EXECUTE] Subnetwork: Deleting hpcimg-primary-subnet in us-central1 +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/hpcimg-primary-subnet]. +Successfully deleted Subnetwork hpcimg-primary-subnet in us-central1. +--- Processing Subnet: lustre-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-06-primary-subnet in us-central1. +[EXECUTE] Subnetwork: Deleting lustre-06-primary-subnet in us-central1 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustre06-slurm-login-001' + +ERROR: Failed to delete Subnetwork lustre-06-primary-subnet in us-central1. Check for other dependencies. +--- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- +WARNING: The following filter keys were not present in any resource : purpose, subnetwork +No dependent addresses found for lustre-test-06-primary-subnet in us-central1. +[EXECUTE] Subnetwork: Deleting lustre-test-06-primary-subnet in us-central1 +ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: + - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-test-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustretest-controller' + +ERROR: Failed to delete Subnetwork lustre-test-06-primary-subnet in us-central1. Check for other dependencies. +--- Thu Nov 27 09:46:30 AM UTC 2025 --- Cleanup Script Run Finished --- + diff --git a/template.txt b/template.txt new file mode 100644 index 0000000000..e1e7eb1964 --- /dev/null +++ b/template.txt @@ -0,0 +1,4227 @@ +[2025-11-30 18:41:02] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:41:02] [INFO] Time Cutoff (General): 2025-11-30T18:41:02+0000 +[2025-11-30 18:41:02] [INFO] Time Cutoff (Images): 2025-10-01T18:41:02+0000 +[2025-11-30 18:41:02] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:41:02] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:41:02] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:41:04] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:41:04] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:41:06] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:41:06] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a163acslur-compute-nodeset-20251013073728349400000002 (Global) +[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a2acslurmf-compute-nodeset-20250919234302964700000001 (Global) +[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3h23d4-compute-a3nodeset-20251114063711441000000003 (Global) +[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3h23d4-controller-default-20251114063701192000000002 (Global) +[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3h23d4-login-login-20251114063653023800000001 (Global) +[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3hca628-compute-a3nodeset-20251017102226551000000004 (Global) +[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3hca628-compute-debugnodeset-20251017102226507900000003 (Global) +[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3hca628-controller-default-20251017102143258500000002 (Global) +[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3hca628-login-login-20251017102143197700000001 (Global) +[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcb15f-compute-a3nodeset-20251116065432482600000003 (Global) +[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcb15f-compute-debugnodeset-20251116065432486000000004 (Global) +[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcb15f-controller-default-20251116065340384400000001 (Global) +[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcb15f-login-login-20251116065340427400000002 (Global) +[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcdf00-compute-a3nodeset-20251115185843671200000004 (Global) +[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcdf00-compute-debugnodeset-20251115185843634300000003 (Global) +[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcdf00-controller-default-20251115185751172200000001 (Global) +[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcdf00-login-login-20251115185752052600000002 (Global) +[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3lavnew-compute-a3nodeset-20251010141633773500000003 (Global) +[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3lavnew-compute-debugnodeset-20251010141633744500000001 (Global) +[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3lavnew-controller-default-20251010141643756900000004 (Global) +[2025-11-30 18:41:10] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:41:10] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:41:12] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:41:12] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:41:12] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:41:12] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:41:15] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:41:15] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:41:18] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:41:18] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:41:19] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:41:19] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:41:19] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:41:19] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:41:19] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:41:19] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:41:19] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:41:19] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:41:19] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:41:19] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:41:19Z (Unix: 1763318479) +[2025-11-30 18:41:19] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:41:21] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:41:21] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 18:41:24] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:41:24] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:41:24] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:41:24] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:41:24] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:41:24] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 18:41:26] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:41:26] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:41:29] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:41:29] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:41:31] [INFO] No Global Address found matching criteria. +[2025-11-30 18:41:31] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 18:41:33] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:41:33] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 18:41:36] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:41:36] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:41:36] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:41:36] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:41:36] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:39] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:39] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 18:41:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:41:41] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 18:41:43] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:41:43] [INFO] CLEANUP RUN FINISHED +[2025-11-30 18:42:34] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:42:34] [INFO] Time Cutoff (General): 2025-11-30T18:42:34+0000 +[2025-11-30 18:42:34] [INFO] Time Cutoff (Images): 2025-10-01T18:42:34+0000 +[2025-11-30 18:42:34] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:42:34] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:42:34] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:42:37] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:42:37] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:42:38] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:42:38] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:42:42] [EXECUTE] Deleting Instance Template: a163acslur-compute-nodeset-20251013073728349400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a163acslur-compute-nodeset-20251013073728349400000002]. +[2025-11-30 18:42:45] [SUCCESS] Deleted a163acslur-compute-nodeset-20251013073728349400000002 +[2025-11-30 18:42:45] [EXECUTE] Deleting Instance Template: a2acslurmf-compute-nodeset-20250919234302964700000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a2acslurmf-compute-nodeset-20250919234302964700000001]. +[2025-11-30 18:42:48] [SUCCESS] Deleted a2acslurmf-compute-nodeset-20250919234302964700000001 +[2025-11-30 18:42:48] [EXECUTE] Deleting Instance Template: a3h23d4-compute-a3nodeset-20251114063711441000000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3h23d4-compute-a3nodeset-20251114063711441000000003]. +[2025-11-30 18:42:51] [SUCCESS] Deleted a3h23d4-compute-a3nodeset-20251114063711441000000003 +[2025-11-30 18:42:51] [EXECUTE] Deleting Instance Template: a3h23d4-controller-default-20251114063701192000000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3h23d4-controller-default-20251114063701192000000002]. +[2025-11-30 18:42:54] [SUCCESS] Deleted a3h23d4-controller-default-20251114063701192000000002 +[2025-11-30 18:42:54] [EXECUTE] Deleting Instance Template: a3h23d4-login-login-20251114063653023800000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3h23d4-login-login-20251114063653023800000001]. +[2025-11-30 18:42:57] [SUCCESS] Deleted a3h23d4-login-login-20251114063653023800000001 +[2025-11-30 18:42:57] [EXECUTE] Deleting Instance Template: a3hca628-compute-a3nodeset-20251017102226551000000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hca628-compute-a3nodeset-20251017102226551000000004]. +[2025-11-30 18:43:00] [SUCCESS] Deleted a3hca628-compute-a3nodeset-20251017102226551000000004 +[2025-11-30 18:43:00] [EXECUTE] Deleting Instance Template: a3hca628-compute-debugnodeset-20251017102226507900000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hca628-compute-debugnodeset-20251017102226507900000003]. +[2025-11-30 18:43:04] [SUCCESS] Deleted a3hca628-compute-debugnodeset-20251017102226507900000003 +[2025-11-30 18:43:04] [EXECUTE] Deleting Instance Template: a3hca628-controller-default-20251017102143258500000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hca628-controller-default-20251017102143258500000002]. +[2025-11-30 18:43:07] [SUCCESS] Deleted a3hca628-controller-default-20251017102143258500000002 +[2025-11-30 18:43:07] [EXECUTE] Deleting Instance Template: a3hca628-login-login-20251017102143197700000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hca628-login-login-20251017102143197700000001]. +[2025-11-30 18:43:10] [SUCCESS] Deleted a3hca628-login-login-20251017102143197700000001 +[2025-11-30 18:43:10] [EXECUTE] Deleting Instance Template: a3hcb15f-compute-a3nodeset-20251116065432482600000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcb15f-compute-a3nodeset-20251116065432482600000003]. +[2025-11-30 18:43:13] [SUCCESS] Deleted a3hcb15f-compute-a3nodeset-20251116065432482600000003 +[2025-11-30 18:43:13] [EXECUTE] Deleting Instance Template: a3hcb15f-compute-debugnodeset-20251116065432486000000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcb15f-compute-debugnodeset-20251116065432486000000004]. +[2025-11-30 18:43:17] [SUCCESS] Deleted a3hcb15f-compute-debugnodeset-20251116065432486000000004 +[2025-11-30 18:43:17] [EXECUTE] Deleting Instance Template: a3hcb15f-controller-default-20251116065340384400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcb15f-controller-default-20251116065340384400000001]. +[2025-11-30 18:43:20] [SUCCESS] Deleted a3hcb15f-controller-default-20251116065340384400000001 +[2025-11-30 18:43:20] [EXECUTE] Deleting Instance Template: a3hcb15f-login-login-20251116065340427400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcb15f-login-login-20251116065340427400000002]. +[2025-11-30 18:43:22] [SUCCESS] Deleted a3hcb15f-login-login-20251116065340427400000002 +[2025-11-30 18:43:22] [EXECUTE] Deleting Instance Template: a3hcdf00-compute-a3nodeset-20251115185843671200000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcdf00-compute-a3nodeset-20251115185843671200000004]. +[2025-11-30 18:43:26] [SUCCESS] Deleted a3hcdf00-compute-a3nodeset-20251115185843671200000004 +[2025-11-30 18:43:26] [EXECUTE] Deleting Instance Template: a3hcdf00-compute-debugnodeset-20251115185843634300000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcdf00-compute-debugnodeset-20251115185843634300000003]. +[2025-11-30 18:43:29] [SUCCESS] Deleted a3hcdf00-compute-debugnodeset-20251115185843634300000003 +[2025-11-30 18:43:29] [EXECUTE] Deleting Instance Template: a3hcdf00-controller-default-20251115185751172200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcdf00-controller-default-20251115185751172200000001]. +[2025-11-30 18:43:32] [SUCCESS] Deleted a3hcdf00-controller-default-20251115185751172200000001 +[2025-11-30 18:43:32] [EXECUTE] Deleting Instance Template: a3hcdf00-login-login-20251115185752052600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcdf00-login-login-20251115185752052600000002]. +[2025-11-30 18:43:35] [SUCCESS] Deleted a3hcdf00-login-login-20251115185752052600000002 +[2025-11-30 18:43:35] [EXECUTE] Deleting Instance Template: a3lavnew-compute-a3nodeset-20251010141633773500000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3lavnew-compute-a3nodeset-20251010141633773500000003]. +[2025-11-30 18:43:38] [SUCCESS] Deleted a3lavnew-compute-a3nodeset-20251010141633773500000003 +[2025-11-30 18:43:38] [EXECUTE] Deleting Instance Template: a3lavnew-compute-debugnodeset-20251010141633744500000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3lavnew-compute-debugnodeset-20251010141633744500000001]. +[2025-11-30 18:43:42] [SUCCESS] Deleted a3lavnew-compute-debugnodeset-20251010141633744500000001 +[2025-11-30 18:43:42] [EXECUTE] Deleting Instance Template: a3lavnew-controller-default-20251010141643756900000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3lavnew-controller-default-20251010141643756900000004]. +[2025-11-30 18:43:45] [SUCCESS] Deleted a3lavnew-controller-default-20251010141643756900000004 +[2025-11-30 18:43:45] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:43:45] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:43:47] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:43:47] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:43:47] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:43:47] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:43:50] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:43:50] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:43:53] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:43:53] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:43:53] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:43:53] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:43:53] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:43:53] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:43:53] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:43:53] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:43:54] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:43:54] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:43:54] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:43:54] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:43:54Z (Unix: 1763318634) +[2025-11-30 18:43:54] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:43:56] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:43:56] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 18:43:59] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:43:59] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:43:59] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:43:59] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:43:59] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:43:59] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 18:44:01] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:44:01] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:44:03] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:44:03] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:44:06] [INFO] No Global Address found matching criteria. +[2025-11-30 18:44:06] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 18:44:08] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:44:08] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 18:44:11] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:44:11] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:44:11] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:44:11] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:44:11] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:14] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:14] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 18:44:16] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:44:16] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 18:44:18] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:44:18] [INFO] CLEANUP RUN FINISHED +[2025-11-30 18:44:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:44:29] [INFO] Time Cutoff (General): 2025-11-30T18:44:29+0000 +[2025-11-30 18:44:29] [INFO] Time Cutoff (Images): 2025-10-01T18:44:29+0000 +[2025-11-30 18:44:29] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:44:29] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:44:29] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:44:31] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:44:31] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:44:33] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:44:33] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3lavnew-login-login-20251010141633747300000002 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m14b2-compute-a3meganodeset-20251024053708898700000004 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m14b2-compute-debugnodeset-20251024053708872100000003 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m14b2-controller-default-20251024053548842100000002 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m14b2-login-login-20251024053548834500000001 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1b32-compute-a3meganodeset-20251023134637080900000004 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1b32-compute-debugnodeset-20251023134637047400000003 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1b32-controller-default-20251023134613677700000001 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1b32-login-login-20251023134613712600000002 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1bc7-compute-a3meganodeset-20251114062112120000000004 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1bc7-compute-debugnodeset-20251114062112091900000003 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1bc7-controller-default-20251114061949993400000001 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1bc7-login-login-20251114061950086100000002 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m2c1a-compute-a3meganodeset-20250828212541355600000004 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m2c1a-compute-debugnodeset-20250828212541351800000003 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m2c1a-controller-default-20250828212418057400000002 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m2c1a-login-login-20250828212416517200000001 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m3500-compute-a3meganodeset-20251114015749275800000004 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m3500-compute-debugnodeset-20251114015749253600000003 (Global) +[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m3500-controller-default-20251114015726857500000001 (Global) +[2025-11-30 18:44:36] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:44:36] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:44:39] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:44:39] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:44:39] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:44:39] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:44:42] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:44:42] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:44:44] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:44:45] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:44:45] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:44:45] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:44:45] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:44:45] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:44:45] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:44:45] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:44:45] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:44:45] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:44:45] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:44:45] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:44:45Z (Unix: 1763318685) +[2025-11-30 18:44:45] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:44:48] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:44:48] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 18:44:50] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:44:50] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:44:50] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:44:50] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:44:50] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:44:50] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 18:44:52] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:44:52] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:44:55] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:44:55] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:44:57] [INFO] No Global Address found matching criteria. +[2025-11-30 18:44:57] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 18:45:00] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:45:00] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 18:45:02] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:45:02] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:45:02] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:45:02] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:45:02] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:05] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 18:45:07] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:45:07] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 18:45:09] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:45:09] [INFO] CLEANUP RUN FINISHED +[2025-11-30 18:45:40] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:45:40] [INFO] Time Cutoff (General): 2025-11-30T18:45:40+0000 +[2025-11-30 18:45:40] [INFO] Time Cutoff (Images): 2025-10-01T18:45:40+0000 +[2025-11-30 18:45:40] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:45:40] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:45:40] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:45:43] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:45:43] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:45:45] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:45:45] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:45:48] [EXECUTE] Deleting Instance Template: a3lavnew-login-login-20251010141633747300000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3lavnew-login-login-20251010141633747300000002]. +[2025-11-30 18:45:51] [SUCCESS] Deleted a3lavnew-login-login-20251010141633747300000002 +[2025-11-30 18:45:51] [EXECUTE] Deleting Instance Template: a3m14b2-compute-a3meganodeset-20251024053708898700000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m14b2-compute-a3meganodeset-20251024053708898700000004]. +[2025-11-30 18:45:54] [SUCCESS] Deleted a3m14b2-compute-a3meganodeset-20251024053708898700000004 +[2025-11-30 18:45:54] [EXECUTE] Deleting Instance Template: a3m14b2-compute-debugnodeset-20251024053708872100000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m14b2-compute-debugnodeset-20251024053708872100000003]. +[2025-11-30 18:45:57] [SUCCESS] Deleted a3m14b2-compute-debugnodeset-20251024053708872100000003 +[2025-11-30 18:45:57] [EXECUTE] Deleting Instance Template: a3m14b2-controller-default-20251024053548842100000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m14b2-controller-default-20251024053548842100000002]. +[2025-11-30 18:46:00] [SUCCESS] Deleted a3m14b2-controller-default-20251024053548842100000002 +[2025-11-30 18:46:00] [EXECUTE] Deleting Instance Template: a3m14b2-login-login-20251024053548834500000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m14b2-login-login-20251024053548834500000001]. +[2025-11-30 18:46:03] [SUCCESS] Deleted a3m14b2-login-login-20251024053548834500000001 +[2025-11-30 18:46:03] [EXECUTE] Deleting Instance Template: a3m1b32-compute-a3meganodeset-20251023134637080900000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1b32-compute-a3meganodeset-20251023134637080900000004]. +[2025-11-30 18:46:06] [SUCCESS] Deleted a3m1b32-compute-a3meganodeset-20251023134637080900000004 +[2025-11-30 18:46:06] [EXECUTE] Deleting Instance Template: a3m1b32-compute-debugnodeset-20251023134637047400000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1b32-compute-debugnodeset-20251023134637047400000003]. +[2025-11-30 18:46:09] [SUCCESS] Deleted a3m1b32-compute-debugnodeset-20251023134637047400000003 +[2025-11-30 18:46:09] [EXECUTE] Deleting Instance Template: a3m1b32-controller-default-20251023134613677700000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1b32-controller-default-20251023134613677700000001]. +[2025-11-30 18:46:12] [SUCCESS] Deleted a3m1b32-controller-default-20251023134613677700000001 +[2025-11-30 18:46:12] [EXECUTE] Deleting Instance Template: a3m1b32-login-login-20251023134613712600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1b32-login-login-20251023134613712600000002]. +[2025-11-30 18:46:15] [SUCCESS] Deleted a3m1b32-login-login-20251023134613712600000002 +[2025-11-30 18:46:15] [EXECUTE] Deleting Instance Template: a3m1bc7-compute-a3meganodeset-20251114062112120000000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1bc7-compute-a3meganodeset-20251114062112120000000004]. +[2025-11-30 18:46:18] [SUCCESS] Deleted a3m1bc7-compute-a3meganodeset-20251114062112120000000004 +[2025-11-30 18:46:18] [EXECUTE] Deleting Instance Template: a3m1bc7-compute-debugnodeset-20251114062112091900000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1bc7-compute-debugnodeset-20251114062112091900000003]. +[2025-11-30 18:46:21] [SUCCESS] Deleted a3m1bc7-compute-debugnodeset-20251114062112091900000003 +[2025-11-30 18:46:21] [EXECUTE] Deleting Instance Template: a3m1bc7-controller-default-20251114061949993400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1bc7-controller-default-20251114061949993400000001]. +[2025-11-30 18:46:24] [SUCCESS] Deleted a3m1bc7-controller-default-20251114061949993400000001 +[2025-11-30 18:46:24] [EXECUTE] Deleting Instance Template: a3m1bc7-login-login-20251114061950086100000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1bc7-login-login-20251114061950086100000002]. +[2025-11-30 18:46:28] [SUCCESS] Deleted a3m1bc7-login-login-20251114061950086100000002 +[2025-11-30 18:46:28] [EXECUTE] Deleting Instance Template: a3m2c1a-compute-a3meganodeset-20250828212541355600000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m2c1a-compute-a3meganodeset-20250828212541355600000004]. +[2025-11-30 18:46:31] [SUCCESS] Deleted a3m2c1a-compute-a3meganodeset-20250828212541355600000004 +[2025-11-30 18:46:31] [EXECUTE] Deleting Instance Template: a3m2c1a-compute-debugnodeset-20250828212541351800000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m2c1a-compute-debugnodeset-20250828212541351800000003]. +[2025-11-30 18:46:34] [SUCCESS] Deleted a3m2c1a-compute-debugnodeset-20250828212541351800000003 +[2025-11-30 18:46:34] [EXECUTE] Deleting Instance Template: a3m2c1a-controller-default-20250828212418057400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m2c1a-controller-default-20250828212418057400000002]. +[2025-11-30 18:46:37] [SUCCESS] Deleted a3m2c1a-controller-default-20250828212418057400000002 +[2025-11-30 18:46:37] [EXECUTE] Deleting Instance Template: a3m2c1a-login-login-20250828212416517200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m2c1a-login-login-20250828212416517200000001]. +[2025-11-30 18:46:40] [SUCCESS] Deleted a3m2c1a-login-login-20250828212416517200000001 +[2025-11-30 18:46:40] [EXECUTE] Deleting Instance Template: a3m3500-compute-a3meganodeset-20251114015749275800000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m3500-compute-a3meganodeset-20251114015749275800000004]. +[2025-11-30 18:46:44] [SUCCESS] Deleted a3m3500-compute-a3meganodeset-20251114015749275800000004 +[2025-11-30 18:46:44] [EXECUTE] Deleting Instance Template: a3m3500-compute-debugnodeset-20251114015749253600000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m3500-compute-debugnodeset-20251114015749253600000003]. +[2025-11-30 18:46:47] [SUCCESS] Deleted a3m3500-compute-debugnodeset-20251114015749253600000003 +[2025-11-30 18:46:47] [EXECUTE] Deleting Instance Template: a3m3500-controller-default-20251114015726857500000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m3500-controller-default-20251114015726857500000001]. +[2025-11-30 18:46:50] [SUCCESS] Deleted a3m3500-controller-default-20251114015726857500000001 +[2025-11-30 18:46:50] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:46:50] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:46:53] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:46:53] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:46:53] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:46:53] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:46:56] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:46:56] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:46:59] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:46:59] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:46:59] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:46:59] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:46:59] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:46:59] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:46:59] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:46:59] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:47:00] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:47:00] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:47:00] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:47:00] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:47:00Z (Unix: 1763318820) +[2025-11-30 18:47:00] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:47:02] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:47:02] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 18:47:05] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:47:05] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:47:05] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:47:05] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:47:05] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:47:05] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 18:47:07] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:47:07] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:47:09] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:47:09] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:47:12] [INFO] No Global Address found matching criteria. +[2025-11-30 18:47:12] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 18:47:14] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:47:14] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 18:47:16] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:47:16] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:47:16] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:47:16] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:47:16] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:19] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 18:47:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:47:22] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 18:47:23] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:47:23] [INFO] CLEANUP RUN FINISHED +[2025-11-30 18:48:04] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:48:04] [INFO] Time Cutoff (General): 2025-11-30T18:48:04+0000 +[2025-11-30 18:48:04] [INFO] Time Cutoff (Images): 2025-10-01T18:48:04+0000 +[2025-11-30 18:48:04] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:48:04] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:48:05] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:48:07] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:48:07] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:48:09] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:48:09] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m3500-login-login-20251114015727053000000002 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m45dc-compute-a3meganodeset-20250819191837179100000004 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m45dc-compute-debugnodeset-20250819191837149700000003 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m45dc-controller-default-20250819191713313600000001 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m45dc-login-login-20250819191713400200000002 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m5a82-compute-a3meganodeset-20250822225405540000000004 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m5a82-compute-debugnodeset-20250822225405514300000003 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m5a82-controller-default-20250822225341573600000001 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m5a82-login-login-20250822225348890200000002 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m7038-compute-a3meganodeset-20251023114646456400000004 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m7038-compute-debugnodeset-20251023114646424100000003 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m7038-controller-default-20251023114621658200000001 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m7038-login-login-20251023114621674300000002 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m9518-compute-a3meganodeset-20251114111413631900000003 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m9518-compute-debugnodeset-20251114111413659500000004 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m9518-controller-default-20251114111348348300000001 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m9518-login-login-20251114111348371900000002 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m96ba-compute-a3meganodeset-20250822235943750600000004 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m96ba-compute-debugnodeset-20250822235943730400000003 (Global) +[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m96ba-controller-default-20250822235920176400000001 (Global) +[2025-11-30 18:48:12] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:48:12] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:48:15] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:48:15] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:48:15] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:48:15] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:48:18] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:48:18] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:48:21] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:48:21] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:48:21] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:48:21] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:48:21] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:48:21] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:48:21] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:48:21] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:48:21] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:48:22] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:48:22] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:48:22] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:48:22Z (Unix: 1763318902) +[2025-11-30 18:48:22] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:48:24] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:48:24] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 18:48:27] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:48:27] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:48:27] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:48:27] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:48:27] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:48:27] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 18:48:29] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:48:29] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:48:31] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:48:31] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:48:34] [INFO] No Global Address found matching criteria. +[2025-11-30 18:48:34] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 18:48:36] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:48:36] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 18:48:39] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:48:39] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:48:39] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:48:39] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:48:39] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:42] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 18:48:44] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:48:44] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 18:48:46] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:48:46] [INFO] CLEANUP RUN FINISHED +./cleanup.sh: line 712: n: command not found +[2025-11-30 18:48:50] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:48:50] [INFO] Time Cutoff (General): 2025-11-30T18:48:50+0000 +[2025-11-30 18:48:50] [INFO] Time Cutoff (Images): 2025-10-01T18:48:50+0000 +[2025-11-30 18:48:50] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:48:50] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:48:51] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:48:53] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:48:53] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:48:55] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:48:55] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:48:58] [EXECUTE] Deleting Instance Template: a3m3500-login-login-20251114015727053000000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m3500-login-login-20251114015727053000000002]. +[2025-11-30 18:49:01] [SUCCESS] Deleted a3m3500-login-login-20251114015727053000000002 +[2025-11-30 18:49:01] [EXECUTE] Deleting Instance Template: a3m45dc-compute-a3meganodeset-20250819191837179100000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m45dc-compute-a3meganodeset-20250819191837179100000004]. +[2025-11-30 18:49:04] [SUCCESS] Deleted a3m45dc-compute-a3meganodeset-20250819191837179100000004 +[2025-11-30 18:49:04] [EXECUTE] Deleting Instance Template: a3m45dc-compute-debugnodeset-20250819191837149700000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m45dc-compute-debugnodeset-20250819191837149700000003]. +[2025-11-30 18:49:07] [SUCCESS] Deleted a3m45dc-compute-debugnodeset-20250819191837149700000003 +[2025-11-30 18:49:07] [EXECUTE] Deleting Instance Template: a3m45dc-controller-default-20250819191713313600000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m45dc-controller-default-20250819191713313600000001]. +[2025-11-30 18:49:11] [SUCCESS] Deleted a3m45dc-controller-default-20250819191713313600000001 +[2025-11-30 18:49:11] [EXECUTE] Deleting Instance Template: a3m45dc-login-login-20250819191713400200000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m45dc-login-login-20250819191713400200000002]. +[2025-11-30 18:49:14] [SUCCESS] Deleted a3m45dc-login-login-20250819191713400200000002 +[2025-11-30 18:49:14] [EXECUTE] Deleting Instance Template: a3m5a82-compute-a3meganodeset-20250822225405540000000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m5a82-compute-a3meganodeset-20250822225405540000000004]. +[2025-11-30 18:49:17] [SUCCESS] Deleted a3m5a82-compute-a3meganodeset-20250822225405540000000004 +[2025-11-30 18:49:17] [EXECUTE] Deleting Instance Template: a3m5a82-compute-debugnodeset-20250822225405514300000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m5a82-compute-debugnodeset-20250822225405514300000003]. +[2025-11-30 18:49:20] [SUCCESS] Deleted a3m5a82-compute-debugnodeset-20250822225405514300000003 +[2025-11-30 18:49:20] [EXECUTE] Deleting Instance Template: a3m5a82-controller-default-20250822225341573600000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m5a82-controller-default-20250822225341573600000001]. +[2025-11-30 18:49:24] [SUCCESS] Deleted a3m5a82-controller-default-20250822225341573600000001 +[2025-11-30 18:49:24] [EXECUTE] Deleting Instance Template: a3m5a82-login-login-20250822225348890200000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m5a82-login-login-20250822225348890200000002]. +[2025-11-30 18:49:27] [SUCCESS] Deleted a3m5a82-login-login-20250822225348890200000002 +[2025-11-30 18:49:27] [EXECUTE] Deleting Instance Template: a3m7038-compute-a3meganodeset-20251023114646456400000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m7038-compute-a3meganodeset-20251023114646456400000004]. +[2025-11-30 18:49:29] [SUCCESS] Deleted a3m7038-compute-a3meganodeset-20251023114646456400000004 +[2025-11-30 18:49:29] [EXECUTE] Deleting Instance Template: a3m7038-compute-debugnodeset-20251023114646424100000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m7038-compute-debugnodeset-20251023114646424100000003]. +[2025-11-30 18:49:33] [SUCCESS] Deleted a3m7038-compute-debugnodeset-20251023114646424100000003 +[2025-11-30 18:49:33] [EXECUTE] Deleting Instance Template: a3m7038-controller-default-20251023114621658200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m7038-controller-default-20251023114621658200000001]. +[2025-11-30 18:49:36] [SUCCESS] Deleted a3m7038-controller-default-20251023114621658200000001 +[2025-11-30 18:49:36] [EXECUTE] Deleting Instance Template: a3m7038-login-login-20251023114621674300000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m7038-login-login-20251023114621674300000002]. +[2025-11-30 18:49:39] [SUCCESS] Deleted a3m7038-login-login-20251023114621674300000002 +[2025-11-30 18:49:39] [EXECUTE] Deleting Instance Template: a3m9518-compute-a3meganodeset-20251114111413631900000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9518-compute-a3meganodeset-20251114111413631900000003]. +[2025-11-30 18:49:42] [SUCCESS] Deleted a3m9518-compute-a3meganodeset-20251114111413631900000003 +[2025-11-30 18:49:42] [EXECUTE] Deleting Instance Template: a3m9518-compute-debugnodeset-20251114111413659500000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9518-compute-debugnodeset-20251114111413659500000004]. +[2025-11-30 18:49:46] [SUCCESS] Deleted a3m9518-compute-debugnodeset-20251114111413659500000004 +[2025-11-30 18:49:46] [EXECUTE] Deleting Instance Template: a3m9518-controller-default-20251114111348348300000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9518-controller-default-20251114111348348300000001]. +[2025-11-30 18:49:49] [SUCCESS] Deleted a3m9518-controller-default-20251114111348348300000001 +[2025-11-30 18:49:49] [EXECUTE] Deleting Instance Template: a3m9518-login-login-20251114111348371900000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9518-login-login-20251114111348371900000002]. +[2025-11-30 18:49:53] [SUCCESS] Deleted a3m9518-login-login-20251114111348371900000002 +[2025-11-30 18:49:53] [EXECUTE] Deleting Instance Template: a3m96ba-compute-a3meganodeset-20250822235943750600000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m96ba-compute-a3meganodeset-20250822235943750600000004]. +[2025-11-30 18:49:55] [SUCCESS] Deleted a3m96ba-compute-a3meganodeset-20250822235943750600000004 +[2025-11-30 18:49:55] [EXECUTE] Deleting Instance Template: a3m96ba-compute-debugnodeset-20250822235943730400000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m96ba-compute-debugnodeset-20250822235943730400000003]. +[2025-11-30 18:49:58] [SUCCESS] Deleted a3m96ba-compute-debugnodeset-20250822235943730400000003 +[2025-11-30 18:49:58] [EXECUTE] Deleting Instance Template: a3m96ba-controller-default-20250822235920176400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m96ba-controller-default-20250822235920176400000001]. +[2025-11-30 18:50:01] [SUCCESS] Deleted a3m96ba-controller-default-20250822235920176400000001 +[2025-11-30 18:50:01] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:50:01] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:50:04] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:50:04] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:50:04] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:50:04] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:50:06] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:50:06] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:50:09] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:50:10] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:50:10] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:50:10] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:50:10] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:50:10] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:50:10] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:50:10] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:50:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:50:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:50:10] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:50:10] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:50:10Z (Unix: 1763319010) +[2025-11-30 18:50:10] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:50:13] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:50:13] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 18:50:15] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:50:15] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:50:15] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:50:15] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:50:15] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:50:15] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 18:50:18] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:50:18] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:50:20] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:50:20] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:50:23] [INFO] No Global Address found matching criteria. +[2025-11-30 18:50:23] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 18:50:25] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:50:25] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 18:50:27] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:50:27] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:50:27] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:50:27] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:50:27] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:30] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 18:50:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:50:33] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 18:50:35] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:50:35] [INFO] CLEANUP RUN FINISHED +[2025-11-30 18:52:49] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:52:49] [INFO] Time Cutoff (General): 2025-11-30T18:52:49+0000 +[2025-11-30 18:52:49] [INFO] Time Cutoff (Images): 2025-10-01T18:52:49+0000 +[2025-11-30 18:52:49] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:52:49] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:52:49] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:52:51] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:52:51] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:52:53] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:52:53] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m96ba-login-login-20250822235920487900000002 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9864-compute-a3meganodeset-20250819063622277500000004 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9864-compute-debugnodeset-20250819063622259700000003 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9864-controller-default-20250819063500745200000001 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9864-login-login-20250819063500917600000002 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-a3meganodeset-20251022121513097300000004 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-debugnodeset-20251022121513061400000003 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9b3091-controller-default-20251022121449886500000001 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9b3091-login-login-20251022121452690600000002 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mc636-compute-a3meganodeset-20250819181357460200000004 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mc636-compute-debugnodeset-20250819181357431100000003 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mc636-controller-default-20250819181334248000000001 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mc636-login-login-20250819181334341700000002 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3me777-compute-a3meganodeset-20251024084138357500000004 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3me777-compute-debugnodeset-20251024084138331700000003 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3me777-controller-default-20251024084114048400000001 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3me777-login-login-20251024084114099200000002 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mega-compute-a3meganodeset-20251118080924120000000004 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mega-compute-debugnodeset-20251118080924115100000003 (Global) +[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mega-controller-default-20251118080901356800000001 (Global) +[2025-11-30 18:52:56] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:52:56] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:52:59] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:52:59] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:52:59] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:52:59] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:53:02] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:53:02] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:53:04] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:53:04] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:53:05] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:53:05] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:53:05] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:53:05] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:53:05] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:53:05] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:53:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:53:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:53:05] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:53:05] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:53:05Z (Unix: 1763319185) +[2025-11-30 18:53:05] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:53:07] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:53:07] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 18:53:10] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:53:10] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:53:10] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:53:10] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:53:10] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:53:10] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 18:53:12] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:53:12] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:53:15] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:53:15] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:53:17] [INFO] No Global Address found matching criteria. +[2025-11-30 18:53:17] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 18:53:19] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:53:19] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 18:53:22] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:53:22] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:53:22] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:53:22] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:53:22] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:25] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 18:53:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:53:27] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 18:53:29] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:53:29] [INFO] CLEANUP RUN FINISHED +[2025-11-30 18:53:59] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:53:59] [INFO] Time Cutoff (General): 2025-11-30T18:53:59+0000 +[2025-11-30 18:53:59] [INFO] Time Cutoff (Images): 2025-10-01T18:53:59+0000 +[2025-11-30 18:53:59] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:53:59] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:53:59] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:54:02] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:54:02] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:54:04] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:54:04] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m96ba-login-login-20250822235920487900000002 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9864-compute-a3meganodeset-20250819063622277500000004 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9864-compute-debugnodeset-20250819063622259700000003 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9864-controller-default-20250819063500745200000001 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9864-login-login-20250819063500917600000002 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-a3meganodeset-20251022121513097300000004 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-debugnodeset-20251022121513061400000003 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9b3091-controller-default-20251022121449886500000001 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9b3091-login-login-20251022121452690600000002 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mc636-compute-a3meganodeset-20250819181357460200000004 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mc636-compute-debugnodeset-20250819181357431100000003 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mc636-controller-default-20250819181334248000000001 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mc636-login-login-20250819181334341700000002 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3me777-compute-a3meganodeset-20251024084138357500000004 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3me777-compute-debugnodeset-20251024084138331700000003 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3me777-controller-default-20251024084114048400000001 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3me777-login-login-20251024084114099200000002 (Global) +[2025-11-30 18:54:07] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mega-compute-debugnodeset-20251118080924115100000003 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mega-controller-default-20251118080901356800000001 (Global) +[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mega-login-login-20251118080901434600000002 (Global) +[2025-11-30 18:54:07] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:54:07] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:54:09] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:54:09] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:54:09] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:54:09] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:54:12] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:54:12] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:54:15] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:54:15] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:54:15] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:54:15] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:54:15] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:54:16] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:54:16] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:54:16] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:54:16] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:54:16] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:54:16] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:54:16] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:54:16Z (Unix: 1763319256) +[2025-11-30 18:54:16] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:54:18] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:54:18] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 18:54:21] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:54:21] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:54:21] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:54:21] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:54:21] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:54:21] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 18:54:23] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:54:23] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:54:25] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:54:25] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:54:28] [INFO] No Global Address found matching criteria. +[2025-11-30 18:54:28] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 18:54:30] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:54:30] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 18:54:32] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:54:32] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:54:32] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:54:32] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:54:32] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:35] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 18:54:37] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:54:37] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 18:54:39] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:54:39] [INFO] CLEANUP RUN FINISHED +[2025-11-30 18:54:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:54:47] [INFO] Time Cutoff (General): 2025-11-30T18:54:47+0000 +[2025-11-30 18:54:47] [INFO] Time Cutoff (Images): 2025-10-01T18:54:47+0000 +[2025-11-30 18:54:47] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:54:47] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:54:48] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:54:50] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:54:50] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:54:51] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:54:52] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m96ba-login-login-20250822235920487900000002 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9864-compute-a3meganodeset-20250819063622277500000004 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9864-compute-debugnodeset-20250819063622259700000003 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9864-controller-default-20250819063500745200000001 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9864-login-login-20250819063500917600000002 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-a3meganodeset-20251022121513097300000004 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-debugnodeset-20251022121513061400000003 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9b3091-controller-default-20251022121449886500000001 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9b3091-login-login-20251022121452690600000002 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3mc636-compute-a3meganodeset-20250819181357460200000004 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3mc636-compute-debugnodeset-20250819181357431100000003 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3mc636-controller-default-20250819181334248000000001 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3mc636-login-login-20250819181334341700000002 (Global) +[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3me777-compute-a3meganodeset-20251024084138357500000004 (Global) +[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3me777-compute-debugnodeset-20251024084138331700000003 (Global) +[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3me777-controller-default-20251024084114048400000001 (Global) +[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3me777-login-login-20251024084114099200000002 (Global) +[2025-11-30 18:54:55] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 18:54:55] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 18:54:55] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 18:54:55] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3qclavek-compute-a3ultranodeset-20251001033444416400000002 (Global) +[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3qclavek-controller-default-20251001033450408100000003 (Global) +[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3qclavek-login-slurm-login-20251001033440486800000001 (Global) +[2025-11-30 18:54:55] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:54:55] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:54:57] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:54:57] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:54:57] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:54:57] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:55:00] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:55:00] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:55:03] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:55:03] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:55:03] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:55:03] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:55:03] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:55:03] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:55:03] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:55:03] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:55:04] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:55:04] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:55:04] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:55:04] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:55:04Z (Unix: 1763319304) +[2025-11-30 18:55:04] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:55:06] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:55:06] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 18:55:09] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:55:09] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:55:09] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:55:09] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:55:09] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:55:09] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 18:55:11] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:55:11] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:55:13] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:55:13] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:55:17] [INFO] No Global Address found matching criteria. +[2025-11-30 18:55:17] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 18:55:19] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:55:19] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 18:55:22] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:55:22] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:55:22] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:55:22] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:55:22] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:24] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 18:55:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:55:27] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 18:55:28] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:55:28] [INFO] CLEANUP RUN FINISHED +./cleanup.sh: line 712: n: command not found +[2025-11-30 18:56:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:56:29] [INFO] Time Cutoff (General): 2025-11-30T18:56:29+0000 +[2025-11-30 18:56:29] [INFO] Time Cutoff (Images): 2025-10-01T18:56:29+0000 +[2025-11-30 18:56:29] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:56:29] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:56:30] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:56:32] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:56:32] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:56:34] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:56:34] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:56:37] [EXECUTE] Deleting Instance Template: a3m96ba-login-login-20250822235920487900000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m96ba-login-login-20250822235920487900000002]. +[2025-11-30 18:56:40] [SUCCESS] Deleted a3m96ba-login-login-20250822235920487900000002 +[2025-11-30 18:56:40] [EXECUTE] Deleting Instance Template: a3m9864-compute-a3meganodeset-20250819063622277500000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9864-compute-a3meganodeset-20250819063622277500000004]. +[2025-11-30 18:56:43] [SUCCESS] Deleted a3m9864-compute-a3meganodeset-20250819063622277500000004 +[2025-11-30 18:56:43] [EXECUTE] Deleting Instance Template: a3m9864-compute-debugnodeset-20250819063622259700000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9864-compute-debugnodeset-20250819063622259700000003]. +[2025-11-30 18:56:47] [SUCCESS] Deleted a3m9864-compute-debugnodeset-20250819063622259700000003 +[2025-11-30 18:56:47] [EXECUTE] Deleting Instance Template: a3m9864-controller-default-20250819063500745200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9864-controller-default-20250819063500745200000001]. +[2025-11-30 18:56:50] [SUCCESS] Deleted a3m9864-controller-default-20250819063500745200000001 +[2025-11-30 18:56:50] [EXECUTE] Deleting Instance Template: a3m9864-login-login-20250819063500917600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9864-login-login-20250819063500917600000002]. +[2025-11-30 18:56:53] [SUCCESS] Deleted a3m9864-login-login-20250819063500917600000002 +[2025-11-30 18:56:53] [EXECUTE] Deleting Instance Template: a3m9b3091-compute-a3meganodeset-20251022121513097300000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9b3091-compute-a3meganodeset-20251022121513097300000004]. +[2025-11-30 18:56:56] [SUCCESS] Deleted a3m9b3091-compute-a3meganodeset-20251022121513097300000004 +[2025-11-30 18:56:56] [EXECUTE] Deleting Instance Template: a3m9b3091-compute-debugnodeset-20251022121513061400000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9b3091-compute-debugnodeset-20251022121513061400000003]. +[2025-11-30 18:57:00] [SUCCESS] Deleted a3m9b3091-compute-debugnodeset-20251022121513061400000003 +[2025-11-30 18:57:00] [EXECUTE] Deleting Instance Template: a3m9b3091-controller-default-20251022121449886500000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9b3091-controller-default-20251022121449886500000001]. +[2025-11-30 18:57:03] [SUCCESS] Deleted a3m9b3091-controller-default-20251022121449886500000001 +[2025-11-30 18:57:03] [EXECUTE] Deleting Instance Template: a3m9b3091-login-login-20251022121452690600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9b3091-login-login-20251022121452690600000002]. +[2025-11-30 18:57:06] [SUCCESS] Deleted a3m9b3091-login-login-20251022121452690600000002 +[2025-11-30 18:57:06] [EXECUTE] Deleting Instance Template: a3mc636-compute-a3meganodeset-20250819181357460200000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3mc636-compute-a3meganodeset-20250819181357460200000004]. +[2025-11-30 18:57:09] [SUCCESS] Deleted a3mc636-compute-a3meganodeset-20250819181357460200000004 +[2025-11-30 18:57:09] [EXECUTE] Deleting Instance Template: a3mc636-compute-debugnodeset-20250819181357431100000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3mc636-compute-debugnodeset-20250819181357431100000003]. +[2025-11-30 18:57:12] [SUCCESS] Deleted a3mc636-compute-debugnodeset-20250819181357431100000003 +[2025-11-30 18:57:12] [EXECUTE] Deleting Instance Template: a3mc636-controller-default-20250819181334248000000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3mc636-controller-default-20250819181334248000000001]. +[2025-11-30 18:57:15] [SUCCESS] Deleted a3mc636-controller-default-20250819181334248000000001 +[2025-11-30 18:57:15] [EXECUTE] Deleting Instance Template: a3mc636-login-login-20250819181334341700000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3mc636-login-login-20250819181334341700000002]. +[2025-11-30 18:57:18] [SUCCESS] Deleted a3mc636-login-login-20250819181334341700000002 +[2025-11-30 18:57:18] [EXECUTE] Deleting Instance Template: a3me777-compute-a3meganodeset-20251024084138357500000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3me777-compute-a3meganodeset-20251024084138357500000004]. +[2025-11-30 18:57:22] [SUCCESS] Deleted a3me777-compute-a3meganodeset-20251024084138357500000004 +[2025-11-30 18:57:22] [EXECUTE] Deleting Instance Template: a3me777-compute-debugnodeset-20251024084138331700000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3me777-compute-debugnodeset-20251024084138331700000003]. +[2025-11-30 18:57:25] [SUCCESS] Deleted a3me777-compute-debugnodeset-20251024084138331700000003 +[2025-11-30 18:57:25] [EXECUTE] Deleting Instance Template: a3me777-controller-default-20251024084114048400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3me777-controller-default-20251024084114048400000001]. +[2025-11-30 18:57:28] [SUCCESS] Deleted a3me777-controller-default-20251024084114048400000001 +[2025-11-30 18:57:28] [EXECUTE] Deleting Instance Template: a3me777-login-login-20251024084114099200000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3me777-login-login-20251024084114099200000002]. +[2025-11-30 18:57:31] [SUCCESS] Deleted a3me777-login-login-20251024084114099200000002 +[2025-11-30 18:57:31] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 18:57:31] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 18:57:31] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 18:57:31] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 18:57:31] [EXECUTE] Deleting Instance Template: a3qclavek-compute-a3ultranodeset-20251001033444416400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3qclavek-compute-a3ultranodeset-20251001033444416400000002]. +[2025-11-30 18:57:34] [SUCCESS] Deleted a3qclavek-compute-a3ultranodeset-20251001033444416400000002 +[2025-11-30 18:57:34] [EXECUTE] Deleting Instance Template: a3qclavek-controller-default-20251001033450408100000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3qclavek-controller-default-20251001033450408100000003]. +[2025-11-30 18:57:38] [SUCCESS] Deleted a3qclavek-controller-default-20251001033450408100000003 +[2025-11-30 18:57:38] [EXECUTE] Deleting Instance Template: a3qclavek-login-slurm-login-20251001033440486800000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3qclavek-login-slurm-login-20251001033440486800000001]. +[2025-11-30 18:57:41] [SUCCESS] Deleted a3qclavek-login-slurm-login-20251001033440486800000001 +[2025-11-30 18:57:41] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:57:41] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:57:43] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:57:43] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:57:43] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:57:43] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:57:46] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:57:46] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:57:49] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:57:49] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:57:49] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:57:49] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:57:49] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:57:49] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:57:49] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:57:49] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:57:50] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:57:50] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:57:50] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:57:50] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:57:50Z (Unix: 1763319470) +[2025-11-30 18:57:50] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:57:52] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:57:52] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 18:57:55] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:57:55] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:57:55] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:57:55] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:57:55] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:57:55] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 18:57:57] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:57:57] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:58:00] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:58:00] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:58:02] [INFO] No Global Address found matching criteria. +[2025-11-30 18:58:02] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 18:58:05] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:58:05] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 18:58:07] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:58:07] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:58:07] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:58:07] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:58:07] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 18:58:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:09] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:10] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 18:58:12] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:58:12] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 18:58:14] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:58:14] [INFO] CLEANUP RUN FINISHED +[2025-11-30 18:58:35] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:58:35] [INFO] Time Cutoff (General): 2025-11-30T18:58:34+0000 +[2025-11-30 18:58:35] [INFO] Time Cutoff (Images): 2025-10-01T18:58:35+0000 +[2025-11-30 18:58:35] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:58:35] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:58:35] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:58:38] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:58:38] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:58:40] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:58:40] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:58:43] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 18:58:43] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 18:58:43] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 18:58:43] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3slurmsy-compute-a3nodeset-20251016115123978200000002 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3slurmsy-controller-default-20251016115129563200000003 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3slurmsy-login-slurm-login-20251016115120300800000001 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3u529e-compute-a3ultranodeset-20251114115423143800000002 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3u529e-controller-default-20251114115425847100000003 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3u529e-login-slurm-login-20251114115420368400000001 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3usarr-compute-a4highnodeset-20251105000020022500000002 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3usarr-controller-default-20251105000025097200000003 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3usarr-login-slurm-login-20251105000015399700000001 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h333e-compute-a4highnodeset-20251120094513362600000002 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h333e-controller-default-20251120094518412300000003 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h333e-login-slurm-login-20251120094509085100000001 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h5c04-compute-a4highnodeset-20251114133459801900000002 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h5c04-controller-default-20251114133505649700000003 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h5c04-login-slurm-login-20251114133456443900000001 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h639c-controller-default-20251126091821328000000003 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h639c-login-slurm-login-20251126091812165600000001 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h68f2-compute-a4highnodeset-20251125205110303400000002 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h68f2-controller-default-20251125205115747500000003 (Global) +[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h68f2-login-slurm-login-20251125205107060800000001 (Global) +[2025-11-30 18:58:43] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:58:43] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:58:46] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:58:46] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:58:46] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:58:46] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:58:49] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:58:49] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:58:52] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:58:52] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:58:52] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:58:52] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:58:52] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:58:52] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:58:52] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:58:52] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:58:53] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:58:53] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:58:53] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:58:53] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:58:53Z (Unix: 1763319533) +[2025-11-30 18:58:53] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:58:55] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:58:55] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 18:58:58] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 18:58:58] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 18:58:58] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 18:58:58] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 18:58:58] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 18:58:58] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 18:59:00] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 18:59:00] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:59:03] [INFO] No Regional Address found matching criteria. +[2025-11-30 18:59:03] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 18:59:05] [INFO] No Global Address found matching criteria. +[2025-11-30 18:59:05] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 18:59:08] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 18:59:08] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 18:59:10] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:59:10] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:59:10] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 18:59:10] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 18:59:10] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:13] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 18:59:15] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 18:59:15] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 18:59:17] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 18:59:17] [INFO] CLEANUP RUN FINISHED +[2025-11-30 18:59:38] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 18:59:38] [INFO] Time Cutoff (General): 2025-11-30T18:59:38+0000 +[2025-11-30 18:59:38] [INFO] Time Cutoff (Images): 2025-10-01T18:59:38+0000 +[2025-11-30 18:59:38] [INFO] Delete Limit per Type: 20 +[2025-11-30 18:59:38] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 18:59:38] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 18:59:41] [INFO] No Service Accounts found matching prefix. +[2025-11-30 18:59:41] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:59:43] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 18:59:43] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 18:59:46] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 18:59:46] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 18:59:46] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 18:59:46] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 18:59:46] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 18:59:46] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 18:59:46] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3u529e-compute-a3ultranodeset-20251114115423143800000002 (Global) +[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3u529e-controller-default-20251114115425847100000003 (Global) +[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3u529e-login-slurm-login-20251114115420368400000001 (Global) +[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3usarr-compute-a4highnodeset-20251105000020022500000002 (Global) +[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3usarr-controller-default-20251105000025097200000003 (Global) +[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3usarr-login-slurm-login-20251105000015399700000001 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h333e-compute-a4highnodeset-20251120094513362600000002 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h333e-controller-default-20251120094518412300000003 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h333e-login-slurm-login-20251120094509085100000001 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h5c04-compute-a4highnodeset-20251114133459801900000002 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h5c04-controller-default-20251114133505649700000003 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h5c04-login-slurm-login-20251114133456443900000001 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h639c-controller-default-20251126091821328000000003 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h639c-login-slurm-login-20251126091812165600000001 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h68f2-compute-a4highnodeset-20251125205110303400000002 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h68f2-controller-default-20251125205115747500000003 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h68f2-login-slurm-login-20251125205107060800000001 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h6c3b-compute-a4highnodeset-20251117134652173800000002 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h6c3b-controller-default-20251117134657496900000003 (Global) +[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h6c3b-login-slurm-login-20251117134647592200000001 (Global) +[2025-11-30 18:59:47] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 18:59:47] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 18:59:49] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 18:59:49] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 18:59:49] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 18:59:49] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 18:59:52] [INFO] No Filestore instances found matching criteria. +[2025-11-30 18:59:52] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 18:59:55] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 18:59:55] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 18:59:55] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 18:59:55] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 18:59:56] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 18:59:56] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 18:59:56] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 18:59:56] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 18:59:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 18:59:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 18:59:56] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 18:59:56] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:59:56Z (Unix: 1763319596) +[2025-11-30 18:59:56] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 18:59:59] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 18:59:59] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:00:01] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:00:02] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:00:02] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:00:02] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:00:02] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:00:02] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:00:04] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:00:04] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:00:07] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:00:07] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:00:09] [INFO] No Global Address found matching criteria. +[2025-11-30 19:00:09] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:00:12] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:00:12] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:00:15] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:00:15] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:00:15] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:00:15] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:00:15] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:19] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:00:22] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:00:22] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:00:24] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:00:24] [INFO] CLEANUP RUN FINISHED +./cleanup.sh: line 712: n: command not found +[2025-11-30 19:01:10] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:01:10] [INFO] Time Cutoff (General): 2025-11-30T19:01:10+0000 +[2025-11-30 19:01:10] [INFO] Time Cutoff (Images): 2025-10-01T19:01:10+0000 +[2025-11-30 19:01:10] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:01:10] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:01:10] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:01:13] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:01:13] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:01:15] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:01:15] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:01:18] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:01:18] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:01:18] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:01:18] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:01:18] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:01:18] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:01:18] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:01:18] [EXECUTE] Deleting Instance Template: a3u529e-compute-a3ultranodeset-20251114115423143800000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3u529e-compute-a3ultranodeset-20251114115423143800000002]. +[2025-11-30 19:01:21] [SUCCESS] Deleted a3u529e-compute-a3ultranodeset-20251114115423143800000002 +[2025-11-30 19:01:21] [EXECUTE] Deleting Instance Template: a3u529e-controller-default-20251114115425847100000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3u529e-controller-default-20251114115425847100000003]. +[2025-11-30 19:01:24] [SUCCESS] Deleted a3u529e-controller-default-20251114115425847100000003 +[2025-11-30 19:01:24] [EXECUTE] Deleting Instance Template: a3u529e-login-slurm-login-20251114115420368400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3u529e-login-slurm-login-20251114115420368400000001]. +[2025-11-30 19:01:27] [SUCCESS] Deleted a3u529e-login-slurm-login-20251114115420368400000001 +[2025-11-30 19:01:27] [EXECUTE] Deleting Instance Template: a3usarr-compute-a4highnodeset-20251105000020022500000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3usarr-compute-a4highnodeset-20251105000020022500000002]. +[2025-11-30 19:01:30] [SUCCESS] Deleted a3usarr-compute-a4highnodeset-20251105000020022500000002 +[2025-11-30 19:01:30] [EXECUTE] Deleting Instance Template: a3usarr-controller-default-20251105000025097200000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3usarr-controller-default-20251105000025097200000003]. +[2025-11-30 19:01:34] [SUCCESS] Deleted a3usarr-controller-default-20251105000025097200000003 +[2025-11-30 19:01:34] [EXECUTE] Deleting Instance Template: a3usarr-login-slurm-login-20251105000015399700000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3usarr-login-slurm-login-20251105000015399700000001]. +[2025-11-30 19:01:37] [SUCCESS] Deleted a3usarr-login-slurm-login-20251105000015399700000001 +[2025-11-30 19:01:37] [EXECUTE] Deleting Instance Template: a4h333e-compute-a4highnodeset-20251120094513362600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h333e-compute-a4highnodeset-20251120094513362600000002]. +[2025-11-30 19:01:40] [SUCCESS] Deleted a4h333e-compute-a4highnodeset-20251120094513362600000002 +[2025-11-30 19:01:40] [EXECUTE] Deleting Instance Template: a4h333e-controller-default-20251120094518412300000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h333e-controller-default-20251120094518412300000003]. +[2025-11-30 19:01:43] [SUCCESS] Deleted a4h333e-controller-default-20251120094518412300000003 +[2025-11-30 19:01:43] [EXECUTE] Deleting Instance Template: a4h333e-login-slurm-login-20251120094509085100000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h333e-login-slurm-login-20251120094509085100000001]. +[2025-11-30 19:01:46] [SUCCESS] Deleted a4h333e-login-slurm-login-20251120094509085100000001 +[2025-11-30 19:01:46] [EXECUTE] Deleting Instance Template: a4h5c04-compute-a4highnodeset-20251114133459801900000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h5c04-compute-a4highnodeset-20251114133459801900000002]. +[2025-11-30 19:01:50] [SUCCESS] Deleted a4h5c04-compute-a4highnodeset-20251114133459801900000002 +[2025-11-30 19:01:50] [EXECUTE] Deleting Instance Template: a4h5c04-controller-default-20251114133505649700000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h5c04-controller-default-20251114133505649700000003]. +[2025-11-30 19:01:53] [SUCCESS] Deleted a4h5c04-controller-default-20251114133505649700000003 +[2025-11-30 19:01:53] [EXECUTE] Deleting Instance Template: a4h5c04-login-slurm-login-20251114133456443900000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h5c04-login-slurm-login-20251114133456443900000001]. +[2025-11-30 19:01:56] [SUCCESS] Deleted a4h5c04-login-slurm-login-20251114133456443900000001 +[2025-11-30 19:01:56] [EXECUTE] Deleting Instance Template: a4h639c-controller-default-20251126091821328000000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h639c-controller-default-20251126091821328000000003]. +[2025-11-30 19:01:59] [SUCCESS] Deleted a4h639c-controller-default-20251126091821328000000003 +[2025-11-30 19:01:59] [EXECUTE] Deleting Instance Template: a4h639c-login-slurm-login-20251126091812165600000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h639c-login-slurm-login-20251126091812165600000001]. +[2025-11-30 19:02:02] [SUCCESS] Deleted a4h639c-login-slurm-login-20251126091812165600000001 +[2025-11-30 19:02:02] [EXECUTE] Deleting Instance Template: a4h68f2-compute-a4highnodeset-20251125205110303400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h68f2-compute-a4highnodeset-20251125205110303400000002]. +[2025-11-30 19:02:06] [SUCCESS] Deleted a4h68f2-compute-a4highnodeset-20251125205110303400000002 +[2025-11-30 19:02:06] [EXECUTE] Deleting Instance Template: a4h68f2-controller-default-20251125205115747500000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h68f2-controller-default-20251125205115747500000003]. +[2025-11-30 19:02:09] [SUCCESS] Deleted a4h68f2-controller-default-20251125205115747500000003 +[2025-11-30 19:02:09] [EXECUTE] Deleting Instance Template: a4h68f2-login-slurm-login-20251125205107060800000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h68f2-login-slurm-login-20251125205107060800000001]. +[2025-11-30 19:02:12] [SUCCESS] Deleted a4h68f2-login-slurm-login-20251125205107060800000001 +[2025-11-30 19:02:12] [EXECUTE] Deleting Instance Template: a4h6c3b-compute-a4highnodeset-20251117134652173800000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h6c3b-compute-a4highnodeset-20251117134652173800000002]. +[2025-11-30 19:02:15] [SUCCESS] Deleted a4h6c3b-compute-a4highnodeset-20251117134652173800000002 +[2025-11-30 19:02:15] [EXECUTE] Deleting Instance Template: a4h6c3b-controller-default-20251117134657496900000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h6c3b-controller-default-20251117134657496900000003]. +[2025-11-30 19:02:19] [SUCCESS] Deleted a4h6c3b-controller-default-20251117134657496900000003 +[2025-11-30 19:02:19] [EXECUTE] Deleting Instance Template: a4h6c3b-login-slurm-login-20251117134647592200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h6c3b-login-slurm-login-20251117134647592200000001]. +[2025-11-30 19:02:22] [SUCCESS] Deleted a4h6c3b-login-slurm-login-20251117134647592200000001 +[2025-11-30 19:02:22] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:02:22] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:02:24] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:02:24] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:02:24] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:02:24] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:02:27] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:02:27] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:02:30] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:02:30] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:02:30] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:02:30] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:02:31] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:02:31] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:02:31] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:02:31] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:02:31] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:02:31] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:02:31] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:02:31] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:02:31Z (Unix: 1763319751) +[2025-11-30 19:02:31] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:02:34] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:02:34] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:02:36] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:02:36] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:02:36] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:02:36] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:02:36] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:02:36] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:02:39] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:02:39] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:02:41] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:02:41] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:02:44] [INFO] No Global Address found matching criteria. +[2025-11-30 19:02:44] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:02:47] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:02:47] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:02:49] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:02:49] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:02:49] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:02:49] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:02:49] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:52] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:02:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:02:54] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:02:56] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:02:56] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:03:03] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:03:03] [INFO] Time Cutoff (General): 2025-11-30T19:03:03+0000 +[2025-11-30 19:03:03] [INFO] Time Cutoff (Images): 2025-10-01T19:03:03+0000 +[2025-11-30 19:03:03] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:03:03] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:03:04] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:03:06] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:03:06] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:03:08] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:03:08] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:03:11] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:03:11] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:03:11] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:03:11] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:03:11] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:03:11] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:03:11] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc0e2-compute-a4highnodeset-20251126202608471600000002 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc0e2-controller-default-20251126202614479200000003 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc0e2-login-slurm-login-20251126202605221400000001 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc1a3-compute-a4highnodeset-20251127090301343400000002 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc1a3-controller-default-20251127090306826200000003 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc1a3-login-slurm-login-20251127090258428600000001 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hcf79-compute-a4highnodeset-20251119140049364000000002 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hcf79-controller-default-20251119140054728100000003 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hcf79-login-slurm-login-20251119140045167600000001 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4he340-controller-default-20251126120236033300000003 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4he340-login-slurm-login-20251126120226527400000001 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hee35-compute-a4highnodeset-20251119063803576500000002 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hee35-controller-default-20251119063809267100000003 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hee35-login-slurm-login-20251119063800421400000001 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-controller-default-20251116112212875400000003 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-login-slurm-login-20251116112205108200000001 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-controller-default-20251116055930192300000003 (Global) +[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-login-slurm-login-20251116055921750900000001 (Global) +[2025-11-30 19:03:11] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:03:11] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:03:14] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:03:14] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:03:14] [DRY-RUN] Would delete Compute Instance: topology-controller us-central1-a +[2025-11-30 19:03:14] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:03:14] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:03:17] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:03:17] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:03:20] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:03:20] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:03:20] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:03:20] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:03:20] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:03:20] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:03:20] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:03:20] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:03:21] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:03:21] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:03:21] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:03:21] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:03:21Z (Unix: 1763319801) +[2025-11-30 19:03:21] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:03:23] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:03:23] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:03:26] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:03:26] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:03:26] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:03:26] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:03:26] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:03:26] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:03:28] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:03:28] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:03:31] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:03:31] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:03:33] [INFO] No Global Address found matching criteria. +[2025-11-30 19:03:33] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:03:36] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:03:36] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:03:38] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:03:38] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:03:38] [DRY-RUN] Would delete Zonal Disk: topology-controller https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[2025-11-30 19:03:38] [DRY-RUN] Would delete Zonal Disk: topology-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[2025-11-30 19:03:38] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:03:38] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:03:38] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:41] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:03:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:03:43] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:03:45] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:03:45] [INFO] CLEANUP RUN FINISHED +./cleanup.sh: line 712: n: command not found +[2025-11-30 19:05:35] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:05:35] [INFO] Time Cutoff (General): 2025-11-30T19:05:35+0000 +[2025-11-30 19:05:35] [INFO] Time Cutoff (Images): 2025-10-01T19:05:35+0000 +[2025-11-30 19:05:35] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:05:35] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:05:36] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:05:38] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:05:38] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:05:40] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:05:40] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:05:43] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:05:43] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:05:43] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:05:43] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:05:43] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:05:43] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:05:43] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc0e2-compute-a4highnodeset-20251126202608471600000002 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc0e2-controller-default-20251126202614479200000003 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc0e2-login-slurm-login-20251126202605221400000001 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc1a3-compute-a4highnodeset-20251127090301343400000002 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc1a3-controller-default-20251127090306826200000003 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc1a3-login-slurm-login-20251127090258428600000001 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hcf79-compute-a4highnodeset-20251119140049364000000002 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hcf79-controller-default-20251119140054728100000003 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hcf79-login-slurm-login-20251119140045167600000001 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4he340-controller-default-20251126120236033300000003 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4he340-login-slurm-login-20251126120226527400000001 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hee35-compute-a4highnodeset-20251119063803576500000002 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hee35-controller-default-20251119063809267100000003 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hee35-login-slurm-login-20251119063800421400000001 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-controller-default-20251116112212875400000003 (Global) +[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-login-slurm-login-20251116112205108200000001 (Global) +[2025-11-30 19:05:44] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 (Global) +[2025-11-30 19:05:44] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-controller-default-20251116055930192300000003 (Global) +[2025-11-30 19:05:44] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-login-slurm-login-20251116055921750900000001 (Global) +[2025-11-30 19:05:44] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:05:44] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:05:46] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:05:46] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-controller us-central1-a +[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-nodeset-0 us-central1-a +[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-nodeset-1 us-central1-a +[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-nodeset-2 us-central1-a +[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-nodeset-3 us-central1-a +[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-nodeset-4 us-central1-a +[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-slurm-login-001 us-central1-a +[2025-11-30 19:05:46] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:05:46] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:05:49] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:05:49] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:05:52] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:05:52] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:05:52] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:05:52] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:05:52] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:05:52] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:05:52] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:05:52] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:05:53] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:05:53] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:05:53] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:05:53] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:05:53Z (Unix: 1763319953) +[2025-11-30 19:05:53] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:05:55] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:05:55] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:05:58] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:05:58] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:05:58] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:05:58] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:05:58] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:05:58] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:06:00] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:06:00] [INFO] --- Processing: Regional Address (Limit: 20) --- +[2025-11-30 19:06:02] [DRY-RUN] Would delete Regional Address: nat-auto-ip-11875105-1-1764529459120987 us-central1 +[2025-11-30 19:06:02] [INFO] --- Processing: Global Address (Limit: 20) --- +[2025-11-30 19:06:05] [INFO] No Global Address found matching criteria. +[2025-11-30 19:06:05] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:06:07] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:06:07] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:06:10] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:06:10] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-controller https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-nodeset-0 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-nodeset-1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-nodeset-2 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-nodeset-3 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-nodeset-4 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-slurm-login-001 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a +[2025-11-30 19:06:10] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:06:10] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:06:10] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:06:12] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:12] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:12] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:13] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:06:15] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:06:15] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:06:17] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:06:17] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:07:05] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:07:05] [INFO] Time Cutoff (General): 2025-11-30T19:07:05+0000 +[2025-11-30 19:07:05] [INFO] Time Cutoff (Images): 2025-10-01T19:07:05+0000 +[2025-11-30 19:07:05] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:07:05] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:07:05] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:07:08] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:07:08] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:07:10] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:07:10] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:07:13] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:07:13] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:07:13] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:07:13] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:07:13] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:07:13] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:07:13] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc0e2-compute-a4highnodeset-20251126202608471600000002 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc0e2-controller-default-20251126202614479200000003 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc0e2-login-slurm-login-20251126202605221400000001 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc1a3-compute-a4highnodeset-20251127090301343400000002 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc1a3-controller-default-20251127090306826200000003 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc1a3-login-slurm-login-20251127090258428600000001 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hcf79-compute-a4highnodeset-20251119140049364000000002 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hcf79-controller-default-20251119140054728100000003 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hcf79-login-slurm-login-20251119140045167600000001 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4he340-controller-default-20251126120236033300000003 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4he340-login-slurm-login-20251126120226527400000001 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hee35-compute-a4highnodeset-20251119063803576500000002 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hee35-controller-default-20251119063809267100000003 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hee35-login-slurm-login-20251119063800421400000001 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-controller-default-20251116112212875400000003 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-login-slurm-login-20251116112205108200000001 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-controller-default-20251116055930192300000003 (Global) +[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-login-slurm-login-20251116055921750900000001 (Global) +[2025-11-30 19:07:13] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:07:13] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:07:16] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:07:17] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:07:17] [SKIP] topology-controller (Protected Substring) +[2025-11-30 19:07:17] [SKIP] topology-nodeset-0 (Protected Substring) +[2025-11-30 19:07:17] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:07:17] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:07:19] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:07:19] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:07:23] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:07:23] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:07:23] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:07:23] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:07:23] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:07:23] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:07:23] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:07:23] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:07:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:07:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:07:23] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:07:24] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:07:23Z (Unix: 1763320043) +[2025-11-30 19:07:24] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:07:26] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:07:26] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:07:28] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:07:28] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:07:28] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:07:28] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:07:28] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:07:28] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:07:31] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:07:31] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:07:33] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:07:33] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:07:35] [INFO] No Global Address found matching criteria. +[2025-11-30 19:07:35] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:07:38] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:07:38] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:07:40] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:07:40] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:07:40] [SKIP] topology-controller-save (Protected Substring) +[2025-11-30 19:07:40] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:07:40] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:07:40] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:07:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:42] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:43] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:07:45] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:07:45] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:07:47] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:07:47] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:08:16] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:08:16] [INFO] Time Cutoff (General): 2025-11-30T14:08:16+0000 +[2025-11-30 19:08:16] [INFO] Time Cutoff (Images): 2025-10-01T19:08:16+0000 +[2025-11-30 19:08:16] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:08:16] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:08:17] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:08:19] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:08:19] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:08:21] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:08:21] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:08:24] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:08:24] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:08:24] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:08:24] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:08:24] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:08:24] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:08:24] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc0e2-compute-a4highnodeset-20251126202608471600000002 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc0e2-controller-default-20251126202614479200000003 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc0e2-login-slurm-login-20251126202605221400000001 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc1a3-compute-a4highnodeset-20251127090301343400000002 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc1a3-controller-default-20251127090306826200000003 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc1a3-login-slurm-login-20251127090258428600000001 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hcf79-compute-a4highnodeset-20251119140049364000000002 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hcf79-controller-default-20251119140054728100000003 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hcf79-login-slurm-login-20251119140045167600000001 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4he340-controller-default-20251126120236033300000003 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4he340-login-slurm-login-20251126120226527400000001 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hee35-compute-a4highnodeset-20251119063803576500000002 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hee35-controller-default-20251119063809267100000003 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hee35-login-slurm-login-20251119063800421400000001 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-controller-default-20251116112212875400000003 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-login-slurm-login-20251116112205108200000001 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-controller-default-20251116055930192300000003 (Global) +[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-login-slurm-login-20251116055921750900000001 (Global) +[2025-11-30 19:08:24] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:08:24] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:08:27] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:08:27] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:08:27] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:08:27] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:08:30] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:08:30] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:08:33] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:08:33] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:08:33] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:08:33] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:08:33] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:08:33] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:08:33] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:08:33] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:08:34] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:08:34] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:08:34] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:08:34] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:08:34Z (Unix: 1763320114) +[2025-11-30 19:08:34] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:08:36] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:08:36] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:08:38] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:08:38] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:08:38] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:08:38] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:08:39] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:08:39] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:08:41] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:08:41] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:08:43] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:08:43] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:08:46] [INFO] No Global Address found matching criteria. +[2025-11-30 19:08:46] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:08:48] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:08:48] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:08:51] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:08:51] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:08:51] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:08:51] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:08:51] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:54] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:54] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:08:56] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:08:56] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:08:58] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:08:58] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:09:08] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:09:08] [INFO] Time Cutoff (General): 2025-11-30T14:09:08+0000 +[2025-11-30 19:09:08] [INFO] Time Cutoff (Images): 2025-10-01T19:09:08+0000 +[2025-11-30 19:09:09] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:09:09] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:09:09] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:09:11] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:09:11] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:09:13] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:09:13] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:09:16] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:09:16] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:09:16] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:09:16] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:09:16] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:09:16] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:09:16] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:09:16] [EXECUTE] Deleting Instance Template: a4hc0e2-compute-a4highnodeset-20251126202608471600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc0e2-compute-a4highnodeset-20251126202608471600000002]. +[2025-11-30 19:09:19] [SUCCESS] Deleted a4hc0e2-compute-a4highnodeset-20251126202608471600000002 +[2025-11-30 19:09:19] [EXECUTE] Deleting Instance Template: a4hc0e2-controller-default-20251126202614479200000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc0e2-controller-default-20251126202614479200000003]. +[2025-11-30 19:09:22] [SUCCESS] Deleted a4hc0e2-controller-default-20251126202614479200000003 +[2025-11-30 19:09:22] [EXECUTE] Deleting Instance Template: a4hc0e2-login-slurm-login-20251126202605221400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc0e2-login-slurm-login-20251126202605221400000001]. +[2025-11-30 19:09:26] [SUCCESS] Deleted a4hc0e2-login-slurm-login-20251126202605221400000001 +[2025-11-30 19:09:26] [EXECUTE] Deleting Instance Template: a4hc1a3-compute-a4highnodeset-20251127090301343400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc1a3-compute-a4highnodeset-20251127090301343400000002]. +[2025-11-30 19:09:29] [SUCCESS] Deleted a4hc1a3-compute-a4highnodeset-20251127090301343400000002 +[2025-11-30 19:09:29] [EXECUTE] Deleting Instance Template: a4hc1a3-controller-default-20251127090306826200000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc1a3-controller-default-20251127090306826200000003]. +[2025-11-30 19:09:32] [SUCCESS] Deleted a4hc1a3-controller-default-20251127090306826200000003 +[2025-11-30 19:09:32] [EXECUTE] Deleting Instance Template: a4hc1a3-login-slurm-login-20251127090258428600000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc1a3-login-slurm-login-20251127090258428600000001]. +[2025-11-30 19:09:35] [SUCCESS] Deleted a4hc1a3-login-slurm-login-20251127090258428600000001 +[2025-11-30 19:09:35] [EXECUTE] Deleting Instance Template: a4hcf79-compute-a4highnodeset-20251119140049364000000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hcf79-compute-a4highnodeset-20251119140049364000000002]. +[2025-11-30 19:09:38] [SUCCESS] Deleted a4hcf79-compute-a4highnodeset-20251119140049364000000002 +[2025-11-30 19:09:38] [EXECUTE] Deleting Instance Template: a4hcf79-controller-default-20251119140054728100000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hcf79-controller-default-20251119140054728100000003]. +[2025-11-30 19:09:41] [SUCCESS] Deleted a4hcf79-controller-default-20251119140054728100000003 +[2025-11-30 19:09:41] [EXECUTE] Deleting Instance Template: a4hcf79-login-slurm-login-20251119140045167600000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hcf79-login-slurm-login-20251119140045167600000001]. +[2025-11-30 19:09:44] [SUCCESS] Deleted a4hcf79-login-slurm-login-20251119140045167600000001 +[2025-11-30 19:09:44] [EXECUTE] Deleting Instance Template: a4he340-controller-default-20251126120236033300000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4he340-controller-default-20251126120236033300000003]. +[2025-11-30 19:09:47] [SUCCESS] Deleted a4he340-controller-default-20251126120236033300000003 +[2025-11-30 19:09:47] [EXECUTE] Deleting Instance Template: a4he340-login-slurm-login-20251126120226527400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4he340-login-slurm-login-20251126120226527400000001]. +[2025-11-30 19:09:50] [SUCCESS] Deleted a4he340-login-slurm-login-20251126120226527400000001 +[2025-11-30 19:09:50] [EXECUTE] Deleting Instance Template: a4hee35-compute-a4highnodeset-20251119063803576500000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hee35-compute-a4highnodeset-20251119063803576500000002]. +[2025-11-30 19:09:53] [SUCCESS] Deleted a4hee35-compute-a4highnodeset-20251119063803576500000002 +[2025-11-30 19:09:53] [EXECUTE] Deleting Instance Template: a4hee35-controller-default-20251119063809267100000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hee35-controller-default-20251119063809267100000003]. +[2025-11-30 19:09:56] [SUCCESS] Deleted a4hee35-controller-default-20251119063809267100000003 +[2025-11-30 19:09:56] [EXECUTE] Deleting Instance Template: a4hee35-login-slurm-login-20251119063800421400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hee35-login-slurm-login-20251119063800421400000001]. +[2025-11-30 19:09:59] [SUCCESS] Deleted a4hee35-login-slurm-login-20251119063800421400000001 +[2025-11-30 19:09:59] [EXECUTE] Deleting Instance Template: a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002]. +[2025-11-30 19:10:02] [SUCCESS] Deleted a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 +[2025-11-30 19:10:02] [EXECUTE] Deleting Instance Template: a4hf1a46f0-controller-default-20251116112212875400000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf1a46f0-controller-default-20251116112212875400000003]. +[2025-11-30 19:10:06] [SUCCESS] Deleted a4hf1a46f0-controller-default-20251116112212875400000003 +[2025-11-30 19:10:06] [EXECUTE] Deleting Instance Template: a4hf1a46f0-login-slurm-login-20251116112205108200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf1a46f0-login-slurm-login-20251116112205108200000001]. +[2025-11-30 19:10:09] [SUCCESS] Deleted a4hf1a46f0-login-slurm-login-20251116112205108200000001 +[2025-11-30 19:10:09] [EXECUTE] Deleting Instance Template: a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002]. +[2025-11-30 19:10:11] [SUCCESS] Deleted a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 +[2025-11-30 19:10:11] [EXECUTE] Deleting Instance Template: a4hf97fa5b-controller-default-20251116055930192300000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf97fa5b-controller-default-20251116055930192300000003]. +[2025-11-30 19:10:15] [SUCCESS] Deleted a4hf97fa5b-controller-default-20251116055930192300000003 +[2025-11-30 19:10:15] [EXECUTE] Deleting Instance Template: a4hf97fa5b-login-slurm-login-20251116055921750900000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf97fa5b-login-slurm-login-20251116055921750900000001]. +[2025-11-30 19:10:18] [SUCCESS] Deleted a4hf97fa5b-login-slurm-login-20251116055921750900000001 +[2025-11-30 19:10:18] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:10:18] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:10:20] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:10:20] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:10:20] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:10:20] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:10:23] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:10:23] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:10:26] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:10:26] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:10:26] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:10:26] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:10:26] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:10:26] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:10:26] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:10:26] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:10:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:10:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:10:27] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:10:27] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:10:27Z (Unix: 1763320227) +[2025-11-30 19:10:27] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:10:29] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:10:29] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:10:32] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:10:32] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:10:32] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:10:32] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:10:32] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:10:32] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:10:34] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:10:34] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:10:37] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:10:37] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:10:39] [INFO] No Global Address found matching criteria. +[2025-11-30 19:10:39] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:10:41] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:10:41] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:10:44] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:10:44] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:10:44] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:10:44] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:10:44] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:47] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:10:49] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:10:49] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:10:51] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:10:51] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:12:04] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:12:04] [INFO] Time Cutoff (General): 2025-11-30T14:12:04+0000 +[2025-11-30 19:12:04] [INFO] Time Cutoff (Images): 2025-10-01T19:12:04+0000 +[2025-11-30 19:12:04] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:12:04] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:12:04] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:12:07] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:12:07] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:12:09] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:12:09] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:12:12] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:12:12] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:12:12] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:12:12] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:12:12] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:12:12] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:12:12] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4hrrsarth-compute-a4highnodeset-20251112094446322500000002 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4hrrsarth-controller-default-20251112094451660400000003 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4hrrsarth-login-slurm-login-20251112094443045200000001 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4newimgek-compute-a4highnodeset-20251121125823424400000002 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4newimgek-controller-default-20251121125828262300000003 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4newimgek-login-slurm-login-20251121125820468400000001 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4oldimgek-compute-a4highnodeset-20251121120827776000000003 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4oldimgek-controller-default-20251121120824903100000001 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4oldimgek-login-slurm-login-20251121120824904000000002 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xdemo-compute-a4xnodeset-20250904073238588700000003 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xdemo-controller-default-20250904073238579200000002 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xdemo-login-slurm-login-20250904073238576200000001 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xneel-compute-a4xnodeset-20251114072028106100000003 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xneel-controller-default-20251114072028094500000002 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xneel-login-slurm-login-20251114072028092100000001 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xqc-compute-a4xnodeset-20250925042600882300000003 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xqc-controller-default-20250925042600869000000001 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xqc-login-slurm-login-20250925042600873800000002 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a58b93slur-compute-nodeset-20250808184941566500000001 (Global) +[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a7f7bcslur-compute-nodeset-20250804215637357700000001 (Global) +[2025-11-30 19:12:12] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:12:12] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:12:14] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:12:14] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:12:14] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:12:15] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:12:17] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:12:17] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:12:20] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:12:20] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:12:20] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:12:20] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:12:21] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:12:21] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:12:21] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:12:21] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:12:21] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:12:21] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:12:21] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:12:21] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:12:21Z (Unix: 1763320341) +[2025-11-30 19:12:21] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:12:23] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:12:23] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:12:26] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:12:26] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:12:26] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:12:26] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:12:26] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:12:26] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:12:28] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:12:28] [INFO] --- Processing: Regional Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:12:31] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:12:31] [INFO] --- Processing: Global Address (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : creationTimestamp, region +[2025-11-30 19:12:33] [INFO] No Global Address found matching criteria. +[2025-11-30 19:12:33] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:12:35] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:12:35] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:12:38] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:12:38] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:12:38] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:12:38] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:12:38] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:41] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:12:43] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:12:43] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:12:45] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:12:45] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:13:49] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:13:49] [INFO] Time Cutoff (General): 2025-11-30T14:13:49+0000 +[2025-11-30 19:13:49] [INFO] Time Cutoff (Images): 2025-10-01T19:13:49+0000 +[2025-11-30 19:13:49] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:13:49] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:13:50] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:13:52] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:13:52] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:13:54] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:13:54] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:13:57] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:13:57] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:13:57] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:13:57] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:13:57] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:13:57] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:13:57] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:13:57] [EXECUTE] Deleting Instance Template: a4hrrsarth-compute-a4highnodeset-20251112094446322500000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hrrsarth-compute-a4highnodeset-20251112094446322500000002]. +[2025-11-30 19:14:00] [SUCCESS] Deleted a4hrrsarth-compute-a4highnodeset-20251112094446322500000002 +[2025-11-30 19:14:00] [EXECUTE] Deleting Instance Template: a4hrrsarth-controller-default-20251112094451660400000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hrrsarth-controller-default-20251112094451660400000003]. +[2025-11-30 19:14:03] [SUCCESS] Deleted a4hrrsarth-controller-default-20251112094451660400000003 +[2025-11-30 19:14:03] [EXECUTE] Deleting Instance Template: a4hrrsarth-login-slurm-login-20251112094443045200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hrrsarth-login-slurm-login-20251112094443045200000001]. +[2025-11-30 19:14:06] [SUCCESS] Deleted a4hrrsarth-login-slurm-login-20251112094443045200000001 +[2025-11-30 19:14:06] [EXECUTE] Deleting Instance Template: a4newimgek-compute-a4highnodeset-20251121125823424400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4newimgek-compute-a4highnodeset-20251121125823424400000002]. +[2025-11-30 19:14:09] [SUCCESS] Deleted a4newimgek-compute-a4highnodeset-20251121125823424400000002 +[2025-11-30 19:14:09] [EXECUTE] Deleting Instance Template: a4newimgek-controller-default-20251121125828262300000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4newimgek-controller-default-20251121125828262300000003]. +[2025-11-30 19:14:12] [SUCCESS] Deleted a4newimgek-controller-default-20251121125828262300000003 +[2025-11-30 19:14:12] [EXECUTE] Deleting Instance Template: a4newimgek-login-slurm-login-20251121125820468400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4newimgek-login-slurm-login-20251121125820468400000001]. +[2025-11-30 19:14:15] [SUCCESS] Deleted a4newimgek-login-slurm-login-20251121125820468400000001 +[2025-11-30 19:14:15] [EXECUTE] Deleting Instance Template: a4oldimgek-compute-a4highnodeset-20251121120827776000000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4oldimgek-compute-a4highnodeset-20251121120827776000000003]. +[2025-11-30 19:14:19] [SUCCESS] Deleted a4oldimgek-compute-a4highnodeset-20251121120827776000000003 +[2025-11-30 19:14:19] [EXECUTE] Deleting Instance Template: a4oldimgek-controller-default-20251121120824903100000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4oldimgek-controller-default-20251121120824903100000001]. +[2025-11-30 19:14:22] [SUCCESS] Deleted a4oldimgek-controller-default-20251121120824903100000001 +[2025-11-30 19:14:22] [EXECUTE] Deleting Instance Template: a4oldimgek-login-slurm-login-20251121120824904000000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4oldimgek-login-slurm-login-20251121120824904000000002]. +[2025-11-30 19:14:25] [SUCCESS] Deleted a4oldimgek-login-slurm-login-20251121120824904000000002 +[2025-11-30 19:14:25] [EXECUTE] Deleting Instance Template: a4xdemo-compute-a4xnodeset-20250904073238588700000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xdemo-compute-a4xnodeset-20250904073238588700000003]. +[2025-11-30 19:14:28] [SUCCESS] Deleted a4xdemo-compute-a4xnodeset-20250904073238588700000003 +[2025-11-30 19:14:28] [EXECUTE] Deleting Instance Template: a4xdemo-controller-default-20250904073238579200000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xdemo-controller-default-20250904073238579200000002]. +[2025-11-30 19:14:31] [SUCCESS] Deleted a4xdemo-controller-default-20250904073238579200000002 +[2025-11-30 19:14:31] [EXECUTE] Deleting Instance Template: a4xdemo-login-slurm-login-20250904073238576200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xdemo-login-slurm-login-20250904073238576200000001]. +[2025-11-30 19:14:34] [SUCCESS] Deleted a4xdemo-login-slurm-login-20250904073238576200000001 +[2025-11-30 19:14:34] [EXECUTE] Deleting Instance Template: a4xneel-compute-a4xnodeset-20251114072028106100000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xneel-compute-a4xnodeset-20251114072028106100000003]. +[2025-11-30 19:14:37] [SUCCESS] Deleted a4xneel-compute-a4xnodeset-20251114072028106100000003 +[2025-11-30 19:14:37] [EXECUTE] Deleting Instance Template: a4xneel-controller-default-20251114072028094500000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xneel-controller-default-20251114072028094500000002]. +[2025-11-30 19:14:40] [SUCCESS] Deleted a4xneel-controller-default-20251114072028094500000002 +[2025-11-30 19:14:40] [EXECUTE] Deleting Instance Template: a4xneel-login-slurm-login-20251114072028092100000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xneel-login-slurm-login-20251114072028092100000001]. +[2025-11-30 19:14:43] [SUCCESS] Deleted a4xneel-login-slurm-login-20251114072028092100000001 +[2025-11-30 19:14:43] [EXECUTE] Deleting Instance Template: a4xqc-compute-a4xnodeset-20250925042600882300000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xqc-compute-a4xnodeset-20250925042600882300000003]. +[2025-11-30 19:14:46] [SUCCESS] Deleted a4xqc-compute-a4xnodeset-20250925042600882300000003 +[2025-11-30 19:14:46] [EXECUTE] Deleting Instance Template: a4xqc-controller-default-20250925042600869000000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xqc-controller-default-20250925042600869000000001]. +[2025-11-30 19:14:49] [SUCCESS] Deleted a4xqc-controller-default-20250925042600869000000001 +[2025-11-30 19:14:49] [EXECUTE] Deleting Instance Template: a4xqc-login-slurm-login-20250925042600873800000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xqc-login-slurm-login-20250925042600873800000002]. +[2025-11-30 19:14:52] [SUCCESS] Deleted a4xqc-login-slurm-login-20250925042600873800000002 +[2025-11-30 19:14:52] [EXECUTE] Deleting Instance Template: a58b93slur-compute-nodeset-20250808184941566500000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a58b93slur-compute-nodeset-20250808184941566500000001]. +[2025-11-30 19:14:56] [SUCCESS] Deleted a58b93slur-compute-nodeset-20250808184941566500000001 +[2025-11-30 19:14:56] [EXECUTE] Deleting Instance Template: a7f7bcslur-compute-nodeset-20250804215637357700000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a7f7bcslur-compute-nodeset-20250804215637357700000001]. +[2025-11-30 19:14:59] [SUCCESS] Deleted a7f7bcslur-compute-nodeset-20250804215637357700000001 +[2025-11-30 19:14:59] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:14:59] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:15:01] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:15:01] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:15:01] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:15:01] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:15:04] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:15:04] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:15:07] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:15:07] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:15:07] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:15:07] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:15:07] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:15:07] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:15:07] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:15:07] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:15:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:15:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:15:08] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:15:08] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:15:08Z (Unix: 1763320508) +[2025-11-30 19:15:08] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:15:10] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:15:10] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:15:12] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:15:12] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:15:12] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:15:12] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:15:12] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:15:12] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:15:15] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:15:15] [INFO] --- Processing: Regional Address (Limit: 20) --- +[2025-11-30 19:15:17] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:15:17] [INFO] --- Processing: Global Address (Limit: 20) --- +[2025-11-30 19:15:19] [INFO] No Global Address found matching criteria. +[2025-11-30 19:15:19] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:15:22] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:15:22] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:15:24] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:15:24] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:15:24] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:15:24] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:15:24] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:27] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:15:29] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:15:29] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:15:31] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:15:31] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:15:41] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:15:41] [INFO] Time Cutoff (General): 2025-11-30T14:15:41+0000 +[2025-11-30 19:15:41] [INFO] Time Cutoff (Images): 2025-10-01T19:15:41+0000 +[2025-11-30 19:15:41] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:15:41] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:15:42] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:15:44] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:15:44] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +[2025-11-30 19:15:46] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:15:46] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:15:49] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:15:49] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:15:49] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:15:49] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:15:49] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:15:49] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:15:49] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: a7f7bcslur-controller-default-20250804215646842800000003 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: a7f7bcslur-login-slurm-login-20250804215637381200000002 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: a94slurmfl-compute-nodeset-20251018073312378100000002 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: aa5daslurm-compute-nodeset-20250710053732511400000001 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: acbslurmsi-compute-nodeset-20250815192152759600000002 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: aslurmflex-compute-nodeset-20251007191438377100000001 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: b168slurmf-compute-nodeset-20251120173723559000000001 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: batch-job-instance-template-20250901212237920900000001 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: batch-job-instance-template-20250912070019961500000001 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: be1slurmfl-compute-nodeset-20251118223535177700000001 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: bfa462slur-compute-nodeset-20250912051104900400000001 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: bfa462slur-controller-default-20250912051114475200000003 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: bfa462slur-login-slurm-login-20250912051104953500000002 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: buildslurm-compute-debugnodeset-20251030080636567600000001 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: buildslurm-controller-default-20251030080646114800000002 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: c0aslurmfl-compute-nodeset-20251010073733464500000002 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: c17slurmfl-compute-nodeset-20250911220802714700000001 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: c2dtest7-compute-c2dnodeset-20250926070704394500000003 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: c2dtest7-controller-default-20250926070704365100000001 (Global) +[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: c2dtest7-login-slurm-login-20250926070704373700000002 (Global) +[2025-11-30 19:15:49] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:15:49] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:15:52] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:15:52] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:15:52] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:15:52] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:15:54] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:15:54] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:15:57] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:15:57] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:15:57] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:15:57] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:15:58] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:15:58] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:15:58] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:15:58] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:15:58] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:15:58] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:15:58] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:15:58] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:15:58Z (Unix: 1763320558) +[2025-11-30 19:15:58] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:16:00] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:16:00] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:16:03] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:16:03] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:16:03] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:16:03] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:16:03] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:16:03] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:16:05] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:16:05] [INFO] --- Processing: Regional Address (Limit: 20) --- +[2025-11-30 19:16:08] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:16:08] [INFO] --- Processing: Global Address (Limit: 20) --- +[2025-11-30 19:16:10] [INFO] No Global Address found matching criteria. +[2025-11-30 19:16:10] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:16:13] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:16:13] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:16:15] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:16:15] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:16:15] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:16:15] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:16:15] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:18] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:16:20] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:16:20] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:16:22] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:16:22] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:16:46] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:16:46] [INFO] Time Cutoff (General): 2025-11-30T14:16:46+0000 +[2025-11-30 19:16:46] [INFO] Time Cutoff (Images): 2025-10-01T19:16:46+0000 +[2025-11-30 19:16:46] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:16:46] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:16:47] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:16:49] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:16:49] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +[2025-11-30 19:16:51] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:16:51] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:16:54] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:16:54] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:16:54] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:16:54] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:16:54] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:16:54] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:16:54] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: a7f7bcslur-controller-default-20250804215646842800000003 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: a7f7bcslur-login-slurm-login-20250804215637381200000002 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: a94slurmfl-compute-nodeset-20251018073312378100000002 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: aa5daslurm-compute-nodeset-20250710053732511400000001 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: acbslurmsi-compute-nodeset-20250815192152759600000002 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: aslurmflex-compute-nodeset-20251007191438377100000001 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: b168slurmf-compute-nodeset-20251120173723559000000001 (Global) +[2025-11-30 19:16:54] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:16:54] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: be1slurmfl-compute-nodeset-20251118223535177700000001 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: bfa462slur-compute-nodeset-20250912051104900400000001 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: bfa462slur-controller-default-20250912051114475200000003 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: bfa462slur-login-slurm-login-20250912051104953500000002 (Global) +[2025-11-30 19:16:54] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:16:54] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c0aslurmfl-compute-nodeset-20251010073733464500000002 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c17slurmfl-compute-nodeset-20250911220802714700000001 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c2dtest7-compute-c2dnodeset-20250926070704394500000003 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c2dtest7-controller-default-20250926070704365100000001 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c2dtest7-login-slurm-login-20250926070704373700000002 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c379slurms-compute-nodeset-20250815194346833100000002 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c379slurms-controller-default-20250815194356672000000003 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c379slurms-login-slurm-login-20250815194346831100000001 (Global) +[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c52cb1fa4h-compute-a4highnodeset-20251107160458966300000002 (Global) +[2025-11-30 19:16:54] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:16:54] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:16:56] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:16:56] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:16:56] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:16:56] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:16:59] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:16:59] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:17:02] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:17:02] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:17:02] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:17:02] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:17:03] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:17:03] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:17:03] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:17:03] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:17:03] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:17:03] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:17:03] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:17:03] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:17:03Z (Unix: 1763320623) +[2025-11-30 19:17:03] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:17:06] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:17:06] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:17:08] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:17:08] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:17:08] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:17:08] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:17:08] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:17:08] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:17:11] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:17:11] [INFO] --- Processing: Regional Address (Limit: 20) --- +[2025-11-30 19:17:13] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:17:13] [INFO] --- Processing: Global Address (Limit: 20) --- +[2025-11-30 19:17:15] [INFO] No Global Address found matching criteria. +[2025-11-30 19:17:15] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:17:18] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:17:18] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:17:20] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:17:20] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:17:20] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:17:20] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:17:20] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:23] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:17:26] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:17:26] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:17:27] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:17:27] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:17:56] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:17:56] [INFO] Time Cutoff (General): 2025-11-30T14:17:56+0000 +[2025-11-30 19:17:56] [INFO] Time Cutoff (Images): 2025-10-01T19:17:56+0000 +[2025-11-30 19:17:56] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:17:57] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:17:57] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:18:00] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:18:00] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +[2025-11-30 19:18:02] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:18:02] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:18:05] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:18:05] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:18:05] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:18:05] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:18:05] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:18:05] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:18:05] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:18:05] [EXECUTE] Deleting Instance Template: a7f7bcslur-controller-default-20250804215646842800000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a7f7bcslur-controller-default-20250804215646842800000003]. +[2025-11-30 19:18:08] [SUCCESS] Deleted a7f7bcslur-controller-default-20250804215646842800000003 +[2025-11-30 19:18:08] [EXECUTE] Deleting Instance Template: a7f7bcslur-login-slurm-login-20250804215637381200000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a7f7bcslur-login-slurm-login-20250804215637381200000002]. +[2025-11-30 19:18:11] [SUCCESS] Deleted a7f7bcslur-login-slurm-login-20250804215637381200000002 +[2025-11-30 19:18:11] [EXECUTE] Deleting Instance Template: a94slurmfl-compute-nodeset-20251018073312378100000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a94slurmfl-compute-nodeset-20251018073312378100000002]. +[2025-11-30 19:18:14] [SUCCESS] Deleted a94slurmfl-compute-nodeset-20251018073312378100000002 +[2025-11-30 19:18:14] [EXECUTE] Deleting Instance Template: aa5daslurm-compute-nodeset-20250710053732511400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/aa5daslurm-compute-nodeset-20250710053732511400000001]. +[2025-11-30 19:18:17] [SUCCESS] Deleted aa5daslurm-compute-nodeset-20250710053732511400000001 +[2025-11-30 19:18:17] [EXECUTE] Deleting Instance Template: acbslurmsi-compute-nodeset-20250815192152759600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/acbslurmsi-compute-nodeset-20250815192152759600000002]. +[2025-11-30 19:18:20] [SUCCESS] Deleted acbslurmsi-compute-nodeset-20250815192152759600000002 +[2025-11-30 19:18:20] [EXECUTE] Deleting Instance Template: aslurmflex-compute-nodeset-20251007191438377100000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/aslurmflex-compute-nodeset-20251007191438377100000001]. +[2025-11-30 19:18:23] [SUCCESS] Deleted aslurmflex-compute-nodeset-20251007191438377100000001 +[2025-11-30 19:18:23] [EXECUTE] Deleting Instance Template: b168slurmf-compute-nodeset-20251120173723559000000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/b168slurmf-compute-nodeset-20251120173723559000000001]. +[2025-11-30 19:18:27] [SUCCESS] Deleted b168slurmf-compute-nodeset-20251120173723559000000001 +[2025-11-30 19:18:27] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:18:27] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:18:27] [EXECUTE] Deleting Instance Template: be1slurmfl-compute-nodeset-20251118223535177700000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/be1slurmfl-compute-nodeset-20251118223535177700000001]. +[2025-11-30 19:18:30] [SUCCESS] Deleted be1slurmfl-compute-nodeset-20251118223535177700000001 +[2025-11-30 19:18:30] [EXECUTE] Deleting Instance Template: bfa462slur-compute-nodeset-20250912051104900400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/bfa462slur-compute-nodeset-20250912051104900400000001]. +[2025-11-30 19:18:33] [SUCCESS] Deleted bfa462slur-compute-nodeset-20250912051104900400000001 +[2025-11-30 19:18:33] [EXECUTE] Deleting Instance Template: bfa462slur-controller-default-20250912051114475200000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/bfa462slur-controller-default-20250912051114475200000003]. +[2025-11-30 19:18:36] [SUCCESS] Deleted bfa462slur-controller-default-20250912051114475200000003 +[2025-11-30 19:18:36] [EXECUTE] Deleting Instance Template: bfa462slur-login-slurm-login-20250912051104953500000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/bfa462slur-login-slurm-login-20250912051104953500000002]. +[2025-11-30 19:18:39] [SUCCESS] Deleted bfa462slur-login-slurm-login-20250912051104953500000002 +[2025-11-30 19:18:39] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:18:39] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:18:39] [EXECUTE] Deleting Instance Template: c0aslurmfl-compute-nodeset-20251010073733464500000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c0aslurmfl-compute-nodeset-20251010073733464500000002]. +[2025-11-30 19:18:43] [SUCCESS] Deleted c0aslurmfl-compute-nodeset-20251010073733464500000002 +[2025-11-30 19:18:43] [EXECUTE] Deleting Instance Template: c17slurmfl-compute-nodeset-20250911220802714700000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c17slurmfl-compute-nodeset-20250911220802714700000001]. +[2025-11-30 19:18:46] [SUCCESS] Deleted c17slurmfl-compute-nodeset-20250911220802714700000001 +[2025-11-30 19:18:46] [EXECUTE] Deleting Instance Template: c2dtest7-compute-c2dnodeset-20250926070704394500000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c2dtest7-compute-c2dnodeset-20250926070704394500000003]. +[2025-11-30 19:18:49] [SUCCESS] Deleted c2dtest7-compute-c2dnodeset-20250926070704394500000003 +[2025-11-30 19:18:49] [EXECUTE] Deleting Instance Template: c2dtest7-controller-default-20250926070704365100000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c2dtest7-controller-default-20250926070704365100000001]. +[2025-11-30 19:18:53] [SUCCESS] Deleted c2dtest7-controller-default-20250926070704365100000001 +[2025-11-30 19:18:53] [EXECUTE] Deleting Instance Template: c2dtest7-login-slurm-login-20250926070704373700000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c2dtest7-login-slurm-login-20250926070704373700000002]. +[2025-11-30 19:18:56] [SUCCESS] Deleted c2dtest7-login-slurm-login-20250926070704373700000002 +[2025-11-30 19:18:56] [EXECUTE] Deleting Instance Template: c379slurms-compute-nodeset-20250815194346833100000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c379slurms-compute-nodeset-20250815194346833100000002]. +[2025-11-30 19:18:59] [SUCCESS] Deleted c379slurms-compute-nodeset-20250815194346833100000002 +[2025-11-30 19:18:59] [EXECUTE] Deleting Instance Template: c379slurms-controller-default-20250815194356672000000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c379slurms-controller-default-20250815194356672000000003]. +[2025-11-30 19:19:02] [SUCCESS] Deleted c379slurms-controller-default-20250815194356672000000003 +[2025-11-30 19:19:02] [EXECUTE] Deleting Instance Template: c379slurms-login-slurm-login-20250815194346831100000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c379slurms-login-slurm-login-20250815194346831100000001]. +[2025-11-30 19:19:06] [SUCCESS] Deleted c379slurms-login-slurm-login-20250815194346831100000001 +[2025-11-30 19:19:06] [EXECUTE] Deleting Instance Template: c52cb1fa4h-compute-a4highnodeset-20251107160458966300000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c52cb1fa4h-compute-a4highnodeset-20251107160458966300000002]. +[2025-11-30 19:19:09] [SUCCESS] Deleted c52cb1fa4h-compute-a4highnodeset-20251107160458966300000002 +[2025-11-30 19:19:09] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:19:09] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:19:12] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:19:12] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:19:12] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:19:12] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +WARNING: The following filter keys were not present in any resource : createTime +[2025-11-30 19:19:14] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:19:14] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:19:17] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:19:17] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:19:18] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:19:18] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:19:18] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:19:18] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:19:18] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:19:18] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:19:18] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:19:18] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:19:18] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:19:18] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:19:18Z (Unix: 1763320758) +[2025-11-30 19:19:18] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:19:21] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:19:21] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:19:23] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:19:23] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:19:23] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:19:23] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:19:23] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:19:23] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:19:26] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:19:26] [INFO] --- Processing: Regional Address (Limit: 20) --- +[2025-11-30 19:19:28] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:19:28] [INFO] --- Processing: Global Address (Limit: 20) --- +[2025-11-30 19:19:30] [INFO] No Global Address found matching criteria. +[2025-11-30 19:19:30] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:19:33] [INFO] Finished processing VPC Peerings. 0 peerings actioned. +[2025-11-30 19:19:33] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:19:35] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:19:35] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:19:35] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:19:35] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:19:35] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:38] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:19:41] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:19:41] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:19:43] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:19:43] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:19:52] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:19:52] [INFO] Time Cutoff (General): 2025-11-30T14:19:52+0000 +[2025-11-30 19:19:52] [INFO] Time Cutoff (Images): 2025-10-01T19:19:52+0000 +[2025-11-30 19:19:52] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:19:52] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:19:53] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:19:55] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:19:55] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +[2025-11-30 19:19:57] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:19:57] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:20:00] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:20:00] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:20:00] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:20:00] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:20:00] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:20:00] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:20:00] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:20:00] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:20:00] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:20:00] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:20:00] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: ca2slurmfl-compute-nodeset-20251021163807550000000002 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: ccslurmfle-compute-nodeset-20251023183424836600000002 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: ce64slurms-compute-nodeset-20250821083541488100000002 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: ce64slurms-controller-default-20250821083551070400000003 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: ce64slurms-login-slurm-login-20250821083541466900000001 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3eslurmsi-compute-nodeset-20250804164233979200000001 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3eslurmsi-controller-default-20250804164243493900000003 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3eslurmsi-login-slurm-login-20250804164233980900000002 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3slurmsim-compute-nodeset-20250801211726043400000001 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3slurmsim-controller-default-20250801211735786100000003 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3slurmsim-login-slurm-login-20250801211726053600000002 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d72c8slurm-compute-nodeset-20251121180541385700000002 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d72c8slurm-controller-default-20251121180552318500000003 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d72c8slurm-login-slurm-login-20251121180541344900000001 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: de3580slur-compute-nodeset-20250725053828889300000001 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: de3580slur-controller-default-20250725053838683600000003 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: de3580slur-login-slurm-login-20250725053828911000000002 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: deb6slurmf-compute-nodeset-20250612173759618500000002 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: dynpoc-compute-computenodeset-20251015002137498100000004 (Global) +[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: dynpoc-compute-debugnodeset-20251015002137496700000003 (Global) +[2025-11-30 19:20:00] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:20:00] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:20:03] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:20:03] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:20:03] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:20:03] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +[2025-11-30 19:20:06] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:20:06] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:20:09] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:20:09] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:20:09] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:20:09] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:20:09] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:20:09] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:20:10] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:20:10] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:20:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:20:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:20:10] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:20:10] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:20:10Z (Unix: 1763320810) +[2025-11-30 19:20:10] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:20:12] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:20:12] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:20:15] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:20:15] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:20:15] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:20:15] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:20:15] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:20:15] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:20:17] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:20:17] [INFO] --- Processing: Regional Address (Limit: 20) --- +[2025-11-30 19:20:20] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:20:20] [INFO] --- Processing: Global Address (Limit: 20) --- +[2025-11-30 19:20:22] [INFO] No Global Address found matching criteria. +[2025-11-30 19:20:22] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:20:25] [INFO] [ACTION] Deleting Service Networking peering on: hpc-enterprise-slurm-v6 +[2025-11-30 19:20:25] [DRY-RUN] Would delete Service Peering: servicenetworking-googleapis-com (Network: hpc-enterprise-slurm-v6) +[2025-11-30 19:20:25] [INFO] Finished processing VPC Peerings. 1 peerings actioned. +[2025-11-30 19:20:25] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:20:28] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:20:28] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:20:28] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:20:28] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:20:28] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:31] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:20:33] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:20:33] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:20:35] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:20:35] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:21:16] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:21:16] [INFO] Time Cutoff (General): 2025-11-30T14:21:16+0000 +[2025-11-30 19:21:16] [INFO] Time Cutoff (Images): 2025-10-01T19:21:16+0000 +[2025-11-30 19:21:16] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:21:16] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:21:17] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:21:19] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:21:19] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +[2025-11-30 19:21:21] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:21:21] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:21:24] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:21:24] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:21:24] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:21:24] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:21:24] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:21:24] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:21:24] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:21:24] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:21:24] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:21:24] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:21:24] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:21:24] [EXECUTE] Deleting Instance Template: ca2slurmfl-compute-nodeset-20251021163807550000000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ca2slurmfl-compute-nodeset-20251021163807550000000002]. +[2025-11-30 19:21:27] [SUCCESS] Deleted ca2slurmfl-compute-nodeset-20251021163807550000000002 +[2025-11-30 19:21:27] [EXECUTE] Deleting Instance Template: ccslurmfle-compute-nodeset-20251023183424836600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ccslurmfle-compute-nodeset-20251023183424836600000002]. +[2025-11-30 19:21:30] [SUCCESS] Deleted ccslurmfle-compute-nodeset-20251023183424836600000002 +[2025-11-30 19:21:30] [EXECUTE] Deleting Instance Template: ce64slurms-compute-nodeset-20250821083541488100000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ce64slurms-compute-nodeset-20250821083541488100000002]. +[2025-11-30 19:21:33] [SUCCESS] Deleted ce64slurms-compute-nodeset-20250821083541488100000002 +[2025-11-30 19:21:33] [EXECUTE] Deleting Instance Template: ce64slurms-controller-default-20250821083551070400000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ce64slurms-controller-default-20250821083551070400000003]. +[2025-11-30 19:21:36] [SUCCESS] Deleted ce64slurms-controller-default-20250821083551070400000003 +[2025-11-30 19:21:36] [EXECUTE] Deleting Instance Template: ce64slurms-login-slurm-login-20250821083541466900000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ce64slurms-login-slurm-login-20250821083541466900000001]. +[2025-11-30 19:21:39] [SUCCESS] Deleted ce64slurms-login-slurm-login-20250821083541466900000001 +[2025-11-30 19:21:39] [EXECUTE] Deleting Instance Template: d3eslurmsi-compute-nodeset-20250804164233979200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3eslurmsi-compute-nodeset-20250804164233979200000001]. +[2025-11-30 19:21:43] [SUCCESS] Deleted d3eslurmsi-compute-nodeset-20250804164233979200000001 +[2025-11-30 19:21:43] [EXECUTE] Deleting Instance Template: d3eslurmsi-controller-default-20250804164243493900000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3eslurmsi-controller-default-20250804164243493900000003]. +[2025-11-30 19:21:46] [SUCCESS] Deleted d3eslurmsi-controller-default-20250804164243493900000003 +[2025-11-30 19:21:46] [EXECUTE] Deleting Instance Template: d3eslurmsi-login-slurm-login-20250804164233980900000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3eslurmsi-login-slurm-login-20250804164233980900000002]. +[2025-11-30 19:21:49] [SUCCESS] Deleted d3eslurmsi-login-slurm-login-20250804164233980900000002 +[2025-11-30 19:21:49] [EXECUTE] Deleting Instance Template: d3slurmsim-compute-nodeset-20250801211726043400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3slurmsim-compute-nodeset-20250801211726043400000001]. +[2025-11-30 19:21:52] [SUCCESS] Deleted d3slurmsim-compute-nodeset-20250801211726043400000001 +[2025-11-30 19:21:52] [EXECUTE] Deleting Instance Template: d3slurmsim-controller-default-20250801211735786100000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3slurmsim-controller-default-20250801211735786100000003]. +[2025-11-30 19:21:55] [SUCCESS] Deleted d3slurmsim-controller-default-20250801211735786100000003 +[2025-11-30 19:21:55] [EXECUTE] Deleting Instance Template: d3slurmsim-login-slurm-login-20250801211726053600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3slurmsim-login-slurm-login-20250801211726053600000002]. +[2025-11-30 19:21:58] [SUCCESS] Deleted d3slurmsim-login-slurm-login-20250801211726053600000002 +[2025-11-30 19:21:58] [EXECUTE] Deleting Instance Template: d72c8slurm-compute-nodeset-20251121180541385700000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d72c8slurm-compute-nodeset-20251121180541385700000002]. +[2025-11-30 19:22:01] [SUCCESS] Deleted d72c8slurm-compute-nodeset-20251121180541385700000002 +[2025-11-30 19:22:01] [EXECUTE] Deleting Instance Template: d72c8slurm-controller-default-20251121180552318500000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d72c8slurm-controller-default-20251121180552318500000003]. +[2025-11-30 19:22:04] [SUCCESS] Deleted d72c8slurm-controller-default-20251121180552318500000003 +[2025-11-30 19:22:04] [EXECUTE] Deleting Instance Template: d72c8slurm-login-slurm-login-20251121180541344900000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d72c8slurm-login-slurm-login-20251121180541344900000001]. +[2025-11-30 19:22:07] [SUCCESS] Deleted d72c8slurm-login-slurm-login-20251121180541344900000001 +[2025-11-30 19:22:07] [EXECUTE] Deleting Instance Template: de3580slur-compute-nodeset-20250725053828889300000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/de3580slur-compute-nodeset-20250725053828889300000001]. +[2025-11-30 19:22:10] [SUCCESS] Deleted de3580slur-compute-nodeset-20250725053828889300000001 +[2025-11-30 19:22:10] [EXECUTE] Deleting Instance Template: de3580slur-controller-default-20250725053838683600000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/de3580slur-controller-default-20250725053838683600000003]. +[2025-11-30 19:22:13] [SUCCESS] Deleted de3580slur-controller-default-20250725053838683600000003 +[2025-11-30 19:22:13] [EXECUTE] Deleting Instance Template: de3580slur-login-slurm-login-20250725053828911000000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/de3580slur-login-slurm-login-20250725053828911000000002]. +[2025-11-30 19:22:16] [SUCCESS] Deleted de3580slur-login-slurm-login-20250725053828911000000002 +[2025-11-30 19:22:16] [EXECUTE] Deleting Instance Template: deb6slurmf-compute-nodeset-20250612173759618500000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/deb6slurmf-compute-nodeset-20250612173759618500000002]. +[2025-11-30 19:22:19] [SUCCESS] Deleted deb6slurmf-compute-nodeset-20250612173759618500000002 +[2025-11-30 19:22:19] [EXECUTE] Deleting Instance Template: dynpoc-compute-computenodeset-20251015002137498100000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/dynpoc-compute-computenodeset-20251015002137498100000004]. +[2025-11-30 19:22:23] [SUCCESS] Deleted dynpoc-compute-computenodeset-20251015002137498100000004 +[2025-11-30 19:22:23] [EXECUTE] Deleting Instance Template: dynpoc-compute-debugnodeset-20251015002137496700000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/dynpoc-compute-debugnodeset-20251015002137496700000003]. +[2025-11-30 19:22:26] [SUCCESS] Deleted dynpoc-compute-debugnodeset-20251015002137496700000003 +[2025-11-30 19:22:26] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:22:26] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:22:28] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:22:28] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:22:28] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:22:28] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +[2025-11-30 19:22:31] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:22:31] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:22:34] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:22:34] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:22:34] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:22:34] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:22:34] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:22:34] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:22:34] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:22:34] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:22:35] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:22:35] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:22:35] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:22:35] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:22:35Z (Unix: 1763320955) +[2025-11-30 19:22:35] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:22:37] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:22:37] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:22:40] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:22:40] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:22:40] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:22:40] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:22:40] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:22:40] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:22:42] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:22:42] [INFO] --- Processing: Regional Address (Limit: 20) --- +[2025-11-30 19:22:45] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:22:45] [INFO] --- Processing: Global Address (Limit: 20) --- +[2025-11-30 19:22:47] [INFO] No Global Address found matching criteria. +[2025-11-30 19:22:47] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:22:49] [INFO] [ACTION] Deleting Service Networking peering on: hpc-enterprise-slurm-v6 +[2025-11-30 19:22:49] [EXECUTE] Deleting Service Peering: servicenetworking-googleapis-com (Network: hpc-enterprise-slurm-v6) +ERROR: (gcloud.services.vpc-peerings.delete) The operation "operations/dcf.p40-508417052821-756785e9-1aea-4f3a-b7df-d81c1cb8c5cd" resulted in a failure "Failed to delete connection; Producer services (e.g. CloudSQL, Cloud Memstore, etc.) are still using this connection. +Help Token: AXcLsyCsa6BMQ5F6c1hxIFPD8mjSOLxYM2QIXmOSrXbB3xGvT_0N_wHZ5T8h59ZN4tAIsXbZYJMbUOA8_D9BSf-69dS_M3lvNEO5FdIEhC4dFWyZ". +Details: "[>, >, >]>>]>>>]>, >, >, >]>]". +[2025-11-30 19:23:05] [ERROR] Failed to delete servicenetworking-googleapis-com +[2025-11-30 19:23:05] [INFO] Finished processing VPC Peerings. 1 peerings actioned. +[2025-11-30 19:23:05] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:23:07] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:23:07] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:23:07] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:23:07] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:23:07] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:10] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:23:13] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:23:13] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:23:15] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:23:15] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:25:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:25:29] [INFO] Time Cutoff (General): 2025-11-30T14:25:29+0000 +[2025-11-30 19:25:29] [INFO] Time Cutoff (Images): 2025-10-01T19:25:29+0000 +[2025-11-30 19:25:29] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:25:29] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:25:30] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- +[2025-11-30 19:25:32] [INFO] No Service Accounts found matching prefix. +[2025-11-30 19:25:32] [INFO] --- Processing: GKE Cluster (Limit: 20) --- +[2025-11-30 19:25:34] [INFO] No GKE Cluster found matching criteria. +[2025-11-30 19:25:34] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:25:37] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:25:37] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:25:37] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:25:37] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:25:37] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:25:37] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:25:37] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:25:37] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:25:37] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:25:37] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:25:37] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: dynpoc-compute-h3nodeset-20251015002137498900000005 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: dynpoc-controller-default-20251015002137476700000001 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: dynpoc-login-slurm-login-20251015002137480600000002 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: e2dbe6slur-compute-nodeset-20251007201508565800000001 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: e3879cslur-compute-nodeset-20250919212831665800000002 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: eaa3beslur-compute-nodeset-20251124070522321600000001 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: ebf828slur-compute-nodeset-20251124071134868500000001 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: ebf828slur-controller-default-20251124071144586200000003 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: ebf828slur-login-slurm-login-20251124071134910400000002 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f4944slurm-compute-nodeset-20250912131529204200000001 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f4e324slur-compute-nodeset-20250811165336641200000001 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f4e324slur-controller-default-20250811165346184100000003 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f4e324slur-login-slurm-login-20250811165336654300000002 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f88073slur-compute-nodeset-20250826031610614500000002 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f88073slur-controller-default-20250826031620053300000003 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f88073slur-login-slurm-login-20250826031610570200000001 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: fa4slurmsi-compute-nodeset-20250808051559443200000002 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: fa4slurmsi-controller-default-20250808051608790400000003 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: fa4slurmsi-login-slurm-login-20250808051559433300000001 (Global) +[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: g4qclav-compute-g4nodeset-20251111194038852400000002 (Global) +[2025-11-30 19:25:37] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:25:37] [INFO] --- Processing: Compute Instance (Limit: 20) --- +[2025-11-30 19:25:40] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:25:40] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:25:40] [SKIP] vertexui-do-not-kill (In Exclusion List) +[2025-11-30 19:25:40] [INFO] --- Processing: Filestore Instances (Limit: 20) --- +[2025-11-30 19:25:43] [INFO] No Filestore instances found matching criteria. +[2025-11-30 19:25:43] [INFO] --- Processing: VM Images (Limit: 20) --- +[2025-11-30 19:25:45] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) +[2025-11-30 19:25:45] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) +[2025-11-30 19:25:46] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) +[2025-11-30 19:25:46] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) +[2025-11-30 19:25:46] [SKIP] harsh-a4-image (In Exclusion List) +[2025-11-30 19:25:46] [SKIP] pbspro0 (In Exclusion List) +[2025-11-30 19:25:46] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) +[2025-11-30 19:25:46] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) +[2025-11-30 19:25:46] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) +[2025-11-30 19:25:46] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) +[2025-11-30 19:25:46] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- +[2025-11-30 19:25:46] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:25:46Z (Unix: 1763321146) +[2025-11-30 19:25:46] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner +Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. + +[2025-11-30 19:25:48] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. +[2025-11-30 19:25:48] [INFO] --- Processing: Cloud Router (Limit: 20) --- +[2025-11-30 19:25:51] [SKIP] default-net-router (In Exclusion List) +[2025-11-30 19:25:51] [SKIP] default-router-australia-southeast1 (In Exclusion List) +[2025-11-30 19:25:51] [SKIP] default-router-us-east4 (In Exclusion List) +[2025-11-30 19:25:51] [SKIP] default-router-us-west1 (In Exclusion List) +[2025-11-30 19:25:51] [SKIP] default-router-us-west4 (In Exclusion List) +[2025-11-30 19:25:51] [INFO] --- Processing: Firewall Rules (Limit: 20) --- +[2025-11-30 19:25:53] [INFO] --- Processing: Compute Addresses --- +[2025-11-30 19:25:53] [INFO] --- Processing: Regional Address (Limit: 20) --- +[2025-11-30 19:25:55] [INFO] No Regional Address found matching criteria. +[2025-11-30 19:25:55] [INFO] --- Processing: Global Address (Limit: 20) --- +[2025-11-30 19:25:58] [INFO] No Global Address found matching criteria. +[2025-11-30 19:25:58] [INFO] --- Processing: VPC Peerings (Limit: 20) --- +[2025-11-30 19:26:01] [INFO] [ACTION] Deleting Service Networking peering on: hpc-enterprise-slurm-v6 +[2025-11-30 19:26:01] [DRY-RUN] Would delete Service Peering: servicenetworking-googleapis-com (Network: hpc-enterprise-slurm-v6) +[2025-11-30 19:26:01] [INFO] Finished processing VPC Peerings. 1 peerings actioned. +[2025-11-30 19:26:01] [INFO] --- Processing: Zonal Disk (Limit: 20) --- +[2025-11-30 19:26:03] [SKIP] image-inspector-550 (In Exclusion List) +[2025-11-30 19:26:03] [SKIP] image-inspector (In Exclusion List) +[2025-11-30 19:26:03] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) +[2025-11-30 19:26:03] [SKIP] vertexui-do-not-kill-data (In Exclusion List) +[2025-11-30 19:26:03] [INFO] --- Processing: Subnetworks (Limit: 20) --- +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:06] [INFO] --- Processing: VPC Networks (Limit: 20) --- +[2025-11-30 19:26:08] [SKIP] hpc-vpc (In Exclusion List) +[2025-11-30 19:26:08] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- +[2025-11-30 19:26:10] [INFO] No 'deleted:serviceAccount' bindings found. +[2025-11-30 19:26:10] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:27:59] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:27:59] [INFO] Time Cutoff (General): 2025-11-30T14:27:59+0000 +[2025-11-30 19:27:59] [INFO] Time Cutoff (Images): 2025-10-01T19:27:59+0000 +[2025-11-30 19:27:59] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:27:59] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:27:59] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:28:03] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:28:03] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:28:03] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:28:03] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:28:03] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:28:03] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:28:03] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:28:03] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:28:03] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:28:03] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:28:03] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: dynpoc-compute-h3nodeset-20251015002137498900000005 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: dynpoc-controller-default-20251015002137476700000001 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: dynpoc-login-slurm-login-20251015002137480600000002 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: e2dbe6slur-compute-nodeset-20251007201508565800000001 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: e3879cslur-compute-nodeset-20250919212831665800000002 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: eaa3beslur-compute-nodeset-20251124070522321600000001 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: ebf828slur-compute-nodeset-20251124071134868500000001 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: ebf828slur-controller-default-20251124071144586200000003 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: ebf828slur-login-slurm-login-20251124071134910400000002 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f4944slurm-compute-nodeset-20250912131529204200000001 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f4e324slur-compute-nodeset-20250811165336641200000001 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f4e324slur-controller-default-20250811165346184100000003 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f4e324slur-login-slurm-login-20250811165336654300000002 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f88073slur-compute-nodeset-20250826031610614500000002 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f88073slur-controller-default-20250826031620053300000003 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f88073slur-login-slurm-login-20250826031610570200000001 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: fa4slurmsi-compute-nodeset-20250808051559443200000002 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: fa4slurmsi-controller-default-20250808051608790400000003 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: fa4slurmsi-login-slurm-login-20250808051559433300000001 (Global) +[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: g4qclav-compute-g4nodeset-20251111194038852400000002 (Global) +[2025-11-30 19:28:03] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:28:03] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:28:17] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:28:17] [INFO] Time Cutoff (General): 2025-11-30T14:28:17+0000 +[2025-11-30 19:28:17] [INFO] Time Cutoff (Images): 2025-10-01T19:28:17+0000 +[2025-11-30 19:28:17] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:28:17] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:28:18] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:28:21] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:28:21] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:28:21] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:28:21] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:28:21] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:28:21] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:28:21] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:28:21] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:28:21] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:28:21] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:28:21] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: dynpoc-compute-h3nodeset-20251015002137498900000005 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: dynpoc-controller-default-20251015002137476700000001 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: dynpoc-login-slurm-login-20251015002137480600000002 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: e2dbe6slur-compute-nodeset-20251007201508565800000001 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: e3879cslur-compute-nodeset-20250919212831665800000002 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: eaa3beslur-compute-nodeset-20251124070522321600000001 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: ebf828slur-compute-nodeset-20251124071134868500000001 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: ebf828slur-controller-default-20251124071144586200000003 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: ebf828slur-login-slurm-login-20251124071134910400000002 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f4944slurm-compute-nodeset-20250912131529204200000001 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f4e324slur-compute-nodeset-20250811165336641200000001 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f4e324slur-controller-default-20250811165346184100000003 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f4e324slur-login-slurm-login-20250811165336654300000002 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f88073slur-compute-nodeset-20250826031610614500000002 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f88073slur-controller-default-20250826031620053300000003 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f88073slur-login-slurm-login-20250826031610570200000001 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: fa4slurmsi-compute-nodeset-20250808051559443200000002 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: fa4slurmsi-controller-default-20250808051608790400000003 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: fa4slurmsi-login-slurm-login-20250808051559433300000001 (Global) +[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: g4qclav-compute-g4nodeset-20251111194038852400000002 (Global) +[2025-11-30 19:28:21] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:28:21] [INFO] CLEANUP RUN FINISHED +./cleanup.sh: line 712: n: command not found +[2025-11-30 19:28:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:28:47] [INFO] Time Cutoff (General): 2025-11-30T14:28:47+0000 +[2025-11-30 19:28:47] [INFO] Time Cutoff (Images): 2025-10-01T19:28:47+0000 +[2025-11-30 19:28:47] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:28:47] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:28:48] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:28:51] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:28:51] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:28:51] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:28:51] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:28:51] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:28:51] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:28:51] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:28:51] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:28:51] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:28:51] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:28:51] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:28:51] [EXECUTE] Deleting Instance Template: dynpoc-compute-h3nodeset-20251015002137498900000005 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/dynpoc-compute-h3nodeset-20251015002137498900000005]. +[2025-11-30 19:28:54] [SUCCESS] Deleted dynpoc-compute-h3nodeset-20251015002137498900000005 +[2025-11-30 19:28:54] [EXECUTE] Deleting Instance Template: dynpoc-controller-default-20251015002137476700000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/dynpoc-controller-default-20251015002137476700000001]. +[2025-11-30 19:28:57] [SUCCESS] Deleted dynpoc-controller-default-20251015002137476700000001 +[2025-11-30 19:28:57] [EXECUTE] Deleting Instance Template: dynpoc-login-slurm-login-20251015002137480600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/dynpoc-login-slurm-login-20251015002137480600000002]. +[2025-11-30 19:29:00] [SUCCESS] Deleted dynpoc-login-slurm-login-20251015002137480600000002 +[2025-11-30 19:29:00] [EXECUTE] Deleting Instance Template: e2dbe6slur-compute-nodeset-20251007201508565800000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/e2dbe6slur-compute-nodeset-20251007201508565800000001]. +[2025-11-30 19:29:03] [SUCCESS] Deleted e2dbe6slur-compute-nodeset-20251007201508565800000001 +[2025-11-30 19:29:03] [EXECUTE] Deleting Instance Template: e3879cslur-compute-nodeset-20250919212831665800000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/e3879cslur-compute-nodeset-20250919212831665800000002]. +[2025-11-30 19:29:07] [SUCCESS] Deleted e3879cslur-compute-nodeset-20250919212831665800000002 +[2025-11-30 19:29:07] [EXECUTE] Deleting Instance Template: eaa3beslur-compute-nodeset-20251124070522321600000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/eaa3beslur-compute-nodeset-20251124070522321600000001]. +[2025-11-30 19:29:10] [SUCCESS] Deleted eaa3beslur-compute-nodeset-20251124070522321600000001 +[2025-11-30 19:29:10] [EXECUTE] Deleting Instance Template: ebf828slur-compute-nodeset-20251124071134868500000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ebf828slur-compute-nodeset-20251124071134868500000001]. +[2025-11-30 19:29:13] [SUCCESS] Deleted ebf828slur-compute-nodeset-20251124071134868500000001 +[2025-11-30 19:29:13] [EXECUTE] Deleting Instance Template: ebf828slur-controller-default-20251124071144586200000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ebf828slur-controller-default-20251124071144586200000003]. +[2025-11-30 19:29:16] [SUCCESS] Deleted ebf828slur-controller-default-20251124071144586200000003 +[2025-11-30 19:29:16] [EXECUTE] Deleting Instance Template: ebf828slur-login-slurm-login-20251124071134910400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ebf828slur-login-slurm-login-20251124071134910400000002]. +[2025-11-30 19:29:19] [SUCCESS] Deleted ebf828slur-login-slurm-login-20251124071134910400000002 +[2025-11-30 19:29:19] [EXECUTE] Deleting Instance Template: f4944slurm-compute-nodeset-20250912131529204200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f4944slurm-compute-nodeset-20250912131529204200000001]. +[2025-11-30 19:29:22] [SUCCESS] Deleted f4944slurm-compute-nodeset-20250912131529204200000001 +[2025-11-30 19:29:22] [EXECUTE] Deleting Instance Template: f4e324slur-compute-nodeset-20250811165336641200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f4e324slur-compute-nodeset-20250811165336641200000001]. +[2025-11-30 19:29:25] [SUCCESS] Deleted f4e324slur-compute-nodeset-20250811165336641200000001 +[2025-11-30 19:29:25] [EXECUTE] Deleting Instance Template: f4e324slur-controller-default-20250811165346184100000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f4e324slur-controller-default-20250811165346184100000003]. +[2025-11-30 19:29:28] [SUCCESS] Deleted f4e324slur-controller-default-20250811165346184100000003 +[2025-11-30 19:29:28] [EXECUTE] Deleting Instance Template: f4e324slur-login-slurm-login-20250811165336654300000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f4e324slur-login-slurm-login-20250811165336654300000002]. +[2025-11-30 19:29:31] [SUCCESS] Deleted f4e324slur-login-slurm-login-20250811165336654300000002 +[2025-11-30 19:29:31] [EXECUTE] Deleting Instance Template: f88073slur-compute-nodeset-20250826031610614500000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f88073slur-compute-nodeset-20250826031610614500000002]. +[2025-11-30 19:29:34] [SUCCESS] Deleted f88073slur-compute-nodeset-20250826031610614500000002 +[2025-11-30 19:29:34] [EXECUTE] Deleting Instance Template: f88073slur-controller-default-20250826031620053300000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f88073slur-controller-default-20250826031620053300000003]. +[2025-11-30 19:29:37] [SUCCESS] Deleted f88073slur-controller-default-20250826031620053300000003 +[2025-11-30 19:29:37] [EXECUTE] Deleting Instance Template: f88073slur-login-slurm-login-20250826031610570200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f88073slur-login-slurm-login-20250826031610570200000001]. +[2025-11-30 19:29:40] [SUCCESS] Deleted f88073slur-login-slurm-login-20250826031610570200000001 +[2025-11-30 19:29:40] [EXECUTE] Deleting Instance Template: fa4slurmsi-compute-nodeset-20250808051559443200000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/fa4slurmsi-compute-nodeset-20250808051559443200000002]. +[2025-11-30 19:29:43] [SUCCESS] Deleted fa4slurmsi-compute-nodeset-20250808051559443200000002 +[2025-11-30 19:29:43] [EXECUTE] Deleting Instance Template: fa4slurmsi-controller-default-20250808051608790400000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/fa4slurmsi-controller-default-20250808051608790400000003]. +[2025-11-30 19:29:46] [SUCCESS] Deleted fa4slurmsi-controller-default-20250808051608790400000003 +[2025-11-30 19:29:46] [EXECUTE] Deleting Instance Template: fa4slurmsi-login-slurm-login-20250808051559433300000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/fa4slurmsi-login-slurm-login-20250808051559433300000001]. +[2025-11-30 19:29:49] [SUCCESS] Deleted fa4slurmsi-login-slurm-login-20250808051559433300000001 +[2025-11-30 19:29:49] [EXECUTE] Deleting Instance Template: g4qclav-compute-g4nodeset-20251111194038852400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4qclav-compute-g4nodeset-20251111194038852400000002]. +[2025-11-30 19:29:52] [SUCCESS] Deleted g4qclav-compute-g4nodeset-20251111194038852400000002 +[2025-11-30 19:29:52] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:29:52] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:30:24] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:30:24] [INFO] Time Cutoff (General): 2025-11-30T14:30:23+0000 +[2025-11-30 19:30:24] [INFO] Time Cutoff (Images): 2025-10-01T19:30:24+0000 +[2025-11-30 19:30:24] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:30:24] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:30:24] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:30:27] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:30:27] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:30:27] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:30:27] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:30:27] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:30:27] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:30:27] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:30:27] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:30:27] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:30:27] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:30:27] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4qclav-controller-default-20251111194048650800000003 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4qclav-login-slurm-login-20251111194038848700000001 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4test-compute-g4nodeset-20250922145102127500000001 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4test-controller-default-20250922145111921600000003 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4test-login-slurm-login-20250922145102128200000002 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4testnewe-compute-g4nodeset-20251023163712389000000001 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4testnewe-controller-default-20251023163722101400000003 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4testnewe-login-slurm-login-20251023163712400900000002 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: h4dqc-compute-h4dnodeset-20250526185335423500000003 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: h4dqc-controller-default-20250526185335395700000001 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: h4dqc-login-slurm-login-20250526185335402700000002 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: h4dsaara-compute-h4dnodeset-20250903085805905100000002 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hclsv60f32-compute-gpu-20250801171014779800000001 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hclsv60f32-compute-ns-20250801171014806600000002 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hclsv60f32-controller-default-20250801171022463800000004 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hclsv60f32-login-slurm-login-20250801171014845800000003 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hpc01-compute-a216nodeset-20251125104737582400000001 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hpc01-compute-a28nodeset-20251125104737596100000002 (Global) +[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hpc01-compute-c2dnodeset-20251125104737610300000004 (Global) +[2025-11-30 19:30:27] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:30:27] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:30:54] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:30:54] [INFO] Time Cutoff (General): 2025-11-30T14:30:54+0000 +[2025-11-30 19:30:54] [INFO] Time Cutoff (Images): 2025-10-01T19:30:54+0000 +[2025-11-30 19:30:54] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:30:54] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:30:54] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:30:57] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:30:57] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:30:57] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:30:58] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:30:58] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:30:58] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:30:58] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:30:58] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:30:58] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:30:58] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:30:58] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:30:58] [EXECUTE] Deleting Instance Template: g4qclav-controller-default-20251111194048650800000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4qclav-controller-default-20251111194048650800000003]. +[2025-11-30 19:31:01] [SUCCESS] Deleted g4qclav-controller-default-20251111194048650800000003 +[2025-11-30 19:31:01] [EXECUTE] Deleting Instance Template: g4qclav-login-slurm-login-20251111194038848700000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4qclav-login-slurm-login-20251111194038848700000001]. +[2025-11-30 19:31:04] [SUCCESS] Deleted g4qclav-login-slurm-login-20251111194038848700000001 +[2025-11-30 19:31:04] [EXECUTE] Deleting Instance Template: g4test-compute-g4nodeset-20250922145102127500000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4test-compute-g4nodeset-20250922145102127500000001]. +[2025-11-30 19:31:07] [SUCCESS] Deleted g4test-compute-g4nodeset-20250922145102127500000001 +[2025-11-30 19:31:07] [EXECUTE] Deleting Instance Template: g4test-controller-default-20250922145111921600000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4test-controller-default-20250922145111921600000003]. +[2025-11-30 19:31:10] [SUCCESS] Deleted g4test-controller-default-20250922145111921600000003 +[2025-11-30 19:31:10] [EXECUTE] Deleting Instance Template: g4test-login-slurm-login-20250922145102128200000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4test-login-slurm-login-20250922145102128200000002]. +[2025-11-30 19:31:13] [SUCCESS] Deleted g4test-login-slurm-login-20250922145102128200000002 +[2025-11-30 19:31:13] [EXECUTE] Deleting Instance Template: g4testnewe-compute-g4nodeset-20251023163712389000000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4testnewe-compute-g4nodeset-20251023163712389000000001]. +[2025-11-30 19:31:16] [SUCCESS] Deleted g4testnewe-compute-g4nodeset-20251023163712389000000001 +[2025-11-30 19:31:16] [EXECUTE] Deleting Instance Template: g4testnewe-controller-default-20251023163722101400000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4testnewe-controller-default-20251023163722101400000003]. +[2025-11-30 19:31:19] [SUCCESS] Deleted g4testnewe-controller-default-20251023163722101400000003 +[2025-11-30 19:31:19] [EXECUTE] Deleting Instance Template: g4testnewe-login-slurm-login-20251023163712400900000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4testnewe-login-slurm-login-20251023163712400900000002]. +[2025-11-30 19:31:22] [SUCCESS] Deleted g4testnewe-login-slurm-login-20251023163712400900000002 +[2025-11-30 19:31:22] [EXECUTE] Deleting Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) +ERROR: (gcloud.compute.instance-templates.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/global/instanceTemplates/gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492' was not found + +[2025-11-30 19:31:24] [ERROR] Failed to delete gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 +[2025-11-30 19:31:24] [EXECUTE] Deleting Instance Template: h4dqc-compute-h4dnodeset-20250526185335423500000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/h4dqc-compute-h4dnodeset-20250526185335423500000003]. +[2025-11-30 19:31:28] [SUCCESS] Deleted h4dqc-compute-h4dnodeset-20250526185335423500000003 +[2025-11-30 19:31:28] [EXECUTE] Deleting Instance Template: h4dqc-controller-default-20250526185335395700000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/h4dqc-controller-default-20250526185335395700000001]. +[2025-11-30 19:31:31] [SUCCESS] Deleted h4dqc-controller-default-20250526185335395700000001 +[2025-11-30 19:31:31] [EXECUTE] Deleting Instance Template: h4dqc-login-slurm-login-20250526185335402700000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/h4dqc-login-slurm-login-20250526185335402700000002]. +[2025-11-30 19:31:34] [SUCCESS] Deleted h4dqc-login-slurm-login-20250526185335402700000002 +[2025-11-30 19:31:34] [EXECUTE] Deleting Instance Template: h4dsaara-compute-h4dnodeset-20250903085805905100000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/h4dsaara-compute-h4dnodeset-20250903085805905100000002]. +[2025-11-30 19:31:37] [SUCCESS] Deleted h4dsaara-compute-h4dnodeset-20250903085805905100000002 +[2025-11-30 19:31:37] [EXECUTE] Deleting Instance Template: hclsv60f32-compute-gpu-20250801171014779800000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hclsv60f32-compute-gpu-20250801171014779800000001]. +[2025-11-30 19:31:40] [SUCCESS] Deleted hclsv60f32-compute-gpu-20250801171014779800000001 +[2025-11-30 19:31:40] [EXECUTE] Deleting Instance Template: hclsv60f32-compute-ns-20250801171014806600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hclsv60f32-compute-ns-20250801171014806600000002]. +[2025-11-30 19:31:43] [SUCCESS] Deleted hclsv60f32-compute-ns-20250801171014806600000002 +[2025-11-30 19:31:43] [EXECUTE] Deleting Instance Template: hclsv60f32-controller-default-20250801171022463800000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hclsv60f32-controller-default-20250801171022463800000004]. +[2025-11-30 19:31:46] [SUCCESS] Deleted hclsv60f32-controller-default-20250801171022463800000004 +[2025-11-30 19:31:46] [EXECUTE] Deleting Instance Template: hclsv60f32-login-slurm-login-20250801171014845800000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hclsv60f32-login-slurm-login-20250801171014845800000003]. +[2025-11-30 19:31:49] [SUCCESS] Deleted hclsv60f32-login-slurm-login-20250801171014845800000003 +[2025-11-30 19:31:49] [EXECUTE] Deleting Instance Template: hpc01-compute-a216nodeset-20251125104737582400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-a216nodeset-20251125104737582400000001]. +[2025-11-30 19:31:52] [SUCCESS] Deleted hpc01-compute-a216nodeset-20251125104737582400000001 +[2025-11-30 19:31:52] [EXECUTE] Deleting Instance Template: hpc01-compute-a28nodeset-20251125104737596100000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-a28nodeset-20251125104737596100000002]. +[2025-11-30 19:31:55] [SUCCESS] Deleted hpc01-compute-a28nodeset-20251125104737596100000002 +[2025-11-30 19:31:55] [EXECUTE] Deleting Instance Template: hpc01-compute-c2dnodeset-20251125104737610300000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-c2dnodeset-20251125104737610300000004]. +[2025-11-30 19:31:58] [SUCCESS] Deleted hpc01-compute-c2dnodeset-20251125104737610300000004 +[2025-11-30 19:31:58] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:31:58] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:32:24] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:32:24] [INFO] Time Cutoff (General): 2025-11-30T14:32:24+0000 +[2025-11-30 19:32:24] [INFO] Time Cutoff (Images): 2025-10-01T19:32:24+0000 +[2025-11-30 19:32:24] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:32:24] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:32:25] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:32:28] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:32:28] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:32:28] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:32:28] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:32:28] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:32:28] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:32:28] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:32:28] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:32:28] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:32:28] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:32:28] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpc01-compute-c2nodeset-20251125104737599200000003 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpc01-compute-c3nodeset-20251125104737611200000005 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpc01-compute-h3nodeset-20251125104737617100000006 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpc01-compute-n2nodeset-20251125104739693200000007 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpc01-login-slurm-login-20251125104742597200000008 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-a216nodeset-20251123152333927800000004 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-a28nodeset-20251123152344956400000009 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c2dnodeset-20251123152343768500000008 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c2nodeset-20251123152333914100000003 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c3nodeset-20251123152334135900000006 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-h3nodeset-20251123152343701700000007 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-n2nodeset-20251123152334134700000005 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-controller-default-20251123152321945800000001 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-login-slurm-login-20251123152333155000000002 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-computenodeset-20251120073345516900000002 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-debugnodeset-20251120073345535400000005 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-h3nodeset-20251120073345521400000003 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcslurm-controller-default-20251120073345359600000001 (Global) +[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcslurm-login-slurm-login-20251120073345530000000004 (Global) +[2025-11-30 19:32:28] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:32:28] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:34:27] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:34:27] [INFO] Time Cutoff (General): 2025-11-30T14:34:27+0000 +[2025-11-30 19:34:27] [INFO] Time Cutoff (Images): 2025-10-01T19:34:27+0000 +[2025-11-30 19:34:27] [INFO] Delete Limit per Type: 20 +[2025-11-30 19:34:27] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:34:28] [INFO] --- Processing: Instance Templates (Limit: 20) --- +[2025-11-30 19:34:31] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) +[2025-11-30 19:34:31] [SKIP] hpc01-compute-c2nodeset-20251125104737599200000003 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpc01-compute-c3nodeset-20251125104737611200000005 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpc01-compute-h3nodeset-20251125104737617100000006 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpc01-compute-n2nodeset-20251125104739693200000007 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpc01-login-slurm-login-20251125104742597200000008 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpcimg-compute-a216nodeset-20251123152333927800000004 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpcimg-compute-a28nodeset-20251123152344956400000009 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpcimg-compute-c2dnodeset-20251123152343768500000008 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpcimg-compute-c2nodeset-20251123152333914100000003 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpcimg-compute-c3nodeset-20251123152334135900000006 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpcimg-compute-h3nodeset-20251123152343701700000007 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpcimg-compute-n2nodeset-20251123152334134700000005 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpcimg-controller-default-20251123152321945800000001 (In Exclusion List) +[2025-11-30 19:34:31] [SKIP] hpcimg-login-slurm-login-20251123152333155000000002 (In Exclusion List) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-computenodeset-20251120073345516900000002 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-debugnodeset-20251120073345535400000005 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-h3nodeset-20251120073345521400000003 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: hpcslurm-controller-default-20251120073345359600000001 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: hpcslurm-login-slurm-login-20251120073345530000000004 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: laveeek29-compute-a3ultranodeset-20251120185220355800000002 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: laveeek29-controller-default-20251120185222245000000003 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: laveeek29-login-slurm-login-20251120185217925800000001 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustre06-compute-lustrenodeset-20251126041817737800000001 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustre06-controller-default-20251126041828459800000003 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustre06-login-slurm-login-20251126041817754000000002 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a216nodeset-20251127075747115200000008 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a28nodeset-20251127075746973800000005 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2dnodeset-20251127075746534500000004 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2nodeset-20251127075746393800000003 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c3nodeset-20251127075747101600000006 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-h3nodeset-20251127075747131700000009 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-n2nodeset-20251127075747112000000007 (Global) +[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-controller-default-20251127075745707800000001 (Global) +[2025-11-30 19:34:31] [INFO] Hit delete limit (20) for Instance Templates. +[2025-11-30 19:34:31] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:34:53] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:34:53] [INFO] Time Cutoff (General): 2025-11-30T14:34:53+0000 +[2025-11-30 19:34:53] [INFO] Time Cutoff (Images): 2025-10-01T19:34:53+0000 +[2025-11-30 19:34:53] [INFO] Delete Limit per Type: 200 +[2025-11-30 19:34:53] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:34:54] [INFO] --- Processing: Instance Templates (Limit: 200) --- +[2025-11-30 19:34:57] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) +[2025-11-30 19:34:57] [SKIP] hpc01-compute-c2nodeset-20251125104737599200000003 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpc01-compute-c3nodeset-20251125104737611200000005 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpc01-compute-h3nodeset-20251125104737617100000006 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpc01-compute-n2nodeset-20251125104739693200000007 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpc01-login-slurm-login-20251125104742597200000008 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpcimg-compute-a216nodeset-20251123152333927800000004 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpcimg-compute-a28nodeset-20251123152344956400000009 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpcimg-compute-c2dnodeset-20251123152343768500000008 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpcimg-compute-c2nodeset-20251123152333914100000003 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpcimg-compute-c3nodeset-20251123152334135900000006 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpcimg-compute-h3nodeset-20251123152343701700000007 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpcimg-compute-n2nodeset-20251123152334134700000005 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpcimg-controller-default-20251123152321945800000001 (In Exclusion List) +[2025-11-30 19:34:57] [SKIP] hpcimg-login-slurm-login-20251123152333155000000002 (In Exclusion List) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-computenodeset-20251120073345516900000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-debugnodeset-20251120073345535400000005 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-h3nodeset-20251120073345521400000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: hpcslurm-controller-default-20251120073345359600000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: hpcslurm-login-slurm-login-20251120073345530000000004 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: laveeek29-compute-a3ultranodeset-20251120185220355800000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: laveeek29-controller-default-20251120185222245000000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: laveeek29-login-slurm-login-20251120185217925800000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustre06-compute-lustrenodeset-20251126041817737800000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustre06-controller-default-20251126041828459800000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustre06-login-slurm-login-20251126041817754000000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a216nodeset-20251127075747115200000008 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a28nodeset-20251127075746973800000005 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2dnodeset-20251127075746534500000004 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2nodeset-20251127075746393800000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c3nodeset-20251127075747101600000006 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-h3nodeset-20251127075747131700000009 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-n2nodeset-20251127075747112000000007 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-controller-default-20251127075745707800000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-login-slurm-login-20251127075746027600000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-a216nodeset-20251127085007177000000008 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-a28nodeset-20251127085007152400000005 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c2dnodeset-20251127085007154700000006 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c2nodeset-20251127085007178300000009 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c3nodeset-20251127085007120300000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-h3nodeset-20251127085007156500000007 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-n2nodeset-20251127085007147000000004 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-controller-default-20251127085005432900000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-login-slurm-login-20251127085005549500000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-a216nodeset-20251127064214383600000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-a28nodeset-20251127064214402000000004 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c2dnodeset-20251127064214467900000009 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c2nodeset-20251127064214424900000005 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c3nodeset-20251127064214452600000007 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-h3nodeset-20251127064214457600000008 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-n2nodeset-20251127064214447800000006 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-controller-default-20251127064213231900000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-login-slurm-login-20251127064213259800000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-a216nodeset-20251126044326259100000008 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-a28nodeset-20251126044326243000000005 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-c2dnodeset-20251126044326150300000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-c2nodeset-20251126044326244400000006 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-c3nodeset-20251126044326214900000004 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-h3nodeset-20251126044326264800000009 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-n2nodeset-20251126044326256800000007 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-controller-default-20251126044324852600000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-login-slurm-login-20251126044324988400000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: mainek-compute-a3ultranodeset-20251121092031664700000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: mainek-controller-default-20251121092028913400000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: mainek-login-slurm-login-20251121092028928700000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: pkbh4dp2-compute-computenodeset-20250412005603828600000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: pkrv6db5de-login-slurm-login-20250801090852525400000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: pzpk-compute-computenodeset-20250411004353418500000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: pzpk-controller-default-20250411004353405200000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesh4d-compute-h4dnodeset-20251016152751862600000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesh4d-controller-default-20251016152751828700000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesh4d-login-slurm-login-20251016152751847100000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesth4d-compute-h4dnodeset-20251013062737759200000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesth4d-controller-default-20251013062737731900000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesth4d-login-slurm-login-20251013062737750800000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-computenodeset-20250912110554713000000004 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-debugnodeset-20250912110554709900000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-h3nodeset-20250912110554735800000005 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: rock8a070a-controller-default-20250912110554351400000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: rock8a070a-login-slurm-login-20250912110554546400000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: simtestnet-compute-a3ultranodeset-20251128190204679000000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: simtestnet-controller-default-20251128190210353600000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: simtestnet-login-slurm-login-20251128190201729100000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm0-compute-a3nodeset-20251009153324036900000004 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm0-compute-debugnodeset-20251009153324023000000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm0-controller-default-20251009153304999900000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm0-login-login-20251009153305041200000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm9ff-compute-debugnodeset-20250925215605513600000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm9ff-controller-default-20250925215558268500000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm9ff-login-slurm-login-20250925215558759400000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmf1c-compute-debugnodeset-20250925064111290200000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmf1c-controller-default-20250925064111186200000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmf1c-login-slurm-login-20250925064111290800000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20250806200217240000000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20250912233249589700000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20251111083658962700000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmsimpl-compute-nodeset-20250708064936486300000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmsimpl-compute-nodeset-20250814191439012800000001 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmsimpl-controller-default-20250708064945457400000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmsimpl-controller-default-20250814191448978300000003 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmsimpl-login-slurm-login-20250814191439021200000002 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: testing2 (Global) +[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: welp-insta-temp (Global) +[2025-11-30 19:34:57] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:36:17] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:36:17] [INFO] Time Cutoff (General): 2025-11-30T14:36:17+0000 +[2025-11-30 19:36:17] [INFO] Time Cutoff (Images): 2025-10-01T19:36:17+0000 +[2025-11-30 19:36:17] [INFO] Delete Limit per Type: 200 +[2025-11-30 19:36:17] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:36:18] [INFO] --- Processing: Instance Templates (Limit: 200) --- +[2025-11-30 19:36:21] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:36:21] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:36:21] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:36:21] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:36:21] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:36:21] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:36:21] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:36:21] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:36:21] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:36:21] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:36:21] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpc01-compute-c2nodeset-20251125104737599200000003 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpc01-compute-c3nodeset-20251125104737611200000005 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpc01-compute-h3nodeset-20251125104737617100000006 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpc01-compute-n2nodeset-20251125104739693200000007 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpc01-login-slurm-login-20251125104742597200000008 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-a216nodeset-20251123152333927800000004 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-a28nodeset-20251123152344956400000009 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c2dnodeset-20251123152343768500000008 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c2nodeset-20251123152333914100000003 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c3nodeset-20251123152334135900000006 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-h3nodeset-20251123152343701700000007 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-n2nodeset-20251123152334134700000005 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-controller-default-20251123152321945800000001 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-login-slurm-login-20251123152333155000000002 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-computenodeset-20251120073345516900000002 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-debugnodeset-20251120073345535400000005 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-h3nodeset-20251120073345521400000003 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcslurm-controller-default-20251120073345359600000001 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcslurm-login-slurm-login-20251120073345530000000004 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: laveeek29-compute-a3ultranodeset-20251120185220355800000002 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: laveeek29-controller-default-20251120185222245000000003 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: laveeek29-login-slurm-login-20251120185217925800000001 (Global) +[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: lustre06-compute-lustrenodeset-20251126041817737800000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustre06-controller-default-20251126041828459800000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustre06-login-slurm-login-20251126041817754000000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a216nodeset-20251127075747115200000008 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a28nodeset-20251127075746973800000005 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2dnodeset-20251127075746534500000004 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2nodeset-20251127075746393800000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c3nodeset-20251127075747101600000006 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-h3nodeset-20251127075747131700000009 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-n2nodeset-20251127075747112000000007 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-controller-default-20251127075745707800000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-login-slurm-login-20251127075746027600000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-a216nodeset-20251127085007177000000008 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-a28nodeset-20251127085007152400000005 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c2dnodeset-20251127085007154700000006 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c2nodeset-20251127085007178300000009 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c3nodeset-20251127085007120300000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-h3nodeset-20251127085007156500000007 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-n2nodeset-20251127085007147000000004 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-controller-default-20251127085005432900000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-login-slurm-login-20251127085005549500000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-a216nodeset-20251127064214383600000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-a28nodeset-20251127064214402000000004 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c2dnodeset-20251127064214467900000009 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c2nodeset-20251127064214424900000005 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c3nodeset-20251127064214452600000007 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-h3nodeset-20251127064214457600000008 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-n2nodeset-20251127064214447800000006 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-controller-default-20251127064213231900000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-login-slurm-login-20251127064213259800000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-a216nodeset-20251126044326259100000008 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-a28nodeset-20251126044326243000000005 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-c2dnodeset-20251126044326150300000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-c2nodeset-20251126044326244400000006 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-c3nodeset-20251126044326214900000004 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-h3nodeset-20251126044326264800000009 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-n2nodeset-20251126044326256800000007 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-controller-default-20251126044324852600000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-login-slurm-login-20251126044324988400000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: mainek-compute-a3ultranodeset-20251121092031664700000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: mainek-controller-default-20251121092028913400000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: mainek-login-slurm-login-20251121092028928700000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: pkbh4dp2-compute-computenodeset-20250412005603828600000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: pkrv6db5de-login-slurm-login-20250801090852525400000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: pzpk-compute-computenodeset-20250411004353418500000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: pzpk-controller-default-20250411004353405200000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesh4d-compute-h4dnodeset-20251016152751862600000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesh4d-controller-default-20251016152751828700000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesh4d-login-slurm-login-20251016152751847100000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesth4d-compute-h4dnodeset-20251013062737759200000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesth4d-controller-default-20251013062737731900000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesth4d-login-slurm-login-20251013062737750800000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-computenodeset-20250912110554713000000004 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-debugnodeset-20250912110554709900000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-h3nodeset-20250912110554735800000005 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: rock8a070a-controller-default-20250912110554351400000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: rock8a070a-login-slurm-login-20250912110554546400000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: simtestnet-compute-a3ultranodeset-20251128190204679000000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: simtestnet-controller-default-20251128190210353600000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: simtestnet-login-slurm-login-20251128190201729100000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm0-compute-a3nodeset-20251009153324036900000004 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm0-compute-debugnodeset-20251009153324023000000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm0-controller-default-20251009153304999900000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm0-login-login-20251009153305041200000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm9ff-compute-debugnodeset-20250925215605513600000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm9ff-controller-default-20250925215558268500000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm9ff-login-slurm-login-20250925215558759400000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmf1c-compute-debugnodeset-20250925064111290200000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmf1c-controller-default-20250925064111186200000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmf1c-login-slurm-login-20250925064111290800000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20250806200217240000000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20250912233249589700000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20251111083658962700000002 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmsimpl-compute-nodeset-20250708064936486300000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmsimpl-compute-nodeset-20250814191439012800000001 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmsimpl-controller-default-20250708064945457400000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmsimpl-controller-default-20250814191448978300000003 (Global) +[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmsimpl-login-slurm-login-20250814191439021200000002 (Global) +[2025-11-30 19:36:22] [SKIP] testing2 (In Exclusion List) +[2025-11-30 19:36:22] [SKIP] welp-insta-temp (In Exclusion List) +[2025-11-30 19:36:22] [INFO] CLEANUP RUN FINISHED +[2025-11-30 19:36:53] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev +[2025-11-30 19:36:53] [INFO] Time Cutoff (General): 2025-11-30T14:36:53+0000 +[2025-11-30 19:36:53] [INFO] Time Cutoff (Images): 2025-10-01T19:36:53+0000 +[2025-11-30 19:36:53] [INFO] Delete Limit per Type: 200 +[2025-11-30 19:36:53] [INFO] Loading exclusions from exclusions.txt... +[2025-11-30 19:36:54] [INFO] --- Processing: Instance Templates (Limit: 200) --- +[2025-11-30 19:36:57] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) +[2025-11-30 19:36:57] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) +[2025-11-30 19:36:57] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) +[2025-11-30 19:36:57] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) +[2025-11-30 19:36:57] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) +[2025-11-30 19:36:57] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) +[2025-11-30 19:36:57] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) +[2025-11-30 19:36:57] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) +[2025-11-30 19:36:57] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) +[2025-11-30 19:36:57] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) +[2025-11-30 19:36:57] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) +[2025-11-30 19:36:57] [EXECUTE] Deleting Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) +ERROR: (gcloud.compute.instance-templates.delete) Could not fetch resource: + - The resource 'projects/hpc-toolkit-dev/global/instanceTemplates/gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492' was not found + +[2025-11-30 19:36:59] [ERROR] Failed to delete gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 +[2025-11-30 19:36:59] [EXECUTE] Deleting Instance Template: hpc01-compute-c2nodeset-20251125104737599200000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-c2nodeset-20251125104737599200000003]. +[2025-11-30 19:37:02] [SUCCESS] Deleted hpc01-compute-c2nodeset-20251125104737599200000003 +[2025-11-30 19:37:02] [EXECUTE] Deleting Instance Template: hpc01-compute-c3nodeset-20251125104737611200000005 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-c3nodeset-20251125104737611200000005]. +[2025-11-30 19:37:05] [SUCCESS] Deleted hpc01-compute-c3nodeset-20251125104737611200000005 +[2025-11-30 19:37:05] [EXECUTE] Deleting Instance Template: hpc01-compute-h3nodeset-20251125104737617100000006 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-h3nodeset-20251125104737617100000006]. +[2025-11-30 19:37:08] [SUCCESS] Deleted hpc01-compute-h3nodeset-20251125104737617100000006 +[2025-11-30 19:37:08] [EXECUTE] Deleting Instance Template: hpc01-compute-n2nodeset-20251125104739693200000007 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-n2nodeset-20251125104739693200000007]. +[2025-11-30 19:37:12] [SUCCESS] Deleted hpc01-compute-n2nodeset-20251125104739693200000007 +[2025-11-30 19:37:12] [EXECUTE] Deleting Instance Template: hpc01-login-slurm-login-20251125104742597200000008 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-login-slurm-login-20251125104742597200000008]. +[2025-11-30 19:37:15] [SUCCESS] Deleted hpc01-login-slurm-login-20251125104742597200000008 +[2025-11-30 19:37:15] [EXECUTE] Deleting Instance Template: hpcimg-compute-a216nodeset-20251123152333927800000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-a216nodeset-20251123152333927800000004]. +[2025-11-30 19:37:18] [SUCCESS] Deleted hpcimg-compute-a216nodeset-20251123152333927800000004 +[2025-11-30 19:37:18] [EXECUTE] Deleting Instance Template: hpcimg-compute-a28nodeset-20251123152344956400000009 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-a28nodeset-20251123152344956400000009]. +[2025-11-30 19:37:21] [SUCCESS] Deleted hpcimg-compute-a28nodeset-20251123152344956400000009 +[2025-11-30 19:37:21] [EXECUTE] Deleting Instance Template: hpcimg-compute-c2dnodeset-20251123152343768500000008 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-c2dnodeset-20251123152343768500000008]. +[2025-11-30 19:37:24] [SUCCESS] Deleted hpcimg-compute-c2dnodeset-20251123152343768500000008 +[2025-11-30 19:37:24] [EXECUTE] Deleting Instance Template: hpcimg-compute-c2nodeset-20251123152333914100000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-c2nodeset-20251123152333914100000003]. +[2025-11-30 19:37:27] [SUCCESS] Deleted hpcimg-compute-c2nodeset-20251123152333914100000003 +[2025-11-30 19:37:27] [EXECUTE] Deleting Instance Template: hpcimg-compute-c3nodeset-20251123152334135900000006 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-c3nodeset-20251123152334135900000006]. +[2025-11-30 19:37:30] [SUCCESS] Deleted hpcimg-compute-c3nodeset-20251123152334135900000006 +[2025-11-30 19:37:30] [EXECUTE] Deleting Instance Template: hpcimg-compute-h3nodeset-20251123152343701700000007 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-h3nodeset-20251123152343701700000007]. +[2025-11-30 19:37:33] [SUCCESS] Deleted hpcimg-compute-h3nodeset-20251123152343701700000007 +[2025-11-30 19:37:34] [EXECUTE] Deleting Instance Template: hpcimg-compute-n2nodeset-20251123152334134700000005 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-n2nodeset-20251123152334134700000005]. +[2025-11-30 19:37:37] [SUCCESS] Deleted hpcimg-compute-n2nodeset-20251123152334134700000005 +[2025-11-30 19:37:37] [EXECUTE] Deleting Instance Template: hpcimg-controller-default-20251123152321945800000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-controller-default-20251123152321945800000001]. +[2025-11-30 19:37:40] [SUCCESS] Deleted hpcimg-controller-default-20251123152321945800000001 +[2025-11-30 19:37:40] [EXECUTE] Deleting Instance Template: hpcimg-login-slurm-login-20251123152333155000000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-login-slurm-login-20251123152333155000000002]. +[2025-11-30 19:37:43] [SUCCESS] Deleted hpcimg-login-slurm-login-20251123152333155000000002 +[2025-11-30 19:37:43] [EXECUTE] Deleting Instance Template: hpcslurm-compute-computenodeset-20251120073345516900000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcslurm-compute-computenodeset-20251120073345516900000002]. +[2025-11-30 19:37:46] [SUCCESS] Deleted hpcslurm-compute-computenodeset-20251120073345516900000002 +[2025-11-30 19:37:46] [EXECUTE] Deleting Instance Template: hpcslurm-compute-debugnodeset-20251120073345535400000005 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcslurm-compute-debugnodeset-20251120073345535400000005]. +[2025-11-30 19:37:49] [SUCCESS] Deleted hpcslurm-compute-debugnodeset-20251120073345535400000005 +[2025-11-30 19:37:49] [EXECUTE] Deleting Instance Template: hpcslurm-compute-h3nodeset-20251120073345521400000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcslurm-compute-h3nodeset-20251120073345521400000003]. +[2025-11-30 19:37:52] [SUCCESS] Deleted hpcslurm-compute-h3nodeset-20251120073345521400000003 +[2025-11-30 19:37:52] [EXECUTE] Deleting Instance Template: hpcslurm-controller-default-20251120073345359600000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcslurm-controller-default-20251120073345359600000001]. +[2025-11-30 19:37:55] [SUCCESS] Deleted hpcslurm-controller-default-20251120073345359600000001 +[2025-11-30 19:37:55] [EXECUTE] Deleting Instance Template: hpcslurm-login-slurm-login-20251120073345530000000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcslurm-login-slurm-login-20251120073345530000000004]. +[2025-11-30 19:37:59] [SUCCESS] Deleted hpcslurm-login-slurm-login-20251120073345530000000004 +[2025-11-30 19:37:59] [EXECUTE] Deleting Instance Template: laveeek29-compute-a3ultranodeset-20251120185220355800000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/laveeek29-compute-a3ultranodeset-20251120185220355800000002]. +[2025-11-30 19:38:02] [SUCCESS] Deleted laveeek29-compute-a3ultranodeset-20251120185220355800000002 +[2025-11-30 19:38:02] [EXECUTE] Deleting Instance Template: laveeek29-controller-default-20251120185222245000000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/laveeek29-controller-default-20251120185222245000000003]. +[2025-11-30 19:38:05] [SUCCESS] Deleted laveeek29-controller-default-20251120185222245000000003 +[2025-11-30 19:38:05] [EXECUTE] Deleting Instance Template: laveeek29-login-slurm-login-20251120185217925800000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/laveeek29-login-slurm-login-20251120185217925800000001]. +[2025-11-30 19:38:08] [SUCCESS] Deleted laveeek29-login-slurm-login-20251120185217925800000001 +[2025-11-30 19:38:08] [EXECUTE] Deleting Instance Template: lustre06-compute-lustrenodeset-20251126041817737800000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustre06-compute-lustrenodeset-20251126041817737800000001]. +[2025-11-30 19:38:11] [SUCCESS] Deleted lustre06-compute-lustrenodeset-20251126041817737800000001 +[2025-11-30 19:38:11] [EXECUTE] Deleting Instance Template: lustre06-controller-default-20251126041828459800000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustre06-controller-default-20251126041828459800000003]. +[2025-11-30 19:38:14] [SUCCESS] Deleted lustre06-controller-default-20251126041828459800000003 +[2025-11-30 19:38:14] [EXECUTE] Deleting Instance Template: lustre06-login-slurm-login-20251126041817754000000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustre06-login-slurm-login-20251126041817754000000002]. +[2025-11-30 19:38:17] [SUCCESS] Deleted lustre06-login-slurm-login-20251126041817754000000002 +[2025-11-30 19:38:17] [EXECUTE] Deleting Instance Template: lustredev0-compute-a216nodeset-20251127075747115200000008 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-a216nodeset-20251127075747115200000008]. +[2025-11-30 19:38:20] [SUCCESS] Deleted lustredev0-compute-a216nodeset-20251127075747115200000008 +[2025-11-30 19:38:20] [EXECUTE] Deleting Instance Template: lustredev0-compute-a28nodeset-20251127075746973800000005 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-a28nodeset-20251127075746973800000005]. +[2025-11-30 19:38:23] [SUCCESS] Deleted lustredev0-compute-a28nodeset-20251127075746973800000005 +[2025-11-30 19:38:23] [EXECUTE] Deleting Instance Template: lustredev0-compute-c2dnodeset-20251127075746534500000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-c2dnodeset-20251127075746534500000004]. +[2025-11-30 19:38:26] [SUCCESS] Deleted lustredev0-compute-c2dnodeset-20251127075746534500000004 +[2025-11-30 19:38:26] [EXECUTE] Deleting Instance Template: lustredev0-compute-c2nodeset-20251127075746393800000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-c2nodeset-20251127075746393800000003]. +[2025-11-30 19:38:29] [SUCCESS] Deleted lustredev0-compute-c2nodeset-20251127075746393800000003 +[2025-11-30 19:38:29] [EXECUTE] Deleting Instance Template: lustredev0-compute-c3nodeset-20251127075747101600000006 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-c3nodeset-20251127075747101600000006]. +[2025-11-30 19:38:32] [SUCCESS] Deleted lustredev0-compute-c3nodeset-20251127075747101600000006 +[2025-11-30 19:38:32] [EXECUTE] Deleting Instance Template: lustredev0-compute-h3nodeset-20251127075747131700000009 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-h3nodeset-20251127075747131700000009]. +[2025-11-30 19:38:35] [SUCCESS] Deleted lustredev0-compute-h3nodeset-20251127075747131700000009 +[2025-11-30 19:38:35] [EXECUTE] Deleting Instance Template: lustredev0-compute-n2nodeset-20251127075747112000000007 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-n2nodeset-20251127075747112000000007]. +[2025-11-30 19:38:38] [SUCCESS] Deleted lustredev0-compute-n2nodeset-20251127075747112000000007 +[2025-11-30 19:38:38] [EXECUTE] Deleting Instance Template: lustredev0-controller-default-20251127075745707800000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-controller-default-20251127075745707800000001]. +[2025-11-30 19:38:42] [SUCCESS] Deleted lustredev0-controller-default-20251127075745707800000001 +[2025-11-30 19:38:42] [EXECUTE] Deleting Instance Template: lustredev0-login-slurm-login-20251127075746027600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-login-slurm-login-20251127075746027600000002]. +[2025-11-30 19:38:45] [SUCCESS] Deleted lustredev0-login-slurm-login-20251127075746027600000002 +[2025-11-30 19:38:45] [EXECUTE] Deleting Instance Template: lustreprod-compute-a216nodeset-20251127085007177000000008 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-a216nodeset-20251127085007177000000008]. +[2025-11-30 19:38:48] [SUCCESS] Deleted lustreprod-compute-a216nodeset-20251127085007177000000008 +[2025-11-30 19:38:48] [EXECUTE] Deleting Instance Template: lustreprod-compute-a28nodeset-20251127085007152400000005 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-a28nodeset-20251127085007152400000005]. +[2025-11-30 19:38:51] [SUCCESS] Deleted lustreprod-compute-a28nodeset-20251127085007152400000005 +[2025-11-30 19:38:51] [EXECUTE] Deleting Instance Template: lustreprod-compute-c2dnodeset-20251127085007154700000006 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-c2dnodeset-20251127085007154700000006]. +[2025-11-30 19:38:53] [SUCCESS] Deleted lustreprod-compute-c2dnodeset-20251127085007154700000006 +[2025-11-30 19:38:54] [EXECUTE] Deleting Instance Template: lustreprod-compute-c2nodeset-20251127085007178300000009 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-c2nodeset-20251127085007178300000009]. +[2025-11-30 19:38:57] [SUCCESS] Deleted lustreprod-compute-c2nodeset-20251127085007178300000009 +[2025-11-30 19:38:57] [EXECUTE] Deleting Instance Template: lustreprod-compute-c3nodeset-20251127085007120300000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-c3nodeset-20251127085007120300000003]. +[2025-11-30 19:39:00] [SUCCESS] Deleted lustreprod-compute-c3nodeset-20251127085007120300000003 +[2025-11-30 19:39:00] [EXECUTE] Deleting Instance Template: lustreprod-compute-h3nodeset-20251127085007156500000007 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-h3nodeset-20251127085007156500000007]. +[2025-11-30 19:39:03] [SUCCESS] Deleted lustreprod-compute-h3nodeset-20251127085007156500000007 +[2025-11-30 19:39:03] [EXECUTE] Deleting Instance Template: lustreprod-compute-n2nodeset-20251127085007147000000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-n2nodeset-20251127085007147000000004]. +[2025-11-30 19:39:06] [SUCCESS] Deleted lustreprod-compute-n2nodeset-20251127085007147000000004 +[2025-11-30 19:39:06] [EXECUTE] Deleting Instance Template: lustreprod-controller-default-20251127085005432900000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-controller-default-20251127085005432900000001]. +[2025-11-30 19:39:10] [SUCCESS] Deleted lustreprod-controller-default-20251127085005432900000001 +[2025-11-30 19:39:10] [EXECUTE] Deleting Instance Template: lustreprod-login-slurm-login-20251127085005549500000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-login-slurm-login-20251127085005549500000002]. +[2025-11-30 19:39:12] [SUCCESS] Deleted lustreprod-login-slurm-login-20251127085005549500000002 +[2025-11-30 19:39:12] [EXECUTE] Deleting Instance Template: lustreqa05-compute-a216nodeset-20251127064214383600000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-a216nodeset-20251127064214383600000003]. +[2025-11-30 19:39:15] [SUCCESS] Deleted lustreqa05-compute-a216nodeset-20251127064214383600000003 +[2025-11-30 19:39:15] [EXECUTE] Deleting Instance Template: lustreqa05-compute-a28nodeset-20251127064214402000000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-a28nodeset-20251127064214402000000004]. +[2025-11-30 19:39:18] [SUCCESS] Deleted lustreqa05-compute-a28nodeset-20251127064214402000000004 +[2025-11-30 19:39:18] [EXECUTE] Deleting Instance Template: lustreqa05-compute-c2dnodeset-20251127064214467900000009 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-c2dnodeset-20251127064214467900000009]. +[2025-11-30 19:39:21] [SUCCESS] Deleted lustreqa05-compute-c2dnodeset-20251127064214467900000009 +[2025-11-30 19:39:21] [EXECUTE] Deleting Instance Template: lustreqa05-compute-c2nodeset-20251127064214424900000005 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-c2nodeset-20251127064214424900000005]. +[2025-11-30 19:39:24] [SUCCESS] Deleted lustreqa05-compute-c2nodeset-20251127064214424900000005 +[2025-11-30 19:39:24] [EXECUTE] Deleting Instance Template: lustreqa05-compute-c3nodeset-20251127064214452600000007 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-c3nodeset-20251127064214452600000007]. +[2025-11-30 19:39:27] [SUCCESS] Deleted lustreqa05-compute-c3nodeset-20251127064214452600000007 +[2025-11-30 19:39:27] [EXECUTE] Deleting Instance Template: lustreqa05-compute-h3nodeset-20251127064214457600000008 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-h3nodeset-20251127064214457600000008]. +[2025-11-30 19:39:30] [SUCCESS] Deleted lustreqa05-compute-h3nodeset-20251127064214457600000008 +[2025-11-30 19:39:30] [EXECUTE] Deleting Instance Template: lustreqa05-compute-n2nodeset-20251127064214447800000006 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-n2nodeset-20251127064214447800000006]. +[2025-11-30 19:39:34] [SUCCESS] Deleted lustreqa05-compute-n2nodeset-20251127064214447800000006 +[2025-11-30 19:39:34] [EXECUTE] Deleting Instance Template: lustreqa05-controller-default-20251127064213231900000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-controller-default-20251127064213231900000001]. +[2025-11-30 19:39:37] [SUCCESS] Deleted lustreqa05-controller-default-20251127064213231900000001 +[2025-11-30 19:39:37] [EXECUTE] Deleting Instance Template: lustreqa05-login-slurm-login-20251127064213259800000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-login-slurm-login-20251127064213259800000002]. +[2025-11-30 19:39:40] [SUCCESS] Deleted lustreqa05-login-slurm-login-20251127064213259800000002 +[2025-11-30 19:39:40] [EXECUTE] Deleting Instance Template: lustretest-compute-a216nodeset-20251126044326259100000008 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-a216nodeset-20251126044326259100000008]. +[2025-11-30 19:39:43] [SUCCESS] Deleted lustretest-compute-a216nodeset-20251126044326259100000008 +[2025-11-30 19:39:43] [EXECUTE] Deleting Instance Template: lustretest-compute-a28nodeset-20251126044326243000000005 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-a28nodeset-20251126044326243000000005]. +[2025-11-30 19:39:46] [SUCCESS] Deleted lustretest-compute-a28nodeset-20251126044326243000000005 +[2025-11-30 19:39:46] [EXECUTE] Deleting Instance Template: lustretest-compute-c2dnodeset-20251126044326150300000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-c2dnodeset-20251126044326150300000003]. +[2025-11-30 19:39:49] [SUCCESS] Deleted lustretest-compute-c2dnodeset-20251126044326150300000003 +[2025-11-30 19:39:49] [EXECUTE] Deleting Instance Template: lustretest-compute-c2nodeset-20251126044326244400000006 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-c2nodeset-20251126044326244400000006]. +[2025-11-30 19:39:52] [SUCCESS] Deleted lustretest-compute-c2nodeset-20251126044326244400000006 +[2025-11-30 19:39:52] [EXECUTE] Deleting Instance Template: lustretest-compute-c3nodeset-20251126044326214900000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-c3nodeset-20251126044326214900000004]. +[2025-11-30 19:39:55] [SUCCESS] Deleted lustretest-compute-c3nodeset-20251126044326214900000004 +[2025-11-30 19:39:55] [EXECUTE] Deleting Instance Template: lustretest-compute-h3nodeset-20251126044326264800000009 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-h3nodeset-20251126044326264800000009]. +[2025-11-30 19:39:58] [SUCCESS] Deleted lustretest-compute-h3nodeset-20251126044326264800000009 +[2025-11-30 19:39:58] [EXECUTE] Deleting Instance Template: lustretest-compute-n2nodeset-20251126044326256800000007 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-n2nodeset-20251126044326256800000007]. +[2025-11-30 19:40:01] [SUCCESS] Deleted lustretest-compute-n2nodeset-20251126044326256800000007 +[2025-11-30 19:40:01] [EXECUTE] Deleting Instance Template: lustretest-controller-default-20251126044324852600000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-controller-default-20251126044324852600000001]. +[2025-11-30 19:40:04] [SUCCESS] Deleted lustretest-controller-default-20251126044324852600000001 +[2025-11-30 19:40:04] [EXECUTE] Deleting Instance Template: lustretest-login-slurm-login-20251126044324988400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-login-slurm-login-20251126044324988400000002]. +[2025-11-30 19:40:07] [SUCCESS] Deleted lustretest-login-slurm-login-20251126044324988400000002 +[2025-11-30 19:40:07] [EXECUTE] Deleting Instance Template: mainek-compute-a3ultranodeset-20251121092031664700000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/mainek-compute-a3ultranodeset-20251121092031664700000003]. +[2025-11-30 19:40:10] [SUCCESS] Deleted mainek-compute-a3ultranodeset-20251121092031664700000003 +[2025-11-30 19:40:10] [EXECUTE] Deleting Instance Template: mainek-controller-default-20251121092028913400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/mainek-controller-default-20251121092028913400000001]. +[2025-11-30 19:40:14] [SUCCESS] Deleted mainek-controller-default-20251121092028913400000001 +[2025-11-30 19:40:14] [EXECUTE] Deleting Instance Template: mainek-login-slurm-login-20251121092028928700000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/mainek-login-slurm-login-20251121092028928700000002]. +[2025-11-30 19:40:17] [SUCCESS] Deleted mainek-login-slurm-login-20251121092028928700000002 +[2025-11-30 19:40:17] [EXECUTE] Deleting Instance Template: pkbh4dp2-compute-computenodeset-20250412005603828600000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/pkbh4dp2-compute-computenodeset-20250412005603828600000002]. +[2025-11-30 19:40:20] [SUCCESS] Deleted pkbh4dp2-compute-computenodeset-20250412005603828600000002 +[2025-11-30 19:40:20] [EXECUTE] Deleting Instance Template: pkrv6db5de-login-slurm-login-20250801090852525400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/pkrv6db5de-login-slurm-login-20250801090852525400000002]. +[2025-11-30 19:40:23] [SUCCESS] Deleted pkrv6db5de-login-slurm-login-20250801090852525400000002 +[2025-11-30 19:40:23] [EXECUTE] Deleting Instance Template: pzpk-compute-computenodeset-20250411004353418500000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/pzpk-compute-computenodeset-20250411004353418500000002]. +[2025-11-30 19:40:26] [SUCCESS] Deleted pzpk-compute-computenodeset-20250411004353418500000002 +[2025-11-30 19:40:26] [EXECUTE] Deleting Instance Template: pzpk-controller-default-20250411004353405200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/pzpk-controller-default-20250411004353405200000001]. +[2025-11-30 19:40:29] [SUCCESS] Deleted pzpk-controller-default-20250411004353405200000001 +[2025-11-30 19:40:29] [EXECUTE] Deleting Instance Template: ractesh4d-compute-h4dnodeset-20251016152751862600000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesh4d-compute-h4dnodeset-20251016152751862600000003]. +[2025-11-30 19:40:32] [SUCCESS] Deleted ractesh4d-compute-h4dnodeset-20251016152751862600000003 +[2025-11-30 19:40:32] [EXECUTE] Deleting Instance Template: ractesh4d-controller-default-20251016152751828700000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesh4d-controller-default-20251016152751828700000001]. +[2025-11-30 19:40:35] [SUCCESS] Deleted ractesh4d-controller-default-20251016152751828700000001 +[2025-11-30 19:40:35] [EXECUTE] Deleting Instance Template: ractesh4d-login-slurm-login-20251016152751847100000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesh4d-login-slurm-login-20251016152751847100000002]. +[2025-11-30 19:40:38] [SUCCESS] Deleted ractesh4d-login-slurm-login-20251016152751847100000002 +[2025-11-30 19:40:38] [EXECUTE] Deleting Instance Template: ractesth4d-compute-h4dnodeset-20251013062737759200000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesth4d-compute-h4dnodeset-20251013062737759200000003]. +[2025-11-30 19:40:41] [SUCCESS] Deleted ractesth4d-compute-h4dnodeset-20251013062737759200000003 +[2025-11-30 19:40:41] [EXECUTE] Deleting Instance Template: ractesth4d-controller-default-20251013062737731900000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesth4d-controller-default-20251013062737731900000001]. +[2025-11-30 19:40:44] [SUCCESS] Deleted ractesth4d-controller-default-20251013062737731900000001 +[2025-11-30 19:40:44] [EXECUTE] Deleting Instance Template: ractesth4d-login-slurm-login-20251013062737750800000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesth4d-login-slurm-login-20251013062737750800000002]. +[2025-11-30 19:40:47] [SUCCESS] Deleted ractesth4d-login-slurm-login-20251013062737750800000002 +[2025-11-30 19:40:47] [EXECUTE] Deleting Instance Template: rock8a070a-compute-computenodeset-20250912110554713000000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/rock8a070a-compute-computenodeset-20250912110554713000000004]. +[2025-11-30 19:40:50] [SUCCESS] Deleted rock8a070a-compute-computenodeset-20250912110554713000000004 +[2025-11-30 19:40:50] [EXECUTE] Deleting Instance Template: rock8a070a-compute-debugnodeset-20250912110554709900000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/rock8a070a-compute-debugnodeset-20250912110554709900000003]. +[2025-11-30 19:40:53] [SUCCESS] Deleted rock8a070a-compute-debugnodeset-20250912110554709900000003 +[2025-11-30 19:40:53] [EXECUTE] Deleting Instance Template: rock8a070a-compute-h3nodeset-20250912110554735800000005 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/rock8a070a-compute-h3nodeset-20250912110554735800000005]. +[2025-11-30 19:40:56] [SUCCESS] Deleted rock8a070a-compute-h3nodeset-20250912110554735800000005 +[2025-11-30 19:40:56] [EXECUTE] Deleting Instance Template: rock8a070a-controller-default-20250912110554351400000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/rock8a070a-controller-default-20250912110554351400000001]. +[2025-11-30 19:41:00] [SUCCESS] Deleted rock8a070a-controller-default-20250912110554351400000001 +[2025-11-30 19:41:00] [EXECUTE] Deleting Instance Template: rock8a070a-login-slurm-login-20250912110554546400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/rock8a070a-login-slurm-login-20250912110554546400000002]. +[2025-11-30 19:41:03] [SUCCESS] Deleted rock8a070a-login-slurm-login-20250912110554546400000002 +[2025-11-30 19:41:03] [EXECUTE] Deleting Instance Template: simtestnet-compute-a3ultranodeset-20251128190204679000000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/simtestnet-compute-a3ultranodeset-20251128190204679000000002]. +[2025-11-30 19:41:06] [SUCCESS] Deleted simtestnet-compute-a3ultranodeset-20251128190204679000000002 +[2025-11-30 19:41:06] [EXECUTE] Deleting Instance Template: simtestnet-controller-default-20251128190210353600000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/simtestnet-controller-default-20251128190210353600000003]. +[2025-11-30 19:41:09] [SUCCESS] Deleted simtestnet-controller-default-20251128190210353600000003 +[2025-11-30 19:41:09] [EXECUTE] Deleting Instance Template: simtestnet-login-slurm-login-20251128190201729100000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/simtestnet-login-slurm-login-20251128190201729100000001]. +[2025-11-30 19:41:12] [SUCCESS] Deleted simtestnet-login-slurm-login-20251128190201729100000001 +[2025-11-30 19:41:12] [EXECUTE] Deleting Instance Template: slurm0-compute-a3nodeset-20251009153324036900000004 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm0-compute-a3nodeset-20251009153324036900000004]. +[2025-11-30 19:41:15] [SUCCESS] Deleted slurm0-compute-a3nodeset-20251009153324036900000004 +[2025-11-30 19:41:15] [EXECUTE] Deleting Instance Template: slurm0-compute-debugnodeset-20251009153324023000000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm0-compute-debugnodeset-20251009153324023000000003]. +[2025-11-30 19:41:18] [SUCCESS] Deleted slurm0-compute-debugnodeset-20251009153324023000000003 +[2025-11-30 19:41:18] [EXECUTE] Deleting Instance Template: slurm0-controller-default-20251009153304999900000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm0-controller-default-20251009153304999900000001]. +[2025-11-30 19:41:21] [SUCCESS] Deleted slurm0-controller-default-20251009153304999900000001 +[2025-11-30 19:41:21] [EXECUTE] Deleting Instance Template: slurm0-login-login-20251009153305041200000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm0-login-login-20251009153305041200000002]. +[2025-11-30 19:41:24] [SUCCESS] Deleted slurm0-login-login-20251009153305041200000002 +[2025-11-30 19:41:24] [EXECUTE] Deleting Instance Template: slurm9ff-compute-debugnodeset-20250925215605513600000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm9ff-compute-debugnodeset-20250925215605513600000003]. +[2025-11-30 19:41:27] [SUCCESS] Deleted slurm9ff-compute-debugnodeset-20250925215605513600000003 +[2025-11-30 19:41:27] [EXECUTE] Deleting Instance Template: slurm9ff-controller-default-20250925215558268500000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm9ff-controller-default-20250925215558268500000001]. +[2025-11-30 19:41:31] [SUCCESS] Deleted slurm9ff-controller-default-20250925215558268500000001 +[2025-11-30 19:41:31] [EXECUTE] Deleting Instance Template: slurm9ff-login-slurm-login-20250925215558759400000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm9ff-login-slurm-login-20250925215558759400000002]. +[2025-11-30 19:41:33] [SUCCESS] Deleted slurm9ff-login-slurm-login-20250925215558759400000002 +[2025-11-30 19:41:33] [EXECUTE] Deleting Instance Template: slurmf1c-compute-debugnodeset-20250925064111290200000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmf1c-compute-debugnodeset-20250925064111290200000002]. +[2025-11-30 19:41:37] [SUCCESS] Deleted slurmf1c-compute-debugnodeset-20250925064111290200000002 +[2025-11-30 19:41:37] [EXECUTE] Deleting Instance Template: slurmf1c-controller-default-20250925064111186200000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmf1c-controller-default-20250925064111186200000001]. +[2025-11-30 19:41:40] [SUCCESS] Deleted slurmf1c-controller-default-20250925064111186200000001 +[2025-11-30 19:41:40] [EXECUTE] Deleting Instance Template: slurmf1c-login-slurm-login-20250925064111290800000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmf1c-login-slurm-login-20250925064111290800000003]. +[2025-11-30 19:41:43] [SUCCESS] Deleted slurmf1c-login-slurm-login-20250925064111290800000003 +[2025-11-30 19:41:43] [EXECUTE] Deleting Instance Template: slurmflex-compute-nodeset-20250806200217240000000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmflex-compute-nodeset-20250806200217240000000001]. +[2025-11-30 19:41:46] [SUCCESS] Deleted slurmflex-compute-nodeset-20250806200217240000000001 +[2025-11-30 19:41:46] [EXECUTE] Deleting Instance Template: slurmflex-compute-nodeset-20250912233249589700000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmflex-compute-nodeset-20250912233249589700000002]. +[2025-11-30 19:41:49] [SUCCESS] Deleted slurmflex-compute-nodeset-20250912233249589700000002 +[2025-11-30 19:41:49] [EXECUTE] Deleting Instance Template: slurmflex-compute-nodeset-20251111083658962700000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmflex-compute-nodeset-20251111083658962700000002]. +[2025-11-30 19:41:52] [SUCCESS] Deleted slurmflex-compute-nodeset-20251111083658962700000002 +[2025-11-30 19:41:52] [EXECUTE] Deleting Instance Template: slurmsimpl-compute-nodeset-20250708064936486300000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmsimpl-compute-nodeset-20250708064936486300000001]. +[2025-11-30 19:41:55] [SUCCESS] Deleted slurmsimpl-compute-nodeset-20250708064936486300000001 +[2025-11-30 19:41:55] [EXECUTE] Deleting Instance Template: slurmsimpl-compute-nodeset-20250814191439012800000001 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmsimpl-compute-nodeset-20250814191439012800000001]. +[2025-11-30 19:41:59] [SUCCESS] Deleted slurmsimpl-compute-nodeset-20250814191439012800000001 +[2025-11-30 19:41:59] [EXECUTE] Deleting Instance Template: slurmsimpl-controller-default-20250708064945457400000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmsimpl-controller-default-20250708064945457400000003]. +[2025-11-30 19:42:02] [SUCCESS] Deleted slurmsimpl-controller-default-20250708064945457400000003 +[2025-11-30 19:42:02] [EXECUTE] Deleting Instance Template: slurmsimpl-controller-default-20250814191448978300000003 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmsimpl-controller-default-20250814191448978300000003]. +[2025-11-30 19:42:05] [SUCCESS] Deleted slurmsimpl-controller-default-20250814191448978300000003 +[2025-11-30 19:42:05] [EXECUTE] Deleting Instance Template: slurmsimpl-login-slurm-login-20250814191439021200000002 (Global) +Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmsimpl-login-slurm-login-20250814191439021200000002]. +[2025-11-30 19:42:08] [SUCCESS] Deleted slurmsimpl-login-slurm-login-20250814191439021200000002 +[2025-11-30 19:42:08] [SKIP] testing2 (In Exclusion List) +[2025-11-30 19:42:08] [SKIP] welp-insta-temp (In Exclusion List) +[2025-11-30 19:42:08] [INFO] CLEANUP RUN FINISHED From f5d98eb2b5594e1a8237582cf6e58e9538cd49fb Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Sun, 7 Dec 2025 18:06:09 +0000 Subject: [PATCH 03/19] Phase 3: Getting cloudbuild yaml+ Label support --- cleanup.sh => tools/cleanup.sh | 585 +++++++++---------------- tools/cloud-build/project-cleanup.yaml | 247 +++-------- exclusions.txt => tools/exclusions.txt | 1 - 3 files changed, 262 insertions(+), 571 deletions(-) rename cleanup.sh => tools/cleanup.sh (51%) rename exclusions.txt => tools/exclusions.txt (99%) diff --git a/cleanup.sh b/tools/cleanup.sh similarity index 51% rename from cleanup.sh rename to tools/cleanup.sh index efdca4e398..49d639f64e 100755 --- a/cleanup.sh +++ b/tools/cleanup.sh @@ -4,33 +4,18 @@ # CONFIGURATION & GLOBAL VARIABLES # ============================================================================== -set -u -set -o pipefail - -# Output Redirection -LOG_FILE="template.txt" -exec >> "$LOG_FILE" 2>&1 - -PROJECT_ID="hpc-toolkit-dev" -DRY_RUN="false" # Set to "false" to actually delete -EXCLUSION_FILE="exclusions.txt" -DELETE_LIMIT=200 -PROTECTED_SUBSTRING="topology" - -# Service Account Config -SA_DELETE_PREFIX="test-sa-" - -# VM Image Config -IMAGE_AGE_DAYS=60 - -# Time Calculations -# Standard Resource Cutoff (1 hour buffer) -CUTOFF_TIME=$(date -d "5 hours ago" -u +%Y-%m-%dT%H:%M:%S%z) -# Image Cutoff (60 days ago) -CUTOFF_TIME_IMAGES=$(date -d "$IMAGE_AGE_DAYS days ago" -u +%Y-%m-%dT%H:%M:%S%z) - # Associative array for exclusions declare -A EXCLUSION_MAP +DELETE_LIMIT=200 +ERROR_COUNT=0 + +# Environment Variables expected from Cloud Build +PROJECT_ID="${PROJECT_ID:-hpc-toolkit-dev}" +DRY_RUN="${DRY_RUN:-true}" +EXCLUSION_FILE="${EXCLUSION_FILE:-tools/exclusions.txt}" +CUTOFF_TIME="${CUTOFF_TIME:-$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%S%z)}" +IMAGE_AGE_DAYS="${IMAGE_AGE_DAYS:-60}" +CUTOFF_TIME_IMAGES="${CUTOFF_TIME_IMAGES:-$(date -d "$IMAGE_AGE_DAYS days ago" -u +%Y-%m-%dT%H:%M:%S%z)}" # ============================================================================== # HELPER FUNCTIONS @@ -47,15 +32,15 @@ check_dependencies() { for cmd in "${dependencies[@]}"; do if ! command -v "$cmd" &> /dev/null; then log "ERROR" "Missing required dependency: $cmd" - exit 1 + exit 1 # Dependencies are critical, we must exit immediately here. fi done } load_exclusions() { if [[ ! -f "$EXCLUSION_FILE" ]]; then - log "WARNING" "Exclusion file not found: $EXCLUSION_FILE. Proceeding without exclusions." - return + log "ERROR" "Exclusion file not found: $EXCLUSION_FILE." + exit 1 fi log "INFO" "Loading exclusions from $EXCLUSION_FILE..." @@ -70,14 +55,41 @@ load_exclusions() { is_excluded() { local resource_name="$1" - if [[ "$resource_name" == *"$PROTECTED_SUBSTRING"* ]]; then - log "SKIP" "$resource_name (Protected Substring)" - return 0 - fi + local labels_str="${2:-}" + if [[ -n "${EXCLUSION_MAP[$resource_name]:-}" ]]; then log "SKIP" "$resource_name (In Exclusion List)" return 0 fi + + if [[ -n "$labels_str" ]]; then + IFS=';' read -ra LABEL_PAIRS <<< "$labels_str" + for PAIR in "${LABEL_PAIRS[@]}"; do + local KEY VAL + KEY="${PAIR%%=*}" + VAL="${PAIR#*=}" + if [[ "$KEY" == "do-not-delete" ]]; then + if [[ "$VAL" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then + local exp_seconds + if ! exp_seconds=$(date -d "$VAL + 1 day" +%s 2>/dev/null); then + log "WARNING" "$resource_name (Label: do-not-delete invalid date value: $VAL)" + else + local current_seconds + current_seconds=$(date +%s) + if [[ "$exp_seconds" -gt "$current_seconds" ]]; then + log "SKIP" "$resource_name (Label: do-not-delete=$VAL, valid until end of day)" + return 0 + else + log "INFO" "$resource_name (Label: do-not-delete=$VAL expired)" + fi + fi + else + log "WARNING" "$resource_name (Label: do-not-delete invalid date format: $VAL, expected YYYY-MM-DD)" + fi + break + fi + done + fi return 1 } @@ -95,6 +107,7 @@ execute_delete() { log "SUCCESS" "Deleted $resource_name" else log "ERROR" "Failed to delete $resource_name" + ((ERROR_COUNT++)) || true fi fi } @@ -107,36 +120,31 @@ process_resources() { local label="$1" local list_command="$2" local delete_command_base="$3" - local scope_type="$4" # location, zone, region, or none + local scope_type="$4" log "INFO" "--- Processing: $label (Limit: $DELETE_LIMIT) ---" local resources if ! resources=$(eval "$list_command"); then log "ERROR" "Failed to list $label" - return + ((ERROR_COUNT++)) || true + return 0 fi if [[ -z "$resources" ]]; then log "INFO" "No $label found matching criteria." - return + return 0 fi local count=0 - while read -r line; do - [[ -z "$line" ]] && continue - local name scope - read -r name scope <<< "$line" - - if [[ -z "$name" ]]; then continue; fi - - # --- LIMIT CHECK --- - if [[ $count -ge $DELETE_LIMIT ]]; then + while IFS=$'\t' read -r name scope labels_str; do + [[ -z "$name" ]] && continue + if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for $label." break fi - if is_excluded "$name"; then continue; fi + if is_excluded "$name" "${labels_str:-}"; then continue; fi local final_cmd="$delete_command_base \"$name\" --quiet" if [[ "$scope_type" != "none" && -n "$scope" ]]; then @@ -144,7 +152,7 @@ process_resources() { fi execute_delete "$label" "$name" "$final_cmd" "${scope:-(Global)}" - ((count++)) + ((count++)) || true done <<< "$resources" } @@ -154,258 +162,174 @@ process_resources() { process_instance_templates() { log "INFO" "--- Processing: Instance Templates (Limit: $DELETE_LIMIT) ---" - - # List templates created before CUTOFF_TIME local templates if ! templates=$(gcloud compute instance-templates list \ --project="$PROJECT_ID" \ --filter="creationTimestamp < '$CUTOFF_TIME'" \ - --format="value(name)" | sort); then + --format="value(name, labels)" | sort); then log "ERROR" "Failed to list instance templates." - return 1 - fi - - if [[ -z "$templates" ]]; then - log "INFO" "No instance templates found matching criteria." - return + ((ERROR_COUNT++)) || true + return 0 fi + if [[ -z "$templates" ]]; then log "INFO" "No instance templates found matching criteria."; return 0; fi local count=0 - while read -r name; do + while IFS=$'\t' read -r name labels_str; do if [[ -z "$name" ]]; then continue; fi - - # --- LIMIT CHECK --- - if [[ $count -ge $DELETE_LIMIT ]]; then - log "INFO" "Hit delete limit ($DELETE_LIMIT) for Instance Templates." - break - fi - - if is_excluded "$name"; then continue; fi - + if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Instance Templates."; break; fi + if is_excluded "$name" "${labels_str:-}"; then continue; fi execute_delete "Instance Template" "$name" \ "gcloud compute instance-templates delete \"$name\" --project=\"$PROJECT_ID\" --quiet" \ "(Global)" - - ((count++)) + ((count++)) || true done <<< "$templates" } process_addresses() { log "INFO" "--- Processing: Compute Addresses ---" - - # These use the Standard Processor, so the limit is handled inside process_resources process_resources "Regional Address" \ - "gcloud compute addresses list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND region:*\" --format=\"value(name,region)\" | sort" \ + "gcloud compute addresses list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND region:*\" --format=\"value(name,region,labels)\" | sort" \ "gcloud compute addresses delete --project=\"$PROJECT_ID\"" \ "region" - process_resources "Global Address" \ - "gcloud compute addresses list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND NOT region:*\" --format=\"value(name)\" | sort" \ + "gcloud compute addresses list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND NOT region:*\" --format=\"value[separator='t'](name, labels)\" | awk 'BEGIN{OFS=\"\t\"} {if (NF==1) print \$1, \"Global\", \"\"; else print \$1, \"Global\", \$2}' | sort" \ "gcloud compute addresses delete --project=\"$PROJECT_ID\" --global" \ "none" } - process_vpc_peerings() { log "INFO" "--- Processing: VPC Peerings (Limit: $DELETE_LIMIT) ---" - local networks_json if ! networks_json=$(gcloud compute networks list --project="$PROJECT_ID" --format="json"); then log "ERROR" "Failed to list networks." - return 1 - fi - - if [[ -z "$networks_json" || "$networks_json" == "[]" ]]; then - log "INFO" "No networks found in project." - return + ((ERROR_COUNT++)) || true + return 0 fi + if [[ -z "$networks_json" || "$networks_json" == "[]" ]]; then log "INFO" "No networks found in project."; return 0; fi local count=0 - - # Use process substitution <(...) to avoid subshell issues so 'count' updates correctly while IFS= read -r net_obj; do if [[ $count -ge $DELETE_LIMIT ]]; then break; fi - local net_name net_name=$(echo "$net_obj" | jq -r '.name') - if [[ -z "$net_name" || "$net_name" == "null" ]]; then continue; fi - # log "DEBUG" "Checking network: $net_name" - - # Check for peerings array inside the network object local peerings_json peerings_json=$(echo "$net_obj" | jq -c '.peerings // []') + if [[ "$peerings_json" == "[]" || "$peerings_json" == "null" ]]; then continue; fi - # If empty array or null, skip - if [[ "$peerings_json" == "[]" || "$peerings_json" == "null" ]]; then - continue - fi - - # Inner loop for peerings while IFS= read -r peering_obj; do if [[ $count -ge $DELETE_LIMIT ]]; then break; fi - local peering_name peering_name=$(echo "$peering_obj" | jq -r '.name') + if [[ -z "$peering_name" || "$peering_name" == "null" ]]; then continue; fi - if [[ -z "$peering_name" || "$peering_name" == "null" ]]; then - log "DEBUG" " Skipping peering with no name" - continue - fi + if is_excluded "$peering_name" || is_excluded "$net_name"; then continue; fi local peer_network peer_network=$(echo "$peering_obj" | jq -r '.network // ""') local state state=$(echo "$peering_obj" | jq -r '.state // ""') - if is_excluded "$peering_name" || is_excluded "$net_name"; then - continue - fi - if [[ "$peering_name" == "servicenetworking-googleapis-com" ]]; then - log "INFO" " [ACTION] Deleting Service Networking peering on: $net_name" execute_delete "Service Peering" "$peering_name" \ "gcloud services vpc-peerings delete --service=servicenetworking.googleapis.com --network=\"$net_name\" --project=\"$PROJECT_ID\" --quiet" \ "(Network: $net_name)" - ((count++)) - elif [[ "$peering_name" == filestore-peer-* ]]; then - log "INFO" " [SKIP] Managed Filestore peering: $peering_name on $net_name. This is tied to a Filestore instance lifecycle." - continue - elif [[ "$peer_network" == *"/global/networks/servicenetworking" ]]; then - log "INFO" " [SKIP] Reverse Service Networking peering: $peering_name on $net_name" - continue + ((count++)) || true + elif [[ "$peering_name" == filestore-peer-* ]]; then continue; + elif [[ "$peer_network" == *"/global/networks/servicenetworking" ]]; then continue; else - # Standard VPC Peering - log "INFO" " [ACTION] Deleting Standard VPC peering: $peering_name on: $net_name (State: $state)" execute_delete "VPC Peering" "$peering_name" \ "gcloud compute networks peerings delete \"$peering_name\" --network=\"$net_name\" --project=\"$PROJECT_ID\" --quiet" \ "(Network: $net_name, State: $state)" - ((count++)) + ((count++)) || true fi - done < <(echo "$peerings_json" | jq -c '.[]') - - done < <(echo "$networks_json" | jq -c '.[]') - - if [[ $count -ge $DELETE_LIMIT ]]; then - log "INFO" "Hit delete limit ($DELETE_LIMIT) for VPC Peerings." - fi + done < <(echo "$peerings_json" | jq -c '.[]') + done < <(echo "$networks_json" | jq -c '.[]') + if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for VPC Peerings."; fi log "INFO" "Finished processing VPC Peerings. $count peerings actioned." } -process_service_accounts() { - log "INFO" "--- Processing: Service Accounts (Prefix: $SA_DELETE_PREFIX) ---" - - # We use 'head -n' here to enforce the limit at the list level - local sas - sas=$(gcloud iam service-accounts list --project="$PROJECT_ID" \ - --filter="email ~ ^$SA_DELETE_PREFIX" \ - --format="value(email)" | head -n "$DELETE_LIMIT") - - if [[ -z "$sas" ]]; then - log "INFO" "No Service Accounts found matching prefix." - return - fi - - for email in $sas; do - if is_excluded "$email"; then continue; fi - execute_delete "Service Account" "$email" \ - "gcloud iam service-accounts delete \"$email\" --project=\"$PROJECT_ID\" --quiet" - done -} - process_iam_deleted_members() { log "INFO" "--- Processing: IAM Role Bindings for Deleted SAs (Limit: $DELETE_LIMIT) ---" - local policy_json if ! policy_json=$(gcloud projects get-iam-policy "$PROJECT_ID" --format=json); then log "ERROR" "Failed to get IAM policy." - return + ((ERROR_COUNT++)) || true + return 0 fi - local deleted_bindings deleted_bindings=$(echo "$policy_json" | jq -r '.bindings[] | .role as $r | .members[] | select(startswith("deleted:serviceAccount:")) | "\($r)\t\(.)"') - - if [[ -z "$deleted_bindings" ]]; then - log "INFO" "No 'deleted:serviceAccount' bindings found." - return - fi + if [[ -z "$deleted_bindings" ]]; then log "INFO" "No 'deleted:serviceAccount' bindings found."; return 0; fi local count=0 - while IFS=$'\t' read -r role member; do if [[ -z "$role" || -z "$member" ]]; then continue; fi - - # --- LIMIT CHECK --- - if [[ $count -ge $DELETE_LIMIT ]]; then - log "INFO" "Hit delete limit ($DELETE_LIMIT) for IAM Bindings." - break - fi + if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for IAM Bindings."; break; fi + + local cmd="gcloud projects remove-iam-policy-binding \"$PROJECT_ID\" --member=\"$member\" --role=\"$role\" --condition=None --quiet" if [[ "$DRY_RUN" == "true" ]]; then log "DRY-RUN" "Would remove IAM binding: $member from role $role" else log "EXECUTE" "Removing IAM binding: $member from role $role" - gcloud projects remove-iam-policy-binding "$PROJECT_ID" \ - --member="$member" --role="$role" --condition=None --quiet >/dev/null || log "ERROR" "Failed to remove binding" + if ! eval "$cmd" >/dev/null; then + log "ERROR" "Failed to remove binding" + ((ERROR_COUNT++)) || true + fi fi - - ((count++)) + ((count++)) || true done <<< "$deleted_bindings" } process_vm_images() { log "INFO" "--- Processing: VM Images (Limit: $DELETE_LIMIT) ---" - local images - images=$(gcloud compute images list --project="$PROJECT_ID" --no-standard-images \ - --format="value(name,creationTimestamp)") - - if [[ -z "$images" ]]; then - log "INFO" "No custom VM images found." - return + if ! images=$(gcloud compute images list --project="$PROJECT_ID" --no-standard-images \ + --format="value(name,creationTimestamp,labels)"); then + log "ERROR" "Failed to list VM images" + ((ERROR_COUNT++)) || true + return 0 fi + if [[ -z "$images" ]]; then log "INFO" "No custom VM images found."; return 0; fi + local cutoff_seconds - cutoff_seconds=$(date -d "$CUTOFF_TIME_IMAGES" +%s) - local count=0 + # Using date check directly; if date fails, we handle it inside the loop + if ! cutoff_seconds=$(date -d "$CUTOFF_TIME_IMAGES" +%s); then + log "ERROR" "Failed to calculate cutoff time for images" + ((ERROR_COUNT++)) || true + return 0 + fi - while read -r name timestamp; do + local count=0 + while IFS=$'\t' read -r name timestamp labels_str; do [[ -z "$name" ]] && continue - - # --- LIMIT CHECK --- - if [[ $count -ge $DELETE_LIMIT ]]; then - log "INFO" "Hit delete limit ($DELETE_LIMIT) for VM Images." - break - fi - - if is_excluded "$name"; then continue; fi + if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for VM Images."; break; fi + if is_excluded "$name" "${labels_str:-}"; then continue; fi local ts_seconds if ! ts_seconds=$(date -d "$timestamp" +%s 2>/dev/null); then log "WARNING" "Could not parse timestamp '$timestamp' for image $name. Skipping." continue fi - if [[ $ts_seconds -lt $cutoff_seconds ]]; then execute_delete "VM Image" "$name" \ "gcloud compute images delete \"$name\" --project=\"$PROJECT_ID\" --quiet" - ((count++)) + ((count++)) || true fi - done <<< "$images" } process_docker_images() { log "INFO" "--- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: $DELETE_LIMIT) ---" - - # 1. Calculate Cutoff (14 Days Ago) using UTC local cutoff_date cutoff_date=$(date -u -d "14 days ago" '+%Y-%m-%dT%H:%M:%SZ') local cutoff_seconds - if ! cutoff_seconds=$(date -u -d "$cutoff_date" +%s); then + if ! cutoff_seconds=$(date -u -d "$cutoff_date" +%s); then log "ERROR" "Failed to calculate cutoff_seconds." - return 1 + ((ERROR_COUNT++)) || true + return 0 fi log "INFO" "Policy: Delete 'test-runner' images updated before $cutoff_date (Unix: $cutoff_seconds)" @@ -413,243 +337,151 @@ process_docker_images() { local repo_name="hpc-toolkit-repo" local package_name="test-runner" local full_package_url="${location}-docker.pkg.dev/${PROJECT_ID}/${repo_name}/${package_name}" - - log "INFO" "Scanning Target Package: $full_package_url" - - # 5. List image versions for "test-runner" using CSV format + local images_output - if ! images_output=$(gcloud artifacts docker images list "$full_package_url" \ - --format="csv[no-heading](uri,updateTime)" \ - --sort-by="updateTime"); then - log "WARNING" "Failed to list images for $full_package_url" - return 1 - fi - - if [[ -z "$images_output" ]]; then - log "INFO" " > No image versions found for $package_name." - return + # Use if ! to catch failure without exiting + if ! images_output=$(gcloud artifacts docker images list "$full_package_url" --format="csv[no-heading](uri,updateTime)" --sort-by="updateTime" 2>/dev/null); then + log "WARNING" "Failed to list images for $full_package_url (Repo might not exist or empty)" + return 0 fi + if [[ -z "$images_output" ]]; then log "INFO" " > No image versions found for $package_name."; return 0; fi local count=0 - # 6. Iterate Images Line by Line (using CSV parsing) while IFS=, read -r full_image_ref update_time; do if [[ -z "$full_image_ref" ]]; then continue; fi + if [[ -z "$update_time" ]]; then continue; fi + if [[ "$full_image_ref" != *"@sha256:"* ]]; then continue; fi - # Sanity check: Ensure we have a valid timestamp - if [[ -z "$update_time" ]]; then - log "DEBUG" " Skipping line, missing timestamp for: $full_image_ref" - continue - fi - - # We only care about images with a digest in the URI (@sha256:...) - if [[ "$full_image_ref" != *"@sha256:"* ]]; then - # log "DEBUG" " Skipping URI without digest: $full_image_ref" - continue - fi - - # --- TIME CHECK --- local image_seconds - if ! image_seconds=$(date -u -d "$update_time" +%s 2>/dev/null); then - log "WARNING" " Could not parse date '$update_time' for $full_image_ref. Skipping." - continue - fi + if ! image_seconds=$(date -u -d "$update_time" +%s 2>/dev/null); then continue; fi - if [[ $image_seconds -ge $cutoff_seconds ]]; then - log "INFO" " [KEEP] .../test-runner... (Updated: $update_time) - Too new" + if [[ $image_seconds -ge $cutoff_seconds ]]; then continue; else - # --- DELETE LOGIC --- - if [[ $count -ge $DELETE_LIMIT ]]; then - log "INFO" "Hit delete limit ($DELETE_LIMIT) for Docker Images." - break # Break the loop, don't exit script - fi - - if is_excluded "$package_name"; then - log "INFO" " [SKIP] $package_name is excluded" - continue - fi - - log "INFO" " [DELETE] $full_image_ref (Updated: $update_time)" + if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Docker Images."; break; fi + if is_excluded "$package_name"; then continue; fi + if is_excluded "$full_image_ref"; then continue; fi execute_delete "Docker Image Version" "$full_image_ref" \ "gcloud artifacts docker images delete \"$full_image_ref\" --project=\"$PROJECT_ID\" --delete-tags --quiet" \ "(Updated: $update_time)" - ((count++)) + ((count++)) || true fi done <<< "$images_output" - log "INFO" "Finished Docker Image processing for $package_name. $count images marked for deletion." } - process_firewalls() { log "INFO" "--- Processing: Firewall Rules (Limit: $DELETE_LIMIT) ---" - local fws - fws=$(gcloud compute firewall-rules list --project="$PROJECT_ID" \ + if ! fws=$(gcloud compute firewall-rules list --project="$PROJECT_ID" \ --filter="creationTimestamp < '$CUTOFF_TIME'" \ - --format="value(name,network)" | sort) - - if [[ -z "$fws" ]]; then - log "INFO" "No Firewall Rules found matching criteria." - return + --format="value(name,network,labels)" | sort); then + log "ERROR" "Failed to list firewall rules" + ((ERROR_COUNT++)) || true + return 0 fi + if [[ -z "$fws" ]]; then log "INFO" "No Firewall Rules found matching criteria."; return 0; fi local count=0 - while read -r name network_uri; do + while IFS=$'\t' read -r name network_uri labels_str; do [[ -z "$name" ]] && continue - - # --- PROTECT DEFAULT NETWORK --- local network_name network_name=$(basename "$network_uri") if [[ "$network_name" == "default" ]]; then continue; fi - - # --- LIMIT CHECK --- - if [[ $count -ge $DELETE_LIMIT ]]; then - log "INFO" "Hit delete limit ($DELETE_LIMIT) for Firewall Rules." - break - fi - - if is_excluded "$name"; then continue; fi - + if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Firewall Rules."; break; fi + if is_excluded "$name" "${labels_str:-}"; then continue; fi execute_delete "Firewall Rule" "$name" \ "gcloud compute firewall-rules delete \"$name\" --project=\"$PROJECT_ID\" --quiet" - - ((count++)) - + ((count++)) || true done <<< "$fws" } process_filestore() { log "INFO" "--- Processing: Filestore Instances (Limit: $DELETE_LIMIT) ---" - - # 1. Fetch JSON output for robust parsing local fs_json - if ! fs_json=$(gcloud filestore instances list \ - --project="$PROJECT_ID" \ - --filter="createTime < '$CUTOFF_TIME'" \ - --format="json"); then # Removed 2>/dev/null to see gcloud errors + if ! fs_json=$(gcloud filestore instances list --project="$PROJECT_ID" --filter="createTime < '$CUTOFF_TIME'" --format="json"); then log "ERROR" "Failed to list Filestore instances." - return 1 - fi - - if [[ -z "$fs_json" || "$fs_json" == "[]" ]]; then - log "INFO" "No Filestore instances found matching criteria." - return + ((ERROR_COUNT++)) || true + return 0 fi + if [[ -z "$fs_json" || "$fs_json" == "[]" ]]; then log "INFO" "No Filestore instances found matching criteria."; return 0; fi - - # 2. Extract Location and Name using jq local fs_list - if ! fs_list=$(echo "$fs_json" | jq -r '.[] | select(.name) | "\(.name | split("/")[3])\t\(.name | split("/")[-1])"'); then + if ! fs_list=$(echo "$fs_json" | jq -r '.[] | select(.name) | "\(.name | split("/")[3])\t\(.name | split("/")[-1])\t\(.labels | to_entries | map("\(.key)=\(.value)") | join(";"))"'); then log "ERROR" "Failed to parse Filestore JSON with jq." - return 1 - fi - - if [[ -z "$fs_list" ]]; then - log "INFO" "No instances found after jq parsing." - return + ((ERROR_COUNT++)) || true + return 0 fi + if [[ -z "$fs_list" ]]; then log "INFO" "No instances found after jq parsing."; return 0; fi local count=0 - - # 3. Iterate - while IFS=$'\t' read -r location name; do - # Trim potential whitespace - location=$(echo "$location" | awk '{$1=$1};1') - name=$(echo "$name" | awk '{$1=$1};1') - - if [[ -z "$location" || -z "$name" ]]; then - log "DEBUG" "Skipping line with empty fields: location='${location}', name='${name}'" - continue - fi - - log "DEBUG" "Processing Instance: Name='${name}', Location='${location}'" - - # --- LIMIT CHECK --- - if [[ $count -ge $DELETE_LIMIT ]]; then - log "INFO" "Hit delete limit ($DELETE_LIMIT) for Filestore." - break - fi - - if is_excluded "$name"; then - log "INFO" "Skipping excluded Filestore: $name" - continue - fi - - # Construct delete command with explicit location + while IFS=$'\t' read -r location name labels_str; do + location=$(echo "$location" | awk '{$1=$1};1'); name=$(echo "$name" | awk '{$1=$1};1') + if [[ -z "$location" || -z "$name" ]]; then continue; fi + if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Filestore."; break; fi + if is_excluded "$name" "${labels_str:-}"; then continue; fi local delete_cmd="gcloud filestore instances delete \"$name\" --project=\"$PROJECT_ID\" --location=\"$location\" --quiet --force" - execute_delete "Filestore" "$name" "$delete_cmd" "($location)" - - ((count++)) - + ((count++)) || true done <<< "$fs_list" - log "INFO" "Finished processing Filestore instances. Attempted to delete $count." } process_subnetworks() { log "INFO" "--- Processing: Subnetworks (Limit: $DELETE_LIMIT) ---" - local subnets - subnets=$(gcloud compute networks subnets list --project="$PROJECT_ID" --filter="creationTimestamp < '$CUTOFF_TIME'" --format="value(name,region,network,selfLink)") + if ! subnets=$(gcloud compute networks subnets list --project="$PROJECT_ID" --filter="creationTimestamp < '$CUTOFF_TIME'" --format="value(name,region,network,selfLink)"); then + log "ERROR" "Failed to list subnets" + ((ERROR_COUNT++)) || true + return 0 + fi local count=0 while IFS=$'\t' read -r name region network_uri self_link; do [[ -z "$name" ]] && continue - - # --- PROTECT DEFAULT NETWORK --- local network_name=$(basename "$network_uri") if [[ "$network_name" == "default" ]]; then continue; fi - - # --- LIMIT CHECK --- - if [[ $count -ge $DELETE_LIMIT ]]; then - log "INFO" "Hit delete limit ($DELETE_LIMIT) for Subnetworks." - break - fi - + if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Subnetworks."; break; fi if is_excluded "$name"; then continue; fi - local dependents=$(gcloud compute addresses list --project="$PROJECT_ID" --filter="purpose=GCE_ENDPOINT AND region=(\"$region\") AND subnetwork=(\"$self_link\")" --format="value(name)") + # Note: listing dependents might fail, wrapping in error check not strictly necessary for deletion loop but good practice + local dependents + dependents=$(gcloud compute addresses list --project="$PROJECT_ID" --filter="purpose=GCE_ENDPOINT AND region=(\"$region\") AND subnetwork=(\"$self_link\")" --format="value(name)" 2>/dev/null || true) + for addr in $dependents; do execute_delete "Dependent Address" "$addr" "gcloud compute addresses delete \"$addr\" --project=\"$PROJECT_ID\" --region=\"$region\" --quiet" done - execute_delete "Subnetwork" "$name" "gcloud compute networks subnets delete \"$name\" --project=\"$PROJECT_ID\" --region=\"$region\" --quiet" - ((count++)) + ((count++)) || true done <<< "$subnets" } process_networks() { log "INFO" "--- Processing: VPC Networks (Limit: $DELETE_LIMIT) ---" - local networks - networks=$(gcloud compute networks list --project="$PROJECT_ID" --filter="creationTimestamp < '$CUTOFF_TIME'" --format="value(name,selfLink)") + if ! networks=$(gcloud compute networks list --project="$PROJECT_ID" --filter="creationTimestamp < '$CUTOFF_TIME'" --format="value(name,selfLink)"); then + log "ERROR" "Failed to list networks" + ((ERROR_COUNT++)) || true + return 0 + fi local count=0 while IFS=$'\t' read -r name self_link; do [[ -z "$name" ]] && continue - - # --- PROTECT DEFAULT NETWORK --- if [[ "$name" == "default" ]]; then continue; fi - - # --- LIMIT CHECK --- - if [[ $count -ge $DELETE_LIMIT ]]; then - log "INFO" "Hit delete limit ($DELETE_LIMIT) for Networks." - break - fi - + if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Networks."; break; fi if is_excluded "$name"; then continue; fi - echo "$name $self_link" - local routes=$(gcloud compute routes list --project="$PROJECT_ID" --filter="network=\"$self_link\"" --format="value(name)") - for r in $routes; do execute_delete "Dep. Route" "$r" "gcloud compute routes delete \"$r\" --project=\"$PROJECT_ID\" --quiet"; done - - local fws=$(gcloud compute firewall-rules list --project="$PROJECT_ID" --filter="network=\"$self_link\"" --format="value(name)") - for fw in $fws; do execute_delete "Dep. FW" "$fw" "gcloud compute firewall-rules delete \"$fw\" --project=\"$PROJECT_ID\" --quiet"; done - + local routes + routes=$(gcloud compute routes list --project="$PROJECT_ID" --filter="network=\"$self_link\"" --format="value(name)" 2>/dev/null || true) + for r in $routes; do if ! is_excluded "$r"; then execute_delete "Dep. Route" "$r" "gcloud compute routes delete \"$r\" --project=\"$PROJECT_ID\" --quiet"; fi; done + + local fws + fws=$(gcloud compute firewall-rules list --project="$PROJECT_ID" --filter="network=\"$self_link\"" --format="value(name)" 2>/dev/null || true) + for fw in $fws; do if ! is_excluded "$fw"; then execute_delete "Dep. FW" "$fw" "gcloud compute firewall-rules delete \"$fw\" --project=\"$PROJECT_ID\" --quiet"; fi; done + execute_delete "Network" "$name" "gcloud compute networks delete \"$name\" --project=\"$PROJECT_ID\" --quiet" - ((count++)) + ((count++)) || true done <<< "$networks" } @@ -662,50 +494,53 @@ main() { log "INFO" "Time Cutoff (General): $CUTOFF_TIME" log "INFO" "Time Cutoff (Images): $CUTOFF_TIME_IMAGES" log "INFO" "Delete Limit per Type: $DELETE_LIMIT" - + log "INFO" "DRY_RUN: $DRY_RUN" + log "INFO" "Exclusion File: $EXCLUSION_FILE" + check_dependencies load_exclusions # --- Phase 1: High Level Resources --- - # process_service_accounts - - # process_resources "GKE Cluster" \ - # "gcloud container clusters list --project=\"$PROJECT_ID\" --filter=\"createTime < '$CUTOFF_TIME'\" --format=\"value(name,location)\" | sort" \ - # "gcloud container clusters delete --project=\"$PROJECT_ID\"" "location" - + process_resources "GKE Cluster" \ + "gcloud container clusters list --project=\"$PROJECT_ID\" --filter=\"createTime < '$CUTOFF_TIME'\" --format=\"value(name,location,resourceLabels)\" | sort" \ + "gcloud container clusters delete --project=\"$PROJECT_ID\"" "location" process_instance_templates + process_resources "Compute Instance" \ + "gcloud compute instances list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,zone,labels)\" | sort" \ + "gcloud compute instances delete --project=\"$PROJECT_ID\" --delete-disks=all" "zone" + process_filestore - # process_resources "Compute Instance" \ - # "gcloud compute instances list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,zone)\" | sort" \ - # "gcloud compute instances delete --project=\"$PROJECT_ID\"" "zone" - - # process_filestore - # # --- Phase 2: Images & Artifacts --- - # process_vm_images - # process_docker_images + # --- Phase 2: Images & Artifacts --- + process_vm_images + process_docker_images # --- Phase 3: Network Infrastructure --- - # process_resources "Cloud Router" \ - # "gcloud compute routers list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,region)\" | sort" \ - # "gcloud compute routers delete --project=\"$PROJECT_ID\"" "region" + process_resources "Cloud Router" \ + "gcloud compute routers list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,region,labels)\" | sort" \ + "gcloud compute routers delete --project=\"$PROJECT_ID\"" "region" + process_firewalls + process_addresses + process_vpc_peerings + process_resources "Zonal Disk" \ + "gcloud compute disks list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND zone:*\" --format=\"value(name,zone,labels)\" | sort" \ + "gcloud compute disks delete --project=\"$PROJECT_ID\"" "zone" + + # --- Phase 4: Networking Hierarchies --- + process_subnetworks + process_networks + + # --- Phase 5: IAM Cleanup --- + process_iam_deleted_members - # process_firewalls - - # process_addresses - # process_vpc_peerings - - # process_resources "Zonal Disk" \ - # "gcloud compute disks list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND zone:*\" --format=\"value(name,zone)\" | sort" \ - # "gcloud compute disks delete --project=\"$PROJECT_ID\"" "zone" - - # # --- Phase 4: Networking Hierarchies --- - # process_subnetworks - # process_networks - - # # --- Phase 5: IAM Cleanup --- - # process_iam_deleted_members - log "INFO" "CLEANUP RUN FINISHED" + + if [[ $ERROR_COUNT -gt 0 ]]; then + log "WARNING" "Finished with $ERROR_COUNT errors during execution." + exit 1 + else + log "SUCCESS" "Finished with 0 errors." + exit 0 + fi } main \ No newline at end of file diff --git a/tools/cloud-build/project-cleanup.yaml b/tools/cloud-build/project-cleanup.yaml index 17260a282a..012da654fe 100644 --- a/tools/cloud-build/project-cleanup.yaml +++ b/tools/cloud-build/project-cleanup.yaml @@ -1,203 +1,60 @@ -# cloudbuild.yaml -substitutions: - _DRY_RUN: "true" # Set to "false" to enable actual deletion - _PROJECT_ID: "hpc-toolkit-dev" - _EXCLUSION_BUCKET: "hpc-ctk1357" # CHANGE: Your bucket name - _EXCLUSION_FILE: "cleanup/exclusions.txt" # CHANGE: Path to the exclusion file in the bucket +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- steps: - name: gcr.io/cloud-builders/gcloud entrypoint: /bin/bash + env: + - "BUILD_ID=${BUILD_ID}" + - "PROJECT_ID=${PROJECT_ID}" + - "DRY_RUN=true" + - "EXCLUSION_FILE=tools/exclusions.txt" args: - - '-c' + - -c - | - set -e # Exit immediately if a command exits with a non-zero status. - set -u # Treat unset variables as an error. - set -o pipefail # Return value of a pipeline is the status of the last command to exit with a non-zero status. - - PROJECT_ID="${_PROJECT_ID}" - DRY_RUN="${_DRY_RUN}" - EXCLUSION_GCS_PATH="gs://${_EXCLUSION_BUCKET}/${_EXCLUSION_FILE}" - - echo "--- STARTING RESOURCE CLEANUP in project $$PROJECT_ID ---" - echo "DRY_RUN mode: $$DRY_RUN" - echo "Fetching exclusion list from: $$EXCLUSION_GCS_PATH" - - # --- Safety Mechanism: Load Exclusion List from GCS --- - EXCLUDE_LIST=() - while IFS= read -r line || [[ -n "$$line" ]]; do - # Trim whitespace and ignore empty lines or comments - trimmed_line=$(echo "$$line" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | grep -v '^#' | grep -v '^$$') - if [[ -n "$$trimmed_line" ]]; then - EXCLUDE_LIST+=("$$trimmed_line") - fi - done < <(gcloud storage cat "$$EXCLUSION_GCS_PATH") - - if [[ $${#EXCLUDE_LIST[@]} -eq 0 ]]; then - echo "WARNING: Exclusion list from $$EXCLUSION_GCS_PATH is empty or the file was not found." - # exit 1 # Consider exiting if the list is mandatory - fi - - echo "Exclusion list loaded with $${#EXCLUDE_LIST[@]} entries:" - printf " - %s\n" "$${EXCLUDE_LIST[@]}" - - is_excluded() { - local resource_name="$$1" - # Always exclude the default network - for excluded in "$${EXCLUDE_LIST[@]}"; do - if [[ "$$resource_name" == "$$excluded" ]]; then - return 0 # True - is excluded - fi - done - return 1 # False - not excluded - } - - run_command() { - if [[ "$$DRY_RUN" == "true" ]]; then - echo "[DRY RUN] Would run: gcloud $$@" - else - echo "[EXECUTE] No running" - fi - } - - echo "--- Deletion Phase 1: GKE Clusters ---" - gcloud container clusters list --project="$$PROJECT_ID" --format="value(name,zone)" | while read -r NAME LOCATION; do - if ! is_excluded "$$NAME"; then - run_command container clusters delete "$$NAME" --project="$$PROJECT_ID" --zone="$$LOCATION" --quiet - else - echo "Skipping GKE Cluster: $$NAME (In exclusion list)" - fi - done - - echo "--- Deletion Phase 1: Compute Instances ---" - gcloud compute instances list --project="$$PROJECT_ID" --format="value(name,zone)" | while read -r NAME ZONE; do - if ! is_excluded "$$NAME"; then - run_command compute instances delete "$$NAME" --project="$$PROJECT_ID" --zone="$$ZONE" --quiet - else - echo "Skipping Instance: $$NAME (In exclusion list)" - fi - done - - echo "--- Deletion Phase 1: Filestore Instances ---" - gcloud filestore instances list --project="$$PROJECT_ID" --format="value(name,location)" | while read -r NAME LOCATION; do - if ! is_excluded "$$NAME"; then - run_command filestore instances delete "$$NAME" --project="$$PROJECT_ID" --location="$$LOCATION" --quiet - else - echo "Skipping Filestore: $$NAME (In exclusion list)" - fi - done - - REGIONS=$(gcloud compute regions list --project="$$PROJECT_ID" --format="value(name)") - - echo "--- Deletion Phase 2: Routers ---" - for REGION in $$REGIONS; do - gcloud compute routers list --project="$$PROJECT_ID" --filter="region:($$REGION)" --format="value(name)" | while read -r NAME; do - if ! is_excluded "$$NAME"; then - run_command compute routers delete "$$NAME" --project="$$PROJECT_ID" --region="$$REGION" --quiet - else - echo "Skipping Router: $$NAME in $$REGION (In exclusion list)" - fi - done - done - - echo "--- Deletion Phase 2: Firewall Rules ---" - gcloud compute firewall-rules list --project="$$PROJECT_ID" --format="value(name,network)" | while read -r NAME NETWORK; do - NETWORK_NAME=$(basename "$$NETWORK") - if ! is_excluded "$$NAME"; then - if [[ "$$NETWORK_NAME" == "default" ]]; then - echo "Skipping Firewall: $$NAME (On default network, not typically managed by tests)" - continue - fi - run_command compute firewall-rules delete "$$NAME" --project="$$PROJECT_ID" --quiet - else - echo "Skipping Firewall: $$NAME (In exclusion list)" - fi - done - - echo "--- Deletion Phase 3: Service Networking Connections ---" - gcloud compute networks list --project="$$PROJECT_ID" --format="value(name)" | while read -r NETWORK_NAME; do - if is_excluded "$$NETWORK_NAME"; then - echo "Skipping Service Networking deletion for excluded network: $$NETWORK_NAME" - continue - fi - - echo "Attempting to remove Service Networking connection from network: $$NETWORK_NAME" - # The --service defaults to servicenetworking.googleapis.com - # This command will not fail the script if the connection doesn't exist. - run_command services vpc-peerings delete --network="$$NETWORK_NAME" --project="$$PROJECT_ID" --quiet - done - - echo "--- Deletion Phase 4: Subnetworks ---" - for REGION in $$REGIONS; do - gcloud compute networks subnets list --project="$$PROJECT_ID" --filter="region:($$REGION)" --format="value(name,network)" | while read -r NAME NETWORK; do - NETWORK_NAME=$(basename "$$NETWORK") - if ! is_excluded "$$NAME"; then - run_command compute networks subnets delete "$$NAME" --project="$$PROJECT_ID" --region="$$REGION" --quiet - else - echo "Skipping Subnet: $$NAME in $$REGION (In exclusion list)" - fi - done - done - - echo "--- Deletion Phase 5: Networks ---" - gcloud compute networks list --project="$$PROJECT_ID" --format="value(name)" | while read -r NAME; do - if ! is_excluded "$$NAME"; then - run_command compute networks delete "$$NAME" --project="$$PROJECT_ID" --quiet - else - echo "Skipping Network: $$NAME (In exclusion list or is default)" - fi - done - - echo "--- Deletion Phase 6: Instance Templates ---" - gcloud compute instance-templates list --project="$$PROJECT_ID" --format="value(name)" | while read -r NAME; do - if ! is_excluded "$$NAME"; then - run_command compute instance-templates delete "$$NAME" --project="$$PROJECT_ID" --quiet - else - echo "Skipping Instance Template: $$NAME (In exclusion list)" - fi - done - - echo "--- Deletion Phase 7: Disks ---" - gcloud compute disks list --project="$$PROJECT_ID" --format="value(name,zone)" | while read -r NAME ZONE; do - if ! is_excluded "$$NAME"; then - run_command compute disks delete "$$NAME" --project="$$PROJECT_ID" --zone="$$ZONE" --quiet - else - echo "Skipping Disk: $$NAME (In exclusion list)" - fi - done - - echo "--- Deletion Phase 8: Addresses ---" - for REGION in $$REGIONS; do - gcloud compute addresses list --project="$$PROJECT_ID" --filter="region:($$REGION)" --format="value(name)" | while read -r NAME; do - if ! is_excluded "$$NAME"; then - run_command compute addresses delete "$$NAME" --project="$$PROJECT_ID" --region="$$REGION" --quiet + set -euo pipefail + + # Install dependencies + echo "Installing jq..." + apt-get update -y && apt-get install -y jq + + # Set time variables + export CUTOFF_TIME=$(date -d '5 hours ago' -u +%Y-%m-%dT%H:%M:%S%z) + export CUTOFF_TIME_IMAGES=$(date -d "60 days ago" -u +%Y-%m-%dT%H:%M:%S%z) + + attempt=1 + max_retries=10 + + while [ "$attempt" -le "$max_retries" ]; do + echo "--- Execution Attempt ${attempt} of ${max_retries} ---" + + if /workspace/tools/cleanup.sh; then + echo "Cleanup completed successfully." + exit 0 else - echo "Skipping Address: $$NAME in $$REGION (In exclusion list)" + echo "Cleanup script failed (returned non-zero exit code)." + + if [ "$attempt" -lt "$max_retries" ]; then + wait_time=$((2 ** (attempt - 1))) + echo "Waiting ${wait_time} seconds before retrying..." + sleep "$wait_time" + attempt=$((attempt + 1)) + else + echo "Max retries reached. Exiting with failure." + exit 1 + fi fi - done - done - gcloud compute addresses list --project="$$PROJECT_ID" --global --format="value(name)" | while read -r NAME; do - if ! is_excluded "$$NAME"; then - run_command compute addresses delete "$$NAME" --project="$$PROJECT_ID" --global --quiet - else - echo "Skipping Global Address: $$NAME (In exclusion list)" - fi - done - - echo "--- Deletion Phase 9: GCS Buckets (CAUTION) ---" - gcloud storage ls --project="$$PROJECT_ID" | while read -r BUCKET; do - BUCKET_NAME=$(echo "$$BUCKET" | sed 's|gs://||; s|/||') - if ! is_excluded "$$BUCKET_NAME"; then - if [[ "$$DRY_RUN" == "true" ]]; then - echo "[DRY RUN - BUCKET] Would run: gcloud storage rm -r $$BUCKET" - else - echo "[EXECUTE - BUCKET] Running: gcloud storage rm -r $$BUCKET" - gcloud storage rm -r "$$BUCKET" - fi - else - echo "Skipping Bucket: $$BUCKET_NAME (In exclusion list)" - fi - done - - echo "--- CLEANUP SCRIPT FINISHED ---" - + done \ No newline at end of file diff --git a/exclusions.txt b/tools/exclusions.txt similarity index 99% rename from exclusions.txt rename to tools/exclusions.txt index 110d6b442f..d7d8e208e3 100644 --- a/exclusions.txt +++ b/tools/exclusions.txt @@ -18,7 +18,6 @@ telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com vertexui-do-not-kill-boot vertexui-do-not-kill-data -default default-router-us-west1 default-router-us-west4 default-net-router From 642091a006f0fb022b1ee48ffbb166b48258f74c Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Tue, 9 Dec 2025 06:34:11 +0000 Subject: [PATCH 04/19] Removing files --- changescript.txt | 3492 --------------------- checking.txt | 3090 ------------------ disk.txt | 3015 ------------------ dockerimages.txt | 1352 -------- filestores.txt | 179 -- firewalls.txt | 2155 ------------- iam.txt | 4836 ---------------------------- images.txt | 7315 ------------------------------------------- instances.txt | 2847 ----------------- network.txt | 2708 ---------------- networks.txt | 2086 ------------ peer.txt | 566 ---- policy-bindings.txt | 2834 ----------------- rdisk.txt | 373 --- routers.txt | 1350 -------- subnetworks.txt | 3453 -------------------- template.txt | 4227 ------------------------- tools/cleanup.sh | 37 +- 18 files changed, 10 insertions(+), 45905 deletions(-) delete mode 100644 changescript.txt delete mode 100644 checking.txt delete mode 100644 disk.txt delete mode 100644 dockerimages.txt delete mode 100644 filestores.txt delete mode 100644 firewalls.txt delete mode 100644 iam.txt delete mode 100644 images.txt delete mode 100644 instances.txt delete mode 100644 network.txt delete mode 100644 networks.txt delete mode 100644 peer.txt delete mode 100644 policy-bindings.txt delete mode 100644 rdisk.txt delete mode 100644 routers.txt delete mode 100644 subnetworks.txt delete mode 100644 template.txt diff --git a/changescript.txt b/changescript.txt deleted file mode 100644 index f198636a4d..0000000000 --- a/changescript.txt +++ /dev/null @@ -1,3492 +0,0 @@ -[2025-11-28 13:44:49] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 13:44:49] [INFO] Time Cutoff (General): 2025-11-28T12:44:49+0000 -[2025-11-28 13:44:49] [INFO] Time Cutoff (Images): 2025-09-29T13:44:49+0000 -[2025-11-28 13:44:49] [INFO] Delete Limit per Type: 10 -[2025-11-28 13:44:49] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 13:44:50] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 13:44:51] [INFO] No Service Accounts found matching prefix. -[2025-11-28 13:44:51] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 13:44:53] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 13:44:53] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 13:44:54] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 13:44:54] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 13:44:54] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 13:44:54] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 13:44:54] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 13:44:54] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 13:44:54] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 13:44:57] [INFO] No Filestore found matching criteria. -[2025-11-28 13:44:57] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 13:44:58] [DRY-RUN] Would delete VM Image: a3lavtest-u22-20250825t071123z -[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3lavtest-u22-20250825t121357z -[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-ctkhar-20250610t181958z -[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-2c1a21-u22-20250828t204934z -[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-45dc0b-u22-20250819t184231z -[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-5a8205-u22-20250822t221818z -[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-96bae0-u22-20250822t232353z -[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-986401-u22-20250819t055812z -[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3m-slurm-c6360d-u22-20250819t173658z -[2025-11-28 13:44:59] [DRY-RUN] Would delete VM Image: a3mergesc-slurm-u22-20250929t131528z -[2025-11-28 13:44:59] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 13:44:59] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 13:45:11] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 13:45:13] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 13:45:13] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 13:45:13] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 13:45:13] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 13:45:13] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 13:45:13] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 13:45:13] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 13:45:15] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 13:45:15] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 13:45:15] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 13:45:16] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:45:16] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:45:17] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:45:17] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 13:45:17] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 13:45:17] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 13:45:17] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 13:45:17] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 13:45:18] [INFO] No Global Address found matching criteria. -[2025-11-28 13:45:18] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 13:45:35] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 13:45:37] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 13:45:37] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 13:45:37] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 13:45:37] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 13:45:37] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 13:45:37] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 13:45:37] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 13:45:37] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 13:45:39] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 13:45:39] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:39] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 13:45:41] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 13:45:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:45:41] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 13:45:42] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 13:45:42] [INFO] CLEANUP RUN FINISHED -[2025-11-28 13:47:53] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 13:47:53] [INFO] Time Cutoff (General): 2025-11-28T12:47:53+0000 -[2025-11-28 13:47:53] [INFO] Time Cutoff (Images): 2025-09-29T13:47:53+0000 -[2025-11-28 13:47:53] [INFO] Delete Limit per Type: 10 -[2025-11-28 13:47:53] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 13:47:54] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 13:47:55] [INFO] No Service Accounts found matching prefix. -[2025-11-28 13:47:55] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 13:47:57] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 13:47:57] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 13:47:59] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 13:47:59] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 13:47:59] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 13:47:59] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 13:47:59] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 13:47:59] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 13:47:59] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 13:48:01] [INFO] No Filestore found matching criteria. -[2025-11-28 13:48:01] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 13:48:03] [EXECUTE] Deleting VM Image: a3lavtest-u22-20250825t071123z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3lavtest-u22-20250825t071123z]. -[2025-11-28 13:48:11] [SUCCESS] Deleted a3lavtest-u22-20250825t071123z -[2025-11-28 13:48:11] [EXECUTE] Deleting VM Image: a3lavtest-u22-20250825t121357z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3lavtest-u22-20250825t121357z]. -[2025-11-28 13:48:17] [SUCCESS] Deleted a3lavtest-u22-20250825t121357z -[2025-11-28 13:48:17] [EXECUTE] Deleting VM Image: a3m-ctkhar-20250610t181958z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-ctkhar-20250610t181958z]. -[2025-11-28 13:48:25] [SUCCESS] Deleted a3m-ctkhar-20250610t181958z -[2025-11-28 13:48:25] [EXECUTE] Deleting VM Image: a3m-slurm-2c1a21-u22-20250828t204934z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-2c1a21-u22-20250828t204934z]. -[2025-11-28 13:48:32] [SUCCESS] Deleted a3m-slurm-2c1a21-u22-20250828t204934z -[2025-11-28 13:48:32] [EXECUTE] Deleting VM Image: a3m-slurm-45dc0b-u22-20250819t184231z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-45dc0b-u22-20250819t184231z]. -[2025-11-28 13:48:41] [SUCCESS] Deleted a3m-slurm-45dc0b-u22-20250819t184231z -[2025-11-28 13:48:41] [EXECUTE] Deleting VM Image: a3m-slurm-5a8205-u22-20250822t221818z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-5a8205-u22-20250822t221818z]. -[2025-11-28 13:48:48] [SUCCESS] Deleted a3m-slurm-5a8205-u22-20250822t221818z -[2025-11-28 13:48:49] [EXECUTE] Deleting VM Image: a3m-slurm-96bae0-u22-20250822t232353z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-96bae0-u22-20250822t232353z]. -[2025-11-28 13:48:56] [SUCCESS] Deleted a3m-slurm-96bae0-u22-20250822t232353z -[2025-11-28 13:48:56] [EXECUTE] Deleting VM Image: a3m-slurm-986401-u22-20250819t055812z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-986401-u22-20250819t055812z]. -[2025-11-28 13:49:03] [SUCCESS] Deleted a3m-slurm-986401-u22-20250819t055812z -[2025-11-28 13:49:03] [EXECUTE] Deleting VM Image: a3m-slurm-c6360d-u22-20250819t173658z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3m-slurm-c6360d-u22-20250819t173658z]. -[2025-11-28 13:49:11] [SUCCESS] Deleted a3m-slurm-c6360d-u22-20250819t173658z -[2025-11-28 13:49:11] [EXECUTE] Deleting VM Image: a3mergesc-slurm-u22-20250929t131528z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3mergesc-slurm-u22-20250929t131528z]. -[2025-11-28 13:49:19] [SUCCESS] Deleted a3mergesc-slurm-u22-20250929t131528z -[2025-11-28 13:49:19] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 13:49:19] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 13:49:32] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 13:49:33] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 13:49:33] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 13:49:33] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 13:49:33] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 13:49:33] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 13:49:33] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 13:49:33] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 13:49:35] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 13:49:35] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 13:49:35] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 13:49:37] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 13:49:37] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 13:49:37] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 13:49:39] [INFO] No Global Address found matching criteria. -[2025-11-28 13:49:39] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 13:49:56] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 13:49:58] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 13:49:58] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 13:49:58] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 13:49:58] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 13:49:58] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 13:49:58] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 13:49:58] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 13:49:58] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 13:50:00] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 13:50:00] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:01] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:01] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:01] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:01] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:01] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:01] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 13:50:02] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 13:50:02] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:50:02] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 13:50:04] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 13:50:04] [INFO] CLEANUP RUN FINISHED -[2025-11-28 13:51:01] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 13:51:01] [INFO] Time Cutoff (General): 2025-11-28T12:51:01+0000 -[2025-11-28 13:51:01] [INFO] Time Cutoff (Images): 2025-09-29T13:51:01+0000 -[2025-11-28 13:51:01] [INFO] Delete Limit per Type: 50 -[2025-11-28 13:51:01] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 13:51:01] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 13:51:03] [INFO] No Service Accounts found matching prefix. -[2025-11-28 13:51:03] [INFO] --- Processing: GKE Cluster (Limit: 50) --- -[2025-11-28 13:51:04] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 13:51:04] [INFO] --- Processing: Compute Instance (Limit: 50) --- -[2025-11-28 13:51:06] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 13:51:06] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 13:51:06] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 13:51:06] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 13:51:06] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 13:51:06] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 13:51:06] [INFO] --- Processing: Filestore (Limit: 50) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 13:51:08] [INFO] No Filestore found matching criteria. -[2025-11-28 13:51:08] [INFO] --- Processing: VM Images (Limit: 50) --- -[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a3u-image-u22-20250325t162635z -[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a3u-pp-u22-20250324t225357z -[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a3u-slurm-561cbc-u22-20250819t055542z -[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a3u-slurm-e26388-u22-20250822t220551z -[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-0a7d41-u22-20250819t061447z -[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-90f096-u22-20250806t160005z -[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-a69479-u22-20250731t113616z -[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-122222-u22-20250825t101805z -[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-1e376d-u22-20250829t132242z -[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-2c330f-u22-20250919t101939z -[2025-11-28 13:51:10] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-3c7a77-u22-20250903t145654z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-5535cc-u22-20250828t101953z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-570431-u22-20250822t204218z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-63819f-u22-20250905t101930z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-6dfb8d-u22-20250912t144538z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-76c357-u22-20250908t085635z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-869494-u22-20250825t042131z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-8b9201-u22-20250917t101932z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-946daf-u22-20250721t172352z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-a6d7f5-u22-20250825t142931z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-aa5bfc-u22-20250916t101953z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ac8613-u22-20250826t060423z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ad3d05-u22-20250918t101923z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-add7ca-u22-20250902t164931z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-b00d3e-u22-20250926t172512z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-bb93b4-u22-20250926t113807z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-bc10ee-u22-20250922t101931z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-c74ec8-u22-20250915t065803z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-d60a9e-u22-20250819t055629z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-e2a125-u22-20250901t101947z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-e3f6d6-u22-20250912t002144z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ec0d72-u22-20250904t101939z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-edeb3b-u22-20250912t074741z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ee0684-u22-20250814t130214z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-f0afe0-u22-20250923t110504z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: a4high-image-builder-20250214t220935z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: arpit-a3-toolkitest-u22-20250825t164840z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250904t073016z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250910t104131z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250910t144314z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: arpit-toolkit-u22-20250826t034632z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: chs-dcgmi-metric-u22-20250925t121709z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: common-slurm-image-20250725t234825z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: cx-a3u-u22-20250701t080501z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: harsh-a4-image -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t085040z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t101254z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t163709z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t214154z -[2025-11-28 13:51:11] [DRY-RUN] Would delete VM Image: pbspro0 -[2025-11-28 13:51:11] [INFO] Hit delete limit (50) for VM Images. -[2025-11-28 13:51:11] [INFO] --- Processing: Docker Images (Limit: 50) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 13:51:23] [INFO] --- Processing: Cloud Router (Limit: 50) --- -[2025-11-28 13:51:25] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 13:51:25] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 13:51:25] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 13:51:25] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 13:51:25] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 13:51:25] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 13:51:25] [INFO] --- Processing: Firewall Rules (Limit: 50) --- -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 13:51:27] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 13:51:27] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 13:51:27] [INFO] --- Processing: Regional Address (Limit: 50) --- -[2025-11-28 13:51:29] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 13:51:29] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 13:51:29] [INFO] --- Processing: Global Address (Limit: 50) --- -[2025-11-28 13:51:31] [INFO] No Global Address found matching criteria. -[2025-11-28 13:51:31] [INFO] --- Processing: Service Networking Connections (Limit: 50) --- -[2025-11-28 13:51:47] [INFO] --- Processing: Zonal Disk (Limit: 50) --- -[2025-11-28 13:51:49] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 13:51:49] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 13:51:49] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 13:51:49] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 13:51:49] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 13:51:49] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 13:51:49] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 13:51:49] [INFO] --- Processing: Subnetworks (Limit: 50) --- -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 13:51:51] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 13:51:51] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:51] [INFO] --- Processing: VPC Networks (Limit: 50) --- -[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 13:51:53] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 13:51:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:51:53] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 50) --- -[2025-11-28 13:51:54] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 13:51:54] [INFO] CLEANUP RUN FINISHED -[2025-11-28 13:58:41] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 13:58:41] [INFO] Time Cutoff (General): 2025-11-28T12:58:41+0000 -[2025-11-28 13:58:41] [INFO] Time Cutoff (Images): 2025-09-29T13:58:41+0000 -[2025-11-28 13:58:41] [INFO] Delete Limit per Type: 10 -[2025-11-28 13:58:41] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 13:58:41] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 13:58:43] [INFO] No Service Accounts found matching prefix. -[2025-11-28 13:58:43] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 13:58:44] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 13:58:44] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 13:58:46] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 13:58:46] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 13:58:46] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 13:58:46] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 13:58:46] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 13:58:46] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 13:58:46] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 13:58:48] [INFO] No Filestore found matching criteria. -[2025-11-28 13:58:48] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 13:58:50] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a3u-pp-u22-20250324t225357z -[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a3u-slurm-561cbc-u22-20250819t055542z -[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a3u-slurm-e26388-u22-20250822t220551z -[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-0a7d41-u22-20250819t061447z -[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-90f096-u22-20250806t160005z -[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-a69479-u22-20250731t113616z -[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-122222-u22-20250825t101805z -[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-1e376d-u22-20250829t132242z -[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-2c330f-u22-20250919t101939z -[2025-11-28 13:58:51] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-3c7a77-u22-20250903t145654z -[2025-11-28 13:58:51] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 13:58:51] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 13:59:03] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 13:59:05] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 13:59:05] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 13:59:05] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 13:59:05] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 13:59:05] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 13:59:05] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 13:59:05] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 13:59:06] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:59:06] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:59:06] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:59:06] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:59:06] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 13:59:07] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 13:59:07] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 13:59:07] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 13:59:08] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 13:59:08] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 13:59:08] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 13:59:10] [INFO] No Global Address found matching criteria. -[2025-11-28 13:59:10] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 13:59:27] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 13:59:28] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 13:59:28] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 13:59:28] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 13:59:28] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 13:59:28] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 13:59:28] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 13:59:28] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 13:59:28] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 13:59:30] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 13:59:30] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:31] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 13:59:32] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 13:59:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 13:59:32] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 13:59:34] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 13:59:34] [INFO] CLEANUP RUN FINISHED -./cleanup.sh: line 521: n: command not found -[2025-11-28 13:59:40] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 13:59:40] [INFO] Time Cutoff (General): 2025-11-28T12:59:40+0000 -[2025-11-28 13:59:40] [INFO] Time Cutoff (Images): 2025-09-29T13:59:40+0000 -[2025-11-28 13:59:40] [INFO] Delete Limit per Type: 10 -[2025-11-28 13:59:40] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 13:59:40] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 13:59:42] [INFO] No Service Accounts found matching prefix. -[2025-11-28 13:59:42] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 13:59:44] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 13:59:44] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 13:59:46] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 13:59:46] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 13:59:46] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 13:59:46] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 13:59:46] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 13:59:46] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 13:59:46] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 13:59:48] [INFO] No Filestore found matching criteria. -[2025-11-28 13:59:48] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 13:59:50] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 13:59:50] [EXECUTE] Deleting VM Image: a3u-pp-u22-20250324t225357z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3u-pp-u22-20250324t225357z]. -[2025-11-28 13:59:57] [SUCCESS] Deleted a3u-pp-u22-20250324t225357z -[2025-11-28 13:59:57] [EXECUTE] Deleting VM Image: a3u-slurm-561cbc-u22-20250819t055542z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3u-slurm-561cbc-u22-20250819t055542z]. -[2025-11-28 14:00:04] [SUCCESS] Deleted a3u-slurm-561cbc-u22-20250819t055542z -[2025-11-28 14:00:04] [EXECUTE] Deleting VM Image: a3u-slurm-e26388-u22-20250822t220551z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3u-slurm-e26388-u22-20250822t220551z]. -[2025-11-28 14:00:11] [SUCCESS] Deleted a3u-slurm-e26388-u22-20250822t220551z -[2025-11-28 14:00:11] [EXECUTE] Deleting VM Image: a4h-slurm-0a7d41-u22-20250819t061447z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-0a7d41-u22-20250819t061447z]. -[2025-11-28 14:00:18] [SUCCESS] Deleted a4h-slurm-0a7d41-u22-20250819t061447z -[2025-11-28 14:00:18] [EXECUTE] Deleting VM Image: a4h-slurm-90f096-u22-20250806t160005z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-90f096-u22-20250806t160005z]. -[2025-11-28 14:00:25] [SUCCESS] Deleted a4h-slurm-90f096-u22-20250806t160005z -[2025-11-28 14:00:25] [EXECUTE] Deleting VM Image: a4h-slurm-a69479-u22-20250731t113616z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-a69479-u22-20250731t113616z]. -[2025-11-28 14:00:32] [SUCCESS] Deleted a4h-slurm-a69479-u22-20250731t113616z -[2025-11-28 14:00:32] [EXECUTE] Deleting VM Image: a4h-slurm-flex-122222-u22-20250825t101805z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-122222-u22-20250825t101805z]. -[2025-11-28 14:00:39] [SUCCESS] Deleted a4h-slurm-flex-122222-u22-20250825t101805z -[2025-11-28 14:00:39] [EXECUTE] Deleting VM Image: a4h-slurm-flex-1e376d-u22-20250829t132242z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-1e376d-u22-20250829t132242z]. -[2025-11-28 14:00:46] [SUCCESS] Deleted a4h-slurm-flex-1e376d-u22-20250829t132242z -[2025-11-28 14:00:46] [EXECUTE] Deleting VM Image: a4h-slurm-flex-2c330f-u22-20250919t101939z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-2c330f-u22-20250919t101939z]. -[2025-11-28 14:00:54] [SUCCESS] Deleted a4h-slurm-flex-2c330f-u22-20250919t101939z -[2025-11-28 14:00:54] [EXECUTE] Deleting VM Image: a4h-slurm-flex-3c7a77-u22-20250903t145654z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-3c7a77-u22-20250903t145654z]. -[2025-11-28 14:01:01] [SUCCESS] Deleted a4h-slurm-flex-3c7a77-u22-20250903t145654z -[2025-11-28 14:01:01] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 14:01:01] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 14:01:14] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 14:01:15] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 14:01:15] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 14:01:15] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 14:01:15] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 14:01:15] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 14:01:15] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 14:01:15] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 14:01:17] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 14:01:17] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 14:01:17] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 14:01:19] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 14:01:19] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 14:01:19] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 14:01:21] [INFO] No Global Address found matching criteria. -[2025-11-28 14:01:21] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 14:01:37] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 14:01:39] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 14:01:39] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 14:01:39] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 14:01:39] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 14:01:39] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 14:01:39] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 14:01:39] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 14:01:39] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 14:01:41] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 14:01:41] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:41] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 14:01:43] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 14:01:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:01:43] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 14:01:44] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 14:01:44] [INFO] CLEANUP RUN FINISHED -[2025-11-28 14:02:39] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 14:02:39] [INFO] Time Cutoff (General): 2025-11-28T13:02:39+0000 -[2025-11-28 14:02:39] [INFO] Time Cutoff (Images): 2025-09-29T14:02:39+0000 -[2025-11-28 14:02:39] [INFO] Delete Limit per Type: 10 -[2025-11-28 14:02:39] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 14:02:39] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 14:02:41] [INFO] No Service Accounts found matching prefix. -[2025-11-28 14:02:41] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 14:02:43] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 14:02:43] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 14:02:44] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 14:02:44] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 14:02:44] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 14:02:44] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 14:02:44] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 14:02:44] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 14:02:44] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 14:02:47] [INFO] No Filestore found matching criteria. -[2025-11-28 14:02:47] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 14:02:49] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-5535cc-u22-20250828t101953z -[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-570431-u22-20250822t204218z -[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-63819f-u22-20250905t101930z -[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-6dfb8d-u22-20250912t144538z -[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-76c357-u22-20250908t085635z -[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-869494-u22-20250825t042131z -[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-8b9201-u22-20250917t101932z -[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-946daf-u22-20250721t172352z -[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-a6d7f5-u22-20250825t142931z -[2025-11-28 14:02:49] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-aa5bfc-u22-20250916t101953z -[2025-11-28 14:02:49] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 14:02:49] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 14:03:01] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 14:03:03] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 14:03:03] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 14:03:03] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 14:03:03] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 14:03:03] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 14:03:03] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 14:03:03] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 14:03:05] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 14:03:05] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 14:03:05] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 14:03:07] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 14:03:07] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 14:03:07] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 14:03:08] [INFO] No Global Address found matching criteria. -[2025-11-28 14:03:08] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 14:03:25] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 14:03:27] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 14:03:27] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 14:03:27] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 14:03:27] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 14:03:27] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 14:03:27] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 14:03:27] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 14:03:27] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 14:03:29] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 14:03:29] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:29] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 14:03:31] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 14:03:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 14:03:31] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 14:03:32] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 14:03:32] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:17:44] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:17:45] [INFO] Time Cutoff (General): 2025-11-28T14:17:44+0000 -[2025-11-28 15:17:45] [INFO] Time Cutoff (Images): 2025-09-29T15:17:44+0000 -[2025-11-28 15:17:45] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:17:45] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:17:45] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 15:17:47] [INFO] No Service Accounts found matching prefix. -[2025-11-28 15:17:47] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 15:17:48] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 15:17:48] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 15:17:50] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:17:50] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:17:50] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:17:50] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:17:50] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:17:50] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 15:17:50] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 15:17:52] [INFO] No Filestore found matching criteria. -[2025-11-28 15:17:52] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:17:54] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-5535cc-u22-20250828t101953z -[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-570431-u22-20250822t204218z -[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-63819f-u22-20250905t101930z -[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-6dfb8d-u22-20250912t144538z -[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-76c357-u22-20250908t085635z -[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-869494-u22-20250825t042131z -[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-8b9201-u22-20250917t101932z -[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-946daf-u22-20250721t172352z -[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-a6d7f5-u22-20250825t142931z -[2025-11-28 15:17:54] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-aa5bfc-u22-20250916t101953z -[2025-11-28 15:17:54] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:17:54] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 15:18:07] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 15:18:09] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 15:18:09] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 15:18:09] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 15:18:09] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 15:18:09] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 15:18:09] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 15:18:09] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 15:18:11] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 15:18:11] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 15:18:11] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 15:18:13] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 15:18:13] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 15:18:13] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 15:18:15] [INFO] No Global Address found matching criteria. -[2025-11-28 15:18:15] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 15:18:31] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 15:18:33] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:18:33] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:18:33] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:18:33] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:18:33] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:18:33] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 15:18:33] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 15:18:33] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 15:18:35] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 15:18:35] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:35] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 15:18:37] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 15:18:37] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:18:37] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 15:18:38] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 15:18:38] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:19:39] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:19:39] [INFO] Time Cutoff (General): 2025-11-28T14:19:39+0000 -[2025-11-28 15:19:39] [INFO] Time Cutoff (Images): 2025-09-29T15:19:39+0000 -[2025-11-28 15:19:39] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:19:39] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:19:39] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 15:19:41] [INFO] No Service Accounts found matching prefix. -[2025-11-28 15:19:41] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 15:19:42] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 15:19:42] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 15:19:44] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:19:44] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:19:44] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:19:44] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:19:44] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:19:44] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 15:19:44] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 15:19:46] [INFO] No Filestore found matching criteria. -[2025-11-28 15:19:46] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:19:48] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:19:48] [EXECUTE] Deleting VM Image: a4h-slurm-flex-5535cc-u22-20250828t101953z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-5535cc-u22-20250828t101953z]. -[2025-11-28 15:19:55] [SUCCESS] Deleted a4h-slurm-flex-5535cc-u22-20250828t101953z -[2025-11-28 15:19:55] [EXECUTE] Deleting VM Image: a4h-slurm-flex-570431-u22-20250822t204218z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-570431-u22-20250822t204218z]. -[2025-11-28 15:20:03] [SUCCESS] Deleted a4h-slurm-flex-570431-u22-20250822t204218z -[2025-11-28 15:20:03] [EXECUTE] Deleting VM Image: a4h-slurm-flex-63819f-u22-20250905t101930z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-63819f-u22-20250905t101930z]. -[2025-11-28 15:20:10] [SUCCESS] Deleted a4h-slurm-flex-63819f-u22-20250905t101930z -[2025-11-28 15:20:10] [EXECUTE] Deleting VM Image: a4h-slurm-flex-6dfb8d-u22-20250912t144538z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-6dfb8d-u22-20250912t144538z]. -[2025-11-28 15:20:18] [SUCCESS] Deleted a4h-slurm-flex-6dfb8d-u22-20250912t144538z -[2025-11-28 15:20:18] [EXECUTE] Deleting VM Image: a4h-slurm-flex-76c357-u22-20250908t085635z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-76c357-u22-20250908t085635z]. -[2025-11-28 15:20:26] [SUCCESS] Deleted a4h-slurm-flex-76c357-u22-20250908t085635z -[2025-11-28 15:20:26] [EXECUTE] Deleting VM Image: a4h-slurm-flex-869494-u22-20250825t042131z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-869494-u22-20250825t042131z]. -[2025-11-28 15:20:33] [SUCCESS] Deleted a4h-slurm-flex-869494-u22-20250825t042131z -[2025-11-28 15:20:33] [EXECUTE] Deleting VM Image: a4h-slurm-flex-8b9201-u22-20250917t101932z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-8b9201-u22-20250917t101932z]. -[2025-11-28 15:20:40] [SUCCESS] Deleted a4h-slurm-flex-8b9201-u22-20250917t101932z -[2025-11-28 15:20:40] [EXECUTE] Deleting VM Image: a4h-slurm-flex-946daf-u22-20250721t172352z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-946daf-u22-20250721t172352z]. -[2025-11-28 15:20:48] [SUCCESS] Deleted a4h-slurm-flex-946daf-u22-20250721t172352z -[2025-11-28 15:20:48] [EXECUTE] Deleting VM Image: a4h-slurm-flex-a6d7f5-u22-20250825t142931z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-a6d7f5-u22-20250825t142931z]. -[2025-11-28 15:20:55] [SUCCESS] Deleted a4h-slurm-flex-a6d7f5-u22-20250825t142931z -[2025-11-28 15:20:55] [EXECUTE] Deleting VM Image: a4h-slurm-flex-aa5bfc-u22-20250916t101953z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-aa5bfc-u22-20250916t101953z]. -[2025-11-28 15:21:03] [SUCCESS] Deleted a4h-slurm-flex-aa5bfc-u22-20250916t101953z -[2025-11-28 15:21:03] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:21:03] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 15:21:15] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 15:21:17] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 15:21:17] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 15:21:17] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 15:21:17] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 15:21:17] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 15:21:17] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 15:21:17] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 15:21:19] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 15:21:19] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 15:21:19] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 15:21:21] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 15:21:21] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 15:21:21] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 15:21:23] [INFO] No Global Address found matching criteria. -[2025-11-28 15:21:23] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 15:21:39] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 15:21:41] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:21:41] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:21:41] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:21:41] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:21:41] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:21:41] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 15:21:41] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 15:21:41] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 15:21:43] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 15:21:43] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:43] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 15:21:45] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 15:21:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:21:45] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 15:21:46] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 15:21:46] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:22:19] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:22:19] [INFO] Time Cutoff (General): 2025-11-28T14:22:19+0000 -[2025-11-28 15:22:19] [INFO] Time Cutoff (Images): 2025-09-29T15:22:19+0000 -[2025-11-28 15:22:19] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:22:19] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:22:19] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 15:22:21] [INFO] No Service Accounts found matching prefix. -[2025-11-28 15:22:21] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -WARNING: The following zones did not respond: europe-west3. List results may be incomplete. -[2025-11-28 15:22:27] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 15:22:27] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 15:22:29] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:22:29] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:22:29] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:22:29] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:22:29] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:22:29] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 15:22:29] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 15:22:31] [INFO] No Filestore found matching criteria. -[2025-11-28 15:22:31] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:22:33] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ac8613-u22-20250826t060423z -[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ad3d05-u22-20250918t101923z -[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-add7ca-u22-20250902t164931z -[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-b00d3e-u22-20250926t172512z -[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-bb93b4-u22-20250926t113807z -[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-bc10ee-u22-20250922t101931z -[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-c74ec8-u22-20250915t065803z -[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-d60a9e-u22-20250819t055629z -[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-e2a125-u22-20250901t101947z -[2025-11-28 15:22:33] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-e3f6d6-u22-20250912t002144z -[2025-11-28 15:22:33] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:22:33] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 15:22:46] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 15:22:47] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 15:22:47] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 15:22:47] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 15:22:47] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 15:22:47] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 15:22:47] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 15:22:48] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 15:22:49] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 15:22:49] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 15:22:49] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 15:22:51] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 15:22:51] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 15:22:51] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 15:22:53] [INFO] No Global Address found matching criteria. -[2025-11-28 15:22:53] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 15:23:09] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 15:23:11] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:23:11] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:23:11] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:23:11] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:23:11] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:23:11] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 15:23:11] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 15:23:11] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 15:23:13] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 15:23:13] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:13] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 15:23:15] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 15:23:15] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:23:15] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 15:23:16] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 15:23:16] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:23:34] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:23:34] [INFO] Time Cutoff (General): 2025-11-28T14:23:34+0000 -[2025-11-28 15:23:34] [INFO] Time Cutoff (Images): 2025-09-29T15:23:34+0000 -[2025-11-28 15:23:34] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:23:34] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:23:34] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 15:23:36] [INFO] No Service Accounts found matching prefix. -[2025-11-28 15:23:36] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 15:23:37] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 15:23:37] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 15:23:39] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:23:39] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:23:39] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:23:39] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:23:39] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:23:39] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 15:23:39] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 15:23:41] [INFO] No Filestore found matching criteria. -[2025-11-28 15:23:41] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:23:43] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:23:43] [EXECUTE] Deleting VM Image: a4h-slurm-flex-ac8613-u22-20250826t060423z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-ac8613-u22-20250826t060423z]. -[2025-11-28 15:23:51] [SUCCESS] Deleted a4h-slurm-flex-ac8613-u22-20250826t060423z -[2025-11-28 15:23:51] [EXECUTE] Deleting VM Image: a4h-slurm-flex-ad3d05-u22-20250918t101923z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-ad3d05-u22-20250918t101923z]. -[2025-11-28 15:23:59] [SUCCESS] Deleted a4h-slurm-flex-ad3d05-u22-20250918t101923z -[2025-11-28 15:23:59] [EXECUTE] Deleting VM Image: a4h-slurm-flex-add7ca-u22-20250902t164931z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-add7ca-u22-20250902t164931z]. -[2025-11-28 15:24:06] [SUCCESS] Deleted a4h-slurm-flex-add7ca-u22-20250902t164931z -[2025-11-28 15:24:06] [EXECUTE] Deleting VM Image: a4h-slurm-flex-b00d3e-u22-20250926t172512z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-b00d3e-u22-20250926t172512z]. -[2025-11-28 15:24:14] [SUCCESS] Deleted a4h-slurm-flex-b00d3e-u22-20250926t172512z -[2025-11-28 15:24:14] [EXECUTE] Deleting VM Image: a4h-slurm-flex-bb93b4-u22-20250926t113807z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-bb93b4-u22-20250926t113807z]. -[2025-11-28 15:24:21] [SUCCESS] Deleted a4h-slurm-flex-bb93b4-u22-20250926t113807z -[2025-11-28 15:24:21] [EXECUTE] Deleting VM Image: a4h-slurm-flex-bc10ee-u22-20250922t101931z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-bc10ee-u22-20250922t101931z]. -[2025-11-28 15:24:28] [SUCCESS] Deleted a4h-slurm-flex-bc10ee-u22-20250922t101931z -[2025-11-28 15:24:28] [EXECUTE] Deleting VM Image: a4h-slurm-flex-c74ec8-u22-20250915t065803z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-c74ec8-u22-20250915t065803z]. -[2025-11-28 15:24:35] [SUCCESS] Deleted a4h-slurm-flex-c74ec8-u22-20250915t065803z -[2025-11-28 15:24:35] [EXECUTE] Deleting VM Image: a4h-slurm-flex-d60a9e-u22-20250819t055629z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-d60a9e-u22-20250819t055629z]. -[2025-11-28 15:24:42] [SUCCESS] Deleted a4h-slurm-flex-d60a9e-u22-20250819t055629z -[2025-11-28 15:24:42] [EXECUTE] Deleting VM Image: a4h-slurm-flex-e2a125-u22-20250901t101947z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-e2a125-u22-20250901t101947z]. -[2025-11-28 15:24:50] [SUCCESS] Deleted a4h-slurm-flex-e2a125-u22-20250901t101947z -[2025-11-28 15:24:50] [EXECUTE] Deleting VM Image: a4h-slurm-flex-e3f6d6-u22-20250912t002144z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-e3f6d6-u22-20250912t002144z]. -[2025-11-28 15:24:57] [SUCCESS] Deleted a4h-slurm-flex-e3f6d6-u22-20250912t002144z -[2025-11-28 15:24:57] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:24:57] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 15:25:10] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 15:25:12] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 15:25:12] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 15:25:12] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 15:25:12] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 15:25:12] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 15:25:12] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 15:25:12] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 15:25:13] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 15:25:13] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 15:25:13] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 15:25:15] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 15:25:15] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 15:25:15] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 15:25:17] [INFO] No Global Address found matching criteria. -[2025-11-28 15:25:17] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 15:25:34] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 15:25:36] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:25:36] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:25:36] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:25:36] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:25:36] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:25:36] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 15:25:36] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 15:25:36] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 15:25:38] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 15:25:38] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:38] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 15:25:40] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 15:25:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:25:40] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 15:25:41] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 15:25:41] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:26:08] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:26:08] [INFO] Time Cutoff (General): 2025-11-28T14:26:08+0000 -[2025-11-28 15:26:08] [INFO] Time Cutoff (Images): 2025-09-29T15:26:08+0000 -[2025-11-28 15:26:08] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:26:08] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:26:09] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 15:26:10] [INFO] No Service Accounts found matching prefix. -[2025-11-28 15:26:10] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 15:26:11] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 15:26:11] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 15:26:13] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:26:13] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:26:13] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:26:13] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:26:13] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:26:13] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 15:26:13] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 15:26:15] [INFO] No Filestore found matching criteria. -[2025-11-28 15:26:15] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:26:17] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ec0d72-u22-20250904t101939z -[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-edeb3b-u22-20250912t074741z -[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-ee0684-u22-20250814t130214z -[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: a4h-slurm-flex-f0afe0-u22-20250923t110504z -[2025-11-28 15:26:18] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: arpit-a3-toolkitest-u22-20250825t164840z -[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250904t073016z -[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250910t104131z -[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: arpit-a3slurm-toolkit-u22-20250910t144314z -[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: arpit-toolkit-u22-20250826t034632z -[2025-11-28 15:26:18] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:26:18] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:26:18] [DRY-RUN] Would delete VM Image: cx-a3u-u22-20250701t080501z -[2025-11-28 15:26:18] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:26:18] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 15:26:30] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 15:26:31] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 15:26:31] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 15:26:31] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 15:26:31] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 15:26:31] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 15:26:31] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 15:26:31] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 15:26:33] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 15:26:33] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 15:26:33] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 15:26:35] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 15:26:35] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 15:26:35] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 15:26:37] [INFO] No Global Address found matching criteria. -[2025-11-28 15:26:37] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 15:26:53] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 15:26:55] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:26:55] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:26:55] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:26:55] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:26:55] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:26:55] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 15:26:55] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 15:26:55] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 15:26:57] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 15:26:57] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:57] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 15:26:58] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 15:26:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:26:58] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 15:27:00] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 15:27:00] [INFO] CLEANUP RUN FINISHED -./cleanup.sh: line 521: n: command not found -[2025-11-28 15:27:10] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:27:10] [INFO] Time Cutoff (General): 2025-11-28T14:27:10+0000 -[2025-11-28 15:27:10] [INFO] Time Cutoff (Images): 2025-09-29T15:27:10+0000 -[2025-11-28 15:27:10] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:27:10] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:27:11] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 15:27:12] [INFO] No Service Accounts found matching prefix. -[2025-11-28 15:27:12] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 15:27:14] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 15:27:14] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 15:27:16] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:27:16] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:27:16] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:27:16] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:27:16] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:27:16] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 15:27:16] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 15:27:18] [INFO] No Filestore found matching criteria. -[2025-11-28 15:27:18] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:27:20] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:27:21] [EXECUTE] Deleting VM Image: a4h-slurm-flex-ec0d72-u22-20250904t101939z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-ec0d72-u22-20250904t101939z]. -[2025-11-28 15:27:27] [SUCCESS] Deleted a4h-slurm-flex-ec0d72-u22-20250904t101939z -[2025-11-28 15:27:27] [EXECUTE] Deleting VM Image: a4h-slurm-flex-edeb3b-u22-20250912t074741z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-edeb3b-u22-20250912t074741z]. -[2025-11-28 15:27:35] [SUCCESS] Deleted a4h-slurm-flex-edeb3b-u22-20250912t074741z -[2025-11-28 15:27:35] [EXECUTE] Deleting VM Image: a4h-slurm-flex-ee0684-u22-20250814t130214z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-ee0684-u22-20250814t130214z]. -[2025-11-28 15:27:42] [SUCCESS] Deleted a4h-slurm-flex-ee0684-u22-20250814t130214z -[2025-11-28 15:27:42] [EXECUTE] Deleting VM Image: a4h-slurm-flex-f0afe0-u22-20250923t110504z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a4h-slurm-flex-f0afe0-u22-20250923t110504z]. -[2025-11-28 15:27:50] [SUCCESS] Deleted a4h-slurm-flex-f0afe0-u22-20250923t110504z -[2025-11-28 15:27:50] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:27:50] [EXECUTE] Deleting VM Image: arpit-a3-toolkitest-u22-20250825t164840z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/arpit-a3-toolkitest-u22-20250825t164840z]. -[2025-11-28 15:27:58] [SUCCESS] Deleted arpit-a3-toolkitest-u22-20250825t164840z -[2025-11-28 15:27:58] [EXECUTE] Deleting VM Image: arpit-a3slurm-toolkit-u22-20250904t073016z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/arpit-a3slurm-toolkit-u22-20250904t073016z]. -[2025-11-28 15:28:05] [SUCCESS] Deleted arpit-a3slurm-toolkit-u22-20250904t073016z -[2025-11-28 15:28:05] [EXECUTE] Deleting VM Image: arpit-a3slurm-toolkit-u22-20250910t104131z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/arpit-a3slurm-toolkit-u22-20250910t104131z]. -[2025-11-28 15:28:12] [SUCCESS] Deleted arpit-a3slurm-toolkit-u22-20250910t104131z -[2025-11-28 15:28:12] [EXECUTE] Deleting VM Image: arpit-a3slurm-toolkit-u22-20250910t144314z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/arpit-a3slurm-toolkit-u22-20250910t144314z]. -[2025-11-28 15:28:20] [SUCCESS] Deleted arpit-a3slurm-toolkit-u22-20250910t144314z -[2025-11-28 15:28:20] [EXECUTE] Deleting VM Image: arpit-toolkit-u22-20250826t034632z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/arpit-toolkit-u22-20250826t034632z]. -[2025-11-28 15:28:28] [SUCCESS] Deleted arpit-toolkit-u22-20250826t034632z -[2025-11-28 15:28:28] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:28:28] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:28:28] [EXECUTE] Deleting VM Image: cx-a3u-u22-20250701t080501z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/cx-a3u-u22-20250701t080501z]. -[2025-11-28 15:28:35] [SUCCESS] Deleted cx-a3u-u22-20250701t080501z -[2025-11-28 15:28:35] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:28:35] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 15:28:47] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 15:28:48] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 15:28:48] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 15:28:48] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 15:28:48] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 15:28:48] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 15:28:48] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 15:28:48] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 15:28:50] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 15:28:50] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 15:28:50] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 15:28:52] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 15:28:52] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 15:28:52] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 15:28:54] [INFO] No Global Address found matching criteria. -[2025-11-28 15:28:54] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 15:29:10] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 15:29:12] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:29:12] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:29:12] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:29:12] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:29:12] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:29:12] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 15:29:12] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 15:29:12] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 15:29:14] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 15:29:14] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:14] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 15:29:16] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 15:29:16] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:29:16] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 15:29:17] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 15:29:17] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:29:25] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:29:25] [INFO] Time Cutoff (General): 2025-11-28T14:29:25+0000 -[2025-11-28 15:29:25] [INFO] Time Cutoff (Images): 2025-09-29T15:29:25+0000 -[2025-11-28 15:29:25] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:29:25] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:29:26] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 15:29:27] [INFO] No Service Accounts found matching prefix. -[2025-11-28 15:29:27] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 15:29:29] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 15:29:29] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 15:29:31] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:29:31] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:29:31] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:29:31] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:29:31] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:29:31] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 15:29:31] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 15:29:34] [INFO] No Filestore found matching criteria. -[2025-11-28 15:29:34] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:29:36] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:29:36] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:29:36] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:29:36] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: harsh-a4-image -[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t085040z -[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t101254z -[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t163709z -[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t214154z -[2025-11-28 15:29:36] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: raasa-a3h-slurm-u20-20250916t084718z -[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250919t102219z -[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t002502z -[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t024123z -[2025-11-28 15:29:36] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t034057z -[2025-11-28 15:29:36] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:29:36] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 15:29:49] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 15:29:51] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 15:29:51] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 15:29:51] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 15:29:51] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 15:29:51] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 15:29:51] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 15:29:51] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 15:29:53] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 15:29:53] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 15:29:53] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 15:29:54] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 15:29:54] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 15:29:54] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 15:29:56] [INFO] No Global Address found matching criteria. -[2025-11-28 15:29:56] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 15:30:16] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 15:30:18] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:30:18] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:30:18] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:30:18] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:30:18] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:30:18] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 15:30:18] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 15:30:18] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 15:30:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 15:30:20] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 15:30:20] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:20] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 15:30:21] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 15:30:21] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:30:21] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 15:30:23] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 15:30:23] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:30:44] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:30:44] [INFO] Time Cutoff (General): 2025-11-28T14:30:44+0000 -[2025-11-28 15:30:44] [INFO] Time Cutoff (Images): 2025-09-29T15:30:44+0000 -[2025-11-28 15:30:44] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:30:45] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:30:45] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 15:30:46] [INFO] No Service Accounts found matching prefix. -[2025-11-28 15:30:46] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 15:30:48] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 15:30:48] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 15:30:50] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:30:50] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:30:50] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:30:50] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:30:50] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:30:50] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 15:30:50] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 15:30:51] [INFO] No Filestore found matching criteria. -[2025-11-28 15:30:51] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:30:53] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:30:54] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:30:54] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:30:54] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:30:54] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t085040z -[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: hpc-exr-2-u22-20250912t101254z -[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t163709z -[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: htcondor-10x-20250901t214154z -[2025-11-28 15:30:54] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: raasa-a3h-slurm-u20-20250916t084718z -[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250919t102219z -[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t002502z -[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t024123z -[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: rac-a3ul-nccl-u22-20250920t034057z -[2025-11-28 15:30:54] [DRY-RUN] Would delete VM Image: rac-a4h-nccl-u22-20250919t085137z -[2025-11-28 15:30:54] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:30:54] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 15:31:06] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 15:31:08] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 15:31:08] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 15:31:08] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 15:31:08] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 15:31:08] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 15:31:08] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 15:31:08] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 15:31:10] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 15:31:10] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 15:31:10] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 15:31:11] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 15:31:11] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 15:31:12] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 15:31:12] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 15:31:13] [INFO] No Global Address found matching criteria. -[2025-11-28 15:31:13] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 15:31:30] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 15:31:32] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:31:32] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:31:32] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:31:32] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:31:32] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:31:32] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 15:31:32] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 15:31:32] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 15:31:34] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 15:31:34] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:34] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 15:31:36] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 15:31:36] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:31:36] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 15:31:37] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 15:31:37] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:32:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:32:47] [INFO] Time Cutoff (General): 2025-11-28T14:32:47+0000 -[2025-11-28 15:32:47] [INFO] Time Cutoff (Images): 2025-09-29T15:32:47+0000 -[2025-11-28 15:32:47] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:32:47] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:32:47] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 15:32:48] [INFO] No Service Accounts found matching prefix. -[2025-11-28 15:32:48] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 15:32:50] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 15:32:50] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 15:32:52] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:32:52] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:32:52] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:32:52] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:32:52] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:32:52] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 15:32:52] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 15:32:54] [INFO] No Filestore found matching criteria. -[2025-11-28 15:32:54] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:32:56] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:32:56] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:32:56] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:32:56] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:32:56] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:32:56] [EXECUTE] Deleting VM Image: hpc-exr-2-u22-20250912t085040z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/hpc-exr-2-u22-20250912t085040z]. -[2025-11-28 15:33:04] [SUCCESS] Deleted hpc-exr-2-u22-20250912t085040z -[2025-11-28 15:33:04] [EXECUTE] Deleting VM Image: hpc-exr-2-u22-20250912t101254z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/hpc-exr-2-u22-20250912t101254z]. -[2025-11-28 15:33:12] [SUCCESS] Deleted hpc-exr-2-u22-20250912t101254z -[2025-11-28 15:33:12] [EXECUTE] Deleting VM Image: htcondor-10x-20250901t163709z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/htcondor-10x-20250901t163709z]. -[2025-11-28 15:33:19] [SUCCESS] Deleted htcondor-10x-20250901t163709z -[2025-11-28 15:33:19] [EXECUTE] Deleting VM Image: htcondor-10x-20250901t214154z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/htcondor-10x-20250901t214154z]. -[2025-11-28 15:33:27] [SUCCESS] Deleted htcondor-10x-20250901t214154z -[2025-11-28 15:33:27] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:33:27] [EXECUTE] Deleting VM Image: raasa-a3h-slurm-u20-20250916t084718z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/raasa-a3h-slurm-u20-20250916t084718z]. -[2025-11-28 15:33:35] [SUCCESS] Deleted raasa-a3h-slurm-u20-20250916t084718z -[2025-11-28 15:33:35] [EXECUTE] Deleting VM Image: rac-a3ul-nccl-u22-20250919t102219z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a3ul-nccl-u22-20250919t102219z]. -[2025-11-28 15:33:42] [SUCCESS] Deleted rac-a3ul-nccl-u22-20250919t102219z -[2025-11-28 15:33:42] [EXECUTE] Deleting VM Image: rac-a3ul-nccl-u22-20250920t002502z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a3ul-nccl-u22-20250920t002502z]. -[2025-11-28 15:33:49] [SUCCESS] Deleted rac-a3ul-nccl-u22-20250920t002502z -[2025-11-28 15:33:49] [EXECUTE] Deleting VM Image: rac-a3ul-nccl-u22-20250920t024123z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a3ul-nccl-u22-20250920t024123z]. -[2025-11-28 15:33:57] [SUCCESS] Deleted rac-a3ul-nccl-u22-20250920t024123z -[2025-11-28 15:33:57] [EXECUTE] Deleting VM Image: rac-a3ul-nccl-u22-20250920t034057z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a3ul-nccl-u22-20250920t034057z]. -[2025-11-28 15:34:05] [SUCCESS] Deleted rac-a3ul-nccl-u22-20250920t034057z -[2025-11-28 15:34:05] [EXECUTE] Deleting VM Image: rac-a4h-nccl-u22-20250919t085137z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a4h-nccl-u22-20250919t085137z]. -[2025-11-28 15:34:12] [SUCCESS] Deleted rac-a4h-nccl-u22-20250919t085137z -[2025-11-28 15:34:12] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:34:12] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 15:34:25] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 15:34:27] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 15:34:27] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 15:34:27] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 15:34:27] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 15:34:27] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 15:34:27] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 15:34:27] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 15:34:28] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 15:34:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 15:34:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 15:34:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 15:34:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 15:34:29] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 15:34:29] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 15:34:30] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 15:34:30] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 15:34:30] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 15:34:32] [INFO] No Global Address found matching criteria. -[2025-11-28 15:34:32] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 15:34:49] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 15:34:51] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:34:51] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:34:51] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:34:51] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:34:51] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:34:51] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 15:34:51] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 15:34:51] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 15:34:53] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 15:34:53] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:53] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 15:34:55] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 15:34:55] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:34:55] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 15:34:56] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 15:34:56] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:36:15] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:36:15] [INFO] Time Cutoff (General): 2025-11-28T14:36:15+0000 -[2025-11-28 15:36:15] [INFO] Time Cutoff (Images): 2025-09-29T15:36:15+0000 -[2025-11-28 15:36:15] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:36:15] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:36:15] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 15:36:17] [INFO] No Service Accounts found matching prefix. -[2025-11-28 15:36:17] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 15:36:18] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 15:36:18] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 15:36:20] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:36:20] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:36:20] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:36:20] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:36:20] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:36:20] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 15:36:20] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 15:36:23] [INFO] No Filestore found matching criteria. -[2025-11-28 15:36:23] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:36:25] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:36:25] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:36:25] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:36:25] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:36:25] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:36:25] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rac-a4h-nccl-u22-20250919t101657z -[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rac-a4h-nccl-u22-20250920t002518z -[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rac-a4h-u22-20250917t065059z -[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t015604z -[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t032350z -[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t040319z -[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t054432z -[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t064626z -[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t155746z -[2025-11-28 15:36:25] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t164626z -[2025-11-28 15:36:25] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:36:25] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 15:36:37] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 15:36:39] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 15:36:39] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 15:36:39] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 15:36:39] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 15:36:39] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 15:36:39] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 15:36:39] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 15:36:41] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 15:36:41] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 15:36:41] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 15:36:43] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 15:36:43] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 15:36:43] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 15:36:44] [INFO] No Global Address found matching criteria. -[2025-11-28 15:36:44] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 15:37:01] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 15:37:03] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:37:03] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:37:03] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:37:03] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:37:03] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:37:03] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 15:37:03] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 15:37:03] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 15:37:05] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 15:37:05] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:05] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 15:37:07] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 15:37:07] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:37:07] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 15:37:08] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 15:37:08] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:37:48] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:37:48] [INFO] Time Cutoff (General): 2025-11-28T14:37:48+0000 -[2025-11-28 15:37:48] [INFO] Time Cutoff (Images): 2025-09-29T15:37:48+0000 -[2025-11-28 15:37:48] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:37:48] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:37:48] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-28 15:37:50] [INFO] No Service Accounts found matching prefix. -[2025-11-28 15:37:50] [INFO] --- Processing: GKE Cluster (Limit: 10) --- -[2025-11-28 15:37:51] [SKIP] gke-a3-nccl-test (Protected Substring) -[2025-11-28 15:37:51] [INFO] --- Processing: Compute Instance (Limit: 10) --- -[2025-11-28 15:37:53] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:37:53] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:37:53] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:37:53] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:37:53] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:37:53] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-28 15:37:53] [INFO] --- Processing: Filestore (Limit: 10) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-28 15:37:56] [INFO] No Filestore found matching criteria. -[2025-11-28 15:37:56] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:37:58] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:37:58] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:37:58] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:37:58] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:37:58] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:37:58] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:37:58] [EXECUTE] Deleting VM Image: rac-a4h-nccl-u22-20250919t101657z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a4h-nccl-u22-20250919t101657z]. -[2025-11-28 15:38:05] [SUCCESS] Deleted rac-a4h-nccl-u22-20250919t101657z -[2025-11-28 15:38:05] [EXECUTE] Deleting VM Image: rac-a4h-nccl-u22-20250920t002518z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a4h-nccl-u22-20250920t002518z]. -[2025-11-28 15:38:12] [SUCCESS] Deleted rac-a4h-nccl-u22-20250920t002518z -[2025-11-28 15:38:12] [EXECUTE] Deleting VM Image: rac-a4h-u22-20250917t065059z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rac-a4h-u22-20250917t065059z]. -[2025-11-28 15:38:20] [SUCCESS] Deleted rac-a4h-u22-20250917t065059z -[2025-11-28 15:38:20] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t015604z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t015604z]. -[2025-11-28 15:38:28] [SUCCESS] Deleted rach-a3ul-u22-20250911t015604z -[2025-11-28 15:38:28] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t032350z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t032350z]. -[2025-11-28 15:38:36] [SUCCESS] Deleted rach-a3ul-u22-20250911t032350z -[2025-11-28 15:38:36] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t040319z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t040319z]. -[2025-11-28 15:38:43] [SUCCESS] Deleted rach-a3ul-u22-20250911t040319z -[2025-11-28 15:38:43] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t054432z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t054432z]. -[2025-11-28 15:38:51] [SUCCESS] Deleted rach-a3ul-u22-20250911t054432z -[2025-11-28 15:38:51] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t064626z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t064626z]. -[2025-11-28 15:38:58] [SUCCESS] Deleted rach-a3ul-u22-20250911t064626z -[2025-11-28 15:38:58] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t155746z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t155746z]. -[2025-11-28 15:39:06] [SUCCESS] Deleted rach-a3ul-u22-20250911t155746z -[2025-11-28 15:39:06] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t164626z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t164626z]. -[2025-11-28 15:39:13] [SUCCESS] Deleted rach-a3ul-u22-20250911t164626z -[2025-11-28 15:39:13] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:39:13] [INFO] --- Processing: Docker Images (Limit: 10) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 15:39:26] [INFO] --- Processing: Cloud Router (Limit: 10) --- -[2025-11-28 15:39:27] [SKIP] default-net-router (In Exclusion List) -[2025-11-28 15:39:27] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-28 15:39:27] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-28 15:39:27] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-28 15:39:27] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-0-router (Protected Substring) -[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-1-router (Protected Substring) -[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-2-router (Protected Substring) -[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-3-router (Protected Substring) -[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-4-router (Protected Substring) -[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-5-router (Protected Substring) -[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-6-router (Protected Substring) -[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-gpunet-7-router (Protected Substring) -[2025-11-28 15:39:27] [SKIP] gke-a3-nccl-test-net-router (Protected Substring) -[2025-11-28 15:39:27] [INFO] --- Processing: Firewall Rules (Limit: 10) --- -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-net-fw-allow-iap-ingress (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-a3-nccl-test-net-fw-allow-internal-traffic (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-67a003b2df18e79e-vms (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-c7463adc3946ab8d-vms (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-64d100f944401533-vms (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-516bc03ebfc0a517-vms (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-0d4ac17bedf9e73e-vms (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-9f0e042d6f100104-vms (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-dda042fc80cb8c79-vms (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-eeb258c14496d164-vms (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-all (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-exkubelet (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-inkubelet (Protected Substring) -[2025-11-28 15:39:29] [SKIP] gke-gke-a3-nccl-test-e0b2ade6-vms (Protected Substring) -[2025-11-28 15:39:29] [INFO] --- Processing: Compute Addresses --- -[2025-11-28 15:39:29] [INFO] --- Processing: Regional Address (Limit: 10) --- -[2025-11-28 15:39:31] [SKIP] gk3-gke-a3-nccl-test-e0b2ade6-1183714c-pe (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-0-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-1-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-2-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-3-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-4-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-5-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-6-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-gpunet-7-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-0 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-1 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-2 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-3 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-4 (Protected Substring) -[2025-11-28 15:39:31] [SKIP] gke-a3-nccl-test-net-nat-ips-us-west4-5 (Protected Substring) -[2025-11-28 15:39:31] [INFO] --- Processing: Global Address (Limit: 10) --- -[2025-11-28 15:39:33] [INFO] No Global Address found matching criteria. -[2025-11-28 15:39:33] [INFO] --- Processing: Service Networking Connections (Limit: 10) --- -[2025-11-28 15:39:49] [INFO] --- Processing: Zonal Disk (Limit: 10) --- -[2025-11-28 15:39:51] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-5cn6 (Protected Substring) -[2025-11-28 15:39:51] [SKIP] gke-gke-a3-nccl-test-a3-megagpu-8g-a3-9e5dbf6e-bv8h (Protected Substring) -[2025-11-28 15:39:51] [SKIP] gke-gke-a3-nccl-test-system-958513d3-zkwh (Protected Substring) -[2025-11-28 15:39:51] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-28 15:39:51] [SKIP] image-inspector (In Exclusion List) -[2025-11-28 15:39:51] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-28 15:39:51] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-28 15:39:51] [INFO] --- Processing: Subnetworks (Limit: 10) --- -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-0-subnet (Protected Substring) -[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-1-subnet (Protected Substring) -[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-2-subnet (Protected Substring) -[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-3-subnet (Protected Substring) -[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-4-subnet (Protected Substring) -[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-5-subnet (Protected Substring) -[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-6-subnet (Protected Substring) -[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-gpunet-7-subnet (Protected Substring) -[2025-11-28 15:39:53] [SKIP] gke-a3-nccl-test-subnet (Protected Substring) -[2025-11-28 15:39:53] [SKIP] gke-gke-a3-nccl-test-8c6e3b41-pe-subnet (Protected Substring) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:53] [INFO] --- Processing: VPC Networks (Limit: 10) --- -[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-0 (Protected Substring) -[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-1 (Protected Substring) -[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-2 (Protected Substring) -[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-3 (Protected Substring) -[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-4 (Protected Substring) -[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-5 (Protected Substring) -[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-6 (Protected Substring) -[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-gpunet-7 (Protected Substring) -[2025-11-28 15:39:55] [SKIP] gke-a3-nccl-test-net (Protected Substring) -[2025-11-28 15:39:55] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-28 15:39:55] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 10) --- -[2025-11-28 15:39:57] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-28 15:39:57] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:41:13] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:41:13] [INFO] Time Cutoff (General): 2025-11-28T14:41:13+0000 -[2025-11-28 15:41:13] [INFO] Time Cutoff (Images): 2025-09-29T15:41:13+0000 -[2025-11-28 15:41:13] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:41:13] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:41:14] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:41:16] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:41:16] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:41:16] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:41:16] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:41:16] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:41:16] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t180026z -[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t191956z -[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t210446z -[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250911t223305z -[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250912t030352z -[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250912t043440z -[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ul-u22-20250912t054729z -[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rach-a3ult-u22-20250916t052511z -[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rasa-a3h-slurm-u20-20250915t114049z -[2025-11-28 15:41:16] [DRY-RUN] Would delete VM Image: rasa-a3h-slurm-u20-20250915t131548z -[2025-11-28 15:41:16] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:41:16] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:41:31] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:41:31] [INFO] Time Cutoff (General): 2025-11-28T14:41:31+0000 -[2025-11-28 15:41:31] [INFO] Time Cutoff (Images): 2025-09-29T15:41:31+0000 -[2025-11-28 15:41:31] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:41:31] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:41:31] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:41:33] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:41:34] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:41:34] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:41:34] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:41:34] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:41:34] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:41:34] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t180026z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t180026z]. -[2025-11-28 15:41:42] [SUCCESS] Deleted rach-a3ul-u22-20250911t180026z -[2025-11-28 15:41:42] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t191956z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t191956z]. -[2025-11-28 15:41:48] [SUCCESS] Deleted rach-a3ul-u22-20250911t191956z -[2025-11-28 15:41:48] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t210446z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t210446z]. -[2025-11-28 15:41:56] [SUCCESS] Deleted rach-a3ul-u22-20250911t210446z -[2025-11-28 15:41:56] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250911t223305z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250911t223305z]. -[2025-11-28 15:42:03] [SUCCESS] Deleted rach-a3ul-u22-20250911t223305z -[2025-11-28 15:42:03] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250912t030352z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250912t030352z]. -[2025-11-28 15:42:10] [SUCCESS] Deleted rach-a3ul-u22-20250912t030352z -[2025-11-28 15:42:10] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250912t043440z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250912t043440z]. -[2025-11-28 15:42:17] [SUCCESS] Deleted rach-a3ul-u22-20250912t043440z -[2025-11-28 15:42:17] [EXECUTE] Deleting VM Image: rach-a3ul-u22-20250912t054729z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ul-u22-20250912t054729z]. -[2025-11-28 15:42:25] [SUCCESS] Deleted rach-a3ul-u22-20250912t054729z -[2025-11-28 15:42:25] [EXECUTE] Deleting VM Image: rach-a3ult-u22-20250916t052511z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rach-a3ult-u22-20250916t052511z]. -[2025-11-28 15:42:32] [SUCCESS] Deleted rach-a3ult-u22-20250916t052511z -[2025-11-28 15:42:32] [EXECUTE] Deleting VM Image: rasa-a3h-slurm-u20-20250915t114049z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rasa-a3h-slurm-u20-20250915t114049z]. -[2025-11-28 15:42:40] [SUCCESS] Deleted rasa-a3h-slurm-u20-20250915t114049z -[2025-11-28 15:42:40] [EXECUTE] Deleting VM Image: rasa-a3h-slurm-u20-20250915t131548z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/rasa-a3h-slurm-u20-20250915t131548z]. -[2025-11-28 15:42:47] [SUCCESS] Deleted rasa-a3h-slurm-u20-20250915t131548z -[2025-11-28 15:42:47] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:42:47] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:43:01] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:43:01] [INFO] Time Cutoff (General): 2025-11-28T14:43:01+0000 -[2025-11-28 15:43:01] [INFO] Time Cutoff (Images): 2025-09-29T15:43:01+0000 -[2025-11-28 15:43:01] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:43:01] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:43:02] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:43:03] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:43:04] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:43:04] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:43:04] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:43:04] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:43:04] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: rocka4h-rocky9-20250908t175724z -[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: rocka4hf-rocky9-20250910t040750z -[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sa-chs-ops-u22-20250925t092448z -[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: salsa-a3h-slurm-u22-20250919t083351z -[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sar-a3h-slurm-u20-20250912t102555z -[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t094000z -[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t104421z -[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: saral-saara-a3h-slurm-u22-20250926t054333z -[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sarasa-a3h-slurm-u22-20250919t125328z -[2025-11-28 15:43:04] [DRY-RUN] Would delete VM Image: sasa-a3h-slurm-u22-20250919t064159z -[2025-11-28 15:43:04] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:43:04] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:48:50] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:48:50] [INFO] Time Cutoff (General): 2025-11-28T14:48:50+0000 -[2025-11-28 15:48:50] [INFO] Time Cutoff (Images): 2025-09-29T15:48:50+0000 -[2025-11-28 15:48:50] [INFO] Delete Limit per Type: 10 -[2025-11-28 15:48:50] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:48:51] [INFO] --- Processing: VM Images (Limit: 10) --- -[2025-11-28 15:48:53] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:48:53] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:48:53] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:48:53] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:48:53] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:48:53] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:48:53] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-28 15:48:53] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sa-chs-ops-u22-20250925t092448z -[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: salsa-a3h-slurm-u22-20250919t083351z -[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sar-a3h-slurm-u20-20250912t102555z -[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t094000z -[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t104421z -[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: saral-saara-a3h-slurm-u22-20250926t054333z -[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sarasa-a3h-slurm-u22-20250919t125328z -[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sasa-a3h-slurm-u22-20250919t064159z -[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: sl-saara-a3hm-slurm-u22-20250925t103201z -[2025-11-28 15:48:53] [DRY-RUN] Would delete VM Image: slara-saara-a3h-slurm-u22-20250926t035224z -[2025-11-28 15:48:53] [INFO] Hit delete limit (10) for VM Images. -[2025-11-28 15:48:53] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:49:16] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:49:16] [INFO] Time Cutoff (General): 2025-11-28T14:49:16+0000 -[2025-11-28 15:49:16] [INFO] Time Cutoff (Images): 2025-09-29T15:49:16+0000 -[2025-11-28 15:49:16] [INFO] Delete Limit per Type: 20 -[2025-11-28 15:49:16] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:49:16] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-28 15:49:18] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:49:18] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:49:19] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:49:19] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:49:19] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:49:19] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:49:19] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-28 15:49:19] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sa-chs-ops-u22-20250925t092448z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: salsa-a3h-slurm-u22-20250919t083351z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sar-a3h-slurm-u20-20250912t102555z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t094000z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sara-a3h-slurm-u20-20250915t104421z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: saral-saara-a3h-slurm-u22-20250926t054333z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sarasa-a3h-slurm-u22-20250919t125328z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sasa-a3h-slurm-u22-20250919t064159z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: sl-saara-a3hm-slurm-u22-20250925t103201z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slara-saara-a3h-slurm-u22-20250926t035224z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slm-saara-a3h-slurm-u22-20250925t114209z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slr-saara-a3h-slurm-u22-20250925t151755z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slr-saara-a3h-slurm-u22-20250925t160703z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slra-saara-a3h-slurm-u22-20250925t163630z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slsa-a3h-slurm-u20-20250918t115814z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slsa-a3h-slurm-u20-20250918t124720z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250825t103536z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250825t113315z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250825t120950z -[2025-11-28 15:49:19] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250825t135812z -[2025-11-28 15:49:19] [INFO] Hit delete limit (20) for VM Images. -[2025-11-28 15:49:19] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:50:02] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:50:02] [INFO] Time Cutoff (General): 2025-11-28T14:50:02+0000 -[2025-11-28 15:50:02] [INFO] Time Cutoff (Images): 2025-09-29T15:50:02+0000 -[2025-11-28 15:50:02] [INFO] Delete Limit per Type: 20 -[2025-11-28 15:50:02] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:50:03] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-28 15:50:05] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:50:05] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:50:05] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:50:05] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:50:05] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:50:05] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:50:05] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-28 15:50:05] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-28 15:50:05] [EXECUTE] Deleting VM Image: sa-chs-ops-u22-20250925t092448z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sa-chs-ops-u22-20250925t092448z]. -[2025-11-28 15:50:13] [SUCCESS] Deleted sa-chs-ops-u22-20250925t092448z -[2025-11-28 15:50:13] [EXECUTE] Deleting VM Image: salsa-a3h-slurm-u22-20250919t083351z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/salsa-a3h-slurm-u22-20250919t083351z]. -[2025-11-28 15:50:20] [SUCCESS] Deleted salsa-a3h-slurm-u22-20250919t083351z -[2025-11-28 15:50:20] [EXECUTE] Deleting VM Image: sar-a3h-slurm-u20-20250912t102555z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sar-a3h-slurm-u20-20250912t102555z]. -[2025-11-28 15:50:27] [SUCCESS] Deleted sar-a3h-slurm-u20-20250912t102555z -[2025-11-28 15:50:27] [EXECUTE] Deleting VM Image: sara-a3h-slurm-u20-20250915t094000z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sara-a3h-slurm-u20-20250915t094000z]. -[2025-11-28 15:50:35] [SUCCESS] Deleted sara-a3h-slurm-u20-20250915t094000z -[2025-11-28 15:50:35] [EXECUTE] Deleting VM Image: sara-a3h-slurm-u20-20250915t104421z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sara-a3h-slurm-u20-20250915t104421z]. -[2025-11-28 15:50:42] [SUCCESS] Deleted sara-a3h-slurm-u20-20250915t104421z -[2025-11-28 15:50:42] [EXECUTE] Deleting VM Image: saral-saara-a3h-slurm-u22-20250926t054333z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/saral-saara-a3h-slurm-u22-20250926t054333z]. -[2025-11-28 15:50:50] [SUCCESS] Deleted saral-saara-a3h-slurm-u22-20250926t054333z -[2025-11-28 15:50:50] [EXECUTE] Deleting VM Image: sarasa-a3h-slurm-u22-20250919t125328z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sarasa-a3h-slurm-u22-20250919t125328z]. -[2025-11-28 15:50:57] [SUCCESS] Deleted sarasa-a3h-slurm-u22-20250919t125328z -[2025-11-28 15:50:57] [EXECUTE] Deleting VM Image: sasa-a3h-slurm-u22-20250919t064159z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sasa-a3h-slurm-u22-20250919t064159z]. -[2025-11-28 15:51:05] [SUCCESS] Deleted sasa-a3h-slurm-u22-20250919t064159z -[2025-11-28 15:51:05] [EXECUTE] Deleting VM Image: sl-saara-a3hm-slurm-u22-20250925t103201z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/sl-saara-a3hm-slurm-u22-20250925t103201z]. -[2025-11-28 15:51:12] [SUCCESS] Deleted sl-saara-a3hm-slurm-u22-20250925t103201z -[2025-11-28 15:51:12] [EXECUTE] Deleting VM Image: slara-saara-a3h-slurm-u22-20250926t035224z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slara-saara-a3h-slurm-u22-20250926t035224z]. -[2025-11-28 15:51:20] [SUCCESS] Deleted slara-saara-a3h-slurm-u22-20250926t035224z -[2025-11-28 15:51:20] [EXECUTE] Deleting VM Image: slm-saara-a3h-slurm-u22-20250925t114209z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slm-saara-a3h-slurm-u22-20250925t114209z]. -[2025-11-28 15:51:28] [SUCCESS] Deleted slm-saara-a3h-slurm-u22-20250925t114209z -[2025-11-28 15:51:28] [EXECUTE] Deleting VM Image: slr-saara-a3h-slurm-u22-20250925t151755z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slr-saara-a3h-slurm-u22-20250925t151755z]. -[2025-11-28 15:51:35] [SUCCESS] Deleted slr-saara-a3h-slurm-u22-20250925t151755z -[2025-11-28 15:51:35] [EXECUTE] Deleting VM Image: slr-saara-a3h-slurm-u22-20250925t160703z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slr-saara-a3h-slurm-u22-20250925t160703z]. -[2025-11-28 15:51:43] [SUCCESS] Deleted slr-saara-a3h-slurm-u22-20250925t160703z -[2025-11-28 15:51:43] [EXECUTE] Deleting VM Image: slra-saara-a3h-slurm-u22-20250925t163630z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slra-saara-a3h-slurm-u22-20250925t163630z]. -[2025-11-28 15:51:51] [SUCCESS] Deleted slra-saara-a3h-slurm-u22-20250925t163630z -[2025-11-28 15:51:51] [EXECUTE] Deleting VM Image: slsa-a3h-slurm-u20-20250918t115814z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slsa-a3h-slurm-u20-20250918t115814z]. -[2025-11-28 15:51:59] [SUCCESS] Deleted slsa-a3h-slurm-u20-20250918t115814z -[2025-11-28 15:51:59] [EXECUTE] Deleting VM Image: slsa-a3h-slurm-u20-20250918t124720z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slsa-a3h-slurm-u20-20250918t124720z]. -[2025-11-28 15:52:06] [SUCCESS] Deleted slsa-a3h-slurm-u20-20250918t124720z -[2025-11-28 15:52:06] [EXECUTE] Deleting VM Image: slurm-a3mega-20250825t103536z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250825t103536z]. -[2025-11-28 15:52:13] [SUCCESS] Deleted slurm-a3mega-20250825t103536z -[2025-11-28 15:52:13] [EXECUTE] Deleting VM Image: slurm-a3mega-20250825t113315z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250825t113315z]. -[2025-11-28 15:52:21] [SUCCESS] Deleted slurm-a3mega-20250825t113315z -[2025-11-28 15:52:21] [EXECUTE] Deleting VM Image: slurm-a3mega-20250825t120950z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250825t120950z]. -[2025-11-28 15:52:28] [SUCCESS] Deleted slurm-a3mega-20250825t120950z -[2025-11-28 15:52:28] [EXECUTE] Deleting VM Image: slurm-a3mega-20250825t135812z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250825t135812z]. -[2025-11-28 15:52:36] [SUCCESS] Deleted slurm-a3mega-20250825t135812z -[2025-11-28 15:52:36] [INFO] Hit delete limit (20) for VM Images. -[2025-11-28 15:52:36] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:53:05] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:53:05] [INFO] Time Cutoff (General): 2025-11-28T14:53:05+0000 -[2025-11-28 15:53:05] [INFO] Time Cutoff (Images): 2025-09-29T15:53:05+0000 -[2025-11-28 15:53:05] [INFO] Delete Limit per Type: 20 -[2025-11-28 15:53:05] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:53:05] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-28 15:53:07] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:53:07] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:53:07] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:53:07] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:53:07] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:53:07] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:53:07] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-28 15:53:07] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250828t190928z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250902t123613z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250902t150129z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250903t032910z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250917t065710z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250912t071120z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250916t093232z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250917t061957z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-gcp-next-hpc-rocky-linux-8-1739990978 -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-gcp-next-hpc-rocky-linux-8-1740100297 -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t145301z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t153428z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t200206z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t210655z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250901t115638z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250902t024825z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250903t141250z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250904t070453z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t022348z -[2025-11-28 15:53:07] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t064250z -[2025-11-28 15:53:07] [INFO] Hit delete limit (20) for VM Images. -[2025-11-28 15:53:07] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:53:56] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:53:56] [INFO] Time Cutoff (General): 2025-11-28T14:53:56+0000 -[2025-11-28 15:53:56] [INFO] Time Cutoff (Images): 2025-09-29T15:53:56+0000 -[2025-11-28 15:53:56] [INFO] Delete Limit per Type: 20 -[2025-11-28 15:53:56] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:53:57] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-28 15:53:59] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:53:59] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:53:59] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:53:59] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:53:59] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:53:59] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:53:59] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-28 15:53:59] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-28 15:53:59] [EXECUTE] Deleting VM Image: slurm-a3mega-20250828t190928z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250828t190928z]. -[2025-11-28 15:54:06] [SUCCESS] Deleted slurm-a3mega-20250828t190928z -[2025-11-28 15:54:06] [EXECUTE] Deleting VM Image: slurm-a3mega-20250902t123613z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250902t123613z]. -[2025-11-28 15:54:14] [SUCCESS] Deleted slurm-a3mega-20250902t123613z -[2025-11-28 15:54:14] [EXECUTE] Deleting VM Image: slurm-a3mega-20250902t150129z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250902t150129z]. -[2025-11-28 15:54:21] [SUCCESS] Deleted slurm-a3mega-20250902t150129z -[2025-11-28 15:54:21] [EXECUTE] Deleting VM Image: slurm-a3mega-20250903t032910z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250903t032910z]. -[2025-11-28 15:54:27] [SUCCESS] Deleted slurm-a3mega-20250903t032910z -[2025-11-28 15:54:27] [EXECUTE] Deleting VM Image: slurm-a3mega-20250917t065710z - - -Command killed by keyboard interrupt - -[2025-11-28 15:55:20] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:55:20] [INFO] Time Cutoff (General): 2025-11-28T14:55:19+0000 -[2025-11-28 15:55:20] [INFO] Time Cutoff (Images): 2025-09-29T15:55:19+0000 -[2025-11-28 15:55:20] [INFO] Delete Limit per Type: 20 -[2025-11-28 15:55:20] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:55:20] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-28 15:55:22] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:55:22] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:55:22] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:55:22] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:55:22] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:55:22] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:55:22] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-28 15:55:22] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-28 15:55:22] [DRY-RUN] Would delete VM Image: slurm-a3mega-20250917t065710z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250912t071120z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250916t093232z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-dlvm-20250917t061957z -[2025-11-28 15:55:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-28 15:55:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t145301z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t153428z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t200206z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t210655z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250901t115638z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250902t024825z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250903t141250z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250904t070453z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t022348z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t064250z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250921t061944z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: test-a4-nccl-01-u22-20250409t085350z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: test-a4-nccl-u22-20250409t065232z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: test-a4-nccl-u22-20250409t074128z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: try-a3hh-slurm-u22-20250925t075018z -[2025-11-28 15:55:23] [DRY-RUN] Would delete VM Image: try-a3hhi-slurm-u22-20250925t091429z -[2025-11-28 15:55:23] [INFO] Hit delete limit (20) for VM Images. -[2025-11-28 15:55:23] [INFO] CLEANUP RUN FINISHED -[2025-11-28 15:55:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 15:55:47] [INFO] Time Cutoff (General): 2025-11-28T14:55:47+0000 -[2025-11-28 15:55:47] [INFO] Time Cutoff (Images): 2025-09-29T15:55:47+0000 -[2025-11-28 15:55:47] [INFO] Delete Limit per Type: 20 -[2025-11-28 15:55:47] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 15:55:48] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-28 15:55:49] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 15:55:50] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 15:55:50] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 15:55:50] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 15:55:50] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 15:55:50] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 15:55:50] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-28 15:55:50] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-28 15:55:50] [EXECUTE] Deleting VM Image: slurm-a3mega-20250917t065710z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-a3mega-20250917t065710z]. -[2025-11-28 15:55:57] [SUCCESS] Deleted slurm-a3mega-20250917t065710z -[2025-11-28 15:55:57] [EXECUTE] Deleting VM Image: slurm-dlvm-20250912t071120z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-dlvm-20250912t071120z]. -[2025-11-28 15:56:05] [SUCCESS] Deleted slurm-dlvm-20250912t071120z -[2025-11-28 15:56:05] [EXECUTE] Deleting VM Image: slurm-dlvm-20250916t093232z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-dlvm-20250916t093232z]. -[2025-11-28 15:56:12] [SUCCESS] Deleted slurm-dlvm-20250916t093232z -[2025-11-28 15:56:12] [EXECUTE] Deleting VM Image: slurm-dlvm-20250917t061957z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-dlvm-20250917t061957z]. -[2025-11-28 15:56:19] [SUCCESS] Deleted slurm-dlvm-20250917t061957z -[2025-11-28 15:56:19] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-28 15:56:19] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-28 15:56:19] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t145301z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250828t145301z]. -[2025-11-28 15:56:26] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250828t145301z -[2025-11-28 15:56:26] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250828t153428z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250828t153428z]. -[2025-11-28 15:56:33] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250828t153428z -[2025-11-28 15:56:33] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t200206z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250830t200206z]. -[2025-11-28 15:56:40] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250830t200206z -[2025-11-28 15:56:40] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250830t210655z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250830t210655z]. -[2025-11-28 15:56:48] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250830t210655z -[2025-11-28 15:56:48] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250901t115638z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250901t115638z]. -[2025-11-28 15:56:56] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250901t115638z -[2025-11-28 15:56:56] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250902t024825z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250902t024825z]. -[2025-11-28 15:57:03] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250902t024825z -[2025-11-28 15:57:03] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250903t141250z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250903t141250z]. -[2025-11-28 15:57:09] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250903t141250z -[2025-11-28 15:57:09] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250904t070453z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250904t070453z]. -[2025-11-28 15:57:16] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250904t070453z -[2025-11-28 15:57:16] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t022348z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250919t022348z]. -[2025-11-28 15:57:23] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250919t022348z -[2025-11-28 15:57:23] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250919t064250z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250919t064250z]. -[2025-11-28 15:57:29] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250919t064250z -[2025-11-28 15:57:29] [EXECUTE] Deleting VM Image: slurm-ubuntu2404-accelerator-arm64-64k-20250921t061944z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/slurm-ubuntu2404-accelerator-arm64-64k-20250921t061944z]. -[2025-11-28 15:57:36] [SUCCESS] Deleted slurm-ubuntu2404-accelerator-arm64-64k-20250921t061944z -[2025-11-28 15:57:36] [EXECUTE] Deleting VM Image: test-a4-nccl-01-u22-20250409t085350z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/test-a4-nccl-01-u22-20250409t085350z]. -[2025-11-28 15:57:43] [SUCCESS] Deleted test-a4-nccl-01-u22-20250409t085350z -[2025-11-28 15:57:43] [EXECUTE] Deleting VM Image: test-a4-nccl-u22-20250409t065232z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/test-a4-nccl-u22-20250409t065232z]. -[2025-11-28 15:57:50] [SUCCESS] Deleted test-a4-nccl-u22-20250409t065232z -[2025-11-28 15:57:50] [EXECUTE] Deleting VM Image: test-a4-nccl-u22-20250409t074128z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/test-a4-nccl-u22-20250409t074128z]. -[2025-11-28 15:57:58] [SUCCESS] Deleted test-a4-nccl-u22-20250409t074128z -[2025-11-28 15:57:58] [EXECUTE] Deleting VM Image: try-a3hh-slurm-u22-20250925t075018z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/try-a3hh-slurm-u22-20250925t075018z]. -[2025-11-28 15:58:05] [SUCCESS] Deleted try-a3hh-slurm-u22-20250925t075018z -[2025-11-28 15:58:05] [EXECUTE] Deleting VM Image: try-a3hhi-slurm-u22-20250925t091429z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/try-a3hhi-slurm-u22-20250925t091429z]. -[2025-11-28 15:58:12] [SUCCESS] Deleted try-a3hhi-slurm-u22-20250925t091429z -[2025-11-28 15:58:13] [INFO] Hit delete limit (20) for VM Images. -[2025-11-28 15:58:13] [INFO] CLEANUP RUN FINISHED -[2025-11-28 16:00:00] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:00:00] [INFO] Time Cutoff (General): 2025-11-28T15:00:00+0000 -[2025-11-28 16:00:00] [INFO] Time Cutoff (Images): 2025-09-29T16:00:00+0000 -[2025-11-28 16:00:00] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:00:00] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:00:01] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-28 16:00:03] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 16:00:03] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 16:00:03] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 16:00:03] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 16:00:03] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 16:00:03] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 16:00:03] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-28 16:00:03] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-28 16:00:03] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-28 16:00:03] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-28 16:00:03] [DRY-RUN] Would delete VM Image: ysaarj-a3h-slurm-u22-20250925t053055z -[2025-11-28 16:00:03] [DRY-RUN] Would delete VM Image: ysaarj-a3h-slurm-u22-20250925t062142z -[2025-11-28 16:00:04] [DRY-RUN] Would delete VM Image: ysaj-a3h-slurm-u22-20250924t145223z -[2025-11-28 16:00:04] [INFO] CLEANUP RUN FINISHED -[2025-11-28 16:00:40] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:00:40] [INFO] Time Cutoff (General): 2025-11-28T15:00:40+0000 -[2025-11-28 16:00:40] [INFO] Time Cutoff (Images): 2025-09-29T16:00:40+0000 -[2025-11-28 16:00:40] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:00:40] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:00:40] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-28 16:00:42] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-28 16:00:42] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-28 16:00:42] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-28 16:00:42] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-28 16:00:42] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-28 16:00:42] [SKIP] pbspro0 (In Exclusion List) -[2025-11-28 16:00:42] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-28 16:00:42] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-28 16:00:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-28 16:00:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-28 16:00:43] [EXECUTE] Deleting VM Image: ysaarj-a3h-slurm-u22-20250925t053055z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/ysaarj-a3h-slurm-u22-20250925t053055z]. -[2025-11-28 16:00:50] [SUCCESS] Deleted ysaarj-a3h-slurm-u22-20250925t053055z -[2025-11-28 16:00:50] [EXECUTE] Deleting VM Image: ysaarj-a3h-slurm-u22-20250925t062142z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/ysaarj-a3h-slurm-u22-20250925t062142z]. -[2025-11-28 16:00:57] [SUCCESS] Deleted ysaarj-a3h-slurm-u22-20250925t062142z -[2025-11-28 16:00:57] [EXECUTE] Deleting VM Image: ysaj-a3h-slurm-u22-20250924t145223z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/ysaj-a3h-slurm-u22-20250924t145223z]. -[2025-11-28 16:01:05] [SUCCESS] Deleted ysaj-a3h-slurm-u22-20250924t145223z -[2025-11-28 16:01:05] [INFO] CLEANUP RUN FINISHED -[2025-11-28 16:01:17] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:01:17] [INFO] Time Cutoff (General): 2025-11-28T15:01:17+0000 -[2025-11-28 16:01:17] [INFO] Time Cutoff (Images): 2025-09-29T16:01:17+0000 -[2025-11-28 16:01:17] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:01:17] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:01:18] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 16:01:28] [INFO] CLEANUP RUN FINISHED -[2025-11-28 16:02:00] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:02:00] [INFO] Time Cutoff (General): 2025-11-28T15:02:00+0000 -[2025-11-28 16:02:00] [INFO] Time Cutoff (Images): 2025-09-29T16:02:00+0000 -[2025-11-28 16:02:00] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:02:00] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:02:01] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 16:02:13] [INFO] CLEANUP RUN FINISHED diff --git a/checking.txt b/checking.txt deleted file mode 100644 index 869b83ae88..0000000000 --- a/checking.txt +++ /dev/null @@ -1,3090 +0,0 @@ -[2025-11-29 11:10:50] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 11:10:50] [INFO] Time Cutoff (General): 2025-11-29T10:10:50+0000 -[2025-11-29 11:10:50] [INFO] Time Cutoff (Images): 2025-09-30T11:10:50+0000 -[2025-11-29 11:10:50] [INFO] Delete Limit per Type: 20 -[2025-11-29 11:10:50] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 11:10:50] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 11:10:52] [INFO] No Service Accounts found matching prefix. -[2025-11-29 11:10:52] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 11:10:53] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 11:10:53] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 11:10:55] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 11:10:55] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 11:10:55] [EXECUTE] Deleting Compute Instance: simtestnet-controller us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/simtestnet-controller]. -[2025-11-29 11:11:49] [SUCCESS] Deleted simtestnet-controller -[2025-11-29 11:11:49] [EXECUTE] Deleting Compute Instance: simtestnet-slurm-login-001 us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/simtestnet-slurm-login-001]. -[2025-11-29 11:12:42] [SUCCESS] Deleted simtestnet-slurm-login-001 -[2025-11-29 11:12:42] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 11:12:42] [INFO] --- Processing: Filestore (Limit: 20) --- -[2025-11-29 11:12:44] [EXECUTE] Deleting Filestore: simtestnet-5a54389c (Global) -ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. -The [instance] resource is not properly specified. -Failed to find attribute [zone]. The attribute can be set in the following ways: -- provide the argument `instance` on the command line with a fully specified name -- provide the argument `--zone` on the command line -- provide the argument `region` on the command line -- provide the argument `location` on the command line -- set the property `filestore/zone` -- set the property `filestore/region` -- set the property `filestore/location` -[2025-11-29 11:12:46] [ERROR] Failed to delete simtestnet-5a54389c -[2025-11-29 11:12:46] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 11:12:47] [EXECUTE] Deleting VM Image: a3mergesca-slurm-u22-20250929t173859z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3mergesca-slurm-u22-20250929t173859z]. -[2025-11-29 11:12:55] [SUCCESS] Deleted a3mergesca-slurm-u22-20250929t173859z -[2025-11-29 11:12:55] [EXECUTE] Deleting VM Image: a3mergesla-slurm-u22-20250930t043239z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3mergesla-slurm-u22-20250930t043239z]. -[2025-11-29 11:13:03] [SUCCESS] Deleted a3mergesla-slurm-u22-20250930t043239z -[2025-11-29 11:13:03] [EXECUTE] Deleting VM Image: a3qclavek-u22-20250930t085450z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3qclavek-u22-20250930t085450z]. -[2025-11-29 11:13:10] [SUCCESS] Deleted a3qclavek-u22-20250930t085450z -[2025-11-29 11:13:10] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 11:13:10] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 11:13:10] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 11:13:10] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 11:13:10] [EXECUTE] Deleting VM Image: ctk-swarnabm-sep29-03-u22-20250929t161201z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/ctk-swarnabm-sep29-03-u22-20250929t161201z]. -[2025-11-29 11:13:18] [SUCCESS] Deleted ctk-swarnabm-sep29-03-u22-20250929t161201z -[2025-11-29 11:13:18] [EXECUTE] Deleting VM Image: ctk-swarnabm-sep29-04-u22-20250930t022418z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/ctk-swarnabm-sep29-04-u22-20250930t022418z]. -[2025-11-29 11:13:26] [SUCCESS] Deleted ctk-swarnabm-sep29-04-u22-20250930t022418z -[2025-11-29 11:13:26] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 11:13:26] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 11:13:26] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 11:13:26] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 11:13:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 11:13:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 11:13:27] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 11:13:28] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 11:13:28] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 11:13:28] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 11:13:28] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 11:13:28] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-29 11:13:28] [EXECUTE] Deleting Cloud Router: simtestnet-net-0-router us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/simtestnet-net-0-router]. -[2025-11-29 11:13:32] [SUCCESS] Deleted simtestnet-net-0-router -[2025-11-29 11:13:32] [EXECUTE] Deleting Cloud Router: simtestnet-net-1-router us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/simtestnet-net-1-router]. -[2025-11-29 11:13:35] [SUCCESS] Deleted simtestnet-net-1-router -[2025-11-29 11:13:35] [EXECUTE] Deleting Cloud Router: simtestnet-net-router us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/simtestnet-net-router]. -[2025-11-29 11:13:39] [SUCCESS] Deleted simtestnet-net-router -[2025-11-29 11:13:39] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-29 11:13:41] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 11:13:41] [INFO] --- Processing: Regional Address (Limit: 20) --- -[2025-11-29 11:13:42] [EXECUTE] Deleting Regional Address: simtestnet-net-0-nat-ips-us-central1-0 us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-0-nat-ips-us-central1-0]. -[2025-11-29 11:13:45] [SUCCESS] Deleted simtestnet-net-0-nat-ips-us-central1-0 -[2025-11-29 11:13:45] [EXECUTE] Deleting Regional Address: simtestnet-net-0-nat-ips-us-central1-1 us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-0-nat-ips-us-central1-1]. -[2025-11-29 11:13:47] [SUCCESS] Deleted simtestnet-net-0-nat-ips-us-central1-1 -[2025-11-29 11:13:47] [EXECUTE] Deleting Regional Address: simtestnet-net-1-nat-ips-us-central1-0 us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-1-nat-ips-us-central1-0]. -[2025-11-29 11:13:49] [SUCCESS] Deleted simtestnet-net-1-nat-ips-us-central1-0 -[2025-11-29 11:13:49] [EXECUTE] Deleting Regional Address: simtestnet-net-1-nat-ips-us-central1-1 us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-1-nat-ips-us-central1-1]. -[2025-11-29 11:13:51] [SUCCESS] Deleted simtestnet-net-1-nat-ips-us-central1-1 -[2025-11-29 11:13:51] [EXECUTE] Deleting Regional Address: simtestnet-net-nat-ips-us-central1-0 us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-nat-ips-us-central1-0]. -[2025-11-29 11:13:54] [SUCCESS] Deleted simtestnet-net-nat-ips-us-central1-0 -[2025-11-29 11:13:54] [EXECUTE] Deleting Regional Address: simtestnet-net-nat-ips-us-central1-1 us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/simtestnet-net-nat-ips-us-central1-1]. -[2025-11-29 11:13:56] [SUCCESS] Deleted simtestnet-net-nat-ips-us-central1-1 -[2025-11-29 11:13:56] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 11:13:58] [INFO] No Global Address found matching criteria. -[2025-11-29 11:13:58] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 11:14:08] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 11:14:10] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 11:14:10] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 11:14:10] [EXECUTE] Deleting Zonal Disk: simtestnet-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/simtestnet-controller-save]. -[2025-11-29 11:14:12] [SUCCESS] Deleted simtestnet-controller-save -[2025-11-29 11:14:12] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 11:14:12] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 11:14:12] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 11:14:13] [SKIP] hpc-vpc (In Exclusion List) -WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork -[2025-11-29 11:14:15] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-0]. -[2025-11-29 11:14:27] [SUCCESS] Deleted simtestnet-mrdma-sub-0 -WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork -[2025-11-29 11:14:28] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-1]. -[2025-11-29 11:14:38] [SUCCESS] Deleted simtestnet-mrdma-sub-1 -WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork -[2025-11-29 11:14:40] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-2 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-2]. -[2025-11-29 11:14:51] [SUCCESS] Deleted simtestnet-mrdma-sub-2 -WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork -[2025-11-29 11:14:52] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-3 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-3]. -[2025-11-29 11:15:03] [SUCCESS] Deleted simtestnet-mrdma-sub-3 -WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork -[2025-11-29 11:15:05] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-4]. -[2025-11-29 11:15:16] [SUCCESS] Deleted simtestnet-mrdma-sub-4 -WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork -[2025-11-29 11:15:18] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-5 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-5]. -[2025-11-29 11:15:29] [SUCCESS] Deleted simtestnet-mrdma-sub-5 -WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork -[2025-11-29 11:15:31] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-6 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-6]. -[2025-11-29 11:15:42] [SUCCESS] Deleted simtestnet-mrdma-sub-6 -WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork -[2025-11-29 11:15:44] [EXECUTE] Deleting Subnetwork: simtestnet-mrdma-sub-7 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-mrdma-sub-7]. -[2025-11-29 11:15:55] [SUCCESS] Deleted simtestnet-mrdma-sub-7 -WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork -[2025-11-29 11:15:57] [EXECUTE] Deleting Subnetwork: simtestnet-primary-subnet -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-primary-subnet]. -[2025-11-29 11:16:09] [SUCCESS] Deleted simtestnet-primary-subnet -WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork -[2025-11-29 11:16:11] [EXECUTE] Deleting Subnetwork: simtestnet-sub-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-sub-0]. -[2025-11-29 11:16:38] [SUCCESS] Deleted simtestnet-sub-0 -WARNING: The following filter keys were not present in any resource : purpose, region, subnetwork -[2025-11-29 11:16:40] [EXECUTE] Deleting Subnetwork: simtestnet-sub-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/simtestnet-sub-1]. -[2025-11-29 11:16:50] [SUCCESS] Deleted simtestnet-sub-1 -[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:51] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 11:16:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:16:54] [EXECUTE] Deleting Dep. Route: default-route-2106bb9792a0cd0a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-2106bb9792a0cd0a]. -[2025-11-29 11:17:04] [SUCCESS] Deleted default-route-2106bb9792a0cd0a -[2025-11-29 11:17:04] [EXECUTE] Deleting Dep. Route: default-route-4853b5b69eb20ef0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-4853b5b69eb20ef0]. -[2025-11-29 11:17:26] [SUCCESS] Deleted default-route-4853b5b69eb20ef0 -[2025-11-29 11:17:26] [EXECUTE] Deleting Dep. Route: default-route-c4bfdfb86628f2ec -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-c4bfdfb86628f2ec]. -[2025-11-29 11:17:37] [SUCCESS] Deleted default-route-c4bfdfb86628f2ec -[2025-11-29 11:17:37] [EXECUTE] Deleting Dep. Route: peering-route-7922d039802e0f43 - - -Command killed by keyboard interrupt - -[2025-11-29 11:18:30] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 11:18:30] [INFO] Time Cutoff (General): 2025-11-29T10:18:30+0000 -[2025-11-29 11:18:30] [INFO] Time Cutoff (Images): 2025-09-30T11:18:30+0000 -[2025-11-29 11:18:30] [INFO] Delete Limit per Type: 20 -[2025-11-29 11:18:30] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 11:18:31] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 11:18:33] [INFO] No Service Accounts found matching prefix. -[2025-11-29 11:18:33] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 11:18:34] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 11:18:34] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 11:18:36] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 11:18:36] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 11:18:36] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 11:18:36] [INFO] --- Processing: Filestore (Limit: 20) --- -[2025-11-29 11:18:38] [EXECUTE] Deleting Filestore: simtestnet-5a54389c (Global) -ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. -The [instance] resource is not properly specified. -Failed to find attribute [zone]. The attribute can be set in the following ways: -- provide the argument `instance` on the command line with a fully specified name -- provide the argument `--zone` on the command line -- provide the argument `region` on the command line -- provide the argument `location` on the command line -- set the property `filestore/zone` -- set the property `filestore/region` -- set the property `filestore/location` -[2025-11-29 11:18:39] [ERROR] Failed to delete simtestnet-5a54389c -[2025-11-29 11:18:39] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 11:18:41] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 11:18:41] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 11:18:42] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 11:18:42] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 11:18:42] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 11:18:42] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 11:18:42] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 11:18:42] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 11:18:42] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 11:18:42] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 11:18:42] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 11:18:44] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 11:18:44] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 11:18:44] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 11:18:44] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 11:18:44] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-29 11:18:44] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-29 11:18:46] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 11:18:46] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 11:18:48] [INFO] No Regional Address found matching criteria. -[2025-11-29 11:18:48] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 11:18:49] [INFO] No Global Address found matching criteria. -[2025-11-29 11:18:49] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 11:18:59] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 11:19:01] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 11:19:01] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 11:19:01] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 11:19:01] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 11:19:01] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:19:03] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 11:19:05] [SKIP] hpc-vpc (In Exclusion List) -simtestnet-net https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net -[2025-11-29 11:19:07] [EXECUTE] Deleting Dep. Route: peering-route-7922d039802e0f43 -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -[2025-11-29 11:19:08] [ERROR] Failed to delete peering-route-7922d039802e0f43 -[2025-11-29 11:19:10] [EXECUTE] Deleting Network: simtestnet-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net]. -[2025-11-29 11:19:42] [SUCCESS] Deleted simtestnet-net -simtestnet-net-0 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net-0 -[2025-11-29 11:19:43] [EXECUTE] Deleting Dep. Route: peering-route-7922d039802e0f43 -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -[2025-11-29 11:19:45] [ERROR] Failed to delete peering-route-7922d039802e0f43 -[2025-11-29 11:19:47] [EXECUTE] Deleting Network: simtestnet-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net-0]. -[2025-11-29 11:20:39] [SUCCESS] Deleted simtestnet-net-0 -simtestnet-net-1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net-1 -[2025-11-29 11:20:42] [EXECUTE] Deleting Network: simtestnet-net-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-net-1]. -[2025-11-29 11:21:21] [SUCCESS] Deleted simtestnet-net-1 -simtestnet-rdma-net https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-rdma-net -[2025-11-29 11:21:24] [EXECUTE] Deleting Network: simtestnet-rdma-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/simtestnet-rdma-net]. -[2025-11-29 11:21:53] [SUCCESS] Deleted simtestnet-rdma-net -[2025-11-29 11:21:53] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 11:21:54] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 11:21:54] [INFO] CLEANUP RUN FINISHED -[2025-11-29 11:24:15] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 11:24:15] [INFO] Time Cutoff (General): 2025-11-29T10:24:15+0000 -[2025-11-29 11:24:15] [INFO] Time Cutoff (Images): 2025-09-30T11:24:15+0000 -[2025-11-29 11:24:15] [INFO] Delete Limit per Type: 20 -[2025-11-29 11:24:15] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 11:24:15] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 11:24:17] [INFO] No Service Accounts found matching prefix. -[2025-11-29 11:24:17] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 11:24:19] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 11:24:19] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 11:24:20] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 11:24:21] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 11:24:21] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 11:24:21] [INFO] --- Processing: Filestore (Limit: 20) --- -[2025-11-29 11:24:23] [EXECUTE] Deleting Filestore: simtestnet-5a54389c (Global) -ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. -The [instance] resource is not properly specified. -Failed to find attribute [zone]. The attribute can be set in the following ways: -- provide the argument `instance` on the command line with a fully specified name -- provide the argument `--zone` on the command line -- provide the argument `region` on the command line -- provide the argument `location` on the command line -- set the property `filestore/zone` -- set the property `filestore/region` -- set the property `filestore/location` -[2025-11-29 11:24:24] [ERROR] Failed to delete simtestnet-5a54389c -[2025-11-29 11:24:24] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 11:24:26] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 11:24:26] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 11:24:27] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 11:24:27] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 11:24:27] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 11:24:27] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 11:24:27] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 11:24:27] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 11:24:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 11:24:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 11:24:27] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 11:24:29] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 11:24:29] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 11:24:29] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 11:24:29] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 11:24:29] [SKIP] default-router-us-west4 (In Exclusion List) -./cleanup.sh: line 544: process_firewalls: command not found -[2025-11-29 11:24:29] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 11:24:29] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 11:24:30] [INFO] No Regional Address found matching criteria. -[2025-11-29 11:24:30] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 11:24:32] [INFO] No Global Address found matching criteria. -[2025-11-29 11:24:32] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 11:24:36] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 11:24:37] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 11:24:37] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 11:24:37] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 11:24:37] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 11:24:37] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:40] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 11:24:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 11:24:41] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 11:24:43] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 11:24:43] [INFO] CLEANUP RUN FINISHED -[2025-11-29 11:26:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 11:26:47] [INFO] Time Cutoff (General): 2025-11-29T10:26:47+0000 -[2025-11-29 11:26:47] [INFO] Time Cutoff (Images): 2025-09-30T11:26:47+0000 -[2025-11-29 11:26:47] [INFO] Delete Limit per Type: 20 -[2025-11-29 11:26:47] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 11:26:48] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 11:26:49] [INFO] No Service Accounts found matching prefix. -[2025-11-29 11:26:49] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 11:26:51] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 11:26:51] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 11:26:53] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 11:26:53] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 11:26:53] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 11:26:53] [INFO] --- Processing: Filestore (Limit: 20) --- -[2025-11-29 11:26:55] [EXECUTE] Deleting Filestore: simtestnet-5a54389c (Global) -ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. -The [instance] resource is not properly specified. -Failed to find attribute [zone]. The attribute can be set in the following ways: -- provide the argument `instance` on the command line with a fully specified name -- provide the argument `--zone` on the command line -- provide the argument `region` on the command line -- provide the argument `location` on the command line -- set the property `filestore/zone` -- set the property `filestore/region` -- set the property `filestore/location` -[2025-11-29 11:26:56] [ERROR] Failed to delete simtestnet-5a54389c -[2025-11-29 11:26:56] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 11:26:58] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 11:26:58] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 11:26:58] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 11:26:58] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 11:26:58] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 11:26:58] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 11:26:58] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 11:26:58] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 11:26:59] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 11:26:59] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 11:26:59] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 11:27:01] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 11:27:01] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 11:27:01] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 11:27:01] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 11:27:01] [SKIP] default-router-us-west4 (In Exclusion List) -./cleanup.sh: line 545: process_firewalls: command not found -[2025-11-29 11:27:01] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 11:27:01] [INFO] --- Processing: Regional Address (Limit: 20) --- -Traceback (most recent call last): - File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 193, in - main() - File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 187, in main - gcloud_main = _import_gcloud_main() - ^^^^^^^^^^^^^^^^^^^^^ - File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 90, in _import_gcloud_main - import googlecloudsdk.gcloud_main - File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/gcloud_main.py", line 42, in - from googlecloudsdk.core.credentials import creds_context_managers - File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/core/credentials/creds_context_managers.py", line 28, in - from googlecloudsdk.api_lib.iamcredentials import util as iamcred_util - File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/api_lib/iamcredentials/util.py", line 26, in - from apitools.base.py import http_wrapper - File "/usr/bin/../lib/google-cloud-sdk/lib/third_party/apitools/base/py/http_wrapper.py", line 39, in - from oauth2client.client import HttpAccessTokenRefreshError as TokenRefreshError # noqa - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/bin/../lib/google-cloud-sdk/lib/third_party/oauth2client/client.py", line 52, in - from oauth2client import crypt - File "/usr/bin/../lib/google-cloud-sdk/lib/third_party/oauth2client/crypt.py", line 41, in - from oauth2client import _openssl_crypt - File "/usr/bin/../lib/google-cloud-sdk/lib/third_party/oauth2client/_openssl_crypt.py", line 16, in - from OpenSSL import crypto - File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/OpenSSL/__init__.py", line 8, in - from OpenSSL import SSL, crypto - File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/OpenSSL/SSL.py", line 35, in - from OpenSSL.crypto import ( - File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/OpenSSL/crypto.py", line 22, in - from cryptography import utils, x509 - File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/cryptography/x509/__init__.py", line 8, in - from cryptography.x509.base import ( - File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/cryptography/x509/base.py", line 15, in - from cryptography.hazmat.primitives import hashes, serialization - File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/cryptography/hazmat/primitives/serialization/__init__.py", line 25, in - from cryptography.hazmat.primitives.serialization.ssh import ( - File "/usr/lib/google-cloud-sdk/platform/bundledpythonunix/lib/python3.12/site-packages/cryptography/hazmat/primitives/serialization/ssh.py", line 19, in - from cryptography.hazmat.primitives.asymmetric import ( - File "", line 1360, in _find_and_load - File "", line 1331, in _find_and_load_unlocked - File "", line 935, in _load_unlocked - File "", line 995, in exec_module - File "", line 1128, in get_code - File "", line 757, in _compile_bytecode -KeyboardInterrupt -[2025-11-29 11:28:45] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 11:28:45] [INFO] Time Cutoff (General): 2025-11-29T10:28:45+0000 -[2025-11-29 11:28:45] [INFO] Time Cutoff (Images): 2025-09-30T11:28:45+0000 -[2025-11-29 11:28:45] [INFO] Delete Limit per Type: 20 -[2025-11-29 11:28:45] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 11:28:45] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 11:28:47] [INFO] No Service Accounts found matching prefix. -[2025-11-29 11:28:47] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 11:28:48] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 11:28:48] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 11:28:50] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 11:28:50] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 11:28:50] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 11:28:50] [INFO] --- Processing: Filestore (Limit: 20) --- -[2025-11-29 11:28:52] [EXECUTE] Deleting Filestore: simtestnet-5a54389c (Global) -ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. -The [instance] resource is not properly specified. -Failed to find attribute [zone]. The attribute can be set in the following ways: -- provide the argument `instance` on the command line with a fully specified name -- provide the argument `--zone` on the command line -- provide the argument `region` on the command line -- provide the argument `location` on the command line -- set the property `filestore/zone` -- set the property `filestore/region` -- set the property `filestore/location` -[2025-11-29 11:28:53] [ERROR] Failed to delete simtestnet-5a54389c -[2025-11-29 11:28:53] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 11:28:55] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 11:28:55] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 11:28:55] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 11:28:55] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 11:28:56] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 11:28:56] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 11:28:56] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 11:28:56] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 11:28:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 11:28:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 11:28:56] [INFO] --- Processing: Cloud Router (Limit: 20) --- - - -Command killed by keyboard interrupt - -[2025-11-29 19:02:55] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 19:02:55] [INFO] Time Cutoff (General): 2025-11-29T18:02:55+0000 -[2025-11-29 19:02:55] [INFO] Time Cutoff (Images): 2025-09-30T19:02:55+0000 -[2025-11-29 19:02:55] [INFO] Delete Limit per Type: 20 -[2025-11-29 19:02:55] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 19:02:55] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 19:02:58] [INFO] No Service Accounts found matching prefix. -[2025-11-29 19:02:58] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 19:03:00] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 19:03:00] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 19:03:02] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:03:02] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:03:02] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 19:03:02] [INFO] --- Processing: Filestore (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 19:03:04] [INFO] No Filestore found matching criteria. -[2025-11-29 19:03:04] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 19:03:07] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 19:03:07] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 19:03:07] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 19:03:07] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 19:03:07] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 19:03:07] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 19:03:07] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 19:03:07] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 19:03:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 19:03:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 19:03:08] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 19:03:10] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 19:03:10] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 19:03:10] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 19:03:10] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 19:03:10] [SKIP] default-router-us-west4 (In Exclusion List) -./cleanup.sh: line 570: process_firewalls: command not found -[2025-11-29 19:03:10] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 19:03:10] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:03:12] [INFO] No Regional Address found matching criteria. -[2025-11-29 19:03:12] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:03:14] [INFO] No Global Address found matching criteria. -[2025-11-29 19:03:14] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 19:03:17] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 19:03:19] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:03:19] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:03:19] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 19:03:19] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 19:03:19] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 19:03:21] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:22] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 19:03:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:03:24] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 19:03:25] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 19:03:25] [INFO] CLEANUP RUN FINISHED -[2025-11-29 19:14:16] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 19:14:16] [INFO] Time Cutoff (General): 2025-11-29T19:14:16+0000 -[2025-11-29 19:14:16] [INFO] Time Cutoff (Images): 2025-09-30T19:14:16+0000 -[2025-11-29 19:14:16] [INFO] Delete Limit per Type: 20 -[2025-11-29 19:14:16] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 19:14:16] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 19:14:18] [INFO] No Service Accounts found matching prefix. -[2025-11-29 19:14:18] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 19:14:19] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 19:14:20] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 19:14:23] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:14:23] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:14:23] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 19:14:23] [INFO] --- Processing: Filestore (Limit: 20) --- -[2025-11-29 19:14:25] [DRY-RUN] Would delete Filestore: simk (Global) -[2025-11-29 19:14:25] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 19:14:27] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 19:14:28] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 19:14:28] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 19:14:28] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 19:14:28] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 19:14:28] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 19:14:28] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 19:14:28] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 19:14:28] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 19:14:28] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 19:14:28] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 19:14:30] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 19:14:30] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 19:14:30] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 19:14:30] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 19:14:30] [SKIP] default-router-us-west4 (In Exclusion List) -./cleanup.sh: line 570: process_firewalls: command not found -[2025-11-29 19:14:30] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 19:14:30] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:14:32] [INFO] No Regional Address found matching criteria. -[2025-11-29 19:14:32] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:14:34] [INFO] No Global Address found matching criteria. -[2025-11-29 19:14:34] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 19:14:37] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 19:14:39] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:14:39] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:14:39] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 19:14:39] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 19:14:39] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 19:14:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:42] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 19:14:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:14:44] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 19:14:45] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 19:14:45] [INFO] CLEANUP RUN FINISHED -[2025-11-29 19:14:54] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 19:14:54] [INFO] Time Cutoff (General): 2025-11-29T19:14:54+0000 -[2025-11-29 19:14:54] [INFO] Time Cutoff (Images): 2025-09-30T19:14:54+0000 -[2025-11-29 19:14:54] [INFO] Delete Limit per Type: 20 -[2025-11-29 19:14:54] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 19:14:55] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 19:14:57] [INFO] No Service Accounts found matching prefix. -[2025-11-29 19:14:57] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 19:14:58] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 19:14:58] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 19:15:02] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:15:02] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:15:02] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 19:15:02] [INFO] --- Processing: Filestore (Limit: 20) --- -[2025-11-29 19:15:04] [EXECUTE] Deleting Filestore: simk (Global) -ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. -The [instance] resource is not properly specified. -Failed to find attribute [zone]. The attribute can be set in the following ways: -- provide the argument `instance` on the command line with a fully specified name -- provide the argument `--zone` on the command line -- provide the argument `region` on the command line -- provide the argument `location` on the command line -- set the property `filestore/zone` -- set the property `filestore/region` -- set the property `filestore/location` -[2025-11-29 19:15:05] [ERROR] Failed to delete simk -[2025-11-29 19:15:05] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 19:15:07] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 19:15:07] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 19:15:08] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 19:15:08] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 19:15:08] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 19:15:08] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 19:15:08] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 19:15:08] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 19:15:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 19:15:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 19:15:08] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 19:15:10] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 19:15:10] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 19:15:10] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 19:15:10] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 19:15:10] [SKIP] default-router-us-west4 (In Exclusion List) -./cleanup.sh: line 570: process_firewalls: command not found -[2025-11-29 19:15:10] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 19:15:10] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:15:12] [INFO] No Regional Address found matching criteria. -[2025-11-29 19:15:12] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:15:14] [INFO] No Global Address found matching criteria. -[2025-11-29 19:15:14] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 19:15:18] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 19:15:20] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:15:20] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:15:20] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 19:15:20] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 19:15:20] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:22] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 19:15:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:15:23] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 19:15:25] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 19:15:25] [INFO] CLEANUP RUN FINISHED -[2025-11-29 19:16:44] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 19:16:44] [INFO] Time Cutoff (General): 2025-11-29T19:16:44+0000 -[2025-11-29 19:16:44] [INFO] Time Cutoff (Images): 2025-09-30T19:16:44+0000 -[2025-11-29 19:16:44] [INFO] Delete Limit per Type: 20 -[2025-11-29 19:16:44] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 19:16:44] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 19:16:46] [INFO] No Service Accounts found matching prefix. -[2025-11-29 19:16:46] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 19:16:47] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 19:16:47] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 19:16:50] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:16:50] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:16:50] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 19:16:50] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -[2025-11-29 19:16:52] [DEBUG] Filestore JSON Output: [ - { - "createTime": "2025-11-29T19:05:02.285881388Z", - "customPerformanceSupported": true, - "fileShares": [ - { - "capacityGb": "1024", - "name": "hhh" - } - ], - "name": "projects/hpc-toolkit-dev/locations/us-central1/instances/simk", - "networks": [ - { - "connectMode": "DIRECT_PEERING", - "ipAddresses": [ - "10.203.98.194" - ], - "modes": [ - "MODE_IPV4" - ], - "network": "hpc-vpc", - "reservedIpRange": "10.203.98.192/26" - } - ], - "performanceLimits": { - "maxIops": "12000", - "maxReadIops": "12000", - "maxReadThroughputBps": "125829120", - "maxWriteIops": "4000", - "maxWriteThroughputBps": "104857600" - }, - "protocol": "NFS_V3", - "state": "READY", - "tier": "REGIONAL" - } -] -[2025-11-29 19:16:52] [DEBUG] Parsed Filestore List (Location\tName): -[2025-11-29 19:16:52] [DEBUG] us-central1 simk -[2025-11-29 19:16:52] [DEBUG] Processing Instance: Name='simk', Location='us-central1' -[2025-11-29 19:16:52] [DRY-RUN] Would delete Filestore: simk (us-central1) -[2025-11-29 19:16:52] [INFO] Finished processing Filestore instances. Attempted to delete 1. -[2025-11-29 19:16:52] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 19:16:55] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 19:16:55] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 19:16:55] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 19:16:55] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 19:16:55] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 19:16:55] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 19:16:55] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 19:16:55] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 19:16:55] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 19:16:55] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 19:16:55] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 19:16:57] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 19:16:57] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 19:16:57] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 19:16:57] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 19:16:57] [SKIP] default-router-us-west4 (In Exclusion List) -./cleanup.sh: line 567: process_firewalls: command not found -[2025-11-29 19:16:57] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 19:16:57] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:16:59] [INFO] No Regional Address found matching criteria. -[2025-11-29 19:16:59] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:17:01] [INFO] No Global Address found matching criteria. -[2025-11-29 19:17:01] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 19:17:05] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 19:17:07] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:17:07] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:17:07] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 19:17:07] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 19:17:07] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:09] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 19:17:11] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:11] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 19:17:12] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 19:17:12] [INFO] CLEANUP RUN FINISHED -[2025-11-29 19:17:20] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 19:17:20] [INFO] Time Cutoff (General): 2025-11-29T19:17:20+0000 -[2025-11-29 19:17:20] [INFO] Time Cutoff (Images): 2025-09-30T19:17:20+0000 -[2025-11-29 19:17:20] [INFO] Delete Limit per Type: 20 -[2025-11-29 19:17:20] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 19:17:21] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 19:17:22] [INFO] No Service Accounts found matching prefix. -[2025-11-29 19:17:22] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 19:17:24] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 19:17:24] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 19:17:26] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:17:26] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:17:26] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 19:17:26] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -[2025-11-29 19:17:28] [DEBUG] Filestore JSON Output: [ - { - "createTime": "2025-11-29T19:05:02.285881388Z", - "customPerformanceSupported": true, - "fileShares": [ - { - "capacityGb": "1024", - "name": "hhh" - } - ], - "name": "projects/hpc-toolkit-dev/locations/us-central1/instances/simk", - "networks": [ - { - "connectMode": "DIRECT_PEERING", - "ipAddresses": [ - "10.203.98.194" - ], - "modes": [ - "MODE_IPV4" - ], - "network": "hpc-vpc", - "reservedIpRange": "10.203.98.192/26" - } - ], - "performanceLimits": { - "maxIops": "12000", - "maxReadIops": "12000", - "maxReadThroughputBps": "125829120", - "maxWriteIops": "4000", - "maxWriteThroughputBps": "104857600" - }, - "protocol": "NFS_V3", - "state": "READY", - "tier": "REGIONAL" - } -] -[2025-11-29 19:17:28] [DEBUG] Parsed Filestore List (Location\tName): -[2025-11-29 19:17:28] [DEBUG] us-central1 simk -[2025-11-29 19:17:28] [DEBUG] Processing Instance: Name='simk', Location='us-central1' -[2025-11-29 19:17:28] [DRY-RUN] Would delete Filestore: simk (us-central1) -[2025-11-29 19:17:28] [INFO] Finished processing Filestore instances. Attempted to delete 1. -[2025-11-29 19:17:28] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 19:17:30] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 19:17:30] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 19:17:30] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 19:17:30] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 19:17:30] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 19:17:30] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 19:17:30] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 19:17:30] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 19:17:31] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 19:17:31] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 19:17:31] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 19:17:32] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 19:17:32] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 19:17:32] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 19:17:32] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 19:17:32] [SKIP] default-router-us-west4 (In Exclusion List) -./cleanup.sh: line 567: process_firewalls: command not found -[2025-11-29 19:17:32] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 19:17:32] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:17:34] [INFO] No Regional Address found matching criteria. -[2025-11-29 19:17:34] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:17:36] [INFO] No Global Address found matching criteria. -[2025-11-29 19:17:36] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 19:17:40] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 19:17:42] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:17:42] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:17:42] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 19:17:42] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 19:17:42] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:44] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 19:17:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:17:46] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 19:17:47] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 19:17:47] [INFO] CLEANUP RUN FINISHED -[2025-11-29 19:17:50] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 19:17:50] [INFO] Time Cutoff (General): 2025-11-29T19:17:50+0000 -[2025-11-29 19:17:50] [INFO] Time Cutoff (Images): 2025-09-30T19:17:50+0000 -[2025-11-29 19:17:50] [INFO] Delete Limit per Type: 20 -[2025-11-29 19:17:50] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 19:17:50] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 19:17:52] [INFO] No Service Accounts found matching prefix. -[2025-11-29 19:17:52] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 19:17:53] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 19:17:53] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 19:17:56] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:17:56] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:17:56] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 19:17:56] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -[2025-11-29 19:17:58] [DEBUG] Filestore JSON Output: [ - { - "createTime": "2025-11-29T19:05:02.285881388Z", - "customPerformanceSupported": true, - "fileShares": [ - { - "capacityGb": "1024", - "name": "hhh" - } - ], - "name": "projects/hpc-toolkit-dev/locations/us-central1/instances/simk", - "networks": [ - { - "connectMode": "DIRECT_PEERING", - "ipAddresses": [ - "10.203.98.194" - ], - "modes": [ - "MODE_IPV4" - ], - "network": "hpc-vpc", - "reservedIpRange": "10.203.98.192/26" - } - ], - "performanceLimits": { - "maxIops": "12000", - "maxReadIops": "12000", - "maxReadThroughputBps": "125829120", - "maxWriteIops": "4000", - "maxWriteThroughputBps": "104857600" - }, - "protocol": "NFS_V3", - "state": "READY", - "tier": "REGIONAL" - } -] -[2025-11-29 19:17:58] [DEBUG] Parsed Filestore List (Location\tName): -[2025-11-29 19:17:58] [DEBUG] us-central1 simk -[2025-11-29 19:17:58] [DEBUG] Processing Instance: Name='simk', Location='us-central1' -[2025-11-29 19:17:58] [EXECUTE] Deleting Filestore: simk (us-central1) -Waiting for [operation-1764443880298-644c09ab6193d-0d3292e2-0685018e] to finish... -..............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................done. -[2025-11-29 19:22:49] [SUCCESS] Deleted simk -[2025-11-29 19:22:49] [INFO] Finished processing Filestore instances. Attempted to delete 1. -[2025-11-29 19:22:49] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 19:22:51] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 19:22:51] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 19:22:52] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 19:22:52] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 19:22:52] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 19:22:52] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 19:22:52] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 19:22:52] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 19:22:52] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 19:22:52] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 19:22:52] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 19:22:54] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 19:22:54] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 19:22:54] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 19:22:54] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 19:22:54] [SKIP] default-router-us-west4 (In Exclusion List) -./cleanup.sh: line 567: process_firewalls: command not found -[2025-11-29 19:22:54] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 19:22:54] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:22:56] [INFO] No Regional Address found matching criteria. -[2025-11-29 19:22:56] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:22:58] [INFO] No Global Address found matching criteria. -[2025-11-29 19:22:58] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 19:23:01] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 19:23:03] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:23:03] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:23:03] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 19:23:03] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 19:23:03] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:06] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 19:23:07] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:23:07] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 19:23:09] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 19:23:09] [INFO] CLEANUP RUN FINISHED -[2025-11-29 19:26:33] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 19:26:33] [INFO] Time Cutoff (General): 2025-11-29T19:26:33+0000 -[2025-11-29 19:26:33] [INFO] Time Cutoff (Images): 2025-09-30T19:26:33+0000 -[2025-11-29 19:26:33] [INFO] Delete Limit per Type: 20 -[2025-11-29 19:26:33] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 19:26:33] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 19:26:35] [INFO] No Service Accounts found matching prefix. -[2025-11-29 19:26:35] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 19:26:37] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 19:26:37] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 19:26:38] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:26:38] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:26:38] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 19:26:38] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 19:26:41] [INFO] No Filestore instances found matching criteria. -[2025-11-29 19:26:41] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 19:26:43] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 19:26:43] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 19:26:43] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 19:26:43] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 19:26:44] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 19:26:44] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 19:26:44] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 19:26:44] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 19:26:44] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 19:26:44] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 19:26:44] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 19:26:46] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 19:26:46] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 19:26:46] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 19:26:46] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 19:26:46] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-29 19:26:46] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-29 19:26:48] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 19:26:48] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:26:49] [INFO] No Regional Address found matching criteria. -[2025-11-29 19:26:49] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 19:26:51] [INFO] No Global Address found matching criteria. -[2025-11-29 19:26:51] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 19:26:55] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 19:26:57] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 19:26:57] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 19:26:57] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 19:26:57] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 19:26:57] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:26:59] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 19:27:01] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 19:27:01] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 19:27:02] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 19:27:02] [INFO] CLEANUP RUN FINISHED - -[2025-11-29 20:50:13] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 20:50:13] [INFO] Time Cutoff (General): 2025-11-29T20:50:13+0000 -[2025-11-29 20:50:13] [INFO] Time Cutoff (Images): 2025-09-30T20:50:13+0000 -[2025-11-29 20:50:13] [INFO] Delete Limit per Type: 20 -[2025-11-29 20:50:13] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 20:50:13] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 20:50:15] [INFO] No Service Accounts found matching prefix. -[2025-11-29 20:50:15] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 20:50:17] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 20:50:17] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 20:50:19] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 20:50:19] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 20:50:19] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 20:50:19] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 20:50:21] [INFO] No Filestore instances found matching criteria. -[2025-11-29 20:50:21] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 20:50:24] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 20:50:24] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 20:50:24] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 20:50:24] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 20:50:24] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 20:50:24] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 20:50:24] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 20:50:24] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 20:50:24] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 20:50:24] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 20:50:24] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 20:50:26] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 20:50:26] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 20:50:26] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 20:50:26] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 20:50:26] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-29 20:50:26] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-29 20:50:28] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 20:50:28] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 20:50:30] [INFO] No Regional Address found matching criteria. -[2025-11-29 20:50:30] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 20:50:32] [INFO] No Global Address found matching criteria. -[2025-11-29 20:50:32] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 20:50:36] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 20:50:38] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 20:50:38] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 20:50:38] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 20:50:38] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 20:50:38] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:41] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 20:50:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:50:43] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 20:50:44] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 20:50:44] [INFO] CLEANUP RUN FINISHED -[2025-11-29 20:51:05] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 20:51:06] [INFO] Time Cutoff (General): 2025-11-29T20:51:05+0000 -[2025-11-29 20:51:06] [INFO] Time Cutoff (Images): 2025-09-30T20:51:05+0000 -[2025-11-29 20:51:06] [INFO] Delete Limit per Type: 20 -[2025-11-29 20:51:06] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 20:51:06] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 20:51:08] [INFO] No Service Accounts found matching prefix. -[2025-11-29 20:51:08] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 20:51:09] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 20:51:09] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 20:51:11] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 20:51:11] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 20:51:11] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 20:51:11] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 20:51:14] [INFO] No Filestore instances found matching criteria. -[2025-11-29 20:51:14] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 20:51:16] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 20:51:16] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 20:51:16] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 20:51:16] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 20:51:16] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 20:51:16] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 20:51:16] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 20:51:16] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 20:51:17] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 20:51:17] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 20:51:17] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[2025-11-29 20:51:17] [INFO] Policy: Delete images updated before Sat Nov 15 08:51:17 PM UTC 2025 -[2025-11-29 20:51:20] [INFO] Scanning Target Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:29] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:30] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:32] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:33] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:34] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:35] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:36] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:37] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:38] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025) - Too new (<14 days) -[2025-11-29 20:51:40] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 20:51:42] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 20:51:42] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 20:51:42] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 20:51:42] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 20:51:42] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-29 20:51:42] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-29 20:51:44] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 20:51:44] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 20:51:46] [INFO] No Regional Address found matching criteria. -[2025-11-29 20:51:46] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 20:51:48] [INFO] No Global Address found matching criteria. -[2025-11-29 20:51:48] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 20:51:52] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 20:51:54] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 20:51:54] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 20:51:54] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 20:51:54] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 20:51:54] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:56] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 20:51:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 20:51:58] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 20:52:00] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 20:52:00] [INFO] CLEANUP RUN FINISHED -[2025-11-29 20:59:28] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-29 20:59:28] [INFO] Time Cutoff (General): 2025-11-29T20:59:28+0000 -[2025-11-29 20:59:28] [INFO] Time Cutoff (Images): 2025-09-30T20:59:28+0000 -[2025-11-29 20:59:28] [INFO] Delete Limit per Type: 20 -[2025-11-29 20:59:28] [INFO] Loading exclusions from exclusions.txt... -[2025-11-29 20:59:29] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-29 20:59:30] [INFO] No Service Accounts found matching prefix. -[2025-11-29 20:59:30] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 20:59:32] [INFO] No GKE Cluster found matching criteria. -[2025-11-29 20:59:32] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-29 20:59:34] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 20:59:34] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 20:59:34] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-29 20:59:34] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-29 20:59:36] [INFO] No Filestore instances found matching criteria. -[2025-11-29 20:59:36] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-29 20:59:38] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-29 20:59:39] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-29 20:59:39] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-29 20:59:39] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-29 20:59:39] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-29 20:59:39] [SKIP] pbspro0 (In Exclusion List) -[2025-11-29 20:59:39] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-29 20:59:39] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-29 20:59:39] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-29 20:59:39] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-29 20:59:39] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[2025-11-29 20:59:39] [INFO] Policy: Delete images updated before Sat Nov 15 08:59:39 PM UTC 2025 (Timestamp: 1763240379) -[2025-11-29 20:59:43] [INFO] Scanning Target Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:52] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:53] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:55] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:56] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:57] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 20:59:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:00] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:01] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:02] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:04] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner (Sat Nov 29 12:00:00 AM UTC 2025 | TS: 1764374400 >= Cutoff: 1763240379) - Too new -[2025-11-29 21:00:04] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-29 21:00:06] [SKIP] default-net-router (In Exclusion List) -[2025-11-29 21:00:06] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-29 21:00:06] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-29 21:00:06] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-29 21:00:06] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-29 21:00:06] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-29 21:00:08] [INFO] --- Processing: Compute Addresses --- -[2025-11-29 21:00:08] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 21:00:10] [INFO] No Regional Address found matching criteria. -[2025-11-29 21:00:10] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-29 21:00:12] [INFO] No Global Address found matching criteria. -[2025-11-29 21:00:12] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-29 21:00:16] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-29 21:00:18] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-29 21:00:18] [SKIP] image-inspector (In Exclusion List) -[2025-11-29 21:00:18] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-29 21:00:18] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-29 21:00:18] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:20] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-29 21:00:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-29 21:00:22] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-29 21:00:24] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-29 21:00:24] [INFO] CLEANUP RUN FINISHED diff --git a/disk.txt b/disk.txt deleted file mode 100644 index c4d91c3c2d..0000000000 --- a/disk.txt +++ /dev/null @@ -1,3015 +0,0 @@ ---- Thu Nov 27 12:05:43 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T08:05:43+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -lustredev0-controller us-central1-a -lustredev0-slurm-login-001 us-central1-a - gcloud compute instances delete "lustredev0-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "lustredev0-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -lustre-dev-06-net-router us-central1 -[DRY RUN] Cloud Router: Would delete lustre-dev-06-net-router in us-central1 - Command: gcloud compute routers delete "lustre-dev-06-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -lustre-dev-06-net-fw-allow-iap-ingress -lustre-dev-06-net-fw-allow-internal-traffic -[DRY RUN] Firewall Rule: Would delete lustre-dev-06-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "lustre-dev-06-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete lustre-dev-06-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "lustre-dev-06-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -lustre-dev-06-primary-subnet in us-central1 ---- Processing Subnet: lustre-dev-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-dev-06-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete lustre-dev-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-dev-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-dev-06-net -lustre-qa-05-net ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-9a732912a95e848c for network lustre-dev-06-net -[DRY RUN] Route: Would delete default-route-r-fff77e2697290c10 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete lustre-dev-06-net-fw-allow-iap-ingress for network lustre-dev-06-net -[DRY RUN] Firewall Rule: Would delete lustre-dev-06-net-fw-allow-internal-traffic for network lustre-dev-06-net -[DRY RUN] Network: Would delete lustre-dev-06-net - Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -The following Zonal Disks are targeted for deletion: -a3h23d4-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hca628-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hcb15f-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hcdf00-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsa628c1-ff9f3704-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsa628c1-ff9f3704-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsb15fa8-57b77541-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsb15fa8-57b77541-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsdf0061-6327333f-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsdf0061-6327333f-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3lavnew-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3m0280-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m1312-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -a3m14b2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m1b32-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m1bc7-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m26af-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3m2fb0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -a3m3500-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m3d4c-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m4246-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3m49e4-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3m5577-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m6293-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3m7038-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m72ec-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m816d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -a3m9051-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m9518-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3mbc98-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3h23d4-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hca628-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hcb15f-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hcdf00-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsa628c1-ff9f3704-boot-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsa628c1-ff9f3704-nfs-instance-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsb15fa8-57b77541-boot-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsb15fa8-57b77541-nfs-instance-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsdf0061-6327333f-boot-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3hnfsdf0061-6327333f-nfs-instance-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3lavnew-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m0280-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m1312-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m14b2-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m1b32-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m1bc7-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m26af-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m2fb0-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m3500-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m3d4c-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m4246-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m49e4-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m5577-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m6293-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m7038-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m72ec-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m816d-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m9051-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3m9518-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3mbc98-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet ---- Thu Nov 27 12:06:11 PM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 12:06:33 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T08:06:33+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -lustredev0-controller us-central1-a -lustredev0-slurm-login-001 us-central1-a -[EXECUTE] Instance: Deleting lustredev0-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustredev0-controller]. -[EXECUTE] Instance: Deleting lustredev0-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustredev0-slurm-login-001]. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -lustre-dev-06-net-router us-central1 -[EXECUTE] Cloud Router: Deleting lustre-dev-06-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/lustre-dev-06-net-router]. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -lustre-dev-06-net-fw-allow-iap-ingress -lustre-dev-06-net-fw-allow-internal-traffic -[EXECUTE] Firewall Rule: Deleting lustre-dev-06-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-dev-06-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting lustre-dev-06-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-dev-06-net-fw-allow-internal-traffic]. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -lustre-dev-06-primary-subnet in us-central1 ---- Processing Subnet: lustre-dev-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-dev-06-primary-subnet in us-central1. -[EXECUTE] Subnetwork: Deleting lustre-dev-06-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-dev-06-primary-subnet]. -Successfully deleted Subnetwork lustre-dev-06-primary-subnet in us-central1. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-dev-06-net -lustre-qa-05-net ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-9a732912a95e848c for network lustre-dev-06-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-9a732912a95e848c]. -[EXECUTE] Route: Deleting peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b99c802ac7b2e10 -[EXECUTE] Route: Deleting peering-route-7869e60dfba46542 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-7869e60dfba46542 -[EXECUTE] Route: Deleting peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-87752b9a8f2ebae2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-dev-06-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-dev-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-8b558489' - -ERROR: Failed to delete Network lustre-dev-06-net. Check for remaining dependencies. ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b91a4552351d170 -[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-6f7c1d8537c80540 -[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-qa-05-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' - -ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -The following Zonal Disks are targeted for deletion: -a3h23d4-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hca628-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hcb15f-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hcdf00-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsa628c1-ff9f3704-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsa628c1-ff9f3704-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsb15fa8-57b77541-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsb15fa8-57b77541-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsdf0061-6327333f-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3hnfsdf0061-6327333f-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3lavnew-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -a3m0280-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m1312-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -a3m14b2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m1b32-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m1bc7-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m26af-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3m2fb0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -a3m3500-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m3d4c-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m4246-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3m49e4-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3m5577-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m6293-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3m7038-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m72ec-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m816d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -a3m9051-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3m9518-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3mbc98-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -[EXECUTE] Zonal Disk: Deleting a3h23d4-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3h23d4-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3hca628-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hca628-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3hcb15f-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hcb15f-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3hcdf00-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hcdf00-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3hnfsa628c1-ff9f3704-boot-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsa628c1-ff9f3704-boot-disk]. -[EXECUTE] Zonal Disk: Deleting a3hnfsa628c1-ff9f3704-nfs-instance-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsa628c1-ff9f3704-nfs-instance-disk]. -[EXECUTE] Zonal Disk: Deleting a3hnfsb15fa8-57b77541-boot-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsb15fa8-57b77541-boot-disk]. -[EXECUTE] Zonal Disk: Deleting a3hnfsb15fa8-57b77541-nfs-instance-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsb15fa8-57b77541-nfs-instance-disk]. -[EXECUTE] Zonal Disk: Deleting a3hnfsdf0061-6327333f-boot-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsdf0061-6327333f-boot-disk]. -[EXECUTE] Zonal Disk: Deleting a3hnfsdf0061-6327333f-nfs-instance-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3hnfsdf0061-6327333f-nfs-instance-disk]. -[EXECUTE] Zonal Disk: Deleting a3lavnew-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/a3lavnew-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m0280-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m0280-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m1312-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/a3m1312-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m14b2-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m14b2-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m1b32-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m1b32-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m1bc7-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m1bc7-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m26af-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3m26af-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m2fb0-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/a3m2fb0-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m3500-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m3500-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m3d4c-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m3d4c-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m4246-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3m4246-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m49e4-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3m49e4-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m5577-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m5577-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m6293-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3m6293-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m7038-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m7038-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m72ec-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m72ec-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m816d-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/a3m816d-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m9051-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m9051-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3m9518-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3m9518-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3mbc98-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3mbc98-controller-save]. ---- Thu Nov 27 12:10:18 PM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 12:14:16 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T08:14:16+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-dev-06-net -lustre-qa-05-net ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-dev-06-net - Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -The following Zonal Disks are targeted for deletion: -a3mc60d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -a3me1d2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3me62f-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3me777-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3med3e-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3mega-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3mfe07-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a4h333e-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4h5c04-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4h639c-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4h68f2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4h6c3b-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hc0e2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hcf79-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4he340-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hee35-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hfa418-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hrrsarth-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4newimgek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b -a4oldimgek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b -a7f7bcslur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -bfa462slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -c2dtest7-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -c379slurms-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -c52cb1fa4h-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -ce64slurms-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -d3eslurmsi-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -d3slurmsim-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -d72c8slurm-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -de3580slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3mc60d-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3me1d2-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3me62f-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3me777-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3med3e-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3mega-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a3mfe07-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4h333e-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4h5c04-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4h639c-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4h68f2-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4h6c3b-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc0e2-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hcf79-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4he340-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hee35-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hfa418-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hrrsarth-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4newimgek-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4oldimgek-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a7f7bcslur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "bfa462slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "c2dtest7-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "c379slurms-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "c52cb1fa4h-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "ce64slurms-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "d3eslurmsi-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "d3slurmsim-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "d72c8slurm-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "de3580slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet ---- Thu Nov 27 12:14:42 PM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 12:15:16 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T08:15:16+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-dev-06-net -lustre-qa-05-net ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b99c802ac7b2e10 -[EXECUTE] Route: Deleting peering-route-7869e60dfba46542 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-7869e60dfba46542 -[EXECUTE] Route: Deleting peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-87752b9a8f2ebae2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-dev-06-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-dev-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-8b558489' - -ERROR: Failed to delete Network lustre-dev-06-net. Check for remaining dependencies. ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b91a4552351d170 -[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-6f7c1d8537c80540 -[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-qa-05-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' - -ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -The following Zonal Disks are targeted for deletion: -a3mc60d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -a3me1d2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3me62f-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3me777-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3med3e-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -a3mega-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a3mfe07-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -a4h333e-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4h5c04-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4h639c-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4h68f2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4h6c3b-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hc0e2-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hcf79-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4he340-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hee35-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hfa418-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hrrsarth-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4newimgek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b -a4oldimgek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b -a7f7bcslur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -bfa462slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -c2dtest7-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -c379slurms-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -c52cb1fa4h-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -ce64slurms-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -d3eslurmsi-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -d3slurmsim-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -d72c8slurm-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -de3580slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[EXECUTE] Zonal Disk: Deleting a3mc60d-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/a3mc60d-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3me1d2-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3me1d2-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3me62f-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3me62f-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3me777-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3me777-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3med3e-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/a3med3e-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3mega-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3mega-controller-save]. -[EXECUTE] Zonal Disk: Deleting a3mfe07-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/a3mfe07-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4h333e-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4h333e-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4h5c04-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4h5c04-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4h639c-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4h639c-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4h68f2-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4h68f2-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4h6c3b-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4h6c3b-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4hc0e2-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hc0e2-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4hcf79-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hcf79-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4he340-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4he340-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4hee35-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hee35-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4hfa418-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hfa418-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4hrrsarth-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hrrsarth-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4newimgek-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b/disks/a4newimgek-controller-save]. -[EXECUTE] Zonal Disk: Deleting a4oldimgek-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b/disks/a4oldimgek-controller-save]. -[EXECUTE] Zonal Disk: Deleting a7f7bcslur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/a7f7bcslur-controller-save]. -[EXECUTE] Zonal Disk: Deleting bfa462slur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/bfa462slur-controller-save]. -[EXECUTE] Zonal Disk: Deleting c2dtest7-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/c2dtest7-controller-save]. -[EXECUTE] Zonal Disk: Deleting c379slurms-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/c379slurms-controller-save]. -[EXECUTE] Zonal Disk: Deleting c52cb1fa4h-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/c52cb1fa4h-controller-save]. -[EXECUTE] Zonal Disk: Deleting ce64slurms-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/ce64slurms-controller-save]. -[EXECUTE] Zonal Disk: Deleting d3eslurmsi-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/d3eslurmsi-controller-save]. -[EXECUTE] Zonal Disk: Deleting d3slurmsim-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/d3slurmsim-controller-save]. -[EXECUTE] Zonal Disk: Deleting d72c8slurm-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/d72c8slurm-controller-save]. -[EXECUTE] Zonal Disk: Deleting de3580slur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/de3580slur-controller-save]. ---- Thu Nov 27 12:17:20 PM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 12:18:27 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T08:18:27+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -a4h-slurm-net-router us-central1 -[DRY RUN] Cloud Router: Would delete a4h-slurm-net-router in us-central1 - Command: gcloud compute routers delete "a4h-slurm-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -a4h-slurm-net-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-internal-traffic -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -a4h-slurm-c1a329-primary-subnet in us-central1 ---- Processing Subnet: a4h-slurm-c1a329-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-c1a329-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-c1a329-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-c1a329-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -a4h-slurm-net -lustre-dev-06-net -lustre-qa-05-net ---- Processing Network: a4h-slurm-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-c4140f28d05b01fa for network a4h-slurm-net -[DRY RUN] Route: Would delete default-route-c9eb018ca4302458 for network a4h-slurm-net -[DRY RUN] Route: Would delete default-route-e0f1d431390409f8 for network a4h-slurm-net -[DRY RUN] Route: Would delete default-route-r-563d93643dd6c2ce for network a4h-slurm-net -[DRY RUN] Route: Would delete default-route-r-6fc73fd854e13923 for network a4h-slurm-net -[DRY RUN] Route: Would delete default-route-r-d9145f71426839a1 for network a4h-slurm-net -[DRY RUN] Route: Would delete peering-route-6bf11df1af8e59ac for network a4h-slurm-net -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete a4h-slurm-c1a329 for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic for network a4h-slurm-net -[DRY RUN] Network: Would delete a4h-slurm-net - Command: gcloud compute networks delete "a4h-slurm-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-dev-06-net - Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -The following Zonal Disks are targeted for deletion: -dynpoc-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -ebf828slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4a36-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -f4e324slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -f88073slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -fa4slurmsi-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -g4qclav-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -hpcdy-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -hpcdydis-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -hpcimg-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -hpcslurm-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -laveeek29-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b -lustre06-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[DRY RUN] Zonal Disk: gcloud compute disks delete "dynpoc-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "ebf828slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "f4e324slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "f88073slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "fa4slurmsi-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "g4qclav-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "hpcdy-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "hpcdydis-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "hpcimg-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "hpcslurm-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "laveeek29-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "lustre06-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet ---- Thu Nov 27 12:18:59 PM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 01:21:16 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T09:21:16+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -a4hc1a3-a4highnodeset-0 us-central1-b -a4hc1a3-a4highnodeset-1 us-central1-b -a4hc1a3-controller us-central1-b -a4hc1a3-slurm-login-001 us-central1-b -lustreprod-controller us-central1-a -lustreprod-slurm-login-001 us-central1-a - gcloud compute instances delete "a4hc1a3-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4hc1a3-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4hc1a3-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4hc1a3-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "lustreprod-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "lustreprod-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -a4h-slurm-net-0-router us-central1 -a4h-slurm-net-1-router us-central1 -a4h-slurm-net-router us-central1 -lustre-prod-06-net-router us-central1 -[DRY RUN] Cloud Router: Would delete a4h-slurm-net-0-router in us-central1 - Command: gcloud compute routers delete "a4h-slurm-net-0-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Cloud Router: Would delete a4h-slurm-net-1-router in us-central1 - Command: gcloud compute routers delete "a4h-slurm-net-1-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Cloud Router: Would delete a4h-slurm-net-router in us-central1 - Command: gcloud compute routers delete "a4h-slurm-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Cloud Router: Would delete lustre-prod-06-net-router in us-central1 - Command: gcloud compute routers delete "lustre-prod-06-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -a4h-slurm-c1a329 -a4h-slurm-internal-0 -a4h-slurm-internal-1 -a4h-slurm-net-0-fw-allow-iap-ingress -a4h-slurm-net-1-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-internal-traffic -lustre-prod-06-net-fw-allow-iap-ingress -lustre-prod-06-net-fw-allow-internal-traffic -[DRY RUN] Firewall Rule: Would delete a4h-slurm-c1a329 - Command: gcloud compute firewall-rules delete "a4h-slurm-c1a329" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 - Command: gcloud compute firewall-rules delete "a4h-slurm-internal-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 - Command: gcloud compute firewall-rules delete "a4h-slurm-internal-1" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete lustre-prod-06-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "lustre-prod-06-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete lustre-prod-06-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "lustre-prod-06-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -a4h-slurm-c1a329-primary-subnet in us-central1 -a4h-slurm-mrdma-sub-0 in us-central1 -a4h-slurm-mrdma-sub-1 in us-central1 -a4h-slurm-mrdma-sub-2 in us-central1 -a4h-slurm-mrdma-sub-3 in us-central1 -a4h-slurm-mrdma-sub-4 in us-central1 -a4h-slurm-mrdma-sub-5 in us-central1 -a4h-slurm-mrdma-sub-6 in us-central1 -a4h-slurm-mrdma-sub-7 in us-central1 -a4h-slurm-sub-0 in us-central1 -a4h-slurm-sub-1 in us-central1 -lustre-prod-06-primary-subnet in us-central1 ---- Processing Subnet: a4h-slurm-c1a329-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-c1a329-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-c1a329-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-c1a329-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: a4h-slurm-mrdma-sub-0 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-0 in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: a4h-slurm-mrdma-sub-1 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-1 in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-1 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-1" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: a4h-slurm-mrdma-sub-2 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-2 in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-2 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-2" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: a4h-slurm-mrdma-sub-3 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-3 in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-3 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-3" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: a4h-slurm-mrdma-sub-4 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-4 in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-4 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-4" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: a4h-slurm-mrdma-sub-5 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-5 in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-5 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-5" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: a4h-slurm-mrdma-sub-6 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-6 in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-6 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-6" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: a4h-slurm-mrdma-sub-7 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-7 in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-7 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-7" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: a4h-slurm-sub-0 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-sub-0 in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: a4h-slurm-sub-1 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-sub-1 in us-central1. -[DRY RUN] Subnetwork: Would delete a4h-slurm-sub-1 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-sub-1" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: lustre-prod-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-prod-06-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete lustre-prod-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-prod-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -a4h-slurm-net-0 -a4h-slurm-net-1 -a4h-slurm-net -a4h-slurm-rdma-net -lustre-dev-06-net -lustre-prod-06-net -lustre-qa-05-net ---- Processing Network: a4h-slurm-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-c9eb018ca4302458 for network a4h-slurm-net-0 -[DRY RUN] Route: Would delete default-route-r-6fc73fd854e13923 for network a4h-slurm-net-0 -[DRY RUN] Route: Would delete peering-route-6bf11df1af8e59ac for network a4h-slurm-net-0 -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete a4h-slurm-c1a329 for network a4h-slurm-net-0 -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 for network a4h-slurm-net-0 -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress for network a4h-slurm-net-0 -[DRY RUN] Network: Would delete a4h-slurm-net-0 - Command: gcloud compute networks delete "a4h-slurm-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: a4h-slurm-net-1 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-c4140f28d05b01fa for network a4h-slurm-net-1 -[DRY RUN] Route: Would delete default-route-r-563d93643dd6c2ce for network a4h-slurm-net-1 -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 for network a4h-slurm-net-1 -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress for network a4h-slurm-net-1 -[DRY RUN] Network: Would delete a4h-slurm-net-1 - Command: gcloud compute networks delete "a4h-slurm-net-1" --project="hpc-toolkit-dev" --quiet ---- Processing Network: a4h-slurm-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-c4140f28d05b01fa for network a4h-slurm-net -[DRY RUN] Route: Would delete default-route-c9eb018ca4302458 for network a4h-slurm-net -[DRY RUN] Route: Would delete default-route-e0f1d431390409f8 for network a4h-slurm-net -[DRY RUN] Route: Would delete default-route-r-563d93643dd6c2ce for network a4h-slurm-net -[DRY RUN] Route: Would delete default-route-r-6fc73fd854e13923 for network a4h-slurm-net -[DRY RUN] Route: Would delete default-route-r-d9145f71426839a1 for network a4h-slurm-net -[DRY RUN] Route: Would delete peering-route-6bf11df1af8e59ac for network a4h-slurm-net -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete a4h-slurm-c1a329 for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress for network a4h-slurm-net -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic for network a4h-slurm-net -[DRY RUN] Network: Would delete a4h-slurm-net - Command: gcloud compute networks delete "a4h-slurm-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: a4h-slurm-rdma-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-r-19088df9372a47db for network a4h-slurm-rdma-net -[DRY RUN] Route: Would delete default-route-r-2bee294d1f7bbb6f for network a4h-slurm-rdma-net -[DRY RUN] Route: Would delete default-route-r-338b3187b72833ab for network a4h-slurm-rdma-net -[DRY RUN] Route: Would delete default-route-r-96dc060624d84425 for network a4h-slurm-rdma-net -[DRY RUN] Route: Would delete default-route-r-b7660fe09523cc85 for network a4h-slurm-rdma-net -[DRY RUN] Route: Would delete default-route-r-c9124f49ccbd9cd1 for network a4h-slurm-rdma-net -[DRY RUN] Route: Would delete default-route-r-ce12262b3077e680 for network a4h-slurm-rdma-net -[DRY RUN] Route: Would delete default-route-r-f87d27e61d42aad8 for network a4h-slurm-rdma-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete a4h-slurm-rdma-net - Command: gcloud compute networks delete "a4h-slurm-rdma-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-dev-06-net - Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-prod-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-efa6510eac5f44f0 for network lustre-prod-06-net -[DRY RUN] Route: Would delete default-route-r-c93d4441b25056d3 for network lustre-prod-06-net -[DRY RUN] Route: Would delete peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net -[DRY RUN] Route: Would delete peering-route-eebb81463c1f952f for network lustre-prod-06-net -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete lustre-prod-06-net-fw-allow-iap-ingress for network lustre-prod-06-net -[DRY RUN] Firewall Rule: Would delete lustre-prod-06-net-fw-allow-internal-traffic for network lustre-prod-06-net -[DRY RUN] Network: Would delete lustre-prod-06-net - Command: gcloud compute networks delete "lustre-prod-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -The following Zonal Disks are targeted for deletion: -a4hc1a3-a4highnodeset-0 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hc1a3-a4highnodeset-1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hc1a3-controller https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hc1a3-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -a4hc1a3-slurm-login-001 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -dynpoc-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -ebf828slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4a36-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -f4e324slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -f88073slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -fa4slurmsi-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -g4qclav-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -hpcdy-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc1a3-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc1a3-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc1a3-controller" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc1a3-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "a4hc1a3-slurm-login-001" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "dynpoc-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "ebf828slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4691-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-4a36-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mds0-mdt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mgs0-mgt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-mgs0-mnt0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss0-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss1-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "exascaler-cloud-a2a0-oss2-ost0-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "f4e324slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "f88073slur-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "fa4slurmsi-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "g4qclav-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "hpcdy-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c" --quiet ---- Thu Nov 27 01:22:18 PM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 01:22:39 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T09:22:39+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -a4hc1a3-a4highnodeset-0 us-central1-b -a4hc1a3-a4highnodeset-1 us-central1-b -a4hc1a3-controller us-central1-b -a4hc1a3-slurm-login-001 us-central1-b -lustreprod-controller us-central1-a -lustreprod-slurm-login-001 us-central1-a -[EXECUTE] Instance: Deleting a4hc1a3-a4highnodeset-0 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc1a3-a4highnodeset-0]. -[EXECUTE] Instance: Deleting a4hc1a3-a4highnodeset-1 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc1a3-a4highnodeset-1]. -[EXECUTE] Instance: Deleting a4hc1a3-controller in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc1a3-controller]. -[EXECUTE] Instance: Deleting a4hc1a3-slurm-login-001 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc1a3-slurm-login-001]. -[EXECUTE] Instance: Deleting lustreprod-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustreprod-controller]. -[EXECUTE] Instance: Deleting lustreprod-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustreprod-slurm-login-001]. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -a4h-slurm-net-0-router us-central1 -a4h-slurm-net-1-router us-central1 -a4h-slurm-net-router us-central1 -lustre-prod-06-net-router us-central1 -[EXECUTE] Cloud Router: Deleting a4h-slurm-net-0-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-0-router]. -[EXECUTE] Cloud Router: Deleting a4h-slurm-net-1-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-1-router]. -[EXECUTE] Cloud Router: Deleting a4h-slurm-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-router]. -[EXECUTE] Cloud Router: Deleting lustre-prod-06-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/lustre-prod-06-net-router]. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -a4h-slurm-c1a329 -a4h-slurm-internal-0 -a4h-slurm-internal-1 -a4h-slurm-net-0-fw-allow-iap-ingress -a4h-slurm-net-1-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-internal-traffic -lustre-prod-06-net-fw-allow-iap-ingress -lustre-prod-06-net-fw-allow-internal-traffic -[EXECUTE] Firewall Rule: Deleting a4h-slurm-c1a329 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-c1a329]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-internal-0]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-internal-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-internal-1]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-1-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-1-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting lustre-prod-06-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-prod-06-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting lustre-prod-06-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-prod-06-net-fw-allow-internal-traffic]. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -a4h-slurm-c1a329-primary-subnet in us-central1 -a4h-slurm-mrdma-sub-0 in us-central1 -a4h-slurm-mrdma-sub-1 in us-central1 -a4h-slurm-mrdma-sub-2 in us-central1 -a4h-slurm-mrdma-sub-3 in us-central1 -a4h-slurm-mrdma-sub-4 in us-central1 -a4h-slurm-mrdma-sub-5 in us-central1 -a4h-slurm-mrdma-sub-6 in us-central1 -a4h-slurm-mrdma-sub-7 in us-central1 -a4h-slurm-sub-0 in us-central1 -a4h-slurm-sub-1 in us-central1 -lustre-prod-06-primary-subnet in us-central1 ---- Processing Subnet: a4h-slurm-c1a329-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-c1a329-primary-subnet in us-central1. -[EXECUTE] Subnetwork: Deleting a4h-slurm-c1a329-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-c1a329-primary-subnet]. -Successfully deleted Subnetwork a4h-slurm-c1a329-primary-subnet in us-central1. ---- Processing Subnet: a4h-slurm-mrdma-sub-0 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-0 in us-central1. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-0 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-0]. -Successfully deleted Subnetwork a4h-slurm-mrdma-sub-0 in us-central1. ---- Processing Subnet: a4h-slurm-mrdma-sub-1 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-1 in us-central1. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-1 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-1]. -Successfully deleted Subnetwork a4h-slurm-mrdma-sub-1 in us-central1. ---- Processing Subnet: a4h-slurm-mrdma-sub-2 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-2 in us-central1. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-2 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-2]. -Successfully deleted Subnetwork a4h-slurm-mrdma-sub-2 in us-central1. ---- Processing Subnet: a4h-slurm-mrdma-sub-3 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-3 in us-central1. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-3 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-3]. -Successfully deleted Subnetwork a4h-slurm-mrdma-sub-3 in us-central1. ---- Processing Subnet: a4h-slurm-mrdma-sub-4 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-4 in us-central1. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-4 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-4]. -Successfully deleted Subnetwork a4h-slurm-mrdma-sub-4 in us-central1. ---- Processing Subnet: a4h-slurm-mrdma-sub-5 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-5 in us-central1. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-5 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-5]. -Successfully deleted Subnetwork a4h-slurm-mrdma-sub-5 in us-central1. ---- Processing Subnet: a4h-slurm-mrdma-sub-6 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-6 in us-central1. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-6 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-6]. -Successfully deleted Subnetwork a4h-slurm-mrdma-sub-6 in us-central1. ---- Processing Subnet: a4h-slurm-mrdma-sub-7 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-mrdma-sub-7 in us-central1. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-7 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-7]. -Successfully deleted Subnetwork a4h-slurm-mrdma-sub-7 in us-central1. ---- Processing Subnet: a4h-slurm-sub-0 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-sub-0 in us-central1. -[EXECUTE] Subnetwork: Deleting a4h-slurm-sub-0 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-sub-0]. -Successfully deleted Subnetwork a4h-slurm-sub-0 in us-central1. ---- Processing Subnet: a4h-slurm-sub-1 in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for a4h-slurm-sub-1 in us-central1. -[EXECUTE] Subnetwork: Deleting a4h-slurm-sub-1 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-sub-1]. -Successfully deleted Subnetwork a4h-slurm-sub-1 in us-central1. ---- Processing Subnet: lustre-prod-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-prod-06-primary-subnet in us-central1. -[EXECUTE] Subnetwork: Deleting lustre-prod-06-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-prod-06-primary-subnet]. -Successfully deleted Subnetwork lustre-prod-06-primary-subnet in us-central1. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -a4h-slurm-net-0 -a4h-slurm-net-1 -a4h-slurm-net -a4h-slurm-rdma-net -lustre-dev-06-net -lustre-prod-06-net -lustre-qa-05-net ---- Processing Network: a4h-slurm-net-0 --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-c9eb018ca4302458 for network a4h-slurm-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-c9eb018ca4302458]. -[EXECUTE] Route: Deleting peering-route-6bf11df1af8e59ac for network a4h-slurm-net-0 -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-6bf11df1af8e59ac -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting a4h-slurm-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4h-slurm-net-0]. -Successfully deleted Network a4h-slurm-net-0. ---- Processing Network: a4h-slurm-net-1 --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-c4140f28d05b01fa for network a4h-slurm-net-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-c4140f28d05b01fa]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting a4h-slurm-net-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4h-slurm-net-1]. -Successfully deleted Network a4h-slurm-net-1. ---- Processing Network: a4h-slurm-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-e0f1d431390409f8 for network a4h-slurm-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-e0f1d431390409f8]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting a4h-slurm-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4h-slurm-net]. -Successfully deleted Network a4h-slurm-net. ---- Processing Network: a4h-slurm-rdma-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting a4h-slurm-rdma-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4h-slurm-rdma-net]. -Successfully deleted Network a4h-slurm-rdma-net. ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b99c802ac7b2e10 -[EXECUTE] Route: Deleting peering-route-7869e60dfba46542 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-7869e60dfba46542 -[EXECUTE] Route: Deleting peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-87752b9a8f2ebae2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-dev-06-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-dev-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-8b558489' - -ERROR: Failed to delete Network lustre-dev-06-net. Check for remaining dependencies. ---- Processing Network: lustre-prod-06-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-efa6510eac5f44f0 for network lustre-prod-06-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-efa6510eac5f44f0]. -[EXECUTE] Route: Deleting peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-0a55a53b5c4fdb82 -[EXECUTE] Route: Deleting peering-route-eebb81463c1f952f for network lustre-prod-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-eebb81463c1f952f -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-prod-06-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-prod-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-297f4f1a' - -ERROR: Failed to delete Network lustre-prod-06-net. Check for remaining dependencies. ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b91a4552351d170 -[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-6f7c1d8537c80540 -[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-qa-05-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' - -ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -The following Zonal Disks are targeted for deletion: -a4hc1a3-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -dynpoc-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -ebf828slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4691-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -exascaler-cloud-4a36-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-4a36-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-mds0-mdt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-mgs0-mgt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-mgs0-mnt0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-oss0-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-oss1-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -exascaler-cloud-a2a0-oss2-ost0-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -f4e324slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -f88073slur-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -fa4slurmsi-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -g4qclav-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -hpcdy-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -hpcdydis-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -hpcimg-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -hpcslurm-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -laveeek29-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b -[EXECUTE] Zonal Disk: Deleting a4hc1a3-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/a4hc1a3-controller-save]. -[EXECUTE] Zonal Disk: Deleting dynpoc-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/dynpoc-controller-save]. -[EXECUTE] Zonal Disk: Deleting ebf828slur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/ebf828slur-controller-save]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-mds0-mdt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-mds0-mdt0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-mgs0-mgt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-mgs0-mgt0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-mgs0-mnt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-mgs0-mnt0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-oss0-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-oss0-ost0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-oss1-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-oss1-ost0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4691-oss2-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/exascaler-cloud-4691-oss2-ost0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-mds0-mdt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-mds0-mdt0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-mgs0-mgt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-mgs0-mgt0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-mgs0-mnt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-mgs0-mnt0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-oss0-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-oss0-ost0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-oss1-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-oss1-ost0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-4a36-oss2-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-4a36-oss2-ost0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-mds0-mdt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-mds0-mdt0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-mgs0-mgt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-mgs0-mgt0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-mgs0-mnt0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-mgs0-mnt0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-oss0-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-oss0-ost0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-oss1-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-oss1-ost0-disk]. -[EXECUTE] Zonal Disk: Deleting exascaler-cloud-a2a0-oss2-ost0-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/exascaler-cloud-a2a0-oss2-ost0-disk]. -[EXECUTE] Zonal Disk: Deleting f4e324slur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/f4e324slur-controller-save]. -[EXECUTE] Zonal Disk: Deleting f88073slur-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/f88073slur-controller-save]. -[EXECUTE] Zonal Disk: Deleting fa4slurmsi-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/fa4slurmsi-controller-save]. -[EXECUTE] Zonal Disk: Deleting g4qclav-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/g4qclav-controller-save]. -[EXECUTE] Zonal Disk: Deleting hpcdy-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/hpcdy-controller-save]. -[EXECUTE] Zonal Disk: Deleting hpcdydis-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/disks/hpcdydis-controller-save]. -[EXECUTE] Zonal Disk: Deleting hpcimg-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/hpcimg-controller-save]. -[EXECUTE] Zonal Disk: Deleting hpcslurm-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/hpcslurm-controller-save]. -[EXECUTE] Zonal Disk: Deleting laveeek29-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b/disks/laveeek29-controller-save]. ---- Thu Nov 27 01:41:20 PM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 01:42:00 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T09:42:00+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Resource: gke-gke-a3-nccl-test-system-5a732ef0-nglr (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-5a732ef0-nglr in us-west4-b (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-bbd87395-all (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-bbd87395-all (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-bbd87395-exkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-bbd87395-exkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-bbd87395-inkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-bbd87395-inkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-bbd87395-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-bbd87395-vms (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -Skip Resource: gke-a3-nccl-test-gpunet-0-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-1-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-2-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-3-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-4-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-5-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-6-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-7-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-gke-a3-nccl-test-93689389-pe-subnet (Contains protected substring: gke-a3-nccl-test) -No Subnetworks found to delete in this run after filtering. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Resource: gke-a3-nccl-test-gpunet-0 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-0 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-1 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-2 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-3 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-5 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-6 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-7 (In exclusion list) -Skip Resource: gke-a3-nccl-test-net (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-net (In exclusion list) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-dev-06-net -lustre-prod-06-net -lustre-qa-05-net ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-dev-06-net - Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-prod-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net -[DRY RUN] Route: Would delete peering-route-eebb81463c1f952f for network lustre-prod-06-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-prod-06-net - Command: gcloud compute networks delete "lustre-prod-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -The following Zonal Disks are targeted for deletion: -lustre06-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -lustredev0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -lustreprod-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -lustreqa05-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -lustretest-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -mainek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b -monitoring-8323fe-fb7b1106-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c -monitoring-8323fe-fb7b1106-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c -packer-07ae14 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-119ae1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-29c351 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-2c0c79 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b -packer-2f2917 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-3e3b45 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-3f3cd1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -packer-40dd5c https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -packer-469685 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-4dd90f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -packer-536b09 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -packer-5458a5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-578a64 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-58b950 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-59a068 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-5a390a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-5c41cc https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b -packer-6b4d6d https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-74179b https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -packer-825add https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-84a235 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b -packer-90b969 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -[DRY RUN] Zonal Disk: gcloud compute disks delete "lustre06-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "lustredev0-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "lustreprod-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "lustreqa05-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "lustretest-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "mainek-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "monitoring-8323fe-fb7b1106-boot-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "monitoring-8323fe-fb7b1106-nfs-instance-disk" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-07ae14" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-119ae1" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-29c351" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-2c0c79" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-2f2917" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-3e3b45" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-3f3cd1" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-40dd5c" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-469685" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-4dd90f" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-536b09" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-5458a5" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-578a64" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-58b950" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-59a068" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-5a390a" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-5c41cc" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-6b4d6d" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-74179b" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-825add" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-84a235" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-90b969" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet ---- Thu Nov 27 01:42:31 PM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 01:42:55 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T09:42:55+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-bbd87395-all (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-bbd87395-all (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-bbd87395-exkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-bbd87395-exkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-bbd87395-inkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-bbd87395-inkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-bbd87395-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-bbd87395-vms (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -Skip Resource: gke-a3-nccl-test-gpunet-0-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-1-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-2-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-3-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-4-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-5-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-6-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-gpunet-7-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-a3-nccl-test-subnet (Contains protected substring: gke-a3-nccl-test) -Skip Resource: gke-gke-a3-nccl-test-93689389-pe-subnet (Contains protected substring: gke-a3-nccl-test) -No Subnetworks found to delete in this run after filtering. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Resource: gke-a3-nccl-test-gpunet-0 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-0 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-1 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-2 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-3 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-5 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-6 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7 (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-gpunet-7 (In exclusion list) -Skip Resource: gke-a3-nccl-test-net (Contains protected substring: gke-a3-nccl-test) -Skip Network: gke-a3-nccl-test-net (In exclusion list) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-dev-06-net -lustre-prod-06-net -lustre-qa-05-net ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b99c802ac7b2e10 -[EXECUTE] Route: Deleting peering-route-7869e60dfba46542 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-7869e60dfba46542 -[EXECUTE] Route: Deleting peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-87752b9a8f2ebae2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-dev-06-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-dev-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-8b558489' - -ERROR: Failed to delete Network lustre-dev-06-net. Check for remaining dependencies. ---- Processing Network: lustre-prod-06-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-0a55a53b5c4fdb82 -[EXECUTE] Route: Deleting peering-route-eebb81463c1f952f for network lustre-prod-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-eebb81463c1f952f -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-prod-06-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-prod-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-297f4f1a' - -ERROR: Failed to delete Network lustre-prod-06-net. Check for remaining dependencies. ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b91a4552351d170 -[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-6f7c1d8537c80540 -[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-qa-05-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' - -ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -The following Zonal Disks are targeted for deletion: -lustre06-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -lustredev0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -lustreprod-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -lustreqa05-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -lustretest-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -mainek-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b -monitoring-8323fe-fb7b1106-boot-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c -monitoring-8323fe-fb7b1106-nfs-instance-disk https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c -packer-07ae14 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-119ae1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-29c351 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-2c0c79 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b -packer-2f2917 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-3e3b45 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-3f3cd1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -packer-40dd5c https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -packer-469685 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-4dd90f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -packer-536b09 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -packer-5458a5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-578a64 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-58b950 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-59a068 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-5a390a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-5c41cc https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b -packer-6b4d6d https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-74179b https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -packer-825add https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-84a235 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b -packer-90b969 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -[EXECUTE] Zonal Disk: Deleting lustre06-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/lustre06-controller-save]. -[EXECUTE] Zonal Disk: Deleting lustredev0-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/lustredev0-controller-save]. -[EXECUTE] Zonal Disk: Deleting lustreprod-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/lustreprod-controller-save]. -[EXECUTE] Zonal Disk: Deleting lustreqa05-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/lustreqa05-controller-save]. -[EXECUTE] Zonal Disk: Deleting lustretest-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/lustretest-controller-save]. -[EXECUTE] Zonal Disk: Deleting mainek-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b/disks/mainek-controller-save]. -[EXECUTE] Zonal Disk: Deleting monitoring-8323fe-fb7b1106-boot-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/disks/monitoring-8323fe-fb7b1106-boot-disk]. -[EXECUTE] Zonal Disk: Deleting monitoring-8323fe-fb7b1106-nfs-instance-disk in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/disks/monitoring-8323fe-fb7b1106-nfs-instance-disk]. -[EXECUTE] Zonal Disk: Deleting packer-07ae14 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-07ae14]. -[EXECUTE] Zonal Disk: Deleting packer-119ae1 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-119ae1]. -[EXECUTE] Zonal Disk: Deleting packer-29c351 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-29c351]. -[EXECUTE] Zonal Disk: Deleting packer-2c0c79 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b/disks/packer-2c0c79]. -[EXECUTE] Zonal Disk: Deleting packer-2f2917 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-2f2917]. -[EXECUTE] Zonal Disk: Deleting packer-3e3b45 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-3e3b45]. -[EXECUTE] Zonal Disk: Deleting packer-3f3cd1 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/packer-3f3cd1]. -[EXECUTE] Zonal Disk: Deleting packer-40dd5c in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/packer-40dd5c]. -[EXECUTE] Zonal Disk: Deleting packer-469685 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-469685]. -[EXECUTE] Zonal Disk: Deleting packer-4dd90f in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/disks/packer-4dd90f]. -[EXECUTE] Zonal Disk: Deleting packer-536b09 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/packer-536b09]. -[EXECUTE] Zonal Disk: Deleting packer-5458a5 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-5458a5]. -[EXECUTE] Zonal Disk: Deleting packer-578a64 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-578a64]. -[EXECUTE] Zonal Disk: Deleting packer-58b950 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-58b950]. -[EXECUTE] Zonal Disk: Deleting packer-59a068 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-59a068]. -[EXECUTE] Zonal Disk: Deleting packer-5a390a in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-5a390a]. -[EXECUTE] Zonal Disk: Deleting packer-5c41cc in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b/disks/packer-5c41cc]. -[EXECUTE] Zonal Disk: Deleting packer-6b4d6d in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-6b4d6d]. -[EXECUTE] Zonal Disk: Deleting packer-74179b in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/packer-74179b]. -[EXECUTE] Zonal Disk: Deleting packer-825add in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-825add]. -[EXECUTE] Zonal Disk: Deleting packer-84a235 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b/disks/packer-84a235]. -[EXECUTE] Zonal Disk: Deleting packer-90b969 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-90b969]. ---- Thu Nov 27 01:45:25 PM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 02:10:35 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T10:10:35+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-dev-06-net -lustre-prod-06-net -lustre-qa-05-net ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-dev-06-net - Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-prod-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net -[DRY RUN] Route: Would delete peering-route-eebb81463c1f952f for network lustre-prod-06-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-prod-06-net - Command: gcloud compute networks delete "lustre-prod-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: vertexui-do-not-kill-boot in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) -Skip Zonal Disk: vertexui-do-not-kill-data in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) -The following Zonal Disks are targeted for deletion: -packer-92ca3a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-9e2891 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-a392b5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -packer-a575bb https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-a947ca https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-aa9b5a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-b0173f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-b20310 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -packer-b70fdc https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-b9da76 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-bafb89 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-c44786 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-ce39a2 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-d02e9e https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-d0b2dd https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-d7777f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-e11a23 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b -packer-e861ff https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -packer-ea78e4 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-ec1ba5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-f7d004 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-f9879b https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-ffda40 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -pvc-c70cba5c-a091-449a-9659-b84bd4f11316 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central2-b -ractesh4d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -ractesth4d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -slurm0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-92ca3a" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-9e2891" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-a392b5" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-a575bb" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-a947ca" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-aa9b5a" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-b0173f" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-b20310" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-b70fdc" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-b9da76" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-bafb89" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-c44786" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-ce39a2" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-d02e9e" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-d0b2dd" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-d7777f" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-e11a23" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-e861ff" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-ea78e4" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-ec1ba5" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-f7d004" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-f9879b" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "packer-ffda40" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "pvc-c70cba5c-a091-449a-9659-b84bd4f11316" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central2-b" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "ractesh4d-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "ractesth4d-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a" --quiet -[DRY RUN] Zonal Disk: gcloud compute disks delete "slurm0-controller-save" --project="hpc-toolkit-dev" --zone="https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a" --quiet ---- Thu Nov 27 02:11:06 PM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 02:13:44 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T10:13:44+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-dev-06-net -lustre-prod-06-net -lustre-qa-05-net ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b99c802ac7b2e10 -[EXECUTE] Route: Deleting peering-route-7869e60dfba46542 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-7869e60dfba46542 -[EXECUTE] Route: Deleting peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-87752b9a8f2ebae2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-dev-06-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-dev-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-8b558489' - -ERROR: Failed to delete Network lustre-dev-06-net. Check for remaining dependencies. ---- Processing Network: lustre-prod-06-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-0a55a53b5c4fdb82 -[EXECUTE] Route: Deleting peering-route-eebb81463c1f952f for network lustre-prod-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-eebb81463c1f952f -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-prod-06-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-prod-06-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-297f4f1a' - -ERROR: Failed to delete Network lustre-prod-06-net. Check for remaining dependencies. ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b91a4552351d170 -[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-6f7c1d8537c80540 -[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-qa-05-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' - -ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: vertexui-do-not-kill-boot in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) -Skip Zonal Disk: vertexui-do-not-kill-data in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) -The following Zonal Disks are targeted for deletion: -packer-92ca3a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-9e2891 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-a392b5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -packer-a575bb https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-a947ca https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-aa9b5a https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-b0173f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-b20310 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -packer-b70fdc https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -packer-b9da76 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-bafb89 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-c44786 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-ce39a2 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-d02e9e https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-d0b2dd https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-d7777f https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-e11a23 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b -packer-e861ff https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -packer-ea78e4 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-ec1ba5 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-f7d004 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-f9879b https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -packer-ffda40 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -pvc-c70cba5c-a091-449a-9659-b84bd4f11316 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central2-b -ractesh4d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -ractesth4d-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -slurm0-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -[EXECUTE] Zonal Disk: Deleting packer-92ca3a in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-92ca3a]. -[EXECUTE] Zonal Disk: Deleting packer-9e2891 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-9e2891]. -[EXECUTE] Zonal Disk: Deleting packer-a392b5 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/packer-a392b5]. -[EXECUTE] Zonal Disk: Deleting packer-a575bb in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-a575bb]. -[EXECUTE] Zonal Disk: Deleting packer-a947ca in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-a947ca]. -[EXECUTE] Zonal Disk: Deleting packer-aa9b5a in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-aa9b5a]. -[EXECUTE] Zonal Disk: Deleting packer-b0173f in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-b0173f]. -[EXECUTE] Zonal Disk: Deleting packer-b20310 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/disks/packer-b20310]. -[EXECUTE] Zonal Disk: Deleting packer-b70fdc in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/disks/packer-b70fdc]. -[EXECUTE] Zonal Disk: Deleting packer-b9da76 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-b9da76]. -[EXECUTE] Zonal Disk: Deleting packer-bafb89 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-bafb89]. -[EXECUTE] Zonal Disk: Deleting packer-c44786 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-c44786]. -[EXECUTE] Zonal Disk: Deleting packer-ce39a2 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-ce39a2]. -[EXECUTE] Zonal Disk: Deleting packer-d02e9e in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-d02e9e]. -[EXECUTE] Zonal Disk: Deleting packer-d0b2dd in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-d0b2dd]. -[EXECUTE] Zonal Disk: Deleting packer-d7777f in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-d7777f]. -[EXECUTE] Zonal Disk: Deleting packer-e11a23 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b/disks/packer-e11a23]. -[EXECUTE] Zonal Disk: Deleting packer-e861ff in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/packer-e861ff]. -[EXECUTE] Zonal Disk: Deleting packer-ea78e4 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-ea78e4]. -[EXECUTE] Zonal Disk: Deleting packer-ec1ba5 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-ec1ba5]. -[EXECUTE] Zonal Disk: Deleting packer-f7d004 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-f7d004]. -[EXECUTE] Zonal Disk: Deleting packer-f9879b in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/packer-f9879b]. -[EXECUTE] Zonal Disk: Deleting packer-ffda40 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/packer-ffda40]. -[EXECUTE] Zonal Disk: Deleting pvc-c70cba5c-a091-449a-9659-b84bd4f11316 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central2-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central2-b/disks/pvc-c70cba5c-a091-449a-9659-b84bd4f11316]. -[EXECUTE] Zonal Disk: Deleting ractesh4d-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/ractesh4d-controller-save]. -[EXECUTE] Zonal Disk: Deleting ractesth4d-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/disks/ractesth4d-controller-save]. -[EXECUTE] Zonal Disk: Deleting slurm0-controller-save in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/disks/slurm0-controller-save]. ---- Deletion Phase 4b: Regional Persistent Disks (Top 30) --- -WARNING: The following filter keys were not present in any resource : region -No Regional Disks found matching criteria or list command failed. ---- Thu Nov 27 02:16:09 PM UTC 2025 --- Cleanup Script Run Finished --- - diff --git a/dockerimages.txt b/dockerimages.txt deleted file mode 100644 index e1286f2154..0000000000 --- a/dockerimages.txt +++ /dev/null @@ -1,1352 +0,0 @@ -[2025-11-28 16:04:25] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:04:25] [INFO] Time Cutoff (General): 2025-11-28T15:04:25+0000 -[2025-11-28 16:04:25] [INFO] Time Cutoff (Images): 2025-09-29T16:04:25+0000 -[2025-11-28 16:04:25] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:04:25] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:04:25] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 16:04:29] [INFO] CLEANUP RUN FINISHED -[2025-11-28 16:06:10] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:06:10] [INFO] Time Cutoff (General): 2025-11-28T15:06:10+0000 -[2025-11-28 16:06:10] [INFO] Time Cutoff (Images): 2025-09-29T16:06:10+0000 -[2025-11-28 16:06:10] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:06:10] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:06:11] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 16:39:05] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:39:05] [INFO] Time Cutoff (General): 2025-11-28T15:39:05+0000 -[2025-11-28 16:39:05] [INFO] Time Cutoff (Images): 2025-09-29T16:39:05+0000 -[2025-11-28 16:39:05] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:39:05] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:39:05] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 16:39:09] [DEBUG] Found repositories: -[2025-11-28 16:39:09] [DEBUG] gcr.io - release - cleanup-repo - gcf-artifacts - h4d - hpc-toolkit-repo - slurm -[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. -[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. -[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. -[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. -[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. -[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. -[2025-11-28 16:39:09] [DEBUG] Skipping empty line in repo list. -[2025-11-28 16:39:09] [INFO] Finished processing Docker Images. Deleted 0 images. -[2025-11-28 16:41:41] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:41:41] [INFO] Time Cutoff (General): 2025-11-28T15:41:41+0000 -[2025-11-28 16:41:41] [INFO] Time Cutoff (Images): 2025-09-29T16:41:41+0000 -[2025-11-28 16:41:41] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:41:41] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:41:42] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 16:41:46] [DEBUG] Found repositories output: -[2025-11-28 16:41:46] [DEBUG] gcr.io - release - cleanup-repo - gcf-artifacts - h4d - hpc-toolkit-repo - slurm -[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: gcr.io -[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: release -[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: cleanup-repo -[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: gcf-artifacts -[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: h4d -[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: hpc-toolkit-repo -[2025-11-28 16:41:46] [DEBUG] Skipping line: repo_name is empty. Line content was: slurm -[2025-11-28 16:41:46] [INFO] Finished processing Docker Images. Deleted 0 images. -[2025-11-28 16:44:38] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:44:38] [INFO] Time Cutoff (General): 2025-11-28T15:44:38+0000 -[2025-11-28 16:44:38] [INFO] Time Cutoff (Images): 2025-09-29T16:44:38+0000 -[2025-11-28 16:44:38] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:44:38] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:44:38] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: gcr.io. Skipping. -[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: release. Skipping. -[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: cleanup-repo. Skipping. -[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: gcf-artifacts. Skipping. -[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: h4d. Skipping. -[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: hpc-toolkit-repo. Skipping. -[2025-11-28 16:44:42] [WARNING] Could not parse location or name from ID: slurm. Skipping. -[2025-11-28 16:54:35] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:54:35] [INFO] Time Cutoff (General): 2025-11-28T15:54:35+0000 -[2025-11-28 16:54:35] [INFO] Time Cutoff (Images): 2025-09-29T16:54:35+0000 -[2025-11-28 16:54:35] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:54:35] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:54:36] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 16:55:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:55:29] [INFO] Time Cutoff (General): 2025-11-28T15:55:29+0000 -[2025-11-28 16:55:29] [INFO] Time Cutoff (Images): 2025-09-29T16:55:29+0000 -[2025-11-28 16:55:29] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:55:29] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:55:30] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 16:58:03] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 16:58:03] [INFO] Time Cutoff (General): 2025-11-28T15:58:03+0000 -[2025-11-28 16:58:03] [INFO] Time Cutoff (Images): 2025-09-29T16:58:03+0000 -[2025-11-28 16:58:03] [INFO] Delete Limit per Type: 20 -[2025-11-28 16:58:03] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 16:58:03] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[2025-11-28 16:58:03] [INFO] Listing repositories... -[2025-11-28 16:58:07] [INFO] Found 8 repositories. Scanning... -[2025-11-28 16:58:07] [INFO] Scanning Repository: Listing items under project hpc-toolkit-dev-docker.pkg.dev/hpc-toolkit-dev/across all locations. -[2025-11-28 16:58:08] [WARNING] Failed to list images in Listing items under project hpc-toolkit-dev-docker.pkg.dev/hpc-toolkit-dev/across all locations. (Check permissions?) -[2025-11-28 17:00:39] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:00:39] [INFO] Time Cutoff (General): 2025-11-28T16:00:39+0000 -[2025-11-28 17:00:39] [INFO] Time Cutoff (Images): 2025-09-29T17:00:39+0000 -[2025-11-28 17:00:39] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:00:39] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:00:39] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[2025-11-28 17:00:39] [INFO] Listing repositories... -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 17:00:43] [INFO] Found 7 repositories. Scanning... -[2025-11-28 17:03:35] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:03:35] [INFO] Time Cutoff (General): 2025-11-28T16:03:35+0000 -[2025-11-28 17:03:35] [INFO] Time Cutoff (Images): 2025-09-29T17:03:35+0000 -[2025-11-28 17:03:35] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:03:35] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:03:36] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[2025-11-28 17:03:36] [INFO] Listing repositories... -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 17:06:38] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:06:38] [INFO] Time Cutoff (General): 2025-11-28T16:06:38+0000 -[2025-11-28 17:06:38] [INFO] Time Cutoff (Images): 2025-09-29T17:06:38+0000 -[2025-11-28 17:06:38] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:06:38] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:06:38] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 17:08:27] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:08:27] [INFO] Time Cutoff (General): 2025-11-28T16:08:27+0000 -[2025-11-28 17:08:27] [INFO] Time Cutoff (Images): 2025-09-29T17:08:27+0000 -[2025-11-28 17:08:27] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:08:27] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:08:27] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -gcr.io -[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: gcr.io -release -[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: release -cleanup-repo -[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: cleanup-repo -gcf-artifacts -[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: gcf-artifacts -h4d -[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: h4d -hpc-toolkit-repo -[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: hpc-toolkit-repo -slurm -[2025-11-28 17:08:31] [DEBUG] Skipping non-resource line: slurm -[2025-11-28 17:14:25] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:14:25] [INFO] Time Cutoff (General): 2025-11-28T16:14:25+0000 -[2025-11-28 17:14:25] [INFO] Time Cutoff (Images): 2025-09-29T17:14:25+0000 -[2025-11-28 17:14:25] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:14:25] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:14:26] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - -[2025-11-28 17:16:22] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:16:22] [INFO] Time Cutoff (General): 2025-11-28T16:16:22+0000 -[2025-11-28 17:16:22] [INFO] Time Cutoff (Images): 2025-09-29T17:16:22+0000 -[2025-11-28 17:16:22] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:16:22] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:16:22] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -Listing items under project hpc-toolkit-dev, across all locations. - - gcr.io - release - cleanup-repo - gcf-artifacts - h4d - hpc-toolkit-repo - slurm -./cleanup.sh: line 339: location: unbound variable -[2025-11-28 17:21:04] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:21:04] [INFO] Time Cutoff (General): 2025-11-28T16:21:04+0000 -[2025-11-28 17:21:04] [INFO] Time Cutoff (Images): 2025-09-29T17:21:04+0000 -[2025-11-28 17:21:05] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:21:05] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:21:05] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[2025-11-28 17:21:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:21:47] [INFO] Time Cutoff (General): 2025-11-28T16:21:47+0000 -[2025-11-28 17:21:47] [INFO] Time Cutoff (Images): 2025-09-29T17:21:47+0000 -[2025-11-28 17:21:47] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:21:47] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:21:47] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[ - { - "createTime": "2025-10-13T08:03:59.173812Z", - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us/repositories/gcr.io", - "sizeBytes": "39598014", - "updateTime": "2025-10-13T12:09:09.310894Z", - "vulnerabilityScanningConfig": { - "enablementState": "SCANNING_ACTIVE" - } - }, - { - "cleanupPolicyDryRun": true, - "createTime": "2023-11-01T17:59:10.830905Z", - "dockerConfig": {}, - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us/repositories/release", - "updateTime": "2023-11-01T17:59:10.830905Z", - "vulnerabilityScanningConfig": { - "enablementState": "SCANNING_ACTIVE", - "lastEnableTime": "2023-11-01T17:59:09.991552887Z" - } - }, - { - "createTime": "2025-10-22T06:35:13.473857Z", - "description": "cleanup Docker images", - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/cleanup-repo", - "satisfiesPzi": true, - "sizeBytes": "245560553", - "updateTime": "2025-10-22T06:46:30.327071Z", - "vulnerabilityScanningConfig": { - "enablementState": "SCANNING_ACTIVE", - "lastEnableTime": "2025-10-22T06:35:12.884195815Z" - } - }, - { - "createTime": "2025-10-03T10:00:14.065755Z", - "description": "This repository is created and used by Cloud Functions for storing function docker images.", - "format": "DOCKER", - "labels": { - "goog-managed-by": "cloudfunctions" - }, - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/gcf-artifacts", - "satisfiesPzi": true, - "updateTime": "2025-10-03T10:30:13.100631Z", - "vulnerabilityScanningConfig": { - "enablementState": "SCANNING_ACTIVE", - "lastEnableTime": "2025-10-03T10:00:13.385328838Z" - } - }, - { - "cleanupPolicyDryRun": true, - "createTime": "2025-04-24T06:52:17.227166Z", - "dockerConfig": {}, - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/h4d", - "satisfiesPzi": true, - "sizeBytes": "280691926", - "updateTime": "2025-11-14T10:23:27.244749Z", - "vulnerabilityScanningConfig": { - "enablementConfig": "INHERITED", - "enablementState": "SCANNING_ACTIVE", - "lastEnableTime": "2025-04-24T06:52:16.563653918Z" - } - }, - { - "cleanupPolicyDryRun": true, - "createTime": "2021-11-09T08:19:59.809355Z", - "description": "Repo for HPC Toolkit build artifacts", - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/hpc-toolkit-repo", - "satisfiesPzi": true, - "sizeBytes": "721938296284", - "updateTime": "2025-11-27T18:50:53.202608Z", - "vulnerabilityScanningConfig": { - "enablementState": "SCANNING_ACTIVE" - } - }, - { - "cleanupPolicyDryRun": true, - "createTime": "2025-09-26T04:16:29.290075Z", - "dockerConfig": {}, - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us-west4/repositories/slurm", - "satisfiesPzi": true, - "sizeBytes": "1127322284", - "updateTime": "2025-09-26T21:27:32.377895Z", - "vulnerabilityScanningConfig": { - "enablementConfig": "INHERITED", - "enablementState": "SCANNING_ACTIVE", - "lastEnableTime": "2025-09-26T04:16:21.552547084Z" - } - } -] -[2025-11-28 17:24:49] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:24:49] [INFO] Time Cutoff (General): 2025-11-28T16:24:49+0000 -[2025-11-28 17:24:49] [INFO] Time Cutoff (Images): 2025-09-29T17:24:49+0000 -[2025-11-28 17:24:49] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:24:49] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:24:49] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[2025-11-28 17:26:16] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:26:16] [INFO] Time Cutoff (General): 2025-11-28T16:26:16+0000 -[2025-11-28 17:26:16] [INFO] Time Cutoff (Images): 2025-09-29T17:26:16+0000 -[2025-11-28 17:26:16] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:26:16] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:26:17] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[ - { - "createTime": "2025-10-13T08:03:59.173812Z", - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us/repositories/gcr.io", - "sizeBytes": "39598014", - "updateTime": "2025-10-13T12:09:09.310894Z", - "vulnerabilityScanningConfig": { - "enablementState": "SCANNING_ACTIVE" - } - }, - { - "cleanupPolicyDryRun": true, - "createTime": "2023-11-01T17:59:10.830905Z", - "dockerConfig": {}, - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us/repositories/release", - "updateTime": "2023-11-01T17:59:10.830905Z", - "vulnerabilityScanningConfig": { - "enablementState": "SCANNING_ACTIVE", - "lastEnableTime": "2023-11-01T17:59:09.991552887Z" - } - }, - { - "createTime": "2025-10-22T06:35:13.473857Z", - "description": "cleanup Docker images", - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/cleanup-repo", - "satisfiesPzi": true, - "sizeBytes": "245560553", - "updateTime": "2025-10-22T06:46:30.327071Z", - "vulnerabilityScanningConfig": { - "enablementState": "SCANNING_ACTIVE", - "lastEnableTime": "2025-10-22T06:35:12.884195815Z" - } - }, - { - "createTime": "2025-10-03T10:00:14.065755Z", - "description": "This repository is created and used by Cloud Functions for storing function docker images.", - "format": "DOCKER", - "labels": { - "goog-managed-by": "cloudfunctions" - }, - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/gcf-artifacts", - "satisfiesPzi": true, - "updateTime": "2025-10-03T10:30:13.100631Z", - "vulnerabilityScanningConfig": { - "enablementState": "SCANNING_ACTIVE", - "lastEnableTime": "2025-10-03T10:00:13.385328838Z" - } - }, - { - "cleanupPolicyDryRun": true, - "createTime": "2025-04-24T06:52:17.227166Z", - "dockerConfig": {}, - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/h4d", - "satisfiesPzi": true, - "sizeBytes": "280691926", - "updateTime": "2025-11-14T10:23:27.244749Z", - "vulnerabilityScanningConfig": { - "enablementConfig": "INHERITED", - "enablementState": "SCANNING_ACTIVE", - "lastEnableTime": "2025-04-24T06:52:16.563653918Z" - } - }, - { - "cleanupPolicyDryRun": true, - "createTime": "2021-11-09T08:19:59.809355Z", - "description": "Repo for HPC Toolkit build artifacts", - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us-central1/repositories/hpc-toolkit-repo", - "satisfiesPzi": true, - "sizeBytes": "721938296284", - "updateTime": "2025-11-27T18:50:53.202608Z", - "vulnerabilityScanningConfig": { - "enablementState": "SCANNING_ACTIVE" - } - }, - { - "cleanupPolicyDryRun": true, - "createTime": "2025-09-26T04:16:29.290075Z", - "dockerConfig": {}, - "format": "DOCKER", - "mode": "STANDARD_REPOSITORY", - "name": "projects/hpc-toolkit-dev/locations/us-west4/repositories/slurm", - "satisfiesPzi": true, - "sizeBytes": "1127322284", - "updateTime": "2025-09-26T21:27:32.377895Z", - "vulnerabilityScanningConfig": { - "enablementConfig": "INHERITED", - "enablementState": "SCANNING_ACTIVE", - "lastEnableTime": "2025-09-26T04:16:21.552547084Z" - } - } -] -null gcr.io -null release -null cleanup-repo -null gcf-artifacts -null h4d -null hpc-toolkit-repo -null slurm -Here: null gcr.io -Here: null release -Here: null cleanup-repo -Here: null gcf-artifacts -Here: null h4d -Here: null hpc-toolkit-repo -Here: null slurm -[2025-11-28 17:29:00] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:29:00] [INFO] Time Cutoff (General): 2025-11-28T16:29:00+0000 -[2025-11-28 17:29:00] [INFO] Time Cutoff (Images): 2025-09-29T17:29:00+0000 -[2025-11-28 17:29:00] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:29:00] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:29:00] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[2025-11-28 17:29:03] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io -[2025-11-28 17:29:05] [DRY-RUN] Would delete Docker Image: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go -[2025-11-28 17:29:05] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/release -[2025-11-28 17:29:07] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo -[2025-11-28 17:29:08] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo/scriptimage -[2025-11-28 17:29:08] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/gcf-artifacts -[2025-11-28 17:29:09] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d -[2025-11-28 17:29:11] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-gke-irdma-health-check -[2025-11-28 17:29:11] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-irdma-webhook-server -[2025-11-28 17:29:11] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo -[2025-11-28 17:29:17] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler -[2025-11-28 17:29:17] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim -[2025-11-28 17:29:17] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -[2025-11-28 17:29:17] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -[2025-11-28 17:29:17] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -[2025-11-28 17:29:17] [INFO] Scanning Repository: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm -[2025-11-28 17:29:19] [DRY-RUN] Would delete Docker Image: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm/slurmd-pyxis -[2025-11-28 17:32:51] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:32:51] [INFO] Time Cutoff (General): 2025-11-28T16:32:51+0000 -[2025-11-28 17:32:51] [INFO] Time Cutoff (Images): 2025-09-29T17:32:51+0000 -[2025-11-28 17:32:51] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:32:51] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:32:51] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[2025-11-28 17:32:55] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io -[2025-11-28 17:32:57] [DRY-RUN] Would delete Docker Image: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go -[2025-11-28 17:32:57] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/release -[2025-11-28 17:32:58] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo -[2025-11-28 17:33:00] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo/scriptimage -[2025-11-28 17:33:00] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/gcf-artifacts -[2025-11-28 17:33:01] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d -[2025-11-28 17:33:03] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-gke-irdma-health-check -[2025-11-28 17:33:03] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-irdma-webhook-server -[2025-11-28 17:33:03] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo -[2025-11-28 17:33:13] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler -[2025-11-28 17:33:13] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim -[2025-11-28 17:33:13] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -[2025-11-28 17:33:13] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -[2025-11-28 17:33:13] [DRY-RUN] Would delete Docker Image: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -[2025-11-28 17:33:13] [INFO] Scanning Repository: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm -[2025-11-28 17:33:15] [DRY-RUN] Would delete Docker Image: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm/slurmd-pyxis -[2025-11-28 17:37:11] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:37:11] [INFO] Time Cutoff (General): 2025-11-28T16:37:11+0000 -[2025-11-28 17:37:11] [INFO] Time Cutoff (Images): 2025-09-29T17:37:11+0000 -[2025-11-28 17:37:11] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:37:11] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:37:11] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[2025-11-28 17:37:15] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io -[2025-11-28 17:37:16] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/release -[2025-11-28 17:37:18] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo -[2025-11-28 17:37:19] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/gcf-artifacts -[2025-11-28 17:37:21] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d -[2025-11-28 17:37:22] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo -[2025-11-28 17:37:29] [INFO] Scanning Repository: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm -[2025-11-28 17:38:40] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-28 17:38:40] [INFO] Time Cutoff (General): 2025-11-28T16:38:40+0000 -[2025-11-28 17:38:40] [INFO] Time Cutoff (Images): 2025-09-29T17:38:40+0000 -[2025-11-28 17:38:41] [INFO] Delete Limit per Type: 20 -[2025-11-28 17:38:41] [INFO] Loading exclusions from exclusions.txt... -[2025-11-28 17:38:41] [INFO] --- Processing: Docker Images (Artifact Registry) (Limit: 20) --- -[2025-11-28 17:38:45] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/gcr.io -us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go -us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go -us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go -us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go -us-docker.pkg.dev/hpc-toolkit-dev/gcr.io/irdma-healthcheck-webhook-go -[2025-11-28 17:38:46] [INFO] Scanning Repository: us-docker.pkg.dev/hpc-toolkit-dev/release - - -[2025-11-28 17:38:48] [INFO] > No images found. -[2025-11-28 17:38:48] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo -us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo/scriptimage -us-central1-docker.pkg.dev/hpc-toolkit-dev/cleanup-repo/scriptimage -[2025-11-28 17:38:49] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/gcf-artifacts - - -[2025-11-28 17:38:51] [INFO] > No images found. -[2025-11-28 17:38:51] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d -us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-gke-irdma-health-check -us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-irdma-webhook-server -us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-gke-irdma-health-check -us-central1-docker.pkg.dev/hpc-toolkit-dev/h4d/cluster-toolkit-irdma-webhook-server -[2025-11-28 17:38:52] [INFO] Scanning Repository: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/compiler -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/ghpc-slim -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/hpc-toolkit-builder -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/http -us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -[2025-11-28 17:38:59] [INFO] Scanning Repository: us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm -us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm/slurmd-pyxis -us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm/slurmd-pyxis -us-west4-docker.pkg.dev/hpc-toolkit-dev/slurm/slurmd-pyxis diff --git a/filestores.txt b/filestores.txt deleted file mode 100644 index 022161d854..0000000000 --- a/filestores.txt +++ /dev/null @@ -1,179 +0,0 @@ ---- Thu Nov 27 03:41:05 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Targeting resources created before: 2025-11-26T23:41:05+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -./cleanup.sh: line 77: ---: command not found ---- Deletion Phase 1: GKE Clusters (Top 20) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -The following Filestore instances are targeted for deletion in this run: -a4h-slurm-c0e262-f5260d85 -./cleanup.sh: line 182: NAME: unbound variable ---- Thu Nov 27 03:42:30 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Targeting resources created before: 2025-11-26T23:42:30+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -./cleanup.sh: line 77: ---: command not found ---- Deletion Phase 1: GKE Clusters (Top 20) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Thu Nov 27 03:42:37 AM UTC 2025 --- Cleanup Script Run Finished --- - diff --git a/firewalls.txt b/firewalls.txt deleted file mode 100644 index cbc2696d86..0000000000 --- a/firewalls.txt +++ /dev/null @@ -1,2155 +0,0 @@ ---- Thu Nov 27 03:47:02 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-26T23:47:02+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-all (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-all (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-vms (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -a4h-slurm-c0e262 -a4h-slurm-internal-0 -a4h-slurm-internal-1 -a4h-slurm-net-0-fw-allow-iap-ingress -a4h-slurm-net-1-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-internal-traffic -mainek-net-fw-allow-iap-ingress -mainek-net-fw-allow-internal-traffic -managed-lustre-03-net-fw-allow-iap-ingress -[DRY RUN] Firewall Rule: Would delete a4h-slurm-c0e262 - Command: gcloud compute firewall-rules delete "a4h-slurm-c0e262" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 - Command: gcloud compute firewall-rules delete "a4h-slurm-internal-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 - Command: gcloud compute firewall-rules delete "a4h-slurm-internal-1" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete managed-lustre-03-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "managed-lustre-03-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet ---- Thu Nov 27 03:47:13 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 03:49:36 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-26T23:49:36+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - managed-lustre-03-net-fw-allow-iap-ingress ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-all (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-all (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-vms (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: managed-lustre-03-net-fw-allow-iap-ingress (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -a4h-slurm-c0e262 -a4h-slurm-internal-0 -a4h-slurm-internal-1 -a4h-slurm-net-0-fw-allow-iap-ingress -a4h-slurm-net-1-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-internal-traffic -mainek-net-fw-allow-iap-ingress -mainek-net-fw-allow-internal-traffic -managed-lustre-03-net-fw-allow-internal-traffic -[DRY RUN] Firewall Rule: Would delete a4h-slurm-c0e262 - Command: gcloud compute firewall-rules delete "a4h-slurm-c0e262" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 - Command: gcloud compute firewall-rules delete "a4h-slurm-internal-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 - Command: gcloud compute firewall-rules delete "a4h-slurm-internal-1" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete managed-lustre-03-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "managed-lustre-03-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet ---- Thu Nov 27 03:49:46 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 03:50:25 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-26T23:50:25+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-all (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-all (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-vms (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -a4h-slurm-c0e262 -a4h-slurm-internal-0 -a4h-slurm-internal-1 -a4h-slurm-net-0-fw-allow-iap-ingress -a4h-slurm-net-1-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-internal-traffic -mainek-net-fw-allow-iap-ingress -mainek-net-fw-allow-internal-traffic -managed-lustre-03-net-fw-allow-iap-ingress -[DRY RUN] Firewall Rule: Would delete a4h-slurm-c0e262 - Command: gcloud compute firewall-rules delete "a4h-slurm-c0e262" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-0 - Command: gcloud compute firewall-rules delete "a4h-slurm-internal-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-internal-1 - Command: gcloud compute firewall-rules delete "a4h-slurm-internal-1" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-0-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-1-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4h-slurm-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "a4h-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mainek-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "mainek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete managed-lustre-03-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "managed-lustre-03-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet ---- Thu Nov 27 03:50:37 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 03:51:13 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-26T23:51:13+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-all (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-all (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-vms (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -a4h-slurm-c0e262 -a4h-slurm-internal-0 -a4h-slurm-internal-1 -a4h-slurm-net-0-fw-allow-iap-ingress -a4h-slurm-net-1-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-iap-ingress -a4h-slurm-net-fw-allow-internal-traffic -mainek-net-fw-allow-iap-ingress -mainek-net-fw-allow-internal-traffic -managed-lustre-03-net-fw-allow-iap-ingress -[EXECUTE] Firewall Rule: Deleting a4h-slurm-c0e262 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-c0e262]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-internal-0]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-internal-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-internal-1]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-1-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-1-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4h-slurm-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4h-slurm-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting mainek-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting mainek-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting managed-lustre-03-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/managed-lustre-03-net-fw-allow-iap-ingress]. ---- Thu Nov 27 03:52:42 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 04:01:50 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T00:01:50+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-0-cd329e7681dcd885-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-1-7c5eefba7a3794dd-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-2-a471ab1bfba32e34-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-3-ecde7ab14b91f3fa-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-4-706f8cbf69c0667b-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-5-00641f6a2fe63813-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-6-6d9463728e1194f8-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl--gke-a3-nccl-test-gpunet-7-75cf5c746c638c3d-vms (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-all (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-all (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-exkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-inkubelet (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-d5d4e9ce-vms (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-gke-a3-nccl-test-d5d4e9ce-vms (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -managed-lustre-03-net-fw-allow-internal-traffic -mglsa-net-fw-allow-iap-ingress -mglsa-net-fw-allow-internal-traffic -mglsard-net-fw-allow-iap-ingress -mglsard-net-fw-allow-internal-traffic -ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress -ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic -ml-gke-net-fw-allow-iap-ingress -ml-gke-net-fw-allow-internal-traffic -monitoring-8323fe-net-fw-allow-iap-ingress -[DRY RUN] Firewall Rule: Would delete managed-lustre-03-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "managed-lustre-03-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mglsa-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "mglsa-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mglsa-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "mglsa-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mglsard-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "mglsard-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mglsard-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "mglsard-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete ml-gke-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "ml-gke-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete ml-gke-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "ml-gke-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete monitoring-8323fe-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "monitoring-8323fe-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet ---- Thu Nov 27 04:02:00 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 05:08:31 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:08:31+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -managed-lustre-03-net-fw-allow-internal-traffic -mglsa-net-fw-allow-iap-ingress -mglsa-net-fw-allow-internal-traffic -mglsard-net-fw-allow-iap-ingress -mglsard-net-fw-allow-internal-traffic -ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress -ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic -ml-gke-net-fw-allow-iap-ingress -ml-gke-net-fw-allow-internal-traffic -monitoring-8323fe-net-fw-allow-iap-ingress -[DRY RUN] Firewall Rule: Would delete managed-lustre-03-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "managed-lustre-03-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mglsa-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "mglsa-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mglsa-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "mglsa-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mglsard-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "mglsard-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete mglsard-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "mglsard-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete ml-gke-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "ml-gke-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete ml-gke-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "ml-gke-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete monitoring-8323fe-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "monitoring-8323fe-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet ---- Thu Nov 27 05:08:43 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 05:09:35 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:09:35+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -managed-lustre-03-net-fw-allow-internal-traffic -mglsa-net-fw-allow-iap-ingress -mglsa-net-fw-allow-internal-traffic -mglsard-net-fw-allow-iap-ingress -mglsard-net-fw-allow-internal-traffic -ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress -ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic -ml-gke-net-fw-allow-iap-ingress -ml-gke-net-fw-allow-internal-traffic -monitoring-8323fe-net-fw-allow-iap-ingress -[EXECUTE] Firewall Rule: Deleting managed-lustre-03-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/managed-lustre-03-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting mglsa-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mglsa-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting mglsa-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mglsa-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting mglsard-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mglsard-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting mglsard-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mglsard-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/ml-gke-e2e-a8fae6-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/ml-gke-e2e-a8fae6-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting ml-gke-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/ml-gke-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting ml-gke-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/ml-gke-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting monitoring-8323fe-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/monitoring-8323fe-net-fw-allow-iap-ingress]. ---- Thu Nov 27 05:10:55 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 05:11:11 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:11:11+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-7-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -monitoring-8323fe-net-fw-allow-internal-traffic -sa-chs-ops-internal-0 -sa-chs-ops-net-0-fw-allow-iap-ingress -sispot3u-internal-0 -sispot3u-net-0-fw-allow-iap-ingress -slurm-a3-base-sysnet-fw-allow-iap-ingress -slurm-a3-base-sysnet-fw-allow-internal-traffic -sp-helmtest1-net-fw-allow-iap-ingress -sp-helmtest1-net-fw-allow-internal-traffic -static-sarthakag-net-fw-allow-iap-ingress -[DRY RUN] Firewall Rule: Would delete monitoring-8323fe-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "monitoring-8323fe-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete sa-chs-ops-internal-0 - Command: gcloud compute firewall-rules delete "sa-chs-ops-internal-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete sa-chs-ops-net-0-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "sa-chs-ops-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete sispot3u-internal-0 - Command: gcloud compute firewall-rules delete "sispot3u-internal-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete sispot3u-net-0-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "sispot3u-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete slurm-a3-base-sysnet-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "slurm-a3-base-sysnet-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete slurm-a3-base-sysnet-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "slurm-a3-base-sysnet-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete sp-helmtest1-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "sp-helmtest1-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete sp-helmtest1-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "sp-helmtest1-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete static-sarthakag-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "static-sarthakag-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet ---- Thu Nov 27 05:11:21 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 05:11:39 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:11:39+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 1: GKE Clusters (Top 10) --- -WARNING: The following filter keys were not present in any resource : createTime -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Resource: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-0-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-1-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-2-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-3-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-4-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-5-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-gpunet-6-fw-allow-internal-traffic (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-iap-ingress (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-iap-ingress (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-fw-allow-internal-traffic (Contains protected substring: gke-a3-nccl-test) -Skip Firewall: gke-a3-nccl-test-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -monitoring-8323fe-net-fw-allow-internal-traffic -sa-chs-ops-internal-0 -sa-chs-ops-net-0-fw-allow-iap-ingress -sispot3u-internal-0 -sispot3u-net-0-fw-allow-iap-ingress -slurm-a3-base-sysnet-fw-allow-iap-ingress -slurm-a3-base-sysnet-fw-allow-internal-traffic -sp-helmtest1-net-fw-allow-iap-ingress -sp-helmtest1-net-fw-allow-internal-traffic -static-sarthakag-net-fw-allow-iap-ingress -[EXECUTE] Firewall Rule: Deleting monitoring-8323fe-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/monitoring-8323fe-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting sa-chs-ops-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sa-chs-ops-internal-0]. -[EXECUTE] Firewall Rule: Deleting sa-chs-ops-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sa-chs-ops-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting sispot3u-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sispot3u-internal-0]. -[EXECUTE] Firewall Rule: Deleting sispot3u-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sispot3u-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting slurm-a3-base-sysnet-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/slurm-a3-base-sysnet-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting slurm-a3-base-sysnet-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/slurm-a3-base-sysnet-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting sp-helmtest1-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sp-helmtest1-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting sp-helmtest1-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/sp-helmtest1-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting static-sarthakag-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/static-sarthakag-net-fw-allow-iap-ingress]. ---- Thu Nov 27 05:12:55 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 05:14:50 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:14:50+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 1: GKE Clusters (Top 10) --- -WARNING: The following filter keys were not present in any resource : createTime -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -static-sarthakag-net-fw-allow-internal-traffic -[DRY RUN] Firewall Rule: Would delete static-sarthakag-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "static-sarthakag-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet ---- Thu Nov 27 05:15:00 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 05:15:13 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:15:13+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 1: GKE Clusters (Top 10) --- -WARNING: The following filter keys were not present in any resource : createTime -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -static-sarthakag-net-fw-allow-internal-traffic -[EXECUTE] Firewall Rule: Deleting static-sarthakag-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/static-sarthakag-net-fw-allow-internal-traffic]. ---- Thu Nov 27 05:15:30 AM UTC 2025 --- Cleanup Script Run Finished --- - diff --git a/iam.txt b/iam.txt deleted file mode 100644 index db3313fece..0000000000 --- a/iam.txt +++ /dev/null @@ -1,4836 +0,0 @@ ---- Wed Nov 26 03:34:42 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 43 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic ---- Deletion Phase 2: Service Accounts (Prefix: a3, Top 20) --- -The following Service Accounts are targeted for deletion in this run: -a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[DRY RUN] Service Account: Would delete a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Service Account: Would delete a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:34:44 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:35:42 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 43 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic ---- Deletion Phase 2: Service Accounts (Prefix: a3, Top 20) --- -The following Service Accounts are targeted for deletion in this run: -a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:35:45 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:43:06 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 50 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- -The following Service Accounts are targeted for deletion in this run: -a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:43:09 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:43:24 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 50 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- -The following Service Accounts are targeted for deletion in this run: -a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com ---- Wed Nov 26 03:43:26 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:43:44 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 50 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- -The following Service Accounts are targeted for deletion in this run: -a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3gpu-test-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3gpu-test-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3h-1a8984-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3h-23d450-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3hc-308e7e-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3hc-34802a-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3hc-754952-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3hc-98b097-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3hc-a628c1-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3hc-acec6f-compute@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 03:44:00 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:44:15 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 50 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- -Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -The following Service Accounts are targeted for deletion in this run: -a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:44:18 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:44:43 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- -Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -The following Service Accounts are targeted for deletion in this run: -a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-test-01-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "a3u-test-01-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:44:46 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:45:11 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- -Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -The following Service Accounts are targeted for deletion in this run: -a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3u-test-01-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3hc-b15fa8-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3hc-c27fd0-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3hc-df0061-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3u-shub-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3u-shub-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3u-sp-h10-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3u-sp-h10-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3u-sp-test02-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3u-sp-test02-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting a3u-test-01-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3u-test-01-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 03:45:26 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:45:36 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- -Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -The following Service Accounts are targeted for deletion in this run: -a3u-test-01-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "a3u-test-01-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:45:38 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:45:52 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: a3, Top 10) --- -Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -The following Service Accounts are targeted for deletion in this run: -a3u-test-01-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting a3u-test-01-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [a3u-test-01-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 03:45:55 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:46:46 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: ag, Top 10) --- -The following Service Accounts are targeted for deletion in this run: -ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -agkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -agkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -agrkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -agrkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "agkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "agkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "agrkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "agrkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:46:48 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:47:15 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: ag, Top 10) --- -The following Service Accounts are targeted for deletion in this run: -ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -agkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -agkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -agrkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -agrkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting agkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [agkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting agkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [agkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting agrkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [agrkhu-a2h-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting agrkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [agrkhu-a2h-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 03:47:29 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:47:44 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: kh, Top 10) --- -The following Service Accounts are targeted for deletion in this run: -kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:47:46 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:48:03 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: kh, Top 10) --- -The following Service Accounts are targeted for deletion in this run: -kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 03:48:13 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:48:40 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: enter, Top 10) --- -The following Service Accounts are targeted for deletion in this run: -enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:48:42 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:48:53 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: enter, Top 30) --- -The following Service Accounts are targeted for deletion in this run: -enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:48:55 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:49:07 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: enter, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-c8c582-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e12bc9-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e12bc9-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e12bc9-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e34500-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e34500-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e34500-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-c8c582-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-e12bc9-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-e12bc9-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-e12bc9-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-e34500-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-e34500-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-e34500-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:49:09 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:49:20 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: enter, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-c8c582-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e12bc9-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e12bc9-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e12bc9-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e34500-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e34500-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-e34500-login@hpc-toolkit-dev.iam.gserviceaccount.com -enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com -enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com -enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-3ff825-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-c8c582-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-c8c582-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-c8c582-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-c8c582-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-e12bc9-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-e12bc9-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-e12bc9-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-e12bc9-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-e12bc9-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-e12bc9-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-e34500-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-e34500-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-e34500-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-e34500-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-e34500-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-e34500-login@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 03:50:20 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:50:31 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: enter, Top 50) --- -No Service Accounts matching prefix "enter" found to delete in this run. ---- Wed Nov 26 03:50:33 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:59:00 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a2high, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-a2high-0d9dc0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-0d9dc0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-23b433-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-23b433-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-2b0192-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-3c4787-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-697aa6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-697aa6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-b8013c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-b8013c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-d0dde5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-d0dde5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-eb5c24-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-eb5c24-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-ed0848-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-ed0848-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "gke-a2high-0d9dc0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-0d9dc0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-23b433-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-23b433-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-2b0192-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-3c4787-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-697aa6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-697aa6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-b8013c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-b8013c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-d0dde5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-d0dde5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-eb5c24-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-eb5c24-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-ed0848-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a2high-ed0848-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:59:02 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:59:15 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a2high, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-a2high-0d9dc0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-0d9dc0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-23b433-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-23b433-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-2b0192-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-3c4787-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-697aa6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-697aa6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-b8013c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-b8013c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-d0dde5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-d0dde5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-eb5c24-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-eb5c24-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-ed0848-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a2high-ed0848-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting gke-a2high-0d9dc0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-0d9dc0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-0d9dc0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-0d9dc0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-23b433-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-23b433-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-23b433-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-23b433-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-2b0192-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-2b0192-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-3c4787-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-3c4787-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-697aa6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-697aa6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-697aa6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-697aa6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-b8013c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-b8013c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-b8013c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-b8013c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-d0dde5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-d0dde5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-d0dde5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-d0dde5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-eb5c24-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-eb5c24-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-eb5c24-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-eb5c24-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-ed0848-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-ed0848-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a2high-ed0848-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a2high-ed0848-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 03:59:41 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:00:18 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 51 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a3high-, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:00:20 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:00:49 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 57 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a3high-, Top 50) --- -Skip Service Account: gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -The following Service Accounts are targeted for deletion in this run: -gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:00:51 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:01:04 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 57 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a3high-, Top 50) --- -Skip Service Account: gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -The following Service Accounts are targeted for deletion in this run: -gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-180d30-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-242b29-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-3c28e6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-5afe17-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-6e74f0-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-7e9d0e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-7f6328-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-8a7b3c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-8c5c8a-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-b886ee-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 04:01:25 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:01:59 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 57 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a3ultra-, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-a3ultra-38a428-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-38a428-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-3dae81-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-3dae81-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-4c44e7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-4c44e7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-6a0980-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-6a0980-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-7274d4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-7274d4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-75f21e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-75f21e-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-7c2246-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-7c2246-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-80a51f-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-80a51f-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-a5f9fe-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-a5f9fe-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-ba0d18-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-ba0d18-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-bd974d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-bd974d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-c1cc47-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-c1cc47-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-d90ea2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-d90ea2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-e06534-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-e06534-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-e1e6b2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-e1e6b2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "gke-a3ultra-38a428-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-38a428-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-3dae81-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-3dae81-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-4c44e7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-4c44e7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-6a0980-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-6a0980-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-7274d4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-7274d4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-75f21e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-75f21e-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-7c2246-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-7c2246-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-80a51f-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-80a51f-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-a5f9fe-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-a5f9fe-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-ba0d18-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-ba0d18-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-bd974d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-bd974d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-c1cc47-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-c1cc47-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-d90ea2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-d90ea2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-e06534-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-e06534-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-e1e6b2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a3ultra-e1e6b2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:02:01 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:02:20 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 57 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a3ultra-, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-a3ultra-38a428-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-38a428-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-3dae81-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-3dae81-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-4c44e7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-4c44e7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-6a0980-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-6a0980-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-7274d4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-7274d4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-75f21e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-75f21e-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-7c2246-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-7c2246-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-80a51f-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-80a51f-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-a5f9fe-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-a5f9fe-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-ba0d18-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-ba0d18-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-bd974d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-bd974d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-c1cc47-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-c1cc47-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-d90ea2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-d90ea2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-e06534-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-e06534-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-e1e6b2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3ultra-e1e6b2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting gke-a3ultra-38a428-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-38a428-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-38a428-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-38a428-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-3dae81-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-3dae81-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-3dae81-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-3dae81-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-4c44e7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-4c44e7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-4c44e7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-4c44e7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-6a0980-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-6a0980-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-6a0980-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-6a0980-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-7274d4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-7274d4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-7274d4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-7274d4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-75f21e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-75f21e-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-75f21e-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-75f21e-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-7c2246-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-7c2246-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-7c2246-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-7c2246-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-80a51f-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-80a51f-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-80a51f-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-80a51f-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-a5f9fe-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-a5f9fe-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-a5f9fe-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-a5f9fe-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-ba0d18-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-ba0d18-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-ba0d18-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-ba0d18-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-bd974d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-bd974d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-bd974d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-bd974d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-c1cc47-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-c1cc47-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-c1cc47-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-c1cc47-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-d90ea2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-d90ea2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-d90ea2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-d90ea2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-e06534-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-e06534-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-e06534-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-e06534-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-e1e6b2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-e1e6b2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a3ultra-e1e6b2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a3ultra-e1e6b2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 04:03:08 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:03:28 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 57 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a4-, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:03:30 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:03:59 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a4-, Top 50) --- -Skip Service Account: gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -The following Service Accounts are targeted for deletion in this run: -gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:04:01 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:04:14 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a4-, Top 50) --- -Skip Service Account: gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -The following Service Accounts are targeted for deletion in this run: -gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4-825044-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4-825044-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4-a6fef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4-a6fef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4-b579b5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4-d35653-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4-parul-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4-parul-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 04:04:31 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:05:01 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-dws, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-dws-fs-5ac270-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dws-fs-899431-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dws-fs-c9bf69-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dwsfs-cba5fb-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dwsfs-cba5fb-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dws-fs-fe875c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dws-fs-fe875c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dws-fs-fffe70-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "gke-dws-fs-5ac270-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-dws-fs-899431-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-dws-fs-c9bf69-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-dwsfs-cba5fb-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-dwsfs-cba5fb-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-dws-fs-fe875c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-dws-fs-fe875c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-dws-fs-fffe70-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:05:03 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:05:15 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-dws, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-dws-fs-5ac270-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dws-fs-899431-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dws-fs-c9bf69-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dwsfs-cba5fb-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dwsfs-cba5fb-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dws-fs-fe875c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dws-fs-fe875c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-dws-fs-fffe70-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting gke-dws-fs-5ac270-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-dws-fs-5ac270-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-dws-fs-899431-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-dws-fs-899431-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-dws-fs-c9bf69-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-dws-fs-c9bf69-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-dwsfs-cba5fb-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-dwsfs-cba5fb-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-dwsfs-cba5fb-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-dwsfs-cba5fb-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-dws-fs-fe875c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-dws-fs-fe875c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-dws-fs-fe875c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-dws-fs-fe875c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-dws-fs-fffe70-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-dws-fs-fffe70-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 04:05:28 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:06:06 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-ml, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-ml-136c4c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-ml-136c4c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-ml-6e40a9-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-ml-6e40a9-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-ml-7535ac-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-ml-7535ac-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "gke-ml-136c4c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-ml-136c4c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-ml-6e40a9-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-ml-6e40a9-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-ml-7535ac-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-ml-7535ac-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:06:09 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:06:20 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-ml, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-ml-136c4c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-ml-136c4c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-ml-6e40a9-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-ml-6e40a9-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-ml-7535ac-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-ml-7535ac-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting gke-ml-136c4c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-ml-136c4c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-ml-136c4c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-ml-136c4c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-ml-6e40a9-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-ml-6e40a9-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-ml-6e40a9-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-ml-6e40a9-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-ml-7535ac-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-ml-7535ac-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-ml-7535ac-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-ml-7535ac-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 04:06:31 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:08:29 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: ml-gke, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -ml-gke-65f28f-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-c60a00-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-c60a00-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-cc0c06-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-61611c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-61611c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-7df443-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-7df443-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-a8fae6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-a8fae6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-aa06c5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-aa06c5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-b3b573-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-b3b573-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-b8a492-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-b8a492-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-ceae83-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-ceae83-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-fb384b-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-fb384b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "ml-gke-65f28f-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-c60a00-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-c60a00-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-cc0c06-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-61611c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-61611c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-7df443-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-7df443-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-a8fae6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-a8fae6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-aa06c5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-aa06c5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-b3b573-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-b3b573-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-b8a492-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-b8a492-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-ceae83-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-ceae83-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-fb384b-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "ml-gke-e2e-fb384b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:08:31 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:09:15 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: ml-gke, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -ml-gke-65f28f-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-c60a00-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-c60a00-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-cc0c06-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-61611c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-61611c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-7df443-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-7df443-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-a8fae6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-a8fae6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-aa06c5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-aa06c5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-b3b573-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-b3b573-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-b8a492-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-b8a492-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-ceae83-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-ceae83-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-fb384b-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -ml-gke-e2e-fb384b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting ml-gke-65f28f-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-65f28f-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-c60a00-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-c60a00-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-c60a00-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-c60a00-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-cc0c06-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-cc0c06-gke-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-61611c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-61611c-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-61611c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-61611c-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-7df443-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-7df443-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-7df443-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-7df443-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-a8fae6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-a8fae6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-a8fae6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-a8fae6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-aa06c5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-aa06c5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-aa06c5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-aa06c5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-b3b573-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-b3b573-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-b3b573-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-b3b573-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-b8a492-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-b8a492-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-b8a492-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-b8a492-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-ceae83-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-ceae83-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-ceae83-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-ceae83-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-fb384b-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-fb384b-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting ml-gke-e2e-fb384b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [ml-gke-e2e-fb384b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 04:09:47 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:10:30 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: poornima, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "poornima-gke-h4d-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "poornima-gke-h4d-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "poornima-gke-h4d-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "poornima-gke-h4d-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "poornima-gke-h4d-4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "poornima-gke-h4d-4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "poornima-gke-h4d-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "poornima-gke-h4d-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:10:32 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:10:44 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: poornima, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -poornima-gke-h4d-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting poornima-gke-h4d-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [poornima-gke-h4d-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting poornima-gke-h4d-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [poornima-gke-h4d-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting poornima-gke-h4d-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [poornima-gke-h4d-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting poornima-gke-h4d-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [poornima-gke-h4d-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting poornima-gke-h4d-4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [poornima-gke-h4d-4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting poornima-gke-h4d-4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [poornima-gke-h4d-4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting poornima-gke-h4d-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [poornima-gke-h4d-5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting poornima-gke-h4d-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [poornima-gke-h4d-5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 04:11:01 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:12:27 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a4x, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4x-pb1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4x-pb1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4x-pb1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-a4x-pb1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:12:29 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:12:41 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-a4x, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4x-pb1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4x-pb1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4x-pb1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4x-pb1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-a4x-pb1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-a4x-pb1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 04:12:49 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:13:28 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-hyperdisk, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-hyperdisk-4491a7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-hyperdisk-4491a7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-hyperdisk-450674-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-hyperdisk-450674-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "gke-hyperdisk-4491a7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-hyperdisk-4491a7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-hyperdisk-450674-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-hyperdisk-450674-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:13:30 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:13:38 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-hyperdisk, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-hyperdisk-4491a7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-hyperdisk-4491a7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-hyperdisk-450674-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-hyperdisk-450674-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting gke-hyperdisk-4491a7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-hyperdisk-4491a7-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-hyperdisk-4491a7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-hyperdisk-4491a7-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-hyperdisk-450674-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-hyperdisk-450674-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-hyperdisk-450674-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-hyperdisk-450674-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 04:13:46 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:15:49 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-h4d, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-h4d-f6a672-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - gcloud iam service-accounts delete "gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet - gcloud iam service-accounts delete "gke-h4d-f6a672-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 04:15:52 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:16:04 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: gke-h4d, Top 50) --- -The following Service Accounts are targeted for deletion in this run: -gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-h4d-f6a672-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -[EXECUTE] Service Account: Deleting gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com] -[EXECUTE] Service Account: Deleting gke-h4d-f6a672-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -deleted service account [gke-h4d-f6a672-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com] ---- Wed Nov 26 04:16:10 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:17:26 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 50 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 2: Service Accounts (Prefix: a3m-, Top 50) --- -Skip Service Account: a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -Skip Service Account: a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com (In exclusion list) -No Service Accounts matching prefix "a3m-" found to delete in this run. ---- Wed Nov 26 04:17:28 PM UTC 2025 --- Cleanup Script Run Finished --- -[2025-11-30 16:42:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 16:42:47] [INFO] Time Cutoff (General): 2025-11-30T16:42:47+0000 -[2025-11-30 16:42:47] [INFO] Time Cutoff (Images): 2025-10-01T16:42:47+0000 -[2025-11-30 16:42:47] [INFO] Delete Limit per Type: 200 -[2025-11-30 16:42:47] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 16:42:47] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 16:42:49] [INFO] No Service Accounts found matching prefix. -[2025-11-30 16:42:49] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 16:42:51] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 16:42:51] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 16:42:54] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 16:42:54] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 16:42:54] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 16:42:54] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 16:42:56] [INFO] No Filestore instances found matching criteria. -[2025-11-30 16:42:56] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 16:42:59] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 16:42:59] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 16:42:59] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 16:42:59] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 16:43:00] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 16:43:00] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 16:43:00] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 16:43:00] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 16:43:00] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 16:43:00] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 16:43:00] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 16:43:00] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T16:43:00Z (Unix: 1763311380) -[2025-11-30 16:43:00] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763311380]) -[2025-11-30 16:43:03] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 16:43:03] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 16:43:05] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 16:43:05] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 16:43:05] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 16:43:05] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 16:43:05] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 16:43:05] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 16:43:08] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 16:43:08] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 16:43:10] [INFO] No Regional Address found matching criteria. -[2025-11-30 16:43:10] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 16:43:13] [INFO] No Global Address found matching criteria. -[2025-11-30 16:43:13] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- -[2025-11-30 16:43:17] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 16:43:20] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 16:43:20] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 16:43:20] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 16:43:20] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 16:43:20] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:25] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 16:43:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:43:27] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/artifactregistry.reader -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/artifactregistry.reader -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/artifactregistry.reader -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/artifactregistry.reader -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/artifactregistry.reader -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/artifactregistry.reader -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/artifactregistry.reader -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/artifactregistry.reader -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/artifactregistry.reader -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/artifactregistry.reader -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/artifactregistry.reader -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/compute.instanceAdmin.v1 -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/compute.instanceAdmin.v1 -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/compute.instanceAdmin.v1 -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/compute.instanceAdmin.v1 -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/compute.instanceAdmin.v1 -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/compute.instanceAdmin.v1 -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/container.admin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/container.admin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/iam.serviceAccountUser -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/iam.serviceAccountUser -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/iam.serviceAccountUser -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/iam.serviceAccountUser -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/iam.serviceAccountUser -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/iam.serviceAccountUser -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/logging.logWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/monitoring.metricWriter -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/monitoring.viewer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/monitoring.viewer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/monitoring.viewer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/monitoring.viewer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/monitoring.viewer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/monitoring.viewer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/monitoring.viewer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/monitoring.viewer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/monitoring.viewer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/monitoring.viewer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/monitoring.viewer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/pubsub.admin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/pubsub.admin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/pubsub.admin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/pubsub.admin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/pubsub.admin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/pubsub.admin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/stackdriver.resourceMetadata.writer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/stackdriver.resourceMetadata.writer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/stackdriver.resourceMetadata.writer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/stackdriver.resourceMetadata.writer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/stackdriver.resourceMetadata.writer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/stackdriver.resourceMetadata.writer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/stackdriver.resourceMetadata.writer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/stackdriver.resourceMetadata.writer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/stackdriver.resourceMetadata.writer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/stackdriver.resourceMetadata.writer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/stackdriver.resourceMetadata.writer -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/storage.objectAdmin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/storage.objectAdmin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/storage.objectAdmin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/storage.objectAdmin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/storage.objectAdmin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/storage.objectAdmin -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/storage.objectCreator -[2025-11-30 16:43:29] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/storage.objectCreator -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/storage.objectCreator -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/storage.objectCreator -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/storage.objectCreator -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/storage.objectCreator -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [DRY-RUN] Would remove IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/storage.objectViewer -[2025-11-30 16:43:30] [INFO] CLEANUP RUN FINISHED -[2025-11-30 16:44:09] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 16:44:09] [INFO] Time Cutoff (General): 2025-11-30T16:44:09+0000 -[2025-11-30 16:44:09] [INFO] Time Cutoff (Images): 2025-10-01T16:44:09+0000 -[2025-11-30 16:44:09] [INFO] Delete Limit per Type: 200 -[2025-11-30 16:44:09] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 16:44:09] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 16:44:11] [INFO] No Service Accounts found matching prefix. -[2025-11-30 16:44:11] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 16:44:13] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 16:44:13] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 16:44:16] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 16:44:16] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 16:44:16] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 16:44:16] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 16:44:19] [INFO] No Filestore instances found matching criteria. -[2025-11-30 16:44:19] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 16:44:22] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 16:44:22] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 16:44:22] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 16:44:22] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 16:44:22] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 16:44:22] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 16:44:22] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 16:44:22] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 16:44:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 16:44:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 16:44:23] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 16:44:23] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T16:44:23Z (Unix: 1763311463) -[2025-11-30 16:44:23] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763311463]) -[2025-11-30 16:44:25] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 16:44:25] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 16:44:28] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 16:44:28] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 16:44:28] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 16:44:28] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 16:44:28] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 16:44:28] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 16:44:30] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 16:44:30] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 16:44:33] [INFO] No Regional Address found matching criteria. -[2025-11-30 16:44:33] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 16:44:35] [INFO] No Global Address found matching criteria. -[2025-11-30 16:44:35] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- -[2025-11-30 16:44:40] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 16:44:43] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 16:44:43] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 16:44:43] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 16:44:43] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 16:44:43] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:46] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 16:44:48] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 16:44:48] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 16:44:50] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:44:53] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:44:57] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:00] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:03] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:06] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:09] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:13] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:16] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:20] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:23] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:26] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:29] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:32] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:35] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:39] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:42] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:45] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/container.admin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:48] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/container.admin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:52] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:55] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:45:58] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:01] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:04] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:08] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:10] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:14] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:17] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:20] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:23] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:26] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:30] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:33] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:37] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:40] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:43] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:46] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:50] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:53] [EXECUTE] Removing IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:56] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:46:59] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:02] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:05] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:09] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:12] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:16] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:18] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:21] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:24] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:28] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:31] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:34] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:37] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:40] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:44] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:47] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:50] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:53] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:56] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:47:59] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:02] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:05] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:08] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:11] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:14] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:17] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:20] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:23] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:27] [EXECUTE] Removing IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:30] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:33] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:36] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:40] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:43] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:46] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:49] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:52] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:55] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:48:58] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:01] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:04] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:08] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:11] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:14] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:17] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:20] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:23] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:26] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:29] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:32] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:35] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:38] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:41] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:45] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:48] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:51] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:54] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:49:57] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:00] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:04] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:07] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:10] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:13] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:16] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:19] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:22] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:26] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:29] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:32] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:35] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:38] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:41] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:44] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:48] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106107645271893451575 from role roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:51] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108046394722571947718 from role roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:54] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110989820921341156941 from role roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:50:57] [EXECUTE] Removing IAM binding: deleted:serviceAccount:laveka3h-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110918127125500353643 from role roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:00] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100298979397651775766 from role roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:03] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103612858320802615747 from role roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:06] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100341644059205559544 from role roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:09] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116618512454733727102 from role roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:12] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112141509073872326290 from role roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:15] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106716121623512790522 from role roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:18] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118012390928776530416 from role roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:21] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111938377086534463354 from role roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:24] [EXECUTE] Removing IAM binding: deleted:serviceAccount:a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114670329988045244730 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:28] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-49a022-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102266394755914873155 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:31] [EXECUTE] Removing IAM binding: deleted:serviceAccount:gke-a3mega-82065d-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106399507956488414675 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:34] [EXECUTE] Removing IAM binding: deleted:serviceAccount:h4d-res-swarnabm4-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118128979637791643441 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:37] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108358414623729926243 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:40] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpc-01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116860138243001812308 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:43] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111545836663584555673 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:47] [EXECUTE] Removing IAM binding: deleted:serviceAccount:hpcimg-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113514583702688531621 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:50] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101004292855767006490 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:53] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-dev-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116808054413179159974 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:56] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110229264639258472440 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:51:59] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-prod-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113229608651101418627 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:52:03] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100031193511509603385 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:52:06] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-qa-05-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116336971662174231507 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:52:09] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103923876255070407008 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:52:12] [EXECUTE] Removing IAM binding: deleted:serviceAccount:lustre-test-06-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105852558605686242546 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:52:16] [EXECUTE] Removing IAM binding: deleted:serviceAccount:mglsard-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110578885383284599265 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:52:19] [EXECUTE] Removing IAM binding: deleted:serviceAccount:oncall-a3high-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107341277428348485917 from role roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. -[2025-11-30 16:52:22] [INFO] CLEANUP RUN FINISHED -[2025-11-30 17:31:37] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 17:31:37] [INFO] Time Cutoff (General): 2025-11-30T17:31:37+0000 -[2025-11-30 17:31:37] [INFO] Time Cutoff (Images): 2025-10-01T17:31:37+0000 -[2025-11-30 17:31:37] [INFO] Delete Limit per Type: 200 -[2025-11-30 17:31:37] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 17:31:37] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 17:31:40] [INFO] No Service Accounts found matching prefix. -[2025-11-30 17:31:40] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 17:31:42] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 17:31:42] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 17:31:45] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 17:31:45] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 17:31:45] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 17:31:45] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 17:31:48] [INFO] No Filestore instances found matching criteria. -[2025-11-30 17:31:48] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 17:31:51] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 17:31:51] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 17:31:51] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 17:31:51] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 17:31:51] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 17:31:51] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 17:31:51] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 17:31:51] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 17:31:51] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 17:31:51] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 17:31:52] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 17:31:52] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T17:31:52Z (Unix: 1763314312) -[2025-11-30 17:31:52] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763314312]) -[2025-11-30 17:31:54] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 17:31:54] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 17:31:57] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 17:31:57] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 17:31:57] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 17:31:57] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 17:31:57] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 17:31:57] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 17:31:59] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 17:31:59] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 17:32:01] [INFO] No Regional Address found matching criteria. -[2025-11-30 17:32:01] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 17:32:04] [INFO] No Global Address found matching criteria. -[2025-11-30 17:32:04] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- -[2025-11-30 17:32:08] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 17:32:11] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 17:32:11] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 17:32:11] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 17:32:11] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 17:32:11] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:14] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 17:32:16] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:32:16] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 17:32:18] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 17:32:18] [INFO] CLEANUP RUN FINISHED diff --git a/images.txt b/images.txt deleted file mode 100644 index d0d7ac0ba1..0000000000 --- a/images.txt +++ /dev/null @@ -1,7315 +0,0 @@ -[2025-11-30 14:49:34] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 14:49:34] [INFO] Time Cutoff (General): 2025-11-30T14:49:34+0000 -[2025-11-30 14:49:34] [INFO] Time Cutoff (Images): 2025-10-01T14:49:34+0000 -[2025-11-30 14:49:34] [INFO] Delete Limit per Type: 20 -[2025-11-30 14:49:34] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 14:49:34] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 14:49:37] [INFO] No Service Accounts found matching prefix. -[2025-11-30 14:49:37] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 14:49:39] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 14:49:39] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 14:49:41] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 14:49:41] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 14:49:41] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 14:49:41] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 14:49:44] [INFO] No Filestore instances found matching criteria. -[2025-11-30 14:49:44] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 14:49:47] [DRY-RUN] Would delete VM Image: a3qc-u22-20251001t070936z -[2025-11-30 14:49:47] [DRY-RUN] Would delete VM Image: a3qch-u22-20251001t093852z -[2025-11-30 14:49:47] [DRY-RUN] Would delete VM Image: a3qclavek-u22-20251001t025923z -[2025-11-30 14:49:47] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 14:49:48] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 14:49:48] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 14:49:48] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 14:49:48] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 14:49:48] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 14:49:48] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 14:49:48] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 14:49:48] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 14:49:48] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 14:49:48] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 14:49:48] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T14:49:48Z (Unix: 1763304588) -[2025-11-30 14:49:48] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 (Updated: 2024-03-27T23:09:45 [TS: 1711580985] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 (Updated: 2024-03-27T23:09:45) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 (Updated: 2024-03-29T01:38:38 [TS: 1711676318] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 (Updated: 2024-03-29T01:38:38) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d (Updated: 2024-03-29T07:18:04 [TS: 1711696684] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d (Updated: 2024-03-29T07:18:04) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 (Updated: 2024-03-30T07:18:27 [TS: 1711783107] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 (Updated: 2024-03-30T07:18:27) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 (Updated: 2024-03-31T07:18:48 [TS: 1711869528] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 (Updated: 2024-03-31T07:18:48) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 (Updated: 2024-04-01T07:18:16 [TS: 1711955896] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 (Updated: 2024-04-01T07:18:16) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 (Updated: 2024-04-02T07:18:23 [TS: 1712042303] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 (Updated: 2024-04-02T07:18:23) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c (Updated: 2024-04-03T07:19:36 [TS: 1712128776] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c (Updated: 2024-04-03T07:19:36) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb (Updated: 2024-04-04T00:38:49 [TS: 1712191129] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb (Updated: 2024-04-04T00:38:49) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 (Updated: 2024-04-04T07:18:02 [TS: 1712215082] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 (Updated: 2024-04-04T07:18:02) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 (Updated: 2024-04-05T07:18:24 [TS: 1712301504] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 (Updated: 2024-04-05T07:18:24) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba (Updated: 2024-04-06T07:18:49 [TS: 1712387929] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba (Updated: 2024-04-06T07:18:49) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 (Updated: 2024-04-07T07:19:00 [TS: 1712474340] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 (Updated: 2024-04-07T07:19:00) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 (Updated: 2024-04-08T07:18:15 [TS: 1712560695] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 (Updated: 2024-04-08T07:18:15) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 (Updated: 2024-04-09T07:18:35 [TS: 1712647115] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 (Updated: 2024-04-09T07:18:35) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb (Updated: 2024-04-10T07:18:20 [TS: 1712733500] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb (Updated: 2024-04-10T07:18:20) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d (Updated: 2024-04-11T07:18:36 [TS: 1712819916] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d (Updated: 2024-04-11T07:18:36) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 (Updated: 2024-04-12T07:18:04 [TS: 1712906284] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 (Updated: 2024-04-12T07:18:04) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d (Updated: 2024-04-13T07:18:53 [TS: 1712992733] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d (Updated: 2024-04-13T07:18:53) -[2025-11-30 14:49:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae (Updated: 2024-04-14T07:18:44 [TS: 1713079124] < Cutoff: [TS: 1763304588]) -[2025-11-30 14:49:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae (Updated: 2024-04-14T07:18:44) -[2025-11-30 14:49:54] [INFO] Hit delete limit (20) for Docker Images. -[2025-11-30 14:49:54] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 14:49:55] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 14:49:57] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 14:49:57] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 14:49:57] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 14:49:57] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 14:49:57] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 14:49:57] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 14:49:59] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 14:49:59] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 14:50:02] [INFO] No Regional Address found matching criteria. -[2025-11-30 14:50:02] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 14:50:04] [INFO] No Global Address found matching criteria. -[2025-11-30 14:50:04] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-30 14:50:09] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 14:50:11] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 14:50:11] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 14:50:11] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 14:50:11] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 14:50:11] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:14] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 14:50:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:50:17] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 14:50:18] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 14:50:18] [INFO] CLEANUP RUN FINISHED -[2025-11-30 14:50:31] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 14:50:31] [INFO] Time Cutoff (General): 2025-11-30T14:50:31+0000 -[2025-11-30 14:50:31] [INFO] Time Cutoff (Images): 2025-10-01T14:50:31+0000 -[2025-11-30 14:50:31] [INFO] Delete Limit per Type: 20 -[2025-11-30 14:50:31] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 14:50:32] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 14:50:34] [INFO] No Service Accounts found matching prefix. -[2025-11-30 14:50:34] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 14:50:36] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 14:50:36] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 14:50:39] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 14:50:39] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 14:50:39] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 14:50:39] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 14:50:41] [INFO] No Filestore instances found matching criteria. -[2025-11-30 14:50:41] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 14:50:44] [EXECUTE] Deleting VM Image: a3qc-u22-20251001t070936z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3qc-u22-20251001t070936z]. -[2025-11-30 14:50:53] [SUCCESS] Deleted a3qc-u22-20251001t070936z -[2025-11-30 14:50:53] [EXECUTE] Deleting VM Image: a3qch-u22-20251001t093852z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3qch-u22-20251001t093852z]. -[2025-11-30 14:51:00] [SUCCESS] Deleted a3qch-u22-20251001t093852z -[2025-11-30 14:51:00] [EXECUTE] Deleting VM Image: a3qclavek-u22-20251001t025923z -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/images/a3qclavek-u22-20251001t025923z]. -[2025-11-30 14:51:09] [SUCCESS] Deleted a3qclavek-u22-20251001t025923z -[2025-11-30 14:51:09] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 14:51:10] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 14:51:10] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 14:51:10] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 14:51:10] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 14:51:10] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 14:51:10] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 14:51:10] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 14:51:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 14:51:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 14:51:11] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 14:51:11] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T14:51:11Z (Unix: 1763304671) -[2025-11-30 14:51:11] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 14:51:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 (Updated: 2024-03-27T23:09:45 [TS: 1711580985] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 (Updated: 2024-03-27T23:09:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/280fef8f-40e8-4723-b732-f1bc3ba2e3cc] to complete... -.....done. -[2025-11-30 14:51:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c751b0ec63746e520bc43046d8f363e3d78c450125bcefe3750144144c553539 -[2025-11-30 14:51:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 (Updated: 2024-03-29T01:38:38 [TS: 1711676318] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 (Updated: 2024-03-29T01:38:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4c40cc8e-d225-4ae7-aff7-a8f08dd0755e] to complete... -.....done. -[2025-11-30 14:51:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f9ee9bef1bdb79d070b925e69deef795e186dfc172dc57bbb3c163a562c0a148 -[2025-11-30 14:51:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d (Updated: 2024-03-29T07:18:04 [TS: 1711696684] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d (Updated: 2024-03-29T07:18:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1b2fe78c-9f92-40c4-a748-ff5ac25d719d] to complete... -.....done. -[2025-11-30 14:51:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b25c98173ac73ca0edeb61023d53166a30686bff31692e80bd3a93baecd4894d -[2025-11-30 14:51:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 (Updated: 2024-03-30T07:18:27 [TS: 1711783107] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 (Updated: 2024-03-30T07:18:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/03c29cd7-c320-4a30-80a8-9c99fe572afe] to complete... -.....done. -[2025-11-30 14:51:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7be198fb937da1d98a4027439a5e13d3fbd7b3ea6a7d10b2e5908e27a2843c25 -[2025-11-30 14:51:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 (Updated: 2024-03-31T07:18:48 [TS: 1711869528] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 (Updated: 2024-03-31T07:18:48) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8d5277a8-a066-460d-b549-ad4684e28755] to complete... -.....done. -[2025-11-30 14:51:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:421b81d67580408ccb3d099536d23e8aa409e0f4306245feea86f215f1139ce6 -[2025-11-30 14:51:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 (Updated: 2024-04-01T07:18:16 [TS: 1711955896] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 (Updated: 2024-04-01T07:18:16) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d7d21773-b0af-411a-b105-505261372cf9] to complete... -.....done. -[2025-11-30 14:51:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c2f8ea1b1bcefc453d0555d7b1072f6af9f17b96a4341f2c1967f386c61efa4 -[2025-11-30 14:51:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 (Updated: 2024-04-02T07:18:23 [TS: 1712042303] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 (Updated: 2024-04-02T07:18:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/79833c81-b677-4423-ae90-3a766aad2a9d] to complete... -.....done. -[2025-11-30 14:51:41] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3780313e7699fba44f6d0d29d20ccd096a90714cb151316541e2f8fe9d7914f0 -[2025-11-30 14:51:41] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c (Updated: 2024-04-03T07:19:36 [TS: 1712128776] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:41] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c (Updated: 2024-04-03T07:19:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ca10467d-ccb0-4def-abff-5304e4326b54] to complete... -.....done. -[2025-11-30 14:51:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:582ac53b32570301c6b3d4b048ec2cb9dacee267efbd1f8cad5345bcea41954c -[2025-11-30 14:51:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb (Updated: 2024-04-04T00:38:49 [TS: 1712191129] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb (Updated: 2024-04-04T00:38:49) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/36123b73-ce5b-4237-943d-8d34e5ebcf85] to complete... -.....done. -[2025-11-30 14:51:48] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3dc5df306f588c5997918e098f9937cfd1e79427ff64f6cdf944742102dd8cb -[2025-11-30 14:51:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 (Updated: 2024-04-04T07:18:02 [TS: 1712215082] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 (Updated: 2024-04-04T07:18:02) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c7e2f6f2-9966-4630-833d-2c67cfa3092b] to complete... -.....done. -[2025-11-30 14:51:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:24372da8419e7e9a669f7f5cba2b4d77246a1f2aad6e1835d560ff961b81b037 -[2025-11-30 14:51:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 (Updated: 2024-04-05T07:18:24 [TS: 1712301504] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 (Updated: 2024-04-05T07:18:24) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/95e449b6-bdff-4186-8803-a226776d77e1] to complete... -......done. -[2025-11-30 14:51:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cbca3ddfc3dff4a94148daeee2005e17e89e695a0f92ac118a535feb6f3fe29 -[2025-11-30 14:51:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba (Updated: 2024-04-06T07:18:49 [TS: 1712387929] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba (Updated: 2024-04-06T07:18:49) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1e2e3b9f-c8d2-43f8-a2d9-b598ef6c1758] to complete... -.....done. -[2025-11-30 14:51:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cb516cdce813c6b62fd73c91eaa3a77f00cf5ca2cab18eb72ee6c36b74c71ba -[2025-11-30 14:51:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 (Updated: 2024-04-07T07:19:00 [TS: 1712474340] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:51:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 (Updated: 2024-04-07T07:19:00) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d300e354-55a2-4133-b8d9-e60a88c2ea4d] to complete... -......done. -[2025-11-30 14:52:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:606ca10c4d6fb004b9b636343d32d2114aedbd3ddd044f530f4b875395166af6 -[2025-11-30 14:52:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 (Updated: 2024-04-08T07:18:15 [TS: 1712560695] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:52:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 (Updated: 2024-04-08T07:18:15) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/35066762-f7bd-4245-be58-c0d2e050e758] to complete... -......done. -[2025-11-30 14:52:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b84b78e90676eb57ed19d847f4f7c0547779c6a39e174674292185e6a609b293 -[2025-11-30 14:52:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 (Updated: 2024-04-09T07:18:35 [TS: 1712647115] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:52:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 (Updated: 2024-04-09T07:18:35) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c37cbc0a-5992-4a07-98c9-9436ee23b8bb] to complete... -.....done. -[2025-11-30 14:52:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:221e4844341214f8ab448b87d0f37ade16173033443abac739c869ef0b22abf8 -[2025-11-30 14:52:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb (Updated: 2024-04-10T07:18:20 [TS: 1712733500] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:52:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb (Updated: 2024-04-10T07:18:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a5ce4155-a5d5-4544-8205-dde4a26392b8] to complete... -.....done. -[2025-11-30 14:52:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66e055187a41d204d97c3d5a1af7b0d5e83c92c469a008478a9f4b57d286f2eb -[2025-11-30 14:52:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d (Updated: 2024-04-11T07:18:36 [TS: 1712819916] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:52:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d (Updated: 2024-04-11T07:18:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5cb25c3a-8273-42d7-9622-14f2868c1f81] to complete... -.....done. -[2025-11-30 14:52:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19375b429842ea5006382a83a7f31d9dd32523131747bc055c66699d3175771d -[2025-11-30 14:52:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 (Updated: 2024-04-12T07:18:04 [TS: 1712906284] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:52:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 (Updated: 2024-04-12T07:18:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f8f63255-a4e6-458b-bc25-f6e870909f77] to complete... -.....done. -[2025-11-30 14:52:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bd352807d3228d99159973c14fe68e8c3abc1b53d0b7fa74d66e83286f40b7f8 -[2025-11-30 14:52:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d (Updated: 2024-04-13T07:18:53 [TS: 1712992733] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:52:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d (Updated: 2024-04-13T07:18:53) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1f21e30e-b6dc-4dc2-ba8f-caf8aa4ff7af] to complete... -.....done. -[2025-11-30 14:52:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:18f932cdbf6dc17a582d48f241d2e7d651c0f9c688411a787043c6df7a5e996d -[2025-11-30 14:52:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae (Updated: 2024-04-14T07:18:44 [TS: 1713079124] < Cutoff: [TS: 1763304671]) -[2025-11-30 14:52:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae (Updated: 2024-04-14T07:18:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a8f32c5f-d036-4f58-ac01-b2489872c85e] to complete... -.....done. -[2025-11-30 14:52:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ef8c2cf3b63a86143d886da2cfc81effc17b8d36074d101b66da4294f73f9ae -[2025-11-30 14:52:27] [INFO] Hit delete limit (20) for Docker Images. -[2025-11-30 14:52:27] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 14:52:27] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 14:52:29] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 14:52:29] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 14:52:29] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 14:52:29] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 14:52:29] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 14:52:29] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 14:52:31] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 14:52:32] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 14:52:34] [INFO] No Regional Address found matching criteria. -[2025-11-30 14:52:34] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 14:52:36] [INFO] No Global Address found matching criteria. -[2025-11-30 14:52:36] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-30 14:52:41] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 14:52:44] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 14:52:44] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 14:52:44] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 14:52:44] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 14:52:44] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:47] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 14:52:50] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:52:50] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 14:52:52] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 14:52:52] [INFO] CLEANUP RUN FINISHED -[2025-11-30 14:53:18] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 14:53:18] [INFO] Time Cutoff (General): 2025-11-30T14:53:18+0000 -[2025-11-30 14:53:18] [INFO] Time Cutoff (Images): 2025-10-01T14:53:18+0000 -[2025-11-30 14:53:18] [INFO] Delete Limit per Type: 20 -[2025-11-30 14:53:18] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 14:53:19] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 14:53:21] [INFO] No Service Accounts found matching prefix. -[2025-11-30 14:53:21] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 14:53:23] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 14:53:23] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 14:53:25] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 14:53:25] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 14:53:25] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 14:53:25] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 14:53:28] [INFO] No Filestore instances found matching criteria. -[2025-11-30 14:53:28] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 14:53:31] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 14:53:31] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 14:53:31] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 14:53:31] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 14:53:31] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 14:53:31] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 14:53:31] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 14:53:31] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 14:53:32] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 14:53:32] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 14:53:32] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 14:53:32] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T14:53:32Z (Unix: 1763304812) -[2025-11-30 14:53:32] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19 [TS: 1713165559] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17 [TS: 1713251897] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57 [TS: 1713338277] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08 [TS: 1713424748] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26 [TS: 1713511166] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36 [TS: 1713597516] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57 [TS: 1713683937] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38 [TS: 1713770258] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51 [TS: 1713856671] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09 [TS: 1713943149] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19 [TS: 1714029499] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26 [TS: 1714115966] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09 [TS: 1714202349] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58 [TS: 1714288738] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09 [TS: 1714375269] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18 [TS: 1714461498] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48 [TS: 1714547868] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44 [TS: 1714634264] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43 [TS: 1714720723] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43) -[2025-11-30 14:53:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00 [TS: 1714807080] < Cutoff: [TS: 1763304812]) -[2025-11-30 14:53:38] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00) -[2025-11-30 14:53:38] [INFO] Hit delete limit (20) for Docker Images. -[2025-11-30 14:53:38] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 14:53:38] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 14:53:41] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 14:53:41] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 14:53:41] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 14:53:41] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 14:53:41] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 14:53:41] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 14:53:43] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 14:53:44] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 14:53:46] [INFO] No Regional Address found matching criteria. -[2025-11-30 14:53:46] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 14:53:48] [INFO] No Global Address found matching criteria. -[2025-11-30 14:53:48] [INFO] --- Processing: Service Networking Connections (Limit: 20) --- -[2025-11-30 14:53:53] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 14:53:55] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 14:53:55] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 14:53:55] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 14:53:55] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 14:53:55] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:53:58] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 14:54:01] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:01] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 14:54:02] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 14:54:02] [INFO] CLEANUP RUN FINISHED -[2025-11-30 14:54:08] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 14:54:08] [INFO] Time Cutoff (General): 2025-11-30T14:54:08+0000 -[2025-11-30 14:54:08] [INFO] Time Cutoff (Images): 2025-10-01T14:54:08+0000 -[2025-11-30 14:54:08] [INFO] Delete Limit per Type: 200 -[2025-11-30 14:54:08] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 14:54:09] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 14:54:11] [INFO] No Service Accounts found matching prefix. -[2025-11-30 14:54:11] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 14:54:13] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 14:54:13] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 14:54:15] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 14:54:15] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 14:54:16] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 14:54:16] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 14:54:18] [INFO] No Filestore instances found matching criteria. -[2025-11-30 14:54:18] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 14:54:21] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 14:54:21] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 14:54:21] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 14:54:21] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 14:54:22] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 14:54:22] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 14:54:22] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 14:54:22] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 14:54:22] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 14:54:22] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 14:54:22] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 14:54:22] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T14:54:22Z (Unix: 1763304862) -[2025-11-30 14:54:22] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19 [TS: 1713165559] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17 [TS: 1713251897] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57 [TS: 1713338277] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08 [TS: 1713424748] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26 [TS: 1713511166] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36 [TS: 1713597516] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57 [TS: 1713683937] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38 [TS: 1713770258] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51 [TS: 1713856671] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09 [TS: 1713943149] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19 [TS: 1714029499] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26 [TS: 1714115966] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09 [TS: 1714202349] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58 [TS: 1714288738] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09 [TS: 1714375269] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18 [TS: 1714461498] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48 [TS: 1714547868] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44 [TS: 1714634264] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43 [TS: 1714720723] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00 [TS: 1714807080] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 (Updated: 2024-05-05T07:19:28 [TS: 1714893568] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 (Updated: 2024-05-05T07:19:28) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 (Updated: 2024-05-06T07:18:08 [TS: 1714979888] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 (Updated: 2024-05-06T07:18:08) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 (Updated: 2024-05-07T07:20:44 [TS: 1715066444] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 (Updated: 2024-05-07T07:20:44) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 (Updated: 2024-05-08T07:18:32 [TS: 1715152712] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 (Updated: 2024-05-08T07:18:32) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b (Updated: 2024-05-09T07:19:05 [TS: 1715239145] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b (Updated: 2024-05-09T07:19:05) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd (Updated: 2024-05-10T07:18:32 [TS: 1715325512] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd (Updated: 2024-05-10T07:18:32) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 (Updated: 2024-05-11T07:18:53 [TS: 1715411933] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 (Updated: 2024-05-11T07:18:53) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 (Updated: 2024-05-12T07:18:59 [TS: 1715498339] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 (Updated: 2024-05-12T07:18:59) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 (Updated: 2024-05-13T07:18:49 [TS: 1715584729] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 (Updated: 2024-05-13T07:18:49) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc (Updated: 2024-05-14T07:16:18 [TS: 1715670978] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc (Updated: 2024-05-14T07:16:18) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 (Updated: 2024-05-15T07:18:59 [TS: 1715757539] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 (Updated: 2024-05-15T07:18:59) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 (Updated: 2024-05-16T07:18:34 [TS: 1715843914] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 (Updated: 2024-05-16T07:18:34) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 (Updated: 2024-05-17T07:18:04 [TS: 1715930284] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 (Updated: 2024-05-17T07:18:04) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 (Updated: 2024-05-18T07:18:46 [TS: 1716016726] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 (Updated: 2024-05-18T07:18:46) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd (Updated: 2024-05-19T07:19:20 [TS: 1716103160] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd (Updated: 2024-05-19T07:19:20) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c (Updated: 2024-05-20T07:18:54 [TS: 1716189534] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c (Updated: 2024-05-20T07:18:54) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce (Updated: 2024-05-21T07:18:14 [TS: 1716275894] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce (Updated: 2024-05-21T07:18:14) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e (Updated: 2024-05-22T07:17:56 [TS: 1716362276] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e (Updated: 2024-05-22T07:17:56) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 (Updated: 2024-05-23T07:17:57 [TS: 1716448677] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 (Updated: 2024-05-23T07:17:57) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 (Updated: 2024-05-24T07:17:47 [TS: 1716535067] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 (Updated: 2024-05-24T07:17:47) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 (Updated: 2024-05-25T07:18:10 [TS: 1716621490] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 (Updated: 2024-05-25T07:18:10) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 (Updated: 2024-05-26T07:18:34 [TS: 1716707914] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 (Updated: 2024-05-26T07:18:34) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 (Updated: 2024-05-27T07:18:35 [TS: 1716794315] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 (Updated: 2024-05-27T07:18:35) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf (Updated: 2024-05-28T07:18:07 [TS: 1716880687] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf (Updated: 2024-05-28T07:18:07) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 (Updated: 2024-05-29T07:17:33 [TS: 1716967053] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 (Updated: 2024-05-29T07:17:33) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d (Updated: 2024-05-30T07:18:38 [TS: 1717053518] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d (Updated: 2024-05-30T07:18:38) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 (Updated: 2024-05-31T07:18:40 [TS: 1717139920] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 (Updated: 2024-05-31T07:18:40) -[2025-11-30 14:54:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de (Updated: 2024-06-01T07:18:20 [TS: 1717226300] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:28] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de (Updated: 2024-06-01T07:18:20) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb (Updated: 2024-06-02T07:19:05 [TS: 1717312745] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb (Updated: 2024-06-02T07:19:05) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 (Updated: 2024-06-03T07:18:42 [TS: 1717399122] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 (Updated: 2024-06-03T07:18:42) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 (Updated: 2024-06-04T07:18:28 [TS: 1717485508] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 (Updated: 2024-06-04T07:18:28) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 (Updated: 2024-06-05T07:18:24 [TS: 1717571904] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 (Updated: 2024-06-05T07:18:24) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 (Updated: 2024-06-06T07:20:38 [TS: 1717658438] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 (Updated: 2024-06-06T07:20:38) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d (Updated: 2024-06-07T07:18:27 [TS: 1717744707] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d (Updated: 2024-06-07T07:18:27) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 (Updated: 2024-06-08T07:18:28 [TS: 1717831108] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 (Updated: 2024-06-08T07:18:28) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e (Updated: 2024-06-09T07:18:23 [TS: 1717917503] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e (Updated: 2024-06-09T07:18:23) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 (Updated: 2024-06-10T07:18:11 [TS: 1718003891] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 (Updated: 2024-06-10T07:18:11) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 (Updated: 2024-06-11T07:18:43 [TS: 1718090323] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 (Updated: 2024-06-11T07:18:43) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 (Updated: 2024-06-12T07:18:47 [TS: 1718176727] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 (Updated: 2024-06-12T07:18:47) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a (Updated: 2024-06-13T07:19:10 [TS: 1718263150] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a (Updated: 2024-06-13T07:19:10) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c (Updated: 2024-06-14T07:18:33 [TS: 1718349513] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c (Updated: 2024-06-14T07:18:33) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 (Updated: 2024-06-15T07:19:01 [TS: 1718435941] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 (Updated: 2024-06-15T07:19:01) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 (Updated: 2024-06-16T07:18:57 [TS: 1718522337] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 (Updated: 2024-06-16T07:18:57) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 (Updated: 2024-06-17T07:17:57 [TS: 1718608677] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 (Updated: 2024-06-17T07:17:57) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 (Updated: 2024-06-18T07:18:50 [TS: 1718695130] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 (Updated: 2024-06-18T07:18:50) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 (Updated: 2024-06-19T07:18:24 [TS: 1718781504] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 (Updated: 2024-06-19T07:18:24) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c (Updated: 2024-06-20T07:18:11 [TS: 1718867891] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c (Updated: 2024-06-20T07:18:11) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 (Updated: 2024-06-20T17:43:06 [TS: 1718905386] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 (Updated: 2024-06-20T17:43:06) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a (Updated: 2024-06-21T07:20:29 [TS: 1718954429] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a (Updated: 2024-06-21T07:20:29) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 (Updated: 2024-06-21T18:43:44 [TS: 1718995424] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 (Updated: 2024-06-21T18:43:44) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 (Updated: 2024-06-21T20:07:06 [TS: 1719000426] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 (Updated: 2024-06-21T20:07:06) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db (Updated: 2024-06-21T22:45:30 [TS: 1719009930] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db (Updated: 2024-06-21T22:45:30) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 (Updated: 2024-06-21T23:36:46 [TS: 1719013006] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 (Updated: 2024-06-21T23:36:46) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e (Updated: 2024-06-22T07:18:01 [TS: 1719040681] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e (Updated: 2024-06-22T07:18:01) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 (Updated: 2024-06-23T07:18:54 [TS: 1719127134] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 (Updated: 2024-06-23T07:18:54) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 (Updated: 2024-06-24T07:18:00 [TS: 1719213480] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 (Updated: 2024-06-24T07:18:00) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e (Updated: 2024-06-24T16:40:01 [TS: 1719247201] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e (Updated: 2024-06-24T16:40:01) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 (Updated: 2024-06-24T17:31:54 [TS: 1719250314] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 (Updated: 2024-06-24T17:31:54) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 (Updated: 2024-06-25T07:19:35 [TS: 1719299975] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 (Updated: 2024-06-25T07:19:35) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 (Updated: 2024-06-26T07:19:18 [TS: 1719386358] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 (Updated: 2024-06-26T07:19:18) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf (Updated: 2024-06-27T07:18:30 [TS: 1719472710] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf (Updated: 2024-06-27T07:18:30) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b (Updated: 2024-06-28T07:18:41 [TS: 1719559121] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b (Updated: 2024-06-28T07:18:41) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de (Updated: 2024-06-29T07:18:57 [TS: 1719645537] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de (Updated: 2024-06-29T07:18:57) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 (Updated: 2024-06-30T07:20:51 [TS: 1719732051] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 (Updated: 2024-06-30T07:20:51) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 (Updated: 2024-07-01T07:18:08 [TS: 1719818288] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 (Updated: 2024-07-01T07:18:08) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 (Updated: 2024-07-02T07:18:47 [TS: 1719904727] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 (Updated: 2024-07-02T07:18:47) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 (Updated: 2024-07-03T07:20:11 [TS: 1719991211] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 (Updated: 2024-07-03T07:20:11) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f (Updated: 2024-07-04T07:18:39 [TS: 1720077519] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f (Updated: 2024-07-04T07:18:39) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 (Updated: 2024-07-05T07:18:56 [TS: 1720163936] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 (Updated: 2024-07-05T07:18:56) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 (Updated: 2024-07-06T07:16:34 [TS: 1720250194] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 (Updated: 2024-07-06T07:16:34) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 (Updated: 2024-07-07T07:19:21 [TS: 1720336761] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 (Updated: 2024-07-07T07:19:21) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 (Updated: 2024-07-08T07:19:37 [TS: 1720423177] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 (Updated: 2024-07-08T07:19:37) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 (Updated: 2024-07-09T07:19:28 [TS: 1720509568] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 (Updated: 2024-07-09T07:19:28) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f (Updated: 2024-07-10T07:19:10 [TS: 1720595950] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f (Updated: 2024-07-10T07:19:10) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 (Updated: 2024-07-11T07:18:45 [TS: 1720682325] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 (Updated: 2024-07-11T07:18:45) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 (Updated: 2024-07-12T07:19:53 [TS: 1720768793] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 (Updated: 2024-07-12T07:19:53) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d (Updated: 2024-07-13T07:19:14 [TS: 1720855154] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d (Updated: 2024-07-13T07:19:14) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 (Updated: 2024-07-14T07:19:26 [TS: 1720941566] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 (Updated: 2024-07-14T07:19:26) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 (Updated: 2024-07-15T07:19:03 [TS: 1721027943] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 (Updated: 2024-07-15T07:19:03) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 (Updated: 2024-07-16T07:20:23 [TS: 1721114423] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 (Updated: 2024-07-16T07:20:23) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 (Updated: 2024-07-17T07:17:53 [TS: 1721200673] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 (Updated: 2024-07-17T07:17:53) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 (Updated: 2024-07-18T07:19:07 [TS: 1721287147] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 (Updated: 2024-07-18T07:19:07) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 (Updated: 2024-07-19T07:18:50 [TS: 1721373530] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 (Updated: 2024-07-19T07:18:50) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 (Updated: 2024-07-20T07:18:11 [TS: 1721459891] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 (Updated: 2024-07-20T07:18:11) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 (Updated: 2024-07-21T07:19:01 [TS: 1721546341] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 (Updated: 2024-07-21T07:19:01) -[2025-11-30 14:54:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a (Updated: 2024-07-22T07:19:06 [TS: 1721632746] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:29] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a (Updated: 2024-07-22T07:19:06) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 (Updated: 2024-07-23T07:19:03 [TS: 1721719143] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 (Updated: 2024-07-23T07:19:03) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 (Updated: 2024-07-24T07:18:45 [TS: 1721805525] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 (Updated: 2024-07-24T07:18:45) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd (Updated: 2024-07-25T07:18:14 [TS: 1721891894] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd (Updated: 2024-07-25T07:18:14) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 (Updated: 2024-07-26T07:18:21 [TS: 1721978301] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 (Updated: 2024-07-26T07:18:21) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 (Updated: 2024-07-27T07:20:10 [TS: 1722064810] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 (Updated: 2024-07-27T07:20:10) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 (Updated: 2024-07-28T07:18:54 [TS: 1722151134] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 (Updated: 2024-07-28T07:18:54) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 (Updated: 2024-07-29T07:17:00 [TS: 1722237420] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 (Updated: 2024-07-29T07:17:00) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 (Updated: 2024-07-30T07:18:35 [TS: 1722323915] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 (Updated: 2024-07-30T07:18:35) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 (Updated: 2024-07-31T07:19:04 [TS: 1722410344] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 (Updated: 2024-07-31T07:19:04) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb (Updated: 2024-08-01T07:18:18 [TS: 1722496698] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb (Updated: 2024-08-01T07:18:18) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 (Updated: 2024-08-02T07:18:39 [TS: 1722583119] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 (Updated: 2024-08-02T07:18:39) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd (Updated: 2024-08-03T07:18:31 [TS: 1722669511] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd (Updated: 2024-08-03T07:18:31) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 (Updated: 2024-08-04T07:19:22 [TS: 1722755962] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 (Updated: 2024-08-04T07:19:22) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a (Updated: 2024-08-05T07:19:47 [TS: 1722842387] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a (Updated: 2024-08-05T07:19:47) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e (Updated: 2024-08-06T07:18:59 [TS: 1722928739] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e (Updated: 2024-08-06T07:18:59) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 (Updated: 2024-08-07T07:19:22 [TS: 1723015162] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 (Updated: 2024-08-07T07:19:22) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 (Updated: 2024-08-08T07:18:44 [TS: 1723101524] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 (Updated: 2024-08-08T07:18:44) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 (Updated: 2024-08-09T07:19:24 [TS: 1723187964] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 (Updated: 2024-08-09T07:19:24) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a (Updated: 2024-08-10T07:18:12 [TS: 1723274292] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a (Updated: 2024-08-10T07:18:12) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f (Updated: 2024-08-11T07:18:52 [TS: 1723360732] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f (Updated: 2024-08-11T07:18:52) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff (Updated: 2024-08-12T07:18:33 [TS: 1723447113] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff (Updated: 2024-08-12T07:18:33) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff (Updated: 2024-08-13T07:19:14 [TS: 1723533554] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff (Updated: 2024-08-13T07:19:14) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a (Updated: 2024-08-14T07:20:03 [TS: 1723620003] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a (Updated: 2024-08-14T07:20:03) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 (Updated: 2024-08-15T07:18:25 [TS: 1723706305] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 (Updated: 2024-08-15T07:18:25) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b (Updated: 2024-08-16T07:18:57 [TS: 1723792737] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b (Updated: 2024-08-16T07:18:57) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b (Updated: 2024-08-17T07:19:21 [TS: 1723879161] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b (Updated: 2024-08-17T07:19:21) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e (Updated: 2024-08-18T07:19:19 [TS: 1723965559] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e (Updated: 2024-08-18T07:19:19) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 (Updated: 2024-08-19T07:19:35 [TS: 1724051975] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 (Updated: 2024-08-19T07:19:35) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 (Updated: 2024-08-20T07:19:04 [TS: 1724138344] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 (Updated: 2024-08-20T07:19:04) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 (Updated: 2024-08-21T07:19:04 [TS: 1724224744] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 (Updated: 2024-08-21T07:19:04) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3b91af27e525f56f3ea343a5b051fc87a0f446e614700ba93b62d0eaebaa270 (Updated: 2024-08-22T07:18:34 [TS: 1724311114] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3b91af27e525f56f3ea343a5b051fc87a0f446e614700ba93b62d0eaebaa270 (Updated: 2024-08-22T07:18:34) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:701c37147c8af20d38046af61d1b92ffa9f17043b68ac37b20fed22e3b3f8e79 (Updated: 2024-08-23T07:19:29 [TS: 1724397569] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:701c37147c8af20d38046af61d1b92ffa9f17043b68ac37b20fed22e3b3f8e79 (Updated: 2024-08-23T07:19:29) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bf48f10d95c8d54302ad4f8867103021165ecc19863e553ca631e80257563c9 (Updated: 2024-08-24T07:18:56 [TS: 1724483936] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bf48f10d95c8d54302ad4f8867103021165ecc19863e553ca631e80257563c9 (Updated: 2024-08-24T07:18:56) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df1f07f0794958aa141b2333b7e891c76069a030a4df49df26ed55f685487e05 (Updated: 2024-08-25T07:18:09 [TS: 1724570289] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df1f07f0794958aa141b2333b7e891c76069a030a4df49df26ed55f685487e05 (Updated: 2024-08-25T07:18:09) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:984af1e0f55f42155e6ec839142c7c38f04ca1c1a25e19574de1d76c7a94758c (Updated: 2024-08-26T07:19:22 [TS: 1724656762] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:984af1e0f55f42155e6ec839142c7c38f04ca1c1a25e19574de1d76c7a94758c (Updated: 2024-08-26T07:19:22) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e85e777472660994c5d895d3ba68f320a661307ea61591b315086e52460586a (Updated: 2024-08-27T07:19:37 [TS: 1724743177] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e85e777472660994c5d895d3ba68f320a661307ea61591b315086e52460586a (Updated: 2024-08-27T07:19:37) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e33a34d6aef4605164895bba151e8a8ddaaa850e4feeece12c8328c477cbf94 (Updated: 2024-08-28T07:19:10 [TS: 1724829550] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e33a34d6aef4605164895bba151e8a8ddaaa850e4feeece12c8328c477cbf94 (Updated: 2024-08-28T07:19:10) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19aefe718d2632c0206b274ffee14c8cce3139254c73c13a8c85ce08e193f724 (Updated: 2024-08-29T07:21:40 [TS: 1724916100] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:19aefe718d2632c0206b274ffee14c8cce3139254c73c13a8c85ce08e193f724 (Updated: 2024-08-29T07:21:40) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:256f6245d97633997bd4c3cc2573f97f39b348629a527a5b3fb8e22b7b857d15 (Updated: 2024-08-30T07:19:17 [TS: 1725002357] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:256f6245d97633997bd4c3cc2573f97f39b348629a527a5b3fb8e22b7b857d15 (Updated: 2024-08-30T07:19:17) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 (Updated: 2024-08-31T07:19:15 [TS: 1725088755] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 (Updated: 2024-08-31T07:19:15) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 (Updated: 2024-09-01T07:18:35 [TS: 1725175115] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 (Updated: 2024-09-01T07:18:35) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 (Updated: 2024-09-02T07:19:08 [TS: 1725261548] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 (Updated: 2024-09-02T07:19:08) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 (Updated: 2024-09-03T07:18:34 [TS: 1725347914] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 (Updated: 2024-09-03T07:18:34) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 (Updated: 2024-09-04T07:19:12 [TS: 1725434352] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 (Updated: 2024-09-04T07:19:12) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd (Updated: 2024-09-04T15:00:21 [TS: 1725462021] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd (Updated: 2024-09-04T15:00:21) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 (Updated: 2024-09-05T07:19:39 [TS: 1725520779] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 (Updated: 2024-09-05T07:19:39) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 (Updated: 2024-09-06T07:19:00 [TS: 1725607140] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 (Updated: 2024-09-06T07:19:00) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 (Updated: 2024-09-07T07:20:06 [TS: 1725693606] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 (Updated: 2024-09-07T07:20:06) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 (Updated: 2024-09-08T07:20:00 [TS: 1725780000] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 (Updated: 2024-09-08T07:20:00) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f (Updated: 2024-09-09T07:18:55 [TS: 1725866335] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f (Updated: 2024-09-09T07:18:55) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 (Updated: 2024-09-10T07:18:23 [TS: 1725952703] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 (Updated: 2024-09-10T07:18:23) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 (Updated: 2024-09-11T07:18:27 [TS: 1726039107] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 (Updated: 2024-09-11T07:18:27) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b (Updated: 2024-09-12T07:19:28 [TS: 1726125568] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b (Updated: 2024-09-12T07:19:28) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 (Updated: 2024-09-13T07:18:59 [TS: 1726211939] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 (Updated: 2024-09-13T07:18:59) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 (Updated: 2024-09-14T07:21:55 [TS: 1726298515] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:30] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 (Updated: 2024-09-14T07:21:55) -[2025-11-30 14:54:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 (Updated: 2024-09-15T07:18:46 [TS: 1726384726] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 (Updated: 2024-09-15T07:18:46) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 (Updated: 2024-09-16T07:19:43 [TS: 1726471183] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 (Updated: 2024-09-16T07:19:43) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 (Updated: 2024-09-17T07:18:21 [TS: 1726557501] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 (Updated: 2024-09-17T07:18:21) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af (Updated: 2024-09-18T07:18:46 [TS: 1726643926] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af (Updated: 2024-09-18T07:18:46) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 (Updated: 2024-09-19T07:19:58 [TS: 1726730398] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 (Updated: 2024-09-19T07:19:58) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c (Updated: 2024-09-20T07:19:34 [TS: 1726816774] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c (Updated: 2024-09-20T07:19:34) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 (Updated: 2024-09-21T07:18:42 [TS: 1726903122] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 (Updated: 2024-09-21T07:18:42) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f (Updated: 2024-09-22T07:19:13 [TS: 1726989553] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f (Updated: 2024-09-22T07:19:13) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 (Updated: 2024-09-23T07:19:27 [TS: 1727075967] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 (Updated: 2024-09-23T07:19:27) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a (Updated: 2024-09-24T07:19:03 [TS: 1727162343] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a (Updated: 2024-09-24T07:19:03) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 (Updated: 2024-09-25T07:19:26 [TS: 1727248766] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 (Updated: 2024-09-25T07:19:26) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 (Updated: 2024-09-26T07:18:34 [TS: 1727335114] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 (Updated: 2024-09-26T07:18:34) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 (Updated: 2024-09-27T07:18:51 [TS: 1727421531] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 (Updated: 2024-09-27T07:18:51) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 (Updated: 2024-09-28T07:19:08 [TS: 1727507948] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 (Updated: 2024-09-28T07:19:08) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b (Updated: 2024-09-29T07:19:59 [TS: 1727594399] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b (Updated: 2024-09-29T07:19:59) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 (Updated: 2024-09-30T07:18:47 [TS: 1727680727] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 (Updated: 2024-09-30T07:18:47) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 (Updated: 2024-10-01T07:18:16 [TS: 1727767096] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 (Updated: 2024-10-01T07:18:16) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 (Updated: 2024-10-02T07:19:22 [TS: 1727853562] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 (Updated: 2024-10-02T07:19:22) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d (Updated: 2024-10-03T07:19:40 [TS: 1727939980] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d (Updated: 2024-10-03T07:19:40) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c (Updated: 2024-10-04T07:20:00 [TS: 1728026400] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c (Updated: 2024-10-04T07:20:00) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 (Updated: 2024-10-05T07:20:18 [TS: 1728112818] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 (Updated: 2024-10-05T07:20:18) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d (Updated: 2024-10-06T07:19:45 [TS: 1728199185] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d (Updated: 2024-10-06T07:19:45) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 (Updated: 2024-10-07T07:20:12 [TS: 1728285612] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 (Updated: 2024-10-07T07:20:12) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 (Updated: 2024-10-08T07:19:06 [TS: 1728371946] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 (Updated: 2024-10-08T07:19:06) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 (Updated: 2024-10-09T07:19:20 [TS: 1728458360] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 (Updated: 2024-10-09T07:19:20) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e (Updated: 2024-10-10T07:19:58 [TS: 1728544798] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e (Updated: 2024-10-10T07:19:58) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf (Updated: 2024-10-11T07:19:31 [TS: 1728631171] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf (Updated: 2024-10-11T07:19:31) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 (Updated: 2024-10-12T07:19:51 [TS: 1728717591] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 (Updated: 2024-10-12T07:19:51) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 (Updated: 2024-10-13T07:20:18 [TS: 1728804018] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 (Updated: 2024-10-13T07:20:18) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 (Updated: 2024-10-14T07:18:56 [TS: 1728890336] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 (Updated: 2024-10-14T07:18:56) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 (Updated: 2024-10-15T07:18:42 [TS: 1728976722] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 (Updated: 2024-10-15T07:18:42) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 (Updated: 2024-10-16T07:18:53 [TS: 1729063133] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 (Updated: 2024-10-16T07:18:53) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 (Updated: 2024-10-17T07:19:38 [TS: 1729149578] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 (Updated: 2024-10-17T07:19:38) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 (Updated: 2024-10-18T07:21:27 [TS: 1729236087] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 (Updated: 2024-10-18T07:21:27) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 (Updated: 2024-10-19T07:21:08 [TS: 1729322468] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 (Updated: 2024-10-19T07:21:08) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 (Updated: 2024-10-20T07:20:13 [TS: 1729408813] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 (Updated: 2024-10-20T07:20:13) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 (Updated: 2024-10-21T07:19:27 [TS: 1729495167] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 (Updated: 2024-10-21T07:19:27) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 (Updated: 2024-10-22T07:19:36 [TS: 1729581576] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 (Updated: 2024-10-22T07:19:36) -[2025-11-30 14:54:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 (Updated: 2024-10-23T07:18:47 [TS: 1729667927] < Cutoff: [TS: 1763304862]) -[2025-11-30 14:54:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 (Updated: 2024-10-23T07:18:47) -[2025-11-30 14:54:31] [INFO] Hit delete limit (200) for Docker Images. -[2025-11-30 14:54:31] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 14:54:31] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 14:54:34] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 14:54:34] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 14:54:34] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 14:54:34] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 14:54:34] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 14:54:34] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 14:54:36] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 14:54:36] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 14:54:38] [INFO] No Regional Address found matching criteria. -[2025-11-30 14:54:38] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 14:54:41] [INFO] No Global Address found matching criteria. -[2025-11-30 14:54:41] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- -[2025-11-30 14:54:46] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 14:54:48] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 14:54:48] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 14:54:48] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 14:54:48] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 14:54:48] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:51] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 14:54:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 14:54:53] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 14:54:55] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 14:54:55] [INFO] CLEANUP RUN FINISHED -[2025-11-30 14:54:59] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 14:54:59] [INFO] Time Cutoff (General): 2025-11-30T14:54:59+0000 -[2025-11-30 14:54:59] [INFO] Time Cutoff (Images): 2025-10-01T14:54:59+0000 -[2025-11-30 14:54:59] [INFO] Delete Limit per Type: 200 -[2025-11-30 14:54:59] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 14:54:59] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 14:55:01] [INFO] No Service Accounts found matching prefix. -[2025-11-30 14:55:01] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 14:55:03] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 14:55:03] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 14:55:05] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 14:55:05] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 14:55:05] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 14:55:05] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 14:55:08] [INFO] No Filestore instances found matching criteria. -[2025-11-30 14:55:08] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 14:55:11] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 14:55:11] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 14:55:11] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 14:55:11] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 14:55:11] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 14:55:11] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 14:55:11] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 14:55:11] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 14:55:12] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 14:55:12] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 14:55:12] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 14:55:12] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T14:55:12Z (Unix: 1763304912) -[2025-11-30 14:55:12] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 14:55:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19 [TS: 1713165559] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 (Updated: 2024-04-15T07:19:19) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fdbb17db-84a0-49ed-9453-cadd0b3aaf04] to complete... -.....done. -[2025-11-30 14:55:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:32f07d0c00c71484d01e294554ee5d8cc43d4b7a2d3bcbf65a7814ee071ea255 -[2025-11-30 14:55:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17 [TS: 1713251897] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe (Updated: 2024-04-16T07:18:17) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2d2b4fd8-6c71-407a-829c-7edb3611ef06] to complete... -.....done. -[2025-11-30 14:55:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a2b119988f2ebab32a437888971d1f091c1f0a902b2ec0e759faaa39ed2abe -[2025-11-30 14:55:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57 [TS: 1713338277] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 (Updated: 2024-04-17T07:17:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7ee525c9-c9b8-4493-b95d-f1f1e5c7c2bb] to complete... -.....done. -[2025-11-30 14:55:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a12cee6766e9efc981b6aa88d2bb055ca346cfe1ae7bf1223ec378d1ef8ae68 -[2025-11-30 14:55:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08 [TS: 1713424748] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 (Updated: 2024-04-18T07:19:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b0c80bb2-8d1e-4a80-bb56-1215d9f124d3] to complete... -......done. -[2025-11-30 14:55:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aee50c2857d3700cfb31666ca7afaa5a01b84668a2b74a494a5f7d6cd8178e88 -[2025-11-30 14:55:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26 [TS: 1713511166] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a (Updated: 2024-04-19T07:19:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b441399d-cdd1-4b57-8bf2-0bca58eecec8] to complete... -.....done. -[2025-11-30 14:55:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b77ef32b0dd026616c62c4d804b1b02fcd15e328c2c78b97a6ded213108e7a -[2025-11-30 14:55:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36 [TS: 1713597516] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c (Updated: 2024-04-20T07:18:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0bdff779-3814-4f6a-8490-2e949eb47612] to complete... -.....done. -[2025-11-30 14:55:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f158d44901eee9fbb68c48ddfdcdd2da506359d6ad83561d0585bd5af52783c -[2025-11-30 14:55:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57 [TS: 1713683937] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 (Updated: 2024-04-21T07:18:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/449226a3-aa7f-4d94-a7de-ea3f23785646] to complete... -.....done. -[2025-11-30 14:55:41] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:88624527047b2790379c3956b62af5428507661244a4c703acecc26813124c53 -[2025-11-30 14:55:41] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38 [TS: 1713770258] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:41] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 (Updated: 2024-04-22T07:17:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/14fe9a67-bc2e-41b6-a946-818a8e3531ab] to complete... -.....done. -[2025-11-30 14:55:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:66667a3fd2ff70c92196eaea87f44c6c4aaf3d25df6fe8212452d117ba002412 -[2025-11-30 14:55:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51 [TS: 1713856671] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 (Updated: 2024-04-23T07:17:51) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/efa195f6-a784-4348-baa2-98a69a2c09df] to complete... -.....done. -[2025-11-30 14:55:48] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2453f39b7f7c684137812ac68a65c3f137b766178975e1aab8be5ec5b1d238a4 -[2025-11-30 14:55:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09 [TS: 1713943149] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa (Updated: 2024-04-24T07:19:09) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7d8906eb-4fb1-4577-ae4d-0dafce979c00] to complete... -......done. -[2025-11-30 14:55:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a499e29116fc7d7f7eddc55167f3a74d387e00e400708bc6c8a00309c1546aa -[2025-11-30 14:55:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19 [TS: 1714029499] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 (Updated: 2024-04-25T07:18:19) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d11827a6-b7c1-4553-ab33-46c4272da5ff] to complete... -.....done. -[2025-11-30 14:55:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fc292b38e4824faf577ddef1c7df5d3823866be22a2701e67b7ed3c4619391e9 -[2025-11-30 14:55:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26 [TS: 1714115966] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d (Updated: 2024-04-26T07:19:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e6dda756-3406-4e4d-8492-317c2770f9db] to complete... -.....done. -[2025-11-30 14:55:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8ae853c026d11df524d3ee8cee51c858b4c6f5ef62a75ce1f963e418c1b282d -[2025-11-30 14:55:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09 [TS: 1714202349] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:55:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 (Updated: 2024-04-27T07:19:09) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f72967bd-f7ee-4098-b988-c9f3d6333b6b] to complete... -.....done. -[2025-11-30 14:56:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:06826aa75f404c909c8c17ed0d0263ca6d644641afe31f4c0e24cd9c940ba820 -[2025-11-30 14:56:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58 [TS: 1714288738] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 (Updated: 2024-04-28T07:18:58) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ea5a78e6-38b7-470e-b38c-5610eaf6520f] to complete... -.....done. -[2025-11-30 14:56:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d3ee4abc444c1e984165d941ef4f71a2c937583cff1eb011ba4d1fcb9b1e4b1 -[2025-11-30 14:56:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09 [TS: 1714375269] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 (Updated: 2024-04-29T07:21:09) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/24d36492-952c-42d8-9810-0f7f88cf7d86] to complete... -.....done. -[2025-11-30 14:56:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e54f6382a08f34f9c942b46dd64ebbe3bd61422e844d5702e72526fa08794308 -[2025-11-30 14:56:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18 [TS: 1714461498] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f (Updated: 2024-04-30T07:18:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/84a8eedc-4865-4f31-8f62-70a3c990ed19] to complete... -.....done. -[2025-11-30 14:56:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87adce64b22862ed306b87af122257651a7a1b10ca412bb559a7e5194c45b89f -[2025-11-30 14:56:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48 [TS: 1714547868] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 (Updated: 2024-05-01T07:17:48) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1b9dd06a-d5d4-4624-bc92-a99b2a1af57f] to complete... -......done. -[2025-11-30 14:56:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0ff12d9aac108ecb0eca915bcecd1986c83f0a1fad7305db7b74c405941d6e5 -[2025-11-30 14:56:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44 [TS: 1714634264] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 (Updated: 2024-05-02T07:17:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/98f7868d-eab0-4b02-b940-4b93d19051d3] to complete... -.....done. -[2025-11-30 14:56:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14d0dfb8073056503bf040d97dd80c4238c10acc8e0e09043981a524f8da0070 -[2025-11-30 14:56:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43 [TS: 1714720723] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc (Updated: 2024-05-03T07:18:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/21632ad0-cd26-42a5-a3e9-6fba7fcbfbfd] to complete... -.....done. -[2025-11-30 14:56:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:62df260a21281c10c98cdc8e022c0fd8668ffe9cc85f169e577e9de125b754bc -[2025-11-30 14:56:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00 [TS: 1714807080] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 (Updated: 2024-05-04T07:18:00) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0ae5fb9e-54ad-4286-a6de-e6c460711756] to complete... -.....done. -[2025-11-30 14:56:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca35962505ac40e5528f76500034e952eba24e419093abea606fdac569bcf2e4 -[2025-11-30 14:56:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 (Updated: 2024-05-05T07:19:28 [TS: 1714893568] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 (Updated: 2024-05-05T07:19:28) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d1f73bc7-1c11-4029-bf03-65bbeeddefd3] to complete... -......done. -[2025-11-30 14:56:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e167a9327b7a73de2120ca9425389c761bc50b54efdbe8bb5a0bed9a17487005 -[2025-11-30 14:56:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 (Updated: 2024-05-06T07:18:08 [TS: 1714979888] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 (Updated: 2024-05-06T07:18:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e0a71f50-88e2-4fc4-a6fb-ae06270f00bd] to complete... -.....done. -[2025-11-30 14:56:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aab0617a48136f405503f4017a67900d30034ae874b4a65b65505d2f17d4fbc4 -[2025-11-30 14:56:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 (Updated: 2024-05-07T07:20:44 [TS: 1715066444] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 (Updated: 2024-05-07T07:20:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0ad96f72-aaa4-43bc-9902-fb836fd35d1c] to complete... -.....done. -[2025-11-30 14:56:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b72e84f82040d97e01dda834701af2f13b35bcdc48af5c825af03210c0ab7526 -[2025-11-30 14:56:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 (Updated: 2024-05-08T07:18:32 [TS: 1715152712] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 (Updated: 2024-05-08T07:18:32) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f7e45e9f-22d9-47a9-80f6-1583f80508c7] to complete... -.....done. -[2025-11-30 14:56:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1abc2db7146087c789cc4b9d8452c5874de6c0ec2fa96607e933f5b531a8f4f9 -[2025-11-30 14:56:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b (Updated: 2024-05-09T07:19:05 [TS: 1715239145] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b (Updated: 2024-05-09T07:19:05) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/693edaf6-e9bd-4e9b-9145-d1c04e5e617d] to complete... -.....done. -[2025-11-30 14:56:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a6b12ae207df6d5f1fa2b29e393866ccbdbd13675795e899555eece10748dd0b -[2025-11-30 14:56:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd (Updated: 2024-05-10T07:18:32 [TS: 1715325512] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd (Updated: 2024-05-10T07:18:32) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/738386a2-1ca0-453e-9e37-8f155596602f] to complete... -.....done. -[2025-11-30 14:56:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8bf27f9346431601c4800fc14d005460f7dc4a41f29589e4ae73b2913013c1dd -[2025-11-30 14:56:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 (Updated: 2024-05-11T07:18:53 [TS: 1715411933] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 (Updated: 2024-05-11T07:18:53) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8fb12ac3-485d-4e13-a57e-5af50568d261] to complete... -.....done. -[2025-11-30 14:56:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15620a09bb730d74ccd4da2dd6c7e792e665c506dc713568b7b9a5e46ba6a404 -[2025-11-30 14:56:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 (Updated: 2024-05-12T07:18:59 [TS: 1715498339] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 (Updated: 2024-05-12T07:18:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cd0a0897-12d1-4736-85ff-f55e141335ad] to complete... -.....done. -[2025-11-30 14:56:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7a01cb8729a2e1f014884a792f745904ef6b7393b362dd64fa057ebaf230af61 -[2025-11-30 14:56:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 (Updated: 2024-05-13T07:18:49 [TS: 1715584729] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 (Updated: 2024-05-13T07:18:49) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1a3c05ea-c11f-40a3-b4f5-1c1e7258e925] to complete... -......done. -[2025-11-30 14:56:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe7983956a6751f054826e7e72bce785a7392f08a54f591461c7843110749095 -[2025-11-30 14:56:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc (Updated: 2024-05-14T07:16:18 [TS: 1715670978] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:56:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc (Updated: 2024-05-14T07:16:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/64a36cbc-6e34-4ae1-bc80-36778391f068] to complete... -.....done. -[2025-11-30 14:57:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0510cb27b44ed45d5cc68580ec2380e409f8d3a6b13755be692925a36c560ecc -[2025-11-30 14:57:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 (Updated: 2024-05-15T07:18:59 [TS: 1715757539] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 (Updated: 2024-05-15T07:18:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4bcae04e-a220-4eb3-a02a-5222ac476c4c] to complete... -......done. -[2025-11-30 14:57:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c5fe6d5b6c24b28e9cf08291574dd2549346f56b2f9f80491a099a2a85733989 -[2025-11-30 14:57:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 (Updated: 2024-05-16T07:18:34 [TS: 1715843914] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 (Updated: 2024-05-16T07:18:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c5a5a145-44ff-478a-9e94-661c9b038c00] to complete... -.....done. -[2025-11-30 14:57:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:854b18298de08fb774d3c5f8255c499d502c1e5f4ffce85a1085c8d787dc6640 -[2025-11-30 14:57:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 (Updated: 2024-05-17T07:18:04 [TS: 1715930284] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 (Updated: 2024-05-17T07:18:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bd291f25-af70-41ab-9459-01c5aea175d0] to complete... -......done. -[2025-11-30 14:57:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca223e406eb99bb69d9af3cb127a0295f47a27ea36341b50d1c429430d76ead7 -[2025-11-30 14:57:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 (Updated: 2024-05-18T07:18:46 [TS: 1716016726] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 (Updated: 2024-05-18T07:18:46) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/47bcc8e7-de77-4438-8731-dcf6d5c3c0ad] to complete... -.....done. -[2025-11-30 14:57:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:82db6159f2cb3b3a1700936cb32ee16d5e2f294546474bfac5e706e06c0a5a55 -[2025-11-30 14:57:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd (Updated: 2024-05-19T07:19:20 [TS: 1716103160] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd (Updated: 2024-05-19T07:19:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c304f58d-7edd-46ee-b8f0-ad1e0d4913e3] to complete... -.....done. -[2025-11-30 14:57:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:77b5fdeae80952a64b5de463373d61500083db1467b016b36d0f953baaa1d3cd -[2025-11-30 14:57:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c (Updated: 2024-05-20T07:18:54 [TS: 1716189534] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c (Updated: 2024-05-20T07:18:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b5cb8a87-1cc4-4ce3-ad79-aecd60172fc3] to complete... -.....done. -[2025-11-30 14:57:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f1ff2737ad3212425c64774f69fb447158cee3df2de7b36003472a817de1d5c -[2025-11-30 14:57:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce (Updated: 2024-05-21T07:18:14 [TS: 1716275894] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce (Updated: 2024-05-21T07:18:14) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4d34d297-f956-4914-ac69-ea66b91af928] to complete... -......done. -[2025-11-30 14:57:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71e7cf20289a883bba8522d059fb51c490cc3406942500a5dbaaf2fe35e481ce -[2025-11-30 14:57:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e (Updated: 2024-05-22T07:17:56 [TS: 1716362276] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e (Updated: 2024-05-22T07:17:56) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2bf45745-42cf-4fe6-843c-f8f703bb57d3] to complete... -.....done. -[2025-11-30 14:57:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8cb81a0f7717d31d9a07f2f1eceacdc679cbc74cfd54d06175bd9567b0426c0e -[2025-11-30 14:57:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 (Updated: 2024-05-23T07:17:57 [TS: 1716448677] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 (Updated: 2024-05-23T07:17:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b4f7a340-1e92-4b07-a3c9-5a5ea0d7945b] to complete... -.....done. -[2025-11-30 14:57:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:134e93edb5de5eebe502914055f40c59d98496277d51c115f4a3c2954bb0d427 -[2025-11-30 14:57:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 (Updated: 2024-05-24T07:17:47 [TS: 1716535067] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 (Updated: 2024-05-24T07:17:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6d7d8367-1dbc-4864-82d7-56b8ba4e1b10] to complete... -.....done. -[2025-11-30 14:57:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2f42ef4fa291e2d5bf14fc758d7f2c6b299410490cbb4cd1cc88637404bb5f7 -[2025-11-30 14:57:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 (Updated: 2024-05-25T07:18:10 [TS: 1716621490] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 (Updated: 2024-05-25T07:18:10) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e6251c19-2355-4c03-9f53-8d44e2ace8ec] to complete... -.....done. -[2025-11-30 14:57:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3f3cab0200355d7edfa4f8c8abc2ed3475843dd8faf619aa7b9e3362a84daeb8 -[2025-11-30 14:57:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 (Updated: 2024-05-26T07:18:34 [TS: 1716707914] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 (Updated: 2024-05-26T07:18:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/337a0709-ddfd-4099-bea4-b0486f84ec10] to complete... -.....done. -[2025-11-30 14:57:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cab686f31bc57aa7094e731ed669488a5c71f572e637c98dfd22f87f677b4e02 -[2025-11-30 14:57:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 (Updated: 2024-05-27T07:18:35 [TS: 1716794315] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 (Updated: 2024-05-27T07:18:35) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/059e6c66-2752-4bd9-b345-fe672fd0684a] to complete... -.....done. -[2025-11-30 14:57:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0887db60c2be722a67742df672fa5a3612ab9a91135547f70198503b051adaa7 -[2025-11-30 14:57:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf (Updated: 2024-05-28T07:18:07 [TS: 1716880687] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf (Updated: 2024-05-28T07:18:07) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d828038b-88dd-443e-bc61-89b320887c02] to complete... -.....done. -[2025-11-30 14:57:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3294b7f5c49c2f1d2b414bdab8b26f7a08e2f847006206b4139ccbb30b055cf -[2025-11-30 14:57:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 (Updated: 2024-05-29T07:17:33 [TS: 1716967053] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 (Updated: 2024-05-29T07:17:33) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a836a188-4f4b-4e0f-9ee6-ca2942059814] to complete... -.....done. -[2025-11-30 14:57:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a53fcf14f80679e5c2489ab789d7b98cb571330c13c4b38ef6f72c2836443fc1 -[2025-11-30 14:57:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d (Updated: 2024-05-30T07:18:38 [TS: 1717053518] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d (Updated: 2024-05-30T07:18:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/26e66eb0-0e4d-4fe6-aa1f-a029ef6de4f6] to complete... -.....done. -[2025-11-30 14:57:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b78d56871e3599d8da3c5a65d41da22b5152e4dcdcc3b4f2834f805266a8f29d -[2025-11-30 14:57:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 (Updated: 2024-05-31T07:18:40 [TS: 1717139920] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:57:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 (Updated: 2024-05-31T07:18:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5dd2f14a-2266-4d15-88d6-d7f5c655d534] to complete... -.....done. -[2025-11-30 14:58:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74795226448cd2cd2e8992cebb4a1e236b1b5ce93326d825125b6f22e68b21c0 -[2025-11-30 14:58:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de (Updated: 2024-06-01T07:18:20 [TS: 1717226300] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de (Updated: 2024-06-01T07:18:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d2eca114-25ee-431b-a6d9-1d98a0f30c67] to complete... -.....done. -[2025-11-30 14:58:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be3f424284a20afc8fb79bf400b9aba51625f9db1c7276492807c4ab5d3620de -[2025-11-30 14:58:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb (Updated: 2024-06-02T07:19:05 [TS: 1717312745] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb (Updated: 2024-06-02T07:19:05) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9531cd8d-4b21-4ba8-941a-387f1e9bc71b] to complete... -......done. -[2025-11-30 14:58:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40666c936d0fec19d634a73b1dde008997cd9ce6035b6707cdd0dda42cf18feb -[2025-11-30 14:58:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 (Updated: 2024-06-03T07:18:42 [TS: 1717399122] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 (Updated: 2024-06-03T07:18:42) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7b029aeb-523b-48ee-bda2-a01354e1587c] to complete... -.....done. -[2025-11-30 14:58:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba64de9c9aacbf76f7754e6b66e5c561d864b8967d274ff468eca2907b0c69a2 -[2025-11-30 14:58:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 (Updated: 2024-06-04T07:18:28 [TS: 1717485508] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 (Updated: 2024-06-04T07:18:28) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/68b265d3-002a-4cfa-ba1f-1fe40ff6372d] to complete... -.....done. -[2025-11-30 14:58:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b8c78d570e994d31508ccd8591b1830fa4194c4ef671097517b7eedb86c011 -[2025-11-30 14:58:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 (Updated: 2024-06-05T07:18:24 [TS: 1717571904] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 (Updated: 2024-06-05T07:18:24) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/594858ff-d5ee-4e5f-91b5-7477b9a84612] to complete... -......done. -[2025-11-30 14:58:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff1a9dd3d0527a1e13efe0eeaa83dc2299f525d45ad9c54c01367c51b6979f16 -[2025-11-30 14:58:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 (Updated: 2024-06-06T07:20:38 [TS: 1717658438] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 (Updated: 2024-06-06T07:20:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e59d0392-8c2f-47ef-bbe1-4d4058bc13c1] to complete... -......done. -[2025-11-30 14:58:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae12ec508e1623fb607c3cda08398a5e4d14d456fca2f9d8173b25ca7f73e74 -[2025-11-30 14:58:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d (Updated: 2024-06-07T07:18:27 [TS: 1717744707] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d (Updated: 2024-06-07T07:18:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4dbdd5be-aa1a-43d3-ad6d-6fac43de8a74] to complete... -.....done. -[2025-11-30 14:58:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1aac49512fdb239616b5d734f499b88d30a42ddaac320943bb6629e3a863a8d -[2025-11-30 14:58:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 (Updated: 2024-06-08T07:18:28 [TS: 1717831108] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 (Updated: 2024-06-08T07:18:28) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c6a10fb9-3a55-483c-9ab7-556491107ad7] to complete... -.....done. -[2025-11-30 14:58:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c874fdada9ea57df6177a295b387571ffa171052da955abdb3fa7187d7fade0 -[2025-11-30 14:58:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e (Updated: 2024-06-09T07:18:23 [TS: 1717917503] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e (Updated: 2024-06-09T07:18:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/14253a47-a556-4848-8219-50d3a4d484bc] to complete... -.....done. -[2025-11-30 14:58:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:538f1a3c4fb2ff72098fddda3fa8ec84a2ecd1b6aead61c94b457a1409d70e7e -[2025-11-30 14:58:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 (Updated: 2024-06-10T07:18:11 [TS: 1718003891] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 (Updated: 2024-06-10T07:18:11) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/270588a0-8ce3-4ae6-abc6-8e725f199342] to complete... -.....done. -[2025-11-30 14:58:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d3e65eb1cfb187b9ad00043366b35a87886f3c618016f5349834b6a7b6bcee6 -[2025-11-30 14:58:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 (Updated: 2024-06-11T07:18:43 [TS: 1718090323] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 (Updated: 2024-06-11T07:18:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2501e5d5-5251-4ce3-8e93-6ffb13530929] to complete... -.....done. -[2025-11-30 14:58:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca6dbc1e4090e6e924126b594c0bc465e23532e403db78fbf5475e16c7339f02 -[2025-11-30 14:58:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 (Updated: 2024-06-12T07:18:47 [TS: 1718176727] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 (Updated: 2024-06-12T07:18:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/470e51e9-0439-424c-9883-a754214b5e31] to complete... -.....done. -[2025-11-30 14:58:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:993e3bf037536c758f740b9f5e428b31e7d648b8d0387e2b209a45841931c651 -[2025-11-30 14:58:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a (Updated: 2024-06-13T07:19:10 [TS: 1718263150] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a (Updated: 2024-06-13T07:19:10) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/27e6ec1a-1642-48ee-b681-ab4ac5703c7b] to complete... -......done. -[2025-11-30 14:58:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a00363f9df85674d8a921b8434c33145f48cf7f22aeb0974eb5e1ba5a3707e8a -[2025-11-30 14:58:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c (Updated: 2024-06-14T07:18:33 [TS: 1718349513] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c (Updated: 2024-06-14T07:18:33) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4a7234b8-39c8-482e-bedf-fd22ebeed203] to complete... -.....done. -[2025-11-30 14:58:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29aed2697dd516e7d5f595219a12decfb9179b13fa7998fac7bb97111a3ce36c -[2025-11-30 14:58:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 (Updated: 2024-06-15T07:19:01 [TS: 1718435941] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 (Updated: 2024-06-15T07:19:01) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/93ad0cc5-8044-4645-81c2-006f70eebd8f] to complete... -.....done. -[2025-11-30 14:58:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed5a5f5ebd4858eafbc488a696594c38579e5763e135cccabd5fd0f2c1b64163 -[2025-11-30 14:58:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 (Updated: 2024-06-16T07:18:57 [TS: 1718522337] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 (Updated: 2024-06-16T07:18:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f7d00588-e25e-447d-812b-1c84db008dfc] to complete... -.....done. -[2025-11-30 14:58:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bae9c2baf536d539b587790c446156f915de80a402d16c3f6389e42b9b58d664 -[2025-11-30 14:58:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 (Updated: 2024-06-17T07:17:57 [TS: 1718608677] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:58:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 (Updated: 2024-06-17T07:17:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a45744ad-e6a2-4598-994c-00a5c8d3256d] to complete... -.....done. -[2025-11-30 14:59:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:093b605f0c65c93d1644301c290e48a006b5a1ba7cd4c4a9add3c745367739e9 -[2025-11-30 14:59:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 (Updated: 2024-06-18T07:18:50 [TS: 1718695130] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 (Updated: 2024-06-18T07:18:50) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/84311460-80c7-498f-90af-2f124687686f] to complete... -.....done. -[2025-11-30 14:59:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb0774b24842b0b042413bfd1f8072b16e3a53feff6f782842d7eddba12968b0 -[2025-11-30 14:59:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 (Updated: 2024-06-19T07:18:24 [TS: 1718781504] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 (Updated: 2024-06-19T07:18:24) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1f08f0cb-11c0-4bd6-9228-348bbc230dfd] to complete... -......done. -[2025-11-30 14:59:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bb39a6a61b891c883fea093663cb54cd736476a929fa47a28693bf3d15569876 -[2025-11-30 14:59:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c (Updated: 2024-06-20T07:18:11 [TS: 1718867891] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c (Updated: 2024-06-20T07:18:11) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/da006e24-8cb1-4d03-a9ca-ffc3930feccc] to complete... -.....done. -[2025-11-30 14:59:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:325ad57acc69d6371ccad8578fca3afaa3658ceae3080f8360eeced5d895740c -[2025-11-30 14:59:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 (Updated: 2024-06-20T17:43:06 [TS: 1718905386] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 (Updated: 2024-06-20T17:43:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f09b92f2-2256-43bc-843a-1e945ee40e30] to complete... -.....done. -[2025-11-30 14:59:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6032041f2d7c4f544d808f836172c0055711193701026bb434e198134b52c5d2 -[2025-11-30 14:59:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a (Updated: 2024-06-21T07:20:29 [TS: 1718954429] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a (Updated: 2024-06-21T07:20:29) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7520ce0d-9cfc-4cf8-ad86-a36ad8558993] to complete... -.....done. -[2025-11-30 14:59:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e05294ae0845bbd6b0ca962a085e11bdbeab428ccb7669f9a92dbcdf717d71a -[2025-11-30 14:59:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 (Updated: 2024-06-21T18:43:44 [TS: 1718995424] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 (Updated: 2024-06-21T18:43:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7fadf1ca-eb50-4d40-a00e-d950f4107705] to complete... -.....done. -[2025-11-30 14:59:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a8f82a09fb1d7a579e9d662fb8c227751be0bfb0fd15de31b8ab952dcb538400 -[2025-11-30 14:59:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 (Updated: 2024-06-21T20:07:06 [TS: 1719000426] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 (Updated: 2024-06-21T20:07:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f017fafc-2960-4fd1-91e1-ff5cbad072a8] to complete... -.....done. -[2025-11-30 14:59:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b889638fcb421da619c2e12f2463f1e73662beb12f6ed0593b611aeedc14e648 -[2025-11-30 14:59:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db (Updated: 2024-06-21T22:45:30 [TS: 1719009930] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db (Updated: 2024-06-21T22:45:30) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e6190cac-b89a-4ef9-8135-735f882af9fa] to complete... -.....done. -[2025-11-30 14:59:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e328457e4f9e57fccd866a3f83704da13355ab5208f182e1609d94571ab07db -[2025-11-30 14:59:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 (Updated: 2024-06-21T23:36:46 [TS: 1719013006] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 (Updated: 2024-06-21T23:36:46) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8133e7f1-63fe-4775-96c3-e0206cd87636] to complete... -.....done. -[2025-11-30 14:59:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1806ab7e65f25be5fca9ba51ee051b50645c4aa6a99e5e9ac03f1f22d9791088 -[2025-11-30 14:59:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e (Updated: 2024-06-22T07:18:01 [TS: 1719040681] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e (Updated: 2024-06-22T07:18:01) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/597bc3da-70bb-4739-93e1-c95cec863e5a] to complete... -.....done. -[2025-11-30 14:59:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1cd8e2cca5cbb929d8316225b124ac3c21149885644b836635458e5444d1fd5e -[2025-11-30 14:59:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 (Updated: 2024-06-23T07:18:54 [TS: 1719127134] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 (Updated: 2024-06-23T07:18:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/02545905-5c63-4994-9eb7-d5107a6e954d] to complete... -.....done. -[2025-11-30 14:59:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dfe17cb1d471890a165d9d2c159abbae7d75117e1f7ed2b264e8eb1dcbbfa71 -[2025-11-30 14:59:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 (Updated: 2024-06-24T07:18:00 [TS: 1719213480] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 (Updated: 2024-06-24T07:18:00) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/190c0e84-9d75-4665-9aa2-51dda9cd1c18] to complete... -.....done. -[2025-11-30 14:59:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c35ddcd84b0fe2d75329b75f98388f1f64543827238bb1ce4c6cd14fb967bd7 -[2025-11-30 14:59:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e (Updated: 2024-06-24T16:40:01 [TS: 1719247201] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e (Updated: 2024-06-24T16:40:01) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/43247d19-97ea-490d-8c24-83fdacbbc804] to complete... -......done. -[2025-11-30 14:59:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1578233fbde82e50a05f8aaaa9043837d01c5d61cf145a073dbb1462eb59b75e -[2025-11-30 14:59:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 (Updated: 2024-06-24T17:31:54 [TS: 1719250314] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 (Updated: 2024-06-24T17:31:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 - -Tags: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner:test-kubectl -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8b9bc5ef-47de-4afd-ac6e-7e7bcfdcacd8] to complete... -.....done. -[2025-11-30 14:59:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3627745c28bf19f028906592c17478682909b422e79a5de89bbe9b5322136ea5 -[2025-11-30 14:59:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 (Updated: 2024-06-25T07:19:35 [TS: 1719299975] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 (Updated: 2024-06-25T07:19:35) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/01c9ac72-0340-4e85-b263-9f7fff326a4a] to complete... -.....done. -[2025-11-30 14:59:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f19f2e26c7c73005c3e544e2785465cf84302a05455c6fbc8c1e9926c64795c4 -[2025-11-30 14:59:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 (Updated: 2024-06-26T07:19:18 [TS: 1719386358] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 (Updated: 2024-06-26T07:19:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b0cd5be1-27f8-4aec-8bff-aefbef5e5305] to complete... -......done. -[2025-11-30 14:59:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a780302798c73d9c1c6ae11f3eef88bfa72d33f14309ceb0f8cadcbf67d101a7 -[2025-11-30 14:59:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf (Updated: 2024-06-27T07:18:30 [TS: 1719472710] < Cutoff: [TS: 1763304912]) -[2025-11-30 14:59:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf (Updated: 2024-06-27T07:18:30) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f8da73d3-2020-4533-b249-87309c114809] to complete... -.....done. -[2025-11-30 15:00:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2b527f5c35b5d45057ad459ac1eef056b6b4c0e667739f0539518c7afa163fcf -[2025-11-30 15:00:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b (Updated: 2024-06-28T07:18:41 [TS: 1719559121] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b (Updated: 2024-06-28T07:18:41) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/69217e24-d9de-4c5a-94e3-13893feed995] to complete... -.....done. -[2025-11-30 15:00:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce5f938315a6f39f2535e28f3eb430451ed2b00774bc21441296125ec5a35e3b -[2025-11-30 15:00:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de (Updated: 2024-06-29T07:18:57 [TS: 1719645537] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de (Updated: 2024-06-29T07:18:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f61199da-077b-4461-ad81-b2e2962129fb] to complete... -.....done. -[2025-11-30 15:00:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:002b4df02ffd6901261432eb1202b84b5b40d63a996dda6c45a08ea970e111de -[2025-11-30 15:00:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 (Updated: 2024-06-30T07:20:51 [TS: 1719732051] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 (Updated: 2024-06-30T07:20:51) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4f95586a-e372-4383-96c9-387dd4383757] to complete... -......done. -[2025-11-30 15:00:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bdd65144a96a67b04e082466570afdfe0e649ee26d3202b772b5290cc74858d4 -[2025-11-30 15:00:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 (Updated: 2024-07-01T07:18:08 [TS: 1719818288] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 (Updated: 2024-07-01T07:18:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bf9bd343-1029-416c-8c26-8abe239bb73d] to complete... -.....done. -[2025-11-30 15:00:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b75a545f43dd45362173d36c829ab9ba322cc9a8f95acff0df18cdee0bd47ae3 -[2025-11-30 15:00:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 (Updated: 2024-07-02T07:18:47 [TS: 1719904727] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 (Updated: 2024-07-02T07:18:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2ec2483e-a897-4b85-8144-32d8c84c52fe] to complete... -.....done. -[2025-11-30 15:00:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d80502779c8aa4445a9e65d4a88d68238b975b9cd03ea6d22b6432e626f3f28 -[2025-11-30 15:00:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 (Updated: 2024-07-03T07:20:11 [TS: 1719991211] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 (Updated: 2024-07-03T07:20:11) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/35bf6fa9-e13f-4af1-b9a8-5b5128a99114] to complete... -.....done. -[2025-11-30 15:00:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2041615e2929c2f6dfe025f377904f4a17e437c65b4361269eb9fa73b5b5e468 -[2025-11-30 15:00:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f (Updated: 2024-07-04T07:18:39 [TS: 1720077519] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f (Updated: 2024-07-04T07:18:39) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/811fbec7-dc28-4309-a838-8c07ef3cf73b] to complete... -.....done. -[2025-11-30 15:00:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dc4f1956154ac7511c8b93282556175f2a05577208f5214f7c0e2792b62d52f -[2025-11-30 15:00:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 (Updated: 2024-07-05T07:18:56 [TS: 1720163936] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 (Updated: 2024-07-05T07:18:56) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/53fe5206-b951-44de-80b5-476383806bad] to complete... -.....done. -[2025-11-30 15:00:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4eb44a6f917bf10a724c82297437903e61ad138fae5da502cc2584fa4db58b63 -[2025-11-30 15:00:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 (Updated: 2024-07-06T07:16:34 [TS: 1720250194] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 (Updated: 2024-07-06T07:16:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/17b471f7-20d3-423e-a7de-8a657f304861] to complete... -.....done. -[2025-11-30 15:00:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:170af85946e3a0306fcdbd74c3c0cde5c3c174414a3dcb86bdab83ebfbe1c421 -[2025-11-30 15:00:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 (Updated: 2024-07-07T07:19:21 [TS: 1720336761] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 (Updated: 2024-07-07T07:19:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/144e1b99-f7a9-4853-aeab-43045cced4d0] to complete... -.....done. -[2025-11-30 15:00:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babc8da54e8cae46955bc2320b2af7266923bdb18d68f249286306482e45dab2 -[2025-11-30 15:00:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 (Updated: 2024-07-08T07:19:37 [TS: 1720423177] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 (Updated: 2024-07-08T07:19:37) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/beac3e5a-36b6-4f55-9df7-c40cb2b373de] to complete... -......done. -[2025-11-30 15:00:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1ae47d8272a25687baeb5efdb8f84c44351b75f4a12ccb2e842872d16ba460b5 -[2025-11-30 15:00:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 (Updated: 2024-07-09T07:19:28 [TS: 1720509568] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 (Updated: 2024-07-09T07:19:28) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d2c043f4-256b-4aa7-a3ac-7fb37fcfb5b6] to complete... -......done. -[2025-11-30 15:00:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b1365d65ecb4010f898ac145e3ce22dd1003227599d7a88d6ab20299f1f5ef24 -[2025-11-30 15:00:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f (Updated: 2024-07-10T07:19:10 [TS: 1720595950] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f (Updated: 2024-07-10T07:19:10) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bcacade1-5023-4579-a83c-f4b3aa837745] to complete... -......done. -[2025-11-30 15:00:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e50c84a058654f81525876075d59ff72cfbf0ab1a903f3d6a527adc962be4f -[2025-11-30 15:00:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 (Updated: 2024-07-11T07:18:45 [TS: 1720682325] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 (Updated: 2024-07-11T07:18:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a57c96c8-4799-4d19-8f5f-2bb0cc0dc89b] to complete... -......done. -[2025-11-30 15:00:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7c9667685fa6f6d83d9e7afc681fd51f506e139947075538ff9f56224855d316 -[2025-11-30 15:00:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 (Updated: 2024-07-12T07:19:53 [TS: 1720768793] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 (Updated: 2024-07-12T07:19:53) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/22e31337-4ed9-4515-8361-c6f34e74c543] to complete... -......done. -[2025-11-30 15:00:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:506f82098ebec9d2f543a2decfbcbfc63b8f45a8e49dcef8ac647269aab80131 -[2025-11-30 15:00:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d (Updated: 2024-07-13T07:19:14 [TS: 1720855154] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d (Updated: 2024-07-13T07:19:14) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4882351c-2a43-4e08-a348-a0c4a61f6e3c] to complete... -.....done. -[2025-11-30 15:00:59] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:60a4641c2ad5330252a5cc2a348586173e0a5a7de99e40926dd7696ef719db6d -[2025-11-30 15:00:59] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 (Updated: 2024-07-14T07:19:26 [TS: 1720941566] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:00:59] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 (Updated: 2024-07-14T07:19:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4a19e534-ebb8-4850-8d79-9584d50893f4] to complete... -.....done. -[2025-11-30 15:01:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:100226b850ec3ddaf3e350c8e2bbd671f57d12e9a737cf783f73deae9c71dfd4 -[2025-11-30 15:01:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 (Updated: 2024-07-15T07:19:03 [TS: 1721027943] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 (Updated: 2024-07-15T07:19:03) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/af5c3390-8ab8-4f03-9b4c-1a368ae70025] to complete... -.....done. -[2025-11-30 15:01:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818eb404c33b3302347372317f0616b6cedde36755318b70a93b0a82ca0c2059 -[2025-11-30 15:01:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 (Updated: 2024-07-16T07:20:23 [TS: 1721114423] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 (Updated: 2024-07-16T07:20:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/acd58f1e-8f64-4252-826b-3fb0e90788fc] to complete... -.....done. -[2025-11-30 15:01:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ced0f1555c1728d3244d1a530a21bb111e44cbfb382cb6f695fcf11a17dc126 -[2025-11-30 15:01:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 (Updated: 2024-07-17T07:17:53 [TS: 1721200673] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 (Updated: 2024-07-17T07:17:53) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5151d5ea-0890-4293-8526-7e1e06e5c7e1] to complete... -.....done. -[2025-11-30 15:01:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f1f0ddba0d9790d2007748a4a900711aaaf9c16fc9e3d93ea7a8ccf72726c85 -[2025-11-30 15:01:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 (Updated: 2024-07-18T07:19:07 [TS: 1721287147] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 (Updated: 2024-07-18T07:19:07) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4c19ac17-5502-46fb-97a7-99d7df10e881] to complete... -.....done. -[2025-11-30 15:01:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a03f3ad9b4c99b01bb90bb68b4c2c08d9382c321eac140d61bf7eed165431272 -[2025-11-30 15:01:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 (Updated: 2024-07-19T07:18:50 [TS: 1721373530] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 (Updated: 2024-07-19T07:18:50) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e267c491-ee80-4fde-b3ea-5f2253ec01d1] to complete... -.....done. -[2025-11-30 15:01:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4373e17f02ddb47f00bd0ed13b55f96c99a1b11ad72e08e7610dad401d293872 -[2025-11-30 15:01:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 (Updated: 2024-07-20T07:18:11 [TS: 1721459891] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 (Updated: 2024-07-20T07:18:11) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4989c3f1-a41b-49df-8225-33347580711b] to complete... -......done. -[2025-11-30 15:01:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5efc7335a378779508cc43dda86319f87b320eb434c2ca6692cb5f7e6f4f0165 -[2025-11-30 15:01:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 (Updated: 2024-07-21T07:19:01 [TS: 1721546341] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 (Updated: 2024-07-21T07:19:01) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cff1ddc4-13ec-41e4-b1f4-d7afb9034fff] to complete... -......done. -[2025-11-30 15:01:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b7bffc856cd4b56b37a4949f704ff4e2f33d352fd896b4f4122f3698826c3040 -[2025-11-30 15:01:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a (Updated: 2024-07-22T07:19:06 [TS: 1721632746] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a (Updated: 2024-07-22T07:19:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/94ca1c60-87cd-49de-af02-d1118255a792] to complete... -.....done. -[2025-11-30 15:01:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b16cc20e462da0412ef3835c4c3f4b3420e91a92e8f630f0b09bd93fa45f365a -[2025-11-30 15:01:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 (Updated: 2024-07-23T07:19:03 [TS: 1721719143] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 (Updated: 2024-07-23T07:19:03) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8bb8dda1-cc97-4f29-a0f7-00576771588b] to complete... -.....done. -[2025-11-30 15:01:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc9ff5d5c20b44d3241bf805551bbb9543d8fed3599f3f2bcf03ccf7f890dce5 -[2025-11-30 15:01:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 (Updated: 2024-07-24T07:18:45 [TS: 1721805525] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 (Updated: 2024-07-24T07:18:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/15563df9-b44d-45aa-845b-26cee7b5bfc9] to complete... -.....done. -[2025-11-30 15:01:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a6e82a6893acc2b71a7ded04d85b7f9d7b9d401886820351d67711bc115fef3 -[2025-11-30 15:01:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd (Updated: 2024-07-25T07:18:14 [TS: 1721891894] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd (Updated: 2024-07-25T07:18:14) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/590f2894-9273-42f1-9801-5e8aa72de232] to complete... -.....done. -[2025-11-30 15:01:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f742a18b62e518f3bff32d3f8235373cf18c4470b94a7d044693dc70c259c3dd -[2025-11-30 15:01:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 (Updated: 2024-07-26T07:18:21 [TS: 1721978301] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 (Updated: 2024-07-26T07:18:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7a1d588e-12d7-40c1-ba87-142c4bf32bf6] to complete... -.....done. -[2025-11-30 15:01:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38de2046bd421111dfe86611dae02de688ccf2e58e5246b2dd68742000fbacf5 -[2025-11-30 15:01:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 (Updated: 2024-07-27T07:20:10 [TS: 1722064810] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 (Updated: 2024-07-27T07:20:10) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/11ef9a6d-f165-4b6a-9922-2151438673e1] to complete... -.....done. -[2025-11-30 15:01:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfa9a8ca7f040abb831c7563c991856d6cbfd4ca7bf044cf895122d2acd6a594 -[2025-11-30 15:01:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 (Updated: 2024-07-28T07:18:54 [TS: 1722151134] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 (Updated: 2024-07-28T07:18:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/384b1d85-e8e7-4046-8175-934d10c07ae0] to complete... -.....done. -[2025-11-30 15:01:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43ff06836d1f31f1e0a8314f60fb5e3992faa7a2b20befc3379403b920f969d6 -[2025-11-30 15:01:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 (Updated: 2024-07-29T07:17:00 [TS: 1722237420] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 (Updated: 2024-07-29T07:17:00) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/10d5b400-c0cf-46ca-81fb-e1a9799bff33] to complete... -.....done. -[2025-11-30 15:01:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0147914a71cf474c25cb863d089bd6018d9c6405eab95f348abef84851e2f048 -[2025-11-30 15:01:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 (Updated: 2024-07-30T07:18:35 [TS: 1722323915] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:01:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 (Updated: 2024-07-30T07:18:35) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7ec300f0-24cd-4069-8c4c-5d10a5c69188] to complete... -.....done. -[2025-11-30 15:02:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b4fa5bd46737157928b8be92f597acf2860025a50eb148cfc40e1f7fae84a35 -[2025-11-30 15:02:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 (Updated: 2024-07-31T07:19:04 [TS: 1722410344] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 (Updated: 2024-07-31T07:19:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f8d86a90-aad2-4815-8e62-e291ca253f26] to complete... -.....done. -[2025-11-30 15:02:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f452783a5598efb3af3bd20061764f5f0e794f9ab7b09239e3d2f7884c44d736 -[2025-11-30 15:02:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb (Updated: 2024-08-01T07:18:18 [TS: 1722496698] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb (Updated: 2024-08-01T07:18:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/43caa55c-bfd0-4f69-b78a-cff5a9f173b8] to complete... -......done. -[2025-11-30 15:02:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea754aa4916422e43b76c55edb2c0491166ad6419a54c08f31dd6de907d39fb -[2025-11-30 15:02:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 (Updated: 2024-08-02T07:18:39 [TS: 1722583119] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 (Updated: 2024-08-02T07:18:39) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0cc5a41c-aeec-497f-869a-d3c2eacb9c77] to complete... -.....done. -[2025-11-30 15:02:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fb577e609b318b55a1c4451daafbae4ccb4832f9470d76890944ff0a3eca0793 -[2025-11-30 15:02:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd (Updated: 2024-08-03T07:18:31 [TS: 1722669511] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd (Updated: 2024-08-03T07:18:31) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6586a7d0-5d6d-4a81-8450-39ac6933aeee] to complete... -.....done. -[2025-11-30 15:02:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:96a986ac7e00a89acca86155613e772bb63fbcb06451d78d42567f1e3ac10cdd -[2025-11-30 15:02:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 (Updated: 2024-08-04T07:19:22 [TS: 1722755962] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 (Updated: 2024-08-04T07:19:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d6e4435f-2da2-40f2-95b0-0edf462bc84d] to complete... -.....done. -[2025-11-30 15:02:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9452d2fe66f8fe7ffe41455f2e10fec908ac269a77f8c0039aab9f07923274c7 -[2025-11-30 15:02:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a (Updated: 2024-08-05T07:19:47 [TS: 1722842387] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a (Updated: 2024-08-05T07:19:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c0897b36-bff0-4e48-a8a8-c579403bc638] to complete... -......done. -[2025-11-30 15:02:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d6fdf48e3e4deb80a449032ecb42d2aa135d3836e1cfe9367c7c16706d92e8a -[2025-11-30 15:02:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e (Updated: 2024-08-06T07:18:59 [TS: 1722928739] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e (Updated: 2024-08-06T07:18:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c21af908-daaf-4e08-a6a0-9ce470ba8adf] to complete... -......done. -[2025-11-30 15:02:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46358a07f9875fbffb109208776552fa8fa206408626f57e230d65f375dc952e -[2025-11-30 15:02:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 (Updated: 2024-08-07T07:19:22 [TS: 1723015162] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 (Updated: 2024-08-07T07:19:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/30db16db-915a-497f-aab4-64d7543e1bb6] to complete... -......done. -[2025-11-30 15:02:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:203a88718c03cf4c1ed37042c08c786308bd68d81a1d6031960ffec366c79638 -[2025-11-30 15:02:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 (Updated: 2024-08-08T07:18:44 [TS: 1723101524] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 (Updated: 2024-08-08T07:18:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4e8ff82e-f27d-4433-bbdd-378399d8b8bc] to complete... -......done. -[2025-11-30 15:02:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc618fb167e07c1fa02df6e9e063f1d9b915a80f7cc181ce6ec285e434c177f8 -[2025-11-30 15:02:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 (Updated: 2024-08-09T07:19:24 [TS: 1723187964] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 (Updated: 2024-08-09T07:19:24) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7a0c4f06-3ba0-49c5-a870-ce582d6410de] to complete... -.....done. -[2025-11-30 15:02:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc4f3b4f7a1ffe0715765bb66ee921eea706555c1c6e4b40d62944152850244 -[2025-11-30 15:02:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a (Updated: 2024-08-10T07:18:12 [TS: 1723274292] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a (Updated: 2024-08-10T07:18:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/63ab4150-940d-42ea-961f-bd03d6bdff7f] to complete... -......done. -[2025-11-30 15:02:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:624f5d8175a5ac7bda2a8c3319f650bf7645ac76446360657357e1071c65773a -[2025-11-30 15:02:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f (Updated: 2024-08-11T07:18:52 [TS: 1723360732] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f (Updated: 2024-08-11T07:18:52) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7a64fc06-22a9-475f-8330-791bf800ee75] to complete... -......done. -[2025-11-30 15:02:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0ba31ca005517a542dc6180618100b692f6ab2a2e2f8edfa8a5c7abb0dc9451f -[2025-11-30 15:02:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff (Updated: 2024-08-12T07:18:33 [TS: 1723447113] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff (Updated: 2024-08-12T07:18:33) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8ad78b33-39e5-4070-a5d7-809c4dbb9328] to complete... -.....done. -[2025-11-30 15:02:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e0767c264764a28da9d23f47cc61f2bef40ef326dce16889f491c70b6166cff -[2025-11-30 15:02:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff (Updated: 2024-08-13T07:19:14 [TS: 1723533554] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff (Updated: 2024-08-13T07:19:14) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/75c535fb-8c98-4097-b953-8cfb50d1f24a] to complete... -......done. -[2025-11-30 15:02:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bbcf0d2970b23b4b8e4cbd9d0da590d15d4f7ccf64e5408a40b7f4b7a2479eff -[2025-11-30 15:02:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a (Updated: 2024-08-14T07:20:03 [TS: 1723620003] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a (Updated: 2024-08-14T07:20:03) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bfe3661a-25d5-4ab8-8fe6-5c92b95449fb] to complete... -......done. -[2025-11-30 15:02:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0078e5d48e7c43b3771a1e02ebd643d9574b12399de930e5a93efd1f090d9f5a -[2025-11-30 15:02:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 (Updated: 2024-08-15T07:18:25 [TS: 1723706305] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:02:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 (Updated: 2024-08-15T07:18:25) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0f2bc4c7-6030-4138-9247-4721f4fd5f53] to complete... -.....done. -[2025-11-30 15:03:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0f3c7483938b82de65379ebae0ea3841fb829aa65064bbdbe13c98578bab8d9 -[2025-11-30 15:03:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b (Updated: 2024-08-16T07:18:57 [TS: 1723792737] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:03:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b (Updated: 2024-08-16T07:18:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/06e7f6f6-8a4e-4dff-9609-c7cb5b17e3fe] to complete... -.....done. -[2025-11-30 15:03:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e579afadaca75c818767244961efdce886751723ba7ea228581def1f4d00730b -[2025-11-30 15:03:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b (Updated: 2024-08-17T07:19:21 [TS: 1723879161] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:03:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b (Updated: 2024-08-17T07:19:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b9a596d0-5655-4608-9c72-ee787bb2b756] to complete... -......done. -[2025-11-30 15:03:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:76ebf98af07b5862bef17a47cf17281ea1b6892b3c6cc548bced533cafc06a0b -[2025-11-30 15:03:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e (Updated: 2024-08-18T07:19:19 [TS: 1723965559] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:03:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e (Updated: 2024-08-18T07:19:19) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fb060761-f36f-4033-8e38-653eec90e4c5] to complete... -......done. -[2025-11-30 15:03:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:852b05ccdcabce6692fddbf3e91779beb9cc8a3139db20f96abbd90b67463c4e -[2025-11-30 15:03:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 (Updated: 2024-08-19T07:19:35 [TS: 1724051975] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:03:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 (Updated: 2024-08-19T07:19:35) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/60fe7d1f-8e4a-4219-b9e1-7edc3b604dd9] to complete... -......done. -[2025-11-30 15:03:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:766b34a198c7710a5340cfd63c9fb4ca192cfaa9d732164874188ff66619d714 -[2025-11-30 15:03:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 (Updated: 2024-08-20T07:19:04 [TS: 1724138344] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:03:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 (Updated: 2024-08-20T07:19:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/950d1860-c53c-428d-8742-830d14ec46a8] to complete... -.....done. -[2025-11-30 15:03:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ee596c377eed607ccd96ad71047098381691a31d29456ae2c3f009fd0f18b450 -[2025-11-30 15:03:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 (Updated: 2024-08-21T07:19:04 [TS: 1724224744] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:03:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 (Updated: 2024-08-21T07:19:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9afca90e-6d1a-4b2d-9501-41bb35689475] to complete... -.....done. -[2025-11-30 15:03:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dcc23bdc6e56fb04bf6697e194d3ca4a08eb19c08a373b2135df9a32c174c689 -[2025-11-30 15:03:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3b91af27e525f56f3ea343a5b051fc87a0f446e614700ba93b62d0eaebaa270 (Updated: 2024-08-22T07:18:34 [TS: 1724311114] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:03:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f3b91af27e525f56f3ea343a5b051fc87a0f446e614700ba93b62d0eaebaa270 (Updated: 2024-08-22T07:18:34) -[2025-11-30 15:03:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:256f6245d97633997bd4c3cc2573f97f39b348629a527a5b3fb8e22b7b857d15 -[2025-11-30 15:03:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 (Updated: 2024-08-31T07:19:15 [TS: 1725088755] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:03:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 (Updated: 2024-08-31T07:19:15) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/734c08b3-3eb7-466c-b798-f2abc3e40093] to complete... -.....done. -[2025-11-30 15:04:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c10e7818cba052173d8d9a63c31f932503e3a8f9f4e06a05cc4c21e4d638566 -[2025-11-30 15:04:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 (Updated: 2024-09-01T07:18:35 [TS: 1725175115] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 (Updated: 2024-09-01T07:18:35) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9b63ff5b-6d50-486b-9410-559ccb5d4442] to complete... -......done. -[2025-11-30 15:04:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a40b5e97021550088bb61f3162544daf5c0d2f1d0caa678423dd59ec97000180 -[2025-11-30 15:04:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 (Updated: 2024-09-02T07:19:08 [TS: 1725261548] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 (Updated: 2024-09-02T07:19:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c4033f09-ab33-40cd-b001-4b6ecc533c12] to complete... -.....done. -[2025-11-30 15:04:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6f9cfe264ccda3e17ba31e5e96e6e07cf4f45c4d55f80cafc38af45ec614818 -[2025-11-30 15:04:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 (Updated: 2024-09-03T07:18:34 [TS: 1725347914] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 (Updated: 2024-09-03T07:18:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d0f1d9ff-de22-4ecb-a9ff-b651bdea0f3e] to complete... -.....done. -[2025-11-30 15:04:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:babe3940e985af5b358ead8543a501f30ea61999899e35f7c8afb72cf3fa5110 -[2025-11-30 15:04:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 (Updated: 2024-09-04T07:19:12 [TS: 1725434352] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 (Updated: 2024-09-04T07:19:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cc2caa33-7952-40a1-99aa-4069ee500e72] to complete... -......done. -[2025-11-30 15:04:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b8834b6bcc5398bab4e9d3dbbaa29d729477ecedb8e428fb50df106b82d40da0 -[2025-11-30 15:04:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd (Updated: 2024-09-04T15:00:21 [TS: 1725462021] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd (Updated: 2024-09-04T15:00:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd - -Tags: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner:test -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8b70d4d9-d4e2-4b31-9692-95f8a2001225] to complete... -......done. -[2025-11-30 15:04:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:145b8840cae6ed7ff8906af1590707c89a6a02473fc1236365f251c01dcabebd -[2025-11-30 15:04:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 (Updated: 2024-09-05T07:19:39 [TS: 1725520779] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 (Updated: 2024-09-05T07:19:39) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/dc9682fe-5606-4b37-8b3b-1b842374e9cc] to complete... -......done. -[2025-11-30 15:04:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9c1207cc6df49756e45cd61a4d7f613e738b02a42dfe11dcf84890a781a4c05 -[2025-11-30 15:04:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 (Updated: 2024-09-06T07:19:00 [TS: 1725607140] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 (Updated: 2024-09-06T07:19:00) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/33b7771b-d4c0-4894-9781-af3b33c4dbf6] to complete... -......done. -[2025-11-30 15:04:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f9cb6edb3298e129752f6347398d96807ecafb461ddc863119b06ac08a1df3 -[2025-11-30 15:04:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 (Updated: 2024-09-07T07:20:06 [TS: 1725693606] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 (Updated: 2024-09-07T07:20:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/22525bf9-d25a-448b-add5-1e49908e3a53] to complete... -......done. -[2025-11-30 15:04:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:daf0df0c5ffb582680a79b3e7be5d620f76c8eef9012bb556356f0f83545ecb2 -[2025-11-30 15:04:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 (Updated: 2024-09-08T07:20:00 [TS: 1725780000] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 (Updated: 2024-09-08T07:20:00) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/348f2339-6895-4af7-8bbe-a1c0b9f1c7d3] to complete... -......done. -[2025-11-30 15:04:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a2ff64546d34aa92e3a5d1f36b3545238ad9e9692774622cd2fbf4fa6568815 -[2025-11-30 15:04:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f (Updated: 2024-09-09T07:18:55 [TS: 1725866335] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f (Updated: 2024-09-09T07:18:55) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fe4435b9-2be0-4d26-b096-8e95902d1747] to complete... -.....done. -[2025-11-30 15:04:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:323386e293e630d3f39921cb39b73a8d59134245696f30096b28b8072515ef3f -[2025-11-30 15:04:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 (Updated: 2024-09-10T07:18:23 [TS: 1725952703] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 (Updated: 2024-09-10T07:18:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a869b618-95c7-47f1-8a02-21f14b99a29c] to complete... -......done. -[2025-11-30 15:04:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edeede7cb7d90e90ad1079792bc1c2c00ade77f6fb53d1fdd3568ff5250e97c8 -[2025-11-30 15:04:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 (Updated: 2024-09-11T07:18:27 [TS: 1726039107] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 (Updated: 2024-09-11T07:18:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ebcb23b0-25dd-4534-a2bf-4ab0a104c2d6] to complete... -......done. -[2025-11-30 15:04:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7b7d9435c55cc04c75de0d772f8b54ee39a81d65535c0896248760bf39149c2 -[2025-11-30 15:04:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b (Updated: 2024-09-12T07:19:28 [TS: 1726125568] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b (Updated: 2024-09-12T07:19:28) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/350ced73-0482-4755-93fa-8dd81b2f8158] to complete... -......done. -[2025-11-30 15:04:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8fdee420375182f43b3bf2d1e242e54abca7f8995d5ae0b769845da9d4e70c2b -[2025-11-30 15:04:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 (Updated: 2024-09-13T07:18:59 [TS: 1726211939] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 (Updated: 2024-09-13T07:18:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/46daa7e5-4637-46b7-8161-bd40ddc9ee73] to complete... -.....done. -[2025-11-30 15:04:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6c5f5b4c7490d05f3daa2ee25c7e2678e07b032ea23780c649f3774b42284d4 -[2025-11-30 15:04:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 (Updated: 2024-09-14T07:21:55 [TS: 1726298515] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 (Updated: 2024-09-14T07:21:55) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/be551cdd-1642-47ee-828d-4e05d48c3a86] to complete... -......done. -[2025-11-30 15:04:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb8f1d5f36f132a3546021ae696158e4944582769186bafe48c2b640083ddef0 -[2025-11-30 15:04:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 (Updated: 2024-09-15T07:18:46 [TS: 1726384726] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:04:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 (Updated: 2024-09-15T07:18:46) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b1e095bd-a455-419b-91e4-ec520392eb19] to complete... -......done. -[2025-11-30 15:05:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:881512a90ec926e4ee337bf95c0da01c57a4966a68db405b130dfa68eeb923b9 -[2025-11-30 15:05:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 (Updated: 2024-09-16T07:19:43 [TS: 1726471183] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 (Updated: 2024-09-16T07:19:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8153c94c-fe6a-48e7-bd26-ed027f4f30a5] to complete... -......done. -[2025-11-30 15:05:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a79a1c0c5e02bb53a1d5f876e3ab7c9da0cd221e081c110333e372bbcb747b72 -[2025-11-30 15:05:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 (Updated: 2024-09-17T07:18:21 [TS: 1726557501] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 (Updated: 2024-09-17T07:18:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0f68dbbb-7eb7-4e49-8f39-c91e72dbc51e] to complete... -......done. -[2025-11-30 15:05:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4aafcdb1f04656b2a2b32b52fc3baa40a2be905b203b57e90a568df9ce8e0927 -[2025-11-30 15:05:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af (Updated: 2024-09-18T07:18:46 [TS: 1726643926] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af (Updated: 2024-09-18T07:18:46) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bf6a3e87-d241-4c40-a2b6-de65f1f75dc8] to complete... -......done. -[2025-11-30 15:05:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:21ff44ac6e7c6febfd526088c3dc8e285c2c1fff849cc585890240116b8190af -[2025-11-30 15:05:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 (Updated: 2024-09-19T07:19:58 [TS: 1726730398] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 (Updated: 2024-09-19T07:19:58) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0b5f4818-9c32-49ea-91ea-ce0c2d1f4d67] to complete... -......done. -[2025-11-30 15:05:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f50518242e8e8b8b7b8a2f090dc91c4895e20f84ac97b96dfb56a6673b715991 -[2025-11-30 15:05:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c (Updated: 2024-09-20T07:19:34 [TS: 1726816774] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c (Updated: 2024-09-20T07:19:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7adaecf8-a513-4c69-acd2-1a6c1c7a85a6] to complete... -......done. -[2025-11-30 15:05:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b124d506d65084474b48f2b52bd9b3226d0a47811ea3a41326a0e82cbcd1264c -[2025-11-30 15:05:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 (Updated: 2024-09-21T07:18:42 [TS: 1726903122] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 (Updated: 2024-09-21T07:18:42) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/23d8ef73-80a8-4b44-88dd-d33552703102] to complete... -......done. -[2025-11-30 15:05:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e898abc53cbee92d10d83eb1a2747f9987cae8c93d048ed8c7f142fda2c24807 -[2025-11-30 15:05:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f (Updated: 2024-09-22T07:19:13 [TS: 1726989553] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f (Updated: 2024-09-22T07:19:13) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8c6974e5-5298-4d03-a125-d96f6aecbece] to complete... -.....done. -[2025-11-30 15:05:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1224ab31648230a59d76a6ff28992ec57445d0323f40175276f7fee8af74f47f -[2025-11-30 15:05:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 (Updated: 2024-09-23T07:19:27 [TS: 1727075967] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 (Updated: 2024-09-23T07:19:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0578c740-b239-46c4-b7af-8e33399d0ccd] to complete... -.....done. -[2025-11-30 15:05:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0f32280145a2a31b50dd648793402856225e4515f6e10c9b819630496f53e0c9 -[2025-11-30 15:05:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a (Updated: 2024-09-24T07:19:03 [TS: 1727162343] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a (Updated: 2024-09-24T07:19:03) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/416c0a72-39ba-47f4-8533-f16b7bbed847] to complete... -.....done. -[2025-11-30 15:05:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdbf873a0a239ec99d597a03e71369a6aff5a099228314e3c3782eea5f5123a -[2025-11-30 15:05:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 (Updated: 2024-09-25T07:19:26 [TS: 1727248766] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 (Updated: 2024-09-25T07:19:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9cf6e646-0df7-41e4-b0d6-adf0332b6383] to complete... -......done. -[2025-11-30 15:05:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cabd587fd610538ae5a74a0166d51090d05c21613faacf8ee943cec964677f73 -[2025-11-30 15:05:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 (Updated: 2024-09-26T07:18:34 [TS: 1727335114] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 (Updated: 2024-09-26T07:18:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b895d6a2-77e0-4f1d-afaf-e901dc15cf5f] to complete... -......done. -[2025-11-30 15:05:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23823b9d244e07bd2dad0441bf168fb2e4e6c2268a3f7281af6fd4b5d59e1ea3 -[2025-11-30 15:05:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 (Updated: 2024-09-27T07:18:51 [TS: 1727421531] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 (Updated: 2024-09-27T07:18:51) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fd999bbe-bfbf-4fe9-bdaa-603e77e64416] to complete... -......done. -[2025-11-30 15:05:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a4131913d60641db1b8dd50b88fb2f364edc86be1435529bdc415d9599cb8c6 -[2025-11-30 15:05:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 (Updated: 2024-09-28T07:19:08 [TS: 1727507948] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 (Updated: 2024-09-28T07:19:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/25c4c2f6-9b35-42c7-9e6e-8496c4250948] to complete... -......done. -[2025-11-30 15:05:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1858b88005fa515943dc2817ca28260b86120e15e32cfea2fff14274b07022d6 -[2025-11-30 15:05:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b (Updated: 2024-09-29T07:19:59 [TS: 1727594399] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b (Updated: 2024-09-29T07:19:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ac70b5e7-42ec-43ca-9914-ef9545e43871] to complete... -......done. -[2025-11-30 15:05:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be80ffd13144b7529d3234628c6b4a28c7583189d26b1083ab488436dc88b19b -[2025-11-30 15:05:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 (Updated: 2024-09-30T07:18:47 [TS: 1727680727] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:05:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 (Updated: 2024-09-30T07:18:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bc5e27f5-4c04-4dc6-8d9f-b17c77a25130] to complete... -......done. -[2025-11-30 15:06:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3d565d09c7f41ff1ef422c92c62cb00d2456fae1b5afcb7acb46483936594e2 -[2025-11-30 15:06:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 (Updated: 2024-10-01T07:18:16 [TS: 1727767096] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 (Updated: 2024-10-01T07:18:16) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4285abe6-3421-477f-85e4-351c69bd5d11] to complete... -......done. -[2025-11-30 15:06:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6512c0c94751acd7a1b0875555dc2c0dc37e2be08515c6955dd3c4c1c89d1627 -[2025-11-30 15:06:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 (Updated: 2024-10-02T07:19:22 [TS: 1727853562] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 (Updated: 2024-10-02T07:19:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8f297a13-1fe9-4430-bde6-320b14ef9c69] to complete... -......done. -[2025-11-30 15:06:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:360c8973c03873ff98090b828fef43a448c1db89f9c9adb08945a143c116f2d8 -[2025-11-30 15:06:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d (Updated: 2024-10-03T07:19:40 [TS: 1727939980] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d (Updated: 2024-10-03T07:19:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/06953728-a6b9-4ef7-9784-4d439aebb515] to complete... -......done. -[2025-11-30 15:06:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d3b890989b182e379442e2bae6187bedd955fc207674258b70aef3960e46717d -[2025-11-30 15:06:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c (Updated: 2024-10-04T07:20:00 [TS: 1728026400] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c (Updated: 2024-10-04T07:20:00) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7a74c22f-f6dc-44b4-9292-6d863e87e1b0] to complete... -.....done. -[2025-11-30 15:06:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef5e36e9a1056a11f313da40161b807630203a00d7fb1aa9e625bca3e6885b4c -[2025-11-30 15:06:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 (Updated: 2024-10-05T07:20:18 [TS: 1728112818] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 (Updated: 2024-10-05T07:20:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8099fc49-39f3-48e2-b54e-860e6544c1ad] to complete... -.....done. -[2025-11-30 15:06:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74db3e764a7029ca53bbe4e3df1d94bb684e65dfd4296b83035657144842b109 -[2025-11-30 15:06:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d (Updated: 2024-10-06T07:19:45 [TS: 1728199185] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d (Updated: 2024-10-06T07:19:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/280d8a74-209d-4233-9329-02ea48f1216a] to complete... -......done. -[2025-11-30 15:06:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1e4aacd3bd3022b865fa87ed8c8063b1e2d1f0880f4a3d930dd2015cfea390d -[2025-11-30 15:06:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 (Updated: 2024-10-07T07:20:12 [TS: 1728285612] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 (Updated: 2024-10-07T07:20:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6487728b-299a-4ea6-9f78-cce63d6c6000] to complete... -......done. -[2025-11-30 15:06:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:08dff41486eb9bed91bb05933426eb417308568260f5de09254e56569b2c2e66 -[2025-11-30 15:06:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 (Updated: 2024-10-08T07:19:06 [TS: 1728371946] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 (Updated: 2024-10-08T07:19:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c2163135-6630-4a48-b988-d5f7baca107c] to complete... -......done. -[2025-11-30 15:06:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7eca5ae9c08e9672547aeb552e3f359d5d2ac6d53f2f5a63c98eef055c163ed2 -[2025-11-30 15:06:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 (Updated: 2024-10-09T07:19:20 [TS: 1728458360] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 (Updated: 2024-10-09T07:19:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f6f4eb44-712c-414f-8ee0-f304bc03e78f] to complete... -......done. -[2025-11-30 15:06:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b53d9471730f24e3859d3b3829708c1c08e62cfc2fbf5e4de9cf9ca189dd7514 -[2025-11-30 15:06:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e (Updated: 2024-10-10T07:19:58 [TS: 1728544798] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e (Updated: 2024-10-10T07:19:58) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4b684cf0-cb2c-41ff-90b8-d98b32098c61] to complete... -......done. -[2025-11-30 15:06:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:11efe80b3753468e0f3d244ae561d02c5e537dc7e3b8793ce8550bcce431968e -[2025-11-30 15:06:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf (Updated: 2024-10-11T07:19:31 [TS: 1728631171] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf (Updated: 2024-10-11T07:19:31) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d25e04d1-8231-42dc-b502-8dccb1c76093] to complete... -......done. -[2025-11-30 15:06:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e9d2cd5390e93b8eca6b0ce045f8a47b589788dc23b89ea09fe6c4752f60ecf -[2025-11-30 15:06:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 (Updated: 2024-10-12T07:19:51 [TS: 1728717591] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 (Updated: 2024-10-12T07:19:51) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d34c18a6-5aab-4631-b690-1c265f3353b7] to complete... -......done. -[2025-11-30 15:06:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:125e0c6361acfd4d413ee3d9ea6f1259c0b431d228c24efce681a9f12de84ae5 -[2025-11-30 15:06:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 (Updated: 2024-10-13T07:20:18 [TS: 1728804018] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 (Updated: 2024-10-13T07:20:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/313c36a8-d2a2-4e4e-bb1b-3ffa8e83c52e] to complete... -......done. -[2025-11-30 15:06:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6b86fdef2f0aac228b69db840004474616091ebc8cc15652f5fc3ec11c53b97 -[2025-11-30 15:06:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 (Updated: 2024-10-14T07:18:56 [TS: 1728890336] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 (Updated: 2024-10-14T07:18:56) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3ccecd5a-5289-446f-81ee-bb8f956cf425] to complete... -......done. -[2025-11-30 15:06:59] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfbd90eb991e7dce61f5ea025a9b031db61a24fbc145b8f53c6cdd5428e8acb0 -[2025-11-30 15:06:59] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 (Updated: 2024-10-15T07:18:42 [TS: 1728976722] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:06:59] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 (Updated: 2024-10-15T07:18:42) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/78be52f7-724a-414f-8cea-df672f17544d] to complete... -.....done. -[2025-11-30 15:07:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72a8a70609f7c7b9d40ed906dcc0ab49cbf8df37eca9321720f360d7f633c7d3 -[2025-11-30 15:07:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 (Updated: 2024-10-16T07:18:53 [TS: 1729063133] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:07:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 (Updated: 2024-10-16T07:18:53) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b21da8ff-d534-43df-a503-926aa026fc41] to complete... -.....done. -[2025-11-30 15:07:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1979a1eb0120d5183b824cbcceef852e2dd3d872d486443d5c19fcf5e616d8 -[2025-11-30 15:07:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 (Updated: 2024-10-17T07:19:38 [TS: 1729149578] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:07:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 (Updated: 2024-10-17T07:19:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/724f27ab-0add-42e9-9ce5-d154fe885be8] to complete... -......done. -[2025-11-30 15:07:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3d1af927f3f3c99847679444830dd88833ef88e6fde46cb6b45a15593d390895 -[2025-11-30 15:07:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 (Updated: 2024-10-18T07:21:27 [TS: 1729236087] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:07:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 (Updated: 2024-10-18T07:21:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/039a7dfa-8125-4176-90de-6b5b4f91a9d9] to complete... -......done. -[2025-11-30 15:07:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b3a791132afd82d3eda0e351ab54ef790d42815047b5ead587fd9946f8361c8 -[2025-11-30 15:07:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 (Updated: 2024-10-19T07:21:08 [TS: 1729322468] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:07:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 (Updated: 2024-10-19T07:21:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d7357cb4-57cf-4433-8b94-ae599dfec4ee] to complete... -......done. -[2025-11-30 15:07:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5cc6e7d75ed2ed5ad7883ebfa71624dfa705b88fd811b40f20a74000e49eaf18 -[2025-11-30 15:07:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 (Updated: 2024-10-20T07:20:13 [TS: 1729408813] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:07:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 (Updated: 2024-10-20T07:20:13) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4c0b6f9b-18db-4b22-93aa-7556feb36b52] to complete... -.....done. -[2025-11-30 15:07:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d03692b81d9e5c0ec677a555c925822d7037db6ee7ffcdae29e4d0f39c5ab12 -[2025-11-30 15:07:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 (Updated: 2024-10-21T07:19:27 [TS: 1729495167] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:07:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 (Updated: 2024-10-21T07:19:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b86cf1b3-407a-43ee-9797-79132a3565e4] to complete... -......done. -[2025-11-30 15:07:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:51d47a9c9dcb4a50c250ab483c0a6032dbac31cb0ae0ffc22d4a60e824529875 -[2025-11-30 15:07:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 (Updated: 2024-10-22T07:19:36 [TS: 1729581576] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:07:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 (Updated: 2024-10-22T07:19:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3db0b93f-b2a7-4e36-b9bb-f793d3d0fbdf] to complete... -......done. -[2025-11-30 15:07:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:edf414fb9c0935906f4f8b5f402b70404b4bd128be2f4e947885d87a3d1174d8 -[2025-11-30 15:07:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 (Updated: 2024-10-23T07:18:47 [TS: 1729667927] < Cutoff: [TS: 1763304912]) -[2025-11-30 15:07:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 (Updated: 2024-10-23T07:18:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ed80c11c-a015-4dc2-9251-fafe5894b861] to complete... -.....done. -[2025-11-30 15:07:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1926e241552d65a2d9a83de98a953bbe57f6921c57c7ff2bc46f4cdd95f5c315 -[2025-11-30 15:07:34] [INFO] Hit delete limit (200) for Docker Images. -[2025-11-30 15:07:34] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 15:07:34] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 15:07:37] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 15:07:37] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 15:07:37] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 15:07:37] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 15:07:37] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 15:07:37] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 15:07:39] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 15:07:39] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:07:42] [INFO] No Regional Address found matching criteria. -[2025-11-30 15:07:42] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:07:44] [INFO] No Global Address found matching criteria. -[2025-11-30 15:07:44] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- -[2025-11-30 15:07:48] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 15:07:51] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:07:51] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:07:51] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 15:07:51] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 15:07:51] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 15:07:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:54] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 15:07:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:07:56] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 15:07:58] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 15:07:58] [INFO] CLEANUP RUN FINISHED -[2025-11-30 15:08:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 15:08:29] [INFO] Time Cutoff (General): 2025-11-30T15:08:29+0000 -[2025-11-30 15:08:29] [INFO] Time Cutoff (Images): 2025-10-01T15:08:29+0000 -[2025-11-30 15:08:29] [INFO] Delete Limit per Type: 200 -[2025-11-30 15:08:29] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 15:08:29] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 15:08:32] [INFO] No Service Accounts found matching prefix. -[2025-11-30 15:08:32] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:08:34] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 15:08:34] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 15:08:37] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:08:37] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:08:37] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 15:08:37] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:08:40] [INFO] No Filestore instances found matching criteria. -[2025-11-30 15:08:40] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 15:08:43] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 15:08:43] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 15:08:43] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 15:08:43] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 15:08:43] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 15:08:43] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 15:08:43] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 15:08:43] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 15:08:44] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 15:08:44] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 15:08:44] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 15:08:44] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:08:44Z (Unix: 1763305724) -[2025-11-30 15:08:44] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 (Updated: 2024-10-24T07:21:12 [TS: 1729754472] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 (Updated: 2024-10-24T07:21:12) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 (Updated: 2024-10-25T07:19:11 [TS: 1729840751] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 (Updated: 2024-10-25T07:19:11) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea (Updated: 2024-10-26T07:19:04 [TS: 1729927144] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea (Updated: 2024-10-26T07:19:04) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 (Updated: 2024-10-27T07:18:59 [TS: 1730013539] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 (Updated: 2024-10-27T07:18:59) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f (Updated: 2024-10-28T07:19:53 [TS: 1730099993] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f (Updated: 2024-10-28T07:19:53) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa (Updated: 2024-10-29T07:21:32 [TS: 1730186492] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa (Updated: 2024-10-29T07:21:32) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c (Updated: 2024-10-30T07:20:09 [TS: 1730272809] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c (Updated: 2024-10-30T07:20:09) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca (Updated: 2024-10-31T07:19:50 [TS: 1730359190] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca (Updated: 2024-10-31T07:19:50) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb (Updated: 2024-11-01T07:19:35 [TS: 1730445575] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb (Updated: 2024-11-01T07:19:35) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 (Updated: 2024-11-02T07:19:20 [TS: 1730531960] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 (Updated: 2024-11-02T07:19:20) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b (Updated: 2024-11-03T07:19:34 [TS: 1730618374] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b (Updated: 2024-11-03T07:19:34) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc (Updated: 2024-11-04T08:20:03 [TS: 1730708403] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc (Updated: 2024-11-04T08:20:03) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed (Updated: 2024-11-05T08:19:36 [TS: 1730794776] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed (Updated: 2024-11-05T08:19:36) -[2025-11-30 15:08:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e (Updated: 2024-11-06T08:19:42 [TS: 1730881182] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:50] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e (Updated: 2024-11-06T08:19:42) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 (Updated: 2024-11-07T08:22:39 [TS: 1730967759] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 (Updated: 2024-11-07T08:22:39) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 (Updated: 2024-11-08T08:19:05 [TS: 1731053945] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 (Updated: 2024-11-08T08:19:05) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 (Updated: 2024-11-09T08:19:52 [TS: 1731140392] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 (Updated: 2024-11-09T08:19:52) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 (Updated: 2024-11-10T08:19:21 [TS: 1731226761] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 (Updated: 2024-11-10T08:19:21) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e (Updated: 2024-11-11T08:18:44 [TS: 1731313124] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e (Updated: 2024-11-11T08:18:44) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c (Updated: 2024-11-12T08:20:12 [TS: 1731399612] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c (Updated: 2024-11-12T08:20:12) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d (Updated: 2024-11-13T08:18:46 [TS: 1731485926] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d (Updated: 2024-11-13T08:18:46) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 (Updated: 2024-11-14T08:19:32 [TS: 1731572372] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 (Updated: 2024-11-14T08:19:32) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa (Updated: 2024-11-15T08:18:37 [TS: 1731658717] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa (Updated: 2024-11-15T08:18:37) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 (Updated: 2024-11-16T08:19:38 [TS: 1731745178] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 (Updated: 2024-11-16T08:19:38) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 (Updated: 2024-11-17T08:19:08 [TS: 1731831548] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 (Updated: 2024-11-17T08:19:08) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c (Updated: 2024-11-18T08:19:40 [TS: 1731917980] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c (Updated: 2024-11-18T08:19:40) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e (Updated: 2024-11-19T08:19:29 [TS: 1732004369] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e (Updated: 2024-11-19T08:19:29) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 (Updated: 2024-11-20T08:19:24 [TS: 1732090764] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 (Updated: 2024-11-20T08:19:24) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf (Updated: 2024-11-21T08:19:48 [TS: 1732177188] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf (Updated: 2024-11-21T08:19:48) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 (Updated: 2024-11-22T08:19:50 [TS: 1732263590] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 (Updated: 2024-11-22T08:19:50) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb (Updated: 2024-11-23T08:18:52 [TS: 1732349932] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb (Updated: 2024-11-23T08:18:52) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 (Updated: 2024-11-24T08:19:55 [TS: 1732436395] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 (Updated: 2024-11-24T08:19:55) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 (Updated: 2024-11-25T08:18:26 [TS: 1732522706] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 (Updated: 2024-11-25T08:18:26) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e (Updated: 2024-11-26T08:19:19 [TS: 1732609159] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e (Updated: 2024-11-26T08:19:19) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f (Updated: 2024-11-27T08:19:05 [TS: 1732695545] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f (Updated: 2024-11-27T08:19:05) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af (Updated: 2024-11-28T08:18:54 [TS: 1732781934] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af (Updated: 2024-11-28T08:18:54) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 (Updated: 2024-11-29T08:19:25 [TS: 1732868365] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 (Updated: 2024-11-29T08:19:25) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db (Updated: 2024-11-30T08:19:26 [TS: 1732954766] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db (Updated: 2024-11-30T08:19:26) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b (Updated: 2024-12-01T08:20:11 [TS: 1733041211] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b (Updated: 2024-12-01T08:20:11) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c (Updated: 2024-12-02T08:18:21 [TS: 1733127501] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c (Updated: 2024-12-02T08:18:21) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff (Updated: 2024-12-03T08:19:17 [TS: 1733213957] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff (Updated: 2024-12-03T08:19:17) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 (Updated: 2024-12-04T08:20:29 [TS: 1733300429] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 (Updated: 2024-12-04T08:20:29) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 (Updated: 2024-12-04T23:35:41 [TS: 1733355341] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 (Updated: 2024-12-04T23:35:41) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e (Updated: 2024-12-05T08:19:43 [TS: 1733386783] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e (Updated: 2024-12-05T08:19:43) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 (Updated: 2024-12-06T08:19:51 [TS: 1733473191] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 (Updated: 2024-12-06T08:19:51) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da (Updated: 2024-12-07T08:19:02 [TS: 1733559542] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da (Updated: 2024-12-07T08:19:02) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d (Updated: 2024-12-08T08:18:49 [TS: 1733645929] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d (Updated: 2024-12-08T08:18:49) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e (Updated: 2024-12-09T08:18:56 [TS: 1733732336] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e (Updated: 2024-12-09T08:18:56) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f (Updated: 2024-12-10T08:20:41 [TS: 1733818841] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f (Updated: 2024-12-10T08:20:41) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba (Updated: 2024-12-11T08:19:22 [TS: 1733905162] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba (Updated: 2024-12-11T08:19:22) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 (Updated: 2024-12-12T08:19:21 [TS: 1733991561] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 (Updated: 2024-12-12T08:19:21) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb (Updated: 2024-12-13T08:19:59 [TS: 1734077999] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb (Updated: 2024-12-13T08:19:59) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 (Updated: 2024-12-14T08:19:12 [TS: 1734164352] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 (Updated: 2024-12-14T08:19:12) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b (Updated: 2024-12-15T08:19:07 [TS: 1734250747] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b (Updated: 2024-12-15T08:19:07) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 (Updated: 2024-12-16T08:18:40 [TS: 1734337120] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 (Updated: 2024-12-16T08:18:40) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 (Updated: 2024-12-17T08:19:36 [TS: 1734423576] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 (Updated: 2024-12-17T08:19:36) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 (Updated: 2024-12-18T08:19:28 [TS: 1734509968] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 (Updated: 2024-12-18T08:19:28) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 (Updated: 2024-12-19T08:19:45 [TS: 1734596385] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 (Updated: 2024-12-19T08:19:45) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 (Updated: 2024-12-20T08:20:22 [TS: 1734682822] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 (Updated: 2024-12-20T08:20:22) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b (Updated: 2024-12-21T08:18:43 [TS: 1734769123] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b (Updated: 2024-12-21T08:18:43) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 (Updated: 2024-12-22T08:19:03 [TS: 1734855543] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 (Updated: 2024-12-22T08:19:03) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 (Updated: 2024-12-23T08:19:14 [TS: 1734941954] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 (Updated: 2024-12-23T08:19:14) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 (Updated: 2024-12-24T08:19:47 [TS: 1735028387] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 (Updated: 2024-12-24T08:19:47) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c (Updated: 2024-12-25T08:19:25 [TS: 1735114765] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c (Updated: 2024-12-25T08:19:25) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d (Updated: 2024-12-26T08:19:08 [TS: 1735201148] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d (Updated: 2024-12-26T08:19:08) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b (Updated: 2024-12-27T08:19:18 [TS: 1735287558] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b (Updated: 2024-12-27T08:19:18) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 (Updated: 2024-12-28T08:20:48 [TS: 1735374048] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 (Updated: 2024-12-28T08:20:48) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 (Updated: 2024-12-29T08:19:41 [TS: 1735460381] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 (Updated: 2024-12-29T08:19:41) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a (Updated: 2024-12-30T08:19:44 [TS: 1735546784] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a (Updated: 2024-12-30T08:19:44) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 (Updated: 2024-12-31T08:19:20 [TS: 1735633160] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 (Updated: 2024-12-31T08:19:20) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a (Updated: 2025-01-01T08:20:09 [TS: 1735719609] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a (Updated: 2025-01-01T08:20:09) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 (Updated: 2025-01-02T08:19:36 [TS: 1735805976] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 (Updated: 2025-01-02T08:19:36) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 (Updated: 2025-01-03T08:19:51 [TS: 1735892391] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 (Updated: 2025-01-03T08:19:51) -[2025-11-30 15:08:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee (Updated: 2025-01-04T08:19:50 [TS: 1735978790] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:51] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee (Updated: 2025-01-04T08:19:50) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 (Updated: 2025-01-05T08:19:40 [TS: 1736065180] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 (Updated: 2025-01-05T08:19:40) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 (Updated: 2025-01-06T08:19:23 [TS: 1736151563] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 (Updated: 2025-01-06T08:19:23) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 (Updated: 2025-01-07T08:19:20 [TS: 1736237960] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 (Updated: 2025-01-07T08:19:20) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 (Updated: 2025-01-08T08:19:34 [TS: 1736324374] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 (Updated: 2025-01-08T08:19:34) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 (Updated: 2025-01-09T08:19:09 [TS: 1736410749] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 (Updated: 2025-01-09T08:19:09) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf (Updated: 2025-01-10T08:18:36 [TS: 1736497116] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf (Updated: 2025-01-10T08:18:36) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe (Updated: 2025-01-11T08:19:12 [TS: 1736583552] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe (Updated: 2025-01-11T08:19:12) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 (Updated: 2025-01-12T08:19:47 [TS: 1736669987] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 (Updated: 2025-01-12T08:19:47) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 (Updated: 2025-01-13T08:19:33 [TS: 1736756373] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 (Updated: 2025-01-13T08:19:33) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 (Updated: 2025-01-15T08:19:08 [TS: 1736929148] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 (Updated: 2025-01-15T08:19:08) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e (Updated: 2025-01-16T08:19:33 [TS: 1737015573] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e (Updated: 2025-01-16T08:19:33) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 (Updated: 2025-01-17T08:22:38 [TS: 1737102158] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 (Updated: 2025-01-17T08:22:38) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f (Updated: 2025-01-18T08:20:38 [TS: 1737188438] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f (Updated: 2025-01-18T08:20:38) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 (Updated: 2025-01-19T08:23:33 [TS: 1737275013] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 (Updated: 2025-01-19T08:23:33) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c (Updated: 2025-01-20T08:20:58 [TS: 1737361258] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c (Updated: 2025-01-20T08:20:58) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 (Updated: 2025-01-21T08:21:03 [TS: 1737447663] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 (Updated: 2025-01-21T08:21:03) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a (Updated: 2025-01-22T08:20:37 [TS: 1737534037] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a (Updated: 2025-01-22T08:20:37) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 (Updated: 2025-01-23T08:20:21 [TS: 1737620421] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 (Updated: 2025-01-23T08:20:21) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 (Updated: 2025-01-24T08:19:44 [TS: 1737706784] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 (Updated: 2025-01-24T08:19:44) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 (Updated: 2025-01-25T08:20:25 [TS: 1737793225] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 (Updated: 2025-01-25T08:20:25) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c (Updated: 2025-01-26T08:19:43 [TS: 1737879583] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c (Updated: 2025-01-26T08:19:43) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d (Updated: 2025-01-27T08:21:26 [TS: 1737966086] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d (Updated: 2025-01-27T08:21:26) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e (Updated: 2025-01-28T08:22:47 [TS: 1738052567] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e (Updated: 2025-01-28T08:22:47) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 (Updated: 2025-01-29T08:20:27 [TS: 1738138827] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 (Updated: 2025-01-29T08:20:27) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 (Updated: 2025-01-30T08:20:39 [TS: 1738225239] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 (Updated: 2025-01-30T08:20:39) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 (Updated: 2025-01-31T08:20:22 [TS: 1738311622] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 (Updated: 2025-01-31T08:20:22) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 (Updated: 2025-02-01T08:20:50 [TS: 1738398050] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 (Updated: 2025-02-01T08:20:50) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 (Updated: 2025-02-02T08:20:43 [TS: 1738484443] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 (Updated: 2025-02-02T08:20:43) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 (Updated: 2025-02-03T08:20:14 [TS: 1738570814] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 (Updated: 2025-02-03T08:20:14) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae (Updated: 2025-02-04T08:20:52 [TS: 1738657252] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae (Updated: 2025-02-04T08:20:52) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b (Updated: 2025-02-05T08:20:48 [TS: 1738743648] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b (Updated: 2025-02-05T08:20:48) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d (Updated: 2025-02-06T08:20:11 [TS: 1738830011] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d (Updated: 2025-02-06T08:20:11) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c (Updated: 2025-02-07T08:21:04 [TS: 1738916464] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c (Updated: 2025-02-07T08:21:04) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef (Updated: 2025-02-08T08:20:54 [TS: 1739002854] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef (Updated: 2025-02-08T08:20:54) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 (Updated: 2025-02-09T08:20:35 [TS: 1739089235] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 (Updated: 2025-02-09T08:20:35) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 (Updated: 2025-02-10T08:20:55 [TS: 1739175655] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 (Updated: 2025-02-10T08:20:55) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af (Updated: 2025-02-11T08:19:45 [TS: 1739261985] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af (Updated: 2025-02-11T08:19:45) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 (Updated: 2025-02-12T08:23:12 [TS: 1739348592] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 (Updated: 2025-02-12T08:23:12) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 (Updated: 2025-02-13T08:21:40 [TS: 1739434900] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 (Updated: 2025-02-13T08:21:40) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 (Updated: 2025-02-14T08:20:29 [TS: 1739521229] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 (Updated: 2025-02-14T08:20:29) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 (Updated: 2025-02-15T08:20:08 [TS: 1739607608] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 (Updated: 2025-02-15T08:20:08) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f (Updated: 2025-02-16T08:20:40 [TS: 1739694040] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f (Updated: 2025-02-16T08:20:40) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 (Updated: 2025-02-17T08:24:07 [TS: 1739780647] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 (Updated: 2025-02-17T08:24:07) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 (Updated: 2025-02-18T08:17:40 [TS: 1739866660] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 (Updated: 2025-02-18T08:17:40) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 (Updated: 2025-02-19T08:21:02 [TS: 1739953262] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 (Updated: 2025-02-19T08:21:02) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c (Updated: 2025-02-20T08:18:01 [TS: 1740039481] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c (Updated: 2025-02-20T08:18:01) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 (Updated: 2025-02-21T08:20:07 [TS: 1740126007] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 (Updated: 2025-02-21T08:20:07) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 (Updated: 2025-02-25T08:20:52 [TS: 1740471652] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 (Updated: 2025-02-25T08:20:52) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 (Updated: 2025-02-26T08:21:06 [TS: 1740558066] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 (Updated: 2025-02-26T08:21:06) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 (Updated: 2025-02-27T08:21:06 [TS: 1740644466] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 (Updated: 2025-02-27T08:21:06) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 (Updated: 2025-02-28T08:21:16 [TS: 1740730876] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 (Updated: 2025-02-28T08:21:16) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c (Updated: 2025-03-01T08:20:02 [TS: 1740817202] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c (Updated: 2025-03-01T08:20:02) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 (Updated: 2025-03-02T08:21:22 [TS: 1740903682] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 (Updated: 2025-03-02T08:21:22) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 (Updated: 2025-03-03T08:17:41 [TS: 1740989861] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 (Updated: 2025-03-03T08:17:41) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b (Updated: 2025-03-04T08:22:52 [TS: 1741076572] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b (Updated: 2025-03-04T08:22:52) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 (Updated: 2025-03-05T08:21:41 [TS: 1741162901] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 (Updated: 2025-03-05T08:21:41) -[2025-11-30 15:08:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 (Updated: 2025-03-06T08:21:32 [TS: 1741249292] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:52] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 (Updated: 2025-03-06T08:21:32) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc (Updated: 2025-03-07T08:19:43 [TS: 1741335583] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc (Updated: 2025-03-07T08:19:43) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c (Updated: 2025-03-08T08:19:23 [TS: 1741421963] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c (Updated: 2025-03-08T08:19:23) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a (Updated: 2025-03-09T08:18:08 [TS: 1741508288] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a (Updated: 2025-03-09T08:18:08) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b (Updated: 2025-03-10T07:21:20 [TS: 1741591280] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b (Updated: 2025-03-10T07:21:20) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e (Updated: 2025-03-11T07:20:43 [TS: 1741677643] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e (Updated: 2025-03-11T07:20:43) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e (Updated: 2025-03-12T07:19:34 [TS: 1741763974] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e (Updated: 2025-03-12T07:19:34) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 (Updated: 2025-03-13T07:20:51 [TS: 1741850451] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 (Updated: 2025-03-13T07:20:51) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 (Updated: 2025-03-14T07:20:37 [TS: 1741936837] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 (Updated: 2025-03-14T07:20:37) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 (Updated: 2025-03-15T07:21:23 [TS: 1742023283] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 (Updated: 2025-03-15T07:21:23) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 (Updated: 2025-03-16T07:20:44 [TS: 1742109644] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 (Updated: 2025-03-16T07:20:44) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 (Updated: 2025-03-17T07:21:09 [TS: 1742196069] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 (Updated: 2025-03-17T07:21:09) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 (Updated: 2025-03-18T07:22:07 [TS: 1742282527] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 (Updated: 2025-03-18T07:22:07) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac (Updated: 2025-03-19T07:21:36 [TS: 1742368896] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac (Updated: 2025-03-19T07:21:36) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a (Updated: 2025-03-20T07:21:15 [TS: 1742455275] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a (Updated: 2025-03-20T07:21:15) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 (Updated: 2025-03-21T07:20:54 [TS: 1742541654] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 (Updated: 2025-03-21T07:20:54) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 (Updated: 2025-03-22T07:21:14 [TS: 1742628074] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 (Updated: 2025-03-22T07:21:14) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f (Updated: 2025-03-23T07:20:50 [TS: 1742714450] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f (Updated: 2025-03-23T07:20:50) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca (Updated: 2025-03-24T07:21:04 [TS: 1742800864] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca (Updated: 2025-03-24T07:21:04) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f (Updated: 2025-03-25T07:21:17 [TS: 1742887277] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f (Updated: 2025-03-25T07:21:17) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 (Updated: 2025-03-26T07:20:59 [TS: 1742973659] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 (Updated: 2025-03-26T07:20:59) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a (Updated: 2025-03-27T07:20:48 [TS: 1743060048] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a (Updated: 2025-03-27T07:20:48) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 (Updated: 2025-03-28T07:20:36 [TS: 1743146436] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 (Updated: 2025-03-28T07:20:36) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 (Updated: 2025-03-29T07:21:17 [TS: 1743232877] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 (Updated: 2025-03-29T07:21:17) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 (Updated: 2025-03-30T07:21:38 [TS: 1743319298] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 (Updated: 2025-03-30T07:21:38) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 (Updated: 2025-03-31T07:20:18 [TS: 1743405618] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 (Updated: 2025-03-31T07:20:18) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 (Updated: 2025-04-01T07:21:20 [TS: 1743492080] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 (Updated: 2025-04-01T07:21:20) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d (Updated: 2025-04-02T07:20:23 [TS: 1743578423] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d (Updated: 2025-04-02T07:20:23) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 (Updated: 2025-04-03T07:20:49 [TS: 1743664849] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 (Updated: 2025-04-03T07:20:49) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b (Updated: 2025-04-04T07:20:41 [TS: 1743751241] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b (Updated: 2025-04-04T07:20:41) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a (Updated: 2025-04-05T07:20:21 [TS: 1743837621] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a (Updated: 2025-04-05T07:20:21) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 (Updated: 2025-04-06T07:23:45 [TS: 1743924225] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 (Updated: 2025-04-06T07:23:45) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 (Updated: 2025-04-07T07:20:24 [TS: 1744010424] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 (Updated: 2025-04-07T07:20:24) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee (Updated: 2025-04-08T07:22:43 [TS: 1744096963] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee (Updated: 2025-04-08T07:22:43) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 (Updated: 2025-04-09T07:20:20 [TS: 1744183220] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 (Updated: 2025-04-09T07:20:20) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 (Updated: 2025-04-10T07:21:00 [TS: 1744269660] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 (Updated: 2025-04-10T07:21:00) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 (Updated: 2025-04-11T07:20:54 [TS: 1744356054] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 (Updated: 2025-04-11T07:20:54) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 (Updated: 2025-04-12T07:21:40 [TS: 1744442500] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 (Updated: 2025-04-12T07:21:40) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 (Updated: 2025-04-13T07:20:59 [TS: 1744528859] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 (Updated: 2025-04-13T07:20:59) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c (Updated: 2025-04-14T07:20:24 [TS: 1744615224] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c (Updated: 2025-04-14T07:20:24) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 (Updated: 2025-04-15T07:20:46 [TS: 1744701646] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 (Updated: 2025-04-15T07:20:46) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 (Updated: 2025-04-16T07:20:12 [TS: 1744788012] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 (Updated: 2025-04-16T07:20:12) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 (Updated: 2025-04-17T07:19:55 [TS: 1744874395] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 (Updated: 2025-04-17T07:19:55) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb (Updated: 2025-04-18T07:20:41 [TS: 1744960841] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb (Updated: 2025-04-18T07:20:41) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa (Updated: 2025-04-19T07:21:15 [TS: 1745047275] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa (Updated: 2025-04-19T07:21:15) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 (Updated: 2025-04-20T07:21:25 [TS: 1745133685] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 (Updated: 2025-04-20T07:21:25) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d (Updated: 2025-04-21T07:21:59 [TS: 1745220119] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d (Updated: 2025-04-21T07:21:59) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a (Updated: 2025-04-22T07:21:34 [TS: 1745306494] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a (Updated: 2025-04-22T07:21:34) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 (Updated: 2025-04-23T07:21:06 [TS: 1745392866] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 (Updated: 2025-04-23T07:21:06) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 (Updated: 2025-04-24T07:21:59 [TS: 1745479319] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 (Updated: 2025-04-24T07:21:59) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 (Updated: 2025-04-25T07:20:22 [TS: 1745565622] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 (Updated: 2025-04-25T07:20:22) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d (Updated: 2025-04-26T07:20:43 [TS: 1745652043] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d (Updated: 2025-04-26T07:20:43) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da (Updated: 2025-04-27T07:21:38 [TS: 1745738498] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da (Updated: 2025-04-27T07:21:38) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 (Updated: 2025-04-28T07:21:01 [TS: 1745824861] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 (Updated: 2025-04-28T07:21:01) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 (Updated: 2025-04-29T07:21:25 [TS: 1745911285] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 (Updated: 2025-04-29T07:21:25) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 (Updated: 2025-04-30T07:20:20 [TS: 1745997620] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 (Updated: 2025-04-30T07:20:20) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 (Updated: 2025-05-01T07:21:24 [TS: 1746084084] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 (Updated: 2025-05-01T07:21:24) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 (Updated: 2025-05-02T07:21:04 [TS: 1746170464] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 (Updated: 2025-05-02T07:21:04) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 (Updated: 2025-05-03T07:21:59 [TS: 1746256919] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:53] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 (Updated: 2025-05-03T07:21:59) -[2025-11-30 15:08:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 (Updated: 2025-05-04T07:21:07 [TS: 1746343267] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 (Updated: 2025-05-04T07:21:07) -[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 (Updated: 2025-05-05T07:21:39 [TS: 1746429699] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 (Updated: 2025-05-05T07:21:39) -[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 (Updated: 2025-05-06T07:21:39 [TS: 1746516099] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 (Updated: 2025-05-06T07:21:39) -[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb (Updated: 2025-05-07T07:22:18 [TS: 1746602538] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb (Updated: 2025-05-07T07:22:18) -[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d (Updated: 2025-05-08T07:22:24 [TS: 1746688944] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d (Updated: 2025-05-08T07:22:24) -[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b (Updated: 2025-05-09T07:21:16 [TS: 1746775276] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b (Updated: 2025-05-09T07:21:16) -[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 (Updated: 2025-05-10T07:21:20 [TS: 1746861680] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 (Updated: 2025-05-10T07:21:20) -[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 (Updated: 2025-05-11T07:20:26 [TS: 1746948026] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 (Updated: 2025-05-11T07:20:26) -[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 (Updated: 2025-05-12T07:21:06 [TS: 1747034466] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 (Updated: 2025-05-12T07:21:06) -[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 (Updated: 2025-05-13T07:21:27 [TS: 1747120887] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 (Updated: 2025-05-13T07:21:27) -[2025-11-30 15:08:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 (Updated: 2025-05-14T07:21:24 [TS: 1747207284] < Cutoff: [TS: 1763305724]) -[2025-11-30 15:08:54] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 (Updated: 2025-05-14T07:21:24) -[2025-11-30 15:08:54] [INFO] Hit delete limit (200) for Docker Images. -[2025-11-30 15:08:54] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 15:08:54] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 15:08:56] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 15:08:56] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 15:08:56] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 15:08:56] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 15:08:56] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 15:08:56] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 15:08:59] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 15:08:59] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:09:01] [INFO] No Regional Address found matching criteria. -[2025-11-30 15:09:01] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:09:03] [INFO] No Global Address found matching criteria. -[2025-11-30 15:09:03] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- -[2025-11-30 15:09:08] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 15:09:10] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:09:10] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:09:10] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 15:09:10] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 15:09:10] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:13] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 15:09:16] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:09:16] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 15:09:17] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 15:09:17] [INFO] CLEANUP RUN FINISHED -[2025-11-30 15:10:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 15:10:29] [INFO] Time Cutoff (General): 2025-11-30T15:10:29+0000 -[2025-11-30 15:10:29] [INFO] Time Cutoff (Images): 2025-10-01T15:10:29+0000 -[2025-11-30 15:10:29] [INFO] Delete Limit per Type: 200 -[2025-11-30 15:10:29] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 15:10:30] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 15:10:32] [INFO] No Service Accounts found matching prefix. -[2025-11-30 15:10:32] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:10:34] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 15:10:34] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 15:10:37] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:10:37] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:10:37] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 15:10:37] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:10:40] [INFO] No Filestore instances found matching criteria. -[2025-11-30 15:10:40] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 15:10:43] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 15:10:43] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 15:10:43] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 15:10:43] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 15:10:43] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 15:10:43] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 15:10:43] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 15:10:43] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 15:10:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 15:10:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 15:10:44] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 15:10:44] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:10:44Z (Unix: 1763305844) -[2025-11-30 15:10:44] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 15:10:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 (Updated: 2024-10-24T07:21:12 [TS: 1729754472] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:10:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 (Updated: 2024-10-24T07:21:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1d72fb41-5046-4044-a316-d2b78c8f8cb3] to complete... -.....done. -[2025-11-30 15:10:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1d444d4ac1bca4417fca6e43c4c54d85213846d8029f7e21892738f7d923cf57 -[2025-11-30 15:10:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 (Updated: 2024-10-25T07:19:11 [TS: 1729840751] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:10:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 (Updated: 2024-10-25T07:19:11) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3de8a708-6e9a-4a23-b990-68d5f9007748] to complete... -.....done. -[2025-11-30 15:10:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec95f2c8312765361af379b87be722fa7c1360efe31a80208ae596fccaeb9401 -[2025-11-30 15:10:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea (Updated: 2024-10-26T07:19:04 [TS: 1729927144] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:10:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea (Updated: 2024-10-26T07:19:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/395c03bc-1131-4bff-b308-a7b41251ba7b] to complete... -......done. -[2025-11-30 15:11:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5363e83e6acd8b95e21d9dfd957a87d244bcdf85c7da409a0c6c73d6803bbeea -[2025-11-30 15:11:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 (Updated: 2024-10-27T07:18:59 [TS: 1730013539] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 (Updated: 2024-10-27T07:18:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/02096d61-a872-4a5e-a333-bd255c67dd7b] to complete... -......done. -[2025-11-30 15:11:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fd439f60236d7bd1ffc82a352f5506f7b15eb8a2502cdb3773131e530ddca248 -[2025-11-30 15:11:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f (Updated: 2024-10-28T07:19:53 [TS: 1730099993] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f (Updated: 2024-10-28T07:19:53) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fb615cd5-97ab-4b29-bc51-c993addcff73] to complete... -......done. -[2025-11-30 15:11:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df80742ac37ad948ad828daf26dab595768cc060e21dd51a20d35856045fa78f -[2025-11-30 15:11:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa (Updated: 2024-10-29T07:21:32 [TS: 1730186492] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa (Updated: 2024-10-29T07:21:32) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0246878c-20aa-45a0-9613-9dbb21d0568f] to complete... -......done. -[2025-11-30 15:11:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bfbe592d048279df3ba2de97da030d7c2efcf142f09bfea2e0dd0b01448b97aa -[2025-11-30 15:11:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c (Updated: 2024-10-30T07:20:09 [TS: 1730272809] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c (Updated: 2024-10-30T07:20:09) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cfa4b060-0d32-4142-94b6-b0e97028c7d8] to complete... -.....done. -[2025-11-30 15:11:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:72e7c1da630151f74ffe294019307af6d281bddd7c1650a2d6748a71661e827c -[2025-11-30 15:11:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca (Updated: 2024-10-31T07:19:50 [TS: 1730359190] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca (Updated: 2024-10-31T07:19:50) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1ba620f4-2f1f-4c24-9b6a-f313785b452a] to complete... -.....done. -[2025-11-30 15:11:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:346f890bfb707d74c48facf5cabef357562e806e2db6647b816ca0a5aeaaa1ca -[2025-11-30 15:11:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb (Updated: 2024-11-01T07:19:35 [TS: 1730445575] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb (Updated: 2024-11-01T07:19:35) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2f84b137-270b-4543-945d-d9ae70c2410e] to complete... -......done. -[2025-11-30 15:11:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca835aa984f253055fce45a74546163e42dc0ba7e5869984ef0d5537340bdbb -[2025-11-30 15:11:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 (Updated: 2024-11-02T07:19:20 [TS: 1730531960] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 (Updated: 2024-11-02T07:19:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b5ecf31d-9a9c-474e-b87d-5c2d174b1de7] to complete... -......done. -[2025-11-30 15:11:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b90cffa10a3f82c3bafc4122f4df842de331a93de342070ea205b824aaffc540 -[2025-11-30 15:11:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b (Updated: 2024-11-03T07:19:34 [TS: 1730618374] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b (Updated: 2024-11-03T07:19:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5f3e8277-dd77-4433-8d65-a183fc08a534] to complete... -......done. -[2025-11-30 15:11:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:992ed731de15af1e735bdacc24bc5891c462e1c317db6fef571e187c09a9ca3b -[2025-11-30 15:11:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc (Updated: 2024-11-04T08:20:03 [TS: 1730708403] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc (Updated: 2024-11-04T08:20:03) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/27069aa2-141e-45fc-8d80-c7e7be7b5fe7] to complete... -......done. -[2025-11-30 15:11:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c38ff21b901c8fbad530660ad60168a9d1ef6819094a2f6aeedf1de39001d8cc -[2025-11-30 15:11:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed (Updated: 2024-11-05T08:19:36 [TS: 1730794776] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed (Updated: 2024-11-05T08:19:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/908020ef-fac3-4535-a381-3d44e555daa4] to complete... -......done. -[2025-11-30 15:11:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5dd1f4c07ed2341f114a4883c4012d1769d99406dcfccf24b954fdba82a3cbed -[2025-11-30 15:11:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e (Updated: 2024-11-06T08:19:42 [TS: 1730881182] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e (Updated: 2024-11-06T08:19:42) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/74ab1756-6c67-499c-8b7b-e058f065f6bb] to complete... -.....done. -[2025-11-30 15:11:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:915b8f530a440d5b9183f240ca4438fc292c59dd0f677ba1e6ed0c14d807105e -[2025-11-30 15:11:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 (Updated: 2024-11-07T08:22:39 [TS: 1730967759] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 (Updated: 2024-11-07T08:22:39) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/90d71dca-2654-4cdb-8293-79c6ee79a908] to complete... -.....done. -[2025-11-30 15:11:48] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eea65806a4dded1f467426b53941a60f3fe7d46a953f1150ec854f3da2b0d5c1 -[2025-11-30 15:11:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 (Updated: 2024-11-08T08:19:05 [TS: 1731053945] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 (Updated: 2024-11-08T08:19:05) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f3003f49-8aa2-4f19-91ee-e9bce6e44c19] to complete... -......done. -[2025-11-30 15:11:52] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:524fa568590810cc23d288471f7ec28e6d682671c400879d9f486bfe62e8ec96 -[2025-11-30 15:11:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 (Updated: 2024-11-09T08:19:52 [TS: 1731140392] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:52] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 (Updated: 2024-11-09T08:19:52) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9c7788c1-18ab-4b6b-927f-2a3412c69f8e] to complete... -......done. -[2025-11-30 15:11:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd47b4daccba10222274d5752cdaa8bcf9e9e1a2bdb9a4cf732531a79f30f777 -[2025-11-30 15:11:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 (Updated: 2024-11-10T08:19:21 [TS: 1731226761] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 (Updated: 2024-11-10T08:19:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/95594b98-7af2-4441-93a4-e449d360acda] to complete... -.....done. -[2025-11-30 15:11:59] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27f2d140cea0c010e1fdfb0f2d7f0c92d310a68e1f6e09cef0c62c94d3564279 -[2025-11-30 15:11:59] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e (Updated: 2024-11-11T08:18:44 [TS: 1731313124] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:11:59] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e (Updated: 2024-11-11T08:18:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f678e910-b8be-4213-a8c3-3030be9a422d] to complete... -.....done. -[2025-11-30 15:12:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:905e2b49133fe65cb3db1de317ee35a724bad2c0fac7381cf2e2f0baccacd99e -[2025-11-30 15:12:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c (Updated: 2024-11-12T08:20:12 [TS: 1731399612] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c (Updated: 2024-11-12T08:20:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bf899df0-0928-4103-ae7d-3474e906632a] to complete... -......done. -[2025-11-30 15:12:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:173f8305e07148ef8b8c4addeb8ee548475ba1a95d1483162294e087dae65f5c -[2025-11-30 15:12:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d (Updated: 2024-11-13T08:18:46 [TS: 1731485926] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d (Updated: 2024-11-13T08:18:46) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/80a2932b-60b5-430e-9c03-e061fac2c639] to complete... -......done. -[2025-11-30 15:12:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8d3756c8626039e44bfd1a42fcb44821b2e9d82728285a39c8ebd8c7333094d -[2025-11-30 15:12:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 (Updated: 2024-11-14T08:19:32 [TS: 1731572372] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 (Updated: 2024-11-14T08:19:32) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d61ebc54-ae66-4ed6-bb92-db4795d2fba4] to complete... -.....done. -[2025-11-30 15:12:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:024adb2ee143d813e87de7dc491c8dda3aa9b2ee87c0105dbe469ff255443527 -[2025-11-30 15:12:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa (Updated: 2024-11-15T08:18:37 [TS: 1731658717] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa (Updated: 2024-11-15T08:18:37) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2e06c71b-5735-49d1-919f-a44390d1b65c] to complete... -.....done. -[2025-11-30 15:12:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebead272df7e5298a825fbf276dae26d7b8473b4e954f825bc794645c4c060aa -[2025-11-30 15:12:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 (Updated: 2024-11-16T08:19:38 [TS: 1731745178] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 (Updated: 2024-11-16T08:19:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/20d51519-7b64-44de-85b8-5ea71ac24823] to complete... -.....done. -[2025-11-30 15:12:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:755c2a51540136458481d0d167d4b0eba8baaf000f8d683667b8444c73babb69 -[2025-11-30 15:12:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 (Updated: 2024-11-17T08:19:08 [TS: 1731831548] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 (Updated: 2024-11-17T08:19:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/98cffb83-76f1-4e75-9f90-f551e1c139a1] to complete... -......done. -[2025-11-30 15:12:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e72c1c02a77eb3399fc65bba857e15cecfc2dd5c285eb17aab90182678a9573 -[2025-11-30 15:12:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c (Updated: 2024-11-18T08:19:40 [TS: 1731917980] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c (Updated: 2024-11-18T08:19:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/89f233e7-7dc3-4216-9ab9-a79aa856975d] to complete... -......done. -[2025-11-30 15:12:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be8f68b0966c5c97f08ffa0ea5b4ce155995d0878689f9f7d8df550e8464d79c -[2025-11-30 15:12:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e (Updated: 2024-11-19T08:19:29 [TS: 1732004369] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e (Updated: 2024-11-19T08:19:29) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6e7a3baf-1ec2-427b-95ce-1203f0150f7a] to complete... -......done. -[2025-11-30 15:12:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399e4d6e9d05186c5a7bfafdf6cc9e361d1b818d53d9fc6d3a2f2ce913b3d79e -[2025-11-30 15:12:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 (Updated: 2024-11-20T08:19:24 [TS: 1732090764] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 (Updated: 2024-11-20T08:19:24) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/059d29e8-cce5-4970-9b02-52b04188f041] to complete... -.....done. -[2025-11-30 15:12:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1196cb88bae5dcc04a70a6c9db95e9d7884d0a93b8d8a2b7b5681f35581e940 -[2025-11-30 15:12:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf (Updated: 2024-11-21T08:19:48 [TS: 1732177188] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf (Updated: 2024-11-21T08:19:48) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d4aa1482-d133-41e4-8a83-bb0cd334acbe] to complete... -......done. -[2025-11-30 15:12:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:625f44932a179eec758b3bd540028848787ac0062c847f617b9bd05e4e2490bf -[2025-11-30 15:12:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 (Updated: 2024-11-22T08:19:50 [TS: 1732263590] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 (Updated: 2024-11-22T08:19:50) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/21c13c9b-fc83-4fe9-b780-cc92769af79f] to complete... -......done. -[2025-11-30 15:12:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52950417e2943eb3f561f34fcb7b84f29b2526f4c67d8dd497f629816e81fb34 -[2025-11-30 15:12:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb (Updated: 2024-11-23T08:18:52 [TS: 1732349932] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb (Updated: 2024-11-23T08:18:52) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c3493829-3493-4cb8-babe-e8733460d3d8] to complete... -......done. -[2025-11-30 15:12:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55d7c8d9fae32536538d43a822a37c21dae284094b68d06e1d3220301b2143fb -[2025-11-30 15:12:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 (Updated: 2024-11-24T08:19:55 [TS: 1732436395] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 (Updated: 2024-11-24T08:19:55) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bfd83599-ec9c-4944-80d8-6643b6c82c5c] to complete... -.....done. -[2025-11-30 15:12:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:02dacd013151d5f79407e655a0d7e243187598a74cf3db326f8363f6d61f1785 -[2025-11-30 15:12:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 (Updated: 2024-11-25T08:18:26 [TS: 1732522706] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 (Updated: 2024-11-25T08:18:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8b465f77-040c-492c-97ca-8c55a29ab780] to complete... -......done. -[2025-11-30 15:12:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d5abcb89cd4d213c083176fa05708112d3735970ccae0ee69eecde1451c5fc72 -[2025-11-30 15:12:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e (Updated: 2024-11-26T08:19:19 [TS: 1732609159] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:12:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e (Updated: 2024-11-26T08:19:19) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2e49a45b-963a-4ed5-8fda-de2386c86f39] to complete... -.....done. -[2025-11-30 15:13:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:df4f0d06a8052b4b79f3f908bb7edef72420415cdee8233794151eb27edc2d8e -[2025-11-30 15:13:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f (Updated: 2024-11-27T08:19:05 [TS: 1732695545] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f (Updated: 2024-11-27T08:19:05) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a425b84f-6836-420a-8987-8bc4b50a95f7] to complete... -......done. -[2025-11-30 15:13:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5011744bee7c0183c89b34d877a4039d677c150163167bda4dc7dfb04619039f -[2025-11-30 15:13:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af (Updated: 2024-11-28T08:18:54 [TS: 1732781934] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af (Updated: 2024-11-28T08:18:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e2d546a4-f58d-484f-9d11-268ecde1dbd4] to complete... -.....done. -[2025-11-30 15:13:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7835deafc71583f7d3d5678e07f46f3033806a00f7b58ce18e75734837eb18af -[2025-11-30 15:13:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 (Updated: 2024-11-29T08:19:25 [TS: 1732868365] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 (Updated: 2024-11-29T08:19:25) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/200ea5c8-0603-4866-a491-152d7866ba66] to complete... -......done. -[2025-11-30 15:13:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cc2f453e0de3fb5ba2b681c676a2ca5dd8e1b0f15ec2603ad9359ff688aae475 -[2025-11-30 15:13:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db (Updated: 2024-11-30T08:19:26 [TS: 1732954766] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db (Updated: 2024-11-30T08:19:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2b506b8f-3309-4a37-96c8-d14015a38604] to complete... -.....done. -[2025-11-30 15:13:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c3a2c3f2573758cce3225c3da64a76f9bf9237d7715095bb7aa96eb6e87839db -[2025-11-30 15:13:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b (Updated: 2024-12-01T08:20:11 [TS: 1733041211] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b (Updated: 2024-12-01T08:20:11) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7cc6888a-c2d4-4176-82ba-aea96989929a] to complete... -.....done. -[2025-11-30 15:13:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ca12d63180b640c4276845fbd8d3a29b7c0211b64f191524a7aa8e99bf91b9b -[2025-11-30 15:13:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c (Updated: 2024-12-02T08:18:21 [TS: 1733127501] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c (Updated: 2024-12-02T08:18:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/28f95d0f-4dbf-4043-b45d-b2086a837b18] to complete... -......done. -[2025-11-30 15:13:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a2429bb4ca7d2265808434e00f3ce7406c3fa63b5dbbf44fe1a2cf14446ec13c -[2025-11-30 15:13:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff (Updated: 2024-12-03T08:19:17 [TS: 1733213957] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff (Updated: 2024-12-03T08:19:17) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ff423dc4-8ff3-4d19-9545-a06179659408] to complete... -.....done. -[2025-11-30 15:13:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ada1f7baa6df08a5adbbf637f8a3ed5bb8a801e7c05cd36a4619f327b72a4ff -[2025-11-30 15:13:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 (Updated: 2024-12-04T08:20:29 [TS: 1733300429] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 (Updated: 2024-12-04T08:20:29) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f6b39aae-8d7d-4df0-94a1-cb934d1235f4] to complete... -.....done. -[2025-11-30 15:13:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2935decab50077ed81396d430d2806dfcb386e95d2509af5e0128999df5c09d8 -[2025-11-30 15:13:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 (Updated: 2024-12-04T23:35:41 [TS: 1733355341] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 (Updated: 2024-12-04T23:35:41) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/915fc1b0-cc0c-462d-bf79-30c623e4111c] to complete... -......done. -[2025-11-30 15:13:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9e7eddb59b4779e63de6e8e896597ca66d553769a0864519a4d382e6be66ce00 -[2025-11-30 15:13:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e (Updated: 2024-12-05T08:19:43 [TS: 1733386783] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e (Updated: 2024-12-05T08:19:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8499967c-7465-4379-a907-2a830827bf91] to complete... -.....done. -[2025-11-30 15:13:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34de01ae27b5c71a47d8550eec271f201d8e8e3a0db545bb18d69d7b50e8090e -[2025-11-30 15:13:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 (Updated: 2024-12-06T08:19:51 [TS: 1733473191] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 (Updated: 2024-12-06T08:19:51) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9a8bd1d5-f131-4f70-8ee9-a598c447836c] to complete... -......done. -[2025-11-30 15:13:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f414509d3dac80b2739d5f127d8c5b3a76488fc714e1cd7fe74a598147c05145 -[2025-11-30 15:13:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da (Updated: 2024-12-07T08:19:02 [TS: 1733559542] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da (Updated: 2024-12-07T08:19:02) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bcb835c6-30b7-4591-aca6-dd7f853680ee] to complete... -......done. -[2025-11-30 15:13:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9346d062a03b582144bf45355bd2b7a7936385d7004440833ba3c29c8f36d0da -[2025-11-30 15:13:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d (Updated: 2024-12-08T08:18:49 [TS: 1733645929] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d (Updated: 2024-12-08T08:18:49) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6a0a6bed-2f06-48d0-ac50-861d46964648] to complete... -......done. -[2025-11-30 15:13:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f99b82be877a0e84a7f20d38b3328743f85dde3c4a3a67d0ba86163b86b9f78d -[2025-11-30 15:13:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e (Updated: 2024-12-09T08:18:56 [TS: 1733732336] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e (Updated: 2024-12-09T08:18:56) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d7b8649b-d5ea-4308-815d-5aff95ce3ba4] to complete... -......done. -[2025-11-30 15:13:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3e29aaf8eb6e48804897bcd09c51676c2ae91c991f6b5e46bde3b3949459421e -[2025-11-30 15:13:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f (Updated: 2024-12-10T08:20:41 [TS: 1733818841] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f (Updated: 2024-12-10T08:20:41) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e075644b-a6b1-48a7-88bd-4fd8a60ff043] to complete... -......done. -[2025-11-30 15:13:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a4721386933d0b5bbeed38ac4ac2011cba58372043796e6d1a4518a8c55330f -[2025-11-30 15:13:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba (Updated: 2024-12-11T08:19:22 [TS: 1733905162] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:13:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba (Updated: 2024-12-11T08:19:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eead26ba-ee59-4a46-b1e2-542e761cc320] to complete... -......done. -[2025-11-30 15:14:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:25297fdf9273c8f0c87a205c3a06506d08255e70811ab71523196eabd7aee5ba -[2025-11-30 15:14:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 (Updated: 2024-12-12T08:19:21 [TS: 1733991561] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 (Updated: 2024-12-12T08:19:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/aa386bf8-39d6-46c7-b09f-153a0c4263a5] to complete... -.....done. -[2025-11-30 15:14:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4526270acd6a1ef7cd4095df8492c4101dcef4feb3f6f602b00bb536e8226181 -[2025-11-30 15:14:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb (Updated: 2024-12-13T08:19:59 [TS: 1734077999] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb (Updated: 2024-12-13T08:19:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/28e66f6f-4acb-420e-b798-02cd16857a99] to complete... -.....done. -[2025-11-30 15:14:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9891f13d65c4be95959de5a354d14fd942336359887a3b55336599352e1329bb -[2025-11-30 15:14:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 (Updated: 2024-12-14T08:19:12 [TS: 1734164352] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 (Updated: 2024-12-14T08:19:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/14499298-53ca-484d-a5f4-107826a5fc45] to complete... -......done. -[2025-11-30 15:14:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dd89c4d884d0bdc0ac64eb4372d9c3f1bcf66dc6b347eb5f4989d72d17ad78a2 -[2025-11-30 15:14:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b (Updated: 2024-12-15T08:19:07 [TS: 1734250747] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b (Updated: 2024-12-15T08:19:07) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/db3a4ba6-eaa6-4625-bab1-c2349190db13] to complete... -.....done. -[2025-11-30 15:14:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69cf63db819ca53d3648ad1a4306c00324c90eef310a97e18f72adf75dcc2c3b -[2025-11-30 15:14:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 (Updated: 2024-12-16T08:18:40 [TS: 1734337120] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 (Updated: 2024-12-16T08:18:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cf627d8f-b714-4538-af41-c9b9e125f514] to complete... -......done. -[2025-11-30 15:14:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b0c3f67d862193366a10f42ad94d4d2be04b7ab4ecb2d5d8ef378c04349b1eb9 -[2025-11-30 15:14:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 (Updated: 2024-12-17T08:19:36 [TS: 1734423576] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 (Updated: 2024-12-17T08:19:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2a3c7787-ae3a-477a-bcea-f55635ae87a5] to complete... -.....done. -[2025-11-30 15:14:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3a824491045fbf3d6efa7af6d3eaf0c3d6a39060c1403f29be563a8032b1ce85 -[2025-11-30 15:14:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 (Updated: 2024-12-18T08:19:28 [TS: 1734509968] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 (Updated: 2024-12-18T08:19:28) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/111726e2-7009-4e6b-9292-a34da8ae7774] to complete... -......done. -[2025-11-30 15:14:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9250c3327403ae62ea9efd0933bd6ae3cf565b292cb335e6fa0592dcd11d2284 -[2025-11-30 15:14:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 (Updated: 2024-12-19T08:19:45 [TS: 1734596385] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 (Updated: 2024-12-19T08:19:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9ddb08d5-d5c1-4efb-9072-1971d959bda5] to complete... -......done. -[2025-11-30 15:14:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e725d698a6d1d218400d49f6eee12b30106546152a487eeb8b00981c0eb9e461 -[2025-11-30 15:14:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 (Updated: 2024-12-20T08:20:22 [TS: 1734682822] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 (Updated: 2024-12-20T08:20:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f7254158-8896-49ce-85d7-ebbc79126364] to complete... -.....done. -[2025-11-30 15:14:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4cc6bf09da06d01d6864e970ac3862f208f783a7beb498d4e8d5c5a3638d4aa8 -[2025-11-30 15:14:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b (Updated: 2024-12-21T08:18:43 [TS: 1734769123] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b (Updated: 2024-12-21T08:18:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/17b0a4e9-2e49-4273-a483-0edeb749430b] to complete... -......done. -[2025-11-30 15:14:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:071d565dda4a4aa7a61c3dcddc2e38b306899c1b1ae313768dcc64fdaf276e0b -[2025-11-30 15:14:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 (Updated: 2024-12-22T08:19:03 [TS: 1734855543] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 (Updated: 2024-12-22T08:19:03) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c49e96e4-faae-490a-99cc-9ad29ade3d0f] to complete... -.....done. -[2025-11-30 15:14:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cd5938969e52c99ca755e7381d602a09e6304daf0d3de36fa600461db38e6f5 -[2025-11-30 15:14:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 (Updated: 2024-12-23T08:19:14 [TS: 1734941954] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 (Updated: 2024-12-23T08:19:14) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a712c0a0-3b1c-48a5-85fb-7083d9189258] to complete... -.....done. -[2025-11-30 15:14:48] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:feb1c4604a486c6f79bbcaf4625ebc2159db695f4d28ee65d5e62e578b476226 -[2025-11-30 15:14:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 (Updated: 2024-12-24T08:19:47 [TS: 1735028387] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 (Updated: 2024-12-24T08:19:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/31470aaf-d18e-4e4f-bd14-8b6cc6cbceaa] to complete... -......done. -[2025-11-30 15:14:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:98f192dc43038c36d689a203ba04d57dd2891613eb0199587f65d4f0a6335264 -[2025-11-30 15:14:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c (Updated: 2024-12-25T08:19:25 [TS: 1735114765] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c (Updated: 2024-12-25T08:19:25) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/237f0e19-5c0b-4405-9581-ad1fdd1de6ef] to complete... -.....done. -[2025-11-30 15:14:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cb3d86342ac76ace77ed13d4c500d3fba2066501b1aa2dc27e9dca2175ee094c -[2025-11-30 15:14:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d (Updated: 2024-12-26T08:19:08 [TS: 1735201148] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d (Updated: 2024-12-26T08:19:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cd8e148f-f667-47aa-815f-9ea7f38d86b1] to complete... -......done. -[2025-11-30 15:14:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46def83a803aeb0140bbb4d27cb0adaf2976c46dde749f4128ef9bc7720ac56d -[2025-11-30 15:14:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b (Updated: 2024-12-27T08:19:18 [TS: 1735287558] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:14:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b (Updated: 2024-12-27T08:19:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/17f120df-8744-4a79-b668-69fe6875d33d] to complete... -.....done. -[2025-11-30 15:15:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3c14cee74ddfdd81c9204272fb7e5ae34cafc4a69596097562c9b35cd54b4b -[2025-11-30 15:15:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 (Updated: 2024-12-28T08:20:48 [TS: 1735374048] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 (Updated: 2024-12-28T08:20:48) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f89727ee-b535-4afa-adb2-71e2103a930e] to complete... -.....done. -[2025-11-30 15:15:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03d1e4f611311609201d75eeb990098410c5abc4f50628153eb66eadd671bd98 -[2025-11-30 15:15:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 (Updated: 2024-12-29T08:19:41 [TS: 1735460381] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 (Updated: 2024-12-29T08:19:41) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c8d94839-5258-4a80-889e-483f2a303027] to complete... -.....done. -[2025-11-30 15:15:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:29e66e98ff8c4e9420199ee9aa160028cddeaba20d658bb1ec2878c88af0d7c0 -[2025-11-30 15:15:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a (Updated: 2024-12-30T08:19:44 [TS: 1735546784] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a (Updated: 2024-12-30T08:19:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/40b2ad56-f54c-4960-bec4-4933a34dacf6] to complete... -......done. -[2025-11-30 15:15:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2ff07bf1378ae9e99d5c03a6a079caf091bf3581e41d0c0946a2398cbbe7a8a -[2025-11-30 15:15:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 (Updated: 2024-12-31T08:19:20 [TS: 1735633160] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 (Updated: 2024-12-31T08:19:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/64223c45-2771-4835-bb21-cf21cea7ef4f] to complete... -.....done. -[2025-11-30 15:15:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:91de0bcfe608a9f6d3ead6e22b2322001cfa2730de57c5aa7d6ef5cba8afd625 -[2025-11-30 15:15:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a (Updated: 2025-01-01T08:20:09 [TS: 1735719609] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a (Updated: 2025-01-01T08:20:09) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8c0e6484-e4ea-4f83-8694-da8db62b099d] to complete... -......done. -[2025-11-30 15:15:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6c1a8c2dd0899b2c4abdf1be2cabae3856845eac581401449df861dbb6b009a -[2025-11-30 15:15:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 (Updated: 2025-01-02T08:19:36 [TS: 1735805976] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 (Updated: 2025-01-02T08:19:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f9d381f8-594d-43fa-a3b4-13ead53891b9] to complete... -.....done. -[2025-11-30 15:15:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4554eaa1293fb49e00563fdd9604044b688a862de1ab1183d8d7f715329c0fe2 -[2025-11-30 15:15:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 (Updated: 2025-01-03T08:19:51 [TS: 1735892391] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 (Updated: 2025-01-03T08:19:51) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/765cfec0-1291-48ca-89dc-682ac1223e11] to complete... -.....done. -[2025-11-30 15:15:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1385cb71eabdb2725086c2df2b70e6462a6e262499aa1c6ef47d50a0047a2ee9 -[2025-11-30 15:15:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee (Updated: 2025-01-04T08:19:50 [TS: 1735978790] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee (Updated: 2025-01-04T08:19:50) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/221b1a8f-874c-480a-9ae2-24ce17f5ea03] to complete... -......done. -[2025-11-30 15:15:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:446cada7613862e954c68771473671191705c195a5ea6633facc93cb9d83d5ee -[2025-11-30 15:15:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 (Updated: 2025-01-05T08:19:40 [TS: 1736065180] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 (Updated: 2025-01-05T08:19:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/65d93cad-f11b-4269-9cec-7b1a501c1f62] to complete... -......done. -[2025-11-30 15:15:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1540e08d75093a7a9d5453b43182fa7005e9062936c42c05324b2c419a815024 -[2025-11-30 15:15:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 (Updated: 2025-01-06T08:19:23 [TS: 1736151563] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 (Updated: 2025-01-06T08:19:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7260811b-0cbb-460e-8123-b47d96de6a03] to complete... -.....done. -[2025-11-30 15:15:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f49bb8c9a0f59f61f487a7fac55fe16b07807110a4933db8c0ff848fdff6f076 -[2025-11-30 15:15:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 (Updated: 2025-01-07T08:19:20 [TS: 1736237960] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 (Updated: 2025-01-07T08:19:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/38cc0369-3b20-4814-9f0c-3831a4189aa8] to complete... -......done. -[2025-11-30 15:15:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:044e4aa7a044914af5cd6ffc3646f03f0248d8e70955eb3e536469760330c2d4 -[2025-11-30 15:15:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 (Updated: 2025-01-08T08:19:34 [TS: 1736324374] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 (Updated: 2025-01-08T08:19:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bdbdaae3-7daf-438e-b877-6d7a2d90886c] to complete... -.....done. -[2025-11-30 15:15:45] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6893c7e327468fc258ac4620d551f79b007469bd150458ffdf1924cc017b3e99 -[2025-11-30 15:15:45] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 (Updated: 2025-01-09T08:19:09 [TS: 1736410749] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:45] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 (Updated: 2025-01-09T08:19:09) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/26ff91e3-1c3a-4324-a0e3-86f81b58deb7] to complete... -.....done. -[2025-11-30 15:15:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6563890dc3ebbeecf5d3fbbe8e0dc4c7ccc419d1755c51b7604d91a7bd6524e3 -[2025-11-30 15:15:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf (Updated: 2025-01-10T08:18:36 [TS: 1736497116] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf (Updated: 2025-01-10T08:18:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7db598b0-d8ba-4320-b345-3683a9324247] to complete... -......done. -[2025-11-30 15:15:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fbb2e149c778f2df975401a70f4aeddcd51c104dd96458542e35c427d8c14caf -[2025-11-30 15:15:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe (Updated: 2025-01-11T08:19:12 [TS: 1736583552] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe (Updated: 2025-01-11T08:19:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a1e90c63-f71d-484d-97b8-ad5244cd13e2] to complete... -.....done. -[2025-11-30 15:15:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:85da7591ac407c14e8f334ccd64d169dbb19b8fd21a9671f65c435851817fbbe -[2025-11-30 15:15:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 (Updated: 2025-01-12T08:19:47 [TS: 1736669987] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:15:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 (Updated: 2025-01-12T08:19:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5e353a84-2938-44a4-a72e-82c72994bad0] to complete... -.....done. -[2025-11-30 15:16:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b6aa42d1b2d5ac00a58b50cc1ea6c12a93a681a9bf974b6bbdaed2f9a719e7e5 -[2025-11-30 15:16:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 (Updated: 2025-01-13T08:19:33 [TS: 1736756373] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 (Updated: 2025-01-13T08:19:33) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b8ee05d9-a556-433e-83fe-2147254e5edd] to complete... -.....done. -[2025-11-30 15:16:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:14f89c3d8fe5085a25d79bde0256255bf658d90127ec8ba4f6acfb944f3b9412 -[2025-11-30 15:16:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 (Updated: 2025-01-15T08:19:08 [TS: 1736929148] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 (Updated: 2025-01-15T08:19:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0f13fce1-f4e5-429b-9359-a23c118bcf62] to complete... -.....done. -[2025-11-30 15:16:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:667534ca88d813c987f364121891ac671a13f085233d81e0e768d5badb977e11 -[2025-11-30 15:16:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e (Updated: 2025-01-16T08:19:33 [TS: 1737015573] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e (Updated: 2025-01-16T08:19:33) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c0458cd0-bc7b-4219-97a0-b4d1f5ed2239] to complete... -......done. -[2025-11-30 15:16:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5a5e8909b775757fef3b76180b6d47cd3d4eb181e31f861791577fef03a1ae2e -[2025-11-30 15:16:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 (Updated: 2025-01-17T08:22:38 [TS: 1737102158] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 (Updated: 2025-01-17T08:22:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/deb08cdd-51a3-44b9-933d-69b96681a2df] to complete... -......done. -[2025-11-30 15:16:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4b73d6103991b526f4f34be20e11822812d2074c25840bb78f8bd46eb12f483 -[2025-11-30 15:16:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f (Updated: 2025-01-18T08:20:38 [TS: 1737188438] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f (Updated: 2025-01-18T08:20:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cbc4a5fe-2eab-44c2-a441-c8ec91e4cf73] to complete... -.....done. -[2025-11-30 15:16:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2acf27b7115cee74eb1ebc9fc5b979fdc046c25ae2a46b2a86ea90a1a85d435f -[2025-11-30 15:16:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 (Updated: 2025-01-19T08:23:33 [TS: 1737275013] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 (Updated: 2025-01-19T08:23:33) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/62210de8-e0fc-4921-9f98-ed1641bb85cb] to complete... -......done. -[2025-11-30 15:16:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ca8d7e852543fd98957f31736f476e1be8dd1f8daee4613e7fbe2ae4e1682347 -[2025-11-30 15:16:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c (Updated: 2025-01-20T08:20:58 [TS: 1737361258] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c (Updated: 2025-01-20T08:20:58) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ed167d2f-4201-41c7-9ad7-38386eb648a5] to complete... -.....done. -[2025-11-30 15:16:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c1c5f0833c20a59f33ff1184e8fcf90c7afc3393f46217fc1141a80e0a5a6b0c -[2025-11-30 15:16:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 (Updated: 2025-01-21T08:21:03 [TS: 1737447663] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 (Updated: 2025-01-21T08:21:03) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/26015732-c98a-4ad4-b672-179bec651261] to complete... -.....done. -[2025-11-30 15:16:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3cb28a919d00f2d0f1ce0172ed6837f46f6f292ecc92926da3f6abb98d1da955 -[2025-11-30 15:16:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a (Updated: 2025-01-22T08:20:37 [TS: 1737534037] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a (Updated: 2025-01-22T08:20:37) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d23e711a-f007-4574-9e61-26cbf6202d60] to complete... -.....done. -[2025-11-30 15:16:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b300751098f0007472b0eb54a58451756235c14fb82902252c6e366e1168197a -[2025-11-30 15:16:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 (Updated: 2025-01-23T08:20:21 [TS: 1737620421] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 (Updated: 2025-01-23T08:20:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/547f8e39-d72d-4d97-bce0-81a2d5343055] to complete... -.....done. -[2025-11-30 15:16:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40591313a6e5413b740c91007594dd6d580cdd717c56a3ed08431ec808ec9287 -[2025-11-30 15:16:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 (Updated: 2025-01-24T08:19:44 [TS: 1737706784] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 (Updated: 2025-01-24T08:19:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5304569a-fd90-44dc-be73-f54d5ec73fdb] to complete... -......done. -[2025-11-30 15:16:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efd05f9048bc090d30fbe5ae466b09529bfda834a3d8963168b47eb5a4487242 -[2025-11-30 15:16:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 (Updated: 2025-01-25T08:20:25 [TS: 1737793225] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 (Updated: 2025-01-25T08:20:25) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/83d47863-f0ee-41e0-ad15-f7de3aa0de23] to complete... -.....done. -[2025-11-30 15:16:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d0dff82c71f03bcbaf637585c3911f6af7bacc2d09a768aebde1f87ff2718352 -[2025-11-30 15:16:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c (Updated: 2025-01-26T08:19:43 [TS: 1737879583] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c (Updated: 2025-01-26T08:19:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5ffeb95e-44d1-4a34-9e49-badaeac69d8d] to complete... -.....done. -[2025-11-30 15:16:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dacc16fe4a22be833355caf2565d7b75f367b064bb87e0a3949cc1dfbfbdbb3c -[2025-11-30 15:16:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d (Updated: 2025-01-27T08:21:26 [TS: 1737966086] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d (Updated: 2025-01-27T08:21:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bc62280e-3850-4eec-8251-446e5dfc3339] to complete... -......done. -[2025-11-30 15:16:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:031083012a83de5a5e67687d727a57b0efd6456214339d07651b95b7bedac91d -[2025-11-30 15:16:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e (Updated: 2025-01-28T08:22:47 [TS: 1738052567] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e (Updated: 2025-01-28T08:22:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f378ff13-619d-459e-9f64-b84ee0f80c2d] to complete... -......done. -[2025-11-30 15:16:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40f3afd60d68a637ad9ab613dc7fe2f00f1793f2cb25fa6530aa714d90af8c0e -[2025-11-30 15:16:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 (Updated: 2025-01-29T08:20:27 [TS: 1738138827] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 (Updated: 2025-01-29T08:20:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b1891a54-223e-4b32-b5b9-20f94b6731ad] to complete... -......done. -[2025-11-30 15:16:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9f616e8ec2b01f1db1b0f78a3e51f94f4c599905d0a9db8b1ff875f0b7cb91c7 -[2025-11-30 15:16:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 (Updated: 2025-01-30T08:20:39 [TS: 1738225239] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:16:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 (Updated: 2025-01-30T08:20:39) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/622a93e2-9461-4fd9-b0fc-f2aa880cda6e] to complete... -......done. -[2025-11-30 15:17:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b97adc4ac6350cd58314bd0da0953da3192b27f6b68631ca14df8d9987da1cc1 -[2025-11-30 15:17:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 (Updated: 2025-01-31T08:20:22 [TS: 1738311622] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 (Updated: 2025-01-31T08:20:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eb173b80-f08e-4868-9bf4-a6bda5fbfad8] to complete... -.....done. -[2025-11-30 15:17:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:104dcc362697ef4a020ed62b0409164520602743e19ca4fffd89a8de056054a8 -[2025-11-30 15:17:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 (Updated: 2025-02-01T08:20:50 [TS: 1738398050] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 (Updated: 2025-02-01T08:20:50) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7145a20c-038f-4c5f-aca2-724b90678ef9] to complete... -.....done. -[2025-11-30 15:17:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c14dcd233f2dcb7841b4047f3210ecffc88013e89a423db477432586e825cc8 -[2025-11-30 15:17:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 (Updated: 2025-02-02T08:20:43 [TS: 1738484443] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 (Updated: 2025-02-02T08:20:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7800581e-1c48-4191-970d-85cc24b3651b] to complete... -.....done. -[2025-11-30 15:17:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f4e5b7883032e43c281b6ba20871e5a081827af0b94e20bc6e0ad356b01f6485 -[2025-11-30 15:17:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 (Updated: 2025-02-03T08:20:14 [TS: 1738570814] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 (Updated: 2025-02-03T08:20:14) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5261cc68-2b99-4d4b-8c49-fb058c2740fa] to complete... -.....done. -[2025-11-30 15:17:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:830e5671a304dd5fc5e9e509ff09c3daaa4877dba9f21e1e2288a4918de6d877 -[2025-11-30 15:17:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae (Updated: 2025-02-04T08:20:52 [TS: 1738657252] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae (Updated: 2025-02-04T08:20:52) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9a8871e3-8495-410d-b607-46056e6b5d44] to complete... -......done. -[2025-11-30 15:17:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87bc6ceb38d5d0b3f83f8c4aaa9a45650c8ea4dcf0b42927f6b035a083f151ae -[2025-11-30 15:17:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b (Updated: 2025-02-05T08:20:48 [TS: 1738743648] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b (Updated: 2025-02-05T08:20:48) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0f46ff97-65af-4586-a2af-fffa12f72335] to complete... -.....done. -[2025-11-30 15:17:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c42f2c076f5004ec3b5338d3b4494a584ef220cc5acc51435679947816d2274b -[2025-11-30 15:17:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d (Updated: 2025-02-06T08:20:11 [TS: 1738830011] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d (Updated: 2025-02-06T08:20:11) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b2cb5f5a-d555-488e-a7aa-3956f9fda431] to complete... -......done. -[2025-11-30 15:17:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68f327310bc3309ee7f057090e50625ea8283bac2b23a24c2ec23b0b9b61b13d -[2025-11-30 15:17:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c (Updated: 2025-02-07T08:21:04 [TS: 1738916464] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c (Updated: 2025-02-07T08:21:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/79744cbb-3db1-4bd8-880c-994827fc13d6] to complete... -.....done. -[2025-11-30 15:17:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5c52072c393b6525761a31a503c1c5d9d71017dbd5fee719a37a93aba217a00c -[2025-11-30 15:17:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef (Updated: 2025-02-08T08:20:54 [TS: 1739002854] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef (Updated: 2025-02-08T08:20:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9e5c0385-97d1-4374-b00f-21f90f461ea3] to complete... -.....done. -[2025-11-30 15:17:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ca471ba5cc61e994a9e625b1f692ced99c12e53379960766007712a657e2cef -[2025-11-30 15:17:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 (Updated: 2025-02-09T08:20:35 [TS: 1739089235] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 (Updated: 2025-02-09T08:20:35) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/79dd60a5-24eb-4cf8-8c7f-1cfdb402d2eb] to complete... -.....done. -[2025-11-30 15:17:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8c9e78f5e834ce6a648e60a551f337089c6127f248fa803a8871ab559b862a36 -[2025-11-30 15:17:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 (Updated: 2025-02-10T08:20:55 [TS: 1739175655] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 (Updated: 2025-02-10T08:20:55) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/99c9df64-cbeb-44de-97f3-c404f68fc8cc] to complete... -.....done. -[2025-11-30 15:17:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3b83f1dcd66ae3c59b65aedf102ad895134704d379554f8a7a836ea00abc9a99 -[2025-11-30 15:17:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af (Updated: 2025-02-11T08:19:45 [TS: 1739261985] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af (Updated: 2025-02-11T08:19:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a2fba2a7-4223-4fd8-8c32-f20651144da0] to complete... -.....done. -[2025-11-30 15:17:45] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6cdca62329cc852827ed82c923c252e19b1733b38068599b5777e1c304e963af -[2025-11-30 15:17:45] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 (Updated: 2025-02-12T08:23:12 [TS: 1739348592] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:45] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 (Updated: 2025-02-12T08:23:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7fc977b9-ea13-48f5-aaef-0d1c23469bb2] to complete... -......done. -[2025-11-30 15:17:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c10c8f3952c21bca9f4ae5291dea0c489a64e75cd6cd703e67e7c8a44e2d9c73 -[2025-11-30 15:17:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 (Updated: 2025-02-13T08:21:40 [TS: 1739434900] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 (Updated: 2025-02-13T08:21:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/608946ac-8589-4c71-b607-13c3eb211554] to complete... -.....done. -[2025-11-30 15:17:52] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dfda1ce6ac1a23b050ccd8f270a634b842bd7c92a55e54c30c3d64789f6b76d1 -[2025-11-30 15:17:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 (Updated: 2025-02-14T08:20:29 [TS: 1739521229] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:52] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 (Updated: 2025-02-14T08:20:29) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0d8b5f70-5ab9-40b6-b1b5-2a8868e07e2c] to complete... -.....done. -[2025-11-30 15:17:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5d4e28c116003d7ee393578c6034c5d0e4c272986cfdeb3620979cda570eeab8 -[2025-11-30 15:17:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 (Updated: 2025-02-15T08:20:08 [TS: 1739607608] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 (Updated: 2025-02-15T08:20:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e4ab18ec-53a8-4c10-bdd2-3f6af00ab576] to complete... -......done. -[2025-11-30 15:17:59] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4e3e85061a8eaf72dc03a59cb9ca7e9d53a7fed4239d9d27fd108a21f1bac593 -[2025-11-30 15:17:59] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f (Updated: 2025-02-16T08:20:40 [TS: 1739694040] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:17:59] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f (Updated: 2025-02-16T08:20:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e42c1bfd-6a84-49a2-b663-1f630339e84d] to complete... -.....done. -[2025-11-30 15:18:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:46e79266a93417ff674b4c390dff3178c110d533b12f28dc8b0ccc5e578d9a0f -[2025-11-30 15:18:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 (Updated: 2025-02-17T08:24:07 [TS: 1739780647] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 (Updated: 2025-02-17T08:24:07) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eb8098e3-5eab-41b3-8565-e664c8968c4a] to complete... -......done. -[2025-11-30 15:18:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b544e2d663d1df9b719d1a137499007fc1e26f5f93e4ac9f8188358c8342c9c7 -[2025-11-30 15:18:06] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 (Updated: 2025-02-18T08:17:40 [TS: 1739866660] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:06] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 (Updated: 2025-02-18T08:17:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a6f79ba4-fcfa-4462-9e52-319bc4bdbcb9] to complete... -.....done. -[2025-11-30 15:18:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:933cd1ca648f5893d2d383dbcc644be652cf0aed1093663cb3993a0d9e48f9d3 -[2025-11-30 15:18:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 (Updated: 2025-02-19T08:21:02 [TS: 1739953262] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 (Updated: 2025-02-19T08:21:02) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d96b73f9-f27a-4923-b7eb-cbfa23e8f19e] to complete... -.....done. -[2025-11-30 15:18:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0f720156c0db2af7601723a50694cf0e5bc48df844010d8ad85676d7c73ff1 -[2025-11-30 15:18:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c (Updated: 2025-02-20T08:18:01 [TS: 1740039481] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c (Updated: 2025-02-20T08:18:01) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c8d0615b-8179-46a8-9bde-cba2db5e9c79] to complete... -.....done. -[2025-11-30 15:18:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7dc21dc5d8876a7d7cd66fd1ac29c7f9621a67be740040523106f4520b91ac3c -[2025-11-30 15:18:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 (Updated: 2025-02-21T08:20:07 [TS: 1740126007] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 (Updated: 2025-02-21T08:20:07) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/00e479ec-5af7-4db9-a9fb-b4e9389fc354] to complete... -......done. -[2025-11-30 15:18:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4baab8dde3a4d63a3f2ca7d77cccbec4c15519b29253b6cbe1f8505d0a931263 -[2025-11-30 15:18:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 (Updated: 2025-02-25T08:20:52 [TS: 1740471652] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 (Updated: 2025-02-25T08:20:52) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5c092c38-328b-483a-b2dd-83548f53aeaa] to complete... -......done. -[2025-11-30 15:18:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e6c6b6d9ddd7ce0672bfc34767c6f1b14c4297d831f3d5bf8d96b4c70152d0a5 -[2025-11-30 15:18:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 (Updated: 2025-02-26T08:21:06 [TS: 1740558066] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 (Updated: 2025-02-26T08:21:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4bacb1cc-25e7-4e19-94f1-56d4466611b3] to complete... -......done. -[2025-11-30 15:18:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1472a875c0935574bff63a31c30b25f533baf4b7ed3d5a05ddab0aa2630e15e2 -[2025-11-30 15:18:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 (Updated: 2025-02-27T08:21:06 [TS: 1740644466] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 (Updated: 2025-02-27T08:21:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d583fd59-9f57-40e8-9e86-a0fe35d28054] to complete... -......done. -[2025-11-30 15:18:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cba3233e3ea638d5d704fdcb184d1a1fe7b449fc01e2855376066b2c3f0d92b4 -[2025-11-30 15:18:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 (Updated: 2025-02-28T08:21:16 [TS: 1740730876] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 (Updated: 2025-02-28T08:21:16) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/dc2af5ad-a0b6-4097-a032-9cd47a6735dd] to complete... -......done. -[2025-11-30 15:18:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b603a2855dd49a0e81659bd4f645af84a1de6fa7b41d098eff3c33a534386bb8 -[2025-11-30 15:18:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c (Updated: 2025-03-01T08:20:02 [TS: 1740817202] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c (Updated: 2025-03-01T08:20:02) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9f9a369c-3175-4395-a6da-c7f19cb7323e] to complete... -.....done. -[2025-11-30 15:18:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:384c589a80560e6c529a6b97da3e2287dc53b2e9703ba6d4d92901e7a36ef22c -[2025-11-30 15:18:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 (Updated: 2025-03-02T08:21:22 [TS: 1740903682] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 (Updated: 2025-03-02T08:21:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0282f1d3-de95-4b5e-9789-84b2b3b42ff6] to complete... -.....done. -[2025-11-30 15:18:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9312d6b0fb313dd8885aaa192a974526fc0c833ba528770e3f745997daefef94 -[2025-11-30 15:18:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 (Updated: 2025-03-03T08:17:41 [TS: 1740989861] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 (Updated: 2025-03-03T08:17:41) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5b6b6571-406a-459c-b9b9-fd1bd52d8994] to complete... -.....done. -[2025-11-30 15:18:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b351aa15f086768b5c1b43719bedfe4bb2149d4b5b2c11d3a18b5172170990e6 -[2025-11-30 15:18:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b (Updated: 2025-03-04T08:22:52 [TS: 1741076572] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b (Updated: 2025-03-04T08:22:52) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/baceb358-fe46-43a3-bc91-44c3022f8042] to complete... -.....done. -[2025-11-30 15:18:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d16013a87fa5443388730ad31c19dbd73a59aeeed04276c15bdc6b3bb518a03b -[2025-11-30 15:18:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 (Updated: 2025-03-05T08:21:41 [TS: 1741162901] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 (Updated: 2025-03-05T08:21:41) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0afb0c25-ec0b-43b9-b557-b3bd15d2c159] to complete... -.....done. -[2025-11-30 15:18:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f2e2c390a1c7578f2017efc1a117e5029a2ff36088f1160801f96d76a81a76a0 -[2025-11-30 15:18:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 (Updated: 2025-03-06T08:21:32 [TS: 1741249292] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 (Updated: 2025-03-06T08:21:32) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/97e5379d-a50e-4fae-882a-c00b46bc1d55] to complete... -.....done. -[2025-11-30 15:18:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ac8cf20310eb5405d0dddb0f9c90344d3b7d32b0b4228ef5203d70134cebecf3 -[2025-11-30 15:18:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc (Updated: 2025-03-07T08:19:43 [TS: 1741335583] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:18:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc (Updated: 2025-03-07T08:19:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/de48f5de-06b9-428b-8caa-51a58ee2a3cb] to complete... -.....done. -[2025-11-30 15:19:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b85f1c7e8d4378139dbda431807c40333c8ab5e9ed5bf47598bc82a1819dacc -[2025-11-30 15:19:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c (Updated: 2025-03-08T08:19:23 [TS: 1741421963] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c (Updated: 2025-03-08T08:19:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/21aed277-ebaa-4d63-b317-e971a6296fca] to complete... -.....done. -[2025-11-30 15:19:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:84ce5aacbf5d6ffbc36a0d335482a390c0558c7b968d4b41aa09604cab0f068c -[2025-11-30 15:19:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a (Updated: 2025-03-09T08:18:08 [TS: 1741508288] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a (Updated: 2025-03-09T08:18:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/90e81efe-d447-4f64-8568-ab0296805023] to complete... -.....done. -[2025-11-30 15:19:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09183bf02d7265a2b595bb3f0ecd6e05fb544c1146c1bc6c041909bb8d45b07a -[2025-11-30 15:19:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b (Updated: 2025-03-10T07:21:20 [TS: 1741591280] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b (Updated: 2025-03-10T07:21:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e36f1439-3fc6-49b4-be48-b63c8ca04886] to complete... -.....done. -[2025-11-30 15:19:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3067e77d50f13be299a184247d5054b5976a859d44f4648a4ea58f1fb6d3440b -[2025-11-30 15:19:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e (Updated: 2025-03-11T07:20:43 [TS: 1741677643] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e (Updated: 2025-03-11T07:20:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5914289e-6d0f-4c69-b39b-4134ca8793ec] to complete... -.....done. -[2025-11-30 15:19:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad2a8a3405acd0f9be687d07983b29931c5f80aa11b954797039a00c40528e -[2025-11-30 15:19:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e (Updated: 2025-03-12T07:19:34 [TS: 1741763974] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e (Updated: 2025-03-12T07:19:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1577694f-3577-4746-8dce-cbbc499cc475] to complete... -.....done. -[2025-11-30 15:19:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:30a705c6beafb3db34ccc092c7302ce638d8b88b2b995bc33f1a68277c4a237e -[2025-11-30 15:19:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 (Updated: 2025-03-13T07:20:51 [TS: 1741850451] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 (Updated: 2025-03-13T07:20:51) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e6313482-221e-4869-8816-ad627694ef62] to complete... -.....done. -[2025-11-30 15:19:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7caf003c09b1179403fd226b149ea2bcf03ca4f61e37ada7bc94f120cc43a71 -[2025-11-30 15:19:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 (Updated: 2025-03-14T07:20:37 [TS: 1741936837] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 (Updated: 2025-03-14T07:20:37) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bc09cee9-0217-419a-9eab-df4dfb090e68] to complete... -.....done. -[2025-11-30 15:19:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6c077d17c3069535d3f902abfc9f50311f8ad0f28c4f091cfaa93d9555f57c21 -[2025-11-30 15:19:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 (Updated: 2025-03-15T07:21:23 [TS: 1742023283] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 (Updated: 2025-03-15T07:21:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8e44511e-9b39-4dc6-8f92-96065bc5056f] to complete... -.....done. -[2025-11-30 15:19:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f8a6d10a4c6df0e5b39718aa2a89bf62ead095f730d83e03934023c3c0e1ff55 -[2025-11-30 15:19:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 (Updated: 2025-03-16T07:20:44 [TS: 1742109644] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 (Updated: 2025-03-16T07:20:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b4da2193-082e-4d6a-b642-4f3b9556b6fc] to complete... -......done. -[2025-11-30 15:19:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b21a96d98276ebc341efdd462a7b6d525d67e5e717b1b783113626639b837897 -[2025-11-30 15:19:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 (Updated: 2025-03-17T07:21:09 [TS: 1742196069] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 (Updated: 2025-03-17T07:21:09) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e09b9c22-8238-448b-8608-560a33bd5efc] to complete... -.....done. -[2025-11-30 15:19:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:068c9dc938d9c9123514a73d87f00d857afe44a052ecdb4a246339990704b9f1 -[2025-11-30 15:19:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 (Updated: 2025-03-18T07:22:07 [TS: 1742282527] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 (Updated: 2025-03-18T07:22:07) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/82286eac-44ec-4fcd-8311-f321e6c8368f] to complete... -......done. -[2025-11-30 15:19:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7e0ecbdaf1ba7a7ad07989bf3b8c638c636d05aac7219ee2a6ed62a84d7a4773 -[2025-11-30 15:19:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac (Updated: 2025-03-19T07:21:36 [TS: 1742368896] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac (Updated: 2025-03-19T07:21:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c9f2c33f-f619-4b4d-8b85-da3ed41357e0] to complete... -.....done. -[2025-11-30 15:19:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01098afd80e065f4c25e9b87aa3ccd28c5128fc6ad08c89287358476c5bf8bac -[2025-11-30 15:19:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a (Updated: 2025-03-20T07:21:15 [TS: 1742455275] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a (Updated: 2025-03-20T07:21:15) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a2e13152-c6a2-4e19-b5b1-28f3fdda7790] to complete... -.....done. -[2025-11-30 15:19:45] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1be9705868fa981c1d7df9ed8411cb503aef31766818d582e1d80d3eb54f690a -[2025-11-30 15:19:45] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 (Updated: 2025-03-21T07:20:54 [TS: 1742541654] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:45] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 (Updated: 2025-03-21T07:20:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7bf6c370-194d-495b-93c3-00d64f8bb812] to complete... -......done. -[2025-11-30 15:19:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:689f63e43ae97e80d5bc2ed2ebe2e271f097f93c64c766020891aa9fca7df1a5 -[2025-11-30 15:19:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 (Updated: 2025-03-22T07:21:14 [TS: 1742628074] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 (Updated: 2025-03-22T07:21:14) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f54e03ee-6a6d-44ce-a2e5-3f2f7f8b4403] to complete... -......done. -[2025-11-30 15:19:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:78adeb260680554077d5f1a726503f75957de63d453b84bd2026879e337bc407 -[2025-11-30 15:19:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f (Updated: 2025-03-23T07:20:50 [TS: 1742714450] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f (Updated: 2025-03-23T07:20:50) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3a19055e-a37b-4a94-adcc-fb03de837b06] to complete... -.....done. -[2025-11-30 15:19:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:295c76cf2f2cc79a9897bd8908e6e616a8d29ec59e696364a188db7262574e7f -[2025-11-30 15:19:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca (Updated: 2025-03-24T07:21:04 [TS: 1742800864] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:19:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca (Updated: 2025-03-24T07:21:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/af12c77c-cc82-4f1e-8c4f-e7ca397e7cd0] to complete... -.....done. -[2025-11-30 15:20:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d01c4e287af5f3d18b4f9a25adb50c6ae7d7d1bbba59eace0f8e1e9345857ca -[2025-11-30 15:20:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f (Updated: 2025-03-25T07:21:17 [TS: 1742887277] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f (Updated: 2025-03-25T07:21:17) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3d7ac828-bdb0-4f70-b21e-47b6ab26e3f4] to complete... -.....done. -[2025-11-30 15:20:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c090c4ca453d8e30200cc8faa9aca1e693a1e32b300f6a0983ab59bb4a0a904f -[2025-11-30 15:20:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 (Updated: 2025-03-26T07:20:59 [TS: 1742973659] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 (Updated: 2025-03-26T07:20:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d703d169-8470-4090-bdf5-53e3c6823ff2] to complete... -.....done. -[2025-11-30 15:20:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9963b8b99bcc0df3162e8222cf7b1f6149dfeed45f0674c4e7b51a36fc3eb9f2 -[2025-11-30 15:20:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a (Updated: 2025-03-27T07:20:48 [TS: 1743060048] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a (Updated: 2025-03-27T07:20:48) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/919d2591-0773-4aa8-bbae-fcf200a69249] to complete... -.....done. -[2025-11-30 15:20:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f79370393099dd3d6beb35c4abcde16cbaf6120e61b1e259e5084c43a00810a -[2025-11-30 15:20:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 (Updated: 2025-03-28T07:20:36 [TS: 1743146436] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 (Updated: 2025-03-28T07:20:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/07ccbd80-5be0-42d1-8b78-0e96d180b9c9] to complete... -.....done. -[2025-11-30 15:20:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de36816fc8a513ee9ba6b8bdcdc83c6d37a838149e33ab891f76a326973b0594 -[2025-11-30 15:20:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 (Updated: 2025-03-29T07:21:17 [TS: 1743232877] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 (Updated: 2025-03-29T07:21:17) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7de707fe-b1e7-453a-8801-538125db413a] to complete... -.....done. -[2025-11-30 15:20:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5b7282ebeb8e7f4ceacdd4a2bdf3af50f152b0af2c764bfdb75d92e3d67bab73 -[2025-11-30 15:20:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 (Updated: 2025-03-30T07:21:38 [TS: 1743319298] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 (Updated: 2025-03-30T07:21:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/418cb23a-e880-4352-9ca2-54711dbb2314] to complete... -.....done. -[2025-11-30 15:20:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f71f5e75957a2ce7328e3ec6d3e3f682e3e5ded73fdde05b340a5a647760a815 -[2025-11-30 15:20:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 (Updated: 2025-03-31T07:20:18 [TS: 1743405618] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 (Updated: 2025-03-31T07:20:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0131a587-8a3f-458b-b07c-6eaf57c9cac3] to complete... -.....done. -[2025-11-30 15:20:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:986faa7e3c6a46161786591fc1d3efbcc3a480e5dd63acb063371c4527caea46 -[2025-11-30 15:20:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 (Updated: 2025-04-01T07:21:20 [TS: 1743492080] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 (Updated: 2025-04-01T07:21:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/600c2c2d-43f9-4615-bf48-b3d96f573afe] to complete... -.....done. -[2025-11-30 15:20:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:556a9cb62bdb2114190593e6082473538d7294ed9707b36c8375b91044c08211 -[2025-11-30 15:20:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d (Updated: 2025-04-02T07:20:23 [TS: 1743578423] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d (Updated: 2025-04-02T07:20:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/93615534-b7e1-40a9-a54d-ddd0c024b455] to complete... -.....done. -[2025-11-30 15:20:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:089de19596c47c943cbe6f8e216bfa45f71bdad7d67c677bd220ff28be91d12d -[2025-11-30 15:20:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 (Updated: 2025-04-03T07:20:49 [TS: 1743664849] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 (Updated: 2025-04-03T07:20:49) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/571447de-1879-4fde-9954-b35e6ce99df5] to complete... -.....done. -[2025-11-30 15:20:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c65bc4d93bc533ac96ad8ba812c1c0c76f5b4dc32c20367e62c7f7d498d1f8c3 -[2025-11-30 15:20:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b (Updated: 2025-04-04T07:20:41 [TS: 1743751241] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b (Updated: 2025-04-04T07:20:41) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2041092c-d1eb-4a0d-855d-77c828884b4c] to complete... -.....done. -[2025-11-30 15:20:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d35a6edb2958a336fb4eebee5acc51545f90fc0a7af177f3d70c16b26c6ee0b -[2025-11-30 15:20:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a (Updated: 2025-04-05T07:20:21 [TS: 1743837621] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a (Updated: 2025-04-05T07:20:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/06c2386d-3789-4acd-ba0a-dd57aaa7e7df] to complete... -.....done. -[2025-11-30 15:20:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2f89ae92f1055408b8032152bd36db4c63865a9622e43accd59566245446b76a -[2025-11-30 15:20:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 (Updated: 2025-04-06T07:23:45 [TS: 1743924225] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 (Updated: 2025-04-06T07:23:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/67832465-d1cf-4e5a-b8f9-c5663f221380] to complete... -.....done. -[2025-11-30 15:20:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9a8051621c3051d3b3e17cf0f8e3c4af2c5c0ed334cda4294e08feb13db693c9 -[2025-11-30 15:20:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 (Updated: 2025-04-07T07:20:24 [TS: 1744010424] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 (Updated: 2025-04-07T07:20:24) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eae5c461-0c20-4556-befd-b08ed91074ea] to complete... -.....done. -[2025-11-30 15:20:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b4e053f002b4e7053dce429f7e5a9dc4fbc7e3deba12ae028eafced1e4c3c2a2 -[2025-11-30 15:20:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee (Updated: 2025-04-08T07:22:43 [TS: 1744096963] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee (Updated: 2025-04-08T07:22:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b48c5f0f-a002-4f2e-a99d-2fcc8871b980] to complete... -.....done. -[2025-11-30 15:20:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:36250d9e4eab49f6fe1e2efcef990a85969e2841ab5076f6b651c30859f528ee -[2025-11-30 15:20:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 (Updated: 2025-04-09T07:20:20 [TS: 1744183220] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 (Updated: 2025-04-09T07:20:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/93704621-002e-4175-83d5-dc6584a66c40] to complete... -.....done. -[2025-11-30 15:20:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c9e9bf73cfa4b0b0280277a99b1ea7195319d3d9090130d9ff5bb3c91f9cc86 -[2025-11-30 15:20:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 (Updated: 2025-04-10T07:21:00 [TS: 1744269660] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:20:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 (Updated: 2025-04-10T07:21:00) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e2d9d0f6-edc0-4d57-9c08-44b2ef70d700] to complete... -.....done. -[2025-11-30 15:21:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a944defac24fa871679aefcbeca0c9bd7eef26ffe3047a77051070c3434e26e1 -[2025-11-30 15:21:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 (Updated: 2025-04-11T07:20:54 [TS: 1744356054] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 (Updated: 2025-04-11T07:20:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/56fb8f24-0081-4c12-92b2-4ee5361d756f] to complete... -.....done. -[2025-11-30 15:21:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:34bbcc0c8eac8625d8ab4b56a284aa86029296be43c36259ee41b22f74ac11e1 -[2025-11-30 15:21:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 (Updated: 2025-04-12T07:21:40 [TS: 1744442500] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 (Updated: 2025-04-12T07:21:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bb71c750-5d2e-4996-9aef-f573502c91d6] to complete... -.....done. -[2025-11-30 15:21:06] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:787f5c765f495484eb403d72616aa89035e36850543826bcb59134b2b5dea879 -[2025-11-30 15:21:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 (Updated: 2025-04-13T07:20:59 [TS: 1744528859] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 (Updated: 2025-04-13T07:20:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c0f699b8-7456-4990-bba7-fd1f407096f5] to complete... -.....done. -[2025-11-30 15:21:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895fa0b08ea66c2e2a8779713d7f7accc001074fdefae98986501d3a524ffff6 -[2025-11-30 15:21:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c (Updated: 2025-04-14T07:20:24 [TS: 1744615224] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c (Updated: 2025-04-14T07:20:24) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cccafaa5-65e5-48e3-bb51-27c8a2063873] to complete... -.....done. -[2025-11-30 15:21:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dbc467a3adadb1a2fac1140c3afb0eaa262ce8b9f621b164a1114db30045c73c -[2025-11-30 15:21:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 (Updated: 2025-04-15T07:20:46 [TS: 1744701646] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 (Updated: 2025-04-15T07:20:46) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/457bb2a4-234f-4ed5-817a-d2107b62119a] to complete... -......done. -[2025-11-30 15:21:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ba835edadcb39760fe401ee4c0241ded7a9230a1d86aa2cdcf502ef5d0b89061 -[2025-11-30 15:21:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 (Updated: 2025-04-16T07:20:12 [TS: 1744788012] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 (Updated: 2025-04-16T07:20:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1782a945-717e-4ac9-b1b0-0a6c256570bf] to complete... -.....done. -[2025-11-30 15:21:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:add801913c1ffdbc47978f724190d14b7753c4461476401f56ea4577a8e13f00 -[2025-11-30 15:21:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 (Updated: 2025-04-17T07:19:55 [TS: 1744874395] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 (Updated: 2025-04-17T07:19:55) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3fe2f825-3f75-429b-bc4e-66c791c70d95] to complete... -.....done. -[2025-11-30 15:21:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112a34107db592f4a180c4843d3d964b691e1edbab0c03b55e7a62127b77bf0 -[2025-11-30 15:21:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb (Updated: 2025-04-18T07:20:41 [TS: 1744960841] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb (Updated: 2025-04-18T07:20:41) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c65df95f-34e6-404b-a0ca-879a87f6693b] to complete... -......done. -[2025-11-30 15:21:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa88aa7ae43620d66e3fe687dbe869be9d743d60026bd488ec5e3d7219c742cb -[2025-11-30 15:21:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa (Updated: 2025-04-19T07:21:15 [TS: 1745047275] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa (Updated: 2025-04-19T07:21:15) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e9741419-65bb-46c4-af17-b68bf77e3a68] to complete... -.....done. -[2025-11-30 15:21:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a3a65454e4a409f974ea5699cc40876983ef7924db90f56f641fa270b675eaa -[2025-11-30 15:21:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 (Updated: 2025-04-20T07:21:25 [TS: 1745133685] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 (Updated: 2025-04-20T07:21:25) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2f4c8846-5322-4309-b16b-33ceab778ae0] to complete... -.....done. -[2025-11-30 15:21:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ebb1d9b284bed5c0cca05fef0a9704b4bb46389ea72f46cd014b159d71b30f30 -[2025-11-30 15:21:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d (Updated: 2025-04-21T07:21:59 [TS: 1745220119] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d (Updated: 2025-04-21T07:21:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6f7abf11-ac26-437a-ac79-0ae1d9cd9fb2] to complete... -.....done. -[2025-11-30 15:21:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:399fd0abb2fef786546d23e8f52b08892c8013c79eb48bd4d98b639a6577971d -[2025-11-30 15:21:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a (Updated: 2025-04-22T07:21:34 [TS: 1745306494] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a (Updated: 2025-04-22T07:21:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/10934be6-c3ce-4ebe-9736-e1ad8a0c3f5c] to complete... -.....done. -[2025-11-30 15:21:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:818d60f5601f6c6820577c00ca6320ce0bda908520c8cb1dff8bf6d4eb4db49a -[2025-11-30 15:21:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 (Updated: 2025-04-23T07:21:06 [TS: 1745392866] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 (Updated: 2025-04-23T07:21:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2638cda2-663d-4626-9983-258237333034] to complete... -.....done. -[2025-11-30 15:21:45] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242955f2b46a02a60e4e7f2adf8201adb8d7b7c4339c58851a8ab0345df33e71 -[2025-11-30 15:21:45] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 (Updated: 2025-04-24T07:21:59 [TS: 1745479319] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:45] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 (Updated: 2025-04-24T07:21:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b00b6823-40ba-4f9c-9081-cc928b6057df] to complete... -.....done. -[2025-11-30 15:21:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dec1087972b62e560385d35d4ef9478fdb86674b8417a01ac25620b894ea3d5 -[2025-11-30 15:21:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 (Updated: 2025-04-25T07:20:22 [TS: 1745565622] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 (Updated: 2025-04-25T07:20:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e788c620-84fe-47dd-8283-a74aac3c3e80] to complete... -.....done. -[2025-11-30 15:21:52] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:65260063fbce5470d8e600d46c1038d83636bcc7d70d95032974ab66a21e9dd2 -[2025-11-30 15:21:52] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d (Updated: 2025-04-26T07:20:43 [TS: 1745652043] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:52] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d (Updated: 2025-04-26T07:20:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/355147be-d407-41af-9358-e2387e5f1739] to complete... -......done. -[2025-11-30 15:21:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:547c036f0fa62d253ff727f82c9a49c85374aaf2e390aaec627c5bcbd38e518d -[2025-11-30 15:21:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da (Updated: 2025-04-27T07:21:38 [TS: 1745738498] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:21:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da (Updated: 2025-04-27T07:21:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7fef88a4-1a39-4d36-9e80-74ff9aa51bbe] to complete... -.....done. -[2025-11-30 15:22:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b796b0058432ccabcb8cbf37b09d60944391e1c568e18c3d39c59c9681c270da -[2025-11-30 15:22:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 (Updated: 2025-04-28T07:21:01 [TS: 1745824861] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 (Updated: 2025-04-28T07:21:01) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d0495266-18e8-4d24-9407-657fb84a2579] to complete... -......done. -[2025-11-30 15:22:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e974f1403240fa777febb842bd1e243e0a77353834f12799a3a85a660e6cd7d6 -[2025-11-30 15:22:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 (Updated: 2025-04-29T07:21:25 [TS: 1745911285] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 (Updated: 2025-04-29T07:21:25) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2238de96-f54c-42de-846d-18ef52ff5dcd] to complete... -......done. -[2025-11-30 15:22:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d393768b1e102c2715cef107b461217d3fddb429823779b703304650384e0ed6 -[2025-11-30 15:22:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 (Updated: 2025-04-30T07:20:20 [TS: 1745997620] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 (Updated: 2025-04-30T07:20:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/786e4953-e669-4ba3-acb9-afa982faa47e] to complete... -.....done. -[2025-11-30 15:22:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:27c87481d5ed17059a7499868676819f888155ed657704b825243ace641a4414 -[2025-11-30 15:22:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 (Updated: 2025-05-01T07:21:24 [TS: 1746084084] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 (Updated: 2025-05-01T07:21:24) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eec93fa2-f697-4ac4-9bbe-07468d2549c9] to complete... -......done. -[2025-11-30 15:22:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f96a9942f6ef609f17503d94ab4fbdea2a9999bd626c516a12340f66706d590 -[2025-11-30 15:22:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 (Updated: 2025-05-02T07:21:04 [TS: 1746170464] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 (Updated: 2025-05-02T07:21:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/17ecec7a-5ed2-4c81-ac4c-a16be4fa08b0] to complete... -......done. -[2025-11-30 15:22:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae869bfb4dd703ad037bbc6269d52c30c60ab7197f3d167c78dbf5d990e0cbe0 -[2025-11-30 15:22:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 (Updated: 2025-05-03T07:21:59 [TS: 1746256919] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 (Updated: 2025-05-03T07:21:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/254afc4b-74eb-4e4d-a47d-fa44d2c04b1f] to complete... -.....done. -[2025-11-30 15:22:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f789f7e4c706546e4cdb126d50d95f4c7da780b648a79970cc796396212ca226 -[2025-11-30 15:22:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 (Updated: 2025-05-04T07:21:07 [TS: 1746343267] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 (Updated: 2025-05-04T07:21:07) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6474c9d5-ffe8-464d-8c75-2a9a05658f9b] to complete... -......done. -[2025-11-30 15:22:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05c2fc0cb771267af1334a151ffd5c989723ba583a4addc39a933e564da6add5 -[2025-11-30 15:22:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 (Updated: 2025-05-05T07:21:39 [TS: 1746429699] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 (Updated: 2025-05-05T07:21:39) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/69c2a80b-8b08-4b1a-ab85-7d2f57c3e1f1] to complete... -.....done. -[2025-11-30 15:22:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc5052e0939578eab65b23e9185b2a1a410bd33c747efb38a9149d1fa89dbda0 -[2025-11-30 15:22:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 (Updated: 2025-05-06T07:21:39 [TS: 1746516099] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 (Updated: 2025-05-06T07:21:39) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0d725914-bd53-466c-8c22-66eab151706e] to complete... -.....done. -[2025-11-30 15:22:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:71c5ed242899b4faa953f01aeb0f56768f3099c7ca796afc30420cb41498f647 -[2025-11-30 15:22:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb (Updated: 2025-05-07T07:22:18 [TS: 1746602538] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb (Updated: 2025-05-07T07:22:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/deb3bbe0-c114-474a-aa04-be50573dce26] to complete... -.....done. -[2025-11-30 15:22:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54586d69bc05a6e8e85d06c642b9a19e007fd68e2099d47383852107cbcd5fdb -[2025-11-30 15:22:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d (Updated: 2025-05-08T07:22:24 [TS: 1746688944] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d (Updated: 2025-05-08T07:22:24) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b71fe70e-f9a4-4a67-a093-76b4f6a72291] to complete... -.....done. -[2025-11-30 15:22:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:845a57988746ac2b360e44566ea24821385f53513cfa9f5d7bb6ae6e5757598d -[2025-11-30 15:22:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b (Updated: 2025-05-09T07:21:16 [TS: 1746775276] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b (Updated: 2025-05-09T07:21:16) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cfd9113f-abbe-42c6-8c4e-7850bbce9e8e] to complete... -......done. -[2025-11-30 15:22:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6202d481471a1e41483c19d2e3555e50db5204325231522d2d4d743ac1798a6b -[2025-11-30 15:22:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 (Updated: 2025-05-10T07:21:20 [TS: 1746861680] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 (Updated: 2025-05-10T07:21:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8db16416-b432-45e9-b1fe-7e9a3e31d0bf] to complete... -.....done. -[2025-11-30 15:22:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2afc68fbc27709bc4130f5d124d36bd9025d4a01c1c4dfe0462b5fae20fdee01 -[2025-11-30 15:22:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 (Updated: 2025-05-11T07:20:26 [TS: 1746948026] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 (Updated: 2025-05-11T07:20:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5b865889-906d-4a53-9ad7-a71d95eb46d3] to complete... -......done. -[2025-11-30 15:22:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce90426ff667268833f2236bc8b8d725507d3d2ae2931b243df132e9a91599b9 -[2025-11-30 15:22:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 (Updated: 2025-05-12T07:21:06 [TS: 1747034466] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 (Updated: 2025-05-12T07:21:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d8b68b09-df98-4a1d-8cec-95e1cf67bc5f] to complete... -......done. -[2025-11-30 15:22:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d692176fd2c15b3a1337bbc65e50e92cd11d3df6c19194f65959fb7b423e57f9 -[2025-11-30 15:22:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 (Updated: 2025-05-13T07:21:27 [TS: 1747120887] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 (Updated: 2025-05-13T07:21:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/403db624-0675-4396-813e-186f87098604] to complete... -......done. -[2025-11-30 15:22:59] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ad8a1f3066c074fc8a7feac5856aee44040fe85ff94d6ec12eec82bb9a4edf29 -[2025-11-30 15:22:59] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 (Updated: 2025-05-14T07:21:24 [TS: 1747207284] < Cutoff: [TS: 1763305844]) -[2025-11-30 15:22:59] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 (Updated: 2025-05-14T07:21:24) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5684dced-0dd8-40e1-95d0-0634f6d6daa9] to complete... -......done. -[2025-11-30 15:23:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b473e2eb4a96f2093c3e85c7ecd6d3d4c5191873a1476003d76eaa2ff0623a68 -[2025-11-30 15:23:03] [INFO] Hit delete limit (200) for Docker Images. -[2025-11-30 15:23:03] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 15:23:03] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 15:23:06] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 15:23:06] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 15:23:06] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 15:23:06] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 15:23:06] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 15:23:06] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 15:23:08] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 15:23:08] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:23:11] [INFO] No Regional Address found matching criteria. -[2025-11-30 15:23:11] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:23:13] [INFO] No Global Address found matching criteria. -[2025-11-30 15:23:13] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- -[2025-11-30 15:23:17] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 15:23:20] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:23:20] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:23:20] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 15:23:20] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 15:23:20] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:23] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 15:23:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:23:25] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 15:23:27] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 15:23:27] [INFO] CLEANUP RUN FINISHED -[2025-11-30 15:24:51] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 15:24:51] [INFO] Time Cutoff (General): 2025-11-30T15:24:51+0000 -[2025-11-30 15:24:51] [INFO] Time Cutoff (Images): 2025-10-01T15:24:51+0000 -[2025-11-30 15:24:51] [INFO] Delete Limit per Type: 200 -[2025-11-30 15:24:51] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 15:24:51] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 15:24:54] [INFO] No Service Accounts found matching prefix. -[2025-11-30 15:24:54] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:24:56] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 15:24:56] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 15:24:58] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:24:58] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:24:58] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 15:24:58] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:25:01] [INFO] No Filestore instances found matching criteria. -[2025-11-30 15:25:01] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 15:25:04] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 15:25:04] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 15:25:05] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 15:25:05] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 15:25:05] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 15:25:05] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 15:25:05] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 15:25:05] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 15:25:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 15:25:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 15:25:05] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 15:25:05] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:25:05Z (Unix: 1763306705) -[2025-11-30 15:25:05] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 (Updated: 2025-05-15T07:21:31 [TS: 1747293691] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 (Updated: 2025-05-15T07:21:31) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 (Updated: 2025-05-16T07:21:17 [TS: 1747380077] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 (Updated: 2025-05-16T07:21:17) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da (Updated: 2025-05-17T07:21:15 [TS: 1747466475] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da (Updated: 2025-05-17T07:21:15) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d (Updated: 2025-05-18T07:21:43 [TS: 1747552903] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d (Updated: 2025-05-18T07:21:43) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de (Updated: 2025-05-19T07:21:57 [TS: 1747639317] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de (Updated: 2025-05-19T07:21:57) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 (Updated: 2025-05-20T07:22:12 [TS: 1747725732] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 (Updated: 2025-05-20T07:22:12) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 (Updated: 2025-05-21T07:19:51 [TS: 1747811991] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 (Updated: 2025-05-21T07:19:51) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f (Updated: 2025-05-22T07:21:21 [TS: 1747898481] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f (Updated: 2025-05-22T07:21:21) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 (Updated: 2025-05-23T07:22:28 [TS: 1747984948] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 (Updated: 2025-05-23T07:22:28) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c (Updated: 2025-05-24T07:21:57 [TS: 1748071317] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c (Updated: 2025-05-24T07:21:57) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be (Updated: 2025-05-25T07:21:23 [TS: 1748157683] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be (Updated: 2025-05-25T07:21:23) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 (Updated: 2025-05-26T07:28:04 [TS: 1748244484] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 (Updated: 2025-05-26T07:28:04) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 (Updated: 2025-05-27T07:19:54 [TS: 1748330394] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 (Updated: 2025-05-27T07:19:54) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a (Updated: 2025-05-28T07:20:28 [TS: 1748416828] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a (Updated: 2025-05-28T07:20:28) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d (Updated: 2025-05-29T07:21:22 [TS: 1748503282] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:09] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d (Updated: 2025-05-29T07:21:22) -[2025-11-30 15:25:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 (Updated: 2025-05-30T07:21:07 [TS: 1748589667] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 (Updated: 2025-05-30T07:21:07) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 (Updated: 2025-05-31T07:20:46 [TS: 1748676046] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 (Updated: 2025-05-31T07:20:46) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 (Updated: 2025-06-01T07:20:52 [TS: 1748762452] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 (Updated: 2025-06-01T07:20:52) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a (Updated: 2025-06-02T07:23:55 [TS: 1748849035] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a (Updated: 2025-06-02T07:23:55) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 (Updated: 2025-06-03T07:21:05 [TS: 1748935265] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 (Updated: 2025-06-03T07:21:05) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd (Updated: 2025-06-04T07:22:23 [TS: 1749021743] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd (Updated: 2025-06-04T07:22:23) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 (Updated: 2025-06-05T07:21:27 [TS: 1749108087] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 (Updated: 2025-06-05T07:21:27) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf (Updated: 2025-06-06T07:22:27 [TS: 1749194547] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf (Updated: 2025-06-06T07:22:27) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c (Updated: 2025-06-07T07:22:15 [TS: 1749280935] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c (Updated: 2025-06-07T07:22:15) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 (Updated: 2025-06-08T07:21:06 [TS: 1749367266] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 (Updated: 2025-06-08T07:21:06) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f (Updated: 2025-06-09T07:20:04 [TS: 1749453604] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f (Updated: 2025-06-09T07:20:04) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f (Updated: 2025-06-10T07:20:54 [TS: 1749540054] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f (Updated: 2025-06-10T07:20:54) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 (Updated: 2025-06-11T07:20:59 [TS: 1749626459] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 (Updated: 2025-06-11T07:20:59) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 (Updated: 2025-06-12T07:19:36 [TS: 1749712776] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 (Updated: 2025-06-12T07:19:36) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 (Updated: 2025-06-13T07:21:52 [TS: 1749799312] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 (Updated: 2025-06-13T07:21:52) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 (Updated: 2025-06-14T07:21:53 [TS: 1749885713] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 (Updated: 2025-06-14T07:21:53) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 (Updated: 2025-06-15T07:20:57 [TS: 1749972057] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 (Updated: 2025-06-15T07:20:57) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 (Updated: 2025-06-16T07:20:18 [TS: 1750058418] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 (Updated: 2025-06-16T07:20:18) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe (Updated: 2025-06-17T07:21:49 [TS: 1750144909] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe (Updated: 2025-06-17T07:21:49) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 (Updated: 2025-06-18T07:20:13 [TS: 1750231213] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 (Updated: 2025-06-18T07:20:13) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 (Updated: 2025-06-19T07:20:59 [TS: 1750317659] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 (Updated: 2025-06-19T07:20:59) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b (Updated: 2025-06-20T07:21:01 [TS: 1750404061] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b (Updated: 2025-06-20T07:21:01) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f (Updated: 2025-06-21T07:22:20 [TS: 1750490540] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f (Updated: 2025-06-21T07:22:20) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb (Updated: 2025-06-22T07:21:21 [TS: 1750576881] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb (Updated: 2025-06-22T07:21:21) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 (Updated: 2025-06-23T07:21:18 [TS: 1750663278] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 (Updated: 2025-06-23T07:21:18) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 (Updated: 2025-06-24T07:21:15 [TS: 1750749675] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 (Updated: 2025-06-24T07:21:15) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 (Updated: 2025-06-25T07:21:11 [TS: 1750836071] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 (Updated: 2025-06-25T07:21:11) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba (Updated: 2025-06-26T07:20:29 [TS: 1750922429] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba (Updated: 2025-06-26T07:20:29) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c (Updated: 2025-06-27T07:20:18 [TS: 1751008818] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c (Updated: 2025-06-27T07:20:18) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb (Updated: 2025-06-28T07:20:26 [TS: 1751095226] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb (Updated: 2025-06-28T07:20:26) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c (Updated: 2025-06-29T07:21:11 [TS: 1751181671] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c (Updated: 2025-06-29T07:21:11) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 (Updated: 2025-06-30T07:22:37 [TS: 1751268157] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 (Updated: 2025-06-30T07:22:37) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f (Updated: 2025-07-01T07:20:54 [TS: 1751354454] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f (Updated: 2025-07-01T07:20:54) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 (Updated: 2025-07-02T07:21:52 [TS: 1751440912] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 (Updated: 2025-07-02T07:21:52) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb (Updated: 2025-07-03T07:20:06 [TS: 1751527206] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb (Updated: 2025-07-03T07:20:06) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e (Updated: 2025-07-04T07:21:34 [TS: 1751613694] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e (Updated: 2025-07-04T07:21:34) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 (Updated: 2025-07-05T07:21:32 [TS: 1751700092] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 (Updated: 2025-07-05T07:21:32) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf (Updated: 2025-07-06T07:21:21 [TS: 1751786481] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf (Updated: 2025-07-06T07:21:21) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 (Updated: 2025-07-07T07:22:25 [TS: 1751872945] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 (Updated: 2025-07-07T07:22:25) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 (Updated: 2025-07-08T07:21:26 [TS: 1751959286] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 (Updated: 2025-07-08T07:21:26) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 (Updated: 2025-07-09T07:22:45 [TS: 1752045765] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 (Updated: 2025-07-09T07:22:45) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 (Updated: 2025-07-10T07:21:44 [TS: 1752132104] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 (Updated: 2025-07-10T07:21:44) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b (Updated: 2025-07-11T07:21:43 [TS: 1752218503] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b (Updated: 2025-07-11T07:21:43) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 (Updated: 2025-07-12T07:20:14 [TS: 1752304814] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 (Updated: 2025-07-12T07:20:14) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 (Updated: 2025-07-13T07:22:16 [TS: 1752391336] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 (Updated: 2025-07-13T07:22:16) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 (Updated: 2025-07-14T07:20:21 [TS: 1752477621] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 (Updated: 2025-07-14T07:20:21) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a (Updated: 2025-07-15T07:21:57 [TS: 1752564117] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a (Updated: 2025-07-15T07:21:57) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 (Updated: 2025-07-16T07:20:47 [TS: 1752650447] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 (Updated: 2025-07-16T07:20:47) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d (Updated: 2025-07-17T07:20:43 [TS: 1752736843] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d (Updated: 2025-07-17T07:20:43) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 (Updated: 2025-07-18T07:22:46 [TS: 1752823366] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 (Updated: 2025-07-18T07:22:46) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 (Updated: 2025-07-19T07:21:08 [TS: 1752909668] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 (Updated: 2025-07-19T07:21:08) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 (Updated: 2025-07-20T07:20:44 [TS: 1752996044] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 (Updated: 2025-07-20T07:20:44) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 (Updated: 2025-07-21T07:20:42 [TS: 1753082442] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 (Updated: 2025-07-21T07:20:42) -[2025-11-30 15:25:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 (Updated: 2025-07-22T07:21:21 [TS: 1753168881] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:10] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 (Updated: 2025-07-22T07:21:21) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 (Updated: 2025-07-23T07:19:58 [TS: 1753255198] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 (Updated: 2025-07-23T07:19:58) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 (Updated: 2025-07-24T07:20:48 [TS: 1753341648] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 (Updated: 2025-07-24T07:20:48) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 (Updated: 2025-07-25T07:22:43 [TS: 1753428163] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 (Updated: 2025-07-25T07:22:43) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c (Updated: 2025-07-26T07:20:47 [TS: 1753514447] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c (Updated: 2025-07-26T07:20:47) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a (Updated: 2025-07-27T07:19:47 [TS: 1753600787] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a (Updated: 2025-07-27T07:19:47) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c (Updated: 2025-07-28T07:22:46 [TS: 1753687366] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c (Updated: 2025-07-28T07:22:46) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 (Updated: 2025-07-29T07:23:16 [TS: 1753773796] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 (Updated: 2025-07-29T07:23:16) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e (Updated: 2025-07-30T07:22:14 [TS: 1753860134] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e (Updated: 2025-07-30T07:22:14) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 (Updated: 2025-07-31T07:21:38 [TS: 1753946498] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 (Updated: 2025-07-31T07:21:38) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 (Updated: 2025-08-01T07:20:39 [TS: 1754032839] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 (Updated: 2025-08-01T07:20:39) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 (Updated: 2025-08-02T07:20:51 [TS: 1754119251] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 (Updated: 2025-08-02T07:20:51) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d (Updated: 2025-08-03T07:20:40 [TS: 1754205640] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d (Updated: 2025-08-03T07:20:40) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 (Updated: 2025-08-04T07:21:29 [TS: 1754292089] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 (Updated: 2025-08-04T07:21:29) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 (Updated: 2025-08-05T07:20:11 [TS: 1754378411] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 (Updated: 2025-08-05T07:20:11) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 (Updated: 2025-08-06T07:20:49 [TS: 1754464849] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 (Updated: 2025-08-06T07:20:49) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 (Updated: 2025-08-07T07:21:48 [TS: 1754551308] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 (Updated: 2025-08-07T07:21:48) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 (Updated: 2025-08-08T07:20:18 [TS: 1754637618] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 (Updated: 2025-08-08T07:20:18) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c (Updated: 2025-08-09T07:21:20 [TS: 1754724080] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c (Updated: 2025-08-09T07:21:20) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c (Updated: 2025-08-10T07:21:33 [TS: 1754810493] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c (Updated: 2025-08-10T07:21:33) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e (Updated: 2025-08-11T07:21:44 [TS: 1754896904] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e (Updated: 2025-08-11T07:21:44) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd (Updated: 2025-08-12T07:21:45 [TS: 1754983305] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd (Updated: 2025-08-12T07:21:45) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 (Updated: 2025-08-13T07:20:21 [TS: 1755069621] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 (Updated: 2025-08-13T07:20:21) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 (Updated: 2025-08-14T07:21:25 [TS: 1755156085] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 (Updated: 2025-08-14T07:21:25) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd (Updated: 2025-08-15T07:19:57 [TS: 1755242397] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd (Updated: 2025-08-15T07:19:57) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a (Updated: 2025-08-16T07:20:46 [TS: 1755328846] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a (Updated: 2025-08-16T07:20:46) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 (Updated: 2025-08-17T07:21:09 [TS: 1755415269] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 (Updated: 2025-08-17T07:21:09) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 (Updated: 2025-08-18T07:21:01 [TS: 1755501661] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 (Updated: 2025-08-18T07:21:01) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 (Updated: 2025-08-19T07:20:59 [TS: 1755588059] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 (Updated: 2025-08-19T07:20:59) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 (Updated: 2025-08-20T07:21:18 [TS: 1755674478] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 (Updated: 2025-08-20T07:21:18) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 (Updated: 2025-08-21T07:21:02 [TS: 1755760862] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 (Updated: 2025-08-21T07:21:02) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 (Updated: 2025-08-22T07:20:17 [TS: 1755847217] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 (Updated: 2025-08-22T07:20:17) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e (Updated: 2025-08-23T07:22:26 [TS: 1755933746] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e (Updated: 2025-08-23T07:22:26) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b (Updated: 2025-08-24T07:21:13 [TS: 1756020073] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b (Updated: 2025-08-24T07:21:13) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 (Updated: 2025-08-25T07:23:27 [TS: 1756106607] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 (Updated: 2025-08-25T07:23:27) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 (Updated: 2025-08-26T07:21:52 [TS: 1756192912] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 (Updated: 2025-08-26T07:21:52) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 (Updated: 2025-08-27T07:22:57 [TS: 1756279377] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 (Updated: 2025-08-27T07:22:57) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f (Updated: 2025-08-28T07:23:00 [TS: 1756365780] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f (Updated: 2025-08-28T07:23:00) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c (Updated: 2025-08-29T07:21:43 [TS: 1756452103] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c (Updated: 2025-08-29T07:21:43) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 (Updated: 2025-08-30T07:22:13 [TS: 1756538533] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 (Updated: 2025-08-30T07:22:13) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c (Updated: 2025-08-31T07:21:37 [TS: 1756624897] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c (Updated: 2025-08-31T07:21:37) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e (Updated: 2025-09-01T07:22:45 [TS: 1756711365] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e (Updated: 2025-09-01T07:22:45) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd (Updated: 2025-09-02T07:20:23 [TS: 1756797623] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd (Updated: 2025-09-02T07:20:23) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 (Updated: 2025-09-03T07:22:13 [TS: 1756884133] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 (Updated: 2025-09-03T07:22:13) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 (Updated: 2025-09-04T07:19:51 [TS: 1756970391] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 (Updated: 2025-09-04T07:19:51) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc (Updated: 2025-09-05T07:20:48 [TS: 1757056848] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc (Updated: 2025-09-05T07:20:48) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 (Updated: 2025-09-06T07:21:41 [TS: 1757143301] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 (Updated: 2025-09-06T07:21:41) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 (Updated: 2025-09-07T07:22:23 [TS: 1757229743] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 (Updated: 2025-09-07T07:22:23) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 (Updated: 2025-09-08T07:22:09 [TS: 1757316129] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 (Updated: 2025-09-08T07:22:09) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 (Updated: 2025-09-09T07:22:21 [TS: 1757402541] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 (Updated: 2025-09-09T07:22:21) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 (Updated: 2025-09-10T07:22:45 [TS: 1757488965] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 (Updated: 2025-09-10T07:22:45) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b (Updated: 2025-09-11T07:21:36 [TS: 1757575296] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b (Updated: 2025-09-11T07:21:36) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 (Updated: 2025-09-12T07:21:47 [TS: 1757661707] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 (Updated: 2025-09-12T07:21:47) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df (Updated: 2025-09-13T07:22:23 [TS: 1757748143] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df (Updated: 2025-09-13T07:22:23) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 (Updated: 2025-09-14T07:21:44 [TS: 1757834504] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 (Updated: 2025-09-14T07:21:44) -[2025-11-30 15:25:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 (Updated: 2025-09-15T07:21:21 [TS: 1757920881] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:11] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 (Updated: 2025-09-15T07:21:21) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa (Updated: 2025-09-16T07:18:40 [TS: 1758007120] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa (Updated: 2025-09-16T07:18:40) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 (Updated: 2025-09-17T07:22:13 [TS: 1758093733] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 (Updated: 2025-09-17T07:22:13) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 (Updated: 2025-09-18T07:22:40 [TS: 1758180160] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 (Updated: 2025-09-18T07:22:40) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 (Updated: 2025-09-19T07:20:23 [TS: 1758266423] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 (Updated: 2025-09-19T07:20:23) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 (Updated: 2025-09-20T07:21:10 [TS: 1758352870] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 (Updated: 2025-09-20T07:21:10) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 (Updated: 2025-09-21T07:22:13 [TS: 1758439333] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 (Updated: 2025-09-21T07:22:13) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e (Updated: 2025-09-22T07:21:54 [TS: 1758525714] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e (Updated: 2025-09-22T07:21:54) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 (Updated: 2025-09-23T07:23:02 [TS: 1758612182] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 (Updated: 2025-09-23T07:23:02) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 (Updated: 2025-09-23T07:23:05 [TS: 1758612185] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 (Updated: 2025-09-23T07:23:05) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 (Updated: 2025-09-24T07:20:14 [TS: 1758698414] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 (Updated: 2025-09-24T07:20:14) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 (Updated: 2025-09-24T07:20:17 [TS: 1758698417] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 (Updated: 2025-09-24T07:20:17) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 (Updated: 2025-09-25T07:21:15 [TS: 1758784875] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 (Updated: 2025-09-25T07:21:15) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae (Updated: 2025-09-25T07:21:18 [TS: 1758784878] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae (Updated: 2025-09-25T07:21:18) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 (Updated: 2025-10-09T07:22:47 [TS: 1759994567] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 (Updated: 2025-10-09T07:22:47) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda (Updated: 2025-10-09T07:22:51 [TS: 1759994571] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda (Updated: 2025-10-09T07:22:51) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef (Updated: 2025-10-10T07:21:41 [TS: 1760080901] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef (Updated: 2025-10-10T07:21:41) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c (Updated: 2025-10-10T07:21:44 [TS: 1760080904] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c (Updated: 2025-10-10T07:21:44) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 (Updated: 2025-10-11T07:21:16 [TS: 1760167276] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 (Updated: 2025-10-11T07:21:16) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e (Updated: 2025-10-11T07:21:19 [TS: 1760167279] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e (Updated: 2025-10-11T07:21:19) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 (Updated: 2025-10-12T07:20:46 [TS: 1760253646] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 (Updated: 2025-10-12T07:20:46) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be (Updated: 2025-10-12T07:20:50 [TS: 1760253650] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be (Updated: 2025-10-12T07:20:50) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca (Updated: 2025-10-13T07:22:50 [TS: 1760340170] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca (Updated: 2025-10-13T07:22:50) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef (Updated: 2025-10-13T07:22:53 [TS: 1760340173] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef (Updated: 2025-10-13T07:22:53) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 (Updated: 2025-10-14T07:21:23 [TS: 1760426483] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 (Updated: 2025-10-14T07:21:23) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 (Updated: 2025-10-14T07:21:26 [TS: 1760426486] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 (Updated: 2025-10-14T07:21:26) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 (Updated: 2025-10-15T07:21:13 [TS: 1760512873] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 (Updated: 2025-10-15T07:21:13) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c (Updated: 2025-10-15T07:21:17 [TS: 1760512877] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c (Updated: 2025-10-15T07:21:17) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 (Updated: 2025-10-16T07:20:31 [TS: 1760599231] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 (Updated: 2025-10-16T07:20:31) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 (Updated: 2025-10-16T07:20:37 [TS: 1760599237] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 (Updated: 2025-10-16T07:20:37) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be (Updated: 2025-10-17T07:22:22 [TS: 1760685742] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be (Updated: 2025-10-17T07:22:22) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 (Updated: 2025-10-17T07:22:28 [TS: 1760685748] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 (Updated: 2025-10-17T07:22:28) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 (Updated: 2025-10-18T07:19:49 [TS: 1760771989] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 (Updated: 2025-10-18T07:19:49) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 (Updated: 2025-10-18T07:19:55 [TS: 1760771995] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 (Updated: 2025-10-18T07:19:55) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 (Updated: 2025-10-19T07:20:50 [TS: 1760858450] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 (Updated: 2025-10-19T07:20:50) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 (Updated: 2025-10-19T07:20:57 [TS: 1760858457] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 (Updated: 2025-10-19T07:20:57) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a (Updated: 2025-10-20T07:22:34 [TS: 1760944954] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a (Updated: 2025-10-20T07:22:34) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 (Updated: 2025-10-20T07:22:40 [TS: 1760944960] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 (Updated: 2025-10-20T07:22:40) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 (Updated: 2025-10-21T07:23:22 [TS: 1761031402] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 (Updated: 2025-10-21T07:23:22) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 (Updated: 2025-10-21T07:23:25 [TS: 1761031405] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 (Updated: 2025-10-21T07:23:25) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 (Updated: 2025-10-22T07:21:05 [TS: 1761117665] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 (Updated: 2025-10-22T07:21:05) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 (Updated: 2025-10-22T07:21:08 [TS: 1761117668] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 (Updated: 2025-10-22T07:21:08) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 (Updated: 2025-10-23T07:21:12 [TS: 1761204072] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 (Updated: 2025-10-23T07:21:12) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 (Updated: 2025-10-23T07:21:15 [TS: 1761204075] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 (Updated: 2025-10-23T07:21:15) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 (Updated: 2025-10-24T07:21:00 [TS: 1761290460] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 (Updated: 2025-10-24T07:21:00) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f (Updated: 2025-10-24T07:21:03 [TS: 1761290463] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f (Updated: 2025-10-24T07:21:03) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 (Updated: 2025-10-25T07:22:28 [TS: 1761376948] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 (Updated: 2025-10-25T07:22:28) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 (Updated: 2025-10-25T07:22:32 [TS: 1761376952] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 (Updated: 2025-10-25T07:22:32) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 (Updated: 2025-10-26T07:17:54 [TS: 1761463074] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 (Updated: 2025-10-26T07:17:54) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 (Updated: 2025-10-26T07:18:06 [TS: 1761463086] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 (Updated: 2025-10-26T07:18:06) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e (Updated: 2025-10-27T07:20:57 [TS: 1761549657] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e (Updated: 2025-10-27T07:20:57) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e (Updated: 2025-10-27T07:21:00 [TS: 1761549660] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e (Updated: 2025-10-27T07:21:00) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 (Updated: 2025-10-28T07:21:03 [TS: 1761636063] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 (Updated: 2025-10-28T07:21:03) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c (Updated: 2025-10-28T07:21:09 [TS: 1761636069] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c (Updated: 2025-10-28T07:21:09) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a (Updated: 2025-10-29T07:21:55 [TS: 1761722515] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a (Updated: 2025-10-29T07:21:55) -[2025-11-30 15:25:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 (Updated: 2025-10-29T07:22:01 [TS: 1761722521] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:12] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 (Updated: 2025-10-29T07:22:01) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 (Updated: 2025-10-30T07:21:11 [TS: 1761808871] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 (Updated: 2025-10-30T07:21:11) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf (Updated: 2025-10-30T07:21:17 [TS: 1761808877] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf (Updated: 2025-10-30T07:21:17) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 (Updated: 2025-10-31T07:22:12 [TS: 1761895332] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 (Updated: 2025-10-31T07:22:12) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 (Updated: 2025-10-31T07:22:18 [TS: 1761895338] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 (Updated: 2025-10-31T07:22:18) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa (Updated: 2025-11-01T07:22:50 [TS: 1761981770] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa (Updated: 2025-11-01T07:22:50) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 (Updated: 2025-11-01T07:22:54 [TS: 1761981774] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 (Updated: 2025-11-01T07:22:54) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 (Updated: 2025-11-02T07:21:16 [TS: 1762068076] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 (Updated: 2025-11-02T07:21:16) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 (Updated: 2025-11-02T07:21:20 [TS: 1762068080] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 (Updated: 2025-11-02T07:21:20) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 (Updated: 2025-11-03T08:23:08 [TS: 1762158188] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 (Updated: 2025-11-03T08:23:08) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a (Updated: 2025-11-03T08:23:12 [TS: 1762158192] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a (Updated: 2025-11-03T08:23:12) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 (Updated: 2025-11-04T08:17:45 [TS: 1762244265] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 (Updated: 2025-11-04T08:17:45) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 (Updated: 2025-11-04T08:17:48 [TS: 1762244268] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 (Updated: 2025-11-04T08:17:48) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 (Updated: 2025-11-05T08:20:13 [TS: 1762330813] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 (Updated: 2025-11-05T08:20:13) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 (Updated: 2025-11-05T08:20:17 [TS: 1762330817] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 (Updated: 2025-11-05T08:20:17) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a (Updated: 2025-11-06T08:20:55 [TS: 1762417255] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a (Updated: 2025-11-06T08:20:55) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 (Updated: 2025-11-06T08:20:58 [TS: 1762417258] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 (Updated: 2025-11-06T08:20:58) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd (Updated: 2025-11-07T08:18:30 [TS: 1762503510] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd (Updated: 2025-11-07T08:18:30) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a (Updated: 2025-11-07T08:18:33 [TS: 1762503513] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a (Updated: 2025-11-07T08:18:33) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af (Updated: 2025-11-08T08:18:22 [TS: 1762589902] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af (Updated: 2025-11-08T08:18:22) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 (Updated: 2025-11-08T08:18:26 [TS: 1762589906] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 (Updated: 2025-11-08T08:18:26) -[2025-11-30 15:25:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd (Updated: 2025-11-09T08:21:42 [TS: 1762676502] < Cutoff: [TS: 1763306705]) -[2025-11-30 15:25:13] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd (Updated: 2025-11-09T08:21:42) -[2025-11-30 15:25:13] [INFO] Hit delete limit (200) for Docker Images. -[2025-11-30 15:25:13] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 15:25:13] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 15:25:15] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 15:25:15] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 15:25:15] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 15:25:15] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 15:25:15] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 15:25:15] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 15:25:18] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 15:25:18] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:25:20] [INFO] No Regional Address found matching criteria. -[2025-11-30 15:25:20] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:25:23] [INFO] No Global Address found matching criteria. -[2025-11-30 15:25:23] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- -[2025-11-30 15:25:27] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 15:25:30] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:25:30] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:25:30] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 15:25:30] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 15:25:30] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:32] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:33] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 15:25:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:25:35] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 15:25:37] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 15:25:37] [INFO] CLEANUP RUN FINISHED -[2025-11-30 15:26:40] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 15:26:40] [INFO] Time Cutoff (General): 2025-11-30T15:26:40+0000 -[2025-11-30 15:26:40] [INFO] Time Cutoff (Images): 2025-10-01T15:26:40+0000 -[2025-11-30 15:26:40] [INFO] Delete Limit per Type: 200 -[2025-11-30 15:26:40] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 15:26:41] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 15:26:44] [INFO] No Service Accounts found matching prefix. -[2025-11-30 15:26:44] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:26:46] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 15:26:46] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 15:26:49] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:26:49] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:26:49] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 15:26:49] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:26:52] [INFO] No Filestore instances found matching criteria. -[2025-11-30 15:26:52] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 15:26:55] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 15:26:55] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 15:26:55] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 15:26:55] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 15:26:55] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 15:26:55] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 15:26:55] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 15:26:55] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 15:26:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 15:26:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 15:26:56] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 15:26:56] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:26:56Z (Unix: 1763306816) -[2025-11-30 15:26:56] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 15:27:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 (Updated: 2025-05-15T07:21:31 [TS: 1747293691] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 (Updated: 2025-05-15T07:21:31) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/dd64c01e-245f-411d-af04-01a07018e5fd] to complete... -......done. -[2025-11-30 15:27:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:de5ecf57eff17d6ee35d538e6dc8bc8916d13fd1d5547405fea95db79378b506 -[2025-11-30 15:27:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 (Updated: 2025-05-16T07:21:17 [TS: 1747380077] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 (Updated: 2025-05-16T07:21:17) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0e68f557-1e12-444c-af99-5d75e4759d05] to complete... -.....done. -[2025-11-30 15:27:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e109f4fc9615491d900a1f698c9766c2b555894ed60414d016757b85a1f5a12 -[2025-11-30 15:27:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da (Updated: 2025-05-17T07:21:15 [TS: 1747466475] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da (Updated: 2025-05-17T07:21:15) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f8eff88c-80b2-4d55-92c2-76868b25fa98] to complete... -.....done. -[2025-11-30 15:27:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e9a2e7f95d8e1e3612f14f10445794ed680735eadafb814c4fa57234bc6913da -[2025-11-30 15:27:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d (Updated: 2025-05-18T07:21:43 [TS: 1747552903] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d (Updated: 2025-05-18T07:21:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6909be50-62d0-438d-9fdc-743de9d1ff34] to complete... -.....done. -[2025-11-30 15:27:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b9bd8c6782b9b41e7633a513aa4c84635a7aa7780d61b0bf526971643aa9624d -[2025-11-30 15:27:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de (Updated: 2025-05-19T07:21:57 [TS: 1747639317] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de (Updated: 2025-05-19T07:21:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/88c0a1a5-03ba-45a4-bdc7-6cc526928dd3] to complete... -.....done. -[2025-11-30 15:27:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:704068358ee24e29eaeb0b48aec7a5d1155a5cf6e2d4db6cdc74871a7ecf40de -[2025-11-30 15:27:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 (Updated: 2025-05-20T07:22:12 [TS: 1747725732] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 (Updated: 2025-05-20T07:22:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/77bb28e8-ba96-4a90-9896-0ffac7d5b846] to complete... -.....done. -[2025-11-30 15:27:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:57d95cd984bc80bf70e97d51dd3f82283717f76cc4cbe9bbd0bb1d7a853e91c4 -[2025-11-30 15:27:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 (Updated: 2025-05-21T07:19:51 [TS: 1747811991] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 (Updated: 2025-05-21T07:19:51) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7ab4dc78-adba-489a-adea-5afd2f7d628d] to complete... -.....done. -[2025-11-30 15:27:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4162485ecfa8bf450eb95705e274807b25b4603baa766b37a53a34d0ef49a98 -[2025-11-30 15:27:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f (Updated: 2025-05-22T07:21:21 [TS: 1747898481] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f (Updated: 2025-05-22T07:21:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b033446a-9573-424a-b53e-7f6731ff4926] to complete... -.....done. -[2025-11-30 15:27:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:601f09661aaa05c9fb9844967ee03a9a48e33125a2e2202c65afa0431aefe91f -[2025-11-30 15:27:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 (Updated: 2025-05-23T07:22:28 [TS: 1747984948] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 (Updated: 2025-05-23T07:22:28) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/12347116-b964-4e06-b05b-8f7444f3d1cd] to complete... -.....done. -[2025-11-30 15:27:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:826fccdb985e2a6c321bed2ec446ec8ed9c4be090e99f2477665b9fcf1088bb6 -[2025-11-30 15:27:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c (Updated: 2025-05-24T07:21:57 [TS: 1748071317] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c (Updated: 2025-05-24T07:21:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/596db79a-b74d-45bd-aa19-e790ea172040] to complete... -......done. -[2025-11-30 15:27:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ddf957811bdc83f2cbb45c3f2f97adefd40891b6fcef271ead54919c5738622c -[2025-11-30 15:27:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be (Updated: 2025-05-25T07:21:23 [TS: 1748157683] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be (Updated: 2025-05-25T07:21:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/50afc1a9-eb5c-4f4f-ad2a-e577c45983c7] to complete... -.....done. -[2025-11-30 15:27:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00ab4699e10c4a80f27af9caa7d6b46da1e8263bce1bf41fefa9b6cc8c3273be -[2025-11-30 15:27:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 (Updated: 2025-05-26T07:28:04 [TS: 1748244484] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 (Updated: 2025-05-26T07:28:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5caef63e-0db0-4f6d-b9f6-83be2350dc0a] to complete... -.....done. -[2025-11-30 15:27:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0fa0a9a5a7a9eee01317f0564163cdb92acf31c545581cb6627dd4ab51690fb7 -[2025-11-30 15:27:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 (Updated: 2025-05-27T07:19:54 [TS: 1748330394] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 (Updated: 2025-05-27T07:19:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/82deb512-b38d-4efa-9826-ef5a78bcbc10] to complete... -.....done. -[2025-11-30 15:27:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5de49427c1bac669d2d16be7a306bf7664169b77979d1b94286e0c5cc0f6d995 -[2025-11-30 15:27:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a (Updated: 2025-05-28T07:20:28 [TS: 1748416828] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a (Updated: 2025-05-28T07:20:28) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/51755698-a4a5-4c79-9f6f-1918e2c8b793] to complete... -.....done. -[2025-11-30 15:27:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b930c9eacb73cbf362a1b051307db09f94a7c90a9fbf693c29d76853b4ecd5a -[2025-11-30 15:27:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d (Updated: 2025-05-29T07:21:22 [TS: 1748503282] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d (Updated: 2025-05-29T07:21:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2335d05b-70b7-4404-8dc7-7e9451ba2ac4] to complete... -.....done. -[2025-11-30 15:27:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e0972f6ece2953ff16bd16e66ff738b8ec23ad3ceac168d5b303b8e498d0fd9d -[2025-11-30 15:27:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 (Updated: 2025-05-30T07:21:07 [TS: 1748589667] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 (Updated: 2025-05-30T07:21:07) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/061ee211-0be7-4971-99c4-17e0a30bed95] to complete... -.....done. -[2025-11-30 15:27:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6df7747b92659d7d41552cffe125880b4042e571473723cf456ae42806839201 -[2025-11-30 15:27:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 (Updated: 2025-05-31T07:20:46 [TS: 1748676046] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:27:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 (Updated: 2025-05-31T07:20:46) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ab03e21b-4029-419f-abd5-b1e2ec1c8b65] to complete... -.....done. -[2025-11-30 15:28:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4f0319e00f0f6cebe23c5ddd6fd73460d27c8885bd3900d76cd552944becef4 -[2025-11-30 15:28:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 (Updated: 2025-06-01T07:20:52 [TS: 1748762452] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 (Updated: 2025-06-01T07:20:52) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/362825fc-af7b-4921-8dc0-728be3e5d609] to complete... -.....done. -[2025-11-30 15:28:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:858874289a12a2268e9d5581bff4efeeb055064d27bdea57beacd87264385e68 -[2025-11-30 15:28:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a (Updated: 2025-06-02T07:23:55 [TS: 1748849035] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a (Updated: 2025-06-02T07:23:55) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f5065687-1a44-4b5a-b165-d44e5e2681a0] to complete... -......done. -[2025-11-30 15:28:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:897941e4a90e91c7d931d86ec591a0a7b7192cc5c09ed57c7f177d66652aa68a -[2025-11-30 15:28:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 (Updated: 2025-06-03T07:21:05 [TS: 1748935265] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 (Updated: 2025-06-03T07:21:05) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3e5cb436-cd37-44bc-b0af-317f1f5abc60] to complete... -.....done. -[2025-11-30 15:28:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e247b317e6faf070f631eda46c2b917e169cf3feee161fbc6cfdb9ee80243c19 -[2025-11-30 15:28:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd (Updated: 2025-06-04T07:22:23 [TS: 1749021743] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd (Updated: 2025-06-04T07:22:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/80c687e8-979b-4b48-a497-748e018f7170] to complete... -......done. -[2025-11-30 15:28:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7fa6b477f1fd4be8d41e589ed9454f014b66b0932d74000abe99c6bfc1c089bd -[2025-11-30 15:28:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 (Updated: 2025-06-05T07:21:27 [TS: 1749108087] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 (Updated: 2025-06-05T07:21:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/266d57b5-0a51-4fea-a982-d8518e68319a] to complete... -.....done. -[2025-11-30 15:28:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d18cad2cc7096c71c19b2e5776e181a7631a11210d4eeb4a2313c783eaa0531 -[2025-11-30 15:28:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf (Updated: 2025-06-06T07:22:27 [TS: 1749194547] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf (Updated: 2025-06-06T07:22:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4d27c469-2f39-4c46-9db2-1e17880f017e] to complete... -.....done. -[2025-11-30 15:28:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:22cce3882156f01b0e78895c38672cc1135ff7d99e7c785a47966fd032019daf -[2025-11-30 15:28:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c (Updated: 2025-06-07T07:22:15 [TS: 1749280935] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c (Updated: 2025-06-07T07:22:15) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bacc7e17-11de-440e-9fc0-d431e5dfe8a6] to complete... -.....done. -[2025-11-30 15:28:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fe63a0e0edad358df4b35740a4d60a95107efc043f3942fe3fe29c938025171c -[2025-11-30 15:28:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 (Updated: 2025-06-08T07:21:06 [TS: 1749367266] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 (Updated: 2025-06-08T07:21:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a6f0d1c7-96dc-438c-ae33-0d1df8014aee] to complete... -.....done. -[2025-11-30 15:28:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eb55829ec989ab68e852c1436b54f825c9f6111b71b8d4b0606bbb1ac8653c54 -[2025-11-30 15:28:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f (Updated: 2025-06-09T07:20:04 [TS: 1749453604] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f (Updated: 2025-06-09T07:20:04) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b5467a09-5203-4b85-b82f-5f99b515eb44] to complete... -.....done. -[2025-11-30 15:28:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:09b77513802c57700f1d93ba0d64acd20384e4d4d7c34e173163448b4ebb948f -[2025-11-30 15:28:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f (Updated: 2025-06-10T07:20:54 [TS: 1749540054] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f (Updated: 2025-06-10T07:20:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2a09dca9-84b8-4153-a3cb-5b2aca6140fb] to complete... -.....done. -[2025-11-30 15:28:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73f1671e30d168ee4db0dbe81bf6dbed7e8d60996b699cab2108cf27d4dcdf4f -[2025-11-30 15:28:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 (Updated: 2025-06-11T07:20:59 [TS: 1749626459] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 (Updated: 2025-06-11T07:20:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/547c8e26-dbe6-49f2-808e-f0f67f883784] to complete... -......done. -[2025-11-30 15:28:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ab738ef88a01ab92c95fd461b7127c9864476df22d5c1105447daef4e091599 -[2025-11-30 15:28:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 (Updated: 2025-06-12T07:19:36 [TS: 1749712776] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 (Updated: 2025-06-12T07:19:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9a3cc358-a13e-4216-b6cd-d3077eed7c2e] to complete... -.....done. -[2025-11-30 15:28:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7ee65112c0981f622d05e6a9b7ed488ce626f35938f7b8ece2ff08f5c2974148 -[2025-11-30 15:28:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 (Updated: 2025-06-13T07:21:52 [TS: 1749799312] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 (Updated: 2025-06-13T07:21:52) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/81e696f9-e0ec-4326-89c7-623f85f250eb] to complete... -.....done. -[2025-11-30 15:28:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6e820896ae60ff084cad8f2578ee658da0168208368a5f3c3e6b2091bbd92694 -[2025-11-30 15:28:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 (Updated: 2025-06-14T07:21:53 [TS: 1749885713] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 (Updated: 2025-06-14T07:21:53) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/616e1f3d-51f5-4ebe-851d-46266a5723c4] to complete... -.....done. -[2025-11-30 15:28:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:86e349936cdb90ab09f2ea29ff919af7c626e154c9026372ecefae67bdf9e6f9 -[2025-11-30 15:28:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 (Updated: 2025-06-15T07:20:57 [TS: 1749972057] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 (Updated: 2025-06-15T07:20:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3f478b30-d35a-47b1-a9cd-1e391ff0ee30] to complete... -.....done. -[2025-11-30 15:28:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1e3e81b8ae9b107ed339182bc2763d71e69cbdf3d224c16ede7692949d363d01 -[2025-11-30 15:28:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 (Updated: 2025-06-16T07:20:18 [TS: 1750058418] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 (Updated: 2025-06-16T07:20:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1c76ab08-3161-4401-895b-3ade123d1890] to complete... -......done. -[2025-11-30 15:28:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9263a67ae8ec5f33a1d3470d8049ba098fcc4165d7784fbc659a3b0cd5772434 -[2025-11-30 15:28:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe (Updated: 2025-06-17T07:21:49 [TS: 1750144909] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:28:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe (Updated: 2025-06-17T07:21:49) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/05a89e17-f248-4c3e-8e9b-03d1044cfcdc] to complete... -.....done. -[2025-11-30 15:29:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0c45c3e6ac248419b5ae1700756f8e234c064faff60cda9d4f83d32bdb45adfe -[2025-11-30 15:29:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 (Updated: 2025-06-18T07:20:13 [TS: 1750231213] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 (Updated: 2025-06-18T07:20:13) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/25101083-fdb8-4b84-85bd-67cabd30fd9a] to complete... -.....done. -[2025-11-30 15:29:04] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73306ac75be615066102987da4cd2b3220763395019933e425825c9f1c90e273 -[2025-11-30 15:29:04] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 (Updated: 2025-06-19T07:20:59 [TS: 1750317659] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:04] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 (Updated: 2025-06-19T07:20:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b645df05-937c-4faa-8fd2-45fd27356faa] to complete... -......done. -[2025-11-30 15:29:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5bb5b1854cf3a2f6cbe01c1db6f778f986506741e9df7ea7e27d91f87a828e3 -[2025-11-30 15:29:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b (Updated: 2025-06-20T07:21:01 [TS: 1750404061] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b (Updated: 2025-06-20T07:21:01) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4ac9df5b-c155-4ab4-af44-cbe3a2a21984] to complete... -.....done. -[2025-11-30 15:29:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6500a2517d7ae5bf4a977b5d932f25840185dd0b5aa78e37f4f86564700e172b -[2025-11-30 15:29:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f (Updated: 2025-06-21T07:22:20 [TS: 1750490540] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f (Updated: 2025-06-21T07:22:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d3a9e7c9-7c07-476a-8fb8-46fd7e86b495] to complete... -.....done. -[2025-11-30 15:29:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7572503d2f9dd3044c68fc69b5ce9629af9cc0042096ae3d075963f652330b3f -[2025-11-30 15:29:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb (Updated: 2025-06-22T07:21:21 [TS: 1750576881] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb (Updated: 2025-06-22T07:21:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eb89a0c0-0956-4658-af00-ef87c5d94c85] to complete... -.....done. -[2025-11-30 15:29:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7f7374b822464a229d09c8e5795ee425292fd7ebc1575daa40965037929e14eb -[2025-11-30 15:29:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 (Updated: 2025-06-23T07:21:18 [TS: 1750663278] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 (Updated: 2025-06-23T07:21:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/315d43a0-bcf1-416c-add0-6ce5a883034f] to complete... -......done. -[2025-11-30 15:29:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a54ede10966c08a155bd49bc858f072cb343f0140d1887ea0351057bdea7c0b1 -[2025-11-30 15:29:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 (Updated: 2025-06-24T07:21:15 [TS: 1750749675] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 (Updated: 2025-06-24T07:21:15) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6773eb36-884d-45a7-8739-c4e5290b12ba] to complete... -.....done. -[2025-11-30 15:29:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3818b1e12922faa703f83dc59e9a8b43432fab1c7eb44838f2bbd996883ebc2 -[2025-11-30 15:29:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 (Updated: 2025-06-25T07:21:11 [TS: 1750836071] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 (Updated: 2025-06-25T07:21:11) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/eb6b2f8d-57de-4d56-a554-7905f7a31f6b] to complete... -.....done. -[2025-11-30 15:29:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:838cecf896f786a8af6251896e5a6202178e566b4a2e23f5bc4ca6abdb45e449 -[2025-11-30 15:29:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba (Updated: 2025-06-26T07:20:29 [TS: 1750922429] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba (Updated: 2025-06-26T07:20:29) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/db1c55b0-9f43-40ab-b8f0-f2677771e2ab] to complete... -.....done. -[2025-11-30 15:29:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:87229727f11b95e048b5b60d7318e7ff63822282baf816e957a29d18ea3faaba -[2025-11-30 15:29:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c (Updated: 2025-06-27T07:20:18 [TS: 1751008818] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c (Updated: 2025-06-27T07:20:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ee175c44-cc0f-464f-8ef7-36d20de2c9a9] to complete... -.....done. -[2025-11-30 15:29:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2146745c972aa3a3a0e6ae063512fafcbd7335984c85180206bc5f90fef55e5c -[2025-11-30 15:29:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb (Updated: 2025-06-28T07:20:26 [TS: 1751095226] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb (Updated: 2025-06-28T07:20:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a55b3ca4-eda6-4e25-b245-e3bf9e6e4ab9] to complete... -.....done. -[2025-11-30 15:29:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcde260f427fedf8327b67851d23434a3e171dea23deab391ae1a229d527d5cb -[2025-11-30 15:29:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c (Updated: 2025-06-29T07:21:11 [TS: 1751181671] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c (Updated: 2025-06-29T07:21:11) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/72c895ea-3cd7-4ffd-ad23-ac104f0b998d] to complete... -.....done. -[2025-11-30 15:29:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cfbfc958ffc708017ab60bfe5c343f771bbc0f1f00f2e7620dbfd841291f067c -[2025-11-30 15:29:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 (Updated: 2025-06-30T07:22:37 [TS: 1751268157] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 (Updated: 2025-06-30T07:22:37) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/413fa74f-8f29-44d9-921f-97f856b30267] to complete... -.....done. -[2025-11-30 15:29:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1500259ec07c258dd80c2df390291410bb06b6b9f756750b5cc8b94f1e93c3f8 -[2025-11-30 15:29:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f (Updated: 2025-07-01T07:20:54 [TS: 1751354454] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f (Updated: 2025-07-01T07:20:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/89a43850-b016-42ed-82f5-6d68726e2a18] to complete... -.....done. -[2025-11-30 15:29:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6058d3b401469de5585bffcefa4bee7f056f7a9daa721bbeaf42b791a0fbc95f -[2025-11-30 15:29:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 (Updated: 2025-07-02T07:21:52 [TS: 1751440912] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 (Updated: 2025-07-02T07:21:52) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0e040ef4-b2df-421e-aced-9f20b59fe47f] to complete... -.....done. -[2025-11-30 15:29:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:012bca92adb12186cd43d4543c0570a14a758a79e9cf9ddaeb74893100cd5d08 -[2025-11-30 15:29:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb (Updated: 2025-07-03T07:20:06 [TS: 1751527206] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb (Updated: 2025-07-03T07:20:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/269b3e69-07cd-4636-ba32-48dc62e4d61f] to complete... -.....done. -[2025-11-30 15:29:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:db888026d5e2c9089a03baceefc2a488d55dcc56aee166c6714932064e9899fb -[2025-11-30 15:29:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e (Updated: 2025-07-04T07:21:34 [TS: 1751613694] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:29:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e (Updated: 2025-07-04T07:21:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/367e73c5-2fa3-489e-affe-5c182ef1f215] to complete... -.....done. -[2025-11-30 15:30:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e789363d740a800098d34ac62d2253a41357c38604a358199ba21c7e14df5d2e -[2025-11-30 15:30:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 (Updated: 2025-07-05T07:21:32 [TS: 1751700092] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 (Updated: 2025-07-05T07:21:32) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6616a7bd-fb9f-4a49-903a-d1e16c032f3b] to complete... -......done. -[2025-11-30 15:30:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63d97ae38b35a49f8d78340a53fc3abdc2b402a9ca36c2ba8bb7d075d32da5b1 -[2025-11-30 15:30:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf (Updated: 2025-07-06T07:21:21 [TS: 1751786481] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf (Updated: 2025-07-06T07:21:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/416917dd-c2cf-4c24-aa04-a960b5f37eee] to complete... -......done. -[2025-11-30 15:30:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:903b2a3be824e33ce16fadfcab9745d347648abd8de2f3063173337e69ca7ddf -[2025-11-30 15:30:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 (Updated: 2025-07-07T07:22:25 [TS: 1751872945] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 (Updated: 2025-07-07T07:22:25) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9814478d-649b-44b9-b160-8db3a4c81c7d] to complete... -.....done. -[2025-11-30 15:30:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:04c57bfe68e4e01a89cc8ee7e9ec0badc32bfd483222c765780fe012aa475545 -[2025-11-30 15:30:13] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 (Updated: 2025-07-08T07:21:26 [TS: 1751959286] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:13] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 (Updated: 2025-07-08T07:21:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/46bd48bd-826d-466c-932c-7a9c52c1adef] to complete... -.....done. -[2025-11-30 15:30:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c0512a6a028fb3ddc0e4d5b6579bf3512a529def58e43d4ec5276d547c95613 -[2025-11-30 15:30:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 (Updated: 2025-07-09T07:22:45 [TS: 1752045765] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 (Updated: 2025-07-09T07:22:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f1eb9bb1-e3f7-4c90-b5f5-aeb47edb2c17] to complete... -.....done. -[2025-11-30 15:30:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8d9438c51a119371befe646aa73edcf27909135b186deac1a105b8d4a1b89ba7 -[2025-11-30 15:30:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 (Updated: 2025-07-10T07:21:44 [TS: 1752132104] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 (Updated: 2025-07-10T07:21:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c4d28673-55fc-4d6e-825d-b7b0d4b5c401] to complete... -......done. -[2025-11-30 15:30:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a32a377c936e400d010e22a74099935942fcdeb1c682b8407923a7df26e90f03 -[2025-11-30 15:30:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b (Updated: 2025-07-11T07:21:43 [TS: 1752218503] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b (Updated: 2025-07-11T07:21:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7241bee7-7f5c-4441-bd29-9fa3de2687b1] to complete... -.....done. -[2025-11-30 15:30:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:53ae1d8ad8a692b98440a7e281832aa6ffd1e651deb2b974fbfc66979ddeb09b -[2025-11-30 15:30:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 (Updated: 2025-07-12T07:20:14 [TS: 1752304814] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 (Updated: 2025-07-12T07:20:14) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d81fc468-5e43-4e2c-92dd-18fe2bdbaec1] to complete... -.....done. -[2025-11-30 15:30:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f6df2891f2b27cae77c337b95109ca21d6ac5987b38edf6bfc7326a64f3cc222 -[2025-11-30 15:30:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 (Updated: 2025-07-13T07:22:16 [TS: 1752391336] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 (Updated: 2025-07-13T07:22:16) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ebe9fd6c-3195-4c45-b7ca-cea46b9a181a] to complete... -.....done. -[2025-11-30 15:30:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48447dcb7d85affeeff1358843cd5b54ab0434994f29369e6cf77be5134109d8 -[2025-11-30 15:30:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 (Updated: 2025-07-14T07:20:21 [TS: 1752477621] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 (Updated: 2025-07-14T07:20:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6d0f1a80-677b-474d-a79a-fb2ab8bcb6fb] to complete... -.....done. -[2025-11-30 15:30:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3868e68939c1827808814d9f20e611c575445aecc28d5c318cf80d2a88c3f812 -[2025-11-30 15:30:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a (Updated: 2025-07-15T07:21:57 [TS: 1752564117] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a (Updated: 2025-07-15T07:21:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/068cb159-da1a-4671-8865-308cffb998ae] to complete... -......done. -[2025-11-30 15:30:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b5caf832b1fcc8cb0d145845271e672f4b566ec21857ad0189ff31b16ca829a -[2025-11-30 15:30:41] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 (Updated: 2025-07-16T07:20:47 [TS: 1752650447] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:41] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 (Updated: 2025-07-16T07:20:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d5e9cf1c-eff4-457e-8291-4e32780485a9] to complete... -.....done. -[2025-11-30 15:30:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9dea5b5faec57bf0df8978207baccf7088c1e224c62db8c8d9a49e63f451e486 -[2025-11-30 15:30:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d (Updated: 2025-07-17T07:20:43 [TS: 1752736843] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d (Updated: 2025-07-17T07:20:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ed86b53e-e238-4add-b8e4-0b1c6302f8c1] to complete... -.....done. -[2025-11-30 15:30:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:33072331152d54ddb150a7347b6e73d66ecc0ca43c3dcda61456b763f076a86d -[2025-11-30 15:30:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 (Updated: 2025-07-18T07:22:46 [TS: 1752823366] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 (Updated: 2025-07-18T07:22:46) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/362e6ff0-7f50-4f9d-8b9d-c9a0f7e8b7a0] to complete... -.....done. -[2025-11-30 15:30:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:52d760f0f7dad34f75543e92818030ac5cf6b1411409b5b059764cf9c5d90e05 -[2025-11-30 15:30:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 (Updated: 2025-07-19T07:21:08 [TS: 1752909668] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 (Updated: 2025-07-19T07:21:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c9676628-863d-4941-bd0c-68e07fed481e] to complete... -.....done. -[2025-11-30 15:30:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:366de37a96cb771b6f12b35542e82f5c65732d47866695e7598aa9687e3f5649 -[2025-11-30 15:30:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 (Updated: 2025-07-20T07:20:44 [TS: 1752996044] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 (Updated: 2025-07-20T07:20:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/32b5f987-3fb9-4714-9976-3e9e46fb4af5] to complete... -.....done. -[2025-11-30 15:30:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:326cc43438318c4a6d1c102ef6bfcdc3f3200dddaa9a9ad0b716bf3557883082 -[2025-11-30 15:30:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 (Updated: 2025-07-21T07:20:42 [TS: 1753082442] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:30:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 (Updated: 2025-07-21T07:20:42) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/aafd4a78-d13e-4d74-ade5-45a598f08473] to complete... -.....done. -[2025-11-30 15:31:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f0c8150716d706fcfeb0b7de5ec7175b764c444b0e5dbbda68c45c45f372e517 -[2025-11-30 15:31:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 (Updated: 2025-07-22T07:21:21 [TS: 1753168881] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 (Updated: 2025-07-22T07:21:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f59135cf-c887-4cb6-8744-82a38010d581] to complete... -.....done. -[2025-11-30 15:31:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:856a10d6554de318285c82e1c740b55e8aa836d866e91e800fd592fc82fb3265 -[2025-11-30 15:31:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 (Updated: 2025-07-23T07:19:58 [TS: 1753255198] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 (Updated: 2025-07-23T07:19:58) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/18bbb4a6-db80-4b3e-8749-370e1f4122f2] to complete... -.....done. -[2025-11-30 15:31:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e2b6a75e9271626a7e094b78d49d9265d91fdaa9ab5b6a22bcfee6347ec6812 -[2025-11-30 15:31:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 (Updated: 2025-07-24T07:20:48 [TS: 1753341648] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 (Updated: 2025-07-24T07:20:48) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/908b321c-53c9-4d5c-83b9-038a3aaf0169] to complete... -......done. -[2025-11-30 15:31:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bc4b935ff71080fa8ba95682a14e51b4b66cee51b4d4efdd5c7786fcaa13bab1 -[2025-11-30 15:31:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 (Updated: 2025-07-25T07:22:43 [TS: 1753428163] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 (Updated: 2025-07-25T07:22:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/07b26725-df87-4f42-8823-19e510a41785] to complete... -......done. -[2025-11-30 15:31:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:69a1e4e3373bcdf04be4d8609924d988bcb27d123504438695f1df16139bf3c3 -[2025-11-30 15:31:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c (Updated: 2025-07-26T07:20:47 [TS: 1753514447] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c (Updated: 2025-07-26T07:20:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/07c7a440-7803-4e55-8b67-a28f6c892542] to complete... -......done. -[2025-11-30 15:31:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1c43d16f5390f4973a11145a51e9575e4b209a574a1faf28b920eda2c7b7587c -[2025-11-30 15:31:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a (Updated: 2025-07-27T07:19:47 [TS: 1753600787] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a (Updated: 2025-07-27T07:19:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6840aad2-7da6-4f1e-bf40-c37bb52e136c] to complete... -......done. -[2025-11-30 15:31:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c078e7e77e33d9a7c57ac4392c6429b0cafaf2e4d66f9bb03d388312ce8fb14a -[2025-11-30 15:31:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c (Updated: 2025-07-28T07:22:46 [TS: 1753687366] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c (Updated: 2025-07-28T07:22:46) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/53fff091-b58d-46fa-af18-4288c48e4249] to complete... -.....done. -[2025-11-30 15:31:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4ad57b389cfe248191f61ff9c263055b0870aed9893ba9d25c87e0ad79e78b9c -[2025-11-30 15:31:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 (Updated: 2025-07-29T07:23:16 [TS: 1753773796] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 (Updated: 2025-07-29T07:23:16) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b6f40155-01ee-48a3-bb59-cee8eb1f30be] to complete... -.....done. -[2025-11-30 15:31:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3120413239121f64c2bf5e693fbfff633193de82cae126cceddb8ec1c755e329 -[2025-11-30 15:31:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e (Updated: 2025-07-30T07:22:14 [TS: 1753860134] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e (Updated: 2025-07-30T07:22:14) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bc77c010-ea51-4243-9e09-2b2ab5bc79ae] to complete... -.....done. -[2025-11-30 15:31:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf5a3e21a2fe08a2ec886d575c84e643473e76253c3189ae45b80954dabbab3e -[2025-11-30 15:31:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 (Updated: 2025-07-31T07:21:38 [TS: 1753946498] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 (Updated: 2025-07-31T07:21:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9579e56a-1e06-45ab-86bb-de534dfe3784] to complete... -.....done. -[2025-11-30 15:31:38] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:159d6452ce9060832d1173b75ee0aeaa027364fd533c561ebf2d21c89282a8a1 -[2025-11-30 15:31:38] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 (Updated: 2025-08-01T07:20:39 [TS: 1754032839] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:38] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 (Updated: 2025-08-01T07:20:39) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0c2dd2a7-6980-4238-9c4e-ae5d6a35e450] to complete... -.....done. -[2025-11-30 15:31:41] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:da0ce0ca6afea9e9f8f9e60d232a35fae255d0bd8f2a9e5dcd51a409cd3d3182 -[2025-11-30 15:31:41] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 (Updated: 2025-08-02T07:20:51 [TS: 1754119251] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:41] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 (Updated: 2025-08-02T07:20:51) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d82adf63-3c14-4780-abc7-bd6f9b56f23a] to complete... -.....done. -[2025-11-30 15:31:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c1198ed6e95db877bc4f5d01bf91fe5c37204a582d4ad7a3117693972f920a7 -[2025-11-30 15:31:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d (Updated: 2025-08-03T07:20:40 [TS: 1754205640] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d (Updated: 2025-08-03T07:20:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/08fc3f03-8a5e-4b84-9eca-91ebab75c6d7] to complete... -.....done. -[2025-11-30 15:31:48] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3c65d33c007eef7a11240b25c72dc8a01c97d97913770843c636c60c13d5162d -[2025-11-30 15:31:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 (Updated: 2025-08-04T07:21:29 [TS: 1754292089] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 (Updated: 2025-08-04T07:21:29) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7db1396c-60c1-4c5c-a9b9-3577fd4b6e6c] to complete... -.....done. -[2025-11-30 15:31:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:17365ea7a2c437f6434e80515577ab678e6c81014798e745d4089aa0fc7af687 -[2025-11-30 15:31:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 (Updated: 2025-08-05T07:20:11 [TS: 1754378411] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 (Updated: 2025-08-05T07:20:11) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/389b3e2e-8e63-4ddd-86ff-e383674cbaf5] to complete... -.....done. -[2025-11-30 15:31:55] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae3ebb1914f68483ded8d8ae3946a13ea4a47b715b405617ba49116ce8fb7b11 -[2025-11-30 15:31:55] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 (Updated: 2025-08-06T07:20:49 [TS: 1754464849] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:55] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 (Updated: 2025-08-06T07:20:49) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fa0cde11-ddff-4fe9-bbcd-267538765da7] to complete... -.....done. -[2025-11-30 15:31:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf2584586235de108e3bb633b2a69dc0ac79505aa1e8c1bbf248f53e45888269 -[2025-11-30 15:31:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 (Updated: 2025-08-07T07:21:48 [TS: 1754551308] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:31:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 (Updated: 2025-08-07T07:21:48) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6d02ffa5-5b49-4082-a5e0-d0581fcf7268] to complete... -.....done. -[2025-11-30 15:32:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b316bcee5d2e3820b074b1b17b5089483b91684f964855decf653f9748b61720 -[2025-11-30 15:32:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 (Updated: 2025-08-08T07:20:18 [TS: 1754637618] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 (Updated: 2025-08-08T07:20:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ce44895f-5e13-46fd-9725-f6d5cd9c89bf] to complete... -.....done. -[2025-11-30 15:32:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:97d08fd24f72f5ef238212244aac3b47c68ca38f13757f5263eb55b203cebe08 -[2025-11-30 15:32:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c (Updated: 2025-08-09T07:21:20 [TS: 1754724080] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c (Updated: 2025-08-09T07:21:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8b0f84e4-46d1-471a-be65-635ad0e8e5da] to complete... -.....done. -[2025-11-30 15:32:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5353986f7f61e09cb2bc26f7292bcedb95db8f66258a9dc8c8a502527f722a7c -[2025-11-30 15:32:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c (Updated: 2025-08-10T07:21:33 [TS: 1754810493] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c (Updated: 2025-08-10T07:21:33) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/24b1e3ce-7c1e-4802-ba5b-125e90b666d1] to complete... -.....done. -[2025-11-30 15:32:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7236715dd8b8bfbcf787b91271e8a97292d5f18caffb40c54fb917c0431f7d0c -[2025-11-30 15:32:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e (Updated: 2025-08-11T07:21:44 [TS: 1754896904] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e (Updated: 2025-08-11T07:21:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c9663d3b-71b5-45e5-9a11-0d5b6d922317] to complete... -.....done. -[2025-11-30 15:32:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:74f068701cb87ebfe9bf3bb9e693d810a17af0625b6b4e5b3ed38087051e477e -[2025-11-30 15:32:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd (Updated: 2025-08-12T07:21:45 [TS: 1754983305] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd (Updated: 2025-08-12T07:21:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2ced74b3-aec1-4f03-896b-00b28e03569a] to complete... -.....done. -[2025-11-30 15:32:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:493f4e7121441266ff48bcdbc16fd291933e237ed5412030c4f5a57469e0a9bd -[2025-11-30 15:32:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 (Updated: 2025-08-13T07:20:21 [TS: 1755069621] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 (Updated: 2025-08-13T07:20:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/360b70f4-793e-468f-9ca7-bbc6f45eb1a3] to complete... -.....done. -[2025-11-30 15:32:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c124fdfe8384631137823bd77123996e440b4245c8135f8e39eb0dc29b3ff14 -[2025-11-30 15:32:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 (Updated: 2025-08-14T07:21:25 [TS: 1755156085] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 (Updated: 2025-08-14T07:21:25) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1570def0-11ff-406b-8f6e-efb44a28d8be] to complete... -.....done. -[2025-11-30 15:32:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:15747282182d04656124be3b769cef910de825eac94239ab762de8701fef85b0 -[2025-11-30 15:32:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd (Updated: 2025-08-15T07:19:57 [TS: 1755242397] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd (Updated: 2025-08-15T07:19:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/29a50ecc-61d5-4d37-9476-fc586d97e5ae] to complete... -.....done. -[2025-11-30 15:32:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1fae201a0a12b8fa8a4144ab036e2a27c712a6ae1b7160b64982e8f08ceb9cd -[2025-11-30 15:32:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a (Updated: 2025-08-16T07:20:46 [TS: 1755328846] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a (Updated: 2025-08-16T07:20:46) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7a965f06-ea39-46ea-8347-9b5c1eb57fd2] to complete... -.....done. -[2025-11-30 15:32:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9180d01dfd9b0aaf038d6c07c945bbf629c432df164fc35d112b5884ab16797a -[2025-11-30 15:32:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 (Updated: 2025-08-17T07:21:09 [TS: 1755415269] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 (Updated: 2025-08-17T07:21:09) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d4b577a7-82ee-4bb0-b9e9-e9527200ac99] to complete... -......done. -[2025-11-30 15:32:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:dc0145d6185598f168f84e224b90dba2e238d5b43562820fac0cf110a0a043a6 -[2025-11-30 15:32:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 (Updated: 2025-08-18T07:21:01 [TS: 1755501661] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 (Updated: 2025-08-18T07:21:01) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6317acb1-3206-4377-9df7-46e41e8081b4] to complete... -.....done. -[2025-11-30 15:32:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f3234ba8a49c54d9ad0f1ac37c6792cfac504544612c41979ae0ecbe169e867 -[2025-11-30 15:32:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 (Updated: 2025-08-19T07:20:59 [TS: 1755588059] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 (Updated: 2025-08-19T07:20:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8994b351-13c9-4ad5-9151-af0e006ae554] to complete... -.....done. -[2025-11-30 15:32:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed157b15ddec8b0f5e5ee060bb32296c1097666bc3e718738ed2df41b56daea6 -[2025-11-30 15:32:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 (Updated: 2025-08-20T07:21:18 [TS: 1755674478] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 (Updated: 2025-08-20T07:21:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0cac5019-7308-4402-a8f8-e838744d8e7c] to complete... -.....done. -[2025-11-30 15:32:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b79607fbaf7304c8b7d7f5b2ab66df981846bb1ed819e1bf2234b9009007b2e5 -[2025-11-30 15:32:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 (Updated: 2025-08-21T07:21:02 [TS: 1755760862] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 (Updated: 2025-08-21T07:21:02) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/14c46048-89ae-436b-806c-23151d3aeca6] to complete... -.....done. -[2025-11-30 15:32:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6a5e23629300092954de862ee494b475e1f8e9e797fb16731caef656cc07fa46 -[2025-11-30 15:32:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 (Updated: 2025-08-22T07:20:17 [TS: 1755847217] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 (Updated: 2025-08-22T07:20:17) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6832d74a-d10c-4ed3-a29b-35ce7dcc5372] to complete... -.....done. -[2025-11-30 15:32:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a316a4f550884dac6a8f5d24f4a88aa5317bd267dce06e58af62c68edeb6c137 -[2025-11-30 15:32:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e (Updated: 2025-08-23T07:22:26 [TS: 1755933746] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e (Updated: 2025-08-23T07:22:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a3ef0c15-ea36-4aae-bfdc-2e4a670a82e9] to complete... -.....done. -[2025-11-30 15:32:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6fbe24ef470b1fb1d88734b3131eff83058870d8066a31ee28ce31e9dc7780e -[2025-11-30 15:32:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b (Updated: 2025-08-24T07:21:13 [TS: 1756020073] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:32:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b (Updated: 2025-08-24T07:21:13) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/33c3810a-8d07-43c4-980f-fcb0032c3296] to complete... -.....done. -[2025-11-30 15:33:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:03030f3841369391bcf15cd4314edd86ee66548becc35bb30a0f4d92f92f661b -[2025-11-30 15:33:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 (Updated: 2025-08-25T07:23:27 [TS: 1756106607] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 (Updated: 2025-08-25T07:23:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0d66e745-d8f4-4d00-a1ee-799f1cc0e42a] to complete... -.....done. -[2025-11-30 15:33:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7d1f4f21115901ed34033f2d121fc059618ac3d5d8c55c13e0014bb1e5011b25 -[2025-11-30 15:33:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 (Updated: 2025-08-26T07:21:52 [TS: 1756192912] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 (Updated: 2025-08-26T07:21:52) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/dab1b37a-647b-45eb-8ef9-981fa52bb02f] to complete... -.....done. -[2025-11-30 15:33:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c26627375ea8edb6bd5ec16123a1f77fc12df081780e87c778728e2dcf6f5e63 -[2025-11-30 15:33:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 (Updated: 2025-08-27T07:22:57 [TS: 1756279377] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 (Updated: 2025-08-27T07:22:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/284394f5-ea90-49e6-a514-f0b1482e0e64] to complete... -.....done. -[2025-11-30 15:33:11] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bf34866a5b81b6b2581155fe98a94b7b139003c928c72a327b337d7539132165 -[2025-11-30 15:33:11] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f (Updated: 2025-08-28T07:23:00 [TS: 1756365780] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:11] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f (Updated: 2025-08-28T07:23:00) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/467a04fc-cfcf-467b-bb3e-a486435137f8] to complete... -.....done. -[2025-11-30 15:33:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a7a9d6745dda23d8d9788f8a3940c02ada4d00b0b1a3babdf1fb0a1af224ab1f -[2025-11-30 15:33:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c (Updated: 2025-08-29T07:21:43 [TS: 1756452103] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c (Updated: 2025-08-29T07:21:43) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/113e4b72-8058-4d4c-9b56-7d23d9680cc8] to complete... -.....done. -[2025-11-30 15:33:18] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ae02ef573b801d6621ed5ce9a4c0d487e5e9ce8955b58ff35a86b702924e446c -[2025-11-30 15:33:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 (Updated: 2025-08-30T07:22:13 [TS: 1756538533] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 (Updated: 2025-08-30T07:22:13) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a16715e0-98ae-4a7d-b761-4388aa7b8d5f] to complete... -.....done. -[2025-11-30 15:33:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e1678005d522f8209a3be7d7d543e761897c93eac0f5ec79a365c33c253bcdf2 -[2025-11-30 15:33:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c (Updated: 2025-08-31T07:21:37 [TS: 1756624897] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c (Updated: 2025-08-31T07:21:37) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e96b6202-6eeb-459d-aaad-333304f8fdc9] to complete... -.....done. -[2025-11-30 15:33:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:753618afe7505108a238af17d6f8aee11bdf576206d16fa85f28e329919a8a5c -[2025-11-30 15:33:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e (Updated: 2025-09-01T07:22:45 [TS: 1756711365] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e (Updated: 2025-09-01T07:22:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0ab8537d-2963-4f4a-a5e5-a03b811a0697] to complete... -......done. -[2025-11-30 15:33:28] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e4d7fb0c18477712610fd59e8e068c0e73959f6401843224e33a46ee9e8ea11e -[2025-11-30 15:33:28] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd (Updated: 2025-09-02T07:20:23 [TS: 1756797623] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:28] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd (Updated: 2025-09-02T07:20:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8ad31bea-933e-4d49-a2be-b5566c8b2a95] to complete... -.....done. -[2025-11-30 15:33:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0d23d71ffd75865736fe0fd0abbf349698808253e32e61a990bb57ed131b86cd -[2025-11-30 15:33:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 (Updated: 2025-09-03T07:22:13 [TS: 1756884133] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 (Updated: 2025-09-03T07:22:13) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/34a407d5-8f28-4f54-aa4d-ea7952fd263f] to complete... -.....done. -[2025-11-30 15:33:35] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2cca3c01b1f1f35cf2be344090051eb4505cc9aa626c3d3ab5ad227028f844d5 -[2025-11-30 15:33:35] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 (Updated: 2025-09-04T07:19:51 [TS: 1756970391] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:35] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 (Updated: 2025-09-04T07:19:51) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/10f4099c-c3b2-4158-8b43-e2037db31a9a] to complete... -.....done. -[2025-11-30 15:33:39] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0568ff318d688f59c8393a5ee235e166df2183b5788b988ad09431e09441b599 -[2025-11-30 15:33:39] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc (Updated: 2025-09-05T07:20:48 [TS: 1757056848] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:39] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc (Updated: 2025-09-05T07:20:48) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8b213ef9-bcfe-437f-88cc-a5cc43ae7c93] to complete... -.....done. -[2025-11-30 15:33:42] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ff0570f792602d06d67c122eb731af0d5fbdc1371531b27a678ccefa6dcba8bc -[2025-11-30 15:33:42] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 (Updated: 2025-09-06T07:21:41 [TS: 1757143301] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:42] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 (Updated: 2025-09-06T07:21:41) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/40789e58-fcf5-4021-a523-63c2e22aa78c] to complete... -.....done. -[2025-11-30 15:33:46] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:374493772f1e7cda3cf2bfc6170c0515a8130ebb2824415b5757dcd79a263f56 -[2025-11-30 15:33:46] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 (Updated: 2025-09-07T07:22:23 [TS: 1757229743] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:46] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 (Updated: 2025-09-07T07:22:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9b860017-bb21-4b4d-bdcb-3d8d8bf72fb0] to complete... -.....done. -[2025-11-30 15:33:49] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35d574bd45012cb52a2ef4d326577923757310077b810897495c51f14dd0d936 -[2025-11-30 15:33:49] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 (Updated: 2025-09-08T07:22:09 [TS: 1757316129] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:49] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 (Updated: 2025-09-08T07:22:09) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b92b7c80-5fa0-40ad-9b3d-c8f8c848095e] to complete... -.....done. -[2025-11-30 15:33:53] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c859c776b55ddc7b4e83bf6efd1766dfb7cd44c78236cd49f9c38f7a883e2db8 -[2025-11-30 15:33:53] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 (Updated: 2025-09-09T07:22:21 [TS: 1757402541] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:53] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 (Updated: 2025-09-09T07:22:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c2072c55-61b9-4337-9f33-79fb0ffd2b4b] to complete... -......done. -[2025-11-30 15:33:56] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:be5b2b1d72351904ab30de3c166ffb425cb1ec229f8b965502b860f976fad330 -[2025-11-30 15:33:56] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 (Updated: 2025-09-10T07:22:45 [TS: 1757488965] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:33:56] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 (Updated: 2025-09-10T07:22:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e956c9ae-7b6b-4b04-b077-3fe3804e24a2] to complete... -......done. -[2025-11-30 15:34:00] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e58d08b4edf4454bd33b146bec7f588ba4e1fcc4b2a5465b79a519f17f971499 -[2025-11-30 15:34:00] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b (Updated: 2025-09-11T07:21:36 [TS: 1757575296] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:00] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b (Updated: 2025-09-11T07:21:36) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/de596c8f-d4f5-46b0-a49a-0ae0e296a61c] to complete... -.....done. -[2025-11-30 15:34:03] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:671e5199a8b526b45dda17f541af1d4ba2b03b68926e03135898bd6b4126670b -[2025-11-30 15:34:03] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 (Updated: 2025-09-12T07:21:47 [TS: 1757661707] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:03] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 (Updated: 2025-09-12T07:21:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a1e0675e-f7ad-4c10-8492-85238658a17b] to complete... -.....done. -[2025-11-30 15:34:07] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ce277d0ff224dcc5a00eabbe2abe91319429d1b2a78215314779dd4bea1298b4 -[2025-11-30 15:34:07] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df (Updated: 2025-09-13T07:22:23 [TS: 1757748143] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:07] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df (Updated: 2025-09-13T07:22:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cbd9d8ca-e848-490a-883a-886e79395928] to complete... -.....done. -[2025-11-30 15:34:10] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6d4dca82cb9dfa941bb662f95b49436172377599676738590448a07623c434df -[2025-11-30 15:34:10] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 (Updated: 2025-09-14T07:21:44 [TS: 1757834504] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:10] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 (Updated: 2025-09-14T07:21:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4480d01e-290f-4327-9347-2292b8d499f5] to complete... -.....done. -[2025-11-30 15:34:14] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5bb3492a90cd699e1ef3381851486dbb8ee48f9931154d059e20f7ebd65bd411 -[2025-11-30 15:34:14] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 (Updated: 2025-09-15T07:21:21 [TS: 1757920881] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:14] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 (Updated: 2025-09-15T07:21:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/021ae34b-955a-4726-b8f4-136adfa25c5c] to complete... -.....done. -[2025-11-30 15:34:17] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1681784e969a3f71a3a950c7979a231742821f7f9f6d63716111ef98218f1b75 -[2025-11-30 15:34:17] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa (Updated: 2025-09-16T07:18:40 [TS: 1758007120] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:17] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa (Updated: 2025-09-16T07:18:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8238eb6e-28f7-4182-8bc4-2818ed6773c8] to complete... -.....done. -[2025-11-30 15:34:20] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6101a89e81554cfc87750f15265fe704c3f32c51980471e95328905fc4442cfa -[2025-11-30 15:34:20] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 (Updated: 2025-09-17T07:22:13 [TS: 1758093733] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:20] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 (Updated: 2025-09-17T07:22:13) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ecb140c8-9b11-497d-aa3d-6a12f1a1ed1d] to complete... -.....done. -[2025-11-30 15:34:24] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:895b65e6a50b46891d16a1b1f179de6d159fec5dd3d16a1a75fc1eca1d1db563 -[2025-11-30 15:34:24] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 (Updated: 2025-09-18T07:22:40 [TS: 1758180160] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:24] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 (Updated: 2025-09-18T07:22:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8f85b98b-c9b4-437a-b89c-d046a71a2ae0] to complete... -.....done. -[2025-11-30 15:34:27] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9429b1e307c50faa0d7ef828754939e6a1e53c321e4797070f5ba28950d29e67 -[2025-11-30 15:34:27] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 (Updated: 2025-09-19T07:20:23 [TS: 1758266423] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:27] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 (Updated: 2025-09-19T07:20:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f582295f-dea6-4365-bb39-4ccd442a8e0f] to complete... -.....done. -[2025-11-30 15:34:31] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0e77ab354779970feddd24655a79d5a488d93dbc57c5c93b69ba4150b2d2db15 -[2025-11-30 15:34:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 (Updated: 2025-09-20T07:21:10 [TS: 1758352870] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:31] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 (Updated: 2025-09-20T07:21:10) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0d9bfebf-924e-4e57-9443-45575ed5cded] to complete... -.....done. -[2025-11-30 15:34:34] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:05a7abfe8018d4547b753a00192db306c2b33902a12f87f90a91991add449c84 -[2025-11-30 15:34:34] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 (Updated: 2025-09-21T07:22:13 [TS: 1758439333] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:34] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 (Updated: 2025-09-21T07:22:13) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2b8c84d1-8989-4a3a-9abe-c0df99d441dc] to complete... -.....done. -[2025-11-30 15:34:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9afa7304307069e87e78840dbf54c8733f82c98e0e9a9eb36e37f58654ad9e4 -[2025-11-30 15:34:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e (Updated: 2025-09-22T07:21:54 [TS: 1758525714] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e (Updated: 2025-09-22T07:21:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/24f2581b-b2c6-4328-991b-9f29b88bc76e] to complete... -.....done. -[2025-11-30 15:34:41] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a5b318347c50e0d08d276a3ee422766224dccbca941bc20fad211219923eeb9e -[2025-11-30 15:34:41] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 (Updated: 2025-09-23T07:23:02 [TS: 1758612182] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:41] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 (Updated: 2025-09-23T07:23:02) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9554246b-7236-4704-90f3-73d5db54fbf3] to complete... -......done. -[2025-11-30 15:34:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ef430cfd6bd3acf5477b56edd9654ae6b2ae0bb014b5e4a3a1a76cc0b145cfa4 -[2025-11-30 15:34:44] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 (Updated: 2025-09-23T07:23:05 [TS: 1758612185] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:44] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 (Updated: 2025-09-23T07:23:05) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/488522e0-3b0d-4771-8108-e4fd4b1cc685] to complete... -.....done. -[2025-11-30 15:34:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:3029c98b128bfe20af054385a73e1b45f45ebbad33229fb5f7f6d3cc15956150 -[2025-11-30 15:34:48] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 (Updated: 2025-09-24T07:20:14 [TS: 1758698414] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:48] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 (Updated: 2025-09-24T07:20:14) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8339acfb-a632-49be-aaf0-31ef24649f2a] to complete... -......done. -[2025-11-30 15:34:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:160f96ab188ccf06d513915864c26f4f0539402de577e4b8008ae34bf6bb4c35 -[2025-11-30 15:34:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 (Updated: 2025-09-24T07:20:17 [TS: 1758698417] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 (Updated: 2025-09-24T07:20:17) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c629ddbf-76ee-47db-bee2-4a75791abbeb] to complete... -.....done. -[2025-11-30 15:34:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d50f51ec0b8dc0cf7ee18d13932169cc2d643c8fe1bbb5c102b657e83c85e6f4 -[2025-11-30 15:34:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 (Updated: 2025-09-25T07:21:15 [TS: 1758784875] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 (Updated: 2025-09-25T07:21:15) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/41c2d152-bfba-4da9-9722-750ba0d6ea95] to complete... -.....done. -[2025-11-30 15:34:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9a6d2bbb91ff2c979f4cde66f866eb3af4e664c74d901dad1a8f1b4723e1fc6 -[2025-11-30 15:34:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae (Updated: 2025-09-25T07:21:18 [TS: 1758784878] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:34:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae (Updated: 2025-09-25T07:21:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/73551225-09c7-45e2-a837-51a3ec093851] to complete... -.....done. -[2025-11-30 15:35:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:131385c6647749593d07472f41212e085603655e84bdd62d2379d1b0e15b7fae -[2025-11-30 15:35:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 (Updated: 2025-10-09T07:22:47 [TS: 1759994567] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 (Updated: 2025-10-09T07:22:47) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b44f9112-ba3b-4268-9984-ed6b1ec615e8] to complete... -.....done. -[2025-11-30 15:35:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b4f77b51fe6df74a540190a8b33e967ed766a372b4cb6b251a7c574b2e65382 -[2025-11-30 15:35:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda (Updated: 2025-10-09T07:22:51 [TS: 1759994571] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda (Updated: 2025-10-09T07:22:51) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a76448cb-aba6-43eb-832b-3a21c9bcdb5c] to complete... -.....done. -[2025-11-30 15:35:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ed259e28e3c078f44fefecdc16afc0127d66bb3ef48fd903030006f6a023fbda -[2025-11-30 15:35:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef (Updated: 2025-10-10T07:21:41 [TS: 1760080901] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef (Updated: 2025-10-10T07:21:41) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1790af12-f417-4d99-9b88-d4ac2fd8941a] to complete... -.....done. -[2025-11-30 15:35:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1b2c4cfd3e1fa7ecd660389dd4f71fc45c5c5d455b579e483d45481d9807ccef -[2025-11-30 15:35:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c (Updated: 2025-10-10T07:21:44 [TS: 1760080904] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c (Updated: 2025-10-10T07:21:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/4a681845-39f9-4baa-a2df-300fdd6d0bf4] to complete... -.....done. -[2025-11-30 15:35:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01ad3c71f7a503050e73b8f69fb58bd041f7f836bccd7b1afadef09738bfd95c -[2025-11-30 15:35:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 (Updated: 2025-10-11T07:21:16 [TS: 1760167276] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 (Updated: 2025-10-11T07:21:16) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b61f91de-55d6-49a9-a167-e6048ec92cd5] to complete... -......done. -[2025-11-30 15:35:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8f895b1a84e8274d52106e3bb96ccda2835286f58084337336defc4c62b74645 -[2025-11-30 15:35:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e (Updated: 2025-10-11T07:21:19 [TS: 1760167279] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e (Updated: 2025-10-11T07:21:19) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/107d8573-fcef-4ad9-a1a7-9adb4fd461e0] to complete... -.....done. -[2025-11-30 15:35:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:44295e497e0b4a87a0692049afdbf0666d981e4f81793dd05857380e1cf9e37e -[2025-11-30 15:35:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 (Updated: 2025-10-12T07:20:46 [TS: 1760253646] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 (Updated: 2025-10-12T07:20:46) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b7a80d1a-ff03-45d4-8cb3-0769bf6b99f2] to complete... -.....done. -[2025-11-30 15:35:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d37389899ab1bdf99ae0c1fa3241d868d1e8bc9e6bdad62819dd240d299f7539 -[2025-11-30 15:35:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be (Updated: 2025-10-12T07:20:50 [TS: 1760253650] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be (Updated: 2025-10-12T07:20:50) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c188c146-8f54-44d5-9f3a-5511a0ab2abd] to complete... -.....done. -[2025-11-30 15:35:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:40b67de75558b708697026212eafd393882f72e2af467160bd233b4bf91e37be -[2025-11-30 15:35:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca (Updated: 2025-10-13T07:22:50 [TS: 1760340170] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca (Updated: 2025-10-13T07:22:50) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/11cb77c7-2861-40ff-86e9-dff1ac7ca81d] to complete... -.....done. -[2025-11-30 15:35:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c8817502b90f8fda3f874978007c7ea55d171124259a50ebe36752b6bfb0a8ca -[2025-11-30 15:35:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef (Updated: 2025-10-13T07:22:53 [TS: 1760340173] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef (Updated: 2025-10-13T07:22:53) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8d869620-afee-4902-9af3-2551965874f6] to complete... -.....done. -[2025-11-30 15:35:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:01e6f6b559c9297d8ba198243a027646a1b232bcd0bbcb4368f5261077c543ef -[2025-11-30 15:35:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 (Updated: 2025-10-14T07:21:23 [TS: 1760426483] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 (Updated: 2025-10-14T07:21:23) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/914bac45-2917-472f-93e1-ff5f262def91] to complete... -.....done. -[2025-11-30 15:35:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:668dad0d5b6a8201df8f160d4d734a95c3e8ad0bc12cfe97749f9c2c0a9fcb31 -[2025-11-30 15:35:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 (Updated: 2025-10-14T07:21:26 [TS: 1760426486] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 (Updated: 2025-10-14T07:21:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/43f5fd27-f421-412e-8a59-131396132b31] to complete... -.....done. -[2025-11-30 15:35:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a9ce49c77e6588702ca46a69a59cb4012d165f8cd81e60529172edc04c5c0cb3 -[2025-11-30 15:35:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 (Updated: 2025-10-15T07:21:13 [TS: 1760512873] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 (Updated: 2025-10-15T07:21:13) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a6521bfa-bb27-492b-9eb5-8456b5f4c774] to complete... -.....done. -[2025-11-30 15:35:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:efe9795c6aae555aea15514e3c0eb741e6a851cae103629d3d8786df579cf388 -[2025-11-30 15:35:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c (Updated: 2025-10-15T07:21:17 [TS: 1760512877] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c (Updated: 2025-10-15T07:21:17) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b47c2435-066f-44a6-aea4-55a1502f303c] to complete... -.....done. -[2025-11-30 15:35:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fabc48e5b462a9fa145b7b8be5903c4cf071de6b8a355ae316d9c77c58c6174c -[2025-11-30 15:35:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 (Updated: 2025-10-16T07:20:31 [TS: 1760599231] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 (Updated: 2025-10-16T07:20:31) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7830fe9c-6486-4fe2-9ec3-4253557283c0] to complete... -......done. -[2025-11-30 15:35:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9011ed54d56854fdd3f79828f73e89fec22e86e34a4883eb492f84184affeff1 -[2025-11-30 15:35:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 (Updated: 2025-10-16T07:20:37 [TS: 1760599237] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 (Updated: 2025-10-16T07:20:37) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/3f282dd8-c17a-4e37-98df-a33162f9d5bd] to complete... -......done. -[2025-11-30 15:35:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:a19568b25f5b8de8f40a1dc7b49a8ba69c03486c6dfc66040ad67cc3642eb747 -[2025-11-30 15:35:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be (Updated: 2025-10-17T07:22:22 [TS: 1760685742] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:35:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be (Updated: 2025-10-17T07:22:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/30e5ad30-e8bb-4a26-9dd6-afd1e0acc4b7] to complete... -......done. -[2025-11-30 15:36:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8501ea1980118aec24fb2decb20ef6f22691d5e7b09ee5eb61d4d90416ac96be -[2025-11-30 15:36:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 (Updated: 2025-10-17T07:22:28 [TS: 1760685748] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 (Updated: 2025-10-17T07:22:28) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b7d654f9-8e5f-42cb-9d1b-b0aa2ea2d611] to complete... -.....done. -[2025-11-30 15:36:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b72454668ec593961eebe6652872037974c16090ed589817e1e9ed9b5f68703 -[2025-11-30 15:36:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 (Updated: 2025-10-18T07:19:49 [TS: 1760771989] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 (Updated: 2025-10-18T07:19:49) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2fd81796-dbe4-4440-9f17-a0d3accd0acb] to complete... -.....done. -[2025-11-30 15:36:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8ef4b5bad78ee08835861cfd1b1b7de830bf8b5d3ca3e1dade35ea24306904f7 -[2025-11-30 15:36:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 (Updated: 2025-10-18T07:19:55 [TS: 1760771995] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 (Updated: 2025-10-18T07:19:55) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e06aa0db-0336-467c-834a-46b0a17e3302] to complete... -.....done. -[2025-11-30 15:36:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:916dfb3f9f90f78bf296045f981388a2710b044a94fac44c98c7dad0cb5562e1 -[2025-11-30 15:36:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 (Updated: 2025-10-19T07:20:50 [TS: 1760858450] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 (Updated: 2025-10-19T07:20:50) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/22088a1b-2ef3-4325-98d0-3f98e8c4fc93] to complete... -.....done. -[2025-11-30 15:36:16] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:81718eee73765633caa24b26632c27a78fe4e2e06fd755acf25705fb9edd0b78 -[2025-11-30 15:36:16] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 (Updated: 2025-10-19T07:20:57 [TS: 1760858457] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 (Updated: 2025-10-19T07:20:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/437fb55a-b67b-4a79-b6df-63decf026a6a] to complete... -.....done. -[2025-11-30 15:36:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d09441cf6af286ee49d04b63833b3defe3d962e99d8e9c437fa542332a08d545 -[2025-11-30 15:36:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a (Updated: 2025-10-20T07:22:34 [TS: 1760944954] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a (Updated: 2025-10-20T07:22:34) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/627712dc-e249-410b-81e6-0678437e64d7] to complete... -.....done. -[2025-11-30 15:36:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:75e17e3bfea3fe8230e3a5298ad658faa5f7f54b532230f7ae9ebde2de7f1b0a -[2025-11-30 15:36:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 (Updated: 2025-10-20T07:22:40 [TS: 1760944960] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 (Updated: 2025-10-20T07:22:40) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b55c9456-44eb-4777-b40f-701bc9142890] to complete... -.....done. -[2025-11-30 15:36:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d114b409b4a82a34c1dbdd4f5022acd6973ad1e1f10664a088fc2731641889e4 -[2025-11-30 15:36:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 (Updated: 2025-10-21T07:23:22 [TS: 1761031402] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 (Updated: 2025-10-21T07:23:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/020036f2-1648-44b0-bcf4-b4ebd02465d9] to complete... -.....done. -[2025-11-30 15:36:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4b8908a5c8b41d8e4ab0abfb6237d8ef1cfbe0059aefa6a99222ed641ee39159 -[2025-11-30 15:36:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 (Updated: 2025-10-21T07:23:25 [TS: 1761031405] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 (Updated: 2025-10-21T07:23:25) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a09c56a3-4402-4d13-b800-14bf968621bf] to complete... -.....done. -[2025-11-30 15:36:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0b6473deaca7c07e096dbb92302c69072c3df4bbbdeb0438e124c663c9b62307 -[2025-11-30 15:36:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 (Updated: 2025-10-22T07:21:05 [TS: 1761117665] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 (Updated: 2025-10-22T07:21:05) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1e96b748-9cc1-4b52-9770-5de09d0be1cf] to complete... -.....done. -[2025-11-30 15:36:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4d571d2c3d753b5f27e0d3dc5eab9f9a9cf08b7fefe3142ea6c7f81c2d3cefe2 -[2025-11-30 15:36:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 (Updated: 2025-10-22T07:21:08 [TS: 1761117668] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 (Updated: 2025-10-22T07:21:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5e8d9721-5872-4330-b819-2f8a11e01b98] to complete... -.....done. -[2025-11-30 15:36:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c0aed4972c2b245a408a1a28b1e0a39f8a98411be909d39f1ad9069ae82f859 -[2025-11-30 15:36:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 (Updated: 2025-10-23T07:21:12 [TS: 1761204072] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 (Updated: 2025-10-23T07:21:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7475419b-cccf-400d-a776-37a5ddc9742c] to complete... -.....done. -[2025-11-30 15:36:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5ca314ec2dfb73bd62320d97cb06beefb2e0b57d9a2023188b7a3d5cc0dd1d38 -[2025-11-30 15:36:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 (Updated: 2025-10-23T07:21:15 [TS: 1761204075] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 (Updated: 2025-10-23T07:21:15) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/80d05e30-872b-488e-b383-853368e1cf90] to complete... -.....done. -[2025-11-30 15:36:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d73882289b9c535633d17d8ee59024f5a2f355e2a06d7780b44c14db06c8d8c1 -[2025-11-30 15:36:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 (Updated: 2025-10-24T07:21:00 [TS: 1761290460] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 (Updated: 2025-10-24T07:21:00) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a51d3e66-2907-421f-80f1-65a74131908e] to complete... -.....done. -[2025-11-30 15:36:50] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:64ce4e5b26379597c76fb12be59dfc4ba697a6cea1b78d56d13c2971654f7e13 -[2025-11-30 15:36:50] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f (Updated: 2025-10-24T07:21:03 [TS: 1761290463] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:50] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f (Updated: 2025-10-24T07:21:03) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/8607ef22-a786-466a-8c21-cdbf06430939] to complete... -.....done. -[2025-11-30 15:36:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73ef1b3af65f7d3a0036b6dd2ab923dd526e025fab24908397fbb7c5749deb5f -[2025-11-30 15:36:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 (Updated: 2025-10-25T07:22:28 [TS: 1761376948] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 (Updated: 2025-10-25T07:22:28) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/df43520c-3876-43c7-8d49-71bb0ef36d69] to complete... -.....done. -[2025-11-30 15:36:57] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eabff057f6a31f23bdb3aa056d02a54f2bdcb99ef872580308783af0136a0966 -[2025-11-30 15:36:57] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 (Updated: 2025-10-25T07:22:32 [TS: 1761376952] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:36:57] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 (Updated: 2025-10-25T07:22:32) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7f02acad-2731-4cd3-ad0f-65a8c22f98e8] to complete... -......done. -[2025-11-30 15:37:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8dfcf18c7379137d6fc95b1bbb6dcabf6dc5c965a5e982be005c50f55cf77e20 -[2025-11-30 15:37:01] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 (Updated: 2025-10-26T07:17:54 [TS: 1761463074] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:01] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 (Updated: 2025-10-26T07:17:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/933c44e0-ea39-46d2-beab-4b2b7e988f3c] to complete... -.....done. -[2025-11-30 15:37:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f1565bdf0dd11a7989c21ce7a30ee56721726c54450c5e6f334e201f8849287 -[2025-11-30 15:37:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 (Updated: 2025-10-26T07:18:06 [TS: 1761463086] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 (Updated: 2025-10-26T07:18:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/46877bba-ca7b-4f9e-adff-3c0e3fb5bc32] to complete... -.....done. -[2025-11-30 15:37:08] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:43f460e0a096c17786e4417f1a36ed05c492e24e034d540aca9e4380a4996083 -[2025-11-30 15:37:08] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e (Updated: 2025-10-27T07:20:57 [TS: 1761549657] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:08] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e (Updated: 2025-10-27T07:20:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b91950e5-fc0a-4c3a-92d3-28b14aed5009] to complete... -.....done. -[2025-11-30 15:37:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:fcb3a4f2069c428f59e4553cebfd508bae26796e20ecb57a1b9478833e24a89e -[2025-11-30 15:37:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e (Updated: 2025-10-27T07:21:00 [TS: 1761549660] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e (Updated: 2025-10-27T07:21:00) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bc70b6de-6624-4017-9a97-cacad3ebb674] to complete... -.....done. -[2025-11-30 15:37:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9b9fe15a34ae1f43013eca5c206f9df0a6a20d6aea8673068762b1165d10c48e -[2025-11-30 15:37:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 (Updated: 2025-10-28T07:21:03 [TS: 1761636063] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:15] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 (Updated: 2025-10-28T07:21:03) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/771f2787-6a8d-401c-ab51-fa8faa47c5f5] to complete... -.....done. -[2025-11-30 15:37:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:9c5b5b482a1563cc40e3cf2a3fb98d738c4153c5dd1fa5d48ae7d88eaa7b3f44 -[2025-11-30 15:37:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c (Updated: 2025-10-28T07:21:09 [TS: 1761636069] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c (Updated: 2025-10-28T07:21:09) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/2a204351-0a43-4e4a-93ee-cfa5923db1bf] to complete... -.....done. -[2025-11-30 15:37:22] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d6fe69cdb4f6470ae991aab73d581a23e71105f43b3687337026f84d3216f45c -[2025-11-30 15:37:22] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a (Updated: 2025-10-29T07:21:55 [TS: 1761722515] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:22] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a (Updated: 2025-10-29T07:21:55) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/ced28ac0-eda7-451e-a0e9-02dc79e97c32] to complete... -.....done. -[2025-11-30 15:37:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d33b34ec63c9e3149bab505e0ed1b4104f18e027b96fec1896043e6f34a5ab8a -[2025-11-30 15:37:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 (Updated: 2025-10-29T07:22:01 [TS: 1761722521] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 (Updated: 2025-10-29T07:22:01) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/361023c0-dd12-4bc4-ba07-ff55f92c9d25] to complete... -.....done. -[2025-11-30 15:37:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5112de7257d96509c777ffcef3189e1df3ec4496749e6c800f9f8689bf66f568 -[2025-11-30 15:37:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 (Updated: 2025-10-30T07:21:11 [TS: 1761808871] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 (Updated: 2025-10-30T07:21:11) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/20a7fd0f-33d1-46da-b931-544c62bd11fa] to complete... -.....done. -[2025-11-30 15:37:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2d01e12ab84335b6bebddc35ebb36bb2d2dd9fa9e74a397615f1595203ef31c8 -[2025-11-30 15:37:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf (Updated: 2025-10-30T07:21:17 [TS: 1761808877] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf (Updated: 2025-10-30T07:21:17) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/70a44954-781d-49af-b475-5b4715260b3f] to complete... -.....done. -[2025-11-30 15:37:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6adb9008a0499b4f14ba194a032e5292a8d5956d002be7aaed3e14e5bb024dbf -[2025-11-30 15:37:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 (Updated: 2025-10-31T07:22:12 [TS: 1761895332] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 (Updated: 2025-10-31T07:22:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/586252f7-a883-4c13-808a-52b9c1537c6b] to complete... -.....done. -[2025-11-30 15:37:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:23cdb23364d86a6bad2dcf5b7a948a0341bca63f14f7b83d5d5ade2f2c3bed76 -[2025-11-30 15:37:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 (Updated: 2025-10-31T07:22:18 [TS: 1761895338] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 (Updated: 2025-10-31T07:22:18) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6fcb9c7b-2fdb-4d77-abdb-23b977d34225] to complete... -.....done. -[2025-11-30 15:37:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7128384be69b40213b794aa2384f60796a06c7ddc667066c8f6b44b905c8fbd5 -[2025-11-30 15:37:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa (Updated: 2025-11-01T07:22:50 [TS: 1761981770] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa (Updated: 2025-11-01T07:22:50) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/e114b7ab-034f-464c-833a-af8d5ba58fea] to complete... -......done. -[2025-11-30 15:37:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:988d1920bc464ed3c0a65f594c6753f569665f26128546260866735e776e00aa -[2025-11-30 15:37:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 (Updated: 2025-11-01T07:22:54 [TS: 1761981774] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 (Updated: 2025-11-01T07:22:54) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/14ff1dc4-9560-45aa-8194-fc430049705f] to complete... -.....done. -[2025-11-30 15:37:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:0084aa13e0828de21613c5d5a9d428cfb46dbe4bcbb28000a345714bb2daf2d1 -[2025-11-30 15:37:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 (Updated: 2025-11-02T07:21:16 [TS: 1762068076] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 (Updated: 2025-11-02T07:21:16) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7548581d-b8c6-41b2-8cf4-59be0475de00] to complete... -.....done. -[2025-11-30 15:37:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:67752a183877955e1efe25e1db71bc256b4d1beeacac2526492786819a314ea6 -[2025-11-30 15:37:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 (Updated: 2025-11-02T07:21:20 [TS: 1762068080] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 (Updated: 2025-11-02T07:21:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d1e0b52f-131b-4d3e-9918-05014f1f1ea7] to complete... -.....done. -[2025-11-30 15:37:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eaf71d5074c5c0bcbbf19b98728a034efcde0f7277777a280c63f4c27ee2f0d5 -[2025-11-30 15:37:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 (Updated: 2025-11-03T08:23:08 [TS: 1762158188] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:37:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 (Updated: 2025-11-03T08:23:08) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0046fdef-0839-4104-8f6b-fcd70567270f] to complete... -.....done. -[2025-11-30 15:38:01] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cd8edc47a491a2849948ea8d248f83f1cf16d893b10a631074495c8d188464f2 -[2025-11-30 15:38:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a (Updated: 2025-11-03T08:23:12 [TS: 1762158192] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a (Updated: 2025-11-03T08:23:12) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/dc38a112-2d49-4dcc-b535-492d5203d99a] to complete... -.....done. -[2025-11-30 15:38:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:aa6a3399223819dcd4f80000ba30f26cb3194d744d889a0db8f8620b9888a30a -[2025-11-30 15:38:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 (Updated: 2025-11-04T08:17:45 [TS: 1762244265] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 (Updated: 2025-11-04T08:17:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/82e158ab-89a5-4923-83c0-b85517487154] to complete... -.....done. -[2025-11-30 15:38:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:ec58d8b6d95f4fc651d6b7fc76231d01bb543c1c6b344b74bdc6b5ceda4fd633 -[2025-11-30 15:38:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 (Updated: 2025-11-04T08:17:48 [TS: 1762244268] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 (Updated: 2025-11-04T08:17:48) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/b0243605-3924-484a-8718-77c7798c7b6d] to complete... -......done. -[2025-11-30 15:38:12] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b704f8ffa12e93375edfa63d27c4ab4c34975f34f33fd5d171e348fb685204d5 -[2025-11-30 15:38:12] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 (Updated: 2025-11-05T08:20:13 [TS: 1762330813] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:12] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 (Updated: 2025-11-05T08:20:13) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/a9ccfb7d-1285-47c9-9e67-76c95e7e2411] to complete... -.....done. -[2025-11-30 15:38:15] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d7fe11ed77fd533e0a15c36785bc0b6ba024556c8e949331a7e59a6647a052e9 -[2025-11-30 15:38:15] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 (Updated: 2025-11-05T08:20:17 [TS: 1762330817] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:16] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 (Updated: 2025-11-05T08:20:17) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9385310f-c275-4271-a36f-34cf67b3b027] to complete... -.....done. -[2025-11-30 15:38:19] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1aae4ad155b7ea88300e742e13c83e8b1b46774da7a392e6d5ceaf17c1cd8191 -[2025-11-30 15:38:19] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a (Updated: 2025-11-06T08:20:55 [TS: 1762417255] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:19] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a (Updated: 2025-11-06T08:20:55) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/f7fbaf5a-f17a-4c83-8944-191641c21331] to complete... -......done. -[2025-11-30 15:38:23] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:d9498f5d27c79b9dd55b3f7312dd7101f13ed09390edbd4bfd547536b7ea3f1a -[2025-11-30 15:38:23] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 (Updated: 2025-11-06T08:20:58 [TS: 1762417258] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:23] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 (Updated: 2025-11-06T08:20:58) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/001b962b-2941-427a-99bb-44465efae926] to complete... -.....done. -[2025-11-30 15:38:26] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2ffbec52b573cde82b463fc32589c7b291fcf9bb4c388c3d6e8a9210afbf91f7 -[2025-11-30 15:38:26] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd (Updated: 2025-11-07T08:18:30 [TS: 1762503510] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:26] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd (Updated: 2025-11-07T08:18:30) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/fbf6a82f-fc89-428d-99ba-e14693841c4f] to complete... -......done. -[2025-11-30 15:38:30] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf645953062f98c097990c3bf45ac70a752ed9581937922df6226427660efbd -[2025-11-30 15:38:30] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a (Updated: 2025-11-07T08:18:33 [TS: 1762503513] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:30] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a (Updated: 2025-11-07T08:18:33) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/13102179-62f5-455e-b23a-00ecf0b80516] to complete... -.....done. -[2025-11-30 15:38:33] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6908593992753e16687a53d9136b542281cbc6ed37b0d49d3fa0317ff6c8ab1a -[2025-11-30 15:38:33] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af (Updated: 2025-11-08T08:18:22 [TS: 1762589902] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:33] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af (Updated: 2025-11-08T08:18:22) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/7b3344f4-d0f1-4217-b8bb-475a45bbc53c] to complete... -.....done. -[2025-11-30 15:38:37] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:38fd9deee40bf03b4fb089a6d2048cdee51a0da2d7f112e1af70ccf3d4f035af -[2025-11-30 15:38:37] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 (Updated: 2025-11-08T08:18:26 [TS: 1762589906] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:37] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 (Updated: 2025-11-08T08:18:26) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/1b1f81c0-34b4-4f2c-a58b-d8272761af3d] to complete... -.....done. -[2025-11-30 15:38:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c6097a0cf460c33342ae99fca8819b3585827fad41f9599c39543fb1d5102951 -[2025-11-30 15:38:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd (Updated: 2025-11-09T08:21:42 [TS: 1762676502] < Cutoff: [TS: 1763306816]) -[2025-11-30 15:38:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd (Updated: 2025-11-09T08:21:42) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/5b819612-8b72-4af6-a6c8-c00242f777ae] to complete... -.....done. -[2025-11-30 15:38:44] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:68918cdd62beb633ce16a65220dc850f75ff7139f79e53232cb0ee6caa26f4dd -[2025-11-30 15:38:44] [INFO] Hit delete limit (200) for Docker Images. -[2025-11-30 15:38:44] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 15:38:44] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 15:38:46] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 15:38:46] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 15:38:46] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 15:38:46] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 15:38:46] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 15:38:46] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 15:38:49] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 15:38:49] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:38:51] [INFO] No Regional Address found matching criteria. -[2025-11-30 15:38:51] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:38:53] [INFO] No Global Address found matching criteria. -[2025-11-30 15:38:53] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- -[2025-11-30 15:38:58] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 15:39:01] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:39:01] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:39:01] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 15:39:01] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 15:39:01] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:04] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 15:39:07] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:07] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 15:39:09] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 15:39:09] [INFO] CLEANUP RUN FINISHED -[2025-11-30 15:39:14] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 15:39:14] [INFO] Time Cutoff (General): 2025-11-30T15:39:14+0000 -[2025-11-30 15:39:14] [INFO] Time Cutoff (Images): 2025-10-01T15:39:14+0000 -[2025-11-30 15:39:14] [INFO] Delete Limit per Type: 200 -[2025-11-30 15:39:14] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 15:39:15] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 15:39:17] [INFO] No Service Accounts found matching prefix. -[2025-11-30 15:39:17] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:39:19] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 15:39:19] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 15:39:21] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:39:21] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:39:21] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 15:39:21] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:39:24] [INFO] No Filestore instances found matching criteria. -[2025-11-30 15:39:24] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 15:39:27] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 15:39:27] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 15:39:27] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 15:39:28] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 15:39:28] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 15:39:28] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 15:39:28] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 15:39:28] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 15:39:28] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 15:39:28] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 15:39:28] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 15:39:28] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:39:28Z (Unix: 1763307568) -[2025-11-30 15:39:28] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 (Updated: 2025-11-09T08:21:45 [TS: 1762676505] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 (Updated: 2025-11-09T08:21:45) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb (Updated: 2025-11-10T08:20:16 [TS: 1762762816] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb (Updated: 2025-11-10T08:20:16) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 (Updated: 2025-11-10T08:20:19 [TS: 1762762819] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 (Updated: 2025-11-10T08:20:19) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e (Updated: 2025-11-11T08:19:38 [TS: 1762849178] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e (Updated: 2025-11-11T08:19:38) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a (Updated: 2025-11-11T08:19:44 [TS: 1762849184] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a (Updated: 2025-11-11T08:19:44) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe (Updated: 2025-11-12T08:20:59 [TS: 1762935659] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe (Updated: 2025-11-12T08:20:59) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 (Updated: 2025-11-12T08:21:06 [TS: 1762935666] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 (Updated: 2025-11-12T08:21:06) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 (Updated: 2025-11-13T08:20:14 [TS: 1763022014] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 (Updated: 2025-11-13T08:20:14) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 (Updated: 2025-11-13T08:20:21 [TS: 1763022021] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 (Updated: 2025-11-13T08:20:21) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 (Updated: 2025-11-14T08:20:20 [TS: 1763108420] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 (Updated: 2025-11-14T08:20:20) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d (Updated: 2025-11-14T08:20:27 [TS: 1763108427] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d (Updated: 2025-11-14T08:20:27) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 (Updated: 2025-11-15T08:21:45 [TS: 1763194905] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 (Updated: 2025-11-15T08:21:45) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 (Updated: 2025-11-15T08:21:52 [TS: 1763194912] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 (Updated: 2025-11-15T08:21:52) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 (Updated: 2025-11-16T08:21:57 [TS: 1763281317] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 (Updated: 2025-11-16T08:21:57) -[2025-11-30 15:39:31] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae (Updated: 2025-11-16T08:22:05 [TS: 1763281325] < Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [DRY-RUN] Would delete Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae (Updated: 2025-11-16T08:22:05) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763307568]) -[2025-11-30 15:39:31] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 15:39:31] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 15:39:34] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 15:39:34] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 15:39:34] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 15:39:34] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 15:39:34] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 15:39:34] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 15:39:36] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 15:39:36] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:39:39] [INFO] No Regional Address found matching criteria. -[2025-11-30 15:39:39] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:39:41] [INFO] No Global Address found matching criteria. -[2025-11-30 15:39:41] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- -[2025-11-30 15:39:46] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 15:39:49] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:39:49] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:39:49] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 15:39:49] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 15:39:49] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:51] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 15:39:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:39:54] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 15:39:55] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 15:39:55] [INFO] CLEANUP RUN FINISHED -[2025-11-30 15:39:59] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 15:39:59] [INFO] Time Cutoff (General): 2025-11-30T15:39:59+0000 -[2025-11-30 15:39:59] [INFO] Time Cutoff (Images): 2025-10-01T15:39:59+0000 -[2025-11-30 15:39:59] [INFO] Delete Limit per Type: 200 -[2025-11-30 15:39:59] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 15:40:00] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 15:40:02] [INFO] No Service Accounts found matching prefix. -[2025-11-30 15:40:02] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:40:04] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 15:40:04] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 15:40:08] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:40:08] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:40:08] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 15:40:08] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 15:40:11] [INFO] No Filestore instances found matching criteria. -[2025-11-30 15:40:11] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 15:40:14] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 15:40:14] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 15:40:14] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 15:40:14] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 15:40:14] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 15:40:15] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 15:40:15] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 15:40:15] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 15:40:15] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 15:40:15] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 15:40:15] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 15:40:15] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T15:40:15Z (Unix: 1763307615) -[2025-11-30 15:40:15] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 15:40:18] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 (Updated: 2025-11-09T08:21:45 [TS: 1762676505] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:18] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 (Updated: 2025-11-09T08:21:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6faafbb1-7528-404f-90d3-f46225e56bf0] to complete... -.....done. -[2025-11-30 15:40:21] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b95cebd0f2515e523e5699b81b47cfc6c8359d76cc18675ed3700f5a9102157 -[2025-11-30 15:40:21] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb (Updated: 2025-11-10T08:20:16 [TS: 1762762816] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:21] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb (Updated: 2025-11-10T08:20:16) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/25eb1ae8-2d01-4e1a-bb6e-a9e19e437f93] to complete... -.....done. -[2025-11-30 15:40:25] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:54fecdba17bd31051c45fef1512fea29c7c5b7d7261c48b1cdc2d841d5bd9cbb -[2025-11-30 15:40:25] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 (Updated: 2025-11-10T08:20:19 [TS: 1762762819] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:25] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 (Updated: 2025-11-10T08:20:19) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6ad2fe8f-2df3-494b-b021-b159c6e6abae] to complete... -.....done. -[2025-11-30 15:40:29] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:cdf1fcdb7f22ca0ba996959d2a7dbd972bf9ad13bbfa584ec9ee6134f1997cc1 -[2025-11-30 15:40:29] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e (Updated: 2025-11-11T08:19:38 [TS: 1762849178] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:29] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e (Updated: 2025-11-11T08:19:38) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/bf01f721-a319-4e11-9287-fcca3bc7f36b] to complete... -.....done. -[2025-11-30 15:40:32] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:73a8e0e8d52b020e3206d6198a0ed3f79cc03006c3b69933858e91460619475e -[2025-11-30 15:40:32] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a (Updated: 2025-11-11T08:19:44 [TS: 1762849184] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:32] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a (Updated: 2025-11-11T08:19:44) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/cb532182-020d-4250-a0cc-35d6b848f5cc] to complete... -.....done. -[2025-11-30 15:40:36] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:7b779bf0e4dbbe5b4ef080d2d3d576a0d110dacf24a7a208e491398e645abd9a -[2025-11-30 15:40:36] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe (Updated: 2025-11-12T08:20:59 [TS: 1762935659] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:36] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe (Updated: 2025-11-12T08:20:59) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d2267cfc-f79e-4baa-ad94-3522995188fe] to complete... -......done. -[2025-11-30 15:40:40] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f362d49e1911c1378c7dc4de5f9a30f1f726247a33b0356c3bdad27c3d4aa8fe -[2025-11-30 15:40:40] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 (Updated: 2025-11-12T08:21:06 [TS: 1762935666] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:40] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 (Updated: 2025-11-12T08:21:06) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/9944c4ff-eb4f-442d-b56a-db74e63546e7] to complete... -.....done. -[2025-11-30 15:40:43] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8b6a88916c8ea926977066a057960b358623665a44c4b451553941784a685228 -[2025-11-30 15:40:43] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 (Updated: 2025-11-13T08:20:14 [TS: 1763022014] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:43] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 (Updated: 2025-11-13T08:20:14) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/99699167-777a-4f47-8315-b0e11d5af629] to complete... -......done. -[2025-11-30 15:40:47] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:638bdcbfd6f5a13508e9e48adcd61530fe2f2a90becbe302e54ffcf54bf03a12 -[2025-11-30 15:40:47] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 (Updated: 2025-11-13T08:20:21 [TS: 1763022021] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:47] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 (Updated: 2025-11-13T08:20:21) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/07a1793a-fde4-4285-a39e-517fb0ea81ab] to complete... -.....done. -[2025-11-30 15:40:51] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:31ac3d79a5d18c0798d6ffffd04f09159d9c88e1be24bfc9d0ecb6952e996062 -[2025-11-30 15:40:51] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 (Updated: 2025-11-14T08:20:20 [TS: 1763108420] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:51] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 (Updated: 2025-11-14T08:20:20) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/c2d3adb2-c23e-436b-b639-f212b0c9dc45] to complete... -......done. -[2025-11-30 15:40:54] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:202f4b04c2a7f562472a7a0435b001f8c8a78f9a2c5013ce2f2e27b43d7e0fb3 -[2025-11-30 15:40:54] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d (Updated: 2025-11-14T08:20:27 [TS: 1763108427] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:54] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d (Updated: 2025-11-14T08:20:27) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/455a2d68-2594-4371-b0ad-7fd6a67fb03f] to complete... -.....done. -[2025-11-30 15:40:58] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5df4d527ec15eac30144e3f806e42056b4ae14670e297e12c3daabfee130d25d -[2025-11-30 15:40:58] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 (Updated: 2025-11-15T08:21:45 [TS: 1763194905] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:40:58] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 (Updated: 2025-11-15T08:21:45) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d5d4fc28-dd0d-4dde-93b5-d897b729f785] to complete... -.....done. -[2025-11-30 15:41:02] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:48ba52c98be1697d4a8daf37328b1f8cd34e19aeb248e227943594dd723255e6 -[2025-11-30 15:41:02] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 (Updated: 2025-11-15T08:21:52 [TS: 1763194912] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:02] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 (Updated: 2025-11-15T08:21:52) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/d291f5b5-c122-4a88-96c1-e505c9e36400] to complete... -.....done. -[2025-11-30 15:41:05] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:862f11be51b0a4f141ab1be46fdefda66e7393ad4cb48e92f2684e22f51d39c4 -[2025-11-30 15:41:05] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 (Updated: 2025-11-16T08:21:57 [TS: 1763281317] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:05] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 (Updated: 2025-11-16T08:21:57) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/0c4ddb0c-6cff-4106-beef-ed8e183d6a15] to complete... -......done. -[2025-11-30 15:41:09] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f39f032d82aa2a748f3c41c80a14904239874964ce360ffb13e4aeae45711c81 -[2025-11-30 15:41:09] [INFO] [DELETE] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae (Updated: 2025-11-16T08:22:05 [TS: 1763281325] < Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:09] [EXECUTE] Deleting Docker Image Version: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae (Updated: 2025-11-16T08:22:05) -Digests: -- us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae -Delete request issued. -Waiting for operation [projects/hpc-toolkit-dev/locations/us-central1/operations/6820bcba-dc10-40e0-829c-f29db84577e0] to complete... -.....done. -[2025-11-30 15:41:13] [SUCCESS] Deleted us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:8e17137c239bce36cc3182d458145964f86265a40bccd9daa0705787bbcd82ae -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763307615]) -[2025-11-30 15:41:13] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 15:41:13] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 15:41:16] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 15:41:16] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 15:41:16] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 15:41:16] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 15:41:16] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 15:41:16] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 15:41:18] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 15:41:18] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:41:21] [INFO] No Regional Address found matching criteria. -[2025-11-30 15:41:21] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 15:41:23] [INFO] No Global Address found matching criteria. -[2025-11-30 15:41:23] [INFO] --- Processing: Service Networking Connections (Limit: 200) --- -[2025-11-30 15:41:27] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 15:41:30] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 15:41:30] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 15:41:30] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 15:41:30] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 15:41:30] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:33] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 15:41:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 15:41:35] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 15:41:37] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 15:41:37] [INFO] CLEANUP RUN FINISHED diff --git a/instances.txt b/instances.txt deleted file mode 100644 index 48a10b7c0c..0000000000 --- a/instances.txt +++ /dev/null @@ -1,2847 +0,0 @@ ---- Wed Nov 26 09:53:14 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 26 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -The following GKE clusters are targeted for deletion in this run: -h4d-res-swarnabm4-3 us-central1 -ml-gke-e2e-a8fae6 asia-southeast1 - gcloud container clusters delete "h4d-res-swarnabm4-3" --project="hpc-toolkit-dev" --location="us-central1" --quiet - gcloud container clusters delete "ml-gke-e2e-a8fae6" --project="hpc-toolkit-dev" --location="asia-southeast1" --quiet ---- Wed Nov 26 09:53:16 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 09:53:42 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 26 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -The following GKE clusters are targeted for deletion in this run: -h4d-res-swarnabm4-3 us-central1 -ml-gke-e2e-a8fae6 asia-southeast1 -[EXECUTE] GKE Cluster: Deleting h4d-res-swarnabm4-3 in us-central1 -Deleting cluster h4d-res-swarnabm4-3... -................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................done. -Deleted [https://container.googleapis.com/v1/projects/hpc-toolkit-dev/zones/us-central1/clusters/h4d-res-swarnabm4-3]. -[EXECUTE] GKE Cluster: Deleting ml-gke-e2e-a8fae6 in asia-southeast1 -Deleting cluster ml-gke-e2e-a8fae6... -................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................done. -Deleted [https://container.googleapis.com/v1/projects/hpc-toolkit-dev/zones/asia-southeast1/clusters/ml-gke-e2e-a8fae6]. ---- Wed Nov 26 10:02:05 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 10:03:24 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 26 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -a1c0c3topo-nodeset-2 us-central1-a -a3hca628-controller us-west1-a -a3hca628-login-001 us-west1-a -a3hcb15f-controller us-west1-a -a3hcb15f-login-001 us-west1-a -a3hcdf00-controller us-west1-a -a3hcdf00-login-001 us-west1-a -a3hnfsa628c1-ff9f3704-nfs-instance us-west1-a -a3hnfsb15fa8-57b77541-nfs-instance us-west1-a -a3hnfsdf0061-6327333f-nfs-instance us-west1-a -[DRY RUN] Instance: Would delete a1c0c3topo-nodeset-2 in us-central1-a - Command: gcloud compute instances delete "a1c0c3topo-nodeset-2" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet -[DRY RUN] Instance: Would delete a3hca628-controller in us-west1-a - Command: gcloud compute instances delete "a3hca628-controller" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet -[DRY RUN] Instance: Would delete a3hca628-login-001 in us-west1-a - Command: gcloud compute instances delete "a3hca628-login-001" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet -[DRY RUN] Instance: Would delete a3hcb15f-controller in us-west1-a - Command: gcloud compute instances delete "a3hcb15f-controller" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet -[DRY RUN] Instance: Would delete a3hcb15f-login-001 in us-west1-a - Command: gcloud compute instances delete "a3hcb15f-login-001" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet -[DRY RUN] Instance: Would delete a3hcdf00-controller in us-west1-a - Command: gcloud compute instances delete "a3hcdf00-controller" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet -[DRY RUN] Instance: Would delete a3hcdf00-login-001 in us-west1-a - Command: gcloud compute instances delete "a3hcdf00-login-001" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet -[DRY RUN] Instance: Would delete a3hnfsa628c1-ff9f3704-nfs-instance in us-west1-a - Command: gcloud compute instances delete "a3hnfsa628c1-ff9f3704-nfs-instance" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet -[DRY RUN] Instance: Would delete a3hnfsb15fa8-57b77541-nfs-instance in us-west1-a - Command: gcloud compute instances delete "a3hnfsb15fa8-57b77541-nfs-instance" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet -[DRY RUN] Instance: Would delete a3hnfsdf0061-6327333f-nfs-instance in us-west1-a - Command: gcloud compute instances delete "a3hnfsdf0061-6327333f-nfs-instance" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet ---- Wed Nov 26 10:03:28 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 10:03:57 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 26 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -a1c0c3topo-nodeset-2 us-central1-a -a3hca628-controller us-west1-a -a3hca628-login-001 us-west1-a -a3hcb15f-controller us-west1-a -a3hcb15f-login-001 us-west1-a -a3hcdf00-controller us-west1-a -a3hcdf00-login-001 us-west1-a -a3hnfsa628c1-ff9f3704-nfs-instance us-west1-a -a3hnfsb15fa8-57b77541-nfs-instance us-west1-a -a3hnfsdf0061-6327333f-nfs-instance us-west1-a -[EXECUTE] Instance: Deleting a1c0c3topo-nodeset-2 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/a1c0c3topo-nodeset-2]. -[EXECUTE] Instance: Deleting a3hca628-controller in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hca628-controller]. -[EXECUTE] Instance: Deleting a3hca628-login-001 in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hca628-login-001]. -[EXECUTE] Instance: Deleting a3hcb15f-controller in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hcb15f-controller]. -[EXECUTE] Instance: Deleting a3hcb15f-login-001 in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hcb15f-login-001]. -[EXECUTE] Instance: Deleting a3hcdf00-controller in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hcdf00-controller]. -[EXECUTE] Instance: Deleting a3hcdf00-login-001 in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hcdf00-login-001]. -[EXECUTE] Instance: Deleting a3hnfsa628c1-ff9f3704-nfs-instance in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hnfsa628c1-ff9f3704-nfs-instance]. -[EXECUTE] Instance: Deleting a3hnfsb15fa8-57b77541-nfs-instance in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hnfsb15fa8-57b77541-nfs-instance]. -[EXECUTE] Instance: Deleting a3hnfsdf0061-6327333f-nfs-instance in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/a3hnfsdf0061-6327333f-nfs-instance]. ---- Wed Nov 26 10:12:55 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 10:13:53 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 26 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -a3mega-controller us-west4-a -a3mega-login-001 us-west4-a -a4h639c-a4highnodeset-0 us-central1-b -a4h639c-a4highnodeset-1 us-central1-b -a4h639c-controller us-central1-b -a4h639c-slurm-login-001 us-central1-b -a7f7bcslur-controller us-central1-a -a7f7bcslur-slurm-login-001 us-central1-a -c379slurms-controller us-central1-a -ce64slurms-controller us-central1-a - gcloud compute instances delete "a3mega-controller" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet - gcloud compute instances delete "a3mega-login-001" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet - gcloud compute instances delete "a4h639c-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4h639c-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4h639c-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4h639c-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a7f7bcslur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "a7f7bcslur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "c379slurms-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "ce64slurms-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet ---- Wed Nov 26 10:13:57 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 10:14:29 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 26 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -a3mega-controller us-west4-a -a3mega-login-001 us-west4-a -a4h639c-a4highnodeset-0 us-central1-b -a4h639c-a4highnodeset-1 us-central1-b -a4h639c-controller us-central1-b -a4h639c-slurm-login-001 us-central1-b -a7f7bcslur-controller us-central1-a -a7f7bcslur-slurm-login-001 us-central1-a -c379slurms-controller us-central1-a -ce64slurms-controller us-central1-a -[EXECUTE] Instance: Deleting a3mega-controller in us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/instances/a3mega-controller]. -[EXECUTE] Instance: Deleting a3mega-login-001 in us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/instances/a3mega-login-001]. -[EXECUTE] Instance: Deleting a4h639c-a4highnodeset-0 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4h639c-a4highnodeset-0]. -[EXECUTE] Instance: Deleting a4h639c-a4highnodeset-1 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4h639c-a4highnodeset-1]. -[EXECUTE] Instance: Deleting a4h639c-controller in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4h639c-controller]. -[EXECUTE] Instance: Deleting a4h639c-slurm-login-001 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4h639c-slurm-login-001]. -[EXECUTE] Instance: Deleting a7f7bcslur-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/a7f7bcslur-controller]. -[EXECUTE] Instance: Deleting a7f7bcslur-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/a7f7bcslur-slurm-login-001]. -[EXECUTE] Instance: Deleting c379slurms-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/c379slurms-controller]. -[EXECUTE] Instance: Deleting ce64slurms-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/ce64slurms-controller]. ---- Wed Nov 26 10:28:03 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 10:28:21 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 26 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -ce64slurms-slurm-login-001 us-central1-a -cluster0vk-nodeset1-0 us-central1-c -cluster0vk-nodeset1-1 us-central1-c -cluster8ix-nodeset1-0 us-east4-b -cluster8ix-nodeset1-1 us-east4-b -clustermce-nodeset1-0 us-east5-a -clustermce-nodeset1-1 us-east5-a -clustermce-nodeset1-2 us-east5-a -clusterum7-nodeset1-1 us-central1-c -clusterum7-nodeset1-2 us-central1-c - gcloud compute instances delete "ce64slurms-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "clustermce-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clustermce-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet ---- Wed Nov 26 10:28:25 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 10:31:34 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 26 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -ce64slurms-slurm-login-001 us-central1-a -cluster0vk-nodeset1-0 us-central1-c -cluster0vk-nodeset1-1 us-central1-c -cluster8ix-nodeset1-0 us-east4-b -cluster8ix-nodeset1-1 us-east4-b -clustermce-nodeset1-0 us-east5-a -clustermce-nodeset1-2 us-east5-a -clusterum7-nodeset1-0 us-central1-c -clusterum7-nodeset1-1 us-central1-c -clusterum7-nodeset1-2 us-central1-c - gcloud compute instances delete "ce64slurms-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "clustermce-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clusterum7-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet ---- Wed Nov 26 10:31:38 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 10:31:55 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 26 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -ce64slurms-slurm-login-001 us-central1-a -cluster0vk-nodeset1-0 us-central1-c -cluster0vk-nodeset1-1 us-central1-c -cluster8ix-nodeset1-0 us-east4-b -cluster8ix-nodeset1-1 us-east4-b -clustermce-nodeset1-2 us-east5-a -clusterum7-nodeset1-0 us-central1-c -clusterum7-nodeset1-1 us-central1-c -clusterum7-nodeset1-2 us-central1-c -clusterum7-nodeset1-3 us-central1-c -[EXECUTE] Instance: Deleting ce64slurms-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/ce64slurms-slurm-login-001]. -[EXECUTE] Instance: Deleting cluster0vk-nodeset1-0 in us-central1-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-0]. -[EXECUTE] Instance: Deleting cluster0vk-nodeset1-1 in us-central1-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-1]. -[EXECUTE] Instance: Deleting cluster8ix-nodeset1-0 in us-east4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/instances/cluster8ix-nodeset1-0]. -[EXECUTE] Instance: Deleting cluster8ix-nodeset1-1 in us-east4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east4-b/instances/cluster8ix-nodeset1-1]. -[EXECUTE] Instance: Deleting clustermce-nodeset1-2 in us-east5-a -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-east5-a/instances/clustermce-nodeset1-2' was not found - ---- Wed Nov 26 10:43:27 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 28 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -cluster0vk-nodeset1-0 us-central1-c -cluster0vk-nodeset1-1 us-central1-c -cluster8ix-nodeset1-0 us-east4-b -cluster8ix-nodeset1-1 us-east4-b -clustermce-nodeset1-0 us-east5-a -clustermce-nodeset1-1 us-east5-a -clustermce-nodeset1-2 us-east5-a -clusterum7-nodeset1-0 us-central1-c -clusterum7-nodeset1-1 us-central1-c -clusterum7-nodeset1-2 us-central1-c - gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "clustermce-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clustermce-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clusterum7-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet ---- Wed Nov 26 10:43:32 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 10:44:42 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 28 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -cluster0vk-nodeset1-0 us-central1-c -cluster0vk-nodeset1-1 us-central1-c -cluster8ix-nodeset1-0 us-east4-b -cluster8ix-nodeset1-1 us-east4-b -clustermce-nodeset1-0 us-east5-a -clustermce-nodeset1-1 us-east5-a -clustermce-nodeset1-2 us-east5-a -clusterum7-nodeset1-0 us-central1-c -clusterum7-nodeset1-1 us-central1-c -clusterum7-nodeset1-2 us-central1-c - gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "clustermce-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clustermce-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clusterum7-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet ---- Wed Nov 26 10:44:47 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 10:45:01 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 28 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -cluster0vk-nodeset1-0 us-central1-c -cluster0vk-nodeset1-1 us-central1-c -cluster8ix-nodeset1-0 us-east4-b -cluster8ix-nodeset1-1 us-east4-b -clustermce-nodeset1-0 us-east5-a -clustermce-nodeset1-1 us-east5-a -clustermce-nodeset1-2 us-east5-a -clusterum7-nodeset1-0 us-central1-c -clusterum7-nodeset1-1 us-central1-c -clusterum7-nodeset1-2 us-central1-c -[EXECUTE] Instance: Deleting cluster0vk-nodeset1-0 in us-central1-c -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-0' was not found - ---- Wed Nov 26 10:47:32 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 28 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -cluster0vk-nodeset1-0 us-central1-c -cluster0vk-nodeset1-1 us-central1-c -cluster8ix-nodeset1-0 us-east4-b -cluster8ix-nodeset1-1 us-east4-b -clustermce-nodeset1-1 us-east5-a -clustermce-nodeset1-2 us-east5-a -clusterum7-nodeset1-0 us-central1-c -clusterum7-nodeset1-1 us-central1-c -clusterum7-nodeset1-2 us-central1-c -clusterum7-nodeset1-3 us-central1-c - gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "clustermce-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clusterum7-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-3" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet ---- Wed Nov 26 10:47:36 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 10:47:49 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 28 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -cluster0vk-nodeset1-0 us-central1-c -cluster0vk-nodeset1-1 us-central1-c -cluster8ix-nodeset1-0 us-east4-b -cluster8ix-nodeset1-1 us-east4-b -clustermce-nodeset1-1 us-east5-a -clustermce-nodeset1-2 us-east5-a -clusterum7-nodeset1-0 us-central1-c -clusterum7-nodeset1-1 us-central1-c -clusterum7-nodeset1-2 us-central1-c -clusterum7-nodeset1-3 us-central1-c -[EXECUTE] Instance: Deleting cluster0vk-nodeset1-0 in us-central1-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-0]. -[EXECUTE] Instance: Deleting cluster0vk-nodeset1-1 in us-central1-c -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-1' was not found - -[EXECUTE] Instance: Deleting cluster8ix-nodeset1-0 in us-east4-b -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-east4-b/instances/cluster8ix-nodeset1-0' was not found - -[EXECUTE] Instance: Deleting cluster8ix-nodeset1-1 in us-east4-b -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-east4-b/instances/cluster8ix-nodeset1-1' was not found - -[EXECUTE] Instance: Deleting clustermce-nodeset1-1 in us-east5-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east5-a/instances/clustermce-nodeset1-1]. -[EXECUTE] Instance: Deleting clustermce-nodeset1-2 in us-east5-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-east5-a/instances/clustermce-nodeset1-2]. -[EXECUTE] Instance: Deleting clusterum7-nodeset1-0 in us-central1-c -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-0' was not found - -[EXECUTE] Instance: Deleting clusterum7-nodeset1-1 in us-central1-c -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-1' was not found - -[EXECUTE] Instance: Deleting clusterum7-nodeset1-2 in us-central1-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-2]. -[EXECUTE] Instance: Deleting clusterum7-nodeset1-3 in us-central1-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-3]. ---- Wed Nov 26 10:58:13 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 10:58:47 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 28 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -cluster0vk-nodeset1-0 us-central1-c -cluster8ix-nodeset1-0 us-east4-b -cluster8ix-nodeset1-1 us-east4-b -clustermce-nodeset1-0 us-east5-a -clustermce-nodeset1-1 us-east5-a -clustermce-nodeset1-2 us-east5-a -clusterum7-nodeset1-0 us-central1-c -clusterum7-nodeset1-1 us-central1-c -clusterum7-nodeset1-2 us-central1-c -clusterum7-nodeset1-4 us-central1-c - gcloud compute instances delete "cluster0vk-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "cluster8ix-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east4-b" --quiet - gcloud compute instances delete "clustermce-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clustermce-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clustermce-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-east5-a" --quiet - gcloud compute instances delete "clusterum7-nodeset1-0" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-2" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-4" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet ---- Wed Nov 26 10:58:51 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 11:00:14 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 38 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - cluster0vk-nodeset1-0 - - cluster8ix-nodeset1-0 - - cluster8ix-nodeset1-1 - - clustermce-nodeset1-0 - - clustermce-nodeset1-1 - - clustermce-nodeset1-2 - - clusterum7-nodeset1-0 - - clusterum7-nodeset1-1 - - clusterum7-nodeset1-2 - - clusterum7-nodeset1-4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: cluster0vk-nodeset1-0 in us-central1-c (In exclusion list) -Skip Instance: cluster8ix-nodeset1-0 in us-east4-b (In exclusion list) -Skip Instance: clustermce-nodeset1-0 in us-east5-a (In exclusion list) -Skip Instance: clustermce-nodeset1-1 in us-east5-a (In exclusion list) -Skip Instance: clustermce-nodeset1-2 in us-east5-a (In exclusion list) -Skip Instance: clusterum7-nodeset1-0 in us-central1-c (In exclusion list) -Skip Instance: clusterum7-nodeset1-1 in us-central1-c (In exclusion list) -Skip Instance: clusterum7-nodeset1-2 in us-central1-c (In exclusion list) -The following Instances are targeted for deletion in this run: -cluster0vk-nodeset1-1 us-central1-c -clusterum7-nodeset1-5 us-central1-c -clusteurop-nodeset1-0 europe-west1-b -d3eslurmsi-controller us-central1-a -d3eslurmsi-slurm-login-001 us-central1-a -d3slurmsim-controller us-central1-a -d3slurmsim-slurm-login-001 us-central1-a -d72c8slurm-controller us-central1-a -d72c8slurm-slurm-login-001 us-central1-a -de3580slur-controller us-central1-a - gcloud compute instances delete "cluster0vk-nodeset1-1" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusterum7-nodeset1-5" --project="hpc-toolkit-dev" --zone="us-central1-c" --quiet - gcloud compute instances delete "clusteurop-nodeset1-0" --project="hpc-toolkit-dev" --zone="europe-west1-b" --quiet - gcloud compute instances delete "d3eslurmsi-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "d3eslurmsi-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "d3slurmsim-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "d3slurmsim-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "d72c8slurm-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "d72c8slurm-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "de3580slur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet ---- Wed Nov 26 11:00:18 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 11:00:45 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 38 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - cluster0vk-nodeset1-0 - - cluster8ix-nodeset1-0 - - cluster8ix-nodeset1-1 - - clustermce-nodeset1-0 - - clustermce-nodeset1-1 - - clustermce-nodeset1-2 - - clusterum7-nodeset1-0 - - clusterum7-nodeset1-1 - - clusterum7-nodeset1-2 - - clusterum7-nodeset1-4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: cluster0vk-nodeset1-0 in us-central1-c (In exclusion list) -Skip Instance: cluster8ix-nodeset1-0 in us-east4-b (In exclusion list) -Skip Instance: cluster8ix-nodeset1-1 in us-east4-b (In exclusion list) -Skip Instance: clustermce-nodeset1-0 in us-east5-a (In exclusion list) -Skip Instance: clustermce-nodeset1-1 in us-east5-a (In exclusion list) -Skip Instance: clustermce-nodeset1-2 in us-east5-a (In exclusion list) -Skip Instance: clusterum7-nodeset1-0 in us-central1-c (In exclusion list) -Skip Instance: clusterum7-nodeset1-1 in us-central1-c (In exclusion list) -Skip Instance: clusterum7-nodeset1-2 in us-central1-c (In exclusion list) -The following Instances are targeted for deletion in this run: -cluster0vk-nodeset1-1 us-central1-c -clusterum7-nodeset1-3 us-central1-c -clusterum7-nodeset1-5 us-central1-c -clusteurop-nodeset1-0 europe-west1-b -clusteurop-nodeset1-1 europe-west1-b -d3eslurmsi-controller us-central1-a -d3eslurmsi-slurm-login-001 us-central1-a -d3slurmsim-controller us-central1-a -d3slurmsim-slurm-login-001 us-central1-a -d72c8slurm-controller us-central1-a -[EXECUTE] Instance: Deleting cluster0vk-nodeset1-1 in us-central1-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/cluster0vk-nodeset1-1]. -[EXECUTE] Instance: Deleting clusterum7-nodeset1-3 in us-central1-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-3]. -[EXECUTE] Instance: Deleting clusterum7-nodeset1-5 in us-central1-c -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-central1-c/instances/clusterum7-nodeset1-5' was not found - -[EXECUTE] Instance: Deleting clusteurop-nodeset1-0 in europe-west1-b -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/europe-west1-b/instances/clusteurop-nodeset1-0' was not found - -[EXECUTE] Instance: Deleting clusteurop-nodeset1-1 in europe-west1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west1-b/instances/clusteurop-nodeset1-1]. -[EXECUTE] Instance: Deleting d3eslurmsi-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d3eslurmsi-controller]. -[EXECUTE] Instance: Deleting d3eslurmsi-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d3eslurmsi-slurm-login-001]. -[EXECUTE] Instance: Deleting d3slurmsim-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d3slurmsim-controller]. -[EXECUTE] Instance: Deleting d3slurmsim-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d3slurmsim-slurm-login-001]. -[EXECUTE] Instance: Deleting d72c8slurm-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d72c8slurm-controller]. ---- Wed Nov 26 11:12:55 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 11:27:49 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 38 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - cluster0vk-nodeset1-0 - - cluster8ix-nodeset1-0 - - cluster8ix-nodeset1-1 - - clustermce-nodeset1-0 - - clustermce-nodeset1-1 - - clustermce-nodeset1-2 - - clusterum7-nodeset1-0 - - clusterum7-nodeset1-1 - - clusterum7-nodeset1-2 - - clusterum7-nodeset1-4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -d72c8slurm-slurm-login-001 us-central1-a -de3580slur-controller us-central1-a -de3580slur-nodeset-0 us-central1-a -de3580slur-nodeset-1 us-central1-a -de3580slur-nodeset-2 us-central1-a -de3580slur-nodeset-3 us-central1-a -de3580slur-nodeset-4 us-central1-a -de3580slur-slurm-login-001 us-central1-a -dynpoc-controller us-central1-a -dynpoc-slurm-login-001 us-central1-a - gcloud compute instances delete "d72c8slurm-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "de3580slur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "de3580slur-nodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "de3580slur-nodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "de3580slur-nodeset-2" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "de3580slur-nodeset-3" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "de3580slur-nodeset-4" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "de3580slur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "dynpoc-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "dynpoc-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet ---- Wed Nov 26 11:27:53 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 11:28:14 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 38 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - cluster0vk-nodeset1-0 - - cluster8ix-nodeset1-0 - - cluster8ix-nodeset1-1 - - clustermce-nodeset1-0 - - clustermce-nodeset1-1 - - clustermce-nodeset1-2 - - clusterum7-nodeset1-0 - - clusterum7-nodeset1-1 - - clusterum7-nodeset1-2 - - clusterum7-nodeset1-4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip GKE Cluster: mglsard (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -d72c8slurm-slurm-login-001 us-central1-a -de3580slur-controller us-central1-a -de3580slur-nodeset-0 us-central1-a -de3580slur-nodeset-1 us-central1-a -de3580slur-nodeset-2 us-central1-a -de3580slur-nodeset-3 us-central1-a -de3580slur-nodeset-4 us-central1-a -de3580slur-slurm-login-001 us-central1-a -dynpoc-controller us-central1-a -dynpoc-slurm-login-001 us-central1-a -[EXECUTE] Instance: Deleting d72c8slurm-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d72c8slurm-slurm-login-001]. -[EXECUTE] Instance: Deleting de3580slur-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-controller]. -[EXECUTE] Instance: Deleting de3580slur-nodeset-0 in us-central1-a -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-nodeset-0' was not found - -[EXECUTE] Instance: Deleting de3580slur-nodeset-1 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-nodeset-1]. -[EXECUTE] Instance: Deleting de3580slur-nodeset-2 in us-central1-a -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-nodeset-2' was not found - -[EXECUTE] Instance: Deleting de3580slur-nodeset-3 in us-central1-a -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-nodeset-3' was not found - -[EXECUTE] Instance: Deleting de3580slur-nodeset-4 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-nodeset-4]. -[EXECUTE] Instance: Deleting de3580slur-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/de3580slur-slurm-login-001]. -[EXECUTE] Instance: Deleting dynpoc-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/dynpoc-controller]. -[EXECUTE] Instance: Deleting dynpoc-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/dynpoc-slurm-login-001]. ---- Wed Nov 26 11:38:44 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 11:44:26 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 38 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - cluster0vk-nodeset1-0 - - cluster8ix-nodeset1-0 - - cluster8ix-nodeset1-1 - - clustermce-nodeset1-0 - - clustermce-nodeset1-1 - - clustermce-nodeset1-2 - - clusterum7-nodeset1-0 - - clusterum7-nodeset1-1 - - clusterum7-nodeset1-2 - - clusterum7-nodeset1-4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -ebf828slur-controller us-central1-a -ebf828slur-slurm-login-001 us-central1-a -exascaler-cloud-4691-mds0 us-central1-a -exascaler-cloud-4691-mgs0 us-central1-a -exascaler-cloud-4691-oss0 us-central1-a -exascaler-cloud-4691-oss1 us-central1-a -exascaler-cloud-4691-oss2 us-central1-a -exascaler-cloud-4a36-mds0 europe-west4-c -exascaler-cloud-4a36-mgs0 europe-west4-c -exascaler-cloud-4a36-oss0 europe-west4-c - gcloud compute instances delete "ebf828slur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "ebf828slur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "exascaler-cloud-4691-mds0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "exascaler-cloud-4691-mgs0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "exascaler-cloud-4691-oss0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "exascaler-cloud-4691-oss1" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "exascaler-cloud-4691-oss2" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "exascaler-cloud-4a36-mds0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "exascaler-cloud-4a36-mgs0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "exascaler-cloud-4a36-oss0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet ---- Wed Nov 26 11:44:30 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 11:45:08 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 38 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - cluster0vk-nodeset1-0 - - cluster8ix-nodeset1-0 - - cluster8ix-nodeset1-1 - - clustermce-nodeset1-0 - - clustermce-nodeset1-1 - - clustermce-nodeset1-2 - - clusterum7-nodeset1-0 - - clusterum7-nodeset1-1 - - clusterum7-nodeset1-2 - - clusterum7-nodeset1-4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -ebf828slur-controller us-central1-a -ebf828slur-slurm-login-001 us-central1-a -exascaler-cloud-4691-mds0 us-central1-a -exascaler-cloud-4691-mgs0 us-central1-a -exascaler-cloud-4691-oss0 us-central1-a -exascaler-cloud-4691-oss1 us-central1-a -exascaler-cloud-4691-oss2 us-central1-a -exascaler-cloud-4a36-mds0 europe-west4-c -exascaler-cloud-4a36-mgs0 europe-west4-c -exascaler-cloud-4a36-oss0 europe-west4-c -[EXECUTE] Instance: Deleting ebf828slur-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/ebf828slur-controller]. -[EXECUTE] Instance: Deleting ebf828slur-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/ebf828slur-slurm-login-001]. -[EXECUTE] Instance: Deleting exascaler-cloud-4691-mds0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/exascaler-cloud-4691-mds0]. -[EXECUTE] Instance: Deleting exascaler-cloud-4691-mgs0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/exascaler-cloud-4691-mgs0]. -[EXECUTE] Instance: Deleting exascaler-cloud-4691-oss0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/exascaler-cloud-4691-oss0]. -[EXECUTE] Instance: Deleting exascaler-cloud-4691-oss1 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/exascaler-cloud-4691-oss1]. -[EXECUTE] Instance: Deleting exascaler-cloud-4691-oss2 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/exascaler-cloud-4691-oss2]. -[EXECUTE] Instance: Deleting exascaler-cloud-4a36-mds0 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-4a36-mds0]. -[EXECUTE] Instance: Deleting exascaler-cloud-4a36-mgs0 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-4a36-mgs0]. -[EXECUTE] Instance: Deleting exascaler-cloud-4a36-oss0 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-4a36-oss0]. ---- Wed Nov 26 11:50:39 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 11:51:04 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 38 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - cluster0vk-nodeset1-0 - - cluster8ix-nodeset1-0 - - cluster8ix-nodeset1-1 - - clustermce-nodeset1-0 - - clustermce-nodeset1-1 - - clustermce-nodeset1-2 - - clusterum7-nodeset1-0 - - clusterum7-nodeset1-1 - - clusterum7-nodeset1-2 - - clusterum7-nodeset1-4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -exascaler-cloud-4a36-oss1 europe-west4-c -exascaler-cloud-4a36-oss2 europe-west4-c -exascaler-cloud-a2a0-mds0 europe-west4-c -exascaler-cloud-a2a0-mgs0 europe-west4-c -exascaler-cloud-a2a0-oss0 europe-west4-c -exascaler-cloud-a2a0-oss1 europe-west4-c -exascaler-cloud-a2a0-oss2 europe-west4-c -f4e324slur-controller us-central1-a -f4e324slur-slurm-login-001 us-central1-a -f88073slur-controller us-central1-a - gcloud compute instances delete "exascaler-cloud-4a36-oss1" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "exascaler-cloud-4a36-oss2" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "exascaler-cloud-a2a0-mds0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "exascaler-cloud-a2a0-mgs0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "exascaler-cloud-a2a0-oss0" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "exascaler-cloud-a2a0-oss1" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "exascaler-cloud-a2a0-oss2" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "f4e324slur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "f4e324slur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "f88073slur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet ---- Wed Nov 26 11:51:08 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 11:51:22 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 38 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - cluster0vk-nodeset1-0 - - cluster8ix-nodeset1-0 - - cluster8ix-nodeset1-1 - - clustermce-nodeset1-0 - - clustermce-nodeset1-1 - - clustermce-nodeset1-2 - - clusterum7-nodeset1-0 - - clusterum7-nodeset1-1 - - clusterum7-nodeset1-2 - - clusterum7-nodeset1-4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -exascaler-cloud-4a36-oss1 europe-west4-c -exascaler-cloud-4a36-oss2 europe-west4-c -exascaler-cloud-a2a0-mds0 europe-west4-c -exascaler-cloud-a2a0-mgs0 europe-west4-c -exascaler-cloud-a2a0-oss0 europe-west4-c -exascaler-cloud-a2a0-oss1 europe-west4-c -exascaler-cloud-a2a0-oss2 europe-west4-c -f4e324slur-controller us-central1-a -f4e324slur-slurm-login-001 us-central1-a -f88073slur-controller us-central1-a -[EXECUTE] Instance: Deleting exascaler-cloud-4a36-oss1 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-4a36-oss1]. -[EXECUTE] Instance: Deleting exascaler-cloud-4a36-oss2 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-4a36-oss2]. -[EXECUTE] Instance: Deleting exascaler-cloud-a2a0-mds0 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-a2a0-mds0]. -[EXECUTE] Instance: Deleting exascaler-cloud-a2a0-mgs0 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-a2a0-mgs0]. -[EXECUTE] Instance: Deleting exascaler-cloud-a2a0-oss0 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-a2a0-oss0]. -[EXECUTE] Instance: Deleting exascaler-cloud-a2a0-oss1 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-a2a0-oss1]. -[EXECUTE] Instance: Deleting exascaler-cloud-a2a0-oss2 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/exascaler-cloud-a2a0-oss2]. -[EXECUTE] Instance: Deleting f4e324slur-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/f4e324slur-controller]. -[EXECUTE] Instance: Deleting f4e324slur-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/f4e324slur-slurm-login-001]. -[EXECUTE] Instance: Deleting f88073slur-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/f88073slur-controller]. ---- Wed Nov 26 11:57:54 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 11:59:13 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 28 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -f88073slur-slurm-login-001 us-central1-a -fa4slurmsi-controller us-central1-a -fa4slurmsi-slurm-login-001 us-central1-a -g4qclav-controller us-central1-b -g4qclav-g4nodeset-0 us-central1-b -g4qclav-slurm-login-001 us-central1-b -gke-1395b4-0 us-central1-a -hpcdy-controller europe-west4-c -hpcdydis-controller europe-west4-c -hpcdydis-slurm-login-001 europe-west4-c - gcloud compute instances delete "f88073slur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "fa4slurmsi-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "fa4slurmsi-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "g4qclav-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "g4qclav-g4nodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "g4qclav-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "gke-1395b4-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "hpcdy-controller" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "hpcdydis-controller" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "hpcdydis-slurm-login-001" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet ---- Wed Nov 26 11:59:18 AM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 12:00:36 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 28 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -f88073slur-slurm-login-001 us-central1-a -fa4slurmsi-controller us-central1-a -fa4slurmsi-slurm-login-001 us-central1-a -g4qclav-controller us-central1-b -g4qclav-g4nodeset-0 us-central1-b -g4qclav-slurm-login-001 us-central1-b -gke-1395b4-0 us-central1-a -hpcdy-controller europe-west4-c -hpcdydis-controller europe-west4-c -hpcdydis-slurm-login-001 europe-west4-c -[EXECUTE] Instance: Deleting f88073slur-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/f88073slur-slurm-login-001]. -[EXECUTE] Instance: Deleting fa4slurmsi-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/fa4slurmsi-controller]. -[EXECUTE] Instance: Deleting fa4slurmsi-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/fa4slurmsi-slurm-login-001]. -[EXECUTE] Instance: Deleting g4qclav-controller in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/g4qclav-controller]. -[EXECUTE] Instance: Deleting g4qclav-g4nodeset-0 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/g4qclav-g4nodeset-0]. -[EXECUTE] Instance: Deleting g4qclav-slurm-login-001 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/g4qclav-slurm-login-001]. -[EXECUTE] Instance: Deleting gke-1395b4-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/gke-1395b4-0]. -[EXECUTE] Instance: Deleting hpcdy-controller in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/hpcdy-controller]. -[EXECUTE] Instance: Deleting hpcdydis-controller in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/hpcdydis-controller]. -[EXECUTE] Instance: Deleting hpcdydis-slurm-login-001 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/hpcdydis-slurm-login-001]. ---- Wed Nov 26 12:17:01 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 12:50:16 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 28 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -The following Instances are targeted for deletion in this run: -a4he340-a4highnodeset-0 us-central1-b -a4he340-a4highnodeset-1 us-central1-b -a4he340-controller us-central1-b -a4he340-slurm-login-001 us-central1-b -hpcdy-slurm-login-001 europe-west4-c -hpcimg-controller us-central1-a -hpcimg-slurm-login-001 us-central1-a -image-inspector-550 us-west1-a -image-inspector us-west1-a -instance-20250625-210718 australia-southeast1-c - gcloud compute instances delete "a4he340-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4he340-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4he340-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4he340-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "hpcdy-slurm-login-001" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "hpcimg-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "hpcimg-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "image-inspector-550" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet - gcloud compute instances delete "image-inspector" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet - gcloud compute instances delete "instance-20250625-210718" --project="hpc-toolkit-dev" --zone="australia-southeast1-c" --quiet ---- Wed Nov 26 12:50:20 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 12:51:11 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -The following Instances are targeted for deletion in this run: -a4he340-a4highnodeset-0 us-central1-b -a4he340-a4highnodeset-1 us-central1-b -a4he340-controller us-central1-b -a4he340-slurm-login-001 us-central1-b -hpcdy-slurm-login-001 europe-west4-c -hpcimg-controller us-central1-a -hpcimg-slurm-login-001 us-central1-a -instance-20250625-210718 australia-southeast1-c -instance-20250918-073640 us-central1-b -instance-20251124-055307 us-central1-b - gcloud compute instances delete "a4he340-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4he340-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4he340-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4he340-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "hpcdy-slurm-login-001" --project="hpc-toolkit-dev" --zone="europe-west4-c" --quiet - gcloud compute instances delete "hpcimg-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "hpcimg-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "instance-20250625-210718" --project="hpc-toolkit-dev" --zone="australia-southeast1-c" --quiet - gcloud compute instances delete "instance-20250918-073640" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "instance-20251124-055307" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet ---- Wed Nov 26 12:51:15 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 12:51:45 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -The following Instances are targeted for deletion in this run: -a4he340-a4highnodeset-0 us-central1-b -a4he340-a4highnodeset-1 us-central1-b -a4he340-controller us-central1-b -a4he340-slurm-login-001 us-central1-b -hpcdy-slurm-login-001 europe-west4-c -hpcimg-controller us-central1-a -hpcimg-slurm-login-001 us-central1-a -instance-20250625-210718 australia-southeast1-c -instance-20250918-073640 us-central1-b -instance-20251124-055307 us-central1-b -[EXECUTE] Instance: Deleting a4he340-a4highnodeset-0 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4he340-a4highnodeset-0]. -[EXECUTE] Instance: Deleting a4he340-a4highnodeset-1 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4he340-a4highnodeset-1]. -[EXECUTE] Instance: Deleting a4he340-controller in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4he340-controller]. -[EXECUTE] Instance: Deleting a4he340-slurm-login-001 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4he340-slurm-login-001]. -[EXECUTE] Instance: Deleting hpcdy-slurm-login-001 in europe-west4-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-c/instances/hpcdy-slurm-login-001]. -[EXECUTE] Instance: Deleting hpcimg-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/hpcimg-controller]. -[EXECUTE] Instance: Deleting hpcimg-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/hpcimg-slurm-login-001]. -[EXECUTE] Instance: Deleting instance-20250625-210718 in australia-southeast1-c -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/australia-southeast1-c/instances/instance-20250625-210718]. -[EXECUTE] Instance: Deleting instance-20250918-073640 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/instance-20250918-073640]. -[EXECUTE] Instance: Deleting instance-20251124-055307 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/instance-20251124-055307]. ---- Wed Nov 26 01:04:17 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:05:06 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -The following Instances are targeted for deletion in this run: -khu-h4d-cluster-test-0 us-central1-a -khu-h4d-cluster-test-1 us-central1-a -khushi-ansible-deb11-0 us-central1-a -khushi-ansible-deb12-0 us-central1-a -khushi-ansible-failed-deb11-0 us-central1-a -khushi-ansible-failed-deb12-0 us-central1-a -khushi-ansible-failed-rhel8-0 us-central1-a -khushi-ansible-failed-rhel9-0 us-central1-a -khushi-ansible-failed-rocky8-0 us-central1-a -khushi-ansible-failed-rocky9-0 us-central1-a - gcloud compute instances delete "khu-h4d-cluster-test-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khu-h4d-cluster-test-1" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-deb11-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-deb12-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-failed-deb11-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-failed-deb12-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-failed-rhel8-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-failed-rhel9-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-failed-rocky8-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-failed-rocky9-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet ---- Wed Nov 26 01:05:11 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:05:28 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -The following Instances are targeted for deletion in this run: -khu-h4d-cluster-test-0 us-central1-a -khu-h4d-cluster-test-1 us-central1-a -khushi-ansible-deb11-0 us-central1-a -khushi-ansible-deb12-0 us-central1-a -khushi-ansible-failed-deb11-0 us-central1-a -khushi-ansible-failed-deb12-0 us-central1-a -khushi-ansible-failed-rhel8-0 us-central1-a -khushi-ansible-failed-rhel9-0 us-central1-a -khushi-ansible-failed-rocky8-0 us-central1-a -khushi-ansible-failed-rocky9-0 us-central1-a -[EXECUTE] Instance: Deleting khu-h4d-cluster-test-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-h4d-cluster-test-0]. -[EXECUTE] Instance: Deleting khu-h4d-cluster-test-1 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-h4d-cluster-test-1]. -[EXECUTE] Instance: Deleting khushi-ansible-deb11-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-deb11-0]. -[EXECUTE] Instance: Deleting khushi-ansible-deb12-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-deb12-0]. -[EXECUTE] Instance: Deleting khushi-ansible-failed-deb11-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-deb11-0]. -[EXECUTE] Instance: Deleting khushi-ansible-failed-deb12-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-deb12-0]. -[EXECUTE] Instance: Deleting khushi-ansible-failed-rhel8-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-rhel8-0]. -[EXECUTE] Instance: Deleting khushi-ansible-failed-rhel9-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-rhel9-0]. -[EXECUTE] Instance: Deleting khushi-ansible-failed-rocky8-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-rocky8-0]. -[EXECUTE] Instance: Deleting khushi-ansible-failed-rocky9-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-rocky9-0]. ---- Wed Nov 26 01:16:48 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:17:01 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -The following Instances are targeted for deletion in this run: -khushi-ansible-failed-ubuntu2204-0 us-central1-a -khushi-ansible-failed-ubuntu2404-0 us-central1-a -khushi-ansible-failed-ubuntu2404arm-0 us-central1-a -khushi-ansible-rhel8-0 us-central1-a -khushi-ansible-rhel9-0 us-central1-a -khushi-ansible-rocky8-0 us-central1-a -khushi-ansible-rocky9-0 us-central1-a -khushi-ansible-ubuntu2204-0 us-central1-a -khushi-ansible-ubuntu2404-0 us-central1-a -khushi-ansible-ubuntu2404arm-0 us-central1-a -khu-test-ansible-sleep-ubuntu2204-0 us-central1-a -khu-test-ansible-sleep-ubuntu2404-0 us-central1-a -khu-test-ansible-sleep-ubuntu2404arm-0 us-central1-a -khu-test-ansible-ubuntu2204-0 us-central1-a -khu-test-ansible-ubuntu2404-0 us-central1-a -khu-test-ansible-ubuntu2404arm-0 us-central1-a -ml-gke-e2e-a8fae6-0 asia-southeast1-b -my-a3-spot-vm us-central1-a -packer-119ae1 us-west1-a -packer-2c0c79 asia-southeast1-b -packer-3e3b45 us-west1-a -packer-59a068 us-west1-a -packer-5c41cc us-south1-b -packer-84a235 europe-west4-b -packer-90b969 us-west4-a -packer-a947ca us-west1-a -temp-image-check us-west1-a -test us-central1-a -testvm01 us-central1-a -testvm727 us-central1-b - gcloud compute instances delete "khushi-ansible-failed-ubuntu2204-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-failed-ubuntu2404-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-failed-ubuntu2404arm-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-rhel8-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-rhel9-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-rocky8-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-rocky9-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-ubuntu2204-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-ubuntu2404-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khushi-ansible-ubuntu2404arm-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khu-test-ansible-sleep-ubuntu2204-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khu-test-ansible-sleep-ubuntu2404-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khu-test-ansible-sleep-ubuntu2404arm-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khu-test-ansible-ubuntu2204-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khu-test-ansible-ubuntu2404-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "khu-test-ansible-ubuntu2404arm-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "ml-gke-e2e-a8fae6-0" --project="hpc-toolkit-dev" --zone="asia-southeast1-b" --quiet - gcloud compute instances delete "my-a3-spot-vm" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "packer-119ae1" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet - gcloud compute instances delete "packer-2c0c79" --project="hpc-toolkit-dev" --zone="asia-southeast1-b" --quiet - gcloud compute instances delete "packer-3e3b45" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet - gcloud compute instances delete "packer-59a068" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet - gcloud compute instances delete "packer-5c41cc" --project="hpc-toolkit-dev" --zone="us-south1-b" --quiet - gcloud compute instances delete "packer-84a235" --project="hpc-toolkit-dev" --zone="europe-west4-b" --quiet - gcloud compute instances delete "packer-90b969" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet - gcloud compute instances delete "packer-a947ca" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet - gcloud compute instances delete "temp-image-check" --project="hpc-toolkit-dev" --zone="us-west1-a" --quiet - gcloud compute instances delete "test" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "testvm01" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "testvm727" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet ---- Wed Nov 26 01:17:06 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:17:30 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -The following Instances are targeted for deletion in this run: -khushi-ansible-failed-ubuntu2204-0 us-central1-a -khushi-ansible-failed-ubuntu2404-0 us-central1-a -khushi-ansible-failed-ubuntu2404arm-0 us-central1-a -khushi-ansible-rhel8-0 us-central1-a -khushi-ansible-rhel9-0 us-central1-a -khushi-ansible-rocky8-0 us-central1-a -khushi-ansible-rocky9-0 us-central1-a -khushi-ansible-ubuntu2204-0 us-central1-a -khushi-ansible-ubuntu2404-0 us-central1-a -khushi-ansible-ubuntu2404arm-0 us-central1-a -khu-test-ansible-sleep-ubuntu2204-0 us-central1-a -khu-test-ansible-sleep-ubuntu2404-0 us-central1-a -khu-test-ansible-sleep-ubuntu2404arm-0 us-central1-a -khu-test-ansible-ubuntu2204-0 us-central1-a -khu-test-ansible-ubuntu2404-0 us-central1-a -khu-test-ansible-ubuntu2404arm-0 us-central1-a -ml-gke-e2e-a8fae6-0 asia-southeast1-b -my-a3-spot-vm us-central1-a -packer-119ae1 us-west1-a -packer-2c0c79 asia-southeast1-b -packer-3e3b45 us-west1-a -packer-59a068 us-west1-a -packer-5c41cc us-south1-b -packer-84a235 europe-west4-b -packer-90b969 us-west4-a -packer-a947ca us-west1-a -temp-image-check us-west1-a -test us-central1-a -testvm01 us-central1-a -testvm727 us-central1-b -[EXECUTE] Instance: Deleting khushi-ansible-failed-ubuntu2204-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-ubuntu2204-0]. -[EXECUTE] Instance: Deleting khushi-ansible-failed-ubuntu2404-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-ubuntu2404-0]. -[EXECUTE] Instance: Deleting khushi-ansible-failed-ubuntu2404arm-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-failed-ubuntu2404arm-0]. -[EXECUTE] Instance: Deleting khushi-ansible-rhel8-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-rhel8-0]. -[EXECUTE] Instance: Deleting khushi-ansible-rhel9-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-rhel9-0]. -[EXECUTE] Instance: Deleting khushi-ansible-rocky8-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-rocky8-0]. -[EXECUTE] Instance: Deleting khushi-ansible-rocky9-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-rocky9-0]. -[EXECUTE] Instance: Deleting khushi-ansible-ubuntu2204-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-ubuntu2204-0]. -[EXECUTE] Instance: Deleting khushi-ansible-ubuntu2404-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-ubuntu2404-0]. -[EXECUTE] Instance: Deleting khushi-ansible-ubuntu2404arm-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khushi-ansible-ubuntu2404arm-0]. -[EXECUTE] Instance: Deleting khu-test-ansible-sleep-ubuntu2204-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-sleep-ubuntu2204-0]. -[EXECUTE] Instance: Deleting khu-test-ansible-sleep-ubuntu2404-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-sleep-ubuntu2404-0]. -[EXECUTE] Instance: Deleting khu-test-ansible-sleep-ubuntu2404arm-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-sleep-ubuntu2404arm-0]. -[EXECUTE] Instance: Deleting khu-test-ansible-ubuntu2204-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-ubuntu2204-0]. -[EXECUTE] Instance: Deleting khu-test-ansible-ubuntu2404-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-ubuntu2404-0]. -[EXECUTE] Instance: Deleting khu-test-ansible-ubuntu2404arm-0 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/khu-test-ansible-ubuntu2404arm-0]. -[EXECUTE] Instance: Deleting ml-gke-e2e-a8fae6-0 in asia-southeast1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b/instances/ml-gke-e2e-a8fae6-0]. -[EXECUTE] Instance: Deleting my-a3-spot-vm in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/my-a3-spot-vm]. -[EXECUTE] Instance: Deleting packer-119ae1 in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/packer-119ae1]. -[EXECUTE] Instance: Deleting packer-2c0c79 in asia-southeast1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/asia-southeast1-b/instances/packer-2c0c79]. -[EXECUTE] Instance: Deleting packer-3e3b45 in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/packer-3e3b45]. -[EXECUTE] Instance: Deleting packer-59a068 in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/packer-59a068]. -[EXECUTE] Instance: Deleting packer-5c41cc in us-south1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-south1-b/instances/packer-5c41cc]. -[EXECUTE] Instance: Deleting packer-84a235 in europe-west4-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/europe-west4-b/instances/packer-84a235]. -[EXECUTE] Instance: Deleting packer-90b969 in us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/instances/packer-90b969]. -[EXECUTE] Instance: Deleting packer-a947ca in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/packer-a947ca]. -[EXECUTE] Instance: Deleting temp-image-check in us-west1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a/instances/temp-image-check]. -[EXECUTE] Instance: Deleting test in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/test]. -[EXECUTE] Instance: Deleting testvm01 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/testvm01]. -[EXECUTE] Instance: Deleting testvm727 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/testvm727]. ---- Wed Nov 26 01:41:23 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:41:38 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -today24 us-central1-b - gcloud compute instances delete "today24" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet ---- Wed Nov 26 01:41:42 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:41:56 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -today24 us-central1-b -[EXECUTE] Instance: Deleting today24 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/today24]. ---- Wed Nov 26 01:42:48 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Thu Nov 27 03:21:06 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Targeting resources created before: 2025-11-26T23:21:06+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -./cleanup.sh: line 72: ---: command not found ---- Deletion Phase 1: GKE Clusters (Top 20) --- -The following GKE clusters are targeted for deletion in this run: -gke-a3-nccl-test us-west4 - gcloud container clusters delete "gke-a3-nccl-test" --project="hpc-toolkit-dev" --location="us-west4" --quiet ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -a4hc0e2-a4highnodeset-0 us-central1-b -a4hc0e2-a4highnodeset-1 us-central1-b -a4hc0e2-controller us-central1-b -a4hc0e2-slurm-login-001 us-central1-b -gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk us-west4-a -gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 us-west4-a -gke-gke-a3-nccl-test-system-17f71453-fns8 us-west4-c - gcloud compute instances delete "a4hc0e2-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4hc0e2-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4hc0e2-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4hc0e2-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet - gcloud compute instances delete "gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet - gcloud compute instances delete "gke-gke-a3-nccl-test-system-17f71453-fns8" --project="hpc-toolkit-dev" --zone="us-west4-c" --quiet ---- Thu Nov 27 03:21:10 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 03:27:17 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Targeting resources created before: 2025-11-26T23:27:17+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -./cleanup.sh: line 77: ---: command not found ---- Deletion Phase 1: GKE Clusters (Top 20) --- -./cleanup.sh: line 50: resource_name: unbound variable -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -./cleanup.sh: line 50: resource_name: unbound variable -No Instances found to delete in this run. ---- Thu Nov 27 03:27:22 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 03:27:52 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Targeting resources created before: 2025-11-26T23:27:52+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -./cleanup.sh: line 77: ---: command not found ---- Deletion Phase 1: GKE Clusters (Top 20) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -a4hc0e2-a4highnodeset-0 us-central1-b -a4hc0e2-a4highnodeset-1 us-central1-b -a4hc0e2-controller us-central1-b -a4hc0e2-slurm-login-001 us-central1-b - gcloud compute instances delete "a4hc0e2-a4highnodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4hc0e2-a4highnodeset-1" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4hc0e2-controller" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet - gcloud compute instances delete "a4hc0e2-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-b" --quiet ---- Thu Nov 27 03:27:57 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 03:28:56 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 20 resources of each type per run. -Targeting resources created before: 2025-11-26T23:28:56+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -./cleanup.sh: line 77: ---: command not found ---- Deletion Phase 1: GKE Clusters (Top 20) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -a4hc0e2-a4highnodeset-0 us-central1-b -a4hc0e2-a4highnodeset-1 us-central1-b -a4hc0e2-controller us-central1-b -a4hc0e2-slurm-login-001 us-central1-b -[EXECUTE] Instance: Deleting a4hc0e2-a4highnodeset-0 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc0e2-a4highnodeset-0]. -[EXECUTE] Instance: Deleting a4hc0e2-a4highnodeset-1 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc0e2-a4highnodeset-1]. -[EXECUTE] Instance: Deleting a4hc0e2-controller in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc0e2-controller]. -[EXECUTE] Instance: Deleting a4hc0e2-slurm-login-001 in us-central1-b -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-b/instances/a4hc0e2-slurm-login-001]. ---- Thu Nov 27 03:35:47 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 03:36:13 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Targeting resources created before: 2025-11-26T23:36:13+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -./cleanup.sh: line 77: ---: command not found ---- Deletion Phase 1: GKE Clusters (Top 20) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -The following Filestore instances are targeted for deletion in this run: -a4h-slurm-c0e262-f5260d85 -[DRY RUN] Filestore Instance: Would delete a4h-slurm-c0e262-f5260d85 in - Command: gcloud filestore instances delete "a4h-slurm-c0e262-f5260d85" --project="hpc-toolkit-dev" --location="" --quiet --force ---- Thu Nov 27 03:36:19 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 03:36:45 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 20 resources of each type per run. -Targeting resources created before: 2025-11-26T23:36:45+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -./cleanup.sh: line 77: ---: command not found ---- Deletion Phase 1: GKE Clusters (Top 20) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -The following Filestore instances are targeted for deletion in this run: -a4h-slurm-c0e262-f5260d85 -[EXECUTE] Filestore Instance: Deleting a4h-slurm-c0e262-f5260d85 in -ERROR: (gcloud.filestore.instances.delete) Error parsing [instance]. -The [instance] resource is not properly specified. -Failed to find attribute [zone]. The attribute can be set in the following ways: -- provide the argument `instance` on the command line with a fully specified name -- provide the argument `--zone` on the command line -- provide the argument `region` on the command line -- provide the argument `location` on the command line -- set the property `filestore/zone` -- set the property `filestore/region` -- set the property `filestore/location` ---- Thu Nov 27 03:36:52 AM UTC 2025 --- Cleanup Script Run Finished --- - diff --git a/network.txt b/network.txt deleted file mode 100644 index 03333e56ef..0000000000 --- a/network.txt +++ /dev/null @@ -1,2708 +0,0 @@ ---- Thu Nov 27 09:55:54 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T05:55:54+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -lustre-06-primary-subnet in us-central1 -lustre-test-06-primary-subnet in us-central1 ---- Processing Subnet: lustre-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-06-primary-subnet in us-central1. -[EXECUTE] Subnetwork: Deleting lustre-06-primary-subnet in us-central1 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustre06-slurm-login-001' - -ERROR: Failed to delete Subnetwork lustre-06-primary-subnet in us-central1. Check for other dependencies. ---- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-test-06-primary-subnet in us-central1. -[EXECUTE] Subnetwork: Deleting lustre-test-06-primary-subnet in us-central1 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-test-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustretest-controller' - -ERROR: Failed to delete Subnetwork lustre-test-06-primary-subnet in us-central1. Check for other dependencies. ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -The following Networks are targeted for deletion in this run: -a3u-onspot-slurm-829610-net-0 -a3u-slurm-net -a4hsarthakag-net -a4htest-net-0 -a4newimgek-net -a4newimgek-net-0 -a4newimgek-net-1 -a4newimgek-rdma-net -a4oldimgek-net -a4oldimgek-net-0 -a4oldimgek-net-1 -a4oldimgek-rdma-net -a4oldimg-net -a4xlavhpcnew-a4x-net-0 -a4xlavhpcnew-a4x-net-1 -a4xlavhpcnew-a4x-rdma-net -a4xslurm-net -cx-a3u-net-0 -db451c7-ml-slurm-v6-net -dynpoc-net -g4-dwsq-1-net-1 -g4qclav-net -gke-1395b4-net -gke-managed-lustre-basic-net -h4d-cluster-rdma-net-0 -h4dqc-net -h4dqc-rdma-net-0 -h4d-res-swarnabm4-3-net -h4d-res-swarnabm4-3-rdma-net -hanu-a3u-net -[EXECUTE] Network: Deleting a3u-onspot-slurm-829610-net-0 -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/a3u-onspot-slurm-829610-net-0' is already being used by 'projects/hpc-toolkit-dev/global/routes/default-route-ea3b20e196a82d45' - -[EXECUTE] Network: Deleting a3u-slurm-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/global/networks/a3u-slurm-net' was not found - -[EXECUTE] Network: Deleting a4hsarthakag-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/a4hsarthakag-net' is already being used by 'projects/hpc-toolkit-dev/global/routes/default-route-6884ebdc9d4edd99' - -[EXECUTE] Network: Deleting a4htest-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4htest-net-0]. -[EXECUTE] Network: Deleting a4newimgek-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4newimgek-net]. -[EXECUTE] Network: Deleting a4newimgek-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4newimgek-net-0]. -[EXECUTE] Network: Deleting a4newimgek-net-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4newimgek-net-1]. -[EXECUTE] Network: Deleting a4newimgek-rdma-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4newimgek-rdma-net]. -[EXECUTE] Network: Deleting a4oldimgek-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4oldimgek-net]. -[EXECUTE] Network: Deleting a4oldimgek-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4oldimgek-net-0]. -[EXECUTE] Network: Deleting a4oldimgek-net-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4oldimgek-net-1]. -[EXECUTE] Network: Deleting a4oldimgek-rdma-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4oldimgek-rdma-net]. -[EXECUTE] Network: Deleting a4oldimg-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4oldimg-net]. -[EXECUTE] Network: Deleting a4xlavhpcnew-a4x-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4xlavhpcnew-a4x-net-0]. -[EXECUTE] Network: Deleting a4xlavhpcnew-a4x-net-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4xlavhpcnew-a4x-net-1]. -[EXECUTE] Network: Deleting a4xlavhpcnew-a4x-rdma-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4xlavhpcnew-a4x-rdma-net]. -[EXECUTE] Network: Deleting a4xslurm-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4xslurm-net]. -[EXECUTE] Network: Deleting cx-a3u-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/cx-a3u-net-0]. -[EXECUTE] Network: Deleting db451c7-ml-slurm-v6-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/db451c7-ml-slurm-v6-net]. -[EXECUTE] Network: Deleting dynpoc-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/dynpoc-net]. -[EXECUTE] Network: Deleting g4-dwsq-1-net-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/g4-dwsq-1-net-1]. -[EXECUTE] Network: Deleting g4qclav-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/g4qclav-net]. -[EXECUTE] Network: Deleting gke-1395b4-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/gke-1395b4-net]. -[EXECUTE] Network: Deleting gke-managed-lustre-basic-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/gke-managed-lustre-basic-net' is already being used by 'projects/hpc-toolkit-dev/global/firewalls/gke-managed-lustre-basic-net-fw-allow-iap-ingress' - -[EXECUTE] Network: Deleting h4d-cluster-rdma-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/h4d-cluster-rdma-net-0]. -[EXECUTE] Network: Deleting h4dqc-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/h4dqc-net]. -[EXECUTE] Network: Deleting h4dqc-rdma-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/h4dqc-rdma-net-0]. -[EXECUTE] Network: Deleting h4d-res-swarnabm4-3-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/h4d-res-swarnabm4-3-net]. -[EXECUTE] Network: Deleting h4d-res-swarnabm4-3-rdma-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/h4d-res-swarnabm4-3-rdma-net]. -[EXECUTE] Network: Deleting hanu-a3u-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hanu-a3u-net]. -./cleanup.sh: line 470: syntax error near unexpected token `then' -./cleanup.sh: line 470: ` then' ---- Thu Nov 27 10:14:54 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T06:14:54+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -lustre-06-primary-subnet in us-central1 -lustre-test-06-primary-subnet in us-central1 ---- Processing Subnet: lustre-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-06-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-test-06-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -a3u-onspot-slurm-829610-net-0 -a4hsarthakag-net -gke-managed-lustre-basic-net -hanu-test-net -hpc-01-net -hpcdydis-net -hpcdy-net -hpc-exr-2-net-0 -hpcimg-net -hpc-lustre-test-02-net -khu-h4d-cluster-test-net -khu-h4d-cluster-test-rdma-net-0 -laveeek29-net-0 -laveeek29-net-1 -laveeek29-net -laveeek29-rdma-net -lavoldchk-net -lavold-net -lavrohek29-net-0 -lustre-06-net -lustre-test-06-net -mainek-net-0 -mainek-net-1 -mainek-net -mainek-rdma-net -managed-lustre-03-net -mglsa-net -mglsard-net -ml-gke-e2e-a8fae6-net -ml-gke-net ---- Processing Network: a3u-onspot-slurm-829610-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-ea3b20e196a82d45 for network a3u-onspot-slurm-829610-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete a3u-onspot-slurm-829610-net-0 - Command: gcloud compute networks delete "a3u-onspot-slurm-829610-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: a4hsarthakag-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-6884ebdc9d4edd99 for network a4hsarthakag-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete a4hsarthakag-net - Command: gcloud compute networks delete "a4hsarthakag-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: gke-managed-lustre-basic-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-98972e97de7c239b for network gke-managed-lustre-basic-net -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete gke-managed-lustre-basic-net-fw-allow-iap-ingress for network gke-managed-lustre-basic-net -[DRY RUN] Firewall Rule: Would delete gke-managed-lustre-basic-net-fw-allow-internal-traffic for network gke-managed-lustre-basic-net -[DRY RUN] Network: Would delete gke-managed-lustre-basic-net - Command: gcloud compute networks delete "gke-managed-lustre-basic-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hanu-test-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-d487416773e0d26a for network hanu-test-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hanu-test-net - Command: gcloud compute networks delete "hanu-test-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpc-01-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-bff8e31ecb82ca1f for network hpc-01-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpc-01-net - Command: gcloud compute networks delete "hpc-01-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpcdydis-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-83226775efe0e36f for network hpcdydis-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpcdydis-net - Command: gcloud compute networks delete "hpcdydis-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpcdy-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-11ccedffb21834f9 for network hpcdy-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpcdy-net - Command: gcloud compute networks delete "hpcdy-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpc-exr-2-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-ca20f8f43f32a6ad for network hpc-exr-2-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpc-exr-2-net-0 - Command: gcloud compute networks delete "hpc-exr-2-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpcimg-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-78cfceb7232d782e for network hpcimg-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpcimg-net - Command: gcloud compute networks delete "hpcimg-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpc-lustre-test-02-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-0d79bd2bca25d91e for network hpc-lustre-test-02-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpc-lustre-test-02-net - Command: gcloud compute networks delete "hpc-lustre-test-02-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: khu-h4d-cluster-test-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-87a3ee47eaab9ab6 for network khu-h4d-cluster-test-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete khu-h4d-cluster-test-net - Command: gcloud compute networks delete "khu-h4d-cluster-test-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: khu-h4d-cluster-test-rdma-net-0 --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete khu-h4d-cluster-test-rdma-net-0 - Command: gcloud compute networks delete "khu-h4d-cluster-test-rdma-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: laveeek29-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-1a384a0ae7610067 for network laveeek29-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete laveeek29-net-0 - Command: gcloud compute networks delete "laveeek29-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: laveeek29-net-1 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-70d6a0878f6e1100 for network laveeek29-net-1 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete laveeek29-net-1 - Command: gcloud compute networks delete "laveeek29-net-1" --project="hpc-toolkit-dev" --quiet ---- Processing Network: laveeek29-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-1a384a0ae7610067 for network laveeek29-net -[DRY RUN] Route: Would delete default-route-3f3b00742a1eca0f for network laveeek29-net -[DRY RUN] Route: Would delete default-route-70d6a0878f6e1100 for network laveeek29-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete laveeek29-net - Command: gcloud compute networks delete "laveeek29-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: laveeek29-rdma-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete laveeek29-rdma-net - Command: gcloud compute networks delete "laveeek29-rdma-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lavoldchk-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-5df8aadab663ab36 for network lavoldchk-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lavoldchk-net - Command: gcloud compute networks delete "lavoldchk-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lavold-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-6e021fe7f303c1fe for network lavold-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lavold-net - Command: gcloud compute networks delete "lavold-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lavrohek29-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-3af10cb6b3f7804e for network lavrohek29-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lavrohek29-net-0 - Command: gcloud compute networks delete "lavrohek29-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-a798499d8bf40405 for network lustre-06-net -[DRY RUN] Route: Would delete default-route-r-d1138650da68a40d for network lustre-06-net -[DRY RUN] Route: Would delete peering-route-ce292d9a6fb17bbc for network lustre-06-net -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete lustre-06-net-fw-allow-iap-ingress for network lustre-06-net -[DRY RUN] Firewall Rule: Would delete lustre-06-net-fw-allow-internal-traffic for network lustre-06-net -[DRY RUN] Network: Would delete lustre-06-net - Command: gcloud compute networks delete "lustre-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-test-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-799f13a785b25782 for network lustre-test-06-net -[DRY RUN] Route: Would delete default-route-r-802e2cad48aae840 for network lustre-test-06-net -[DRY RUN] Route: Would delete peering-route-2273ac2f3dccd077 for network lustre-test-06-net -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete lustre-test-06-net-fw-allow-iap-ingress for network lustre-test-06-net -[DRY RUN] Firewall Rule: Would delete lustre-test-06-net-fw-allow-internal-traffic for network lustre-test-06-net -[DRY RUN] Network: Would delete lustre-test-06-net - Command: gcloud compute networks delete "lustre-test-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mainek-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-4dce75664c4c0cf2 for network mainek-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mainek-net-0 - Command: gcloud compute networks delete "mainek-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mainek-net-1 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-ceb2655d4bb6b336 for network mainek-net-1 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mainek-net-1 - Command: gcloud compute networks delete "mainek-net-1" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mainek-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-4dce75664c4c0cf2 for network mainek-net -[DRY RUN] Route: Would delete default-route-936046db1078bf03 for network mainek-net -[DRY RUN] Route: Would delete default-route-ceb2655d4bb6b336 for network mainek-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mainek-net - Command: gcloud compute networks delete "mainek-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mainek-rdma-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mainek-rdma-net - Command: gcloud compute networks delete "mainek-rdma-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: managed-lustre-03-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-912e02dbc2c43267 for network managed-lustre-03-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete managed-lustre-03-net - Command: gcloud compute networks delete "managed-lustre-03-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mglsa-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-111ca8b8a3346fb4 for network mglsa-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mglsa-net - Command: gcloud compute networks delete "mglsa-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mglsard-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-4afd256f7ddc3be2 for network mglsard-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mglsard-net - Command: gcloud compute networks delete "mglsard-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: ml-gke-e2e-a8fae6-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-2c3a152cc75c7587 for network ml-gke-e2e-a8fae6-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete ml-gke-e2e-a8fae6-net - Command: gcloud compute networks delete "ml-gke-e2e-a8fae6-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: ml-gke-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-1dadbcd4c82ffb80 for network ml-gke-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete ml-gke-net - Command: gcloud compute networks delete "ml-gke-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Thu Nov 27 10:17:13 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 10:17:29 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T06:17:29+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 56 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -lustre06-controller us-central1-a -lustre06-slurm-login-001 us-central1-a -lustretest-controller us-central1-a -lustretest-slurm-login-001 us-central1-a - gcloud compute instances delete "lustre06-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "lustre06-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "lustretest-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "lustretest-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet ---- Deletion Phase 1: Filestore Instances (Top 30) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -lustre-06-primary-subnet in us-central1 -lustre-test-06-primary-subnet in us-central1 ---- Processing Subnet: lustre-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-06-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-test-06-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -a3u-onspot-slurm-829610-net-0 -a4hsarthakag-net -gke-managed-lustre-basic-net -hanu-test-net -hpc-01-net -hpcdydis-net -hpcdy-net -hpc-exr-2-net-0 -hpcimg-net -hpc-lustre-test-02-net -khu-h4d-cluster-test-net -khu-h4d-cluster-test-rdma-net-0 -laveeek29-net-0 -laveeek29-net-1 -laveeek29-net -laveeek29-rdma-net -lavoldchk-net -lavold-net -lavrohek29-net-0 -lustre-06-net -lustre-test-06-net -mainek-net-0 -mainek-net-1 -mainek-net -mainek-rdma-net -managed-lustre-03-net -mglsa-net -mglsard-net -ml-gke-e2e-a8fae6-net -ml-gke-net ---- Processing Network: a3u-onspot-slurm-829610-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-ea3b20e196a82d45 for network a3u-onspot-slurm-829610-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete a3u-onspot-slurm-829610-net-0 - Command: gcloud compute networks delete "a3u-onspot-slurm-829610-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: a4hsarthakag-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-6884ebdc9d4edd99 for network a4hsarthakag-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete a4hsarthakag-net - Command: gcloud compute networks delete "a4hsarthakag-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: gke-managed-lustre-basic-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-98972e97de7c239b for network gke-managed-lustre-basic-net -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete gke-managed-lustre-basic-net-fw-allow-iap-ingress for network gke-managed-lustre-basic-net -[DRY RUN] Firewall Rule: Would delete gke-managed-lustre-basic-net-fw-allow-internal-traffic for network gke-managed-lustre-basic-net -[DRY RUN] Network: Would delete gke-managed-lustre-basic-net - Command: gcloud compute networks delete "gke-managed-lustre-basic-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hanu-test-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-d487416773e0d26a for network hanu-test-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hanu-test-net - Command: gcloud compute networks delete "hanu-test-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpc-01-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-bff8e31ecb82ca1f for network hpc-01-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpc-01-net - Command: gcloud compute networks delete "hpc-01-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpcdydis-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-83226775efe0e36f for network hpcdydis-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpcdydis-net - Command: gcloud compute networks delete "hpcdydis-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpcdy-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-11ccedffb21834f9 for network hpcdy-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpcdy-net - Command: gcloud compute networks delete "hpcdy-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpc-exr-2-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-ca20f8f43f32a6ad for network hpc-exr-2-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpc-exr-2-net-0 - Command: gcloud compute networks delete "hpc-exr-2-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpcimg-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-78cfceb7232d782e for network hpcimg-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpcimg-net - Command: gcloud compute networks delete "hpcimg-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: hpc-lustre-test-02-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-0d79bd2bca25d91e for network hpc-lustre-test-02-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete hpc-lustre-test-02-net - Command: gcloud compute networks delete "hpc-lustre-test-02-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: khu-h4d-cluster-test-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-87a3ee47eaab9ab6 for network khu-h4d-cluster-test-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete khu-h4d-cluster-test-net - Command: gcloud compute networks delete "khu-h4d-cluster-test-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: khu-h4d-cluster-test-rdma-net-0 --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete khu-h4d-cluster-test-rdma-net-0 - Command: gcloud compute networks delete "khu-h4d-cluster-test-rdma-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: laveeek29-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-1a384a0ae7610067 for network laveeek29-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete laveeek29-net-0 - Command: gcloud compute networks delete "laveeek29-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: laveeek29-net-1 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-70d6a0878f6e1100 for network laveeek29-net-1 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete laveeek29-net-1 - Command: gcloud compute networks delete "laveeek29-net-1" --project="hpc-toolkit-dev" --quiet ---- Processing Network: laveeek29-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-1a384a0ae7610067 for network laveeek29-net -[DRY RUN] Route: Would delete default-route-3f3b00742a1eca0f for network laveeek29-net -[DRY RUN] Route: Would delete default-route-70d6a0878f6e1100 for network laveeek29-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete laveeek29-net - Command: gcloud compute networks delete "laveeek29-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: laveeek29-rdma-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete laveeek29-rdma-net - Command: gcloud compute networks delete "laveeek29-rdma-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lavoldchk-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-5df8aadab663ab36 for network lavoldchk-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lavoldchk-net - Command: gcloud compute networks delete "lavoldchk-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lavold-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-6e021fe7f303c1fe for network lavold-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lavold-net - Command: gcloud compute networks delete "lavold-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lavrohek29-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-3af10cb6b3f7804e for network lavrohek29-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lavrohek29-net-0 - Command: gcloud compute networks delete "lavrohek29-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-a798499d8bf40405 for network lustre-06-net -[DRY RUN] Route: Would delete default-route-r-d1138650da68a40d for network lustre-06-net -[DRY RUN] Route: Would delete peering-route-ce292d9a6fb17bbc for network lustre-06-net -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete lustre-06-net-fw-allow-iap-ingress for network lustre-06-net -[DRY RUN] Firewall Rule: Would delete lustre-06-net-fw-allow-internal-traffic for network lustre-06-net -[DRY RUN] Network: Would delete lustre-06-net - Command: gcloud compute networks delete "lustre-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-test-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-799f13a785b25782 for network lustre-test-06-net -[DRY RUN] Route: Would delete default-route-r-802e2cad48aae840 for network lustre-test-06-net -[DRY RUN] Route: Would delete peering-route-2273ac2f3dccd077 for network lustre-test-06-net -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete lustre-test-06-net-fw-allow-iap-ingress for network lustre-test-06-net -[DRY RUN] Firewall Rule: Would delete lustre-test-06-net-fw-allow-internal-traffic for network lustre-test-06-net -[DRY RUN] Network: Would delete lustre-test-06-net - Command: gcloud compute networks delete "lustre-test-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mainek-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-4dce75664c4c0cf2 for network mainek-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mainek-net-0 - Command: gcloud compute networks delete "mainek-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mainek-net-1 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-ceb2655d4bb6b336 for network mainek-net-1 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mainek-net-1 - Command: gcloud compute networks delete "mainek-net-1" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mainek-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-4dce75664c4c0cf2 for network mainek-net -[DRY RUN] Route: Would delete default-route-936046db1078bf03 for network mainek-net -[DRY RUN] Route: Would delete default-route-ceb2655d4bb6b336 for network mainek-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mainek-net - Command: gcloud compute networks delete "mainek-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mainek-rdma-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mainek-rdma-net - Command: gcloud compute networks delete "mainek-rdma-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: managed-lustre-03-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-912e02dbc2c43267 for network managed-lustre-03-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete managed-lustre-03-net - Command: gcloud compute networks delete "managed-lustre-03-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mglsa-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-111ca8b8a3346fb4 for network mglsa-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mglsa-net - Command: gcloud compute networks delete "mglsa-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mglsard-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-4afd256f7ddc3be2 for network mglsard-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mglsard-net - Command: gcloud compute networks delete "mglsard-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: ml-gke-e2e-a8fae6-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-2c3a152cc75c7587 for network ml-gke-e2e-a8fae6-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete ml-gke-e2e-a8fae6-net - Command: gcloud compute networks delete "ml-gke-e2e-a8fae6-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: ml-gke-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-1dadbcd4c82ffb80 for network ml-gke-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete ml-gke-net - Command: gcloud compute networks delete "ml-gke-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Thu Nov 27 10:19:45 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 10:19:57 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T06:19:57+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 50 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -lustre06-controller us-central1-a -lustre06-slurm-login-001 us-central1-a -lustretest-controller us-central1-a -lustretest-slurm-login-001 us-central1-a -[EXECUTE] Instance: Deleting lustre06-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustre06-controller]. -[EXECUTE] Instance: Deleting lustre06-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustre06-slurm-login-001]. -[EXECUTE] Instance: Deleting lustretest-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustretest-controller]. -[EXECUTE] Instance: Deleting lustretest-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustretest-slurm-login-001]. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -lustre-06-net-router us-central1 -lustre-test-06-net-router us-central1 -[EXECUTE] Cloud Router: Deleting lustre-06-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/lustre-06-net-router]. -[EXECUTE] Cloud Router: Deleting lustre-test-06-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/lustre-test-06-net-router]. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -lustre-06-net-fw-allow-iap-ingress -lustre-06-net-fw-allow-internal-traffic -lustre-test-06-net-fw-allow-iap-ingress -lustre-test-06-net-fw-allow-internal-traffic -[EXECUTE] Firewall Rule: Deleting lustre-06-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-06-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting lustre-06-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-06-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting lustre-test-06-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-test-06-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting lustre-test-06-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-test-06-net-fw-allow-internal-traffic]. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -lustre-06-primary-subnet in us-central1 -lustre-test-06-primary-subnet in us-central1 ---- Processing Subnet: lustre-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-06-primary-subnet in us-central1. -[EXECUTE] Subnetwork: Deleting lustre-06-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-06-primary-subnet]. -Successfully deleted Subnetwork lustre-06-primary-subnet in us-central1. ---- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-test-06-primary-subnet in us-central1. -[EXECUTE] Subnetwork: Deleting lustre-test-06-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-test-06-primary-subnet]. -Successfully deleted Subnetwork lustre-test-06-primary-subnet in us-central1. ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -a3u-onspot-slurm-829610-net-0 -a4hsarthakag-net -gke-managed-lustre-basic-net -hanu-test-net -hpc-01-net -hpcdydis-net -hpcdy-net -hpc-exr-2-net-0 -hpcimg-net -hpc-lustre-test-02-net -khu-h4d-cluster-test-net -khu-h4d-cluster-test-rdma-net-0 -laveeek29-net-0 -laveeek29-net-1 -laveeek29-net -laveeek29-rdma-net -lavoldchk-net -lavold-net -lavrohek29-net-0 -lustre-06-net -lustre-test-06-net -mainek-net-0 -mainek-net-1 -mainek-net -mainek-rdma-net -managed-lustre-03-net -mglsa-net -mglsard-net -ml-gke-e2e-a8fae6-net -ml-gke-net ---- Processing Network: a3u-onspot-slurm-829610-net-0 --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-ea3b20e196a82d45 for network a3u-onspot-slurm-829610-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-ea3b20e196a82d45]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting a3u-onspot-slurm-829610-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a3u-onspot-slurm-829610-net-0]. -Successfully deleted Network a3u-onspot-slurm-829610-net-0. ---- Processing Network: a4hsarthakag-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-6884ebdc9d4edd99 for network a4hsarthakag-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-6884ebdc9d4edd99]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting a4hsarthakag-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/a4hsarthakag-net]. -Successfully deleted Network a4hsarthakag-net. ---- Processing Network: gke-managed-lustre-basic-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-98972e97de7c239b for network gke-managed-lustre-basic-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-98972e97de7c239b]. -Checking for dependent firewall rules... -[EXECUTE] Firewall Rule: Deleting gke-managed-lustre-basic-net-fw-allow-iap-ingress for network gke-managed-lustre-basic-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/gke-managed-lustre-basic-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting gke-managed-lustre-basic-net-fw-allow-internal-traffic for network gke-managed-lustre-basic-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/gke-managed-lustre-basic-net-fw-allow-internal-traffic]. -[EXECUTE] Network: Deleting gke-managed-lustre-basic-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/gke-managed-lustre-basic-net]. -Successfully deleted Network gke-managed-lustre-basic-net. ---- Processing Network: hanu-test-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-d487416773e0d26a for network hanu-test-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-d487416773e0d26a]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting hanu-test-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hanu-test-net]. -Successfully deleted Network hanu-test-net. ---- Processing Network: hpc-01-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-bff8e31ecb82ca1f for network hpc-01-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-bff8e31ecb82ca1f]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting hpc-01-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpc-01-net]. -Successfully deleted Network hpc-01-net. ---- Processing Network: hpcdydis-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-83226775efe0e36f for network hpcdydis-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-83226775efe0e36f]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting hpcdydis-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpcdydis-net]. -Successfully deleted Network hpcdydis-net. ---- Processing Network: hpcdy-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-11ccedffb21834f9 for network hpcdy-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-11ccedffb21834f9]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting hpcdy-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpcdy-net]. -Successfully deleted Network hpcdy-net. ---- Processing Network: hpc-exr-2-net-0 --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-ca20f8f43f32a6ad for network hpc-exr-2-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-ca20f8f43f32a6ad]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting hpc-exr-2-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpc-exr-2-net-0]. -Successfully deleted Network hpc-exr-2-net-0. ---- Processing Network: hpcimg-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-78cfceb7232d782e for network hpcimg-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-78cfceb7232d782e]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting hpcimg-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpcimg-net]. -Successfully deleted Network hpcimg-net. ---- Processing Network: hpc-lustre-test-02-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-0d79bd2bca25d91e for network hpc-lustre-test-02-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-0d79bd2bca25d91e]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting hpc-lustre-test-02-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/hpc-lustre-test-02-net]. -Successfully deleted Network hpc-lustre-test-02-net. ---- Processing Network: khu-h4d-cluster-test-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-87a3ee47eaab9ab6 for network khu-h4d-cluster-test-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-87a3ee47eaab9ab6]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting khu-h4d-cluster-test-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/khu-h4d-cluster-test-net]. -Successfully deleted Network khu-h4d-cluster-test-net. ---- Processing Network: khu-h4d-cluster-test-rdma-net-0 --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting khu-h4d-cluster-test-rdma-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/khu-h4d-cluster-test-rdma-net-0]. -Successfully deleted Network khu-h4d-cluster-test-rdma-net-0. ---- Processing Network: laveeek29-net-0 --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-1a384a0ae7610067 for network laveeek29-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-1a384a0ae7610067]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting laveeek29-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/laveeek29-net-0]. -Successfully deleted Network laveeek29-net-0. ---- Processing Network: laveeek29-net-1 --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-70d6a0878f6e1100 for network laveeek29-net-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-70d6a0878f6e1100]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting laveeek29-net-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/laveeek29-net-1]. -Successfully deleted Network laveeek29-net-1. ---- Processing Network: laveeek29-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-3f3b00742a1eca0f for network laveeek29-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-3f3b00742a1eca0f]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting laveeek29-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/laveeek29-net]. -Successfully deleted Network laveeek29-net. ---- Processing Network: laveeek29-rdma-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting laveeek29-rdma-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/laveeek29-rdma-net]. -Successfully deleted Network laveeek29-rdma-net. ---- Processing Network: lavoldchk-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-5df8aadab663ab36 for network lavoldchk-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-5df8aadab663ab36]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lavoldchk-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/lavoldchk-net]. -Successfully deleted Network lavoldchk-net. ---- Processing Network: lavold-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-6e021fe7f303c1fe for network lavold-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-6e021fe7f303c1fe]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lavold-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/lavold-net]. -Successfully deleted Network lavold-net. ---- Processing Network: lavrohek29-net-0 --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-3af10cb6b3f7804e for network lavrohek29-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-3af10cb6b3f7804e]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lavrohek29-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/lavrohek29-net-0]. -Successfully deleted Network lavrohek29-net-0. ---- Processing Network: lustre-06-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-a798499d8bf40405 for network lustre-06-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-a798499d8bf40405]. -[EXECUTE] Route: Deleting peering-route-ce292d9a6fb17bbc for network lustre-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-ce292d9a6fb17bbc -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-06-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/lustre-06-net]. -Successfully deleted Network lustre-06-net. ---- Processing Network: lustre-test-06-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-799f13a785b25782 for network lustre-test-06-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-799f13a785b25782]. -[EXECUTE] Route: Deleting peering-route-2273ac2f3dccd077 for network lustre-test-06-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-2273ac2f3dccd077 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-test-06-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/lustre-test-06-net]. -Successfully deleted Network lustre-test-06-net. ---- Processing Network: mainek-net-0 --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-4dce75664c4c0cf2 for network mainek-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-4dce75664c4c0cf2]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting mainek-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mainek-net-0]. -Successfully deleted Network mainek-net-0. ---- Processing Network: mainek-net-1 --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-ceb2655d4bb6b336 for network mainek-net-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-ceb2655d4bb6b336]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting mainek-net-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mainek-net-1]. -Successfully deleted Network mainek-net-1. ---- Processing Network: mainek-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-936046db1078bf03 for network mainek-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-936046db1078bf03]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting mainek-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mainek-net]. -Successfully deleted Network mainek-net. ---- Processing Network: mainek-rdma-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting mainek-rdma-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mainek-rdma-net]. -Successfully deleted Network mainek-rdma-net. ---- Processing Network: managed-lustre-03-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-912e02dbc2c43267 for network managed-lustre-03-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-912e02dbc2c43267]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting managed-lustre-03-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/managed-lustre-03-net]. -Successfully deleted Network managed-lustre-03-net. ---- Processing Network: mglsa-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-111ca8b8a3346fb4 for network mglsa-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-111ca8b8a3346fb4]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting mglsa-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/mglsa-net' is already being used by 'projects/hpc-toolkit-dev/regions/us-central1/routers/mglsa-net-router' - -ERROR: Failed to delete Network mglsa-net. Check for remaining dependencies. ---- Processing Network: mglsard-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-4afd256f7ddc3be2 for network mglsard-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-4afd256f7ddc3be2]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting mglsard-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/mglsard-net' is already being used by 'projects/hpc-toolkit-dev/regions/us-west4/routers/mglsard-net-router' - -ERROR: Failed to delete Network mglsard-net. Check for remaining dependencies. ---- Processing Network: ml-gke-e2e-a8fae6-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-2c3a152cc75c7587 for network ml-gke-e2e-a8fae6-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-2c3a152cc75c7587]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting ml-gke-e2e-a8fae6-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/ml-gke-e2e-a8fae6-net]. -Successfully deleted Network ml-gke-e2e-a8fae6-net. ---- Processing Network: ml-gke-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-1dadbcd4c82ffb80 for network ml-gke-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-1dadbcd4c82ffb80]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting ml-gke-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/ml-gke-net]. -Successfully deleted Network ml-gke-net. ---- Network Deletion Process Complete --- ---- Thu Nov 27 10:50:43 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 11:01:01 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T07:01:01+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 50 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -lustreqa05-controller us-central1-a -lustreqa05-slurm-login-001 us-central1-a - gcloud compute instances delete "lustreqa05-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "lustreqa05-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -lustre-qa-05-net-router us-central1 -[DRY RUN] Cloud Router: Would delete lustre-qa-05-net-router in us-central1 - Command: gcloud compute routers delete "lustre-qa-05-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -lustre-qa-05-net-fw-allow-iap-ingress -lustre-qa-05-net-fw-allow-internal-traffic -[DRY RUN] Firewall Rule: Would delete lustre-qa-05-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "lustre-qa-05-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete lustre-qa-05-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "lustre-qa-05-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -lustre-qa-05-primary-subnet in us-central1 ---- Processing Subnet: lustre-qa-05-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-qa-05-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete lustre-qa-05-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-qa-05-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-qa-05-net -mglsa-net -mglsard-net -monitoring-8323fe-net -pkrv6ff952b-net -sa-chs-ops-net-0 -sarthakagrr-net -sispot3u-net-0 -slurm-a3-base-sysnet -sp-helmtest1-net -static-sarthakag-net -test-ssd-psc ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-659a1f3173aa28ed for network lustre-qa-05-net -[DRY RUN] Route: Would delete default-route-r-58923476b12e6284 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -[DRY RUN] Firewall Rule: Would delete lustre-qa-05-net-fw-allow-iap-ingress for network lustre-qa-05-net -[DRY RUN] Firewall Rule: Would delete lustre-qa-05-net-fw-allow-internal-traffic for network lustre-qa-05-net -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mglsa-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mglsa-net - Command: gcloud compute networks delete "mglsa-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mglsard-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mglsard-net - Command: gcloud compute networks delete "mglsard-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: monitoring-8323fe-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-b3d726b99719ebcd for network monitoring-8323fe-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete monitoring-8323fe-net - Command: gcloud compute networks delete "monitoring-8323fe-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: pkrv6ff952b-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-e6359607fabd1429 for network pkrv6ff952b-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete pkrv6ff952b-net - Command: gcloud compute networks delete "pkrv6ff952b-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: sa-chs-ops-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-e4841ee375dabbb7 for network sa-chs-ops-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete sa-chs-ops-net-0 - Command: gcloud compute networks delete "sa-chs-ops-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: sarthakagrr-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-60f6e9638a61ada1 for network sarthakagrr-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete sarthakagrr-net - Command: gcloud compute networks delete "sarthakagrr-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: sispot3u-net-0 --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-5330704133972804 for network sispot3u-net-0 -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete sispot3u-net-0 - Command: gcloud compute networks delete "sispot3u-net-0" --project="hpc-toolkit-dev" --quiet ---- Processing Network: slurm-a3-base-sysnet --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-6345135f78abc28d for network slurm-a3-base-sysnet -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete slurm-a3-base-sysnet - Command: gcloud compute networks delete "slurm-a3-base-sysnet" --project="hpc-toolkit-dev" --quiet ---- Processing Network: sp-helmtest1-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-b4f451a5b7595629 for network sp-helmtest1-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete sp-helmtest1-net - Command: gcloud compute networks delete "sp-helmtest1-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: static-sarthakag-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-31f39ec480a9048d for network static-sarthakag-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete static-sarthakag-net - Command: gcloud compute networks delete "static-sarthakag-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: test-ssd-psc --- -Checking for dependent routes... -[DRY RUN] Route: Would delete default-route-6484ee065c661d7a for network test-ssd-psc -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete test-ssd-psc - Command: gcloud compute networks delete "test-ssd-psc" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Thu Nov 27 11:02:06 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 11:02:47 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T07:02:47+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 50 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -lustreqa05-controller us-central1-a -lustreqa05-slurm-login-001 us-central1-a -[EXECUTE] Instance: Deleting lustreqa05-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustreqa05-controller]. -[EXECUTE] Instance: Deleting lustreqa05-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustreqa05-slurm-login-001]. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -lustre-qa-05-net-router us-central1 -[EXECUTE] Cloud Router: Deleting lustre-qa-05-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/lustre-qa-05-net-router]. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -lustre-qa-05-net-fw-allow-iap-ingress -lustre-qa-05-net-fw-allow-internal-traffic -[EXECUTE] Firewall Rule: Deleting lustre-qa-05-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-qa-05-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting lustre-qa-05-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lustre-qa-05-net-fw-allow-internal-traffic]. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -lustre-qa-05-primary-subnet in us-central1 ---- Processing Subnet: lustre-qa-05-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-qa-05-primary-subnet in us-central1. -[EXECUTE] Subnetwork: Deleting lustre-qa-05-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-qa-05-primary-subnet]. -Successfully deleted Subnetwork lustre-qa-05-primary-subnet in us-central1. ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-qa-05-net -mglsa-net -mglsard-net -monitoring-8323fe-net -pkrv6ff952b-net -sa-chs-ops-net-0 -sarthakagrr-net -sispot3u-net-0 -slurm-a3-base-sysnet -sp-helmtest1-net -static-sarthakag-net -test-ssd-psc ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-659a1f3173aa28ed for network lustre-qa-05-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-659a1f3173aa28ed]. -[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b91a4552351d170 -[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-6f7c1d8537c80540 -[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-qa-05-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' - -ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. ---- Processing Network: mglsa-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting mglsa-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/mglsa-net' is already being used by 'projects/hpc-toolkit-dev/regions/us-central1/routers/mglsa-net-router' - -ERROR: Failed to delete Network mglsa-net. Check for remaining dependencies. ---- Processing Network: mglsard-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting mglsard-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/mglsard-net' is already being used by 'projects/hpc-toolkit-dev/regions/us-west4/routers/mglsard-net-router' - -ERROR: Failed to delete Network mglsard-net. Check for remaining dependencies. ---- Processing Network: monitoring-8323fe-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-b3d726b99719ebcd for network monitoring-8323fe-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-b3d726b99719ebcd]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting monitoring-8323fe-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/monitoring-8323fe-net]. -Successfully deleted Network monitoring-8323fe-net. ---- Processing Network: pkrv6ff952b-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-e6359607fabd1429 for network pkrv6ff952b-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-e6359607fabd1429]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting pkrv6ff952b-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/pkrv6ff952b-net]. -Successfully deleted Network pkrv6ff952b-net. ---- Processing Network: sa-chs-ops-net-0 --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-e4841ee375dabbb7 for network sa-chs-ops-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-e4841ee375dabbb7]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting sa-chs-ops-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/sa-chs-ops-net-0]. -Successfully deleted Network sa-chs-ops-net-0. ---- Processing Network: sarthakagrr-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-60f6e9638a61ada1 for network sarthakagrr-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-60f6e9638a61ada1]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting sarthakagrr-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/sarthakagrr-net]. -Successfully deleted Network sarthakagrr-net. ---- Processing Network: sispot3u-net-0 --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-5330704133972804 for network sispot3u-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-5330704133972804]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting sispot3u-net-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/sispot3u-net-0]. -Successfully deleted Network sispot3u-net-0. ---- Processing Network: slurm-a3-base-sysnet --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-6345135f78abc28d for network slurm-a3-base-sysnet -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-6345135f78abc28d]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting slurm-a3-base-sysnet -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/slurm-a3-base-sysnet]. -Successfully deleted Network slurm-a3-base-sysnet. ---- Processing Network: sp-helmtest1-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-b4f451a5b7595629 for network sp-helmtest1-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-b4f451a5b7595629]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting sp-helmtest1-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/sp-helmtest1-net]. -Successfully deleted Network sp-helmtest1-net. ---- Processing Network: static-sarthakag-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-31f39ec480a9048d for network static-sarthakag-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-31f39ec480a9048d]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting static-sarthakag-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/static-sarthakag-net]. -Successfully deleted Network static-sarthakag-net. ---- Processing Network: test-ssd-psc --- -Checking for dependent routes... -[EXECUTE] Route: Deleting default-route-6484ee065c661d7a for network test-ssd-psc -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/routes/default-route-6484ee065c661d7a]. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting test-ssd-psc -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/test-ssd-psc]. -Successfully deleted Network test-ssd-psc. ---- Network Deletion Process Complete --- ---- Thu Nov 27 11:13:01 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 11:13:51 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T07:13:51+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -mglsa-net-router us-central1 -mglsard-net-router us-west4 -[DRY RUN] Cloud Router: Would delete mglsa-net-router in us-central1 - Command: gcloud compute routers delete "mglsa-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Cloud Router: Would delete mglsard-net-router in us-west4 - Command: gcloud compute routers delete "mglsard-net-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. ---- Thu Nov 27 11:17:54 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T07:17:54+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -mglsa-net-router us-central1 -mglsard-net-router us-west4 -[EXECUTE] Cloud Router: Deleting mglsa-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/mglsa-net-router]. -[EXECUTE] Cloud Router: Deleting mglsard-net-router in us-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west4/routers/mglsard-net-router]. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. ---- Thu Nov 27 11:19:04 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T07:19:04+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. -The following Subnetworks (and their dependent addresses) are targeted for deletion: - in ---- Processing Subnet: in --- -ERROR: (gcloud.compute.addresses.list) could not parse resource [] -ERROR: Failed to list dependent addresses for in . -WARNING: Skipping deletion of Subnet in . ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-qa-05-net -mglsa-net -mglsard-net ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mglsa-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mglsa-net - Command: gcloud compute networks delete "mglsa-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: mglsard-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete mglsard-net - Command: gcloud compute networks delete "mglsard-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Thu Nov 27 11:19:31 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 11:19:45 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T07:19:45+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. -The following Subnetworks (and their dependent addresses) are targeted for deletion: - in ---- Processing Subnet: in --- -ERROR: (gcloud.compute.addresses.list) could not parse resource [] -ERROR: Failed to list dependent addresses for in . -WARNING: Skipping deletion of Subnet in . ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-qa-05-net -mglsa-net -mglsard-net ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[EXECUTE] Route: Deleting peering-route-3b91a4552351d170 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-3b91a4552351d170 -[EXECUTE] Route: Deleting peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-6f7c1d8537c80540 -[EXECUTE] Route: Deleting peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -ERROR: (gcloud.compute.routes.delete) Could not fetch resource: - - The auto-generated peering route cannot be deleted. - -ERROR: Failed to delete route peering-route-c2ff29e0ce578be2 -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting lustre-qa-05-net -ERROR: (gcloud.compute.networks.delete) Could not fetch resource: - - The network resource 'projects/hpc-toolkit-dev/global/networks/lustre-qa-05-net' is already being used by 'projects/hpc-toolkit-dev/global/addresses/global-psconnect-ip-25305862' - -ERROR: Failed to delete Network lustre-qa-05-net. Check for remaining dependencies. ---- Processing Network: mglsa-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting mglsa-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mglsa-net]. -Successfully deleted Network mglsa-net. ---- Processing Network: mglsard-net --- -Checking for dependent routes... -No dependent routes found. -Checking for dependent firewall rules... -No dependent firewall rules found. -[EXECUTE] Network: Deleting mglsard-net -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/networks/mglsard-net]. -Successfully deleted Network mglsard-net. ---- Network Deletion Process Complete --- ---- Thu Nov 27 11:21:42 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 11:35:00 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T07:35:00+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. -The following Subnetworks (and their dependent addresses) are targeted for deletion: - in ---- Processing Subnet: in --- -ERROR: (gcloud.compute.addresses.list) could not parse resource [] -ERROR: Failed to list dependent addresses for in . -WARNING: Skipping deletion of Subnet in . ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-qa-05-net ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Thu Nov 27 11:35:24 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 11:54:19 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T07:54:19+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-qa-05-net ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Thu Nov 27 11:54:37 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 11:55:13 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T07:55:13+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-qa-05-net ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Thu Nov 27 11:55:29 AM UTC 2025 --- Cleanup Script Run Finished --- - diff --git a/networks.txt b/networks.txt deleted file mode 100644 index d93f7d4dc6..0000000000 --- a/networks.txt +++ /dev/null @@ -1,2086 +0,0 @@ ---- Wed Nov 26 02:16:54 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 37 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -The following Firewall Rules are targeted for deletion in this run: -a3mega-sys-net-shu7-fw-allow-iap-ingress -a3mega-sys-net-shu7-fw-allow-internal-traffic -a3u-slurm-net-fw-allow-iap-ingress -a3u-slurm-net-fw-allow-internal-traffic -a4hsarthakag-net-fw-allow-iap-ingress -a4hsarthakag-net-fw-allow-internal-traffic -a4htest-internal-0 -a4htest-net-0-fw-allow-iap-ingress -a4newimgek-internal-0 -a4newimgek-internal-1 -a4newimgek-net-0-fw-allow-iap-ingress -a4newimgek-net-1-fw-allow-iap-ingress -a4newimgek-net-fw-allow-iap-ingress -a4newimgek-net-fw-allow-internal-traffic -a4oldimgek-internal-0 -a4oldimgek-internal-1 -a4oldimgek-net-0-fw-allow-iap-ingress -a4oldimgek-net-1-fw-allow-iap-ingress -a4oldimgek-net-fw-allow-iap-ingress -a4oldimgek-net-fw-allow-internal-traffic -[DRY RUN] Firewall Rule: Would delete a3mega-sys-net-shu7-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a3mega-sys-net-shu7-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a3mega-sys-net-shu7-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "a3mega-sys-net-shu7-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a3u-slurm-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a3u-slurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a3u-slurm-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "a3u-slurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4hsarthakag-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4hsarthakag-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4hsarthakag-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "a4hsarthakag-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4htest-internal-0 - Command: gcloud compute firewall-rules delete "a4htest-internal-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4htest-net-0-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4htest-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4newimgek-internal-0 - Command: gcloud compute firewall-rules delete "a4newimgek-internal-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4newimgek-internal-1 - Command: gcloud compute firewall-rules delete "a4newimgek-internal-1" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4newimgek-net-0-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4newimgek-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4newimgek-net-1-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4newimgek-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4newimgek-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4newimgek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4newimgek-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "a4newimgek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4oldimgek-internal-0 - Command: gcloud compute firewall-rules delete "a4oldimgek-internal-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4oldimgek-internal-1 - Command: gcloud compute firewall-rules delete "a4oldimgek-internal-1" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4oldimgek-net-0-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4oldimgek-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4oldimgek-net-1-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4oldimgek-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4oldimgek-net-fw-allow-iap-ingress - Command: gcloud compute firewall-rules delete "a4oldimgek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Firewall Rule: Would delete a4oldimgek-net-fw-allow-internal-traffic - Command: gcloud compute firewall-rules delete "a4oldimgek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 02:17:04 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:17:44 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 37 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -The following Firewall Rules are targeted for deletion in this run: -a3mega-sys-net-shu7-fw-allow-iap-ingress -a3mega-sys-net-shu7-fw-allow-internal-traffic -a3u-slurm-net-fw-allow-iap-ingress -a3u-slurm-net-fw-allow-internal-traffic -a4hsarthakag-net-fw-allow-iap-ingress -a4hsarthakag-net-fw-allow-internal-traffic -a4htest-internal-0 -a4htest-net-0-fw-allow-iap-ingress -a4newimgek-internal-0 -a4newimgek-internal-1 -a4newimgek-net-0-fw-allow-iap-ingress -a4newimgek-net-1-fw-allow-iap-ingress -a4newimgek-net-fw-allow-iap-ingress -a4newimgek-net-fw-allow-internal-traffic -a4oldimgek-internal-0 -a4oldimgek-internal-1 -a4oldimgek-net-0-fw-allow-iap-ingress -a4oldimgek-net-1-fw-allow-iap-ingress -a4oldimgek-net-fw-allow-iap-ingress -a4oldimgek-net-fw-allow-internal-traffic -[EXECUTE] Firewall Rule: Deleting a3mega-sys-net-shu7-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a3mega-sys-net-shu7-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a3mega-sys-net-shu7-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a3mega-sys-net-shu7-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting a3u-slurm-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a3u-slurm-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a3u-slurm-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a3u-slurm-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting a4hsarthakag-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4hsarthakag-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4hsarthakag-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4hsarthakag-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting a4htest-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4htest-internal-0]. -[EXECUTE] Firewall Rule: Deleting a4htest-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4htest-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4newimgek-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-internal-0]. -[EXECUTE] Firewall Rule: Deleting a4newimgek-internal-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-internal-1]. -[EXECUTE] Firewall Rule: Deleting a4newimgek-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4newimgek-net-1-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-net-1-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4newimgek-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4newimgek-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4newimgek-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting a4oldimgek-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-internal-0]. -[EXECUTE] Firewall Rule: Deleting a4oldimgek-internal-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-internal-1]. -[EXECUTE] Firewall Rule: Deleting a4oldimgek-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4oldimgek-net-1-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-net-1-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4oldimgek-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4oldimgek-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimgek-net-fw-allow-internal-traffic]. ---- Wed Nov 26 02:20:54 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:21:04 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 37 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -a4oldimg-net-fw-allow-iap-ingress -a4oldimg-net-fw-allow-internal-traffic -a4xlavhpcnew-a4x-internal-0 -a4xlavhpcnew-a4x-internal-1 -a4xlavhpcnew-a4x-net-0-fw-allow-iap-ingress -a4xlavhpcnew-a4x-net-0-fw-allow-internal-traffic -a4xlavhpcnew-a4x-net-0-fw-allow-ssh-ingress -a4xlavhpcnew-a4x-net-1-fw-allow-iap-ingress -a4xlavhpcnew-a4x-net-1-fw-allow-internal-traffic -a4xslurm-net-fw-allow-iap-ingress -a4xslurm-net-fw-allow-internal-traffic -cx-a3u-internal-0 -cx-a3u-net-0-fw-allow-iap-ingress -db451c7-ml-slurm-v6-net-fw-allow-iap-ingress -db451c7-ml-slurm-v6-net-fw-allow-internal-traffic -dynpoc-net-fw-allow-iap-ingress -dynpoc-net-fw-allow-internal-traffic -g4qclav-net-fw-allow-iap-ingress -g4qclav-net-fw-allow-internal-traffic -gke-1395b4-net-fw-allow-iap-ingress - gcloud compute firewall-rules delete "a4oldimg-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "a4oldimg-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-internal-0" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-internal-1" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-net-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-net-0-fw-allow-ssh-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "a4xlavhpcnew-a4x-net-1-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "a4xslurm-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "a4xslurm-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "cx-a3u-internal-0" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "cx-a3u-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "db451c7-ml-slurm-v6-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "db451c7-ml-slurm-v6-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "dynpoc-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "dynpoc-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "g4qclav-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "g4qclav-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-1395b4-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 02:21:13 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:21:47 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 37 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -a4oldimg-net-fw-allow-iap-ingress -a4oldimg-net-fw-allow-internal-traffic -a4xlavhpcnew-a4x-internal-0 -a4xlavhpcnew-a4x-internal-1 -a4xlavhpcnew-a4x-net-0-fw-allow-iap-ingress -a4xlavhpcnew-a4x-net-0-fw-allow-internal-traffic -a4xlavhpcnew-a4x-net-0-fw-allow-ssh-ingress -a4xlavhpcnew-a4x-net-1-fw-allow-iap-ingress -a4xlavhpcnew-a4x-net-1-fw-allow-internal-traffic -a4xslurm-net-fw-allow-iap-ingress -a4xslurm-net-fw-allow-internal-traffic -cx-a3u-internal-0 -cx-a3u-net-0-fw-allow-iap-ingress -db451c7-ml-slurm-v6-net-fw-allow-iap-ingress -db451c7-ml-slurm-v6-net-fw-allow-internal-traffic -dynpoc-net-fw-allow-iap-ingress -dynpoc-net-fw-allow-internal-traffic -g4qclav-net-fw-allow-iap-ingress -g4qclav-net-fw-allow-internal-traffic -gke-1395b4-net-fw-allow-iap-ingress -[EXECUTE] Firewall Rule: Deleting a4oldimg-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimg-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4oldimg-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4oldimg-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-internal-0]. -[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-internal-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-internal-1]. -[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-net-0-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-net-0-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-net-0-fw-allow-ssh-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-net-0-fw-allow-ssh-ingress]. -[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-net-1-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-net-1-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4xlavhpcnew-a4x-net-1-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xlavhpcnew-a4x-net-1-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting a4xslurm-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xslurm-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting a4xslurm-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/a4xslurm-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting cx-a3u-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/cx-a3u-internal-0]. -[EXECUTE] Firewall Rule: Deleting cx-a3u-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/cx-a3u-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting db451c7-ml-slurm-v6-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/db451c7-ml-slurm-v6-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting db451c7-ml-slurm-v6-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/db451c7-ml-slurm-v6-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting dynpoc-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/dynpoc-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting dynpoc-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/dynpoc-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting g4qclav-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/g4qclav-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting g4qclav-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/g4qclav-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting gke-1395b4-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/gke-1395b4-net-fw-allow-iap-ingress]. ---- Wed Nov 26 02:24:23 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:24:39 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 37 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -gke-1395b4-net-fw-allow-internal-traffic -gke-managed-lustre-basic-net-fw-allow-iap-ingress -gke-managed-lustre-basic-net-fw-allow-internal-traffic -h4dqc-net-fw-allow-iap-ingress -h4dqc-net-fw-allow-internal-traffic -h4dqc-rdma-0 -h4dqc-rdma-net-0-fw-allow-iap-ingress -h4dqc-rdma-net-0-fw-allow-internal-traffic -h4d-res-swarnabm4-3-internal -h4d-res-swarnabm4-3-net-fw-allow-iap-ingress -h4d-res-swarnabm4-3-net-fw-allow-internal-traffic -h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress -h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic -hpc-01-net-fw-allow-iap-ingress -hpc-01-net-fw-allow-internal-traffic -hpcdydis-net-fw-allow-iap-ingress -hpcdydis-net-fw-allow-internal-traffic -hpcdy-net-fw-allow-iap-ingress -hpcdy-net-fw-allow-internal-traffic -hpc-exr-2-internal-0 - gcloud compute firewall-rules delete "gke-1395b4-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-managed-lustre-basic-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-managed-lustre-basic-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-rdma-0" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-internal" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-01-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-01-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdy-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdy-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-exr-2-internal-0" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 02:24:48 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:25:53 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 37 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -gke-1395b4-net-fw-allow-internal-traffic -gke-managed-lustre-basic-net-fw-allow-iap-ingress -gke-managed-lustre-basic-net-fw-allow-internal-traffic -h4dqc-net-fw-allow-iap-ingress -h4dqc-net-fw-allow-internal-traffic -h4dqc-rdma-0 -h4dqc-rdma-net-0-fw-allow-iap-ingress -h4dqc-rdma-net-0-fw-allow-internal-traffic -h4d-res-swarnabm4-3-internal -h4d-res-swarnabm4-3-net-fw-allow-iap-ingress -h4d-res-swarnabm4-3-net-fw-allow-internal-traffic -h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress -h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic -hpc-01-net-fw-allow-iap-ingress -hpc-01-net-fw-allow-internal-traffic -hpcdydis-net-fw-allow-iap-ingress -hpcdydis-net-fw-allow-internal-traffic -hpcdy-net-fw-allow-iap-ingress -hpcdy-net-fw-allow-internal-traffic -hpc-exr-2-internal-0 - gcloud compute firewall-rules delete "gke-1395b4-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-managed-lustre-basic-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-managed-lustre-basic-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-rdma-0" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-internal" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-01-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-01-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdy-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdy-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-exr-2-internal-0" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 02:26:03 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:26:46 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 39 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -gke-1395b4-net-fw-allow-internal-traffic -h4dqc-net-fw-allow-iap-ingress -h4dqc-net-fw-allow-internal-traffic -h4dqc-rdma-0 -h4dqc-rdma-net-0-fw-allow-iap-ingress -h4dqc-rdma-net-0-fw-allow-internal-traffic -h4d-res-swarnabm4-3-internal -h4d-res-swarnabm4-3-net-fw-allow-iap-ingress -h4d-res-swarnabm4-3-net-fw-allow-internal-traffic -h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress -h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic -hpc-01-net-fw-allow-iap-ingress -hpc-01-net-fw-allow-internal-traffic -hpcdydis-net-fw-allow-iap-ingress -hpcdydis-net-fw-allow-internal-traffic -hpcdy-net-fw-allow-iap-ingress -hpcdy-net-fw-allow-internal-traffic -hpc-exr-2-internal-0 -hpc-exr-2-net-0-fw-allow-iap-ingress -hpcimg-net-fw-allow-iap-ingress - gcloud compute firewall-rules delete "gke-1395b4-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-rdma-0" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4dqc-rdma-net-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-internal" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-01-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-01-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdydis-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdy-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcdy-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-exr-2-internal-0" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-exr-2-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpcimg-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 02:26:56 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:27:17 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 39 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -gke-1395b4-net-fw-allow-internal-traffic -h4dqc-net-fw-allow-iap-ingress -h4dqc-net-fw-allow-internal-traffic -h4dqc-rdma-0 -h4dqc-rdma-net-0-fw-allow-iap-ingress -h4dqc-rdma-net-0-fw-allow-internal-traffic -h4d-res-swarnabm4-3-internal -h4d-res-swarnabm4-3-net-fw-allow-iap-ingress -h4d-res-swarnabm4-3-net-fw-allow-internal-traffic -h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress -h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic -hpc-01-net-fw-allow-iap-ingress -hpc-01-net-fw-allow-internal-traffic -hpcdydis-net-fw-allow-iap-ingress -hpcdydis-net-fw-allow-internal-traffic -hpcdy-net-fw-allow-iap-ingress -hpcdy-net-fw-allow-internal-traffic -hpc-exr-2-internal-0 -hpc-exr-2-net-0-fw-allow-iap-ingress -hpcimg-net-fw-allow-iap-ingress -[EXECUTE] Firewall Rule: Deleting gke-1395b4-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/gke-1395b4-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting h4dqc-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4dqc-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting h4dqc-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4dqc-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting h4dqc-rdma-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4dqc-rdma-0]. -[EXECUTE] Firewall Rule: Deleting h4dqc-rdma-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4dqc-rdma-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting h4dqc-rdma-net-0-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4dqc-rdma-net-0-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting h4d-res-swarnabm4-3-internal -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4d-res-swarnabm4-3-internal]. -[EXECUTE] Firewall Rule: Deleting h4d-res-swarnabm4-3-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4d-res-swarnabm4-3-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting h4d-res-swarnabm4-3-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4d-res-swarnabm4-3-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4d-res-swarnabm4-3-rdma-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/h4d-res-swarnabm4-3-rdma-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting hpc-01-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-01-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting hpc-01-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-01-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting hpcdydis-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcdydis-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting hpcdydis-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcdydis-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting hpcdy-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcdy-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting hpcdy-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcdy-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting hpc-exr-2-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-exr-2-internal-0]. -[EXECUTE] Firewall Rule: Deleting hpc-exr-2-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-exr-2-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting hpcimg-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcimg-net-fw-allow-iap-ingress]. ---- Wed Nov 26 02:29:58 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:11:18 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 39 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -a8a55slurm-nodeset-0 us-central1-a - gcloud compute instances delete "a8a55slurm-nodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -hpcimg-net-fw-allow-internal-traffic -hpc-lustre-test-02-net-fw-allow-iap-ingress -hpc-lustre-test-02-net-fw-allow-internal-traffic -khu-h4d-cluster-test-net-fw-allow-iap-ingress -khu-h4d-cluster-test-net-fw-allow-internal-traffic -khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress -laveeek29-internal-0 -laveeek29-internal-1 -laveeek29-net-0-fw-allow-iap-ingress -laveeek29-net-1-fw-allow-iap-ingress -laveeek29-net-fw-allow-iap-ingress -laveeek29-net-fw-allow-internal-traffic -lavoldchk-net-fw-allow-iap-ingress -lavoldchk-net-fw-allow-internal-traffic -lavrohek29-internal-0 -lavrohek29-net-0-fw-allow-iap-ingress -lustre-06-net-fw-allow-iap-ingress -lustre-06-net-fw-allow-internal-traffic -lustre-test-06-net-fw-allow-iap-ingress -lustre-test-06-net-fw-allow-internal-traffic - gcloud compute firewall-rules delete "hpcimg-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-lustre-test-02-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-lustre-test-02-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "khu-h4d-cluster-test-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "khu-h4d-cluster-test-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-internal-0" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-internal-1" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lavoldchk-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lavoldchk-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lavrohek29-internal-0" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lavrohek29-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lustre-06-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lustre-06-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lustre-test-06-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lustre-test-06-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:11:28 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:12:06 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 43 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -a8a55slurm-nodeset-0 us-central1-a - gcloud compute instances delete "a8a55slurm-nodeset-0" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -hpcimg-net-fw-allow-internal-traffic -hpc-lustre-test-02-net-fw-allow-iap-ingress -hpc-lustre-test-02-net-fw-allow-internal-traffic -khu-h4d-cluster-test-net-fw-allow-iap-ingress -khu-h4d-cluster-test-net-fw-allow-internal-traffic -khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress -laveeek29-internal-0 -laveeek29-internal-1 -laveeek29-net-0-fw-allow-iap-ingress -laveeek29-net-1-fw-allow-iap-ingress -laveeek29-net-fw-allow-iap-ingress -laveeek29-net-fw-allow-internal-traffic -lavoldchk-net-fw-allow-iap-ingress -lavoldchk-net-fw-allow-internal-traffic -lavrohek29-internal-0 -lavrohek29-net-0-fw-allow-iap-ingress -mainek-internal-0 -mainek-internal-1 -mainek-net-0-fw-allow-iap-ingress -mainek-net-1-fw-allow-iap-ingress - gcloud compute firewall-rules delete "hpcimg-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-lustre-test-02-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "hpc-lustre-test-02-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "khu-h4d-cluster-test-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "khu-h4d-cluster-test-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-internal-0" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-internal-1" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "laveeek29-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lavoldchk-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lavoldchk-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lavrohek29-internal-0" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "lavrohek29-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "mainek-internal-0" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "mainek-internal-1" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "mainek-net-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "mainek-net-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:12:15 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:12:45 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 43 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -hpcimg-net-fw-allow-internal-traffic -hpc-lustre-test-02-net-fw-allow-iap-ingress -hpc-lustre-test-02-net-fw-allow-internal-traffic -khu-h4d-cluster-test-net-fw-allow-iap-ingress -khu-h4d-cluster-test-net-fw-allow-internal-traffic -khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress -laveeek29-internal-0 -laveeek29-internal-1 -laveeek29-net-0-fw-allow-iap-ingress -laveeek29-net-1-fw-allow-iap-ingress -laveeek29-net-fw-allow-iap-ingress -laveeek29-net-fw-allow-internal-traffic -lavoldchk-net-fw-allow-iap-ingress -lavoldchk-net-fw-allow-internal-traffic -lavrohek29-internal-0 -lavrohek29-net-0-fw-allow-iap-ingress -mainek-internal-0 -mainek-internal-1 -mainek-net-0-fw-allow-iap-ingress -mainek-net-1-fw-allow-iap-ingress -[EXECUTE] Firewall Rule: Deleting hpcimg-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpcimg-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting hpc-lustre-test-02-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-lustre-test-02-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting hpc-lustre-test-02-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/hpc-lustre-test-02-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting khu-h4d-cluster-test-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/khu-h4d-cluster-test-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting khu-h4d-cluster-test-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/khu-h4d-cluster-test-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/khu-h4d-cluster-test-rdma-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting laveeek29-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-internal-0]. -[EXECUTE] Firewall Rule: Deleting laveeek29-internal-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-internal-1]. -[EXECUTE] Firewall Rule: Deleting laveeek29-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting laveeek29-net-1-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-net-1-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting laveeek29-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting laveeek29-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/laveeek29-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting lavoldchk-net-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lavoldchk-net-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting lavoldchk-net-fw-allow-internal-traffic -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lavoldchk-net-fw-allow-internal-traffic]. -[EXECUTE] Firewall Rule: Deleting lavrohek29-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lavrohek29-internal-0]. -[EXECUTE] Firewall Rule: Deleting lavrohek29-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/lavrohek29-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting mainek-internal-0 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-internal-0]. -[EXECUTE] Firewall Rule: Deleting mainek-internal-1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-internal-1]. -[EXECUTE] Firewall Rule: Deleting mainek-net-0-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-net-0-fw-allow-iap-ingress]. -[EXECUTE] Firewall Rule: Deleting mainek-net-1-fw-allow-iap-ingress -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/firewalls/mainek-net-1-fw-allow-iap-ingress]. ---- Wed Nov 26 03:15:43 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:19:25 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 43 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -gke-a3mega-7b5154-remote-node-0 us-west4-a - gcloud compute instances delete "gke-a3mega-7b5154-remote-node-0" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -gke-a3mega-7b5154-gpunet-0-router us-west4 -gke-a3mega-7b5154-gpunet-1-router us-west4 -gke-a3mega-7b5154-gpunet-2-router us-west4 -gke-a3mega-7b5154-gpunet-3-router us-west4 -gke-a3mega-7b5154-gpunet-4-router us-west4 -gke-a3mega-7b5154-gpunet-5-router us-west4 -gke-a3mega-7b5154-gpunet-6-router us-west4 -gke-a3mega-7b5154-gpunet-7-router us-west4 -gke-a3mega-7b5154-net-router us-west4 - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-0-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-1-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-2-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-3-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-4-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-5-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-6-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-7-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-net-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -The following Firewall Rules are targeted for deletion in this run: -gke-a3mega-7b5154-gpunet-0-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-0-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-1-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-1-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-2-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-2-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-3-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-3-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-4-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-4-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-5-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-5-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-6-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-6-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-7-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-7-fw-allow-internal-traffic -gke-a3mega-7b5154-net-fw-allow-iap-ingress -gke-a3mega-7b5154-net-fw-allow-internal-traffic -mainek-net-fw-allow-iap-ingress -mainek-net-fw-allow-internal-traffic - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-1-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-2-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-2-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-3-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-3-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-4-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-4-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-5-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-5-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-6-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-6-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-7-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-7-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "mainek-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "mainek-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:19:35 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:23:12 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 43 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic ---- Deletion Phase 1: GKE Clusters (Top 20) --- -The following GKE clusters are targeted for deletion in this run: -gke-a3mega-7b5154 us-west4 - gcloud container clusters delete "gke-a3mega-7b5154" --project="hpc-toolkit-dev" --location="us-west4" ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -d87d8bslur-controller us-central1-a -d87d8bslur-slurm-login-001 us-central1-a -gke-a3mega-7b5154-remote-node-0 us-west4-a -gke-gke-a3mega-7b5154-default-pool-92973a4c-zdc6 us-west4-a -gke-gke-a3mega-7b5154-default-pool-b8fa115c-f11p us-west4-b -gke-gke-a3mega-7b5154-default-pool-cab11a55-856w us-west4-c - gcloud compute instances delete "d87d8bslur-controller" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "d87d8bslur-slurm-login-001" --project="hpc-toolkit-dev" --zone="us-central1-a" --quiet - gcloud compute instances delete "gke-a3mega-7b5154-remote-node-0" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet - gcloud compute instances delete "gke-gke-a3mega-7b5154-default-pool-92973a4c-zdc6" --project="hpc-toolkit-dev" --zone="us-west4-a" --quiet - gcloud compute instances delete "gke-gke-a3mega-7b5154-default-pool-b8fa115c-f11p" --project="hpc-toolkit-dev" --zone="us-west4-b" --quiet - gcloud compute instances delete "gke-gke-a3mega-7b5154-default-pool-cab11a55-856w" --project="hpc-toolkit-dev" --zone="us-west4-c" --quiet ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -gke-a3mega-7b5154-gpunet-0-router us-west4 -gke-a3mega-7b5154-gpunet-1-router us-west4 -gke-a3mega-7b5154-gpunet-2-router us-west4 -gke-a3mega-7b5154-gpunet-3-router us-west4 -gke-a3mega-7b5154-gpunet-4-router us-west4 -gke-a3mega-7b5154-gpunet-5-router us-west4 -gke-a3mega-7b5154-gpunet-6-router us-west4 -gke-a3mega-7b5154-gpunet-7-router us-west4 -gke-a3mega-7b5154-net-router us-west4 - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-0-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-1-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-2-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-3-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-4-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-5-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-6-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-gpunet-7-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "gke-a3mega-7b5154-net-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet ---- Deletion Phase 3: Firewall Rules (Top 20) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -The following Firewall Rules are targeted for deletion in this run: -gke-a3mega-7b5154-gpunet-0-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-0-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-1-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-1-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-2-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-2-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-3-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-3-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-4-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-4-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-5-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-5-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-6-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-6-fw-allow-internal-traffic -gke-a3mega-7b5154-gpunet-7-fw-allow-iap-ingress -gke-a3mega-7b5154-gpunet-7-fw-allow-internal-traffic -gke-a3mega-7b5154-net-fw-allow-iap-ingress -gke-a3mega-7b5154-net-fw-allow-internal-traffic -gke-gke-a3mega-7b5154-2358afec-all -gke-gke-a3mega-7b5154-2358afec-exkubelet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-0-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-0-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-1-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-1-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-2-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-2-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-3-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-3-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-4-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-4-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-5-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-5-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-6-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-6-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-7-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-gpunet-7-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-net-fw-allow-iap-ingress" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-a3mega-7b5154-net-fw-allow-internal-traffic" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-gke-a3mega-7b5154-2358afec-all" --project="hpc-toolkit-dev" --quiet - gcloud compute firewall-rules delete "gke-gke-a3mega-7b5154-2358afec-exkubelet" --project="hpc-toolkit-dev" --quiet ---- Wed Nov 26 03:23:21 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 03:23:41 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 43 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic ---- Deletion Phase 1: GKE Clusters (Top 20) --- -The following GKE clusters are targeted for deletion in this run: -gke-a3mega-7b5154 us-west4 -[EXECUTE] GKE Cluster: Deleting gke-a3mega-7b5154 in us-west4 -The following clusters will be deleted. - - [gke-a3mega-7b5154] in [us-west4] - -Do you want to continue (Y/n)? -ERROR: (gcloud.container.clusters.delete) This prompt could not be answered because you are not in an interactive session. You can re-run the command with the --quiet flag to accept default answers for all prompts. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -The following Instances are targeted for deletion in this run: -d87d8bslur-controller us-central1-a -d87d8bslur-slurm-login-001 us-central1-a -gke-a3mega-7b5154-remote-node-0 us-west4-a -gke-gke-a3mega-7b5154-default-pool-92973a4c-zdc6 us-west4-a -gke-gke-a3mega-7b5154-default-pool-b8fa115c-f11p us-west4-b -gke-gke-a3mega-7b5154-default-pool-cab11a55-856w us-west4-c -[EXECUTE] Instance: Deleting d87d8bslur-controller in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d87d8bslur-controller]. -[EXECUTE] Instance: Deleting d87d8bslur-slurm-login-001 in us-central1-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a/instances/d87d8bslur-slurm-login-001]. -[EXECUTE] Instance: Deleting gke-a3mega-7b5154-remote-node-0 in us-west4-a -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west4-a/instances/gke-a3mega-7b5154-remote-node-0]. -[EXECUTE] Instance: Deleting gke-gke-a3mega-7b5154-default-pool-92973a4c-zdc6 in us-west4-a -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-west4-a/instances/gke-gke-a3mega-7b5154-default-pool-92973a4c-zdc6' was not found - -[EXECUTE] Instance: Deleting gke-gke-a3mega-7b5154-default-pool-b8fa115c-f11p in us-west4-b -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-west4-b/instances/gke-gke-a3mega-7b5154-default-pool-b8fa115c-f11p' was not found - -[EXECUTE] Instance: Deleting gke-gke-a3mega-7b5154-default-pool-cab11a55-856w in us-west4-c -ERROR: (gcloud.compute.instances.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/zones/us-west4-c/instances/gke-gke-a3mega-7b5154-default-pool-cab11a55-856w' was not found - -./cleanup.sh: line 134: syntax error near unexpected token `(' -./cleanup.sh: line 134: `echo "--- Deletion Phase 1: Filestore Instances (Top $DELETE_LIMIT) ---"' ---- Thu Nov 27 09:50:25 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T05:50:25+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -lustre-06-primary-subnet in us-central1 -lustre-test-06-primary-subnet in us-central1 ---- Processing Subnet: lustre-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-06-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-test-06-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -The following Networks are targeted for deletion in this run: -a3mega-sys-net-shu7 -a3u-onspot-slurm-829610-net-0 -a3u-slurm-net -a4hsarthakag-net -a4htest-net-0 -a4newimgek-net -a4newimgek-net-0 -a4newimgek-net-1 -a4newimgek-rdma-net -a4oldimgek-net -a4oldimgek-net-0 -a4oldimgek-net-1 -a4oldimgek-rdma-net -a4oldimg-net -a4xlavhpcnew-a4x-net-0 -a4xlavhpcnew-a4x-net-1 -a4xlavhpcnew-a4x-rdma-net -a4xslurm-net -cx-a3u-net-0 -db451c7-ml-slurm-v6-net -dynpoc-net -g4-dwsq-1-net-1 -g4qclav-net -gke-1395b4-net -gke-managed-lustre-basic-net -h4d-cluster-rdma-net-0 -h4dqc-net -h4dqc-rdma-net-0 -h4d-res-swarnabm4-3-net -h4d-res-swarnabm4-3-rdma-net -[DRY RUN] Network: Would delete a3mega-sys-net-shu7 - Command: gcloud compute networks delete "a3mega-sys-net-shu7" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a3u-onspot-slurm-829610-net-0 - Command: gcloud compute networks delete "a3u-onspot-slurm-829610-net-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a3u-slurm-net - Command: gcloud compute networks delete "a3u-slurm-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4hsarthakag-net - Command: gcloud compute networks delete "a4hsarthakag-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4htest-net-0 - Command: gcloud compute networks delete "a4htest-net-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4newimgek-net - Command: gcloud compute networks delete "a4newimgek-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4newimgek-net-0 - Command: gcloud compute networks delete "a4newimgek-net-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4newimgek-net-1 - Command: gcloud compute networks delete "a4newimgek-net-1" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4newimgek-rdma-net - Command: gcloud compute networks delete "a4newimgek-rdma-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4oldimgek-net - Command: gcloud compute networks delete "a4oldimgek-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4oldimgek-net-0 - Command: gcloud compute networks delete "a4oldimgek-net-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4oldimgek-net-1 - Command: gcloud compute networks delete "a4oldimgek-net-1" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4oldimgek-rdma-net - Command: gcloud compute networks delete "a4oldimgek-rdma-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4oldimg-net - Command: gcloud compute networks delete "a4oldimg-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4xlavhpcnew-a4x-net-0 - Command: gcloud compute networks delete "a4xlavhpcnew-a4x-net-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4xlavhpcnew-a4x-net-1 - Command: gcloud compute networks delete "a4xlavhpcnew-a4x-net-1" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4xlavhpcnew-a4x-rdma-net - Command: gcloud compute networks delete "a4xlavhpcnew-a4x-rdma-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete a4xslurm-net - Command: gcloud compute networks delete "a4xslurm-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete cx-a3u-net-0 - Command: gcloud compute networks delete "cx-a3u-net-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete db451c7-ml-slurm-v6-net - Command: gcloud compute networks delete "db451c7-ml-slurm-v6-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete dynpoc-net - Command: gcloud compute networks delete "dynpoc-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete g4-dwsq-1-net-1 - Command: gcloud compute networks delete "g4-dwsq-1-net-1" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete g4qclav-net - Command: gcloud compute networks delete "g4qclav-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete gke-1395b4-net - Command: gcloud compute networks delete "gke-1395b4-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete gke-managed-lustre-basic-net - Command: gcloud compute networks delete "gke-managed-lustre-basic-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete h4d-cluster-rdma-net-0 - Command: gcloud compute networks delete "h4d-cluster-rdma-net-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete h4dqc-net - Command: gcloud compute networks delete "h4dqc-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete h4dqc-rdma-net-0 - Command: gcloud compute networks delete "h4dqc-rdma-net-0" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete h4d-res-swarnabm4-3-net - Command: gcloud compute networks delete "h4d-res-swarnabm4-3-net" --project="hpc-toolkit-dev" --quiet -[DRY RUN] Network: Would delete h4d-res-swarnabm4-3-rdma-net - Command: gcloud compute networks delete "h4d-res-swarnabm4-3-rdma-net" --project="hpc-toolkit-dev" --quiet ---- Thu Nov 27 09:50:44 AM UTC 2025 --- Cleanup Script Run Finished --- - diff --git a/peer.txt b/peer.txt deleted file mode 100644 index 6ae514f4e8..0000000000 --- a/peer.txt +++ /dev/null @@ -1,566 +0,0 @@ -[2025-11-30 17:48:42] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 17:48:42] [INFO] Time Cutoff (General): 2025-11-30T17:48:42+0000 -[2025-11-30 17:48:42] [INFO] Time Cutoff (Images): 2025-10-01T17:48:42+0000 -[2025-11-30 17:48:42] [INFO] Delete Limit per Type: 200 -[2025-11-30 17:48:42] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 17:48:43] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 17:48:45] [INFO] No Service Accounts found matching prefix. -[2025-11-30 17:48:45] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 17:48:47] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 17:48:47] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 17:48:49] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 17:48:49] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 17:48:49] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 17:48:49] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 17:48:52] [INFO] No Filestore instances found matching criteria. -[2025-11-30 17:48:52] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 17:48:55] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 17:48:55] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 17:48:55] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 17:48:55] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 17:48:55] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 17:48:56] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 17:48:56] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 17:48:56] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 17:48:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 17:48:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 17:48:56] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 17:48:56] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T17:48:56Z (Unix: 1763315336) -[2025-11-30 17:48:56] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:58] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:59] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763315336]) -[2025-11-30 17:48:59] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 17:48:59] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 17:49:01] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 17:49:01] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 17:49:01] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 17:49:01] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 17:49:01] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 17:49:01] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 17:49:03] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 17:49:03] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 17:49:05] [INFO] No Regional Address found matching criteria. -[2025-11-30 17:49:05] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 17:49:08] [INFO] No Global Address found matching criteria. -[2025-11-30 17:49:08] [INFO] --- Processing: VPC Peerings (Limit: 200) --- -[2025-11-30 17:49:12] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:12] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 17:49:15] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 17:49:15] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 17:49:15] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 17:49:15] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 17:49:15] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:17] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:18] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 17:49:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:49:20] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 17:49:22] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 17:49:22] [INFO] CLEANUP RUN FINISHED -./cleanup.sh: line 636: syntax error near unexpected token `in' -./cleanup.sh: line 636: `in' -[2025-11-30 17:58:51] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 17:58:51] [INFO] Time Cutoff (General): 2025-11-30T17:58:51+0000 -[2025-11-30 17:58:51] [INFO] Time Cutoff (Images): 2025-10-01T17:58:51+0000 -[2025-11-30 17:58:51] [INFO] Delete Limit per Type: 200 -[2025-11-30 17:58:51] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 17:58:52] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 17:58:54] [INFO] No Service Accounts found matching prefix. -[2025-11-30 17:58:54] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 17:58:56] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 17:58:56] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 17:58:58] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 17:58:58] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 17:58:58] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 17:58:58] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 17:59:01] [INFO] No Filestore instances found matching criteria. -[2025-11-30 17:59:01] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 17:59:04] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 17:59:04] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 17:59:04] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 17:59:04] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 17:59:04] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 17:59:04] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 17:59:04] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 17:59:04] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 17:59:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 17:59:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 17:59:05] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 17:59:05] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T17:59:05Z (Unix: 1763315945) -[2025-11-30 17:59:05] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763315945]) -[2025-11-30 17:59:07] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 17:59:07] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 17:59:10] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 17:59:10] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 17:59:10] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 17:59:10] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 17:59:10] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 17:59:10] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 17:59:12] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 17:59:12] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 17:59:15] [INFO] No Regional Address found matching criteria. -[2025-11-30 17:59:15] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 17:59:17] [INFO] No Global Address found matching criteria. -[2025-11-30 17:59:17] [INFO] --- Processing: VPC Peerings (Limit: 200) --- -[2025-11-30 17:59:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:24] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 17:59:26] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 17:59:26] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 17:59:26] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 17:59:26] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 17:59:26] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:29] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 17:59:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 17:59:31] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 17:59:33] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 17:59:33] [INFO] CLEANUP RUN FINISHED -[2025-11-30 18:05:23] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:05:23] [INFO] Time Cutoff (General): 2025-11-30T18:05:23+0000 -[2025-11-30 18:05:23] [INFO] Time Cutoff (Images): 2025-10-01T18:05:23+0000 -[2025-11-30 18:05:23] [INFO] Delete Limit per Type: 200 -[2025-11-30 18:05:23] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:05:23] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:05:26] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:05:26] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:05:27] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:05:27] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 18:05:30] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:05:30] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:05:30] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:05:30] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:05:33] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:05:33] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 18:05:35] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:05:36] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:05:36] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:05:36] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:05:36] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:05:36] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:05:36] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:05:36] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:05:36] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:05:36] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:05:36] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 18:05:36] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:05:36Z (Unix: 1763316336) -[2025-11-30 18:05:36] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763316336]) -[2025-11-30 18:05:39] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:05:39] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 18:05:41] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:05:41] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:05:41] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:05:41] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:05:41] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:05:41] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 18:05:44] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:05:44] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:05:46] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:05:46] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:05:48] [INFO] No Global Address found matching criteria. -[2025-11-30 18:05:48] [INFO] --- Processing: VPC Peerings (Limit: 200) --- -[2025-11-30 18:05:55] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:05:55] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 18:05:57] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:05:57] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:05:57] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:05:57] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:05:57] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:00] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 18:06:02] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:06:02] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 18:06:04] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:06:04] [INFO] CLEANUP RUN FINISHED -[2025-11-30 18:06:34] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:06:34] [INFO] Time Cutoff (General): 2025-11-30T18:06:34+0000 -[2025-11-30 18:06:34] [INFO] Time Cutoff (Images): 2025-10-01T18:06:34+0000 -[2025-11-30 18:06:34] [INFO] Delete Limit per Type: 200 -[2025-11-30 18:06:34] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:06:35] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:06:37] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:06:37] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:06:39] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:06:39] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 18:06:42] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:06:42] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:06:42] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:06:42] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:06:44] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:06:44] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 18:06:47] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:06:47] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:06:47] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:06:47] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:06:47] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:06:48] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:06:48] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:06:48] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:06:48] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:06:48] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:06:48] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 18:06:48] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:06:48Z (Unix: 1763316408) -[2025-11-30 18:06:48] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:50] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:51] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763316408]) -[2025-11-30 18:06:51] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:06:51] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 18:06:53] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:06:53] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:06:53] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:06:53] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:06:53] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:06:53] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 18:06:55] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:06:55] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:06:58] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:06:58] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:07:00] [INFO] No Global Address found matching criteria. -[2025-11-30 18:07:00] [INFO] --- Processing: VPC Peerings (Limit: 200) --- -[2025-11-30 18:07:02] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:02] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:07:02] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 18:07:05] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:07:05] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:07:05] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:07:05] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:07:05] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 18:07:07] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:07] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:07] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:08] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 18:07:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:07:10] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 18:07:12] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:07:12] [INFO] CLEANUP RUN FINISHED -./cleanup.sh: line 235: syntax error near unexpected token `done' -./cleanup.sh: line 235: ` done' -./cleanup.sh: line 235: syntax error near unexpected token `done' -./cleanup.sh: line 235: ` done' -./cleanup.sh: line 235: syntax error near unexpected token `done' -./cleanup.sh: line 235: ` done' -./cleanup.sh: line 235: syntax error near unexpected token `done' -./cleanup.sh: line 235: ` done' -[2025-11-30 18:29:28] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:29:28] [INFO] Time Cutoff (General): 2025-11-30T18:29:28+0000 -[2025-11-30 18:29:28] [INFO] Time Cutoff (Images): 2025-10-01T18:29:28+0000 -[2025-11-30 18:29:28] [INFO] Delete Limit per Type: 200 -[2025-11-30 18:29:28] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:29:28] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:29:31] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:29:31] [INFO] --- Processing: GKE Cluster (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:29:33] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:29:33] [INFO] --- Processing: Compute Instance (Limit: 200) --- -[2025-11-30 18:29:36] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:29:36] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:29:36] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:29:36] [INFO] --- Processing: Filestore Instances (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:29:39] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:29:39] [INFO] --- Processing: VM Images (Limit: 200) --- -[2025-11-30 18:29:42] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:29:43] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:29:43] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:29:43] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:29:43] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:29:43] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:29:43] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:29:43] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:29:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:29:43] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:29:43] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 200) --- -[2025-11-30 18:29:43] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:29:43Z (Unix: 1763317783) -[2025-11-30 18:29:43] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1fc1e00175b3700ed5d99c0f2dcc29f247ad5fe2a077710784c22937c187a719 (Updated: 2025-11-17T08:20:06 [TS: 1763367606] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:653b88835ab33bb89001d38d4695716c5018396a9c1e0c502d5d4e06338e3184 (Updated: 2025-11-17T08:20:17 [TS: 1763367617] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c13171c30dc1aa3d6ba3c34867fff6d39150e3fbd6b137c790fa551d372c3522 (Updated: 2025-11-18T08:20:47 [TS: 1763454047] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6236258042a997cc8e02e2f083051a54fb35ad3ff2abaedabbce6b423ffdde93 (Updated: 2025-11-18T08:20:55 [TS: 1763454055] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4c613ee2b8ed7ffae384bc4b2fda4ee21088403307fe51e8b0ac955e7a89328d (Updated: 2025-11-18T18:49:58 [TS: 1763491798] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:f24fa3856c03c6b6544d930fbbcc43ad357d9f138d1286f0675188aa0dec0f77 (Updated: 2025-11-18T18:50:13 [TS: 1763491813] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:b3a646a9fad927984980aef685aa581a60d5dc71c8c59bd8facada59ab77eed4 (Updated: 2025-11-19T18:51:39 [TS: 1763578299] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4af5db61700b8193a5a66f43de34b556d8c9f5863e980f6dae209e81e6aa17d5 (Updated: 2025-11-19T18:51:45 [TS: 1763578305] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:212b05a0a1c98b2d4563fb1d98bad05752b8c93aa2f1bdb5ac0f79f3070d4cf8 (Updated: 2025-11-20T18:49:17 [TS: 1763664557] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:55460dca917fe8dddcf0cfbfdd12807b9cecd829b30709d4cba8c60586885c73 (Updated: 2025-11-20T18:49:24 [TS: 1763664564] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2e61d182ab84124fac9fe2e5dcb0fd9be383cb66bd3d2a277cb8c1591f381790 (Updated: 2025-11-22T08:20:43 [TS: 1763799643] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6f2e2a759e9f543f6b3a177d3e00326f1050bd7ba7a08d1e61b3b3e50a9fa175 (Updated: 2025-11-22T08:20:52 [TS: 1763799652] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e5fa39311fc457f4efcb60de5ceb650c822ae6a42e6dd12f758dc84d3f9e699 (Updated: 2025-11-23T08:17:47 [TS: 1763885867] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:1efa59c424c2dacdf48f28745bd942bfcef4625cfc7dc254748bdc5cbb5fc222 (Updated: 2025-11-23T08:17:54 [TS: 1763885874] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:35ec3b3c50826e42ba2de89ff70e4665b4ace4636180092863221132af98dbc7 (Updated: 2025-11-24T08:21:56 [TS: 1763972516] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:eba09b99da72473216349995f209a523b8afd0f6b9267ef7733c4439d8c17ad2 (Updated: 2025-11-24T08:22:03 [TS: 1763972523] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:c4040d6826710ffbe9fb83a55acda55c023feead80e477f0243ee3020fd290e6 (Updated: 2025-11-24T18:50:51 [TS: 1764010251] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:00bf2c87e858b285f2623e0adc51bf6770989112457fee5c07f8b102bcdcea2b (Updated: 2025-11-24T18:50:57 [TS: 1764010257] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:e79b8ff506e79f05a06c60b882b9718164ef4aa1ea72faffb50fb3db34c0217f (Updated: 2025-11-25T18:51:48 [TS: 1764096708] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:bef4aa2caca0a52bf1e2a7ba6c33a1d66e7524f20b6ac731e2ebb7eec013e47f (Updated: 2025-11-25T18:51:54 [TS: 1764096714] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:5e1a2f8e6f92ca443b0eb2252ffc0ed863dde4835046c5e8a4f435a9067530f1 (Updated: 2025-11-26T18:47:53 [TS: 1764182873] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:242c018d4024df0ff4273df37ae9d097e84b9dd633632655973d7224b2fc9db0 (Updated: 2025-11-26T18:47:59 [TS: 1764182879] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:63838c0a300bb40209deb226acfc4132381b32610de89cfa3705b7efd5c1b393 (Updated: 2025-11-27T18:50:46 [TS: 1764269446] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:657b36041ee460dd7275ec8b63e90965a82b14f5691147ef7fd43a90256b6f63 (Updated: 2025-11-27T18:50:53 [TS: 1764269453] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:4f84e97c1a57fce13fa7892cb453168b59c696c6b0fca8954f7ac3dba7a9faf5 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:6b33f72b4aa26059e5283a5951a3942da2f4d316ff5b0a7ffc62c9221fcff118 (Updated: 2025-11-28T18:47:55 [TS: 1764355675] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] [KEEP] us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner@sha256:2dadc2e85ec041d14dda5a32a5693622f855e326ac2ac4baa3abef86f809c3e1 (Updated: 2025-11-28T18:48:01 [TS: 1764355681] >= Cutoff: [TS: 1763317783]) -[2025-11-30 18:29:46] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:29:46] [INFO] --- Processing: Cloud Router (Limit: 200) --- -[2025-11-30 18:29:48] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:29:48] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:29:48] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:29:48] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:29:48] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:29:48] [INFO] --- Processing: Firewall Rules (Limit: 200) --- -[2025-11-30 18:29:51] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:29:51] [INFO] --- Processing: Regional Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:29:53] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:29:53] [INFO] --- Processing: Global Address (Limit: 200) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:29:56] [INFO] No Global Address found matching criteria. -[2025-11-30 18:29:56] [INFO] --- Processing: VPC Peerings (Limit: 200) --- -[2025-11-30 18:29:58] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:29:58] [INFO] --- Processing: Zonal Disk (Limit: 200) --- -[2025-11-30 18:30:01] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:30:01] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:30:01] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:30:01] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:30:01] [INFO] --- Processing: Subnetworks (Limit: 200) --- -[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:03] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:04] [INFO] --- Processing: VPC Networks (Limit: 200) --- -[2025-11-30 18:30:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:30:06] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 200) --- -[2025-11-30 18:30:08] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:30:08] [INFO] CLEANUP RUN FINISHED diff --git a/policy-bindings.txt b/policy-bindings.txt deleted file mode 100644 index 1d98b7f105..0000000000 --- a/policy-bindings.txt +++ /dev/null @@ -1,2834 +0,0 @@ ---- Wed Nov 26 04:43:43 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 7: Clean up IAM Policy Bindings for Deleted Service Accounts --- -IAM Binding Cleanup: Total members before: 820 -Found the following deleted service account members in IAM policy: -deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 -deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 -deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 -deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 -deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 -deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 -deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 -deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 -deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 -deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 -deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 -deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 -deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 -deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 -deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 -deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 -deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 -deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 -deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 -deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 -deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 -deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 -deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 -deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 -deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 -deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 -deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 -deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 -deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 -deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 -deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 -deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 -deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 -deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 -deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 -deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 -deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 -deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 -deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 -deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 -deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 -deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 -deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 -deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 -deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 -deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 -deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 -deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 -deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 -deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 -deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 -deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 -deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 -deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 -deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 -deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 -deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 -deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 -deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 -deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 -deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 -deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 -deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 -deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 -deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 -deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 -deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 -deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 -deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 -deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 -deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 -deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 -deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 -deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 -deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 -deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 -deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 ---- Processing member: deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/container.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/container.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/container.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/storage.objectCreator - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009" --role="roles/storage.objectCreator" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/compute.instanceAdmin.v1 - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/compute.instanceAdmin.v1" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/compute.instanceAdmin.v1. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/iam.serviceAccountUser - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/iam.serviceAccountUser" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/iam.serviceAccountUser. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/pubsub.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/pubsub.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/pubsub.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/container.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/container.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/container.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/container.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/container.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/container.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/container.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/container.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/container.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/container.admin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/container.admin" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/container.admin. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/storage.objectViewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808" --role="roles/storage.objectViewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 --- -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/artifactregistry.reader - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/artifactregistry.reader" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/artifactregistry.reader. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/logging.logWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/logging.logWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/logging.logWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.metricWriter - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/monitoring.metricWriter" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.metricWriter. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.viewer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/monitoring.viewer" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.viewer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/stackdriver.resourceMetadata.writer - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/stackdriver.resourceMetadata.writer" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/stackdriver.resourceMetadata.writer. -[DRY RUN] IAM Binding: Would remove deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/storage.objectAdmin - gcloud projects remove-iam-policy-binding "hpc-toolkit-dev" --member="deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067" --role="roles/storage.objectAdmin" --all --quiet - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/storage.objectAdmin. -IAM Binding Cleanup: Total members after: 820 ---- Wed Nov 26 04:43:48 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 04:44:21 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 7: Clean up IAM Policy Bindings for Deleted Service Accounts --- -IAM Binding Cleanup: Total members before: 820 -Found the following deleted service account members in IAM policy: -deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 -deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 -deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 -deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 -deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 -deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 -deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 -deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 -deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 -deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 -deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 -deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 -deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 -deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 -deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 -deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 -deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 -deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 -deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 -deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 -deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 -deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 -deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 -deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 -deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 -deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 -deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 -deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 -deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 -deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 -deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 -deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 -deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 -deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 -deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 -deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 -deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 -deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 -deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 -deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 -deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 -deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 -deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 -deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 -deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 -deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 -deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 -deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 -deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 -deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 -deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 -deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 -deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 -deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 -deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 -deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 -deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 -deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 -deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 -deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 -deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 -deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 -deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 -deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 -deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 -deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 -deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 -deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 -deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 -deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 -deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 -deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 -deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 -deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 -deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 -deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 -deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 ---- Processing member: deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101475817502629942845 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/container.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/container.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:ag-gke-tpu-v6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104159397688709546678 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106341295562021316043 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108360312193154854405 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1b416d-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106615711181208052484 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114358636371718475632 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104103656763101151153 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-1d8822-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106502772439408519928 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106103725899010944853 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107371174339107562345 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-29d66e-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102073203262695871191 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117767132285195942251 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101670033252095088906 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-320d31-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104698970246978916120 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102221563581992422063 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107526635410240741765 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-3f8b97-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112409625151054255208 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101429231762837102213 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100971980399386178257 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-638341-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=110913990482047299747 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100756140101120174579 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102387572362294792573 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-653e01-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108342646872433471146 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114518834845461283939 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102018915464650020045 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-68dd2c-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104741559576432750765 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109452268727402667101 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=118354828433242040261 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-793e73-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101553699724587843943 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106122334748930688022 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105499114147468619716 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-90d00b-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106585139286205369312 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115016087037729491264 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108205636504239969878 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-a62d22-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106333934056253553879 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104601095433300061620 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106739176685305973973 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-d29807-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112214137158866295217 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117810919737040565864 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111227089311057847581 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-ed7d77-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=100888377455606545773 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/storage.objectCreator -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-compute@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112212556803145894009 from roles/storage.objectCreator. ---- Processing member: deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/compute.instanceAdmin.v1 -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/compute.instanceAdmin.v1. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/iam.serviceAccountUser -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/iam.serviceAccountUser. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/pubsub.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/pubsub.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-controller@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101996054312533149996 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:enter-f9adcc-login@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115503881361892222247 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104714705409816147169 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-dev-3-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104801100712118491366 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112123930486182374840 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-e5aef0-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106566036189738414692 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102412800971401241781 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104361086793238433523 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=111091130132207729332 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=117139256165531925962 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=104163988185629957590 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105155249984798026063 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=105239310415191486584 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107941544239948516878 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3high-prod-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109970974749371929426 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/container.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/container.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a3ultra-6e798b-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=113617082448798426329 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=102358440604393847231 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/container.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/container.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4-f91aaa-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106564737601905388996 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=108284320142999666288 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-a4x-parul1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114875394419607890044 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116096002584712461433 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:gke-h4d-4e3b59-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=106104532889376799992 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112292323113102660461 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-002-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107591637470483231608 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=114206326534246072595 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-003-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107140004877531624323 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103996282248867400888 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:h4d-swarnabm-d2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=101414939641097332714 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116114759572229326541 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/container.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/container.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:kh-tpu-7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=109211865400099783499 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=116849799394507390017 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/container.admin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/container.admin. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khuag-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=103850156696668208947 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:khush-tpu-v7x-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=112929959525907030655 from roles/storage.objectAdmin. ---- Processing member: deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/storage.objectViewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=115003836717767651808 from roles/storage.objectViewer. ---- Processing member: deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 --- -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/artifactregistry.reader -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/artifactregistry.reader. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/logging.logWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/logging.logWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.metricWriter -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.metricWriter. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.viewer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/monitoring.viewer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/stackdriver.resourceMetadata.writer -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/stackdriver.resourceMetadata.writer. -[EXECUTE] IAM Binding: Removing deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/storage.objectAdmin -Updated IAM policy for project [hpc-toolkit-dev]. - Successfully removed binding for deleted:serviceAccount:poornima-g4-conn-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com?uid=107128100965143257067 from roles/storage.objectAdmin. -IAM Binding Cleanup: Total members after: 445 ---- Wed Nov 26 05:01:01 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 05:15:36 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 7: Clean up IAM Policy Bindings for Deleted Service Accounts --- -IAM Binding Cleanup: Total members before: 432 -IAM Binding Cleanup: No deleted service accounts found in IAM policy. -IAM Binding Cleanup: Total members after: 432 ---- Wed Nov 26 05:15:40 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 05:15:59 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 7: Clean up IAM Policy Bindings for Deleted Service Accounts --- -IAM Binding Cleanup: Total members before: 432 -IAM Binding Cleanup: No deleted service accounts found in IAM policy. -IAM Binding Cleanup: Total members after: 432 ---- Wed Nov 26 05:16:03 PM UTC 2025 --- Cleanup Script Run Finished --- diff --git a/rdisk.txt b/rdisk.txt deleted file mode 100644 index 8107f7ef42..0000000000 --- a/rdisk.txt +++ /dev/null @@ -1,373 +0,0 @@ ---- Thu Nov 27 02:16:12 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T10:16:12+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Traceback (most recent call last): - File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 193, in - main() - File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 187, in main - gcloud_main = _import_gcloud_main() - ^^^^^^^^^^^^^^^^^^^^^ - File "/usr/bin/../lib/google-cloud-sdk/lib/gcloud.py", line 90, in _import_gcloud_main - import googlecloudsdk.gcloud_main - File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/gcloud_main.py", line 42, in - from googlecloudsdk.core.credentials import creds_context_managers - File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/core/credentials/creds_context_managers.py", line 29, in - from googlecloudsdk.core.credentials import store - File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/core/credentials/store.py", line 34, in - from googlecloudsdk.api_lib.auth import external_account as auth_external_account - File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/api_lib/auth/external_account.py", line 24, in - from googlecloudsdk.core.credentials import creds as c_creds - File "/usr/bin/../lib/google-cloud-sdk/lib/googlecloudsdk/core/credentials/creds.py", line 33, in - from google.auth import compute_engine as google_auth_compute_engine - File "/usr/bin/../lib/google-cloud-sdk/lib/third_party/google/auth/compute_engine/__init__.py", line 18, in - from google.auth.compute_engine.credentials import Credentials - File "", line 1360, in _find_and_load - File "", line 1331, in _find_and_load_unlocked - File "", line 935, in _load_unlocked - File "", line 995, in exec_module - File "", line 1091, in get_code - File "", line 1190, in get_data -KeyboardInterrupt ---- Thu Nov 27 02:16:41 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T10:16:41+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-dev-06-net -lustre-prod-06-net -lustre-qa-05-net ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-dev-06-net - Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-prod-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net -[DRY RUN] Route: Would delete peering-route-eebb81463c1f952f for network lustre-prod-06-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-prod-06-net - Command: gcloud compute networks delete "lustre-prod-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: vertexui-do-not-kill-boot in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) -Skip Zonal Disk: vertexui-do-not-kill-data in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) -No Zonal Disks to delete in this run. ---- Deletion Phase 4b: Regional Persistent Disks (Top 30) --- -WARNING: The following filter keys were not present in any resource : region -No Regional Disks found matching criteria or list command failed. ---- Thu Nov 27 02:17:13 PM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 02:18:53 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T13:18:53+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 47 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - default - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -Skip Filestore Instance: a4h-slurm-c1a329-8ab4ad47 (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-3148a0ff (Location not found in list output) -Skip Filestore Instance: lustre-dev-06-be0d3c5c (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-5b1cfd08 (Location not found in list output) -Skip Filestore Instance: lustre-prod-06-90dc8167 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-b7382e84 (Location not found in list output) -Skip Filestore Instance: lustre-qa-05-c0248923 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -No Subnetworks found to delete in this run after filtering. ---- Subnetwork Deletion Phase Complete --- ---- Deletion Phase 6: Networks (Top 30) --- -Skip Network: default (Is default) -Skip Network: hpc-vpc (In exclusion list) -The following Networks and their dependencies are targeted for deletion in this run: -lustre-dev-06-net -lustre-prod-06-net -lustre-qa-05-net ---- Processing Network: lustre-dev-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b99c802ac7b2e10 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-7869e60dfba46542 for network lustre-dev-06-net -[DRY RUN] Route: Would delete peering-route-87752b9a8f2ebae2 for network lustre-dev-06-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-dev-06-net - Command: gcloud compute networks delete "lustre-dev-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-prod-06-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-0a55a53b5c4fdb82 for network lustre-prod-06-net -[DRY RUN] Route: Would delete peering-route-eebb81463c1f952f for network lustre-prod-06-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-prod-06-net - Command: gcloud compute networks delete "lustre-prod-06-net" --project="hpc-toolkit-dev" --quiet ---- Processing Network: lustre-qa-05-net --- -Checking for dependent routes... -[DRY RUN] Route: Would delete peering-route-3b91a4552351d170 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-6f7c1d8537c80540 for network lustre-qa-05-net -[DRY RUN] Route: Would delete peering-route-c2ff29e0ce578be2 for network lustre-qa-05-net -Checking for dependent firewall rules... -No dependent firewall rules found. -[DRY RUN] Network: Would delete lustre-qa-05-net - Command: gcloud compute networks delete "lustre-qa-05-net" --project="hpc-toolkit-dev" --quiet ---- Network Deletion Process Complete --- ---- Deletion Phase 4a: Zonal Persistent Disks (Top 30) --- -Skip Zonal Disk: image-inspector-550 in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: image-inspector in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-a (In exclusion list) -Skip Zonal Disk: vertexui-do-not-kill-boot in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) -Skip Zonal Disk: vertexui-do-not-kill-data in https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-west1-b (In exclusion list) -No Zonal Disks to delete in this run. ---- Deletion Phase 4b: Regional Persistent Disks (Top 30) --- -WARNING: The following filter keys were not present in any resource : region -No Regional Disks found matching criteria or list command failed. ---- Thu Nov 27 02:19:25 PM UTC 2025 --- Cleanup Script Run Finished --- - diff --git a/routers.txt b/routers.txt deleted file mode 100644 index 74b7af4e48..0000000000 --- a/routers.txt +++ /dev/null @@ -1,1350 +0,0 @@ ---- Wed Nov 26 01:51:04 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 10) --- -The following Cloud Routers are targeted for deletion in this run: -a3mega-sys-net-shu7-router us-west4 -a3u-slurm-net-router us-south1 -a4hsarthakag-net-router us-central1 -a4htest-net-0-router us-central1 -a4newimgek-net-0-router us-south1 -a4newimgek-net-1-router us-south1 -a4newimgek-net-router europe-west4 -a4oldimgek-net-0-router europe-west4 -a4oldimgek-net-1-router europe-west4 -a4oldimgek-net-router europe-west4 - gcloud compute routers delete "a3mega-sys-net-shu7-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "a3u-slurm-net-router" --project="hpc-toolkit-dev" --region="us-south1" --quiet - gcloud compute routers delete "a4hsarthakag-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "a4htest-net-0-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "a4newimgek-net-0-router" --project="hpc-toolkit-dev" --region="us-south1" --quiet - gcloud compute routers delete "a4newimgek-net-1-router" --project="hpc-toolkit-dev" --region="us-south1" --quiet - gcloud compute routers delete "a4newimgek-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet - gcloud compute routers delete "a4oldimgek-net-0-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet - gcloud compute routers delete "a4oldimgek-net-1-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet - gcloud compute routers delete "a4oldimgek-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet ---- Wed Nov 26 01:51:11 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:51:59 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 10) --- -The following Cloud Routers are targeted for deletion in this run: -a3mega-sys-net-shu7-router us-west4 -a3u-slurm-net-router us-south1 -a4hsarthakag-net-router us-central1 -a4htest-net-0-router us-central1 -a4newimgek-net-0-router us-south1 -a4newimgek-net-1-router us-south1 -a4newimgek-net-router europe-west4 -a4oldimgek-net-0-router europe-west4 -a4oldimgek-net-1-router europe-west4 -a4oldimgek-net-router europe-west4 -[EXECUTE] Cloud Router: Deleting a3mega-sys-net-shu7-router in us-west4 -[EXECUTE] Cloud Router: Deleting a3u-slurm-net-router in us-south1 -[EXECUTE] Cloud Router: Deleting a4hsarthakag-net-router in us-central1 -[EXECUTE] Cloud Router: Deleting a4htest-net-0-router in us-central1 -[EXECUTE] Cloud Router: Deleting a4newimgek-net-0-router in us-south1 -[EXECUTE] Cloud Router: Deleting a4newimgek-net-1-router in us-south1 -[EXECUTE] Cloud Router: Deleting a4newimgek-net-router in europe-west4 -[EXECUTE] Cloud Router: Deleting a4oldimgek-net-0-router in europe-west4 -[EXECUTE] Cloud Router: Deleting a4oldimgek-net-1-router in europe-west4 -[EXECUTE] Cloud Router: Deleting a4oldimgek-net-router in europe-west4 ---- Wed Nov 26 01:52:07 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:52:24 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 10) --- -The following Cloud Routers are targeted for deletion in this run: -a3mega-sys-net-shu7-router us-west4 -a3u-slurm-net-router us-south1 -a4hsarthakag-net-router us-central1 -a4htest-net-0-router us-central1 -a4newimgek-net-0-router us-south1 -a4newimgek-net-1-router us-south1 -a4newimgek-net-router europe-west4 -a4oldimgek-net-0-router europe-west4 -a4oldimgek-net-1-router europe-west4 -a4oldimgek-net-router europe-west4 -[EXECUTE] Cloud Router: Deleting a3mega-sys-net-shu7-router in us-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west4/routers/a3mega-sys-net-shu7-router]. -[EXECUTE] Cloud Router: Deleting a3u-slurm-net-router in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/routers/a3u-slurm-net-router]. -[EXECUTE] Cloud Router: Deleting a4hsarthakag-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4hsarthakag-net-router]. -[EXECUTE] Cloud Router: Deleting a4htest-net-0-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4htest-net-0-router]. -[EXECUTE] Cloud Router: Deleting a4newimgek-net-0-router in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/routers/a4newimgek-net-0-router]. -[EXECUTE] Cloud Router: Deleting a4newimgek-net-1-router in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/routers/a4newimgek-net-1-router]. -[EXECUTE] Cloud Router: Deleting a4newimgek-net-router in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/a4newimgek-net-router]. -[EXECUTE] Cloud Router: Deleting a4oldimgek-net-0-router in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/a4oldimgek-net-0-router]. -[EXECUTE] Cloud Router: Deleting a4oldimgek-net-1-router in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/a4oldimgek-net-1-router]. -[EXECUTE] Cloud Router: Deleting a4oldimgek-net-router in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/a4oldimgek-net-router]. ---- Wed Nov 26 01:53:20 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:56:35 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 30 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 10) --- -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -a4oldimg-net-router europe-west4 -a4xlavhpcnew-a4x-net-0-router us-west8 -a4xlavhpcnew-a4x-net-1-router us-west8 -a4xslurm-net-router us-west8 -cx-a3u-net-0-router europe-west1 -db451c7-ml-slurm-v6-net-router asia-southeast1 -default-net-router us-central1 -default-router-australia-southeast1 australia-southeast1 -default-router-us-east4 us-east4 -dynpoc-net-router us-central1 - gcloud compute routers delete "a4oldimg-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet - gcloud compute routers delete "a4xlavhpcnew-a4x-net-0-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet - gcloud compute routers delete "a4xlavhpcnew-a4x-net-1-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet - gcloud compute routers delete "a4xslurm-net-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet - gcloud compute routers delete "cx-a3u-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "db451c7-ml-slurm-v6-net-router" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet - gcloud compute routers delete "default-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "default-router-australia-southeast1" --project="hpc-toolkit-dev" --region="australia-southeast1" --quiet - gcloud compute routers delete "default-router-us-east4" --project="hpc-toolkit-dev" --region="us-east4" --quiet - gcloud compute routers delete "dynpoc-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Wed Nov 26 01:56:43 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:57:33 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 33 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -a4oldimg-net-router europe-west4 -a4xlavhpcnew-a4x-net-0-router us-west8 -a4xlavhpcnew-a4x-net-1-router us-west8 -a4xslurm-net-router us-west8 -cx-a3u-net-0-router europe-west1 -db451c7-ml-slurm-v6-net-router asia-southeast1 -dynpoc-net-router us-central1 -g4-dwsq-1-net-1-router us-central1 -g4qclav-net-router us-central1 -gke-1395b4-net-router us-central1 - gcloud compute routers delete "a4oldimg-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet - gcloud compute routers delete "a4xlavhpcnew-a4x-net-0-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet - gcloud compute routers delete "a4xlavhpcnew-a4x-net-1-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet - gcloud compute routers delete "a4xslurm-net-router" --project="hpc-toolkit-dev" --region="us-west8" --quiet - gcloud compute routers delete "cx-a3u-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "db451c7-ml-slurm-v6-net-router" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet - gcloud compute routers delete "dynpoc-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "g4-dwsq-1-net-1-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "g4qclav-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "gke-1395b4-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Wed Nov 26 01:57:40 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:57:54 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 33 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -a4oldimg-net-router europe-west4 -a4xlavhpcnew-a4x-net-0-router us-west8 -a4xlavhpcnew-a4x-net-1-router us-west8 -a4xslurm-net-router us-west8 -cx-a3u-net-0-router europe-west1 -db451c7-ml-slurm-v6-net-router asia-southeast1 -dynpoc-net-router us-central1 -g4-dwsq-1-net-1-router us-central1 -g4qclav-net-router us-central1 -gke-1395b4-net-router us-central1 -[EXECUTE] Cloud Router: Deleting a4oldimg-net-router in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/a4oldimg-net-router]. -[EXECUTE] Cloud Router: Deleting a4xlavhpcnew-a4x-net-0-router in us-west8 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/routers/a4xlavhpcnew-a4x-net-0-router]. -[EXECUTE] Cloud Router: Deleting a4xlavhpcnew-a4x-net-1-router in us-west8 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/routers/a4xlavhpcnew-a4x-net-1-router]. -[EXECUTE] Cloud Router: Deleting a4xslurm-net-router in us-west8 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/routers/a4xslurm-net-router]. -[EXECUTE] Cloud Router: Deleting cx-a3u-net-0-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/cx-a3u-net-0-router]. -[EXECUTE] Cloud Router: Deleting db451c7-ml-slurm-v6-net-router in asia-southeast1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/asia-southeast1/routers/db451c7-ml-slurm-v6-net-router]. -[EXECUTE] Cloud Router: Deleting dynpoc-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/dynpoc-net-router]. -[EXECUTE] Cloud Router: Deleting g4-dwsq-1-net-1-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/g4-dwsq-1-net-1-router]. -[EXECUTE] Cloud Router: Deleting g4qclav-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/g4qclav-net-router]. -[EXECUTE] Cloud Router: Deleting gke-1395b4-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/gke-1395b4-net-router]. ---- Wed Nov 26 01:58:47 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 01:59:14 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 33 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -gke-managed-lustre-basic-net-router us-central1 -h4dqc-net-router us-central1 -h4d-res-swarnabm4-3-net-router us-central1 -hpc-01-net-router us-central1 -hpcdydis-net-router europe-west4 -hpcdy-net-router europe-west4 -hpc-exr-2-net-0-router europe-west1 -hpcimg-net-router us-central1 -hpc-lustre-test-02-net-router us-central1 -khu-h4d-cluster-test-net-router us-central1 - gcloud compute routers delete "gke-managed-lustre-basic-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "h4dqc-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "h4d-res-swarnabm4-3-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "hpc-01-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "hpcdydis-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet - gcloud compute routers delete "hpcdy-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet - gcloud compute routers delete "hpc-exr-2-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "hpcimg-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "hpc-lustre-test-02-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "khu-h4d-cluster-test-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Wed Nov 26 01:59:23 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:02:27 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 33 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -gke-managed-lustre-basic-net-router us-central1 -h4dqc-net-router us-central1 -h4d-res-swarnabm4-3-net-router us-central1 -hpc-01-net-router us-central1 -hpcdydis-net-router europe-west4 -hpcdy-net-router europe-west4 -hpc-exr-2-net-0-router europe-west1 -hpcimg-net-router us-central1 -hpc-lustre-test-02-net-router us-central1 -khu-h4d-cluster-test-net-router us-central1 - gcloud compute routers delete "gke-managed-lustre-basic-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "h4dqc-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "h4d-res-swarnabm4-3-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "hpc-01-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "hpcdydis-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet - gcloud compute routers delete "hpcdy-net-router" --project="hpc-toolkit-dev" --region="europe-west4" --quiet - gcloud compute routers delete "hpc-exr-2-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "hpcimg-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "hpc-lustre-test-02-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "khu-h4d-cluster-test-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Wed Nov 26 02:02:35 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:03:11 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 33 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -gke-managed-lustre-basic-net-router us-central1 -h4dqc-net-router us-central1 -h4d-res-swarnabm4-3-net-router us-central1 -hpc-01-net-router us-central1 -hpcdydis-net-router europe-west4 -hpcdy-net-router europe-west4 -hpc-exr-2-net-0-router europe-west1 -hpcimg-net-router us-central1 -hpc-lustre-test-02-net-router us-central1 -khu-h4d-cluster-test-net-router us-central1 -[EXECUTE] Cloud Router: Deleting gke-managed-lustre-basic-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/gke-managed-lustre-basic-net-router]. -[EXECUTE] Cloud Router: Deleting h4dqc-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/h4dqc-net-router]. -[EXECUTE] Cloud Router: Deleting h4d-res-swarnabm4-3-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/h4d-res-swarnabm4-3-net-router]. -[EXECUTE] Cloud Router: Deleting hpc-01-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/hpc-01-net-router]. -[EXECUTE] Cloud Router: Deleting hpcdydis-net-router in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/hpcdydis-net-router]. -[EXECUTE] Cloud Router: Deleting hpcdy-net-router in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/routers/hpcdy-net-router]. -[EXECUTE] Cloud Router: Deleting hpc-exr-2-net-0-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/hpc-exr-2-net-0-router]. -[EXECUTE] Cloud Router: Deleting hpcimg-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/hpcimg-net-router]. -[EXECUTE] Cloud Router: Deleting hpc-lustre-test-02-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/hpc-lustre-test-02-net-router]. -[EXECUTE] Cloud Router: Deleting khu-h4d-cluster-test-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/khu-h4d-cluster-test-net-router]. ---- Wed Nov 26 02:04:01 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:04:14 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 33 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -laveeek29-net-0-router europe-west1 -laveeek29-net-1-router europe-west1 -laveeek29-net-router europe-west1 -lavoldchk-net-router europe-west1 -lavrohek29-net-0-router europe-west1 -lustre-06-net-router us-central1 -lustre-test-06-net-router us-central1 -mainek-net-0-router europe-west1 -mainek-net-1-router europe-west1 -mainek-net-router europe-west1 -managed-lustre-03-net-router us-central1 -mglsa-net-router us-central1 -mglsard-net-router us-west4 -ml-gke-e2e-a8fae6-net-router asia-southeast1 -ml-gke-net-router us-central1 -monitoring-8323fe-net-router us-central1 -sa-chs-ops-net-0-router europe-west1 -sispot3u-net-0-router europe-west1 -slurm-a3-base-sysnet-router us-west1 -sp-helmtest1-net-router us-central1 - gcloud compute routers delete "laveeek29-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "laveeek29-net-1-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "laveeek29-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "lavoldchk-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "lavrohek29-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "lustre-06-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "lustre-test-06-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "mainek-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "mainek-net-1-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "mainek-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "managed-lustre-03-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "mglsa-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "mglsard-net-router" --project="hpc-toolkit-dev" --region="us-west4" --quiet - gcloud compute routers delete "ml-gke-e2e-a8fae6-net-router" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet - gcloud compute routers delete "ml-gke-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "monitoring-8323fe-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "sa-chs-ops-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "sispot3u-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "slurm-a3-base-sysnet-router" --project="hpc-toolkit-dev" --region="us-west1" --quiet - gcloud compute routers delete "sp-helmtest1-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Wed Nov 26 02:04:22 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:05:46 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 37 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -laveeek29-net-0-router europe-west1 -laveeek29-net-1-router europe-west1 -laveeek29-net-router europe-west1 -lavoldchk-net-router europe-west1 -lavrohek29-net-0-router europe-west1 -mainek-net-0-router europe-west1 -mainek-net-1-router europe-west1 -mainek-net-router europe-west1 -managed-lustre-03-net-router us-central1 -ml-gke-e2e-a8fae6-net-router asia-southeast1 -ml-gke-net-router us-central1 -monitoring-8323fe-net-router us-central1 -sa-chs-ops-net-0-router europe-west1 -sispot3u-net-0-router europe-west1 -slurm-a3-base-sysnet-router us-west1 -sp-helmtest1-net-router us-central1 -static-sarthakag-net-router us-central1 - gcloud compute routers delete "laveeek29-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "laveeek29-net-1-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "laveeek29-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "lavoldchk-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "lavrohek29-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "mainek-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "mainek-net-1-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "mainek-net-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "managed-lustre-03-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "ml-gke-e2e-a8fae6-net-router" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet - gcloud compute routers delete "ml-gke-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "monitoring-8323fe-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "sa-chs-ops-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "sispot3u-net-0-router" --project="hpc-toolkit-dev" --region="europe-west1" --quiet - gcloud compute routers delete "slurm-a3-base-sysnet-router" --project="hpc-toolkit-dev" --region="us-west1" --quiet - gcloud compute routers delete "sp-helmtest1-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet - gcloud compute routers delete "static-sarthakag-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Wed Nov 26 02:05:53 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:06:46 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 37 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -laveeek29-net-0-router europe-west1 -laveeek29-net-1-router europe-west1 -laveeek29-net-router europe-west1 -lavoldchk-net-router europe-west1 -lavrohek29-net-0-router europe-west1 -mainek-net-0-router europe-west1 -mainek-net-1-router europe-west1 -mainek-net-router europe-west1 -managed-lustre-03-net-router us-central1 -ml-gke-e2e-a8fae6-net-router asia-southeast1 -ml-gke-net-router us-central1 -monitoring-8323fe-net-router us-central1 -sa-chs-ops-net-0-router europe-west1 -sispot3u-net-0-router europe-west1 -slurm-a3-base-sysnet-router us-west1 -sp-helmtest1-net-router us-central1 -static-sarthakag-net-router us-central1 -[EXECUTE] Cloud Router: Deleting laveeek29-net-0-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/laveeek29-net-0-router]. -[EXECUTE] Cloud Router: Deleting laveeek29-net-1-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/laveeek29-net-1-router]. -[EXECUTE] Cloud Router: Deleting laveeek29-net-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/laveeek29-net-router]. -[EXECUTE] Cloud Router: Deleting lavoldchk-net-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/lavoldchk-net-router]. -[EXECUTE] Cloud Router: Deleting lavrohek29-net-0-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/lavrohek29-net-0-router]. -[EXECUTE] Cloud Router: Deleting mainek-net-0-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/mainek-net-0-router]. -[EXECUTE] Cloud Router: Deleting mainek-net-1-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/mainek-net-1-router]. -[EXECUTE] Cloud Router: Deleting mainek-net-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/mainek-net-router]. -[EXECUTE] Cloud Router: Deleting managed-lustre-03-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/managed-lustre-03-net-router]. -[EXECUTE] Cloud Router: Deleting ml-gke-e2e-a8fae6-net-router in asia-southeast1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/asia-southeast1/routers/ml-gke-e2e-a8fae6-net-router]. -[EXECUTE] Cloud Router: Deleting ml-gke-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/ml-gke-net-router]. -[EXECUTE] Cloud Router: Deleting monitoring-8323fe-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/monitoring-8323fe-net-router]. -[EXECUTE] Cloud Router: Deleting sa-chs-ops-net-0-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/sa-chs-ops-net-0-router]. -[EXECUTE] Cloud Router: Deleting sispot3u-net-0-router in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/routers/sispot3u-net-0-router]. -[EXECUTE] Cloud Router: Deleting slurm-a3-base-sysnet-router in us-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west1/routers/slurm-a3-base-sysnet-router]. -[EXECUTE] Cloud Router: Deleting sp-helmtest1-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/sp-helmtest1-net-router]. -[EXECUTE] Cloud Router: Deleting static-sarthakag-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/static-sarthakag-net-router]. ---- Wed Nov 26 02:08:16 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Wed Nov 26 02:08:27 PM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 20 resources of each type per run. -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 37 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router ---- Deletion Phase 1: GKE Clusters (Top 20) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 20) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 20) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 2: Cloud Routers (Top 20) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Wed Nov 26 02:08:34 PM UTC 2025 --- Cleanup Script Run Finished --- ---- Thu Nov 27 03:43:35 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-26T23:43:35+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -./cleanup.sh: line 77: ---: command not found ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -a4h-slurm-net-0-router us-central1 -a4h-slurm-net-1-router us-central1 -a4h-slurm-net-router us-central1 -[DRY RUN] Cloud Router: Would delete a4h-slurm-net-0-router in us-central1 - Command: gcloud compute routers delete "a4h-slurm-net-0-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Cloud Router: Would delete a4h-slurm-net-1-router in us-central1 - Command: gcloud compute routers delete "a4h-slurm-net-1-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Cloud Router: Would delete a4h-slurm-net-router in us-central1 - Command: gcloud compute routers delete "a4h-slurm-net-router" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Thu Nov 27 03:43:45 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 03:44:29 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-26T23:44:29+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 59 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com ---- Deletion Phase 1: GKE Clusters (Top 10) --- -Skip Resource: gke-a3-nccl-test (Contains protected substring: gke-a3-nccl-test) -Skip GKE Cluster: gke-a3-nccl-test (In exclusion list) -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-0xxk in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-a3-megagpu-8g-a3-44f10dba-8mx7 in us-west4-a (In exclusion list) -Skip Resource: gke-gke-a3-nccl-test-system-17f71453-fns8 (Contains protected substring: gke-a3-nccl-test) -Skip Instance: gke-gke-a3-nccl-test-system-17f71453-fns8 in us-west4-c (In exclusion list) -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-0-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-0-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-1-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-1-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-2-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-2-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-3-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-3-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-4-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-4-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-5-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-5-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-6-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-6-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-gpunet-7-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-gpunet-7-router (In exclusion list) -Skip Resource: gke-a3-nccl-test-net-router (Contains protected substring: gke-a3-nccl-test) -Skip Cloud Router: gke-a3-nccl-test-net-router (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -The following Cloud Routers are targeted for deletion in this run: -a4h-slurm-net-0-router us-central1 -a4h-slurm-net-1-router us-central1 -a4h-slurm-net-router us-central1 -[EXECUTE] Cloud Router: Deleting a4h-slurm-net-0-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-0-router]. -[EXECUTE] Cloud Router: Deleting a4h-slurm-net-1-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-1-router]. -[EXECUTE] Cloud Router: Deleting a4h-slurm-net-router in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/routers/a4h-slurm-net-router]. ---- Thu Nov 27 03:44:48 AM UTC 2025 --- Cleanup Script Run Finished --- diff --git a/subnetworks.txt b/subnetworks.txt deleted file mode 100644 index a8bae75926..0000000000 --- a/subnetworks.txt +++ /dev/null @@ -1,3453 +0,0 @@ ---- Thu Nov 27 05:20:28 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:20:28+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 10) --- -WARNING: The following filter keys were not present in any resource : createTime -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 10 per region) --- -Processing Subnetworks in region: africa-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in africa-south1 in this run. -Processing Subnetworks in region: asia-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-east1 in this run. -Processing Subnetworks in region: asia-east2 -No Subnetworks found to delete in asia-east2 in this run. -Processing Subnetworks in region: asia-northeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-northeast1 in this run. -Processing Subnetworks in region: asia-northeast2 -No Subnetworks found to delete in asia-northeast2 in this run. -Processing Subnetworks in region: asia-northeast3 -No Subnetworks found to delete in asia-northeast3 in this run. -Processing Subnetworks in region: asia-south1 -No Subnetworks found to delete in asia-south1 in this run. -Processing Subnetworks in region: asia-south2 -No Subnetworks found to delete in asia-south2 in this run. -Processing Subnetworks in region: asia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in asia-southeast1 are targeted for deletion in this run: -db451c7-ml-slurm-v6-primary-subnet asia-southeast1 -ml-gke-e2e-a8fae6-subnet asia-southeast1 -[DRY RUN] Subnetwork: Would delete db451c7-ml-slurm-v6-primary-subnet in asia-southeast1 - Command: gcloud compute networks subnets delete "db451c7-ml-slurm-v6-primary-subnet" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet -[DRY RUN] Subnetwork: Would delete ml-gke-e2e-a8fae6-subnet in asia-southeast1 - Command: gcloud compute networks subnets delete "ml-gke-e2e-a8fae6-subnet" --project="hpc-toolkit-dev" --region="asia-southeast1" --quiet -Processing Subnetworks in region: asia-southeast2 -No Subnetworks found to delete in asia-southeast2 in this run. -Processing Subnetworks in region: asia-southeast3 -No Subnetworks found to delete in asia-southeast3 in this run. -Processing Subnetworks in region: australia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in australia-southeast1 in this run. -Processing Subnetworks in region: australia-southeast2 -No Subnetworks found to delete in australia-southeast2 in this run. -Processing Subnetworks in region: europe-central2 -No Subnetworks found to delete in europe-central2 in this run. -Processing Subnetworks in region: europe-north1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north1 in this run. -Processing Subnetworks in region: europe-north2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north2 in this run. -Processing Subnetworks in region: europe-southwest1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-southwest1 in this run. -Processing Subnetworks in region: europe-west1 -WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in europe-west1 are targeted for deletion in this run: -cx-a3u-sub-0 europe-west1 -hanu-a3u-primary-subnet europe-west1 -hpc-exr-2-sub-0 europe-west1 -laveeek29-mrdma-sub-0 europe-west1 -laveeek29-mrdma-sub-1 europe-west1 -laveeek29-mrdma-sub-2 europe-west1 -laveeek29-mrdma-sub-3 europe-west1 -laveeek29-mrdma-sub-4 europe-west1 -laveeek29-mrdma-sub-5 europe-west1 -laveeek29-mrdma-sub-6 europe-west1 -[DRY RUN] Subnetwork: Would delete cx-a3u-sub-0 in europe-west1 - Command: gcloud compute networks subnets delete "cx-a3u-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete hanu-a3u-primary-subnet in europe-west1 - Command: gcloud compute networks subnets delete "hanu-a3u-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete hpc-exr-2-sub-0 in europe-west1 - Command: gcloud compute networks subnets delete "hpc-exr-2-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-0 in europe-west1 - Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-1 in europe-west1 - Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-1" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-2 in europe-west1 - Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-2" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-3 in europe-west1 - Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-3" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-4 in europe-west1 - Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-4" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-5 in europe-west1 - Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-5" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-6 in europe-west1 - Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-6" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -Processing Subnetworks in region: europe-west10 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west10 in this run. -Processing Subnetworks in region: europe-west12 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west12 in this run. -Processing Subnetworks in region: europe-west15 -No Subnetworks found to delete in europe-west15 in this run. -Processing Subnetworks in region: europe-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west2 in this run. -Processing Subnetworks in region: europe-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west3 in this run. -Processing Subnetworks in region: europe-west4 -The following Subnetworks in europe-west4 are targeted for deletion in this run: -a4newimgek-primary-subnet europe-west4 -a4oldimgek-mrdma-sub-0 europe-west4 -a4oldimgek-mrdma-sub-1 europe-west4 -a4oldimgek-mrdma-sub-2 europe-west4 -a4oldimgek-mrdma-sub-3 europe-west4 -a4oldimgek-mrdma-sub-4 europe-west4 -a4oldimgek-mrdma-sub-5 europe-west4 -a4oldimgek-mrdma-sub-6 europe-west4 -a4oldimgek-mrdma-sub-7 europe-west4 -a4oldimgek-primary-subnet europe-west4 -[DRY RUN] Subnetwork: Would delete a4newimgek-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "a4newimgek-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-0 in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-0" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-1 in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-1" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-2 in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-2" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-3 in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-3" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-4 in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-4" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-5 in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-5" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-6 in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-6" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete a4oldimgek-mrdma-sub-7 in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimgek-mrdma-sub-7" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete a4oldimgek-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimgek-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -Processing Subnetworks in region: europe-west6 -No Subnetworks found to delete in europe-west6 in this run. -Processing Subnetworks in region: europe-west8 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west8 in this run. -Processing Subnetworks in region: europe-west9 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west9 in this run. -Processing Subnetworks in region: me-central1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central1 in this run. -Processing Subnetworks in region: me-central2 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central2 in this run. -Processing Subnetworks in region: me-west1 -No Subnetworks found to delete in me-west1 in this run. -Processing Subnetworks in region: northamerica-northeast1 -No Subnetworks found to delete in northamerica-northeast1 in this run. -Processing Subnetworks in region: northamerica-northeast2 -No Subnetworks found to delete in northamerica-northeast2 in this run. -Processing Subnetworks in region: northamerica-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in northamerica-south1 in this run. -Processing Subnetworks in region: southamerica-east1 -No Subnetworks found to delete in southamerica-east1 in this run. -Processing Subnetworks in region: southamerica-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in southamerica-west1 in this run. -Processing Subnetworks in region: us-central1 -The following Subnetworks in us-central1 are targeted for deletion in this run: -a4h-slurm-c0e262-primary-subnet us-central1 -a4h-slurm-mrdma-sub-0 us-central1 -a4h-slurm-mrdma-sub-1 us-central1 -a4h-slurm-mrdma-sub-2 us-central1 -a4h-slurm-mrdma-sub-3 us-central1 -a4h-slurm-mrdma-sub-4 us-central1 -a4h-slurm-mrdma-sub-5 us-central1 -a4h-slurm-mrdma-sub-6 us-central1 -a4h-slurm-mrdma-sub-7 us-central1 -a4h-slurm-sub-0 us-central1 -[DRY RUN] Subnetwork: Would delete a4h-slurm-c0e262-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-c0e262-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-1 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-1" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-2 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-2" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-3 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-3" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-4 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-4" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-5 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-5" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-6 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-6" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete a4h-slurm-mrdma-sub-7 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-mrdma-sub-7" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete a4h-slurm-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet -Processing Subnetworks in region: us-central2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-central2 in this run. -Processing Subnetworks in region: us-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east1 in this run. -Processing Subnetworks in region: us-east4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east4 in this run. -Processing Subnetworks in region: us-east5 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east5 in this run. -Processing Subnetworks in region: us-east7 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east7 in this run. -Processing Subnetworks in region: us-south1 -The following Subnetworks in us-south1 are targeted for deletion in this run: -a3u-slurm-3224ec-primary-subnet us-south1 -a4newimgek-mrdma-sub-0 us-south1 -a4newimgek-mrdma-sub-1 us-south1 -a4newimgek-mrdma-sub-2 us-south1 -a4newimgek-mrdma-sub-3 us-south1 -a4newimgek-mrdma-sub-4 us-south1 -a4newimgek-mrdma-sub-5 us-south1 -a4newimgek-mrdma-sub-6 us-south1 -a4newimgek-mrdma-sub-7 us-south1 -a4newimgek-sub-0 us-south1 -[DRY RUN] Subnetwork: Would delete a3u-slurm-3224ec-primary-subnet in us-south1 - Command: gcloud compute networks subnets delete "a3u-slurm-3224ec-primary-subnet" --project="hpc-toolkit-dev" --region="us-south1" --quiet -[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-0 in us-south1 - Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-0" --project="hpc-toolkit-dev" --region="us-south1" --quiet -[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-1 in us-south1 - Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-1" --project="hpc-toolkit-dev" --region="us-south1" --quiet -[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-2 in us-south1 - Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-2" --project="hpc-toolkit-dev" --region="us-south1" --quiet -[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-3 in us-south1 - Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-3" --project="hpc-toolkit-dev" --region="us-south1" --quiet -[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-4 in us-south1 - Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-4" --project="hpc-toolkit-dev" --region="us-south1" --quiet -[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-5 in us-south1 - Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-5" --project="hpc-toolkit-dev" --region="us-south1" --quiet -[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-6 in us-south1 - Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-6" --project="hpc-toolkit-dev" --region="us-south1" --quiet -[DRY RUN] Subnetwork: Would delete a4newimgek-mrdma-sub-7 in us-south1 - Command: gcloud compute networks subnets delete "a4newimgek-mrdma-sub-7" --project="hpc-toolkit-dev" --region="us-south1" --quiet -[DRY RUN] Subnetwork: Would delete a4newimgek-sub-0 in us-south1 - Command: gcloud compute networks subnets delete "a4newimgek-sub-0" --project="hpc-toolkit-dev" --region="us-south1" --quiet -Processing Subnetworks in region: us-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in us-west1 are targeted for deletion in this run: -slurm-a3-base-sysnet-subnet us-west1 -[DRY RUN] Subnetwork: Would delete slurm-a3-base-sysnet-subnet in us-west1 - Command: gcloud compute networks subnets delete "slurm-a3-base-sysnet-subnet" --project="hpc-toolkit-dev" --region="us-west1" --quiet -Processing Subnetworks in region: us-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west2 in this run. -Processing Subnetworks in region: us-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west3 in this run. -Processing Subnetworks in region: us-west4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in us-west4 are targeted for deletion in this run: -a3mega-sys-subnet us-west4 -mglsard-subnet us-west4 -[DRY RUN] Subnetwork: Would delete a3mega-sys-subnet in us-west4 - Command: gcloud compute networks subnets delete "a3mega-sys-subnet" --project="hpc-toolkit-dev" --region="us-west4" --quiet -[DRY RUN] Subnetwork: Would delete mglsard-subnet in us-west4 - Command: gcloud compute networks subnets delete "mglsard-subnet" --project="hpc-toolkit-dev" --region="us-west4" --quiet -Processing Subnetworks in region: us-west8 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in us-west8 are targeted for deletion in this run: -a4xlavhpcnew-a4x-sub-0 us-west8 -a4xlavhpcnew-a4x-sub-1 us-west8 -a4x-mrdma-sub-0 us-west8 -a4x-mrdma-sub-1 us-west8 -a4x-mrdma-sub-2 us-west8 -a4x-mrdma-sub-3 us-west8 -a4xslurm-primary-subnet us-west8 -[DRY RUN] Subnetwork: Would delete a4xlavhpcnew-a4x-sub-0 in us-west8 - Command: gcloud compute networks subnets delete "a4xlavhpcnew-a4x-sub-0" --project="hpc-toolkit-dev" --region="us-west8" --quiet -[DRY RUN] Subnetwork: Would delete a4xlavhpcnew-a4x-sub-1 in us-west8 - Command: gcloud compute networks subnets delete "a4xlavhpcnew-a4x-sub-1" --project="hpc-toolkit-dev" --region="us-west8" --quiet -[DRY RUN] Subnetwork: Would delete a4x-mrdma-sub-0 in us-west8 - Command: gcloud compute networks subnets delete "a4x-mrdma-sub-0" --project="hpc-toolkit-dev" --region="us-west8" --quiet -[DRY RUN] Subnetwork: Would delete a4x-mrdma-sub-1 in us-west8 - Command: gcloud compute networks subnets delete "a4x-mrdma-sub-1" --project="hpc-toolkit-dev" --region="us-west8" --quiet -[DRY RUN] Subnetwork: Would delete a4x-mrdma-sub-2 in us-west8 - Command: gcloud compute networks subnets delete "a4x-mrdma-sub-2" --project="hpc-toolkit-dev" --region="us-west8" --quiet -[DRY RUN] Subnetwork: Would delete a4x-mrdma-sub-3 in us-west8 - Command: gcloud compute networks subnets delete "a4x-mrdma-sub-3" --project="hpc-toolkit-dev" --region="us-west8" --quiet -[DRY RUN] Subnetwork: Would delete a4xslurm-primary-subnet in us-west8 - Command: gcloud compute networks subnets delete "a4xslurm-primary-subnet" --project="hpc-toolkit-dev" --region="us-west8" --quiet ---- Thu Nov 27 05:22:15 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 05:24:37 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:24:37+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 10 per region) --- -Processing Subnetworks in region: africa-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in africa-south1 in this run. -Processing Subnetworks in region: asia-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-east1 in this run. -Processing Subnetworks in region: asia-east2 -No Subnetworks found to delete in asia-east2 in this run. -Processing Subnetworks in region: asia-northeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-northeast1 in this run. -Processing Subnetworks in region: asia-northeast2 -No Subnetworks found to delete in asia-northeast2 in this run. -Processing Subnetworks in region: asia-northeast3 -No Subnetworks found to delete in asia-northeast3 in this run. -Processing Subnetworks in region: asia-south1 -No Subnetworks found to delete in asia-south1 in this run. -Processing Subnetworks in region: asia-south2 -No Subnetworks found to delete in asia-south2 in this run. -Processing Subnetworks in region: asia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in asia-southeast1 are targeted for deletion in this run: -db451c7-ml-slurm-v6-primary-subnet asia-southeast1 -ml-gke-e2e-a8fae6-subnet asia-southeast1 -[EXECUTE] Subnetwork: Deleting db451c7-ml-slurm-v6-primary-subnet in asia-southeast1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/asia-southeast1/subnetworks/db451c7-ml-slurm-v6-primary-subnet]. -[EXECUTE] Subnetwork: Deleting ml-gke-e2e-a8fae6-subnet in asia-southeast1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/asia-southeast1/subnetworks/ml-gke-e2e-a8fae6-subnet]. -Processing Subnetworks in region: asia-southeast2 -No Subnetworks found to delete in asia-southeast2 in this run. -Processing Subnetworks in region: asia-southeast3 -No Subnetworks found to delete in asia-southeast3 in this run. -Processing Subnetworks in region: australia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in australia-southeast1 in this run. -Processing Subnetworks in region: australia-southeast2 -No Subnetworks found to delete in australia-southeast2 in this run. -Processing Subnetworks in region: europe-central2 -No Subnetworks found to delete in europe-central2 in this run. -Processing Subnetworks in region: europe-north1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north1 in this run. -Processing Subnetworks in region: europe-north2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north2 in this run. -Processing Subnetworks in region: europe-southwest1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-southwest1 in this run. -Processing Subnetworks in region: europe-west1 -WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in europe-west1 are targeted for deletion in this run: -cx-a3u-sub-0 europe-west1 -hanu-a3u-primary-subnet europe-west1 -hpc-exr-2-sub-0 europe-west1 -laveeek29-mrdma-sub-0 europe-west1 -laveeek29-mrdma-sub-1 europe-west1 -laveeek29-mrdma-sub-2 europe-west1 -laveeek29-mrdma-sub-3 europe-west1 -laveeek29-mrdma-sub-4 europe-west1 -laveeek29-mrdma-sub-5 europe-west1 -laveeek29-mrdma-sub-6 europe-west1 -[EXECUTE] Subnetwork: Deleting cx-a3u-sub-0 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/cx-a3u-sub-0]. -[EXECUTE] Subnetwork: Deleting hanu-a3u-primary-subnet in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/hanu-a3u-primary-subnet]. -[EXECUTE] Subnetwork: Deleting hpc-exr-2-sub-0 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/hpc-exr-2-sub-0]. -[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-0 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-0]. -[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-1 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-1]. -[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-2 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-2]. -[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-3 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-3]. -[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-4 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-4]. -[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-5 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-5]. -[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-6 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-6]. -Processing Subnetworks in region: europe-west10 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west10 in this run. -Processing Subnetworks in region: europe-west12 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west12 in this run. -Processing Subnetworks in region: europe-west15 -No Subnetworks found to delete in europe-west15 in this run. -Processing Subnetworks in region: europe-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west2 in this run. -Processing Subnetworks in region: europe-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west3 in this run. -Processing Subnetworks in region: europe-west4 -The following Subnetworks in europe-west4 are targeted for deletion in this run: -a4newimgek-primary-subnet europe-west4 -a4oldimgek-mrdma-sub-0 europe-west4 -a4oldimgek-mrdma-sub-1 europe-west4 -a4oldimgek-mrdma-sub-2 europe-west4 -a4oldimgek-mrdma-sub-3 europe-west4 -a4oldimgek-mrdma-sub-4 europe-west4 -a4oldimgek-mrdma-sub-5 europe-west4 -a4oldimgek-mrdma-sub-6 europe-west4 -a4oldimgek-mrdma-sub-7 europe-west4 -a4oldimgek-primary-subnet europe-west4 -[EXECUTE] Subnetwork: Deleting a4newimgek-primary-subnet in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4newimgek-primary-subnet]. -[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-0 in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-0]. -[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-1 in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-1]. -[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-2 in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-2]. -[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-3 in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-3]. -[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-4 in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-4]. -[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-5 in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-5]. -[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-6 in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-6]. -[EXECUTE] Subnetwork: Deleting a4oldimgek-mrdma-sub-7 in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-mrdma-sub-7]. -[EXECUTE] Subnetwork: Deleting a4oldimgek-primary-subnet in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-primary-subnet]. -Processing Subnetworks in region: europe-west6 -No Subnetworks found to delete in europe-west6 in this run. -Processing Subnetworks in region: europe-west8 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west8 in this run. -Processing Subnetworks in region: europe-west9 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west9 in this run. -Processing Subnetworks in region: me-central1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central1 in this run. -Processing Subnetworks in region: me-central2 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central2 in this run. -Processing Subnetworks in region: me-west1 -No Subnetworks found to delete in me-west1 in this run. -Processing Subnetworks in region: northamerica-northeast1 -No Subnetworks found to delete in northamerica-northeast1 in this run. -Processing Subnetworks in region: northamerica-northeast2 -No Subnetworks found to delete in northamerica-northeast2 in this run. -Processing Subnetworks in region: northamerica-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in northamerica-south1 in this run. -Processing Subnetworks in region: southamerica-east1 -No Subnetworks found to delete in southamerica-east1 in this run. -Processing Subnetworks in region: southamerica-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in southamerica-west1 in this run. -Processing Subnetworks in region: us-central1 -The following Subnetworks in us-central1 are targeted for deletion in this run: -a4h-slurm-c0e262-primary-subnet us-central1 -a4h-slurm-mrdma-sub-0 us-central1 -a4h-slurm-mrdma-sub-1 us-central1 -a4h-slurm-mrdma-sub-2 us-central1 -a4h-slurm-mrdma-sub-3 us-central1 -a4h-slurm-mrdma-sub-4 us-central1 -a4h-slurm-mrdma-sub-5 us-central1 -a4h-slurm-mrdma-sub-6 us-central1 -a4h-slurm-mrdma-sub-7 us-central1 -a4h-slurm-sub-0 us-central1 -[EXECUTE] Subnetwork: Deleting a4h-slurm-c0e262-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-c0e262-primary-subnet]. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-0 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-0]. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-1 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-1]. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-2 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-2]. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-3 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-3]. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-4 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-4]. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-5 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-5]. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-6 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-6]. -[EXECUTE] Subnetwork: Deleting a4h-slurm-mrdma-sub-7 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-mrdma-sub-7]. -[EXECUTE] Subnetwork: Deleting a4h-slurm-sub-0 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-sub-0]. -Processing Subnetworks in region: us-central2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-central2 in this run. -Processing Subnetworks in region: us-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east1 in this run. -Processing Subnetworks in region: us-east4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east4 in this run. -Processing Subnetworks in region: us-east5 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east5 in this run. -Processing Subnetworks in region: us-east7 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east7 in this run. -Processing Subnetworks in region: us-south1 -The following Subnetworks in us-south1 are targeted for deletion in this run: -a3u-slurm-3224ec-primary-subnet us-south1 -a4newimgek-mrdma-sub-0 us-south1 -a4newimgek-mrdma-sub-1 us-south1 -a4newimgek-mrdma-sub-2 us-south1 -a4newimgek-mrdma-sub-3 us-south1 -a4newimgek-mrdma-sub-4 us-south1 -a4newimgek-mrdma-sub-5 us-south1 -a4newimgek-mrdma-sub-6 us-south1 -a4newimgek-mrdma-sub-7 us-south1 -a4newimgek-sub-0 us-south1 -[EXECUTE] Subnetwork: Deleting a3u-slurm-3224ec-primary-subnet in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a3u-slurm-3224ec-primary-subnet]. -[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-0 in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-0]. -[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-1 in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-1]. -[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-2 in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-2]. -[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-3 in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-3]. -[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-4 in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-4]. -[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-5 in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-5]. -[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-6 in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-6]. -[EXECUTE] Subnetwork: Deleting a4newimgek-mrdma-sub-7 in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-mrdma-sub-7]. -[EXECUTE] Subnetwork: Deleting a4newimgek-sub-0 in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-sub-0]. -Processing Subnetworks in region: us-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in us-west1 are targeted for deletion in this run: -slurm-a3-base-sysnet-subnet us-west1 -[EXECUTE] Subnetwork: Deleting slurm-a3-base-sysnet-subnet in us-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west1/subnetworks/slurm-a3-base-sysnet-subnet]. -Processing Subnetworks in region: us-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west2 in this run. -Processing Subnetworks in region: us-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west3 in this run. -Processing Subnetworks in region: us-west4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in us-west4 are targeted for deletion in this run: -a3mega-sys-subnet us-west4 -mglsard-subnet us-west4 -[EXECUTE] Subnetwork: Deleting a3mega-sys-subnet in us-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west4/subnetworks/a3mega-sys-subnet]. -[EXECUTE] Subnetwork: Deleting mglsard-subnet in us-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west4/subnetworks/mglsard-subnet]. -Processing Subnetworks in region: us-west8 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in us-west8 are targeted for deletion in this run: -a4xlavhpcnew-a4x-sub-0 us-west8 -a4xlavhpcnew-a4x-sub-1 us-west8 -a4x-mrdma-sub-0 us-west8 -a4x-mrdma-sub-1 us-west8 -a4x-mrdma-sub-2 us-west8 -a4x-mrdma-sub-3 us-west8 -a4xslurm-primary-subnet us-west8 -[EXECUTE] Subnetwork: Deleting a4xlavhpcnew-a4x-sub-0 in us-west8 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4xlavhpcnew-a4x-sub-0]. -[EXECUTE] Subnetwork: Deleting a4xlavhpcnew-a4x-sub-1 in us-west8 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4xlavhpcnew-a4x-sub-1]. -[EXECUTE] Subnetwork: Deleting a4x-mrdma-sub-0 in us-west8 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4x-mrdma-sub-0]. -[EXECUTE] Subnetwork: Deleting a4x-mrdma-sub-1 in us-west8 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4x-mrdma-sub-1]. -[EXECUTE] Subnetwork: Deleting a4x-mrdma-sub-2 in us-west8 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4x-mrdma-sub-2]. -[EXECUTE] Subnetwork: Deleting a4x-mrdma-sub-3 in us-west8 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4x-mrdma-sub-3]. -[EXECUTE] Subnetwork: Deleting a4xslurm-primary-subnet in us-west8 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-west8/subnetworks/a4xslurm-primary-subnet]. ---- Thu Nov 27 05:38:32 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 05:38:42 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:38:42+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 10 per region) --- -Processing Subnetworks in region: africa-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in africa-south1 in this run. -Processing Subnetworks in region: asia-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-east1 in this run. -Processing Subnetworks in region: asia-east2 -No Subnetworks found to delete in asia-east2 in this run. -Processing Subnetworks in region: asia-northeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-northeast1 in this run. -Processing Subnetworks in region: asia-northeast2 -No Subnetworks found to delete in asia-northeast2 in this run. -Processing Subnetworks in region: asia-northeast3 -No Subnetworks found to delete in asia-northeast3 in this run. -Processing Subnetworks in region: asia-south1 -No Subnetworks found to delete in asia-south1 in this run. -Processing Subnetworks in region: asia-south2 -No Subnetworks found to delete in asia-south2 in this run. -Processing Subnetworks in region: asia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-southeast1 in this run. -Processing Subnetworks in region: asia-southeast2 -No Subnetworks found to delete in asia-southeast2 in this run. -Processing Subnetworks in region: asia-southeast3 -No Subnetworks found to delete in asia-southeast3 in this run. -Processing Subnetworks in region: australia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in australia-southeast1 in this run. -Processing Subnetworks in region: australia-southeast2 -No Subnetworks found to delete in australia-southeast2 in this run. -Processing Subnetworks in region: europe-central2 -No Subnetworks found to delete in europe-central2 in this run. -Processing Subnetworks in region: europe-north1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north1 in this run. -Processing Subnetworks in region: europe-north2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north2 in this run. -Processing Subnetworks in region: europe-southwest1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-southwest1 in this run. -Processing Subnetworks in region: europe-west1 -WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in europe-west1 are targeted for deletion in this run: -laveeek29-mrdma-sub-7 europe-west1 -laveeek29-primary-subnet europe-west1 -laveeek29-sub-0 europe-west1 -laveeek29-sub-1 europe-west1 -lavoldchk-primary-subnet europe-west1 -lavrohek29-sub-0 europe-west1 -mainek-mrdma-sub-0 europe-west1 -mainek-mrdma-sub-1 europe-west1 -mainek-mrdma-sub-2 europe-west1 -mainek-mrdma-sub-3 europe-west1 -[DRY RUN] Subnetwork: Would delete laveeek29-mrdma-sub-7 in europe-west1 - Command: gcloud compute networks subnets delete "laveeek29-mrdma-sub-7" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete laveeek29-primary-subnet in europe-west1 - Command: gcloud compute networks subnets delete "laveeek29-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete laveeek29-sub-0 in europe-west1 - Command: gcloud compute networks subnets delete "laveeek29-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete laveeek29-sub-1 in europe-west1 - Command: gcloud compute networks subnets delete "laveeek29-sub-1" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete lavoldchk-primary-subnet in europe-west1 - Command: gcloud compute networks subnets delete "lavoldchk-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete lavrohek29-sub-0 in europe-west1 - Command: gcloud compute networks subnets delete "lavrohek29-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-0 in europe-west1 - Command: gcloud compute networks subnets delete "mainek-mrdma-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-1 in europe-west1 - Command: gcloud compute networks subnets delete "mainek-mrdma-sub-1" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-2 in europe-west1 - Command: gcloud compute networks subnets delete "mainek-mrdma-sub-2" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-3 in europe-west1 - Command: gcloud compute networks subnets delete "mainek-mrdma-sub-3" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -Processing Subnetworks in region: europe-west10 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west10 in this run. -Processing Subnetworks in region: europe-west12 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west12 in this run. -Processing Subnetworks in region: europe-west15 -No Subnetworks found to delete in europe-west15 in this run. -Processing Subnetworks in region: europe-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west2 in this run. -Processing Subnetworks in region: europe-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west3 in this run. -Processing Subnetworks in region: europe-west4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in europe-west4 are targeted for deletion in this run: -a4oldimgek-sub-0 europe-west4 -a4oldimgek-sub-1 europe-west4 -a4oldimg-primary-subnet europe-west4 -hpcdydis-primary-subnet europe-west4 -hpcdy-primary-subnet europe-west4 -[DRY RUN] Subnetwork: Would delete a4oldimgek-sub-0 in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimgek-sub-0" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete a4oldimgek-sub-1 in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimgek-sub-1" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete a4oldimg-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "a4oldimg-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -Processing Subnetworks in region: europe-west6 -No Subnetworks found to delete in europe-west6 in this run. -Processing Subnetworks in region: europe-west8 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west8 in this run. -Processing Subnetworks in region: europe-west9 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west9 in this run. -Processing Subnetworks in region: me-central1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central1 in this run. -Processing Subnetworks in region: me-central2 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central2 in this run. -Processing Subnetworks in region: me-west1 -No Subnetworks found to delete in me-west1 in this run. -Processing Subnetworks in region: northamerica-northeast1 -No Subnetworks found to delete in northamerica-northeast1 in this run. -Processing Subnetworks in region: northamerica-northeast2 -No Subnetworks found to delete in northamerica-northeast2 in this run. -Processing Subnetworks in region: northamerica-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in northamerica-south1 in this run. -Processing Subnetworks in region: southamerica-east1 -No Subnetworks found to delete in southamerica-east1 in this run. -Processing Subnetworks in region: southamerica-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in southamerica-west1 in this run. -Processing Subnetworks in region: us-central1 -Skip Subnet: default (On default network) -The following Subnetworks in us-central1 are targeted for deletion in this run: -a4h-slurm-sub-1 us-central1 -a4htest-sub-0 us-central1 -dynpoc-primary-subnet us-central1 -g4qclav-primary-subnet us-central1 -gke-1395b4-subnet us-central1 -h4d-cluster-rdma-sub-0 us-central1 -h4dqc-primary-subnet us-central1 -h4dqc-rdma-sub-0 us-central1 -h4d-res-swarnabm4-3-rdma-sub us-central1 -h4d-res-swarnabm4-3-sub us-central1 -[DRY RUN] Subnetwork: Would delete a4h-slurm-sub-1 in us-central1 - Command: gcloud compute networks subnets delete "a4h-slurm-sub-1" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete a4htest-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "a4htest-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete dynpoc-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "dynpoc-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete g4qclav-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "g4qclav-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete gke-1395b4-subnet in us-central1 - Command: gcloud compute networks subnets delete "gke-1395b4-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete h4d-cluster-rdma-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "h4d-cluster-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete h4dqc-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "h4dqc-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete h4dqc-rdma-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "h4dqc-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete h4d-res-swarnabm4-3-rdma-sub in us-central1 - Command: gcloud compute networks subnets delete "h4d-res-swarnabm4-3-rdma-sub" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete h4d-res-swarnabm4-3-sub in us-central1 - Command: gcloud compute networks subnets delete "h4d-res-swarnabm4-3-sub" --project="hpc-toolkit-dev" --region="us-central1" --quiet -Processing Subnetworks in region: us-central2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-central2 in this run. -Processing Subnetworks in region: us-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east1 in this run. -Processing Subnetworks in region: us-east4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east4 in this run. -Processing Subnetworks in region: us-east5 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east5 in this run. -Processing Subnetworks in region: us-east7 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east7 in this run. -Processing Subnetworks in region: us-south1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in us-south1 are targeted for deletion in this run: -a4newimgek-sub-1 us-south1 -hanu-test-subnet us-south1 -[DRY RUN] Subnetwork: Would delete a4newimgek-sub-1 in us-south1 - Command: gcloud compute networks subnets delete "a4newimgek-sub-1" --project="hpc-toolkit-dev" --region="us-south1" --quiet -[DRY RUN] Subnetwork: Would delete hanu-test-subnet in us-south1 - Command: gcloud compute networks subnets delete "hanu-test-subnet" --project="hpc-toolkit-dev" --region="us-south1" --quiet -Processing Subnetworks in region: us-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west1 in this run. -Processing Subnetworks in region: us-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west2 in this run. -Processing Subnetworks in region: us-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west3 in this run. -Processing Subnetworks in region: us-west4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west4 in this run. -Processing Subnetworks in region: us-west8 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west8 in this run. ---- Thu Nov 27 05:40:32 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 05:41:03 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:41:03+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 10 per region) --- -Processing Subnetworks in region: africa-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in africa-south1 in this run. -Processing Subnetworks in region: asia-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-east1 in this run. -Processing Subnetworks in region: asia-east2 -No Subnetworks found to delete in asia-east2 in this run. -Processing Subnetworks in region: asia-northeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-northeast1 in this run. -Processing Subnetworks in region: asia-northeast2 -No Subnetworks found to delete in asia-northeast2 in this run. -Processing Subnetworks in region: asia-northeast3 -No Subnetworks found to delete in asia-northeast3 in this run. -Processing Subnetworks in region: asia-south1 -No Subnetworks found to delete in asia-south1 in this run. -Processing Subnetworks in region: asia-south2 -No Subnetworks found to delete in asia-south2 in this run. -Processing Subnetworks in region: asia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-southeast1 in this run. -Processing Subnetworks in region: asia-southeast2 -No Subnetworks found to delete in asia-southeast2 in this run. -Processing Subnetworks in region: asia-southeast3 -No Subnetworks found to delete in asia-southeast3 in this run. -Processing Subnetworks in region: australia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in australia-southeast1 in this run. -Processing Subnetworks in region: australia-southeast2 -No Subnetworks found to delete in australia-southeast2 in this run. -Processing Subnetworks in region: europe-central2 -No Subnetworks found to delete in europe-central2 in this run. -Processing Subnetworks in region: europe-north1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north1 in this run. -Processing Subnetworks in region: europe-north2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north2 in this run. -Processing Subnetworks in region: europe-southwest1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-southwest1 in this run. -Processing Subnetworks in region: europe-west1 -WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in europe-west1 are targeted for deletion in this run: -laveeek29-mrdma-sub-7 europe-west1 -laveeek29-primary-subnet europe-west1 -laveeek29-sub-0 europe-west1 -laveeek29-sub-1 europe-west1 -lavoldchk-primary-subnet europe-west1 -lavrohek29-sub-0 europe-west1 -mainek-mrdma-sub-0 europe-west1 -mainek-mrdma-sub-1 europe-west1 -mainek-mrdma-sub-2 europe-west1 -mainek-mrdma-sub-3 europe-west1 -[EXECUTE] Subnetwork: Deleting laveeek29-mrdma-sub-7 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-mrdma-sub-7]. -[EXECUTE] Subnetwork: Deleting laveeek29-primary-subnet in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-primary-subnet]. -[EXECUTE] Subnetwork: Deleting laveeek29-sub-0 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-sub-0]. -[EXECUTE] Subnetwork: Deleting laveeek29-sub-1 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/laveeek29-sub-1]. -[EXECUTE] Subnetwork: Deleting lavoldchk-primary-subnet in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/lavoldchk-primary-subnet]. -[EXECUTE] Subnetwork: Deleting lavrohek29-sub-0 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/lavrohek29-sub-0]. -[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-0 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-0]. -[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-1 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-1]. -[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-2 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-2]. -[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-3 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-3]. -Processing Subnetworks in region: europe-west10 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west10 in this run. -Processing Subnetworks in region: europe-west12 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west12 in this run. -Processing Subnetworks in region: europe-west15 -No Subnetworks found to delete in europe-west15 in this run. -Processing Subnetworks in region: europe-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west2 in this run. -Processing Subnetworks in region: europe-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west3 in this run. -Processing Subnetworks in region: europe-west4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in europe-west4 are targeted for deletion in this run: -a4oldimgek-sub-0 europe-west4 -a4oldimgek-sub-1 europe-west4 -a4oldimg-primary-subnet europe-west4 -hpcdydis-primary-subnet europe-west4 -hpcdy-primary-subnet europe-west4 -[EXECUTE] Subnetwork: Deleting a4oldimgek-sub-0 in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-sub-0]. -[EXECUTE] Subnetwork: Deleting a4oldimgek-sub-1 in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimgek-sub-1]. -[EXECUTE] Subnetwork: Deleting a4oldimg-primary-subnet in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/a4oldimg-primary-subnet]. -[EXECUTE] Subnetwork: Deleting hpcdydis-primary-subnet in europe-west4 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdydis-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-mds0-internal-address' - -[EXECUTE] Subnetwork: Deleting hpcdy-primary-subnet in europe-west4 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdy-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss2-internal-address' - -Processing Subnetworks in region: europe-west6 -No Subnetworks found to delete in europe-west6 in this run. -Processing Subnetworks in region: europe-west8 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west8 in this run. -Processing Subnetworks in region: europe-west9 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west9 in this run. -Processing Subnetworks in region: me-central1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central1 in this run. -Processing Subnetworks in region: me-central2 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central2 in this run. -Processing Subnetworks in region: me-west1 -No Subnetworks found to delete in me-west1 in this run. -Processing Subnetworks in region: northamerica-northeast1 -No Subnetworks found to delete in northamerica-northeast1 in this run. -Processing Subnetworks in region: northamerica-northeast2 -No Subnetworks found to delete in northamerica-northeast2 in this run. -Processing Subnetworks in region: northamerica-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in northamerica-south1 in this run. -Processing Subnetworks in region: southamerica-east1 -No Subnetworks found to delete in southamerica-east1 in this run. -Processing Subnetworks in region: southamerica-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in southamerica-west1 in this run. -Processing Subnetworks in region: us-central1 -Skip Subnet: default (On default network) -The following Subnetworks in us-central1 are targeted for deletion in this run: -a4h-slurm-sub-1 us-central1 -a4htest-sub-0 us-central1 -dynpoc-primary-subnet us-central1 -g4qclav-primary-subnet us-central1 -gke-1395b4-subnet us-central1 -h4d-cluster-rdma-sub-0 us-central1 -h4dqc-primary-subnet us-central1 -h4dqc-rdma-sub-0 us-central1 -h4d-res-swarnabm4-3-rdma-sub us-central1 -h4d-res-swarnabm4-3-sub us-central1 -[EXECUTE] Subnetwork: Deleting a4h-slurm-sub-1 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/a4h-slurm-sub-1]. -[EXECUTE] Subnetwork: Deleting a4htest-sub-0 in us-central1 ---- Thu Nov 27 05:50:14 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:50:14+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 10 per region) --- -Processing Subnetworks in region: africa-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in africa-south1 in this run. -Processing Subnetworks in region: asia-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-east1 in this run. -Processing Subnetworks in region: asia-east2 -No Subnetworks found to delete in asia-east2 in this run. -Processing Subnetworks in region: asia-northeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-northeast1 in this run. -Processing Subnetworks in region: asia-northeast2 -No Subnetworks found to delete in asia-northeast2 in this run. -Processing Subnetworks in region: asia-northeast3 -No Subnetworks found to delete in asia-northeast3 in this run. -Processing Subnetworks in region: asia-south1 -No Subnetworks found to delete in asia-south1 in this run. -Processing Subnetworks in region: asia-south2 -No Subnetworks found to delete in asia-south2 in this run. -Processing Subnetworks in region: asia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-southeast1 in this run. -Processing Subnetworks in region: asia-southeast2 -No Subnetworks found to delete in asia-southeast2 in this run. -Processing Subnetworks in region: asia-southeast3 -No Subnetworks found to delete in asia-southeast3 in this run. -Processing Subnetworks in region: australia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in australia-southeast1 in this run. -Processing Subnetworks in region: australia-southeast2 -No Subnetworks found to delete in australia-southeast2 in this run. -Processing Subnetworks in region: europe-central2 -No Subnetworks found to delete in europe-central2 in this run. -Processing Subnetworks in region: europe-north1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north1 in this run. -Processing Subnetworks in region: europe-north2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north2 in this run. -Processing Subnetworks in region: europe-southwest1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-southwest1 in this run. -Processing Subnetworks in region: europe-west1 -WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in europe-west1 are targeted for deletion in this run: -mainek-mrdma-sub-4 europe-west1 -mainek-mrdma-sub-5 europe-west1 -mainek-mrdma-sub-6 europe-west1 -mainek-mrdma-sub-7 europe-west1 -mainek-primary-subnet europe-west1 -mainek-sub-0 europe-west1 -mainek-sub-1 europe-west1 -sa-chs-ops-sub-0 europe-west1 -sispot3u-sub-0 europe-west1 -[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-4 in europe-west1 - Command: gcloud compute networks subnets delete "mainek-mrdma-sub-4" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-5 in europe-west1 - Command: gcloud compute networks subnets delete "mainek-mrdma-sub-5" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-6 in europe-west1 - Command: gcloud compute networks subnets delete "mainek-mrdma-sub-6" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete mainek-mrdma-sub-7 in europe-west1 - Command: gcloud compute networks subnets delete "mainek-mrdma-sub-7" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete mainek-primary-subnet in europe-west1 - Command: gcloud compute networks subnets delete "mainek-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete mainek-sub-0 in europe-west1 - Command: gcloud compute networks subnets delete "mainek-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete mainek-sub-1 in europe-west1 - Command: gcloud compute networks subnets delete "mainek-sub-1" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete sa-chs-ops-sub-0 in europe-west1 - Command: gcloud compute networks subnets delete "sa-chs-ops-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -[DRY RUN] Subnetwork: Would delete sispot3u-sub-0 in europe-west1 - Command: gcloud compute networks subnets delete "sispot3u-sub-0" --project="hpc-toolkit-dev" --region="europe-west1" --quiet -Processing Subnetworks in region: europe-west10 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west10 in this run. -Processing Subnetworks in region: europe-west12 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west12 in this run. -Processing Subnetworks in region: europe-west15 -No Subnetworks found to delete in europe-west15 in this run. -Processing Subnetworks in region: europe-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west2 in this run. -Processing Subnetworks in region: europe-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west3 in this run. -Processing Subnetworks in region: europe-west4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in europe-west4 are targeted for deletion in this run: -hpcdydis-primary-subnet europe-west4 -hpcdy-primary-subnet europe-west4 -[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -Processing Subnetworks in region: europe-west6 -No Subnetworks found to delete in europe-west6 in this run. -Processing Subnetworks in region: europe-west8 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west8 in this run. -Processing Subnetworks in region: europe-west9 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west9 in this run. -Processing Subnetworks in region: me-central1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central1 in this run. -Processing Subnetworks in region: me-central2 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central2 in this run. -Processing Subnetworks in region: me-west1 -No Subnetworks found to delete in me-west1 in this run. -Processing Subnetworks in region: northamerica-northeast1 -No Subnetworks found to delete in northamerica-northeast1 in this run. -Processing Subnetworks in region: northamerica-northeast2 -No Subnetworks found to delete in northamerica-northeast2 in this run. -Processing Subnetworks in region: northamerica-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in northamerica-south1 in this run. -Processing Subnetworks in region: southamerica-east1 -No Subnetworks found to delete in southamerica-east1 in this run. -Processing Subnetworks in region: southamerica-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in southamerica-west1 in this run. -Processing Subnetworks in region: us-central1 -Skip Subnet: default (On default network) -The following Subnetworks in us-central1 are targeted for deletion in this run: -dynpoc-primary-subnet us-central1 -g4qclav-primary-subnet us-central1 -gke-1395b4-subnet us-central1 -h4d-cluster-rdma-sub-0 us-central1 -h4dqc-primary-subnet us-central1 -h4dqc-rdma-sub-0 us-central1 -h4d-res-swarnabm4-3-rdma-sub us-central1 -h4d-res-swarnabm4-3-sub us-central1 -hpc-01-primary-subnet us-central1 -hpcimg-primary-subnet us-central1 -[DRY RUN] Subnetwork: Would delete dynpoc-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "dynpoc-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete g4qclav-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "g4qclav-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete gke-1395b4-subnet in us-central1 - Command: gcloud compute networks subnets delete "gke-1395b4-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete h4d-cluster-rdma-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "h4d-cluster-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete h4dqc-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "h4dqc-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete h4dqc-rdma-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "h4dqc-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete h4d-res-swarnabm4-3-rdma-sub in us-central1 - Command: gcloud compute networks subnets delete "h4d-res-swarnabm4-3-rdma-sub" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete h4d-res-swarnabm4-3-sub in us-central1 - Command: gcloud compute networks subnets delete "h4d-res-swarnabm4-3-sub" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete hpc-01-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "hpc-01-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete hpcimg-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "hpcimg-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -Processing Subnetworks in region: us-central2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-central2 in this run. -Processing Subnetworks in region: us-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east1 in this run. -Processing Subnetworks in region: us-east4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east4 in this run. -Processing Subnetworks in region: us-east5 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east5 in this run. -Processing Subnetworks in region: us-east7 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east7 in this run. -Processing Subnetworks in region: us-south1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in us-south1 are targeted for deletion in this run: -a4newimgek-sub-1 us-south1 -hanu-test-subnet us-south1 -[DRY RUN] Subnetwork: Would delete a4newimgek-sub-1 in us-south1 - Command: gcloud compute networks subnets delete "a4newimgek-sub-1" --project="hpc-toolkit-dev" --region="us-south1" --quiet -[DRY RUN] Subnetwork: Would delete hanu-test-subnet in us-south1 - Command: gcloud compute networks subnets delete "hanu-test-subnet" --project="hpc-toolkit-dev" --region="us-south1" --quiet -Processing Subnetworks in region: us-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west1 in this run. -Processing Subnetworks in region: us-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west2 in this run. -Processing Subnetworks in region: us-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west3 in this run. -Processing Subnetworks in region: us-west4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west4 in this run. -Processing Subnetworks in region: us-west8 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west8 in this run. ---- Thu Nov 27 05:52:26 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 05:52:57 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T01:52:57+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -Skip Filestore Instance: a4h-slurm-c0e262-f5260d85 (Location not found in list output) -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 10 per region) --- -Processing Subnetworks in region: africa-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in africa-south1 in this run. -Processing Subnetworks in region: asia-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-east1 in this run. -Processing Subnetworks in region: asia-east2 -No Subnetworks found to delete in asia-east2 in this run. -Processing Subnetworks in region: asia-northeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-northeast1 in this run. -Processing Subnetworks in region: asia-northeast2 -No Subnetworks found to delete in asia-northeast2 in this run. -Processing Subnetworks in region: asia-northeast3 -No Subnetworks found to delete in asia-northeast3 in this run. -Processing Subnetworks in region: asia-south1 -No Subnetworks found to delete in asia-south1 in this run. -Processing Subnetworks in region: asia-south2 -No Subnetworks found to delete in asia-south2 in this run. -Processing Subnetworks in region: asia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-southeast1 in this run. -Processing Subnetworks in region: asia-southeast2 -No Subnetworks found to delete in asia-southeast2 in this run. -Processing Subnetworks in region: asia-southeast3 -No Subnetworks found to delete in asia-southeast3 in this run. -Processing Subnetworks in region: australia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in australia-southeast1 in this run. -Processing Subnetworks in region: australia-southeast2 -No Subnetworks found to delete in australia-southeast2 in this run. -Processing Subnetworks in region: europe-central2 -No Subnetworks found to delete in europe-central2 in this run. -Processing Subnetworks in region: europe-north1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north1 in this run. -Processing Subnetworks in region: europe-north2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north2 in this run. -Processing Subnetworks in region: europe-southwest1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-southwest1 in this run. -Processing Subnetworks in region: europe-west1 -WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in europe-west1 are targeted for deletion in this run: -mainek-mrdma-sub-4 europe-west1 -mainek-mrdma-sub-5 europe-west1 -mainek-mrdma-sub-6 europe-west1 -mainek-mrdma-sub-7 europe-west1 -mainek-primary-subnet europe-west1 -mainek-sub-0 europe-west1 -mainek-sub-1 europe-west1 -sa-chs-ops-sub-0 europe-west1 -sispot3u-sub-0 europe-west1 -[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-4 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-4]. -[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-5 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-5]. -[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-6 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-6]. -[EXECUTE] Subnetwork: Deleting mainek-mrdma-sub-7 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-mrdma-sub-7]. -[EXECUTE] Subnetwork: Deleting mainek-primary-subnet in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-primary-subnet]. -[EXECUTE] Subnetwork: Deleting mainek-sub-0 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-sub-0]. -[EXECUTE] Subnetwork: Deleting mainek-sub-1 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/mainek-sub-1]. -[EXECUTE] Subnetwork: Deleting sa-chs-ops-sub-0 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/sa-chs-ops-sub-0]. -[EXECUTE] Subnetwork: Deleting sispot3u-sub-0 in europe-west1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west1/subnetworks/sispot3u-sub-0]. -Processing Subnetworks in region: europe-west10 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west10 in this run. -Processing Subnetworks in region: europe-west12 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west12 in this run. -Processing Subnetworks in region: europe-west15 -No Subnetworks found to delete in europe-west15 in this run. -Processing Subnetworks in region: europe-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west2 in this run. -Processing Subnetworks in region: europe-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west3 in this run. -Processing Subnetworks in region: europe-west4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in europe-west4 are targeted for deletion in this run: -hpcdydis-primary-subnet europe-west4 -hpcdy-primary-subnet europe-west4 -[EXECUTE] Subnetwork: Deleting hpcdydis-primary-subnet in europe-west4 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdydis-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-mds0-internal-address' - -[EXECUTE] Subnetwork: Deleting hpcdy-primary-subnet in europe-west4 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdy-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss2-internal-address' - -Processing Subnetworks in region: europe-west6 -No Subnetworks found to delete in europe-west6 in this run. -Processing Subnetworks in region: europe-west8 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west8 in this run. -Processing Subnetworks in region: europe-west9 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west9 in this run. -Processing Subnetworks in region: me-central1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central1 in this run. -Processing Subnetworks in region: me-central2 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central2 in this run. -Processing Subnetworks in region: me-west1 -No Subnetworks found to delete in me-west1 in this run. -Processing Subnetworks in region: northamerica-northeast1 -No Subnetworks found to delete in northamerica-northeast1 in this run. -Processing Subnetworks in region: northamerica-northeast2 -No Subnetworks found to delete in northamerica-northeast2 in this run. -Processing Subnetworks in region: northamerica-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in northamerica-south1 in this run. -Processing Subnetworks in region: southamerica-east1 -No Subnetworks found to delete in southamerica-east1 in this run. -Processing Subnetworks in region: southamerica-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in southamerica-west1 in this run. -Processing Subnetworks in region: us-central1 -Skip Subnet: default (On default network) -The following Subnetworks in us-central1 are targeted for deletion in this run: -dynpoc-primary-subnet us-central1 -g4qclav-primary-subnet us-central1 -gke-1395b4-subnet us-central1 -h4d-cluster-rdma-sub-0 us-central1 -h4dqc-primary-subnet us-central1 -h4dqc-rdma-sub-0 us-central1 -h4d-res-swarnabm4-3-rdma-sub us-central1 -h4d-res-swarnabm4-3-sub us-central1 -hpc-01-primary-subnet us-central1 -hpcimg-primary-subnet us-central1 -[EXECUTE] Subnetwork: Deleting dynpoc-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/dynpoc-primary-subnet]. -[EXECUTE] Subnetwork: Deleting g4qclav-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/g4qclav-primary-subnet]. -[EXECUTE] Subnetwork: Deleting gke-1395b4-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/gke-1395b4-subnet]. -[EXECUTE] Subnetwork: Deleting h4d-cluster-rdma-sub-0 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/h4d-cluster-rdma-sub-0]. -[EXECUTE] Subnetwork: Deleting h4dqc-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/h4dqc-primary-subnet]. -[EXECUTE] Subnetwork: Deleting h4dqc-rdma-sub-0 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/h4dqc-rdma-sub-0]. -[EXECUTE] Subnetwork: Deleting h4d-res-swarnabm4-3-rdma-sub in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/h4d-res-swarnabm4-3-rdma-sub]. -[EXECUTE] Subnetwork: Deleting h4d-res-swarnabm4-3-sub in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/h4d-res-swarnabm4-3-sub]. -[EXECUTE] Subnetwork: Deleting hpc-01-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/hpc-01-primary-subnet]. -[EXECUTE] Subnetwork: Deleting hpcimg-primary-subnet in us-central1 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/hpcimg-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-oss1-internal-address' - -Processing Subnetworks in region: us-central2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-central2 in this run. -Processing Subnetworks in region: us-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east1 in this run. -Processing Subnetworks in region: us-east4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east4 in this run. -Processing Subnetworks in region: us-east5 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east5 in this run. -Processing Subnetworks in region: us-east7 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east7 in this run. -Processing Subnetworks in region: us-south1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in us-south1 are targeted for deletion in this run: -a4newimgek-sub-1 us-south1 -hanu-test-subnet us-south1 -[EXECUTE] Subnetwork: Deleting a4newimgek-sub-1 in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/a4newimgek-sub-1]. -[EXECUTE] Subnetwork: Deleting hanu-test-subnet in us-south1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-south1/subnetworks/hanu-test-subnet]. -Processing Subnetworks in region: us-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west1 in this run. -Processing Subnetworks in region: us-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west2 in this run. -Processing Subnetworks in region: us-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west3 in this run. -Processing Subnetworks in region: us-west4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west4 in this run. -Processing Subnetworks in region: us-west8 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west8 in this run. ---- Thu Nov 27 06:00:45 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 09:31:39 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T05:31:39+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 10 per region) --- -Processing Subnetworks in region: africa-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in africa-south1 in this run. -Processing Subnetworks in region: asia-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-east1 in this run. -Processing Subnetworks in region: asia-east2 -No Subnetworks found to delete in asia-east2 in this run. -Processing Subnetworks in region: asia-northeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-northeast1 in this run. -Processing Subnetworks in region: asia-northeast2 -No Subnetworks found to delete in asia-northeast2 in this run. -Processing Subnetworks in region: asia-northeast3 -No Subnetworks found to delete in asia-northeast3 in this run. -Processing Subnetworks in region: asia-south1 -No Subnetworks found to delete in asia-south1 in this run. -Processing Subnetworks in region: asia-south2 -No Subnetworks found to delete in asia-south2 in this run. -Processing Subnetworks in region: asia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in asia-southeast1 in this run. -Processing Subnetworks in region: asia-southeast2 -No Subnetworks found to delete in asia-southeast2 in this run. -Processing Subnetworks in region: asia-southeast3 -No Subnetworks found to delete in asia-southeast3 in this run. -Processing Subnetworks in region: australia-southeast1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in australia-southeast1 in this run. -Processing Subnetworks in region: australia-southeast2 -No Subnetworks found to delete in australia-southeast2 in this run. -Processing Subnetworks in region: europe-central2 -No Subnetworks found to delete in europe-central2 in this run. -Processing Subnetworks in region: europe-north1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north1 in this run. -Processing Subnetworks in region: europe-north2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-north2 in this run. -Processing Subnetworks in region: europe-southwest1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-southwest1 in this run. -Processing Subnetworks in region: europe-west1 -WARNING: --filter : operator evaluation is changing for consistency across Google APIs. region:europe-west1 currently matches but will not match in the near future. Run `gcloud topic filters` for details. -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west1 in this run. -Processing Subnetworks in region: europe-west10 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west10 in this run. -Processing Subnetworks in region: europe-west12 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west12 in this run. -Processing Subnetworks in region: europe-west15 -No Subnetworks found to delete in europe-west15 in this run. -Processing Subnetworks in region: europe-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west2 in this run. -Processing Subnetworks in region: europe-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in europe-west3 in this run. -Processing Subnetworks in region: europe-west4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in europe-west4 are targeted for deletion in this run: -hpcdydis-primary-subnet europe-west4 -hpcdy-primary-subnet europe-west4 -[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -Processing Subnetworks in region: europe-west6 -No Subnetworks found to delete in europe-west6 in this run. -Processing Subnetworks in region: europe-west8 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west8 in this run. -Processing Subnetworks in region: europe-west9 -Skip Subnet: default (On default network) -No Subnetworks found to delete in europe-west9 in this run. -Processing Subnetworks in region: me-central1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central1 in this run. -Processing Subnetworks in region: me-central2 -Skip Subnet: default (On default network) -No Subnetworks found to delete in me-central2 in this run. -Processing Subnetworks in region: me-west1 -No Subnetworks found to delete in me-west1 in this run. -Processing Subnetworks in region: northamerica-northeast1 -No Subnetworks found to delete in northamerica-northeast1 in this run. -Processing Subnetworks in region: northamerica-northeast2 -No Subnetworks found to delete in northamerica-northeast2 in this run. -Processing Subnetworks in region: northamerica-south1 -Skip Subnet: default (On default network) -No Subnetworks found to delete in northamerica-south1 in this run. -Processing Subnetworks in region: southamerica-east1 -No Subnetworks found to delete in southamerica-east1 in this run. -Processing Subnetworks in region: southamerica-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in southamerica-west1 in this run. -Processing Subnetworks in region: us-central1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -The following Subnetworks in us-central1 are targeted for deletion in this run: -hpcimg-primary-subnet us-central1 -hpc-lustre-test-02-primary-subnet us-central1 -khu-h4d-cluster-test-primary-subnet us-central1 -khu-h4d-cluster-test-rdma-sub-0 us-central1 -lustre-06-primary-subnet us-central1 -lustre-test-06-primary-subnet us-central1 -managed-lustre-03-primary-subnet us-central1 -ml-gke-subnet us-central1 -monitoring-8323fe-primary-subnet us-central1 -sarthakagrr-primary-subnet us-central1 -[DRY RUN] Subnetwork: Would delete hpcimg-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "hpcimg-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete hpc-lustre-test-02-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "hpc-lustre-test-02-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-rdma-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete managed-lustre-03-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "managed-lustre-03-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete ml-gke-subnet in us-central1 - Command: gcloud compute networks subnets delete "ml-gke-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete monitoring-8323fe-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "monitoring-8323fe-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete sarthakagrr-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "sarthakagrr-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -Processing Subnetworks in region: us-central2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-central2 in this run. -Processing Subnetworks in region: us-east1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east1 in this run. -Processing Subnetworks in region: us-east4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east4 in this run. -Processing Subnetworks in region: us-east5 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east5 in this run. -Processing Subnetworks in region: us-east7 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-east7 in this run. -Processing Subnetworks in region: us-south1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-south1 in this run. -Processing Subnetworks in region: us-west1 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west1 in this run. -Processing Subnetworks in region: us-west2 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west2 in this run. -Processing Subnetworks in region: us-west3 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west3 in this run. -Processing Subnetworks in region: us-west4 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west4 in this run. -Processing Subnetworks in region: us-west8 -Skip Subnet: default (On default network) -Skip Subnet: hpc-vpc (In exclusion list) -No Subnetworks found to delete in us-west8 in this run. ---- Thu Nov 27 09:33:27 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 09:37:08 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 10 resources of each type per run. -Targeting resources created before: 2025-11-27T05:37:08+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 10) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 10) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 10) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 10) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 10) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 10 across all regions) --- -Skip Subnet: default in africa-south1 (On default network) -Skip Subnet: default in asia-east1 (On default network) -Skip Subnet: default in asia-northeast1 (On default network) -Skip Subnet: default in asia-southeast1 (On default network) -Skip Subnet: default in australia-southeast1 (On default network) -Skip Subnet: default in europe-north1 (On default network) -Skip Subnet: default in europe-north2 (On default network) -Skip Subnet: default in europe-southwest1 (On default network) -Skip Subnet: default in europe-west10 (On default network) -Skip Subnet: default in europe-west12 (On default network) -Skip Subnet: default in europe-west1 (On default network) -Skip Subnet: default in europe-west2 (On default network) -Skip Subnet: default in europe-west3 (On default network) -Skip Subnet: default in europe-west4 (On default network) -Skip Subnet: default in europe-west8 (On default network) -Skip Subnet: default in europe-west9 (On default network) -Skip Subnet: default in me-central1 (On default network) -Skip Subnet: default in me-central2 (On default network) -Skip Subnet: default in northamerica-south1 (On default network) -Skip Subnet: default in southamerica-west1 (On default network) -Skip Subnet: default in us-central1 (On default network) -Skip Subnet: default in us-central2 (On default network) -Skip Subnet: default in us-east1 (On default network) -Skip Subnet: default in us-east4 (On default network) -Skip Subnet: default in us-east5 (On default network) -Skip Subnet: default in us-east7 (On default network) -Skip Subnet: default in us-south1 (On default network) -Skip Subnet: default in us-west1 (On default network) -Skip Subnet: default in us-west2 (On default network) -Skip Subnet: default in us-west3 (On default network) -Skip Subnet: default in us-west4 (On default network) -Skip Subnet: default in us-west8 (On default network) -Skip Subnet: hpc-vpc in asia-east1 (In exclusion list) -Skip Subnet: hpc-vpc in asia-northeast1 (In exclusion list) -Skip Subnet: hpc-vpc in asia-southeast1 (In exclusion list) -Skip Subnet: hpc-vpc in australia-southeast1 (In exclusion list) -Skip Subnet: hpc-vpc in europe-north1 (In exclusion list) -Skip Subnet: hpc-vpc in europe-north2 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west1 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west2 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west3 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west4 (In exclusion list) -Skip Subnet: hpc-vpc in southamerica-west1 (In exclusion list) -Skip Subnet: hpc-vpc in us-central1 (In exclusion list) -Skip Subnet: hpc-vpc in us-central2 (In exclusion list) -Skip Subnet: hpc-vpc in us-east1 (In exclusion list) -Skip Subnet: hpc-vpc in us-east4 (In exclusion list) -Skip Subnet: hpc-vpc in us-east5 (In exclusion list) -Skip Subnet: hpc-vpc in us-east7 (In exclusion list) -Skip Subnet: hpc-vpc in us-south1 (In exclusion list) -Skip Subnet: hpc-vpc in us-west1 (In exclusion list) -Skip Subnet: hpc-vpc in us-west2 (In exclusion list) -Skip Subnet: hpc-vpc in us-west3 (In exclusion list) -Skip Subnet: hpc-vpc in us-west4 (In exclusion list) -Skip Subnet: hpc-vpc in us-west8 (In exclusion list) -The following Subnetworks are targeted for deletion in this run: -hpcdydis-primary-subnet europe-west4 -hpcdy-primary-subnet europe-west4 -hpcimg-primary-subnet us-central1 -hpc-lustre-test-02-primary-subnet us-central1 -khu-h4d-cluster-test-primary-subnet us-central1 -khu-h4d-cluster-test-rdma-sub-0 us-central1 -lustre-06-primary-subnet us-central1 -lustre-test-06-primary-subnet us-central1 -managed-lustre-03-primary-subnet us-central1 -ml-gke-subnet us-central1 -[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete hpcimg-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "hpcimg-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete hpc-lustre-test-02-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "hpc-lustre-test-02-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-rdma-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete managed-lustre-03-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "managed-lustre-03-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete ml-gke-subnet in us-central1 - Command: gcloud compute networks subnets delete "ml-gke-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Subnetwork Deletion Process Complete --- ---- Thu Nov 27 09:37:20 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 09:37:41 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T05:37:41+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -Skip Subnet: default in africa-south1 (On default network) -Skip Subnet: default in asia-east1 (On default network) -Skip Subnet: default in asia-northeast1 (On default network) -Skip Subnet: default in asia-southeast1 (On default network) -Skip Subnet: default in australia-southeast1 (On default network) -Skip Subnet: default in europe-north1 (On default network) -Skip Subnet: default in europe-north2 (On default network) -Skip Subnet: default in europe-southwest1 (On default network) -Skip Subnet: default in europe-west10 (On default network) -Skip Subnet: default in europe-west12 (On default network) -Skip Subnet: default in europe-west1 (On default network) -Skip Subnet: default in europe-west2 (On default network) -Skip Subnet: default in europe-west3 (On default network) -Skip Subnet: default in europe-west4 (On default network) -Skip Subnet: default in europe-west8 (On default network) -Skip Subnet: default in europe-west9 (On default network) -Skip Subnet: default in me-central1 (On default network) -Skip Subnet: default in me-central2 (On default network) -Skip Subnet: default in northamerica-south1 (On default network) -Skip Subnet: default in southamerica-west1 (On default network) -Skip Subnet: default in us-central1 (On default network) -Skip Subnet: default in us-central2 (On default network) -Skip Subnet: default in us-east1 (On default network) -Skip Subnet: default in us-east4 (On default network) -Skip Subnet: default in us-east5 (On default network) -Skip Subnet: default in us-east7 (On default network) -Skip Subnet: default in us-south1 (On default network) -Skip Subnet: default in us-west1 (On default network) -Skip Subnet: default in us-west2 (On default network) -Skip Subnet: default in us-west3 (On default network) -Skip Subnet: default in us-west4 (On default network) -Skip Subnet: default in us-west8 (On default network) -Skip Subnet: hpc-vpc in asia-east1 (In exclusion list) -Skip Subnet: hpc-vpc in asia-northeast1 (In exclusion list) -Skip Subnet: hpc-vpc in asia-southeast1 (In exclusion list) -Skip Subnet: hpc-vpc in australia-southeast1 (In exclusion list) -Skip Subnet: hpc-vpc in europe-north1 (In exclusion list) -Skip Subnet: hpc-vpc in europe-north2 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west1 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west2 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west3 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west4 (In exclusion list) -Skip Subnet: hpc-vpc in southamerica-west1 (In exclusion list) -Skip Subnet: hpc-vpc in us-central1 (In exclusion list) -Skip Subnet: hpc-vpc in us-central2 (In exclusion list) -Skip Subnet: hpc-vpc in us-east1 (In exclusion list) -Skip Subnet: hpc-vpc in us-east4 (In exclusion list) -Skip Subnet: hpc-vpc in us-east5 (In exclusion list) -Skip Subnet: hpc-vpc in us-east7 (In exclusion list) -Skip Subnet: hpc-vpc in us-south1 (In exclusion list) -Skip Subnet: hpc-vpc in us-west1 (In exclusion list) -Skip Subnet: hpc-vpc in us-west2 (In exclusion list) -Skip Subnet: hpc-vpc in us-west3 (In exclusion list) -Skip Subnet: hpc-vpc in us-west4 (In exclusion list) -Skip Subnet: hpc-vpc in us-west8 (In exclusion list) -The following Subnetworks are targeted for deletion in this run: -hpcdydis-primary-subnet europe-west4 -hpcdy-primary-subnet europe-west4 -hpcimg-primary-subnet us-central1 -hpc-lustre-test-02-primary-subnet us-central1 -khu-h4d-cluster-test-primary-subnet us-central1 -khu-h4d-cluster-test-rdma-sub-0 us-central1 -lustre-06-primary-subnet us-central1 -lustre-test-06-primary-subnet us-central1 -managed-lustre-03-primary-subnet us-central1 -ml-gke-subnet us-central1 -monitoring-8323fe-primary-subnet us-central1 -sarthakagrr-primary-subnet us-central1 -sp-helmtest1-subnet us-central1 -static-sarthakag-primary-subnet us-central1 -[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete hpcimg-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "hpcimg-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete hpc-lustre-test-02-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "hpc-lustre-test-02-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete khu-h4d-cluster-test-rdma-sub-0 in us-central1 - Command: gcloud compute networks subnets delete "khu-h4d-cluster-test-rdma-sub-0" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete managed-lustre-03-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "managed-lustre-03-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete ml-gke-subnet in us-central1 - Command: gcloud compute networks subnets delete "ml-gke-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete monitoring-8323fe-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "monitoring-8323fe-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete sarthakagrr-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "sarthakagrr-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete sp-helmtest1-subnet in us-central1 - Command: gcloud compute networks subnets delete "sp-helmtest1-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete static-sarthakag-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "static-sarthakag-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Subnetwork Deletion Process Complete --- ---- Thu Nov 27 09:37:53 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 09:38:17 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T05:38:17+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -Skip Subnet: default in africa-south1 (On default network) -Skip Subnet: default in asia-east1 (On default network) -Skip Subnet: default in asia-northeast1 (On default network) -Skip Subnet: default in asia-southeast1 (On default network) -Skip Subnet: default in australia-southeast1 (On default network) -Skip Subnet: default in europe-north1 (On default network) -Skip Subnet: default in europe-north2 (On default network) -Skip Subnet: default in europe-southwest1 (On default network) -Skip Subnet: default in europe-west10 (On default network) -Skip Subnet: default in europe-west12 (On default network) -Skip Subnet: default in europe-west1 (On default network) -Skip Subnet: default in europe-west2 (On default network) -Skip Subnet: default in europe-west3 (On default network) -Skip Subnet: default in europe-west4 (On default network) -Skip Subnet: default in europe-west8 (On default network) -Skip Subnet: default in europe-west9 (On default network) -Skip Subnet: default in me-central1 (On default network) -Skip Subnet: default in me-central2 (On default network) -Skip Subnet: default in northamerica-south1 (On default network) -Skip Subnet: default in southamerica-west1 (On default network) -Skip Subnet: default in us-central1 (On default network) -Skip Subnet: default in us-central2 (On default network) -Skip Subnet: default in us-east1 (On default network) -Skip Subnet: default in us-east4 (On default network) -Skip Subnet: default in us-east5 (On default network) -Skip Subnet: default in us-east7 (On default network) -Skip Subnet: default in us-south1 (On default network) -Skip Subnet: default in us-west1 (On default network) -Skip Subnet: default in us-west2 (On default network) -Skip Subnet: default in us-west3 (On default network) -Skip Subnet: default in us-west4 (On default network) -Skip Subnet: default in us-west8 (On default network) -Skip Subnet: hpc-vpc in asia-east1 (In exclusion list) -Skip Subnet: hpc-vpc in asia-northeast1 (In exclusion list) -Skip Subnet: hpc-vpc in asia-southeast1 (In exclusion list) -Skip Subnet: hpc-vpc in australia-southeast1 (In exclusion list) -Skip Subnet: hpc-vpc in europe-north1 (In exclusion list) -Skip Subnet: hpc-vpc in europe-north2 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west1 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west2 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west3 (In exclusion list) -Skip Subnet: hpc-vpc in europe-west4 (In exclusion list) -Skip Subnet: hpc-vpc in southamerica-west1 (In exclusion list) -Skip Subnet: hpc-vpc in us-central1 (In exclusion list) -Skip Subnet: hpc-vpc in us-central2 (In exclusion list) -Skip Subnet: hpc-vpc in us-east1 (In exclusion list) -Skip Subnet: hpc-vpc in us-east4 (In exclusion list) -Skip Subnet: hpc-vpc in us-east5 (In exclusion list) -Skip Subnet: hpc-vpc in us-east7 (In exclusion list) -Skip Subnet: hpc-vpc in us-south1 (In exclusion list) -Skip Subnet: hpc-vpc in us-west1 (In exclusion list) -Skip Subnet: hpc-vpc in us-west2 (In exclusion list) -Skip Subnet: hpc-vpc in us-west3 (In exclusion list) -Skip Subnet: hpc-vpc in us-west4 (In exclusion list) -Skip Subnet: hpc-vpc in us-west8 (In exclusion list) -The following Subnetworks are targeted for deletion in this run: -hpcdydis-primary-subnet europe-west4 -hpcdy-primary-subnet europe-west4 -hpcimg-primary-subnet us-central1 -hpc-lustre-test-02-primary-subnet us-central1 -khu-h4d-cluster-test-primary-subnet us-central1 -khu-h4d-cluster-test-rdma-sub-0 us-central1 -lustre-06-primary-subnet us-central1 -lustre-test-06-primary-subnet us-central1 -managed-lustre-03-primary-subnet us-central1 -ml-gke-subnet us-central1 -monitoring-8323fe-primary-subnet us-central1 -sarthakagrr-primary-subnet us-central1 -sp-helmtest1-subnet us-central1 -static-sarthakag-primary-subnet us-central1 -[EXECUTE] Subnetwork: Deleting hpcdydis-primary-subnet in europe-west4 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdydis-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-mds0-internal-address' - -ERROR: Failed to delete Subnetwork hpcdydis-primary-subnet in europe-west4 -[EXECUTE] Subnetwork: Deleting hpcdy-primary-subnet in europe-west4 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdy-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss2-internal-address' - -ERROR: Failed to delete Subnetwork hpcdy-primary-subnet in europe-west4 -[EXECUTE] Subnetwork: Deleting hpcimg-primary-subnet in us-central1 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/hpcimg-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-oss1-internal-address' - -ERROR: Failed to delete Subnetwork hpcimg-primary-subnet in us-central1 -[EXECUTE] Subnetwork: Deleting hpc-lustre-test-02-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/hpc-lustre-test-02-primary-subnet]. -[EXECUTE] Subnetwork: Deleting khu-h4d-cluster-test-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/khu-h4d-cluster-test-primary-subnet]. -[EXECUTE] Subnetwork: Deleting khu-h4d-cluster-test-rdma-sub-0 in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/khu-h4d-cluster-test-rdma-sub-0]. -[EXECUTE] Subnetwork: Deleting lustre-06-primary-subnet in us-central1 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustre06-slurm-login-001' - -ERROR: Failed to delete Subnetwork lustre-06-primary-subnet in us-central1 -[EXECUTE] Subnetwork: Deleting lustre-test-06-primary-subnet in us-central1 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-test-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustretest-controller' - -ERROR: Failed to delete Subnetwork lustre-test-06-primary-subnet in us-central1 -[EXECUTE] Subnetwork: Deleting managed-lustre-03-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/managed-lustre-03-primary-subnet]. -[EXECUTE] Subnetwork: Deleting ml-gke-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/ml-gke-subnet]. -[EXECUTE] Subnetwork: Deleting monitoring-8323fe-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/monitoring-8323fe-primary-subnet]. -[EXECUTE] Subnetwork: Deleting sarthakagrr-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/sarthakagrr-primary-subnet]. -[EXECUTE] Subnetwork: Deleting sp-helmtest1-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/sp-helmtest1-subnet]. -[EXECUTE] Subnetwork: Deleting static-sarthakag-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/static-sarthakag-primary-subnet]. ---- Subnetwork Deletion Process Complete --- ---- Thu Nov 27 09:41:11 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 09:42:33 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: true -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T05:42:33+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -hpcdydis-primary-subnet in europe-west4 -hpcdy-primary-subnet in europe-west4 -hpcimg-primary-subnet in us-central1 -lustre-06-primary-subnet in us-central1 -lustre-test-06-primary-subnet in us-central1 ---- Processing Subnet: hpcdydis-primary-subnet in europe-west4 --- -Found dependent addresses for hpcdydis-primary-subnet in europe-west4: -exascaler-cloud-a2a0-mds0-internal-address -exascaler-cloud-a2a0-mgs0-internal-address -exascaler-cloud-a2a0-oss0-internal-address -exascaler-cloud-a2a0-oss1-internal-address -exascaler-cloud-a2a0-oss2-internal-address -[DRY RUN] Address: Would delete exascaler-cloud-a2a0-mds0-internal-address in europe-west4 - Command: gcloud compute addresses delete "exascaler-cloud-a2a0-mds0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-a2a0-mgs0-internal-address in europe-west4 - Command: gcloud compute addresses delete "exascaler-cloud-a2a0-mgs0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-a2a0-oss0-internal-address in europe-west4 - Command: gcloud compute addresses delete "exascaler-cloud-a2a0-oss0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-a2a0-oss1-internal-address in europe-west4 - Command: gcloud compute addresses delete "exascaler-cloud-a2a0-oss1-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-a2a0-oss2-internal-address in europe-west4 - Command: gcloud compute addresses delete "exascaler-cloud-a2a0-oss2-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete hpcdydis-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdydis-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet ---- Processing Subnet: hpcdy-primary-subnet in europe-west4 --- -Found dependent addresses for hpcdy-primary-subnet in europe-west4: -exascaler-cloud-4a36-mds0-internal-address -exascaler-cloud-4a36-mgs0-internal-address -exascaler-cloud-4a36-oss0-internal-address -exascaler-cloud-4a36-oss1-internal-address -exascaler-cloud-4a36-oss2-internal-address -[DRY RUN] Address: Would delete exascaler-cloud-4a36-mds0-internal-address in europe-west4 - Command: gcloud compute addresses delete "exascaler-cloud-4a36-mds0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-4a36-mgs0-internal-address in europe-west4 - Command: gcloud compute addresses delete "exascaler-cloud-4a36-mgs0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-4a36-oss0-internal-address in europe-west4 - Command: gcloud compute addresses delete "exascaler-cloud-4a36-oss0-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-4a36-oss1-internal-address in europe-west4 - Command: gcloud compute addresses delete "exascaler-cloud-4a36-oss1-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-4a36-oss2-internal-address in europe-west4 - Command: gcloud compute addresses delete "exascaler-cloud-4a36-oss2-internal-address" --project="hpc-toolkit-dev" --region="europe-west4" --quiet -[DRY RUN] Subnetwork: Would delete hpcdy-primary-subnet in europe-west4 - Command: gcloud compute networks subnets delete "hpcdy-primary-subnet" --project="hpc-toolkit-dev" --region="europe-west4" --quiet ---- Processing Subnet: hpcimg-primary-subnet in us-central1 --- -Found dependent addresses for hpcimg-primary-subnet in us-central1: -exascaler-cloud-4691-mds0-internal-address -exascaler-cloud-4691-mgs0-internal-address -exascaler-cloud-4691-oss0-internal-address -exascaler-cloud-4691-oss1-internal-address -exascaler-cloud-4691-oss2-internal-address -[DRY RUN] Address: Would delete exascaler-cloud-4691-mds0-internal-address in us-central1 - Command: gcloud compute addresses delete "exascaler-cloud-4691-mds0-internal-address" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-4691-mgs0-internal-address in us-central1 - Command: gcloud compute addresses delete "exascaler-cloud-4691-mgs0-internal-address" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-4691-oss0-internal-address in us-central1 - Command: gcloud compute addresses delete "exascaler-cloud-4691-oss0-internal-address" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-4691-oss1-internal-address in us-central1 - Command: gcloud compute addresses delete "exascaler-cloud-4691-oss1-internal-address" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Address: Would delete exascaler-cloud-4691-oss2-internal-address in us-central1 - Command: gcloud compute addresses delete "exascaler-cloud-4691-oss2-internal-address" --project="hpc-toolkit-dev" --region="us-central1" --quiet -[DRY RUN] Subnetwork: Would delete hpcimg-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "hpcimg-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: lustre-06-primary-subnet in us-central1 --- -No dependent addresses found for lustre-06-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete lustre-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- -No dependent addresses found for lustre-test-06-primary-subnet in us-central1. -[DRY RUN] Subnetwork: Would delete lustre-test-06-primary-subnet in us-central1 - Command: gcloud compute networks subnets delete "lustre-test-06-primary-subnet" --project="hpc-toolkit-dev" --region="us-central1" --quiet ---- Thu Nov 27 09:42:54 AM UTC 2025 --- Cleanup Script Run Finished --- - ---- Thu Nov 27 09:44:01 AM UTC 2025 --- STARTING RESOURCE CLEANUP in project hpc-toolkit-dev --- -DRY_RUN mode: false -Will attempt to delete up to 30 resources of each type per run. -Targeting resources created before: 2025-11-27T05:44:01+0000 -Reading exclusion list from local file: exclusions.txt -Exclusion list loaded with 60 entries: - - vertexui-do-not-kill - - hpc-ctk1357 - - hpc-toolkit-dev@appspot.gserviceaccount.com - - build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com - - cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com - - 508417052821-compute@developer.gserviceaccount.com - - hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com - - htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com - - pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com - - test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com - - telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - vertexui-do-not-kill-boot - - vertexui-do-not-kill-data - - lustretest-controller - - lustretest-slurm-login-001 - - lustre06-controller - - lustre06-slurm-login-001 - - default - - mglsard - - default-router-us-west1 - - default-router-us-west4 - - default-net-router - - default-router-australia-southeast1 - - default-router-us-east4 - - image-inspector-550 - - image-inspector - - lustre-06-net-router - - lustre-test-06-net-router - - mglsa-net-router - - mglsard-net-router - - gke-managed-lustre-basic-net-fw-allow-iap-ingress - - gke-managed-lustre-basic-net-fw-allow-internal-traffic - - lustre-06-net-fw-allow-iap-ingress - - lustre-06-net-fw-allow-internal-traffic - - lustre-test-06-net-fw-allow-iap-ingress - - lustre-test-06-net-fw-allow-internal-traffic - - a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com - - hpc-vpc ---- Deletion Phase 1: GKE Clusters (Top 30) --- -No GKE clusters found to delete in this run. ---- Deletion Phase 1: Compute Instances (Top 30) --- -Skip Instance: image-inspector-550 in us-west1-a (In exclusion list) -Skip Instance: image-inspector in us-west1-a (In exclusion list) -Skip Instance: lustre06-controller in us-central1-a (In exclusion list) -Skip Instance: lustre06-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: lustretest-controller in us-central1-a (In exclusion list) -Skip Instance: lustretest-slurm-login-001 in us-central1-a (In exclusion list) -Skip Instance: vertexui-do-not-kill in us-west1-b (In exclusion list) -No Instances found to delete in this run. ---- Deletion Phase 1: Filestore Instances (Top 30) --- -No Filestore instances found to delete in this run. ---- Deletion Phase 3: Cloud Routers (Top 30) --- -Skip Cloud Router: default-net-router (In exclusion list) -Skip Cloud Router: default-router-australia-southeast1 (In exclusion list) -Skip Cloud Router: default-router-us-east4 (In exclusion list) -Skip Cloud Router: default-router-us-west1 (In exclusion list) -Skip Cloud Router: default-router-us-west4 (In exclusion list) -Skip Cloud Router: lustre-06-net-router (In exclusion list) -Skip Cloud Router: lustre-test-06-net-router (In exclusion list) -Skip Cloud Router: mglsa-net-router (In exclusion list) -Skip Cloud Router: mglsard-net-router (In exclusion list) -No Cloud Routers found to delete in this run. ---- Deletion Phase 4: Firewall Rules (Top 30) --- -Skip Firewall: a3hc-308e7e (On default network) -Skip Firewall: a3hc-754952 (On default network) -Skip Firewall: a3hc-a628c1 (On default network) -Skip Firewall: a3hc-b15fa8 (On default network) -Skip Firewall: a3hc-c27fd0 (On default network) -Skip Firewall: a3hc-df0061 (On default network) -Skip Firewall: a3mc-c9f69e (On default network) -Skip Firewall: a3mc-f7e2b2 (On default network) -Skip Firewall: allow-internal (On default network) -Skip Firewall: allow-ssh (On default network) -Skip Firewall: default-allow-http (On default network) -Skip Firewall: default-allow-https (On default network) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: gke-managed-lustre-basic-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-06-net-fw-allow-internal-traffic (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-iap-ingress (In exclusion list) -Skip Firewall: lustre-test-06-net-fw-allow-internal-traffic (In exclusion list) -No Firewall Rules found to delete in this run. ---- Deletion Phase 5: Subnetworks (Top 30 across all regions) --- -The following Subnetworks (and their dependent addresses) are targeted for deletion: -hpcdydis-primary-subnet in europe-west4 -hpcdy-primary-subnet in europe-west4 -hpcimg-primary-subnet in us-central1 -lustre-06-primary-subnet in us-central1 -lustre-test-06-primary-subnet in us-central1 ---- Processing Subnet: hpcdydis-primary-subnet in europe-west4 --- -Found dependent addresses for hpcdydis-primary-subnet in europe-west4: -exascaler-cloud-a2a0-mds0-internal-address -exascaler-cloud-a2a0-mgs0-internal-address -exascaler-cloud-a2a0-oss0-internal-address -exascaler-cloud-a2a0-oss1-internal-address -exascaler-cloud-a2a0-oss2-internal-address -[EXECUTE] Address: Deleting exascaler-cloud-a2a0-mds0-internal-address in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-mds0-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-a2a0-mgs0-internal-address in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-mgs0-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-a2a0-oss0-internal-address in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-oss0-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-a2a0-oss1-internal-address in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-oss1-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-a2a0-oss2-internal-address in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-a2a0-oss2-internal-address]. -[EXECUTE] Subnetwork: Deleting hpcdydis-primary-subnet in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdydis-primary-subnet]. -Successfully deleted Subnetwork hpcdydis-primary-subnet in europe-west4. ---- Processing Subnet: hpcdy-primary-subnet in europe-west4 --- -Found dependent addresses for hpcdy-primary-subnet in europe-west4: -exascaler-cloud-4a36-mds0-internal-address -exascaler-cloud-4a36-mgs0-internal-address -exascaler-cloud-4a36-oss0-internal-address -exascaler-cloud-4a36-oss1-internal-address -exascaler-cloud-4a36-oss2-internal-address -[EXECUTE] Address: Deleting exascaler-cloud-4a36-mds0-internal-address in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-mds0-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-4a36-mgs0-internal-address in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-mgs0-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-4a36-oss0-internal-address in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss0-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-4a36-oss1-internal-address in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss1-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-4a36-oss2-internal-address in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/addresses/exascaler-cloud-4a36-oss2-internal-address]. -[EXECUTE] Subnetwork: Deleting hpcdy-primary-subnet in europe-west4 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/europe-west4/subnetworks/hpcdy-primary-subnet]. -Successfully deleted Subnetwork hpcdy-primary-subnet in europe-west4. ---- Processing Subnet: hpcimg-primary-subnet in us-central1 --- -Found dependent addresses for hpcimg-primary-subnet in us-central1: -exascaler-cloud-4691-mds0-internal-address -exascaler-cloud-4691-mgs0-internal-address -exascaler-cloud-4691-oss0-internal-address -exascaler-cloud-4691-oss1-internal-address -exascaler-cloud-4691-oss2-internal-address -[EXECUTE] Address: Deleting exascaler-cloud-4691-mds0-internal-address in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-mds0-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-4691-mgs0-internal-address in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-mgs0-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-4691-oss0-internal-address in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-oss0-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-4691-oss1-internal-address in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-oss1-internal-address]. -[EXECUTE] Address: Deleting exascaler-cloud-4691-oss2-internal-address in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/addresses/exascaler-cloud-4691-oss2-internal-address]. -[EXECUTE] Subnetwork: Deleting hpcimg-primary-subnet in us-central1 -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/regions/us-central1/subnetworks/hpcimg-primary-subnet]. -Successfully deleted Subnetwork hpcimg-primary-subnet in us-central1. ---- Processing Subnet: lustre-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-06-primary-subnet in us-central1. -[EXECUTE] Subnetwork: Deleting lustre-06-primary-subnet in us-central1 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustre06-slurm-login-001' - -ERROR: Failed to delete Subnetwork lustre-06-primary-subnet in us-central1. Check for other dependencies. ---- Processing Subnet: lustre-test-06-primary-subnet in us-central1 --- -WARNING: The following filter keys were not present in any resource : purpose, subnetwork -No dependent addresses found for lustre-test-06-primary-subnet in us-central1. -[EXECUTE] Subnetwork: Deleting lustre-test-06-primary-subnet in us-central1 -ERROR: (gcloud.compute.networks.subnets.delete) Could not fetch resource: - - The subnetwork resource 'projects/hpc-toolkit-dev/regions/us-central1/subnetworks/lustre-test-06-primary-subnet' is already being used by 'projects/hpc-toolkit-dev/zones/us-central1-a/instances/lustretest-controller' - -ERROR: Failed to delete Subnetwork lustre-test-06-primary-subnet in us-central1. Check for other dependencies. ---- Thu Nov 27 09:46:30 AM UTC 2025 --- Cleanup Script Run Finished --- - diff --git a/template.txt b/template.txt deleted file mode 100644 index e1e7eb1964..0000000000 --- a/template.txt +++ /dev/null @@ -1,4227 +0,0 @@ -[2025-11-30 18:41:02] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:41:02] [INFO] Time Cutoff (General): 2025-11-30T18:41:02+0000 -[2025-11-30 18:41:02] [INFO] Time Cutoff (Images): 2025-10-01T18:41:02+0000 -[2025-11-30 18:41:02] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:41:02] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:41:02] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:41:04] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:41:04] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:41:06] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:41:06] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a163acslur-compute-nodeset-20251013073728349400000002 (Global) -[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a2acslurmf-compute-nodeset-20250919234302964700000001 (Global) -[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3h23d4-compute-a3nodeset-20251114063711441000000003 (Global) -[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3h23d4-controller-default-20251114063701192000000002 (Global) -[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3h23d4-login-login-20251114063653023800000001 (Global) -[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3hca628-compute-a3nodeset-20251017102226551000000004 (Global) -[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3hca628-compute-debugnodeset-20251017102226507900000003 (Global) -[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3hca628-controller-default-20251017102143258500000002 (Global) -[2025-11-30 18:41:09] [DRY-RUN] Would delete Instance Template: a3hca628-login-login-20251017102143197700000001 (Global) -[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcb15f-compute-a3nodeset-20251116065432482600000003 (Global) -[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcb15f-compute-debugnodeset-20251116065432486000000004 (Global) -[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcb15f-controller-default-20251116065340384400000001 (Global) -[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcb15f-login-login-20251116065340427400000002 (Global) -[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcdf00-compute-a3nodeset-20251115185843671200000004 (Global) -[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcdf00-compute-debugnodeset-20251115185843634300000003 (Global) -[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcdf00-controller-default-20251115185751172200000001 (Global) -[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3hcdf00-login-login-20251115185752052600000002 (Global) -[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3lavnew-compute-a3nodeset-20251010141633773500000003 (Global) -[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3lavnew-compute-debugnodeset-20251010141633744500000001 (Global) -[2025-11-30 18:41:10] [DRY-RUN] Would delete Instance Template: a3lavnew-controller-default-20251010141643756900000004 (Global) -[2025-11-30 18:41:10] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:41:10] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:41:12] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:41:12] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:41:12] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:41:12] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:41:15] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:41:15] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:41:18] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:41:18] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:41:19] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:41:19] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:41:19] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:41:19] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:41:19] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:41:19] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:41:19] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:41:19] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:41:19] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:41:19] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:41:19Z (Unix: 1763318479) -[2025-11-30 18:41:19] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:41:21] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:41:21] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 18:41:24] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:41:24] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:41:24] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:41:24] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:41:24] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:41:24] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 18:41:26] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:41:26] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:41:29] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:41:29] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:41:31] [INFO] No Global Address found matching criteria. -[2025-11-30 18:41:31] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 18:41:33] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:41:33] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 18:41:36] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:41:36] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:41:36] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:41:36] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:41:36] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:39] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:39] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 18:41:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:41:41] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 18:41:43] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:41:43] [INFO] CLEANUP RUN FINISHED -[2025-11-30 18:42:34] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:42:34] [INFO] Time Cutoff (General): 2025-11-30T18:42:34+0000 -[2025-11-30 18:42:34] [INFO] Time Cutoff (Images): 2025-10-01T18:42:34+0000 -[2025-11-30 18:42:34] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:42:34] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:42:34] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:42:37] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:42:37] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:42:38] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:42:38] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:42:42] [EXECUTE] Deleting Instance Template: a163acslur-compute-nodeset-20251013073728349400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a163acslur-compute-nodeset-20251013073728349400000002]. -[2025-11-30 18:42:45] [SUCCESS] Deleted a163acslur-compute-nodeset-20251013073728349400000002 -[2025-11-30 18:42:45] [EXECUTE] Deleting Instance Template: a2acslurmf-compute-nodeset-20250919234302964700000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a2acslurmf-compute-nodeset-20250919234302964700000001]. -[2025-11-30 18:42:48] [SUCCESS] Deleted a2acslurmf-compute-nodeset-20250919234302964700000001 -[2025-11-30 18:42:48] [EXECUTE] Deleting Instance Template: a3h23d4-compute-a3nodeset-20251114063711441000000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3h23d4-compute-a3nodeset-20251114063711441000000003]. -[2025-11-30 18:42:51] [SUCCESS] Deleted a3h23d4-compute-a3nodeset-20251114063711441000000003 -[2025-11-30 18:42:51] [EXECUTE] Deleting Instance Template: a3h23d4-controller-default-20251114063701192000000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3h23d4-controller-default-20251114063701192000000002]. -[2025-11-30 18:42:54] [SUCCESS] Deleted a3h23d4-controller-default-20251114063701192000000002 -[2025-11-30 18:42:54] [EXECUTE] Deleting Instance Template: a3h23d4-login-login-20251114063653023800000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3h23d4-login-login-20251114063653023800000001]. -[2025-11-30 18:42:57] [SUCCESS] Deleted a3h23d4-login-login-20251114063653023800000001 -[2025-11-30 18:42:57] [EXECUTE] Deleting Instance Template: a3hca628-compute-a3nodeset-20251017102226551000000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hca628-compute-a3nodeset-20251017102226551000000004]. -[2025-11-30 18:43:00] [SUCCESS] Deleted a3hca628-compute-a3nodeset-20251017102226551000000004 -[2025-11-30 18:43:00] [EXECUTE] Deleting Instance Template: a3hca628-compute-debugnodeset-20251017102226507900000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hca628-compute-debugnodeset-20251017102226507900000003]. -[2025-11-30 18:43:04] [SUCCESS] Deleted a3hca628-compute-debugnodeset-20251017102226507900000003 -[2025-11-30 18:43:04] [EXECUTE] Deleting Instance Template: a3hca628-controller-default-20251017102143258500000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hca628-controller-default-20251017102143258500000002]. -[2025-11-30 18:43:07] [SUCCESS] Deleted a3hca628-controller-default-20251017102143258500000002 -[2025-11-30 18:43:07] [EXECUTE] Deleting Instance Template: a3hca628-login-login-20251017102143197700000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hca628-login-login-20251017102143197700000001]. -[2025-11-30 18:43:10] [SUCCESS] Deleted a3hca628-login-login-20251017102143197700000001 -[2025-11-30 18:43:10] [EXECUTE] Deleting Instance Template: a3hcb15f-compute-a3nodeset-20251116065432482600000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcb15f-compute-a3nodeset-20251116065432482600000003]. -[2025-11-30 18:43:13] [SUCCESS] Deleted a3hcb15f-compute-a3nodeset-20251116065432482600000003 -[2025-11-30 18:43:13] [EXECUTE] Deleting Instance Template: a3hcb15f-compute-debugnodeset-20251116065432486000000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcb15f-compute-debugnodeset-20251116065432486000000004]. -[2025-11-30 18:43:17] [SUCCESS] Deleted a3hcb15f-compute-debugnodeset-20251116065432486000000004 -[2025-11-30 18:43:17] [EXECUTE] Deleting Instance Template: a3hcb15f-controller-default-20251116065340384400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcb15f-controller-default-20251116065340384400000001]. -[2025-11-30 18:43:20] [SUCCESS] Deleted a3hcb15f-controller-default-20251116065340384400000001 -[2025-11-30 18:43:20] [EXECUTE] Deleting Instance Template: a3hcb15f-login-login-20251116065340427400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcb15f-login-login-20251116065340427400000002]. -[2025-11-30 18:43:22] [SUCCESS] Deleted a3hcb15f-login-login-20251116065340427400000002 -[2025-11-30 18:43:22] [EXECUTE] Deleting Instance Template: a3hcdf00-compute-a3nodeset-20251115185843671200000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcdf00-compute-a3nodeset-20251115185843671200000004]. -[2025-11-30 18:43:26] [SUCCESS] Deleted a3hcdf00-compute-a3nodeset-20251115185843671200000004 -[2025-11-30 18:43:26] [EXECUTE] Deleting Instance Template: a3hcdf00-compute-debugnodeset-20251115185843634300000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcdf00-compute-debugnodeset-20251115185843634300000003]. -[2025-11-30 18:43:29] [SUCCESS] Deleted a3hcdf00-compute-debugnodeset-20251115185843634300000003 -[2025-11-30 18:43:29] [EXECUTE] Deleting Instance Template: a3hcdf00-controller-default-20251115185751172200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcdf00-controller-default-20251115185751172200000001]. -[2025-11-30 18:43:32] [SUCCESS] Deleted a3hcdf00-controller-default-20251115185751172200000001 -[2025-11-30 18:43:32] [EXECUTE] Deleting Instance Template: a3hcdf00-login-login-20251115185752052600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3hcdf00-login-login-20251115185752052600000002]. -[2025-11-30 18:43:35] [SUCCESS] Deleted a3hcdf00-login-login-20251115185752052600000002 -[2025-11-30 18:43:35] [EXECUTE] Deleting Instance Template: a3lavnew-compute-a3nodeset-20251010141633773500000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3lavnew-compute-a3nodeset-20251010141633773500000003]. -[2025-11-30 18:43:38] [SUCCESS] Deleted a3lavnew-compute-a3nodeset-20251010141633773500000003 -[2025-11-30 18:43:38] [EXECUTE] Deleting Instance Template: a3lavnew-compute-debugnodeset-20251010141633744500000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3lavnew-compute-debugnodeset-20251010141633744500000001]. -[2025-11-30 18:43:42] [SUCCESS] Deleted a3lavnew-compute-debugnodeset-20251010141633744500000001 -[2025-11-30 18:43:42] [EXECUTE] Deleting Instance Template: a3lavnew-controller-default-20251010141643756900000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3lavnew-controller-default-20251010141643756900000004]. -[2025-11-30 18:43:45] [SUCCESS] Deleted a3lavnew-controller-default-20251010141643756900000004 -[2025-11-30 18:43:45] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:43:45] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:43:47] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:43:47] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:43:47] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:43:47] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:43:50] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:43:50] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:43:53] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:43:53] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:43:53] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:43:53] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:43:53] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:43:53] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:43:53] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:43:53] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:43:54] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:43:54] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:43:54] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:43:54] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:43:54Z (Unix: 1763318634) -[2025-11-30 18:43:54] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:43:56] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:43:56] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 18:43:59] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:43:59] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:43:59] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:43:59] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:43:59] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:43:59] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 18:44:01] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:44:01] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:44:03] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:44:03] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:44:06] [INFO] No Global Address found matching criteria. -[2025-11-30 18:44:06] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 18:44:08] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:44:08] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 18:44:11] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:44:11] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:44:11] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:44:11] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:44:11] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:14] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:14] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 18:44:16] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:44:16] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 18:44:18] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:44:18] [INFO] CLEANUP RUN FINISHED -[2025-11-30 18:44:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:44:29] [INFO] Time Cutoff (General): 2025-11-30T18:44:29+0000 -[2025-11-30 18:44:29] [INFO] Time Cutoff (Images): 2025-10-01T18:44:29+0000 -[2025-11-30 18:44:29] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:44:29] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:44:29] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:44:31] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:44:31] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:44:33] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:44:33] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3lavnew-login-login-20251010141633747300000002 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m14b2-compute-a3meganodeset-20251024053708898700000004 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m14b2-compute-debugnodeset-20251024053708872100000003 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m14b2-controller-default-20251024053548842100000002 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m14b2-login-login-20251024053548834500000001 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1b32-compute-a3meganodeset-20251023134637080900000004 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1b32-compute-debugnodeset-20251023134637047400000003 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1b32-controller-default-20251023134613677700000001 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1b32-login-login-20251023134613712600000002 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1bc7-compute-a3meganodeset-20251114062112120000000004 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1bc7-compute-debugnodeset-20251114062112091900000003 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1bc7-controller-default-20251114061949993400000001 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m1bc7-login-login-20251114061950086100000002 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m2c1a-compute-a3meganodeset-20250828212541355600000004 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m2c1a-compute-debugnodeset-20250828212541351800000003 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m2c1a-controller-default-20250828212418057400000002 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m2c1a-login-login-20250828212416517200000001 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m3500-compute-a3meganodeset-20251114015749275800000004 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m3500-compute-debugnodeset-20251114015749253600000003 (Global) -[2025-11-30 18:44:36] [DRY-RUN] Would delete Instance Template: a3m3500-controller-default-20251114015726857500000001 (Global) -[2025-11-30 18:44:36] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:44:36] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:44:39] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:44:39] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:44:39] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:44:39] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:44:42] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:44:42] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:44:44] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:44:45] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:44:45] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:44:45] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:44:45] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:44:45] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:44:45] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:44:45] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:44:45] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:44:45] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:44:45] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:44:45] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:44:45Z (Unix: 1763318685) -[2025-11-30 18:44:45] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:44:48] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:44:48] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 18:44:50] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:44:50] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:44:50] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:44:50] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:44:50] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:44:50] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 18:44:52] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:44:52] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:44:55] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:44:55] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:44:57] [INFO] No Global Address found matching criteria. -[2025-11-30 18:44:57] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 18:45:00] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:45:00] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 18:45:02] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:45:02] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:45:02] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:45:02] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:45:02] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:05] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 18:45:07] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:45:07] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 18:45:09] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:45:09] [INFO] CLEANUP RUN FINISHED -[2025-11-30 18:45:40] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:45:40] [INFO] Time Cutoff (General): 2025-11-30T18:45:40+0000 -[2025-11-30 18:45:40] [INFO] Time Cutoff (Images): 2025-10-01T18:45:40+0000 -[2025-11-30 18:45:40] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:45:40] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:45:40] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:45:43] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:45:43] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:45:45] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:45:45] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:45:48] [EXECUTE] Deleting Instance Template: a3lavnew-login-login-20251010141633747300000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3lavnew-login-login-20251010141633747300000002]. -[2025-11-30 18:45:51] [SUCCESS] Deleted a3lavnew-login-login-20251010141633747300000002 -[2025-11-30 18:45:51] [EXECUTE] Deleting Instance Template: a3m14b2-compute-a3meganodeset-20251024053708898700000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m14b2-compute-a3meganodeset-20251024053708898700000004]. -[2025-11-30 18:45:54] [SUCCESS] Deleted a3m14b2-compute-a3meganodeset-20251024053708898700000004 -[2025-11-30 18:45:54] [EXECUTE] Deleting Instance Template: a3m14b2-compute-debugnodeset-20251024053708872100000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m14b2-compute-debugnodeset-20251024053708872100000003]. -[2025-11-30 18:45:57] [SUCCESS] Deleted a3m14b2-compute-debugnodeset-20251024053708872100000003 -[2025-11-30 18:45:57] [EXECUTE] Deleting Instance Template: a3m14b2-controller-default-20251024053548842100000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m14b2-controller-default-20251024053548842100000002]. -[2025-11-30 18:46:00] [SUCCESS] Deleted a3m14b2-controller-default-20251024053548842100000002 -[2025-11-30 18:46:00] [EXECUTE] Deleting Instance Template: a3m14b2-login-login-20251024053548834500000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m14b2-login-login-20251024053548834500000001]. -[2025-11-30 18:46:03] [SUCCESS] Deleted a3m14b2-login-login-20251024053548834500000001 -[2025-11-30 18:46:03] [EXECUTE] Deleting Instance Template: a3m1b32-compute-a3meganodeset-20251023134637080900000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1b32-compute-a3meganodeset-20251023134637080900000004]. -[2025-11-30 18:46:06] [SUCCESS] Deleted a3m1b32-compute-a3meganodeset-20251023134637080900000004 -[2025-11-30 18:46:06] [EXECUTE] Deleting Instance Template: a3m1b32-compute-debugnodeset-20251023134637047400000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1b32-compute-debugnodeset-20251023134637047400000003]. -[2025-11-30 18:46:09] [SUCCESS] Deleted a3m1b32-compute-debugnodeset-20251023134637047400000003 -[2025-11-30 18:46:09] [EXECUTE] Deleting Instance Template: a3m1b32-controller-default-20251023134613677700000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1b32-controller-default-20251023134613677700000001]. -[2025-11-30 18:46:12] [SUCCESS] Deleted a3m1b32-controller-default-20251023134613677700000001 -[2025-11-30 18:46:12] [EXECUTE] Deleting Instance Template: a3m1b32-login-login-20251023134613712600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1b32-login-login-20251023134613712600000002]. -[2025-11-30 18:46:15] [SUCCESS] Deleted a3m1b32-login-login-20251023134613712600000002 -[2025-11-30 18:46:15] [EXECUTE] Deleting Instance Template: a3m1bc7-compute-a3meganodeset-20251114062112120000000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1bc7-compute-a3meganodeset-20251114062112120000000004]. -[2025-11-30 18:46:18] [SUCCESS] Deleted a3m1bc7-compute-a3meganodeset-20251114062112120000000004 -[2025-11-30 18:46:18] [EXECUTE] Deleting Instance Template: a3m1bc7-compute-debugnodeset-20251114062112091900000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1bc7-compute-debugnodeset-20251114062112091900000003]. -[2025-11-30 18:46:21] [SUCCESS] Deleted a3m1bc7-compute-debugnodeset-20251114062112091900000003 -[2025-11-30 18:46:21] [EXECUTE] Deleting Instance Template: a3m1bc7-controller-default-20251114061949993400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1bc7-controller-default-20251114061949993400000001]. -[2025-11-30 18:46:24] [SUCCESS] Deleted a3m1bc7-controller-default-20251114061949993400000001 -[2025-11-30 18:46:24] [EXECUTE] Deleting Instance Template: a3m1bc7-login-login-20251114061950086100000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m1bc7-login-login-20251114061950086100000002]. -[2025-11-30 18:46:28] [SUCCESS] Deleted a3m1bc7-login-login-20251114061950086100000002 -[2025-11-30 18:46:28] [EXECUTE] Deleting Instance Template: a3m2c1a-compute-a3meganodeset-20250828212541355600000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m2c1a-compute-a3meganodeset-20250828212541355600000004]. -[2025-11-30 18:46:31] [SUCCESS] Deleted a3m2c1a-compute-a3meganodeset-20250828212541355600000004 -[2025-11-30 18:46:31] [EXECUTE] Deleting Instance Template: a3m2c1a-compute-debugnodeset-20250828212541351800000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m2c1a-compute-debugnodeset-20250828212541351800000003]. -[2025-11-30 18:46:34] [SUCCESS] Deleted a3m2c1a-compute-debugnodeset-20250828212541351800000003 -[2025-11-30 18:46:34] [EXECUTE] Deleting Instance Template: a3m2c1a-controller-default-20250828212418057400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m2c1a-controller-default-20250828212418057400000002]. -[2025-11-30 18:46:37] [SUCCESS] Deleted a3m2c1a-controller-default-20250828212418057400000002 -[2025-11-30 18:46:37] [EXECUTE] Deleting Instance Template: a3m2c1a-login-login-20250828212416517200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m2c1a-login-login-20250828212416517200000001]. -[2025-11-30 18:46:40] [SUCCESS] Deleted a3m2c1a-login-login-20250828212416517200000001 -[2025-11-30 18:46:40] [EXECUTE] Deleting Instance Template: a3m3500-compute-a3meganodeset-20251114015749275800000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m3500-compute-a3meganodeset-20251114015749275800000004]. -[2025-11-30 18:46:44] [SUCCESS] Deleted a3m3500-compute-a3meganodeset-20251114015749275800000004 -[2025-11-30 18:46:44] [EXECUTE] Deleting Instance Template: a3m3500-compute-debugnodeset-20251114015749253600000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m3500-compute-debugnodeset-20251114015749253600000003]. -[2025-11-30 18:46:47] [SUCCESS] Deleted a3m3500-compute-debugnodeset-20251114015749253600000003 -[2025-11-30 18:46:47] [EXECUTE] Deleting Instance Template: a3m3500-controller-default-20251114015726857500000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m3500-controller-default-20251114015726857500000001]. -[2025-11-30 18:46:50] [SUCCESS] Deleted a3m3500-controller-default-20251114015726857500000001 -[2025-11-30 18:46:50] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:46:50] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:46:53] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:46:53] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:46:53] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:46:53] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:46:56] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:46:56] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:46:59] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:46:59] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:46:59] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:46:59] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:46:59] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:46:59] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:46:59] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:46:59] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:47:00] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:47:00] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:47:00] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:47:00] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:47:00Z (Unix: 1763318820) -[2025-11-30 18:47:00] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:47:02] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:47:02] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 18:47:05] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:47:05] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:47:05] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:47:05] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:47:05] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:47:05] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 18:47:07] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:47:07] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:47:09] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:47:09] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:47:12] [INFO] No Global Address found matching criteria. -[2025-11-30 18:47:12] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 18:47:14] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:47:14] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 18:47:16] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:47:16] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:47:16] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:47:16] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:47:16] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:19] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 18:47:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:47:22] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 18:47:23] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:47:23] [INFO] CLEANUP RUN FINISHED -[2025-11-30 18:48:04] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:48:04] [INFO] Time Cutoff (General): 2025-11-30T18:48:04+0000 -[2025-11-30 18:48:04] [INFO] Time Cutoff (Images): 2025-10-01T18:48:04+0000 -[2025-11-30 18:48:04] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:48:04] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:48:05] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:48:07] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:48:07] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:48:09] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:48:09] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m3500-login-login-20251114015727053000000002 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m45dc-compute-a3meganodeset-20250819191837179100000004 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m45dc-compute-debugnodeset-20250819191837149700000003 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m45dc-controller-default-20250819191713313600000001 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m45dc-login-login-20250819191713400200000002 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m5a82-compute-a3meganodeset-20250822225405540000000004 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m5a82-compute-debugnodeset-20250822225405514300000003 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m5a82-controller-default-20250822225341573600000001 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m5a82-login-login-20250822225348890200000002 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m7038-compute-a3meganodeset-20251023114646456400000004 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m7038-compute-debugnodeset-20251023114646424100000003 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m7038-controller-default-20251023114621658200000001 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m7038-login-login-20251023114621674300000002 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m9518-compute-a3meganodeset-20251114111413631900000003 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m9518-compute-debugnodeset-20251114111413659500000004 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m9518-controller-default-20251114111348348300000001 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m9518-login-login-20251114111348371900000002 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m96ba-compute-a3meganodeset-20250822235943750600000004 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m96ba-compute-debugnodeset-20250822235943730400000003 (Global) -[2025-11-30 18:48:12] [DRY-RUN] Would delete Instance Template: a3m96ba-controller-default-20250822235920176400000001 (Global) -[2025-11-30 18:48:12] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:48:12] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:48:15] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:48:15] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:48:15] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:48:15] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:48:18] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:48:18] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:48:21] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:48:21] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:48:21] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:48:21] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:48:21] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:48:21] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:48:21] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:48:21] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:48:21] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:48:22] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:48:22] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:48:22] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:48:22Z (Unix: 1763318902) -[2025-11-30 18:48:22] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:48:24] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:48:24] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 18:48:27] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:48:27] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:48:27] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:48:27] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:48:27] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:48:27] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 18:48:29] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:48:29] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:48:31] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:48:31] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:48:34] [INFO] No Global Address found matching criteria. -[2025-11-30 18:48:34] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 18:48:36] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:48:36] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 18:48:39] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:48:39] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:48:39] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:48:39] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:48:39] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:42] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 18:48:44] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:48:44] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 18:48:46] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:48:46] [INFO] CLEANUP RUN FINISHED -./cleanup.sh: line 712: n: command not found -[2025-11-30 18:48:50] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:48:50] [INFO] Time Cutoff (General): 2025-11-30T18:48:50+0000 -[2025-11-30 18:48:50] [INFO] Time Cutoff (Images): 2025-10-01T18:48:50+0000 -[2025-11-30 18:48:50] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:48:50] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:48:51] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:48:53] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:48:53] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:48:55] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:48:55] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:48:58] [EXECUTE] Deleting Instance Template: a3m3500-login-login-20251114015727053000000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m3500-login-login-20251114015727053000000002]. -[2025-11-30 18:49:01] [SUCCESS] Deleted a3m3500-login-login-20251114015727053000000002 -[2025-11-30 18:49:01] [EXECUTE] Deleting Instance Template: a3m45dc-compute-a3meganodeset-20250819191837179100000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m45dc-compute-a3meganodeset-20250819191837179100000004]. -[2025-11-30 18:49:04] [SUCCESS] Deleted a3m45dc-compute-a3meganodeset-20250819191837179100000004 -[2025-11-30 18:49:04] [EXECUTE] Deleting Instance Template: a3m45dc-compute-debugnodeset-20250819191837149700000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m45dc-compute-debugnodeset-20250819191837149700000003]. -[2025-11-30 18:49:07] [SUCCESS] Deleted a3m45dc-compute-debugnodeset-20250819191837149700000003 -[2025-11-30 18:49:07] [EXECUTE] Deleting Instance Template: a3m45dc-controller-default-20250819191713313600000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m45dc-controller-default-20250819191713313600000001]. -[2025-11-30 18:49:11] [SUCCESS] Deleted a3m45dc-controller-default-20250819191713313600000001 -[2025-11-30 18:49:11] [EXECUTE] Deleting Instance Template: a3m45dc-login-login-20250819191713400200000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m45dc-login-login-20250819191713400200000002]. -[2025-11-30 18:49:14] [SUCCESS] Deleted a3m45dc-login-login-20250819191713400200000002 -[2025-11-30 18:49:14] [EXECUTE] Deleting Instance Template: a3m5a82-compute-a3meganodeset-20250822225405540000000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m5a82-compute-a3meganodeset-20250822225405540000000004]. -[2025-11-30 18:49:17] [SUCCESS] Deleted a3m5a82-compute-a3meganodeset-20250822225405540000000004 -[2025-11-30 18:49:17] [EXECUTE] Deleting Instance Template: a3m5a82-compute-debugnodeset-20250822225405514300000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m5a82-compute-debugnodeset-20250822225405514300000003]. -[2025-11-30 18:49:20] [SUCCESS] Deleted a3m5a82-compute-debugnodeset-20250822225405514300000003 -[2025-11-30 18:49:20] [EXECUTE] Deleting Instance Template: a3m5a82-controller-default-20250822225341573600000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m5a82-controller-default-20250822225341573600000001]. -[2025-11-30 18:49:24] [SUCCESS] Deleted a3m5a82-controller-default-20250822225341573600000001 -[2025-11-30 18:49:24] [EXECUTE] Deleting Instance Template: a3m5a82-login-login-20250822225348890200000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m5a82-login-login-20250822225348890200000002]. -[2025-11-30 18:49:27] [SUCCESS] Deleted a3m5a82-login-login-20250822225348890200000002 -[2025-11-30 18:49:27] [EXECUTE] Deleting Instance Template: a3m7038-compute-a3meganodeset-20251023114646456400000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m7038-compute-a3meganodeset-20251023114646456400000004]. -[2025-11-30 18:49:29] [SUCCESS] Deleted a3m7038-compute-a3meganodeset-20251023114646456400000004 -[2025-11-30 18:49:29] [EXECUTE] Deleting Instance Template: a3m7038-compute-debugnodeset-20251023114646424100000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m7038-compute-debugnodeset-20251023114646424100000003]. -[2025-11-30 18:49:33] [SUCCESS] Deleted a3m7038-compute-debugnodeset-20251023114646424100000003 -[2025-11-30 18:49:33] [EXECUTE] Deleting Instance Template: a3m7038-controller-default-20251023114621658200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m7038-controller-default-20251023114621658200000001]. -[2025-11-30 18:49:36] [SUCCESS] Deleted a3m7038-controller-default-20251023114621658200000001 -[2025-11-30 18:49:36] [EXECUTE] Deleting Instance Template: a3m7038-login-login-20251023114621674300000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m7038-login-login-20251023114621674300000002]. -[2025-11-30 18:49:39] [SUCCESS] Deleted a3m7038-login-login-20251023114621674300000002 -[2025-11-30 18:49:39] [EXECUTE] Deleting Instance Template: a3m9518-compute-a3meganodeset-20251114111413631900000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9518-compute-a3meganodeset-20251114111413631900000003]. -[2025-11-30 18:49:42] [SUCCESS] Deleted a3m9518-compute-a3meganodeset-20251114111413631900000003 -[2025-11-30 18:49:42] [EXECUTE] Deleting Instance Template: a3m9518-compute-debugnodeset-20251114111413659500000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9518-compute-debugnodeset-20251114111413659500000004]. -[2025-11-30 18:49:46] [SUCCESS] Deleted a3m9518-compute-debugnodeset-20251114111413659500000004 -[2025-11-30 18:49:46] [EXECUTE] Deleting Instance Template: a3m9518-controller-default-20251114111348348300000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9518-controller-default-20251114111348348300000001]. -[2025-11-30 18:49:49] [SUCCESS] Deleted a3m9518-controller-default-20251114111348348300000001 -[2025-11-30 18:49:49] [EXECUTE] Deleting Instance Template: a3m9518-login-login-20251114111348371900000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9518-login-login-20251114111348371900000002]. -[2025-11-30 18:49:53] [SUCCESS] Deleted a3m9518-login-login-20251114111348371900000002 -[2025-11-30 18:49:53] [EXECUTE] Deleting Instance Template: a3m96ba-compute-a3meganodeset-20250822235943750600000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m96ba-compute-a3meganodeset-20250822235943750600000004]. -[2025-11-30 18:49:55] [SUCCESS] Deleted a3m96ba-compute-a3meganodeset-20250822235943750600000004 -[2025-11-30 18:49:55] [EXECUTE] Deleting Instance Template: a3m96ba-compute-debugnodeset-20250822235943730400000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m96ba-compute-debugnodeset-20250822235943730400000003]. -[2025-11-30 18:49:58] [SUCCESS] Deleted a3m96ba-compute-debugnodeset-20250822235943730400000003 -[2025-11-30 18:49:58] [EXECUTE] Deleting Instance Template: a3m96ba-controller-default-20250822235920176400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m96ba-controller-default-20250822235920176400000001]. -[2025-11-30 18:50:01] [SUCCESS] Deleted a3m96ba-controller-default-20250822235920176400000001 -[2025-11-30 18:50:01] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:50:01] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:50:04] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:50:04] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:50:04] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:50:04] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:50:06] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:50:06] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:50:09] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:50:10] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:50:10] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:50:10] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:50:10] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:50:10] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:50:10] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:50:10] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:50:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:50:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:50:10] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:50:10] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:50:10Z (Unix: 1763319010) -[2025-11-30 18:50:10] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:50:13] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:50:13] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 18:50:15] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:50:15] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:50:15] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:50:15] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:50:15] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:50:15] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 18:50:18] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:50:18] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:50:20] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:50:20] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:50:23] [INFO] No Global Address found matching criteria. -[2025-11-30 18:50:23] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 18:50:25] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:50:25] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 18:50:27] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:50:27] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:50:27] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:50:27] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:50:27] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:30] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 18:50:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:50:33] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 18:50:35] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:50:35] [INFO] CLEANUP RUN FINISHED -[2025-11-30 18:52:49] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:52:49] [INFO] Time Cutoff (General): 2025-11-30T18:52:49+0000 -[2025-11-30 18:52:49] [INFO] Time Cutoff (Images): 2025-10-01T18:52:49+0000 -[2025-11-30 18:52:49] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:52:49] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:52:49] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:52:51] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:52:51] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:52:53] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:52:53] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m96ba-login-login-20250822235920487900000002 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9864-compute-a3meganodeset-20250819063622277500000004 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9864-compute-debugnodeset-20250819063622259700000003 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9864-controller-default-20250819063500745200000001 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9864-login-login-20250819063500917600000002 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-a3meganodeset-20251022121513097300000004 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-debugnodeset-20251022121513061400000003 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9b3091-controller-default-20251022121449886500000001 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3m9b3091-login-login-20251022121452690600000002 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mc636-compute-a3meganodeset-20250819181357460200000004 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mc636-compute-debugnodeset-20250819181357431100000003 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mc636-controller-default-20250819181334248000000001 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mc636-login-login-20250819181334341700000002 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3me777-compute-a3meganodeset-20251024084138357500000004 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3me777-compute-debugnodeset-20251024084138331700000003 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3me777-controller-default-20251024084114048400000001 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3me777-login-login-20251024084114099200000002 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mega-compute-a3meganodeset-20251118080924120000000004 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mega-compute-debugnodeset-20251118080924115100000003 (Global) -[2025-11-30 18:52:56] [DRY-RUN] Would delete Instance Template: a3mega-controller-default-20251118080901356800000001 (Global) -[2025-11-30 18:52:56] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:52:56] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:52:59] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:52:59] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:52:59] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:52:59] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:53:02] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:53:02] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:53:04] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:53:04] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:53:05] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:53:05] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:53:05] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:53:05] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:53:05] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:53:05] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:53:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:53:05] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:53:05] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:53:05] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:53:05Z (Unix: 1763319185) -[2025-11-30 18:53:05] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:53:07] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:53:07] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 18:53:10] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:53:10] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:53:10] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:53:10] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:53:10] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:53:10] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 18:53:12] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:53:12] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:53:15] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:53:15] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:53:17] [INFO] No Global Address found matching criteria. -[2025-11-30 18:53:17] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 18:53:19] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:53:19] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 18:53:22] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:53:22] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:53:22] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:53:22] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:53:22] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:25] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 18:53:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:53:27] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 18:53:29] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:53:29] [INFO] CLEANUP RUN FINISHED -[2025-11-30 18:53:59] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:53:59] [INFO] Time Cutoff (General): 2025-11-30T18:53:59+0000 -[2025-11-30 18:53:59] [INFO] Time Cutoff (Images): 2025-10-01T18:53:59+0000 -[2025-11-30 18:53:59] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:53:59] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:53:59] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:54:02] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:54:02] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:54:04] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:54:04] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m96ba-login-login-20250822235920487900000002 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9864-compute-a3meganodeset-20250819063622277500000004 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9864-compute-debugnodeset-20250819063622259700000003 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9864-controller-default-20250819063500745200000001 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9864-login-login-20250819063500917600000002 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-a3meganodeset-20251022121513097300000004 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-debugnodeset-20251022121513061400000003 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9b3091-controller-default-20251022121449886500000001 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3m9b3091-login-login-20251022121452690600000002 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mc636-compute-a3meganodeset-20250819181357460200000004 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mc636-compute-debugnodeset-20250819181357431100000003 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mc636-controller-default-20250819181334248000000001 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mc636-login-login-20250819181334341700000002 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3me777-compute-a3meganodeset-20251024084138357500000004 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3me777-compute-debugnodeset-20251024084138331700000003 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3me777-controller-default-20251024084114048400000001 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3me777-login-login-20251024084114099200000002 (Global) -[2025-11-30 18:54:07] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mega-compute-debugnodeset-20251118080924115100000003 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mega-controller-default-20251118080901356800000001 (Global) -[2025-11-30 18:54:07] [DRY-RUN] Would delete Instance Template: a3mega-login-login-20251118080901434600000002 (Global) -[2025-11-30 18:54:07] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:54:07] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:54:09] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:54:09] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:54:09] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:54:09] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:54:12] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:54:12] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:54:15] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:54:15] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:54:15] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:54:15] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:54:15] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:54:16] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:54:16] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:54:16] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:54:16] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:54:16] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:54:16] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:54:16] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:54:16Z (Unix: 1763319256) -[2025-11-30 18:54:16] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:54:18] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:54:18] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 18:54:21] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:54:21] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:54:21] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:54:21] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:54:21] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:54:21] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 18:54:23] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:54:23] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:54:25] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:54:25] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:54:28] [INFO] No Global Address found matching criteria. -[2025-11-30 18:54:28] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 18:54:30] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:54:30] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 18:54:32] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:54:32] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:54:32] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:54:32] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:54:32] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:35] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 18:54:37] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:54:37] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 18:54:39] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:54:39] [INFO] CLEANUP RUN FINISHED -[2025-11-30 18:54:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:54:47] [INFO] Time Cutoff (General): 2025-11-30T18:54:47+0000 -[2025-11-30 18:54:47] [INFO] Time Cutoff (Images): 2025-10-01T18:54:47+0000 -[2025-11-30 18:54:47] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:54:47] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:54:48] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:54:50] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:54:50] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:54:51] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:54:52] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m96ba-login-login-20250822235920487900000002 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9864-compute-a3meganodeset-20250819063622277500000004 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9864-compute-debugnodeset-20250819063622259700000003 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9864-controller-default-20250819063500745200000001 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9864-login-login-20250819063500917600000002 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-a3meganodeset-20251022121513097300000004 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9b3091-compute-debugnodeset-20251022121513061400000003 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9b3091-controller-default-20251022121449886500000001 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3m9b3091-login-login-20251022121452690600000002 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3mc636-compute-a3meganodeset-20250819181357460200000004 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3mc636-compute-debugnodeset-20250819181357431100000003 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3mc636-controller-default-20250819181334248000000001 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3mc636-login-login-20250819181334341700000002 (Global) -[2025-11-30 18:54:54] [DRY-RUN] Would delete Instance Template: a3me777-compute-a3meganodeset-20251024084138357500000004 (Global) -[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3me777-compute-debugnodeset-20251024084138331700000003 (Global) -[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3me777-controller-default-20251024084114048400000001 (Global) -[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3me777-login-login-20251024084114099200000002 (Global) -[2025-11-30 18:54:55] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 18:54:55] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 18:54:55] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 18:54:55] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3qclavek-compute-a3ultranodeset-20251001033444416400000002 (Global) -[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3qclavek-controller-default-20251001033450408100000003 (Global) -[2025-11-30 18:54:55] [DRY-RUN] Would delete Instance Template: a3qclavek-login-slurm-login-20251001033440486800000001 (Global) -[2025-11-30 18:54:55] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:54:55] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:54:57] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:54:57] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:54:57] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:54:57] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:55:00] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:55:00] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:55:03] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:55:03] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:55:03] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:55:03] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:55:03] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:55:03] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:55:03] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:55:03] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:55:04] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:55:04] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:55:04] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:55:04] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:55:04Z (Unix: 1763319304) -[2025-11-30 18:55:04] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:55:06] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:55:06] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 18:55:09] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:55:09] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:55:09] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:55:09] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:55:09] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:55:09] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 18:55:11] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:55:11] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:55:13] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:55:13] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:55:17] [INFO] No Global Address found matching criteria. -[2025-11-30 18:55:17] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 18:55:19] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:55:19] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 18:55:22] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:55:22] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:55:22] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:55:22] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:55:22] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:24] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 18:55:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:55:27] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 18:55:28] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:55:28] [INFO] CLEANUP RUN FINISHED -./cleanup.sh: line 712: n: command not found -[2025-11-30 18:56:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:56:29] [INFO] Time Cutoff (General): 2025-11-30T18:56:29+0000 -[2025-11-30 18:56:29] [INFO] Time Cutoff (Images): 2025-10-01T18:56:29+0000 -[2025-11-30 18:56:29] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:56:29] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:56:30] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:56:32] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:56:32] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:56:34] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:56:34] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:56:37] [EXECUTE] Deleting Instance Template: a3m96ba-login-login-20250822235920487900000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m96ba-login-login-20250822235920487900000002]. -[2025-11-30 18:56:40] [SUCCESS] Deleted a3m96ba-login-login-20250822235920487900000002 -[2025-11-30 18:56:40] [EXECUTE] Deleting Instance Template: a3m9864-compute-a3meganodeset-20250819063622277500000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9864-compute-a3meganodeset-20250819063622277500000004]. -[2025-11-30 18:56:43] [SUCCESS] Deleted a3m9864-compute-a3meganodeset-20250819063622277500000004 -[2025-11-30 18:56:43] [EXECUTE] Deleting Instance Template: a3m9864-compute-debugnodeset-20250819063622259700000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9864-compute-debugnodeset-20250819063622259700000003]. -[2025-11-30 18:56:47] [SUCCESS] Deleted a3m9864-compute-debugnodeset-20250819063622259700000003 -[2025-11-30 18:56:47] [EXECUTE] Deleting Instance Template: a3m9864-controller-default-20250819063500745200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9864-controller-default-20250819063500745200000001]. -[2025-11-30 18:56:50] [SUCCESS] Deleted a3m9864-controller-default-20250819063500745200000001 -[2025-11-30 18:56:50] [EXECUTE] Deleting Instance Template: a3m9864-login-login-20250819063500917600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9864-login-login-20250819063500917600000002]. -[2025-11-30 18:56:53] [SUCCESS] Deleted a3m9864-login-login-20250819063500917600000002 -[2025-11-30 18:56:53] [EXECUTE] Deleting Instance Template: a3m9b3091-compute-a3meganodeset-20251022121513097300000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9b3091-compute-a3meganodeset-20251022121513097300000004]. -[2025-11-30 18:56:56] [SUCCESS] Deleted a3m9b3091-compute-a3meganodeset-20251022121513097300000004 -[2025-11-30 18:56:56] [EXECUTE] Deleting Instance Template: a3m9b3091-compute-debugnodeset-20251022121513061400000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9b3091-compute-debugnodeset-20251022121513061400000003]. -[2025-11-30 18:57:00] [SUCCESS] Deleted a3m9b3091-compute-debugnodeset-20251022121513061400000003 -[2025-11-30 18:57:00] [EXECUTE] Deleting Instance Template: a3m9b3091-controller-default-20251022121449886500000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9b3091-controller-default-20251022121449886500000001]. -[2025-11-30 18:57:03] [SUCCESS] Deleted a3m9b3091-controller-default-20251022121449886500000001 -[2025-11-30 18:57:03] [EXECUTE] Deleting Instance Template: a3m9b3091-login-login-20251022121452690600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3m9b3091-login-login-20251022121452690600000002]. -[2025-11-30 18:57:06] [SUCCESS] Deleted a3m9b3091-login-login-20251022121452690600000002 -[2025-11-30 18:57:06] [EXECUTE] Deleting Instance Template: a3mc636-compute-a3meganodeset-20250819181357460200000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3mc636-compute-a3meganodeset-20250819181357460200000004]. -[2025-11-30 18:57:09] [SUCCESS] Deleted a3mc636-compute-a3meganodeset-20250819181357460200000004 -[2025-11-30 18:57:09] [EXECUTE] Deleting Instance Template: a3mc636-compute-debugnodeset-20250819181357431100000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3mc636-compute-debugnodeset-20250819181357431100000003]. -[2025-11-30 18:57:12] [SUCCESS] Deleted a3mc636-compute-debugnodeset-20250819181357431100000003 -[2025-11-30 18:57:12] [EXECUTE] Deleting Instance Template: a3mc636-controller-default-20250819181334248000000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3mc636-controller-default-20250819181334248000000001]. -[2025-11-30 18:57:15] [SUCCESS] Deleted a3mc636-controller-default-20250819181334248000000001 -[2025-11-30 18:57:15] [EXECUTE] Deleting Instance Template: a3mc636-login-login-20250819181334341700000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3mc636-login-login-20250819181334341700000002]. -[2025-11-30 18:57:18] [SUCCESS] Deleted a3mc636-login-login-20250819181334341700000002 -[2025-11-30 18:57:18] [EXECUTE] Deleting Instance Template: a3me777-compute-a3meganodeset-20251024084138357500000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3me777-compute-a3meganodeset-20251024084138357500000004]. -[2025-11-30 18:57:22] [SUCCESS] Deleted a3me777-compute-a3meganodeset-20251024084138357500000004 -[2025-11-30 18:57:22] [EXECUTE] Deleting Instance Template: a3me777-compute-debugnodeset-20251024084138331700000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3me777-compute-debugnodeset-20251024084138331700000003]. -[2025-11-30 18:57:25] [SUCCESS] Deleted a3me777-compute-debugnodeset-20251024084138331700000003 -[2025-11-30 18:57:25] [EXECUTE] Deleting Instance Template: a3me777-controller-default-20251024084114048400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3me777-controller-default-20251024084114048400000001]. -[2025-11-30 18:57:28] [SUCCESS] Deleted a3me777-controller-default-20251024084114048400000001 -[2025-11-30 18:57:28] [EXECUTE] Deleting Instance Template: a3me777-login-login-20251024084114099200000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3me777-login-login-20251024084114099200000002]. -[2025-11-30 18:57:31] [SUCCESS] Deleted a3me777-login-login-20251024084114099200000002 -[2025-11-30 18:57:31] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 18:57:31] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 18:57:31] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 18:57:31] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 18:57:31] [EXECUTE] Deleting Instance Template: a3qclavek-compute-a3ultranodeset-20251001033444416400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3qclavek-compute-a3ultranodeset-20251001033444416400000002]. -[2025-11-30 18:57:34] [SUCCESS] Deleted a3qclavek-compute-a3ultranodeset-20251001033444416400000002 -[2025-11-30 18:57:34] [EXECUTE] Deleting Instance Template: a3qclavek-controller-default-20251001033450408100000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3qclavek-controller-default-20251001033450408100000003]. -[2025-11-30 18:57:38] [SUCCESS] Deleted a3qclavek-controller-default-20251001033450408100000003 -[2025-11-30 18:57:38] [EXECUTE] Deleting Instance Template: a3qclavek-login-slurm-login-20251001033440486800000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3qclavek-login-slurm-login-20251001033440486800000001]. -[2025-11-30 18:57:41] [SUCCESS] Deleted a3qclavek-login-slurm-login-20251001033440486800000001 -[2025-11-30 18:57:41] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:57:41] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:57:43] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:57:43] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:57:43] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:57:43] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:57:46] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:57:46] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:57:49] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:57:49] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:57:49] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:57:49] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:57:49] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:57:49] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:57:49] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:57:49] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:57:50] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:57:50] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:57:50] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:57:50] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:57:50Z (Unix: 1763319470) -[2025-11-30 18:57:50] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:57:52] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:57:52] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 18:57:55] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:57:55] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:57:55] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:57:55] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:57:55] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:57:55] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 18:57:57] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:57:57] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:58:00] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:58:00] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:58:02] [INFO] No Global Address found matching criteria. -[2025-11-30 18:58:02] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 18:58:05] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:58:05] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 18:58:07] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:58:07] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:58:07] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:58:07] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:58:07] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 18:58:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:09] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:10] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 18:58:12] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:58:12] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 18:58:14] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:58:14] [INFO] CLEANUP RUN FINISHED -[2025-11-30 18:58:35] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:58:35] [INFO] Time Cutoff (General): 2025-11-30T18:58:34+0000 -[2025-11-30 18:58:35] [INFO] Time Cutoff (Images): 2025-10-01T18:58:35+0000 -[2025-11-30 18:58:35] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:58:35] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:58:35] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:58:38] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:58:38] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:58:40] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:58:40] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:58:43] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 18:58:43] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 18:58:43] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 18:58:43] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3slurmsy-compute-a3nodeset-20251016115123978200000002 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3slurmsy-controller-default-20251016115129563200000003 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3slurmsy-login-slurm-login-20251016115120300800000001 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3u529e-compute-a3ultranodeset-20251114115423143800000002 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3u529e-controller-default-20251114115425847100000003 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3u529e-login-slurm-login-20251114115420368400000001 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3usarr-compute-a4highnodeset-20251105000020022500000002 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3usarr-controller-default-20251105000025097200000003 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a3usarr-login-slurm-login-20251105000015399700000001 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h333e-compute-a4highnodeset-20251120094513362600000002 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h333e-controller-default-20251120094518412300000003 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h333e-login-slurm-login-20251120094509085100000001 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h5c04-compute-a4highnodeset-20251114133459801900000002 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h5c04-controller-default-20251114133505649700000003 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h5c04-login-slurm-login-20251114133456443900000001 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h639c-controller-default-20251126091821328000000003 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h639c-login-slurm-login-20251126091812165600000001 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h68f2-compute-a4highnodeset-20251125205110303400000002 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h68f2-controller-default-20251125205115747500000003 (Global) -[2025-11-30 18:58:43] [DRY-RUN] Would delete Instance Template: a4h68f2-login-slurm-login-20251125205107060800000001 (Global) -[2025-11-30 18:58:43] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:58:43] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:58:46] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:58:46] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:58:46] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:58:46] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:58:49] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:58:49] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:58:52] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:58:52] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:58:52] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:58:52] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:58:52] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:58:52] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:58:52] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:58:52] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:58:53] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:58:53] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:58:53] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:58:53] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:58:53Z (Unix: 1763319533) -[2025-11-30 18:58:53] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:58:55] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:58:55] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 18:58:58] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 18:58:58] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 18:58:58] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 18:58:58] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 18:58:58] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 18:58:58] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 18:59:00] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 18:59:00] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:59:03] [INFO] No Regional Address found matching criteria. -[2025-11-30 18:59:03] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 18:59:05] [INFO] No Global Address found matching criteria. -[2025-11-30 18:59:05] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 18:59:08] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 18:59:08] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 18:59:10] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:59:10] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:59:10] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 18:59:10] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 18:59:10] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:13] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 18:59:15] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 18:59:15] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 18:59:17] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 18:59:17] [INFO] CLEANUP RUN FINISHED -[2025-11-30 18:59:38] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 18:59:38] [INFO] Time Cutoff (General): 2025-11-30T18:59:38+0000 -[2025-11-30 18:59:38] [INFO] Time Cutoff (Images): 2025-10-01T18:59:38+0000 -[2025-11-30 18:59:38] [INFO] Delete Limit per Type: 20 -[2025-11-30 18:59:38] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 18:59:38] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 18:59:41] [INFO] No Service Accounts found matching prefix. -[2025-11-30 18:59:41] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:59:43] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 18:59:43] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 18:59:46] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 18:59:46] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 18:59:46] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 18:59:46] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 18:59:46] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 18:59:46] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 18:59:46] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3u529e-compute-a3ultranodeset-20251114115423143800000002 (Global) -[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3u529e-controller-default-20251114115425847100000003 (Global) -[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3u529e-login-slurm-login-20251114115420368400000001 (Global) -[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3usarr-compute-a4highnodeset-20251105000020022500000002 (Global) -[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3usarr-controller-default-20251105000025097200000003 (Global) -[2025-11-30 18:59:46] [DRY-RUN] Would delete Instance Template: a3usarr-login-slurm-login-20251105000015399700000001 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h333e-compute-a4highnodeset-20251120094513362600000002 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h333e-controller-default-20251120094518412300000003 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h333e-login-slurm-login-20251120094509085100000001 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h5c04-compute-a4highnodeset-20251114133459801900000002 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h5c04-controller-default-20251114133505649700000003 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h5c04-login-slurm-login-20251114133456443900000001 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h639c-controller-default-20251126091821328000000003 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h639c-login-slurm-login-20251126091812165600000001 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h68f2-compute-a4highnodeset-20251125205110303400000002 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h68f2-controller-default-20251125205115747500000003 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h68f2-login-slurm-login-20251125205107060800000001 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h6c3b-compute-a4highnodeset-20251117134652173800000002 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h6c3b-controller-default-20251117134657496900000003 (Global) -[2025-11-30 18:59:47] [DRY-RUN] Would delete Instance Template: a4h6c3b-login-slurm-login-20251117134647592200000001 (Global) -[2025-11-30 18:59:47] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 18:59:47] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 18:59:49] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 18:59:49] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 18:59:49] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 18:59:49] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 18:59:52] [INFO] No Filestore instances found matching criteria. -[2025-11-30 18:59:52] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 18:59:55] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 18:59:55] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 18:59:55] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 18:59:55] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 18:59:56] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 18:59:56] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 18:59:56] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 18:59:56] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 18:59:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 18:59:56] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 18:59:56] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 18:59:56] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T18:59:56Z (Unix: 1763319596) -[2025-11-30 18:59:56] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 18:59:59] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 18:59:59] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:00:01] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:00:02] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:00:02] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:00:02] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:00:02] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:00:02] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:00:04] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:00:04] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:00:07] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:00:07] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:00:09] [INFO] No Global Address found matching criteria. -[2025-11-30 19:00:09] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:00:12] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:00:12] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:00:15] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:00:15] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:00:15] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:00:15] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:00:15] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:19] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:00:22] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:00:22] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:00:24] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:00:24] [INFO] CLEANUP RUN FINISHED -./cleanup.sh: line 712: n: command not found -[2025-11-30 19:01:10] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:01:10] [INFO] Time Cutoff (General): 2025-11-30T19:01:10+0000 -[2025-11-30 19:01:10] [INFO] Time Cutoff (Images): 2025-10-01T19:01:10+0000 -[2025-11-30 19:01:10] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:01:10] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:01:10] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:01:13] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:01:13] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:01:15] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:01:15] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:01:18] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:01:18] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:01:18] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:01:18] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:01:18] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:01:18] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:01:18] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:01:18] [EXECUTE] Deleting Instance Template: a3u529e-compute-a3ultranodeset-20251114115423143800000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3u529e-compute-a3ultranodeset-20251114115423143800000002]. -[2025-11-30 19:01:21] [SUCCESS] Deleted a3u529e-compute-a3ultranodeset-20251114115423143800000002 -[2025-11-30 19:01:21] [EXECUTE] Deleting Instance Template: a3u529e-controller-default-20251114115425847100000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3u529e-controller-default-20251114115425847100000003]. -[2025-11-30 19:01:24] [SUCCESS] Deleted a3u529e-controller-default-20251114115425847100000003 -[2025-11-30 19:01:24] [EXECUTE] Deleting Instance Template: a3u529e-login-slurm-login-20251114115420368400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3u529e-login-slurm-login-20251114115420368400000001]. -[2025-11-30 19:01:27] [SUCCESS] Deleted a3u529e-login-slurm-login-20251114115420368400000001 -[2025-11-30 19:01:27] [EXECUTE] Deleting Instance Template: a3usarr-compute-a4highnodeset-20251105000020022500000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3usarr-compute-a4highnodeset-20251105000020022500000002]. -[2025-11-30 19:01:30] [SUCCESS] Deleted a3usarr-compute-a4highnodeset-20251105000020022500000002 -[2025-11-30 19:01:30] [EXECUTE] Deleting Instance Template: a3usarr-controller-default-20251105000025097200000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3usarr-controller-default-20251105000025097200000003]. -[2025-11-30 19:01:34] [SUCCESS] Deleted a3usarr-controller-default-20251105000025097200000003 -[2025-11-30 19:01:34] [EXECUTE] Deleting Instance Template: a3usarr-login-slurm-login-20251105000015399700000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a3usarr-login-slurm-login-20251105000015399700000001]. -[2025-11-30 19:01:37] [SUCCESS] Deleted a3usarr-login-slurm-login-20251105000015399700000001 -[2025-11-30 19:01:37] [EXECUTE] Deleting Instance Template: a4h333e-compute-a4highnodeset-20251120094513362600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h333e-compute-a4highnodeset-20251120094513362600000002]. -[2025-11-30 19:01:40] [SUCCESS] Deleted a4h333e-compute-a4highnodeset-20251120094513362600000002 -[2025-11-30 19:01:40] [EXECUTE] Deleting Instance Template: a4h333e-controller-default-20251120094518412300000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h333e-controller-default-20251120094518412300000003]. -[2025-11-30 19:01:43] [SUCCESS] Deleted a4h333e-controller-default-20251120094518412300000003 -[2025-11-30 19:01:43] [EXECUTE] Deleting Instance Template: a4h333e-login-slurm-login-20251120094509085100000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h333e-login-slurm-login-20251120094509085100000001]. -[2025-11-30 19:01:46] [SUCCESS] Deleted a4h333e-login-slurm-login-20251120094509085100000001 -[2025-11-30 19:01:46] [EXECUTE] Deleting Instance Template: a4h5c04-compute-a4highnodeset-20251114133459801900000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h5c04-compute-a4highnodeset-20251114133459801900000002]. -[2025-11-30 19:01:50] [SUCCESS] Deleted a4h5c04-compute-a4highnodeset-20251114133459801900000002 -[2025-11-30 19:01:50] [EXECUTE] Deleting Instance Template: a4h5c04-controller-default-20251114133505649700000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h5c04-controller-default-20251114133505649700000003]. -[2025-11-30 19:01:53] [SUCCESS] Deleted a4h5c04-controller-default-20251114133505649700000003 -[2025-11-30 19:01:53] [EXECUTE] Deleting Instance Template: a4h5c04-login-slurm-login-20251114133456443900000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h5c04-login-slurm-login-20251114133456443900000001]. -[2025-11-30 19:01:56] [SUCCESS] Deleted a4h5c04-login-slurm-login-20251114133456443900000001 -[2025-11-30 19:01:56] [EXECUTE] Deleting Instance Template: a4h639c-controller-default-20251126091821328000000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h639c-controller-default-20251126091821328000000003]. -[2025-11-30 19:01:59] [SUCCESS] Deleted a4h639c-controller-default-20251126091821328000000003 -[2025-11-30 19:01:59] [EXECUTE] Deleting Instance Template: a4h639c-login-slurm-login-20251126091812165600000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h639c-login-slurm-login-20251126091812165600000001]. -[2025-11-30 19:02:02] [SUCCESS] Deleted a4h639c-login-slurm-login-20251126091812165600000001 -[2025-11-30 19:02:02] [EXECUTE] Deleting Instance Template: a4h68f2-compute-a4highnodeset-20251125205110303400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h68f2-compute-a4highnodeset-20251125205110303400000002]. -[2025-11-30 19:02:06] [SUCCESS] Deleted a4h68f2-compute-a4highnodeset-20251125205110303400000002 -[2025-11-30 19:02:06] [EXECUTE] Deleting Instance Template: a4h68f2-controller-default-20251125205115747500000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h68f2-controller-default-20251125205115747500000003]. -[2025-11-30 19:02:09] [SUCCESS] Deleted a4h68f2-controller-default-20251125205115747500000003 -[2025-11-30 19:02:09] [EXECUTE] Deleting Instance Template: a4h68f2-login-slurm-login-20251125205107060800000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h68f2-login-slurm-login-20251125205107060800000001]. -[2025-11-30 19:02:12] [SUCCESS] Deleted a4h68f2-login-slurm-login-20251125205107060800000001 -[2025-11-30 19:02:12] [EXECUTE] Deleting Instance Template: a4h6c3b-compute-a4highnodeset-20251117134652173800000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h6c3b-compute-a4highnodeset-20251117134652173800000002]. -[2025-11-30 19:02:15] [SUCCESS] Deleted a4h6c3b-compute-a4highnodeset-20251117134652173800000002 -[2025-11-30 19:02:15] [EXECUTE] Deleting Instance Template: a4h6c3b-controller-default-20251117134657496900000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h6c3b-controller-default-20251117134657496900000003]. -[2025-11-30 19:02:19] [SUCCESS] Deleted a4h6c3b-controller-default-20251117134657496900000003 -[2025-11-30 19:02:19] [EXECUTE] Deleting Instance Template: a4h6c3b-login-slurm-login-20251117134647592200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4h6c3b-login-slurm-login-20251117134647592200000001]. -[2025-11-30 19:02:22] [SUCCESS] Deleted a4h6c3b-login-slurm-login-20251117134647592200000001 -[2025-11-30 19:02:22] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:02:22] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:02:24] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:02:24] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:02:24] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:02:24] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:02:27] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:02:27] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:02:30] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:02:30] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:02:30] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:02:30] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:02:31] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:02:31] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:02:31] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:02:31] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:02:31] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:02:31] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:02:31] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:02:31] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:02:31Z (Unix: 1763319751) -[2025-11-30 19:02:31] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:02:34] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:02:34] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:02:36] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:02:36] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:02:36] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:02:36] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:02:36] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:02:36] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:02:39] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:02:39] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:02:41] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:02:41] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:02:44] [INFO] No Global Address found matching criteria. -[2025-11-30 19:02:44] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:02:47] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:02:47] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:02:49] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:02:49] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:02:49] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:02:49] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:02:49] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:52] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:02:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:02:54] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:02:56] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:02:56] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:03:03] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:03:03] [INFO] Time Cutoff (General): 2025-11-30T19:03:03+0000 -[2025-11-30 19:03:03] [INFO] Time Cutoff (Images): 2025-10-01T19:03:03+0000 -[2025-11-30 19:03:03] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:03:03] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:03:04] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:03:06] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:03:06] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:03:08] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:03:08] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:03:11] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:03:11] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:03:11] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:03:11] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:03:11] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:03:11] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:03:11] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc0e2-compute-a4highnodeset-20251126202608471600000002 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc0e2-controller-default-20251126202614479200000003 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc0e2-login-slurm-login-20251126202605221400000001 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc1a3-compute-a4highnodeset-20251127090301343400000002 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc1a3-controller-default-20251127090306826200000003 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hc1a3-login-slurm-login-20251127090258428600000001 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hcf79-compute-a4highnodeset-20251119140049364000000002 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hcf79-controller-default-20251119140054728100000003 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hcf79-login-slurm-login-20251119140045167600000001 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4he340-controller-default-20251126120236033300000003 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4he340-login-slurm-login-20251126120226527400000001 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hee35-compute-a4highnodeset-20251119063803576500000002 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hee35-controller-default-20251119063809267100000003 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hee35-login-slurm-login-20251119063800421400000001 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-controller-default-20251116112212875400000003 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-login-slurm-login-20251116112205108200000001 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-controller-default-20251116055930192300000003 (Global) -[2025-11-30 19:03:11] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-login-slurm-login-20251116055921750900000001 (Global) -[2025-11-30 19:03:11] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:03:11] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:03:14] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:03:14] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:03:14] [DRY-RUN] Would delete Compute Instance: topology-controller us-central1-a -[2025-11-30 19:03:14] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:03:14] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:03:17] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:03:17] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:03:20] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:03:20] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:03:20] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:03:20] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:03:20] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:03:20] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:03:20] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:03:20] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:03:21] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:03:21] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:03:21] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:03:21] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:03:21Z (Unix: 1763319801) -[2025-11-30 19:03:21] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:03:23] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:03:23] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:03:26] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:03:26] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:03:26] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:03:26] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:03:26] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:03:26] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:03:28] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:03:28] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:03:31] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:03:31] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:03:33] [INFO] No Global Address found matching criteria. -[2025-11-30 19:03:33] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:03:36] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:03:36] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:03:38] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:03:38] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:03:38] [DRY-RUN] Would delete Zonal Disk: topology-controller https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[2025-11-30 19:03:38] [DRY-RUN] Would delete Zonal Disk: topology-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[2025-11-30 19:03:38] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:03:38] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:03:38] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:41] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:03:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:03:43] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:03:45] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:03:45] [INFO] CLEANUP RUN FINISHED -./cleanup.sh: line 712: n: command not found -[2025-11-30 19:05:35] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:05:35] [INFO] Time Cutoff (General): 2025-11-30T19:05:35+0000 -[2025-11-30 19:05:35] [INFO] Time Cutoff (Images): 2025-10-01T19:05:35+0000 -[2025-11-30 19:05:35] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:05:35] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:05:36] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:05:38] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:05:38] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:05:40] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:05:40] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:05:43] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:05:43] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:05:43] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:05:43] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:05:43] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:05:43] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:05:43] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc0e2-compute-a4highnodeset-20251126202608471600000002 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc0e2-controller-default-20251126202614479200000003 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc0e2-login-slurm-login-20251126202605221400000001 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc1a3-compute-a4highnodeset-20251127090301343400000002 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc1a3-controller-default-20251127090306826200000003 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hc1a3-login-slurm-login-20251127090258428600000001 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hcf79-compute-a4highnodeset-20251119140049364000000002 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hcf79-controller-default-20251119140054728100000003 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hcf79-login-slurm-login-20251119140045167600000001 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4he340-controller-default-20251126120236033300000003 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4he340-login-slurm-login-20251126120226527400000001 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hee35-compute-a4highnodeset-20251119063803576500000002 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hee35-controller-default-20251119063809267100000003 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hee35-login-slurm-login-20251119063800421400000001 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-controller-default-20251116112212875400000003 (Global) -[2025-11-30 19:05:43] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-login-slurm-login-20251116112205108200000001 (Global) -[2025-11-30 19:05:44] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 (Global) -[2025-11-30 19:05:44] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-controller-default-20251116055930192300000003 (Global) -[2025-11-30 19:05:44] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-login-slurm-login-20251116055921750900000001 (Global) -[2025-11-30 19:05:44] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:05:44] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:05:46] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:05:46] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-controller us-central1-a -[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-nodeset-0 us-central1-a -[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-nodeset-1 us-central1-a -[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-nodeset-2 us-central1-a -[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-nodeset-3 us-central1-a -[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-nodeset-4 us-central1-a -[2025-11-30 19:05:46] [DRY-RUN] Would delete Compute Instance: topology-slurm-login-001 us-central1-a -[2025-11-30 19:05:46] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:05:46] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:05:49] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:05:49] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:05:52] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:05:52] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:05:52] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:05:52] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:05:52] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:05:52] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:05:52] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:05:52] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:05:53] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:05:53] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:05:53] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:05:53] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:05:53Z (Unix: 1763319953) -[2025-11-30 19:05:53] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:05:55] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:05:55] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:05:58] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:05:58] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:05:58] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:05:58] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:05:58] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:05:58] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:06:00] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:06:00] [INFO] --- Processing: Regional Address (Limit: 20) --- -[2025-11-30 19:06:02] [DRY-RUN] Would delete Regional Address: nat-auto-ip-11875105-1-1764529459120987 us-central1 -[2025-11-30 19:06:02] [INFO] --- Processing: Global Address (Limit: 20) --- -[2025-11-30 19:06:05] [INFO] No Global Address found matching criteria. -[2025-11-30 19:06:05] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:06:07] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:06:07] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:06:10] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:06:10] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-controller https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-controller-save https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-nodeset-0 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-nodeset-1 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-nodeset-2 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-nodeset-3 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-nodeset-4 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[2025-11-30 19:06:10] [DRY-RUN] Would delete Zonal Disk: topology-slurm-login-001 https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/zones/us-central1-a -[2025-11-30 19:06:10] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:06:10] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:06:10] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:06:12] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:12] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:12] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:13] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:06:15] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:06:15] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:06:17] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:06:17] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:07:05] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:07:05] [INFO] Time Cutoff (General): 2025-11-30T19:07:05+0000 -[2025-11-30 19:07:05] [INFO] Time Cutoff (Images): 2025-10-01T19:07:05+0000 -[2025-11-30 19:07:05] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:07:05] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:07:05] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:07:08] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:07:08] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:07:10] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:07:10] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:07:13] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:07:13] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:07:13] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:07:13] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:07:13] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:07:13] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:07:13] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc0e2-compute-a4highnodeset-20251126202608471600000002 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc0e2-controller-default-20251126202614479200000003 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc0e2-login-slurm-login-20251126202605221400000001 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc1a3-compute-a4highnodeset-20251127090301343400000002 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc1a3-controller-default-20251127090306826200000003 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hc1a3-login-slurm-login-20251127090258428600000001 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hcf79-compute-a4highnodeset-20251119140049364000000002 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hcf79-controller-default-20251119140054728100000003 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hcf79-login-slurm-login-20251119140045167600000001 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4he340-controller-default-20251126120236033300000003 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4he340-login-slurm-login-20251126120226527400000001 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hee35-compute-a4highnodeset-20251119063803576500000002 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hee35-controller-default-20251119063809267100000003 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hee35-login-slurm-login-20251119063800421400000001 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-controller-default-20251116112212875400000003 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-login-slurm-login-20251116112205108200000001 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-controller-default-20251116055930192300000003 (Global) -[2025-11-30 19:07:13] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-login-slurm-login-20251116055921750900000001 (Global) -[2025-11-30 19:07:13] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:07:13] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:07:16] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:07:17] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:07:17] [SKIP] topology-controller (Protected Substring) -[2025-11-30 19:07:17] [SKIP] topology-nodeset-0 (Protected Substring) -[2025-11-30 19:07:17] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:07:17] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:07:19] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:07:19] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:07:23] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:07:23] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:07:23] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:07:23] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:07:23] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:07:23] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:07:23] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:07:23] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:07:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:07:23] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:07:23] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:07:24] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:07:23Z (Unix: 1763320043) -[2025-11-30 19:07:24] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:07:26] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:07:26] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:07:28] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:07:28] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:07:28] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:07:28] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:07:28] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:07:28] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:07:31] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:07:31] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:07:33] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:07:33] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:07:35] [INFO] No Global Address found matching criteria. -[2025-11-30 19:07:35] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:07:38] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:07:38] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:07:40] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:07:40] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:07:40] [SKIP] topology-controller-save (Protected Substring) -[2025-11-30 19:07:40] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:07:40] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:07:40] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:07:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:42] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:43] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:07:45] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:07:45] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:07:47] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:07:47] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:08:16] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:08:16] [INFO] Time Cutoff (General): 2025-11-30T14:08:16+0000 -[2025-11-30 19:08:16] [INFO] Time Cutoff (Images): 2025-10-01T19:08:16+0000 -[2025-11-30 19:08:16] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:08:16] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:08:17] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:08:19] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:08:19] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:08:21] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:08:21] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:08:24] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:08:24] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:08:24] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:08:24] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:08:24] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:08:24] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:08:24] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc0e2-compute-a4highnodeset-20251126202608471600000002 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc0e2-controller-default-20251126202614479200000003 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc0e2-login-slurm-login-20251126202605221400000001 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc1a3-compute-a4highnodeset-20251127090301343400000002 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc1a3-controller-default-20251127090306826200000003 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hc1a3-login-slurm-login-20251127090258428600000001 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hcf79-compute-a4highnodeset-20251119140049364000000002 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hcf79-controller-default-20251119140054728100000003 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hcf79-login-slurm-login-20251119140045167600000001 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4he340-controller-default-20251126120236033300000003 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4he340-login-slurm-login-20251126120226527400000001 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hee35-compute-a4highnodeset-20251119063803576500000002 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hee35-controller-default-20251119063809267100000003 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hee35-login-slurm-login-20251119063800421400000001 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-controller-default-20251116112212875400000003 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf1a46f0-login-slurm-login-20251116112205108200000001 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-controller-default-20251116055930192300000003 (Global) -[2025-11-30 19:08:24] [DRY-RUN] Would delete Instance Template: a4hf97fa5b-login-slurm-login-20251116055921750900000001 (Global) -[2025-11-30 19:08:24] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:08:24] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:08:27] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:08:27] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:08:27] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:08:27] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:08:30] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:08:30] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:08:33] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:08:33] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:08:33] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:08:33] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:08:33] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:08:33] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:08:33] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:08:33] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:08:34] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:08:34] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:08:34] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:08:34] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:08:34Z (Unix: 1763320114) -[2025-11-30 19:08:34] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:08:36] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:08:36] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:08:38] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:08:38] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:08:38] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:08:38] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:08:39] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:08:39] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:08:41] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:08:41] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:08:43] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:08:43] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:08:46] [INFO] No Global Address found matching criteria. -[2025-11-30 19:08:46] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:08:48] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:08:48] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:08:51] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:08:51] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:08:51] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:08:51] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:08:51] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:53] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:54] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:54] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:08:56] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:08:56] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:08:58] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:08:58] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:09:08] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:09:08] [INFO] Time Cutoff (General): 2025-11-30T14:09:08+0000 -[2025-11-30 19:09:08] [INFO] Time Cutoff (Images): 2025-10-01T19:09:08+0000 -[2025-11-30 19:09:09] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:09:09] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:09:09] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:09:11] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:09:11] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:09:13] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:09:13] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:09:16] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:09:16] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:09:16] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:09:16] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:09:16] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:09:16] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:09:16] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:09:16] [EXECUTE] Deleting Instance Template: a4hc0e2-compute-a4highnodeset-20251126202608471600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc0e2-compute-a4highnodeset-20251126202608471600000002]. -[2025-11-30 19:09:19] [SUCCESS] Deleted a4hc0e2-compute-a4highnodeset-20251126202608471600000002 -[2025-11-30 19:09:19] [EXECUTE] Deleting Instance Template: a4hc0e2-controller-default-20251126202614479200000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc0e2-controller-default-20251126202614479200000003]. -[2025-11-30 19:09:22] [SUCCESS] Deleted a4hc0e2-controller-default-20251126202614479200000003 -[2025-11-30 19:09:22] [EXECUTE] Deleting Instance Template: a4hc0e2-login-slurm-login-20251126202605221400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc0e2-login-slurm-login-20251126202605221400000001]. -[2025-11-30 19:09:26] [SUCCESS] Deleted a4hc0e2-login-slurm-login-20251126202605221400000001 -[2025-11-30 19:09:26] [EXECUTE] Deleting Instance Template: a4hc1a3-compute-a4highnodeset-20251127090301343400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc1a3-compute-a4highnodeset-20251127090301343400000002]. -[2025-11-30 19:09:29] [SUCCESS] Deleted a4hc1a3-compute-a4highnodeset-20251127090301343400000002 -[2025-11-30 19:09:29] [EXECUTE] Deleting Instance Template: a4hc1a3-controller-default-20251127090306826200000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc1a3-controller-default-20251127090306826200000003]. -[2025-11-30 19:09:32] [SUCCESS] Deleted a4hc1a3-controller-default-20251127090306826200000003 -[2025-11-30 19:09:32] [EXECUTE] Deleting Instance Template: a4hc1a3-login-slurm-login-20251127090258428600000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hc1a3-login-slurm-login-20251127090258428600000001]. -[2025-11-30 19:09:35] [SUCCESS] Deleted a4hc1a3-login-slurm-login-20251127090258428600000001 -[2025-11-30 19:09:35] [EXECUTE] Deleting Instance Template: a4hcf79-compute-a4highnodeset-20251119140049364000000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hcf79-compute-a4highnodeset-20251119140049364000000002]. -[2025-11-30 19:09:38] [SUCCESS] Deleted a4hcf79-compute-a4highnodeset-20251119140049364000000002 -[2025-11-30 19:09:38] [EXECUTE] Deleting Instance Template: a4hcf79-controller-default-20251119140054728100000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hcf79-controller-default-20251119140054728100000003]. -[2025-11-30 19:09:41] [SUCCESS] Deleted a4hcf79-controller-default-20251119140054728100000003 -[2025-11-30 19:09:41] [EXECUTE] Deleting Instance Template: a4hcf79-login-slurm-login-20251119140045167600000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hcf79-login-slurm-login-20251119140045167600000001]. -[2025-11-30 19:09:44] [SUCCESS] Deleted a4hcf79-login-slurm-login-20251119140045167600000001 -[2025-11-30 19:09:44] [EXECUTE] Deleting Instance Template: a4he340-controller-default-20251126120236033300000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4he340-controller-default-20251126120236033300000003]. -[2025-11-30 19:09:47] [SUCCESS] Deleted a4he340-controller-default-20251126120236033300000003 -[2025-11-30 19:09:47] [EXECUTE] Deleting Instance Template: a4he340-login-slurm-login-20251126120226527400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4he340-login-slurm-login-20251126120226527400000001]. -[2025-11-30 19:09:50] [SUCCESS] Deleted a4he340-login-slurm-login-20251126120226527400000001 -[2025-11-30 19:09:50] [EXECUTE] Deleting Instance Template: a4hee35-compute-a4highnodeset-20251119063803576500000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hee35-compute-a4highnodeset-20251119063803576500000002]. -[2025-11-30 19:09:53] [SUCCESS] Deleted a4hee35-compute-a4highnodeset-20251119063803576500000002 -[2025-11-30 19:09:53] [EXECUTE] Deleting Instance Template: a4hee35-controller-default-20251119063809267100000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hee35-controller-default-20251119063809267100000003]. -[2025-11-30 19:09:56] [SUCCESS] Deleted a4hee35-controller-default-20251119063809267100000003 -[2025-11-30 19:09:56] [EXECUTE] Deleting Instance Template: a4hee35-login-slurm-login-20251119063800421400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hee35-login-slurm-login-20251119063800421400000001]. -[2025-11-30 19:09:59] [SUCCESS] Deleted a4hee35-login-slurm-login-20251119063800421400000001 -[2025-11-30 19:09:59] [EXECUTE] Deleting Instance Template: a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002]. -[2025-11-30 19:10:02] [SUCCESS] Deleted a4hf1a46f0-compute-a4highnodeset-20251116112207965900000002 -[2025-11-30 19:10:02] [EXECUTE] Deleting Instance Template: a4hf1a46f0-controller-default-20251116112212875400000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf1a46f0-controller-default-20251116112212875400000003]. -[2025-11-30 19:10:06] [SUCCESS] Deleted a4hf1a46f0-controller-default-20251116112212875400000003 -[2025-11-30 19:10:06] [EXECUTE] Deleting Instance Template: a4hf1a46f0-login-slurm-login-20251116112205108200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf1a46f0-login-slurm-login-20251116112205108200000001]. -[2025-11-30 19:10:09] [SUCCESS] Deleted a4hf1a46f0-login-slurm-login-20251116112205108200000001 -[2025-11-30 19:10:09] [EXECUTE] Deleting Instance Template: a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002]. -[2025-11-30 19:10:11] [SUCCESS] Deleted a4hf97fa5b-compute-a4highnodeset-20251116055925024400000002 -[2025-11-30 19:10:11] [EXECUTE] Deleting Instance Template: a4hf97fa5b-controller-default-20251116055930192300000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf97fa5b-controller-default-20251116055930192300000003]. -[2025-11-30 19:10:15] [SUCCESS] Deleted a4hf97fa5b-controller-default-20251116055930192300000003 -[2025-11-30 19:10:15] [EXECUTE] Deleting Instance Template: a4hf97fa5b-login-slurm-login-20251116055921750900000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hf97fa5b-login-slurm-login-20251116055921750900000001]. -[2025-11-30 19:10:18] [SUCCESS] Deleted a4hf97fa5b-login-slurm-login-20251116055921750900000001 -[2025-11-30 19:10:18] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:10:18] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:10:20] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:10:20] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:10:20] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:10:20] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:10:23] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:10:23] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:10:26] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:10:26] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:10:26] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:10:26] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:10:26] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:10:26] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:10:26] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:10:26] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:10:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:10:27] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:10:27] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:10:27] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:10:27Z (Unix: 1763320227) -[2025-11-30 19:10:27] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:10:29] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:10:29] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:10:32] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:10:32] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:10:32] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:10:32] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:10:32] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:10:32] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:10:34] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:10:34] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:10:37] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:10:37] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:10:39] [INFO] No Global Address found matching criteria. -[2025-11-30 19:10:39] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:10:41] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:10:41] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:10:44] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:10:44] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:10:44] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:10:44] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:10:44] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:46] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:47] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:10:49] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:10:49] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:10:51] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:10:51] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:12:04] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:12:04] [INFO] Time Cutoff (General): 2025-11-30T14:12:04+0000 -[2025-11-30 19:12:04] [INFO] Time Cutoff (Images): 2025-10-01T19:12:04+0000 -[2025-11-30 19:12:04] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:12:04] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:12:04] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:12:07] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:12:07] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:12:09] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:12:09] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:12:12] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:12:12] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:12:12] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:12:12] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:12:12] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:12:12] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:12:12] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4hrrsarth-compute-a4highnodeset-20251112094446322500000002 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4hrrsarth-controller-default-20251112094451660400000003 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4hrrsarth-login-slurm-login-20251112094443045200000001 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4newimgek-compute-a4highnodeset-20251121125823424400000002 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4newimgek-controller-default-20251121125828262300000003 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4newimgek-login-slurm-login-20251121125820468400000001 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4oldimgek-compute-a4highnodeset-20251121120827776000000003 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4oldimgek-controller-default-20251121120824903100000001 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4oldimgek-login-slurm-login-20251121120824904000000002 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xdemo-compute-a4xnodeset-20250904073238588700000003 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xdemo-controller-default-20250904073238579200000002 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xdemo-login-slurm-login-20250904073238576200000001 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xneel-compute-a4xnodeset-20251114072028106100000003 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xneel-controller-default-20251114072028094500000002 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xneel-login-slurm-login-20251114072028092100000001 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xqc-compute-a4xnodeset-20250925042600882300000003 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xqc-controller-default-20250925042600869000000001 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a4xqc-login-slurm-login-20250925042600873800000002 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a58b93slur-compute-nodeset-20250808184941566500000001 (Global) -[2025-11-30 19:12:12] [DRY-RUN] Would delete Instance Template: a7f7bcslur-compute-nodeset-20250804215637357700000001 (Global) -[2025-11-30 19:12:12] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:12:12] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:12:14] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:12:14] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:12:14] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:12:15] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:12:17] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:12:17] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:12:20] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:12:20] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:12:20] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:12:20] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:12:21] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:12:21] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:12:21] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:12:21] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:12:21] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:12:21] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:12:21] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:12:21] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:12:21Z (Unix: 1763320341) -[2025-11-30 19:12:21] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:12:23] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:12:23] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:12:26] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:12:26] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:12:26] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:12:26] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:12:26] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:12:26] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:12:28] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:12:28] [INFO] --- Processing: Regional Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:12:31] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:12:31] [INFO] --- Processing: Global Address (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : creationTimestamp, region -[2025-11-30 19:12:33] [INFO] No Global Address found matching criteria. -[2025-11-30 19:12:33] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:12:35] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:12:35] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:12:38] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:12:38] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:12:38] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:12:38] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:12:38] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:40] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:41] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:12:43] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:12:43] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:12:45] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:12:45] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:13:49] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:13:49] [INFO] Time Cutoff (General): 2025-11-30T14:13:49+0000 -[2025-11-30 19:13:49] [INFO] Time Cutoff (Images): 2025-10-01T19:13:49+0000 -[2025-11-30 19:13:49] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:13:49] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:13:50] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:13:52] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:13:52] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:13:54] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:13:54] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:13:57] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:13:57] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:13:57] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:13:57] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:13:57] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:13:57] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:13:57] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:13:57] [EXECUTE] Deleting Instance Template: a4hrrsarth-compute-a4highnodeset-20251112094446322500000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hrrsarth-compute-a4highnodeset-20251112094446322500000002]. -[2025-11-30 19:14:00] [SUCCESS] Deleted a4hrrsarth-compute-a4highnodeset-20251112094446322500000002 -[2025-11-30 19:14:00] [EXECUTE] Deleting Instance Template: a4hrrsarth-controller-default-20251112094451660400000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hrrsarth-controller-default-20251112094451660400000003]. -[2025-11-30 19:14:03] [SUCCESS] Deleted a4hrrsarth-controller-default-20251112094451660400000003 -[2025-11-30 19:14:03] [EXECUTE] Deleting Instance Template: a4hrrsarth-login-slurm-login-20251112094443045200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4hrrsarth-login-slurm-login-20251112094443045200000001]. -[2025-11-30 19:14:06] [SUCCESS] Deleted a4hrrsarth-login-slurm-login-20251112094443045200000001 -[2025-11-30 19:14:06] [EXECUTE] Deleting Instance Template: a4newimgek-compute-a4highnodeset-20251121125823424400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4newimgek-compute-a4highnodeset-20251121125823424400000002]. -[2025-11-30 19:14:09] [SUCCESS] Deleted a4newimgek-compute-a4highnodeset-20251121125823424400000002 -[2025-11-30 19:14:09] [EXECUTE] Deleting Instance Template: a4newimgek-controller-default-20251121125828262300000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4newimgek-controller-default-20251121125828262300000003]. -[2025-11-30 19:14:12] [SUCCESS] Deleted a4newimgek-controller-default-20251121125828262300000003 -[2025-11-30 19:14:12] [EXECUTE] Deleting Instance Template: a4newimgek-login-slurm-login-20251121125820468400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4newimgek-login-slurm-login-20251121125820468400000001]. -[2025-11-30 19:14:15] [SUCCESS] Deleted a4newimgek-login-slurm-login-20251121125820468400000001 -[2025-11-30 19:14:15] [EXECUTE] Deleting Instance Template: a4oldimgek-compute-a4highnodeset-20251121120827776000000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4oldimgek-compute-a4highnodeset-20251121120827776000000003]. -[2025-11-30 19:14:19] [SUCCESS] Deleted a4oldimgek-compute-a4highnodeset-20251121120827776000000003 -[2025-11-30 19:14:19] [EXECUTE] Deleting Instance Template: a4oldimgek-controller-default-20251121120824903100000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4oldimgek-controller-default-20251121120824903100000001]. -[2025-11-30 19:14:22] [SUCCESS] Deleted a4oldimgek-controller-default-20251121120824903100000001 -[2025-11-30 19:14:22] [EXECUTE] Deleting Instance Template: a4oldimgek-login-slurm-login-20251121120824904000000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4oldimgek-login-slurm-login-20251121120824904000000002]. -[2025-11-30 19:14:25] [SUCCESS] Deleted a4oldimgek-login-slurm-login-20251121120824904000000002 -[2025-11-30 19:14:25] [EXECUTE] Deleting Instance Template: a4xdemo-compute-a4xnodeset-20250904073238588700000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xdemo-compute-a4xnodeset-20250904073238588700000003]. -[2025-11-30 19:14:28] [SUCCESS] Deleted a4xdemo-compute-a4xnodeset-20250904073238588700000003 -[2025-11-30 19:14:28] [EXECUTE] Deleting Instance Template: a4xdemo-controller-default-20250904073238579200000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xdemo-controller-default-20250904073238579200000002]. -[2025-11-30 19:14:31] [SUCCESS] Deleted a4xdemo-controller-default-20250904073238579200000002 -[2025-11-30 19:14:31] [EXECUTE] Deleting Instance Template: a4xdemo-login-slurm-login-20250904073238576200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xdemo-login-slurm-login-20250904073238576200000001]. -[2025-11-30 19:14:34] [SUCCESS] Deleted a4xdemo-login-slurm-login-20250904073238576200000001 -[2025-11-30 19:14:34] [EXECUTE] Deleting Instance Template: a4xneel-compute-a4xnodeset-20251114072028106100000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xneel-compute-a4xnodeset-20251114072028106100000003]. -[2025-11-30 19:14:37] [SUCCESS] Deleted a4xneel-compute-a4xnodeset-20251114072028106100000003 -[2025-11-30 19:14:37] [EXECUTE] Deleting Instance Template: a4xneel-controller-default-20251114072028094500000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xneel-controller-default-20251114072028094500000002]. -[2025-11-30 19:14:40] [SUCCESS] Deleted a4xneel-controller-default-20251114072028094500000002 -[2025-11-30 19:14:40] [EXECUTE] Deleting Instance Template: a4xneel-login-slurm-login-20251114072028092100000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xneel-login-slurm-login-20251114072028092100000001]. -[2025-11-30 19:14:43] [SUCCESS] Deleted a4xneel-login-slurm-login-20251114072028092100000001 -[2025-11-30 19:14:43] [EXECUTE] Deleting Instance Template: a4xqc-compute-a4xnodeset-20250925042600882300000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xqc-compute-a4xnodeset-20250925042600882300000003]. -[2025-11-30 19:14:46] [SUCCESS] Deleted a4xqc-compute-a4xnodeset-20250925042600882300000003 -[2025-11-30 19:14:46] [EXECUTE] Deleting Instance Template: a4xqc-controller-default-20250925042600869000000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xqc-controller-default-20250925042600869000000001]. -[2025-11-30 19:14:49] [SUCCESS] Deleted a4xqc-controller-default-20250925042600869000000001 -[2025-11-30 19:14:49] [EXECUTE] Deleting Instance Template: a4xqc-login-slurm-login-20250925042600873800000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a4xqc-login-slurm-login-20250925042600873800000002]. -[2025-11-30 19:14:52] [SUCCESS] Deleted a4xqc-login-slurm-login-20250925042600873800000002 -[2025-11-30 19:14:52] [EXECUTE] Deleting Instance Template: a58b93slur-compute-nodeset-20250808184941566500000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a58b93slur-compute-nodeset-20250808184941566500000001]. -[2025-11-30 19:14:56] [SUCCESS] Deleted a58b93slur-compute-nodeset-20250808184941566500000001 -[2025-11-30 19:14:56] [EXECUTE] Deleting Instance Template: a7f7bcslur-compute-nodeset-20250804215637357700000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a7f7bcslur-compute-nodeset-20250804215637357700000001]. -[2025-11-30 19:14:59] [SUCCESS] Deleted a7f7bcslur-compute-nodeset-20250804215637357700000001 -[2025-11-30 19:14:59] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:14:59] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:15:01] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:15:01] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:15:01] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:15:01] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:15:04] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:15:04] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:15:07] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:15:07] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:15:07] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:15:07] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:15:07] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:15:07] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:15:07] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:15:07] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:15:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:15:08] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:15:08] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:15:08] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:15:08Z (Unix: 1763320508) -[2025-11-30 19:15:08] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:15:10] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:15:10] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:15:12] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:15:12] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:15:12] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:15:12] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:15:12] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:15:12] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:15:15] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:15:15] [INFO] --- Processing: Regional Address (Limit: 20) --- -[2025-11-30 19:15:17] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:15:17] [INFO] --- Processing: Global Address (Limit: 20) --- -[2025-11-30 19:15:19] [INFO] No Global Address found matching criteria. -[2025-11-30 19:15:19] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:15:22] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:15:22] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:15:24] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:15:24] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:15:24] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:15:24] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:15:24] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:27] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:15:29] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:15:29] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:15:31] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:15:31] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:15:41] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:15:41] [INFO] Time Cutoff (General): 2025-11-30T14:15:41+0000 -[2025-11-30 19:15:41] [INFO] Time Cutoff (Images): 2025-10-01T19:15:41+0000 -[2025-11-30 19:15:41] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:15:41] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:15:42] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:15:44] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:15:44] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -[2025-11-30 19:15:46] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:15:46] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:15:49] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:15:49] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:15:49] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:15:49] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:15:49] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:15:49] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:15:49] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: a7f7bcslur-controller-default-20250804215646842800000003 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: a7f7bcslur-login-slurm-login-20250804215637381200000002 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: a94slurmfl-compute-nodeset-20251018073312378100000002 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: aa5daslurm-compute-nodeset-20250710053732511400000001 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: acbslurmsi-compute-nodeset-20250815192152759600000002 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: aslurmflex-compute-nodeset-20251007191438377100000001 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: b168slurmf-compute-nodeset-20251120173723559000000001 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: batch-job-instance-template-20250901212237920900000001 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: batch-job-instance-template-20250912070019961500000001 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: be1slurmfl-compute-nodeset-20251118223535177700000001 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: bfa462slur-compute-nodeset-20250912051104900400000001 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: bfa462slur-controller-default-20250912051114475200000003 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: bfa462slur-login-slurm-login-20250912051104953500000002 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: buildslurm-compute-debugnodeset-20251030080636567600000001 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: buildslurm-controller-default-20251030080646114800000002 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: c0aslurmfl-compute-nodeset-20251010073733464500000002 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: c17slurmfl-compute-nodeset-20250911220802714700000001 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: c2dtest7-compute-c2dnodeset-20250926070704394500000003 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: c2dtest7-controller-default-20250926070704365100000001 (Global) -[2025-11-30 19:15:49] [DRY-RUN] Would delete Instance Template: c2dtest7-login-slurm-login-20250926070704373700000002 (Global) -[2025-11-30 19:15:49] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:15:49] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:15:52] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:15:52] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:15:52] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:15:52] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:15:54] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:15:54] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:15:57] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:15:57] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:15:57] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:15:57] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:15:58] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:15:58] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:15:58] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:15:58] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:15:58] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:15:58] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:15:58] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:15:58] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:15:58Z (Unix: 1763320558) -[2025-11-30 19:15:58] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:16:00] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:16:00] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:16:03] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:16:03] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:16:03] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:16:03] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:16:03] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:16:03] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:16:05] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:16:05] [INFO] --- Processing: Regional Address (Limit: 20) --- -[2025-11-30 19:16:08] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:16:08] [INFO] --- Processing: Global Address (Limit: 20) --- -[2025-11-30 19:16:10] [INFO] No Global Address found matching criteria. -[2025-11-30 19:16:10] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:16:13] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:16:13] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:16:15] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:16:15] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:16:15] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:16:15] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:16:15] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:18] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:16:20] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:16:20] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:16:22] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:16:22] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:16:46] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:16:46] [INFO] Time Cutoff (General): 2025-11-30T14:16:46+0000 -[2025-11-30 19:16:46] [INFO] Time Cutoff (Images): 2025-10-01T19:16:46+0000 -[2025-11-30 19:16:46] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:16:46] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:16:47] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:16:49] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:16:49] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -[2025-11-30 19:16:51] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:16:51] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:16:54] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:16:54] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:16:54] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:16:54] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:16:54] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:16:54] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:16:54] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: a7f7bcslur-controller-default-20250804215646842800000003 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: a7f7bcslur-login-slurm-login-20250804215637381200000002 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: a94slurmfl-compute-nodeset-20251018073312378100000002 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: aa5daslurm-compute-nodeset-20250710053732511400000001 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: acbslurmsi-compute-nodeset-20250815192152759600000002 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: aslurmflex-compute-nodeset-20251007191438377100000001 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: b168slurmf-compute-nodeset-20251120173723559000000001 (Global) -[2025-11-30 19:16:54] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:16:54] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: be1slurmfl-compute-nodeset-20251118223535177700000001 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: bfa462slur-compute-nodeset-20250912051104900400000001 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: bfa462slur-controller-default-20250912051114475200000003 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: bfa462slur-login-slurm-login-20250912051104953500000002 (Global) -[2025-11-30 19:16:54] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:16:54] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c0aslurmfl-compute-nodeset-20251010073733464500000002 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c17slurmfl-compute-nodeset-20250911220802714700000001 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c2dtest7-compute-c2dnodeset-20250926070704394500000003 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c2dtest7-controller-default-20250926070704365100000001 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c2dtest7-login-slurm-login-20250926070704373700000002 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c379slurms-compute-nodeset-20250815194346833100000002 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c379slurms-controller-default-20250815194356672000000003 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c379slurms-login-slurm-login-20250815194346831100000001 (Global) -[2025-11-30 19:16:54] [DRY-RUN] Would delete Instance Template: c52cb1fa4h-compute-a4highnodeset-20251107160458966300000002 (Global) -[2025-11-30 19:16:54] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:16:54] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:16:56] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:16:56] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:16:56] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:16:56] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:16:59] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:16:59] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:17:02] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:17:02] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:17:02] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:17:02] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:17:03] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:17:03] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:17:03] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:17:03] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:17:03] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:17:03] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:17:03] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:17:03] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:17:03Z (Unix: 1763320623) -[2025-11-30 19:17:03] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:17:06] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:17:06] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:17:08] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:17:08] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:17:08] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:17:08] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:17:08] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:17:08] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:17:11] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:17:11] [INFO] --- Processing: Regional Address (Limit: 20) --- -[2025-11-30 19:17:13] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:17:13] [INFO] --- Processing: Global Address (Limit: 20) --- -[2025-11-30 19:17:15] [INFO] No Global Address found matching criteria. -[2025-11-30 19:17:15] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:17:18] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:17:18] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:17:20] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:17:20] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:17:20] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:17:20] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:17:20] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:23] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:17:26] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:17:26] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:17:27] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:17:27] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:17:56] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:17:56] [INFO] Time Cutoff (General): 2025-11-30T14:17:56+0000 -[2025-11-30 19:17:56] [INFO] Time Cutoff (Images): 2025-10-01T19:17:56+0000 -[2025-11-30 19:17:56] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:17:57] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:17:57] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:18:00] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:18:00] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -[2025-11-30 19:18:02] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:18:02] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:18:05] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:18:05] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:18:05] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:18:05] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:18:05] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:18:05] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:18:05] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:18:05] [EXECUTE] Deleting Instance Template: a7f7bcslur-controller-default-20250804215646842800000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a7f7bcslur-controller-default-20250804215646842800000003]. -[2025-11-30 19:18:08] [SUCCESS] Deleted a7f7bcslur-controller-default-20250804215646842800000003 -[2025-11-30 19:18:08] [EXECUTE] Deleting Instance Template: a7f7bcslur-login-slurm-login-20250804215637381200000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a7f7bcslur-login-slurm-login-20250804215637381200000002]. -[2025-11-30 19:18:11] [SUCCESS] Deleted a7f7bcslur-login-slurm-login-20250804215637381200000002 -[2025-11-30 19:18:11] [EXECUTE] Deleting Instance Template: a94slurmfl-compute-nodeset-20251018073312378100000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/a94slurmfl-compute-nodeset-20251018073312378100000002]. -[2025-11-30 19:18:14] [SUCCESS] Deleted a94slurmfl-compute-nodeset-20251018073312378100000002 -[2025-11-30 19:18:14] [EXECUTE] Deleting Instance Template: aa5daslurm-compute-nodeset-20250710053732511400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/aa5daslurm-compute-nodeset-20250710053732511400000001]. -[2025-11-30 19:18:17] [SUCCESS] Deleted aa5daslurm-compute-nodeset-20250710053732511400000001 -[2025-11-30 19:18:17] [EXECUTE] Deleting Instance Template: acbslurmsi-compute-nodeset-20250815192152759600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/acbslurmsi-compute-nodeset-20250815192152759600000002]. -[2025-11-30 19:18:20] [SUCCESS] Deleted acbslurmsi-compute-nodeset-20250815192152759600000002 -[2025-11-30 19:18:20] [EXECUTE] Deleting Instance Template: aslurmflex-compute-nodeset-20251007191438377100000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/aslurmflex-compute-nodeset-20251007191438377100000001]. -[2025-11-30 19:18:23] [SUCCESS] Deleted aslurmflex-compute-nodeset-20251007191438377100000001 -[2025-11-30 19:18:23] [EXECUTE] Deleting Instance Template: b168slurmf-compute-nodeset-20251120173723559000000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/b168slurmf-compute-nodeset-20251120173723559000000001]. -[2025-11-30 19:18:27] [SUCCESS] Deleted b168slurmf-compute-nodeset-20251120173723559000000001 -[2025-11-30 19:18:27] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:18:27] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:18:27] [EXECUTE] Deleting Instance Template: be1slurmfl-compute-nodeset-20251118223535177700000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/be1slurmfl-compute-nodeset-20251118223535177700000001]. -[2025-11-30 19:18:30] [SUCCESS] Deleted be1slurmfl-compute-nodeset-20251118223535177700000001 -[2025-11-30 19:18:30] [EXECUTE] Deleting Instance Template: bfa462slur-compute-nodeset-20250912051104900400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/bfa462slur-compute-nodeset-20250912051104900400000001]. -[2025-11-30 19:18:33] [SUCCESS] Deleted bfa462slur-compute-nodeset-20250912051104900400000001 -[2025-11-30 19:18:33] [EXECUTE] Deleting Instance Template: bfa462slur-controller-default-20250912051114475200000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/bfa462slur-controller-default-20250912051114475200000003]. -[2025-11-30 19:18:36] [SUCCESS] Deleted bfa462slur-controller-default-20250912051114475200000003 -[2025-11-30 19:18:36] [EXECUTE] Deleting Instance Template: bfa462slur-login-slurm-login-20250912051104953500000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/bfa462slur-login-slurm-login-20250912051104953500000002]. -[2025-11-30 19:18:39] [SUCCESS] Deleted bfa462slur-login-slurm-login-20250912051104953500000002 -[2025-11-30 19:18:39] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:18:39] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:18:39] [EXECUTE] Deleting Instance Template: c0aslurmfl-compute-nodeset-20251010073733464500000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c0aslurmfl-compute-nodeset-20251010073733464500000002]. -[2025-11-30 19:18:43] [SUCCESS] Deleted c0aslurmfl-compute-nodeset-20251010073733464500000002 -[2025-11-30 19:18:43] [EXECUTE] Deleting Instance Template: c17slurmfl-compute-nodeset-20250911220802714700000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c17slurmfl-compute-nodeset-20250911220802714700000001]. -[2025-11-30 19:18:46] [SUCCESS] Deleted c17slurmfl-compute-nodeset-20250911220802714700000001 -[2025-11-30 19:18:46] [EXECUTE] Deleting Instance Template: c2dtest7-compute-c2dnodeset-20250926070704394500000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c2dtest7-compute-c2dnodeset-20250926070704394500000003]. -[2025-11-30 19:18:49] [SUCCESS] Deleted c2dtest7-compute-c2dnodeset-20250926070704394500000003 -[2025-11-30 19:18:49] [EXECUTE] Deleting Instance Template: c2dtest7-controller-default-20250926070704365100000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c2dtest7-controller-default-20250926070704365100000001]. -[2025-11-30 19:18:53] [SUCCESS] Deleted c2dtest7-controller-default-20250926070704365100000001 -[2025-11-30 19:18:53] [EXECUTE] Deleting Instance Template: c2dtest7-login-slurm-login-20250926070704373700000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c2dtest7-login-slurm-login-20250926070704373700000002]. -[2025-11-30 19:18:56] [SUCCESS] Deleted c2dtest7-login-slurm-login-20250926070704373700000002 -[2025-11-30 19:18:56] [EXECUTE] Deleting Instance Template: c379slurms-compute-nodeset-20250815194346833100000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c379slurms-compute-nodeset-20250815194346833100000002]. -[2025-11-30 19:18:59] [SUCCESS] Deleted c379slurms-compute-nodeset-20250815194346833100000002 -[2025-11-30 19:18:59] [EXECUTE] Deleting Instance Template: c379slurms-controller-default-20250815194356672000000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c379slurms-controller-default-20250815194356672000000003]. -[2025-11-30 19:19:02] [SUCCESS] Deleted c379slurms-controller-default-20250815194356672000000003 -[2025-11-30 19:19:02] [EXECUTE] Deleting Instance Template: c379slurms-login-slurm-login-20250815194346831100000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c379slurms-login-slurm-login-20250815194346831100000001]. -[2025-11-30 19:19:06] [SUCCESS] Deleted c379slurms-login-slurm-login-20250815194346831100000001 -[2025-11-30 19:19:06] [EXECUTE] Deleting Instance Template: c52cb1fa4h-compute-a4highnodeset-20251107160458966300000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/c52cb1fa4h-compute-a4highnodeset-20251107160458966300000002]. -[2025-11-30 19:19:09] [SUCCESS] Deleted c52cb1fa4h-compute-a4highnodeset-20251107160458966300000002 -[2025-11-30 19:19:09] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:19:09] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:19:12] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:19:12] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:19:12] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:19:12] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -WARNING: The following filter keys were not present in any resource : createTime -[2025-11-30 19:19:14] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:19:14] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:19:17] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:19:17] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:19:18] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:19:18] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:19:18] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:19:18] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:19:18] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:19:18] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:19:18] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:19:18] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:19:18] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:19:18] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:19:18Z (Unix: 1763320758) -[2025-11-30 19:19:18] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:19:21] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:19:21] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:19:23] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:19:23] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:19:23] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:19:23] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:19:23] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:19:23] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:19:26] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:19:26] [INFO] --- Processing: Regional Address (Limit: 20) --- -[2025-11-30 19:19:28] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:19:28] [INFO] --- Processing: Global Address (Limit: 20) --- -[2025-11-30 19:19:30] [INFO] No Global Address found matching criteria. -[2025-11-30 19:19:30] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:19:33] [INFO] Finished processing VPC Peerings. 0 peerings actioned. -[2025-11-30 19:19:33] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:19:35] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:19:35] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:19:35] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:19:35] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:19:35] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:38] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:19:41] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:19:41] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:19:43] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:19:43] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:19:52] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:19:52] [INFO] Time Cutoff (General): 2025-11-30T14:19:52+0000 -[2025-11-30 19:19:52] [INFO] Time Cutoff (Images): 2025-10-01T19:19:52+0000 -[2025-11-30 19:19:52] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:19:52] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:19:53] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:19:55] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:19:55] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -[2025-11-30 19:19:57] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:19:57] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:20:00] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:20:00] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:20:00] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:20:00] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:20:00] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:20:00] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:20:00] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:20:00] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:20:00] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:20:00] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:20:00] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: ca2slurmfl-compute-nodeset-20251021163807550000000002 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: ccslurmfle-compute-nodeset-20251023183424836600000002 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: ce64slurms-compute-nodeset-20250821083541488100000002 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: ce64slurms-controller-default-20250821083551070400000003 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: ce64slurms-login-slurm-login-20250821083541466900000001 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3eslurmsi-compute-nodeset-20250804164233979200000001 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3eslurmsi-controller-default-20250804164243493900000003 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3eslurmsi-login-slurm-login-20250804164233980900000002 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3slurmsim-compute-nodeset-20250801211726043400000001 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3slurmsim-controller-default-20250801211735786100000003 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d3slurmsim-login-slurm-login-20250801211726053600000002 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d72c8slurm-compute-nodeset-20251121180541385700000002 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d72c8slurm-controller-default-20251121180552318500000003 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: d72c8slurm-login-slurm-login-20251121180541344900000001 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: de3580slur-compute-nodeset-20250725053828889300000001 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: de3580slur-controller-default-20250725053838683600000003 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: de3580slur-login-slurm-login-20250725053828911000000002 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: deb6slurmf-compute-nodeset-20250612173759618500000002 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: dynpoc-compute-computenodeset-20251015002137498100000004 (Global) -[2025-11-30 19:20:00] [DRY-RUN] Would delete Instance Template: dynpoc-compute-debugnodeset-20251015002137496700000003 (Global) -[2025-11-30 19:20:00] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:20:00] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:20:03] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:20:03] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:20:03] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:20:03] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -[2025-11-30 19:20:06] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:20:06] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:20:09] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:20:09] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:20:09] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:20:09] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:20:09] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:20:09] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:20:10] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:20:10] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:20:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:20:10] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:20:10] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:20:10] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:20:10Z (Unix: 1763320810) -[2025-11-30 19:20:10] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:20:12] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:20:12] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:20:15] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:20:15] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:20:15] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:20:15] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:20:15] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:20:15] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:20:17] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:20:17] [INFO] --- Processing: Regional Address (Limit: 20) --- -[2025-11-30 19:20:20] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:20:20] [INFO] --- Processing: Global Address (Limit: 20) --- -[2025-11-30 19:20:22] [INFO] No Global Address found matching criteria. -[2025-11-30 19:20:22] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:20:25] [INFO] [ACTION] Deleting Service Networking peering on: hpc-enterprise-slurm-v6 -[2025-11-30 19:20:25] [DRY-RUN] Would delete Service Peering: servicenetworking-googleapis-com (Network: hpc-enterprise-slurm-v6) -[2025-11-30 19:20:25] [INFO] Finished processing VPC Peerings. 1 peerings actioned. -[2025-11-30 19:20:25] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:20:28] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:20:28] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:20:28] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:20:28] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:20:28] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:30] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:31] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:31] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:20:33] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:20:33] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:20:35] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:20:35] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:21:16] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:21:16] [INFO] Time Cutoff (General): 2025-11-30T14:21:16+0000 -[2025-11-30 19:21:16] [INFO] Time Cutoff (Images): 2025-10-01T19:21:16+0000 -[2025-11-30 19:21:16] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:21:16] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:21:17] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:21:19] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:21:19] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -[2025-11-30 19:21:21] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:21:21] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:21:24] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:21:24] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:21:24] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:21:24] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:21:24] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:21:24] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:21:24] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:21:24] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:21:24] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:21:24] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:21:24] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:21:24] [EXECUTE] Deleting Instance Template: ca2slurmfl-compute-nodeset-20251021163807550000000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ca2slurmfl-compute-nodeset-20251021163807550000000002]. -[2025-11-30 19:21:27] [SUCCESS] Deleted ca2slurmfl-compute-nodeset-20251021163807550000000002 -[2025-11-30 19:21:27] [EXECUTE] Deleting Instance Template: ccslurmfle-compute-nodeset-20251023183424836600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ccslurmfle-compute-nodeset-20251023183424836600000002]. -[2025-11-30 19:21:30] [SUCCESS] Deleted ccslurmfle-compute-nodeset-20251023183424836600000002 -[2025-11-30 19:21:30] [EXECUTE] Deleting Instance Template: ce64slurms-compute-nodeset-20250821083541488100000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ce64slurms-compute-nodeset-20250821083541488100000002]. -[2025-11-30 19:21:33] [SUCCESS] Deleted ce64slurms-compute-nodeset-20250821083541488100000002 -[2025-11-30 19:21:33] [EXECUTE] Deleting Instance Template: ce64slurms-controller-default-20250821083551070400000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ce64slurms-controller-default-20250821083551070400000003]. -[2025-11-30 19:21:36] [SUCCESS] Deleted ce64slurms-controller-default-20250821083551070400000003 -[2025-11-30 19:21:36] [EXECUTE] Deleting Instance Template: ce64slurms-login-slurm-login-20250821083541466900000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ce64slurms-login-slurm-login-20250821083541466900000001]. -[2025-11-30 19:21:39] [SUCCESS] Deleted ce64slurms-login-slurm-login-20250821083541466900000001 -[2025-11-30 19:21:39] [EXECUTE] Deleting Instance Template: d3eslurmsi-compute-nodeset-20250804164233979200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3eslurmsi-compute-nodeset-20250804164233979200000001]. -[2025-11-30 19:21:43] [SUCCESS] Deleted d3eslurmsi-compute-nodeset-20250804164233979200000001 -[2025-11-30 19:21:43] [EXECUTE] Deleting Instance Template: d3eslurmsi-controller-default-20250804164243493900000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3eslurmsi-controller-default-20250804164243493900000003]. -[2025-11-30 19:21:46] [SUCCESS] Deleted d3eslurmsi-controller-default-20250804164243493900000003 -[2025-11-30 19:21:46] [EXECUTE] Deleting Instance Template: d3eslurmsi-login-slurm-login-20250804164233980900000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3eslurmsi-login-slurm-login-20250804164233980900000002]. -[2025-11-30 19:21:49] [SUCCESS] Deleted d3eslurmsi-login-slurm-login-20250804164233980900000002 -[2025-11-30 19:21:49] [EXECUTE] Deleting Instance Template: d3slurmsim-compute-nodeset-20250801211726043400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3slurmsim-compute-nodeset-20250801211726043400000001]. -[2025-11-30 19:21:52] [SUCCESS] Deleted d3slurmsim-compute-nodeset-20250801211726043400000001 -[2025-11-30 19:21:52] [EXECUTE] Deleting Instance Template: d3slurmsim-controller-default-20250801211735786100000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3slurmsim-controller-default-20250801211735786100000003]. -[2025-11-30 19:21:55] [SUCCESS] Deleted d3slurmsim-controller-default-20250801211735786100000003 -[2025-11-30 19:21:55] [EXECUTE] Deleting Instance Template: d3slurmsim-login-slurm-login-20250801211726053600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d3slurmsim-login-slurm-login-20250801211726053600000002]. -[2025-11-30 19:21:58] [SUCCESS] Deleted d3slurmsim-login-slurm-login-20250801211726053600000002 -[2025-11-30 19:21:58] [EXECUTE] Deleting Instance Template: d72c8slurm-compute-nodeset-20251121180541385700000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d72c8slurm-compute-nodeset-20251121180541385700000002]. -[2025-11-30 19:22:01] [SUCCESS] Deleted d72c8slurm-compute-nodeset-20251121180541385700000002 -[2025-11-30 19:22:01] [EXECUTE] Deleting Instance Template: d72c8slurm-controller-default-20251121180552318500000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d72c8slurm-controller-default-20251121180552318500000003]. -[2025-11-30 19:22:04] [SUCCESS] Deleted d72c8slurm-controller-default-20251121180552318500000003 -[2025-11-30 19:22:04] [EXECUTE] Deleting Instance Template: d72c8slurm-login-slurm-login-20251121180541344900000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/d72c8slurm-login-slurm-login-20251121180541344900000001]. -[2025-11-30 19:22:07] [SUCCESS] Deleted d72c8slurm-login-slurm-login-20251121180541344900000001 -[2025-11-30 19:22:07] [EXECUTE] Deleting Instance Template: de3580slur-compute-nodeset-20250725053828889300000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/de3580slur-compute-nodeset-20250725053828889300000001]. -[2025-11-30 19:22:10] [SUCCESS] Deleted de3580slur-compute-nodeset-20250725053828889300000001 -[2025-11-30 19:22:10] [EXECUTE] Deleting Instance Template: de3580slur-controller-default-20250725053838683600000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/de3580slur-controller-default-20250725053838683600000003]. -[2025-11-30 19:22:13] [SUCCESS] Deleted de3580slur-controller-default-20250725053838683600000003 -[2025-11-30 19:22:13] [EXECUTE] Deleting Instance Template: de3580slur-login-slurm-login-20250725053828911000000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/de3580slur-login-slurm-login-20250725053828911000000002]. -[2025-11-30 19:22:16] [SUCCESS] Deleted de3580slur-login-slurm-login-20250725053828911000000002 -[2025-11-30 19:22:16] [EXECUTE] Deleting Instance Template: deb6slurmf-compute-nodeset-20250612173759618500000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/deb6slurmf-compute-nodeset-20250612173759618500000002]. -[2025-11-30 19:22:19] [SUCCESS] Deleted deb6slurmf-compute-nodeset-20250612173759618500000002 -[2025-11-30 19:22:19] [EXECUTE] Deleting Instance Template: dynpoc-compute-computenodeset-20251015002137498100000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/dynpoc-compute-computenodeset-20251015002137498100000004]. -[2025-11-30 19:22:23] [SUCCESS] Deleted dynpoc-compute-computenodeset-20251015002137498100000004 -[2025-11-30 19:22:23] [EXECUTE] Deleting Instance Template: dynpoc-compute-debugnodeset-20251015002137496700000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/dynpoc-compute-debugnodeset-20251015002137496700000003]. -[2025-11-30 19:22:26] [SUCCESS] Deleted dynpoc-compute-debugnodeset-20251015002137496700000003 -[2025-11-30 19:22:26] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:22:26] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:22:28] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:22:28] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:22:28] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:22:28] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -[2025-11-30 19:22:31] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:22:31] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:22:34] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:22:34] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:22:34] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:22:34] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:22:34] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:22:34] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:22:34] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:22:34] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:22:35] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:22:35] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:22:35] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:22:35] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:22:35Z (Unix: 1763320955) -[2025-11-30 19:22:35] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:22:37] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:22:37] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:22:40] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:22:40] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:22:40] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:22:40] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:22:40] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:22:40] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:22:42] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:22:42] [INFO] --- Processing: Regional Address (Limit: 20) --- -[2025-11-30 19:22:45] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:22:45] [INFO] --- Processing: Global Address (Limit: 20) --- -[2025-11-30 19:22:47] [INFO] No Global Address found matching criteria. -[2025-11-30 19:22:47] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:22:49] [INFO] [ACTION] Deleting Service Networking peering on: hpc-enterprise-slurm-v6 -[2025-11-30 19:22:49] [EXECUTE] Deleting Service Peering: servicenetworking-googleapis-com (Network: hpc-enterprise-slurm-v6) -ERROR: (gcloud.services.vpc-peerings.delete) The operation "operations/dcf.p40-508417052821-756785e9-1aea-4f3a-b7df-d81c1cb8c5cd" resulted in a failure "Failed to delete connection; Producer services (e.g. CloudSQL, Cloud Memstore, etc.) are still using this connection. -Help Token: AXcLsyCsa6BMQ5F6c1hxIFPD8mjSOLxYM2QIXmOSrXbB3xGvT_0N_wHZ5T8h59ZN4tAIsXbZYJMbUOA8_D9BSf-69dS_M3lvNEO5FdIEhC4dFWyZ". -Details: "[>, >, >]>>]>>>]>, >, >, >]>]". -[2025-11-30 19:23:05] [ERROR] Failed to delete servicenetworking-googleapis-com -[2025-11-30 19:23:05] [INFO] Finished processing VPC Peerings. 1 peerings actioned. -[2025-11-30 19:23:05] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:23:07] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:23:07] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:23:07] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:23:07] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:23:07] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:10] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:23:13] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:23:13] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:23:15] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:23:15] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:25:29] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:25:29] [INFO] Time Cutoff (General): 2025-11-30T14:25:29+0000 -[2025-11-30 19:25:29] [INFO] Time Cutoff (Images): 2025-10-01T19:25:29+0000 -[2025-11-30 19:25:29] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:25:29] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:25:30] [INFO] --- Processing: Service Accounts (Prefix: test-sa-) --- -[2025-11-30 19:25:32] [INFO] No Service Accounts found matching prefix. -[2025-11-30 19:25:32] [INFO] --- Processing: GKE Cluster (Limit: 20) --- -[2025-11-30 19:25:34] [INFO] No GKE Cluster found matching criteria. -[2025-11-30 19:25:34] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:25:37] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:25:37] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:25:37] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:25:37] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:25:37] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:25:37] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:25:37] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:25:37] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:25:37] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:25:37] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:25:37] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: dynpoc-compute-h3nodeset-20251015002137498900000005 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: dynpoc-controller-default-20251015002137476700000001 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: dynpoc-login-slurm-login-20251015002137480600000002 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: e2dbe6slur-compute-nodeset-20251007201508565800000001 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: e3879cslur-compute-nodeset-20250919212831665800000002 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: eaa3beslur-compute-nodeset-20251124070522321600000001 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: ebf828slur-compute-nodeset-20251124071134868500000001 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: ebf828slur-controller-default-20251124071144586200000003 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: ebf828slur-login-slurm-login-20251124071134910400000002 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f4944slurm-compute-nodeset-20250912131529204200000001 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f4e324slur-compute-nodeset-20250811165336641200000001 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f4e324slur-controller-default-20250811165346184100000003 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f4e324slur-login-slurm-login-20250811165336654300000002 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f88073slur-compute-nodeset-20250826031610614500000002 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f88073slur-controller-default-20250826031620053300000003 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: f88073slur-login-slurm-login-20250826031610570200000001 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: fa4slurmsi-compute-nodeset-20250808051559443200000002 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: fa4slurmsi-controller-default-20250808051608790400000003 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: fa4slurmsi-login-slurm-login-20250808051559433300000001 (Global) -[2025-11-30 19:25:37] [DRY-RUN] Would delete Instance Template: g4qclav-compute-g4nodeset-20251111194038852400000002 (Global) -[2025-11-30 19:25:37] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:25:37] [INFO] --- Processing: Compute Instance (Limit: 20) --- -[2025-11-30 19:25:40] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:25:40] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:25:40] [SKIP] vertexui-do-not-kill (In Exclusion List) -[2025-11-30 19:25:40] [INFO] --- Processing: Filestore Instances (Limit: 20) --- -[2025-11-30 19:25:43] [INFO] No Filestore instances found matching criteria. -[2025-11-30 19:25:43] [INFO] --- Processing: VM Images (Limit: 20) --- -[2025-11-30 19:25:45] [SKIP] a3u-image-u22-20250325t162635z (In Exclusion List) -[2025-11-30 19:25:45] [SKIP] a4high-image-builder-20250214t220935z (In Exclusion List) -[2025-11-30 19:25:46] [SKIP] chs-dcgmi-metric-u22-20250925t121709z (In Exclusion List) -[2025-11-30 19:25:46] [SKIP] common-slurm-image-20250725t234825z (In Exclusion List) -[2025-11-30 19:25:46] [SKIP] harsh-a4-image (In Exclusion List) -[2025-11-30 19:25:46] [SKIP] pbspro0 (In Exclusion List) -[2025-11-30 19:25:46] [SKIP] rocka4h-rocky9-20250908t175724z (In Exclusion List) -[2025-11-30 19:25:46] [SKIP] rocka4hf-rocky9-20250910t040750z (In Exclusion List) -[2025-11-30 19:25:46] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1739990978 (In Exclusion List) -[2025-11-30 19:25:46] [SKIP] slurm-gcp-next-hpc-rocky-linux-8-1740100297 (In Exclusion List) -[2025-11-30 19:25:46] [INFO] --- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: 20) --- -[2025-11-30 19:25:46] [INFO] Policy: Delete 'test-runner' images updated before 2025-11-16T19:25:46Z (Unix: 1763321146) -[2025-11-30 19:25:46] [INFO] Scanning Target Package: us-central1-docker.pkg.dev/hpc-toolkit-dev/hpc-toolkit-repo/test-runner -Listing items under project hpc-toolkit-dev, location us-central1, repository hpc-toolkit-repo. - -[2025-11-30 19:25:48] [INFO] Finished Docker Image processing for test-runner. 0 images marked for deletion. -[2025-11-30 19:25:48] [INFO] --- Processing: Cloud Router (Limit: 20) --- -[2025-11-30 19:25:51] [SKIP] default-net-router (In Exclusion List) -[2025-11-30 19:25:51] [SKIP] default-router-australia-southeast1 (In Exclusion List) -[2025-11-30 19:25:51] [SKIP] default-router-us-east4 (In Exclusion List) -[2025-11-30 19:25:51] [SKIP] default-router-us-west1 (In Exclusion List) -[2025-11-30 19:25:51] [SKIP] default-router-us-west4 (In Exclusion List) -[2025-11-30 19:25:51] [INFO] --- Processing: Firewall Rules (Limit: 20) --- -[2025-11-30 19:25:53] [INFO] --- Processing: Compute Addresses --- -[2025-11-30 19:25:53] [INFO] --- Processing: Regional Address (Limit: 20) --- -[2025-11-30 19:25:55] [INFO] No Regional Address found matching criteria. -[2025-11-30 19:25:55] [INFO] --- Processing: Global Address (Limit: 20) --- -[2025-11-30 19:25:58] [INFO] No Global Address found matching criteria. -[2025-11-30 19:25:58] [INFO] --- Processing: VPC Peerings (Limit: 20) --- -[2025-11-30 19:26:01] [INFO] [ACTION] Deleting Service Networking peering on: hpc-enterprise-slurm-v6 -[2025-11-30 19:26:01] [DRY-RUN] Would delete Service Peering: servicenetworking-googleapis-com (Network: hpc-enterprise-slurm-v6) -[2025-11-30 19:26:01] [INFO] Finished processing VPC Peerings. 1 peerings actioned. -[2025-11-30 19:26:01] [INFO] --- Processing: Zonal Disk (Limit: 20) --- -[2025-11-30 19:26:03] [SKIP] image-inspector-550 (In Exclusion List) -[2025-11-30 19:26:03] [SKIP] image-inspector (In Exclusion List) -[2025-11-30 19:26:03] [SKIP] vertexui-do-not-kill-boot (In Exclusion List) -[2025-11-30 19:26:03] [SKIP] vertexui-do-not-kill-data (In Exclusion List) -[2025-11-30 19:26:03] [INFO] --- Processing: Subnetworks (Limit: 20) --- -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:06] [INFO] --- Processing: VPC Networks (Limit: 20) --- -[2025-11-30 19:26:08] [SKIP] hpc-vpc (In Exclusion List) -[2025-11-30 19:26:08] [INFO] --- Processing: IAM Role Bindings for Deleted SAs (Limit: 20) --- -[2025-11-30 19:26:10] [INFO] No 'deleted:serviceAccount' bindings found. -[2025-11-30 19:26:10] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:27:59] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:27:59] [INFO] Time Cutoff (General): 2025-11-30T14:27:59+0000 -[2025-11-30 19:27:59] [INFO] Time Cutoff (Images): 2025-10-01T19:27:59+0000 -[2025-11-30 19:27:59] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:27:59] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:27:59] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:28:03] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:28:03] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:28:03] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:28:03] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:28:03] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:28:03] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:28:03] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:28:03] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:28:03] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:28:03] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:28:03] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: dynpoc-compute-h3nodeset-20251015002137498900000005 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: dynpoc-controller-default-20251015002137476700000001 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: dynpoc-login-slurm-login-20251015002137480600000002 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: e2dbe6slur-compute-nodeset-20251007201508565800000001 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: e3879cslur-compute-nodeset-20250919212831665800000002 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: eaa3beslur-compute-nodeset-20251124070522321600000001 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: ebf828slur-compute-nodeset-20251124071134868500000001 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: ebf828slur-controller-default-20251124071144586200000003 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: ebf828slur-login-slurm-login-20251124071134910400000002 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f4944slurm-compute-nodeset-20250912131529204200000001 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f4e324slur-compute-nodeset-20250811165336641200000001 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f4e324slur-controller-default-20250811165346184100000003 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f4e324slur-login-slurm-login-20250811165336654300000002 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f88073slur-compute-nodeset-20250826031610614500000002 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f88073slur-controller-default-20250826031620053300000003 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: f88073slur-login-slurm-login-20250826031610570200000001 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: fa4slurmsi-compute-nodeset-20250808051559443200000002 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: fa4slurmsi-controller-default-20250808051608790400000003 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: fa4slurmsi-login-slurm-login-20250808051559433300000001 (Global) -[2025-11-30 19:28:03] [DRY-RUN] Would delete Instance Template: g4qclav-compute-g4nodeset-20251111194038852400000002 (Global) -[2025-11-30 19:28:03] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:28:03] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:28:17] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:28:17] [INFO] Time Cutoff (General): 2025-11-30T14:28:17+0000 -[2025-11-30 19:28:17] [INFO] Time Cutoff (Images): 2025-10-01T19:28:17+0000 -[2025-11-30 19:28:17] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:28:17] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:28:18] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:28:21] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:28:21] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:28:21] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:28:21] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:28:21] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:28:21] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:28:21] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:28:21] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:28:21] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:28:21] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:28:21] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: dynpoc-compute-h3nodeset-20251015002137498900000005 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: dynpoc-controller-default-20251015002137476700000001 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: dynpoc-login-slurm-login-20251015002137480600000002 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: e2dbe6slur-compute-nodeset-20251007201508565800000001 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: e3879cslur-compute-nodeset-20250919212831665800000002 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: eaa3beslur-compute-nodeset-20251124070522321600000001 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: ebf828slur-compute-nodeset-20251124071134868500000001 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: ebf828slur-controller-default-20251124071144586200000003 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: ebf828slur-login-slurm-login-20251124071134910400000002 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f4944slurm-compute-nodeset-20250912131529204200000001 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f4e324slur-compute-nodeset-20250811165336641200000001 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f4e324slur-controller-default-20250811165346184100000003 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f4e324slur-login-slurm-login-20250811165336654300000002 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f88073slur-compute-nodeset-20250826031610614500000002 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f88073slur-controller-default-20250826031620053300000003 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: f88073slur-login-slurm-login-20250826031610570200000001 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: fa4slurmsi-compute-nodeset-20250808051559443200000002 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: fa4slurmsi-controller-default-20250808051608790400000003 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: fa4slurmsi-login-slurm-login-20250808051559433300000001 (Global) -[2025-11-30 19:28:21] [DRY-RUN] Would delete Instance Template: g4qclav-compute-g4nodeset-20251111194038852400000002 (Global) -[2025-11-30 19:28:21] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:28:21] [INFO] CLEANUP RUN FINISHED -./cleanup.sh: line 712: n: command not found -[2025-11-30 19:28:47] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:28:47] [INFO] Time Cutoff (General): 2025-11-30T14:28:47+0000 -[2025-11-30 19:28:47] [INFO] Time Cutoff (Images): 2025-10-01T19:28:47+0000 -[2025-11-30 19:28:47] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:28:47] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:28:48] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:28:51] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:28:51] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:28:51] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:28:51] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:28:51] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:28:51] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:28:51] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:28:51] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:28:51] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:28:51] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:28:51] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:28:51] [EXECUTE] Deleting Instance Template: dynpoc-compute-h3nodeset-20251015002137498900000005 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/dynpoc-compute-h3nodeset-20251015002137498900000005]. -[2025-11-30 19:28:54] [SUCCESS] Deleted dynpoc-compute-h3nodeset-20251015002137498900000005 -[2025-11-30 19:28:54] [EXECUTE] Deleting Instance Template: dynpoc-controller-default-20251015002137476700000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/dynpoc-controller-default-20251015002137476700000001]. -[2025-11-30 19:28:57] [SUCCESS] Deleted dynpoc-controller-default-20251015002137476700000001 -[2025-11-30 19:28:57] [EXECUTE] Deleting Instance Template: dynpoc-login-slurm-login-20251015002137480600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/dynpoc-login-slurm-login-20251015002137480600000002]. -[2025-11-30 19:29:00] [SUCCESS] Deleted dynpoc-login-slurm-login-20251015002137480600000002 -[2025-11-30 19:29:00] [EXECUTE] Deleting Instance Template: e2dbe6slur-compute-nodeset-20251007201508565800000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/e2dbe6slur-compute-nodeset-20251007201508565800000001]. -[2025-11-30 19:29:03] [SUCCESS] Deleted e2dbe6slur-compute-nodeset-20251007201508565800000001 -[2025-11-30 19:29:03] [EXECUTE] Deleting Instance Template: e3879cslur-compute-nodeset-20250919212831665800000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/e3879cslur-compute-nodeset-20250919212831665800000002]. -[2025-11-30 19:29:07] [SUCCESS] Deleted e3879cslur-compute-nodeset-20250919212831665800000002 -[2025-11-30 19:29:07] [EXECUTE] Deleting Instance Template: eaa3beslur-compute-nodeset-20251124070522321600000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/eaa3beslur-compute-nodeset-20251124070522321600000001]. -[2025-11-30 19:29:10] [SUCCESS] Deleted eaa3beslur-compute-nodeset-20251124070522321600000001 -[2025-11-30 19:29:10] [EXECUTE] Deleting Instance Template: ebf828slur-compute-nodeset-20251124071134868500000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ebf828slur-compute-nodeset-20251124071134868500000001]. -[2025-11-30 19:29:13] [SUCCESS] Deleted ebf828slur-compute-nodeset-20251124071134868500000001 -[2025-11-30 19:29:13] [EXECUTE] Deleting Instance Template: ebf828slur-controller-default-20251124071144586200000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ebf828slur-controller-default-20251124071144586200000003]. -[2025-11-30 19:29:16] [SUCCESS] Deleted ebf828slur-controller-default-20251124071144586200000003 -[2025-11-30 19:29:16] [EXECUTE] Deleting Instance Template: ebf828slur-login-slurm-login-20251124071134910400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ebf828slur-login-slurm-login-20251124071134910400000002]. -[2025-11-30 19:29:19] [SUCCESS] Deleted ebf828slur-login-slurm-login-20251124071134910400000002 -[2025-11-30 19:29:19] [EXECUTE] Deleting Instance Template: f4944slurm-compute-nodeset-20250912131529204200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f4944slurm-compute-nodeset-20250912131529204200000001]. -[2025-11-30 19:29:22] [SUCCESS] Deleted f4944slurm-compute-nodeset-20250912131529204200000001 -[2025-11-30 19:29:22] [EXECUTE] Deleting Instance Template: f4e324slur-compute-nodeset-20250811165336641200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f4e324slur-compute-nodeset-20250811165336641200000001]. -[2025-11-30 19:29:25] [SUCCESS] Deleted f4e324slur-compute-nodeset-20250811165336641200000001 -[2025-11-30 19:29:25] [EXECUTE] Deleting Instance Template: f4e324slur-controller-default-20250811165346184100000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f4e324slur-controller-default-20250811165346184100000003]. -[2025-11-30 19:29:28] [SUCCESS] Deleted f4e324slur-controller-default-20250811165346184100000003 -[2025-11-30 19:29:28] [EXECUTE] Deleting Instance Template: f4e324slur-login-slurm-login-20250811165336654300000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f4e324slur-login-slurm-login-20250811165336654300000002]. -[2025-11-30 19:29:31] [SUCCESS] Deleted f4e324slur-login-slurm-login-20250811165336654300000002 -[2025-11-30 19:29:31] [EXECUTE] Deleting Instance Template: f88073slur-compute-nodeset-20250826031610614500000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f88073slur-compute-nodeset-20250826031610614500000002]. -[2025-11-30 19:29:34] [SUCCESS] Deleted f88073slur-compute-nodeset-20250826031610614500000002 -[2025-11-30 19:29:34] [EXECUTE] Deleting Instance Template: f88073slur-controller-default-20250826031620053300000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f88073slur-controller-default-20250826031620053300000003]. -[2025-11-30 19:29:37] [SUCCESS] Deleted f88073slur-controller-default-20250826031620053300000003 -[2025-11-30 19:29:37] [EXECUTE] Deleting Instance Template: f88073slur-login-slurm-login-20250826031610570200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/f88073slur-login-slurm-login-20250826031610570200000001]. -[2025-11-30 19:29:40] [SUCCESS] Deleted f88073slur-login-slurm-login-20250826031610570200000001 -[2025-11-30 19:29:40] [EXECUTE] Deleting Instance Template: fa4slurmsi-compute-nodeset-20250808051559443200000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/fa4slurmsi-compute-nodeset-20250808051559443200000002]. -[2025-11-30 19:29:43] [SUCCESS] Deleted fa4slurmsi-compute-nodeset-20250808051559443200000002 -[2025-11-30 19:29:43] [EXECUTE] Deleting Instance Template: fa4slurmsi-controller-default-20250808051608790400000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/fa4slurmsi-controller-default-20250808051608790400000003]. -[2025-11-30 19:29:46] [SUCCESS] Deleted fa4slurmsi-controller-default-20250808051608790400000003 -[2025-11-30 19:29:46] [EXECUTE] Deleting Instance Template: fa4slurmsi-login-slurm-login-20250808051559433300000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/fa4slurmsi-login-slurm-login-20250808051559433300000001]. -[2025-11-30 19:29:49] [SUCCESS] Deleted fa4slurmsi-login-slurm-login-20250808051559433300000001 -[2025-11-30 19:29:49] [EXECUTE] Deleting Instance Template: g4qclav-compute-g4nodeset-20251111194038852400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4qclav-compute-g4nodeset-20251111194038852400000002]. -[2025-11-30 19:29:52] [SUCCESS] Deleted g4qclav-compute-g4nodeset-20251111194038852400000002 -[2025-11-30 19:29:52] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:29:52] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:30:24] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:30:24] [INFO] Time Cutoff (General): 2025-11-30T14:30:23+0000 -[2025-11-30 19:30:24] [INFO] Time Cutoff (Images): 2025-10-01T19:30:24+0000 -[2025-11-30 19:30:24] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:30:24] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:30:24] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:30:27] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:30:27] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:30:27] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:30:27] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:30:27] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:30:27] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:30:27] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:30:27] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:30:27] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:30:27] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:30:27] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4qclav-controller-default-20251111194048650800000003 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4qclav-login-slurm-login-20251111194038848700000001 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4test-compute-g4nodeset-20250922145102127500000001 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4test-controller-default-20250922145111921600000003 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4test-login-slurm-login-20250922145102128200000002 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4testnewe-compute-g4nodeset-20251023163712389000000001 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4testnewe-controller-default-20251023163722101400000003 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: g4testnewe-login-slurm-login-20251023163712400900000002 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: h4dqc-compute-h4dnodeset-20250526185335423500000003 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: h4dqc-controller-default-20250526185335395700000001 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: h4dqc-login-slurm-login-20250526185335402700000002 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: h4dsaara-compute-h4dnodeset-20250903085805905100000002 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hclsv60f32-compute-gpu-20250801171014779800000001 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hclsv60f32-compute-ns-20250801171014806600000002 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hclsv60f32-controller-default-20250801171022463800000004 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hclsv60f32-login-slurm-login-20250801171014845800000003 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hpc01-compute-a216nodeset-20251125104737582400000001 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hpc01-compute-a28nodeset-20251125104737596100000002 (Global) -[2025-11-30 19:30:27] [DRY-RUN] Would delete Instance Template: hpc01-compute-c2dnodeset-20251125104737610300000004 (Global) -[2025-11-30 19:30:27] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:30:27] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:30:54] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:30:54] [INFO] Time Cutoff (General): 2025-11-30T14:30:54+0000 -[2025-11-30 19:30:54] [INFO] Time Cutoff (Images): 2025-10-01T19:30:54+0000 -[2025-11-30 19:30:54] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:30:54] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:30:54] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:30:57] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:30:57] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:30:57] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:30:58] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:30:58] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:30:58] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:30:58] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:30:58] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:30:58] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:30:58] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:30:58] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:30:58] [EXECUTE] Deleting Instance Template: g4qclav-controller-default-20251111194048650800000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4qclav-controller-default-20251111194048650800000003]. -[2025-11-30 19:31:01] [SUCCESS] Deleted g4qclav-controller-default-20251111194048650800000003 -[2025-11-30 19:31:01] [EXECUTE] Deleting Instance Template: g4qclav-login-slurm-login-20251111194038848700000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4qclav-login-slurm-login-20251111194038848700000001]. -[2025-11-30 19:31:04] [SUCCESS] Deleted g4qclav-login-slurm-login-20251111194038848700000001 -[2025-11-30 19:31:04] [EXECUTE] Deleting Instance Template: g4test-compute-g4nodeset-20250922145102127500000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4test-compute-g4nodeset-20250922145102127500000001]. -[2025-11-30 19:31:07] [SUCCESS] Deleted g4test-compute-g4nodeset-20250922145102127500000001 -[2025-11-30 19:31:07] [EXECUTE] Deleting Instance Template: g4test-controller-default-20250922145111921600000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4test-controller-default-20250922145111921600000003]. -[2025-11-30 19:31:10] [SUCCESS] Deleted g4test-controller-default-20250922145111921600000003 -[2025-11-30 19:31:10] [EXECUTE] Deleting Instance Template: g4test-login-slurm-login-20250922145102128200000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4test-login-slurm-login-20250922145102128200000002]. -[2025-11-30 19:31:13] [SUCCESS] Deleted g4test-login-slurm-login-20250922145102128200000002 -[2025-11-30 19:31:13] [EXECUTE] Deleting Instance Template: g4testnewe-compute-g4nodeset-20251023163712389000000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4testnewe-compute-g4nodeset-20251023163712389000000001]. -[2025-11-30 19:31:16] [SUCCESS] Deleted g4testnewe-compute-g4nodeset-20251023163712389000000001 -[2025-11-30 19:31:16] [EXECUTE] Deleting Instance Template: g4testnewe-controller-default-20251023163722101400000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4testnewe-controller-default-20251023163722101400000003]. -[2025-11-30 19:31:19] [SUCCESS] Deleted g4testnewe-controller-default-20251023163722101400000003 -[2025-11-30 19:31:19] [EXECUTE] Deleting Instance Template: g4testnewe-login-slurm-login-20251023163712400900000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/g4testnewe-login-slurm-login-20251023163712400900000002]. -[2025-11-30 19:31:22] [SUCCESS] Deleted g4testnewe-login-slurm-login-20251023163712400900000002 -[2025-11-30 19:31:22] [EXECUTE] Deleting Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) -ERROR: (gcloud.compute.instance-templates.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/global/instanceTemplates/gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492' was not found - -[2025-11-30 19:31:24] [ERROR] Failed to delete gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 -[2025-11-30 19:31:24] [EXECUTE] Deleting Instance Template: h4dqc-compute-h4dnodeset-20250526185335423500000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/h4dqc-compute-h4dnodeset-20250526185335423500000003]. -[2025-11-30 19:31:28] [SUCCESS] Deleted h4dqc-compute-h4dnodeset-20250526185335423500000003 -[2025-11-30 19:31:28] [EXECUTE] Deleting Instance Template: h4dqc-controller-default-20250526185335395700000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/h4dqc-controller-default-20250526185335395700000001]. -[2025-11-30 19:31:31] [SUCCESS] Deleted h4dqc-controller-default-20250526185335395700000001 -[2025-11-30 19:31:31] [EXECUTE] Deleting Instance Template: h4dqc-login-slurm-login-20250526185335402700000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/h4dqc-login-slurm-login-20250526185335402700000002]. -[2025-11-30 19:31:34] [SUCCESS] Deleted h4dqc-login-slurm-login-20250526185335402700000002 -[2025-11-30 19:31:34] [EXECUTE] Deleting Instance Template: h4dsaara-compute-h4dnodeset-20250903085805905100000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/h4dsaara-compute-h4dnodeset-20250903085805905100000002]. -[2025-11-30 19:31:37] [SUCCESS] Deleted h4dsaara-compute-h4dnodeset-20250903085805905100000002 -[2025-11-30 19:31:37] [EXECUTE] Deleting Instance Template: hclsv60f32-compute-gpu-20250801171014779800000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hclsv60f32-compute-gpu-20250801171014779800000001]. -[2025-11-30 19:31:40] [SUCCESS] Deleted hclsv60f32-compute-gpu-20250801171014779800000001 -[2025-11-30 19:31:40] [EXECUTE] Deleting Instance Template: hclsv60f32-compute-ns-20250801171014806600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hclsv60f32-compute-ns-20250801171014806600000002]. -[2025-11-30 19:31:43] [SUCCESS] Deleted hclsv60f32-compute-ns-20250801171014806600000002 -[2025-11-30 19:31:43] [EXECUTE] Deleting Instance Template: hclsv60f32-controller-default-20250801171022463800000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hclsv60f32-controller-default-20250801171022463800000004]. -[2025-11-30 19:31:46] [SUCCESS] Deleted hclsv60f32-controller-default-20250801171022463800000004 -[2025-11-30 19:31:46] [EXECUTE] Deleting Instance Template: hclsv60f32-login-slurm-login-20250801171014845800000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hclsv60f32-login-slurm-login-20250801171014845800000003]. -[2025-11-30 19:31:49] [SUCCESS] Deleted hclsv60f32-login-slurm-login-20250801171014845800000003 -[2025-11-30 19:31:49] [EXECUTE] Deleting Instance Template: hpc01-compute-a216nodeset-20251125104737582400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-a216nodeset-20251125104737582400000001]. -[2025-11-30 19:31:52] [SUCCESS] Deleted hpc01-compute-a216nodeset-20251125104737582400000001 -[2025-11-30 19:31:52] [EXECUTE] Deleting Instance Template: hpc01-compute-a28nodeset-20251125104737596100000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-a28nodeset-20251125104737596100000002]. -[2025-11-30 19:31:55] [SUCCESS] Deleted hpc01-compute-a28nodeset-20251125104737596100000002 -[2025-11-30 19:31:55] [EXECUTE] Deleting Instance Template: hpc01-compute-c2dnodeset-20251125104737610300000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-c2dnodeset-20251125104737610300000004]. -[2025-11-30 19:31:58] [SUCCESS] Deleted hpc01-compute-c2dnodeset-20251125104737610300000004 -[2025-11-30 19:31:58] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:31:58] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:32:24] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:32:24] [INFO] Time Cutoff (General): 2025-11-30T14:32:24+0000 -[2025-11-30 19:32:24] [INFO] Time Cutoff (Images): 2025-10-01T19:32:24+0000 -[2025-11-30 19:32:24] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:32:24] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:32:25] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:32:28] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:32:28] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:32:28] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:32:28] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:32:28] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:32:28] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:32:28] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:32:28] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:32:28] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:32:28] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:32:28] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpc01-compute-c2nodeset-20251125104737599200000003 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpc01-compute-c3nodeset-20251125104737611200000005 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpc01-compute-h3nodeset-20251125104737617100000006 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpc01-compute-n2nodeset-20251125104739693200000007 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpc01-login-slurm-login-20251125104742597200000008 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-a216nodeset-20251123152333927800000004 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-a28nodeset-20251123152344956400000009 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c2dnodeset-20251123152343768500000008 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c2nodeset-20251123152333914100000003 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c3nodeset-20251123152334135900000006 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-h3nodeset-20251123152343701700000007 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-compute-n2nodeset-20251123152334134700000005 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-controller-default-20251123152321945800000001 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcimg-login-slurm-login-20251123152333155000000002 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-computenodeset-20251120073345516900000002 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-debugnodeset-20251120073345535400000005 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-h3nodeset-20251120073345521400000003 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcslurm-controller-default-20251120073345359600000001 (Global) -[2025-11-30 19:32:28] [DRY-RUN] Would delete Instance Template: hpcslurm-login-slurm-login-20251120073345530000000004 (Global) -[2025-11-30 19:32:28] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:32:28] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:34:27] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:34:27] [INFO] Time Cutoff (General): 2025-11-30T14:34:27+0000 -[2025-11-30 19:34:27] [INFO] Time Cutoff (Images): 2025-10-01T19:34:27+0000 -[2025-11-30 19:34:27] [INFO] Delete Limit per Type: 20 -[2025-11-30 19:34:27] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:34:28] [INFO] --- Processing: Instance Templates (Limit: 20) --- -[2025-11-30 19:34:31] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) -[2025-11-30 19:34:31] [SKIP] hpc01-compute-c2nodeset-20251125104737599200000003 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpc01-compute-c3nodeset-20251125104737611200000005 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpc01-compute-h3nodeset-20251125104737617100000006 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpc01-compute-n2nodeset-20251125104739693200000007 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpc01-login-slurm-login-20251125104742597200000008 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpcimg-compute-a216nodeset-20251123152333927800000004 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpcimg-compute-a28nodeset-20251123152344956400000009 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpcimg-compute-c2dnodeset-20251123152343768500000008 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpcimg-compute-c2nodeset-20251123152333914100000003 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpcimg-compute-c3nodeset-20251123152334135900000006 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpcimg-compute-h3nodeset-20251123152343701700000007 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpcimg-compute-n2nodeset-20251123152334134700000005 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpcimg-controller-default-20251123152321945800000001 (In Exclusion List) -[2025-11-30 19:34:31] [SKIP] hpcimg-login-slurm-login-20251123152333155000000002 (In Exclusion List) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-computenodeset-20251120073345516900000002 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-debugnodeset-20251120073345535400000005 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-h3nodeset-20251120073345521400000003 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: hpcslurm-controller-default-20251120073345359600000001 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: hpcslurm-login-slurm-login-20251120073345530000000004 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: laveeek29-compute-a3ultranodeset-20251120185220355800000002 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: laveeek29-controller-default-20251120185222245000000003 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: laveeek29-login-slurm-login-20251120185217925800000001 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustre06-compute-lustrenodeset-20251126041817737800000001 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustre06-controller-default-20251126041828459800000003 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustre06-login-slurm-login-20251126041817754000000002 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a216nodeset-20251127075747115200000008 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a28nodeset-20251127075746973800000005 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2dnodeset-20251127075746534500000004 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2nodeset-20251127075746393800000003 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c3nodeset-20251127075747101600000006 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-h3nodeset-20251127075747131700000009 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-compute-n2nodeset-20251127075747112000000007 (Global) -[2025-11-30 19:34:31] [DRY-RUN] Would delete Instance Template: lustredev0-controller-default-20251127075745707800000001 (Global) -[2025-11-30 19:34:31] [INFO] Hit delete limit (20) for Instance Templates. -[2025-11-30 19:34:31] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:34:53] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:34:53] [INFO] Time Cutoff (General): 2025-11-30T14:34:53+0000 -[2025-11-30 19:34:53] [INFO] Time Cutoff (Images): 2025-10-01T19:34:53+0000 -[2025-11-30 19:34:53] [INFO] Delete Limit per Type: 200 -[2025-11-30 19:34:53] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:34:54] [INFO] --- Processing: Instance Templates (Limit: 200) --- -[2025-11-30 19:34:57] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) -[2025-11-30 19:34:57] [SKIP] hpc01-compute-c2nodeset-20251125104737599200000003 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpc01-compute-c3nodeset-20251125104737611200000005 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpc01-compute-h3nodeset-20251125104737617100000006 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpc01-compute-n2nodeset-20251125104739693200000007 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpc01-login-slurm-login-20251125104742597200000008 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpcimg-compute-a216nodeset-20251123152333927800000004 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpcimg-compute-a28nodeset-20251123152344956400000009 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpcimg-compute-c2dnodeset-20251123152343768500000008 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpcimg-compute-c2nodeset-20251123152333914100000003 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpcimg-compute-c3nodeset-20251123152334135900000006 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpcimg-compute-h3nodeset-20251123152343701700000007 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpcimg-compute-n2nodeset-20251123152334134700000005 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpcimg-controller-default-20251123152321945800000001 (In Exclusion List) -[2025-11-30 19:34:57] [SKIP] hpcimg-login-slurm-login-20251123152333155000000002 (In Exclusion List) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-computenodeset-20251120073345516900000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-debugnodeset-20251120073345535400000005 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-h3nodeset-20251120073345521400000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: hpcslurm-controller-default-20251120073345359600000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: hpcslurm-login-slurm-login-20251120073345530000000004 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: laveeek29-compute-a3ultranodeset-20251120185220355800000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: laveeek29-controller-default-20251120185222245000000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: laveeek29-login-slurm-login-20251120185217925800000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustre06-compute-lustrenodeset-20251126041817737800000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustre06-controller-default-20251126041828459800000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustre06-login-slurm-login-20251126041817754000000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a216nodeset-20251127075747115200000008 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a28nodeset-20251127075746973800000005 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2dnodeset-20251127075746534500000004 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2nodeset-20251127075746393800000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c3nodeset-20251127075747101600000006 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-h3nodeset-20251127075747131700000009 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-compute-n2nodeset-20251127075747112000000007 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-controller-default-20251127075745707800000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustredev0-login-slurm-login-20251127075746027600000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-a216nodeset-20251127085007177000000008 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-a28nodeset-20251127085007152400000005 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c2dnodeset-20251127085007154700000006 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c2nodeset-20251127085007178300000009 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c3nodeset-20251127085007120300000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-h3nodeset-20251127085007156500000007 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-compute-n2nodeset-20251127085007147000000004 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-controller-default-20251127085005432900000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreprod-login-slurm-login-20251127085005549500000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-a216nodeset-20251127064214383600000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-a28nodeset-20251127064214402000000004 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c2dnodeset-20251127064214467900000009 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c2nodeset-20251127064214424900000005 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c3nodeset-20251127064214452600000007 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-h3nodeset-20251127064214457600000008 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-n2nodeset-20251127064214447800000006 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-controller-default-20251127064213231900000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustreqa05-login-slurm-login-20251127064213259800000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-a216nodeset-20251126044326259100000008 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-a28nodeset-20251126044326243000000005 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-c2dnodeset-20251126044326150300000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-c2nodeset-20251126044326244400000006 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-c3nodeset-20251126044326214900000004 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-h3nodeset-20251126044326264800000009 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-compute-n2nodeset-20251126044326256800000007 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-controller-default-20251126044324852600000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: lustretest-login-slurm-login-20251126044324988400000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: mainek-compute-a3ultranodeset-20251121092031664700000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: mainek-controller-default-20251121092028913400000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: mainek-login-slurm-login-20251121092028928700000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: pkbh4dp2-compute-computenodeset-20250412005603828600000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: pkrv6db5de-login-slurm-login-20250801090852525400000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: pzpk-compute-computenodeset-20250411004353418500000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: pzpk-controller-default-20250411004353405200000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesh4d-compute-h4dnodeset-20251016152751862600000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesh4d-controller-default-20251016152751828700000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesh4d-login-slurm-login-20251016152751847100000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesth4d-compute-h4dnodeset-20251013062737759200000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesth4d-controller-default-20251013062737731900000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: ractesth4d-login-slurm-login-20251013062737750800000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-computenodeset-20250912110554713000000004 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-debugnodeset-20250912110554709900000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-h3nodeset-20250912110554735800000005 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: rock8a070a-controller-default-20250912110554351400000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: rock8a070a-login-slurm-login-20250912110554546400000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: simtestnet-compute-a3ultranodeset-20251128190204679000000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: simtestnet-controller-default-20251128190210353600000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: simtestnet-login-slurm-login-20251128190201729100000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm0-compute-a3nodeset-20251009153324036900000004 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm0-compute-debugnodeset-20251009153324023000000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm0-controller-default-20251009153304999900000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm0-login-login-20251009153305041200000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm9ff-compute-debugnodeset-20250925215605513600000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm9ff-controller-default-20250925215558268500000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurm9ff-login-slurm-login-20250925215558759400000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmf1c-compute-debugnodeset-20250925064111290200000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmf1c-controller-default-20250925064111186200000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmf1c-login-slurm-login-20250925064111290800000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20250806200217240000000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20250912233249589700000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20251111083658962700000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmsimpl-compute-nodeset-20250708064936486300000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmsimpl-compute-nodeset-20250814191439012800000001 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmsimpl-controller-default-20250708064945457400000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmsimpl-controller-default-20250814191448978300000003 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: slurmsimpl-login-slurm-login-20250814191439021200000002 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: testing2 (Global) -[2025-11-30 19:34:57] [DRY-RUN] Would delete Instance Template: welp-insta-temp (Global) -[2025-11-30 19:34:57] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:36:17] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:36:17] [INFO] Time Cutoff (General): 2025-11-30T14:36:17+0000 -[2025-11-30 19:36:17] [INFO] Time Cutoff (Images): 2025-10-01T19:36:17+0000 -[2025-11-30 19:36:17] [INFO] Delete Limit per Type: 200 -[2025-11-30 19:36:17] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:36:18] [INFO] --- Processing: Instance Templates (Limit: 200) --- -[2025-11-30 19:36:21] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:36:21] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:36:21] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:36:21] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:36:21] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:36:21] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:36:21] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:36:21] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:36:21] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:36:21] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:36:21] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpc01-compute-c2nodeset-20251125104737599200000003 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpc01-compute-c3nodeset-20251125104737611200000005 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpc01-compute-h3nodeset-20251125104737617100000006 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpc01-compute-n2nodeset-20251125104739693200000007 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpc01-login-slurm-login-20251125104742597200000008 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-a216nodeset-20251123152333927800000004 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-a28nodeset-20251123152344956400000009 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c2dnodeset-20251123152343768500000008 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c2nodeset-20251123152333914100000003 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-c3nodeset-20251123152334135900000006 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-h3nodeset-20251123152343701700000007 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-compute-n2nodeset-20251123152334134700000005 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-controller-default-20251123152321945800000001 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcimg-login-slurm-login-20251123152333155000000002 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-computenodeset-20251120073345516900000002 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-debugnodeset-20251120073345535400000005 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcslurm-compute-h3nodeset-20251120073345521400000003 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcslurm-controller-default-20251120073345359600000001 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: hpcslurm-login-slurm-login-20251120073345530000000004 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: laveeek29-compute-a3ultranodeset-20251120185220355800000002 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: laveeek29-controller-default-20251120185222245000000003 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: laveeek29-login-slurm-login-20251120185217925800000001 (Global) -[2025-11-30 19:36:21] [DRY-RUN] Would delete Instance Template: lustre06-compute-lustrenodeset-20251126041817737800000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustre06-controller-default-20251126041828459800000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustre06-login-slurm-login-20251126041817754000000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a216nodeset-20251127075747115200000008 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-a28nodeset-20251127075746973800000005 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2dnodeset-20251127075746534500000004 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c2nodeset-20251127075746393800000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-c3nodeset-20251127075747101600000006 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-h3nodeset-20251127075747131700000009 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-compute-n2nodeset-20251127075747112000000007 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-controller-default-20251127075745707800000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustredev0-login-slurm-login-20251127075746027600000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-a216nodeset-20251127085007177000000008 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-a28nodeset-20251127085007152400000005 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c2dnodeset-20251127085007154700000006 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c2nodeset-20251127085007178300000009 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-c3nodeset-20251127085007120300000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-h3nodeset-20251127085007156500000007 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-compute-n2nodeset-20251127085007147000000004 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-controller-default-20251127085005432900000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreprod-login-slurm-login-20251127085005549500000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-a216nodeset-20251127064214383600000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-a28nodeset-20251127064214402000000004 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c2dnodeset-20251127064214467900000009 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c2nodeset-20251127064214424900000005 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-c3nodeset-20251127064214452600000007 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-h3nodeset-20251127064214457600000008 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-compute-n2nodeset-20251127064214447800000006 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-controller-default-20251127064213231900000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustreqa05-login-slurm-login-20251127064213259800000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-a216nodeset-20251126044326259100000008 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-a28nodeset-20251126044326243000000005 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-c2dnodeset-20251126044326150300000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-c2nodeset-20251126044326244400000006 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-c3nodeset-20251126044326214900000004 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-h3nodeset-20251126044326264800000009 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-compute-n2nodeset-20251126044326256800000007 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-controller-default-20251126044324852600000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: lustretest-login-slurm-login-20251126044324988400000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: mainek-compute-a3ultranodeset-20251121092031664700000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: mainek-controller-default-20251121092028913400000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: mainek-login-slurm-login-20251121092028928700000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: pkbh4dp2-compute-computenodeset-20250412005603828600000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: pkrv6db5de-login-slurm-login-20250801090852525400000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: pzpk-compute-computenodeset-20250411004353418500000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: pzpk-controller-default-20250411004353405200000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesh4d-compute-h4dnodeset-20251016152751862600000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesh4d-controller-default-20251016152751828700000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesh4d-login-slurm-login-20251016152751847100000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesth4d-compute-h4dnodeset-20251013062737759200000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesth4d-controller-default-20251013062737731900000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: ractesth4d-login-slurm-login-20251013062737750800000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-computenodeset-20250912110554713000000004 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-debugnodeset-20250912110554709900000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: rock8a070a-compute-h3nodeset-20250912110554735800000005 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: rock8a070a-controller-default-20250912110554351400000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: rock8a070a-login-slurm-login-20250912110554546400000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: simtestnet-compute-a3ultranodeset-20251128190204679000000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: simtestnet-controller-default-20251128190210353600000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: simtestnet-login-slurm-login-20251128190201729100000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm0-compute-a3nodeset-20251009153324036900000004 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm0-compute-debugnodeset-20251009153324023000000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm0-controller-default-20251009153304999900000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm0-login-login-20251009153305041200000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm9ff-compute-debugnodeset-20250925215605513600000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm9ff-controller-default-20250925215558268500000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurm9ff-login-slurm-login-20250925215558759400000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmf1c-compute-debugnodeset-20250925064111290200000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmf1c-controller-default-20250925064111186200000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmf1c-login-slurm-login-20250925064111290800000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20250806200217240000000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20250912233249589700000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmflex-compute-nodeset-20251111083658962700000002 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmsimpl-compute-nodeset-20250708064936486300000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmsimpl-compute-nodeset-20250814191439012800000001 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmsimpl-controller-default-20250708064945457400000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmsimpl-controller-default-20250814191448978300000003 (Global) -[2025-11-30 19:36:22] [DRY-RUN] Would delete Instance Template: slurmsimpl-login-slurm-login-20250814191439021200000002 (Global) -[2025-11-30 19:36:22] [SKIP] testing2 (In Exclusion List) -[2025-11-30 19:36:22] [SKIP] welp-insta-temp (In Exclusion List) -[2025-11-30 19:36:22] [INFO] CLEANUP RUN FINISHED -[2025-11-30 19:36:53] [INFO] STARTING RESOURCE CLEANUP: hpc-toolkit-dev -[2025-11-30 19:36:53] [INFO] Time Cutoff (General): 2025-11-30T14:36:53+0000 -[2025-11-30 19:36:53] [INFO] Time Cutoff (Images): 2025-10-01T19:36:53+0000 -[2025-11-30 19:36:53] [INFO] Delete Limit per Type: 200 -[2025-11-30 19:36:53] [INFO] Loading exclusions from exclusions.txt... -[2025-11-30 19:36:54] [INFO] --- Processing: Instance Templates (Limit: 200) --- -[2025-11-30 19:36:57] [SKIP] a3mega-compute-a3meganodeset-20251118080924120000000004 (In Exclusion List) -[2025-11-30 19:36:57] [SKIP] a3mega-compute-debugnodeset-20251118080924115100000003 (In Exclusion List) -[2025-11-30 19:36:57] [SKIP] a3mega-controller-default-20251118080901356800000001 (In Exclusion List) -[2025-11-30 19:36:57] [SKIP] a3mega-login-login-20251118080901434600000002 (In Exclusion List) -[2025-11-30 19:36:57] [SKIP] a3slurmsy-compute-a3nodeset-20251016115123978200000002 (In Exclusion List) -[2025-11-30 19:36:57] [SKIP] a3slurmsy-controller-default-20251016115129563200000003 (In Exclusion List) -[2025-11-30 19:36:57] [SKIP] a3slurmsy-login-slurm-login-20251016115120300800000001 (In Exclusion List) -[2025-11-30 19:36:57] [SKIP] batch-job-instance-template-20250901212237920900000001 (In Exclusion List) -[2025-11-30 19:36:57] [SKIP] batch-job-instance-template-20250912070019961500000001 (In Exclusion List) -[2025-11-30 19:36:57] [SKIP] buildslurm-compute-debugnodeset-20251030080636567600000001 (In Exclusion List) -[2025-11-30 19:36:57] [SKIP] buildslurm-controller-default-20251030080646114800000002 (In Exclusion List) -[2025-11-30 19:36:57] [EXECUTE] Deleting Instance Template: gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 (Global) -ERROR: (gcloud.compute.instance-templates.delete) Could not fetch resource: - - The resource 'projects/hpc-toolkit-dev/global/instanceTemplates/gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492' was not found - -[2025-11-30 19:36:59] [ERROR] Failed to delete gke-gke-a3ultra-7c7a-a3-ultragpu-8g-a-b8871492 -[2025-11-30 19:36:59] [EXECUTE] Deleting Instance Template: hpc01-compute-c2nodeset-20251125104737599200000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-c2nodeset-20251125104737599200000003]. -[2025-11-30 19:37:02] [SUCCESS] Deleted hpc01-compute-c2nodeset-20251125104737599200000003 -[2025-11-30 19:37:02] [EXECUTE] Deleting Instance Template: hpc01-compute-c3nodeset-20251125104737611200000005 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-c3nodeset-20251125104737611200000005]. -[2025-11-30 19:37:05] [SUCCESS] Deleted hpc01-compute-c3nodeset-20251125104737611200000005 -[2025-11-30 19:37:05] [EXECUTE] Deleting Instance Template: hpc01-compute-h3nodeset-20251125104737617100000006 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-h3nodeset-20251125104737617100000006]. -[2025-11-30 19:37:08] [SUCCESS] Deleted hpc01-compute-h3nodeset-20251125104737617100000006 -[2025-11-30 19:37:08] [EXECUTE] Deleting Instance Template: hpc01-compute-n2nodeset-20251125104739693200000007 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-compute-n2nodeset-20251125104739693200000007]. -[2025-11-30 19:37:12] [SUCCESS] Deleted hpc01-compute-n2nodeset-20251125104739693200000007 -[2025-11-30 19:37:12] [EXECUTE] Deleting Instance Template: hpc01-login-slurm-login-20251125104742597200000008 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpc01-login-slurm-login-20251125104742597200000008]. -[2025-11-30 19:37:15] [SUCCESS] Deleted hpc01-login-slurm-login-20251125104742597200000008 -[2025-11-30 19:37:15] [EXECUTE] Deleting Instance Template: hpcimg-compute-a216nodeset-20251123152333927800000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-a216nodeset-20251123152333927800000004]. -[2025-11-30 19:37:18] [SUCCESS] Deleted hpcimg-compute-a216nodeset-20251123152333927800000004 -[2025-11-30 19:37:18] [EXECUTE] Deleting Instance Template: hpcimg-compute-a28nodeset-20251123152344956400000009 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-a28nodeset-20251123152344956400000009]. -[2025-11-30 19:37:21] [SUCCESS] Deleted hpcimg-compute-a28nodeset-20251123152344956400000009 -[2025-11-30 19:37:21] [EXECUTE] Deleting Instance Template: hpcimg-compute-c2dnodeset-20251123152343768500000008 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-c2dnodeset-20251123152343768500000008]. -[2025-11-30 19:37:24] [SUCCESS] Deleted hpcimg-compute-c2dnodeset-20251123152343768500000008 -[2025-11-30 19:37:24] [EXECUTE] Deleting Instance Template: hpcimg-compute-c2nodeset-20251123152333914100000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-c2nodeset-20251123152333914100000003]. -[2025-11-30 19:37:27] [SUCCESS] Deleted hpcimg-compute-c2nodeset-20251123152333914100000003 -[2025-11-30 19:37:27] [EXECUTE] Deleting Instance Template: hpcimg-compute-c3nodeset-20251123152334135900000006 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-c3nodeset-20251123152334135900000006]. -[2025-11-30 19:37:30] [SUCCESS] Deleted hpcimg-compute-c3nodeset-20251123152334135900000006 -[2025-11-30 19:37:30] [EXECUTE] Deleting Instance Template: hpcimg-compute-h3nodeset-20251123152343701700000007 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-h3nodeset-20251123152343701700000007]. -[2025-11-30 19:37:33] [SUCCESS] Deleted hpcimg-compute-h3nodeset-20251123152343701700000007 -[2025-11-30 19:37:34] [EXECUTE] Deleting Instance Template: hpcimg-compute-n2nodeset-20251123152334134700000005 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-compute-n2nodeset-20251123152334134700000005]. -[2025-11-30 19:37:37] [SUCCESS] Deleted hpcimg-compute-n2nodeset-20251123152334134700000005 -[2025-11-30 19:37:37] [EXECUTE] Deleting Instance Template: hpcimg-controller-default-20251123152321945800000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-controller-default-20251123152321945800000001]. -[2025-11-30 19:37:40] [SUCCESS] Deleted hpcimg-controller-default-20251123152321945800000001 -[2025-11-30 19:37:40] [EXECUTE] Deleting Instance Template: hpcimg-login-slurm-login-20251123152333155000000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcimg-login-slurm-login-20251123152333155000000002]. -[2025-11-30 19:37:43] [SUCCESS] Deleted hpcimg-login-slurm-login-20251123152333155000000002 -[2025-11-30 19:37:43] [EXECUTE] Deleting Instance Template: hpcslurm-compute-computenodeset-20251120073345516900000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcslurm-compute-computenodeset-20251120073345516900000002]. -[2025-11-30 19:37:46] [SUCCESS] Deleted hpcslurm-compute-computenodeset-20251120073345516900000002 -[2025-11-30 19:37:46] [EXECUTE] Deleting Instance Template: hpcslurm-compute-debugnodeset-20251120073345535400000005 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcslurm-compute-debugnodeset-20251120073345535400000005]. -[2025-11-30 19:37:49] [SUCCESS] Deleted hpcslurm-compute-debugnodeset-20251120073345535400000005 -[2025-11-30 19:37:49] [EXECUTE] Deleting Instance Template: hpcslurm-compute-h3nodeset-20251120073345521400000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcslurm-compute-h3nodeset-20251120073345521400000003]. -[2025-11-30 19:37:52] [SUCCESS] Deleted hpcslurm-compute-h3nodeset-20251120073345521400000003 -[2025-11-30 19:37:52] [EXECUTE] Deleting Instance Template: hpcslurm-controller-default-20251120073345359600000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcslurm-controller-default-20251120073345359600000001]. -[2025-11-30 19:37:55] [SUCCESS] Deleted hpcslurm-controller-default-20251120073345359600000001 -[2025-11-30 19:37:55] [EXECUTE] Deleting Instance Template: hpcslurm-login-slurm-login-20251120073345530000000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/hpcslurm-login-slurm-login-20251120073345530000000004]. -[2025-11-30 19:37:59] [SUCCESS] Deleted hpcslurm-login-slurm-login-20251120073345530000000004 -[2025-11-30 19:37:59] [EXECUTE] Deleting Instance Template: laveeek29-compute-a3ultranodeset-20251120185220355800000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/laveeek29-compute-a3ultranodeset-20251120185220355800000002]. -[2025-11-30 19:38:02] [SUCCESS] Deleted laveeek29-compute-a3ultranodeset-20251120185220355800000002 -[2025-11-30 19:38:02] [EXECUTE] Deleting Instance Template: laveeek29-controller-default-20251120185222245000000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/laveeek29-controller-default-20251120185222245000000003]. -[2025-11-30 19:38:05] [SUCCESS] Deleted laveeek29-controller-default-20251120185222245000000003 -[2025-11-30 19:38:05] [EXECUTE] Deleting Instance Template: laveeek29-login-slurm-login-20251120185217925800000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/laveeek29-login-slurm-login-20251120185217925800000001]. -[2025-11-30 19:38:08] [SUCCESS] Deleted laveeek29-login-slurm-login-20251120185217925800000001 -[2025-11-30 19:38:08] [EXECUTE] Deleting Instance Template: lustre06-compute-lustrenodeset-20251126041817737800000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustre06-compute-lustrenodeset-20251126041817737800000001]. -[2025-11-30 19:38:11] [SUCCESS] Deleted lustre06-compute-lustrenodeset-20251126041817737800000001 -[2025-11-30 19:38:11] [EXECUTE] Deleting Instance Template: lustre06-controller-default-20251126041828459800000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustre06-controller-default-20251126041828459800000003]. -[2025-11-30 19:38:14] [SUCCESS] Deleted lustre06-controller-default-20251126041828459800000003 -[2025-11-30 19:38:14] [EXECUTE] Deleting Instance Template: lustre06-login-slurm-login-20251126041817754000000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustre06-login-slurm-login-20251126041817754000000002]. -[2025-11-30 19:38:17] [SUCCESS] Deleted lustre06-login-slurm-login-20251126041817754000000002 -[2025-11-30 19:38:17] [EXECUTE] Deleting Instance Template: lustredev0-compute-a216nodeset-20251127075747115200000008 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-a216nodeset-20251127075747115200000008]. -[2025-11-30 19:38:20] [SUCCESS] Deleted lustredev0-compute-a216nodeset-20251127075747115200000008 -[2025-11-30 19:38:20] [EXECUTE] Deleting Instance Template: lustredev0-compute-a28nodeset-20251127075746973800000005 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-a28nodeset-20251127075746973800000005]. -[2025-11-30 19:38:23] [SUCCESS] Deleted lustredev0-compute-a28nodeset-20251127075746973800000005 -[2025-11-30 19:38:23] [EXECUTE] Deleting Instance Template: lustredev0-compute-c2dnodeset-20251127075746534500000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-c2dnodeset-20251127075746534500000004]. -[2025-11-30 19:38:26] [SUCCESS] Deleted lustredev0-compute-c2dnodeset-20251127075746534500000004 -[2025-11-30 19:38:26] [EXECUTE] Deleting Instance Template: lustredev0-compute-c2nodeset-20251127075746393800000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-c2nodeset-20251127075746393800000003]. -[2025-11-30 19:38:29] [SUCCESS] Deleted lustredev0-compute-c2nodeset-20251127075746393800000003 -[2025-11-30 19:38:29] [EXECUTE] Deleting Instance Template: lustredev0-compute-c3nodeset-20251127075747101600000006 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-c3nodeset-20251127075747101600000006]. -[2025-11-30 19:38:32] [SUCCESS] Deleted lustredev0-compute-c3nodeset-20251127075747101600000006 -[2025-11-30 19:38:32] [EXECUTE] Deleting Instance Template: lustredev0-compute-h3nodeset-20251127075747131700000009 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-h3nodeset-20251127075747131700000009]. -[2025-11-30 19:38:35] [SUCCESS] Deleted lustredev0-compute-h3nodeset-20251127075747131700000009 -[2025-11-30 19:38:35] [EXECUTE] Deleting Instance Template: lustredev0-compute-n2nodeset-20251127075747112000000007 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-compute-n2nodeset-20251127075747112000000007]. -[2025-11-30 19:38:38] [SUCCESS] Deleted lustredev0-compute-n2nodeset-20251127075747112000000007 -[2025-11-30 19:38:38] [EXECUTE] Deleting Instance Template: lustredev0-controller-default-20251127075745707800000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-controller-default-20251127075745707800000001]. -[2025-11-30 19:38:42] [SUCCESS] Deleted lustredev0-controller-default-20251127075745707800000001 -[2025-11-30 19:38:42] [EXECUTE] Deleting Instance Template: lustredev0-login-slurm-login-20251127075746027600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustredev0-login-slurm-login-20251127075746027600000002]. -[2025-11-30 19:38:45] [SUCCESS] Deleted lustredev0-login-slurm-login-20251127075746027600000002 -[2025-11-30 19:38:45] [EXECUTE] Deleting Instance Template: lustreprod-compute-a216nodeset-20251127085007177000000008 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-a216nodeset-20251127085007177000000008]. -[2025-11-30 19:38:48] [SUCCESS] Deleted lustreprod-compute-a216nodeset-20251127085007177000000008 -[2025-11-30 19:38:48] [EXECUTE] Deleting Instance Template: lustreprod-compute-a28nodeset-20251127085007152400000005 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-a28nodeset-20251127085007152400000005]. -[2025-11-30 19:38:51] [SUCCESS] Deleted lustreprod-compute-a28nodeset-20251127085007152400000005 -[2025-11-30 19:38:51] [EXECUTE] Deleting Instance Template: lustreprod-compute-c2dnodeset-20251127085007154700000006 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-c2dnodeset-20251127085007154700000006]. -[2025-11-30 19:38:53] [SUCCESS] Deleted lustreprod-compute-c2dnodeset-20251127085007154700000006 -[2025-11-30 19:38:54] [EXECUTE] Deleting Instance Template: lustreprod-compute-c2nodeset-20251127085007178300000009 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-c2nodeset-20251127085007178300000009]. -[2025-11-30 19:38:57] [SUCCESS] Deleted lustreprod-compute-c2nodeset-20251127085007178300000009 -[2025-11-30 19:38:57] [EXECUTE] Deleting Instance Template: lustreprod-compute-c3nodeset-20251127085007120300000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-c3nodeset-20251127085007120300000003]. -[2025-11-30 19:39:00] [SUCCESS] Deleted lustreprod-compute-c3nodeset-20251127085007120300000003 -[2025-11-30 19:39:00] [EXECUTE] Deleting Instance Template: lustreprod-compute-h3nodeset-20251127085007156500000007 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-h3nodeset-20251127085007156500000007]. -[2025-11-30 19:39:03] [SUCCESS] Deleted lustreprod-compute-h3nodeset-20251127085007156500000007 -[2025-11-30 19:39:03] [EXECUTE] Deleting Instance Template: lustreprod-compute-n2nodeset-20251127085007147000000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-compute-n2nodeset-20251127085007147000000004]. -[2025-11-30 19:39:06] [SUCCESS] Deleted lustreprod-compute-n2nodeset-20251127085007147000000004 -[2025-11-30 19:39:06] [EXECUTE] Deleting Instance Template: lustreprod-controller-default-20251127085005432900000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-controller-default-20251127085005432900000001]. -[2025-11-30 19:39:10] [SUCCESS] Deleted lustreprod-controller-default-20251127085005432900000001 -[2025-11-30 19:39:10] [EXECUTE] Deleting Instance Template: lustreprod-login-slurm-login-20251127085005549500000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreprod-login-slurm-login-20251127085005549500000002]. -[2025-11-30 19:39:12] [SUCCESS] Deleted lustreprod-login-slurm-login-20251127085005549500000002 -[2025-11-30 19:39:12] [EXECUTE] Deleting Instance Template: lustreqa05-compute-a216nodeset-20251127064214383600000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-a216nodeset-20251127064214383600000003]. -[2025-11-30 19:39:15] [SUCCESS] Deleted lustreqa05-compute-a216nodeset-20251127064214383600000003 -[2025-11-30 19:39:15] [EXECUTE] Deleting Instance Template: lustreqa05-compute-a28nodeset-20251127064214402000000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-a28nodeset-20251127064214402000000004]. -[2025-11-30 19:39:18] [SUCCESS] Deleted lustreqa05-compute-a28nodeset-20251127064214402000000004 -[2025-11-30 19:39:18] [EXECUTE] Deleting Instance Template: lustreqa05-compute-c2dnodeset-20251127064214467900000009 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-c2dnodeset-20251127064214467900000009]. -[2025-11-30 19:39:21] [SUCCESS] Deleted lustreqa05-compute-c2dnodeset-20251127064214467900000009 -[2025-11-30 19:39:21] [EXECUTE] Deleting Instance Template: lustreqa05-compute-c2nodeset-20251127064214424900000005 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-c2nodeset-20251127064214424900000005]. -[2025-11-30 19:39:24] [SUCCESS] Deleted lustreqa05-compute-c2nodeset-20251127064214424900000005 -[2025-11-30 19:39:24] [EXECUTE] Deleting Instance Template: lustreqa05-compute-c3nodeset-20251127064214452600000007 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-c3nodeset-20251127064214452600000007]. -[2025-11-30 19:39:27] [SUCCESS] Deleted lustreqa05-compute-c3nodeset-20251127064214452600000007 -[2025-11-30 19:39:27] [EXECUTE] Deleting Instance Template: lustreqa05-compute-h3nodeset-20251127064214457600000008 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-h3nodeset-20251127064214457600000008]. -[2025-11-30 19:39:30] [SUCCESS] Deleted lustreqa05-compute-h3nodeset-20251127064214457600000008 -[2025-11-30 19:39:30] [EXECUTE] Deleting Instance Template: lustreqa05-compute-n2nodeset-20251127064214447800000006 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-compute-n2nodeset-20251127064214447800000006]. -[2025-11-30 19:39:34] [SUCCESS] Deleted lustreqa05-compute-n2nodeset-20251127064214447800000006 -[2025-11-30 19:39:34] [EXECUTE] Deleting Instance Template: lustreqa05-controller-default-20251127064213231900000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-controller-default-20251127064213231900000001]. -[2025-11-30 19:39:37] [SUCCESS] Deleted lustreqa05-controller-default-20251127064213231900000001 -[2025-11-30 19:39:37] [EXECUTE] Deleting Instance Template: lustreqa05-login-slurm-login-20251127064213259800000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustreqa05-login-slurm-login-20251127064213259800000002]. -[2025-11-30 19:39:40] [SUCCESS] Deleted lustreqa05-login-slurm-login-20251127064213259800000002 -[2025-11-30 19:39:40] [EXECUTE] Deleting Instance Template: lustretest-compute-a216nodeset-20251126044326259100000008 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-a216nodeset-20251126044326259100000008]. -[2025-11-30 19:39:43] [SUCCESS] Deleted lustretest-compute-a216nodeset-20251126044326259100000008 -[2025-11-30 19:39:43] [EXECUTE] Deleting Instance Template: lustretest-compute-a28nodeset-20251126044326243000000005 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-a28nodeset-20251126044326243000000005]. -[2025-11-30 19:39:46] [SUCCESS] Deleted lustretest-compute-a28nodeset-20251126044326243000000005 -[2025-11-30 19:39:46] [EXECUTE] Deleting Instance Template: lustretest-compute-c2dnodeset-20251126044326150300000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-c2dnodeset-20251126044326150300000003]. -[2025-11-30 19:39:49] [SUCCESS] Deleted lustretest-compute-c2dnodeset-20251126044326150300000003 -[2025-11-30 19:39:49] [EXECUTE] Deleting Instance Template: lustretest-compute-c2nodeset-20251126044326244400000006 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-c2nodeset-20251126044326244400000006]. -[2025-11-30 19:39:52] [SUCCESS] Deleted lustretest-compute-c2nodeset-20251126044326244400000006 -[2025-11-30 19:39:52] [EXECUTE] Deleting Instance Template: lustretest-compute-c3nodeset-20251126044326214900000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-c3nodeset-20251126044326214900000004]. -[2025-11-30 19:39:55] [SUCCESS] Deleted lustretest-compute-c3nodeset-20251126044326214900000004 -[2025-11-30 19:39:55] [EXECUTE] Deleting Instance Template: lustretest-compute-h3nodeset-20251126044326264800000009 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-h3nodeset-20251126044326264800000009]. -[2025-11-30 19:39:58] [SUCCESS] Deleted lustretest-compute-h3nodeset-20251126044326264800000009 -[2025-11-30 19:39:58] [EXECUTE] Deleting Instance Template: lustretest-compute-n2nodeset-20251126044326256800000007 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-compute-n2nodeset-20251126044326256800000007]. -[2025-11-30 19:40:01] [SUCCESS] Deleted lustretest-compute-n2nodeset-20251126044326256800000007 -[2025-11-30 19:40:01] [EXECUTE] Deleting Instance Template: lustretest-controller-default-20251126044324852600000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-controller-default-20251126044324852600000001]. -[2025-11-30 19:40:04] [SUCCESS] Deleted lustretest-controller-default-20251126044324852600000001 -[2025-11-30 19:40:04] [EXECUTE] Deleting Instance Template: lustretest-login-slurm-login-20251126044324988400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/lustretest-login-slurm-login-20251126044324988400000002]. -[2025-11-30 19:40:07] [SUCCESS] Deleted lustretest-login-slurm-login-20251126044324988400000002 -[2025-11-30 19:40:07] [EXECUTE] Deleting Instance Template: mainek-compute-a3ultranodeset-20251121092031664700000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/mainek-compute-a3ultranodeset-20251121092031664700000003]. -[2025-11-30 19:40:10] [SUCCESS] Deleted mainek-compute-a3ultranodeset-20251121092031664700000003 -[2025-11-30 19:40:10] [EXECUTE] Deleting Instance Template: mainek-controller-default-20251121092028913400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/mainek-controller-default-20251121092028913400000001]. -[2025-11-30 19:40:14] [SUCCESS] Deleted mainek-controller-default-20251121092028913400000001 -[2025-11-30 19:40:14] [EXECUTE] Deleting Instance Template: mainek-login-slurm-login-20251121092028928700000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/mainek-login-slurm-login-20251121092028928700000002]. -[2025-11-30 19:40:17] [SUCCESS] Deleted mainek-login-slurm-login-20251121092028928700000002 -[2025-11-30 19:40:17] [EXECUTE] Deleting Instance Template: pkbh4dp2-compute-computenodeset-20250412005603828600000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/pkbh4dp2-compute-computenodeset-20250412005603828600000002]. -[2025-11-30 19:40:20] [SUCCESS] Deleted pkbh4dp2-compute-computenodeset-20250412005603828600000002 -[2025-11-30 19:40:20] [EXECUTE] Deleting Instance Template: pkrv6db5de-login-slurm-login-20250801090852525400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/pkrv6db5de-login-slurm-login-20250801090852525400000002]. -[2025-11-30 19:40:23] [SUCCESS] Deleted pkrv6db5de-login-slurm-login-20250801090852525400000002 -[2025-11-30 19:40:23] [EXECUTE] Deleting Instance Template: pzpk-compute-computenodeset-20250411004353418500000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/pzpk-compute-computenodeset-20250411004353418500000002]. -[2025-11-30 19:40:26] [SUCCESS] Deleted pzpk-compute-computenodeset-20250411004353418500000002 -[2025-11-30 19:40:26] [EXECUTE] Deleting Instance Template: pzpk-controller-default-20250411004353405200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/pzpk-controller-default-20250411004353405200000001]. -[2025-11-30 19:40:29] [SUCCESS] Deleted pzpk-controller-default-20250411004353405200000001 -[2025-11-30 19:40:29] [EXECUTE] Deleting Instance Template: ractesh4d-compute-h4dnodeset-20251016152751862600000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesh4d-compute-h4dnodeset-20251016152751862600000003]. -[2025-11-30 19:40:32] [SUCCESS] Deleted ractesh4d-compute-h4dnodeset-20251016152751862600000003 -[2025-11-30 19:40:32] [EXECUTE] Deleting Instance Template: ractesh4d-controller-default-20251016152751828700000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesh4d-controller-default-20251016152751828700000001]. -[2025-11-30 19:40:35] [SUCCESS] Deleted ractesh4d-controller-default-20251016152751828700000001 -[2025-11-30 19:40:35] [EXECUTE] Deleting Instance Template: ractesh4d-login-slurm-login-20251016152751847100000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesh4d-login-slurm-login-20251016152751847100000002]. -[2025-11-30 19:40:38] [SUCCESS] Deleted ractesh4d-login-slurm-login-20251016152751847100000002 -[2025-11-30 19:40:38] [EXECUTE] Deleting Instance Template: ractesth4d-compute-h4dnodeset-20251013062737759200000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesth4d-compute-h4dnodeset-20251013062737759200000003]. -[2025-11-30 19:40:41] [SUCCESS] Deleted ractesth4d-compute-h4dnodeset-20251013062737759200000003 -[2025-11-30 19:40:41] [EXECUTE] Deleting Instance Template: ractesth4d-controller-default-20251013062737731900000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesth4d-controller-default-20251013062737731900000001]. -[2025-11-30 19:40:44] [SUCCESS] Deleted ractesth4d-controller-default-20251013062737731900000001 -[2025-11-30 19:40:44] [EXECUTE] Deleting Instance Template: ractesth4d-login-slurm-login-20251013062737750800000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/ractesth4d-login-slurm-login-20251013062737750800000002]. -[2025-11-30 19:40:47] [SUCCESS] Deleted ractesth4d-login-slurm-login-20251013062737750800000002 -[2025-11-30 19:40:47] [EXECUTE] Deleting Instance Template: rock8a070a-compute-computenodeset-20250912110554713000000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/rock8a070a-compute-computenodeset-20250912110554713000000004]. -[2025-11-30 19:40:50] [SUCCESS] Deleted rock8a070a-compute-computenodeset-20250912110554713000000004 -[2025-11-30 19:40:50] [EXECUTE] Deleting Instance Template: rock8a070a-compute-debugnodeset-20250912110554709900000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/rock8a070a-compute-debugnodeset-20250912110554709900000003]. -[2025-11-30 19:40:53] [SUCCESS] Deleted rock8a070a-compute-debugnodeset-20250912110554709900000003 -[2025-11-30 19:40:53] [EXECUTE] Deleting Instance Template: rock8a070a-compute-h3nodeset-20250912110554735800000005 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/rock8a070a-compute-h3nodeset-20250912110554735800000005]. -[2025-11-30 19:40:56] [SUCCESS] Deleted rock8a070a-compute-h3nodeset-20250912110554735800000005 -[2025-11-30 19:40:56] [EXECUTE] Deleting Instance Template: rock8a070a-controller-default-20250912110554351400000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/rock8a070a-controller-default-20250912110554351400000001]. -[2025-11-30 19:41:00] [SUCCESS] Deleted rock8a070a-controller-default-20250912110554351400000001 -[2025-11-30 19:41:00] [EXECUTE] Deleting Instance Template: rock8a070a-login-slurm-login-20250912110554546400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/rock8a070a-login-slurm-login-20250912110554546400000002]. -[2025-11-30 19:41:03] [SUCCESS] Deleted rock8a070a-login-slurm-login-20250912110554546400000002 -[2025-11-30 19:41:03] [EXECUTE] Deleting Instance Template: simtestnet-compute-a3ultranodeset-20251128190204679000000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/simtestnet-compute-a3ultranodeset-20251128190204679000000002]. -[2025-11-30 19:41:06] [SUCCESS] Deleted simtestnet-compute-a3ultranodeset-20251128190204679000000002 -[2025-11-30 19:41:06] [EXECUTE] Deleting Instance Template: simtestnet-controller-default-20251128190210353600000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/simtestnet-controller-default-20251128190210353600000003]. -[2025-11-30 19:41:09] [SUCCESS] Deleted simtestnet-controller-default-20251128190210353600000003 -[2025-11-30 19:41:09] [EXECUTE] Deleting Instance Template: simtestnet-login-slurm-login-20251128190201729100000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/simtestnet-login-slurm-login-20251128190201729100000001]. -[2025-11-30 19:41:12] [SUCCESS] Deleted simtestnet-login-slurm-login-20251128190201729100000001 -[2025-11-30 19:41:12] [EXECUTE] Deleting Instance Template: slurm0-compute-a3nodeset-20251009153324036900000004 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm0-compute-a3nodeset-20251009153324036900000004]. -[2025-11-30 19:41:15] [SUCCESS] Deleted slurm0-compute-a3nodeset-20251009153324036900000004 -[2025-11-30 19:41:15] [EXECUTE] Deleting Instance Template: slurm0-compute-debugnodeset-20251009153324023000000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm0-compute-debugnodeset-20251009153324023000000003]. -[2025-11-30 19:41:18] [SUCCESS] Deleted slurm0-compute-debugnodeset-20251009153324023000000003 -[2025-11-30 19:41:18] [EXECUTE] Deleting Instance Template: slurm0-controller-default-20251009153304999900000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm0-controller-default-20251009153304999900000001]. -[2025-11-30 19:41:21] [SUCCESS] Deleted slurm0-controller-default-20251009153304999900000001 -[2025-11-30 19:41:21] [EXECUTE] Deleting Instance Template: slurm0-login-login-20251009153305041200000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm0-login-login-20251009153305041200000002]. -[2025-11-30 19:41:24] [SUCCESS] Deleted slurm0-login-login-20251009153305041200000002 -[2025-11-30 19:41:24] [EXECUTE] Deleting Instance Template: slurm9ff-compute-debugnodeset-20250925215605513600000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm9ff-compute-debugnodeset-20250925215605513600000003]. -[2025-11-30 19:41:27] [SUCCESS] Deleted slurm9ff-compute-debugnodeset-20250925215605513600000003 -[2025-11-30 19:41:27] [EXECUTE] Deleting Instance Template: slurm9ff-controller-default-20250925215558268500000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm9ff-controller-default-20250925215558268500000001]. -[2025-11-30 19:41:31] [SUCCESS] Deleted slurm9ff-controller-default-20250925215558268500000001 -[2025-11-30 19:41:31] [EXECUTE] Deleting Instance Template: slurm9ff-login-slurm-login-20250925215558759400000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurm9ff-login-slurm-login-20250925215558759400000002]. -[2025-11-30 19:41:33] [SUCCESS] Deleted slurm9ff-login-slurm-login-20250925215558759400000002 -[2025-11-30 19:41:33] [EXECUTE] Deleting Instance Template: slurmf1c-compute-debugnodeset-20250925064111290200000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmf1c-compute-debugnodeset-20250925064111290200000002]. -[2025-11-30 19:41:37] [SUCCESS] Deleted slurmf1c-compute-debugnodeset-20250925064111290200000002 -[2025-11-30 19:41:37] [EXECUTE] Deleting Instance Template: slurmf1c-controller-default-20250925064111186200000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmf1c-controller-default-20250925064111186200000001]. -[2025-11-30 19:41:40] [SUCCESS] Deleted slurmf1c-controller-default-20250925064111186200000001 -[2025-11-30 19:41:40] [EXECUTE] Deleting Instance Template: slurmf1c-login-slurm-login-20250925064111290800000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmf1c-login-slurm-login-20250925064111290800000003]. -[2025-11-30 19:41:43] [SUCCESS] Deleted slurmf1c-login-slurm-login-20250925064111290800000003 -[2025-11-30 19:41:43] [EXECUTE] Deleting Instance Template: slurmflex-compute-nodeset-20250806200217240000000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmflex-compute-nodeset-20250806200217240000000001]. -[2025-11-30 19:41:46] [SUCCESS] Deleted slurmflex-compute-nodeset-20250806200217240000000001 -[2025-11-30 19:41:46] [EXECUTE] Deleting Instance Template: slurmflex-compute-nodeset-20250912233249589700000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmflex-compute-nodeset-20250912233249589700000002]. -[2025-11-30 19:41:49] [SUCCESS] Deleted slurmflex-compute-nodeset-20250912233249589700000002 -[2025-11-30 19:41:49] [EXECUTE] Deleting Instance Template: slurmflex-compute-nodeset-20251111083658962700000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmflex-compute-nodeset-20251111083658962700000002]. -[2025-11-30 19:41:52] [SUCCESS] Deleted slurmflex-compute-nodeset-20251111083658962700000002 -[2025-11-30 19:41:52] [EXECUTE] Deleting Instance Template: slurmsimpl-compute-nodeset-20250708064936486300000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmsimpl-compute-nodeset-20250708064936486300000001]. -[2025-11-30 19:41:55] [SUCCESS] Deleted slurmsimpl-compute-nodeset-20250708064936486300000001 -[2025-11-30 19:41:55] [EXECUTE] Deleting Instance Template: slurmsimpl-compute-nodeset-20250814191439012800000001 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmsimpl-compute-nodeset-20250814191439012800000001]. -[2025-11-30 19:41:59] [SUCCESS] Deleted slurmsimpl-compute-nodeset-20250814191439012800000001 -[2025-11-30 19:41:59] [EXECUTE] Deleting Instance Template: slurmsimpl-controller-default-20250708064945457400000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmsimpl-controller-default-20250708064945457400000003]. -[2025-11-30 19:42:02] [SUCCESS] Deleted slurmsimpl-controller-default-20250708064945457400000003 -[2025-11-30 19:42:02] [EXECUTE] Deleting Instance Template: slurmsimpl-controller-default-20250814191448978300000003 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmsimpl-controller-default-20250814191448978300000003]. -[2025-11-30 19:42:05] [SUCCESS] Deleted slurmsimpl-controller-default-20250814191448978300000003 -[2025-11-30 19:42:05] [EXECUTE] Deleting Instance Template: slurmsimpl-login-slurm-login-20250814191439021200000002 (Global) -Deleted [https://www.googleapis.com/compute/v1/projects/hpc-toolkit-dev/global/instanceTemplates/slurmsimpl-login-slurm-login-20250814191439021200000002]. -[2025-11-30 19:42:08] [SUCCESS] Deleted slurmsimpl-login-slurm-login-20250814191439021200000002 -[2025-11-30 19:42:08] [SKIP] testing2 (In Exclusion List) -[2025-11-30 19:42:08] [SKIP] welp-insta-temp (In Exclusion List) -[2025-11-30 19:42:08] [INFO] CLEANUP RUN FINISHED diff --git a/tools/cleanup.sh b/tools/cleanup.sh index 49d639f64e..961cfe58d0 100755 --- a/tools/cleanup.sh +++ b/tools/cleanup.sh @@ -6,7 +6,6 @@ # Associative array for exclusions declare -A EXCLUSION_MAP -DELETE_LIMIT=200 ERROR_COUNT=0 # Environment Variables expected from Cloud Build @@ -122,7 +121,7 @@ process_resources() { local delete_command_base="$3" local scope_type="$4" - log "INFO" "--- Processing: $label (Limit: $DELETE_LIMIT) ---" + log "INFO" "--- Processing: $label ---" local resources if ! resources=$(eval "$list_command"); then @@ -139,10 +138,6 @@ process_resources() { local count=0 while IFS=$'\t' read -r name scope labels_str; do [[ -z "$name" ]] && continue - if [[ $count -ge $DELETE_LIMIT ]]; then - log "INFO" "Hit delete limit ($DELETE_LIMIT) for $label." - break - fi if is_excluded "$name" "${labels_str:-}"; then continue; fi @@ -161,7 +156,7 @@ process_resources() { # ============================================================================== process_instance_templates() { - log "INFO" "--- Processing: Instance Templates (Limit: $DELETE_LIMIT) ---" + log "INFO" "--- Processing: Instance Templates ---" local templates if ! templates=$(gcloud compute instance-templates list \ --project="$PROJECT_ID" \ @@ -176,7 +171,6 @@ process_instance_templates() { local count=0 while IFS=$'\t' read -r name labels_str; do if [[ -z "$name" ]]; then continue; fi - if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Instance Templates."; break; fi if is_excluded "$name" "${labels_str:-}"; then continue; fi execute_delete "Instance Template" "$name" \ "gcloud compute instance-templates delete \"$name\" --project=\"$PROJECT_ID\" --quiet" \ @@ -198,7 +192,7 @@ process_addresses() { } process_vpc_peerings() { - log "INFO" "--- Processing: VPC Peerings (Limit: $DELETE_LIMIT) ---" + log "INFO" "--- Processing: VPC Peerings---" local networks_json if ! networks_json=$(gcloud compute networks list --project="$PROJECT_ID" --format="json"); then log "ERROR" "Failed to list networks." @@ -209,7 +203,6 @@ process_vpc_peerings() { local count=0 while IFS= read -r net_obj; do - if [[ $count -ge $DELETE_LIMIT ]]; then break; fi local net_name net_name=$(echo "$net_obj" | jq -r '.name') if [[ -z "$net_name" || "$net_name" == "null" ]]; then continue; fi @@ -219,7 +212,6 @@ process_vpc_peerings() { if [[ "$peerings_json" == "[]" || "$peerings_json" == "null" ]]; then continue; fi while IFS= read -r peering_obj; do - if [[ $count -ge $DELETE_LIMIT ]]; then break; fi local peering_name peering_name=$(echo "$peering_obj" | jq -r '.name') if [[ -z "$peering_name" || "$peering_name" == "null" ]]; then continue; fi @@ -246,12 +238,11 @@ process_vpc_peerings() { fi done < <(echo "$peerings_json" | jq -c '.[]') done < <(echo "$networks_json" | jq -c '.[]') - if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for VPC Peerings."; fi log "INFO" "Finished processing VPC Peerings. $count peerings actioned." } process_iam_deleted_members() { - log "INFO" "--- Processing: IAM Role Bindings for Deleted SAs (Limit: $DELETE_LIMIT) ---" + log "INFO" "--- Processing: IAM Role Bindings for Deleted SAs ---" local policy_json if ! policy_json=$(gcloud projects get-iam-policy "$PROJECT_ID" --format=json); then log "ERROR" "Failed to get IAM policy." @@ -265,7 +256,6 @@ process_iam_deleted_members() { local count=0 while IFS=$'\t' read -r role member; do if [[ -z "$role" || -z "$member" ]]; then continue; fi - if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for IAM Bindings."; break; fi local cmd="gcloud projects remove-iam-policy-binding \"$PROJECT_ID\" --member=\"$member\" --role=\"$role\" --condition=None --quiet" @@ -283,7 +273,7 @@ process_iam_deleted_members() { } process_vm_images() { - log "INFO" "--- Processing: VM Images (Limit: $DELETE_LIMIT) ---" + log "INFO" "--- Processing: VM Images ---" local images if ! images=$(gcloud compute images list --project="$PROJECT_ID" --no-standard-images \ --format="value(name,creationTimestamp,labels)"); then @@ -305,7 +295,6 @@ process_vm_images() { local count=0 while IFS=$'\t' read -r name timestamp labels_str; do [[ -z "$name" ]] && continue - if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for VM Images."; break; fi if is_excluded "$name" "${labels_str:-}"; then continue; fi local ts_seconds @@ -322,7 +311,7 @@ process_vm_images() { } process_docker_images() { - log "INFO" "--- Processing: Docker Images for 'test-runner' (Artifact Registry) (Limit: $DELETE_LIMIT) ---" + log "INFO" "--- Processing: Docker Images for 'test-runner' (Artifact Registry) ---" local cutoff_date cutoff_date=$(date -u -d "14 days ago" '+%Y-%m-%dT%H:%M:%SZ') local cutoff_seconds @@ -357,7 +346,6 @@ process_docker_images() { if [[ $image_seconds -ge $cutoff_seconds ]]; then continue; else - if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Docker Images."; break; fi if is_excluded "$package_name"; then continue; fi if is_excluded "$full_image_ref"; then continue; fi @@ -371,7 +359,7 @@ process_docker_images() { } process_firewalls() { - log "INFO" "--- Processing: Firewall Rules (Limit: $DELETE_LIMIT) ---" + log "INFO" "--- Processing: Firewall Rules ---" local fws if ! fws=$(gcloud compute firewall-rules list --project="$PROJECT_ID" \ --filter="creationTimestamp < '$CUTOFF_TIME'" \ @@ -388,7 +376,6 @@ process_firewalls() { local network_name network_name=$(basename "$network_uri") if [[ "$network_name" == "default" ]]; then continue; fi - if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Firewall Rules."; break; fi if is_excluded "$name" "${labels_str:-}"; then continue; fi execute_delete "Firewall Rule" "$name" \ "gcloud compute firewall-rules delete \"$name\" --project=\"$PROJECT_ID\" --quiet" @@ -397,7 +384,7 @@ process_firewalls() { } process_filestore() { - log "INFO" "--- Processing: Filestore Instances (Limit: $DELETE_LIMIT) ---" + log "INFO" "--- Processing: Filestore Instances ---" local fs_json if ! fs_json=$(gcloud filestore instances list --project="$PROJECT_ID" --filter="createTime < '$CUTOFF_TIME'" --format="json"); then log "ERROR" "Failed to list Filestore instances." @@ -418,7 +405,6 @@ process_filestore() { while IFS=$'\t' read -r location name labels_str; do location=$(echo "$location" | awk '{$1=$1};1'); name=$(echo "$name" | awk '{$1=$1};1') if [[ -z "$location" || -z "$name" ]]; then continue; fi - if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Filestore."; break; fi if is_excluded "$name" "${labels_str:-}"; then continue; fi local delete_cmd="gcloud filestore instances delete \"$name\" --project=\"$PROJECT_ID\" --location=\"$location\" --quiet --force" execute_delete "Filestore" "$name" "$delete_cmd" "($location)" @@ -428,7 +414,7 @@ process_filestore() { } process_subnetworks() { - log "INFO" "--- Processing: Subnetworks (Limit: $DELETE_LIMIT) ---" + log "INFO" "--- Processing: Subnetworks ---" local subnets if ! subnets=$(gcloud compute networks subnets list --project="$PROJECT_ID" --filter="creationTimestamp < '$CUTOFF_TIME'" --format="value(name,region,network,selfLink)"); then log "ERROR" "Failed to list subnets" @@ -441,7 +427,6 @@ process_subnetworks() { [[ -z "$name" ]] && continue local network_name=$(basename "$network_uri") if [[ "$network_name" == "default" ]]; then continue; fi - if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Subnetworks."; break; fi if is_excluded "$name"; then continue; fi # Note: listing dependents might fail, wrapping in error check not strictly necessary for deletion loop but good practice @@ -457,7 +442,7 @@ process_subnetworks() { } process_networks() { - log "INFO" "--- Processing: VPC Networks (Limit: $DELETE_LIMIT) ---" + log "INFO" "--- Processing: VPC Networks ---" local networks if ! networks=$(gcloud compute networks list --project="$PROJECT_ID" --filter="creationTimestamp < '$CUTOFF_TIME'" --format="value(name,selfLink)"); then log "ERROR" "Failed to list networks" @@ -469,7 +454,6 @@ process_networks() { while IFS=$'\t' read -r name self_link; do [[ -z "$name" ]] && continue if [[ "$name" == "default" ]]; then continue; fi - if [[ $count -ge $DELETE_LIMIT ]]; then log "INFO" "Hit delete limit ($DELETE_LIMIT) for Networks."; break; fi if is_excluded "$name"; then continue; fi local routes @@ -493,7 +477,6 @@ main() { log "INFO" "STARTING RESOURCE CLEANUP: $PROJECT_ID" log "INFO" "Time Cutoff (General): $CUTOFF_TIME" log "INFO" "Time Cutoff (Images): $CUTOFF_TIME_IMAGES" - log "INFO" "Delete Limit per Type: $DELETE_LIMIT" log "INFO" "DRY_RUN: $DRY_RUN" log "INFO" "Exclusion File: $EXCLUSION_FILE" From 9248a90e1d7cc183a2d6bdfdd7340b46f48e185f Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Tue, 9 Dec 2025 16:18:45 +0000 Subject: [PATCH 05/19] edit --- .../a3mega-slurm-deployment.yaml | 19 ++++++-------- tools/exclusions.txt | 26 ------------------- 2 files changed, 8 insertions(+), 37 deletions(-) diff --git a/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-deployment.yaml b/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-deployment.yaml index bb2f717768..afd5f817cc 100644 --- a/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-deployment.yaml +++ b/examples/machine-learning/a3-megagpu-8g/a3mega-slurm-deployment.yaml @@ -16,15 +16,15 @@ terraform_backend_defaults: type: gcs configuration: - bucket: customer-bucket + bucket: simranka vars: - deployment_name: a3mega-base - project_id: customer-project - region: customer-region - zone: customer-zone - network_name_system: a3mega-sys-net - subnetwork_name_system: a3mega-sys-subnet + deployment_name: deletion-test + project_id: hpc-toolkit-dev + region: europe-west1 + zone: europe-west1-c + network_name_system: deletion-test-sys-net + subnetwork_name_system: deletion-test-sys-subnet enable_ops_agent: true enable_nvidia_dcgm: true enable_nvidia_persistenced: true @@ -32,7 +32,4 @@ vars: final_image_family: slurm-a3mega slurm_cluster_name: a3mega a3mega_cluster_size: 2 # supply cluster size - a3mega_reservation_name: "" # supply reservation name - # Additional provisioning models (pick only one), can be used to substitute `a3mega_reservation_name`: - a3mega_dws_flex_enabled: false # To make use of DWS Flex-Start, for more info visit: https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/docs/slurm-dws-flex.md - a3mega_enable_spot_vm: false # To make use of Spot VMs, for more info visit: https://cloud.google.com/compute/docs/instances/spot + a3mega_enable_spot_vm: true # To make use of Spot VMs, for more info visit: https://cloud.google.com/compute/docs/instances/spot diff --git a/tools/exclusions.txt b/tools/exclusions.txt index d7d8e208e3..263dd1d856 100644 --- a/tools/exclusions.txt +++ b/tools/exclusions.txt @@ -27,22 +27,7 @@ image-inspector-550 image-inspector gke-managed-lustre-basic-net-fw-allow-iap-ingress gke-managed-lustre-basic-net-fw-allow-internal-traffic -a3m-sp-h13-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h2-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h2-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h5-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -a3m-sp-h5-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-prod-11-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-prod-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-prod-6-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-prod-6-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-test-1-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a3high-test-1-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-gke-np-sa@hpc-toolkit-dev.iam.gserviceaccount.com -gke-a4-gke-wl-sa@hpc-toolkit-dev.iam.gserviceaccount.com hpc-vpc allow-internal allow-ssh @@ -56,16 +41,5 @@ rocka4hf-rocky9-20250910t040750z rocka4h-rocky9-20250908t175724z slurm-gcp-next-hpc-rocky-linux-8-1739990978 slurm-gcp-next-hpc-rocky-linux-8-1740100297 -a3mega-compute-a3meganodeset-20251118080924120000000004 -a3mega-compute-debugnodeset-20251118080924115100000003 -a3mega-controller-default-20251118080901356800000001 -a3mega-login-login-20251118080901434600000002 -a3slurmsy-compute-a3nodeset-20251016115123978200000002 -a3slurmsy-controller-default-20251016115129563200000003 -a3slurmsy-login-slurm-login-20251016115120300800000001 -batch-job-instance-template-20250901212237920900000001 -batch-job-instance-template-20250912070019961500000001 -buildslurm-compute-debugnodeset-20251030080636567600000001 -buildslurm-controller-default-20251030080646114800000002 welp-insta-temp testing2 \ No newline at end of file From 402ba0a9ff8792ae16a2668e3cc1637423941cc9 Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Tue, 9 Dec 2025 16:19:51 +0000 Subject: [PATCH 06/19] too be deleted --- .../artifacts/DO_NOT_MODIFY_THIS_DIRECTORY | 1 + .../.ghpc/artifacts/expanded_blueprint.yaml | 651 +++++ deletion-test/.gitignore | 48 + deletion-test/build_script/main.tf | 86 + .../embedded/community/modules/README.md | 7 + .../modules/compute/gke-nodeset/README.md | 55 + .../modules/compute/gke-nodeset/main.tf | 64 + .../modules/compute/gke-nodeset/metadata.yaml | 20 + .../modules/compute/gke-nodeset/output.tf | 18 + .../compute/gke-nodeset/persistent_volumes.tf | 50 + .../templates/nodeset-general.yaml.tftpl | 203 ++ .../modules/compute/gke-nodeset/variables.tf | 118 + .../modules/compute/gke-nodeset/versions.tf | 27 + .../modules/compute/gke-partition/README.md | 39 + .../modules/compute/gke-partition/main.tf | 47 + .../compute/gke-partition/metadata.yaml | 19 + .../compute/gke-partition/variables.tf | 43 + .../modules/compute/gke-partition/versions.tf | 27 + .../compute/htcondor-execute-point/README.md | 271 ++ .../htcondor-execute-point/compute_image.tf | 30 + .../files/htcondor_configure.yml | 74 + .../files/htcondor_configure_autoscaler.yml | 98 + .../compute/htcondor-execute-point/main.tf | 218 ++ .../htcondor-execute-point/metadata.yaml | 20 + .../compute/htcondor-execute-point/outputs.tf | 25 + .../templates/condor_config.tftpl | 31 + .../download-condor-config.ps1.tftpl | 34 + .../htcondor-execute-point/variables.tf | 265 ++ .../htcondor-execute-point/versions.tf | 34 + .../community/modules/compute/mig/README.md | 45 + .../community/modules/compute/mig/main.tf | 85 + .../modules/compute/mig/metadata.yaml | 21 + .../community/modules/compute/mig/outputs.tf | 18 + .../modules/compute/mig/variables.tf | 86 + .../community/modules/compute/mig/versions.tf | 27 + .../modules/compute/notebook/README.md | 112 + .../modules/compute/notebook/main.tf | 96 + .../modules/compute/notebook/metadata.yaml | 20 + .../modules/compute/notebook/variables.tf | 111 + .../modules/compute/notebook/versions.tf | 29 + .../README.md | 135 + .../main.tf | 128 + .../metadata.yaml | 20 + .../outputs.tf | 36 + .../source_image_logic.tf | 30 + .../variables.tf | 402 +++ .../versions.tf | 22 + .../README.md | 85 + .../schedmd-slurm-gcp-v6-nodeset-tpu/main.tf | 59 + .../metadata.yaml | 21 + .../outputs.tf | 39 + .../variables.tf | 171 ++ .../versions.tf | 23 + .../schedmd-slurm-gcp-v6-nodeset/README.md | 227 ++ .../schedmd-slurm-gcp-v6-nodeset/main.tf | 232 ++ .../metadata.yaml | 21 + .../schedmd-slurm-gcp-v6-nodeset/outputs.tf | 112 + .../source_image_logic.tf | 30 + .../schedmd-slurm-gcp-v6-nodeset/variables.tf | 641 +++++ .../schedmd-slurm-gcp-v6-nodeset/versions.tf | 29 + .../schedmd-slurm-gcp-v6-partition/README.md | 105 + .../schedmd-slurm-gcp-v6-partition/main.tf | 41 + .../metadata.yaml | 20 + .../schedmd-slurm-gcp-v6-partition/outputs.tf | 54 + .../variables.tf | 311 +++ .../versions.tf | 23 + .../container/artifact-registry/README.md | 157 ++ .../container/artifact-registry/main.tf | 268 ++ .../container/artifact-registry/metadata.yaml | 21 + .../container/artifact-registry/outputs.tf | 18 + .../container/artifact-registry/validation.tf | 49 + .../container/artifact-registry/variables.tf | 122 + .../container/artifact-registry/versions.tf | 27 + .../database/bigquery-dataset/README.md | 76 + .../modules/database/bigquery-dataset/main.tf | 32 + .../database/bigquery-dataset/metadata.yaml | 19 + .../database/bigquery-dataset/outputs.tf | 20 + .../database/bigquery-dataset/variables.tf | 36 + .../database/bigquery-dataset/versions.tf | 29 + .../modules/database/bigquery-table/README.md | 87 + .../modules/database/bigquery-table/main.tf | 37 + .../database/bigquery-table/metadata.yaml | 19 + .../database/bigquery-table/outputs.tf | 28 + .../database/bigquery-table/variables.tf | 46 + .../database/bigquery-table/versions.tf | 29 + .../slurm-cloudsql-federation/README.md | 107 + .../slurm-cloudsql-federation/main.tf | 165 ++ .../slurm-cloudsql-federation/metadata.yaml | 21 + .../slurm-cloudsql-federation/outputs.tf | 27 + .../slurm-cloudsql-federation/variables.tf | 173 ++ .../slurm-cloudsql-federation/versions.tf | 36 + .../file-system/DDN-EXAScaler/README.md | 158 ++ .../modules/file-system/DDN-EXAScaler/main.tf | 72 + .../file-system/DDN-EXAScaler/metadata.yaml | 22 + .../file-system/DDN-EXAScaler/outputs.tf | 90 + .../file-system/DDN-EXAScaler/variables.tf | 502 ++++ .../file-system/DDN-EXAScaler/versions.tf | 24 + .../modules/file-system/Intel-DAOS/README.md | 1 + .../modules/file-system/nfs-server/README.md | 152 ++ .../modules/file-system/nfs-server/main.tf | 131 + .../file-system/nfs-server/metadata.yaml | 19 + .../modules/file-system/nfs-server/outputs.tf | 53 + .../nfs-server/scripts/install-nfs-client.sh | 37 + .../scripts/install-nfs-server.sh.tpl | 35 + .../file-system/nfs-server/scripts/mount.sh | 58 + .../file-system/nfs-server/scripts/mount.yaml | 39 + .../file-system/nfs-server/variables.tf | 194 ++ .../file-system/nfs-server/versions.tf | 37 + .../file-system/sycomp-scale/README.md | 35 + .../modules/file-system/weka-client/README.md | 182 ++ .../file-system/weka-client/metadata.yaml | 18 + .../file-system/weka-client/outputs.tf | 71 + .../templates/install-weka-client.yaml.tftpl | 133 + .../weka-client/templates/mount-weka.sh.tftpl | 101 + .../templates/mount-weka.yaml.tftpl | 54 + .../file-system/weka-client/variables.tf | 39 + .../file-system/weka-client/versions.tf | 19 + .../FSI_MonteCarlo.ipynb | 125 + .../files/fsi-montecarlo-on-batch/README.md | 97 + .../fsi-montecarlo-on-batch/iteration.sh | 23 + .../files/fsi-montecarlo-on-batch/main.tf | 102 + .../fsi-montecarlo-on-batch/mc_run.tpl.py | 157 ++ .../fsi-montecarlo-on-batch/mc_run.tpl.yaml | 36 + .../fsi-montecarlo-on-batch/mc_run_reqs.txt | 9 + .../fsi-montecarlo-on-batch/metadata.yaml | 18 + .../fsi-montecarlo-on-batch/variables.tf | 51 + .../files/fsi-montecarlo-on-batch/versions.tf | 43 + .../internal/slurm-gcp/instance/README.md | 100 + .../internal/slurm-gcp/instance/main.tf | 126 + .../internal/slurm-gcp/instance/outputs.tf | 41 + .../internal/slurm-gcp/instance/variables.tf | 119 + .../internal/slurm-gcp/instance/versions.tf | 31 + .../slurm-gcp/instance_template/README.md | 87 + .../files/startup_sh_unlinted | 169 ++ .../slurm-gcp/instance_template/main.tf | 171 ++ .../slurm-gcp/instance_template/outputs.tf | 43 + .../slurm-gcp/instance_template/variables.tf | 431 ++++ .../slurm-gcp/instance_template/versions.tf | 25 + .../internal_instance_template/README.md | 89 + .../internal_instance_template/main.tf | 234 ++ .../internal_instance_template/outputs.tf | 33 + .../internal_instance_template/variables.tf | 398 +++ .../internal_instance_template/versions.tf | 30 + .../internal/slurm-gcp/login/README.md | 52 + .../modules/internal/slurm-gcp/login/main.tf | 112 + .../internal/slurm-gcp/login/outputs.tf | 25 + .../internal/slurm-gcp/login/variables.tf | 188 ++ .../internal/slurm-gcp/login/versions.tf | 29 + .../internal/slurm-gcp/nodeset_tpu/README.md | 95 + .../internal/slurm-gcp/nodeset_tpu/main.tf | 121 + .../internal/slurm-gcp/nodeset_tpu/outputs.tf | 30 + .../slurm-gcp/nodeset_tpu/variables.tf | 158 ++ .../slurm-gcp/nodeset_tpu/versions.tf | 30 + .../dependencies-installer/README.md | 61 + .../helm_install/README.md | 64 + .../helm_install/main.tf | 75 + .../helm_install/metadata.yaml | 19 + .../helm_install/variables.tf | 212 ++ .../helm_install/versions.tf | 24 + .../kubernetes_manifest/README.md | 40 + .../kubernetes_manifest/main.tf | 104 + .../kubernetes_manifest/metadata.yaml | 19 + .../kubernetes_manifest/variables.tf | 69 + .../kubernetes_manifest/versions.tf | 24 + .../management/dependencies-installer/main.tf | 183 ++ .../dependencies-installer/metadata.yaml | 19 + .../dependencies-installer/providers.tf | 25 + .../dependencies-installer/variables.tf | 70 + .../dependencies-installer/versions.tf | 30 + .../network/private-service-access/README.md | 122 + .../network/private-service-access/main.tf | 61 + .../private-service-access/metadata.yaml | 20 + .../network/private-service-access/outputs.tf | 43 + .../private-service-access/variables.tf | 59 + .../private-service-access/versions.tf | 37 + .../modules/project/new-project/README.md | 128 + .../modules/project/service-account/README.md | 111 + .../modules/project/service-account/main.tf | 37 + .../project/service-account/metadata.yaml | 19 + .../project/service-account/outputs.tf | 36 + .../project/service-account/variables.tf | 113 + .../project/service-account/versions.tf | 22 + .../project/service-enablement/README.md | 70 + .../project/service-enablement/main.tf | 28 + .../project/service-enablement/metadata.yaml | 19 + .../project/service-enablement/variables.tf | 31 + .../project/service-enablement/versions.tf | 29 + .../modules/pubsub/bigquery-sub/README.md | 87 + .../modules/pubsub/bigquery-sub/main.tf | 57 + .../modules/pubsub/bigquery-sub/metadata.yaml | 19 + .../modules/pubsub/bigquery-sub/outputs.tf | 20 + .../modules/pubsub/bigquery-sub/variables.tf | 51 + .../modules/pubsub/bigquery-sub/versions.tf | 35 + .../community/modules/pubsub/topic/README.md | 82 + .../community/modules/pubsub/topic/main.tf | 48 + .../modules/pubsub/topic/metadata.yaml | 19 + .../community/modules/pubsub/topic/outputs.tf | 26 + .../modules/pubsub/topic/variables.tf | 74 + .../modules/pubsub/topic/versions.tf | 32 + .../chrome-remote-desktop/README.md | 113 + .../chrome-remote-desktop/main.tf | 111 + .../chrome-remote-desktop/metadata.yaml | 18 + .../chrome-remote-desktop/outputs.tf | 25 + .../scripts/configure-chrome-desktop.yml | 61 + .../scripts/configure-grid-drivers.yml | 163 ++ .../scripts/disable-sleep.yml | 39 + .../chrome-remote-desktop/variables.tf | 277 ++ .../chrome-remote-desktop/versions.tf | 19 + .../scheduler/htcondor-access-point/README.md | 187 ++ .../files/htcondor_configure.yml | 120 + .../scheduler/htcondor-access-point/main.tf | 338 +++ .../htcondor-access-point/metadata.yaml | 20 + .../htcondor-access-point/outputs.tf | 25 + .../templates/condor_config.tftpl | 70 + .../htcondor-access-point/variables.tf | 266 ++ .../htcondor-access-point/versions.tf | 37 + .../htcondor-central-manager/README.md | 159 ++ .../files/htcondor_configure.yml | 72 + .../htcondor-central-manager/main.tf | 226 ++ .../htcondor-central-manager/metadata.yaml | 20 + .../htcondor-central-manager/outputs.tf | 30 + .../templates/condor_config.tftpl | 31 + .../htcondor-central-manager/variables.tf | 192 ++ .../htcondor-central-manager/versions.tf | 33 + .../scheduler/htcondor-pool-secrets/README.md | 172 ++ .../files/htcondor_secrets.yml | 102 + .../scheduler/htcondor-pool-secrets/main.tf | 168 ++ .../htcondor-pool-secrets/metadata.yaml | 20 + .../htcondor-pool-secrets/outputs.tf | 50 + .../templates/fetch-idtoken.ps1.tftpl | 26 + .../htcondor-pool-secrets/variables.tf | 67 + .../htcondor-pool-secrets/versions.tf | 33 + .../htcondor-service-accounts/README.md | 128 + .../htcondor-service-accounts/main.tf | 51 + .../htcondor-service-accounts/metadata.yaml | 19 + .../htcondor-service-accounts/outputs.tf | 30 + .../htcondor-service-accounts/variables.tf | 56 + .../htcondor-service-accounts/versions.tf | 19 + .../scheduler/htcondor-setup/README.md | 118 + .../modules/scheduler/htcondor-setup/main.tf | 68 + .../scheduler/htcondor-setup/metadata.yaml | 21 + .../scheduler/htcondor-setup/outputs.tf | 27 + .../scheduler/htcondor-setup/variables.tf | 55 + .../scheduler/htcondor-setup/versions.tf | 19 + .../schedmd-slurm-gcp-v6-controller/README.md | 405 +++ .../controller.tf | 213 ++ .../etc/htc-slurm.conf.tpl | 65 + .../etc/htc-slurmdbd.conf.tpl | 34 + .../etc/long-prolog-slurm.conf.tpl | 71 + .../schedmd-slurm-gcp-v6-controller/login.tf | 50 + .../schedmd-slurm-gcp-v6-controller/main.tf | 35 + .../metadata.yaml | 21 + .../modules/cleanup_compute/README.md | 42 + .../modules/cleanup_compute/main.tf | 46 + .../scripts/cleanup_compute.sh | 100 + .../modules/cleanup_compute/variables.tf | 71 + .../modules/cleanup_compute/versions.tf | 27 + .../modules/cleanup_tpu/README.md | 79 + .../modules/cleanup_tpu/main.tf | 32 + .../cleanup_tpu/scripts/cleanup_tpu.sh | 63 + .../modules/cleanup_tpu/variables.tf | 60 + .../modules/cleanup_tpu/versions.tf | 27 + .../modules/slurm_files/README.md | 121 + .../modules/slurm_files/etc/cgroup.conf.tpl | 7 + .../modules/slurm_files/etc/slurm.conf.tpl | 67 + .../modules/slurm_files/etc/slurmdbd.conf.tpl | 31 + .../slurm_files/files/external_epilog.sh | 18 + .../slurm_files/files/external_prolog.sh | 18 + .../slurm_files/files/setup_external.sh | 117 + .../modules/slurm_files/main.tf | 406 +++ .../modules/slurm_files/outputs.tf | 45 + .../modules/slurm_files/scripts/conf.py | 658 +++++ .../modules/slurm_files/scripts/file_cache.py | 80 + .../slurm_files/scripts/get_tpu_vmcount.py | 76 + .../slurm_files/scripts/job_submit.lua.tpl | 103 + .../modules/slurm_files/scripts/load_bq.py | 352 +++ .../slurm_files/scripts/local_pubsub.py | 196 ++ .../modules/slurm_files/scripts/mig_flex.py | 254 ++ .../slurm_files/scripts/requirements-dev.txt | 9 + .../slurm_files/scripts/requirements.txt | 18 + .../modules/slurm_files/scripts/resume.py | 703 ++++++ .../slurm_files/scripts/resume_wrapper.sh | 40 + .../modules/slurm_files/scripts/setup.py | 660 +++++ .../scripts/setup_network_storage.py | 327 +++ .../modules/slurm_files/scripts/slurmsync.py | 679 +++++ .../modules/slurm_files/scripts/sort_nodes.py | 171 ++ .../modules/slurm_files/scripts/suspend.py | 126 + .../slurm_files/scripts/suspend_wrapper.sh | 28 + .../slurm_files/scripts/tests/common.py | 116 + .../slurm_files/scripts/tests/test_conf.py | 226 ++ .../slurm_files/scripts/tests/test_resume.py | 175 ++ .../scripts/tests/test_topology.py | 215 ++ .../slurm_files/scripts/tests/test_util.py | 668 +++++ .../slurm_files/scripts/tools/gpu-test | 133 + .../slurm_files/scripts/tools/task-epilog | 67 + .../slurm_files/scripts/tools/task-prolog | 70 + .../modules/slurm_files/scripts/tpu.py | 331 +++ .../modules/slurm_files/scripts/util.py | 2224 +++++++++++++++++ .../slurm_files/scripts/watch_delete_vm_op.py | 124 + .../modules/slurm_files/variables.tf | 504 ++++ .../modules/slurm_files/versions.tf | 37 + .../outputs.tf | 62 + .../partition.tf | 174 ++ .../slurm_files.tf | 191 ++ .../source_image_logic.tf | 30 + .../variables.tf | 814 ++++++ .../variables_controller_instance.tf | 382 +++ .../versions.tf | 33 + .../schedmd-slurm-gcp-v6-login/README.md | 130 + .../schedmd-slurm-gcp-v6-login/main.tf | 115 + .../schedmd-slurm-gcp-v6-login/metadata.yaml | 21 + .../schedmd-slurm-gcp-v6-login/outputs.tf | 18 + .../source_image_logic.tf | 30 + .../schedmd-slurm-gcp-v6-login/variables.tf | 419 ++++ .../schedmd-slurm-gcp-v6-login/versions.tf | 23 + .../modules/scheduler/slinky/README.md | 172 ++ .../modules/scheduler/slinky/main.tf | 197 ++ .../modules/scheduler/slinky/metadata.yaml | 19 + .../modules/scheduler/slinky/outputs.tf | 23 + .../modules/scheduler/slinky/providers.tf | 23 + .../modules/scheduler/slinky/variables.tf | 127 + .../modules/scheduler/slinky/versions.tf | 28 + .../scripts/htcondor-install/README.md | 149 ++ .../htcondor-install/files/autoscaler.py | 417 ++++ .../install-htcondor-autoscaler-deps.yml | 46 + .../files/install-htcondor.yaml | 94 + .../modules/scripts/htcondor-install/main.tf | 51 + .../scripts/htcondor-install/metadata.yaml | 18 + .../scripts/htcondor-install/outputs.tf | 30 + .../templates/install-htcondor.ps1.tftpl | 59 + .../scripts/htcondor-install/variables.tf | 51 + .../scripts/htcondor-install/versions.tf | 19 + .../modules/scripts/ramble-execute/README.md | 116 + .../modules/scripts/ramble-execute/main.tf | 71 + .../scripts/ramble-execute/metadata.yaml | 18 + .../modules/scripts/ramble-execute/outputs.tf | 53 + .../templates/ramble_execute.yml.tpl | 59 + .../scripts/ramble-execute/variables.tf | 114 + .../scripts/ramble-execute/versions.tf | 25 + .../modules/scripts/ramble-setup/README.md | 128 + .../modules/scripts/ramble-setup/main.tf | 113 + .../scripts/ramble-setup/metadata.yaml | 18 + .../modules/scripts/ramble-setup/outputs.tf | 61 + .../scripts/install_ramble_deps.yml | 50 + .../install_ramble_python_deps.yml.tftpl | 28 + .../templates/ramble_setup.yml.tftpl | 157 ++ .../modules/scripts/ramble-setup/variables.tf | 97 + .../modules/scripts/ramble-setup/versions.tf | 30 + .../modules/scripts/spack-execute/README.md | 141 ++ .../modules/scripts/spack-execute/main.tf | 70 + .../scripts/spack-execute/metadata.yaml | 18 + .../modules/scripts/spack-execute/outputs.tf | 45 + .../templates/execute_commands.yml.tpl | 59 + .../scripts/spack-execute/variables.tf | 103 + .../modules/scripts/spack-execute/versions.tf | 25 + .../modules/scripts/spack-setup/README.md | 382 +++ .../modules/scripts/spack-setup/main.tf | 120 + .../modules/scripts/spack-setup/metadata.yaml | 19 + .../modules/scripts/spack-setup/outputs.tf | 56 + .../scripts/install_spack_deps.yml | 50 + .../templates/spack_setup.yml.tftpl | 157 ++ .../modules/scripts/spack-setup/variables.tf | 106 + .../modules/scripts/spack-setup/versions.tf | 30 + .../scripts/wait-for-startup/README.md | 87 + .../modules/scripts/wait-for-startup/main.tf | 47 + .../scripts/wait-for-startup/metadata.yaml | 19 + .../scripts/wait-for-startup/outputs.tf | 15 + .../scripts/wait-for-startup-status.sh | 138 + .../scripts/wait-for-startup/variables.tf | 54 + .../scripts/wait-for-startup/versions.tf | 29 + .../scripts/windows-startup-script/README.md | 109 + .../scripts/windows-startup-script/main.tf | 34 + .../windows-startup-script/metadata.yaml | 18 + .../scripts/windows-startup-script/outputs.tf | 20 + .../templates/install_gpu_driver.ps1.tftpl | 38 + .../templates/setx_http_proxy.ps1 | 21 + .../windows-startup-script/variables.tf | 54 + .../windows-startup-script/versions.tf | 23 + .../modules/embedded/modules/README.md | 554 ++++ .../compute/gke-job-template/README.md | 133 + .../modules/compute/gke-job-template/main.tf | 181 ++ .../compute/gke-job-template/metadata.yaml | 18 + .../compute/gke-job-template/outputs.tf | 27 + .../templates/gke-job-base.yaml.tftpl | 128 + .../compute/gke-job-template/variables.tf | 206 ++ .../compute/gke-job-template/versions.tf | 28 + .../modules/compute/gke-node-pool/README.md | 388 +++ .../compute/gke-node-pool/disk_definitions.tf | 38 + .../sample-tcpx-workload-job.yaml | 50 + .../sample-tcpxo-workload-job.yaml | 70 + .../scripts/enable-tcpx-in-workload.py | 185 ++ .../scripts/enable-tcpxo-in-workload.py | 186 ++ .../compute/gke-node-pool/gpu_direct.tf | 87 + .../compute/gke-node-pool/guest_cpus.tf | 32 + .../modules/compute/gke-node-pool/main.tf | 482 ++++ .../compute/gke-node-pool/metadata.yaml | 21 + .../modules/compute/gke-node-pool/outputs.tf | 152 ++ .../gke-node-pool/reservation_definitions.tf | 107 + .../gke-node-pool/threads_per_core_calc.tf | 42 + .../compute/gke-node-pool/variables.tf | 487 ++++ .../modules/compute/gke-node-pool/versions.tf | 38 + .../modules/compute/resource-policy/README.md | 82 + .../modules/compute/resource-policy/main.tf | 48 + .../compute/resource-policy/metadata.yaml | 19 + .../compute/resource-policy/outputs.tf | 30 + .../compute/resource-policy/variables.tf | 64 + .../compute/resource-policy/versions.tf | 34 + .../modules/compute/vm-instance/README.md | 257 ++ .../compute/vm-instance/compute_image.tf | 30 + .../modules/compute/vm-instance/main.tf | 334 +++ .../modules/compute/vm-instance/metadata.yaml | 19 + .../modules/compute/vm-instance/outputs.tf | 50 + .../startup_from_network_storage.tf | 65 + .../vm-instance/threads_per_core_calc.tf | 42 + .../modules/compute/vm-instance/variables.tf | 452 ++++ .../modules/compute/vm-instance/versions.tf | 41 + .../cloud-storage-bucket/README.md | 170 ++ .../file-system/cloud-storage-bucket/main.tf | 126 + .../cloud-storage-bucket/metadata.yaml | 18 + .../cloud-storage-bucket/outputs.tf | 69 + .../scripts/install-gcs-fuse.sh | 44 + .../cloud-storage-bucket/scripts/mount.sh | 58 + .../cloud-storage-bucket/variables.tf | 254 ++ .../cloud-storage-bucket/versions.tf | 39 + .../modules/file-system/filestore/README.md | 248 ++ .../modules/file-system/filestore/main.tf | 116 + .../file-system/filestore/metadata.yaml | 19 + .../modules/file-system/filestore/outputs.tf | 62 + .../filestore/scripts/install-nfs-client.sh | 37 + .../file-system/filestore/scripts/mount.sh | 58 + .../file-system/filestore/variables.tf | 189 ++ .../modules/file-system/filestore/versions.tf | 36 + .../gke-persistent-volume/README.md | 200 ++ .../file-system/gke-persistent-volume/main.tf | 155 ++ .../gke-persistent-volume/metadata.yaml | 18 + .../gke-persistent-volume/outputs.tf | 31 + .../templates/filestore-pv.yaml.tftpl | 26 + .../templates/filestore-pvc.yaml.tftpl | 18 + .../templates/gcs-pv.yaml.tftpl | 24 + .../templates/gcs-pvc.yaml.tftpl | 21 + .../templates/managed-lustre-pv.yaml.tftpl | 26 + .../templates/managed-lustre-pvc.yaml.tftpl | 18 + .../templates/namespace.yaml.tftpl | 5 + .../gke-persistent-volume/variables.tf | 93 + .../gke-persistent-volume/versions.tf | 30 + .../modules/file-system/gke-storage/README.md | 134 + .../modules/file-system/gke-storage/main.tf | 86 + .../file-system/gke-storage/metadata.yaml | 18 + .../file-system/gke-storage/outputs.tf | 28 + .../hyperdisk-balanced-pvc.yaml.tftpl | 17 + .../hyperdisk-extreme-pvc.yaml.tftpl | 17 + .../hyperdisk-throughput-pvc.yaml.tftpl | 17 + .../namespace.yaml.tftpl | 5 + .../parallelstore-pvc.yaml.tftpl | 17 + .../hyperdisk-balanced-sc.yaml.tftpl | 25 + .../hyperdisk-extreme-sc.yaml.tftpl | 24 + .../hyperdisk-throughput-sc.yaml.tftpl | 24 + .../storage-class/parallelstore-sc.yaml.tftpl | 21 + .../file-system/gke-storage/variables.tf | 144 ++ .../file-system/gke-storage/versions.tf | 21 + .../file-system/managed-lustre/README.md | 289 +++ .../file-system/managed-lustre/main.tf | 104 + .../file-system/managed-lustre/metadata.yaml | 19 + .../file-system/managed-lustre/outputs.tf | 43 + .../scripts/install-managed-lustre-client.sh | 84 + .../managed-lustre/scripts/mount.sh | 58 + .../file-system/managed-lustre/variables.tf | 131 + .../file-system/managed-lustre/versions.tf | 36 + .../file-system/netapp-storage-pool/README.md | 193 ++ .../file-system/netapp-storage-pool/main.tf | 56 + .../netapp-storage-pool/metadata.yaml | 20 + .../netapp-storage-pool/outputs.tf | 23 + .../netapp-storage-pool/variables.tf | 133 + .../netapp-storage-pool/versions.tf | 37 + .../file-system/netapp-volume/README.md | 201 ++ .../modules/file-system/netapp-volume/main.tf | 92 + .../file-system/netapp-volume/metadata.yaml | 19 + .../file-system/netapp-volume/outputs.tf | 66 + .../scripts/install-nfs-client.sh | 37 + .../netapp-volume/scripts/mount.sh | 66 + .../file-system/netapp-volume/variables.tf | 133 + .../file-system/netapp-volume/versions.tf | 32 + .../file-system/parallelstore/README.md | 196 ++ .../modules/file-system/parallelstore/main.tf | 74 + .../file-system/parallelstore/metadata.yaml | 19 + .../file-system/parallelstore/outputs.tf | 47 + .../scripts/install-daos-client.sh | 112 + .../templates/mount-daos.sh.tftpl | 110 + .../file-system/parallelstore/variables.tf | 137 + .../file-system/parallelstore/versions.tf | 36 + .../pre-existing-network-storage/README.md | 192 ++ .../metadata.yaml | 18 + .../pre-existing-network-storage/outputs.tf | 124 + .../scripts/install-daos-client.sh | 112 + .../scripts/install-gcs-fuse.sh | 44 + .../scripts/install-managed-lustre-client.sh | 84 + .../scripts/install-nfs-client.sh | 37 + .../scripts/mount.sh | 58 + .../ddn_exascaler_luster_client_install.tftpl | 50 + .../templates/mount-daos.sh.tftpl | 110 + .../pre-existing-network-storage/variables.tf | 67 + .../pre-existing-network-storage/versions.tf | 19 + .../modules/internal/gpu-definition/README.md | 47 + .../modules/internal/gpu-definition/main.tf | 98 + .../internal/instance_validations/README.md | 30 + .../internal/instance_validations/main.tf | 52 + .../instance_validations/variables.tf | 23 + .../internal/instance_validations/versions.tf | 17 + .../internal/network-attachment/README.md | 54 + .../internal/network-attachment/main.tf | 70 + .../internal/network-attachment/metadata.yaml | 19 + .../modules/internal/tpu-definition/README.md | 85 + .../modules/internal/tpu-definition/main.tf | 69 + .../internal/tpu-definition/outputs.tf | 40 + .../internal/tpu-definition/variables.tf | 29 + .../modules/internal/vpc_peering/README.md | 56 + .../modules/internal/vpc_peering/main.tf | 80 + .../internal/vpc_peering/metadata.yaml | 19 + .../management/kubectl-apply/README.md | 244 ++ .../kubectl-apply/helm_install/README.md | 64 + .../kubectl-apply/helm_install/main.tf | 79 + .../kubectl-apply/helm_install/metadata.yaml | 19 + .../kubectl-apply/helm_install/variables.tf | 212 ++ .../kubectl-apply/helm_install/versions.tf | 24 + .../jobset/jobset-helm-values.yaml | 25 + .../kubectl-apply/kubectl/README.md | 55 + .../management/kubectl-apply/kubectl/main.tf | 92 + .../kubectl-apply/kubectl/metadata.yaml | 19 + .../kubectl-apply/kubectl/variables.tf | 51 + .../kubectl-apply/kubectl/versions.tf | 26 + .../kueue/kueue-helm-values.yaml | 30 + .../modules/management/kubectl-apply/main.tf | 271 ++ .../management/kubectl-apply/metadata.yaml | 19 + .../management/kubectl-apply/providers.tf | 33 + .../management/kubectl-apply/variables.tf | 192 ++ .../management/kubectl-apply/versions.tf | 42 + .../modules/monitoring/dashboard/README.md | 86 + .../dashboard/dashboards/Empty.json.tpl | 17 + .../dashboard/dashboards/HPC.json.tpl | 595 +++++ .../modules/monitoring/dashboard/main.tf | 35 + .../monitoring/dashboard/metadata.yaml | 19 + .../modules/monitoring/dashboard/outputs.tf | 23 + .../modules/monitoring/dashboard/variables.tf | 52 + .../modules/monitoring/dashboard/versions.tf | 29 + .../modules/network/firewall-rules/README.md | 111 + .../modules/network/firewall-rules/main.tf | 60 + .../network/firewall-rules/metadata.yaml | 19 + .../network/firewall-rules/variables.tf | 88 + .../network/firewall-rules/versions.tf | 29 + .../modules/network/gpu-rdma-vpc/README.md | 143 ++ .../modules/network/gpu-rdma-vpc/main.tf | 79 + .../network/gpu-rdma-vpc/metadata.yaml | 19 + .../modules/network/gpu-rdma-vpc/outputs.tf | 59 + .../modules/network/gpu-rdma-vpc/variables.tf | 164 ++ .../modules/network/gpu-rdma-vpc/versions.tf | 19 + .../modules/network/multivpc/README.md | 136 + .../embedded/modules/network/multivpc/main.tf | 78 + .../modules/network/multivpc/metadata.yaml | 19 + .../modules/network/multivpc/outputs.tf | 50 + .../modules/network/multivpc/variables.tf | 201 ++ .../modules/network/multivpc/versions.tf | 19 + .../network/pre-existing-subnetwork/README.md | 94 + .../network/pre-existing-subnetwork/main.tf | 38 + .../pre-existing-subnetwork/metadata.yaml | 21 + .../pre-existing-subnetwork/outputs.tf | 35 + .../pre-existing-subnetwork/variables.tf | 39 + .../pre-existing-subnetwork/versions.tf | 29 + .../network/pre-existing-vpc/README.md | 110 + .../modules/network/pre-existing-vpc/main.tf | 53 + .../network/pre-existing-vpc/metadata.yaml | 19 + .../network/pre-existing-vpc/outputs.tf | 50 + .../network/pre-existing-vpc/variables.tf | 37 + .../network/pre-existing-vpc/versions.tf | 29 + .../embedded/modules/network/vpc/README.md | 237 ++ .../embedded/modules/network/vpc/main.tf | 256 ++ .../modules/network/vpc/metadata.yaml | 19 + .../embedded/modules/network/vpc/outputs.tf | 68 + .../embedded/modules/network/vpc/variables.tf | 301 +++ .../embedded/modules/network/vpc/versions.tf | 19 + .../modules/packer/custom-image/README.md | 320 +++ .../modules/packer/custom-image/image.pkr.hcl | 216 ++ .../modules/packer/custom-image/metadata.yaml | 21 + .../packer/custom-image/variables.pkr.hcl | 276 ++ .../packer/custom-image/versions.pkr.hcl | 25 + .../scheduler/batch-job-template/README.md | 197 ++ .../batch-job-template/compute_image.tf | 30 + .../scheduler/batch-job-template/main.tf | 149 ++ .../batch-job-template/metadata.yaml | 22 + .../scheduler/batch-job-template/outputs.tf | 80 + .../startup_from_network_storage.tf | 65 + .../templates/batch-job-base.yaml.tftpl | 53 + .../templates/batch-submit.sh.tftpl | 10 + .../scheduler/batch-job-template/variables.tf | 240 ++ .../scheduler/batch-job-template/versions.tf | 37 + .../scheduler/batch-login-node/README.md | 127 + .../scheduler/batch-login-node/main.tf | 127 + .../scheduler/batch-login-node/metadata.yaml | 21 + .../scheduler/batch-login-node/outputs.tf | 37 + .../scheduler/batch-login-node/variables.tf | 151 ++ .../scheduler/batch-login-node/versions.tf | 29 + .../modules/scheduler/gke-cluster/README.md | 220 ++ .../modules/scheduler/gke-cluster/main.tf | 470 ++++ .../scheduler/gke-cluster/metadata.yaml | 19 + .../modules/scheduler/gke-cluster/outputs.tf | 104 + .../templates/gke-network-paramset.yaml.tftpl | 9 + .../templates/network-object.yaml.tftpl | 11 + .../scheduler/gke-cluster/variables.tf | 533 ++++ .../modules/scheduler/gke-cluster/versions.tf | 39 + .../pre-existing-gke-cluster/README.md | 116 + .../pre-existing-gke-cluster/main.tf | 70 + .../pre-existing-gke-cluster/metadata.yaml | 19 + .../pre-existing-gke-cluster/outputs.tf | 33 + .../templates/gke-network-paramset.yaml.tftpl | 9 + .../templates/network-object.yaml.tftpl | 11 + .../pre-existing-gke-cluster/variables.tf | 61 + .../pre-existing-gke-cluster/versions.tf | 30 + .../modules/scripts/startup-script/README.md | 355 +++ .../startup-script/files/configure-ssh.yml | 37 + .../startup-script/files/configure_proxy.sh | 54 + .../files/early_run_hotfixes.sh | 32 + .../startup-script/files/get_from_bucket.sh | 73 + .../startup-script/files/install_ansible.sh | 247 ++ .../files/install_cloud_rdma_drivers.sh | 38 + .../startup-script/files/install_docker.yml | 113 + .../files/install_gpu_network_wait_online.yml | 56 + .../files/install_managed_lustre.yml | 33 + .../files/install_monitoring_agent.sh | 144 ++ .../files/running-script-warning.sh | 26 + .../startup-script/files/setup-raid.yml | 100 + .../startup-script/files/setup-ssh-keys.sh | 19 + .../startup-script/files/setup-ssh-keys.yml | 40 + .../files/startup-script-stdlib-body.sh | 39 + .../files/startup-script-stdlib-head.sh | 266 ++ .../modules/scripts/startup-script/main.tf | 306 +++ .../scripts/startup-script/metadata.yaml | 19 + .../modules/scripts/startup-script/outputs.tf | 39 + .../templates/startup-script-custom.tftpl | 65 + .../scripts/startup-script/variables.tf | 298 +++ .../scripts/startup-script/versions.tf | 37 + deletion-test/build_script/outputs.tf | 21 + deletion-test/build_script/providers.tf | 27 + deletion-test/build_script/variables.tf | 55 + deletion-test/build_script/versions.tf | 30 + deletion-test/cluster/main.tf | 227 ++ .../embedded/community/modules/README.md | 7 + .../modules/compute/gke-nodeset/README.md | 55 + .../modules/compute/gke-nodeset/main.tf | 64 + .../modules/compute/gke-nodeset/metadata.yaml | 20 + .../modules/compute/gke-nodeset/output.tf | 18 + .../compute/gke-nodeset/persistent_volumes.tf | 50 + .../templates/nodeset-general.yaml.tftpl | 203 ++ .../modules/compute/gke-nodeset/variables.tf | 118 + .../modules/compute/gke-nodeset/versions.tf | 27 + .../modules/compute/gke-partition/README.md | 39 + .../modules/compute/gke-partition/main.tf | 47 + .../compute/gke-partition/metadata.yaml | 19 + .../compute/gke-partition/variables.tf | 43 + .../modules/compute/gke-partition/versions.tf | 27 + .../compute/htcondor-execute-point/README.md | 271 ++ .../htcondor-execute-point/compute_image.tf | 30 + .../files/htcondor_configure.yml | 74 + .../files/htcondor_configure_autoscaler.yml | 98 + .../compute/htcondor-execute-point/main.tf | 218 ++ .../htcondor-execute-point/metadata.yaml | 20 + .../compute/htcondor-execute-point/outputs.tf | 25 + .../templates/condor_config.tftpl | 31 + .../download-condor-config.ps1.tftpl | 34 + .../htcondor-execute-point/variables.tf | 265 ++ .../htcondor-execute-point/versions.tf | 34 + .../community/modules/compute/mig/README.md | 45 + .../community/modules/compute/mig/main.tf | 85 + .../modules/compute/mig/metadata.yaml | 21 + .../community/modules/compute/mig/outputs.tf | 18 + .../modules/compute/mig/variables.tf | 86 + .../community/modules/compute/mig/versions.tf | 27 + .../modules/compute/notebook/README.md | 112 + .../modules/compute/notebook/main.tf | 96 + .../modules/compute/notebook/metadata.yaml | 20 + .../modules/compute/notebook/variables.tf | 111 + .../modules/compute/notebook/versions.tf | 29 + .../README.md | 135 + .../main.tf | 128 + .../metadata.yaml | 20 + .../outputs.tf | 36 + .../source_image_logic.tf | 30 + .../variables.tf | 402 +++ .../versions.tf | 22 + .../README.md | 85 + .../schedmd-slurm-gcp-v6-nodeset-tpu/main.tf | 59 + .../metadata.yaml | 21 + .../outputs.tf | 39 + .../variables.tf | 171 ++ .../versions.tf | 23 + .../schedmd-slurm-gcp-v6-nodeset/README.md | 227 ++ .../schedmd-slurm-gcp-v6-nodeset/main.tf | 232 ++ .../metadata.yaml | 21 + .../schedmd-slurm-gcp-v6-nodeset/outputs.tf | 112 + .../source_image_logic.tf | 30 + .../schedmd-slurm-gcp-v6-nodeset/variables.tf | 641 +++++ .../schedmd-slurm-gcp-v6-nodeset/versions.tf | 29 + .../schedmd-slurm-gcp-v6-partition/README.md | 105 + .../schedmd-slurm-gcp-v6-partition/main.tf | 41 + .../metadata.yaml | 20 + .../schedmd-slurm-gcp-v6-partition/outputs.tf | 54 + .../variables.tf | 311 +++ .../versions.tf | 23 + .../container/artifact-registry/README.md | 157 ++ .../container/artifact-registry/main.tf | 268 ++ .../container/artifact-registry/metadata.yaml | 21 + .../container/artifact-registry/outputs.tf | 18 + .../container/artifact-registry/validation.tf | 49 + .../container/artifact-registry/variables.tf | 122 + .../container/artifact-registry/versions.tf | 27 + .../database/bigquery-dataset/README.md | 76 + .../modules/database/bigquery-dataset/main.tf | 32 + .../database/bigquery-dataset/metadata.yaml | 19 + .../database/bigquery-dataset/outputs.tf | 20 + .../database/bigquery-dataset/variables.tf | 36 + .../database/bigquery-dataset/versions.tf | 29 + .../modules/database/bigquery-table/README.md | 87 + .../modules/database/bigquery-table/main.tf | 37 + .../database/bigquery-table/metadata.yaml | 19 + .../database/bigquery-table/outputs.tf | 28 + .../database/bigquery-table/variables.tf | 46 + .../database/bigquery-table/versions.tf | 29 + .../slurm-cloudsql-federation/README.md | 107 + .../slurm-cloudsql-federation/main.tf | 165 ++ .../slurm-cloudsql-federation/metadata.yaml | 21 + .../slurm-cloudsql-federation/outputs.tf | 27 + .../slurm-cloudsql-federation/variables.tf | 173 ++ .../slurm-cloudsql-federation/versions.tf | 36 + .../file-system/DDN-EXAScaler/README.md | 158 ++ .../modules/file-system/DDN-EXAScaler/main.tf | 72 + .../file-system/DDN-EXAScaler/metadata.yaml | 22 + .../file-system/DDN-EXAScaler/outputs.tf | 90 + .../file-system/DDN-EXAScaler/variables.tf | 502 ++++ .../file-system/DDN-EXAScaler/versions.tf | 24 + .../modules/file-system/Intel-DAOS/README.md | 1 + .../modules/file-system/nfs-server/README.md | 152 ++ .../modules/file-system/nfs-server/main.tf | 131 + .../file-system/nfs-server/metadata.yaml | 19 + .../modules/file-system/nfs-server/outputs.tf | 53 + .../nfs-server/scripts/install-nfs-client.sh | 37 + .../scripts/install-nfs-server.sh.tpl | 35 + .../file-system/nfs-server/scripts/mount.sh | 58 + .../file-system/nfs-server/scripts/mount.yaml | 39 + .../file-system/nfs-server/variables.tf | 194 ++ .../file-system/nfs-server/versions.tf | 37 + .../file-system/sycomp-scale/README.md | 35 + .../modules/file-system/weka-client/README.md | 182 ++ .../file-system/weka-client/metadata.yaml | 18 + .../file-system/weka-client/outputs.tf | 71 + .../templates/install-weka-client.yaml.tftpl | 133 + .../weka-client/templates/mount-weka.sh.tftpl | 101 + .../templates/mount-weka.yaml.tftpl | 54 + .../file-system/weka-client/variables.tf | 39 + .../file-system/weka-client/versions.tf | 19 + .../FSI_MonteCarlo.ipynb | 125 + .../files/fsi-montecarlo-on-batch/README.md | 97 + .../fsi-montecarlo-on-batch/iteration.sh | 23 + .../files/fsi-montecarlo-on-batch/main.tf | 102 + .../fsi-montecarlo-on-batch/mc_run.tpl.py | 157 ++ .../fsi-montecarlo-on-batch/mc_run.tpl.yaml | 36 + .../fsi-montecarlo-on-batch/mc_run_reqs.txt | 9 + .../fsi-montecarlo-on-batch/metadata.yaml | 18 + .../fsi-montecarlo-on-batch/variables.tf | 51 + .../files/fsi-montecarlo-on-batch/versions.tf | 43 + .../internal/slurm-gcp/instance/README.md | 100 + .../internal/slurm-gcp/instance/main.tf | 126 + .../internal/slurm-gcp/instance/outputs.tf | 41 + .../internal/slurm-gcp/instance/variables.tf | 119 + .../internal/slurm-gcp/instance/versions.tf | 31 + .../slurm-gcp/instance_template/README.md | 87 + .../files/startup_sh_unlinted | 169 ++ .../slurm-gcp/instance_template/main.tf | 171 ++ .../slurm-gcp/instance_template/outputs.tf | 43 + .../slurm-gcp/instance_template/variables.tf | 431 ++++ .../slurm-gcp/instance_template/versions.tf | 25 + .../internal_instance_template/README.md | 89 + .../internal_instance_template/main.tf | 234 ++ .../internal_instance_template/outputs.tf | 33 + .../internal_instance_template/variables.tf | 398 +++ .../internal_instance_template/versions.tf | 30 + .../internal/slurm-gcp/login/README.md | 52 + .../modules/internal/slurm-gcp/login/main.tf | 112 + .../internal/slurm-gcp/login/outputs.tf | 25 + .../internal/slurm-gcp/login/variables.tf | 188 ++ .../internal/slurm-gcp/login/versions.tf | 29 + .../internal/slurm-gcp/nodeset_tpu/README.md | 95 + .../internal/slurm-gcp/nodeset_tpu/main.tf | 121 + .../internal/slurm-gcp/nodeset_tpu/outputs.tf | 30 + .../slurm-gcp/nodeset_tpu/variables.tf | 158 ++ .../slurm-gcp/nodeset_tpu/versions.tf | 30 + .../dependencies-installer/README.md | 61 + .../helm_install/README.md | 64 + .../helm_install/main.tf | 75 + .../helm_install/metadata.yaml | 19 + .../helm_install/variables.tf | 212 ++ .../helm_install/versions.tf | 24 + .../kubernetes_manifest/README.md | 40 + .../kubernetes_manifest/main.tf | 104 + .../kubernetes_manifest/metadata.yaml | 19 + .../kubernetes_manifest/variables.tf | 69 + .../kubernetes_manifest/versions.tf | 24 + .../management/dependencies-installer/main.tf | 183 ++ .../dependencies-installer/metadata.yaml | 19 + .../dependencies-installer/providers.tf | 25 + .../dependencies-installer/variables.tf | 70 + .../dependencies-installer/versions.tf | 30 + .../network/private-service-access/README.md | 122 + .../network/private-service-access/main.tf | 61 + .../private-service-access/metadata.yaml | 20 + .../network/private-service-access/outputs.tf | 43 + .../private-service-access/variables.tf | 59 + .../private-service-access/versions.tf | 37 + .../modules/project/new-project/README.md | 128 + .../modules/project/service-account/README.md | 111 + .../modules/project/service-account/main.tf | 37 + .../project/service-account/metadata.yaml | 19 + .../project/service-account/outputs.tf | 36 + .../project/service-account/variables.tf | 113 + .../project/service-account/versions.tf | 22 + .../project/service-enablement/README.md | 70 + .../project/service-enablement/main.tf | 28 + .../project/service-enablement/metadata.yaml | 19 + .../project/service-enablement/variables.tf | 31 + .../project/service-enablement/versions.tf | 29 + .../modules/pubsub/bigquery-sub/README.md | 87 + .../modules/pubsub/bigquery-sub/main.tf | 57 + .../modules/pubsub/bigquery-sub/metadata.yaml | 19 + .../modules/pubsub/bigquery-sub/outputs.tf | 20 + .../modules/pubsub/bigquery-sub/variables.tf | 51 + .../modules/pubsub/bigquery-sub/versions.tf | 35 + .../community/modules/pubsub/topic/README.md | 82 + .../community/modules/pubsub/topic/main.tf | 48 + .../modules/pubsub/topic/metadata.yaml | 19 + .../community/modules/pubsub/topic/outputs.tf | 26 + .../modules/pubsub/topic/variables.tf | 74 + .../modules/pubsub/topic/versions.tf | 32 + .../chrome-remote-desktop/README.md | 113 + .../chrome-remote-desktop/main.tf | 111 + .../chrome-remote-desktop/metadata.yaml | 18 + .../chrome-remote-desktop/outputs.tf | 25 + .../scripts/configure-chrome-desktop.yml | 61 + .../scripts/configure-grid-drivers.yml | 163 ++ .../scripts/disable-sleep.yml | 39 + .../chrome-remote-desktop/variables.tf | 277 ++ .../chrome-remote-desktop/versions.tf | 19 + .../scheduler/htcondor-access-point/README.md | 187 ++ .../files/htcondor_configure.yml | 120 + .../scheduler/htcondor-access-point/main.tf | 338 +++ .../htcondor-access-point/metadata.yaml | 20 + .../htcondor-access-point/outputs.tf | 25 + .../templates/condor_config.tftpl | 70 + .../htcondor-access-point/variables.tf | 266 ++ .../htcondor-access-point/versions.tf | 37 + .../htcondor-central-manager/README.md | 159 ++ .../files/htcondor_configure.yml | 72 + .../htcondor-central-manager/main.tf | 226 ++ .../htcondor-central-manager/metadata.yaml | 20 + .../htcondor-central-manager/outputs.tf | 30 + .../templates/condor_config.tftpl | 31 + .../htcondor-central-manager/variables.tf | 192 ++ .../htcondor-central-manager/versions.tf | 33 + .../scheduler/htcondor-pool-secrets/README.md | 172 ++ .../files/htcondor_secrets.yml | 102 + .../scheduler/htcondor-pool-secrets/main.tf | 168 ++ .../htcondor-pool-secrets/metadata.yaml | 20 + .../htcondor-pool-secrets/outputs.tf | 50 + .../templates/fetch-idtoken.ps1.tftpl | 26 + .../htcondor-pool-secrets/variables.tf | 67 + .../htcondor-pool-secrets/versions.tf | 33 + .../htcondor-service-accounts/README.md | 128 + .../htcondor-service-accounts/main.tf | 51 + .../htcondor-service-accounts/metadata.yaml | 19 + .../htcondor-service-accounts/outputs.tf | 30 + .../htcondor-service-accounts/variables.tf | 56 + .../htcondor-service-accounts/versions.tf | 19 + .../scheduler/htcondor-setup/README.md | 118 + .../modules/scheduler/htcondor-setup/main.tf | 68 + .../scheduler/htcondor-setup/metadata.yaml | 21 + .../scheduler/htcondor-setup/outputs.tf | 27 + .../scheduler/htcondor-setup/variables.tf | 55 + .../scheduler/htcondor-setup/versions.tf | 19 + .../schedmd-slurm-gcp-v6-controller/README.md | 405 +++ .../controller.tf | 213 ++ .../etc/htc-slurm.conf.tpl | 65 + .../etc/htc-slurmdbd.conf.tpl | 34 + .../etc/long-prolog-slurm.conf.tpl | 71 + .../schedmd-slurm-gcp-v6-controller/login.tf | 50 + .../schedmd-slurm-gcp-v6-controller/main.tf | 35 + .../metadata.yaml | 21 + .../modules/cleanup_compute/README.md | 42 + .../modules/cleanup_compute/main.tf | 46 + .../scripts/cleanup_compute.sh | 100 + .../modules/cleanup_compute/variables.tf | 71 + .../modules/cleanup_compute/versions.tf | 27 + .../modules/cleanup_tpu/README.md | 79 + .../modules/cleanup_tpu/main.tf | 32 + .../cleanup_tpu/scripts/cleanup_tpu.sh | 63 + .../modules/cleanup_tpu/variables.tf | 60 + .../modules/cleanup_tpu/versions.tf | 27 + .../modules/slurm_files/README.md | 121 + .../build/slurm-gcp-devel-controller.zip | Bin 0 -> 84485 bytes .../slurm_files/build/slurm-gcp-devel.zip | Bin 0 -> 63549 bytes .../modules/slurm_files/etc/cgroup.conf.tpl | 7 + .../modules/slurm_files/etc/slurm.conf.tpl | 67 + .../modules/slurm_files/etc/slurmdbd.conf.tpl | 31 + .../slurm_files/files/external_epilog.sh | 18 + .../slurm_files/files/external_prolog.sh | 18 + .../slurm_files/files/setup_external.sh | 117 + .../modules/slurm_files/main.tf | 406 +++ .../modules/slurm_files/outputs.tf | 45 + .../modules/slurm_files/scripts/conf.py | 658 +++++ .../modules/slurm_files/scripts/file_cache.py | 80 + .../slurm_files/scripts/get_tpu_vmcount.py | 76 + .../slurm_files/scripts/job_submit.lua.tpl | 103 + .../modules/slurm_files/scripts/load_bq.py | 352 +++ .../slurm_files/scripts/local_pubsub.py | 196 ++ .../modules/slurm_files/scripts/mig_flex.py | 254 ++ .../slurm_files/scripts/requirements-dev.txt | 9 + .../slurm_files/scripts/requirements.txt | 18 + .../modules/slurm_files/scripts/resume.py | 703 ++++++ .../slurm_files/scripts/resume_wrapper.sh | 40 + .../modules/slurm_files/scripts/setup.py | 660 +++++ .../scripts/setup_network_storage.py | 327 +++ .../modules/slurm_files/scripts/slurmsync.py | 679 +++++ .../modules/slurm_files/scripts/sort_nodes.py | 171 ++ .../modules/slurm_files/scripts/suspend.py | 126 + .../slurm_files/scripts/suspend_wrapper.sh | 28 + .../slurm_files/scripts/tests/common.py | 116 + .../slurm_files/scripts/tests/test_conf.py | 226 ++ .../slurm_files/scripts/tests/test_resume.py | 175 ++ .../scripts/tests/test_topology.py | 215 ++ .../slurm_files/scripts/tests/test_util.py | 668 +++++ .../slurm_files/scripts/tools/gpu-test | 133 + .../slurm_files/scripts/tools/task-epilog | 67 + .../slurm_files/scripts/tools/task-prolog | 70 + .../modules/slurm_files/scripts/tpu.py | 331 +++ .../modules/slurm_files/scripts/util.py | 2224 +++++++++++++++++ .../slurm_files/scripts/watch_delete_vm_op.py | 124 + .../modules/slurm_files/variables.tf | 504 ++++ .../modules/slurm_files/versions.tf | 37 + .../outputs.tf | 62 + .../partition.tf | 174 ++ .../slurm_files.tf | 191 ++ .../source_image_logic.tf | 30 + .../variables.tf | 814 ++++++ .../variables_controller_instance.tf | 382 +++ .../versions.tf | 33 + .../schedmd-slurm-gcp-v6-login/README.md | 130 + .../schedmd-slurm-gcp-v6-login/main.tf | 115 + .../schedmd-slurm-gcp-v6-login/metadata.yaml | 21 + .../schedmd-slurm-gcp-v6-login/outputs.tf | 18 + .../source_image_logic.tf | 30 + .../schedmd-slurm-gcp-v6-login/variables.tf | 419 ++++ .../schedmd-slurm-gcp-v6-login/versions.tf | 23 + .../modules/scheduler/slinky/README.md | 172 ++ .../modules/scheduler/slinky/main.tf | 197 ++ .../modules/scheduler/slinky/metadata.yaml | 19 + .../modules/scheduler/slinky/outputs.tf | 23 + .../modules/scheduler/slinky/providers.tf | 23 + .../modules/scheduler/slinky/variables.tf | 127 + .../modules/scheduler/slinky/versions.tf | 28 + .../scripts/htcondor-install/README.md | 149 ++ .../htcondor-install/files/autoscaler.py | 417 ++++ .../install-htcondor-autoscaler-deps.yml | 46 + .../files/install-htcondor.yaml | 94 + .../modules/scripts/htcondor-install/main.tf | 51 + .../scripts/htcondor-install/metadata.yaml | 18 + .../scripts/htcondor-install/outputs.tf | 30 + .../templates/install-htcondor.ps1.tftpl | 59 + .../scripts/htcondor-install/variables.tf | 51 + .../scripts/htcondor-install/versions.tf | 19 + .../modules/scripts/ramble-execute/README.md | 116 + .../modules/scripts/ramble-execute/main.tf | 71 + .../scripts/ramble-execute/metadata.yaml | 18 + .../modules/scripts/ramble-execute/outputs.tf | 53 + .../templates/ramble_execute.yml.tpl | 59 + .../scripts/ramble-execute/variables.tf | 114 + .../scripts/ramble-execute/versions.tf | 25 + .../modules/scripts/ramble-setup/README.md | 128 + .../modules/scripts/ramble-setup/main.tf | 113 + .../scripts/ramble-setup/metadata.yaml | 18 + .../modules/scripts/ramble-setup/outputs.tf | 61 + .../scripts/install_ramble_deps.yml | 50 + .../install_ramble_python_deps.yml.tftpl | 28 + .../templates/ramble_setup.yml.tftpl | 157 ++ .../modules/scripts/ramble-setup/variables.tf | 97 + .../modules/scripts/ramble-setup/versions.tf | 30 + .../modules/scripts/spack-execute/README.md | 141 ++ .../modules/scripts/spack-execute/main.tf | 70 + .../scripts/spack-execute/metadata.yaml | 18 + .../modules/scripts/spack-execute/outputs.tf | 45 + .../templates/execute_commands.yml.tpl | 59 + .../scripts/spack-execute/variables.tf | 103 + .../modules/scripts/spack-execute/versions.tf | 25 + .../modules/scripts/spack-setup/README.md | 382 +++ .../modules/scripts/spack-setup/main.tf | 120 + .../modules/scripts/spack-setup/metadata.yaml | 19 + .../modules/scripts/spack-setup/outputs.tf | 56 + .../scripts/install_spack_deps.yml | 50 + .../templates/spack_setup.yml.tftpl | 157 ++ .../modules/scripts/spack-setup/variables.tf | 106 + .../modules/scripts/spack-setup/versions.tf | 30 + .../scripts/wait-for-startup/README.md | 87 + .../modules/scripts/wait-for-startup/main.tf | 47 + .../scripts/wait-for-startup/metadata.yaml | 19 + .../scripts/wait-for-startup/outputs.tf | 15 + .../scripts/wait-for-startup-status.sh | 138 + .../scripts/wait-for-startup/variables.tf | 54 + .../scripts/wait-for-startup/versions.tf | 29 + .../scripts/windows-startup-script/README.md | 109 + .../scripts/windows-startup-script/main.tf | 34 + .../windows-startup-script/metadata.yaml | 18 + .../scripts/windows-startup-script/outputs.tf | 20 + .../templates/install_gpu_driver.ps1.tftpl | 38 + .../templates/setx_http_proxy.ps1 | 21 + .../windows-startup-script/variables.tf | 54 + .../windows-startup-script/versions.tf | 23 + .../modules/embedded/modules/README.md | 554 ++++ .../compute/gke-job-template/README.md | 133 + .../modules/compute/gke-job-template/main.tf | 181 ++ .../compute/gke-job-template/metadata.yaml | 18 + .../compute/gke-job-template/outputs.tf | 27 + .../templates/gke-job-base.yaml.tftpl | 128 + .../compute/gke-job-template/variables.tf | 206 ++ .../compute/gke-job-template/versions.tf | 28 + .../modules/compute/gke-node-pool/README.md | 388 +++ .../compute/gke-node-pool/disk_definitions.tf | 38 + .../sample-tcpx-workload-job.yaml | 50 + .../sample-tcpxo-workload-job.yaml | 70 + .../scripts/enable-tcpx-in-workload.py | 185 ++ .../scripts/enable-tcpxo-in-workload.py | 186 ++ .../compute/gke-node-pool/gpu_direct.tf | 87 + .../compute/gke-node-pool/guest_cpus.tf | 32 + .../modules/compute/gke-node-pool/main.tf | 482 ++++ .../compute/gke-node-pool/metadata.yaml | 21 + .../modules/compute/gke-node-pool/outputs.tf | 152 ++ .../gke-node-pool/reservation_definitions.tf | 107 + .../gke-node-pool/threads_per_core_calc.tf | 42 + .../compute/gke-node-pool/variables.tf | 487 ++++ .../modules/compute/gke-node-pool/versions.tf | 38 + .../modules/compute/resource-policy/README.md | 82 + .../modules/compute/resource-policy/main.tf | 48 + .../compute/resource-policy/metadata.yaml | 19 + .../compute/resource-policy/outputs.tf | 30 + .../compute/resource-policy/variables.tf | 64 + .../compute/resource-policy/versions.tf | 34 + .../modules/compute/vm-instance/README.md | 257 ++ .../compute/vm-instance/compute_image.tf | 30 + .../modules/compute/vm-instance/main.tf | 334 +++ .../modules/compute/vm-instance/metadata.yaml | 19 + .../modules/compute/vm-instance/outputs.tf | 50 + .../startup_from_network_storage.tf | 65 + .../vm-instance/threads_per_core_calc.tf | 42 + .../modules/compute/vm-instance/variables.tf | 452 ++++ .../modules/compute/vm-instance/versions.tf | 41 + .../cloud-storage-bucket/README.md | 170 ++ .../file-system/cloud-storage-bucket/main.tf | 126 + .../cloud-storage-bucket/metadata.yaml | 18 + .../cloud-storage-bucket/outputs.tf | 69 + .../scripts/install-gcs-fuse.sh | 44 + .../cloud-storage-bucket/scripts/mount.sh | 58 + .../cloud-storage-bucket/variables.tf | 254 ++ .../cloud-storage-bucket/versions.tf | 39 + .../modules/file-system/filestore/README.md | 248 ++ .../modules/file-system/filestore/main.tf | 116 + .../file-system/filestore/metadata.yaml | 19 + .../modules/file-system/filestore/outputs.tf | 62 + .../filestore/scripts/install-nfs-client.sh | 37 + .../file-system/filestore/scripts/mount.sh | 58 + .../file-system/filestore/variables.tf | 189 ++ .../modules/file-system/filestore/versions.tf | 36 + .../gke-persistent-volume/README.md | 200 ++ .../file-system/gke-persistent-volume/main.tf | 155 ++ .../gke-persistent-volume/metadata.yaml | 18 + .../gke-persistent-volume/outputs.tf | 31 + .../templates/filestore-pv.yaml.tftpl | 26 + .../templates/filestore-pvc.yaml.tftpl | 18 + .../templates/gcs-pv.yaml.tftpl | 24 + .../templates/gcs-pvc.yaml.tftpl | 21 + .../templates/managed-lustre-pv.yaml.tftpl | 26 + .../templates/managed-lustre-pvc.yaml.tftpl | 18 + .../templates/namespace.yaml.tftpl | 5 + .../gke-persistent-volume/variables.tf | 93 + .../gke-persistent-volume/versions.tf | 30 + .../modules/file-system/gke-storage/README.md | 134 + .../modules/file-system/gke-storage/main.tf | 86 + .../file-system/gke-storage/metadata.yaml | 18 + .../file-system/gke-storage/outputs.tf | 28 + .../hyperdisk-balanced-pvc.yaml.tftpl | 17 + .../hyperdisk-extreme-pvc.yaml.tftpl | 17 + .../hyperdisk-throughput-pvc.yaml.tftpl | 17 + .../namespace.yaml.tftpl | 5 + .../parallelstore-pvc.yaml.tftpl | 17 + .../hyperdisk-balanced-sc.yaml.tftpl | 25 + .../hyperdisk-extreme-sc.yaml.tftpl | 24 + .../hyperdisk-throughput-sc.yaml.tftpl | 24 + .../storage-class/parallelstore-sc.yaml.tftpl | 21 + .../file-system/gke-storage/variables.tf | 144 ++ .../file-system/gke-storage/versions.tf | 21 + .../file-system/managed-lustre/README.md | 289 +++ .../file-system/managed-lustre/main.tf | 104 + .../file-system/managed-lustre/metadata.yaml | 19 + .../file-system/managed-lustre/outputs.tf | 43 + .../scripts/install-managed-lustre-client.sh | 84 + .../managed-lustre/scripts/mount.sh | 58 + .../file-system/managed-lustre/variables.tf | 131 + .../file-system/managed-lustre/versions.tf | 36 + .../file-system/netapp-storage-pool/README.md | 193 ++ .../file-system/netapp-storage-pool/main.tf | 56 + .../netapp-storage-pool/metadata.yaml | 20 + .../netapp-storage-pool/outputs.tf | 23 + .../netapp-storage-pool/variables.tf | 133 + .../netapp-storage-pool/versions.tf | 37 + .../file-system/netapp-volume/README.md | 201 ++ .../modules/file-system/netapp-volume/main.tf | 92 + .../file-system/netapp-volume/metadata.yaml | 19 + .../file-system/netapp-volume/outputs.tf | 66 + .../scripts/install-nfs-client.sh | 37 + .../netapp-volume/scripts/mount.sh | 66 + .../file-system/netapp-volume/variables.tf | 133 + .../file-system/netapp-volume/versions.tf | 32 + .../file-system/parallelstore/README.md | 196 ++ .../modules/file-system/parallelstore/main.tf | 74 + .../file-system/parallelstore/metadata.yaml | 19 + .../file-system/parallelstore/outputs.tf | 47 + .../scripts/install-daos-client.sh | 112 + .../templates/mount-daos.sh.tftpl | 110 + .../file-system/parallelstore/variables.tf | 137 + .../file-system/parallelstore/versions.tf | 36 + .../pre-existing-network-storage/README.md | 192 ++ .../metadata.yaml | 18 + .../pre-existing-network-storage/outputs.tf | 124 + .../scripts/install-daos-client.sh | 112 + .../scripts/install-gcs-fuse.sh | 44 + .../scripts/install-managed-lustre-client.sh | 84 + .../scripts/install-nfs-client.sh | 37 + .../scripts/mount.sh | 58 + .../ddn_exascaler_luster_client_install.tftpl | 50 + .../templates/mount-daos.sh.tftpl | 110 + .../pre-existing-network-storage/variables.tf | 67 + .../pre-existing-network-storage/versions.tf | 19 + .../modules/internal/gpu-definition/README.md | 47 + .../modules/internal/gpu-definition/main.tf | 98 + .../internal/instance_validations/README.md | 30 + .../internal/instance_validations/main.tf | 52 + .../instance_validations/variables.tf | 23 + .../internal/instance_validations/versions.tf | 17 + .../internal/network-attachment/README.md | 54 + .../internal/network-attachment/main.tf | 70 + .../internal/network-attachment/metadata.yaml | 19 + .../modules/internal/tpu-definition/README.md | 85 + .../modules/internal/tpu-definition/main.tf | 69 + .../internal/tpu-definition/outputs.tf | 40 + .../internal/tpu-definition/variables.tf | 29 + .../modules/internal/vpc_peering/README.md | 56 + .../modules/internal/vpc_peering/main.tf | 80 + .../internal/vpc_peering/metadata.yaml | 19 + .../management/kubectl-apply/README.md | 244 ++ .../kubectl-apply/helm_install/README.md | 64 + .../kubectl-apply/helm_install/main.tf | 79 + .../kubectl-apply/helm_install/metadata.yaml | 19 + .../kubectl-apply/helm_install/variables.tf | 212 ++ .../kubectl-apply/helm_install/versions.tf | 24 + .../jobset/jobset-helm-values.yaml | 25 + .../kubectl-apply/kubectl/README.md | 55 + .../management/kubectl-apply/kubectl/main.tf | 92 + .../kubectl-apply/kubectl/metadata.yaml | 19 + .../kubectl-apply/kubectl/variables.tf | 51 + .../kubectl-apply/kubectl/versions.tf | 26 + .../kueue/kueue-helm-values.yaml | 30 + .../modules/management/kubectl-apply/main.tf | 271 ++ .../management/kubectl-apply/metadata.yaml | 19 + .../management/kubectl-apply/providers.tf | 33 + .../management/kubectl-apply/variables.tf | 192 ++ .../management/kubectl-apply/versions.tf | 42 + .../modules/monitoring/dashboard/README.md | 86 + .../dashboard/dashboards/Empty.json.tpl | 17 + .../dashboard/dashboards/HPC.json.tpl | 595 +++++ .../modules/monitoring/dashboard/main.tf | 35 + .../monitoring/dashboard/metadata.yaml | 19 + .../modules/monitoring/dashboard/outputs.tf | 23 + .../modules/monitoring/dashboard/variables.tf | 52 + .../modules/monitoring/dashboard/versions.tf | 29 + .../modules/network/firewall-rules/README.md | 111 + .../modules/network/firewall-rules/main.tf | 60 + .../network/firewall-rules/metadata.yaml | 19 + .../network/firewall-rules/variables.tf | 88 + .../network/firewall-rules/versions.tf | 29 + .../modules/network/gpu-rdma-vpc/README.md | 143 ++ .../modules/network/gpu-rdma-vpc/main.tf | 79 + .../network/gpu-rdma-vpc/metadata.yaml | 19 + .../modules/network/gpu-rdma-vpc/outputs.tf | 59 + .../modules/network/gpu-rdma-vpc/variables.tf | 164 ++ .../modules/network/gpu-rdma-vpc/versions.tf | 19 + .../modules/network/multivpc/README.md | 136 + .../embedded/modules/network/multivpc/main.tf | 78 + .../modules/network/multivpc/metadata.yaml | 19 + .../modules/network/multivpc/outputs.tf | 50 + .../modules/network/multivpc/variables.tf | 201 ++ .../modules/network/multivpc/versions.tf | 19 + .../network/pre-existing-subnetwork/README.md | 94 + .../network/pre-existing-subnetwork/main.tf | 38 + .../pre-existing-subnetwork/metadata.yaml | 21 + .../pre-existing-subnetwork/outputs.tf | 35 + .../pre-existing-subnetwork/variables.tf | 39 + .../pre-existing-subnetwork/versions.tf | 29 + .../network/pre-existing-vpc/README.md | 110 + .../modules/network/pre-existing-vpc/main.tf | 53 + .../network/pre-existing-vpc/metadata.yaml | 19 + .../network/pre-existing-vpc/outputs.tf | 50 + .../network/pre-existing-vpc/variables.tf | 37 + .../network/pre-existing-vpc/versions.tf | 29 + .../embedded/modules/network/vpc/README.md | 237 ++ .../embedded/modules/network/vpc/main.tf | 256 ++ .../modules/network/vpc/metadata.yaml | 19 + .../embedded/modules/network/vpc/outputs.tf | 68 + .../embedded/modules/network/vpc/variables.tf | 301 +++ .../embedded/modules/network/vpc/versions.tf | 19 + .../modules/packer/custom-image/README.md | 320 +++ .../modules/packer/custom-image/image.pkr.hcl | 216 ++ .../modules/packer/custom-image/metadata.yaml | 21 + .../packer/custom-image/variables.pkr.hcl | 276 ++ .../packer/custom-image/versions.pkr.hcl | 25 + .../scheduler/batch-job-template/README.md | 197 ++ .../batch-job-template/compute_image.tf | 30 + .../scheduler/batch-job-template/main.tf | 149 ++ .../batch-job-template/metadata.yaml | 22 + .../scheduler/batch-job-template/outputs.tf | 80 + .../startup_from_network_storage.tf | 65 + .../templates/batch-job-base.yaml.tftpl | 53 + .../templates/batch-submit.sh.tftpl | 10 + .../scheduler/batch-job-template/variables.tf | 240 ++ .../scheduler/batch-job-template/versions.tf | 37 + .../scheduler/batch-login-node/README.md | 127 + .../scheduler/batch-login-node/main.tf | 127 + .../scheduler/batch-login-node/metadata.yaml | 21 + .../scheduler/batch-login-node/outputs.tf | 37 + .../scheduler/batch-login-node/variables.tf | 151 ++ .../scheduler/batch-login-node/versions.tf | 29 + .../modules/scheduler/gke-cluster/README.md | 220 ++ .../modules/scheduler/gke-cluster/main.tf | 470 ++++ .../scheduler/gke-cluster/metadata.yaml | 19 + .../modules/scheduler/gke-cluster/outputs.tf | 104 + .../templates/gke-network-paramset.yaml.tftpl | 9 + .../templates/network-object.yaml.tftpl | 11 + .../scheduler/gke-cluster/variables.tf | 533 ++++ .../modules/scheduler/gke-cluster/versions.tf | 39 + .../pre-existing-gke-cluster/README.md | 116 + .../pre-existing-gke-cluster/main.tf | 70 + .../pre-existing-gke-cluster/metadata.yaml | 19 + .../pre-existing-gke-cluster/outputs.tf | 33 + .../templates/gke-network-paramset.yaml.tftpl | 9 + .../templates/network-object.yaml.tftpl | 11 + .../pre-existing-gke-cluster/variables.tf | 61 + .../pre-existing-gke-cluster/versions.tf | 30 + .../modules/scripts/startup-script/README.md | 355 +++ .../startup-script/files/configure-ssh.yml | 37 + .../startup-script/files/configure_proxy.sh | 54 + .../files/early_run_hotfixes.sh | 32 + .../startup-script/files/get_from_bucket.sh | 73 + .../startup-script/files/install_ansible.sh | 247 ++ .../files/install_cloud_rdma_drivers.sh | 38 + .../startup-script/files/install_docker.yml | 113 + .../files/install_gpu_network_wait_online.yml | 56 + .../files/install_managed_lustre.yml | 33 + .../files/install_monitoring_agent.sh | 144 ++ .../files/running-script-warning.sh | 26 + .../startup-script/files/setup-raid.yml | 100 + .../startup-script/files/setup-ssh-keys.sh | 19 + .../startup-script/files/setup-ssh-keys.yml | 40 + .../files/startup-script-stdlib-body.sh | 39 + .../files/startup-script-stdlib-head.sh | 266 ++ .../modules/scripts/startup-script/main.tf | 306 +++ .../scripts/startup-script/metadata.yaml | 19 + .../modules/scripts/startup-script/outputs.tf | 39 + .../templates/startup-script-custom.tftpl | 65 + .../scripts/startup-script/variables.tf | 298 +++ .../scripts/startup-script/versions.tf | 37 + deletion-test/cluster/outputs.tf | 20 + deletion-test/cluster/providers.tf | 27 + deletion-test/cluster/variables.tf | 120 + deletion-test/cluster/versions.tf | 30 + deletion-test/instructions.txt | 60 + deletion-test/primary/main.tf | 40 + .../embedded/community/modules/README.md | 7 + .../modules/compute/gke-nodeset/README.md | 55 + .../modules/compute/gke-nodeset/main.tf | 64 + .../modules/compute/gke-nodeset/metadata.yaml | 20 + .../modules/compute/gke-nodeset/output.tf | 18 + .../compute/gke-nodeset/persistent_volumes.tf | 50 + .../templates/nodeset-general.yaml.tftpl | 203 ++ .../modules/compute/gke-nodeset/variables.tf | 118 + .../modules/compute/gke-nodeset/versions.tf | 27 + .../modules/compute/gke-partition/README.md | 39 + .../modules/compute/gke-partition/main.tf | 47 + .../compute/gke-partition/metadata.yaml | 19 + .../compute/gke-partition/variables.tf | 43 + .../modules/compute/gke-partition/versions.tf | 27 + .../compute/htcondor-execute-point/README.md | 271 ++ .../htcondor-execute-point/compute_image.tf | 30 + .../files/htcondor_configure.yml | 74 + .../files/htcondor_configure_autoscaler.yml | 98 + .../compute/htcondor-execute-point/main.tf | 218 ++ .../htcondor-execute-point/metadata.yaml | 20 + .../compute/htcondor-execute-point/outputs.tf | 25 + .../templates/condor_config.tftpl | 31 + .../download-condor-config.ps1.tftpl | 34 + .../htcondor-execute-point/variables.tf | 265 ++ .../htcondor-execute-point/versions.tf | 34 + .../community/modules/compute/mig/README.md | 45 + .../community/modules/compute/mig/main.tf | 85 + .../modules/compute/mig/metadata.yaml | 21 + .../community/modules/compute/mig/outputs.tf | 18 + .../modules/compute/mig/variables.tf | 86 + .../community/modules/compute/mig/versions.tf | 27 + .../modules/compute/notebook/README.md | 112 + .../modules/compute/notebook/main.tf | 96 + .../modules/compute/notebook/metadata.yaml | 20 + .../modules/compute/notebook/variables.tf | 111 + .../modules/compute/notebook/versions.tf | 29 + .../README.md | 135 + .../main.tf | 128 + .../metadata.yaml | 20 + .../outputs.tf | 36 + .../source_image_logic.tf | 30 + .../variables.tf | 402 +++ .../versions.tf | 22 + .../README.md | 85 + .../schedmd-slurm-gcp-v6-nodeset-tpu/main.tf | 59 + .../metadata.yaml | 21 + .../outputs.tf | 39 + .../variables.tf | 171 ++ .../versions.tf | 23 + .../schedmd-slurm-gcp-v6-nodeset/README.md | 227 ++ .../schedmd-slurm-gcp-v6-nodeset/main.tf | 232 ++ .../metadata.yaml | 21 + .../schedmd-slurm-gcp-v6-nodeset/outputs.tf | 112 + .../source_image_logic.tf | 30 + .../schedmd-slurm-gcp-v6-nodeset/variables.tf | 641 +++++ .../schedmd-slurm-gcp-v6-nodeset/versions.tf | 29 + .../schedmd-slurm-gcp-v6-partition/README.md | 105 + .../schedmd-slurm-gcp-v6-partition/main.tf | 41 + .../metadata.yaml | 20 + .../schedmd-slurm-gcp-v6-partition/outputs.tf | 54 + .../variables.tf | 311 +++ .../versions.tf | 23 + .../container/artifact-registry/README.md | 157 ++ .../container/artifact-registry/main.tf | 268 ++ .../container/artifact-registry/metadata.yaml | 21 + .../container/artifact-registry/outputs.tf | 18 + .../container/artifact-registry/validation.tf | 49 + .../container/artifact-registry/variables.tf | 122 + .../container/artifact-registry/versions.tf | 27 + .../database/bigquery-dataset/README.md | 76 + .../modules/database/bigquery-dataset/main.tf | 32 + .../database/bigquery-dataset/metadata.yaml | 19 + .../database/bigquery-dataset/outputs.tf | 20 + .../database/bigquery-dataset/variables.tf | 36 + .../database/bigquery-dataset/versions.tf | 29 + .../modules/database/bigquery-table/README.md | 87 + .../modules/database/bigquery-table/main.tf | 37 + .../database/bigquery-table/metadata.yaml | 19 + .../database/bigquery-table/outputs.tf | 28 + .../database/bigquery-table/variables.tf | 46 + .../database/bigquery-table/versions.tf | 29 + .../slurm-cloudsql-federation/README.md | 107 + .../slurm-cloudsql-federation/main.tf | 165 ++ .../slurm-cloudsql-federation/metadata.yaml | 21 + .../slurm-cloudsql-federation/outputs.tf | 27 + .../slurm-cloudsql-federation/variables.tf | 173 ++ .../slurm-cloudsql-federation/versions.tf | 36 + .../file-system/DDN-EXAScaler/README.md | 158 ++ .../modules/file-system/DDN-EXAScaler/main.tf | 72 + .../file-system/DDN-EXAScaler/metadata.yaml | 22 + .../file-system/DDN-EXAScaler/outputs.tf | 90 + .../file-system/DDN-EXAScaler/variables.tf | 502 ++++ .../file-system/DDN-EXAScaler/versions.tf | 24 + .../modules/file-system/Intel-DAOS/README.md | 1 + .../modules/file-system/nfs-server/README.md | 152 ++ .../modules/file-system/nfs-server/main.tf | 131 + .../file-system/nfs-server/metadata.yaml | 19 + .../modules/file-system/nfs-server/outputs.tf | 53 + .../nfs-server/scripts/install-nfs-client.sh | 37 + .../scripts/install-nfs-server.sh.tpl | 35 + .../file-system/nfs-server/scripts/mount.sh | 58 + .../file-system/nfs-server/scripts/mount.yaml | 39 + .../file-system/nfs-server/variables.tf | 194 ++ .../file-system/nfs-server/versions.tf | 37 + .../file-system/sycomp-scale/README.md | 35 + .../modules/file-system/weka-client/README.md | 182 ++ .../file-system/weka-client/metadata.yaml | 18 + .../file-system/weka-client/outputs.tf | 71 + .../templates/install-weka-client.yaml.tftpl | 133 + .../weka-client/templates/mount-weka.sh.tftpl | 101 + .../templates/mount-weka.yaml.tftpl | 54 + .../file-system/weka-client/variables.tf | 39 + .../file-system/weka-client/versions.tf | 19 + .../FSI_MonteCarlo.ipynb | 125 + .../files/fsi-montecarlo-on-batch/README.md | 97 + .../fsi-montecarlo-on-batch/iteration.sh | 23 + .../files/fsi-montecarlo-on-batch/main.tf | 102 + .../fsi-montecarlo-on-batch/mc_run.tpl.py | 157 ++ .../fsi-montecarlo-on-batch/mc_run.tpl.yaml | 36 + .../fsi-montecarlo-on-batch/mc_run_reqs.txt | 9 + .../fsi-montecarlo-on-batch/metadata.yaml | 18 + .../fsi-montecarlo-on-batch/variables.tf | 51 + .../files/fsi-montecarlo-on-batch/versions.tf | 43 + .../internal/slurm-gcp/instance/README.md | 100 + .../internal/slurm-gcp/instance/main.tf | 126 + .../internal/slurm-gcp/instance/outputs.tf | 41 + .../internal/slurm-gcp/instance/variables.tf | 119 + .../internal/slurm-gcp/instance/versions.tf | 31 + .../slurm-gcp/instance_template/README.md | 87 + .../files/startup_sh_unlinted | 169 ++ .../slurm-gcp/instance_template/main.tf | 171 ++ .../slurm-gcp/instance_template/outputs.tf | 43 + .../slurm-gcp/instance_template/variables.tf | 431 ++++ .../slurm-gcp/instance_template/versions.tf | 25 + .../internal_instance_template/README.md | 89 + .../internal_instance_template/main.tf | 234 ++ .../internal_instance_template/outputs.tf | 33 + .../internal_instance_template/variables.tf | 398 +++ .../internal_instance_template/versions.tf | 30 + .../internal/slurm-gcp/login/README.md | 52 + .../modules/internal/slurm-gcp/login/main.tf | 112 + .../internal/slurm-gcp/login/outputs.tf | 25 + .../internal/slurm-gcp/login/variables.tf | 188 ++ .../internal/slurm-gcp/login/versions.tf | 29 + .../internal/slurm-gcp/nodeset_tpu/README.md | 95 + .../internal/slurm-gcp/nodeset_tpu/main.tf | 121 + .../internal/slurm-gcp/nodeset_tpu/outputs.tf | 30 + .../slurm-gcp/nodeset_tpu/variables.tf | 158 ++ .../slurm-gcp/nodeset_tpu/versions.tf | 30 + .../dependencies-installer/README.md | 61 + .../helm_install/README.md | 64 + .../helm_install/main.tf | 75 + .../helm_install/metadata.yaml | 19 + .../helm_install/variables.tf | 212 ++ .../helm_install/versions.tf | 24 + .../kubernetes_manifest/README.md | 40 + .../kubernetes_manifest/main.tf | 104 + .../kubernetes_manifest/metadata.yaml | 19 + .../kubernetes_manifest/variables.tf | 69 + .../kubernetes_manifest/versions.tf | 24 + .../management/dependencies-installer/main.tf | 183 ++ .../dependencies-installer/metadata.yaml | 19 + .../dependencies-installer/providers.tf | 25 + .../dependencies-installer/variables.tf | 70 + .../dependencies-installer/versions.tf | 30 + .../network/private-service-access/README.md | 122 + .../network/private-service-access/main.tf | 61 + .../private-service-access/metadata.yaml | 20 + .../network/private-service-access/outputs.tf | 43 + .../private-service-access/variables.tf | 59 + .../private-service-access/versions.tf | 37 + .../modules/project/new-project/README.md | 128 + .../modules/project/service-account/README.md | 111 + .../modules/project/service-account/main.tf | 37 + .../project/service-account/metadata.yaml | 19 + .../project/service-account/outputs.tf | 36 + .../project/service-account/variables.tf | 113 + .../project/service-account/versions.tf | 22 + .../project/service-enablement/README.md | 70 + .../project/service-enablement/main.tf | 28 + .../project/service-enablement/metadata.yaml | 19 + .../project/service-enablement/variables.tf | 31 + .../project/service-enablement/versions.tf | 29 + .../modules/pubsub/bigquery-sub/README.md | 87 + .../modules/pubsub/bigquery-sub/main.tf | 57 + .../modules/pubsub/bigquery-sub/metadata.yaml | 19 + .../modules/pubsub/bigquery-sub/outputs.tf | 20 + .../modules/pubsub/bigquery-sub/variables.tf | 51 + .../modules/pubsub/bigquery-sub/versions.tf | 35 + .../community/modules/pubsub/topic/README.md | 82 + .../community/modules/pubsub/topic/main.tf | 48 + .../modules/pubsub/topic/metadata.yaml | 19 + .../community/modules/pubsub/topic/outputs.tf | 26 + .../modules/pubsub/topic/variables.tf | 74 + .../modules/pubsub/topic/versions.tf | 32 + .../chrome-remote-desktop/README.md | 113 + .../chrome-remote-desktop/main.tf | 111 + .../chrome-remote-desktop/metadata.yaml | 18 + .../chrome-remote-desktop/outputs.tf | 25 + .../scripts/configure-chrome-desktop.yml | 61 + .../scripts/configure-grid-drivers.yml | 163 ++ .../scripts/disable-sleep.yml | 39 + .../chrome-remote-desktop/variables.tf | 277 ++ .../chrome-remote-desktop/versions.tf | 19 + .../scheduler/htcondor-access-point/README.md | 187 ++ .../files/htcondor_configure.yml | 120 + .../scheduler/htcondor-access-point/main.tf | 338 +++ .../htcondor-access-point/metadata.yaml | 20 + .../htcondor-access-point/outputs.tf | 25 + .../templates/condor_config.tftpl | 70 + .../htcondor-access-point/variables.tf | 266 ++ .../htcondor-access-point/versions.tf | 37 + .../htcondor-central-manager/README.md | 159 ++ .../files/htcondor_configure.yml | 72 + .../htcondor-central-manager/main.tf | 226 ++ .../htcondor-central-manager/metadata.yaml | 20 + .../htcondor-central-manager/outputs.tf | 30 + .../templates/condor_config.tftpl | 31 + .../htcondor-central-manager/variables.tf | 192 ++ .../htcondor-central-manager/versions.tf | 33 + .../scheduler/htcondor-pool-secrets/README.md | 172 ++ .../files/htcondor_secrets.yml | 102 + .../scheduler/htcondor-pool-secrets/main.tf | 168 ++ .../htcondor-pool-secrets/metadata.yaml | 20 + .../htcondor-pool-secrets/outputs.tf | 50 + .../templates/fetch-idtoken.ps1.tftpl | 26 + .../htcondor-pool-secrets/variables.tf | 67 + .../htcondor-pool-secrets/versions.tf | 33 + .../htcondor-service-accounts/README.md | 128 + .../htcondor-service-accounts/main.tf | 51 + .../htcondor-service-accounts/metadata.yaml | 19 + .../htcondor-service-accounts/outputs.tf | 30 + .../htcondor-service-accounts/variables.tf | 56 + .../htcondor-service-accounts/versions.tf | 19 + .../scheduler/htcondor-setup/README.md | 118 + .../modules/scheduler/htcondor-setup/main.tf | 68 + .../scheduler/htcondor-setup/metadata.yaml | 21 + .../scheduler/htcondor-setup/outputs.tf | 27 + .../scheduler/htcondor-setup/variables.tf | 55 + .../scheduler/htcondor-setup/versions.tf | 19 + .../schedmd-slurm-gcp-v6-controller/README.md | 405 +++ .../controller.tf | 213 ++ .../etc/htc-slurm.conf.tpl | 65 + .../etc/htc-slurmdbd.conf.tpl | 34 + .../etc/long-prolog-slurm.conf.tpl | 71 + .../schedmd-slurm-gcp-v6-controller/login.tf | 50 + .../schedmd-slurm-gcp-v6-controller/main.tf | 35 + .../metadata.yaml | 21 + .../modules/cleanup_compute/README.md | 42 + .../modules/cleanup_compute/main.tf | 46 + .../scripts/cleanup_compute.sh | 100 + .../modules/cleanup_compute/variables.tf | 71 + .../modules/cleanup_compute/versions.tf | 27 + .../modules/cleanup_tpu/README.md | 79 + .../modules/cleanup_tpu/main.tf | 32 + .../cleanup_tpu/scripts/cleanup_tpu.sh | 63 + .../modules/cleanup_tpu/variables.tf | 60 + .../modules/cleanup_tpu/versions.tf | 27 + .../modules/slurm_files/README.md | 121 + .../modules/slurm_files/etc/cgroup.conf.tpl | 7 + .../modules/slurm_files/etc/slurm.conf.tpl | 67 + .../modules/slurm_files/etc/slurmdbd.conf.tpl | 31 + .../slurm_files/files/external_epilog.sh | 18 + .../slurm_files/files/external_prolog.sh | 18 + .../slurm_files/files/setup_external.sh | 117 + .../modules/slurm_files/main.tf | 406 +++ .../modules/slurm_files/outputs.tf | 45 + .../modules/slurm_files/scripts/conf.py | 658 +++++ .../modules/slurm_files/scripts/file_cache.py | 80 + .../slurm_files/scripts/get_tpu_vmcount.py | 76 + .../slurm_files/scripts/job_submit.lua.tpl | 103 + .../modules/slurm_files/scripts/load_bq.py | 352 +++ .../slurm_files/scripts/local_pubsub.py | 196 ++ .../modules/slurm_files/scripts/mig_flex.py | 254 ++ .../slurm_files/scripts/requirements-dev.txt | 9 + .../slurm_files/scripts/requirements.txt | 18 + .../modules/slurm_files/scripts/resume.py | 703 ++++++ .../slurm_files/scripts/resume_wrapper.sh | 40 + .../modules/slurm_files/scripts/setup.py | 660 +++++ .../scripts/setup_network_storage.py | 327 +++ .../modules/slurm_files/scripts/slurmsync.py | 679 +++++ .../modules/slurm_files/scripts/sort_nodes.py | 171 ++ .../modules/slurm_files/scripts/suspend.py | 126 + .../slurm_files/scripts/suspend_wrapper.sh | 28 + .../slurm_files/scripts/tests/common.py | 116 + .../slurm_files/scripts/tests/test_conf.py | 226 ++ .../slurm_files/scripts/tests/test_resume.py | 175 ++ .../scripts/tests/test_topology.py | 215 ++ .../slurm_files/scripts/tests/test_util.py | 668 +++++ .../slurm_files/scripts/tools/gpu-test | 133 + .../slurm_files/scripts/tools/task-epilog | 67 + .../slurm_files/scripts/tools/task-prolog | 70 + .../modules/slurm_files/scripts/tpu.py | 331 +++ .../modules/slurm_files/scripts/util.py | 2224 +++++++++++++++++ .../slurm_files/scripts/watch_delete_vm_op.py | 124 + .../modules/slurm_files/variables.tf | 504 ++++ .../modules/slurm_files/versions.tf | 37 + .../outputs.tf | 62 + .../partition.tf | 174 ++ .../slurm_files.tf | 191 ++ .../source_image_logic.tf | 30 + .../variables.tf | 814 ++++++ .../variables_controller_instance.tf | 382 +++ .../versions.tf | 33 + .../schedmd-slurm-gcp-v6-login/README.md | 130 + .../schedmd-slurm-gcp-v6-login/main.tf | 115 + .../schedmd-slurm-gcp-v6-login/metadata.yaml | 21 + .../schedmd-slurm-gcp-v6-login/outputs.tf | 18 + .../source_image_logic.tf | 30 + .../schedmd-slurm-gcp-v6-login/variables.tf | 419 ++++ .../schedmd-slurm-gcp-v6-login/versions.tf | 23 + .../modules/scheduler/slinky/README.md | 172 ++ .../modules/scheduler/slinky/main.tf | 197 ++ .../modules/scheduler/slinky/metadata.yaml | 19 + .../modules/scheduler/slinky/outputs.tf | 23 + .../modules/scheduler/slinky/providers.tf | 23 + .../modules/scheduler/slinky/variables.tf | 127 + .../modules/scheduler/slinky/versions.tf | 28 + .../scripts/htcondor-install/README.md | 149 ++ .../htcondor-install/files/autoscaler.py | 417 ++++ .../install-htcondor-autoscaler-deps.yml | 46 + .../files/install-htcondor.yaml | 94 + .../modules/scripts/htcondor-install/main.tf | 51 + .../scripts/htcondor-install/metadata.yaml | 18 + .../scripts/htcondor-install/outputs.tf | 30 + .../templates/install-htcondor.ps1.tftpl | 59 + .../scripts/htcondor-install/variables.tf | 51 + .../scripts/htcondor-install/versions.tf | 19 + .../modules/scripts/ramble-execute/README.md | 116 + .../modules/scripts/ramble-execute/main.tf | 71 + .../scripts/ramble-execute/metadata.yaml | 18 + .../modules/scripts/ramble-execute/outputs.tf | 53 + .../templates/ramble_execute.yml.tpl | 59 + .../scripts/ramble-execute/variables.tf | 114 + .../scripts/ramble-execute/versions.tf | 25 + .../modules/scripts/ramble-setup/README.md | 128 + .../modules/scripts/ramble-setup/main.tf | 113 + .../scripts/ramble-setup/metadata.yaml | 18 + .../modules/scripts/ramble-setup/outputs.tf | 61 + .../scripts/install_ramble_deps.yml | 50 + .../install_ramble_python_deps.yml.tftpl | 28 + .../templates/ramble_setup.yml.tftpl | 157 ++ .../modules/scripts/ramble-setup/variables.tf | 97 + .../modules/scripts/ramble-setup/versions.tf | 30 + .../modules/scripts/spack-execute/README.md | 141 ++ .../modules/scripts/spack-execute/main.tf | 70 + .../scripts/spack-execute/metadata.yaml | 18 + .../modules/scripts/spack-execute/outputs.tf | 45 + .../templates/execute_commands.yml.tpl | 59 + .../scripts/spack-execute/variables.tf | 103 + .../modules/scripts/spack-execute/versions.tf | 25 + .../modules/scripts/spack-setup/README.md | 382 +++ .../modules/scripts/spack-setup/main.tf | 120 + .../modules/scripts/spack-setup/metadata.yaml | 19 + .../modules/scripts/spack-setup/outputs.tf | 56 + .../scripts/install_spack_deps.yml | 50 + .../templates/spack_setup.yml.tftpl | 157 ++ .../modules/scripts/spack-setup/variables.tf | 106 + .../modules/scripts/spack-setup/versions.tf | 30 + .../scripts/wait-for-startup/README.md | 87 + .../modules/scripts/wait-for-startup/main.tf | 47 + .../scripts/wait-for-startup/metadata.yaml | 19 + .../scripts/wait-for-startup/outputs.tf | 15 + .../scripts/wait-for-startup-status.sh | 138 + .../scripts/wait-for-startup/variables.tf | 54 + .../scripts/wait-for-startup/versions.tf | 29 + .../scripts/windows-startup-script/README.md | 109 + .../scripts/windows-startup-script/main.tf | 34 + .../windows-startup-script/metadata.yaml | 18 + .../scripts/windows-startup-script/outputs.tf | 20 + .../templates/install_gpu_driver.ps1.tftpl | 38 + .../templates/setx_http_proxy.ps1 | 21 + .../windows-startup-script/variables.tf | 54 + .../windows-startup-script/versions.tf | 23 + .../modules/embedded/modules/README.md | 554 ++++ .../compute/gke-job-template/README.md | 133 + .../modules/compute/gke-job-template/main.tf | 181 ++ .../compute/gke-job-template/metadata.yaml | 18 + .../compute/gke-job-template/outputs.tf | 27 + .../templates/gke-job-base.yaml.tftpl | 128 + .../compute/gke-job-template/variables.tf | 206 ++ .../compute/gke-job-template/versions.tf | 28 + .../modules/compute/gke-node-pool/README.md | 388 +++ .../compute/gke-node-pool/disk_definitions.tf | 38 + .../sample-tcpx-workload-job.yaml | 50 + .../sample-tcpxo-workload-job.yaml | 70 + .../scripts/enable-tcpx-in-workload.py | 185 ++ .../scripts/enable-tcpxo-in-workload.py | 186 ++ .../compute/gke-node-pool/gpu_direct.tf | 87 + .../compute/gke-node-pool/guest_cpus.tf | 32 + .../modules/compute/gke-node-pool/main.tf | 482 ++++ .../compute/gke-node-pool/metadata.yaml | 21 + .../modules/compute/gke-node-pool/outputs.tf | 152 ++ .../gke-node-pool/reservation_definitions.tf | 107 + .../gke-node-pool/threads_per_core_calc.tf | 42 + .../compute/gke-node-pool/variables.tf | 487 ++++ .../modules/compute/gke-node-pool/versions.tf | 38 + .../modules/compute/resource-policy/README.md | 82 + .../modules/compute/resource-policy/main.tf | 48 + .../compute/resource-policy/metadata.yaml | 19 + .../compute/resource-policy/outputs.tf | 30 + .../compute/resource-policy/variables.tf | 64 + .../compute/resource-policy/versions.tf | 34 + .../modules/compute/vm-instance/README.md | 257 ++ .../compute/vm-instance/compute_image.tf | 30 + .../modules/compute/vm-instance/main.tf | 334 +++ .../modules/compute/vm-instance/metadata.yaml | 19 + .../modules/compute/vm-instance/outputs.tf | 50 + .../startup_from_network_storage.tf | 65 + .../vm-instance/threads_per_core_calc.tf | 42 + .../modules/compute/vm-instance/variables.tf | 452 ++++ .../modules/compute/vm-instance/versions.tf | 41 + .../cloud-storage-bucket/README.md | 170 ++ .../file-system/cloud-storage-bucket/main.tf | 126 + .../cloud-storage-bucket/metadata.yaml | 18 + .../cloud-storage-bucket/outputs.tf | 69 + .../scripts/install-gcs-fuse.sh | 44 + .../cloud-storage-bucket/scripts/mount.sh | 58 + .../cloud-storage-bucket/variables.tf | 254 ++ .../cloud-storage-bucket/versions.tf | 39 + .../modules/file-system/filestore/README.md | 248 ++ .../modules/file-system/filestore/main.tf | 116 + .../file-system/filestore/metadata.yaml | 19 + .../modules/file-system/filestore/outputs.tf | 62 + .../filestore/scripts/install-nfs-client.sh | 37 + .../file-system/filestore/scripts/mount.sh | 58 + .../file-system/filestore/variables.tf | 189 ++ .../modules/file-system/filestore/versions.tf | 36 + .../gke-persistent-volume/README.md | 200 ++ .../file-system/gke-persistent-volume/main.tf | 155 ++ .../gke-persistent-volume/metadata.yaml | 18 + .../gke-persistent-volume/outputs.tf | 31 + .../templates/filestore-pv.yaml.tftpl | 26 + .../templates/filestore-pvc.yaml.tftpl | 18 + .../templates/gcs-pv.yaml.tftpl | 24 + .../templates/gcs-pvc.yaml.tftpl | 21 + .../templates/managed-lustre-pv.yaml.tftpl | 26 + .../templates/managed-lustre-pvc.yaml.tftpl | 18 + .../templates/namespace.yaml.tftpl | 5 + .../gke-persistent-volume/variables.tf | 93 + .../gke-persistent-volume/versions.tf | 30 + .../modules/file-system/gke-storage/README.md | 134 + .../modules/file-system/gke-storage/main.tf | 86 + .../file-system/gke-storage/metadata.yaml | 18 + .../file-system/gke-storage/outputs.tf | 28 + .../hyperdisk-balanced-pvc.yaml.tftpl | 17 + .../hyperdisk-extreme-pvc.yaml.tftpl | 17 + .../hyperdisk-throughput-pvc.yaml.tftpl | 17 + .../namespace.yaml.tftpl | 5 + .../parallelstore-pvc.yaml.tftpl | 17 + .../hyperdisk-balanced-sc.yaml.tftpl | 25 + .../hyperdisk-extreme-sc.yaml.tftpl | 24 + .../hyperdisk-throughput-sc.yaml.tftpl | 24 + .../storage-class/parallelstore-sc.yaml.tftpl | 21 + .../file-system/gke-storage/variables.tf | 144 ++ .../file-system/gke-storage/versions.tf | 21 + .../file-system/managed-lustre/README.md | 289 +++ .../file-system/managed-lustre/main.tf | 104 + .../file-system/managed-lustre/metadata.yaml | 19 + .../file-system/managed-lustre/outputs.tf | 43 + .../scripts/install-managed-lustre-client.sh | 84 + .../managed-lustre/scripts/mount.sh | 58 + .../file-system/managed-lustre/variables.tf | 131 + .../file-system/managed-lustre/versions.tf | 36 + .../file-system/netapp-storage-pool/README.md | 193 ++ .../file-system/netapp-storage-pool/main.tf | 56 + .../netapp-storage-pool/metadata.yaml | 20 + .../netapp-storage-pool/outputs.tf | 23 + .../netapp-storage-pool/variables.tf | 133 + .../netapp-storage-pool/versions.tf | 37 + .../file-system/netapp-volume/README.md | 201 ++ .../modules/file-system/netapp-volume/main.tf | 92 + .../file-system/netapp-volume/metadata.yaml | 19 + .../file-system/netapp-volume/outputs.tf | 66 + .../scripts/install-nfs-client.sh | 37 + .../netapp-volume/scripts/mount.sh | 66 + .../file-system/netapp-volume/variables.tf | 133 + .../file-system/netapp-volume/versions.tf | 32 + .../file-system/parallelstore/README.md | 196 ++ .../modules/file-system/parallelstore/main.tf | 74 + .../file-system/parallelstore/metadata.yaml | 19 + .../file-system/parallelstore/outputs.tf | 47 + .../scripts/install-daos-client.sh | 112 + .../templates/mount-daos.sh.tftpl | 110 + .../file-system/parallelstore/variables.tf | 137 + .../file-system/parallelstore/versions.tf | 36 + .../pre-existing-network-storage/README.md | 192 ++ .../metadata.yaml | 18 + .../pre-existing-network-storage/outputs.tf | 124 + .../scripts/install-daos-client.sh | 112 + .../scripts/install-gcs-fuse.sh | 44 + .../scripts/install-managed-lustre-client.sh | 84 + .../scripts/install-nfs-client.sh | 37 + .../scripts/mount.sh | 58 + .../ddn_exascaler_luster_client_install.tftpl | 50 + .../templates/mount-daos.sh.tftpl | 110 + .../pre-existing-network-storage/variables.tf | 67 + .../pre-existing-network-storage/versions.tf | 19 + .../modules/internal/gpu-definition/README.md | 47 + .../modules/internal/gpu-definition/main.tf | 98 + .../internal/instance_validations/README.md | 30 + .../internal/instance_validations/main.tf | 52 + .../instance_validations/variables.tf | 23 + .../internal/instance_validations/versions.tf | 17 + .../internal/network-attachment/README.md | 54 + .../internal/network-attachment/main.tf | 70 + .../internal/network-attachment/metadata.yaml | 19 + .../modules/internal/tpu-definition/README.md | 85 + .../modules/internal/tpu-definition/main.tf | 69 + .../internal/tpu-definition/outputs.tf | 40 + .../internal/tpu-definition/variables.tf | 29 + .../modules/internal/vpc_peering/README.md | 56 + .../modules/internal/vpc_peering/main.tf | 80 + .../internal/vpc_peering/metadata.yaml | 19 + .../management/kubectl-apply/README.md | 244 ++ .../kubectl-apply/helm_install/README.md | 64 + .../kubectl-apply/helm_install/main.tf | 79 + .../kubectl-apply/helm_install/metadata.yaml | 19 + .../kubectl-apply/helm_install/variables.tf | 212 ++ .../kubectl-apply/helm_install/versions.tf | 24 + .../jobset/jobset-helm-values.yaml | 25 + .../kubectl-apply/kubectl/README.md | 55 + .../management/kubectl-apply/kubectl/main.tf | 92 + .../kubectl-apply/kubectl/metadata.yaml | 19 + .../kubectl-apply/kubectl/variables.tf | 51 + .../kubectl-apply/kubectl/versions.tf | 26 + .../kueue/kueue-helm-values.yaml | 30 + .../modules/management/kubectl-apply/main.tf | 271 ++ .../management/kubectl-apply/metadata.yaml | 19 + .../management/kubectl-apply/providers.tf | 33 + .../management/kubectl-apply/variables.tf | 192 ++ .../management/kubectl-apply/versions.tf | 42 + .../modules/monitoring/dashboard/README.md | 86 + .../dashboard/dashboards/Empty.json.tpl | 17 + .../dashboard/dashboards/HPC.json.tpl | 595 +++++ .../modules/monitoring/dashboard/main.tf | 35 + .../monitoring/dashboard/metadata.yaml | 19 + .../modules/monitoring/dashboard/outputs.tf | 23 + .../modules/monitoring/dashboard/variables.tf | 52 + .../modules/monitoring/dashboard/versions.tf | 29 + .../modules/network/firewall-rules/README.md | 111 + .../modules/network/firewall-rules/main.tf | 60 + .../network/firewall-rules/metadata.yaml | 19 + .../network/firewall-rules/variables.tf | 88 + .../network/firewall-rules/versions.tf | 29 + .../modules/network/gpu-rdma-vpc/README.md | 143 ++ .../modules/network/gpu-rdma-vpc/main.tf | 79 + .../network/gpu-rdma-vpc/metadata.yaml | 19 + .../modules/network/gpu-rdma-vpc/outputs.tf | 59 + .../modules/network/gpu-rdma-vpc/variables.tf | 164 ++ .../modules/network/gpu-rdma-vpc/versions.tf | 19 + .../modules/network/multivpc/README.md | 136 + .../embedded/modules/network/multivpc/main.tf | 78 + .../modules/network/multivpc/metadata.yaml | 19 + .../modules/network/multivpc/outputs.tf | 50 + .../modules/network/multivpc/variables.tf | 201 ++ .../modules/network/multivpc/versions.tf | 19 + .../network/pre-existing-subnetwork/README.md | 94 + .../network/pre-existing-subnetwork/main.tf | 38 + .../pre-existing-subnetwork/metadata.yaml | 21 + .../pre-existing-subnetwork/outputs.tf | 35 + .../pre-existing-subnetwork/variables.tf | 39 + .../pre-existing-subnetwork/versions.tf | 29 + .../network/pre-existing-vpc/README.md | 110 + .../modules/network/pre-existing-vpc/main.tf | 53 + .../network/pre-existing-vpc/metadata.yaml | 19 + .../network/pre-existing-vpc/outputs.tf | 50 + .../network/pre-existing-vpc/variables.tf | 37 + .../network/pre-existing-vpc/versions.tf | 29 + .../embedded/modules/network/vpc/README.md | 237 ++ .../embedded/modules/network/vpc/main.tf | 256 ++ .../modules/network/vpc/metadata.yaml | 19 + .../embedded/modules/network/vpc/outputs.tf | 68 + .../embedded/modules/network/vpc/variables.tf | 301 +++ .../embedded/modules/network/vpc/versions.tf | 19 + .../modules/packer/custom-image/README.md | 320 +++ .../modules/packer/custom-image/image.pkr.hcl | 216 ++ .../modules/packer/custom-image/metadata.yaml | 21 + .../packer/custom-image/variables.pkr.hcl | 276 ++ .../packer/custom-image/versions.pkr.hcl | 25 + .../scheduler/batch-job-template/README.md | 197 ++ .../batch-job-template/compute_image.tf | 30 + .../scheduler/batch-job-template/main.tf | 149 ++ .../batch-job-template/metadata.yaml | 22 + .../scheduler/batch-job-template/outputs.tf | 80 + .../startup_from_network_storage.tf | 65 + .../templates/batch-job-base.yaml.tftpl | 53 + .../templates/batch-submit.sh.tftpl | 10 + .../scheduler/batch-job-template/variables.tf | 240 ++ .../scheduler/batch-job-template/versions.tf | 37 + .../scheduler/batch-login-node/README.md | 127 + .../scheduler/batch-login-node/main.tf | 127 + .../scheduler/batch-login-node/metadata.yaml | 21 + .../scheduler/batch-login-node/outputs.tf | 37 + .../scheduler/batch-login-node/variables.tf | 151 ++ .../scheduler/batch-login-node/versions.tf | 29 + .../modules/scheduler/gke-cluster/README.md | 220 ++ .../modules/scheduler/gke-cluster/main.tf | 470 ++++ .../scheduler/gke-cluster/metadata.yaml | 19 + .../modules/scheduler/gke-cluster/outputs.tf | 104 + .../templates/gke-network-paramset.yaml.tftpl | 9 + .../templates/network-object.yaml.tftpl | 11 + .../scheduler/gke-cluster/variables.tf | 533 ++++ .../modules/scheduler/gke-cluster/versions.tf | 39 + .../pre-existing-gke-cluster/README.md | 116 + .../pre-existing-gke-cluster/main.tf | 70 + .../pre-existing-gke-cluster/metadata.yaml | 19 + .../pre-existing-gke-cluster/outputs.tf | 33 + .../templates/gke-network-paramset.yaml.tftpl | 9 + .../templates/network-object.yaml.tftpl | 11 + .../pre-existing-gke-cluster/variables.tf | 61 + .../pre-existing-gke-cluster/versions.tf | 30 + .../modules/scripts/startup-script/README.md | 355 +++ .../startup-script/files/configure-ssh.yml | 37 + .../startup-script/files/configure_proxy.sh | 54 + .../files/early_run_hotfixes.sh | 32 + .../startup-script/files/get_from_bucket.sh | 73 + .../startup-script/files/install_ansible.sh | 247 ++ .../files/install_cloud_rdma_drivers.sh | 38 + .../startup-script/files/install_docker.yml | 113 + .../files/install_gpu_network_wait_online.yml | 56 + .../files/install_managed_lustre.yml | 33 + .../files/install_monitoring_agent.sh | 144 ++ .../files/running-script-warning.sh | 26 + .../startup-script/files/setup-raid.yml | 100 + .../startup-script/files/setup-ssh-keys.sh | 19 + .../startup-script/files/setup-ssh-keys.yml | 40 + .../files/startup-script-stdlib-body.sh | 39 + .../files/startup-script-stdlib-head.sh | 266 ++ .../modules/scripts/startup-script/main.tf | 306 +++ .../scripts/startup-script/metadata.yaml | 19 + .../modules/scripts/startup-script/outputs.tf | 39 + .../templates/startup-script-custom.tftpl | 65 + .../scripts/startup-script/variables.tf | 298 +++ .../scripts/startup-script/versions.tf | 37 + deletion-test/primary/outputs.tf | 37 + deletion-test/primary/providers.tf | 27 + deletion-test/primary/variables.tf | 55 + deletion-test/primary/versions.tf | 30 + .../slurm-build/slurm-image/README.md | 320 +++ .../slurm-build/slurm-image/image.pkr.hcl | 216 ++ .../slurm-build/slurm-image/metadata.yaml | 21 + .../slurm-build/slurm-image/variables.pkr.hcl | 276 ++ .../slurm-build/slurm-image/versions.pkr.hcl | 25 + 1931 files changed, 189062 insertions(+) create mode 100644 deletion-test/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY create mode 100644 deletion-test/.ghpc/artifacts/expanded_blueprint.yaml create mode 100644 deletion-test/.gitignore create mode 100644 deletion-test/build_script/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/output.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/notebook/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/notebook/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/notebook/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/notebook/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/notebook/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/validation.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/Intel-DAOS/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/sycomp-scale/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb create mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh create mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt create mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/providers.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/new-project/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables_controller_instance.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/providers.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/README.md create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/main.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpx-in-workload.py create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/compute_image.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/mount.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/mount.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-managed-lustre-client.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/templates/mount-daos.sh.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/gpu-definition/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/gpu-definition/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/instance_validations/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/instance_validations/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/instance_validations/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/instance_validations/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/network-attachment/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/network-attachment/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/network-attachment/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/providers.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl create mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl create mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/firewall-rules/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/network/firewall-rules/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/firewall-rules/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/network/firewall-rules/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/firewall-rules/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/packer/custom-image/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/packer/custom-image/image.pkr.hcl create mode 100644 deletion-test/build_script/modules/embedded/modules/packer/custom-image/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/packer/custom-image/variables.pkr.hcl create mode 100644 deletion-test/build_script/modules/embedded/modules/packer/custom-image/versions.pkr.hcl create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/README.md create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_docker.yml create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/main.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/metadata.yaml create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/outputs.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/variables.tf create mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/versions.tf create mode 100644 deletion-test/build_script/outputs.tf create mode 100644 deletion-test/build_script/providers.tf create mode 100644 deletion-test/build_script/variables.tf create mode 100644 deletion-test/build_script/versions.tf create mode 100644 deletion-test/cluster/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/output.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/notebook/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/notebook/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/notebook/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/notebook/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/notebook/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/validation.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/Intel-DAOS/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/sycomp-scale/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb create mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh create mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt create mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/providers.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/new-project/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel-controller.zip create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel.zip create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables_controller_instance.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/providers.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/README.md create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/main.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpx-in-workload.py create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/compute_image.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/mount.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/mount.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-managed-lustre-client.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/templates/mount-daos.sh.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/gpu-definition/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/gpu-definition/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/instance_validations/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/instance_validations/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/instance_validations/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/instance_validations/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/network-attachment/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/network-attachment/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/network-attachment/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/providers.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl create mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl create mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/firewall-rules/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/network/firewall-rules/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/firewall-rules/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/network/firewall-rules/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/firewall-rules/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/packer/custom-image/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/packer/custom-image/image.pkr.hcl create mode 100644 deletion-test/cluster/modules/embedded/modules/packer/custom-image/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/packer/custom-image/variables.pkr.hcl create mode 100644 deletion-test/cluster/modules/embedded/modules/packer/custom-image/versions.pkr.hcl create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/README.md create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_docker.yml create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/main.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/metadata.yaml create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/outputs.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/variables.tf create mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/versions.tf create mode 100644 deletion-test/cluster/outputs.tf create mode 100644 deletion-test/cluster/providers.tf create mode 100644 deletion-test/cluster/variables.tf create mode 100644 deletion-test/cluster/versions.tf create mode 100644 deletion-test/instructions.txt create mode 100644 deletion-test/primary/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/output.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/notebook/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/notebook/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/notebook/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/notebook/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/sycomp-scale/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb create mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh create mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt create mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/providers.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/new-project/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-enablement/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-enablement/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-enablement/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-enablement/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml create mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml create mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml create mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables_controller_instance.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/providers.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf create mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpx-in-workload.py create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/mount.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-managed-lustre-client.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/mount-daos.sh.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/internal/gpu-definition/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/internal/gpu-definition/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/internal/instance_validations/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/internal/instance_validations/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/internal/instance_validations/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/internal/instance_validations/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/internal/network-attachment/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/internal/network-attachment/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/internal/tpu-definition/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/internal/tpu-definition/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/internal/tpu-definition/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/internal/tpu-definition/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/internal/vpc_peering/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/internal/vpc_peering/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/providers.tf create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl create mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl create mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/firewall-rules/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/network/firewall-rules/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/network/firewall-rules/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/firewall-rules/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/packer/custom-image/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl create mode 100644 deletion-test/primary/modules/embedded/modules/packer/custom-image/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl create mode 100644 deletion-test/primary/modules/embedded/modules/packer/custom-image/versions.pkr.hcl create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/README.md create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/main.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/outputs.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/variables.tf create mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/versions.tf create mode 100644 deletion-test/primary/outputs.tf create mode 100644 deletion-test/primary/providers.tf create mode 100644 deletion-test/primary/variables.tf create mode 100644 deletion-test/primary/versions.tf create mode 100644 deletion-test/slurm-build/slurm-image/README.md create mode 100644 deletion-test/slurm-build/slurm-image/image.pkr.hcl create mode 100644 deletion-test/slurm-build/slurm-image/metadata.yaml create mode 100644 deletion-test/slurm-build/slurm-image/variables.pkr.hcl create mode 100644 deletion-test/slurm-build/slurm-image/versions.pkr.hcl diff --git a/deletion-test/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY b/deletion-test/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY new file mode 100644 index 0000000000..56f49c329a --- /dev/null +++ b/deletion-test/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY @@ -0,0 +1 @@ +Files in this directory are managed by gcluster. Do not modify them manually! diff --git a/deletion-test/.ghpc/artifacts/expanded_blueprint.yaml b/deletion-test/.ghpc/artifacts/expanded_blueprint.yaml new file mode 100644 index 0000000000..4c9f667e28 --- /dev/null +++ b/deletion-test/.ghpc/artifacts/expanded_blueprint.yaml @@ -0,0 +1,651 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +blueprint_name: a3mega-slurm +ghpc_version: v1.74.0-31-gedd55b924-dirty +validators: + - validator: test_deployment_variable_not_used + skip: true +vars: + a3mega_cluster_size: 2 + a3mega_dws_flex_enabled: false + a3mega_enable_spot_vm: true + a3mega_partition_name: a3mega + a3mega_reservation_name: "" + deployment_name: deletion-test + disk_size_gb: 200 + enable_controller_public_ips: true + enable_login_public_ips: true + enable_nvidia_dcgm: true + enable_nvidia_persistenced: true + enable_ops_agent: true + final_image_family: slurm-a3mega + instance_image: + family: ((var.final_image_family)) + project: ((var.project_id)) + labels: + ghpc_blueprint: a3mega-slurm + ghpc_deployment: ((var.deployment_name)) + local_mount_homefs: /home + localssd_mountpoint: /mnt/localssd + network_name_system: deletion-test-sys-net + project_id: hpc-toolkit-dev + region: europe-west1 + slurm_cluster_name: a3mega + source_image_family: ubuntu-accelerator-2204-amd64-with-nvidia-570 + source_image_project_id: + - ubuntu-os-accelerator-images + subnetwork_name_system: deletion-test-sys-subnet + sys_net_range: 172.16.0.0/16 + zone: europe-west1-c +deployment_groups: + - group: primary + terraform_backend: + type: gcs + configuration: + bucket: simranka + prefix: (("a3mega-slurm/${var.deployment_name}/primary")) + terraform_providers: + google: + source: hashicorp/google + version: '>= 6.9.0, <= 7.12.0' + configuration: + project: ((var.project_id)) + region: ((var.region)) + zone: ((var.zone)) + google-beta: + source: hashicorp/google-beta + version: '>= 6.9.0, <= 7.12.0' + configuration: + project: ((var.project_id)) + region: ((var.region)) + zone: ((var.zone)) + modules: + - source: modules/network/vpc + kind: terraform + id: sysnet + outputs: + - name: network_name + - name: subnetwork_name + - name: network_id + description: Automatically-generated output exported for use by later deployment groups + sensitive: true + - name: subnetwork_self_link + description: Automatically-generated output exported for use by later deployment groups + sensitive: true + settings: + deployment_name: ((var.deployment_name)) + labels: ((var.labels)) + mtu: 8244 + network_address_range: ((var.sys_net_range)) + network_name: ((var.network_name_system)) + project_id: ((var.project_id)) + region: ((var.region)) + subnetworks: + - description: primary subnetwork in gsc-sys-net + new_bits: 4 + subnet_name: ((var.subnetwork_name_system)) + subnet_private_access: true + subnet_region: ((var.region)) + - group: build_script + terraform_backend: + type: gcs + configuration: + bucket: simranka + prefix: (("a3mega-slurm/${var.deployment_name}/build_script")) + terraform_providers: + google: + source: hashicorp/google + version: '>= 6.9.0, <= 7.12.0' + configuration: + project: ((var.project_id)) + region: ((var.region)) + zone: ((var.zone)) + google-beta: + source: hashicorp/google-beta + version: '>= 6.9.0, <= 7.12.0' + configuration: + project: ((var.project_id)) + region: ((var.region)) + zone: ((var.zone)) + modules: + - source: modules/scripts/startup-script + kind: terraform + id: image_build_script + outputs: + - name: startup_script + description: Automatically-generated output exported for use by later deployment groups + sensitive: true + settings: + configure_ssh_host_patterns: + - 10.0.0.* + - 10.1.0.* + - 10.2.0.* + - 10.3.0.* + - 10.4.0.* + - 10.5.0.* + - 10.6.0.* + - 10.7.0.* + - (("${var.slurm_cluster_name}*")) + deployment_name: ((var.deployment_name)) + docker: + enabled: true + world_writable: true + enable_gpu_network_wait_online: true + install_ansible: true + labels: ((var.labels)) + project_id: ((var.project_id)) + region: ((var.region)) + runners: + - content: | + --- + - name: Hold nvidia packages + hosts: all + become: true + vars: + nvidia_packages_to_hold: + - libnvidia-cfg1-*-server + - libnvidia-compute-*-server + - libnvidia-nscq-* + - nvidia-compute-utils-*-server + - nvidia-fabricmanager-* + - nvidia-utils-*-server + - nvidia-imex-* + tasks: + - name: Hold nvidia packages + ansible.builtin.command: + argv: + - apt-mark + - hold + - "{{ item }}" + loop: "{{ nvidia_packages_to_hold }}" + destination: hold-nvidia-packages.yml + type: ansible-local + - content: | + #!/bin/bash + set -e -o pipefail + apt-mark hold google-compute-engine + apt-mark hold google-compute-engine-oslogin + apt-mark hold google-guest-agent + apt-mark hold google-osconfig-agent + destination: prevent_google_compute_upgrades.sh + type: shell + - content: | + { + "reboot": false, + "install_cuda": false, + "install_ompi": true, + "install_lustre": false, + "install_managed_lustre": false, + "install_gcsfuse": true, + "monitoring_agent": "cloud-ops", + "use_open_drivers": true + } + destination: /var/tmp/slurm_vars.json + type: data + - content: | + #!/bin/bash + set -e -o pipefail + apt-get update + apt-get install -y git + ansible-galaxy role install googlecloudplatform.google_cloud_ops_agents + ansible-pull \ + -U https://github.com/GoogleCloudPlatform/slurm-gcp -C 6.10.6 \ + -i localhost, --limit localhost --connection=local \ + -e @/var/tmp/slurm_vars.json \ + ansible/playbook.yml + destination: install_slurm.sh + type: shell + - content: | + --- + - name: Install updated gVNIC driver from GitHub + hosts: all + become: true + vars: + package_url: https://github.com/GoogleCloudPlatform/compute-virtual-ethernet-linux/releases/download/v1.4.3/gve-dkms_1.4.3_all.deb + package_filename: /tmp/{{ package_url | basename }} + tasks: + - name: Install driver dependencies + ansible.builtin.apt: + name: + - dkms + - name: Download gVNIC package + ansible.builtin.get_url: + url: "{{ package_url }}" + dest: "{{ package_filename }}" + - name: Install updated gVNIC + ansible.builtin.apt: + deb: "{{ package_filename }}" + state: present + destination: update-gvnic.yml + type: ansible-local + - content: | + #!/bin/bash + set -ex -o pipefail + add-nvidia-repositories -y + apt update -y + apt install -y cuda-toolkit-12-8 + apt install -y nvidia-container-toolkit + apt install -y datacenter-gpu-manager-4-cuda12 + apt install -y datacenter-gpu-manager-4-dev + destination: install-cuda-toolkit.sh + type: shell + - content: | + * - memlock unlimited + * - nproc unlimited + * - stack unlimited + * - nofile 1048576 + * - cpu unlimited + * - rtprio unlimited + destination: /etc/security/limits.d/99-unlimited.conf + type: data + - content: | + ENROOT_CONFIG_PATH ${HOME}/.enroot + ENROOT_RUNTIME_PATH /mnt/localssd/${UID}/enroot/runtime + ENROOT_CACHE_PATH /mnt/localssd/${UID}/enroot/cache + ENROOT_DATA_PATH /mnt/localssd/${UID}/enroot/data + ENROOT_TEMP_PATH /mnt/localssd/${UID}/enroot + destination: /etc/enroot/enroot.conf + type: data + - content: '(("---\n- name: Install CUDA & DCGM & Configure Ops Agent\n hosts: all\n become: true\n vars:\n enable_ops_agent: ${var.enable_ops_agent}\n enable_nvidia_dcgm: ${var.enable_nvidia_dcgm}\n tasks:\n - name: Create nvidia-persistenced override directory\n ansible.builtin.file:\n path: /etc/systemd/system/nvidia-persistenced.service.d\n state: directory\n owner: root\n group: root\n mode: 0o755\n - name: Configure nvidia-persistenced override\n ansible.builtin.copy:\n dest: /etc/systemd/system/nvidia-persistenced.service.d/persistence_mode.conf\n owner: root\n group: root\n mode: 0o644\n content: |\n [Service]\n ExecStart=\n ExecStart=/usr/bin/nvidia-persistenced --user nvidia-persistenced --verbose\n notify: Reload SystemD\n handlers:\n - name: Reload SystemD\n ansible.builtin.systemd:\n daemon_reload: true\n post_tasks:\n - name: Enable Google Cloud Ops Agent\n ansible.builtin.service:\n name: google-cloud-ops-agent.service\n state: \"{{ ''started'' if enable_ops_agent else ''stopped'' }}\"\n enabled: \"{{ enable_ops_agent }}\"\n - name: Disable NVIDIA DCGM by default (enable during boot on GPU nodes)\n ansible.builtin.service:\n name: nvidia-dcgm.service\n state: stopped\n enabled: \"{{ enable_nvidia_dcgm }}\"\n - name: Disable nvidia-persistenced SystemD unit (enable during boot on GPU nodes)\n ansible.builtin.service:\n name: nvidia-persistenced.service\n state: stopped\n enabled: false\n"))' + destination: configure_gpu_monitoring.yml + type: ansible-local + - content: | + --- + - name: Install DMBABUF import helper + hosts: all + become: true + tasks: + - name: Setup apt-transport-artifact-registry repository + ansible.builtin.apt_repository: + repo: deb http://packages.cloud.google.com/apt apt-transport-artifact-registry-stable main + state: present + - name: Install driver dependencies + ansible.builtin.apt: + name: + - dkms + - apt-transport-artifact-registry + - name: Setup gpudirect-tcpxo apt repository + ansible.builtin.apt_repository: + repo: deb [arch=all trusted=yes ] ar+https://us-apt.pkg.dev/projects/gce-ai-infra gpudirect-tcpxo-apt main + state: present + - name: Install DMABUF import helper DKMS package + ansible.builtin.apt: + name: dmabuf-import-helper + state: present + destination: install_dmabuf.yml + type: ansible-local + - content: | + --- + - name: Setup GPUDirect-TCPXO aperture devices + hosts: all + become: true + tasks: + - name: Mount aperture devices to /dev and make writable + ansible.builtin.copy: + dest: /etc/udev/rules.d/00-a3-megagpu.rules + owner: root + group: root + mode: 0o644 + content: | + ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x1ae0", ATTR{device}=="0x0084", TAG+="systemd", \ + RUN+="/usr/bin/mkdir --mode=0755 -p /dev/aperture_devices", \ + RUN+="/usr/bin/systemd-mount --type=none --options=bind --collect %S/%p /dev/aperture_devices/%k", \ + RUN+="/usr/bin/bash -c '/usr/bin/chmod 0666 /dev/aperture_devices/%k/resource*'" + notify: Update initramfs + handlers: + - name: Update initramfs + ansible.builtin.command: /usr/sbin/update-initramfs -u -k all + destination: aperture_devices.yml + type: ansible-local + - content: | + #!/bin/bash + # IMPORTANT: This script should be run *last* in any sequence of setup steps + # that use 'gsutil' or other gcloud commands. + # This is because removing the Snap version of the GCloud SDK can temporarily + # break existing 'gsutil' paths, which might disrupt other scripts still running + # that rely on the Snap-installed version. + + set -e -o pipefail + + # Remove the previously installed Google Cloud SDK (google-cloud-cli) and + # the LXD container manager, both of which might have been installed via Snap. + # This step is crucial to prevent conflicts with the upcoming APT installation + # and address potential issues with Snapd and NFS mounts in specific environments + snap remove google-cloud-cli lxd + # Install key and google-cloud-cli from apt repo + GCLOUD_APT_SOURCE="/etc/apt/sources.list.d/google-cloud-sdk.list" + if [ ! -f "${GCLOUD_APT_SOURCE}" ]; then + # indentation matters in EOT below; do not blindly edit! + cat < "${GCLOUD_APT_SOURCE}" + deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt cloud-sdk main + EOT + fi + curl -o /usr/share/keyrings/cloud.google.asc https://packages.cloud.google.com/apt/doc/apt-key.gpg + apt-get update + apt-get install --assume-yes google-cloud-cli + # Clean up the bash executable hash for subsequent steps using gsutil + hash -r + destination: remove_snap_gcloud.sh + type: shell + - group: slurm-build + terraform_backend: + type: gcs + configuration: + bucket: simranka + prefix: (("a3mega-slurm/${var.deployment_name}/slurm-build")) + modules: + - source: modules/packer/custom-image + kind: packer + id: slurm-image + use: + - image_build_script + - sysnet + settings: + deployment_name: ((var.deployment_name)) + disk_size: ((var.disk_size_gb)) + image_family: ((var.final_image_family)) + labels: ((var.labels)) + machine_type: c2-standard-8 + metadata: + user-data: | + #cloud-config + write_files: + - path: /etc/apt/apt.conf.d/20auto-upgrades + permissions: '0644' + owner: root + content: | + APT::Periodic::Update-Package-Lists "0"; + APT::Periodic::Unattended-Upgrade "0"; + omit_external_ip: false + project_id: ((var.project_id)) + source_image_family: ((var.source_image_family)) + source_image_project_id: ((var.source_image_project_id)) + startup_script: ((module.image_build_script.startup_script)) + subnetwork_name: ((module.sysnet.subnetwork_name)) + zone: ((var.zone)) + - group: cluster + terraform_backend: + type: gcs + configuration: + bucket: simranka + prefix: (("a3mega-slurm/${var.deployment_name}/cluster")) + terraform_providers: + google: + source: hashicorp/google + version: '>= 6.9.0, <= 7.12.0' + configuration: + project: ((var.project_id)) + region: ((var.region)) + zone: ((var.zone)) + google-beta: + source: hashicorp/google-beta + version: '>= 6.9.0, <= 7.12.0' + configuration: + project: ((var.project_id)) + region: ((var.region)) + zone: ((var.zone)) + modules: + - source: modules/file-system/cloud-storage-bucket + kind: terraform + id: data-bucket + settings: + deployment_name: ((var.deployment_name)) + labels: ((var.labels)) + local_mount: /gcs + mount_options: defaults,rw,_netdev,implicit_dirs,allow_other,implicit_dirs,file_mode=777,dir_mode=777 + project_id: ((var.project_id)) + random_suffix: true + region: ((var.region)) + - source: modules/network/multivpc + kind: terraform + id: gpunets + settings: + deployment_name: ((var.deployment_name)) + global_ip_address_range: 10.0.0.0/9 + network_count: 8 + network_name_prefix: (("${var.deployment_name}-gpunet")) + project_id: ((var.project_id)) + region: ((var.region)) + subnetwork_cidr_suffix: 20 + - source: community/modules/network/private-service-access + kind: terraform + id: private_service_access + use: + - sysnet + settings: + labels: ((var.labels)) + network_id: ((module.sysnet.network_id)) + project_id: ((var.project_id)) + - source: modules/file-system/filestore + kind: terraform + id: homefs + use: + - sysnet + - private_service_access + outputs: + - name: network_storage + settings: + connect_mode: ((module.private_service_access.connect_mode)) + deletion_protection: + enabled: true + reason: Avoid data loss + deployment_name: ((var.deployment_name)) + filestore_tier: HIGH_SCALE_SSD + labels: ((var.labels)) + local_mount: /home + mount_options: defaults,hard + network_id: ((module.sysnet.network_id)) + project_id: ((var.project_id)) + region: ((var.region)) + reserved_ip_range: ((module.private_service_access.reserved_ip_range)) + size_gb: 10240 + zone: ((var.zone)) + - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + kind: terraform + id: debug_nodeset + use: + - sysnet + settings: + disk_size_gb: ((var.disk_size_gb)) + instance_image: ((var.instance_image)) + labels: ((var.labels)) + machine_type: n2-standard-2 + name: debug_nodeset + node_count_dynamic_max: 4 + node_count_static: 0 + project_id: ((var.project_id)) + region: ((var.region)) + subnetwork_self_link: ((module.sysnet.subnetwork_self_link)) + zone: ((var.zone)) + - source: community/modules/compute/schedmd-slurm-gcp-v6-partition + kind: terraform + id: debug_partition + use: + - debug_nodeset + settings: + exclusive: false + nodeset: ((flatten([module.debug_nodeset.nodeset]))) + partition_name: debug + - source: modules/scripts/startup-script + kind: terraform + id: a3mega_startup + settings: + deployment_name: ((var.deployment_name)) + docker: + daemon_config: '(("{\n \"data-root\": \"${var.localssd_mountpoint}/docker\"\n}\n"))' + enabled: true + world_writable: true + labels: ((var.labels)) + local_ssd_filesystem: + mountpoint: ((var.localssd_mountpoint)) + permissions: "1777" + project_id: ((var.project_id)) + region: ((var.region)) + runners: + - content: | + --- + - name: Configure Slurm to depend upon aperture devices + hosts: all + become: true + vars: {} + tasks: + - name: Ensure slurmd starts after aperture devices are ready + ansible.builtin.copy: + dest: /etc/systemd/system/slurmd.service.d/aperture.conf + owner: root + group: root + mode: 0o644 + content: | + [Service] + ExecCondition=/usr/bin/test -d /dev/aperture_devices/ + notify: Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + destination: slurm_aperture.yml + type: ansible-local + - content: '(("---\n- name: Enable NVIDIA DCGM on GPU nodes\n hosts: all\n become: true\n vars:\n enable_ops_agent: ${var.enable_ops_agent}\n enable_nvidia_dcgm: ${var.enable_nvidia_dcgm}\n enable_nvidia_persistenced: ${var.enable_nvidia_persistenced}\n tasks:\n - name: Update Ops Agent configuration\n ansible.builtin.blockinfile:\n path: /etc/google-cloud-ops-agent/config.yaml\n insertafter: EOF\n block: |\n metrics:\n receivers:\n dcgm:\n type: dcgm\n service:\n pipelines:\n dcgm:\n receivers:\n - dcgm\n notify:\n - Restart Google Cloud Ops Agent\n handlers:\n - name: Restart Google Cloud Ops Agent\n ansible.builtin.service:\n name: google-cloud-ops-agent.service\n state: \"{{ ''restarted'' if enable_ops_agent else ''stopped'' }}\"\n enabled: \"{{ enable_ops_agent }}\"\n post_tasks:\n - name: Enable Google Cloud Ops Agent\n ansible.builtin.service:\n name: google-cloud-ops-agent.service\n state: \"{{ ''started'' if enable_ops_agent else ''stopped'' }}\"\n enabled: \"{{ enable_ops_agent }}\"\n - name: Enable NVIDIA DCGM\n ansible.builtin.service:\n name: nvidia-dcgm.service\n state: \"{{ ''started'' if enable_nvidia_dcgm else ''stopped'' }}\"\n enabled: \"{{ enable_nvidia_dcgm }}\"\n - name: Enable NVIDIA Persistence Daemon\n ansible.builtin.service:\n name: nvidia-persistenced.service\n state: \"{{ ''started'' if enable_nvidia_persistenced else ''stopped'' }}\"\n enabled: \"{{ enable_nvidia_persistenced }}\"\n"))' + destination: enable_dcgm.yml + type: ansible-local + - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + kind: terraform + id: a3mega_nodeset + use: + - sysnet + - gpunets + settings: + additional_networks: ((flatten([module.gpunets.additional_networks]))) + advanced_machine_features: + threads_per_core: null + bandwidth_tier: gvnic_enabled + disk_size_gb: ((var.disk_size_gb)) + disk_type: pd-ssd + dws_flex: + enabled: ((var.a3mega_dws_flex_enabled)) + enable_public_ips: false + enable_spot_vm: ((var.a3mega_enable_spot_vm)) + instance_image: ((var.instance_image)) + labels: ((var.labels)) + machine_type: a3-megagpu-8g + name: a3mega_nodeset + node_conf: + CoresPerSocket: 52 + ThreadsPerCore: 2 + node_count_dynamic_max: 0 + node_count_static: ((var.a3mega_cluster_size)) + on_host_maintenance: TERMINATE + project_id: ((var.project_id)) + region: ((var.region)) + reservation_name: ((var.a3mega_reservation_name)) + startup_script: ((module.a3mega_startup.startup_script)) + subnetwork_self_link: ((module.sysnet.subnetwork_self_link)) + zone: ((var.zone)) + - source: community/modules/compute/schedmd-slurm-gcp-v6-partition + kind: terraform + id: a3mega_partition + use: + - a3mega_nodeset + settings: + exclusive: false + is_default: true + nodeset: ((flatten([module.a3mega_nodeset.nodeset]))) + partition_conf: + OverSubscribe: EXCLUSIVE + ResumeTimeout: 900 + SuspendTimeout: 600 + partition_name: ((var.a3mega_partition_name)) + - source: modules/scripts/startup-script + kind: terraform + id: controller_startup + settings: + deployment_name: ((var.deployment_name)) + labels: ((var.labels)) + project_id: ((var.project_id)) + region: ((var.region)) + runners: + - content: (("#!/bin/bash\nSLURM_ROOT=/opt/apps/adm/slurm\nmkdir -m 0755 -p \"$${SLURM_ROOT}/scripts\"\nmkdir -p \"$${SLURM_ROOT}/partition-${var.a3mega_partition_name}-prolog_slurmd.d\"\nmkdir -p \"$${SLURM_ROOT}/partition-${var.a3mega_partition_name}-epilog_slurmd.d\"\nmkdir -p \"$${SLURM_ROOT}/prolog_slurmd.d\"\nmkdir -p \"$${SLURM_ROOT}/epilog_slurmd.d\"\n# enable the use of password-free sudo within Slurm jobs on all compute nodes\n# feature is restricted to users with OS Admin Login IAM role\n# https://cloud.google.com/iam/docs/understanding-roles#compute.osAdminLogin\ncurl -s -o \"$${SLURM_ROOT}/scripts/sudo-oslogin\" \\\n https://raw.githubusercontent.com/GoogleCloudPlatform/slurm-gcp/master/tools/prologs-epilogs/sudo-oslogin\nchmod 0755 \"$${SLURM_ROOT}/scripts/sudo-oslogin\"\nln -s \"$${SLURM_ROOT}/scripts/sudo-oslogin\" \"$${SLURM_ROOT}/prolog_slurmd.d/sudo-oslogin.prolog_slurmd\"\nln -s \"$${SLURM_ROOT}/scripts/sudo-oslogin\" \"$${SLURM_ROOT}/epilog_slurmd.d/sudo-oslogin.epilog_slurmd\"\ncurl -s -o \"$${SLURM_ROOT}/scripts/rxdm\" \\\n https://raw.githubusercontent.com/GoogleCloudPlatform/slurm-gcp/master/tools/prologs-epilogs/receive-data-path-manager-mega\nchmod 0755 \"$${SLURM_ROOT}/scripts/rxdm\"\nln -s \"$${SLURM_ROOT}/scripts/rxdm\" \"$${SLURM_ROOT}/partition-${var.a3mega_partition_name}-prolog_slurmd.d/rxdm.prolog_slurmd\"\nln -s \"$${SLURM_ROOT}/scripts/rxdm\" \"$${SLURM_ROOT}/partition-${var.a3mega_partition_name}-epilog_slurmd.d/rxdm.epilog_slurmd\"\n# enable a GPU health check that runs at the completion of all jobs on A3mega nodes\nln -s \"/slurm/scripts/tools/gpu-test\" \"$${SLURM_ROOT}/partition-${var.a3mega_partition_name}-epilog_slurmd.d/gpu-test.epilog_slurmd\"\n")) + destination: stage_scripts.sh + type: shell + - content: | + #!/bin/bash + # reset enroot to defaults of files under /home and running under /run + # allows basic enroot testing with reduced I/O performance + rm -f /etc/enroot/enroot.conf + destination: reset_enroot.sh + type: shell + - source: community/modules/scheduler/schedmd-slurm-gcp-v6-login + kind: terraform + id: slurm_login + use: + - sysnet + settings: + disk_size_gb: ((var.disk_size_gb)) + disk_type: pd-balanced + enable_login_public_ips: ((var.enable_login_public_ips)) + instance_image: ((var.instance_image)) + labels: ((var.labels)) + machine_type: c2-standard-4 + name_prefix: login + project_id: ((var.project_id)) + region: ((var.region)) + subnetwork_self_link: ((module.sysnet.subnetwork_self_link)) + zone: ((var.zone)) + - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + kind: terraform + id: slurm_controller + use: + - sysnet + - a3mega_partition + - debug_partition + - slurm_login + - homefs + - data-bucket + settings: + controller_startup_script: ((module.controller_startup.startup_script)) + deployment_name: ((var.deployment_name)) + disk_size_gb: ((var.disk_size_gb)) + enable_cleanup_compute: true + enable_controller_public_ips: ((var.enable_controller_public_ips)) + enable_external_prolog_epilog: true + instance_image: ((var.instance_image)) + labels: ((var.labels)) + login_nodes: ((flatten([module.slurm_login.login_nodes]))) + login_startup_script: | + #!/bin/bash + # reset enroot to defaults of files under /home and running under /run + # allows basic enroot testing with reduced I/O performance + rm -f /etc/enroot/enroot.conf + machine_type: c2-standard-8 + network_storage: ((flatten([module.data-bucket.network_storage, flatten([module.homefs.network_storage])]))) + nodeset: ((flatten([module.debug_partition.nodeset, flatten([module.a3mega_partition.nodeset])]))) + nodeset_dyn: ((flatten([module.debug_partition.nodeset_dyn, flatten([module.a3mega_partition.nodeset_dyn])]))) + nodeset_tpu: ((flatten([module.debug_partition.nodeset_tpu, flatten([module.a3mega_partition.nodeset_tpu])]))) + partitions: ((flatten([module.debug_partition.partitions, flatten([module.a3mega_partition.partitions])]))) + project_id: ((var.project_id)) + prolog_scripts: + - content: | + #!/bin/bash + hostname | tee /etc/hostname + filename: set_hostname_for_enroot.sh + region: ((var.region)) + slurm_cluster_name: ((var.slurm_cluster_name)) + slurm_conf_tpl: modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl + subnetwork_self_link: ((module.sysnet.subnetwork_self_link)) + zone: ((var.zone)) +terraform_backend_defaults: + type: gcs + configuration: + bucket: simranka diff --git a/deletion-test/.gitignore b/deletion-test/.gitignore new file mode 100644 index 0000000000..1e44b25074 --- /dev/null +++ b/deletion-test/.gitignore @@ -0,0 +1,48 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +# Cache objects +packer_cache/ + +# https://www.packer.io/guides/hcl/variables +# Exclude all .pkrvars.hcl files, which are likely to contain sensitive data, +# such as password, private keys, and other secrets. These should not be part of +# version control as they are data points which are potentially sensitive and +# subject to change depending on the environment. +# +*.pkrvars.hcl + +# For built boxes +*.box diff --git a/deletion-test/build_script/main.tf b/deletion-test/build_script/main.tf new file mode 100644 index 0000000000..18d593011a --- /dev/null +++ b/deletion-test/build_script/main.tf @@ -0,0 +1,86 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + backend "gcs" { + bucket = "simranka" + prefix = "a3mega-slurm/deletion-test/build_script" + } +} + +module "image_build_script" { + source = "./modules/embedded/modules/scripts/startup-script" + configure_ssh_host_patterns = ["10.0.0.*", "10.1.0.*", "10.2.0.*", "10.3.0.*", "10.4.0.*", "10.5.0.*", "10.6.0.*", "10.7.0.*", "${var.slurm_cluster_name}*"] + deployment_name = var.deployment_name + docker = { + enabled = true + world_writable = true + } + enable_gpu_network_wait_online = true + install_ansible = true + labels = var.labels + project_id = var.project_id + region = var.region + runners = [{ + content = "---\n- name: Hold nvidia packages\n hosts: all\n become: true\n vars:\n nvidia_packages_to_hold:\n - libnvidia-cfg1-*-server\n - libnvidia-compute-*-server\n - libnvidia-nscq-*\n - nvidia-compute-utils-*-server\n - nvidia-fabricmanager-*\n - nvidia-utils-*-server\n - nvidia-imex-*\n tasks:\n - name: Hold nvidia packages\n ansible.builtin.command:\n argv:\n - apt-mark\n - hold\n - \"{{ item }}\"\n loop: \"{{ nvidia_packages_to_hold }}\"\n" + destination = "hold-nvidia-packages.yml" + type = "ansible-local" + }, { + content = "#!/bin/bash\nset -e -o pipefail\napt-mark hold google-compute-engine\napt-mark hold google-compute-engine-oslogin\napt-mark hold google-guest-agent\napt-mark hold google-osconfig-agent\n" + destination = "prevent_google_compute_upgrades.sh" + type = "shell" + }, { + content = "{\n \"reboot\": false,\n \"install_cuda\": false,\n \"install_ompi\": true,\n \"install_lustre\": false,\n \"install_managed_lustre\": false,\n \"install_gcsfuse\": true,\n \"monitoring_agent\": \"cloud-ops\",\n \"use_open_drivers\": true\n}\n" + destination = "/var/tmp/slurm_vars.json" + type = "data" + }, { + content = "#!/bin/bash\nset -e -o pipefail\napt-get update\napt-get install -y git\nansible-galaxy role install googlecloudplatform.google_cloud_ops_agents\nansible-pull \\\n -U https://github.com/GoogleCloudPlatform/slurm-gcp -C 6.10.6 \\\n -i localhost, --limit localhost --connection=local \\\n -e @/var/tmp/slurm_vars.json \\\n ansible/playbook.yml\n" + destination = "install_slurm.sh" + type = "shell" + }, { + content = "---\n- name: Install updated gVNIC driver from GitHub\n hosts: all\n become: true\n vars:\n package_url: https://github.com/GoogleCloudPlatform/compute-virtual-ethernet-linux/releases/download/v1.4.3/gve-dkms_1.4.3_all.deb\n package_filename: /tmp/{{ package_url | basename }}\n tasks:\n - name: Install driver dependencies\n ansible.builtin.apt:\n name:\n - dkms\n - name: Download gVNIC package\n ansible.builtin.get_url:\n url: \"{{ package_url }}\"\n dest: \"{{ package_filename }}\"\n - name: Install updated gVNIC\n ansible.builtin.apt:\n deb: \"{{ package_filename }}\"\n state: present\n" + destination = "update-gvnic.yml" + type = "ansible-local" + }, { + content = "#!/bin/bash\nset -ex -o pipefail\nadd-nvidia-repositories -y\napt update -y\napt install -y cuda-toolkit-12-8\napt install -y nvidia-container-toolkit\napt install -y datacenter-gpu-manager-4-cuda12\napt install -y datacenter-gpu-manager-4-dev\n" + destination = "install-cuda-toolkit.sh" + type = "shell" + }, { + content = "* - memlock unlimited\n* - nproc unlimited\n* - stack unlimited\n* - nofile 1048576\n* - cpu unlimited\n* - rtprio unlimited\n" + destination = "/etc/security/limits.d/99-unlimited.conf" + type = "data" + }, { + content = "ENROOT_CONFIG_PATH $${HOME}/.enroot\nENROOT_RUNTIME_PATH /mnt/localssd/$${UID}/enroot/runtime\nENROOT_CACHE_PATH /mnt/localssd/$${UID}/enroot/cache\nENROOT_DATA_PATH /mnt/localssd/$${UID}/enroot/data\nENROOT_TEMP_PATH /mnt/localssd/$${UID}/enroot\n" + destination = "/etc/enroot/enroot.conf" + type = "data" + }, { + content = "---\n- name: Install CUDA & DCGM & Configure Ops Agent\n hosts: all\n become: true\n vars:\n enable_ops_agent: ${var.enable_ops_agent}\n enable_nvidia_dcgm: ${var.enable_nvidia_dcgm}\n tasks:\n - name: Create nvidia-persistenced override directory\n ansible.builtin.file:\n path: /etc/systemd/system/nvidia-persistenced.service.d\n state: directory\n owner: root\n group: root\n mode: 0o755\n - name: Configure nvidia-persistenced override\n ansible.builtin.copy:\n dest: /etc/systemd/system/nvidia-persistenced.service.d/persistence_mode.conf\n owner: root\n group: root\n mode: 0o644\n content: |\n [Service]\n ExecStart=\n ExecStart=/usr/bin/nvidia-persistenced --user nvidia-persistenced --verbose\n notify: Reload SystemD\n handlers:\n - name: Reload SystemD\n ansible.builtin.systemd:\n daemon_reload: true\n post_tasks:\n - name: Enable Google Cloud Ops Agent\n ansible.builtin.service:\n name: google-cloud-ops-agent.service\n state: \"{{ 'started' if enable_ops_agent else 'stopped' }}\"\n enabled: \"{{ enable_ops_agent }}\"\n - name: Disable NVIDIA DCGM by default (enable during boot on GPU nodes)\n ansible.builtin.service:\n name: nvidia-dcgm.service\n state: stopped\n enabled: \"{{ enable_nvidia_dcgm }}\"\n - name: Disable nvidia-persistenced SystemD unit (enable during boot on GPU nodes)\n ansible.builtin.service:\n name: nvidia-persistenced.service\n state: stopped\n enabled: false\n" + destination = "configure_gpu_monitoring.yml" + type = "ansible-local" + }, { + content = "---\n- name: Install DMBABUF import helper\n hosts: all\n become: true\n tasks:\n - name: Setup apt-transport-artifact-registry repository\n ansible.builtin.apt_repository:\n repo: deb http://packages.cloud.google.com/apt apt-transport-artifact-registry-stable main\n state: present\n - name: Install driver dependencies\n ansible.builtin.apt:\n name:\n - dkms\n - apt-transport-artifact-registry\n - name: Setup gpudirect-tcpxo apt repository\n ansible.builtin.apt_repository:\n repo: deb [arch=all trusted=yes ] ar+https://us-apt.pkg.dev/projects/gce-ai-infra gpudirect-tcpxo-apt main\n state: present\n - name: Install DMABUF import helper DKMS package\n ansible.builtin.apt:\n name: dmabuf-import-helper\n state: present\n" + destination = "install_dmabuf.yml" + type = "ansible-local" + }, { + content = "---\n- name: Setup GPUDirect-TCPXO aperture devices\n hosts: all\n become: true\n tasks:\n - name: Mount aperture devices to /dev and make writable\n ansible.builtin.copy:\n dest: /etc/udev/rules.d/00-a3-megagpu.rules\n owner: root\n group: root\n mode: 0o644\n content: |\n ACTION==\"add\", SUBSYSTEM==\"pci\", ATTR{vendor}==\"0x1ae0\", ATTR{device}==\"0x0084\", TAG+=\"systemd\", \\\n RUN+=\"/usr/bin/mkdir --mode=0755 -p /dev/aperture_devices\", \\\n RUN+=\"/usr/bin/systemd-mount --type=none --options=bind --collect %S/%p /dev/aperture_devices/%k\", \\\n RUN+=\"/usr/bin/bash -c '/usr/bin/chmod 0666 /dev/aperture_devices/%k/resource*'\"\n notify: Update initramfs\n handlers:\n - name: Update initramfs\n ansible.builtin.command: /usr/sbin/update-initramfs -u -k all\n" + destination = "aperture_devices.yml" + type = "ansible-local" + }, { + content = "#!/bin/bash\n# IMPORTANT: This script should be run *last* in any sequence of setup steps\n# that use 'gsutil' or other gcloud commands.\n# This is because removing the Snap version of the GCloud SDK can temporarily\n# break existing 'gsutil' paths, which might disrupt other scripts still running\n# that rely on the Snap-installed version.\n\nset -e -o pipefail\n\n# Remove the previously installed Google Cloud SDK (google-cloud-cli) and\n# the LXD container manager, both of which might have been installed via Snap.\n# This step is crucial to prevent conflicts with the upcoming APT installation\n# and address potential issues with Snapd and NFS mounts in specific environments\nsnap remove google-cloud-cli lxd\n# Install key and google-cloud-cli from apt repo\nGCLOUD_APT_SOURCE=\"/etc/apt/sources.list.d/google-cloud-sdk.list\"\nif [ ! -f \"$${GCLOUD_APT_SOURCE}\" ]; then\n # indentation matters in EOT below; do not blindly edit!\n cat < \"$${GCLOUD_APT_SOURCE}\"\ndeb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt cloud-sdk main\nEOT\nfi\ncurl -o /usr/share/keyrings/cloud.google.asc https://packages.cloud.google.com/apt/doc/apt-key.gpg\napt-get update\napt-get install --assume-yes google-cloud-cli\n# Clean up the bash executable hash for subsequent steps using gsutil\nhash -r\n" + destination = "remove_snap_gcloud.sh" + type = "shell" + }] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/README.md b/deletion-test/build_script/modules/embedded/community/modules/README.md new file mode 100644 index 0000000000..0c83b9d30c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/README.md @@ -0,0 +1,7 @@ +# Community Modules + +This directory contains modules that rely on partner resources, have been +contributed by outside developers or are in early development by the Cluster Toolkit +team. The modules in this directory are listed alongside core modules in the +[core modules README](../../modules/README.md). There you can also learn more +about general use and how to write custom Cluster Toolkit modules. diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/README.md new file mode 100644 index 0000000000..0ee685d93e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/README.md @@ -0,0 +1,55 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 4.84 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [home\_pv](#module\_home\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | +| [kubectl\_apply](#module\_kubectl\_apply) | ../../../../modules/management/kubectl-apply | n/a | +| [slurm\_key\_pv](#module\_slurm\_key\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.gke_nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | +| [filestore\_id](#input\_filestore\_id) | An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `list(string)` | n/a | yes | +| [image](#input\_image) | The image for slurm daemon | `string` | n/a | yes | +| [instance\_templates](#input\_instance\_templates) | The URLs of Instance Templates | `list(string)` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| n/a | yes | +| [node\_count\_static](#input\_node\_count\_static) | The number of static nodes in node-pool | `number` | n/a | yes | +| [node\_pool\_names](#input\_node\_pool\_names) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `list(string)` | n/a | yes | +| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used in slurm controller | `string` | n/a | yes | +| [slurm\_controller\_instance](#input\_slurm\_controller\_instance) | Slurm cluster controller instance | `any` | n/a | yes | +| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | +| [subnetwork](#input\_subnetwork) | Primary subnetwork object | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset\_name](#output\_nodeset\_name) | Name of the new Slinky nodset | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/main.tf new file mode 100644 index 0000000000..8b2f1deeac --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/main.tf @@ -0,0 +1,64 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +### GKE NodeSet +locals { + manifest_path = "${path.module}/templates/nodeset-general.yaml.tftpl" +} + +module "kubectl_apply" { + source = "../../../../modules/management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + apply_manifests = [{ + source = local.manifest_path, + template_vars = { + slurm_namespace = var.slurm_namespace, + nodeset_name = "${var.slurm_cluster_name}-${var.nodeset_name}", + nodeset_cr_name = "${var.slurm_cluster_name}-${var.nodeset_name}", + controller_name = "${var.slurm_cluster_name}-controller", + node_pool_name = var.node_pool_names[0], + node_count = var.node_count_static, + image = var.image, + home_pvc = module.home_pv.pvc_name + slurm_key_pvc = module.slurm_key_pv.pvc_name + } + }] +} + +data "google_storage_bucket" "this" { + name = var.slurm_bucket[0].name + + depends_on = [var.slurm_bucket] +} + +### Slurm NodeSet +locals { + nodeset = { + gke_nodepool = var.node_pool_names[0] + nodeset_name = var.nodeset_name + node_count_static = var.node_count_static + subnetwork = "https://www.googleapis.com/compute/v1/projects/${var.project_id}/regions/${var.subnetwork.region}/subnetworks/${var.subnetwork.name}" + instance_template = var.instance_templates[0] + } +} + +resource "google_storage_bucket_object" "gke_nodeset_config" { + bucket = data.google_storage_bucket.this.name + name = "${var.slurm_bucket_dir}/nodeset_configs/${var.nodeset_name}.yaml" + content = yamlencode(local.nodeset) +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml new file mode 100644 index 0000000000..ea2cfc221e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/output.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/output.tf new file mode 100644 index 0000000000..15970ff0b7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/output.tf @@ -0,0 +1,18 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_name" { + description = "Name of the new Slinky nodset" + value = local.nodeset.nodeset_name +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf new file mode 100644 index 0000000000..8a190c4019 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf @@ -0,0 +1,50 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + slurm_key_storage = { + server_ip = var.slurm_controller_instance.network_interface[0].network_ip + remote_mount = "/slurm/key_distribution" # defined in /community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py + client_install_runner = {} + mount_runner = {} + fs_type = "" + local_mount = "" + mount_options = "" + } +} + +module "slurm_key_pv" { + source = "../../../../modules/file-system/gke-persistent-volume" + labels = {} + capacity_gib = 1 + cluster_id = var.cluster_id + filestore_id = "projects/empty/locations/empty/instances/empty" # this does not apply since this NFS is not a filestore + namespace = var.slurm_namespace + network_storage = local.slurm_key_storage + pv_name = "slurm-key-pv" + pvc_name = "slurm-key-pvc" +} + +# Assume the var.network_storage[0] will be home and only one home pv is accepted for now. +module "home_pv" { + source = "../../../../modules/file-system/gke-persistent-volume" + labels = {} + capacity_gib = 1024 + cluster_id = var.cluster_id + filestore_id = var.filestore_id[0] + network_storage = var.network_storage[0] + namespace = var.slurm_namespace + pv_name = "home-pv" + pvc_name = "home-pvc" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl new file mode 100644 index 0000000000..a5a4a5e7ac --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl @@ -0,0 +1,203 @@ +apiVersion: slinky.slurm.net/v1alpha1 +kind: NodeSet +metadata: + annotations: + meta.helm.sh/release-name: slurm + meta.helm.sh/release-namespace: ${slurm_namespace} + labels: + app.kubernetes.io/component: compute + app.kubernetes.io/instance: slurm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: slurmd + app.kubernetes.io/part-of: slurm + app.kubernetes.io/version: "24.11" + helm.sh/chart: slurm-0.3.0 + nodeset.slinky.slurm.net/name: ${nodeset_name} + name: ${nodeset_name} + namespace: ${slurm_namespace} +spec: + clusterName: slurm + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + replicas: ${node_count} + revisionHistoryLimit: 0 + selector: + matchLabels: + app.kubernetes.io/instance: slurm + app.kubernetes.io/name: slurmd + nodeset.slinky.slurm.net/name: ${nodeset_name} + serviceName: slurm-compute + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: slurmd + labels: + app.kubernetes.io/component: compute + app.kubernetes.io/instance: slurm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: slurmd + app.kubernetes.io/part-of: slurm + app.kubernetes.io/version: "24.11" + helm.sh/chart: slurm-0.3.0 + nodeset.slinky.slurm.net/name: ${nodeset_name} + spec: + automountServiceAccountToken: false + containers: + - args: + - -g + - -- + - bash + - -c + - | + mkdir -p /usr/local/lib/slurm + ln -s /usr/lib/x86_64-linux-gnu/slurm/spank_pyxis.so /usr/local/lib/slurm/spank_pyxis.so + /usr/local/bin/entrypoint.sh -Z --conf-server ${controller_name}:6825 -N $NODE_NAME + command: + - tini + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_CPUS + value: "0" + - name: POD_MEMORY + value: "0" + image: ${image} + imagePullPolicy: IfNotPresent + name: slurmd + ports: + - containerPort: 6818 + name: slurmd + protocol: TCP + readinessProbe: + exec: + command: + - scontrol + - show + - slurmd + resources: {} + securityContext: + capabilities: + add: + - BPF + - NET_ADMIN + - SYS_ADMIN + - SYS_NICE + privileged: true + volumeMounts: + - mountPath: /etc/slurm + name: etc-slurm + - mountPath: /run + name: run + - mountPath: /var/spool/slurmd + name: slurm-spool + - mountPath: /var/log/slurm + name: slurm-log + - mountPath: /home + name: home-pvc + dnsConfig: + searches: + - ${controller_name} + hostNetwork: true + initContainers: + - command: + - tini + - -g + - -- + - bash + - -c + - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# + SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env + contains:\n# SLURM_USER - username or UID\n\nfunction init::common() {\n\tlocal + dir\n\n\tdir=/var/spool/slurmd\n\tmkdir -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" + \"$dir\"\n\tchmod -v 700 \"$dir\"\n\n\tdir=/var/spool/slurmctld\n\tmkdir + -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" \"$dir\"\n\tchmod + -v 700 \"$dir\"\n}\n\nfunction init::slurm() {\n\tSLURM_MOUNT=/mnt/slurm\n\tSLURM_DIR=/mnt/etc/slurm\n\n\t# + Workaround to ephemeral volumes not supporting securityContext\n\t# https://github.com/kubernetes/kubernetes/issues/81089\n\n\t# + Copy Slurm config files, secrets, and scripts\n\tmkdir -p \"$SLURM_DIR\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -name \"*.conf\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -name \"*.key\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" + -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind \"$${SLURM_MOUNT}\" -type + f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 | xargs + -0r cp -vt \"$${SLURM_DIR}\"\n\n\t# Set general permissions and ownership\n\tfind + \"$${SLURM_DIR}\" -type f -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind + \"$${SLURM_DIR}\" -type f -name \"*.conf\" -print0 | xargs -0r chmod -v 644\n\tfind + \"$${SLURM_DIR}\" -type f -name \"*.key\" -print0 | xargs -0r chmod -v 600\n\tfind + \"$${SLURM_DIR}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" + -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind \"$${SLURM_DIR}\" + -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 + | xargs -0r chmod -v 755\n\n\t# Inject secrets into certain config files\n\tlocal + dbd_conf=\"slurmdbd.conf\"\n\tif [[ -f \"$${SLURM_MOUNT}/$${dbd_conf}\" ]]; + then\n\t\techo \"Injecting secrets from environment into: $${dbd_conf}\"\n\t\trm + -f \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tenvsubst <\"$${SLURM_MOUNT}/$${dbd_conf}\" + >\"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" + \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchmod -v 600 \"$${SLURM_DIR}/$${dbd_conf}\"\n\tfi\n\n\t# + Display Slurm directory files\n\tls -lAF \"$${SLURM_DIR}\"\n}\n\nfunction + main() {\n\tinit::common\n\tinit::slurm\n}\nmain\n" + env: + - name: SLURM_USER + value: slurm + image: ${image} + imagePullPolicy: IfNotPresent + name: init + resources: {} + volumeMounts: + - mountPath: /mnt/slurm + name: slurm-config + - mountPath: /mnt/etc/slurm + name: etc-slurm + - command: + - tini + - -g + - -- + - bash + - -c + - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# + SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env + contains:\n# SOCKET - Named socket to read from\n\nmkdir -v -p \"$(dirname + \"$SOCKET\")\"\nrm -f \"$SOCKET\"\nif ! [ -f \"$SOCKET\" ]; then\n\tmkfifo + -m 777 \"$SOCKET\"\nfi\nwhile IFS=\"\" read data; do\n\techo $data\ndone + <\"$SOCKET\"\n" + env: + - name: SOCKET + value: /var/log/slurm/slurmd.log + image: ghcr.io/slinkyproject/sackd:24.11-ubuntu24.04 + imagePullPolicy: IfNotPresent + name: logfile + resources: {} + restartPolicy: Always + volumeMounts: + - mountPath: /var/log/slurm + name: slurm-log + nodeSelector: + cloud.google.com/gke-nodepool: ${node_pool_name} + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Equal + value: present + volumes: + - emptyDir: + medium: Memory + name: etc-slurm + - emptyDir: {} + name: run + - name: slurm-config + persistentVolumeClaim: + claimName: ${slurm_key_pvc} + - emptyDir: + medium: Memory + name: slurm-spool + - emptyDir: + medium: Memory + name: slurm-log + - name: home-pvc + persistentVolumeClaim: + claimName: ${home_pvc} + updateStrategy: + rollingUpdate: + maxUnavailable: 20% + type: RollingUpdate diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/variables.tf new file mode 100644 index 0000000000..c091a0da86 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/variables.tf @@ -0,0 +1,118 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" + type = string +} + +variable "slurm_cluster_name" { + type = string + description = "Cluster name, used in slurm controller" + + validation { + condition = var.slurm_cluster_name != null && can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) + error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." + } +} + +variable "slurm_controller_instance" { + type = any + description = "Slurm cluster controller instance" +} + +variable "image" { + description = "The image for slurm daemon" + type = string + nullable = false +} + +variable "node_pool_names" { + description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." + type = list(string) + nullable = false +} + +variable "node_count_static" { + description = "The number of static nodes in node-pool" + type = number +} + +variable "subnetwork" { + description = "Primary subnetwork object" + type = any +} + +variable "slurm_namespace" { + description = "slurm namespace for charts" + type = string + default = "slurm" +} + +variable "nodeset_name" { + description = "The nodeset name" + type = string + default = "gkenodeset" +} + +variable "slurm_bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + type = string + nullable = false +} + +variable "slurm_bucket" { + description = "GCS Bucket of Slurm cluster file storage." + type = any + nullable = true +} + +variable "instance_templates" { + description = "The URLs of Instance Templates" + type = list(string) + nullable = false +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + + validation { + condition = length(var.network_storage) == 1 && var.network_storage[0].local_mount == "/home" + error_message = "The 'network_storage' variable must contain exactly one element, and that element's 'local_mount' attribute must be \"/home\"." + } +} + +variable "filestore_id" { + description = "An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." + type = list(string) + + validation { + condition = length(var.filestore_id) == 1 + error_message = "The 'filestore_id' variable must contain exactly one element." + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/versions.tf new file mode 100644 index 0000000000..3d7237cb92 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-nodeset/v1.51.0" + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/README.md new file mode 100644 index 0000000000..2a7c363a87 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/README.md @@ -0,0 +1,39 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 4.84 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [has\_tpu](#input\_has\_tpu) | If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu. | `bool` | `false` | no | +| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | +| [partition\_name](#input\_partition\_name) | The partition name | `string` | `"gke"` | no | +| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/main.tf new file mode 100644 index 0000000000..2949fd6594 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/main.tf @@ -0,0 +1,47 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "google_storage_bucket" "this" { + name = var.slurm_bucket[0].name + + depends_on = [var.slurm_bucket] +} + +### Slurm Partition +locals { + partition_conf = { + "PowerDownOnIdle" = "NO" + "SuspendTime" = "INFINITE" + "SuspendTimeout" = var.has_tpu ? 240 : 120 + "ResumeTimeout" = var.has_tpu ? 600 : 300 + } + + partition = { + partition_name = var.partition_name + partition_conf = local.partition_conf + + partition_nodeset = [var.nodeset_name] + partition_nodeset_tpu = [] + partition_nodeset_dyn = [] + # Options + enable_job_exclusive = true + power_down_on_idle = false + } +} + +resource "google_storage_bucket_object" "parition_config" { + bucket = data.google_storage_bucket.this.name + name = "${var.slurm_bucket_dir}/partition_configs/${var.partition_name}.yaml" + content = yamlencode(local.partition) +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/metadata.yaml new file mode 100644 index 0000000000..557e1fc2ae --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/variables.tf new file mode 100644 index 0000000000..3aeed2e59a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/variables.tf @@ -0,0 +1,43 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "has_tpu" { + description = "If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu." + type = bool + default = false +} + +variable "nodeset_name" { + description = "The nodeset name" + type = string + default = "gkenodeset" +} + +variable "partition_name" { + description = "The partition name" + type = string + default = "gke" +} + +variable "slurm_bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + type = string + nullable = false +} + +variable "slurm_bucket" { + description = "GCS Bucket of Slurm cluster file storage." + type = any + nullable = true +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/versions.tf new file mode 100644 index 0000000000..aede55263c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-partition/v1.51.0" + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/README.md new file mode 100644 index 0000000000..4f65411ddf --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/README.md @@ -0,0 +1,271 @@ +## Description + +This module performs the following tasks: + +- create an instance template from which execute points will be created +- create a managed instance group ([MIG][mig]) for execute points +- create a Toolkit runner to configure the autoscaler to scale the MIG + +It is expected to be used with the [htcondor-install] and [htcondor-setup] +modules. + +[htcondor-install]: ../../scripts/htcondor-install/README.md +[htcondor-setup]: ../../scheduler/htcondor-setup/README.md +[mig]: https://cloud.google.com/compute/docs/instance-groups/ + +### Known limitations + +This module may be used multiple times in a blueprint to create sets of +execute points in an HTCondor pool. If used more than 1 time, the setting +[name_prefix](#input_name_prefix) must be set to a value that is unique across +all uses of the htcondor-execute-point module. If you do not follow this +constraint, you will likely receive an error while running `terraform apply` +similar to that shown below. + +```text +Error: Invalid value for variable + + on modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf line 136, in module "startup_script": + 136: runners = local.all_runners + ├──────────────── + │ var.runners is list of map of string with 5 elements + +All startup-script runners must have a unique destination. +``` + +### How to configure jobs to select execute points + +HTCondor access points provisioned by the Toolkit are specially configured to +honor an attribute named `RequireId` in each [Job ClassAd][jobad]. This value +must be set to the ID of a MIG created by an instance of this module. The +[htcondor-access-point] module includes a setting `var.default_mig_id` that will +set this value automatically to the MIG ID corresponding to the module's +execute points. If this setting is left unset each job must specify `+RequireId` +explicitly. In all cases, the default value can be overridden explicitly as shown +below: + +```text +universe = vanilla +executable = /bin/echo +arguments = "Hello, World!" +output = out.$(ClusterId).$(ProcId) +error = err.$(ClusterId).$(ProcId) +log = log.$(ClusterId).$(ProcId) +request_cpus = 1 +request_memory = 100MB ++RequireId = "htcondor-pool-ep-mig" +queue +``` + +[htcondor-access-point]: ../../scheduler/htcondor-access-point/README.md +[jobad]: https://htcondor.readthedocs.io/en/latest/users-manual/matchmaking-with-classads.html + +### Example + +A full example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +The following code snippet creates a pool with 2 sets of HTCondor execute +points, one using On-demand pricing and the other using Spot pricing. They use +a startup script and network created in previous steps. + +```yaml +- id: htcondor_execute_point + source: community/modules/compute/htcondor-execute-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + min_idle: 2 + +- id: htcondor_execute_point_spot + source: community/modules/compute/htcondor-execute-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + spot: true + +- id: htcondor_access + source: community/modules/scheduler/htcondor-access-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + - htcondor_execute_point + - htcondor_execute_point_spot + settings: + default_mig_id: $(htcondor_execute_point.mig_id) + enable_public_ips: true + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - access_point_ips + - access_point_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## Behavior of Managed Instance Group (MIG) + +Regional [MIGs][mig] are used to provision Execute Points. By default, VMs +will be provisioned in any of the zones available in that region, however, it +can be constrained to run in fewer zones (or a single zone) using +[var.zones](#input_zones). + +When the configuration of an Execute Point is changed, the MIG can be configured +to [replace the VM][replacement] using a "proactive" or "opportunistic" policy. +By default, the policy is set to opportunistic. In practice, this means that +Execute Points will _NOT_ be automatically replaced by Terraform when changes to +the instance template / HTCondor configuration are made. We recommend leaving +this at the default value as it will allow the HTCondor autoscaler to replace +VMs when they become idle without disrupting running jobs. + +However, if it is desired [var.update_policy](#input_update_policy) can be set +to "PROACTIVE" to enable automatic replacement. This will disrupt running jobs +and send them back to the queue. Alternatively, one can leave the setting at +the default value of "OPPORTUNISTIC" and update: + +- intentionally by issuing an update via Cloud Console or using gcloud (below) +- VMs becomes unhealthy or are otherwise automatically replaced (e.g. regular + Google Cloud maintenance) + +For example, to manually update all instances in a MIG: + +```text +gcloud compute instance-groups managed update-instances \ + <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + +## Known Issues + +When using OS Login with "external users" (outside of the Google Cloud +organization), then Docker universe jobs will fail and cause the Docker daemon +to crash. This stems from the use of POSIX user ids (uid) outside the range +supported by Docker. Please consider disabling OS Login if this atypical +situation applies. + +```yaml +vars: + # add setting below to existing deployment variables + enable_oslogin: DISABLE +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 4.0 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.0 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [execute\_point\_instance\_template](#module\_execute\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [mig](#module\_mig) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.execute_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.execute_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Disk type for template | `string` | `"pd-balanced"` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape across zones for instance group managing execute points | `string` | `"ANY"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [execute\_point\_runner](#input\_execute\_point\_runner) | A list of Toolkit runners for configuring an HTCondor execute point | `list(map(string))` | `[]` | no | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account for HTCondor execute point (e-mail format) | `string` | n/a | yes | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | HTCondor execute point VM image

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to HTConodr execute points | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor execute points | `string` | `"n2-standard-4"` | no | +| [max\_size](#input\_max\_size) | Maximum size of the HTCondor execute point pool. | `number` | `5` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor execute points | `map(string)` | `{}` | no | +| [min\_idle](#input\_min\_idle) | Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max\_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly. | `number` | `0` | no | +| [name\_prefix](#input\_name\_prefix) | Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network HTCondor execute points will join | `string` | `"default"` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HTCondor execute points will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which HTCondor execute points will be created | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork HTCondor execute points will join | `string` | `null` | no | +| [target\_size](#input\_target\_size) | Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size. | `number` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | +| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | Startup script to run at boot-time for Windows-based HTCondor execute points | `list(string)` | `[]` | no | +| [zones](#input\_zones) | Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [autoscaler\_runner](#output\_autoscaler\_runner) | Toolkit runner to configure the HTCondor autoscaler | +| [mig\_id](#output\_mig\_id) | ID of the managed instance group containing the execute points | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml new file mode 100644 index 0000000000..375ae036cd --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml @@ -0,0 +1,74 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Role + hosts: localhost + become: true + vars: + spool_dir: /var/lib/condor/spool + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + handlers: + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml new file mode 100644 index 0000000000..a85158fdfc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml @@ -0,0 +1,98 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This playbook makes the assumption that a virtual environment has been created +# with the autoscaler and its dependencies previously installed. A runner that +# does this is provided as an output of the htcondor-install module within the +# Cluster Toolkit at community/modules/scripts/htcondor-install. + +--- +- name: Configure HTCondor Autoscaler + hosts: all + vars: + python: /usr/local/htcondor/bin/python3 + autoscaler: /usr/local/htcondor/bin/autoscaler.py + systemd_override_path: /etc/systemd/system + become: true + tasks: + - name: User must supply HTCondor role + ansible.builtin.assert: + that: + - project_id is defined + - region is defined + - zone is defined + - mig_id is defined + - max_size is defined + - name: Create SystemD service for HTCondor autoscaler + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@.service" + mode: 0644 + content: | + [Unit] + Description=HTCondor Autoscaler MIG: %i + + [Service] + User=condor + Type=oneshot + ExecStart={{ python }} {{ autoscaler }} --p $PROJECT_ID --r $REGION --z $ZONE --mz --g %i --c $MAX_SIZE --i $MIN_IDLE + notify: + - Reload SystemD + - name: Create SystemD override directory for autoscaler configuration + ansible.builtin.file: + path: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d" + state: directory + owner: root + group: root + mode: 0755 + - name: Create autoscaler configuration + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d/miglimit.conf" + mode: 0644 + content: | + [Service] + Environment=PROJECT_ID={{ project_id }} + Environment=REGION={{ region }} + Environment=ZONE={{ zone }} + Environment=MAX_SIZE={{ max_size }} + Environment=MIN_IDLE={{ min_idle }} + notify: + - Reload SystemD + - name: Create SystemD timer for HTCondor autoscaler + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@.timer" + mode: 0644 + content: | + [Unit] + Description=Run HTCondor Autoscaler Periodically + + [Timer] + OnCalendar=minutely + AccuracySec=1us + RandomizedDelaySec=30 + # the directive below is ignored harmlessly on CentOS 7; this has impact + # that timing averages to 1 minute but is not precisely 1 minute; still + # useful to ensure that timers for different MIGs do not overlap + FixedRandomDelay=true + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + post_tasks: + - name: Activate HTCondor Autoscaler timer + ansible.builtin.systemd: + name: htcondor-autoscaler@{{ mig_id }}.timer + enabled: true + state: started diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf new file mode 100644 index 0000000000..7b0df94987 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf @@ -0,0 +1,218 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-execute-point", ghpc_role = "compute" }) +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + zones = coalescelist(var.zones, data.google_compute_zones.available.names) + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + windows_startup_ps1 = join("\n\n", flatten([var.windows_startup_ps1, local.execute_config_windows_startup_ps1])) + + is_windows_image = anytrue([for l in data.google_compute_image.compute_image.licenses : length(regexall("windows-cloud", l)) > 0]) + windows_startup_metadata = local.is_windows_image && local.windows_startup_ps1 != "" ? { + windows-startup-script-ps1 = local.windows_startup_ps1 + } : {} + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.windows_startup_metadata, + local.network_storage_metadata, + local.enable_oslogin, + local.disable_automatic_updates_metadata, + var.metadata + ) + + autoscaler_runner = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_configure_autoscaler.yml") + "destination" = "htcondor_configure_autoscaler_${module.mig.instance_group_manager.name}.yml" + "args" = join(" ", [ + "-e project_id=${var.project_id}", + "-e region=${var.region}", + "-e zone=${local.zones[0]}", # this value is required, but ignored by regional MIG autoscaler + "-e mig_id=${module.mig.instance_group_manager.name}", + "-e max_size=${var.max_size}", + "-e min_idle=${var.min_idle}", + ]) + } + + execute_config = templatefile("${path.module}/templates/condor_config.tftpl", { + htcondor_role = "get_htcondor_execute", + central_manager_ips = var.central_manager_ips, + guest_accelerator = local.guest_accelerator, + }) + + execute_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.execute_config.output_name}" + execute_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e htcondor_role=get_htcondor_execute", + "-e config_object=${local.execute_object}", + ]) + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.execute_point_runner, + [local.execute_runner], + ) + + execute_config_windows_startup_ps1 = templatefile( + "${path.module}/templates/download-condor-config.ps1.tftpl", + { + config_object = local.execute_object, + } + ) + + name_prefix = "${var.deployment_name}-${var.name_prefix}-ep" +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +resource "null_resource" "execute_config" { + triggers = { + config = local.execute_config + } +} + +resource "google_storage_bucket_object" "execute_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.execute_config.id), 0, 4)}" + content = local.execute_config + bucket = var.htcondor_bucket_name +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +module "execute_point_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.execute_point_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + gpu = one(local.guest_accelerator) + preemptible = var.spot + startup_script = local.is_windows_image ? null : module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.compute_image.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config +} + +module "mig" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = var.target_size + hostname = local.name_prefix + mig_name = local.name_prefix + instance_template = module.execute_point_instance_template.self_link + + health_check_name = "health-htcondor-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "SUBSTITUTE" + max_surge_fixed = length(local.zones) + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf new file mode 100644 index 0000000000..b31f40130f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "autoscaler_runner" { + value = local.autoscaler_runner + description = "Toolkit runner to configure the HTCondor autoscaler" +} + +output "mig_id" { + value = module.mig.instance_group_manager.name + description = "ID of the managed instance group containing the execute points" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl new file mode 100644 index 0000000000..c8f5ce31a8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl @@ -0,0 +1,31 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:${htcondor_role} +CONDOR_HOST = ${join(",", central_manager_ips)} + +# StartD configuration settings +%{ if length(guest_accelerator) > 0 ~} +use feature:GPUs +%{ endif ~} +use feature:PartitionableSlot +use feature:CommonCloudAttributesGoogle("-c created-by") +UPDATE_INTERVAL = 30 +TRUST_UID_DOMAIN = True +STARTER_ALLOW_RUNAS_OWNER = True +RUNBENCHMARKS = False diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl new file mode 100644 index 0000000000..19789f122e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl @@ -0,0 +1,34 @@ +# create directory for local condor_config customizations +$config_dir = 'C:\Condor\config' +if(!(test-path -PathType container -Path $config_dir)) +{ + New-Item -ItemType Directory -Path $config_dir +} + +# update local condor_config if blueprint has changed +$config_file = "$config_dir\50-ghpc-managed" +if (Test-Path -Path $config_file -PathType Leaf) +{ + $local_hash = gcloud --format="value(md5_hash)" storage hash $config_file +} +else +{ + $local_hash = "INVALID-HASH" +} + +$remote_hash = gcloud --format="value(md5_hash)" storage hash ${config_object} +if ($local_hash -cne $remote_hash) +{ + Write-Output "Updating condor configuration" + gcloud storage cp ${config_object} $config_file + if ($LASTEXITCODE -ne 0) + { + throw "Could not download HTCondor configuration; exiting startup script" + } + Restart-Service condor +} + +# ignored if service is already running; must be here to handle case where +# machine is rebooted, but configuration has previously been downloaded +# and service is disabled from automatic start +Start-Service condor diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf new file mode 100644 index 0000000000..aab8a54c2d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf @@ -0,0 +1,265 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HTCondor execute points will be created" + type = string +} + +variable "region" { + description = "The region in which HTCondor execute points will be created" + type = string +} + +variable "zones" { + description = "Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region." + type = list(string) + default = [] + nullable = false +} + +variable "distribution_policy_target_shape" { + description = "Target shape across zones for instance group managing execute points" + type = string + default = "ANY" +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to HTConodr execute points" + type = map(string) +} + +variable "machine_type" { + description = "Machine type to use for HTCondor execute points" + type = string + default = "n2-standard-4" +} + +variable "execute_point_runner" { + description = "A list of Toolkit runners for configuring an HTCondor execute point" + type = list(map(string)) + default = [] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "instance_image" { + description = <<-EOD + HTCondor execute point VM image + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "execute_point_service_account_email" { + description = "Service account for HTCondor execute point (e-mail format)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_self_link" { + description = "The self link of the network HTCondor execute points will join" + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork HTCondor execute points will join" + type = string + default = null +} + +variable "target_size" { + description = "Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size." + type = number + default = null +} + +variable "max_size" { + description = "Maximum size of the HTCondor execute point pool." + type = number + default = 5 +} + +variable "min_idle" { + description = "Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly." + type = number + default = 0 +} + +variable "metadata" { + description = "Metadata to add to HTCondor execute points" + type = map(string) + default = {} +} + +# this default is deliberately the opposite of vm-instance because of observed +# issues running HTCondor docker universe jobs with OS Login enabled and running +# jobs as a user with uid>2^31; these uids occur when users outside the GCP +# organization login to a VM and OS Login is enabled. +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "spot" { + description = "Provision VMs using discounted Spot pricing, allowing for preemption" + type = bool + default = false +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 100 +} + +variable "disk_type" { + description = "Disk type for template" + type = string + default = "pd-balanced" +} + +variable "windows_startup_ps1" { + description = "Startup script to run at boot-time for Windows-based HTCondor execute points" + type = list(string) + default = [] + nullable = false +} + +variable "central_manager_ips" { + description = "List of IP addresses of HTCondor Central Managers" + type = list(string) +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The HTCondor module supports 0 or 1 models of accelerator card on each execute point" + } +} + +variable "name_prefix" { + description = "Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module" + type = string + nullable = false + validation { + condition = length(var.name_prefix) > 0 + error_message = "var.name_prefix must be a set to a non-empty string and must also be unique across all instances of htcondor-execute-point" + } +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" + type = string + default = "OPPORTUNISTIC" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf new file mode 100644 index 0000000000..729dc3cda5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.1" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.74.0" + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/README.md new file mode 100644 index 0000000000..278207b04a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/README.md @@ -0,0 +1,45 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_group_manager.mig](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_group_manager) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [base\_instance\_name](#input\_base\_instance\_name) | Base name for the instances in the MIG | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will be used to name MIG if `var.name` is not provided | `string` | n/a | yes | +| [ghpc\_module\_id](#input\_ghpc\_module\_id) | Internal GHPC field, do not set this value | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the MIG | `map(string)` | n/a | yes | +| [name](#input\_name) | Name of the MIG. If not provided, will be generated from `var.deployment_name` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the MIG will be created | `string` | n/a | yes | +| [target\_size](#input\_target\_size) | Target number of instances in the MIG | `number` | `0` | no | +| [versions](#input\_versions) | Application versions managed by this instance group. Each version deals with a specific instance template | 
list(object({
    name              = string
    instance_template = string
    target_size = optional(object({
      fixed   = optional(number)
      percent = optional(number)
    }))
  }))
| n/a | yes | +| [wait\_for\_instances](#input\_wait\_for\_instances) | Whether to wait for all instances to be created/updated before returning | `bool` | `false` | no | +| [zone](#input\_zone) | Compute Platform zone. Required, currently only zonal MIGs are supported | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [self\_link](#output\_self\_link) | The URL of the created MIG | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/main.tf new file mode 100644 index 0000000000..0e7cf186c2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/main.tf @@ -0,0 +1,85 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "mig", ghpc_role = "compute" }) +} + +locals { + sanitized_deploy_name = try(replace(lower(var.deployment_name), "/[^a-z0-9]/", ""), null) + sanitized_module_id = try(replace(lower(var.ghpc_module_id), "/[^a-z0-9]/", ""), null) + synth_mig_name = try("${local.sanitized_deploy_name}-${local.sanitized_module_id}", null) + + mig_name = var.name == null ? local.synth_mig_name : var.name + base_instance_name = var.base_instance_name == null ? local.mig_name : var.base_instance_name +} + +resource "google_compute_instance_group_manager" "mig" { + # REQUIRED + name = local.mig_name + base_instance_name = local.base_instance_name + zone = var.zone + + dynamic "version" { + for_each = var.versions + content { + name = version.value.name + instance_template = version.value.instance_template + dynamic "target_size" { + for_each = version.value.target_size != null ? [version.value.target_size] : [] + content { + fixed = target_size.value.fixed + percent = target_size.value.percent + } + } + } + } + + # OPTIONAL + project = var.project_id + target_size = var.target_size + wait_for_instances = var.wait_for_instances + + all_instances_config { + # TODO: validate that template metadata not getting wiped out + # TODO: validate that template labels not getting wiped out + labels = local.labels + } + + # OMITTED: + # * description + # * named_port + # * list_managed_instances_results + # * target_pools - specific for Load Balancers usage + # * wait_for_instances_status + # * auto_healing_policies + # * stateful_disk + # * stateful_internal_ip + # * update_policy + # * params + + + lifecycle { + precondition { + condition = local.mig_name != null + error_message = "Could not come up with a name for the MIG, specify `var.name`" + } + + precondition { + condition = local.base_instance_name != null + error_message = "Could not come up with a base_instance_name, specify `var.base_instance_name`" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/metadata.yaml new file mode 100644 index 0000000000..97a4fa9a89 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com +ghpc: + inject_module_id: ghpc_module_id diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/outputs.tf new file mode 100644 index 0000000000..23c66a3535 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "self_link" { + description = "The URL of the created MIG" + value = google_compute_instance_group_manager.mig.self_link +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/variables.tf new file mode 100644 index 0000000000..b6c3c0e78a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/variables.tf @@ -0,0 +1,86 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Project in which the MIG will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the deployment, will be used to name MIG if `var.name` is not provided" + type = string +} + +variable "labels" { + description = "Labels to add to the MIG" + type = map(string) +} + +variable "zone" { + description = "Compute Platform zone. Required, currently only zonal MIGs are supported" + type = string +} + + +variable "versions" { + description = <<-EOD + Application versions managed by this instance group. Each version deals with a specific instance template + EOD + type = list(object({ + name = string + instance_template = string + target_size = optional(object({ + fixed = optional(number) + percent = optional(number) + })) + })) + + validation { + condition = length(var.versions) > 0 + error_message = "At least one version must be provided" + } + +} + + +variable "ghpc_module_id" { + description = "Internal GHPC field, do not set this value" + type = string + default = null +} + +variable "name" { + description = "Name of the MIG. If not provided, will be generated from `var.deployment_name`" + type = string + default = null +} + +variable "base_instance_name" { + description = "Base name for the instances in the MIG" + type = string + default = null +} + + +variable "target_size" { + description = "Target number of instances in the MIG" + type = number + default = 0 +} + +variable "wait_for_instances" { + description = "Whether to wait for all instances to be created/updated before returning" + type = bool + default = false +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/versions.tf new file mode 100644 index 0000000000..4147447b44 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = "> 5.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:mig/v1.74.0" + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/README.md new file mode 100644 index 0000000000..1dcacc57e9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/README.md @@ -0,0 +1,112 @@ +# Description + +This module creates the Vertex AI Notebook, to be used in tutorials. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage + +This is a simple usage, using the default network: + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: + name_prefix: my-bucket + local_mount: /home/jupyter/my-bucket + + - id: notebook + source: community/modules/compute/notebook + use: [bucket] + settings: + name_prefix: notebook + machine_type: n1-standard-4 + +``` + +If the user wants do specify a custom subnetwork, or specific external IP restrictions, they can use the `network_interfaces` variable, here is an example on how to use a Shared VPC Subnet with an ephemeral external IP: + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: + name_prefix: my-bucket + local_mount: /home/jupyter/my-bucket + + - id: notebook + source: community/modules/compute/notebook + use: [bucket] + settings: + name_prefix: notebook + machine_type: n1-standard-4 + network_interfaces: + - network: "projects/HOST_PROJECT_ID/global/networks/SHARED_VPC_NAME" + subnet: "projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME" + nic_type: "VIRTIO_NET" +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 5.34 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 5.34 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.mount_script](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_workbench_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/workbench_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the notebook. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name, can be provided from the google-cloud-storage module | `string` | `null` | no | +| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "tf-latest-cpu",
  "name": null,
  "project": "deeplearning-platform-release"
}
| no | +| [labels](#input\_labels) | Labels to add to the resource Key-value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | The machine type to employ | `string` | n/a | yes | +| [mount\_runner](#input\_mount\_runner) | mount content from the google-cloud-storage module | `map(string)` | n/a | yes | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces for the VM instance. Each network interface is represented by an object with the following fields:

- network: (Optional) The name of the Virtual Private Cloud (VPC) network that this VM instance is connected to.

- subnet: (Optional) The name of the subnetwork within the specified VPC that this VM instance is connected to.

- nic\_type: (Optional) The type of vNIC to be used on this interface. Possible values are: `VIRTIO_NET`, `GVNIC`.

- access\_configs: (Optional) An array of access configurations for this network interface. The access\_config object contains:
* external\_ip: (Required) An external IP address associated with this instance. Specify an unused static external IP address available to the project or leave this field undefined to use an IP from a shared ephemeral IP address pool. If you specify a static external IP address, it must live in the same region as the zone of the instance. | 
list(object({
    network  = optional(string)
    subnet   = optional(string)
    nic_type = optional(string)
    access_configs = optional(list(object({
      external_ip = optional(string)
    })))
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | ID of project in which the notebook will be created. | `string` | n/a | yes | +| [service\_account\_email](#input\_service\_account\_email) | If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account | `string` | `null` | no | +| [zone](#input\_zone) | The zone to deploy to | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/main.tf new file mode 100644 index 0000000000..cd3ce3b4ea --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/main.tf @@ -0,0 +1,96 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "notebook", ghpc_role = "compute" }) +} + +locals { + suffix = random_id.resource_name_suffix.hex + #name = "thenotebook" + name = "notebook-${var.deployment_name}-${local.suffix}" + bucket = replace(var.gcs_bucket_path, "gs://", "") + post_script_filename = "mount-${local.suffix}.sh" + + # mount_runner_args is defined in the file: cluster-toolkit/modules/file-system/cloud-storage-bucket/outputs.tf + mount_args = split(" ", var.mount_runner.args) + + unused = local.mount_args[0] + remote_mount = local.mount_args[1] + local_mount = local.mount_args[2] + fs_type = local.mount_args[3] + # These options provide a "rw" mount of the GCS bucket + mount_options = "defaults,_netdev,allow_other,implicit_dirs,gid=1000,uid=1000" + + content0 = var.mount_runner.content + content1 = replace(local.content0, "$1", local.unused) + content2 = replace(local.content1, "$2", local.remote_mount) + content3 = replace(local.content2, "$3", local.local_mount) + content4 = replace(local.content3, "$4", local.fs_type) + content5 = replace(local.content4, "$5", local.mount_options) + +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket_object" "mount_script" { + name = local.post_script_filename + content = local.content5 + bucket = local.bucket +} + +resource "google_workbench_instance" "instance" { + name = local.name + location = var.zone + project = var.project_id + labels = local.labels + gce_setup { + machine_type = var.machine_type + metadata = { + post-startup-script = "${var.gcs_bucket_path}/${google_storage_bucket_object.mount_script.name}" + } + vm_image { + project = var.instance_image.project + family = var.instance_image.family + } + + dynamic "service_accounts" { + for_each = var.service_account_email == null ? [] : [1] + content { + email = var.service_account_email + } + } + + dynamic "network_interfaces" { + for_each = var.network_interfaces + content { + network = network_interfaces.value.network + subnet = network_interfaces.value.subnet + nic_type = network_interfaces.value.nic_type + + dynamic "access_configs" { + for_each = network_interfaces.value.access_configs != null ? network_interfaces.value.access_configs : [] + content { + external_ip = access_configs.value.external_ip + } + } + } + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/metadata.yaml new file mode 100644 index 0000000000..4a7d5397ca --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - notebooks.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/variables.tf new file mode 100644 index 0000000000..4359de8c10 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/variables.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which the notebook will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment; used as part of name of the notebook." + type = string + # notebook name can have: lowercase letters, numbers, or hyphens (-) and cannot end with a hyphen + validation { + error_message = "The notebook name uses 'deployment_name' -- can only have: lowercase letters, numbers, or hyphens" + condition = can(regex("^[a-z0-9]+(?:-[a-z0-9]+)*$", var.deployment_name)) + } +} + +variable "zone" { + description = "The zone to deploy to" + type = string +} + +variable "machine_type" { + description = "The machine type to employ" + type = string +} + +variable "labels" { + description = "Labels to add to the resource Key-value pairs." + type = map(string) +} + +variable "instance_image" { + description = "Instance Image" + type = map(string) + default = { + project = "deeplearning-platform-release" + family = "tf-latest-cpu" + name = null + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "gcs_bucket_path" { + description = "Bucket name, can be provided from the google-cloud-storage module" + type = string + default = null +} + +variable "mount_runner" { + description = "mount content from the google-cloud-storage module" + type = map(string) + + validation { + condition = (length(split(" ", var.mount_runner.args)) == 5) + error_message = "There must be 5 elements in the Mount Runner Arguments: ${var.mount_runner.args} \n " + } +} + +variable "service_account_email" { + description = "If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account" + type = string + default = null +} + +variable "network_interfaces" { + type = list(object({ + network = optional(string) + subnet = optional(string) + nic_type = optional(string) + access_configs = optional(list(object({ + external_ip = optional(string) + }))) + })) + default = [] + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | +| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name    = string
    device_name  = string
    disk_size_gb = number
    disk_type    = string
    disk_labels  = map(string)
    auto_delete  = bool
    boot         = bool
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | +| [feature](#input\_feature) | The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used. | `string` | `null` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | Path to the Slurm bucket. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster. | `string` | n/a | yes | +| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_template\_self\_link](#output\_instance\_template\_self\_link) | The URI of the template. | +| [node\_name\_prefix](#output\_node\_name\_prefix) | The prefix to be used for the node names.

Make sure that nodes are named `-`
This temporary required for proper functioning of the nodes.
While Slurm scheduler uses "features" to bind node and nodeset,
the SlurmGCP relies on node names for this (to be switched to features as well). | +| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf new file mode 100644 index 0000000000..31d9f14ae7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf @@ -0,0 +1,128 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset-dynamic", ghpc_role = "compute" }) +} + +module "instance_validation" { + source = "../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + nodeset_name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + feature = coalesce(var.feature, local.nodeset_name) + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + universe_domain = { "universe_domain" = var.universe_domain } + + metadata = merge( + local.disable_automatic_updates_metadata, + local.universe_domain, + { slurmd_feature = local.feature }, + var.metadata + ) + + nodeset = { + nodeset_name = local.nodeset_name + nodeset_feature : local.feature + startup_script = local.ghpc_startup_script + network_storage = var.network_storage + } + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + } + ] + + public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + ghpc_startup_script = [{ + filename = "ghpc_nodeset_startup.sh" + content = var.startup_script + }] + +} + +module "slurm_nodeset_template" { + source = "../../internal/slurm-gcp/instance_template" + + project_id = var.project_id + region = var.region + name_prefix = local.nodeset_name + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = "compute" + slurm_bucket_path = var.slurm_bucket_path + metadata = local.metadata + + additional_disks = local.additional_disks + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(local.labels, var.disk_labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + + bandwidth_tier = var.bandwidth_tier + can_ip_forward = var.can_ip_forward + + advanced_machine_features = var.advanced_machine_features + enable_confidential_vm = var.enable_confidential_vm + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + labels = local.labels + machine_type = var.machine_type + + min_cpu_platform = var.min_cpu_platform + on_host_maintenance = var.on_host_maintenance + termination_action = try(var.spot_instance_config.termination_action, null) + preemptible = var.preemptible + spot = var.enable_spot_vm + service_account = local.service_account + gpu = one(local.guest_accelerator) # requires gpu_definition.tf + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + subnetwork = var.subnetwork_self_link + additional_networks = var.additional_networks + access_config = local.access_config + tags = concat([var.slurm_cluster_name], var.tags) +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml new file mode 100644 index 0000000000..a99e59d09f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [compute.googleapis.com] +ghpc: + inject_module_id: name diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf new file mode 100644 index 0000000000..2d2d1415cf --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_dyn" { + description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset +} + +output "instance_template_self_link" { + description = "The URI of the template." + value = module.slurm_nodeset_template.self_link +} + +output "node_name_prefix" { + description = <<-EOD + The prefix to be used for the node names. + + Make sure that nodes are named `-` + This temporary required for proper functioning of the nodes. + While Slurm scheduler uses "features" to bind node and nodeset, + the SlurmGCP relies on node names for this (to be switched to features as well). + EOD + value = "${var.slurm_cluster_name}-${local.nodeset_name}" + +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf new file mode 100644 index 0000000000..ec6206e317 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf @@ -0,0 +1,402 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "feature" { + type = string + description = "The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used." + default = null +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "slurm_cluster_name" { + description = "Name of the Slurm cluster." + type = string +} + +variable "slurm_bucket_path" { + description = "Path to the Slurm bucket." + type = string +} + + +variable "machine_type" { + description = "Compute Platform machine type to use for this partition compute nodes." + type = string + default = "c2-standard-60" +} + +variable "metadata" { + type = map(string) + description = "Metadata, provided as a map." + default = {} +} + +variable "instance_image" { + description = <<-EOD + Defines the image that will be used in the Slurm node group VM instances. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + + For more information on creating custom images that comply with Slurm on GCP + see the "Slurm on GCP Custom Images" section in docs/vm-images.md. + EOD + type = map(string) + default = { + family = "slurm-gcp-6-11-hpc-rocky-linux-8" + project = "schedmd-slurm-public" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations + description = <<-EOD + A flag that designates that the user is aware that they are requesting + to use a custom and potentially incompatible image for this Slurm on + GCP module. + + If the field is set to false, only the compatible families and project + names will be accepted. The deployment will fail with any other image + family or name. If set to true, no checks will be done. + + See: https://goo.gle/hpc-slurm-images + EOD + type = bool + default = false +} + + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "tags" { + type = list(string) + description = "Network tag list." + default = [] +} + +variable "disk_type" { + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + type = string + default = "pd-standard" +} + +variable "disk_size_gb" { + description = "Size of boot disk to create for the partition compute nodes." + type = number + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "additional_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = list(object({ + disk_name = string + device_name = string + disk_size_gb = number + disk_type = string + disk_labels = map(string) + auto_delete = bool + boot = bool + })) + default = [] +} + +variable "enable_confidential_vm" { + type = bool + description = "Enable the Confidential VM configuration. Note: the instance image must support option." + default = false +} + +variable "enable_shielded_vm" { + type = bool + description = "Enable the Shielded VM configuration. Note: the instance image must support option." + default = false +} + +variable "shielded_instance_config" { + type = object({ + enable_integrity_monitoring = bool + enable_secure_boot = bool + enable_vtpm = bool + }) + description = <<-EOD + Shielded VM configuration for the instance. Note: not used unless + enable_shielded_vm is 'true'. + - enable_integrity_monitoring : Compare the most recent boot measurements to the + integrity policy baseline and return a pair of pass/fail results depending on + whether they match or not. + - enable_secure_boot : Verify the digital signature of all boot components, and + halt the boot process if signature verification fails. + - enable_vtpm : Use a virtualized trusted platform module, which is a + specialized computer chip you can use to encrypt objects like keys and + certificates. + EOD + default = { + enable_integrity_monitoring = true + enable_secure_boot = true + enable_vtpm = true + } +} + + +variable "enable_oslogin" { + type = bool + description = <<-EOD + Enables Google Cloud os-login for user login and authentication for VMs. + See https://cloud.google.com/compute/docs/oslogin + EOD + default = true +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example." + type = bool + default = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "labels" { + description = "Labels to add to partition compute instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "The name of the minimum CPU platform that you want the instance to use." + type = string + default = null +} + +variable "on_host_maintenance" { + type = string + description = <<-EOD + Instance availability Policy. + + Note: Placement groups are not supported when on_host_maintenance is set to + "MIGRATE" and will be deactivated regardless of the value of + enable_placement. To support enable_placement, ensure on_host_maintenance is + set to "TERMINATE". + EOD + default = "TERMINATE" +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." + } +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + + +variable "service_account_email" { + description = "Service account e-mail address to attach to the compute instances." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the compute instances." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "enable_spot_vm" { + description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." + type = bool + default = false +} + +variable "spot_instance_config" { + description = "Configuration for spot VMs." + type = object({ + termination_action = string + }) + default = null +} + +variable "bandwidth_tier" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | +| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | +| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | +| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `null` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling worker nodes allowed in this partition.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of worker nodes to be statically created.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | +| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. | `string` | `""` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the TPU-vm. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the TPU-vm. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | +| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | `"2.14.0"` | no | +| [zone](#input\_zone) | Zone in which to create compute VMs. TPU partitions can only specify a single zone. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf new file mode 100644 index 0000000000..ac9b119702 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# locals { +# # This label allows for billing report tracking based on module. +# labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) +# } + +locals { + name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + nodeset_tpu = { + node_count_static = var.node_count_static + node_count_dynamic_max = var.node_count_dynamic_max + nodeset_name = local.name + node_type = var.node_type + + accelerator_config = var.accelerator_config + tf_version = var.tf_version + preemptible = var.preemptible + preserve_tpu = var.preserve_tpu + + data_disks = var.data_disks + docker_image = var.docker_image + + enable_public_ip = var.enable_public_ips + # TODO: rename to subnetwork_self_link, requires changes to the scripts + subnetwork = var.subnetwork_self_link + service_account = local.service_account + zone = var.zone + + project_id = var.project_id + reserved = var.reserved + network_storage = var.network_storage + } + + node_type_core_count = var.node_type == "" ? 0 : tonumber(regex("-(.*)", var.node_type)[0]) + + accelerator_core_list = var.accelerator_config.topology == "" ? [0, 0] : regexall("\\d+", var.accelerator_config.topology) + accelerator_core_count = length(local.accelerator_core_list) > 2 ? (local.accelerator_core_list[0] * local.accelerator_core_list[1] * local.accelerator_core_list[2]) * 2 : (local.accelerator_core_list[0] * local.accelerator_core_list[1]) * 2 + + tpu_core_count = local.accelerator_core_count == 0 ? local.node_type_core_count : local.accelerator_core_count +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml new file mode 100644 index 0000000000..95b6d1c730 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name + has_to_be_used: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf new file mode 100644 index 0000000000..8cb7b8663e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf @@ -0,0 +1,39 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_tpu" { + description = "Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset_tpu + + precondition { + condition = (var.node_type == "") != (var.accelerator_config == { topology : "", version : "" }) + error_message = "Either a node_type or an accelerator_config must be provided." + } + + precondition { + condition = ((local.tpu_core_count / 8) <= var.node_count_dynamic_max) || ((local.tpu_core_count / 8) <= var.node_count_static) + error_message = <<-EOD + When using TPUs there should be at least one node per every 8 cores. + Currently there are ${local.tpu_core_count} cores but only ${var.node_count_static} static nodes and ${var.node_count_dynamic_max} dynamic nodes. + EOD + } + + precondition { + condition = (var.node_count_dynamic_max % (local.tpu_core_count / 8) == 0) && (var.node_count_static % (local.tpu_core_count / 8) == 0) + error_message = <<-EOD + The number of worker nodes should be a multiple of ${local.tpu_core_count / 8}. + This is to ensure each node has a TPU machine for job scheduling. + EOD + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf new file mode 100644 index 0000000000..367b0bee09 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf @@ -0,0 +1,171 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "node_count_static" { + description = <<-EOD + Number of worker nodes to be statically created. + For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). + See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. + EOD + type = number + default = 0 +} + +variable "node_count_dynamic_max" { + description = <<-EOD + Maximum number of auto-scaling worker nodes allowed in this partition. + For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). + See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. + EOD + type = number + default = 0 +} + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "enable_public_ips" { + description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." + type = bool + default = false +} + +variable "disable_public_ips" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `enable_public_ips` instead." + type = bool + default = null + validation { + condition = var.disable_public_ips == null + error_message = "DEPRECATED: Use `enable_public_ips` instead." + } +} + +variable "node_type" { + description = "Specify a node type to base the vm configuration upon it." + type = string + default = "" +} + +variable "accelerator_config" { + description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." + type = object({ + topology = string + version = string + }) + default = { + topology = "" + version = "" + } + validation { + condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], var.accelerator_config.version) + error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" + } + validation { + condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) + error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." + } +} + +variable "tf_version" { + description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." + type = string + default = "2.14.0" +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + +variable "preserve_tpu" { + description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" + type = bool + default = false +} + +variable "zone" { + description = "Zone in which to create compute VMs. TPU partitions can only specify a single zone." + type = string +} + +variable "data_disks" { + description = "The data disks to include in the TPU node" + type = list(string) + default = [] +} + +variable "docker_image" { + description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" + type = string + default = null +} + +variable "subnetwork_self_link" { + type = string + description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." +} + +variable "service_account_email" { + description = "Service account e-mail address to attach to the TPU-vm." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the TPU-vm." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "service_account" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + } +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "reserved" { + description = "Specify whether TPU-vms in this nodeset are created under a reservation." + type = bool + default = false +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + })) + default = [] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf new file mode 100644 index 0000000000..398eeffdda --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.3" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.74.0" + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md new file mode 100644 index 0000000000..7c9e32debf --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md @@ -0,0 +1,227 @@ +## Description + +This module creates a nodeset data structure intended to be input to the +[schedmd-slurm-gcp-v6-partition](../schedmd-slurm-gcp-v6-partition/) module. + +Nodesets allow adding heterogeneous node types to a partition, and hence +running jobs that mix multiple node characteristics. See the [heterogeneous jobs +section][hetjobs] of the SchedMD documentation for more information. + +To specify nodes from a specific nodesets in a partition, the [`--nodelist`] +(or `-w`) flag can be used, for example: + +```bash +srun -N 3 -p compute --nodelist cluster-compute-group-[0-2] hostname +``` + +Where the 3 nodes will be selected from the nodes `cluster-compute-group-[0-2]` +in the compute partition. + +Additionally, depending on how the nodes differ, a constraint can be added via +the [`--constraint`] (or `-C`) flag or other flags such as `--mincpus` can be +used to specify nodes with the desired characteristics. + +[`--nodelist`]: https://slurm.schedmd.com/srun.html#OPT_nodelist +[`--constraint`]: https://slurm.schedmd.com/srun.html#OPT_constraint +[hetjobs]: https://slurm.schedmd.com/heterogeneous_jobs.html + +### Example + +The following code snippet creates a partition module using the `nodeset` +module as input with: + +* a max node count of 200 +* VM machine type of `c2-standard-30` +* partition name of "compute" +* default nodeset name of "ghpc" +* connected to the `network` module via `use` +* nodes mounted to homefs via `use` + +```yaml +- id: nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + node_count_dynamic_max: 200 + machine_type: c2-standard-30 + +- id: compute_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + use: + - homefs + - nodeset + settings: + partition_name: compute +``` + +## Custom Images + +For more information on creating valid custom images for the node group VM +instances or for custom instance templates, see our [vm-images.md] documentation +page. + +[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images + +## GPU Support + +More information on GPU support in Slurm on GCP and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) + +### Compute VM Zone Policies + +The Slurm on GCP nodeset module allows you to specify additional zones in +which to create VMs through [bulk creation][bulk]. This is valuable when +configuring partitions with popular VM families and you desire access to +more compute resources across zones. + +[bulk]: https://cloud.google.com/compute/docs/instances/multiple/about-bulk-creation +[networkpricing]: https://cloud.google.com/vpc/network-pricing + +> **_WARNING:_** Lenient zone policies can lead to additional egress costs when +> moving large amounts of data between zones in the same region. For example, +> traffic between VMs and traffic from VMs to shared filesystems such as +> Filestore. For more information on egress fees, see the +> [Network Pricing][networkpricing] Google Cloud documentation. +> +> To avoid egress charges, ensure your compute nodes are created in a single +> zone by setting var.zone and leaving var.zones to its default value of the +> empty list. +> +> **_NOTE:_** If a new zone is added to the region while the cluster is active, +> nodes in the partition may be created in that zone. In this case, the +> partition may need to be redeployed to ensure the newly added zone is denied. + +In the zonal example below, the nodeset's zone implicitly defaults to the +deployment variable `vars.zone`: + +```yaml +vars: + zone: us-central1-f + +- id: zonal-nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset +``` + +In the example below, we enable creation in additional zones: + +```yaml +vars: + zone: us-central1-f + +- id: multi-zonal-nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + settings: + zones: + - us-central1-a + - us-central1-b +``` + +## Support +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4 | +| [google](#requirement\_google) | >= 5.11 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 5.11 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.machine_type_zone_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [google_compute_machine_types.machine_types_by_zone](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | +| [google_compute_reservation.reservation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_topology](#input\_accelerator\_topology) | Specifies the shape of the Accelerator (GPU/TPU) slice. | `string` | `null` | no | +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = optional(string)
    subnetwork         = string
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    stack_type         = optional(string)
    queue_count        = optional(number)
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | +| [dws\_flex](#input\_dws\_flex) | If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes.
See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler
Options:
- enable: Enable DWS Flex Start
- max\_run\_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week).
- use\_job\_duration: Use the job duration to determine the max\_run\_duration, if job duration is not set, max\_run\_duration will be used.
- use\_bulk\_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances

Limitations:
- CAN NOT be used with reservations;
- CAN NOT be used with placement groups; | 
object({
    enabled          = optional(bool, true)
    max_run_duration = optional(number, 604800) # one week
    use_job_duration = optional(bool, false)
    use_bulk_insert  = optional(bool, false)
  })
| 
{
  "enabled": false
}
| no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_maintenance\_reservation](#input\_enable\_maintenance\_reservation) | Enables slurm reservation for scheduled maintenance. | `bool` | `false` | no | +| [enable\_opportunistic\_maintenance](#input\_enable\_opportunistic\_maintenance) | On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_placement](#input\_enable\_placement) | Use placement policy for VMs in this nodeset.
See: https://cloud.google.com/compute/docs/instances/placement-policies-overview
To set max\_distance of used policy, use `placement_max_distance` variable.

Enabled by default, reasons for users to disable it:
- If non-dense reservation is used, user can avoid extra-cost of creating placement policies;
- If user wants to avoid "all or nothing" VM provisioning behaviour;
- If user wants to intentionally have "spread" VMs (e.g. for reliability reasons) | `bool` | `true` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | +| [future\_reservation](#input\_future\_reservation) | If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT\_ID/zones/ZONE/futureReservations/FUTURE\_RESERVATION\_NAME'.
See https://cloud.google.com/compute/docs/instances/future-reservations-overview | `string` | `""` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_properties](#input\_instance\_properties) | Override the instance properties. Used to test features not supported by Slurm GCP,
recommended for advanced usage only.
See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert
If any sub-field (e.g. scheduling) is set, it will override the values computed by
SlurmGCP and ignoring values of provided vars. | `any` | `null` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for compute nodes. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | +| [maintenance\_interval](#input\_maintenance\_interval) | Sets the maintenance interval for instances in this nodeset.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. | `string` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_conf](#input\_node\_conf) | Map of Slurm node line configuration. | `map(any)` | `{}` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling nodes allowed in this partition. | `number` | `10` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | +| [placement\_max\_distance](#input\_placement\_max\_distance) | Maximum distance between nodes in the placement group. Requires enable\_placement to be true. Values must be supported by the chosen machine type. | `number` | `null` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | +| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]
- RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]

Must be a "SPECIFIC" reservation
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [zone](#input\_zone) | Zone in which to create compute VMs. Additional zones in the same region can be specified in var.zones. | `string` | n/a | yes | +| [zone\_target\_shape](#input\_zone\_target\_shape) | Strategy for distributing VMs across zones in a region.
ANY
GCE picks zones for creating VM instances to fulfill the requested number of VMs
within present resource constraints and to maximize utilization of unused zonal
reservations.
ANY\_SINGLE\_ZONE (default)
GCE always selects a single zone for all the VMs, optimizing for resource quotas,
available reservations and general capacity.
BALANCED
GCE prioritizes acquisition of resources, scheduling VMs in zones where resources
are available while distributing VMs as evenly as possible across allowed zones
to minimize the impact of zonal failure. | `string` | `"ANY_SINGLE_ZONE"` | no | +| [zones](#input\_zones) | Additional zones in which to allow creation of partition nodes. Google Cloud
will find zone based on availability, quota and reservations.
Should not be set if SPECIFIC reservation is used. | `set(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf new file mode 100644 index 0000000000..da6aae33ee --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf @@ -0,0 +1,232 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) +} + +module "instance_validation" { + source = "../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata + ) + + name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.disk_resource_manager_tags + } + ] + + public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + ghpc_startup_script = [{ + filename = "ghpc_nodeset_startup.sh" + content = var.startup_script + }] + + termination_action = (var.dws_flex.enabled && !var.dws_flex.use_bulk_insert) ? "DELETE" : try(var.spot_instance_config.termination_action, null) + + nodeset = { + node_count_static = var.node_count_static + node_count_dynamic_max = var.node_count_dynamic_max + node_conf = var.node_conf + nodeset_name = local.name + dws_flex = var.dws_flex + + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(local.labels, var.disk_labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + + bandwidth_tier = var.bandwidth_tier + can_ip_forward = var.can_ip_forward + + enable_confidential_vm = var.enable_confidential_vm + enable_placement = var.enable_placement + placement_max_distance = var.placement_max_distance + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + gpu = one(local.guest_accelerator) + accelerator_topology = var.accelerator_topology + + labels = local.labels + machine_type = terraform_data.machine_type_zone_validation.output + advanced_machine_features = var.advanced_machine_features + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + region = var.region + resource_manager_tags = var.resource_manager_tags + service_account = local.service_account + shielded_instance_config = var.shielded_instance_config + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + subnetwork_self_link = var.subnetwork_self_link + additional_networks = var.additional_networks + access_config = local.access_config + tags = var.tags + spot = var.enable_spot_vm + termination_action = local.termination_action + reservation_name = local.reservation_name + future_reservation = local.future_reservation + maintenance_interval = var.maintenance_interval + instance_properties_json = jsonencode(var.instance_properties) + + zone_target_shape = var.zone_target_shape + zone_policy_allow = local.zones + zone_policy_deny = local.zones_deny + + startup_script = local.ghpc_startup_script + network_storage = var.network_storage + + enable_maintenance_reservation = var.enable_maintenance_reservation + enable_opportunistic_maintenance = var.enable_opportunistic_maintenance + } +} + +locals { + zones = setunion(var.zones, [var.zone]) + zones_deny = setsubtract(data.google_compute_zones.available.names, local.zones) +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region + + lifecycle { + postcondition { + condition = length(setsubtract(local.zones, self.names)) == 0 + error_message = <<-EOD + Invalid zones=${jsonencode(setsubtract(local.zones, self.names))} + Available zones=${jsonencode(self.names)} + EOD + } + } +} + +locals { + res_match = regex("^(?P(?Pprojects/(?P[a-z0-9-]+)/reservations/)?(?P[a-z0-9-]+)(?P/reservationBlocks/[a-z0-9-]+)?)?$", var.reservation_name) + + res_short_name = local.res_match.name + res_project = coalesce(local.res_match.project, var.project_id) + res_prefix = coalesce(local.res_match.prefix, "projects/${local.res_project}/reservations/") + res_suffix = local.res_match.suffix == null ? "" : local.res_match.suffix + + reservation_name = local.res_match.whole == null ? "" : "${local.res_prefix}${local.res_short_name}${local.res_suffix}" +} + +locals { + fr_match = regex("^(?Pprojects/(?P[a-z0-9-]+)/zones/(?P[a-z0-9-]+)/futureReservations/)?(?P[a-z0-9-]+)?$", var.future_reservation) + + fr_name = local.fr_match.name + fr_project = coalesce(local.fr_match.project, var.project_id) + fr_zone = coalesce(local.fr_match.zone, var.zone) + + future_reservation = var.future_reservation == "" ? "" : "projects/${local.fr_project}/zones/${local.fr_zone}/futureReservations/${local.fr_name}" +} + + +# tflint-ignore: terraform_unused_declarations +data "google_compute_reservation" "reservation" { + count = length(local.reservation_name) > 0 ? 1 : 0 + + name = local.res_short_name + project = local.res_project + zone = var.zone + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "Couldn't find the reservation ${var.reservation_name}" + } + + postcondition { + condition = coalesce(self.specific_reservation_required, true) + error_message = < 0] +} + +resource "terraform_data" "machine_type_zone_validation" { + input = var.machine_type + lifecycle { + precondition { + condition = length(local.zones_with_machine_type) > 0 + error_message = <<-EOT + machine type ${var.machine_type} is not available in any of the zones ${jsonencode(local.zones)}". To list zones in which it is available, run: + + gcloud compute machine-types list --filter="name=${var.machine_type}" + EOT + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml new file mode 100644 index 0000000000..95b6d1c730 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name + has_to_be_used: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf new file mode 100644 index 0000000000..18ed74e2d5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf @@ -0,0 +1,112 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset" { + description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset + + precondition { + condition = !contains([ + "c3-:pd-standard", + "h3-:pd-standard", + "h3-:pd-ssd", + ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") + error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." + } + + precondition { + condition = var.reservation_name == "" || length(var.zones) == 0 + error_message = <<-EOD + If a reservation is specified, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.accelerator_topology == null || var.enable_placement + error_message = "accelerator_topology requires enable_placement to be set to true." + } + + precondition { + condition = (var.accelerator_topology == null) || try(tonumber(split("x", var.accelerator_topology)[1]) % local.guest_accelerator[0].count == 0, false) + error_message = "accelerator_topology must be divisible by number of gpus in machine." + } + + precondition { + condition = var.placement_max_distance == null || var.enable_placement + error_message = "placement_max_distance requires enable_placement to be set to true." + } + + precondition { + condition = !(startswith(var.machine_type, "a3-") && var.placement_max_distance == 1) + error_message = "A3 machines do not support a placement_max_distance of 1." + } + + precondition { + condition = var.reservation_name == "" || !var.dws_flex.enabled + error_message = "Cannot use reservations with DWS Flex." + } + + precondition { + condition = !var.enable_placement || !var.dws_flex.enabled + error_message = "Cannot use DWS Flex with `enable_placement`." + } + + precondition { + condition = length(var.zones) == 0 || !var.dws_flex.enabled + error_message = <<-EOD + If a DWS Flex is enabled, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.on_host_maintenance == "TERMINATE" || !var.dws_flex.enabled + error_message = "If DWS Flex is used, `on_host_maintenance` should be set to 'TERMINATE'" + } + + precondition { + condition = !var.enable_spot_vm || !var.dws_flex.enabled + error_message = "Cannot use both Flex-Start and Spot VMs for provisioning." + } + + precondition { + condition = var.reservation_name == "" || var.future_reservation == "" + error_message = "Cannot use reservations and future reservations in the same nodeset" + } + + precondition { + condition = !var.enable_placement || var.future_reservation == "" + error_message = "Cannot use `enable_placement` with future reservations." + } + + precondition { + condition = var.future_reservation == "" || length(var.zones) == 0 + error_message = <<-EOD + If a future reservation is specified, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.future_reservation == "" || local.fr_zone == var.zone + error_message = <<-EOD + The zone of the deployment must match that of the future reservation + EOD + } + + precondition { + condition = var.node_count_dynamic_max > 0 || var.node_count_static > 0 + error_message = <<-EOD + This nodeset contains zero nodes, there should be at least one static or dynamic node + EOD + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf new file mode 100644 index 0000000000..06ef5aac6f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf @@ -0,0 +1,641 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "node_conf" { + description = "Map of Slurm node line configuration." + type = map(any) + default = {} + validation { + condition = lookup(var.node_conf, "Sockets", null) == null + error_message = <<-EOD + `Sockets` field is in conflict with `SocketsPerBoard` which is automatically generated by SlurmGCP. + Instead, you can override the following fields: `Boards`, `SocketsPerBoard`, `CoresPerSocket`, and `ThreadsPerCore`. + See: https://slurm.schedmd.com/slurm.conf.html#OPT_Boards and https://slurm.schedmd.com/slurm.conf.html#OPT_Sockets_1 + EOD + } +} + +variable "node_count_static" { + description = "Number of nodes to be statically created." + type = number + default = 0 +} + +variable "node_count_dynamic_max" { + description = "Maximum number of auto-scaling nodes allowed in this partition." + type = number + default = 10 +} + +## VM Definition +variable "instance_template" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Instance template can not be specified for compute nodes." + type = string + default = null + validation { + condition = var.instance_template == null + error_message = "DEPRECATED: Instance template can not be specified for compute nodes." + } +} + +variable "machine_type" { + description = "Compute Platform machine type to use for this partition compute nodes." + type = string + default = "c2-standard-60" +} + +variable "metadata" { + type = map(string) + description = "Metadata, provided as a map." + default = {} +} + +variable "instance_image" { + description = <<-EOD + Defines the image that will be used in the Slurm node group VM instances. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + + For more information on creating custom images that comply with Slurm on GCP + see the "Slurm on GCP Custom Images" section in docs/vm-images.md. + EOD + type = map(string) + default = { + family = "slurm-gcp-6-11-hpc-rocky-linux-8" + project = "schedmd-slurm-public" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations + description = <<-EOD + A flag that designates that the user is aware that they are requesting + to use a custom and potentially incompatible image for this Slurm on + GCP module. + + If the field is set to false, only the compatible families and project + names will be accepted. The deployment will fail with any other image + family or name. If set to true, no checks will be done. + + See: https://goo.gle/hpc-slurm-images + EOD + type = bool + default = false +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "tags" { + type = list(string) + description = "Network tag list." + default = [] +} + +variable "disk_type" { + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + type = string + default = "pd-standard" +} + +variable "disk_size_gb" { + description = "Size of boot disk to create for the partition compute nodes." + type = number + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string)) + auto_delete = optional(bool) + boot = optional(bool) + disk_resource_manager_tags = optional(map(string)) + })) + default = [] +} + +variable "enable_confidential_vm" { + type = bool + description = "Enable the Confidential VM configuration. Note: the instance image must support option." + default = false +} + +variable "enable_shielded_vm" { + type = bool + description = "Enable the Shielded VM configuration. Note: the instance image must support option." + default = false +} + +variable "shielded_instance_config" { + type = object({ + enable_integrity_monitoring = bool + enable_secure_boot = bool + enable_vtpm = bool + }) + description = <<-EOD + Shielded VM configuration for the instance. Note: not used unless + enable_shielded_vm is 'true'. + - enable_integrity_monitoring : Compare the most recent boot measurements to the + integrity policy baseline and return a pair of pass/fail results depending on + whether they match or not. + - enable_secure_boot : Verify the digital signature of all boot components, and + halt the boot process if signature verification fails. + - enable_vtpm : Use a virtualized trusted platform module, which is a + specialized computer chip you can use to encrypt objects like keys and + certificates. + EOD + default = { + enable_integrity_monitoring = true + enable_secure_boot = true + enable_vtpm = true + } +} + + +variable "enable_oslogin" { + type = bool + description = <<-EOD + Enables Google Cloud os-login for user login and authentication for VMs. + See https://cloud.google.com/compute/docs/oslogin + EOD + default = true +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example." + type = bool + default = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "labels" { + description = "Labels to add to partition compute instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "The name of the minimum CPU platform that you want the instance to use." + type = string + default = null +} + +variable "on_host_maintenance" { + type = string + description = <<-EOD + Instance availability Policy. + + Note: Placement groups are not supported when on_host_maintenance is set to + "MIGRATE" and will be deactivated regardless of the value of + enable_placement. To support enable_placement, ensure on_host_maintenance is + set to "TERMINATE". + EOD + default = "TERMINATE" +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." + } +} + +variable "accelerator_topology" { + type = string + description = "Specifies the shape of the Accelerator (GPU/TPU) slice." + nullable = true + default = null +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + + +variable "service_account_email" { + description = "Service account e-mail address to attach to the compute instances." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the compute instances." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "service_account" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + } +} + +variable "enable_spot_vm" { + description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." + type = bool + default = false +} + +variable "spot_instance_config" { + description = "Configuration for spot VMs." + type = object({ + termination_action = string + }) + default = null +} + +variable "bandwidth_tier" { + description = < 0 + error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME[/reservationBlocks/BLOCK_ID]', [...] are optional parts." + } +} + +variable "future_reservation" { + description = <<-EOD + If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zones/ZONE/futureReservations/FUTURE_RESERVATION_NAME'. + See https://cloud.google.com/compute/docs/instances/future-reservations-overview + EOD + type = string + default = "" + nullable = false + + validation { + condition = length(regexall("^(projects/([a-z0-9-]+)/zones/([a-z0-9-]+)/futureReservations/([a-z0-9-]+))?$", var.future_reservation)) > 0 || length(regexall("^([a-z0-9-]+)$", var.future_reservation)) > 0 + error_message = "Future reservation must be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zone/ZONE/futureReservations/FUTURE_RESERVATION_NAME'." + } +} + +variable "maintenance_interval" { + description = <<-EOD + Sets the maintenance interval for instances in this nodeset. + See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. + EOD + type = string + default = null +} + +variable "startup_script" { + description = "Startup script used by VMs in this nodeset" + type = string + default = "# no-op" +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + })) + default = [] +} + + +variable "instance_properties" { + description = <<-EOD + Override the instance properties. Used to test features not supported by Slurm GCP, + recommended for advanced usage only. + See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert + If any sub-field (e.g. scheduling) is set, it will override the values computed by + SlurmGCP and ignoring values of provided vars. + EOD + type = any + default = null +} + + +variable "enable_maintenance_reservation" { + type = bool + description = "Enables slurm reservation for scheduled maintenance." + default = false +} + + +variable "enable_opportunistic_maintenance" { + type = bool + description = "On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle." + default = false +} + + +variable "dws_flex" { + description = <<-EOD + If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes. + See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler + Options: + - enable: Enable DWS Flex Start + - max_run_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week). + - use_job_duration: Use the job duration to determine the max_run_duration, if job duration is not set, max_run_duration will be used. + - use_bulk_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances + + Limitations: + - CAN NOT be used with reservations; + - CAN NOT be used with placement groups; + + EOD + + type = object({ + enabled = optional(bool, true) + max_run_duration = optional(number, 604800) # one week + use_job_duration = optional(bool, false) + use_bulk_insert = optional(bool, false) + }) + default = { + enabled = false + } + validation { + condition = var.dws_flex.max_run_duration >= 600 && var.dws_flex.max_run_duration <= 604800 + error_message = "Max duration must be at least than 10 minutes, and cannot be more than one week." + } +} + +variable "placement_max_distance" { + type = number + description = "Maximum distance between nodes in the placement group. Requires enable_placement to be true. Values must be supported by the chosen machine type." + nullable = true + default = null + + validation { + condition = coalesce(var.placement_max_distance, 1) >= 1 && coalesce(var.placement_max_distance, 3) <= 3 + error_message = "Invalid value for placement_max_distance. Valid values are null, 1, 2, or 3." + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf new file mode 100644 index 0000000000..e014c318e4 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.4" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.11" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.74.0" + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md new file mode 100644 index 0000000000..d3dbcd959e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md @@ -0,0 +1,105 @@ +## Description + +This module creates a compute partition that can be used as input to the +[schedmd-slurm-gcp-v6-controller](../../scheduler/schedmd-slurm-gcp-v6-controller/README.md). + +The partition module is designed to work alongside the +[schedmd-slurm-gcp-v6-nodeset](../schedmd-slurm-gcp-v6-nodeset/README.md) +module. A partition can be made up of one or +more nodesets, provided either through `use` (preferred) or defined manually +in the `nodeset` variable. + +### Example + +The following code snippet creates a partition module with: + +* 2 nodesets added via `use`. + * The first nodeset is made up of machines of type `c2-standard-30`. + * The second nodeset is made up of machines of type `c2-standard-60`. + * Both nodesets have a maximum count of 200 dynamically created nodes. +* partition name of "compute". +* connected to the `network` module via `use`. +* nodes mounted to homefs via `use`. + +```yaml +- id: nodeset_1 + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + name: c30 + node_count_dynamic_max: 200 + machine_type: c2-standard-30 + +- id: nodeset_2 + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + name: c60 + node_count_dynamic_max: 200 + machine_type: c2-standard-60 + +- id: compute_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + use: + - homefs + - nodeset_1 + - nodeset_2 + settings: + partition_name: compute +``` + +## Support + +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [exclusive](#input\_exclusive) | Exclusive job access to nodes. When set to true nodes execute single job and are deleted
after job exits. If set to false, multiple jobs can be scheduled on one node. | `bool` | `true` | no | +| [is\_default](#input\_is\_default) | Sets this partition as the default partition by updating the partition\_conf.
If "Default" is already set in partition\_conf, this variable will have no effect. | `bool` | `false` | no | +| [network\_storage](#input\_network\_storage) | DEPRECATED | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | A list of nodesets.
For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip     = string
      remote_mount  = string
      local_mount   = string
      fs_type       = string
      mount_options = string
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | +| [partition\_conf](#input\_partition\_conf) | Slurm partition configuration as a map.
See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION | `map(string)` | `{}` | no | +| [partition\_name](#input\_partition\_name) | The name of the slurm partition. | `string` | n/a | yes | +| [resume\_timeout](#input\_resume\_timeout) | Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'ResumeTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. | `number` | `null` | no | +| [suspend\_time](#input\_suspend\_time) | Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram.
This sets 'SuspendTime' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details.
NOTE: use value -1 to exclude partition from suspend.
NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. | `number` | `300` | no | +| [suspend\_timeout](#input\_suspend\_timeout) | Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'SuspendTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. | `number` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Details of a nodesets in this partition | +| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of a dynamic nodesets in this partition | +| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of a TPU nodesets in this partition | +| [partitions](#output\_partitions) | Details of a slurm partition | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf new file mode 100644 index 0000000000..1618c64280 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf @@ -0,0 +1,41 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + use_static = [for ns in concat(var.nodeset, var.nodeset_tpu) : ns.nodeset_name if ns.node_count_static > 0] + + has_node = length(var.nodeset) > 0 + has_dyn = length(var.nodeset_dyn) > 0 + has_tpu = length(var.nodeset_tpu) > 0 + has_flex = length([for ns in var.nodeset : ns.dws_flex.enabled if ns.dws_flex.enabled]) > 0 +} + +locals { + partition_conf = merge({ + "Default" = var.is_default ? "YES" : null + "SuspendTime" = var.suspend_time < 0 ? "INFINITE" : var.suspend_time + "SuspendTimeout" = var.suspend_timeout != null ? var.suspend_timeout : (local.has_tpu ? 240 : 120) + }, var.partition_conf, { "ResumeTimeout" = local.has_flex ? 65535 : try(var.partition_conf["ResumeTimeout"], coalesce(var.resume_timeout, (local.has_tpu ? 600 : 300))) }) + + partition = { + partition_name = var.partition_name + partition_conf = local.partition_conf + + partition_nodeset = [for ns in var.nodeset : ns.nodeset_name] + partition_nodeset_tpu = [for ns in var.nodeset_tpu : ns.nodeset_name] + partition_nodeset_dyn = [for ns in var.nodeset_dyn : ns.nodeset_name] + # Options + enable_job_exclusive = var.exclusive + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml new file mode 100644 index 0000000000..13ea127b3c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + has_to_be_used: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf new file mode 100644 index 0000000000..35dece64fb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf @@ -0,0 +1,54 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "partitions" { + description = "Details of a slurm partition" + + value = [local.partition] + + precondition { + condition = (length(local.use_static) == 0) || !var.exclusive + error_message = <<-EOD + Can't use static nodes within partition with `var.exclusive` set to `true`. + NOTE: Partition's `var.exclusive` is set to `true` by default. Set it to `false` explicitly to use static nodes. + EOD + } + + precondition { + # Can not mix TPU with other non-TPU nodesets due to SlurmGCP specific limitations; + # Can not mix dynamic with non-dynamic nodesets due to Slurms inability to + # turn off "power management" at nodeset level (can only do it at partition or node level). + condition = sum([for b in [local.has_node, local.has_dyn, local.has_tpu] : b ? 1 : 0]) == 1 + error_message = "Partition must contain exactly one type of nodeset." + } +} + +output "nodeset" { + description = "Details of a nodesets in this partition" + + value = var.nodeset +} + +output "nodeset_tpu" { + description = "Details of a TPU nodesets in this partition" + + value = var.nodeset_tpu +} + + +output "nodeset_dyn" { + description = "Details of a dynamic nodesets in this partition" + + value = var.nodeset_dyn +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf new file mode 100644 index 0000000000..a1c85adb90 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf @@ -0,0 +1,311 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "partition_name" { + description = "The name of the slurm partition." + type = string + + validation { + condition = can(regex("^[a-z](?:[a-z0-9]*)$", var.partition_name)) + error_message = "Variable 'partition_name' must be a match of regex '^[a-z](?:[a-z0-9]*)$'." + } +} + +variable "partition_conf" { + description = <<-EOD + Slurm partition configuration as a map. + See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION + EOD + type = map(string) + default = {} +} + +variable "is_default" { + description = <<-EOD + Sets this partition as the default partition by updating the partition_conf. + If "Default" is already set in partition_conf, this variable will have no effect. + EOD + type = bool + default = false +} + +variable "exclusive" { + description = <<-EOD + Exclusive job access to nodes. When set to true nodes execute single job and are deleted + after job exits. If set to false, multiple jobs can be scheduled on one node. + EOD + type = bool + default = true +} + +variable "nodeset" { + description = <<-EOD + A list of nodesets. + For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset + EOD + type = list(object({ + node_count_static = optional(number, 0) + node_count_dynamic_max = optional(number, 1) + node_conf = optional(map(string), {}) + nodeset_name = string + additional_disks = optional(list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string), {}) + auto_delete = optional(bool, true) + boot = optional(bool, false) + disk_resource_manager_tags = optional(map(string), {}) + })), []) + bandwidth_tier = optional(string, "platform_default") + can_ip_forward = optional(bool, false) + disk_auto_delete = optional(bool, true) + disk_labels = optional(map(string), {}) + disk_resource_manager_tags = optional(map(string), {}) + disk_size_gb = optional(number) + disk_type = optional(string) + enable_confidential_vm = optional(bool, false) + enable_placement = optional(bool, false) + placement_max_distance = optional(number, null) + enable_oslogin = optional(bool, true) + enable_shielded_vm = optional(bool, false) + enable_maintenance_reservation = optional(bool, false) + enable_opportunistic_maintenance = optional(bool, false) + gpu = optional(object({ + count = number + type = string + })) + accelerator_topology = optional(string, null) + dws_flex = object({ + enabled = bool + max_run_duration = number + use_job_duration = bool + use_bulk_insert = bool + }) + labels = optional(map(string), {}) + machine_type = optional(string) + advanced_machine_features = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + maintenance_interval = optional(string) + instance_properties_json = string + metadata = optional(map(string), {}) + min_cpu_platform = optional(string) + network_tier = optional(string, "STANDARD") + network_storage = optional(list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = optional(map(string)) + mount_runner = optional(map(string)) + })), []) + on_host_maintenance = optional(string) + preemptible = optional(bool, false) + region = optional(string) + resource_manager_tags = optional(map(string), {}) + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + shielded_instance_config = optional(object({ + enable_integrity_monitoring = optional(bool, true) + enable_secure_boot = optional(bool, true) + enable_vtpm = optional(bool, true) + })) + source_image_family = optional(string) + source_image_project = optional(string) + source_image = optional(string) + subnetwork_self_link = string + additional_networks = optional(list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + }))) + access_config = optional(list(object({ + nat_ip = string + network_tier = string + }))) + spot = optional(bool, false) + tags = optional(list(string), []) + termination_action = optional(string) + reservation_name = optional(string) + future_reservation = string + startup_script = optional(list(object({ + filename = string + content = string })), []) + + zone_target_shape = string + zone_policy_allow = set(string) + zone_policy_deny = set(string) + })) + default = [] + + validation { + condition = length(distinct(var.nodeset[*].nodeset_name)) == length(var.nodeset) + error_message = "All nodesets must have a unique name." + } +} + +variable "nodeset_tpu" { + description = "Define TPU nodesets, as a list." + type = list(object({ + node_count_static = optional(number, 0) + node_count_dynamic_max = optional(number, 5) + nodeset_name = string + enable_public_ip = optional(bool, false) + node_type = string + accelerator_config = optional(object({ + topology = string + version = string + }), { + topology = "" + version = "" + }) + tf_version = string + preemptible = optional(bool, false) + preserve_tpu = optional(bool, false) + zone = string + data_disks = optional(list(string), []) + docker_image = optional(string, "") + network_storage = optional(list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + })), []) + subnetwork = string + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + project_id = string + reserved = optional(string, false) + })) + default = [] + + validation { + condition = length(distinct([for x in var.nodeset_tpu : x.nodeset_name])) == length(var.nodeset_tpu) + error_message = "All TPU nodesets must have a unique name." + } +} + +variable "nodeset_dyn" { + description = "Defines dynamic nodesets, as a list." + type = list(object({ + nodeset_name = string + nodeset_feature = string + })) + default = [] + + validation { + condition = length(distinct([for x in var.nodeset_dyn : x.nodeset_name])) == length(var.nodeset_dyn) + error_message = "All dynamic nodesets must have a unique name." + } +} + +variable "resume_timeout" { + description = <<-EOD + Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use. + If null is given, then a smart default will be chosen depending on nodesets in partition. + This sets 'ResumeTimeout' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. + EOD + type = number + default = null + + validation { + condition = var.resume_timeout == null ? true : var.resume_timeout > 0 && var.resume_timeout < 65536 + error_message = "Value must be > 0 and < 65536" + } +} + +variable "suspend_time" { + description = <<-EOD + Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram. + This sets 'SuspendTime' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details. + NOTE: use value -1 to exclude partition from suspend. + NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. + EOD + type = number + default = 300 + + validation { + condition = var.suspend_time >= -1 + error_message = "Value must be >= -1." + } +} + +variable "suspend_timeout" { + description = <<-EOD + Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown. + If null is given, then a smart default will be chosen depending on nodesets in partition. + This sets 'SuspendTimeout' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. + EOD + type = number + default = null + + validation { + condition = var.suspend_timeout == null ? true : var.suspend_timeout > 0 + error_message = "Value must be > 0." + } +} + + +# tflint-ignore: terraform_unused_declarations +variable "network_storage" { + description = "DEPRECATED" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] + validation { + condition = length(var.network_storage) == 0 + error_message = <<-EOD + network_storage in partition module is deprecated and should not be set. + To add network storage to compute nodes, use network_storage of nodeset module instead. + EOD + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf new file mode 100644 index 0000000000..d388f4bfdd --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.3" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.74.0" + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/README.md b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/README.md new file mode 100644 index 0000000000..994f1500ba --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/README.md @@ -0,0 +1,157 @@ +## Description + +This module provides ways to create and manage Google Cloud Artifact Registry repositories. + +Currently this module is built to support repositories in Docker format although there are placeholder variables for other types which may work too. Remote repositories with pull-through cache functionality integrated with Google Secret Manager is currently supported. The aim of this module is to eventually offer feature parity with this [Terraform module](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository#nested_remote_repository_config), allowing creation of repositories in various formats, including Docker, Maven, NPM, Python, APT, YUM, and COMMON. + +This module is best suited for managing artifact repositories in HPC/AI containerized environments where artifacts need to be shared across distributed systems. It includes IAM role configurations and secret access handling for seamless integration with CI/CD pipelines and other services too. + +It is designed to help facilitate containerized workloads running in the Cluster Toolkit with SLURM leveraging [Enroot](https://github.com/NVIDIA/enroot) and [Pyxis](https://github.com/NVIDIA/pyxis). Docker repositories can store container images that are used in job submissions, enabling efficient and scalable execution of containerized HPC or AI based workloads. + +## Usage + +### Service Account / APIs + +You will need to enable the relevant APIs and create a Service Account for your cluster with the following Artifact Registry permissions. + +```yaml + - id: services-api + source: community/modules/project/service-enablement + settings: + gcp_service_list: + - secretmanager.googleapis.com + - cloudbuild.googleapis.com + - artifactregistry.googleapis.com + + - source: community/modules/project/service-account + kind: terraform + id: hpc_service_account + settings: + project_id: project_name + name: service_account_name + project_roles: + - artifactregistry.reader + - artifactregistry.writer + - secretmanager.secretAccessor +``` + +### Deployment + +Create a standard Docker repository. + +```yaml +- id: registry + source: community/modules/container/artifact-registry + settings: + repo_mode: STANDARD_REPOSITORY + format: DOCKER +``` + +Mirror of public Docker Hub repository. + +```yaml +- id: dockerhub_registry + source: community/modules/container/artifact-registry + settings: + repo_mode: REMOTE_REPOSITORY + format: DOCKER + repo_public_repository: DOCKER_HUB +``` + +Mirror of NVIDIA's [NGC Catalog](https://catalog.ngc.nvidia.com/containers). [API key](https://org.ngc.nvidia.com/setup/api-key) used in blueprint is stored in Secret Manager. + +```yaml +- id: ngc_registry + source: community/modules/container/artifact-registry + settings: + repo_mode: REMOTE_REPOSITORY + format: DOCKER + repo_mirror_url: "https://nvcr.io" + repo_username: $oauthtoken + repo_password: api_key_here + use_upstream_credentials: True +``` + +### Container Operations + +Retrieve `$REPOSITORY_NAME` from [Artifact Registry](https://console.cloud.google.com/artifacts) or by using `gcloud`. + +```yaml +gcloud artifacts repositories list --project="${PROJECT_ID}" +``` + +Pulling containers from your mirrored internal Artifact Repositories. + +Pull [Ubuntu](https://hub.docker.com/_/ubuntu) from Docker Hub mirror. + +```yaml +docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/library/ubuntu:latest +``` + +Pull [Pytorch](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch) from NGC Catalog mirror. + +```yaml +docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/nvidia/pytorch:24.11-py3 +``` + +Alternatively, proceed with running SLURM's [NVIDIA/pyxis](https://github.com/NVIDIA/pyxis) plugin, which will now be able to pull and use these containers directly from the mirrored repositories. + +Note: only Docker registries have been tested so far. Placeholders do exist for other registry types which may or may not work. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_artifact_registry_repository.artifact_registry](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository) | resource | +| [google_secret_manager_secret.repo_password_secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_version.repo_password_secret_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [random_password.repo_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | +| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment. | `string` | n/a | yes | +| [format](#input\_format) | Artifact Registry format (e.g., DOCKER). | `string` | `"DOCKER"` | no | +| [labels](#input\_labels) | Labels to add to the artifact registry. Key-value pairs. | `map(string)` | `{}` | no | +| [project\_id](#input\_project\_id) | Project ID where the artifact registry and secret are created. | `string` | n/a | yes | +| [region](#input\_region) | Region for the artifact registry. | `string` | n/a | yes | +| [repo\_mirror\_url](#input\_repo\_mirror\_url) | For REMOTE\_REPOSITORY, URL for a custom or common mirror. | `string` | `null` | no | +| [repo\_mode](#input\_repo\_mode) | Artifact Registry mode (STANDARD\_REPOSITORY, REMOTE\_REPOSITORY, etc.). | `string` | `"STANDARD_REPOSITORY"` | no | +| [repo\_password](#input\_repo\_password) | Optional password/API key. If null, one will be randomly generated. | `string` | `null` | no | +| [repo\_public\_repository](#input\_repo\_public\_repository) | For REMOTE\_REPOSITORY, name of a known public repo as per the Terraform module
(e.g., DOCKER\_HUB) or null for custom repo. | `string` | `null` | no | +| [repo\_username](#input\_repo\_username) | Username for external repository. | `string` | `null` | no | +| [repository\_base](#input\_repository\_base) | For APT/YUM public repos, repository\_base (e.g., 'DEBIAN', 'UBUNTU'). | `string` | `null` | no | +| [repository\_path](#input\_repository\_path) | For APT/YUM public repos, repository\_path (e.g., 'debian/dists/buster'). | `string` | `null` | no | +| [use\_upstream\_credentials](#input\_use\_upstream\_credentials) | Configure Service Account to use upstream credentials for REMOTE\_REPOSITORY:
If true, a username/password is used for the REMOTE\_REPOSITORY mirror.
If false (or if repo\_password == null), no password is created at all.
Note: Blueprint credentials will be stored in Secrets Manager. | `bool` | `false` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | (Optional) A list of objects to enable user-managed replication.
Each object can have:
location = string
kms\_key\_name = optional(string)
If empty, auto replication is used. | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [registry\_url](#output\_registry\_url) | The URL of the created artifact registry. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/main.tf b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/main.tf new file mode 100644 index 0000000000..c3406af607 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/main.tf @@ -0,0 +1,268 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "artifact-registry", ghpc_role = "container" }) +} + +locals { + # Auto (i.e., empty) vs user-managed replication + auto = length(var.user_managed_replication) == 0 ? true : false + + # For remote custom repositories, parse out host to create a base_component name + mirror_url_no_proto = var.repo_mirror_url != null ? replace(replace(var.repo_mirror_url, "https://", ""), "http://", "") : "" + mirror_host = local.mirror_url_no_proto != "" ? split("/", local.mirror_url_no_proto)[0] : "" + + base_component = replace( + replace( + replace( + lower( + local.mirror_host != "" + ? "${var.format}-${var.repo_mode}-${local.mirror_host}" + : "${var.format}-${var.repo_mode}-nohost" + ), + "\\.", "-" + ), + "/", "-" + ), + "_", "-" + ) + + repository_suffix = random_id.resource_name_suffix.hex + + # The final name for the artifact registry repository + repository_name = replace( + replace( + lower( + format("%s-%s", local.base_component, local.repository_suffix) + ), + ".", "-" + ), + "/", "-" + ) + + # The secret name is derived from the repository name + # with a suffix like "-secret". + derived_secret_name = format("%s-secret", local.repository_name) +} + +############################## +# PASSWORD / SECRET +############################## + +# Only create a random password if user didn't supply one +resource "random_password" "repo_password" { + count = var.use_upstream_credentials && var.repo_password == null ? 1 : 0 + length = 24 + special = true + override_special = "_-#=." +} + +resource "google_secret_manager_secret" "repo_password_secret" { + count = var.use_upstream_credentials ? 1 : 0 + project = var.project_id + + # Derive the secret ID from the repository name + secret_id = local.derived_secret_name + + labels = local.labels + + replication { + dynamic "auto" { + for_each = local.auto ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = local.auto ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = replicas.value.kms_key_name != null ? [1] : [] + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_version" "repo_password_secret_version" { + count = var.use_upstream_credentials ? 1 : 0 + secret = google_secret_manager_secret.repo_password_secret[0].id + + # If user provided a password, use it. Otherwise use the random password. + secret_data = var.repo_password != null ? var.repo_password : random_password.repo_password[0].result +} + +############################## +# IAM BINDINGS +############################## + +############################## +# ARTIFACT REGISTRY +############################## + +resource "random_id" "resource_name_suffix" { + byte_length = 2 +} + +resource "google_artifact_registry_repository" "artifact_registry" { + project = var.project_id + location = var.region + format = var.format + mode = var.repo_mode + description = var.deployment_name + labels = local.labels + repository_id = local.repository_name + + # Only create remote_repository_config if REMOTE_REPOSITORY + dynamic "remote_repository_config" { + for_each = var.repo_mode == "REMOTE_REPOSITORY" ? [1] : [] + content { + description = "Pull-through cache" + + dynamic "docker_repository" { + for_each = var.format == "DOCKER" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "docker_repository" { + for_each = var.format == "DOCKER" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "maven_repository" { + for_each = var.format == "MAVEN" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "maven_repository" { + for_each = var.format == "MAVEN" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "npm_repository" { + for_each = var.format == "NPM" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "npm_repository" { + for_each = var.format == "NPM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "python_repository" { + for_each = var.format == "PYTHON" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "python_repository" { + for_each = var.format == "PYTHON" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "apt_repository" { + for_each = var.format == "APT" && var.repo_public_repository != null ? [1] : [] + content { + public_repository { + repository_base = var.repository_base + repository_path = var.repository_path + } + } + } + + dynamic "apt_repository" { + for_each = var.format == "APT" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "yum_repository" { + for_each = var.format == "YUM" && var.repo_public_repository != null ? [1] : [] + content { + public_repository { + repository_base = var.repository_base + repository_path = var.repository_path + } + } + } + + dynamic "yum_repository" { + for_each = var.format == "YUM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "common_repository" { + for_each = var.format == "COMMON" ? [1] : [] + content { + uri = var.repo_mirror_url + } + } + + # Only enable upstream credentials if user wants it + dynamic "upstream_credentials" { + for_each = var.use_upstream_credentials ? [1] : [] + content { + username_password_credentials { + username = var.repo_username + password_secret_version = google_secret_manager_secret_version.repo_password_secret_version[0].name + } + } + } + } + } + + depends_on = [ + google_secret_manager_secret.repo_password_secret, + google_secret_manager_secret_version.repo_password_secret_version, + ] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/metadata.yaml new file mode 100644 index 0000000000..6b68c98a54 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - secretmanager.googleapis.com + - artifactregistry.googleapis.com + - cloudbuild.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/outputs.tf new file mode 100644 index 0000000000..92b6dbb165 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "registry_url" { + description = "The URL of the created artifact registry." + value = "${var.region}-docker.pkg.dev/${var.project_id}/${var.deployment_name}" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/validation.tf b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/validation.tf new file mode 100644 index 0000000000..a795060fb7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/validation.tf @@ -0,0 +1,49 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "terraform_data" "input_validation" { + lifecycle { + precondition { + condition = ( + var.repo_password == null || + (var.use_upstream_credentials && var.repo_mode == "REMOTE_REPOSITORY") + ) + error_message = "repo_password may be set only when repo_mode=REMOTE_REPOSITORY and use_upstream_credentials=true." + } + + precondition { + condition = ( + !var.use_upstream_credentials || + var.repo_mode == "REMOTE_REPOSITORY" + ) + error_message = "use_upstream_credentials is allowed only when repo_mode is REMOTE_REPOSITORY." + } + + precondition { + condition = ( + var.repo_mode != "REMOTE_REPOSITORY" || + (var.repo_public_repository != null || var.repo_mirror_url != null) + ) + error_message = "For a REMOTE_REPOSITORY you must set repo_public_repository or repo_mirror_url." + } + + precondition { + condition = ( + !contains(["APT", "YUM"], var.format) || + (var.repository_base != null && var.repository_path != null) + ) + error_message = "APT/YUM formats require repository_base and repository_path." + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/variables.tf new file mode 100644 index 0000000000..9a4eecb921 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/variables.tf @@ -0,0 +1,122 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Project ID where the artifact registry and secret are created." + type = string +} + +variable "region" { + description = "Region for the artifact registry." + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment." + type = string +} + +variable "labels" { + description = "Labels to add to the artifact registry. Key-value pairs." + type = map(string) + default = {} +} + +variable "repo_password" { + description = "Optional password/API key. If null, one will be randomly generated." + type = string + default = null +} + +variable "user_managed_replication" { + description = <<-DOC + (Optional) A list of objects to enable user-managed replication. + Each object can have: + location = string + kms_key_name = optional(string) + If empty, auto replication is used. + DOC + type = list(object({ + location = string + kms_key_name = optional(string) + })) + default = [] +} + +variable "format" { + description = "Artifact Registry format (e.g., DOCKER)." + type = string + default = "DOCKER" +} + +variable "repo_mode" { + description = "Artifact Registry mode (STANDARD_REPOSITORY, REMOTE_REPOSITORY, etc.)." + type = string + default = "STANDARD_REPOSITORY" + + validation { + condition = can(regex("^(STANDARD_REPOSITORY|REMOTE_REPOSITORY|VIRTUAL_REPOSITORY)$", var.repo_mode)) + error_message = "repo_mode must be one of STANDARD_REPOSITORY, REMOTE_REPOSITORY, or VIRTUAL_REPOSITORY." + } +} + +variable "repo_public_repository" { + description = <<-DOC + For REMOTE_REPOSITORY, name of a known public repo as per the Terraform module + (e.g., DOCKER_HUB) or null for custom repo. + DOC + type = string + default = null + + # To Do: implement validation + # validation { + # condition = ((var.repo_mode != "REMOTE_REPOSITORY" && var.repo_public_repository == null) || (var.repo_mode == "REMOTE_REPOSITORY" && (var.repo_public_repository != null || var.repo_mirror_url != null))) + # error_message = "If repo_mode is REMOTE_REPOSITORY, you must set either repo_public_repository or repo_mirror_url. Otherwise, leave them null." + # } +} + +variable "repo_mirror_url" { + description = "For REMOTE_REPOSITORY, URL for a custom or common mirror." + type = string + default = null +} + +variable "use_upstream_credentials" { + description = <<-DOC + Configure Service Account to use upstream credentials for REMOTE_REPOSITORY: + If true, a username/password is used for the REMOTE_REPOSITORY mirror. + If false (or if repo_password == null), no password is created at all. + Note: Blueprint credentials will be stored in Secrets Manager. + DOC + type = bool + default = false +} + +variable "repo_username" { + description = "Username for external repository." + type = string + default = null +} + +variable "repository_base" { + description = "For APT/YUM public repos, repository_base (e.g., 'DEBIAN', 'UBUNTU')." + type = string + default = null +} + +variable "repository_path" { + description = "For APT/YUM public repos, repository_path (e.g., 'debian/dists/buster')." + type = string + default = null +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/versions.tf new file mode 100644 index 0000000000..392a7131d2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/README.md b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/README.md new file mode 100644 index 0000000000..23bf87398a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/README.md @@ -0,0 +1,76 @@ +## Description + +Creates a BigQuery dataset. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage +This is a simple usage. + +```yaml + - id: bq-dataset + source: community/modules/database/bigquery-dataset + settings: + dataset_id: my_dataset +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_dataset.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | The name of the dataset to be created | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the dataset. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [dataset\_id](#output\_dataset\_id) | Name of the dataset that was created. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/main.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/main.tf new file mode 100644 index 0000000000..1a9c4bba60 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/main.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-dataset", ghpc_role = "database" }) +} +locals { + dataset_id = var.dataset_id != null ? var.dataset_id : replace("${var.deployment_name}_dataset_${random_id.resource_name_suffix.hex}", "-", "_") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_bigquery_dataset" "pbsb" { + dataset_id = local.dataset_id + project = var.project_id + labels = local.labels +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml new file mode 100644 index 0000000000..87ff9357e4 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigquery.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf new file mode 100644 index 0000000000..9cd8e5df31 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "dataset_id" { + description = "Name of the dataset that was created." + value = google_bigquery_dataset.pbsb.dataset_id +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/variables.tf new file mode 100644 index 0000000000..90c229af6b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/variables.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "dataset_id" { + description = "The name of the dataset to be created" + type = string + default = null +} + +variable "labels" { + description = "Labels to add to the dataset. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/versions.tf new file mode 100644 index 0000000000..12ddbe842d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/README.md b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/README.md new file mode 100644 index 0000000000..ef67cfef01 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/README.md @@ -0,0 +1,87 @@ +## Description + +Creates a BigQuery table with a specified schema. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage + +```yaml +id: bq-table + source: community/modules/database/bigquery-table + use: [bq-dataset] + settings: + table_schema: + ' + [ + { + "name": "id", "type": "STRING" + } + ] + ' +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_table.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Dataset name to be used to create the new BQ Table | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the tables. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [table\_id](#input\_table\_id) | Table name to be used to create the new BQ Table | `string` | `null` | no | +| [table\_schema](#input\_table\_schema) | Schema used to create the new BQ Table | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [dataset\_id](#output\_dataset\_id) | ID of BQ dataset | +| [table\_id](#output\_table\_id) | ID of created BQ table | +| [table\_name](#output\_table\_name) | Name of created BQ table | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/main.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/main.tf new file mode 100644 index 0000000000..73f3923e00 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/main.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-table", ghpc_role = "database" }) +} + +locals { + table_id = var.table_id != null ? var.table_id : replace("${var.deployment_name}_table_${random_id.resource_name_suffix.hex}", "-", "_") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_bigquery_table" "pbsb" { + deletion_protection = false + project = var.project_id + table_id = local.table_id + dataset_id = var.dataset_id + schema = var.table_schema + labels = local.labels +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/metadata.yaml new file mode 100644 index 0000000000..87ff9357e4 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigquery.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/outputs.tf new file mode 100644 index 0000000000..4220ec1390 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/outputs.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "table_name" { + description = "Name of created BQ table" + value = google_bigquery_table.pbsb.friendly_name +} +output "table_id" { + description = "ID of created BQ table" + value = google_bigquery_table.pbsb.table_id +} +output "dataset_id" { + description = "ID of BQ dataset" + value = google_bigquery_table.pbsb.dataset_id +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/variables.tf new file mode 100644 index 0000000000..ec474b4e64 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/variables.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "labels" { + description = "Labels to add to the tables. Key-value pairs." + type = map(string) +} + +variable "table_id" { + description = "Table name to be used to create the new BQ Table" + type = string + default = null +} + +variable "dataset_id" { + description = "Dataset name to be used to create the new BQ Table" + type = string +} + +variable "table_schema" { + description = "Schema used to create the new BQ Table" + type = string +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/versions.tf new file mode 100644 index 0000000000..12ddbe842d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md new file mode 100644 index 0000000000..08364c175b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md @@ -0,0 +1,107 @@ +## Description + +terraform-google-sql makes it easy to create a Google CloudSQL instance and +implement high availability settings. This module is meant for use with +Terraform 0.13+ and tested using Terraform 1.0+. + +The cloudsql created here is used to integrate with the slurm cluster to enable +accounting data storage. + +### Example + +```yaml +- id: cloudsql + source: community/modules/database/slurm-cloudsql-federation + use: [network] + settings: + sql_instance_name: slurm-sql6-demo + tier: "db-f1-micro" +``` + +This creates a cloud sql instance, including a database, user that would allow +the slurm cluster to use as an external DB. In addition, it will allow BigQuery +to run federated query through it. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | +| [google](#requirement\_google) | >= 3.83 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_connection.connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_connection) | resource | +| [google_compute_address.psc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_forwarding_rule.psc_consumer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_forwarding_rule) | resource | +| [google_sql_database.database](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database) | resource | +| [google_sql_database_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance) | resource | +| [google_sql_user.users](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [random_password.password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [authorized\_networks](#input\_authorized\_networks) | IP address ranges as authorized networks of the Cloud SQL for MySQL instances | `list(string)` | `[]` | no | +| [data\_cache\_enabled](#input\_data\_cache\_enabled) | Whether data cache is enabled for the instance. Can be used with ENTERPRISE\_PLUS edition. | `bool` | `false` | no | +| [database\_flags](#input\_database\_flags) | Database flags to set on instance. | `map(string)` | `{}` | no | +| [database\_version](#input\_database\_version) | The version of the database to be created. | `string` | `"MYSQL_8_0"` | no | +| [deletion\_protection](#input\_deletion\_protection) | Whether or not to allow Terraform to destroy the instance. | `string` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [disk\_autoresize](#input\_disk\_autoresize) | Set to false to disable automatic disk grow. | `bool` | `true` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of the database disk in GiB. | `number` | `null` | no | +| [edition](#input\_edition) | value | `string` | `"ENTERPRISE"` | no | +| [enable\_backups](#input\_enable\_backups) | Set true to enable backups | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is going to be created in.:
`projects//global/networks/`" | `string` | n/a | yes | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [query\_insights](#input\_query\_insights) | Query insights configuration. | 
object({
    enabled                 = optional(bool, false)
    query_plans_per_minute  = optional(number)
    query_string_length     = optional(number)
    record_application_tags = optional(bool)
    record_client_address   = optional(bool)
  })
| `{}` | no | +| [region](#input\_region) | The region where SQL instance will be configured | `string` | n/a | yes | +| [sql\_instance\_name](#input\_sql\_instance\_name) | name given to the sql instance for ease of identificaion | `string` | n/a | yes | +| [sql\_password](#input\_sql\_password) | Password for the SQL database. | `any` | `null` | no | +| [sql\_username](#input\_sql\_username) | Username for the SQL database | `string` | `"slurm"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self link of the network where Cloud SQL instance PSC endpoint will be created | `string` | `null` | no | +| [tier](#input\_tier) | The machine type to use for the SQL instance | `string` | n/a | yes | +| [use\_psc\_connection](#input\_use\_psc\_connection) | Create Private Service Connection instead of using Private Service Access peering | `bool` | `false` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cloudsql](#output\_cloudsql) | Describes the cloudsql instance. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf new file mode 100644 index 0000000000..9b518a1b5f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf @@ -0,0 +1,165 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "slurm-cloudsql-federation", ghpc_role = "database" }) +} + +locals { + user_managed_replication = var.user_managed_replication +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "random_password" "password" { + length = 12 + special = false +} + +locals { + sql_instance_name = var.sql_instance_name == null ? "${var.deployment_name}-sql-${random_id.resource_name_suffix.hex}" : var.sql_instance_name + sql_password = var.sql_password == null ? random_password.password.result : var.sql_password +} + + +resource "google_sql_database_instance" "instance" { + project = var.project_id + depends_on = [var.private_vpc_connection_peering] + name = local.sql_instance_name + region = var.region + deletion_protection = var.deletion_protection + database_version = var.database_version + + settings { + disk_size = var.disk_size_gb + disk_autoresize = var.disk_autoresize + edition = var.edition + tier = var.tier + user_labels = local.labels + + dynamic "data_cache_config" { + for_each = var.edition == "ENTERPRISE_PLUS" ? [""] : [] + content { + data_cache_enabled = var.data_cache_enabled + } + } + + dynamic "database_flags" { + for_each = var.database_flags + content { + name = database_flags.key + value = database_flags.value + } + } + + insights_config { + query_insights_enabled = var.query_insights.enabled + query_plans_per_minute = var.query_insights.query_plans_per_minute + query_string_length = var.query_insights.query_string_length + record_application_tags = var.query_insights.record_application_tags + record_client_address = var.query_insights.record_client_address + } + + ip_configuration { + ipv4_enabled = false + private_network = var.use_psc_connection ? null : var.network_id + enable_private_path_for_google_cloud_services = true + + dynamic "authorized_networks" { + for_each = var.use_psc_connection ? [] : var.authorized_networks + iterator = ip_range + + content { + value = ip_range.value + } + } + dynamic "psc_config" { + for_each = var.use_psc_connection ? [""] : [] + content { + psc_enabled = true + allowed_consumer_projects = [var.project_id] + } + } + } + + backup_configuration { + enabled = var.enable_backups + # to allow easy switching between ENTERPRISE and ENTERPRISE_PLUS + transaction_log_retention_days = 7 + } + } + lifecycle { + precondition { + condition = var.disk_autoresize && var.disk_size_gb == null || !var.disk_autoresize + error_message = "If setting disk_size_gb set disk_autorize to false to prevent re-provisioning of the instance after disk auto-expansion." + } + } +} + + + +resource "google_compute_address" "psc" { + count = var.use_psc_connection ? 1 : 0 + project = var.project_id + name = local.sql_instance_name + address_type = "INTERNAL" + region = var.region + subnetwork = var.subnetwork_self_link + labels = local.labels +} + +resource "google_compute_forwarding_rule" "psc_consumer" { + count = var.use_psc_connection ? 1 : 0 + name = local.sql_instance_name + project = var.project_id + region = var.region + subnetwork = var.subnetwork_self_link + ip_address = google_compute_address.psc[0].self_link + load_balancing_scheme = "" + recreate_closed_psc = true + target = google_sql_database_instance.instance.psc_service_attachment_link +} + +resource "google_sql_database" "database" { + project = var.project_id + name = "slurm_accounting" + instance = google_sql_database_instance.instance.name +} + +resource "google_sql_user" "users" { + project = var.project_id + name = var.sql_username + instance = google_sql_database_instance.instance.name + password = local.sql_password +} + +resource "google_bigquery_connection" "connection" { + provider = google + project = var.project_id + location = var.region + cloud_sql { + instance_id = google_sql_database_instance.instance.connection_name + database = google_sql_database.database.name + type = "MYSQL" + credential { + username = google_sql_user.users.name + password = google_sql_user.users.password + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml new file mode 100644 index 0000000000..fc0cae0859 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigqueryconnection.googleapis.com + - sqladmin.googleapis.com + - servicenetworking.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf new file mode 100644 index 0000000000..0d05221cd8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "cloudsql" { + description = "Describes the cloudsql instance." + sensitive = true + value = { + server_ip = var.use_psc_connection ? google_compute_address.psc[0].address : google_sql_database_instance.instance.ip_address[0].ip_address + user = google_sql_user.users.name + password = google_sql_user.users.password + db_name = google_sql_database.database.name + user_managed_replication = local.user_managed_replication + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf new file mode 100644 index 0000000000..a2f150419e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf @@ -0,0 +1,173 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "authorized_networks" { + description = "IP address ranges as authorized networks of the Cloud SQL for MySQL instances" + type = list(string) + default = [] + nullable = false +} + +variable "database_version" { + description = "The version of the database to be created." + type = string + default = "MYSQL_8_0" + validation { + condition = contains(["MYSQL_5_7", "MYSQL_8_0", "MYSQL_8_4"], var.database_version) + error_message = "The database version must be either MYSQL_5_7, MYSQL_8_0 or MYSQL_8_4." + } +} + +variable "data_cache_enabled" { + description = "Whether data cache is enabled for the instance. Can be used with ENTERPRISE_PLUS edition." + type = bool + default = false +} + +variable "database_flags" { + description = "Database flags to set on instance." + type = map(string) + default = {} + nullable = false +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "disk_autoresize" { + description = "Set to false to disable automatic disk grow." + type = bool + default = true +} + +variable "disk_size_gb" { + description = "Size of the database disk in GiB." + type = number + default = null +} + +variable "edition" { + description = "value" + type = string + validation { + condition = contains(["ENTERPRISE", "ENTERPRISE_PLUS"], var.edition) + error_message = "The database edition must be either ENTERPRISE or ENTERPRISE_PLUS" + } + default = "ENTERPRISE" +} + +variable "enable_backups" { + description = "Set true to enable backups" + type = bool + default = false +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "query_insights" { + description = "Query insights configuration." + nullable = false + default = {} + type = object({ + enabled = optional(bool, false) + query_plans_per_minute = optional(number) + query_string_length = optional(number) + record_application_tags = optional(bool) + record_client_address = optional(bool) + }) +} + +variable "region" { + description = "The region where SQL instance will be configured" + type = string +} + +variable "tier" { + description = "The machine type to use for the SQL instance" + type = string +} + +variable "sql_instance_name" { + description = "name given to the sql instance for ease of identificaion" + type = string +} + +variable "deletion_protection" { + description = "Whether or not to allow Terraform to destroy the instance." + type = string + default = false +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} + +variable "sql_username" { + description = "Username for the SQL database" + type = string + default = "slurm" +} + +variable "sql_password" { + description = "Password for the SQL database." + type = any + default = null +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is going to be created in.: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "private_vpc_connection_peering" { + description = "The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation." + type = string + default = null +} + +variable "subnetwork_self_link" { + description = "Self link of the network where Cloud SQL instance PSC endpoint will be created" + type = string + default = null +} + +variable "user_managed_replication" { + type = list(object({ + location = string + kms_key_name = optional(string) + })) + description = "Replication parameters that will be used for defined secrets" + default = [] +} + +variable "use_psc_connection" { + description = "Create Private Service Connection instead of using Private Service Access peering" + type = bool + default = false +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf new file mode 100644 index 0000000000..7e672858b6 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" + } + + required_version = ">= 0.13.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md new file mode 100644 index 0000000000..d39a58afe1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md @@ -0,0 +1,158 @@ +> [!WARNING] +> This module is deprecated and will be removed on July 1, 2025. The +> recommended replacement is the +> [GCP Managed Lustre module](../../../../modules/file-system/managed-lustre/README.md) + +## Description +This module creates a DDN EXAScaler Cloud Lustre file system using code based on DDN's +[exascaler-cloud-terraform](https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp) (`scripts/2.2.2` is last release with GCP-specific module). + +More information about the architecture can be found at +[Overview of Lustre and EXAScaler Cloud][architecture]. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +> **Warning**: This file system has a license cost as described in the pricing +> section of the [DDN EXAScaler Cloud Marketplace Solution][marketplace]. +> +> **Note**: By default security.public_key is set to `null`, therefore the +> admin user is not created. To ensure the admin user is created, provide a +> public key via the security setting. +> +> **Note**: This module's instances require access to Google APIs and +> therefore, instances must have public IP address or it must be used in a +> subnetwork where [Private Google Access][private-google-access] is enabled. + +[private-google-access]: https://cloud.google.com/vpc/docs/configure-private-google-access +[marketplace]: https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud +[architecture]: https://cloud.google.com/architecture/parallel-file-systems-for-hpc#overview_of_lustre_and_exascaler_cloud + +## Mounting + +To mount the DDN EXAScaler Lustre file system you must first install the DDN +Lustre client and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +the [hpc-enterprise-slurm.yaml](../../../../examples/hpc-enterprise-slurm.yaml) for an +example of using this module with Slurm. + +If mounting is not automatically handled as described above, the DDN-EXAScaler +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + # This file system has an associated license cost. + # https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud + - id: lustrefs + source: community/modules/file-system/DDN-EXAScaler + use: [network1] + settings: {local_mount: /scratch} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(lustrefs.install_ddn_lustre_client_runner) + - $(lustrefs.mount_runner) + +``` + +See [additional documentation][ddn-install-docs] from DDN EXAScaler. + +[ddn-install-docs]: https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp#install-new-exascaler-cloud-clients +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## Support + +EXAScaler Cloud includes self-help support with access to publicly available +documents and videos. Premium support includes 24x7x365 access to DDN's experts, +along with support community access, automated notifications of updates and +other premium support features. For more information, visit +[EXAscaler Cloud on GCP][exa-gcp]. + +[exa-gcp]: https://console.cloud.google.com/marketplace/product/ddnstorage/exascaler-cloud + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [ddn\_exascaler](#module\_ddn\_exascaler) | github.com/DDNStorage/exascaler-cloud-terraform//gcp | a3355d50deebe45c0556b45bd599059b7c06988d | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [boot](#input\_boot) | Boot disk properties | 
object({
    disk_type   = string
    auto_delete = bool
    script_url  = string
  })
| 
{
  "auto_delete": true,
  "disk_type": "pd-standard",
  "script_url": null
}
| no | +| [cls](#input\_cls) | Compute client properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 0,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-2",
  "public_ip": true
}
| no | +| [clt](#input\_clt) | Compute client target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 0,
  "disk_size": 256,
  "disk_type": "pd-standard"
}
| no | +| [fsname](#input\_fsname) | EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long | `string` | `"exacloud"` | no | +| [image](#input\_image) | DEPRECATED: Source image properties | `any` | `null` | no | +| [instance\_image](#input\_instance\_image) | Source image properties

Expected Fields:
name: Unavailable with this module.
family: The image family to use.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp",
  "project": "ddn-public"
}
| no | +| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. Key-value pairs. | `map(string)` | `{}` | no | +| [local\_mount](#input\_local\_mount) | Mountpoint (at the client instances) for this EXAScaler system | `string` | `"/shared"` | no | +| [mds](#input\_mds) | Metadata server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | +| [mdt](#input\_mdt) | Metadata target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | +| [mgs](#input\_mgs) | Management server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | +| [mgt](#input\_mgt) | Management target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | +| [mnt](#input\_mnt) | Monitoring target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | +| [network\_properties](#input\_network\_properties) | Network options. 'network\_self\_link' or 'network\_properties' must be provided. | 
object({
    routing = string
    tier    = string
    id      = string
    auto    = bool
    mtu     = number
    new     = bool
    nat     = bool
  })
| `null` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self-link of the VPC network to where the system is connected. Ignored if 'network\_properties' is provided. 'network\_self\_link' or 'network\_properties' must be provided. | `string` | `null` | no | +| [oss](#input\_oss) | Object Storage server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 3,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-16",
  "public_ip": true
}
| no | +| [ost](#input\_ost) | Object Storage target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | +| [prefix](#input\_prefix) | EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud') | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Compute Platform project that will host the EXAScaler filesystem | `string` | n/a | yes | +| [security](#input\_security) | Security options | 
object({
    admin              = string
    public_key         = string
    block_project_keys = bool
    enable_os_login    = bool
    enable_local       = bool
    enable_ssh         = bool
    enable_http        = bool
    ssh_source_ranges  = list(string)
    http_source_ranges = list(string)
  })
| 
{
  "admin": "stack",
  "block_project_keys": false,
  "enable_http": false,
  "enable_local": false,
  "enable_os_login": true,
  "enable_ssh": false,
  "http_source_ranges": [
    "0.0.0.0/0"
  ],
  "public_key": null,
  "ssh_source_ranges": [
    "0.0.0.0/0"
  ]
}
| no | +| [service\_account](#input\_service\_account) | Service account name used by deploy application | 
object({
    new   = bool
    email = string
  })
| 
{
  "email": null,
  "new": false
}
| no | +| [subnetwork\_address](#input\_subnetwork\_address) | The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork\_properties' is provided. | `string` | `null` | no | +| [subnetwork\_properties](#input\_subnetwork\_properties) | Subnetwork properties. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | 
object({
    address = string
    private = bool
    id      = string
    new     = bool
  })
| `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork\_properties' is provided. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | `string` | `null` | no | +| [waiter](#input\_waiter) | Waiter to check progress and result for deployment. | `string` | `null` | no | +| [zone](#input\_zone) | Compute Platform zone where the servers will be located | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_config\_script](#output\_client\_config\_script) | Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler. | +| [http\_console](#output\_http\_console) | HTTP address to access the system web console. | +| [install\_ddn\_lustre\_client\_runner](#output\_install\_ddn\_lustre\_client\_runner) | Runner that encapsulates the `client_config_script` output on this module. | +| [mount\_command](#output\_mount\_command) | Command to mount the file system. `client_config_script` must be run first. | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the DDN EXAScaler Lustre file system | +| [network\_storage](#output\_network\_storage) | Describes a EXAScaler system to be mounted by other systems. | +| [private\_addresses](#output\_private\_addresses) | Private IP addresses for all instances. | +| [ssh\_console](#output\_ssh\_console) | Instructions to ssh into the instances. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf new file mode 100644 index 0000000000..6a2fc4b702 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ddn-exascaler", ghpc_role = "file-system" }) +} + +locals { + + network_id = var.network_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.network_self_link)[0] : null + named_net = { + routing = "REGIONAL" + tier = "STANDARD" + id = local.network_id + auto = false + mtu = 1500 + new = false + nat = false + } + + subnetwork_id = var.subnetwork_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.subnetwork_self_link)[0] : null + named_subnet = { + address = var.subnetwork_address + private = true + id = local.subnetwork_id + new = false + } +} + +module "ddn_exascaler" { + source = "github.com/DDNStorage/exascaler-cloud-terraform//gcp?ref=a3355d50deebe45c0556b45bd599059b7c06988d" + fsname = var.fsname + zone = var.zone + project = var.project_id + prefix = var.prefix + labels = local.labels + security = var.security + service_account = var.service_account + waiter = var.waiter + network = var.network_properties == null ? local.named_net : var.network_properties + subnetwork = var.subnetwork_properties == null ? local.named_subnet : var.subnetwork_properties + boot = var.boot + image = var.instance_image + mgs = var.mgs + mgt = var.mgt + mnt = var.mnt + mds = var.mds + mdt = var.mdt + oss = var.oss + ost = var.ost + cls = var.cls + clt = var.clt +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml new file mode 100644 index 0000000000..b995bd4358 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - deploymentmanager.googleapis.com + - iam.googleapis.com + - runtimeconfig.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf new file mode 100644 index 0000000000..2e9ae732ae --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf @@ -0,0 +1,90 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +output "private_addresses" { + description = "Private IP addresses for all instances." + value = module.ddn_exascaler.private_addresses +} + +output "ssh_console" { + description = "Instructions to ssh into the instances." + value = module.ddn_exascaler.ssh_console +} + +output "client_config_script" { + description = "Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler." + value = module.ddn_exascaler.client_config +} + +output "install_ddn_lustre_client_runner" { + description = "Runner that encapsulates the `client_config_script` output on this module." + value = local.client_install_runner +} + +locals { + client_install_runner = { + "type" = "shell" + "content" = module.ddn_exascaler.client_config + "destination" = "install_ddn_lustre_client.sh" + } + + # Mount command provided by DDN does not support custom local mount + split_mount_cmd = split(" ", module.ddn_exascaler.mount_command) + split_mount_cmd_wo_mountpoint = slice(local.split_mount_cmd, 0, length(local.split_mount_cmd) - 1) + mount_cmd = "${join(" ", local.split_mount_cmd_wo_mountpoint)} ${var.local_mount}" + mount_cmd_w_mkdir = "mkdir -p ${var.local_mount} && ${local.mount_cmd}" + mount_runner = { + "type" = "shell" + "content" = local.mount_cmd_w_mkdir + "destination" = "mount-ddn-lustre.sh" + } +} + +output "mount_command" { + description = "Command to mount the file system. `client_config_script` must be run first." + value = local.mount_cmd_w_mkdir +} + +output "mount_runner" { + description = "Runner to mount the DDN EXAScaler Lustre file system" + value = local.mount_runner +} + +output "http_console" { + description = "HTTP address to access the system web console." + value = module.ddn_exascaler.http_console +} + +output "network_storage" { + description = "Describes a EXAScaler system to be mounted by other systems." + value = { + server_ip = split(":", split(" ", module.ddn_exascaler.mount_command)[3])[0] + remote_mount = length(regexall("^/.*", var.fsname)) > 0 ? var.fsname : format("/%s", var.fsname) + local_mount = var.local_mount != null ? var.local_mount : format("/mnt/%s", var.fsname) + fs_type = "lustre" + mount_options = "" + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } + depends_on = [ + module.ddn_exascaler + ] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf new file mode 100644 index 0000000000..68bcc8a8ba --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf @@ -0,0 +1,502 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +# EXAScaler filesystem name +# only alphanumeric characters are allowed, +# and the value must be 1-8 characters long +variable "fsname" { + description = "EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long" + type = string + default = "exacloud" +} + +# Project ID to manage resources +# https://cloud.google.com/resource-manager/docs/creating-managing-projects +variable "project_id" { + description = "Compute Platform project that will host the EXAScaler filesystem" + type = string +} + +# Zone name to manage resources +# https://cloud.google.com/compute/docs/regions-zones +variable "zone" { + description = "Compute Platform zone where the servers will be located" + type = string +} + +# Service account name used by deploy application +# https://cloud.google.com/iam/docs/service-accounts +# new: create a new custom service account or use an existing one: true or false +# email: existing service account email address, will be using if new is false +# set email = null to use the default compute service account +variable "service_account" { + description = "Service account name used by deploy application" + type = object({ + new = bool + email = string + }) + default = { + new = false + email = null + } +} + +# Waiter to check progress and result for deployment. +# To use Google Deployment Manager: +# waiter = "deploymentmanager" +# To use generic Google Cloud SDK command line: +# waiter = "sdk" +# If you don’t want to wait until the deployment is complete: +# waiter = null +# https://cloud.google.com/deployment-manager/runtime-configurator/creating-a-waiter +variable "waiter" { + description = "Waiter to check progress and result for deployment." + type = string + default = null +} + +# Security options +# admin: optional user name for remote SSH access +# Set admin = null to disable creation admin user +# public_key: path to the SSH public key on the local host +# Set public_key = null to disable creation admin user +# block_project_keys: true or false +# Block project-wide public SSH keys if you want to restrict +# deployment to only user with deployment-level public SSH key. +# https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys +# enable_os_login: true or false +# Enable or disable OS Login feature. +# Please note, enabling this option disables other security options: +# admin, public_key and block_project_keys. +# https://cloud.google.com/compute/docs/instances/managing-instance-access#enable_oslogin +# enable_local: true or false, enable or disable firewall rules for local access +# enable_ssh: true or false, enable or disable remote SSH access +# ssh_source_ranges: source IP ranges for remote SSH access in CIDR notation +# enable_http: true or false, enable or disable remote HTTP access +# http_source_ranges: source IP ranges for remote HTTP access in CIDR notation +variable "security" { + description = "Security options" + type = object({ + admin = string + public_key = string + block_project_keys = bool + enable_os_login = bool + enable_local = bool + enable_ssh = bool + enable_http = bool + ssh_source_ranges = list(string) + http_source_ranges = list(string) + }) + + default = { + admin = "stack" + public_key = null + block_project_keys = false + enable_os_login = true + enable_local = false + enable_ssh = false + enable_http = false + ssh_source_ranges = [ + "0.0.0.0/0" + ] + http_source_ranges = [ + "0.0.0.0/0" + ] + } +} + +variable "network_self_link" { + description = "The self-link of the VPC network to where the system is connected. Ignored if 'network_properties' is provided. 'network_self_link' or 'network_properties' must be provided." + type = string + default = null +} + +# Network properties +# https://cloud.google.com/vpc/docs/vpc +# routing: network-wide routing mode: REGIONAL or GLOBAL +# tier: networking tier for VM interfaces: STANDARD or PREMIUM +# id: existing network id, will be using if new is false +# auto: create subnets in each region automatically: false or true +# mtu: maximum transmission unit in bytes: 1460 - 1500 +# new: create a new network or use an existing one: true or false +# nat: allow instances without external IP to communicate with the outside world: true or false +variable "network_properties" { + description = "Network options. 'network_self_link' or 'network_properties' must be provided." + type = object({ + routing = string + tier = string + id = string + auto = bool + mtu = number + new = bool + nat = bool + }) + + default = null +} + +variable "subnetwork_self_link" { + description = "The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork_properties' is provided. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." + type = string + default = null +} + +variable "subnetwork_address" { + description = "The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork_properties' is provided." + type = string + default = null +} + +# Subnetwork properties +# https://cloud.google.com/vpc/docs/vpc +# address: IP range of internal addresses for a new subnetwork +# private: when enabled VMs in this subnetwork without external +# IP addresses can access Google APIs and services by using +# Private Google Access: true or false +# https://cloud.google.com/vpc/docs/private-access-options +# id: existing subnetwork id, will be using if new is false +# new: create a new subnetwork or use an existing one: true or false +variable "subnetwork_properties" { + description = "Subnetwork properties. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." + type = object({ + address = string + private = bool + id = string + new = bool + }) + default = null +} +# Boot disk properties +# disk_type: pd-standard, pd-ssd or pd-balanced +# auto_delete: true or false +# whether the disk will be auto-deleted when the instance is deleted +variable "boot" { + description = "Boot disk properties" + type = object({ + disk_type = string + auto_delete = bool + script_url = string + }) + default = { + disk_type = "pd-standard" + auto_delete = true + script_url = null + } +} + +# Source image properties +# project: project name +# family: image family name +# name: !!DEPRECATED!! - image name +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: Source image properties" + type = any + # Omitting type checking so validation can provide more useful error message + # type = object({ + # project = string + # family = string + # }) + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + Source image properties + + Expected Fields: + name: Unavailable with this module. + family: The image family to use. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "ddn-public" + family = "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp" + } + + validation { + condition = !can(coalesce(var.instance_image.name)) + error_message = "In var.instance_image, the \"name\" field is not used, please use the \"family\" setting." + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, the \"family\" field must be a string set to the image family." + } +} + +# Management server properties +# node_type: type of management server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of management servers +variable "mgs" { + description = "Management server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-32" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 1 + } +} + +# Management target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of management target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of management target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of management target in GB (scratch disk size must be exactly 375) +# disk_count: number of management targets +# disk_raid: create striped management target, true or false +variable "mgt" { + description = "Management target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 128 + disk_count = 1 + disk_raid = false + } +} + + +# Monitoring target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of monitoring target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of monitoring target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of monitoring target in GB (scratch disk size must be exactly 375) +# disk_count: number of monitoring targets +# disk_raid: create striped monitoring target, true or false +variable "mnt" { + description = "Monitoring target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 128 + disk_count = 1 + disk_raid = false + } +} + +# Metadata server properties +# node_type: type of metadata server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of metadata servers +variable "mds" { + description = "Metadata server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-32" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 1 + } +} + +# Metadata target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of metadata target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of metadata target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of metadata target in GB (scratch disk size must be exactly 375) +# disk_count: number of metadata targets +# disk_raid: create striped metadata target, true or false +variable "mdt" { + description = "Metadata target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-ssd" + disk_size = 3500 + disk_count = 1 + disk_raid = false + } +} + +# Object Storage server properties +# node_type: type of storage server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of storage servers +variable "oss" { + description = "Object Storage server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-16" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 3 + } +} + +# Object Storage target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of storage target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of storage target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of storage target in GB (scratch disk size must be exactly 375) +# disk_count: number of storage targets +# disk_raid: create striped storage target, true or false +variable "ost" { + description = "Object Storage target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-ssd" + disk_size = 3500 + disk_count = 1 + disk_raid = false + } +} + +# Compute client properties +# node_type: type of compute client +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of compute clients +variable "cls" { + description = "Compute client properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-2" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 0 + } +} +# Compute client target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of compute target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of compute target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of compute target in GB (scratch disk size must be exactly 375) +# disk_count: number of compute targets +variable "clt" { + description = "Compute client target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 256 + disk_count = 0 + } +} +variable "local_mount" { + description = "Mountpoint (at the client instances) for this EXAScaler system" + type = string + default = "/shared" +} + +variable "prefix" { + description = "EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud')" + type = string + default = null +} + +variable "labels" { + description = "Labels to add to EXAScaler Cloud deployment. Key-value pairs." + type = map(string) + default = {} +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf new file mode 100644 index 0000000000..2981b4dd75 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/Intel-DAOS/README.md b/deletion-test/build_script/modules/embedded/community/modules/file-system/Intel-DAOS/README.md new file mode 100644 index 0000000000..04db0acb8c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/Intel-DAOS/README.md @@ -0,0 +1 @@ +> **_NOTE:_** Cluster Toolkit is dropping support for the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository. The DAOS example blueprints (`hpc-slurm-daos.yaml` and `pfs-daos.yaml`) have been removed from the Cluster Toolkit. We recommend migrating to the first-party [Parallelstore](../../../../modules/file-system/parallelstore/) module for similar functionality. To help with this transition, see the Parallelstore example blueprints ([pfs-parallelstore.yaml](../../../../examples/pfs-parallelstore.yaml) and [ps-slurm.yaml](../../../../examples/ps-slurm.yaml)). If the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository is necessary, we recommend using the last Cluster Toolkit [v1.41.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/releases/tag/v1.41.0). diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/README.md b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/README.md new file mode 100644 index 0000000000..66aaaa46af --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/README.md @@ -0,0 +1,152 @@ +## Description + +This module creates a Network File Sharing (NFS) file system based on a VM +instance and [compute disk][disk]. This file system can share directories and +files with other clients over a network. `nfs-server` can be used by +[vm-instance](../../../../modules/compute/vm-instance/README.md) and SchedMD +community modules that create compute VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +If you are using Hyperdisk storage, check the possible disk size, IOPS, and throughput values for each disk type in the [Hyperdisk limits documentation](https://cloud.google.com/compute/docs/disks/hyperdisks#limits-disk). + +> **_WARNING:_** This module has only been tested against the HPC centos7 OS +> disk image (the default). Using other images may work, but have not been +> verified. + +[disk]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk + +### Example + +```yaml +- id: homefs + source: community/modules/file-system/nfs-server + use: [network1] +``` + +This creates a NFS on a virtual machine which allow other VMs to mount the +volume as an external file system. + +> **_NOTE:_** All disks are destroyed along with the instance, during a `gcluster destroy`/`terraform destroy` event. However, you can setup data retention with `create_boot_snapshot_before_destroy` (boot disk) and `create_snapshot_before_destroy` (data disk). + +## Mounting + +To mount the NFS Server you must first ensure that the NFS client has been +installed the and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +See the [hpc-centos-ss.yaml] test config for an example of using this module +with a `vm-instance` module. + +If mounting is not automatically handled as described above, the `nfs-server` +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + - id: nfs + source: community/modules/file-system/nfs-server + use: [network1] + settings: {local_mounts: [/mnt1]} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(nfs.install_nfs_client_runner) + - $(nfs.mount_runner) + +``` + +[hpc-centos-ss.yaml]: ../../../../tools/validate_configs/test_configs/hpc-centos-ss.yaml +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 6.14 | +| [null](#requirement\_null) | >= 3.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.14 | +| [null](#provider\_null) | >= 3.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_disk.attached_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_instance.compute_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource | +| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_default_service_account.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_default_service_account) | data source | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [auto\_delete\_disk](#input\_auto\_delete\_disk) | DEPRECATED: Whether or not the NFS disk should be auto-deleted | `string` | `null` | no | +| [boot\_disk\_size](#input\_boot\_disk\_size) | Storage size in GB for the boot disk | `number` | `null` | no | +| [boot\_disk\_type](#input\_boot\_disk\_type) | Storage type for the boot disk | `string` | `null` | no | +| [create\_boot\_snapshot\_before\_destroy](#input\_create\_boot\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the boot disk | `bool` | `false` | no | +| [create\_snapshot\_before\_destroy](#input\_create\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the NFS data disk | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the NFS instance if no name is specified. | `string` | n/a | yes | +| [disk\_size](#input\_disk\_size) | Storage size in GB for the NFS data disk | `number` | `"100"` | no | +| [image](#input\_image) | DEPRECATED: The VM image used by the NFS server | `string` | `null` | no | +| [instance\_image](#input\_instance\_image) | The VM image used by the NFS server.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to the NFS instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mounts](#input\_local\_mounts) | Mountpoint for this NFS compute instance | `list(string)` | 
[
  "/data"
]
| no | +| [machine\_type](#input\_machine\_type) | Type of the VM instance to use | `string` | `"n2d-standard-2"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the NFS VM. | `string` | `"default"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [provisioned\_iops](#input\_provisioned\_iops) | Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput | `number` | `null` | no | +| [provisioned\_throughput](#input\_provisioned\_throughput) | Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme | `number` | `null` | no | +| [scopes](#input\_scopes) | Scopes to apply to the controller | `list(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [service\_account](#input\_service\_account) | Service Account for the NFS server | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the NFS VM. | `string` | `null` | no | +| [type](#input\_type) | Storage type for the NFS data disk | `string` | `"pd-ssd"` | no | +| [zone](#input\_zone) | The zone name where the NFS instance located in. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | +| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | +| [network\_storage](#output\_network\_storage) | export of all desired folder directories | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/main.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/main.tf new file mode 100644 index 0000000000..a00d2681ba --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/main.tf @@ -0,0 +1,131 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "nfs-server", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + server_ip = google_compute_instance.compute_instance.network_interface[0].network_ip + fs_type = "nfs" + mount_options = "defaults,hard,intr" + install_nfs_client_runners = [for mount in var.local_mounts : + { + "type" = "shell" + "source" = "${path.module}/scripts/install-nfs-client.sh" + "destination" = "install-nfs${replace(mount, "/", "_")}.sh" + } + ] + mount_runners = [for mount in var.local_mounts : + { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"/exports${mount}\" \"${mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(mount, "/", "_")}.sh" + } + ] + ansible_mount_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/mount.yaml" + "destination" = "mount.yaml" + } +} + +data "google_compute_default_service_account" "default" {} + +resource "google_compute_disk" "attached_disk" { + project = var.project_id + name = "${local.name}-nfs-instance-disk" + size = var.disk_size + type = var.type + zone = var.zone + labels = local.labels + provisioned_iops = var.provisioned_iops + provisioned_throughput = var.provisioned_throughput + create_snapshot_before_destroy = var.create_snapshot_before_destroy +} + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project +} + +resource "null_resource" "image" { + triggers = { + name = try(var.instance_image.name, null), + family = try(var.instance_image.family, null), + project = var.instance_image.project + } +} + +resource "google_compute_disk" "boot_disk" { + project = var.project_id + + name = "${local.name}-boot-disk" + size = var.boot_disk_size + type = var.boot_disk_type + image = data.google_compute_image.compute_image.self_link + labels = local.labels + zone = var.zone + create_snapshot_before_destroy = var.create_boot_snapshot_before_destroy + + lifecycle { + replace_triggered_by = [null_resource.image] + ignore_changes = [ + image + ] + } +} + +resource "google_compute_instance" "compute_instance" { + project = var.project_id + name = "${local.name}-nfs-instance" + zone = var.zone + machine_type = var.machine_type + + boot_disk { + auto_delete = false + source = google_compute_disk.boot_disk.self_link + device_name = google_compute_disk.boot_disk.name + } + + attached_disk { + source = google_compute_disk.attached_disk.id + device_name = "attached_disk" + } + + network_interface { + network = var.network_self_link + subnetwork = var.subnetwork_self_link + } + + service_account { + email = var.service_account == null ? data.google_compute_default_service_account.default.email : var.service_account + scopes = var.scopes + } + + metadata = var.metadata + metadata_startup_script = templatefile("${path.module}/scripts/install-nfs-server.sh.tpl", { local_mounts = var.local_mounts }) + + labels = local.labels +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/outputs.tf new file mode 100644 index 0000000000..e23b94e2b2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/outputs.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +# render the content for each folder +output "network_storage" { + description = "export of all desired folder directories" + value = [for i, mount in var.local_mounts : { + remote_mount = "/exports${mount}" + local_mount = mount + fs_type = local.fs_type + mount_options = local.mount_options + server_ip = local.server_ip + client_install_runner = local.install_nfs_client_runners[i] + mount_runner = local.mount_runners[i] + } + ] +} + +output "install_nfs_client" { + description = "Script for installing NFS client" + value = file("${path.module}/scripts/install-nfs-client.sh") +} + +output "install_nfs_client_runner" { + description = "Runner to install NFS client using the startup-script module" + value = local.install_nfs_client_runners[0] +} + +output "mount_runner" { + description = <<-EOT + Runner to mount the file-system using an ansible playbook. The startup-script + module will automatically handle installation of ansible. + - id: example-startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(your-fs-id.mount_runner) + ... + EOT + value = local.ansible_mount_runner +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl new file mode 100644 index 0000000000..1b06a5f032 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl @@ -0,0 +1,35 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -ex + +if [ ! -d "/exports" ]; then # first load, format and mount the disk + # See https://cloud.google.com/compute/docs/disks/add-persistent-disk + uuid=$(uuidgen) + mkfs.ext4 -F -m 0 -U "$uuid" -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/disk/by-id/google-attached_disk + + mkdir /exports + echo "UUID=$uuid /exports ext4 discard,defaults 0 0" >> /etc/fstab + mount --target /exports/ + + %{ for mount in local_mounts ~} + mkdir -p /exports${mount} + chmod 755 /exports${mount} + echo '/exports${mount} *(rw,sync,no_root_squash)' >> "/etc/exports" + %{ endfor ~} +fi + +systemctl start nfs-server rpcbind +systemctl enable nfs-server +exportfs -r diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml new file mode 100644 index 0000000000..f7fbe58d5e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml @@ -0,0 +1,39 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Mounts the file systems specified in the metadata network_storage key + hosts: localhost + become: true + vars: + meta_key: "network_storage" + url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" + tasks: + - name: Read metadata network_storage information + ansible.builtin.uri: + url: "{{ url }}/{{ meta_key }}" + method: GET + headers: + Metadata-Flavor: "Google" + register: storage + - name: Mount file systems + ansible.posix.mount: + src: "{{ item.server_ip }}:/{{ item.remote_mount }}" + path: "{{ item.local_mount }}" + opts: "{{ item.mount_options }}" + boot: true + fstype: "{{ item.fs_type }}" + state: "mounted" + loop: "{{ storage.json }}" diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/variables.tf new file mode 100644 index 0000000000..9a58da641e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/variables.tf @@ -0,0 +1,194 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the NFS instance if no name is specified." + type = string +} + +variable "name" { + description = "The resource name of the instance." + type = string + default = null +} + +variable "zone" { + description = "The zone name where the NFS instance located in." + type = string +} + +variable "boot_disk_size" { + description = "Storage size in GB for the boot disk" + type = number + default = null +} + +variable "boot_disk_type" { + description = "Storage type for the boot disk" + type = string + default = null +} + +variable "create_boot_snapshot_before_destroy" { + description = "Whether to create a snapshot before destroying the boot disk" + type = bool + default = false +} + +variable "disk_size" { + description = "Storage size in GB for the NFS data disk" + type = number + default = "100" +} + +variable "type" { + description = "Storage type for the NFS data disk" + type = string + default = "pd-ssd" +} + +variable "create_snapshot_before_destroy" { + description = "Whether to create a snapshot before destroying the NFS data disk" + type = bool + default = false +} + +variable "provisioned_iops" { + description = "Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput" + type = number + default = null +} + +variable "provisioned_throughput" { + description = "Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme" + type = number + default = null +} + +# Deprecated, replaced by instance_image +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: The VM image used by the NFS server" + type = string + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + The VM image used by the NFS server. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +# Deprecated, replaced by create_snapshot_before_destroy and create_boot_snapshot_before_destroy +# tflint-ignore: terraform_unused_declarations +variable "auto_delete_disk" { + description = "DEPRECATED: Whether or not the NFS disk should be auto-deleted" + type = string + default = null + + validation { + condition = var.auto_delete_disk == null + error_message = "The 'var.auto_delete_disk' setting is broken in Cluster Toolkit versions >1.25.0 and deprecated in versions >1.48.0, please use 'var.create_snapshot_before_destroy' and 'var.create_boot_snapshot_before_destroy' instead." + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the NFS VM." + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the NFS VM." + type = string + default = null +} + +variable "machine_type" { + description = "Type of the VM instance to use" + type = string + default = "n2d-standard-2" +} + +variable "labels" { + description = "Labels to add to the NFS instance. Key-value pairs." + type = map(string) +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "service_account" { + description = "Service Account for the NFS server" + type = string + default = null +} + +variable "scopes" { + description = "Scopes to apply to the controller" + type = list(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "local_mounts" { + description = "Mountpoint for this NFS compute instance" + type = list(string) + default = ["/data"] + + validation { + condition = alltrue([ + for m in var.local_mounts : substr(m, 0, 1) == "/" + ]) + error_message = "Local mountpoints have to start with '/'." + } + validation { + condition = length(var.local_mounts) > 0 + error_message = "At least one local mount must be specified in var.local_mounts." + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/versions.tf new file mode 100644 index 0000000000..63443806b8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.14" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/sycomp-scale/README.md b/deletion-test/build_script/modules/embedded/community/modules/file-system/sycomp-scale/README.md new file mode 100644 index 0000000000..79ff12bc18 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/sycomp-scale/README.md @@ -0,0 +1,35 @@ +## Description + +This document provides information on how to deploy an instance of [Sycomp Intelligent Data Storage Platform](https://sycomp.com/solution/hpc/storage/) on Google Cloud Platform ([GCP](https://cloud.google.com/)) using the Google Cluster Toolkit. + +> **_NOTE:_** +> Sycomp Storage on GCP does not require an HPC Toolkit wrapper. +> Terraform modules are sourced directly from GitLab. + +Terraform modules for Sycomp Intelligent Data Storage Platform are downloaded on deployment using the Google Cloud Toolkit. + +The Terraform module parameters are documented in the `README.md` files in the respective module directories of the source GitLab repository. The main modules are: + +- `sycomp-scale` +- `sycomp-scale-expansion` + +## Examples + +The community examples folder (community/examples/sycomp/) contains four example blueprints that you can use to deploy or expand a Sycomp Storage cluster. + +- [community/examples/sycomp/sycomp-storage.yaml][sycomp-storage-yaml] - + Blueprint for deploying a Sycomp Storage cluster consisting of 3 storage servers. + +- [community/examples/sycomp/sycomp-storage-expansion.yaml][sycomp-storage-expansion-yaml] - + Blueprint for expanding the above created cluster from 3 to 4 storage servers. + +- [community/examples/sycomp/sycomp-storage-ece.yaml][sycomp-storage-ece-yaml] - + Blueprint for deploying a Sycomp Storage cluster consisting of 7 storage servers with ECE (Erasure Code Edition) software RAID. + +- [community/examples/sycomp/sycomp-storage-slurm.yaml][sycomp-storage-slurm-yaml] - + Blueprint for deploying a Slurm cluster and Sycomp Storage cluster with 3 servers. The Slurm compute nodes are configured as NFS clients and have the ability to use the Sycomp Storage filesystem. + +[sycomp-storage-yaml]: ../../../examples/sycomp/sycomp-storage.yaml +[sycomp-storage-expansion-yaml]: ../../../examples/sycomp/sycomp-storage-expansion.yaml +[sycomp-storage-ece-yaml]: ../../../examples/sycomp/sycomp-storage-ece.yaml +[sycomp-storage-slurm-yaml]: ../../../examples/sycomp/sycomp-storage-slurm.yaml diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/README.md b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/README.md new file mode 100644 index 0000000000..0e2a936167 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/README.md @@ -0,0 +1,182 @@ +## Description + +This module provides scripts for client installation and mounting [WEKA] +filesystems. Client supports both UDP and DPDK modes and allows customization of +mount parameters using Compute VM instance metadata. + +For deploying Weka cluster please consult [WEKA installation on GCP]. + +[WEKA]: https://www.weka.io/ +[WEKA installation on GCP]: https://docs.weka.io/planning-and-installation/weka-installation-on-gcp + +## Prerequisites + +* up and running Weka cluster +* running on a [supported OS](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#operating-system) +* [open firewall](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#required-ports) + between WEKA backend servers and clients +* VPC peering configuration: + * if clients share VPCs created for WEKA cluster, no additional configuration + is necessary + * if dedicated VPCs are in use for clients, then WEKA VPCs needs to be peered + with VPCs that are used as: + * primary interface on client + * interfaces dedicated for DPDK client + * if dedicated VPCs are in use for clients, then those VPCs needs to be peered + with each other + +## Mounting +This example creates mount scripts that will mount `default` filesystem from +`10.0.0.3` WEKA backend: + +```yaml + - id: wekafs + source: community/modules/file-system/weka-client + settings: + local_mount: /scratch + server_ip: 10.0.0.3 + remote_mount: default + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: $(wekafs.runners) +``` + +If you need to add mount script along other runners, remember to add all 4 +runners provided by this script as shown in this example: + +```yaml + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(wekafs.client_install_runner) + - $(wekafs.mount_runner) + - type: shell + content: | + #!/bin/bash + + echo Sample + destination: sample-script.sh +``` + +To use the client within Slurm partition, with DPDK, remember to set additional +networks, and configure metadata. In this example, all four additional interfaces +are dedicated to WEKA DPDK + +```yaml + - id: c2_60_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + - mount-at-startup # as defined in previous examples + settings: + bandwidth_tier: virtio_enabled # Weka requires VirtIO, from WEKA 4.4.1, DPDK is also supported on gVNIC + additional_networks: + - subnetwork: weka-client-1 + nic_type: VIRTIO_NET + - subnetwork: weka-client-2 + nic_type: VIRTIO_NET + - subnetwork: weka-client-3 + nic_type: VIRTIO_NET + - subnetwork: weka-client-4 + nic_type: VIRTIO_NET + machine_type: c2-standard-60 + metadata: + weka-data_interfaces: 1,2,3,4 # allocate interfaces 1, 2, 3 and 4 to DPDK + weka-mode: dpdk + weka-options: num_cores=4,dpdk_base_memory_mb=16 + node_conf: + # From https://docs.weka.io/planning-and-installation/bare-metal/planning-a-weka-system-installation + # do not set RealMem as this is set automatically by Cluster Toolkit + CoreSpecCount: 4 + MemSpecLimit: 5120 +``` + +Due to the fact, that client installation takes ~6-7 minutes, if you use WEKA together with Slurm and do not bundle +client in the instance image, you may need to increase the timeout for startups scripts. + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + settings: + compute_startup_scripts_timeout: 600 + login_startup_scripts_timeout: 600 + ... + - id: compute_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + settings: + resume_timeout: 600 + ... +``` + +## Supported VM metadata options +Client scripts do support following metadata keys: +* `weka-mode` - one of `udp` or `dpdk`. Defaults to `udp`. Sets client mode. +* `weka-data_interfaces` - comma separated list of interface identifiers, + specifying which interfaces are dedicated for data plane. Set to `1` to + dedicate second interface of instance for WEKA DPDK. Set to `2,5` to dedicate + third and sixth interface of instance for WEKA DPDK. +* `weka-mgmt_interface` - identifier of management interface, defaults to `0`, + which means to use primary interface as management interface. +* `weka-options` - additional [mount command options](https://docs.weka.io/weka-filesystems-and-object-stores/mounting-filesystems#mount-command-options) + to pass to `mount` command + +## Adding client to the OS image +To save time during the mount command install and precompile DPDK drivers in the +OS image. Following scripts compiles DPDK driver for currently running kernel. + +```shell +#!/bin/bash + +set -e -o pipefail + +echo Downloading and installing Weka client +curl --max-time 10 "{{ weka backend endpoint }}/dist/v1/install" | sh +WEKA_VERSION=$(weka -v | sed -e 's/^[^0-9]*//') +echo Installing Weka version: ${WEKA_VERSION} +weka version get "${WEKA_VERSION}" +weka version set "${WEKA_VERSION}" +# run setup for the second time, if it fails for the first time +weka local setup weka || weka local setup weka +weka version prepare "${WEKA_VERSION}" +weka local stop +weka local rm -f --all +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | +| [mount\_options](#input\_mount\_options) | Mount options for filesystem shared by all clients. | `string` | `""` | no | +| [remote\_mount](#input\_remote\_mount) | Weka filesystem name. | `string` | n/a | yes | +| [server\_ip](#input\_server\_ip) | Weka backend IP address used for bootstrapping. | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_install\_runner](#output\_client\_install\_runner) | Ansible runner that performs client installation needed to use file system. | +| [mount\_runner](#output\_mount\_runner) | Ansible runner that mounts the file system. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/metadata.yaml new file mode 100644 index 0000000000..419bc3fe46 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/outputs.tf new file mode 100644 index 0000000000..0bd9098d80 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/outputs.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + template_args = { + local_mount = var.local_mount + mount_options = var.mount_options == "" ? "" : "-o ${var.mount_options}" + remote_mount = var.remote_mount + server_ip = var.server_ip + service_name = "weka-mount${replace(var.local_mount, "/", "-")}" + } + mount_script = templatefile("${path.module}/templates/mount-weka.sh.tftpl", local.template_args) + + mount_runner_ansible = { + type = "ansible-local" + content = templatefile( + "${path.module}/templates/mount-weka.yaml.tftpl", + merge( + local.template_args, + { mount_weka_script = local.mount_script } + ) + ) + destination = "mount_filesystem${replace(var.local_mount, "/", "_")}.yaml" + } + + client_install_runner = { + type = "ansible-local" + content = templatefile("${path.module}/templates/install-weka-client.yaml.tftpl", local.template_args) + destination = "install_filesystem${replace(var.local_mount, "/", "_")}.yaml" + } +} + +# currently WEKA mounts are not compatible with network_storage logic, as WEKA volumes needs to be mounted by +# systemd script and not /etc/fstab entry, as the mount command needs to have network configuration which may change +# between restarts +# +#output "network_storage" { +# description = "Describes a remote network storage to be mounted by fs-tab." +# value = { +# server_ip = var.server_ip +# remote_mount = var.remote_mount +# local_mount = var.local_mount +# fs_type = var.fs_type +# mount_options = var.mount_options +# client_install_runner = local.client_install_runner +# mount_runner = local.mount_runner +# } +#} +# +output "client_install_runner" { + description = "Ansible runner that performs client installation needed to use file system." + value = local.client_install_runner +} + +output "mount_runner" { + description = "Ansible runner that mounts the file system." + value = local.mount_runner_ansible +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl new file mode 100644 index 0000000000..ddc3acdb5d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl @@ -0,0 +1,133 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Mounts the file systems specified in the metadata network_storage key + hosts: localhost + become: true + vars: + meta_key: "network_storage" + url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" + tasks: + - name: Check if weka is installed + ansible.builtin.stat: + path: /usr/bin/weka + register: weka_binary + + - name: Create temporary location for installation script + ansible.builtin.tempfile: + state: file + register: + install_script + when: not weka_binary.stat.exists + + - name: Download WEKA client + ansible.builtin.get_url: + url: http://${server_ip}:14000/dist/v1/install + dest: "{{ install_script.path }}" + mode: "700" + when: not weka_binary.stat.exists + + - name: Run WEKA installation script + ansible.builtin.shell: + cmd: "{{ install_script.path }}" + when: not weka_binary.stat.exists + register: weka_install_result + changed_when: weka_install_result.rc == 0 + + - name: Read metadata network_storage information + ansible.builtin.uri: + url: "{{ url }}/weka-version" + method: GET + headers: + Metadata-Flavor: "Google" + status_code: + - 200 + - 404 + register: get_weka_version + + - name: Set WEKA version from metadata server + ansible.builtin.set_fact: + weka_version: "{{ get_weka_version.body }}" + when: get_weka_version.status == 200 + + - name: Get version of WEKA installation client + ansible.builtin.shell: + cmd: weka -v | sed -e 's/^[^0-9.]*\([0-9.]*\)[^0-9.]*$/\1/' + register: get_weka_client_version + changed_when: get_weka_client_version.rc == 0 + + - name: Set WEKA version from WEKA installation client + ansible.builtin.set_fact: + weka_version: "{{ get_weka_client_version.stdout }}" + when: get_weka_version.status == 404 + + - name: Download user-defined WEKA version + ansible.builtin.shell: + cmd: weka version get {{ weka_version }} + register: result + changed_when: result.rc == 0 + + - name: Set user-defined WEKA version + ansible.builtin.shell: + cmd: weka version set {{ weka_version }} + register: result + changed_when: result.rc == 0 + + - name: Setup WEKA client + ansible.builtin.shell: + cmd: weka local setup weka + register: setup_1_result + changed_when: setup_1_result.rc == 0 + failed_when: false # ignore errors + + - name: Setup WEKA client (2nd try) + ansible.builtin.shell: + cmd: weka local setup weka + register: result + changed_when: result.rc == 0 + when: setup_1_result.rc != 0 + + - name: Prepare WEKA version + ansible.builtin.shell: + cmd: weka version prepare {{ weka_version }} + register: result + changed_when: result.rc == 0 + + - name: Stop WEKA client + ansible.builtin.shell: + cmd: weka local stop + async: 30 + poll: 10 + register: weka_stop + changed_when: weka_stop.get("rc") == 0 # when killed by async, rc is not defined + failed_when: false # ignore errors + + - name: Stop WEKA client (2nd try) + ansible.builtin.shell: + cmd: weka local stop + async: 30 + poll: 10 + register: result + changed_when: result.rc == 0 + failed_when: false # ignore errors + when: weka_stop.get("rc") != 0 + + - name: Remove WEKA containers + ansible.builtin.shell: + cmd: weka local rm -f --all + register: result + changed_when: result.rc == 0 + failed_when: false # ignore errors diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl new file mode 100644 index 0000000000..19c6dc1fdc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl @@ -0,0 +1,101 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# shellcheck disable=SC2034 +METADATA_BASE_URL="http://metadata.google.internal/computeMetadata/v1/instance" +# shellcheck disable=SC2034 +ATTR_URL="$${METADATA_BASE_URL}/attributes/weka-" +NET_URL="$${METADATA_BASE_URL}/network-interfaces" + +# shellcheck disable=SC1083 +WEKA_MODE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mode || echo -n udp) +# shellcheck disable=SC1083 +WEKA_DATA_INTERFACES=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}data_interfaces || exit 0) +# shellcheck disable=SC1083 +WEKA_MGMT_INTERFACE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mgmt_interface || echo -n 0) +# shellcheck disable=SC1083 +WEKA_OPTIONS=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}options || exit 0) + +WEKA_OPTIONS="$${WEKA_OPTIONS:+-o $WEKA_OPTIONS}" + +netmask_to_cidr () { + c=0 + # shellcheck disable=SC2086,SC1083 + x=0$( printf '%o' $${1//./ } ) + while [ "$x" -gt 0 ]; do + c=$(( c + x%2 )) + x=$(( x >> 1)) + done + echo $c ; +} + +# detect network interface naming scheme +if [[ -e /sys/class/net/eth0 ]] ; then + DEVICE_NAME="eth" + DEVICE_INDEX_BASE=0 +elif [[ -e /sys/class/net/ens4 ]] ; then + DEVICE_NAME="ens" + DEVICE_INDEX_BASE=4 +else + echo "Can't detect device names. Both /sys/class/net/eth0 and /sys/class/net/ens4 do not exists" + exit 1 +fi + +# ensure that /etc/hosts contains entry for hostname pointing to primary interface +NEW_IP=$(ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) | head -n 1 | sed -e 's/^.*inet \([0-9\.]\+\)\/.*$/\1/') +if [ -n "$NEW_IP" ] ; then + HOSTNAME=$(hostname) + sed -i -e "/$HOSTNAME/s/^[0-9\.]\+ $HOSTNAME/$NEW_IP $HOSTNAME/" /etc/hosts +else + echo "Failed to find primary interface address" + ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) + exit 1 +fi + +# shellcheck disable=SC2154 +echo "Mounting Weka ${server_ip}/${remote_mount} to ${local_mount}" +mkdir -p "${local_mount}" +service weka-agent start +if [[ $WEKA_MODE == "udp" ]] ; then + # shellcheck disable=SC2086,SC2154,SC2086 + mount -t wekafs ${mount_options} -o net=udp $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" + +elif [[ $WEKA_MODE == "dpdk" ]] ; then + declare -a DATA_INTERFACES + # split WEKA_DATA_INTERFACES by comma into array + # shellcheck disable=SC2034 + IFS=',' read -r -a DATA_INTERFACES <<< "$WEKA_DATA_INTERFACES" + + DATA_OPTIONS="" + # shellcheck disable=SC2066 + for interface in "$${DATA_INTERFACES[@]}" ; do + INTERFACE_IP=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/ip") + INTERFACE_MASK=$(netmask_to_cidr "$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/subnetmask")") + INTERFACE_GATEWAY=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/gateway") + + DATA_OPTIONS+="-o net=$DEVICE_NAME$((DEVICE_INDEX_BASE + interface))/$INTERFACE_IP/$INTERFACE_MASK/$INTERFACE_GATEWAY " + done + + # shellcheck disable=SC2086 + mount -t wekafs \ + ${mount_options} \ + -o mgmt_ip="$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$WEKA_MGMT_INTERFACE/ip")" \ + $DATA_OPTIONS $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" +else + echo "Unknown weka:mode metadata value: $${WEKA_MODE}. Allowed values: udp and dpdk" + exit 1 +fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl new file mode 100644 index 0000000000..84587103a9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl @@ -0,0 +1,54 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Mount the WEKA file systems + hosts: localhost + become: true + vars: + local_mount: "${local_mount}" + remote_mount: "${remote_mount}" + server_ip: "${server_ip}" + service_name: "weka-mount-${replace(local_mount, "/", "_")}" + tasks: + - name: Create mount script + ansible.builtin.copy: + dest: "/etc/{{ service_name }}.sh" + mode: "0755" + content: | + ${indent(8, mount_weka_script)} + + - name: Create systemd service for weka mount + ansible.builtin.copy: + dest: "/etc/systemd/system/{{ service_name }}.service" + mode: "0644" + content: | + [Install] + WantedBy=multi-user.target + [Unit] + Description=Mount Weka {{ server_ip }}/{{ remote_mount }} at {{ local_mount }} + After=network-online.target + Wants=network-online.target + [Service] + RemainAfterExit=true + Type=oneshot + ExecStart=/bin/bash -c "/etc/{{ service_name }}.sh" + + - name: Enable and start weka mount service + ansible.builtin.systemd: + name: "{{ service_name }}" + daemon_reload: true + enabled: true + state: started diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/variables.tf new file mode 100644 index 0000000000..f07961d64c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/variables.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "local_mount" { + description = "The mount point where the contents of the device may be accessed after mounting." + type = string + default = "/mnt" +} + +variable "mount_options" { + description = "Mount options for filesystem shared by all clients." + type = string + default = "" + nullable = false +} + +variable "remote_mount" { + description = "Weka filesystem name." + type = string +} + +variable "server_ip" { + description = "Weka backend IP address used for bootstrapping." + type = string + default = "" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/versions.tf new file mode 100644 index 0000000000..9e6af1fa7f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.14.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb new file mode 100644 index 0000000000..f13726f691 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "project_id = \"${project_id}\"\n", + "dataset_id = \"${dataset_id}\"\n", + "table_id = \"${table_id}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ONI1Xo0-KtAD", + "outputId": "fb9ca475-e4ec-4cd0-e0e6-14f409eefd7a" + }, + "outputs": [], + "source": [ + "from google.cloud import bigquery\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "\n", + "client = bigquery.Client(project=project_id)\n", + "\n", + "df = client.query(f'''\n", + "SELECT ticker, cast(price AS FLOAT64) AS price, CAST(OFFSET as INTEGER) AS offset, start_date, end_date, iteration\n", + "FROM `{project_id}.{dataset_id}.{table_id}`,\n", + "UNNEST(simulation_results) as NUMERIC with OFFSET\n", + "WHERE epoch_time IN\n", + " # Get the latest simulation runs for each Ticker Symbol\n", + "(SELECT MAX(epoch_time) FROM `{project_id}.{dataset_id}.{table_id}` GROUP BY ticker)\n", + "'''\n", + ").to_dataframe()\n", + "# Display the data\n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define a function to plot the data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def plot_ticker(t,df):\n", + "\n", + " dtf = df[(df.ticker==t) &(df.offset == 250)].price.describe(include=[np.float64], percentiles=[.05, .01, .001])\n", + " cellText = []\n", + " for v in dtf.values:\n", + " cellText.append([v])\n", + " \n", + " pltf = df[df.ticker==t].pivot(index='offset', columns='iteration', values='price')\n", + " \n", + " fig = plt.figure(figsize=(10,5))\n", + " ax1 = fig.add_subplot(122)\n", + " pltf.plot(legend=False, ax=ax1, xlabel='Time(days)', ylabel='US$', title=f\"{ df[(df.ticker == t) & (df.offset == 0) & (df.iteration == 4)]}\")\n", + " ax2 = fig.add_subplot(121)\n", + " font_size=10\n", + " bbox=[0, 0, .5, 1]\n", + " ax2.axis('off')\n", + " mpl_table = ax2.table(cellText = cellText, rowLabels=dtf.index.values, bbox=bbox)\n", + " mpl_table.auto_set_font_size(False)\n", + " mpl_table.set_fontsize(font_size)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 808 + }, + "id": "jvBmb_KceX7z", + "outputId": "42a3ba9f-b68f-4c7b-d928-0fedeed9216c" + }, + "outputs": [], + "source": [ + "ticker_list = df.ticker.unique()\n", + "for t in ticker_list:\n", + " plot_ticker(t,df)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md new file mode 100644 index 0000000000..e54893a1bb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md @@ -0,0 +1,97 @@ +## Description + +Copy files to a target GCS bucket. + +Primarily used for FSI - MonteCarlo Tutorial **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: +../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage +This copies the module files to the specified GCS bucket. It is expected that +the bucket will be mounted on the target VM. + +Some of the files are templates, and `main.tf` translates the files with the +passed variable values. This way the user does not have to change things like +pointing to the correct bigquery table or adding in the project_id. + +```yaml + - id: fsi_tutorial_files + source: community/modules/files/fsi-montecarlo-on-batch + use: [bq-dataset, bq-table, fsi_bucket, pubsub_topic] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 3.83 | +| [http](#requirement\_http) | ~> 3.0 | +| [random](#requirement\_random) | ~> 3.0 | +| [template](#requirement\_template) | ~> 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [http](#provider\_http) | ~> 3.0 | +| [random](#provider\_random) | ~> 3.0 | +| [template](#provider\_template) | ~> 2.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.get_iteration_sh](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.get_mc_reqs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.get_requirements](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.ipynb_obj_fsi](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.mc_obj_yaml](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.mc_run](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.run_batch_py](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [http_http.batch_py](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | +| [http_http.batch_requirements](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | +| [template_file.ipynb_fsi](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | +| [template_file.mc_run_py](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | +| [template_file.mc_run_yaml](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Bigquery dataset id | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to run project | `string` | n/a | yes | +| [table\_id](#input\_table\_id) | Bigquery table id | `string` | n/a | yes | +| [topic\_id](#input\_topic\_id) | Pubsub Topic Name | `string` | n/a | yes | +| [topic\_schema](#input\_topic\_schema) | Pubsub Topic schema | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh new file mode 100644 index 0000000000..50aa865a31 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ticker=("GOOG" "AMZN" "MSFT" "NVDA" "META" "TSLA" "PEP" "COST") +echo "BI: $BATCH_TASK_INDEX" +echo "TI: ${ticker[$BATCH_TASK_INDEX]}" +python3 -m pip install -r /mnt/disks/fsi/mc_run_reqs.txt +python3 /mnt/disks/fsi/mc_run.py \ + --ticker "${ticker[$BATCH_TASK_INDEX]}" \ + --iterations 500 \ + --start_date 2022-01-01 diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf new file mode 100644 index 0000000000..83dc7fe9cf --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf @@ -0,0 +1,102 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + bucket = replace(var.gcs_bucket_path, "gs://", "") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +data "template_file" "mc_run_py" { + template = file("${path.module}/mc_run.tpl.py") + vars = { + project_id = var.project_id + topic_id = var.topic_id + topic_schema = var.topic_schema + dataset_id = var.dataset_id + table_id = var.table_id + } +} + +resource "google_storage_bucket_object" "mc_run" { + name = "mc_run.py" + content = data.template_file.mc_run_py.rendered + bucket = local.bucket +} + +data "template_file" "mc_run_yaml" { + template = file("${path.module}/mc_run.tpl.yaml") + vars = { + project_id = var.project_id + bucket_name = local.bucket + region = var.region + } +} + +resource "google_storage_bucket_object" "mc_obj_yaml" { + name = "mc_run.yaml" + content = data.template_file.mc_run_yaml.rendered + bucket = local.bucket +} + +data "template_file" "ipynb_fsi" { + template = file("${path.module}/FSI_MonteCarlo.ipynb") + vars = { + project_id = var.project_id + dataset_id = var.dataset_id + table_id = var.table_id + } +} +resource "google_storage_bucket_object" "ipynb_obj_fsi" { + name = "FSI_MonteCarlo.ipynb" + content = data.template_file.ipynb_fsi.rendered + bucket = local.bucket +} + +data "http" "batch_py" { + url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/batch.py" +} + +resource "google_storage_bucket_object" "run_batch_py" { + name = "batch.py" + content = data.http.batch_py.response_body + bucket = local.bucket +} + +data "http" "batch_requirements" { + url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/requirements.txt" +} + +resource "google_storage_bucket_object" "get_requirements" { + name = "requirements.txt" + content = data.http.batch_requirements.response_body + bucket = local.bucket +} + +resource "google_storage_bucket_object" "get_iteration_sh" { + name = "iteration.sh" + content = file("${path.module}/iteration.sh") + bucket = local.bucket +} + +resource "google_storage_bucket_object" "get_mc_reqs" { + name = "mc_run_reqs.txt" + content = file("${path.module}/mc_run_reqs.txt") + bucket = local.bucket +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py new file mode 100644 index 0000000000..4e0a64e363 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Run MC simulation for VaR portfolio risk +""" + +import avro.schema +import io +import google.auth +import numpy +import time +import yfinance as yf + +from absl import app +from absl import flags +from avro.io import DatumWriter, BinaryEncoder, BinaryDecoder, DatumReader +from datetime import datetime +from datetime import timedelta +from google.cloud import pubsub_v1, bigquery +from google.cloud.pubsub import SchemaServiceClient + +PROJECT_ID = '${project_id}' +INCOMING_TOPIC_ID = '${topic_id}' +INCOMING_TOPIC_SCHEMA = '${topic_schema}' +DATASET_ID = '${dataset_id}' +TABLE_ID = '${table_id}' + + +FLAGS = flags.FLAGS + +flags.DEFINE_string("ticker", 'GOOG', "Nasdaq Stock Ticker to run, default GOOG") +flags.DEFINE_string("start_date", '2022-01-01' , "Start data for data query, default 2022-01-01") +flags.DEFINE_integer("calendar_days", 365 , "How many calendar days to include in the calculation") +flags.DEFINE_integer("epoch_time", f'{int(time.time())}' , "Epoch time, number of seconds since January 1st, 1970 at 00:00:00 UTC.") +flags.DEFINE_integer("iterations", 100 , "Number of iterations to run.") +flags.DEFINE_boolean("print_raw", False, "Dump raw data.") + +class VaRSimulator: + + def __init__(self): + pass + + def get_data(self): + self.get_historical_data_yahoo() + + def get_historical_data_yahoo(self): + + # get historical market data: https://pypi.org/project/yfinance/ + + self.raw_data = yf.Ticker(self.ticker).history(start=self.start_date, end=self.end_date ) + self.data = self.raw_data.Close + + def print_raw(self): + print(self.get_stats()) + print(type(self.raw_data)) + print(self.raw_data) + + def get_stats(self): + close = self.data + self.first = close[0] + self.last = close[-1] + self.trading_days = len(close) + self.cagr = (self.last / self.first) ** (365.0/self.calendar_days) -1.0 + self.volatility = self.data.pct_change().std() + return(self.first, self.last, self.trading_days, self.cagr, self.volatility) + + def run_simulation(self): + + returns = numpy.random.normal(self.cagr/self.trading_days, self.volatility, self.trading_days) + 1 + returns = numpy.insert(returns,0,1.0) + self.simulation_results = self.last * returns.cumprod() + return(self.simulation_results) + + def create_object(self): + self.object = { + "ticker": self.ticker, + "epoch_time": self.epoch_time, + "iteration": self.iteration, + "start_date": self.start_date, + "end_date": self.end_date, + "simulation_results": list(map(lambda x: {"price":x}, self.simulation_results)) + } + return(self.object) + + +class PubsubToBiquery: + + def __init__(self): + + the_time = int(time.time()) + + self.project_id = PROJECT_ID + + self.publisher_client = pubsub_v1.PublisherClient() + self.topic_path = self.publisher_client.topic_path(self.project_id, INCOMING_TOPIC_ID) + + self.schema_client = SchemaServiceClient() + self.schema_path = self.schema_client.schema_path(self.project_id, INCOMING_TOPIC_SCHEMA) + + pubsub_schema = self.schema_client.get_schema(request={"name": self.schema_path}) + avro_schema = avro.schema.parse(pubsub_schema.definition) + + self.writer = DatumWriter(avro_schema) + + + def publish_record(self,record): + + byte_stream = io.BytesIO() + encoder = BinaryEncoder(byte_stream) + self.writer.write(record, encoder) + data = byte_stream.getvalue() + byte_stream.flush() + future = self.publisher_client.publish(self.topic_path, data) + if(FLAGS.print_raw): + print(f"Published message ID: {future.result()}") + + +def main(argv): + + vr = VaRSimulator() + pbbq = PubsubToBiquery() + + vr.ticker =FLAGS.ticker + vr.start_date =FLAGS.start_date + vr.end_date =f'{(datetime.strptime(FLAGS.start_date,"%Y-%m-%d") + timedelta(days = FLAGS.calendar_days)).date()}' + vr.calendar_days = FLAGS.calendar_days + vr.epoch_time = FLAGS.epoch_time + vr.iteration = 1 + + vr.get_data() + vr.get_stats() + + for i in range(FLAGS.iterations): + vr.iteration = i + vr.run_simulation() + pbbq.publish_record(vr.create_object()) + + if(FLAGS.print_raw): + vr.print_raw() + + +if __name__ == "__main__": + """ This is executed when run from the command line """ + app.run(main) diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml new file mode 100644 index 0000000000..7f7de4840b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +project_id: "${project_id}" +region: "${region}" + +job_prefix: 'fsi-' +machine_type: "n2-standard-2" +volumes: +- {bucket_name: "${bucket_name}", gcs_path: "/mnt/disks/fsi"} + +container: + image_uri: "python" + entry_point: "/bin/bash" + commands: ["/mnt/disks/fsi/iteration.sh", "$BATCH_TASK_INDEX"] + +task_count: 8 #optional +parallelism: 4 #optional +task_count_per_node: 2 #optional +cpu_milli: 1000 #optional +memory_mib: 102400 #optional + + +labels: + env: "monte" + type: "carlo" diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt new file mode 100644 index 0000000000..105ed70ad2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt @@ -0,0 +1,9 @@ +absl-py +avro +google-auth +google-cloud +google-cloud-batch +google-cloud-pubsub +google-cloud-bigquery +yfinance +PyYAML diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml new file mode 100644 index 0000000000..268c8faa9a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [storage.googleapis.com] diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf new file mode 100644 index 0000000000..eddf3c9478 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which GCS bucket will be created." + type = string +} + +variable "gcs_bucket_path" { + description = "Bucket name" + type = string + default = null +} + +variable "topic_id" { + description = "Pubsub Topic Name" + type = string +} + +variable "topic_schema" { + description = "Pubsub Topic schema" + type = string +} + +variable "dataset_id" { + description = "Bigquery dataset id" + type = string +} + +variable "table_id" { + description = "Bigquery table id" + type = string +} + +variable "region" { + description = "Region to run project" + type = string +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf new file mode 100644 index 0000000000..86dcb4dc52 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + http = { + source = "hashicorp/http" + version = "~> 3.0" + } + template = { + source = "hashicorp/template" + version = "~> 2.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md new file mode 100644 index 0000000000..ae8462d763 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md @@ -0,0 +1,100 @@ +# Module: Slurm Instance + + + +- [Module: Slurm Instance](#module-slurm-instance) + - [Overview](#overview) + - [Module API](#module-api) + + + +## Overview + +This module creates a [compute instance](../../../../docs/glossary.md#vm) from +[instance template](../../../../docs/glossary.md#instance-template) for a +[Slurm cluster](../slurm_cluster/README.md). + +> **NOTE:** This module is only intended to be used by Slurm modules. For +> general usage, please consider using: +> +> - [terraform-google-modules/vm/google//modules/compute_instance](https://registry.terraform.io/modules/terraform-google-modules/vm/google/latest/submodules/compute_instance). +> **WARNING:** The source image is not modified. Make sure to use a compatible +> source image. + +## Module API + +For the terraform module API reference, please see +[README_TF.md](./README_TF.md). + + +Copyright (C) SchedMD LLC. +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.0 | +| [google](#requirement\_google) | >= 3.43 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.43 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_from_template.slurm_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | +| [null_resource.replace_trigger](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_instance_template.base](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | +| [hostname](#input\_hostname) | Hostname of instances | `string` | n/a | yes | +| [instance\_template](#input\_instance\_template) | Instance template self\_link used to create compute instances | `string` | n/a | yes | +| [network](#input\_network) | Network to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | +| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | +| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | +| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | +| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances | `list(string)` | `[]` | no | +| [subnetwork](#input\_subnetwork) | Subnet to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The project that subnetwork belongs to | `string` | `null` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [available\_zones](#output\_available\_zones) | List of available zones in region | +| [instances\_details](#output\_instances\_details) | List of all details for compute instances | +| [instances\_self\_links](#output\_instances\_self\_links) | List of self-links for compute instances | +| [names](#output\_names) | List of available zones in region | +| [slurm\_instances](#output\_slurm\_instances) | List of all resource objects for compute instances | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf new file mode 100644 index 0000000000..2af9008a0e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf @@ -0,0 +1,126 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########## +# LOCALS # +########## + +locals { + num_instances = length(var.static_ips) == 0 ? var.num_instances : length(var.static_ips) + + # local.static_ips is the same as var.static_ips with a dummy element appended + # at the end of the list to work around "list does not have any elements so cannot + # determine type" error when var.static_ips is empty + static_ips = concat(var.static_ips, ["NOT_AN_IP"]) + + network_interfaces = [for index in range(local.num_instances) : + concat([ + { + access_config = var.access_config + alias_ip_range = [] + ipv6_access_config = [] + network = var.network + network_ip = length(var.static_ips) == 0 ? "" : element(local.static_ips, index) + nic_type = null + queue_count = null + stack_type = null + subnetwork = var.subnetwork + subnetwork_project = var.subnetwork_project + } + ], + var.additional_networks + ) + ] +} + +################ +# DATA SOURCES # +################ + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +data "google_compute_instance_template" "base" { + project = var.project_id + name = var.instance_template +} + +############# +# INSTANCES # +############# +resource "null_resource" "replace_trigger" { + triggers = { + trigger = var.replace_trigger + } +} + +# TODO: `internal/slurm-gcp/login` is ONLY user of `internal/slurm-gcp/instance` +# Remove this module, add functionality (+ prune generality) to the login module directly. +resource "google_compute_instance_from_template" "slurm_instance" { + count = local.num_instances + name = format("%s-%s", var.hostname, format("%03d", count.index + 1)) + project = var.project_id + zone = var.zone == null ? data.google_compute_zones.available.names[count.index % length(data.google_compute_zones.available.names)] : var.zone + + allow_stopping_for_update = true + + dynamic "network_interface" { + for_each = local.network_interfaces[count.index] + iterator = nic + content { + dynamic "access_config" { + for_each = nic.value.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = nic.value.alias_ip_range + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + dynamic "ipv6_access_config" { + for_each = nic.value.ipv6_access_config + iterator = access_config + content { + network_tier = access_config.value.network_tier + } + } + network = nic.value.network + network_ip = nic.value.network_ip + nic_type = nic.value.nic_type + queue_count = nic.value.queue_count + subnetwork = nic.value.subnetwork + subnetwork_project = nic.value.subnetwork_project + } + } + + source_instance_template = data.google_compute_instance_template.base.self_link + # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 + # we have to explicitly override instance labels instead of inheriting them from template. + labels = data.google_compute_instance_template.base.labels + + + lifecycle { + replace_triggered_by = [null_resource.replace_trigger.id] + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf new file mode 100644 index 0000000000..4eba78a7e8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf @@ -0,0 +1,41 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "slurm_instances" { + description = "List of all resource objects for compute instances" + value = google_compute_instance_from_template.slurm_instance +} + +output "instances_self_links" { + description = "List of self-links for compute instances" + value = google_compute_instance_from_template.slurm_instance[*].self_link +} + +output "instances_details" { + description = "List of all details for compute instances" + value = google_compute_instance_from_template.slurm_instance[*] +} + +output "available_zones" { + description = "List of available zones in region" + value = data.google_compute_zones.available.names +} + +output "names" { + description = "List of available zones in region" + value = google_compute_instance_from_template.slurm_instance[*].name +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf new file mode 100644 index 0000000000..11111a2c05 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf @@ -0,0 +1,119 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The GCP project ID" + default = null +} + +variable "network" { + description = "Network to deploy to. Only one of network or subnetwork should be specified." + type = string + default = "" +} + +variable "subnetwork" { + description = "Subnet to deploy to. Only one of network or subnetwork should be specified." + type = string + default = "" +} + +variable "subnetwork_project" { + description = "The project that subnetwork belongs to" + type = string + default = null +} + +variable "hostname" { + description = "Hostname of instances" + type = string +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })) + nullable = false +} + +variable "static_ips" { + description = "List of static IPs for VM instances" + type = list(string) + default = [] +} + +variable "access_config" { + description = "Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet." + type = list(object({ + nat_ip = string + network_tier = string + })) + default = [] +} + +variable "num_instances" { + description = "Number of instances to create. This value is ignored if static_ips is provided." + type = number + default = 1 +} + +variable "instance_template" { + description = "Instance template self_link used to create compute instances" + type = string +} + +variable "region" { + description = "Region where the instances should be created." + type = string + default = null +} + +variable "zone" { + description = "Zone where the instances should be created. If not specified, instances will be spread across available zones in the region." + type = string + default = null +} + +######### +# SLURM # +######### + +variable "replace_trigger" { + description = "Trigger value to replace the instances." + type = string + default = "" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf new file mode 100644 index 0000000000..a3e84c09bf --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf @@ -0,0 +1,31 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 1.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.43" + } + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md new file mode 100644 index 0000000000..87394bef6a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md @@ -0,0 +1,87 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.0 | +| [local](#requirement\_local) | ~> 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | ~> 2.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_template](#module\_instance\_template) | ../internal_instance_template | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.startup](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    disk_type                  = optional(string)
    disk_size_gb               = optional(number)
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `virtio_enabled` setting will only enable VirtioNet and will not enable TIER\_1.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard. | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See
https://cloud.google.com/compute/docs/gpus more details.
- type : the GPU type
- count : number of GPUs | 
object({
    type  = string
    count = number
  })
| `null` | no | +| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"n1-standard-1"` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Prefix for template resource. | `string` | `"default"` | no | +| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network
attribute for Legacy or Auto subnetted networks and subnetwork for custom
subnetted networks. | `string` | `null` | no | +| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | +| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the instances. See
'main.tf:local.service\_account' for the default. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming. | `string` | n/a | yes | +| [slurm\_instance\_role](#input\_slurm\_instance\_role) | Slurm instance type. Must be one of: controller; login; compute; or null. | `string` | n/a | yes | +| [source\_image](#input\_source\_image) | Source disk image. | `string` | `""` | no | +| [source\_image\_family](#input\_source\_image\_family) | Source image family. | `string` | `""` | no | +| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. If it is not provided, the provider project is used. | `string` | `""` | no | +| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must
exist in the same region this instance will be created in. Either network or
subnetwork must be provided. | `string` | `null` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [termination\_action](#input\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_template](#output\_instance\_template) | Instance template details | +| [labels](#output\_labels) | Labels attached to the instance template | +| [name](#output\_name) | Name of instance template | +| [self\_link](#output\_self\_link) | Self\_link of instance template | +| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | +| [tags](#output\_tags) | Tags that will be associated with instance(s) | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted new file mode 100644 index 0000000000..2edaa942d2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted @@ -0,0 +1,169 @@ +#!/bin/bash +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SLURM_DIR=/slurm +FLAGFILE=$SLURM_DIR/slurm_configured_do_not_remove +SCRIPTS_DIR=$SLURM_DIR/scripts +if [[ -z "$HOME" ]]; then + # google-startup-scripts.service lacks environment variables + HOME="$(getent passwd "$(whoami)" | cut -d: -f6)" +fi + +# Temporary workaround for transition period when some of older images +# don't have "baked in" python yet. +# TODO: Remove +SLURM_PY="/slurm/python/venv/bin/python3.13" +SYSTEM_PY="/usr/bin/python3" +if [[ ! -e "$SLURM_PY" ]]; then + echo "Symlink $SLURM_PY does not exist. Creating symlink to $SYSTEM_PY" + mkdir -p /slurm/python/venv/bin + ln -s "$SYSTEM_PY" "$SLURM_PY" +fi + +METADATA_SERVER="metadata.google.internal" +URL="http://$METADATA_SERVER/computeMetadata/v1" +CURL="curl -sS --fail --header Metadata-Flavor:Google" + +PING_METADATA="ping -q -w1 -c1 $METADATA_SERVER" +echo "INFO: $PING_METADATA" +for i in $(seq 10); do + [ $i -gt 1 ] && sleep 5; + $PING_METADATA > /dev/null && s=0 && break || s=$?; + echo "ERROR: Failed to contact metadata server, will retry" +done +if [ $s -ne 0 ]; then + echo "ERROR: Unable to contact metadata server, aborting" + wall -n '*** Slurm setup failed in the startup script! see `journalctl -u google-startup-scripts` ***' + exit 1 +else + echo "INFO: Successfully contacted metadata server" +fi + +PING_GOOGLE="ping -q -w1 -c1 8.8.8.8" +echo "INFO: $PING_GOOGLE" +for i in $(seq 5); do + [ $i -gt 1 ] && sleep 2; + $PING_GOOGLE > /dev/null && s=0 && break || s=$?; + echo "failed to ping Google DNS, will retry" +done +if [ $s -ne 0 ]; then + echo "WARNING: No internet access detected" +else + echo "INFO: Internet access detected" +fi + +mkdir -p $SCRIPTS_DIR +UNIVERSE_DOMAIN="$($CURL $URL/instance/attributes/universe_domain)" +BUCKET="$($CURL $URL/instance/attributes/slurm_bucket_path)" +if [[ -z $BUCKET ]]; then + echo "ERROR: No bucket path detected." + exit 1 +fi + +SCRIPTS_ZIP="$HOME/slurm-gcp-scripts.zip" +export CLOUDSDK_CORE_UNIVERSE_DOMAIN="$UNIVERSE_DOMAIN" + +INSTANCE_ROLE="$($CURL $URL/instance/attributes/slurm_instance_role)" + +if [ "$INSTANCE_ROLE" == "controller" ]; then + DEVEL_ZIP="slurm-gcp-devel-controller.zip" +else + DEVEL_ZIP="slurm-gcp-devel.zip" +fi +until gcloud storage cp "$BUCKET/$DEVEL_ZIP" "$SCRIPTS_ZIP"; do + echo "WARN: Could not download SlurmGCP scripts, retrying in 5 seconds." + # Remove marker used to determine if gcloud is being used in a GCE VM. + # This can get mistakenly set to False in some cases. + rm -f /root/.config/gcloud/gce + sleep 5 +done +unzip -o "$SCRIPTS_ZIP" -d "$SCRIPTS_DIR" +rm -rf "$SCRIPTS_ZIP" + +#temporary hack to not make the script fail on TPU vm +chown slurm:slurm -R "$SCRIPTS_DIR" || true +chmod 700 -R "$SCRIPTS_DIR" + + +if [ -f $FLAGFILE ]; then + echo "WARNING: Slurm was previously configured, quitting" + exit 0 +fi +touch $FLAGFILE + +function tpu_setup { + #allow the following command to fail, as this attribute does not exist for regular nodes + docker_image=$($CURL $URL/instance/attributes/slurm_docker_image 2> /dev/null || true) + if [ -z $docker_image ]; then #Not a tpu node, do not do anything + return + fi + if [ "$OS_ENV" == "slurm_container" ]; then #Already inside the slurm container, we should continue starting + return + fi + + #given a input_string like "WORKER_0:Joseph;WORKER_1:richard;WORKER_2:edward;WORKER_3:john" and a number 1, this function will print richard + parse_metadata() { + local number=$1 + local input_string=$2 + local word=$(echo "$input_string" | awk -v n="$number" -F ':|;' '{ for (i = 1; i <= NF; i+=2) if ($(i) == "WORKER_"n) print $(i+1) }') + echo "$word" + } + + input_string=$($CURL $URL/instance/attributes/slurm_names) + worker_id=$($CURL $URL/instance/attributes/tpu-env | awk '/WORKER_ID/ {print $2}' | tr -d \') + real_name=$(parse_metadata $worker_id $input_string) + + #Prepare to docker pull with gcloud + mkdir -p /root/.docker + cat << EOF > /root/.docker/config.json +{ + "credHelpers": { + "gcr.io": "gcloud", + "us-docker.pkg.dev": "gcloud" + } +} +EOF + #cgroup detection + CGV=1 + CGROUP_FLAGS="-v /sys/fs/cgroup:/sys/fs/cgroup:rw" + if [ -f /sys/fs/cgroup/cgroup.controllers ]; then #CGV2 + CGV=2 + fi + if [ $CGV == 2 ]; then + CGROUP_FLAGS="--cgroup-parent=docker.slice --cgroupns=private --tmpfs /run --tmpfs /run/lock --tmpfs /tmp" + if [ ! -f /etc/systemd/system/docker.slice ]; then #In case that there is no slice prepared for hosting the containers create it + printf "[Unit]\nDescription=docker slice\nBefore=slices.target\n[Slice]\nCPUAccounting=true\nMemoryAccounting=true" > /etc/systemd/system/docker.slice + systemctl start docker.slice + fi + fi + #for the moment always use --privileged, as systemd might not work properly otherwise + TPU_FLAGS="--privileged" + # TPU_FLAGS="--cap-add SYS_RESOURCE --device /dev/accel0 --device /dev/accel1 --device /dev/accel2 --device /dev/accel3" + # if [ $CGV == 2 ]; then #In case that we are in CGV2 for systemd to work correctly for the moment we go with privileged + # TPU_FLAGS="--privileged" + # fi + + docker run -d $CGROUP_FLAGS $TPU_FLAGS --net=host --name=slurmd --hostname=$real_name --entrypoint=/usr/bin/systemd --restart unless-stopped $docker_image + exit 0 +} + +tpu_setup #will do nothing for normal nodes or the container spawned inside TPU + +echo "INFO: Running python cluster setup script" +SETUP_SCRIPT_FILE=$SCRIPTS_DIR/setup.py +chmod +x $SETUP_SCRIPT_FILE +exec $SETUP_SCRIPT_FILE diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf new file mode 100644 index 0000000000..c91bbc4fd1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf @@ -0,0 +1,171 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "instance_validation" { + source = "../../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +########## +# LOCALS # +########## + +locals { + additional_disks = [ + for disk in var.additional_disks : { + disk_name = disk.disk_name + device_name = disk.device_name + auto_delete = disk.auto_delete + source = disk.source + boot = disk.boot + disk_size_gb = disk.disk_size_gb + disk_type = disk.disk_type + disk_labels = merge( + disk.disk_labels, + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) + disk_resource_manager_tags = disk.disk_resource_manager_tags + } + ] + + service_account = { + email = try(var.service_account.email, null) + scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) + } + + source_image_family = ( + var.source_image_family != "" && var.source_image_family != null + ? var.source_image_family + : "slurm-gcp-6-11-hpc-rocky-linux-8" + ) + source_image_project = ( + var.source_image_project != "" && var.source_image_project != null + ? var.source_image_project + : "projects/schedmd-slurm-public/global/images/family" + ) + + source_image = ( + var.source_image != null + ? var.source_image + : "" + ) + + + name_prefix = "${var.slurm_cluster_name}-${var.slurm_instance_role}-${var.name_prefix}" + + total_egress_bandwidth_tier = var.bandwidth_tier == "tier_1_enabled" ? "TIER_1" : "DEFAULT" + + nic_type_map = { + platform_default = null + virtio_enabled = "VIRTIO_NET" + gvnic_enabled = "GVNIC" + tier_1_enabled = "GVNIC" + } + nic_type = lookup(local.nic_type_map, var.bandwidth_tier, null) + + labels = merge(var.labels, + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) +} + +######## +# DATA # +######## + +data "local_file" "startup" { + filename = "${path.module}/files/startup_sh_unlinted" +} + +############ +# TEMPLATE # +############ + +module "instance_template" { + source = "../internal_instance_template" + + project_id = var.project_id + + # Network + can_ip_forward = var.can_ip_forward + network_ip = var.network_ip + network = var.network + nic_type = local.nic_type + region = var.region + subnetwork_project = var.subnetwork_project + subnetwork = var.subnetwork + tags = var.tags + total_egress_bandwidth_tier = local.total_egress_bandwidth_tier + additional_networks = var.additional_networks + access_config = var.access_config + + # Instance + machine_type = var.machine_type + min_cpu_platform = var.min_cpu_platform + name_prefix = local.name_prefix + gpu = var.gpu + service_account = local.service_account + shielded_instance_config = var.shielded_instance_config + advanced_machine_features = var.advanced_machine_features + enable_confidential_vm = var.enable_confidential_vm + enable_shielded_vm = var.enable_shielded_vm + preemptible = var.preemptible + spot = var.spot + on_host_maintenance = var.on_host_maintenance + labels = local.labels + instance_termination_action = var.termination_action + resource_manager_tags = var.resource_manager_tags + + # Metadata + startup_script = coalesce(var.internal_startup_script, data.local_file.startup.content) + metadata = merge( + var.metadata, + { + enable-oslogin = upper(var.enable_oslogin) + slurm_bucket_path = var.slurm_bucket_path + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) + + # Image + source_image_project = local.source_image_project + source_image_family = local.source_image_family + source_image = local.source_image + + # Disk + disk_type = var.disk_type + disk_size_gb = var.disk_size_gb + auto_delete = var.disk_auto_delete + disk_labels = merge( + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + var.disk_labels, + ) + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + + max_run_duration = var.max_run_duration + provisioning_model = var.provisioning_model + reservation_affinity = var.reservation_affinity +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf new file mode 100644 index 0000000000..65da41052e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "instance_template" { + description = "Instance template details" + value = module.instance_template +} + +output "self_link" { + description = "Self_link of instance template" + value = module.instance_template.self_link +} + +output "name" { + description = "Name of instance template" + value = module.instance_template.name +} + +output "tags" { + description = "Tags that will be associated with instance(s)" + value = module.instance_template.tags +} + +output "service_account" { + description = "Service account object, includes email and scopes." + value = module.instance_template.service_account +} + +output "labels" { + description = "Labels attached to the instance template" + value = local.labels +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf new file mode 100644 index 0000000000..35dd9c376f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf @@ -0,0 +1,431 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +########### +# GENERAL # +########### + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "on_host_maintenance" { + type = string + description = "Instance availability Policy" + default = "MIGRATE" +} + +variable "labels" { + type = map(string) + description = "Labels, provided as a map" + default = {} +} + +variable "enable_oslogin" { + type = bool + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >=0.13.0 | +| [google](#requirement\_google) | >= 3.88 | +| [google-beta](#requirement\_google-beta) | >= 6.13.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.88 | +| [google-beta](#provider\_google-beta) | >= 6.13.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance_template.tpl](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_template) | resource | +| [google_project.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    auto_delete                = bool
    boot                       = bool
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = map(string)
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | +| [alias\_ip\_range](#input\_alias\_ip\_range) | An array of alias IP ranges for this network interface. Can only be specified for network interfaces on subnet-mode networks.
ip\_cidr\_range: The IP CIDR range represented by this alias IP range. This IP CIDR range must belong to the specified subnetwork and cannot contain IP addresses reserved by system or used by other network interfaces. At the time of writing only a netmask (e.g. /24) may be supplied, with a CIDR format resulting in an API error.
subnetwork\_range\_name: The subnetwork secondary range name specifying the secondary range from which to allocate the IP CIDR range for this alias IP range. If left unspecified, the primary range of the subnetwork will be used. | 
object({
    ip_cidr_range         = string
    subnetwork_range_name = string
  })
| `null` | no | +| [auto\_delete](#input\_auto\_delete) | Whether or not the boot disk should be auto-deleted | `string` | `"true"` | no | +| [automatic\_restart](#input\_automatic\_restart) | (Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user). | `bool` | `true` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example | `string` | `"false"` | no | +| [disk\_encryption\_key](#input\_disk\_encryption\_key) | The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance | `string` | `null` | no | +| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `string` | `"100"` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Whether to enable the Confidential VM configuration on the instance. Note that the instance image must support Confidential VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Whether to enable the Shielded VM configuration on the instance. Note that the instance image must support Shielded VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | +| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See https://cloud.google.com/compute/docs/gpus more details | 
object({
    type  = string
    count = number
  })
| `null` | no | +| [instance\_termination\_action](#input\_instance\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | +| [ipv6\_access\_config](#input\_ipv6\_access\_config) | IPv6 access configurations. Currently a max of 1 IPv6 access configuration is supported. If not specified, the instance will have no external IPv6 Internet access. | 
list(object({
    network_tier = string
  }))
| `[]` | no | +| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create, e.g. n1-standard-1 | `string` | `"n1-standard-1"` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Name prefix for the instance template | `string` | n/a | yes | +| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks. | `string` | `""` | no | +| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | +| [nic\_type](#input\_nic\_type) | The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO\_NET. | `string` | `null` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | +| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | +| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#service_account. | 
object({
    email  = optional(string)
    scopes = set(string)
  })
| n/a | yes | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Not used unless enable\_shielded\_vm is true. Shielded VM configuration for the instance. | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [source\_image](#input\_source\_image) | Source disk image. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `""` | no | +| [source\_image\_family](#input\_source\_image\_family) | Source image family. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `"centos-7"` | no | +| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. The default project contains CentOS images. | `string` | `"centos-cloud"` | no | +| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | +| [stack\_type](#input\_stack\_type) | The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4\_ONLY. | `string` | `null` | no | +| [startup\_script](#input\_startup\_script) | User startup script to run when instances spin up | `string` | `""` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided. | `string` | `""` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [total\_egress\_bandwidth\_tier](#input\_total\_egress\_bandwidth\_tier) | Network bandwidth tier. Note: machine\_type must be a supported type. Values are 'TIER\_1' or 'DEFAULT'.
See https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration for details. | `string` | `"DEFAULT"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [name](#output\_name) | Name of instance template | +| [self\_link](#output\_self\_link) | Self-link of instance template | +| [service\_account](#output\_service\_account) | value | +| [tags](#output\_tags) | Tags that will be associated with instance(s) | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf new file mode 100644 index 0000000000..f8d2813ece --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf @@ -0,0 +1,234 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "instance_validation" { + source = "../../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +######### +# Locals +######### + +locals { + source_image = var.source_image != "" ? var.source_image : "centos-7-v20201112" + source_image_family = var.source_image_family != "" ? var.source_image_family : "centos-7" + source_image_project = var.source_image_project != "" ? var.source_image_project : "centos-cloud" + + boot_disk = [ + { + source_image = var.source_image != "" ? format("${local.source_image_project}/${local.source_image}") : format("${local.source_image_project}/${local.source_image_family}") + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_labels = var.disk_labels + auto_delete = var.auto_delete + disk_resource_manager_tags = var.disk_resource_manager_tags + boot = "true" + }, + ] + + all_disks = concat(local.boot_disk, var.additional_disks) + + # NOTE: Even if all the shielded_instance_config or confidential_instance_config + # values are false, if the config block exists and an unsupported image is chosen, + # the apply will fail so we use a single-value array with the default value to + # initialize the block only if it is enabled. + shielded_vm_configs = var.enable_shielded_vm ? [true] : [] + + gpu_enabled = var.gpu != null + alias_ip_range_enabled = var.alias_ip_range != null + preemptible = var.preemptible || var.spot + on_host_maintenance = ( + local.preemptible || var.enable_confidential_vm || local.gpu_enabled + ? "TERMINATE" + : var.on_host_maintenance + ) + automatic_restart = ( + # must be false when preemptible is true + local.preemptible ? false : var.automatic_restart + ) + + nic_type = var.total_egress_bandwidth_tier == "TIER_1" ? "GVNIC" : var.nic_type + + + provisioning_model = coalesce(var.provisioning_model, local.preemptible ? "SPOT" : "STANDARD") +} + +data "google_project" "this" { + project_id = var.project_id +} + +#################### +# Instance Template +#################### +resource "google_compute_instance_template" "tpl" { + provider = google-beta + name_prefix = "${var.name_prefix}-" + project = var.project_id + machine_type = var.machine_type + labels = var.labels + metadata = var.metadata + tags = var.tags + can_ip_forward = var.can_ip_forward + metadata_startup_script = var.startup_script + region = var.region + min_cpu_platform = var.min_cpu_platform + resource_manager_tags = var.resource_manager_tags + + service_account { + email = coalesce(var.service_account.email, "${data.google_project.this.number}-compute@developer.gserviceaccount.com") + scopes = lookup(var.service_account, "scopes", null) + } + + dynamic "disk" { + for_each = local.all_disks + content { + auto_delete = lookup(disk.value, "auto_delete", null) + boot = lookup(disk.value, "boot", null) + device_name = lookup(disk.value, "device_name", null) + disk_name = lookup(disk.value, "disk_name", null) + disk_size_gb = lookup(disk.value, "disk_size_gb", lookup(disk.value, "disk_type", null) == "local-ssd" ? "375" : null) + disk_type = lookup(disk.value, "disk_type", null) + interface = lookup(disk.value, "interface", lookup(disk.value, "disk_type", null) == "local-ssd" ? "NVME" : null) + mode = lookup(disk.value, "mode", null) + source = lookup(disk.value, "source", null) + source_image = lookup(disk.value, "source_image", null) + type = lookup(disk.value, "disk_type", null) == "local-ssd" ? "SCRATCH" : "PERSISTENT" + labels = (lookup(disk.value, "source", null) != null || lookup(disk.value, "disk_type", null) == "local-ssd") ? null : lookup(disk.value, "disk_labels", null) + resource_manager_tags = lookup(disk.value, "disk_resource_manager_tags", {}) + + dynamic "disk_encryption_key" { + for_each = compact([var.disk_encryption_key == null ? null : 1]) + content { + kms_key_self_link = var.disk_encryption_key + } + } + } + } + + network_interface { + network = var.network + subnetwork = var.subnetwork + subnetwork_project = var.subnetwork_project + network_ip = try(coalesce(var.network_ip), null) + nic_type = local.nic_type + stack_type = var.stack_type + dynamic "access_config" { + for_each = var.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = var.ipv6_access_config + content { + network_tier = ipv6_access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = local.alias_ip_range_enabled ? [var.alias_ip_range] : [] + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + } + + dynamic "network_interface" { + for_each = var.additional_networks + content { + network = network_interface.value.network + subnetwork = network_interface.value.subnetwork + subnetwork_project = network_interface.value.subnetwork_project + network_ip = try(coalesce(network_interface.value.network_ip), null) + nic_type = try(coalesce(network_interface.value.nic_type), null) + dynamic "access_config" { + for_each = network_interface.value.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = network_interface.value.ipv6_access_config + content { + network_tier = ipv6_access_config.value.network_tier + } + } + } + } + + network_performance_config { + total_egress_bandwidth_tier = coalesce(var.total_egress_bandwidth_tier, "DEFAULT") + } + + lifecycle { + create_before_destroy = "true" + } + + scheduling { + preemptible = local.preemptible + provisioning_model = local.provisioning_model + automatic_restart = local.automatic_restart + on_host_maintenance = local.on_host_maintenance + instance_termination_action = var.instance_termination_action + + dynamic "max_run_duration" { + for_each = var.max_run_duration != null ? [var.max_run_duration] : [] + content { + seconds = max_run_duration.value + } + } + } + + dynamic "reservation_affinity" { + for_each = var.reservation_affinity != null ? [var.reservation_affinity] : [] + content { + type = reservation_affinity.value.type + } + } + + advanced_machine_features { + enable_nested_virtualization = var.advanced_machine_features.enable_nested_virtualization + threads_per_core = var.advanced_machine_features.threads_per_core + turbo_mode = var.advanced_machine_features.turbo_mode + visible_core_count = var.advanced_machine_features.visible_core_count + performance_monitoring_unit = var.advanced_machine_features.performance_monitoring_unit + enable_uefi_networking = var.advanced_machine_features.enable_uefi_networking + } + + dynamic "shielded_instance_config" { + for_each = local.shielded_vm_configs + content { + enable_secure_boot = lookup(var.shielded_instance_config, "enable_secure_boot", shielded_instance_config.value) + enable_vtpm = lookup(var.shielded_instance_config, "enable_vtpm", shielded_instance_config.value) + enable_integrity_monitoring = lookup(var.shielded_instance_config, "enable_integrity_monitoring", shielded_instance_config.value) + } + } + + confidential_instance_config { + enable_confidential_compute = var.enable_confidential_vm + } + + dynamic "guest_accelerator" { + for_each = local.gpu_enabled ? [var.gpu] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf new file mode 100644 index 0000000000..69f8d3b98c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "self_link" { + description = "Self-link of instance template" + value = google_compute_instance_template.tpl.self_link +} + +output "name" { + description = "Name of instance template" + value = google_compute_instance_template.tpl.name +} + +output "tags" { + description = "Tags that will be associated with instance(s)" + value = google_compute_instance_template.tpl.tags +} + +output "service_account" { + description = "value" + value = google_compute_instance_template.tpl.service_account[0] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf new file mode 100644 index 0000000000..c285c3fea5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf @@ -0,0 +1,398 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + type = string + description = "The GCP project ID" + default = null +} + +variable "name_prefix" { + description = "Name prefix for the instance template" + type = string +} + +variable "machine_type" { + description = "Machine type to create, e.g. n1-standard-1" + type = string + default = "n1-standard-1" +} + +variable "min_cpu_platform" { + description = "Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform" + type = string + default = null +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example" + type = string + default = "false" +} + +variable "tags" { + type = list(string) + description = "Network tags, provided as a list" + default = [] +} + +variable "labels" { + type = map(string) + description = "Labels, provided as a map" + default = {} +} + +variable "preemptible" { + type = bool + description = "Allow the instance to be preempted" + default = false +} + +variable "spot" { + description = <<-EOD + Provision as a SPOT preemptible instance. + See https://cloud.google.com/compute/docs/instances/spot for more details. + EOD + type = bool + default = false +} + +variable "instance_termination_action" { + description = <<-EOD + Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'. + See https://cloud.google.com/compute/docs/instances/spot for more details. + EOD + type = string + default = null +} + +variable "automatic_restart" { + type = bool + description = "(Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user)." + default = true +} + +variable "on_host_maintenance" { + type = string + description = "Instance availability Policy" + default = "MIGRATE" +} + +variable "region" { + type = string + description = "Region where the instance template should be created." + nullable = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +####### +# disk +####### +variable "source_image" { + description = "Source disk image. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." + type = string + default = "" +} + +variable "source_image_family" { + description = "Source image family. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." + type = string + default = "centos-7" +} + +variable "source_image_project" { + description = "Project where the source image comes from. The default project contains CentOS images." + type = string + default = "centos-cloud" +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = string + default = "100" +} + +variable "disk_type" { + description = "Boot disk type, can be either pd-ssd, local-ssd, or pd-standard" + type = string + default = "pd-standard" +} + +variable "disk_labels" { + description = "Labels to be assigned to boot disk, provided as a map" + type = map(string) + default = {} +} + +variable "disk_encryption_key" { + description = "The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance" + type = string + default = null +} + +variable "auto_delete" { + description = "Whether or not the boot disk should be auto-deleted" + type = string + default = "true" +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + description = "List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name" + type = list(object({ + source = optional(string) + disk_name = optional(string) + device_name = string + auto_delete = bool + boot = bool + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = map(string) + disk_resource_manager_tags = map(string) + })) + default = [] +} + +#################### +# network_interface +#################### +variable "network" { + description = "The name or self_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks." + type = string + default = "" +} + +variable "nic_type" { + description = "The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO_NET." + type = string + default = null +} + +variable "subnetwork" { + description = "The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided." + type = string + default = "" +} + +variable "subnetwork_project" { + description = "The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used." + type = string + default = null +} + +variable "network_ip" { + description = "Private IP address to assign to the instance if desired." + type = string + default = "" +} + +variable "stack_type" { + description = "The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4_ONLY." + type = string + default = null +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + })) +} + +variable "total_egress_bandwidth_tier" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 6.41 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.41 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance](#module\_instance) | ../instance | n/a | +| [template](#module\_template) | ../instance_template | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | +| [login\_nodes](#input\_login\_nodes) | Slurm login instance definitions. | 
object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  })
| n/a | yes | +| [network\_storage](#input\_network\_storage) | Storage to mounted on login instances
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path to directory in the bucket for configs | `string` | n/a | yes | +| [slurm\_bucket\_name](#input\_slurm\_bucket\_name) | Name of the bucket for configs | `string` | n/a | yes | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name | `string` | n/a | yes | +| [startup\_scripts](#input\_startup\_scripts) | List of scripts to be ran on login VMs startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | +| [startup\_scripts\_timeout](#input\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script. If any script exceeds this timeout,
then the instance setup process is considered failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instances](#output\_instances) | VM instances of login nodes | +| [service\_account](#output\_service\_account) | Service Account used by login VMs | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf new file mode 100644 index 0000000000..605461f7e6 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf @@ -0,0 +1,112 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "template" { + source = "../instance_template" + + project_id = var.project_id + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = "login" + slurm_bucket_path = var.slurm_bucket_path + name_prefix = local.name + + additional_disks = var.login_nodes.additional_disks + bandwidth_tier = var.login_nodes.bandwidth_tier + can_ip_forward = var.login_nodes.can_ip_forward + advanced_machine_features = var.login_nodes.advanced_machine_features + disk_auto_delete = var.login_nodes.disk_auto_delete + disk_labels = var.login_nodes.disk_labels + disk_resource_manager_tags = var.login_nodes.disk_resource_manager_tags + disk_size_gb = var.login_nodes.disk_size_gb + disk_type = var.login_nodes.disk_type + enable_confidential_vm = var.login_nodes.enable_confidential_vm + enable_oslogin = var.login_nodes.enable_oslogin + enable_shielded_vm = var.login_nodes.enable_shielded_vm + gpu = var.login_nodes.gpu + labels = var.login_nodes.labels + machine_type = var.login_nodes.machine_type + metadata = merge(var.login_nodes.metadata, { + "universe_domain" = var.universe_domain, + "slurm_login_group" = local.name + }) + min_cpu_platform = var.login_nodes.min_cpu_platform + on_host_maintenance = var.login_nodes.on_host_maintenance + preemptible = var.login_nodes.preemptible + region = var.login_nodes.region + resource_manager_tags = var.login_nodes.resource_manager_tags + service_account = var.login_nodes.service_account + shielded_instance_config = var.login_nodes.shielded_instance_config + source_image_family = var.login_nodes.source_image_family + source_image_project = var.login_nodes.source_image_project + source_image = var.login_nodes.source_image + spot = var.login_nodes.spot + subnetwork = var.login_nodes.subnetwork + tags = concat([var.slurm_cluster_name], var.login_nodes.tags) + termination_action = var.login_nodes.termination_action + + internal_startup_script = var.internal_startup_script +} + +module "instance" { + source = "../instance" + + access_config = var.login_nodes.access_config + hostname = "${var.slurm_cluster_name}-${local.name}" + + project_id = var.project_id + + instance_template = module.template.self_link + num_instances = var.login_nodes.num_instances + + additional_networks = var.login_nodes.additional_networks + region = var.login_nodes.region + static_ips = var.login_nodes.static_ips + subnetwork = var.login_nodes.subnetwork + zone = var.login_nodes.zone + + replace_trigger = var.replace_trigger +} + +resource "google_storage_bucket_object" "startup_scripts" { + for_each = { + for s in var.startup_scripts : format( + "slurm-login-%s-script-%s", local.name, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_") + ) => s.content + } + + bucket = var.slurm_bucket_name + name = "${var.slurm_bucket_dir}/${each.key}" + content = each.value + source_md5hash = md5(each.value) +} + +locals { + name = var.login_nodes.group_name # short hand + + config = { + group_name = local.name + startup_scripts_timeout = var.startup_scripts_timeout + network_storage = var.network_storage + } +} + +resource "google_storage_bucket_object" "config" { + bucket = var.slurm_bucket_name + name = "${var.slurm_bucket_dir}/login_group_configs/${local.name}.yaml" + content = yamlencode(local.config) + source_md5hash = md5(yamlencode(local.config)) + + # To ensure that login group "is not ready" until all startup scripts are written down + depends_on = [google_storage_bucket_object.startup_scripts] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf new file mode 100644 index 0000000000..04de18a188 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "service_account" { + value = module.template.service_account + description = "Service Account used by login VMs" +} + +output "instances" { + value = module.instance.slurm_instances + description = "VM instances of login nodes" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf new file mode 100644 index 0000000000..3efd862942 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf @@ -0,0 +1,188 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "slurm_cluster_name" { + type = string + description = "Cluster name" +} + +variable "slurm_bucket_path" { + type = string + description = "GCS Bucket URI of Slurm cluster file storage." +} + + +variable "slurm_bucket_name" { + type = string + description = "Name of the bucket for configs" +} + +variable "slurm_bucket_dir" { + type = string + description = "Path to directory in the bucket for configs" +} + + +variable "universe_domain" { + description = "Domain address for alternate API universe" + type = string + default = "googleapis.com" +} + +variable "login_nodes" { + description = "Slurm login instance definitions." + type = object({ + group_name = string + access_config = optional(list(object({ + nat_ip = string + network_tier = string + }))) + additional_disks = optional(list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string), {}) + auto_delete = optional(bool, true) + boot = optional(bool, false) + disk_resource_manager_tags = optional(map(string), {}) + })), []) + additional_networks = optional(list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })), []) + bandwidth_tier = optional(string, "platform_default") + can_ip_forward = optional(bool, false) + disk_auto_delete = optional(bool, true) + disk_labels = optional(map(string), {}) + disk_resource_manager_tags = optional(map(string), {}) + disk_size_gb = optional(number) + disk_type = optional(string, "n1-standard-1") + enable_confidential_vm = optional(bool, false) + enable_oslogin = optional(bool, true) + enable_shielded_vm = optional(bool, false) + gpu = optional(object({ + count = number + type = string + })) + labels = optional(map(string), {}) + machine_type = optional(string) + advanced_machine_features = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + metadata = optional(map(string), {}) + min_cpu_platform = optional(string) + num_instances = optional(number, 1) + on_host_maintenance = optional(string) + preemptible = optional(bool, false) + region = optional(string) + resource_manager_tags = optional(map(string), {}) + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + shielded_instance_config = optional(object({ + enable_integrity_monitoring = optional(bool, true) + enable_secure_boot = optional(bool, true) + enable_vtpm = optional(bool, true) + })) + source_image_family = optional(string) + source_image_project = optional(string) + source_image = optional(string) + static_ips = optional(list(string), []) + subnetwork = string + spot = optional(bool, false) + tags = optional(list(string), []) + zone = optional(string) + termination_action = optional(string) + }) +} + + +variable "startup_scripts" { + description = "List of scripts to be ran on login VMs startup." + type = list(object({ + filename = string + content = string + })) + default = [] +} + +variable "startup_scripts_timeout" { + description = < + +- [Module: Slurm Nodeset (TPU)](#module-slurm-nodeset-tpu) + - [Overview](#overview) + - [Module API](#module-api) + + + +## Overview + +This is a submodule of [slurm_cluster](../../../slurm_cluster/README.md). It +creates a Slurm TPU nodeset for [slurm_partition](../slurm_partition/README.md). + +## Module API + +For the terraform module API reference, please see +[README_TF.md](./README_TF.md). + + +Copyright (C) SchedMD LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.2 | +| [google](#requirement\_google) | >= 3.53 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.53 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.nodeset_tpu](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_subnetwork.nodeset_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | +| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | +| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `""` | no | +| [enable\_public\_ip](#input\_enable\_public\_ip) | Enables IP address to access the Internet. | `bool` | `false` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of nodes allowed in this partition to be created dynamically. | `number` | `0` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | +| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. Not needed if you use accelerator\_config | `string` | `null` | no | +| [nodeset\_name](#input\_nodeset\_name) | Name of Slurm nodeset. | `string` | n/a | yes | +| [preemptible](#input\_preemptible) | Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details. | `bool` | `false` | no | +| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `true` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the TPU-vm.
If none is given, the default service account and scopes will be used. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | +| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | n/a | yes | +| [zone](#input\_zone) | Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Nodeset details. | +| [nodeset\_name](#output\_nodeset\_name) | Nodeset name. | +| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf new file mode 100644 index 0000000000..1a6a9cfba1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf @@ -0,0 +1,121 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########### +# NODESET # +########### + +locals { + node_conf_hw = { + Mem334CPU96 = { + CPUs = 96 + Boards = 1 + Sockets = 2 + CoresPerSocket = 24 + ThreadsPerCore = 2 + RealMemory = 307200 + } + Mem400CPU240 = { + CPUs = 240 + Boards = 1 + Sockets = 2 + CoresPerSocket = 60 + ThreadsPerCore = 2 + RealMemory = 400000 + } + } + node_conf_mappings = { + "v2" = local.node_conf_hw.Mem334CPU96 + "v3" = local.node_conf_hw.Mem334CPU96 + "v4" = local.node_conf_hw.Mem400CPU240 + } + simple_nodes = ["v2-8", "v3-8", "v4-8"] +} + +locals { + snetwork = data.google_compute_subnetwork.nodeset_subnetwork.name + region = join("-", slice(split("-", var.zone), 0, 2)) + tpu_fam = var.accelerator_config.version != "" ? lower(var.accelerator_config.version) : split("-", var.node_type)[0] + #If subnetwork is specified and it does not have private_ip_google_access, we need to have public IPs on the TPU + #if no subnetwork is specified, the default one will be used, this does not have private_ip_google_access so we need public IPs too + pub_need = !data.google_compute_subnetwork.nodeset_subnetwork.private_ip_google_access + can_preempt = var.node_type != null ? contains(local.simple_nodes, var.node_type) : false + nodeset_tpu = { + nodeset_name = var.nodeset_name + node_conf = local.node_conf_mappings[local.tpu_fam] + node_type = var.node_type + accelerator_config = var.accelerator_config + tf_version = var.tf_version + preemptible = local.can_preempt ? var.preemptible : false + reserved = var.reserved + node_count_dynamic_max = var.node_count_dynamic_max + node_count_static = var.node_count_static + enable_public_ip = var.enable_public_ip + zone = var.zone + service_account = var.service_account != null ? var.service_account : local.service_account + preserve_tpu = local.can_preempt ? var.preserve_tpu : false + data_disks = var.data_disks + docker_image = var.docker_image != "" ? var.docker_image : "us-docker.pkg.dev/schedmd-slurm-public/tpu/slurm-gcp-6-9:tf-${var.tf_version}" + subnetwork = local.snetwork + network_storage = var.network_storage + } + + service_account = { + email = try(var.service_account.email, null) + scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) + } +} + +data "google_compute_subnetwork" "nodeset_subnetwork" { + name = var.subnetwork + region = local.region + project = var.project_id + + self_link = ( + length(regexall("/projects/([^/]*)", var.subnetwork)) > 0 + && length(regexall("/regions/([^/]*)", var.subnetwork)) > 0 + ? var.subnetwork + : null + ) +} + +resource "null_resource" "nodeset_tpu" { + triggers = { + nodeset = sha256(jsonencode(local.nodeset_tpu)) + } + lifecycle { + precondition { + condition = sum([var.node_count_dynamic_max, var.node_count_static]) > 0 + error_message = "Sum of node_count_dynamic_max and node_count_static must be > 0." + } + precondition { + condition = !(var.preemptible && var.reserved) + error_message = "Nodeset cannot be preemptible and reserved at the same time." + } + precondition { + condition = !(var.subnetwork == null && !var.enable_public_ip) + error_message = "Using the default subnetwork for the TPU nodeset requires enable_public_ip set to true." + } + precondition { + condition = !(var.subnetwork != null && (local.pub_need && !var.enable_public_ip)) + error_message = "The subnetwork specified does not have Private Google Access enabled. This is required when enable_public_ip is set to false." + } + precondition { + condition = !(var.node_type == null && (var.accelerator_config.topology == "" && var.accelerator_config.version == "")) + error_message = "Either a node type or an accelerator_config must be provided." + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf new file mode 100644 index 0000000000..fce700d567 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "nodeset_name" { + description = "Nodeset name." + value = local.nodeset_tpu.nodeset_name +} + +output "nodeset" { + description = "Nodeset details." + value = local.nodeset_tpu +} + +output "service_account" { + description = "Service account object, includes email and scopes." + value = local.service_account +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf new file mode 100644 index 0000000000..a8c470dec9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf @@ -0,0 +1,158 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "nodeset_name" { + description = "Name of Slurm nodeset." + type = string + + validation { + condition = can(regex("^[a-z](?:[a-z0-9]{0,14})$", var.nodeset_name)) + error_message = "Variable 'nodeset_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,14})$'." + } +} + +variable "node_type" { + description = "Specify a node type to base the vm configuration upon it. Not needed if you use accelerator_config" + type = string + default = null +} + +variable "accelerator_config" { + description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." + type = object({ + topology = string + version = string + }) + default = { + topology = "" + version = "" + } + validation { + condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], upper(var.accelerator_config.version)) + error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" + } + validation { + condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) + error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." + } +} + +variable "docker_image" { + description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" + type = string + default = "" +} + +variable "tf_version" { + description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." + type = string +} + +variable "zone" { + description = "Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it." + type = string + + validation { + condition = can(coalesce(var.zone)) + error_message = "Zone cannot be null or empty." + } +} + +variable "preemptible" { + description = "Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details." + type = bool + default = false +} + +variable "reserved" { + description = "Specify whether TPU-vms in this nodeset are created under a reservation." + type = bool + default = false +} + +variable "preserve_tpu" { + description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" + type = bool + default = true +} + +variable "node_count_static" { + description = "Number of nodes to be statically created." + type = number + default = 0 + + validation { + condition = var.node_count_static >= 0 + error_message = "Value must be >= 0." + } +} + +variable "node_count_dynamic_max" { + description = "Maximum number of nodes allowed in this partition to be created dynamically." + type = number + default = 0 + + validation { + condition = var.node_count_dynamic_max >= 0 + error_message = "Value must be >= 0." + } +} + +variable "enable_public_ip" { + description = "Enables IP address to access the Internet." + type = bool + default = false +} + +variable "data_disks" { + type = list(string) + description = "The data disks to include in the TPU node" + default = [] +} + +variable "subnetwork" { + description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." + type = string +} + +variable "service_account" { + type = object({ + email = string + scopes = set(string) + }) + description = < +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | +| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | +| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | +| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | +| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | +| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "v0.7.2")
  })
| `{}` | no | +| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "v0.11.4")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | +| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md new file mode 100644 index 0000000000..1957899617 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md @@ -0,0 +1,64 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [helm](#provider\_helm) | ~> 2.17 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | +| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | +| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | +| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | +| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | +| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | +| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | +| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | +| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | +| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | +| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | +| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | +| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | +| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | +| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | +| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | +| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | +| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | +| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | +| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | +| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | +| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | +| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | +| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | +| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | +| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | +| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | +| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | +| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | +| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf new file mode 100644 index 0000000000..bd2383b772 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf @@ -0,0 +1,75 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "helm_release" "apply_chart" { + # Required Identification + name = var.release_name + chart = var.chart_name + + # Chart Source & Version + repository = var.chart_repository + version = var.chart_version + devel = var.devel + + # Target Namespace + namespace = var.namespace + create_namespace = var.create_namespace + + # Values Configuration + values = var.values_yaml + + dynamic "set" { + for_each = var.set_values + content { + name = set.value.name + value = set.value.value + type = set.value.type + } + } + + # Installation/Upgrade Behavior + description = var.description + atomic = var.atomic + cleanup_on_fail = var.cleanup_on_fail + dependency_update = var.dependency_update + disable_crd_hooks = var.disable_crd_hooks + disable_openapi_validation = var.disable_openapi_validation + disable_webhooks = var.disable_webhooks + force_update = var.force_update + lint = var.lint + max_history = var.max_history + recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI + render_subchart_notes = var.render_subchart_notes + reset_values = var.reset_values + reuse_values = var.reuse_values + skip_crds = var.skip_crds + timeout = var.timeout + wait = var.wait + wait_for_jobs = var.wait_for_jobs + + # Verification & Credentials + keyring = var.keyring + pass_credentials = var.pass_credentials + verify = var.verify + + # Post Rendering + dynamic "postrender" { + # Only include the block if var.postrender is not null + for_each = var.postrender == null ? [] : [var.postrender] + content { + binary_path = postrender.value.binary_path + } + } + +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml new file mode 100644 index 0000000000..e18197e2b7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf new file mode 100644 index 0000000000..04e8e214fc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf @@ -0,0 +1,212 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +# --- Required --- +variable "release_name" { + description = "Name of the Helm release." + type = string +} + +variable "chart_name" { + description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." + type = string +} + +# --- Chart Location & Version --- +variable "chart_repository" { + description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." + type = string + default = null +} + +variable "chart_version" { + description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." + type = string + default = null +} + +variable "devel" { + description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." + type = bool + default = false +} + +# --- Namespace --- +variable "namespace" { + description = "Kubernetes namespace to install the Helm release into." + type = string + default = "default" +} + +variable "create_namespace" { + description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." + type = bool + default = true # Common convenience setting +} + +# --- Values Customization --- +variable "values_yaml" { + description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." + type = list(string) + default = [] +} + +variable "set_values" { + description = "List of objects defining values to set ('helm install --set')." + type = list(object({ + name = string # Path to the value (e.g., 'service.type', 'replicaCount') + value = string # The value to set + type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') + })) + default = [] +} + +# --- Installation/Upgrade Behavior --- +variable "description" { + description = "Set an optional description for the Helm release." + type = string + default = null +} + +variable "atomic" { + description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." + type = bool + default = false +} + +variable "wait" { + description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." + type = bool + default = true # Often a good default for dependencies +} + +variable "wait_for_jobs" { + description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." + type = bool + default = false # Helm CLI default is false +} + +variable "timeout" { + description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." + type = number + default = 300 # 5 minutes (Helm CLI default) +} + +variable "cleanup_on_fail" { + description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." + type = bool + default = false +} + +variable "dependency_update" { + description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." + type = bool + default = false +} + +variable "disable_crd_hooks" { + description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." + type = bool + default = false +} + +variable "disable_openapi_validation" { + description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." + type = bool + default = false +} + +variable "disable_webhooks" { + description = "Prevent hooks from running ('helm install --no-hooks')." + type = bool + default = false +} + +variable "force_update" { + description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." + type = bool + default = false +} + +variable "lint" { + description = "Run the helm chart linter during the plan ('helm lint')." + type = bool + default = false +} + +variable "max_history" { + description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." + type = number + default = null # Terraform provider defaults to Helm's default (usually 10) +} + +variable "recreate_pods" { + description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." + type = bool + default = false +} + +variable "render_subchart_notes" { + description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." + type = bool + default = false +} + +variable "reset_values" { + description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." + type = bool + default = false +} + +variable "reuse_values" { + description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." + type = bool + default = false # Helm CLI default is false +} + +variable "skip_crds" { + description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." + type = bool + default = false +} + +# --- Verification & Credentials --- +variable "keyring" { + description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." + type = string + default = null # Defaults to Helm's default keyring location +} + +variable "pass_credentials" { + description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." + type = bool + default = false +} + +variable "verify" { + description = "Verify the package before installing it ('helm install --verify')." + type = bool + default = false +} + +# --- Advanced Rendering --- +variable "postrender" { + description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." + type = object({ + binary_path = string # Path to the post-renderer executable + }) + default = null # Disabled by default +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf new file mode 100644 index 0000000000..09d912e2c9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md new file mode 100644 index 0000000000..46bfe51a32 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md @@ -0,0 +1,40 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.23 | + +## Providers + +| Name | Version | +|------|---------| +| [kubernetes](#provider\_kubernetes) | ~> 2.23 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubernetes_manifest.apply_manifests](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | +| [field\_manager](#input\_field\_manager) | (Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts. | 
object({
    name            = optional(string, null)
    force_conflicts = optional(bool, false)
  })
| `null` | no | +| [resource\_timeouts](#input\_resource\_timeouts) | (Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met. | 
object({
    create = optional(string, null)
    update = optional(string, null)
    delete = optional(string, null)
  })
| 
{
  "create": "15m",
  "delete": "5m",
  "update": "10m"
}
| no | +| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `""` | no | +| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | +| [wait\_for\_fields](#input\_wait\_for\_fields) | (Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern. | `map(string)` | `{}` | no | +| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf new file mode 100644 index 0000000000..f97f26038d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf @@ -0,0 +1,104 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + yaml_separator = "\n---" + + # --- 1. Determine the primary source of YAML content --- + # Prioritize 'content' variable if provided + primary_content_body = var.content != "" ? var.content : null + + # --- 2. Handle 'source_path' based on its type (File vs. Directory) --- + + # Check if source_path is a directory (indicated by trailing slash) + is_directory = endswith(var.source_path, "/") + directory_absolute_path = local.is_directory ? abspath(var.source_path) : null + + # Check if source_path is a single yaml or tftpl file (only if not a directory) + is_single_file = !local.is_directory && ( + length(regexall("\\.yaml$", lower(var.source_path))) > 0 || + length(regexall("\\.tftpl$", lower(var.source_path))) > 0 + ) + single_file_raw_content = local.is_single_file ? ( + length(regexall("\\.tftpl$", lower(var.source_path))) > 0 ? + templatefile(abspath(var.source_path), var.template_vars) : + file(abspath(var.source_path)) + ) : null + + # Docs from primary_content_body + docs_from_primary_source = [ + for doc in split(local.yaml_separator, coalesce(local.primary_content_body, local.single_file_raw_content, "")) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + + # Docs from .yaml files in a directory + directory_yaml_files = local.is_directory ? fileset(local.directory_absolute_path, "*.yaml") : [] + docs_from_directory_yamls = flatten([ + for file_name in local.directory_yaml_files : + [ + for doc in split(local.yaml_separator, file(format("%s/%s", local.directory_absolute_path, file_name))) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + ]) + + # Docs from .tftpl files in a directory + directory_template_files = local.is_directory ? fileset(local.directory_absolute_path, "*.tftpl") : [] + docs_from_directory_templates = flatten([ + for file_name in local.directory_template_files : + [ + for doc in split(local.yaml_separator, templatefile(format("%s/%s", local.directory_absolute_path, file_name), var.template_vars)) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + ]) + + all_parsed_docs = concat( + local.docs_from_primary_source, + local.docs_from_directory_yamls, + local.docs_from_directory_templates + ) + + # --- 5. Create the final map for `for_each` (keys must be unique strings) --- + docs_map = tomap({ + for index, doc in local.all_parsed_docs : index => doc + if length(trimspace(doc)) > 0 + }) +} + +# Apply all manifest files dynamically +resource "kubernetes_manifest" "apply_manifests" { + for_each = local.docs_map + manifest = yamldecode(each.value) + timeouts { + create = var.resource_timeouts.create + update = var.resource_timeouts.update + delete = var.resource_timeouts.delete + } + + dynamic "wait" { + for_each = var.wait_for_rollout ? [1] : [] + content { + rollout = var.wait_for_rollout + fields = var.wait_for_fields + } + } + + # Configure the 'field_manager' block dynamically + dynamic "field_manager" { + for_each = var.field_manager != null ? [var.field_manager] : [] + content { + name = field_manager.value.name + force_conflicts = field_manager.value.force_conflicts + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml new file mode 100644 index 0000000000..e18197e2b7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf new file mode 100644 index 0000000000..0b846189ea --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf @@ -0,0 +1,69 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +variable "content" { + description = "The YAML body to apply to gke cluster." + type = string + default = null +} + +variable "source_path" { + description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." + type = string + default = "" +} + +variable "template_vars" { + description = "The values to populate template file(s) with." + type = any + default = null +} + +variable "wait_for_rollout" { + description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." + type = bool + default = true +} + + +variable "wait_for_fields" { + description = "(Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern." + type = map(string) + default = {} +} + +variable "resource_timeouts" { + description = "(Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met." + type = object({ + create = optional(string, null) + update = optional(string, null) + delete = optional(string, null) + }) + default = { + create = "15m" # Default create timeout, also covers waiting for initial conditions + update = "10m" # Default update timeout, also covers waiting for update conditions + delete = "5m" # Default delete timeout + } +} + +variable "field_manager" { + description = "(Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts." + type = object({ + name = optional(string, null) + force_conflicts = optional(bool, false) + }) + default = null +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf new file mode 100644 index 0000000000..61786b06de --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + # Defines the providers that this module depends on and their versions. + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.23" + } + } + required_version = ">= 1.3" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/main.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/main.tf new file mode 100644 index 0000000000..8db4870452 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/main.tf @@ -0,0 +1,183 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] + project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] + + install_gpu_operator = try(var.gpu_operator.install, false) + install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) +} + +data "google_container_cluster" "gke_cluster" { + project = local.project_id + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +module "install_kueue" { + source = "./helm_install" + depends_on = [var.gke_cluster_exists] + + release_name = "kueue" + + chart_name = "oci://registry.k8s.io/kueue/charts/kueue" + chart_version = var.kueue.version # Specify your desired Kueue version + + create_namespace = true # Helm can also create the namespace + wait = true + timeout = 600 # seconds +} + +module "install_jobset" { + source = "./helm_install" + depends_on = [var.gke_cluster_exists, module.install_kueue] + release_name = "jobset-controller" # The release name for your JobSet installation + chart_name = "oci://registry.k8s.io/jobset/charts/jobset" # The Helm repository URL for nvidia charts + chart_version = var.jobset.version + create_namespace = true + namespace = "jobset-system" +} + +module "install_nvidia_dra_driver" { + count = local.install_nvidia_dra_driver ? 1 : 0 + depends_on = [var.gke_cluster_exists] + source = "./helm_install" + + release_name = "nvidia-dra-driver-gpu" # The release name + chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts + chart_name = "nvidia-dra-driver-gpu" # The chart name + chart_version = var.nvidia_dra_driver.version # The chart version + namespace = "nvidia-dra-driver-gpu" # The target namespace + create_namespace = true # Equivalent to --create-namespace + + # Use the 'values' argument to pass the YAML content + # This corresponds to the -f <(cat < +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.2 | +| [google](#requirement\_google) | >= 6.40 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.40 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_global_address.private_ip_alloc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_global_address) | resource | +| [google_compute_network_peering_routes_config.private_vpc_peering_routes_gcnv](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering_routes_config) | resource | +| [google_service_networking_connection.private_vpc_connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_networking_connection) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [address](#input\_address) | The IP address or beginning of the address range allocated for the Private Service Access. | `string` | `null` | no | +| [deletion\_policy](#input\_deletion\_policy) | The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to supporting resources. Key-value pairs. | `map(string)` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to configure Private Service Access:
`projects//global/networks/`" | `string` | n/a | yes | +| [prefix\_length](#input\_prefix\_length) | The prefix length of the IP range allocated for the Private Service Access. | `number` | `16` | no | +| [project\_id](#input\_project\_id) | ID of project in which Private Service Access will be created. | `string` | n/a | yes | +| [service\_name](#input\_service\_name) | The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'. | `string` | `"servicenetworking.googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cidr\_range](#output\_cidr\_range) | CIDR range of the created google\_compute\_global\_address | +| [connect\_mode](#output\_connect\_mode) | Services that use Private Service Access typically specify connect\_mode
"PRIVATE\_SERVICE\_ACCESS". This output value sets connect\_mode and additionally
blocks terraform actions until the VPC connection has been created. | +| [private\_vpc\_connection\_peering](#output\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection that was created by the service provider. | +| [reserved\_ip\_range](#output\_reserved\_ip\_range) | Named IP range to be used by services connected with Private Service Access. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/main.tf b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/main.tf new file mode 100644 index 0000000000..429e4d93f0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/main.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "private-service-access", ghpc_role = "network" }) +} + +locals { + split_network_id = split("/", var.network_id) + network_name = local.split_network_id[4] + network_project = local.split_network_id[1] +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_compute_global_address" "private_ip_alloc" { + provider = google + name = "global-psconnect-ip-${random_id.resource_name_suffix.hex}" + project = var.project_id + purpose = "VPC_PEERING" + address_type = "INTERNAL" + network = var.network_id + prefix_length = var.prefix_length + labels = local.labels + address = var.address +} + +resource "google_service_networking_connection" "private_vpc_connection" { + network = var.network_id + service = var.service_name + reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name] + deletion_policy = var.deletion_policy + update_on_creation_fail = var.deletion_policy == "ABANDON" ? true : null +} + +# Google Cloud NetApp Volumes need enablement of custom_route import and export +resource "google_compute_network_peering_routes_config" "private_vpc_peering_routes_gcnv" { + count = var.service_name == "netapp.servicenetworking.goog" ? 1 : 0 + project = local.network_project + network = local.network_name + peering = google_service_networking_connection.private_vpc_connection.peering + + export_custom_routes = true + import_custom_routes = true +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/metadata.yaml new file mode 100644 index 0000000000..93e8b3970e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - servicenetworking.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/outputs.tf new file mode 100644 index 0000000000..296f2e9140 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/outputs.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "private_vpc_connection_peering" { + description = "The name of the VPC Network peering connection that was created by the service provider." + sensitive = true + value = google_service_networking_connection.private_vpc_connection.peering +} + +output "connect_mode" { + description = <<-EOT + Services that use Private Service Access typically specify connect_mode + "PRIVATE_SERVICE_ACCESS". This output value sets connect_mode and additionally + blocks terraform actions until the VPC connection has been created. + EOT + value = "PRIVATE_SERVICE_ACCESS" + depends_on = [ + google_service_networking_connection.private_vpc_connection, + ] +} + +output "reserved_ip_range" { + description = "Named IP range to be used by services connected with Private Service Access." + value = google_compute_global_address.private_ip_alloc.name +} + +output "cidr_range" { + description = "CIDR range of the created google_compute_global_address" + value = "${google_compute_global_address.private_ip_alloc.address}/${google_compute_global_address.private_ip_alloc.prefix_length}" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/variables.tf new file mode 100644 index 0000000000..4b0a3e796f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/variables.tf @@ -0,0 +1,59 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "address" { + description = "The IP address or beginning of the address range allocated for the Private Service Access." + type = string + default = null +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to configure Private Service Access: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "labels" { + description = "Labels to add to supporting resources. Key-value pairs." + type = map(string) +} + +variable "prefix_length" { + description = "The prefix length of the IP range allocated for the Private Service Access." + type = number + default = 16 +} + +variable "project_id" { + description = "ID of project in which Private Service Access will be created." + type = string +} + +variable "service_name" { + description = "The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'." + type = string + default = "servicenetworking.googleapis.com" +} + +variable "deletion_policy" { + description = "The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON." + type = string + default = null +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/versions.tf new file mode 100644 index 0000000000..df2914cdb9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.40" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" + } + + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" + } + + required_version = ">= 1.2" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/new-project/README.md b/deletion-test/build_script/modules/embedded/community/modules/project/new-project/README.md new file mode 100644 index 0000000000..5e5cabe9d5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/new-project/README.md @@ -0,0 +1,128 @@ +## Description + +This module allows you to create opinionated Google Cloud Platform projects. It +creates projects and configures aspects like Shared VPC connectivity, IAM +access, Service Accounts, and API enablement to follow best practices. + +This module is meant for use with Terraform 0.13. + +**Note:** This module has been removed from the Cluster Toolkit. The upstream module (`terraform-google-project-factory`) is now the recommended way to create and manage GCP projects. + +### Example + +```yaml +- id: project + source: github.com/terraform-google-modules/terraform-google-project-factory?rev=v17.0.0&depth=1 +``` + +This creates a new project with pre-defined project ID, a designated folder and +organization and associated billing account which will be used to pay for +services consumed. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [project\_factory](#module\_project\_factory) | terraform-google-modules/project-factory/google | ~> 11.3 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [activate\_api\_identities](#input\_activate\_api\_identities) | The list of service identities (Google Managed service account for the API) to force-create for the project (e.g. in order to grant additional roles).
APIs in this list will automatically be appended to `activate_apis`.
Not including the API in this list will follow the default behaviour for identity creation (which is usually when the first resource using the API is created).
Any roles (e.g. service agent role) must be explicitly listed. See https://cloud.google.com/iam/docs/understanding-roles#service-agent-roles-roles for a list of related roles. | 
list(object({
    api   = string
    roles = list(string)
  }))
| `[]` | no | +| [activate\_apis](#input\_activate\_apis) | The list of apis to activate within the project | `list(string)` | 
[
  "compute.googleapis.com",
  "serviceusage.googleapis.com",
  "storage.googleapis.com"
]
| no | +| [auto\_create\_network](#input\_auto\_create\_network) | Create the default network | `bool` | `false` | no | +| [billing\_account](#input\_billing\_account) | The ID of the billing account to associate this project with | `string` | n/a | yes | +| [bucket\_force\_destroy](#input\_bucket\_force\_destroy) | Force the deletion of all objects within the GCS bucket when deleting the bucket (optional) | `bool` | `false` | no | +| [bucket\_labels](#input\_bucket\_labels) | A map of key/value label pairs to assign to the bucket (optional) | `map(string)` | `{}` | no | +| [bucket\_location](#input\_bucket\_location) | The location for a GCS bucket to create (optional) | `string` | `"US"` | no | +| [bucket\_name](#input\_bucket\_name) | A name for a GCS bucket to create (in the bucket\_project project), useful for Terraform state (optional) | `string` | `""` | no | +| [bucket\_project](#input\_bucket\_project) | A project to create a GCS bucket (bucket\_name) in, useful for Terraform state (optional) | `string` | `""` | no | +| [bucket\_ula](#input\_bucket\_ula) | Enable Uniform Bucket Level Access | `bool` | `true` | no | +| [bucket\_versioning](#input\_bucket\_versioning) | Enable versioning for a GCS bucket to create (optional) | `bool` | `false` | no | +| [budget\_alert\_pubsub\_topic](#input\_budget\_alert\_pubsub\_topic) | The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}` | `string` | `null` | no | +| [budget\_alert\_spent\_percents](#input\_budget\_alert\_spent\_percents) | A list of percentages of the budget to alert on when threshold is exceeded | `list(number)` | 
[
  0.5,
  0.7,
  1
]
| no | +| [budget\_amount](#input\_budget\_amount) | The amount to use for a budget alert | `number` | `null` | no | +| [budget\_display\_name](#input\_budget\_display\_name) | The display name of the budget. If not set defaults to `Budget For ` | `string` | `null` | no | +| [budget\_monitoring\_notification\_channels](#input\_budget\_monitoring\_notification\_channels) | A list of monitoring notification channels in the form `[projects/{project_id}/notificationChannels/{channel_id}]`. A maximum of 5 channels are allowed. | `list(string)` | `[]` | no | +| [consumer\_quotas](#input\_consumer\_quotas) | The quotas configuration you want to override for the project. | 
list(object({
    service = string,
    metric  = string,
    limit   = string,
    value   = string,
  }))
| `[]` | no | +| [create\_project\_sa](#input\_create\_project\_sa) | Whether the default service account for the project shall be created | `bool` | `true` | no | +| [default\_network\_tier](#input\_default\_network\_tier) | Default Network Service Tier for resources created in this project. If unset, the value will not be modified. See https://cloud.google.com/network-tiers/docs/using-network-service-tiers and https://cloud.google.com/network-tiers. | `string` | `""` | no | +| [default\_service\_account](#input\_default\_service\_account) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | `string` | `"keep"` | no | +| [disable\_dependent\_services](#input\_disable\_dependent\_services) | Whether services that are enabled and which depend on this service should also be disabled when this service is destroyed. | `bool` | `true` | no | +| [disable\_services\_on\_destroy](#input\_disable\_services\_on\_destroy) | Whether project services will be disabled when the resources are destroyed | `bool` | `true` | no | +| [domain](#input\_domain) | The domain name (optional). | `string` | `""` | no | +| [enable\_shared\_vpc\_host\_project](#input\_enable\_shared\_vpc\_host\_project) | If this project is a shared VPC host project. If true, you must *not* set svpc\_host\_project\_id variable. Default is false. | `bool` | `false` | no | +| [folder\_id](#input\_folder\_id) | The ID of a folder to host this project | `string` | `""` | no | +| [grant\_services\_network\_role](#input\_grant\_services\_network\_role) | Whether or not to grant service agents the network roles on the host project | `bool` | `true` | no | +| [grant\_services\_security\_admin\_role](#input\_grant\_services\_security\_admin\_role) | Whether or not to grant Kubernetes Engine Service Agent the Security Admin role on the host project so it can manage firewall rules | `bool` | `false` | no | +| [group\_name](#input\_group\_name) | A group to control the project by being assigned group\_role (defaults to project editor) | `string` | `""` | no | +| [group\_role](#input\_group\_role) | The role to give the controlling group (group\_name) over the project (defaults to project editor) | `string` | `"roles/editor"` | no | +| [labels](#input\_labels) | Map of labels for project | `map(string)` | `{}` | no | +| [lien](#input\_lien) | Add a lien on the project to prevent accidental deletion | `bool` | `false` | no | +| [name](#input\_name) | The name for the project | `string` | `null` | no | +| [org\_id](#input\_org\_id) | The organization ID. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The ID to give the project. If not provided, the `name` will be used. | `string` | `""` | no | +| [project\_sa\_name](#input\_project\_sa\_name) | Default service account name for the project. | `string` | `"project-service-account"` | no | +| [random\_project\_id](#input\_random\_project\_id) | Adds a suffix of 4 random characters to the `project_id` | `bool` | `false` | no | +| [sa\_role](#input\_sa\_role) | A role to give the default Service Account for the project (defaults to none) | `string` | `""` | no | +| [shared\_vpc\_subnets](#input\_shared\_vpc\_subnets) | List of subnets fully qualified subnet IDs (ie. projects/$project\_id/regions/$region/subnetworks/$subnet\_id) | `list(string)` | `[]` | no | +| [svpc\_host\_project\_id](#input\_svpc\_host\_project\_id) | The ID of the host project which hosts the shared VPC | `string` | `""` | no | +| [usage\_bucket\_name](#input\_usage\_bucket\_name) | Name of a GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | +| [usage\_bucket\_prefix](#input\_usage\_bucket\_prefix) | Prefix in the GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | +| [vpc\_service\_control\_attach\_enabled](#input\_vpc\_service\_control\_attach\_enabled) | Whether the project will be attached to a VPC Service Control Perimeter | `bool` | `false` | no | +| [vpc\_service\_control\_perimeter\_name](#input\_vpc\_service\_control\_perimeter\_name) | The name of a VPC Service Control Perimeter to add the created project to | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [api\_s\_account](#output\_api\_s\_account) | API service account email | +| [api\_s\_account\_fmt](#output\_api\_s\_account\_fmt) | API service account email formatted for terraform use | +| [budget\_name](#output\_budget\_name) | The name of the budget if created | +| [domain](#output\_domain) | The organization's domain | +| [enabled\_api\_identities](#output\_enabled\_api\_identities) | Enabled API identities in the project | +| [enabled\_apis](#output\_enabled\_apis) | Enabled APIs in the project | +| [group\_email](#output\_group\_email) | The email of the G Suite group with group\_name | +| [project\_bucket\_self\_link](#output\_project\_bucket\_self\_link) | Project's bucket selfLink | +| [project\_bucket\_url](#output\_project\_bucket\_url) | Project's bucket url | +| [project\_id](#output\_project\_id) | ID of the project that was created | +| [project\_name](#output\_project\_name) | Name of the project that was created | +| [project\_number](#output\_project\_number) | Number of the project that was created | +| [service\_account\_display\_name](#output\_service\_account\_display\_name) | The display name of the default service account | +| [service\_account\_email](#output\_service\_account\_email) | The email of the default service account | +| [service\_account\_id](#output\_service\_account\_id) | The id of the default service account | +| [service\_account\_name](#output\_service\_account\_name) | The fully-qualified name of the default service account | +| [service\_account\_unique\_id](#output\_service\_account\_unique\_id) | The unique id of the default service account | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/README.md b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/README.md new file mode 100644 index 0000000000..0f5c10c7e4 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/README.md @@ -0,0 +1,111 @@ +## Description + +Allows creation of service accounts for a Google Cloud Platform project. + +### Example + +```yaml +- id: service_acct + source: community/modules/project/service-account + settings: + project_id: $(vars.project_id) + name: instance_acct + project_roles: + - logging.logWriter + - monitoring.metricWriter + - storage.objectViewer +``` + +This creates a service account in GCP project "project_id" with the name +"instance_acct". It will have the 3 roles listed for all resources within the +project. + +### Usage with startup-script module + +When this module is used in conjunction with the [startup-script] module, the +service account must be granted (at least) read access to the bucket. This can +be achieved by granting project-wide access as shown above or by specifying the +service account as a bucket viewer in the startup-script module: + +```yaml +- id: service_acct + source: community/modules/project/service-account + settings: + project_id: $(vars.project_id) + name: instance_acct + project_roles: + - logging.logWriter + - monitoring.metricWriter +- id: script + source: modules/scripts/startup-script + settings: + bucket_viewers: + - $(service_acct.service_account_iam_email) +``` + +[startup-script]: ../../../../modules/scripts/startup-script/README.md + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [service\_account](#module\_service\_account) | terraform-google-modules/service-accounts/google | ~> 4.2 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [billing\_account\_id](#input\_billing\_account\_id) | If assigning billing role, specify a billing account (default is to assign at the organizational level). | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment (will be prepended to service account name) | `string` | n/a | yes | +| [description](#input\_description) | Description of the created service account. | `string` | `"Service Account"` | no | +| [descriptions](#input\_descriptions) | Deprecated; create single service accounts using var.description. | `list(string)` | `null` | no | +| [display\_name](#input\_display\_name) | Display name of the created service account. | `string` | `"Service Account"` | no | +| [generate\_keys](#input\_generate\_keys) | Generate keys for service account. | `bool` | `false` | no | +| [grant\_billing\_role](#input\_grant\_billing\_role) | Grant billing user role. | `bool` | `false` | no | +| [grant\_xpn\_roles](#input\_grant\_xpn\_roles) | Grant roles for shared VPC management. | `bool` | `true` | no | +| [name](#input\_name) | Name of the service account to create. | `string` | n/a | yes | +| [names](#input\_names) | Deprecated; create single service accounts using var.name. | `list(string)` | `null` | no | +| [org\_id](#input\_org\_id) | Id of the organization for org-level roles. | `string` | `""` | no | +| [prefix](#input\_prefix) | Deprecated; prefix now set using var.deployment\_name | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | +| [project\_roles](#input\_project\_roles) | List of roles to grant to service account (e.g. "storage.objectViewer" or "compute.instanceAdmin.v1" | `list(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [key](#output\_key) | Service account key (if creation was requested) | +| [service\_account\_email](#output\_service\_account\_email) | Service account e-mail address | +| [service\_account\_iam\_email](#output\_service\_account\_iam\_email) | Service account IAM binding format (serviceAccount:name@example.com) | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/main.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/main.tf new file mode 100644 index 0000000000..e8a69be642 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/main.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + display_name = "${var.display_name} (${var.deployment_name})" + description = "${var.description} (${var.deployment_name})" +} + +module "service_account" { + source = "terraform-google-modules/service-accounts/google" + version = "~> 4.2" + + billing_account_id = var.billing_account_id + description = local.description + display_name = local.display_name + generate_keys = var.generate_keys + grant_billing_role = var.grant_billing_role + grant_xpn_roles = var.grant_xpn_roles + names = [var.name] + org_id = var.org_id + prefix = var.deployment_name + project_id = var.project_id + project_roles = [for role in var.project_roles : "${var.project_id}=>roles/${role}"] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/metadata.yaml new file mode 100644 index 0000000000..c4dcdffdf4 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/outputs.tf new file mode 100644 index 0000000000..f9c9be05c8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/outputs.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "key" { + description = "Service account key (if creation was requested)" + value = module.service_account.key +} + +output "service_account_email" { + description = "Service account e-mail address" + value = module.service_account.email + depends_on = [ + module.service_account, + ] +} + +output "service_account_iam_email" { + description = "Service account IAM binding format (serviceAccount:name@example.com)" + value = module.service_account.iam_email + depends_on = [ + module.service_account, + ] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/variables.tf new file mode 100644 index 0000000000..53267f47e7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/variables.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "billing_account_id" { + description = "If assigning billing role, specify a billing account (default is to assign at the organizational level)." + type = string + default = "" +} + +variable "deployment_name" { + description = "Name of the deployment (will be prepended to service account name)" + type = string +} + +variable "description" { + description = "Description of the created service account." + type = string + default = "Service Account" +} + +# tflint-ignore: terraform_unused_declarations +variable "descriptions" { + description = "Deprecated; create single service accounts using var.description." + type = list(string) + default = null + + validation { + condition = var.descriptions == null + error_message = "var.descriptions has been deprecated in favor of creating single accounts with var.description" + } +} + +variable "display_name" { + description = "Display name of the created service account." + type = string + default = "Service Account" +} + +variable "generate_keys" { + description = "Generate keys for service account." + type = bool + default = false +} + +variable "grant_billing_role" { + description = "Grant billing user role." + type = bool + default = false +} + +variable "grant_xpn_roles" { + description = "Grant roles for shared VPC management." + type = bool + default = true +} + +variable "name" { + description = "Name of the service account to create." + type = string +} + +# tflint-ignore: terraform_unused_declarations +variable "names" { + description = "Deprecated; create single service accounts using var.name." + type = list(string) + default = null + + validation { + condition = var.names == null + error_message = "var.names has been deprecated in favor of creating single accounts with var.name" + } +} + +variable "org_id" { + description = "Id of the organization for org-level roles." + type = string + default = "" +} + +# tflint-ignore: terraform_unused_declarations +variable "prefix" { + description = "Deprecated; prefix now set using var.deployment_name" + type = string + default = null + + validation { + condition = var.prefix == null + error_message = "var.prefix has been deprecated in favor of setting prefix with var.deployment_name" + } +} + +variable "project_id" { + description = "ID of the project" + type = string +} + +variable "project_roles" { + description = "List of roles to grant to service account (e.g. \"storage.objectViewer\" or \"compute.instanceAdmin.v1\"" + type = list(string) +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/versions.tf new file mode 100644 index 0000000000..38e6e71945 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/versions.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/README.md b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/README.md new file mode 100644 index 0000000000..266eac26ec --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/README.md @@ -0,0 +1,70 @@ +## Description + +Allows management of multiple API services for a Google Cloud Platform project. + +### Example + +```yaml +- id: services-api + source: community/modules/project/service-enablement + settings: + gcp_service_list: [ + "file.googleapis.com", + "compute.googleapis.com" + ] +``` + +This allows the project to enable both the filestore API as well as the compute API. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_project_service.gcp_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [disable\_on\_destroy](#input\_disable\_on\_destroy) | Disable services on destroy if they were enabled (or already enabled) during apply (default: false) | `bool` | `false` | no | +| [gcp\_service\_list](#input\_gcp\_service\_list) | list of APIs to be enabled for the project | `list(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/main.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/main.tf new file mode 100644 index 0000000000..965e93c549 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/main.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +resource "google_project_service" "gcp_services" { + count = length(var.gcp_service_list) + project = var.project_id + service = var.gcp_service_list[count.index] + timeouts { + create = "30m" + update = "40m" + } + + disable_dependent_services = true + disable_on_destroy = var.disable_on_destroy +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/metadata.yaml new file mode 100644 index 0000000000..c594c8f819 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - serviceusage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/variables.tf new file mode 100644 index 0000000000..08f13999fe --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "ID of the project" + type = string +} + +variable "gcp_service_list" { + description = "list of APIs to be enabled for the project" + type = list(string) +} + +variable "disable_on_destroy" { + description = "Disable services on destroy if they were enabled (or already enabled) during apply (default: false)" + type = bool + default = false +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/versions.tf new file mode 100644 index 0000000000..07f25fb045 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/README.md b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/README.md new file mode 100644 index 0000000000..052e6aee23 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/README.md @@ -0,0 +1,87 @@ +# Description + +This module creates a Bigquery Pub/Sub Subscription. + +Primarily used for FSI - MonteCarlo Tutorial: +**[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Example + +The following example creates a Bigquery subscription using a Bigquery table and +Pub/Sub topic. + +```yaml + - id: bq_subscription + source: community/modules/pubsub/bigquery-sub + use: [bq-table, pubsub_topic] +``` + +Also see usages in this +[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_project_iam_member.editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_project_iam_member.viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_pubsub_subscription.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Name of the dataset that was created. Can be provided by the bigquery-table module | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [subscription\_id](#input\_subscription\_id) | The name of the pubsub subscription to be created | `string` | `null` | no | +| [table\_id](#input\_table\_id) | ID of created BQ table. Can be provided by the bigquery-table module | `string` | n/a | yes | +| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [subscription\_id](#output\_subscription\_id) | Name of the subscription that was created. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf new file mode 100644 index 0000000000..8edbc6b24e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf @@ -0,0 +1,57 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-sub", ghpc_role = "pubsub" }) +} + +locals { + subscription_id = var.subscription_id != null ? var.subscription_id : "${var.deployment_name}_subscription_${random_id.resource_name_suffix.hex}" +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_project_iam_member" "viewer" { + project = data.google_project.project.project_id + role = "roles/bigquery.metadataViewer" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +resource "google_project_iam_member" "editor" { + project = data.google_project.project.project_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +resource "google_pubsub_subscription" "example" { + depends_on = [google_project_iam_member.editor, google_project_iam_member.viewer] + name = local.subscription_id + topic = var.topic_id + project = var.project_id + labels = local.labels + bigquery_config { + table = "${var.project_id}.${var.dataset_id}.${var.table_id}" + use_topic_schema = true + write_metadata = true + } + +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml new file mode 100644 index 0000000000..9aedef48dc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - pubsub.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf new file mode 100644 index 0000000000..fc81859503 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "subscription_id" { + description = "Name of the subscription that was created." + value = google_pubsub_subscription.example.name +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf new file mode 100644 index 0000000000..ee4dbbed8e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "topic_id" { + description = "The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module" + type = string +} + +variable "subscription_id" { + description = "The name of the pubsub subscription to be created" + type = string + default = null +} + +variable "dataset_id" { + description = "Name of the dataset that was created. Can be provided by the bigquery-table module" + type = string +} + +variable "table_id" { + description = "ID of created BQ table. Can be provided by the bigquery-table module" + type = string +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf new file mode 100644 index 0000000000..46ad6e17c8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" + } + required_version = ">= 1.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/README.md b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/README.md new file mode 100644 index 0000000000..177f799dc6 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/README.md @@ -0,0 +1,82 @@ +## Description + +Creates a Pub/Sub topic + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +### Example + +The following example creates a Pub/Sub topic. + +```yaml + - id: pubsub_topic + source: community/modules/pubsub/topic +``` + +Also see usages in this +[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_pubsub_schema.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_schema) | resource | +| [google_pubsub_topic.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [schema\_id](#input\_schema\_id) | The name of the pubsub schema to be created | `string` | `null` | no | +| [schema\_json](#input\_schema\_json) | The JSON definition of the pubsub topic schema | `string` | `"{ \n \"name\" : \"Avro\", \n \"type\" : \"record\", \n \"fields\" : \n [\n {\"name\" : \"ticker\", \"type\" : \"string\"},\n {\"name\" : \"epoch_time\", \"type\" : \"int\"},\n {\"name\" : \"iteration\", \"type\" : \"int\"},\n {\"name\" : \"start_date\", \"type\" : \"string\"},\n {\"name\" : \"end_date\", \"type\" : \"string\"},\n {\n \"name\":\"simulation_results\",\n \"type\":{\n \"type\": \"array\", \n \"items\":{\n \"name\":\"Child\",\n \"type\":\"record\",\n \"fields\":[\n {\"name\":\"price\", \"type\":\"double\"}\n ]\n }\n }\n }\n ]\n }\n"` | no | +| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to be created | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [topic\_id](#output\_topic\_id) | Name of the topic that was created. | +| [topic\_schema](#output\_topic\_schema) | Name of the topic schema that was created. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/main.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/main.tf new file mode 100644 index 0000000000..4ba68fb5d0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "topic", ghpc_role = "pubsub" }) +} + +locals { + topic_id = var.topic_id != null ? var.topic_id : "${var.deployment_name}_topic_${random_id.resource_name_suffix.hex}" + schema_id = var.schema_id != null ? var.schema_id : "${var.deployment_name}_schema_${random_id.resource_name_suffix.hex}" +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_pubsub_topic" "example" { + name = local.topic_id + depends_on = [google_pubsub_schema.example] + project = var.project_id + labels = local.labels + schema_settings { + schema = "projects/${var.project_id}/schemas/${local.schema_id}" + encoding = "BINARY" + } +} + +resource "google_pubsub_schema" "example" { + name = local.schema_id + project = var.project_id + type = "AVRO" + + definition = var.schema_json +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/metadata.yaml new file mode 100644 index 0000000000..9aedef48dc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - pubsub.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/outputs.tf new file mode 100644 index 0000000000..3ea9d951b2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/outputs.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "topic_id" { + description = "Name of the topic that was created." + value = google_pubsub_topic.example.name +} + + +output "topic_schema" { + description = "Name of the topic schema that was created." + value = local.schema_id +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/variables.tf new file mode 100644 index 0000000000..dca575d21d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/variables.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "topic_id" { + description = "The name of the pubsub topic to be created" + type = string + default = null +} + +variable "schema_id" { + description = "The name of the pubsub schema to be created" + type = string + default = null +} + +variable "schema_json" { + description = "The JSON definition of the pubsub topic schema" + type = string + default = < **Note**: This is an experimental module. This module has only been tested in +> limited capacity with the Cluster Toolkit. The module interface may have undergo +> breaking changes in the future. + +### Example + +The following example will create a single GPU accelerated remote desktop. + +```yaml + - id: remote-desktop + source: community/modules/remote-desktop/chrome-remote-desktop + use: [network1] + settings: + install_nvidia_driver: true +``` + +### Setting up the Remote Desktop + +1. Once the remote desktop has been deployed, navigate to https://remotedesktop.google.com/headless. +1. Click through `Begin`, `Next`, & `Authorize`. +1. Copy the code snippet for `Debian Linux`. +1. SSH into the remote desktop machine. It will be listed under + [VM Instances](https://console.cloud.google.com/compute/instances) in the + Google Cloud web console. +1. Run the copied command and follow instructions to set up a PIN. +1. You should now see your machine listed on the + [Chrome Remote Desktop page](https://remotedesktop.google.com/access) under `Remote devices`. +1. Click on your machine and enter PIN if prompted. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.12.31 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [client\_startup\_script](#module\_client\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | +| [instances](#module\_instances) | ../../../../modules/compute/vm-instance | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | +| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image th
at supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-v
m-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. Cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | +| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-balanced"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs on the internet. | `bool` | `true` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required | 
list(object({
    type  = string,
    count = number
  }))
| 
[
  {
    "count": 1,
    "type": "nvidia-tesla-t4-vws"
  }
]
| no | +| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Installs the nvidia driver (true/false). For details, see https://cloud.google.com/compute/docs/gpus/install-drivers-gpu | `bool` | n/a | yes | +| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | +| [instance\_image](#input\_instance\_image) | Image used to build chrome remote desktop node. The default image is
name="debian-12-bookworm-v20250610" and project="debian-cloud".
NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues.

An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud".

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "name": "debian-12-bookworm-v20250610",
  "project": "debian-cloud"
}
| no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation. Must be N1 family if GPU is used. | `string` | `"n1-standard-8"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface
**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.
Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "RDMA", "IRDMA", "MRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. | `string` | `"default"` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `"TERMINATE"` | no | +| [project\_id](#input\_project\_id) | Project in which Google Cloud resources will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/cloud-platform"
  ]
}
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core | `number` | `2` | no | +| [zone](#input\_zone) | Default zone for creating resources | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_name](#output\_instance\_name) | Name of the first instance created, if any. | +| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf new file mode 100644 index 0000000000..a5cf7c5d37 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "chrome-remote-desktop", ghpc_role = "remote-desktop" }) +} + +locals { + + user_startup_script_runners = var.startup_script == null ? [] : [ + { + type = "shell" + content = var.startup_script + destination = "user_startup_script.sh" + } + ] + + configure_nvidia_driver_runners = var.install_nvidia_driver == false ? [] : [ + { + type = "ansible-local" + content = file("${path.module}/scripts/configure-grid-drivers.yml") + destination = "/usr/local/ghpc/configure-grid-drivers.yml" + } + ] + + configure_chrome_remote_desktop_runners = [ + { + type = "ansible-local" + content = file("${path.module}/scripts/configure-chrome-desktop.yml") + destination = "/usr/local/ghpc/configure-chrome-desktop.yml" + } + ] + + disable_sleep = [ + { + type = "ansible-local" + content = file("${path.module}/scripts/disable-sleep.yml") + destination = "/usr/local/ghpc/disable-sleep.yml" + } + ] +} + +module "client_startup_script" { + source = "../../../../modules/scripts/startup-script" + + deployment_name = var.deployment_name + project_id = var.project_id + region = var.region + labels = local.labels + + runners = flatten([ + local.user_startup_script_runners, + local.configure_nvidia_driver_runners, + local.configure_chrome_remote_desktop_runners, + local.disable_sleep + ]) +} + +module "instances" { + source = "../../../../modules/compute/vm-instance" + + instance_count = var.instance_count + name_prefix = var.name_prefix + add_deployment_name_before_prefix = var.add_deployment_name_before_prefix + provisioning_model = var.spot ? "SPOT" : null + + deployment_name = var.deployment_name + project_id = var.project_id + region = var.region + zone = var.zone + labels = local.labels + + machine_type = var.machine_type + service_account_email = var.service_account.email + metadata = var.metadata + startup_script = module.client_startup_script.startup_script + enable_oslogin = var.enable_oslogin + + instance_image = var.instance_image + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + auto_delete_boot_disk = var.auto_delete_boot_disk + + disable_public_ips = !var.enable_public_ips + network_self_link = var.network_self_link + subnetwork_self_link = var.subnetwork_self_link + network_interfaces = var.network_interfaces + bandwidth_tier = var.bandwidth_tier + tags = var.tags + + threads_per_core = var.threads_per_core + guest_accelerator = var.guest_accelerator + on_host_maintenance = var.on_host_maintenance + + network_storage = var.network_storage + +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf new file mode 100644 index 0000000000..bcf8ece52d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "script to load and run all runners, as a string value." + value = module.client_startup_script.startup_script +} + +output "instance_name" { + description = "Name of the first instance created, if any." + value = var.instance_count > 0 ? module.instances.name[0] : null +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml new file mode 100644 index 0000000000..391aa86433 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml @@ -0,0 +1,61 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Ensure Desktop OS and Chrome Remote Desktop is installed + hosts: localhost + become: true + module_defaults: + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + tasks: + - name: Install desktop packages + ansible.builtin.apt: + name: + - xfce4 + - xfce4-goodies + state: present + register: apt_result + retries: 10 + delay: 30 + until: apt_result is success + + - name: Download and configure CRD + ansible.builtin.get_url: + url: https://dl.google.com/linux/direct/chrome-remote-desktop_current_amd64.deb + dest: /tmp/chrome-remote-desktop_current_amd64.deb + mode: "0755" + timeout: 30 + + - name: Install CRD + ansible.builtin.apt: + deb: /tmp/chrome-remote-desktop_current_amd64.deb + environment: + DEBIAN_FRONTEND: noninteractive + register: apt_result + retries: 10 + delay: 30 + until: apt_result is success + + - name: Configure CRD to use Xfce by default + ansible.builtin.copy: + dest: /etc/chrome-remote-desktop-session + content: "exec /etc/X11/Xsession /usr/bin/xfce4-session" + mode: 0644 + + - name: Start Chrome remote desktop + ansible.builtin.command: /etc/init.d/chrome-remote-desktop start + register: result + changed_when: result.rc == 0 diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml new file mode 100644 index 0000000000..daae08176d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml @@ -0,0 +1,163 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Ensure nvidia grid drivers and other binaries are installed + hosts: localhost + become: true + vars: + dist_settings: + bullseye: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-510.85.02-grid.run + grid_ver: vGPU14.2 + bookworm: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-550.54.15-grid.run + grid_ver: vGPU17.1 + jammy: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - gcc-12 # must match compiler used to build kernel on latest Ubuntu 22 + - pkg-config # observed to be necessary for GRID driver installation on latest Ubuntu 22 + - libglvnd-dev # observed to be necessary for GRID driver installation on latest Ubuntu 22 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-525.125.06-grid.run + grid_ver: vGPU15.3 + tasks: + - name: Fail if using wrong OS + ansible.builtin.assert: + that: + - ansible_os_family in ["Debian", "Ubuntu"] + - ansible_distribution_release in dist_settings.keys() | list + fail_msg: "ansible_os_family: {{ ansible_os_family }} or ansible_distribution_release: {{ansible_distribution_release}} was not acceptable." + + - name: Check if GRID driver installed + ansible.builtin.command: which nvidia-smi + register: nvidiasmi_result + ignore_errors: true + changed_when: false + + - name: Install binaries for GRID drivers + ansible.builtin.apt: + name: '{{ dist_settings[ansible_distribution_release]["packages"] }}' + state: present + update_cache: true + register: apt_result + retries: 6 + delay: 10 + until: apt_result is success + + - name: Install GRID driver if not existing + when: nvidiasmi_result is failed + block: + - name: Download GPU driver + ansible.builtin.get_url: + url: https://storage.googleapis.com/nvidia-drivers-us-public/GRID/{{ dist_settings[ansible_distribution_release]["grid_ver"] }}/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} + dest: /tmp/ + mode: "0755" + timeout: 30 + + - name: Stop gdm service + ansible.builtin.systemd: + name: gdm + state: stopped + + - name: Install GPU driver + ansible.builtin.shell: | + #jinja2: trim_blocks: "True" + {% if ansible_distribution_release == "jammy" %} + CC=gcc-12 /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent + {% else %} + /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent + {% endif %} + register: result + changed_when: result.rc == 0 + + - name: Download VirtualGL driver + ansible.builtin.get_url: + url: https://sourceforge.net/projects/virtualgl/files/3.0.2/virtualgl_3.0.2_amd64.deb/download + dest: /tmp/virtualgl_3.0.2_amd64.deb + mode: "0755" + timeout: 30 + + - name: Install VirtualGL + ansible.builtin.command: gdebi /tmp/virtualgl_3.0.2_amd64.deb --non-interactive + register: result + changed_when: result.rc == 0 + + - name: Fix headless Nvidia issue + block: + - name: Lookup gpu info + ansible.builtin.command: nvidia-xconfig --query-gpu-info + register: gpu_info + failed_when: gpu_info.rc != 0 + changed_when: false + + - name: Extract PCI ID + ansible.builtin.shell: | + set -o pipefail + echo "{{ gpu_info.stdout }}" | grep "PCI BusID " | head -n 1 | cut -d':' -f2-99 | xargs + args: + executable: /bin/bash + register: pci_id + changed_when: false + + - name: Configure nvidia-xconfig + ansible.builtin.command: nvidia-xconfig -a --allow-empty-initial-configuration --enable-all-gpus --virtual=1920x1200 --busid={{ pci_id.stdout }} + register: result + changed_when: result.rc == 0 + + - name: Set HardDPMS to false + ansible.builtin.replace: + path: /etc/X11/xorg.conf + regexp: "Section \"Device\"" + replace: "Section \"Device\"\n Option \"HardDPMS\" \"false\"" + + - name: Configure VirtualGL for X + ansible.builtin.command: vglserver_config +glx +s +f -t + register: result + changed_when: result.rc == 0 + + - name: Configure gdm for X + block: + - name: Configure default display manager + ansible.builtin.copy: + dest: /etc/X11/default-display-manager + content: "/usr/sbin/gdm3" + mode: 0644 + + - name: Switch boot target to gui + ansible.builtin.command: systemctl set-default graphical.target + register: result + changed_when: result.rc == 0 + + - name: Start gdm service + ansible.builtin.systemd: + name: gdm + daemon_reload: true + state: started diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml new file mode 100644 index 0000000000..6767b05fb2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml @@ -0,0 +1,39 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Mask sleep, suspend, hibernate, and hybrid-sleep targets + hosts: localhost + become: true + tasks: + + - name: Mask sleep target + ansible.builtin.systemd: + name: sleep.target + masked: true + + - name: Mask suspend target + ansible.builtin.systemd: + name: suspend.target + masked: true + + - name: Mask hibernate target + ansible.builtin.systemd: + name: hibernate.target + masked: true + + - name: Mask hybrid-sleep target + ansible.builtin.systemd: + name: hybrid-sleep.target + masked: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf new file mode 100644 index 0000000000..ac4c3b1869 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf @@ -0,0 +1,277 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which Google Cloud resources will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. Cloud resource names will include this value." + type = string + #default = "chrome-remote-desktop" +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zone" { + description = "Default zone for creating resources" + type = string +} + +variable "instance_count" { + description = "Number of instances" + type = number + default = 1 +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "instance_image" { + description = <<-EOD + Image used to build chrome remote desktop node. The default image is + name="debian-12-bookworm-v20250610" and project="debian-cloud". + NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues. + + An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud". + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "debian-cloud" + name = "debian-12-bookworm-v20250610" + } +} + +variable "disk_size_gb" { + description = "Size of disk for instances." + type = number + default = 200 +} + +variable "disk_type" { + description = "Disk type for instances." + type = string + default = "pd-balanced" +} + +variable "auto_delete_boot_disk" { + description = "Controls if boot disk should be auto-deleted when instance is deleted." + type = bool + default = true +} + +variable "name_prefix" { + description = <<-EOT + An optional name for all VM and disk resources. + If not supplied, `deployment_name` will be used. + When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, + then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". + EOT + type = string + default = null +} + +variable "add_deployment_name_before_prefix" { + description = <<-EOT + If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. + See `name_prefix` for further details on resource naming behavior. + EOT + type = bool + default = false +} + +variable "enable_public_ips" { + description = "If set to true, instances will have public IPs on the internet." + type = bool + default = true +} + +variable "machine_type" { + description = "Machine type to use for the instance creation. Must be N1 family if GPU is used." + type = string + default = "n1-standard-8" +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "service_account" { + description = "Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account." + type = object({ + email = string, + scopes = set(string) + }) + default = { + email = null + scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the VM." + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the VM." + type = string + default = null +} + +variable "network_interfaces" { + description = <<-EOT + A list of network interfaces. The options match that of the terraform + network_interface block of google_compute_instance. For descriptions of the + subfields or more information see the documentation: + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + **_NOTE:_** If `network_interfaces` are set, `network_self_link` and + `subnetwork_self_link` will be ignored, even if they are provided through + the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply + to network interfaces defined in this variable. + Subfields: + network (string, required if subnetwork is not supplied) + subnetwork (string, required if network is not supplied) + subnetwork_project (string, optional) + network_ip (string, optional) + nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "RDMA", "IRDMA", "MRDMA"]) + stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) + queue_count (number, optional) + access_config (object, optional) + ipv6_access_config (object, optional) + alias_ip_range (list(object), optional) + EOT + type = list(object({ + network = string, + subnetwork = string, + subnetwork_project = string, + network_ip = string, + nic_type = string, + stack_type = string, + queue_count = number, + access_config = list(object({ + nat_ip = string, + public_ptr_domain_name = string, + network_tier = string + })), + ipv6_access_config = list(object({ + public_ptr_domain_name = string, + network_tier = string + })), + alias_ip_range = list(object({ + ip_cidr_range = string, + subnetwork_range_name = string + })) + })) + default = [] +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "startup_script" { + description = "Startup script used on the instance" + type = string + default = null +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required" + type = list(object({ + type = string, + count = number + })) + default = [{ + type = "nvidia-tesla-t4-vws" + count = 1 + }] +} + +variable "threads_per_core" { + description = "Sets the number of threads per physical core" + type = number + default = 2 +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" + type = string + default = "TERMINATE" +} + +variable "bandwidth_tier" { + description = <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +This mode can be switched to proactive (automatic) replacement by setting +[var.update_policy](#input_update_policy) to "PROACTIVE". In this case we +recommend the use of Filestore to store the job queue state ("spool") and +setting [var.spool_parent_dir][#input_spool_parent_dir] to its mount point: + +```yaml + - id: spoolfs + source: modules/file-system/filestore + use: + - network1 + settings: + filestore_tier: ENTERPRISE + local_mount: /shared + +... + + - id: htcondor_access + source: community/modules/scheduler/htcondor-access-point + use: + - network1 + - spoolfs + - htcondor_secrets + - htcondor_setup + - htcondor_cm + - htcondor_execute_point_group + settings: + spool_parent_dir: /shared +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 3.83 | +| [null](#requirement\_null) | >= 3.0 | +| [random](#requirement\_random) | ~> 3.6 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [null](#provider\_null) | >= 3.0 | +| [random](#provider\_random) | ~> 3.6 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [access\_point\_instance\_template](#module\_access\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [htcondor\_ap](#module\_htcondor\_ap) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_address.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_region_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_region_disk) | resource | +| [google_storage_bucket_object.ap_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.ap_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_shuffle.zones](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/shuffle) | resource | +| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_instance.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | +| [google_compute_region_instance_group.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_runner](#input\_access\_point\_runner) | A list of Toolkit runners for configuring an HTCondor access point | `list(map(string))` | `[]` | no | +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account for access point (e-mail format) | `string` | n/a | yes | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [autoscaler\_runner](#input\_autoscaler\_runner) | A list of Toolkit runners for configuring autoscaling daemons | `list(map(string))` | `[]` | no | +| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | +| [default\_mig\_id](#input\_default\_mig\_id) | Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | +| [disk\_type](#input\_disk\_type) | Boot disk size in GB | `string` | `"pd-balanced"` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape acoss zones for instance group managing high availability of access point | `string` | `"ANY_SINGLE_ZONE"` | no | +| [enable\_high\_availability](#input\_enable\_high\_availability) | Provision HTCondor access point in high availability mode | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | Enable Public IPs on the access points | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor and Toolkit support installed."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | +| [mig\_id](#input\_mig\_id) | List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module) | `list(string)` | `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spool\_disk\_size\_gb](#input\_spool\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | +| [spool\_disk\_type](#input\_spool\_disk\_type) | Boot disk size in GB | `string` | `"pd-ssd"` | no | +| [spool\_parent\_dir](#input\_spool\_parent\_dir) | HTCondor access point configuration SPOOL will be set to subdirectory named "spool" | `string` | `"/var/lib/condor"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | +| [zones](#input\_zones) | Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_ips](#output\_access\_point\_ips) | IP addresses of the access points provisioned by this module | +| [access\_point\_name](#output\_access\_point\_name) | Name of the access point provisioned by this module | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml new file mode 100644 index 0000000000..6a2f50c831 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml @@ -0,0 +1,120 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Access Point + hosts: localhost + become: true + vars: + spool_dir: /var/lib/condor/spool + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + htcondor_spool_disk_device: /dev/disk/by-id/google-htcondor-spool-disk + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + - name: Configure HTCondor SchedD + when: htcondor_role == 'get_htcondor_submit' + block: + - name: Format spool disk + community.general.filesystem: + fstype: ext4 + state: present + dev: "{{ htcondor_spool_disk_device }}" + # RUN TUNE2FS + - name: Mount spool (creates mount point) + ansible.posix.mount: + path: "{{ spool_dir }}" + src: "{{ htcondor_spool_disk_device }}" + fstype: ext4 + opts: defaults + state: mounted + - name: Ensure spool free space + ansible.builtin.command: tune2fs -r 0 {{ htcondor_spool_disk_device }} + - name: Setup spool directory + ansible.builtin.file: + path: "{{ spool_dir }}" + state: directory + owner: condor + group: condor + mode: 0755 + recurse: true + - name: Create SystemD override directory for HTCondor + ansible.builtin.file: + path: /etc/systemd/system/condor.service.d + state: directory + owner: root + group: root + mode: 0755 + - name: Ensure HTCondor starts after shared filesystem is mounted + ansible.builtin.copy: + dest: /etc/systemd/system/condor.service.d/mount-spool.conf + mode: 0644 + content: | + [Unit] + RequiresMountsFor={{ spool_dir }} + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor configuration complete; startup-script may still be executing ********" diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf new file mode 100644 index 0000000000..fdbcf5c32f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf @@ -0,0 +1,338 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-access-point", ghpc_role = "scheduler" }) +} + +locals { + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + metadata = merge( + local.network_storage_metadata, + local.enable_oslogin_metadata, + local.disable_automatic_updates_metadata, + var.metadata + ) + + host_count = 1 + name_prefix = "${var.deployment_name}-ap" + + example_runner = { + type = "data" + destination = "/var/tmp/helloworld.sub" + content = <<-EOT + universe = vanilla + executable = /bin/sleep + arguments = 1000 + output = out.$(ClusterId).$(ProcId) + error = err.$(ClusterId).$(ProcId) + log = log.$(ClusterId).$(ProcId) + request_cpus = 1 + request_memory = 100MB + queue + EOT + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.access_point_runner, + [local.schedd_runner], + var.autoscaler_runner, + [local.example_runner] + ) + + ap_config = templatefile("${path.module}/templates/condor_config.tftpl", { + htcondor_role = "get_htcondor_submit", + central_manager_ips = var.central_manager_ips + spool_dir = "${var.spool_parent_dir}/spool", + mig_ids = var.mig_id, + default_mig_id = var.default_mig_id + }) + + ap_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.ap_config.output_name}" + schedd_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e htcondor_role=get_htcondor_submit", + "-e config_object=${local.ap_object}", + "-e spool_dir=${var.spool_parent_dir}/spool", + "-e htcondor_spool_disk_device=/dev/disk/by-id/google-${local.spool_disk_device_name}", + ]) + } + + access_point_ips = google_compute_address.ap.address + access_point_name = data.google_compute_instance.ap.name + + spool_disk_resource_name = "${var.deployment_name}-spool-disk" + spool_disk_device_name = "htcondor-spool-disk" + spool_disk_source = try(google_compute_disk.spool[0].name, google_compute_region_disk.spool[0].self_link) + + zones = coalescelist(var.zones, random_shuffle.zones.result) + + vm_family = split("-", var.machine_type)[0] + regional_pd_families = ["e2", "n1", "n2", "n2d"] +} + +data "google_compute_image" "htcondor" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + postcondition { + condition = self.disk_size_gb <= var.disk_size_gb + error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region + + lifecycle { + postcondition { + condition = alltrue([ + for z in var.zones : contains(self.names, z) + ]) + error_message = "Each entry in var.zones must be a zone in var.region: ${var.region}" + } + } +} + +resource "random_shuffle" "zones" { + input = data.google_compute_zones.available.names + result_count = var.enable_high_availability ? 2 : 1 +} + +data "google_compute_region_instance_group" "ap" { + self_link = module.htcondor_ap.self_link + lifecycle { + postcondition { + condition = length(self.instances) == local.host_count + error_message = "There should be ${local.host_count} access points found" + } + } +} + +data "google_compute_instance" "ap" { + self_link = data.google_compute_region_instance_group.ap.instances[0].instance +} + +resource "null_resource" "ap_config" { + triggers = { + config = local.ap_config + } +} + +resource "google_storage_bucket_object" "ap_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.ap_config.id), 0, 4)}" + content = local.ap_config + bucket = var.htcondor_bucket_name + + lifecycle { + precondition { + condition = var.default_mig_id == "" || contains(var.mig_id, var.default_mig_id) + error_message = "If set, var.default_mig_id must be an element in var.mig_id" + } + + # by construction, this precondition only fails when the user has set + # var.zones to a non-empty list of length not equal to 2 + precondition { + condition = !var.enable_high_availability || length(local.zones) == 2 + error_message = "When using HTCondor access point high availability, var.zones must be of length 2." + } + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +resource "google_compute_region_disk" "spool" { + count = var.enable_high_availability ? 1 : 0 + name = local.spool_disk_resource_name + labels = local.labels + type = var.spool_disk_type + region = var.region + size = var.spool_disk_size_gb + + replica_zones = local.zones + + lifecycle { + precondition { + condition = var.spool_disk_size_gb >= 200 + error_message = "When using HTCondor access point high availability, var.spool_disk_size_gb must be set to 200 or greater." + } + + precondition { + condition = contains(local.regional_pd_families, local.vm_family) + error_message = "When using HTCondor access point high availability, var.machine_type must be one of ${jsonencode(local.regional_pd_families)}." + } + } +} + +resource "google_compute_disk" "spool" { + count = var.enable_high_availability ? 0 : 1 + name = local.spool_disk_resource_name + labels = local.labels + type = var.spool_disk_type + zone = local.zones[0] + size = var.spool_disk_size_gb +} + +resource "google_compute_address" "ap" { + project = var.project_id + name = local.name_prefix + region = var.region + subnetwork = var.subnetwork_self_link + address_type = "INTERNAL" + purpose = "GCE_ENDPOINT" +} + +module "access_point_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.access_point_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + preemptible = false + startup_script = module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.htcondor.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + network_ip = google_compute_address.ap.id + + # spool disk + additional_disks = [ + { + source = local.spool_disk_source + device_name = local.spool_disk_device_name + } + ] +} + +module "htcondor_ap" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = local.host_count + hostname = local.name_prefix + instance_template = module.access_point_instance_template.self_link + + health_check_name = "health-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) + max_surge_fixed = 0 # must be 0 to preserve hostnames + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + + stateful_disks = [{ + device_name = local.spool_disk_device_name + delete_rule = "ON_PERMANENT_INSTANCE_DELETION" + }] + stateful_ips = var.enable_public_ips ? [{ + interface_name = "nic0" + delete_rule = "ON_PERMANENT_INSTANCE_DELETION" + is_external = true + }] : [] + + # the timeouts below are default for resource + wait_for_instances = true + mig_timeouts = { + create = "15m" + delete = "15m" + update = "15m" + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf new file mode 100644 index 0000000000..f7424c6d5d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "access_point_ips" { + description = "IP addresses of the access points provisioned by this module" + value = local.access_point_ips +} + +output "access_point_name" { + description = "Name of the access point provisioned by this module" + value = local.access_point_name +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl new file mode 100644 index 0000000000..214fbc726f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl @@ -0,0 +1,70 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:${htcondor_role} +CONDOR_HOST = ${join(",", central_manager_ips)} + +SPOOL = ${spool_dir} +SCHEDD_INTERVAL = 30 +TRUST_UID_DOMAIN = True +SUBMIT_ATTRS = RunAsOwner +RunAsOwner = True + +# When a job matches to a machine, add machine attributes to the job for +# condor_history (e.g. VM Instance ID) +use feature:JobsHaveInstanceIDs +SYSTEM_JOB_MACHINE_ATTRS = $(SYSTEM_JOB_MACHINE_ATTRS) \ + CloudVMType CloudZone CloudInterruptible +SYSTEM_JOB_MACHINE_ATTRS_HISTORY_LENGTH = 10 + +# Add Cloud attributes to SchedD ClassAd +use feature:ScheddCronOneShot(cloud, $(LIBEXEC)/common-cloud-attributes-google.py) +SCHEDD_CRON_cloud_PREFIX = Cloud + +# aid the user by automatically using RequireSpot in their Requirements, unless +# the user has explicitly used CloudInterruptible +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SPOT +JOB_TRANSFORM_SPOT @=end + REQUIREMENTS ! isUndefined(RequireSpot) && ! unresolved(Requirements, "^CloudInterruptible$") + SET Requirements ($(MY.Requirements)) && (CloudInterruptible is My.RequireSpot) +@end + +# help the user by enforcing that RequireSpot is undefined or a boolean +SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) SPOT +SUBMIT_REQUIREMENT_SPOT = isUndefined(RequireSpot) || isBoolean(RequireSpot) +SUBMIT_REQUIREMENT_SPOT_REASON = "If +RequireSpot is defined, it must be either True or False" + +%{ if length(mig_ids) > 0 ~} +MIG_IDS = "${join(" ", mig_ids)}" +MIG_ID_LIST = split($(MIG_IDS)) +%{ if default_mig_id != "" ~} +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) ID_DEFAULT +JOB_TRANSFORM_ID_DEFAULT @=end + DEFAULT RequireId "${default_mig_id}" +@end +%{ endif ~} +SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) MIGID +SUBMIT_REQUIREMENT_MIGID = !isUndefined(RequireId) && member(RequireId, $(MIG_ID_LIST)) +SUBMIT_REQUIREMENT_MIGID_REASON = strcat("Jobs must set +RequireId to one of following values surrounded by quotation marks:\n", $(MIG_IDS)) + +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) MIGID +JOB_TRANSFORM_MIGID @=end + REQUIREMENTS ! isUndefined(RequireId) && ! unresolved(Requirements, "^CloudCreatedBy$") + SET Requirements ($(MY.Requirements)) && regexp(strcat("/", My.RequireId, "$"), CloudCreatedBy) +@end +%{ endif ~} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf new file mode 100644 index 0000000000..f54a88ac2e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf @@ -0,0 +1,266 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zones" { + description = "Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region." + type = list(string) + default = [] + nullable = false + + validation { + condition = length(var.zones) <= 2 + error_message = "Set var.zones to the empty list or up to 2 zones in var.region" + } +} + +variable "distribution_policy_target_shape" { + description = "Target shape acoss zones for instance group managing high availability of access point" + type = string + default = "ANY_SINGLE_ZONE" +} + +variable "network_self_link" { + description = "The self link of the network in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "access_point_service_account_email" { + description = "Service account for access point (e-mail format)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 32 + nullable = false +} + +variable "disk_type" { + description = "Boot disk size in GB" + type = string + default = "pd-balanced" + nullable = false +} + +variable "spool_disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 32 + nullable = false +} + +variable "spool_disk_type" { + description = "Boot disk size in GB" + type = string + default = "pd-ssd" + nullable = false +} + +variable "metadata" { + description = "Metadata to add to HTCondor central managers" + type = map(string) + default = {} +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + nullable = false + validation { + condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "enable_high_availability" { + description = "Provision HTCondor access point in high availability mode" + type = bool + default = false +} + +variable "instance_image" { + description = <<-EOD + Custom VM image with HTCondor and Toolkit support installed." + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "machine_type" { + description = "Machine type to use for HTCondor central managers" + type = string + default = "n2-standard-4" +} + +variable "access_point_runner" { + description = "A list of Toolkit runners for configuring an HTCondor access point" + type = list(map(string)) + default = [] +} + +variable "autoscaler_runner" { + description = "A list of Toolkit runners for configuring autoscaling daemons" + type = list(map(string)) + default = [] +} + +variable "spool_parent_dir" { + description = "HTCondor access point configuration SPOOL will be set to subdirectory named \"spool\"" + type = string + default = "/var/lib/condor" +} + +variable "central_manager_ips" { + description = "List of IP addresses of HTCondor Central Managers" + type = list(string) +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "enable_public_ips" { + description = "Enable Public IPs on the access points" + type = bool + default = false +} + +variable "mig_id" { + description = "List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module)" + type = list(string) + default = [] + nullable = false + + validation { + condition = length(var.mig_id) > 0 + error_message = "At least 1 MIG containing execute points must be provided to this module" + } +} + +variable "default_mig_id" { + description = "Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id" + type = string + default = "" + nullable = false +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" + type = string + default = "OPPORTUNISTIC" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf new file mode 100644 index 0000000000..0d07e7abf1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.74.0" + } + + required_version = ">= 1.1" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md new file mode 100644 index 0000000000..dfab563a55 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md @@ -0,0 +1,159 @@ +## Description + +This module provisions a highly available HTCondor central manager using a [Managed +Instance Group (MIG)][mig] with auto-healing. + +[mig]: https://cloud.google.com/compute/docs/instance-groups + +## Usage + +This module provisions an HTCondor central manager with a standard +configuration. For the node to function correctly, you must supply the input +variable described below: + +- [var.central_manager_runner](#input_central_manager_runner) + - Runner must download a POOL password / signing key and create an [IDTOKEN] + with no scopes (full authorization). + +A reference implementation is included in the Toolkit module +[htcondor-pool-secrets]. You may substitute implementations so long as they +duplicate the functionality in the references. Usage is demonstrated in the +[HTCondor example][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- +[htcondor-pool-secrets]: ../htcondor-pool-secrets/README.md +[IDTOKEN]: https://htcondor.readthedocs.io/en/latest/admin-manual/security.html#introducing-idtokens + +## Behavior of Managed Instance Group (MIG) + +A regional [MIG][mig] is used to provision the central manager, although only +1 node will ever be active at a time. By default, the node will be provisioned +in any of the zones available in that region, however, it can be constrained to +run in fewer zones (or a single zone) using [var.zones](#input_zones). + +When the configuration of the Central Manager is changed, the MIG can be +configured to [replace the VM][replacement] using a "proactive" or +"opportunistic" policy. By default, the Central Manager replacement policy is +set to proactive. In practice, this means that the Central Manager will be +replaced by Terraform when changes to the instance template / HTCondor +configuration are made. The Central Manager is safe to replace automatically as +it gathers its state information from periodic messages exchanged with the rest +of the HTCondor pool. + +This mode can be configured by setting [var.update_policy](#input_update_policy) +to either "PROACTIVE" (default) or "OPPORTUNISTIC". If set to opportunistic +replacement, the Central Manager will be replaced only when: + +- intentionally by issuing an update via Cloud Console or using gcloud (below) +- the VM becomes unhealthy or is otherwise automatically replaced (e.g. regular + Google Cloud maintenance) + +For example, to manually update all instances in a MIG: + +```text +gcloud compute instance-groups managed update-instances \ + <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + +## Limiting inter-zone egress + +Because all the elements of the HTCondor pool use regional MIGs, they may be +subject to [interzone egress fees][network-pricing]. The primary traffic between +nodes of an HTCondor pool running embarrassingly parallel jobs is expected to +be limited to API traffic for job scheduling and monitoring. Please review the +[network pricing][network-pricing] documentation and determine if this cost is +a concern. If it is, use [var.zones](#input_zones) to constrain each node within +your HTCondor pool to operate within a single zone. + +[network-pricing]: https://cloud.google.com/vpc/network-pricing + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1.0 | +| [google](#requirement\_google) | >= 3.83 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [central\_manager\_instance\_template](#module\_central\_manager\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [htcondor\_cm](#module\_htcondor\_cm) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_address.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_storage_bucket_object.cm_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.cm_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_instance.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | +| [google_compute_region_instance_group.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [central\_manager\_runner](#input\_central\_manager\_runner) | A list of Toolkit runners for configuring an HTCondor central manager | `list(map(string))` | `[]` | no | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for central manager (can be supplied by htcondor-setup module) | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `20` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape for instance group managing high availability of central manager | `string` | `"ANY_SINGLE_ZONE"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor installed using the htcondor-install module."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor central manager will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Central Manager ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle). | `string` | `"PROACTIVE"` | no | +| [zones](#input\_zones) | Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [central\_manager\_ips](#output\_central\_manager\_ips) | IP addresses of the central managers provisioned by this module | +| [central\_manager\_name](#output\_central\_manager\_name) | Name of the central managers provisioned by this module | +| [list\_instances\_command](#output\_list\_instances\_command) | Command to list central managers provisioned by this module | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml new file mode 100644 index 0000000000..7408af6370 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml @@ -0,0 +1,72 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor central manager + hosts: localhost + become: true + vars: + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + handlers: + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf new file mode 100644 index 0000000000..d288a91144 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf @@ -0,0 +1,226 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-central-manager", ghpc_role = "scheduler" }) +} + +locals { + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + metadata = merge( + local.network_storage_metadata, + local.enable_oslogin_metadata, + local.disable_automatic_updates_metadata, + var.metadata + ) + + name_prefix = "${var.deployment_name}-cm" + + cm_config = templatefile("${path.module}/templates/condor_config.tftpl", {}) + + cm_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.cm_config.output_name}" + schedd_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e config_object=${local.cm_object}", + ]) + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.central_manager_runner, + [local.schedd_runner] + ) + + central_manager_ips = google_compute_address.cm.address + central_manager_name = data.google_compute_instance.cm.name + + list_instances_command = "gcloud compute instance-groups list-instances ${data.google_compute_region_instance_group.cm.name} --region ${var.region} --project ${var.project_id}" + + zones = coalescelist(var.zones, data.google_compute_zones.available.names) +} + +data "google_compute_image" "htcondor" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + postcondition { + condition = self.disk_size_gb <= var.disk_size_gb + error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +data "google_compute_region_instance_group" "cm" { + self_link = module.htcondor_cm.self_link + lifecycle { + postcondition { + condition = length(self.instances) == 1 + error_message = "There should only be 1 central manager found" + } + } +} + +data "google_compute_instance" "cm" { + self_link = data.google_compute_region_instance_group.cm.instances[0].instance +} + +resource "null_resource" "cm_config" { + triggers = { + config = local.cm_config + } +} + +resource "google_storage_bucket_object" "cm_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.cm_config.id), 0, 4)}" + content = local.cm_config + bucket = var.htcondor_bucket_name +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +resource "google_compute_address" "cm" { + project = var.project_id + name = local.name_prefix + region = var.region + subnetwork = var.subnetwork_self_link + address_type = "INTERNAL" + purpose = "GCE_ENDPOINT" +} + +module "central_manager_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.central_manager_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + preemptible = false + startup_script = module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.htcondor.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + network_ip = google_compute_address.cm.id +} + +module "htcondor_cm" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = 1 + hostname = local.name_prefix + instance_template = module.central_manager_instance_template.self_link + + health_check_name = "health-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) + max_surge_fixed = 0 # must be 0 to preserve hostnames + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + + # the timeouts below are default for resource + wait_for_instances = true + mig_timeouts = { + create = "15m" + delete = "15m" + update = "15m" + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf new file mode 100644 index 0000000000..a6272e7ca2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "list_instances_command" { + description = "Command to list central managers provisioned by this module" + value = local.list_instances_command +} + +output "central_manager_ips" { + description = "IP addresses of the central managers provisioned by this module" + value = local.central_manager_ips +} + +output "central_manager_name" { + description = "Name of the central managers provisioned by this module" + value = local.central_manager_name +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl new file mode 100644 index 0000000000..5b9676457e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl @@ -0,0 +1,31 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:get_htcondor_central_manager +CONDOR_HOST = $(IPV4_ADDRESS) + +# Central Manager configuration settings +# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-collector-configuration-file-entries +# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-negotiator-configuration-file-entries +# set classad lifetime (expiration) to ~5x the update interval for all daemons +# defaults to 900s +CLASSAD_LIFETIME = 180 +COLLECTOR_UPDATE_INTERVAL = 30 +NEGOTIATOR_UPDATE_INTERVAL = 30 +NEGOTIATOR_DEPTH_FIRST = True +NEGOTIATOR_UPDATE_AFTER_CYCLE = True diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf new file mode 100644 index 0000000000..7f85861c3f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf @@ -0,0 +1,192 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor central manager will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zones" { + description = "Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region." + type = list(string) + default = [] + nullable = false +} + +variable "distribution_policy_target_shape" { + description = "Target shape for instance group managing high availability of central manager" + type = string + default = "ANY_SINGLE_ZONE" +} + +variable "network_self_link" { + description = "The self link of the network in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "central_manager_service_account_email" { + description = "Service account e-mail for central manager (can be supplied by htcondor-setup module)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 20 + nullable = false +} + +variable "metadata" { + description = "Metadata to add to HTCondor central managers" + type = map(string) + default = {} +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + nullable = false + validation { + condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "instance_image" { + description = <<-EOD + Custom VM image with HTCondor installed using the htcondor-install module." + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "machine_type" { + description = "Machine type to use for HTCondor central managers" + type = string + default = "n2-standard-4" +} + +variable "central_manager_runner" { + description = "A list of Toolkit runners for configuring an HTCondor central manager" + type = list(map(string)) + default = [] +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Central Manager (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)." + type = string + default = "PROACTIVE" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf new file mode 100644 index 0000000000..4dee3adac7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.74.0" + } + + required_version = ">= 1.1.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md new file mode 100644 index 0000000000..7158e7bac6 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md @@ -0,0 +1,172 @@ +## Description + +This module is responsible for the following actions: + +- store an HTCondor Pool password in Google Cloud Secret Manager + - will generate a new password if one is not supplied +- create a secret in Google Cloud Secret Manager in which the HTCondor central + manager can place IDTOKENs (JWT Authorizations) for execute points to download +- create a Toolkit runner for the central manager + - download the POOL password / signing key + - create a local IDTOKEN for itself + - upload the execute point IDTOKEN secret +- create a Toolkit runner for access points + - download the POOL password / signing key + - create a local IDTOKEN for itself +- create a Toolkit runner for execute points + - Fetch the IDTOKEN secret generated by the central manager + +It is expected to be used with the [htcondor-install] and +[htcondor-execute-point] modules. + +[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[htcondor-install]: ../../scripts/htcondor-setup/README.md +[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md + +[htcrole]: https://htcondor.readthedocs.io/en/latest/getting-htcondor/admin-quick-start.html#what-get-htcondor-does-to-configure-a-role + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_setup + + - id: htcondor_startup_central_manager + source: modules/scripts/startup-script + settings: + runners: + - $(htcondor_install.install_htcondor_runner) + - $(htcondor_secrets.central_manager_runner) + - $(htcondor_setup.central_manager_runner) + +- id: htcondor_cm + source: modules/compute/vm-instance + use: + - network1 + - htcondor_startup_central_manager + settings: + name_prefix: cm0 + machine_type: c2-standard-4 + disable_public_ips: true + service_account: + email: $(htcondor_setup.central_manager_service_account) + scopes: + - cloud-platform + network_interfaces: + - network: null + subnetwork: $(network1.subnetwork_self_link) + subnetwork_project: $(vars.project_id) + network_ip: $(htcondor_setup.central_manager_internal_ip) + stack_type: null + access_config: [] + ipv6_access_config: [] + alias_ip_range: [] + nic_type: VIRTIO_NET + queue_count: null + outputs: + - internal_ip +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 4.84 | +| [random](#requirement\_random) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | +| [random](#provider\_random) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_secret_manager_secret.execute_point_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_iam_member.access_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.central_manager_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.central_manager_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.execute_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_version.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [random_password.pool](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [pool\_password](#input\_pool\_password) | HTCondor Pool Password | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [trust\_domain](#input\_trust\_domain) | Trust domain for HTCondor pool (if not supplied, will be set based on project\_id) | `string` | `""` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_runner](#output\_access\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor access point | +| [central\_manager\_runner](#output\_central\_manager\_runner) | Toolkit Runner to download pool secrets to an HTCondor central manager | +| [execute\_point\_runner](#output\_execute\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor execute point | +| [pool\_password\_secret\_id](#output\_pool\_password\_secret\_id) | Google Cloud Secret Manager ID containing HTCondor Pool Password | +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | PowerShell script to download pool secrets to an HTCondor execute point | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml new file mode 100644 index 0000000000..538c809c2a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml @@ -0,0 +1,102 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Secrets + hosts: localhost + become: true + vars: + condor_config_root: /etc/condor + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - password_id is defined + - trust_domain is defined + - name: Set Pool Trust Domain + ansible.builtin.copy: + dest: "{{ condor_config_root }}/config.d/51-ghpc-trust-domain" + mode: 0644 + content: | + # these lines must appear AFTER any "use role:" settings + UID_DOMAIN = {{ trust_domain }} + TRUST_DOMAIN = {{ trust_domain }} + - name: Get HTCondor Pool password (token signing key) + when: htcondor_role != 'get_htcondor_execute' + ansible.builtin.shell: | + set -e -o pipefail +o history + POOL_PASSWORD=$(gcloud secrets versions access latest --secret={{ password_id }}) + echo -n "$POOL_PASSWORD" | sh -c "condor_store_cred add -c -i -" + args: + creates: "{{ condor_config_root }}/passwords.d/POOL" + executable: /bin/bash + - name: Configure HTCondor Central Manager + when: htcondor_role == 'get_htcondor_central_manager' + block: + - name: Create IDTOKEN for Central Manager + ansible.builtin.shell: | + umask 0077 + condor_token_create -identity condor@{{ trust_domain }} \ + -token condor@{{ trust_domain }} + args: + creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" + - name: Create IDTOKEN secret for Execute Points + when: xp_idtoken_secret_id | length > 0 + changed_when: true + ansible.builtin.shell: | + umask 0077 + TMPFILE=$(mktemp) + condor_token_create -authz READ -authz ADVERTISE_MASTER \ + -authz ADVERTISE_STARTD -identity condor@{{ trust_domain }} > "$TMPFILE" + gcloud secrets versions add --data-file "$TMPFILE" {{ xp_idtoken_secret_id }} + rm -f "$TMPFILE" + - name: Configure HTCondor SchedD + when: htcondor_role == 'get_htcondor_submit' + block: + - name: Create IDTOKEN to advertise access point + ansible.builtin.shell: | + umask 0077 + # DAEMON authorization can likely be removed in future when scopes + # needed to trigger a negotiation cycle are changed. Suggest review + # https://opensciencegrid.atlassian.net/jira/software/c/projects/HTCONDOR/issues/?filter=allissues + condor_token_create -authz READ -authz ADVERTISE_MASTER \ + -authz ADVERTISE_SCHEDD -authz DAEMON -identity condor@{{ trust_domain }} \ + -token condor@{{ trust_domain }} + args: + creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" + - name: Configure HTCondor StartD + when: htcondor_role == 'get_htcondor_execute' + block: + - name: Create SystemD override directory for HTCondor Execute Point + ansible.builtin.file: + path: /etc/systemd/system/condor.service.d + state: directory + owner: root + group: root + mode: 0755 + - name: Fetch IDTOKEN to advertise execute point + ansible.builtin.copy: + dest: "/etc/systemd/system/condor.service.d/htcondor-token-fetcher.conf" + mode: 0644 + content: | + [Service] + ExecStartPre=gcloud secrets versions access latest --secret {{ xp_idtoken_secret_id }} \ + --out-file {{ condor_config_root }}/tokens.d/condor@{{ trust_domain }} + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf new file mode 100644 index 0000000000..1a7c761760 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf @@ -0,0 +1,168 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-pool-secrets", ghpc_role = "scheduler" }) +} + +locals { + pool_password = coalesce(var.pool_password, random_password.pool.result) + auto = length(var.user_managed_replication) == 0 ? "" : "-user" + access_point_service_account_iam_email = "serviceAccount:${var.access_point_service_account_email}" + central_manager_service_account_iam_email = "serviceAccount:${var.central_manager_service_account_email}" + execute_point_service_account_iam_email = "serviceAccount:${var.execute_point_service_account_email}" + + trust_domain = coalesce(var.trust_domain, "c.${var.project_id}.internal") + + runner_cm = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_central_manager", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + + runner_access = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_submit", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + + runner_execute = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_execute", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + windows_startup_ps1 = templatefile( + "${path.module}/templates/fetch-idtoken.ps1.tftpl", + { + trust_domain = local.trust_domain, + xp_idtoken_secret_id = google_secret_manager_secret.execute_point_idtoken.secret_id, + } + ) +} + +resource "random_password" "pool" { + length = 24 + special = true + override_special = "_-#=." +} + +resource "google_secret_manager_secret" "pool_password" { + secret_id = "${var.deployment_name}-pool-password${local.auto}" + + labels = local.labels + + replication { + dynamic "auto" { + for_each = length(var.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_version" "pool_password" { + secret = google_secret_manager_secret.pool_password.id + secret_data = local.pool_password +} + +# this secret will be populated by the Central Manager +resource "google_secret_manager_secret" "execute_point_idtoken" { + secret_id = "${var.deployment_name}-execute-point-idtoken${local.auto}" + + labels = local.labels + + replication { + dynamic "auto" { + for_each = length(var.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_iam_member" "central_manager_password" { + secret_id = google_secret_manager_secret.pool_password.id + role = "roles/secretmanager.secretAccessor" + member = local.central_manager_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "central_manager_idtoken" { + secret_id = google_secret_manager_secret.execute_point_idtoken.id + role = "roles/secretmanager.secretVersionManager" + member = local.central_manager_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "access_point" { + secret_id = google_secret_manager_secret.pool_password.id + role = "roles/secretmanager.secretAccessor" + member = local.access_point_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "execute_point" { + secret_id = google_secret_manager_secret.execute_point_idtoken.id + role = "roles/secretmanager.secretAccessor" + member = local.execute_point_service_account_iam_email +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml new file mode 100644 index 0000000000..4b0bdbd616 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com + - secretmanager.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf new file mode 100644 index 0000000000..81c4986b16 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "pool_password_secret_id" { + description = "Google Cloud Secret Manager ID containing HTCondor Pool Password" + value = google_secret_manager_secret.pool_password.secret_id + sensitive = true +} + +output "central_manager_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor central manager" + value = local.runner_cm + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "access_point_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor access point" + value = local.runner_access + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "execute_point_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor execute point" + value = local.runner_execute + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "windows_startup_ps1" { + description = "PowerShell script to download pool secrets to an HTCondor execute point" + value = local.windows_startup_ps1 +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl new file mode 100644 index 0000000000..04c96291ee --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl @@ -0,0 +1,26 @@ +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +$config_dir = 'C:\Condor\config' +if(!(test-path -PathType container -Path $config_dir)) +{ + New-Item -ItemType Directory -Path $config_dir +} +$config_file = "$config_dir\51-ghpc-trust-domain" + +$config_string = @' +# these lines must appear AFTER any "use role:" settings +UID_DOMAIN = ${trust_domain} +TRUST_DOMAIN = ${trust_domain} +'@ + +Set-Content -Path "$config_file" -Value "$config_string" + +# obtain IDTOKEN for authentication by StartD to Central Manager +gcloud secrets versions access latest --secret ${xp_idtoken_secret_id} ` + --out-file C:\condor\tokens.d\condor@${trust_domain} + +if ($LASTEXITCODE -ne 0) +{ + throw "Could not download HTCondor IDTOKEN; exiting startup script" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf new file mode 100644 index 0000000000..22ef3644e8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "access_point_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "central_manager_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "execute_point_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "pool_password" { + description = "HTCondor Pool Password" + type = string + sensitive = true + default = null +} + +variable "trust_domain" { + description = "Trust domain for HTCondor pool (if not supplied, will be set based on project_id)" + type = string + default = "" +} + +variable "user_managed_replication" { + type = list(object({ + location = string + kms_key_name = optional(string) + })) + description = "Replication parameters that will be used for defined secrets" + default = [] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf new file mode 100644 index 0000000000..d8a1d96f5f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md new file mode 100644 index 0000000000..5a403c0a38 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md @@ -0,0 +1,128 @@ +## Description + +This module creates the service accounts for use by the primary elements of an +[HTCondor pool][pool]: + +- Central Managers +- Access Points +- Execute Points + +Each service account is assigned common roles necessary for the VM to function +properly. In particular, nearly every VM requires the ability to read from Cloud +Storage buckets and write Cloud Logging entries. These roles are configurable +as described below. + +[pool]: https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-admin-manual.html#the-different-roles-a-machine-can-play + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_service_accounts + source: community/modules/scheduler/htcondor-service-accounts + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + - htcondor_service_accounts + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_service_accounts + +- id: htcondor_cm + source: community/modules/scheduler/htcondor-central-manager + use: + - network1 + - htcondor_secrets + - htcondor_service_accounts + - htcondor_setup + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - central_manager_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [access\_point\_service\_account](#module\_access\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | +| [central\_manager\_service\_account](#module\_central\_manager\_service\_account) | ../../../../community/modules/project/service-account | n/a | +| [execute\_point\_service\_account](#module\_execute\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_roles](#input\_access\_point\_roles) | Project-wide roles for HTCondor Access Point service account | `list(string)` | 
[
  "compute.instanceAdmin.v1",
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [central\_manager\_roles](#input\_central\_manager\_roles) | Project-wide roles for HTCondor Central Manager service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_roles](#input\_execute\_point\_roles) | Project-wide roles for HTCondor Execute Point service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_service\_account\_email](#output\_access\_point\_service\_account\_email) | HTCondor Access Point Service Account (e-mail format) | +| [central\_manager\_service\_account\_email](#output\_central\_manager\_service\_account\_email) | HTCondor Central Manager Service Account (e-mail format) | +| [execute\_point\_service\_account\_email](#output\_execute\_point\_service\_account\_email) | HTCondor Execute Point Service Account (e-mail format) | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf new file mode 100644 index 0000000000..9d97b18642 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# NB: the community/modules/project/service-account module will not output the +# service account e-mail address until all IAM bindings have been created; if +# underlying implementation changes, this module should declare explicit +# depends_on the IAM bindings to prevent race conditions for services that +# require them + +module "access_point_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Access Point" + deployment_name = var.deployment_name + name = "access" + project_roles = var.access_point_roles +} + +module "execute_point_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Execute Point" + deployment_name = var.deployment_name + name = "execute" + project_roles = var.execute_point_roles +} + +module "central_manager_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Central Manager" + deployment_name = var.deployment_name + name = "cm" + project_roles = var.central_manager_roles +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml new file mode 100644 index 0000000000..c4dcdffdf4 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf new file mode 100644 index 0000000000..28f3a79457 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "access_point_service_account_email" { + description = "HTCondor Access Point Service Account (e-mail format)" + value = module.access_point_service_account.service_account_email +} + +output "central_manager_service_account_email" { + description = "HTCondor Central Manager Service Account (e-mail format)" + value = module.central_manager_service_account.service_account_email +} + +output "execute_point_service_account_email" { + description = "HTCondor Execute Point Service Account (e-mail format)" + value = module.execute_point_service_account.service_account_email +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf new file mode 100644 index 0000000000..ee186e0971 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "access_point_roles" { + description = "Project-wide roles for HTCondor Access Point service account" + type = list(string) + default = [ + "compute.instanceAdmin.v1", + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} + +variable "central_manager_roles" { + description = "Project-wide roles for HTCondor Central Manager service account" + type = list(string) + default = [ + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} + +variable "execute_point_roles" { + description = "Project-wide roles for HTCondor Execute Point service account" + type = list(string) + default = [ + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/README.md new file mode 100644 index 0000000000..1722702ceb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/README.md @@ -0,0 +1,118 @@ +## Description + +This module creates a bucket in which to store HTCondor configurations and +a firewall rule that allows Managed Instance Group health checks to probe the +health of HTCondor VMs. + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_service_accounts + source: community/modules/scheduler/htcondor-service-accounts + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + - htcondor_service_accounts + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_service_accounts + +- id: htcondor_cm + source: community/modules/scheduler/htcondor-central-manager + use: + - network1 + - htcondor_secrets + - htcondor_service_accounts + - htcondor_setup + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - central_manager_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [health\_check\_firewall\_rule](#module\_health\_check\_firewall\_rule) | ../../../../modules/network/firewall-rules | n/a | +| [htcondor\_bucket](#module\_htcondor\_bucket) | ../../../../modules/file-system/cloud-storage-bucket | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account e-mail for HTCondor Access Point | `string` | n/a | yes | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for HTCondor Central Manager | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account e-mail for HTCondor Execute Points | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which Central Managers will be placed. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [htcondor\_bucket\_name](#output\_htcondor\_bucket\_name) | Name of the HTCondor configuration bucket | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf new file mode 100644 index 0000000000..e048362663 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-setup", ghpc_role = "scheduler" }) +} + +locals { + service_account_iam_email = [ + "serviceAccount:${var.access_point_service_account_email}", + "serviceAccount:${var.central_manager_service_account_email}", + "serviceAccount:${var.execute_point_service_account_email}", + ] + service_account_email = [ + var.access_point_service_account_email, + var.central_manager_service_account_email, + var.execute_point_service_account_email, + ] +} + +module "health_check_firewall_rule" { + source = "../../../../modules/network/firewall-rules" + + subnetwork_self_link = var.subnetwork_self_link + + ingress_rules = [{ + name = "allow-health-check-${var.deployment_name}" + description = "Allow Managed Instance Group Health Checks for HTCondor VMs" + direction = "INGRESS" + source_ranges = [ + "130.211.0.0/22", + "35.191.0.0/16", + ] + target_service_accounts = local.service_account_email + allow = [{ + protocol = "tcp" + ports = ["9618"] + }] + }] +} + +module "htcondor_bucket" { + source = "../../../../modules/file-system/cloud-storage-bucket" + + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + name_prefix = "${var.deployment_name}-htcondor-config" + random_suffix = true + labels = local.labels + viewers = local.service_account_iam_email + + use_deployment_name_in_bucket_name = false +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml new file mode 100644 index 0000000000..7b4918b962 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - iam.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf new file mode 100644 index 0000000000..a44223faee --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "htcondor_bucket_name" { + description = "Name of the HTCondor configuration bucket" + value = module.htcondor_bucket.gcs_bucket_name + + # ensure that all IAM bindings to the bucket and firewall rules are active + # before this modules output is allowed to propagate + depends_on = [ + module.htcondor_bucket, + module.health_check_firewall_rule + ] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf new file mode 100644 index 0000000000..147a2ca88d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which Central Managers will be placed." + type = string +} + +variable "access_point_service_account_email" { + description = "Service account e-mail for HTCondor Access Point" + type = string +} + +variable "central_manager_service_account_email" { + description = "Service account e-mail for HTCondor Central Manager" + type = string +} + +variable "execute_point_service_account_email" { + description = "Service account e-mail for HTCondor Execute Points" + type = string +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md new file mode 100644 index 0000000000..43254cbfa8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md @@ -0,0 +1,405 @@ +## Description + +This module creates a slurm controller node via the internal +[slurm\_instance\_template] module. + +More information about Slurm On GCP can be found at the +[project's GitHub page][slurm-gcp] and in the +[Slurm on Google Cloud User Guide][slurm-ug]. + +The [user guide][slurm-ug] provides detailed instructions on customizing and +enhancing the Slurm on GCP cluster as well as recommendations on configuring the +controller for optimal performance at different scales. + +[slurm\_instance\_template]: /community/modules/internal/slurm-gcp/instance_template/README.md +[slurm-ug]: https://goo.gle/slurm-gcp-user-guide. +[enable\_cleanup\_compute]: #input\_enable\_cleanup\_compute +[enable\_cleanup\_subscriptions]: #input\_enable\_cleanup\_subscriptions +[enable\_reconfigure]: #input\_enable\_reconfigure + +### Example + +```yaml +- id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: + - network + - homefs + - compute_partition + settings: + machine_type: c2-standard-8 +``` + +This creates a controller node with the following attributes: + +* connected to the primary subnetwork of `network` +* the filesystem with the ID `homefs` (defined elsewhere in the blueprint) + mounted +* One partition with the ID `compute_partition` (defined elsewhere in the + blueprint) +* machine type upgraded from the default `c2-standard-4` to `c2-standard-8` + +### Live Cluster Reconfiguration + +The `schedmd-slurm-gcp-v6-controller` module supports the reconfiguration of +partitions and slurm configuration in a running, active cluster. + +To reconfigure a running cluster: + +1. Edit the blueprint with the desired configuration changes +2. Call `gcluster create -w` to overwrite the deployment directory +3. Follow instructions in terminal to deploy + +The following are examples of updates that can be made to a running cluster: + +* Add or remove a partition to the cluster +* Resize an existing partition +* Attach new network storage to an existing partition + +> **NOTE**: Changing the VM `machine_type` of a partition may not work. +> It is better to create a new partition and delete the old one. + +## Custom Images + +For more information on creating valid custom images for the controller VM +instance or for custom instance templates, see our [vm-images.md] documentation +page. + +[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images + +## GPU Support + +More information on GPU support in Slurm on GCP and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) + +## Reservation for Scheduled Maintenance + +A [maintenance event](https://cloud.google.com/compute/docs/instances/host-maintenance-overview#maintenanceevents) is when a compute engine stops a VM to perform a hardware or +software update which is determined by the host maintenance policy. This can +also affect the running jobs if the maintenance kicks in. Now, Customers can +protect jobs from getting terminated due to maintenance using the cluster +toolkit. You can enable creation of reservation for scheduled maintenance for +your compute nodeset and Slurm will reserve your node for maintenance during the +maintenance window. If you try to schedule any jobs which overlap with the +maintenance reservation, Slurm would not schedule any job. + +You can specify in your blueprint like + +```yaml + - id: compute_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network] + settings: + enable_maintenance_reservation: true +``` + +To enable creation of reservation for maintenance. + +While running job on slurm cluster, you can specify total run time of the job +using [-t flag](https://slurm.schedmd.com/srun.html#OPT_time).This would only +run the job outside of the maintenance window. + +```shell +srun -n1 -pcompute -t 10:00 +``` + +Currently upcoming maintenance notification is supported in ALPHA version of +compute API. You can update the API version from your blueprint, + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + settings: + endpoint_versions: + compute: "alpha" +``` + +## Opportunistic GCP maintenance in Slurm + +Customers can also enable running GCP maintenance as Slurm job opportunistically +to perform early maintenance. If a node is detected for maintenance, Slurm will +create a job to perform maintenance and put it in the job queue. + +If [backfill](https://slurm.schedmd.com/sched_config.html#backfill) scheduler is +used, Slurm will backfill maintenance job if it can find any empty time window. + +Customer can also choose builtin scheduler type. In this case, Slurm would run +maintenance job in strictly priority order. If the maintenance job doesn't kick +in, then forced maintenance will take place at scheduled window. + +Customer can enable this feature at nodeset level by, + +```yaml + - id: debug_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network] + settings: + enable_opportunistic_maintenance: true +``` + +## Placement Max Distance + +When using +[enable_placement](../../../../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md#input_enable_placement) +with Slurm, Google Compute Engine will attempt to place VMs as physically close +together as possible. Capacity constraints at the time of VM creation may still +force VMs to be spread across multiple racks. Google provides the `max-distance` +flag which can used to control the maximum spreading allowed. Read more about +`max-distance` in the +[official docs](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies +). + +You can use the `placement_max_distance` setting on the nodeset module to control the `max-distance` behavior. See the following example: + +```yaml + - id: nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [ network ] + settings: + machine_type: c2-standard-4 + node_count_dynamic_max: 30 + enable_placement: true + placement_max_distance: 1 + +> [!NOTE] +> `schedmd-slurm-gcp-v6-nodeset.settings.enable_placement: true` must also be +> set for placement_max_distance to take effect. + +In the above case using a value of 1 will restrict VM to be placed on the same +rack. You can confirm that the `max-distance` was applied by calling the +following command while jobs are running: + +```shell +gcloud beta compute resource-policies list \ + --format='yaml(name,groupPlacementPolicy.maxDistance)' +``` + +> [!WARNING] +> If a zone lacks capacity, using a lower `max-distance` value (such as 1) is +> more likely to cause VMs creation to fail. + +## TreeWidth and Node Communication + +Slurm uses a fan out mechanism to communicate large groups of nodes. The shape +of this fan out tree is determined by the +[TreeWidth](https://slurm.schedmd.com/slurm.conf.html#OPT_TreeWidth) +configuration variable. + +In the cloud, this fan out mechanism can become unstable when nodes restart with +new IP addresses. You can enforce that all nodes communicate directly with the +controller by setting TreeWidth to a value >= largest partition. + +If the largest partition was 200 nodes, configure the blueprint as follows: + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + ... + settings: + cloud_parameters: + tree_width: 200 +``` + +The default has been set to 128. Values above this have not been fully tested +and may cause congestion on the controller. A more scalable solution is under +way. + +## ResumeRate and Node Resumption + +The `ResumeRate` parameter in `slurm.conf` controls the maximum number of nodes +that Slurm attempts to resume (power up) per minute. This is particularly +important in cloud environments where auto-scaling can lead to a large number of +nodes starting concurrently. + +When many nodes start simultaneously, they can place a heavy load on shared +resources, especially shared filesystems, as they all try to mount filesystems +and access configuration files at the same time. By limiting the `ResumeRate`, +you can stagger the node startup process, reducing the peak load on these shared +resources and improving overall cluster stability during scaling events. + +For example, to limit the node resumption rate to 100 nodes per minute, +configure the blueprint as follows: + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + ... + settings: + cloud_parameters: + resume_rate: 100 +``` + +Adjust this value based on the capabilities of your shared filesystem and the +expected scaling behavior of your cluster. + +## Support +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 6.41 | +| [google-beta](#requirement\_google-beta) | >= 6.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.41 | +| [google-beta](#provider\_google-beta) | >= 6.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [bucket](#module\_bucket) | terraform-google-modules/cloud-storage/google | >= 6.1 | +| [daos\_network\_storage\_scripts](#module\_daos\_network\_storage\_scripts) | ../../../../modules/scripts/startup-script | n/a | +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [login](#module\_login) | ../../internal/slurm-gcp/login | n/a | +| [nodeset\_cleanup](#module\_nodeset\_cleanup) | ./modules/cleanup_compute | n/a | +| [nodeset\_cleanup\_tpu](#module\_nodeset\_cleanup\_tpu) | ./modules/cleanup_tpu | n/a | +| [slurm\_controller\_template](#module\_slurm\_controller\_template) | ../../internal/slurm-gcp/instance_template | n/a | +| [slurm\_files](#module\_slurm\_files) | ./modules/slurm_files | n/a | +| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | +| [slurm\_nodeset\_tpu](#module\_slurm\_nodeset\_tpu) | ../../internal/slurm-gcp/nodeset_tpu | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance_from_template.controller](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_from_template) | resource | +| [google_compute_disk.controller_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_secret_manager_secret.cloudsql](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_iam_member.cloudsql_secret_accessor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_version.cloudsql_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [google_storage_bucket_iam_member.legacy_readers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | +| [google_storage_bucket_iam_member.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | +| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_project.controller_project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = string
    device_name                = string
    disk_type                  = string
    disk_size_gb               = number
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. If not specified, then one will be chosen based on slurm\_cluster\_name. | `string` | `null` | no | +| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket.
Ignored when 'create\_bucket' is true. | `string` | `null` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | +| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Defaults inherited from [Slurm GCP repo](https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/terraform/slurm_cluster/modules/slurm_files/README_TF.md#input_cloud_parameters) | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | +| [cloudsql](#input\_cloudsql) | Use this database instead of the one on the controller.
server\_ip : Address of the database server.
user : The user to access the database as.
password : The password, given the user, to access the given database. (sensitive)
db\_name : The database to access.
user\_managed\_replication : The list of location and (optional) kms\_key\_name for secret | 
object({
    server_ip = string
    user      = string
    password  = string # sensitive
    db_name   = string
    user_managed_replication = optional(list(object({
      location     = string
      kms_key_name = optional(string)
    })), [])
  })
| `null` | no | +| [compute\_startup\_script](#input\_compute\_startup\_script) | DEPRECATED: `compute_startup_script` has been deprecated.
Use `startup_script` of nodeset module instead. | `any` | `null` | no | +| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script in compute nodes. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | +| [controller\_project\_id](#input\_controller\_project\_id) | Optionally. Provision controller and config bucket in the different project | `string` | `null` | no | +| [controller\_startup\_script](#input\_controller\_startup\_script) | Startup script used by the controller VM. | `string` | `"# no-op"` | no | +| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    type = string
    size = number
  })
| 
{
  "size": 50,
  "type": "pd-ssd"
}
| no | +| [create\_bucket](#input\_create\_bucket) | Create GCS bucket instead of using an existing one. | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment. | `string` | n/a | yes | +| [disable\_controller\_public\_ips](#input\_disable\_controller\_public\_ips) | DEPRECATED: Use `enable_controller_public_ips` instead. | `bool` | `null` | no | +| [disable\_default\_mounts](#input\_disable\_default\_mounts) | DEPRECATED: Use `enable_default_mounts` instead. | `bool` | `null` | no | +| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | +| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | `true` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_controller\_public\_ips](#input\_enable\_controller\_public\_ips) | If set to true. The controller will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. | `bool` | `false` | no | +| [enable\_default\_mounts](#input\_enable\_default\_mounts) | Enable default global network storage from the controller
- /home
- /opt/apps | `bool` | `true` | no | +| [enable\_devel](#input\_enable\_devel) | DEPRECATED: `enable_devel` is always on. | `bool` | `null` | no | +| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/tools/prologs-epilogs/README.md | `bool` | `null` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | +| [enable\_slurm\_gcp\_plugins](#input\_enable\_slurm\_gcp\_plugins) | DEPRECATED: Slurm GCP plugins have been deprecated.
Instead of 'max\_hops' plugin please use the 'placement\_max\_distance' nodeset property.
Instead of 'enable\_vpmu' plugin please use 'advanced\_machine\_features.performance\_monitoring\_unit' nodeset property. | `any` | `null` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": "beta"
}
| no | +| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for controller. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | +| [login\_network\_storage](#input\_login\_network\_storage) | An array of network attached storage mounts to be configured on all login nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [login\_nodes](#input\_login\_nodes) | List of slurm login instance definitions. | 
list(object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  }))
| `[]` | no | +| [login\_startup\_script](#input\_login\_startup\_script) | Startup script used by the login VMs. | `string` | `"# no-op"` | no | +| [login\_startup\_scripts\_timeout](#input\_login\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in login\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = optional(map(string))
    mount_runner          = optional(map(string))
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | Define nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | +| [partitions](#input\_partitions) | Cluster partitions as a list. See module slurm\_partition. | 
list(object({
    partition_name        = string
    partition_conf        = optional(map(string), {})
    partition_nodeset     = optional(list(string), [])
    partition_nodeset_dyn = optional(list(string), [])
    partition_nodeset_tpu = optional(list(string), [])
    enable_job_exclusive  = optional(bool, false)
  }))
| `[]` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [region](#input\_region) | The default region to place resources in. | `string` | n/a | yes | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the controller instance. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the controller instance. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming and slurm accounting.
If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). | `string` | `null` | no | +| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | +| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | +| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Post deployment instructions. | +| [slurm\_bucket](#output\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | +| [slurm\_bucket\_dir](#output\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | +| [slurm\_bucket\_name](#output\_slurm\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | +| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | Bucket path used by cluster. | +| [slurm\_cluster\_name](#output\_slurm\_cluster\_name) | Slurm cluster name. | +| [slurm\_controller\_instance](#output\_slurm\_controller\_instance) | Compute instance of controller node | +| [slurm\_login\_instances](#output\_slurm\_login\_instances) | Compute instances of login nodes | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf new file mode 100644 index 0000000000..4a887b99cf --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf @@ -0,0 +1,213 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.disk_resource_manager_tags + } + ] + + state_disk = var.controller_state_disk != null ? [{ + source = google_compute_disk.controller_disk[0].name + device_name = google_compute_disk.controller_disk[0].name + disk_labels = null + auto_delete = false + boot = false + }] : [] + + synth_def_sa_email = "${data.google_project.controller_project.number}-compute@developer.gserviceaccount.com" + + service_account = { + email = coalesce(var.service_account_email, local.synth_def_sa_email) + scopes = var.service_account_scopes + } + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata, + local.universe_domain + ) + + controller_project_id = coalesce(var.controller_project_id, var.project_id) +} + +data "google_project" "controller_project" { + project_id = local.controller_project_id +} + +resource "google_compute_disk" "controller_disk" { + count = var.controller_state_disk != null ? 1 : 0 + + project = local.controller_project_id + name = "${local.slurm_cluster_name}-controller-save" + type = var.controller_state_disk.type + size = var.controller_state_disk.size + zone = var.zone +} + +# INSTANCE TEMPLATE +module "slurm_controller_template" { + source = "../../internal/slurm-gcp/instance_template" + + project_id = local.controller_project_id + region = var.region + slurm_instance_role = "controller" + slurm_cluster_name = local.slurm_cluster_name + labels = local.labels + + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(var.disk_labels, local.labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = concat(local.additional_disks, local.state_disk) + + bandwidth_tier = var.bandwidth_tier + slurm_bucket_path = module.slurm_files.slurm_bucket_path + can_ip_forward = var.can_ip_forward + advanced_machine_features = var.advanced_machine_features + resource_manager_tags = var.resource_manager_tags + + enable_confidential_vm = var.enable_confidential_vm + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + gpu = one(module.gpu.guest_accelerator) + + machine_type = var.machine_type + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + service_account = local.service_account + + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + subnetwork = var.subnetwork_self_link + + tags = concat([local.slurm_cluster_name], var.tags) + # termination_action = TODO: add support for termination_action (?) +} + +# INSTANCE +resource "google_compute_instance_from_template" "controller" { + provider = google-beta + + name = "${local.slurm_cluster_name}-controller" + project = local.controller_project_id + zone = var.zone + source_instance_template = module.slurm_controller_template.self_link + # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 + # we have to explicitly override instance labels instead of inheriting them from template. + labels = module.slurm_controller_template.labels + + allow_stopping_for_update = true + + # Can't rely on template to specify nics due to usage of static_ip + network_interface { + dynamic "access_config" { + for_each = var.enable_controller_public_ips ? ["unit"] : [] + content { + nat_ip = null + network_tier = null + } + } + network_ip = length(var.static_ips) == 0 ? "" : var.static_ips[0] + subnetwork = var.subnetwork_self_link + } + + dynamic "network_interface" { + for_each = var.controller_network_attachment != null ? [1] : [] + content { + network_attachment = var.controller_network_attachment + } + } +} + +moved { + from = module.slurm_controller_instance.google_compute_instance_from_template.slurm_instance[0] + to = google_compute_instance_from_template.controller +} + +# SECRETS: CLOUDSQL +resource "google_secret_manager_secret" "cloudsql" { + count = var.cloudsql != null ? 1 : 0 + + secret_id = "${local.slurm_cluster_name}-slurm-secret-cloudsql" + project = var.project_id + + replication { + dynamic "auto" { + for_each = length(var.cloudsql.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.cloudsql.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = nonsensitive(var.cloudsql.user_managed_replication) + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } + + labels = { + slurm_cluster_name = local.slurm_cluster_name + } +} + +resource "google_secret_manager_secret_version" "cloudsql_version" { + count = var.cloudsql != null ? 1 : 0 + + secret = google_secret_manager_secret.cloudsql[0].id + secret_data = jsonencode(var.cloudsql) +} + +resource "google_secret_manager_secret_iam_member" "cloudsql_secret_accessor" { + count = var.cloudsql != null ? 1 : 0 + + secret_id = google_secret_manager_secret.cloudsql[0].id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${local.service_account.email}" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl new file mode 100644 index 0000000000..219bdc5227 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl @@ -0,0 +1,65 @@ +# slurm.conf +# https://slurm.schedmd.com/high_throughput.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxArraySize=10001 +MaxJobCount=500000 +MaxNodeCount=65536 +MinJobAge=60 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +SlurmctldDebug=error +SlurmdDebug=error + +# +# +# TIMERS +MessageTimeout=60 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=120 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl new file mode 100644 index 0000000000..93ac47e341 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl @@ -0,0 +1,34 @@ +# slurmdbd.conf +# https://slurm.schedmd.com/slurmdbd.conf.html + +DebugLevel=info +PidFile=/var/run/slurm/slurmdbd.pid + +# https://slurm.schedmd.com/slurmdbd.conf.html#OPT_CommitDelay +CommitDelay=1 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +AuthType=auth/{auth_key} +AuthAltTypes=auth/jwt +AuthAltParameters=jwt_key={state_save}/jwt_hs256.key + +DbdHost={control_host} + +LogFile={slurmlog}/slurmdbd.log + +SlurmUser=slurm + +StorageLoc={db_name} + +StorageType=accounting_storage/mysql +StorageHost={db_host} +StoragePort={db_port} +StorageUser={db_user} +StoragePass={db_pass} + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl new file mode 100644 index 0000000000..d3f2615a68 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl @@ -0,0 +1,71 @@ +# slurm.conf +# https://slurm.schedmd.com/slurm.conf.html +# https://slurm.schedmd.com/configurator.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxNodeCount=64000 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +AccountingStoreFlags=job_comment +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/cgroup +SlurmctldDebug=info +SlurmdDebug=info +DebugFlags=Power + +# +# +# TIMERS +MessageTimeout=600 +BatchStartTimeout=600 +PrologEpilogTimeout=600 +PrologFlags=Contain + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=600 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf new file mode 100644 index 0000000000..21e915a125 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf @@ -0,0 +1,50 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +locals { + # TODO: deprecate `var.login_[ startup_script, startup_scripts_timeout, network_storage]` + # in favour of vars defined in user-facing login module + ghpc_startup_login = [{ + filename = "ghpc_startup.sh" + content = var.login_startup_script + }] + + login_startup_scripts = concat(local.common_scripts, local.ghpc_startup_login) +} + +module "login" { + source = "../../internal/slurm-gcp/login" + for_each = { for x in var.login_nodes : x.group_name => x } + + project_id = var.project_id + + slurm_cluster_name = local.slurm_cluster_name + slurm_bucket_path = module.slurm_files.slurm_bucket_path + slurm_bucket_name = module.slurm_files.bucket_name + slurm_bucket_dir = module.slurm_files.bucket_dir + + login_nodes = each.value + + startup_scripts = local.login_startup_scripts + startup_scripts_timeout = var.login_startup_scripts_timeout + + network_storage = var.login_network_storage + + universe_domain = var.universe_domain + + # trigger replacement of login nodes when the controller instance is replaced + # Needed for re-mounting volumes hosted on controller + replace_trigger = google_compute_instance_from_template.controller.self_link +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf new file mode 100644 index 0000000000..7622bdffef --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-controller", ghpc_role = "scheduler" }) +} + +locals { + # Since deployment name may be used to create a cluster name, we remove any invalid character from the beginning + # Also, slurm imposed a lot of restrictions to this name, so we format it to an acceptable string + tmp_cluster_name = substr(replace(lower(var.deployment_name), "/^[^a-z]*|[^a-z0-9]/", ""), 0, 10) + slurm_cluster_name = coalesce(var.slurm_cluster_name, local.tmp_cluster_name) + + universe_domain = { "universe_domain" = var.universe_domain } +} + +# See +# * slurm_files.tf +# * controller.tf +# * partition.tf +# * login.tf diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml new file mode 100644 index 0000000000..7b4918b962 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - iam.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md new file mode 100644 index 0000000000..002bf14145 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md @@ -0,0 +1,42 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name         = string
    subnetwork_self_link = string
    additional_networks = list(object({
      subnetwork = string
    }))
  })
| n/a | yes | +| [nodeset\_template](#input\_nodeset\_template) | Self link of the nodeset template | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf new file mode 100644 index 0000000000..bd8773cf84 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf @@ -0,0 +1,46 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + cleanup_dependencies_agg = flatten([ + var.nodeset.subnetwork_self_link, + var.nodeset.additional_networks[*].subnetwork, + var.nodeset_template]) +} + +# Can not use variadic list in `depends_on`, wrap it into a collection of `null_resource` +resource "null_resource" "dependencies" { + count = length(local.cleanup_dependencies_agg) +} + +resource "null_resource" "script" { + count = var.enable_cleanup_compute ? 1 : 0 + + triggers = { + project_id = var.project_id + cluster_name = var.slurm_cluster_name + nodeset_name = var.nodeset.nodeset_name + universe_domain = var.universe_domain + compute_endpoint_version = var.endpoint_versions.compute + gcloud_path_override = var.gcloud_path_override + } + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/cleanup_compute.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" + when = destroy + } + + # Ensure that clean up is done before attempt to delete the networks + depends_on = [null_resource.dependencies] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh new file mode 100644 index 0000000000..a98243d464 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +project="$1" +cluster_name="$2" +nodeset_name="$3" +universe_domain="$4" +compute_endpoint_version="$5" +gcloud_dir="$6" +MAX_ATTEMPTS=3 + +if [[ $# -ne 5 ]] && [[ $# -ne 6 ]]; then + echo "Usage: $0 []" + exit 1 +fi + +if [[ -n "${gcloud_dir}" ]]; then + export PATH="$gcloud_dir:$PATH" +fi + +export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" +export CLOUDSDK_CORE_PROJECT="${project}" + +if ! type -P gcloud 1>/dev/null; then + echo "gcloud is not available and your compute resources are not being cleaned up" + echo "https://console.cloud.google.com/compute/instances?project=${project}" + exit 1 +fi + +tmpfile=$(mktemp) # have to use a temp file, since `< <(gcloud ...)` doesn't work nicely with `head` +trap 'rm -f "$tmpfile"' EXIT + +echo "Deleting managed instance groups" +mig_filter="name:${cluster_name}-${nodeset_name}-*" +gcloud compute instance-groups managed list --format="value(self_link)" --filter="${mig_filter}" >"$tmpfile" +while batch="$(head -n 5)" && [[ ${#batch} -gt 0 ]]; do + groups=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line + # The lack of quotes around ${groups} is intentional and causes each new space-separated "word" to + # be treated as independent arguments. See PR#2523 + # shellcheck disable=SC2086 + for _ in $( #occasionally MIGs will fail to delete due to some active transformation happening, so let's retry + seq 1 $MAX_ATTEMPTS + ); do + if gcloud compute instance-groups managed delete --quiet ${groups}; then + break + fi + echo "MIG deletion failed, retrying" + done +done <"$tmpfile" +true >"$tmpfile" # Wipe contents of tmp file + +echo "Deleting compute nodes" +node_filter="name:${cluster_name}-${nodeset_name}-* labels.slurm_cluster_name=${cluster_name} AND labels.slurm_instance_role=compute" + +running_nodes_filter="${node_filter} AND status!=STOPPING" +# List all currently running instances and attempt to delete them +gcloud compute instances list --format="value(selfLink)" --filter="${running_nodes_filter}" >"$tmpfile" +# Do 500 instances at a time +while batch="$(head -n 500)" && [[ ${#batch} -gt 0 ]]; do + nodes=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line + # The lack of quotes around ${nodes} is intentional and causes each new space-separated "word" to + # be treated as independent arguments. See PR#2523 + # shellcheck disable=SC2086 + gcloud compute instances delete --quiet ${nodes} || echo "Failed to delete some instances" +done <"$tmpfile" + +# In case if controller tries to delete the nodes as well, +# wait until nodes in STOPPING state are deleted, before deleting the resource policies +stopping_nodes_filter="${node_filter} AND status=STOPPING" +while true; do + node=$(gcloud compute instances list --format="value(name)" --filter="${stopping_nodes_filter}" --limit=1) + if [[ -z "${node}" ]]; then + break + fi + echo "Waiting for instances to be deleted: ${node}" + sleep 5 +done + +echo "Deleting resource policies" +policies_filter="name:${cluster_name}-slurmgcp-managed-${nodeset_name}-*" +gcloud compute resource-policies list --format="value(selfLink)" --filter="${policies_filter}" | while read -r line; do + echo "Deleting resource policy: $line" + gcloud compute resource-policies delete --quiet "${line}" || { + echo "Failed to delete resource policy: $line" + } +done diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf new file mode 100644 index 0000000000..b6da69931c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf @@ -0,0 +1,71 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "Project ID" +} + + +variable "slurm_cluster_name" { + type = string + description = "Name of the Slurm cluster" +} + +variable "enable_cleanup_compute" { + description = < [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | 3.2.3 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf new file mode 100644 index 0000000000..ec86a03a24 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "null_resource" "script" { + count = var.enable_cleanup_compute ? 1 : 0 + + triggers = { + project_id = var.project_id + cluster_name = var.slurm_cluster_name + nodeset_name = var.nodeset.nodeset_name + zone = var.nodeset.zone + universe_domain = var.universe_domain + compute_endpoint_version = var.endpoint_versions.compute + gcloud_path_override = var.gcloud_path_override + } + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/cleanup_tpu.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.zone} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" + when = destroy + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh new file mode 100644 index 0000000000..c724e342c3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +project="$1" +cluster_name="$2" +nodeset_name="$3" +zone="$4" +universe_domain="$5" +compute_endpoint_version="$6" +gcloud_dir="$7" + +if [[ $# -ne 6 ]] && [[ $# -ne 7 ]]; then + echo "Usage: $0 []" + exit 1 +fi + +if [[ -n "${gcloud_dir}" ]]; then + export PATH="$gcloud_dir:$PATH" +fi + +export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" +export CLOUDSDK_CORE_PROJECT="${project}" + +if ! type -P gcloud 1>/dev/null; then + echo "gcloud is not available and your compute resources are not being cleaned up" + echo "https://console.cloud.google.com/compute/instances?project=${project}" + exit 1 +fi + +echo "Deleting TPU nodes" +node_filter="name~${cluster_name}-${nodeset_name}" +running_nodes_filter="${node_filter} AND state!=DELETING" + +# List all currently running nodes and attempt to delete them +gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${running_nodes_filter}" | while read -r name; do + echo "Deleting TPU node: $name" + gcloud compute tpus tpu-vm delete --async --zone="${zone}" --quiet "${name}" || echo "Failed to delete $name" +done + +# Wait until nodes in DELETING state are deleted, before deleting the resource policies +deleting_nodes_filter="${node_filter} AND state=DELETING" +while true; do + node=$(gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${deleting_nodes_filter}" --limit=1) + if [[ -z "${node}" ]]; then + break + fi + echo "Waiting for nodes to be deleted: ${node}" + sleep 5 +done diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf new file mode 100644 index 0000000000..1ac6f64b75 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf @@ -0,0 +1,60 @@ +/** + * Copyright (C) Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "Project ID" +} + +variable "slurm_cluster_name" { + type = string + description = "Name of the Slurm cluster" +} + +variable "enable_cleanup_compute" { + description = < +Copyright (C) SchedMD LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.3 | +| [archive](#requirement\_archive) | ~> 2.0 | +| [google](#requirement\_google) | >= 6.41 | +| [local](#requirement\_local) | ~> 2.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [archive](#provider\_archive) | ~> 2.0 | +| [google](#provider\_google) | >= 6.41 | +| [local](#provider\_local) | ~> 2.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.controller_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.devel](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.devel_compute](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_dyn_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_tpu_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.task_epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.task_prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [random_uuid.cluster_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/uuid) | resource | +| [archive_file.slurm_gcp_devel_compute_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | +| [archive_file.slurm_gcp_devel_controller_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | +| [local_file.chs_gpu_health_check](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.external_epilog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.external_prolog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.setup_external](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. | `string` | `null` | no | +| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket to use. | `string` | n/a | yes | +| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | +| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Default behavior defined in scripts/conf.py | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | +| [cloudsql\_secret](#input\_cloudsql\_secret) | Secret URI to cloudsql secret. | `string` | `null` | no | +| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in compute\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | +| [controller\_startup\_scripts](#input\_controller\_startup\_scripts) | List of scripts to be ran on controller VM startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | +| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    device_name = string
  })
| 
{
  "device_name": null
}
| no | +| [disable\_default\_mounts](#input\_disable\_default\_mounts) | Disable default global network storage from the controller
- /home
- /apps | `bool` | `false` | no | +| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. Not for production use. | `bool` | `false` | no | +| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/v5/tools/prologs-epilogs/README.md | `bool` | `false` | no | +| [enable\_hybrid](#input\_enable\_hybrid) | Enables use of hybrid controller mode. When true, controller\_hybrid\_config will
be used instead of controller\_instance\_config and will disable login instances. | `bool` | `false` | no | +| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": null
}
| no | +| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | +| [google\_app\_cred\_path](#input\_google\_app\_cred\_path) | Path to Google Application Credentials. | `string` | `null` | no | +| [install\_dir](#input\_install\_dir) | Directory where the hybrid configuration directory will be installed on the
on-premise controller (e.g. /etc/slurm/hybrid). This updates the prefix path
for the resume and suspend scripts in the generated `cloud.conf` file.

This variable should be used when the TerraformHost and the SlurmctldHost
are different.

This will default to var.output\_dir if null. | `string` | `null` | no | +| [munge\_mount](#input\_munge\_mount) | Remote munge mount for compute and login nodes to acquire the munge.key.
By default, the munge mount server will be assumed to be the
`var.slurm_control_host` (or `var.slurm_control_addr` if non-null) when
`server_ip=null`. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| 
{
  "fs_type": "nfs",
  "mount_options": "",
  "remote_mount": "/etc/munge/",
  "server_ip": null
}
| no | +| [network\_storage](#input\_network\_storage) | Storage to mounted on all instances.
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | Cluster nodenets, as a list. | `list(any)` | `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Cluster nodenets (dynamic), as a list. | `list(any)` | `[]` | no | +| [nodeset\_startup\_scripts](#input\_nodeset\_startup\_scripts) | List of scripts to be ran on compute VM startup in the specific nodeset. | 
map(list(object({
    filename = string
    content  = string
  })))
| `{}` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Cluster nodenets (TPU), as a list. | `list(any)` | `[]` | no | +| [output\_dir](#input\_output\_dir) | Directory where this module will write its files to. These files include:
cloud.conf; cloud\_gres.conf; config.yaml; resume.py; suspend.py; and util.py. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The GCP project ID. | `string` | n/a | yes | +| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [slurm\_bin\_dir](#input\_slurm\_bin\_dir) | Path to directory of Slurm binary commands (e.g. scontrol, sinfo). If 'null',
then it will be assumed that binaries are in $PATH. | `string` | `null` | no | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | The cluster name, used for resource naming and slurm accounting. | `string` | n/a | yes | +| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | +| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | +| [slurm\_control\_addr](#input\_slurm\_control\_addr) | The IP address or a name by which the address can be identified.

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | +| [slurm\_control\_host](#input\_slurm\_control\_host) | The short, or long, hostname of the machine where Slurm control daemon is
executed (i.e. the name returned by the command "hostname -s").

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | +| [slurm\_control\_host\_port](#input\_slurm\_control\_host\_port) | The port number that the Slurm controller, slurmctld, listens to for work.

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldPort | `string` | `"6818"` | no | +| [slurm\_key\_mount](#input\_slurm\_key\_mount) | Remote mount for compute and login nodes to acquire the slurm.key. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| `null` | no | +| [slurm\_log\_dir](#input\_slurm\_log\_dir) | Directory where Slurm logs to. | `string` | `"/var/log/slurm"` | no | +| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | +| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [bucket\_dir](#output\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | +| [bucket\_name](#output\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | +| [config](#output\_config) | Cluster configuration. | +| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl new file mode 100644 index 0000000000..ffeb167cfc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl @@ -0,0 +1,7 @@ +# cgroup.conf +# https://slurm.schedmd.com/cgroup.conf.html + +ConstrainCores=yes +ConstrainRamSpace=yes +ConstrainSwapSpace=no +ConstrainDevices=yes diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl new file mode 100644 index 0000000000..4951289842 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl @@ -0,0 +1,67 @@ +# slurm.conf +# https://slurm.schedmd.com/slurm.conf.html +# https://slurm.schedmd.com/configurator.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxNodeCount=64000 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +AccountingStoreFlags=job_comment +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/cgroup +SlurmctldDebug=info +SlurmdDebug=info +DebugFlags=Power + +# +# +# TIMERS +MessageTimeout=60 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=120 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl new file mode 100644 index 0000000000..8c90a9dfbe --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl @@ -0,0 +1,31 @@ +# slurmdbd.conf +# https://slurm.schedmd.com/slurmdbd.conf.html + +DebugLevel=info +PidFile=/var/run/slurm/slurmdbd.pid + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +AuthType=auth/{auth_key} +AuthAltTypes=auth/jwt +AuthAltParameters=jwt_key={state_save}/jwt_hs256.key + +DbdHost={control_host} + +LogFile={slurmlog}/slurmdbd.log + +SlurmUser=slurm + +StorageLoc={db_name} + +StorageType=accounting_storage/mysql +StorageHost={db_host} +StoragePort={db_port} +StorageUser={db_user} +StoragePass={db_pass} + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh new file mode 100644 index 0000000000..db514fc9e5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -x /opt/apps/adm/slurm/slurm_epilog ]]; then + exec /opt/apps/adm/slurm/slurm_epilog +fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh new file mode 100644 index 0000000000..37a91bb1ea --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -x /opt/apps/adm/slurm/slurm_prolog ]]; then + exec /opt/apps/adm/slurm/slurm_prolog +fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh new file mode 100644 index 0000000000..0877ff3b19 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SLURM_EXTERNAL_ROOT="/opt/apps/adm/slurm" +SLURM_MUX_FILE="slurm_mux" + +mkdir -p "${SLURM_EXTERNAL_ROOT}" +mkdir -p "${SLURM_EXTERNAL_ROOT}/logs" +mkdir -p "${SLURM_EXTERNAL_ROOT}/etc" + +# create common prolog / epilog "multiplex" script +if [ ! -f "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" ]; then + # indentation matters in EOT below; do not blindly edit! + cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CMD="${0##*/}" +# Locate script +BASE=$(readlink -f $0) +BASE=${BASE%/*} + +export CLUSTER_ADM_BASE=${BASE} + +# Source config file if it exists for extra DEBUG settings +# used below +SLURM_MUX_CONF=${CLUSTER_ADM_BASE}/etc/slurm_mux.conf +if [[ -r ${SLURM_MUX_CONF} ]]; then + source ${SLURM_MUX_CONF} +fi + +# Setup logging if configured and directory exists +LOGFILE="/dev/null" +if [[ -d ${DEBUG_SLURM_MUX_LOG_DIR} && ${DEBUG_SLURM_MUX_ENABLE_LOG} == "yes" ]]; then + LOGFILE="${DEBUG_SLURM_MUX_LOG_DIR}/${CMD}-${SLURM_SCRIPT_CONTEXT}-job-${SLURMD_NODENAME}.log" + exec >>${LOGFILE} 2>&1 +fi + +# Global scriptlets +for SCRIPTLET in ${BASE}/${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do + if [[ -x ${SCRIPTLET} ]]; then + echo "Running ${SCRIPTLET}" + ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 + echo "Running ${SCRIPTLET} returned $?" + fi +done + +# Per partition scriptlets +for SCRIPTLET in ${BASE}/partition-${SLURM_JOB_PARTITION}-${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do + if [[ -x ${SCRIPTLET} ]]; then + echo "Running ${SCRIPTLET}" + ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 + echo "Running ${SCRIPTLET} returned $?" + fi +done +EOT +fi + +# ensure proper permissions on slurm_mux script +chmod 0755 "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" + +# create default slurm_mux configuration file +if [ ! -f "${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" ]; then + cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" +# these settings are intended for temporary debugging purposes only; leaving +# them enabled will write files for each job to a shared NFS directory without +# any automated cleanup +DEBUG_SLURM_MUX_LOG_DIR=/opt/apps/adm/slurm/logs +DEBUG_SLURM_MUX_ENABLE_LOG=no +EOT +fi + +# create epilog symbolic link +if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_epilog" ]; then + cd ${SLURM_EXTERNAL_ROOT} + # delete existing file if necessary + rm -f slurm_epilog + ln -s ${SLURM_MUX_FILE} slurm_epilog + cd - >/dev/null +fi + +# create prolog symbolic link +if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_prolog" ]; then + cd ${SLURM_EXTERNAL_ROOT} + # delete existing file if necessary + rm -f slurm_prolog + ln -s ${SLURM_MUX_FILE} slurm_prolog + cd - >/dev/null +fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf new file mode 100644 index 0000000000..e63b2d1100 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf @@ -0,0 +1,406 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + scripts_dir = abspath("${path.module}/scripts") + + bucket_dir = coalesce(var.bucket_dir, format("%s-files", var.slurm_cluster_name)) +} + +######## +# DATA # +######## + +data "google_storage_bucket" "this" { + name = var.bucket_name +} + +########## +# RANDOM # +########## + +resource "random_uuid" "cluster_id" { +} + +################## +# CLUSTER CONFIG # +################## + +locals { + config = { + enable_bigquery_load = var.enable_bigquery_load + cloudsql_secret = var.cloudsql_secret + cluster_id = random_uuid.cluster_id.result + project = var.project_id + slurm_cluster_name = var.slurm_cluster_name + enable_slurm_auth = var.enable_slurm_auth + bucket_path = local.bucket_path + enable_debug_logging = var.enable_debug_logging + extra_logging_flags = var.extra_logging_flags + controller_state_disk = var.controller_state_disk + + # storage + disable_default_mounts = var.disable_default_mounts + network_storage = var.network_storage + + # timeouts + controller_startup_scripts_timeout = var.controller_startup_scripts_timeout + compute_startup_scripts_timeout = var.compute_startup_scripts_timeout + + munge_mount = local.munge_mount + slurm_key_mount = var.slurm_key_mount + + # slurm conf + prolog_scripts = [for k, v in google_storage_bucket_object.prolog_scripts : k] + epilog_scripts = [for k, v in google_storage_bucket_object.epilog_scripts : k] + task_prolog_scripts = [for k, v in google_storage_bucket_object.task_prolog_scripts : k] + task_epilog_scripts = [for k, v in google_storage_bucket_object.task_epilog_scripts : k] + cloud_parameters = var.cloud_parameters + + # hybrid + hybrid = var.enable_hybrid + google_app_cred_path = var.enable_hybrid ? local.google_app_cred_path : null + output_dir = var.enable_hybrid ? local.output_dir : null + install_dir = var.enable_hybrid ? local.install_dir : null + slurm_control_host = var.enable_hybrid ? var.slurm_control_host : null + slurm_control_host_port = var.enable_hybrid ? local.slurm_control_host_port : null + slurm_control_addr = var.enable_hybrid ? var.slurm_control_addr : null + slurm_bin_dir = var.enable_hybrid ? local.slurm_bin_dir : null + slurm_log_dir = var.enable_hybrid ? local.slurm_log_dir : null + controller_network_attachment = var.controller_network_attachment + + + # config files templates + slurmdbd_conf_tpl = file(coalesce(var.slurmdbd_conf_tpl, "${local.etc_dir}/slurmdbd.conf.tpl")) + slurm_conf_tpl = var.slurm_conf_template != null ? var.slurm_conf_template : file(coalesce(var.slurm_conf_tpl, "${local.etc_dir}/slurm.conf.tpl")) + cgroup_conf_tpl = file(coalesce(var.cgroup_conf_tpl, "${local.etc_dir}/cgroup.conf.tpl")) + + # Providers + endpoint_versions = var.endpoint_versions + } + + x_nodeset = toset(var.nodeset[*].nodeset_name) + x_nodeset_dyn = toset(var.nodeset_dyn[*].nodeset_name) + x_nodeset_tpu = toset(var.nodeset_tpu[*].nodeset.nodeset_name) + x_nodeset_overlap = setintersection([], local.x_nodeset, local.x_nodeset_dyn, local.x_nodeset_tpu) + + etc_dir = abspath("${path.module}/etc") + + bucket_path = format("%s/%s", data.google_storage_bucket.this.url, local.bucket_dir) + + slurm_control_host_port = coalesce(var.slurm_control_host_port, "6818") + + google_app_cred_path = var.google_app_cred_path != null ? abspath(var.google_app_cred_path) : null + slurm_bin_dir = var.slurm_bin_dir != null ? abspath(var.slurm_bin_dir) : null + slurm_log_dir = var.slurm_log_dir != null ? abspath(var.slurm_log_dir) : null + + munge_mount = var.enable_hybrid ? { + server_ip = lookup(var.munge_mount, "server_ip", coalesce(var.slurm_control_addr, var.slurm_control_host)) + remote_mount = lookup(var.munge_mount, "remote_mount", "/etc/munge/") + fs_type = lookup(var.munge_mount, "fs_type", "nfs") + mount_options = lookup(var.munge_mount, "mount_options", "") + } : null + + output_dir = can(coalesce(var.output_dir)) ? abspath(var.output_dir) : abspath(".") + install_dir = can(coalesce(var.install_dir)) ? abspath(var.install_dir) : local.output_dir +} + +resource "google_storage_bucket_object" "config" { + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/config.yaml" + content = yamlencode(local.config) + source_md5hash = md5(yamlencode(local.config)) + + # Take dependency on all other "config artifacts" so creation of `config.yaml` + # can be used as a signal for setup.py that "everything is ready". + # Some of following files, particularly mount scripts for new NFSes, can take a while to be created. + depends_on = [ + google_storage_bucket_object.controller_startup_scripts, + google_storage_bucket_object.nodeset_startup_scripts, + google_storage_bucket_object.prolog_scripts, + google_storage_bucket_object.epilog_scripts, + google_storage_bucket_object.task_prolog_scripts, + google_storage_bucket_object.task_epilog_scripts + ] +} + +resource "google_storage_bucket_object" "nodeset_config" { + for_each = { for ns in var.nodeset : ns.nodeset_name => merge(ns, { + instance_properties = jsondecode(ns.instance_properties_json) + }) } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +resource "google_storage_bucket_object" "nodeset_dyn_config" { + for_each = { for ns in var.nodeset_dyn : ns.nodeset_name => ns } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_dyn_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +resource "google_storage_bucket_object" "nodeset_tpu_config" { + for_each = { for n in var.nodeset_tpu[*].nodeset : n.nodeset_name => n } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_tpu_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +######### +# DEVEL # +######### + +locals { + build_dir = abspath("${path.module}/build") + + slurm_gcp_devel_controller_zip = "slurm-gcp-devel-controller.zip" + slurm_gcp_devel_compute_zip = "slurm-gcp-devel.zip" + slurm_gcp_devel_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_controller_zip) + slurm_gcp_devel_compute_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_compute_zip) + + controller_files = [ + "tools/gpu-test", + "tools/task-epilog", + "tools/task-prolog", + "conf.py", + "file_cache.py", + "get_tpu_vmcount.py", + "job_submit.lua.tpl", + "load_bq.py", + "local_pubsub.py", + "mig_flex.py", + "resume_wrapper.sh", + "resume.py", + "setup_network_storage.py", + "setup.py", + "slurmsync.py", + "sort_nodes.py", + "suspend_wrapper.sh", + "suspend.py", + "tpu.py", + "util.py", + "watch_delete_vm_op.py", + ] + + compute_files = [ + "tools/gpu-test", + "tools/task-epilog", + "tools/task-prolog", + "file_cache.py", + "get_tpu_vmcount.py", + "job_submit.lua.tpl", + "local_pubsub.py", + "mig_flex.py", + "setup_network_storage.py", + "setup.py", + "slurmsync.py", + "sort_nodes.py", + "suspend.py", + "tpu.py", + "util.py", + "watch_delete_vm_op.py", + ] +} + +data "archive_file" "slurm_gcp_devel_controller_zip" { + output_path = "${local.build_dir}/${local.slurm_gcp_devel_controller_zip}" + type = "zip" + + dynamic "source" { + for_each = local.controller_files + content { + content = file("${local.scripts_dir}/${source.value}") + filename = source.value + } + } +} + +data "archive_file" "slurm_gcp_devel_compute_zip" { + output_path = "${local.build_dir}/${local.slurm_gcp_devel_compute_zip}" + type = "zip" + + dynamic "source" { + for_each = local.compute_files + content { + content = file("${local.scripts_dir}/${source.value}") + filename = source.value + } + } +} + +resource "google_storage_bucket_object" "devel" { + bucket = var.bucket_name + name = local.slurm_gcp_devel_zip_bucket + source = data.archive_file.slurm_gcp_devel_controller_zip.output_path + source_md5hash = data.archive_file.slurm_gcp_devel_controller_zip.output_md5 +} + +resource "google_storage_bucket_object" "devel_compute" { + bucket = var.bucket_name + name = local.slurm_gcp_devel_compute_zip_bucket + source = data.archive_file.slurm_gcp_devel_compute_zip.output_path + source_md5hash = data.archive_file.slurm_gcp_devel_compute_zip.output_md5 +} + +########### +# SCRIPTS # +########### + +resource "google_storage_bucket_object" "controller_startup_scripts" { + for_each = { + for x in local.controller_startup_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-controller-script-%s", local.bucket_dir, each.key) + content = each.value.content + source_md5hash = md5(each.value.content) +} + +resource "google_storage_bucket_object" "nodeset_startup_scripts" { + for_each = { for x in flatten([ + for nodeset, scripts in var.nodeset_startup_scripts + : [for s in scripts + : { + content = s.content, + name = format("slurm-nodeset-%s-script-%s", nodeset, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_")) } + ]]) : x.name => x.content } + + bucket = var.bucket_name + name = format("%s/%s", local.bucket_dir, each.key) + content = each.value + source_md5hash = md5(each.value) +} + +resource "google_storage_bucket_object" "prolog_scripts" { + for_each = { + for x in local.prolog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-prolog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +resource "google_storage_bucket_object" "epilog_scripts" { + for_each = { + for x in local.epilog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-epilog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +resource "google_storage_bucket_object" "task_prolog_scripts" { + for_each = { + for x in local.task_prolog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-task_prolog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +resource "google_storage_bucket_object" "task_epilog_scripts" { + for_each = { + for x in local.task_epilog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-task_epilog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +############################ +# DATA: CHS GPU HEALTH CHECK +############################ + +data "local_file" "chs_gpu_health_check" { + filename = "${path.module}/scripts/tools/gpu-test" +} + +################################ +# DATA: EXTERNAL PROLOG/EPILOG # +################################ + +data "local_file" "external_epilog" { + filename = "${path.module}/files/external_epilog.sh" +} + +data "local_file" "external_prolog" { + filename = "${path.module}/files/external_prolog.sh" +} + +data "local_file" "setup_external" { + filename = "${path.module}/files/setup_external.sh" +} + +locals { + external_epilog = [{ + filename = "z_external_epilog.sh" + content = data.local_file.external_epilog.content + source = null + }] + external_prolog = [{ + filename = "z_external_prolog.sh" + content = data.local_file.external_prolog.content + source = null + }] + setup_external = [{ + filename = "z_setup_external.sh" + content = data.local_file.setup_external.content + }] + chs_gpu_health_check = [{ + filename = "a_chs_gpu_health_check.sh" + content = data.local_file.chs_gpu_health_check.content + source = null + }] + + chs_prolog = var.enable_chs_gpu_health_check_prolog ? local.chs_gpu_health_check : [] + ext_prolog = var.enable_external_prolog_epilog ? local.external_prolog : [] + prolog_scripts = concat(local.chs_prolog, local.ext_prolog, var.prolog_scripts) + task_prolog_scripts = var.task_prolog_scripts + + chs_epilog = var.enable_chs_gpu_health_check_epilog ? local.chs_gpu_health_check : [] + ext_epilog = var.enable_external_prolog_epilog ? local.external_epilog : [] + epilog_scripts = concat(local.chs_epilog, local.ext_epilog, var.epilog_scripts) + task_epilog_scripts = var.task_epilog_scripts + + controller_startup_scripts = var.enable_external_prolog_epilog ? concat(local.setup_external, var.controller_startup_scripts) : var.controller_startup_scripts + + +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf new file mode 100644 index 0000000000..111c997d62 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "slurm_bucket_path" { + description = "GCS Bucket URI of Slurm cluster file storage." + value = local.bucket_path +} + +output "bucket_name" { + description = "GCS Bucket name of Slurm cluster file storage." + value = data.google_storage_bucket.this.name +} + +output "bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + value = local.bucket_dir +} + +output "config" { + description = "Cluster configuration." + value = local.config + + precondition { + condition = var.enable_hybrid ? can(coalesce(var.slurm_control_host)) : true + error_message = "Input slurm_control_host is required." + } + + precondition { + condition = length(local.x_nodeset_overlap) == 0 + error_message = "All nodeset names must be unique among all nodeset types." + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py new file mode 100644 index 0000000000..89ceefa3df --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py @@ -0,0 +1,658 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Iterable, Dict, Set, Tuple +from itertools import chain +from collections import defaultdict +import json +from pathlib import Path +import util +from util import dirs, slurmdirs +import tpu +from addict import Dict as NSDict # type: ignore + +FILE_PREAMBLE = """ +# Warning: +# This file is managed by a script. Manual modifications will be overwritten. +""" + + + +def dict_to_conf(conf, delim=" ") -> str: + """convert dict to delimited slurm-style key-value pairs""" + + def filter_conf(pair): + k, v = pair + if isinstance(v, list): + v = ",".join(str(el) for el in v if el is not None) + return k, (v if bool(v) or v == 0 else None) + + return delim.join( + f"{k}={v}" for k, v in map(filter_conf, conf.items()) if v is not None + ) + + +TOPOLOGY_PLUGIN_TREE = "topology/tree" + +def topology_plugin(lkp: util.Lookup) -> str: + """ + Returns configured topology plugin, defaults to `topology/tree`. + """ + cp, key = lkp.cfg.cloud_parameters, "topology_plugin" + if key not in cp or cp[key] is None: + return TOPOLOGY_PLUGIN_TREE + return cp[key] + +def conflines(lkp: util.Lookup) -> str: + params = lkp.cfg.cloud_parameters + def get(key, default): + """ + Returns the value of the key in params if it exists and is not None, + otherwise returns supplied default. + We can't rely on the `dict.get` method because the value could be `None` as + well as empty NSDict, depending on type of the `cfg.cloud_parameters`. + TODO: Simplify once NSDict is removed from the codebase. + """ + if key not in params or params[key] is None: + return default + return params[key] + + no_comma_params = get("no_comma_params", False) + + any_gpus = any( + lkp.template_info(nodeset.instance_template).gpu + for nodeset in lkp.cfg.nodeset.values() + ) + + any_tpu = any( + tpu_nodeset is not None + for part in lkp.cfg.partitions.values() + for tpu_nodeset in part.partition_nodeset_tpu + ) + + any_gke = any( + lkp.nodeset_is_gke(nodeset) + for nodeset in lkp.cfg.nodeset.values() + ) + + any_dynamic = any(bool(p.partition_feature) for p in lkp.cfg.partitions.values()) + comma_params = { + "LaunchParameters": [ + "enable_nss_slurm", + "use_interactive_step", + ], + "SlurmctldParameters": [ + "cloud_reg_addrs" if any_dynamic or any_tpu or any_gke else "cloud_dns", + "enable_configless", + "idle_on_node_suspend", + ], + "GresTypes": [ + "gpu" if any_gpus else None, + ], + } + + scripts_dir = lkp.cfg.install_dir or dirs.scripts + prolog_path = Path(dirs.custom_scripts / "prolog.d") + epilog_path = Path(dirs.custom_scripts / "epilog.d") + task_prolog_path = Path(dirs.custom_scripts / "task_prolog.d") + task_epilog_path = Path(dirs.custom_scripts / "task_epilog.d") + default_tree_width = 65533 if any_dynamic else 128 + + conf_options = { + **(comma_params if not no_comma_params else {}), + "Prolog": f"{prolog_path}/*" if lkp.cfg.prolog_scripts else None, + "Epilog": f"{epilog_path}/*" if lkp.cfg.epilog_scripts else None, + "TaskProlog": f"{task_prolog_path}/task-prolog" if lkp.cfg.task_prolog_scripts else None, + "TaskEpilog": f"{task_epilog_path}/task-epilog" if lkp.cfg.task_epilog_scripts else None, + "PrologFlags": get("prolog_flags", None), + "SwitchType": get("switch_type", None), + "PrivateData": get("private_data", []), + "SchedulerParameters": get("scheduler_parameters", [ + "bf_continue", + "salloc_wait_nodes", + "ignore_prefer_validation", + ]), + "ResumeProgram": f"{scripts_dir}/resume_wrapper.sh", + "ResumeFailProgram": f"{scripts_dir}/suspend_wrapper.sh", + "ResumeRate": get("resume_rate", 0), + "ResumeTimeout": get("resume_timeout", 300), + "SuspendProgram": f"{scripts_dir}/suspend_wrapper.sh", + "SuspendRate": get("suspend_rate", 0), + "SuspendTimeout": get("suspend_timeout", 300), + "SlurmdTimeout": get("slurmd_timeout", 300), + "UnkillableStepTimeout": get("unkillable_step_timeout", 300), + "TreeWidth": get("tree_width", default_tree_width), + "JobSubmitPlugins": "lua" if any_tpu else None, + "TopologyPlugin": topology_plugin(lkp), + "TopologyParam": get("topology_param", "SwitchAsNodeRank"), + } + return dict_to_conf(conf_options, delim="\n") + + + + +def nodeset_lines(nodeset, lkp: util.Lookup) -> str: + template_info = lkp.template_info(nodeset.instance_template) + machine_conf = lkp.template_machine_conf(nodeset.instance_template) + + # follow https://slurm.schedmd.com/slurm.conf.html#OPT_Boards + # by setting Boards, SocketsPerBoard, CoresPerSocket, and ThreadsPerCore + gres = f"gpu:{template_info.gpu.count}" if template_info.gpu else None + node_conf = { + "RealMemory": machine_conf.memory, + "Boards": machine_conf.boards, + "SocketsPerBoard": machine_conf.sockets_per_board, + "CoresPerSocket": machine_conf.cores_per_socket, + "ThreadsPerCore": machine_conf.threads_per_core, + "CPUs": machine_conf.cpus, + "Gres": gres, + **nodeset.node_conf, + } + nodelist = lkp.nodelist(nodeset) + + return "\n".join( + map( + dict_to_conf, + [ + {"NodeName": nodelist, "State": "CLOUD", **node_conf}, + {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, + ], + ) + ) + + +def nodeset_tpu_lines(nodeset, lkp: util.Lookup) -> str: + nodelist = lkp.nodelist(nodeset) + return "\n".join( + map( + dict_to_conf, + [ + {"NodeName": nodelist, "State": "CLOUD", **nodeset.node_conf}, + {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, + ], + ) + ) + + +def nodeset_dyn_lines(nodeset): + """generate slurm NodeSet definition for dynamic nodeset""" + return dict_to_conf( + {"NodeSet": nodeset.nodeset_name, "Feature": nodeset.nodeset_feature} + ) + + +def partitionlines(partition, lkp: util.Lookup) -> str: + """Make a partition line for the slurm.conf""" + MIN_MEM_PER_CPU = 100 + + def defmempercpu(nodeset_name: str) -> int: + nodeset = lkp.cfg.nodeset.get(nodeset_name) + template = nodeset.instance_template + machine = lkp.template_machine_conf(template) + mem_spec_limit = int(nodeset.node_conf.get("MemSpecLimit", 0)) + return max(MIN_MEM_PER_CPU, (machine.memory - mem_spec_limit) // machine.cpus) + + defmem = min( + map(defmempercpu, partition.partition_nodeset), default=MIN_MEM_PER_CPU + ) + + nodesets = list( + chain( + partition.partition_nodeset, + partition.partition_nodeset_dyn, + partition.partition_nodeset_tpu, + ) + ) + + is_tpu = len(partition.partition_nodeset_tpu) > 0 + is_dyn = len(partition.partition_nodeset_dyn) > 0 + + oversub_exlusive = partition.enable_job_exclusive or is_tpu + power_down_on_idle = partition.enable_job_exclusive and not is_dyn + + line_elements = { + "PartitionName": partition.partition_name, + "Nodes": ",".join(nodesets), + "State": "UP", + "DefMemPerCPU": defmem, + "SuspendTime": 300, + "Oversubscribe": "Exclusive" if oversub_exlusive else None, + "PowerDownOnIdle": "YES" if power_down_on_idle else None, + **partition.partition_conf, + } + + return dict_to_conf(line_elements) + + +def suspend_exc_lines(lkp: util.Lookup) -> Iterable[str]: + static_nodelists = [] + for ns in lkp.power_managed_nodesets(): + if ns.node_count_static: + nodelist = lkp.nodelist_range(ns.nodeset_name, 0, ns.node_count_static) + static_nodelists.append(nodelist) + suspend_exc_nodes = {"SuspendExcNodes": static_nodelists} + + dyn_parts = [ + p.partition_name + for p in lkp.cfg.partitions.values() + if len(p.partition_nodeset_dyn) > 0 + ] + suspend_exc_parts = {"SuspendExcParts": [*dyn_parts]} + + return filter( + None, + [ + dict_to_conf(suspend_exc_nodes) if static_nodelists else None, + dict_to_conf(suspend_exc_parts), + ], + ) + + +def make_cloud_conf(lkp: util.Lookup) -> str: + """generate cloud.conf snippet""" + lines = [ + FILE_PREAMBLE, + conflines(lkp), + *(nodeset_lines(n, lkp) for n in lkp.cfg.nodeset.values()), + *(nodeset_dyn_lines(n) for n in lkp.cfg.nodeset_dyn.values()), + *(nodeset_tpu_lines(n, lkp) for n in lkp.cfg.nodeset_tpu.values()), + *(partitionlines(p, lkp) for p in lkp.cfg.partitions.values()), + *(suspend_exc_lines(lkp)), + ] + return "\n\n".join(filter(None, lines)) + + +def gen_cloud_conf(lkp: util.Lookup) -> None: + content = make_cloud_conf(lkp) + + conf_file = lkp.etc_dir / "cloud.conf" + conf_file.write_text(content) + util.chown_slurm(conf_file, mode=0o644) + + +def install_slurm_conf(lkp: util.Lookup) -> None: + """install slurm.conf""" + if lkp.cfg.ompi_version: + mpi_default = "pmi2" + else: + mpi_default = "none" + + conf_options = { + "name": lkp.cfg.slurm_cluster_name, + "control_addr": lkp.control_addr if lkp.control_addr else lkp.hostname_fqdn, + "control_host": lkp.control_host, + "accounting_storage_host": lkp.control_addr if lkp.cfg.controller_network_attachment else lkp.control_host, + "control_host_port": lkp.control_host_port, + "scripts": dirs.scripts, + "slurmlog": dirs.log, + "state_save": slurmdirs.state, + "mpi_default": mpi_default, + "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", + } + + conf = lkp.cfg.slurm_conf_tpl.format(**conf_options) + + conf_file = lkp.etc_dir / "slurm.conf" + conf_file.write_text(conf) + util.chown_slurm(conf_file, mode=0o644) + + +def install_slurmdbd_conf(lkp: util.Lookup) -> None: + """install slurmdbd.conf""" + conf_options = { + "control_host": lkp.control_host, + "slurmlog": dirs.log, + "state_save": slurmdirs.state, + "db_name": "slurm_acct_db", + "db_user": "slurm", + "db_pass": '""', + "db_host": "localhost", + "db_port": "3306", + "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", + } + + if lkp.cfg.cloudsql_secret: + secret_name = f"{lkp.cfg.slurm_cluster_name}-slurm-secret-cloudsql" + payload = json.loads(util.access_secret_version(lkp.project, secret_name)) + + if payload["db_name"] and payload["db_name"] != "": + conf_options["db_name"] = payload["db_name"] + if payload["user"] and payload["user"] != "": + conf_options["db_user"] = payload["user"] + if payload["password"] and payload["password"] != "": + conf_options["db_pass"] = payload["password"] + + db_host_str = payload["server_ip"].split(":") + if db_host_str[0]: + conf_options["db_host"] = db_host_str[0] + conf_options["db_port"] = ( + db_host_str[1] if len(db_host_str) >= 2 else "3306" + ) + + conf = lkp.cfg.slurmdbd_conf_tpl.format(**conf_options) + + conf_file = lkp.etc_dir / "slurmdbd.conf" + conf_file.write_text(conf) + util.chown_slurm(conf_file, 0o600) + + +def install_cgroup_conf(lkp: util.Lookup) -> None: + """install cgroup.conf""" + conf_file = lkp.etc_dir / "cgroup.conf" + conf_file.write_text(lkp.cfg.cgroup_conf_tpl) + util.chown_slurm(conf_file, mode=0o600) + + +def install_jobsubmit_lua(lkp: util.Lookup) -> None: + """install job_submit.lua if there are tpu nodes in the cluster""" + if not any( + tpu_nodeset is not None + for part in lkp.cfg.partitions.values() + for tpu_nodeset in part.partition_nodeset_tpu + ): + return # No TPU partitions, no need for job_submit.lua + + scripts_dir = lkp.cfg.slurm_scripts_dir or dirs.scripts + tpl = (scripts_dir / "job_submit.lua.tpl").read_text() + conf = tpl.format(scripts_dir=scripts_dir) + + conf_file = lkp.etc_dir / "job_submit.lua" + conf_file.write_text(conf) + util.chown_slurm(conf_file, 0o600) + + +def gen_cloud_gres_conf_lines(lkp: util.Lookup) -> str: + """generate cloud_gres.conf's content""" + + gpu_nodes = defaultdict(list) + for nodeset in lkp.cfg.nodeset.values(): + ti = lkp.template_info(nodeset.instance_template) + gpu_count = ti.gpu.count if ti.gpu else 0 + gpu_type = ti.gpu.type if ti.gpu else None + if gpu_count: + gpu_nodes[(gpu_count, gpu_type)].append(lkp.nodelist(nodeset)) + + lines = [ + dict_to_conf( + { + "NodeName": names, + "Name": "gpu", + "Type": gpu_type, + "File": "/dev/nvidia{}".format(f"[0-{gpu_count-1}]" if gpu_count > 1 else "0"), + } + ) + for (gpu_count, gpu_type), names in gpu_nodes.items() + ] + lines.append("\n") + return "\n".join(lines) + + +def gen_cloud_gres_conf(lkp: util.Lookup) -> None: + """create cloud_gres.conf file""" + + content = FILE_PREAMBLE + gen_cloud_gres_conf_lines(lkp) + + conf_file = lkp.etc_dir / "cloud_gres.conf" + conf_file.write_text(content) + util.chown_slurm(conf_file, mode=0o600) + + +def install_gres_conf(lkp: util.Lookup) -> None: + conf_file = lkp.etc_dir / "cloud_gres.conf" + gres_conf = lkp.etc_dir / "gres.conf" + if not gres_conf.exists(): + gres_conf.symlink_to(conf_file) + util.chown_slurm(gres_conf, mode=0o600) + + +class Switch: + """ + Represents a switch in the topology.conf file. + NOTE: It's class user job to make sure that there is no leaf-less Switches in the tree + """ + + def __init__( + self, + name: str, + nodes: Optional[Iterable[str]] = None, + switches: Optional[Dict[str, "Switch"]] = None, + ): + self.name = name + self.nodes = nodes or [] + self.switches = switches or {} + + def conf_line(self) -> str: + d = {"SwitchName": self.name} + if self.nodes: + d["Nodes"] = util.to_hostlist(self.nodes) + if self.switches: + d["Switches"] = util.to_hostlist(self.switches.keys()) + return dict_to_conf(d) + + def render_conf_lines(self) -> Iterable[str]: + yield self.conf_line() + for s in sorted(self.switches.values(), key=lambda s: s.name): + yield from s.render_conf_lines() + +class TopologySummary: + """ + Represents a summary of the topology, to make judgements about changes. + To be stored in JSON file along side of topology.conf to simplify parsing. + """ + def __init__( + self, + physical_host: Optional[Dict[str, str]] = None, + down_nodes: Optional[Iterable[str]] = None, + tpu_nodes: Optional[Iterable[str]] = None, + ) -> None: + self.physical_host = physical_host or {} + self.down_nodes = set(down_nodes or []) + self.tpu_nodes = set(tpu_nodes or []) + + + @classmethod + def path(cls, lkp: util.Lookup) -> Path: + return lkp.etc_dir / "cloud_topology.summary.json" + + @classmethod + def loads(cls, s: str) -> "TopologySummary": + d = json.loads(s) + return cls( + physical_host=d.get("physical_host"), + down_nodes=d.get("down_nodes"), + tpu_nodes=d.get("tpu_nodes"), + ) + + @classmethod + def load(cls, lkp: util.Lookup) -> "TopologySummary": + p = cls.path(lkp) + if not p.exists(): + return cls() # Return empty instance + return cls.loads(p.read_text()) + + def dumps(self) -> str: + return json.dumps( + { + "physical_host": self.physical_host, + "down_nodes": list(self.down_nodes), + "tpu_nodes": list(self.tpu_nodes), + }, + indent=2) + + def dump(self, lkp: util.Lookup) -> None: + TopologySummary.path(lkp).write_text(self.dumps()) + + def _nodenames(self) -> Set[str]: + return set(self.physical_host) | self.down_nodes | self.tpu_nodes + + def requires_reconfigure(self, prev: "TopologySummary") -> bool: + """ + Reconfigure IFF one of the following occurs: + * A node is added + * A node get a non-empty physicalHost + """ + if len(self._nodenames() - prev._nodenames()) > 0: + return True + for n, ph in self.physical_host.items(): + if ph and ph != prev.physical_host.get(n): + return True + return False + +class TopologyBuilder: + def __init__(self) -> None: + self._r = Switch("") # fake root, not part of the tree + self.summary = TopologySummary() + + def add(self, path: List[str], nodes: Iterable[str]) -> None: + n = self._r + assert path + for p in path: + n = n.switches.setdefault(p, Switch(p)) + n.nodes = [*n.nodes, *nodes] + + def render_conf_lines(self) -> Iterable[str]: + if not self._r.switches: + return [] # type: ignore + for s in sorted(self._r.switches.values(), key=lambda s: s.name): + yield from s.render_conf_lines() + + def compress(self) -> "TopologyBuilder": + compressed = TopologyBuilder() + compressed.summary = self.summary + def _walk( + u: Switch, c: Switch + ): # u: uncompressed node, c: its counterpart in compressed tree + pref = f"{c.name}_" if c != compressed._r else "s" + for i, us in enumerate(sorted(u.switches.values(), key=lambda s: s.name)): + cs = Switch(f"{pref}{i}", nodes=us.nodes) + c.switches[cs.name] = cs + _walk(us, cs) + + _walk(self._r, compressed._r) + return compressed + + +def add_tpu_nodeset_topology(nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup): + tpuobj = tpu.TPU.make(nodeset.nodeset_name, lkp) + static, dynamic = lkp.nodenames(nodeset) + + pref = ["tpu-root", f"ns_{nodeset.nodeset_name}"] + if tpuobj.vmcount == 1: # Put all nodes in one switch + all_nodes = list(chain(static, dynamic)) + bldr.add(pref, all_nodes) + bldr.summary.tpu_nodes.update(all_nodes) + return + + # Chunk nodes into sub-switches of size `vmcount` + chunk_num = 0 + for nodenames in (static, dynamic): + for nodeschunk in util.chunked(nodenames, n=tpuobj.vmcount): + chunk_name = f"{nodeset.nodeset_name}-{chunk_num}" + chunk_num += 1 + bldr.add([*pref, chunk_name], nodeschunk) + bldr.summary.tpu_nodes.update(nodeschunk) + +_SLURM_TOPO_ROOT = "slurm-root" + +def _make_physical_path(physical_host: str) -> List[str]: + assert physical_host.startswith("/"), f"Unexpected physicalHost: {physical_host}" + parts = physical_host[1:].split("/") + # Due to issues with Slurm's topology plugin, we can not use all components of `physicalHost`, + # trim it down to `cluster/rack`. + short_path = parts[:2] + return [_SLURM_TOPO_ROOT, *short_path] + +def add_nodeset_topology( + nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup +) -> None: + up_nodes = set() + default_path = [_SLURM_TOPO_ROOT, f"ns_{nodeset.nodeset_name}"] + + for inst in lkp.instances().values(): + try: + if lkp.node_nodeset_name(inst.name) != nodeset.nodeset_name: + continue + except Exception: + continue + + phys_host = inst.resource_status.physical_host or "" + bldr.summary.physical_host[inst.name] = phys_host + up_nodes.add(inst.name) + + if phys_host: + bldr.add(_make_physical_path(phys_host), [inst.name]) + else: + bldr.add(default_path, [inst.name]) + + down_nodes = [] + for node in chain(*lkp.nodenames(nodeset)): + if node not in up_nodes: + down_nodes.append(node) + if down_nodes: + bldr.add(default_path, down_nodes) + bldr.summary.down_nodes.update(down_nodes) + +def gen_topology(lkp: util.Lookup) -> TopologyBuilder: + bldr = TopologyBuilder() + for ns in lkp.cfg.nodeset_tpu.values(): + add_tpu_nodeset_topology(ns, bldr, lkp) + for ns in lkp.cfg.nodeset.values(): + add_nodeset_topology(ns, bldr, lkp) + return bldr + +def gen_topology_conf(lkp: util.Lookup) -> Tuple[bool, TopologySummary]: + """ + Generates slurm topology.conf. + Returns whether the topology.conf got updated. + """ + topo = gen_topology(lkp).compress() + conf_file = lkp.etc_dir / "cloud_topology.conf" + + with open(conf_file, "w") as f: + f.writelines(FILE_PREAMBLE + "\n") + for line in topo.render_conf_lines(): + f.write(line) + f.write("\n") + f.write("\n") + + prev_summary = TopologySummary.load(lkp) + return topo.summary.requires_reconfigure(prev_summary), topo.summary + +def install_topology_conf(lkp: util.Lookup) -> None: + conf_file = lkp.etc_dir / "cloud_topology.conf" + summary_file = lkp.etc_dir / "cloud_topology.summary.json" + topo_conf = lkp.etc_dir / "topology.conf" + + if not topo_conf.exists(): + topo_conf.symlink_to(conf_file) + + util.chown_slurm(conf_file, mode=0o600) + util.chown_slurm(summary_file, mode=0o600) + + +def gen_controller_configs(lkp: util.Lookup) -> None: + install_slurm_conf(lkp) + install_slurmdbd_conf(lkp) + gen_cloud_conf(lkp) + gen_cloud_gres_conf(lkp) + install_gres_conf(lkp) + install_cgroup_conf(lkp) + install_jobsubmit_lua(lkp) + + if topology_plugin(lkp) == TOPOLOGY_PLUGIN_TREE: + _, summary = gen_topology_conf(lkp) + summary.dump(lkp) + install_topology_conf(lkp) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py new file mode 100644 index 0000000000..cd2e41e5af --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py @@ -0,0 +1,80 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any +from pathlib import Path +import shutil +import pickle + +import logging +log = logging.getLogger() + +# Can't reuse tool from util.py to avoid circular dependencies +# TODO: break down util.py for better modularity. +def _chown_slurm(path: Path) -> None: + shutil.chown(path, user="slurm", group="slurm") + +class FileCache: + def __init__(self, path: Path): + self.path = path + + def get(self, key: str) -> Any | None: + p = self.path / key + if not p.exists(): + return None + + try: + with p.open("rb") as f: + return pickle.load(f) + + except Exception as e: + log.warning(f"Failed to read cached value at {p}: {e}") + return None + + def set(self, key: str, data: Any) -> None: + p = self.path / key + + try: + # Create & chown before writing to minimize chances + # of ending up with root-owned corrupted file that can't be cleaned up + # TODO: restrict usage of cache by root to avoid all this complexity + # or have a cache per user. + p.touch(exist_ok=True) + _chown_slurm(p) + with p.open("wb") as f: + pickle.dump(data, f) + + except Exception as e: + log.warning(f"Failed to write cached value at {p}: {e}") + + +class NoCache: + def get(self, key: str) -> Any: + log.warning("No cache used") + return None + + def set(self, key: str, data: Any) -> None: + log.warning("No cache used") + + +def cache(name: str) -> FileCache | NoCache: + try: + path = Path("/tmp/slurm_gcp_cache/") / name + if not path.exists(): + path.mkdir(exist_ok=True, parents=True) + _chown_slurm(path) + return FileCache(path) + except: + log.exception(f"Failed to create cache, fallback to NoCache") + return NoCache() diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py new file mode 100644 index 0000000000..df0fd8ebe0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py @@ -0,0 +1,76 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright 2024 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import util +import tpu + + +def get_vmcount_of_tpu_part(part): + res = 0 + lkp = util.lookup() + for ns in lkp.cfg.partitions[part].partition_nodeset_tpu: + tpu_obj = tpu.TPU.make(ns, lkp) + if res == 0: + res = tpu_obj.vmcount + else: + if res != tpu_obj.vmcount: + # this should not happen, that in the same partition there are different vmcount nodesets + return -1 + return res + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--partitions", + "-p", + help="The partition(s) to retrieve the TPU vmcount value for.", + ) + args = parser.parse_args() + if not args.partitions: + exit(0) + + # useful exit code + # partition does not exists in config.yaml, thus do not exist in slurm + PART_INVALID = -1 + # in the same partition there are nodesets with different vmcounts + DIFF_VMCOUNTS_SAME_PART = -2 + # partition is a list of partitions in which at least two of them have different vmcount + DIFF_PART_DIFFERENT_VMCOUNTS = -3 + vmcounts = [] + # valid equals to 0 means that we are ok, otherwise it will be set to one of the previously defined exit codes + valid = 0 + for part in args.partitions.split(","): + if part not in util.lookup().cfg.partitions: + valid = PART_INVALID + break + else: + if util.lookup().partition_is_tpu(part): + vmcount = get_vmcount_of_tpu_part(part) + if vmcount == -1: + valid = DIFF_VMCOUNTS_SAME_PART + break + vmcounts.append(vmcount) + else: + vmcounts.append(0) + # this means that there are different vmcounts for these partitions + if valid == 0 and len(set(vmcounts)) != 1: + valid = DIFF_PART_DIFFERENT_VMCOUNTS + if valid != 0: + print(f"VMCOUNT:{valid}") + else: + print(f"VMCOUNT:{vmcounts[0]}") diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl new file mode 100644 index 0000000000..810a0742b0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl @@ -0,0 +1,103 @@ +SCRIPTS_DIR = "{scripts_dir}" +NO_VAL = 4294967294 +-- get_tpu_vmcount.py exit code +PART_INVALID = -1 -- partition does not exists in config.yaml, thus do not exist in slurm +DIFF_VMCOUNTS_SAME_PART = -2 -- in the same partition there are nodesets with different vmcounts +DIFF_PART_DIFFERENT_VMCOUNTS = -3 -- partition is a list of partitions in which at least two of them have different vmcount +UNKWOWN_ERROR = -4 -- get_tpu_vmcount.py did not return a valid response + +function get_part(job_desc, part_list) + if job_desc.partition then + return job_desc.partition + end + for name, val in pairs(part_list) do + if val.flag_default == 1 then + return name + end + end + return nil +end + +function os.capture(cmd, raw) + local handle = assert(io.popen(cmd, 'r')) + local output = assert(handle:read('*a')) + handle:close() + return output +end + +function get_vmcount(part) + local cmd = SCRIPTS_DIR .. "/get_tpu_vmcount.py -p " .. part + local out = os.capture(cmd, true) + for line in out:gmatch("(.-)\r?\n") do + local tag, val = line:match("([^:]+):([^:]+)") + if tag == "VMCOUNT" then + return tonumber(val) + end + end + return UNKWOWN_ERROR +end + +function slurm_job_submit(job_desc, part_list, submit_uid) + local part = get_part(job_desc, part_list) + local vmcount = get_vmcount(part) + -- Only do something if the job is in a TPU partition, if vmcount is 0, it implies that the partition(s) specified are not TPU ones + if vmcount == 0 then + return slurm.SUCCESS + end + -- This is a TPU job, but as the vmcount is 1 it can he handled the same way + if vmcount == 1 then + return slurm.SUCCESS + end + -- Check for errors + if vmcount == PART_INVALID then + slurm.log_user("Invalid partition specified " .. part) + return slurm.FAILURE + end + if vmcount == DIFF_VMCOUNTS_SAME_PART then + slurm.log_user("In partition(s) " .. part .. + " there are more than one tpu nodeset vmcount, this should not happen.") + return slurm.ERROR + end + if vmcount == DIFF_PART_DIFFERENT_VMCOUNTS then + slurm.log_user("In partition list " .. part .. + " there are more than one TPU types, cannot determine which is the correct vmcount to use, please retry with only one partition.") + return slurm.FAILURE + end + if vmcount == UNKWOWN_ERROR then + slurm.log_user("Something went wrong while executing get_tpu_vmcount.py.") + return slurm.ERROR + end + -- This is surely a TPU node + if vmcount > 1 then + local min_nodes = job_desc.min_nodes + local max_nodes = job_desc.max_nodes + -- if not specified assume it is one, this should be improved taking into account the cpus, mem, and other factors + if min_nodes == NO_VAL then + min_nodes = 1 + max_nodes = 1 + end + -- as max_nodes can be higher than the nodes in the partition, we are not able to calculate with certainty the nodes that this job will have if this value is set to something + -- different than min_nodes + if min_nodes ~= max_nodes then + slurm.log_user("Max nodes cannot be set different than min nodes for the TPU partitions.") + return slurm.ERROR + end + -- Set the number of switches to the number of nodes originally requested by the job, as the job requests "TPU groups" + job_desc.req_switch = min_nodes + + -- Apply the node increase into the job description. + job_desc.min_nodes = min_nodes * vmcount + job_desc.max_nodes = max_nodes * vmcount + -- if job_desc.features then + -- slurm.log_user("Features: %s",job_desc.features) + -- end + end + + return slurm.SUCCESS +end + +function slurm_job_modify(job_desc, job_rec, part_list, modify_uid) + return slurm.SUCCESS +end + +return slurm.SUCCESS diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py new file mode 100644 index 0000000000..cabd6e3e9f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py @@ -0,0 +1,352 @@ +#!/slurm/python/venv/bin/python3.13 +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Callable, Any +import argparse +import os +import shelve +import uuid +from collections import namedtuple +from datetime import datetime, timedelta, timezone +from pathlib import Path +from pprint import pprint + +import util +from google.api_core import exceptions, retry +from google.cloud import bigquery as bq +from google.cloud.bigquery import SchemaField # type: ignore +from util import lookup, run + +SACCT = "sacct" +script = Path(__file__).resolve() + +DEFAULT_TIMESTAMP_FILE = script.parent / "bq_timestamp" +timestamp_file = Path(os.environ.get("TIMESTAMP_FILE", DEFAULT_TIMESTAMP_FILE)) +# The maximum request to insert_rows is 10MB, each sacct row is about 1200 bytes or ~ 8000 rows. +# Set to 5000 for a little wiggle room. +BQ_ROW_BATCH_SIZE = 5000 + +# cluster_id_file = script.parent / 'cluster_uuid' +# try: +# cluster_id = cluster_id_file.read_text().rstrip() +# except FileNotFoundError: +# cluster_id = uuid.uuid4().hex +# cluster_id_file.write_text(cluster_id) + +job_idx_cache_path = script.parent / "bq_job_idx_cache" + +SLURM_TIME_FORMAT = r"%Y-%m-%dT%H:%M:%S" + + +def make_datetime(time_string): + if time_string == "None": + return None + return datetime.strptime(time_string, SLURM_TIME_FORMAT).replace( + tzinfo=timezone.utc + ) + + +def make_time_interval(seconds): + sign = 1 + if seconds < 0: + sign = -1 + seconds = abs(seconds) + d, r = divmod(seconds, 60 * 60 * 24) + h, r = divmod(r, 60 * 60) + m, s = divmod(r, 60) + d *= sign + h *= sign + return f"{d}D {h:02}:{m:02}:{s}" + + +converters: Dict[str, Callable[[Any], Any]] = { + "DATETIME": make_datetime, + "INTERVAL": make_time_interval, + "STRING": str, + "INT64": lambda n: int(n or 0), +} + + +def schema_field(field_name, data_type, description, required=False): + return SchemaField( + field_name, + data_type, + description=description, + mode="REQUIRED" if required else "NULLABLE", + ) + + +schema_fields = [ + schema_field("cluster_name", "STRING", "cluster name", required=True), + schema_field("cluster_id", "STRING", "UUID for the cluster", required=True), + schema_field("entry_uuid", "STRING", "entry UUID for the job row", required=True), + schema_field( + "job_db_uuid", "STRING", "job db index from the slurm database", required=True + ), + schema_field("job_id_raw", "INT64", "raw job id", required=True), + schema_field("job_id", "STRING", "job id", required=True), + schema_field("state", "STRING", "final job state", required=True), + schema_field("job_name", "STRING", "job name"), + schema_field("partition", "STRING", "job partition"), + schema_field("submit_time", "DATETIME", "job submit time"), + schema_field("start_time", "DATETIME", "job start time"), + schema_field("end_time", "DATETIME", "job end time"), + schema_field("elapsed_raw", "INT64", "STRING", "job run time in seconds"), + # schema_field("elapsed_time", "INTERVAL", "STRING", "job run time interval"), + schema_field("timelimit_raw", "STRING", "job timelimit in minutes"), + schema_field("timelimit", "STRING", "job timelimit"), + # schema_field("num_tasks", "INT64", "number of allocated tasks in job"), + schema_field("nodelist", "STRING", "names of nodes allocated to job"), + schema_field("user", "STRING", "user responsible for job"), + schema_field("uid", "INT64", "uid of job user"), + schema_field("group", "STRING", "group of job user"), + schema_field("gid", "INT64", "gid of job user"), + schema_field("wckey", "STRING", "job wckey"), + schema_field("qos", "STRING", "job qos"), + schema_field("comment", "STRING", "job comment"), + schema_field("admin_comment", "STRING", "job admin comment"), + # extra will be added in 23.02 + # schema_field("extra", "STRING", "job extra field"), + schema_field("exitcode", "STRING", "job exit code"), + schema_field("alloc_cpus", "INT64", "count of allocated CPUs"), + schema_field("alloc_nodes", "INT64", "number of nodes allocated to job"), + schema_field("alloc_tres", "STRING", "allocated trackable resources (TRES)"), + # schema_field("system_cpu", "INTERVAL", "cpu time used by parent processes"), + # schema_field("cpu_time", "INTERVAL", "CPU time used (elapsed * cpu count)"), + schema_field("cpu_time_raw", "INT64", "CPU time used (elapsed * cpu count)"), + # schema_field("ave_cpu", "INT64", "Average CPU time of all tasks in job"), + # schema_field( + # "tres_usage_tot", + # "STRING", + # "Tres total usage by all tasks in job", + # ), +] + + +slurm_field_map = { + "job_db_uuid": "DBIndex", + "job_id_raw": "JobIDRaw", + "job_id": "JobID", + "state": "State", + "job_name": "JobName", + "partition": "Partition", + "submit_time": "Submit", + "start_time": "Start", + "end_time": "End", + "elapsed_raw": "ElapsedRaw", + "elapsed_time": "Elapsed", + "timelimit_raw": "TimelimitRaw", + "timelimit": "Timelimit", + "num_tasks": "NTasks", + "nodelist": "Nodelist", + "user": "User", + "uid": "Uid", + "group": "Group", + "gid": "Gid", + "wckey": "Wckey", + "qos": "Qos", + "comment": "Comment", + "admin_comment": "AdminComment", + # "extra": "Extra", + "exit_code": "ExitCode", + "alloc_cpus": "AllocCPUs", + "alloc_nodes": "AllocNodes", + "alloc_tres": "AllocTres", + "system_cpu": "SystemCPU", + "cpu_time": "CPUTime", + "cpu_time_raw": "CPUTimeRaw", + "ave_cpu": "AveCPU", + "tres_usage_tot": "TresUsageInTot", +} + +# new field name is the key for job_schema. Used to lookup the datatype when +# creating the job rows +job_schema = {field.name: field for field in schema_fields} +# Order is important here, as that is how they are parsed from sacct output +Job = namedtuple("Job", job_schema.keys()) # type: ignore +# ... see https://github.com/python/mypy/issues/848 + +client = bq.Client( + project=lookup().cfg.project, + credentials=util.default_credentials(), + client_options=util.create_client_options(util.ApiEndpoint.BQ), +) +dataset_id = f"{lookup().cfg.slurm_cluster_name}_job_data" +dataset = bq.DatasetReference(project=lookup().project, dataset_id=dataset_id) +table = bq.Table( + bq.TableReference(dataset, f"jobs_{lookup().cfg.slurm_cluster_name}"), schema_fields +) + + +class JobInsertionFailed(Exception): + pass + + +def make_job_row(job): + job_row = { + field_name: converters[field.field_type](job[field_name]) + for field_name, field in job_schema.items() + if field_name in job + } + job_row["entry_uuid"] = uuid.uuid4().hex + job_row["cluster_id"] = lookup().cfg.cluster_id + job_row["cluster_name"] = lookup().cfg.slurm_cluster_name + return job_row + + +def load_slurm_jobs(start, end): + states = ",".join( + ( + "BOOT_FAIL", + "CANCELLED", + "COMPLETED", + "DEADLINE", + "FAILED", + "NODE_FAIL", + "OUT_OF_MEMORY", + "PREEMPTED", + "REQUEUED", + "REVOKED", + "TIMEOUT", + ) + ) + start_iso = start.isoformat(timespec="seconds") + end_iso = end.isoformat(timespec="seconds") + # slurm_fields and bq_fields will be in matching order + slurm_fields = ",".join(slurm_field_map.values()) + bq_fields = slurm_field_map.keys() + cmd = ( + f"{SACCT} --start {start_iso} --end {end_iso} -X -D --format={slurm_fields} " + f"--state={states} --parsable2 --noheader --allusers --duplicates" + ) + text = run(cmd).stdout.splitlines() + # zip pairs bq_fields with the value from sacct + jobs = [dict(zip(bq_fields, line.split("|"))) for line in text] + + # The job index cache allows us to avoid sending duplicate jobs. This avoids a race condition with updating the database. + with shelve.open(str(job_idx_cache_path), flag="r") as job_idx_cache: + job_rows = [ + make_job_row(job) + for job in jobs + if str(job["job_db_uuid"]) not in job_idx_cache + ] + return job_rows + + +def init_table(): + global dataset + global table + dataset = client.create_dataset(dataset, exists_ok=True) # type: ignore + table = client.create_table(table, exists_ok=True) + until_found = retry.Retry(predicate=retry.if_exception_type(exceptions.NotFound)) + table = client.get_table(table, retry=until_found) + # cannot add required fields to an existing schema + table.schema = schema_fields + table = client.update_table(table, ["schema"]) + + +def purge_job_idx_cache(): + purge_time = datetime.now() - timedelta(minutes=30) + with shelve.open(str(job_idx_cache_path), writeback=True) as cache: + to_delete = [] + for idx, stamp in cache.items(): + if stamp < purge_time: + to_delete.append(idx) + for idx in to_delete: + del cache[idx] + + +def bq_submit(jobs): + try: + result = client.insert_rows(table, jobs) + except exceptions.NotFound as e: + print(f"failed to upload job data, table not yet found: {e}") + raise e + except Exception as e: + print(f"failed to upload job data: {e}") + raise e + if result: + pprint(jobs) + pprint(result) + raise JobInsertionFailed("failed to upload job data to big query") + print(f"successfully loaded {len(jobs)} jobs") + + +def get_time_window(): + if not timestamp_file.is_file(): + timestamp_file.touch() + try: + timestamp = datetime.strptime( + timestamp_file.read_text().rstrip(), SLURM_TIME_FORMAT + ) + # time window will overlap the previous by 10 minutes. Duplicates will be filtered out by the job_idx_cache + start = timestamp - timedelta(minutes=10) + except ValueError: + # timestamp 1 is 1 second after the epoch; timestamp 0 is special for sacct + start = datetime.fromtimestamp(1) + # end is now() truncated to the last second + end = datetime.now().replace(microsecond=0) + return start, end + + +def write_timestamp(time): + timestamp_file.write_text(time.isoformat(timespec="seconds")) + + +def update_job_idx_cache(jobs, timestamp): + with shelve.open(str(job_idx_cache_path), writeback=True) as job_idx_cache: + for job in jobs: + job_idx = str(job["job_db_uuid"]) + job_idx_cache[job_idx] = timestamp + + +def main(): + if not lookup().cfg.enable_bigquery_load: + print("bigquery load is not currently enabled") + exit(0) + init_table() + + start, end = get_time_window() + jobs = load_slurm_jobs(start, end) + # on failure, an exception will cause the timestamp not to be rewritten. So + # it will try again next time. If some writes succeed, we don't currently + # have a way to not submit duplicates next time. + if jobs: + num_batches = (len(jobs) - 1) // BQ_ROW_BATCH_SIZE + 1 + print( + f"loading {num_batches} batches of BigQuery data in batches of size : {BQ_ROW_BATCH_SIZE}" + ) + for batch_indx, job_indx in enumerate(range(0, len(jobs), BQ_ROW_BATCH_SIZE)): + print(f"loading BigQuery data batch {batch_indx} of {num_batches}") + bq_submit(jobs[job_indx : job_indx + BQ_ROW_BATCH_SIZE]) + write_timestamp(end) + update_job_idx_cache(jobs, end) + + +parser = argparse.ArgumentParser(description="submit slurm job data to big query") +parser.add_argument( + "timestamp_file", + nargs="?", + action="store", + type=Path, + help="specify timestamp file for reading and writing the time window start. Precedence over TIMESTAMP_FILE env var.", +) + +purge_job_idx_cache() +if __name__ == "__main__": + args = parser.parse_args() + if args.timestamp_file: + timestamp_file = args.timestamp_file.resolve() + main() diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py new file mode 100644 index 0000000000..d4a4477f83 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py @@ -0,0 +1,196 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Implementation of message queue that mimics interface of GCP (PubSub)[https://cloud.google.com/pubsub] + +Messages are stored on controller state disk (to survive controller re-creation) with following layout: + +// +├- +| └- +└- .staging + └- + └- + +One message is one immutable file, that will be deleted after acknowledgement. +NOTE: Implementation assumes that both `` and `.staging/` are on the same disk device, +so it can rely on atomic "move / rename" operation. +""" +from typing import Any +import util +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +import os +import uuid + +import logging +log = logging.getLogger() + + +@dataclass(frozen=True) +class Message: + id: str + created: datetime + data: Any + + def to_json(self) -> dict[str, str]: + return dict( + id=self.id, + created=self.created.isoformat(), + data=self.data) + + @classmethod + def from_json(cls, data: dict[str, str]) -> 'Message': + return cls( + id=data['id'], + created=datetime.fromisoformat(data['created']), + data=data['data']) + +class Topic: + """ + Acts as PubSub topic (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics). + We can have multiple instances of + """ + def __init__(self, path: Path, staging: Path) -> None: + self._path = path + self._staging = staging + + def _gen_id(self, created: datetime) -> str: + ts = created.strftime("%Y_%m_%d-%H_%M_%S") + suf = str(uuid.uuid4())[:8] + return f"{ts}-{suf}" + + def publish(self, data: Any) -> None: + created = util.now() + id = self._gen_id(created) + msg = Message(id=id, created=created, data=data) + + staged = self._staging / msg.id + dst = self._path / msg.id + + # Write to stagin area first then perform atomic move + # to prevent "reads of partial writes" + staged.write_text(json.dumps(msg.to_json())) + util.chown_slurm(staged) + staged.rename(dst) + + +class Subscription: + """ + Acts as PubSub subscription (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) + with following settings: + + ``` + ackDeadlineSeconds = +Inf # don't resend message that was already being delivered but not acked yet + retainAckedMessages = False # don't persist messages that were already acked + enableMessageOrdering = True # delivers messages in chronoligical order + messageRetentionDuration = +Inf # don't expire messages + deadLetterPolicy = None # "deadlettering" is disabled, subscriber should take care of any poisonous messages + retryPolicy = { # NACKed message will be re-delievered after some time + minimumBackoff = 30s # NOTE: Practically there is no timer, but Subscription instance will not try to re-deliver NACKed messages. + maximumBackoff = 30s # Assumes that slurmsync runs every 30+ sec. + } + ``` + + IMPORTANT: Should only be run as part of slurmsync, + this is our way to ensure that at most one instance exists at a time. + There is no concurancy safeguards in place, avoid multithreaded `pull`, + while multithreaded `ack` & `modify_ack_deadline` are OK. + """ + + def __init__(self, path: Path) -> None: + self._path: Path = path + # contains ALL messages pulled by this subscription instance + # both acked, nacked, and still being processed + # used to prevent double delivery within lifetime of subscription (slurmsync) + self._pulled: set[str] = set() + + def _delete(self, id: str) -> None: + log.debug(f"removing {id}") + try: + os.unlink(self._path / id) + except: + log.exception(f"Failed to remove message {id}") + + def _read_msg(self, id: str) -> Message | None: + try: + with open(self._path / id, 'r') as f: + content = json.loads(f.read()) + return Message.from_json(content) + except Exception: + log.exception(f"Failed to read message {id}") + self._delete(id) # delete message to reduce "deadlettering" + return None + + def pull(self, max_messages: int) -> list[Message]: + if not self._path.exists(): + log.warning(f"Topic {self._path} does not exist") + return [] + res = [] + ls = sorted(os.listdir(self._path)) + for name in ls: + msg = self._read_msg(name) + if msg is not None and msg.id not in self._pulled: + self._pulled.add(msg.id) + res.append(msg) + + if len(res) >= max_messages: + break + return res + + + def ack(self, ids: list[str]) -> None: + for id in ids: + self._delete(id) + + + def modify_ack_deadline(self, ids: list[str], deadline: int) -> None: + """ + Modifies the ack deadline for a specific message. + IMPORTANT: Only accepts deadline=0, which is a way to NACK + Any other values are also meaningless due to ackDeadlineSeconds==+Inf + """ + assert deadline == 0 # no op, next subscriber (slurmsync) will pick this up + + +# Topics and Subscriptions are singletons +# TODO: consider making thread-safe +_topics = {} +_subscriptions = {} + +def _make_path(name: str) -> Path: + p = util.slurmdirs.state / "pubsub" / name + p.mkdir(parents=True, exist_ok=True) + util.chown_slurm(p) + return p + +def _make_staging_path(name: str) -> Path: + p = util.slurmdirs.state / "pubsub" / ".staging" / name + p.mkdir(parents=True, exist_ok=True) + util.chown_slurm(p) + return p + +def topic(name: str) -> Topic: + if name not in _topics: + _topics[name] = Topic(_make_path(name), _make_staging_path(name)) + return _topics[name] + +def subscription(name: str) -> Subscription: + if name not in _subscriptions: + _subscriptions[name] = Subscription(_make_path(name)) + return _subscriptions[name] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py new file mode 100644 index 0000000000..8ea3d0657e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py @@ -0,0 +1,254 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional + +import util +import uuid +from addict import Dict as NSDict # type: ignore +from datetime import datetime, timedelta +from collections import defaultdict +import logging +from time import sleep + +log = logging.getLogger() + +DWS_EOL_RESERVATION_DURATION = 10 # minutes + +def _duration(flex_options: NSDict, job_id: Optional[int], lkp: util.Lookup) -> int: + dur = flex_options.max_run_duration + if not job_id or not flex_options.use_job_duration: + return dur + + job = lkp.job(job_id) + if not job or not job.duration: + return dur + + if timedelta(minutes=10) <= job.duration <= timedelta(weeks=1): + return int(job.duration.total_seconds()) + + log.info("Job TimeLimit cannot be less than 10 minutes or exceed one week") + return dur + +def _create_slurm_reservation(node_name: str, boot_time: datetime, run_duration: int, lkp: util.Lookup): + """ + Create a Slurm reservation starting at EOL - buffer time. + """ + eol = boot_time + timedelta(seconds=run_duration) + start_str = eol.strftime("%Y-%m-%dT%H:%M:%S") + reservation_name = f"dws-eol-{node_name}" + log.debug(f"creating slurm reservation for {node_name}") + try: + util.run(f"{lkp.scontrol} create reservation user=slurm starttime={start_str} duration={DWS_EOL_RESERVATION_DURATION} nodes={node_name} reservationname={reservation_name} flags=maint,ignore_jobs") + except Exception as e: + log.error(f"Failed to create reservation for {node_name}: {e}") + +def _delete_slurm_reservation(node_name: str, lkp: util.Lookup): + """ + Delete the Slurm reservation for the given node. + """ + reservation_name = f"dws-eol-{node_name}" + try: + util.run(f"{lkp.scontrol} delete reservation {reservation_name}") + log.debug(f"Deleted Slurm reservation {reservation_name} for {node_name}") + except Exception as e: + log.error(f"Failed to delete reservation for {node_name}: {e}") + +def resume_flex_chunk(nodes: List[str], job_id: Optional[int], lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + assert len(nodeset.zone_policy_allow) > 0 + region = lkp.node_region(model) + + assert nodeset.dws_flex.enabled + + uid = str(uuid.uuid4())[:8] + if job_id: + mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-job-{job_id}-{uid}" + else: + mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-{uid}" + + # Create MIG + req = lkp.compute.regionInstanceGroupManagers().insert( + project=lkp.project, + region=region, + body=dict( + name=mig_name, + versions=[dict(instanceTemplate=nodeset.instance_template)], + targetSize=0, + distributionPolicy=dict( + zones=[ + dict(zone=f"zones/{z}") for z in nodeset.zone_policy_allow + ], + targetShape="ANY_SINGLE_ZONE" ), + updatePolicy = dict(instanceRedistributionType = "NONE" ), + instanceLifecyclePolicy=dict(defaultActionOnFailure= "DO_NOTHING" ), # TODO(FLEX): Not supported yet, migrate once supported + ) + ) + util.log_api_request(req) + op = req.execute() + res = util.wait_for_operation(op) + assert "error" not in res, f"{res}" + + # Create resize request + duration_seconds = _duration(nodeset.dws_flex, job_id, lkp) + req = lkp.compute.regionInstanceGroupManagerResizeRequests().insert( + project=lkp.project, + region=region, + instanceGroupManager=mig_name, + body=dict( + name="initial-resize", + instances=[dict(name=n) for n in nodes], + requested_run_duration=dict( + seconds=duration_seconds + ) + ) + ) + util.log_api_request(req) + op = req.execute() + res = util.wait_for_operation(op) + + # Create Slurm reservations if use_job_duration is set + if nodeset.dws_flex.use_job_duration: + # Get run duration (seconds) + run_duration = duration_seconds + for node_name in nodes: + # Fetch instance creation time from GCP instance (via util.py) + instance = lkp.instance(node_name) + if(instance and instance.creation_timestamp): + log.debug("creating with creation_timestamp") + boot_time = instance.creation_timestamp # Already a datetime object + else: + boot_time = datetime.utcnow() + log.debug("creating with utcnow time: {boot_time}") + _create_slurm_reservation(node_name, boot_time, run_duration, lkp) + + assert "error" not in res, f"{res}" + +def _suspend_flex_mig(mig_self_link: str, nodes: List[str], lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + assert len(nodeset.zone_policy_allow) > 0 + region = lkp.node_region(model) + project=lkp.project + instanceGroupManager=util.trim_self_link(mig_self_link) + + links = [ + f"zones/{inst.zone}/instances/{inst.name}" + for inst in [ + lkp.instance(node) for node in nodes + ] if inst + ] + + target_mig=lkp.get_mig(lkp.project, region, instanceGroupManager) + assert target_mig + + # TODO(FLEX): This will not work if MIG didn't obtain capacity yet. + # The request will fail and MIG will continue provisioning. + # Instead whole MIG should be deleted. + # + All other instances in MIG are not provisioned also, safe to delete + # - Need to come up will clear test to differentiate non-provisioned MIG and single VM being down; + # Particularly CRITICAL due to ActionOnFailure=DO_NOTHING + # - Need to `down_nodes_notify_jobs` for all nodes in MIG, make sure that it doesn't interfere with Slurm suspend-flow. + + if target_mig["targetSize"] == len(nodes): #We can just delete the whole MIG in this case + req = lkp.compute.regionInstanceGroupManagers().delete( + project=project, + region=region, + instanceGroupManager=instanceGroupManager, + ) + else: + req = lkp.compute.regionInstanceGroupManagers().deleteInstances( + project=project, + region=region, + instanceGroupManager=instanceGroupManager, + body=dict( + instances=links, + skipInstancesOnValidationError=True, + ) + ) + + util.log_api_request(req) + op = req.execute() + + res = util.wait_for_operation(op) + + # Delete Slurm reservations for nodes being deprovisioned + for node_name in nodes: + log.info("delete dws reservation") + _delete_slurm_reservation(node_name, lkp) + + assert "error" not in res, f"{res}" + +def _suspend_provisioning_inst(nodes:List[str], node_template:str, lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + assert len(nodeset.zone_policy_allow) > 0 + region = lkp.node_region(model) + + mig_list=lkp.get_mig_list(lkp.project, region) + + # FLEX (#TODO): If we enter this conditional it's likely this was called so early that MIG creation hasn't started + # Consider potentially retrying? No natural mechanism for retry currently but we could + # perhaps use slurmsync and then try it again to ensure it wasn't a case of being too early. + # This is important since we're now enabling long ResumeTimeout (Slurm won't call suspend on node within reasonable timeframe) + # so until we do this is slurmsync this is a temporary workaround. + + if not mig_list or not mig_list.get("items"): + log.info("No matching MIG found to delete! Retrying...") + sleep(5) + mig_list=lkp.get_mig_list(lkp.project, region) + if not mig_list or not mig_list.get("items"): + return + + for mig in mig_list["items"]: + if mig["instanceTemplate"] == node_template: + if mig["currentActions"]["creating"] > 0 and mig["targetSize"] == mig["currentActions"]["creating"]: + req = lkp.compute.regionInstanceGroupManagers().delete( + project=lkp.project, + region=region, + instanceGroupManager=util.trim_self_link(mig["selfLink"]), + ) + + util.log_api_request(req) + op = req.execute() + + res = util.wait_for_operation(op) + assert "error" not in res, f"{res}" + return + + log.info("No matching MIG found to delete!") + +def suspend_flex_nodes(nodes: List[str], lkp: util.Lookup) -> None: + by_mig = defaultdict(list) + not_provisioned = defaultdict(list) + for node in nodes: + inst = lkp.instance(node) + if not inst: + not_provisioned[lkp.node_template(node)].append(node) + else: + mig = inst.metadata.get("created-by") + if not mig: + log.error(f"Can not suspend {node}, can not find associated MIG") + continue + by_mig[mig].append(node) + + for mig, nodes in by_mig.items(): + _suspend_flex_mig(mig, nodes, lkp) + + for node_template, nodes in not_provisioned.items(): + _suspend_provisioning_inst(nodes, node_template, lkp) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt new file mode 100644 index 0000000000..2ab3162ccf --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt @@ -0,0 +1,9 @@ +pytest +pytest-mock +pytest_unordered +mock + +types-mock +types-httplib2 +types-requests +types-PyYAML diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt new file mode 100644 index 0000000000..e923e53dbf --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt @@ -0,0 +1,18 @@ +addict==2.4.0 +google-api-core==2.19.0 +google-api-python-client==2.93.0 +google-auth==2.40.3 +google-auth-httplib2==0.1.0 +google-cloud-bigquery==3.11.3 +google-cloud-core==2.3.3 +google-cloud-secret-manager~=2.22 +google-cloud-storage==2.10.0 +google-cloud-tpu==1.10.0 +google-resumable-media==2.5.0 +googleapis-common-protos==1.59.1 +grpcio==1.60.0 +grpcio-status==1.60.0 +httplib2==0.22.0 +more-executors==2.11.4 +pyyaml==6.0.2 +requests==2.32.4 diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py new file mode 100644 index 0000000000..ea0012a0b1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py @@ -0,0 +1,703 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Dict, Any +import argparse +from datetime import timedelta +import shlex +import json +import logging +import os +import yaml +import collections +from pathlib import Path +from dataclasses import dataclass +from addict import Dict as NSDict # type: ignore + +import util +from util import ( + chunked, + ensure_execute, + execute_with_futures, + log_api_request, + map_with_futures, + run, + separate, + to_hostlist, + trim_self_link, + wait_for_operation, +) +from util import lookup, ReservationDetails +import tpu +import mig_flex + +log = logging.getLogger() + +PLACEMENT_MAX_CNT = 1500 +# Placement group needs to be the same for an entire bulk_insert hence +# if placement is used the actual BULK_INSERT_LIMIT will be +# max([1000, PLACEMENT_MAX_CNT]) +BULK_INSERT_LIMIT = 5000 + +# https://cloud.google.com/compute/docs/instance-groups#types_of_managed_instance_groups +ZONAL_MIG_SIZE_LIMIT = 1000 + + +@dataclass(frozen=True) +class ResumeJobData: + job_id: int + partition: str + nodes_alloc: List[str] + +@dataclass(frozen=True) +class ResumeData: + jobs: List[ResumeJobData] + + +def get_resume_file_data() -> Optional[ResumeData]: + if not (path := os.getenv("SLURM_RESUME_FILE")): + log.error("SLURM_RESUME_FILE was not in environment. Cannot get detailed job, node, partition allocation data.") + return None + blob = Path(path).read_text() + log.debug(f"Resume data: {blob}") + data = json.loads(blob) + + jobs = [] + for jo in data.get("jobs", []): + job = ResumeJobData( + job_id = jo.get("job_id"), + partition = jo.get("partition"), + nodes_alloc = util.to_hostnames(jo.get("nodes_alloc")), + ) + jobs.append(job) + return ResumeData(jobs=jobs) + +def instance_properties(nodeset: NSDict, model:str, placement_group:Optional[str], labels:Optional[dict], job_id:Optional[int]): + props = NSDict() + + if labels: # merge in extra labels on instance and disks + template_link = lookup().node_template(model) + template_info = lookup().template_info(template_link) + + props.labels = {**template_info.labels, **labels} + + for disk in template_info.disks: + if disk.initializeParams.get("diskType", "local-ssd") == "local-ssd": + continue # do not label local ssd + disk.initializeParams.labels.update(labels) + props.disks = template_info.disks + + if placement_group: + props.resourcePolicies = [placement_group] + + if reservation := lookup().nodeset_reservation(nodeset): + update_reservation_props(reservation, props, placement_group, reservation.calendar) + + if (fr := lookup().future_reservation(nodeset)) and fr.specific: + assert fr.active_reservation + update_reservation_props(fr.active_reservation, props, placement_group, fr.calendar) + + if props.resourcePolicies: + props.scheduling.onHostMaintenance = "TERMINATE" + + if nodeset.maintenance_interval: + props.scheduling.maintenanceInterval = nodeset.maintenance_interval + + if nodeset.dws_flex.enabled and nodeset.dws_flex.use_bulk_insert: + update_props_dws(props, nodeset.dws_flex, job_id) + + # Override with properties explicit specified in the nodeset + props.update(nodeset.get("instance_properties") or {}) + return props + +def update_reservation_props(reservation:ReservationDetails, props:NSDict, placement_group:Optional[str], calendar_mode:bool) -> None: + props.reservationAffinity = { + "consumeReservationType": "SPECIFIC_RESERVATION", + "key": f"compute.{util.universe_domain()}/reservation-name", + "values": [reservation.bulk_insert_name], + } + + if reservation.dense or calendar_mode: + props.scheduling.provisioningModel = "RESERVATION_BOUND" + + # Figure out `resourcePolicies` + if reservation.policies: # use ones already attached to reservations + props.resourcePolicies = reservation.policies + elif reservation.dense and placement_group: # use once created by Slurm + props.resourcePolicies = [placement_group] + else: # vanilla reservations don't support external policies + props.resourcePolicies = [] + log.info( + f"reservation {reservation.bulk_insert_name} is being used with resourcePolicies: {props.resourcePolicies}") + +def update_props_dws(props: NSDict, dws_flex: NSDict, job_id: Optional[int]) -> None: + props.scheduling.onHostMaintenance = "TERMINATE" + props.scheduling.instanceTerminationAction = "DELETE" + props.reservationAffinity['consumeReservationType'] = "NO_RESERVATION" + props.scheduling.maxRunDuration['seconds'] = dws_flex_duration(dws_flex, job_id) + +def dws_flex_duration(dws_flex: NSDict, job_id: Optional[int]) -> int: + max_duration = dws_flex.max_run_duration + if dws_flex.use_job_duration and job_id is not None and (job := lookup().job(job_id)) and job.duration: + if timedelta(seconds=30) <= job.duration <= timedelta(weeks=1): + max_duration = int(job.duration.total_seconds()) + else: + log.info("Job TimeLimit cannot be less than 30 seconds or exceed one week") + return max_duration + +def create_instances_request(nodes: List[str], placement_group: Optional[str], excl_job_id: Optional[int]): + """Call regionInstances.bulkInsert to create instances""" + assert 0 < len(nodes) <= BULK_INSERT_LIMIT + + # model here indicates any node that can be used to describe the rest + model = next(iter(nodes)) + log.debug(f"create_instances_request: {model} placement: {placement_group}") + + nodeset = lookup().node_nodeset(model) + template = lookup().node_template(model) + labels = {"slurm_job_id": excl_job_id} if excl_job_id else None + + body = dict( + count = len(nodes), + sourceInstanceTemplate = template, + # key is instance name, value overwrites properties (no overwrites) + perInstanceProperties = {k: {} for k in nodes}, + instanceProperties = instance_properties( + nodeset, model, placement_group, labels, excl_job_id + ), + ) + + if placement_group and excl_job_id is not None: + pass # do not set minCount to force "all or nothing" behavior + else: + body["minCount"] = 1 + + zone_allow = nodeset.zone_policy_allow or [] + zone_deny = nodeset.zone_policy_deny or [] + + if len(zone_allow) == 1: # if only one zone is used, use zonal BulkInsert API, as less prone to errors + api_method = lookup().compute.instances().bulkInsert + method_args = {"zone": zone_allow[0]} + else: + api_method = lookup().compute.regionInstances().bulkInsert + method_args = {"region": lookup().node_region(model)} + + body["locationPolicy"] = dict( + locations = { + **{ f"zones/{z}": {"preference": "ALLOW"} for z in zone_allow }, + **{ f"zones/{z}": {"preference": "DENY"} for z in zone_deny }}, + targetShape = nodeset.zone_target_shape, + ) + + req = api_method( + project=lookup().project, + body=body, + **method_args) + log.debug(f"new request: endpoint={req.methodId} nodes={to_hostlist(nodes)}") + log_api_request(req) + return req + +@dataclass() +class PlacementAndNodes: + placement: Optional[str] + nodes: List[str] + +@dataclass(frozen=True) +class BulkChunk: + nodes: List[str] + prefix: str # - + chunk_idx: int + excl_job_id: Optional[int] + placement_group: Optional[str] = None + + @property + def name(self): + if self.placement_group is not None: + return f"{self.prefix}:job{self.excl_job_id}:{self.placement_group}:{self.chunk_idx}" + if self.excl_job_id is not None: + return f"{self.prefix}:job{self.excl_job_id}:{self.chunk_idx}" + return f"{self.prefix}:{self.chunk_idx}" + + +def group_nodes_bulk(nodes: List[str], resume_data: Optional[ResumeData], lkp: util.Lookup): + """group nodes by nodeset, placement_group, exclusive_job_id if any""" + if resume_data is None: # all nodes will be considered jobless + resume_data = ResumeData(jobs=[]) + + nodes_set = set(nodes) # turn into set to simplify intersection + non_excl = nodes_set.copy() + groups : Dict[Optional[int], List[PlacementAndNodes]] = {} # excl_job_id|none -> PlacementAndNodes + + # expand all exclusive job nodelists + for job in resume_data.jobs: + if not lkp.cfg.partitions[job.partition].enable_job_exclusive: + continue + + groups[job.job_id] = [] + # placement group assignment is based on all allocated nodes, ... + for pn in create_placements(job.nodes_alloc, job.job_id, lkp): + groups[job.job_id].append( + PlacementAndNodes( + placement=pn.placement, + #... but we only want to handle nodes in nodes_resume in this run. + nodes = sorted(set(pn.nodes) & nodes_set) + )) + non_excl.difference_update(job.nodes_alloc) + + groups[None] = create_placements(sorted(non_excl), excl_job_id=None, lkp=lkp) + + def chunk_nodes(nodes: List[str]): + if not nodes: + return [] + + model = nodes[0] + + if lkp.is_flex_node(model): + chunk_size = ZONAL_MIG_SIZE_LIMIT + elif lkp.node_is_tpu(model): + ns_name = lkp.node_nodeset_name(model) + chunk_size = tpu.TPU.make(ns_name, lkp).vmcount + else: + chunk_size = BULK_INSERT_LIMIT + + return chunked(nodes, n=chunk_size) + + chunks = [ + BulkChunk( + nodes=nodes_chunk, + prefix=lkp.node_prefix(nodes_chunk[0]), # - + excl_job_id = job_id, + placement_group=pn.placement, + chunk_idx=i) + + for job_id, placements in groups.items() + for pn in placements if pn.nodes + for i, nodes_chunk in enumerate(chunk_nodes(pn.nodes)) + ] + return {chunk.name: chunk for chunk in chunks} + + +def resume_nodes(nodes: List[str], resume_data: Optional[ResumeData]): + """resume nodes in nodelist""" + lkp = lookup() + # Prevent dormant nodes associated with a reservation from being resumed + nodes, dormant_res_nodes = util.separate(lkp.is_dormant_res_node, nodes) + + if dormant_res_nodes: + log.warning(f"Resume was unable to resume reservation nodes={dormant_res_nodes}") + down_nodes_notify_jobs(dormant_res_nodes, "Reservation is not active, nodes cannot be resumed", resume_data) + + nodes, flex_managed = util.separate(lkp.is_provisioning_flex_node, nodes) + if flex_managed: + log.warning(f"Resume was unable to resume nodes={flex_managed} already managed by MIGs") + down_nodes_notify_jobs(flex_managed, "VM is managed MIG, can not be resumed", resume_data) + + if not nodes: + log.info("No nodes to resume") + return + + nodes = sorted(nodes, key=lkp.node_prefix) + grouped_nodes = group_nodes_bulk(nodes, resume_data, lkp) + + if log.isEnabledFor(logging.DEBUG): + grouped_nodelists = { + group: to_hostlist(chunk.nodes) for group, chunk in grouped_nodes.items() + } + log.debug( + "node bulk groups: \n{}".format(yaml.safe_dump(grouped_nodelists).rstrip()) + ) + + tpu_chunks, flex_chunks = [], [] + bi_inserts = {} + + for group, chunk in grouped_nodes.items(): + model = chunk.nodes[0] + + if lkp.node_is_tpu(model): + tpu_chunks.append(chunk.nodes) + elif lkp.is_flex_node(model): + flex_chunks.append(chunk) + else: + bi_inserts[group] = create_instances_request( + chunk.nodes, chunk.placement_group, chunk.excl_job_id + ) + + for chunk in flex_chunks: + mig_flex.resume_flex_chunk(chunk.nodes, chunk.excl_job_id, lkp) + + # execute all bulkInsert requests with batch + bulk_ops = dict( + zip(bi_inserts.keys(), map_with_futures(ensure_execute, bi_inserts.values())) + ) + log.debug(f"bulk_ops={yaml.safe_dump(bulk_ops)}") + started = { + group: op for group, op in bulk_ops.items() if not isinstance(op, Exception) + } + failed = { + group: err for group, err in bulk_ops.items() if isinstance(err, Exception) + } + if failed: + failed_reqs = [str(e) for e in failed.items()] + log.error("bulkInsert API failures: {}".format("; ".join(failed_reqs))) + for ident, exc in failed.items(): + down_nodes_notify_jobs(grouped_nodes[ident].nodes, f"GCP Error: {exc._get_reason()}", resume_data) # type: ignore + + if log.isEnabledFor(logging.DEBUG): + for group, op in started.items(): + group_nodes = grouped_nodelists[group] + name = op["name"] + gid = op["operationGroupId"] + log.debug( + f"new bulkInsert operation started: group={group} nodes={group_nodes} name={name} operationGroupId={gid}" + ) + # wait for all bulkInserts to complete and log any errors + bulk_operations = {group: wait_for_operation(op) for group, op in started.items()} + + # Start TPU after regular nodes so that regular nodes are not affected by the slower TPU nodes + execute_with_futures(tpu.start_tpu, tpu_chunks) + + for group, op in bulk_operations.items(): + _handle_bulk_insert_op(op, grouped_nodes[group].nodes, resume_data) + + +def _get_failed_zonal_instance_inserts(bulk_op: Any, zone: str, lkp: util.Lookup) -> list[Any]: + group_id = bulk_op["operationGroupId"] + user = bulk_op["user"] + started = bulk_op["startTime"] + ended = bulk_op["endTime"] + + fltr = f'(user eq "{user}") AND (operationType eq "insert") AND (creationTimestamp > "{started}") AND (creationTimestamp < "{ended}")' + act = lkp.compute.zoneOperations() + req = act.list(project=lkp.project, zone=zone, filter=fltr) + ops = [] + while req is not None: + result = util.ensure_execute(req) + for op in result.get("items", []): + if op.get("operationGroupId") == group_id and "error" in op: + ops.append(op) + req = act.list_next(req, result) + return ops + + +def _get_failed_instance_inserts(bulk_op: Any, lkp: util.Lookup) -> list[Any]: + zones = set() # gather zones that had failed inserts + for loc, stat in bulk_op.get("instancesBulkInsertOperationMetadata", {}).get("perLocationStatus", {}).items(): + pref, zone = loc.split("/", 1) + if not pref == "zones": + log.error(f"Unexpected location: {loc} in operation {bulk_op['name']}") + continue + if stat.get("targetVmCount", 0) != stat.get("createdVmCount", 0): + zones.add(zone) + + res = [] + for zone in zones: + res.extend(_get_failed_zonal_instance_inserts(bulk_op, zone, lkp)) + return res + +def _handle_bulk_insert_op(op: Dict, nodes: List[str], resume_data: Optional[ResumeData]) -> None: + """ + Handles **DONE** BulkInsert operations + """ + assert op["operationType"] == "bulkInsert" and op["status"] == "DONE", f"unexpected op: {op}" + + group_id = op["operationGroupId"] + if "error" in op: + error = op["error"]["errors"][0] + log.error( + f"bulkInsert operation error: {error['code']} name={op['name']} operationGroupId={group_id} nodes={to_hostlist(nodes)}" + ) + + created = 0 + for status in op["instancesBulkInsertOperationMetadata"]["perLocationStatus"].values(): + created += status.get("createdVmCount", 0) + if created == len(nodes): + log.info(f"created {len(nodes)} instances: nodes={to_hostlist(nodes)}") + return # no need to gather status of insert-operations. + + # TODO: don't gather insert-operations per bulkInsert request, instead aggregate it + # across all bulkInserts (goes one level above this function) + failed = _get_failed_instance_inserts(op, util.lookup()) + + # Multiple errors are possible, group by all of them (joined string codes) + by_error_inserts = util.groupby_unsorted( + failed, + lambda op: "+".join(err["code"] for err in op["error"]["errors"]), + ) + for code, failed_ops in by_error_inserts: + failed_ops = list(failed_ops) + failed_nodes = [trim_self_link(op["targetLink"]) for op in failed_ops] + hostlist = util.to_hostlist(failed_nodes) + log.error( + f"{len(failed_nodes)} instances failed to start: {code} ({hostlist}) operationGroupId={group_id}" + ) + + msg = "; ".join( + f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" + for err in failed_ops[0]["error"]["errors"] + ) + if code != "RESOURCE_ALREADY_EXISTS": + down_nodes_notify_jobs(failed_nodes, f"GCP Error: {msg}", resume_data) + log.error( + f"errors from insert for node '{failed_nodes[0]}' ({failed_ops[0]['name']}): {msg}" + ) + + +def down_nodes_notify_jobs(nodes: List[str], reason: str, resume_data: Optional[ResumeData]) -> None: + """set nodes down with reason""" + nodes_set = set(nodes) # turn into set to speed up intersection + jobs = resume_data.jobs if resume_data else [] + reason_quoted = shlex.quote(reason) + + for job in jobs: + if not (set(job.nodes_alloc) & nodes_set): + continue + run(f"{lookup().scontrol} update jobid={job.job_id} admincomment={reason_quoted}", check=False) + run(f"{lookup().scontrol} notify {job.job_id} {reason_quoted}", check=False) + + nodelist = util.to_hostlist(nodes) + log.error(f"Marking nodes {nodelist} as DOWN, reason: {reason}") + run(f"{lookup().scontrol} update nodename={nodelist} state=down reason={reason_quoted}", check=False) + + + + +def create_placement_request(pg_name: str, region: str, max_distance: Optional[int], accelerator_topology: Optional[str]): + config = { + "name": pg_name, + "region": region, + "groupPlacementPolicy": { + "collocation": "COLLOCATED", + "maxDistance": max_distance, + "gpuTopology": accelerator_topology, + }, + } + + request = lookup().compute.resourcePolicies().insert( + project=lookup().project, region=region, body=config + ) + log_api_request(request) + return request + + +def create_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + nodeset_map = collections.defaultdict(list) + for node in nodes: # split nodes on nodesets + nodeset_map[lkp.node_nodeset_name(node)].append(node) + + placements = [] + for _, ns_nodes in nodeset_map.items(): + placements.extend(create_nodeset_placements(ns_nodes, excl_job_id, lkp)) + return placements + + +def _allocate_nodes_to_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + # canned result for no placement policies created + no_pp = [PlacementAndNodes(placement=None, nodes=nodes)] + + model = nodes[0] + nodeset = lkp.node_nodeset(model) + + is_slice = bool(getattr(nodeset, 'accelerator_topology', None)) + + excl_job_placement = (excl_job_id is not None) and (not is_slice) + + if excl_job_placement and len(nodes) < 2: + return no_pp # don't create placement_policy for just one node + + if lkp.is_flex_node(model): + return no_pp # TODO(FLEX): Add support for workload policies + if lkp.node_is_tpu(model): + return no_pp + if not (nodeset.enable_placement and valid_placement_node(model)): + return no_pp + + max_count = calculate_chunk_size(nodeset, lkp) + + name_prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-{nodeset.nodeset_name}" + + if excl_job_placement: # simply chunk given nodes by max size of placement + return [ + PlacementAndNodes(placement=f"{name_prefix}-{excl_job_id}-{i}", nodes=chunk) + for i, chunk in enumerate(chunked(nodes, n=max_count)) + ] + + # split whole nodeset (not only nodes to resume) into chunks of max size of placement + # create placements (most likely already exists) placements for requested nodes + chunks = collections.defaultdict(list) # chunk_id -> nodes + invalid = [] + + for node in nodes: + try: + chunk = lkp.node_index(node) // max_count + chunks[chunk].append(node) + except: + invalid.append(node) + + placements = [ + # NOTE: use 0 instead of job_id for consistency with previous SlurmGCP behavior + PlacementAndNodes(placement=f"{name_prefix}-0-{c_id}", nodes=c_nodes) + for c_id, c_nodes in chunks.items() + ] + + if invalid: + placements.append(PlacementAndNodes(placement=None, nodes=invalid)) + log.error(f"Could not find placement for nodes with unexpected names: {to_hostlist(invalid)}") + + return placements + +def calculate_hosts_per_topo(accelerator_topology: str, machine_type: NSDict) -> int: + # Calculate total number of hosts per topology (Assumes format: '1x72') + try: + top_split = [int(x) for x in accelerator_topology.split("x")] + except Exception as e: + log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") + raise e + + if len(machine_type.accelerators) == 0: + gpus_per_machine = 0 + else: + gpus_per_machine = machine_type.accelerators[0].count + + if len(top_split) != 2: + log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") + elif top_split[0] <= 0 or top_split[1] <= 0: + log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") + elif gpus_per_machine <= 0: + log.error(f"The machine type has no accelerators. Cannot use accelerator topology {accelerator_topology}.") + elif top_split[1] % gpus_per_machine: + log.error(f"The GPU count {gpus_per_machine} per node is not a factor of the accelerator topology {accelerator_topology}") + + return (top_split[0] * top_split[1]) // gpus_per_machine + +def calculate_chunk_size(nodeset: NSDict, lkp: util.Lookup) -> int: + # Calculates the chunk size based on max distance value received or accelerator topology + # Assuming nodeset is not tpu + machine_type = lkp.template_info(nodeset.instance_template).machine_type + max_distance = nodeset.placement_max_distance + accelerator_topology = nodeset.accelerator_topology + + # Look for accelerator topology first + if accelerator_topology: + hosts_per_topo = calculate_hosts_per_topo(accelerator_topology, machine_type) + return hosts_per_topo + + if max_distance == 1: + return 22 + elif max_distance == 2: + if machine_type.family.startswith("a3"): + return 256 + else: + return 150 + elif max_distance == 3: + return 1500 + else: + return PLACEMENT_MAX_CNT + +def create_nodeset_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + placements = _allocate_nodes_to_placements(nodes, excl_job_id, lkp) + region = lkp.node_region(nodes[0]) + max_distance = lkp.node_nodeset(nodes[0]).get('placement_max_distance') + accelerator_topology = lkp.nodeset_accelerator_topology(lkp.node_nodeset_name(nodes[0])) + + if log.isEnabledFor(logging.DEBUG): + debug_p = {p.placement: to_hostlist(p.nodes) for p in placements} + log.debug( + f"creating {len(placements)} placement groups: \n{yaml.safe_dump(debug_p).rstrip()}" + ) + + requests = { + p.placement: create_placement_request(p.placement, region, max_distance, accelerator_topology) for p in placements if p.placement + } + if not requests: + return placements + # TODO: aggregate all requests for whole resume and execute them at once (don't limit to nodeset/job) + ops = dict( + zip(requests.keys(), map_with_futures(ensure_execute, requests.values())) + ) + + def classify_result(item): + op = item[1] + if not isinstance(op, Exception): + return "submitted" + if all(e.get("reason") == "alreadyExists" for e in op.error_details): # type: ignore + return "redundant" + return "failed" + + grouped_ops = dict(util.groupby_unsorted(list(ops.items()), classify_result)) + submitted, redundant, failed = ( + dict(grouped_ops.get(key, {})) for key in ("submitted", "redundant", "failed") + ) + if redundant: + log.warning( + "placement policies already exist: {}".format(",".join(redundant.keys())) + ) + if failed: + reqs = [f"{e}" for _, e in failed.values()] + log.fatal("failed to create placement policies: {}".format("; ".join(reqs))) + operations = {group: wait_for_operation(op) for group, op in submitted.items()} + for group, op in operations.items(): + if "error" in op: + msg = "; ".join( + f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" + for err in op["error"]["errors"] + ) + log.error( + f"placement group failed to create: '{group}' ({op['name']}): {msg}" + ) + + log.info( + f"created {len(operations)} placement groups ({to_hostlist(operations.keys())})" + ) + return placements + + +def valid_placement_node(node: str) -> bool: + invalid_types = frozenset(["e2", "t2d", "n1", "t2a", "m1", "m2", "m3"]) + mt = lookup().node_template_info(node).machineType + if mt.split("-")[0] in invalid_types: + log.warn(f"Unsupported machine type for placement policy: {mt}.") + log.warn( + f"Please do not use any the following machine types with placement policy: ({','.join(invalid_types)})" + ) + return False + return True + + +def main(nodelist: str) -> None: + """main called when run as script""" + log.debug(f"ResumeProgram {nodelist}") + # Filter out nodes not in config.yaml + other_nodes, nodes = separate( + lookup().is_power_managed_node, util.to_hostnames(nodelist) + ) + if other_nodes: + log.error( + f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" + ) + + if not nodes: + log.info("No nodes to resume") + return + resume_data = get_resume_file_data() + log.info(f"resume {util.to_hostlist(nodes)}") + resume_nodes(nodes, resume_data) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("nodelist", help="list of nodes to resume") + args = util.init_log_and_parse(parser) + main(args.nodelist) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh new file mode 100644 index 0000000000..023d246f01 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PYTHON_SCRIPT="${SCRIPT_DIR}/resume.py" + +# Capture all arguments passed by Slurm (the nodelist). +ALL_ARGS=("$@") + +# This array will hold extra argument for resume.py, like the resume data file. +UNIQUE_RESUME_FILE="" + +# Handle SLURM_RESUME_FILE if provided +if [ -n "${SLURM_RESUME_FILE-}" ] && [ -f "$SLURM_RESUME_FILE" ]; then + SAFE_DIR="/tmp/slurm_resume_data" + mkdir -p "$SAFE_DIR" + + UNIQUE_RESUME_FILE="${SAFE_DIR}/resumedata.$$.json" + cp "$SLURM_RESUME_FILE" "$UNIQUE_RESUME_FILE" +fi + +SLURM_RESUME_FILE="${UNIQUE_RESUME_FILE}" +setsid "${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & + +exit 0 diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py new file mode 100644 index 0000000000..846524adf2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py @@ -0,0 +1,660 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import shutil +import subprocess +import stat +import time +import yaml +from pathlib import Path +import functools + +import util +from util import ( + lookup, + dirs, + slurmdirs, + run, + install_custom_scripts, +) +import conf +import slurmsync + +from setup_network_storage import ( + setup_network_storage, + setup_nfs_exports, +) + + +log = logging.getLogger() + + +MOTD_HEADER = """ + SSSSSSS + SSSSSSSSS + SSSSSSSSS + SSSSSSSSS + SSSS SSSSSSS SSSS + SSSSSS SSSSSS + SSSSSS SSSSSSS SSSSSS + SSSS SSSSSSSSS SSSS + SSS SSSSSSSSS SSS + SSSSS SSSS SSSSSSSSS SSSS SSSSS + SSS SSSSSS SSSSSSSSS SSSSSS SSS + SSSSSS SSSSSSS SSSSSS + SSS SSSSSS SSSSSS SSS + SSSSS SSSS SSSSSSS SSSS SSSSS + S SSS SSSSSSSSS SSS S + SSS SSSS SSSSSSSSS SSSS SSS + S SSS SSSSSS SSSSSSSSS SSSSSS SSS S + SSSSS SSSSSS SSSSSSSSS SSSSSS SSSSS + S SSSSS SSSS SSSSSSS SSSS SSSSS S + S SSS SSS SSS SSS S + S S S S + SSS + SSS + SSS + SSS + SSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSS SSSSSSSSSSSSSSSSSSSS +SSSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSSS SSSSSSSSSSSSSSSSSSSSSS +SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSSS SSS SSSSSSSSSSSSSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSS SSS SSSSSSSSSSSSS SSSS SSSS SSSS SSSS + +""" +_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" + +def start_motd(): + """advise in motd that slurm is currently configuring""" + wall_msg = "*** Slurm is currently being configured in the background. ***" + motd_msg = MOTD_HEADER + wall_msg + "\n\n" + Path("/etc/motd").write_text(motd_msg) + util.run(f"wall -n '{wall_msg}'", timeout=30) + + +def end_motd(broadcast=True): + """modify motd to signal that setup is complete""" + Path("/etc/motd").write_text(MOTD_HEADER) + + if not broadcast: + return + + run( + "wall -n '*** Slurm {} setup complete ***'".format(lookup().instance_role), + timeout=30, + ) + if not lookup().is_controller: + run( + """wall -n ' +/home on the controller was mounted over the existing /home. +Log back in to ensure your home directory is correct. +'""", + timeout=30, + ) + + +def failed_motd(): + """modify motd to signal that setup is failed""" + wall_msg = f"*** Slurm setup failed! Please view log: {util.get_log_path()} ***" + motd_msg = MOTD_HEADER + wall_msg + "\n\n" + Path("/etc/motd").write_text(motd_msg) + util.run(f"wall -n '{wall_msg}'", timeout=30) + + +def _startup_script_timeout(lkp: util.Lookup) -> int: + if lkp.is_controller: + return lkp.cfg.get("controller_startup_scripts_timeout", 300) + elif lkp.instance_role == "compute": + return lkp.cfg.get("compute_startup_scripts_timeout", 300) + elif lkp.is_login_node: + return lkp.cfg.login_groups[util.instance_login_group()].get("startup_scripts_timeout", 300) + return 300 + + +def run_custom_scripts(): + """run custom scripts based on instance_role""" + custom_dir = dirs.custom_scripts + if lookup().is_controller: + # controller has all scripts, but only runs controller.d + custom_dirs = [custom_dir / "controller.d"] + elif lookup().instance_role == "compute": + # compute setup with nodeset.d + custom_dirs = [custom_dir / "nodeset.d"] + elif lookup().is_login_node: + # login setup with only login.d + custom_dirs = [custom_dir / "login.d"] + else: + # Unknown role: run nothing + custom_dirs = [] + + timeout = _startup_script_timeout(lookup()) + + custom_scripts = [ + p + for d in custom_dirs + for p in d.rglob("*") + if p.is_file() and not p.name.endswith(".disabled") + ] + print_scripts = ",".join(str(s.relative_to(custom_dir)) for s in custom_scripts) + log.debug(f"custom scripts to run: {custom_dir}/({print_scripts})") + + try: + for script in custom_scripts: + log.info(f"running script {script.name} with timeout={timeout}") + result = run(str(script), timeout=timeout, check=False, shell=True) + runlog = ( + f"{script.name} returncode={result.returncode}\n" + f"stdout={result.stdout}stderr={result.stderr}" + ) + log.info(runlog) + result.check_returncode() + except OSError as e: + log.error(f"script {script} is not executable") + raise e + except subprocess.TimeoutExpired as e: + log.error(f"script {script} did not complete within timeout={timeout}") + raise e + except Exception as e: + log.exception(f"script {script} encountered an exception") + raise e + +def mount_save_state_disk(): + disk_name = f"/dev/disk/by-id/google-{lookup().cfg.controller_state_disk.device_name}" + mount_point = util.slurmdirs.state + fs_type = "ext4" + + rdevice = util.run(f"realpath {disk_name}").stdout.strip() + file_output = util.run(f"file -s {rdevice}").stdout.strip() + if "filesystem" not in file_output: + util.run(f"mkfs -t {fs_type} -q {rdevice}") + + fstab_entry = f"{disk_name} {mount_point} {fs_type}" + with open("/etc/fstab", "r") as f: + fstab = f.readlines() + if fstab_entry not in fstab: + with open("/etc/fstab", "a") as f: + f.write(f"{fstab_entry} defaults 0 0\n") + + util.run(f"systemctl daemon-reload") + + os.makedirs(mount_point, exist_ok=True) + util.run(f"mount {mount_point}") + + util.chown_slurm(mount_point) + + +def setup_jwt_key(): + jwt_key = Path(slurmdirs.state / "jwt_hs256.key") + + if jwt_key.exists(): + log.info("JWT key already exists. Skipping key generation.") + else: + run("dd if=/dev/urandom bs=32 count=1 > " + str(jwt_key), shell=True) + + util.chown_slurm(jwt_key, mode=0o400) + + +def _generate_key(p: Path) -> None: + run(f"dd if=/dev/random of={p} bs=1024 count=1") + + +def setup_key(lkp: util.Lookup) -> None: + file_name = "munge.key" + dir = dirs.munge + + if lkp.cfg.enable_slurm_auth: + file_name = "slurm.key" + dir = slurmdirs.etc + + dst = Path(dir / file_name) + + if lkp.cfg.controller_state_disk.device_name: + # Copy key from persistent state disk + persist = slurmdirs.state / file_name + if not persist.exists(): + _generate_key(persist) + + shutil.copyfile(persist, dst) + if lkp.cfg.enable_slurm_auth: + util.chown_slurm(dst, mode=0o400) + util.chown_slurm(persist, mode=0o400) + else: + shutil.chown(dst, user="munge", group="munge") + os.chmod(dst, stat.S_IRUSR) + else: + if dst.exists(): + log.info("key already exists. Skipping key generation.") + else: + _generate_key(dst) + if lkp.cfg.enable_slurm_auth: + util.chown_slurm(dst, mode=0o400) + else: + shutil.chown(dst, user="munge", group="munge") + os.chmod(dst, stat.S_IRUSR) + + if lkp.cfg.enable_slurm_auth: + # Put key into shared volume for distribution + distributed = util.slurmdirs.key_distribution / file_name + shutil.copyfile(dst, distributed) + util.chown_slurm(distributed, mode=0o400) + # Munge is distributed from /etc/munge. + else: + run("systemctl restart munge", timeout=30) + + +def setup_nss_slurm(): + """install and configure nss_slurm""" + # setup nss_slurm + util.mkdirp(Path("/var/spool/slurmd")) + run( + "ln -s {}/lib/libnss_slurm.so.2 /usr/lib64/libnss_slurm.so.2".format( + slurmdirs.prefix + ), + check=False, + ) + run(r"sed -i 's/\(^\(passwd\|group\):\s\+\)/\1slurm /g' /etc/nsswitch.conf") + + +def setup_sudoers(): + content = """ +# Allow SlurmUser to manage the slurm daemons +slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmd.service +slurm ALL= NOPASSWD: /usr/bin/systemctl restart sackd.service +slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmctld.service +""" + sudoers_file = Path("/etc/sudoers.d/slurm") + sudoers_file.write_text(content) + sudoers_file.chmod(0o0440) + + +def setup_maintenance_script(): + perform_maintenance = """#!/bin/bash + +#SBATCH --priority=low +#SBATCH --time=180 + +VM_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") +ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" | cut -d '/' -f 4) + +gcloud compute instances perform-maintenance $VM_NAME \ + --zone=$ZONE +""" + + + with open(_MAINTENANCE_SBATCH_SCRIPT_PATH, "w") as f: + f.write(perform_maintenance) + + util.chown_slurm(_MAINTENANCE_SBATCH_SCRIPT_PATH, mode=0o755) + + +def update_system_config(file, content): + """Add system defaults options for service files""" + sysconfig = Path("/etc/sysconfig") + default = Path("/etc/default") + + if sysconfig.exists(): + conf_dir = sysconfig + elif default.exists(): + conf_dir = default + else: + raise Exception("Cannot determine system configuration directory.") + + slurmd_file = Path(conf_dir, file) + slurmd_file.write_text(content) + +def _symlink_mysql_datadir(lkp: util.Lookup) -> None: + """ Symlink /var/lib/mysql to controller state disk if needed. """ + if not lkp.cfg.controller_state_disk.device_name: + return + + datadir = Path("/var/lib/mysql") + dst = slurmdirs.state / "mysql" + + if dst.exists(): + run(f"rm -rf {datadir}") + else: + shutil.move(datadir, dst) + + datadir.symlink_to(dst, target_is_directory=True) + shutil.chown(datadir, user="mysql", group="mysql") + run(f"chown -R mysql:mysql {dst}") + +def configure_mysql(lkp: util.Lookup) -> None: + cnfdir = Path("/etc/my.cnf.d") + if not cnfdir.exists(): + cnfdir = Path("/etc/mysql/conf.d") + if not (cnfdir / "mysql_slurm.cnf").exists(): + (cnfdir / "mysql_slurm.cnf").write_text( + """ +[mysqld] +bind-address=127.0.0.1 +innodb_buffer_pool_size=1024M +innodb_log_file_size=64M +innodb_lock_wait_timeout=900 +""" + ) + + run("systemctl stop mariadb", timeout=30) + _symlink_mysql_datadir(lkp) + + run("systemctl enable mariadb", timeout=30) + run("systemctl restart mariadb", timeout=30) + + db_name = "slurm_acct_db" + + + cmd = "mysql -u root -e" + for host in ("localhost", lkp.control_host): + run(f"""{cmd} "drop user if exists 'slurm'@'{host}'";""", timeout=30) + run(f"""{cmd} "create user 'slurm'@'{host}'";""", timeout=30) + run(f"""{cmd} "grant all on {db_name}.* TO 'slurm'@'{host}'";""", timeout=30) + + +def configure_dirs(): + for p in dirs.values(): + util.mkdirp(p) + + for p in (dirs.slurm, dirs.scripts, dirs.custom_scripts): + util.chown_slurm(p) + + for p in slurmdirs.values(): + util.mkdirp(p) + util.chown_slurm(p) + + for sl, tgt in ( # create symlinks + (Path("/etc/slurm"), slurmdirs.etc), + (dirs.scripts / "etc", slurmdirs.etc), + (dirs.scripts / "log", dirs.log), + ): + if sl.exists() and sl.is_symlink(): + sl.unlink() + sl.symlink_to(tgt) + + # copy auxiliary scripts + for dst_folder, src_file in ((lookup().cfg.slurm_bin_dir, + Path("sort_nodes.py")), + (dirs.custom_scripts / "task_prolog.d", + Path("tools/task-prolog")), + (dirs.custom_scripts / "task_epilog.d", + Path("tools/task-epilog"))): + dst = Path(dst_folder) / src_file.name + util.mkdirp(dst.parent) + shutil.copyfile(util.scripts_dir / src_file, dst) + os.chmod(dst, 0o755) + + +def self_report_controller_address(lkp: util.Lookup) -> None: + if not lkp.cfg.controller_network_attachment: + return # only self report address if network attachment is used + data = { "slurm_control_addr": lkp.cfg.slurm_control_addr } + bucket, prefix = util._get_bucket_and_common_prefix() + blob = util.storage_client().bucket(bucket).blob(f"{prefix}/controller_addr.yaml") + with blob.open('w') as f: + f.write(yaml.dump(data)) + +def setup_controller(): + """Run controller setup""" + log.info("Setting up controller") + lkp = util.lookup() + util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) + install_custom_scripts() + conf.gen_controller_configs(lkp) + + if lkp.cfg.controller_state_disk.device_name != None: + mount_save_state_disk() + + setup_jwt_key() + setup_key(lkp) + + setup_sudoers() + setup_network_storage() + + run_custom_scripts() + + if not lkp.cfg.cloudsql_secret: + configure_mysql(lkp) + + run("systemctl enable slurmdbd", timeout=30) + run("systemctl restart slurmdbd", timeout=30) + + # Wait for slurmdbd to come up + time.sleep(5) + + sacctmgr = f"{slurmdirs.prefix}/bin/sacctmgr -i" + result = run( + f"{sacctmgr} add cluster {lkp.cfg.slurm_cluster_name}", timeout=30, check=False + ) + if "already exists" in result.stdout: + log.info(result.stdout) + elif result.returncode > 1: + result.check_returncode() # will raise error + + run("systemctl enable slurmctld", timeout=30) + run("systemctl restart slurmctld", timeout=30) + + run("systemctl enable slurmrestd", timeout=30) + run("systemctl restart slurmrestd", timeout=30) + + # Export at the end to signal that everything is up + run("systemctl enable nfs-server", timeout=30) + run("systemctl start nfs-server", timeout=30) + + setup_nfs_exports() + run("systemctl enable --now slurmcmd.timer", timeout=30) + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status slurmdbd", timeout=30) + run("systemctl status slurmctld", timeout=30) + run("systemctl status slurmrestd", timeout=30) + + try: + slurmsync.sync_instances() + except Exception: + log.exception("Failed to sync instances, will try next time.") + + run("systemctl enable slurm_load_bq.timer", timeout=30) + run("systemctl start slurm_load_bq.timer", timeout=30) + run("systemctl status slurm_load_bq.timer", timeout=30) + + # Add script to perform maintenance + setup_maintenance_script() + + self_report_controller_address(lkp) + + log.info("Done setting up controller") + pass + + +def setup_login(): + """run login node setup""" + log.info("Setting up login") + + lkp = lookup() + slurmctld_host = f"{lkp.control_host}" + if lkp.control_addr: + slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" + sackd_options = [ + f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', + ] + sysconf = f"""SACKD_OPTIONS='{" ".join(sackd_options)}'""" + update_system_config("sackd", sysconf) + install_custom_scripts() + + setup_network_storage() + setup_sudoers() + if not lkp.cfg.enable_slurm_auth: + run("systemctl restart munge", timeout=30) + run("systemctl enable sackd", timeout=30) + run("systemctl restart sackd", timeout=30) + run("systemctl enable --now slurmcmd.timer", timeout=30) + + run_custom_scripts() + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status sackd", timeout=30) + + log.info("Done setting up login") + + +def setup_compute(): + """run compute node setup""" + log.info("Setting up compute") + + lkp = lookup() + util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) + slurmctld_host = f"{lkp.control_host}" + if lkp.control_addr: + slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" + slurmd_options = [ + f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', + ] + + try: + slurmd_feature = util.instance_metadata("attributes/slurmd_feature", silent=True) + except util.MetadataNotFoundError: + slurmd_feature = None + + if slurmd_feature is not None: + slurmd_options.append(f'--conf="Feature={slurmd_feature}"') + slurmd_options.append("-Z") + + sysconf = f"""SLURMD_OPTIONS='{" ".join(slurmd_options)}'""" + update_system_config("slurmd", sysconf) + install_custom_scripts() + + setup_nss_slurm() + setup_network_storage() + + has_gpu = run("lspci | grep --ignore-case 'NVIDIA' | wc -l", shell=True).returncode + if has_gpu: + run("nvidia-smi") + + run_custom_scripts() + + setup_sudoers() + if not lkp.cfg.enable_slurm_auth: + run("systemctl restart munge", timeout=30) + run("systemctl enable slurmd", timeout=30) + run("systemctl restart slurmd", timeout=30) + run("systemctl enable --now slurmcmd.timer", timeout=30) + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status slurmd", timeout=30) + + log.info("Done setting up compute") + +def setup_cloud_ops() -> None: + """Add health checks, deployment info, and updated setup path to cloud ops config.""" + cloudOpsStatus = run( + "systemctl is-active --quiet google-cloud-ops-agent.service", check=False + ).returncode + + if cloudOpsStatus != 0: + return + + with open("/etc/google-cloud-ops-agent/config.yaml", "r") as f: + file = yaml.safe_load(f) + + # Update setup receiver path + file["logging"]["receivers"]["setup"]["include_paths"] = ["/var/log/slurm/setup.log"] + + cluster_info = { + 'type':'modify_fields', + 'fields': { + 'labels."cluster_name"':{ + 'static_value':f"{lookup().cfg.slurm_cluster_name}" + }, + 'labels."hostname"':{ + 'static_value': f"{lookup().hostname}" + } + } + } + + file["logging"]["processors"]["add_cluster_info"] = cluster_info + file["logging"]["service"]["pipelines"]["slurmlog_pipeline"]["processors"].append("add_cluster_info") + file["logging"]["service"]["pipelines"]["slurmlog2_pipeline"]["processors"].append("add_cluster_info") + + with open("/etc/google-cloud-ops-agent/config.yaml", "w") as f: + yaml.safe_dump(file, f, sort_keys=False) + + retries = 2 + for _ in range(retries): + try: + run("systemctl restart google-cloud-ops-agent.service", timeout=120) + break + except subprocess.TimeoutExpired: + log.error("google-cloud-ops-agent.service did not restart within 120s.") + result=run("cat /var/log/google-cloud-ops-agent/subagents/logging-module.log", timeout=120, shell=True) + if result.stdout: + log.error(f"Logs for google-cloud-ops-agent (logging-module.log file):\n{result.stdout}") + raise + + +def main(): + start_motd() + + log.info("Starting setup, fetching config") + sleep_seconds = 5 + while True: + try: + _, cfg = util.fetch_config() + util.update_config(cfg) + break + except util.DeffetiveStoredConfigError as e: + log.warning(f"config is not ready yet: {e}, sleeping for {sleep_seconds}s") + except Exception as e: + log.exception(f"unexpected error while fetching config, sleeping for {sleep_seconds}s") + time.sleep(sleep_seconds) + log.info("Config fetched") + setup_cloud_ops() + configure_dirs() + # call the setup function for the instance type + { + "controller": setup_controller, + "compute": setup_compute, + "login": setup_login, + }.get( + lookup().instance_role, + lambda: log.fatal(f"Unknown node role: {lookup().instance_role}"))() + + end_motd() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--slurmd-feature", dest="slurmd_feature", help="Unused, to be removed.") + _ = util.init_log_and_parse(parser) + + try: + main() + except Exception: + log.exception("Aborting setup...") + failed_motd() diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py new file mode 100644 index 0000000000..095f42e758 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py @@ -0,0 +1,327 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import os +import sys +import stat +import time +import logging +import uuid + +import shutil +from pathlib import Path +from concurrent.futures import as_completed +from addict import Dict as NSDict # type: ignore + +import util +from util import NSMount, lookup, run, dirs, separate +from more_executors import Executors, ExceptionRetryPolicy + + +log = logging.getLogger() + +def mounts_by_local(mounts: list[NSMount]) -> dict[str, NSMount]: + """convert list of mounts to dict of mounts, local_mount as key""" + return {str(m.local_mount.resolve()): m for m in mounts} + + +def _get_default_mounts(lkp: util.Lookup) -> list[NSMount]: + if lkp.cfg.disable_default_mounts: + return [] + return [ + NSMount( + server_ip=lkp.controller_mount_server_ip(), + remote_mount=path, + local_mount=path, + fs_type="nfs", + mount_options="defaults,hard,intr", + ) + for path in ( + dirs.home, + dirs.apps, + ) + ] + +def get_slurm_bucket_mount() -> NSMount: + bucket, path = util._get_bucket_and_common_prefix() + return NSMount( + fs_type="gcsfuse", + server_ip="", + remote_mount=Path(bucket), + local_mount=dirs.slurm_bucket_mount, + mount_options=f"defaults,_netdev,implicit_dirs,only_dir={path}", + ) + +def resolve_network_storage() -> List[NSMount]: + """Combine appropriate network_storage fields to a single list""" + lkp = lookup() + + # create dict of mounts, local_mount: mount_info + mounts = mounts_by_local(_get_default_mounts(lkp)) + + if lkp.is_controller and util.should_mount_slurm_bucket(): + mounts.update(mounts_by_local([get_slurm_bucket_mount()])) + + # On non-controller instances, entries in network_storage could overwrite + # default exports from the controller. Be careful, of course + common = [lkp.normalize_ns_mount(m) for m in lkp.cfg.network_storage] + mounts.update(mounts_by_local(common)) + + if lkp.is_login_node: + login_group = lkp.cfg.login_groups[util.instance_login_group()] + login_ns = [lkp.normalize_ns_mount(m) for m in login_group.network_storage] + mounts.update(mounts_by_local(login_ns)) + + if lkp.instance_role == "compute": + try: + nodeset = lkp.node_nodeset() + except Exception: + pass # external nodename, skip lookup + else: + nodeset_ns = [lkp.normalize_ns_mount(m) for m in nodeset.network_storage] + mounts.update(mounts_by_local(nodeset_ns)) + + return list(mounts.values()) + + +def is_controller_mount(mount) -> bool: + # NOTE: Valid Lustre server_ip can take the form of '@tcp' + server_ip = mount.server_ip.split("@")[0] + mount_addr = util.host_lookup(server_ip) + return mount_addr == lookup().control_host_addr + +def setup_network_storage(): + """prepare network fs mounts and add them to fstab""" + log.info("Set up network storage") + + all_mounts = resolve_network_storage() + if lookup().is_controller: + mounts, _ = separate(is_controller_mount, all_mounts) + else: + mounts = all_mounts + + # Determine fstab entries and write them out + fstab_entries = [] + for mount in mounts: + local_mount = mount.local_mount + fs_type = mount.fs_type + server_ip = mount.server_ip or "" + src = mount.remote_mount if fs_type == "gcsfuse" else f"{server_ip}:{mount.remote_mount}" + + log.info(f"Setting up mount ({fs_type}) {src} to {local_mount}") + util.mkdirp(local_mount) + + mount_options = mount.mount_options.split(",") if mount.mount_options else [] + if "_netdev" not in mount_options: + mount_options += ["_netdev"] + options_line = ",".join(mount_options) + + + fstab_entries.append(f"{src} {local_mount} {fs_type} {options_line} 0 0") + + fstab = Path("/etc/fstab") + if not Path(fstab.with_suffix(".bak")).is_file(): + shutil.copy2(fstab, fstab.with_suffix(".bak")) + shutil.copy2(fstab.with_suffix(".bak"), fstab) + with open(fstab, "a") as f: + f.write("\n") + for entry in fstab_entries: + f.write(entry) + f.write("\n") + + mount_fstab(mounts, log) + if lookup().cfg.enable_slurm_auth: + slurm_key_mount_handler() + else: + munge_mount_handler() + + +def mount_fstab(mounts: list[NSMount], log): + """Wait on each mount, then make sure all fstab is mounted""" + def mount_path(path: Path): + log.info(f"Waiting for '{path}' to be mounted...") + try: + run(f"mount {path}", timeout=120) + except Exception as e: + exc_type, _, _ = sys.exc_info() + log.error(f"mount of path '{path}' failed: {exc_type}: {e}") + raise e + log.info(f"Mount point '{path}' was mounted.") + + MAX_MOUNT_TIMEOUT = 60 * 5 + future_list = [] + retry_policy = ExceptionRetryPolicy( + max_attempts=120, exponent=1.6, sleep=1.0, max_sleep=16.0 + ) + with Executors.thread_pool().with_timeout(MAX_MOUNT_TIMEOUT).with_retry( + retry_policy=retry_policy + ) as exe: + for m in mounts: + future = exe.submit(mount_path, m.local_mount) + future_list.append(future) + + # Iterate over futures, checking for exceptions + for future in as_completed(future_list): + try: + future.result() + except Exception as e: + raise e + + +def munge_mount_handler(): + if lookup().is_controller: + return + mnt = lookup().munge_mount + + log.info(f"Mounting munge share to: {mnt.local_mount}") + mnt.local_mount.mkdir() + if mnt.fs_type == "gcsfuse": + cmd = [ + "gcsfuse", + f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, + mnt.server_ip, + str(mnt.local_mount), + ] + else: + cmd = [ + "mount", + f"--types={mnt.fs_type}", + f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, + f"{mnt.server_ip}:{mnt.remote_mount}", + str(mnt.local_mount), + ] + # wait max 240s for munge mount + timeout = 240 + for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): + try: + run(cmd, timeout=timeout) + break + except Exception as e: + log.error( + f"munge mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" + ) + time.sleep(wait) + err = e + continue + else: + raise err + + munge_key = Path(dirs.munge / "munge.key") + log.info(f"Copy munge.key from: {mnt.local_mount}") + shutil.copy2(Path(mnt.local_mount / "munge.key"), munge_key) + + log.info("Restrict permissions of munge.key") + shutil.chown(munge_key, user="munge", group="munge") + os.chmod(munge_key, stat.S_IRUSR) + + log.info(f"Unmount {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + run(f"fusermount -u {mnt.local_mount}", timeout=120) + else: + run(f"umount {mnt.local_mount}", timeout=120) + shutil.rmtree(mnt.local_mount) + +def slurm_key_mount_handler(): + if lookup().is_controller: + return + mnt = lookup().slurm_key_mount + + log.info(f"Mounting slurm_key share to: {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + cmd = [ + "gcsfuse", + f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, + mnt.server_ip, + str(mnt.local_mount), + ] + else: + cmd = [ + "mount", + f"--types={mnt.fs_type}", + f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, + f"{mnt.server_ip}:{mnt.remote_mount}", + str(mnt.local_mount), + ] + timeout = 120 # wait max 120s to mount + for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): + try: + run(cmd, timeout=timeout) + break + except Exception as e: + log.error( + f"slurm key mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" + ) + time.sleep(wait) + err = e + continue + else: + raise err + + file_name = "slurm.key" + dst = Path(util.slurmdirs.etc / file_name) + log.info(f"Copy slurm.key from: {mnt.local_mount}") + shutil.copy2(mnt.local_mount / file_name, dst) + + log.info("Restrict permissions of slurm.key") + util.chown_slurm(dst, mode=0o400) + + log.info(f"Unmount {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + run(f"fusermount -u {mnt.local_mount}", timeout=120) + else: + run(f"umount {mnt.local_mount}", timeout=120) + shutil.rmtree(mnt.local_mount) + + +def setup_nfs_exports(): + """nfs export all needed directories""" + lkp = util.lookup() + assert lkp.is_controller + + # The controller only needs to set up exports for cluster-internal mounts + exported_mounts = [m for m in resolve_network_storage() if is_controller_mount(m)] + + # key by remote mount path since that is what needs exporting + to_export = {m.remote_mount: "*(rw,no_subtree_check,no_root_squash)" for m in exported_mounts} + + key_mount = lkp.slurm_key_mount if lkp.cfg.enable_slurm_auth else lkp.munge_mount + if is_controller_mount(key_mount): + # Export key mount as read-only + to_export[key_mount.remote_mount] = "*(ro,no_subtree_check,no_root_squash)" + + if util.should_mount_slurm_bucket(): + mnt = get_slurm_bucket_mount() + # FSID is required for virtual filesystem that is not based on a device + # Also export it as read-only + fsid=str(uuid.uuid4()) + to_export[mnt.local_mount] = f"*(ro,no_subtree_check,no_root_squash,fsid={fsid})" + + # export path if corresponding selector boolean is True + lines = [] + for path,options in to_export.items(): + util.mkdirp(Path(path)) + run(rf"sed -i '\#{path}#d' /etc/exports", timeout=30) + lines.append(f"{path} {options}") + + exportsd = Path("/etc/exports.d") + util.mkdirp(exportsd) + with (exportsd / "slurm.exports").open("w") as f: + f.write("\n") + f.write("\n".join(lines)) + run("exportfs -a", timeout=30) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py new file mode 100644 index 0000000000..1bfdd5acce --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py @@ -0,0 +1,679 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import fcntl +import json +import logging +import re +import sys +import shlex +from datetime import datetime, timedelta +from itertools import chain +from pathlib import Path +from dataclasses import dataclass +from typing import Dict, Tuple, List, Optional, Protocol, Any +from functools import lru_cache + +import util +from util import ( + batch_execute, + ensure_execute, + execute_with_futures, + FutureReservation, + install_custom_scripts, + run, + separate, + to_hostlist, + NodeState, + chunked, + dirs, +) +from util import lookup +from suspend import delete_instances +import tpu +import conf +import watch_delete_vm_op + +log = logging.getLogger() + +TOT_REQ_CNT = 1000 +_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" + +class NodeAction(Protocol): + def apply(self, nodes:List[str]) -> None: + ... + + def __hash__(self): + ... + +@dataclass(frozen=True) +class NodeActionPowerUp(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to resume ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=power_up") + +@dataclass(frozen=True) +class NodeActionIdle(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} nodes to idle ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=resume") + +@dataclass(frozen=True) +class NodeActionPowerDown(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to power down ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=power_down") + + +@dataclass(frozen=True) +class NodeActionPowerDownForce(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to power down ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=power_down_force") + + +@dataclass(frozen=True) +class NodeActionDelete(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to delete ({hostlist})") + delete_instances(nodes) + +@dataclass(frozen=True) +class NodeActionPrempt(): + def apply(self, nodes:List[str]) -> None: + NodeActionDown(reason="Preempted instance").apply(nodes) + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances restarted ({hostlist})") + start_instances(nodes) + +@dataclass(frozen=True) +class NodeActionUnchanged(): + def apply(self, nodes:List[str]) -> None: + pass + +@dataclass(frozen=True) +class NodeActionDown(): + reason: str + + def apply(self, nodes: List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} nodes set down ({hostlist}) with reason={self.reason}") + run(f"{lookup().scontrol} update nodename={hostlist} state=down reason={shlex.quote(self.reason)}") + +@dataclass(frozen=True) +class NodeActionUnknown(): + slurm_state: Optional[NodeState] + instance_state: Optional[str] + + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.error(f"{len(nodes)} nodes have unexpected {self.slurm_state} and instance state:{self.instance_state}, ({hostlist})") + +def start_instance_op(node: str) -> Any: + inst = lookup().instance(node) + assert inst + + return lookup().compute.instances().start( + project=lookup().project, + zone=inst.zone, + instance=inst.name, + ) + + +def start_instances(node_list): + log.info("{} instances to start ({})".format(len(node_list), ",".join(node_list))) + lkp = lookup() + # TODO: use code from resume.py to assign proper placement + normal, tpu_nodes = separate(lkp.node_is_tpu, node_list) + ops = {node: start_instance_op(node) for node in normal} + + done, failed = batch_execute(ops) + + tpu_start_data = [] + for ns, nodes in util.groupby_unsorted(tpu_nodes, lkp.node_nodeset_name): + tpuobj = tpu.TPU.make(ns, lkp) + for snodes in chunked(nodes, n=tpuobj.vmcount): + tpu_start_data.append({"tpu": tpuobj, "node": snodes}) + execute_with_futures(tpu.start_tpu, tpu_start_data) + + +def _find_dynamic_node_status() -> NodeAction: + # TODO: cover more cases: + # * delete dead dynamic nodes + # * delete orhpaned instances + return NodeActionUnchanged() # don't touch dynamic nodes + +def get_fr_action(fr: FutureReservation, state:Optional[NodeState]) -> Optional[NodeAction]: + now = util.now() + if state is None: + return None # handle like any other node + if fr.start_time < now < fr.end_time: + return None # handle like any other node + + if state.base == "DOWN": + return NodeActionUnchanged() + if fr.start_time >= now: + msg = f"Waiting for reservation:{fr.name} to start at {fr.start_time}" + else: + msg = f"Reservation:{fr.name} is after its end-time" + return NodeActionDown(reason=msg) + +def _find_tpu_node_action(nodename, state) -> NodeAction: + lkp = lookup() + tpuobj = tpu.TPU.make(lkp.node_nodeset_name(nodename), lkp) + inst = tpuobj.get_node(nodename) + # If we do not find the node but it is from a Tpu that has multiple vms look for the master node + if inst is None and tpuobj.vmcount > 1: + # Get the tpu slurm nodelist of the nodes in the same tpu group as nodename + nodelist = run( + f"{lkp.scontrol} show topo {nodename}" + + " | awk -F'=' '/Level=0/ { print $NF }'", + shell=True, + ).stdout + l_nodelist = util.to_hostnames(nodelist) + group_names = set(l_nodelist) + # get the list of all the existing tpus in the nodeset + tpus_list = set(tpuobj.list_node_names()) + # In the intersection there must be only one node that is the master + tpus_int = list(group_names.intersection(tpus_list)) + if len(tpus_int) > 1: + log.error( + f"More than one cloud tpu node for tpu group {nodelist}, there should be only one that should be {l_nodelist[0]}, but we have found {tpus_int}" + ) + return NodeActionUnknown(slurm_state=state, instance_state=None) + if len(tpus_int) == 1: + inst = tpuobj.get_node(tpus_int[0]) + # if len(tpus_int ==0) this case is not relevant as this would be the case always that a TPU group is not running + if inst is None: + if state.base == "DOWN" and "POWERED_DOWN" in state.flags: + return NodeActionIdle() + if "POWERING_DOWN" in state.flags: + return NodeActionIdle() + if "COMPLETING" in state.flags: + return NodeActionDown(reason="Unbacked instance") + if state.base != "DOWN" and not ( + set(("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN")) + & state.flags + ): + return NodeActionDown(reason="Unbacked instance") + if lkp.is_static_node(nodename): + return NodeActionPowerUp() + elif ( + state is not None + and "POWERED_DOWN" not in state.flags + and "POWERING_DOWN" not in state.flags + and inst.state == tpu.TPU.State.STOPPED + ): + if tpuobj.preemptible: + return NodeActionPrempt() + if state.base != "DOWN": + return NodeActionDown(reason="Instance terminated") + elif ( + state is None or "POWERED_DOWN" in state.flags + ) and inst.state == tpu.TPU.State.READY: + return NodeActionDelete() + elif state is None: + # if state is None here, the instance exists but it's not in Slurm + return NodeActionUnknown(slurm_state=state, instance_state=inst.status) + + return NodeActionUnchanged() + +def get_node_action(nodename: str) -> NodeAction: + """Determine node/instance status that requires action""" + lkp = lookup() + state = lkp.node_state(nodename) + + if lkp.node_is_gke(nodename): + return NodeActionUnchanged() + + if lkp.node_is_fr(nodename): + fr = lkp.future_reservation(lkp.node_nodeset(nodename)) + assert fr + if action := get_fr_action(fr, state): + return action + + if lkp.node_is_dyn(nodename): + return _find_dynamic_node_status() + + if lkp.node_is_tpu(nodename): + return _find_tpu_node_action(nodename, state) + + # split below is workaround for VMs whose hostname is FQDN + inst = lkp.instance(nodename.split(".")[0]) + power_flags = frozenset( + ("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN") + ) & (state.flags if state is not None else set()) + + if (state is None) and (inst is None): + # Should never happen + return NodeActionUnknown(None, None) + if inst is None: + assert state is not None # to keep type-checker happy + if "POWERING_UP" in state.flags: + return NodeActionUnchanged() + if state.base == "DOWN" and "POWERED_DOWN" in state.flags: + return NodeActionIdle() + if "POWERING_DOWN" in state.flags: + return NodeActionIdle() + if "COMPLETING" in state.flags: + return NodeActionDown(reason="Unbacked instance") + if state.base != "DOWN" and not power_flags: + return NodeActionDown(reason="Unbacked instance") + if state.base == "DOWN" and not power_flags: + return NodeActionPowerDown() + if "NOT_RESPONDING" in state.flags: + return NodeActionPowerDown() + if "POWERED_DOWN" in state.flags and lkp.is_static_node(nodename): + return NodeActionPowerUp() + elif ( + state is not None + and "POWERED_DOWN" not in state.flags + and "POWERING_DOWN" not in state.flags + and inst.status == "TERMINATED" + ): + if inst.scheduling.preemptible: + return NodeActionPrempt() + if state.base != "DOWN": + return NodeActionDown(reason="Instance terminated") + elif (state is None or "POWERED_DOWN" in state.flags) and inst.status == "RUNNING": + log.info("%s is potential orphan node", nodename) + threshold = timedelta(seconds=90) + age = util.now() - inst.creation_timestamp + log.info(f"{nodename} state: {state}, age: {age}") + if age < threshold: + log.info(f"{nodename} not marked as orphan, it started less than {threshold.seconds}s ago ({age.seconds}s)") + return NodeActionUnchanged() + return NodeActionDelete() + elif state is None: + # if state is None here, the instance exists but it's not in Slurm + return NodeActionUnknown(slurm_state=state, instance_state=inst.status) + elif lkp.is_flex_node(nodename) and "POWERING_UP" in state.flags: + threshold = timedelta(seconds=int(lkp.cfg.compute_startup_scripts_timeout) * 2) #extra buffer for unexpectedly long startup scripts + if util.now() - inst.creation_timestamp > threshold: + log.info(f"{nodename} was unable to join the cluster after {threshold.seconds}s, potential failure on VM startup. Powering down...") + return NodeActionPowerDownForce() + return NodeActionUnchanged() + + +def delete_resource_policies(links: list[str], lkp: util.Lookup) -> None: + requests = {} + for link in links: + name = util.trim_self_link(link) + region = util.parse_self_link(link).region + requests[name] = lkp.compute.resourcePolicies().delete(project=lkp.project, region=region, resourcePolicy=name) + + def swallow_err(_: str) -> None: + pass + + done, failed = batch_execute(requests, log_err=swallow_err) + if failed: + # Filter out resourceInUseByAnotherResource errors , they are expected to happen + def ignore_err(e) -> bool: + return "resourceInUseByAnotherResource" in str(e) + + failures = [f"{n}: {e}" for n, (_, e) in failed.items() if not ignore_err(e)] + if failures: + log.error(f"some placement groups failed to delete: {failures}") + log.info( + f"deleted {len(done)} of {len(links)} placement groups ({to_hostlist(done.keys())})" + ) + + + +@lru_cache +def _get_resource_policies_in_region(lkp: util.Lookup, region: str) -> list[Any]: + res = [] + act = lkp.compute.resourcePolicies() + op = act.list(project=lkp.project, region=region) + prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-" + while op is not None: + result = ensure_execute(op) + res.extend([p for p in result.get("items", []) if p.get("name", "").startswith(prefix)]) + op = act.list_next(op, result) + return res + + +@lru_cache +def _get_resource_policies(lkp: util.Lookup) -> list[Any]: + res = [] + for region in lkp.cluster_regions(): + res.extend(_get_resource_policies_in_region(lkp, region)) + return res + +def sync_placement_groups(): + """Delete placement policies that are for jobs that have completed/terminated""" + keep_states = frozenset( + [ + "RUNNING", + "CONFIGURING", + "STOPPED", + "SUSPENDED", + "COMPLETING", + "PENDING", + ] + ) + + lkp = lookup() + keep_jobs = { + str(job.id) + for job in lkp.get_jobs() + if job.job_state in keep_states + } + keep_jobs.add("0") # Job 0 is a placeholder for static node placement + + to_delete = [] + pg_regex = re.compile( + rf"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-(?P[^\s\-]+)-(?P\d+)-(?P\d+)" + ) + + for pg in _get_resource_policies(lkp): + name = pg["name"] + + if (mtch := pg_regex.match(name)) is None: + log.warning(f"Unexpected resource policy {name=}") + continue + if mtch.group("job_id") not in keep_jobs: + to_delete.append(pg["selfLink"]) + + if to_delete: + delete_resource_policies(to_delete, lkp) + + +def sync_instances(): + compute_instances = { + name for name, inst in lookup().instances().items() if inst.role == "compute" + } + slurm_nodes = set(lookup().slurm_nodes().keys()) + log.debug(f"reconciling {len(compute_instances)} GCP instances and {len(slurm_nodes)} Slurm nodes.") + + for action, nodes in util.groupby_unsorted(list(compute_instances | slurm_nodes), get_node_action): + action.apply(list(nodes)) + + +def reconfigure_slurm(): + update_msg = "*** slurm configuration was updated ***" + if lookup().cfg.hybrid: + # terraform handles generating the config.yaml, don't do it here + return + + upd, cfg_new = util.fetch_config() + if not upd: + log.debug("No changes in config detected.") + return + log.debug("Changes in config detected. Reconfiguring Slurm now.") + util.update_config(cfg_new) + + if lookup().is_controller: + conf.gen_controller_configs(lookup()) + log.info("Restarting slurmctld to make changes take effect.") + try: + # TODO: consider removing "restart" since "reconfigure" should restart slurmctld as well + run("sudo systemctl restart slurmctld.service", check=False) + util.scontrol_reconfigure(lookup()) + except Exception: + log.exception("failed to reconfigure slurmctld") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + elif lookup().instance_role_safe == "compute": + log.info("Restarting slurmd to make changes take effect.") + run("systemctl restart slurmd") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + elif lookup().is_login_node: + log.info("Restarting sackd to make changes take effect.") + run("systemctl restart sackd") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + + +def update_topology(lkp: util.Lookup) -> None: + if conf.topology_plugin(lkp) != conf.TOPOLOGY_PLUGIN_TREE: + return + updated, summary = conf.gen_topology_conf(lkp) + if updated: + log.info("Topology configuration updated. Reconfiguring Slurm.") + util.scontrol_reconfigure(lkp) + # Safe summary only after Slurm got reconfigured, so summary reflects Slurm POV + summary.dump(lkp) + + +def delete_reservation(lkp: util.Lookup, reservation_name: str) -> None: + util.run(f"{lkp.scontrol} delete reservation {reservation_name}") + + +def create_reservation(lkp: util.Lookup, reservation_name: str, node: str, start_time: datetime) -> None: + # Format time to be compatible with slurm reservation. + formatted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S') + + util.run(f"{lkp.scontrol} create reservation user=slurm starttime={formatted_start_time} duration=180 nodes={node} reservationname={reservation_name} flags=maint,ignore_jobs") + + +def get_slurm_reservation_maintenance(lkp: util.Lookup) -> Dict[str, datetime]: + res = util.run(f"{lkp.scontrol} show reservation --json") + all_reservations = json.loads(res.stdout) + reservation_map = {} + + for reservation in all_reservations['reservations']: + name = reservation.get('name') + nodes = reservation.get('node_list') + time_epoch = reservation.get('start_time', {}).get('number') + + if name is None or nodes is None or time_epoch is None: + continue + + if reservation.get('node_count') != 1: + continue + + if name != f"{nodes}_maintenance": + continue + + reservation_map[name] = datetime.fromtimestamp(time_epoch) + + return reservation_map + +@lru_cache +def get_upcoming_maintenance(lkp: util.Lookup) -> Dict[str, Tuple[str, datetime]]: + upc_maint_map = {} + + for node, inst in lkp.instances().items(): + if inst.resource_status.upcoming_maintenance: + upc_maint_map[node + "_maintenance"] = (node, inst.resource_status.upcoming_maintenance.window_start_time) + + return upc_maint_map + + +def sync_maintenance_reservation(lkp: util.Lookup) -> None: + upc_maint_map = get_upcoming_maintenance(lkp) # map reservation_name -> (node_name, time) + log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") + + curr_reservation_map = get_slurm_reservation_maintenance(lkp) # map reservation_name -> time + log.debug(f"curr-reservation-map: {curr_reservation_map}") + + del_reservation = set(curr_reservation_map.keys() - upc_maint_map.keys()) + create_reservation_map = {} + + for res_name, (node, start_time) in upc_maint_map.items(): + try: + enabled = lkp.node_nodeset(node).enable_maintenance_reservation + except Exception: + enabled = False + + if not enabled: + if res_name in curr_reservation_map: + del_reservation.add(res_name) + continue + + if res_name in curr_reservation_map: + diff = curr_reservation_map[res_name] - start_time + if abs(diff) <= timedelta(seconds=1): + continue + else: + del_reservation.add(res_name) + create_reservation_map[res_name] = (node, start_time) + else: + create_reservation_map[res_name] = (node, start_time) + + log.debug(f"del-reservation: {del_reservation}") + for res_name in del_reservation: + delete_reservation(lkp, res_name) + + log.debug(f"create-reservation-map: {create_reservation_map}") + for res_name, (node, start_time) in create_reservation_map.items(): + create_reservation(lkp, res_name, node, start_time) + + +def delete_maintenance_job(job_name: str) -> None: + util.run(f"scancel --name={job_name}") + + +def create_maintenance_job(job_name: str, node: str) -> None: + util.run(f"sbatch --job-name={job_name} --nodelist={node} {_MAINTENANCE_SBATCH_SCRIPT_PATH}") + + +def get_slurm_maintenance_job(lkp: util.Lookup) -> Dict[str, str]: + jobs = {} + + for job in lkp.get_jobs(): + if job.name is None or job.required_nodes is None or job.job_state is None: + continue + + if job.name != f"{job.required_nodes}_maintenance": + continue + + if job.job_state != "PENDING": + continue + + jobs[job.name] = job.required_nodes + + return jobs + + +def sync_opportunistic_maintenance(lkp: util.Lookup) -> None: + upc_maint_map = get_upcoming_maintenance(lkp) # map job_name -> (node_name, time) + log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") + + curr_jobs = get_slurm_maintenance_job(lkp) # map job_name -> node. + log.debug(f"curr-maintenance-job-map: {curr_jobs}") + + del_jobs = set(curr_jobs.keys() - upc_maint_map.keys()) + create_jobs = {} + + for job_name, (node, _) in upc_maint_map.items(): + try: + enabled = lkp.node_nodeset(node).enable_opportunistic_maintenance + except Exception: + enabled = False + + if not enabled: + if job_name in curr_jobs: + del_jobs.add(job_name) + continue + + if job_name not in curr_jobs: + create_jobs[job_name] = node + + log.debug(f"del-maintenance-job: {del_jobs}") + for job_name in del_jobs: + delete_maintenance_job(job_name) + + log.debug(f"create-maintenance-job: {create_jobs}") + for job_name, node in create_jobs.items(): + create_maintenance_job(job_name, node) + + + +def sync_flex_migs(lkp: util.Lookup) -> None: + pass + + +def process_messages(lkp: util.Lookup) -> None: + try: + watch_delete_vm_op.watch_vm_delete_ops(lkp) + except: + log.exception("failed during watching delete VM operations") + + +def main(): + lkp = lookup() + if util.should_mount_slurm_bucket() and not lkp.is_controller: + return + try: + reconfigure_slurm() + except Exception: + log.exception("failed to reconfigure slurm") + if lkp.is_controller: + try: + process_messages(lkp) + except: + log.exception("failed to process messages") + + try: + sync_instances() + except Exception: + log.exception("failed to sync instances") + + try: + sync_flex_migs(lkp) + except Exception: + log.exception("failed to sync DWS Flex MIGs") + + try: + sync_placement_groups() + except Exception: + log.exception("failed to sync placement groups") + + try: + update_topology(lkp) + except Exception: + log.exception("failed to update topology") + + try: + sync_maintenance_reservation(lkp) + except Exception: + log.exception("failed to sync slurm reservation for scheduled maintenance") + + try: + sync_opportunistic_maintenance(lkp) + except Exception: + log.exception("failed to sync opportunistic reservation for scheduled maintenance") + + + try: + # TODO: it performs 1 to 4 GCS list requests, + # use cached version, combine with `_list_config_blobs` + install_custom_scripts(check_hash=True) + except Exception: + log.exception("failed to sync custom scripts") + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + _ = util.init_log_and_parse(parser) + + pid_file = (Path("/tmp") / Path(__file__).name).with_suffix(".pid") + with pid_file.open("w") as fp: + try: + fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) + main() + except BlockingIOError: + sys.exit(0) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py new file mode 100644 index 0000000000..ae36c54222 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py @@ -0,0 +1,171 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script sorts nodes based on their `physicalHost`. + +See https://cloud.google.com/compute/docs/instances/use-compact-placement-policies + +You can reduce latency in tightly coupled HPC workloads (including distributed ML training) +by deploying them to machines that are located close together. +For example, if you deploy your workload on a single physical rack, you can expect lower latency +than if your workload is spread across multiple racks. +Sending data across multiple rack requires sending data through additional network switches. + +Example usage: +``` my_sbatch.sh +#SBATCH --ntasks-per-node=8 +#SBATCH --nodes=64 + +export SLURM_HOSTFILE=$(sort_nodes.py) + +srun -l hostname | sort +``` +""" +import os +import subprocess +import uuid +from typing import List, Optional, Dict +from collections import OrderedDict + +def order(paths: List[List[str]]) -> List[str]: + """ + Orders the leaves of the tree in a way that minimizes the sum of distance in between + each pair of neighboring nodes in the resulting order. + The resulting order will always start from the first node in the input list. + The ordering is "stable" with respect to the input order of the leaves i.e. + given a choice between two nodes (identical in other ways) it will select "nodelist-smallest" one. + + Returns a list of nodenames, ordered as described above. + """ + if not paths: return [] + class Vert: + "Represents a vertex in a *network* tree." + def __init__(self, name: str, parent: Optional["Vert"]): + self.name = name + self.parent = parent + # Use `OrderedDict` to preserve insertion order + # TODO: once we move to Python 3.7+ use regular `dict` since it has the same guarantee + self.children: OrderedDict = OrderedDict() + + # build a tree, children are ordered by insertion order + root = Vert("", None) + for path in paths: + n = root + for v in path: + if v not in n.children: + n.children[v] = Vert(v, n) + n = n.children[v] + + # walk the tree in insertion order, gather leaves + result = [] + def gather_nodes(v: Vert) -> None: + if not v.children: # this is a Slurm node + result.append(v.name) + for u in v.children.values(): + gather_nodes(u) + gather_nodes(root) + return result + + +class Instance: + def __init__(self, name: str, zone: str, physical_host: Optional[str]): + self.name = name + self.zone = zone + self.physical_host = physical_host + + +def make_path(node_name: str, inst: Optional[Instance]) -> List[str]: + if not inst: # node with unknown instance (e.g. hybrid cluster) + return ["unknown", node_name] + zone = f"zone_{inst.zone}" + if not inst.physical_host: # node without physical host info (e.g. no placement policy) + return [zone, "unknown", node_name] + + assert inst.physical_host.startswith("/"), f"Unexpected physicalHost: {inst.physical_host}" + parts = inst.physical_host[1:].split("/") + if len(parts) >= 4: + return [*parts, node_name] + return [zone, *parts, node_name] + + +def to_hostnames(nodelist: str) -> List[str]: + cmd = ["scontrol", "show", "hostnames", nodelist] + out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout + return [n.decode("utf-8") for n in out.splitlines()] + + +def get_instances(node_names: List[str]) -> Dict[str, Optional[Instance]]: + fmt = ( + "--format=csv[no-heading,separator=','](zone,resourceStatus.physicalHost,name)" + ) + cmd = ["gcloud", "compute", "instances", "list", fmt] + + scp = os.path.commonprefix(node_names) + if scp: + cmd.append(f"--filter=name~'{scp}.*'") + out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout + d = {} + for line in out.splitlines(): + zone, physical_host, name = line.decode("utf-8").split(",") + d[name] = Instance(name, zone, physical_host) + return {n: d.get(n) for n in node_names} + + +def main(args) -> None: + nodelist = args.nodelist or os.getenv("SLURM_NODELIST") + if not nodelist: + raise ValueError("nodelist is not provided and SLURM_NODELIST is not set") + + if args.ntasks_per_node is None: + args.ntasks_per_node = int(os.getenv("SLURM_NTASKS_PER_NODE", "") or 1) + assert args.ntasks_per_node > 0 + + output = args.output or f"hosts.{uuid.uuid4()}" + + node_names = to_hostnames(nodelist) + instannces = get_instances(node_names) + paths = [make_path(n, instannces[n]) for n in node_names] + ordered = order(paths) + + with open(output, "w") as f: + for node in ordered: + for _ in range(args.ntasks_per_node): + f.write(node) + f.write("\n") + print(output) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--nodelist", + type=str, + help="Slurm 'hostlist expression' of nodes to sort, if not set the value of SLURM_NODELIST environment variable will be used", + ) + parser.add_argument( + "--ntasks-per-node", + type=int, + help="""Number of times to repeat each node in resulting sorted list. +If not set, the value of SLURM_NTASKS_PER_NODE environment variable will be used, +if neither is set, defaults to 1""", + ) + parser.add_argument( + "--output", type=str, help="Output file to write, defaults to 'hosts.'" + ) + args = parser.parse_args() + main(args) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py new file mode 100644 index 0000000000..ecef70f1cc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py @@ -0,0 +1,126 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Any +import argparse +import logging + +import util +from util import ( + log_api_request, + batch_execute, + to_hostlist, + separate, +) +from util import lookup +import tpu +import mig_flex +import watch_delete_vm_op + +log = logging.getLogger() + +TOT_REQ_CNT = 1000 + + +def truncate_iter(iterable, max_count): + end = "..." + _iter = iter(iterable) + for i, el in enumerate(_iter, start=1): + if i >= max_count: + yield end + break + yield el + + +def delete_instance_request(name: str) -> Any: + inst = lookup().instance(name) + assert inst + + request = lookup().compute.instances().delete( + project=lookup().project, + zone=inst.zone, + instance=name, + ) + log_api_request(request) + return request + + +def delete_instances(instances): + """delete instances individually""" + invalid, valid = separate(lambda inst: bool(lookup().instance(inst)), instances) + if len(invalid) > 0: + log.debug("instances do not exist: {}".format(",".join(invalid))) + if len(valid) == 0: + log.debug("No instances to delete") + return + + requests = {inst: delete_instance_request(inst) for inst in valid} + + log.info(f"to delete {len(valid)} instances ({to_hostlist(valid)})") + ops, failed = batch_execute(requests) + for node, (_, err) in failed.items(): + log.error(f"instance {node} failed to delete: {err}") + + log.info(f"deleting {len(ops)} instances {to_hostlist(ops.keys())}") + + topic = watch_delete_vm_op.watch_delete_vm_op_topic() + for node, op in ops.items(): + topic.publish(op, node) + + + + +def suspend_nodes(nodes: List[str]) -> None: + lkp = lookup() + other_nodes, tpu_nodes = util.separate(lkp.node_is_tpu, nodes) + bulk_nodes, flex_nodes = util.separate(lkp.is_flex_node, other_nodes) + + mig_flex.suspend_flex_nodes(flex_nodes, lkp) + delete_instances(bulk_nodes) + tpu.delete_tpu_instances(tpu_nodes) + + +def main(nodelist): + """main called when run as script""" + log.debug(f"SuspendProgram {nodelist}") + + # Filter out nodes not in config.yaml + other_nodes, pm_nodes = separate( + lookup().is_power_managed_node, util.to_hostnames(nodelist) + ) + if other_nodes: + log.debug( + f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" + ) + if pm_nodes: + log.debug(f"Suspending nodes '{to_hostlist(pm_nodes)}' from '{nodelist}'") + else: + log.debug("No cloud nodes to suspend") + return + + log.info(f"suspend {nodelist}") + suspend_nodes(pm_nodes) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("nodelist", help="list of nodes to suspend") + args = util.init_log_and_parse(parser) + + main(args.nodelist) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh new file mode 100644 index 0000000000..9079e4e4b0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PYTHON_SCRIPT="${SCRIPT_DIR}/suspend.py" + +# Capture all arguments passed by Slurm (the nodelist). +ALL_ARGS=("$@") + +"${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & +disown + +exit 0 diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py new file mode 100644 index 0000000000..0ce7fb5ec4 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py @@ -0,0 +1,116 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Any +import sys +from dataclasses import dataclass, field +from datetime import datetime + +SCRIPTS_DIR = "community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts" +if SCRIPTS_DIR not in sys.path: + sys.path.append(SCRIPTS_DIR) # TODO: make this more robust + +import util + + +SOME_TS = datetime.fromisoformat("2018-09-03T20:56:35.450686+00:00") +# TODO: use "real" classes once they are defined (instead of NSDict) + +@dataclass +class Placeholder: + pass + +@dataclass +class TstNodeset: + nodeset_name: str = "cantor" + node_count_static: int = 0 + node_count_dynamic_max: int = 0 + node_conf: dict[str, Any] = field(default_factory=dict) + instance_template: Optional[str] = None + reservation_name: Optional[str] = "" + zone_policy_allow: Optional[list[str]] = field(default_factory=list) + enable_placement: bool = True + placement_max_distance: Optional[int] = None + accelerator_topology: Optional[str] = "" + future_reservation: Optional[str] = "" + +@dataclass +class TstPartition: + partition_name: str = "euler" + partition_nodeset: list[str] = field(default_factory=list) + partition_nodeset_tpu: list[str] = field(default_factory=list) + enable_job_exclusive: bool = False + +@dataclass +class TstCfg: + slurm_cluster_name: str = "m22" + cloud_parameters: dict[str, Any] = field(default_factory=dict) + + partitions: dict[str, TstPartition] = field(default_factory=dict) + nodeset: dict[str, TstNodeset] = field(default_factory=dict) + nodeset_tpu: dict[str, TstNodeset] = field(default_factory=dict) + nodeset_dyn: dict[str, TstNodeset] = field(default_factory=dict) + + install_dir: Optional[str] = None + output_dir: Optional[str] = None + + prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + task_prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + task_epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + + +@dataclass +class TstTPU: # to prevent client initialization durint "TPU.__init__" + vmcount: int + +@dataclass +class TstMachineConf: + cpus: int + memory: int + sockets: int + sockets_per_board: int + cores_per_socket: int + boards: int + threads_per_core: int + + +@dataclass +class TstTemplateInfo: + gpu: Optional[util.AcceleratorInfo] + +def tstInstance(name: str, physical_host: Optional[str] = None): + return util.Instance( + name=name, + zone="anorien", + status="RUNNING", + creation_timestamp=SOME_TS, + resource_status=util.InstanceResourceStatus( + physical_host=physical_host, + upcoming_maintenance=None, + ), + scheduling=util.NSDict(), + role="compute", + metadata={}, + ) + +def make_to_hostnames_mock(tbl: Optional[dict[str, list[str]]]): + tbl = tbl or {} + + def se(k: str) -> list[str]: + if k not in tbl: + raise AssertionError(f"to_hostnames mock: unexpected nodelist: '{k}'") + return tbl[k] + + return se diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py new file mode 100644 index 0000000000..6bd6762748 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py @@ -0,0 +1,226 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from mock import Mock +from common import TstNodeset, TstCfg, TstMachineConf, TstTemplateInfo, Placeholder + +import addict # type: ignore +import conf +import util + + +def test_nodeset_tpu_lines(): + nodeset = TstNodeset( + "turbo", + node_count_static=2, + node_count_dynamic_max=3, + node_conf={"red": "velvet"}, + ) + assert conf.nodeset_tpu_lines(nodeset, util.Lookup(TstCfg())) == "\n".join( + [ + "NodeName=m22-turbo-[0-4] State=CLOUD red=velvet", + "NodeSet=turbo Nodes=m22-turbo-[0-4]", + ] + ) + + +def test_nodeset_lines(): + nodeset = TstNodeset( + "turbo", + node_count_static=2, + node_count_dynamic_max=3, + node_conf={"red": "velvet", "CPUs": 55}, + ) + lkp = util.Lookup(TstCfg()) + lkp.template_info = Mock(return_value=TstTemplateInfo( + gpu=util.AcceleratorInfo(type="Popov", count=33) + )) + mc = TstMachineConf( + cpus=5, + memory=6, + sockets=7, + sockets_per_board=8, + boards=9, + threads_per_core=10, + cores_per_socket=11, + ) + lkp.template_machine_conf = Mock(return_value=mc) # type: ignore[method-assign] + assert conf.nodeset_lines(nodeset, lkp) == "\n".join( + [ + "NodeName=m22-turbo-[0-4] State=CLOUD RealMemory=6 Boards=9 SocketsPerBoard=8 CoresPerSocket=11 ThreadsPerCore=10 CPUs=55 Gres=gpu:33 red=velvet", + "NodeSet=turbo Nodes=m22-turbo-[0-4]", + ] + ) + + +@pytest.mark.parametrize( + "value,want", + [ + ({"a": 1}, "a=1"), + ({"a": "two"}, "a=two"), + ({"a": [3, 4]}, "a=3,4"), + ({"a": ["five", "six"]}, "a=five,six"), + ({"a": None}, ""), + ({"a": ["seven", None, 8]}, "a=seven,8"), + ({"a": 1, "b": "two"}, "a=1 b=two"), + ({"a": 1, "b": None, "c": "three"}, "a=1 c=three"), + ({"a": 0, "b": None, "c": 0.0, "e": ""}, "a=0 c=0.0"), + ({"a": [0, 0.0, None, "X", "", "Y"]}, "a=0,0.0,X,,Y"), + ]) +def test_dict_to_conf(value: dict, want: str): + assert conf.dict_to_conf(value) == want + + + +@pytest.mark.parametrize( + "cfg,want", + [ + (TstCfg( + install_dir="ukulele", + ), + """LaunchParameters=enable_nss_slurm,use_interactive_step +SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend +SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +SlurmdTimeout=300 +UnkillableStepTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + (TstCfg( + install_dir="ukulele", + cloud_parameters={ + "no_comma_params": True, + "private_data": None, + "scheduler_parameters": None, + "resume_rate": None, + "resume_timeout": None, + "suspend_rate": None, + "suspend_timeout": None, + "unkillable_step_timeout": None, + "slurmd_timeout": None, + "topology_plugin": None, + "topology_param": None, + "tree_width": None, + }, + ), + """SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +SlurmdTimeout=300 +UnkillableStepTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + (TstCfg( + install_dir="ukulele", + cloud_parameters={ + "no_comma_params": True, + "private_data": [ + "events", + "jobs", + ], + "scheduler_parameters": [ + "bf_busy_nodes", + "bf_continue", + "ignore_prefer_validation", + "nohold_on_prolog_fail", + ], + "resume_rate": 1, + "resume_timeout": 2, + "suspend_rate": 3, + "suspend_timeout": 4, + "slurmd_timeout": 5, + "unkillable_step_timeout": 6, + "tree_width": 7, + "topology_plugin": "guess", + "topology_param": "yellow", + }, + ), + """PrivateData=events,jobs +SchedulerParameters=bf_busy_nodes,bf_continue,ignore_prefer_validation,nohold_on_prolog_fail +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=1 +ResumeTimeout=2 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=3 +SuspendTimeout=4 +SlurmdTimeout=5 +UnkillableStepTimeout=6 +TreeWidth=7 +TopologyPlugin=guess +TopologyParam=yellow"""), + (TstCfg( + install_dir="ukulele", + task_prolog_scripts=[Placeholder()], + task_epilog_scripts=[Placeholder()], + ), + """LaunchParameters=enable_nss_slurm,use_interactive_step +SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend +TaskProlog=/slurm/custom_scripts/task_prolog.d/task-prolog +TaskEpilog=/slurm/custom_scripts/task_epilog.d/task-epilog +SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +SlurmdTimeout=300 +UnkillableStepTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + ]) +def test_conflines(cfg, want): + assert conf.conflines(util.Lookup(cfg)) == want + + cfg.cloud_parameters = addict.Dict(cfg.cloud_parameters) + assert conf.conflines(util.Lookup(cfg)) == want + + +@pytest.mark.parametrize( + "cfg,gputype,gpucount,want", + [ + (TstCfg(), + "", + 0, + "\n"), + (TstCfg( + nodeset={"turbo": TstNodeset("turbo")} + ), + "Popov", + 8, + "Name=gpu Type=Popov File=/dev/nvidia[0-7]\n\n"), + ]) +def test_gen_cloud_gres_conf_lines(cfg, gputype, gpucount, want): + lkp = util.Lookup(cfg) + lkp.template_info = Mock(return_value=TstTemplateInfo( + gpu=util.AcceleratorInfo(type=gputype, count=gpucount) + )) + assert conf.gen_cloud_gres_conf_lines(lkp) == want diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py new file mode 100644 index 0000000000..77f1229605 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py @@ -0,0 +1,175 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import os +import pytest +import unittest.mock +import unittest +import tempfile + +from common import TstCfg, TstNodeset, TstPartition, TstTPU # needed to import util +import util +import resume +from resume import ResumeData, ResumeJobData, BulkChunk, PlacementAndNodes + +def test_get_resume_file_data_no_env(): + with unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": ""}): + assert resume.get_resume_file_data() is None + + +def test_get_resume_file_data(): + with tempfile.NamedTemporaryFile() as f: + f.write(b"""{ + "jobs": [ + { + "extra": null, + "job_id": 1, + "features": null, + "nodes_alloc": "green-[0-2]", + "nodes_resume": "green-[0-1]", + "oversubscribe": "OK", + "partition": "red", + "reservation": null + } + ], + "all_nodes_resume": "green-[0-1]" +}""") + f.flush() + with ( + unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": f.name}), + unittest.mock.patch("util.to_hostnames") as mock_to_hostnames, + ): + mock_to_hostnames.return_value = ["green-0", "green-1", "green-2"] + assert resume.get_resume_file_data() == ResumeData(jobs=[ + ResumeJobData( + job_id = 1, + partition="red", + nodes_alloc=["green-0", "green-1", "green-2"], + ) + ]) + mock_to_hostnames.assert_called_once_with("green-[0-2]") + + +@unittest.mock.patch("tpu.TPU.make") +@unittest.mock.patch("resume.create_placements") +def test_group_nodes_bulk(mock_create_placements, mock_tpu): + cfg = TstCfg( + nodeset={ + "n": TstNodeset(nodeset_name="n"), + }, + nodeset_tpu={ + "t": TstNodeset(nodeset_name="t"), + }, + partitions={ + "p1": TstPartition( + partition_name="p1", + enable_job_exclusive=True, + ), + "p2": TstPartition( + partition_name="p2", + partition_nodeset_tpu=["t"], + enable_job_exclusive=True, + ) + } + ) + lkp = util.Lookup(cfg) + + def mock_create_placements_se(nodes, excl_job_id, lkp): + args = (set(nodes), excl_job_id) + if ({'c-n-1', 'c-n-2', 'c-t-8', 'c-t-9'}, None) == args: + return [ + PlacementAndNodes("g0", ["c-n-1", "c-n-2"]), + PlacementAndNodes(None, ['c-t-8', 'c-t-9']), + ] + if ({"c-n-0", "c-n-8"}, 1) == args: + return [ + PlacementAndNodes("g10", ["c-n-0"]), + PlacementAndNodes("g11", ["c-n-8"]), + ] + if ({'c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5'}, 2) == args: + return [ + PlacementAndNodes(None, ['c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5']) + ] + raise AssertionError(f"unexpected invocation: '{args}'") + mock_create_placements.side_effect = mock_create_placements_se + + def mock_tpu_se(ns: str, lkp) -> TstTPU: + if ns == "t": + return TstTPU(vmcount=2) + raise AssertionError(f"unexpected invocation: '{ns}'") + mock_tpu.side_effect = mock_tpu_se + + got = resume.group_nodes_bulk( + ["c-n-0", "c-n-1", "c-n-2", "c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-8", "c-t-9"], + ResumeData(jobs=[ + ResumeJobData(job_id=1, partition="p1", nodes_alloc=["c-n-0", "c-n-8"]), + ResumeJobData(job_id=2, partition="p2", nodes_alloc=["c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-4", "c-t-5"]), + ]), lkp) + mock_create_placements.assert_called() + assert got == { + "c-n:jobNone:g0:0": BulkChunk( + nodes=["c-n-1", "c-n-2"], prefix="c-n", chunk_idx=0, excl_job_id=None, placement_group="g0"), + "c-n:job1:g10:0": BulkChunk( + nodes=["c-n-0"], prefix="c-n", chunk_idx=0, excl_job_id=1, placement_group="g10"), + "c-t:0": BulkChunk( + nodes=["c-t-8", "c-t-9"], prefix="c-t", chunk_idx=0, excl_job_id=None, placement_group=None), + "c-t:job2:0": BulkChunk( + nodes=["c-t-0", "c-t-1"], prefix="c-t", chunk_idx=0, excl_job_id=2, placement_group=None), + "c-t:job2:1": BulkChunk( + nodes=["c-t-2", "c-t-3"], prefix="c-t", chunk_idx=1, excl_job_id=2, placement_group=None), + } + + +@pytest.mark.parametrize( + "nodes,excl_job_id,expected", + [ + ( # TPU - no placements + ["c-t-0", "c-t-2"], 4, [PlacementAndNodes(None, ["c-t-0", "c-t-2"])] + ), + ( # disabled placements - no placemens + ["c-x-0", "c-x-2"], 4, [PlacementAndNodes(None, ["c-x-0", "c-x-2"])] + ), + ( # excl_job + ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], 4, [ + PlacementAndNodes("c-slurmgcp-managed-n-4-0", ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"]) + ] + ), + ( # no excl_job + ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], None, [ + PlacementAndNodes("c-slurmgcp-managed-n-0-0", ["c-n-0", "c-n-2"]), + PlacementAndNodes('c-slurmgcp-managed-n-0-1', ['c-n-2011']), + PlacementAndNodes(None, ["c-n-uno"]), + ] + ), + ], +) +def test_allocate_nodes_to_placements(nodes: list[str], excl_job_id: Optional[int], expected: list[PlacementAndNodes]): + cfg = TstCfg( + slurm_cluster_name="c", + nodeset={ + "n": TstNodeset(nodeset_name="n", enable_placement=True), + "x": TstNodeset(nodeset_name="x", enable_placement=False) + }, + nodeset_tpu={ + "t": TstNodeset(nodeset_name="t") + }) + lkp = util.Lookup(cfg) + + with unittest.mock.patch("resume.valid_placement_node") as mock_valid_placement_node: + mock_valid_placement_node.return_value = True + lkp.template_info = unittest.mock.Mock(return_value=unittest.mock.Mock(machine_type=unittest.mock.Mock(family="n1"))) + + assert resume._allocate_nodes_to_placements(nodes, excl_job_id, lkp) == expected diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py new file mode 100644 index 0000000000..df9f3a0137 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py @@ -0,0 +1,215 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import json +import mock +from pytest_unordered import unordered +from common import TstCfg, TstNodeset, TstTPU, tstInstance +import sort_nodes + +import util +import conf +import tempfile + +PRELUDE = """ +# Warning: +# This file is managed by a script. Manual modifications will be overwritten. + +""" + +def test_gen_topology_conf_empty(): + out_dir = tempfile.mkdtemp() + cfg = TstCfg(output_dir=out_dir) + conf.gen_topology_conf(util.Lookup(cfg)) + assert open(out_dir + "/cloud_topology.conf").read() == PRELUDE + "\n" + + +@mock.patch("tpu.TPU.make") +def test_gen_topology_conf(tpu_mock): + output_dir = tempfile.mkdtemp() + cfg = TstCfg( + nodeset_tpu={ + "a": TstNodeset("bold", node_count_static=4, node_count_dynamic_max=5), + "b": TstNodeset("slim", node_count_dynamic_max=3), + }, + nodeset={ + "c": TstNodeset("green", node_count_static=2, node_count_dynamic_max=3), + "d": TstNodeset("blue", node_count_static=7), + "e": TstNodeset("pink", node_count_dynamic_max=4), + }, + output_dir=output_dir, + ) + + def tpu_se(ns: str, lkp) -> TstTPU: + if ns == "bold": + return TstTPU(vmcount=3) + if ns == "slim": + return TstTPU(vmcount=1) + raise AssertionError(f"unexpected TPU name: '{ns}'") + + tpu_mock.side_effect = tpu_se + + lkp = util.Lookup(cfg) + lkp.instances = lambda: { n.name: n for n in [ # type: ignore[assignment] + # nodeset blue + tstInstance("m22-blue-0"), # no physicalHost + tstInstance("m22-blue-0", physical_host="/a/a/a"), + tstInstance("m22-blue-1", physical_host="/a/a/b"), + tstInstance("m22-blue-2", physical_host="/a/b/a"), + tstInstance("m22-blue-3", physical_host="/b/a/a"), + # nodeset green + tstInstance("m22-green-3", physical_host="/a/a/c"), + ]} + + uncompressed = conf.gen_topology(lkp) + want_uncompressed = [ + #NOTE: the switch names are not unique, it's not valid content for topology.conf + # The uniquefication and compression of names are done in the compress() method + "SwitchName=slurm-root Switches=a,b,ns_blue,ns_green,ns_pink", + # "physical" topology + 'SwitchName=a Switches=a,b', + 'SwitchName=a Nodes=m22-blue-[0-1],m22-green-3', + 'SwitchName=b Nodes=m22-blue-2', + 'SwitchName=b Switches=a', + 'SwitchName=a Nodes=m22-blue-3', + # topology "by nodeset" + "SwitchName=ns_blue Nodes=m22-blue-[4-6]", + "SwitchName=ns_green Nodes=m22-green-[0-2,4]", + "SwitchName=ns_pink Nodes=m22-pink-[0-3]", + # TPU topology + "SwitchName=tpu-root Switches=ns_bold,ns_slim", + "SwitchName=ns_bold Switches=bold-[0-3]", + "SwitchName=bold-0 Nodes=m22-bold-[0-2]", + "SwitchName=bold-1 Nodes=m22-bold-3", + "SwitchName=bold-2 Nodes=m22-bold-[4-6]", + "SwitchName=bold-3 Nodes=m22-bold-[7-8]", + "SwitchName=ns_slim Nodes=m22-slim-[0-2]"] + assert list(uncompressed.render_conf_lines()) == want_uncompressed + + compressed = uncompressed.compress() + want_compressed = [ + "SwitchName=s0 Switches=s0_[0-4]", # root + # "physical" topology + 'SwitchName=s0_0 Switches=s0_0_[0-1]', # /a + 'SwitchName=s0_0_0 Nodes=m22-blue-[0-1],m22-green-3', # /a/a + 'SwitchName=s0_0_1 Nodes=m22-blue-2', # /a/b + 'SwitchName=s0_1 Switches=s0_1_0', # /b + 'SwitchName=s0_1_0 Nodes=m22-blue-3', # /b/a + # topology "by nodeset" + "SwitchName=s0_2 Nodes=m22-blue-[4-6]", + "SwitchName=s0_3 Nodes=m22-green-[0-2,4]", + "SwitchName=s0_4 Nodes=m22-pink-[0-3]", + # TPU topology + "SwitchName=s1 Switches=s1_[0-1]", + "SwitchName=s1_0 Switches=s1_0_[0-3]", + "SwitchName=s1_0_0 Nodes=m22-bold-[0-2]", + "SwitchName=s1_0_1 Nodes=m22-bold-3", + "SwitchName=s1_0_2 Nodes=m22-bold-[4-6]", + "SwitchName=s1_0_3 Nodes=m22-bold-[7-8]", + "SwitchName=s1_1 Nodes=m22-slim-[0-2]"] + assert list(compressed.render_conf_lines()) == want_compressed + + upd, summary = conf.gen_topology_conf(lkp) + assert upd == True + want_written = PRELUDE + "\n".join(want_compressed) + "\n\n" + assert open(output_dir + "/cloud_topology.conf").read() == want_written + + summary.dump(lkp) + summary_got = json.loads(open(output_dir + "/cloud_topology.summary.json").read()) + + assert summary_got == { + "down_nodes": unordered( + [f"m22-blue-{i}" for i in (4,5,6)] + + [f"m22-green-{i}" for i in (0,1,2,4)] + + [f"m22-pink-{i}" for i in range(4)]), + "tpu_nodes": unordered( + [f"m22-bold-{i}" for i in range(9)] + + [f"m22-slim-{i}" for i in range(3)]), + 'physical_host': { + 'm22-blue-0': '/a/a/a', + 'm22-blue-1': '/a/a/b', + 'm22-blue-2': '/a/b/a', + 'm22-blue-3': '/b/a/a', + 'm22-green-3': '/a/a/c'}, + } + + + +def test_gen_topology_conf_update(): + cfg = TstCfg( + nodeset={ + "c": TstNodeset("green", node_count_static=2), + }, + output_dir=tempfile.mkdtemp(), + ) + lkp = util.Lookup(cfg) + lkp.instances = lambda: { # type: ignore[assignment] + # no instances + } + + # initial generation - reconfigure + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # add node: node_count_static 2 -> 3 - reconfigure + lkp.cfg.nodeset["c"].node_count_static = 3 + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # remove node: node_count_static 3 -> 2 - no reconfigure + lkp.cfg.nodeset["c"].node_count_static = 2 + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + # set empty physicalHost - no reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + # set physicalHost - reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/c")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # change physicalHost - reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/z")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # shut down node - no reconfigure + lkp.instances = lambda: {} # type: ignore[assignment] + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + +@pytest.mark.parametrize( + "paths,expected", + [ + (["z/n-0", "z/n-1", "z/n-2", "z/n-3", "z/n-4", "z/n-10"], ['n-0', 'n-1', 'n-2', 'n-3', 'n-4', 'n-10']), + (["y/n-0", "z/n-1", "x/n-2", "x/n-3", "y/n-4", "g/n-10"], ['n-0', 'n-4', 'n-1', 'n-2', 'n-3', 'n-10']), + ]) +def test_sort_nodes_order(paths: list[str], expected: list[str]) -> None: + paths_expanded = [l.split("/") for l in paths] + assert sort_nodes.order(paths_expanded) == expected diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py new file mode 100644 index 0000000000..69617d0301 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py @@ -0,0 +1,668 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Type + +import pytest +from mock import Mock +from datetime import datetime, timezone, timedelta +import unittest + +from common import TstNodeset, TstCfg # needed to import util +import util +from util import NodeState, MachineType, AcceleratorInfo, UpcomingMaintenance, InstanceResourceStatus, FutureReservation, ReservationDetails +from google.api_core.client_options import ClientOptions # noqa: E402 +from addict import Dict as NSDict # type: ignore + +# Note: need to install pytest-mock + +@pytest.mark.parametrize( + "name,expected", + [ + ( + "az-buka-23", + { + "cluster": "az", + "nodeset": "buka", + "node": "23", + "prefix": "az-buka", + "range": None, + "suffix": "23", + }, + ), + ( + "az-buka-xyzf", + { + "cluster": "az", + "nodeset": "buka", + "node": "xyzf", + "prefix": "az-buka", + "range": None, + "suffix": "xyzf", + }, + ), + ( + "az-buka-[2-3]", + { + "cluster": "az", + "nodeset": "buka", + "node": "[2-3]", + "prefix": "az-buka", + "range": "[2-3]", + "suffix": None, + }, + ), + ], +) +def test_node_desc(name, expected): + assert util.lookup()._node_desc(name) == expected + + +@pytest.mark.parametrize( + "name,expected", + [ + ("az-buka-23", 23), + ("az-buka-0", 0), + ("az-buka", Exception), + ("az-buka-xyzf", ValueError), + ("az-buka-[2-3]", ValueError), + ], +) +def test_node_index(name, expected): + if type(expected) is type and issubclass(expected, Exception): + with pytest.raises(expected): + util.lookup().node_index(name) + else: + assert util.lookup().node_index(name) == expected + + +@pytest.mark.parametrize( + "name", + [ + "az-buka", + ], +) +def test_node_desc_fail(name): + with pytest.raises(Exception): + util.lookup()._node_desc(name) + + +@pytest.mark.parametrize( + "names,expected", + [ + ("pedro,pedro-1,pedro-2,pedro-01,pedro-02", "pedro,pedro-[1-2,01-02]"), + ("pedro,,pedro-1,,pedro-2", "pedro,pedro-[1-2]"), + ("pedro-8,pedro-9,pedro-10,pedro-11", "pedro-[8-9,10-11]"), + ("pedro-08,pedro-09,pedro-10,pedro-11", "pedro-[08-11]"), + ("pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-09]"), + ("pedro-10,pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-10]"), + ("pedro-8,pedro-9,juan-10,juan-11", "juan-[10-11],pedro-[8-9]"), + ("az,buki,vedi", "az,buki,vedi"), + ("a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12", "a[0-9,10-12]"), + ("a0,a2,a4,a6,a7,a8,a11,a12", "a[0,2,4,6-8,11-12]"), + ("seas7-0,seas7-1", "seas7-[0-1]"), + ], +) +def test_to_hostlist(names, expected): + assert util.to_hostlist(names.split(",")) == expected + + +@pytest.mark.parametrize( + "api,ep_ver,expected", + [ + ( + util.ApiEndpoint.BQ, + "v1", + ClientOptions(api_endpoint="https://bq.googleapis.com/v1/"), + ), + ( + util.ApiEndpoint.COMPUTE, + "staging_v1", + ClientOptions(api_endpoint="https://compute.googleapis.com/staging_v1/"), + ), + ( + util.ApiEndpoint.SECRET, + "v1", + ClientOptions(api_endpoint="https://secret_manager.googleapis.com/v1/"), + ), + ( + util.ApiEndpoint.STORAGE, + "beta", + ClientOptions(api_endpoint="https://storage.googleapis.com/beta/"), + ), + ( + util.ApiEndpoint.TPU, + "alpha", + ClientOptions(api_endpoint="https://tpu.googleapis.com/alpha/"), + ), + ], +) +def test_create_client_options( + api: util.ApiEndpoint, ep_ver: str, expected: ClientOptions, mocker +): + ud_mock = mocker.patch("util.universe_domain") + ep_mock = mocker.patch("util.endpoint_version") + ud_mock.return_value = "googleapis.com" + ep_mock.return_value = ep_ver + assert util.create_client_options(api).__repr__() == expected.__repr__() + + + +@pytest.mark.parametrize( + "nodeset,err", + [ + (TstNodeset(reservation_name="projects/x/reservations/y"), AssertionError), # no zones + (TstNodeset( + reservation_name="projects/x/reservations/y", + zone_policy_allow=["eine", "zwei"]), AssertionError), # multiples zones + (TstNodeset( + reservation_name="robin", + zone_policy_allow=["eine"]), ValueError), # invalid name + (TstNodeset( + reservation_name="projects/reservations/y", + zone_policy_allow=["eine"]), ValueError), # invalid name + (TstNodeset( + reservation_name="projects/x/zones/z/reservations/y", + zone_policy_allow=["eine"]), ValueError), # invalid name + ] +) +def test_nodeset_reservation_err(nodeset, err): + lkp = util.Lookup(TstCfg()) + lkp._get_reservation = Mock() + with pytest.raises(err): + lkp.nodeset_reservation(nodeset) + lkp._get_reservation.assert_not_called() # type: ignore + +@pytest.mark.parametrize( + "nodeset,policies,expected", + [ + (TstNodeset(), [], None), # no reservation + (TstNodeset( + reservation_name="projects/bobin/reservations/robin", + zone_policy_allow=["eine"]), + [], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=[], + deployment_type=None, + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/bobin/reservations/robin")), + (TstNodeset( + reservation_name="projects/bobin/reservations/robin", + zone_policy_allow=["eine"]), + ["seven/wanders", "five/red/apples", "yum"], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=["wanders", "apples", "yum"], + deployment_type=None, + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/bobin/reservations/robin")), + (TstNodeset( + reservation_name="projects/bobin/reservations/robin/snek/cheese-brie-6", + zone_policy_allow=["eine"]), + [], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=[], + deployment_type=None, + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/bobin/reservations/robin/snek/cheese-brie-6")), + + ]) + +def test_nodeset_reservation_ok(nodeset, policies, expected): + lkp = util.Lookup(TstCfg()) + lkp._get_reservation = Mock() + + if not expected: + assert lkp.nodeset_reservation(nodeset) is None + lkp._get_reservation.assert_not_called() # type: ignore + return + + lkp._get_reservation.return_value = { # type: ignore + "resourcePolicies": {i: p for i, p in enumerate(policies)}, + } + assert lkp.nodeset_reservation(nodeset) == expected + lkp._get_reservation.assert_called_once_with(expected.project, expected.zone, expected.name) # type: ignore + +@pytest.mark.parametrize( + "job_info,expected_job", + [ + ( + """JobId=123 + TimeLimit=02:00:00 + JobName=myjob + JobState=PENDING + ReqNodeList=node-[1-10]""", + util.Job( + id=123, + duration=timedelta(days=0, hours=2, minutes=0, seconds=0), + name="myjob", + job_state="PENDING", + required_nodes="node-[1-10]" + ), + ), + ( + """JobId=456 + JobName=anotherjob + JobState=PENDING + ReqNodeList=node-group1""", + util.Job( + id=456, + duration=None, + name="anotherjob", + job_state="PENDING", + required_nodes="node-group1" + ), + ), + ( + """JobId=789 + TimeLimit=00:30:00 + JobState=COMPLETED""", + util.Job( + id=789, + duration=timedelta(minutes=30), + name=None, + job_state="COMPLETED", + required_nodes=None + ), + ), + ( + """JobId=101112 + TimeLimit=1-00:30:00 + JobState=COMPLETED, + ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", + util.Job( + id=101112, + duration=timedelta(days=1, hours=0, minutes=30, seconds=0), + name=None, + job_state="COMPLETED", + required_nodes="node-[1-10],grob-pop-[2,1,44-77]" + ), + ), + ( + """JobId=131415 + TimeLimit=1-00:30:00 + JobName=mynode-1_maintenance + JobState=COMPLETED, + ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", + util.Job( + id=131415, + duration=timedelta(days=1, hours=0, minutes=30, seconds=0), + name="mynode-1_maintenance", + job_state="COMPLETED", + required_nodes="node-[1-10],grob-pop-[2,1,44-77]" + ), + ), + ], +) +def test_parse_job_info(job_info, expected_job): + lkp = util.Lookup(TstCfg()) + assert lkp._parse_job_info(job_info) == expected_job + + + +@pytest.mark.parametrize( + "node,state,want", + [ + ("c-n-2", NodeState("DOWN", frozenset([])), NodeState("DOWN", frozenset([]))), # happy scenario + ("c-d-vodoo", None, None), # dynamic nodeset + ("c-x-44", None, None), # unknown(removed) nodeset + ("c-n-7", None, None), # Out of bounds: c-n-[0-4] - downsized nodeset + ("c-t-7", None, None), # Out of bounds: c-t-[0-4] - downsized nodeset TPU + ("c-n-2", None, RuntimeError), # something is wrong + ("c-t-2", None, RuntimeError), # something is wrong, but TPU + + # Check boundaries match [0-5) + ("c-n-5", None, None), # out of boundaries + ("c-n-4", None, RuntimeError), # within boundaries + ]) +def test_node_state(node: str, state: Optional[NodeState], want: NodeState | None | Type[Exception]): + cfg = TstCfg( + slurm_cluster_name="c", + nodeset={ + "n": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, + nodeset_tpu={ + "t": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, + nodeset_dyn={ + "d": TstNodeset()}, + ) + lkp = util.Lookup(cfg) + lkp.slurm_nodes = lambda: {node: state} if state else {} # type: ignore[assignment] + # ... see https://github.com/python/typeshed/issues/6347 + + if type(want) is type and issubclass(want, Exception): + with pytest.raises(want): + lkp.node_state(node) + else: + assert lkp.node_state(node) == want + + + +@pytest.mark.parametrize( + "jo,want", + [ + ({ + "accelerators": [ { "guestAcceleratorCount": 1, "guestAcceleratorType": "nvidia-tesla-a100" } ], + "creationTimestamp": "1969-12-31T16:00:00.000-08:00", + "description": "Accelerator Optimized: 1 NVIDIA Tesla A100 GPU, 12 vCPUs, 85GB RAM", + "guestCpus": 12, + "id": "1000012", + "imageSpaceGb": 0, + "isSharedCpu": False, + "kind": "compute#machineType", + "maximumPersistentDisks": 128, + "maximumPersistentDisksSizeGb": "263168", + "memoryMb": 87040, + "name": "a2-highgpu-1g", + "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/us-central1-a/machineTypes/a2-highgpu-1g", + "zone": "us-central1-a" + }, MachineType( + name="a2-highgpu-1g", + guest_cpus=12, + memory_mb=87040, + accelerators=[ + AcceleratorInfo(type="nvidia-tesla-a100", count=1) + ] + )), + ({ + "architecture": "X86_64", + "creationTimestamp": "1969-12-31T16:00:00.000-08:00", + "description": "8 vCPUs, 32 GB RAM", + "guestCpus": 8, + "id": "1210008", + "imageSpaceGb": 0, + "isSharedCpu": False, + "kind": "compute#machineType", + "maximumPersistentDisks": 128, + "maximumPersistentDisksSizeGb": "263168", + "memoryMb": 32768, + "name": "t2d-standard-8", + "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/europe-north2-b/machineTypes/t2d-standard-8", + "zone": "europe-north2-b" + }, MachineType( + name="t2d-standard-8", + guest_cpus=8, + memory_mb=32768, + accelerators=[] + )), + ]) +def test_MachineType_from_json(jo: dict, want: MachineType): + assert MachineType.from_json(jo) == want + + +@pytest.mark.parametrize( + "template,expected", + [ + ( + NSDict({ + "machine_type": MachineType( + name="e2", + guest_cpus=12, + memory_mb=87040, + accelerators=[]), + }), + None + ), + ( + NSDict({ + "machine_type": MachineType( + name="tpu-machine", + guest_cpus=12, + memory_mb=87040, + accelerators=[ + AcceleratorInfo(type="tpu-v6", count=1) + ]), + }), + None + ), + ( + NSDict({ + "machine_type": MachineType( + name="a2-highgpu-1g", + guest_cpus=12, + memory_mb=87040, + accelerators=[AcceleratorInfo(type="nvidia-tesla-a100", count=1)] + ), + }), + AcceleratorInfo(type="nvidia-tesla-a100", count=1) + ), + ( + NSDict({ + "machine_type": MachineType( + name="a2-highgpu-1g", + guest_cpus=12, + memory_mb=87040, + accelerators=[]), + "guestAccelerators":[ { "acceleratorCount": 1, "acceleratorType": "nvidia-tesla-a100" } ], + }), + AcceleratorInfo(type="nvidia-tesla-a100", count=1) + ), + ], +) +def test_get_template_gpu(template, expected): + assert util.get_template_gpu(template) == expected + + +UTC, PST = timezone.utc, timezone(timedelta(hours=-8)) + +@pytest.mark.parametrize( + "got,want", + [ + # from instance.creationTimestamp: + ("2024-11-30T12:47:51.676-08:00", datetime(2024, 11, 30, 12, 47, 51, 676000, tzinfo=PST)), + # from futureReservation.creationTimestamp + ("2024-11-05T15:23:33.702-08:00", datetime(2024, 11, 5, 15, 23, 33, 702000, tzinfo=PST)), + # from futureReservation.timeWindow.endTime + ("2025-01-15T00:00:00Z", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), + # fallback to UTC if no tz is specified + ("2025-01-15T00:00:00", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), + ]) +def test_parse_gcp_timestamp(got: str, want: datetime): + assert util.parse_gcp_timestamp(got) == want + + +@pytest.mark.parametrize( + "got,want", + [ + (None, None), + (dict( + windowStartTime="2025-01-15T00:00:00Z", + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + (dict( + startTimeWindow=dict( + earliest="2025-01-15T00:00:00Z"), + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + (dict( + windowStartTime="2025-01-15T00:00:00Z", + startTimeWindow=dict( + earliest="2025-01-25T00:00:00Z"), # ignored + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + ]) +def tests_parse_UpcomingMaintenance_OK(got: dict, want: Optional[UpcomingMaintenance]): + assert UpcomingMaintenance.from_json(got) == want + + +@pytest.mark.parametrize( + "got", + [ + {}, + dict( + windowStartTime=dict( + earliest="2025-01-15T00:00:00Z")), + ]) +def tests_parse_UpcomingMaintenance_FAIL(got: dict): + with pytest.raises(ValueError): + UpcomingMaintenance.from_json(got) + + +@pytest.mark.parametrize( + "got,want", + [ + (None, InstanceResourceStatus( + physical_host=None, + upcoming_maintenance=None)), + ({}, InstanceResourceStatus( + physical_host=None, + upcoming_maintenance=None)), + (dict( + physicalHost="/aaa/bbb/ccc"), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=None)), + (dict( # invalid upcomingMaintenance field to be ignored + physicalHost="/aaa/bbb/ccc", + upcomingMaintenance="maintenance is upon us"), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=None)), + (dict( + physicalHost="/aaa/bbb/ccc", + upcomingMaintenance=dict(windowStartTime="2025-01-15T00:00:00Z")), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=UpcomingMaintenance( + window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC)))), + ]) +def test_parse_InstanceResourceStatus(got: dict, want: Optional[InstanceResourceStatus]): + assert InstanceResourceStatus.from_json(got) == want + + +@pytest.mark.parametrize( + "link,component_name,expected", + [ + ( + "mylink/regions/us-cental1/other", + "regions", + "us-cental1" + ), + ( + "mylink/global/other", + "regions", + None + ), + ], +) +def test_get_self_link_component(link, component_name, expected): + assert util.get_self_link_component(link, component_name) == expected + + +def test_future_reservation_none(): + lkp = util.Lookup(TstCfg()) + assert lkp.future_reservation(TstNodeset()) == None + + +def test_future_reservation_declined(): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-03T23:30:00Z" }, + status = {"procurementStatus": "DECLINED"}, + reservationMode = "CALENDAR", + specificReservationRequired = True, + )) + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 3, 23, 30, tzinfo=timezone.utc), + reservation_mode="CALENDAR", + active_reservation=None) + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + +@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 13, 0, 0, tzinfo=timezone.utc)) +def test_future_reservation_active(_): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, + status = { + "procurementStatus": "FULFILLED", + "autoCreatedReservations": [ + "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" + ], + }, + specificReservationRequired = True, + )) + lkp._get_reservation = Mock(return_value=dict()) + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), + reservation_mode=None, + active_reservation=ReservationDetails( + project='manhattan', + zone='danger', + name='melon', + policies=[], + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/manhattan/reservations/melon", + deployment_type=None)) + + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + lkp._get_reservation.assert_called_once_with("manhattan", "danger", "melon") + +@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 28, 0, 0, tzinfo=timezone.utc)) +def test_future_reservation_inactive(_): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, + status = { + "procurementStatus": "FULFILLED", + "autoCreatedReservations": [ + "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" + ], + }, + reservationMode = "DEFAULT", + specificReservationRequired = True, + )) + lkp._get_reservation = Mock() + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), + reservation_mode="DEFAULT", + active_reservation=None) + + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + lkp._get_reservation.assert_not_called() diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test new file mode 100644 index 0000000000..a583642015 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test @@ -0,0 +1,133 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +unset CUDA_VISIBLE_DEVICES + +LOG_FILE="/var/log/slurm/chs_health_check.log" +TMP_DCGM_OUT="/tmp/dcgm.out" +TMP_ECC_ERRORS_OUT="/tmp/ecc_errors.out" + +log_step() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" +} + +# Fail gracefully if nvidia-smi or dcgmi doesn't exist +if ! type -P nvidia-smi 1>/dev/null; then + log_step "nvidia-smi not found - this script requires nvidia-smi to function" + exit 0 +fi + +if ! type -P dcgmi 1>/dev/null; then + log_step "dcgmi not found - this script requires dcgmi to function" + exit 0 +fi + +if ! type -P nv-hostengine 1>/dev/null; then + log_step "nv-hostengine not found - this script requires nv-hostengine to function" + exit 0 +fi + +################################################### +# Disable running health checks +################################################### +# Check if the environment variable '$SLURM_JOB_EXTRA' is set and contains the +# substring 'healthchecks_prolog=off' +if [[ -n "$SLURM_JOB_EXTRA" ]]; then + log_step "Environment variable SLURM_JOB_EXTRA is set. Checking if it contains healthchecks_prolog=off." + # Check if the value of the variable matches the string "healthchecks_prolog=off" + if [[ "$SLURM_JOB_EXTRA" == *"healthchecks_prolog=off"* ]]; then + log_step "Environment variable SLURM_JOB_EXTRA matches substring healthchecks_prolog=off. Skipping health checks." + exit 0 + else + log_step "Environment variable SLURM_JOB_EXTRA does NOT match substring healthchecks_prolog=off. Attempting to run health checks." + fi +else + log_step "Environment variable SLURM_JOB_EXTRA is NOT set. Attempting to run health checks." +fi + +# Exit if GPU isn't H/B 100/200 +GPU_MODEL=$(nvidia-smi --query-gpu=name --format=csv,noheader) +if ! [[ "$GPU_MODEL" =~ [BH][1-2]00 ]]; then + log_step "No Supported GPU detected" + exit 0 +fi + +NUMGPUS=$(nvidia-smi -L | wc -l) + +# Check that all GPUs are healthy via DCGM and check for ECC errors +if [ $NUMGPUS -gt 0 ]; then + log_step "Execute DCGM health check, ECC error check, and NVLink error check for GPUs" + GPULIST=$(nvidia-smi --query-gpu=index --format=csv,noheader | tr '\n' ',' | sed 's/,$//') + rm -f $TMP_DCGM_OUT + rm -f $TMP_ECC_ERRORS_OUT + + # Run DCGM checks + START_HOSTENGINE=false + if ! pidof nv-hostengine > /dev/null; then + log_step "Starting nv-hostengine..." + nv-hostengine >> "$LOG_FILE" 2>&1 + sleep 1 # Give it a moment to start up + START_HOSTENGINE=true + fi + GROUPID=$(dcgmi group -c gpuinfo | awk '{print $NF}' | tr -d ' ') + dcgmi group -g $GROUPID -a $GPULIST >> "$LOG_FILE" 2>&1 + dcgmi diag -g $GROUPID -r 1 > "$TMP_DCGM_OUT" 2>&1 + cat "$TMP_DCGM_OUT" >> "$LOG_FILE" + dcgmi group -d $GROUPID >> "$LOG_FILE" 2>&1 + + # Terminate the host engine if it was manually started + if [ "$START_HOSTENGINE" = true ]; then + log_step "Terminating nv-hostengine..." + nv-hostengine -t >> "$LOG_FILE" 2>&1 + fi + + # Check for DCGM failures + DCGM_FAILED=0 + if grep -i fail "$TMP_DCGM_OUT" > /dev/null; then + DCGM_FAILED=1 + fi + + # Check for ECC errors + nvidia-smi --query-gpu=ecc.errors.uncorrected.volatile.total --format=csv,noheader > "$TMP_ECC_ERRORS_OUT" + cat "$TMP_ECC_ERRORS_OUT" >> "$LOG_FILE" + ECC_ERRORS=$(awk -F', ' '{sum += $2} END {print sum}' "$TMP_ECC_ERRORS_OUT") + log_step "ECC Errors: $ECC_ERRORS" + + # Check for NVLink errors + NVLINK_ERRORS=$(nvidia-smi nvlink -sc 0bz -i 0 2>/dev/null | grep -i "Error Count" | awk '{sum += $3} END {print sum}') + # Set to 0 if empty/null + NVLINK_ERRORS=${NVLINK_ERRORS:-0} + log_step "NVLink Errors: $NVLINK_ERRORS" + + if [ $DCGM_FAILED -eq 1 ] || \ + [ $ECC_ERRORS -gt 0 ] || \ + [ $NVLINK_ERRORS -gt 0 ]; then + REASON="GPU issues detected: " + if [ $DCGM_FAILED -eq 1 ]; then + REASON+="DCGM test failed, " + fi + if [ $ECC_ERRORS -gt 0 ]; then + REASON+="ECC errors found ($ECC_ERRORS double-bit errors), " + fi + if [ $NVLINK_ERRORS -gt 0 ]; then + REASON+="NVLink errors detected ($NVLINK_ERRORS errors), " + fi + REASON+="see $LOG_FILE" + log_step "$REASON" + exit 1 + fi +fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog new file mode 100644 index 0000000000..a22ddea9e5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Main TaskEpilog Script +# This script executes all *.sh scripts found in /slurm/custom_scripts/task_epilog.d/ +# +# slurm.conf configuration: +# TaskEpilog=/slurm/scripts/tools/task-epilog + +# Directory containing the individual task epilog scripts +EPILOG_D_DIR="/slurm/custom_scripts/task_epilog.d" + +# --- Output Handling for TaskEpilog --- +# The stdout and stderr of this script (and the sub-scripts it calls) +# are typically captured by Slurm and written to the job's output/error file +# or a separate Slurm log, depending on configuration. +# Unlike TaskProlog, stdout is not typically parsed for special commands +# like 'export' or 'print' to affect the (now finished) task's environment. +# +# --- Error Handling --- +# If any script in EPILOG_D_DIR exits with a non-zero status, +# this main script will also exit with a non-zero status. +# Slurm will log this. Depending on Slurm's configuration, +# frequent epilog failures might lead to node issues or alerts. +set -e # Exit immediately if a command exits with a non-zero status. + +# Check if the directory exists +if [[ ! -d "$EPILOG_D_DIR" ]]; then + # Log in task stdout and exit if the directory is missing. This likely indicates a configuration error. + echo "print TaskEpilog Error: Directory '$EPILOG_D_DIR' not found. Check Slurm configuration." + exit 1 +fi + +# Find and execute all *.sh scripts in the directory +# Scripts will be executed in reverse alphabetical order of their filenames. +find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do + if [[ -x "$script" ]]; then + # Execute the script. Its stdout will be captured by this wrapper. + # Its stderr will also be passed through. + # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. + "$script" + else + # Log in task stdout a warning if a *.sh file is found but is not executable + echo "print TaskEpilog Warning: Script '$script' is not executable and will be skipped." + fi +done + +# Check if any scripts were found and executed +if [[ $(find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then + # Log in task stdout if no scripts were found to execute + echo "print TaskEpilog Info: No executable *.sh scripts found in $EPILOG_D_DIR." +fi + +# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) +exit 0 diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog new file mode 100644 index 0000000000..feddb23209 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Main TaskProlog Script +# This script executes all *.sh scripts found in /slurm/custom_scripts/task_prolog.d/ +# +# slurm.conf configuration: +# TaskProlog=/slurm/scripts/tools/task-prolog + +# Directory containing the individual task prolog scripts +PROLOG_D_DIR="/slurm/custom_scripts/task_prolog.d" + +# --- Output Handling for TaskProlog --- +# Slurm's TaskProlog can interpret specific stdout lines: +# - "export NAME=value" : Sets an environment variable for the task. +# - "unset NAME" : Unsets an environment variable for the task. +# - "print message" : Prints a message to the task's standard output. +# +# This wrapper script will concatenate the stdout of all sub-scripts. +# If sub-scripts need to set/unset environment variables or print messages +# for the task, they should output the appropriate "export", "unset", or "print" +# commands to their own stdout. + +# --- Error Handling --- +# If any script in PROLOG_D_DIR exits with a non-zero status, +# this main script will also exit with a non-zero status. +# This will typically cause the task to fail. +set -e # Exit immediately if a command exits with a non-zero status. + +# Check if the directory exists +if [[ ! -d "$PROLOG_D_DIR" ]]; then + # Log in task stdout and exit if the directory is missing. All jobs will be failed. + echo "print TaskProlog Error: Directory '$PROLOG_D_DIR' not found. Check Slurm configuration." + exit 1 +fi + +# Find and execute all *.sh scripts in the directory +# Scripts will be executed in reverse alphabetical order of their filenames. +find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do + if [[ -x "$script" ]]; then + # Execute the script. Its stdout will be captured by this wrapper. + # Its stderr will also be passed through. + # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. + "$script" + else + # Log a warning in task stdout if a *.sh file is found but is not executable + echo "print TaskProlog Warning: Script '$script' is not executable and will be skipped." + fi +done + +# Check if any scripts were found and executed +if [[ $(find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then + # Log in task stdout if no scripts were found to execute + echo "print TaskProlog Info: No executable *.sh scripts found in $PROLOG_D_DIR." +fi + +# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) +exit 0 diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py new file mode 100644 index 0000000000..531f0348dc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py @@ -0,0 +1,331 @@ +# mypy: ignore-errors +# This implementation of TPU integration is to be deprecated + +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import socket +import logging +from pathlib import Path +import yaml + +import util +from util import create_client_options, ApiEndpoint + +from google.cloud import tpu_v2 as tpu # noqa: E402 +import google.api_core.exceptions as gExceptions # noqa: E402 + +log = logging.getLogger() + +_tpu_cache = {} + +class TPU: + """Class for handling the TPU-vm nodes""" + + State = tpu.types.cloud_tpu.Node.State + TPUS_PER_VM = 4 + __expected_states = { + "create": State.READY, + "start": State.READY, + "stop": State.STOPPED, + } + + __tpu_version_mapping = { + "V2": tpu.AcceleratorConfig().Type.V2, + "V3": tpu.AcceleratorConfig().Type.V3, + "V4": tpu.AcceleratorConfig().Type.V4, + } + + @classmethod + def make(cls, nodeset_name: str, lkp: util.Lookup) -> "TPU": + key = (id(lkp), nodeset_name) + if key not in _tpu_cache: + nodeset = lkp.cfg.nodeset_tpu[nodeset_name] + _tpu_cache[key] = cls(nodeset, lkp) + return _tpu_cache[key] + + + def __init__(self, nodeset: object, lkp: util.Lookup): + self._nodeset = nodeset + self.lkp = lkp + self._parent = f"projects/{lkp.project}/locations/{nodeset.zone}" + co = create_client_options(ApiEndpoint.TPU) + self._client = tpu.TpuClient(client_options=co) + self.data_disks = [] + for data_disk in nodeset.data_disks: + ad = tpu.AttachedDisk() + ad.source_disk = data_disk + ad.mode = tpu.AttachedDisk.DiskMode.DISK_MODE_UNSPECIFIED + self.data_disks.append(ad) + ns_ac = nodeset.accelerator_config + if ns_ac.topology != "" and ns_ac.version != "": + ac = tpu.AcceleratorConfig() + ac.topology = ns_ac.topology + ac.type_ = self.__tpu_version_mapping[ns_ac.version] + self.ac = ac + else: + req = tpu.GetAcceleratorTypeRequest( + name=f"{self._parent}/acceleratorTypes/{nodeset.node_type}" + ) + self.ac = self._client.get_accelerator_type(req).accelerator_configs[0] + self.vmcount = self.__calc_vm_from_topology(self.ac.topology) + + @property + def nodeset(self): + return self._nodeset + + @property + def preserve_tpu(self): + return self._nodeset.preserve_tpu + + @property + def node_type(self): + return self._nodeset.node_type + + @property + def tf_version(self): + return self._nodeset.tf_version + + @property + def enable_public_ip(self): + return self._nodeset.enable_public_ip + + @property + def preemptible(self): + return self._nodeset.preemptible + + @property + def reserved(self): + return self._nodeset.reserved + + @property + def service_account(self): + return self._nodeset.service_account + + @property + def zone(self): + return self._nodeset.zone + + def check_node_type(self): + if self.node_type is None: + return False + try: + request = tpu.GetAcceleratorTypeRequest( + name=f"{self._parent}/acceleratorTypes/{self.node_type}" + ) + return self._client.get_accelerator_type(request=request) is not None + except Exception: + return False + + def check_tf_version(self): + try: + request = tpu.GetRuntimeVersionRequest( + name=f"{self._parent}/runtimeVersions/{self.tf_version}" + ) + return self._client.get_runtime_version(request=request) is not None + except Exception: + return False + + def __calc_vm_from_topology(self, topology): + topo = topology.split("x") + tot = 1 + for num in topo: + tot = tot * int(num) + return tot // self.TPUS_PER_VM + + def __check_resp(self, response, op_name): + des_state = self.__expected_states.get(op_name) + # If the state is not in the table just print the response + if des_state is None: + return False + if response.__class__.__name__ != "Node": # If the response is not a node fail + return False + if response.state == des_state: + return True + return False + + def list_nodes(self): + try: + request = tpu.ListNodesRequest(parent=self._parent) + res = self._client.list_nodes(request=request) + except gExceptions.NotFound: + res = None + return res + + def list_node_names(self): + return [node.name.split("/")[-1] for node in self.list_nodes()] + + def start_node(self, nodename): + request = tpu.StartNodeRequest(name=f"{self._parent}/nodes/{nodename}") + resp = self._client.start_node(request=request).result() + return self.__check_resp(resp, "start") + + def stop_node(self, nodename): + request = tpu.StopNodeRequest(name=f"{self._parent}/nodes/{nodename}") + resp = self._client.stop_node(request=request).result() + return self.__check_resp(resp, "stop") + + def get_node(self, nodename): + try: + request = tpu.GetNodeRequest(name=f"{self._parent}/nodes/{nodename}") + res = self._client.get_node(request=request) + except gExceptions.NotFound: + res = None + return res + + def _register_node(self, nodename, ip_addr): + dns_name = socket.getnameinfo((ip_addr, 0), 0)[0] + util.run( + f"{self.lkp.scontrol} update nodename={nodename} nodeaddr={ip_addr} nodehostname={dns_name}" + ) + + def create_node(self, nodename): + if self.vmcount > 1 and not isinstance(nodename, list): + log.error( + f"Tried to create a {self.vmcount} node TPU on nodeset {self._nodeset.nodeset_name} but only received one nodename {nodename}" + ) + return False + if self.vmcount > 1 and ( + isinstance(nodename, list) and len(nodename) != self.vmcount + ): + log.error( + f"Expected to receive a list of {self.vmcount} nodenames for TPU node creation in nodeset {self._nodeset.nodeset_name}, but received this list {nodename}" + ) + return False + + node = tpu.Node() + node.accelerator_config = self.ac + node.runtime_version = f"tpu-vm-tf-{self.tf_version}" + startup_script = """ + #!/bin/bash + echo "startup script not found > /var/log/startup_error.log" + """ + with open( + Path(self.lkp.cfg.slurm_scripts_dir or util.dirs.scripts) / "startup.sh", "r" + ) as script: + startup_script = script.read() + if isinstance(nodename, list): + node_id = nodename[0] + slurm_names = [] + wid = 0 + for node_wid in nodename: + slurm_names.append(f"WORKER_{wid}:{node_wid}") + wid += 1 + else: + node_id = nodename + slurm_names = [f"WORKER_0:{nodename}"] + node.metadata = { + "slurm_docker_image": self.nodeset.docker_image, + "startup-script": startup_script, + "slurm_instance_role": "compute", + "slurm_cluster_name": self.lkp.cfg.slurm_cluster_name, + "slurm_bucket_path": self.lkp.cfg.bucket_path, + "slurm_names": ";".join(slurm_names), + "universe_domain": util.universe_domain(), + } + node.tags = [self.lkp.cfg.slurm_cluster_name] + if self.nodeset.service_account: + node.service_account.email = self.nodeset.service_account.email + node.service_account.scope = self.nodeset.service_account.scopes + node.scheduling_config.preemptible = self.preemptible + node.scheduling_config.reserved = self.reserved + node.network_config.subnetwork = self.nodeset.subnetwork + node.network_config.enable_external_ips = self.enable_public_ip + if self.data_disks: + node.data_disks = self.data_disks + + request = tpu.CreateNodeRequest(parent=self._parent, node=node, node_id=node_id) + resp = self._client.create_node(request=request).result() + if not self.__check_resp(resp, "create"): + return False + if isinstance(nodename, list): + for node_id, net_endpoint in zip(nodename, resp.network_endpoints): + self._register_node(node_id, net_endpoint.ip_address) + else: + ip_add = resp.network_endpoints[0].ip_address + self._register_node(nodename, ip_add) + return True + + def delete_node(self, nodename): + request = tpu.DeleteNodeRequest(name=f"{self._parent}/nodes/{nodename}") + try: + resp = self._client.delete_node(request=request).result() + if resp: + return self.get_node(nodename=nodename) is None + return False + except gExceptions.NotFound: + # log only error if vmcount is 1 as for other tpu vm count, this could be "phantom" nodes + if self.vmcount == 1: + log.error(f"Tpu single node {nodename} not found") + else: + # for the TPU nodes that consist in more than one vm, only the first node of the TPU a.k.a. the master node will + # exist as real TPU nodes, so the other ones are expected to not be found, check the hostname of the node that has + # not been found, and if it ends in 0, it means that is the master node and it should have been found, and in consequence + # log an error + nodehostname = yaml.safe_load( + util.run(f"{self.lkp.scontrol} --yaml show node {nodename}").stdout.rstrip() + )["nodes"][0]["hostname"] + if nodehostname.split("-")[-1] == "0": + log.error(f"TPU master node {nodename} not found") + else: + log.info(f"Deleted TPU 'phantom' node {nodename}") + # If the node is not found it is tecnichally deleted, so return success. + return True + +def _stop_tpu(node: str) -> None: + lkp = util.lookup() + tpuobj = TPU.make(lkp.node_nodeset_name(node), lkp) + if tpuobj.nodeset.preserve_tpu and tpuobj.vmcount == 1: + log.info(f"stopping node {node}") + if tpuobj.stop_node(node): + return + log.error("Error stopping node {node} will delete instead") + log.info(f"deleting node {node}") + if not tpuobj.delete_node(node): + log.error("Error deleting node {node}") + + +def delete_tpu_instances(instances: List[str]) -> None: + util.execute_with_futures(_stop_tpu, instances) + + +def start_tpu(node: List[str]): + lkp = util.lookup() + tpuobj = TPU.make(lkp.node_nodeset_name(node[0]), lkp) + + if len(node) == 1: + node = node[0] + log.debug( + f"Will create a TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node}" + ) + tpunode = tpuobj.get_node(node) + if tpunode is None: + if not tpuobj.create_node(nodename=node): + log.error("Error creating tpu node {node}") + else: + if tpuobj.preserve_tpu: + if not tpuobj.start_node(nodename=node): + log.error("Error starting tpu node {node}") + else: + log.info( + f"Tpu node {node} is already created, but will not start it because nodeset does not have preserve_tpu option active." + ) + else: + log.debug( + f"Will create a multi-vm TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node[0]}" + ) + if not tpuobj.create_node(nodename=node): + log.error("Error creating tpu node {node}") diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py new file mode 100644 index 0000000000..217fd0bca2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py @@ -0,0 +1,2224 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable, List, Tuple, Optional, Any, Dict, Sequence, Type, Callable, Union +import argparse +import base64 +from dataclasses import dataclass, field +from datetime import timedelta, datetime, timezone +import hashlib +import inspect +import json +import logging +import logging.config +import logging.handlers +import math +import os +import re +import shlex +import shutil +import socket +import subprocess +import sys +from enum import Enum +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager +from functools import lru_cache, reduce, wraps +from itertools import chain, islice +from pathlib import Path +from time import sleep, time + +# TODO: remove "type: ignore" once moved to newer version of libraries +from google.cloud import secretmanager +from google.cloud import storage # type: ignore + +import google.auth # type: ignore +from google.oauth2 import service_account # type: ignore +import googleapiclient.discovery # type: ignore +import google_auth_httplib2 # type: ignore +from googleapiclient.http import set_user_agent # type: ignore +from google.api_core.client_options import ClientOptions +import httplib2 + +import google.api_core.exceptions as gExceptions + +import requests as requests_lib + +import yaml +from addict import Dict as NSDict # type: ignore +import file_cache + +USER_AGENT = "Slurm_GCP_Scripts/1.5 (GPN:SchedMD)" +ENV_CONFIG_YAML = os.getenv("SLURM_CONFIG_YAML") +if ENV_CONFIG_YAML: + CONFIG_FILE = Path(ENV_CONFIG_YAML) +else: + CONFIG_FILE = Path(__file__).with_name("config.yaml") +API_REQ_LIMIT = 2000 + + +def mkdirp(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + + +scripts_dir = next( + p for p in (Path(__file__).parent, Path("/slurm/scripts")) if p.is_dir() +) + + +# load all directories as Paths into a dict-like namespace +dirs = NSDict( + home = Path("/home"), + apps = Path("/opt/apps"), + slurm = Path("/slurm"), + scripts = scripts_dir, + custom_scripts = Path("/slurm/custom_scripts"), + munge = Path("/etc/munge"), + secdisk = Path("/mnt/disks/sec"), + log = Path("/var/log/slurm"), + slurm_bucket_mount = Path("/slurm/bucket"), +) + +slurmdirs = NSDict( + prefix = Path("/usr/local"), + etc = Path("/usr/local/etc/slurm"), + state = Path("/var/spool/slurm"), + key_distribution = Path("/slurm/key_distribution"), +) + + +# TODO: Remove this hack (relies on undocumented behavior of PyYAML) +# No need to represent NSDict and Path once we move to properly typed & serializable config. +yaml.SafeDumper.yaml_representers[ + None # type: ignore +] = lambda self, data: yaml.representer.SafeRepresenter.represent_str(self, str(data)) # type: ignore + + +class ApiEndpoint(Enum): + COMPUTE = "compute" + BQ = "bq" + STORAGE = "storage" + TPU = "tpu" + SECRET = "secret_manager" + + +@dataclass(frozen=True) +class AcceleratorInfo: + type: str + count: int + + @classmethod + def from_json(cls, jo: dict) -> "AcceleratorInfo": + return cls( + type=jo["guestAcceleratorType"], + count=jo["guestAcceleratorCount"]) + +@dataclass(frozen=True) +class MachineType: + name: str + guest_cpus: int + memory_mb: int + accelerators: List[AcceleratorInfo] + + @classmethod + def from_json(cls, jo: dict) -> "MachineType": + return cls( + name=jo["name"], + guest_cpus=jo["guestCpus"], + memory_mb=jo["memoryMb"], + accelerators=[ + AcceleratorInfo.from_json(a) for a in jo.get("accelerators", [])], + ) + + @property + def family(self) -> str: + # TODO: doesn't work with N1 custom machine types + # See https://cloud.google.com/compute/docs/instances/creating-instance-with-custom-machine-type#create + return self.name.split("-")[0] + + @property + def supports_smt(self) -> bool: + # https://cloud.google.com/compute/docs/cpu-platforms + if self.family in ("t2a", "t2d", "h3", "c4a", "h4d",): + return False + if self.guest_cpus == 1: + return False + return True + + @property + def sockets(self) -> int: + return { + "h3": 2, + "h4d": 2, + "c2d": 2 if self.guest_cpus > 56 else 1, + "a3": 2, + "c2": 2 if self.guest_cpus > 30 else 1, + "c3": 2 if self.guest_cpus > 88 else 1, + "c3d": 2 if self.guest_cpus > 180 else 1, + "c4": 2 if self.guest_cpus > 96 else 1, + "c4d": 2 if self.guest_cpus > 192 else 1, + }.get( + self.family, + 1, # assume 1 socket for all other families + ) + + +@dataclass(frozen=True) +class UpcomingMaintenance: + window_start_time: datetime + + @classmethod + def from_json(cls, jo: Optional[dict]) -> Optional["UpcomingMaintenance"]: + if jo is None: + return None + try: + if "windowStartTime" in jo: + ts = parse_gcp_timestamp(jo["windowStartTime"]) + elif "startTimeWindow" in jo: + ts = parse_gcp_timestamp(jo["startTimeWindow"]["earliest"]) + else: + raise Exception("Neither windowStartTime nor startTimeWindow are found") + except BaseException as e: + raise ValueError(f"Unexpected format for upcomingMaintenance: {jo}") from e + return cls(window_start_time=ts) + +@dataclass(frozen=True) +class InstanceResourceStatus: + physical_host: Optional[str] + upcoming_maintenance: Optional[UpcomingMaintenance] + + @classmethod + def from_json(cls, jo: Optional[dict]) -> "InstanceResourceStatus": + if not jo: + return cls( + physical_host=None, + upcoming_maintenance=None, + ) + + try: + maint = UpcomingMaintenance.from_json(jo.get("upcomingMaintenance")) + except ValueError as e: + log.exception("Failed to parse upcomingMaintenance, ignoring") + maint = None # intentionally swallow exception + + return cls( + physical_host=jo.get("physicalHost"), + upcoming_maintenance=maint, + ) + + +@dataclass(frozen=True) +class Instance: + name: str + zone: str + status: str + creation_timestamp: datetime + role: Optional[str] + resource_status: InstanceResourceStatus + metadata: Dict[str, str] + # TODO: use proper InstanceScheduling class + scheduling: NSDict + + @classmethod + def from_json(cls, jo: dict) -> "Instance": + return cls( + name=jo["name"], + zone=trim_self_link(jo["zone"]), + status=jo["status"], + creation_timestamp=parse_gcp_timestamp(jo["creationTimestamp"]), + resource_status=InstanceResourceStatus.from_json(jo.get("resourceStatus")), + scheduling=NSDict(jo.get("scheduling")), + role = jo.get("labels", {}).get("slurm_instance_role"), + metadata = {k["key"]: k["value"] for k in jo.get("metadata", {}).get("items", [])} + ) + + +@dataclass(frozen=True) +class NSMount: + server_ip: str + local_mount: Path + remote_mount: Path + fs_type: str + mount_options: str + +@lru_cache(maxsize=1) +def default_credentials(): + return google.auth.default()[0] + + +@lru_cache(maxsize=1) +def authentication_project(): + return google.auth.default()[1] + + +DEFAULT_UNIVERSE_DOMAIN = "googleapis.com" + + +def now() -> datetime: + """ + Return current time as timezone-aware datetime. + + IMPORTANT: DO NOT use `datetime.now()`, unless you explicitly need to have tz-naive datetime. + Otherwise there is a risk of getting: "cannot compare naive and aware datetimes" error, + since all timetstamps we receive from GCP API are tz-aware. + + Another motivation for this function is to allow to mock time in tests. + """ + return datetime.now(timezone.utc) + +def parse_gcp_timestamp(s: str) -> datetime: + """ + Parse timestamp strings returned by GCP API into datetime. + Works with both Zulu and non-Zulu timestamps. + NOTE: It always return tz-aware datetime (fallbacks to UTC and logs error). + """ + # Requires Python >= 3.7 + # TODO: Remove this "hack" of trimming the Z from timestamps once we move to Python 3.11 + # (context: https://discuss.python.org/t/parse-z-timezone-suffix-in-datetime/2220/30) + ts = datetime.fromisoformat(s.replace('Z', '+00:00')) + if ts.tzinfo is None: # fallback to UTC + log.error(f"Received timestamp without timezone info: {s}") + ts = ts.replace(tzinfo=timezone.utc) + return ts + + +def universe_domain() -> str: + try: + return instance_metadata("attributes/universe_domain") + except MetadataNotFoundError: + return DEFAULT_UNIVERSE_DOMAIN + + +def endpoint_version(api: ApiEndpoint) -> Optional[str]: + return lookup().endpoint_versions.get(api.value, None) + + +@lru_cache(maxsize=1) +def get_credentials() -> Optional[service_account.Credentials]: + """Get credentials for service account""" + key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if key_path is not None: + credentials = service_account.Credentials.from_service_account_file( + key_path, scopes=[f"https://www.{universe_domain()}/auth/cloud-platform"] + ) + else: + credentials = default_credentials() + + return credentials + + +@lru_cache(maxsize=1) +def get_dev_key() -> Optional[str]: + """Get dev key for project (uses json or yaml format)""" + try: + with open("/etc/slurm/slurm_vars.yaml", 'r') as file: + data = yaml.safe_load(file) + return data['google_developer_key'] + except: + return None + + +def create_client_options(api: ApiEndpoint) -> ClientOptions: + """Create client options for cloud endpoints""" + ver = endpoint_version(api) + ud = universe_domain() + options = {} + if ud and ud != DEFAULT_UNIVERSE_DOMAIN: + options["universe_domain"] = ud + if ver: + options["api_endpoint"] = f"https://{api.value}.{ud}/{ver}/" + co = ClientOptions(**options) + log.debug(f"Using ClientOptions = {co} for API: {api.value}") + return co + +log = logging.getLogger() + + +def access_secret_version(project_id, secret_id, version_id="latest"): + """ + Access the payload for the given secret version if one exists. The version + can be a version number as a string (e.g. "5") or an alias (e.g. "latest"). + """ + co = create_client_options(ApiEndpoint.SECRET) + client = secretmanager.SecretManagerServiceClient(client_options=co) + name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" + try: + response = client.access_secret_version(request={"name": name}) + log.debug(f"Secret '{name}' was found.") + payload = response.payload.data.decode("UTF-8") + except gExceptions.NotFound: + log.debug(f"Secret '{name}' was not found!") + payload = None + + return payload + + +def parse_self_link(self_link: str): + """Parse a selfLink url, extracting all useful values + https://.../v1/projects//regions//... + {'project': , 'region': , ...} + can also extract zone, instance (name), image, etc + """ + link_patt = re.compile(r"(?P[^\/\s]+)s\/(?P[^\s\/]+)") + return NSDict(link_patt.findall(self_link)) + + +def parse_bucket_uri(uri: str): + """ + Parse a bucket url + E.g. gs:/// + """ + pattern = re.compile(r"gs://(?P[^/\s]+)/(?P([^/\s]+)(/[^/\s]+)*)") + matches = pattern.match(uri) + assert matches, f"Unexpected bucker URI: '{uri}'" + return matches.group("bucket"), matches.group("path") + + +def get_template_gpu(template): + """get gpu info from machine type or guest accelerators""" + gpu_keyword = "nvidia" + gpu = None + if template.machine_type.accelerators: + tma = template.machine_type.accelerators[0] + if gpu_keyword in tma.type.lower(): + gpu = tma + elif template.guestAccelerators: + tga = template.guestAccelerators[0] + if gpu_keyword in tga.acceleratorType.lower(): + gpu = AcceleratorInfo( + type=tga.acceleratorType, + count=tga.acceleratorCount) + return gpu + + +def trim_self_link(link: str): + """get resource name from self link url, eg. + https://.../v1/projects//regions/ + -> + """ + try: + return link[link.rindex("/") + 1 :] + except ValueError: + raise Exception(f"'/' not found, not a self link: '{link}' ") + + +def get_self_link_component(link: str, component_name: str): + """ + Extracts a component (e.g., 'region', 'project') from a self-link URL. + Args: + link: The self-link URL string. + component_name: The name of the component to extract (e.g., 'regions', 'projects'). + Returns: + The extracted component value (e.g., '', ''), + or None if the component is not found in the link. + """ + search_string = f"/{component_name}/" + start_index = link.rfind(search_string) + + if start_index == -1: + return None + + start_index += len(search_string) + end_index = link.find("/", start_index) + + if end_index == -1: + # If no further slash, the rest of the string is the component + return link[start_index:] + else: + return link[start_index:end_index] + + +def execute_with_futures(func, seq): + with ThreadPoolExecutor() as exe: + futures = [] + for i in seq: + future = exe.submit(func, i) + futures.append(future) + for future in as_completed(futures): + result = future.exception() + if result is not None: + raise result + + +def map_with_futures(func, seq): + with ThreadPoolExecutor() as exe: + futures = [] + for i in seq: + future = exe.submit(func, i) + futures.append(future) + for future in futures: + # Will be result or raise Exception + res = None + try: + res = future.result() + except Exception as e: + res = e + yield res + +def should_mount_slurm_bucket() -> bool: + try: + return instance_metadata("attributes/slurm_bucket_mount", silent=True).lower() == "true" + except MetadataNotFoundError: + return False + + +def _get_bucket_and_common_prefix() -> Tuple[str, str]: + uri = instance_metadata("attributes/slurm_bucket_path") + return parse_bucket_uri(uri) + +def blob_get(file): + bucket_name, path = _get_bucket_and_common_prefix() + blob_name = f"{path}/{file}" + return storage_client().get_bucket(bucket_name).blob(blob_name) + + +def blob_list(prefix="", delimiter=None): + bucket_name, path = _get_bucket_and_common_prefix() + blob_prefix = f"{path}/{prefix}" + # Note: The call returns a response only when the iterator is consumed. + blobs = storage_client().list_blobs( + bucket_name, prefix=blob_prefix, delimiter=delimiter + ) + return [blob for blob in blobs] + +def file_list(prefix="", subpath="") -> List[os.DirEntry]: + path = dirs.slurm_bucket_mount + file_prefix = f"{path}/{subpath}" + try: + files = os.scandir(file_prefix) + return [file for file in files if file.name.startswith(prefix)] + except: + return [] + # Not considering lack of file's existence as fatal (we may check for files we know don't exist). + # Responsibility of callee to determine if it is fatal or not, blob_list returns empty iterator in similar cases. + +def hash_file(fullpath: Path) -> str: + with open(fullpath, "rb") as f: + file_hash = hashlib.md5() + chunk = f.read(8192) + while chunk: + file_hash.update(chunk) + chunk = f.read(8192) + return base64.b64encode(file_hash.digest()).decode("utf-8") + + +def install_custom_scripts(check_hash:bool=False): + """download custom scripts from gcs bucket""" + role, tokens = lookup().instance_role, [] + + mounted_scripts=False + if should_mount_slurm_bucket() and role != "controller": + mounted_scripts=True + + all_prolog_tokens = ["prolog", "epilog", "task_prolog", "task_epilog"] + if role == "controller": + tokens = ["controller"] + all_prolog_tokens + elif role == "compute": + tokens = [f"nodeset-{lookup().node_nodeset_name()}"] + all_prolog_tokens + elif role == "login": + tokens = [f"login-{instance_login_group()}"] + + prefixes = [f"slurm-{tok}-script" for tok in tokens] + + # TODO: use single `blob_list`, to reduce ~4x number of GCS requests + if mounted_scripts: + source_collection = list(chain.from_iterable(file_list(prefix=p) for p in prefixes)) + else: + source_collection = list(chain.from_iterable(blob_list(prefix=p) for p in prefixes)) + + script_pattern = re.compile(r"^slurm-(?P\S+)-script-(?P\S+)") + for source in source_collection: + if mounted_scripts: + m = script_pattern.match(source.name) + else: + m = script_pattern.match(Path(source.name).name) + + if not m: + log.warning(f"found blob that doesn't match expected pattern: {source.name}") + continue + path_parts = m["path"].split("-") + path_parts[0] += ".d" + stem, _, ext = m["name"].rpartition("_") + filename = ".".join((stem, ext)) + + path = Path(*path_parts, filename) + fullpath = (dirs.custom_scripts / path).resolve() + mkdirp(fullpath.parent) + + for par in path.parents: + chown_slurm(dirs.custom_scripts / par) + need_update = True + + if check_hash and fullpath.exists() and isinstance(source,storage.Blob): + # TODO: MD5 reported by gcloud may differ from the one calculated here (e.g. if blob got gzipped), + # consider using gCRC32C + need_update = hash_file(fullpath) != source.md5_hash + + log.info(f"installing custom script: {path} from {source.name}") + + if isinstance(source,os.DirEntry): + shutil.copy(source.path, fullpath) #Needs to be copied since mounted nfs is read-only + chown_slurm(fullpath, mode=0o755) + + elif need_update: + with fullpath.open("wb") as f: + source.download_to_file(f) + chown_slurm(fullpath, mode=0o755) + +def compute_service(version="beta"): + """Make thread-safe compute service handle + creates a new Http for each request + """ + credentials = get_credentials() + dev_key = get_dev_key() + + def build_request(http, *args, **kwargs): + new_http = set_user_agent(httplib2.Http(), USER_AGENT) + if credentials is not None: + new_http = google_auth_httplib2.AuthorizedHttp(credentials, http=new_http) + return googleapiclient.http.HttpRequest(new_http, *args, **kwargs) + + ver = endpoint_version(ApiEndpoint.COMPUTE) + disc_url = googleapiclient.discovery.DISCOVERY_URI + if ver: + version = ver + disc_url = disc_url.replace(DEFAULT_UNIVERSE_DOMAIN, universe_domain()) + + log.debug(f"Using version={version} of Google Compute Engine API") + return googleapiclient.discovery.build( + "compute", + version, + requestBuilder=build_request, + credentials=credentials, + developerKey=dev_key, + discoveryServiceUrl=disc_url, + cache_discovery=False, # See https://github.com/googleapis/google-api-python-client/issues/299 + ) + +def storage_client() -> storage.Client: + """ + Config-independent storage client + """ + ud = universe_domain() + co = {} + if ud and ud != DEFAULT_UNIVERSE_DOMAIN: + co["universe_domain"] = ud + return storage.Client(client_options=ClientOptions(**co)) + + +class DeffetiveStoredConfigError(Exception): + """ + Raised when config can not be loaded and assembled from bucket + """ + pass + + +def _fill_cfg_defaults(cfg: NSDict) -> NSDict: + if not cfg.slurm_log_dir: + cfg.slurm_log_dir = dirs.log + if not cfg.slurm_bin_dir: + cfg.slurm_bin_dir = slurmdirs.prefix / "bin" + if not cfg.slurm_control_host: + try: + control_dns_name = instance_metadata("attributes/slurm_control_dns", silent=True) + cfg.slurm_control_host = control_dns_name + except MetadataNotFoundError: + cfg.slurm_control_host = f"{cfg.slurm_cluster_name}-controller" + if not cfg.slurm_control_host_port: + cfg.slurm_control_host_port = "6820-6830" + return cfg + +@dataclass +class _ConfigBlobs: + """ + "Private" class that represent a collection of GCS blobs for configuration + """ + core: storage.Blob + controller_addr: Optional[storage.Blob] + partition: List[storage.Blob] = field(default_factory=list) + nodeset: List[storage.Blob] = field(default_factory=list) + nodeset_dyn: List[storage.Blob] = field(default_factory=list) + nodeset_tpu: List[storage.Blob] = field(default_factory=list) + login_group: List[storage.Blob] = field(default_factory=list) + + @property + def hash(self) -> str: + h = hashlib.md5() + all = [self.core] + self.partition + self.nodeset + self.nodeset_dyn + self.nodeset_tpu + if self.controller_addr: + all.append(self.controller_addr) + + # sort blobs so hash is consistent + for blob in sorted(all, key=lambda b: b.name): + h.update(blob.md5_hash.encode("utf-8")) + return h.hexdigest() + +@dataclass +class _ConfigFiles: + """ + "Private" class that represent a collection of files for configuration + """ + core: Path + controller_addr: Optional[Path] + partition: List[Path] = field(default_factory=list) + nodeset: List[Path] = field(default_factory=list) + nodeset_dyn: List[Path] = field(default_factory=list) + nodeset_tpu: List[Path] = field(default_factory=list) + login_group: List[Path] = field(default_factory=list) + +def _list_config_blobs() -> _ConfigBlobs: + _, common_prefix = _get_bucket_and_common_prefix() + + core: Optional[storage.Blob] = None + controller_addr: Optional[storage.Blob] = None + rest: Dict[str, List[storage.Blob]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} + + is_controller = instance_role() == "controller" + + for blob in blob_list(prefix=""): + if blob.name == f"{common_prefix}/config.yaml": + core = blob + if blob.name == f"{common_prefix}/controller_addr.yaml" and not is_controller: + # Don't add this config blobs for controller to avoid "double reconfiguration": + # Initially this file doesn't exist and produce later by `setup_controller`; + # Appearance of this blob would trigger change in combined hash of config files; + # Ignore existence of this file for controller, assume that + # no other instance nodes will proceed with configuration until this file is created. + controller_addr = blob + for key in rest.keys(): + if blob.name.startswith(f"{common_prefix}/{key}_configs/"): + rest[key].append(blob) + + if core is None: + raise DeffetiveStoredConfigError(f"{common_prefix}/config.yaml not found in bucket") + + return _ConfigBlobs(core=core, controller_addr=controller_addr, **rest) + +def _list_config_files() -> _ConfigFiles: + file_dir = dirs.slurm_bucket_mount + core: Optional[Path] = None + controller_addr: Optional[Path] = None + rest: Dict[str, List[Path]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} + + if Path(f"{file_dir}/config.yaml").exists(): + core = Path(f"{file_dir}/config.yaml") + + for key in rest.keys(): + for f in file_list(subpath=f"{key}_configs"): + rest[key].append(f.path) + + if core is None: + raise Exception(f"config.yaml was not found in mounted folder: {dirs.slurm_bucket_mount}") #Intentionally not using DeffetiveStoredConfigError as this is considered a fatal error + + return _ConfigFiles(core=core, controller_addr=None, **rest) + +def _fetch_config(old_hash: Optional[str]) -> Optional[Tuple[NSDict, str]]: + """Fetch config from bucket, returns None if no changes are detected.""" + blobs = _list_config_blobs() + if old_hash == blobs.hash: + return None + + def _download(bs) -> List[Any]: + return [yaml.safe_load(b.download_as_text()) for b in bs] + + return _assemble_config( + core=_download([blobs.core])[0], + controller_addr=_download([blobs.controller_addr])[0] if blobs.controller_addr else None, + partitions=_download(blobs.partition), + nodesets=_download(blobs.nodeset), + nodesets_dyn=_download(blobs.nodeset_dyn), + nodesets_tpu=_download(blobs.nodeset_tpu), + login_groups=_download(blobs.login_group), + ), blobs.hash + +def _fetch_mounted_config() -> Optional[Tuple[NSDict, str]]: + if not dirs.slurm_bucket_mount.is_mount(): + raise Exception(f"{dirs.slurm_bucket_mount} is not mounted") + + files = _list_config_files() + + def _load(files) -> List[Any]: + file_yaml=[] + for file in files: + with open(file, "r") as f: + file_yaml.append(yaml.safe_load(f)) + return file_yaml + + return _assemble_config( + core=_load([files.core])[0], + controller_addr=None, + partitions=_load(files.partition), + nodesets=_load(files.nodeset), + nodesets_dyn=_load(files.nodeset_dyn), + nodesets_tpu=_load(files.nodeset_tpu), + login_groups=_load(files.login_group), + ) + +def controller_lookup_self_ip() -> str: + assert instance_role() == "controller" + # Get IP of LAST network-interface + # TODO: Consider change order of NICs definition, so right NIC is always @0. + idx = instance_metadata("network-interfaces").split()[-1] # either `0/` or `1/` + return instance_metadata(f"network-interfaces/{idx}ip") + +def _assemble_config( + core: Any, + controller_addr: Optional[Any], + partitions: List[Any], + nodesets: List[Any], + nodesets_dyn: List[Any], + nodesets_tpu: List[Any], + login_groups: List[Any], + ) -> NSDict: + cfg = NSDict(core) + + if cfg.controller_network_attachment: + # lookup controller address + if instance_role() == "controller": + # ignore stored value of `controller_addr`, it will be overwritten during `setup_controller` + cfg.slurm_control_addr = controller_lookup_self_ip() + else: + if not controller_addr: + raise DeffetiveStoredConfigError("controller_addr.yaml not found in bucket") + cfg.slurm_control_addr = controller_addr["slurm_control_addr"] + + # add partition configs + for p_yaml in partitions: + p_cfg = NSDict(p_yaml) + assert p_cfg.get("partition_name"), "partition_name is required" + p_name = p_cfg.partition_name + assert p_name not in cfg.partitions, f"partition {p_name} already defined" + cfg.partitions[p_name] = p_cfg + + # add nodeset configs + ns_names = set() + def _add_nodesets(yamls: List[Any], target: dict): + for ns_yaml in yamls: + ns_cfg = NSDict(ns_yaml) + assert ns_cfg.get("nodeset_name"), "nodeset_name is required" + ns_name = ns_cfg.nodeset_name + assert ns_name not in ns_names, f"nodeset {ns_name} already defined" + target[ns_name] = ns_cfg + ns_names.add(ns_name) + + _add_nodesets(nodesets, cfg.nodeset) + _add_nodesets(nodesets_dyn, cfg.nodeset_dyn) + _add_nodesets(nodesets_tpu, cfg.nodeset_tpu) + + # validate that configs for all referenced nodesets are present + for p in cfg.partitions.values(): + for ns_name in chain(p.partition_nodeset, p.partition_nodeset_dyn, p.partition_nodeset_tpu): + if ns_name not in ns_names: + raise DeffetiveStoredConfigError(f"nodeset {ns_name} not defined in config") + + for lg_yaml in login_groups: + lg_cfg = NSDict(lg_yaml) + assert lg_cfg.get("group_name"), "group_name is required" + lg_name = lg_cfg.group_name + assert lg_name not in cfg.login_groups + cfg.login_groups[lg_name] = lg_cfg + + if instance_role() == "login": + group = instance_login_group() + if group not in cfg.login_groups: + raise DeffetiveStoredConfigError(f"login group '{group}' does not exist in config") + + return _fill_cfg_defaults(cfg) + +def fetch_config() -> Tuple[bool, NSDict]: + """ + Fetches config from bucket and saves it locally + Returns True if new (updated) config was fetched + """ + hash_file = Path("/slurm/scripts/.config.hash") + old_hash = hash_file.read_text() if hash_file.exists() else None + + if should_mount_slurm_bucket() and instance_role() != "controller": + cfg = _fetch_mounted_config() + CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) + chown_slurm(CONFIG_FILE) + return False, cfg + + cfg_and_hash = _fetch_config(old_hash=old_hash) + + if not cfg_and_hash: + return False, _load_config() + + cfg, hash = cfg_and_hash + hash_file.write_text(hash) + chown_slurm(hash_file) + CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) + chown_slurm(CONFIG_FILE) + return True, cfg + +def owned_file_handler(filename): + """create file handler""" + chown_slurm(filename) + return logging.handlers.WatchedFileHandler(filename, delay=True) + +def get_log_path() -> Path: + """ + Returns path to log file for the current script. + e.g. resume.py -> /var/log/slurm/resume.log + """ + cfg_log_dir = lookup().cfg.slurm_log_dir + log_dir = Path(cfg_log_dir) if cfg_log_dir else dirs.log + return (log_dir / Path(sys.argv[0]).name).with_suffix(".log") + +def init_log_and_parse(parser: argparse.ArgumentParser) -> argparse.Namespace: + parser.add_argument( + "--debug", + "-d", + dest="loglevel", + action="store_const", + const=logging.DEBUG, + default=logging.INFO, + help="Enable debugging output", + ) + parser.add_argument( + "--trace-api", + "-t", + action="store_true", + help="Enable detailed api request output", + ) + args = parser.parse_args() + loglevel = args.loglevel + if lookup().cfg.enable_debug_logging: + loglevel = logging.DEBUG + if args.trace_api: + lookup().cfg.extra_logging_flags["trace_api"] = True + # Configure root logger + logging.config.dictConfig({ + "version": 1, + "disable_existing_loggers": True, + "formatters": { + "standard": { + "format": "%(levelname)s: %(message)s", + }, + "stamp": { + "format": "%(asctime)s %(levelname)s: %(message)s", + }, + }, + "handlers": { + "stdout_handler": { + "level": logging.DEBUG, + "formatter": "standard", + "class": "logging.StreamHandler", + "stream": sys.stdout, + }, + "file_handler": { + "()": owned_file_handler, + "level": logging.DEBUG, + "formatter": "stamp", + "filename": get_log_path(), + }, + }, + "root": { + "handlers": ["stdout_handler", "file_handler"], + "level": loglevel, + }, + }) + + sys.excepthook = _handle_exception + + return args + + +def log_api_request(request): + """log.trace info about a compute API request""" + if not lookup().cfg.extra_logging_flags.get("trace_api"): + return + # output the whole request object as pretty yaml + # the body is nested json, so load it as well + rep = json.loads(request.to_json()) + if rep.get("body", None) is not None: + rep["body"] = json.loads(rep["body"]) + pretty_req = yaml.safe_dump(rep).rstrip() + # label log message with the calling function + log.debug(f"{inspect.stack()[1].function}:\n{pretty_req}") + + +def _handle_exception(exc_type, exc_value, exc_trace): + """log exceptions other than KeyboardInterrupt""" + if not issubclass(exc_type, KeyboardInterrupt): + log.exception("Fatal exception", exc_info=(exc_type, exc_value, exc_trace)) + sys.__excepthook__(exc_type, exc_value, exc_trace) + + +def run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + timeout=None, + check=True, + universal_newlines=True, + **kwargs, +): + """Wrapper for subprocess.run() with convenient defaults""" + if isinstance(args, list): + args = list(filter(lambda x: x is not None, args)) + args = " ".join(args) + if not shell and isinstance(args, str): + args = shlex.split(args) + log.debug(f"run: {args}") + try: + result = subprocess.run( + args, + stdout=stdout, + stderr=stderr, + shell=shell, + timeout=timeout, + check=check, + universal_newlines=universal_newlines, + **kwargs, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + log_subprocess(e) + raise + log_subprocess(result) + return result + +def log_subprocess(subj: subprocess.CalledProcessError | subprocess.TimeoutExpired | subprocess.CompletedProcess) -> None: + match subj: + case subprocess.CompletedProcess(returncode=0): + # Do not log successful runs, to not overwhelm logs (e.g. scontrol show jobs --json) + # TODO: consider still doing it in DEBUG or trim output to few KBs. + return + case subprocess.CompletedProcess(): # non-zero returncode + log.error(f"Command '{subj.args}' returned exit status {subj.returncode}.") + case subprocess.CalledProcessError() | subprocess.TimeoutExpired(): + log.error(str(subj)) + + + def normalize(out: None | str | bytes) -> None | str: + """ + Turns stderr and stdout into string: + > A bytes sequence, or a string if run() was called with an encoding, errors, or text=True. None if was not captured. + """ + match out: + case None: + return None + case str(): + return out.strip() + case bytes(): + return out.decode().strip() + case _: + return repr(out) + + if stdout := normalize(subj.stdout): + log.error(f"stdout: {stdout}") + if stderr := normalize(subj.stderr): + log.error(f"stderr: {stderr}") + + +def chown_slurm(path: Path, mode=None) -> None: + if path.exists(): + if mode: + path.chmod(mode) + else: + mkdirp(path.parent) + if mode: + path.touch(mode=mode) + else: + path.touch() + try: + shutil.chown(path, user="slurm", group="slurm") + except LookupError: + log.warning(f"User 'slurm' does not exist. Cannot 'chown slurm:slurm {path}'.") + except PermissionError: + log.warning(f"Not authorized to 'chown slurm:slurm {path}'.") + except Exception as err: + log.error(err) + + +@contextmanager +def cd(path): + """Change working directory for context""" + prev = Path.cwd() + os.chdir(path) + try: + yield + finally: + os.chdir(prev) + + +def cached_property(f): + return property(lru_cache()(f)) + + +def retry(max_retries: int, init_wait_time: float, warn_msg: str, exc_type: Type[Exception]): + """Retries functions that raises the exception exc_type. + Retry time is increased by a factor of two for every iteration. + + Args: + max_retries (int): Maximum number of retries + init_wait_time (float): Initial wait time in secs + warn_msg (str): Message to print during retries + exc_type (Exception): Exception type to check for + """ + + if max_retries <= 0: + raise ValueError("Incorrect value for max_retries, must be >= 1") + if init_wait_time <= 0.0: + raise ValueError("Invalid value for init_wait_time, must be > 0.0") + + def decorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + retry = 0 + secs = init_wait_time + captured_exc: Optional[BaseException] = None + while retry < max_retries: + try: + return f(*args, **kwargs) + except exc_type as e: + captured_exc = e + log.warn(f"{warn_msg}, retrying in {secs}") + sleep(secs) + retry += 1 + secs *= 2 + assert captured_exc + raise captured_exc + + return wrapper + + return decorator + + +def separate(pred: Callable[[Any], bool], coll: Iterable[Any]) -> Tuple[List[Any], List[Any]]: + """filter into 2 lists based on pred returning True or False + returns ([False], [True]) + """ + res: Tuple[List[Any], List[Any]] = ([],[]) + for el in coll: + res[pred(el)].append(el) + return res + + +def chunked(iterable, n=API_REQ_LIMIT): + """group iterator into chunks of max size n""" + it = iter(iterable) + while True: + chunk = list(islice(it, n)) + if not chunk: + return + yield chunk + +def groupby_unsorted(seq: Sequence[Any], key): + indices = defaultdict(list) + for i, el in enumerate(seq): + indices[key(el)].append(i) + for k, idxs in indices.items(): + yield k, (seq[i] for i in idxs) + + +@lru_cache(maxsize=32) +def find_ratio(a, n, s, r0=None): + """given the start (a), count (n), and sum (s), find the ratio required""" + if n == 2: + return s / a - 1 + an = a * n + if n == 1 or s == an: + return 1 + if r0 is None: + # we only need to know which side of 1 to guess, and the iteration will work + r0 = 1.1 if an < s else 0.9 + + # geometric sum formula + def f(r): + return a * (1 - r**n) / (1 - r) - s + + # derivative of f + def df(r): + rm1 = r - 1 + rn = r**n + return (a * (rn * (n * rm1 - r) + r)) / (r * rm1**2) + + MIN_DR = 0.0001 # negligible change + r = r0 + # print(f"r(0)={r0}") + MAX_TRIES = 64 + for i in range(1, MAX_TRIES + 1): + try: + dr = f(r) / df(r) + except ZeroDivisionError: + log.error(f"Failed to find ratio due to zero division! Returning r={r0}") + return r0 + r = r - dr + # print(f"r({i})={r}") + # if the change in r is small, we are close enough + if abs(dr) < MIN_DR: + break + else: + log.error(f"Could not find ratio after {MAX_TRIES}! Returning r={r0}") + return r0 + return r + + +def backoff_delay(start, timeout=None, ratio=None, count: int = 0): + """generates `count` waits starting at `start` + sum of waits is `timeout` or each one is `ratio` bigger than the last + the last wait is always 0""" + # timeout or ratio must be set but not both + assert (timeout is None) ^ (ratio is None) + assert ratio is None or ratio > 0 + assert timeout is None or timeout >= start + assert (count > 1 or timeout is not None) and isinstance(count, int) + assert start > 0 + + if count == 0: + # Equation for auto-count is tuned to have a max of + # ~int(timeout) counts with a start wait of <0.01. + # Increasing start wait decreases count eg. + # backoff_delay(10, timeout=60) -> count = 5 + count = int( + (timeout / ((start + 0.05) ** (1 / 2)) + 2) // math.log(timeout + 2) + ) + + yield start + # if ratio is set: + # timeout = start * (1 - ratio**(count - 1)) / (1 - ratio) + if ratio is None: + ratio = find_ratio(start, count - 1, timeout) + + wait = start + # we have start and 0, so we only need to generate count - 2 + for _ in range(count - 2): + wait *= ratio + yield wait + yield 0 + return + + +ROOT_URL = "http://metadata.google.internal/computeMetadata/v1" + +class MetadataNotFoundError(Exception): + pass + +def get_metadata(path:str, silent=False) -> str: + """Get metadata relative to metadata/computeMetadata/v1""" + HEADERS = {"Metadata-Flavor": "Google"} + url = f"{ROOT_URL}/{path}" + try: + resp = requests_lib.get(url, headers=HEADERS) + resp.raise_for_status() + return resp.text + except requests_lib.exceptions.HTTPError: + if not silent: + log.warning(f"metadata not found ({url})") + raise MetadataNotFoundError(f"failed to get_metadata from {url}") + + +@lru_cache(maxsize=None) +def instance_metadata(path: str, silent:bool=False) -> str: + return get_metadata(f"instance/{path}", silent=silent) + +def instance_role(): + return instance_metadata("attributes/slurm_instance_role") + + +def instance_login_group(): + return instance_metadata("attributes/slurm_login_group") + + +def natural_sort(text): + def atoi(text): + return int(text) if text.isdigit() else text + + return [atoi(w) for w in re.split(r"(\d+)", text)] + + +def to_hostlist(names: Iterable[str]) -> str: + """ + Fast implementation of `hostlist` that doesn't invoke `scontrol` + IMPORTANT: + * Acts as `scontrol show hostlistsorted`, i.e. original order is not preserved + * Achieves worse compression than `scontrol show hostlist` for some cases + """ + pref = defaultdict(list) + tokenizer = re.compile(r"^(.*?)(\d*)$") + for name in filter(None, names): + matches = tokenizer.match(name) + assert matches, name + p, s = matches.groups() + pref[p].append(s) + + def _compress_suffixes(ss: List[str]) -> List[str]: + cur, res = None, [] + + def cur_repr(): + assert cur + nums, strs = cur + if nums[0] == nums[1]: + return strs[0] + return f"{strs[0]}-{strs[1]}" + + for s in sorted(ss, key=int): + n = int(s) + if cur is None: + cur = ((n, n), (s, s)) + continue + + nums, strs = cur + if n == nums[1] + 1: + cur = ((nums[0], n), (strs[0], s)) + else: + res.append(cur_repr()) + cur = ((n, n), (s, s)) + if cur: + res.append(cur_repr()) + return res + + res = [] + for p in sorted(pref.keys()): + sl = defaultdict(list) + for s in pref[p]: + sl[len(s)].append(s) + cs = [] + for ln in sorted(sl.keys()): + if ln == 0: + res.append(p) + else: + cs.extend(_compress_suffixes(sl[ln])) + if not cs: + continue + if len(cs) == 1 and "-" not in cs[0]: + res.append(f"{p}{cs[0]}") + else: + res.append(f"{p}[{','.join(cs)}]") + return ",".join(res) + +@lru_cache(maxsize=None) +def to_hostnames(nodelist: str) -> List[str]: + """make list of hostnames from hostlist expression""" + if not nodelist: + return [] # avoid degenerate invocation of scontrol + if isinstance(nodelist, str): + hostlist = nodelist + else: + hostlist = ",".join(nodelist) + hostnames = run(f"{lookup().scontrol} show hostnames {hostlist}").stdout.splitlines() + return hostnames + + +def retry_exception(exc) -> bool: + """return true for exceptions that should always be retried""" + msg = str(exc) + retry_errors = ( + "Rate Limit Exceeded", + "Quota Exceeded", + "Quota exceeded", + ) + return any(err in msg for err in retry_errors) + + +def ensure_execute(request): + """Handle rate limits and socket time outs""" + + for retry, wait in enumerate(backoff_delay(0.5, timeout=10 * 60, count=20)): + try: + return request.execute() + except googleapiclient.errors.HttpError as e: + if retry_exception(e): + log.error(f"retry:{retry} '{e}'") + sleep(wait) + continue + raise + + except socket.timeout as e: + # socket timed out, try again + log.debug(e) + + except Exception as e: + log.error(e, exc_info=True) + raise + + break + + +def batch_execute(requests, retry_cb=None, log_err=log.error): + """execute list or dict as batch requests + retry if retry_cb returns true + """ + BATCH_LIMIT = 1000 + if not isinstance(requests, dict): + requests = {str(k): v for k, v in enumerate(requests)} # rid generated here + done = {} + failed = {} + timestamps: List[float] = [] + rate_limited = False + + def batch_callback(rid, resp, exc): + nonlocal rate_limited + if exc is not None: + log_err(f"compute request exception {rid}: {exc}") + if retry_exception(exc): + rate_limited = True + else: + req = requests.pop(rid) + failed[rid] = (req, exc) + else: + # if retry_cb is set, don't move to done until it returns false + if retry_cb is None or not retry_cb(resp): + requests.pop(rid) + done[rid] = resp + + def batch_request(reqs): + batch = lookup().compute.new_batch_http_request(callback=batch_callback) + for rid, req in reqs: + batch.add(req, request_id=rid) + return batch + + while requests: + if timestamps: + timestamps = [stamp for stamp in timestamps if stamp > time()] + if rate_limited and timestamps: + stamp = next(iter(timestamps)) + sleep(max(stamp - time(), 0)) + rate_limited = False + # up to API_REQ_LIMIT (2000) requests + # in chunks of up to BATCH_LIMIT (1000) + batches = [ + batch_request(chunk) + for chunk in chunked(islice(requests.items(), API_REQ_LIMIT), BATCH_LIMIT) + ] + timestamps.append(time() + 100) + with ThreadPoolExecutor() as exe: + futures = [] + for batch in batches: + future = exe.submit(ensure_execute, batch) + futures.append(future) + for future in futures: + result = future.exception() + if result is not None: + raise result + + return done, failed + + +def get_operation_req(lkp: "Lookup", name: str, region: Optional[str]=None, zone: Optional[str]=None) -> Any: + if zone: + return lkp.compute.zoneOperations().get(project=lkp.project, zone=zone, operation=name) + elif region: + return lkp.compute.regionOperations().get(project=lkp.project, region=region, operation=name) + return lkp.compute.globalOperations().get(project=lkp.project, operation=name) + +def wait_request(operation, project: str): + """makes the appropriate wait request for a given operation""" + if "zone" in operation: + req = lookup().compute.zoneOperations().wait( + project=project, + zone=trim_self_link(operation["zone"]), + operation=operation["name"], + ) + elif "region" in operation: + req = lookup().compute.regionOperations().wait( + project=project, + region=trim_self_link(operation["region"]), + operation=operation["name"], + ) + else: + req = lookup().compute.globalOperations().wait( + project=project, operation=operation["name"] + ) + return req + + +def wait_for_operation(operation) -> Dict[str, Any]: + """wait for given operation""" + project = parse_self_link(operation["selfLink"]).project + wait_req = wait_request(operation, project=project) + + while True: + result = ensure_execute(wait_req) + if result["status"] == "DONE": + log_errors = " with errors" if "error" in result else "" + log.debug( + f"operation complete{log_errors}: type={result['operationType']}, name={result['name']}" + ) + return result + + + +def getThreadsPerCore(template) -> int: + if not template.machine_type.supports_smt: + return 1 + return template.advancedMachineFeatures.threadsPerCore or 2 + + +@retry( + max_retries=9, + init_wait_time=1, + warn_msg="Temporary failure in name resolution", + exc_type=socket.gaierror, +) +def host_lookup(host_name: str) -> str: + return socket.gethostbyname(host_name) + + +class Dumper(yaml.SafeDumper): + """Add representers for pathlib.Path and NSDict for yaml serialization""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.add_representer(NSDict, self.represent_nsdict) + self.add_multi_representer(Path, self.represent_path) + + @staticmethod + def represent_nsdict(dumper, data): + return dumper.represent_mapping("tag:yaml.org,2002:map", data.items()) + + @staticmethod + def represent_path(dumper, path): + return dumper.represent_scalar("tag:yaml.org,2002:str", str(path)) + + +@dataclass(frozen=True) +class ReservationDetails: + project: str + zone: str + name: str + policies: List[str] # names (not URLs) of resource policies + bulk_insert_name: str # name in format suitable for bulk insert (currently identical to user supplied name in long format) + deployment_type: Optional[str] + reservation_mode: Optional[str] + assured_count: int + delete_at_time: Optional[datetime] + + @property + def dense(self) -> bool: + return self.deployment_type == "DENSE" + + @property + def calendar(self) -> bool: + return self.reservation_mode == "CALENDAR" + +@dataclass(frozen=True) +class FutureReservation: + project: str + zone: str + name: str + specific: bool + start_time: datetime + end_time: datetime + reservation_mode: Optional[str] + active_reservation: Optional[ReservationDetails] + + @property + def calendar(self) -> bool: + return self.reservation_mode == "CALENDAR" + +@dataclass +class Job: + id: int + name: Optional[str] = None + required_nodes: Optional[str] = None + job_state: Optional[str] = None + duration: Optional[timedelta] = None + +@dataclass(frozen=True) +class NodeState: + base: str + flags: frozenset + +class Lookup: + """Wrapper class for cached data access""" + + def __init__(self, cfg): + self._cfg = cfg + + @property + def cfg(self): + return self._cfg + + @property + def project(self): + return self.cfg.project or authentication_project() + + @cached_property + def control_addr(self) -> Optional[str]: + return self.cfg.get("slurm_control_addr", None) + + @property + def control_host(self): + return self.cfg.slurm_control_host + + @cached_property + def control_host_addr(self): + return self.control_addr or host_lookup(self.cfg.slurm_control_host) + + @property + def control_host_port(self): + return self.cfg.slurm_control_host_port + + @property + def endpoint_versions(self): + return self.cfg.endpoint_versions + + @property + def scontrol(self): + return Path(self.cfg.slurm_bin_dir or "") / "scontrol" + + @cached_property + def instance_role(self): + return instance_role() + + @cached_property + def instance_role_safe(self): + try: + role = self.instance_role + except Exception as e: + log.error(e) + role = None + return role + + @property + def is_controller(self): + return self.instance_role_safe == "controller" + + @property + def is_login_node(self): + return self.instance_role_safe == "login" + + @cached_property + def compute(self): + # TODO evaluate when we need to use google_app_cred_path + if self.cfg.google_app_cred_path: + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.cfg.google_app_cred_path + return compute_service() + + @cached_property + def hostname(self): + return socket.gethostname() + + @cached_property + def hostname_fqdn(self): + return socket.getfqdn() + + @cached_property + def zone(self): + return instance_metadata("zone") + + node_desc_regex = re.compile( + r"^(?P(?P[^\s\-]+)-(?P\S+))-(?P(?P\w+)|(?P\[[\d,-]+\]))$" + ) + + @lru_cache(maxsize=None) + def _node_desc(self, node_name): + """Get parts from node name""" + if not node_name: + node_name = self.hostname + # workaround below is for VMs whose hostname is FQDN + node_name_short = node_name.split(".")[0] + m = self.node_desc_regex.match(node_name_short) + if not m: + raise Exception(f"node name {node_name} is not valid") + return m.groupdict() + + def node_prefix(self, node_name=None): + return self._node_desc(node_name)["prefix"] + + def node_index(self, node: str) -> int: + """ node_index("cluster-nodeset-45") == 45 """ + suff = self._node_desc(node)["suffix"] + + if suff is None: + raise ValueError(f"Node {node} name does not end with numeric index") + return int(suff) + + def node_nodeset_name(self, node_name=None): + return self._node_desc(node_name)["nodeset"] + + def node_nodeset(self, node_name=None): + nodeset_name = self.node_nodeset_name(node_name) + if nodeset_name in self.cfg.nodeset_tpu: + return self.cfg.nodeset_tpu[nodeset_name] + + return self.cfg.nodeset[nodeset_name] + + def partition_is_tpu(self, part: str) -> bool: + """check if partition with name part contains a nodeset of type tpu""" + return len(self.cfg.partitions[part].partition_nodeset_tpu) > 0 + + + def node_is_tpu(self, node_name=None): + nodeset_name = self.node_nodeset_name(node_name) + return self.cfg.nodeset_tpu.get(nodeset_name) is not None + + def nodeset_is_tpu(self, nodeset_name=None) -> bool: + return self.cfg.nodeset_tpu.get(nodeset_name) is not None + + def node_is_fr(self, node_name:str) -> bool: + return bool(self.node_nodeset(node_name).future_reservation) + + def is_dormant_res_node(self, node_name:str) -> bool: + fr = self.future_reservation(self.node_nodeset(node_name)) + res = self.nodeset_reservation(self.node_nodeset(node_name)) + + if fr is None and res is None: + return False + + if fr: + return fr.active_reservation is None + + if res: + if res.calendar: + # If reservation is calendar based, check if it is past the delete_at_time + if res.delete_at_time is not None and now() >= res.delete_at_time: + log.debug(f"DWS calendar reservation {res.bulk_insert_name} is past deletion time {res.delete_at_time}, skipping resume.") + return True + + # If assured_count is 0 do not resume nodes as they are not active yet + if res.delete_at_time is not None and res.assured_count <= 0: + log.debug(f"DWS calendar reservation {res.bulk_insert_name} is not active yet, skipping resume.") + return True + + return False + + def node_is_dyn(self, node_name=None) -> bool: + nodeset = self.node_nodeset_name(node_name) + return self.cfg.nodeset_dyn.get(nodeset) is not None + + def node_is_gke(self, node_name=None) -> bool: + return self.nodeset_is_gke(self.node_nodeset(node_name)) + + def nodeset_is_gke(self, nodeset=None) -> bool: + return "gke_nodepool" in nodeset + + def node_template(self, node_name=None) -> str: + """ Self link of nodeset template """ + return self.node_nodeset(node_name).instance_template + + def node_template_info(self, node_name=None): + return self.template_info(self.node_template(node_name)) + + def node_region(self, node_name=None): + nodeset = self.node_nodeset(node_name) + return parse_self_link(nodeset.subnetwork).region + + def nodeset_accelerator_topology(self, nodeset_name: str) -> Optional[str]: + if not self.nodeset_is_tpu(nodeset_name): + return getattr(self.cfg.nodeset[nodeset_name], 'accelerator_topology', None) + return None + + def nodeset_prefix(self, nodeset_name): + return f"{self.cfg.slurm_cluster_name}-{nodeset_name}" + + def nodelist_range(self, nodeset_name: str, start: int, count: int) -> str: + assert 0 <= start and 0 < count + pref = self.nodeset_prefix(nodeset_name) + if count == 1: + return f"{pref}-{start}" + return f"{pref}-[{start}-{start + count - 1}]" + + def static_dynamic_sizes(self, nodeset: NSDict) -> Tuple[int, int]: + return (nodeset.node_count_static or 0, nodeset.node_count_dynamic_max or 0) + + def nodelist(self, nodeset) -> str: + cnt = sum(self.static_dynamic_sizes(nodeset)) + if cnt == 0: + return "" + return self.nodelist_range(nodeset.nodeset_name, 0, cnt) + + def nodenames(self, nodeset) -> Tuple[Iterable[str], Iterable[str]]: + pref = self.nodeset_prefix(nodeset.nodeset_name) + s_count, d_count = self.static_dynamic_sizes(nodeset) + return ( + (f"{pref}-{i}" for i in range(s_count)), + (f"{pref}-{i}" for i in range(s_count, s_count + d_count)), + ) + + def power_managed_nodesets(self) -> Iterable[NSDict]: + return chain(self.cfg.nodeset.values(), self.cfg.nodeset_tpu.values()) + + def is_power_managed_node(self, node_name: str) -> bool: + try: + ns = self.node_nodeset(node_name) + if ns is None: + return False + idx = int(self._node_desc(node_name)["suffix"]) + return idx < sum(self.static_dynamic_sizes(ns)) + except Exception: + return False + + def is_static_node(self, node_name: str) -> bool: + if not self.is_power_managed_node(node_name): + return False + idx = int(self._node_desc(node_name)["suffix"]) + return idx < self.node_nodeset(node_name).node_count_static + + @lru_cache(maxsize=None) + def slurm_nodes(self) -> Dict[str, NodeState]: + def parse_line(node_line) -> Tuple[str, NodeState]: + """turn node,state line to (node, NodeState)""" + # state flags include: CLOUD, COMPLETING, DRAIN, FAIL, POWERED_DOWN, + # POWERING_DOWN + node, fullstate = node_line.split(",") + state = fullstate.split("+") + state_tuple = NodeState(base=state[0], flags=frozenset(state[1:])) + return (node, state_tuple) + + cmd = ( + f"{self.scontrol} show nodes | " + r"grep -oP '^NodeName=\K(\S+)|\s+State=\K(\S+)' | " + r"paste -sd',\n'" + ) + node_lines = run(cmd, shell=True).stdout.rstrip().splitlines() + nodes = { + node: state + for node, state in map(parse_line, node_lines) + if "CLOUD" in state.flags or "DYNAMIC_NORM" in state.flags + } + return nodes + + def node_state(self, nodename: str) -> Optional[NodeState]: + state = self.slurm_nodes().get(nodename) + if state is not None: + return state + + # state is None => Slurm doesn't know this node, + # there are two reasons: + # * happy: + # * node belongs to removed nodeset + # * node belongs to downsized portion of nodeset + # * dynamic node that didn't register itself + # * unhappy: + # * there is a drift in Slurm and SlurmGCP configurations + # * `slurm_nodes` function failed to handle `scontrol show nodes`, + # TODO: make `slurm_nodes` robust by using `scontrol show nodes --json` + # In either of "unhappy" cases it's too dangerous to proceed - abort slurmsync. + try: + ns = self.node_nodeset(nodename) + except: + log.info(f"Unknown node {nodename}, belongs to unknown nodeset") + return None # Can't find nodeset, may be belongs to removed nodeset + + if self.node_is_dyn(nodename): + log.info(f"Unknown node {nodename}, belongs to dynamic nodeset") + return None # we can't make any judjment for dynamic nodes + + cnt = sum(self.static_dynamic_sizes(ns)) + if self.node_index(nodename) >= cnt: + log.info(f"Unknown node {nodename}, out of nodeset size boundaries ({cnt})") + return None # node belongs to downsized nodeset + + raise RuntimeError(f"Slurm does not recognize node {nodename}, potential misconfiguration.") + + + @lru_cache(maxsize=1) + def instances(self) -> Dict[str, Instance]: + instance_information_fields = [ + "creationTimestamp", + "name", + "resourceStatus", + "scheduling", + "status", + "labels.slurm_instance_role", + "zone", + "metadata", + ] + + instance_fields = ",".join(sorted(instance_information_fields)) + fields = f"items.zones.instances({instance_fields}),nextPageToken" + flt = f"labels.slurm_cluster_name={self.cfg.slurm_cluster_name} AND name:{self.cfg.slurm_cluster_name}-*" + act = self.compute.instances() + op = act.aggregatedList(project=self.project, fields=fields, filter=flt) + + instances = {} + while op is not None: + result = ensure_execute(op) + for zone in result.get("items", {}).values(): + for jo in zone.get("instances", []): + inst = Instance.from_json(jo) + if inst.name in instances: + log.error(f"Duplicate VM name {inst.name} across multiple zones") + instances[inst.name] = inst + op = act.aggregatedList_next(op, result) + return instances + + def instance(self, instance_name: str) -> Optional[Instance]: + return self.instances().get(instance_name) + + @lru_cache() + def _get_reservation(self, project: str, zone: str, name: str) -> Any: + """See https://cloud.google.com/compute/docs/reference/rest/v1/reservations""" + return self.compute.reservations().get( + project=project, zone=zone, reservation=name).execute() + + @lru_cache() + def get_mig(self, project: str, region: str, self_link:str) -> Any: + """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" + return self.compute.regionInstanceGroupManagers().get(project=project, region=region, instanceGroupManager=self_link).execute() + + @lru_cache + def get_mig_instances(self, project: str, region: str, self_link:str) -> Any: + return self.compute.regionInstanceGroupManagers().listManagedInstances(project=project, region=region, instanceGroupManager=self_link).execute() + + @lru_cache() + def get_mig_list(self, project: str, region: str) -> Any: + """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" + return self.compute.regionInstanceGroupManagers().list(project=project, region=region).execute() + + @lru_cache() + def _get_future_reservation(self, project:str, zone:str, name: str) -> Any: + """See https://cloud.google.com/compute/docs/reference/rest/v1/futureReservations""" + return self.compute.futureReservations().get(project=project, zone=zone, futureReservation=name).execute() + + def get_reservation_details(self, project:str, zone:str, name:str, bulk_insert_name:str) -> ReservationDetails: + reservation = self._get_reservation(project, zone, name) + + # Converts policy URLs to names, e.g.: + # projects/111111/regions/us-central1/resourcePolicies/zebra -> zebra + policies = [u.split("/")[-1] for u in reservation.get("resourcePolicies", {}).values()] + + return ReservationDetails( + project=project, + zone=zone, + name=name, + policies=policies, + deployment_type=reservation.get("deploymentType"), + reservation_mode=reservation.get("reservationMode"), + assured_count=int(reservation.get("specificReservation", {}).get("assuredCount", 0)), + delete_at_time=parse_gcp_timestamp(reservation.get("deleteAtTime")) if reservation.get("deleteAtTime") else None, + bulk_insert_name=bulk_insert_name) + + def nodeset_reservation(self, nodeset: NSDict) -> Optional[ReservationDetails]: + if not nodeset.reservation_name: + return None + + zones = list(nodeset.zone_policy_allow or []) + assert len(zones) == 1, "Only single zone is supported if using a reservation" + zone = zones[0] + + regex = re.compile(r'^projects/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$') + if not (match := regex.match(nodeset.reservation_name)): + raise ValueError( + f"Invalid reservation name: '{nodeset.reservation_name}', expected format is 'projects/PROJECT/reservations/NAME'" + ) + + project, name = match.group("project", "reservation") + return self.get_reservation_details(project, zone, name, nodeset.reservation_name) + + def future_reservation(self, nodeset: NSDict) -> Optional[FutureReservation]: + if not nodeset.future_reservation: + return None + + active_reservation = None + match = re.search(r'^projects/(?P[^/]+)/zones/(?P[^/]+)/futureReservations/(?P[^/]+)(/.*)?$', nodeset.future_reservation) + assert match, f"Invalid future reservation name '{nodeset.future_reservation}'" + project, zone, name = match.group("project","zone","name") + fr = self._get_future_reservation(project,zone,name) + + start_time = parse_gcp_timestamp(fr["timeWindow"]["startTime"]) + end_time = parse_gcp_timestamp(fr["timeWindow"]["endTime"]) + + if "autoCreatedReservations" in fr["status"] and (res:=fr["status"]["autoCreatedReservations"][0]): + if start_time <= now() <=end_time: + match = re.search(r'projects/(?P[^/]+)/zones/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$',res) + assert match, f"Unexpected reservation name '{res}'" + res_name = match.group("name") + bulk_insert_name = f"projects/{project}/reservations/{res_name}" + active_reservation = self.get_reservation_details(project, zone, res_name, bulk_insert_name) + + return FutureReservation( + project=project, + zone=zone, + name=name, + specific=fr["specificReservationRequired"], + start_time=start_time, + end_time=end_time, + reservation_mode=fr.get("reservationMode"), + active_reservation=active_reservation + ) + + @lru_cache(maxsize=1) + def machine_types(self): + field_names = "name,zone,guestCpus,memoryMb,accelerators" + fields = f"items.zones.machineTypes({field_names}),nextPageToken" + + machines: Dict[str, Dict[str, Any]] = defaultdict(dict) + act = self.compute.machineTypes() + op = act.aggregatedList(project=self.project, fields=fields) + while op is not None: + result = ensure_execute(op) + machine_iter = chain.from_iterable( + scope.get("machineTypes", []) for scope in result["items"].values() + ) + for machine in machine_iter: + name = machine["name"] + zone = machine["zone"] + machines[name][zone] = machine + + op = act.aggregatedList_next(op, result) + return machines + + def machine_type(self, name: str) -> MachineType: + custom_patt = re.compile( + r"((?P\w+)-)?custom-(?P\d+)-(?P\d+)" + ) + if match := custom_patt.match(name): + return MachineType( + name=name, + guest_cpus=int(match.group("cpus")), + memory_mb=int(match.group("mem")), + accelerators=[], + ) + + machines = self.machine_types() + if name not in machines: + raise Exception(f"machine type {name} not found") + per_zone = machines[name] + assert per_zone + return MachineType.from_json( + next(iter(per_zone.values())) # pick the first/any zone + ) + + def template_machine_conf(self, template_link): + template = self.template_info(template_link) + machine = template.machine_type + + machine_conf = NSDict() + machine_conf.boards = 1 # No information, assume 1 + machine_conf.sockets = machine.sockets + # the value below for SocketsPerBoard must be type int + machine_conf.sockets_per_board = machine_conf.sockets // machine_conf.boards + machine_conf.threads_per_core = 1 + _div = 2 if getThreadsPerCore(template) == 1 else 1 + machine_conf.cpus = ( + int(machine.guest_cpus / _div) if machine.supports_smt else machine.guest_cpus + ) + machine_conf.cores_per_socket = int(machine_conf.cpus / machine_conf.sockets) + # Because the actual memory on the host will be different than + # what is configured (e.g. kernel will take it). From + # experiments, about 16 MB per GB are used (plus about 400 MB + # buffer for the first couple of GB's. Using 30 MB to be safe. + gb = machine.memory_mb // 1024 + machine_conf.memory = machine.memory_mb - (400 + (30 * gb)) + return machine_conf + + @lru_cache(maxsize=None) + def template_info(self, template_link): + template_name = trim_self_link(template_link) + cache = file_cache.cache("template_cache") + + if cached := cache.get(template_name): + return NSDict(cached) + + region = get_self_link_component(template_link, "regions") + + template = ensure_execute( + self.compute.instanceTemplates().get( + project=self.project, instanceTemplate=template_name + ) if region is None else + self.compute.regionInstanceTemplates().get( + project=self.project, region=region, instanceTemplate=template_name + ) + ).get("properties") + template = NSDict(template) + # name and link are not in properties, so stick them in + template.name = template_name + template.link = template_link + template.machine_type = self.machine_type(template.machineType) + # TODO delete metadata to reduce memory footprint? + # del template.metadata + + template.gpu = get_template_gpu(template) + + cache.set(template_name, template.to_dict()) + return template + + def _parse_job_info(self, job_info: str) -> Job: + """Extract job details""" + if match:= re.search(r"JobId=(\d+)", job_info): + job_id = int(match.group(1)) + else: + raise ValueError(f"Job ID not found in the job info: {job_info}") + + if match:= re.search(r"TimeLimit=(?:(\d+)-)?(\d{2}):(\d{2}):(\d{2})", job_info): + days, hours, minutes, seconds = match.groups() + duration = timedelta( + days=int(days) if days else 0, + hours=int(hours), + minutes=int(minutes), + seconds=int(seconds) + ) + else: + duration = None + + if match := re.search(r"JobName=([^\n]+)", job_info): + name = match.group(1) + else: + name = None + + if match := re.search(r"JobState=(\w+)", job_info): + job_state = match.group(1) + else: + job_state = None + + if match := re.search(r"ReqNodeList=([^ ]+)", job_info): + required_nodes = match.group(1) + else: + required_nodes = None + + return Job(id=job_id, duration=duration, name=name, job_state=job_state, required_nodes=required_nodes) + + @lru_cache + def get_jobs(self) -> List[Job]: + res = run(f"{self.scontrol} show jobs", timeout=30) + + return [self._parse_job_info(job) for job in res.stdout.split("\n\n")[:-1]] + + @lru_cache + def job(self, job_id: int) -> Optional[Job]: + job_info = run(f"{self.scontrol} show jobid {job_id}", check=False).stdout.rstrip() + if not job_info: + return None + + return self._parse_job_info(job_info=job_info) + + @property + def etc_dir(self) -> Path: + return Path(self.cfg.output_dir or slurmdirs.etc) + + def controller_mount_server_ip(self) -> str: + return self.control_addr or self.control_host + + def normalize_ns_mount(self, ns: Union[dict, NSMount]) -> NSMount: + if isinstance(ns, NSMount): + return ns + + server_ip = ns.get("server_ip") or "$controller" + if server_ip == "$controller": + server_ip = self.controller_mount_server_ip() + + return NSMount( + server_ip=server_ip, + local_mount=Path(ns["local_mount"]), + remote_mount=Path(ns["remote_mount"]), + fs_type=ns["fs_type"], + mount_options=ns["mount_options"], + ) + + @property + def munge_mount(self) -> NSMount: + if self.cfg.munge_mount: + mnt = self.cfg.munge_mount + mnt.local_mount = mnt.local_mount or "/mnt/munge" + return self.normalize_ns_mount(mnt) + else: + return NSMount( + server_ip=self.controller_mount_server_ip(), + local_mount=Path("/mnt/munge"), + remote_mount=dirs.munge, + fs_type="nfs", + mount_options="defaults,hard,intr,_netdev", + ) + + @property + def slurm_key_mount(self) -> NSMount: + if self.cfg.slurm_key_mount: + mnt = self.cfg.slurm_key_mount + mnt.local_mount = mnt.local_mount or slurmdirs.key_distribution + return self.normalize_ns_mount(mnt) + else: + return NSMount( + server_ip=self.controller_mount_server_ip(), + local_mount=slurmdirs.key_distribution, + remote_mount=slurmdirs.key_distribution, + fs_type="nfs", + mount_options="defaults,hard,intr,_netdev", + ) + + def is_flex_node(self, node: str) -> bool: + try: + nodeset = self.node_nodeset(node) + if nodeset.dws_flex.use_bulk_insert: + return False #For legacy flex support + return bool(nodeset.dws_flex.enabled) + except: + return False + + def is_provisioning_flex_node(self, node:str) -> bool: + if not self.is_flex_node(node): + return False + if self.instance(node) is not None: + return True + + nodeset = self.node_nodeset(node) + zones = nodeset.zone_policy_allow + assert len(zones) > 0 + region = self.node_region(node) + + potential_migs=[] + mig_list=self.get_mig_list(self.project, region) + + if not mig_list or not mig_list.get("items"): + return False + + for mig in mig_list["items"]: + if not mig.get("instanceTemplate"): #possibly an old MIG + return False + if mig["instanceTemplate"] == self.node_template(node) and mig["currentActions"]["creating"] > 0: + potential_migs.append(self.get_mig_instances(self.project, region, trim_self_link(mig["selfLink"]))) + + if not potential_migs: + return False + + for instance_collection in potential_migs[0]["managedInstances"]: + if node in instance_collection["name"] and instance_collection["currentAction"]=="CREATING": + return True + return False + + def cluster_regions(self) -> list[str]: + """ + Returns all regions used in cluster + NOTE: only concerned with normal nodesets, + neither TPU, nor dynamic, nor login node, nor controller node are considered + """ + res = set() + for nodeset in self.cfg.nodeset.values(): + res.add(parse_self_link(nodeset.subnetwork).region) + return list(res) + + + +_lkp: Optional[Lookup] = None + +def _load_config() -> NSDict: + return NSDict(yaml.safe_load(CONFIG_FILE.read_text())) + +def lookup() -> Lookup: + global _lkp + if _lkp is None: + try: + cfg = _load_config() + except FileNotFoundError: + log.error(f"config file not found: {CONFIG_FILE}") + cfg = NSDict() # TODO: fail here, once all code paths are covered (mainly init_logging) + _lkp = Lookup(cfg) + return _lkp + +def update_config(cfg: NSDict) -> None: + global _lkp + _lkp = Lookup(cfg) + +def scontrol_reconfigure(lkp: Lookup) -> None: + log.info("Running systemctl restart slurmctld.service") + run("sudo systemctl restart slurmctld.service", timeout=30) + log.info("Running scontrol reconfigure") + run(f"{lkp.scontrol} reconfigure") diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py new file mode 100644 index 0000000000..d1d77a1833 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py @@ -0,0 +1,124 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + + +from dataclasses import dataclass, asdict +import util +import local_pubsub + +import logging +log = logging.getLogger() + +# Name of the topic +TOPIC = "watch_delete_vm_op" + +@dataclass(frozen=True) +class WatchDeleteVmOp_Message: + op_name: str + zone: str + node: str + +class WatchDeleteVmOp_Topic: + def __init__(self, topic: local_pubsub.Topic) -> None: + self._t = topic + + def publish(self, op: dict[str, Any], node: str) -> None: + assert op.get("operationType") == "delete" + assert op.get("zone") + assert node + + msg = WatchDeleteVmOp_Message(op_name=op["name"], zone=op["zone"], node=node) + self._t.publish(data=asdict(msg)) + + +def watch_delete_vm_op_topic() -> WatchDeleteVmOp_Topic: + return WatchDeleteVmOp_Topic(local_pubsub.topic(TOPIC)) + + +def _watch_op(lkp: util.Lookup, m: WatchDeleteVmOp_Message) -> bool: + """ + Processes VM delete-operation. + If operation is still running - do nothing + If operation failed - log error & remove op from watch list + If operation is done - remove op from watch list do nothing + + To avoid querying status for each op individually, use list of VM instances as + a source of data. Don't query op for instance X if instance X is not present + (presumably deleted). + NOTE: This optimization can lead to false-positives - + absence of error-logs in case op failed, but VM got deleted by other means. + + Returns True if message should be marked as processed (ack). + """ + + inst = lkp.instance(m.node) + + if not inst: + log.debug(f"Stop watching op {m.op_name}, VM {m.node} appears to be deleted") + return True # ack, potentially false-positive + + if inst.status == "TERMINATED": + log.debug(f"Stop watching op {m.op_name}, VM {m.node} is TERMINATED") + return True # ack, potentially false-positive + + if inst.status == "STOPPING": + log.debug(f"Skipping op {m.op_name}, VM {m.node} is STOPPING") + return False # try later + + try: + op = util.get_operation_req(lkp, m.op_name, zone=m.zone).execute() + except: + # TODO: consider less conservative handling, but be careful not to cause deadlettering. + log.exception(f"Failed to get operation {m.op_name}, will not retry") + return True # ack (remove) + + if op["status"] != "DONE": + log.debug(f"Watching op {m.op_name} is still not done ({op['status']})") + return False # try later + + if "error" in op: + log.error(f"Operation {m.op_name} to delete {m.node} finished with error: {op['error']}") + else: + log.debug(f"Operation {m.op_name} to delete {m.node} successfully finished") + return True # ack + + +def watch_vm_delete_ops(lkp: util.Lookup) -> None: + sub = local_pubsub.subscription(TOPIC) + + # Pull once instead of "pulling until empty", motivation: + # Bulk of cases processed by `_watch_op` relies on freshness of `lkp.instances`, + # `lkp.instances` are fetched once during run of `slurmsync`. + # Therefore we shouldn't try to re-process messages that has been already NACKed in this run, + # since they will be handled with the same `lkp.instance` as a previous attempt. + msgs = sub.pull(max_messages=1000) # 1000 is arbitrary number to be adjusted if needed. + log.debug(f"Processing {len(msgs)} delete VM operations") + # TODO: handle messages in butches to improve latency + for m in msgs: + try: + dm = WatchDeleteVmOp_Message(**m.data) + ack = _watch_op(lkp, dm) + except Exception: + log.exception(f"Failed to process the message {m.id}, removing") + ack = True + if ack: + sub.ack([m.id]) + else: + sub.modify_ack_deadline([m.id], deadline=0) # NACK + + + + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf new file mode 100644 index 0000000000..71905a0342 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf @@ -0,0 +1,504 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "bucket_name" { + description = <<-EOD + Name of GCS bucket to use. + EOD + type = string +} + +variable "bucket_dir" { + description = "Bucket directory for cluster files to be put into." + type = string + default = null +} + +variable "enable_debug_logging" { + type = bool + description = "Enables debug logging mode. Not for production use." + default = false +} + +variable "extra_logging_flags" { + type = map(bool) + description = "The only available flag is `trace_api`" + default = {} +} + +variable "project_id" { + description = "The GCP project ID." + type = string +} + +variable "enable_slurm_auth" { + description = < x... } + nodeset_map = { for k, vs in local.nodeset_map_ell : k => vs[0] } + + nodeset_tpu_map_ell = { for x in var.nodeset_tpu : x.nodeset_name => x... } + nodeset_tpu_map = { for k, vs in local.nodeset_tpu_map_ell : k => vs[0] } + + nodeset_dyn_map_ell = { for x in var.nodeset_dyn : x.nodeset_name => x... } + nodeset_dyn_map = { for k, vs in local.nodeset_dyn_map_ell : k => vs[0] } + + + no_reservation_affinity = { type : "NO_RESERVATION" } +} + +# NODESET +module "slurm_nodeset_template" { + source = "../../internal/slurm-gcp/instance_template" + for_each = local.nodeset_map + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + slurm_instance_role = "compute" + slurm_bucket_path = module.slurm_files.slurm_bucket_path + + additional_disks = each.value.additional_disks + bandwidth_tier = each.value.bandwidth_tier + can_ip_forward = each.value.can_ip_forward + advanced_machine_features = each.value.advanced_machine_features + disk_auto_delete = each.value.disk_auto_delete + disk_labels = each.value.disk_labels + disk_resource_manager_tags = each.value.disk_resource_manager_tags + disk_size_gb = each.value.disk_size_gb + disk_type = each.value.disk_type + enable_confidential_vm = each.value.enable_confidential_vm + enable_oslogin = each.value.enable_oslogin + enable_shielded_vm = each.value.enable_shielded_vm + gpu = each.value.gpu + labels = merge(each.value.labels, { slurm_nodeset = each.value.nodeset_name }) + machine_type = each.value.machine_type + metadata = merge(each.value.metadata, local.universe_domain) + min_cpu_platform = each.value.min_cpu_platform + name_prefix = each.value.nodeset_name + on_host_maintenance = each.value.on_host_maintenance + preemptible = each.value.preemptible + region = each.value.region + resource_manager_tags = each.value.resource_manager_tags + spot = each.value.spot + termination_action = each.value.termination_action + service_account = each.value.service_account + shielded_instance_config = each.value.shielded_instance_config + source_image_family = each.value.source_image_family + source_image_project = each.value.source_image_project + source_image = each.value.source_image + subnetwork = each.value.subnetwork_self_link + additional_networks = each.value.additional_networks + access_config = each.value.access_config + tags = concat([local.slurm_cluster_name], each.value.tags) + + max_run_duration = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? each.value.dws_flex.max_run_duration : null + provisioning_model = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? "FLEX_START" : null + reservation_affinity = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? local.no_reservation_affinity : null +} + +module "nodeset_cleanup" { + source = "./modules/cleanup_compute" + for_each = local.nodeset_map + + nodeset = each.value + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + enable_cleanup_compute = var.enable_cleanup_compute + universe_domain = var.universe_domain + endpoint_versions = var.endpoint_versions + gcloud_path_override = var.gcloud_path_override + nodeset_template = module.slurm_nodeset_template[each.value.nodeset_name].self_link +} + +locals { + nodesets = [for name, ns in local.nodeset_map : { + nodeset_name = ns.nodeset_name + node_conf = ns.node_conf + dws_flex = ns.dws_flex + instance_template = module.slurm_nodeset_template[ns.nodeset_name].self_link + node_count_dynamic_max = ns.node_count_dynamic_max + node_count_static = ns.node_count_static + subnetwork = ns.subnetwork_self_link + reservation_name = ns.reservation_name + future_reservation = ns.future_reservation + maintenance_interval = ns.maintenance_interval + instance_properties_json = ns.instance_properties_json + enable_placement = ns.enable_placement + placement_max_distance = ns.placement_max_distance + network_storage = ns.network_storage + zone_target_shape = ns.zone_target_shape + zone_policy_allow = ns.zone_policy_allow + zone_policy_deny = ns.zone_policy_deny + enable_maintenance_reservation = ns.enable_maintenance_reservation + enable_opportunistic_maintenance = ns.enable_opportunistic_maintenance + accelerator_topology = ns.accelerator_topology + }] +} + +# NODESET TPU +module "slurm_nodeset_tpu" { + source = "../../internal/slurm-gcp/nodeset_tpu" + for_each = local.nodeset_tpu_map + + project_id = var.project_id + node_count_dynamic_max = each.value.node_count_dynamic_max + node_count_static = each.value.node_count_static + nodeset_name = each.value.nodeset_name + zone = each.value.zone + node_type = each.value.node_type + accelerator_config = each.value.accelerator_config + tf_version = each.value.tf_version + preemptible = each.value.preemptible + preserve_tpu = each.value.preserve_tpu + enable_public_ip = each.value.enable_public_ip + service_account = each.value.service_account + data_disks = each.value.data_disks + docker_image = each.value.docker_image + subnetwork = each.value.subnetwork +} + +module "nodeset_cleanup_tpu" { + source = "./modules/cleanup_tpu" + for_each = local.nodeset_tpu_map + + nodeset = { + nodeset_name = each.value.nodeset_name + zone = each.value.zone + } + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + enable_cleanup_compute = var.enable_cleanup_compute + universe_domain = var.universe_domain + endpoint_versions = var.endpoint_versions + gcloud_path_override = var.gcloud_path_override + + depends_on = [ + # Depend on controller network, as a best effort to avoid + # subnetwork resourceInUseByAnotherResource error + var.subnetwork_self_link + ] +} + +resource "google_storage_bucket_object" "parition_config" { + for_each = { for p in var.partitions : p.partition_name => p } + + bucket = module.slurm_files.bucket_name + name = "${module.slurm_files.bucket_dir}/partition_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +moved { + from = module.slurm_files.google_storage_bucket_object.parition_config + to = google_storage_bucket_object.parition_config +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf new file mode 100644 index 0000000000..218c36e392 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf @@ -0,0 +1,191 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# BUCKET + +locals { + synt_suffix = substr(md5("${local.controller_project_id}${var.deployment_name}"), 0, 5) + synth_bucket_name = "${local.slurm_cluster_name}${local.synt_suffix}" + + bucket_name = var.create_bucket ? module.bucket[0].name : var.bucket_name +} + +module "bucket" { + source = "terraform-google-modules/cloud-storage/google" + version = ">= 6.1" + + count = var.create_bucket ? 1 : 0 + + location = var.region + names = [local.synth_bucket_name] + prefix = "slurm" + project_id = local.controller_project_id + + force_destroy = { + (local.synth_bucket_name) = true + } + + labels = merge(local.labels, { + slurm_cluster_name = local.slurm_cluster_name + }) +} + +# BUCKET IAMs +locals { + compute_sa = toset(flatten([for x in module.slurm_nodeset_template : x.service_account])) + compute_tpu_sa = toset(flatten([for x in module.slurm_nodeset_tpu : x.service_account])) + login_sa = toset(flatten([for x in module.login : x.service_account])) + + viewers = toset(flatten([ + "serviceAccount:${module.slurm_controller_template.service_account.email}", + formatlist("serviceAccount:%s", [for x in local.compute_sa : x.email]), + formatlist("serviceAccount:%s", [for x in local.compute_tpu_sa : x.email if x.email != null]), + formatlist("serviceAccount:%s", [for x in local.login_sa : x.email]), + ])) +} + + +resource "google_storage_bucket_iam_member" "viewers" { + for_each = local.viewers + bucket = local.bucket_name + role = "roles/storage.objectViewer" + member = each.value +} + +resource "google_storage_bucket_iam_member" "legacy_readers" { + for_each = local.viewers + bucket = local.bucket_name + role = "roles/storage.legacyBucketReader" + member = each.value +} + +locals { + daos_ns = [ + for ns in var.network_storage : + ns if ns.fs_type == "daos" + ] + + daos_client_install_runners = [ + for ns in local.daos_ns : + ns.client_install_runner if ns.client_install_runner != null + ] + + daos_mount_runners = [ + for ns in local.daos_ns : + ns.mount_runner if ns.mount_runner != null + ] + + daos_network_storage_runners = concat( + local.daos_client_install_runners, + local.daos_mount_runners, + ) + + daos_install_mount_script = { + filename = "ghpc_daos_mount.sh" + content = length(local.daos_ns) > 0 ? module.daos_network_storage_scripts[0].startup_script : "" + } + + common_scripts = length(local.daos_ns) > 0 ? [local.daos_install_mount_script] : [] +} + +# SLURM FILES +locals { + ghpc_startup_script_controller = concat( + local.common_scripts, + [{ + filename = "ghpc_startup.sh" + content = var.controller_startup_script + }]) + + controller_state_disk = { + device_name : try(google_compute_disk.controller_disk[0].name, null) + } + + + nodeset_startup_scripts = { for k, v in local.nodeset_map : k => concat(local.common_scripts, v.startup_script) } +} + +module "daos_network_storage_scripts" { + count = length(local.daos_ns) > 0 ? 1 : 0 + + source = "../../../../modules/scripts/startup-script" + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.daos_network_storage_runners +} + +module "slurm_files" { + source = "./modules/slurm_files" + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + bucket_dir = var.bucket_dir + bucket_name = local.bucket_name + controller_network_attachment = var.controller_network_attachment + + slurmdbd_conf_tpl = var.slurmdbd_conf_tpl + slurm_conf_tpl = var.slurm_conf_tpl + slurm_conf_template = var.slurm_conf_template + cgroup_conf_tpl = var.cgroup_conf_tpl + cloud_parameters = var.cloud_parameters + cloudsql_secret = try( + one(google_secret_manager_secret_version.cloudsql_version[*].id), + null) + + controller_startup_scripts = local.ghpc_startup_script_controller + controller_startup_scripts_timeout = var.controller_startup_scripts_timeout + nodeset_startup_scripts = local.nodeset_startup_scripts + compute_startup_scripts_timeout = var.compute_startup_scripts_timeout + controller_state_disk = local.controller_state_disk + + enable_debug_logging = var.enable_debug_logging + extra_logging_flags = var.extra_logging_flags + + enable_slurm_auth = var.enable_slurm_auth + + enable_bigquery_load = var.enable_bigquery_load + enable_external_prolog_epilog = var.enable_external_prolog_epilog + enable_chs_gpu_health_check_prolog = var.enable_chs_gpu_health_check_prolog + enable_chs_gpu_health_check_epilog = var.enable_chs_gpu_health_check_epilog + epilog_scripts = var.epilog_scripts + prolog_scripts = var.prolog_scripts + task_epilog_scripts = var.task_epilog_scripts + task_prolog_scripts = var.task_prolog_scripts + + disable_default_mounts = !var.enable_default_mounts + network_storage = [ + for storage in var.network_storage : { + server_ip = storage.server_ip, + remote_mount = storage.remote_mount, + local_mount = storage.local_mount, + fs_type = storage.fs_type, + mount_options = storage.mount_options + } + if storage.fs_type != "daos" + ] + + nodeset = local.nodesets + nodeset_dyn = values(local.nodeset_dyn_map) + # Use legacy format for now + nodeset_tpu = values(module.slurm_nodeset_tpu)[*] + + + depends_on = [module.bucket] + + # Providers + endpoint_versions = var.endpoint_versions +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf new file mode 100644 index 0000000000..85ad10fa21 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf @@ -0,0 +1,814 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########### +# GENERAL # +########### + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "deployment_name" { + description = "Name of the deployment." + type = string +} + +variable "slurm_cluster_name" { + type = string + description = <<-EOD + Cluster name, used for resource naming and slurm accounting. + If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). + EOD + default = null + + validation { + condition = var.slurm_cluster_name == null || can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) + error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." + } +} + +variable "region" { + type = string + description = "The default region to place resources in." +} + +variable "zone" { + type = string + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disable\_login\_public\_ips](#input\_disable\_login\_public\_ips) | DEPRECATED: Use `enable_login_public_ips` instead. | `bool` | `null` | no | +| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_login\_public\_ips](#input\_enable\_login\_public\_ips) | If set to true. The login node will have a random public IP assigned to it. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for login nodes. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Unique name prefix for login nodes. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all login groups. | `string` | n/a | yes | +| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the login instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the login instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [login\_nodes](#output\_login\_nodes) | Slurm login instance definition. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf new file mode 100644 index 0000000000..6ebe5902dc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf @@ -0,0 +1,115 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-login", ghpc_role = "scheduler" }) +} + +module "instance_validation" { + source = "../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata + ) + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.disk_resource_manager_tags + } + ] + + public_access_config = [{ nat_ip = null, network_tier = null }] + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + # lower, replace `_` with `-`, and remove any non-alphanumeric characters + group_name = replace( + replace( + lower(var.name_prefix), + "_", "-"), + "/[^-a-z0-9]/", "") + + + login_node = { + group_name = local.group_name + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(var.disk_labels, local.labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + additional_networks = var.additional_networks + + can_ip_forward = var.can_ip_forward + advanced_machine_features = var.advanced_machine_features + + enable_confidential_vm = var.enable_confidential_vm + access_config = var.enable_login_public_ips ? local.public_access_config : [] + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + gpu = one(local.guest_accelerator) + labels = local.labels + machine_type = var.machine_type + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + num_instances = var.num_instances + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + region = var.region + resource_manager_tags = var.resource_manager_tags + zone = var.zone + + service_account = local.service_account + + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + static_ips = var.static_ips + bandwidth_tier = var.bandwidth_tier + + subnetwork = var.subnetwork_self_link + tags = var.tags + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml new file mode 100644 index 0000000000..47f003258e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name_prefix + has_to_be_used: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf new file mode 100644 index 0000000000..e700542794 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "login_nodes" { + description = "Slurm login instance definition." + value = [local.login_node] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf new file mode 100644 index 0000000000..7c1a2e06b5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf @@ -0,0 +1,419 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +variable "project_id" { # tflint-ignore: terraform_unused_declarations + type = string + description = "Project ID to create resources in." +} + +variable "region" { + type = string + description = "Region where the instances should be created." + default = null +} + +variable "zone" { + type = string + description = <<-EOD + Zone where the instances should be created. If not specified, instances will be + spread across available zones in the region. + EOD + default = null +} + +variable "name_prefix" { + type = string + description = <<-EOD + Unique name prefix for login nodes. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all login groups. + EOD +} + +variable "num_instances" { + type = number + description = "Number of instances to create. This value is ignored if static_ips is provided." + default = 1 +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} +} + +variable "disk_type" { + type = string + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + default = "pd-ssd" +} + +variable "disk_size_gb" { + type = number + description = "Boot disk size in GB." + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + type = list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string)) + auto_delete = optional(bool) + boot = optional(bool) + disk_resource_manager_tags = optional(map(string)) + })) + description = "List of maps of disks." + default = [] +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })) + nullable = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "disable_smt" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + type = bool + default = null + validation { + condition = var.disable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "static_ips" { + type = list(string) + description = "List of static IPs for VM instances." + default = [] +} + +variable "bandwidth_tier" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 6.16 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.16 | +| [helm](#provider\_helm) | ~> 2.17 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.cert_manager](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.prometheus](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.slurm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.slurm_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cert\_manager\_chart\_version](#input\_cert\_manager\_chart\_version) | Version of the Cert Manager chart to install. | `string` | `"v1.18.2"` | no | +| [cert\_manager\_values](#input\_cert\_manager\_values) | Value overrides for the Cert Manager release | `any` | 
{
  "crds": {
    "enabled": true
  }
}
| no | +| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | +| [install\_kube\_prometheus\_stack](#input\_install\_kube\_prometheus\_stack) | Install the Kube Prometheus Stack. | `bool` | `false` | no | +| [install\_slurm\_chart](#input\_install\_slurm\_chart) | Install slurm-operator chart. | `bool` | `true` | no | +| [install\_slurm\_operator\_chart](#input\_install\_slurm\_operator\_chart) | Install slurm-operator chart. | `bool` | `true` | no | +| [node\_pool\_names](#input\_node\_pool\_names) | Names of node pools, for use in node affinities (Slinky system components). | `list(string)` | `null` | no | +| [project\_id](#input\_project\_id) | The project ID that hosts the GKE cluster. | `string` | n/a | yes | +| [prometheus\_chart\_version](#input\_prometheus\_chart\_version) | Version of the Kube Prometheus Stack chart to install. | `string` | `"77.0.1"` | no | +| [prometheus\_values](#input\_prometheus\_values) | Value overrides for the Prometheus release | `any` | 
{
  "installCRDs": true
}
| no | +| [slurm\_chart\_version](#input\_slurm\_chart\_version) | Version of the Slurm chart to install. | `string` | `"0.3.1"` | no | +| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | +| [slurm\_operator\_chart\_version](#input\_slurm\_operator\_chart\_version) | Version of the Slurm Operator chart to install. | `string` | `"0.3.1"` | no | +| [slurm\_operator\_namespace](#input\_slurm\_operator\_namespace) | slurm namespace for charts | `string` | `"slinky"` | no | +| [slurm\_operator\_repository](#input\_slurm\_operator\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | +| [slurm\_operator\_values](#input\_slurm\_operator\_values) | Value overrides for the Slinky release | `any` | `{}` | no | +| [slurm\_repository](#input\_slurm\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | +| [slurm\_values](#input\_slurm\_values) | Value overrides for the Slurm release | `any` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [slurm\_namespace](#output\_slurm\_namespace) | namespace for the slurm chart | +| [slurm\_operator\_namespace](#output\_slurm\_operator\_namespace) | namespace for the slinky operator chart | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/main.tf new file mode 100644 index 0000000000..aff33b73a0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/main.tf @@ -0,0 +1,197 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] + project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] + + # Define affinity settings when node pools are specified + node_pool_affinity = var.node_pool_names != null ? { + nodeAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [{ + matchExpressions = [{ + key = "cloud.google.com/gke-nodepool" + operator = "In" + values = var.node_pool_names + }] + }] + } + } + } : {} +} + +data "google_client_config" "default" {} + +data "google_container_cluster" "gke_cluster" { + project = local.project_id + name = local.cluster_name + location = local.cluster_location +} + +resource "helm_release" "cert_manager" { + name = "cert-manager" + chart = "cert-manager" + repository = "https://charts.jetstack.io" + version = var.cert_manager_chart_version + namespace = "cert-manager" + create_namespace = true + + values = concat( + [yamlencode({ + affinity = local.node_pool_affinity + webhook = { + affinity = local.node_pool_affinity + } + cainjector = { + affinity = local.node_pool_affinity + } + startupapicheck = { + affinity = local.node_pool_affinity + } + })], + [yamlencode(var.cert_manager_values)] + ) +} + +resource "helm_release" "slurm_operator" { + count = var.install_slurm_operator_chart ? 1 : 0 + name = "slurm-operator" + chart = "slurm-operator" + repository = var.slurm_operator_repository + version = var.slurm_operator_chart_version + namespace = var.slurm_operator_namespace + create_namespace = true + + # The Cert Manager webhook deployment must be running to provision the Operator + depends_on = [ + helm_release.cert_manager + ] + + values = concat( + [yamlencode({ + operator = { + affinity = local.node_pool_affinity + } + webhook = { + affinity = local.node_pool_affinity + } + })], + [yamlencode(var.slurm_operator_values)] + ) +} + +resource "helm_release" "slurm" { + count = var.install_slurm_chart ? 1 : 0 + name = "slurm" + chart = "slurm" + repository = var.slurm_repository + version = var.slurm_chart_version + namespace = var.slurm_namespace + create_namespace = true + + # The Slurm Operator must be running to provision Slurm clusters/nodesets + depends_on = [ + helm_release.slurm_operator + ] + + values = concat( + [yamlencode({ + controller = { + affinity = local.node_pool_affinity + } + accounting = { + affinity = local.node_pool_affinity + } + mariadb = { + primary = { + affinity = local.node_pool_affinity + } + secondary = { + affinity = local.node_pool_affinity + } + } + restapi = { + affinity = local.node_pool_affinity + } + slurm-exporter = { + exporter = { + affinity = local.node_pool_affinity + } + } + })], + [yamlencode(var.slurm_values)] + ) +} + +resource "helm_release" "prometheus" { + count = var.install_kube_prometheus_stack ? 1 : 0 + name = "prometheus" + chart = "kube-prometheus-stack" + repository = "https://prometheus-community.github.io/helm-charts" + version = var.prometheus_chart_version + namespace = "prometheus" + create_namespace = true + + values = concat( + [yamlencode({ + crds = { + upgradeJob = { + affinity = local.node_pool_affinity + } + } + alertmanager = { + alertmanagerSpec = { + affinity = local.node_pool_affinity + } + } + prometheusOperator = { + admissionWebhooks = { + deployment = { + affinity = local.node_pool_affinity + } + patch = { + affinity = local.node_pool_affinity + } + } + affinity = local.node_pool_affinity + } + prometheus = { + prometheusSpec = { + affinity = local.node_pool_affinity + } + } + thanosRuler = { + thanosRulerSpec = { + affinity = local.node_pool_affinity + } + } + kube-state-metrics = { + affinity = local.node_pool_affinity + } + grafana = { + affinity = local.node_pool_affinity + imageRenderer = { + affinity = local.node_pool_affinity + } + } + prometheus-windows-exporter = { + affinity = local.node_pool_affinity + } + })], + [yamlencode(var.prometheus_values)] + ) +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/metadata.yaml new file mode 100644 index 0000000000..e18197e2b7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/outputs.tf new file mode 100644 index 0000000000..8ea6385905 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/outputs.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "slurm_namespace" { + description = "namespace for the slurm chart" + value = var.slurm_namespace +} + +output "slurm_operator_namespace" { + description = "namespace for the slinky operator chart" + value = var.slurm_operator_namespace +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/providers.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/providers.tf new file mode 100644 index 0000000000..313d6dc58e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/providers.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +provider "helm" { + kubernetes { + host = "https://${data.google_container_cluster.gke_cluster.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode( + data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate, + ) + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/variables.tf new file mode 100644 index 0000000000..8acaf78562 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/variables.tf @@ -0,0 +1,127 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The project ID that hosts the GKE cluster." + type = string +} + +variable "cluster_id" { + description = "An identifier for the GKE cluster resource with format projects//locations//clusters/." + type = string + nullable = false +} + +variable "node_pool_names" { + description = "Names of node pools, for use in node affinities (Slinky system components)." + type = list(string) + default = null +} + +variable "cert_manager_chart_version" { + description = "Version of the Cert Manager chart to install." + type = string + default = "v1.18.2" +} + +variable "cert_manager_values" { + description = "Value overrides for the Cert Manager release" + type = any + default = { + crds = { + enabled = true + } + } +} + +variable "slurm_operator_chart_version" { + description = "Version of the Slurm Operator chart to install." + type = string + default = "0.3.1" +} + +variable "slurm_operator_values" { + description = "Value overrides for the Slinky release" + type = any + default = {} +} + +variable "slurm_chart_version" { + description = "Version of the Slurm chart to install." + type = string + default = "0.3.1" +} + +variable "slurm_values" { + description = "Value overrides for the Slurm release" + type = any + default = {} +} + +variable "install_kube_prometheus_stack" { + # Components detailed at https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack + description = "Install the Kube Prometheus Stack." + type = bool + default = false +} + +variable "prometheus_chart_version" { + description = "Version of the Kube Prometheus Stack chart to install." + type = string + default = "77.0.1" +} + +variable "prometheus_values" { + description = "Value overrides for the Prometheus release" + type = any + default = { + installCRDs = true + } +} + +variable "slurm_namespace" { + description = "slurm namespace for charts" + type = string + default = "slurm" +} + +variable "slurm_operator_namespace" { + description = "slurm namespace for charts" + type = string + default = "slinky" +} + +variable "install_slurm_chart" { + description = "Install slurm-operator chart." + type = bool + default = true +} + +variable "install_slurm_operator_chart" { + description = "Install slurm-operator chart." + type = bool + default = true +} + +variable "slurm_repository" { + description = "Value overrides for the Slinky release" + type = string + default = "oci://ghcr.io/slinkyproject/charts" +} + +variable "slurm_operator_repository" { + description = "Value overrides for the Slinky release" + type = string + default = "oci://ghcr.io/slinkyproject/charts" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/versions.tf new file mode 100644 index 0000000000..ae4327aeef --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/versions.tf @@ -0,0 +1,28 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + google = { + source = "hashicorp/google" + version = ">= 6.16" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/README.md new file mode 100644 index 0000000000..71a862fd6c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/README.md @@ -0,0 +1,149 @@ +## Description + +This module creates a Toolkit runner that will install HTCondor on RedHat 7 or +8 and its derivative operating systems. These include the CentOS 7 and Rocky +Linux 8 releases of the [HPC VM Image][hpcvmimage]. It may also function on +RedHat 9 and derivatives, however it is not yet supported. Please report any +[issues] on these 3 distributions or open a [discussion] to request support on +Debian or Ubuntu distributions. + +[issues]: https://github.com/GoogleCloudPlatform/hpc-toolkit/issues +[discussion]: https://github.com/GoogleCloudPlatform/hpc-toolkit/discussions + +It also exports a list of Google Cloud APIs which must be enabled prior to +provisioning an HTCondor Pool. + +It is expected to be used with the [htcondor-setup] and +[htcondor-execute-point] modules. + +[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[htcondor-setup]: ../../scheduler/htcondor-setup/README.md +[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md + +### Example + +The following code snippet uses this module to create startup scripts that +install the HTCondor software into a custom VM image. + +```yaml +deployment_groups: +- group: primary + modules: + - id: network1 + source: modules/network/vpc + outputs: + - network_name + + - id: htcondor_install + source: community/modules/scripts/htcondor-install + + - id: htcondor_install_script + source: modules/scripts/startup-script + use: + - htcondor_install + +- group: packer + modules: + - id: custom-image + source: modules/packer/custom-image + kind: packer + use: + - network1 + - htcondor_install_script + settings: + disk_size: 50 + source_image_family: hpc-rocky-linux-8 + image_family: "htcondor-10x" +``` + +A full example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +## Important note + +All POSIX users and HTCondor jobs can act as the service account attached to +VMs within the pool. This enables the use of IAM restrictions via service +accounts but also allows users to access services to which system daemons need +access (e.g. to create Cloud Logging entries). If this is undesirable, one can +restrict access to the instance metadata server to the `root` and `condor` +users. This will allow system services to use the service account, but not +other POSIX users or HTCondor jobs. The firewall example below is appropriate +for CentOS 7. + +```shell +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 1 \ + -m owner --uid-owner root -p tcp -d metadata.google.internal --dport 80 -j ACCEPT +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 2 \ + -m owner --uid-owner condor -p tcp -d metadata.google.internal --dport 80 -j ACCEPT +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 3 \ + -p tcp -d metadata.google.internal --dport 80 -j DROP +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 4 \ + -p tcp -d metadata.google.internal --dport 8080 -j DROP +firewall-cmd --permanent --zone=public --add-port=9618/tcp +firewall-cmd --reload +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [condor\_version](#input\_condor\_version) | Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: "23.0.0","23.*")) | `string` | `"23.*"` | no | +| [enable\_docker](#input\_enable\_docker) | Install and enable docker daemon alongside HTCondor | `bool` | `true` | no | +| [http\_proxy](#input\_http\_proxy) | Set system default web (http and https) proxy for Windows HTCondor installation | `string` | `""` | no | +| [python\_windows\_installer\_url](#input\_python\_windows\_installer\_url) | URL of Python installer for Windows | `string` | `"https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcp\_service\_list](#output\_gcp\_service\_list) | Google Cloud APIs required by HTCondor | +| [runners](#output\_runners) | Runner to install HTCondor using startup-scripts | +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | Windows PowerShell script to install HTCondor | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py new file mode 100644 index 0000000000..77bafa0310 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py @@ -0,0 +1,417 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script for resizing managed instance group (MIG) cluster size based +# on the number of jobs in the Condor Queue. + +from absl import app +from absl import flags +from collections import OrderedDict +from datetime import datetime +from pprint import pprint +from googleapiclient import discovery +from oauth2client.client import GoogleCredentials + +import argparse +import os +import math +import time +import htcondor +import classad + +parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument("--p", required=True, help="Project id", type=str) +parser.add_argument( + "--z", + required=True, + help="Name of GCP zone where the managed instance group is located", + type=str, +) +parser.add_argument( + "--r", + required=True, + help="Name of GCP region where the managed instance group is located", + type=str, +) +parser.add_argument( + "--mz", + required=False, + help="Enabled multizone (regional) managed instance group", + action="store_true", +) +parser.add_argument( + "--g", required=True, help="Name of the managed instance group", type=str +) +parser.add_argument( + "--i", + default=0, + help="Minimum number of idle compute instances", + type=int +) +parser.add_argument( + "--c", required=True, help="Maximum number of compute instances", type=int +) +parser.add_argument( + "--v", + default=0, + help="Increase output verbosity. 1-show basic debug info. 2-show detail debug info", + type=int, + choices=[0, 1, 2], +) +parser.add_argument( + "--d", + default=0, + help="Dry Run, default=0, if 1, then no scaling actions", + type=int, + choices=[0, 1], +) + +args = parser.parse_args() + +class AutoScaler: + def __init__(self, multizone=False): + + self.multizone = multizone + # Obtain credentials + self.credentials = GoogleCredentials.get_application_default() + self.service = discovery.build("compute", "v1", credentials=self.credentials) + + if self.multizone: + self.instanceGroupManagers = self.service.regionInstanceGroupManagers() + else: + self.instanceGroupManagers = self.service.instanceGroupManagers() + + # Remove specified instances from MIG and decrease MIG size + def deleteFromMig(self, node_self_links): + requestDelInstance = self.instanceGroupManagers.deleteInstances( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + body={ "instances": node_self_links }, + ) + + # execute if not a dry-run + if not self.dryrun: + response = requestDelInstance.execute() + if self.debug > 0: + pprint(response) + return response + return "Dry Run" + + def getInstanceTemplateInfo(self): + requestTemplateName = self.instanceGroupManagers.get( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + fields="instanceTemplate", + ) + responseTemplateName = requestTemplateName.execute() + template_name = "" + + if self.debug > 1: + print("Request for the template name") + pprint(responseTemplateName) + + if len(responseTemplateName) > 0: + template_url = responseTemplateName.get("instanceTemplate") + template_url_partitioned = template_url.split("/") + template_name = template_url_partitioned[len(template_url_partitioned) - 1] + + requestInstanceTemplate = self.service.instanceTemplates().get( + project=self.project, instanceTemplate=template_name, fields="properties" + ) + responseInstanceTemplateInfo = requestInstanceTemplate.execute() + + if self.debug > 1: + print("Template information") + pprint(responseInstanceTemplateInfo["properties"]) + + machine_type = responseInstanceTemplateInfo["properties"]["machineType"] + is_spot = responseInstanceTemplateInfo["properties"]["scheduling"][ + "preemptible" + ] + if self.debug > 0: + print("Machine Type: " + machine_type) + print("Is spot: " + str(is_spot)) + request = self.service.machineTypes().get( + project=self.project, zone=self.zone, machineType=machine_type + ) + response = request.execute() + guest_cpus = response["guestCpus"] + if self.debug > 1: + print("Machine information") + pprint(responseInstanceTemplateInfo["properties"]) + if self.debug > 0: + print("Guest CPUs: " + str(guest_cpus)) + + instanceTemplateInfo = { + "machine_type": machine_type, + "is_spot": is_spot, + "guest_cpus": guest_cpus, + } + return instanceTemplateInfo + + def scale(self): + # diagnosis + if self.debug > 1: + print("Launching autoscaler.py with the following arguments:") + print("project_id: " + self.project) + print("zone: " + self.zone) + print("region: " + self.region) + print(f"multizone: {self.multizone}") + print("group_manager: " + self.instance_group_manager) + print("computeinstancelimit: " + str(self.compute_instance_limit)) + print("debuglevel: " + str(self.debug)) + + if self.multizone: + self.zoneargs = {"region": self.region} + else: + self.zoneargs = {"zone": self.zone} + + # Each HTCondor scheduler (SchedD), maintains a list of jobs under its + # stewardship. A full list of Job ClassAd attributes can be found at + # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html + schedd = htcondor.Schedd() + # encourage the job queue to start a new negotiation cycle; there are + # internal unconfigurable rate limits so not guaranteed; this is not + # strictly required for success, but may reduce latency of autoscaling + schedd.reschedule() + REQUEST_CPUS_ATTRIBUTE = "RequestCpus" + REQUEST_GPUS_ATTRIBUTE = "RequestGpus" + REQUEST_MEMORY_ATTRIBUTE = "RequestMemory" + job_attributes = [ + REQUEST_CPUS_ATTRIBUTE, + REQUEST_GPUS_ATTRIBUTE, + REQUEST_MEMORY_ATTRIBUTE, + ] + + instanceTemplateInfo = self.getInstanceTemplateInfo() + self.is_spot = instanceTemplateInfo["is_spot"] + self.cores_per_node = instanceTemplateInfo["guest_cpus"] + print(f"MIG is configured for Spot pricing: {self.is_spot}") + print("Number of CPU per compute node: " + str(self.cores_per_node)) + + # this query will constrain the search for jobs to those that either + # require spot VMs or do not require Spot VMs based on whether the + # VM instance template is configured for Spot pricing + spot_query = classad.ExprTree(f"RequireId == \"{self.instance_group_manager}\"") + + # For purpose of scaling a Managed Instance Group, count only jobs that + # are idle and likely participated in a negotiation cycle (there does + # not appear to be a single classad attribute for this). + # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html#JobStatus + LAST_CYCLE_ATTRIBUTE = "LastNegotiationCycleTime0" + coll = htcondor.Collector() + negotiator_ad = coll.query(htcondor.AdTypes.Negotiator, projection=[LAST_CYCLE_ATTRIBUTE]) + if len(negotiator_ad) != 1: + print(f"There should be exactly 1 negotiator in the pool. There is {len(negotiator_ad)}") + exit() + last_negotiation_cycle_time = negotiator_ad[0].get(LAST_CYCLE_ATTRIBUTE) + if not last_negotiation_cycle_time: + print(f"The negotiator has not yet started a match cycle. Exiting auto-scaling.") + exit() + + print(f"Last negotiation cycle occurred at: {datetime.fromtimestamp(last_negotiation_cycle_time)}") + idle_job_query = classad.ExprTree(f"JobStatus == 1 && QDate < {last_negotiation_cycle_time}") + idle_job_ads = schedd.query(constraint=idle_job_query.and_(spot_query), + projection=job_attributes) + + total_idle_request_cpus = sum(j[REQUEST_CPUS_ATTRIBUTE] for j in idle_job_ads) + print(f"Total CPUs requested by idle jobs: {total_idle_request_cpus}") + + if self.debug > 1: + print("Information about the compute instance template") + pprint(instanceTemplateInfo) + + # Calculate the minimum number of instances that, for fully packed + # execute points, could satisfy current job queue + min_hosts_for_idle_jobs = math.ceil(total_idle_request_cpus / self.cores_per_node) + if self.debug > 0: + print(f"Minimum hosts needed: {total_idle_request_cpus} / {self.cores_per_node} = {min_hosts_for_idle_jobs}") + + # Get current number of instances in the MIG + requestGroupInfo = self.instanceGroupManagers.get( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + ) + responseGroupInfo = requestGroupInfo.execute() + current_target = responseGroupInfo["targetSize"] + print(f"Current MIG target size: {current_target}") + + # Find instances that are being modified by the MIG (currentAction is + # any value other than "NONE"). A common reason an instance is modified + # is it because it has failed a health check. + reqModifyingInstances = self.instanceGroupManagers.listManagedInstances( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + filter="currentAction != \"NONE\"", + orderBy="creationTimestamp desc" + ) + respModifyingInstances = reqModifyingInstances.execute() + + # Find VMs that are idle (no dynamic slots created from partitionable + # slots) in the MIG handled by this autoscaler + filter_idle_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots==0") + filter_claimed_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots>0") + filter_mig = classad.ExprTree(f"regexp(\".*/{self.instance_group_manager}$\", CloudCreatedBy)") + # A full list of Machine (StartD) ClassAd attributes can be found at + # https://htcondor.readthedocs.io/en/latest/classad-attributes/machine-classad-attributes.html + idle_node_ads = coll.query(htcondor.AdTypes.Startd, + constraint=filter_idle_vms.and_(filter_mig), + projection=["Machine", "CloudZone"]) + + NODENAME_ATTRIBUTE = "Machine" + claimed_node_ads = coll.query(htcondor.AdTypes.Startd, + constraint=filter_claimed_vms.and_(filter_mig), + projection=[NODENAME_ATTRIBUTE]) + claimed_nodes = [ ad[NODENAME_ATTRIBUTE].split(".")[0] for ad in claimed_node_ads] + + # treat OrderedDict as a set by ignoring key values; this set will + # contain VMs we would consider deleting, in inverse order of + # their readiness to join pool (creating, unhealthy, healthy+idle) + idle_nodes = OrderedDict() + try: + modifyingInstances = respModifyingInstances["managedInstances"] + except KeyError: + modifyingInstances = [] + + print(f"There are {len(modifyingInstances)} VMs being modified by the managed instance group") + + # there is potential for nodes in MIG health check "VERIFYING" state + # to have already joined the pool and be running jobs + for instance in modifyingInstances: + self_link = instance["instance"] + node_name = self_link.rsplit("/", 1)[-1] + if node_name not in claimed_nodes: + idle_nodes[self_link] = "modifying" + + for ad in idle_node_ads: + node = ad["Machine"].split(".")[0] + zone = ad["CloudZone"] + self_link = "https://www.googleapis.com/compute/v1/projects/" + \ + self.project + "/zones/" + zone + "/instances/" + node + # there is potential for nodes in MIG health check "VERIFYING" state + # to have already joined the pool and be idle; delete them last + if self_link in idle_nodes: + idle_nodes.move_to_end(self_link) + idle_nodes[self_link] = "idle" + n_idle = len(idle_nodes) + + print(f"There are {n_idle} VMs being modified or idle in the pool") + if self.debug > 1: + print("Listing idle nodes:") + pprint(idle_nodes) + + # always keep size tending toward the minimum idle VMs requested + new_target = current_target + self.compute_instance_min_idle - n_idle + min_hosts_for_idle_jobs + if new_target > self.compute_instance_limit: + self.size = self.compute_instance_limit + print(f"MIG target size will be limited by {self.compute_instance_limit}") + else: + self.size = new_target + + print(f"New MIG target size: {self.size}") + + if self.debug > 1: + print("MIG Information:") + print(responseGroupInfo) + + if self.size == current_target: + if current_target == 0: + print("Queue is empty") + print("Running correct number of VMs to handle queue") + exit() + + if self.size < current_target: + print("Scaling down. Looking for nodes that can be shut down") + + if self.debug > 1: + print("Compute node busy status:") + for node in idle_nodes: + print(node) + + # Shut down idle nodes up to our calculated limit + nodes_to_delete = list(idle_nodes.keys())[0:current_target-self.size] + for node in nodes_to_delete: + print(f"Attempting to delete: {node.rsplit('/',1)[-1]}") + respDel = self.deleteFromMig(nodes_to_delete) + + if self.debug > 1: + print("Scaling down complete") + + if self.size > current_target: + print( + "Scaling up. Need to increase number of instances to " + str(self.size) + ) + # Request to resize + request = self.instanceGroupManagers.resize( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + size=self.size, + ) + response = request.execute() + if self.debug > 1: + print("Requesting to increase MIG size") + pprint(response) + print("Scaling up complete") + + +def main(): + + scaler = AutoScaler(args.mz) + + # Project ID + scaler.project = args.p # Ex:'slurm-var-demo' + + # Name of the zone where the managed instance group is located + scaler.zone = args.z # Ex: 'us-central1-f' + + # Name of the region where the managed instance group is located + scaler.region = args.r # Ex: 'us-central1' + + # The name of the managed instance group. + scaler.instance_group_manager = args.g # Ex: 'condor-compute-igm' + + # Default number of cores per instance, will be replaced with actual value + scaler.cores_per_node = 4 + + # Default number of running instances that the managed instance group should maintain at any given time. This number will go up and down based on the load (number of jobs in the queue) + scaler.size = 0 + + scaler.compute_instance_min_idle = args.i + + # Dry run: : 0, run scaling; 1, only provide info. + scaler.dryrun = args.d > 0 + + # Debug level: 1-print debug information, 2 - print detail debug information + scaler.debug = 0 + if args.v: + scaler.debug = args.v + + # Limit for the maximum number of compute instance. If zero (default setting), no limit will be enforced by the script + scaler.compute_instance_limit = 0 + if args.c: + scaler.compute_instance_limit = abs(args.c) + + scaler.scale() + + +if __name__ == "__main__": + main() diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml new file mode 100644 index 0000000000..db989f9d40 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml @@ -0,0 +1,46 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install but do not activate HTCondor autoscaler + become: true + hosts: localhost + tasks: + - name: Install Python 3 pip + ansible.builtin.package: + name: python3-pip + state: present + - name: Create virtual environment for HTCondor autoscaler + ansible.builtin.pip: + name: pip + version: 21.3.1 # last Python 3.6-compatible release + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv + - name: Install latest setuptools + ansible.builtin.pip: + name: setuptools + version: 59.6.0 # last Python 3.6-compatible release + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv + - name: Install HTCondor autoscaler dependencies + with_items: + - oauth2client + - google-api-python-client + - absl-py + - htcondor + ansible.builtin.pip: + name: "{{ item }}" + state: present # rely on pip resolver to pick latest compatible releases + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml new file mode 100644 index 0000000000..4d3abbbfd6 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml @@ -0,0 +1,94 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The instructions for installing HTCondor may change with time, although we +# anticipate that they will stay fixed for the 23.0 releases. Find up-to-date +# recommendations at: +## https://htcondor.readthedocs.io/en/latest/getting-htcondor/from-our-repositories.html + +--- +- name: Ensure HTCondor is installed + hosts: all + vars: + enable_docker: true + htcondor_key: https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-23.0-Key + docker_key: https://download.docker.com/linux/centos/gpg + become: true + module_defaults: + ansible.builtin.yum: + lock_timeout: 300 + tasks: + - name: Enable EPEL repository + ansible.builtin.yum: + name: + - epel-release + - name: Directly install RPM verification keys + ansible.builtin.rpm_key: + state: present + key: "{{ item }}" + loop: + - "{{ htcondor_key }}" + - "{{ docker_key }}" + register: key_install + retries: 10 + delay: 60 + until: key_install is success + - name: Enable HTCondor LTS Release repository + ansible.builtin.yum_repository: + name: htcondor-feature + description: HTCondor LTS Release (23.0) + file: htcondor + baseurl: https://research.cs.wisc.edu/htcondor/repo/23.0/el$releasever/$basearch/release + gpgkey: "{{ htcondor_key }}" + gpgcheck: true + repo_gpgcheck: true + priority: "90" + - name: Install HTCondor + ansible.builtin.yum: + name: condor-{{ condor_version | default("23.*") | string }} + state: present + - name: Ensure token directory + ansible.builtin.file: + path: /etc/condor/tokens.d + mode: 0700 + owner: root + group: root + - name: Install Docker and configure HTCondor to use it + when: enable_docker | bool # allows string to be passed at CLI + block: + - name: Setup Docker repo + ansible.builtin.yum_repository: + name: docker-ce-stable + description: Docker CE Stable - $basearch + baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable + enabled: yes + gpgcheck: yes + gpgkey: "{{ docker_key }}" + - name: Install Docker + ansible.builtin.yum: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin + - name: Enable Docker + ansible.builtin.service: + name: docker + state: started + enabled: true + - name: Add condor to docker group + ansible.builtin.user: + name: condor + groups: docker + append: yes diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/main.tf new file mode 100644 index 0000000000..0853e035f4 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + runners = [ + { + "type" = "ansible-local" + "source" = "${path.module}/files/install-htcondor.yaml" + "destination" = "install-htcondor.yaml" + "args" = join(" ", [ + "-e enable_docker=${var.enable_docker}", + "-e condor_version=${var.condor_version}", + ]) + }, + { + "type" = "ansible-local" + "content" = file("${path.module}/files/install-htcondor-autoscaler-deps.yml") + "destination" = "install-htcondor-autoscaler-deps.yml" + }, + { + "type" = "data" + "content" = file("${path.module}/files/autoscaler.py") + "destination" = "/usr/local/htcondor/bin/autoscaler.py" + }, + ] + + install_htcondor_ps1 = templatefile( + "${path.module}/templates/install-htcondor.ps1.tftpl", { + condor_version = var.condor_version, + http_proxy = var.http_proxy, + python_windows_installer_url = var.python_windows_installer_url, + }) + + required_apis = [ + "compute.googleapis.com", + "secretmanager.googleapis.com", + ] +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf new file mode 100644 index 0000000000..c7951737ff --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "runners" { + description = "Runner to install HTCondor using startup-scripts" + value = local.runners +} + +output "windows_startup_ps1" { + description = "Windows PowerShell script to install HTCondor" + value = local.install_htcondor_ps1 +} + +output "gcp_service_list" { + description = "Google Cloud APIs required by HTCondor" + value = local.required_apis +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl new file mode 100644 index 0000000000..7492da3c12 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl @@ -0,0 +1,59 @@ +#Requires -RunAsAdministrator + +# Windows 2016 needs forced upgrade to TLS 1.2 +[Net.ServicePointManager]::SecurityProtocol = 'Tls12' + +# important for catching exception in Invoke-WebRequest +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +%{ if http_proxy != "" ~} +[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") +%{ endif ~} + +# do not show progress bar when running Invoke-WebRequest +$ProgressPreference = 'SilentlyContinue' + +# download C Runtime DLL necessary for HTCondor installer +$runtime_installer = 'C:\vc_redist.x64.exe' +Invoke-WebRequest https://aka.ms/vs/17/release/vc_redist.x64.exe -OutFile "$runtime_installer" +Start-Process -FilePath "$runtime_installer" -Wait -ArgumentList "/norestart /quiet /log c:\vc_redist_log.txt" +Remove-Item "$runtime_installer" + +# download HTCondor installer +$htcondor_installer = 'C:\htcondor.msi' +%{ if condor_version == "23.*" } +Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/current/condor-Windows-x64.msi -OutFile "$htcondor_installer" +%{ else ~} +Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/${condor_version}/release/condor-${condor_version}-Windows-x64.msi -OutFile "$htcondor_installer" +%{ endif ~} +$args='/qn /l* condor-install-log.txt /i' +$args=$args + " $htcondor_installer" +$args=$args + ' NEWPOOL="N"' +$args=$args + ' RUNJOBS="N"' +$args=$args + ' SUBMITJOBS="N"' +$args=$args + ' INSTALLDIR="C:\Condor"' +Start-Process "msiexec.exe" -Wait -ArgumentList "$args" +Remove-Item "$htcondor_installer" + +# do not start HTCondor on boot by default. Allow startup script to download +# configuration first and then start HTCondor +Set-Service -StartupType Manual condor + +# remove settings from condor_config that we want to override in configuration step +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^CONDOR_HOST' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^INSTALL_USER' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^DAEMON_LIST' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^use SECURITY' -NotMatch) + +# install Python so that custom ClassAd hooks can execute +$python_installer = 'C:\python-installer.exe' +Invoke-WebRequest -Uri "${python_windows_installer_url}" -OutFile "$python_installer" +Start-Process -FilePath "$python_installer" -Wait -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1 Include_test=0' +%{ if http_proxy == "" ~} +Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --no-warn-script-location requests" +%{ else ~} +Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --proxy ${http_proxy} --no-warn-script-location requests" +%{ endif ~} +Invoke-WebRequest -Uri "https://raw.githubusercontent.com/htcondor/htcondor/main/src/condor_scripts/common-cloud-attributes-google.py" -OutFile "C:\Condor\bin\common-cloud-attributes-google.py" +Remove-Item "$python_installer" diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/variables.tf new file mode 100644 index 0000000000..1afdf4e0eb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "enable_docker" { + description = "Install and enable docker daemon alongside HTCondor" + type = bool + default = true +} + +variable "condor_version" { + description = "Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: \"23.0.0\",\"23.*\"))" + type = string + default = "23.*" + + validation { + error_message = "var.condor_version must be set to \"23.*\" for latest 23.0 release or to a specific \"23.0.y\" release." + condition = var.condor_version == "23.*" || ( + length(split(".", var.condor_version)) == 3 && alltrue([ + for v in split(".", var.condor_version) : can(tonumber(v)) + ]) && split(".", var.condor_version)[0] == "23" + && split(".", var.condor_version)[1] == "0" + ) + } +} + +variable "http_proxy" { + description = "Set system default web (http and https) proxy for Windows HTCondor installation" + type = string + default = "" + nullable = false +} + +variable "python_windows_installer_url" { + description = "URL of Python installer for Windows" + type = string + default = "https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe" + nullable = false +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/README.md new file mode 100644 index 0000000000..55c2fc7e4e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/README.md @@ -0,0 +1,116 @@ +## Description + +This module will create a startup-script runner that will execute Ramble commands. + +Ramble is a multi-platform experimentation framework capable of driving +software installation, acquiring input files, configuring experiments, and +extracting results. For more information about Ramble, see: +https://github.com/GoogleCloudPlatform/ramble + +This module outputs a startup script runner, which can be combined with other +startup script runners to execute a set of Ramble commands. + +Ramble makes extensive use of Spack. It must be installed with a Toolkit runner +generated by the [spack-setup module](../spack-setup/README.md) following the +[basic example](#basic-example) below. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +# Examples + +## Basic Example + +Below is a basic example of using this module. + +```yaml + - id: spack + source: community/modules/scripts/spack-setup + + - id: ramble-setup + source: community/modules/scripts/ramble-setup + + - id: ramble-execute + source: community/modules/scripts/ramble-execute + use: [spack, ramble-setup] + settings: + commands: + - ramble list +``` + +This example shows installing Spack and Ramble with their own modules +(spack-setup and ramble-setup respectively). Then the ramble-execute module +is added to simply list all applications Ramble knows about. + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | +| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [log\_file](#input\_log\_file) | Log file to write output from Ramble execute steps into | `string` | `"/var/log/ramble-execute.log"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by an instance of ramble-setup.
Can be defined explicitly, or by chaining an instance of a ramble-setup module
through a `use` setting. | `string` | n/a | yes | +| [ramble\_runner](#input\_ramble\_runner) | Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | +| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting.
Defaults to /etc/profile.d/spack.sh if not set. | `string` | `"/etc/profile.d/spack.sh"` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the ramble-setup module. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble startup script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for ramble, to be reused by ramble-execute module. | +| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | +| [ramble\_runner](#output\_ramble\_runner) | Runner to execute Ramble commands using an ansible playbook. The startup-script module
will automatically handle installation of ansible. | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to Spack profile script. | +| [startup\_script](#output\_startup\_script) | Ramble startup script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/main.tf new file mode 100644 index 0000000000..7ef0b029e3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/main.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ramble-execute", ghpc_role = "scripts" }) +} + +locals { + commands_content = var.commands == null ? "echo 'no ramble commands provided'" : indent(4, yamlencode(var.commands)) + + execute_contents = templatefile( + "${path.module}/templates/ramble_execute.yml.tpl", + { + pre_script = "if [ -f ${var.spack_profile_script_path} ]; then . ${var.spack_profile_script_path}; fi; . ${var.ramble_profile_script_path}" + log_file = var.log_file + commands = local.commands_content + system_user_name = var.system_user_name + } + ) + + data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] + + execute_md5 = substr(md5(local.execute_contents), 0, 4) + execute_runner = { + type = "ansible-local" + content = local.execute_contents + destination = "ramble_execute_${local.execute_md5}.yml" + } + + previous_runners = var.ramble_runner != null ? [var.ramble_runner] : [] + runners = concat(local.previous_runners, local.data_runners, [local.execute_runner]) + + # Destinations should be unique while also being known at time of apply + combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) + combined_md5 = substr(md5(local.combined_unique_string), 0, 4) + combined_runner = { + type = "shell" + content = module.startup_script.startup_script + destination = "combined_install_ramble_${local.combined_md5}.sh" + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = var.gcs_bucket_path +} + +resource "local_file" "debug_file_ansible_execute" { + content = local.execute_contents + filename = "${path.module}/debug_execute_${local.execute_md5}.yml" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf new file mode 100644 index 0000000000..4e6c3a44d8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Ramble startup script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Ramble startup script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "ramble_runner" { + description = <<-EOT + Runner to execute Ramble commands using an ansible playbook. The startup-script module + will automatically handle installation of ansible. + EOT + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for ramble, to be reused by ramble-execute module." + value = var.gcs_bucket_path +} + +output "spack_profile_script_path" { + description = "Path to Spack profile script." + value = var.spack_profile_script_path +} + +output "ramble_profile_script_path" { + description = "Path to Ramble profile script." + value = var.ramble_profile_script_path +} + +output "system_user_name" { + description = "The system user used to execute commands." + value = var.system_user_name +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl new file mode 100644 index 0000000000..0e98f3aa2c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Execute Commands + hosts: localhost + vars: + pre_script: ${pre_script} + log_file: ${log_file} + commands: ${commands} + system_user_name: ${system_user_name} + tasks: + - name: Execute command block + block: + - name: Print commands to be executed + ansible.builtin.debug: + msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" + + - name: Streaming log info + ansible.builtin.debug: + msg: | + Logs from commands will not be printed here until success (or failure) + Streaming logs can be found at {{ log_file }} + + - name: Ensure user can write to log file + ansible.builtin.file: + path: "{{ log_file }}" + state: touch + owner: "{{ system_user_name }}" + + - name: Execute commands + ansible.builtin.shell: | + set -eo pipefail + { + {{ pre_script }} + echo " === Starting commands ===" + {{ commands }} + echo " === Finished commands ===" + } 2>&1 | tee -a {{ log_file }} + args: + executable: /bin/bash + register: output + become: true + become_user: "{{ system_user_name }}" + + always: + - name: Print commands output + ansible.builtin.debug: + var: output.stdout_lines diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/variables.tf new file mode 100644 index 0000000000..ec67228df5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/variables.tf @@ -0,0 +1,114 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing spack scripts." + type = string +} + +variable "region" { + description = "Region to place bucket containing spack scripts." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "log_file" { + description = "Log file to write output from Ramble execute steps into" + default = "/var/log/ramble-execute.log" + type = string +} + +variable "data_files" { + description = <<-EOT + A list of files to be transferred prior to running commands. + It must specify one of 'source' (absolute local file path) or 'content' (string). + It must specify a 'destination' with absolute path where file should be placed. + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) + error_message = "All destinations must be absolute paths and start with '/'." + } + validation { + condition = alltrue([ + for r in var.data_files : + can(r["content"]) != can(r["source"]) + ]) + error_message = "A data_file must specify either 'content' or 'source', but never both." + } + validation { + condition = alltrue([ + for r in var.data_files : + lookup(r, "content", lookup(r, "source", null)) != null + ]) + error_message = "A data_file must specify a non-null 'content' or 'source'." + } +} + +variable "commands" { + description = "String of commands to run within this module" + default = null + type = string +} + +variable "ramble_runner" { + description = "Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module." + type = object({ + type = string + content = string + destination = string + }) +} + +variable "system_user_name" { + description = "Name of the system user used to execute commands. Generally passed from the ramble-setup module." + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string +} + +variable "spack_profile_script_path" { + description = <<-EOT + Path to the Spack profile.d script. + Can be defined explicitly, or by chaining an instance of a spack-setup module + through a `use` setting. + Defaults to /etc/profile.d/spack.sh if not set. + EOT + type = string + default = "/etc/profile.d/spack.sh" +} + +variable "ramble_profile_script_path" { + description = <<-EOT + Path to the Ramble profile.d script. Created by an instance of ramble-setup. + Can be defined explicitly, or by chaining an instance of a ramble-setup module + through a `use` setting. + EOT + type = string +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/versions.tf new file mode 100644 index 0000000000..9b23317323 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/versions.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/README.md new file mode 100644 index 0000000000..9891088105 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/README.md @@ -0,0 +1,128 @@ +## Description + +This module will create a set of startup-script runners that will setup Ramble, +and install Ramble’s dependencies. + +Ramble is a multi-platform experimentation framework capable of driving +software installation, acquiring input files, configuring experiments, and +extracting results. For more information about ramble, see: +https://github.com/GoogleCloudPlatform/ramble + +This module outputs two startup script runners, which can be added to startup +scripts to setup, ramble and its dependencies. + +For this module to be completely functional, it depends on a spack +installation. For more information, see Cluster-Toolkit’s Spack module. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +# Examples + +## Basic Example + +```yaml +- id: ramble-setup + source: community/modules/scripts/ramble-setup +``` + +This example simply installs ramble on a VM. + +## Full Example + +```yaml +- id: ramble-setup + source: community/modules/scripts/ramble-setup + settings: + install_dir: /ramble + ramble_url: https://github.com/GoogleCloudPlatform/ramble + ramble_ref: v0.2.1 + log_file: /var/log/ramble.log + chown_owner: “owner” + chgrp_group: “user_group” + chmod_mode: “a+r” +``` + +This example simply installs ramble into a VM at the location `/ramble`, checks +out the v0.2.1 tag, changes the owner and group to “owner” and “user_group”, +and chmod’s the clone to make it world readable. + +Also see a more complete [Ramble example blueprint](../../../examples/ramble.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 4.42 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [chmod\_mode](#input\_chmod\_mode) | Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify).
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | +| [install\_dir](#input\_install\_dir) | Destination directory of installation of Ramble. | `string` | `"/apps/ramble"` | no | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by this module | `string` | `"/etc/profile.d/ramble.sh"` | no | +| [ramble\_ref](#input\_ramble\_ref) | Git ref to checkout for Ramble. | `string` | `"develop"` | no | +| [ramble\_url](#input\_ramble\_url) | URL for Ramble repository to clone. | `string` | `"https://github.com/GoogleCloudPlatform/ramble"` | no | +| [ramble\_virtualenv\_path](#input\_ramble\_virtualenv\_path) | Virtual environment path in which to install Ramble Python interpreter and other dependencies | `string` | `"/usr/local/ramble-python"` | no | +| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | +| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Ramble. It will be created if it does not exist. | `string` | `"ramble"` | no | +| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble installation script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module. | +| [ramble\_path](#output\_ramble\_path) | Location ramble is installed into. | +| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | +| [ramble\_ref](#output\_ramble\_ref) | Git ref the ramble install is checked out to use | +| [ramble\_runner](#output\_ramble\_runner) | Runner to be used with startup-script module or passed to ramble-execute module.
- installs Ramble dependencies
- installs Ramble
- generates profile.d script to enable access to Ramble
This is safe to run in parallel by multiple machines. | +| [startup\_script](#output\_startup\_script) | Ramble installation script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/main.tf new file mode 100644 index 0000000000..4389af7d33 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/main.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ramble-setup", ghpc_role = "scripts" }) +} + +locals { + profile_script = <<-EOF + if [ -f ${var.install_dir}/share/ramble/setup-env.sh ]; then + test -t 1 && echo "** Ramble's python virtualenv (/usr/local/ramble-python) is activated. Call 'deactivate' to deactivate." + VIRTUAL_ENV_DISABLE_PROMPT=1 . ${var.ramble_virtualenv_path}/bin/activate + . ${var.install_dir}/share/ramble/setup-env.sh + fi + EOF + + script_content = templatefile( + "${path.module}/templates/ramble_setup.yml.tftpl", + { + sw_name = "ramble" + profile_script = indent(4, yamlencode(local.profile_script)) + install_dir = var.install_dir + git_url = var.ramble_url + git_ref = var.ramble_ref + chmod_mode = var.chmod_mode + system_user_name = var.system_user_name + system_user_uid = var.system_user_uid + system_user_gid = var.system_user_gid + finalize_setup_script = "echo 'no finalize setup script'" + profile_script_path = var.ramble_profile_script_path + } + ) + + install_ramble_deps_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/install_ramble_deps.yml" + "destination" = "install_ramble_deps.yml" + "args" = "-e virtualenv_path=${var.ramble_virtualenv_path}" + } + + python_reqs_content = templatefile( + "${path.module}/templates/install_ramble_python_deps.yml.tftpl", + { + install_dir = var.install_dir + virtualenv_path = var.ramble_virtualenv_path + } + ) + + python_reqs_runner = { + "type" = "ansible-local" + "content" = local.python_reqs_content + "destination" = "install_ramble_reqs.yml" + } + + install_ramble_runner = { + "type" = "ansible-local" + "content" = local.script_content + "destination" = "install_ramble.yml" + } + + bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}"), 0, 8) + # Max bucket name length is 63, so truncate deployment_name if necessary. + # The string "-ramble-scripts-" is 16 characters and bucket_md5 is 8 characters, + # leaving 63-16-8=39 chars for deployment_name. + bucket_name = "${substr(var.deployment_name, 0, 39)}-ramble-scripts-${local.bucket_md5}" + runners = [local.install_ramble_deps_runner, local.install_ramble_runner, local.python_reqs_runner] + + combined_runner = { + "type" = "shell" + "content" = module.startup_script.startup_script + "destination" = "ramble-install-and-setup.sh" + } + +} + +resource "google_storage_bucket" "bucket" { + project = var.project_id + name = local.bucket_name + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" +} + +resource "local_file" "debug_file_shell_install" { + content = local.script_content + filename = "${path.module}/debug_install.yml" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf new file mode 100644 index 0000000000..e587470eac --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Ramble installation script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Ramble installation script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "ramble_runner" { + description = <<-EOT + Runner to be used with startup-script module or passed to ramble-execute module. + - installs Ramble dependencies + - installs Ramble + - generates profile.d script to enable access to Ramble + This is safe to run in parallel by multiple machines. + EOT + value = local.combined_runner +} + +output "ramble_path" { + description = "Location ramble is installed into." + value = var.install_dir +} + +output "ramble_ref" { + description = "Git ref the ramble install is checked out to use" + value = var.ramble_ref +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module." + value = "gs://${google_storage_bucket.bucket.name}" +} + +output "ramble_profile_script_path" { + description = "Path to Ramble profile script." + value = var.ramble_profile_script_path +} + +output "system_user_name" { + description = "The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands." + value = var.system_user_name +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml new file mode 100644 index 0000000000..b7905bbe9e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml @@ -0,0 +1,50 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Create python virtual env for a tool + become: yes + hosts: localhost + vars: + virtualenv_path: ${virtualenv_path} + tasks: + - name: Install dependencies through system package manager + ansible.builtin.package: + name: + - python3 + - python3-pip + - git + register: package + changed_when: package.changed + retries: 5 + delay: 10 + until: package is success + + - name: Create virtualenv for tool + # Python 3.6 is minimum we wish to support due to ease of installation on + # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip + # supported by 3.6. Additionally, recent versions of pip are necessary for + # proper dependency resolution of real-world problems with google-cloud-* + # (and third-party) Python packages (20.3+ probably effective minimum). + ansible.builtin.pip: + name: pip>=21.3.1 + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv + + - name: Add google-cloud-storage to virtualenv + ansible.builtin.pip: + name: google-cloud-storage + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl new file mode 100644 index 0000000000..ea14780a58 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl @@ -0,0 +1,28 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Install Python Requirements + hosts: localhost + vars: + install_dir: ${install_dir} + virtualenv_path: ${virtualenv_path} + tasks: + + - name: Install dependencies + ansible.builtin.pip: + requirements: "{{ install_dir }}/requirements.txt" + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl new file mode 100644 index 0000000000..ca48a5afa0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl @@ -0,0 +1,157 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install Software + hosts: localhost + vars: + sw_name: ${sw_name} + profile_script: ${profile_script} + install_dir: ${install_dir} + git_url: ${git_url} + git_ref: ${git_ref} + chmod_mode: ${chmod_mode} + system_user_name: ${system_user_name} + system_user_uid: ${system_user_uid} + system_user_gid: ${system_user_gid} + finalize_setup_script: ${finalize_setup_script} + profile_script_path: ${profile_script_path} + tasks: + - name: Print software name + ansible.builtin.debug: + msg: "Running installation for software: {{ sw_name }}" + + - name: Add profile script for software + ansible.builtin.copy: + dest: "{{ profile_script_path }}" + mode: '0644' + content: "{{ profile_script }}" + when: profile_script + + - name: Look up user to use for install + block: + + - name: Check if user already exists + ansible.builtin.getent: + database: passwd + key: "{{ system_user_name }}" + + - name: Look up existing user details + ansible.builtin.user: + name: "{{ system_user_name }}" + register: system_user + + rescue: + - name: User did not exist, create group for system user + ansible.builtin.group: + name: "{{ system_user_name }}" + gid: "{{ system_user_gid }}" + system: true + register: system_group + + - name: Create system user + ansible.builtin.user: + name: "{{ system_user_name }}" + comment: "{{ sw_name }} installation" + uid: "{{ system_user_uid }}" + group: "{{ system_group.name }}" + system: true + register: system_user + + - name: Create parent of install directory + ansible.builtin.file: + path: "{{ install_dir | dirname }}" + state: directory + + - name: Set lock dir + ansible.builtin.set_fact: + lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" + + - name: Acquire lock + ansible.builtin.command: + mkdir "{{ lock_dir }}" + register: lock_out + changed_when: lock_out.rc == 0 + failed_when: false + + - name: Add hostname to lock_dir + ansible.builtin.file: + path: "{{ lock_dir }}/{{ ansible_hostname }}" + state: touch + when: lock_out.rc == 0 + + - name: Clone branch or tag into installation directory + ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} + failed_when: false + register: clone_res + when: lock_out.rc == 0 + + - name: Clone commit hash into installation directory + ansible.builtin.command: "{{ item }}" + with_items: + - git clone {{ git_url }} {{ install_dir }} + - git -C {{ install_dir }} checkout {{ git_ref }} + when: lock_out.rc == 0 and clone_res.rc != 0 + + - name: Transfer ownership to system user + ansible.builtin.file: + path: "{{ install_dir }}" + owner: "{{ system_user.name }}" + group: "{{ system_user.group }}" + recurse: true + follow: false + when: lock_out.rc == 0 + + - name: Finalize setup + ansible.builtin.shell: "{{ finalize_setup_script }}" + when: lock_out.rc == 0 and finalize_setup_script + become: true + become_user: "{{ system_user.name }}" + + - name: Apply chmod + ansible.builtin.file: + path: "{{ install_dir }}" + mode: "{{ chmod_mode | default(omit, true) }}" + recurse: true + follow: false + when: (lock_out.rc == 0) and (chmod_mode != None) + + - name: Release lock + ansible.builtin.file: + path: "{{ lock_dir }}/done" + state: touch + when: lock_out.rc == 0 + + - name: Wait for lock + block: + - name: Wait for lock + ansible.builtin.wait_for: + path: "{{ lock_dir }}/done" + state: present + timeout: 600 + sleep: 10 + when: lock_out.rc != 0 + + rescue: + - name: Timed out on waiting for lock, get lock directory contents + ansible.builtin.find: + paths: "{{ lock_dir }}" + register: lock_dir_contents + + - name: Print lock directory contents, it should contain name of host that is holding lock + ansible.builtin.debug: + msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" + + - name: Failed to get lock + ansible.builtin.fail: + msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/variables.tf new file mode 100644 index 0000000000..0d3a8eed05 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/variables.tf @@ -0,0 +1,97 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "install_dir" { + description = "Destination directory of installation of Ramble." + default = "/apps/ramble" + type = string +} + +variable "ramble_url" { + description = "URL for Ramble repository to clone." + default = "https://github.com/GoogleCloudPlatform/ramble" + type = string +} + +variable "ramble_ref" { + description = "Git ref to checkout for Ramble." + default = "develop" + type = string +} + +variable "chmod_mode" { + description = <<-EOT + Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify). + For usage information see: + https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode + EOT + default = "" + type = string + nullable = false +} + +variable "system_user_name" { + description = "Name of system user that will perform installation of Ramble. It will be created if it does not exist." + default = "ramble" + type = string + nullable = false +} + +variable "system_user_uid" { + description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." + default = 1104762904 + type = number + nullable = false +} + +variable "system_user_gid" { + description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." + default = 1104762904 + type = number + nullable = false +} + +variable "ramble_virtualenv_path" { + description = "Virtual environment path in which to install Ramble Python interpreter and other dependencies" + default = "/usr/local/ramble-python" + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing startup script." + type = string +} + +variable "region" { + description = "Region to place bucket containing startup script." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "ramble_profile_script_path" { + description = "Path to the Ramble profile.d script. Created by this module" + type = string + default = "/etc/profile.d/ramble.sh" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/versions.tf new file mode 100644 index 0000000000..936b4a5b80 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/README.md new file mode 100644 index 0000000000..8cbb75fb42 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/README.md @@ -0,0 +1,141 @@ +## Description + +This module creates a script that defines a software build using Spack and +performs any additional customization to a Spack installation. + +There are two main variable inputs that can be used to define a Spack build: +`data_files` and `commands`. + +- `data_files`: Any files specified will be transferred to the machine running + outputted script. Data file `content` can be defined inline in the blueprint + or can point to a `source`, an absolute local path of a file. This can be used + to transfer environment definition files, config definition files, GPG keys, + or software licenses. `data_files` are transferred before `commands` are run. +- `commands`: A script that is run. This can be used to perform actions such as + installation of compilers & packages, environment creation, adding a build + cache, and modifying the spack configuration. + +## Example + +The `spack-execute` module should `use` a `spack-setup` module. This will +prepend the installation of Spack and its dependencies to the build. Then +`spack-execute` can be used by a module that takes `startup-script` as an input. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: builder-vm + source: modules/compute/vm-instance + use: [network1, spack-build] +``` + +To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. + +[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml + +### Using with `startup-script` module + +The `spack-runner` output can be used by the `startup-script` module. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(spack-build.spack-runner) + - type: shell + destination: "my-script.sh" + content: echo 'hello world' + + - id: workstation + source: modules/compute/vm-instance + use: [network1, startup-script] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | +| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [log\_file](#input\_log\_file) | Defines the logfile that script output will be written to | `string` | `"/var/log/spack.log"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by an instance of spack-setup.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting. | `string` | n/a | yes | +| [spack\_runner](#input\_spack\_runner) | Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | +| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the spack-setup module. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack startup script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | +| [spack\_runner](#output\_spack\_runner) | Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules. | +| [startup\_script](#output\_startup\_script) | Spack startup script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/main.tf new file mode 100644 index 0000000000..04ebcf7d49 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "spack-execute", ghpc_role = "scripts" }) +} + +locals { + commands_content = var.commands == null ? "echo 'no spack commands provided'" : indent(4, yamlencode(var.commands)) + + execute_contents = templatefile( + "${path.module}/templates/execute_commands.yml.tpl", + { + pre_script = ". ${var.spack_profile_script_path}" + log_file = var.log_file + commands = local.commands_content + system_user_name = var.system_user_name + } + ) + + data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] + + execute_md5 = substr(md5(local.execute_contents), 0, 4) + execute_runner = { + type = "ansible-local" + content = local.execute_contents + destination = "spack_execute_${local.execute_md5}.yml" + } + + runners = concat([var.spack_runner], local.data_runners, [local.execute_runner]) + + # Destinations should be unique while also being known at time of apply + combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) + combined_md5 = substr(md5(local.combined_unique_string), 0, 4) + combined_runner = { + type = "shell" + content = module.startup_script.startup_script + destination = "combined_install_spack_${local.combined_md5}.sh" + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = var.gcs_bucket_path +} + +resource "local_file" "debug_file_ansible_execute" { + content = local.execute_contents + filename = "${path.module}/debug_execute_${local.execute_md5}.yml" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/outputs.tf new file mode 100644 index 0000000000..4a52532d51 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Spack startup script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Spack startup script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "spack_runner" { + description = "Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules." + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." + value = var.gcs_bucket_path +} + +output "spack_profile_script_path" { + description = "Path to the Spack profile.d script." + value = var.spack_profile_script_path +} + +output "system_user_name" { + description = "The system user used to execute commands." + value = var.system_user_name +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl new file mode 100644 index 0000000000..0e98f3aa2c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Execute Commands + hosts: localhost + vars: + pre_script: ${pre_script} + log_file: ${log_file} + commands: ${commands} + system_user_name: ${system_user_name} + tasks: + - name: Execute command block + block: + - name: Print commands to be executed + ansible.builtin.debug: + msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" + + - name: Streaming log info + ansible.builtin.debug: + msg: | + Logs from commands will not be printed here until success (or failure) + Streaming logs can be found at {{ log_file }} + + - name: Ensure user can write to log file + ansible.builtin.file: + path: "{{ log_file }}" + state: touch + owner: "{{ system_user_name }}" + + - name: Execute commands + ansible.builtin.shell: | + set -eo pipefail + { + {{ pre_script }} + echo " === Starting commands ===" + {{ commands }} + echo " === Finished commands ===" + } 2>&1 | tee -a {{ log_file }} + args: + executable: /bin/bash + register: output + become: true + become_user: "{{ system_user_name }}" + + always: + - name: Print commands output + ansible.builtin.debug: + var: output.stdout_lines diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/variables.tf new file mode 100644 index 0000000000..851cd1aed8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/variables.tf @@ -0,0 +1,103 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing spack scripts." + type = string +} + +variable "region" { + description = "Region to place bucket containing spack scripts." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "log_file" { + description = "Defines the logfile that script output will be written to" + default = "/var/log/spack.log" + type = string +} + +variable "data_files" { + description = <<-EOT + A list of files to be transferred prior to running commands. + It must specify one of 'source' (absolute local file path) or 'content' (string). + It must specify a 'destination' with absolute path where file should be placed. + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) + error_message = "All destinations must be absolute paths and start with '/'." + } + validation { + condition = alltrue([ + for r in var.data_files : + can(r["content"]) != can(r["source"]) + ]) + error_message = "A data_file must specify either 'content' or 'source', but never both." + } + validation { + condition = alltrue([ + for r in var.data_files : + lookup(r, "content", lookup(r, "source", null)) != null + ]) + error_message = "A data_file must specify a non-null 'content' or 'source'." + } +} + +variable "commands" { + description = "String of commands to run within this module" + type = string + default = null +} + +variable "spack_runner" { + description = "Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module." + type = object({ + type = string + content = string + destination = string + }) +} + +variable "system_user_name" { + description = "Name of the system user used to execute commands. Generally passed from the spack-setup module." + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string +} + +variable "spack_profile_script_path" { + description = <<-EOT + Path to the Spack profile.d script. Created by an instance of spack-setup. + Can be defined explicitly, or by chaining an instance of a spack-setup module + through a `use` setting. + EOT + type = string +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/versions.tf new file mode 100644 index 0000000000..09583c3d43 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/versions.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.0.0" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/README.md new file mode 100644 index 0000000000..01d3e6d389 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/README.md @@ -0,0 +1,382 @@ +## Description + +This module can be used to setup and install Spack on a VM. To actually run +Spack commands to install other software use the +[spack-execute](../spack-execute/) module. + +This module generates a script that performs the following: + +1. Install system dependencies needed for Spack +1. Clone Spack into a predefined directory +1. Check out a specific version of Spack + +There are several options on how to consume the outputs of this module: + +> [!IMPORTANT] +> Breaking changes between after v1.21.0. `spack-install` module replaced by +> `spack-setup` and `spack-execute` modules. +> [Details Below](#deprecations-and-breaking-changes) + +## Examples + +### `use` `spack-setup` with `spack-execute` + +This will prepend the `spack-setup` script to the `spack-execute` commands. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: builder + source: modules/compute/vm-instance + use: [network1, spack-build] +``` + +### `use` `spack-setup` with `vm-instance` or Slurm module + +This will run `spack-setup` scripts on the downstream compute resource. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-installer + source: modules/compute/vm-instance + use: [network1, spack-setup] +``` + +OR + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: [network1, partition1, spack-setup] +``` + +### Build `starup-script` with `spack-runner` output + +This will use the generated `spack-setup` script as one step in `startup-script`. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(spack-setup.spack-runner) + - type: shell + destination: "my-script.sh" + content: echo 'hello world' + + - id: workstation + source: modules/compute/vm-instance + use: [network1, startup-script] +``` + +To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. + +[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml + +## Environment Setup + +### Activating Spack + +[Spack installation] produces a setup script that adds `spack` to your `PATH` as +well as some other command-line integration tools. This script can be found at +`/share/spack/setup-env.sh`. This script will be automatically +added to bash startup by any machine that runs the `spack_runner`. + +If you have multiple machines that all want to use the same shared Spack +installation you can just have both machines run the `spack_runner`. + +[Spack installation]: https://spack-tutorial.readthedocs.io/en/latest/tutorial_basics.html#installing-spack + +### Managing Spack Python dependencies + +Spack is configured with [SPACK_PYTHON] to ensure that Spack itself uses a +Python virtual environment with a supported copy of Python with the package +`google-cloud-storage` pre-installed. This enables Spack to use mirrors and +[build caches][builds] on Google Cloud Storage. It does not configure Python +packages *inside* Spack virtual environments. If you need to add more Python +dependencies for Spack itself, use the `spack python` command: + +```shell +sudo -i spack python -m pip install package-name +``` + +[SPACK_PYTHON]: https://spack.readthedocs.io/en/latest/getting_started.html#shell-support +[builds]: https://spack.readthedocs.io/en/latest/binary_caches.html + +## Spack Permissions + +### System `spack` user is created - Default + +By default this module will create a `spack` linux user and group with +consistent UID and GID. This user and group will own the Spack installation. To +allow a user to manually add Spack packages to the system Spack installation, +you can add the user to the spack group: + +```sh +sudo usermod -a -G spack +``` + +Log out and back in so the group change will take effect, then `` will +be able to call `spack install `. + +> [!NOTE] +> A background persistent SSH connections may prevent the group change from +> taking effect. + +You can use the `system_user_name`, `system_user_uid`, and `system_user_gid` to +customize the name and ids of the system user. While unlikely, it is possible +that the default `system_user_uid` or `system_user_gid` could conflict with +existing UIDs. + +### Use and existing user + +Alternatively, if `system_user_name` is a user already on the system, then this +existing user will be used for Spack installation. + +#### OS Login User + +If OS Login is enabled (default for most Cluster Toolkit modules) then you can +provide an OS Login user name: + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + system_user_name: username_company_com +``` + +This will work even if the user has not yet logged onto the machine. When the +specified user does log on to the machine they will be able to call +`spack install` without any further configuration. + +#### Pre-configured user + +You can also use a startup script to configure a user: + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + system_user_name: special-user + + - id: startup + source: modules/scripts/startup-script + settings: + runners: + - type: shell + destination: "create_user.sh" + content: | + #!/bin/bash + sudo useradd -u 799 special-user + sudo groupadd -g 922 org-group + sudo usermod -g org-group special-user + - $(spack-setup.spack_runner) + + - id: spack-vms + source: modules/compute/vm-instance + use: [network1, startup] + settings: + name_prefix: spack-vm + machine_type: n2d-standard-2 + instance_count: 5 +``` + +### Chaining spack installations + +If there is a need to have a non-root user to install spack packages it is +recommended to create a separate installation for that user and chain Spack installations +([Spack docs](https://spack.readthedocs.io/en/latest/chain.html#chaining-spack-installations)). + +Steps to chain Spack installations: + +1. Get the version of the system Spack: + + ```sh + $ spack --version + + 0.20.0 (e493ab31c6f81a9e415a4b0e0e2263374c61e758) + # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + # Note commit hash and use in next step + ``` + +1. Clone a new spack installation: + + ```sh + git clone -c feature.manyFiles=true https://github.com/spack/spack.git /spack + git -C /spack checkout + ``` + +1. Point the new Spack installation to the system Spack installation. Create a + file at `/spack/etc/spack/upstreams.yaml` with the following + contents: + + ```yaml + upstreams: + spack-instance-1: + install_tree: /sw/spack/opt/spack/ + ``` + +1. Add the following line to your `.bashrc` to make sure the new `spack` is in + your `PATH`. + + ```sh + . /spack/share/spack/setup-env.sh + ``` + +## Deprecations and Breaking Changes + +The old `spack-install` module has been replaced by the `spack-setup` and +`spack-execute` modules. Generally this change strives to allow for a more +flexible definition of a Spack build by using native Spack commands. + +For every deprecated variable from `spack-install` there is documentation on how +to perform the equivalent action using `commands` and `data_files`. The +documentation can be found on the [inputs table](#inputs) below. + +Below is a simple example of the same functionality shown before and after the +breaking changes. + +```yaml + # Before + - id: spack-install + source: community/modules/scripts/spack-install + settings: + install_dir: /sw/spack + compilers: + - gcc@10.3.0 target=x86_64 + packages: + - intel-mpi@2018.4.274%gcc@10.3.0 + +- id: spack-startup + source: modules/scripts/startup-script + settings: + runners: + - $(spack.install_spack_deps_runner) + - $(spack.install_spack_runner) +``` + +```yaml + # After + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + install_dir: /sw/spack + + - id: spack-execute + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + spack load gcc@10.3.0 target=x86_64 + spack compiler find --scope site + spack install intel-mpi@2018.4.274%gcc@10.3.0 + +- id: spack-startup + source: modules/scripts/startup-script + settings: + runners: + - $(spack-execute.spack-runner) +``` + +Although the old `spack-install` module will no longer be maintained, it is +still possible to use the old module in a blueprint by referencing an old +version from GitHub. Note the source line in the following example. + +```yaml + - id: spack-install + source: github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/spack-install?ref=v1.22.1&depth=1 +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 4.42 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [chmod\_mode](#input\_chmod\_mode) | `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification.
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `"g+w"` | no | +| [configure\_for\_google](#input\_configure\_for\_google) | When true, the spack installation will be configured to pull from Google's Spack binary cache. | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | +| [install\_dir](#input\_install\_dir) | Directory to install spack into. | `string` | `"/sw/spack"` | no | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by this module | `string` | `"/etc/profile.d/spack.sh"` | no | +| [spack\_ref](#input\_spack\_ref) | Git ref to checkout for spack. | `string` | `"v0.20.0"` | no | +| [spack\_url](#input\_spack\_url) | URL to clone the spack repo from. | `string` | `"https://github.com/spack/spack"` | no | +| [spack\_virtualenv\_path](#input\_spack\_virtualenv\_path) | Virtual environment path in which to install Spack Python interpreter and other dependencies | `string` | `"/usr/local/spack-python"` | no | +| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Spack. It will be created if it does not exist. | `string` | `"spack"` | no | +| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack installation script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | +| [spack\_path](#output\_spack\_path) | Path to the root of the spack installation | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | +| [spack\_runner](#output\_spack\_runner) | Runner to be used with startup-script module or passed to spack-execute module.
- installs Spack dependencies
- installs Spack
- generates profile.d script to enable access to Spack
This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. | +| [startup\_script](#output\_startup\_script) | Spack installation script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Spack. It can be reused by spack-execute module to install spack packages. | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/main.tf new file mode 100644 index 0000000000..d45f5d1be3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/main.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "spack-setup", ghpc_role = "scripts" }) +} + +locals { + profile_script = <<-EOF + SPACK_PYTHON=${var.spack_virtualenv_path}/bin/python3 + if [ -f ${var.install_dir}/share/spack/setup-env.sh ]; then + test -t 1 && echo "Running Spack setup, this may take a moment on first login." + . ${var.install_dir}/share/spack/setup-env.sh + fi + EOF + + supported_cache_versions = ["v0.19.0", "v0.20.0"] + cache_version = contains(local.supported_cache_versions, var.spack_ref) ? var.spack_ref : "latest" + add_google_mirror_script = !var.configure_for_google ? "" : <<-EOF + if ! spack mirror list | grep -q google_binary_cache; then + spack mirror add --scope site google_binary_cache gs://spack/${local.cache_version} + spack buildcache keys --install --trust + fi + EOF + + finalize_setup_script = <<-EOF + set -e + . ${var.spack_profile_script_path} + spack config --scope site add 'packages:all:permissions:read:world' + spack config --scope site add 'packages:all:permissions:write:group' + spack gpg init + spack compiler find --scope site + ${local.add_google_mirror_script} + # perform fast install to make sure Spack is fully initialized + spack install xz + spack uninstall --yes-to-all xz + EOF + + script_content = templatefile( + "${path.module}/templates/spack_setup.yml.tftpl", + { + sw_name = "spack" + profile_script = indent(4, yamlencode(local.profile_script)) + install_dir = var.install_dir + git_url = var.spack_url + git_ref = var.spack_ref + chmod_mode = var.chmod_mode + system_user_name = var.system_user_name + system_user_uid = var.system_user_uid + system_user_gid = var.system_user_gid + finalize_setup_script = indent(4, yamlencode(local.finalize_setup_script)) + profile_script_path = var.spack_profile_script_path + } + ) + + install_spack_deps_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/install_spack_deps.yml" + "destination" = "install_spack_deps.yml" + "args" = "-e virtualenv_path=${var.spack_virtualenv_path}" + } + install_spack_runner = { + "type" = "ansible-local" + "content" = local.script_content + "destination" = "install_spack.yml" + } + + bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}.${local.script_content}"), 0, 8) + # Max bucket name length is 63, so truncate deployment_name if necessary. + # The string "-spack-scripts-" is 15 characters and bucket_md5 is 8 characters, + # leaving 63-15-8=40 chars for deployment_name. Using 39 so it has the same prefix as the + # ramble-setup module's GCS bucket. + bucket_name = "${substr(var.deployment_name, 0, 39)}-spack-scripts-${local.bucket_md5}" + runners = [local.install_spack_deps_runner, local.install_spack_runner] + + combined_runner = { + "type" = "shell" + "content" = module.startup_script.startup_script + "destination" = "spack-install-and-setup.sh" + } +} + +resource "google_storage_bucket" "bucket" { + project = var.project_id + name = local.bucket_name + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" +} + +resource "local_file" "debug_file_shell_install" { + content = local.script_content + filename = "${path.module}/debug_install.yml" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml new file mode 100644 index 0000000000..2ada34471f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/outputs.tf new file mode 100644 index 0000000000..d94b9757db --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/outputs.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Spack installation script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Spack installation script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "spack_path" { + description = "Path to the root of the spack installation" + value = var.install_dir +} + +output "spack_runner" { + description = <<-EOT + Runner to be used with startup-script module or passed to spack-execute module. + - installs Spack dependencies + - installs Spack + - generates profile.d script to enable access to Spack + This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. + EOT + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." + value = "gs://${google_storage_bucket.bucket.name}" +} + +output "spack_profile_script_path" { + description = "Path to the Spack profile.d script." + value = var.spack_profile_script_path +} + +output "system_user_name" { + description = "The system user used to install Spack. It can be reused by spack-execute module to install spack packages." + value = var.system_user_name +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml new file mode 100644 index 0000000000..b7905bbe9e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml @@ -0,0 +1,50 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Create python virtual env for a tool + become: yes + hosts: localhost + vars: + virtualenv_path: ${virtualenv_path} + tasks: + - name: Install dependencies through system package manager + ansible.builtin.package: + name: + - python3 + - python3-pip + - git + register: package + changed_when: package.changed + retries: 5 + delay: 10 + until: package is success + + - name: Create virtualenv for tool + # Python 3.6 is minimum we wish to support due to ease of installation on + # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip + # supported by 3.6. Additionally, recent versions of pip are necessary for + # proper dependency resolution of real-world problems with google-cloud-* + # (and third-party) Python packages (20.3+ probably effective minimum). + ansible.builtin.pip: + name: pip>=21.3.1 + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv + + - name: Add google-cloud-storage to virtualenv + ansible.builtin.pip: + name: google-cloud-storage + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl new file mode 100644 index 0000000000..ca48a5afa0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl @@ -0,0 +1,157 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install Software + hosts: localhost + vars: + sw_name: ${sw_name} + profile_script: ${profile_script} + install_dir: ${install_dir} + git_url: ${git_url} + git_ref: ${git_ref} + chmod_mode: ${chmod_mode} + system_user_name: ${system_user_name} + system_user_uid: ${system_user_uid} + system_user_gid: ${system_user_gid} + finalize_setup_script: ${finalize_setup_script} + profile_script_path: ${profile_script_path} + tasks: + - name: Print software name + ansible.builtin.debug: + msg: "Running installation for software: {{ sw_name }}" + + - name: Add profile script for software + ansible.builtin.copy: + dest: "{{ profile_script_path }}" + mode: '0644' + content: "{{ profile_script }}" + when: profile_script + + - name: Look up user to use for install + block: + + - name: Check if user already exists + ansible.builtin.getent: + database: passwd + key: "{{ system_user_name }}" + + - name: Look up existing user details + ansible.builtin.user: + name: "{{ system_user_name }}" + register: system_user + + rescue: + - name: User did not exist, create group for system user + ansible.builtin.group: + name: "{{ system_user_name }}" + gid: "{{ system_user_gid }}" + system: true + register: system_group + + - name: Create system user + ansible.builtin.user: + name: "{{ system_user_name }}" + comment: "{{ sw_name }} installation" + uid: "{{ system_user_uid }}" + group: "{{ system_group.name }}" + system: true + register: system_user + + - name: Create parent of install directory + ansible.builtin.file: + path: "{{ install_dir | dirname }}" + state: directory + + - name: Set lock dir + ansible.builtin.set_fact: + lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" + + - name: Acquire lock + ansible.builtin.command: + mkdir "{{ lock_dir }}" + register: lock_out + changed_when: lock_out.rc == 0 + failed_when: false + + - name: Add hostname to lock_dir + ansible.builtin.file: + path: "{{ lock_dir }}/{{ ansible_hostname }}" + state: touch + when: lock_out.rc == 0 + + - name: Clone branch or tag into installation directory + ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} + failed_when: false + register: clone_res + when: lock_out.rc == 0 + + - name: Clone commit hash into installation directory + ansible.builtin.command: "{{ item }}" + with_items: + - git clone {{ git_url }} {{ install_dir }} + - git -C {{ install_dir }} checkout {{ git_ref }} + when: lock_out.rc == 0 and clone_res.rc != 0 + + - name: Transfer ownership to system user + ansible.builtin.file: + path: "{{ install_dir }}" + owner: "{{ system_user.name }}" + group: "{{ system_user.group }}" + recurse: true + follow: false + when: lock_out.rc == 0 + + - name: Finalize setup + ansible.builtin.shell: "{{ finalize_setup_script }}" + when: lock_out.rc == 0 and finalize_setup_script + become: true + become_user: "{{ system_user.name }}" + + - name: Apply chmod + ansible.builtin.file: + path: "{{ install_dir }}" + mode: "{{ chmod_mode | default(omit, true) }}" + recurse: true + follow: false + when: (lock_out.rc == 0) and (chmod_mode != None) + + - name: Release lock + ansible.builtin.file: + path: "{{ lock_dir }}/done" + state: touch + when: lock_out.rc == 0 + + - name: Wait for lock + block: + - name: Wait for lock + ansible.builtin.wait_for: + path: "{{ lock_dir }}/done" + state: present + timeout: 600 + sleep: 10 + when: lock_out.rc != 0 + + rescue: + - name: Timed out on waiting for lock, get lock directory contents + ansible.builtin.find: + paths: "{{ lock_dir }}" + register: lock_dir_contents + + - name: Print lock directory contents, it should contain name of host that is holding lock + ansible.builtin.debug: + msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" + + - name: Failed to get lock + ansible.builtin.fail: + msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/variables.tf new file mode 100644 index 0000000000..85baeec401 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/variables.tf @@ -0,0 +1,106 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +# spack-setup variables + +variable "install_dir" { + description = "Directory to install spack into." + type = string + default = "/sw/spack" +} + +variable "spack_url" { + description = "URL to clone the spack repo from." + type = string + default = "https://github.com/spack/spack" +} + +variable "spack_ref" { + description = "Git ref to checkout for spack." + type = string + default = "v0.20.0" +} + +variable "configure_for_google" { + description = "When true, the spack installation will be configured to pull from Google's Spack binary cache." + type = bool + default = true +} + + +variable "chmod_mode" { + description = <<-EOT + `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification. + For usage information see: + https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode + EOT + default = "g+w" + type = string + nullable = false +} + +variable "system_user_name" { + description = "Name of system user that will perform installation of Spack. It will be created if it does not exist." + default = "spack" + type = string + nullable = false +} + +variable "system_user_uid" { + description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." + default = 1104762903 + type = number + nullable = false +} + +variable "system_user_gid" { + description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." + default = 1104762903 + type = number + nullable = false +} + +variable "spack_virtualenv_path" { + description = "Virtual environment path in which to install Spack Python interpreter and other dependencies" + default = "/usr/local/spack-python" + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing startup script." + type = string +} + +variable "region" { + description = "Region to place bucket containing startup script." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "spack_profile_script_path" { + description = "Path to the Spack profile.d script. Created by this module" + type = string + default = "/etc/profile.d/spack.sh" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/versions.tf new file mode 100644 index 0000000000..ff1180fc1b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/README.md new file mode 100644 index 0000000000..ee9c057c39 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/README.md @@ -0,0 +1,87 @@ +## Description + +This module will insert a dependency on the completion of the startup script +for one or more specified compute VMs and report back if it fails. This can be useful when running +post-boot installation scripts that require the startup script to finish setting up a node. + +> **_WARNING:_**: this module is experimental and not fully supported. + +### Additional Dependencies + +* [**gcloud**](https://cloud.google.com/sdk/gcloud) must be present in the path + of the machine where `terraform apply` is run. + +### Example + +```yaml +- id: workstation + source: modules/compute/vm-instance + use: + - network1 + - my-startup-script + settings: + instance_count: 4 + +# Wait for all instances of the above VM to finish running startup scripts. +- id: wait + source: community/modules/scripts/wait-for-startup + settings: + instance_names: $(workstation.name) +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.validate_instance_names](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.wait_for_startup](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | +| [instance\_name](#input\_instance\_name) | Name of the instance we are waiting for (can be null if 'instance\_names' is not empty) | `string` | `null` | no | +| [instance\_names](#input\_instance\_names) | A list of instance names we are waiting for, in addition to the one mentioned in 'instance\_name' (if any) | `list(string)` | `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [timeout](#input\_timeout) | Timeout in seconds | `number` | `1200` | no | +| [zone](#input\_zone) | The GCP zone where the instance is running | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/main.tf new file mode 100644 index 0000000000..3f6b416251 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + combined_instance_names = concat(var.instance_names, [var.instance_name]) +} + +resource "null_resource" "validate_instance_names" { + lifecycle { + precondition { + condition = var.instance_name != null || length(var.instance_names) > 0 + error_message = "At least one instance name must be provided" + } + } +} + +resource "null_resource" "wait_for_startup" { + count = length(local.combined_instance_names) + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/wait-for-startup-status.sh" + environment = { + INSTANCE_NAME = self.triggers.instance_name + ZONE = var.zone + PROJECT_ID = var.project_id + TIMEOUT = var.timeout + GCLOUD_PATH = var.gcloud_path_override + } + } + + triggers = { + instance_name = local.combined_instance_names[count.index] + } +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf new file mode 100644 index 0000000000..11a2ddf118 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf @@ -0,0 +1,15 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh new file mode 100644 index 0000000000..fae5833121 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z "${INSTANCE_NAME}" ]]; then + echo "INSTANCE_NAME is unset... exiting" + exit 0 +fi +if [[ -z "${ZONE}" ]]; then + echo "ZONE is unset" + exit 1 +fi +if [[ -z "${PROJECT_ID}" ]]; then + echo "PROJECT_ID is unset" + exit 1 +fi +if [[ -z "${TIMEOUT}" ]]; then + echo "TIMEOUT is unset" + exit 1 +fi + +if [[ -n "${GCLOUD_PATH}" ]]; then + export PATH="$GCLOUD_PATH:$PATH" +fi + +echo "Waiting for startup: instance_name='${INSTANCE_NAME}', zone='${ZONE}', project_id='${PROJECT_ID}', timeout_seconds='${TIMEOUT}'" + +# Wrapper around grep that swallows the error status code 1 +c1grep() { grep "$@" || test $? = 1; } + +now=$(date +%s) + +# If VM was created more than 30 days ago, serial port logs may no longer exist. +# Exit without errors if the instance is older than 30 days. +logsExpiryDays=30 +createdTimestampIso=$(gcloud compute instances describe "${INSTANCE_NAME}" --project "${PROJECT_ID}" --zone "${ZONE}" --format "value(creationTimestamp)") +earliestAllowedCreatedTimestamp=$(date -d "${createdTimestampIso} +${logsExpiryDays} day" +%s) +if [[ "$earliestAllowedCreatedTimestamp" -lt "$now" ]]; then + echo "Instance was created more than 30 days ago - serial port 1 logs are likely expired... exiting" + exit 0 +fi + +deadline=$((now + TIMEOUT)) +error_file=$(mktemp) +fetch_cmd="gcloud compute instances get-serial-port-output ${INSTANCE_NAME} --port 1 --zone ${ZONE} --project ${PROJECT_ID}" +# Match string for all finish types of the old guest agent and successful +# finishes on the new guest agent +FINISH_LINE="startup-script exit status" +# Match string for failures on the new guest agent +FINISH_LINE_ERR="Script \"startup-script\" failed with error:" + +# NEW: Accept also these finish lines as success. +STARTUP_SCRIPT_SUCCEEDED_LINE="google-startup-scripts.service: Succeeded." +STARTUP_SCRIPT_FINISHED_LINE="Finished Google Compute Engine Startup Scripts." +STARTUP_SCRIPT_SERVICE_FINISHED_LINE="Finished google-startup-scripts.service - Google Compute Engine Startup Scripts." + +NON_FATAL_ERRORS=( + "Internal error" +) + +until [[ now -gt deadline ]]; do + ser_log=$( + set -o pipefail + ${fetch_cmd} 2>"${error_file}" | + c1grep "${FINISH_LINE}\|${FINISH_LINE_ERR}\|${STARTUP_SCRIPT_SUCCEEDED_LINE}\|${STARTUP_SCRIPT_FINISHED_LINE}\|${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}" + ) || { + err=$(cat "${error_file}") + echo "$err" + fatal_error="true" + for e in "${NON_FATAL_ERRORS[@]}"; do + if [[ $err = *"$e"* ]]; then + fatal_error="false" + break + fi + done + + if [[ $fatal_error = "true" ]]; then + exit 1 + fi + } + if [[ -n "${ser_log}" ]]; then break; fi + sleep 5 + now=$(date +%s) +done + +# This line checks for an exit code - the assumption is that there is a number +# at the end of the line and it is an exit code. +# Modified to correctly extract the last numeric exit status from the relevant log line. +LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=Script \"startup-script\" failed with error: exit status )[0-9]+" | tail -n 1) +if [[ -z "${LAST_EXIT_STATUS}" ]]; then + LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=startup-script exit status )[0-9]+" | tail -n 1) +fi + +# This specific text is monitored for in tests, do not change. +INSPECT_OUTPUT_TEXT="To inspect the startup script output, please run:" + +# --- Prioritize explicit failure from the script itself --- +if [[ "${LAST_EXIT_STATUS}" == 1 ]]; then + echo "startup-script finished with errors, ${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" + exit 1 +# --- Then explicit success from the script itself --- +elif [[ "${LAST_EXIT_STATUS}" == 0 ]]; then + echo "startup-script finished successfully" + exit 0 +elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SUCCEEDED_LINE}"; then + echo "startup-script finished successfully (startup script succeeded line detected)" + exit 0 +elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_FINISHED_LINE}"; then + echo "startup-script finished successfully (startup script finished line detected)" + exit 0 +elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}"; then + echo "startup-script finished successfully (startup script service finished line detected)" + exit 0 +# --- If we reached deadline, it's a timeout --- +elif [[ now -ge deadline ]]; then + echo "startup-script timed out after ${TIMEOUT} seconds" + echo "${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" + exit 1 +# --- All other cases are considered failure or invalid state --- +else + echo "Invalid or undetermined startup script status. Last detected exit status: '${LAST_EXIT_STATUS}'" + echo "${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" + exit "${LAST_EXIT_STATUS}" +fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf new file mode 100644 index 0000000000..fe6410a920 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "instance_name" { + description = "Name of the instance we are waiting for (can be null if 'instance_names' is not empty)" + type = string + default = null +} + +variable "instance_names" { + description = "A list of instance names we are waiting for, in addition to the one mentioned in 'instance_name' (if any)" + type = list(string) + default = [] +} + +variable "zone" { + description = "The GCP zone where the instance is running" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "timeout" { + description = "Timeout in seconds" + type = number + default = 1200 + validation { + condition = var.timeout >= 0 + error_message = "The timeout should be non-negative" + } +} + +variable "gcloud_path_override" { + description = "Directory of the gcloud executable to be used during cleanup" + type = string + default = "" + nullable = false +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf new file mode 100644 index 0000000000..8cd43b944e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/README.md new file mode 100644 index 0000000000..fc25bc0a55 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/README.md @@ -0,0 +1,109 @@ +## Description + +This module contains a set of scripts to be used in customizing Windows VMs at +boot or during image building. Please note that the installation of NVIDIA GPU +drivers takes, at minimum, 30-60 minutes. It is therefore recommended to build +a custom image and reuse it as shown below, rather than install GPU drivers at +boot time. + +> NOTE: the output `windows_startup_ps1` must be passed explicitly as shown +> below when used with Packer modules. This is due to a limitation in the `use` +> keyword and inputs of type `list` in Packer modules; this does not impact +> Terraform modules + +### NVIDIA Drivers and CUDA Toolkit + +Many Google Cloud VM families include or can have NVIDIA GPUs attached to them. +This module supports GPU applications by enabling you to easily install +a compatible release of NVIDIA drivers and of the CUDA Toolkit. The script is +the [solution recommended by our documentation][docs] and is [directly sourced +from GitHub][script-src]. + +[docs]: https://cloud.google.com/compute/docs/gpus/install-drivers-gpu#windows +[script-src]: https://github.com/GoogleCloudPlatform/compute-gpu-installation/blob/24dac3004360e0696c49560f2da2cd60fcb80107/windows/install_gpu_driver.ps1 + +```yaml +- group: primary + modules: + - id: network1 + source: modules/network/vpc + settings: + enable_iap_rdp_ingress: true + enable_iap_winrm_ingress: true + + - id: windows_startup + source: community/modules/scripts/windows-startup-script + settings: + install_nvidia_driver: true + +- group: packer + modules: + - id: image + source: modules/packer/custom-image + kind: packer + use: + - network1 + - windows_startup + settings: + source_image_family: windows-2016 + machine_type: n1-standard-8 + accelerator_count: 1 + accelerator_type: nvidia-tesla-t4 + disk_size: 75 + disk_type: pd-ssd + omit_external_ip: false + state_timeout: 15m +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [http\_proxy](#input\_http\_proxy) | Set http and https proxy for use by Invoke-WebRequest commands | `string` | `""` | no | +| [http\_proxy\_set\_environment](#input\_http\_proxy\_set\_environment) | Set system default environment variables http\_proxy and https\_proxy for all commands | `bool` | `false` | no | +| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install\_nvidia\_driver\_script | `bool` | `false` | no | +| [install\_nvidia\_driver\_args](#input\_install\_nvidia\_driver\_args) | Arguments to supply to NVIDIA driver install script | `string` | `"/s /n"` | no | +| [install\_nvidia\_driver\_script](#input\_install\_nvidia\_driver\_script) | Install script for NVIDIA drivers specified by http/https URL | `string` | `"https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe"` | no | +| [no\_proxy](#input\_no\_proxy) | Environment variables no\_proxy (only used if var.http\_proxy\_set\_environment is enabled) | `string` | `"169.254.169.254,metadata,metadata.google.internal,.googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | A string list of scripts selected by this module | + diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/main.tf new file mode 100644 index 0000000000..5e6bc8b94d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/main.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + setx_http_proxy_ps1 = !var.http_proxy_set_environment ? [] : [ + templatefile("${path.module}/templates/setx_http_proxy.ps1", { + "http_proxy" : var.http_proxy, + "no_proxy" : var.no_proxy, + }) + ] + + nvidia_ps1 = !var.install_nvidia_driver ? [] : [ + templatefile("${path.module}/templates/install_gpu_driver.ps1.tftpl", { + "url" : var.install_nvidia_driver_script + "args" : var.install_nvidia_driver_args + "http_proxy" : var.http_proxy, + }) + ] + + startup_ps1 = concat(local.setx_http_proxy_ps1, local.nvidia_ps1) +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf new file mode 100644 index 0000000000..006ea312ad --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "windows_startup_ps1" { + description = "A string list of scripts selected by this module" + value = local.startup_ps1 +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl new file mode 100644 index 0000000000..55c4a2a3cd --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl @@ -0,0 +1,38 @@ +#Requires -RunAsAdministrator + +# Windows 2016 needs forced upgrade to TLS 1.2 +[Net.ServicePointManager]::SecurityProtocol = 'Tls12' + +# important for catching exception in Invoke-WebRequest +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +%{ if http_proxy != "" } +[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") +%{ endif } + +# Create the folder for the driver download +$file_dir = 'C:\NVIDIA-Driver\nvidia_installer_windows.exe' +if (!(Test-Path -Path 'C:\NVIDIA-Driver')) { + New-Item -Path 'C:\' -Name 'NVIDIA-Driver' -ItemType 'directory' | Out-Null +} + +# Download the file to a specified directory +Write-Output "Downloading ${url} to $file_dir" +# Disabling progress bar has surprising large (10-100x) impact on speed +$ProgressPreference = 'SilentlyContinue' +try { + Invoke-WebRequest -Uri "${url}" -OutFile "$file_dir" +} catch { + Write-Output "$_" + throw "Failed to download ${url}; exiting startup script" +} + +# Install the file with the specified path from earlier as well as the RunAs admin option +Write-Output "Executing $file_dir with arguments '${args}'" +try { + Start-Process -FilePath "$file_dir" -ArgumentList '${args}' -Wait +} catch { + Write-Output "$_" + throw "Could not install NVIDIA driver; exiting startup script" +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 new file mode 100644 index 0000000000..ca4d13f98b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 @@ -0,0 +1,21 @@ +<# + Copyright 2025 "Google LLC" + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#> + +#Requires -RunAsAdministrator + +setx http_proxy ${http_proxy} /m +setx https_proxy ${http_proxy} /m +setx no_proxy ${no_proxy} /m diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf new file mode 100644 index 0000000000..9e4fb9e67d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "install_nvidia_driver" { + description = "Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install_nvidia_driver_script" + type = bool + default = false +} + +variable "install_nvidia_driver_script" { + description = "Install script for NVIDIA drivers specified by http/https URL" + type = string + default = "https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe" +} + +variable "install_nvidia_driver_args" { + description = "Arguments to supply to NVIDIA driver install script" + type = string + default = "/s /n" +} + +variable "http_proxy" { + description = "Set http and https proxy for use by Invoke-WebRequest commands" + type = string + default = "" + nullable = false +} + +variable "http_proxy_set_environment" { + description = "Set system default environment variables http_proxy and https_proxy for all commands" + type = bool + default = false + nullable = false +} + +variable "no_proxy" { + description = "Environment variables no_proxy (only used if var.http_proxy_set_environment is enabled)" + type = string + default = "169.254.169.254,metadata,metadata.google.internal,.googleapis.com" + nullable = false +} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf new file mode 100644 index 0000000000..dfeeac34f8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/build_script/modules/embedded/modules/README.md b/deletion-test/build_script/modules/embedded/modules/README.md new file mode 100644 index 0000000000..6886b3f330 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/README.md @@ -0,0 +1,554 @@ +# Modules + +This directory contains a set of core modules built for the Cluster Toolkit. Modules +describe the building blocks of an AI/ML and HPC deployment. The expected fields in a +module are listed in more detail [below](#module-fields). Blueprints can be +extended in functionality by incorporating [modules from GitHub +repositories][ghmods]. + +[ghmods]: #github-modules + +## All Modules + +Modules from various sources are all listed here for visibility. Badges are used +to indicate the source and status of many of these resources. + +Modules listed below with the ![core-badge] badge are located in this +folder and are tested and maintained by the Cluster Toolkit team. + +Modules labeled with the ![community-badge] badge are contributed by +the community (including the Cluster Toolkit team, partners, etc.). Community modules +are located in the [community folder](../community/modules/README.md). + +Modules labeled with the ![deprecated-badge] badge are now deprecated and may be +removed in the future. Customers are advised to transition to alternatives. + +Modules that are still in development and less stable are labeled with the +![experimental-badge] badge. + +[core-badge]: https://img.shields.io/badge/-core-blue?style=plastic +[community-badge]: https://img.shields.io/badge/-community-%23b8def4?style=plastic +[stable-badge]: https://img.shields.io/badge/-stable-lightgrey?style=plastic +[experimental-badge]: https://img.shields.io/badge/-experimental-%23febfa2?style=plastic +[deprecated-badge]: https://img.shields.io/badge/-deprecated-%23fea2a2?style=plastic + +### Compute + +* **[vm-instance]** ![core-badge] : Creates one or more VM instances. +* **[schedmd-slurm-gcp-v6-partition]** ![core-badge] : + Creates a partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. +* **[schedmd-slurm-gcp-v6-nodeset]** ![core-badge] : + Creates a nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. +* **[schedmd-slurm-gcp-v6-nodeset-tpu]** ![core-badge] : + Creates a TPU nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. +* **[schedmd-slurm-gcp-v6-nodeset-dynamic]** ![core-badge] ![experimental-badge]: + Creates a dynamic nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module and instance template. +* **[gke-node-pool]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes node pool using GKE. +* **[resource-policy]** ![core-badge] ![experimental-badge] : Create a resource policy for compute engines that can be applied to gke-node-pool's nodes. +* **[gke-job-template]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes job file to be used with a [gke-node-pool]. +* **[htcondor-execute-point]** ![community-badge] ![experimental-badge] : + Manages a group of execute points for use in an [HTCondor + pool][htcondor-setup]. +* **[mig]** ![community-badge] ![experimental-badge] : Creates a Managed Instance Group. +* **[notebook]** ![community-badge] ![experimental-badge] : Creates a Vertex AI + Notebook. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[gke-nodeset]** ![community-badge] ![experimental-badge] : Create a slinky nodeset to be used by the [gke-partition] module. +* **[gke-partition]** ![community-badge] ![experimental-badge] : Creates a slinky partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. + +[vm-instance]: compute/vm-instance/README.md +[gke-node-pool]: ../modules/compute/gke-node-pool/README.md +[resource-policy]: ../modules/compute/resource-policy/README.md +[gke-job-template]: ../modules/compute/gke-job-template/README.md +[schedmd-slurm-gcp-v6-partition]: ../community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md +[schedmd-slurm-gcp-v6-nodeset]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md +[schedmd-slurm-gcp-v6-nodeset-tpu]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md +[schedmd-slurm-gcp-v6-nodeset-dynamic]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md +[htcondor-execute-point]: ../community/modules/compute/htcondor-execute-point/README.md +[mig]: ../community/modules/compute/mig/README.md +[notebook]: ../community/modules/compute/notebook/README.md +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +### Database + +* **[slurm-cloudsql-federation]** ![community-badge] ![experimental-badge] : + Creates a [Google SQL Instance](https://cloud.google.com/sql/) meant to be + integrated with a [slurm-controller][schedmd-slurm-gcp-v6-controller]. +* **[bigquery-dataset]** ![community-badge] ![experimental-badge] : Creates a BQ + dataset. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[bigquery-table]** ![community-badge] ![experimental-badge] : Creates a BQ + table. Primarily used for + [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. + +[slurm-cloudsql-federation]: ../community/modules/database/slurm-cloudsql-federation/README.md +[bigquery-dataset]: ../community/modules/database/bigquery-dataset/README.md +[bigquery-table]: ../community/modules/database/bigquery-table/README.md +[fsi-montecarlo-on-batch]: ../community/modules/files/fsi-montecarlo-on-batch/README.md + +### File System + +* **[filestore]** ![core-badge] : Creates a + [filestore](https://cloud.google.com/filestore) file system. +* **[parallelstore]** ![core-badge] ![experimental-badge]: Creates a + [parallelstore](https://cloud.google.com/parallelstore) file system. +* **[pre-existing-network-storage]** ![core-badge] : Specifies a + pre-existing file system that can be mounted on a VM. +* **[managed-lustre]** ![core-badge] ![experimental-badge]: Creates a + [managed-lustred](https://cloud.google.com/managed-lustre) file system. +* **[DDN-EXAScaler]** ![community-badge] ![deprecated-badge] : Creates + a [DDN EXAscaler lustre](https://www.ddn.com/partners/google-cloud-platform/) + file system. This module is deprecated and will be removed by July 1, 2025. Consider migrating to managed-lustre. +* **[cloud-storage-bucket]** ![core-badge] : Creates a Google Cloud Storage (GCS) bucket. +* **[gke-persistent-volume]** ![core-badge] ![experimental-badge] : Creates + persistent volumes and persistent volume claims for shared storage. +* **[nfs-server]** ![community-badge] ![experimental-badge] : Creates a VM and + configures an NFS server that can be mounted by other VM. +* **[weka-client]** ![community-badge] ![experimental-badge] : Installs client + and mounts [WEKA](https://www.weka.io/) filesystems. + +[filestore]: file-system/filestore/README.md +[parallelstore]: file-system/parallelstore/README.md +[pre-existing-network-storage]: file-system/pre-existing-network-storage/README.md +[managed-lustre]: file-system/managed-lustre/README.md +[ddn-exascaler]: ../community/modules/file-system/DDN-EXAScaler/README.md +[nfs-server]: ../community/modules/file-system/nfs-server/README.md +[cloud-storage-bucket]: file-system/cloud-storage-bucket/README.md +[gke-persistent-volume]: file-system/gke-persistent-volume/README.md +[weka-client]: ../community/modules/file-system/weka-client/README.md + +### Monitoring + +* **[dashboard]** ![core-badge] : Creates a + [monitoring dashboard](https://cloud.google.com/monitoring/dashboards) for + visually tracking a Cluster Toolkit deployment. + +[dashboard]: monitoring/dashboard/README.md + +### Network + +* **[vpc]** ![core-badge] : Creates a + [Virtual Private Cloud (VPC)](https://cloud.google.com/vpc) network with + regional subnetworks and firewall rules. +* **[multivpc]** ![core-badge] ![experimental-badge]: Creates a variable + number of VPC networks using the [vpc] module. +* **[pre-existing-vpc]** ![core-badge] : Used to connect newly + built components to a pre-existing VPC network. +* **[firewall-rules]** ![core-badge] ![experimental-badge] : Add custom firewall + rules to existing networks (commonly used with [pre-existing-vpc]). +* **[private-service-access]** ![community-badge] ![experimental-badge] : + Configures Private Services Access for a VPC network (commonly used with [filestore] and [slurm-cloudsql-federation]). + +[vpc]: network/vpc/README.md +[multivpc]: network/multivpc/README.md +[pre-existing-vpc]: network/pre-existing-vpc/README.md +[firewall-rules]: network/firewall-rules/README.md +[private-service-access]: ../community/modules/network/private-service-access/README.md + +### Packer + +* **[custom-image]** ![core-badge] : Creates a custom VM Image + based on the GCP HPC VM image. + +[custom-image]: packer/custom-image/README.md + +### Project + +* **[service-account]** ![community-badge] ![experimental-badge] : Creates [service + accounts](https://cloud.google.com/iam/docs/service-accounts) for a GCP + project. +* **[service-enablement]** ![community-badge] ![experimental-badge] : Allows enabling + various APIs for a Google Cloud Project. + +[service-account]: ../community/modules/project/service-account/README.md +[service-enablement]: ../community/modules/project/service-enablement/README.md + +### Pub/Sub + +* **[topic]** ![community-badge] ![experimental-badge] : Creates a +Pub/Sub topic. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[bigquery-sub]** ![community-badge] ![experimental-badge] : Creates a +Pub/Sub subscription. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. + +[topic]: ../community/modules/pubsub/topic/README.md +[bigquery-sub]: ../community/modules/pubsub/bigquery-sub/README.md + +### Remote Desktop + +* **[chrome-remote-desktop]** ![community-badge] ![experimental-badge] : Creates + a GPU accelerated Chrome Remote Desktop. + +[chrome-remote-desktop]: ../community/modules/remote-desktop/chrome-remote-desktop/README.md + +### Scheduler + +* **[batch-job-template]** ![core-badge] : Creates a Google Cloud Batch job + template that works with other Toolkit modules. +* **[batch-login-node]** ![core-badge] : Creates a VM that can be used for + submission of Google Cloud Batch jobs. +* **[gke-cluster]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes cluster using GKE. +* **[pre-existing-gke-cluster]** ![core-badge] ![experimental-badge] : Retrieves an existing GKE cluster. Substitute for ([gke-cluster]) module. +* **[schedmd-slurm-gcp-v6-controller]** ![core-badge] : + Creates a Slurm controller node. +* **[schedmd-slurm-gcp-v6-login]** ![core-badge] : + Creates a Slurm login node. +* **[htcondor-setup]** ![community-badge] ![experimental-badge] : Creates the + base infrastructure for an HTCondor pool (service accounts and Cloud Storage bucket). +* **[htcondor-pool-secrets]** ![community-badge] ![experimental-badge] : Creates + and manages access to the secrets necessary for secure operation of an + HTCondor pool. +* **[htcondor-access-point]** ![community-badge] ![experimental-badge] : Creates + a regional instance group managing a highly available HTCondor access point + (login node). + +[batch-job-template]: ../modules/scheduler/batch-job-template/README.md +[batch-login-node]: ../modules/scheduler/batch-login-node/README.md +[gke-cluster]: ../modules/scheduler/gke-cluster/README.md +[pre-existing-gke-cluster]: ../modules/scheduler/pre-existing-gke-cluster/README.md +[htcondor-setup]: ../community/modules/scheduler/htcondor-setup/README.md +[htcondor-pool-secrets]: ../community/modules/scheduler/htcondor-pool-secrets/README.md +[htcondor-access-point]: ../community/modules/scheduler/htcondor-access-point/README.md +[schedmd-slurm-gcp-v6-controller]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md +[schedmd-slurm-gcp-v6-login]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md + +### Scripts + +* **[startup-script]** ![core-badge] : Creates a customizable startup script + that can be fed into compute VMs. +* **[windows-startup-script]** ![community-badge] ![experimental-badge]: Creates + Windows PowerShell (PS1) scripts that can be used to customize Windows VMs + and VM images. +* **[htcondor-install]** ![community-badge] ![experimental-badge] : Creates + a startup script to install HTCondor and exports a list of required APIs +* **[ramble-execute]** ![community-badge] ![experimental-badge] : Creates a + startup script to execute + [Ramble](https://github.com/GoogleCloudPlatform/ramble) commands on a target + VM +* **[ramble-setup]** ![community-badge] ![experimental-badge] : Creates a + startup script to install + [Ramble](https://github.com/GoogleCloudPlatform/ramble) on an instance or a + slurm login or controller. +* **[spack-setup]** ![community-badge] ![experimental-badge] : Creates a startup + script to install [Spack](https://github.com/spack/spack) on an instance or a + slurm login or controller. +* **[spack-execute]** ![community-badge] ![experimental-badge] : Defines a + software build using [Spack](https://github.com/spack/spack). +* **[wait-for-startup]** ![community-badge] ![experimental-badge] : Waits for + successful completion of a startup script on a compute VM. + +[startup-script]: scripts/startup-script/README.md +[windows-startup-script]: ../community/modules/scripts/windows-startup-script/README.md +[htcondor-install]: ../community/modules/scripts/htcondor-install/README.md +[kubernetes-operations]: ../community/modules/scripts/kubernetes-operations/README.md +[ramble-execute]: ../community/modules/scripts/ramble-execute/README.md +[ramble-setup]: ../community/modules/scripts/ramble-setup/README.md +[spack-setup]: ../community/modules/scripts/spack-setup/README.md +[spack-execute]: ../community/modules/scripts/spack-execute/README.md +[wait-for-startup]: ../community/modules/scripts/wait-for-startup/README.md + +## Module Fields + +### ID (Required) + +The `id` field is used to uniquely identify and reference a defined module. +ID's are used in [variables](../examples/README.md#variables) and become the +name of each module when writing the terraform `main.tf` file. They are also +used in the [use](#use-optional) and [outputs](#outputs-optional) lists +described below. + +For terraform modules, the ID will be rendered into the terraform module label +at the top level main.tf file. + +### Source (Required) + +The source is a path or URL that points to the source files for Packer or +Terraform modules. A source can either be a filesystem path or a URL to a git +repository: + +* Filesystem paths + * modules embedded in the `gcluster` executable + * modules in the local filesystem +* Remote modules using [Terraform URL syntax](https://developer.hashicorp.com/terraform/language/modules/sources) + * Hosted on [GitHub](https://developer.hashicorp.com/terraform/language/modules/sources#github) + * Google Cloud Storage [Buckets](https://developer.hashicorp.com/terraform/language/modules/sources#gcs-bucket) + * Generic [git repositories](https://developer.hashicorp.com/terraform/language/modules/sources#generic-git-repository) + + when modules are in a subdirectory of the git repository, a special + double-slash `//` notation can be required as described below + +An important distinction is that those URLs are natively supported by Terraform so +they are not copied to your deployment directory. Packer does not have native +support for git-hosted modules so the Toolkit will copy these modules into the +deployment folder on your behalf. + +#### Embedded Modules + +Embedded modules are added to the gcluster binary during compilation and cannot +be edited. To refer to embedded modules, set the source path to +`modules/<>` or `community/modules/<>`. + +The paths match the modules in the repository structure for [core modules](./) +and [community modules](../community/modules/). Because the modules are embedded +during compilation, your local copies may differ unless you recompile gcluster. + +For example, this example snippet uses the embedded pre-existing-vpc module: + +```yaml + - id: network1 + source: modules/network/pre-existing-vpc +``` + +#### Local Modules + +Local modules point to a module in the file system and can easily be edited. +They are very useful during module development. To use a local module, set +the source to a path starting with `/`, `./`, or `../`. For instance, the +following module definition refers the local pre-existing-vpc modules. + +```yaml + - id: network1 + source: modules/network/pre-existing-vpc +``` + +> **_NOTE:_** Relative paths (beginning with `.` or `..` must be relative to the +> working directory from which `gcluster` is executed. This example would have to be +> run from a local copy of the Cluster Toolkit repository. An alternative is to use +> absolute paths to modules. + +#### GitHub-hosted Modules and Packages + +To use a Terraform module available on GitHub, set the source to a path starting +with `github.com` (HTTPS) or `git@github.com` (SSH). For instance, the following +module definition sources the Toolkit vpc module: + +```yaml + - id: network1 + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc +``` + +This example uses the [double-slash notation][tfsubdir] (`//`) to indicate that +the Toolkit is a "package" of multiple modules whose root directory is the root +of the git repository. The remainder of the path indicates the sub-directory of +the vpc module. + +The example above uses the default `main` branch of the Toolkit. Specific +[revisions][tfrev] can be selected with any valid [git reference][gitref]. +(git branch, commit hash or tag). If the git reference is a tag or branch, we +recommend setting `&depth=1` to reduce the data transferred over the network. +This option cannot be set when the reference is a commit hash. The following +examples select the vpc module on the active `develop` branch and also an older +release of the filestore module: + +```yaml + - id: network1 + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc?ref=develop + ... + - id: homefs + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/file-system/filestore?ref=v1.22.1&depth=1 +``` + +Because Terraform modules natively support this syntax, gcluster will not copy +GitHub-hosted modules into your deployment folder. Terraform will download them +into a hidden folder when you run `terraform init`. + +[tfrev]: https://www.terraform.io/language/modules/sources#selecting-a-revision +[gitref]: https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection#_single_revisions +[tfsubdir]: https://www.terraform.io/language/modules/sources#modules-in-package-sub-directories + +##### GitHub-hosted Packer modules + +Packer does not natively support GitHub-hosted modules so `gcluster create` will +copy modules into your deployment folder. + +If the module uses `//` package notation, `gcluster create` will copy the entire +repository to the module path: `deployment_name/group_name/module_id`. However, +when `gcluster deploy` is invoked, it will run Packer from the subdirectory +`deployment_name/group_name/module_id/subdirectory/after/double_slash`. + +If the module does not use `//` package notation, `gcluster create` will copy +only the final directory in the path to `deployment_name/group_name/module_id`. + +In all cases, `gcluster create` will remove the `.git` directory from the packer +module to ensure that you can manage the entire deployment directory with its +own git versioning. + +##### GitHub over SSH + +Get module from GitHub over SSH: + +```yaml + - id: network1 + source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc +``` + +Specific versions can be selected as for HTTPS: + +```yaml + - id: network1 + source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc?ref=v1.22.1&depth=1 +``` + +##### Generic Git Modules + +To use a Terraform module available in a non-GitHub git repository such as +gitlab, set the source to a path starting `git::`. Two Standard git protocols +are supported, `git::https://` for HTTPS or `git::git@github.com` for SSH. + +Additional formatting and features after `git::` are identical to that of the +[GitHub Modules](#github-modules) described above. + +#### Google Cloud Storage Modules + +To use a Terraform module available in a Google Cloud Storage bucket, set the source +to a URL with the special `gcs::` prefix, followed by a [GCS bucket object URL](https://cloud.google.com/storage/docs/request-endpoints#typical). + +For example: `gcs::https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH_TO_MODULE` + +### Kind (May be Required) + +`kind` refers to the way in which a module is deployed. Currently, `kind` can be +either `terraform` or `packer`. It must be specified for modules of type +`packer`. If omitted, it will default to `terraform`. + +### Settings (May Be Required) + +The settings field is a map that supplies any user-defined variables for each +module. Settings values can be simple strings, numbers or booleans, but can +also support complex data types like maps and lists of variable depth. These +settings will become the values for the variables defined in either the +`variables.tf` file for Terraform or `variable.pkr.hcl` file for Packer. + +For some modules, there are mandatory variables that must be set, +therefore `settings` is a required field in that case. In many situations, a +combination of sensible defaults, deployment variables and used modules can +populated all required settings and therefore the settings field can be omitted. + +### Use (Optional) + +The `use` field is a powerful way of linking a module to one or more other +modules. When a module "uses" another module, the outputs of the used +module are compared to the settings of the current module. If they have +matching names and the setting has no explicit value, then it will be set to +the used module's output. For example, see the following blueprint snippet: + +```yaml +modules: +- id: network1 + source: modules/network/vpc + +- id: workstation + source: modules/compute/vm-instance + use: [network1] + settings: + ... +``` + +In this snippet, the VM instance `workstation` uses the outputs of vpc +`network1`. + +In this case both `network_self_link` and `subnetwork_self_link` in the +[workstation settings](compute/vm-instance/README.md#Inputs) will be set +to `$(network1.network_self_link)` and `$(network1.subnetwork_self_link)` which +refer to the [network1 outputs](network/vpc/README#Outputs) +of the same names. + +The order of precedence that `gcluster` uses in determining when to infer a setting +value is in the following priority order: + +1. Explicitly set in the blueprint using the `settings` field +1. Output from a used module, taken in the order provided in the `use` list +1. Deployment variable (`vars`) of the same name +1. Default value for the setting + +> **_NOTE:_** See the +> [network storage documentation](./../docs/network_storage.md) for more +> information about mounting network storage file systems via the `use` field. + +### Outputs (Optional) + +The `outputs` field adds the output of individual Terraform modules to the +output of its deployment group. This enables the value to be available via +`terraform output`. This can useful for displaying the IP of a login node or +printing instructions on how to use a module, as we have in the +[monitoring dashboard module](monitoring/dashboard/README.md#Outputs). + +The outputs field is a lists that it can be in either of two formats: a string +equal to the name of the module output, or a map specifying the `name`, +`description`, and whether the value is `sensitive` and should be suppressed +from the standard output of Terraform commands. An example is shown below +that displays the internal and public IP addresses of a VM created by the +vm-instance module: + +```yaml + - id: vm + source: modules/compute/vm-instance + use: + - network1 + settings: + machine_type: e2-medium + outputs: + - internal_ip + - name: external_ip + description: "External IP of VM" + sensitive: true +``` + +The outputs shown after running Terraform apply will resemble: + +```text +Apply complete! Resources: 7 added, 0 changed, 0 destroyed. + +Outputs: + +external_ip_simplevm = +internal_ip_simplevm = [ + "10.128.0.19", +] +``` + +### Required Services (APIs) (optional) + +Each Toolkit module depends upon Google Cloud services ("APIs") being enabled +in the project used by the AI/ML and HPC environment. For example, the [creation of +VMs](compute/vm-instance/) requires the Compute Engine API +(compute.googleapis.com). The [startup-script](scripts/startup-script/) module +requires the Cloud Storage API (storage.googleapis.com) for storage of the +scripts themselves. Each module included in the Toolkit source code describes +its required APIs internally. The Toolkit will merge the requirements from all +modules and [automatically validate](../README.md#blueprint-validation) that all +APIs are enabled in the project specified by `$(vars.project_id)`. + +## Common Settings + +The following common naming conventions should be used to decrease the verbosity +needed to define a blueprint. This is intentional to allow multiple +modules to share inferred settings from deployment variables or from other +modules listed under the `use` field. + +For example, if all modules are to be created in a single region, that region +can be defined as a deployment variable named `region`, which is shared between +all modules without an explicit setting. Similarly, if many modules need to be +connected to the same VPC network, they all can add the vpc module ID to their +`use` list so that `network_self_link` would be inferred from that vpc module rather +than having to set it manually. + +* **project_id**: The GCP project ID in which to create the GCP resources. +* **deployment_name**: The name of the current deployment of a blueprint. This + can help to avoid naming conflicts of modules when multiple deployments are + created from the same blueprint. +* **region**: The GCP + [region](https://cloud.google.com/compute/docs/regions-zones) the module + will be created in. +* **zone**: The GCP [zone](https://cloud.google.com/compute/docs/regions-zones) + the module will be created in. +* **labels**: + [Labels](https://cloud.google.com/resource-manager/docs/creating-managing-labels) + added to the module. In order to include any module in advanced + monitoring, labels must be exposed. We strongly recommend that all modules + expose this variable. + +## Writing Custom Cluster Toolkit Modules + +Modules are flexible by design, however we define some [best practices](../docs/module-guidelines.md) when +creating a new module meant to be used with the Cluster Toolkit. diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/README.md b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/README.md new file mode 100644 index 0000000000..f807cd727e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/README.md @@ -0,0 +1,133 @@ +## Description + +This module is used to create a Kubernetes job template file. + +The job template file can be submitted as is or used as a template for further +customization. Add the `instructions` output to a blueprint (as shown below) to +get instructions on how to use `kubectl` to submit the job. + +This module is designed to `use` one or more `gke-node-pool` modules. The job +will be configured to run on any of the specified node pools. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a GKE job template file. + +```yaml + - id: job-template + source: modules/compute/gke-job-template + use: [compute_pool] + settings: + node_count: 3 + outputs: [instructions] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### Storage Options + +This module natively supports: + +* Filestore as a shared file system between pods/nodes. +* Pod level ephemeral storage options: + * memory backed emptyDir + * local SSD backed emptyDir + * SSD persistent disk backed ephemeral volume + * balanced persistent disk backed ephemeral volume + +See the [storage-gke.yaml blueprint](../../../examples/storage-gke.yaml) and the +associated [documentation](../../../../examples/README.md#storage-gkeyaml--) for +examples of how to use Filestore and ephemeral storage with this module. + +### Requested Resources + +When one or more `gke-node-pool` modules are referenced with the `use` field. +The requested resources will be populated to achieve a 1 pod per node packing +while still leaving some headroom for required system pods. + +This functionality can be overridden by specifying the desired cpu requirement +using the `requested_cpu_per_pod` setting. + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.2 | +| [local](#requirement\_local) | >= 2.0.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allocatable\_cpu\_per\_node](#input\_allocatable\_cpu\_per\_node) | The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | +| [allocatable\_gpu\_per\_node](#input\_allocatable\_gpu\_per\_node) | The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | +| [backoff\_limit](#input\_backoff\_limit) | Controls the number of retries before considering a Job as failed. Set to zero for shared fate. | `number` | `0` | no | +| [command](#input\_command) | The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes. | `list(string)` | 
[
  "hostname"
]
| no | +| [completion\_mode](#input\_completion\_mode) | Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information | `string` | `"Indexed"` | no | +| [ephemeral\_volumes](#input\_ephemeral\_volumes) | Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB. | 
list(object({
    type       = string
    mount_path = string
    size_gb    = number
  }))
| `[]` | no | +| [has\_gpu](#input\_has\_gpu) | Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module. | `list(bool)` | 
[
  false
]
| no | +| [image](#input\_image) | The container image the job should use. | `string` | `"debian"` | no | +| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account to run the job as. If null then no service account is specified. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the GKE job template. Key-value pairs. | `map(string)` | n/a | yes | +| [machine\_family](#input\_machine\_family) | The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria. | `string` | `null` | no | +| [name](#input\_name) | The name of the job. | `string` | `"my-job"` | no | +| [node\_count](#input\_node\_count) | How many nodes the job should run in parallel. | `number` | `1` | no | +| [node\_pool\_names](#input\_node\_pool\_names) | A list of node pool names on which to run the job. Can be populated via `use` field. | `list(string)` | `[]` | no | +| [node\_selectors](#input\_node\_selectors) | A list of node selectors to use to place the job. | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | +| [persistent\_volume\_claims](#input\_persistent\_volume\_claims) | A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module. | 
list(object({
    name          = string
    namespace     = string
    mount_path    = string
    mount_options = string
    storage_type  = string
  }))
| `[]` | no | +| [random\_name\_sufix](#input\_random\_name\_sufix) | Appends a random suffix to the job name to avoid clashes. | `bool` | `true` | no | +| [requested\_cpu\_per\_pod](#input\_requested\_cpu\_per\_pod) | The requested cpu per pod. If null, allocatable\_cpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_cpu\_per\_node. | `number` | `-1` | no | +| [requested\_gpu\_per\_pod](#input\_requested\_gpu\_per\_pod) | The requested gpu per pod. If null, allocatable\_gpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_gpu\_per\_node. | `number` | `-1` | no | +| [restart\_policy](#input\_restart\_policy) | Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed. | `string` | `"Never"` | no | +| [security\_context](#input\_security\_context) | The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | +| [tolerations](#input\_tolerations) | Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field. | 
list(object({
    key      = string
    operator = string
    value    = string
    effect   = string
  }))
| 
[
  {
    "effect": "NoSchedule",
    "key": "user-workload",
    "operator": "Equal",
    "value": "true"
  }
]
| no | +| [tpu\_accelerator\_type](#input\_tpu\_accelerator\_type) | The TPU accelerator type label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | +| [tpu\_chips\_per\_node](#input\_tpu\_chips\_per\_node) | The number of TPU chips per node. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | +| [tpu\_topology](#input\_tpu\_topology) | The TPU topology label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for submitting the GKE job. | + diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/main.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/main.tf new file mode 100644 index 0000000000..e84138bb3f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/main.tf @@ -0,0 +1,181 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-job-template", ghpc_role = "compute" }) +} + +locals { + tpu_accelerator_node_selector = var.tpu_accelerator_type[0] != null ? [{ + key = "cloud.google.com/gke-tpu-accelerator" + value = var.tpu_accelerator_type[0] + }] : [] + + tpu_topology_node_selector = var.tpu_topology[0] != null ? [{ + key = "cloud.google.com/gke-tpu-topology" + value = var.tpu_topology[0] + }] : [] +} + +locals { + # Start with the minimum cpu available of used node pools + min_allocatable_cpu = min(var.allocatable_cpu_per_node...) + full_node_cpu_request = ( + local.min_allocatable_cpu > 2 ? # if large enough + local.min_allocatable_cpu - 1 : # leave headroom for 1 cpu + local.min_allocatable_cpu / 2 + 0.1 # else take just over half + ) - (local.any_gcs ? 0.25 : 0) # save room for gcs side car + + cpu_request = ( + var.requested_cpu_per_pod >= 0 ? # if user supplied requested cpu + var.requested_cpu_per_pod : # then honor it + ( # else + local.min_allocatable_cpu >= 0 ? # if allocatable cpu was supplied + local.full_node_cpu_request : # then claim the full node + -1 # else do not set a limit + ) + ) + millicpu = floor(local.cpu_request * 1000) + cpu_request_string = local.millicpu >= 0 ? "${local.millicpu}m" : null + full_node_request = local.min_allocatable_cpu >= 0 && var.requested_cpu_per_pod < 0 + + memory_request_value = try(sum([for ed in var.ephemeral_volumes : + ed.size_gb + if ed.type == "memory" + ]), 0) + memory_request_string = local.memory_request_value > 0 ? "${local.memory_request_value}Gi" : null + + ephemeral_request_value = try(sum([for ed in var.ephemeral_volumes : + ed.size_gb + if ed.type == "local-ssd" + ]), 0) + ephemeral_request_string = local.ephemeral_request_value > 0 ? "${local.ephemeral_request_value}Gi" : null + + uses_local_ssd = anytrue([for ed in var.ephemeral_volumes : + ed.type == "local-ssd" + ]) + local_ssd_node_selector = local.uses_local_ssd ? [{ + key = "cloud.google.com/gke-ephemeral-storage-local-ssd" + value = "true" + }] : [] + + # Setup limit for GPUs per pod + min_allocatable_gpu = min(var.allocatable_gpu_per_node...) + min_allocatable_gpu_per_pod = local.min_allocatable_gpu > 0 ? local.min_allocatable_gpu : null + gpu_limit_per_pod = var.requested_gpu_per_pod > 0 ? var.requested_gpu_per_pod : local.min_allocatable_gpu_per_pod + gpu_limit_string = alltrue(var.has_gpu) ? tostring(local.gpu_limit_per_pod) : null + + empty_dir_volumes = [for ed in var.ephemeral_volumes : + { + name = replace(trim(ed.mount_path, "/"), "/", "-") + mount_path = ed.mount_path + size_limit = "${ed.size_gb}Gi" + in_memory = ed.type == "memory" + } + if contains(["memory", "local-ssd"], ed.type) + ] + + ephemeral_pd_volumes = [for pd in var.ephemeral_volumes : + { + name = replace(trim(pd.mount_path, "/"), "/", "-") + mount_path = pd.mount_path + storage_class_name = pd.type == "pd-ssd" ? "premium-rwo" : "standard-rwo" + storage = "${pd.size_gb}Gi" + } + if contains(["pd-balanced", "pd-ssd"], pd.type) + ] + + pvc_volumes = [for pvc in var.persistent_volume_claims : + { + name = replace(trim(pvc.mount_path, "/"), "/", "-") + mount_path = pvc.mount_path + claim_name = pvc.name + } + ] + + volume_mounts = [for v in concat(local.empty_dir_volumes, local.ephemeral_pd_volumes, local.pvc_volumes) : + { + name = v.name + mount_path = v.mount_path + } + ] + + suffix = var.random_name_sufix ? "-${random_id.resource_name_suffix.hex}" : "" + machine_family_node_selector = var.machine_family != null ? [{ + key = "cloud.google.com/machine-family" + value = var.machine_family + }] : [] + node_selectors = concat(local.machine_family_node_selector, local.local_ssd_node_selector, local.tpu_accelerator_node_selector, local.tpu_topology_node_selector, var.node_selectors) + + any_gcs = anytrue([for pvc in var.persistent_volume_claims : + pvc.storage_type == "gcs" + ]) + + job_template_contents = templatefile( + "${path.module}/templates/gke-job-base.yaml.tftpl", + { + name = var.name + suffix = local.suffix + image = var.image + command = var.command + node_count = var.node_count + completion_mode = var.completion_mode + k8s_service_account_name = var.k8s_service_account_name + node_pool_names = var.node_pool_names + node_selectors = local.node_selectors + tpu_limit = var.tpu_chips_per_node[0] + full_node_request = local.full_node_request + cpu_request = local.cpu_request_string + gpu_limit = local.gpu_limit_string + restart_policy = var.restart_policy + backoff_limit = var.backoff_limit + tolerations = distinct(var.tolerations) + security_context = var.security_context + labels = local.labels + + empty_dir_volumes = local.empty_dir_volumes + ephemeral_pd_volumes = local.ephemeral_pd_volumes + pvc_volumes = local.pvc_volumes + volume_mounts = local.volume_mounts + memory_request = local.memory_request_string + ephemeral_request = local.ephemeral_request_string + gcs_annotation = local.any_gcs + } + ) + + job_template_output_path = "${path.root}/${var.name}${local.suffix}.yaml" + +} + +resource "random_id" "resource_name_suffix" { + byte_length = 2 + keepers = { + timestamp = timestamp() + } +} + +resource "local_file" "job_template" { + content = local.job_template_contents + filename = local.job_template_output_path + + lifecycle { + precondition { + condition = local.any_gcs ? var.k8s_service_account_name != null : true + error_message = "When using GCS, a kubernetes service account with workload identity is required. gke-cluster module will perform this setup when var.configure_workload_identity_sa is set to true." + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/outputs.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/outputs.tf new file mode 100644 index 0000000000..adf78e936d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instructions" { + description = "Instructions for submitting the GKE job." + value = <<-EOT + A GKE job file has been created locally at: + ${abspath(local.job_template_output_path)} + + Use the following commands to: + Submit your job: + kubectl create -f ${abspath(local.job_template_output_path)} + EOT +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl new file mode 100644 index 0000000000..11df39ce2c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl @@ -0,0 +1,128 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ${name}${suffix} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + parallelism: ${node_count} + completions: ${node_count} + completionMode: ${completion_mode} + template: + %{~ if gcs_annotation ~} + metadata: + annotations: + gke-gcsfuse/volumes: "true" + %{~ endif ~} + spec: + %{~ if length(security_context) > 0 ~} + securityContext: + %{~ for context in security_context ~} + ${context.key}: ${context.value} + %{~ endfor ~} + %{~ endif ~} + %{~ if k8s_service_account_name != null ~} + serviceAccountName: ${k8s_service_account_name} + %{~ endif ~} + %{~ if length(node_pool_names) > 0 ~} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + %{~ for node_pool in node_pool_names ~} + - ${node_pool} + %{~ endfor ~} + %{~ endif ~} + %{~ if length(node_selectors) > 0 ~} + nodeSelector: + %{~ for selector in node_selectors ~} + ${selector.key}: "${selector.value}" + %{~ endfor ~} + %{~ endif ~} + tolerations: + %{~ for toleration in tolerations ~} + - key: ${toleration.key} + operator: ${toleration.operator} + value: "${toleration.value}" + effect: ${toleration.effect} + %{~ endfor ~} + containers: + - name: ${name}-container + image: ${image} + command: + %{for s in command}- ${indent(8, yamlencode(s))}%{~ endfor } + %{~ if gpu_limit != null || cpu_request != null || tpu_limit != null ~} + resources: + %{~ if gpu_limit != null || tpu_limit != null ~} + limits: + %{~ if gpu_limit != null ~} + # GPUs should only be specified as limits + # https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/ + nvidia.com/gpu: ${gpu_limit} + %{~ endif ~} + %{~ if tpu_limit != null ~} + google.com/tpu: ${tpu_limit} + %{~ endif ~} + %{~ endif ~} + %{~ if cpu_request != null || memory_request != null || ephemeral_request != null || tpu_limit != null ~} + requests: + %{~ if full_node_request ~} + # cpu request attempts full node per pod + %{~ endif ~} + %{~ if cpu_request != null ~} + cpu: ${cpu_request} + %{~ endif ~} + %{~ if tpu_limit != null ~} + google.com/tpu: ${tpu_limit} + %{~ endif ~} + %{~ if memory_request != null ~} + memory: ${memory_request} + %{~ endif ~} + %{~ if ephemeral_request != null ~} + ephemeral-storage: ${ephemeral_request} + %{~ endif ~} + %{~ endif ~} + %{~ endif ~} + %{~ if length(volume_mounts) > 0 ~} + volumeMounts: + %{~ for v in volume_mounts ~} + - name: ${v.name} + mountPath: ${v.mount_path} + %{~ endfor ~} + %{~ endif ~} + %{~ if length(volume_mounts) > 0 ~} + volumes: + %{~ for ed in empty_dir_volumes ~} + - name: ${ed.name} + emptyDir: + sizeLimit: ${ed.size_limit} + %{~ if ed.in_memory ~} + medium: "Memory" + %{~ endif ~} + %{~ endfor ~} + %{~ for pd in ephemeral_pd_volumes ~} + - name: ${pd.name} + ephemeral: + volumeClaimTemplate: + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: ${pd.storage_class_name} + resources: + requests: + storage: ${pd.storage} + %{~ endfor ~} + %{~ for pvc in pvc_volumes ~} + - name: ${pvc.name} + persistentVolumeClaim: + claimName: ${pvc.claim_name} + %{~ endfor ~} + %{~ endif ~} + restartPolicy: ${restart_policy} + backoffLimit: ${backoff_limit} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/variables.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/variables.tf new file mode 100644 index 0000000000..fd83f2b692 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/variables.tf @@ -0,0 +1,206 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "name" { + description = "The name of the job." + type = string + default = "my-job" +} + +variable "node_count" { + description = "How many nodes the job should run in parallel." + type = number + default = 1 +} + +variable "completion_mode" { + description = "Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information" + type = string + default = "Indexed" +} + +variable "command" { + description = "The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes." + type = list(string) + default = ["hostname"] +} + +variable "image" { + description = "The container image the job should use." + type = string + default = "debian" +} + +variable "k8s_service_account_name" { + description = "Kubernetes service account to run the job as. If null then no service account is specified." + type = string + default = null +} + +variable "node_pool_names" { + description = "A list of node pool names on which to run the job. Can be populated via `use` field." + type = list(string) + default = [] +} + +variable "allocatable_cpu_per_node" { + description = "The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." + type = list(number) + default = [-1] +} + +variable "has_gpu" { + description = "Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module." + type = list(bool) + default = [false] +} + +variable "requested_cpu_per_pod" { + description = "The requested cpu per pod. If null, allocatable_cpu_per_node will be used to claim whole nodes. If provided will override allocatable_cpu_per_node." + type = number + default = -1 +} + +variable "allocatable_gpu_per_node" { + description = "The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." + type = list(number) + default = [-1] +} + +variable "requested_gpu_per_pod" { + description = "The requested gpu per pod. If null, allocatable_gpu_per_node will be used to claim whole nodes. If provided will override allocatable_gpu_per_node." + type = number + default = -1 +} + +variable "tolerations" { + description = "Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field." + type = list(object({ + key = string + operator = string + value = string + effect = string + })) + default = [ + { + key = "user-workload" + operator = "Equal" + value = "true" + effect = "NoSchedule" + } + ] +} + +variable "security_context" { + description = "The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/" + type = list(object({ + key = string + value = string + })) + default = [] +} + +variable "machine_family" { + description = "The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria." + type = string + default = null +} + +variable "node_selectors" { + description = "A list of node selectors to use to place the job." + type = list(object({ + key = string + value = string + })) + default = [] +} + +variable "restart_policy" { + description = "Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed." + type = string + default = "Never" +} + +variable "backoff_limit" { + description = "Controls the number of retries before considering a Job as failed. Set to zero for shared fate." + type = number + default = 0 +} + +variable "random_name_sufix" { + description = "Appends a random suffix to the job name to avoid clashes." + type = bool + default = true +} + +variable "persistent_volume_claims" { + description = "A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module." + type = list(object({ + name = string + namespace = string + mount_path = string + mount_options = string + storage_type = string + })) + default = [] +} + +variable "ephemeral_volumes" { + description = "Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB." + type = list(object({ + type = string + mount_path = string + size_gb = number + })) + default = [] + validation { + condition = alltrue([ + for v in var.ephemeral_volumes : + contains(["pd-balanced", "pd-ssd", "memory", "local-ssd"], v.type) + ]) + error_message = "Type must be one of 'pd-balanced', 'pd-ssd', 'memory', 'local-ssd'." + } + validation { + condition = alltrue([ + for v in var.ephemeral_volumes : + substr(v.mount_path, 0, 1) == "/" + ]) + error_message = "Mount path must start with the '/' character." + } +} + +variable "labels" { + description = "Labels to add to the GKE job template. Key-value pairs." + type = map(string) +} + +variable "tpu_accelerator_type" { + description = "The TPU accelerator type label. Populated from gke-node-pool via `use` field." + type = list(string) + default = [null] +} + +variable "tpu_topology" { + description = "The TPU topology label. Populated from gke-node-pool via `use` field." + type = list(string) + default = [null] +} + +variable "tpu_chips_per_node" { + description = "The number of TPU chips per node. Populated from gke-node-pool via `use` field." + type = list(string) + default = [null] +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/versions.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/versions.tf new file mode 100644 index 0000000000..0f902ac8c5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/versions.tf @@ -0,0 +1,28 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.2" + + required_providers { + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/README.md b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/README.md new file mode 100644 index 0000000000..b25d905252 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/README.md @@ -0,0 +1,388 @@ +## Description + +This module creates a Google Kubernetes Engine +([GKE](https://cloud.google.com/kubernetes-engine)) node pool. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a GKE node group. + +```yaml + - id: compute_pool + source: modules/compute/gke-node-pool + use: [gke_cluster] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### Taints and Tolerations + +By default node pools created with this module will be tainted with +`user-workload=true:NoSchedule` to prevent system pods from being scheduled. +User jobs targeting the node pool should include this toleration. This behavior +can be overridden using the `taints` setting. See +[docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-taints) for +more info. + +### Local SSD Storage +GKE offers two options for managing locally attached SSDs. + +The first, and recommended, option is for GKE to manage the ephemeral storage +space on the node, which will then be automatically attached to pods which +request an `emptyDir` volume. This can be accomplished using the +[`local_ssd_count_ephemeral_storage`] variable. + +The second, more complex, option is for GCP to attach these nodes as raw block +storage. In this case, the cluster administrator is responsible for software +RAID settings, partitioning, formatting and mounting these disks on the host +OS. Still, this may be desired behavior in use cases which aren't supported +by an `emptyDir` volume (for example, a `ReadOnlyMany` or `ReadWriteMany` PV). +This can be accomplished using the [`local_ssd_count_nvme_block`] variable. + +The [`local_ssd_count_ephemeral_storage`] and [`local_ssd_count_nvme_block`] +variables are mutually exclusive and cannot be mixed together. + +Also, the number of SSDs which can be attached to a node depends on the +[machine type](https://cloud.google.com/compute/docs/disks#local_ssd_machine_type_restrictions). + +See [docs](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd) +for more info. + +[`local_ssd_count_ephemeral_storage`]: #input\_local\_ssd\_count\_ephemeral\_storage +[`local_ssd_count_nvme_block`]: #input\_local\_ssd\_count\_nvme\_block + +### Considerations with GPUs + +When a GPU is attached to a node an additional taint is automatically added: +`nvidia.com/gpu=present:NoSchedule`. For jobs to get placed on these nodes, the +equivalent toleration is required. The `gke-job-template` module will +automatically apply this toleration when using a node pool with GPUs. + +Nvidia GPU drivers must be installed. The recommended approach for GKE to install +GPU dirvers is by applying a DaemonSet to the cluster. See +[these instructions](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus#cos). + +However, in some cases it may be desired to compile a different driver (such as +a desire to install a newer version, compatibility with the +[Nvidia GPU-operator](https://github.com/NVIDIA/gpu-operator) or other +use-cases). In this case, ensure that you turn off the +[enable_secure_boot](#input\_enable\_secure\_boot) option to allow unsigned +kernel modules to be loaded. + +#### Maximize GPU network bandwidth with GPUDirect and multi-networking +For A3 Series machines to achieve optimal performance , GKE provide two networking stacks for remote direct memory access (RDMA): + +- A3 High machine types (a3-highgpu-8g): utilize GPUDirect-TCPX to reduce the overhead required to transfer packet payloads to and from GPUs, which significantly improves throughput at scale compared to GPUs that don't use GPUDirect. +- A3 Mega machine types (a3-megagpu-8g): utilize GPUDirect-TCPXO to improve GPU to GPU communication, and further improves GPU to VM communication. + +To achieve this, when creating nodepools with A3 Series machine type, pass in a multivpc module to the gke-node-pool module, and the gke-node-pool module would detect the eligible machine type and enable GPUDirect for it. More specifically, the below components will be installed in the nodepool for enabling GPUDirect. + +- Install NCCL plugin for GPUDirect [TCPX](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpx) or [TCPXO](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpxo) +- Install [NRI](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nri_device_injector) device injector plugin +- Provide support for injecting GPUDirect required components(annotations, volumes, rxdm sidecar etc.) into the user workload in the form of Kubernetes Job. + - Provide sample workload to showcase how it will be updated with the required components injected, and how it can be deployed. + - Allow user to use the provided script to update their own workload and deploy. + +The GPUDirect supports included in the Cluster Toolkit aim to automate the [GPUDirect User Guid](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#install-gpudirect-tcpx-nccl) and provide better usability. + +> **_NOTE:_** You must [enable multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) feature when creating the GKE cluster. When gke-cluster depends on multivpc (with the use keyword), multi networking will be automatically enabled on the cluster creation. +> When gke-cluster or pre-existing-gke-cluster depends on multivpc (with the use keyword), the [network objects](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment) required for multi networking will be created on the cluster. + +### GPUs Examples + +There are several ways to add GPUs to a GKE node pool. See +[docs](https://cloud.google.com/compute/docs/gpus) for more info on GPUs. + +The following is a node pool that uses `a2`, `a3` or `g2` machine types which has a +fixed number of attached GPUs, let's call these machine types as "pre-defined gpu machine families": + +```yaml + - id: simple-a2-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g +``` + +> **Note**: It is not necessary to define the [`guest_accelerator`] setting when +> using pre-defined gpu machine families as information about GPUs, such as type, count and +> `gpu_driver_installation_config`, is automatically inferred from the machine type. +> Optional fields such as `gpu_partition_size` need to be specified only if they have +> non-default values. + +The following scenarios require the [`guest_accelerator`] block is specified: + +- To partition an A100 GPU into multiple GPUs on an A2 family machine. +- To specify a time sharing configuration on a GPUs. +- To attach a GPU to an N1 family machine. + +The following is an example of +[partitioning](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus-multi) +an A100 GPU: + +> **Note**: In the following example, `type`, `count` and `gpu_driver_installation_config` are picked up automatically. + +```yaml + - id: multi-instance-gpu-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g + guest_accelerator: + - gpu_partition_size: 1g.5gb +``` + +[`guest_accelerator`]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_guest_accelerator + +The following is an example of +[GPU time sharing](https://cloud.google.com/kubernetes-engine/docs/concepts/timesharing-gpus) +(with partitioned GPUs): + +```yaml + - id: time-sharing-gpu-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g + guest_accelerator: + - gpu_partition_size: 1g.5gb + gpu_sharing_config: + gpu_sharing_strategy: TIME_SHARING + max_shared_clients_per_gpu: 3 +``` + +Following is an example of using a GPU attached to an `n1` machine: + +```yaml + - id: t4-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: n1-standard-16 + guest_accelerator: + - type: nvidia-tesla-t4 + count: 2 +``` + +The following is an example of using a GPU (with sharing config) attached to an `n1` machine: + +```yaml + - id: n1-t4-pool + source: community/modules/compute/gke-node-pool + use: [gke_cluster] + settings: + name: n1-t4-pool + machine_type: n1-standard-1 + guest_accelerator: + - type: nvidia-tesla-t4 + count: 2 + gpu_driver_installation_config: + gpu_driver_version: "LATEST" + gpu_sharing_config: + max_shared_clients_per_gpu: 2 + gpu_sharing_strategy: "TIME_SHARING" +``` + +Finally, the following is adding multivpc to a node pool: + +```yaml + - id: network + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: gke-cluster + source: modules/scheduler/gke-cluster + use: [network, multinetwork] + settings: + cluster_name: $(vars.deployment_name) + + - id: a3-megagpu_pool + source: modules/compute/gke-node-pool + use: [gke-cluster, multinetwork] + settings: + machine_type: a3-megagpu-8g + ... +``` + +## Using GCE Reservations +You can reserve Google Compute Engine instances in a specific zone to ensure resources are available for their workloads when needed. For more details on how to manage reservations, see [Reserving Compute Engine zonal resources](https://cloud.google.com/compute/docs/instances/reserving-zonal-resources). + +After creating a reservation, you can consume the reserved GCE VM instances in GKE. GKE clusters deployed using Cluster Toolkit support the same consumption modes as Compute Engine: NO_RESERVATION(default), ANY_RESERVATION, SPECIFIC_RESERVATION. + +This can be accomplished using [`reservation_affinity`](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/compute/gke-node-pool/README.md#input_reservation_affinity). + +```yaml +# Target any reservation +reservation_affinity: + consume_reservation_type: ANY_RESERVATION + +# Target a specific reservation +reservation_affinity: + consume_reservation_type: SPECIFIC_RESERVATION + specific_reservations: + - name: specific-reservation-1 +``` + +The following requirements need to be satisfied for the node pool nodes to be able to use a specific reservation: +1. A reservation with the name must exist in the specified project(`var.project_id`) and one of the specified zones(`var.zones`). +2. Its consumption type must be `specific`. +3. Its GCE VM Properties must match with those of the Node Pool; Machine type, Accelerators (GPU Type and count), Local SSD disk type and count. + +If you want to utilise a shared reservation, the owner project of the shared reservation needs to be explicitly specified like the following. Note that a shared reservation can be used by the project that hosts the reservation (owner project) and by the projects the reservation is shared with (consumer projects). See how to [create and use a shared reservation](https://cloud.google.com/compute/docs/instances/reservations-shared). + +```yaml +reservation_affinity: + consume_reservation_type: SPECIFIC_RESERVATION + specific_reservations: + - name: specific-reservation-shared + project: shared_reservation_owner_project_id +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 7.2 | +| [google-beta](#requirement\_google-beta) | >= 7.2 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 7.2 | +| [google-beta](#provider\_google-beta) | >= 7.2 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | +| [tpu](#module\_tpu) | ../../internal/tpu-definition | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_container_node_pool.node_pool](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | +| [null_resource.enable_tcpx_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.enable_tcpxo_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.install_dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_machine_types.machine_info](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | +| [google_compute_region_instance_template.instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_template) | data source | +| [google_compute_reservation.specific_reservations](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [auto\_repair](#input\_auto\_repair) | Whether the nodes will be automatically repaired. | `bool` | `true` | no | +| [auto\_upgrade](#input\_auto\_upgrade) | Whether the nodes will be automatically upgraded. | `bool` | `false` | no | +| [autoscaling\_total\_max\_nodes](#input\_autoscaling\_total\_max\_nodes) | Total maximum number of nodes in the NodePool. | `number` | `1000` | no | +| [autoscaling\_total\_min\_nodes](#input\_autoscaling\_total\_min\_nodes) | Total minimum number of nodes in the NodePool. | `number` | `0` | no | +| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | +| [compact\_placement](#input\_compact\_placement) | DEPRECATED: Use `placement_policy` | `bool` | `null` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for each node. | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Disk type for each node. | `string` | `null` | no | +| [enable\_flex\_start](#input\_enable\_flex\_start) | If true, start the node pool with Flex Start provisioning model.
To learn more about flex-start mode, please refer to
https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and
https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest | `bool` | `false` | no | +| [enable\_gcfs](#input\_enable\_gcfs) | Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config). | `bool` | `false` | no | +| [enable\_numa\_aware\_scheduling](#input\_enable\_numa\_aware\_scheduling) | Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling. | `bool` | `false` | no | +| [enable\_private\_nodes](#input\_enable\_private\_nodes) | Whether nodes have internal IP addresses only. | `bool` | `true` | no | +| [enable\_queued\_provisioning](#input\_enable\_queued\_provisioning) | If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool. | `bool` | `false` | no | +| [enable\_secure\_boot](#input\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | +| [gke\_version](#input\_gke\_version) | GKE version | `string` | n/a | yes | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = optional(string)
    count = optional(number, 0)
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | +| [host\_maintenance\_interval](#input\_host\_maintenance\_interval) | Specifies the frequency of planned maintenance events. | `string` | `""` | no | +| [image\_type](#input\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | +| [initial\_node\_count](#input\_initial\_node\_count) | The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static\_node\_count and must be set to a value between autoscaling\_total\_min\_nodes and autoscaling\_total\_max\_nodes. | `number` | `null` | no | +| [internal\_ghpc\_module\_id](#input\_internal\_ghpc\_module\_id) | DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide). | `string` | n/a | yes | +| [is\_reservation\_active](#input\_is\_reservation\_active) | Whether the specified reservation is already created. | `bool` | `true` | no | +| [kubernetes\_labels](#input\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_count\_ephemeral\_storage](#input\_local\_ssd\_count\_ephemeral\_storage) | The number of local SSDs to attach to each node to back ephemeral storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | +| [local\_ssd\_count\_nvme\_block](#input\_local\_ssd\_count\_nvme\_block) | The number of local SSDs to attach to each node to back block storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | +| [machine\_type](#input\_machine\_type) | The name of a Google Compute Engine machine type. | `string` | `"c2-standard-60"` | no | +| [max\_pods\_per\_node](#input\_max\_pods\_per\_node) | The maximum number of pods per node in this node pool. This will force replacement. | `number` | `null` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [name](#input\_name) | The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix.
If setting manually, ensure a unique value across all gke-node-pools. | `string` | `null` | no | +| [num\_node\_pools](#input\_num\_node\_pools) | Number of node pools to create. This is same as num\_slices. | `number` | `1` | no | +| [num\_slices](#input\_num\_slices) | Number of TPUs slices to create. This is same as num\_node\_pools. | `number` | `1` | no | +| [placement\_policy](#input\_placement\_policy) | Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| 
{
  "name": null,
  "tpu_topology": null,
  "type": null
}
| no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Reservation resource to consume. When targeting SPECIFIC\_RESERVATION, specific\_reservations needs be specified.
Even though specific\_reservations is a list, only one reservation is allowed by the NodePool API.
It is assumed that the specified reservation exists and has available capacity.
For a shared reservation, specify the project\_id as well in which it was created.
To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared | 
object({
    consume_reservation_type = string
    specific_reservations = optional(list(object({
      name    = string
      project = optional(string)
    })))
  })
| 
{
  "consume_reservation_type": "NO_RESERVATION",
  "specific_reservations": []
}
| no | +| [run\_workload\_script](#input\_run\_workload\_script) | Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only. | `bool` | `true` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [static\_node\_count](#input\_static\_node\_count) | The static number of nodes in the node pool. If set, autoscaling will be disabled. | `number` | `null` | no | +| [taints](#input\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | +| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | +| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | +| [total\_max\_nodes](#input\_total\_max\_nodes) | DEPRECATED: Use autoscaling\_total\_max\_nodes. | `number` | `null` | no | +| [total\_min\_nodes](#input\_total\_min\_nodes) | DEPRECATED: Use autoscaling\_total\_min\_nodes. | `number` | `null` | no | +| [upgrade\_settings](#input\_upgrade\_settings) | Defines node pool upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | +| [zones](#input\_zones) | A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable. | `list(string)` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [allocatable\_cpu\_per\_node](#output\_allocatable\_cpu\_per\_node) | Number of CPUs available for scheduling pods on each node. | +| [allocatable\_gpu\_per\_node](#output\_allocatable\_gpu\_per\_node) | Number of GPUs available for scheduling pods on each node. | +| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | +| [guest\_accelerator](#output\_guest\_accelerator) | The accelerator type of the nodes. | +| [has\_gpu](#output\_has\_gpu) | Boolean value indicating whether nodes in the pool are configured with GPUs. | +| [instance\_templates](#output\_instance\_templates) | The URLs of Instance Templates | +| [instructions](#output\_instructions) | Instructions for submitting the sample GPUDirect enabled job. | +| [machine\_type](#output\_machine\_type) | Machine Type | +| [node\_count\_static](#output\_node\_count\_static) | The number of static nodes in node-pool. | +| [node\_pool\_names](#output\_node\_pool\_names) | Names of the node pools. | +| [static\_gpu\_count](#output\_static\_gpu\_count) | Total number of GPUs in the node pool. Available only for static node pools. | +| [tolerations](#output\_tolerations) | Tolerations needed for a pod to be scheduled on this node pool. | +| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | +| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | +| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | + diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf new file mode 100644 index 0000000000..0c1c255255 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +## Required variables: +# local_ssd_count_ephemeral_storage +# local_ssd_count_nvme_block +# machine_type + +locals { + + local_ssd_machines = { + "a3-highgpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, + "a3-megagpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, + "a3-ultragpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, + "a4-highgpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, + } + + generated_local_ssd_config = lookup(local.local_ssd_machines, var.machine_type, { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = null }) + + # Select in priority order: + # (1) var.local_ssd_count_ephemeral_storage and var.local_ssd_count_nvme_block if any is not null + # (2) local.local_ssd_machines if not empty + # (3) default to null value for both local_ssd_count_ephemeral_storage and local_ssd_count_nvme_block + local_ssd_config = (var.local_ssd_count_ephemeral_storage == null && var.local_ssd_count_nvme_block == null) ? local.generated_local_ssd_config : { local_ssd_count_ephemeral_storage = var.local_ssd_count_ephemeral_storage, local_ssd_count_nvme_block = var.local_ssd_count_nvme_block } +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml new file mode 100644 index 0000000000..1106f63479 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml @@ -0,0 +1,50 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: my-sample-job +spec: + parallelism: 2 + completions: 2 + completionMode: Indexed + template: + spec: + containers: + - name: nccl-test + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx-dev:v3.1.9 + imagePullPolicy: Always + command: + - /bin/sh + - -c + - | + service ssh restart; + sleep infinity; + env: + - name: LD_LIBRARY_PATH + value: /usr/local/nvidia/lib64 + volumeMounts: + - name: config-volume + mountPath: /configs + resources: + limits: + nvidia.com/gpu: 8 + volumes: + - name: config-volume + configMap: + name: nccl-configmap + defaultMode: 0777 + restartPolicy: Never + backoffLimit: 0 diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml new file mode 100644 index 0000000000..bce6720681 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml @@ -0,0 +1,70 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: my-sample-job +spec: + parallelism: 2 + completions: 2 + completionMode: Indexed + template: + spec: + hostname: host1 + subdomain: nccl-host-1 + containers: + - name: nccl-test + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.14 + imagePullPolicy: Always + command: + - /bin/sh + - -c + - | + set -ex + chmod 755 /scripts/demo-run-nccl-test-tcpxo-via-mpi.sh + cat >/scripts/allgather.sh < 0: + container["env"].extend(env_vars) + container["volumeMounts"].extend(volume_mounts) + +if __name__ == "__main__": + main() diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py new file mode 100644 index 0000000000..db9fb3e7ff --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py @@ -0,0 +1,186 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +import argparse +import os + +def main(): + parser = argparse.ArgumentParser(description="TCPXO Job Manifest Generator") + parser.add_argument("-f", "--file", required=True, help="Path to your job template YAML file") + parser.add_argument("-r", "--rxdm", required=True, help="RxDM version") + + args = parser.parse_args() + + # Get the YAML file from the user + if not args.file: + args.file = input("Please provide the path to your job template YAML file: ") + + # Get component versions from user + if not args.rxdm: + args.rxdm = input("Enter the RxDM version: ") + + # Load and modify the YAML + with open(args.file, "r") as file: + job_manifest = yaml.load(file, Loader=yaml.BaseLoader) + + # Update annotations + add_annotations(job_manifest) + + # Update volumes + add_volumes(job_manifest) + + # Update tolerations + add_tolerations(job_manifest) + + # Add tcpxo-daemon container + add_tcpxo_daemon_container(job_manifest, args.rxdm) + + # Update environment variables and volumeMounts for GPU containers + update_gpu_containers(job_manifest) + + # Generate the new YAML file + updated_job = str(yaml.dump(job_manifest, default_flow_style=False, width=1000, default_style="|", sort_keys=False)).replace("|-", "") + + new_file_name = args.file.replace(".yaml", "-tcpxo.yaml") + with open(new_file_name, "w", encoding="utf-8") as file: + file.write(updated_job) + + # Step 7: Provide instructions to the user + print("\nA new manifest has been generated and updated to have TCPXO enabled based on the provided workload") + print("It can be found in {path}".format(path=os.path.abspath(new_file_name))) + print("You can use the following commands to submit the sample job:") + print(" kubectl create -f {path}".format(path=os.path.abspath(new_file_name))) + +def add_annotations(job_manifest): + annotations = { + 'devices.gke.io/container.tcpxo-daemon':"""|+ +- path: /dev/nvidia0 +- path: /dev/nvidia1 +- path: /dev/nvidia2 +- path: /dev/nvidia3 +- path: /dev/nvidia4 +- path: /dev/nvidia5 +- path: /dev/nvidia6 +- path: /dev/nvidia7 +- path: /dev/nvidiactl +- path: /dev/nvidia-uvm +- path: /dev/dmabuf_import_helper""", + "networking.gke.io/default-interface": "eth0", + "networking.gke.io/interfaces": """| +[ + {"interfaceName":"eth0","network":"default"}, + {"interfaceName":"eth1","network":"vpc1"}, + {"interfaceName":"eth2","network":"vpc2"}, + {"interfaceName":"eth3","network":"vpc3"}, + {"interfaceName":"eth4","network":"vpc4"}, + {"interfaceName":"eth5","network":"vpc5"}, + {"interfaceName":"eth6","network":"vpc6"}, + {"interfaceName":"eth7","network":"vpc7"}, + {"interfaceName":"eth8","network":"vpc8"} +]""", + } + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("metadata", {}) + + # Add/update annotations + pod_template_spec = job_manifest["spec"]["template"]["metadata"] + if "annotations" in pod_template_spec: + pod_template_spec["annotations"].update(annotations) + else: + pod_template_spec["annotations"] = annotations + +def add_tolerations(job_manifest): + tolerations = [ + {"key": "user-workload", "operator": "Equal", "value": """\"true\"""", "effect": "NoSchedule"}, + ] + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add tolerations + pod_spec = job_manifest["spec"]["template"]["spec"] + if "tolerations" in pod_spec: + pod_spec["tolerations"].extend(tolerations) + else: + pod_spec["tolerations"] = tolerations + +def add_volumes(job_manifest): + volumes = [ + {"name": "nvidia-install-dir-host", "hostPath": {"path": "/home/kubernetes/bin/nvidia"}}, + {"name": "sys", "hostPath": {"path": "/sys"}}, + {"name": "proc-sys", "hostPath": {"path": "/proc/sys"}}, + {"name": "aperture-devices", "hostPath": {"path": "/dev/aperture_devices"}}, + ] + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add volumes + pod_spec = job_manifest["spec"]["template"]["spec"] + if "volumes" in pod_spec: + pod_spec["volumes"].extend(volumes) + else: + pod_spec["volumes"] = volumes + + +def add_tcpxo_daemon_container(job_template, rxdm_version): + tcpxo_daemon_container = { + "name": "tcpxo-daemon", + "image": f"us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:{rxdm_version}", # Use provided RxDM version + "imagePullPolicy": "Always", + "command": ["/bin/sh", "-c"], + "args": [ + """| + set -ex + chmod 755 /fts/entrypoint_rxdm_container.sh + /fts/entrypoint_rxdm_container.sh --num_hops=2 --num_nics=8 --uid= --alsologtostderr""" + ], + "securityContext": { + "capabilities": {"add": ["NET_ADMIN", "NET_BIND_SERVICE"]} + }, + "volumeMounts": [ + {"name": "nvidia-install-dir-host", "mountPath": "/usr/local/nvidia"}, + {"name": "sys", "mountPath": "/hostsysfs"}, + {"name": "proc-sys", "mountPath": "/hostprocsysfs"}, + ], + "env": [{"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}], + } + + # Create path if it doesn't exist + job_template.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add container + pod_spec = job_template["spec"]["template"]["spec"] + pod_spec.setdefault("containers", []).insert(0, tcpxo_daemon_container) + +def update_gpu_containers(job_manifest): + env_vars = [ + {"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}, + {"name": "NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY", "value": "/dev/aperture_devices"}, + ] + volume_mounts = [{"name": "aperture-devices", "mountPath": "/dev/aperture_devices"}] + + pod_spec = job_manifest.get("spec", {}).get("template", {}).get("spec", {}) + for container in pod_spec.get("containers", []): + # Create path if it doesn't exist + container.setdefault("env", []) + container.setdefault("volumeMounts", []) + if int(container.get("resources", {}).get("limits", {}).get("nvidia.com/gpu", 0)) > 0: + container["env"].extend(env_vars) + container["volumeMounts"].extend(volume_mounts) + +if __name__ == "__main__": + main() diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf new file mode 100644 index 0000000000..d23d050986 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf @@ -0,0 +1,87 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# Enable GPUDirect for A3 and A3Mega VMs, this involve multiple kubectl steps to integrate with the created cluster +# 1. Install NCCL plugin daemonset +# 2. Install NRI plugin daemonset +# 3. Update provided workload to inject rxdm sidecar and other required annotation, volume etc. +locals { + workload_path_tcpx = "${path.module}/gpu-direct-workload/sample-tcpx-workload-job.yaml" + workload_path_tcpxo = "${path.module}/gpu-direct-workload/sample-tcpxo-workload-job.yaml" + + gpu_direct_settings = { + "a3-highgpu-8g" = { + # Manifest to be installed for enabling TCPX on a3-highgpu-8g machines + gpu_direct_manifests = [ + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-tcpx-installer.yaml", # nccl_plugin v3.1.9 for tcpx + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-config.yaml", # nccl_configmap + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/nri_device_injector/nri-device-injector.yaml", # nri_plugin + ] + updated_workload_path = replace(local.workload_path_tcpx, ".yaml", "-tcpx.yaml") + rxdm_version = "v2.0.12" # matching nccl-tcpx-installer version v3.1.9 + min_additional_networks = 4 + major_minor_version_acceptable_map = { + "1.27" = "1.27.7-gke.1121000" + "1.28" = "1.28.8-gke.1095000" + "1.29" = "1.29.3-gke.1093000" + "1.30" = "1.30.2-gke.1023000" + } + } + "a3-megagpu-8g" = { + # Manifest to be installed for enabling TCPXO on a3-megagpu-8g machines + gpu_direct_manifests = [ + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/gpudirect-tcpxo/nccl-tcpxo-installer.yaml", # nccl_plugin v1.0.14 for tcpxo + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/nri_device_injector/nri-device-injector.yaml", # nri_plugin + ] + updated_workload_path = replace(local.workload_path_tcpxo, ".yaml", "-tcpxo.yaml") + rxdm_version = "v1.0.20" # matching nccl-tcpxo-installer version v1.0.14 + min_additional_networks = 8 + major_minor_version_acceptable_map = { + "1.28" = "1.28.9-gke.1250000" + "1.29" = "1.29.4-gke.1542000" + "1.30" = "1.30.4-gke.1129000" + "1.31" = "1.31.1-gke.2008000" + "1.32" = "1.32.2-gke.1489001" + } + } + } + + min_additional_networks = try(local.gpu_direct_settings[var.machine_type].min_additional_networks, 0) + + gke_version_regex = "(\\d+\\.\\d+)\\.(\\d+)-gke\\.(\\d+)" # GKE version format: 1.X.Y-gke.Z , regex output: ["1.X" , "Y", "Z"] + + gke_version_parts = regex(local.gke_version_regex, var.gke_version) + gke_version_major = local.gke_version_parts[0] + + major_minor_version_acceptable_map = try(local.gpu_direct_setting[var.machine_type].major_minor_version_acceptable_map, null) + minor_version_acceptable = try(contains(keys(local.major_minor_version_acceptable_map), local.gke_version_major), false) ? local.major_minor_version_acceptable_map[local.gke_version_major] : "1.0.0-gke.0" + minor_version_acceptable_parts = regex(local.gke_version_regex, local.minor_version_acceptable) + gke_gpudirect_compatible = local.gke_version_parts[1] > local.minor_version_acceptable_parts[1] || (local.gke_version_parts[1] == local.minor_version_acceptable_parts[1] && local.gke_version_parts[2] >= local.minor_version_acceptable_parts[2]) +} + +check "gpu_direct_check_multi_vpc" { + assert { + condition = length(var.additional_networks) >= local.min_additional_networks + error_message = "To achieve optimal performance for ${var.machine_type} machine, at least ${local.min_additional_networks} additional vpc is recommended. You could configure it in the blueprint through modules/network/multivpc with network_count set as ${local.min_additional_networks}" + } +} + +check "gke_version_requirements" { + assert { + condition = local.gke_gpudirect_compatible + error_message = "GPUDirect is not supported on GKE version ${var.gke_version} for ${var.machine_type} machine. For supported version details visit https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#requirements" + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf new file mode 100644 index 0000000000..1ddc7ba8c3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +data "google_compute_machine_types" "machine_info" { + for_each = var.zones == null ? toset([]) : toset(var.zones) + + project = var.project_id + zone = each.key + filter = "name = \"${var.machine_type}\"" +} + +locals { + valid_machine_info = { + for zone, data in data.google_compute_machine_types.machine_info : + zone => data.machine_types if length(data.machine_types) > 0 + } + + guest_cpus = try(local.valid_machine_info[0].guest_cpus, 0) +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/main.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/main.tf new file mode 100644 index 0000000000..05314497fc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/main.tf @@ -0,0 +1,482 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-node-pool", ghpc_role = "compute" }) +} + +locals { + upgrade_settings = { + strategy = var.upgrade_settings.strategy + max_surge = coalesce(var.upgrade_settings.max_surge, 0) + max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) + } +} + +module "gpu" { + source = "../../internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + has_gpu = length(local.guest_accelerator) > 0 + allocatable_gpu_per_node = local.has_gpu ? max(local.guest_accelerator[*].count...) : -1 + is_static_node_pool_with_gpus = var.static_node_count != null && local.allocatable_gpu_per_node != -1 + static_gpu_count = local.is_static_node_pool_with_gpus ? var.static_node_count * local.allocatable_gpu_per_node : 0 + gpu_taint = local.has_gpu ? [{ + key = "nvidia.com/gpu" + value = "present" + effect = "NO_SCHEDULE" + }] : [] + + autoscale_set = var.autoscaling_total_min_nodes != 0 || var.autoscaling_total_max_nodes != 1000 + static_node_set = var.static_node_count != null + initial_node_set = try(var.initial_node_count > 0, false) + + module_unique_id = replace(lower(var.internal_ghpc_module_id), "/[^a-z0-9\\-]/", "") +} + + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] +} + +module "tpu" { + source = "../../internal/tpu-definition" + + machine_type = var.machine_type + placement_policy = var.placement_policy +} + + +data "google_container_cluster" "gke_cluster" { + name = local.cluster_name + location = local.cluster_location +} + +resource "google_container_node_pool" "node_pool" { + provider = google-beta + + count = max(var.num_node_pools, var.num_slices) + + name = (max(var.num_node_pools, var.num_slices) == 1) ? coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])) : join("-", [coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])), count.index]) + cluster = var.cluster_id + node_locations = var.zones + + node_count = var.static_node_count + dynamic "autoscaling" { + for_each = local.static_node_set ? [] : [1] + content { + total_min_node_count = var.autoscaling_total_min_nodes + total_max_node_count = var.autoscaling_total_max_nodes + location_policy = "ANY" + } + } + + initial_node_count = var.initial_node_count + + max_pods_per_node = var.max_pods_per_node + + management { + auto_repair = var.auto_repair + auto_upgrade = var.auto_upgrade + } + + upgrade_settings { + strategy = local.upgrade_settings.strategy + max_surge = local.upgrade_settings.max_surge + max_unavailable = local.upgrade_settings.max_unavailable + } + + dynamic "placement_policy" { + for_each = var.placement_policy.type != null ? [1] : [] + content { + type = var.placement_policy.type + policy_name = var.placement_policy.name + tpu_topology = module.tpu.is_tpu ? var.placement_policy.tpu_topology : null + } + } + + dynamic "queued_provisioning" { + for_each = var.enable_queued_provisioning ? [1] : [] + content { + enabled = true + } + } + + node_config { + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + resource_labels = local.labels + labels = var.kubernetes_labels + service_account = var.service_account_email + oauth_scopes = var.service_account_scopes + machine_type = var.machine_type + spot = var.spot + image_type = var.image_type + flex_start = var.enable_flex_start + max_run_duration = var.max_run_duration != null ? "${var.max_run_duration}s" : null + + dynamic "guest_accelerator" { + for_each = local.guest_accelerator + iterator = ga + content { + type = coalesce(ga.value.type, try(local.generated_guest_accelerator[0].type, "")) + count = coalesce(try(ga.value.count, 0) > 0 ? ga.value.count : try(local.generated_guest_accelerator[0].count, "0")) + + gpu_partition_size = try(ga.value.gpu_partition_size, null) + + dynamic "gpu_driver_installation_config" { + # in case user did not specify guest_accelerator settings, we need a try to default to [] + for_each = try([ga.value.gpu_driver_installation_config], [{ gpu_driver_version = "DEFAULT" }]) + iterator = gdic + content { + gpu_driver_version = gdic.value.gpu_driver_version + } + } + + dynamic "gpu_sharing_config" { + for_each = try(ga.value.gpu_sharing_config == null, true) ? [] : [ga.value.gpu_sharing_config] + iterator = gsc + content { + gpu_sharing_strategy = gsc.value.gpu_sharing_strategy + max_shared_clients_per_gpu = gsc.value.max_shared_clients_per_gpu + } + } + } + } + + dynamic "taint" { + for_each = concat(var.taints, local.gpu_taint, module.tpu.tpu_taint) + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } + + dynamic "ephemeral_storage_local_ssd_config" { + for_each = local.local_ssd_config.local_ssd_count_ephemeral_storage != null ? [1] : [] + content { + local_ssd_count = local.local_ssd_config.local_ssd_count_ephemeral_storage + } + } + + dynamic "local_nvme_ssd_block_config" { + for_each = local.local_ssd_config.local_ssd_count_nvme_block != null ? [1] : [] + content { + local_ssd_count = local.local_ssd_config.local_ssd_count_nvme_block + } + } + + shielded_instance_config { + enable_secure_boot = var.enable_secure_boot + enable_integrity_monitoring = true + } + + dynamic "gcfs_config" { + for_each = var.enable_gcfs ? [1] : [] + content { + enabled = true + } + } + + gvnic { + enabled = var.image_type == "COS_CONTAINERD" + } + + dynamic "advanced_machine_features" { + for_each = local.set_threads_per_core ? [1] : [] + content { + threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf + } + } + + # Implied by Workload Identity + workload_metadata_config { + mode = "GKE_METADATA" + } + # Implied by workload identity. + metadata = { + "disable-legacy-endpoints" = "true" + } + + linux_node_config { + sysctls = { + "net.ipv4.tcp_rmem" = "4096 87380 16777216" + "net.ipv4.tcp_wmem" = "4096 16384 16777216" + } + } + + reservation_affinity { + consume_reservation_type = var.reservation_affinity.consume_reservation_type + key = local.is_valid_reservation ? local.reservation_resource_api_label : null + values = local.is_valid_reservation ? (var.is_reservation_active ? local.active_reservation_values : local.default_reservation_values) : null + } + + dynamic "host_maintenance_policy" { + for_each = var.host_maintenance_interval != "" ? [1] : [] + content { + maintenance_interval = var.host_maintenance_interval + } + } + + kubelet_config { + cpu_manager_policy = var.enable_numa_aware_scheduling ? "static" : null + dynamic "topology_manager" { + for_each = var.enable_numa_aware_scheduling ? [1] : [] + content { + policy = "restricted" + } + } + dynamic "memory_manager" { + for_each = var.enable_numa_aware_scheduling ? [1] : [] + content { + policy = "Static" + } + } + } + } + + network_config { + dynamic "additional_node_network_configs" { + for_each = var.additional_networks + + content { + network = additional_node_network_configs.value.network + subnetwork = additional_node_network_configs.value.subnetwork + } + } + + enable_private_nodes = var.enable_private_nodes + } + + timeouts { + create = var.timeout_create + update = var.timeout_update + } + + lifecycle { + ignore_changes = [ + node_config[0].labels, + initial_node_count, + # Ignore local/ephemeral ssd configs as they are tied to machine types. + node_config[0].ephemeral_storage_local_ssd_config, + node_config[0].local_nvme_ssd_block_config, + ] + precondition { + condition = (var.max_pods_per_node == null) || (data.google_container_cluster.gke_cluster.networking_mode == "VPC_NATIVE") + error_message = "max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." + } + precondition { + condition = !local.static_node_set || !local.autoscale_set + error_message = "static_node_count cannot be set with either autoscaling_total_min_nodes or autoscaling_total_max_nodes." + } + precondition { + condition = !local.static_node_set || !local.initial_node_set + error_message = "initial_node_count cannot be set with static_node_count." + } + precondition { + condition = !local.initial_node_set || (coalesce(var.initial_node_count, 0) >= var.autoscaling_total_min_nodes && coalesce(var.initial_node_count, 0) <= var.autoscaling_total_max_nodes) + error_message = "initial_node_count must be between autoscaling_total_min_nodes and autoscaling_total_max_nodes included." + } + precondition { + condition = !(coalesce(local.local_ssd_config.local_ssd_count_ephemeral_storage, 0) > 0 && coalesce(local.local_ssd_config.local_ssd_count_nvme_block, 0) > 0) + error_message = "Only one of local_ssd_count_ephemeral_storage or local_ssd_count_nvme_block can be set to a non-zero value." + } + precondition { + condition = ( + (var.reservation_affinity.consume_reservation_type != "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 0) || + (var.reservation_affinity.consume_reservation_type == "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 1) + ) + error_message = <<-EOT + When using NO_RESERVATION or ANY_RESERVATION as the `consume_reservation_type`, `specific_reservations` cannot be set. + On the other hand, with SPECIFIC_RESERVATION you must set `specific_reservations`. + EOT + } + precondition { + condition = ( + (local.input_specific_reservations_count == 0) || + ((length(local.verified_specific_reservations) == 1 || !var.is_reservation_active) && + length(local.specific_reservation_requirement_violations) == 0) + ) + error_message = <<-EOT + Check if your reservation is configured correctly: + - A reservation with the name must exist in the specified project and one of the specified zones + + - Its consumption type must be "specific" + %{for property in local.specific_reservation_requirement_violations} + - ${local.specific_reservation_requirement_violation_messages[property]} + %{endfor} + EOT + } + precondition { + condition = ( + (local.input_specific_reservations_count == 0) || + (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) == 0) || + (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) > 0 && try(local.input_reservation_projects[0], var.project_id) == var.project_id) + ) + error_message = "Shared extended reservations are not supported by GKE." + } + precondition { + condition = contains(["SURGE"], local.upgrade_settings.strategy) + error_message = "Only SURGE strategy is supported" + } + precondition { + condition = local.upgrade_settings.max_unavailable >= 0 + error_message = "max_unavailable should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_surge >= 0 + error_message = "max_surge should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 + error_message = "At least one of max_unavailable or max_surge must greater than 0" + } + precondition { + condition = var.placement_policy.type != "COMPACT" || (var.zones != null ? (length(var.zones) == 1) : false) + error_message = "Compact placement is only available for node pools operating in a single zone." + } + precondition { + condition = var.placement_policy.type != "COMPACT" || local.upgrade_settings.strategy != "BLUE_GREEN" + error_message = "Compact placement is not supported with blue-green upgrades." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.placement_policy.type == "COMPACT") + error_message = "placement_policy cannot be COMPACT when enable_queued_provisioning is true." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.reservation_affinity.consume_reservation_type != "NO_RESERVATION") + error_message = "reservation_affinity should be NO_RESERVATION when enable_queued_provisioning is true." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.autoscaling_total_min_nodes != 0) + error_message = "autoscaling_total_min_nodes should be 0 when enable_queued_provisioning is true." + } + precondition { + condition = !(var.num_node_pools > 1 && var.num_slices > 1) + error_message = "num_node_pools is for CPUs and GPUS, and num_slices is for TPUs. Both cannot be set at the same time to create a group of identical nodepools / slices." + } + precondition { + condition = !(var.num_node_pools == 0 && var.num_slices == 0) + error_message = "Either num_node_pools (for CPUs and GPUS) or num_slices (for TPUs) should be set to a positive integer value." + } + precondition { + condition = !(var.num_node_pools < 0 || var.num_slices < 0) + error_message = "Negative integer value of num_node_pools or num_slices is not valid. Please use a positive integer value to set num_node_pools for CPUs and GPUS, and num_slices for TPUs." + } + precondition { + condition = var.enable_flex_start == true ? (var.auto_repair == false) : true + error_message = "enable_flex_start needs node auto_repair set to false." + } + precondition { + condition = var.enable_flex_start == true ? (var.static_node_count == null) : true + error_message = "enable_flex_start does not work with static_node_count. static_node_count should be set to null." + } + precondition { + condition = var.enable_flex_start == true ? (var.reservation_affinity.consume_reservation_type == "NO_RESERVATION") : true + error_message = "enable_flex_start only works with reservation_affinity consume_reservation_type NO_RESERVATION." + } + precondition { + condition = var.enable_flex_start == true ? (var.spot == false) : true + error_message = "Both enable_flex_start and spot consumption option cannot be set to true at the same time." + } + } +} + +locals { + supported_machine_types_for_install_dependencies = ["a3-highgpu-8g", "a3-megagpu-8g"] +} + +# Replicates GKE's naming logic for its instance templates. The full +# pattern is "gke-{cluster_name}-{nodepool_name}-{hash}". +# +# This code builds the "{cluster_name}-{nodepool_name}" prefix, which is +# capped at 32 characters plus a dash '-' in between, by truncating names if needed: +# - If both names > 16 chars, both are cut to 16. +# - If one name > 16, it's shortened so the combined name length is 32. +data "google_compute_region_instance_template" "instance_template" { + for_each = { for idx, np in google_container_node_pool.node_pool : idx => np } + project = var.project_id + filter = "name: gke-${ + (length(local.cluster_name) <= 16 && length(each.value.name) <= 16) ? "${local.cluster_name}-${each.value.name}" : + (length(local.cluster_name) > 16 && length(each.value.name) > 16) ? "${substr(local.cluster_name, 0, 16)}-${substr(each.value.name, 0, 16)}" : + (length(local.cluster_name) > 16) ? "${substr(local.cluster_name, 0, 32 - length(each.value.name))}-${each.value.name}" : + "${local.cluster_name}-${substr(each.value.name, 0, 32 - length(local.cluster_name))}" + }*" + most_recent = true +} + +resource "null_resource" "install_dependencies" { + count = var.run_workload_script && contains(local.supported_machine_types_for_install_dependencies, var.machine_type) ? 1 : 0 + provisioner "local-exec" { + command = "pip3 install pyyaml" + } +} + +locals { + gpu_direct_setting = lookup(local.gpu_direct_settings, var.machine_type, { gpu_direct_manifests = [], updated_workload_path = "", rxdm_version = "" }) +} + +# execute script to inject rxdm sidecar into workload to enable tcpx for a3-highgpu-8g VM workload +resource "null_resource" "enable_tcpx_in_workload" { + count = var.run_workload_script && var.machine_type == "a3-highgpu-8g" ? 1 : 0 + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpx-in-workload.py --file ${local.workload_path_tcpx} --rxdm ${local.gpu_direct_setting.rxdm_version}" + } + + depends_on = [null_resource.install_dependencies] +} + +# execute script to inject rxdm sidecar into workload to enable tcpxo for a3-megagpu-8g VM workload +resource "null_resource" "enable_tcpxo_in_workload" { + count = var.run_workload_script && var.machine_type == "a3-megagpu-8g" ? 1 : 0 + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py --file ${local.workload_path_tcpxo} --rxdm ${local.gpu_direct_setting.rxdm_version}" + } + + depends_on = [null_resource.install_dependencies] +} + +# apply manifest to enable tcpx +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + apply_manifests = flatten([ + for manifest in local.gpu_direct_setting.gpu_direct_manifests : [ + { + source = manifest + } + ] + ]) +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/metadata.yaml new file mode 100644 index 0000000000..e980d595a2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com +ghpc: + inject_module_id: internal_ghpc_module_id diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/outputs.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/outputs.tf new file mode 100644 index 0000000000..44e1c3d971 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/outputs.tf @@ -0,0 +1,152 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "node_pool_names" { + description = "Names of the node pools." + value = google_container_node_pool.node_pool[*].name +} + +locals { + # Shared core machines only have 1 cpu allocatable, even if they have 2 cpu capacity + vcpu = local.machine_shared_core ? 1 : local.guest_cpus + useable_cpu = local.set_threads_per_core ? local.threads_per_core * local.vcpu / 2 : local.vcpu + + # allocatable resource definition: https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes#cpu_reservations + second_core = local.useable_cpu > 1 ? 1 : 0 + third_fourth_core = local.useable_cpu == 3 ? 1 : local.useable_cpu > 3 ? 2 : 0 + cores_above_four = local.useable_cpu > 4 ? local.useable_cpu - 4 : 0 + + allocatable_cpu = 0.94 + (0.99 * local.second_core) + (0.995 * local.third_fourth_core) + (0.9975 * local.cores_above_four) +} + +output "allocatable_cpu_per_node" { + description = "Number of CPUs available for scheduling pods on each node." + value = local.allocatable_cpu +} + +output "has_gpu" { + description = "Boolean value indicating whether nodes in the pool are configured with GPUs." + value = local.has_gpu +} + +output "allocatable_gpu_per_node" { + description = "Number of GPUs available for scheduling pods on each node." + value = local.allocatable_gpu_per_node +} + +output "static_gpu_count" { + description = "Total number of GPUs in the node pool. Available only for static node pools." + value = local.static_gpu_count +} + +locals { + translate_toleration = { + PREFER_NO_SCHEDULE = "PreferNoSchedule" + NO_SCHEDULE = "NoSchedule" + NO_EXECUTE = "NoExecute" + } + taints = google_container_node_pool.node_pool[0].node_config[0].taint + tolerations = [for taint in local.taints : { + key = taint.key + operator = "Equal" + value = taint.value + effect = lookup(local.translate_toleration, taint.effect, null) + }] +} + +output "tolerations" { + description = "Tolerations needed for a pod to be scheduled on this node pool." + value = local.tolerations +} + +locals { + gpu_direct_enabled = var.machine_type == "a3-highgpu-8g" || var.machine_type == "a3-megagpu-8g" + script_path = { + a3-highgpu-8g = "enable-tcpx-in-workload.py", + a3-megagpu-8g = "enable-tcpxo-in-workload.py" + } + nccl_path = var.machine_type == "a3-highgpu-8g" ? "configs" : "scripts" + gpu_direct_instruction = <<-EOT + Since you are using ${var.machine_type} machine type that has GPUDirect support, your nodepool had been configured with the required plugins. + To fully utilize GPUDirect you will need to add some components into your workload manifest. Details below: + + A sample GKE job that has GPUDirect enabled and NCCL test included has been generated locally at: + ${abspath(local.gpu_direct_setting.updated_workload_path)} + + You can use the following commands to submit the sample job: + kubectl create -f ${abspath(local.gpu_direct_setting.updated_workload_path)} + After submitting the sample job, you can validate the GPU performance by initiating NCCL test included in the sample workload: + NCCL test can be initiated from any one of the sample job Pods and coordinate with the peer Pods: + export POD_NAME=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | head -n 1) + export PEER_POD_IPS=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.status.podIP}}{{" "}}{{end}}') + kubectl exec --stdin --tty --container=nccl-test $POD_NAME -- /${local.nccl_path}/allgather.sh $PEER_POD_IPS + + If you would like to enable GPUDirect for your own workload, please follow the below steps: + export WORKLOAD_PATH=<> + python3 ${abspath("${path.module}/gpu-direct-workload/scripts/${lookup(local.script_path, var.machine_type, "")}")} --file $WORKLOAD_PATH --rxdm ${local.gpu_direct_setting.rxdm_version} + **WARNING** + The "--rxdm" version is tied to the nccl-tcpx/o-installer that had been deployed to your cluster, changing it to other value might have impact on performance + **WARNING** + + Or you can also follow our GPUDirect user guide to update your workload + https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#add-gpudirect-manifests + EOT +} + +output "instructions" { + description = "Instructions for submitting the sample GPUDirect enabled job." + value = local.gpu_direct_enabled ? local.gpu_direct_instruction : null +} + +output "node_count_static" { + description = "The number of static nodes in node-pool." + value = coalesce(var.static_node_count, var.initial_node_count, 0) +} + +output "guest_accelerator" { + description = "The accelerator type of the nodes." + value = local.guest_accelerator +} + +output "cluster_id" { + description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." + value = var.cluster_id +} + +output "machine_type" { + description = "Machine Type" + value = var.machine_type +} + +output "instance_templates" { + description = "The URLs of Instance Templates" + value = [for key, template in data.google_compute_region_instance_template.instance_template : template.self_link] +} + +output "tpu_accelerator_type" { + description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." + value = module.tpu.is_tpu ? module.tpu.tpu_accelerator_type : null +} + +output "tpu_topology" { + description = "The topology of the TPU slice (e.g., '4x4')." + value = module.tpu.is_tpu ? module.tpu.tpu_topology : null +} + +output "tpu_chips_per_node" { + description = "The number of TPU chips on each node in the pool." + value = module.tpu.is_tpu ? module.tpu.tpu_chips_per_node : null +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf new file mode 100644 index 0000000000..7c29e3902a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf @@ -0,0 +1,107 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# Split the input into three different lists where the details of a given reservation are at the same index across these lists. +locals { + # Specific block of an extended reservation can be targeted with exr-one/reservationBlocks/exr-one-block-1 + # Data source needs to be queried with the reservation name only. So, we extract the reservation name + input_reservation_names = [for r in try(var.reservation_affinity.specific_reservations, []) : split("/", r.name)[0]] + input_reservation_projects = [for r in try(var.reservation_affinity.specific_reservations, []) : coalesce(r.project, var.project_id)] + # We, also, remember the suffix "/reservationBlocks/exr-one-block-1" for use elsewhere afterwards + input_reservation_suffixes = [for r in try(var.reservation_affinity.specific_reservations, []) : substr(r.name, length(split("/", r.name)[0]), -1)] + # Adding this variable to by-pass the machine-type validation for TPUs + is_tpu = var.placement_policy.tpu_topology != null +} + +data "google_compute_reservation" "specific_reservations" { + for_each = ( + local.input_specific_reservations_count == 0 ? + {} : + { + for pair in flatten([ + for zone in try(var.zones, []) : [ + for i, reservation_name in try(local.input_reservation_names, []) : { + key : "${local.input_reservation_projects[i]}/${zone}/${reservation_name}" + zone : zone + reservation_name : reservation_name + project : local.input_reservation_projects[i] + } + ] + ]) : + pair.key => pair + } + ) + name = each.value.reservation_name + zone = each.value.zone + project = each.value.project +} + +locals { + generated_guest_accelerator = module.gpu.machine_type_guest_accelerator + reservation_resource_api_label = "compute.googleapis.com/reservation-name" + input_specific_reservations_count = try(length(var.reservation_affinity.specific_reservations), 0) + + # Filter specific reservations + verified_specific_reservations = [for k, v in data.google_compute_reservation.specific_reservations : v if(v.specific_reservation != null && v.specific_reservation_required == true)] + + # Build two maps to be used to compare the VM properties between reservations and the node pool + # Validation of only machine-type for CPUs and and both machine-type and guest-accelerators for GPUs + # Skip this for TPUs ( returns an empty list to skip the machine-type validation for aggregate TPU reservations) + reservation_vm_properties = local.is_tpu ? [] : [for reservation in local.verified_specific_reservations : { + "machine_type" : try(reservation.specific_reservation[0].instance_properties[0].machine_type, "") + "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs + { for acc in try(reservation.specific_reservation[0].instance_properties[0].guest_accelerators, []) : acc.accelerator_type => acc.accelerator_count } + ) : {} # If no GPUs, it's an empty map {} + }] + + nodepool_vm_properties = { + "machine_type" : var.machine_type + "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs + { for acc in try(local.guest_accelerator, []) : coalesce(acc.type, try(local.generated_guest_accelerator[0].type, "")) => coalesce(acc.count, try(local.generated_guest_accelerator[0].count, 0)) } + ) : {} # If no GPUs, it's an empty map {} + } + + # Compare two maps by counting the keys that mismatch. + # Know that in map comparison the order of keys does not matter. That is {NVME: x, SCSI: y} and {SCSI: y, NVME: x} are equal + # As of this writing, there is only one reservation supported by the Node Pool API. So, directly accessing it from the list + specific_reservation_requirement_violations = length(local.reservation_vm_properties) == 0 ? [] : [for k, v in local.nodepool_vm_properties : k if v != local.reservation_vm_properties[0][k]] + + specific_reservation_requirement_violation_messages = { + "machine_type" : <<-EOT + The reservation has "${try(local.reservation_vm_properties[0].machine_type, "")}" machine type and the node pool has "${local.nodepool_vm_properties.machine_type}". Check the relevant node pool setting: "machine_type" + EOT + "guest_accelerators" : <<-EOT + The reservation has ${jsonencode(try(local.reservation_vm_properties[0].guest_accelerators, {}))} accelerators and the node pool has ${jsonencode(try(local.nodepool_vm_properties.guest_accelerators, {}))}. Check the relevant node pool setting: "guest_accelerator". When unspecified, for the machine_type=${var.machine_type}, the default is guest_accelerator=${jsonencode(try(local.generated_guest_accelerator, [{}]))}. + EOT + } +} + +locals { + # Check if reservation is valid, that is, if it exists, there should be only 1 verified specific reservation or the reservation doesn't exist + is_valid_reservation = length(local.verified_specific_reservations) == 1 || !var.is_reservation_active + + # Build the list of reservation names when var.is_reservation_active is true + active_reservation_values = [ + for i, r in local.verified_specific_reservations : + length(local.input_reservation_suffixes[i]) > 0 ? + format("%s%s", r.name, local.input_reservation_suffixes[i]) : + "projects/${r.project}/reservations/${r.name}" + ] + + # Define a default reservation value if no specific reservations are present + specific_reservation_name = length(local.input_reservation_names) > 0 ? local.input_reservation_names[0] : "" + default_reservation_values = ["projects/${var.project_id}/reservations/${local.specific_reservation_name}"] +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf new file mode 100644 index 0000000000..e582db33da --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' + +# "inputs": +# var.machine_type: Machine type for the instance being evaluated. +# var.threads_per_core : Sets the number of threads per physical core, where 0 +# has behavior described in description. + +# "outputs": +# local.set_threads_per_core: bool that tells if threads per core should be set, +# to be used with a dynamic block. +# local.threads_per_core: actual threads_per_core to be used. + +locals { + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + machine_shared_core = length(local.machine_vals) <= 2 + machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) + + smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) + smt_capable_vcpu = local.machine_vcpus >= 2 + + smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core + set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) + threads_per_core = var.threads_per_core == 2 ? 2 : 1 +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/variables.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/variables.tf new file mode 100644 index 0000000000..b44ea28d57 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/variables.tf @@ -0,0 +1,487 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" + type = string +} + +variable "zones" { + description = "A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable." + type = list(string) + default = null +} + +variable "name" { + description = <<-EOD + The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix. + If setting manually, ensure a unique value across all gke-node-pools. + EOD + type = string + default = null + + validation { + # Check if the variable is null OR if it matches the GCP resource naming regex. + condition = var.name == null || can(regex("^[a-z]([-a-z0-9]{0,34}[a-z0-9])?$", var.name)) + error_message = <<-EOD + If provided, the node pool name must be between 1 and 36 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. + Underscores are not allowed. A shorter length is enforced to accommodate a suffix when creating multiple node pools. + EOD + } +} + +variable "internal_ghpc_module_id" { + description = "DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide)." + type = string +} + +variable "machine_type" { + description = "The name of a Google Compute Engine machine type." + type = string + default = "c2-standard-60" +} + +variable "disk_size_gb" { + description = "Size of disk for each node." + type = number + default = 100 +} + +variable "disk_type" { + description = "Disk type for each node." + type = string + default = null +} + +variable "enable_gcfs" { + description = "Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config)." + type = bool + default = false +} + +variable "enable_secure_boot" { + description = "Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info." + type = bool + default = true +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = optional(string) + count = optional(number, 0) + gpu_driver_installation_config = optional(object({ + gpu_driver_version = string + }), { gpu_driver_version = "DEFAULT" }) + gpu_partition_size = optional(string) + gpu_sharing_config = optional(object({ + gpu_sharing_strategy = string + max_shared_clients_per_gpu = number + })) + })) + default = [] + nullable = false + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.count != null]) + error_message = "var.guest_accelerator[*].count cannot be null" + } + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.count >= 0]) + error_message = "var.guest_accelerator[*].count must never be negative" + } + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.gpu_driver_installation_config != null]) + error_message = "var.guest_accelerator[*].gpu_driver_installation_config must not be null; leave unset to enable GKE to select default GPU driver installation" + } +} + +variable "image_type" { + description = "The default image type used by NAP once a new node pool is being created. Use either COS_CONTAINERD or UBUNTU_CONTAINERD." + type = string + default = "COS_CONTAINERD" +} + +variable "local_ssd_count_ephemeral_storage" { + description = <<-EOT + The number of local SSDs to attach to each node to back ephemeral storage. + Uses NVMe interfaces. Must be supported by `machine_type`. + When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. + [See above](#local-ssd-storage) for more info. + EOT + type = number + default = null +} + +variable "local_ssd_count_nvme_block" { + description = <<-EOT + The number of local SSDs to attach to each node to back block storage. + Uses NVMe interfaces. Must be supported by `machine_type`. + When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. + [See above](#local-ssd-storage) for more info. + + EOT + type = number + default = null +} + +variable "autoscaling_total_min_nodes" { + description = "Total minimum number of nodes in the NodePool." + type = number + default = 0 +} + +variable "autoscaling_total_max_nodes" { + description = "Total maximum number of nodes in the NodePool." + type = number + default = 1000 +} + +variable "static_node_count" { + description = "The static number of nodes in the node pool. If set, autoscaling will be disabled." + type = number + default = null +} + +variable "is_reservation_active" { + description = "Whether the specified reservation is already created." + type = bool + default = true +} + +variable "auto_repair" { + description = "Whether the nodes will be automatically repaired." + type = bool + default = true +} + +variable "auto_upgrade" { + description = "Whether the nodes will be automatically upgraded." + type = bool + default = false +} + +variable "threads_per_core" { + description = <<-EOT + Sets the number of threads per physical core. By setting threads_per_core + to 2, Simultaneous Multithreading (SMT) is enabled extending the total number + of virtual cores. For example, a machine of type c2-standard-60 will have 60 + virtual cores with threads_per_core equal to 2. With threads_per_core equal + to 1 (SMT turned off), only the 30 physical cores will be available on the VM. + + The default value of \"0\" will turn off SMT for supported machine types, and + will fall back to GCE defaults for unsupported machine types (t2d, shared-core + instances, or instances with less than 2 vCPU). + + Disabling SMT can be more performant in many HPC workloads, therefore it is + disabled by default where compatible. + + null = SMT configuration will use the GCE defaults for the machine type + 0 = SMT will be disabled where compatible (default) + 1 = SMT will always be disabled (will fail on incompatible machine types) + 2 = SMT will always be enabled (will fail on incompatible machine types) + EOT + type = number + default = 0 + + validation { + condition = var.threads_per_core == null || try(var.threads_per_core >= 0, false) && try(var.threads_per_core <= 2, false) + error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." + } +} + +variable "spot" { + description = "Provision VMs using discounted Spot pricing, allowing for preemption" + type = bool + default = false +} + +# tflint-ignore: terraform_unused_declarations +variable "compact_placement" { + description = "DEPRECATED: Use `placement_policy`" + type = bool + default = null + validation { + condition = var.compact_placement == null + error_message = "`compact_placement` is deprecated. Use `placement_policy` instead" + } +} + +variable "placement_policy" { + description = <<-EOT + Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool. + It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + EOT + + type = object({ + type = string + name = optional(string) + tpu_topology = optional(string) + }) + default = { + type = null + name = null + tpu_topology = null + } + validation { + condition = var.placement_policy.type == null || try(contains(["COMPACT"], var.placement_policy.type), false) + error_message = "`COMPACT` is the only supported value for `placement_policy.type`." + } +} + +variable "service_account_email" { + description = "Service account e-mail address to use with the node pool" + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to to use with the node pool." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "taints" { + description = "Taints to be applied to the system node pool." + type = list(object({ + key = string + value = any + effect = string + })) + default = [] +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} + +variable "kubernetes_labels" { + description = <<-EOT + Kubernetes labels to be applied to each node in the node group. Key-value pairs. + (The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) + EOT + type = map(string) + default = null +} + +variable "timeout_create" { + description = "Timeout for creating a node pool" + type = string + default = null +} + +variable "timeout_update" { + description = "Timeout for updating a node pool" + type = string + default = null +} + +# Deprecated + +# tflint-ignore: terraform_unused_declarations +variable "total_min_nodes" { + description = "DEPRECATED: Use autoscaling_total_min_nodes." + type = number + default = null + validation { + condition = var.total_min_nodes == null + error_message = "total_min_nodes was renamed to autoscaling_total_min_nodes and is deprecated; use autoscaling_total_min_nodes" + } +} + +# tflint-ignore: terraform_unused_declarations +variable "total_max_nodes" { + description = "DEPRECATED: Use autoscaling_total_max_nodes." + type = number + default = null + validation { + condition = var.total_max_nodes == null + error_message = "total_max_nodes was renamed to autoscaling_total_max_nodes and is deprecated; use autoscaling_total_max_nodes" + } +} + +# tflint-ignore: terraform_unused_declarations +variable "service_account" { + description = "DEPRECATED: use service_account_email and scopes." + type = object({ + email = string, + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "service_account is deprecated and replaced with service_account_email and scopes." + } +} + +variable "additional_networks" { + description = "Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool" + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + })) + nullable = false +} + +variable "reservation_affinity" { + description = <<-EOT + Reservation resource to consume. When targeting SPECIFIC_RESERVATION, specific_reservations needs be specified. + Even though specific_reservations is a list, only one reservation is allowed by the NodePool API. + It is assumed that the specified reservation exists and has available capacity. + For a shared reservation, specify the project_id as well in which it was created. + To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared + EOT + type = object({ + consume_reservation_type = string + specific_reservations = optional(list(object({ + name = string + project = optional(string) + }))) + }) + default = { + consume_reservation_type = "NO_RESERVATION" + specific_reservations = [] + } + validation { + condition = contains(["NO_RESERVATION", "ANY_RESERVATION", "SPECIFIC_RESERVATION"], var.reservation_affinity.consume_reservation_type) + error_message = "Accepted values are: {NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION}" + } +} + +variable "host_maintenance_interval" { + description = "Specifies the frequency of planned maintenance events." + type = string + default = "" + nullable = false + validation { + condition = contains(["", "PERIODIC", "AS_NEEDED"], var.host_maintenance_interval) + error_message = "Invalid host_maintenance_interval value. Must be PERIODIC, AS_NEEDED or the empty string" + } +} + +variable "initial_node_count" { + description = "The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static_node_count and must be set to a value between autoscaling_total_min_nodes and autoscaling_total_max_nodes." + type = number + default = null +} + +variable "gke_version" { + description = "GKE version" + type = string +} + +variable "max_pods_per_node" { + description = "The maximum number of pods per node in this node pool. This will force replacement." + type = number + default = null +} + +variable "upgrade_settings" { + description = <<-EOT + Defines node pool upgrade settings. It is highly recommended that you define all max_surge and max_unavailable. + If max_surge is not specified, it would be set to a default value of 0. + If max_unavailable is not specified, it would be set to a default value of 1. + EOT + type = object({ + strategy = string + max_surge = optional(number) + max_unavailable = optional(number) + }) + default = { + strategy = "SURGE" + max_surge = 0 + max_unavailable = 1 + } +} + +variable "run_workload_script" { + description = "Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only." + type = bool + default = true +} + +variable "enable_queued_provisioning" { + description = "If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool." + type = bool + default = false +} + +variable "enable_flex_start" { + description = <<-EOT + If true, start the node pool with Flex Start provisioning model. + To learn more about flex-start mode, please refer to + https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and + https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest + EOT + type = bool + default = false +} + +variable "max_run_duration" { + description = "The duration (in whole seconds) of the instance. Instance will run and be terminated after then." + type = number + default = null +} + +variable "enable_private_nodes" { + description = "Whether nodes have internal IP addresses only." + type = bool + default = true +} + +variable "num_node_pools" { + description = "Number of node pools to create. This is same as num_slices." + type = number + default = 1 +} + +variable "num_slices" { + description = "Number of TPUs slices to create. This is same as num_node_pools." + type = number + default = 1 +} + +variable "enable_numa_aware_scheduling" { + description = "Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling." + type = bool + default = false +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/versions.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/versions.tf new file mode 100644 index 0000000000..f018d04fc5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/versions.tf @@ -0,0 +1,38 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 7.2" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 7.2" + } + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/README.md b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/README.md new file mode 100644 index 0000000000..3b769e8761 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/README.md @@ -0,0 +1,82 @@ +## Description + +This modules create a [resource policy for compute engines](https://cloud.google.com/compute/docs/instances/placement-policies-overview). This policy can be passed to a gke-node-pool module to apply the policy on the node-pool's nodes. + +Note: By default, you can't apply compact placement policies with a max distance value to A3 VMs. To request access to this feature, contact your [Technical Account Manager (TAM)](https://cloud.google.com/tam) or the [Sales team](https://cloud.google.com/contact). + +### Example + +The following example creates a group placement resource policy and applies it to a gke-node-pool. + +```yaml + - id: group_placement_1 + source: modules/compute/resource-policy + settings: + name: gp-np-1 + group_placement_max_distance: 2 + + - id: node_pool_1 + source: modules/compute/gke-node-pool + use: [group_placement_1] + settings: + machine_type: e2-standard-8 + outputs: [instructions] +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google-beta](#requirement\_google-beta) | >= 6.29.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google-beta](#provider\_google-beta) | >= 6.29.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_resource_policy.policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [group\_placement\_max\_distance](#input\_group\_placement\_max\_distance) | The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | `number` | `0` | no | +| [name](#input\_name) | The resource policy's name. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The project ID for the resource policy. | `string` | n/a | yes | +| [region](#input\_region) | The region for the the resource policy. | `string` | n/a | yes | +| [workload\_policy](#input\_workload\_policy) | Describes the workload policy | 
object({
    type                  = optional(string, null)
    max_topology_distance = optional(string, null)
    accelerator_topology  = optional(string, null)
  })
| 
{
  "accelerator_topology": null,
  "max_topology_distance": null,
  "type": null
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [placement\_policy](#output\_placement\_policy) | Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions.
The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. | + diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/main.tf b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/main.tf new file mode 100644 index 0000000000..906424ca7c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + name = "${var.name}-${random_id.resource_name_suffix.hex}" +} + +resource "google_compute_resource_policy" "policy" { + name = local.name + region = var.region + project = var.project_id + provider = google-beta + + dynamic "workload_policy" { + for_each = var.workload_policy.type != null ? [1] : [] + + content { + type = var.workload_policy.type + max_topology_distance = var.workload_policy.max_topology_distance + accelerator_topology = var.workload_policy.accelerator_topology + } + } + + dynamic "group_placement_policy" { + for_each = var.group_placement_max_distance > 0 ? [1] : [] + + content { + collocation = "COLLOCATED" + max_distance = var.group_placement_max_distance + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/outputs.tf b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/outputs.tf new file mode 100644 index 0000000000..c1dc65bcbb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "placement_policy" { + description = <<-EOT + Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. + It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. + EOT + + value = { + type = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? "COMPACT" : null + name = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? local.name : null + tpu_topology = (var.workload_policy.type != null) ? var.workload_policy.accelerator_topology : null + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/variables.tf b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/variables.tf new file mode 100644 index 0000000000..92434326ca --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/variables.tf @@ -0,0 +1,64 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID for the resource policy." + type = string +} + +variable "region" { + description = "The region for the the resource policy." + type = string +} + +variable "name" { + description = "The resource policy's name." + type = string + + validation { + # Check if the variable matches the GCP resource naming regex. + condition = can(regex("^[a-z]([-a-z0-9]{0,52}[a-z0-9])?$", var.name)) + error_message = <<-EOD + The resource policy name must be between 1 and 54 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. + Underscores are not allowed. A shorter length is enforced to accommodate a random suffix. + EOD + } +} + +variable "group_placement_max_distance" { + description = <<-EOT + The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + EOT + + type = number + default = 0 +} + +variable "workload_policy" { + description = "Describes the workload policy" + type = object({ + type = optional(string, null) + max_topology_distance = optional(string, null) + accelerator_topology = optional(string, null) + }) + default = { + type = null + max_topology_distance = null + accelerator_topology = null + } + nullable = false +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/versions.tf b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/versions.tf new file mode 100644 index 0000000000..f235fbade3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:resource-policy/v1.37.2" + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/README.md b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/README.md new file mode 100644 index 0000000000..0c4737e0d9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/README.md @@ -0,0 +1,257 @@ +## Description + +This module creates one or more +[compute VM instances](https://cloud.google.com/compute/docs/instances). + +### Example + +```yaml +- id: compute + source: modules/compute/vm-instance + use: [network1] + settings: + instance_count: 8 + name_prefix: compute + machine_type: c2-standard-60 +``` + +This creates a cluster of 8 compute VMs that are: + +* named `compute-[0-7]` +* on the network defined by the `network1` module +* of type c2-standard-60 + +> **_NOTE:_** Simultaneous Multithreading (SMT) is deactivated by default +> (threads_per_core=1), which means only the physical cores are visible on the +> VM. With SMT disabled, a machine of type c2-standard-60 will only have the 30 +> physical cores visible. To change this, set `threads_per_core=2` under +> settings. + +### VPC Networks + +There are two methods for adding network connectivity to the `vm-instance` +module. The first is shown in the example above, where a `vpc` module or +`pre-existing-vpc` module is used by the `vm-instance` module. When this +happens, the `network_self_link` and `subnetwork_self_link` outputs from the +network are provided as input to the `vm-instance` and a network interface is +defined based on that. This can also be done updating the `network_self_link` and +`subnetwork_self_link` settings directly. + +The alternative option can be used when more than one network needs to be added +to the `vm-instance` or further customization is needed beyond what is provided +via other variables. For this option, the `network_interfaces` variable can be +used to set up one or more network interfaces on the VM instance. The format is +consistent with the terraform `google_compute_instance` `network_interface` +block, and more information can be found in the +[terraform docs][network-interface-tf]. + +> **_NOTE:_** When supplying the `network_interfaces` variable, networks +> associated with the `vm-instance` via use will be ignored in favor of the +> networks added in `network_interfaces`. In addition, `bandwidth_tier` and +> `disable_public_ips` will not apply to networks defined in +> `network_interfaces`. + +[network-interface-tf]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + +### SSH key metadata + +This module will ignore all changes to the `ssh-keys` metadata field that are +typically set by [external Google Cloud tools that automate SSH access][gcpssh] +when not using OS Login. For example, clicking on the Google Cloud Console SSH +button next to VMs in the VM Instances list will temporarily modify VM metadata +to include a dynamically-generated SSH public key. + +[gcpssh]: https://cloud.google.com/compute/docs/connect/add-ssh-keys#metadata + +### Placement + +The `placement_policy` variable can be used to control where your VM instances +are physically located relative to each other within a zone. See the official +placement [guide][guide-link] and [api][api-link] documentation. + +[guide-link]: https://cloud.google.com/compute/docs/instances/define-instance-placement +[api-link]: https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement + +Use the following settings for compact placement: + +```yaml + ... + settings: + instance_count: 4 + machine_type: c2-standard-60 + placement_policy: + collocation: "COLLOCATED" +``` + +By default the above placement policy will always result in the most compact set +of VMs available. If you would like that provisioning failed if some level of +compactness is not obtainable, you can enforce this with the [`max_distance` +setting](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies): + +```yaml + ... + settings: + instance_count: 4 + machine_type: c2-standard-60 + placement_policy: + collocation: "COLLOCATED" + max_distance: 1 +``` + +Use the following settings for spread placement: + +```yaml + ... + settings: + instance_count: 4 + machine_type: n2-standard-4 + placement_policy: + availability_domain_count: 2 +``` + +When `vm_count` is not set, as shown in the examples above, then the VMs will be +added to the placement policy incrementally. This is the **recommended way** to +use placement policies. + +If `vm_count` is specified then VMs will stay in pending state until the +specified number of VMs are created. See the warning below if using this field. + +> [!WARNING] +> When creating a compact placement using `vm_count` with more than 10 VMs, you +> must add `-parallelism=` argument on apply. For example if you have 15 VMs +> in a placement group: `terraform apply -parallelism=15`. This is because +> terraform self limits to 10 parallel requests by default but the create +> instance requests will not succeed until all VMs in the placement group have +> been requested, forming a deadlock. + +### GPU Support + +More information on GPU support in `vm-instance` and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../docs/gpu-support.md) + +## Lifecycle + +The `vm-instance` module will be replaced when the `instance_image` variable is +changed and `terraform apply` is run on the deployment group folder or +`gcluster deploy` is run. However, it will not be automatically replaced if a new +image is created in a family. + +To selectively replace the vm-instance(s), consider running terraform +`apply -replace` such as: + +> See https://developer.hashicorp.com/terraform/cli/commands/plan#replace-address for precise syntax terraform apply -replace=ADDRESS + +```shell +terraform state list +# search for the module ID and resource +terraform apply -replace="address" +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 4.73.0 | +| [google-beta](#requirement\_google-beta) | >= 6.13.0 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.73.0 | +| [google-beta](#provider\_google-beta) | >= 6.13.0 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | +| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance) | resource | +| [google-beta_google_compute_resource_policy.placement_policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | +| [google_compute_address.compute_ip](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_disk.additional_disks](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.replace_vm_trigger_from_placement](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | +| [additional\_persistent\_disks](#input\_additional\_persistent\_disks) | Configurations of additional disks to be included on the partition nodes. | 
object({
    count = optional(number, 0)
    type  = optional(string, "pd-balanced")
    size  = optional(number, 200)
  })
| `{}` | no | +| [allocate\_ip](#input\_allocate\_ip) | If not null, allocate IPs with the given configuration. See details at
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address | 
object({
    address_type = optional(string, "INTERNAL")
    purpose      = optional(string),
    network_tier = optional(string),
    ip_version   = optional(string, "IPV4"),
  })
| `null` | no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | +| [automatic\_restart](#input\_automatic\_restart) | Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user). | `bool` | `null` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will optionally be used name resources according to `name_prefix` | `string` | n/a | yes | +| [disable\_public\_ips](#input\_disable\_public\_ips) | If set to true, instances will not have public IPs | `bool` | `false` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | +| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-standard"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | +| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_count](#input\_local\_ssd\_count) | The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd. | `number` | `0` | no | +| [local\_ssd\_interface](#input\_local\_ssd\_interface) | Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set. | `string` | `"NVME"` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface

**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply
to network interfaces defined in this variable.

Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "MRDMA", "IRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. Can use "default" for the default network. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `null` | no | +| [placement\_policy](#input\_placement\_policy) | Control where your VM instances are physically located relative to each other within a zone.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy#nested_group_placement_policy | `any` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [provisioning\_model](#input\_provisioning\_model) | Provisioning model for cloud instance. | `string` | `null` | no | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME
- RESERVATION\_NAME

Must be a "SPECIFIC\_RESERVATION"
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | +| [service\_account](#input\_service\_account) | DEPRECATED - Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [spot](#input\_spot) | DEPRECATED - Use `provisioning_model` instead. | `bool` | `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | +| [zone](#input\_zone) | Compute Platform zone | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [external\_ip](#output\_external\_ip) | External IP of the instances (if enabled) | +| [instructions](#output\_instructions) | Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions. | +| [internal\_ip](#output\_internal\_ip) | Internal IP of the instances | +| [name](#output\_name) | Names of instances created | +| [self\_link](#output\_self\_link) | The tuple URIs of the created instances | + diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/compute_image.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/main.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/main.tf new file mode 100644 index 0000000000..0a8c7d354e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/main.tf @@ -0,0 +1,334 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "vm-instance", ghpc_role = "compute" }) +} + +module "gpu" { + source = "../../internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + native_fstype = [] + startup_script = local.startup_from_network_storage != null ? ( + { startup-script = local.startup_from_network_storage }) : {} + network_storage = var.network_storage != null ? ( + { network_storage = jsonencode(var.network_storage) }) : {} + + prefix_optional_deployment_name = var.name_prefix != null ? var.name_prefix : var.deployment_name + prefix_always_deployment_name = var.name_prefix != null ? "${var.deployment_name}-${var.name_prefix}" : var.deployment_name + resource_prefix = var.add_deployment_name_before_prefix ? local.prefix_always_deployment_name : local.prefix_optional_deployment_name + + enable_gvnic = var.bandwidth_tier != "not_enabled" + enable_tier_1 = var.bandwidth_tier == "tier_1_enabled" + + provisioning_model = var.provisioning_model + + spot = var.provisioning_model == "SPOT" + + # compact_placement : true when placement policy is provided and collocation set; false if unset + compact_placement = try(var.placement_policy.collocation, null) != null + + gpu_attached = contains(["a2", "g2"], local.machine_family) || length(local.guest_accelerator) > 0 + + # both of these must be false if either compact placement or preemptible/spot instances are used + # automatic restart is tolerant of GPUs while on host maintenance is not + automatic_restart_default = local.compact_placement || local.spot ? false : null + on_host_maintenance_default = local.compact_placement || local.spot || local.gpu_attached ? "TERMINATE" : "MIGRATE" + + automatic_restart = ( + var.automatic_restart != null + ? var.automatic_restart + : local.automatic_restart_default + ) + + on_host_maintenance = ( + var.on_host_maintenance != null + ? var.on_host_maintenance + : local.on_host_maintenance_default + ) + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + # Network Interfaces + # Support for `use` input and base network parameters like `network_self_link` and `subnetwork_self_link` + empty_access_config = { + nat_ip = null, + public_ptr_domain_name = null, + network_tier = null + } + default_network_interface = { + network = var.network_self_link + subnetwork = var.subnetwork_self_link + subnetwork_project = null # will populate from subnetwork_self_link + network_ip = null + nic_type = local.enable_gvnic ? "GVNIC" : null + stack_type = null + queue_count = null + access_config = var.disable_public_ips ? [] : [local.empty_access_config] + ipv6_access_config = [] + alias_ip_range = [] + } + network_interfaces = coalescelist(var.network_interfaces, [local.default_network_interface]) + network_interfaces_with_ips = var.allocate_ip == null ? local.network_interfaces : [ + for i, interface in local.network_interfaces : + merge(interface, { + network_ip = google_compute_address.compute_ip[i].address + }) + ] +} + +resource "null_resource" "image" { + triggers = { + name = try(var.instance_image.name, null), + family = try(var.instance_image.family, null), + project = try(var.instance_image.project, null) + } +} + +resource "google_compute_disk" "additional_disks" { + project = var.project_id + + count = var.instance_count * var.additional_persistent_disks.count + + # NB: this resource array must be sliced accounting for var.instance_count + name = "${local.resource_prefix}-disk-${count.index}" + type = var.additional_persistent_disks.type + size = var.additional_persistent_disks.size + labels = local.labels + zone = var.zone +} + +resource "google_compute_resource_policy" "placement_policy" { + project = var.project_id + provider = google-beta + + count = var.placement_policy != null ? 1 : 0 + name = "${local.resource_prefix}-vm-instance-placement" + group_placement_policy { + vm_count = try(var.placement_policy.vm_count, null) + availability_domain_count = try(var.placement_policy.availability_domain_count, null) + collocation = try(var.placement_policy.collocation, null) + max_distance = try(var.placement_policy.max_distance, null) + } +} + +resource "null_resource" "replace_vm_trigger_from_placement" { + triggers = { + vm_count = try(tostring(var.placement_policy.vm_count), "") + availability_domain_count = try(tostring(var.placement_policy.availability_domain_count), "") + max_distance = try(tostring(var.placement_policy.max_distance), "") + collocation = try(var.placement_policy.collocation, "") + } +} + +resource "google_compute_address" "compute_ip" { + project = var.project_id + + count = var.allocate_ip != null ? length(local.network_interfaces) : 0 + + name = "${local.resource_prefix}-${count.index}" + + address = local.network_interfaces[count.index].network_ip + region = var.region + network = can(coalesce(local.network_interfaces[count.index].subnetwork)) ? null : local.network_interfaces[count.index].network + subnetwork = local.network_interfaces[count.index].subnetwork + address_type = var.allocate_ip.address_type + purpose = var.allocate_ip.purpose + network_tier = var.allocate_ip.network_tier + ip_version = var.allocate_ip.ip_version +} + +resource "google_compute_instance" "compute_vm" { + project = var.project_id + provider = google-beta + + count = var.instance_count + + depends_on = [var.network_self_link, var.network_storage] + + name = "${local.resource_prefix}-${count.index}" + min_cpu_platform = var.min_cpu_platform + machine_type = var.machine_type + zone = var.zone + + resource_policies = google_compute_resource_policy.placement_policy[*].self_link + + tags = var.tags + labels = local.labels + + boot_disk { + initialize_params { + image = data.google_compute_image.compute_image.self_link + size = var.disk_size_gb + type = var.disk_type + labels = local.labels + } + + device_name = "${local.resource_prefix}-boot-disk-${count.index}" + auto_delete = var.auto_delete_boot_disk + } + + dynamic "attached_disk" { + for_each = slice( + google_compute_disk.additional_disks, + var.additional_persistent_disks.count * count.index, + var.additional_persistent_disks.count * count.index + var.additional_persistent_disks.count, + ) + + content { + source = attached_disk.value.self_link + device_name = "additional-disk-${attached_disk.key}" + mode = "READ_WRITE" + } + } + + dynamic "scratch_disk" { + for_each = range(var.local_ssd_count) + content { + interface = var.local_ssd_interface + } + } + + dynamic "network_interface" { + for_each = local.network_interfaces_with_ips + + content { + network = network_interface.value.network + subnetwork = network_interface.value.subnetwork + subnetwork_project = network_interface.value.subnetwork_project + network_ip = network_interface.value.network_ip + nic_type = network_interface.value.nic_type + stack_type = network_interface.value.stack_type + queue_count = network_interface.value.queue_count + dynamic "access_config" { + for_each = network_interface.value.access_config + content { + nat_ip = access_config.value.nat_ip + public_ptr_domain_name = access_config.value.public_ptr_domain_name + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = network_interface.value.ipv6_access_config + content { + public_ptr_domain_name = ipv6_access_config.value.public_ptr_domain_name + network_tier = ipv6_access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = network_interface.value.alias_ip_range + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + } + } + + network_performance_config { + total_egress_bandwidth_tier = local.enable_tier_1 ? "TIER_1" : "DEFAULT" + } + + service_account { + email = var.service_account_email + scopes = var.service_account_scopes + } + + dynamic "guest_accelerator" { + for_each = local.guest_accelerator + content { + count = guest_accelerator.value.count + type = guest_accelerator.value.type + } + } + + scheduling { + on_host_maintenance = local.on_host_maintenance + automatic_restart = local.automatic_restart + preemptible = local.spot + provisioning_model = local.provisioning_model + } + + dynamic "advanced_machine_features" { + for_each = local.set_threads_per_core ? [1] : [] + content { + threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf + } + } + + dynamic "reservation_affinity" { + for_each = var.reservation_name == "" ? [] : [1] + content { + type = "SPECIFIC_RESERVATION" + specific_reservation { + key = "compute.googleapis.com/reservation-name" + values = [var.reservation_name] + } + } + } + + metadata = merge( + local.network_storage, + local.startup_script, + local.enable_oslogin, + local.disable_automatic_updates_metadata, + var.metadata + ) + + lifecycle { + ignore_changes = [ + metadata["ssh-keys"], + ] + + replace_triggered_by = [ + null_resource.replace_vm_trigger_from_placement + ] + + precondition { + condition = (length(var.network_interfaces) == 0) != (var.network_self_link == null && var.subnetwork_self_link == null) + error_message = "Exactly one of network_interfaces or network_self_link/subnetwork_self_link must be specified." + } + precondition { + condition = alltrue([for interface in var.network_interfaces : interface.network_ip == null]) || var.instance_count == 1 + error_message = <<-EOT + The network_ip cannot be statically set on vm-instance when the VM instance_count is greater than 1. + Either set the network_ip to null to allow it to be set dynamically for all instances, or create modules for each VM instance with its own network interface. + EOT + } + precondition { + condition = !contains([ + "c3-:pd-standard", + "h3-:pd-standard", + "h3-:pd-ssd", + ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") + error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/outputs.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/outputs.tf new file mode 100644 index 0000000000..eab8cb56bd --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "name" { + description = "Names of instances created" + value = google_compute_instance.compute_vm[*].name +} + +output "self_link" { + description = "The tuple URIs of the created instances" + value = google_compute_instance.compute_vm[*].self_link +} + +output "external_ip" { + description = "External IP of the instances (if enabled)" + value = try(google_compute_instance.compute_vm[*].network_interface[0].access_config[0].nat_ip, []) +} + +output "internal_ip" { + description = "Internal IP of the instances" + value = google_compute_instance.compute_vm[*].network_interface[0].network_ip +} + +locals { + first_instance_link = try(google_compute_instance.compute_vm[0].self_link, "no-instance") + ssh_instructions = <<-EOT + Use the following commands to SSH into the first VM created: + gcloud compute ssh ${local.first_instance_link} --project ${var.project_id} + If not accessible from the public internet, use an SSH tunnel through IAP: + gcloud compute ssh ${local.first_instance_link} --tunnel-through-iap --project ${var.project_id} + EOT +} + +output "instructions" { + description = "Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions." + value = var.instance_count > 0 ? local.ssh_instructions : "No instances were created." +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf new file mode 100644 index 0000000000..02bc58e4f7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "inputs": +# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. +# var.network_storage : to be passed into metadata somewhere else (not here) +# var.startup_script : to be changed into a more complete file system with all the fs runners + +# "outputs": +# local.startup_from_network_storage : A full startup script with all the runners that are not supported +# natively and were included in the network_storage structure + +locals { + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + # Pull out runners to include in startup script + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + startup_script_runner = [{ + content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" + destination = "passed_startup_script.sh" + type = "shell" + }] + + full_runner_list = concat( + local.storage_client_install_runners, + local.mount_runners, + local.startup_script_runner + ) + + startup_from_network_storage = module.netstorage_startup_script.startup_script +} + +module "netstorage_startup_script" { + source = "../../scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.full_runner_list +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf new file mode 100644 index 0000000000..e582db33da --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' + +# "inputs": +# var.machine_type: Machine type for the instance being evaluated. +# var.threads_per_core : Sets the number of threads per physical core, where 0 +# has behavior described in description. + +# "outputs": +# local.set_threads_per_core: bool that tells if threads per core should be set, +# to be used with a dynamic block. +# local.threads_per_core: actual threads_per_core to be used. + +locals { + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + machine_shared_core = length(local.machine_vals) <= 2 + machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) + + smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) + smt_capable_vcpu = local.machine_vcpus >= 2 + + smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core + set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) + threads_per_core = var.threads_per_core == 2 ? 2 : 1 +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/variables.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/variables.tf new file mode 100644 index 0000000000..5519b8cd40 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/variables.tf @@ -0,0 +1,452 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "instance_count" { + description = "Number of instances" + type = number + default = 1 +} + +variable "instance_image" { + description = "Instance Image" + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "disk_size_gb" { + description = "Size of disk for instances." + type = number + default = 200 +} + +variable "disk_type" { + description = "Disk type for instances." + type = string + default = "pd-standard" +} + +variable "auto_delete_boot_disk" { + description = "Controls if boot disk should be auto-deleted when instance is deleted." + type = bool + default = true +} + +variable "local_ssd_count" { + description = "The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd." + type = number + default = 0 +} + +variable "local_ssd_interface" { + description = "Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set." + type = string + default = "NVME" +} + +variable "additional_persistent_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = object({ + count = optional(number, 0) + type = optional(string, "pd-balanced") + size = optional(number, 200) + }) + default = {} +} + +variable "name_prefix" { + description = <<-EOT + An optional name for all VM and disk resources. + If not supplied, `deployment_name` will be used. + When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, + then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". + EOT + type = string + default = null +} + +variable "add_deployment_name_before_prefix" { + description = <<-EOT + If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. + See `name_prefix` for further details on resource naming behavior. + EOT + type = bool + default = false +} + +variable "disable_public_ips" { + description = "If set to true, instances will not have public IPs" + type = bool + default = false +} + +variable "machine_type" { + description = "Machine type to use for the instance creation" + type = string + default = "c2-standard-60" +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "deployment_name" { + description = "Name of the deployment, will optionally be used name resources according to `name_prefix`" + type = string +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} + +variable "service_account_email" { + description = "Service account e-mail address to use with the node pool" + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to to use with the node pool." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +# tflint-ignore: terraform_unused_declarations +variable "service_account" { + description = "DEPRECATED - Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string, + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "The 'service_account' setting is deprecated, please use 'var.service_account_email' and 'var.service_account_scopes' instead." + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the VM. Can use \"default\" for the default network." + type = string + default = null +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the VM." + type = string + default = null +} + +variable "network_interfaces" { + description = <<-EOT + A list of network interfaces. The options match that of the terraform + network_interface block of google_compute_instance. For descriptions of the + subfields or more information see the documentation: + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + + **_NOTE:_** If `network_interfaces` are set, `network_self_link` and + `subnetwork_self_link` will be ignored, even if they are provided through + the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply + to network interfaces defined in this variable. + + Subfields: + network (string, required if subnetwork is not supplied) + subnetwork (string, required if network is not supplied) + subnetwork_project (string, optional) + network_ip (string, optional) + nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "MRDMA", "IRDMA"]) + stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) + queue_count (number, optional) + access_config (object, optional) + ipv6_access_config (object, optional) + alias_ip_range (list(object), optional) + EOT + type = list(object({ + network = string, + subnetwork = string, + subnetwork_project = string, + network_ip = string, + nic_type = string, + stack_type = string, + queue_count = number, + access_config = list(object({ + nat_ip = string, + public_ptr_domain_name = string, + network_tier = string + })), + ipv6_access_config = list(object({ + public_ptr_domain_name = string, + network_tier = string + })), + alias_ip_range = list(object({ + ip_cidr_range = string, + subnetwork_range_name = string + })) + })) + default = [] + validation { + condition = alltrue([ + for ni in var.network_interfaces : (ni.network == null) != (ni.subnetwork == null) + ]) + error_message = "All additional network interfaces must define exactly one of \"network\" or \"subnetwork\"." + } + validation { + condition = alltrue([ + for ni in var.network_interfaces : ni.nic_type == "GVNIC" || ni.nic_type == "VIRTIO_NET" || ni.nic_type == "MRDMA" || ni.nic_type == "IRDMA" || ni.nic_type == null + ]) + error_message = "In the variable network_interfaces, field \"nic_type\" must be \"GVNIC\", \"VIRTIO_NET\", \"MRDMA\", \"IRDMA\", or null." + } + validation { + condition = alltrue([ + for ni in var.network_interfaces : ni.stack_type == "IPV4_ONLY" || ni.stack_type == "IPV4_IPV6" || ni.stack_type == null + ]) + error_message = "In the variable network_interfaces, field \"stack_type\" must be either \"IPV4_ONLY\", \"IPV4_IPV6\" or null." + } +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "zone" { + description = "Compute Platform zone" + type = string +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "startup_script" { + description = "Startup script used on the instance" + type = string + default = null +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false +} + +variable "automatic_restart" { + description = "Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user)." + type = bool + default = null +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" + type = string + default = null + validation { + condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) + error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." + } +} + +variable "bandwidth_tier" { + description = <= 0, false) && try(var.threads_per_core <= 2, false) + error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." + } + +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "allocate_ip" { + description = <<-EOT + If not null, allocate IPs with the given configuration. See details at + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address + EOT + type = object({ + address_type = optional(string, "INTERNAL") + purpose = optional(string), + network_tier = optional(string), + ip_version = optional(string, "IPV4"), + }) + default = null +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "reservation_name" { + description = <<-EOD + Name of the reservation to use for VM resources, should be in one of the following formats: + - projects/PROJECT_ID/reservations/RESERVATION_NAME + - RESERVATION_NAME + + Must be a "SPECIFIC_RESERVATION" + Set to empty string if using no reservation or automatically-consumed reservations + EOD + type = string + default = "" + nullable = false + + validation { + condition = length(regexall("^((projects/([a-z0-9-]+)/reservations/)?([a-z0-9-]+))?$", var.reservation_name)) > 0 + error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME', [...] is an optional part." + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/versions.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/versions.tf new file mode 100644 index 0000000000..0429782c6d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/versions.tf @@ -0,0 +1,41 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.73.0" + } + + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.13.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/README.md new file mode 100644 index 0000000000..285a20bde2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/README.md @@ -0,0 +1,170 @@ +## Description + +This module creates a [Google Cloud Storage (GCS) bucket](https://cloud.google.com/storage). + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +### Example + +The following example will create a bucket named `simulation-results-xxxxxxxx`, +where `xxxxxxxx` is a randomly generated id. + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: + name_prefix: simulation-results + random_suffix: true +``` + +> **_NOTE:_** Use of `random_suffix` may cause the following error when used +> with other modules: +> `value depends on resource attributes that cannot be determined until apply`. +> To resolve this set `random_suffix` to `false` (default). + + + +> **_NOTE:_** Bucket namespace is shared by all users of Google Cloud so it is +> possible to have a bucket name clash with an existing bucket that is not in +> your project. To resolve this try to use a more unique name, or set the +> `random_suffix` variable to `true`. + +## Naming of Bucket + +There are potentially three parts to the bucket name. Each of these parts are +configurable in the blueprint. + +1. A **custom prefix**, provided by the user in the blueprint \ +Provide the custom prefix using the `name_prefix` setting. + +1. The **deployment name**, included by default \ +The deployment name can be excluded by setting `use_deployment_name_in_bucket_name: false`. + +1. A **random id** suffix, excluded by default \ +The random id can be included by setting `random_suffix: true`. + +If none of these are provided (no `name_prefix`, +`use_deployment_name_in_bucket_name: false`, & `random_suffix: false`), then the +bucket name will default to `no-bucket-name-provided`. + +Since bucket namespace is shared by all users of Google Cloud, it is more likely +to experience naming clashes than with other resources. In many cases, adding +the `random_suffix` will resolve the naming clash issue. + +> **Warning**: If a bucket is created with a `random_suffix` and then used as +> the bucket for a startup script in the same deployment group this will cause a +> `not known at apply time` error in terraform. The solution is to either create +> the bucket in a separate deployment group or to remove the random suffix. + +## Mounting + +To mount the Cloud Storage bucket you must first ensure that the GCS Fuse client +has been installed and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. + +If mounting is not automatically handled as described above, the +`cloud-storage-bucket` module outputs runners that can be used with the +`startup-script` module to install the client and mount the file system. See the +following example: + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: {local_mount: /data} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(bucket.client_install_runner) + - $(bucket.mount_runner) +``` + +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | +| [google-beta](#requirement\_google-beta) | >= 6.9.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [google-beta](#provider\_google-beta) | >= 6.9.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource | +| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [autoclass](#input\_autoclass) | Configure bucket autoclass setup

The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern.

The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time.
Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE')

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/autoclass | 
object({
    enabled                = optional(bool, false)
    terminal_storage_class = optional(string, null)
  })
| 
{
  "enabled": false
}
| no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the GCS bucket. | `string` | n/a | yes | +| [enable\_hierarchical\_namespace](#input\_enable\_hierarchical\_namespace) | If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket. | `bool` | `false` | no | +| [enable\_object\_retention](#input\_enable\_object\_retention) | If true, enables retention policy at per object level for the bucket.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/object-lock | `bool` | `false` | no | +| [enable\_versioning](#input\_enable\_versioning) | If true, enables versioning for the bucket. | `bool` | `false` | no | +| [force\_destroy](#input\_force\_destroy) | If true will destroy bucket with all objects stored within. | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to the GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | +| [lifecycle\_rules](#input\_lifecycle\_rules) | List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule | 
list(object({
    # Object with keys:
    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.
    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.
    action = object({
      type          = string
      storage_class = optional(string)
    })

    # Object with keys:
    # - age - (Optional) Minimum age of an object in days to satisfy this condition.
    # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field.
    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.
    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".
    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY.
    # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition.
    # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition.
    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.
    # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition.
    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.
    # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object.
    # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent.
    condition = object({
      age                        = optional(number)
      send_age_if_zero           = optional(bool)
      created_before             = optional(string)
      with_state                 = optional(string)
      matches_storage_class      = optional(string)
      matches_prefix             = optional(string)
      matches_suffix             = optional(string)
      num_newer_versions         = optional(number)
      custom_time_before         = optional(string)
      days_since_custom_time     = optional(number)
      days_since_noncurrent_time = optional(number)
      noncurrent_time_before     = optional(string)
    })
  }))
| `[]` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | +| [mount\_options](#input\_mount\_options) | Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories) | `string` | `"defaults,_netdev,implicit_dirs"` | no | +| [name\_prefix](#input\_name\_prefix) | Name Prefix. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | +| [public\_access\_prevention](#input\_public\_access\_prevention) | Bucket public access can be controlled by setting a value of either `inherited` or `enforced`.
When set to `enforced`, public access to the bucket is blocked.
If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/public-access-prevention | `string` | `null` | no | +| [random\_suffix](#input\_random\_suffix) | If true, a random id will be appended to the suffix of the bucket name. | `bool` | `false` | no | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [retention\_policy\_period](#input\_retention\_policy\_period) | If defined, this will configure retention\_policy with retention\_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/bucket-lock | `number` | `null` | no | +| [soft\_delete\_retention\_duration](#input\_soft\_delete\_retention\_duration) | If defined, this will configure soft\_delete\_policy with retention\_duration\_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days).
Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/soft-delete | `number` | `null` | no | +| [storage\_class](#input\_storage\_class) | The storage class of the GCS bucket. | `string` | `"REGIONAL"` | no | +| [uniform\_bucket\_level\_access](#input\_uniform\_bucket\_level\_access) | Allow uniform control access to the bucket. | `bool` | `true` | no | +| [use\_deployment\_name\_in\_bucket\_name](#input\_use\_deployment\_name\_in\_bucket\_name) | If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments. | `bool` | `true` | no | +| [viewers](#input\_viewers) | A list of additional accounts that can read packages from this bucket | `set(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use gcs fuse. | +| [gcs\_bucket\_name](#output\_gcs\_bucket\_name) | Bucket name. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | The gsutil bucket path with format of `gs://`. | +| [mount\_runner](#output\_mount\_runner) | Runner that mounts the cloud storage bucket with gcs fuse. | +| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | + diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf new file mode 100644 index 0000000000..81ba0ca6a9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf @@ -0,0 +1,126 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "cloud-storage-bucket", ghpc_role = "file-system" }) +} + +locals { + prefix = var.name_prefix != null ? var.name_prefix : "" + deployment = var.use_deployment_name_in_bucket_name ? var.deployment_name : "" + suffix = var.random_suffix ? random_id.resource_name_suffix.hex : "" + first_dash = (local.prefix != "" && (local.deployment != "" || local.suffix != "")) ? "-" : "" + second_dash = local.deployment != "" && local.suffix != "" ? "-" : "" + composite_name = "${local.prefix}${local.first_dash}${local.deployment}${local.second_dash}${local.suffix}" + name = local.composite_name == "" ? "no-bucket-name-provided" : local.composite_name +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket" "bucket" { + provider = google-beta + project = var.project_id + name = local.name + uniform_bucket_level_access = var.uniform_bucket_level_access + location = var.region + storage_class = var.storage_class + labels = local.labels + force_destroy = var.force_destroy + public_access_prevention = var.public_access_prevention + enable_object_retention = var.enable_object_retention + hierarchical_namespace { + enabled = var.enable_hierarchical_namespace + } + + dynamic "autoclass" { + for_each = var.autoclass.enabled ? [1] : [] + content { + enabled = var.autoclass.enabled + terminal_storage_class = var.autoclass.terminal_storage_class + } + } + + dynamic "soft_delete_policy" { + for_each = var.soft_delete_retention_duration == null ? [] : [1] + content { + retention_duration_seconds = var.soft_delete_retention_duration + } + } + + dynamic "retention_policy" { + for_each = var.retention_policy_period == null ? [] : [1] + content { + retention_period = var.retention_policy_period + } + } + + dynamic "versioning" { + for_each = var.enable_versioning ? [1] : [] + content { + enabled = var.enable_versioning + } + } + + dynamic "lifecycle_rule" { + for_each = var.lifecycle_rules + content { + action { + type = lifecycle_rule.value.action.type + storage_class = lookup(lifecycle_rule.value.action, "storage_class", null) + } + condition { + age = lookup(lifecycle_rule.value.condition, "age", null) + send_age_if_zero = lookup(lifecycle_rule.value.condition, "send_age_if_zero", null) + created_before = lookup(lifecycle_rule.value.condition, "created_before", null) + with_state = lookup(lifecycle_rule.value.condition, "with_state", contains(keys(lifecycle_rule.value.condition), "is_live") ? (lifecycle_rule.value.condition["is_live"] ? "LIVE" : null) : null) + matches_storage_class = lifecycle_rule.value.condition["matches_storage_class"] != null ? split(",", lifecycle_rule.value.condition["matches_storage_class"]) : null + matches_prefix = lifecycle_rule.value.condition["matches_prefix"] != null ? split(",", lifecycle_rule.value.condition["matches_prefix"]) : null + matches_suffix = lifecycle_rule.value.condition["matches_suffix"] != null ? split(",", lifecycle_rule.value.condition["matches_suffix"]) : null + num_newer_versions = lookup(lifecycle_rule.value.condition, "num_newer_versions", null) + custom_time_before = lookup(lifecycle_rule.value.condition, "custom_time_before", null) + days_since_custom_time = lookup(lifecycle_rule.value.condition, "days_since_custom_time", null) + days_since_noncurrent_time = lookup(lifecycle_rule.value.condition, "days_since_noncurrent_time", null) + noncurrent_time_before = lookup(lifecycle_rule.value.condition, "noncurrent_time_before", null) + } + } + } + + lifecycle { + precondition { + condition = !var.autoclass.enabled || !var.enable_hierarchical_namespace + error_message = "Hierarchical namespace is not compatible with Autoclass enabled." + } + + precondition { + condition = !var.enable_hierarchical_namespace || var.uniform_bucket_level_access + error_message = "Hierarchical namespace is not compatible with Uniform bucket level access disabled." + } + + precondition { + condition = !var.enable_versioning || !var.enable_hierarchical_namespace + error_message = "Hierarchical namespace is not compatible with Object versioning enabled." + } + } +} + +resource "google_storage_bucket_iam_binding" "viewers" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.objectViewer" + members = var.viewers +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf new file mode 100644 index 0000000000..29ddfef2d2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a remote network storage to be mounted by fs-tab." + value = { + remote_mount = local.name + local_mount = var.local_mount + fs_type = "gcsfuse" + mount_options = var.mount_options + server_ip = "" + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } +} + +locals { + client_install_runner = { + "type" = "shell" + "content" = file("${path.module}/scripts/install-gcs-fuse.sh") + "destination" = "install-gcsfuse${replace(var.local_mount, "/", "_")}.sh" + } + + mount_runner = { + "type" = "shell" + "destination" = "mount_gcs${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"not-used\" \"${local.name}\" \"${var.local_mount}\" \"gcsfuse\" \"${var.mount_options}\"" + "content" = file("${path.module}/scripts/mount.sh") + } +} + +output "client_install_runner" { + description = "Runner that performs client installation needed to use gcs fuse." + value = local.client_install_runner +} + +output "mount_runner" { + description = "Runner that mounts the cloud storage bucket with gcs fuse." + value = local.mount_runner +} + +output "gcs_bucket_path" { + description = "The gsutil bucket path with format of `gs://`." + # cannot use resource attribute, will cause lookup failure in startup-script + value = "gs://${local.name}" + + # needed to make sure bucket contents are deleted before bucket + depends_on = [ + google_storage_bucket.bucket + ] +} + +output "gcs_bucket_name" { + description = "Bucket name." + value = google_storage_bucket.bucket.name +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh new file mode 100644 index 0000000000..f8a990260b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ ! "$(which gcsfuse)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then + tee /etc/yum.repos.d/gcsfuse.repo >/dev/null </dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf new file mode 100644 index 0000000000..9804e4b268 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf @@ -0,0 +1,254 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which GCS bucket will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment; used as part of name of the GCS bucket." + type = string +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "labels" { + description = "Labels to add to the GCS bucket. Key-value pairs." + type = map(string) +} + +variable "local_mount" { + description = "The mount point where the contents of the device may be accessed after mounting." + type = string + default = "/mnt" +} + +variable "mount_options" { + description = "Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories)" + type = string + default = "defaults,_netdev,implicit_dirs" +} + +variable "name_prefix" { + description = "Name Prefix." + type = string + default = null +} + +variable "use_deployment_name_in_bucket_name" { + description = "If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments." + type = bool + default = true +} + +variable "random_suffix" { + description = "If true, a random id will be appended to the suffix of the bucket name." + type = bool + default = false +} + +variable "force_destroy" { + description = "If true will destroy bucket with all objects stored within." + type = bool + default = false +} + +variable "viewers" { + description = "A list of additional accounts that can read packages from this bucket" + type = set(string) + default = [] + + validation { + error_message = "All bucket viewers must be in IAM style: user:user@example.com, serviceAccount:sa@example.com, or group:group@example.com." + condition = alltrue([ + for viewer in var.viewers : length(regexall("^(user|serviceAccount|group):", viewer)) > 0 + ]) + } +} + +variable "enable_hierarchical_namespace" { + description = "If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket." + type = bool + default = false +} + +variable "uniform_bucket_level_access" { + description = "Allow uniform control access to the bucket." + type = bool + default = true +} + +variable "storage_class" { + description = "The storage class of the GCS bucket." + type = string + default = "REGIONAL" + validation { + condition = contains([ + "STANDARD", + "MULTI_REGIONAL", + "REGIONAL", + "NEARLINE", + "COLDLINE", + "ARCHIVE" + ], var.storage_class) + error_message = "Allowed values for GCS storage_class are 'STANDARD', 'MULTI_REGIONAL', 'REGIONAL', 'NEARLINE', 'COLDLINE', 'ARCHIVE'.\nhttps://cloud.google.com/storage/docs/storage-classes" + } +} + +variable "autoclass" { + description = <<-EOT + Configure bucket autoclass setup + + The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern. + + The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time. + Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE') + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/autoclass + EOT + type = object({ + enabled = optional(bool, false) + terminal_storage_class = optional(string, null) + }) + default = { + enabled = false + } + nullable = false + validation { + condition = !can(coalesce(var.autoclass.terminal_storage_class)) || var.autoclass.enabled + error_message = "Cannot set bucket var.autoclass.terminal_storage_class unless var.autoclass.enabled is true" + } +} + +variable "public_access_prevention" { + description = <<-EOT + Bucket public access can be controlled by setting a value of either `inherited` or `enforced`. + When set to `enforced`, public access to the bucket is blocked. + If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/public-access-prevention + EOT + type = string + default = null + validation { + condition = var.public_access_prevention == null ? true : contains([ + "inherited", + "enforced" + ], var.public_access_prevention) + error_message = "Allowed values for public_access_prevention are 'inherited', 'enforced'.\n" + } +} + +variable "soft_delete_retention_duration" { + description = <<-EOT + If defined, this will configure soft_delete_policy with retention_duration_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days). + Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/soft-delete + EOT + type = number + default = null + validation { + condition = var.soft_delete_retention_duration == null ? true : var.soft_delete_retention_duration == 0 || var.soft_delete_retention_duration >= 604800 && var.soft_delete_retention_duration <= 7776000 + error_message = "var.soft_delete_retention_duration value can be 0 or in between 604800(7 days) and 7776000(90 days)." + } +} + +variable "enable_versioning" { + description = "If true, enables versioning for the bucket." + type = bool + default = false +} + +variable "lifecycle_rules" { + description = "List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule" + type = list(object({ + # Object with keys: + # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass. + # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule. + action = object({ + type = string + storage_class = optional(string) + }) + + # Object with keys: + # - age - (Optional) Minimum age of an object in days to satisfy this condition. + # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field. + # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition. + # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY". + # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY. + # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition. + # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition. + # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition. + # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition. + # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true. + # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object. + # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent. + condition = object({ + age = optional(number) + send_age_if_zero = optional(bool) + created_before = optional(string) + with_state = optional(string) + matches_storage_class = optional(string) + matches_prefix = optional(string) + matches_suffix = optional(string) + num_newer_versions = optional(number) + custom_time_before = optional(string) + days_since_custom_time = optional(number) + days_since_noncurrent_time = optional(number) + noncurrent_time_before = optional(string) + }) + })) + default = [] +} + +variable "retention_policy_period" { + description = <<-EOT + If defined, this will configure retention_policy with retention_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/bucket-lock + EOT + type = number + default = null + validation { + condition = var.retention_policy_period == null ? true : var.retention_policy_period > 0 && var.retention_policy_period <= 3155760000 + error_message = "var.soft_delete_policy_retention_duration value must be in between 1 and 3155760000(100 years) seconds." + } +} + +variable "enable_object_retention" { + description = <<-EOT + If true, enables retention policy at per object level for the bucket. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/object-lock + EOT + type = bool + default = false +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf new file mode 100644 index 0000000000..217ee2f3a2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.9.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" + } + required_version = ">= 0.14.0" +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/README.md new file mode 100644 index 0000000000..3bf251828e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/README.md @@ -0,0 +1,248 @@ +## Description + +This module creates a [filestore](https://cloud.google.com/filestore) +instance. Filestore is a high performance network file system that can be +mounted to one or more compute VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +### Deletion protection + +We recommend considering enabling [Filestore deletion protection][fdp]. Deletion +protection will prevent unintentional deletion of an entire Filestore instance. +It does not prevent deletion of files within the Filestore instance when mounted +by a VM. It is not available on some [tiers](#filestore-tiers), including the +default BASIC\_HDD tier or BASIC\_SSD tier. Follow the documentation link for +up to date details. + +Usage can be enabled in a blueprint with, for example: + +```yaml + - id: homefs + source: modules/file-system/filestore + use: [network] + settings: + deletion_protection: + enabled: true + reason: Avoid data loss + filestore_tier: ZONAL + local_mount: /home + size_gb: 1024 +``` + +[fdp]: https://cloud.google.com/filestore/docs/deletion-protection + +### Filestore tiers + +At the time of writing, Filestore supports 5 [tiers of service][tiers] that are +specified in the Toolkit using the following names: + +- Basic HDD: "BASIC\_HDD" ([preferred][tierapi]) or "STANDARD" (deprecated) +- Basic SSD: "BASIC\_SSD" ([preferred][tierapi]) or "PREMIUM" (deprecated) +- Zonal: "ZONAL" +- Enterprise: "ENTERPRISE" +- Regional: "REGIONAL" + +[tierapi]: https://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier + +**Please review the minimum storage requirements for each tier**. The Terraform +module can only enforce the minimum value of the `size_gb` parameter for the +lowest tier of service. If you supply a value that is too low, Filestore +creation will fail when you run `terraform apply`. + +[tiers]: https://cloud.google.com/filestore/docs/service-tiers + +### Filestore protocols and mount options +After Filestore instance is created, you can mount this to the compute node +using different mount options. Toolkit uses [default mount options](https://linux.die.net/man/8/mount) +for all tier services. Filestore has recommended mount options for different +service tiers which may overall improve performance. These can be found here: +[recommended mount options.](https://cloud.google.com/filestore/docs/mounting-fileshares) +While creating filestore module, you can overwrite these mount options as +mentioned below. + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /homefs + mount_options: defaults,hard,timeo=600,retrans=3,_netdev +``` + +Filestore supports NFS protocols `NFS_V3` (default) and `NFS_V4_1`. Protocol support depends on the selected tier: +- `NFS_V3`: Supported on all tiers (`BASIC_HDD`, `BASIC_SSD`, `HIGH_SCALE_SSD`, `ZONAL`, `ENTERPRISE`). +- `NFS_V4_1`: Supported only on `HIGH_SCALE_SSD`, `ZONAL`, `REGIONAL`, and `ENTERPRISE`. +This can be specified at creation time via the `protocol` variable. By default, `NFS_V3` is used for compatibility. +See the example below and [this page](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#protocol-1) for more information. + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /homefs + protocol: NFS_V4_1 + filestore_tier: ZONAL +``` + +### Filestore quota + +Your project must have unused quota for Cloud Filestore in the region you will +provision the storage. This can be found by browsing to the [Quota tab within IAM +& Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. +Please note that there are separate quota limits for HDD and SSD storage. + +All projects begin with 0 available quota for High Scale SSD tier. To use this +tier, [make a request and wait for it to be approved][hs-ssd-quota]. + +[hs-ssd-quota]: https://cloud.google.com/filestore/docs/high-scale + +### Example - Basic HDD + +The Filestore instance defined below will have the following attributes: + +- (default) `BASIC_HDD` tier +- (default) 1TiB capacity +- `homefs` module ID +- mount point at `/home` +- connected to the network defined in the `network1` module + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /home +``` + +### Example - High Scale SSD + +The Filestore instance defined below will have the following attributes: + +- `HIGH_SCALE_SSD` tier +- 10TiB capacity +- `highscale` module ID +- mount point at `/projects` +- connected to the VPC network defined in the `network1` module + +```yaml +- id: highscale + source: modules/file-system/filestore + use: [network1] + settings: + filestore_tier: HIGH_SCALE_SSD + size_gb: 10240 + local_mount: /projects +``` + +## Mounting + +To mount the Filestore instance you must first ensure that the NFS client has +been installed and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +See the [hpc-slurm](../../../examples/hpc-slurm.yaml) for +an example of using this module with Slurm. + +If mounting is not automatically handled as described above, the `filestore` +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + - id: filestore + source: modules/file-system/filestore + use: [network1] + settings: {local_mount: /scratch} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(filestore.install_nfs_client_runner) + - $(filestore.mount_runner) + +``` + +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 6.4 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.4 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_filestore_instance.filestore_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [connect\_mode](#input\_connect\_mode) | Used to select mode - supported values DIRECT\_PEERING and PRIVATE\_SERVICE\_ACCESS. | `string` | `"DIRECT_PEERING"` | no | +| [deletion\_protection](#input\_deletion\_protection) | Configure Filestore instance deletion protection | 
object({
    enabled = optional(bool, false)
    reason  = optional(string)
  })
| 
{
  "enabled": false
}
| no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the filestore instance if no name is specified. | `string` | n/a | yes | +| [description](#input\_description) | A description of the filestore instance. | `string` | `""` | no | +| [filestore\_share\_name](#input\_filestore\_share\_name) | Name of the file system share on the instance. | `string` | `"nfsshare"` | no | +| [filestore\_tier](#input\_filestore\_tier) | The service tier of the instance. | `string` | `"BASIC_HDD"` | no | +| [labels](#input\_labels) | Labels to add to the filestore instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mount](#input\_local\_mount) | Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md)). | `string` | `"/shared"` | no | +| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"defaults,_netdev"` | no | +| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [nfs\_export\_options](#input\_nfs\_export\_options) | Define NFS export options. | 
list(object({
    access_mode = optional(string)
    ip_ranges   = optional(list(string))
    squash_mode = optional(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | ID of project in which Filestore instance will be created. | `string` | n/a | yes | +| [protocol](#input\_protocol) | NFS protocol version. Default is NFS\_V3. NFS\_V4\_1 is only supported with HIGH\_SCALE\_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers. | `string` | `"NFS_V3"` | no | +| [region](#input\_region) | Location for Filestore instances at Enterprise tier. | `string` | n/a | yes | +| [reserved\_ip\_range](#input\_reserved\_ip\_range) | Reserved IP range for Filestore instance. Users are encouraged to set to null
for automatic selection. If supplied, it must be:

CIDR format when var.connect\_mode == "DIRECT\_PEERING"
Named IP Range when var.connect\_mode == "PRIVATE\_SERVICE\_ACCESS"

See Cloud documentation for more details:

https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range | `string` | `null` | no | +| [size\_gb](#input\_size\_gb) | Storage size of the filestore instance in GB. | `number` | `1024` | no | +| [zone](#input\_zone) | Location for Filestore instances below Enterprise tier. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | +| [filestore\_id](#output\_filestore\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | +| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | +| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | +| [network\_storage](#output\_network\_storage) | Describes a filestore instance. | + diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/main.tf new file mode 100644 index 0000000000..ce035dbb2b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/main.tf @@ -0,0 +1,116 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "filestore", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + is_high_capacity_tier = contains(["HIGH_SCALE_SSD", "ZONAL", "REGIONAL"], var.filestore_tier) && var.size_gb >= 10240 && var.size_gb <= 102400 + + timeouts = local.is_high_capacity_tier ? [1] : [] + server_ip = google_filestore_instance.filestore_instance.networks[0].ip_addresses[0] + remote_mount = format("/%s", google_filestore_instance.filestore_instance.file_shares[0].name) + fs_type = "nfs" + mount_options = var.mount_options + + install_nfs_client_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-nfs-client.sh" + "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" + } + mount_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" + } + + # id format: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network#id + split_network_id = split("/", var.network_id) + network_name = local.split_network_id[4] + network_project = local.split_network_id[1] + shared_vpc = local.network_project != var.project_id +} + +resource "google_filestore_instance" "filestore_instance" { + project = var.project_id + + name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + description = var.description + location = contains(["ENTERPRISE", "REGIONAL"], var.filestore_tier) ? var.region : var.zone + tier = var.filestore_tier + protocol = var.protocol + + deletion_protection_enabled = var.deletion_protection.enabled + deletion_protection_reason = var.deletion_protection.reason + + file_shares { + capacity_gb = var.size_gb + name = var.filestore_share_name + dynamic "nfs_export_options" { + for_each = var.nfs_export_options + content { + access_mode = nfs_export_options.value.access_mode + ip_ranges = nfs_export_options.value.ip_ranges + squash_mode = nfs_export_options.value.squash_mode + } + } + } + + labels = local.labels + + networks { + network = local.shared_vpc ? var.network_id : local.network_name + connect_mode = var.connect_mode + modes = ["MODE_IPV4"] + reserved_ip_range = var.reserved_ip_range + } + + dynamic "timeouts" { + for_each = local.timeouts + content { + create = "1h" + update = "1h" + delete = "1h" + } + } + + lifecycle { + precondition { + condition = ( + var.reserved_ip_range == null || + var.connect_mode == "PRIVATE_SERVICE_ACCESS" || + var.connect_mode == "DIRECT_PEERING" && can(cidrhost(var.reserved_ip_range, 0)) && contains(["24", "29"], try(split("/", var.reserved_ip_range)[1], "")) + ) + error_message = <<-EOT + If connect_mode is set to DIRECT_PEERING and reserved_ip_range is + specified then it must be a CIDR IP range with suffix range size 29 for + BASIC_HDD or BASIC_SSD tiers. Otherwise the range size must be 24. + EOT + } + + precondition { + condition = !startswith(var.filestore_tier, "BASIC") || var.protocol != "NFS_V4_1" + error_message = "NFS_V4_1 is not supported on BASIC Filestore tiers." + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/metadata.yaml new file mode 100644 index 0000000000..5298336f09 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - file.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/outputs.tf new file mode 100644 index 0000000000..9bdb3bdc7b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/outputs.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a filestore instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = local.mount_options + client_install_runner = local.install_nfs_client_runner + mount_runner = local.mount_runner + } +} + +output "install_nfs_client" { + description = "Script for installing NFS client" + value = file("${path.module}/scripts/install-nfs-client.sh") +} + +output "install_nfs_client_runner" { + description = "Runner to install NFS client using the startup-script module" + value = local.install_nfs_client_runner +} + +output "mount_runner" { + description = <<-EOT + Runner to mount the file-system using an ansible playbook. The startup-script + module will automatically handle installation of ansible. + - id: example-startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(your-fs-id.mount_runner) + ... + EOT + value = local.mount_runner +} + +output "filestore_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" + value = google_filestore_instance.filestore_instance.id +} + +output "capacity_gib" { + description = "File share capacity in GiB." + value = google_filestore_instance.filestore_instance.file_shares[0].capacity_gb +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/mount.sh b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/variables.tf new file mode 100644 index 0000000000..2d7e9258c0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/variables.tf @@ -0,0 +1,189 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which Filestore instance will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the filestore instance if no name is specified." + type = string +} + +variable "zone" { + description = "Location for Filestore instances below Enterprise tier." + type = string +} + +variable "region" { + description = "Location for Filestore instances at Enterprise tier." + type = string +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is connected given in the format: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "name" { + description = "The resource name of the instance." + type = string + default = null +} + +variable "filestore_share_name" { + description = "Name of the file system share on the instance." + type = string + default = "nfsshare" +} + +variable "local_mount" { + description = "Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md))." + type = string + default = "/shared" +} + +variable "size_gb" { + description = "Storage size of the filestore instance in GB." + type = number + default = 1024 + validation { + condition = var.size_gb >= 1024 + error_message = "No Filestore tier supports less than 1024GiB.\nSee https://cloud.google.com/filestore/docs/service-tiers." + } +} + +variable "filestore_tier" { + description = "The service tier of the instance." + type = string + default = "BASIC_HDD" + validation { + condition = var.filestore_tier != "STANDARD" + error_message = "The preferred name for STANDARD tier is now BASIC_HDD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." + } + validation { + condition = var.filestore_tier != "PREMIUM" + error_message = "The preferred name for PREMIUM tier is now BASIC_SSD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." + } + validation { + condition = contains([ + "BASIC_HDD", + "BASIC_SSD", + "HIGH_SCALE_SSD", + "ZONAL", + "REGIONAL", + "ENTERPRISE" + ], var.filestore_tier) + # Avoid adding the legacy tier name in error_message, for e.g. 'HIGH_SCALE_SSD', 'ENTERPRISE'. + # As we want to steer the customer to new one's, but also support the legacy ones for older customers. + error_message = "Allowed values for filestore_tier are 'BASIC_HDD','BASIC_SSD','ZONAL','REGIONAL'.\nhttps://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#tier\nhttps://cloud.google.com/filestore/docs/reference/rest/v1/Tier." + } +} + +variable "labels" { + description = "Labels to add to the filestore instance. Key-value pairs." + type = map(string) +} + +variable "connect_mode" { + description = "Used to select mode - supported values DIRECT_PEERING and PRIVATE_SERVICE_ACCESS." + type = string + default = "DIRECT_PEERING" + nullable = false + validation { + condition = contains(["DIRECT_PEERING", "PRIVATE_SERVICE_ACCESS"], var.connect_mode) + error_message = "Allowed values for connect_mode are \"DIRECT_PEERING\" or \"PRIVATE_SERVICE_ACCESS\"." + } +} + +variable "nfs_export_options" { + description = "Define NFS export options." + type = list(object({ + access_mode = optional(string) + ip_ranges = optional(list(string)) + squash_mode = optional(string) + })) + default = [] + nullable = false +} + +variable "reserved_ip_range" { + description = <<-EOT + Reserved IP range for Filestore instance. Users are encouraged to set to null + for automatic selection. If supplied, it must be: + + CIDR format when var.connect_mode == "DIRECT_PEERING" + Named IP Range when var.connect_mode == "PRIVATE_SERVICE_ACCESS" + + See Cloud documentation for more details: + + https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range + EOT + type = string + default = null + nullable = true +} + +variable "mount_options" { + description = "NFS mount options to mount file system." + type = string + default = "defaults,_netdev" +} + +variable "deletion_protection" { + description = "Configure Filestore instance deletion protection" + type = object({ + enabled = optional(bool, false) + reason = optional(string) + }) + default = { + enabled = false + } + nullable = false + + validation { + condition = !can(coalesce(var.deletion_protection.reason)) || var.deletion_protection.enabled + error_message = "Cannot set Filestore var.deletion_protection.reason unless var.deletion_protection.enabled is true" + } +} + +variable "protocol" { + description = "NFS protocol version. Default is NFS_V3. NFS_V4_1 is only supported with HIGH_SCALE_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers." + type = string + default = "NFS_V3" + validation { + condition = contains(["NFS_V3", "NFS_V4_1"], var.protocol) + error_message = "Allowed values for protocol are 'NFS_V3' or 'NFS_V4_1'." + } +} + +variable "description" { + description = "A description of the filestore instance." + type = string + default = "" + validation { + condition = length(var.description) <= 2048 + error_message = "Filestore description must be 2048 characters or fewer" + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/versions.tf new file mode 100644 index 0000000000..1ba0e7967e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.4" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/README.md new file mode 100644 index 0000000000..88ae4511e3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/README.md @@ -0,0 +1,200 @@ +## Description + +This module creates Kubernetes Persistent Volumes (PV) and Persistent Volume +Claims (PVC) that can be used by a [gke-job-template]. + +`gke-persistent-volume` works with Filestore, Google Cloud Storage and Managed Lustre. Each +`gke-persistent-volume` can only be used with a single file system so if multiple +shared file systems are used then multiple `gke-persistent-volume` modules are +needed in the blueprint. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a Filestore and then uses the +`gke-persistent-volume` module to use the Filestore as shared storage in a +`gke-job-template`. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] + settings: + master_authorized_networks: + - display_name: deployment-machine + cidr_block: /32 + + - id: datafs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /data + + - id: datafs-pv + source: modules/file-system/gke-persistent-volume + use: [datafs, gke_cluster] + + - id: job-template + source: modules/compute/gke-job-template + use: [datafs-pv, compute_pool, gke_cluster] +``` + +The following example creates a GCS bucket and then uses the +`gke-persistent-volume` module to use the bucket as shared storage in a +`gke-job-template`. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] + settings: + master_authorized_networks: + - display_name: deployment-machine + cidr_block: /32 + + - id: data-bucket + source: modules/file-system/cloud-storage-bucket + settings: + local_mount: /data + + - id: datagcs-pv + source: modules/file-system/gke-persistent-volume + use: [data-bucket, gke_cluster] + + - id: job-template + source: modules/compute/gke-job-template + use: [datagcs-pv, compute_pool, gke_cluster] +``` + +The following example creates a Managed Lustre and then uses the +`gke-persistent-volume` module to use the Lustre as shared storage in a +`gke-job-template`. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] + settings: + master_authorized_networks: + - display_name: deployment-machine + cidr_block: /32 + + - id: data-managedlustre + source: modules/file-system/managed-lustre + settings: + local_mount: /data + + - id: datalustre-pv + source: modules/file-system/gke-persistent-volume + use: [data-managedlustre, gke_cluster] + + - id: job-template + source: modules/compute/gke-job-template + use: [datalustre-pv, compute_pool, gke_cluster] +``` + +See example +[storage-gke.yaml](../../../../examples/README.md#storage-gkeyaml--) blueprint +for a complete example. + +### Authorized Network + +Since the `gke-persistent-volume` module is making calls to the Kubernetes API +to create Kubernetes entities, the machine performing the deployment must be +authorized to connect to the Kubernetes API. You can add the +`master_authorized_networks` settings block, as shown in the example above, with +the IP address of the machine performing the deployment. This will ensure that +the deploying machine can connect to the cluster. + +### Connecting Via Use + +The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit +modules. For example the `gke-persistent-volume` module can `use` a +`gke-cluster` module and a `filestore` module, as shown in the example above. + +```mermaid + graph TD; + vpc--> |OneToMany| gke-cluster; + gke-cluster--> |OneToMany| gke-node-pool; + gke-node-pool--> |ManyToMany| gke-job-template; + gke-cluster--> |OneToMany| gke-persistent-volume; + gke-persistent-volume--> |ManyToMany| gke-job-template; + vpc--> |OneToMany| filestore; + vpc--> |OneToMany| gcs; + vpc--> |OneToMany| managed-lustre; + filestore--> |OneToOne| gke-persistent-volume; + gcs--> |OneToOne| gke-persistent-volume; + managed-lustre--> |OneToOne| gke-persistent-volume; + ``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [kubectl](#provider\_kubectl) | >= 1.7.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubectl_manifest.pv](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.pvc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.pvc_namespace](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [capacity\_gib](#input\_capacity\_gib) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | +| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | +| [filestore\_id](#input\_filestore\_id) | An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | +| [gcs\_bucket\_name](#input\_gcs\_bucket\_name) | The gcs bucket to be used with the persistent volume. | `string` | `null` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [lustre\_id](#input\_lustre\_id) | An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | +| [network\_storage](#input\_network\_storage) | Network attached storage mount to be configured. | 
object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  })
| n/a | yes | +| [pv\_name](#input\_pv\_name) | The name for PV. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | +| [pvc\_name](#input\_pvc\_name) | The name for PVC. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object describing the Kubernetes PersistentVolumeClaim created by this module. | +| [pvc\_name](#output\_pvc\_name) | The name of the Kubernetes PVC created by this module. | + diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/main.tf new file mode 100644 index 0000000000..818ebaf595 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/main.tf @@ -0,0 +1,155 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-persistent-volume", ghpc_role = "file-system" }) +} + +locals { + # Flags indicating which storage type is active based on input variables. + storage_type_active = { + gcs = var.gcs_bucket_name != null + lustre = var.lustre_id != null + filestore = var.filestore_id != null + } + + # Determine the active storage type name. + active_types = [for type, is_active in local.storage_type_active : type if is_active] + + # The precondition in kubectl_manifest.pv ensures exactly one type is active. + storage_type = length(local.active_types) > 0 ? local.active_types[0] : "unknown" + + # Map containing the base name derivation logic for each storage type. + base_name_map = { + gcs = var.gcs_bucket_name + lustre = var.lustre_id != null ? split("/", var.lustre_id)[5] : null + filestore = var.filestore_id != null ? split("/", var.filestore_id)[5] : null + } + # Retrieve the base name for the active storage type. + base_name = local.base_name_map[local.storage_type] + + # PV and PVC names + pv_name = var.pv_name != null ? var.pv_name : "${local.base_name}-pv" + pvc_name = var.pvc_name != null ? var.pvc_name : "${local.base_name}-pvc" + + # Template file paths + pv_templates = { + gcs = "${path.module}/templates/gcs-pv.yaml.tftpl" + lustre = "${path.module}/templates/managed-lustre-pv.yaml.tftpl" + filestore = "${path.module}/templates/filestore-pv.yaml.tftpl" + } + pvc_templates = { + gcs = "${path.module}/templates/gcs-pvc.yaml.tftpl" + lustre = "${path.module}/templates/managed-lustre-pvc.yaml.tftpl" + filestore = "${path.module}/templates/filestore-pvc.yaml.tftpl" + } + + # Common variables for all PVC templates + common_pvc_vars = { + pv_name = local.pv_name + pvc_name = local.pvc_name + labels = local.labels + capacity = "${var.capacity_gib}Gi" + namespace = var.namespace + } + + # Common variables for all PV templates + common_pv_vars = { + pv_name = local.pv_name + capacity = "${var.capacity_gib}Gi" + labels = local.labels + } + + # Variables for PV templates, merging common vars with type-specific ones. + pv_template_vars = { + gcs = merge(local.common_pv_vars, { + mount_options = var.gcs_bucket_name != null ? split(",", var.network_storage.mount_options) : [] + bucket_name = var.gcs_bucket_name + namespace = var.namespace + pvc_name = local.pvc_name + }) + lustre = merge(local.common_pv_vars, { + location = var.lustre_id != null ? split("/", var.lustre_id)[3] : null + project = split("/", var.cluster_id)[1] + instance_name = local.base_name + server_ip = var.lustre_id != null ? split("@", var.network_storage.server_ip)[0] : null + filesystem_name = var.network_storage.remote_mount + pvc_name = local.pvc_name + namespace = var.namespace + }) + filestore = merge(local.common_pv_vars, { + location = var.filestore_id != null ? split("/", var.filestore_id)[3] : null + filestore_name = local.base_name + share_name = trimprefix(var.network_storage.remote_mount, "/") + ip_address = var.network_storage.server_ip + pvc_name = local.pvc_name + namespace = var.namespace + }) + } + + # Rendered YAML contents + pv_content = templatefile( + local.pv_templates[local.storage_type], + local.pv_template_vars[local.storage_type] + ) + pvc_content = templatefile( + local.pvc_templates[local.storage_type], + local.common_pvc_vars + ) + + # GKE Cluster details + cluster_name = split("/", var.cluster_id)[5] + cluster_location = split("/", var.cluster_id)[3] +} + +data "google_container_cluster" "gke_cluster" { + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +provider "kubectl" { + host = "https://${data.google_container_cluster.gke_cluster.endpoint}" + cluster_ca_certificate = base64decode(data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) + token = data.google_client_config.default.access_token + load_config_file = false +} + +resource "kubectl_manifest" "pvc_namespace" { + count = var.namespace != "default" ? 1 : 0 + + yaml_body = templatefile("${path.module}/templates/namespace.yaml.tftpl", { + namespace = var.namespace + }) +} + +resource "kubectl_manifest" "pv" { + yaml_body = local.pv_content + + lifecycle { + precondition { + condition = length(local.active_types) == 1 + error_message = "Exactly one of gcs_bucket_name, filestore_id, or lustre_id must be set." + } + } +} + +resource "kubectl_manifest" "pvc" { + yaml_body = local.pvc_content + depends_on = [kubectl_manifest.pv, kubectl_manifest.pvc_namespace] +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf new file mode 100644 index 0000000000..60cf2dbe0f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "persistent_volume_claims" { + description = "An object describing the Kubernetes PersistentVolumeClaim created by this module." + value = { + name = local.pvc_name + namespace = var.namespace + mount_path = var.network_storage.local_mount + mount_options = var.network_storage.mount_options + storage_type = local.storage_type + } +} + +output "pvc_name" { + description = "The name of the Kubernetes PVC created by this module." + value = local.pvc_name +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl new file mode 100644 index 0000000000..06a1276c1e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl @@ -0,0 +1,26 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${pv_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + storageClassName: "" + capacity: + storage: ${capacity} + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + volumeMode: Filesystem + csi: + driver: filestore.csi.storage.gke.io + volumeHandle: "modeInstance/${location}/${filestore_name}/${share_name}" + volumeAttributes: + ip: ${ip_address} + volume: ${share_name} + claimRef: + name: ${pvc_name} + namespace: ${namespace} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl new file mode 100644 index 0000000000..83cfb3bc8c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl @@ -0,0 +1,18 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: ${pv_name} + resources: + requests: + storage: ${capacity} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl new file mode 100644 index 0000000000..aa0e570a8b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${pv_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + storageClassName: "" + capacity: + storage: ${capacity} + accessModes: + - ReadWriteMany + %{~ if mount_options != null ~} + mountOptions: + %{~ for key in mount_options ~} + - ${key} + %{~ endfor ~} + %{~ endif ~} + csi: + driver: gcsfuse.csi.storage.gke.io + volumeHandle: ${bucket_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl new file mode 100644 index 0000000000..4d02c85629 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: ${pv_name} + resources: + requests: + storage: ${capacity} + claimRef: + name: ${pvc_name} + namespace: ${namespace} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl new file mode 100644 index 0000000000..2b3b5e7738 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl @@ -0,0 +1,26 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${pv_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + storageClassName: "" + capacity: + storage: ${capacity} + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + volumeMode: Filesystem + claimRef: + namespace: ${namespace} + name: ${pvc_name} + csi: + driver: lustre.csi.storage.gke.io + volumeHandle: "${project}/${location}/${instance_name}/default-pool/default-container" + volumeAttributes: + ip: ${server_ip} + filesystem: ${filesystem_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl new file mode 100644 index 0000000000..83cfb3bc8c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl @@ -0,0 +1,18 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: ${pv_name} + resources: + requests: + storage: ${capacity} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl new file mode 100644 index 0000000000..fa7647e33f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf new file mode 100644 index 0000000000..fd281756e7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf @@ -0,0 +1,93 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "cluster_id" { + description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" + type = string +} + +variable "network_storage" { + description = "Network attached storage mount to be configured." + type = object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + }) +} + +variable "filestore_id" { + description = "An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." + type = string + default = null + validation { + condition = ( + var.filestore_id == null || + try(length(split("/", var.filestore_id)), 0) == 6 + ) + error_message = "filestore_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." + } +} + +variable "lustre_id" { + description = "An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." + type = string + default = null + validation { + condition = ( + var.lustre_id == null || + try(length(split("/", var.lustre_id)), 0) == 6 + ) + error_message = "lustre_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." + } +} + +variable "gcs_bucket_name" { + description = "The gcs bucket to be used with the persistent volume." + type = string + default = null +} + +variable "capacity_gib" { + description = "The storage capacity with which to create the persistent volume." + type = number +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} + +variable "namespace" { + description = "Kubernetes namespace to deploy the storage PVC/PV" + type = string + default = "default" +} + +variable "pv_name" { + description = "The name for PV. IF not set, a name will be generated based on the storage name." + type = string + default = null +} + +variable "pvc_name" { + description = "The name for PVC. IF not set, a name will be generated based on the storage name." + type = string + default = null +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf new file mode 100644 index 0000000000..fa1c3e2b3f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf @@ -0,0 +1,30 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.7.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.74.0" + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/README.md new file mode 100644 index 0000000000..78ef5402aa --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/README.md @@ -0,0 +1,134 @@ +## Description + +This module creates Kubernetes Storage Class (SC) that can be used by a Persistent Volume Claim (PVC) +to dynamically provision GCP storage resources like Parallelstore. + +### Example + +The following example uses the `gke-storage` module to creates a Parallelstore Storage Class and Persistent Volume Claim, +then use them in a `gke-job-template` to dynamically provision the resource. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network] + settings: + enable_parallelstore_csi: true + + # Private Service Access (PSA) requires the compute.networkAdmin role which is + # included in the Owner role, but not Editor. + # PSA is required for all Parallelstore functionality. + # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: gke_storage + source: modules/file-system/gke-storage + use: [ gke_cluster, private_service_access ] + settings: + storage_type: Parallelstore + access_mode: ReadWriteMany + sc_volume_binding_mode: Immediate + sc_reclaim_policy: Delete + sc_topology_zones: [$(vars.zone)] + pvc_count: 2 + capacity_gb: 12000 + + - id: job_template + source: modules/compute/gke-job-template + use: [gke_storage, compute_pool] +``` + +See example +[gke-managed-parallelstore.yaml](../../../examples/README.md#gke-managed-parallelstoreyaml--) blueprint +for a complete example. + +### Authorized Network + +Since the `gke-storage` module is making calls to the Kubernetes API +to create Kubernetes entities, the machine performing the deployment must be +authorized to connect to the Kubernetes API. You can add the +`master_authorized_networks` settings block, as shown in the example above, with +the IP address of the machine performing the deployment. This will ensure that +the deploying machine can connect to the cluster. + +### Connecting Via Use + +The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit +modules. For example the `gke-storage` module can `use` a +`gke-cluster` module and a `private_service_access` module, as shown in the example above. + +```mermaid +graph TD; + vpc-->|OneToMany|gke-cluster; + gke-cluster-->|OneToMany|gke-node-pool; + gke-node-pool-->|ManyToMany|gke-job-template; + gke-cluster-->|OneToMany|gke-storage; + gke-storage-->|ManyToMany|gke-job-template; +``` + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_mode](#input\_access\_mode) | The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)
Valid access modes:
- ReadWriteOnce
- ReadOnlyMany
- ReadWriteMany
- ReadWriteOncePod | `string` | n/a | yes | +| [capacity\_gb](#input\_capacity\_gb) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | +| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [mount\_options](#input\_mount\_options) | Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class. | `string` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [pv\_mount\_path](#input\_pv\_mount\_path) | Path within the container at which the volume should be mounted. Must not contain ':'. | `string` | `"/data"` | no | +| [pvc\_count](#input\_pvc\_count) | How many PersistentVolumeClaims that will be created | `number` | `1` | no | +| [sc\_reclaim\_policy](#input\_sc\_reclaim\_policy) | Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted.
[More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming)
Supported value:
- Retain
- Delete | `string` | n/a | yes | +| [sc\_topology\_zones](#input\_sc\_topology\_zones) | Zone location that allow the volumes to be dynamically provisioned. | `list(string)` | `null` | no | +| [sc\_volume\_binding\_mode](#input\_sc\_volume\_binding\_mode) | Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound.
Supported value:
- Immediate
- WaitForFirstConsumer | `string` | `"WaitForFirstConsumer"` | no | +| [storage\_type](#input\_storage\_type) | The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview)
to used. This module currently support dynamic provisioning for the below storage options
- Parallelstore
- Hyperdisk-balanced
- Hyperdisk-throughput
- Hyperdisk-extreme | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object that describes a k8s PVC created by this module. | + diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/main.tf new file mode 100644 index 0000000000..9c9a641f79 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/main.tf @@ -0,0 +1,86 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-storage", ghpc_role = "file-system" }) +} + +locals { + storage_type = lower(var.storage_type) + storage_class_name = "${local.storage_type}-sc" + pvc_name_prefix = "${local.storage_type}-pvc" +} + +check "private_vpc_connection_peering" { + assert { + condition = lower(var.storage_type) != "parallelstore" ? true : var.private_vpc_connection_peering != null + error_message = <<-EOT + Parallelstore must be run within the same VPC as the GKE cluster and have private services access enabled. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access. + If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). + EOT + } +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + # count = var.pvc_count + apply_manifests = flatten( + [ + # create StorageClass in the cluster + { + content = templatefile( + "${path.module}/storage-class/${local.storage_class_name}.yaml.tftpl", + { + name = local.storage_class_name + labels = local.labels + volume_binding_mode = var.sc_volume_binding_mode + reclaim_policy = var.sc_reclaim_policy + topology_zones = var.sc_topology_zones + }) + }, + var.namespace != "default" ? [{ + content = templatefile( + "${path.module}/persistent-volume-claim/namespace.yaml.tftpl", + { + namespace = var.namespace + }) + }] : [], + # create PersistentVolumeClaim in the cluster + flatten([ + for idx in range(var.pvc_count) : [ + { + content = templatefile( + "${path.module}/persistent-volume-claim/${(local.pvc_name_prefix)}.yaml.tftpl", + { + pvc_name = "${local.pvc_name_prefix}-${idx}" + labels = local.labels + capacity = "${var.capacity_gb}Gi" + access_mode = var.access_mode + storage_class_name = local.storage_class_name + namespace = var.namespace + } + ) + } + ] + ]) + ]) +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/metadata.yaml new file mode 100644 index 0000000000..8722823274 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/outputs.tf new file mode 100644 index 0000000000..ce80cdb266 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/outputs.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "persistent_volume_claims" { + description = "An object that describes a k8s PVC created by this module." + value = flatten([ + for idx in range(var.pvc_count) : [{ + name = "${local.pvc_name_prefix}-${idx}" + namespace = var.namespace + mount_path = "${var.pv_mount_path}/${local.pvc_name_prefix}-${idx}" + mount_options = var.mount_options + storage_type = local.storage_type + }] + ]) +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl new file mode 100644 index 0000000000..fa7647e33f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl new file mode 100644 index 0000000000..46e1f023d3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl @@ -0,0 +1,25 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + type: hyperdisk-balanced + provisioned-throughput-on-create: "250Mi" + provisioned-iops-on-create: "7000" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl new file mode 100644 index 0000000000..445020d001 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl @@ -0,0 +1,24 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + %{~ endfor ~} + type: hyperdisk-extreme + provisioned-iops-on-create: "50000" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl new file mode 100644 index 0000000000..ec404aec45 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl @@ -0,0 +1,24 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + type: hyperdisk-throughput + provisioned-throughput-on-create: "250Mi" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl new file mode 100644 index 0000000000..e6b8ea8d3e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl @@ -0,0 +1,21 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: parallelstore.csi.storage.gke.io +parameters: +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/variables.tf new file mode 100644 index 0000000000..dba1c33b77 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/variables.tf @@ -0,0 +1,144 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" + type = string +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} + +variable "storage_type" { + description = <<-EOT + The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview) + to used. This module currently support dynamic provisioning for the below storage options + - Parallelstore + - Hyperdisk-balanced + - Hyperdisk-throughput + - Hyperdisk-extreme + EOT + type = string + nullable = false + validation { + condition = var.storage_type == null ? false : contains(["parallelstore", "hyperdisk-balanced", "hyperdisk-throughput", "hyperdisk-extreme"], lower(var.storage_type)) + error_message = "Allowed string values for var.storage_type are \"Parallelstore\", \"Hyperdisk-balanced\", \"Hyperdisk-throughput\", \"Hyperdisk-extreme\"." + } +} + +variable "access_mode" { + description = <<-EOT + The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) + Valid access modes: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + EOT + type = string + nullable = false + validation { + condition = var.access_mode == null ? false : contains(["readwriteonce", "readonlymany", "readwritemany", "readwriteoncepod"], lower(var.access_mode)) + error_message = "Allowed string values for var.access_mode are \"ReadWriteOnce\", \"ReadOnlyMany\", \"ReadWriteMany\", \"ReadWriteOncePod\"." + } +} + +variable "sc_volume_binding_mode" { + description = <<-EOT + Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound. + Supported value: + - Immediate + - WaitForFirstConsumer + EOT + type = string + default = "WaitForFirstConsumer" + validation { + condition = var.sc_volume_binding_mode == null ? true : contains(["immediate", "waitforfirstconsumer"], lower(var.sc_volume_binding_mode)) + error_message = "Allowed string values for var.sc_volume_binding_mode are \"Immediate\", \"WaitForFirstConsumer\"." + } +} + +variable "sc_reclaim_policy" { + description = <<-EOT + Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted. + [More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming) + Supported value: + - Retain + - Delete + EOT + type = string + nullable = false + validation { + condition = var.sc_reclaim_policy == null ? true : contains(["retain", "delete"], lower(var.sc_reclaim_policy)) + error_message = "Allowed string values for var.sc_reclaim_policy are \"Retain\", \"Delete\"." + } +} + +variable "sc_topology_zones" { + description = "Zone location that allow the volumes to be dynamically provisioned." + type = list(string) + default = null +} + +variable "pvc_count" { + description = "How many PersistentVolumeClaims that will be created" + type = number + default = 1 +} + +variable "pv_mount_path" { + description = "Path within the container at which the volume should be mounted. Must not contain ':'." + type = string + default = "/data" + validation { + condition = var.pv_mount_path == null ? true : !strcontains(var.pv_mount_path, ":") + error_message = "pv_mount_path must not contain ':', please correct it and retry" + } +} + +variable "mount_options" { + description = "Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class." + type = string + default = null +} + +variable "capacity_gb" { + description = "The storage capacity with which to create the persistent volume." + type = number +} + +variable "private_vpc_connection_peering" { + description = <<-EOT + The name of the VPC Network peering connection. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access and + If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). + EOT + type = string + default = null +} + +variable "namespace" { + description = "Kubernetes namespace to deploy the storage PVC/PV" + type = string + default = "default" +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/versions.tf new file mode 100644 index 0000000000..bcc803e41e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/versions.tf @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.74.0" + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/README.md new file mode 100644 index 0000000000..28530a379f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/README.md @@ -0,0 +1,289 @@ +## Description + +This module creates a [Managed Lustre](https://cloud.google.com/managed-lustre) +instance. Managed Lustre is a high performance network file system that can be +mounted to one or more VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +### Supported Operating Systems + +A Managed Lustre instance can be used with Slurm cluster or compute +VM running Ubuntu 20.04, 22.04 or Rocky Linux 8 (including the HPC flavor). + +### Managed Lustre Access + +Managed Lustre must be enabled for your project by Google staff. Please contact +your sales representative for further steps. + +### Example - New VPC + +For Managed Lustre instance, the snippet below creates new VPC and configures +private-service-access for this newly created network. Both items are required +to be passed to the Lustre module to ensure that they're built in order and +that the correct subnetwork has private service access. + +```yaml + - id: network + source: modules/network/vpc + + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] +``` + +### Example - Slurm + +When using Slurm you must take into consideration whether or not you are using +an official image from the `schedmd-slurm-public` project or building your own. +The Lustre client modules are pre-installed in the official images. With the +official images, Lustre can be used as follows: + +```yaml +- id: managed_lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + +# Other modules: nodesets, partitions, login, etc. + +- id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: + - network + - lustre_partition + - managed_lustre + - slurm_login + settings: + machine_type: n2-standard-4 + enable_controller_public_ips: true +``` + +For custom images you must install the modules during the image build as the +Slurm cluster will not run the installation script like it does for the +standard VMs. + +Assuming you have a startup script for the Slurm image building, you can add +this Ansible playbook to correctly install the Lustre drivers into the image +(for Slurm-GCP versions greater than 6.10.0): + +```yaml +- type: data + destination: /var/tmp/slurm_vars.json + content: | + { + "reboot": false, + "install_cuda": false, + "install_gcsfuse": true, + "install_lustre": false, + "install_managed_lustre": true, + "install_nvidia_repo": true, + "install_ompi": true, + "allow_kernel_upgrades": false, + "monitoring_agent": "cloud-ops", + } +``` + +The `install_managed_lustre: true` line specifies that slurm-gcp should install +the correct modules within the slurm image. This runner should be placed +ahead of the script that calls the ansible build of the slurm-gcp image. + +### Example - Existing VPC + +If you want to use existing network with private-service-access configured, you need +to manually provide `private_vpc_connection_peering` to the Managed Lustre module. +You can get this details from the Google Cloud Console UI in `VPC network peering` +section. Below is the example of using existing network and creating Managed Lustre. +If existing network is not configured with private-service-access, you can follow +[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) +to set it up. + +```yaml + - id: network + source: modules/network/pre-existing-vpc + settings: + network_name: // Add network name + subnetwork_name: // Add subnetwork name + + - id: lustre + source: modules/file-system/managed-lustre + use: [network] + settings: + private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" +``` + +### Example - GKE compatibility + +By default the Managed Lustre instance that is deployed is not compatible with +GKE. To enable the compatibility use the `gke_support_enabled: true` option. +This creates a file `/etc/modprobe/lnet.conf` that changes the listening port +to 6988. + +```yaml + - id: managed-lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + gke_support_enabled: true +``` + +> [!WARNING] +> +> 1. VMs cannot connect to both GKE compatible and GKE incompatible lustre +> instances at the same time as they connect to different ports. Lustre can +> only listen to one port at a time. +> +> 2. Setting `gke_support_enabled: true` will not affect Slurm nodes, GKE +> compatibility must be built into the Slurm image. + +### Example - Importing data from GSC Bucket + +One option with the Managed Lustre instance is to import data from a GSC bucket +upon the lustre instance creation. To do this, use the `import_gcs_bucket_uri` +variable to dictate the bucket to pull data from. The data will be imported +under the directory specified by `local_mount` (`/shared` if unspecified). + +> [!NOTE] +> +> 1. This is a one way operation. Once the data has been copied to the lustre +> instance it will not be updated with any changes made to the GCS bucket. +> +> 2. Once the lustre instance has been created in Terraform, the copy process +> will proceed in the background. Data may not be appear in the mounted +> directory for a period of time after the deployment has completed (see below). + +```yaml +- id: managed_lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + import_gcs_bucket_uri: gs:// +``` + +> [!WARNING] +> Please follow [this guide](https://cloud.google.com/managed-lustre/docs/transfer-data#required_permissions) +> to set up the correct IAM permissions for importing data from GCS to lustre. +> Without this, the copy process may fail silently leaving an empty lustre +> instance. + +If an import is requested, gcluster will output a json response similar to: + +```json +{ + "name": "projects//locations//operations/", + "metadata": { + "@type": "type.googleapis.com/google.cloud.lustre.v1.ImportDataMetadata", + "createTime": "", + "target": "projects//locations//instances/", + "requestedCancellation": false, + "apiVersion": "v1" + }, + "done": false +} +``` + +You can retrieve more information about the transfer using the following +command, substituting with values from the json response above: + +```bash +gcloud lustre operations describe --location --project +``` + +This will provide information on if the transfer is complete or if any errors +have occurred. See more at +[Get operation](https://cloud.google.com/managed-lustre/docs/transfer-data#get_operation). + +## License + + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 6.27.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.27.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_lustre_instance.lustre_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/lustre_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | +| [google_storage_bucket.lustre_import_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the Lustre instance if no name is specified. | `string` | n/a | yes | +| [description](#input\_description) | Description of the created Lustre instance. | `string` | `"Lustre Instance"` | no | +| [gke\_support\_enabled](#input\_gke\_support\_enabled) | Set to true to create Managed Lustre instance with GKE compatibility.
Note: This does not work with Slurm, the Slurm image must be built with
the correct compatibility. | `bool` | `false` | no | +| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to managed lustre. Data will
be imported to the local\_mount directory. Changing this value will not
trigger a redeployment, to prevent data deletion. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the Managed Lustre instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mount](#input\_local\_mount) | Local mount point for the Managed Lustre instance. | `string` | `"/shared"` | no | +| [mount\_options](#input\_mount\_options) | Mounting options for the file system. | `string` | `"defaults,_netdev"` | no | +| [name](#input\_name) | Name of the Lustre instance | `string` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | Network self-link this instance will be on, required for checking private service access | `string` | n/a | yes | +| [per\_unit\_storage\_throughput](#input\_per\_unit\_storage\_throughput) | Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000. | `number` | `500` | no | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | ID of project in which Lustre instance will be created. | `string` | n/a | yes | +| [remote\_mount](#input\_remote\_mount) | Remote mount point of the Managed Lustre instance | `string` | n/a | yes | +| [size\_gib](#input\_size\_gib) | Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations | `number` | `36000` | no | +| [zone](#input\_zone) | Location for the Lustre instance. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | +| [install\_managed\_lustre\_client](#output\_install\_managed\_lustre\_client) | Script for installing Managed Lustre client | +| [lustre\_id](#output\_lustre\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | +| [network\_storage](#output\_network\_storage) | Describes a Managed Lustre instance. | + diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/main.tf new file mode 100644 index 0000000000..a969c53673 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/main.tf @@ -0,0 +1,104 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "managed-lustre", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +data "google_compute_network_peering" "private_peering" { + name = var.private_vpc_connection_peering + network = var.network_self_link +} + +locals { + server_ip = split(":", google_lustre_instance.lustre_instance.mount_point)[0] + remote_mount = split(":", google_lustre_instance.lustre_instance.mount_point)[1] + fs_type = "lustre" + mount_options = var.mount_options + instance_id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + destination_path = "/" + + install_managed_lustre_client_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-managed-lustre-client.sh" + "destination" = "install-managed-lustre-client${replace(var.local_mount, "/", "_")}.sh" + "args" = var.gke_support_enabled ? "1" : "0" + } + mount_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" + } + + bucket_count = try(length(data.google_storage_bucket.lustre_import_bucket), 0) +} + +data "google_storage_bucket" "lustre_import_bucket" { + count = try(length(var.import_gcs_bucket_uri) > 0, false) ? 1 : 0 + + name = split("//", var.import_gcs_bucket_uri)[1] +} + +resource "google_lustre_instance" "lustre_instance" { + project = var.project_id + + description = var.description + instance_id = local.instance_id + location = var.zone + + filesystem = var.remote_mount + capacity_gib = var.size_gib + per_unit_storage_throughput = var.per_unit_storage_throughput + + labels = local.labels + network = var.network_id + + gke_support_enabled = var.gke_support_enabled + + timeouts { + create = "1h" + update = "1h" + delete = "1h" + } + + depends_on = [var.private_vpc_connection_peering, data.google_storage_bucket.lustre_import_bucket] + + lifecycle { + precondition { + condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" + error_message = "The subnetwork that the lustre instance is hosted on must have private service access." + } + } + + provisioner "local-exec" { + command = < 0 ]]; then + curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + -d '{"gcsPath": {"uri":"${coalesce(var.import_gcs_bucket_uri, "gs://")}"}, "lustrePath": {"path":"${local.destination_path}"}}' \ + https://lustre.googleapis.com/v1/projects/${var.project_id}/locations/${var.zone}/instances/${local.instance_id}:importData + fi + EOF + interpreter = ["bash", "-c"] + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/metadata.yaml new file mode 100644 index 0000000000..66da9827b6 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - lustre.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/outputs.tf new file mode 100644 index 0000000000..6de815524a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/outputs.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a Managed Lustre instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = local.mount_options + client_install_runner = local.install_managed_lustre_client_runner + mount_runner = local.mount_runner + } +} + +output "install_managed_lustre_client" { + description = "Script for installing Managed Lustre client" + value = file("${path.module}/scripts/install-managed-lustre-client.sh") +} + +output "lustre_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" + value = google_lustre_instance.lustre_instance.id +} + +output "capacity_gib" { + description = "File share capacity in GiB." + value = google_lustre_instance.lustre_instance.capacity_gib +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh new file mode 100644 index 0000000000..878130ab47 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install Managed Lustre client modules +# Based on these instructions: https://cloud.google.com/managed-lustre/docs/connect-from-compute-engine + +# The client modules currently only support Rocky 8, and Ubuntu 20.04/22.04 + +set -e + +GKE_ENABLED=$1 + +# Update lnet to enable GKE supported Lustre instance +if [[ $GKE_ENABLED == "1" ]]; then + if [[ -f "/etc/modprobe.d/lnet.conf" ]] && grep -Fq "options lnet accept_port" /etc/modprobe.d/lnet.conf; then + echo "Lnet accept port already set, continuing without updating /etc/modprobe.d/lnet.conf" + else + echo "options lnet accept_port=6988" >>/etc/modprobe.d/lnet.conf + fi +fi + +if grep -q lustre /proc/filesystems; then + echo "Skipping managed lustre client install as it is already supported" + exit 0 +fi + +# Get distro information +. /etc/os-release +DIST="NA" +if [[ $NAME == *"Ubuntu"* ]]; then + if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then + DIST="Ubuntu" + fi +elif [[ $NAME == *"Rocky"* ]]; then + if [[ $VERSION_ID == "8"* ]]; then + DIST="Rocky" + fi +fi + +if [[ ${DIST} == "Ubuntu" ]]; then + KEY_LOC=/etc/apt/keyrings + KEY_NAME=gcp-ar-repo.gpg + # Download new repo key + mkdir -p "${KEY_LOC}" + wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null + + # Set up apt repo + echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list + + # Install modules + apt update + apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) +elif [[ ${DIST} == "Rocky" ]]; then + # Set up yum repo + touch /etc/yum.repos.d/artifact-registry.repo + tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF + [lustre-client-rocky-8] + name=lustre-client-rocky-8 + baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + # Install modules + yum makecache + yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client +fi + +if [[ $DIST != "NA" ]]; then + # Load the new lustre client module + modprobe lustre +fi diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/variables.tf new file mode 100644 index 0000000000..65607af66d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/variables.tf @@ -0,0 +1,131 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which Lustre instance will be created." + type = string +} + +variable "description" { + description = "Description of the created Lustre instance." + type = string + default = "Lustre Instance" +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the Lustre instance if no name is specified." + type = string +} + +variable "zone" { + description = "Location for the Lustre instance." + type = string +} + +variable "name" { + description = "Name of the Lustre instance" + type = string +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is connected given in the format: + `projects//global/networks/`" + EOT + type = string + nullable = false + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "network_self_link" { + description = "Network self-link this instance will be on, required for checking private service access" + type = string + nullable = false +} + +variable "remote_mount" { + description = "Remote mount point of the Managed Lustre instance" + type = string + nullable = false +} + +variable "local_mount" { + description = "Local mount point for the Managed Lustre instance." + type = string + default = "/shared" +} + +variable "size_gib" { + description = "Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations" + type = number + default = 36000 +} + +variable "per_unit_storage_throughput" { + description = "Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000." + type = number + default = 500 +} + +variable "labels" { + description = "Labels to add to the Managed Lustre instance. Key-value pairs." + type = map(string) +} + +variable "mount_options" { + description = "Mounting options for the file system." + type = string + default = "defaults,_netdev" +} + +variable "private_vpc_connection_peering" { + description = <<-EOT + The name of the VPC Network peering connection. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access and + If using existing VPC with private-service-access enabled, set this manually." + EOT + type = string + nullable = false +} + +variable "gke_support_enabled" { + description = <<-EOT + Set to true to create Managed Lustre instance with GKE compatibility. + Note: This does not work with Slurm, the Slurm image must be built with + the correct compatibility. + EOT + type = bool + nullable = false + default = false +} + +variable "import_gcs_bucket_uri" { + description = <<-EOT + The name of the GCS bucket to import data from to managed lustre. Data will + be imported to the local_mount directory. Changing this value will not + trigger a redeployment, to prevent data deletion. + EOT + type = string + default = null + + validation { + condition = startswith(coalesce(var.import_gcs_bucket_uri, "gs://"), "gs://") + error_message = "The GCS bucket uri must start with 'gs://'" + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/versions.tf new file mode 100644 index 0000000000..2322c9a8fd --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.27.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/README.md new file mode 100644 index 0000000000..82332f3406 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/README.md @@ -0,0 +1,193 @@ +## Description + +This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) +storage pool. + +NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. +NetApp Volume provides: + +- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x +- a [rich feature set][service-levels] +- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) +- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data +- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost + +Support for NetApp Volumes is split into two modules. + +- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. +- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +### NetApp storage pool service levels + +The netapp-storage-pool module currently supports the following NetApp Volumes [service levels][service-levels]: + +- Standard: 16 KiBps throughput per provisioned KiB of volume capacity. +- Premium: 64 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. +- Extreme: 128 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. + +Check the [service level matrix][service-levels] for additional information on capability differences between service levels. Flex service levels are currently not supported, but you can connect to existing Flex volumes using the [pre-existing-network-storage module][pre-existing]. + +### On-boarding NetApp Volumes +NetApp Volumes uses [Private Service Access](https://cloud.google.com/vpc/docs/private-services-access) (PSA) to connect volumes to your network. Before you create a storage pool, make sure to [connect NetApp Volumes to your network](https://cloud.google.com/netapp/volumes/docs/get-started/configure-access/networking). + +Example of creating a storage pool using a new network: + +```yaml +deployment_groups: +- group: primary + modules: + - id: network + source: modules/network/vpc + settings: + region: $(vars.region) + + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + service_name: "netapp.servicenetworking.goog" + deletion_policy: "ABANDON" + + - id: netapp_pool + source: modules/file-system/netapp-storage-pool + use: [network, private_service_access] + settings: + pool_name: $(vars.deployment_name)-eda-pool + capacity_gib: 20000 + service_level: "EXTREME" + region: $(vars.region) +``` + +Example of creating a storage pool using an existing network which was already PSA-peered with NetApp Volume: + +```yaml +deployment_groups: + - group: primary + modules: + - id: network + source: modules/network/pre-existing-vpc + settings: + project_id: $(vars.project_id) + region: $(vars.region) + network_name: $(vars.network) + + - id: netapp_pool + source: modules/file-system/netapp-storage-pool + use: [network] + settings: + pool_name: "eda-pool" + capacity_gib: 20000 + service_level: "EXTREME" + region: $(vars.region) +``` + +### Storage pool example + +The following example shows all available parameters in use: + +```yaml + - id: netapp_pool + source: modules/file-system/netapp-storage-pool + use: [network, private_service_access] + settings: + pool_name: "mypool" + region: "us-west4" + capacity_gib: 2048 + service_level: "EXTREME" + active_directory_policy: "projects/myproject/locations/us-east4/activeDirectories/my-ad" + cmek_policy: "projects/myproject/locations/us-east4/kmsConfigs/my-cmek-policy" + ldap_enabled: false + allow_auto_tiering: false + description: "Demo storage pool" + labels: + owner: bob +``` + +### NetApp Volumes quota + +Your project must have unused quota for NetApp Volumes in the region you will +provision the storage pool. This can be found by browsing to the [Quota tab within IAM & Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. +Please note that there are separate quota limits for Standard and Premium/Extreme service levels. + +See also NetApp Volumes [default quotas](https://cloud.google.com/netapp/volumes/docs/quotas#netapp-volumes-default-quotas). + +[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels +[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering +[pre-existing]: ../pre-existing-network-storage/README.md +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [google](#requirement\_google) | >= 6.45.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.45.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_netapp_storage_pool.netapp_storage_pool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_storage_pool) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [active\_directory\_policy](#input\_active\_directory\_policy) | The ID of the Active Directory policy to apply to the storage pool in the format:
`projects//locations//activeDirectoryPolicies/` | `string` | `null` | no | +| [allow\_auto\_tiering](#input\_allow\_auto\_tiering) | Whether to allow automatic tiering for the storage pool. | `bool` | `false` | no | +| [capacity\_gib](#input\_capacity\_gib) | The capacity of the storage pool in GiB. | `number` | `2048` | no | +| [cmek\_policy](#input\_cmek\_policy) | The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format:
`projects//locations//kmsConfigs/` | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used as name of the NetApp storage pool if no name is specified. | `string` | n/a | yes | +| [description](#input\_description) | A description of the NetApp storage pool. | `string` | `""` | no | +| [labels](#input\_labels) | Labels to add to the NetApp storage pool. Key-value pairs. | `map(string)` | n/a | yes | +| [ldap\_enabled](#input\_ldap\_enabled) | Whether to enable LDAP for the storage pool. | `bool` | `false` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | Network self-link the pool will be on, required for checking private service access | `string` | n/a | yes | +| [pool\_name](#input\_pool\_name) | The name of the storage pool. Leave empty to generate name based on deployment name. | `string` | `null` | no | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the private VPC connection peering. | `string` | `"sn-netapp-prod"` | no | +| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | +| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | +| [service\_level](#input\_service\_level) | The service level of the storage pool. | `string` | `"PREMIUM"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gb](#output\_capacity\_gb) | Storage pool capacity in GiB. | +| [netapp\_storage\_pool\_id](#output\_netapp\_storage\_pool\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}` | + diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/main.tf new file mode 100644 index 0000000000..b9d63c11c3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/main.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "netapp-storage-pool", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +data "google_compute_network_peering" "private_peering" { + name = var.private_vpc_connection_peering + network = var.network_self_link +} + +resource "google_netapp_storage_pool" "netapp_storage_pool" { + project = var.project_id + + name = var.pool_name != null ? var.pool_name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + location = var.region + network = var.network_id + service_level = var.service_level + capacity_gib = var.capacity_gib + + active_directory = var.active_directory_policy + kms_config = var.cmek_policy + ldap_enabled = var.ldap_enabled + allow_auto_tiering = var.allow_auto_tiering + + description = var.description + labels = local.labels + + depends_on = [data.google_compute_network_peering.private_peering] + + lifecycle { + precondition { + condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" + error_message = "The network for the storage pool must have private service access." + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml new file mode 100644 index 0000000000..7a5291f9d5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - netapp.googleapis.com + - servicenetworking.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf new file mode 100644 index 0000000000..91379631c6 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "netapp_storage_pool_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}`" + value = google_netapp_storage_pool.netapp_storage_pool.id +} + +output "capacity_gb" { + description = "Storage pool capacity in GiB." + value = google_netapp_storage_pool.netapp_storage_pool.capacity_gib +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf new file mode 100644 index 0000000000..04f19fd3fb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf @@ -0,0 +1,133 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which the NetApp storage pool will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the deployment, used as name of the NetApp storage pool if no name is specified." + type = string +} + +variable "region" { + description = "Location for NetApp storage pool." + type = string +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "network_self_link" { + description = "Network self-link the pool will be on, required for checking private service access" + type = string + nullable = false +} + +variable "private_vpc_connection_peering" { + description = "The name of the private VPC connection peering." + type = string + default = "sn-netapp-prod" +} + +variable "pool_name" { + description = "The name of the storage pool. Leave empty to generate name based on deployment name." + type = string + default = null +} + +variable "service_level" { + description = "The service level of the storage pool." + type = string + default = "PREMIUM" + validation { + condition = contains(["STANDARD", "PREMIUM", "EXTREME"], var.service_level) + error_message = "Allowed values for service_level are 'STANDARD', 'PREMIUM', or 'EXTREME'." + } +} + +variable "capacity_gib" { + description = "The capacity of the storage pool in GiB." + type = number + default = 2048 + validation { + condition = var.capacity_gib >= 2048 + error_message = "The minimum capacity for the storage pool is 2048 GiB." + } +} + +variable "active_directory_policy" { + description = <<-EOT + The ID of the Active Directory policy to apply to the storage pool in the format: + `projects//locations//activeDirectoryPolicies/` + EOT + type = string + default = null + validation { + condition = var.active_directory_policy == null ? true : length(split("/", var.active_directory_policy)) == 6 + error_message = "The active directory policy must be provided in the following format: projects//locations//activeDirectoryPolicies/." + } +} + +variable "cmek_policy" { + description = <<-EOT + The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format: + `projects//locations//kmsConfigs/` + EOT + type = string + default = null + validation { + condition = var.cmek_policy == null ? true : length(split("/", var.cmek_policy)) == 6 + error_message = "The CMEK policy must be provided in the following format: projects//locations//kmsConfigs/." + } +} + +variable "ldap_enabled" { + description = "Whether to enable LDAP for the storage pool." + type = bool + default = false +} + +variable "allow_auto_tiering" { + description = "Whether to allow automatic tiering for the storage pool." + type = bool + default = false +} + +variable "description" { + description = "A description of the NetApp storage pool." + type = string + default = "" + validation { + condition = length(var.description) <= 2048 + error_message = "NetApp storage pool description must be 2048 characters or fewer" + } +} + +variable "labels" { + description = "Labels to add to the NetApp storage pool. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf new file mode 100644 index 0000000000..f6501116cd --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.45.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" + } + + required_version = ">= 1.5.7" +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/README.md new file mode 100644 index 0000000000..6aaaf0cb05 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/README.md @@ -0,0 +1,201 @@ +## Description + +This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) +volume. + +NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. +NetApp Volume provides: + +- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x +- a [rich feature set][service-levels] +- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) +- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data +- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost + +Support for NetApp Volumes is split into two modules. + +- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. +- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +## Deletion protection +The netapp-volume module currently doesn't implement volume deletion protection. If you create a volume with Cluster Toolkit by using this module, Cluster Toolkit will also delete it when you run `gcluster destroy`. All the data in the volume will be gone. If you want to retain the volume instead, it is advised to [use existing volumes not created by Cluster Toolkit](#using-existing-volumes-not-created-by-cluster-toolkit). + +## Volumes overview +Volumes are filesystem containers which can be shared using NFS or SMB filesharing protocols. Volumes *live* inside of [storage pools](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview), which can be provisioned using the [netapp-storage-pool] module. Volumes inherit fundamental settings from the pool. They *consume* capacity provided by the pool. You can create one or multiple volumes *inside* a pool. + +[netapp-storage-pool]: ../netapp-storage-pool/README.md +[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels +[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering +[pre-existing]: ../pre-existing-network-storage/README.md +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## Volume examples +The following examples show the use of netapp-volume. They builds on top of an storage pool which can be provisioned using the [netapp-storage-pool][netapp-storage-pool] module. + +### Example with minimal parameters + +```yaml + - id: home_volume + source: modules/file-system/netapp-volume + use: [netapp_pool] # Create this pool using the netapp-storage-pool module + settings: + volume_name: "eda-home" + capacity_gib: 1024 # Size up to available capacity in the pool + local_mount: "/eda-home" # Mount point at client when client uses USE directive + protocols: ["NFSV3"] + region: $(vars.region) + # Default export policy exports to "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" and no_root_squash +``` + +### Example with all parameters + +```yaml + - id: shared_volume + source: modules/file-system/netapp-volume + use: [netapp_pool] # Create this pool using the netapp-storage-pool module + settings: + volume_name: "eda-shared" + capacity_gib: 25000 # Size up to available capacity in the pool + large_capacity: true + local_mount: "/shared" # Mount point at client when client uses USE directive + mount_options: "rw" # Allows customizing mount options for special workloads + protocols: ["NFSV3","NFSV4"] # List of protocols. ["NFSV3], ["NFSv4] or ["NFSV3, "NFSV4"] + region: $(vars.region) + unix_permissions: "0777" # Specify default permissions for roo inode owned by root:root + # If no export policy is specified, a permissive default policy will be applied, which is: + # allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" # RFC1918 + # has_root_access = true # no_root_squash enabled + # access_type = "READ_WRITE" + export_policy: + - allowed_clients: "10.10.20.8,10.10.20.9" + has_root_access: true # no_root_squash enabled + access_type: "READ_WRITE" + nfsv3: false # allow only NFSv4 for these hosts + nfsv4: true + - allowed_clients: "10.0.0.0/8" + has_root_access: false # no_root_squash disabled + access_type: "READ_WRITE" + nfsv3: true # allow only NFSv3 for these hosts + nfsv4: false + tiering_policy: # Enable auto-tiering. Requires auto-tiering enabled storage pool + tier_action: "ENABLED" + cooling_threshold_days: 31 # tier data blocks which have not been touched for 31 days + + description: "Shared volume for EDA job" + labels: + owner: bob +``` + +## Protocol support +Since Cluster Toolkit is currently built to provision Linux-based compute clusters, this module supports NFSv3 and NFSv4.1 only. SMB is blocked. + +## Large volumes +Volumes larger than 15 TiB can be created as [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). Such volumes can grow up to 3 PiB and can scale read performance up to 29 GiBps. They provide six IP addresses to the volume. They are exported via the `server_ips` output. When connecting a large volume to a client using the USE directive, cluster toolkit currently uses the first IP only. This will be improved in the future. + +This feature is allow-listed GA. To request allow-listing, see [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). + +## Auto-tiering support +For auto-tiering enabled storage pools you can enable auto-tiering on the volume. For more information, see [manage auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering). + +## Using existing volumes not created by Cluster Toolkit +NetApp Volumes volumes are regular NFS exports. You can use the [pre-existing-network-storage] module to integrate them into Cluster Toolkit. + +Example code: + +```yaml +- id: homefs + source: modules/file-system/pre-existing-network-storage + settings: + server_ip: ## Set server IP here ## + remote_mount: nfsshare + local_mount: /home + fs_type: nfs +``` + +This creates a resource in Cluster Toolkit which references the specified NFS export, which will be mounted at `/home` by clients which mount if via USE directive. + +Note that the `server_ip` must be known before deployment and this module does not allow +to specify a list of IPs for large volumes. + +[pre-existing-network-storage]: ../pre-existing-network-storage/README.md + +## FlexCache support +NetApp FlexCache technology accelerates data access, reduces WAN latency and lowers WAN bandwidth costs for read-intensive workloads, especially where clients need to access the same data repeatedly. When you create a FlexCache volume, you create a remote cache of an already existing (origin) volume that contains only the actively accessed data (hot data) of the origin volume. + +The FlexCache support in Google Cloud NetApp Volumes allows you to provision a cache volume in your Google network to improve performance for hybrid cloud environments. A FlexCache volume can help you transition workloads to the hybrid cloud by caching data from an on-premises data center to cloud. + +Deploying FlexCache volumes requires manual steps on the ONTAP origin side, which are not automated. Therefore this module has no support to deploy FlexCache volumes today. Deploy them manually and use the [pre-existing-network-storage](#using-existing-volumes-not-created-by-cluster-toolkit) instead. + +## License + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [google](#requirement\_google) | >= 6.45.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.45.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_netapp_volume.netapp_volume](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_volume) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [capacity\_gib](#input\_capacity\_gib) | The capacity of the volume in GiB. | `number` | `1024` | no | +| [description](#input\_description) | A description of the NetApp volume. | `string` | `""` | no | +| [export\_policy\_rules](#input\_export\_policy\_rules) | Define NFS export policy. | 
list(object({
    allowed_clients = optional(string)
    has_root_access = optional(bool, false)
    access_type     = optional(string, "READ_WRITE")
    nfsv3           = optional(bool)
    nfsv4           = optional(bool)
  }))
| 
[
  {
    "access_type": "READ_WRITE",
    "allowed_clients": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
    "has_root_access": true
  }
]
| no | +| [labels](#input\_labels) | Labels to add to the NetApp volume. Key-value pairs. | `map(string)` | n/a | yes | +| [large\_capacity](#input\_large\_capacity) | If true, the volume will be created with large capacity.
Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. | `bool` | `false` | no | +| [local\_mount](#input\_local\_mount) | Mountpoint for this volume. | `string` | `"/shared"` | no | +| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"rw,hard,rsize=65536,wsize=65536,tcp"` | no | +| [netapp\_storage\_pool\_id](#input\_netapp\_storage\_pool\_id) | The ID of the NetApp storage pool to use for the volume. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | +| [protocols](#input\_protocols) | The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported. | `list(string)` | 
[
  "NFSV3"
]
| no | +| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | +| [tiering\_policy](#input\_tiering\_policy) | Define the tiering policy for the NetApp volume. | 
object({
    tier_action            = optional(string)
    cooling_threshold_days = optional(number)
  })
| `null` | no | +| [unix\_permissions](#input\_unix\_permissions) | UNIX permissions for root inode in the volume. | `string` | `"0777"` | no | +| [volume\_name](#input\_volume\_name) | The name of the volume. Needs to be unique within the storage pool. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gb](#output\_capacity\_gb) | Volume capacity in GiB. | +| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | +| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | +| [netapp\_volume\_id](#output\_netapp\_volume\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}` | +| [network\_storage](#output\_network\_storage) | Describes a NetApp Volumes volume. | +| [server\_ips](#output\_server\_ips) | List of IP addresses of the volume. | + diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/main.tf new file mode 100644 index 0000000000..d8345bf347 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/main.tf @@ -0,0 +1,92 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "netapp-volume", ghpc_role = "file-system" }) +} + +# resource "random_id" "resource_name_suffix" { +# byte_length = 4 +# } + +locals { + full_path = split(":", google_netapp_volume.netapp_volume.mount_options[0].export_full) + server_ip = local.full_path[0] + remote_mount = local.full_path[1] + # Large volumes will have 6 IPs + server_ips = [for ip in google_netapp_volume.netapp_volume.mount_options[*].export_full : split(":", ip)[0]] + fs_type = "nfs" + mount_options = var.mount_options + + install_nfs_client_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-nfs-client.sh" + "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" + } + mount_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${join(",", local.server_ips)}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" + } + + split_pool_id = split("/", var.netapp_storage_pool_id) + pool_name = local.split_pool_id[5] +} + +resource "google_netapp_volume" "netapp_volume" { + project = var.project_id + + name = var.volume_name + share_name = var.volume_name + location = var.region + protocols = var.protocols + capacity_gib = var.capacity_gib + large_capacity = var.large_capacity + multiple_endpoints = var.large_capacity == true ? true : null + storage_pool = local.pool_name + unix_permissions = var.unix_permissions + + dynamic "tiering_policy" { + for_each = var.tiering_policy == null ? [] : [0] + content { + cooling_threshold_days = lookup(var.tiering_policy, "cooling_threshold_days", null) + tier_action = lookup(var.tiering_policy, "tier_action", null) + } + } + + description = var.description + labels = local.labels + + dynamic "export_policy" { + for_each = var.export_policy_rules == null ? [] : [0] + content { + dynamic "rules" { + for_each = var.export_policy_rules + content { + access_type = rules.value.access_type + allowed_clients = rules.value.allowed_clients + has_root_access = rules.value.has_root_access + nfsv3 = rules.value.nfsv3 == null ? contains([for p in var.protocols : lower(p)], "nfsv3") : rules.value.nfsv3 + nfsv4 = rules.value.nfsv4 == null ? contains([for p in var.protocols : lower(p)], "nfsv4") : rules.value.nfsv4 + } + } + } + } + + depends_on = [var.netapp_storage_pool_id] +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/metadata.yaml new file mode 100644 index 0000000000..e4a7aaaa14 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - netapp.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/outputs.tf new file mode 100644 index 0000000000..641eae007a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/outputs.tf @@ -0,0 +1,66 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +output "network_storage" { + description = "Describes a NetApp Volumes volume." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = local.mount_options + client_install_runner = local.install_nfs_client_runner + mount_runner = local.mount_runner + } +} + +output "install_nfs_client" { + description = "Script for installing NFS client" + value = file("${path.module}/scripts/install-nfs-client.sh") +} + +output "install_nfs_client_runner" { + description = "Runner to install NFS client using the startup-script module" + value = local.install_nfs_client_runner +} + +output "mount_runner" { + description = <<-EOT + Runner to mount the file-system using an ansible playbook. The startup-script + module will automatically handle installation of ansible. + - id: example-startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(your-fs-id.mount_runner) + ... + EOT + value = local.mount_runner +} + +output "netapp_volume_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}`" + value = google_netapp_volume.netapp_volume.id +} + +output "capacity_gb" { + description = "Volume capacity in GiB." + value = google_netapp_volume.netapp_volume.capacity_gib +} + +output "server_ips" { + description = "List of IP addresses of the volume." + value = local.server_ips +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..1b1595e5a4 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsupported distribution' + return 1 + fi +fi diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh new file mode 100644 index 0000000000..8253d40a24 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IPS=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +# accept a list of colon-separated IPs and randomly pick one to enable load balancing +# In recent changes cluster toolkit doesn't seem to use this file anymore, +# which makes all mounts use the first IP in the list. Needs to be investigated in future. +IFS="," read -r -a arrIPS <<<"${SERVER_IPS}" +rand1=$(od -vAn -t d -N1 /dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/variables.tf new file mode 100644 index 0000000000..272558ff77 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/variables.tf @@ -0,0 +1,133 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which the NetApp storage pool will be created." + type = string +} + +variable "netapp_storage_pool_id" { + description = "The ID of the NetApp storage pool to use for the volume." + type = string + validation { + condition = length(split("/", var.netapp_storage_pool_id)) == 6 + error_message = "The storage pool id must be provided in the following format: projects//locations//storagePools/." + } +} + +variable "region" { + description = "Location for NetApp storage pool." + type = string +} + +variable "volume_name" { + description = "The name of the volume. Needs to be unique within the storage pool." + type = string + default = null +} + +variable "capacity_gib" { + description = "The capacity of the volume in GiB." + type = number + default = 1024 + validation { + condition = var.capacity_gib >= 100 + error_message = "The minimum capacity for the volume is 100 GiB." + } +} + +variable "protocols" { + description = "The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported." + type = list(string) + default = ["NFSV3"] + validation { + condition = alltrue([for p in var.protocols : contains(["NFSV3", "NFSV4"], p)]) + error_message = "Allowed values for protocols are 'NFSV3' or 'NFSV4'." + } +} + +variable "description" { + description = "A description of the NetApp volume." + type = string + default = "" + validation { + condition = length(var.description) <= 2048 + error_message = "NetApp volume description must be 2048 characters or fewer" + } +} + +variable "labels" { + description = "Labels to add to the NetApp volume. Key-value pairs." + type = map(string) +} + +variable "local_mount" { + description = "Mountpoint for this volume." + type = string + default = "/shared" +} + +variable "mount_options" { + description = "NFS mount options to mount file system." + type = string + default = "rw,hard,rsize=65536,wsize=65536,tcp" +} + +variable "large_capacity" { + description = <<-EOT + If true, the volume will be created with large capacity. + Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. + EOT + type = bool + default = false +} + +variable "unix_permissions" { + description = "UNIX permissions for root inode in the volume." + type = string + default = "0777" + validation { + condition = length(var.unix_permissions) <= 4 + error_message = "UNIX permissions must be a 4-digit octal number." + } +} + +variable "tiering_policy" { + description = "Define the tiering policy for the NetApp volume." + type = object({ + tier_action = optional(string) + cooling_threshold_days = optional(number) + }) + default = null +} + +variable "export_policy_rules" { + description = "Define NFS export policy." + type = list(object({ + allowed_clients = optional(string) + has_root_access = optional(bool, false) + access_type = optional(string, "READ_WRITE") + nfsv3 = optional(bool) + nfsv4 = optional(bool) + })) + # Permissive default if user does not specify nfs_export_options. Allow all RFC1918 CIDRS with no_root_squash + default = [{ + allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", + has_root_access = true, + access_type = "READ_WRITE", + }] + nullable = true +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/versions.tf new file mode 100644 index 0000000000..c624d5100b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/versions.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.45.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" + } + + required_version = ">= 1.5.7" +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/README.md new file mode 100644 index 0000000000..0b942f067f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/README.md @@ -0,0 +1,196 @@ +## Description + +This module creates [parallelstore](https://cloud.google.com/parallelstore) +instance. Parallelstore is Google Cloud's first party parallel file system +service based on [Intel DAOS](https://docs.daos.io/v2.2/) + +### Supported Operating Systems + +A parallelstore instance can be used with Slurm cluster or compute +VM running Ubuntu 22.04, debian 12 or HPC Rocky Linux 8. + +### Parallelstore Quota + +To get access to a private preview of Parallelstore APIs, your project needs to +be allowlisted. To set this up, please work with your account representative. + +### Parallelstore mount options + +After parallelstore instance is created, you can specify mount options depending +upon your workload. DAOS is configured to deliver the best user experience for +interactive workloads with aggressive caching. If you are running parallel +workloads concurrently accessing the sane files from multiple client nodes, it +is recommended to disable the writeback cache to avoid cross-client consistency +issues. You can specify different mount options as follows, + +```yaml + - id: parallelstore + source: modules/file-system/parallelstore + use: [network, ps_connect] + settings: + mount_options: "disable-wb-cache,thread-count=20,eq-count=8" +``` + +### Example - New VPC + +For parallelstore instance, Below snippet creates new VPC and configures private-service-access +for this newly created network. + +```yaml + - id: network + source: modules/network/vpc + + # Private Service Access (PSA) requires the compute.networkAdmin role which is + # included in the Owner role, but not Editor. + # PSA is required for all Parallelstore functionality. + # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: parallelstore + source: modules/file-system/parallelstore + use: [network, private_service_access] +``` + +### Example - Existing VPC + +If you want to use existing network with private-service-access configured, you need +to manually provide `private_vpc_connection_peering` to the parallelstore module. +You can get this details from the Google Cloud Console UI in `VPC network peering` +section. Below is the example of using existing network and creating parallelstore. +If existing network is not configured with private-service-access, you can follow +[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) +to set it up. + +```yaml + - id: network + source: modules/network/pre-existing-vpc + settings: + network_name: // Add network name + subnetwork_name: // Add subnetwork name + + - id: parallelstore + source: modules/file-system/parallelstore + use: [network] + settings: + private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" +``` + +### Import data from GCS bucket + +You can import data from your GCS bucket to parallelstore instance. Important to +note that data may not be available to the instance immediately. This depends on +latency and size of data. Below is the example of importing data from bucket. + +```yaml + - id: parallelstore + source: modules/file-system/parallelstore + use: [network] + settings: + import_gcs_bucket_uri: gs://gcs-bucket/folder-path + import_destination_path: /gcs/import/ +``` + +Here you can replace `import_gcs_bucket_uri` with the uri of sub folder within GCS +bucket and `import_destination_path` with local directory within parallelstore +instance. + +### Additional configuration for DAOS agent and dfuse +Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + daos_agent_config: | + credential_config: + cache_expiration: 1m +``` + +Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/parallelstore + settings: + dfuse_environment: + D_LOG_FILE: /tmp/client.log + D_APPEND_PID_TO_LOG: 1 + D_LOG_MASK: debug +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13 | +| [google](#requirement\_google) | >= 6.13.0 | +| [null](#requirement\_null) | ~> 3.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.13.0 | +| [null](#provider\_null) | ~> 3.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_parallelstore_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/parallelstore_instance) | resource | +| [null_resource.hydration](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [daos\_agent\_config](#input\_daos\_agent\_config) | Additional configuration to be added to daos\_config.yml | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. | `string` | n/a | yes | +| [dfuse\_environment](#input\_dfuse\_environment) | Additional environment variables for DFuse process | `map(string)` | `{}` | no | +| [directory\_stripe](#input\_directory\_stripe) | The parallelstore stripe level for directories. | `string` | `null` | no | +| [file\_stripe](#input\_file\_stripe) | The parallelstore stripe level for files. | `string` | `null` | no | +| [import\_destination\_path](#input\_import\_destination\_path) | The name of local path to import data on parallelstore instance from GCS bucket. | `string` | `null` | no | +| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to parallelstore. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to parallel store instance. | `map(string)` | `{}` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/parallelstore"` | no | +| [mount\_options](#input\_mount\_options) | Options describing various aspects of the parallelstore instance. | `string` | `"disable-wb-cache,thread-count=16,eq-count=8"` | no | +| [name](#input\_name) | Name of parallelstore instance. | `string` | `null` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [size\_gb](#input\_size\_gb) | Storage size of the parallelstore instance in GB. | `number` | `12000` | no | +| [zone](#input\_zone) | Location for parallelstore instance. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions to monitor import-data operation from GCS bucket to parallelstore. | +| [network\_storage](#output\_network\_storage) | Describes a parallelstore instance. | + diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/main.tf new file mode 100644 index 0000000000..acc2a0551e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/main.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "parallelstore", ghpc_role = "file-system" }) +} + +locals { + fs_type = "daos" + server_ip = "" + remote_mount = "" + id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + access_points = jsonencode(google_parallelstore_instance.instance.access_points) + destination_path = var.import_destination_path == null ? "/" : var.import_destination_path + + client_install_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-daos-client.sh" + "destination" = "install_daos_client.sh" + } + + mount_runner = { + "type" = "shell" + "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { + access_points = local.access_points + daos_agent_config = var.daos_agent_config + dfuse_environment = var.dfuse_environment + local_mount = var.local_mount + mount_options = join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) + }) + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + } +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_parallelstore_instance" "instance" { + project = var.project_id + instance_id = local.id + location = var.zone + capacity_gib = var.size_gb + network = var.network_id + file_stripe_level = var.file_stripe + directory_stripe_level = var.directory_stripe + + labels = local.labels + + depends_on = [var.private_vpc_connection_peering] +} + +resource "null_resource" "hydration" { + count = var.import_gcs_bucket_uri != null ? 1 : 0 + + depends_on = [resource.google_parallelstore_instance.instance] + provisioner "local-exec" { + command = "curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -d '{\"source_gcs_bucket\": {\"uri\":\"${var.import_gcs_bucket_uri}\"}, \"destination_parallelstore\": {\"path\":\"${local.destination_path}\"}}' https://parallelstore.googleapis.com/v1beta/projects/${var.project_id}/locations/${var.zone}/instances/${local.id}:importData" + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/metadata.yaml new file mode 100644 index 0000000000..c0994d15bb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - parallelstore.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/outputs.tf new file mode 100644 index 0000000000..f6e817ac8a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/outputs.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + operation_instructions = <<-EOT + Data is being imported from GCS bucket to parallelstore instance. It may + not be available immediately. + EOT +} + +output "network_storage" { + description = "Describes a parallelstore instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = var.mount_options + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } + + precondition { + condition = var.import_gcs_bucket_uri != null || var.import_destination_path == null + error_message = <<-EOD + Please specify import_gcs_bucket_uri to import data to parallelstore instance. + EOD + } +} + +output "instructions" { + description = "Instructions to monitor import-data operation from GCS bucket to parallelstore." + value = var.import_gcs_bucket_uri != null ? local.operation_instructions : "Data is not imported from GCS bucket." +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh new file mode 100644 index 0000000000..e96eadb56a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || + { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 +fi + +if [ -x /bin/daos ]; then + echo "DAOS already installed" + daos version +else + # Install the DAOS client library + # The following commands should be executed on each client vm. + ## For Rocky linux 8 / RedHat 8. + if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then + # 1) Add the Parallelstore package repository + cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF + [parallelstore-v2-6-el${OS_VERSION_MAJOR}] + name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 + baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + + ## TODO: Remove disable automatic update script after issue is fixed. + if [ -x /usr/bin/google_disable_automatic_updates ]; then + /usr/bin/google_disable_automatic_updates + fi + dnf clean all + dnf makecache + + # 2) Install daos-client + dnf install -y epel-release # needed for capstone + dnf install -y daos-client + + # 3) Upgrade libfabric + dnf upgrade -y libfabric + + # For Ubuntu 22.04 and debian 12, + elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then + # shellcheck disable=SC2034 + DEBIAN_FRONTEND=noninteractive + + # 1) Add the Parallelstore package repository + curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg + echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list + + apt-get update + + # 2) Install daos-client + apt-get install -y daos-client + + # 3) Create daos_agent.service (comes pre-installed with RedHat) + if ! getent passwd daos_agent >/dev/null 2>&1; then + useradd daos_agent + fi + cat >/etc/systemd/system/daos_agent.service <<-EOF + [Unit] + Description=DAOS Agent + StartLimitIntervalSec=60 + Wants=network-online.target + After=network-online.target + + [Service] + Type=notify + User=daos_agent + Group=daos_agent + RuntimeDirectory=daos_agent + RuntimeDirectoryMode=0755 + ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml + StandardOutput=journal + StandardError=journal + Restart=always + RestartSec=10 + LimitMEMLOCK=infinity + LimitCORE=infinity + StartLimitBurst=5 + + [Install] + WantedBy=multi-user.target + EOF + else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + fi +fi + +exit 0 diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl new file mode 100644 index 0000000000..c6f5d53660 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl @@ -0,0 +1,110 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; } && { [[ "$${OS_VERSION_MAJOR}" = "8" ]] || [[ "$${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "$${OS_ID}" = "ubuntu" ]] && [[ "$${OS_VERSION}" = "22.04" ]]; } || + { [[ "$${OS_ID}" = "debian" ]] && [[ "$${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system $${OS_ID} $${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + +fi + +# Edit agent config +daos_config=/etc/daos/daos_agent.yml + +# rewrite $daos_config from scratch +mv $${daos_config} $${daos_config}.orig + +exclude_fabric_ifaces="" +# Get names of network interfaces not in first PCI slot +# The first PCI slot is a standard network adapter while remaining interfaces +# are typically network cards dedicated to GPU or workload communication +if [[ "$${OS_ID}" == "debian" ]] || [[ "$${OS_ID}" = "ubuntu" ]]; then + extra_interfaces=$(find /sys/class/net/ -not -name 'enp0s*' -regextype posix-extended -regex '.*/enp[0-9]+s.*' -printf '"%f"\n' | paste -s -d ',') +elif [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; then + extra_interfaces=$(find /sys/class/net/ -not -name eth0 -regextype posix-extended -regex '.*/eth[0-9]+' -printf '"%f"\n' | paste -s -d ',') +fi + +cat > $daos_config </etc/systemd/system/"$${service_name}" </global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "import_gcs_bucket_uri" { + description = "The name of the GCS bucket to import data from to parallelstore." + type = string + default = null +} + +variable "import_destination_path" { + description = "The name of local path to import data on parallelstore instance from GCS bucket." + type = string + default = null +} + +variable "file_stripe" { + description = "The parallelstore stripe level for files." + type = string + default = null + validation { + condition = var.file_stripe == null ? true : contains([ + "FILE_STRIPE_LEVEL_UNSPECIFIED", + "FILE_STRIPE_LEVEL_MIN", + "FILE_STRIPE_LEVEL_BALANCED", + "FILE_STRIPE_LEVEL_MAX", + ], var.file_stripe) + error_message = "var.file_stripe must be set to \"FILE_STRIPE_LEVEL_UNSPECIFIED\", \"FILE_STRIPE_LEVEL_MIN\", \"FILE_STRIPE_LEVEL_BALANCED\", or \"FILE_STRIPE_LEVEL_MAX\"" + } +} + +variable "directory_stripe" { + description = "The parallelstore stripe level for directories." + type = string + default = null + validation { + condition = var.directory_stripe == null ? true : contains([ + "DIRECTORY_STRIPE_LEVEL_UNSPECIFIED", + "DIRECTORY_STRIPE_LEVEL_MIN", + "DIRECTORY_STRIPE_LEVEL_BALANCED", + "DIRECTORY_STRIPE_LEVEL_MAX", + ], var.directory_stripe) + error_message = "var.directory_stripe must be set to \"DIRECTORY_STRIPE_LEVEL_UNSPECIFIED\", \"DIRECTORY_STRIPE_LEVEL_MIN\", \"DIRECTORY_STRIPE_LEVEL_BALANCED\", or \"DIRECTORY_STRIPE_LEVEL_MAX\"" + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/versions.tf new file mode 100644 index 0000000000..174b5281e4 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.13.0" + } + + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/README.md new file mode 100644 index 0000000000..47cf1518a1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/README.md @@ -0,0 +1,192 @@ +## Description + +This module defines a file-system that already exists (i.e. it does not create +a new file system) in a way that can be shared with other modules. This allows +a compute VM to mount a filesystem that is not part of the current deployment +group. + +The pre-existing network storage can be referenced in the same way as any Cluster +Toolkit supported file-system such as [filestore](../filestore/README.md). + +For more information on network storage options in the Cluster Toolkit, see +the extended [Network Storage documentation](../../../docs/network_storage.md). + +### Example + +```yaml +- id: homefs + source: modules/file-system/pre-existing-network-storage + settings: + server_ip: ## Set server IP here ## + remote_mount: nfsshare + local_mount: /home + fs_type: nfs +``` + +This creates a pre-existing-network-storage module in terraform at the +provided IP in `server_ip` of type nfs that will be mounted at `/home`. Note +that the `server_ip` must be known before deployment. + +The following is an example of using `pre-existing-network-storage` with a GCS +bucket: + +```yaml +- id: data-bucket + source: modules/file-system/pre-existing-network-storage + settings: + remote_mount: my-bucket-name + local_mount: /data + fs_type: gcsfuse + mount_options: defaults,_netdev,implicit_dirs +``` + +The `implicit_dirs` mount option allows object paths to be treated as if they +were directories. This is important when working with files that were created by +another source, but there may have performance impacts. The `_netdev` mount option +denotes that the storage device requires network access. + +The following is an example of using `pre-existing-network-storage` with the `lustre` +filesystem: + +```yaml +- id: lustrefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: lustre + server_ip: 192.168.227.11@tcp + local_mount: /scratch + remote_mount: /exacloud +``` + +Note the use of the MGS NID (Network ID) in the `server_ip` field - in +particular, note the `@tcp` suffix. + +The following is an example of using `pre-existing-network-storage` with the +`managed_lustre` filesystem: + +```yaml +- id: lustrefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: managed_lustre + server_ip: 192.168.227.11@tcp + local_mount: /scratch + remote_mount: /mg_lustre +``` + +This is similar to the `lustre` filesystem, with the exception that it connects +with a managed Lustre instance hosted by GCP. Currently only Rocky 8 and +Ubuntu 20.04 and Ubuntu 22.04 are supported. + +The following is an example of using `pre-existing-network-storage` with the `daos` +filesystem. In order to use existing `parallelstore` instance, `fs_type` needs to be +explicitly mentioned in blueprint. The `remote_mount` option refers to `access_points` +for `parallelstore` instance. + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 +``` + +Parallelstore supports additional options for its mountpoints under `parallelstore_options` setting. +Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 + parallelstore_options: + daos_agent_config: | + credential_config: + cache_expiration: 1m +``` + +Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 + parallelstore_options: + dfuse_environment: + D_LOG_FILE: /tmp/client.log + D_APPEND_PID_TO_LOG: 1 + D_LOG_MASK: debug +``` + +### Mounting + +For the `fs_type` listed below, this module will provide `client_install_runner` +and `mount_runner` outputs. These can be used to create a startup script to +mount the network storage system. + +Supported `fs_type`: + +- nfs +- lustre +- managed_lustre +- gcsfuse +- daos + +[scripts/mount.sh](./scripts/mount.sh) is used as the contents of +`mount_runner`. This script will update `/etc/fstab` and mount the network +storage. This script will fail if the specified `local_mount` is already being +used by another entry in `/etc/fstab`. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. + +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## License + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [fs\_type](#input\_fs\_type) | Type of file system to be mounted (e.g., nfs, lustre) | `string` | `"nfs"` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | +| [managed\_lustre\_options](#input\_managed\_lustre\_options) | Managed Lustre specific options:
gke\_support\_enabled (bool, default = false)
Note: gke\_support\_enabled does not work with Slurm, the Slurm image must be built with
the correct compatibility. | 
object({
    gke_support_enabled = optional(bool, false)
  })
| `{}` | no | +| [mount\_options](#input\_mount\_options) | Options describing various aspects of the file system. Consider adding setting to 'defaults,\_netdev,implicit\_dirs' when using gcsfuse. | `string` | `"defaults,_netdev"` | no | +| [parallelstore\_options](#input\_parallelstore\_options) | Parallelstore specific options | 
object({
    daos_agent_config = optional(string, "")
    dfuse_environment = optional(map(string), {})
  })
| `{}` | no | +| [remote\_mount](#input\_remote\_mount) | Remote FS name or export. This is the exported directory for nfs, fs name for lustre, and bucket name (without gs://) for gcsfuse. | `string` | n/a | yes | +| [server\_ip](#input\_server\_ip) | The device name as supplied to fs-tab, excluding remote fs-name(for nfs, that is the server IP, for lustre [:]). This can be omitted for gcsfuse. | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use file system. | +| [mount\_runner](#output\_mount\_runner) | Runner that mounts the file system. | +| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | + diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf new file mode 100644 index 0000000000..203b6dfdac --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf @@ -0,0 +1,124 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a remote network storage to be mounted by fs-tab." + value = { + server_ip = var.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = var.mount_options + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } +} + +locals { + # Update remote mount to include a slash if the fs_type requires one to exist + remote_mount_with_slash = length(regexall("^/.*", var.remote_mount)) > 0 ? ( + var.remote_mount + ) : format("/%s", var.remote_mount) + remote_mount = contains(local.mount_vanilla_supported_fstype, local.fs_type) ? ( + local.remote_mount_with_slash + ) : var.remote_mount + + ml_gke_support_enabled = coalesce(try(var.managed_lustre_options.gke_support_enabled, false), false) + + # Collapse fs_type lustre and managed lustre for most uses, only needs to be + # different for client installation + fs_type = strcontains(var.fs_type, "lustre") ? "lustre" : var.fs_type + + # Client Install + ddn_lustre_client_install_script = templatefile( + "${path.module}/templates/ddn_exascaler_luster_client_install.tftpl", + { + server_ip = split("@", var.server_ip)[0] + remote_mount = local.remote_mount + local_mount = var.local_mount + } + ) + managed_lustre_client_install_script = file("${path.module}/scripts/install-managed-lustre-client.sh") + nfs_client_install_script = file("${path.module}/scripts/install-nfs-client.sh") + gcs_fuse_install_script = file("${path.module}/scripts/install-gcs-fuse.sh") + daos_client_install_script = file("${path.module}/scripts/install-daos-client.sh") + + install_scripts = { + "lustre" = local.ddn_lustre_client_install_script + "managed_lustre" = local.managed_lustre_client_install_script + "nfs" = local.nfs_client_install_script + "gcsfuse" = local.gcs_fuse_install_script + "daos" = local.daos_client_install_script + } + + client_install_runner = { + "type" = "shell" + "content" = lookup(local.install_scripts, var.fs_type, "echo 'skipping: client_install_runner not yet supported for ${var.fs_type}'") + "destination" = "install_filesystem_client${replace(var.local_mount, "/", "_")}.sh" + "args" = local.ml_gke_support_enabled ? "1" : "" + } + + mount_vanilla_supported_fstype = ["lustre", "nfs"] + mount_runner_vanilla = { + "type" = "shell" + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"${var.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" + "content" = ( + contains(local.mount_vanilla_supported_fstype, local.fs_type) ? + file("${path.module}/scripts/mount.sh") : + "echo 'skipping: mount_runner not yet supported for ${var.fs_type}'" + ) + } + gcsbucket = trimprefix(var.remote_mount, "gs://") + mount_runner_gcsfuse = { + "type" = "shell" + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"not-used\" \"${local.gcsbucket}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" + "content" = file("${path.module}/scripts/mount.sh") + } + + mount_runner_daos = { + "type" = "shell" + "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { + access_points = var.remote_mount + daos_agent_config = var.parallelstore_options.daos_agent_config + dfuse_environment = var.parallelstore_options.dfuse_environment + local_mount = var.local_mount + # avoid passing "--" as mount option to dfuse + mount_options = length(var.mount_options) == 0 ? "" : join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) + }) + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + } + + mount_scripts = { + "lustre" = local.mount_runner_vanilla + "nfs" = local.mount_runner_vanilla + "gcsfuse" = local.mount_runner_gcsfuse + "daos" = local.mount_runner_daos + } + + mount_runner = lookup(local.mount_scripts, local.fs_type, local.mount_runner_vanilla) +} + +output "client_install_runner" { + description = "Runner that performs client installation needed to use file system." + value = local.client_install_runner +} + +output "mount_runner" { + description = "Runner that mounts the file system." + value = local.mount_runner +} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh new file mode 100644 index 0000000000..e96eadb56a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || + { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 +fi + +if [ -x /bin/daos ]; then + echo "DAOS already installed" + daos version +else + # Install the DAOS client library + # The following commands should be executed on each client vm. + ## For Rocky linux 8 / RedHat 8. + if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then + # 1) Add the Parallelstore package repository + cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF + [parallelstore-v2-6-el${OS_VERSION_MAJOR}] + name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 + baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + + ## TODO: Remove disable automatic update script after issue is fixed. + if [ -x /usr/bin/google_disable_automatic_updates ]; then + /usr/bin/google_disable_automatic_updates + fi + dnf clean all + dnf makecache + + # 2) Install daos-client + dnf install -y epel-release # needed for capstone + dnf install -y daos-client + + # 3) Upgrade libfabric + dnf upgrade -y libfabric + + # For Ubuntu 22.04 and debian 12, + elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then + # shellcheck disable=SC2034 + DEBIAN_FRONTEND=noninteractive + + # 1) Add the Parallelstore package repository + curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg + echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list + + apt-get update + + # 2) Install daos-client + apt-get install -y daos-client + + # 3) Create daos_agent.service (comes pre-installed with RedHat) + if ! getent passwd daos_agent >/dev/null 2>&1; then + useradd daos_agent + fi + cat >/etc/systemd/system/daos_agent.service <<-EOF + [Unit] + Description=DAOS Agent + StartLimitIntervalSec=60 + Wants=network-online.target + After=network-online.target + + [Service] + Type=notify + User=daos_agent + Group=daos_agent + RuntimeDirectory=daos_agent + RuntimeDirectoryMode=0755 + ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml + StandardOutput=journal + StandardError=journal + Restart=always + RestartSec=10 + LimitMEMLOCK=infinity + LimitCORE=infinity + StartLimitBurst=5 + + [Install] + WantedBy=multi-user.target + EOF + else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + fi +fi + +exit 0 diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh new file mode 100644 index 0000000000..f8a990260b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ ! "$(which gcsfuse)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then + tee /etc/yum.repos.d/gcsfuse.repo >/dev/null <>/etc/modprobe.d/lnet.conf + fi +fi + +if grep -q lustre /proc/filesystems; then + echo "Skipping managed lustre client install as it is already supported" + exit 0 +fi + +# Get distro information +. /etc/os-release +DIST="NA" +if [[ $NAME == *"Ubuntu"* ]]; then + if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then + DIST="Ubuntu" + fi +elif [[ $NAME == *"Rocky"* ]]; then + if [[ $VERSION_ID == "8"* ]]; then + DIST="Rocky" + fi +fi + +if [[ ${DIST} == "Ubuntu" ]]; then + KEY_LOC=/etc/apt/keyrings + KEY_NAME=gcp-ar-repo.gpg + # Download new repo key + mkdir -p "${KEY_LOC}" + wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null + + # Set up apt repo + echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list + + # Install modules + apt update + apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) +elif [[ ${DIST} == "Rocky" ]]; then + # Set up yum repo + touch /etc/yum.repos.d/artifact-registry.repo + tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF + [lustre-client-rocky-8] + name=lustre-client-rocky-8 + baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + # Install modules + yum makecache + yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client +fi + +if [[ $DIST != "NA" ]]; then + # Load the new lustre client module + modprobe lustre +fi diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl new file mode 100644 index 0000000000..f5f0291e85 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl @@ -0,0 +1,50 @@ +#!/bin/sh + +# Copyright 2022 DataDirect Networks +# Modifications Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prior Art: https://github.com/DDNStorage/exascaler-cloud-terraform/blob/78deadbb2c1fa7e4603cf9605b0f7d1782117954/gcp/templates/client-script.tftpl + +# install new EXAScaler Cloud clients: +# all instances must be in the same zone +# and connected to the same network and subnet +# to set up EXAScaler Cloud filesystem on a new client instance, +# run the following commands on the client with root privileges: +set -e +if [[ ! -z $(cat /proc/filesystems | grep lustre) ]]; then + echo "Skipping lustre client install as it is already supported" + exit 0 +fi + +cat >/etc/esc-client.conf< $daos_config </etc/systemd/system/"$${service_name}" < +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string
    count = number
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [guest\_accelerator](#output\_guest\_accelerator) | Sanitized list of the type and count of accelerator cards attached to the instance. | +| [machine\_type\_guest\_accelerator](#output\_machine\_type\_guest\_accelerator) | List of the type and count of accelerator cards attached to the specified machine type. | + diff --git a/deletion-test/build_script/modules/embedded/modules/internal/gpu-definition/main.tf b/deletion-test/build_script/modules/embedded/modules/internal/gpu-definition/main.tf new file mode 100644 index 0000000000..f0861cddc9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/gpu-definition/main.tf @@ -0,0 +1,98 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "machine_type" { + description = "Machine type to use for the instance creation" + type = string +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string + count = number + gpu_driver_installation_config = optional(object({ + gpu_driver_version = string + }), { gpu_driver_version = "DEFAULT" }) + gpu_partition_size = optional(string) + gpu_sharing_config = optional(object({ + gpu_sharing_strategy = string + max_shared_clients_per_gpu = number + })) + })) + default = [] + nullable = false +} + +locals { + # example state; terraform will ignore diffs if last element of URL matches + # guest_accelerator = [ + # { + # count = 1 + # type = "https://www.googleapis.com/compute/beta/projects/PROJECT/zones/ZONE/acceleratorTypes/nvidia-tesla-a100" + # }, + # ] + accelerator_machines = { + "a2-highgpu-1g" = { type = "nvidia-tesla-a100", count = 1 }, + "a2-highgpu-2g" = { type = "nvidia-tesla-a100", count = 2 }, + "a2-highgpu-4g" = { type = "nvidia-tesla-a100", count = 4 }, + "a2-highgpu-8g" = { type = "nvidia-tesla-a100", count = 8 }, + "a2-megagpu-16g" = { type = "nvidia-tesla-a100", count = 16 }, + "a2-ultragpu-1g" = { type = "nvidia-a100-80gb", count = 1 }, + "a2-ultragpu-2g" = { type = "nvidia-a100-80gb", count = 2 }, + "a2-ultragpu-4g" = { type = "nvidia-a100-80gb", count = 4 }, + "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, + "a3-highgpu-1g" = { type = "nvidia-h100-80gb", count = 1 }, + "a3-highgpu-2g" = { type = "nvidia-h100-80gb", count = 2 }, + "a3-highgpu-4g" = { type = "nvidia-h100-80gb", count = 4 }, + "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, + "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, + "a4-highgpu-8g-lowmem" = { type = "nvidia-b200", count = 8 }, + "a4-highgpu-8g" = { type = "nvidia-b200", count = 8 }, + "a4x-highgpu-4g" = { type = "nvidia-gb200", count = 4 }, + "a4x-highgpu-4g-nolssd" = { type = "nvidia-gb200", count = 4 }, + "g2-standard-4" = { type = "nvidia-l4", count = 1 }, + "g2-standard-8" = { type = "nvidia-l4", count = 1 }, + "g2-standard-12" = { type = "nvidia-l4", count = 1 }, + "g2-standard-16" = { type = "nvidia-l4", count = 1 }, + "g2-standard-24" = { type = "nvidia-l4", count = 2 }, + "g2-standard-32" = { type = "nvidia-l4", count = 1 }, + "g2-standard-48" = { type = "nvidia-l4", count = 4 }, + "g2-standard-96" = { type = "nvidia-l4", count = 8 }, + } + generated_guest_accelerator = try([local.accelerator_machines[var.machine_type]], []) + + # Select in priority order: + # (1) var.guest_accelerator if not empty + # (2) local.generated_guest_accelerator if not empty + # (3) default to empty list if both are empty + guest_accelerator = try(coalescelist(var.guest_accelerator, local.generated_guest_accelerator), []) +} + +output "guest_accelerator" { + description = "Sanitized list of the type and count of accelerator cards attached to the instance." + value = local.guest_accelerator +} + +output "machine_type_guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the specified machine type." + value = local.generated_guest_accelerator +} + +terraform { + required_version = ">= 1.3" +} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/README.md b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/README.md new file mode 100644 index 0000000000..21746fe0d8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/README.md @@ -0,0 +1,30 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [disk\_type](#input\_disk\_type) | The disk type to validate. | `string` | n/a | yes | +| [machine\_type](#input\_machine\_type) | The machine type to validate. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/main.tf b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/main.tf new file mode 100644 index 0000000000..d89d7edfec --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/main.tf @@ -0,0 +1,52 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +check "disk_type_c4_compatibility" { + assert { + condition = !(can(regex("^c4-", var.machine_type)) && var.disk_type == "pd-ssd") + error_message = "The C4 machine series does not support pd-ssd. Please use hyperdisk-balanced or another compatible disk type." + } +} + + +check "disk_type_c2_compatibility" { + assert { + condition = !(can(regex("^c2-", var.machine_type)) && can(regex("hyperdisk", var.disk_type))) + error_message = "The C2 machine series does not support Hyperdisk as a boot disk. Please use a compatible disk type like pd-ssd, pd-standard, or pd-balanced." + } +} + + +check "disk_type_pd_extreme_compatibility" { + assert { + condition = var.disk_type != "pd-extreme" || can(regex("^(m1-|m2-|m3-|n2-|n2d-)", var.machine_type)) + error_message = "pd-extreme disks are only supported for M1, M2, M3, N2, and N2D machine series." + } +} + + +check "disk_type_hyperdisk_extreme_compatibility" { + assert { + condition = var.disk_type != "hyperdisk-extreme" || can(regex("^(c3-|m1-|m3-|n2-)", var.machine_type)) + error_message = "hyperdisk-extreme disks are only supported for C3, M1, M3, and N2 machine series." + } +} + + +check "disk_type_hyperdisk_throughput_compatibility" { + assert { + condition = var.disk_type != "hyperdisk-throughput" || can(regex("^(c3-|c3d-|n4-|n2-|n2d-|n1-|t2d-|m1-)", var.machine_type)) + error_message = "hyperdisk-throughput disks are only supported for C3, C3D, N4, N2, N2D, N1, T2D, and M1 machine series." + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/variables.tf b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/variables.tf new file mode 100644 index 0000000000..23478051b3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/variables.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "machine_type" { + type = string + description = "The machine type to validate." +} + +variable "disk_type" { + type = string + description = "The disk type to validate." +} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/versions.tf b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/versions.tf new file mode 100644 index 0000000000..4702005614 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/versions.tf @@ -0,0 +1,17 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 0.15.0" +} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/README.md b/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/README.md new file mode 100644 index 0000000000..8aa9270a0a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/README.md @@ -0,0 +1,54 @@ + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | +| [google-beta](#requirement\_google-beta) | >= 6.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google-beta](#provider\_google-beta) | >= 6.0.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_network_attachment.self](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_network_attachment) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [connection\_preference](#input\_connection\_preference) | The connection preference of service attachment. | `string` | `"ACCEPT_AUTOMATIC"` | no | +| [name](#input\_name) | Name of the resource. Provided by the client when the resource is created | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The ID of the project in which the resource belongs. | `string` | n/a | yes | +| [region](#input\_region) | Region where the network attachment resides | `string` | n/a | yes | +| [subnetwork\_self\_links](#input\_subnetwork\_self\_links) | An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment. | `list(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [self\_link](#output\_self\_link) | Server-defined URL for the resource. | + diff --git a/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/main.tf b/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/main.tf new file mode 100644 index 0000000000..bbbece7085 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +variable "connection_preference" { + type = string + description = "The connection preference of service attachment." + default = "ACCEPT_AUTOMATIC" +} + +variable "subnetwork_self_links" { + type = list(string) + description = " An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment." +} + +variable "name" { + type = string + description = "Name of the resource. Provided by the client when the resource is created" +} + +variable "project_id" { + type = string + description = "The ID of the project in which the resource belongs." +} + +variable "region" { + type = string + description = "Region where the network attachment resides" +} + + +resource "google_compute_network_attachment" "self" { + provider = google-beta + + project = var.project_id + region = var.region + name = var.name + connection_preference = var.connection_preference + subnetworks = var.subnetwork_self_links +} + + +output "self_link" { + value = google_compute_network_attachment.self.self_link + description = "Server-defined URL for the resource." +} + +terraform { + required_version = ">= 0.15.0" + + required_providers { + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.0.0" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/metadata.yaml new file mode 100644 index 0000000000..e80fc96b9c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/README.md b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/README.md new file mode 100644 index 0000000000..610d82c1b9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/README.md @@ -0,0 +1,85 @@ +## Description + +This is an internal helper module designed to encapsulate and centralize all hardware-specific logic for Google Cloud TPUs. It is intended to be called by parent modules like `gke-node-pool` to determine if a node pool is TPU-based and to retrieve its specific attributes. + +This module's primary responsibilities are: + +* Reliably detect if a node pool is for TPUs by checking its `placement_policy`. +* Determine the correct GKE `tpu-accelerator` label based on the machine type family. +* Determine the `number of chips per node` based on the specific machine type. +* Generate the standard **Kubernetes taint** that should be applied to TPU nodes. + +This follows the same design pattern as the `gpu-definition` internal module, promoting a clean separation of concerns within the gke-node-pool module. + +## Usage + +This module is not intended for direct use in a blueprint. It should be called from a parent module like `gke-node-pool`. + +```yaml +module "tpu" { + source = "../../internal/tpu-definition" + + # Pass the parent module's variables to this module + machine_type = var.machine_type + placement_policy = var.placement_policy +} + +# Example of consuming the module's outputs in the parent module +locals { + # The tpu_taint is then used in the node_config's dynamic "taint" block + tpu_taint = module.tpu.tpu_taint +} +``` + +## License + + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [machine\_type](#input\_machine\_type) | The machine type of the node pool. | `string` | n/a | yes | +| [placement\_policy](#input\_placement\_policy) | The placement policy for the node pool. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [is\_tpu](#output\_is\_tpu) | Boolean value indicating if the node pool is for TPUs. | +| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | +| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | +| [tpu\_taint](#output\_tpu\_taint) | A list containing the standard TPU taint object if the node pool is for TPUs. | +| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | + diff --git a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/main.tf b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/main.tf new file mode 100644 index 0000000000..c8ee417d71 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/main.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # Determine if this is a TPU node pool by checking if the machine_type exists in our authoritative map of TPU machine types. + is_tpu = contains(keys(local.tpu_chip_count_map), var.machine_type) + + tpu_taint = local.is_tpu ? [{ + key = "google.com/tpu" + value = "present" + effect = "NO_SCHEDULE" + }] : [] + + # Map of machine prefixes to GKE accelerator labels. + tpu_accelerator_map = { + "ct4p" = "tpu-v4-podslice" # TPU v4 + "ct5lp" = "tpu-v5-lite-podslice" # TPU v5e + "ct5p" = "tpu-v5p-slice" # TPU v5p + "ct6e" = "tpu-v6e-slice" # TPU v6e + "tpu7x" = "tpu7x" # TPU v7x + } + + # Map specific GCE machine types to the number of TPU chips per node (VM). + # The machine-type map must be updated to reflect new TPU releases with reference to public documentation: https://docs.cloud.google.com/tpu/docs/intro-to-tpu + tpu_chip_count_map = { + # v4 - ct4p + "ct4p-hightpu-4t" = 4 + + # v5e - ct5lp + "ct5lp-hightpu-1t" = 1 + "ct5lp-hightpu-4t" = 4 + "ct5lp-hightpu-8t" = 8 + + # v5p - ct5p + "ct5p-hightpu-1t" = 1 + "ct5p-hightpu-2t" = 2 + "ct5p-hightpu-4t" = 4 + + # v6e - ct6e + "ct6e-standard-1t" = 1 + "ct6e-standard-4t" = 4 + "ct6e-standard-8t" = 8 + + # v7x - tpu7x + "tpu7x-standard-4t" = 4 + } + + # Robustly extract the machine family prefix (e.g., "ct6e"). + tpu_machine_family = local.is_tpu ? element(split("-", var.machine_type), 0) : "" + tpu_accelerator_type = local.is_tpu ? lookup(local.tpu_accelerator_map, local.tpu_machine_family, null) : null + tpu_chips_per_node = local.is_tpu ? lookup(local.tpu_chip_count_map, var.machine_type, null) : null +} + +terraform { + required_version = ">= 1.3" +} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/outputs.tf b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/outputs.tf new file mode 100644 index 0000000000..fa3c21fa34 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "is_tpu" { + description = "Boolean value indicating if the node pool is for TPUs." + value = local.is_tpu +} + +output "tpu_accelerator_type" { + description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." + value = local.tpu_accelerator_type +} + +output "tpu_topology" { + description = "The topology of the TPU slice (e.g., '4x4')." + value = local.is_tpu ? var.placement_policy.tpu_topology : null +} + +output "tpu_chips_per_node" { + description = "The number of TPU chips on each node in the pool." + value = local.tpu_chips_per_node +} + +output "tpu_taint" { + description = "A list containing the standard TPU taint object if the node pool is for TPUs." + value = local.tpu_taint +} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/variables.tf b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/variables.tf new file mode 100644 index 0000000000..254488c02d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/variables.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "machine_type" { + description = "The machine type of the node pool." + type = string +} + +variable "placement_policy" { + description = "The placement policy for the node pool." + type = object({ + type = string + name = optional(string) + tpu_topology = optional(string) + }) +} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/README.md b/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/README.md new file mode 100644 index 0000000000..aefac9d187 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/README.md @@ -0,0 +1,56 @@ + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_network_peering.peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [export\_custom\_routes](#input\_export\_custom\_routes) | (Optional) Whether to export the custom routes to the peer network. Defaults to false. | `bool` | `null` | no | +| [import\_custom\_routes](#input\_import\_custom\_routes) | (Optional) Whether to import the custom routes from the peer network. Defaults to false. | `bool` | `null` | no | +| [import\_subnet\_routes\_with\_public\_ip](#input\_import\_subnet\_routes\_with\_public\_ip) | (Optional) Whether subnet routes with public IP range are imported. | `bool` | `null` | no | +| [name](#input\_name) | Name of the peering. | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | The primary network of the peering. | `string` | n/a | yes | +| [peer\_network\_self\_link](#input\_peer\_network\_self\_link) | The peer network in the peering. The peer network may belong to a different project. | `string` | n/a | yes | +| [stack\_type](#input\_stack\_type) | (Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [peering\_name](#output\_peering\_name) | Name of the peering. | + diff --git a/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/main.tf b/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/main.tf new file mode 100644 index 0000000000..386fa9377b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/main.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "name" { + type = string + description = "Name of the peering." +} + +variable "network_self_link" { + type = string + description = "The primary network of the peering." +} + +variable "peer_network_self_link" { + type = string + description = "The peer network in the peering. The peer network may belong to a different project." +} + +variable "export_custom_routes" { + type = bool + description = "(Optional) Whether to export the custom routes to the peer network. Defaults to false." + default = null +} + +variable "import_custom_routes" { + type = bool + description = "(Optional) Whether to import the custom routes from the peer network. Defaults to false." + default = null +} + +variable "import_subnet_routes_with_public_ip" { + type = bool + description = "(Optional) Whether subnet routes with public IP range are imported. " + default = null +} + +variable "stack_type" { + type = string + description = "(Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. " + default = null +} + +resource "google_compute_network_peering" "peering" { + name = var.name + network = var.network_self_link + peer_network = var.peer_network_self_link + export_custom_routes = var.export_custom_routes + import_custom_routes = var.import_custom_routes + import_subnet_routes_with_public_ip = var.import_subnet_routes_with_public_ip + stack_type = var.stack_type +} + +output "peering_name" { + value = google_compute_network_peering.peering.name + description = "Name of the peering." +} + +terraform { + required_version = ">= 0.15.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/metadata.yaml new file mode 100644 index 0000000000..e80fc96b9c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/README.md b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/README.md new file mode 100644 index 0000000000..d7054eb725 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/README.md @@ -0,0 +1,244 @@ +## Description + +This module simplifies the following functionality: + +* Applying Kubernetes manifests to GKE clusters: It provides flexible options for specifying manifests, allowing you to either directly embed them as strings content or reference them from URLs, files, templates, or entire .yaml and .tftpl files in directories. +* Deploying commonly used infrastructure like [Kueue](https://kueue.sigs.k8s.io/docs/) or [Jobset](https://jobset.sigs.k8s.io/docs/). + +> Note: Kueue can work with a variety of frameworks out of the box, find them [here](https://kueue.sigs.k8s.io/docs/tasks/run/) + +### Explanation + +* **Manifest:** + * **Raw String:** Specify manifests directly within the module configuration using the `content: manifest_body` format. + * **File/Template/Directory Reference:** Set `source` to the path to: + * A single URL to a manifest file. Ex.: `https://github.com/.../myrepo/manifest.yaml`. + + > **Note:** Applying from a URL has important limitations. Please review the [Considerations & Callouts for Applying from URLs](#applying-manifests-from-urls-considerations--callouts) section below. + * A single local YAML manifest file (`.yaml`). Ex.: `./manifest.yaml`. + * A template file (`.tftpl`) to generate a manifest. Ex.: `./template.yaml.tftpl`. You can pass the variables to format the template file in `template_vars`. + * A directory containing multiple YAML or template files. Ex: `./manifests/`. You can pass the variables to format the template files in `template_vars`. + +#### Manifest Example + +```yaml +- id: existing-gke-cluster + source: modules/scheduler/pre-existing-gke-cluster + settings: + project_id: $(vars.project_id) + cluster_name: my-gke-cluster + region: us-central1 + +- id: kubectl-apply + source: modules/management/kubectl-apply + use: [existing-gke-cluster] + settings: + - content: | + apiVersion: v1 + kind: Namespace + metadata: + name: my-namespace + - source: "https://github.com/kubernetes-sigs/jobset/releases/download/v0.6.0/manifests.yaml" + - source: $(ghpc_stage("manifests/configmap1.yaml")) + - source: $(ghpc_stage("manifests/configmap2.yaml.tftpl")) + template_vars: {name: "dev-config", public: "false"} + - source: $(ghpc_stage("manifests"))/ + template_vars: {name: "dev-config", public: "false"} +``` + +#### Pre-build infrastructure Example + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml")) + jobset: + install: true +``` + +The `config_path` field in `kueue` installation accepts a template file, too. You will need to provide variables for the template using `config_template_vars` field. + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) + config_template_vars: {name: "dev-config", public: "false"} + jobset: + install: true +``` + +You can specify a particular kueue version that you would like to use using the `version` flag. By default, we recommend customers to [use v0.10.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L68). You can find the list of supported kueue versions [here](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L18). + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + version: v0.10.0 + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) + config_template_vars: {name: "dev-config", public: "false"} + jobset: + install: true +``` + +> **_NOTE:_** +> +> The `project_id` and `region` settings would be inferred from the deployment variables of the same name, but they are included here for clarity. +> +> Terraform may apply resources in parallel, leading to potential dependency issues. If a resource's dependencies aren't ready, it will be applied again up to 15 times. + +## Callouts + +### Applying Manifests from URLs: Considerations & Callouts + +While this module supports applying manifests directly from remote `http://` or `https://` URLs, this method introduces complexities not present when using local files. For production environments, we recommend sourcing manifests from local paths or a version-controlled Git repository. Moreover, this method will be deprecated soon. Hence we recommend to use other methods to source manifests. + +If you choose to use the URL method, be aware of the following potential issues and their solutions. + +#### **1. Apply Order and Race Conditions** + +The module applies manifests from the `apply_manifests` list in parallel. This can create a **race condition** if one manifest depends on another. The most common example is applying a manifest with custom resources (like a `ClusterQueue`) at the same time as the manifest that defines it (the `CustomResourceDefinition` or CRD). + +There is **no guarantee** that the CRD will be applied before the resource that uses it. This can lead to non-deterministic deployment failures with errors like: + +```Error: resource [kueue.x-k8s.io/v1beta1/ClusterQueue] isn't valid for cluster``` + +##### **Recommended Workaround: Two-Stage Apply** + +To ensure a reliable deployment, you must manually enforce the correct order of operations. + +1. **Initial Deployment:** In your blueprint, include **only** the manifest(s) containing the `CustomResourceDefinition` (CRD) resources in the `apply_manifests` list. + + *Example `settings` for the first run:* + + ```yaml + settings: + apply_manifests: + # This manifest contains the CRDs for Kueue + - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" + server_side_apply: true + ``` + +2. **Run the deployment** (`gcluster deploy` or `terraform apply`). + +3. **Second Deployment:** Once the first apply is successful, **add** the manifests containing your custom resources (like `ClusterQueue`, `LocalQueue`) to the list. + + *Example `settings` for the second run:* + + ```yaml + settings: + apply_manifests: + # The CRD manifest is still present + - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" + server_side_apply: true + + # Now, add your configuration manifest + - source: "https://gist.githubusercontent.com/YourUser/..." # Your configuration URL + server_side_apply: true + ``` + +4. **Run the deployment command again.** Since the CRDs are now guaranteed to exist in the cluster, this second apply will succeed reliably. + +#### **2. Large Manifests (CRDs)** + +* **Issue:** Applying very large manifests can fail with a `metadata.annotations: Too long` error. +* **Solution:** Enable Server-Side Apply by setting `server_side_apply: true` for the manifest entry. + +#### **3. Conflicts on Re-application** + +* **Issue:** Re-running a deployment after a partial failure can cause server-side apply field manager `conflicts`. +* **Solution:** Forcibly take ownership of the resource fields by setting `force_conflicts: true`. + +#### **4. Terraform Template Files (`.tftpl`)** + +* **Limitation:** This module **cannot** render a template file (`.tftpl`) when sourced from a remote URL. +* **Workaround:** You must render the template into a pure YAML file locally, host that rendered file at a URL, and provide the URL of the rendered file in your blueprint. + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 7.2 | +| [helm](#requirement\_helm) | ~> 2.17 | +| [http](#requirement\_http) | ~> 3.0 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 7.2 | +| [http](#provider\_http) | ~> 3.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [configure\_kueue](#module\_configure\_kueue) | ./kubectl | n/a | +| [install\_gib](#module\_install\_gib) | ./kubectl | n/a | +| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | +| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | +| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | +| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | +| [kubectl\_apply\_manifests](#module\_kubectl\_apply\_manifests) | ./kubectl | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.gib_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.initial_gib_version](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.jobset_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.kueue_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | +| [http_http.manifest_from_url](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [apply\_manifests](#input\_apply\_manifests) | A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md).
NOTE: The `enable` input acts as a FF to apply a manifest or not. By default it is always set to `true`. | 
list(object({
    enable            = optional(bool, true)
    content           = optional(string, null)
    source            = optional(string, null)
    template_vars     = optional(map(any), null)
    server_side_apply = optional(bool, false)
    wait_for_rollout  = optional(bool, true)
  }))
| `[]` | no | +| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | +| [gib](#input\_gib) | Install the NCCL gIB plugin | 
object({
    install = bool
    path    = string
    template_vars = object({
      image   = optional(string, "us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib")
      version = string
      node_affinity = optional(any, {
        requiredDuringSchedulingIgnoredDuringExecution = {
          nodeSelectorTerms = [{
            matchExpressions = [{
              key      = "cloud.google.com/gke-gpu",
              operator = "In",
              values   = ["true"]
            }]
          }]
        }
      })
      accelerator_count = number
      max_unavailable   = optional(string, "50%")
    })
  })
| 
{
  "install": false,
  "path": "",
  "template_vars": {
    "accelerator_count": 0,
    "version": ""
  }
}
| no | +| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | +| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "0.10.1")
  })
| `{}` | no | +| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "0.13.3")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | +| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | +| [target\_architecture](#input\_target\_architecture) | The target architecture for the GKE nodes and gIB plugin (e.g., 'x86\_64' or 'arm64'). | `string` | `"x86_64"` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/README.md b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/README.md new file mode 100644 index 0000000000..1957899617 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/README.md @@ -0,0 +1,64 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [helm](#provider\_helm) | ~> 2.17 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | +| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | +| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | +| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | +| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | +| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | +| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | +| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | +| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | +| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | +| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | +| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | +| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | +| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | +| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | +| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | +| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | +| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | +| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | +| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | +| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | +| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | +| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | +| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | +| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | +| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | +| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | +| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | +| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | +| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf new file mode 100644 index 0000000000..8cc09bd3e2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf @@ -0,0 +1,79 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "helm_release" "apply_chart" { + # Required Identification + name = var.release_name + chart = var.chart_name + + # Chart Source & Version + repository = var.chart_repository + version = var.chart_version + devel = var.devel + + # Target Namespace + namespace = var.namespace + create_namespace = var.create_namespace + + # Values Configuration + values = var.values_yaml + + dynamic "set" { + for_each = var.set_values + content { + name = set.value.name + value = set.value.value + type = set.value.type + } + } + + # Installation/Upgrade Behavior + description = var.description + atomic = var.atomic + cleanup_on_fail = var.cleanup_on_fail + dependency_update = var.dependency_update + disable_crd_hooks = var.disable_crd_hooks + disable_openapi_validation = var.disable_openapi_validation + disable_webhooks = var.disable_webhooks + force_update = var.force_update + lint = var.lint + max_history = var.max_history + recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI + render_subchart_notes = var.render_subchart_notes + reset_values = var.reset_values + reuse_values = var.reuse_values + skip_crds = var.skip_crds + timeout = var.timeout + wait = var.wait + wait_for_jobs = var.wait_for_jobs + + # Verification & Credentials + keyring = var.keyring + pass_credentials = var.pass_credentials + verify = var.verify + + # Post Rendering + dynamic "postrender" { + # Only include the block if var.postrender is not null + for_each = var.postrender == null ? [] : [var.postrender] + content { + binary_path = postrender.value.binary_path + } + } + + # Lifecycle block (optional - generally avoid complex lifecycle in generic modules) + # lifecycle { + # ignore_changes = [] + # } +} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf new file mode 100644 index 0000000000..04e8e214fc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf @@ -0,0 +1,212 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +# --- Required --- +variable "release_name" { + description = "Name of the Helm release." + type = string +} + +variable "chart_name" { + description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." + type = string +} + +# --- Chart Location & Version --- +variable "chart_repository" { + description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." + type = string + default = null +} + +variable "chart_version" { + description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." + type = string + default = null +} + +variable "devel" { + description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." + type = bool + default = false +} + +# --- Namespace --- +variable "namespace" { + description = "Kubernetes namespace to install the Helm release into." + type = string + default = "default" +} + +variable "create_namespace" { + description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." + type = bool + default = true # Common convenience setting +} + +# --- Values Customization --- +variable "values_yaml" { + description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." + type = list(string) + default = [] +} + +variable "set_values" { + description = "List of objects defining values to set ('helm install --set')." + type = list(object({ + name = string # Path to the value (e.g., 'service.type', 'replicaCount') + value = string # The value to set + type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') + })) + default = [] +} + +# --- Installation/Upgrade Behavior --- +variable "description" { + description = "Set an optional description for the Helm release." + type = string + default = null +} + +variable "atomic" { + description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." + type = bool + default = false +} + +variable "wait" { + description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." + type = bool + default = true # Often a good default for dependencies +} + +variable "wait_for_jobs" { + description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." + type = bool + default = false # Helm CLI default is false +} + +variable "timeout" { + description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." + type = number + default = 300 # 5 minutes (Helm CLI default) +} + +variable "cleanup_on_fail" { + description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." + type = bool + default = false +} + +variable "dependency_update" { + description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." + type = bool + default = false +} + +variable "disable_crd_hooks" { + description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." + type = bool + default = false +} + +variable "disable_openapi_validation" { + description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." + type = bool + default = false +} + +variable "disable_webhooks" { + description = "Prevent hooks from running ('helm install --no-hooks')." + type = bool + default = false +} + +variable "force_update" { + description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." + type = bool + default = false +} + +variable "lint" { + description = "Run the helm chart linter during the plan ('helm lint')." + type = bool + default = false +} + +variable "max_history" { + description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." + type = number + default = null # Terraform provider defaults to Helm's default (usually 10) +} + +variable "recreate_pods" { + description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." + type = bool + default = false +} + +variable "render_subchart_notes" { + description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." + type = bool + default = false +} + +variable "reset_values" { + description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." + type = bool + default = false +} + +variable "reuse_values" { + description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." + type = bool + default = false # Helm CLI default is false +} + +variable "skip_crds" { + description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." + type = bool + default = false +} + +# --- Verification & Credentials --- +variable "keyring" { + description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." + type = string + default = null # Defaults to Helm's default keyring location +} + +variable "pass_credentials" { + description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." + type = bool + default = false +} + +variable "verify" { + description = "Verify the package before installing it ('helm install --verify')." + type = bool + default = false +} + +# --- Advanced Rendering --- +variable "postrender" { + description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." + type = object({ + binary_path = string # Path to the post-renderer executable + }) + default = null # Disabled by default +} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf new file mode 100644 index 0000000000..09d912e2c9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml new file mode 100644 index 0000000000..92fc1bca22 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml @@ -0,0 +1,25 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For referencing the original jobset helm chart values, pull the latest jobset chart version +# `helm pull oci://registry.k8s.io/jobset/charts/jobset --version=0.10.1` (latest helm chart version) + +controller: + # It ensures the Jobset pod(s) can be scheduled on GKE clusters where the + # system node pool uses the default "gke-managed-components" taint. + tolerations: + - key: "components.gke.io/gke-managed-components" + operator: "Equal" + value: "true" + effect: "NoSchedule" diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/README.md b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/README.md new file mode 100644 index 0000000000..691f4dc34a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/README.md @@ -0,0 +1,55 @@ + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | + +## Providers + +| Name | Version | +|------|---------| +| [kubectl](#provider\_kubectl) | >= 1.7.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubectl_manifest.apply_doc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_path_documents.templates](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | +| [kubectl_path_documents.yamls](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | +| [force\_conflicts](#input\_force\_conflicts) | The force\_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller) | `bool` | `false` | no | +| [server\_side\_apply](#input\_server\_side\_apply) | Allow using kubectl server-side apply method. | `bool` | `false` | no | +| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `null` | no | +| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | +| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf new file mode 100644 index 0000000000..acf1d3c908 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf @@ -0,0 +1,92 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + yaml_separator = "\n---" + + # This locals block processes manifest inputs from one of four methods, + # evaluated in order of precedence using coalesce. + + # --- METHOD 1: Direct Content Input --- + # Used when manifest content is passed directly as a string. + content_yaml_body = var.content + + # Fallback for safe path checking in subsequent methods. + null_safe_source = coalesce(var.source_path, " ") + + # --- METHOD 2: Single Local YAML File --- + # Used when var.source_path points to a local .yaml file. + yaml_file = length(regexall("\\.yaml(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null + yaml_file_content = local.yaml_file != null ? file(local.yaml_file) : null + + # --- METHOD 3: Single Local Template File --- + # Used when var.source_path points to a local .tftpl file. + template_file = length(regexall("\\.tftpl(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null + template_file_content = local.template_file != null ? templatefile(local.template_file, var.template_vars) : null + + # --- CONSOLIDATE & PROCESS --- + # Coalesce finds the first non-null content from the methods above. + yaml_body = coalesce(local.content_yaml_body, local.yaml_file_content, local.template_file_content, " ") + # Ensure only valid YAML is processed + # It explicitly tests if the content can be decoded before including it. + yaml_body_docs = compact(flatten([ + for doc in split(local.yaml_separator, local.yaml_body) : [ + for content in [trimspace(doc)] : ( + # Use a temporary local variable and can() to test for successful YAML decoding. + # This handles malformed documents (like comment blocks) which cause yamldecode() to fail. + can(yamldecode(content)) && length(yamldecode(content)) > 0 ? content : null + ) + ] + ])) + + # --- METHOD 4: Directory of Files --- + # If no content was found via the methods above AND the source path looks like a directory, + # we assume this is the desired method. The data blocks below will handle it. + directory = length(local.yaml_body_docs) == 0 && endswith(local.null_safe_source, "/") ? abspath(var.source_path) : null + + # --- FINAL AGGREGATION --- + # Combine documents from single-source methods and directory-scan methods into one list. + docs_list = concat(try(local.yaml_body_docs, []), try(data.kubectl_path_documents.yamls[0].documents, []), try(data.kubectl_path_documents.templates[0].documents, [])) + docs_map = tomap({ + for index, doc in local.docs_list : index => doc + }) +} + +data "kubectl_path_documents" "yamls" { + count = local.directory != null ? 1 : 0 + pattern = "${local.directory}/*.yaml" +} + +data "kubectl_path_documents" "templates" { + count = local.directory != null ? 1 : 0 + pattern = "${local.directory}/*.tftpl" + vars = var.template_vars +} + +resource "kubectl_manifest" "apply_doc" { + for_each = local.docs_map + yaml_body = each.value + server_side_apply = var.server_side_apply + wait_for_rollout = var.wait_for_rollout + force_conflicts = var.force_conflicts + + lifecycle { + precondition { + condition = !var.force_conflicts || var.server_side_apply + error_message = "The 'force_conflicts' variable can only be set to true when 'server_side_apply' is also true." + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf new file mode 100644 index 0000000000..7bf34e089c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "content" { + description = "The YAML body to apply to gke cluster." + type = string + default = null +} + +variable "source_path" { + description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." + type = string + default = null +} + +variable "template_vars" { + description = "The values to populate template file(s) with." + type = any + default = null +} + +variable "server_side_apply" { + description = "Allow using kubectl server-side apply method." + type = bool + default = false +} + +variable "wait_for_rollout" { + description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." + type = bool + default = true +} + +variable "force_conflicts" { + description = "The force_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller)" + type = bool + default = false +} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf new file mode 100644 index 0000000000..cce452239f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.7.0" + } + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml new file mode 100644 index 0000000000..7c0bef7013 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml @@ -0,0 +1,30 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For referencing the original Kueue helm chart values, pull the latest helm chart version +# `helm pull oci://registry.k8s.io/kueue/charts/kueue --version=0.13.3` (latest helm chart version) + +controllerManager: + # -- Enables the Topology-Aware Scheduling feature gate. + featureGates: + - name: TopologyAwareScheduling + enabled: true + + # It ensures the Kueue pod can schedule on GKE clusters where the + # system node pool uses the default "gke-managed-components" taint. + tolerations: + - key: "components.gke.io/gke-managed-components" + operator: "Equal" + value: "true" + effect: "NoSchedule" diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/main.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/main.tf new file mode 100644 index 0000000000..73a15ad1ab --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/main.tf @@ -0,0 +1,271 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] + project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] + + # 1. First, Identify manifests that are explicitly enabled. + enabled_manifests = { + for index, manifest in var.apply_manifests : index => manifest + if try(manifest.enable, true) + } + + # 2. Identify URL-based manifests + url_manifests = { + for index, manifest in local.enabled_manifests : index => manifest + if try(manifest.source, null) != null && (startswith(manifest.source, "http://") || startswith(manifest.source, "https://")) + } + + # 3. Rebuild the map by populating the 'content' field for URLs based manifest + processed_apply_manifests_map = tomap({ + for index, manifest in local.enabled_manifests : tostring(index) => { + # If this manifest was a URL, its content is the body from the HTTP call. + content = contains(keys(local.url_manifests), tostring(index)) ? data.http.manifest_from_url[tostring(index)].body : manifest.content + + # If this was a URL, its source path is now null. Otherwise, use original. + source = contains(keys(local.url_manifests), tostring(index)) ? null : manifest.source + + # Pass other vars + template_vars = manifest.template_vars + server_side_apply = manifest.server_side_apply + wait_for_rollout = manifest.wait_for_rollout + } + }) + + install_kueue = try(var.kueue.install, false) + install_jobset = try(var.jobset.install, false) + install_gpu_operator = try(var.gpu_operator.install, false) + install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) + install_gib = try(var.gib.install, false) +} + +data "http" "manifest_from_url" { + for_each = local.url_manifests + url = each.value.source +} + +data "google_container_cluster" "gke_cluster" { + project = local.project_id + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +module "kubectl_apply_manifests" { + for_each = local.processed_apply_manifests_map + source = "./kubectl" + depends_on = [var.gke_cluster_exists] + + content = each.value.content + source_path = each.value.source + template_vars = each.value.template_vars + server_side_apply = each.value.server_side_apply + wait_for_rollout = each.value.wait_for_rollout + + providers = { + kubectl = kubectl + } +} + +module "install_kueue" { + source = "./helm_install" + count = local.install_kueue ? 1 : 0 + wait = false + timeout = 1200 + release_name = "kueue" + chart_repository = "oci://registry.k8s.io/kueue/charts" + chart_name = "kueue" + chart_version = var.kueue.version + namespace = "kueue-system" + create_namespace = true + values_yaml = [ + file("${path.module}/kueue/kueue-helm-values.yaml") + ] + + depends_on = [var.gke_cluster_exists] +} + +module "configure_kueue" { + source = "./kubectl" + source_path = local.install_kueue ? try(var.kueue.config_path, "") : null + template_vars = local.install_kueue ? try(var.kueue.config_template_vars, null) : null + depends_on = [module.install_kueue] + + server_side_apply = true + wait_for_rollout = true + + providers = { + kubectl = kubectl + } +} + +module "install_jobset" { + source = "./helm_install" + count = local.install_jobset ? 1 : 0 + wait = false + timeout = 1200 + release_name = "jobset" + chart_repository = "oci://registry.k8s.io/jobset/charts" + chart_name = "jobset" + chart_version = var.jobset.version + namespace = "jobset-system" + create_namespace = true + values_yaml = [ + file("${path.module}/jobset/jobset-helm-values.yaml") + ] + depends_on = [var.gke_cluster_exists, module.configure_kueue] +} + +module "install_nvidia_dra_driver" { + count = local.install_nvidia_dra_driver ? 1 : 0 + depends_on = [module.kubectl_apply_manifests, var.gke_cluster_exists, module.configure_kueue] + source = "./helm_install" + + release_name = "nvidia-dra-driver-gpu" # The release name + chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts + chart_name = "nvidia-dra-driver-gpu" # The chart name + chart_version = var.nvidia_dra_driver.version # The chart version + namespace = "nvidia-dra-driver-gpu" # The target namespace + create_namespace = true # Equivalent to --create-namespace + + # Use the 'values' argument to pass the YAML content + # This corresponds to the -f <(cat < +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_monitoring_dashboard.dashboard](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_dashboard) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [base\_dashboard](#input\_base\_dashboard) | Baseline dashboard template, select from HPC or Empty | `string` | `"HPC"` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the monitoring dashboard instance. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [title](#input\_title) | Title of the created dashboard | `string` | `"Cluster Toolkit Dashboard"` | no | +| [widgets](#input\_widgets) | List of additional widgets to add to the base dashboard. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for accessing the monitoring dashboard | + diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl new file mode 100644 index 0000000000..f25cbbd2c6 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl @@ -0,0 +1,17 @@ +{ + "displayName": "${title}: ${deployment_name}", + "gridLayout": { + "columns": 2, + "widgets": [ + { + "text": { + "content": "Metrics from the ${deployment_name} deployment of the Cluster Toolkit.", + "format": "MARKDOWN" + }, + "title": "${title}" + }%{ for widget in widgets ~}, + ${widget} + %{endfor ~} + ] + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl new file mode 100644 index 0000000000..5b20435a9a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl @@ -0,0 +1,595 @@ +{ + "displayName": "${title}: ${deployment_name}", + "labels": ${jsonencode(labels)}, + "gridLayout": { + "columns": 2, + "widgets": [ + { + "text": { + "content": "HPC metrics from the ${deployment_name} deployment of the Cluster Toolkit.", + "format": "MARKDOWN" + }, + "title": "${title}" + }, + { + "title": "VM Instance - Memory utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/memory/percent_used\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - CPU Utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", + "pickTimeSeriesFilter": { + "direction": "TOP", + "numTimeSeries": 20, + "rankingMethod": "METHOD_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - CPU utilization (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + }, + "unitOverride": "%" + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk read operations", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/read_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk write operations", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/write_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk Read Bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/disk/read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk Write Bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/disk/write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Throttled read bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Throttled write bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Received packets", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/received_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - Sent packets", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - Received bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/received_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Sent bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_MEAN", + "groupByFields": [ + "metric.label.\"instance_name\"", + "metric.label.\"loadbalanced\"", + "resource.label.\"project_id\"", + "resource.label.\"instance_id\"", + "resource.label.\"zone\"" + ], + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Network Traffic Bytes (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/interface/traffic\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Network Packets (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/interface/packets\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "TCP connections", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/network/tcp_connections\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + }, + "unitOverride": "1" + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - CPU utilization for steal", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "STACKED_BAR", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MAX" + }, + "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\" metric.label.\"cpu_state\"=\"steal\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - CPU utilization [MEAN]", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }%{ for widget in widgets ~}, + ${widget} + %{endfor ~} + ] + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/main.tf b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/main.tf new file mode 100644 index 0000000000..df3c5c36b0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/main.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "dashboard", ghpc_role = "monitoring" }) +} + +locals { + dash_path = "${path.module}/dashboards/${var.base_dashboard}.json.tpl" +} + +resource "google_monitoring_dashboard" "dashboard" { + dashboard_json = templatefile(local.dash_path, { + widgets = var.widgets + deployment_name = var.deployment_name + title = var.title + labels = local.labels + } + ) + project = var.project_id +} diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/metadata.yaml new file mode 100644 index 0000000000..de1a10f57d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - stackdriver.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/outputs.tf b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/outputs.tf new file mode 100644 index 0000000000..b7ff35fb0e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/outputs.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instructions" { + description = "Instructions for accessing the monitoring dashboard" + value = <<-EOT + A monitoring dashboard has been created. To view, navigate to the following URL: + https://console.cloud.google.com/monitoring/dashboards/builder${regex("/[0-9a-z-]*$", google_monitoring_dashboard.dashboard.id)} + EOT +} diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/variables.tf b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/variables.tf new file mode 100644 index 0000000000..8194f8b73a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/variables.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "base_dashboard" { + description = "Baseline dashboard template, select from HPC or Empty" + type = string + default = "HPC" + validation { + condition = contains(["HPC", "Empty"], var.base_dashboard) + error_message = "Must set var.base_dashboard to either \"HPC\" or \"Empty\"." + } +} + +variable "title" { + description = "Title of the created dashboard" + type = string + default = "Cluster Toolkit Dashboard" +} + +variable "widgets" { + description = "List of additional widgets to add to the base dashboard." + type = list(string) + default = [] +} + +variable "labels" { + description = "Labels to add to the monitoring dashboard instance. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/versions.tf b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/versions.tf new file mode 100644 index 0000000000..2717fe79f6 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/README.md b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/README.md new file mode 100644 index 0000000000..057f4b649d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/README.md @@ -0,0 +1,111 @@ +## Description + +This module facilitates the creation of custom firewall rules for existing +networks. + +## Example usage + +This module can be used by other Toolkit modules to create application-specific +firewall rules or in conjunction with the [pre-existing-vpc] module to enable +traffic in existing networks. The snippet below is drawn from the +[ml-slurm.yaml] example: + +```yaml +- group: primary + modules: + - id: network + source: modules/network/pre-existing-vpc + + # this example anticipates that the VPC default network has internal traffic + # allowed and IAP tunneling for SSH connections + - id: firewall_rule + source: modules/network/firewall-rules + use: + - network + settings: + ingress_rules: + - name: $(vars.deployment_name)-allow-internal-traffic + description: Allow internal traffic + destination_ranges: + - $(network.subnetwork_address) + source_ranges: + - $(network.subnetwork_address) + allow: + - protocol: tcp + ports: + - 0-65535 + - protocol: udp + ports: + - 0-65535 + - protocol: icmp + - name: $(vars.deployment_name)-allow-iap-ssh + description: Allow IAP-tunneled SSH connections + destination_ranges: + - $(network.subnetwork_address) + source_ranges: + - 35.235.240.0/20 + allow: + - protocol: tcp + ports: + - 22 +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [firewall\_rule](#module\_firewall\_rule) | terraform-google-modules/network/google//modules/firewall-rules | ~> 12.0 | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.pga_check](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [google_compute_subnetwork.subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [egress\_rules](#input\_egress\_rules) | List of egress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | +| [ingress\_rules](#input\_ingress\_rules) | List of ingress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | +| [network\_name](#input\_network\_name) | The name of the network to create firewall rules in | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The project ID to host the network in | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork whose global network firewall rules will be modified. | `string` | n/a | yes | + +## Outputs + +No outputs. + + +[pre-existing-vpc]: ../pre-existing-vpc/README.md +[ml-slurm.yaml]: ../../../examples/ml-slurm.yaml diff --git a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/main.tf b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/main.tf new file mode 100644 index 0000000000..05241278ad --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/main.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + use_subnetwork_data = (var.project_id == null || var.network_name == null) && var.subnetwork_self_link != null +} + +# the google_compute_network data source does not allow identification by +# self_link, which uniquely identifies subnet, project, and network +data "google_compute_subnetwork" "subnetwork" { + # Only instantiate this data source if needed + count = local.use_subnetwork_data ? 1 : 0 + self_link = var.subnetwork_self_link +} + +locals { + # Derived values from data source, null if data source is not used + derived_project_id = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].project : null + derived_network_name = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].network : null + + # Effective values: Use var if provided, otherwise use derived value + effective_project_id = coalesce(var.project_id, local.derived_project_id) + effective_network_name = coalesce(var.network_name, local.derived_network_name) +} + +# Module-level check for Private Google Access on the subnetwork. +# This check is only relevant if subnetwork_self_link was provided and used. +resource "terraform_data" "pga_check" { + count = local.use_subnetwork_data ? 1 : 0 + + lifecycle { + precondition { + condition = data.google_compute_subnetwork.subnetwork[0].private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.subnetwork[0].name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } + } +} + +module "firewall_rule" { + source = "terraform-google-modules/network/google//modules/firewall-rules" + version = "~> 12.0" + project_id = local.effective_project_id + network_name = local.effective_network_name + + ingress_rules = var.ingress_rules + egress_rules = var.egress_rules +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/variables.tf b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/variables.tf new file mode 100644 index 0000000000..05e9be4425 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/variables.tf @@ -0,0 +1,88 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork whose global network firewall rules will be modified." + type = string +} + +variable "project_id" { + description = "The project ID to host the network in" + type = string + default = null +} + +variable "network_name" { + description = "The name of the network to create firewall rules in" + type = string + default = null +} + +variable "ingress_rules" { + description = "List of ingress rules" + default = [] + type = list(object({ + name = string + description = optional(string, null) + disabled = optional(bool, null) + priority = optional(number, null) + destination_ranges = optional(list(string), []) + source_ranges = optional(list(string), []) + source_tags = optional(list(string)) + source_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + target_service_accounts = optional(list(string)) + + allow = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + deny = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + log_config = optional(object({ + metadata = string + })) + })) +} + +variable "egress_rules" { + description = "List of egress rules" + default = [] + type = list(object({ + name = string + description = optional(string, null) + disabled = optional(bool, null) + priority = optional(number, null) + destination_ranges = optional(list(string), []) + source_ranges = optional(list(string), []) + source_tags = optional(list(string)) + source_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + target_service_accounts = optional(list(string)) + + allow = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + deny = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + log_config = optional(object({ + metadata = string + })) + })) +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/versions.tf new file mode 100644 index 0000000000..9061dd3ae5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.74.0" + } + + required_version = ">= 1.5" +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/README.md b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/README.md new file mode 100644 index 0000000000..abbfe3b97b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/README.md @@ -0,0 +1,143 @@ +## Description + +This module accomplishes the following: + +* Creates one [VPC network][cft-network] + * Each VPC contains a variable number of subnetworks as specified in the + `subnetworks_template` variable + * Each subnetwork contains distinct IP address ranges +* Outputs the following unique parameters + * `subnetwork_interfaces` which is compatible with Slurm and vm-instance + modules + * `subnetwork_interfaces_gke` which is compatible with GKE modules + +This module is a simplified version of the VPC module and its main difference +is the variable `subnetwork_template` which is the template for all subnetworks +created within the network. This template contains the following values: + +1. `count`: The number of subnetworks to be created +1. `name_prefix`: The prefix for the subnetwork names +1. `ip_range`: [CIDR-formatted IP range][cidr] +1. `region`: The region where the subnetwork will be deployed + +> [!WARNING] +> The `ip_range` should be always be large enough to split into `count` +> subnetworks and the number of required connections within. + +[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v10.0.0 +[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation + +### Example + +This snippet uses the gpu-vpc module to create a new VPC network named +`test-rdma-net` with 8 subnetworks named `test-mrdma-sub-#` where # ranges from +0 to 7. The subnetworks will split the `ip_range` evenly, starting from bit 16 +(0 indexed). The networks are ingested by the Slurm nodeset within the +`additional_networks` setting. + +```yaml + - id: rdma-net + source: modules/network/gpu-rdma-vpc + settings: + network_name: test-rdma-net + network_profile: https://www.googleapis.com/compute/beta/projects/$(vars.project_id)/global/networkProfiles/$(vars.zone)-vpc-roce + network_routing_mode: REGIONAL + subnetworks_template: + name_prefix: test-mrdma-sub + count: 8 + ip_range: 192.168.0.0/16 + region: $(vars.region) + + - id: a3_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network0] + settings: + machine_type: a3-ultragpu-8g + additional_networks: + $(concat( + [{ + network=null, + subnetwork=network1.subnetwork_self_link, + subnetwork_project=vars.project_id, + nic_type="GVNIC", + queue_count=null, + network_ip="", + stack_type=null, + access_config=[], + ipv6_access_config=[], + alias_ip_range=[] + }], + rdma-net.subnetwork_interfaces + )) + ... +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | DEPRECATED: enable\_internal\_traffic can not be specified for gpu-rdma-vpc. | `bool` | `null` | no | +| [firewall\_log\_config](#input\_firewall\_log\_config) | DEPRECATED: firewall\_log\_config can not be specified for gpu-rdma-vpc. | `string` | `null` | no | +| [firewall\_rules](#input\_firewall\_rules) | DEPRECATED: firewall\_rules can not be specified for gpu-rdma-vpc. | `any` | `null` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}} | `string` | n/a | yes | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "REGIONAL") | `string` | `"REGIONAL"` | no | +| [nic\_type](#input\_nic\_type) | NIC type for use in modules that use the output | `string` | `"MRDMA"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | +| [subnetworks\_template](#input\_subnetworks\_template) | Specifications for the subnetworks that will be created within this VPC.

count (number, required, number of subnets to create, default is 8)
name\_prefix (string, required, subnet name prefix, default is deployment name)
ip\_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16)
region (string, optional, region to deploy subnets to, defaults to vars.region) | 
object({
    count       = number
    name_prefix = string
    ip_range    = string
    region      = optional(string)
  })
| 
{
  "count": 8,
  "ip_range": "192.168.0.0/16",
  "name_prefix": null,
  "region": null
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [network\_id](#output\_network\_id) | ID of the new VPC network | +| [network\_name](#output\_network\_name) | Name of the new VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | +| [subnetwork\_interfaces](#output\_subnetwork\_interfaces) | Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules) | +| [subnetwork\_interfaces\_gke](#output\_subnetwork\_interfaces\_gke) | Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool) | +| [subnetwork\_name\_prefix](#output\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | +| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | + diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/main.tf b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/main.tf new file mode 100644 index 0000000000..e37db01976 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/main.tf @@ -0,0 +1,79 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + autoname = replace(var.deployment_name, "_", "-") + network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name + subnet_prefix = var.subnetworks_template.name_prefix == null ? "${local.autoname}-subnet" : var.subnetworks_template.name_prefix + + new_bits = ceil(log(var.subnetworks_template.count, 2)) + template_subnetworks = [for i in range(var.subnetworks_template.count) : + { + subnet_name = "${local.subnet_prefix}-${i}" + subnet_region = try(var.subnetworks_template.region, var.region) + subnet_ip = cidrsubnet(var.subnetworks_template.ip_range, local.new_bits, i) + } + ] + + firewall_rules = [] + + output_subnets = [ + for subnet in module.vpc.subnets : { + network = null + subnetwork = subnet.self_link + subnetwork_project = null # will populate from subnetwork_self_link + network_ip = null + nic_type = var.nic_type + stack_type = null + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } + ] + + output_subnets_gke = [ + for i in range(length(module.vpc.subnets)) : { + network = local.network_name + subnetwork = local.template_subnetworks[i].subnet_name + subnetwork_project = var.project_id + network_ip = null + nic_type = var.nic_type + stack_type = null + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } + ] +} + +module "vpc" { + source = "terraform-google-modules/network/google" + version = "~> 12.0" + + network_name = local.network_name + project_id = var.project_id + auto_create_subnetworks = false + subnets = local.template_subnetworks + routing_mode = var.network_routing_mode + mtu = var.mtu + description = var.network_description + shared_vpc_host = var.shared_vpc_host + delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes + firewall_rules = local.firewall_rules + network_profile = var.network_profile +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf new file mode 100644 index 0000000000..0a21f1d3f2 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the new VPC network" + value = module.vpc.network_name + depends_on = [module.vpc] +} + +output "network_id" { + description = "ID of the new VPC network" + value = module.vpc.network_id + depends_on = [module.vpc] +} + +output "network_self_link" { + description = "Self link of the new VPC network" + value = module.vpc.network_self_link + depends_on = [module.vpc] +} + +output "subnetworks" { + description = "Full list of subnetwork objects belonging to the new VPC network" + value = module.vpc.subnets + depends_on = [module.vpc] +} + +output "subnetwork_interfaces" { + description = "Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules)" + value = local.output_subnets + depends_on = [module.vpc] +} + +# The output subnetwork_interfaces is compatible with vm-instance module but not with gke-node-pool +# See https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/99493df21cecf6a092c45298bf7a45e0343cf622/modules/compute/vm-instance/variables.tf#L220 +# So, we need a separate output that makes the network and subnetwork names available +output "subnetwork_interfaces_gke" { + description = "Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool)" + value = local.output_subnets_gke + depends_on = [module.vpc] +} + +output "subnetwork_name_prefix" { + description = "Prefix of the RDMA subnetwork names" + value = var.subnetworks_template.name_prefix +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf new file mode 100644 index 0000000000..a30fb50e7d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf @@ -0,0 +1,164 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "network_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" + type = string + default = null +} + +variable "region" { + description = "The default region for Cloud resources" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "subnetworks_template" { + description = <<-EOT + Specifications for the subnetworks that will be created within this VPC. + + count (number, required, number of subnets to create, default is 8) + name_prefix (string, required, subnet name prefix, default is deployment name) + ip_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16) + region (string, optional, region to deploy subnets to, defaults to vars.region) + EOT + nullable = false + type = object({ + count = number + name_prefix = string + ip_range = string + region = optional(string) + }) + default = { + count = 8 + name_prefix = null + ip_range = "192.168.0.0/16" + region = null + } + + validation { + condition = var.subnetworks_template.count > 0 + error_message = "Number of subnetworks must be greater than 0" + } + + validation { + condition = can(cidrhost(var.subnetworks_template.ip_range, 0)) + error_message = "IP address range must be in CIDR format." + } +} + +variable "network_routing_mode" { + type = string + default = "REGIONAL" + description = "The network routing mode (default \"REGIONAL\")" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "shared_vpc_host" { + type = bool + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + default = false +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_internal_traffic" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." + type = bool + default = null + validation { + condition = var.enable_internal_traffic == null + error_message = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." + } +} + +variable "firewall_rules" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." + type = any + default = null + validation { + condition = var.firewall_rules == null + error_message = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." + } +} + +variable "firewall_log_config" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." + type = string + default = null + validation { + condition = var.firewall_log_config == null + error_message = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." + } +} + +variable "network_profile" { + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + EOT + type = string + nullable = false + + validation { + condition = can(coalesce(var.network_profile)) + error_message = "var.network_profile must be specified and not an empty string" + } +} + +variable "nic_type" { + description = "NIC type for use in modules that use the output" + type = string + nullable = true + default = "MRDMA" + + validation { + condition = contains(["MRDMA"], var.nic_type) + error_message = "The nic_type must be \"MRDMA\"." + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf new file mode 100644 index 0000000000..71b7106734 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.15.0" +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/multivpc/README.md b/deletion-test/build_script/modules/embedded/modules/network/multivpc/README.md new file mode 100644 index 0000000000..973e6b32c9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/multivpc/README.md @@ -0,0 +1,136 @@ +## Description + +This module accomplishes the following: + +* Creates 2 to 8 [VPC networks][vpc] + * Each VPC contains exactly 1 subnetwork + * Each subnetwork contains distinct IP address ranges +* Outputs the `additional_networks` parameter, which is compatible with Slurm + modules + +There are 4 variables that differentiate this module from the standard VPC +module. + +1. `network_prefix`: The name prefix of the VPCs to be created. All + networks and subnetworks will start with this and end with a unique number. +1. `network_count`: The number of VPCs to be created. +1. `global_ip_address_range`: [CIDR-formatted IP range][cidr] +1. `network_cidr_suffix`: The CIDR suffix that defines the address + space that the individual VPCs will cover. + +> [!WARNING] +> The `network_cidr_suffix` should be always be larger than the CIDR suffix on +> `global_ip_address_range`. The difference between these two suffixes should +> be large enough to accommodate the number of VPCs that are being deployed +> (e.g. CIDR suffix bit difference <= `ceil(log2(network_count)))`). + +> [!NOTE] +> For deployments that need multiple VPCs that do not meet this use-case, users +> should deploy multiple individual VPC modules. + +[vpc]: ../vpc/README.md +[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation + +### Example + +This snippet uses the multivpc module to create 8 new VPC networks named +`multivpc-net-#` where # ranges from 0 to 7. Additionally, it creates 1 +subnetwork in each VPC. + +```yaml + - id: network + source: modules/network/vpc + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: a3_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network, multinetwork] + settings: + machine_type: a3-highgpu-8g + ... +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [vpcs](#module\_vpcs) | ../vpc | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.global_ip_cidr_suffix](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | +| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | +| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | +| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | +| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | +| [global\_ip\_address\_range](#input\_global\_ip\_address\_range) | IP address range (CIDR) that will span entire set of VPC networks | `string` | `"172.16.0.0/12"` | no | +| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT) | `number` | `2` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_count](#input\_network\_count) | The number of vpc nettworks to create | `number` | `4` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_interface\_defaults](#input\_network\_interface\_defaults) | The template of the network settings to be used on all vpcs. | 
object({
    network            = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string, "GVNIC")
    stack_type         = optional(string, "IPV4_ONLY")
    queue_count        = optional(string)
    access_config = optional(list(object({
      nat_ip                 = string
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  })
| 
{
  "access_config": [],
  "alias_ip_range": [],
  "ipv6_access_config": [],
  "network": null,
  "network_ip": "",
  "nic_type": "GVNIC",
  "queue_count": null,
  "stack_type": "IPV4_ONLY",
  "subnetwork": null,
  "subnetwork_project": null
}
| no | +| [network\_name\_prefix](#input\_network\_name\_prefix) | The base name of the vpcs and their subnets, will be appended with a sequence number | `string` | `""` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network dynamic routing mode | `string` | `"REGIONAL"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [subnetwork\_cidr\_suffix](#input\_subnetwork\_cidr\_suffix) | The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network. | `number` | `16` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [additional\_networks](#output\_additional\_networks) | Network interfaces for each subnetwork created by this module | +| [network\_ids](#output\_network\_ids) | IDs of the new VPC network | +| [network\_names](#output\_network\_names) | Names of the new VPC networks | +| [network\_self\_links](#output\_network\_self\_links) | Self link of the new VPC network | +| [subnetwork\_addresses](#output\_subnetwork\_addresses) | IP address range of the primary subnetwork | +| [subnetwork\_names](#output\_subnetwork\_names) | Names of the subnetwork created in each network | +| [subnetwork\_self\_links](#output\_subnetwork\_self\_links) | Self link of the primary subnetwork | + diff --git a/deletion-test/build_script/modules/embedded/modules/network/multivpc/main.tf b/deletion-test/build_script/modules/embedded/modules/network/multivpc/main.tf new file mode 100644 index 0000000000..ad06e793c1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/multivpc/main.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # this input variable is validated to be in CIDR format + network_name = coalesce(replace(var.network_name_prefix, "_", "-"), replace(var.deployment_name, "_", "-")) + global_ip_cidr_prefix = split("/", var.global_ip_address_range)[0] + global_ip_cidr_suffix = split("/", var.global_ip_address_range)[1] + global_ip_cidr_valid = "${local.global_ip_cidr_prefix}/${terraform_data.global_ip_cidr_suffix.output}" + subnetwork_new_bits = var.subnetwork_cidr_suffix - local.global_ip_cidr_suffix + maximum_subnetworks = pow(2, local.subnetwork_new_bits) + additional_networks = [ + for vpc in module.vpcs : + merge(var.network_interface_defaults, { + network = vpc.network_name + subnetwork = vpc.subnetwork_name + subnetwork_project = var.project_id + }) + ] +} + +resource "terraform_data" "global_ip_cidr_suffix" { + input = local.global_ip_cidr_suffix + lifecycle { + precondition { + condition = local.maximum_subnetworks >= var.network_count + error_message = < 1 + error_message = "The minimum VPCs able to be created by this module is 2. Use the standard Toolkit module at modules/network/vpc for count = 1" + } + validation { + condition = var.network_count <= 8 + error_message = "The maximum VPCs able to be created by this module is 8" + } +} + +variable "global_ip_address_range" { + description = "IP address range (CIDR) that will span entire set of VPC networks" + type = string + default = "172.16.0.0/12" + + validation { + condition = can(cidrhost(var.global_ip_address_range, 0)) + error_message = "var.global_ip_address_range must be an IPv4 CIDR range (e.g. \"172.16.0.0/12\")." + } +} + +variable "subnetwork_cidr_suffix" { + description = "The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network." + type = number + default = 16 +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "network_routing_mode" { + type = string + default = "REGIONAL" + description = "The network dynamic routing mode" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "ips_per_nat" { + type = number + description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT)" + default = 2 +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_iap_ssh_ingress" { + type = bool + description = "Enable a firewall rule to allow SSH access using IAP tunnels" + default = true +} + +variable "enable_iap_rdp_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" + default = false +} + +variable "enable_iap_winrm_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" + default = false +} + +variable "enable_internal_traffic" { + type = bool + description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" + default = true +} + +variable "extra_iap_ports" { + type = list(string) + description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" + default = [] +} + +variable "allowed_ssh_ip_ranges" { + type = list(string) + description = "A list of CIDR IP ranges from which to allow ssh access" + default = [] + + validation { + condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) + error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." + } +} + +variable "firewall_rules" { + type = any + description = "List of firewall rules" + default = [] +} + +variable "network_interface_defaults" { + type = object({ + network = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + network_ip = optional(string, "") + nic_type = optional(string, "GVNIC") + stack_type = optional(string, "IPV4_ONLY") + queue_count = optional(string) + access_config = optional(list(object({ + nat_ip = string + network_tier = string + public_ptr_domain_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + public_ptr_domain_name = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + }) + description = "The template of the network settings to be used on all vpcs." + default = { + network = null + subnetwork = null + subnetwork_project = null + network_ip = "" + nic_type = "GVNIC" + stack_type = "IPV4_ONLY" + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } +} + +variable "network_profile" { + type = string + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown + EOT + default = null +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/multivpc/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/multivpc/versions.tf new file mode 100644 index 0000000000..e75a67f7b6 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/multivpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.4.0" +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/README.md b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/README.md new file mode 100644 index 0000000000..4d63b17091 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/README.md @@ -0,0 +1,94 @@ +## Description + +This module discovers a subnetwork that already exists in Google Cloud and +outputs subnetwork attributes that uniquely identify it for use by other modules. + +For example, the blueprint below discovers the referred to subnetwork. +With the `use` keyword, the [vm-instance] module accepts the `subnetwork_self_link` +input variables that uniquely identify the subnetwork in which the VM will be created. + +[vpc]: ../vpc/README.md +[vm-instance]: ../../compute/vm-instance/README.md + +> **_NOTE:_** Additional IAM work is needed for this to work correctly. + +### Example + +```yaml +- id: network + source: modules/network/pre-existing-subnetwork + settings: + subnetwork_self_link: https://www.googleapis.com/compute/v1/projects/name-of-host-project/regions/REGION/subnetworks/SUBNETNAME + +- id: example_vm + source: modules/compute/vm-instance + use: + - network + settings: + name_prefix: example + machine_type: c2-standard-4 +``` + +As described in documentation: +[https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork] + +If subnetwork_self_link is provided then name,region,project is ignored. + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [project](#input\_project) | Name of the project that owns the subnetwork | `string` | `null` | no | +| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | `null` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self-link of the subnet in the VPC | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | +| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | + diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/main.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/main.tf new file mode 100644 index 0000000000..9fb206f969 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/main.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +data "google_compute_subnetwork" "primary_subnetwork" { + name = var.subnetwork_name + region = var.region + project = var.project + self_link = var.subnetwork_self_link + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The subnetwork: ${coalesce(var.subnetwork_name, var.subnetwork_self_link)} could not be found." + } + } +} + +# Module-level check for Private Google Access on the subnetwork +check "private_google_access_enabled_subnetwork" { + assert { + condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml new file mode 100644 index 0000000000..6a6f1e5757 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com +ghpc: + has_to_be_used: true diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf new file mode 100644 index 0000000000..868708dc6b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "subnetwork" { + description = "Full subnetwork object in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork +} + +output "subnetwork_name" { + description = "Name of the subnetwork in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.name +} + +output "subnetwork_self_link" { + description = "Subnetwork self-link in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.self_link +} + +output "subnetwork_address" { + description = "Subnetwork IP range in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf new file mode 100644 index 0000000000..d5191843e8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "subnetwork_self_link" { + description = "Self-link of the subnet in the VPC" + type = string + default = null +} + +variable "project" { + description = "Name of the project that owns the subnetwork" + type = string + default = null +} + +variable "subnetwork_name" { + description = "Name of the pre-existing VPC subnetwork" + type = string + default = null +} + +variable "region" { + description = "Region in which to search for primary subnetwork" + type = string + default = null +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf new file mode 100644 index 0000000000..917d948433 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.74.0" + } + + required_version = ">= 1.5" +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/README.md b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/README.md new file mode 100644 index 0000000000..38a1840c2d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/README.md @@ -0,0 +1,110 @@ +## Description + +This module discovers a VPC network that already exists in Google Cloud and +outputs network attributes that uniquely identify it for use by other modules. +The module outputs are aligned with the [vpc module][vpc] so that it can be used +as a drop-in substitute when a VPC already exists. + +For example, the blueprint below discovers the "default" global network and the +"default" regional subnetwork in us-central1. With the `use` keyword, the +[vm-instance] module accepts the `network_self_link` and `subnetwork_self_link` +input variables that uniquely identify the network and subnetwork in which the +VM will be created. + +[vpc]: ../vpc/README.md +[vm-instance]: ../../compute/vm-instance/README.md + +### Example + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + settings: + project_id: $(vars.project_id) + region: us-central1 + +- id: example_vm + source: modules/compute/vm-instance + use: + - network1 + settings: + name_prefix: example + machine_type: c2-standard-4 +``` + +> **_NOTE:_** The `project_id` and `region` settings would be inferred from the +> deployment variables of the same name, but they are included here for clarity. + +### Use shared-vpc + +If a network is created in different project, this module can be used to +reference the network. To use a network from a different project first make sure +you have a [cloud nat][cloudnat] and [IAP][iap] forwarding. For more details, +refer [shared-vpc][shared-vpc-doc] + +[cloudnat]: https://cloud.google.com/nat/docs/overview +[iap]: https://cloud.google.com/iap/docs/using-tcp-forwarding +[shared-vpc-doc]: ../../../examples/README.md#hpc-slurm-sharedvpcyaml-community-badge-experimental-badge + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_network.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network) | data source | +| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [network\_name](#input\_network\_name) | Name of the existing VPC network | `string` | `"default"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | n/a | yes | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork; defaults to var.network\_name if set to null. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [network\_id](#output\_network\_id) | ID of the existing VPC network | +| [network\_name](#output\_network\_name) | Name of the existing VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the existing VPC network | +| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | +| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | + diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/main.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/main.tf new file mode 100644 index 0000000000..ed332bab72 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/main.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +data "google_compute_network" "vpc" { + name = var.network_name + project = var.project_id + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The network: ${var.network_name} could not be found in project: ${var.project_id}." + } + } +} + +locals { + subnetwork_name = var.subnetwork_name != null ? var.subnetwork_name : var.network_name +} + +data "google_compute_subnetwork" "primary_subnetwork" { + name = local.subnetwork_name + region = var.region + project = var.project_id + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The subnetwork: ${local.subnetwork_name} could not be found in project: ${var.project_id} and region: ${var.region}." + } + } +} + +# Module-level check for Private Google Access on the subnetwork +check "private_google_access_enabled_subnetwork" { + assert { + condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/outputs.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/outputs.tf new file mode 100644 index 0000000000..00861af5ca --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the existing VPC network" + value = data.google_compute_network.vpc.name +} + +output "network_id" { + description = "ID of the existing VPC network" + value = data.google_compute_network.vpc.id +} + +output "network_self_link" { + description = "Self link of the existing VPC network" + value = data.google_compute_network.vpc.self_link +} + +output "subnetwork" { + description = "Full subnetwork object in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork +} + +output "subnetwork_name" { + description = "Name of the subnetwork in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.name +} + +output "subnetwork_self_link" { + description = "Subnetwork self-link in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.self_link +} + +output "subnetwork_address" { + description = "Subnetwork IP range in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/variables.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/variables.tf new file mode 100644 index 0000000000..291a81604a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "network_name" { + description = "Name of the existing VPC network" + type = string + default = "default" +} + +variable "subnetwork_name" { + description = "Name of the pre-existing VPC subnetwork; defaults to var.network_name if set to null." + type = string + default = null +} + +variable "region" { + description = "Region in which to search for primary subnetwork" + type = string +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/versions.tf new file mode 100644 index 0000000000..81fe5aeff3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.74.0" + } + + required_version = ">= 1.5" +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/README.md b/deletion-test/build_script/modules/embedded/modules/network/vpc/README.md new file mode 100644 index 0000000000..2c2b1aa1a3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/vpc/README.md @@ -0,0 +1,237 @@ +## Description + +This module creates a new [VPC network][vpc] with 1 or more subnetworks and +a [Cloud Router][router] for every region with a subnetwork. By default, it will +create: + +* A [Cloud NAT][nat] to enable outbound access to the public internet for VMs + without public IP addresses; VMs with public IP addresses bypass the NAT to + directly access the public internet +* A firewall rule that enables inbound SSH access from [Identity-Aware + Proxy][iap] +* A firewall rule that enables all traffic internal to the network + +This behavior is optional and can be configured as [described below](#inputs). +This module is based on networking support in the [Cloud Foundation +Toolkit][cft]. We recommend following the [documentation for the network +module][cft-network] and [submodules][cft-network-submodules] for more details. +In particular, the detailed structure of input variables can be found for: + +* [var.firewall\_rules](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/firewall-rules#inputs) +* [var.secondary\_ranges](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets#inputs) + +[vpc]: https://cloud.google.com/vpc +[router]: https://github.com/terraform-google-modules/terraform-google-cloud-router +[nat]: https://github.com/terraform-google-modules/terraform-google-cloud-nat +[iap]: https://cloud.google.com/iap +[cft]: https://cloud.google.com/foundation-toolkit +[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0 +[cft-network-submodules]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules + +Additionally, [Google Private Access][gpa] is enabled by default on all +subnetworks unless it is explicitly disabled. This setting ensures that all VMs +can use Google services such as [Cloud Storage][gcs] even if they do not have +public IP addresses or Cloud NAT is disabled. + +[gpa]: https://cloud.google.com/vpc/docs/private-google-access +[gcs]: https://cloud.google.com/storage + +### Example + +This creates a new VPC network named `cluster-net`. + +```yaml + - id: network1 + source: modules/network/vpc + settings: + network_name: cluster-net +``` + +### Deprecation warning + +The variables listed below have been deprecated and will be removed in a future +release. Until they are removed,You may continue to use them in Toolkit +blueprints with the same functionality as documented in the [Toolkit 1.0 +release][vpc1.0]. + +* Deprecated variables + * `var.primary_subnetwork` + * `var.additional_subnetworks` + * `var.subnetwork_size` + +[vpc1.0]: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/v1.0.0/modules/network/vpc/README.md + +The following variables have been added to support explicit IP ranges for +subnetworks while retaining existing functionality. We advise adopting them even +if not using explicit IP ranges . The Toolkit ***does not support*** mixing +deprecated variables with the new replacements. The new functionality is +described in [more detail below](#subnetworks). + +* New variables to adopt + * `var.subnetworks` + * A value for this can be generated by merging `var.primary_subnetwork` and + `var.additional_subnetworks` into a single list + * `var.default_primary_subnetwork_size` + * This variable has been renamed for clarity; its value can be directly + copied from an explicit setting for `var.subnetwork_size`; if your blueprint + does not have an explicit setting, the default values are the same + +### Subnetworks + +This module will always provision at least 1 "primary" subnetwork in which most +resources are expected to be provisioned. This primary subnetwork is determined +by + +1. The first element of [var.subnetworks](#input_subnetworks) if it is not the + empty list +2. A default subnetwork automatically calculated from + * [var.subnetwork_name](#input_subnetwork_name) + * [var.region](#input_region) + * [var.network_address_range](#input_network_address_range) + * [var.default_primary_subnetwork_size](#input_default_primary_subnetwork_size) + +If `var.subnetworks` is provided then the primary subnetwork name is taken +explicitly from it and `var.subnetwork_name` is ignored. + +`var.subnetworks` behaves identically to the [Cloud Foundation Toolkit subnets +module][cftsubnets] with the lone exception that one can provide ***one*** of +the following settings for each subnetwork: + +* `new_bits` +* `subnet_ip` + +If each subnetwork defines `subnet_ip` then these are taken to be their explicit +CIDR IP ranges. If each subnetwork defines `new_bits`, then these are taken to +be the size of the CIDR subnetwork (in bits). IP ranges for each subnetwork are +calculated using `var.network_address_range` as the base IP, producing the most +compact set of subnetworks possible. + +> **_NOTE:_** we do not presently support the modification of individual subnetworks +> when using this module to provision more than 1 subnetwork using automatically +> calculated IP ranges based upon `new_bits`. Doing so will cause IP ranges to be +> recalculated for each subnetwork. We advise appending new subnetworks to the end +> of `var.subnetworks`. + +[cftsubnets]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets + +### SSH Access + +By default a firewall rule is created to allow inbound SSH access from +[Identity-Aware Proxy][iap]. A user must have the `IAP-Secured Tunnel User` +(`roles/iap.tunnelResourceAccessor`) IAM role to be able to SSH over IAP. + +To allow regular SSH access from a known IP address you can add the following +`firewall_rules` setting to the `vpc` module: + +```yaml + - id: network1 + source: modules/network/vpc + settings: + firewall_rules: + - name: ssh-my-machine + direction: INGRESS + ranges: [/32] + allow: + - protocol: tcp + ports: [22] +``` + +> **Note**: You must populate the above example with the source IP address from +> which you plan to SSH from. You can use a service like +> [whatismyip.com](https://whatismyip.com) to determine your IP address. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +| Name | Version | +|------|---------| +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 7.3 | +| [nat\_ip\_addresses](#module\_nat\_ip\_addresses) | terraform-google-modules/address/google | ~> 4.1 | +| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.cloud_nat_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.network_profile_firewall_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.secondary_ranges_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_subnetworks](#input\_additional\_subnetworks) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `list(map(string))` | `null` | no | +| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | +| [default\_primary\_subnetwork\_size](#input\_default\_primary\_subnetwork\_size) | The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks | `number` | `15` | no | +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_cloud\_nat](#input\_enable\_cloud\_nat) | Enable the creation of Cloud NATs. | `bool` | `true` | no | +| [enable\_cloud\_router](#input\_enable\_cloud\_router) | Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview | `bool` | `true` | no | +| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | +| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | +| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | +| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | +| [firewall\_log\_config](#input\_firewall\_log\_config) | Firewall log configuration for Toolkit firewall rules (var.enable\_iap\_ssh\_ingress and others) | `string` | `"DISABLE_LOGGING"` | no | +| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | +| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples | `number` | `2` | no | +| [labels](#input\_labels) | Labels to add to network resources that support labels. Key-value pairs of strings. | `map(string)` | `{}` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_address\_range](#input\_network\_address\_range) | IP address range (CIDR) for global network | `string` | `"10.0.0.0/9"` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "GLOBAL") | `string` | `"GLOBAL"` | no | +| [primary\_subnetwork](#input\_primary\_subnetwork) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `map(string)` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [secondary\_ranges](#input\_secondary\_ranges) | "Secondary ranges associated with the subnets.
This will be deprecated in favour of secondary\_ranges\_list at a later date.
Please migrate to using the same." | `map(list(object({ range_name = string, ip_cidr_range = string })))` | `{}` | no | +| [secondary\_ranges\_list](#input\_secondary\_ranges\_list) | "List of secondary ranges associated with the subnetworks.
Each subnetwork must be specified at most once in this list." | 
list(object({
    subnetwork_name = string,
    ranges = list(object({
      range_name    = string,
      ip_cidr_range = string
    }))
  }))
| `[]` | no | +| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-primary-subnet") | `string` | `null` | no | +| [subnetwork\_size](#input\_subnetwork\_size) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `number` | `null` | no | +| [subnetworks](#input\_subnetworks) | List of subnetworks to create within the VPC. If left empty, it will be
replaced by a single, default subnetwork constructed from other parameters
(e.g. var.region). In all cases, the first subnetwork in the list is identified
by outputs as a "primary" subnetwork.

subnet\_name (string, required, name of subnet)
subnet\_region (string, required, region of subnet)
subnet\_ip (string, mutually exclusive with new\_bits, CIDR-formatted IP range for subnetwork)
new\_bits (number, mutually exclusive with subnet\_ip, CIDR bits used to calculate subnetwork range)
subnet\_private\_access (bool, optional, Enable Private Access on subnetwork)
subnet\_flow\_logs (map(string), optional, Configure Flow Logs see terraform-google-network module)
description (string, optional, Description of Network)
purpose (string, optional, related to Load Balancing)
role (string, optional, related to Load Balancing) | `list(map(string))` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nat\_ips](#output\_nat\_ips) | External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used) | +| [network\_id](#output\_network\_id) | ID of the new VPC network | +| [network\_name](#output\_network\_name) | Name of the new VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | +| [subnetwork](#output\_subnetwork) | Primary subnetwork object | +| [subnetwork\_address](#output\_subnetwork\_address) | IP address range of the primary subnetwork | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the primary subnetwork | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Self link of the primary subnetwork | +| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | + diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/main.tf b/deletion-test/build_script/modules/embedded/modules/network/vpc/main.tf new file mode 100644 index 0000000000..24c8eb22bd --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/vpc/main.tf @@ -0,0 +1,256 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +resource "terraform_data" "secondary_ranges_validation" { + lifecycle { + precondition { + condition = !(length(var.secondary_ranges) > 0 && length(var.secondary_ranges_list) > 0) + error_message = "Only one of var.secondary_ranges or var.secondary_ranges_list should be specified" + } + } +} + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "vpc", ghpc_role = "network" }) +} + +locals { + autoname = replace(var.deployment_name, "_", "-") + network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name + subnetwork_name = var.subnetwork_name == null ? "${local.autoname}-primary-subnet" : var.subnetwork_name + + # define a default subnetwork for cases in which no explicit subnetworks are + # defined in var.subnetworks + default_primary_subnetwork_cidr_block = cidrsubnet(var.network_address_range, var.default_primary_subnetwork_size, 0) + default_primary_subnetwork = { + subnet_name = local.subnetwork_name + subnet_ip = local.default_primary_subnetwork_cidr_block + subnet_region = var.region + subnet_private_access = true + subnet_flow_logs = false + description = "primary subnetwork in ${local.network_name}" + purpose = null + role = null + } + + # Identify user-supplied primary subnetwork + # (1) explicit var.subnetworks[0] + # (2) implicit local default subnetwork + input_primary_subnetwork = coalesce(try(var.subnetworks[0], null), local.default_primary_subnetwork) + + # Identify user-supplied additional subnetworks + # (1) explicit var.subnetworks[1:end] + # (2) empty list + input_additional_subnetworks = try(slice(var.subnetworks, 1, length(var.subnetworks)), []) + + # at this point we have constructed a list of subnetworks but need to extract + # user-provided CIDR blocks or calculate them from user-provided new_bits + # after we complete deprecation, local.all_subnetworks can be replaced with + # var.subnetworks (or local.default_primary_subnetwork if that is null) + input_subnetworks = concat([local.input_primary_subnetwork], local.input_additional_subnetworks) + subnetworks_cidr_blocks = try( + local.input_subnetworks[*]["subnet_ip"], + cidrsubnets(var.network_address_range, local.input_subnetworks[*]["new_bits"]...) + ) + + # merge in the CIDR blocks (even when already there) and remove new_bits + subnetworks = [for i, subnet in local.input_subnetworks : + merge({ for k, v in subnet : k => v if k != "new_bits" }, { "subnet_ip" = local.subnetworks_cidr_blocks[i] }) + ] + + # gather the unique regions for purposes of creating Router/NAT + cloud_router_regions = var.enable_cloud_router ? distinct([for subnet in local.subnetworks : subnet.subnet_region]) : [] + cloud_nat_regions = var.enable_cloud_nat ? local.cloud_router_regions : [] + + # this comprehension should have 1 and only 1 match + output_primary_subnetwork = one([for k, v in module.vpc.subnets : v if k == "${local.subnetworks[0].subnet_region}/${local.subnetworks[0].subnet_name}"]) + output_primary_subnetwork_name = local.output_primary_subnetwork.name + output_primary_subnetwork_self_link = local.output_primary_subnetwork.self_link + output_primary_subnetwork_ip_cidr_range = local.output_primary_subnetwork.ip_cidr_range + + iap_ports = distinct(concat(compact([ + var.enable_iap_rdp_ingress ? "3389" : "", + var.enable_iap_ssh_ingress ? "22" : "", + var.enable_iap_winrm_ingress ? "5986" : "", + ]), var.extra_iap_ports)) + + firewall_log_api_values = { + "DISABLE_LOGGING" = null + "INCLUDE_ALL_METADATA" = { metadata = "INCLUDE_ALL_METADATA" }, + "EXCLUDE_ALL_METADATA" = { metadata = "EXCLUDE_ALL_METADATA" }, + } + firewall_log_config = lookup(local.firewall_log_api_values, var.firewall_log_config, null) + + allow_iap_ingress = { + name = "${local.network_name}-fw-allow-iap-ingress" + description = "allow TCP access via Identity-Aware Proxy" + direction = "INGRESS" + priority = null + ranges = ["35.235.240.0/20"] + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = local.iap_ports + }] + deny = [] + log_config = local.firewall_log_config + } + + allow_ssh_ingress = { + name = "${local.network_name}-fw-allow-ssh-ingress" + description = "allow SSH access" + direction = "INGRESS" + priority = null + ranges = var.allowed_ssh_ip_ranges + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = ["22"] + }] + deny = [] + log_config = local.firewall_log_config + } + + allow_internal_traffic = { + name = "${local.network_name}-fw-allow-internal-traffic" + priority = null + description = "allow traffic between nodes of this VPC" + direction = "INGRESS" + ranges = [var.network_address_range] + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = ["0-65535"] + }, { + protocol = "udp" + ports = ["0-65535"] + }, { + protocol = "icmp" + ports = null + }, + ] + deny = [] + log_config = local.firewall_log_config + } + + firewall_rules = concat( + var.firewall_rules, + length(var.allowed_ssh_ip_ranges) > 0 ? [local.allow_ssh_ingress] : [], + var.enable_internal_traffic ? [local.allow_internal_traffic] : [], + length(local.iap_ports) > 0 ? [local.allow_iap_ingress] : [] + ) + + secondary_ranges_map = { + for secondary_range in var.secondary_ranges_list : + secondary_range.subnetwork_name => secondary_range.ranges + } +} + +resource "terraform_data" "network_profile_firewall_validation" { + lifecycle { + precondition { + condition = !(try(strcontains(var.network_profile, "roce"), false) && length(local.firewall_rules) > 0) + error_message = "If var.network_profile contains 'roce', var.firewall_rules must be empty and var.enable_internal_traffic must be false, please see: https://cloud.google.com/vpc/docs/rdma-network-profiles#additional_features_that_dont_apply_to_traffic_from_rdma_nics" + } + } +} + +module "vpc" { + source = "terraform-google-modules/network/google" + version = "~> 12.0" + + depends_on = [terraform_data.network_profile_firewall_validation] + + network_name = local.network_name + project_id = var.project_id + auto_create_subnetworks = false + subnets = local.subnetworks + secondary_ranges = length(local.secondary_ranges_map) > 0 ? local.secondary_ranges_map : var.secondary_ranges + routing_mode = var.network_routing_mode + mtu = var.mtu + description = var.network_description + shared_vpc_host = var.shared_vpc_host + delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes + firewall_rules = local.firewall_rules + network_profile = var.network_profile +} + +resource "terraform_data" "cloud_nat_validation" { + lifecycle { + precondition { + condition = var.enable_cloud_router == true || var.enable_cloud_nat == false + error_message = <<-EOD + "Cannot have Cloud NAT without a Cloud Router. If you desire Cloud NAT functionality please set `enable_cloud_router` to true." + EOD + } + } +} + +# This use of the module may appear odd when var.ips_per_nat = 0. The module +# will be called for all regions with subnetworks but names will be set to the +# empty list. This is a perfectly valid value (the default!). In this scenario, +# no IP addresses are created and all module outputs are empty lists. +# +# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/variables.tf#L27 +# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/outputs.tf +module "nat_ip_addresses" { + source = "terraform-google-modules/address/google" + version = "~> 4.1" + + depends_on = [terraform_data.cloud_nat_validation] + + for_each = toset(local.cloud_nat_regions) + + project_id = var.project_id + region = each.value + # an external, regional (not global) IP address is suited for a regional NAT + address_type = "EXTERNAL" + global = false + labels = local.labels + names = [for idx in range(var.ips_per_nat) : "${local.network_name}-nat-ips-${each.value}-${idx}"] +} + +module "cloud_router" { + source = "terraform-google-modules/cloud-router/google" + version = "~> 7.3" + + depends_on = [terraform_data.cloud_nat_validation] + + for_each = toset(local.cloud_router_regions) + + project = var.project_id + name = "${local.network_name}-router" + region = each.value + network = module.vpc.network_name + # in scenario with no NAT IPs, no NAT is created even if router is created + # https://github.com/terraform-google-modules/terraform-google-cloud-router/blob/v2.0.0/nat.tf#L18-L20 + nats = length(module.nat_ip_addresses[each.value].self_links) == 0 ? [] : [ + { + name : "cloud-nat-${each.value}", + nat_ips : module.nat_ip_addresses[each.value].self_links + }, + ] +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/network/vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/outputs.tf b/deletion-test/build_script/modules/embedded/modules/network/vpc/outputs.tf new file mode 100644 index 0000000000..c2ee6bdf6b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/vpc/outputs.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the new VPC network" + value = module.vpc.network_name + depends_on = [module.vpc, module.cloud_router] +} + +output "network_id" { + description = "ID of the new VPC network" + value = module.vpc.network_id + depends_on = [module.vpc, module.cloud_router] +} + +output "network_self_link" { + description = "Self link of the new VPC network" + value = module.vpc.network_self_link + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetworks" { + description = "Full list of subnetwork objects belonging to the new VPC network" + value = module.vpc.subnets + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork" { + description = "Primary subnetwork object" + value = local.output_primary_subnetwork + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_name" { + description = "Name of the primary subnetwork" + value = local.output_primary_subnetwork_name + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_self_link" { + description = "Self link of the primary subnetwork" + value = local.output_primary_subnetwork_self_link + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_address" { + description = "IP address range of the primary subnetwork" + value = local.output_primary_subnetwork_ip_cidr_range + depends_on = [module.vpc, module.cloud_router] +} + +output "nat_ips" { + description = "External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used)" + value = flatten([for ipmod in module.nat_ip_addresses : ipmod.addresses]) +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/variables.tf b/deletion-test/build_script/modules/embedded/modules/network/vpc/variables.tf new file mode 100644 index 0000000000..e036189404 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/vpc/variables.tf @@ -0,0 +1,301 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "labels" { + description = "Labels to add to network resources that support labels. Key-value pairs of strings." + type = map(string) + default = {} + nullable = false +} + +variable "network_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" + type = string + default = null +} + +variable "subnetwork_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-primary-subnet\")" + type = string + default = null +} + +# tflint-ignore: terraform_unused_declarations +variable "subnetwork_size" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = number + default = null + validation { + condition = var.subnetwork_size == null + error_message = "subnetwork_size is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +variable "default_primary_subnetwork_size" { + description = "The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks" + type = number + default = 15 +} + +variable "region" { + description = "The default region for Cloud resources" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "network_address_range" { + description = "IP address range (CIDR) for global network" + type = string + default = "10.0.0.0/9" + + validation { + condition = can(cidrhost(var.network_address_range, 0)) + error_message = "IP address range must be in CIDR format." + } +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "subnetworks" { + description = <<-EOT + List of subnetworks to create within the VPC. If left empty, it will be + replaced by a single, default subnetwork constructed from other parameters + (e.g. var.region). In all cases, the first subnetwork in the list is identified + by outputs as a "primary" subnetwork. + + subnet_name (string, required, name of subnet) + subnet_region (string, required, region of subnet) + subnet_ip (string, mutually exclusive with new_bits, CIDR-formatted IP range for subnetwork) + new_bits (number, mutually exclusive with subnet_ip, CIDR bits used to calculate subnetwork range) + subnet_private_access (bool, optional, Enable Private Access on subnetwork) + subnet_flow_logs (map(string), optional, Configure Flow Logs see terraform-google-network module) + description (string, optional, Description of Network) + purpose (string, optional, related to Load Balancing) + role (string, optional, related to Load Balancing) + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_name"]) + ]) + error_message = "All subnetworks must define \"subnet_name\"." + } + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_region"]) + ]) + error_message = "All subnetworks must define \"subnet_region\"." + } + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_ip"]) != can(s["new_bits"]) + ]) + error_message = "All subnetworks must define exactly one of \"subnet_ip\" or \"new_bits\"." + } + validation { + condition = alltrue([for s in var.subnetworks : can(s["subnet_ip"])]) || alltrue([for s in var.subnetworks : can(s["new_bits"])]) + error_message = "All subnetworks must make same choice of \"subnet_ip\" or \"new_bits\"." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "primary_subnetwork" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = map(string) + default = null + validation { + condition = var.primary_subnetwork == null + error_message = "primary_subnetwork is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "additional_subnetworks" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = list(map(string)) + default = null + validation { + condition = var.additional_subnetworks == null + error_message = "additional_subnetworks is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +variable "secondary_ranges" { + type = map(list(object({ range_name = string, ip_cidr_range = string }))) + description = <<-EOT + "Secondary ranges associated with the subnets. + This will be deprecated in favour of secondary_ranges_list at a later date. + Please migrate to using the same." + EOT + default = {} +} + +variable "secondary_ranges_list" { + type = list(object({ + subnetwork_name = string, + ranges = list(object({ + range_name = string, + ip_cidr_range = string + })) + })) + description = <<-EOT + "List of secondary ranges associated with the subnetworks. + Each subnetwork must be specified at most once in this list." + EOT + default = [] + validation { + condition = (length(var.secondary_ranges_list[*].subnetwork_name) == + length(distinct(var.secondary_ranges_list[*].subnetwork_name))) + error_message = "Each subnetwork should be specified at most once in this list. Remove any duplicates." + } +} + +variable "network_routing_mode" { + type = string + default = "GLOBAL" + description = "The network routing mode (default \"GLOBAL\")" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "ips_per_nat" { + type = number + description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples" + default = 2 +} + +variable "shared_vpc_host" { + type = bool + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + default = false +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_iap_ssh_ingress" { + type = bool + description = "Enable a firewall rule to allow SSH access using IAP tunnels" + default = true +} + +variable "enable_iap_rdp_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" + default = false +} + +variable "enable_iap_winrm_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" + default = false +} + +variable "enable_internal_traffic" { + type = bool + description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" + default = true +} + +variable "enable_cloud_router" { + type = bool + description = "Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview" + default = true +} + +variable "enable_cloud_nat" { + type = bool + description = "Enable the creation of Cloud NATs." + default = true +} + +variable "extra_iap_ports" { + type = list(string) + description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" + default = [] +} + +variable "allowed_ssh_ip_ranges" { + type = list(string) + description = "A list of CIDR IP ranges from which to allow ssh access" + default = [] + + validation { + condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) + error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." + } +} + +variable "firewall_rules" { + type = any + description = "List of firewall rules" + default = [] +} + +variable "firewall_log_config" { + type = string + description = "Firewall log configuration for Toolkit firewall rules (var.enable_iap_ssh_ingress and others)" + default = "DISABLE_LOGGING" + nullable = false + + validation { + condition = contains([ + "INCLUDE_ALL_METADATA", + "EXCLUDE_ALL_METADATA", + "DISABLE_LOGGING", + ], var.firewall_log_config) + error_message = "var.firewall_log_config must be set to \"DISABLE_LOGGING\", or enable logging with \"INCLUDE_ALL_METADATA\" or \"EXCLUDE_ALL_METADATA\"" + } +} + +variable "network_profile" { + type = string + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown + EOT + default = null +} diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/vpc/versions.tf new file mode 100644 index 0000000000..71b7106734 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/network/vpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.15.0" +} diff --git a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/README.md b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/README.md new file mode 100644 index 0000000000..192d7575a5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/README.md @@ -0,0 +1,320 @@ +# Custom Images in the Cluster Toolkit (formerly HPC Toolkit) + +Please review the +[introduction to image building](../../../docs/image-building.md) for general +information on building custom images using the Toolkit. + +## Introduction + +This module uses [Packer](https://www.packer.io/) to create an image within an +Cluster Toolkit deployment. Packer operates by provisioning a short-lived VM in +Google Cloud on which it executes scripts to customize the boot disk for +repeated use. The VM's boot disk is specified from a source image that defaults +to the [HPC VM Image][hpcimage]. This Packer "template" supports customization +by the following approaches following a [recommended use](#recommended-use): + +- [startup-script metadata][startup-metadata] from [raw string][sss] or + [file][ssf] +- [Shell scripts][shell] uploaded from the Packer execution environment to the + VM +- [Ansible playbooks][ansible] uploaded from the Packer execution environment to + the VM + +They can be specified independently of one another, so that anywhere from 1 to 3 +solutions can be used simultaneously. In the case that 0 scripts are supplied, +the source boot disk is effectively copied to your project without +customization. This can be useful in scenarios where increased control over the +image maintenance lifecycle is desired or when policies restrict the use of +images to internal projects. + +## Minimum requirements + +### Outbound internet access + +Most customization scripts require access to resources on the public internet. +This can be achieved by one of the following 2 approaches: + +1. Using a public IP address on the VM + +- Set [var.omit_external_ip](#input_omit_external_ip) to `false` + +1. Configuring a VPC with a Cloud NAT in the region of the VM + +- Use the [vpc] module which automates NAT creation + +### Inbound internet access + +Read [order of execution](#order-of-execution) below for a discussion of VM +customization solutions and their requirements for inbound SSH access. +[Environments without SSH access](#environments-without-ssh-access) should use +the metadata-based startup-script solution. + +A simple way to enable inbound SSH access is to use the VPC module with +`allowed_ssh_ip_ranges` set to `0.0.0.0/0`. + +### User or service account executing Packer at command line + +The user or service account running Packer must have the permission to create +VMs in the selected VPC network and, if [use\_iap](#input_use_iap) is set, must +have the "IAP-Secured Tunnel User" role. Recommended roles are: + +- `roles/compute.instanceAdmin.v1` +- `roles/iap.tunnelResourceAccessor` + +### VM service account roles + +The service account attached to the temporary build VM created by Packer should +have the ability to write Cloud Logging entries so that you may inspect and +debug build logs. When using the metadata startup-script customization solution, +the service account attached to the temporary build VM created by Packer must +have the permission to modify its own metadata and to read from Cloud Storage +buckets. Recommended roles are: + +- `roles/compute.instanceAdmin.v1` +- `roles/iam.serviceAccountUser` +- `roles/logging.logWriter` +- `roles/monitoring.metricWriter` +- `roles/storage.objectViewer` + +It is recommended to create this service account as a separate step outside a +blueprint due to known delay in [IAM bindings propagation][iamprop]. + +## Example blueprints + +A recommended pattern for building images with this module is to use the +terraform based [startup-script] module along with this packer custom-image +module. Below you can find links to several examples of this pattern, including +usage instructions. + +### [Image Builder] + +The [Image Builder] blueprint demonstrates a solution that builds an image +using: + +- The [HPC VM Image][hpcimage] as a base upon which to customize +- A VPC network with firewall rules that allow IAP-based SSH tunnels +- A Toolkit runner that installs a custom script + +Please review the [examples README] for usage instructions. + +## Order of execution + +The startup script specified in metadata executes in parallel with the other +supported methods. However, the remaining methods execute in a well-defined +order relative to one another. + +1. All shell scripts will execute in the configured order +1. After shell scripts complete, all Ansible playbooks will execute in the + configured order + +> **_NOTE:_** if both [startup_script][sss] and [startup_script_file][ssf] are +> specified, then [startup_script_file][ssf] takes precedence. + +## Recommended use + +Because the [metadata startup script executes in parallel](#order-of-execution) +with the other solutions, conflicts can arise, especially when package managers +(`yum` or `apt`) lock their databases during package installation. Therefore, it +is recommended to choose one of the following approaches: + +1. Specify _either_ [startup_script][sss] _or_ [startup_script_file][ssf] and do + not specify [shell_scripts][shell] or [ansible_playbooks][ansible]. + - This can be especially useful in + [environments that restrict SSH access](#environments-without-ssh-access) +1. Specify any combination of [shell_scripts][shell] and + [ansible_playbooks][ansible] and do not specify [startup_script][sss] or + [startup_script_file][ssf]. + +If any of the startup script approaches fail by returning a code other than 0, +Packer will determine that the build has failed and refuse to save the image. + +## External access with SSH + +The [shell scripts][shell] and [Ansible playbooks][ansible] customization +solutions both require SSH access to the VM from the Packer execution +environment. SSH access can be enabled one of 2 ways: + +1. The VM is created without a public IP address and SSH tunnels are created + using [Identity-Aware Proxy (IAP)][iaptunnel]. + - Allow [use_iap](#input_use_iap) to take on its default value of `true` +1. The VM is created with an IP address on the public internet and firewall + rules allow SSH access from the Packer execution environment. + - Set `omit_external_ip = false` (or `omit_external_ip: false` in a + blueprint) + - Add firewall rules that open SSH to the VM + +The Packer template defaults to using to the 1st IAP-based solution because it +is more secure (no exposure to public internet) and because the [vpc] module +automatically sets up all necessary firewall rules for SSH tunneling and +outbound-only access to the internet through [Cloud NAT][cloudnat]. + +In either SSH solution, customization scripts should be supplied as files in the +[shell_scripts][shell] and [ansible_playbooks][ansible] settings. + +## Environments without SSH access + +Many network environments disallow SSH access to VMs. In these environments, the +[metadata-based startup scripts][startup-metadata] are appropriate because they +execute entirely independently of the Packer execution environment. + +In this scenario, a single scripts should be supplied in the form of a string to +the [startup_script][sss] input variable. This solution integrates well with +Toolkit runners. Runners operate by using a single startup script whose behavior +is extended by downloading and executing a customizable set of runners from +Cloud Storage at startup. + +> **_NOTE:_** Packer will attempt to use SSH if either [shell_scripts][shell] or +> [ansible_playbooks][ansible] are set to non-empty values. Leave them at their +> default, empty values to ensure access by SSH is disabled. + +## Supplying startup script as a string + +The [startup_script][sss] parameter accepts scripts formatted as strings. In +Packer and Terraform, multi-line strings can be specified using +[heredoc syntax](https://www.terraform.io/language/expressions/strings#heredoc-strings) +in an input [Packer variables file][pkrvars] (`*.pkrvars.hcl`) For example, the +following snippet defines a multi-line bash script followed by an integer +representing the size, in GiB, of the resulting image: + +```hcl +startup_script = <<-EOT + #!/bin/bash + yum install -y epel-release + yum install -y jq + EOT + +disk_size = 100 +``` + +In a blueprint, the equivalent syntax is: + +```yaml +... + settings: + startup_script: | + #!/bin/bash + yum install -y epel-release + yum install -y jq + disk_size: 100 +... +``` + +## Monitoring startup script execution + +When using startup script customization, Packer will print very limited output +to the console. For example: + +```text +==> example.googlecompute.toolkit_image: Waiting for any running startup script to finish... +==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... +==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... +==> example.googlecompute.toolkit_image: Startup script, if any, has finished running. +``` + +### Debugging startup-script failures + +> [!NOTE] +> There can be a delay in the propagation of the logs from the instance to +> Cloud Logging, so it may require waiting a few minutes to see the full logs. + +If the Packer image build fails, the module will output a `gcloud` command +that can be used directly to review startup-script execution. + +## License + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + +```text + http://www.apache.org/licenses/LICENSE-2.0 +``` + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_count](#input\_accelerator\_count) | Number of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `number` | `null` | no | +| [accelerator\_type](#input\_accelerator\_type) | Type of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `string` | `null` | no | +| [ansible\_playbooks](#input\_ansible\_playbooks) | A list of Ansible playbook configurations that will be uploaded to customize the VM image | 
list(object({
    playbook_file   = string
    galaxy_file     = string
    extra_arguments = list(string)
  }))
| `[]` | no | +| [communicator](#input\_communicator) | Communicator to use for provisioners that require access to VM ("ssh" or "winrm") | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name | `string` | n/a | yes | +| [disk\_size](#input\_disk\_size) | Size of disk image in GB | `number` | `null` | no | +| [disk\_type](#input\_disk\_type) | Type of persistent disk to provision | `string` | `"pd-balanced"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [image\_family](#input\_image\_family) | The family name of the image to be built. Defaults to `deployment_name` | `string` | `null` | no | +| [image\_name](#input\_image\_name) | The name of the image to be built. If not supplied, it will be set to image\_family-$ISO\_TIMESTAMP | `string` | `null` | no | +| [image\_storage\_locations](#input\_image\_storage\_locations) | Storage location, either regional or multi-regional, where snapshot content is to be stored and only accepts 1 value.
See https://developer.hashicorp.com/packer/plugins/builders/googlecompute#image_storage_locations | `list(string)` | `null` | no | +| [labels](#input\_labels) | Labels to apply to the short-lived VM | `map(string)` | `null` | no | +| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2-standard-4"` | no | +| [manifest\_file](#input\_manifest\_file) | File to which to write Packer build manifest | `string` | `"packer-manifest.json"` | no | +| [metadata](#input\_metadata) | Instance metadata for the builder VM (use var.startup\_script or var.startup\_script\_file to set startup-script metadata) | `map(string)` | `{}` | no | +| [network\_project\_id](#input\_network\_project\_id) | Project ID of Shared VPC network | `string` | `null` | no | +| [omit\_external\_ip](#input\_omit\_external\_ip) | Provision the image building VM without a public IP address | `bool` | `true` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which to create VM and image | `string` | n/a | yes | +| [scopes](#input\_scopes) | DEPRECATED: use var.service\_account\_scopes | `set(string)` | `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | The service account email to use. If null or 'default', then the default Compute Engine service account will be used. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Service account scopes to attach to the instance. See
https://cloud.google.com/compute/docs/access/service-accounts. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shell\_scripts](#input\_shell\_scripts) | A list of paths to local shell scripts which will be uploaded to customize the VM image | `list(string)` | `[]` | no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [source\_image](#input\_source\_image) | Source OS image to build from | `string` | `null` | no | +| [source\_image\_family](#input\_source\_image\_family) | Alternative to source\_image. Specify image family to build from latest image in family | `string` | `"hpc-rocky-linux-8"` | no | +| [source\_image\_project\_id](#input\_source\_image\_project\_id) | A list of project IDs to search for the source image. Packer will search the
first project ID in the list first, and fall back to the next in the list,
until it finds the source image. | `list(string)` | `null` | no | +| [ssh\_username](#input\_ssh\_username) | Username to use for SSH access to VM | `string` | `"hpc-toolkit-packer"` | no | +| [startup\_script](#input\_startup\_script) | Startup script (as raw string) used to build the custom Linux VM image (overridden by var.startup\_script\_file if both are set) | `string` | `null` | no | +| [startup\_script\_file](#input\_startup\_script\_file) | File path to local shell script that will be used to customize the Linux VM image (overrides var.startup\_script) | `string` | `null` | no | +| [state\_timeout](#input\_state\_timeout) | The time to wait for instance state changes, including image creation | `string` | `"10m"` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of subnetwork in which to provision image building VM | `string` | n/a | yes | +| [tags](#input\_tags) | Assign network tags to apply firewall rules to VM instance | `list(string)` | `null` | no | +| [use\_iap](#input\_use\_iap) | Use IAP proxy when connecting by SSH | `bool` | `true` | no | +| [use\_os\_login](#input\_use\_os\_login) | Use OS Login when connecting by SSH | `bool` | `false` | no | +| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | A list of strings containing PowerShell scripts which will customize a Windows VM image (requires WinRM communicator) | `list(string)` | `[]` | no | +| [wrap\_startup\_script](#input\_wrap\_startup\_script) | Wrap startup script with Packer-generated wrapper | `bool` | `true` | no | +| [zone](#input\_zone) | Cloud zone in which to provision image building VM | `string` | n/a | yes | + +## Outputs + +No outputs. + + +[ansible]: #input_ansible_playbooks +[cloudnat]: https://cloud.google.com/nat/docs/overview +[examples readme]: ../../../examples/README.md#image-builderyaml- +[hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[iamprop]: https://cloud.google.com/iam/docs/access-change-propagation +[iaptunnel]: https://cloud.google.com/iap/docs/using-tcp-forwarding +[image builder]: ../../../examples/image-builder.yaml +[logging-console]: https://console.cloud.google.com/logs/ +[logging-read-docs]: https://cloud.google.com/sdk/gcloud/reference/logging/read +[pkrvars]: https://www.packer.io/guides/hcl/variables#from-a-file +[shell]: #input_shell_scripts +[ssf]: #input_startup_script_file +[sss]: #input_startup_script +[startup-metadata]: https://cloud.google.com/compute/docs/instances/startup-scripts/linux +[startup-script]: ../../../modules/scripts/startup-script +[vpc]: ../../network/vpc/README.md diff --git a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/image.pkr.hcl b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/image.pkr.hcl new file mode 100644 index 0000000000..9282cf7433 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/image.pkr.hcl @@ -0,0 +1,216 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "custom-image", ghpc_role = "packer" }) + + # construct a unique image name from the image family + image_family = var.image_family != null ? var.image_family : var.deployment_name + image_name_default = "${local.image_family}-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" + image_name = var.image_name != null ? var.image_name : local.image_name_default + + # construct vm image name for use when getting logs + instance_name = "packer-${substr(uuidv4(), 0, 6)}" + + # default to explicit var.communicator, otherwise in-order: ssh/winrm/none + shell_script_communicator = length(var.shell_scripts) > 0 ? "ssh" : "" + ansible_playbook_communicator = length(var.ansible_playbooks) > 0 ? "ssh" : "" + powershell_script_communicator = length(var.windows_startup_ps1) > 0 ? "winrm" : "" + communicator = coalesce( + var.communicator, + local.shell_script_communicator, + local.ansible_playbook_communicator, + local.powershell_script_communicator, + "none" + ) + + # must not enable IAP when no communicator is in use + use_iap = local.communicator == "none" ? false : var.use_iap + + # construct metadata from startup_script and metadata variables + startup_script_metadata = var.startup_script == null ? {} : { startup-script = var.startup_script } + + linux_user_metadata = { + block-project-ssh-keys = "TRUE" + shutdown-script = <<-EOT + #!/bin/bash + userdel -r ${var.ssh_username} + sed -i '/${var.ssh_username}/d' /var/lib/google/google_users + EOT + } + windows_packer_user = "packer_user" + windows_user_metadata = { + sysprep-specialize-script-cmd = "winrm quickconfig -quiet & net user /add ${local.windows_packer_user} & net localgroup administrators ${local.windows_packer_user} /add & winrm set winrm/config/service/auth @{Basic=\\\"true\\\"}" + windows-shutdown-script-cmd = <<-EOT + net user /delete ${local.windows_packer_user} + EOT + } + user_metadata = local.communicator == "winrm" ? local.windows_user_metadata : local.linux_user_metadata + + # merge metadata such that var.metadata always overrides user management + # metadata but always allow var.startup_script to override var.metadata + metadata = merge( + local.user_metadata, + var.metadata, + local.startup_script_metadata, + ) + + # determine best value for on_host_maintenance if not supplied by user + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + gpu_attached = contains(["a2", "g2"], local.machine_family) || var.accelerator_type != null + on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" + on_host_maintenance = ( + var.on_host_maintenance != null + ? var.on_host_maintenance + : local.on_host_maintenance_default + ) + + accelerator_type = var.accelerator_type == null ? null : "projects/${var.project_id}/zones/${var.zone}/acceleratorTypes/${var.accelerator_type}" + + winrm_username = local.communicator == "winrm" ? "packer_user" : null + winrm_insecure = local.communicator == "winrm" ? true : null + winrm_use_ssl = local.communicator == "winrm" ? true : null + + enable_integrity_monitoring = var.enable_shielded_vm && var.shielded_instance_config.enable_integrity_monitoring + enable_secure_boot = var.enable_shielded_vm && var.shielded_instance_config.enable_secure_boot + enable_vtpm = var.enable_shielded_vm && var.shielded_instance_config.enable_vtpm + + image_licenses = [ + "projects/click-to-deploy-images/global/licenses/hpc-toolkit-vm-image" + ] +} + +source "googlecompute" "toolkit_image" { + communicator = local.communicator + project_id = var.project_id + image_name = local.image_name + image_family = local.image_family + image_labels = local.labels + instance_name = local.instance_name + machine_type = var.machine_type + accelerator_type = local.accelerator_type + accelerator_count = var.accelerator_count + on_host_maintenance = local.on_host_maintenance + disk_size = var.disk_size + disk_type = var.disk_type + omit_external_ip = var.omit_external_ip + use_internal_ip = var.omit_external_ip + subnetwork = var.subnetwork_name + network_project_id = var.network_project_id + service_account_email = var.service_account_email + scopes = var.service_account_scopes + source_image = var.source_image + source_image_family = var.source_image_family + source_image_project_id = var.source_image_project_id + ssh_username = var.ssh_username + tags = var.tags + use_iap = local.use_iap + use_os_login = var.use_os_login + winrm_username = local.winrm_username + winrm_insecure = local.winrm_insecure + winrm_use_ssl = local.winrm_use_ssl + zone = var.zone + labels = local.labels + metadata = local.metadata + startup_script_file = var.startup_script_file + wrap_startup_script = var.wrap_startup_script + state_timeout = var.state_timeout + image_storage_locations = var.image_storage_locations + enable_secure_boot = local.enable_secure_boot + enable_vtpm = local.enable_vtpm + enable_integrity_monitoring = local.enable_integrity_monitoring + image_licenses = local.image_licenses +} + +build { + name = var.deployment_name + sources = ["sources.googlecompute.toolkit_image"] + + # using dynamic blocks to create provisioners ensures that there are no + # provisioner blocks when none are provided and we can use the none + # communicator when using startup-script + + # provisioner "shell" blocks + dynamic "provisioner" { + labels = ["shell"] + for_each = var.shell_scripts + content { + execute_command = "sudo -H sh -c '{{ .Vars }} {{ .Path }}'" + script = provisioner.value + } + } + + # provisioner "powershell" blocks + dynamic "provisioner" { + labels = ["powershell"] + for_each = var.windows_startup_ps1 + content { + inline = split("\n", provisioner.value) + } + } + + dynamic "provisioner" { + labels = ["powershell"] + for_each = length(var.windows_startup_ps1) > 0 ? [1] : [] + content { + inline = [ + "GCESysprep -no_shutdown" + ] + } + } + + # provisioner "ansible-local" blocks + # this installs custom roles/collections from ansible-galaxy in /home/packer + # which will be removed at the end; consider modifying /etc/ansible/ansible.cfg + dynamic "provisioner" { + labels = ["ansible-local"] + for_each = var.ansible_playbooks + content { + playbook_file = provisioner.value.playbook_file + galaxy_file = provisioner.value.galaxy_file + extra_arguments = provisioner.value.extra_arguments + } + } + + post-processor "manifest" { + output = var.manifest_file + strip_path = true + custom_data = { + built-by = "cloud-hpc-toolkit" + } + } + + # If there is an error during image creation, print out command for getting packer VM logs + error-cleanup-provisioner "shell-local" { + environment_vars = [ + "PRJ_ID=${var.project_id}", + "INST_NAME=${local.instance_name}", + "ZONE=${var.zone}", + ] + inline_shebang = "/bin/bash -e" + inline = [ + "type -P gcloud > /dev/null || exit 0", + "INST_ID=$(gcloud compute instances describe $INST_NAME --project $PRJ_ID --format=\"value(id)\" --zone=$ZONE)", + "echo 'Error building image try checking logs:'", + join(" ", ["echo \"gcloud logging --project $PRJ_ID read", + "'logName=(\\\"projects/$PRJ_ID/logs/GCEMetadataScripts\\\" OR \\\"projects/$PRJ_ID/logs/google_metadata_script_runner\\\") AND resource.labels.instance_id=$INST_ID'", + "--format=\\\"table(timestamp, resource.labels.instance_id, jsonPayload.message)\\\"", + "--order=asc\"" + ] + ) + ] + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/metadata.yaml new file mode 100644 index 0000000000..23108c4e17 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - logging.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/variables.pkr.hcl b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/variables.pkr.hcl new file mode 100644 index 0000000000..3cede102ce --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/variables.pkr.hcl @@ -0,0 +1,276 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "deployment_name" { + description = "Cluster Toolkit deployment name" + type = string +} + +variable "project_id" { + description = "Project in which to create VM and image" + type = string +} + +variable "machine_type" { + description = "VM machine type on which to build new image" + type = string + default = "n2-standard-4" +} + +variable "disk_size" { + description = "Size of disk image in GB" + type = number + default = null +} + +variable "disk_type" { + description = "Type of persistent disk to provision" + type = string + default = "pd-balanced" +} + +variable "zone" { + description = "Cloud zone in which to provision image building VM" + type = string +} + +variable "network_project_id" { + description = "Project ID of Shared VPC network" + type = string + default = null +} + +variable "subnetwork_name" { + description = "Name of subnetwork in which to provision image building VM" + type = string +} + +variable "omit_external_ip" { + description = "Provision the image building VM without a public IP address" + type = bool + default = true +} + +variable "tags" { + description = "Assign network tags to apply firewall rules to VM instance" + type = list(string) + default = null +} + +variable "image_family" { + description = "The family name of the image to be built. Defaults to `deployment_name`" + type = string + default = null +} + +variable "image_name" { + description = "The name of the image to be built. If not supplied, it will be set to image_family-$ISO_TIMESTAMP" + type = string + default = null +} + +variable "source_image_project_id" { + description = < +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 4.0 | +| [local](#requirement\_local) | >= 2.0.0 | +| [null](#requirement\_null) | ~> 3.0 | +| [random](#requirement\_random) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.0 | +| [local](#provider\_local) | >= 2.0.0 | +| [null](#provider\_null) | ~> 3.0 | +| [random](#provider\_random) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_template](#module\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.submit_script](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [null_resource.submit_job](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.submit_job_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used for the job\_id | `string` | n/a | yes | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs | `bool` | `true` | no | +| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions. Valid inputs are `"alpha"`, `"beta"` and "" (empty string for default version) | `string` | `""` | no | +| [image](#input\_image) | DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided. | `any` | `null` | no | +| [instance\_image](#input\_instance\_image) | Google Cloud Batch compute node image. Ignored if `instance_template` is provided.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [instance\_template](#input\_instance\_template) | Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions. | `string` | `null` | no | +| [job\_filename](#input\_job\_filename) | The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified | `string` | `null` | no | +| [job\_id](#input\_job\_id) | An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs. | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided. | `map(string)` | n/a | yes | +| [log\_policy](#input\_log\_policy) | Create a block to define log policy.
When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging.
When set to `PATH`, path must be added to generated template.
When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. | `string` | `"CLOUD_LOGGING"` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided. | `string` | `"n2-standard-4"` | no | +| [mpi\_mode](#input\_mpi\_mode) | Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`. | `bool` | `false` | no | +| [native\_batch\_mounting](#input\_native\_batch\_mounting) | Batch can mount some fs\_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts. | `bool` | `true` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which to run the Google Cloud Batch job | `string` | n/a | yes | +| [runnable](#input\_runnable) | A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`. | `string` | `null` | no | +| [runnables](#input\_runnables) | A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template. | 
list(object({
    script = string
  }))
| `null` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/devstorage.read_only",
    "https://www.googleapis.com/auth/logging.write",
    "https://www.googleapis.com/auth/monitoring.write",
    "https://www.googleapis.com/auth/servicecontrol",
    "https://www.googleapis.com/auth/service.management.readonly",
    "https://www.googleapis.com/auth/trace.append"
  ]
}
| no | +| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided. | `string` | `null` | no | +| [submit](#input\_submit) | When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply. | `bool` | `false` | no | +| [subnetwork](#input\_subnetwork) | The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided. | `any` | `null` | no | +| [task\_count](#input\_task\_count) | Number of parallel tasks | `number` | `1` | no | +| [task\_count\_per\_node](#input\_task\_count\_per\_node) | Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value. | `number` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcloud\_version](#output\_gcloud\_version) | The version of gcloud to be used. | +| [instance\_template](#output\_instance\_template) | Instance template used by the Batch job. | +| [instructions](#output\_instructions) | Instructions for submitting the Batch job. | +| [job\_data](#output\_job\_data) | All data associated with the defined job, typically provided as input to clout-batch-login-node. | +| [network\_storage](#output\_network\_storage) | An array of network attached storage mounts used by the Batch job. | +| [startup\_script](#output\_startup\_script) | Startup script run before Google Cloud Batch job starts. | + diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/main.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/main.tf new file mode 100644 index 0000000000..0d681536c9 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/main.tf @@ -0,0 +1,149 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "batch-job-template", ghpc_role = "scheduler" }) +} + +locals { + instance_template = coalesce(var.instance_template, module.instance_template.self_link) + + tasks_per_node = var.task_count_per_node != null ? var.task_count_per_node : (var.mpi_mode ? 1 : null) + + one_line_runnable = coalesce(var.runnable, "## Add your workload here ##") + runnables = coalesce(var.runnables, [{ script = local.one_line_runnable }]) + + job_template_contents = templatefile( + "${path.module}/templates/batch-job-base.yaml.tftpl", + { + synchronized = var.mpi_mode + runnables = local.runnables + task_count = var.task_count + tasks_per_node = local.tasks_per_node + require_hosts_file = var.mpi_mode + permissive_ssh = var.mpi_mode + log_policy = var.log_policy + instance_template = local.instance_template + nfs_volumes = local.native_batch_network_storage + labels = local.labels + } + ) + + submit_job_id = "${var.job_id}-${random_id.submit_job_suffix.hex}" + job_filename = coalesce(var.job_filename, "${var.job_id}.yaml") + job_template_output_path = "${path.root}/${local.job_filename}" + + submit_script_contents = templatefile( + "${path.module}/templates/batch-submit.sh.tftpl", + { + project = var.project_id + location = var.region + config = local_file.job_template.filename + submit_job_id = local.submit_job_id + } + ) + submit_script_output_path = "${path.root}/submit-${var.job_id}.sh" + + subnetwork_name = var.subnetwork != null ? var.subnetwork.name : "default" + subnetwork_project = var.subnetwork != null ? var.subnetwork.project : var.project_id + + # Filter network_storage for native Batch support + native_fstype = var.native_batch_mounting ? ["nfs"] : [] + native_batch_network_storage = [ + for ns in var.network_storage : + ns if contains(local.native_fstype, ns.fs_type) + ] + # other processing happens in startup_from_network_storage.tf + + # this code is similar to code in Packer and vm-instance modules + # it differs in that this module does not (yet) expose var.guest_acclerator + # for attaching GPUs to N1 VMs. For now, identify only A2 types. + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + gpu_attached = contains(["a2", "g2"], local.machine_family) + on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" + + on_host_maintenance = coalesce(var.on_host_maintenance, local.on_host_maintenance_default) + + network_storage_metadata = var.network_storage != null ? ({ network_storage = jsonencode(var.network_storage) }) : {} + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.network_storage_metadata, + local.disable_automatic_updates_metadata + ) +} + +module "instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = var.instance_template == null ? "${var.job_id}-instance-template" : "unused-template" + project_id = var.project_id + subnetwork = local.subnetwork_name + subnetwork_project = local.subnetwork_project + service_account = var.service_account + access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + labels = local.labels + + machine_type = var.machine_type + startup_script = local.startup_from_network_storage + metadata = local.metadata + source_image_family = data.google_compute_image.compute_image.family + source_image = data.google_compute_image.compute_image.name + source_image_project = data.google_compute_image.compute_image.project + on_host_maintenance = local.on_host_maintenance +} + +resource "local_file" "job_template" { + content = local.job_template_contents + filename = local.job_template_output_path + + lifecycle { + precondition { + condition = var.runnable == null || var.runnables == null + error_message = "var.runnable and var.runnables (plural) cannot both be set." + } + } +} + +resource "random_id" "submit_job_suffix" { + byte_length = 4 + keepers = { + always_run = timestamp() + } +} + +resource "local_file" "submit_script" { + content = local.submit_script_contents + filename = local.submit_script_output_path +} + +resource "null_resource" "submit_job" { + depends_on = [local_file.job_template, local_file.submit_script] + count = var.submit ? 1 : 0 + + # A new deployment should always submit a new job. Old finished jobs aren't persistent parts of + # Cloud infrastructure. + triggers = { + always_run = timestamp() + } + + provisioner "local-exec" { + command = local.submit_script_output_path + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml new file mode 100644 index 0000000000..387e810962 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - batch.googleapis.com + - compute.googleapis.com +ghpc: + inject_module_id: job_id diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/outputs.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/outputs.tf new file mode 100644 index 0000000000..0b1295975a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/outputs.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + provided_instance_tpl_msg = "The Batch job template uses the existing VM instance template:" + generated_instance_tpl_msg = "The Batch job template uses a new VM instance template created matching the provided settings:" + submit_msg = <<-EOT + + The job has been submitted. See job status at: + https://console.cloud.google.com/batch/jobsDetail/regions/${var.region}/jobs/${local.submit_job_id}?project=${var.project_id} + EOT +} + +output "instructions" { + description = "Instructions for submitting the Batch job." + value = <<-EOT + + A Batch job template file has been created locally at: + ${abspath(local.job_template_output_path)} + + ${var.instance_template == null ? local.generated_instance_tpl_msg : local.provided_instance_tpl_msg} + ${local.instance_template} + ${var.submit ? local.submit_msg : ""} + + Use the following commands to: + Submit your job${var.submit ? " (Note: job has already been submitted)" : ""}: + gcloud ${var.gcloud_version} batch jobs submit ${local.submit_job_id} --config=${abspath(local.job_template_output_path)} --location=${var.region} --project=${var.project_id} + + Check status: + gcloud ${var.gcloud_version} batch jobs describe ${local.submit_job_id} --location=${var.region} --project=${var.project_id} | grep state: + + Delete job: + gcloud ${var.gcloud_version} batch jobs delete ${local.submit_job_id} --location=${var.region} --project=${var.project_id} + + List all jobs: + gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} + EOT +} + +output "job_data" { + description = "All data associated with the defined job, typically provided as input to clout-batch-login-node." + value = { + template_contents = local.job_template_contents, + filename = local.job_filename, + id = local.submit_job_id + } +} + +output "instance_template" { + description = "Instance template used by the Batch job." + value = local.instance_template +} + +output "network_storage" { + description = "An array of network attached storage mounts used by the Batch job." + value = var.network_storage +} + +output "startup_script" { + description = "Startup script run before Google Cloud Batch job starts." + value = var.startup_script +} + +output "gcloud_version" { + description = "The version of gcloud to be used." + value = var.gcloud_version +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf new file mode 100644 index 0000000000..02bc58e4f7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "inputs": +# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. +# var.network_storage : to be passed into metadata somewhere else (not here) +# var.startup_script : to be changed into a more complete file system with all the fs runners + +# "outputs": +# local.startup_from_network_storage : A full startup script with all the runners that are not supported +# natively and were included in the network_storage structure + +locals { + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + # Pull out runners to include in startup script + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + startup_script_runner = [{ + content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" + destination = "passed_startup_script.sh" + type = "shell" + }] + + full_runner_list = concat( + local.storage_client_install_runners, + local.mount_runners, + local.startup_script_runner + ) + + startup_from_network_storage = module.netstorage_startup_script.startup_script +} + +module "netstorage_startup_script" { + source = "../../scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.full_runner_list +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl new file mode 100644 index 0000000000..83fccde53b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl @@ -0,0 +1,53 @@ +taskGroups: + - taskSpec: + runnables: + %{~ if synchronized ~} + - barrier: + name: "wait-for-node-startup" + %{~ endif ~} + %{~ for runnable in runnables ~} + - script: + text: ${indent(12, chomp(yamlencode(runnable.script)))} + %{~ if synchronized ~} + - barrier: + name: "wait-for-script-to-complete" + %{~ endif ~} + %{~ endfor ~} + %{~ if length(nfs_volumes) > 0 ~} + volumes: + %{~ for index, vol in nfs_volumes ~} + - nfs: + server: "${vol.server_ip}" + remotePath: "${vol.remote_mount}" + %{~ if vol.mount_options != "" && vol.mount_options != null ~} + mountOptions: "${vol.mount_options}" + %{~ endif ~} + mountPath: "${vol.local_mount}" + %{~ endfor ~} + %{~ endif ~} + taskCount: ${task_count} + %{~ if tasks_per_node != null ~} + taskCountPerNode: ${tasks_per_node} + %{~ endif ~} + requireHostsFile: ${require_hosts_file} + permissiveSsh: ${permissive_ssh} +%{~ if instance_template != null } +allocationPolicy: + instances: + - instanceTemplate: "${instance_template}" +%{~ endif } +%{~ if log_policy == "CLOUD_LOGGING" } +logsPolicy: + destination: "CLOUD_LOGGING" +%{ endif } +%{~ if log_policy == "PATH" } +logsPolicy: + destination: "PATH" + logsPath: ## Add logging path here +%{ endif } +%{~ if length(labels) > 0 ~} +labels: +%{ for k, v in labels ~} + ${k}: "${v}" +%{ endfor } +%{~ endif ~} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl new file mode 100644 index 0000000000..25f89c3ceb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl @@ -0,0 +1,10 @@ +#!/bin/bash +set -e -o pipefail +GCLOUD_MAJOR_VERSION=$(gcloud --version | head -n 1 | awk '{print $NF}' | cut -f1 --delimiter=.) +if [ $((GCLOUD_MAJOR_VERSION >= 461)) ]; then + gcloud batch jobs submit ${submit_job_id} --project=${project} --location=${location} --config=${config} + echo "batch job ${submit_job_id} successfully submitted" +else + echo "gcloud must be updated to version 461.0.0 or later." + exit 1 +fi diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/variables.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/variables.tf new file mode 100644 index 0000000000..f65fbd111e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/variables.tf @@ -0,0 +1,240 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "region" { + description = "The region in which to run the Google Cloud Batch job" + type = string +} + +variable "deployment_name" { + description = "Name of the deployment, used for the job_id" + type = string +} + +variable "labels" { + description = "Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided." + type = map(string) +} + +variable "job_id" { + description = "An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs." + type = string +} + +variable "job_filename" { + description = "The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified" + type = string + default = null +} + +variable "gcloud_version" { + description = "The version of the gcloud cli being used. Used for output instructions. Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default version)" + type = string + default = "" + + validation { + condition = contains(["alpha", "beta", ""], var.gcloud_version) + error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." + } +} + +variable "task_count" { + description = "Number of parallel tasks" + type = number + default = 1 +} + +variable "task_count_per_node" { + description = "Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value." + type = number + default = null +} + +variable "mpi_mode" { + description = "Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`." + type = bool + default = false +} + +variable "log_policy" { + description = <<-EOT + Create a block to define log policy. + When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging. + When set to `PATH`, path must be added to generated template. + When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. + EOT + type = string + default = "CLOUD_LOGGING" + + validation { + condition = contains(["CLOUD_LOGGING", "PATH", "DESTINATION_UNSPECIFIED"], var.log_policy) + error_message = "Allowed values for log_policy are 'CLOUD_LOGGING', 'PATH', or 'DESTINATION_UNSPECIFIED'." + } +} + +variable "runnables" { + description = "A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template." + type = list(object({ + script = string + })) + default = null +} + +variable "runnable" { + description = "A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`." + type = string + default = null +} + +variable "instance_template" { + description = "Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions." + type = string + default = null +} + +variable "subnetwork" { + description = "The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided." + type = any + default = null +} + +variable "enable_public_ips" { + description = "If set to true, instances will have public IPs" + type = bool + default = true +} + +variable "service_account" { + description = "Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided." + type = object({ + email = string, + scopes = set(string) + }) + default = { + email = null + scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/trace.append" + ] + } +} + +variable "machine_type" { + description = "Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided." + type = string + default = "n2-standard-4" +} + +variable "startup_script" { + description = "Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided." + type = string + default = null +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided." + type = list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "native_batch_mounting" { + description = "Batch can mount some fs_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts." + type = bool + default = true +} + +# Deprecated, replaced by instance_image +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided." + type = any + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + Google Cloud Batch compute node image. Ignored if `instance_template` is provided. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE`" + type = string + default = null + validation { + condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) + error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." + } +} + +variable "submit" { + description = "When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply." + type = bool + default = false +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/versions.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/versions.tf new file mode 100644 index 0000000000..a1161e1354 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + google = { + source = "hashicorp/google" + version = ">= 4.0" + } + } + required_version = ">= 1.1" +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/README.md b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/README.md new file mode 100644 index 0000000000..c20ca7dbeb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/README.md @@ -0,0 +1,127 @@ +# Description + +This module creates a VM that acts as a login node to test and submit Google +Cloud Batch jobs. It is intended to be used along with the `batch-job-template` +module. + +This login node: + +- Uses the same VM settings as the first provided `batch-job-template`, such as + image, machine type, etc... +- Runs the same `startup-script` as the first provided `batch-job-template`. +- Has the same mounted file systems as the provided `batch-job-template`. +- Contains a folder with job templates generated by `batch-job-template` modules. + +Since the login node has the same mounted storage and is a homogeneous machine +to the Google Cloud Batch compute VMs, it can be used to inspect shared file +systems and test installed software before submitting a Google Cloud Batch job. + +## Example + +```yaml +- id: batch-job + source: modules/scheduler/batch-job-template + ... + +- id: batch-login + source: modules/scheduler/batch-login-node + use: [batch-job] + outputs: [instructions] +``` + +## Authentication + +To submit jobs from the login node, the service account attached to the VM needs +the `Batch Job Administrator` role. In most cases this service account will be +the Compute Engine default service account and will not be granted this role by +default. + +You can grant this role either by adding the `Batch Job Administrator` role to +the service account in the IAM page in the Google Cloud Console, or by running +the following command line: + +```bash +gcloud projects add-iam-policy-binding \ + --member=serviceAccount: \ + --role=roles/batch.jobsAdmin +``` + +## gcloud Batch Access + +Until the Google Cloud Batch API is generally available (GA), it may not be +available in all versions of the `gcloud` cli. You can test if the Google Cloud +Batch commands are available by running `gcloud [alpha|beta|] batch -h`. If the +Google Cloud Batch cli is not available it can generally be mitigated by either +updating `gcloud` by running `gcloud components update`, or using an image that +contains a more recent version of `gcloud`. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [login\_startup\_script](#module\_login\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_from_template.batch_login](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | +| [google_compute_instance_template.batch_instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [batch\_job\_directory](#input\_batch\_job\_directory) | The path of the directory on the login node in which to place the Google Cloud Batch job template | `string` | `"/home/batch-jobs"` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, also used for the job\_id | `string` | n/a | yes | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions.
Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default
version). Typically supplied by a batch-job-template module. If multiple
batch-job-template modules supply the gcloud\_version, only the first will be used. | `string` | `""` | no | +| [instance\_template](#input\_instance\_template) | Login VM instance template self-link. Typically supplied by a
batch-job-template module. If multiple batch-job-template modules supply the
instance\_template, the first will be used. | `string` | n/a | yes | +| [job\_data](#input\_job\_data) | List of jobs and supporting data for each, typically provided via "use" from the batch-job-template module. | 
list(object({
    template_contents = string,
    filename          = string,
    id                = string
  }))
| n/a | yes | +| [job\_filename](#input\_job\_filename) | Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [job\_id](#input\_job\_id) | Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions. | `string` | `null` | no | +| [job\_template\_contents](#input\_job\_template\_contents) | Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the login node. Key-value pairs | `map(string)` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which to create the login node | `string` | n/a | yes | +| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [zone](#input\_zone) | The zone in which to create the login node | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for accessing the login node and submitting Google Cloud Batch jobs | +| [login\_node\_name](#output\_login\_node\_name) | Name of the created VM | + diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/main.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/main.tf new file mode 100644 index 0000000000..6f539af122 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/main.tf @@ -0,0 +1,127 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "batch-login-node", ghpc_role = "scheduler" }) +} + +data "google_compute_instance_template" "batch_instance_template" { + name = var.instance_template +} + +locals { + job_template_runners = [for job in var.job_data : { + content = job.template_contents + destination = "${var.batch_job_directory}/${job.filename}" + type = "data" + }] + + instance_template_metadata = data.google_compute_instance_template.batch_instance_template.metadata + startup_metadata = { startup-script = module.login_startup_script.startup_script } + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + login_metadata = merge(local.instance_template_metadata, local.startup_metadata, local.oslogin_metadata) + + batch_command_instructions = join("\n", [for job in var.job_data : <<-EOT + ## For job: ${job.id} ## + + Submit your job from login node: + gcloud ${var.gcloud_version} batch jobs submit ${job.id} --config=${var.batch_job_directory}/${job.filename} --location=${var.region} --project=${var.project_id} + + Check status: + gcloud ${var.gcloud_version} batch jobs describe ${job.id} --location=${var.region} --project=${var.project_id} | grep state: + + Delete job: + gcloud ${var.gcloud_version} batch jobs delete ${job.id} --location=${var.region} --project=${var.project_id} + + EOT + ]) + + list_all_jobs = <<-EOT + List all jobs: + gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} + EOT + + readme_contents = <<-EOT + # Batch Job Templates + + This folder contains Batch job templates created by the Cluster Toolkit. + These templates can be edited before submitting to Batch to capture more + complex workloads. + + Use the following commands to: + ${local.list_all_jobs} + + ${local.batch_command_instructions} + EOT + + # Construct startup script for network storage + storage_client_install_runners = [ + for i, ns in var.network_storage : merge(ns.client_install_runner, { + destination = "${i}-${ns.client_install_runner.destination}" + }) if ns.client_install_runner != null + ] + mount_runners = [ + for i, ns in var.network_storage : merge(ns.mount_runner, { + destination = "${i}-${ns.mount_runner.destination}" + }) if ns.mount_runner != null + ] + + startup_script_runner = { + content = var.startup_script != null ? var.startup_script : "echo 'Batch job template had no startup script'" + destination = "passed_startup_script.sh" + type = "shell" + } +} + +module "login_startup_script" { + source = "../../scripts/startup-script" + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = concat( + local.storage_client_install_runners, + local.mount_runners, + [local.startup_script_runner], + local.job_template_runners, + [ + { + content = local.readme_contents + destination = "${var.batch_job_directory}/README.md" + type = "data" + } + ] + ) +} + +resource "google_compute_instance_from_template" "batch_login" { + name = "${var.deployment_name}-batch-login" + source_instance_template = var.instance_template + project = var.project_id + zone = var.zone + metadata = local.login_metadata + + service_account { + scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml new file mode 100644 index 0000000000..9af2319b4a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - batch.googleapis.com + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/outputs.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/outputs.tf new file mode 100644 index 0000000000..ea8eccf8d5 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/outputs.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "login_node_name" { + description = "Name of the created VM" + value = google_compute_instance_from_template.batch_login.name +} + +output "instructions" { + description = "Instructions for accessing the login node and submitting Google Cloud Batch jobs" + value = <<-EOT + + Batch job template files will be placed on the Batch login node in the following directory: + ${var.batch_job_directory} + + Use the following commands to: + SSH into the login node: + gcloud compute ssh --zone ${google_compute_instance_from_template.batch_login.zone} ${google_compute_instance_from_template.batch_login.name} --project ${google_compute_instance_from_template.batch_login.project} + + ${local.list_all_jobs} + + ${local.batch_command_instructions} + EOT +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/variables.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/variables.tf new file mode 100644 index 0000000000..3b9caa7001 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/variables.tf @@ -0,0 +1,151 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "deployment_name" { + description = "Name of the deployment, also used for the job_id" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "region" { + description = "The region in which to create the login node" + type = string +} + +variable "zone" { + description = "The zone in which to create the login node" + type = string +} + +variable "labels" { + description = "Labels to add to the login node. Key-value pairs" + type = map(string) +} + +variable "instance_template" { + description = <<-EOT + Login VM instance template self-link. Typically supplied by a + batch-job-template module. If multiple batch-job-template modules supply the + instance_template, the first will be used. + EOT + type = string +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module." + type = list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "startup_script" { + description = "Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module." + type = string + default = null +} + +variable "job_data" { + description = "List of jobs and supporting data for each, typically provided via \"use\" from the batch-job-template module." + type = list(object({ + template_contents = string, + filename = string, + id = string + })) + validation { + condition = length(distinct([for job in var.job_data : job.filename])) == length(var.job_data) + error_message = "All filenames in var.job_data must be unique." + } + validation { + condition = length(distinct([for job in var.job_data : job.id])) == length(var.job_data) + error_message = "All job IDs in var.job_data must be unique." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_template_contents" { + description = "Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module." + type = string + default = null + validation { + condition = var.job_template_contents == null + error_message = "job_template_contents is deprecated. Please use `job_data` instead." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_filename" { + description = "Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module." + type = string + default = null + validation { + condition = var.job_filename == null + error_message = "job_filename is deprecated. Please use `job_data` instead." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_id" { + description = "Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions." + type = string + default = null + validation { + condition = var.job_id == null + error_message = "job_id is deprecated. Please use `job_data` instead." + } +} + +variable "gcloud_version" { + description = <<-EOT + The version of the gcloud cli being used. Used for output instructions. + Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default + version). Typically supplied by a batch-job-template module. If multiple + batch-job-template modules supply the gcloud_version, only the first will be used. + EOT + type = string + default = "" + + validation { + condition = contains(["alpha", "beta", ""], var.gcloud_version) + error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." + } +} + +variable "batch_job_directory" { + description = "The path of the directory on the login node in which to place the Google Cloud Batch job template" + type = string + default = "/home/batch-jobs" +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/versions.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/versions.tf new file mode 100644 index 0000000000..15337a1d7b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/README.md b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/README.md new file mode 100644 index 0000000000..dd4f7fdaa7 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/README.md @@ -0,0 +1,220 @@ +## Description + +This module creates a Google Kubernetes Engine +([GKE](https://cloud.google.com/kubernetes-engine)) cluster. + +### Example + +The following example creates a GKE cluster and a VPC designed to work with GKE. +See [VPC Network](#vpc-network) section for more information about network +requirements. + +```yaml + - id: network1 + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### VPC Network + +This module is configured to create a +[VPC-native cluster](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips). +This means that alias IPs are used and that the subnetwork requires secondary +ranges for pods and services. In the example shown above these secondary ranges +are created in the VPC module. By default the `gke-cluster` module will look for +ranges with the names `pods` and `services`. These names can be configured using +the `pods_ip_range_name` and `services_ip_range_name` settings. + +### Multi-networking + +To [enable Multi-networking](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment), pass multivpc module to gke-cluster module as described in example below. Passing a multivpc module enables multi networking and [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en) on the cluster. + +```yaml + - id: network + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: gke-cluster + source: modules/scheduler/gke-cluster + use: [network, multinetwork] ## enables multi networking and Dataplane V2 on cluster + settings: + cluster_name: $(vars.deployment_name) +``` + +Find an example of multi networking in GKE [here](../../../examples/gke-a3-megagpu.yaml). + +### Cluster Limitations + +The current implementations has the following limitations: + +- Autopilot is disabled +- Auto-provisioning of new node pools is disabled +- Network policies are not supported +- General addon configuration is not supported +- Only regional cluster is supported + +### GKE Inference Gateway + +Setting `enable_inference_gateway` to `true` will enable the `HttpLoadBalancing` +addon and deploy the Inference Gateway CRDs. This feature requires a subnet with +`purpose` set to `REGIONAL_MANAGED_PROXY` in the VPC. For more information, see +the [GKE Inference Gateway documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/serve-with-gke-inference-gateway). + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 7.2 | +| [google-beta](#requirement\_google-beta) | >= 7.2 | +| [kubernetes](#requirement\_kubernetes) | >= 2.36 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 7.2 | +| [google-beta](#provider\_google-beta) | >= 7.2 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | +| [workload\_identity](#module\_workload\_identity) | terraform-google-modules/kubernetes-engine/google//modules/workload-identity | >= 40.0 | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_cluster) | resource | +| [google-beta_google_container_node_pool.system_node_pools](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | +| [google-beta_google_container_engine_versions.version_prefix_filter](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/data-sources/google_container_engine_versions) | data source | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks enables multi networking and creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [authenticator\_security\_group](#input\_authenticator\_security\_group) | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| [autoscaling\_profile](#input\_autoscaling\_profile) | (Beta) Optimize for utilization or availability when deciding to remove nodes. Can be BALANCED or OPTIMIZE\_UTILIZATION. | `string` | `"OPTIMIZE_UTILIZATION"` | no | +| [cloud\_dns\_config](#input\_cloud\_dns\_config) | Configuration for Using Cloud DNS for GKE.

additive\_vpc\_scope\_dns\_domain: This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = "CLOUD\_DNS" and cluster\_dns\_scope = "CLUSTER\_SCOPE" must both be set as well.
cluster\_dns: Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS.
cluster\_dns\_scope: The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE.
cluster\_dns\_domain: The suffix used for all cluster service records. | 
object({
    additive_vpc_scope_dns_domain = optional(string)
    cluster_dns                   = optional(string, "PROVIDER_UNSPECIFIED")
    cluster_dns_scope             = optional(string, "DNS_SCOPE_UNSPECIFIED")
    cluster_dns_domain            = optional(string)
  })
| 
{
  "additive_vpc_scope_dns_domain": null,
  "cluster_dns": "PROVIDER_UNSPECIFIED",
  "cluster_dns_domain": null,
  "cluster_dns_scope": "DNS_SCOPE_UNSPECIFIED"
}
| no | +| [cluster\_availability\_type](#input\_cluster\_availability\_type) | Type of cluster availability. Possible values are: {REGIONAL, ZONAL} | `string` | `"REGIONAL"` | no | +| [cluster\_reference\_type](#input\_cluster\_reference\_type) | How the google\_container\_node\_pool.system\_node\_pools refers to the cluster. Possible values are: {SELF\_LINK, NAME} | `string` | `"SELF_LINK"` | no | +| [configure\_workload\_identity\_sa](#input\_configure\_workload\_identity\_sa) | When true, a kubernetes service account will be created and bound using workload identity to the service account used to create the cluster. | `bool` | `false` | no | +| [default\_max\_pods\_per\_node](#input\_default\_max\_pods\_per\_node) | The default maximum number of pods per node in this cluster. | `number` | `null` | no | +| [deletion\_protection](#input\_deletion\_protection) | "Determines if the cluster can be deleted by gcluster commands or not".
To delete a cluster provisioned with deletion\_protection set to true, you must first set it to false and apply the changes.
Then proceed with deletion as usual. | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`. | `string` | n/a | yes | +| [enable\_dataplane\_v2](#input\_enable\_dataplane\_v2) | Enables [Dataplane v2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2). This setting is immutable on clusters. If null, will default to false unless using multi-networking, in which case it will default to true | `bool` | `null` | no | +| [enable\_dcgm\_monitoring](#input\_enable\_dcgm\_monitoring) | Enable GKE to collect DCGM metrics | `bool` | `false` | no | +| [enable\_external\_dns\_endpoint](#input\_enable\_external\_dns\_endpoint) | Allow [DNS-based approach](https://cloud.google.com/kubernetes-engine/docs/concepts/network-isolation#dns-based_endpoint) for accessing the GKE control plane.
Refer this [dedicated blog](https://cloud.google.com/blog/products/containers-kubernetes/new-dns-based-endpoint-for-the-gke-control-plane) for more details. | `bool` | `false` | no | +| [enable\_filestore\_csi](#input\_enable\_filestore\_csi) | The status of the Filestore Container Storage Interface (CSI) driver addon, which allows the usage of filestore instance as volumes. | `bool` | `false` | no | +| [enable\_gcsfuse\_csi](#input\_enable\_gcsfuse\_csi) | The status of the GCSFuse Container Storage Interface (CSI) driver addon, which allows the usage of a GCS bucket as volumes. | `bool` | `false` | no | +| [enable\_inference\_gateway](#input\_enable\_inference\_gateway) | If true, enables GKE features required for Inference Gateway, including the HttpLoadBalancing addon, and installs required CRDs. | `bool` | `false` | no | +| [enable\_k8s\_beta\_apis](#input\_enable\_k8s\_beta\_apis) | List of Enabled Kubernetes Beta APIs. | `list(string)` | `null` | no | +| [enable\_managed\_lustre\_csi](#input\_enable\_managed\_lustre\_csi) | The status of the Google Compute Engine Managed Lustre Container Storage Interface (CSI) driver addon, which allows the usage of a lustre as volumes. | `bool` | `false` | no | +| [enable\_master\_global\_access](#input\_enable\_master\_global\_access) | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `false` | no | +| [enable\_multi\_networking](#input\_enable\_multi\_networking) | Enables [multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) (Requires GKE Enterprise). This setting is immutable on clusters and enables [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en). If null, will determine state based on if additional\_networks are passed in. | `bool` | `null` | no | +| [enable\_node\_local\_dns\_cache](#input\_enable\_node\_local\_dns\_cache) | Enable GKE NodeLocal DNSCache addon to improve DNS lookup latency | `bool` | `false` | no | +| [enable\_parallelstore\_csi](#input\_enable\_parallelstore\_csi) | The status of the Google Compute Engine Parallelstore Container Storage Interface (CSI) driver addon, which allows the usage of a parallelstore as volumes. | `bool` | `false` | no | +| [enable\_persistent\_disk\_csi](#input\_enable\_persistent\_disk\_csi) | The status of the Google Compute Engine Persistent Disk Container Storage Interface (CSI) driver addon, which allows the usage of a PD as volumes. | `bool` | `true` | no | +| [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | (Beta) Whether the master's internal IP address is used as the cluster endpoint. | `bool` | `true` | no | +| [enable\_private\_ipv6\_google\_access](#input\_enable\_private\_ipv6\_google\_access) | The private IPv6 google access type for the VMs in this subnet. | `bool` | `true` | no | +| [enable\_private\_nodes](#input\_enable\_private\_nodes) | (Beta) Whether nodes have internal IP addresses only. | `bool` | `true` | no | +| [enable\_ray\_operator](#input\_enable\_ray\_operator) | The status of the Ray operator addon, This feature enables Kubernetes APIs for managing and scaling Ray clusters and jobs. You control and are responsible for managing ray.io custom resources in your cluster. This feature is not compatible with GKE clusters that already have another Ray operator installed. Supports clusters on Kubernetes version 1.29.8-gke.1054000 or later. | `bool` | `false` | no | +| [gcp\_public\_cidrs\_access\_enabled](#input\_gcp\_public\_cidrs\_access\_enabled) | Whether the cluster master is accessible via all the Google Compute Engine Public IPs. To view this list of IP addresses look here https://cloud.google.com/compute/docs/faq#find_ip_range | `bool` | `false` | no | +| [k8s\_network\_names](#input\_k8s\_network\_names) | Kubernetes network names details for GKE. If starting index is not specified for gvnic or rdma, it would be set to the default values. | 
object({
    gvnic_prefix      = optional(string, "")
    gvnic_start_index = optional(number, 1)
    gvnic_postfix     = optional(string, "")
    rdma_prefix       = optional(string, "")
    rdma_start_index  = optional(number, 0)
    rdma_postfix      = optional(string, "")
  })
| 
{
  "gvnic_postfix": "",
  "gvnic_prefix": "gvnic-",
  "gvnic_start_index": 1,
  "rdma_postfix": "",
  "rdma_prefix": "rdma-",
  "rdma_start_index": 0
}
| no | +| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account name to use with the gke cluster | `string` | `"workload-identity-k8s-sa"` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [maintenance\_exclusions](#input\_maintenance\_exclusions) | List of maintenance exclusions. A cluster can have up to three. | 
list(object({
    name            = string
    start_time      = string
    end_time        = string
    exclusion_scope = string
  }))
| `[]` | no | +| [maintenance\_start\_time](#input\_maintenance\_start\_time) | Start time for daily maintenance operations. Specified in GMT with `HH:MM` format. | `string` | `"09:00"` | no | +| [master\_authorized\_networks](#input\_master\_authorized\_networks) | External network that can access Kubernetes master through HTTPS. Must be specified in CIDR notation. | 
list(object({
    cidr_block   = string
    display_name = string
  }))
| `[]` | no | +| [master\_ipv4\_cidr\_block](#input\_master\_ipv4\_cidr\_block) | (Beta) The IP range in CIDR notation to use for the hosted master network. | `string` | `"172.16.0.32/28"` | no | +| [min\_master\_version](#input\_min\_master\_version) | The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release. | `string` | `null` | no | +| [name\_suffix](#input\_name\_suffix) | Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`. | `string` | `""` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`. | `string` | n/a | yes | +| [networking\_mode](#input\_networking\_mode) | Determines whether alias IPs or routes will be used for pod IPs in the cluster. Options are VPC\_NATIVE or ROUTES. VPC\_NATIVE enables IP aliasing. The default is VPC\_NATIVE. | `string` | `"VPC_NATIVE"` | no | +| [pods\_ip\_range\_name](#input\_pods\_ip\_range\_name) | The name of the secondary subnet ip range to use for pods. | `string` | `"pods"` | no | +| [prefix\_with\_deployment\_name](#input\_prefix\_with\_deployment\_name) | If true, cluster name will be prefixed by `deployment_name` (ex: -). | `bool` | `true` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [region](#input\_region) | The region to host the cluster in. | `string` | n/a | yes | +| [release\_channel](#input\_release\_channel) | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. | `string` | `"UNSPECIFIED"` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the system node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the system node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [services\_ip\_range\_name](#input\_services\_ip\_range\_name) | The name of the secondary subnet range to use for services. | `string` | `"services"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to host the cluster in. | `string` | n/a | yes | +| [system\_node\_pool\_disk\_size\_gb](#input\_system\_node\_pool\_disk\_size\_gb) | Size of disk for each node of the system node pool. | `number` | `100` | no | +| [system\_node\_pool\_disk\_type](#input\_system\_node\_pool\_disk\_type) | Disk type for each node of the system node pool. | `string` | `null` | no | +| [system\_node\_pool\_enable\_secure\_boot](#input\_system\_node\_pool\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | +| [system\_node\_pool\_enabled](#input\_system\_node\_pool\_enabled) | Create a system node pool. | `bool` | `true` | no | +| [system\_node\_pool\_image\_type](#input\_system\_node\_pool\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | +| [system\_node\_pool\_kubernetes\_labels](#input\_system\_node\_pool\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | +| [system\_node\_pool\_machine\_type](#input\_system\_node\_pool\_machine\_type) | Machine type for the system node pool. | `string` | `"e2-standard-4"` | no | +| [system\_node\_pool\_name](#input\_system\_node\_pool\_name) | Name of the system node pool. | `string` | `"system"` | no | +| [system\_node\_pool\_node\_count](#input\_system\_node\_pool\_node\_count) | The total min and max nodes to be maintained in the system node pool. | 
object({
    total_min_nodes = number
    total_max_nodes = number
  })
| 
{
  "total_max_nodes": 10,
  "total_min_nodes": 2
}
| no | +| [system\_node\_pool\_taints](#input\_system\_node\_pool\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "components.gke.io/gke-managed-components",
    "value": true
  }
]
| no | +| [system\_node\_pool\_zones](#input\_system\_node\_pool\_zones) | The zones to use for the system node pool. If not specified, the cluster default node zone(s) will be used. | `list(string)` | `null` | no | +| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | +| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | +| [upgrade\_settings](#input\_upgrade\_settings) | Defines gke cluster upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | +| [version\_prefix](#input\_version\_prefix) | If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally. | `string` | `"1.31."` | no | +| [zone](#input\_zone) | Zone for a zonal cluster. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | An identifier for the resource with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | +| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | +| [gke\_version](#output\_gke\_version) | GKE cluster's version. | +| [instructions](#output\_instructions) | Instructions on how to connect to the created cluster. | +| [k8s\_service\_account\_name](#output\_k8s\_service\_account\_name) | Name of k8s service account. | + diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/main.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/main.tf new file mode 100644 index 0000000000..6106f8d90f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/main.tf @@ -0,0 +1,470 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-cluster", ghpc_role = "scheduler" }) +} + +locals { + upgrade_settings = { + strategy = var.upgrade_settings.strategy + max_surge = coalesce(var.upgrade_settings.max_surge, 0) + max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) + } +} + +locals { + dash = var.prefix_with_deployment_name && var.name_suffix != "" ? "-" : "" + prefix = var.prefix_with_deployment_name ? var.deployment_name : "" + name_maybe_empty = "${local.prefix}${local.dash}${var.name_suffix}" + name = local.name_maybe_empty != "" ? local.name_maybe_empty : "NO-NAME-GIVEN" + + cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ + security_group = var.authenticator_security_group + }] + + default_sa_email = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" + sa_email = coalesce(var.service_account_email, local.default_sa_email) + + # additional VPCs enable multi networking + derived_enable_multi_networking = coalesce(var.enable_multi_networking, length(var.additional_networks) > 0) + + # multi networking needs enabled Dataplane v2 + derived_enable_dataplane_v2 = coalesce(var.enable_dataplane_v2, local.derived_enable_multi_networking) + + default_monitoring_component = [ + "SYSTEM_COMPONENTS", + "POD", + "DAEMONSET", + "DEPLOYMENT", + "STATEFULSET", + "STORAGE", + "HPA", + "CADVISOR", + "KUBELET" + ] + + default_logging_component = [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] +} + +data "google_project" "project" { + project_id = var.project_id +} + +data "google_container_engine_versions" "version_prefix_filter" { + provider = google-beta + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + version_prefix = var.version_prefix +} + +locals { + master_version = var.min_master_version != null ? var.min_master_version : data.google_container_engine_versions.version_prefix_filter.latest_master_version +} + +resource "google_container_cluster" "gke_cluster" { + provider = google-beta + + project = var.project_id + name = local.name + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + resource_labels = local.labels + networking_mode = var.networking_mode + # decouple node pool lifecycle from cluster life cycle + remove_default_node_pool = true + initial_node_count = 1 # must be set when remove_default_node_pool is set + node_locations = var.system_node_pool_zones + + deletion_protection = var.deletion_protection + + dynamic "enable_k8s_beta_apis" { + for_each = var.enable_k8s_beta_apis != null ? [1] : [] + content { + enabled_apis = var.enable_k8s_beta_apis + } + } + + network = var.network_id + subnetwork = var.subnetwork_self_link + + # Note: the existence of the "master_authorized_networks_config" block enables + # the master authorized networks even if it's empty. + master_authorized_networks_config { + dynamic "cidr_blocks" { + for_each = var.master_authorized_networks + content { + cidr_block = cidr_blocks.value.cidr_block + display_name = cidr_blocks.value.display_name + } + } + gcp_public_cidrs_access_enabled = var.gcp_public_cidrs_access_enabled + } + + private_ipv6_google_access = var.enable_private_ipv6_google_access ? "PRIVATE_IPV6_GOOGLE_ACCESS_TO_GOOGLE" : null + default_max_pods_per_node = var.default_max_pods_per_node + master_auth { + client_certificate_config { + issue_client_certificate = false + } + } + + enable_shielded_nodes = true + + cluster_autoscaling { + # Controls auto provisioning of node-pools + enabled = false + + # Controls autoscaling algorithm of node-pools + autoscaling_profile = var.autoscaling_profile + } + + datapath_provider = local.derived_enable_dataplane_v2 ? "ADVANCED_DATAPATH" : "LEGACY_DATAPATH" + + enable_multi_networking = local.derived_enable_multi_networking + + network_policy { + # Enabling NetworkPolicy for clusters with DatapathProvider=ADVANCED_DATAPATH + # is not allowed. Dataplane V2 will take care of network policy enforcement + # instead. + enabled = false + # GKE Dataplane V2 support. This must be set to PROVIDER_UNSPECIFIED in + # order to let the datapath_provider take effect. + # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/656#issuecomment-720398658 + provider = "PROVIDER_UNSPECIFIED" + } + + private_cluster_config { + enable_private_nodes = var.enable_private_nodes + enable_private_endpoint = var.enable_private_endpoint + master_ipv4_cidr_block = var.master_ipv4_cidr_block + master_global_access_config { + enabled = var.enable_master_global_access + } + } + + ip_allocation_policy { + cluster_secondary_range_name = var.pods_ip_range_name + services_secondary_range_name = var.services_ip_range_name + } + + workload_identity_config { + workload_pool = "${var.project_id}.svc.id.goog" + } + + dynamic "gateway_api_config" { + for_each = var.enable_inference_gateway ? [1] : [] + content { + channel = "CHANNEL_STANDARD" + } + } + + dynamic "authenticator_groups_config" { + for_each = local.cluster_authenticator_security_group + content { + security_group = authenticator_groups_config.value.security_group + } + } + + release_channel { + channel = var.release_channel + } + min_master_version = local.master_version + + maintenance_policy { + daily_maintenance_window { + start_time = var.maintenance_start_time + } + + dynamic "maintenance_exclusion" { + for_each = var.maintenance_exclusions + content { + exclusion_name = maintenance_exclusion.value.name + start_time = maintenance_exclusion.value.start_time + end_time = maintenance_exclusion.value.end_time + exclusion_options { + scope = maintenance_exclusion.value.exclusion_scope + } + } + } + } + + dynamic "dns_config" { + for_each = var.cloud_dns_config != null ? [1] : [] + content { + additive_vpc_scope_dns_domain = var.cloud_dns_config.additive_vpc_scope_dns_domain + cluster_dns = var.cloud_dns_config.cluster_dns + cluster_dns_scope = var.cloud_dns_config.cluster_dns_scope + cluster_dns_domain = var.cloud_dns_config.cluster_dns_domain + } + } + + addons_config { + gcp_filestore_csi_driver_config { + enabled = var.enable_filestore_csi + } + gcs_fuse_csi_driver_config { + enabled = var.enable_gcsfuse_csi + } + gce_persistent_disk_csi_driver_config { + enabled = var.enable_persistent_disk_csi + } + dns_cache_config { + enabled = var.enable_node_local_dns_cache + } + parallelstore_csi_driver_config { + enabled = var.enable_parallelstore_csi + } + ray_operator_config { + enabled = var.enable_ray_operator + } + lustre_csi_driver_config { + enabled = var.enable_managed_lustre_csi + } + dynamic "http_load_balancing" { + for_each = var.enable_inference_gateway ? [1] : [] + content { + disabled = false + } + } + } + + timeouts { + create = var.timeout_create + update = var.timeout_update + } + + node_config { + shielded_instance_config { + enable_secure_boot = var.system_node_pool_enable_secure_boot + enable_integrity_monitoring = true + } + } + + control_plane_endpoints_config { + dns_endpoint_config { + allow_external_traffic = var.enable_external_dns_endpoint + } + } + + lifecycle { + # Ignore all changes to the default node pool. It's being removed after creation. + ignore_changes = [ + node_config, + min_master_version, + ] + precondition { + condition = var.default_max_pods_per_node == null || var.networking_mode == "VPC_NATIVE" + error_message = "default_max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." + } + precondition { + condition = coalesce(var.enable_dataplane_v2, true) || !local.derived_enable_multi_networking + error_message = "'enable_dataplane_v2' cannot be false when enabling multi networking." + } + precondition { + condition = coalesce(var.enable_multi_networking, true) || length(var.additional_networks) == 0 + error_message = "'enable_multi_networking' cannot be false when using multivpc module, which passes additional_networks." + } + } + + monitoring_config { + enable_components = var.enable_dcgm_monitoring ? concat(local.default_monitoring_component, ["DCGM"]) : local.default_monitoring_component + managed_prometheus { + enabled = true + } + } + + logging_config { + enable_components = local.default_logging_component + } +} + +# We define explicit node pools, so that it can be modified without +# having to destroy the entire cluster. +resource "google_container_node_pool" "system_node_pools" { + provider = google-beta + count = var.system_node_pool_enabled ? 1 : 0 + + project = var.project_id + name = var.system_node_pool_name + cluster = var.cluster_reference_type == "NAME" ? google_container_cluster.gke_cluster.name : google_container_cluster.gke_cluster.self_link + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + node_locations = var.system_node_pool_zones + version = local.master_version + + autoscaling { + total_min_node_count = var.system_node_pool_node_count.total_min_nodes + total_max_node_count = var.system_node_pool_node_count.total_max_nodes + } + + upgrade_settings { + strategy = local.upgrade_settings.strategy + max_surge = local.upgrade_settings.max_surge + max_unavailable = local.upgrade_settings.max_unavailable + } + + management { + auto_repair = true + auto_upgrade = true + } + + node_config { + labels = var.system_node_pool_kubernetes_labels + resource_labels = local.labels + service_account = var.service_account_email + oauth_scopes = var.service_account_scopes + machine_type = var.system_node_pool_machine_type + disk_size_gb = var.system_node_pool_disk_size_gb + disk_type = var.system_node_pool_disk_type + + dynamic "taint" { + for_each = var.system_node_pool_taints + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } + + # Forcing the use of the Container-optimized image, as it is the only + # image with the proper logging daemon installed. + # + # cos images use Shielded VMs since v1.13.6-gke.0. + # https://cloud.google.com/kubernetes-engine/docs/how-to/node-images + # + # We use COS_CONTAINERD to be compatible with (optional) gVisor. + # https://cloud.google.com/kubernetes-engine/docs/how-to/sandbox-pods + image_type = var.system_node_pool_image_type + + shielded_instance_config { + enable_secure_boot = var.system_node_pool_enable_secure_boot + enable_integrity_monitoring = true + } + + gvnic { + enabled = var.system_node_pool_image_type == "COS_CONTAINERD" + } + + # Implied by Workload Identity + workload_metadata_config { + mode = "GKE_METADATA" + } + # Implied by workload identity. + metadata = { + "disable-legacy-endpoints" = "true" + } + } + + lifecycle { + ignore_changes = [ + node_config[0].labels, + node_config[0].taint, + version, + ] + precondition { + condition = contains(["SURGE"], local.upgrade_settings.strategy) + error_message = "Only SURGE strategy is supported" + } + precondition { + condition = local.upgrade_settings.max_unavailable >= 0 + error_message = "max_unavailable should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_surge >= 0 + error_message = "max_surge should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 + error_message = "At least one of max_unavailable or max_surge must greater than 0" + } + } +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${google_container_cluster.gke_cluster.endpoint}" + cluster_ca_certificate = base64decode(google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) + token = data.google_client_config.default.access_token +} + +module "workload_identity" { + count = var.configure_workload_identity_sa ? 1 : 0 + source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" + version = ">= 40.0" + + use_existing_gcp_sa = true + name = var.k8s_service_account_name + gcp_sa_name = local.sa_email + project_id = var.project_id + + # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1059 + depends_on = [ + data.google_project.project, + google_container_cluster.gke_cluster + ] +} + +locals { + k8s_service_account_name = one(module.workload_identity[*].k8s_service_account_name) +} + +locals { + # Separate gvnic and rdma networks and assign indexes + gvnic_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "GVNIC")] : + merge(net, { name = "${var.k8s_network_names.gvnic_prefix}${idx + var.k8s_network_names.gvnic_start_index}${var.k8s_network_names.gvnic_postfix}" }) + ] + + rdma_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "RDMA")] : + merge(net, { name = "${var.k8s_network_names.rdma_prefix}${idx + var.k8s_network_names.rdma_start_index}${var.k8s_network_names.rdma_postfix}" }) + ] + + all_networks = concat(local.gvnic_networks, local.rdma_networks) +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = google_container_cluster.gke_cluster.id + project_id = var.project_id + + apply_manifests = concat(flatten([ + for idx, network_info in local.all_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = network_info.name, + network_name = network_info.network + subnetwork_name = network_info.subnetwork, + device_mode = strcontains(upper(network_info.nic_type), "RDMA") ? "RDMA" : "NetDevice" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = network_info.name } + } + ] + ]), + var.enable_inference_gateway ? [ + { + source = "https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.0.0/manifests.yaml", + template_vars = {} + } + ] : [] + ) +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml new file mode 100644 index 0000000000..bd1517ce8f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/outputs.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/outputs.tf new file mode 100644 index 0000000000..3326a5468e --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/outputs.tf @@ -0,0 +1,104 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_id" { + description = "An identifier for the resource with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." + value = google_container_cluster.gke_cluster.id +} + +output "gke_cluster_exists" { + description = "A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations." + value = true + depends_on = [ + google_container_cluster.gke_cluster + ] +} + +locals { + private_endpoint_message = trimspace( + <<-EOT + This cluster was created with 'enable_private_endpoint: true'. + It cannot be accessed from a public IP addresses. + One way to access this cluster is from a VM created in the GKE cluster subnet. + EOT + ) + master_authorized_networks_message = length(var.master_authorized_networks) == 0 ? "" : trimspace( + <<-EOT + The following networks have been authorized to access this cluster: + ${join("\n", [for x in var.master_authorized_networks : " ${x.display_name}: ${x.cidr_block}"])}" + EOT + ) + public_endpoint_message = trimspace( + <<-EOT + To add authorized networks you can allowlist your IP with this command: + gcloud container clusters update ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} \ + --enable-master-authorized-networks \ + --master-authorized-networks /32 + EOT + ) + allowlist_your_ip_message = var.enable_private_endpoint ? local.private_endpoint_message : local.public_endpoint_message + kubernetes_service_account_message = local.k8s_service_account_name == null ? "" : trimspace( + <<-EOT + Use the following Kubernetes Service Account in the default namespace to run your workloads: + ${local.k8s_service_account_name} + The GCP Service Account mapped to this Kubernetes Service Account is: + ${local.sa_email} + EOT + ) + kubernetes_cluster_fetch_credential_message = var.enable_external_dns_endpoint ? trimspace( + <<-EOT + Use the following command to fetch credentials for the created cluster: + gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} \ + --dns-endpoint + EOT + ) : trimspace( + <<-EOT + Use the following command to fetch credentials for the created cluster: + gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} + EOT + ) +} + +output "instructions" { + description = "Instructions on how to connect to the created cluster." + value = trimspace( + <<-EOT + ${local.master_authorized_networks_message} + + ${local.allowlist_your_ip_message} + + ${local.kubernetes_cluster_fetch_credential_message} + + ${local.kubernetes_service_account_message} + EOT + ) +} + +output "k8s_service_account_name" { + description = "Name of k8s service account." + value = local.k8s_service_account_name +} + +output "gke_version" { + description = "GKE cluster's version." + value = google_container_cluster.gke_cluster.master_version +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl new file mode 100644 index 0000000000..d376a1a760 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl @@ -0,0 +1,9 @@ +--- +apiVersion: networking.gke.io/v1 +kind: GKENetworkParamSet +metadata: + name: ${name} +spec: + vpc: ${network_name} + vpcSubnet: ${subnetwork_name} + deviceMode: ${device_mode} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl new file mode 100644 index 0000000000..1571a92692 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl @@ -0,0 +1,11 @@ +--- +apiVersion: networking.gke.io/v1 +kind: Network +metadata: + name: ${name} +spec: + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: ${name} + type: Device diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/variables.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/variables.tf new file mode 100644 index 0000000000..8d863b1730 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/variables.tf @@ -0,0 +1,533 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "name_suffix" { + description = "Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`." + type = string + default = "" +} + +variable "deployment_name" { + description = "Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`." + type = string +} + +variable "prefix_with_deployment_name" { + description = "If true, cluster name will be prefixed by `deployment_name` (ex: -)." + type = bool + default = true +} + +variable "region" { + description = "The region to host the cluster in." + type = string +} + +variable "zone" { + description = "Zone for a zonal cluster." + default = null + type = string +} + +variable "network_id" { + description = "The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`." + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to host the cluster in." + type = string +} + +variable "pods_ip_range_name" { + description = "The name of the secondary subnet ip range to use for pods." + type = string + default = "pods" +} + +variable "services_ip_range_name" { + description = "The name of the secondary subnet range to use for services." + type = string + default = "services" +} + +variable "enable_private_ipv6_google_access" { + description = "The private IPv6 google access type for the VMs in this subnet." + type = bool + default = true +} + +variable "release_channel" { + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`." + type = string + default = "UNSPECIFIED" +} + +variable "min_master_version" { + description = "The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release." + type = string + default = null +} + +variable "version_prefix" { + description = "If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally." + type = string + default = "1.31." +} + +variable "maintenance_start_time" { + description = "Start time for daily maintenance operations. Specified in GMT with `HH:MM` format." + type = string + default = "09:00" +} + +variable "maintenance_exclusions" { + description = "List of maintenance exclusions. A cluster can have up to three." + type = list(object({ + name = string + start_time = string + end_time = string + exclusion_scope = string + })) + default = [] + validation { + condition = alltrue([ + for x in var.maintenance_exclusions : + contains(["NO_UPGRADES", "NO_MINOR_UPGRADES", "NO_MINOR_OR_NODE_UPGRADES"], x.exclusion_scope) + ]) + error_message = "`exclusion_scope` must be set to `NO_UPGRADES` OR `NO_MINOR_UPGRADES` OR `NO_MINOR_OR_NODE_UPGRADES`." + } +} + +variable "cloud_dns_config" { + description = < **_NOTE:_** The `project_id` and `region` settings would be inferred from the +> deployment variables of the same name, but they are included here for clarity. + +### Multi-networking + +To create network objects in GKE cluster, you can pass a multivpc module to a pre-existing-gke-cluster module instead of [applying a manifest manually](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment). + +```yaml + - id: network + source: modules/network/vpc + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: existing-gke-cluster ## multinetworking must be enabled in advance when cluster creation + source: modules/scheduler/pre-existing-gke-cluster + use: [multinetwork] + settings: + cluster_name: $(vars.deployment_name) +``` + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_container_cluster.existing_gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the existing cluster | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project that hosts the existing cluster | `string` | n/a | yes | +| [rdma\_subnetwork\_name\_prefix](#input\_rdma\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | `string` | `null` | no | +| [region](#input\_region) | Region in which to search for the cluster | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | +| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster exists. | +| [gke\_version](#output\_gke\_version) | GKE cluster's version. | + diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf new file mode 100644 index 0000000000..926d2be100 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_container_cluster" "existing_gke_cluster" { + name = var.cluster_name + project = var.project_id + location = var.region +} + +locals { + rdma_networks = [for network_info in var.additional_networks : network_info if strcontains(upper(network_info.nic_type), "RDMA")] + non_rdma_networks = [for network_info in var.additional_networks : network_info if !strcontains(upper(network_info.nic_type), "RDMA")] + apply_manifests_rdma_networks = flatten([ + for idx, network_info in local.rdma_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = "${var.rdma_subnetwork_name_prefix}-${idx}", + network_name = network_info.network + subnetwork_name = "${var.rdma_subnetwork_name_prefix}-${idx}", + device_mode = "RDMA" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = "${var.rdma_subnetwork_name_prefix}-${idx}" } + } + ] + ]) + + apply_manifests_non_rdma_networks = flatten([ + for idx, network_info in local.non_rdma_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = network_info.subnetwork + network_name = network_info.network + subnetwork_name = network_info.subnetwork + device_mode = "NetDevice" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = network_info.subnetwork } + } + ] + ]) +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = data.google_container_cluster.existing_gke_cluster.id + project_id = var.project_id + + apply_manifests = concat(local.apply_manifests_non_rdma_networks, local.apply_manifests_rdma_networks) +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf new file mode 100644 index 0000000000..8884ee30b0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_id" { + description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." + value = data.google_container_cluster.existing_gke_cluster.id +} + +output "gke_cluster_exists" { + description = "A static flag that signals to downstream modules that a cluster exists." + value = true + depends_on = [ + data.google_container_cluster.existing_gke_cluster + ] +} + +output "gke_version" { + description = "GKE cluster's version." + value = data.google_container_cluster.existing_gke_cluster.master_version +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl new file mode 100644 index 0000000000..d376a1a760 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl @@ -0,0 +1,9 @@ +--- +apiVersion: networking.gke.io/v1 +kind: GKENetworkParamSet +metadata: + name: ${name} +spec: + vpc: ${network_name} + vpcSubnet: ${subnetwork_name} + deviceMode: ${device_mode} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl new file mode 100644 index 0000000000..1571a92692 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl @@ -0,0 +1,11 @@ +--- +apiVersion: networking.gke.io/v1 +kind: Network +metadata: + name: ${name} +spec: + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: ${name} + type: Device diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf new file mode 100644 index 0000000000..9e9ed98ed3 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project that hosts the existing cluster" + type = string +} + +variable "cluster_name" { + description = "Name of the existing cluster" + type = string +} + +variable "region" { + description = "Region in which to search for the cluster" + type = string +} + +variable "additional_networks" { + description = "Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster." + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + })) +} + +variable "rdma_subnetwork_name_prefix" { + description = "Prefix of the RDMA subnetwork names" + default = null + type = string +} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf new file mode 100644 index 0000000000..562d8647b1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "> 5.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.74.0" + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/README.md b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/README.md new file mode 100644 index 0000000000..db9094909b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/README.md @@ -0,0 +1,355 @@ +## Description + +This module creates a startup script that will execute a list of runners in the +order they are specified. The runners are copied to a GCS bucket at deployment +time and then copied into the VM as they are executed after startup. + +Each runner receives the following attributes: + +- `destination`: (Required) The name of the file at the destination VM. If an + absolute path is provided, the file will be copied to that path, otherwise + the file will be created in a temporary folder and deleted once the startup + script runs. +- `type`: (Required) The type of the runner, one of the following: + - `shell`: The runner is a shell script and will be executed once copied to + the destination VM. + - `ansible-local`: The runner is an ansible playbook and will run on the VM + with the following command line flags: + + ```shell + ansible-playbook --connection=local --inventory=localhost, \ + --limit localhost <> + ``` + + - `data`: The data or file specified will be copied to `<>`. No + action will be performed after the data is staged. This data can be used by + subsequent runners or simply made available on the VM for later use. +- `content`: (Optional) Content to be uploaded and, if `type` is + either `shell` or `ansible-local`, executed. Must be defined if `source` is + not. +- `source`: (Optional) A path to the file or data you want to upload. Must be + defined if `content` is not. The source path is relative to the deployment + group directory. To ensure correctness of path use `ghpc_stage` function, that + would copy referenced file to the deployment group directory. For example: + + ```yaml + source: $(ghpc_stage("path/to/file")) + ``` + + For more examples with context, see the + [example blueprint snippet](#example). To reference any other source file, an + absolute path must be used. + +- `args`: (Optional) Arguments to be passed to `shell` or `ansible-local` + runners. For `shell` runners, these will be passed as arguments to the script + when it is executed. For `ansible-local` runners, they will be appended to + a list of default arguments that invoke `ansible-playbook` on the localhost. + Therefore`args` should not include any arguments that alter this behavior, + such as `--connection`, `--inventory`, or `--limit`. + +### Runner dependencies + +`ansible-local` runners require Ansible to be installed in the VM before +running. To support other playbook runners in the Cluster Toolkit, we install +version 2.11 of `ansible-core` as well as the larger package of collections +found in `ansible` version 4.10.0. + +If an `ansible-local` runner is found in the list supplied to this module, +a script to install Ansible will be prepended to the list of runners. This +behavior can be disabled by setting `var.prepend_ansible_installer` to `false`. +This script will do the following at VM startup: + +- Install system-wide python3 if not already installed using system package + managers (yum, apt-get, etc) +- Install `python3-distutils` system-wide in debian and ubuntu based + environments. This can be a missing dependency on system installations of + python3 for installing and upgrading pip. +- Install system-wide pip3 if not already installed and upgrade pip3 if the + version is not at least 18.0. +- Install and create a virtual environment located at `/usr/local/ghpc-venv`. +- Install ansible into this virtual environment if the current version of + ansible is not version 2.11 or higher. + +To use the virtual environment created by this script, you can activate it by +running the following command on the VM: + +```shell +source /usr/local/ghpc-venv/bin/activate +``` + +You may also need to provide the correct python interpreter as the python3 +binary in the virtual environment. This can be done by adding the following flag +when calling `ansible-playbook`: + +```shell +-e ansible_python_interpreter=/usr/local/ghpc-venv/bin/activate +``` + +> **_NOTE:_** ansible-playbook and other ansible command line tools will only be +> accessible from the command line (and in your PATH variable) after activating +> this environment. + +### Staging the runners + +Runners will be uploaded to a +[GCS bucket](https://cloud.google.com/storage/docs/creating-buckets). This +bucket will be created by this module and named as +`${var.deployment_name}-startup-scripts-${random_id}`. VMs using the startup +script created by this module will pull the runners content from a GCS bucket +and therefore must have access to GCS. + +> **_NOTE:_** To ensure access to GCS, set the following OAuth scope on the +> instance using the startup scripts: +> `https://www.googleapis.com/auth/devstorage.read_only`. +> +> This is set as a default scope in the [vm-instance], +> [schedMD-slurm-on-gcp-login-node] and [schedMD-slurm-on-gcp-controller] +> modules + +[vm-instance]: ../../compute/vm-instance/README.md +[schedMD-slurm-on-gcp-login-node]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md +[schedMD-slurm-on-gcp-controller]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md + +### Tracking startup script execution + +For more information on how to use startup scripts on Google Cloud Platform, +please refer to +[this document](https://cloud.google.com/compute/docs/instances/startup-scripts/linux). + +To debug startup scripts from a Linux VM created with startup script generated +by this module: + +```shell +sudo DEBUG=1 google_metadata_script_runner startup +``` + +To view outputs from a Linux startup script, run: + +```shell +sudo journalctl -u google-startup-scripts.service +``` + +### Monitoring Agent Installation + +This `startup-script` module has several options for installing a Google +monitoring agent. There are two relevant settings: `install_stackdriver_agent` +and `install_cloud_ops_agent`. + +The _Stackdriver Agent_ also called the _Legacy Cloud Monitoring Agent_ provides +better performance under some HPC workloads. While official documentation +recommends using the _Cloud Ops Agent_, it is recommended to use +`install_stackdriver_agent` when performance is important. + +#### Stackdriver Agent Installation + +If an image or machine already has Cloud Ops Agent installed and you would like +to instead use the Stackdriver Agent, the following script will remove the Cloud +Ops Agent and install the Stackdriver Agent. + +```bash +# Remove Cloud Ops Agent +sudo systemctl stop google-cloud-ops-agent.service +sudo systemctl disable google-cloud-ops-agent.service +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +sudo bash add-google-cloud-ops-agent-repo.sh --uninstall +sudo bash add-google-cloud-ops-agent-repo.sh --remove-repo + +# Install Stackdriver Agent +curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh +sudo bash add-monitoring-agent-repo.sh --also-install +curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh +sudo bash add-logging-agent-repo.sh --also-install +sudo service stackdriver-agent start +sudo service google-fluentd restart +``` + +#### Cloud Ops Agent Installation + +If an image or machine already has the Stackdriver Agent installed and you would +like to instead use the Cloud Ops Agent, the following script will remove the +Stackdriver Agent and install the Cloud Ops Agent. + +```bash +# UnInstall Stackdriver Agent + +sudo systemctl stop stackdriver-agent.service +sudo systemctl disable stackdriver-agent.service +curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh +sudo dpkg --configure -a +sudo bash add-monitoring-agent-repo.sh --uninstall +sudo bash add-monitoring-agent-repo.sh --remove-repo +sudo systemctl stop google-fluentd.service +sudo systemctl disable google-fluentd.service +sudo dpkg --configure -a +curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh +sudo bash add-logging-agent-repo.sh --uninstall +sudo bash add-logging-agent-repo.sh --remove-repo + +# Install ops-agent + +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +sudo bash add-google-cloud-ops-agent-repo.sh --also-install +sudo service google-cloud-ops-agent start +``` + +As a reminder, this should be in a startup script, which should run on all +Compute nodes via the `compute_startup_script` on the controller. + +#### Testing Installation + +You can test if one of the agents is running using the following commands: + +```bash +# For Cloud Ops Agent +$ sudo systemctl is-active google-cloud-ops-agent"*" +active +active +active +active + +# For Legacy Monitoring and Logging Agents +$ sudo service stackdriver-agent status +stackdriver-agent is running [ OK ] +$ sudo service google-fluentd status +google-fluentd is running [ OK ] +``` + +For official documentation see troubleshooting docs: + +- [Cloud Ops Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-install-startup) +- [Legacy Monitoring Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/monitoring/troubleshooting) +- [Legacy Logging Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/logging/troubleshooting) + +### Example + +```yaml +- id: startup + source: modules/scripts/startup-script + settings: + runners: + # Some modules such as filestore have runners as outputs for convenience: + - $(homefs.install_nfs_client_runner) + # These runners can still be created manually: + # - type: shell + # destination: "modules/filestore/scripts/install_nfs_client.sh" + # source: "modules/filestore/scripts/install_nfs_client.sh" + - type: ansible-local + destination: "modules/filestore/scripts/mount.yaml" + source: "modules/filestore/scripts/mount.yaml" + - type: data + source: /tmp/foo.tgz + destination: /tmp/bar.tgz + - type: shell + destination: "decompress.sh" + content: | + #!/bin/sh + echo $2 + tar zxvf /tmp/$1 -C / + args: "bar.tgz 'Expanding file'" + +- id: compute-cluster + source: modules/compute/vm-instance + use: [homefs, startup] +``` + +In the above example, a new GCS bucket is created to upload the startup-scripts. +But in the case where the user wants to reuse existing GCS bucket or folder, +they are able to do so by using the `gcs_bucket_path` as shown in the below example + +```yaml +- id: startup + source: modules/scripts/startup-script + settings: + gcs_bucket_path: gs://user-test-bucket/folder1/folder2 + install_stackdriver_agent: true + +- id: compute-cluster + source: modules/compute/vm-instance + use: [startup] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 6.41 | +| [local](#requirement\_local) | >= 2.0.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.41 | +| [local](#provider\_local) | >= 2.0.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.configs_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | +| [google_storage_bucket_object.scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [local_file.debug_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [ansible\_virtualenv\_path](#input\_ansible\_virtualenv\_path) | Virtual environment path in which to install Ansible | `string` | `"/usr/local/ghpc-venv"` | no | +| [bucket\_viewers](#input\_bucket\_viewers) | Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account) | `list(string)` | `[]` | no | +| [configure\_ssh\_host\_patterns](#input\_configure\_ssh\_host\_patterns) | If specified, it will automate ssh configuration by:
- Defining a Host block for every element of this variable and setting StrictHostKeyChecking to 'No'.
Ex: "hpc*", "hpc01*", "ml*"
- The first time users log-in, it will create ssh keys that are added to the authorized keys list
This requires a shared /home filesystem and relies on specifying the right prefix. | `list(string)` | `[]` | no | +| [debug\_file](#input\_debug\_file) | Path to an optional local to be written with 'startup\_script'. | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used to name GCS bucket for startup scripts. | `string` | n/a | yes | +| [docker](#input\_docker) | Install and configure Docker | 
object({
    enabled        = optional(bool, false)
    world_writable = optional(bool, false)
    daemon_config  = optional(string, "")
  })
| 
{
  "enabled": false
}
| no | +| [enable\_docker\_world\_writable](#input\_enable\_docker\_world\_writable) | DEPRECATED: use var.docker | `bool` | `null` | no | +| [enable\_gpu\_network\_wait\_online](#input\_enable\_gpu\_network\_wait\_online) | Enable a SystemD unit that blocks execution of startup-scripts until after all network interfaces are online. (Works on reboots or boots of an image built using this solution) | `bool` | `false` | no | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | `null` | no | +| [http\_no\_proxy](#input\_http\_no\_proxy) | Domains for which to disable http\_proxy behavior. Honored only if var.http\_proxy is set | `string` | `".google.com,.googleapis.com,metadata.google.internal,localhost,127.0.0.1"` | no | +| [http\_proxy](#input\_http\_proxy) | Web (http and https) proxy configuration for pip, apt, and yum/dnf and interactive shells | `string` | `""` | no | +| [install\_ansible](#input\_install\_ansible) | Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used. | `bool` | `null` | no | +| [install\_cloud\_ops\_agent](#input\_install\_cloud\_ops\_agent) | Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true. | `bool` | `false` | no | +| [install\_cloud\_rdma\_drivers](#input\_install\_cloud\_rdma\_drivers) | If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8. Should not be enabled if using the HPC VM Image. | `bool` | `false` | no | +| [install\_docker](#input\_install\_docker) | DEPRECATED: use var.docker. | `bool` | `null` | no | +| [install\_stackdriver\_agent](#input\_install\_stackdriver\_agent) | Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance. | `bool` | `false` | no | +| [labels](#input\_labels) | Labels for the created GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_filesystem](#input\_local\_ssd\_filesystem) | Create and mount a filesystem from local SSD disks (data will be lost if VMs are powered down without enabling migration); enable by setting mountpoint field to a valid directory path. | 
object({
    fs_type     = optional(string, "ext4")
    mountpoint  = optional(string, "")
    permissions = optional(string, "0755")
  })
| 
{
  "fs_type": "ext4",
  "mountpoint": "",
  "permissions": "0755"
}
| no | +| [managed\_lustre](#input\_managed\_lustre) | Configure Managed Lustre (assumes driver already installed) | 
object({
    enabled = optional(bool, false)
    port    = optional(number, 988)
  })
| 
{
  "enabled": false,
  "port": 988
}
| no | +| [prepend\_ansible\_installer](#input\_prepend\_ansible\_installer) | DEPRECATED. Use `install_ansible=false` to prevent ansible installation. | `bool` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [runners](#input\_runners) | List of runners to run on remote VM.
Runners can be of type ansible-local, shell or data.
A runner must specify one of 'source' or 'content'.
All runners must specify 'destination'. If 'destination' does not include a
path, it will be copied in a temporary folder and deleted after running.
Runners may also pass 'args', which will be passed as argument to shell runners only. | `list(map(string))` | `[]` | no | +| [set\_ofi\_cloud\_rdma\_tunables](#input\_set\_ofi\_cloud\_rdma\_tunables) | Controls whether to enable specific OFI environment variables for workloads using Cloud RDMA networking. Should be false for non-RDMA workloads. | `bool` | `false` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [compute\_startup\_script](#output\_compute\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | +| [controller\_startup\_script](#output\_controller\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | +| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | + diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml new file mode 100644 index 0000000000..02c449c7cb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml @@ -0,0 +1,37 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Configure ssh between nodes + become: true + hosts: localhost + vars: + ssh_config_path: "/etc/ssh/ssh_config" + bashrc: "{{ '/etc/bashrc' if ansible_facts['os_family'] == 'RedHat' else '/etc/bash.bashrc' }}" + setup_ssh_script: "/bin/bash /usr/local/ghpc/setup-ssh-keys.sh" + tasks: + - name: "Set StrictHostKeyChecking to no" + ansible.builtin.blockinfile: + path: "{{ ssh_config_path }}" + block: | + Host "{{ item }}" + StrictHostKeyChecking no + marker: "# {mark} ANSIBLE MANAGED BLOCK {{item}}" + loop: "{{ host_name_prefix }}" + - name: "Create ssh keys in .bashrc if not already done" + ansible.builtin.lineinfile: + path: "{{ bashrc }}" + regexp: '^{{ setup_ssh_script }}' + line: "{{ setup_ssh_script }}" diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh new file mode 100644 index 0000000000..38c7ff9b5c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +web_proxy="${1:-}" +if [ -z "$web_proxy" ]; then + echo "Error: must provide 1 argument identifying http/https proxy" + exit 1 +fi + +# configure pip to use proxy +PIP_CONF=/etc/pip.conf +if [ ! -f "$PIP_CONF" ]; then + cat <<-EOF >"$PIP_CONF" + [global] + proxy=$web_proxy + EOF +fi + +# configure yum or dnf to use proxy +if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + YUM_CONF="/etc/yum.conf" + if ! grep -q '^proxy=.*' "$YUM_CONF"; then + sed --follow-symlinks -i.bak "/^\[main]/a proxy=$web_proxy" "$YUM_CONF" + else + sed --follow-symlinks -i.bak "s,proxy=.*,proxy=$web_proxy," "$YUM_CONF" + fi +fi + +# configure apt to use proxy +if [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release 2>/dev/null || + grep -qi ubuntu /etc/os-release 2>/dev/null; then + APT_CONF_PROXY="/etc/apt/apt.conf.d/99proxy.conf" + if [ ! -f "$APT_CONF_PROXY" ]; then + cat <<-EOF >"$APT_CONF_PROXY" + Acquire::http::Proxy "$web_proxy"; + Acquire::https::Proxy "$web_proxy"; + EOF + fi +fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh new file mode 100644 index 0000000000..682e1352a1 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script applies fixes to VMs that must occur early in boot. For example, +# when yum or apt repositories are misconfigured, preventing most package +# operations from completing successfully. + +source /etc/os-release + +if [[ "$PRETTY_NAME" == "CentOS Linux 7 (Core)" ]]; then + echo "Applying hotfixes for CentOS 7" + if grep -q '^mirrorlist' /etc/yum.repos.d/CentOS-Base.repo; then + echo "Removing mirrorlist from default CentOS 7 repositories" + sed -i '/^mirrorlist/d' /etc/yum.repos.d/CentOS-Base.repo + fi + if grep -q '^#baseurl=http://mirror.centos.org' /etc/yum.repos.d/CentOS-Base.repo; then + echo "Reconfiguring default CentOS 7 repositories to use CentOS Vault" + sed -i 's,^#baseurl=http://mirror.centos.org/,baseurl=http://vault.centos.org/,' /etc/yum.repos.d/CentOS-Base.repo + fi +fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh new file mode 100644 index 0000000000..3a29ae808f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh @@ -0,0 +1,73 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Given a url and filename, download an object to the vardir. When the installed +# version of gcloud is >=402.0.0 (Sept. 2022), then gcloud storage is used to +# fetch from the bucket. Otherwise gsutil is used. Note, the service account for +# the instance must be properly configured with a role having authorization to +# get objects from the bucket. +# +# This function is intended for single file downloads and no attempt is made to +# verify the checksum other than the default behavior of gcloud or gsutil. +# +# This function has no other platform dependencies other than gcloud / gsutil. + +# This code originated from: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 +stdlib::get_from_bucket() { + local OPTIND opt url fname dir="${VARDIR:-/var/lib/startup}" + while getopts ":u:f:d:" opt; do + case "${opt}" in + u) url="${OPTARG}" ;; + f) fname="${OPTARG}" ;; + d) dir="${OPTARG}" ;; + :) + stdlib::mandatory_argument -n stdlib::get_from_bucket -f "$OPTARG" + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error 'Usage: stdlib::get_from_bucket -u -f -d ' + stdlib::info 'For example: stdlib::get_from_bucket -u gs://mybucket/foo.tgz -d /var/tmp' + return "${E_UNKNOWN_ARG}" + ;; + esac + done + # Trivially compute the filename from the URL if unspecified. + if [[ -z ${fname} ]]; then + fname=${url##*/} + stdlib::debug "Computed filename='${fname}' given URL." + fi + [[ -d ${dir} ]] || mkdir "${dir}" + local attempt=0 + local max_retries=7 + # store gcs command as array and then split when called by stdlib::cmd + if stdlib::cmd gcloud help storage cp &>/dev/null; then + gcs_command=(gcloud storage cp --no-user-output-enabled) + else + gcs_command=(gsutil -q cp) + fi + while [[ $attempt -le $max_retries ]]; do + if [[ $attempt -gt 0 ]]; then + local wait=$((2 ** attempt)) + stdlib::error "Retry attempt ${attempt} of ${max_retries} with exponential backoff: ${wait} seconds." + sleep $wait + fi + if stdlib::cmd "${gcs_command[@]}" "${url}" "${dir}/${fname}"; then + break + else + stdlib::error "${gcs_command[*]} reported non-zero exit code fetching ${url}." + ((attempt++)) + fi + done +} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh new file mode 100644 index 0000000000..eac2b2e32a --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh @@ -0,0 +1,247 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +REQ_ANSIBLE_VERSION=2.15 +REQ_ANSIBLE_PIP_VERSION=8.7.0 +REQ_PIP_WHEEL_VERSION=0.45.1 +REQ_PIP_SETUPTOOLS_VERSION=80.8.0 +REQ_PIP_MAJOR_VERSION=25 +REQ_PYTHON3_VERSION=9 + +apt_wait() { + while fuser /var/lib/apt/lists/lock >/dev/null 2>&1; do + echo "Sleeping for apt lists lock" + sleep 3 + done +} + +# Installs any dependencies needed for python based on the OS +install_python_deps() { + # this file is present on both Debian and Ubuntu OSes + if [ -f /etc/debian_version ]; then + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3-setuptools python3-venv + fi +} + +# Gets the name of the python executable for python starting with python3, then +# checking python. Sets the variable to an empty string if neither are found. +get_python_path() { + python_path="" + if command -v python3 1>/dev/null; then + python_path=$(command -v python3) + elif command -v python 1>/dev/null; then + python_path=$(command -v python) + fi +} + +# Returns the python major version. If provided, it will use the first argument +# as the python executable, otherwise it will default to simply "python". +get_python_major_version() { + python_path=${1:-python} + python_major_version=$(${python_path} -c "import sys; print(sys.version_info.major)") +} + +# Returns the python minor version. If provided, it will use the first argument +# as the python executable, otherwise it will default to simply "python". +get_python_minor_version() { + python_path=${1:-python} + python_minor_version=$(${python_path} -c "import sys; print(sys.version_info.minor)") +} + +# Install python3 with the yum package manager. Updates python_path to the +# newly installed packaged. +install_python3_dnf() { + major_version=$(rpm -E "%{rhel}") + set -- "--disablerepo=*" "--enablerepo=baseos,appstream" + if grep -qi 'ID="rhel"' /etc/os-release; then + # Do not set --disablerepo / --enablerepo on RedHat, due to + # complex repo names; clear array + set -- + fi + # On Rocky Linux 9, Python 3.9 is installed by default but this + # has already been dropped by ansible-core for control nodes. + # https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix + # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where + # it is available as "python3*" but must be named explicitly on + # older releases. It also ensures longer support for Ansible. + if [ "${major_version}" -lt "10" ]; then + dnf install "$@" -y python3.12 python3.12-pip + python_path=$(command -v python3.12) + else + dnf install "$@" -y python3 python3-pip + python_path=$(command -v python3) + fi +} + +# Install python3 with the apt package manager. Updates python_path to the +# newly installed packaged. +install_python3_apt() { + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3 python3-setuptools python3-pip python3-venv + python_path=$(command -v python3) +} + +install_python3() { + if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || + [ -f /etc/system-release ]; then + install_python3_dnf + elif [ -f /etc/debian_version ]; then + install_python3_apt + else + echo "Error: Unsupported Distribution" + return 1 + fi +} + +# Install pip3 with the dnf package manager. Updates python_path to the +# newly installed packaged. +install_pip3_dnf() { + major_version=$(rpm -E "%{rhel}") + set -- "--disablerepo=*" "--enablerepo=baseos,appstream" + if grep -qi 'ID="rhel"' /etc/os-release; then + # Do not set --disablerepo / --enablerepo on RedHat, due to complex repo names + # clear array + set -- + fi + # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where + # it is available as "python3*" but must be named explicitly on + # older releases. It also ensures longer support for Ansible. + if [ "${major_version}" -lt "10" ]; then + dnf install "$@" -y python3.12-pip + else + dnf install "$@" -y python3-pip + fi +} + +# Install pip3 with the apt package manager. Updates python_path to the +# newly installed packaged. +install_pip3_apt() { + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3-pip +} + +install_pip3() { + if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || + [ -f /etc/system-release ]; then + install_pip3_dnf + elif [ -f /etc/debian_version ]; then + install_pip3_apt + else + echo "Error: Unsupported Distribution" + return 1 + fi +} + +main() { + if [ $# -gt 1 ]; then + echo "Error: provide only 1 optional argument identifying virtual environment path for Ansible" + return 1 + fi + + venv_path="${1:-/usr/local/ghpc-venv}" + + # Get the python3 executable, or install it if not found + get_python_path + get_python_major_version "${python_path}" + get_python_minor_version "${python_path}" + if [ "${python_path}" = "" ] || [ "${python_major_version}" = "2" ] || [ "${python_minor_version}" -lt "${REQ_PYTHON3_VERSION}" ]; then + if ! install_python3; then + return 1 + fi + get_python_major_version "${python_path}" + get_python_minor_version "${python_path}" + else + install_python_deps + fi + + # Install OS-packaged pip + if ! ${python_path} -m pip --version 2>/dev/null; then + if ! install_pip3; then + return 1 + fi + fi + + # Create pip virtual environment for Cluster Toolkit + ${python_path} -m venv "${venv_path}" --copies + venv_python_path=${venv_path}/bin/python3 + + # Upgrade pip if necessary + pip_version=$(${venv_python_path} -m pip --version | sed -nr 's/^pip ([0-9]+\.[0-9]+).*$/\1/p') + pip_major_version=$(echo "${pip_version}" | cut -d '.' -f 1) + if [ "${pip_major_version}" -lt "${REQ_PIP_MAJOR_VERSION}" ]; then + ${venv_python_path} -m pip install --upgrade pip + fi + + # upgrade wheel if necessary + wheel_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^wheel" || true) + if [ "$wheel_pkg" != "wheel==${REQ_PIP_WHEEL_VERSION}" ]; then + ${venv_python_path} -m pip install -U wheel==${REQ_PIP_WHEEL_VERSION} + fi + + # upgrade setuptools if necessary + setuptools_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^setuptools" || true) + if [ "$setuptools_pkg" != "setuptools==${REQ_PIP_SETUPTOOLS_VERSION}" ]; then + ${venv_python_path} -m pip install -U setuptools==${REQ_PIP_SETUPTOOLS_VERSION} + fi + + # configure ansible to always use correct Python binary + if [ ! -f /etc/ansible/ansible.cfg ]; then + mkdir /etc/ansible + cat <<-EOF >/etc/ansible/ansible.cfg + [defaults] + interpreter_python=${venv_python_path} + stdout_callback=debug + stderr_callback=debug + EOF + fi + + # Install ansible + ansible_version="" + if command -v ansible-playbook 1>/dev/null; then + ansible_version=$(ansible-playbook --version 2>/dev/null | sed -nr 's/^ansible-playbook.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p') + ansible_major_vers=$(echo "${ansible_version}" | cut -d '.' -f 1) + ansible_minor_vers=$(echo "${ansible_version}" | cut -d '.' -f 2) + ansible_req_major_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 1) + ansible_req_minor_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 2) + fi + if [ -z "${ansible_version}" ] || [ "${ansible_major_vers}" -ne "${ansible_req_major_vers}" ] || + [ "${ansible_minor_vers}" -lt "${ansible_req_minor_vers}" ]; then + ${venv_python_path} -m pip install ansible=="${REQ_ANSIBLE_PIP_VERSION}" + fi + while read -r cmd; do + if ! [ -L "/usr/bin/${cmd}" ]; then + ln -s "${venv_path}/bin/${cmd}" "/usr/bin/${cmd}" + fi + done <<-EOF + ansible + ansible-config + ansible-connection + ansible-console + ansible-doc + ansible-galaxy + ansible-inventory + ansible-playbook + ansible-pull + ansible-test + ansible-vault + EOF +} + +main "$@" diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh new file mode 100644 index 0000000000..375792459b --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +OS_ID="$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g')" +OS_VERSION="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g')" +OS_VERSION_MAJOR="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//')" +REBOOT_FILE="/etc/.rdma_reboot" + +if { [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; } && { [ "${OS_VERSION_MAJOR}" = "8" ]; }; then + KMOD_VERSION="$(dnf list installed | awk '$1 ~ /^kmod-idpf-irdma(\.|$)/ {print $2}')" + + # For images that do not already have Cloud RDMA drivers installed + if [ -z "${KMOD_VERSION}" ] && [ -z "${REBOOT_FILE}" ]; then + sudo dnf update -y + sudo dnf install https://depot.ciq.com/public/files/gce-accelerator/irdma-kernel-modules-el8-x86_64/irdma-repos.rpm -y + sudo dnf install kmod-idpf-irdma rdma-core libibverbs-utils librdmacm-utils infiniband-diags perftest -y + sudo touch "${REBOOT_FILE}" + reboot + fi + echo "This image has IRDMA packages already installed, exiting." + exit 0 +else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. Cloud RDMA Drivers are only supported on Rocky Linux 8." + exit 1 +fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_docker.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_docker.yml new file mode 100644 index 0000000000..f9b0abeb14 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_docker.yml @@ -0,0 +1,113 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install and configure Docker + hosts: all + become: true + vars: + docker_data_root: '' + docker_daemon_config: '' + enable_docker_world_writable: false + tasks: + - name: Check if docker is installed + ansible.builtin.stat: + path: /usr/bin/docker + register: docker_binary + - name: Download Docker Installer + ansible.builtin.get_url: + url: https://get.docker.com + dest: /tmp/get-docker.sh + owner: root + group: root + mode: '0644' + when: not docker_binary.stat.exists + - name: Install Docker + ansible.builtin.command: sh /tmp/get-docker.sh + register: docker_installed + changed_when: docker_installed.rc != 0 + when: not docker_binary.stat.exists + - name: Create Docker daemon configuration + ansible.builtin.copy: + dest: /etc/docker/daemon.json + mode: '0644' + content: '{{ docker_daemon_config }}' + validate: /usr/bin/dockerd --validate --config-file %s + when: docker_daemon_config + notify: + - Restart Docker + - name: Create Docker service override directory + ansible.builtin.file: + path: /etc/systemd/system/docker.service.d + state: directory + owner: root + group: root + mode: '0755' + - name: Create Docker service override configuration + ansible.builtin.copy: + dest: /etc/systemd/system/docker.service.d/data-root.conf + mode: '0644' + content: | + [Unit] + {% if docker_data_root %} + RequiresMountsFor={{ docker_data_root }} + {% endif %} + After=mount-localssd-raid.service + - name: Create Docker socket override directory + ansible.builtin.file: + path: /etc/systemd/system/docker.socket.d + state: directory + owner: root + group: root + mode: '0755' + when: enable_docker_world_writable + - name: Create Docker socket override configuration + ansible.builtin.copy: + dest: /etc/systemd/system/docker.socket.d/world-writable.conf + mode: '0644' + content: | + [Socket] + SocketMode=0666 + when: enable_docker_world_writable + notify: + - Reload SystemD + - Recreate Docker socket + - name: Delete Docker socket override configuration + ansible.builtin.file: + path: /etc/systemd/system/docker.socket.d/world-writable.conf + state: absent + when: not enable_docker_world_writable + notify: + - Reload SystemD + - Recreate Docker socket + + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + - name: Recreate Docker socket + ansible.builtin.service: + name: docker.socket + state: restarted + - name: Restart Docker + ansible.builtin.service: + name: docker.service + state: restarted + + post_tasks: + - name: Start Docker + ansible.builtin.service: + name: docker.service + state: started + enabled: true diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml new file mode 100644 index 0000000000..9d295dfc7d --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml @@ -0,0 +1,56 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install network wait service for A3/A4 variants + hosts: all + become: true + tasks: + + - name: Create universal SystemD service for GPU networking delay + when: ansible_os_family == "Debian" + ansible.builtin.copy: + dest: /etc/systemd/system/delay-gpu-network.service + owner: root + group: root + mode: "0644" + content: | + [Unit] + Description=Delay boot on multi-NIC VMs until networks are routable + After=network-online.target + Wants=network-online.target + Before=google-startup-scripts.service + + [Service] + # This condition checks if the machine type is one of the supported A3/A4 variants. + # The service will only run if the machine type matches. + ExecCondition=/bin/bash -c "/usr/bin/curl -s -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/machine-type | grep -qE '(/a3-highgpu-8g|/a3-megagpu-8g|/a3-ultragpu-8g|/a4-highgpu-8g|/a4x-highgpu-4g)$'" + ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -o routable --timeout=180 + ExecStartPost=/bin/sleep 30 + + [Install] + WantedBy=multi-user.target + notify: + - Reload SystemD + + - name: Enable universal GPU network delay service + when: ansible_os_family == "Debian" + ansible.builtin.systemd_service: + name: delay-gpu-network.service + enabled: true + + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml new file mode 100644 index 0000000000..94699471bb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml @@ -0,0 +1,33 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure Managed Lustre (assumes driver already installed) + hosts: all + become: true + vars: + default_lustre_port: 988 + managed_lustre_port: "{{ default_lustre_port }}" + tasks: + # Ideally changes to this file would also trigger an execution of lnetctl + # command to update accept_port but it is unclear if lnetctl supports this. + - name: Update lnet to use non-default port + when: managed_lustre_port | int != {{ default_lustre_port }} + ansible.builtin.copy: + owner: root + group: root + mode: '0644' + dest: /etc/modprobe.d/lnet.conf + content: | + options lnet accept_port={{ managed_lustre_port | int }} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh new file mode 100644 index 0000000000..eb4bf899b8 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +LEGACY_MONITORING_PACKAGE='stackdriver-agent' +LEGACY_MONITORING_SCRIPT_URL='https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh' +LEGACY_LOGGING_PACKAGE='google-fluentd' +LEGACY_LOGGING_SCRIPT_URL='https://dl.google.com/cloudagents/add-logging-agent-repo.sh' + +OPSAGENT_PACKAGE='google-cloud-ops-agent' +OPSAGENT_SCRIPT_URL='https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh' + +ops_or_legacy="${1:-legacy}" + +fail() { + echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" + exit 1 +} + +handle_debian() { + is_legacy_monitoring_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_MONITORING_PACKAGE} | + grep "${LEGACY_MONITORING_PACKAGE} is installed" + } + + is_legacy_logging_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_LOGGING_PACKAGE} | + grep "${LEGACY_LOGGING_PACKAGE} is installed" + } + + is_legacy_installed() { + is_legacy_monitoring_installed || is_legacy_logging_installed + } + + is_opsagent_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${OPSAGENT_PACKAGE} | + grep "${OPSAGENT_PACKAGE} is installed" + } + + install_with_retry() { + MAX_RETRY=50 + RETRY=0 + until [ ${RETRY} -eq ${MAX_RETRY} ] || curl -s "${1}" | bash -s -- --also-install; do + RETRY=$((RETRY + 1)) + echo "WARNING: Installation of ${1} failed on try ${RETRY} of ${MAX_RETRY}" + sleep 5 + done + if [ $RETRY -eq $MAX_RETRY ]; then + echo "ERROR: Installation of ${1} was not successful after ${MAX_RETRY} attempts." + exit 1 + fi + } + + install_opsagent() { + install_with_retry "${OPSAGENT_SCRIPT_URL}" + } + + install_stackdriver_agent() { + install_with_retry "${LEGACY_MONITORING_SCRIPT_URL}" + install_with_retry "${LEGACY_LOGGING_SCRIPT_URL}" + service stackdriver-agent start + service google-fluentd start + } +} + +handle_redhat() { + is_legacy_monitoring_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_MONITORING_PACKAGE} | + grep "${LEGACY_MONITORING_PACKAGE} is installed" + } + + is_legacy_logging_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_LOGGING_PACKAGE} | + grep "${LEGACY_LOGGING_PACKAGE} is installed" + } + + is_legacy_installed() { + is_legacy_monitoring_installed || is_legacy_logging_installed + } + + is_opsagent_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${OPSAGENT_PACKAGE} | + grep "${OPSAGENT_PACKAGE} is installed" + } + + install_opsagent() { + curl -s "${OPSAGENT_SCRIPT_URL}" | bash -s -- --also-install + } + + install_stackdriver_agent() { + curl -sS "${LEGACY_MONITORING_SCRIPT_URL}" | bash -s -- --also-install + curl -sS "${LEGACY_LOGGING_SCRIPT_URL}" | bash -s -- --also-install + service stackdriver-agent start + service google-fluentd start + } +} + +main() { + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + handle_redhat + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + handle_debian + else + fail "Unsupported platform." + fi + + # Handle cases that agent is already installed + if [[ -z "$(is_legacy_monitoring_installed)" && -n $(is_legacy_logging_installed) ]] || + [[ -n "$(is_legacy_monitoring_installed)" && -z $(is_legacy_logging_installed) ]]; then + fail "Bad state: legacy agent is partially installed" + elif [[ "${ops_or_legacy}" == "legacy" ]] && is_legacy_installed; then + echo "Legacy agent is already installed" + exit 0 + elif [[ "${ops_or_legacy}" != "legacy" ]] && is_opsagent_installed; then + echo "Ops agent is already installed" + exit 0 + elif is_legacy_installed || is_opsagent_installed; then + fail "Agent is already installed but does not match requested agent of ${ops_or_legacy}" + fi + + # install agent + if [[ "${ops_or_legacy}" == "legacy" ]]; then + echo "Installing legacy monitoring agent (stackdriver)" + install_stackdriver_agent + else + echo "Installing cloud ops agent" + echo "WARNING: cloud ops agent may have a performance impact. Consider using legacy monitoring agent (stackdriver)." + install_opsagent + fi +} + +main diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh new file mode 100644 index 0000000000..738181aafb --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_COMPLETE_FILE="/run/startup_script_msg" + +# Ensure we're in an interactive terminal and not root +if [ -t 1 ] && [ "$(id -u)" -ne 0 ]; then + # Check if the file has contents otherwise skip + if [ -s "$SCRIPT_COMPLETE_FILE" ]; then + echo + cat "$SCRIPT_COMPLETE_FILE" + echo + fi +fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml new file mode 100644 index 0000000000..d94aac81fd --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml @@ -0,0 +1,100 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Configure local SSDs + become: true + hosts: localhost + vars: + raid_name: localssd + array_dev: /dev/md/{{ raid_name }} + fstype: ext4 + interface: nvme + mode: '0755' + mountpoint: /mnt/{{ raid_name }} + tasks: + - name: Get local SSD devices + ansible.builtin.find: + file_type: link + path: /dev/disk/by-id + patterns: google-local-{{ "nvme-" if interface == "nvme" else "" }}ssd-* + register: local_ssd_devices + + - name: Exit if zero local ssd found + ansible.builtin.meta: end_play + when: local_ssd_devices.files | length == 0 + + - name: Install mdadm + ansible.builtin.package: + name: mdadm + state: present + + # this service will act during the play and upon reboots to ensure that local + # SSD volumes are always assembled into a RAID and re-formatted if necessary; + # there are many scenarios where a VM can be stopped or migrated during + # maintenance and the contents of local SSD will be discarded + - name: Install service to create local SSD RAID and format it + ansible.builtin.copy: + dest: /etc/systemd/system/create-localssd-raid.service + mode: 0644 + content: | + [Unit] + After=local-fs.target + Before=slurmd.service docker.service + ConditionPathExists=!{{ array_dev }} + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/bash -c "/usr/sbin/mdadm --create {{ array_dev }} --name={{ raid_name }} --homehost=any --level=0 --raid-devices={{ local_ssd_devices.files | length }} /dev/disk/by-id/google-local-nvme-ssd-*{{ " --force" if local_ssd_devices.files | length == 1 else "" }}" + ExecStartPost=/usr/sbin/mkfs -t {{ fstype }}{{ " -m 0" if fstype == "ext4" else "" }} {{ array_dev }} + + [Install] + WantedBy=slurmd.service docker.service + + - name: Create RAID array and format + ansible.builtin.systemd: + name: create-localssd-raid.service + state: started + enabled: true + daemon_reload: true + + - name: Install service to mount local SSD array + ansible.builtin.copy: + dest: /etc/systemd/system/mount-localssd-raid.service + mode: 0644 + content: | + [Unit] + After=local-fs.target create-localssd-raid.service + Before=slurmd.service docker.service + Wants=create-localssd-raid.service + ConditionPathIsMountPoint=!{{ mountpoint }} + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/systemd-mount -t {{ fstype }} -o discard,defaults,nofail {{ array_dev }} {{ mountpoint }} + ExecStartPost=/usr/bin/chmod {{ mode }} {{ mountpoint }} + ExecStop=/usr/bin/systemd-umount {{ mountpoint }} + + [Install] + WantedBy=slurmd.service docker.service + + - name: Mount RAID array and set permissions + ansible.builtin.systemd: + name: mount-localssd-raid.service + state: started + enabled: true + daemon_reload: true diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh new file mode 100644 index 0000000000..1c8018fb01 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! -d ~/.ssh/ ]; then + source /usr/local/ghpc-venv/bin/activate + ansible-playbook /usr/local/ghpc/setup-ssh-keys.yml +fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml new file mode 100644 index 0000000000..692896bb9c --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml @@ -0,0 +1,40 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Setup SSH Keys for user + become: false + hosts: localhost + vars: + pub_key_path: "{{ ansible_env.HOME }}/.ssh" + pub_key_file: "{{ pub_key_path }}/id_rsa" + auth_key_file: "{{ pub_key_path }}/authorized_keys" + tasks: + - name: "Create .ssh folder" + ansible.builtin.file: + path: "{{ pub_key_path }}" + state: directory + mode: 0700 + owner: "{{ ansible_user_id }}" + - name: Create keys + community.crypto.openssh_keypair: + path: "{{ pub_key_file }}" + owner: "{{ ansible_user_id }}" + - name: Copy public key to authorized keys + ansible.builtin.copy: + src: "{{ pub_key_file }}.pub" + dest: "{{ auth_key_file }}" + owner: "{{ ansible_user_id }}" + mode: 0644 diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh new file mode 100644 index 0000000000..8ca40bc73f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh @@ -0,0 +1,39 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code contains minor changes from the original: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 + +stdlib::main() { + DELETE_AT_EXIT="$(mktemp -d)" + readonly DELETE_AT_EXIT + + # Initialize state required by other functions, e.g. debug() + stdlib::init + stdlib::debug "Loaded startup-script-stdlib as an executable." + + stdlib::load_config_values + + stdlib::load_runners +} + +# if script is being executed and not sourced. +if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then + stdlib::finish() { + [[ -d ${DELETE_AT_EXIT:-} ]] && rm -rf "${DELETE_AT_EXIT}" + } + trap stdlib::finish EXIT + + stdlib::main "$@" +fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh new file mode 100644 index 0000000000..589a3215ab --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh @@ -0,0 +1,266 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code contains minor changes from the original in: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 + +# Standard library of functions useful for startup scripts. + +# These are outside init_global_vars so logging functions work with the most +# basic case of `source startup-script-stdlib.sh` +readonly SYSLOG_DEBUG_PRIORITY="${SYSLOG_DEBUG_PRIORITY:-syslog.debug}" +readonly SYSLOG_INFO_PRIORITY="${SYSLOG_INFO_PRIORITY:-syslog.info}" +readonly SYSLOG_ERROR_PRIORITY="${SYSLOG_ERROR_PRIORITY:-syslog.error}" +# Global counter of how many times stdlib::init() has been called. +STARTUP_SCRIPT_STDLIB_INITIALIZED=0 + +# Error codes +readonly E_RUN_OR_DIE=5 +readonly E_MISSING_MANDATORY_ARG=9 +readonly E_UNKNOWN_ARG=10 + +SCRIPT_COMPLETE_FILE="/run/startup_script_msg" +SUCCESS_MESSAGE="* NOTICE **: The Cluster Toolkit startup scripts have finished running successfully." +readonly SUCCESS_MESSAGE +ERROR_MESSAGE="** ERROR **: The Cluster Toolkit startup scripts have finished running, but produced an error." +readonly ERROR_MESSAGE +WARNING_MESSAGE="** WARNING **: The Cluster Toolkit startup scripts are currently running." +readonly WARNING_MESSAGE + +stdlib::debug() { + [[ -z ${DEBUG:-} ]] && return 0 + local ds msg + msg="$*" + logger -p "${SYSLOG_DEBUG_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + [[ -n ${QUIET:-} ]] && return 0 + ds="$(date +"${DATE_FMT}") " + echo -e "${BLUE}${ds}Debug [$$]: ${msg}${NC}" >&2 +} + +stdlib::info() { + local ds msg + msg="$*" + logger -p "${SYSLOG_INFO_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + [[ -n ${QUIET:-} ]] && return 0 + ds="$(date +"${DATE_FMT}") " + echo -e "${GREEN}${ds}Info [$$]: ${msg}${NC}" >&2 +} + +stdlib::error() { + local ds msg + msg="$*" + ds="$(date +"${DATE_FMT}") " + logger -p "${SYSLOG_ERROR_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + echo -e "${RED}${ds}Error [$$]: ${msg}${NC}" >&2 +} + +stdlib::announce_runners_start() { + if [ -z "$recursive_proc" ]; then + wall -n "$WARNING_MESSAGE" + echo "$WARNING_MESSAGE" >"$SCRIPT_COMPLETE_FILE" + fi + export recursive_proc=$((${recursive_proc:=0} + 1)) +} + +stdlib::announce_runners_end() { + exit_code=$1 + export recursive_proc=$((${recursive_proc:=0} - 1)) + if [ "$recursive_proc" -le "0" ]; then + if [ "$exit_code" -ne "0" ]; then + wall -n "$ERROR_MESSAGE" + echo "$ERROR_MESSAGE" >"$SCRIPT_COMPLETE_FILE" + else + wall -n "$SUCCESS_MESSAGE" + echo -n "" >"$SCRIPT_COMPLETE_FILE" + fi + fi +} + +# The main initialization function of this library. This should be kept to the +# minimum amount of work required for all functions to operate cleanly. +stdlib::init() { + if [[ ${STARTUP_SCRIPT_STDLIB_INITIALIZED} -gt 0 ]]; then + stdlib::info 'stdlib::init()'" already initialized, no action taken." + return 0 + fi + ((STARTUP_SCRIPT_STDLIB_INITIALIZED++)) || true + stdlib::init_global_vars + stdlib::init_directories + stdlib::debug "stdlib::init(): startup-script-stdlib.sh initialized and ready" +} + +# Initialize global variables. +stdlib::init_global_vars() { + # The program name, used for logging. + readonly PROG="${PROG:-startup-script-stdlib}" + # Date format used for stderr logging. Passed to date + command. + readonly DATE_FMT="${DATE_FMT:-"%a %b %d %H:%M:%S %z %Y"}" + # var directory + readonly VARDIR="${VARDIR:-/var/lib/startup}" + # Override this with file://localhost/tmp/foo/bar in spec test context + readonly METADATA_BASE="${METADATA_BASE:-http://metadata.google.internal}" + + # Color variables + if [[ -n ${COLOR:-} ]]; then + readonly NC='\033[0m' # no color + readonly RED='\033[0;31m' # error + readonly GREEN='\033[0;32m' # info + readonly BLUE='\033[0;34m' # debug + else + readonly NC='' + readonly RED='' + readonly GREEN='' + readonly BLUE='' + fi + + return 0 +} + +stdlib::init_directories() { + if ! [[ -e ${VARDIR} ]]; then + install -d -m 0755 -o 0 -g 0 "${VARDIR}" + fi +} + +## +# Get a metadata key. When used without -o, this function is guaranteed to +# produce no output on STDOUT other than the retrieved value. This is intended +# to support the use case of +# FOO="$(stdlib::metadata_get -k instance/attributes/foo)" +# +# If the requested key does not exist, the error code will be 22 and zero bytes +# written to STDOUT. +stdlib::metadata_get() { + local OPTIND opt key outfile + local metadata="${METADATA_BASE%/}/computeMetadata/v1" + local exit_code + while getopts ":k:o:" opt; do + case "${opt}" in + k) key="${OPTARG}" ;; + o) outfile="${OPTARG}" ;; + :) + stdlib::error "Invalid option: -${OPTARG} requires an argument" + stdlib::metadata_get_usage + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error "Unknown option: -${opt}" + stdlib::metadata_get_usage + return "${E_UNKNOWN_ARG}" + ;; + esac + done + local url="${metadata}/${key#/}" + + stdlib::debug "Getting metadata resource url=${url}" + if [[ -z ${outfile:-} ]]; then + curl --location --silent --connect-timeout 1 --fail \ + -H 'Metadata-Flavor: Google' "$url" 2>/dev/null + exit_code=$? + else + stdlib::cmd curl --location \ + --silent \ + --connect-timeout 1 \ + --fail \ + --output "${outfile}" \ + -H 'Metadata-Flavor: Google' \ + "$url" + exit_code=$? + fi + case "${exit_code}" in + 22 | 37) + stdlib::debug "curl exit_code=${exit_code} for url=${url}" \ + "(Does not exist)" + ;; + esac + return "${exit_code}" +} + +stdlib::metadata_get_usage() { + stdlib::info 'Usage: stdlib::metadata_get -k ' + stdlib::info 'For example: stdlib::metadata_get -k instance/attributes/startup-config' +} + +# Load configuration values in the spirit of /etc/sysconfig defaults, but from +# metadata instead of the filesystem. +stdlib::load_config_values() { + local config_file + local key="instance/attributes/startup-script-config" + # shellcheck disable=SC2119 + config_file="$(stdlib::mktemp)" + stdlib::metadata_get -k "${key}" -o "${config_file}" + local status=$? + case "$status" in + 0) + stdlib::debug "SUCCESS: Configuration data sourced from $key" + ;; + 22 | 37) + stdlib::debug "no configuration data loaded from $key" + ;; + *) + stdlib::error "metadata_get -k $key returned unknown status=${status}" + ;; + esac + # shellcheck source=/dev/null + source "${config_file}" +} + +# Run a command logging the entry and exit. Intended for system level commands +# and operational debugging. Not intended for use with redirection. This is +# not named run() because bats uses a run() function. +stdlib::cmd() { + local exit_code argv=("$@") + stdlib::debug "BEGIN: stdlib::cmd() command=[${argv[*]}]" + "${argv[@]}" + exit_code=$? + stdlib::debug "END: stdlib::cmd() command=[${argv[*]}] exit_code=${exit_code}" + return $exit_code +} + +# Run a command successfully or exit the program with an error. +stdlib::run_or_die() { + if ! stdlib::cmd "$@"; then + stdlib::error "stdlib::run_or_die(): exiting with exit code ${E_RUN_OR_DIE}." + exit "${E_RUN_OR_DIE}" + fi +} + +# Intended to take advantage of automatic cleanup of startup script library +# temporary files without exporting a modified TMPDIR to child processes, which +# would cause the children to have their TMPDIR deleted out from under them. +# shellcheck disable=SC2120 +stdlib::mktemp() { + TMPDIR="${DELETE_AT_EXIT:-${TMPDIR}}" mktemp "$@" +} + +# Return a nice error message if a mandatory argument is missing. +stdlib::mandatory_argument() { + local OPTIND opt name flag + while getopts ":n:f:" opt; do + case "$opt" in + n) name="${OPTARG}" ;; + f) flag="${OPTARG}" ;; + :) + stdlib::error "Invalid argument: -${OPTARG} requires an argument to stdlib::mandatory_argument()" + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error "Unknown argument: -${OPTARG}" + stdlib::info "Usage: stdlib::mandatory_argument -n -f " + return "${E_UNKNOWN_ARG}" + ;; + esac + done + stdlib::error "Invalid argument: -${flag} requires an argument to ${name}()." +} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/main.tf b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/main.tf new file mode 100644 index 0000000000..02124eeddc --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/main.tf @@ -0,0 +1,306 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "startup-script", ghpc_role = "scripts" }) +} + +locals { + monitoring_agent_installer = ( + var.install_cloud_ops_agent || var.install_stackdriver_agent ? + [{ + type = "shell" + source = "${path.module}/files/install_monitoring_agent.sh" + destination = "install_monitoring_agent_automatic.sh" + args = var.install_cloud_ops_agent ? "ops" : "legacy" # install legacy (stackdriver) + }] : + [] + ) + + warnings = [ + { + type = "data" + content = file("${path.module}/files/running-script-warning.sh") + destination = "/etc/profile.d/99-running-script-warning.sh" + } + ] + + configure_ssh = length(var.configure_ssh_host_patterns) > 0 + host_args = { + host_name_prefix = var.configure_ssh_host_patterns + } + + prefix_file = "/tmp/prefix_file.json" + ansible_docker_settings_file = "/tmp/ansible_docker_settings.json" + + docker_config = try(jsondecode(var.docker.daemon_config), {}) + docker_data_root = try(local.docker_config.data-root, null) + + configure_ssh_runners = local.configure_ssh ? [ + { + type = "data" + source = "${path.module}/files/setup-ssh-keys.sh" + destination = "/usr/local/ghpc/setup-ssh-keys.sh" + }, + { + type = "data" + source = "${path.module}/files/setup-ssh-keys.yml" + destination = "/usr/local/ghpc/setup-ssh-keys.yml" + }, + { + type = "data" + content = jsonencode(local.host_args) + destination = local.prefix_file + }, + { + type = "ansible-local" + content = file("${path.module}/files/configure-ssh.yml") + destination = "configure-ssh.yml" + args = "-e @${local.prefix_file}" + } + ] : [] + + proxy_runner = var.http_proxy == "" ? [] : [ + { + type = "data" + destination = "/etc/profile.d/http_proxy.sh" + content = <<-EOT + #!/bin/bash + export http_proxy=${var.http_proxy} + export https_proxy=${var.http_proxy} + export NO_PROXY=${var.http_no_proxy} + EOT + }, + { + type = "shell" + source = "${path.module}/files/configure_proxy.sh" + destination = "configure_proxy.sh" + args = var.http_proxy + } + ] + + ofi_runner = !var.set_ofi_cloud_rdma_tunables ? [] : [ + { + type = "data" + destination = "/etc/profile.d/set_ofi_cloud_rdma_tunables.sh" + content = <<-EOT + #!/bin/bash + export FI_PROVIDER="verbs;ofi_rxm" + export FI_OFI_RXM_USE_RNDV_WRITE=0 + export FI_VERBS_INLINE_SIZE=39 + export I_MPI_FABRICS="shm:ofi" + export FI_UNIVERSE_SIZE=1024 + export I_MPI_ADJUST_ALLTOALL=1 + export I_MPI_ADJUST_IALLTOALL=1 + export I_MPI_ADJUST_BCAST=4 + export I_MPI_ADJUST_IBCAST=1 + EOT + }, + ] + + rdma_runner = !var.install_cloud_rdma_drivers ? [] : [ + { + type = "shell" + source = "${path.module}/files/install_cloud_rdma_drivers.sh" + destination = "install_cloud_rdma_drivers.sh" + } + ] + + docker_runner = !var.docker.enabled ? [] : [ + { + type = "data" + destination = local.ansible_docker_settings_file + content = jsonencode({ + enable_docker_world_writable = var.docker.world_writable + docker_daemon_config = var.docker.daemon_config + docker_data_root = local.docker_data_root + }) + }, + { + type = "ansible-local" + destination = "install_docker.yml" + content = file("${path.module}/files/install_docker.yml") + args = "-e \"@${local.ansible_docker_settings_file}\"" + }, + ] + + managed_lustre_runner = !var.managed_lustre.enabled ? [] : [ + { + type = "ansible-local" + destination = "install_managed_lustre.yml" + content = file("${path.module}/files/install_managed_lustre.yml") + args = "-e managed_lustre_port=${var.managed_lustre.port}" + }, + ] + + gpu_network_wait_online_runner = !var.enable_gpu_network_wait_online ? [] : [ + { + type = "ansible-local" + destination = "install_gpu_network_wait_online.yml" + content = file("${path.module}/files/install_gpu_network_wait_online.yml") + args = "" + }, + ] + + local_ssd_filesystem_enabled = can(coalesce(var.local_ssd_filesystem.mountpoint)) + raid_setup = !local.local_ssd_filesystem_enabled ? [] : [ + { + type = "ansible-local" + destination = "setup-raid.yml" + content = file("${path.module}/files/setup-raid.yml") + args = join(" ", [ + "-e mountpoint=${var.local_ssd_filesystem.mountpoint}", + "-e fs_type=${var.local_ssd_filesystem.fs_type}", + "-e mode=${var.local_ssd_filesystem.permissions}", + ]) + }, + ] + + supplied_ansible_runners = anytrue([for r in var.runners : r.type == "ansible-local"]) + has_ansible_runners = anytrue([ + local.supplied_ansible_runners, + local.configure_ssh, + var.docker.enabled, + var.managed_lustre.enabled, + var.enable_gpu_network_wait_online, + local.local_ssd_filesystem_enabled + ]) + + install_ansible = coalesce(var.install_ansible, local.has_ansible_runners) + ansible_installer = local.install_ansible ? [{ + type = "shell" + source = "${path.module}/files/install_ansible.sh" + destination = "install_ansible_automatic.sh" + args = var.ansible_virtualenv_path + }] : [] + + hotfix_runner = [{ + type = "shell" + source = "${path.module}/files/early_run_hotfixes.sh" + destination = "early_run_hotfixes.sh" + }] + + runners = concat( + local.warnings, + local.hotfix_runner, + local.proxy_runner, + local.ofi_runner, + local.rdma_runner, + local.monitoring_agent_installer, + local.ansible_installer, + local.raid_setup, # order RAID early to ensure filesystem is ready for subsequent runners + local.managed_lustre_runner, + local.configure_ssh_runners, + local.docker_runner, + local.gpu_network_wait_online_runner, + var.runners + ) + + bucket_regex = "^gs://([^/]*)/*(.*)" + gcs_bucket_path_trimmed = var.gcs_bucket_path == null ? null : trimsuffix(var.gcs_bucket_path, "/") + storage_folder_path = local.gcs_bucket_path_trimmed == null ? null : regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[1] + storage_folder_path_prefix = local.storage_folder_path == null || local.storage_folder_path == "" ? "" : "${local.storage_folder_path}/" + + user_provided_bucket_name = try(regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[0], null) + storage_bucket_name = coalesce(one(google_storage_bucket.configs_bucket[*].name), local.user_provided_bucket_name) + + load_runners = templatefile( + "${path.module}/templates/startup-script-custom.tftpl", + { + bucket = local.storage_bucket_name, + http_proxy = var.http_proxy, + no_proxy = var.http_no_proxy, + runners = [ + for runner in local.runners : { + object = google_storage_bucket_object.scripts[basename(runner["destination"])].output_name + type = runner["type"] + destination = runner["destination"] + args = contains(keys(runner), "args") ? runner["args"] : "" + } + ] + } + ) + + stdlib_head = file("${path.module}/files/startup-script-stdlib-head.sh") + get_from_bucket = file("${path.module}/files/get_from_bucket.sh") + stdlib_body = file("${path.module}/files/startup-script-stdlib-body.sh") + + # List representing complete content, to be concatenated together. + stdlib_list = [ + local.stdlib_head, + local.get_from_bucket, + local.load_runners, + local.stdlib_body, + ] + + # Final content output to the user + stdlib = join("", local.stdlib_list) + + runners_map = { for runner in local.runners : + basename(runner["destination"]) => { + content = lookup(runner, "content", null) + source = lookup(runner, "source", null) + } + } +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket" "configs_bucket" { + count = var.gcs_bucket_path == null ? 1 : 0 + project = var.project_id + name = "${var.deployment_name}-startup-scripts-${random_id.resource_name_suffix.hex}" + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +resource "google_storage_bucket_iam_binding" "viewers" { + bucket = local.storage_bucket_name + role = "roles/storage.objectViewer" + members = var.bucket_viewers +} + +resource "google_storage_bucket_object" "scripts" { + # this writes all scripts exactly once into GCS + for_each = local.runners_map + name = "${local.storage_folder_path_prefix}${each.key}-${substr(try(md5(each.value.content), filemd5(each.value.source)), 0, 4)}" + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) + bucket = local.storage_bucket_name + timeouts { + create = "10m" + update = "10m" + } + + lifecycle { + precondition { + condition = !(var.install_cloud_ops_agent && var.install_stackdriver_agent) + error_message = "Only one of var.install_stackdriver_agent or var.install_cloud_ops_agent can be set. Stackdriver is recommended for best performance." + } + } +} + +resource "local_file" "debug_file" { + for_each = toset(var.debug_file != null ? [var.debug_file] : []) + filename = var.debug_file + content = local.stdlib +} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/metadata.yaml new file mode 100644 index 0000000000..2ada34471f --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/outputs.tf b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/outputs.tf new file mode 100644 index 0000000000..6a15082814 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/outputs.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "script to load and run all runners, as a string value." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} + +output "compute_startup_script" { + description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} + +output "controller_startup_script" { + description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl new file mode 100644 index 0000000000..3c894b00b0 --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl @@ -0,0 +1,65 @@ + + +stdlib::run_playbook() { + if [ ! "$(which ansible-playbook)" ]; then + stdlib::error "ansible-playbook not found"\ + "Please install ansible before running ansible-local runners." + exit 1 + fi + ansible-playbook --connection=local --inventory=localhost, --limit localhost $1 $2 + ret_code=$? + return $${ret_code} +} + +stdlib::runner() { + + type=$1 + object=$2 + destination=$3 + tmpdir=$4 + args=$5 + + destpath="$(dirname $destination)" + filename="$(basename $destination)" + + if [ "$destpath" = "." ]; then + destpath=$tmpdir + fi + + stdlib::get_from_bucket -u "gs://${bucket}/$object" -d "$destpath" -f "$filename" + + stdlib::info "=== start executing runner: $object ===" + case "$1" in + ansible-local) stdlib::run_playbook "$destpath/$filename" "$args";; + shell) chmod u+x /$destpath/$filename && $destpath/$filename $args;; + esac + + exit_code=$? + stdlib::info "=== $object finished with exit_code=$exit_code ===" + if [ "$exit_code" -ne "0" ] ; then + stdlib::error "=== execution of $object failed, exiting ===" + stdlib::announce_runners_end "$exit_code" + exit $exit_code + fi +} + +stdlib::load_runners(){ + tmpdir="$(mktemp -d)" + + stdlib::debug "=== BEGIN Running runners ===" + stdlib::announce_runners_start + + %{if http_proxy != "" ~} + stdlib::info "=== Setting HTTP_PROXY,HTTPS_PROXY to ${http_proxy} ===" + export http_proxy=${http_proxy} + export https_proxy=${http_proxy} + export NO_PROXY=${no_proxy} + %{endif ~} + + %{for r in runners ~} + stdlib::runner "${r.type}" "${r.object}" "${r.destination}" $${tmpdir} "${r.args}" + %{endfor ~} + + stdlib::announce_runners_end "0" + stdlib::debug "=== END Running runners ===" +} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/variables.tf b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/variables.tf new file mode 100644 index 0000000000..7080085ece --- /dev/null +++ b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/variables.tf @@ -0,0 +1,298 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used to name GCS bucket for startup scripts." + type = string +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string + default = null +} + +variable "bucket_viewers" { + description = "Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account)" + type = list(string) + default = [] + + validation { + condition = alltrue([ + for u in var.bucket_viewers : length(regexall("^(allUsers$|allAuthenticatedUsers$|user:|group:|serviceAccount:|domain:)", u)) > 0 + ]) + error_message = "Bucket viewer members must begin with user/group/serviceAccount/domain following https://cloud.google.com/iam/docs/reference/rest/v1/Policy#Binding" + } +} + +variable "debug_file" { + description = "Path to an optional local to be written with 'startup_script'." + type = string + default = null +} + +variable "labels" { + description = "Labels for the created GCS bucket. Key-value pairs." + type = map(string) +} + +variable "runners" { + description = < 0 + error_message = "The POSIX permissions for the mountpoint must be represented as a 3 or 4-digit octal" + } + + default = { + fs_type = "ext4" + mountpoint = "" + permissions = "0755" + } + + nullable = false +} + +variable "install_cloud_ops_agent" { + description = "Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true." + type = bool + default = false +} + +variable "install_stackdriver_agent" { + description = "Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance." + type = bool + default = false +} + +variable "install_ansible" { + description = "Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used." + type = bool + default = null +} + +variable "configure_ssh_host_patterns" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 4.84 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [home\_pv](#module\_home\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | +| [kubectl\_apply](#module\_kubectl\_apply) | ../../../../modules/management/kubectl-apply | n/a | +| [slurm\_key\_pv](#module\_slurm\_key\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.gke_nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | +| [filestore\_id](#input\_filestore\_id) | An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `list(string)` | n/a | yes | +| [image](#input\_image) | The image for slurm daemon | `string` | n/a | yes | +| [instance\_templates](#input\_instance\_templates) | The URLs of Instance Templates | `list(string)` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| n/a | yes | +| [node\_count\_static](#input\_node\_count\_static) | The number of static nodes in node-pool | `number` | n/a | yes | +| [node\_pool\_names](#input\_node\_pool\_names) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `list(string)` | n/a | yes | +| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used in slurm controller | `string` | n/a | yes | +| [slurm\_controller\_instance](#input\_slurm\_controller\_instance) | Slurm cluster controller instance | `any` | n/a | yes | +| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | +| [subnetwork](#input\_subnetwork) | Primary subnetwork object | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset\_name](#output\_nodeset\_name) | Name of the new Slinky nodset | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/main.tf new file mode 100644 index 0000000000..8b2f1deeac --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/main.tf @@ -0,0 +1,64 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +### GKE NodeSet +locals { + manifest_path = "${path.module}/templates/nodeset-general.yaml.tftpl" +} + +module "kubectl_apply" { + source = "../../../../modules/management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + apply_manifests = [{ + source = local.manifest_path, + template_vars = { + slurm_namespace = var.slurm_namespace, + nodeset_name = "${var.slurm_cluster_name}-${var.nodeset_name}", + nodeset_cr_name = "${var.slurm_cluster_name}-${var.nodeset_name}", + controller_name = "${var.slurm_cluster_name}-controller", + node_pool_name = var.node_pool_names[0], + node_count = var.node_count_static, + image = var.image, + home_pvc = module.home_pv.pvc_name + slurm_key_pvc = module.slurm_key_pv.pvc_name + } + }] +} + +data "google_storage_bucket" "this" { + name = var.slurm_bucket[0].name + + depends_on = [var.slurm_bucket] +} + +### Slurm NodeSet +locals { + nodeset = { + gke_nodepool = var.node_pool_names[0] + nodeset_name = var.nodeset_name + node_count_static = var.node_count_static + subnetwork = "https://www.googleapis.com/compute/v1/projects/${var.project_id}/regions/${var.subnetwork.region}/subnetworks/${var.subnetwork.name}" + instance_template = var.instance_templates[0] + } +} + +resource "google_storage_bucket_object" "gke_nodeset_config" { + bucket = data.google_storage_bucket.this.name + name = "${var.slurm_bucket_dir}/nodeset_configs/${var.nodeset_name}.yaml" + content = yamlencode(local.nodeset) +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml new file mode 100644 index 0000000000..ea2cfc221e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/output.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/output.tf new file mode 100644 index 0000000000..15970ff0b7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/output.tf @@ -0,0 +1,18 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_name" { + description = "Name of the new Slinky nodset" + value = local.nodeset.nodeset_name +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf new file mode 100644 index 0000000000..8a190c4019 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf @@ -0,0 +1,50 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + slurm_key_storage = { + server_ip = var.slurm_controller_instance.network_interface[0].network_ip + remote_mount = "/slurm/key_distribution" # defined in /community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py + client_install_runner = {} + mount_runner = {} + fs_type = "" + local_mount = "" + mount_options = "" + } +} + +module "slurm_key_pv" { + source = "../../../../modules/file-system/gke-persistent-volume" + labels = {} + capacity_gib = 1 + cluster_id = var.cluster_id + filestore_id = "projects/empty/locations/empty/instances/empty" # this does not apply since this NFS is not a filestore + namespace = var.slurm_namespace + network_storage = local.slurm_key_storage + pv_name = "slurm-key-pv" + pvc_name = "slurm-key-pvc" +} + +# Assume the var.network_storage[0] will be home and only one home pv is accepted for now. +module "home_pv" { + source = "../../../../modules/file-system/gke-persistent-volume" + labels = {} + capacity_gib = 1024 + cluster_id = var.cluster_id + filestore_id = var.filestore_id[0] + network_storage = var.network_storage[0] + namespace = var.slurm_namespace + pv_name = "home-pv" + pvc_name = "home-pvc" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl new file mode 100644 index 0000000000..a5a4a5e7ac --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl @@ -0,0 +1,203 @@ +apiVersion: slinky.slurm.net/v1alpha1 +kind: NodeSet +metadata: + annotations: + meta.helm.sh/release-name: slurm + meta.helm.sh/release-namespace: ${slurm_namespace} + labels: + app.kubernetes.io/component: compute + app.kubernetes.io/instance: slurm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: slurmd + app.kubernetes.io/part-of: slurm + app.kubernetes.io/version: "24.11" + helm.sh/chart: slurm-0.3.0 + nodeset.slinky.slurm.net/name: ${nodeset_name} + name: ${nodeset_name} + namespace: ${slurm_namespace} +spec: + clusterName: slurm + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + replicas: ${node_count} + revisionHistoryLimit: 0 + selector: + matchLabels: + app.kubernetes.io/instance: slurm + app.kubernetes.io/name: slurmd + nodeset.slinky.slurm.net/name: ${nodeset_name} + serviceName: slurm-compute + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: slurmd + labels: + app.kubernetes.io/component: compute + app.kubernetes.io/instance: slurm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: slurmd + app.kubernetes.io/part-of: slurm + app.kubernetes.io/version: "24.11" + helm.sh/chart: slurm-0.3.0 + nodeset.slinky.slurm.net/name: ${nodeset_name} + spec: + automountServiceAccountToken: false + containers: + - args: + - -g + - -- + - bash + - -c + - | + mkdir -p /usr/local/lib/slurm + ln -s /usr/lib/x86_64-linux-gnu/slurm/spank_pyxis.so /usr/local/lib/slurm/spank_pyxis.so + /usr/local/bin/entrypoint.sh -Z --conf-server ${controller_name}:6825 -N $NODE_NAME + command: + - tini + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_CPUS + value: "0" + - name: POD_MEMORY + value: "0" + image: ${image} + imagePullPolicy: IfNotPresent + name: slurmd + ports: + - containerPort: 6818 + name: slurmd + protocol: TCP + readinessProbe: + exec: + command: + - scontrol + - show + - slurmd + resources: {} + securityContext: + capabilities: + add: + - BPF + - NET_ADMIN + - SYS_ADMIN + - SYS_NICE + privileged: true + volumeMounts: + - mountPath: /etc/slurm + name: etc-slurm + - mountPath: /run + name: run + - mountPath: /var/spool/slurmd + name: slurm-spool + - mountPath: /var/log/slurm + name: slurm-log + - mountPath: /home + name: home-pvc + dnsConfig: + searches: + - ${controller_name} + hostNetwork: true + initContainers: + - command: + - tini + - -g + - -- + - bash + - -c + - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# + SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env + contains:\n# SLURM_USER - username or UID\n\nfunction init::common() {\n\tlocal + dir\n\n\tdir=/var/spool/slurmd\n\tmkdir -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" + \"$dir\"\n\tchmod -v 700 \"$dir\"\n\n\tdir=/var/spool/slurmctld\n\tmkdir + -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" \"$dir\"\n\tchmod + -v 700 \"$dir\"\n}\n\nfunction init::slurm() {\n\tSLURM_MOUNT=/mnt/slurm\n\tSLURM_DIR=/mnt/etc/slurm\n\n\t# + Workaround to ephemeral volumes not supporting securityContext\n\t# https://github.com/kubernetes/kubernetes/issues/81089\n\n\t# + Copy Slurm config files, secrets, and scripts\n\tmkdir -p \"$SLURM_DIR\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -name \"*.conf\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -name \"*.key\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" + -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind \"$${SLURM_MOUNT}\" -type + f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 | xargs + -0r cp -vt \"$${SLURM_DIR}\"\n\n\t# Set general permissions and ownership\n\tfind + \"$${SLURM_DIR}\" -type f -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind + \"$${SLURM_DIR}\" -type f -name \"*.conf\" -print0 | xargs -0r chmod -v 644\n\tfind + \"$${SLURM_DIR}\" -type f -name \"*.key\" -print0 | xargs -0r chmod -v 600\n\tfind + \"$${SLURM_DIR}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" + -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind \"$${SLURM_DIR}\" + -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 + | xargs -0r chmod -v 755\n\n\t# Inject secrets into certain config files\n\tlocal + dbd_conf=\"slurmdbd.conf\"\n\tif [[ -f \"$${SLURM_MOUNT}/$${dbd_conf}\" ]]; + then\n\t\techo \"Injecting secrets from environment into: $${dbd_conf}\"\n\t\trm + -f \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tenvsubst <\"$${SLURM_MOUNT}/$${dbd_conf}\" + >\"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" + \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchmod -v 600 \"$${SLURM_DIR}/$${dbd_conf}\"\n\tfi\n\n\t# + Display Slurm directory files\n\tls -lAF \"$${SLURM_DIR}\"\n}\n\nfunction + main() {\n\tinit::common\n\tinit::slurm\n}\nmain\n" + env: + - name: SLURM_USER + value: slurm + image: ${image} + imagePullPolicy: IfNotPresent + name: init + resources: {} + volumeMounts: + - mountPath: /mnt/slurm + name: slurm-config + - mountPath: /mnt/etc/slurm + name: etc-slurm + - command: + - tini + - -g + - -- + - bash + - -c + - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# + SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env + contains:\n# SOCKET - Named socket to read from\n\nmkdir -v -p \"$(dirname + \"$SOCKET\")\"\nrm -f \"$SOCKET\"\nif ! [ -f \"$SOCKET\" ]; then\n\tmkfifo + -m 777 \"$SOCKET\"\nfi\nwhile IFS=\"\" read data; do\n\techo $data\ndone + <\"$SOCKET\"\n" + env: + - name: SOCKET + value: /var/log/slurm/slurmd.log + image: ghcr.io/slinkyproject/sackd:24.11-ubuntu24.04 + imagePullPolicy: IfNotPresent + name: logfile + resources: {} + restartPolicy: Always + volumeMounts: + - mountPath: /var/log/slurm + name: slurm-log + nodeSelector: + cloud.google.com/gke-nodepool: ${node_pool_name} + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Equal + value: present + volumes: + - emptyDir: + medium: Memory + name: etc-slurm + - emptyDir: {} + name: run + - name: slurm-config + persistentVolumeClaim: + claimName: ${slurm_key_pvc} + - emptyDir: + medium: Memory + name: slurm-spool + - emptyDir: + medium: Memory + name: slurm-log + - name: home-pvc + persistentVolumeClaim: + claimName: ${home_pvc} + updateStrategy: + rollingUpdate: + maxUnavailable: 20% + type: RollingUpdate diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/variables.tf new file mode 100644 index 0000000000..c091a0da86 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/variables.tf @@ -0,0 +1,118 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" + type = string +} + +variable "slurm_cluster_name" { + type = string + description = "Cluster name, used in slurm controller" + + validation { + condition = var.slurm_cluster_name != null && can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) + error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." + } +} + +variable "slurm_controller_instance" { + type = any + description = "Slurm cluster controller instance" +} + +variable "image" { + description = "The image for slurm daemon" + type = string + nullable = false +} + +variable "node_pool_names" { + description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." + type = list(string) + nullable = false +} + +variable "node_count_static" { + description = "The number of static nodes in node-pool" + type = number +} + +variable "subnetwork" { + description = "Primary subnetwork object" + type = any +} + +variable "slurm_namespace" { + description = "slurm namespace for charts" + type = string + default = "slurm" +} + +variable "nodeset_name" { + description = "The nodeset name" + type = string + default = "gkenodeset" +} + +variable "slurm_bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + type = string + nullable = false +} + +variable "slurm_bucket" { + description = "GCS Bucket of Slurm cluster file storage." + type = any + nullable = true +} + +variable "instance_templates" { + description = "The URLs of Instance Templates" + type = list(string) + nullable = false +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + + validation { + condition = length(var.network_storage) == 1 && var.network_storage[0].local_mount == "/home" + error_message = "The 'network_storage' variable must contain exactly one element, and that element's 'local_mount' attribute must be \"/home\"." + } +} + +variable "filestore_id" { + description = "An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." + type = list(string) + + validation { + condition = length(var.filestore_id) == 1 + error_message = "The 'filestore_id' variable must contain exactly one element." + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/versions.tf new file mode 100644 index 0000000000..3d7237cb92 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-nodeset/v1.51.0" + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/README.md new file mode 100644 index 0000000000..2a7c363a87 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/README.md @@ -0,0 +1,39 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 4.84 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [has\_tpu](#input\_has\_tpu) | If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu. | `bool` | `false` | no | +| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | +| [partition\_name](#input\_partition\_name) | The partition name | `string` | `"gke"` | no | +| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/main.tf new file mode 100644 index 0000000000..2949fd6594 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/main.tf @@ -0,0 +1,47 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "google_storage_bucket" "this" { + name = var.slurm_bucket[0].name + + depends_on = [var.slurm_bucket] +} + +### Slurm Partition +locals { + partition_conf = { + "PowerDownOnIdle" = "NO" + "SuspendTime" = "INFINITE" + "SuspendTimeout" = var.has_tpu ? 240 : 120 + "ResumeTimeout" = var.has_tpu ? 600 : 300 + } + + partition = { + partition_name = var.partition_name + partition_conf = local.partition_conf + + partition_nodeset = [var.nodeset_name] + partition_nodeset_tpu = [] + partition_nodeset_dyn = [] + # Options + enable_job_exclusive = true + power_down_on_idle = false + } +} + +resource "google_storage_bucket_object" "parition_config" { + bucket = data.google_storage_bucket.this.name + name = "${var.slurm_bucket_dir}/partition_configs/${var.partition_name}.yaml" + content = yamlencode(local.partition) +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/metadata.yaml new file mode 100644 index 0000000000..557e1fc2ae --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/variables.tf new file mode 100644 index 0000000000..3aeed2e59a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/variables.tf @@ -0,0 +1,43 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "has_tpu" { + description = "If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu." + type = bool + default = false +} + +variable "nodeset_name" { + description = "The nodeset name" + type = string + default = "gkenodeset" +} + +variable "partition_name" { + description = "The partition name" + type = string + default = "gke" +} + +variable "slurm_bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + type = string + nullable = false +} + +variable "slurm_bucket" { + description = "GCS Bucket of Slurm cluster file storage." + type = any + nullable = true +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/versions.tf new file mode 100644 index 0000000000..aede55263c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-partition/v1.51.0" + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/README.md new file mode 100644 index 0000000000..4f65411ddf --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/README.md @@ -0,0 +1,271 @@ +## Description + +This module performs the following tasks: + +- create an instance template from which execute points will be created +- create a managed instance group ([MIG][mig]) for execute points +- create a Toolkit runner to configure the autoscaler to scale the MIG + +It is expected to be used with the [htcondor-install] and [htcondor-setup] +modules. + +[htcondor-install]: ../../scripts/htcondor-install/README.md +[htcondor-setup]: ../../scheduler/htcondor-setup/README.md +[mig]: https://cloud.google.com/compute/docs/instance-groups/ + +### Known limitations + +This module may be used multiple times in a blueprint to create sets of +execute points in an HTCondor pool. If used more than 1 time, the setting +[name_prefix](#input_name_prefix) must be set to a value that is unique across +all uses of the htcondor-execute-point module. If you do not follow this +constraint, you will likely receive an error while running `terraform apply` +similar to that shown below. + +```text +Error: Invalid value for variable + + on modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf line 136, in module "startup_script": + 136: runners = local.all_runners + ├──────────────── + │ var.runners is list of map of string with 5 elements + +All startup-script runners must have a unique destination. +``` + +### How to configure jobs to select execute points + +HTCondor access points provisioned by the Toolkit are specially configured to +honor an attribute named `RequireId` in each [Job ClassAd][jobad]. This value +must be set to the ID of a MIG created by an instance of this module. The +[htcondor-access-point] module includes a setting `var.default_mig_id` that will +set this value automatically to the MIG ID corresponding to the module's +execute points. If this setting is left unset each job must specify `+RequireId` +explicitly. In all cases, the default value can be overridden explicitly as shown +below: + +```text +universe = vanilla +executable = /bin/echo +arguments = "Hello, World!" +output = out.$(ClusterId).$(ProcId) +error = err.$(ClusterId).$(ProcId) +log = log.$(ClusterId).$(ProcId) +request_cpus = 1 +request_memory = 100MB ++RequireId = "htcondor-pool-ep-mig" +queue +``` + +[htcondor-access-point]: ../../scheduler/htcondor-access-point/README.md +[jobad]: https://htcondor.readthedocs.io/en/latest/users-manual/matchmaking-with-classads.html + +### Example + +A full example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +The following code snippet creates a pool with 2 sets of HTCondor execute +points, one using On-demand pricing and the other using Spot pricing. They use +a startup script and network created in previous steps. + +```yaml +- id: htcondor_execute_point + source: community/modules/compute/htcondor-execute-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + min_idle: 2 + +- id: htcondor_execute_point_spot + source: community/modules/compute/htcondor-execute-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + spot: true + +- id: htcondor_access + source: community/modules/scheduler/htcondor-access-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + - htcondor_execute_point + - htcondor_execute_point_spot + settings: + default_mig_id: $(htcondor_execute_point.mig_id) + enable_public_ips: true + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - access_point_ips + - access_point_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## Behavior of Managed Instance Group (MIG) + +Regional [MIGs][mig] are used to provision Execute Points. By default, VMs +will be provisioned in any of the zones available in that region, however, it +can be constrained to run in fewer zones (or a single zone) using +[var.zones](#input_zones). + +When the configuration of an Execute Point is changed, the MIG can be configured +to [replace the VM][replacement] using a "proactive" or "opportunistic" policy. +By default, the policy is set to opportunistic. In practice, this means that +Execute Points will _NOT_ be automatically replaced by Terraform when changes to +the instance template / HTCondor configuration are made. We recommend leaving +this at the default value as it will allow the HTCondor autoscaler to replace +VMs when they become idle without disrupting running jobs. + +However, if it is desired [var.update_policy](#input_update_policy) can be set +to "PROACTIVE" to enable automatic replacement. This will disrupt running jobs +and send them back to the queue. Alternatively, one can leave the setting at +the default value of "OPPORTUNISTIC" and update: + +- intentionally by issuing an update via Cloud Console or using gcloud (below) +- VMs becomes unhealthy or are otherwise automatically replaced (e.g. regular + Google Cloud maintenance) + +For example, to manually update all instances in a MIG: + +```text +gcloud compute instance-groups managed update-instances \ + <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + +## Known Issues + +When using OS Login with "external users" (outside of the Google Cloud +organization), then Docker universe jobs will fail and cause the Docker daemon +to crash. This stems from the use of POSIX user ids (uid) outside the range +supported by Docker. Please consider disabling OS Login if this atypical +situation applies. + +```yaml +vars: + # add setting below to existing deployment variables + enable_oslogin: DISABLE +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 4.0 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.0 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [execute\_point\_instance\_template](#module\_execute\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [mig](#module\_mig) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.execute_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.execute_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Disk type for template | `string` | `"pd-balanced"` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape across zones for instance group managing execute points | `string` | `"ANY"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [execute\_point\_runner](#input\_execute\_point\_runner) | A list of Toolkit runners for configuring an HTCondor execute point | `list(map(string))` | `[]` | no | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account for HTCondor execute point (e-mail format) | `string` | n/a | yes | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | HTCondor execute point VM image

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to HTConodr execute points | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor execute points | `string` | `"n2-standard-4"` | no | +| [max\_size](#input\_max\_size) | Maximum size of the HTCondor execute point pool. | `number` | `5` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor execute points | `map(string)` | `{}` | no | +| [min\_idle](#input\_min\_idle) | Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max\_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly. | `number` | `0` | no | +| [name\_prefix](#input\_name\_prefix) | Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network HTCondor execute points will join | `string` | `"default"` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HTCondor execute points will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which HTCondor execute points will be created | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork HTCondor execute points will join | `string` | `null` | no | +| [target\_size](#input\_target\_size) | Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size. | `number` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | +| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | Startup script to run at boot-time for Windows-based HTCondor execute points | `list(string)` | `[]` | no | +| [zones](#input\_zones) | Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [autoscaler\_runner](#output\_autoscaler\_runner) | Toolkit runner to configure the HTCondor autoscaler | +| [mig\_id](#output\_mig\_id) | ID of the managed instance group containing the execute points | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml new file mode 100644 index 0000000000..375ae036cd --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml @@ -0,0 +1,74 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Role + hosts: localhost + become: true + vars: + spool_dir: /var/lib/condor/spool + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + handlers: + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml new file mode 100644 index 0000000000..a85158fdfc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml @@ -0,0 +1,98 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This playbook makes the assumption that a virtual environment has been created +# with the autoscaler and its dependencies previously installed. A runner that +# does this is provided as an output of the htcondor-install module within the +# Cluster Toolkit at community/modules/scripts/htcondor-install. + +--- +- name: Configure HTCondor Autoscaler + hosts: all + vars: + python: /usr/local/htcondor/bin/python3 + autoscaler: /usr/local/htcondor/bin/autoscaler.py + systemd_override_path: /etc/systemd/system + become: true + tasks: + - name: User must supply HTCondor role + ansible.builtin.assert: + that: + - project_id is defined + - region is defined + - zone is defined + - mig_id is defined + - max_size is defined + - name: Create SystemD service for HTCondor autoscaler + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@.service" + mode: 0644 + content: | + [Unit] + Description=HTCondor Autoscaler MIG: %i + + [Service] + User=condor + Type=oneshot + ExecStart={{ python }} {{ autoscaler }} --p $PROJECT_ID --r $REGION --z $ZONE --mz --g %i --c $MAX_SIZE --i $MIN_IDLE + notify: + - Reload SystemD + - name: Create SystemD override directory for autoscaler configuration + ansible.builtin.file: + path: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d" + state: directory + owner: root + group: root + mode: 0755 + - name: Create autoscaler configuration + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d/miglimit.conf" + mode: 0644 + content: | + [Service] + Environment=PROJECT_ID={{ project_id }} + Environment=REGION={{ region }} + Environment=ZONE={{ zone }} + Environment=MAX_SIZE={{ max_size }} + Environment=MIN_IDLE={{ min_idle }} + notify: + - Reload SystemD + - name: Create SystemD timer for HTCondor autoscaler + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@.timer" + mode: 0644 + content: | + [Unit] + Description=Run HTCondor Autoscaler Periodically + + [Timer] + OnCalendar=minutely + AccuracySec=1us + RandomizedDelaySec=30 + # the directive below is ignored harmlessly on CentOS 7; this has impact + # that timing averages to 1 minute but is not precisely 1 minute; still + # useful to ensure that timers for different MIGs do not overlap + FixedRandomDelay=true + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + post_tasks: + - name: Activate HTCondor Autoscaler timer + ansible.builtin.systemd: + name: htcondor-autoscaler@{{ mig_id }}.timer + enabled: true + state: started diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf new file mode 100644 index 0000000000..7b0df94987 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf @@ -0,0 +1,218 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-execute-point", ghpc_role = "compute" }) +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + zones = coalescelist(var.zones, data.google_compute_zones.available.names) + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + windows_startup_ps1 = join("\n\n", flatten([var.windows_startup_ps1, local.execute_config_windows_startup_ps1])) + + is_windows_image = anytrue([for l in data.google_compute_image.compute_image.licenses : length(regexall("windows-cloud", l)) > 0]) + windows_startup_metadata = local.is_windows_image && local.windows_startup_ps1 != "" ? { + windows-startup-script-ps1 = local.windows_startup_ps1 + } : {} + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.windows_startup_metadata, + local.network_storage_metadata, + local.enable_oslogin, + local.disable_automatic_updates_metadata, + var.metadata + ) + + autoscaler_runner = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_configure_autoscaler.yml") + "destination" = "htcondor_configure_autoscaler_${module.mig.instance_group_manager.name}.yml" + "args" = join(" ", [ + "-e project_id=${var.project_id}", + "-e region=${var.region}", + "-e zone=${local.zones[0]}", # this value is required, but ignored by regional MIG autoscaler + "-e mig_id=${module.mig.instance_group_manager.name}", + "-e max_size=${var.max_size}", + "-e min_idle=${var.min_idle}", + ]) + } + + execute_config = templatefile("${path.module}/templates/condor_config.tftpl", { + htcondor_role = "get_htcondor_execute", + central_manager_ips = var.central_manager_ips, + guest_accelerator = local.guest_accelerator, + }) + + execute_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.execute_config.output_name}" + execute_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e htcondor_role=get_htcondor_execute", + "-e config_object=${local.execute_object}", + ]) + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.execute_point_runner, + [local.execute_runner], + ) + + execute_config_windows_startup_ps1 = templatefile( + "${path.module}/templates/download-condor-config.ps1.tftpl", + { + config_object = local.execute_object, + } + ) + + name_prefix = "${var.deployment_name}-${var.name_prefix}-ep" +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +resource "null_resource" "execute_config" { + triggers = { + config = local.execute_config + } +} + +resource "google_storage_bucket_object" "execute_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.execute_config.id), 0, 4)}" + content = local.execute_config + bucket = var.htcondor_bucket_name +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +module "execute_point_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.execute_point_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + gpu = one(local.guest_accelerator) + preemptible = var.spot + startup_script = local.is_windows_image ? null : module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.compute_image.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config +} + +module "mig" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = var.target_size + hostname = local.name_prefix + mig_name = local.name_prefix + instance_template = module.execute_point_instance_template.self_link + + health_check_name = "health-htcondor-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "SUBSTITUTE" + max_surge_fixed = length(local.zones) + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf new file mode 100644 index 0000000000..b31f40130f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "autoscaler_runner" { + value = local.autoscaler_runner + description = "Toolkit runner to configure the HTCondor autoscaler" +} + +output "mig_id" { + value = module.mig.instance_group_manager.name + description = "ID of the managed instance group containing the execute points" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl new file mode 100644 index 0000000000..c8f5ce31a8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl @@ -0,0 +1,31 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:${htcondor_role} +CONDOR_HOST = ${join(",", central_manager_ips)} + +# StartD configuration settings +%{ if length(guest_accelerator) > 0 ~} +use feature:GPUs +%{ endif ~} +use feature:PartitionableSlot +use feature:CommonCloudAttributesGoogle("-c created-by") +UPDATE_INTERVAL = 30 +TRUST_UID_DOMAIN = True +STARTER_ALLOW_RUNAS_OWNER = True +RUNBENCHMARKS = False diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl new file mode 100644 index 0000000000..19789f122e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl @@ -0,0 +1,34 @@ +# create directory for local condor_config customizations +$config_dir = 'C:\Condor\config' +if(!(test-path -PathType container -Path $config_dir)) +{ + New-Item -ItemType Directory -Path $config_dir +} + +# update local condor_config if blueprint has changed +$config_file = "$config_dir\50-ghpc-managed" +if (Test-Path -Path $config_file -PathType Leaf) +{ + $local_hash = gcloud --format="value(md5_hash)" storage hash $config_file +} +else +{ + $local_hash = "INVALID-HASH" +} + +$remote_hash = gcloud --format="value(md5_hash)" storage hash ${config_object} +if ($local_hash -cne $remote_hash) +{ + Write-Output "Updating condor configuration" + gcloud storage cp ${config_object} $config_file + if ($LASTEXITCODE -ne 0) + { + throw "Could not download HTCondor configuration; exiting startup script" + } + Restart-Service condor +} + +# ignored if service is already running; must be here to handle case where +# machine is rebooted, but configuration has previously been downloaded +# and service is disabled from automatic start +Start-Service condor diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf new file mode 100644 index 0000000000..aab8a54c2d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf @@ -0,0 +1,265 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HTCondor execute points will be created" + type = string +} + +variable "region" { + description = "The region in which HTCondor execute points will be created" + type = string +} + +variable "zones" { + description = "Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region." + type = list(string) + default = [] + nullable = false +} + +variable "distribution_policy_target_shape" { + description = "Target shape across zones for instance group managing execute points" + type = string + default = "ANY" +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to HTConodr execute points" + type = map(string) +} + +variable "machine_type" { + description = "Machine type to use for HTCondor execute points" + type = string + default = "n2-standard-4" +} + +variable "execute_point_runner" { + description = "A list of Toolkit runners for configuring an HTCondor execute point" + type = list(map(string)) + default = [] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "instance_image" { + description = <<-EOD + HTCondor execute point VM image + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "execute_point_service_account_email" { + description = "Service account for HTCondor execute point (e-mail format)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_self_link" { + description = "The self link of the network HTCondor execute points will join" + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork HTCondor execute points will join" + type = string + default = null +} + +variable "target_size" { + description = "Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size." + type = number + default = null +} + +variable "max_size" { + description = "Maximum size of the HTCondor execute point pool." + type = number + default = 5 +} + +variable "min_idle" { + description = "Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly." + type = number + default = 0 +} + +variable "metadata" { + description = "Metadata to add to HTCondor execute points" + type = map(string) + default = {} +} + +# this default is deliberately the opposite of vm-instance because of observed +# issues running HTCondor docker universe jobs with OS Login enabled and running +# jobs as a user with uid>2^31; these uids occur when users outside the GCP +# organization login to a VM and OS Login is enabled. +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "spot" { + description = "Provision VMs using discounted Spot pricing, allowing for preemption" + type = bool + default = false +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 100 +} + +variable "disk_type" { + description = "Disk type for template" + type = string + default = "pd-balanced" +} + +variable "windows_startup_ps1" { + description = "Startup script to run at boot-time for Windows-based HTCondor execute points" + type = list(string) + default = [] + nullable = false +} + +variable "central_manager_ips" { + description = "List of IP addresses of HTCondor Central Managers" + type = list(string) +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The HTCondor module supports 0 or 1 models of accelerator card on each execute point" + } +} + +variable "name_prefix" { + description = "Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module" + type = string + nullable = false + validation { + condition = length(var.name_prefix) > 0 + error_message = "var.name_prefix must be a set to a non-empty string and must also be unique across all instances of htcondor-execute-point" + } +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" + type = string + default = "OPPORTUNISTIC" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf new file mode 100644 index 0000000000..729dc3cda5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.1" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.74.0" + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/README.md new file mode 100644 index 0000000000..278207b04a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/README.md @@ -0,0 +1,45 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_group_manager.mig](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_group_manager) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [base\_instance\_name](#input\_base\_instance\_name) | Base name for the instances in the MIG | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will be used to name MIG if `var.name` is not provided | `string` | n/a | yes | +| [ghpc\_module\_id](#input\_ghpc\_module\_id) | Internal GHPC field, do not set this value | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the MIG | `map(string)` | n/a | yes | +| [name](#input\_name) | Name of the MIG. If not provided, will be generated from `var.deployment_name` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the MIG will be created | `string` | n/a | yes | +| [target\_size](#input\_target\_size) | Target number of instances in the MIG | `number` | `0` | no | +| [versions](#input\_versions) | Application versions managed by this instance group. Each version deals with a specific instance template | 
list(object({
    name              = string
    instance_template = string
    target_size = optional(object({
      fixed   = optional(number)
      percent = optional(number)
    }))
  }))
| n/a | yes | +| [wait\_for\_instances](#input\_wait\_for\_instances) | Whether to wait for all instances to be created/updated before returning | `bool` | `false` | no | +| [zone](#input\_zone) | Compute Platform zone. Required, currently only zonal MIGs are supported | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [self\_link](#output\_self\_link) | The URL of the created MIG | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/main.tf new file mode 100644 index 0000000000..0e7cf186c2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/main.tf @@ -0,0 +1,85 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "mig", ghpc_role = "compute" }) +} + +locals { + sanitized_deploy_name = try(replace(lower(var.deployment_name), "/[^a-z0-9]/", ""), null) + sanitized_module_id = try(replace(lower(var.ghpc_module_id), "/[^a-z0-9]/", ""), null) + synth_mig_name = try("${local.sanitized_deploy_name}-${local.sanitized_module_id}", null) + + mig_name = var.name == null ? local.synth_mig_name : var.name + base_instance_name = var.base_instance_name == null ? local.mig_name : var.base_instance_name +} + +resource "google_compute_instance_group_manager" "mig" { + # REQUIRED + name = local.mig_name + base_instance_name = local.base_instance_name + zone = var.zone + + dynamic "version" { + for_each = var.versions + content { + name = version.value.name + instance_template = version.value.instance_template + dynamic "target_size" { + for_each = version.value.target_size != null ? [version.value.target_size] : [] + content { + fixed = target_size.value.fixed + percent = target_size.value.percent + } + } + } + } + + # OPTIONAL + project = var.project_id + target_size = var.target_size + wait_for_instances = var.wait_for_instances + + all_instances_config { + # TODO: validate that template metadata not getting wiped out + # TODO: validate that template labels not getting wiped out + labels = local.labels + } + + # OMITTED: + # * description + # * named_port + # * list_managed_instances_results + # * target_pools - specific for Load Balancers usage + # * wait_for_instances_status + # * auto_healing_policies + # * stateful_disk + # * stateful_internal_ip + # * update_policy + # * params + + + lifecycle { + precondition { + condition = local.mig_name != null + error_message = "Could not come up with a name for the MIG, specify `var.name`" + } + + precondition { + condition = local.base_instance_name != null + error_message = "Could not come up with a base_instance_name, specify `var.base_instance_name`" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/metadata.yaml new file mode 100644 index 0000000000..97a4fa9a89 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com +ghpc: + inject_module_id: ghpc_module_id diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/outputs.tf new file mode 100644 index 0000000000..23c66a3535 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "self_link" { + description = "The URL of the created MIG" + value = google_compute_instance_group_manager.mig.self_link +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/variables.tf new file mode 100644 index 0000000000..b6c3c0e78a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/variables.tf @@ -0,0 +1,86 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Project in which the MIG will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the deployment, will be used to name MIG if `var.name` is not provided" + type = string +} + +variable "labels" { + description = "Labels to add to the MIG" + type = map(string) +} + +variable "zone" { + description = "Compute Platform zone. Required, currently only zonal MIGs are supported" + type = string +} + + +variable "versions" { + description = <<-EOD + Application versions managed by this instance group. Each version deals with a specific instance template + EOD + type = list(object({ + name = string + instance_template = string + target_size = optional(object({ + fixed = optional(number) + percent = optional(number) + })) + })) + + validation { + condition = length(var.versions) > 0 + error_message = "At least one version must be provided" + } + +} + + +variable "ghpc_module_id" { + description = "Internal GHPC field, do not set this value" + type = string + default = null +} + +variable "name" { + description = "Name of the MIG. If not provided, will be generated from `var.deployment_name`" + type = string + default = null +} + +variable "base_instance_name" { + description = "Base name for the instances in the MIG" + type = string + default = null +} + + +variable "target_size" { + description = "Target number of instances in the MIG" + type = number + default = 0 +} + +variable "wait_for_instances" { + description = "Whether to wait for all instances to be created/updated before returning" + type = bool + default = false +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/versions.tf new file mode 100644 index 0000000000..4147447b44 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = "> 5.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:mig/v1.74.0" + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/README.md new file mode 100644 index 0000000000..1dcacc57e9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/README.md @@ -0,0 +1,112 @@ +# Description + +This module creates the Vertex AI Notebook, to be used in tutorials. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage + +This is a simple usage, using the default network: + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: + name_prefix: my-bucket + local_mount: /home/jupyter/my-bucket + + - id: notebook + source: community/modules/compute/notebook + use: [bucket] + settings: + name_prefix: notebook + machine_type: n1-standard-4 + +``` + +If the user wants do specify a custom subnetwork, or specific external IP restrictions, they can use the `network_interfaces` variable, here is an example on how to use a Shared VPC Subnet with an ephemeral external IP: + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: + name_prefix: my-bucket + local_mount: /home/jupyter/my-bucket + + - id: notebook + source: community/modules/compute/notebook + use: [bucket] + settings: + name_prefix: notebook + machine_type: n1-standard-4 + network_interfaces: + - network: "projects/HOST_PROJECT_ID/global/networks/SHARED_VPC_NAME" + subnet: "projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME" + nic_type: "VIRTIO_NET" +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 5.34 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 5.34 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.mount_script](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_workbench_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/workbench_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the notebook. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name, can be provided from the google-cloud-storage module | `string` | `null` | no | +| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "tf-latest-cpu",
  "name": null,
  "project": "deeplearning-platform-release"
}
| no | +| [labels](#input\_labels) | Labels to add to the resource Key-value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | The machine type to employ | `string` | n/a | yes | +| [mount\_runner](#input\_mount\_runner) | mount content from the google-cloud-storage module | `map(string)` | n/a | yes | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces for the VM instance. Each network interface is represented by an object with the following fields:

- network: (Optional) The name of the Virtual Private Cloud (VPC) network that this VM instance is connected to.

- subnet: (Optional) The name of the subnetwork within the specified VPC that this VM instance is connected to.

- nic\_type: (Optional) The type of vNIC to be used on this interface. Possible values are: `VIRTIO_NET`, `GVNIC`.

- access\_configs: (Optional) An array of access configurations for this network interface. The access\_config object contains:
* external\_ip: (Required) An external IP address associated with this instance. Specify an unused static external IP address available to the project or leave this field undefined to use an IP from a shared ephemeral IP address pool. If you specify a static external IP address, it must live in the same region as the zone of the instance. | 
list(object({
    network  = optional(string)
    subnet   = optional(string)
    nic_type = optional(string)
    access_configs = optional(list(object({
      external_ip = optional(string)
    })))
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | ID of project in which the notebook will be created. | `string` | n/a | yes | +| [service\_account\_email](#input\_service\_account\_email) | If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account | `string` | `null` | no | +| [zone](#input\_zone) | The zone to deploy to | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/main.tf new file mode 100644 index 0000000000..cd3ce3b4ea --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/main.tf @@ -0,0 +1,96 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "notebook", ghpc_role = "compute" }) +} + +locals { + suffix = random_id.resource_name_suffix.hex + #name = "thenotebook" + name = "notebook-${var.deployment_name}-${local.suffix}" + bucket = replace(var.gcs_bucket_path, "gs://", "") + post_script_filename = "mount-${local.suffix}.sh" + + # mount_runner_args is defined in the file: cluster-toolkit/modules/file-system/cloud-storage-bucket/outputs.tf + mount_args = split(" ", var.mount_runner.args) + + unused = local.mount_args[0] + remote_mount = local.mount_args[1] + local_mount = local.mount_args[2] + fs_type = local.mount_args[3] + # These options provide a "rw" mount of the GCS bucket + mount_options = "defaults,_netdev,allow_other,implicit_dirs,gid=1000,uid=1000" + + content0 = var.mount_runner.content + content1 = replace(local.content0, "$1", local.unused) + content2 = replace(local.content1, "$2", local.remote_mount) + content3 = replace(local.content2, "$3", local.local_mount) + content4 = replace(local.content3, "$4", local.fs_type) + content5 = replace(local.content4, "$5", local.mount_options) + +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket_object" "mount_script" { + name = local.post_script_filename + content = local.content5 + bucket = local.bucket +} + +resource "google_workbench_instance" "instance" { + name = local.name + location = var.zone + project = var.project_id + labels = local.labels + gce_setup { + machine_type = var.machine_type + metadata = { + post-startup-script = "${var.gcs_bucket_path}/${google_storage_bucket_object.mount_script.name}" + } + vm_image { + project = var.instance_image.project + family = var.instance_image.family + } + + dynamic "service_accounts" { + for_each = var.service_account_email == null ? [] : [1] + content { + email = var.service_account_email + } + } + + dynamic "network_interfaces" { + for_each = var.network_interfaces + content { + network = network_interfaces.value.network + subnet = network_interfaces.value.subnet + nic_type = network_interfaces.value.nic_type + + dynamic "access_configs" { + for_each = network_interfaces.value.access_configs != null ? network_interfaces.value.access_configs : [] + content { + external_ip = access_configs.value.external_ip + } + } + } + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/metadata.yaml new file mode 100644 index 0000000000..4a7d5397ca --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - notebooks.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/variables.tf new file mode 100644 index 0000000000..4359de8c10 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/variables.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which the notebook will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment; used as part of name of the notebook." + type = string + # notebook name can have: lowercase letters, numbers, or hyphens (-) and cannot end with a hyphen + validation { + error_message = "The notebook name uses 'deployment_name' -- can only have: lowercase letters, numbers, or hyphens" + condition = can(regex("^[a-z0-9]+(?:-[a-z0-9]+)*$", var.deployment_name)) + } +} + +variable "zone" { + description = "The zone to deploy to" + type = string +} + +variable "machine_type" { + description = "The machine type to employ" + type = string +} + +variable "labels" { + description = "Labels to add to the resource Key-value pairs." + type = map(string) +} + +variable "instance_image" { + description = "Instance Image" + type = map(string) + default = { + project = "deeplearning-platform-release" + family = "tf-latest-cpu" + name = null + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "gcs_bucket_path" { + description = "Bucket name, can be provided from the google-cloud-storage module" + type = string + default = null +} + +variable "mount_runner" { + description = "mount content from the google-cloud-storage module" + type = map(string) + + validation { + condition = (length(split(" ", var.mount_runner.args)) == 5) + error_message = "There must be 5 elements in the Mount Runner Arguments: ${var.mount_runner.args} \n " + } +} + +variable "service_account_email" { + description = "If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account" + type = string + default = null +} + +variable "network_interfaces" { + type = list(object({ + network = optional(string) + subnet = optional(string) + nic_type = optional(string) + access_configs = optional(list(object({ + external_ip = optional(string) + }))) + })) + default = [] + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | +| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name    = string
    device_name  = string
    disk_size_gb = number
    disk_type    = string
    disk_labels  = map(string)
    auto_delete  = bool
    boot         = bool
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | +| [feature](#input\_feature) | The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used. | `string` | `null` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | Path to the Slurm bucket. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster. | `string` | n/a | yes | +| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_template\_self\_link](#output\_instance\_template\_self\_link) | The URI of the template. | +| [node\_name\_prefix](#output\_node\_name\_prefix) | The prefix to be used for the node names.

Make sure that nodes are named `-`
This temporary required for proper functioning of the nodes.
While Slurm scheduler uses "features" to bind node and nodeset,
the SlurmGCP relies on node names for this (to be switched to features as well). | +| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf new file mode 100644 index 0000000000..31d9f14ae7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf @@ -0,0 +1,128 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset-dynamic", ghpc_role = "compute" }) +} + +module "instance_validation" { + source = "../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + nodeset_name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + feature = coalesce(var.feature, local.nodeset_name) + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + universe_domain = { "universe_domain" = var.universe_domain } + + metadata = merge( + local.disable_automatic_updates_metadata, + local.universe_domain, + { slurmd_feature = local.feature }, + var.metadata + ) + + nodeset = { + nodeset_name = local.nodeset_name + nodeset_feature : local.feature + startup_script = local.ghpc_startup_script + network_storage = var.network_storage + } + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + } + ] + + public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + ghpc_startup_script = [{ + filename = "ghpc_nodeset_startup.sh" + content = var.startup_script + }] + +} + +module "slurm_nodeset_template" { + source = "../../internal/slurm-gcp/instance_template" + + project_id = var.project_id + region = var.region + name_prefix = local.nodeset_name + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = "compute" + slurm_bucket_path = var.slurm_bucket_path + metadata = local.metadata + + additional_disks = local.additional_disks + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(local.labels, var.disk_labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + + bandwidth_tier = var.bandwidth_tier + can_ip_forward = var.can_ip_forward + + advanced_machine_features = var.advanced_machine_features + enable_confidential_vm = var.enable_confidential_vm + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + labels = local.labels + machine_type = var.machine_type + + min_cpu_platform = var.min_cpu_platform + on_host_maintenance = var.on_host_maintenance + termination_action = try(var.spot_instance_config.termination_action, null) + preemptible = var.preemptible + spot = var.enable_spot_vm + service_account = local.service_account + gpu = one(local.guest_accelerator) # requires gpu_definition.tf + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + subnetwork = var.subnetwork_self_link + additional_networks = var.additional_networks + access_config = local.access_config + tags = concat([var.slurm_cluster_name], var.tags) +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml new file mode 100644 index 0000000000..a99e59d09f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [compute.googleapis.com] +ghpc: + inject_module_id: name diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf new file mode 100644 index 0000000000..2d2d1415cf --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_dyn" { + description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset +} + +output "instance_template_self_link" { + description = "The URI of the template." + value = module.slurm_nodeset_template.self_link +} + +output "node_name_prefix" { + description = <<-EOD + The prefix to be used for the node names. + + Make sure that nodes are named `-` + This temporary required for proper functioning of the nodes. + While Slurm scheduler uses "features" to bind node and nodeset, + the SlurmGCP relies on node names for this (to be switched to features as well). + EOD + value = "${var.slurm_cluster_name}-${local.nodeset_name}" + +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf new file mode 100644 index 0000000000..ec6206e317 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf @@ -0,0 +1,402 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "feature" { + type = string + description = "The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used." + default = null +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "slurm_cluster_name" { + description = "Name of the Slurm cluster." + type = string +} + +variable "slurm_bucket_path" { + description = "Path to the Slurm bucket." + type = string +} + + +variable "machine_type" { + description = "Compute Platform machine type to use for this partition compute nodes." + type = string + default = "c2-standard-60" +} + +variable "metadata" { + type = map(string) + description = "Metadata, provided as a map." + default = {} +} + +variable "instance_image" { + description = <<-EOD + Defines the image that will be used in the Slurm node group VM instances. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + + For more information on creating custom images that comply with Slurm on GCP + see the "Slurm on GCP Custom Images" section in docs/vm-images.md. + EOD + type = map(string) + default = { + family = "slurm-gcp-6-11-hpc-rocky-linux-8" + project = "schedmd-slurm-public" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations + description = <<-EOD + A flag that designates that the user is aware that they are requesting + to use a custom and potentially incompatible image for this Slurm on + GCP module. + + If the field is set to false, only the compatible families and project + names will be accepted. The deployment will fail with any other image + family or name. If set to true, no checks will be done. + + See: https://goo.gle/hpc-slurm-images + EOD + type = bool + default = false +} + + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "tags" { + type = list(string) + description = "Network tag list." + default = [] +} + +variable "disk_type" { + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + type = string + default = "pd-standard" +} + +variable "disk_size_gb" { + description = "Size of boot disk to create for the partition compute nodes." + type = number + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "additional_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = list(object({ + disk_name = string + device_name = string + disk_size_gb = number + disk_type = string + disk_labels = map(string) + auto_delete = bool + boot = bool + })) + default = [] +} + +variable "enable_confidential_vm" { + type = bool + description = "Enable the Confidential VM configuration. Note: the instance image must support option." + default = false +} + +variable "enable_shielded_vm" { + type = bool + description = "Enable the Shielded VM configuration. Note: the instance image must support option." + default = false +} + +variable "shielded_instance_config" { + type = object({ + enable_integrity_monitoring = bool + enable_secure_boot = bool + enable_vtpm = bool + }) + description = <<-EOD + Shielded VM configuration for the instance. Note: not used unless + enable_shielded_vm is 'true'. + - enable_integrity_monitoring : Compare the most recent boot measurements to the + integrity policy baseline and return a pair of pass/fail results depending on + whether they match or not. + - enable_secure_boot : Verify the digital signature of all boot components, and + halt the boot process if signature verification fails. + - enable_vtpm : Use a virtualized trusted platform module, which is a + specialized computer chip you can use to encrypt objects like keys and + certificates. + EOD + default = { + enable_integrity_monitoring = true + enable_secure_boot = true + enable_vtpm = true + } +} + + +variable "enable_oslogin" { + type = bool + description = <<-EOD + Enables Google Cloud os-login for user login and authentication for VMs. + See https://cloud.google.com/compute/docs/oslogin + EOD + default = true +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example." + type = bool + default = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "labels" { + description = "Labels to add to partition compute instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "The name of the minimum CPU platform that you want the instance to use." + type = string + default = null +} + +variable "on_host_maintenance" { + type = string + description = <<-EOD + Instance availability Policy. + + Note: Placement groups are not supported when on_host_maintenance is set to + "MIGRATE" and will be deactivated regardless of the value of + enable_placement. To support enable_placement, ensure on_host_maintenance is + set to "TERMINATE". + EOD + default = "TERMINATE" +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." + } +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + + +variable "service_account_email" { + description = "Service account e-mail address to attach to the compute instances." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the compute instances." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "enable_spot_vm" { + description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." + type = bool + default = false +} + +variable "spot_instance_config" { + description = "Configuration for spot VMs." + type = object({ + termination_action = string + }) + default = null +} + +variable "bandwidth_tier" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | +| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | +| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | +| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `null` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling worker nodes allowed in this partition.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of worker nodes to be statically created.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | +| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. | `string` | `""` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the TPU-vm. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the TPU-vm. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | +| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | `"2.14.0"` | no | +| [zone](#input\_zone) | Zone in which to create compute VMs. TPU partitions can only specify a single zone. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf new file mode 100644 index 0000000000..ac9b119702 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# locals { +# # This label allows for billing report tracking based on module. +# labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) +# } + +locals { + name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + nodeset_tpu = { + node_count_static = var.node_count_static + node_count_dynamic_max = var.node_count_dynamic_max + nodeset_name = local.name + node_type = var.node_type + + accelerator_config = var.accelerator_config + tf_version = var.tf_version + preemptible = var.preemptible + preserve_tpu = var.preserve_tpu + + data_disks = var.data_disks + docker_image = var.docker_image + + enable_public_ip = var.enable_public_ips + # TODO: rename to subnetwork_self_link, requires changes to the scripts + subnetwork = var.subnetwork_self_link + service_account = local.service_account + zone = var.zone + + project_id = var.project_id + reserved = var.reserved + network_storage = var.network_storage + } + + node_type_core_count = var.node_type == "" ? 0 : tonumber(regex("-(.*)", var.node_type)[0]) + + accelerator_core_list = var.accelerator_config.topology == "" ? [0, 0] : regexall("\\d+", var.accelerator_config.topology) + accelerator_core_count = length(local.accelerator_core_list) > 2 ? (local.accelerator_core_list[0] * local.accelerator_core_list[1] * local.accelerator_core_list[2]) * 2 : (local.accelerator_core_list[0] * local.accelerator_core_list[1]) * 2 + + tpu_core_count = local.accelerator_core_count == 0 ? local.node_type_core_count : local.accelerator_core_count +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml new file mode 100644 index 0000000000..95b6d1c730 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name + has_to_be_used: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf new file mode 100644 index 0000000000..8cb7b8663e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf @@ -0,0 +1,39 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_tpu" { + description = "Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset_tpu + + precondition { + condition = (var.node_type == "") != (var.accelerator_config == { topology : "", version : "" }) + error_message = "Either a node_type or an accelerator_config must be provided." + } + + precondition { + condition = ((local.tpu_core_count / 8) <= var.node_count_dynamic_max) || ((local.tpu_core_count / 8) <= var.node_count_static) + error_message = <<-EOD + When using TPUs there should be at least one node per every 8 cores. + Currently there are ${local.tpu_core_count} cores but only ${var.node_count_static} static nodes and ${var.node_count_dynamic_max} dynamic nodes. + EOD + } + + precondition { + condition = (var.node_count_dynamic_max % (local.tpu_core_count / 8) == 0) && (var.node_count_static % (local.tpu_core_count / 8) == 0) + error_message = <<-EOD + The number of worker nodes should be a multiple of ${local.tpu_core_count / 8}. + This is to ensure each node has a TPU machine for job scheduling. + EOD + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf new file mode 100644 index 0000000000..367b0bee09 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf @@ -0,0 +1,171 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "node_count_static" { + description = <<-EOD + Number of worker nodes to be statically created. + For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). + See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. + EOD + type = number + default = 0 +} + +variable "node_count_dynamic_max" { + description = <<-EOD + Maximum number of auto-scaling worker nodes allowed in this partition. + For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). + See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. + EOD + type = number + default = 0 +} + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "enable_public_ips" { + description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." + type = bool + default = false +} + +variable "disable_public_ips" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `enable_public_ips` instead." + type = bool + default = null + validation { + condition = var.disable_public_ips == null + error_message = "DEPRECATED: Use `enable_public_ips` instead." + } +} + +variable "node_type" { + description = "Specify a node type to base the vm configuration upon it." + type = string + default = "" +} + +variable "accelerator_config" { + description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." + type = object({ + topology = string + version = string + }) + default = { + topology = "" + version = "" + } + validation { + condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], var.accelerator_config.version) + error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" + } + validation { + condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) + error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." + } +} + +variable "tf_version" { + description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." + type = string + default = "2.14.0" +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + +variable "preserve_tpu" { + description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" + type = bool + default = false +} + +variable "zone" { + description = "Zone in which to create compute VMs. TPU partitions can only specify a single zone." + type = string +} + +variable "data_disks" { + description = "The data disks to include in the TPU node" + type = list(string) + default = [] +} + +variable "docker_image" { + description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" + type = string + default = null +} + +variable "subnetwork_self_link" { + type = string + description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." +} + +variable "service_account_email" { + description = "Service account e-mail address to attach to the TPU-vm." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the TPU-vm." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "service_account" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + } +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "reserved" { + description = "Specify whether TPU-vms in this nodeset are created under a reservation." + type = bool + default = false +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + })) + default = [] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf new file mode 100644 index 0000000000..398eeffdda --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.3" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.74.0" + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md new file mode 100644 index 0000000000..7c9e32debf --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md @@ -0,0 +1,227 @@ +## Description + +This module creates a nodeset data structure intended to be input to the +[schedmd-slurm-gcp-v6-partition](../schedmd-slurm-gcp-v6-partition/) module. + +Nodesets allow adding heterogeneous node types to a partition, and hence +running jobs that mix multiple node characteristics. See the [heterogeneous jobs +section][hetjobs] of the SchedMD documentation for more information. + +To specify nodes from a specific nodesets in a partition, the [`--nodelist`] +(or `-w`) flag can be used, for example: + +```bash +srun -N 3 -p compute --nodelist cluster-compute-group-[0-2] hostname +``` + +Where the 3 nodes will be selected from the nodes `cluster-compute-group-[0-2]` +in the compute partition. + +Additionally, depending on how the nodes differ, a constraint can be added via +the [`--constraint`] (or `-C`) flag or other flags such as `--mincpus` can be +used to specify nodes with the desired characteristics. + +[`--nodelist`]: https://slurm.schedmd.com/srun.html#OPT_nodelist +[`--constraint`]: https://slurm.schedmd.com/srun.html#OPT_constraint +[hetjobs]: https://slurm.schedmd.com/heterogeneous_jobs.html + +### Example + +The following code snippet creates a partition module using the `nodeset` +module as input with: + +* a max node count of 200 +* VM machine type of `c2-standard-30` +* partition name of "compute" +* default nodeset name of "ghpc" +* connected to the `network` module via `use` +* nodes mounted to homefs via `use` + +```yaml +- id: nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + node_count_dynamic_max: 200 + machine_type: c2-standard-30 + +- id: compute_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + use: + - homefs + - nodeset + settings: + partition_name: compute +``` + +## Custom Images + +For more information on creating valid custom images for the node group VM +instances or for custom instance templates, see our [vm-images.md] documentation +page. + +[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images + +## GPU Support + +More information on GPU support in Slurm on GCP and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) + +### Compute VM Zone Policies + +The Slurm on GCP nodeset module allows you to specify additional zones in +which to create VMs through [bulk creation][bulk]. This is valuable when +configuring partitions with popular VM families and you desire access to +more compute resources across zones. + +[bulk]: https://cloud.google.com/compute/docs/instances/multiple/about-bulk-creation +[networkpricing]: https://cloud.google.com/vpc/network-pricing + +> **_WARNING:_** Lenient zone policies can lead to additional egress costs when +> moving large amounts of data between zones in the same region. For example, +> traffic between VMs and traffic from VMs to shared filesystems such as +> Filestore. For more information on egress fees, see the +> [Network Pricing][networkpricing] Google Cloud documentation. +> +> To avoid egress charges, ensure your compute nodes are created in a single +> zone by setting var.zone and leaving var.zones to its default value of the +> empty list. +> +> **_NOTE:_** If a new zone is added to the region while the cluster is active, +> nodes in the partition may be created in that zone. In this case, the +> partition may need to be redeployed to ensure the newly added zone is denied. + +In the zonal example below, the nodeset's zone implicitly defaults to the +deployment variable `vars.zone`: + +```yaml +vars: + zone: us-central1-f + +- id: zonal-nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset +``` + +In the example below, we enable creation in additional zones: + +```yaml +vars: + zone: us-central1-f + +- id: multi-zonal-nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + settings: + zones: + - us-central1-a + - us-central1-b +``` + +## Support +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4 | +| [google](#requirement\_google) | >= 5.11 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 5.11 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.machine_type_zone_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [google_compute_machine_types.machine_types_by_zone](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | +| [google_compute_reservation.reservation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_topology](#input\_accelerator\_topology) | Specifies the shape of the Accelerator (GPU/TPU) slice. | `string` | `null` | no | +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = optional(string)
    subnetwork         = string
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    stack_type         = optional(string)
    queue_count        = optional(number)
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | +| [dws\_flex](#input\_dws\_flex) | If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes.
See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler
Options:
- enable: Enable DWS Flex Start
- max\_run\_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week).
- use\_job\_duration: Use the job duration to determine the max\_run\_duration, if job duration is not set, max\_run\_duration will be used.
- use\_bulk\_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances

Limitations:
- CAN NOT be used with reservations;
- CAN NOT be used with placement groups; | 
object({
    enabled          = optional(bool, true)
    max_run_duration = optional(number, 604800) # one week
    use_job_duration = optional(bool, false)
    use_bulk_insert  = optional(bool, false)
  })
| 
{
  "enabled": false
}
| no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_maintenance\_reservation](#input\_enable\_maintenance\_reservation) | Enables slurm reservation for scheduled maintenance. | `bool` | `false` | no | +| [enable\_opportunistic\_maintenance](#input\_enable\_opportunistic\_maintenance) | On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_placement](#input\_enable\_placement) | Use placement policy for VMs in this nodeset.
See: https://cloud.google.com/compute/docs/instances/placement-policies-overview
To set max\_distance of used policy, use `placement_max_distance` variable.

Enabled by default, reasons for users to disable it:
- If non-dense reservation is used, user can avoid extra-cost of creating placement policies;
- If user wants to avoid "all or nothing" VM provisioning behaviour;
- If user wants to intentionally have "spread" VMs (e.g. for reliability reasons) | `bool` | `true` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | +| [future\_reservation](#input\_future\_reservation) | If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT\_ID/zones/ZONE/futureReservations/FUTURE\_RESERVATION\_NAME'.
See https://cloud.google.com/compute/docs/instances/future-reservations-overview | `string` | `""` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_properties](#input\_instance\_properties) | Override the instance properties. Used to test features not supported by Slurm GCP,
recommended for advanced usage only.
See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert
If any sub-field (e.g. scheduling) is set, it will override the values computed by
SlurmGCP and ignoring values of provided vars. | `any` | `null` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for compute nodes. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | +| [maintenance\_interval](#input\_maintenance\_interval) | Sets the maintenance interval for instances in this nodeset.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. | `string` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_conf](#input\_node\_conf) | Map of Slurm node line configuration. | `map(any)` | `{}` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling nodes allowed in this partition. | `number` | `10` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | +| [placement\_max\_distance](#input\_placement\_max\_distance) | Maximum distance between nodes in the placement group. Requires enable\_placement to be true. Values must be supported by the chosen machine type. | `number` | `null` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | +| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]
- RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]

Must be a "SPECIFIC" reservation
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [zone](#input\_zone) | Zone in which to create compute VMs. Additional zones in the same region can be specified in var.zones. | `string` | n/a | yes | +| [zone\_target\_shape](#input\_zone\_target\_shape) | Strategy for distributing VMs across zones in a region.
ANY
GCE picks zones for creating VM instances to fulfill the requested number of VMs
within present resource constraints and to maximize utilization of unused zonal
reservations.
ANY\_SINGLE\_ZONE (default)
GCE always selects a single zone for all the VMs, optimizing for resource quotas,
available reservations and general capacity.
BALANCED
GCE prioritizes acquisition of resources, scheduling VMs in zones where resources
are available while distributing VMs as evenly as possible across allowed zones
to minimize the impact of zonal failure. | `string` | `"ANY_SINGLE_ZONE"` | no | +| [zones](#input\_zones) | Additional zones in which to allow creation of partition nodes. Google Cloud
will find zone based on availability, quota and reservations.
Should not be set if SPECIFIC reservation is used. | `set(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf new file mode 100644 index 0000000000..da6aae33ee --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf @@ -0,0 +1,232 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) +} + +module "instance_validation" { + source = "../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata + ) + + name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.disk_resource_manager_tags + } + ] + + public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + ghpc_startup_script = [{ + filename = "ghpc_nodeset_startup.sh" + content = var.startup_script + }] + + termination_action = (var.dws_flex.enabled && !var.dws_flex.use_bulk_insert) ? "DELETE" : try(var.spot_instance_config.termination_action, null) + + nodeset = { + node_count_static = var.node_count_static + node_count_dynamic_max = var.node_count_dynamic_max + node_conf = var.node_conf + nodeset_name = local.name + dws_flex = var.dws_flex + + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(local.labels, var.disk_labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + + bandwidth_tier = var.bandwidth_tier + can_ip_forward = var.can_ip_forward + + enable_confidential_vm = var.enable_confidential_vm + enable_placement = var.enable_placement + placement_max_distance = var.placement_max_distance + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + gpu = one(local.guest_accelerator) + accelerator_topology = var.accelerator_topology + + labels = local.labels + machine_type = terraform_data.machine_type_zone_validation.output + advanced_machine_features = var.advanced_machine_features + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + region = var.region + resource_manager_tags = var.resource_manager_tags + service_account = local.service_account + shielded_instance_config = var.shielded_instance_config + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + subnetwork_self_link = var.subnetwork_self_link + additional_networks = var.additional_networks + access_config = local.access_config + tags = var.tags + spot = var.enable_spot_vm + termination_action = local.termination_action + reservation_name = local.reservation_name + future_reservation = local.future_reservation + maintenance_interval = var.maintenance_interval + instance_properties_json = jsonencode(var.instance_properties) + + zone_target_shape = var.zone_target_shape + zone_policy_allow = local.zones + zone_policy_deny = local.zones_deny + + startup_script = local.ghpc_startup_script + network_storage = var.network_storage + + enable_maintenance_reservation = var.enable_maintenance_reservation + enable_opportunistic_maintenance = var.enable_opportunistic_maintenance + } +} + +locals { + zones = setunion(var.zones, [var.zone]) + zones_deny = setsubtract(data.google_compute_zones.available.names, local.zones) +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region + + lifecycle { + postcondition { + condition = length(setsubtract(local.zones, self.names)) == 0 + error_message = <<-EOD + Invalid zones=${jsonencode(setsubtract(local.zones, self.names))} + Available zones=${jsonencode(self.names)} + EOD + } + } +} + +locals { + res_match = regex("^(?P(?Pprojects/(?P[a-z0-9-]+)/reservations/)?(?P[a-z0-9-]+)(?P/reservationBlocks/[a-z0-9-]+)?)?$", var.reservation_name) + + res_short_name = local.res_match.name + res_project = coalesce(local.res_match.project, var.project_id) + res_prefix = coalesce(local.res_match.prefix, "projects/${local.res_project}/reservations/") + res_suffix = local.res_match.suffix == null ? "" : local.res_match.suffix + + reservation_name = local.res_match.whole == null ? "" : "${local.res_prefix}${local.res_short_name}${local.res_suffix}" +} + +locals { + fr_match = regex("^(?Pprojects/(?P[a-z0-9-]+)/zones/(?P[a-z0-9-]+)/futureReservations/)?(?P[a-z0-9-]+)?$", var.future_reservation) + + fr_name = local.fr_match.name + fr_project = coalesce(local.fr_match.project, var.project_id) + fr_zone = coalesce(local.fr_match.zone, var.zone) + + future_reservation = var.future_reservation == "" ? "" : "projects/${local.fr_project}/zones/${local.fr_zone}/futureReservations/${local.fr_name}" +} + + +# tflint-ignore: terraform_unused_declarations +data "google_compute_reservation" "reservation" { + count = length(local.reservation_name) > 0 ? 1 : 0 + + name = local.res_short_name + project = local.res_project + zone = var.zone + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "Couldn't find the reservation ${var.reservation_name}" + } + + postcondition { + condition = coalesce(self.specific_reservation_required, true) + error_message = < 0] +} + +resource "terraform_data" "machine_type_zone_validation" { + input = var.machine_type + lifecycle { + precondition { + condition = length(local.zones_with_machine_type) > 0 + error_message = <<-EOT + machine type ${var.machine_type} is not available in any of the zones ${jsonencode(local.zones)}". To list zones in which it is available, run: + + gcloud compute machine-types list --filter="name=${var.machine_type}" + EOT + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml new file mode 100644 index 0000000000..95b6d1c730 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name + has_to_be_used: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf new file mode 100644 index 0000000000..18ed74e2d5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf @@ -0,0 +1,112 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset" { + description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset + + precondition { + condition = !contains([ + "c3-:pd-standard", + "h3-:pd-standard", + "h3-:pd-ssd", + ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") + error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." + } + + precondition { + condition = var.reservation_name == "" || length(var.zones) == 0 + error_message = <<-EOD + If a reservation is specified, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.accelerator_topology == null || var.enable_placement + error_message = "accelerator_topology requires enable_placement to be set to true." + } + + precondition { + condition = (var.accelerator_topology == null) || try(tonumber(split("x", var.accelerator_topology)[1]) % local.guest_accelerator[0].count == 0, false) + error_message = "accelerator_topology must be divisible by number of gpus in machine." + } + + precondition { + condition = var.placement_max_distance == null || var.enable_placement + error_message = "placement_max_distance requires enable_placement to be set to true." + } + + precondition { + condition = !(startswith(var.machine_type, "a3-") && var.placement_max_distance == 1) + error_message = "A3 machines do not support a placement_max_distance of 1." + } + + precondition { + condition = var.reservation_name == "" || !var.dws_flex.enabled + error_message = "Cannot use reservations with DWS Flex." + } + + precondition { + condition = !var.enable_placement || !var.dws_flex.enabled + error_message = "Cannot use DWS Flex with `enable_placement`." + } + + precondition { + condition = length(var.zones) == 0 || !var.dws_flex.enabled + error_message = <<-EOD + If a DWS Flex is enabled, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.on_host_maintenance == "TERMINATE" || !var.dws_flex.enabled + error_message = "If DWS Flex is used, `on_host_maintenance` should be set to 'TERMINATE'" + } + + precondition { + condition = !var.enable_spot_vm || !var.dws_flex.enabled + error_message = "Cannot use both Flex-Start and Spot VMs for provisioning." + } + + precondition { + condition = var.reservation_name == "" || var.future_reservation == "" + error_message = "Cannot use reservations and future reservations in the same nodeset" + } + + precondition { + condition = !var.enable_placement || var.future_reservation == "" + error_message = "Cannot use `enable_placement` with future reservations." + } + + precondition { + condition = var.future_reservation == "" || length(var.zones) == 0 + error_message = <<-EOD + If a future reservation is specified, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.future_reservation == "" || local.fr_zone == var.zone + error_message = <<-EOD + The zone of the deployment must match that of the future reservation + EOD + } + + precondition { + condition = var.node_count_dynamic_max > 0 || var.node_count_static > 0 + error_message = <<-EOD + This nodeset contains zero nodes, there should be at least one static or dynamic node + EOD + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf new file mode 100644 index 0000000000..06ef5aac6f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf @@ -0,0 +1,641 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "node_conf" { + description = "Map of Slurm node line configuration." + type = map(any) + default = {} + validation { + condition = lookup(var.node_conf, "Sockets", null) == null + error_message = <<-EOD + `Sockets` field is in conflict with `SocketsPerBoard` which is automatically generated by SlurmGCP. + Instead, you can override the following fields: `Boards`, `SocketsPerBoard`, `CoresPerSocket`, and `ThreadsPerCore`. + See: https://slurm.schedmd.com/slurm.conf.html#OPT_Boards and https://slurm.schedmd.com/slurm.conf.html#OPT_Sockets_1 + EOD + } +} + +variable "node_count_static" { + description = "Number of nodes to be statically created." + type = number + default = 0 +} + +variable "node_count_dynamic_max" { + description = "Maximum number of auto-scaling nodes allowed in this partition." + type = number + default = 10 +} + +## VM Definition +variable "instance_template" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Instance template can not be specified for compute nodes." + type = string + default = null + validation { + condition = var.instance_template == null + error_message = "DEPRECATED: Instance template can not be specified for compute nodes." + } +} + +variable "machine_type" { + description = "Compute Platform machine type to use for this partition compute nodes." + type = string + default = "c2-standard-60" +} + +variable "metadata" { + type = map(string) + description = "Metadata, provided as a map." + default = {} +} + +variable "instance_image" { + description = <<-EOD + Defines the image that will be used in the Slurm node group VM instances. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + + For more information on creating custom images that comply with Slurm on GCP + see the "Slurm on GCP Custom Images" section in docs/vm-images.md. + EOD + type = map(string) + default = { + family = "slurm-gcp-6-11-hpc-rocky-linux-8" + project = "schedmd-slurm-public" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations + description = <<-EOD + A flag that designates that the user is aware that they are requesting + to use a custom and potentially incompatible image for this Slurm on + GCP module. + + If the field is set to false, only the compatible families and project + names will be accepted. The deployment will fail with any other image + family or name. If set to true, no checks will be done. + + See: https://goo.gle/hpc-slurm-images + EOD + type = bool + default = false +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "tags" { + type = list(string) + description = "Network tag list." + default = [] +} + +variable "disk_type" { + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + type = string + default = "pd-standard" +} + +variable "disk_size_gb" { + description = "Size of boot disk to create for the partition compute nodes." + type = number + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string)) + auto_delete = optional(bool) + boot = optional(bool) + disk_resource_manager_tags = optional(map(string)) + })) + default = [] +} + +variable "enable_confidential_vm" { + type = bool + description = "Enable the Confidential VM configuration. Note: the instance image must support option." + default = false +} + +variable "enable_shielded_vm" { + type = bool + description = "Enable the Shielded VM configuration. Note: the instance image must support option." + default = false +} + +variable "shielded_instance_config" { + type = object({ + enable_integrity_monitoring = bool + enable_secure_boot = bool + enable_vtpm = bool + }) + description = <<-EOD + Shielded VM configuration for the instance. Note: not used unless + enable_shielded_vm is 'true'. + - enable_integrity_monitoring : Compare the most recent boot measurements to the + integrity policy baseline and return a pair of pass/fail results depending on + whether they match or not. + - enable_secure_boot : Verify the digital signature of all boot components, and + halt the boot process if signature verification fails. + - enable_vtpm : Use a virtualized trusted platform module, which is a + specialized computer chip you can use to encrypt objects like keys and + certificates. + EOD + default = { + enable_integrity_monitoring = true + enable_secure_boot = true + enable_vtpm = true + } +} + + +variable "enable_oslogin" { + type = bool + description = <<-EOD + Enables Google Cloud os-login for user login and authentication for VMs. + See https://cloud.google.com/compute/docs/oslogin + EOD + default = true +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example." + type = bool + default = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "labels" { + description = "Labels to add to partition compute instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "The name of the minimum CPU platform that you want the instance to use." + type = string + default = null +} + +variable "on_host_maintenance" { + type = string + description = <<-EOD + Instance availability Policy. + + Note: Placement groups are not supported when on_host_maintenance is set to + "MIGRATE" and will be deactivated regardless of the value of + enable_placement. To support enable_placement, ensure on_host_maintenance is + set to "TERMINATE". + EOD + default = "TERMINATE" +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." + } +} + +variable "accelerator_topology" { + type = string + description = "Specifies the shape of the Accelerator (GPU/TPU) slice." + nullable = true + default = null +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + + +variable "service_account_email" { + description = "Service account e-mail address to attach to the compute instances." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the compute instances." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "service_account" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + } +} + +variable "enable_spot_vm" { + description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." + type = bool + default = false +} + +variable "spot_instance_config" { + description = "Configuration for spot VMs." + type = object({ + termination_action = string + }) + default = null +} + +variable "bandwidth_tier" { + description = < 0 + error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME[/reservationBlocks/BLOCK_ID]', [...] are optional parts." + } +} + +variable "future_reservation" { + description = <<-EOD + If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zones/ZONE/futureReservations/FUTURE_RESERVATION_NAME'. + See https://cloud.google.com/compute/docs/instances/future-reservations-overview + EOD + type = string + default = "" + nullable = false + + validation { + condition = length(regexall("^(projects/([a-z0-9-]+)/zones/([a-z0-9-]+)/futureReservations/([a-z0-9-]+))?$", var.future_reservation)) > 0 || length(regexall("^([a-z0-9-]+)$", var.future_reservation)) > 0 + error_message = "Future reservation must be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zone/ZONE/futureReservations/FUTURE_RESERVATION_NAME'." + } +} + +variable "maintenance_interval" { + description = <<-EOD + Sets the maintenance interval for instances in this nodeset. + See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. + EOD + type = string + default = null +} + +variable "startup_script" { + description = "Startup script used by VMs in this nodeset" + type = string + default = "# no-op" +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + })) + default = [] +} + + +variable "instance_properties" { + description = <<-EOD + Override the instance properties. Used to test features not supported by Slurm GCP, + recommended for advanced usage only. + See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert + If any sub-field (e.g. scheduling) is set, it will override the values computed by + SlurmGCP and ignoring values of provided vars. + EOD + type = any + default = null +} + + +variable "enable_maintenance_reservation" { + type = bool + description = "Enables slurm reservation for scheduled maintenance." + default = false +} + + +variable "enable_opportunistic_maintenance" { + type = bool + description = "On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle." + default = false +} + + +variable "dws_flex" { + description = <<-EOD + If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes. + See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler + Options: + - enable: Enable DWS Flex Start + - max_run_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week). + - use_job_duration: Use the job duration to determine the max_run_duration, if job duration is not set, max_run_duration will be used. + - use_bulk_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances + + Limitations: + - CAN NOT be used with reservations; + - CAN NOT be used with placement groups; + + EOD + + type = object({ + enabled = optional(bool, true) + max_run_duration = optional(number, 604800) # one week + use_job_duration = optional(bool, false) + use_bulk_insert = optional(bool, false) + }) + default = { + enabled = false + } + validation { + condition = var.dws_flex.max_run_duration >= 600 && var.dws_flex.max_run_duration <= 604800 + error_message = "Max duration must be at least than 10 minutes, and cannot be more than one week." + } +} + +variable "placement_max_distance" { + type = number + description = "Maximum distance between nodes in the placement group. Requires enable_placement to be true. Values must be supported by the chosen machine type." + nullable = true + default = null + + validation { + condition = coalesce(var.placement_max_distance, 1) >= 1 && coalesce(var.placement_max_distance, 3) <= 3 + error_message = "Invalid value for placement_max_distance. Valid values are null, 1, 2, or 3." + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf new file mode 100644 index 0000000000..e014c318e4 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.4" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.11" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.74.0" + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md new file mode 100644 index 0000000000..d3dbcd959e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md @@ -0,0 +1,105 @@ +## Description + +This module creates a compute partition that can be used as input to the +[schedmd-slurm-gcp-v6-controller](../../scheduler/schedmd-slurm-gcp-v6-controller/README.md). + +The partition module is designed to work alongside the +[schedmd-slurm-gcp-v6-nodeset](../schedmd-slurm-gcp-v6-nodeset/README.md) +module. A partition can be made up of one or +more nodesets, provided either through `use` (preferred) or defined manually +in the `nodeset` variable. + +### Example + +The following code snippet creates a partition module with: + +* 2 nodesets added via `use`. + * The first nodeset is made up of machines of type `c2-standard-30`. + * The second nodeset is made up of machines of type `c2-standard-60`. + * Both nodesets have a maximum count of 200 dynamically created nodes. +* partition name of "compute". +* connected to the `network` module via `use`. +* nodes mounted to homefs via `use`. + +```yaml +- id: nodeset_1 + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + name: c30 + node_count_dynamic_max: 200 + machine_type: c2-standard-30 + +- id: nodeset_2 + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + name: c60 + node_count_dynamic_max: 200 + machine_type: c2-standard-60 + +- id: compute_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + use: + - homefs + - nodeset_1 + - nodeset_2 + settings: + partition_name: compute +``` + +## Support + +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [exclusive](#input\_exclusive) | Exclusive job access to nodes. When set to true nodes execute single job and are deleted
after job exits. If set to false, multiple jobs can be scheduled on one node. | `bool` | `true` | no | +| [is\_default](#input\_is\_default) | Sets this partition as the default partition by updating the partition\_conf.
If "Default" is already set in partition\_conf, this variable will have no effect. | `bool` | `false` | no | +| [network\_storage](#input\_network\_storage) | DEPRECATED | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | A list of nodesets.
For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip     = string
      remote_mount  = string
      local_mount   = string
      fs_type       = string
      mount_options = string
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | +| [partition\_conf](#input\_partition\_conf) | Slurm partition configuration as a map.
See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION | `map(string)` | `{}` | no | +| [partition\_name](#input\_partition\_name) | The name of the slurm partition. | `string` | n/a | yes | +| [resume\_timeout](#input\_resume\_timeout) | Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'ResumeTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. | `number` | `null` | no | +| [suspend\_time](#input\_suspend\_time) | Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram.
This sets 'SuspendTime' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details.
NOTE: use value -1 to exclude partition from suspend.
NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. | `number` | `300` | no | +| [suspend\_timeout](#input\_suspend\_timeout) | Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'SuspendTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. | `number` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Details of a nodesets in this partition | +| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of a dynamic nodesets in this partition | +| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of a TPU nodesets in this partition | +| [partitions](#output\_partitions) | Details of a slurm partition | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf new file mode 100644 index 0000000000..1618c64280 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf @@ -0,0 +1,41 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + use_static = [for ns in concat(var.nodeset, var.nodeset_tpu) : ns.nodeset_name if ns.node_count_static > 0] + + has_node = length(var.nodeset) > 0 + has_dyn = length(var.nodeset_dyn) > 0 + has_tpu = length(var.nodeset_tpu) > 0 + has_flex = length([for ns in var.nodeset : ns.dws_flex.enabled if ns.dws_flex.enabled]) > 0 +} + +locals { + partition_conf = merge({ + "Default" = var.is_default ? "YES" : null + "SuspendTime" = var.suspend_time < 0 ? "INFINITE" : var.suspend_time + "SuspendTimeout" = var.suspend_timeout != null ? var.suspend_timeout : (local.has_tpu ? 240 : 120) + }, var.partition_conf, { "ResumeTimeout" = local.has_flex ? 65535 : try(var.partition_conf["ResumeTimeout"], coalesce(var.resume_timeout, (local.has_tpu ? 600 : 300))) }) + + partition = { + partition_name = var.partition_name + partition_conf = local.partition_conf + + partition_nodeset = [for ns in var.nodeset : ns.nodeset_name] + partition_nodeset_tpu = [for ns in var.nodeset_tpu : ns.nodeset_name] + partition_nodeset_dyn = [for ns in var.nodeset_dyn : ns.nodeset_name] + # Options + enable_job_exclusive = var.exclusive + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml new file mode 100644 index 0000000000..13ea127b3c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + has_to_be_used: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf new file mode 100644 index 0000000000..35dece64fb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf @@ -0,0 +1,54 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "partitions" { + description = "Details of a slurm partition" + + value = [local.partition] + + precondition { + condition = (length(local.use_static) == 0) || !var.exclusive + error_message = <<-EOD + Can't use static nodes within partition with `var.exclusive` set to `true`. + NOTE: Partition's `var.exclusive` is set to `true` by default. Set it to `false` explicitly to use static nodes. + EOD + } + + precondition { + # Can not mix TPU with other non-TPU nodesets due to SlurmGCP specific limitations; + # Can not mix dynamic with non-dynamic nodesets due to Slurms inability to + # turn off "power management" at nodeset level (can only do it at partition or node level). + condition = sum([for b in [local.has_node, local.has_dyn, local.has_tpu] : b ? 1 : 0]) == 1 + error_message = "Partition must contain exactly one type of nodeset." + } +} + +output "nodeset" { + description = "Details of a nodesets in this partition" + + value = var.nodeset +} + +output "nodeset_tpu" { + description = "Details of a TPU nodesets in this partition" + + value = var.nodeset_tpu +} + + +output "nodeset_dyn" { + description = "Details of a dynamic nodesets in this partition" + + value = var.nodeset_dyn +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf new file mode 100644 index 0000000000..a1c85adb90 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf @@ -0,0 +1,311 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "partition_name" { + description = "The name of the slurm partition." + type = string + + validation { + condition = can(regex("^[a-z](?:[a-z0-9]*)$", var.partition_name)) + error_message = "Variable 'partition_name' must be a match of regex '^[a-z](?:[a-z0-9]*)$'." + } +} + +variable "partition_conf" { + description = <<-EOD + Slurm partition configuration as a map. + See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION + EOD + type = map(string) + default = {} +} + +variable "is_default" { + description = <<-EOD + Sets this partition as the default partition by updating the partition_conf. + If "Default" is already set in partition_conf, this variable will have no effect. + EOD + type = bool + default = false +} + +variable "exclusive" { + description = <<-EOD + Exclusive job access to nodes. When set to true nodes execute single job and are deleted + after job exits. If set to false, multiple jobs can be scheduled on one node. + EOD + type = bool + default = true +} + +variable "nodeset" { + description = <<-EOD + A list of nodesets. + For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset + EOD + type = list(object({ + node_count_static = optional(number, 0) + node_count_dynamic_max = optional(number, 1) + node_conf = optional(map(string), {}) + nodeset_name = string + additional_disks = optional(list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string), {}) + auto_delete = optional(bool, true) + boot = optional(bool, false) + disk_resource_manager_tags = optional(map(string), {}) + })), []) + bandwidth_tier = optional(string, "platform_default") + can_ip_forward = optional(bool, false) + disk_auto_delete = optional(bool, true) + disk_labels = optional(map(string), {}) + disk_resource_manager_tags = optional(map(string), {}) + disk_size_gb = optional(number) + disk_type = optional(string) + enable_confidential_vm = optional(bool, false) + enable_placement = optional(bool, false) + placement_max_distance = optional(number, null) + enable_oslogin = optional(bool, true) + enable_shielded_vm = optional(bool, false) + enable_maintenance_reservation = optional(bool, false) + enable_opportunistic_maintenance = optional(bool, false) + gpu = optional(object({ + count = number + type = string + })) + accelerator_topology = optional(string, null) + dws_flex = object({ + enabled = bool + max_run_duration = number + use_job_duration = bool + use_bulk_insert = bool + }) + labels = optional(map(string), {}) + machine_type = optional(string) + advanced_machine_features = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + maintenance_interval = optional(string) + instance_properties_json = string + metadata = optional(map(string), {}) + min_cpu_platform = optional(string) + network_tier = optional(string, "STANDARD") + network_storage = optional(list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = optional(map(string)) + mount_runner = optional(map(string)) + })), []) + on_host_maintenance = optional(string) + preemptible = optional(bool, false) + region = optional(string) + resource_manager_tags = optional(map(string), {}) + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + shielded_instance_config = optional(object({ + enable_integrity_monitoring = optional(bool, true) + enable_secure_boot = optional(bool, true) + enable_vtpm = optional(bool, true) + })) + source_image_family = optional(string) + source_image_project = optional(string) + source_image = optional(string) + subnetwork_self_link = string + additional_networks = optional(list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + }))) + access_config = optional(list(object({ + nat_ip = string + network_tier = string + }))) + spot = optional(bool, false) + tags = optional(list(string), []) + termination_action = optional(string) + reservation_name = optional(string) + future_reservation = string + startup_script = optional(list(object({ + filename = string + content = string })), []) + + zone_target_shape = string + zone_policy_allow = set(string) + zone_policy_deny = set(string) + })) + default = [] + + validation { + condition = length(distinct(var.nodeset[*].nodeset_name)) == length(var.nodeset) + error_message = "All nodesets must have a unique name." + } +} + +variable "nodeset_tpu" { + description = "Define TPU nodesets, as a list." + type = list(object({ + node_count_static = optional(number, 0) + node_count_dynamic_max = optional(number, 5) + nodeset_name = string + enable_public_ip = optional(bool, false) + node_type = string + accelerator_config = optional(object({ + topology = string + version = string + }), { + topology = "" + version = "" + }) + tf_version = string + preemptible = optional(bool, false) + preserve_tpu = optional(bool, false) + zone = string + data_disks = optional(list(string), []) + docker_image = optional(string, "") + network_storage = optional(list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + })), []) + subnetwork = string + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + project_id = string + reserved = optional(string, false) + })) + default = [] + + validation { + condition = length(distinct([for x in var.nodeset_tpu : x.nodeset_name])) == length(var.nodeset_tpu) + error_message = "All TPU nodesets must have a unique name." + } +} + +variable "nodeset_dyn" { + description = "Defines dynamic nodesets, as a list." + type = list(object({ + nodeset_name = string + nodeset_feature = string + })) + default = [] + + validation { + condition = length(distinct([for x in var.nodeset_dyn : x.nodeset_name])) == length(var.nodeset_dyn) + error_message = "All dynamic nodesets must have a unique name." + } +} + +variable "resume_timeout" { + description = <<-EOD + Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use. + If null is given, then a smart default will be chosen depending on nodesets in partition. + This sets 'ResumeTimeout' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. + EOD + type = number + default = null + + validation { + condition = var.resume_timeout == null ? true : var.resume_timeout > 0 && var.resume_timeout < 65536 + error_message = "Value must be > 0 and < 65536" + } +} + +variable "suspend_time" { + description = <<-EOD + Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram. + This sets 'SuspendTime' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details. + NOTE: use value -1 to exclude partition from suspend. + NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. + EOD + type = number + default = 300 + + validation { + condition = var.suspend_time >= -1 + error_message = "Value must be >= -1." + } +} + +variable "suspend_timeout" { + description = <<-EOD + Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown. + If null is given, then a smart default will be chosen depending on nodesets in partition. + This sets 'SuspendTimeout' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. + EOD + type = number + default = null + + validation { + condition = var.suspend_timeout == null ? true : var.suspend_timeout > 0 + error_message = "Value must be > 0." + } +} + + +# tflint-ignore: terraform_unused_declarations +variable "network_storage" { + description = "DEPRECATED" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] + validation { + condition = length(var.network_storage) == 0 + error_message = <<-EOD + network_storage in partition module is deprecated and should not be set. + To add network storage to compute nodes, use network_storage of nodeset module instead. + EOD + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf new file mode 100644 index 0000000000..d388f4bfdd --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.3" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.74.0" + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/README.md b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/README.md new file mode 100644 index 0000000000..994f1500ba --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/README.md @@ -0,0 +1,157 @@ +## Description + +This module provides ways to create and manage Google Cloud Artifact Registry repositories. + +Currently this module is built to support repositories in Docker format although there are placeholder variables for other types which may work too. Remote repositories with pull-through cache functionality integrated with Google Secret Manager is currently supported. The aim of this module is to eventually offer feature parity with this [Terraform module](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository#nested_remote_repository_config), allowing creation of repositories in various formats, including Docker, Maven, NPM, Python, APT, YUM, and COMMON. + +This module is best suited for managing artifact repositories in HPC/AI containerized environments where artifacts need to be shared across distributed systems. It includes IAM role configurations and secret access handling for seamless integration with CI/CD pipelines and other services too. + +It is designed to help facilitate containerized workloads running in the Cluster Toolkit with SLURM leveraging [Enroot](https://github.com/NVIDIA/enroot) and [Pyxis](https://github.com/NVIDIA/pyxis). Docker repositories can store container images that are used in job submissions, enabling efficient and scalable execution of containerized HPC or AI based workloads. + +## Usage + +### Service Account / APIs + +You will need to enable the relevant APIs and create a Service Account for your cluster with the following Artifact Registry permissions. + +```yaml + - id: services-api + source: community/modules/project/service-enablement + settings: + gcp_service_list: + - secretmanager.googleapis.com + - cloudbuild.googleapis.com + - artifactregistry.googleapis.com + + - source: community/modules/project/service-account + kind: terraform + id: hpc_service_account + settings: + project_id: project_name + name: service_account_name + project_roles: + - artifactregistry.reader + - artifactregistry.writer + - secretmanager.secretAccessor +``` + +### Deployment + +Create a standard Docker repository. + +```yaml +- id: registry + source: community/modules/container/artifact-registry + settings: + repo_mode: STANDARD_REPOSITORY + format: DOCKER +``` + +Mirror of public Docker Hub repository. + +```yaml +- id: dockerhub_registry + source: community/modules/container/artifact-registry + settings: + repo_mode: REMOTE_REPOSITORY + format: DOCKER + repo_public_repository: DOCKER_HUB +``` + +Mirror of NVIDIA's [NGC Catalog](https://catalog.ngc.nvidia.com/containers). [API key](https://org.ngc.nvidia.com/setup/api-key) used in blueprint is stored in Secret Manager. + +```yaml +- id: ngc_registry + source: community/modules/container/artifact-registry + settings: + repo_mode: REMOTE_REPOSITORY + format: DOCKER + repo_mirror_url: "https://nvcr.io" + repo_username: $oauthtoken + repo_password: api_key_here + use_upstream_credentials: True +``` + +### Container Operations + +Retrieve `$REPOSITORY_NAME` from [Artifact Registry](https://console.cloud.google.com/artifacts) or by using `gcloud`. + +```yaml +gcloud artifacts repositories list --project="${PROJECT_ID}" +``` + +Pulling containers from your mirrored internal Artifact Repositories. + +Pull [Ubuntu](https://hub.docker.com/_/ubuntu) from Docker Hub mirror. + +```yaml +docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/library/ubuntu:latest +``` + +Pull [Pytorch](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch) from NGC Catalog mirror. + +```yaml +docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/nvidia/pytorch:24.11-py3 +``` + +Alternatively, proceed with running SLURM's [NVIDIA/pyxis](https://github.com/NVIDIA/pyxis) plugin, which will now be able to pull and use these containers directly from the mirrored repositories. + +Note: only Docker registries have been tested so far. Placeholders do exist for other registry types which may or may not work. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_artifact_registry_repository.artifact_registry](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository) | resource | +| [google_secret_manager_secret.repo_password_secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_version.repo_password_secret_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [random_password.repo_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | +| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment. | `string` | n/a | yes | +| [format](#input\_format) | Artifact Registry format (e.g., DOCKER). | `string` | `"DOCKER"` | no | +| [labels](#input\_labels) | Labels to add to the artifact registry. Key-value pairs. | `map(string)` | `{}` | no | +| [project\_id](#input\_project\_id) | Project ID where the artifact registry and secret are created. | `string` | n/a | yes | +| [region](#input\_region) | Region for the artifact registry. | `string` | n/a | yes | +| [repo\_mirror\_url](#input\_repo\_mirror\_url) | For REMOTE\_REPOSITORY, URL for a custom or common mirror. | `string` | `null` | no | +| [repo\_mode](#input\_repo\_mode) | Artifact Registry mode (STANDARD\_REPOSITORY, REMOTE\_REPOSITORY, etc.). | `string` | `"STANDARD_REPOSITORY"` | no | +| [repo\_password](#input\_repo\_password) | Optional password/API key. If null, one will be randomly generated. | `string` | `null` | no | +| [repo\_public\_repository](#input\_repo\_public\_repository) | For REMOTE\_REPOSITORY, name of a known public repo as per the Terraform module
(e.g., DOCKER\_HUB) or null for custom repo. | `string` | `null` | no | +| [repo\_username](#input\_repo\_username) | Username for external repository. | `string` | `null` | no | +| [repository\_base](#input\_repository\_base) | For APT/YUM public repos, repository\_base (e.g., 'DEBIAN', 'UBUNTU'). | `string` | `null` | no | +| [repository\_path](#input\_repository\_path) | For APT/YUM public repos, repository\_path (e.g., 'debian/dists/buster'). | `string` | `null` | no | +| [use\_upstream\_credentials](#input\_use\_upstream\_credentials) | Configure Service Account to use upstream credentials for REMOTE\_REPOSITORY:
If true, a username/password is used for the REMOTE\_REPOSITORY mirror.
If false (or if repo\_password == null), no password is created at all.
Note: Blueprint credentials will be stored in Secrets Manager. | `bool` | `false` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | (Optional) A list of objects to enable user-managed replication.
Each object can have:
location = string
kms\_key\_name = optional(string)
If empty, auto replication is used. | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [registry\_url](#output\_registry\_url) | The URL of the created artifact registry. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/main.tf b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/main.tf new file mode 100644 index 0000000000..c3406af607 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/main.tf @@ -0,0 +1,268 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "artifact-registry", ghpc_role = "container" }) +} + +locals { + # Auto (i.e., empty) vs user-managed replication + auto = length(var.user_managed_replication) == 0 ? true : false + + # For remote custom repositories, parse out host to create a base_component name + mirror_url_no_proto = var.repo_mirror_url != null ? replace(replace(var.repo_mirror_url, "https://", ""), "http://", "") : "" + mirror_host = local.mirror_url_no_proto != "" ? split("/", local.mirror_url_no_proto)[0] : "" + + base_component = replace( + replace( + replace( + lower( + local.mirror_host != "" + ? "${var.format}-${var.repo_mode}-${local.mirror_host}" + : "${var.format}-${var.repo_mode}-nohost" + ), + "\\.", "-" + ), + "/", "-" + ), + "_", "-" + ) + + repository_suffix = random_id.resource_name_suffix.hex + + # The final name for the artifact registry repository + repository_name = replace( + replace( + lower( + format("%s-%s", local.base_component, local.repository_suffix) + ), + ".", "-" + ), + "/", "-" + ) + + # The secret name is derived from the repository name + # with a suffix like "-secret". + derived_secret_name = format("%s-secret", local.repository_name) +} + +############################## +# PASSWORD / SECRET +############################## + +# Only create a random password if user didn't supply one +resource "random_password" "repo_password" { + count = var.use_upstream_credentials && var.repo_password == null ? 1 : 0 + length = 24 + special = true + override_special = "_-#=." +} + +resource "google_secret_manager_secret" "repo_password_secret" { + count = var.use_upstream_credentials ? 1 : 0 + project = var.project_id + + # Derive the secret ID from the repository name + secret_id = local.derived_secret_name + + labels = local.labels + + replication { + dynamic "auto" { + for_each = local.auto ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = local.auto ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = replicas.value.kms_key_name != null ? [1] : [] + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_version" "repo_password_secret_version" { + count = var.use_upstream_credentials ? 1 : 0 + secret = google_secret_manager_secret.repo_password_secret[0].id + + # If user provided a password, use it. Otherwise use the random password. + secret_data = var.repo_password != null ? var.repo_password : random_password.repo_password[0].result +} + +############################## +# IAM BINDINGS +############################## + +############################## +# ARTIFACT REGISTRY +############################## + +resource "random_id" "resource_name_suffix" { + byte_length = 2 +} + +resource "google_artifact_registry_repository" "artifact_registry" { + project = var.project_id + location = var.region + format = var.format + mode = var.repo_mode + description = var.deployment_name + labels = local.labels + repository_id = local.repository_name + + # Only create remote_repository_config if REMOTE_REPOSITORY + dynamic "remote_repository_config" { + for_each = var.repo_mode == "REMOTE_REPOSITORY" ? [1] : [] + content { + description = "Pull-through cache" + + dynamic "docker_repository" { + for_each = var.format == "DOCKER" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "docker_repository" { + for_each = var.format == "DOCKER" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "maven_repository" { + for_each = var.format == "MAVEN" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "maven_repository" { + for_each = var.format == "MAVEN" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "npm_repository" { + for_each = var.format == "NPM" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "npm_repository" { + for_each = var.format == "NPM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "python_repository" { + for_each = var.format == "PYTHON" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "python_repository" { + for_each = var.format == "PYTHON" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "apt_repository" { + for_each = var.format == "APT" && var.repo_public_repository != null ? [1] : [] + content { + public_repository { + repository_base = var.repository_base + repository_path = var.repository_path + } + } + } + + dynamic "apt_repository" { + for_each = var.format == "APT" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "yum_repository" { + for_each = var.format == "YUM" && var.repo_public_repository != null ? [1] : [] + content { + public_repository { + repository_base = var.repository_base + repository_path = var.repository_path + } + } + } + + dynamic "yum_repository" { + for_each = var.format == "YUM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "common_repository" { + for_each = var.format == "COMMON" ? [1] : [] + content { + uri = var.repo_mirror_url + } + } + + # Only enable upstream credentials if user wants it + dynamic "upstream_credentials" { + for_each = var.use_upstream_credentials ? [1] : [] + content { + username_password_credentials { + username = var.repo_username + password_secret_version = google_secret_manager_secret_version.repo_password_secret_version[0].name + } + } + } + } + } + + depends_on = [ + google_secret_manager_secret.repo_password_secret, + google_secret_manager_secret_version.repo_password_secret_version, + ] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/metadata.yaml new file mode 100644 index 0000000000..6b68c98a54 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - secretmanager.googleapis.com + - artifactregistry.googleapis.com + - cloudbuild.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/outputs.tf new file mode 100644 index 0000000000..92b6dbb165 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "registry_url" { + description = "The URL of the created artifact registry." + value = "${var.region}-docker.pkg.dev/${var.project_id}/${var.deployment_name}" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/validation.tf b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/validation.tf new file mode 100644 index 0000000000..a795060fb7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/validation.tf @@ -0,0 +1,49 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "terraform_data" "input_validation" { + lifecycle { + precondition { + condition = ( + var.repo_password == null || + (var.use_upstream_credentials && var.repo_mode == "REMOTE_REPOSITORY") + ) + error_message = "repo_password may be set only when repo_mode=REMOTE_REPOSITORY and use_upstream_credentials=true." + } + + precondition { + condition = ( + !var.use_upstream_credentials || + var.repo_mode == "REMOTE_REPOSITORY" + ) + error_message = "use_upstream_credentials is allowed only when repo_mode is REMOTE_REPOSITORY." + } + + precondition { + condition = ( + var.repo_mode != "REMOTE_REPOSITORY" || + (var.repo_public_repository != null || var.repo_mirror_url != null) + ) + error_message = "For a REMOTE_REPOSITORY you must set repo_public_repository or repo_mirror_url." + } + + precondition { + condition = ( + !contains(["APT", "YUM"], var.format) || + (var.repository_base != null && var.repository_path != null) + ) + error_message = "APT/YUM formats require repository_base and repository_path." + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/variables.tf new file mode 100644 index 0000000000..9a4eecb921 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/variables.tf @@ -0,0 +1,122 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Project ID where the artifact registry and secret are created." + type = string +} + +variable "region" { + description = "Region for the artifact registry." + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment." + type = string +} + +variable "labels" { + description = "Labels to add to the artifact registry. Key-value pairs." + type = map(string) + default = {} +} + +variable "repo_password" { + description = "Optional password/API key. If null, one will be randomly generated." + type = string + default = null +} + +variable "user_managed_replication" { + description = <<-DOC + (Optional) A list of objects to enable user-managed replication. + Each object can have: + location = string + kms_key_name = optional(string) + If empty, auto replication is used. + DOC + type = list(object({ + location = string + kms_key_name = optional(string) + })) + default = [] +} + +variable "format" { + description = "Artifact Registry format (e.g., DOCKER)." + type = string + default = "DOCKER" +} + +variable "repo_mode" { + description = "Artifact Registry mode (STANDARD_REPOSITORY, REMOTE_REPOSITORY, etc.)." + type = string + default = "STANDARD_REPOSITORY" + + validation { + condition = can(regex("^(STANDARD_REPOSITORY|REMOTE_REPOSITORY|VIRTUAL_REPOSITORY)$", var.repo_mode)) + error_message = "repo_mode must be one of STANDARD_REPOSITORY, REMOTE_REPOSITORY, or VIRTUAL_REPOSITORY." + } +} + +variable "repo_public_repository" { + description = <<-DOC + For REMOTE_REPOSITORY, name of a known public repo as per the Terraform module + (e.g., DOCKER_HUB) or null for custom repo. + DOC + type = string + default = null + + # To Do: implement validation + # validation { + # condition = ((var.repo_mode != "REMOTE_REPOSITORY" && var.repo_public_repository == null) || (var.repo_mode == "REMOTE_REPOSITORY" && (var.repo_public_repository != null || var.repo_mirror_url != null))) + # error_message = "If repo_mode is REMOTE_REPOSITORY, you must set either repo_public_repository or repo_mirror_url. Otherwise, leave them null." + # } +} + +variable "repo_mirror_url" { + description = "For REMOTE_REPOSITORY, URL for a custom or common mirror." + type = string + default = null +} + +variable "use_upstream_credentials" { + description = <<-DOC + Configure Service Account to use upstream credentials for REMOTE_REPOSITORY: + If true, a username/password is used for the REMOTE_REPOSITORY mirror. + If false (or if repo_password == null), no password is created at all. + Note: Blueprint credentials will be stored in Secrets Manager. + DOC + type = bool + default = false +} + +variable "repo_username" { + description = "Username for external repository." + type = string + default = null +} + +variable "repository_base" { + description = "For APT/YUM public repos, repository_base (e.g., 'DEBIAN', 'UBUNTU')." + type = string + default = null +} + +variable "repository_path" { + description = "For APT/YUM public repos, repository_path (e.g., 'debian/dists/buster')." + type = string + default = null +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/versions.tf new file mode 100644 index 0000000000..392a7131d2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/README.md b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/README.md new file mode 100644 index 0000000000..23bf87398a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/README.md @@ -0,0 +1,76 @@ +## Description + +Creates a BigQuery dataset. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage +This is a simple usage. + +```yaml + - id: bq-dataset + source: community/modules/database/bigquery-dataset + settings: + dataset_id: my_dataset +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_dataset.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | The name of the dataset to be created | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the dataset. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [dataset\_id](#output\_dataset\_id) | Name of the dataset that was created. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/main.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/main.tf new file mode 100644 index 0000000000..1a9c4bba60 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/main.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-dataset", ghpc_role = "database" }) +} +locals { + dataset_id = var.dataset_id != null ? var.dataset_id : replace("${var.deployment_name}_dataset_${random_id.resource_name_suffix.hex}", "-", "_") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_bigquery_dataset" "pbsb" { + dataset_id = local.dataset_id + project = var.project_id + labels = local.labels +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml new file mode 100644 index 0000000000..87ff9357e4 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigquery.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf new file mode 100644 index 0000000000..9cd8e5df31 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "dataset_id" { + description = "Name of the dataset that was created." + value = google_bigquery_dataset.pbsb.dataset_id +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/variables.tf new file mode 100644 index 0000000000..90c229af6b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/variables.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "dataset_id" { + description = "The name of the dataset to be created" + type = string + default = null +} + +variable "labels" { + description = "Labels to add to the dataset. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/versions.tf new file mode 100644 index 0000000000..12ddbe842d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/README.md b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/README.md new file mode 100644 index 0000000000..ef67cfef01 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/README.md @@ -0,0 +1,87 @@ +## Description + +Creates a BigQuery table with a specified schema. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage + +```yaml +id: bq-table + source: community/modules/database/bigquery-table + use: [bq-dataset] + settings: + table_schema: + ' + [ + { + "name": "id", "type": "STRING" + } + ] + ' +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_table.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Dataset name to be used to create the new BQ Table | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the tables. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [table\_id](#input\_table\_id) | Table name to be used to create the new BQ Table | `string` | `null` | no | +| [table\_schema](#input\_table\_schema) | Schema used to create the new BQ Table | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [dataset\_id](#output\_dataset\_id) | ID of BQ dataset | +| [table\_id](#output\_table\_id) | ID of created BQ table | +| [table\_name](#output\_table\_name) | Name of created BQ table | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/main.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/main.tf new file mode 100644 index 0000000000..73f3923e00 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/main.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-table", ghpc_role = "database" }) +} + +locals { + table_id = var.table_id != null ? var.table_id : replace("${var.deployment_name}_table_${random_id.resource_name_suffix.hex}", "-", "_") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_bigquery_table" "pbsb" { + deletion_protection = false + project = var.project_id + table_id = local.table_id + dataset_id = var.dataset_id + schema = var.table_schema + labels = local.labels +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/metadata.yaml new file mode 100644 index 0000000000..87ff9357e4 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigquery.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/outputs.tf new file mode 100644 index 0000000000..4220ec1390 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/outputs.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "table_name" { + description = "Name of created BQ table" + value = google_bigquery_table.pbsb.friendly_name +} +output "table_id" { + description = "ID of created BQ table" + value = google_bigquery_table.pbsb.table_id +} +output "dataset_id" { + description = "ID of BQ dataset" + value = google_bigquery_table.pbsb.dataset_id +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/variables.tf new file mode 100644 index 0000000000..ec474b4e64 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/variables.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "labels" { + description = "Labels to add to the tables. Key-value pairs." + type = map(string) +} + +variable "table_id" { + description = "Table name to be used to create the new BQ Table" + type = string + default = null +} + +variable "dataset_id" { + description = "Dataset name to be used to create the new BQ Table" + type = string +} + +variable "table_schema" { + description = "Schema used to create the new BQ Table" + type = string +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/versions.tf new file mode 100644 index 0000000000..12ddbe842d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md new file mode 100644 index 0000000000..08364c175b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md @@ -0,0 +1,107 @@ +## Description + +terraform-google-sql makes it easy to create a Google CloudSQL instance and +implement high availability settings. This module is meant for use with +Terraform 0.13+ and tested using Terraform 1.0+. + +The cloudsql created here is used to integrate with the slurm cluster to enable +accounting data storage. + +### Example + +```yaml +- id: cloudsql + source: community/modules/database/slurm-cloudsql-federation + use: [network] + settings: + sql_instance_name: slurm-sql6-demo + tier: "db-f1-micro" +``` + +This creates a cloud sql instance, including a database, user that would allow +the slurm cluster to use as an external DB. In addition, it will allow BigQuery +to run federated query through it. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | +| [google](#requirement\_google) | >= 3.83 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_connection.connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_connection) | resource | +| [google_compute_address.psc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_forwarding_rule.psc_consumer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_forwarding_rule) | resource | +| [google_sql_database.database](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database) | resource | +| [google_sql_database_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance) | resource | +| [google_sql_user.users](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [random_password.password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [authorized\_networks](#input\_authorized\_networks) | IP address ranges as authorized networks of the Cloud SQL for MySQL instances | `list(string)` | `[]` | no | +| [data\_cache\_enabled](#input\_data\_cache\_enabled) | Whether data cache is enabled for the instance. Can be used with ENTERPRISE\_PLUS edition. | `bool` | `false` | no | +| [database\_flags](#input\_database\_flags) | Database flags to set on instance. | `map(string)` | `{}` | no | +| [database\_version](#input\_database\_version) | The version of the database to be created. | `string` | `"MYSQL_8_0"` | no | +| [deletion\_protection](#input\_deletion\_protection) | Whether or not to allow Terraform to destroy the instance. | `string` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [disk\_autoresize](#input\_disk\_autoresize) | Set to false to disable automatic disk grow. | `bool` | `true` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of the database disk in GiB. | `number` | `null` | no | +| [edition](#input\_edition) | value | `string` | `"ENTERPRISE"` | no | +| [enable\_backups](#input\_enable\_backups) | Set true to enable backups | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is going to be created in.:
`projects//global/networks/`" | `string` | n/a | yes | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [query\_insights](#input\_query\_insights) | Query insights configuration. | 
object({
    enabled                 = optional(bool, false)
    query_plans_per_minute  = optional(number)
    query_string_length     = optional(number)
    record_application_tags = optional(bool)
    record_client_address   = optional(bool)
  })
| `{}` | no | +| [region](#input\_region) | The region where SQL instance will be configured | `string` | n/a | yes | +| [sql\_instance\_name](#input\_sql\_instance\_name) | name given to the sql instance for ease of identificaion | `string` | n/a | yes | +| [sql\_password](#input\_sql\_password) | Password for the SQL database. | `any` | `null` | no | +| [sql\_username](#input\_sql\_username) | Username for the SQL database | `string` | `"slurm"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self link of the network where Cloud SQL instance PSC endpoint will be created | `string` | `null` | no | +| [tier](#input\_tier) | The machine type to use for the SQL instance | `string` | n/a | yes | +| [use\_psc\_connection](#input\_use\_psc\_connection) | Create Private Service Connection instead of using Private Service Access peering | `bool` | `false` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cloudsql](#output\_cloudsql) | Describes the cloudsql instance. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf new file mode 100644 index 0000000000..9b518a1b5f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf @@ -0,0 +1,165 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "slurm-cloudsql-federation", ghpc_role = "database" }) +} + +locals { + user_managed_replication = var.user_managed_replication +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "random_password" "password" { + length = 12 + special = false +} + +locals { + sql_instance_name = var.sql_instance_name == null ? "${var.deployment_name}-sql-${random_id.resource_name_suffix.hex}" : var.sql_instance_name + sql_password = var.sql_password == null ? random_password.password.result : var.sql_password +} + + +resource "google_sql_database_instance" "instance" { + project = var.project_id + depends_on = [var.private_vpc_connection_peering] + name = local.sql_instance_name + region = var.region + deletion_protection = var.deletion_protection + database_version = var.database_version + + settings { + disk_size = var.disk_size_gb + disk_autoresize = var.disk_autoresize + edition = var.edition + tier = var.tier + user_labels = local.labels + + dynamic "data_cache_config" { + for_each = var.edition == "ENTERPRISE_PLUS" ? [""] : [] + content { + data_cache_enabled = var.data_cache_enabled + } + } + + dynamic "database_flags" { + for_each = var.database_flags + content { + name = database_flags.key + value = database_flags.value + } + } + + insights_config { + query_insights_enabled = var.query_insights.enabled + query_plans_per_minute = var.query_insights.query_plans_per_minute + query_string_length = var.query_insights.query_string_length + record_application_tags = var.query_insights.record_application_tags + record_client_address = var.query_insights.record_client_address + } + + ip_configuration { + ipv4_enabled = false + private_network = var.use_psc_connection ? null : var.network_id + enable_private_path_for_google_cloud_services = true + + dynamic "authorized_networks" { + for_each = var.use_psc_connection ? [] : var.authorized_networks + iterator = ip_range + + content { + value = ip_range.value + } + } + dynamic "psc_config" { + for_each = var.use_psc_connection ? [""] : [] + content { + psc_enabled = true + allowed_consumer_projects = [var.project_id] + } + } + } + + backup_configuration { + enabled = var.enable_backups + # to allow easy switching between ENTERPRISE and ENTERPRISE_PLUS + transaction_log_retention_days = 7 + } + } + lifecycle { + precondition { + condition = var.disk_autoresize && var.disk_size_gb == null || !var.disk_autoresize + error_message = "If setting disk_size_gb set disk_autorize to false to prevent re-provisioning of the instance after disk auto-expansion." + } + } +} + + + +resource "google_compute_address" "psc" { + count = var.use_psc_connection ? 1 : 0 + project = var.project_id + name = local.sql_instance_name + address_type = "INTERNAL" + region = var.region + subnetwork = var.subnetwork_self_link + labels = local.labels +} + +resource "google_compute_forwarding_rule" "psc_consumer" { + count = var.use_psc_connection ? 1 : 0 + name = local.sql_instance_name + project = var.project_id + region = var.region + subnetwork = var.subnetwork_self_link + ip_address = google_compute_address.psc[0].self_link + load_balancing_scheme = "" + recreate_closed_psc = true + target = google_sql_database_instance.instance.psc_service_attachment_link +} + +resource "google_sql_database" "database" { + project = var.project_id + name = "slurm_accounting" + instance = google_sql_database_instance.instance.name +} + +resource "google_sql_user" "users" { + project = var.project_id + name = var.sql_username + instance = google_sql_database_instance.instance.name + password = local.sql_password +} + +resource "google_bigquery_connection" "connection" { + provider = google + project = var.project_id + location = var.region + cloud_sql { + instance_id = google_sql_database_instance.instance.connection_name + database = google_sql_database.database.name + type = "MYSQL" + credential { + username = google_sql_user.users.name + password = google_sql_user.users.password + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml new file mode 100644 index 0000000000..fc0cae0859 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigqueryconnection.googleapis.com + - sqladmin.googleapis.com + - servicenetworking.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf new file mode 100644 index 0000000000..0d05221cd8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "cloudsql" { + description = "Describes the cloudsql instance." + sensitive = true + value = { + server_ip = var.use_psc_connection ? google_compute_address.psc[0].address : google_sql_database_instance.instance.ip_address[0].ip_address + user = google_sql_user.users.name + password = google_sql_user.users.password + db_name = google_sql_database.database.name + user_managed_replication = local.user_managed_replication + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf new file mode 100644 index 0000000000..a2f150419e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf @@ -0,0 +1,173 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "authorized_networks" { + description = "IP address ranges as authorized networks of the Cloud SQL for MySQL instances" + type = list(string) + default = [] + nullable = false +} + +variable "database_version" { + description = "The version of the database to be created." + type = string + default = "MYSQL_8_0" + validation { + condition = contains(["MYSQL_5_7", "MYSQL_8_0", "MYSQL_8_4"], var.database_version) + error_message = "The database version must be either MYSQL_5_7, MYSQL_8_0 or MYSQL_8_4." + } +} + +variable "data_cache_enabled" { + description = "Whether data cache is enabled for the instance. Can be used with ENTERPRISE_PLUS edition." + type = bool + default = false +} + +variable "database_flags" { + description = "Database flags to set on instance." + type = map(string) + default = {} + nullable = false +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "disk_autoresize" { + description = "Set to false to disable automatic disk grow." + type = bool + default = true +} + +variable "disk_size_gb" { + description = "Size of the database disk in GiB." + type = number + default = null +} + +variable "edition" { + description = "value" + type = string + validation { + condition = contains(["ENTERPRISE", "ENTERPRISE_PLUS"], var.edition) + error_message = "The database edition must be either ENTERPRISE or ENTERPRISE_PLUS" + } + default = "ENTERPRISE" +} + +variable "enable_backups" { + description = "Set true to enable backups" + type = bool + default = false +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "query_insights" { + description = "Query insights configuration." + nullable = false + default = {} + type = object({ + enabled = optional(bool, false) + query_plans_per_minute = optional(number) + query_string_length = optional(number) + record_application_tags = optional(bool) + record_client_address = optional(bool) + }) +} + +variable "region" { + description = "The region where SQL instance will be configured" + type = string +} + +variable "tier" { + description = "The machine type to use for the SQL instance" + type = string +} + +variable "sql_instance_name" { + description = "name given to the sql instance for ease of identificaion" + type = string +} + +variable "deletion_protection" { + description = "Whether or not to allow Terraform to destroy the instance." + type = string + default = false +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} + +variable "sql_username" { + description = "Username for the SQL database" + type = string + default = "slurm" +} + +variable "sql_password" { + description = "Password for the SQL database." + type = any + default = null +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is going to be created in.: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "private_vpc_connection_peering" { + description = "The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation." + type = string + default = null +} + +variable "subnetwork_self_link" { + description = "Self link of the network where Cloud SQL instance PSC endpoint will be created" + type = string + default = null +} + +variable "user_managed_replication" { + type = list(object({ + location = string + kms_key_name = optional(string) + })) + description = "Replication parameters that will be used for defined secrets" + default = [] +} + +variable "use_psc_connection" { + description = "Create Private Service Connection instead of using Private Service Access peering" + type = bool + default = false +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf new file mode 100644 index 0000000000..7e672858b6 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" + } + + required_version = ">= 0.13.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md new file mode 100644 index 0000000000..d39a58afe1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md @@ -0,0 +1,158 @@ +> [!WARNING] +> This module is deprecated and will be removed on July 1, 2025. The +> recommended replacement is the +> [GCP Managed Lustre module](../../../../modules/file-system/managed-lustre/README.md) + +## Description +This module creates a DDN EXAScaler Cloud Lustre file system using code based on DDN's +[exascaler-cloud-terraform](https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp) (`scripts/2.2.2` is last release with GCP-specific module). + +More information about the architecture can be found at +[Overview of Lustre and EXAScaler Cloud][architecture]. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +> **Warning**: This file system has a license cost as described in the pricing +> section of the [DDN EXAScaler Cloud Marketplace Solution][marketplace]. +> +> **Note**: By default security.public_key is set to `null`, therefore the +> admin user is not created. To ensure the admin user is created, provide a +> public key via the security setting. +> +> **Note**: This module's instances require access to Google APIs and +> therefore, instances must have public IP address or it must be used in a +> subnetwork where [Private Google Access][private-google-access] is enabled. + +[private-google-access]: https://cloud.google.com/vpc/docs/configure-private-google-access +[marketplace]: https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud +[architecture]: https://cloud.google.com/architecture/parallel-file-systems-for-hpc#overview_of_lustre_and_exascaler_cloud + +## Mounting + +To mount the DDN EXAScaler Lustre file system you must first install the DDN +Lustre client and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +the [hpc-enterprise-slurm.yaml](../../../../examples/hpc-enterprise-slurm.yaml) for an +example of using this module with Slurm. + +If mounting is not automatically handled as described above, the DDN-EXAScaler +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + # This file system has an associated license cost. + # https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud + - id: lustrefs + source: community/modules/file-system/DDN-EXAScaler + use: [network1] + settings: {local_mount: /scratch} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(lustrefs.install_ddn_lustre_client_runner) + - $(lustrefs.mount_runner) + +``` + +See [additional documentation][ddn-install-docs] from DDN EXAScaler. + +[ddn-install-docs]: https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp#install-new-exascaler-cloud-clients +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## Support + +EXAScaler Cloud includes self-help support with access to publicly available +documents and videos. Premium support includes 24x7x365 access to DDN's experts, +along with support community access, automated notifications of updates and +other premium support features. For more information, visit +[EXAscaler Cloud on GCP][exa-gcp]. + +[exa-gcp]: https://console.cloud.google.com/marketplace/product/ddnstorage/exascaler-cloud + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [ddn\_exascaler](#module\_ddn\_exascaler) | github.com/DDNStorage/exascaler-cloud-terraform//gcp | a3355d50deebe45c0556b45bd599059b7c06988d | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [boot](#input\_boot) | Boot disk properties | 
object({
    disk_type   = string
    auto_delete = bool
    script_url  = string
  })
| 
{
  "auto_delete": true,
  "disk_type": "pd-standard",
  "script_url": null
}
| no | +| [cls](#input\_cls) | Compute client properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 0,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-2",
  "public_ip": true
}
| no | +| [clt](#input\_clt) | Compute client target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 0,
  "disk_size": 256,
  "disk_type": "pd-standard"
}
| no | +| [fsname](#input\_fsname) | EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long | `string` | `"exacloud"` | no | +| [image](#input\_image) | DEPRECATED: Source image properties | `any` | `null` | no | +| [instance\_image](#input\_instance\_image) | Source image properties

Expected Fields:
name: Unavailable with this module.
family: The image family to use.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp",
  "project": "ddn-public"
}
| no | +| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. Key-value pairs. | `map(string)` | `{}` | no | +| [local\_mount](#input\_local\_mount) | Mountpoint (at the client instances) for this EXAScaler system | `string` | `"/shared"` | no | +| [mds](#input\_mds) | Metadata server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | +| [mdt](#input\_mdt) | Metadata target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | +| [mgs](#input\_mgs) | Management server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | +| [mgt](#input\_mgt) | Management target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | +| [mnt](#input\_mnt) | Monitoring target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | +| [network\_properties](#input\_network\_properties) | Network options. 'network\_self\_link' or 'network\_properties' must be provided. | 
object({
    routing = string
    tier    = string
    id      = string
    auto    = bool
    mtu     = number
    new     = bool
    nat     = bool
  })
| `null` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self-link of the VPC network to where the system is connected. Ignored if 'network\_properties' is provided. 'network\_self\_link' or 'network\_properties' must be provided. | `string` | `null` | no | +| [oss](#input\_oss) | Object Storage server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 3,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-16",
  "public_ip": true
}
| no | +| [ost](#input\_ost) | Object Storage target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | +| [prefix](#input\_prefix) | EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud') | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Compute Platform project that will host the EXAScaler filesystem | `string` | n/a | yes | +| [security](#input\_security) | Security options | 
object({
    admin              = string
    public_key         = string
    block_project_keys = bool
    enable_os_login    = bool
    enable_local       = bool
    enable_ssh         = bool
    enable_http        = bool
    ssh_source_ranges  = list(string)
    http_source_ranges = list(string)
  })
| 
{
  "admin": "stack",
  "block_project_keys": false,
  "enable_http": false,
  "enable_local": false,
  "enable_os_login": true,
  "enable_ssh": false,
  "http_source_ranges": [
    "0.0.0.0/0"
  ],
  "public_key": null,
  "ssh_source_ranges": [
    "0.0.0.0/0"
  ]
}
| no | +| [service\_account](#input\_service\_account) | Service account name used by deploy application | 
object({
    new   = bool
    email = string
  })
| 
{
  "email": null,
  "new": false
}
| no | +| [subnetwork\_address](#input\_subnetwork\_address) | The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork\_properties' is provided. | `string` | `null` | no | +| [subnetwork\_properties](#input\_subnetwork\_properties) | Subnetwork properties. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | 
object({
    address = string
    private = bool
    id      = string
    new     = bool
  })
| `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork\_properties' is provided. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | `string` | `null` | no | +| [waiter](#input\_waiter) | Waiter to check progress and result for deployment. | `string` | `null` | no | +| [zone](#input\_zone) | Compute Platform zone where the servers will be located | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_config\_script](#output\_client\_config\_script) | Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler. | +| [http\_console](#output\_http\_console) | HTTP address to access the system web console. | +| [install\_ddn\_lustre\_client\_runner](#output\_install\_ddn\_lustre\_client\_runner) | Runner that encapsulates the `client_config_script` output on this module. | +| [mount\_command](#output\_mount\_command) | Command to mount the file system. `client_config_script` must be run first. | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the DDN EXAScaler Lustre file system | +| [network\_storage](#output\_network\_storage) | Describes a EXAScaler system to be mounted by other systems. | +| [private\_addresses](#output\_private\_addresses) | Private IP addresses for all instances. | +| [ssh\_console](#output\_ssh\_console) | Instructions to ssh into the instances. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf new file mode 100644 index 0000000000..6a2fc4b702 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ddn-exascaler", ghpc_role = "file-system" }) +} + +locals { + + network_id = var.network_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.network_self_link)[0] : null + named_net = { + routing = "REGIONAL" + tier = "STANDARD" + id = local.network_id + auto = false + mtu = 1500 + new = false + nat = false + } + + subnetwork_id = var.subnetwork_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.subnetwork_self_link)[0] : null + named_subnet = { + address = var.subnetwork_address + private = true + id = local.subnetwork_id + new = false + } +} + +module "ddn_exascaler" { + source = "github.com/DDNStorage/exascaler-cloud-terraform//gcp?ref=a3355d50deebe45c0556b45bd599059b7c06988d" + fsname = var.fsname + zone = var.zone + project = var.project_id + prefix = var.prefix + labels = local.labels + security = var.security + service_account = var.service_account + waiter = var.waiter + network = var.network_properties == null ? local.named_net : var.network_properties + subnetwork = var.subnetwork_properties == null ? local.named_subnet : var.subnetwork_properties + boot = var.boot + image = var.instance_image + mgs = var.mgs + mgt = var.mgt + mnt = var.mnt + mds = var.mds + mdt = var.mdt + oss = var.oss + ost = var.ost + cls = var.cls + clt = var.clt +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml new file mode 100644 index 0000000000..b995bd4358 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - deploymentmanager.googleapis.com + - iam.googleapis.com + - runtimeconfig.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf new file mode 100644 index 0000000000..2e9ae732ae --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf @@ -0,0 +1,90 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +output "private_addresses" { + description = "Private IP addresses for all instances." + value = module.ddn_exascaler.private_addresses +} + +output "ssh_console" { + description = "Instructions to ssh into the instances." + value = module.ddn_exascaler.ssh_console +} + +output "client_config_script" { + description = "Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler." + value = module.ddn_exascaler.client_config +} + +output "install_ddn_lustre_client_runner" { + description = "Runner that encapsulates the `client_config_script` output on this module." + value = local.client_install_runner +} + +locals { + client_install_runner = { + "type" = "shell" + "content" = module.ddn_exascaler.client_config + "destination" = "install_ddn_lustre_client.sh" + } + + # Mount command provided by DDN does not support custom local mount + split_mount_cmd = split(" ", module.ddn_exascaler.mount_command) + split_mount_cmd_wo_mountpoint = slice(local.split_mount_cmd, 0, length(local.split_mount_cmd) - 1) + mount_cmd = "${join(" ", local.split_mount_cmd_wo_mountpoint)} ${var.local_mount}" + mount_cmd_w_mkdir = "mkdir -p ${var.local_mount} && ${local.mount_cmd}" + mount_runner = { + "type" = "shell" + "content" = local.mount_cmd_w_mkdir + "destination" = "mount-ddn-lustre.sh" + } +} + +output "mount_command" { + description = "Command to mount the file system. `client_config_script` must be run first." + value = local.mount_cmd_w_mkdir +} + +output "mount_runner" { + description = "Runner to mount the DDN EXAScaler Lustre file system" + value = local.mount_runner +} + +output "http_console" { + description = "HTTP address to access the system web console." + value = module.ddn_exascaler.http_console +} + +output "network_storage" { + description = "Describes a EXAScaler system to be mounted by other systems." + value = { + server_ip = split(":", split(" ", module.ddn_exascaler.mount_command)[3])[0] + remote_mount = length(regexall("^/.*", var.fsname)) > 0 ? var.fsname : format("/%s", var.fsname) + local_mount = var.local_mount != null ? var.local_mount : format("/mnt/%s", var.fsname) + fs_type = "lustre" + mount_options = "" + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } + depends_on = [ + module.ddn_exascaler + ] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf new file mode 100644 index 0000000000..68bcc8a8ba --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf @@ -0,0 +1,502 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +# EXAScaler filesystem name +# only alphanumeric characters are allowed, +# and the value must be 1-8 characters long +variable "fsname" { + description = "EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long" + type = string + default = "exacloud" +} + +# Project ID to manage resources +# https://cloud.google.com/resource-manager/docs/creating-managing-projects +variable "project_id" { + description = "Compute Platform project that will host the EXAScaler filesystem" + type = string +} + +# Zone name to manage resources +# https://cloud.google.com/compute/docs/regions-zones +variable "zone" { + description = "Compute Platform zone where the servers will be located" + type = string +} + +# Service account name used by deploy application +# https://cloud.google.com/iam/docs/service-accounts +# new: create a new custom service account or use an existing one: true or false +# email: existing service account email address, will be using if new is false +# set email = null to use the default compute service account +variable "service_account" { + description = "Service account name used by deploy application" + type = object({ + new = bool + email = string + }) + default = { + new = false + email = null + } +} + +# Waiter to check progress and result for deployment. +# To use Google Deployment Manager: +# waiter = "deploymentmanager" +# To use generic Google Cloud SDK command line: +# waiter = "sdk" +# If you don’t want to wait until the deployment is complete: +# waiter = null +# https://cloud.google.com/deployment-manager/runtime-configurator/creating-a-waiter +variable "waiter" { + description = "Waiter to check progress and result for deployment." + type = string + default = null +} + +# Security options +# admin: optional user name for remote SSH access +# Set admin = null to disable creation admin user +# public_key: path to the SSH public key on the local host +# Set public_key = null to disable creation admin user +# block_project_keys: true or false +# Block project-wide public SSH keys if you want to restrict +# deployment to only user with deployment-level public SSH key. +# https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys +# enable_os_login: true or false +# Enable or disable OS Login feature. +# Please note, enabling this option disables other security options: +# admin, public_key and block_project_keys. +# https://cloud.google.com/compute/docs/instances/managing-instance-access#enable_oslogin +# enable_local: true or false, enable or disable firewall rules for local access +# enable_ssh: true or false, enable or disable remote SSH access +# ssh_source_ranges: source IP ranges for remote SSH access in CIDR notation +# enable_http: true or false, enable or disable remote HTTP access +# http_source_ranges: source IP ranges for remote HTTP access in CIDR notation +variable "security" { + description = "Security options" + type = object({ + admin = string + public_key = string + block_project_keys = bool + enable_os_login = bool + enable_local = bool + enable_ssh = bool + enable_http = bool + ssh_source_ranges = list(string) + http_source_ranges = list(string) + }) + + default = { + admin = "stack" + public_key = null + block_project_keys = false + enable_os_login = true + enable_local = false + enable_ssh = false + enable_http = false + ssh_source_ranges = [ + "0.0.0.0/0" + ] + http_source_ranges = [ + "0.0.0.0/0" + ] + } +} + +variable "network_self_link" { + description = "The self-link of the VPC network to where the system is connected. Ignored if 'network_properties' is provided. 'network_self_link' or 'network_properties' must be provided." + type = string + default = null +} + +# Network properties +# https://cloud.google.com/vpc/docs/vpc +# routing: network-wide routing mode: REGIONAL or GLOBAL +# tier: networking tier for VM interfaces: STANDARD or PREMIUM +# id: existing network id, will be using if new is false +# auto: create subnets in each region automatically: false or true +# mtu: maximum transmission unit in bytes: 1460 - 1500 +# new: create a new network or use an existing one: true or false +# nat: allow instances without external IP to communicate with the outside world: true or false +variable "network_properties" { + description = "Network options. 'network_self_link' or 'network_properties' must be provided." + type = object({ + routing = string + tier = string + id = string + auto = bool + mtu = number + new = bool + nat = bool + }) + + default = null +} + +variable "subnetwork_self_link" { + description = "The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork_properties' is provided. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." + type = string + default = null +} + +variable "subnetwork_address" { + description = "The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork_properties' is provided." + type = string + default = null +} + +# Subnetwork properties +# https://cloud.google.com/vpc/docs/vpc +# address: IP range of internal addresses for a new subnetwork +# private: when enabled VMs in this subnetwork without external +# IP addresses can access Google APIs and services by using +# Private Google Access: true or false +# https://cloud.google.com/vpc/docs/private-access-options +# id: existing subnetwork id, will be using if new is false +# new: create a new subnetwork or use an existing one: true or false +variable "subnetwork_properties" { + description = "Subnetwork properties. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." + type = object({ + address = string + private = bool + id = string + new = bool + }) + default = null +} +# Boot disk properties +# disk_type: pd-standard, pd-ssd or pd-balanced +# auto_delete: true or false +# whether the disk will be auto-deleted when the instance is deleted +variable "boot" { + description = "Boot disk properties" + type = object({ + disk_type = string + auto_delete = bool + script_url = string + }) + default = { + disk_type = "pd-standard" + auto_delete = true + script_url = null + } +} + +# Source image properties +# project: project name +# family: image family name +# name: !!DEPRECATED!! - image name +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: Source image properties" + type = any + # Omitting type checking so validation can provide more useful error message + # type = object({ + # project = string + # family = string + # }) + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + Source image properties + + Expected Fields: + name: Unavailable with this module. + family: The image family to use. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "ddn-public" + family = "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp" + } + + validation { + condition = !can(coalesce(var.instance_image.name)) + error_message = "In var.instance_image, the \"name\" field is not used, please use the \"family\" setting." + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, the \"family\" field must be a string set to the image family." + } +} + +# Management server properties +# node_type: type of management server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of management servers +variable "mgs" { + description = "Management server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-32" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 1 + } +} + +# Management target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of management target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of management target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of management target in GB (scratch disk size must be exactly 375) +# disk_count: number of management targets +# disk_raid: create striped management target, true or false +variable "mgt" { + description = "Management target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 128 + disk_count = 1 + disk_raid = false + } +} + + +# Monitoring target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of monitoring target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of monitoring target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of monitoring target in GB (scratch disk size must be exactly 375) +# disk_count: number of monitoring targets +# disk_raid: create striped monitoring target, true or false +variable "mnt" { + description = "Monitoring target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 128 + disk_count = 1 + disk_raid = false + } +} + +# Metadata server properties +# node_type: type of metadata server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of metadata servers +variable "mds" { + description = "Metadata server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-32" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 1 + } +} + +# Metadata target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of metadata target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of metadata target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of metadata target in GB (scratch disk size must be exactly 375) +# disk_count: number of metadata targets +# disk_raid: create striped metadata target, true or false +variable "mdt" { + description = "Metadata target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-ssd" + disk_size = 3500 + disk_count = 1 + disk_raid = false + } +} + +# Object Storage server properties +# node_type: type of storage server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of storage servers +variable "oss" { + description = "Object Storage server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-16" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 3 + } +} + +# Object Storage target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of storage target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of storage target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of storage target in GB (scratch disk size must be exactly 375) +# disk_count: number of storage targets +# disk_raid: create striped storage target, true or false +variable "ost" { + description = "Object Storage target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-ssd" + disk_size = 3500 + disk_count = 1 + disk_raid = false + } +} + +# Compute client properties +# node_type: type of compute client +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of compute clients +variable "cls" { + description = "Compute client properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-2" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 0 + } +} +# Compute client target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of compute target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of compute target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of compute target in GB (scratch disk size must be exactly 375) +# disk_count: number of compute targets +variable "clt" { + description = "Compute client target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 256 + disk_count = 0 + } +} +variable "local_mount" { + description = "Mountpoint (at the client instances) for this EXAScaler system" + type = string + default = "/shared" +} + +variable "prefix" { + description = "EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud')" + type = string + default = null +} + +variable "labels" { + description = "Labels to add to EXAScaler Cloud deployment. Key-value pairs." + type = map(string) + default = {} +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf new file mode 100644 index 0000000000..2981b4dd75 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/Intel-DAOS/README.md b/deletion-test/cluster/modules/embedded/community/modules/file-system/Intel-DAOS/README.md new file mode 100644 index 0000000000..04db0acb8c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/Intel-DAOS/README.md @@ -0,0 +1 @@ +> **_NOTE:_** Cluster Toolkit is dropping support for the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository. The DAOS example blueprints (`hpc-slurm-daos.yaml` and `pfs-daos.yaml`) have been removed from the Cluster Toolkit. We recommend migrating to the first-party [Parallelstore](../../../../modules/file-system/parallelstore/) module for similar functionality. To help with this transition, see the Parallelstore example blueprints ([pfs-parallelstore.yaml](../../../../examples/pfs-parallelstore.yaml) and [ps-slurm.yaml](../../../../examples/ps-slurm.yaml)). If the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository is necessary, we recommend using the last Cluster Toolkit [v1.41.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/releases/tag/v1.41.0). diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/README.md b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/README.md new file mode 100644 index 0000000000..66aaaa46af --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/README.md @@ -0,0 +1,152 @@ +## Description + +This module creates a Network File Sharing (NFS) file system based on a VM +instance and [compute disk][disk]. This file system can share directories and +files with other clients over a network. `nfs-server` can be used by +[vm-instance](../../../../modules/compute/vm-instance/README.md) and SchedMD +community modules that create compute VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +If you are using Hyperdisk storage, check the possible disk size, IOPS, and throughput values for each disk type in the [Hyperdisk limits documentation](https://cloud.google.com/compute/docs/disks/hyperdisks#limits-disk). + +> **_WARNING:_** This module has only been tested against the HPC centos7 OS +> disk image (the default). Using other images may work, but have not been +> verified. + +[disk]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk + +### Example + +```yaml +- id: homefs + source: community/modules/file-system/nfs-server + use: [network1] +``` + +This creates a NFS on a virtual machine which allow other VMs to mount the +volume as an external file system. + +> **_NOTE:_** All disks are destroyed along with the instance, during a `gcluster destroy`/`terraform destroy` event. However, you can setup data retention with `create_boot_snapshot_before_destroy` (boot disk) and `create_snapshot_before_destroy` (data disk). + +## Mounting + +To mount the NFS Server you must first ensure that the NFS client has been +installed the and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +See the [hpc-centos-ss.yaml] test config for an example of using this module +with a `vm-instance` module. + +If mounting is not automatically handled as described above, the `nfs-server` +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + - id: nfs + source: community/modules/file-system/nfs-server + use: [network1] + settings: {local_mounts: [/mnt1]} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(nfs.install_nfs_client_runner) + - $(nfs.mount_runner) + +``` + +[hpc-centos-ss.yaml]: ../../../../tools/validate_configs/test_configs/hpc-centos-ss.yaml +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 6.14 | +| [null](#requirement\_null) | >= 3.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.14 | +| [null](#provider\_null) | >= 3.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_disk.attached_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_instance.compute_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource | +| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_default_service_account.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_default_service_account) | data source | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [auto\_delete\_disk](#input\_auto\_delete\_disk) | DEPRECATED: Whether or not the NFS disk should be auto-deleted | `string` | `null` | no | +| [boot\_disk\_size](#input\_boot\_disk\_size) | Storage size in GB for the boot disk | `number` | `null` | no | +| [boot\_disk\_type](#input\_boot\_disk\_type) | Storage type for the boot disk | `string` | `null` | no | +| [create\_boot\_snapshot\_before\_destroy](#input\_create\_boot\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the boot disk | `bool` | `false` | no | +| [create\_snapshot\_before\_destroy](#input\_create\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the NFS data disk | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the NFS instance if no name is specified. | `string` | n/a | yes | +| [disk\_size](#input\_disk\_size) | Storage size in GB for the NFS data disk | `number` | `"100"` | no | +| [image](#input\_image) | DEPRECATED: The VM image used by the NFS server | `string` | `null` | no | +| [instance\_image](#input\_instance\_image) | The VM image used by the NFS server.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to the NFS instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mounts](#input\_local\_mounts) | Mountpoint for this NFS compute instance | `list(string)` | 
[
  "/data"
]
| no | +| [machine\_type](#input\_machine\_type) | Type of the VM instance to use | `string` | `"n2d-standard-2"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the NFS VM. | `string` | `"default"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [provisioned\_iops](#input\_provisioned\_iops) | Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput | `number` | `null` | no | +| [provisioned\_throughput](#input\_provisioned\_throughput) | Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme | `number` | `null` | no | +| [scopes](#input\_scopes) | Scopes to apply to the controller | `list(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [service\_account](#input\_service\_account) | Service Account for the NFS server | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the NFS VM. | `string` | `null` | no | +| [type](#input\_type) | Storage type for the NFS data disk | `string` | `"pd-ssd"` | no | +| [zone](#input\_zone) | The zone name where the NFS instance located in. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | +| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | +| [network\_storage](#output\_network\_storage) | export of all desired folder directories | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/main.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/main.tf new file mode 100644 index 0000000000..a00d2681ba --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/main.tf @@ -0,0 +1,131 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "nfs-server", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + server_ip = google_compute_instance.compute_instance.network_interface[0].network_ip + fs_type = "nfs" + mount_options = "defaults,hard,intr" + install_nfs_client_runners = [for mount in var.local_mounts : + { + "type" = "shell" + "source" = "${path.module}/scripts/install-nfs-client.sh" + "destination" = "install-nfs${replace(mount, "/", "_")}.sh" + } + ] + mount_runners = [for mount in var.local_mounts : + { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"/exports${mount}\" \"${mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(mount, "/", "_")}.sh" + } + ] + ansible_mount_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/mount.yaml" + "destination" = "mount.yaml" + } +} + +data "google_compute_default_service_account" "default" {} + +resource "google_compute_disk" "attached_disk" { + project = var.project_id + name = "${local.name}-nfs-instance-disk" + size = var.disk_size + type = var.type + zone = var.zone + labels = local.labels + provisioned_iops = var.provisioned_iops + provisioned_throughput = var.provisioned_throughput + create_snapshot_before_destroy = var.create_snapshot_before_destroy +} + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project +} + +resource "null_resource" "image" { + triggers = { + name = try(var.instance_image.name, null), + family = try(var.instance_image.family, null), + project = var.instance_image.project + } +} + +resource "google_compute_disk" "boot_disk" { + project = var.project_id + + name = "${local.name}-boot-disk" + size = var.boot_disk_size + type = var.boot_disk_type + image = data.google_compute_image.compute_image.self_link + labels = local.labels + zone = var.zone + create_snapshot_before_destroy = var.create_boot_snapshot_before_destroy + + lifecycle { + replace_triggered_by = [null_resource.image] + ignore_changes = [ + image + ] + } +} + +resource "google_compute_instance" "compute_instance" { + project = var.project_id + name = "${local.name}-nfs-instance" + zone = var.zone + machine_type = var.machine_type + + boot_disk { + auto_delete = false + source = google_compute_disk.boot_disk.self_link + device_name = google_compute_disk.boot_disk.name + } + + attached_disk { + source = google_compute_disk.attached_disk.id + device_name = "attached_disk" + } + + network_interface { + network = var.network_self_link + subnetwork = var.subnetwork_self_link + } + + service_account { + email = var.service_account == null ? data.google_compute_default_service_account.default.email : var.service_account + scopes = var.scopes + } + + metadata = var.metadata + metadata_startup_script = templatefile("${path.module}/scripts/install-nfs-server.sh.tpl", { local_mounts = var.local_mounts }) + + labels = local.labels +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/outputs.tf new file mode 100644 index 0000000000..e23b94e2b2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/outputs.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +# render the content for each folder +output "network_storage" { + description = "export of all desired folder directories" + value = [for i, mount in var.local_mounts : { + remote_mount = "/exports${mount}" + local_mount = mount + fs_type = local.fs_type + mount_options = local.mount_options + server_ip = local.server_ip + client_install_runner = local.install_nfs_client_runners[i] + mount_runner = local.mount_runners[i] + } + ] +} + +output "install_nfs_client" { + description = "Script for installing NFS client" + value = file("${path.module}/scripts/install-nfs-client.sh") +} + +output "install_nfs_client_runner" { + description = "Runner to install NFS client using the startup-script module" + value = local.install_nfs_client_runners[0] +} + +output "mount_runner" { + description = <<-EOT + Runner to mount the file-system using an ansible playbook. The startup-script + module will automatically handle installation of ansible. + - id: example-startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(your-fs-id.mount_runner) + ... + EOT + value = local.ansible_mount_runner +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl new file mode 100644 index 0000000000..1b06a5f032 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl @@ -0,0 +1,35 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -ex + +if [ ! -d "/exports" ]; then # first load, format and mount the disk + # See https://cloud.google.com/compute/docs/disks/add-persistent-disk + uuid=$(uuidgen) + mkfs.ext4 -F -m 0 -U "$uuid" -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/disk/by-id/google-attached_disk + + mkdir /exports + echo "UUID=$uuid /exports ext4 discard,defaults 0 0" >> /etc/fstab + mount --target /exports/ + + %{ for mount in local_mounts ~} + mkdir -p /exports${mount} + chmod 755 /exports${mount} + echo '/exports${mount} *(rw,sync,no_root_squash)' >> "/etc/exports" + %{ endfor ~} +fi + +systemctl start nfs-server rpcbind +systemctl enable nfs-server +exportfs -r diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml new file mode 100644 index 0000000000..f7fbe58d5e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml @@ -0,0 +1,39 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Mounts the file systems specified in the metadata network_storage key + hosts: localhost + become: true + vars: + meta_key: "network_storage" + url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" + tasks: + - name: Read metadata network_storage information + ansible.builtin.uri: + url: "{{ url }}/{{ meta_key }}" + method: GET + headers: + Metadata-Flavor: "Google" + register: storage + - name: Mount file systems + ansible.posix.mount: + src: "{{ item.server_ip }}:/{{ item.remote_mount }}" + path: "{{ item.local_mount }}" + opts: "{{ item.mount_options }}" + boot: true + fstype: "{{ item.fs_type }}" + state: "mounted" + loop: "{{ storage.json }}" diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/variables.tf new file mode 100644 index 0000000000..9a58da641e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/variables.tf @@ -0,0 +1,194 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the NFS instance if no name is specified." + type = string +} + +variable "name" { + description = "The resource name of the instance." + type = string + default = null +} + +variable "zone" { + description = "The zone name where the NFS instance located in." + type = string +} + +variable "boot_disk_size" { + description = "Storage size in GB for the boot disk" + type = number + default = null +} + +variable "boot_disk_type" { + description = "Storage type for the boot disk" + type = string + default = null +} + +variable "create_boot_snapshot_before_destroy" { + description = "Whether to create a snapshot before destroying the boot disk" + type = bool + default = false +} + +variable "disk_size" { + description = "Storage size in GB for the NFS data disk" + type = number + default = "100" +} + +variable "type" { + description = "Storage type for the NFS data disk" + type = string + default = "pd-ssd" +} + +variable "create_snapshot_before_destroy" { + description = "Whether to create a snapshot before destroying the NFS data disk" + type = bool + default = false +} + +variable "provisioned_iops" { + description = "Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput" + type = number + default = null +} + +variable "provisioned_throughput" { + description = "Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme" + type = number + default = null +} + +# Deprecated, replaced by instance_image +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: The VM image used by the NFS server" + type = string + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + The VM image used by the NFS server. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +# Deprecated, replaced by create_snapshot_before_destroy and create_boot_snapshot_before_destroy +# tflint-ignore: terraform_unused_declarations +variable "auto_delete_disk" { + description = "DEPRECATED: Whether or not the NFS disk should be auto-deleted" + type = string + default = null + + validation { + condition = var.auto_delete_disk == null + error_message = "The 'var.auto_delete_disk' setting is broken in Cluster Toolkit versions >1.25.0 and deprecated in versions >1.48.0, please use 'var.create_snapshot_before_destroy' and 'var.create_boot_snapshot_before_destroy' instead." + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the NFS VM." + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the NFS VM." + type = string + default = null +} + +variable "machine_type" { + description = "Type of the VM instance to use" + type = string + default = "n2d-standard-2" +} + +variable "labels" { + description = "Labels to add to the NFS instance. Key-value pairs." + type = map(string) +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "service_account" { + description = "Service Account for the NFS server" + type = string + default = null +} + +variable "scopes" { + description = "Scopes to apply to the controller" + type = list(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "local_mounts" { + description = "Mountpoint for this NFS compute instance" + type = list(string) + default = ["/data"] + + validation { + condition = alltrue([ + for m in var.local_mounts : substr(m, 0, 1) == "/" + ]) + error_message = "Local mountpoints have to start with '/'." + } + validation { + condition = length(var.local_mounts) > 0 + error_message = "At least one local mount must be specified in var.local_mounts." + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/versions.tf new file mode 100644 index 0000000000..63443806b8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.14" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/sycomp-scale/README.md b/deletion-test/cluster/modules/embedded/community/modules/file-system/sycomp-scale/README.md new file mode 100644 index 0000000000..79ff12bc18 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/sycomp-scale/README.md @@ -0,0 +1,35 @@ +## Description + +This document provides information on how to deploy an instance of [Sycomp Intelligent Data Storage Platform](https://sycomp.com/solution/hpc/storage/) on Google Cloud Platform ([GCP](https://cloud.google.com/)) using the Google Cluster Toolkit. + +> **_NOTE:_** +> Sycomp Storage on GCP does not require an HPC Toolkit wrapper. +> Terraform modules are sourced directly from GitLab. + +Terraform modules for Sycomp Intelligent Data Storage Platform are downloaded on deployment using the Google Cloud Toolkit. + +The Terraform module parameters are documented in the `README.md` files in the respective module directories of the source GitLab repository. The main modules are: + +- `sycomp-scale` +- `sycomp-scale-expansion` + +## Examples + +The community examples folder (community/examples/sycomp/) contains four example blueprints that you can use to deploy or expand a Sycomp Storage cluster. + +- [community/examples/sycomp/sycomp-storage.yaml][sycomp-storage-yaml] - + Blueprint for deploying a Sycomp Storage cluster consisting of 3 storage servers. + +- [community/examples/sycomp/sycomp-storage-expansion.yaml][sycomp-storage-expansion-yaml] - + Blueprint for expanding the above created cluster from 3 to 4 storage servers. + +- [community/examples/sycomp/sycomp-storage-ece.yaml][sycomp-storage-ece-yaml] - + Blueprint for deploying a Sycomp Storage cluster consisting of 7 storage servers with ECE (Erasure Code Edition) software RAID. + +- [community/examples/sycomp/sycomp-storage-slurm.yaml][sycomp-storage-slurm-yaml] - + Blueprint for deploying a Slurm cluster and Sycomp Storage cluster with 3 servers. The Slurm compute nodes are configured as NFS clients and have the ability to use the Sycomp Storage filesystem. + +[sycomp-storage-yaml]: ../../../examples/sycomp/sycomp-storage.yaml +[sycomp-storage-expansion-yaml]: ../../../examples/sycomp/sycomp-storage-expansion.yaml +[sycomp-storage-ece-yaml]: ../../../examples/sycomp/sycomp-storage-ece.yaml +[sycomp-storage-slurm-yaml]: ../../../examples/sycomp/sycomp-storage-slurm.yaml diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/README.md b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/README.md new file mode 100644 index 0000000000..0e2a936167 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/README.md @@ -0,0 +1,182 @@ +## Description + +This module provides scripts for client installation and mounting [WEKA] +filesystems. Client supports both UDP and DPDK modes and allows customization of +mount parameters using Compute VM instance metadata. + +For deploying Weka cluster please consult [WEKA installation on GCP]. + +[WEKA]: https://www.weka.io/ +[WEKA installation on GCP]: https://docs.weka.io/planning-and-installation/weka-installation-on-gcp + +## Prerequisites + +* up and running Weka cluster +* running on a [supported OS](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#operating-system) +* [open firewall](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#required-ports) + between WEKA backend servers and clients +* VPC peering configuration: + * if clients share VPCs created for WEKA cluster, no additional configuration + is necessary + * if dedicated VPCs are in use for clients, then WEKA VPCs needs to be peered + with VPCs that are used as: + * primary interface on client + * interfaces dedicated for DPDK client + * if dedicated VPCs are in use for clients, then those VPCs needs to be peered + with each other + +## Mounting +This example creates mount scripts that will mount `default` filesystem from +`10.0.0.3` WEKA backend: + +```yaml + - id: wekafs + source: community/modules/file-system/weka-client + settings: + local_mount: /scratch + server_ip: 10.0.0.3 + remote_mount: default + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: $(wekafs.runners) +``` + +If you need to add mount script along other runners, remember to add all 4 +runners provided by this script as shown in this example: + +```yaml + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(wekafs.client_install_runner) + - $(wekafs.mount_runner) + - type: shell + content: | + #!/bin/bash + + echo Sample + destination: sample-script.sh +``` + +To use the client within Slurm partition, with DPDK, remember to set additional +networks, and configure metadata. In this example, all four additional interfaces +are dedicated to WEKA DPDK + +```yaml + - id: c2_60_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + - mount-at-startup # as defined in previous examples + settings: + bandwidth_tier: virtio_enabled # Weka requires VirtIO, from WEKA 4.4.1, DPDK is also supported on gVNIC + additional_networks: + - subnetwork: weka-client-1 + nic_type: VIRTIO_NET + - subnetwork: weka-client-2 + nic_type: VIRTIO_NET + - subnetwork: weka-client-3 + nic_type: VIRTIO_NET + - subnetwork: weka-client-4 + nic_type: VIRTIO_NET + machine_type: c2-standard-60 + metadata: + weka-data_interfaces: 1,2,3,4 # allocate interfaces 1, 2, 3 and 4 to DPDK + weka-mode: dpdk + weka-options: num_cores=4,dpdk_base_memory_mb=16 + node_conf: + # From https://docs.weka.io/planning-and-installation/bare-metal/planning-a-weka-system-installation + # do not set RealMem as this is set automatically by Cluster Toolkit + CoreSpecCount: 4 + MemSpecLimit: 5120 +``` + +Due to the fact, that client installation takes ~6-7 minutes, if you use WEKA together with Slurm and do not bundle +client in the instance image, you may need to increase the timeout for startups scripts. + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + settings: + compute_startup_scripts_timeout: 600 + login_startup_scripts_timeout: 600 + ... + - id: compute_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + settings: + resume_timeout: 600 + ... +``` + +## Supported VM metadata options +Client scripts do support following metadata keys: +* `weka-mode` - one of `udp` or `dpdk`. Defaults to `udp`. Sets client mode. +* `weka-data_interfaces` - comma separated list of interface identifiers, + specifying which interfaces are dedicated for data plane. Set to `1` to + dedicate second interface of instance for WEKA DPDK. Set to `2,5` to dedicate + third and sixth interface of instance for WEKA DPDK. +* `weka-mgmt_interface` - identifier of management interface, defaults to `0`, + which means to use primary interface as management interface. +* `weka-options` - additional [mount command options](https://docs.weka.io/weka-filesystems-and-object-stores/mounting-filesystems#mount-command-options) + to pass to `mount` command + +## Adding client to the OS image +To save time during the mount command install and precompile DPDK drivers in the +OS image. Following scripts compiles DPDK driver for currently running kernel. + +```shell +#!/bin/bash + +set -e -o pipefail + +echo Downloading and installing Weka client +curl --max-time 10 "{{ weka backend endpoint }}/dist/v1/install" | sh +WEKA_VERSION=$(weka -v | sed -e 's/^[^0-9]*//') +echo Installing Weka version: ${WEKA_VERSION} +weka version get "${WEKA_VERSION}" +weka version set "${WEKA_VERSION}" +# run setup for the second time, if it fails for the first time +weka local setup weka || weka local setup weka +weka version prepare "${WEKA_VERSION}" +weka local stop +weka local rm -f --all +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | +| [mount\_options](#input\_mount\_options) | Mount options for filesystem shared by all clients. | `string` | `""` | no | +| [remote\_mount](#input\_remote\_mount) | Weka filesystem name. | `string` | n/a | yes | +| [server\_ip](#input\_server\_ip) | Weka backend IP address used for bootstrapping. | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_install\_runner](#output\_client\_install\_runner) | Ansible runner that performs client installation needed to use file system. | +| [mount\_runner](#output\_mount\_runner) | Ansible runner that mounts the file system. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/metadata.yaml new file mode 100644 index 0000000000..419bc3fe46 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/outputs.tf new file mode 100644 index 0000000000..0bd9098d80 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/outputs.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + template_args = { + local_mount = var.local_mount + mount_options = var.mount_options == "" ? "" : "-o ${var.mount_options}" + remote_mount = var.remote_mount + server_ip = var.server_ip + service_name = "weka-mount${replace(var.local_mount, "/", "-")}" + } + mount_script = templatefile("${path.module}/templates/mount-weka.sh.tftpl", local.template_args) + + mount_runner_ansible = { + type = "ansible-local" + content = templatefile( + "${path.module}/templates/mount-weka.yaml.tftpl", + merge( + local.template_args, + { mount_weka_script = local.mount_script } + ) + ) + destination = "mount_filesystem${replace(var.local_mount, "/", "_")}.yaml" + } + + client_install_runner = { + type = "ansible-local" + content = templatefile("${path.module}/templates/install-weka-client.yaml.tftpl", local.template_args) + destination = "install_filesystem${replace(var.local_mount, "/", "_")}.yaml" + } +} + +# currently WEKA mounts are not compatible with network_storage logic, as WEKA volumes needs to be mounted by +# systemd script and not /etc/fstab entry, as the mount command needs to have network configuration which may change +# between restarts +# +#output "network_storage" { +# description = "Describes a remote network storage to be mounted by fs-tab." +# value = { +# server_ip = var.server_ip +# remote_mount = var.remote_mount +# local_mount = var.local_mount +# fs_type = var.fs_type +# mount_options = var.mount_options +# client_install_runner = local.client_install_runner +# mount_runner = local.mount_runner +# } +#} +# +output "client_install_runner" { + description = "Ansible runner that performs client installation needed to use file system." + value = local.client_install_runner +} + +output "mount_runner" { + description = "Ansible runner that mounts the file system." + value = local.mount_runner_ansible +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl new file mode 100644 index 0000000000..ddc3acdb5d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl @@ -0,0 +1,133 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Mounts the file systems specified in the metadata network_storage key + hosts: localhost + become: true + vars: + meta_key: "network_storage" + url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" + tasks: + - name: Check if weka is installed + ansible.builtin.stat: + path: /usr/bin/weka + register: weka_binary + + - name: Create temporary location for installation script + ansible.builtin.tempfile: + state: file + register: + install_script + when: not weka_binary.stat.exists + + - name: Download WEKA client + ansible.builtin.get_url: + url: http://${server_ip}:14000/dist/v1/install + dest: "{{ install_script.path }}" + mode: "700" + when: not weka_binary.stat.exists + + - name: Run WEKA installation script + ansible.builtin.shell: + cmd: "{{ install_script.path }}" + when: not weka_binary.stat.exists + register: weka_install_result + changed_when: weka_install_result.rc == 0 + + - name: Read metadata network_storage information + ansible.builtin.uri: + url: "{{ url }}/weka-version" + method: GET + headers: + Metadata-Flavor: "Google" + status_code: + - 200 + - 404 + register: get_weka_version + + - name: Set WEKA version from metadata server + ansible.builtin.set_fact: + weka_version: "{{ get_weka_version.body }}" + when: get_weka_version.status == 200 + + - name: Get version of WEKA installation client + ansible.builtin.shell: + cmd: weka -v | sed -e 's/^[^0-9.]*\([0-9.]*\)[^0-9.]*$/\1/' + register: get_weka_client_version + changed_when: get_weka_client_version.rc == 0 + + - name: Set WEKA version from WEKA installation client + ansible.builtin.set_fact: + weka_version: "{{ get_weka_client_version.stdout }}" + when: get_weka_version.status == 404 + + - name: Download user-defined WEKA version + ansible.builtin.shell: + cmd: weka version get {{ weka_version }} + register: result + changed_when: result.rc == 0 + + - name: Set user-defined WEKA version + ansible.builtin.shell: + cmd: weka version set {{ weka_version }} + register: result + changed_when: result.rc == 0 + + - name: Setup WEKA client + ansible.builtin.shell: + cmd: weka local setup weka + register: setup_1_result + changed_when: setup_1_result.rc == 0 + failed_when: false # ignore errors + + - name: Setup WEKA client (2nd try) + ansible.builtin.shell: + cmd: weka local setup weka + register: result + changed_when: result.rc == 0 + when: setup_1_result.rc != 0 + + - name: Prepare WEKA version + ansible.builtin.shell: + cmd: weka version prepare {{ weka_version }} + register: result + changed_when: result.rc == 0 + + - name: Stop WEKA client + ansible.builtin.shell: + cmd: weka local stop + async: 30 + poll: 10 + register: weka_stop + changed_when: weka_stop.get("rc") == 0 # when killed by async, rc is not defined + failed_when: false # ignore errors + + - name: Stop WEKA client (2nd try) + ansible.builtin.shell: + cmd: weka local stop + async: 30 + poll: 10 + register: result + changed_when: result.rc == 0 + failed_when: false # ignore errors + when: weka_stop.get("rc") != 0 + + - name: Remove WEKA containers + ansible.builtin.shell: + cmd: weka local rm -f --all + register: result + changed_when: result.rc == 0 + failed_when: false # ignore errors diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl new file mode 100644 index 0000000000..19c6dc1fdc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl @@ -0,0 +1,101 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# shellcheck disable=SC2034 +METADATA_BASE_URL="http://metadata.google.internal/computeMetadata/v1/instance" +# shellcheck disable=SC2034 +ATTR_URL="$${METADATA_BASE_URL}/attributes/weka-" +NET_URL="$${METADATA_BASE_URL}/network-interfaces" + +# shellcheck disable=SC1083 +WEKA_MODE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mode || echo -n udp) +# shellcheck disable=SC1083 +WEKA_DATA_INTERFACES=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}data_interfaces || exit 0) +# shellcheck disable=SC1083 +WEKA_MGMT_INTERFACE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mgmt_interface || echo -n 0) +# shellcheck disable=SC1083 +WEKA_OPTIONS=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}options || exit 0) + +WEKA_OPTIONS="$${WEKA_OPTIONS:+-o $WEKA_OPTIONS}" + +netmask_to_cidr () { + c=0 + # shellcheck disable=SC2086,SC1083 + x=0$( printf '%o' $${1//./ } ) + while [ "$x" -gt 0 ]; do + c=$(( c + x%2 )) + x=$(( x >> 1)) + done + echo $c ; +} + +# detect network interface naming scheme +if [[ -e /sys/class/net/eth0 ]] ; then + DEVICE_NAME="eth" + DEVICE_INDEX_BASE=0 +elif [[ -e /sys/class/net/ens4 ]] ; then + DEVICE_NAME="ens" + DEVICE_INDEX_BASE=4 +else + echo "Can't detect device names. Both /sys/class/net/eth0 and /sys/class/net/ens4 do not exists" + exit 1 +fi + +# ensure that /etc/hosts contains entry for hostname pointing to primary interface +NEW_IP=$(ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) | head -n 1 | sed -e 's/^.*inet \([0-9\.]\+\)\/.*$/\1/') +if [ -n "$NEW_IP" ] ; then + HOSTNAME=$(hostname) + sed -i -e "/$HOSTNAME/s/^[0-9\.]\+ $HOSTNAME/$NEW_IP $HOSTNAME/" /etc/hosts +else + echo "Failed to find primary interface address" + ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) + exit 1 +fi + +# shellcheck disable=SC2154 +echo "Mounting Weka ${server_ip}/${remote_mount} to ${local_mount}" +mkdir -p "${local_mount}" +service weka-agent start +if [[ $WEKA_MODE == "udp" ]] ; then + # shellcheck disable=SC2086,SC2154,SC2086 + mount -t wekafs ${mount_options} -o net=udp $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" + +elif [[ $WEKA_MODE == "dpdk" ]] ; then + declare -a DATA_INTERFACES + # split WEKA_DATA_INTERFACES by comma into array + # shellcheck disable=SC2034 + IFS=',' read -r -a DATA_INTERFACES <<< "$WEKA_DATA_INTERFACES" + + DATA_OPTIONS="" + # shellcheck disable=SC2066 + for interface in "$${DATA_INTERFACES[@]}" ; do + INTERFACE_IP=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/ip") + INTERFACE_MASK=$(netmask_to_cidr "$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/subnetmask")") + INTERFACE_GATEWAY=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/gateway") + + DATA_OPTIONS+="-o net=$DEVICE_NAME$((DEVICE_INDEX_BASE + interface))/$INTERFACE_IP/$INTERFACE_MASK/$INTERFACE_GATEWAY " + done + + # shellcheck disable=SC2086 + mount -t wekafs \ + ${mount_options} \ + -o mgmt_ip="$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$WEKA_MGMT_INTERFACE/ip")" \ + $DATA_OPTIONS $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" +else + echo "Unknown weka:mode metadata value: $${WEKA_MODE}. Allowed values: udp and dpdk" + exit 1 +fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl new file mode 100644 index 0000000000..84587103a9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl @@ -0,0 +1,54 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Mount the WEKA file systems + hosts: localhost + become: true + vars: + local_mount: "${local_mount}" + remote_mount: "${remote_mount}" + server_ip: "${server_ip}" + service_name: "weka-mount-${replace(local_mount, "/", "_")}" + tasks: + - name: Create mount script + ansible.builtin.copy: + dest: "/etc/{{ service_name }}.sh" + mode: "0755" + content: | + ${indent(8, mount_weka_script)} + + - name: Create systemd service for weka mount + ansible.builtin.copy: + dest: "/etc/systemd/system/{{ service_name }}.service" + mode: "0644" + content: | + [Install] + WantedBy=multi-user.target + [Unit] + Description=Mount Weka {{ server_ip }}/{{ remote_mount }} at {{ local_mount }} + After=network-online.target + Wants=network-online.target + [Service] + RemainAfterExit=true + Type=oneshot + ExecStart=/bin/bash -c "/etc/{{ service_name }}.sh" + + - name: Enable and start weka mount service + ansible.builtin.systemd: + name: "{{ service_name }}" + daemon_reload: true + enabled: true + state: started diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/variables.tf new file mode 100644 index 0000000000..f07961d64c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/variables.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "local_mount" { + description = "The mount point where the contents of the device may be accessed after mounting." + type = string + default = "/mnt" +} + +variable "mount_options" { + description = "Mount options for filesystem shared by all clients." + type = string + default = "" + nullable = false +} + +variable "remote_mount" { + description = "Weka filesystem name." + type = string +} + +variable "server_ip" { + description = "Weka backend IP address used for bootstrapping." + type = string + default = "" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/versions.tf new file mode 100644 index 0000000000..9e6af1fa7f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.14.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb new file mode 100644 index 0000000000..f13726f691 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "project_id = \"${project_id}\"\n", + "dataset_id = \"${dataset_id}\"\n", + "table_id = \"${table_id}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ONI1Xo0-KtAD", + "outputId": "fb9ca475-e4ec-4cd0-e0e6-14f409eefd7a" + }, + "outputs": [], + "source": [ + "from google.cloud import bigquery\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "\n", + "client = bigquery.Client(project=project_id)\n", + "\n", + "df = client.query(f'''\n", + "SELECT ticker, cast(price AS FLOAT64) AS price, CAST(OFFSET as INTEGER) AS offset, start_date, end_date, iteration\n", + "FROM `{project_id}.{dataset_id}.{table_id}`,\n", + "UNNEST(simulation_results) as NUMERIC with OFFSET\n", + "WHERE epoch_time IN\n", + " # Get the latest simulation runs for each Ticker Symbol\n", + "(SELECT MAX(epoch_time) FROM `{project_id}.{dataset_id}.{table_id}` GROUP BY ticker)\n", + "'''\n", + ").to_dataframe()\n", + "# Display the data\n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define a function to plot the data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def plot_ticker(t,df):\n", + "\n", + " dtf = df[(df.ticker==t) &(df.offset == 250)].price.describe(include=[np.float64], percentiles=[.05, .01, .001])\n", + " cellText = []\n", + " for v in dtf.values:\n", + " cellText.append([v])\n", + " \n", + " pltf = df[df.ticker==t].pivot(index='offset', columns='iteration', values='price')\n", + " \n", + " fig = plt.figure(figsize=(10,5))\n", + " ax1 = fig.add_subplot(122)\n", + " pltf.plot(legend=False, ax=ax1, xlabel='Time(days)', ylabel='US$', title=f\"{ df[(df.ticker == t) & (df.offset == 0) & (df.iteration == 4)]}\")\n", + " ax2 = fig.add_subplot(121)\n", + " font_size=10\n", + " bbox=[0, 0, .5, 1]\n", + " ax2.axis('off')\n", + " mpl_table = ax2.table(cellText = cellText, rowLabels=dtf.index.values, bbox=bbox)\n", + " mpl_table.auto_set_font_size(False)\n", + " mpl_table.set_fontsize(font_size)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 808 + }, + "id": "jvBmb_KceX7z", + "outputId": "42a3ba9f-b68f-4c7b-d928-0fedeed9216c" + }, + "outputs": [], + "source": [ + "ticker_list = df.ticker.unique()\n", + "for t in ticker_list:\n", + " plot_ticker(t,df)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md new file mode 100644 index 0000000000..e54893a1bb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md @@ -0,0 +1,97 @@ +## Description + +Copy files to a target GCS bucket. + +Primarily used for FSI - MonteCarlo Tutorial **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: +../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage +This copies the module files to the specified GCS bucket. It is expected that +the bucket will be mounted on the target VM. + +Some of the files are templates, and `main.tf` translates the files with the +passed variable values. This way the user does not have to change things like +pointing to the correct bigquery table or adding in the project_id. + +```yaml + - id: fsi_tutorial_files + source: community/modules/files/fsi-montecarlo-on-batch + use: [bq-dataset, bq-table, fsi_bucket, pubsub_topic] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 3.83 | +| [http](#requirement\_http) | ~> 3.0 | +| [random](#requirement\_random) | ~> 3.0 | +| [template](#requirement\_template) | ~> 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [http](#provider\_http) | ~> 3.0 | +| [random](#provider\_random) | ~> 3.0 | +| [template](#provider\_template) | ~> 2.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.get_iteration_sh](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.get_mc_reqs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.get_requirements](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.ipynb_obj_fsi](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.mc_obj_yaml](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.mc_run](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.run_batch_py](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [http_http.batch_py](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | +| [http_http.batch_requirements](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | +| [template_file.ipynb_fsi](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | +| [template_file.mc_run_py](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | +| [template_file.mc_run_yaml](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Bigquery dataset id | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to run project | `string` | n/a | yes | +| [table\_id](#input\_table\_id) | Bigquery table id | `string` | n/a | yes | +| [topic\_id](#input\_topic\_id) | Pubsub Topic Name | `string` | n/a | yes | +| [topic\_schema](#input\_topic\_schema) | Pubsub Topic schema | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh new file mode 100644 index 0000000000..50aa865a31 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ticker=("GOOG" "AMZN" "MSFT" "NVDA" "META" "TSLA" "PEP" "COST") +echo "BI: $BATCH_TASK_INDEX" +echo "TI: ${ticker[$BATCH_TASK_INDEX]}" +python3 -m pip install -r /mnt/disks/fsi/mc_run_reqs.txt +python3 /mnt/disks/fsi/mc_run.py \ + --ticker "${ticker[$BATCH_TASK_INDEX]}" \ + --iterations 500 \ + --start_date 2022-01-01 diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf new file mode 100644 index 0000000000..83dc7fe9cf --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf @@ -0,0 +1,102 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + bucket = replace(var.gcs_bucket_path, "gs://", "") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +data "template_file" "mc_run_py" { + template = file("${path.module}/mc_run.tpl.py") + vars = { + project_id = var.project_id + topic_id = var.topic_id + topic_schema = var.topic_schema + dataset_id = var.dataset_id + table_id = var.table_id + } +} + +resource "google_storage_bucket_object" "mc_run" { + name = "mc_run.py" + content = data.template_file.mc_run_py.rendered + bucket = local.bucket +} + +data "template_file" "mc_run_yaml" { + template = file("${path.module}/mc_run.tpl.yaml") + vars = { + project_id = var.project_id + bucket_name = local.bucket + region = var.region + } +} + +resource "google_storage_bucket_object" "mc_obj_yaml" { + name = "mc_run.yaml" + content = data.template_file.mc_run_yaml.rendered + bucket = local.bucket +} + +data "template_file" "ipynb_fsi" { + template = file("${path.module}/FSI_MonteCarlo.ipynb") + vars = { + project_id = var.project_id + dataset_id = var.dataset_id + table_id = var.table_id + } +} +resource "google_storage_bucket_object" "ipynb_obj_fsi" { + name = "FSI_MonteCarlo.ipynb" + content = data.template_file.ipynb_fsi.rendered + bucket = local.bucket +} + +data "http" "batch_py" { + url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/batch.py" +} + +resource "google_storage_bucket_object" "run_batch_py" { + name = "batch.py" + content = data.http.batch_py.response_body + bucket = local.bucket +} + +data "http" "batch_requirements" { + url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/requirements.txt" +} + +resource "google_storage_bucket_object" "get_requirements" { + name = "requirements.txt" + content = data.http.batch_requirements.response_body + bucket = local.bucket +} + +resource "google_storage_bucket_object" "get_iteration_sh" { + name = "iteration.sh" + content = file("${path.module}/iteration.sh") + bucket = local.bucket +} + +resource "google_storage_bucket_object" "get_mc_reqs" { + name = "mc_run_reqs.txt" + content = file("${path.module}/mc_run_reqs.txt") + bucket = local.bucket +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py new file mode 100644 index 0000000000..4e0a64e363 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Run MC simulation for VaR portfolio risk +""" + +import avro.schema +import io +import google.auth +import numpy +import time +import yfinance as yf + +from absl import app +from absl import flags +from avro.io import DatumWriter, BinaryEncoder, BinaryDecoder, DatumReader +from datetime import datetime +from datetime import timedelta +from google.cloud import pubsub_v1, bigquery +from google.cloud.pubsub import SchemaServiceClient + +PROJECT_ID = '${project_id}' +INCOMING_TOPIC_ID = '${topic_id}' +INCOMING_TOPIC_SCHEMA = '${topic_schema}' +DATASET_ID = '${dataset_id}' +TABLE_ID = '${table_id}' + + +FLAGS = flags.FLAGS + +flags.DEFINE_string("ticker", 'GOOG', "Nasdaq Stock Ticker to run, default GOOG") +flags.DEFINE_string("start_date", '2022-01-01' , "Start data for data query, default 2022-01-01") +flags.DEFINE_integer("calendar_days", 365 , "How many calendar days to include in the calculation") +flags.DEFINE_integer("epoch_time", f'{int(time.time())}' , "Epoch time, number of seconds since January 1st, 1970 at 00:00:00 UTC.") +flags.DEFINE_integer("iterations", 100 , "Number of iterations to run.") +flags.DEFINE_boolean("print_raw", False, "Dump raw data.") + +class VaRSimulator: + + def __init__(self): + pass + + def get_data(self): + self.get_historical_data_yahoo() + + def get_historical_data_yahoo(self): + + # get historical market data: https://pypi.org/project/yfinance/ + + self.raw_data = yf.Ticker(self.ticker).history(start=self.start_date, end=self.end_date ) + self.data = self.raw_data.Close + + def print_raw(self): + print(self.get_stats()) + print(type(self.raw_data)) + print(self.raw_data) + + def get_stats(self): + close = self.data + self.first = close[0] + self.last = close[-1] + self.trading_days = len(close) + self.cagr = (self.last / self.first) ** (365.0/self.calendar_days) -1.0 + self.volatility = self.data.pct_change().std() + return(self.first, self.last, self.trading_days, self.cagr, self.volatility) + + def run_simulation(self): + + returns = numpy.random.normal(self.cagr/self.trading_days, self.volatility, self.trading_days) + 1 + returns = numpy.insert(returns,0,1.0) + self.simulation_results = self.last * returns.cumprod() + return(self.simulation_results) + + def create_object(self): + self.object = { + "ticker": self.ticker, + "epoch_time": self.epoch_time, + "iteration": self.iteration, + "start_date": self.start_date, + "end_date": self.end_date, + "simulation_results": list(map(lambda x: {"price":x}, self.simulation_results)) + } + return(self.object) + + +class PubsubToBiquery: + + def __init__(self): + + the_time = int(time.time()) + + self.project_id = PROJECT_ID + + self.publisher_client = pubsub_v1.PublisherClient() + self.topic_path = self.publisher_client.topic_path(self.project_id, INCOMING_TOPIC_ID) + + self.schema_client = SchemaServiceClient() + self.schema_path = self.schema_client.schema_path(self.project_id, INCOMING_TOPIC_SCHEMA) + + pubsub_schema = self.schema_client.get_schema(request={"name": self.schema_path}) + avro_schema = avro.schema.parse(pubsub_schema.definition) + + self.writer = DatumWriter(avro_schema) + + + def publish_record(self,record): + + byte_stream = io.BytesIO() + encoder = BinaryEncoder(byte_stream) + self.writer.write(record, encoder) + data = byte_stream.getvalue() + byte_stream.flush() + future = self.publisher_client.publish(self.topic_path, data) + if(FLAGS.print_raw): + print(f"Published message ID: {future.result()}") + + +def main(argv): + + vr = VaRSimulator() + pbbq = PubsubToBiquery() + + vr.ticker =FLAGS.ticker + vr.start_date =FLAGS.start_date + vr.end_date =f'{(datetime.strptime(FLAGS.start_date,"%Y-%m-%d") + timedelta(days = FLAGS.calendar_days)).date()}' + vr.calendar_days = FLAGS.calendar_days + vr.epoch_time = FLAGS.epoch_time + vr.iteration = 1 + + vr.get_data() + vr.get_stats() + + for i in range(FLAGS.iterations): + vr.iteration = i + vr.run_simulation() + pbbq.publish_record(vr.create_object()) + + if(FLAGS.print_raw): + vr.print_raw() + + +if __name__ == "__main__": + """ This is executed when run from the command line """ + app.run(main) diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml new file mode 100644 index 0000000000..7f7de4840b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +project_id: "${project_id}" +region: "${region}" + +job_prefix: 'fsi-' +machine_type: "n2-standard-2" +volumes: +- {bucket_name: "${bucket_name}", gcs_path: "/mnt/disks/fsi"} + +container: + image_uri: "python" + entry_point: "/bin/bash" + commands: ["/mnt/disks/fsi/iteration.sh", "$BATCH_TASK_INDEX"] + +task_count: 8 #optional +parallelism: 4 #optional +task_count_per_node: 2 #optional +cpu_milli: 1000 #optional +memory_mib: 102400 #optional + + +labels: + env: "monte" + type: "carlo" diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt new file mode 100644 index 0000000000..105ed70ad2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt @@ -0,0 +1,9 @@ +absl-py +avro +google-auth +google-cloud +google-cloud-batch +google-cloud-pubsub +google-cloud-bigquery +yfinance +PyYAML diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml new file mode 100644 index 0000000000..268c8faa9a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [storage.googleapis.com] diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf new file mode 100644 index 0000000000..eddf3c9478 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which GCS bucket will be created." + type = string +} + +variable "gcs_bucket_path" { + description = "Bucket name" + type = string + default = null +} + +variable "topic_id" { + description = "Pubsub Topic Name" + type = string +} + +variable "topic_schema" { + description = "Pubsub Topic schema" + type = string +} + +variable "dataset_id" { + description = "Bigquery dataset id" + type = string +} + +variable "table_id" { + description = "Bigquery table id" + type = string +} + +variable "region" { + description = "Region to run project" + type = string +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf new file mode 100644 index 0000000000..86dcb4dc52 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + http = { + source = "hashicorp/http" + version = "~> 3.0" + } + template = { + source = "hashicorp/template" + version = "~> 2.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md new file mode 100644 index 0000000000..ae8462d763 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md @@ -0,0 +1,100 @@ +# Module: Slurm Instance + + + +- [Module: Slurm Instance](#module-slurm-instance) + - [Overview](#overview) + - [Module API](#module-api) + + + +## Overview + +This module creates a [compute instance](../../../../docs/glossary.md#vm) from +[instance template](../../../../docs/glossary.md#instance-template) for a +[Slurm cluster](../slurm_cluster/README.md). + +> **NOTE:** This module is only intended to be used by Slurm modules. For +> general usage, please consider using: +> +> - [terraform-google-modules/vm/google//modules/compute_instance](https://registry.terraform.io/modules/terraform-google-modules/vm/google/latest/submodules/compute_instance). +> **WARNING:** The source image is not modified. Make sure to use a compatible +> source image. + +## Module API + +For the terraform module API reference, please see +[README_TF.md](./README_TF.md). + + +Copyright (C) SchedMD LLC. +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.0 | +| [google](#requirement\_google) | >= 3.43 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.43 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_from_template.slurm_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | +| [null_resource.replace_trigger](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_instance_template.base](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | +| [hostname](#input\_hostname) | Hostname of instances | `string` | n/a | yes | +| [instance\_template](#input\_instance\_template) | Instance template self\_link used to create compute instances | `string` | n/a | yes | +| [network](#input\_network) | Network to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | +| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | +| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | +| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | +| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances | `list(string)` | `[]` | no | +| [subnetwork](#input\_subnetwork) | Subnet to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The project that subnetwork belongs to | `string` | `null` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [available\_zones](#output\_available\_zones) | List of available zones in region | +| [instances\_details](#output\_instances\_details) | List of all details for compute instances | +| [instances\_self\_links](#output\_instances\_self\_links) | List of self-links for compute instances | +| [names](#output\_names) | List of available zones in region | +| [slurm\_instances](#output\_slurm\_instances) | List of all resource objects for compute instances | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf new file mode 100644 index 0000000000..2af9008a0e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf @@ -0,0 +1,126 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########## +# LOCALS # +########## + +locals { + num_instances = length(var.static_ips) == 0 ? var.num_instances : length(var.static_ips) + + # local.static_ips is the same as var.static_ips with a dummy element appended + # at the end of the list to work around "list does not have any elements so cannot + # determine type" error when var.static_ips is empty + static_ips = concat(var.static_ips, ["NOT_AN_IP"]) + + network_interfaces = [for index in range(local.num_instances) : + concat([ + { + access_config = var.access_config + alias_ip_range = [] + ipv6_access_config = [] + network = var.network + network_ip = length(var.static_ips) == 0 ? "" : element(local.static_ips, index) + nic_type = null + queue_count = null + stack_type = null + subnetwork = var.subnetwork + subnetwork_project = var.subnetwork_project + } + ], + var.additional_networks + ) + ] +} + +################ +# DATA SOURCES # +################ + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +data "google_compute_instance_template" "base" { + project = var.project_id + name = var.instance_template +} + +############# +# INSTANCES # +############# +resource "null_resource" "replace_trigger" { + triggers = { + trigger = var.replace_trigger + } +} + +# TODO: `internal/slurm-gcp/login` is ONLY user of `internal/slurm-gcp/instance` +# Remove this module, add functionality (+ prune generality) to the login module directly. +resource "google_compute_instance_from_template" "slurm_instance" { + count = local.num_instances + name = format("%s-%s", var.hostname, format("%03d", count.index + 1)) + project = var.project_id + zone = var.zone == null ? data.google_compute_zones.available.names[count.index % length(data.google_compute_zones.available.names)] : var.zone + + allow_stopping_for_update = true + + dynamic "network_interface" { + for_each = local.network_interfaces[count.index] + iterator = nic + content { + dynamic "access_config" { + for_each = nic.value.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = nic.value.alias_ip_range + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + dynamic "ipv6_access_config" { + for_each = nic.value.ipv6_access_config + iterator = access_config + content { + network_tier = access_config.value.network_tier + } + } + network = nic.value.network + network_ip = nic.value.network_ip + nic_type = nic.value.nic_type + queue_count = nic.value.queue_count + subnetwork = nic.value.subnetwork + subnetwork_project = nic.value.subnetwork_project + } + } + + source_instance_template = data.google_compute_instance_template.base.self_link + # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 + # we have to explicitly override instance labels instead of inheriting them from template. + labels = data.google_compute_instance_template.base.labels + + + lifecycle { + replace_triggered_by = [null_resource.replace_trigger.id] + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf new file mode 100644 index 0000000000..4eba78a7e8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf @@ -0,0 +1,41 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "slurm_instances" { + description = "List of all resource objects for compute instances" + value = google_compute_instance_from_template.slurm_instance +} + +output "instances_self_links" { + description = "List of self-links for compute instances" + value = google_compute_instance_from_template.slurm_instance[*].self_link +} + +output "instances_details" { + description = "List of all details for compute instances" + value = google_compute_instance_from_template.slurm_instance[*] +} + +output "available_zones" { + description = "List of available zones in region" + value = data.google_compute_zones.available.names +} + +output "names" { + description = "List of available zones in region" + value = google_compute_instance_from_template.slurm_instance[*].name +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf new file mode 100644 index 0000000000..11111a2c05 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf @@ -0,0 +1,119 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The GCP project ID" + default = null +} + +variable "network" { + description = "Network to deploy to. Only one of network or subnetwork should be specified." + type = string + default = "" +} + +variable "subnetwork" { + description = "Subnet to deploy to. Only one of network or subnetwork should be specified." + type = string + default = "" +} + +variable "subnetwork_project" { + description = "The project that subnetwork belongs to" + type = string + default = null +} + +variable "hostname" { + description = "Hostname of instances" + type = string +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })) + nullable = false +} + +variable "static_ips" { + description = "List of static IPs for VM instances" + type = list(string) + default = [] +} + +variable "access_config" { + description = "Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet." + type = list(object({ + nat_ip = string + network_tier = string + })) + default = [] +} + +variable "num_instances" { + description = "Number of instances to create. This value is ignored if static_ips is provided." + type = number + default = 1 +} + +variable "instance_template" { + description = "Instance template self_link used to create compute instances" + type = string +} + +variable "region" { + description = "Region where the instances should be created." + type = string + default = null +} + +variable "zone" { + description = "Zone where the instances should be created. If not specified, instances will be spread across available zones in the region." + type = string + default = null +} + +######### +# SLURM # +######### + +variable "replace_trigger" { + description = "Trigger value to replace the instances." + type = string + default = "" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf new file mode 100644 index 0000000000..a3e84c09bf --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf @@ -0,0 +1,31 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 1.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.43" + } + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md new file mode 100644 index 0000000000..87394bef6a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md @@ -0,0 +1,87 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.0 | +| [local](#requirement\_local) | ~> 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | ~> 2.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_template](#module\_instance\_template) | ../internal_instance_template | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.startup](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    disk_type                  = optional(string)
    disk_size_gb               = optional(number)
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `virtio_enabled` setting will only enable VirtioNet and will not enable TIER\_1.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard. | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See
https://cloud.google.com/compute/docs/gpus more details.
- type : the GPU type
- count : number of GPUs | 
object({
    type  = string
    count = number
  })
| `null` | no | +| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"n1-standard-1"` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Prefix for template resource. | `string` | `"default"` | no | +| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network
attribute for Legacy or Auto subnetted networks and subnetwork for custom
subnetted networks. | `string` | `null` | no | +| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | +| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the instances. See
'main.tf:local.service\_account' for the default. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming. | `string` | n/a | yes | +| [slurm\_instance\_role](#input\_slurm\_instance\_role) | Slurm instance type. Must be one of: controller; login; compute; or null. | `string` | n/a | yes | +| [source\_image](#input\_source\_image) | Source disk image. | `string` | `""` | no | +| [source\_image\_family](#input\_source\_image\_family) | Source image family. | `string` | `""` | no | +| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. If it is not provided, the provider project is used. | `string` | `""` | no | +| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must
exist in the same region this instance will be created in. Either network or
subnetwork must be provided. | `string` | `null` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [termination\_action](#input\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_template](#output\_instance\_template) | Instance template details | +| [labels](#output\_labels) | Labels attached to the instance template | +| [name](#output\_name) | Name of instance template | +| [self\_link](#output\_self\_link) | Self\_link of instance template | +| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | +| [tags](#output\_tags) | Tags that will be associated with instance(s) | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted new file mode 100644 index 0000000000..2edaa942d2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted @@ -0,0 +1,169 @@ +#!/bin/bash +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SLURM_DIR=/slurm +FLAGFILE=$SLURM_DIR/slurm_configured_do_not_remove +SCRIPTS_DIR=$SLURM_DIR/scripts +if [[ -z "$HOME" ]]; then + # google-startup-scripts.service lacks environment variables + HOME="$(getent passwd "$(whoami)" | cut -d: -f6)" +fi + +# Temporary workaround for transition period when some of older images +# don't have "baked in" python yet. +# TODO: Remove +SLURM_PY="/slurm/python/venv/bin/python3.13" +SYSTEM_PY="/usr/bin/python3" +if [[ ! -e "$SLURM_PY" ]]; then + echo "Symlink $SLURM_PY does not exist. Creating symlink to $SYSTEM_PY" + mkdir -p /slurm/python/venv/bin + ln -s "$SYSTEM_PY" "$SLURM_PY" +fi + +METADATA_SERVER="metadata.google.internal" +URL="http://$METADATA_SERVER/computeMetadata/v1" +CURL="curl -sS --fail --header Metadata-Flavor:Google" + +PING_METADATA="ping -q -w1 -c1 $METADATA_SERVER" +echo "INFO: $PING_METADATA" +for i in $(seq 10); do + [ $i -gt 1 ] && sleep 5; + $PING_METADATA > /dev/null && s=0 && break || s=$?; + echo "ERROR: Failed to contact metadata server, will retry" +done +if [ $s -ne 0 ]; then + echo "ERROR: Unable to contact metadata server, aborting" + wall -n '*** Slurm setup failed in the startup script! see `journalctl -u google-startup-scripts` ***' + exit 1 +else + echo "INFO: Successfully contacted metadata server" +fi + +PING_GOOGLE="ping -q -w1 -c1 8.8.8.8" +echo "INFO: $PING_GOOGLE" +for i in $(seq 5); do + [ $i -gt 1 ] && sleep 2; + $PING_GOOGLE > /dev/null && s=0 && break || s=$?; + echo "failed to ping Google DNS, will retry" +done +if [ $s -ne 0 ]; then + echo "WARNING: No internet access detected" +else + echo "INFO: Internet access detected" +fi + +mkdir -p $SCRIPTS_DIR +UNIVERSE_DOMAIN="$($CURL $URL/instance/attributes/universe_domain)" +BUCKET="$($CURL $URL/instance/attributes/slurm_bucket_path)" +if [[ -z $BUCKET ]]; then + echo "ERROR: No bucket path detected." + exit 1 +fi + +SCRIPTS_ZIP="$HOME/slurm-gcp-scripts.zip" +export CLOUDSDK_CORE_UNIVERSE_DOMAIN="$UNIVERSE_DOMAIN" + +INSTANCE_ROLE="$($CURL $URL/instance/attributes/slurm_instance_role)" + +if [ "$INSTANCE_ROLE" == "controller" ]; then + DEVEL_ZIP="slurm-gcp-devel-controller.zip" +else + DEVEL_ZIP="slurm-gcp-devel.zip" +fi +until gcloud storage cp "$BUCKET/$DEVEL_ZIP" "$SCRIPTS_ZIP"; do + echo "WARN: Could not download SlurmGCP scripts, retrying in 5 seconds." + # Remove marker used to determine if gcloud is being used in a GCE VM. + # This can get mistakenly set to False in some cases. + rm -f /root/.config/gcloud/gce + sleep 5 +done +unzip -o "$SCRIPTS_ZIP" -d "$SCRIPTS_DIR" +rm -rf "$SCRIPTS_ZIP" + +#temporary hack to not make the script fail on TPU vm +chown slurm:slurm -R "$SCRIPTS_DIR" || true +chmod 700 -R "$SCRIPTS_DIR" + + +if [ -f $FLAGFILE ]; then + echo "WARNING: Slurm was previously configured, quitting" + exit 0 +fi +touch $FLAGFILE + +function tpu_setup { + #allow the following command to fail, as this attribute does not exist for regular nodes + docker_image=$($CURL $URL/instance/attributes/slurm_docker_image 2> /dev/null || true) + if [ -z $docker_image ]; then #Not a tpu node, do not do anything + return + fi + if [ "$OS_ENV" == "slurm_container" ]; then #Already inside the slurm container, we should continue starting + return + fi + + #given a input_string like "WORKER_0:Joseph;WORKER_1:richard;WORKER_2:edward;WORKER_3:john" and a number 1, this function will print richard + parse_metadata() { + local number=$1 + local input_string=$2 + local word=$(echo "$input_string" | awk -v n="$number" -F ':|;' '{ for (i = 1; i <= NF; i+=2) if ($(i) == "WORKER_"n) print $(i+1) }') + echo "$word" + } + + input_string=$($CURL $URL/instance/attributes/slurm_names) + worker_id=$($CURL $URL/instance/attributes/tpu-env | awk '/WORKER_ID/ {print $2}' | tr -d \') + real_name=$(parse_metadata $worker_id $input_string) + + #Prepare to docker pull with gcloud + mkdir -p /root/.docker + cat << EOF > /root/.docker/config.json +{ + "credHelpers": { + "gcr.io": "gcloud", + "us-docker.pkg.dev": "gcloud" + } +} +EOF + #cgroup detection + CGV=1 + CGROUP_FLAGS="-v /sys/fs/cgroup:/sys/fs/cgroup:rw" + if [ -f /sys/fs/cgroup/cgroup.controllers ]; then #CGV2 + CGV=2 + fi + if [ $CGV == 2 ]; then + CGROUP_FLAGS="--cgroup-parent=docker.slice --cgroupns=private --tmpfs /run --tmpfs /run/lock --tmpfs /tmp" + if [ ! -f /etc/systemd/system/docker.slice ]; then #In case that there is no slice prepared for hosting the containers create it + printf "[Unit]\nDescription=docker slice\nBefore=slices.target\n[Slice]\nCPUAccounting=true\nMemoryAccounting=true" > /etc/systemd/system/docker.slice + systemctl start docker.slice + fi + fi + #for the moment always use --privileged, as systemd might not work properly otherwise + TPU_FLAGS="--privileged" + # TPU_FLAGS="--cap-add SYS_RESOURCE --device /dev/accel0 --device /dev/accel1 --device /dev/accel2 --device /dev/accel3" + # if [ $CGV == 2 ]; then #In case that we are in CGV2 for systemd to work correctly for the moment we go with privileged + # TPU_FLAGS="--privileged" + # fi + + docker run -d $CGROUP_FLAGS $TPU_FLAGS --net=host --name=slurmd --hostname=$real_name --entrypoint=/usr/bin/systemd --restart unless-stopped $docker_image + exit 0 +} + +tpu_setup #will do nothing for normal nodes or the container spawned inside TPU + +echo "INFO: Running python cluster setup script" +SETUP_SCRIPT_FILE=$SCRIPTS_DIR/setup.py +chmod +x $SETUP_SCRIPT_FILE +exec $SETUP_SCRIPT_FILE diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf new file mode 100644 index 0000000000..c91bbc4fd1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf @@ -0,0 +1,171 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "instance_validation" { + source = "../../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +########## +# LOCALS # +########## + +locals { + additional_disks = [ + for disk in var.additional_disks : { + disk_name = disk.disk_name + device_name = disk.device_name + auto_delete = disk.auto_delete + source = disk.source + boot = disk.boot + disk_size_gb = disk.disk_size_gb + disk_type = disk.disk_type + disk_labels = merge( + disk.disk_labels, + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) + disk_resource_manager_tags = disk.disk_resource_manager_tags + } + ] + + service_account = { + email = try(var.service_account.email, null) + scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) + } + + source_image_family = ( + var.source_image_family != "" && var.source_image_family != null + ? var.source_image_family + : "slurm-gcp-6-11-hpc-rocky-linux-8" + ) + source_image_project = ( + var.source_image_project != "" && var.source_image_project != null + ? var.source_image_project + : "projects/schedmd-slurm-public/global/images/family" + ) + + source_image = ( + var.source_image != null + ? var.source_image + : "" + ) + + + name_prefix = "${var.slurm_cluster_name}-${var.slurm_instance_role}-${var.name_prefix}" + + total_egress_bandwidth_tier = var.bandwidth_tier == "tier_1_enabled" ? "TIER_1" : "DEFAULT" + + nic_type_map = { + platform_default = null + virtio_enabled = "VIRTIO_NET" + gvnic_enabled = "GVNIC" + tier_1_enabled = "GVNIC" + } + nic_type = lookup(local.nic_type_map, var.bandwidth_tier, null) + + labels = merge(var.labels, + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) +} + +######## +# DATA # +######## + +data "local_file" "startup" { + filename = "${path.module}/files/startup_sh_unlinted" +} + +############ +# TEMPLATE # +############ + +module "instance_template" { + source = "../internal_instance_template" + + project_id = var.project_id + + # Network + can_ip_forward = var.can_ip_forward + network_ip = var.network_ip + network = var.network + nic_type = local.nic_type + region = var.region + subnetwork_project = var.subnetwork_project + subnetwork = var.subnetwork + tags = var.tags + total_egress_bandwidth_tier = local.total_egress_bandwidth_tier + additional_networks = var.additional_networks + access_config = var.access_config + + # Instance + machine_type = var.machine_type + min_cpu_platform = var.min_cpu_platform + name_prefix = local.name_prefix + gpu = var.gpu + service_account = local.service_account + shielded_instance_config = var.shielded_instance_config + advanced_machine_features = var.advanced_machine_features + enable_confidential_vm = var.enable_confidential_vm + enable_shielded_vm = var.enable_shielded_vm + preemptible = var.preemptible + spot = var.spot + on_host_maintenance = var.on_host_maintenance + labels = local.labels + instance_termination_action = var.termination_action + resource_manager_tags = var.resource_manager_tags + + # Metadata + startup_script = coalesce(var.internal_startup_script, data.local_file.startup.content) + metadata = merge( + var.metadata, + { + enable-oslogin = upper(var.enable_oslogin) + slurm_bucket_path = var.slurm_bucket_path + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) + + # Image + source_image_project = local.source_image_project + source_image_family = local.source_image_family + source_image = local.source_image + + # Disk + disk_type = var.disk_type + disk_size_gb = var.disk_size_gb + auto_delete = var.disk_auto_delete + disk_labels = merge( + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + var.disk_labels, + ) + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + + max_run_duration = var.max_run_duration + provisioning_model = var.provisioning_model + reservation_affinity = var.reservation_affinity +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf new file mode 100644 index 0000000000..65da41052e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "instance_template" { + description = "Instance template details" + value = module.instance_template +} + +output "self_link" { + description = "Self_link of instance template" + value = module.instance_template.self_link +} + +output "name" { + description = "Name of instance template" + value = module.instance_template.name +} + +output "tags" { + description = "Tags that will be associated with instance(s)" + value = module.instance_template.tags +} + +output "service_account" { + description = "Service account object, includes email and scopes." + value = module.instance_template.service_account +} + +output "labels" { + description = "Labels attached to the instance template" + value = local.labels +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf new file mode 100644 index 0000000000..35dd9c376f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf @@ -0,0 +1,431 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +########### +# GENERAL # +########### + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "on_host_maintenance" { + type = string + description = "Instance availability Policy" + default = "MIGRATE" +} + +variable "labels" { + type = map(string) + description = "Labels, provided as a map" + default = {} +} + +variable "enable_oslogin" { + type = bool + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >=0.13.0 | +| [google](#requirement\_google) | >= 3.88 | +| [google-beta](#requirement\_google-beta) | >= 6.13.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.88 | +| [google-beta](#provider\_google-beta) | >= 6.13.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance_template.tpl](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_template) | resource | +| [google_project.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    auto_delete                = bool
    boot                       = bool
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = map(string)
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | +| [alias\_ip\_range](#input\_alias\_ip\_range) | An array of alias IP ranges for this network interface. Can only be specified for network interfaces on subnet-mode networks.
ip\_cidr\_range: The IP CIDR range represented by this alias IP range. This IP CIDR range must belong to the specified subnetwork and cannot contain IP addresses reserved by system or used by other network interfaces. At the time of writing only a netmask (e.g. /24) may be supplied, with a CIDR format resulting in an API error.
subnetwork\_range\_name: The subnetwork secondary range name specifying the secondary range from which to allocate the IP CIDR range for this alias IP range. If left unspecified, the primary range of the subnetwork will be used. | 
object({
    ip_cidr_range         = string
    subnetwork_range_name = string
  })
| `null` | no | +| [auto\_delete](#input\_auto\_delete) | Whether or not the boot disk should be auto-deleted | `string` | `"true"` | no | +| [automatic\_restart](#input\_automatic\_restart) | (Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user). | `bool` | `true` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example | `string` | `"false"` | no | +| [disk\_encryption\_key](#input\_disk\_encryption\_key) | The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance | `string` | `null` | no | +| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `string` | `"100"` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Whether to enable the Confidential VM configuration on the instance. Note that the instance image must support Confidential VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Whether to enable the Shielded VM configuration on the instance. Note that the instance image must support Shielded VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | +| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See https://cloud.google.com/compute/docs/gpus more details | 
object({
    type  = string
    count = number
  })
| `null` | no | +| [instance\_termination\_action](#input\_instance\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | +| [ipv6\_access\_config](#input\_ipv6\_access\_config) | IPv6 access configurations. Currently a max of 1 IPv6 access configuration is supported. If not specified, the instance will have no external IPv6 Internet access. | 
list(object({
    network_tier = string
  }))
| `[]` | no | +| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create, e.g. n1-standard-1 | `string` | `"n1-standard-1"` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Name prefix for the instance template | `string` | n/a | yes | +| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks. | `string` | `""` | no | +| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | +| [nic\_type](#input\_nic\_type) | The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO\_NET. | `string` | `null` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | +| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | +| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#service_account. | 
object({
    email  = optional(string)
    scopes = set(string)
  })
| n/a | yes | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Not used unless enable\_shielded\_vm is true. Shielded VM configuration for the instance. | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [source\_image](#input\_source\_image) | Source disk image. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `""` | no | +| [source\_image\_family](#input\_source\_image\_family) | Source image family. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `"centos-7"` | no | +| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. The default project contains CentOS images. | `string` | `"centos-cloud"` | no | +| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | +| [stack\_type](#input\_stack\_type) | The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4\_ONLY. | `string` | `null` | no | +| [startup\_script](#input\_startup\_script) | User startup script to run when instances spin up | `string` | `""` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided. | `string` | `""` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [total\_egress\_bandwidth\_tier](#input\_total\_egress\_bandwidth\_tier) | Network bandwidth tier. Note: machine\_type must be a supported type. Values are 'TIER\_1' or 'DEFAULT'.
See https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration for details. | `string` | `"DEFAULT"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [name](#output\_name) | Name of instance template | +| [self\_link](#output\_self\_link) | Self-link of instance template | +| [service\_account](#output\_service\_account) | value | +| [tags](#output\_tags) | Tags that will be associated with instance(s) | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf new file mode 100644 index 0000000000..f8d2813ece --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf @@ -0,0 +1,234 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "instance_validation" { + source = "../../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +######### +# Locals +######### + +locals { + source_image = var.source_image != "" ? var.source_image : "centos-7-v20201112" + source_image_family = var.source_image_family != "" ? var.source_image_family : "centos-7" + source_image_project = var.source_image_project != "" ? var.source_image_project : "centos-cloud" + + boot_disk = [ + { + source_image = var.source_image != "" ? format("${local.source_image_project}/${local.source_image}") : format("${local.source_image_project}/${local.source_image_family}") + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_labels = var.disk_labels + auto_delete = var.auto_delete + disk_resource_manager_tags = var.disk_resource_manager_tags + boot = "true" + }, + ] + + all_disks = concat(local.boot_disk, var.additional_disks) + + # NOTE: Even if all the shielded_instance_config or confidential_instance_config + # values are false, if the config block exists and an unsupported image is chosen, + # the apply will fail so we use a single-value array with the default value to + # initialize the block only if it is enabled. + shielded_vm_configs = var.enable_shielded_vm ? [true] : [] + + gpu_enabled = var.gpu != null + alias_ip_range_enabled = var.alias_ip_range != null + preemptible = var.preemptible || var.spot + on_host_maintenance = ( + local.preemptible || var.enable_confidential_vm || local.gpu_enabled + ? "TERMINATE" + : var.on_host_maintenance + ) + automatic_restart = ( + # must be false when preemptible is true + local.preemptible ? false : var.automatic_restart + ) + + nic_type = var.total_egress_bandwidth_tier == "TIER_1" ? "GVNIC" : var.nic_type + + + provisioning_model = coalesce(var.provisioning_model, local.preemptible ? "SPOT" : "STANDARD") +} + +data "google_project" "this" { + project_id = var.project_id +} + +#################### +# Instance Template +#################### +resource "google_compute_instance_template" "tpl" { + provider = google-beta + name_prefix = "${var.name_prefix}-" + project = var.project_id + machine_type = var.machine_type + labels = var.labels + metadata = var.metadata + tags = var.tags + can_ip_forward = var.can_ip_forward + metadata_startup_script = var.startup_script + region = var.region + min_cpu_platform = var.min_cpu_platform + resource_manager_tags = var.resource_manager_tags + + service_account { + email = coalesce(var.service_account.email, "${data.google_project.this.number}-compute@developer.gserviceaccount.com") + scopes = lookup(var.service_account, "scopes", null) + } + + dynamic "disk" { + for_each = local.all_disks + content { + auto_delete = lookup(disk.value, "auto_delete", null) + boot = lookup(disk.value, "boot", null) + device_name = lookup(disk.value, "device_name", null) + disk_name = lookup(disk.value, "disk_name", null) + disk_size_gb = lookup(disk.value, "disk_size_gb", lookup(disk.value, "disk_type", null) == "local-ssd" ? "375" : null) + disk_type = lookup(disk.value, "disk_type", null) + interface = lookup(disk.value, "interface", lookup(disk.value, "disk_type", null) == "local-ssd" ? "NVME" : null) + mode = lookup(disk.value, "mode", null) + source = lookup(disk.value, "source", null) + source_image = lookup(disk.value, "source_image", null) + type = lookup(disk.value, "disk_type", null) == "local-ssd" ? "SCRATCH" : "PERSISTENT" + labels = (lookup(disk.value, "source", null) != null || lookup(disk.value, "disk_type", null) == "local-ssd") ? null : lookup(disk.value, "disk_labels", null) + resource_manager_tags = lookup(disk.value, "disk_resource_manager_tags", {}) + + dynamic "disk_encryption_key" { + for_each = compact([var.disk_encryption_key == null ? null : 1]) + content { + kms_key_self_link = var.disk_encryption_key + } + } + } + } + + network_interface { + network = var.network + subnetwork = var.subnetwork + subnetwork_project = var.subnetwork_project + network_ip = try(coalesce(var.network_ip), null) + nic_type = local.nic_type + stack_type = var.stack_type + dynamic "access_config" { + for_each = var.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = var.ipv6_access_config + content { + network_tier = ipv6_access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = local.alias_ip_range_enabled ? [var.alias_ip_range] : [] + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + } + + dynamic "network_interface" { + for_each = var.additional_networks + content { + network = network_interface.value.network + subnetwork = network_interface.value.subnetwork + subnetwork_project = network_interface.value.subnetwork_project + network_ip = try(coalesce(network_interface.value.network_ip), null) + nic_type = try(coalesce(network_interface.value.nic_type), null) + dynamic "access_config" { + for_each = network_interface.value.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = network_interface.value.ipv6_access_config + content { + network_tier = ipv6_access_config.value.network_tier + } + } + } + } + + network_performance_config { + total_egress_bandwidth_tier = coalesce(var.total_egress_bandwidth_tier, "DEFAULT") + } + + lifecycle { + create_before_destroy = "true" + } + + scheduling { + preemptible = local.preemptible + provisioning_model = local.provisioning_model + automatic_restart = local.automatic_restart + on_host_maintenance = local.on_host_maintenance + instance_termination_action = var.instance_termination_action + + dynamic "max_run_duration" { + for_each = var.max_run_duration != null ? [var.max_run_duration] : [] + content { + seconds = max_run_duration.value + } + } + } + + dynamic "reservation_affinity" { + for_each = var.reservation_affinity != null ? [var.reservation_affinity] : [] + content { + type = reservation_affinity.value.type + } + } + + advanced_machine_features { + enable_nested_virtualization = var.advanced_machine_features.enable_nested_virtualization + threads_per_core = var.advanced_machine_features.threads_per_core + turbo_mode = var.advanced_machine_features.turbo_mode + visible_core_count = var.advanced_machine_features.visible_core_count + performance_monitoring_unit = var.advanced_machine_features.performance_monitoring_unit + enable_uefi_networking = var.advanced_machine_features.enable_uefi_networking + } + + dynamic "shielded_instance_config" { + for_each = local.shielded_vm_configs + content { + enable_secure_boot = lookup(var.shielded_instance_config, "enable_secure_boot", shielded_instance_config.value) + enable_vtpm = lookup(var.shielded_instance_config, "enable_vtpm", shielded_instance_config.value) + enable_integrity_monitoring = lookup(var.shielded_instance_config, "enable_integrity_monitoring", shielded_instance_config.value) + } + } + + confidential_instance_config { + enable_confidential_compute = var.enable_confidential_vm + } + + dynamic "guest_accelerator" { + for_each = local.gpu_enabled ? [var.gpu] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf new file mode 100644 index 0000000000..69f8d3b98c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "self_link" { + description = "Self-link of instance template" + value = google_compute_instance_template.tpl.self_link +} + +output "name" { + description = "Name of instance template" + value = google_compute_instance_template.tpl.name +} + +output "tags" { + description = "Tags that will be associated with instance(s)" + value = google_compute_instance_template.tpl.tags +} + +output "service_account" { + description = "value" + value = google_compute_instance_template.tpl.service_account[0] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf new file mode 100644 index 0000000000..c285c3fea5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf @@ -0,0 +1,398 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + type = string + description = "The GCP project ID" + default = null +} + +variable "name_prefix" { + description = "Name prefix for the instance template" + type = string +} + +variable "machine_type" { + description = "Machine type to create, e.g. n1-standard-1" + type = string + default = "n1-standard-1" +} + +variable "min_cpu_platform" { + description = "Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform" + type = string + default = null +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example" + type = string + default = "false" +} + +variable "tags" { + type = list(string) + description = "Network tags, provided as a list" + default = [] +} + +variable "labels" { + type = map(string) + description = "Labels, provided as a map" + default = {} +} + +variable "preemptible" { + type = bool + description = "Allow the instance to be preempted" + default = false +} + +variable "spot" { + description = <<-EOD + Provision as a SPOT preemptible instance. + See https://cloud.google.com/compute/docs/instances/spot for more details. + EOD + type = bool + default = false +} + +variable "instance_termination_action" { + description = <<-EOD + Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'. + See https://cloud.google.com/compute/docs/instances/spot for more details. + EOD + type = string + default = null +} + +variable "automatic_restart" { + type = bool + description = "(Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user)." + default = true +} + +variable "on_host_maintenance" { + type = string + description = "Instance availability Policy" + default = "MIGRATE" +} + +variable "region" { + type = string + description = "Region where the instance template should be created." + nullable = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +####### +# disk +####### +variable "source_image" { + description = "Source disk image. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." + type = string + default = "" +} + +variable "source_image_family" { + description = "Source image family. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." + type = string + default = "centos-7" +} + +variable "source_image_project" { + description = "Project where the source image comes from. The default project contains CentOS images." + type = string + default = "centos-cloud" +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = string + default = "100" +} + +variable "disk_type" { + description = "Boot disk type, can be either pd-ssd, local-ssd, or pd-standard" + type = string + default = "pd-standard" +} + +variable "disk_labels" { + description = "Labels to be assigned to boot disk, provided as a map" + type = map(string) + default = {} +} + +variable "disk_encryption_key" { + description = "The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance" + type = string + default = null +} + +variable "auto_delete" { + description = "Whether or not the boot disk should be auto-deleted" + type = string + default = "true" +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + description = "List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name" + type = list(object({ + source = optional(string) + disk_name = optional(string) + device_name = string + auto_delete = bool + boot = bool + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = map(string) + disk_resource_manager_tags = map(string) + })) + default = [] +} + +#################### +# network_interface +#################### +variable "network" { + description = "The name or self_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks." + type = string + default = "" +} + +variable "nic_type" { + description = "The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO_NET." + type = string + default = null +} + +variable "subnetwork" { + description = "The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided." + type = string + default = "" +} + +variable "subnetwork_project" { + description = "The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used." + type = string + default = null +} + +variable "network_ip" { + description = "Private IP address to assign to the instance if desired." + type = string + default = "" +} + +variable "stack_type" { + description = "The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4_ONLY." + type = string + default = null +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + })) +} + +variable "total_egress_bandwidth_tier" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 6.41 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.41 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance](#module\_instance) | ../instance | n/a | +| [template](#module\_template) | ../instance_template | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | +| [login\_nodes](#input\_login\_nodes) | Slurm login instance definitions. | 
object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  })
| n/a | yes | +| [network\_storage](#input\_network\_storage) | Storage to mounted on login instances
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path to directory in the bucket for configs | `string` | n/a | yes | +| [slurm\_bucket\_name](#input\_slurm\_bucket\_name) | Name of the bucket for configs | `string` | n/a | yes | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name | `string` | n/a | yes | +| [startup\_scripts](#input\_startup\_scripts) | List of scripts to be ran on login VMs startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | +| [startup\_scripts\_timeout](#input\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script. If any script exceeds this timeout,
then the instance setup process is considered failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instances](#output\_instances) | VM instances of login nodes | +| [service\_account](#output\_service\_account) | Service Account used by login VMs | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf new file mode 100644 index 0000000000..605461f7e6 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf @@ -0,0 +1,112 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "template" { + source = "../instance_template" + + project_id = var.project_id + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = "login" + slurm_bucket_path = var.slurm_bucket_path + name_prefix = local.name + + additional_disks = var.login_nodes.additional_disks + bandwidth_tier = var.login_nodes.bandwidth_tier + can_ip_forward = var.login_nodes.can_ip_forward + advanced_machine_features = var.login_nodes.advanced_machine_features + disk_auto_delete = var.login_nodes.disk_auto_delete + disk_labels = var.login_nodes.disk_labels + disk_resource_manager_tags = var.login_nodes.disk_resource_manager_tags + disk_size_gb = var.login_nodes.disk_size_gb + disk_type = var.login_nodes.disk_type + enable_confidential_vm = var.login_nodes.enable_confidential_vm + enable_oslogin = var.login_nodes.enable_oslogin + enable_shielded_vm = var.login_nodes.enable_shielded_vm + gpu = var.login_nodes.gpu + labels = var.login_nodes.labels + machine_type = var.login_nodes.machine_type + metadata = merge(var.login_nodes.metadata, { + "universe_domain" = var.universe_domain, + "slurm_login_group" = local.name + }) + min_cpu_platform = var.login_nodes.min_cpu_platform + on_host_maintenance = var.login_nodes.on_host_maintenance + preemptible = var.login_nodes.preemptible + region = var.login_nodes.region + resource_manager_tags = var.login_nodes.resource_manager_tags + service_account = var.login_nodes.service_account + shielded_instance_config = var.login_nodes.shielded_instance_config + source_image_family = var.login_nodes.source_image_family + source_image_project = var.login_nodes.source_image_project + source_image = var.login_nodes.source_image + spot = var.login_nodes.spot + subnetwork = var.login_nodes.subnetwork + tags = concat([var.slurm_cluster_name], var.login_nodes.tags) + termination_action = var.login_nodes.termination_action + + internal_startup_script = var.internal_startup_script +} + +module "instance" { + source = "../instance" + + access_config = var.login_nodes.access_config + hostname = "${var.slurm_cluster_name}-${local.name}" + + project_id = var.project_id + + instance_template = module.template.self_link + num_instances = var.login_nodes.num_instances + + additional_networks = var.login_nodes.additional_networks + region = var.login_nodes.region + static_ips = var.login_nodes.static_ips + subnetwork = var.login_nodes.subnetwork + zone = var.login_nodes.zone + + replace_trigger = var.replace_trigger +} + +resource "google_storage_bucket_object" "startup_scripts" { + for_each = { + for s in var.startup_scripts : format( + "slurm-login-%s-script-%s", local.name, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_") + ) => s.content + } + + bucket = var.slurm_bucket_name + name = "${var.slurm_bucket_dir}/${each.key}" + content = each.value + source_md5hash = md5(each.value) +} + +locals { + name = var.login_nodes.group_name # short hand + + config = { + group_name = local.name + startup_scripts_timeout = var.startup_scripts_timeout + network_storage = var.network_storage + } +} + +resource "google_storage_bucket_object" "config" { + bucket = var.slurm_bucket_name + name = "${var.slurm_bucket_dir}/login_group_configs/${local.name}.yaml" + content = yamlencode(local.config) + source_md5hash = md5(yamlencode(local.config)) + + # To ensure that login group "is not ready" until all startup scripts are written down + depends_on = [google_storage_bucket_object.startup_scripts] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf new file mode 100644 index 0000000000..04de18a188 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "service_account" { + value = module.template.service_account + description = "Service Account used by login VMs" +} + +output "instances" { + value = module.instance.slurm_instances + description = "VM instances of login nodes" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf new file mode 100644 index 0000000000..3efd862942 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf @@ -0,0 +1,188 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "slurm_cluster_name" { + type = string + description = "Cluster name" +} + +variable "slurm_bucket_path" { + type = string + description = "GCS Bucket URI of Slurm cluster file storage." +} + + +variable "slurm_bucket_name" { + type = string + description = "Name of the bucket for configs" +} + +variable "slurm_bucket_dir" { + type = string + description = "Path to directory in the bucket for configs" +} + + +variable "universe_domain" { + description = "Domain address for alternate API universe" + type = string + default = "googleapis.com" +} + +variable "login_nodes" { + description = "Slurm login instance definitions." + type = object({ + group_name = string + access_config = optional(list(object({ + nat_ip = string + network_tier = string + }))) + additional_disks = optional(list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string), {}) + auto_delete = optional(bool, true) + boot = optional(bool, false) + disk_resource_manager_tags = optional(map(string), {}) + })), []) + additional_networks = optional(list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })), []) + bandwidth_tier = optional(string, "platform_default") + can_ip_forward = optional(bool, false) + disk_auto_delete = optional(bool, true) + disk_labels = optional(map(string), {}) + disk_resource_manager_tags = optional(map(string), {}) + disk_size_gb = optional(number) + disk_type = optional(string, "n1-standard-1") + enable_confidential_vm = optional(bool, false) + enable_oslogin = optional(bool, true) + enable_shielded_vm = optional(bool, false) + gpu = optional(object({ + count = number + type = string + })) + labels = optional(map(string), {}) + machine_type = optional(string) + advanced_machine_features = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + metadata = optional(map(string), {}) + min_cpu_platform = optional(string) + num_instances = optional(number, 1) + on_host_maintenance = optional(string) + preemptible = optional(bool, false) + region = optional(string) + resource_manager_tags = optional(map(string), {}) + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + shielded_instance_config = optional(object({ + enable_integrity_monitoring = optional(bool, true) + enable_secure_boot = optional(bool, true) + enable_vtpm = optional(bool, true) + })) + source_image_family = optional(string) + source_image_project = optional(string) + source_image = optional(string) + static_ips = optional(list(string), []) + subnetwork = string + spot = optional(bool, false) + tags = optional(list(string), []) + zone = optional(string) + termination_action = optional(string) + }) +} + + +variable "startup_scripts" { + description = "List of scripts to be ran on login VMs startup." + type = list(object({ + filename = string + content = string + })) + default = [] +} + +variable "startup_scripts_timeout" { + description = < + +- [Module: Slurm Nodeset (TPU)](#module-slurm-nodeset-tpu) + - [Overview](#overview) + - [Module API](#module-api) + + + +## Overview + +This is a submodule of [slurm_cluster](../../../slurm_cluster/README.md). It +creates a Slurm TPU nodeset for [slurm_partition](../slurm_partition/README.md). + +## Module API + +For the terraform module API reference, please see +[README_TF.md](./README_TF.md). + + +Copyright (C) SchedMD LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.2 | +| [google](#requirement\_google) | >= 3.53 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.53 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.nodeset_tpu](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_subnetwork.nodeset_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | +| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | +| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `""` | no | +| [enable\_public\_ip](#input\_enable\_public\_ip) | Enables IP address to access the Internet. | `bool` | `false` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of nodes allowed in this partition to be created dynamically. | `number` | `0` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | +| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. Not needed if you use accelerator\_config | `string` | `null` | no | +| [nodeset\_name](#input\_nodeset\_name) | Name of Slurm nodeset. | `string` | n/a | yes | +| [preemptible](#input\_preemptible) | Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details. | `bool` | `false` | no | +| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `true` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the TPU-vm.
If none is given, the default service account and scopes will be used. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | +| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | n/a | yes | +| [zone](#input\_zone) | Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Nodeset details. | +| [nodeset\_name](#output\_nodeset\_name) | Nodeset name. | +| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf new file mode 100644 index 0000000000..1a6a9cfba1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf @@ -0,0 +1,121 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########### +# NODESET # +########### + +locals { + node_conf_hw = { + Mem334CPU96 = { + CPUs = 96 + Boards = 1 + Sockets = 2 + CoresPerSocket = 24 + ThreadsPerCore = 2 + RealMemory = 307200 + } + Mem400CPU240 = { + CPUs = 240 + Boards = 1 + Sockets = 2 + CoresPerSocket = 60 + ThreadsPerCore = 2 + RealMemory = 400000 + } + } + node_conf_mappings = { + "v2" = local.node_conf_hw.Mem334CPU96 + "v3" = local.node_conf_hw.Mem334CPU96 + "v4" = local.node_conf_hw.Mem400CPU240 + } + simple_nodes = ["v2-8", "v3-8", "v4-8"] +} + +locals { + snetwork = data.google_compute_subnetwork.nodeset_subnetwork.name + region = join("-", slice(split("-", var.zone), 0, 2)) + tpu_fam = var.accelerator_config.version != "" ? lower(var.accelerator_config.version) : split("-", var.node_type)[0] + #If subnetwork is specified and it does not have private_ip_google_access, we need to have public IPs on the TPU + #if no subnetwork is specified, the default one will be used, this does not have private_ip_google_access so we need public IPs too + pub_need = !data.google_compute_subnetwork.nodeset_subnetwork.private_ip_google_access + can_preempt = var.node_type != null ? contains(local.simple_nodes, var.node_type) : false + nodeset_tpu = { + nodeset_name = var.nodeset_name + node_conf = local.node_conf_mappings[local.tpu_fam] + node_type = var.node_type + accelerator_config = var.accelerator_config + tf_version = var.tf_version + preemptible = local.can_preempt ? var.preemptible : false + reserved = var.reserved + node_count_dynamic_max = var.node_count_dynamic_max + node_count_static = var.node_count_static + enable_public_ip = var.enable_public_ip + zone = var.zone + service_account = var.service_account != null ? var.service_account : local.service_account + preserve_tpu = local.can_preempt ? var.preserve_tpu : false + data_disks = var.data_disks + docker_image = var.docker_image != "" ? var.docker_image : "us-docker.pkg.dev/schedmd-slurm-public/tpu/slurm-gcp-6-9:tf-${var.tf_version}" + subnetwork = local.snetwork + network_storage = var.network_storage + } + + service_account = { + email = try(var.service_account.email, null) + scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) + } +} + +data "google_compute_subnetwork" "nodeset_subnetwork" { + name = var.subnetwork + region = local.region + project = var.project_id + + self_link = ( + length(regexall("/projects/([^/]*)", var.subnetwork)) > 0 + && length(regexall("/regions/([^/]*)", var.subnetwork)) > 0 + ? var.subnetwork + : null + ) +} + +resource "null_resource" "nodeset_tpu" { + triggers = { + nodeset = sha256(jsonencode(local.nodeset_tpu)) + } + lifecycle { + precondition { + condition = sum([var.node_count_dynamic_max, var.node_count_static]) > 0 + error_message = "Sum of node_count_dynamic_max and node_count_static must be > 0." + } + precondition { + condition = !(var.preemptible && var.reserved) + error_message = "Nodeset cannot be preemptible and reserved at the same time." + } + precondition { + condition = !(var.subnetwork == null && !var.enable_public_ip) + error_message = "Using the default subnetwork for the TPU nodeset requires enable_public_ip set to true." + } + precondition { + condition = !(var.subnetwork != null && (local.pub_need && !var.enable_public_ip)) + error_message = "The subnetwork specified does not have Private Google Access enabled. This is required when enable_public_ip is set to false." + } + precondition { + condition = !(var.node_type == null && (var.accelerator_config.topology == "" && var.accelerator_config.version == "")) + error_message = "Either a node type or an accelerator_config must be provided." + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf new file mode 100644 index 0000000000..fce700d567 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "nodeset_name" { + description = "Nodeset name." + value = local.nodeset_tpu.nodeset_name +} + +output "nodeset" { + description = "Nodeset details." + value = local.nodeset_tpu +} + +output "service_account" { + description = "Service account object, includes email and scopes." + value = local.service_account +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf new file mode 100644 index 0000000000..a8c470dec9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf @@ -0,0 +1,158 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "nodeset_name" { + description = "Name of Slurm nodeset." + type = string + + validation { + condition = can(regex("^[a-z](?:[a-z0-9]{0,14})$", var.nodeset_name)) + error_message = "Variable 'nodeset_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,14})$'." + } +} + +variable "node_type" { + description = "Specify a node type to base the vm configuration upon it. Not needed if you use accelerator_config" + type = string + default = null +} + +variable "accelerator_config" { + description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." + type = object({ + topology = string + version = string + }) + default = { + topology = "" + version = "" + } + validation { + condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], upper(var.accelerator_config.version)) + error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" + } + validation { + condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) + error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." + } +} + +variable "docker_image" { + description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" + type = string + default = "" +} + +variable "tf_version" { + description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." + type = string +} + +variable "zone" { + description = "Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it." + type = string + + validation { + condition = can(coalesce(var.zone)) + error_message = "Zone cannot be null or empty." + } +} + +variable "preemptible" { + description = "Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details." + type = bool + default = false +} + +variable "reserved" { + description = "Specify whether TPU-vms in this nodeset are created under a reservation." + type = bool + default = false +} + +variable "preserve_tpu" { + description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" + type = bool + default = true +} + +variable "node_count_static" { + description = "Number of nodes to be statically created." + type = number + default = 0 + + validation { + condition = var.node_count_static >= 0 + error_message = "Value must be >= 0." + } +} + +variable "node_count_dynamic_max" { + description = "Maximum number of nodes allowed in this partition to be created dynamically." + type = number + default = 0 + + validation { + condition = var.node_count_dynamic_max >= 0 + error_message = "Value must be >= 0." + } +} + +variable "enable_public_ip" { + description = "Enables IP address to access the Internet." + type = bool + default = false +} + +variable "data_disks" { + type = list(string) + description = "The data disks to include in the TPU node" + default = [] +} + +variable "subnetwork" { + description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." + type = string +} + +variable "service_account" { + type = object({ + email = string + scopes = set(string) + }) + description = < +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | +| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | +| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | +| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | +| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | +| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "v0.7.2")
  })
| `{}` | no | +| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "v0.11.4")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | +| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md new file mode 100644 index 0000000000..1957899617 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md @@ -0,0 +1,64 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [helm](#provider\_helm) | ~> 2.17 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | +| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | +| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | +| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | +| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | +| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | +| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | +| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | +| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | +| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | +| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | +| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | +| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | +| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | +| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | +| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | +| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | +| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | +| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | +| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | +| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | +| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | +| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | +| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | +| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | +| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | +| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | +| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | +| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | +| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf new file mode 100644 index 0000000000..bd2383b772 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf @@ -0,0 +1,75 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "helm_release" "apply_chart" { + # Required Identification + name = var.release_name + chart = var.chart_name + + # Chart Source & Version + repository = var.chart_repository + version = var.chart_version + devel = var.devel + + # Target Namespace + namespace = var.namespace + create_namespace = var.create_namespace + + # Values Configuration + values = var.values_yaml + + dynamic "set" { + for_each = var.set_values + content { + name = set.value.name + value = set.value.value + type = set.value.type + } + } + + # Installation/Upgrade Behavior + description = var.description + atomic = var.atomic + cleanup_on_fail = var.cleanup_on_fail + dependency_update = var.dependency_update + disable_crd_hooks = var.disable_crd_hooks + disable_openapi_validation = var.disable_openapi_validation + disable_webhooks = var.disable_webhooks + force_update = var.force_update + lint = var.lint + max_history = var.max_history + recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI + render_subchart_notes = var.render_subchart_notes + reset_values = var.reset_values + reuse_values = var.reuse_values + skip_crds = var.skip_crds + timeout = var.timeout + wait = var.wait + wait_for_jobs = var.wait_for_jobs + + # Verification & Credentials + keyring = var.keyring + pass_credentials = var.pass_credentials + verify = var.verify + + # Post Rendering + dynamic "postrender" { + # Only include the block if var.postrender is not null + for_each = var.postrender == null ? [] : [var.postrender] + content { + binary_path = postrender.value.binary_path + } + } + +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml new file mode 100644 index 0000000000..e18197e2b7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf new file mode 100644 index 0000000000..04e8e214fc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf @@ -0,0 +1,212 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +# --- Required --- +variable "release_name" { + description = "Name of the Helm release." + type = string +} + +variable "chart_name" { + description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." + type = string +} + +# --- Chart Location & Version --- +variable "chart_repository" { + description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." + type = string + default = null +} + +variable "chart_version" { + description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." + type = string + default = null +} + +variable "devel" { + description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." + type = bool + default = false +} + +# --- Namespace --- +variable "namespace" { + description = "Kubernetes namespace to install the Helm release into." + type = string + default = "default" +} + +variable "create_namespace" { + description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." + type = bool + default = true # Common convenience setting +} + +# --- Values Customization --- +variable "values_yaml" { + description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." + type = list(string) + default = [] +} + +variable "set_values" { + description = "List of objects defining values to set ('helm install --set')." + type = list(object({ + name = string # Path to the value (e.g., 'service.type', 'replicaCount') + value = string # The value to set + type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') + })) + default = [] +} + +# --- Installation/Upgrade Behavior --- +variable "description" { + description = "Set an optional description for the Helm release." + type = string + default = null +} + +variable "atomic" { + description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." + type = bool + default = false +} + +variable "wait" { + description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." + type = bool + default = true # Often a good default for dependencies +} + +variable "wait_for_jobs" { + description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." + type = bool + default = false # Helm CLI default is false +} + +variable "timeout" { + description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." + type = number + default = 300 # 5 minutes (Helm CLI default) +} + +variable "cleanup_on_fail" { + description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." + type = bool + default = false +} + +variable "dependency_update" { + description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." + type = bool + default = false +} + +variable "disable_crd_hooks" { + description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." + type = bool + default = false +} + +variable "disable_openapi_validation" { + description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." + type = bool + default = false +} + +variable "disable_webhooks" { + description = "Prevent hooks from running ('helm install --no-hooks')." + type = bool + default = false +} + +variable "force_update" { + description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." + type = bool + default = false +} + +variable "lint" { + description = "Run the helm chart linter during the plan ('helm lint')." + type = bool + default = false +} + +variable "max_history" { + description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." + type = number + default = null # Terraform provider defaults to Helm's default (usually 10) +} + +variable "recreate_pods" { + description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." + type = bool + default = false +} + +variable "render_subchart_notes" { + description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." + type = bool + default = false +} + +variable "reset_values" { + description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." + type = bool + default = false +} + +variable "reuse_values" { + description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." + type = bool + default = false # Helm CLI default is false +} + +variable "skip_crds" { + description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." + type = bool + default = false +} + +# --- Verification & Credentials --- +variable "keyring" { + description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." + type = string + default = null # Defaults to Helm's default keyring location +} + +variable "pass_credentials" { + description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." + type = bool + default = false +} + +variable "verify" { + description = "Verify the package before installing it ('helm install --verify')." + type = bool + default = false +} + +# --- Advanced Rendering --- +variable "postrender" { + description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." + type = object({ + binary_path = string # Path to the post-renderer executable + }) + default = null # Disabled by default +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf new file mode 100644 index 0000000000..09d912e2c9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md new file mode 100644 index 0000000000..46bfe51a32 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md @@ -0,0 +1,40 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.23 | + +## Providers + +| Name | Version | +|------|---------| +| [kubernetes](#provider\_kubernetes) | ~> 2.23 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubernetes_manifest.apply_manifests](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | +| [field\_manager](#input\_field\_manager) | (Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts. | 
object({
    name            = optional(string, null)
    force_conflicts = optional(bool, false)
  })
| `null` | no | +| [resource\_timeouts](#input\_resource\_timeouts) | (Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met. | 
object({
    create = optional(string, null)
    update = optional(string, null)
    delete = optional(string, null)
  })
| 
{
  "create": "15m",
  "delete": "5m",
  "update": "10m"
}
| no | +| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `""` | no | +| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | +| [wait\_for\_fields](#input\_wait\_for\_fields) | (Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern. | `map(string)` | `{}` | no | +| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf new file mode 100644 index 0000000000..f97f26038d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf @@ -0,0 +1,104 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + yaml_separator = "\n---" + + # --- 1. Determine the primary source of YAML content --- + # Prioritize 'content' variable if provided + primary_content_body = var.content != "" ? var.content : null + + # --- 2. Handle 'source_path' based on its type (File vs. Directory) --- + + # Check if source_path is a directory (indicated by trailing slash) + is_directory = endswith(var.source_path, "/") + directory_absolute_path = local.is_directory ? abspath(var.source_path) : null + + # Check if source_path is a single yaml or tftpl file (only if not a directory) + is_single_file = !local.is_directory && ( + length(regexall("\\.yaml$", lower(var.source_path))) > 0 || + length(regexall("\\.tftpl$", lower(var.source_path))) > 0 + ) + single_file_raw_content = local.is_single_file ? ( + length(regexall("\\.tftpl$", lower(var.source_path))) > 0 ? + templatefile(abspath(var.source_path), var.template_vars) : + file(abspath(var.source_path)) + ) : null + + # Docs from primary_content_body + docs_from_primary_source = [ + for doc in split(local.yaml_separator, coalesce(local.primary_content_body, local.single_file_raw_content, "")) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + + # Docs from .yaml files in a directory + directory_yaml_files = local.is_directory ? fileset(local.directory_absolute_path, "*.yaml") : [] + docs_from_directory_yamls = flatten([ + for file_name in local.directory_yaml_files : + [ + for doc in split(local.yaml_separator, file(format("%s/%s", local.directory_absolute_path, file_name))) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + ]) + + # Docs from .tftpl files in a directory + directory_template_files = local.is_directory ? fileset(local.directory_absolute_path, "*.tftpl") : [] + docs_from_directory_templates = flatten([ + for file_name in local.directory_template_files : + [ + for doc in split(local.yaml_separator, templatefile(format("%s/%s", local.directory_absolute_path, file_name), var.template_vars)) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + ]) + + all_parsed_docs = concat( + local.docs_from_primary_source, + local.docs_from_directory_yamls, + local.docs_from_directory_templates + ) + + # --- 5. Create the final map for `for_each` (keys must be unique strings) --- + docs_map = tomap({ + for index, doc in local.all_parsed_docs : index => doc + if length(trimspace(doc)) > 0 + }) +} + +# Apply all manifest files dynamically +resource "kubernetes_manifest" "apply_manifests" { + for_each = local.docs_map + manifest = yamldecode(each.value) + timeouts { + create = var.resource_timeouts.create + update = var.resource_timeouts.update + delete = var.resource_timeouts.delete + } + + dynamic "wait" { + for_each = var.wait_for_rollout ? [1] : [] + content { + rollout = var.wait_for_rollout + fields = var.wait_for_fields + } + } + + # Configure the 'field_manager' block dynamically + dynamic "field_manager" { + for_each = var.field_manager != null ? [var.field_manager] : [] + content { + name = field_manager.value.name + force_conflicts = field_manager.value.force_conflicts + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml new file mode 100644 index 0000000000..e18197e2b7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf new file mode 100644 index 0000000000..0b846189ea --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf @@ -0,0 +1,69 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +variable "content" { + description = "The YAML body to apply to gke cluster." + type = string + default = null +} + +variable "source_path" { + description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." + type = string + default = "" +} + +variable "template_vars" { + description = "The values to populate template file(s) with." + type = any + default = null +} + +variable "wait_for_rollout" { + description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." + type = bool + default = true +} + + +variable "wait_for_fields" { + description = "(Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern." + type = map(string) + default = {} +} + +variable "resource_timeouts" { + description = "(Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met." + type = object({ + create = optional(string, null) + update = optional(string, null) + delete = optional(string, null) + }) + default = { + create = "15m" # Default create timeout, also covers waiting for initial conditions + update = "10m" # Default update timeout, also covers waiting for update conditions + delete = "5m" # Default delete timeout + } +} + +variable "field_manager" { + description = "(Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts." + type = object({ + name = optional(string, null) + force_conflicts = optional(bool, false) + }) + default = null +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf new file mode 100644 index 0000000000..61786b06de --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + # Defines the providers that this module depends on and their versions. + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.23" + } + } + required_version = ">= 1.3" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/main.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/main.tf new file mode 100644 index 0000000000..8db4870452 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/main.tf @@ -0,0 +1,183 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] + project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] + + install_gpu_operator = try(var.gpu_operator.install, false) + install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) +} + +data "google_container_cluster" "gke_cluster" { + project = local.project_id + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +module "install_kueue" { + source = "./helm_install" + depends_on = [var.gke_cluster_exists] + + release_name = "kueue" + + chart_name = "oci://registry.k8s.io/kueue/charts/kueue" + chart_version = var.kueue.version # Specify your desired Kueue version + + create_namespace = true # Helm can also create the namespace + wait = true + timeout = 600 # seconds +} + +module "install_jobset" { + source = "./helm_install" + depends_on = [var.gke_cluster_exists, module.install_kueue] + release_name = "jobset-controller" # The release name for your JobSet installation + chart_name = "oci://registry.k8s.io/jobset/charts/jobset" # The Helm repository URL for nvidia charts + chart_version = var.jobset.version + create_namespace = true + namespace = "jobset-system" +} + +module "install_nvidia_dra_driver" { + count = local.install_nvidia_dra_driver ? 1 : 0 + depends_on = [var.gke_cluster_exists] + source = "./helm_install" + + release_name = "nvidia-dra-driver-gpu" # The release name + chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts + chart_name = "nvidia-dra-driver-gpu" # The chart name + chart_version = var.nvidia_dra_driver.version # The chart version + namespace = "nvidia-dra-driver-gpu" # The target namespace + create_namespace = true # Equivalent to --create-namespace + + # Use the 'values' argument to pass the YAML content + # This corresponds to the -f <(cat < +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.2 | +| [google](#requirement\_google) | >= 6.40 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.40 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_global_address.private_ip_alloc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_global_address) | resource | +| [google_compute_network_peering_routes_config.private_vpc_peering_routes_gcnv](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering_routes_config) | resource | +| [google_service_networking_connection.private_vpc_connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_networking_connection) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [address](#input\_address) | The IP address or beginning of the address range allocated for the Private Service Access. | `string` | `null` | no | +| [deletion\_policy](#input\_deletion\_policy) | The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to supporting resources. Key-value pairs. | `map(string)` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to configure Private Service Access:
`projects//global/networks/`" | `string` | n/a | yes | +| [prefix\_length](#input\_prefix\_length) | The prefix length of the IP range allocated for the Private Service Access. | `number` | `16` | no | +| [project\_id](#input\_project\_id) | ID of project in which Private Service Access will be created. | `string` | n/a | yes | +| [service\_name](#input\_service\_name) | The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'. | `string` | `"servicenetworking.googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cidr\_range](#output\_cidr\_range) | CIDR range of the created google\_compute\_global\_address | +| [connect\_mode](#output\_connect\_mode) | Services that use Private Service Access typically specify connect\_mode
"PRIVATE\_SERVICE\_ACCESS". This output value sets connect\_mode and additionally
blocks terraform actions until the VPC connection has been created. | +| [private\_vpc\_connection\_peering](#output\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection that was created by the service provider. | +| [reserved\_ip\_range](#output\_reserved\_ip\_range) | Named IP range to be used by services connected with Private Service Access. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/main.tf b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/main.tf new file mode 100644 index 0000000000..429e4d93f0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/main.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "private-service-access", ghpc_role = "network" }) +} + +locals { + split_network_id = split("/", var.network_id) + network_name = local.split_network_id[4] + network_project = local.split_network_id[1] +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_compute_global_address" "private_ip_alloc" { + provider = google + name = "global-psconnect-ip-${random_id.resource_name_suffix.hex}" + project = var.project_id + purpose = "VPC_PEERING" + address_type = "INTERNAL" + network = var.network_id + prefix_length = var.prefix_length + labels = local.labels + address = var.address +} + +resource "google_service_networking_connection" "private_vpc_connection" { + network = var.network_id + service = var.service_name + reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name] + deletion_policy = var.deletion_policy + update_on_creation_fail = var.deletion_policy == "ABANDON" ? true : null +} + +# Google Cloud NetApp Volumes need enablement of custom_route import and export +resource "google_compute_network_peering_routes_config" "private_vpc_peering_routes_gcnv" { + count = var.service_name == "netapp.servicenetworking.goog" ? 1 : 0 + project = local.network_project + network = local.network_name + peering = google_service_networking_connection.private_vpc_connection.peering + + export_custom_routes = true + import_custom_routes = true +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/metadata.yaml new file mode 100644 index 0000000000..93e8b3970e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - servicenetworking.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/outputs.tf new file mode 100644 index 0000000000..296f2e9140 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/outputs.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "private_vpc_connection_peering" { + description = "The name of the VPC Network peering connection that was created by the service provider." + sensitive = true + value = google_service_networking_connection.private_vpc_connection.peering +} + +output "connect_mode" { + description = <<-EOT + Services that use Private Service Access typically specify connect_mode + "PRIVATE_SERVICE_ACCESS". This output value sets connect_mode and additionally + blocks terraform actions until the VPC connection has been created. + EOT + value = "PRIVATE_SERVICE_ACCESS" + depends_on = [ + google_service_networking_connection.private_vpc_connection, + ] +} + +output "reserved_ip_range" { + description = "Named IP range to be used by services connected with Private Service Access." + value = google_compute_global_address.private_ip_alloc.name +} + +output "cidr_range" { + description = "CIDR range of the created google_compute_global_address" + value = "${google_compute_global_address.private_ip_alloc.address}/${google_compute_global_address.private_ip_alloc.prefix_length}" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/variables.tf new file mode 100644 index 0000000000..4b0a3e796f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/variables.tf @@ -0,0 +1,59 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "address" { + description = "The IP address or beginning of the address range allocated for the Private Service Access." + type = string + default = null +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to configure Private Service Access: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "labels" { + description = "Labels to add to supporting resources. Key-value pairs." + type = map(string) +} + +variable "prefix_length" { + description = "The prefix length of the IP range allocated for the Private Service Access." + type = number + default = 16 +} + +variable "project_id" { + description = "ID of project in which Private Service Access will be created." + type = string +} + +variable "service_name" { + description = "The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'." + type = string + default = "servicenetworking.googleapis.com" +} + +variable "deletion_policy" { + description = "The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON." + type = string + default = null +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/versions.tf new file mode 100644 index 0000000000..df2914cdb9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.40" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" + } + + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" + } + + required_version = ">= 1.2" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/new-project/README.md b/deletion-test/cluster/modules/embedded/community/modules/project/new-project/README.md new file mode 100644 index 0000000000..5e5cabe9d5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/new-project/README.md @@ -0,0 +1,128 @@ +## Description + +This module allows you to create opinionated Google Cloud Platform projects. It +creates projects and configures aspects like Shared VPC connectivity, IAM +access, Service Accounts, and API enablement to follow best practices. + +This module is meant for use with Terraform 0.13. + +**Note:** This module has been removed from the Cluster Toolkit. The upstream module (`terraform-google-project-factory`) is now the recommended way to create and manage GCP projects. + +### Example + +```yaml +- id: project + source: github.com/terraform-google-modules/terraform-google-project-factory?rev=v17.0.0&depth=1 +``` + +This creates a new project with pre-defined project ID, a designated folder and +organization and associated billing account which will be used to pay for +services consumed. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [project\_factory](#module\_project\_factory) | terraform-google-modules/project-factory/google | ~> 11.3 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [activate\_api\_identities](#input\_activate\_api\_identities) | The list of service identities (Google Managed service account for the API) to force-create for the project (e.g. in order to grant additional roles).
APIs in this list will automatically be appended to `activate_apis`.
Not including the API in this list will follow the default behaviour for identity creation (which is usually when the first resource using the API is created).
Any roles (e.g. service agent role) must be explicitly listed. See https://cloud.google.com/iam/docs/understanding-roles#service-agent-roles-roles for a list of related roles. | 
list(object({
    api   = string
    roles = list(string)
  }))
| `[]` | no | +| [activate\_apis](#input\_activate\_apis) | The list of apis to activate within the project | `list(string)` | 
[
  "compute.googleapis.com",
  "serviceusage.googleapis.com",
  "storage.googleapis.com"
]
| no | +| [auto\_create\_network](#input\_auto\_create\_network) | Create the default network | `bool` | `false` | no | +| [billing\_account](#input\_billing\_account) | The ID of the billing account to associate this project with | `string` | n/a | yes | +| [bucket\_force\_destroy](#input\_bucket\_force\_destroy) | Force the deletion of all objects within the GCS bucket when deleting the bucket (optional) | `bool` | `false` | no | +| [bucket\_labels](#input\_bucket\_labels) | A map of key/value label pairs to assign to the bucket (optional) | `map(string)` | `{}` | no | +| [bucket\_location](#input\_bucket\_location) | The location for a GCS bucket to create (optional) | `string` | `"US"` | no | +| [bucket\_name](#input\_bucket\_name) | A name for a GCS bucket to create (in the bucket\_project project), useful for Terraform state (optional) | `string` | `""` | no | +| [bucket\_project](#input\_bucket\_project) | A project to create a GCS bucket (bucket\_name) in, useful for Terraform state (optional) | `string` | `""` | no | +| [bucket\_ula](#input\_bucket\_ula) | Enable Uniform Bucket Level Access | `bool` | `true` | no | +| [bucket\_versioning](#input\_bucket\_versioning) | Enable versioning for a GCS bucket to create (optional) | `bool` | `false` | no | +| [budget\_alert\_pubsub\_topic](#input\_budget\_alert\_pubsub\_topic) | The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}` | `string` | `null` | no | +| [budget\_alert\_spent\_percents](#input\_budget\_alert\_spent\_percents) | A list of percentages of the budget to alert on when threshold is exceeded | `list(number)` | 
[
  0.5,
  0.7,
  1
]
| no | +| [budget\_amount](#input\_budget\_amount) | The amount to use for a budget alert | `number` | `null` | no | +| [budget\_display\_name](#input\_budget\_display\_name) | The display name of the budget. If not set defaults to `Budget For ` | `string` | `null` | no | +| [budget\_monitoring\_notification\_channels](#input\_budget\_monitoring\_notification\_channels) | A list of monitoring notification channels in the form `[projects/{project_id}/notificationChannels/{channel_id}]`. A maximum of 5 channels are allowed. | `list(string)` | `[]` | no | +| [consumer\_quotas](#input\_consumer\_quotas) | The quotas configuration you want to override for the project. | 
list(object({
    service = string,
    metric  = string,
    limit   = string,
    value   = string,
  }))
| `[]` | no | +| [create\_project\_sa](#input\_create\_project\_sa) | Whether the default service account for the project shall be created | `bool` | `true` | no | +| [default\_network\_tier](#input\_default\_network\_tier) | Default Network Service Tier for resources created in this project. If unset, the value will not be modified. See https://cloud.google.com/network-tiers/docs/using-network-service-tiers and https://cloud.google.com/network-tiers. | `string` | `""` | no | +| [default\_service\_account](#input\_default\_service\_account) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | `string` | `"keep"` | no | +| [disable\_dependent\_services](#input\_disable\_dependent\_services) | Whether services that are enabled and which depend on this service should also be disabled when this service is destroyed. | `bool` | `true` | no | +| [disable\_services\_on\_destroy](#input\_disable\_services\_on\_destroy) | Whether project services will be disabled when the resources are destroyed | `bool` | `true` | no | +| [domain](#input\_domain) | The domain name (optional). | `string` | `""` | no | +| [enable\_shared\_vpc\_host\_project](#input\_enable\_shared\_vpc\_host\_project) | If this project is a shared VPC host project. If true, you must *not* set svpc\_host\_project\_id variable. Default is false. | `bool` | `false` | no | +| [folder\_id](#input\_folder\_id) | The ID of a folder to host this project | `string` | `""` | no | +| [grant\_services\_network\_role](#input\_grant\_services\_network\_role) | Whether or not to grant service agents the network roles on the host project | `bool` | `true` | no | +| [grant\_services\_security\_admin\_role](#input\_grant\_services\_security\_admin\_role) | Whether or not to grant Kubernetes Engine Service Agent the Security Admin role on the host project so it can manage firewall rules | `bool` | `false` | no | +| [group\_name](#input\_group\_name) | A group to control the project by being assigned group\_role (defaults to project editor) | `string` | `""` | no | +| [group\_role](#input\_group\_role) | The role to give the controlling group (group\_name) over the project (defaults to project editor) | `string` | `"roles/editor"` | no | +| [labels](#input\_labels) | Map of labels for project | `map(string)` | `{}` | no | +| [lien](#input\_lien) | Add a lien on the project to prevent accidental deletion | `bool` | `false` | no | +| [name](#input\_name) | The name for the project | `string` | `null` | no | +| [org\_id](#input\_org\_id) | The organization ID. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The ID to give the project. If not provided, the `name` will be used. | `string` | `""` | no | +| [project\_sa\_name](#input\_project\_sa\_name) | Default service account name for the project. | `string` | `"project-service-account"` | no | +| [random\_project\_id](#input\_random\_project\_id) | Adds a suffix of 4 random characters to the `project_id` | `bool` | `false` | no | +| [sa\_role](#input\_sa\_role) | A role to give the default Service Account for the project (defaults to none) | `string` | `""` | no | +| [shared\_vpc\_subnets](#input\_shared\_vpc\_subnets) | List of subnets fully qualified subnet IDs (ie. projects/$project\_id/regions/$region/subnetworks/$subnet\_id) | `list(string)` | `[]` | no | +| [svpc\_host\_project\_id](#input\_svpc\_host\_project\_id) | The ID of the host project which hosts the shared VPC | `string` | `""` | no | +| [usage\_bucket\_name](#input\_usage\_bucket\_name) | Name of a GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | +| [usage\_bucket\_prefix](#input\_usage\_bucket\_prefix) | Prefix in the GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | +| [vpc\_service\_control\_attach\_enabled](#input\_vpc\_service\_control\_attach\_enabled) | Whether the project will be attached to a VPC Service Control Perimeter | `bool` | `false` | no | +| [vpc\_service\_control\_perimeter\_name](#input\_vpc\_service\_control\_perimeter\_name) | The name of a VPC Service Control Perimeter to add the created project to | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [api\_s\_account](#output\_api\_s\_account) | API service account email | +| [api\_s\_account\_fmt](#output\_api\_s\_account\_fmt) | API service account email formatted for terraform use | +| [budget\_name](#output\_budget\_name) | The name of the budget if created | +| [domain](#output\_domain) | The organization's domain | +| [enabled\_api\_identities](#output\_enabled\_api\_identities) | Enabled API identities in the project | +| [enabled\_apis](#output\_enabled\_apis) | Enabled APIs in the project | +| [group\_email](#output\_group\_email) | The email of the G Suite group with group\_name | +| [project\_bucket\_self\_link](#output\_project\_bucket\_self\_link) | Project's bucket selfLink | +| [project\_bucket\_url](#output\_project\_bucket\_url) | Project's bucket url | +| [project\_id](#output\_project\_id) | ID of the project that was created | +| [project\_name](#output\_project\_name) | Name of the project that was created | +| [project\_number](#output\_project\_number) | Number of the project that was created | +| [service\_account\_display\_name](#output\_service\_account\_display\_name) | The display name of the default service account | +| [service\_account\_email](#output\_service\_account\_email) | The email of the default service account | +| [service\_account\_id](#output\_service\_account\_id) | The id of the default service account | +| [service\_account\_name](#output\_service\_account\_name) | The fully-qualified name of the default service account | +| [service\_account\_unique\_id](#output\_service\_account\_unique\_id) | The unique id of the default service account | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/README.md b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/README.md new file mode 100644 index 0000000000..0f5c10c7e4 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/README.md @@ -0,0 +1,111 @@ +## Description + +Allows creation of service accounts for a Google Cloud Platform project. + +### Example + +```yaml +- id: service_acct + source: community/modules/project/service-account + settings: + project_id: $(vars.project_id) + name: instance_acct + project_roles: + - logging.logWriter + - monitoring.metricWriter + - storage.objectViewer +``` + +This creates a service account in GCP project "project_id" with the name +"instance_acct". It will have the 3 roles listed for all resources within the +project. + +### Usage with startup-script module + +When this module is used in conjunction with the [startup-script] module, the +service account must be granted (at least) read access to the bucket. This can +be achieved by granting project-wide access as shown above or by specifying the +service account as a bucket viewer in the startup-script module: + +```yaml +- id: service_acct + source: community/modules/project/service-account + settings: + project_id: $(vars.project_id) + name: instance_acct + project_roles: + - logging.logWriter + - monitoring.metricWriter +- id: script + source: modules/scripts/startup-script + settings: + bucket_viewers: + - $(service_acct.service_account_iam_email) +``` + +[startup-script]: ../../../../modules/scripts/startup-script/README.md + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [service\_account](#module\_service\_account) | terraform-google-modules/service-accounts/google | ~> 4.2 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [billing\_account\_id](#input\_billing\_account\_id) | If assigning billing role, specify a billing account (default is to assign at the organizational level). | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment (will be prepended to service account name) | `string` | n/a | yes | +| [description](#input\_description) | Description of the created service account. | `string` | `"Service Account"` | no | +| [descriptions](#input\_descriptions) | Deprecated; create single service accounts using var.description. | `list(string)` | `null` | no | +| [display\_name](#input\_display\_name) | Display name of the created service account. | `string` | `"Service Account"` | no | +| [generate\_keys](#input\_generate\_keys) | Generate keys for service account. | `bool` | `false` | no | +| [grant\_billing\_role](#input\_grant\_billing\_role) | Grant billing user role. | `bool` | `false` | no | +| [grant\_xpn\_roles](#input\_grant\_xpn\_roles) | Grant roles for shared VPC management. | `bool` | `true` | no | +| [name](#input\_name) | Name of the service account to create. | `string` | n/a | yes | +| [names](#input\_names) | Deprecated; create single service accounts using var.name. | `list(string)` | `null` | no | +| [org\_id](#input\_org\_id) | Id of the organization for org-level roles. | `string` | `""` | no | +| [prefix](#input\_prefix) | Deprecated; prefix now set using var.deployment\_name | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | +| [project\_roles](#input\_project\_roles) | List of roles to grant to service account (e.g. "storage.objectViewer" or "compute.instanceAdmin.v1" | `list(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [key](#output\_key) | Service account key (if creation was requested) | +| [service\_account\_email](#output\_service\_account\_email) | Service account e-mail address | +| [service\_account\_iam\_email](#output\_service\_account\_iam\_email) | Service account IAM binding format (serviceAccount:name@example.com) | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/main.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/main.tf new file mode 100644 index 0000000000..e8a69be642 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/main.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + display_name = "${var.display_name} (${var.deployment_name})" + description = "${var.description} (${var.deployment_name})" +} + +module "service_account" { + source = "terraform-google-modules/service-accounts/google" + version = "~> 4.2" + + billing_account_id = var.billing_account_id + description = local.description + display_name = local.display_name + generate_keys = var.generate_keys + grant_billing_role = var.grant_billing_role + grant_xpn_roles = var.grant_xpn_roles + names = [var.name] + org_id = var.org_id + prefix = var.deployment_name + project_id = var.project_id + project_roles = [for role in var.project_roles : "${var.project_id}=>roles/${role}"] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/metadata.yaml new file mode 100644 index 0000000000..c4dcdffdf4 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/outputs.tf new file mode 100644 index 0000000000..f9c9be05c8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/outputs.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "key" { + description = "Service account key (if creation was requested)" + value = module.service_account.key +} + +output "service_account_email" { + description = "Service account e-mail address" + value = module.service_account.email + depends_on = [ + module.service_account, + ] +} + +output "service_account_iam_email" { + description = "Service account IAM binding format (serviceAccount:name@example.com)" + value = module.service_account.iam_email + depends_on = [ + module.service_account, + ] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/variables.tf new file mode 100644 index 0000000000..53267f47e7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/variables.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "billing_account_id" { + description = "If assigning billing role, specify a billing account (default is to assign at the organizational level)." + type = string + default = "" +} + +variable "deployment_name" { + description = "Name of the deployment (will be prepended to service account name)" + type = string +} + +variable "description" { + description = "Description of the created service account." + type = string + default = "Service Account" +} + +# tflint-ignore: terraform_unused_declarations +variable "descriptions" { + description = "Deprecated; create single service accounts using var.description." + type = list(string) + default = null + + validation { + condition = var.descriptions == null + error_message = "var.descriptions has been deprecated in favor of creating single accounts with var.description" + } +} + +variable "display_name" { + description = "Display name of the created service account." + type = string + default = "Service Account" +} + +variable "generate_keys" { + description = "Generate keys for service account." + type = bool + default = false +} + +variable "grant_billing_role" { + description = "Grant billing user role." + type = bool + default = false +} + +variable "grant_xpn_roles" { + description = "Grant roles for shared VPC management." + type = bool + default = true +} + +variable "name" { + description = "Name of the service account to create." + type = string +} + +# tflint-ignore: terraform_unused_declarations +variable "names" { + description = "Deprecated; create single service accounts using var.name." + type = list(string) + default = null + + validation { + condition = var.names == null + error_message = "var.names has been deprecated in favor of creating single accounts with var.name" + } +} + +variable "org_id" { + description = "Id of the organization for org-level roles." + type = string + default = "" +} + +# tflint-ignore: terraform_unused_declarations +variable "prefix" { + description = "Deprecated; prefix now set using var.deployment_name" + type = string + default = null + + validation { + condition = var.prefix == null + error_message = "var.prefix has been deprecated in favor of setting prefix with var.deployment_name" + } +} + +variable "project_id" { + description = "ID of the project" + type = string +} + +variable "project_roles" { + description = "List of roles to grant to service account (e.g. \"storage.objectViewer\" or \"compute.instanceAdmin.v1\"" + type = list(string) +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/versions.tf new file mode 100644 index 0000000000..38e6e71945 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/versions.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/README.md b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/README.md new file mode 100644 index 0000000000..266eac26ec --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/README.md @@ -0,0 +1,70 @@ +## Description + +Allows management of multiple API services for a Google Cloud Platform project. + +### Example + +```yaml +- id: services-api + source: community/modules/project/service-enablement + settings: + gcp_service_list: [ + "file.googleapis.com", + "compute.googleapis.com" + ] +``` + +This allows the project to enable both the filestore API as well as the compute API. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_project_service.gcp_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [disable\_on\_destroy](#input\_disable\_on\_destroy) | Disable services on destroy if they were enabled (or already enabled) during apply (default: false) | `bool` | `false` | no | +| [gcp\_service\_list](#input\_gcp\_service\_list) | list of APIs to be enabled for the project | `list(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/main.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/main.tf new file mode 100644 index 0000000000..965e93c549 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/main.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +resource "google_project_service" "gcp_services" { + count = length(var.gcp_service_list) + project = var.project_id + service = var.gcp_service_list[count.index] + timeouts { + create = "30m" + update = "40m" + } + + disable_dependent_services = true + disable_on_destroy = var.disable_on_destroy +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/metadata.yaml new file mode 100644 index 0000000000..c594c8f819 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - serviceusage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/variables.tf new file mode 100644 index 0000000000..08f13999fe --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "ID of the project" + type = string +} + +variable "gcp_service_list" { + description = "list of APIs to be enabled for the project" + type = list(string) +} + +variable "disable_on_destroy" { + description = "Disable services on destroy if they were enabled (or already enabled) during apply (default: false)" + type = bool + default = false +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/versions.tf new file mode 100644 index 0000000000..07f25fb045 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/README.md b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/README.md new file mode 100644 index 0000000000..052e6aee23 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/README.md @@ -0,0 +1,87 @@ +# Description + +This module creates a Bigquery Pub/Sub Subscription. + +Primarily used for FSI - MonteCarlo Tutorial: +**[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Example + +The following example creates a Bigquery subscription using a Bigquery table and +Pub/Sub topic. + +```yaml + - id: bq_subscription + source: community/modules/pubsub/bigquery-sub + use: [bq-table, pubsub_topic] +``` + +Also see usages in this +[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_project_iam_member.editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_project_iam_member.viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_pubsub_subscription.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Name of the dataset that was created. Can be provided by the bigquery-table module | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [subscription\_id](#input\_subscription\_id) | The name of the pubsub subscription to be created | `string` | `null` | no | +| [table\_id](#input\_table\_id) | ID of created BQ table. Can be provided by the bigquery-table module | `string` | n/a | yes | +| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [subscription\_id](#output\_subscription\_id) | Name of the subscription that was created. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf new file mode 100644 index 0000000000..8edbc6b24e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf @@ -0,0 +1,57 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-sub", ghpc_role = "pubsub" }) +} + +locals { + subscription_id = var.subscription_id != null ? var.subscription_id : "${var.deployment_name}_subscription_${random_id.resource_name_suffix.hex}" +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_project_iam_member" "viewer" { + project = data.google_project.project.project_id + role = "roles/bigquery.metadataViewer" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +resource "google_project_iam_member" "editor" { + project = data.google_project.project.project_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +resource "google_pubsub_subscription" "example" { + depends_on = [google_project_iam_member.editor, google_project_iam_member.viewer] + name = local.subscription_id + topic = var.topic_id + project = var.project_id + labels = local.labels + bigquery_config { + table = "${var.project_id}.${var.dataset_id}.${var.table_id}" + use_topic_schema = true + write_metadata = true + } + +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml new file mode 100644 index 0000000000..9aedef48dc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - pubsub.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf new file mode 100644 index 0000000000..fc81859503 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "subscription_id" { + description = "Name of the subscription that was created." + value = google_pubsub_subscription.example.name +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf new file mode 100644 index 0000000000..ee4dbbed8e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "topic_id" { + description = "The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module" + type = string +} + +variable "subscription_id" { + description = "The name of the pubsub subscription to be created" + type = string + default = null +} + +variable "dataset_id" { + description = "Name of the dataset that was created. Can be provided by the bigquery-table module" + type = string +} + +variable "table_id" { + description = "ID of created BQ table. Can be provided by the bigquery-table module" + type = string +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf new file mode 100644 index 0000000000..46ad6e17c8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" + } + required_version = ">= 1.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/README.md b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/README.md new file mode 100644 index 0000000000..177f799dc6 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/README.md @@ -0,0 +1,82 @@ +## Description + +Creates a Pub/Sub topic + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +### Example + +The following example creates a Pub/Sub topic. + +```yaml + - id: pubsub_topic + source: community/modules/pubsub/topic +``` + +Also see usages in this +[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_pubsub_schema.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_schema) | resource | +| [google_pubsub_topic.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [schema\_id](#input\_schema\_id) | The name of the pubsub schema to be created | `string` | `null` | no | +| [schema\_json](#input\_schema\_json) | The JSON definition of the pubsub topic schema | `string` | `"{ \n \"name\" : \"Avro\", \n \"type\" : \"record\", \n \"fields\" : \n [\n {\"name\" : \"ticker\", \"type\" : \"string\"},\n {\"name\" : \"epoch_time\", \"type\" : \"int\"},\n {\"name\" : \"iteration\", \"type\" : \"int\"},\n {\"name\" : \"start_date\", \"type\" : \"string\"},\n {\"name\" : \"end_date\", \"type\" : \"string\"},\n {\n \"name\":\"simulation_results\",\n \"type\":{\n \"type\": \"array\", \n \"items\":{\n \"name\":\"Child\",\n \"type\":\"record\",\n \"fields\":[\n {\"name\":\"price\", \"type\":\"double\"}\n ]\n }\n }\n }\n ]\n }\n"` | no | +| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to be created | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [topic\_id](#output\_topic\_id) | Name of the topic that was created. | +| [topic\_schema](#output\_topic\_schema) | Name of the topic schema that was created. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/main.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/main.tf new file mode 100644 index 0000000000..4ba68fb5d0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "topic", ghpc_role = "pubsub" }) +} + +locals { + topic_id = var.topic_id != null ? var.topic_id : "${var.deployment_name}_topic_${random_id.resource_name_suffix.hex}" + schema_id = var.schema_id != null ? var.schema_id : "${var.deployment_name}_schema_${random_id.resource_name_suffix.hex}" +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_pubsub_topic" "example" { + name = local.topic_id + depends_on = [google_pubsub_schema.example] + project = var.project_id + labels = local.labels + schema_settings { + schema = "projects/${var.project_id}/schemas/${local.schema_id}" + encoding = "BINARY" + } +} + +resource "google_pubsub_schema" "example" { + name = local.schema_id + project = var.project_id + type = "AVRO" + + definition = var.schema_json +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/metadata.yaml new file mode 100644 index 0000000000..9aedef48dc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - pubsub.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/outputs.tf new file mode 100644 index 0000000000..3ea9d951b2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/outputs.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "topic_id" { + description = "Name of the topic that was created." + value = google_pubsub_topic.example.name +} + + +output "topic_schema" { + description = "Name of the topic schema that was created." + value = local.schema_id +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/variables.tf new file mode 100644 index 0000000000..dca575d21d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/variables.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "topic_id" { + description = "The name of the pubsub topic to be created" + type = string + default = null +} + +variable "schema_id" { + description = "The name of the pubsub schema to be created" + type = string + default = null +} + +variable "schema_json" { + description = "The JSON definition of the pubsub topic schema" + type = string + default = < **Note**: This is an experimental module. This module has only been tested in +> limited capacity with the Cluster Toolkit. The module interface may have undergo +> breaking changes in the future. + +### Example + +The following example will create a single GPU accelerated remote desktop. + +```yaml + - id: remote-desktop + source: community/modules/remote-desktop/chrome-remote-desktop + use: [network1] + settings: + install_nvidia_driver: true +``` + +### Setting up the Remote Desktop + +1. Once the remote desktop has been deployed, navigate to https://remotedesktop.google.com/headless. +1. Click through `Begin`, `Next`, & `Authorize`. +1. Copy the code snippet for `Debian Linux`. +1. SSH into the remote desktop machine. It will be listed under + [VM Instances](https://console.cloud.google.com/compute/instances) in the + Google Cloud web console. +1. Run the copied command and follow instructions to set up a PIN. +1. You should now see your machine listed on the + [Chrome Remote Desktop page](https://remotedesktop.google.com/access) under `Remote devices`. +1. Click on your machine and enter PIN if prompted. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.12.31 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [client\_startup\_script](#module\_client\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | +| [instances](#module\_instances) | ../../../../modules/compute/vm-instance | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | +| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image th
at supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-v
m-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. Cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | +| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-balanced"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs on the internet. | `bool` | `true` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required | 
list(object({
    type  = string,
    count = number
  }))
| 
[
  {
    "count": 1,
    "type": "nvidia-tesla-t4-vws"
  }
]
| no | +| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Installs the nvidia driver (true/false). For details, see https://cloud.google.com/compute/docs/gpus/install-drivers-gpu | `bool` | n/a | yes | +| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | +| [instance\_image](#input\_instance\_image) | Image used to build chrome remote desktop node. The default image is
name="debian-12-bookworm-v20250610" and project="debian-cloud".
NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues.

An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud".

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "name": "debian-12-bookworm-v20250610",
  "project": "debian-cloud"
}
| no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation. Must be N1 family if GPU is used. | `string` | `"n1-standard-8"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface
**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.
Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "RDMA", "IRDMA", "MRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. | `string` | `"default"` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `"TERMINATE"` | no | +| [project\_id](#input\_project\_id) | Project in which Google Cloud resources will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/cloud-platform"
  ]
}
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core | `number` | `2` | no | +| [zone](#input\_zone) | Default zone for creating resources | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_name](#output\_instance\_name) | Name of the first instance created, if any. | +| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf new file mode 100644 index 0000000000..a5cf7c5d37 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "chrome-remote-desktop", ghpc_role = "remote-desktop" }) +} + +locals { + + user_startup_script_runners = var.startup_script == null ? [] : [ + { + type = "shell" + content = var.startup_script + destination = "user_startup_script.sh" + } + ] + + configure_nvidia_driver_runners = var.install_nvidia_driver == false ? [] : [ + { + type = "ansible-local" + content = file("${path.module}/scripts/configure-grid-drivers.yml") + destination = "/usr/local/ghpc/configure-grid-drivers.yml" + } + ] + + configure_chrome_remote_desktop_runners = [ + { + type = "ansible-local" + content = file("${path.module}/scripts/configure-chrome-desktop.yml") + destination = "/usr/local/ghpc/configure-chrome-desktop.yml" + } + ] + + disable_sleep = [ + { + type = "ansible-local" + content = file("${path.module}/scripts/disable-sleep.yml") + destination = "/usr/local/ghpc/disable-sleep.yml" + } + ] +} + +module "client_startup_script" { + source = "../../../../modules/scripts/startup-script" + + deployment_name = var.deployment_name + project_id = var.project_id + region = var.region + labels = local.labels + + runners = flatten([ + local.user_startup_script_runners, + local.configure_nvidia_driver_runners, + local.configure_chrome_remote_desktop_runners, + local.disable_sleep + ]) +} + +module "instances" { + source = "../../../../modules/compute/vm-instance" + + instance_count = var.instance_count + name_prefix = var.name_prefix + add_deployment_name_before_prefix = var.add_deployment_name_before_prefix + provisioning_model = var.spot ? "SPOT" : null + + deployment_name = var.deployment_name + project_id = var.project_id + region = var.region + zone = var.zone + labels = local.labels + + machine_type = var.machine_type + service_account_email = var.service_account.email + metadata = var.metadata + startup_script = module.client_startup_script.startup_script + enable_oslogin = var.enable_oslogin + + instance_image = var.instance_image + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + auto_delete_boot_disk = var.auto_delete_boot_disk + + disable_public_ips = !var.enable_public_ips + network_self_link = var.network_self_link + subnetwork_self_link = var.subnetwork_self_link + network_interfaces = var.network_interfaces + bandwidth_tier = var.bandwidth_tier + tags = var.tags + + threads_per_core = var.threads_per_core + guest_accelerator = var.guest_accelerator + on_host_maintenance = var.on_host_maintenance + + network_storage = var.network_storage + +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf new file mode 100644 index 0000000000..bcf8ece52d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "script to load and run all runners, as a string value." + value = module.client_startup_script.startup_script +} + +output "instance_name" { + description = "Name of the first instance created, if any." + value = var.instance_count > 0 ? module.instances.name[0] : null +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml new file mode 100644 index 0000000000..391aa86433 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml @@ -0,0 +1,61 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Ensure Desktop OS and Chrome Remote Desktop is installed + hosts: localhost + become: true + module_defaults: + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + tasks: + - name: Install desktop packages + ansible.builtin.apt: + name: + - xfce4 + - xfce4-goodies + state: present + register: apt_result + retries: 10 + delay: 30 + until: apt_result is success + + - name: Download and configure CRD + ansible.builtin.get_url: + url: https://dl.google.com/linux/direct/chrome-remote-desktop_current_amd64.deb + dest: /tmp/chrome-remote-desktop_current_amd64.deb + mode: "0755" + timeout: 30 + + - name: Install CRD + ansible.builtin.apt: + deb: /tmp/chrome-remote-desktop_current_amd64.deb + environment: + DEBIAN_FRONTEND: noninteractive + register: apt_result + retries: 10 + delay: 30 + until: apt_result is success + + - name: Configure CRD to use Xfce by default + ansible.builtin.copy: + dest: /etc/chrome-remote-desktop-session + content: "exec /etc/X11/Xsession /usr/bin/xfce4-session" + mode: 0644 + + - name: Start Chrome remote desktop + ansible.builtin.command: /etc/init.d/chrome-remote-desktop start + register: result + changed_when: result.rc == 0 diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml new file mode 100644 index 0000000000..daae08176d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml @@ -0,0 +1,163 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Ensure nvidia grid drivers and other binaries are installed + hosts: localhost + become: true + vars: + dist_settings: + bullseye: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-510.85.02-grid.run + grid_ver: vGPU14.2 + bookworm: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-550.54.15-grid.run + grid_ver: vGPU17.1 + jammy: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - gcc-12 # must match compiler used to build kernel on latest Ubuntu 22 + - pkg-config # observed to be necessary for GRID driver installation on latest Ubuntu 22 + - libglvnd-dev # observed to be necessary for GRID driver installation on latest Ubuntu 22 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-525.125.06-grid.run + grid_ver: vGPU15.3 + tasks: + - name: Fail if using wrong OS + ansible.builtin.assert: + that: + - ansible_os_family in ["Debian", "Ubuntu"] + - ansible_distribution_release in dist_settings.keys() | list + fail_msg: "ansible_os_family: {{ ansible_os_family }} or ansible_distribution_release: {{ansible_distribution_release}} was not acceptable." + + - name: Check if GRID driver installed + ansible.builtin.command: which nvidia-smi + register: nvidiasmi_result + ignore_errors: true + changed_when: false + + - name: Install binaries for GRID drivers + ansible.builtin.apt: + name: '{{ dist_settings[ansible_distribution_release]["packages"] }}' + state: present + update_cache: true + register: apt_result + retries: 6 + delay: 10 + until: apt_result is success + + - name: Install GRID driver if not existing + when: nvidiasmi_result is failed + block: + - name: Download GPU driver + ansible.builtin.get_url: + url: https://storage.googleapis.com/nvidia-drivers-us-public/GRID/{{ dist_settings[ansible_distribution_release]["grid_ver"] }}/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} + dest: /tmp/ + mode: "0755" + timeout: 30 + + - name: Stop gdm service + ansible.builtin.systemd: + name: gdm + state: stopped + + - name: Install GPU driver + ansible.builtin.shell: | + #jinja2: trim_blocks: "True" + {% if ansible_distribution_release == "jammy" %} + CC=gcc-12 /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent + {% else %} + /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent + {% endif %} + register: result + changed_when: result.rc == 0 + + - name: Download VirtualGL driver + ansible.builtin.get_url: + url: https://sourceforge.net/projects/virtualgl/files/3.0.2/virtualgl_3.0.2_amd64.deb/download + dest: /tmp/virtualgl_3.0.2_amd64.deb + mode: "0755" + timeout: 30 + + - name: Install VirtualGL + ansible.builtin.command: gdebi /tmp/virtualgl_3.0.2_amd64.deb --non-interactive + register: result + changed_when: result.rc == 0 + + - name: Fix headless Nvidia issue + block: + - name: Lookup gpu info + ansible.builtin.command: nvidia-xconfig --query-gpu-info + register: gpu_info + failed_when: gpu_info.rc != 0 + changed_when: false + + - name: Extract PCI ID + ansible.builtin.shell: | + set -o pipefail + echo "{{ gpu_info.stdout }}" | grep "PCI BusID " | head -n 1 | cut -d':' -f2-99 | xargs + args: + executable: /bin/bash + register: pci_id + changed_when: false + + - name: Configure nvidia-xconfig + ansible.builtin.command: nvidia-xconfig -a --allow-empty-initial-configuration --enable-all-gpus --virtual=1920x1200 --busid={{ pci_id.stdout }} + register: result + changed_when: result.rc == 0 + + - name: Set HardDPMS to false + ansible.builtin.replace: + path: /etc/X11/xorg.conf + regexp: "Section \"Device\"" + replace: "Section \"Device\"\n Option \"HardDPMS\" \"false\"" + + - name: Configure VirtualGL for X + ansible.builtin.command: vglserver_config +glx +s +f -t + register: result + changed_when: result.rc == 0 + + - name: Configure gdm for X + block: + - name: Configure default display manager + ansible.builtin.copy: + dest: /etc/X11/default-display-manager + content: "/usr/sbin/gdm3" + mode: 0644 + + - name: Switch boot target to gui + ansible.builtin.command: systemctl set-default graphical.target + register: result + changed_when: result.rc == 0 + + - name: Start gdm service + ansible.builtin.systemd: + name: gdm + daemon_reload: true + state: started diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml new file mode 100644 index 0000000000..6767b05fb2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml @@ -0,0 +1,39 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Mask sleep, suspend, hibernate, and hybrid-sleep targets + hosts: localhost + become: true + tasks: + + - name: Mask sleep target + ansible.builtin.systemd: + name: sleep.target + masked: true + + - name: Mask suspend target + ansible.builtin.systemd: + name: suspend.target + masked: true + + - name: Mask hibernate target + ansible.builtin.systemd: + name: hibernate.target + masked: true + + - name: Mask hybrid-sleep target + ansible.builtin.systemd: + name: hybrid-sleep.target + masked: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf new file mode 100644 index 0000000000..ac4c3b1869 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf @@ -0,0 +1,277 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which Google Cloud resources will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. Cloud resource names will include this value." + type = string + #default = "chrome-remote-desktop" +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zone" { + description = "Default zone for creating resources" + type = string +} + +variable "instance_count" { + description = "Number of instances" + type = number + default = 1 +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "instance_image" { + description = <<-EOD + Image used to build chrome remote desktop node. The default image is + name="debian-12-bookworm-v20250610" and project="debian-cloud". + NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues. + + An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud". + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "debian-cloud" + name = "debian-12-bookworm-v20250610" + } +} + +variable "disk_size_gb" { + description = "Size of disk for instances." + type = number + default = 200 +} + +variable "disk_type" { + description = "Disk type for instances." + type = string + default = "pd-balanced" +} + +variable "auto_delete_boot_disk" { + description = "Controls if boot disk should be auto-deleted when instance is deleted." + type = bool + default = true +} + +variable "name_prefix" { + description = <<-EOT + An optional name for all VM and disk resources. + If not supplied, `deployment_name` will be used. + When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, + then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". + EOT + type = string + default = null +} + +variable "add_deployment_name_before_prefix" { + description = <<-EOT + If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. + See `name_prefix` for further details on resource naming behavior. + EOT + type = bool + default = false +} + +variable "enable_public_ips" { + description = "If set to true, instances will have public IPs on the internet." + type = bool + default = true +} + +variable "machine_type" { + description = "Machine type to use for the instance creation. Must be N1 family if GPU is used." + type = string + default = "n1-standard-8" +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "service_account" { + description = "Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account." + type = object({ + email = string, + scopes = set(string) + }) + default = { + email = null + scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the VM." + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the VM." + type = string + default = null +} + +variable "network_interfaces" { + description = <<-EOT + A list of network interfaces. The options match that of the terraform + network_interface block of google_compute_instance. For descriptions of the + subfields or more information see the documentation: + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + **_NOTE:_** If `network_interfaces` are set, `network_self_link` and + `subnetwork_self_link` will be ignored, even if they are provided through + the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply + to network interfaces defined in this variable. + Subfields: + network (string, required if subnetwork is not supplied) + subnetwork (string, required if network is not supplied) + subnetwork_project (string, optional) + network_ip (string, optional) + nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "RDMA", "IRDMA", "MRDMA"]) + stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) + queue_count (number, optional) + access_config (object, optional) + ipv6_access_config (object, optional) + alias_ip_range (list(object), optional) + EOT + type = list(object({ + network = string, + subnetwork = string, + subnetwork_project = string, + network_ip = string, + nic_type = string, + stack_type = string, + queue_count = number, + access_config = list(object({ + nat_ip = string, + public_ptr_domain_name = string, + network_tier = string + })), + ipv6_access_config = list(object({ + public_ptr_domain_name = string, + network_tier = string + })), + alias_ip_range = list(object({ + ip_cidr_range = string, + subnetwork_range_name = string + })) + })) + default = [] +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "startup_script" { + description = "Startup script used on the instance" + type = string + default = null +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required" + type = list(object({ + type = string, + count = number + })) + default = [{ + type = "nvidia-tesla-t4-vws" + count = 1 + }] +} + +variable "threads_per_core" { + description = "Sets the number of threads per physical core" + type = number + default = 2 +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" + type = string + default = "TERMINATE" +} + +variable "bandwidth_tier" { + description = <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +This mode can be switched to proactive (automatic) replacement by setting +[var.update_policy](#input_update_policy) to "PROACTIVE". In this case we +recommend the use of Filestore to store the job queue state ("spool") and +setting [var.spool_parent_dir][#input_spool_parent_dir] to its mount point: + +```yaml + - id: spoolfs + source: modules/file-system/filestore + use: + - network1 + settings: + filestore_tier: ENTERPRISE + local_mount: /shared + +... + + - id: htcondor_access + source: community/modules/scheduler/htcondor-access-point + use: + - network1 + - spoolfs + - htcondor_secrets + - htcondor_setup + - htcondor_cm + - htcondor_execute_point_group + settings: + spool_parent_dir: /shared +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 3.83 | +| [null](#requirement\_null) | >= 3.0 | +| [random](#requirement\_random) | ~> 3.6 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [null](#provider\_null) | >= 3.0 | +| [random](#provider\_random) | ~> 3.6 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [access\_point\_instance\_template](#module\_access\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [htcondor\_ap](#module\_htcondor\_ap) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_address.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_region_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_region_disk) | resource | +| [google_storage_bucket_object.ap_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.ap_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_shuffle.zones](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/shuffle) | resource | +| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_instance.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | +| [google_compute_region_instance_group.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_runner](#input\_access\_point\_runner) | A list of Toolkit runners for configuring an HTCondor access point | `list(map(string))` | `[]` | no | +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account for access point (e-mail format) | `string` | n/a | yes | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [autoscaler\_runner](#input\_autoscaler\_runner) | A list of Toolkit runners for configuring autoscaling daemons | `list(map(string))` | `[]` | no | +| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | +| [default\_mig\_id](#input\_default\_mig\_id) | Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | +| [disk\_type](#input\_disk\_type) | Boot disk size in GB | `string` | `"pd-balanced"` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape acoss zones for instance group managing high availability of access point | `string` | `"ANY_SINGLE_ZONE"` | no | +| [enable\_high\_availability](#input\_enable\_high\_availability) | Provision HTCondor access point in high availability mode | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | Enable Public IPs on the access points | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor and Toolkit support installed."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | +| [mig\_id](#input\_mig\_id) | List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module) | `list(string)` | `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spool\_disk\_size\_gb](#input\_spool\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | +| [spool\_disk\_type](#input\_spool\_disk\_type) | Boot disk size in GB | `string` | `"pd-ssd"` | no | +| [spool\_parent\_dir](#input\_spool\_parent\_dir) | HTCondor access point configuration SPOOL will be set to subdirectory named "spool" | `string` | `"/var/lib/condor"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | +| [zones](#input\_zones) | Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_ips](#output\_access\_point\_ips) | IP addresses of the access points provisioned by this module | +| [access\_point\_name](#output\_access\_point\_name) | Name of the access point provisioned by this module | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml new file mode 100644 index 0000000000..6a2f50c831 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml @@ -0,0 +1,120 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Access Point + hosts: localhost + become: true + vars: + spool_dir: /var/lib/condor/spool + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + htcondor_spool_disk_device: /dev/disk/by-id/google-htcondor-spool-disk + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + - name: Configure HTCondor SchedD + when: htcondor_role == 'get_htcondor_submit' + block: + - name: Format spool disk + community.general.filesystem: + fstype: ext4 + state: present + dev: "{{ htcondor_spool_disk_device }}" + # RUN TUNE2FS + - name: Mount spool (creates mount point) + ansible.posix.mount: + path: "{{ spool_dir }}" + src: "{{ htcondor_spool_disk_device }}" + fstype: ext4 + opts: defaults + state: mounted + - name: Ensure spool free space + ansible.builtin.command: tune2fs -r 0 {{ htcondor_spool_disk_device }} + - name: Setup spool directory + ansible.builtin.file: + path: "{{ spool_dir }}" + state: directory + owner: condor + group: condor + mode: 0755 + recurse: true + - name: Create SystemD override directory for HTCondor + ansible.builtin.file: + path: /etc/systemd/system/condor.service.d + state: directory + owner: root + group: root + mode: 0755 + - name: Ensure HTCondor starts after shared filesystem is mounted + ansible.builtin.copy: + dest: /etc/systemd/system/condor.service.d/mount-spool.conf + mode: 0644 + content: | + [Unit] + RequiresMountsFor={{ spool_dir }} + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor configuration complete; startup-script may still be executing ********" diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf new file mode 100644 index 0000000000..fdbcf5c32f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf @@ -0,0 +1,338 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-access-point", ghpc_role = "scheduler" }) +} + +locals { + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + metadata = merge( + local.network_storage_metadata, + local.enable_oslogin_metadata, + local.disable_automatic_updates_metadata, + var.metadata + ) + + host_count = 1 + name_prefix = "${var.deployment_name}-ap" + + example_runner = { + type = "data" + destination = "/var/tmp/helloworld.sub" + content = <<-EOT + universe = vanilla + executable = /bin/sleep + arguments = 1000 + output = out.$(ClusterId).$(ProcId) + error = err.$(ClusterId).$(ProcId) + log = log.$(ClusterId).$(ProcId) + request_cpus = 1 + request_memory = 100MB + queue + EOT + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.access_point_runner, + [local.schedd_runner], + var.autoscaler_runner, + [local.example_runner] + ) + + ap_config = templatefile("${path.module}/templates/condor_config.tftpl", { + htcondor_role = "get_htcondor_submit", + central_manager_ips = var.central_manager_ips + spool_dir = "${var.spool_parent_dir}/spool", + mig_ids = var.mig_id, + default_mig_id = var.default_mig_id + }) + + ap_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.ap_config.output_name}" + schedd_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e htcondor_role=get_htcondor_submit", + "-e config_object=${local.ap_object}", + "-e spool_dir=${var.spool_parent_dir}/spool", + "-e htcondor_spool_disk_device=/dev/disk/by-id/google-${local.spool_disk_device_name}", + ]) + } + + access_point_ips = google_compute_address.ap.address + access_point_name = data.google_compute_instance.ap.name + + spool_disk_resource_name = "${var.deployment_name}-spool-disk" + spool_disk_device_name = "htcondor-spool-disk" + spool_disk_source = try(google_compute_disk.spool[0].name, google_compute_region_disk.spool[0].self_link) + + zones = coalescelist(var.zones, random_shuffle.zones.result) + + vm_family = split("-", var.machine_type)[0] + regional_pd_families = ["e2", "n1", "n2", "n2d"] +} + +data "google_compute_image" "htcondor" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + postcondition { + condition = self.disk_size_gb <= var.disk_size_gb + error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region + + lifecycle { + postcondition { + condition = alltrue([ + for z in var.zones : contains(self.names, z) + ]) + error_message = "Each entry in var.zones must be a zone in var.region: ${var.region}" + } + } +} + +resource "random_shuffle" "zones" { + input = data.google_compute_zones.available.names + result_count = var.enable_high_availability ? 2 : 1 +} + +data "google_compute_region_instance_group" "ap" { + self_link = module.htcondor_ap.self_link + lifecycle { + postcondition { + condition = length(self.instances) == local.host_count + error_message = "There should be ${local.host_count} access points found" + } + } +} + +data "google_compute_instance" "ap" { + self_link = data.google_compute_region_instance_group.ap.instances[0].instance +} + +resource "null_resource" "ap_config" { + triggers = { + config = local.ap_config + } +} + +resource "google_storage_bucket_object" "ap_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.ap_config.id), 0, 4)}" + content = local.ap_config + bucket = var.htcondor_bucket_name + + lifecycle { + precondition { + condition = var.default_mig_id == "" || contains(var.mig_id, var.default_mig_id) + error_message = "If set, var.default_mig_id must be an element in var.mig_id" + } + + # by construction, this precondition only fails when the user has set + # var.zones to a non-empty list of length not equal to 2 + precondition { + condition = !var.enable_high_availability || length(local.zones) == 2 + error_message = "When using HTCondor access point high availability, var.zones must be of length 2." + } + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +resource "google_compute_region_disk" "spool" { + count = var.enable_high_availability ? 1 : 0 + name = local.spool_disk_resource_name + labels = local.labels + type = var.spool_disk_type + region = var.region + size = var.spool_disk_size_gb + + replica_zones = local.zones + + lifecycle { + precondition { + condition = var.spool_disk_size_gb >= 200 + error_message = "When using HTCondor access point high availability, var.spool_disk_size_gb must be set to 200 or greater." + } + + precondition { + condition = contains(local.regional_pd_families, local.vm_family) + error_message = "When using HTCondor access point high availability, var.machine_type must be one of ${jsonencode(local.regional_pd_families)}." + } + } +} + +resource "google_compute_disk" "spool" { + count = var.enable_high_availability ? 0 : 1 + name = local.spool_disk_resource_name + labels = local.labels + type = var.spool_disk_type + zone = local.zones[0] + size = var.spool_disk_size_gb +} + +resource "google_compute_address" "ap" { + project = var.project_id + name = local.name_prefix + region = var.region + subnetwork = var.subnetwork_self_link + address_type = "INTERNAL" + purpose = "GCE_ENDPOINT" +} + +module "access_point_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.access_point_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + preemptible = false + startup_script = module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.htcondor.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + network_ip = google_compute_address.ap.id + + # spool disk + additional_disks = [ + { + source = local.spool_disk_source + device_name = local.spool_disk_device_name + } + ] +} + +module "htcondor_ap" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = local.host_count + hostname = local.name_prefix + instance_template = module.access_point_instance_template.self_link + + health_check_name = "health-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) + max_surge_fixed = 0 # must be 0 to preserve hostnames + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + + stateful_disks = [{ + device_name = local.spool_disk_device_name + delete_rule = "ON_PERMANENT_INSTANCE_DELETION" + }] + stateful_ips = var.enable_public_ips ? [{ + interface_name = "nic0" + delete_rule = "ON_PERMANENT_INSTANCE_DELETION" + is_external = true + }] : [] + + # the timeouts below are default for resource + wait_for_instances = true + mig_timeouts = { + create = "15m" + delete = "15m" + update = "15m" + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf new file mode 100644 index 0000000000..f7424c6d5d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "access_point_ips" { + description = "IP addresses of the access points provisioned by this module" + value = local.access_point_ips +} + +output "access_point_name" { + description = "Name of the access point provisioned by this module" + value = local.access_point_name +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl new file mode 100644 index 0000000000..214fbc726f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl @@ -0,0 +1,70 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:${htcondor_role} +CONDOR_HOST = ${join(",", central_manager_ips)} + +SPOOL = ${spool_dir} +SCHEDD_INTERVAL = 30 +TRUST_UID_DOMAIN = True +SUBMIT_ATTRS = RunAsOwner +RunAsOwner = True + +# When a job matches to a machine, add machine attributes to the job for +# condor_history (e.g. VM Instance ID) +use feature:JobsHaveInstanceIDs +SYSTEM_JOB_MACHINE_ATTRS = $(SYSTEM_JOB_MACHINE_ATTRS) \ + CloudVMType CloudZone CloudInterruptible +SYSTEM_JOB_MACHINE_ATTRS_HISTORY_LENGTH = 10 + +# Add Cloud attributes to SchedD ClassAd +use feature:ScheddCronOneShot(cloud, $(LIBEXEC)/common-cloud-attributes-google.py) +SCHEDD_CRON_cloud_PREFIX = Cloud + +# aid the user by automatically using RequireSpot in their Requirements, unless +# the user has explicitly used CloudInterruptible +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SPOT +JOB_TRANSFORM_SPOT @=end + REQUIREMENTS ! isUndefined(RequireSpot) && ! unresolved(Requirements, "^CloudInterruptible$") + SET Requirements ($(MY.Requirements)) && (CloudInterruptible is My.RequireSpot) +@end + +# help the user by enforcing that RequireSpot is undefined or a boolean +SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) SPOT +SUBMIT_REQUIREMENT_SPOT = isUndefined(RequireSpot) || isBoolean(RequireSpot) +SUBMIT_REQUIREMENT_SPOT_REASON = "If +RequireSpot is defined, it must be either True or False" + +%{ if length(mig_ids) > 0 ~} +MIG_IDS = "${join(" ", mig_ids)}" +MIG_ID_LIST = split($(MIG_IDS)) +%{ if default_mig_id != "" ~} +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) ID_DEFAULT +JOB_TRANSFORM_ID_DEFAULT @=end + DEFAULT RequireId "${default_mig_id}" +@end +%{ endif ~} +SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) MIGID +SUBMIT_REQUIREMENT_MIGID = !isUndefined(RequireId) && member(RequireId, $(MIG_ID_LIST)) +SUBMIT_REQUIREMENT_MIGID_REASON = strcat("Jobs must set +RequireId to one of following values surrounded by quotation marks:\n", $(MIG_IDS)) + +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) MIGID +JOB_TRANSFORM_MIGID @=end + REQUIREMENTS ! isUndefined(RequireId) && ! unresolved(Requirements, "^CloudCreatedBy$") + SET Requirements ($(MY.Requirements)) && regexp(strcat("/", My.RequireId, "$"), CloudCreatedBy) +@end +%{ endif ~} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf new file mode 100644 index 0000000000..f54a88ac2e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf @@ -0,0 +1,266 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zones" { + description = "Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region." + type = list(string) + default = [] + nullable = false + + validation { + condition = length(var.zones) <= 2 + error_message = "Set var.zones to the empty list or up to 2 zones in var.region" + } +} + +variable "distribution_policy_target_shape" { + description = "Target shape acoss zones for instance group managing high availability of access point" + type = string + default = "ANY_SINGLE_ZONE" +} + +variable "network_self_link" { + description = "The self link of the network in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "access_point_service_account_email" { + description = "Service account for access point (e-mail format)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 32 + nullable = false +} + +variable "disk_type" { + description = "Boot disk size in GB" + type = string + default = "pd-balanced" + nullable = false +} + +variable "spool_disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 32 + nullable = false +} + +variable "spool_disk_type" { + description = "Boot disk size in GB" + type = string + default = "pd-ssd" + nullable = false +} + +variable "metadata" { + description = "Metadata to add to HTCondor central managers" + type = map(string) + default = {} +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + nullable = false + validation { + condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "enable_high_availability" { + description = "Provision HTCondor access point in high availability mode" + type = bool + default = false +} + +variable "instance_image" { + description = <<-EOD + Custom VM image with HTCondor and Toolkit support installed." + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "machine_type" { + description = "Machine type to use for HTCondor central managers" + type = string + default = "n2-standard-4" +} + +variable "access_point_runner" { + description = "A list of Toolkit runners for configuring an HTCondor access point" + type = list(map(string)) + default = [] +} + +variable "autoscaler_runner" { + description = "A list of Toolkit runners for configuring autoscaling daemons" + type = list(map(string)) + default = [] +} + +variable "spool_parent_dir" { + description = "HTCondor access point configuration SPOOL will be set to subdirectory named \"spool\"" + type = string + default = "/var/lib/condor" +} + +variable "central_manager_ips" { + description = "List of IP addresses of HTCondor Central Managers" + type = list(string) +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "enable_public_ips" { + description = "Enable Public IPs on the access points" + type = bool + default = false +} + +variable "mig_id" { + description = "List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module)" + type = list(string) + default = [] + nullable = false + + validation { + condition = length(var.mig_id) > 0 + error_message = "At least 1 MIG containing execute points must be provided to this module" + } +} + +variable "default_mig_id" { + description = "Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id" + type = string + default = "" + nullable = false +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" + type = string + default = "OPPORTUNISTIC" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf new file mode 100644 index 0000000000..0d07e7abf1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.74.0" + } + + required_version = ">= 1.1" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md new file mode 100644 index 0000000000..dfab563a55 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md @@ -0,0 +1,159 @@ +## Description + +This module provisions a highly available HTCondor central manager using a [Managed +Instance Group (MIG)][mig] with auto-healing. + +[mig]: https://cloud.google.com/compute/docs/instance-groups + +## Usage + +This module provisions an HTCondor central manager with a standard +configuration. For the node to function correctly, you must supply the input +variable described below: + +- [var.central_manager_runner](#input_central_manager_runner) + - Runner must download a POOL password / signing key and create an [IDTOKEN] + with no scopes (full authorization). + +A reference implementation is included in the Toolkit module +[htcondor-pool-secrets]. You may substitute implementations so long as they +duplicate the functionality in the references. Usage is demonstrated in the +[HTCondor example][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- +[htcondor-pool-secrets]: ../htcondor-pool-secrets/README.md +[IDTOKEN]: https://htcondor.readthedocs.io/en/latest/admin-manual/security.html#introducing-idtokens + +## Behavior of Managed Instance Group (MIG) + +A regional [MIG][mig] is used to provision the central manager, although only +1 node will ever be active at a time. By default, the node will be provisioned +in any of the zones available in that region, however, it can be constrained to +run in fewer zones (or a single zone) using [var.zones](#input_zones). + +When the configuration of the Central Manager is changed, the MIG can be +configured to [replace the VM][replacement] using a "proactive" or +"opportunistic" policy. By default, the Central Manager replacement policy is +set to proactive. In practice, this means that the Central Manager will be +replaced by Terraform when changes to the instance template / HTCondor +configuration are made. The Central Manager is safe to replace automatically as +it gathers its state information from periodic messages exchanged with the rest +of the HTCondor pool. + +This mode can be configured by setting [var.update_policy](#input_update_policy) +to either "PROACTIVE" (default) or "OPPORTUNISTIC". If set to opportunistic +replacement, the Central Manager will be replaced only when: + +- intentionally by issuing an update via Cloud Console or using gcloud (below) +- the VM becomes unhealthy or is otherwise automatically replaced (e.g. regular + Google Cloud maintenance) + +For example, to manually update all instances in a MIG: + +```text +gcloud compute instance-groups managed update-instances \ + <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + +## Limiting inter-zone egress + +Because all the elements of the HTCondor pool use regional MIGs, they may be +subject to [interzone egress fees][network-pricing]. The primary traffic between +nodes of an HTCondor pool running embarrassingly parallel jobs is expected to +be limited to API traffic for job scheduling and monitoring. Please review the +[network pricing][network-pricing] documentation and determine if this cost is +a concern. If it is, use [var.zones](#input_zones) to constrain each node within +your HTCondor pool to operate within a single zone. + +[network-pricing]: https://cloud.google.com/vpc/network-pricing + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1.0 | +| [google](#requirement\_google) | >= 3.83 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [central\_manager\_instance\_template](#module\_central\_manager\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [htcondor\_cm](#module\_htcondor\_cm) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_address.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_storage_bucket_object.cm_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.cm_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_instance.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | +| [google_compute_region_instance_group.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [central\_manager\_runner](#input\_central\_manager\_runner) | A list of Toolkit runners for configuring an HTCondor central manager | `list(map(string))` | `[]` | no | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for central manager (can be supplied by htcondor-setup module) | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `20` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape for instance group managing high availability of central manager | `string` | `"ANY_SINGLE_ZONE"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor installed using the htcondor-install module."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor central manager will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Central Manager ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle). | `string` | `"PROACTIVE"` | no | +| [zones](#input\_zones) | Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [central\_manager\_ips](#output\_central\_manager\_ips) | IP addresses of the central managers provisioned by this module | +| [central\_manager\_name](#output\_central\_manager\_name) | Name of the central managers provisioned by this module | +| [list\_instances\_command](#output\_list\_instances\_command) | Command to list central managers provisioned by this module | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml new file mode 100644 index 0000000000..7408af6370 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml @@ -0,0 +1,72 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor central manager + hosts: localhost + become: true + vars: + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + handlers: + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf new file mode 100644 index 0000000000..d288a91144 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf @@ -0,0 +1,226 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-central-manager", ghpc_role = "scheduler" }) +} + +locals { + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + metadata = merge( + local.network_storage_metadata, + local.enable_oslogin_metadata, + local.disable_automatic_updates_metadata, + var.metadata + ) + + name_prefix = "${var.deployment_name}-cm" + + cm_config = templatefile("${path.module}/templates/condor_config.tftpl", {}) + + cm_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.cm_config.output_name}" + schedd_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e config_object=${local.cm_object}", + ]) + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.central_manager_runner, + [local.schedd_runner] + ) + + central_manager_ips = google_compute_address.cm.address + central_manager_name = data.google_compute_instance.cm.name + + list_instances_command = "gcloud compute instance-groups list-instances ${data.google_compute_region_instance_group.cm.name} --region ${var.region} --project ${var.project_id}" + + zones = coalescelist(var.zones, data.google_compute_zones.available.names) +} + +data "google_compute_image" "htcondor" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + postcondition { + condition = self.disk_size_gb <= var.disk_size_gb + error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +data "google_compute_region_instance_group" "cm" { + self_link = module.htcondor_cm.self_link + lifecycle { + postcondition { + condition = length(self.instances) == 1 + error_message = "There should only be 1 central manager found" + } + } +} + +data "google_compute_instance" "cm" { + self_link = data.google_compute_region_instance_group.cm.instances[0].instance +} + +resource "null_resource" "cm_config" { + triggers = { + config = local.cm_config + } +} + +resource "google_storage_bucket_object" "cm_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.cm_config.id), 0, 4)}" + content = local.cm_config + bucket = var.htcondor_bucket_name +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +resource "google_compute_address" "cm" { + project = var.project_id + name = local.name_prefix + region = var.region + subnetwork = var.subnetwork_self_link + address_type = "INTERNAL" + purpose = "GCE_ENDPOINT" +} + +module "central_manager_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.central_manager_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + preemptible = false + startup_script = module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.htcondor.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + network_ip = google_compute_address.cm.id +} + +module "htcondor_cm" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = 1 + hostname = local.name_prefix + instance_template = module.central_manager_instance_template.self_link + + health_check_name = "health-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) + max_surge_fixed = 0 # must be 0 to preserve hostnames + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + + # the timeouts below are default for resource + wait_for_instances = true + mig_timeouts = { + create = "15m" + delete = "15m" + update = "15m" + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf new file mode 100644 index 0000000000..a6272e7ca2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "list_instances_command" { + description = "Command to list central managers provisioned by this module" + value = local.list_instances_command +} + +output "central_manager_ips" { + description = "IP addresses of the central managers provisioned by this module" + value = local.central_manager_ips +} + +output "central_manager_name" { + description = "Name of the central managers provisioned by this module" + value = local.central_manager_name +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl new file mode 100644 index 0000000000..5b9676457e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl @@ -0,0 +1,31 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:get_htcondor_central_manager +CONDOR_HOST = $(IPV4_ADDRESS) + +# Central Manager configuration settings +# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-collector-configuration-file-entries +# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-negotiator-configuration-file-entries +# set classad lifetime (expiration) to ~5x the update interval for all daemons +# defaults to 900s +CLASSAD_LIFETIME = 180 +COLLECTOR_UPDATE_INTERVAL = 30 +NEGOTIATOR_UPDATE_INTERVAL = 30 +NEGOTIATOR_DEPTH_FIRST = True +NEGOTIATOR_UPDATE_AFTER_CYCLE = True diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf new file mode 100644 index 0000000000..7f85861c3f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf @@ -0,0 +1,192 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor central manager will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zones" { + description = "Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region." + type = list(string) + default = [] + nullable = false +} + +variable "distribution_policy_target_shape" { + description = "Target shape for instance group managing high availability of central manager" + type = string + default = "ANY_SINGLE_ZONE" +} + +variable "network_self_link" { + description = "The self link of the network in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "central_manager_service_account_email" { + description = "Service account e-mail for central manager (can be supplied by htcondor-setup module)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 20 + nullable = false +} + +variable "metadata" { + description = "Metadata to add to HTCondor central managers" + type = map(string) + default = {} +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + nullable = false + validation { + condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "instance_image" { + description = <<-EOD + Custom VM image with HTCondor installed using the htcondor-install module." + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "machine_type" { + description = "Machine type to use for HTCondor central managers" + type = string + default = "n2-standard-4" +} + +variable "central_manager_runner" { + description = "A list of Toolkit runners for configuring an HTCondor central manager" + type = list(map(string)) + default = [] +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Central Manager (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)." + type = string + default = "PROACTIVE" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf new file mode 100644 index 0000000000..4dee3adac7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.74.0" + } + + required_version = ">= 1.1.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md new file mode 100644 index 0000000000..7158e7bac6 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md @@ -0,0 +1,172 @@ +## Description + +This module is responsible for the following actions: + +- store an HTCondor Pool password in Google Cloud Secret Manager + - will generate a new password if one is not supplied +- create a secret in Google Cloud Secret Manager in which the HTCondor central + manager can place IDTOKENs (JWT Authorizations) for execute points to download +- create a Toolkit runner for the central manager + - download the POOL password / signing key + - create a local IDTOKEN for itself + - upload the execute point IDTOKEN secret +- create a Toolkit runner for access points + - download the POOL password / signing key + - create a local IDTOKEN for itself +- create a Toolkit runner for execute points + - Fetch the IDTOKEN secret generated by the central manager + +It is expected to be used with the [htcondor-install] and +[htcondor-execute-point] modules. + +[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[htcondor-install]: ../../scripts/htcondor-setup/README.md +[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md + +[htcrole]: https://htcondor.readthedocs.io/en/latest/getting-htcondor/admin-quick-start.html#what-get-htcondor-does-to-configure-a-role + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_setup + + - id: htcondor_startup_central_manager + source: modules/scripts/startup-script + settings: + runners: + - $(htcondor_install.install_htcondor_runner) + - $(htcondor_secrets.central_manager_runner) + - $(htcondor_setup.central_manager_runner) + +- id: htcondor_cm + source: modules/compute/vm-instance + use: + - network1 + - htcondor_startup_central_manager + settings: + name_prefix: cm0 + machine_type: c2-standard-4 + disable_public_ips: true + service_account: + email: $(htcondor_setup.central_manager_service_account) + scopes: + - cloud-platform + network_interfaces: + - network: null + subnetwork: $(network1.subnetwork_self_link) + subnetwork_project: $(vars.project_id) + network_ip: $(htcondor_setup.central_manager_internal_ip) + stack_type: null + access_config: [] + ipv6_access_config: [] + alias_ip_range: [] + nic_type: VIRTIO_NET + queue_count: null + outputs: + - internal_ip +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 4.84 | +| [random](#requirement\_random) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | +| [random](#provider\_random) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_secret_manager_secret.execute_point_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_iam_member.access_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.central_manager_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.central_manager_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.execute_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_version.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [random_password.pool](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [pool\_password](#input\_pool\_password) | HTCondor Pool Password | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [trust\_domain](#input\_trust\_domain) | Trust domain for HTCondor pool (if not supplied, will be set based on project\_id) | `string` | `""` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_runner](#output\_access\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor access point | +| [central\_manager\_runner](#output\_central\_manager\_runner) | Toolkit Runner to download pool secrets to an HTCondor central manager | +| [execute\_point\_runner](#output\_execute\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor execute point | +| [pool\_password\_secret\_id](#output\_pool\_password\_secret\_id) | Google Cloud Secret Manager ID containing HTCondor Pool Password | +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | PowerShell script to download pool secrets to an HTCondor execute point | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml new file mode 100644 index 0000000000..538c809c2a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml @@ -0,0 +1,102 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Secrets + hosts: localhost + become: true + vars: + condor_config_root: /etc/condor + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - password_id is defined + - trust_domain is defined + - name: Set Pool Trust Domain + ansible.builtin.copy: + dest: "{{ condor_config_root }}/config.d/51-ghpc-trust-domain" + mode: 0644 + content: | + # these lines must appear AFTER any "use role:" settings + UID_DOMAIN = {{ trust_domain }} + TRUST_DOMAIN = {{ trust_domain }} + - name: Get HTCondor Pool password (token signing key) + when: htcondor_role != 'get_htcondor_execute' + ansible.builtin.shell: | + set -e -o pipefail +o history + POOL_PASSWORD=$(gcloud secrets versions access latest --secret={{ password_id }}) + echo -n "$POOL_PASSWORD" | sh -c "condor_store_cred add -c -i -" + args: + creates: "{{ condor_config_root }}/passwords.d/POOL" + executable: /bin/bash + - name: Configure HTCondor Central Manager + when: htcondor_role == 'get_htcondor_central_manager' + block: + - name: Create IDTOKEN for Central Manager + ansible.builtin.shell: | + umask 0077 + condor_token_create -identity condor@{{ trust_domain }} \ + -token condor@{{ trust_domain }} + args: + creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" + - name: Create IDTOKEN secret for Execute Points + when: xp_idtoken_secret_id | length > 0 + changed_when: true + ansible.builtin.shell: | + umask 0077 + TMPFILE=$(mktemp) + condor_token_create -authz READ -authz ADVERTISE_MASTER \ + -authz ADVERTISE_STARTD -identity condor@{{ trust_domain }} > "$TMPFILE" + gcloud secrets versions add --data-file "$TMPFILE" {{ xp_idtoken_secret_id }} + rm -f "$TMPFILE" + - name: Configure HTCondor SchedD + when: htcondor_role == 'get_htcondor_submit' + block: + - name: Create IDTOKEN to advertise access point + ansible.builtin.shell: | + umask 0077 + # DAEMON authorization can likely be removed in future when scopes + # needed to trigger a negotiation cycle are changed. Suggest review + # https://opensciencegrid.atlassian.net/jira/software/c/projects/HTCONDOR/issues/?filter=allissues + condor_token_create -authz READ -authz ADVERTISE_MASTER \ + -authz ADVERTISE_SCHEDD -authz DAEMON -identity condor@{{ trust_domain }} \ + -token condor@{{ trust_domain }} + args: + creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" + - name: Configure HTCondor StartD + when: htcondor_role == 'get_htcondor_execute' + block: + - name: Create SystemD override directory for HTCondor Execute Point + ansible.builtin.file: + path: /etc/systemd/system/condor.service.d + state: directory + owner: root + group: root + mode: 0755 + - name: Fetch IDTOKEN to advertise execute point + ansible.builtin.copy: + dest: "/etc/systemd/system/condor.service.d/htcondor-token-fetcher.conf" + mode: 0644 + content: | + [Service] + ExecStartPre=gcloud secrets versions access latest --secret {{ xp_idtoken_secret_id }} \ + --out-file {{ condor_config_root }}/tokens.d/condor@{{ trust_domain }} + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf new file mode 100644 index 0000000000..1a7c761760 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf @@ -0,0 +1,168 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-pool-secrets", ghpc_role = "scheduler" }) +} + +locals { + pool_password = coalesce(var.pool_password, random_password.pool.result) + auto = length(var.user_managed_replication) == 0 ? "" : "-user" + access_point_service_account_iam_email = "serviceAccount:${var.access_point_service_account_email}" + central_manager_service_account_iam_email = "serviceAccount:${var.central_manager_service_account_email}" + execute_point_service_account_iam_email = "serviceAccount:${var.execute_point_service_account_email}" + + trust_domain = coalesce(var.trust_domain, "c.${var.project_id}.internal") + + runner_cm = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_central_manager", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + + runner_access = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_submit", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + + runner_execute = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_execute", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + windows_startup_ps1 = templatefile( + "${path.module}/templates/fetch-idtoken.ps1.tftpl", + { + trust_domain = local.trust_domain, + xp_idtoken_secret_id = google_secret_manager_secret.execute_point_idtoken.secret_id, + } + ) +} + +resource "random_password" "pool" { + length = 24 + special = true + override_special = "_-#=." +} + +resource "google_secret_manager_secret" "pool_password" { + secret_id = "${var.deployment_name}-pool-password${local.auto}" + + labels = local.labels + + replication { + dynamic "auto" { + for_each = length(var.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_version" "pool_password" { + secret = google_secret_manager_secret.pool_password.id + secret_data = local.pool_password +} + +# this secret will be populated by the Central Manager +resource "google_secret_manager_secret" "execute_point_idtoken" { + secret_id = "${var.deployment_name}-execute-point-idtoken${local.auto}" + + labels = local.labels + + replication { + dynamic "auto" { + for_each = length(var.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_iam_member" "central_manager_password" { + secret_id = google_secret_manager_secret.pool_password.id + role = "roles/secretmanager.secretAccessor" + member = local.central_manager_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "central_manager_idtoken" { + secret_id = google_secret_manager_secret.execute_point_idtoken.id + role = "roles/secretmanager.secretVersionManager" + member = local.central_manager_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "access_point" { + secret_id = google_secret_manager_secret.pool_password.id + role = "roles/secretmanager.secretAccessor" + member = local.access_point_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "execute_point" { + secret_id = google_secret_manager_secret.execute_point_idtoken.id + role = "roles/secretmanager.secretAccessor" + member = local.execute_point_service_account_iam_email +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml new file mode 100644 index 0000000000..4b0bdbd616 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com + - secretmanager.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf new file mode 100644 index 0000000000..81c4986b16 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "pool_password_secret_id" { + description = "Google Cloud Secret Manager ID containing HTCondor Pool Password" + value = google_secret_manager_secret.pool_password.secret_id + sensitive = true +} + +output "central_manager_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor central manager" + value = local.runner_cm + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "access_point_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor access point" + value = local.runner_access + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "execute_point_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor execute point" + value = local.runner_execute + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "windows_startup_ps1" { + description = "PowerShell script to download pool secrets to an HTCondor execute point" + value = local.windows_startup_ps1 +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl new file mode 100644 index 0000000000..04c96291ee --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl @@ -0,0 +1,26 @@ +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +$config_dir = 'C:\Condor\config' +if(!(test-path -PathType container -Path $config_dir)) +{ + New-Item -ItemType Directory -Path $config_dir +} +$config_file = "$config_dir\51-ghpc-trust-domain" + +$config_string = @' +# these lines must appear AFTER any "use role:" settings +UID_DOMAIN = ${trust_domain} +TRUST_DOMAIN = ${trust_domain} +'@ + +Set-Content -Path "$config_file" -Value "$config_string" + +# obtain IDTOKEN for authentication by StartD to Central Manager +gcloud secrets versions access latest --secret ${xp_idtoken_secret_id} ` + --out-file C:\condor\tokens.d\condor@${trust_domain} + +if ($LASTEXITCODE -ne 0) +{ + throw "Could not download HTCondor IDTOKEN; exiting startup script" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf new file mode 100644 index 0000000000..22ef3644e8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "access_point_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "central_manager_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "execute_point_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "pool_password" { + description = "HTCondor Pool Password" + type = string + sensitive = true + default = null +} + +variable "trust_domain" { + description = "Trust domain for HTCondor pool (if not supplied, will be set based on project_id)" + type = string + default = "" +} + +variable "user_managed_replication" { + type = list(object({ + location = string + kms_key_name = optional(string) + })) + description = "Replication parameters that will be used for defined secrets" + default = [] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf new file mode 100644 index 0000000000..d8a1d96f5f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md new file mode 100644 index 0000000000..5a403c0a38 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md @@ -0,0 +1,128 @@ +## Description + +This module creates the service accounts for use by the primary elements of an +[HTCondor pool][pool]: + +- Central Managers +- Access Points +- Execute Points + +Each service account is assigned common roles necessary for the VM to function +properly. In particular, nearly every VM requires the ability to read from Cloud +Storage buckets and write Cloud Logging entries. These roles are configurable +as described below. + +[pool]: https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-admin-manual.html#the-different-roles-a-machine-can-play + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_service_accounts + source: community/modules/scheduler/htcondor-service-accounts + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + - htcondor_service_accounts + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_service_accounts + +- id: htcondor_cm + source: community/modules/scheduler/htcondor-central-manager + use: + - network1 + - htcondor_secrets + - htcondor_service_accounts + - htcondor_setup + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - central_manager_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [access\_point\_service\_account](#module\_access\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | +| [central\_manager\_service\_account](#module\_central\_manager\_service\_account) | ../../../../community/modules/project/service-account | n/a | +| [execute\_point\_service\_account](#module\_execute\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_roles](#input\_access\_point\_roles) | Project-wide roles for HTCondor Access Point service account | `list(string)` | 
[
  "compute.instanceAdmin.v1",
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [central\_manager\_roles](#input\_central\_manager\_roles) | Project-wide roles for HTCondor Central Manager service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_roles](#input\_execute\_point\_roles) | Project-wide roles for HTCondor Execute Point service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_service\_account\_email](#output\_access\_point\_service\_account\_email) | HTCondor Access Point Service Account (e-mail format) | +| [central\_manager\_service\_account\_email](#output\_central\_manager\_service\_account\_email) | HTCondor Central Manager Service Account (e-mail format) | +| [execute\_point\_service\_account\_email](#output\_execute\_point\_service\_account\_email) | HTCondor Execute Point Service Account (e-mail format) | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf new file mode 100644 index 0000000000..9d97b18642 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# NB: the community/modules/project/service-account module will not output the +# service account e-mail address until all IAM bindings have been created; if +# underlying implementation changes, this module should declare explicit +# depends_on the IAM bindings to prevent race conditions for services that +# require them + +module "access_point_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Access Point" + deployment_name = var.deployment_name + name = "access" + project_roles = var.access_point_roles +} + +module "execute_point_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Execute Point" + deployment_name = var.deployment_name + name = "execute" + project_roles = var.execute_point_roles +} + +module "central_manager_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Central Manager" + deployment_name = var.deployment_name + name = "cm" + project_roles = var.central_manager_roles +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml new file mode 100644 index 0000000000..c4dcdffdf4 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf new file mode 100644 index 0000000000..28f3a79457 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "access_point_service_account_email" { + description = "HTCondor Access Point Service Account (e-mail format)" + value = module.access_point_service_account.service_account_email +} + +output "central_manager_service_account_email" { + description = "HTCondor Central Manager Service Account (e-mail format)" + value = module.central_manager_service_account.service_account_email +} + +output "execute_point_service_account_email" { + description = "HTCondor Execute Point Service Account (e-mail format)" + value = module.execute_point_service_account.service_account_email +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf new file mode 100644 index 0000000000..ee186e0971 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "access_point_roles" { + description = "Project-wide roles for HTCondor Access Point service account" + type = list(string) + default = [ + "compute.instanceAdmin.v1", + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} + +variable "central_manager_roles" { + description = "Project-wide roles for HTCondor Central Manager service account" + type = list(string) + default = [ + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} + +variable "execute_point_roles" { + description = "Project-wide roles for HTCondor Execute Point service account" + type = list(string) + default = [ + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/README.md new file mode 100644 index 0000000000..1722702ceb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/README.md @@ -0,0 +1,118 @@ +## Description + +This module creates a bucket in which to store HTCondor configurations and +a firewall rule that allows Managed Instance Group health checks to probe the +health of HTCondor VMs. + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_service_accounts + source: community/modules/scheduler/htcondor-service-accounts + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + - htcondor_service_accounts + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_service_accounts + +- id: htcondor_cm + source: community/modules/scheduler/htcondor-central-manager + use: + - network1 + - htcondor_secrets + - htcondor_service_accounts + - htcondor_setup + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - central_manager_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [health\_check\_firewall\_rule](#module\_health\_check\_firewall\_rule) | ../../../../modules/network/firewall-rules | n/a | +| [htcondor\_bucket](#module\_htcondor\_bucket) | ../../../../modules/file-system/cloud-storage-bucket | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account e-mail for HTCondor Access Point | `string` | n/a | yes | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for HTCondor Central Manager | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account e-mail for HTCondor Execute Points | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which Central Managers will be placed. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [htcondor\_bucket\_name](#output\_htcondor\_bucket\_name) | Name of the HTCondor configuration bucket | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf new file mode 100644 index 0000000000..e048362663 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-setup", ghpc_role = "scheduler" }) +} + +locals { + service_account_iam_email = [ + "serviceAccount:${var.access_point_service_account_email}", + "serviceAccount:${var.central_manager_service_account_email}", + "serviceAccount:${var.execute_point_service_account_email}", + ] + service_account_email = [ + var.access_point_service_account_email, + var.central_manager_service_account_email, + var.execute_point_service_account_email, + ] +} + +module "health_check_firewall_rule" { + source = "../../../../modules/network/firewall-rules" + + subnetwork_self_link = var.subnetwork_self_link + + ingress_rules = [{ + name = "allow-health-check-${var.deployment_name}" + description = "Allow Managed Instance Group Health Checks for HTCondor VMs" + direction = "INGRESS" + source_ranges = [ + "130.211.0.0/22", + "35.191.0.0/16", + ] + target_service_accounts = local.service_account_email + allow = [{ + protocol = "tcp" + ports = ["9618"] + }] + }] +} + +module "htcondor_bucket" { + source = "../../../../modules/file-system/cloud-storage-bucket" + + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + name_prefix = "${var.deployment_name}-htcondor-config" + random_suffix = true + labels = local.labels + viewers = local.service_account_iam_email + + use_deployment_name_in_bucket_name = false +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml new file mode 100644 index 0000000000..7b4918b962 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - iam.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf new file mode 100644 index 0000000000..a44223faee --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "htcondor_bucket_name" { + description = "Name of the HTCondor configuration bucket" + value = module.htcondor_bucket.gcs_bucket_name + + # ensure that all IAM bindings to the bucket and firewall rules are active + # before this modules output is allowed to propagate + depends_on = [ + module.htcondor_bucket, + module.health_check_firewall_rule + ] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf new file mode 100644 index 0000000000..147a2ca88d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which Central Managers will be placed." + type = string +} + +variable "access_point_service_account_email" { + description = "Service account e-mail for HTCondor Access Point" + type = string +} + +variable "central_manager_service_account_email" { + description = "Service account e-mail for HTCondor Central Manager" + type = string +} + +variable "execute_point_service_account_email" { + description = "Service account e-mail for HTCondor Execute Points" + type = string +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md new file mode 100644 index 0000000000..43254cbfa8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md @@ -0,0 +1,405 @@ +## Description + +This module creates a slurm controller node via the internal +[slurm\_instance\_template] module. + +More information about Slurm On GCP can be found at the +[project's GitHub page][slurm-gcp] and in the +[Slurm on Google Cloud User Guide][slurm-ug]. + +The [user guide][slurm-ug] provides detailed instructions on customizing and +enhancing the Slurm on GCP cluster as well as recommendations on configuring the +controller for optimal performance at different scales. + +[slurm\_instance\_template]: /community/modules/internal/slurm-gcp/instance_template/README.md +[slurm-ug]: https://goo.gle/slurm-gcp-user-guide. +[enable\_cleanup\_compute]: #input\_enable\_cleanup\_compute +[enable\_cleanup\_subscriptions]: #input\_enable\_cleanup\_subscriptions +[enable\_reconfigure]: #input\_enable\_reconfigure + +### Example + +```yaml +- id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: + - network + - homefs + - compute_partition + settings: + machine_type: c2-standard-8 +``` + +This creates a controller node with the following attributes: + +* connected to the primary subnetwork of `network` +* the filesystem with the ID `homefs` (defined elsewhere in the blueprint) + mounted +* One partition with the ID `compute_partition` (defined elsewhere in the + blueprint) +* machine type upgraded from the default `c2-standard-4` to `c2-standard-8` + +### Live Cluster Reconfiguration + +The `schedmd-slurm-gcp-v6-controller` module supports the reconfiguration of +partitions and slurm configuration in a running, active cluster. + +To reconfigure a running cluster: + +1. Edit the blueprint with the desired configuration changes +2. Call `gcluster create -w` to overwrite the deployment directory +3. Follow instructions in terminal to deploy + +The following are examples of updates that can be made to a running cluster: + +* Add or remove a partition to the cluster +* Resize an existing partition +* Attach new network storage to an existing partition + +> **NOTE**: Changing the VM `machine_type` of a partition may not work. +> It is better to create a new partition and delete the old one. + +## Custom Images + +For more information on creating valid custom images for the controller VM +instance or for custom instance templates, see our [vm-images.md] documentation +page. + +[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images + +## GPU Support + +More information on GPU support in Slurm on GCP and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) + +## Reservation for Scheduled Maintenance + +A [maintenance event](https://cloud.google.com/compute/docs/instances/host-maintenance-overview#maintenanceevents) is when a compute engine stops a VM to perform a hardware or +software update which is determined by the host maintenance policy. This can +also affect the running jobs if the maintenance kicks in. Now, Customers can +protect jobs from getting terminated due to maintenance using the cluster +toolkit. You can enable creation of reservation for scheduled maintenance for +your compute nodeset and Slurm will reserve your node for maintenance during the +maintenance window. If you try to schedule any jobs which overlap with the +maintenance reservation, Slurm would not schedule any job. + +You can specify in your blueprint like + +```yaml + - id: compute_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network] + settings: + enable_maintenance_reservation: true +``` + +To enable creation of reservation for maintenance. + +While running job on slurm cluster, you can specify total run time of the job +using [-t flag](https://slurm.schedmd.com/srun.html#OPT_time).This would only +run the job outside of the maintenance window. + +```shell +srun -n1 -pcompute -t 10:00 +``` + +Currently upcoming maintenance notification is supported in ALPHA version of +compute API. You can update the API version from your blueprint, + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + settings: + endpoint_versions: + compute: "alpha" +``` + +## Opportunistic GCP maintenance in Slurm + +Customers can also enable running GCP maintenance as Slurm job opportunistically +to perform early maintenance. If a node is detected for maintenance, Slurm will +create a job to perform maintenance and put it in the job queue. + +If [backfill](https://slurm.schedmd.com/sched_config.html#backfill) scheduler is +used, Slurm will backfill maintenance job if it can find any empty time window. + +Customer can also choose builtin scheduler type. In this case, Slurm would run +maintenance job in strictly priority order. If the maintenance job doesn't kick +in, then forced maintenance will take place at scheduled window. + +Customer can enable this feature at nodeset level by, + +```yaml + - id: debug_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network] + settings: + enable_opportunistic_maintenance: true +``` + +## Placement Max Distance + +When using +[enable_placement](../../../../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md#input_enable_placement) +with Slurm, Google Compute Engine will attempt to place VMs as physically close +together as possible. Capacity constraints at the time of VM creation may still +force VMs to be spread across multiple racks. Google provides the `max-distance` +flag which can used to control the maximum spreading allowed. Read more about +`max-distance` in the +[official docs](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies +). + +You can use the `placement_max_distance` setting on the nodeset module to control the `max-distance` behavior. See the following example: + +```yaml + - id: nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [ network ] + settings: + machine_type: c2-standard-4 + node_count_dynamic_max: 30 + enable_placement: true + placement_max_distance: 1 + +> [!NOTE] +> `schedmd-slurm-gcp-v6-nodeset.settings.enable_placement: true` must also be +> set for placement_max_distance to take effect. + +In the above case using a value of 1 will restrict VM to be placed on the same +rack. You can confirm that the `max-distance` was applied by calling the +following command while jobs are running: + +```shell +gcloud beta compute resource-policies list \ + --format='yaml(name,groupPlacementPolicy.maxDistance)' +``` + +> [!WARNING] +> If a zone lacks capacity, using a lower `max-distance` value (such as 1) is +> more likely to cause VMs creation to fail. + +## TreeWidth and Node Communication + +Slurm uses a fan out mechanism to communicate large groups of nodes. The shape +of this fan out tree is determined by the +[TreeWidth](https://slurm.schedmd.com/slurm.conf.html#OPT_TreeWidth) +configuration variable. + +In the cloud, this fan out mechanism can become unstable when nodes restart with +new IP addresses. You can enforce that all nodes communicate directly with the +controller by setting TreeWidth to a value >= largest partition. + +If the largest partition was 200 nodes, configure the blueprint as follows: + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + ... + settings: + cloud_parameters: + tree_width: 200 +``` + +The default has been set to 128. Values above this have not been fully tested +and may cause congestion on the controller. A more scalable solution is under +way. + +## ResumeRate and Node Resumption + +The `ResumeRate` parameter in `slurm.conf` controls the maximum number of nodes +that Slurm attempts to resume (power up) per minute. This is particularly +important in cloud environments where auto-scaling can lead to a large number of +nodes starting concurrently. + +When many nodes start simultaneously, they can place a heavy load on shared +resources, especially shared filesystems, as they all try to mount filesystems +and access configuration files at the same time. By limiting the `ResumeRate`, +you can stagger the node startup process, reducing the peak load on these shared +resources and improving overall cluster stability during scaling events. + +For example, to limit the node resumption rate to 100 nodes per minute, +configure the blueprint as follows: + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + ... + settings: + cloud_parameters: + resume_rate: 100 +``` + +Adjust this value based on the capabilities of your shared filesystem and the +expected scaling behavior of your cluster. + +## Support +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 6.41 | +| [google-beta](#requirement\_google-beta) | >= 6.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.41 | +| [google-beta](#provider\_google-beta) | >= 6.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [bucket](#module\_bucket) | terraform-google-modules/cloud-storage/google | >= 6.1 | +| [daos\_network\_storage\_scripts](#module\_daos\_network\_storage\_scripts) | ../../../../modules/scripts/startup-script | n/a | +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [login](#module\_login) | ../../internal/slurm-gcp/login | n/a | +| [nodeset\_cleanup](#module\_nodeset\_cleanup) | ./modules/cleanup_compute | n/a | +| [nodeset\_cleanup\_tpu](#module\_nodeset\_cleanup\_tpu) | ./modules/cleanup_tpu | n/a | +| [slurm\_controller\_template](#module\_slurm\_controller\_template) | ../../internal/slurm-gcp/instance_template | n/a | +| [slurm\_files](#module\_slurm\_files) | ./modules/slurm_files | n/a | +| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | +| [slurm\_nodeset\_tpu](#module\_slurm\_nodeset\_tpu) | ../../internal/slurm-gcp/nodeset_tpu | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance_from_template.controller](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_from_template) | resource | +| [google_compute_disk.controller_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_secret_manager_secret.cloudsql](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_iam_member.cloudsql_secret_accessor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_version.cloudsql_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [google_storage_bucket_iam_member.legacy_readers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | +| [google_storage_bucket_iam_member.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | +| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_project.controller_project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = string
    device_name                = string
    disk_type                  = string
    disk_size_gb               = number
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. If not specified, then one will be chosen based on slurm\_cluster\_name. | `string` | `null` | no | +| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket.
Ignored when 'create\_bucket' is true. | `string` | `null` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | +| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Defaults inherited from [Slurm GCP repo](https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/terraform/slurm_cluster/modules/slurm_files/README_TF.md#input_cloud_parameters) | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | +| [cloudsql](#input\_cloudsql) | Use this database instead of the one on the controller.
server\_ip : Address of the database server.
user : The user to access the database as.
password : The password, given the user, to access the given database. (sensitive)
db\_name : The database to access.
user\_managed\_replication : The list of location and (optional) kms\_key\_name for secret | 
object({
    server_ip = string
    user      = string
    password  = string # sensitive
    db_name   = string
    user_managed_replication = optional(list(object({
      location     = string
      kms_key_name = optional(string)
    })), [])
  })
| `null` | no | +| [compute\_startup\_script](#input\_compute\_startup\_script) | DEPRECATED: `compute_startup_script` has been deprecated.
Use `startup_script` of nodeset module instead. | `any` | `null` | no | +| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script in compute nodes. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | +| [controller\_project\_id](#input\_controller\_project\_id) | Optionally. Provision controller and config bucket in the different project | `string` | `null` | no | +| [controller\_startup\_script](#input\_controller\_startup\_script) | Startup script used by the controller VM. | `string` | `"# no-op"` | no | +| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    type = string
    size = number
  })
| 
{
  "size": 50,
  "type": "pd-ssd"
}
| no | +| [create\_bucket](#input\_create\_bucket) | Create GCS bucket instead of using an existing one. | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment. | `string` | n/a | yes | +| [disable\_controller\_public\_ips](#input\_disable\_controller\_public\_ips) | DEPRECATED: Use `enable_controller_public_ips` instead. | `bool` | `null` | no | +| [disable\_default\_mounts](#input\_disable\_default\_mounts) | DEPRECATED: Use `enable_default_mounts` instead. | `bool` | `null` | no | +| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | +| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | `true` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_controller\_public\_ips](#input\_enable\_controller\_public\_ips) | If set to true. The controller will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. | `bool` | `false` | no | +| [enable\_default\_mounts](#input\_enable\_default\_mounts) | Enable default global network storage from the controller
- /home
- /opt/apps | `bool` | `true` | no | +| [enable\_devel](#input\_enable\_devel) | DEPRECATED: `enable_devel` is always on. | `bool` | `null` | no | +| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/tools/prologs-epilogs/README.md | `bool` | `null` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | +| [enable\_slurm\_gcp\_plugins](#input\_enable\_slurm\_gcp\_plugins) | DEPRECATED: Slurm GCP plugins have been deprecated.
Instead of 'max\_hops' plugin please use the 'placement\_max\_distance' nodeset property.
Instead of 'enable\_vpmu' plugin please use 'advanced\_machine\_features.performance\_monitoring\_unit' nodeset property. | `any` | `null` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": "beta"
}
| no | +| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for controller. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | +| [login\_network\_storage](#input\_login\_network\_storage) | An array of network attached storage mounts to be configured on all login nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [login\_nodes](#input\_login\_nodes) | List of slurm login instance definitions. | 
list(object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  }))
| `[]` | no | +| [login\_startup\_script](#input\_login\_startup\_script) | Startup script used by the login VMs. | `string` | `"# no-op"` | no | +| [login\_startup\_scripts\_timeout](#input\_login\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in login\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = optional(map(string))
    mount_runner          = optional(map(string))
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | Define nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | +| [partitions](#input\_partitions) | Cluster partitions as a list. See module slurm\_partition. | 
list(object({
    partition_name        = string
    partition_conf        = optional(map(string), {})
    partition_nodeset     = optional(list(string), [])
    partition_nodeset_dyn = optional(list(string), [])
    partition_nodeset_tpu = optional(list(string), [])
    enable_job_exclusive  = optional(bool, false)
  }))
| `[]` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [region](#input\_region) | The default region to place resources in. | `string` | n/a | yes | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the controller instance. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the controller instance. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming and slurm accounting.
If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). | `string` | `null` | no | +| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | +| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | +| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Post deployment instructions. | +| [slurm\_bucket](#output\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | +| [slurm\_bucket\_dir](#output\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | +| [slurm\_bucket\_name](#output\_slurm\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | +| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | Bucket path used by cluster. | +| [slurm\_cluster\_name](#output\_slurm\_cluster\_name) | Slurm cluster name. | +| [slurm\_controller\_instance](#output\_slurm\_controller\_instance) | Compute instance of controller node | +| [slurm\_login\_instances](#output\_slurm\_login\_instances) | Compute instances of login nodes | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf new file mode 100644 index 0000000000..4a887b99cf --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf @@ -0,0 +1,213 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.disk_resource_manager_tags + } + ] + + state_disk = var.controller_state_disk != null ? [{ + source = google_compute_disk.controller_disk[0].name + device_name = google_compute_disk.controller_disk[0].name + disk_labels = null + auto_delete = false + boot = false + }] : [] + + synth_def_sa_email = "${data.google_project.controller_project.number}-compute@developer.gserviceaccount.com" + + service_account = { + email = coalesce(var.service_account_email, local.synth_def_sa_email) + scopes = var.service_account_scopes + } + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata, + local.universe_domain + ) + + controller_project_id = coalesce(var.controller_project_id, var.project_id) +} + +data "google_project" "controller_project" { + project_id = local.controller_project_id +} + +resource "google_compute_disk" "controller_disk" { + count = var.controller_state_disk != null ? 1 : 0 + + project = local.controller_project_id + name = "${local.slurm_cluster_name}-controller-save" + type = var.controller_state_disk.type + size = var.controller_state_disk.size + zone = var.zone +} + +# INSTANCE TEMPLATE +module "slurm_controller_template" { + source = "../../internal/slurm-gcp/instance_template" + + project_id = local.controller_project_id + region = var.region + slurm_instance_role = "controller" + slurm_cluster_name = local.slurm_cluster_name + labels = local.labels + + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(var.disk_labels, local.labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = concat(local.additional_disks, local.state_disk) + + bandwidth_tier = var.bandwidth_tier + slurm_bucket_path = module.slurm_files.slurm_bucket_path + can_ip_forward = var.can_ip_forward + advanced_machine_features = var.advanced_machine_features + resource_manager_tags = var.resource_manager_tags + + enable_confidential_vm = var.enable_confidential_vm + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + gpu = one(module.gpu.guest_accelerator) + + machine_type = var.machine_type + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + service_account = local.service_account + + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + subnetwork = var.subnetwork_self_link + + tags = concat([local.slurm_cluster_name], var.tags) + # termination_action = TODO: add support for termination_action (?) +} + +# INSTANCE +resource "google_compute_instance_from_template" "controller" { + provider = google-beta + + name = "${local.slurm_cluster_name}-controller" + project = local.controller_project_id + zone = var.zone + source_instance_template = module.slurm_controller_template.self_link + # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 + # we have to explicitly override instance labels instead of inheriting them from template. + labels = module.slurm_controller_template.labels + + allow_stopping_for_update = true + + # Can't rely on template to specify nics due to usage of static_ip + network_interface { + dynamic "access_config" { + for_each = var.enable_controller_public_ips ? ["unit"] : [] + content { + nat_ip = null + network_tier = null + } + } + network_ip = length(var.static_ips) == 0 ? "" : var.static_ips[0] + subnetwork = var.subnetwork_self_link + } + + dynamic "network_interface" { + for_each = var.controller_network_attachment != null ? [1] : [] + content { + network_attachment = var.controller_network_attachment + } + } +} + +moved { + from = module.slurm_controller_instance.google_compute_instance_from_template.slurm_instance[0] + to = google_compute_instance_from_template.controller +} + +# SECRETS: CLOUDSQL +resource "google_secret_manager_secret" "cloudsql" { + count = var.cloudsql != null ? 1 : 0 + + secret_id = "${local.slurm_cluster_name}-slurm-secret-cloudsql" + project = var.project_id + + replication { + dynamic "auto" { + for_each = length(var.cloudsql.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.cloudsql.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = nonsensitive(var.cloudsql.user_managed_replication) + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } + + labels = { + slurm_cluster_name = local.slurm_cluster_name + } +} + +resource "google_secret_manager_secret_version" "cloudsql_version" { + count = var.cloudsql != null ? 1 : 0 + + secret = google_secret_manager_secret.cloudsql[0].id + secret_data = jsonencode(var.cloudsql) +} + +resource "google_secret_manager_secret_iam_member" "cloudsql_secret_accessor" { + count = var.cloudsql != null ? 1 : 0 + + secret_id = google_secret_manager_secret.cloudsql[0].id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${local.service_account.email}" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl new file mode 100644 index 0000000000..219bdc5227 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl @@ -0,0 +1,65 @@ +# slurm.conf +# https://slurm.schedmd.com/high_throughput.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxArraySize=10001 +MaxJobCount=500000 +MaxNodeCount=65536 +MinJobAge=60 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +SlurmctldDebug=error +SlurmdDebug=error + +# +# +# TIMERS +MessageTimeout=60 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=120 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl new file mode 100644 index 0000000000..93ac47e341 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl @@ -0,0 +1,34 @@ +# slurmdbd.conf +# https://slurm.schedmd.com/slurmdbd.conf.html + +DebugLevel=info +PidFile=/var/run/slurm/slurmdbd.pid + +# https://slurm.schedmd.com/slurmdbd.conf.html#OPT_CommitDelay +CommitDelay=1 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +AuthType=auth/{auth_key} +AuthAltTypes=auth/jwt +AuthAltParameters=jwt_key={state_save}/jwt_hs256.key + +DbdHost={control_host} + +LogFile={slurmlog}/slurmdbd.log + +SlurmUser=slurm + +StorageLoc={db_name} + +StorageType=accounting_storage/mysql +StorageHost={db_host} +StoragePort={db_port} +StorageUser={db_user} +StoragePass={db_pass} + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl new file mode 100644 index 0000000000..d3f2615a68 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl @@ -0,0 +1,71 @@ +# slurm.conf +# https://slurm.schedmd.com/slurm.conf.html +# https://slurm.schedmd.com/configurator.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxNodeCount=64000 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +AccountingStoreFlags=job_comment +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/cgroup +SlurmctldDebug=info +SlurmdDebug=info +DebugFlags=Power + +# +# +# TIMERS +MessageTimeout=600 +BatchStartTimeout=600 +PrologEpilogTimeout=600 +PrologFlags=Contain + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=600 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf new file mode 100644 index 0000000000..21e915a125 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf @@ -0,0 +1,50 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +locals { + # TODO: deprecate `var.login_[ startup_script, startup_scripts_timeout, network_storage]` + # in favour of vars defined in user-facing login module + ghpc_startup_login = [{ + filename = "ghpc_startup.sh" + content = var.login_startup_script + }] + + login_startup_scripts = concat(local.common_scripts, local.ghpc_startup_login) +} + +module "login" { + source = "../../internal/slurm-gcp/login" + for_each = { for x in var.login_nodes : x.group_name => x } + + project_id = var.project_id + + slurm_cluster_name = local.slurm_cluster_name + slurm_bucket_path = module.slurm_files.slurm_bucket_path + slurm_bucket_name = module.slurm_files.bucket_name + slurm_bucket_dir = module.slurm_files.bucket_dir + + login_nodes = each.value + + startup_scripts = local.login_startup_scripts + startup_scripts_timeout = var.login_startup_scripts_timeout + + network_storage = var.login_network_storage + + universe_domain = var.universe_domain + + # trigger replacement of login nodes when the controller instance is replaced + # Needed for re-mounting volumes hosted on controller + replace_trigger = google_compute_instance_from_template.controller.self_link +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf new file mode 100644 index 0000000000..7622bdffef --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-controller", ghpc_role = "scheduler" }) +} + +locals { + # Since deployment name may be used to create a cluster name, we remove any invalid character from the beginning + # Also, slurm imposed a lot of restrictions to this name, so we format it to an acceptable string + tmp_cluster_name = substr(replace(lower(var.deployment_name), "/^[^a-z]*|[^a-z0-9]/", ""), 0, 10) + slurm_cluster_name = coalesce(var.slurm_cluster_name, local.tmp_cluster_name) + + universe_domain = { "universe_domain" = var.universe_domain } +} + +# See +# * slurm_files.tf +# * controller.tf +# * partition.tf +# * login.tf diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml new file mode 100644 index 0000000000..7b4918b962 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - iam.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md new file mode 100644 index 0000000000..002bf14145 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md @@ -0,0 +1,42 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name         = string
    subnetwork_self_link = string
    additional_networks = list(object({
      subnetwork = string
    }))
  })
| n/a | yes | +| [nodeset\_template](#input\_nodeset\_template) | Self link of the nodeset template | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf new file mode 100644 index 0000000000..bd8773cf84 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf @@ -0,0 +1,46 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + cleanup_dependencies_agg = flatten([ + var.nodeset.subnetwork_self_link, + var.nodeset.additional_networks[*].subnetwork, + var.nodeset_template]) +} + +# Can not use variadic list in `depends_on`, wrap it into a collection of `null_resource` +resource "null_resource" "dependencies" { + count = length(local.cleanup_dependencies_agg) +} + +resource "null_resource" "script" { + count = var.enable_cleanup_compute ? 1 : 0 + + triggers = { + project_id = var.project_id + cluster_name = var.slurm_cluster_name + nodeset_name = var.nodeset.nodeset_name + universe_domain = var.universe_domain + compute_endpoint_version = var.endpoint_versions.compute + gcloud_path_override = var.gcloud_path_override + } + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/cleanup_compute.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" + when = destroy + } + + # Ensure that clean up is done before attempt to delete the networks + depends_on = [null_resource.dependencies] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh new file mode 100644 index 0000000000..a98243d464 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +project="$1" +cluster_name="$2" +nodeset_name="$3" +universe_domain="$4" +compute_endpoint_version="$5" +gcloud_dir="$6" +MAX_ATTEMPTS=3 + +if [[ $# -ne 5 ]] && [[ $# -ne 6 ]]; then + echo "Usage: $0 []" + exit 1 +fi + +if [[ -n "${gcloud_dir}" ]]; then + export PATH="$gcloud_dir:$PATH" +fi + +export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" +export CLOUDSDK_CORE_PROJECT="${project}" + +if ! type -P gcloud 1>/dev/null; then + echo "gcloud is not available and your compute resources are not being cleaned up" + echo "https://console.cloud.google.com/compute/instances?project=${project}" + exit 1 +fi + +tmpfile=$(mktemp) # have to use a temp file, since `< <(gcloud ...)` doesn't work nicely with `head` +trap 'rm -f "$tmpfile"' EXIT + +echo "Deleting managed instance groups" +mig_filter="name:${cluster_name}-${nodeset_name}-*" +gcloud compute instance-groups managed list --format="value(self_link)" --filter="${mig_filter}" >"$tmpfile" +while batch="$(head -n 5)" && [[ ${#batch} -gt 0 ]]; do + groups=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line + # The lack of quotes around ${groups} is intentional and causes each new space-separated "word" to + # be treated as independent arguments. See PR#2523 + # shellcheck disable=SC2086 + for _ in $( #occasionally MIGs will fail to delete due to some active transformation happening, so let's retry + seq 1 $MAX_ATTEMPTS + ); do + if gcloud compute instance-groups managed delete --quiet ${groups}; then + break + fi + echo "MIG deletion failed, retrying" + done +done <"$tmpfile" +true >"$tmpfile" # Wipe contents of tmp file + +echo "Deleting compute nodes" +node_filter="name:${cluster_name}-${nodeset_name}-* labels.slurm_cluster_name=${cluster_name} AND labels.slurm_instance_role=compute" + +running_nodes_filter="${node_filter} AND status!=STOPPING" +# List all currently running instances and attempt to delete them +gcloud compute instances list --format="value(selfLink)" --filter="${running_nodes_filter}" >"$tmpfile" +# Do 500 instances at a time +while batch="$(head -n 500)" && [[ ${#batch} -gt 0 ]]; do + nodes=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line + # The lack of quotes around ${nodes} is intentional and causes each new space-separated "word" to + # be treated as independent arguments. See PR#2523 + # shellcheck disable=SC2086 + gcloud compute instances delete --quiet ${nodes} || echo "Failed to delete some instances" +done <"$tmpfile" + +# In case if controller tries to delete the nodes as well, +# wait until nodes in STOPPING state are deleted, before deleting the resource policies +stopping_nodes_filter="${node_filter} AND status=STOPPING" +while true; do + node=$(gcloud compute instances list --format="value(name)" --filter="${stopping_nodes_filter}" --limit=1) + if [[ -z "${node}" ]]; then + break + fi + echo "Waiting for instances to be deleted: ${node}" + sleep 5 +done + +echo "Deleting resource policies" +policies_filter="name:${cluster_name}-slurmgcp-managed-${nodeset_name}-*" +gcloud compute resource-policies list --format="value(selfLink)" --filter="${policies_filter}" | while read -r line; do + echo "Deleting resource policy: $line" + gcloud compute resource-policies delete --quiet "${line}" || { + echo "Failed to delete resource policy: $line" + } +done diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf new file mode 100644 index 0000000000..b6da69931c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf @@ -0,0 +1,71 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "Project ID" +} + + +variable "slurm_cluster_name" { + type = string + description = "Name of the Slurm cluster" +} + +variable "enable_cleanup_compute" { + description = < [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | 3.2.3 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf new file mode 100644 index 0000000000..ec86a03a24 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "null_resource" "script" { + count = var.enable_cleanup_compute ? 1 : 0 + + triggers = { + project_id = var.project_id + cluster_name = var.slurm_cluster_name + nodeset_name = var.nodeset.nodeset_name + zone = var.nodeset.zone + universe_domain = var.universe_domain + compute_endpoint_version = var.endpoint_versions.compute + gcloud_path_override = var.gcloud_path_override + } + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/cleanup_tpu.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.zone} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" + when = destroy + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh new file mode 100644 index 0000000000..c724e342c3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +project="$1" +cluster_name="$2" +nodeset_name="$3" +zone="$4" +universe_domain="$5" +compute_endpoint_version="$6" +gcloud_dir="$7" + +if [[ $# -ne 6 ]] && [[ $# -ne 7 ]]; then + echo "Usage: $0 []" + exit 1 +fi + +if [[ -n "${gcloud_dir}" ]]; then + export PATH="$gcloud_dir:$PATH" +fi + +export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" +export CLOUDSDK_CORE_PROJECT="${project}" + +if ! type -P gcloud 1>/dev/null; then + echo "gcloud is not available and your compute resources are not being cleaned up" + echo "https://console.cloud.google.com/compute/instances?project=${project}" + exit 1 +fi + +echo "Deleting TPU nodes" +node_filter="name~${cluster_name}-${nodeset_name}" +running_nodes_filter="${node_filter} AND state!=DELETING" + +# List all currently running nodes and attempt to delete them +gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${running_nodes_filter}" | while read -r name; do + echo "Deleting TPU node: $name" + gcloud compute tpus tpu-vm delete --async --zone="${zone}" --quiet "${name}" || echo "Failed to delete $name" +done + +# Wait until nodes in DELETING state are deleted, before deleting the resource policies +deleting_nodes_filter="${node_filter} AND state=DELETING" +while true; do + node=$(gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${deleting_nodes_filter}" --limit=1) + if [[ -z "${node}" ]]; then + break + fi + echo "Waiting for nodes to be deleted: ${node}" + sleep 5 +done diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf new file mode 100644 index 0000000000..1ac6f64b75 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf @@ -0,0 +1,60 @@ +/** + * Copyright (C) Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "Project ID" +} + +variable "slurm_cluster_name" { + type = string + description = "Name of the Slurm cluster" +} + +variable "enable_cleanup_compute" { + description = < +Copyright (C) SchedMD LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.3 | +| [archive](#requirement\_archive) | ~> 2.0 | +| [google](#requirement\_google) | >= 6.41 | +| [local](#requirement\_local) | ~> 2.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [archive](#provider\_archive) | ~> 2.0 | +| [google](#provider\_google) | >= 6.41 | +| [local](#provider\_local) | ~> 2.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.controller_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.devel](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.devel_compute](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_dyn_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_tpu_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.task_epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.task_prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [random_uuid.cluster_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/uuid) | resource | +| [archive_file.slurm_gcp_devel_compute_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | +| [archive_file.slurm_gcp_devel_controller_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | +| [local_file.chs_gpu_health_check](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.external_epilog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.external_prolog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.setup_external](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. | `string` | `null` | no | +| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket to use. | `string` | n/a | yes | +| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | +| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Default behavior defined in scripts/conf.py | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | +| [cloudsql\_secret](#input\_cloudsql\_secret) | Secret URI to cloudsql secret. | `string` | `null` | no | +| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in compute\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | +| [controller\_startup\_scripts](#input\_controller\_startup\_scripts) | List of scripts to be ran on controller VM startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | +| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    device_name = string
  })
| 
{
  "device_name": null
}
| no | +| [disable\_default\_mounts](#input\_disable\_default\_mounts) | Disable default global network storage from the controller
- /home
- /apps | `bool` | `false` | no | +| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. Not for production use. | `bool` | `false` | no | +| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/v5/tools/prologs-epilogs/README.md | `bool` | `false` | no | +| [enable\_hybrid](#input\_enable\_hybrid) | Enables use of hybrid controller mode. When true, controller\_hybrid\_config will
be used instead of controller\_instance\_config and will disable login instances. | `bool` | `false` | no | +| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": null
}
| no | +| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | +| [google\_app\_cred\_path](#input\_google\_app\_cred\_path) | Path to Google Application Credentials. | `string` | `null` | no | +| [install\_dir](#input\_install\_dir) | Directory where the hybrid configuration directory will be installed on the
on-premise controller (e.g. /etc/slurm/hybrid). This updates the prefix path
for the resume and suspend scripts in the generated `cloud.conf` file.

This variable should be used when the TerraformHost and the SlurmctldHost
are different.

This will default to var.output\_dir if null. | `string` | `null` | no | +| [munge\_mount](#input\_munge\_mount) | Remote munge mount for compute and login nodes to acquire the munge.key.
By default, the munge mount server will be assumed to be the
`var.slurm_control_host` (or `var.slurm_control_addr` if non-null) when
`server_ip=null`. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| 
{
  "fs_type": "nfs",
  "mount_options": "",
  "remote_mount": "/etc/munge/",
  "server_ip": null
}
| no | +| [network\_storage](#input\_network\_storage) | Storage to mounted on all instances.
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | Cluster nodenets, as a list. | `list(any)` | `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Cluster nodenets (dynamic), as a list. | `list(any)` | `[]` | no | +| [nodeset\_startup\_scripts](#input\_nodeset\_startup\_scripts) | List of scripts to be ran on compute VM startup in the specific nodeset. | 
map(list(object({
    filename = string
    content  = string
  })))
| `{}` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Cluster nodenets (TPU), as a list. | `list(any)` | `[]` | no | +| [output\_dir](#input\_output\_dir) | Directory where this module will write its files to. These files include:
cloud.conf; cloud\_gres.conf; config.yaml; resume.py; suspend.py; and util.py. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The GCP project ID. | `string` | n/a | yes | +| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [slurm\_bin\_dir](#input\_slurm\_bin\_dir) | Path to directory of Slurm binary commands (e.g. scontrol, sinfo). If 'null',
then it will be assumed that binaries are in $PATH. | `string` | `null` | no | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | The cluster name, used for resource naming and slurm accounting. | `string` | n/a | yes | +| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | +| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | +| [slurm\_control\_addr](#input\_slurm\_control\_addr) | The IP address or a name by which the address can be identified.

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | +| [slurm\_control\_host](#input\_slurm\_control\_host) | The short, or long, hostname of the machine where Slurm control daemon is
executed (i.e. the name returned by the command "hostname -s").

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | +| [slurm\_control\_host\_port](#input\_slurm\_control\_host\_port) | The port number that the Slurm controller, slurmctld, listens to for work.

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldPort | `string` | `"6818"` | no | +| [slurm\_key\_mount](#input\_slurm\_key\_mount) | Remote mount for compute and login nodes to acquire the slurm.key. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| `null` | no | +| [slurm\_log\_dir](#input\_slurm\_log\_dir) | Directory where Slurm logs to. | `string` | `"/var/log/slurm"` | no | +| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | +| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [bucket\_dir](#output\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | +| [bucket\_name](#output\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | +| [config](#output\_config) | Cluster configuration. | +| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel-controller.zip b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel-controller.zip new file mode 100644 index 0000000000000000000000000000000000000000..902c068c2a01a723f897028303de7b2985f2c32e GIT binary patch literal 84485 zcmagEQ;;r9)MiFWwr$(Cb;{OTwr$%!W!tu`{w6wNCSoG`@65Zr&5V_MuV*XD zfPkU`LH@V@{|n;30}}^(b9zUwDPKF6P0ocqk8fz&ZzEC$I-ZG}rIcD$w~XP8=UkI6 zTuqELnb*92q-9$15 zETdiL70W>zKy)P)SB;DKO^5y=W9qnTvxI*W>9DYhK8m#SylsSLm}7{wDJH*7P1gGU z{`dEHGI5g%kw$n4uxfZ2#ckXyeLRA`LOkNJiym`Hgz2wshbfqrEu>v0;M7%#S$V;S zn6!XqLQ|+2$p4mXwOhYLG_s_(3=sPwIgpa-JAWam8OKvg45Wmjopf|f#?>J^KI_1s zK)*PT|NeY*;c%K)H(uu;;OFh*?&j<(H6xJdacXg^P%A$(OF!`r@$hoR@erUevr`u0 z)ME~)cZdc32_?lH9*0g?Co@3;kukzlhMRl`oE8^9SC-I-s~^NoWUkUIFlsr0MOQ%9 zM9d5y3wuQjq&oLe?07ZQ6f0c# z77TX|yONW2&w(KL|0WktJXP&IYg%!cAA_=`rWo3L2!i%fKM`9Lnxn)sWZ0X*iNqt> z6Nz&5X8q;bN_@}8MN+3WPZS3RpV@*vkEBZq-yC}%+BOLqI`s9Fquz)KwkiYtk?$~1 z!B!w=ywDrR1!TpD7gaq*KQ2BR_)pTy3;a4er|S7}i;v6|B9&;`!MB*wGF6>zG?v@vx6N$nWEM>$59seN1eVbt;O=h90=T1p^ zZOT3#TW6*f(lv%jYcoyvs{`=!z>^q|kU)WMx`EG6r=RIIxBeOmQa1fW!9h{z5=1)` zgFMoBO)s%H3#l)ou-sX98+-dsvg`8(N^^UK`~mN?pdB2=OJ}It{+oZCS?{20^^{Y< zU7VW^6z(^R*5slp+5*>UJ2tX3^z%7-81O~5*a9OAB7eRX8a&;8etgC0 znJ#Dp$FjM0$BAh)lu?M$R+nxWO2q0PJQ<;YOeSYT_bjX>L)0dR4R zm6j)JV5~&vY!p=Qf-=Xt;58iUl~%x+D_JuJe^ag0*~pe@51*A|Y?K&Bo|e42r`!}u zt4gnNV-tph23>Pnxovf&*m5})4m{`NHZ6oW>`thjeG_mh3Yf9M{as+`?Sk9Ons~5b=xya{c~U%3DDs4KHz0R~rWtBMvMtcMV{uwv z#2K9%thRq&)fT<|=WBgqc8H%XS2j&0scG6_yLq&=w2Z3~(r4u8NR8Q$jn%-th%!w0 zImOev)wESJz{tW$nGzO>nWFl!e}qLr&XL&E+SVt-;rdOG87v5aUP-H2rU=I^Ub_W} z1%96E9Cn^sK*4@5o`12mzU$I5EM<4FE`EQ0I2+-5MHLo3N+qxNoscaDz~t#OgzVAA zkMA7`*!r`I&z&E!=faOLPc13I4$f5?^%Rsk3oqD@s`@PB!{?k0wTTaij$BAWzFtHO5e`Casym2BSZ@S2PZAXRumi#~e{&_dn=2}Te){c~J{Xw4-3!ylqSiwKQOF403&+RUCGHW z(1%d*vDmf(!3xm@q9mBupVbxTb>Ts_jA+^$MlNAt@3jhjcvt zQ|-^`sb9+;f!m)f^`?Dis;qU_bpt9d?^m>vL3Y_bRzC4RXw$3R?C%l;0#`m!@W>k^ zGD@ij_%?yo?v~aCDo-M6Ow(d!I*t^5D{37c{!S_1wB@wPCG~5FBh6v4iB<{|CF2em z>KL-dGSWLx6MJRnl1`T%EvYf+bI?7Kwq#hti)5tyP2rNH^>pbcg*DG47qjXXWm1yR zC=4`t;d2sm4ZFur# zM#g-7zKTM*R}lUJv{5A5xX?Q<$^QIVORykEk>C!S!{j87veUd>ii#Z21z%Z@gpOf1 zND7!0y~U^!P+(O?$$066rB6Q1BT~b@H@&z6tq#C1W`lU=y#UFMvC7y%&KvPw)H3Wr zL&XmAByj(Xc`zT4VcSmpyz5u(=G4fque=t3^j*>c zs{jw|4{0Y0-0j5n6n}2E89}^&ImTFCqX$ObM0Phf_l5Ktn2vh2b^276JFdJnc+tUG z_ljP|L=Y&B0*6t=I8k72>HZgIhR?YzSe0PGyaONB!+~U^X7xYSp$Ax*H&b}o_%>H` zFIoP^i$|C|bz}XBi+dJQ z@*@i8qxvYh??2VXDmb*FC?2wv?uiW!W>s%^LoC?gwle$%U{9y$f|8R&2OI~v6V-^X zZsZp+7{@S_g5)71x~A)Q!0K5I0JZ9so{x>tB6GaQy`%nXSIU{1k0g>Zn0fdiujg5e z{g)Fy9e-1_BP@)(qztvy{cl<3;2M!R5Bp-MtO_syzkAXssJ+ZuaF<8InU9k}-YSJy zq^dtpujm)B5rX2{E&)I>y>G9N2!o&|$YsX9YP9!w9R-5!hV@n3%F6_|ctQ1%*JS{1 z<@p2Lr@p#McYk?m65oegOzm)QE0jU1Ci~FZCI@T5siO~H9*q|ytn|c=cCW*Cs7x&)qykKd%^{QCOPC}wm6`kZ)-y7W9t8i@@4(Q7XDWEgO6+@y zX`C>YP-08&Pkt^ris{2_JxKmg^M{wd;2BqF+!<^qDTFghrO8A5h68f7b>Tz0SSBZ% zX2XXT-PsI=i;|6ca2}Roo|GcP7!KYj4pYvd)^GuzF_V-PEO$?ut_b{_T>LfWXhLF& zFI*e}%SMBQloS~>y8yDl&oP5N^UWEEkO6A}sTA#k;3%$YzS#2J)y&q#g)al^hHuQ2 z)Pm|)%K)g(Hk5jFCfRH@9cc^HyS*<@QKMSsg#R2wq?FP@M%oZm<`?fE`xE(GC%QC9 zkL~+xAwD7WH(%i#s^9`{5oDc4#)9wy8jf%jr&g`557eU3hyM6xjJ4b&TgYBq51vWb zv=+$`V)PpR-kLN+G5bJnjFw928D4dCA3od22@tL3l_@&CA7z7j zU6W30AnI6lw0Jlhxf`nteE!SdM(fKI9#GG#CyKQ|H}Aaj(ro(B>T>+s+i>fb?iBE_ zyW$l3fVN2V=(LWU-W2p`V&LshGsj=G5%mAS0 zCE6|cvoY{QU|I?=oZ((>MRev16yT_+`>_+zdQCQ@HRna(E!g2s)Rai^cj!n=;0-k{GKh93D^c8Od(=abHp#GWxDeHa^ zGOe!G*>B|%S>`0xm`0t)SVSK)A~ z?I^lQZej906b52#tb6m7_{3IIsk z&836jUOYSdEb?^Y*fs^fhxEmE+icq;7D2jtzBm7*smWRh*-XIT5n=W#Xe35E!u>=r z;$2mO;KB^UNh2jPE=py|RVFNrl8uo!Gz_JHQ@rlJy^^BFK27W@H)U%`Rm6i0oJBlr zJYwH083ZWX&L1#lH0<{W;M_HS$Mn=rLOmdOQs%yPxiZ|R3_lPihq{7wnz6p@+(lZ~ z4ixXCYsUHw{6fYB-x_hK;F0vPS4|KSKbXilYVFnFb1LLIv2AtRF!_H2XGNZ(+Y76B zc#70KlQ#m_svVb`Cj{HN{+f2+|9$J16ML#R^ay{(W8n1jl+9+NO}Z=!h7kA-)`;hj zB4v3%@iGfuQ(}%h7x*w}2*ygbur7tPJ|?@h|0a2nqfnyoc8Lyx#WTn>9=3n<3Y?uG z!>PydEgjC1)o6<#kbHk!B>(K@IFJ31NM&m+T8}Py=z;Ya+`QwH#M$B_GUzZIqcidHW#_U`h0@Rsl@2X#Rmkqy9cNBsNRTf_AjrIsk@ z7;PjVc8%m(BfQq1&pSr^f{KT7b{b>*h!os$snNCv7!s@LuZomc`aC;evk3+)hP1RW z<9Lil@v)Z`p(?cIO=d$qeLb!87p#>O3(Z9f{`$mcW+dKapt5b7yB<`}>GgEkRz>}r zRynaG+!A4#!B=sxFxWIUi|0KTZhi|f=`sGdcAi2ABc6;7Kw})~Gsw-$2nV&Ix zN-eF;_Hj$`+|`yKT#j4``iVetd5ME6K>k8$v6{j!<{Ug2xgUAks?Wiif{>ywhg_|X zL0p4)*xN7>YS28AZ^41%w^&w*Uq{fVLe^#nMq^@+?;oON!fDP6D&to##<&U<+T_9# zy3wE?ysh{<TAK$t!;rvF6~*0*hSbP~%#5zWRFzK$x(MCAE>2@78k z+%vo8--a!6+nC4UUnX>9LkY2p#Rj6TbI}$<0NP6s;#J$(zvRjIro-siP6vUS6h}Z7 z=q>wdp+eD}06SfgN|gme^lmz@HmPW+don^pH&3}AR*v8?PtTmX@YZOE&9L~$m!_~h z9ewmEY%A2L9%eYuMp;0|bmy29^MKA(f}kXkt93Vg$_52CJ>KgZJxazaTL3O4I~{bH zI-<{=A6)rz#pGlWBX7S=tP$e-vz;mxIU67by@KhulX092*)45=s?^TOVyjsGk6eP& z3co_%PTQ`Eaqa2OS8n$rw+l}e-g=|n*=5!b4gqezp*MEIXAz6)!7CnG??+LV`GQJ6 zVIh~HI_QVdK#*z|2+R`I5MHQrK!7jBLxBBYsR^n4`0DrG>`-rOv!4xFqB zlP!g7@2a|irIM*o%{E0&Z3O2d1Ds}ZX!i57B_TD=`dKMjbe-n1eVGPEURga;R4ug! z$BS>PnSE$UvCq`zGMij|y`}quI#w;wDJM{TMsWyjlC~Lw$RY2Bi)e52l-w<~&Sp-M zMtLbl4S6$d7YJfRKDD}~cB;S4U#Pea_bn5?J5$0osyD+uj_W%gMEZ%>XePU?Kg}*c z1GJ@ygZN%;_Ln-dELSL-BSIP-n#d!+0we}kwta-^@SIxP>ZbNKtF@IkF!?$vnPN8x zV4L z|3>(1Gi*(+bt4gNlJ#=^f1a4SxcfL~`E&9N57 zJ=Q!r@ecdWehfixbc9fJx52RzWFys2U~jG3tUJ}FpxDt9{YIs)C|pdQakBVIb?~2{ zwucY9Sm15gucY0zn$|x{NcRd{XztA>BJ|7g_-u>#jFzdpjp8?3Hu&!v1#n4x+OB{{ zdacGHiBBqVXuDw?P6pD9mK~m^M1%ee+zo<>Xw705z6~)ulm+Ab4#Z->HnFV763DN) zHFRX{+$>3AF(8@J^H|;HerT zI2++@%!Xr#9F}QV`WLcIE0j=W#LTRH0Er2 z(jv!wVh2ufnP68FhpJLog&-1i_6p|o9`3cYDT3zl4k#Ifp<{GLE*Icgw25dNdA#D= zer)7WjGyY*lZ>$*|_!g**Hzw0;iSjFtL)QbqXtxW3>o z`W9AsbkN^yb!M~Q%dkGd!>)zE^$^tSw9}I120W64U?!Qv5%L7I6QC7UovAW(FokCYEOZMHkI#dJdcHsD7(;oC$%` z;JO`<^%w|PdmZQIwDsZKdh-;)rM0Aks1u2*s5O@X1d{8u$MlnohTf42{IeV5pr8<5&lUh`4}?MkBIYfPRt&J)WkWZ zn-;VhxpG=NUrr<7tA}2u@lXm7Q*U~qgCLWrq%AX?Ok{1|$w>B6E(T9e>pQ^9@(xY+F{P^tU&j5*osr1Tq#_IllAL4=u`pbhOX#!N21j9gA)^$W)6Y`UPp95 z5Hb@4ws>;qls38+Km9_R?6`j#T>~!fUhZMy1o=25f5xnfZ{h6u(e5)=zPo`CpV$m< z7DRI$We~=lKYlk-YHB)#Z?GieRmEJYyf-R$WhipXTX26>e%Z}G0}R3xh|STbrh4s9 zjZU@XU>h48lqaf-rn+>9_~=#&M=4rNwi9a8KAF^@&inj(AywD;A?#sR)3-)|?7yD? zK~z6)vvNO7;uGv|6pi4GWPPI-%7_9o6fy@_dh@thKrDMaBsKY8R)-{+X9k+P2j7#>qq zM17fmT^R3Z$?w?zTqH1N`4qwo=ObA`wD0`aO;l7+W&iOVhTl1hp5B_TmSvIte`MKZ zKE)vb1p?ZF`k%;x{2y5?%v=pz9o-Dv?Mxip>|OuEONg4B!zL$6@1=(9aWoufV^(ij z_uW8Nao1?8JR%QWR3W0J73p*)u@<1wg!ZM6K~gGJwM`4p)|r!F*7t7zYPgtIFc5Pf*d)7WPLE)DqC~_vX_5Lo#?A|tD1e$5HwF?Qvg4= z?au4{sae-S;}yr#GiWpXdE2_nThn20pO1FcU~hK$!ya0)RS@p2uz84qErF)U0nZi@ z0*F})gc%y;A0_aNsD1-3Q5wWjkSIK~o75=VA`prd;~#l4E%Tlu;tER0a#dv=Lv9UO z9NX6vvYpzR${<2XreIt!*2*ZfMfWRza*n{uY;h2n^iQ^38ck0Pq%>!P ziu>V7J@UbJ_Pd@a0Bw9W$OBQX7&vBI2`;lj)VifZJ6~9&N9X@YU4pgA1jy&^a!f$z?a#* zo+;TNmkh-Mq%28svRS#3rX{hU3a|^3R8dXy->Nnthq8H4AwjI+21ByZhDrXILqKB0 zVg2Z(PmPh&jZgEEms(+c{7bTL==*&WQe1+qrbbI0S-W;oOsW6F9 z#iQBGhy88wxoN)2>dgfEjJQ>^(-$w`*X`%2@4#_y{(jHzey^K-87(>v1abrPuNvhu zBo2%Wj1dMmqrot~asLPz+Wl>U)_va3-`?%lwQnbPfD`hzTO9@y{q9)@@3SMqpB1;c0q{ogGGp-ykyiH6!x5+`^b(BPGI$-wKO0#cYJ%LN% zWndkC=k7!3J>_2`dEz3@btg_Z3J$-QyNlWp(lF5%5IZ=le7H@5pujQ2GPrLLcmgqFoc>GJ~1)5FKe*Y}cn`o+2m3tnQ`sO#2Mct8E+1Iyxj zaR$9YS5KgKsEXWYX66nZD_uOR6}{mNzS17$D7anpye#<*-dT#JVyb~|Ou;QjFocv9 zvU_%D?9KG9{&`-#ysc)a%r^M)>N|(Uy~BSL`rq|XRK=Q_`+wAuhW@`%%i6)%z{Snj z&dQbE*3F3C)zP+CL&srr6xn~Z4ik|xBAMiTm}`_P0=ShC7l$!(EzpbtK~AQ6yr3Pb zWAbXGQL(wNCtXiJwJ;f`5(9ZFBgetKx5LuS$>aORKc&03HyE=}z_rrT&K2PFY{A;m zA-9v)7l&C4`YZy$<)FOvprK)X|`V4;;i6dzimgI5j@jCH5{rSkT zT1ON%0!53i$YR#MuPjKe6r2#o+C1M6R;+yF+qBhCnH@}39VYwo-;jk&RprFxWmVPD z_h$5Fc-E&{BX@vLMgkM_10co`xKwbidaysAN|A^$l+ZK`J2s4t1I(0nR;hq-uDzfK zI6QBM{+8b1hR4VBxKBi*9UPoYDB0_Gy`!1VQyYjyk$4e&N*xeGRUZ_@cd+($w11L@ z&_-YnnC$A(ksdifg_qzy%z%j#Hc0^jkFP$$qFic0#imHwidxzyF$-e*CS8>Zb%vS`fM+yT5)ln|9YSOtE~3OVb6PpBKKetYw~*N+HzKPoTXgQs^jYrz zLs=-a@>LRM=No5I=Cg9~_3^px@SB^%HfxDazF7}<%wa7}#hqzE5?ZrTUK(GW?MJg8 z9JC%?b%_Dxq`v7kf}q}y`=w>!zjT*J)zj>ktj9i8 zTznIU4%~RJ5mKgG+{RlgYWQGh3S;@L+ShXHF_F<3QkU|k^NjqfkghEvakzd55(z7< zs}p)$Tj5$b#s8A`LbLKZ7!^It1bg6bbjE)J(^LjBcs(Yc{?fUn#A4~Fy$P)+vy>b| z6uT;!qdXyT!E@Dg*)}L?XqBWZ?P>RnOI>}^FA5I81m%*E+myl%2Z=YfY~7!Kg5KuG z$_3?rp|Bes=o)h+l<;$jRlI{Ea+W3U{;Ah;3}$K=soiXHixk~TPN|^F5K>q}O)=6< z*K<^05o~uyv7DS-a44~-b&`ABE+qNWseJ~n0oi3>g!j)W6Jr0920j~C%rWoUVI9Tw zp73<#tUkt$-Ix_(yl((?ndo!h6Re+;g64_${uA>0#(Ppu8_~1F;{Zu=ghRK!aVv0; z0lj4acBRXGW_D9Z3dsSKRW!t-sJO;$DdRRuXg_T@E0iy&Sc9OrTU#wDh%sIqi zdfx1*E-rIX5WK?=Bb)eh(F{eWZT*E^@pMr{{ym|4ZlrW1JV`|N2)Wg#U@q|KS_R z*1^csz}V@(da~Ht#^taT@Z#|k|s)f-Dkk>%E*iMS46 z6amHsYOe9@>GP`sz9XMVlDhW2i#UV97JOZdmAU=JyXaQE95OahSH|Tp^yJNU+3yR8 zh!24Db|j9DJ(x@he3iHj2)VPMO5+-Qf0x313c}BRL=7Ed??oSy@@SmWfG4YIujqQuyg&(!}xg z(Vm~Lso0HZoObXa1DQtpfGH24J`@S(&U@FYxFe*6T zcsqO4tjuCY^*`=f9#~}{6vQ|OBX69+@cZj0c4+UrP1l#nKEviuu|ua?urmk+f$*Gpk#SCOYomXb=O!6Kv^nLJPinl4g6!W#mC9 zsi&-dNHHVicci7MU@F*E4i!}f-l6X%+xn@zE|n@7r|F$Pd`k6f=B2dGuC!PzmT%ZJ z-p_cz-_wj7Q{P`A&7# z+gQN@r7U=l%t_`d%~Rj^644gmQllTUG=^HN?tZ(@K{lSlbc?wntTCdyl7xQr?(VxX0%LXJHATp0t9!qW>&A%XH}46rd*<)2c*iRq-lngyxvBrgy`j@(>0yE(If z^(SLz9vlN*3X8=@sF?nVQ7{!sSX^PwU%=dy_0WnK=8{0oLM2Rv32MBq6cWSy(@^aS zW6<|D9JmmIIZX(g&&j2;-4IHP6H>WShUf%^T%en;>pjsQ({t!6LZJx?YYP`GY)=0G zSJ1~;AsnJ#5CJ?qJfv`UC1bRo0P7-1qXZWd=b!?z{d|Bk9s~hDIfq6c8Np~%k^x}z zJYZfoK@#zaa9YMrk&L%WyoXO_SB>^)WfzQ((VFE-a?nyot~$Xia=>DdU^vRL=xi~> zbDZ~(OJCZBTqaR28DK3z@Qk*YNAMZ!89LjQItprh_F*X*O!!863N@RXl$^j)py_3D z&J$yVxRtxRt-EF>J5y^Vq;^g|#$?CnA*o|!aX-CesEJFZ%xZwA)alkJVuJFvtV@x3CAB<84%CC5E zrMrt&MIA{w^pFa_PmY2ugs%#Eb;ec5{p0QT_cxFMtLxJ1{rRX}o<3Uy?&^bF zS=b_PW|DHsJ*ww9i^^f4Cp0zQOX|+9tlX86aElM}P|BDM&8aKu=Ic47PpKKG`93i_ zzA~Z$4ffqdV5`ZKezCCGJ!ss@tn$>>Dh>;83+b)uTbwYE2~j5I*)DU?0%L_0+s*ep z?dz7SjrZ7t4aTOHei(X9d%5z6fZDL?H8->cBc0cOsIpk{H{CQ#2`@N$|1n}MpCS^Q zrn{MB6ZJl<+vtMkK=eC`HR?)E3w3>!)J5HLOYh-kPmQWSrKV8$TUzfHE;NOUjn-eT z0%YPF9XYz@iUy}k+nRa3^`wxs{8w~dBe|Q6TB#Og=*eO`gH);ZwA|8R>_N9QhFvB# z+V-9F^z#~#c_{mmj%A#Rt%A7Go+jC$in?W_EEU@tIJ7fC*kuC=`Kp`lcCZ;=ae{?8 zuynX`ho)Jzk+!pEqtqc`>%#t>Zkw+p @)wAQO`7^A}Dy}~JgKH-uricek%=|#H1 zc3SMAw3^JpCO>M85?q0I)+nyO5thX{{*C4Lt&t?%GaK1->+rJT-cWMTH@E%+t0L|REC zz@4Q7uN!T~y9ln$NwDdM)1*KuHBB*XLuz)r}LhKx>dA%xl?Ti#5}~@-9+$*MliAs=hXY5%`^{KlJRaR9lSvfv z3wS&~hm*6+(pIT7*D!-6>UKL8AF{Q981uJ^hLNzY3?>R$l|$QKZixa>wieLwt3Z$K zrB=>a2}hI{Rp)4PW>?y>uu;vy-EvetvKPk_=B&cj=FUuBGcPZfY)cpYp(o5x)R0eJ>a#iw$+)u~ul;yjlltnq;Tbq4g;fNGdmf1fU{!_cUwMSV|!T^(w*BCq= z<$n&;G;OFFC2=*$ee;jTyuot z)&X3KcqQa`@}hCGB%fMJDR;-f6|Qud+@{cvD1I&EKN@?30Ima3@ZUEOXEGP@0xpPv zfRMIT{m(1=-v_p74;6d3?j`c`zJkCHDyY>A6;7<-4;g<$A08xzwEy5z%$Dd2gj1*H zXzzTDekG%b5$EJ{IV*2Gi*7wLhg9_3E90%VR51^M`;|K$<2m3Xm6iTY)HL*HaF~D} zzR(bHj@p{_#?t|**Y(O-a%X3zZ{v^)8Jb3kV^rnUDHukTQu*o@wi%n0RWB|xFSvhg zu%9|>UV+14S)*ueQna2hdvzi&6BoX))&eZ}FM5O(T(ay{14G(D%spTDn*DTp2 zGKJvecAFMw;k2_GxpaoX5ab|bjDQq<=3@E@#y5HcZ!&{U2S%OF>FW|pFi zC!{L+@^Qj8&5;``FIRzqV)MG#FYWW;!nLvOu&PVT*)3I9mcG{Bf0tZ;SC$-O)rY$; z!?!zY>)Ph``C_+gy6Jr6=)!)*+8v&yyi_40#0HAwmFGm8zF3>Z1fkt6@SL)G7P%TW zg-8C6*bCejpXGAM3o_z~L1mqF(g;{}hLlFKFzJ3TOC8;s_F)Y%i@&K?BbGkSoUt9V zR;DHf7Y7%HCR`0fz}+9{(UhC1mOk*d#bxOvwWke|?U=ICUkHYe7DLeD&F2M#Umjdo zaP++L6!f@iZ%E%Umb~9>rLvyU2A**WI5Io(zkJfUf#Qkt+N+1n-(!qAPS={OjEb~k zf99rE&h3HB64_G3-Q2M^cj&hDK1Uk`=U2s_J?e77k!YP;aYLzb73t>pE$x+gk=ahS zmiux<=L$X!CJyUv=HC8vOTFfN&kgnSo88^%Zz@mnuNmltR4*W7G+Q)qEt(uKr@5qR zWVCLZuG@+v_3Bk9qQUq6=X~F$TIM6auHqcq7^0*!Qo6b_v!zD$)*P%Fj7u1K70??| zQNv%Hd3p=_zJ-mnWF!yEF=xH@E?OK$vW4 z0Ac=T2*DohPRj3 zD7c^UqiIw_+meK>0j*(6X6LSGO@)Gq&t5OQYd7=8iX&^wiAFco?WgOH!FfCHX~TBU zR(~od)qDc=BdM0mcA(={4*yy&SC^yRy`K8$0#0hp68t{q4MJdatN|J+U}&7^^ptzV zxAaHhIIM=>E|3hWBkZkAwMg~bZhFl>%kX`roBEIotj$kcq0@6Zv2h+c%el~>b~fc! zGiind6`oaB_$MS(xm-RU zg!f6(O2Xd3-OhC0@w z&SqYa!Hg`G`P^k7XRUKWnfz?ZAsU~<4{;Ffv*9a zfP>tTR>llFf#{&*EhpGClSgQSC_-eTZna$wT58rv{sdFB7SLB%1X%3YYHEK}iCu7q zQ?5>Av-`Lo?$jya!tZ$kg@@OFvfns3cDID!ooa=!)~ROoG$;27FGuV4L-c-GYHXQf z({Hd&{|aIbPRjSy?c;-zm$eN-VW{3{8lcs}3MthpF$MdL3?qL^AT13=yN|K0RYq0X zYQ&b>G+(3YXZ9LQyv;-m*(~_7aGCH7v$eT?y(i-WfJqFfC7yaamFt#ie-3feA$oOf9S4M@h zBNy42tKYbUdNDz7tDMNubY8*8YPE$rT{gMqMy&F^no-=-Hj2WELbY=atI?>V z#NSV>;EV|pLcFSZ0qOlaj2}l9=~tRg`d<)rkVn06X9()AeigvVunY`jpc&WB!{MOD zX0-aN`Z-5&ZjKrvJZ{_cCy93KkL&5D4S{Bvk&hE&mA>6C+y#M>pgDy!Zbz z1fXf>K+<*}(3fXSc=)4F^lTu@W?xa!nt}p6XAQkw>+s45O&)#LpHd)?B$A_)Bj%( zr*))A6$ieBZKU4l8atha^aU4-rae$~L^dqSSS|n;77#0MSVlv+2A}M%s0G@Fj?PSk zbVoI^5U7}NQJ5T5foepvwLzf5B2|d;)?r?t5eEeZtE_h;XyM0gI0I5@ajnaR72mdL zH^{ptn&nFiQh;ZmmusNQKtH2T#_*k*mn>>CuveGDIBwJeqMoW;Y)ghjx*5dY?;Jcn zU^UpqHVBmx-+wV>moab#+Fm~sp?3U|yqu5X9xq5^2o2r@rb`cgcWvv`fhk6DxBu`G z{;tq=XY}&(ef9D3U<~~D>d4>c@8jje)u-S8#gnf03oD?n--ibb-6jt5riwjS2M4K4 z;=Hi_#|c5O6ogR6n3_2;$dtNHGcMs$f>pX4SDH1n-c%Mw_WlgCm@>pd)>q)iLO3wMnmChMr-MyS%MU$xZ8M^ zJMkqT^oxM_Y|-J6_V_p$i`6*9#S@PxnX*AbP^lOy-gFo7=tZD{hx&_R1`CZqdD`Iv{&p_+fDzV(b5F+yZ@+&6O%`PPbvM(6ywTj{^3>|M_~? zVrow}Qu*#VwAINay7_V@eOK<4$o)#5%pGp^n0j

zZ2>8Loutw`d%}QuG^V?QKJi zEA9_vO_A$a1;Y<~8S{I+jr?9bBw00VbHIkA&jeDIRvffeI2Gpj0^TmUo#SRM+I>UO zA^?o(nxgdMvPB8S7CclL&5j)btdP$_+CmENQYA@#i2QacWez%-xnzB$#@nLFjTc{#SrL_6JPG z0%dj-9wlHH;_1HGJRVYvEQgXTNO47JpgCFc6)~|jO{O43hrTB%rn^X?J0Pjlk274O z?!l&}ezCRf4B+0sc;n^nm3`aax`lvCEurQL*re{iYvM8=?VH{+{8Esa%aKmQk&ReFtw13D-c1Pvsj+IzQA{e5M`A6-+9yqP*f7RlgiG` zwL7+SaC|u~EG+X4UcKok6uF;Ch`Zxop8H9ea0U8O)P%cAN;}!{OjCATcV!>4GMgSX z85$au;lRne1e z{I7si`yJv#WlCMcO0+V|w+m&nmevmSGI&^R)$VIB;du-6bW)fr!%$aH2`c<~kNT9y zZ+01uaij2=JhOz2Q>>bnnkCd;lLh13YQToqW#-VujA1nR=HVZmSrjxj61J|6eM@2c ze88y}xNNJ5#P!dHWl71n*gsOsjpK}E*4z04V!7TL;qbZg;=l-n62`R#TjR`o8dvz~ z*i_=}fu(RJ*IFJz4iRarNme@nxVW2&Cv3eMX4ViT=;}0jZVRm5 zY+Wx?@s4lV{m)ll5?5ENPd4DQk^-19=$-6UFd?m9J~fbJQI&70B{(@s2cI7$*9|Xn z+p$I!BzFJsM(Xds3m0+|vAhD#B*b<2{-8ReS-7KauoX+NLgd>m02?b90I^4jQ- zJ0@)npaIc2F%vSQQIUX=Bi!hm6X6&t#bvCk_y-l{?85iU zWTm2^Q~Oh29MBL30T`0@Kae0MPYzvQR>P1C;f3wj&v{9od`f?F8lVV9D*q6ehFt-J zcx5S>r_zq@ZmVlF0@zY$fv_g1~n3J_c2&&iG z7mVjb^k;8*&LnarieHxL5X{gXs_BhEq)iFUv04Z^O>YwT;x(j>vwv&(sl)x20Paf8d!Xi+XmQ`^&xA3+ipD7j$d zb9C<}_rAmdhu1zt)?6;~ZT=q1*mE~=-JrT13FhVu9_f`Gob?IGj>ls4rZd$VP3d_v zZTQ6$V(Ora?5dfpZy8vzDWfv?{14w#;zU*NvGJl6MMC1-C%2Z~JA&^nt9&4($zelMzT zdjMRPwzyyEWxZE$0dIDmlz-2~mCOn@RdnvZgSd!_#RqMp;H_A3KN6fnyb|8iK;6CU zI0Zg@CwM_S&ZT&#J%7=>^7oh_UmWoa5x$xKCv+X-4Y>0ASLzeR`cI)N^uN&6*3w+x z%*NE?U)Z{=`E9?|?)|IlKf;VTGQlA}gUW4iFUxKsne~*$6;85N2MvTz%bT>JP!d;i z*(vz*dO-NsY|L_L7wp=)AXfI@%1@mYiaFF&#h#8#4ptAT^9y^FsSqN}&czQIpv3Iw zlvBW(DNMnzuA>erjx3_K%9Gvy^DIaWR=*(-Z6dJ64?fTjK(JMXCD{s4LAMAtP}>ol z0Ri%tYH02wDXFRjF=rChbm(%j!6tHG419)#Xlad-GDxVl0H(sf0yfDReALv`meK3i ztzBhs8XA{Irw`h*(9fBVl|NN41*<<4azBQnSio3(Ri9!pYuE*-rtVW}LxBve7O*r3 z5t|vHPIRqqJ88=FS8Ug1jv9bqva3Cr+P1~4NR7>`f{_MFLd4e@xSnM7>PHb}9oTYcc={O5bL>Qp2M} z1sAMSg>+M1u+&qN(&jiDmD?{W5|#lGY|_!D>%3!~`%+9ZnD8nXcb%LsT54 zyceR1{Sg?Y(b6;WF8SsK{NNWh%-qYMB4qfR)aK1--qJAKm?~0HwExHA8DpLms1XXN z5K3ZG*D>%#2Nm?v0tbZrt`4mn{VaNFsd{OuP5d~2WCB0^7MkRKi7=+%_)n`v5Wx@* znr3Y-WwFkFd8iW_(~{U0q>sWB=v-OugXr1ZU?9EY!w7^`jP^h0RZZ$03xuf^=QhB_XwzXu+Fr*73 zrG`f@b7n;kqF=%3Cxg*9{zmVW(eKLW>$sj?4m-CNWWY7Ga0-yZ-Et%IhU7Le>8y&T zgwNKM#CVpFr*EyZxtWK&6GLZ!K|S$puE*!{7Bj4ad6e8^2E)yDx!W7OPN>QBz(0hc$T&G zm}xulxCt6x?m#?mns}>hoQ?D_zU%I85kIAOmoF@n(&@~|tG9GJ>oF^@HQ2|jwC$}1 zol0>bW-?aP#$Nyws9-A1IP}@W*#qC}FR*`&c^co_eR>-!%ah5EL&QVEK!)IuL%(d&4 z&mti~iu^rFPovr*38^h2v(qKC{+q=TjsM+Xkc8cq$}Xg1tO;qxXqjq_Ph<0B;*x9VPKeV*@*zXA~XP*Pb;qH!F|+>&wUwYq)J!Lv3}bCn8}m1(D97a^$6~$P)5o3v_ep zoM+rA2@1Dji*TDb)PtU6C0ffEcz-Ni}BvT)%#q+M)?pP1zu_*}~Y`jG&0|8~b| z9*sx+V015oIz8@h>DiY%QurNVZ!-I*Gxc$j_Uv$An@~G8k?lWg+9G)CHBeXd&ZtL* ztt>3}`%+Mk?k5}e=B6!P=gqPcw_b<=KcqY423nn~-E4I&h zE5El&YYXSyCG5=G&O!UhZ9s2hP0gsVWq3w8_|9$X>BEFWx9gvL{3QAZ;$$4iSeh+> zD{|kKl4neW>#kEiu$zwd)=HHti!Yvk!v6c(#gWv!D+SEo+}~As@}$z%b{I;ahunWHm>W?AVvjjO9&NSQX?6 z%yrr~1j5 zkCJcBVV2I1Z_kavWH49(hn%j62ywk0#Lh(-G$^ z{Ylih>ddb5!g%4OfXRy=&ce>&m8hwWDx0CeIo7>r)mhdbVjIY)ok{Rpi_LdL|Lcx* z+b~Nz&OBo}Q82;9cjLmf#3G;PLy#EFs45UxE@d1771plAL=q!4NyuwyMt6!LgJK)xS~7K{U1ST^K=qcmguJiDLG9` zE~;S^UuQ}?xGSkBS!e=?ZijYXI+5_n{Hy%@kvg_3+{LR{+|aUoH1Fiu5|l_xt}riFpB=yh^^iR$Sf$;#~>*Y z@58bUUvArruGvm+?~~6fH$w|<5$BRf5qNwV5AUT|!UNjU+UV{c+t2E;@cO>qa$vDw z!Ly0a=(K@6G6;FrL2`mmSqc8Rj(?84F4>{`IY?x7vYO)JNX72YnmzDOggVnXYf`n+PMoc`!RcCwZuhWqan#%>m##9B-*JAQ$EG->0DeIcHP%IWS?K@BxACj zuvV>3?x*m`;Lh6-_H;~7?NS!pc1Ht-v$iFe6nUpF4=L4pAQ_#ysJPl2&=k0 zk31Y4JluhjN%t~Oc;|En&{O|Gv(X0dE3NVyw7S+f(k?kUf^qmbt7x)6Cr3zZjo}QN zf)sGesR3R{SSba^eR_;KmF9xG2DsL!Y*4`Ojw~U1SR@jL$)C*tov`l! zOdt$@yqHR}v9Y79A{eG7rJR<-_^IeOo? zKi`L!Ln!SG@H&A0l_-Pss1PE=xVYO6yMP9iv;v)nnHo_rP-=Bnt%nUs@;a$SI)WoB zS{6Jqur<^0aD@ogh8czA!Lp>_;L@aP25hKtXjf8H!0AHSlsf7h5va;<1We4KOHR)F zTnsH$8H?L+vzZ5CH68eI`ObP6r(IX!L49xZeHNM@bD=u5A{~$Vr7!txHo9+fnKV#& z3t-n?sLb%88gD0jj8F3bgMWf7XIkuS@HX&h=0weWh;N}Y!m6=9Tp}#?nGvd$sQmWV z>SHOcExwM&M!_Y@HiUKg9(|(WSH6goMgd8=N zM~tbYW>N#e^~}cWj38B+ljkY2oUrDYl$(J3zMS6gE=E>n-p{t49}ka4XnCTG0zl(* zo6_HIcP1t#P{0Cq<>A7F&+f~UlaW#WjeNd-eVPE|9aOGf``SA`1ZK7oJNt>&DIv-aq=5CU&tL+IcLm#|@KTw~xk^C|oi-bO za^I}@T&rEtl;}FZWw?chgLCQ)L1nKj3PVv9i%K3t_V^SgHn`oSyt~OUlI$cClMCO# zDG}Q|JW&x&seCMBdFVY*C8Yc=M%N;-)yWQS?gE%E&Nf!E}Gq%pfX`uk`+}>4gKhDsXU`M>K zwmppa5>e*hj=z779#rG3=C}j_d$C{jCzBNQ-}Kx%TRbei+2%fSfwh7;CG&yv6b z;=Vxy`z(;A86gMOAgihLP9Nwapez>lkdW9eIDfJNI;k=aNFF88iV5P$A?@h?)-vAMFyW9+mcod;Vja#N z!yTH@M+V!&ZL!;P9qCM=>~=TiyUmQB0xMD9g0yYn%6w+KR*2J7+7n3`f>00Je<3T| z)Y8rvEZvkHE}#&Z%pcaSda?w*^L*{*BKk9FVkxfTz!9dCU_xjWFc!G(Vlu)7?C2}0 zTY!wbY?XwQZz4>OwQe8?=5Ko`Q`DK(GW|%R5h3Ta_p%@NW=gkk;e?up6E&xf_U<$Z z5a3;_4ayBOalz%u&}snFWrkUkV^Vu$GOr|gb*tb;)TA;)&4*a{RwCKov?3G*mcz6* zPDhl_zS-LwyR84f(yom2jl!1(&a+v&>|6rD5^MfwT0JF3F5V|$c)~5*yr0FRtCzQ( zDKgh@`TVi-E_;*YZ7h;U8h93bgTVlyPC|j-p4)&@o%8i$M1mrIYhqYpW<5lFZXkbjF?8r!#pE;84PGv=_e~HBnAVR;N~lC;hzWR(g~yu&16} z5m&Jo7aJg3!msbs@r~fI{U%XsIV7+K{@plR`NaXQc#MGwM>$I%UF(Nx-e&8MF@%8n zpd29hww2^<1&KWzp%6>>`ysL{km5LwVm=DSX&0qc+Xp+KpuT>>Iu@C)K^Ya~>IT|D z6AyFVNfaO7hfyb`CIJ3ZUvG&n$eeByX*SFRg}VG=ecUA6W;Y<(@n;qnGBR*I^o9e$ zLO48TCt0p`GM-|tXqmSn<4hIq1UHqtM z{~WKpz4JCA(2-#E`X58rufjH9|5oBjCat3klgDXsn{|wM5T~)b@5uU|N~&CLwI-KP z1J6)KRhP{+FOYi@^2{EL7@t#LOk)lnJ|3^wd%jV`FEM)k#_SNB%&$z1B+?XnCki+1 zbT3zC|I^bP#Lu3a&-tg~?2b$qxw%bIF^eOIh;gxj$TfO6d9{k9_eN%gLG#((39!v* zhJHi~W4y+^YhU0%@u^?x?G|J^hr;V!dXzF_0M-auZ z0<5qc{9RhDPd6bN1YYOBrei!q@Iv2id|zN2kqV4YKAU-ck)#VPlH#}%g7gAxx8>Aq|Fh$)@al{=LH#|l)*D@McMw;N1v=g?oCeW{ z{IJ;&A?&#B7OsqyrUs(VqQnTe0le|X3nJbpxMrt)tb(~rpbT=H zye==bUl?Y3vOPsmdcV@~WbJGDC2#1(5o`Kwrej;PNovpHaLco@jp56OHIr-*RF(75R zPPp!uo5gk*$wbpeqSvqs9$9`O>jPF4?rVKAc}X5!*Os1B`oKRtbrhj2);@b|SM_<30oJhZ|85d>7tEivPEy#ex z!5DrMTxeo-bKcS26GvjzbT$iy7yb9?bOtRg1i-iZ+ZKJs09!SRzT?nTEN=SZ^tTRpC^z=J+_C15_hMG|MZsh_ACKr9rCy7+^#@t)S3+AN^v^K~H|zWLD* zt=+p)w}Klm+`(r}Max0db_D*e-<`4kbLxlkB^~O!bT;vZeUeKXi`7CukAnB_CCP7C zZ{bM@^jH@9l;l{}FuA0DKs?m-K1JHXYFshxomt;_s4oK5>4uw)-EaHGHyu{MlkCm| zxFDvs^Bl_&u#OPqt6?oRlvwO~9^IbF6Q!|b86`^m%JeBAT_mP<@zM#2qmoM_wl1&2 zV*7|zH2E%pb>vG?n?SAo-GP7%650WRLha_et`O3EFfyk)+f&FTdA|bpm`}n%dzZa! zMjE@4HGI+EGnhSL0-oqa38j~}a4Fa{?1F-VR!TQ^@yx^1?&G10BpEgA^v*w+(iw_C z-*rP2%HA-*g%0swjXMK|<0ijtBCww?;e{{{Y@Gg;^I3u+tm|bUXoI0b(}xpfy=WAR zz%HaISuhjXMqlx&Nwn&c$yJN)=WeX}ce)TM1~zE9g5e*&mj$ws8aeYSZqBc^FvGW{ zm!>_Lt469H$A5z?)m4;G_!h=>A>5!OlUSG!%&L>D>}r1JWojk@o5jVH6NYZ~36t(R z3WN)M&1#om{lUapu7L_;a@0+!y~8@O{Na*G0!3~2GqtI2GBvNB0#iK?fO=vVZj#2? zplDbsI%UMlTZT>xN||`#6CtaZ9DVNjwWIQ2#K8Gl%>zl$rOgFO=%%-a8k>f2sS-yX zj>E-|`X7+xxkkTn#ugxF9#Y`2S`0-AXxS5-lL2khYk&Uo%k_h$tovBtsN%S~Gc5ws zUX4*9(lrF*<{GZ+4hvO|*jv!VqB61cP##A*C)HBASXFtsOe!VUiCM3`!nu0z7jgy+ zs<=9RUP%8ji|27m)#~Tc7)w! zHq4u&ja97JGt5mn;Teqn$qV*R90Im-VmbOH0s7+S63l4dFpIYGpv-ws6#Mm^}%2orHDY|HdZGa z&8-X4s2omR@l0W8e+xx|r*pE-*8;jpy_V;OK%R!1`5Q5XlO=TNItwQ)+@Ucr9-aTn z-k2`O3~E82kErg82A|$9Pf6k#)dNdi!FKyYimREO8xutx6vDoxYv`AF@A<^gpz5)k z@Z9mN5V&y#$<_kWaQTdl_9t1TvEJ>=;pZqlAGj8kPPFWT;PPO5bROKDC|n|-B=(6G zt8i(fUqI}ZtD+e&a5hOrL3AZqZhChAH)TsVg(i_)rrYp}Xu)P;sasF|!2mv4#Ijff z^%Nh+PDWMR^6H{b)!|jy-hn1$>I{)h2k&j=(3JcG0ffq$ZV4VMG%+H9fz(uVbnAf9g);p1+ zng3ZsO_`${8&xRH4RLB;T04d750W~S4piT*CbdR5b&ENnia@iPG{_70B5`9Y8`Y_r zN+P{k_P5Y0uIDk8BYG|XcWl3LmLNfRk$)h?yJ!>>+-kN==W4(2Atu<2gcUcIEe(X_|^B+HcFE z`2jmOkTW0aDc+BU={D*_@B<+WkCy~HwsmbF+zuf!kR7qZQL<0yjv^krgfT~$Lp0)@ zV+<<~eWJ9LpW{Gpq2`u?;qY6TxvbQ#VrQR!@nTRzh$ZD5y<VrYTP>$l z*I|3Q%G4{_YF;kXSZIf)yEpw)-L|c7bTD)cJW-aEKhz@HU(L~E4qkr&55XF?WJfA@ z4*T1Zc>`vQW?Brdp!@;Vq+z0qJZuOJU%5IzH_kmOEK)H; zZj{QFT{c=Jt8h5{V2f6z6*1_{IOe-`udK~pdix=OmQ#v9flH8tKKLoAYqxol%eSQ& z++e70MGkNFtpce1OisRF=J1e)Vq*z*AS4PzIJjxffGxuQEoZ(kLFFPQEK;g^RQOt} z3~oqfE?!&&eDDvx))&4?Id(zxQxT;Wdxf|$PGW$kc{K~xE<*%C!B`E;6hLr^HgnBJ zwud+&W{26}${KWIcU^^Q? z7Y}~yM5wme{nVqm`gZx}c9^_9C-Mb>4K+B~kYpZ1T{&&G{JL6299~$LpNr)jAWBQH zt1Q&HIIr%B(&buvVQDDsA$Ilr+w(e@tk`YlF{$drbPg#8CN<$~T9kZ5LE}beplOtq zt^!xE>q*vB&Q*m zxlp7WV1ND^4FJ<}+0EUKPY{;p3rgl^I;(guo;zb-)91_2EiX^cpOfeR;^qC_8kxC~ zeXha~i8%FJHmp_u<``;&pK~N2b>D`4FKwZKZW+)+L>^_zRgz{7enOr3$G_gr!kdMQ3cHp)Xj`7@jmTyW=!q zD$2Ll%8JWv@UE4TsiUkr+S-Ip(smta9{-fDQTTq0dZmFBFblPp(ahQD`$&bLQt11e zSSh}Ynnu>@7lz9Ki(mKawW5>*+<$dvZv~YNxLQBGoy_B!6-F;QsSTxf{F9czRfUB#wyPJ=@hxZr36&kRJ)%$(_1#N~) z`^HsA^>6K#J`47a8@IvP9g2{GL$n26ea!7HWHfnXYGOU*WXpLbH_gVPPP7Hq_PlkU z9op{;4xLwPxXdVtrAE{yan!eD22bQH)4gfE?lMoK&~cy`^#1SWcACe!^ku!5Y0bEm*Sn}T_L`DVHm#W80ThRUTwWyg zT8sMppjYdTwpZ(9Ra+g=$<68=X$tph!Hzr-$@Xw36&oFHFOvMWB=)o6+BIkeiUOE= z?T5C#^@l;TM(at7X)`Qahj$k&^~s(cs1{}#vUe!Rd|KxXwG*^80`@hy9@b&YG8ojuc(kl?9%$#R*wKXMtcB>7~{mev#`4t^U8 zA5Q7rxxDW+~d{J=4}3 zy6rtVp|!0(K>(e_nHBz*92TEv*Ub2my~J85V&~oGgv5+lt(bc#bFWk&S6liA4wsl~ zxA1~>YF4%t)2SSNF|#H*C*)8m#9q(5LR6~;^x^pPY89r7P=sINso8A=rr?V{F|=ca z9JSqwK9AikH#?y>0;#<1Okj`PUJtuJgt?zyLG_Ws>J_NE1XME|DE4% zGc$+R>to*@z9xw4djB|8%cBvTMfWg08`Iw)at^RR^0(b0b$G))q|KEag0Hc2V)2K~ zx%Hv0+^tYO*n>d!6ceeEr5zO-4?k@-Y5E+0-ul2s4 z{k%by5a4*{Gk%1Om!~a$6zCgQfjhwk?yvX!U<9|}w`cPD+qAE?{_}Bf^OyMQ?l976 zgci51?R~(T70J>hmdNgMe`U?#Oa6QS;0nl7!O}LrfZ`R~O=Z9=HltSaO{iJWaTNaD z9|E|o>pjFomtU_DbH5`aPh1%eZhwGaQI+KtDs$%Q`sCxb$77p-+vAZOY_$8sriG{j zXeWEn033_KU1)Hu(bP7EklC7Md3~KO)Zs22apECUK{#9|c7d^5ufV3u%PbM@*)~biPdaWE) zCwjvk{iqv+MT%*-MZ#7(O3}re1N;J2V+MhAX)Ctfi#>jmWJG-YuO`m$LQ2tkpR+hOz1~(n z!WJW3FDj0|mN?`1M7QCadgfQVwwB9I45oVa9R80?-HbLRYZdIp*eL`^DODom-|C+q6>!P#+v$a1p#=Z~A5)m};sd6OUE z^ZI7gnQLa#A`IP+I2$@<+7U~n3s=yC5WwL;LBB84jqze(=LTzD7_iZ%nm{wTOf={f zvK-!f7}O?><X@_|7ciIN5N|0^i`9y9{R00OW)7#74{?O> z5woEW01MNNb!$9sRgD|Om~)N@?!c7&b{y6_GR1zS&qe0q0jn=jL6koQXiA;z7o8Tc znpqOwFD=X#E!(1Byi84=@-TY6P5Kx=noH@dE%klT-MB11pPTa!7CtfBcSZ$>RzA(g zwk(3-vMIXbVMM`~db)A}XS(7UEDq_g>>f5|=wPB*nOx)@F?^^wmbZJ5VsXxv_Cz$M zisR-i8iOY~8aC|)-Mm?Nnqe0$r6ez8R)|C$ZG9(pHuVUcRs(?Be6uR$+QuiDe? zEq3x59~$sp{?X@tH*q(Cj*inh#(}k&`HhA+Ec$gXHrV~T%RW}AwJQ0LUp^#g+CfTNn(D0B`B4V$ zoaYY(*Hd|lm^3(cf>+zd-n0-eLLn}$lc9rysS};Eg-z0e-4+LWNc~WGMm}2*NJJ^YTYc+zQQZ;C zV5i&KpEih~{(@5COzo)SBo!Ru`!0G4@*S|c+hJxbzbxjS-%{(!rIvf_Rb$?mm>o_{ zd!RbH`k(l@(d8(H%$Uv#5wsHCR0~&0I749!Ek&DiTc^OjaZgK@x+LiwQ9ys(hbQ=& z@LbD<2;62!3tmE-5^+-0z&z+bfJ(&-v8qbvK@F*fOS(^1Ney_n`n1zr5S4@q7+t}Y zeyf5Za2$Nu2-o(OojZ2O3j<`ME0tUjzFZSxN0xkeI;814&%h=Hg{^i^R=2k`h zjtGonR7R(rirfKGIRQMSLu*U`NnLiLvJ|q(owP2o(sYP@b;`o$Nv{Ma#J~_clZ*4E zFrH8t2w-4pnpGexF{Z?yuCI(-lV|Eb%|#;z0J{Cvq{>Bw!`ss3hraAm=hB`Q-VPoe z=jJ%&{N<&+q+F{}fXlTw}!cnpUoOjvXLXY1!Yeu??x#iG~c4|3KGsZ$3>E0CUX za^4QMLf?-*pef$J5nc{pN+F7>JqFX3uIElXSnl@K2%oPJKu z7)?O0rjuzZP5ul5gK?3$k>Ff&;u$N!W2o$Z69bMNg-eufNb+^#x zzw)rnPBD=^*5Q1f|2($NhR4Ty$0?7VYl7+g@dy4N!BA8)Sk?g$0KoGPFYwGDbJGyKI+ZEJz|@ z7!HI1kkhEP;NLfh4XLy&A?MVln97T^uw%i3^$iw)u0L(689mdzS;cDp#65U>Nlz|To9RfH{$UJUIasys+C zQDBCNSg;CGN1am_X`eZ`7m<*{?17{&R05b_EtJe3OJ^`;y}nP&P*;!!vYaI~ZYdey zBPCfHz$EcrMn!2x8R%WLT81qy()8vo9#oG~2E$O&y$LiY%+Wu|P{VIsT2=l2_4C6w z8n%%UZKS6q!0G?@aU(kWBzhzHZLz?|fHV;a=uRHv%ey^vi*Yo#r1ZlTB*xcSOukT;m%nZ`55A+(le|TSSwS3*UF72K4blt^LS(N9Yk|SAw2Fhp0e1oY@@IizSR@Do;Yguv?=}>6_cjFy_g0Yau;=?@#=6 z(A7DEkaN*u$EZlMhWIUs;&E~mPi-XN3ZM~<721i!QV?eF!>sa7{KE2ZakpA@J)LVr zX{e}%nP>|VjyYwT)q~!03F>y3Im;ebrDH;~LA%T2l45RR$gFP!k)8?sBMyug6VpBm zd#38pfNJqiP=yu)bSd~MOEa)dxk;Nlj;CAm^cclzR*;h);Y7`poZDR;*eda{@t&WutpJJsl zGlRy%IVyb&8D{OFKt>Jd0%s+FFk5j@NmO)gE>xlaNHL?3#pXc;KOeVPH5g|Z>BL9$ zX--F525>YfyjpZkx|li{!zZ)CX>;bO!mgTl%h-8GpoDWWq-aIx)n5wVbPFhbUA@i3gvD0` zD|Q;RfM=%)RKfC2W`+E<^krl*%|Q>bxTH3MO*pkjaN>fnu>#7hFAGf0V8FEj_4IT* zIguGJOWT5xA~OT*fvyD9wAP@7#58pyamr$&bzoLxuR$`M5NY37QVY~4OZFNo(4185 znQj<@qZ-0_CL%*Xeh{A-z)+0yF#{v8n{7ndc7WGuT1Dl~#U{!~uJlAOpNwcj+zx;z zmg`lBAs*f~bMWnFYHnN~gSMDG=J}z%MWxj+A6ZSun78=V1&+|*k%!V-u(L09dOpp5 zcv*PWC)zV083P2DP0Jb=W3=d^<7x*2 zWVF(WPXMjg-)8SZ5Oe_25Ndg4($7y6;yaQXR{?_6bnhaIT9&%7WkLqHg~<)Ut({nX zZOZqD%BE1;XYQGbT{>A63RfX;rFipEfNFXf{3%U^b-r5yuOX#m{me4V76Ra2%yrWt z!I8u6qo~wLCMvsL_GNwKTFx>)taIZ#I{aDnt;|_7&`o!YTdb|>-g|Z!2j?T09mSN4 zB`+G9jRPzO9Wu+Ui{HeLlDwjHY;DgxMX?ni1Ep~%25SvH;>hLo zjVeKEa=Fch(jK){TCS2AylI&sxmWuTd@}<9leH=)!P73vXi2tb)CIjAGo;ff*V_dz z44;(OR>xvB_f&*3k+q3l3|Q-+N?UC%Iy5#pTA~@yVz7iPpd|Am)QWf*d^;tTV?p~h zB{Ab}7KFflCLfB?e$C`Q^T`)8V8e=WZh$wuCM097l_WZYC*r4L_z=^R?5d>vSLRmI;E+1g+y@nQVRc$A{9jjX{HQHdr&{=!4)@yV$$D1!`*dM|ih_DL9XI~f?@ zEj#d{B29#*mOYOmBX{BN)^*Axw}zdk$wGVLk=gC@YWQwvk-4Y?eH;;^oqP{HJz%Ks zz+ju(5ipk(KbM;*tmc=Rf}i%Z5v@gE($>O6R3hV;U|l0z$*s4F?M+g^nIgbx-7uI8 z1%5D1yFo^{x=(af56>JGV8gvF%-N@iV9N}z zJqwaV5{wzznByPw^J%PNNO{*3&yT(H!_o0U!SDC($n^WRZihY{-Iz;6=DL|%*P=Y` z&8M)6NnXT=zG;hqtZrSX*YB$N=&gsfN=?BE(%fCq8y@Q{TUOUrD76r5BweA_F_rrHx(Fna0s%bu=vj zfxl{>O938GQ`>QzE}#FB!oN=Hv;xA<2>dDGmPFUfK&)ffffq}6y^1nzmRg{r(4w32 z#ph9@zH|8S+l3SR^TO59;rDIy5Kd7mTFHvrk*tGVd0*Y)zg zbJS74J<}p6#PA81V4vr!eaKB$w@t{-g86+kGh@HH*T=!Vfd2(!S!<9s;K3KB2`5ip zteBYAJ)5OeL{8r0V_xXk>Ug>4ZEaQG1VdwxZSpXgC&?^I-=!6Uo-~uR zAtl|?ruSvsri3#r{<~=tOr$DwWhM#blF}>f2txjnJMTAs^OqM5n z6zC4ava(xY1*Qmg6c}(@4K__i5Ac$UX}NW}q_*k&!v4v2EM7 zZQHhO+qRRA^T)Q6j&ri_8T;{`_o|0_sj=o-bIxxbj87F?)qRyLAGp zBMR;{4hy!oAVf7P-2n)g$R-u~!R7fewR;AREDLV_TlHWclmV*eN-a-u`^-ubt@VWt zpF$Z^XgpC+GZx5M)ah{W1zY`@5|DgYBISFOv^Ta)O(BCA;X-`eYT01dGco|^Bv{>a z=X|9R3+QNQ-ko2x8CsZ@Qbfb1_uaPV#1(A$W7Z)pg97|}Xe+k*V)%4f1_q#GzNfhtd%1Z9x8BJkh zb|97X5nCXP{lh*=-|#k>_Hl_aTcW8^ z&I}TzRK^tFo@Xh_#%L>3Y|(+HeueVEv^u)WD=2NhLeddKn7z?%N+|J^!|v(u^gxt~ z7|iC!!rAR`oeJtnY1jn=vlk<9nhtqWF0IjJG#N58eNi#^MoJpHBuV?GyvSLCLv;o> zuwyJQ(*{3KR&fzOaBMVe8M1-y(2`|SQ8EU*W0GrS?ZJ|QpakeBvgM$F$mp8$b|a#d z1CPNTD$_QUWvByE$_L?i$vF$e2#y;!Dl~Sy$yYB#b;|5owqRTws~{M!LjfnCR8ae^ zjq(LV|5PCPikJo-oB~{28vT>c7&pXBkWNc>PV3;F0lnq3pJB0Hl> z$x#-ZSp>KvwUf%dijq=O8+T%Jad3q>vexGKSH>_F>*T^L^4gpia%`LXK@m&RgT>1l ziSNPtT|>7l@rXuGg)|L&MsSBOKtTS?Q>{tXxt7ppcOlh2O*q5I?HH`!m+N#&TMB+- z8v!u}-KWb;@l?k&dlt<-wjK;lZ{mS@mV7+{jk1)KEX3Z-$)#*RdKAA~!TJ-=nvE-z znwuG|`|-0or@hP!wfjmJEJ_RE`d<96Ri%lG>EL=2RACpUIo*b7`yvNJ4P-?TJuk+yczY$tuBc5;%|2FXiJ774a+SH+^<-Dl z{G6~&t1Ms(4Ji?@OV5jKURST1J#kJk4cINRBmONP*kW9>lWGYrxGSW@`mK{t=CB|p z+FLVs9IQd&GU?{%8w-uMCL}?72S;Be72j(xg#7Gtkpn+lHX(M|Bwg>uP`&l0zRTTW z+74nvtJxaNcRikd{HY^pWCyWfi!T~CF~&iWOSOo)0u6N|)7t5Rc zMjq6vq$s9A6wu4EsZffX)8YjgCDwZ1GCk=vHkOHc5rpQl(6UqEF5ZC8Vbn$k&n(j3 zZCt##nX#q}*P~eo+H5WB&J8`>$D~t^^`ZtD#Y-eLwPx06!sc$@=hc&mthVFMSHo_h zU4-sXZ|P?vdcc4r@1J>2;?`j~FzbRIZb;8v@%Tp=IDk-AFX*oCUgur*McXXsw=}?y z`~Ka>D+JgeTm5ThP+KL~Bq41D#K$TDy^F4a2TM+Np1;YD8DOtt^nO0nwK}xYDsjTQ~_N2-B@q4yOqbwXb^cFVL` z#;{0D%I=O?%uf=0r(JvCWFoNvwI=!fRvzZ)`Wk z*(I^PW4#o6TiMYf>~9qnm{EyisFKLdWA>-AQenRi9RrGKcOsKEV(AA5KC`?WWLkJ$ z2_C2(&2FO}T1)*W@zv1QTvx93=}%uXt-fR^^2u;9qQZ7D41P3F&IgzTR1fhWq~qa& zAOAHR%jpa^I4etP(-u}qxm zu=XIsTnqDbkMY0e9iszc1kT)##@yVG!dl;vMhvUzIZ#*^B!Y-mS5O-bfYQBa4G5Lx! zaBb=~a$x8my1_2I(ad;RWHDzw!{i>=Mpod1hWwSO_%HLnSx(eA06C|qR|54va__$c zr-xv~9W*wCmPQM*R?p5D-_?G-Im0){v9Q+^LEA1BRRvuQTR@)FF1Rhe0S%IUs~P0X zwO-F~!Hd@HhFzbx1xt2on7mAeQ;b2UDa3xXxGK@ry8+yWoS)7A+`&|N%uN|HV=dqy zAFL(~ccT+9rp%K#&_giX2%yxPZ%1J0MHQ|joM z1m_e_SuL(cI|Z#GohaqB)ImZt9=Z`fwlVu_y0_Ge4c0`%_$k`@ub{A_g~4^{1Z})d zjk})J0Y~#@wo5_u`}cph`LMPvP3h=BK=sQ1*EatLl=;8gd_7weXE!@XD?KM?J4XYv z|D=I0Idz;4*d1@)P<@t$;;}PV-2XibMAX{an$kIp7}>?}v~}G72QiY2m;i(TWJ68& z{ZAQmx2#ylPDPh)g4>-WziVM<%iR2{%Snf|lC$ogDfP%Japc$o3;Nglzlrx7vnCGj z%l$cucF zb<~wCte-$h2a0_H4FQRNza97j-`sJ18l18rdkL$NVt1c4FtaXTzFQJM-S{1@MFeE;BDQ(-=hVxAf za?=ObaU5%7_Af@Glb0RGo*?<5^+5s=+Uz~B`}24=b+<>GI6k~R{@JnZT<@MB==J%w z_jY}F!?5Y?9n3tHO?6FWU1+txGI_UuQGdK2J(Xd^XVFsipJ*a|D*}_V7|L>XiQq8G zi;u3*LFU_%O&v4P#1}oPs__39-+xcX9+FGjD7U3fz zqXsZuJdRZ&pI&S(T_oSeAYHaHHm`rJuaWyO-Ta<+%Cu9gJfe zrVj@uR8=wX>6$uMlq?ZuBaMZ%lK;9SUCm$cHHSWQy0reh&gF9yZLQOPz?h&kuc}zw zG8o7gwlHrP7JB~JG~v8pjx!Db&|y@^Rts8?IEx{`vNuPje8@NBxmyPmvI$VlAHHA! zKC&H>HZBQ7I?Cj#&=m)&%%*>}#9<72Ct9Xi5~Bk07Y~*VfWQQ96;!S0;v>@%TlS|3 zq~{BE)sI=LG0o}9mz0=WkhHHYvn(iP zQ_i!%n9mTYWPd|z6Ry_O7OJ}>uRjm*PiejRWV&+pgs6pB ze+W?ZZY~OOai#jiH;hsN&xy(s`4a0dS$41V&rZWljxbk>qtGbjah$ZE^8A*V#26QJ zzf*mc9WAdGOQ~O429H3$1qJ(5x^t%;ko`8e``oO={lf-_#hTO4G~c(r<|72 z3Tl?fo;d1=Bp943(QXjL^t=YHRV#5jw~phGI_}_>bXhvs7IHA z*sa%&Et@3Cc0;>paLg`YH*%0TQ_l$0K80j#92%OP{^sCVm^PUPM&bs`F710rpkqN! zxOWYpZa7c+Hqc+ep>)I`Inz2Tq%muU=gvgfMo2XB8z40s$FrV13s+e)2FMrhYby#6 zR=C-A)SMXKJ%Dd;w<4~Lw$u=1jbBWcJmM!?l+>qh=v8zSfoZY37xzbqpDla_>FPO7 zD2KSt8l(sgWg+xm^{bG&&6BzlExHheeGooxjYtYtjMKhr8;utJLWBLUF_lF zE5wMhOYgjJcOW1;tl+M$^P2S|>UgHrv1@!ayS+o#7H?P0Bz>9!h`%QMBLQ#aFD+g;|YZboc8C9o3r6iT2QS$t8P(-R3)Ti=suh~Wz-Yox z1Jxh1Q6gft7+$g=sD7dKTRVRQTc_-e(vrEO1QTWX0urrUujfizlO#u<;e zfKgL;wV!{2P`9O9M)v$O^$kzlS} z*VdpkM?Jn4A4 zy&gTxv~j5q#KFA4``!!!WQ!HK^fcf~5iIjoT<1ZniXQU5e6<=*k}-o+Y?v#4P7%-n zgVo#Qe53hN!=6vP8pCKyexHmQ>d-+HS7fpg>r}=bno=5}2`~x(?$8fVbEc7A`Lw0sntSLuCQl7L3!L4M|{b{>!IE?nnIhwJ?*?WSq z9P>kME~`G(R+w`{2Fnr*NTT2Fwv zbC~i{Mvih9hW{JxD%k8Z%xLl1Juw)gnW4Mk;Pn`_k2WwRZfMDY(EI{eW(XsvvPAU? zjlxIUI0C-lo1K6hEyGhs3fCc|$Wk^8n+4A|r@|gCuu)i{0ZbH%JJGE3YLoQcP|2YT zy`Q4n-YB;;Qr%FiM6m^GQX6;K9>FngtQHTQB0wO!rJQX^sUFqclv9lilXj~|=;i#R z#_1moI>hNB)o`QBV?9WLk+YtK2GfhtMbNczY7~AN4+n^2&ljT=d$Sr}^~2s<1zLvW z$!9s(I}&lPeTESlN5nL43IJ`3b&DB>OKuxZWd2D*`?(F~iDiLxKKSI5$eQI=OW<$eTFzvs zzQ5ld5cK@sHtxR+8LLHa=+T~eis^;E0g2`zdMEc<%0W?)4RdA753#UMg-nkGDfW+Fxv7>Wous;=ZL`5 zq?_k^AJ@otYrroXEzQG|v!E$m1s#v$UgGeT2X$Z83fvNSu+r+XNdIKd(Y>vmV_RPKxt)NVo%c%y zalaje(1O&q0qVe!D?KP(z(8SHV%!eIhh-wdMc7Hp2*&hBCrhV5$Ht-~I;pEQK6j6c zE}yy!0);Aa9r|C1U2$}**)`#pZ}|TzvO)U~H|ZZpK@-*grO197n}D3GT^wzkJZz2r zqr$jv+z#3ucAnIx7kbh+$D8TT!c=*54w`zrY+Y@}_^v(Y955rnC~GPGf%&H*3%`31 zK>fi;HCx^Od3ty}h3O!H9iy8vQ@t99ql2xu(Y%=v(++k zHG5wuv6G+D+10WWGd24!H268+P`=z;T!a7=M@jbhu}JDS%rL=-gknikwP0t+@pLkA zWfICpqJTf89=Io9j?et;wC3=rdrgAnGbo6Q&h?-sjMT<74#q(h_6!uJs0YY~xoV8) z!@OotDJ4EH6c{aeQmH54GgriRs^KB#ys}dF?BncN0$5T+QJBCFle`E5pIM2Dqn?I@ zP-)RAgR|154gybIRH=)N>1%B04cV;o2gT7L1cL(kVUMsK<;mL9QwKx|LhZFxiLiS< zgolW;)TCMMID_D3iKNvygM>P#%NjCC@Cu}LMZ#xB#O<1BCArnS>mFbH7I!466U;<$ zBo(BX`-laF$Hq3&e8Q9Vh1IDn3IY zrilb9(?(H3jpmcVG^lgz_M?i9q*?zanoIwYEF)M9jMyVuH>&(`)l>7-UMeJFwmFoQ zB$5a&UI=FHB&gc(mByI4JmH8OrSVJ`sm6kpOdVRIQcwT=ntQ~t=gWYhou?<3M(!6G zJ6mf2yMu#+16zm?R%W(JZWd-HM!vB8FKktePgqSXtqgs2VPx+KfkBjYur^vBzdQ0a z`c8v63Wf?wWhvC~91fzKl_Gse7z)sag%sVaE*@%YvInDRDwIz{YFNB(6*? zeM%7;R~2MxIdNZ~kNLo4r-5v#TbI9XbR@-r^c3yi*DS9Zuo}3_GO}FG=8G5^tT3U` z5!k6%3j|eq7aa|FB1qPZr5>?RR@1$8Jj*U6eN_<&`aVEPO6?1lZ*JBjMpjNYVlC^d z)aIlL;)0>fTGZng)U-jdCoNfxI-@<9sH+m{3#~4cZuq2%z%9D@8cRs#mC8;0`*vrn z(4D>oBWs%S;p~b#KxuSHf)*Xy10#!~oya=vE>KQb2elV98TFgcsCJ&0klyW4#E|Mn*}1`_+0 z3J2j^lS#t+mrA|o42Pr8O$D)`v^23WtEJ&7fULZGNJ7?Y=^n%cH(x0M1N(O!)wQY8c~&l=w()QQG2 zFnF)q?qTKzGZB95*~($yxvj-G=JE zB%ogF=Z7e6tHrL_oByhvi<=WuuNGIA1p1AJ%N4RLSP&gT;aFL$fCR6CNp37!K76E{ zAP=u{j%-1T$+n)Xx2;C_S3X=^wPi(J29eG*C9}6VYqRYYxb*(~H_DQ}7N(Otau^Mr zWa40Z7XMqdBA&X8QnJO`q^ALF+r}l9n zh8j6Xg31gTzCkr_@G3gn9!)Pf?=rQPS_xtlM3n@YPf$=SOY*HjYJH~UuMaY&^IKx( zr?lK^OFLsojKDH~58@z`5`}*zvyogikDh*=yEToX8Ul+HI`i(TB;(wg4z%6}j@3KZ zF3eO02*E?q4j>XV;q`VqYq!Lt4$u(Q+o4aG(Q;MkoWFYQWjg+Wsazjk}tBn zjLXRyC*M)5L|BcA@w9**bbEHQ+;3v8GH=F23ryFvl~~(GSC2DuQw(Q@*{T```;I^;n?*y&lVXbC>UbI5w(QVsNdE`uo5ZQe}astBAD?rhl zu`I(9;+)Z!B`{HsRedxP_}z0>oGLF)ithZvsIA5_Zb4&!1(Uv9LQ|R8KO{|XBqQR2 z2GdcRBp$tB9Szh;2iqHDw_&pp;3iB&qOr`$EU1rBn*Nh!#Ayh|o_o}GosFq zr7I8h?f8tQ+O=KZBr7eUVJN;9Ta`{^(qauUIbN^=H;F>XiD-@>{uE|iM0VPw@6K}Lva|iWvQu2cGQBa#f4)U$5X9O0m6&<&R|Y zHgdye^UQAG>@mqA$Kpj1Tgmasoh){q&5|=`wt91ys#dufv9Cp=e6~g4(ZWMj7Y|WU zNk-ZD&NTfPwk0s8BgG_h4R%zna}|?TtQSiwz$AN3fuo7V+pc$pKm#wXT+5fd)Y(%> zLkD?n!nLg=0OUvE;NB*ro*>6hA*UBiyt`ynvdRrqSFO5aJKxOkw65FG<&EPOEu%ZK zxw9x0K;Wcj@wvw~;{u=r;p$VCv5d8M{Sh8=-k(PSI!vj6kH~_piXE0R>d_;MXiMq4kph{5Yy})`xX~rkGl!mD%p#j23CvT@mGM5^ylA&V#PMs70OiY*n+f~6cD#CB{S%zOywgd4(_Os z&|@r_=q1!YxV#s8s*4^_od&%74cc&Gm{JpvUi8!lJ9%f8?`%84Wym$un^YC`&~&gG zbxoNMKmoV-Hj90=48V5raeUo-yu5uoRL^VLQkR>PEhVX$3OxSa707c}sYZXCJh(r; zl(tZpQuETeZ;|h(CWA^5(FQq4k-SQgbQj9cc}sswQj#4`PN|S|vq{UL|AA0bjeh&? zX1{Zsp|~2z{Z6$H8~{xv3yC&cCez4RUCCtjBUa!4t||cG4m8P5Qd!kT9kkN|>FSzC z(4<_UIi3U`9=zoZ29X2r9vEGbqj9Cd?l0JLB_)`}e2GkUo$fgeO?&(Dt2FN-TyM{; zCW*6(HJzDk%gF+UVG>Vs;l~%9aLkB3}d@iqL0~6IX6%6o)4IXw3g-7;GZh z?J`#QAwG4-Q_5@g@0Ln~NK$L8UVATmg9KdkEF5C>*8i@t2Ij#K4NB28Au~8!2WCoT z(4f?i>;7$1e$FBF2l#`U$n`rIpk+UG8gj%0*#h}-jQr7SdEE#1*6=0A!&!xHMBXwq zb?u>@YK@09`#CPd%f_X?+u@}%H7tLY8_4dO&xX{fd+C|dCQ{KHyWB|loV*Qr|G6K( zud-Hx2~L9o!ynh=pH6LOgoQ_*({wuAKNiU~Vc zHUBh@I?<34#U#wncT~&GQvTBSFi%H3@+kkYgKTpvH&4M+ajrlCFvTz%>W){`y+UPU z2gPa=N#d2v*(0C?#)cgWcRPEru|BQ)EUC?%ilz{&7P5k@6b;@eOJWst3-?^kLS$X$ zi)qi3ST^(g-LPFSK!OM0YkJkIPj5l4vMJ%O3$Jkjeiinq6-Qg|K=sPl$R2%jwtE!D zIszO(Wumuw_8wzr=eBcuyE-mCOez} zy~90zRUnQDn{rAlvqjlpqYfrcdk+*X&rzvCizHL;g89fOB)Sg0SbYY4=b`T?-fdld ze`tJo++4Q-!&9(EJ$CVx9e3nYdJKzQ=nc4r<&%%6u;;e+>k}te+-qH7Tx*JpPJc<0 zDPhE5%MN95Uij$PZ%xDXr6e^b)?-FA{y%)4xn&E)H7 z`8+*owOJEYr8RrfwerR5kva?3><>1RnL`PQ7Ri;7xBnEbgm4thN`Xo-qaix4t&EgQ zea--jpxZhUNx^2D-SJo)5-x*I{7Dp7LrkRaeBm2M>cB7t?TwyUX~qUQE_j;JHN&Oe39JD`=l=F4IM zKS1A=<}z(kZH!Ez(R(a|v)NQI-gvwED5tvU?#JNsuN!?kdU_pseSdJ{6GOkT+s?@q zSxqA+FXBxz)qX=1=E)lI20mD6W5!qt-*NTSOkCt$(906B8Vl1{xWE~5S$Ni}05tT1 zs!5K~F07rGR=IC%9j|RK)2Mzln6i>@fW4D`1rmr+jx*6uO3Om{zvnf6w z+ca-4>$}Y<P?cvw?Xie|7xE8sWM{>stzttbw&C zeCWiWcYahWaPHsDhjFO*Vhty;IDFKL!!0g|=T^f1*sAl}e)nws=i>SKgiut+7v0;t z;R5nmR#JL2pP)$#iEkB&m=wsT`@GN?xO95*lI6!WkF3Ii$G z4ke9J+H)e3)~l~UXh%`&RJLYo2`bLV&bM>hD%Zacq$H*Av+noJOS#O zcU#c$d!T_NxuCA@B^04n3+cX^T5WBhY}T5AW|}pfX^GtVvu1Oq3EHFpq?D=zSwVCPYN_Legvi40dE_L+>4P4f3_)8T$Bfm$d>d;?5vrzpe+rEC8(-ad*YaVKy! zp*OK)C-H#@Wv@G*GK%@P{D?I{{UoFFDt76N7*>#Hk8N*@|FC z5RVzY_71XKa2)~It<4Vj8=F8~R4eFKU35?2p014JIPO35xqO6^GJy?-+T`R~n;89$ z(a0L_PO92xQ=3fEYQ94}LF#CkcTw3ln(nj_-oV_(4qwQN(N+8u1l&`eKCB{?y!gGF_ks4ND7ym#Ylov1(0L z1hZ}`>P3h!tu9>wl{r7m73VY0+>Ncy`!m{$g z^yH$@mHnq5WbTEnEzZ!lK(&h=Y18qr% z{uKzW=hTBMBgu;7XR%U*YWtC18`swV5><2jYoRsDqAQ%)`X}qQi4X}(#C=f+X{SlN znL}}ZNjM6=$6)K5!NGm#QA2YCvV-v+WzEx<9}+E@D#j>7ti5Z6C@NA(6m!egHMXla z?0D|=?x^Yke0gv~`Hs{ZQwUV_MlhVGjO>z%8rO@i8q(7v9H6OBSOxji(}B)ULr~S~ zZu=o{R&PAvb8NOe&)vtz_h#gLzjN#M@AdLMzeyQC@S^jrCs1Yn0c+P}cSawxuJvQgUWSZ<c{QeHtcg_NamG* zIB1P11LH3y{)BTmu=wxAtOR7V>O^nQY*8N5SNnqSlE zL1%gv(oNblu3b9Q)eQ~GBZZEEo(>LQUzd(9tbVwC&04+T&@B)les&#$I4fTIGD>&R zHPpe8SNv4^KMl46;*D-)?R$mc{8KUOj3L{w|Ie_tBAvagHTER@AH&+mB5HP>_sgBL zOK)U1=7*>6!`b;1+=r*9kB_H^lW%Kam6f}n%j@39@yEF%xElW6<2~EU*~h&Q*tkpt z^lgKJai4PkL`-_poND-QP}aYg#n{s?p=R=o3eDC06o+UjyM_J`Dl;tLN}#_x*n%Qg zHyMm}xIt09dT+egP~>tbw=w2_{62!m8wZ-^&*A`QC04Pa-O*GL`Gof z2iBr_OqBIpBGd4_WxPM@tfN6HsaBBfT8c~pWs zR(JG24!p1bheeUfBkYO*JUj-$1e&s0fTizQYFxO;tthrK|JW_SLXJ$z7rh#;GJ(J2iO0*Th^emQ-SNfo>Dh9~1`r*faDOIDsc_}ovSc%F|m!G@=sY?7Ygq#dpKYi4)OziS4)pj8|0sC}YYU6Qj zlUzM;i9NX0)3>`JY`@IDgyg-O;oz82O*O4O>O81MB^J_=a_6||UyLp=BwdlFx?J-q zhwb8qKKOwP8z&Waj)^JdS3_i@R4>>nvbD-zggBr&iVNVPTG77E{En5?Zey81?yyNY z0$DM44F{<yD!w15^>j|bIKaYm;X&IyoWgKdNK0=zIIm$qI$Ka8^kJ& zY0pdZ(;I7HwEV2uVXg^^x_X>HU)k`rvTG$2vgTGJP|&)>=GeTRv%yj11Q)k(M*V_j zvq(WYXWwt*+9-tnqiPy7M#06klqVh$+a2VY6{QErJ^nb?TB)j(#~>)`U#>gVE;L6{ zm5mtUQaX|BpN(sTb1=YhD7I)X4r$slLZpLb*jZOyt`$@(w)ZqyX^b?NhFak;UiMlQ zzr_vCFK8)4;Nzm}I#0X|-V;BE(GCwV9|dMH!!fI7L5@$8nimn7rwbaSf_B;YaL)I^ zAejB-+dv02OI}`|M9CtN0)Q~U8WvU7%-Ngm$Qx$;pwpRB~%z%q<_9vevS-Jo*fq-f*72{ z_24$pxUFZGSG|4fAdyTFJx0NQ*4Bpf&q7iq*;FdUw%jFs^9U(6#a0gEm^8o6?M9|L14&M~=VS z`Z^pKrAv!KqGB6H#>>`aGfXi8a+PZ*!-8xEm+!Oozshyc(&0vb|59Qm3wi8}C1UQ4 z#m>_XS_UWF0o4~(ZpeOqu)fEh~XU3Vi%;|KD~WH8}Z2Eu{iecmLBLArTe4jH=l|Hke8*1+oZRy zudsh%y56P_l!nEEagw5?-`Ayo`Pj8+bB?B0&sNC3a3&2>XuKl5bS87Xah0ESOGai> zqO=+_ua8us3aFka7cqK?+bTyPnI}!HkPtV_o#f38uJz>UBRj-qOf%kUHD!G7hP9eQ za&b0~ax@fD3m`tT!!V!OM{pR+@O`W=k@hbs*xHcs%oWT+Wo7Py2@!m}dK0V6NGG0M z)x3AS}&Nr@GL{lyN#7cXr#6QHh<^RLB2XwBV3p0s%{k6h1X z#XCwb+xg=oPbAge_w#s@8dd%_R^&myxaf%_K)WM!_}X_ep0|nUTOQ?JD+?+fHijS{ zeSz^PiS=HEhA<9>IRHc9W_Q(2DC3w&zj9tq(<~1!9;jw?s7V>65d<#MX15Ele6$?a z6!ow?Q$QkLfur~&kASfX*(YDdtt45gkC9S=3Gh&zpq;ZgMdgrA`A38$<0k8u&NQzJ zy;yotO~Qi<7AvWc!ZhH-&sJ8Ir+RkMWaY)Zu!;cnR%Ndp7+2~2i@qnKdAR;*ePkX_hIN=$}`b3~h1l78KT%%%s9BZSlw{M-?+E zvh)=zr?Q~P_HC^O;%3pL!XdG!ZEArPP}L2gVky>XyyiXQ%JyQ;Yq0EnX)?GVz&Q1g z>_RRb=`jb+hqayQ69nZFvL_RvlJsKR&qIi}E_~4o>PRX3HdR(?m5@3q>~n6(>?+(Q zL>s{)A%db+TU_rnol0rT7qG!~i*=@$;uiWDW?s7ElpDA|$PIqrx z9{n!9@ca69BH-3w+lQYW|0nY3AiMazGZf?hLeeQ$qs=wLVXLl%X|+kc^Ihk?AAT+t z*6uoAr;Xc?HyYEtlkVWT6hTezS1@^hZM<1c`P0#sHQVZU>WtfdBt(Dvh5Fx7lM06j zgaH}|2<;~t@&9i@`gs7z$;HXu#Mby{&h%E>vOVZT>RnZmK8h0eCnEQn&qC4cmuch? zRWidi>rzJn7my;(z>YMQpo+yl+=5AbYgnCF5Rry6JAOY76Eb-&>)6Y&J+ZWFQ=HCI z;GScXe!S^BO6wC@fUParuv)REC0V7co}yL#ZV%|{y7(}E_`xdW?b#`^;uug);-F;* z?fl7K$oCzkiZfk+y;hGH)CbQgmk-(^qCuf;W4|=MrC$n_0M5II5ziLfv>Q3oE5y7s z`9-@G6pzLkq~A8pKf?~vn_N!A09;~?YNnhF$z|GxSL(JO{(Y)59#bL>6rxzkGyW!z zQ2rFW)zqS@Gjr0lMOT(p=G_A!{{{Tn9+=A#dHQGLJ^L78$^ zvU3kI`EbrjKXv-^kFA;Zu9J&d?{Q_D-E`YTVTOGS+opzu^<;SFiS#vH!HK z0%$MA)r%C|bjd;*{7SeOm?b78W(aAO22^%rX%U%)oVvPgd|KUr z2F{R&xmqNU#KngQrw+Nl-t`Y}gk+YmmKQ9<^rt#_nH>rX8aOjDiN^sFe~?u)zlQ&! z6)Dh0+Ufa zPglk|_QoBw%WX0pRZsSR^D{oX4q9|ol(Y2(#w2rd%+zWXTd*2{O;$t15}o;!-D1*} zhVC{_P;WZRAj>ULor$scGK4+9p`ElW(Y8orkG#rFkeQt!2_wj(;1g6I3Vf(OdqgcN zuo?d=Cxk`0#Txu7hw<6mOKu-EA}YS2#hT1uQkW`^6%%*QIqk)@m()clovP1!1}pi4 zxs>eSoWhYQEfkl2)(@>=00A`t)FO*pp`XbVRU`5D0IXm|k^D&=)ZOLT4s=ue16o(% zqAho!?#=_3EzieYF{5ZV)-Y-EP_CU#B9qf@J1HeHMZTNqN9`lsZA{bwOC1TiY|HIG zLUf$7W@p$CsT|Avr8t*MIExlkKq6;}`1U%gOZUUB`aU3^mV}0de@)`VKrP_Cj)Yyw z$a*w@3h6_*KO^C(*t|kC4Oh|8UqH8Ayek_B>caWz2WK)~8i#LqH-`Tw#T(4>DphK7 z3<9-AE@Vvf8>RaE?e4@KpCY!Q)Q3!KGFUF3`iVI(8$)tZN9?MVg5M3K99YS+^pYW{ z^g(m%qW`lhHKx?YT=K-pIs?<&U{goCv57Fj*YV|sTcL?;XE8PJr=Q5kbgURJJItUS zv+xj}#iAk~PSvN@hxSd^wFw|yfG6#siZ9s97Au;ww|2dl@p)ZQHhO+qTWgJ5$fh{PIrK z`~kP>-dm^EK6|foK1*0yLMD<@gdV}cnGx$FO3M>V3E_<>=eVH^>QX=6C4_cLFI)?) z*4_5o(IbYwEZvssPcO~wHf$czmOJvFApRZre8!PtmQU~hJ&=Hyv*)_{AxrN6%pCsv zDuMies>J^VlQfLmjoxEH9PECtc9EWeCJ{qCD^}6ejw)=HN6LU&pFeXDAj67_jksK! zGM)|l?u5Q56HW%)a&^DHP7ZVzxG$aRyon4D4Q;H%@3;L*kv8ex3;)|lM3a0k={QPi zCX1$_9V=)Gm6~WI_&f3M7uoTu=|mi5P$z*Ky>R}va74VBZ34L|%8Z+U>B@dj7+*E% zD$o+eNqi52p;-^eTe}43b63)Uq0}&o&RT<55%>vN20kbcZ@f2^V^}hL<5!VDtA*QK08; zumJen_$Z->Fi)EfK)=n zkm2|t-MRzZcE_dzw|@&0=W1 zb^SW9bYQLNW76{f%t(rej1y9kQX=cSJuOIpk$=C=#nV1!@N9avxtDw{xfygk3N37u z<&wuW8qR9%w|RcjZ@?f8qWbesm#H(urpUJD0W09k{o*b@Asfh^pG=nQ$XEE=YrAQ= zzk#_jNdCTuP#)AMR!d-ElN(C~&20ir7*2UoFN*>Psr%-H7{hBkjcAq_m&@Dm=w z7eX$L3F%88$1J4Q1Uiegdy#69xzV6ezE~#C&?HTmuacozwv@ANADnfr#k#(lN*P_M z5c6O^Mgn)!03jZi@_Im76fz1e;3pI}2c&`7M?K24KTDbb^%y2*zKHmzkknnmkyI1M zO-dOS405-o9seyWl2yCp&u#b`EVHO|QU^PUMCR1E1As>t@^q|p$Vd}C z%L8ow=bj=uVC8h@IUdrR-ejlPz9zS&ysxxm*V91Ujv&uixD+T7?Gug27*U@s?I6M{ zKvvbFO>9DjF_Ku(D?n|eLYH*Fjw4l|>zc|!zJ&~iX5e!ll_tF$pb4;9v~I3iw>tl_ z<#5N`Z1b1->|g@NJGPfNTnmkB=pXg#vlSr35OVft>^Lxx5$8J;=1FAmwe+=g_zFe< zYd@->B~Pyl^|Y;e4WjbY8e}_mPY}Lpx#bpr<-*|3c}I42)5Yt+Nw^xqjw09xf^>Bm zv&`0%Qzrh%6q}ktEt)==mpoYysnM#p^Y+MF)s#e4#MGM@x|>B?nwkR!6U&^8E;ocR zXK3M#n2=iF$B6!zh2xJ|HIue7F-|fsF^XumRB0-$ zD@~`BqOl#quW&_i`(6{*_VXvmE34wpUj%VXzWog2{U2E|?pUWx{&Qku`Rp7>CD*Ls z?rvkAghnhJ>5@n&lLl)G0`V^c11lqiYY$IZI!|5rijz|XioC3E&R5vUB3HUfZ9V1i zXUn>~@BsOSKnV<8oZ(&iClkwRNY?Och{w!I&y?yrhe<^i57rZe?~T9I-^gl1Lo4z; zYqH~I|xC40`(uCFcxbHynf zvQii#7~2jX#OzeBRU!82lo{8%9_#n$((+-Q=t9DO9yc7GDcgmuOr%FDz9IipIM)sp z4nHA)fEs^xV*h>N{4eD9|0WK_=) zk#YNJ_Gh_%)qR$iTY>ez ztK>9FCGmuG+V!_>WyDitCfWErXDTbona5 zR@Zdkz@@Q4A2YD|J8a3*vrTtY|7ylCaJ>2%ye|9v;$iFTY-`V(K4t43JieY@zWRCb zWE{2b+|u&u*F9}rCBx6LbijW;#OQULn<_$;QqMjM>@(-+BOusH=Lnx~xmXq^*Ghh; zvJU-|-~&%%>{BPq5HKr=r<_1epEefI>0LweOvNRGW2F`7dgX``Pk5z%Lh$Xol+YXt zlJNIM68-E;nD10W^H1@r6MkC>s86F)EWmHTVQa^()I?UxnpMFn#ts z_QCsBx&YM?_W4A~b~|x~fyAc-(PLJv(9S^`o8UF{H*t(gh8^0ICQE+u`RMxd^=AZC z1l9WXW`ABEkDlT}v#@m*7OMeBBbff+S{nC*LNlrfBmo*N9Ce7V_>Sw3FR9bN-N?f0 z!fi&1q6cFsHgT}IT!hxxlL#y$$|&nwoAyK0g{vd6H0x=SpX;N7`oRqDk<)9%NV^B2NiIZ(wWWv+RR@Aw^U5*VZUC zK^Y8)_Ju4(-rEkCibO86L}9j4Nsi?+*;sBu7dCM$C}o5=As|LAleLGp^#QHjTB_=*R!D(LB)-U&EaPuXRWQN?3*mt)2OL z2_>I;Hqdtfaw17e92k2G;@pMzRpY@0)KfDi-RCMDx9+MLZb+axGvdsItqLxQ^2Z30JVo zv4mJ;k_;0DD`#Do?z}CAJt{iPaG?=eMcaM9Ub(Dc1oA_QoB2$hFyUgSLGN2fZ zO@H!w%{3L8x|v92494ZcS)PhgRfrbn)g#ryg0s;t>r6bK+jp>g19bc~u7p}%#+W6* z-qA2Fz&k8)ltJ$zk)n=GT&Rq=;jNRwj+}S!JMJK~PTs?)&I5ib8WOq4jTF6v>PmnQ zR03NwQgRiGoUo)K4V;3~JRP@@N+*egcv&d=Lrq^0#AjT_Ej7ChxeKcImAIv9vVZH; zE-eg{j@^!5Zk&&!J55)0G+utdlo=3%pb=|d^U#=-WLNQos9b1BrV^GjfIcKpK`Rdr zz`S0Ppnr4&GlA|#OUeVbJMBm;U;7=qm>Yj(>OYH!ed`dq$}r?zG`?zw3U_1c^zL-K zi~hTqm3990^1?iH5RMN0eth-P@~^JJE%y8?`G+%0H?D61{ivT)qS42Y07GN~qlsi5 zF;JixP-_^2`UtGVi6iY&K%*-`d7(}|QG0Au`YC|}K3vX$h}&*|GP!V}Z>RG{oaXM; zwr2hW_V~oum?-FmUB31UOhbOf(*Z$3U$=s46AXM{spC{eRA$+wamM0Q`lNO{*Hho- zwlbl8T6Dj!zHSQMfVZZ5NL^3YG zefTw~sypTvTq*bJSe&ZUNK82 zaAwhv`bKgt^zve|#%MjQOQCsmsDFi_0pN!Z?$E$!ffa+|ZI1L21}OQkik~#TW~J!#HrxMSZkT zzM2FE&T^Cu2>a)oi+*%-YnhoZ_QHqfl~t5h7}{8w?8!?4xb`~Wgl(DhP&S}4+mqnT z^qbmf$uA%5jVf!S@5~+;s0kjr{t25uJpNh}zsA-@n-L-39>FGCU5=6_$Gi(FZjmjQ zn+llG6)FnOy0PxAHY7_}BSwiFWgW1g4UD0ROxXzV0j^>z-LRxE(OAH*gG$0L`xb2Q z4GfQ~bC23(yz2*=m*Kc3Qs22Z;S!7^<~f~5+#gR?#my&%u(eu}QtDZuD&qV!vpx#; zl&@BlO6&s|FD`ZF$>D3#@u{0}j6RF7He6I3Gxrng_1iE);7G0S^?m$K*TlXDj<_;X z5oC|W-42NrF zqI&uG%`U@YkgRwp*&(#|jF!vS2ONT>5kWG;*z3&ZR65|*?`$dQ1wWxfKdmo*AV%|i zZTRv%T0?t;W`_bS?HrG{7k~6FuZm;KYzRjFb46vMy(hZMu0S7d?<@L`C7IsJ4@$QqkVEl)vV&P10o-mqG5?7q zsj^3f>G=@>1l}GM6){V-u3byDpQ8M;i3W18ar`m6Iykfe`)qO7UucbE{cFxq zKW1<-4OfenvoU>2mP1T5KCB;KPGQ|-s8F3al^G!B+lwMD?#HJ-SZ$!0 z3qejU$@qN5{*@KE2AYU-0#=ALx9xkRAYSA0>N_BI>ZNgvvBX?9?Y8S+ zM5l%bUN9sFa}0Do{B(;+0TDYs>cf9N*VW3S;NkJ{@fD6zqziR^C*aTEdT@N&*g3h@ zebaF}P%P&O{o~tTPP3rVYXvV-8nthtL6Ud^QUKMo81Qo#0p~KcZdIG^9ZB@m!zOo^ zv-!uYyVO)%5yWZj2BZLl5`(MTcWMOk?e5{~#-4rATV1UB{rvg{?Zw^I!p76i!QC1# zjd8PkzdAf$K0j@eeDg}q#`6y6#l!v126^D*kA0IqH_*K>1;>V7P%k-v4@!NC#Q<&J zwa1AZ7Yk7pm*mYEXMT%;8 z4>t;p7b!4KYo%vnjol$FxsDFou>9=e3-6EMHqaB_Nc9Dj%+~KN3nw^Jsau+4Y{lzx z9Y%;H^37?Uw}J!Ci-zn2w@5dRbSZ{4R_ES&pW)(B68hA?sHLJPcIzbKJT2sg zliA9<{0$VJlWaV z9uMQP@indWzSFqsg9-OJKfK8;Qc;!I9hwZnNC_OE8Ln?4Rmym8h7xW3?6&7G_Sa;YB|=$ zH`M6jtE1h*oo%qDT2BJT{GH6?&$x;fRhOM_EJus)SOgEx^v~$?l~P*ul9}Lm*&A7r zJC43%gY}h-I}FT-BqQ96Oh6P&WYdyC$Q+%lMx`MZ#)k3gf?lh+5eG=?2cA7=UWyES zpIE(2(oJQz&K4#85s>uXNWU7D(7CuL2FU1w$eZ)ioP<_cW5{SB>j;~$B7=U?&XmAf& zLv-P3NvmG`GBUa8z~UnwS>YI!b(XiT52Z2+B6?Eza0^4499^8uYtrrUt>O~&sYn&~@)y2jnBm>v1*Edjv?4aaze*NI% zbh3zlqSnsbNy57VWw41m29i}LH7S~Areo{NhnBa5szJS@2%xE)7?tTIvrM!)LoX0d z!6OX{X>^kSD7=2&QHFA^w`6W`H}(UjiN|8rQd2KvX`mU$V)M6mWNu(9c)70-_w5(z z#k%Dwq%1ccQ!bm0}}q1ev9jwsDfq((~^vyD!LPYxfeU)^g>F7UpR@uS>pBXQue>r-zr%&)Q1@57B~}UN{?z-A5wtIGny>onG#SGA#F}{Z9*c2#D9PCAG?I+Nx=F1dVb~)F6b5ztrZ0w9V*Y6RhtF1+ zN{uIMzYkv|mf0FMde$YZE2guzu?l`fj3md*Z(ekp2!2rCFjapO<&r0&SRYn+-|sc2{yh!2x>7>;mj4X(Y#NG*$dC7iAbZ*qCT(uvTjMS(l0P99J((&A_+7}vC(fMU|rs+FH&E9E+>tyuYa-eW?J_D>@}X*IpiT89?mwkd|xg^awV-=7)NS&U7?# zy-CxlNFb!M^3S9}ZjHS_Byl$R+_jNy2U+;tLxkK+s;++Cx#Wgq0l8q1fbmZ=@yKEd z$L8#g0{7`6<%-$%CP=6BxP9@FO8?c^(B%u{!g^JRb%78&sf0xX0|TvTod^?wZH&L& z65K~c+4FU|XZxCWu7MLdUnN7lMHR&$sd#hCbFx1v2Bf4hi#GZgD+4ZWqo0pRZ*86s z7;j%M_dH~w)^DbQOssi@D$c%JvMVP^DqNCadt=sl7Vd*`NAocaRbTmdG%f!Ho_bWR zbS9dKog5>2=5#~&(5O1#KPV|z@9R zSx>-<(FM3K$S35=4I3~&l4hcv3`mGJVX;(4tF!67cv?sBf4kf|JggvMXGtTnd|$Wp zuVnUJhCRnwULFOD%6Du$Wwr>6|H$X{9O%YjXuir!Za$+9)(P{1`6q`mmLQJRG zbEojHT9b3ahV`<)wnawb?xd|1N>+cd0;k5;vhyp89z|iwP?VNIUTpUgF|;59po_Fb0Jzrj0kq9)?>YIhG@(j ze!Kx&gDkzqCQ+pCkH}J!k)hZN>llp6r;WZ#y|Xq-D*$GjfO8IfQ#%B%pnJbzS5afSY(NU@Dv) zssSr#ooVdq`+%yCj7P(sMbDTe?kyAQbL}TDoC$T-bmAgV=sH*L78a5b_Ml)L$2|9-1tTAPsxD<_kAEHsJJTkF>FEpUpoot*Aj` z&U86(*r&NRp7Wl}&(MSgXv;%x=zZTmrV&@h!l~Yl2$_Xw-DB_Hm_P@KCRwn8~lUy_#oU*0*)Dse; zVVfd|d!v0XfAqbxLlDOi1Vtr`OKSrwtjZ!6wWOWLcUGnMq!Kr_aJRNd;4DTj;YoBK zQvg8`@A@MxYJ7Gn7jERRn_nMo#kZ_2++62xaJHZCmM=u^Po*nW@I?Gji8!Sf@% zeJSg#ZX4}TM}EgRDoN`xU(V zF&EyWclP-P?iS0|JVh0j`6NUZ;fD~p-XoLRPMr<{c}{m6+`4#9`dej>wl1#-{uv-N zUln(4(X?dWT!f=HjrkcrIlDT<32_A{l4ZN1Oqer_Xh8>!;o*o3>q4jUk(;WjM_6kz zLaCCI?&#S52N)f7yMkS+ylxu4@9NyllRcX_W?|XqV#DwWU4CPkWG(3EK(G62kbfGk zMuC{Rsn-gcH`QUZr}57KT;Vb4Q3VT+Gfsn?*u$@l%#F@~W8Fc}Bu%eqv%3B;i~{-| z9PPF;$?+fU{SE+Z<8DQ@J^!{JzqTFyM)*&^_{VT~>f`6wOW2?P*AVZ=f&p@LFthw= z7&kh-oi|!yDc-tyz>~J#h1QfL?Ix;R|)i6RPiYZ=oHK#{t0izt2+l?moW?d{9U%kj7|nKrc> zl{GmVp6(tl4tKYv$A^XpWgcbJRHR4`%Spd9;$CS-(~As*d8#Q&?=1SjN0a^L;wY+a zJJF?T-BOW9G0r$J9L}*QQg9*6Vri>1pri>g}xP-B$u`7q6?QhttQ`0kP|M5N^f?`Rnt3{4!;(RAmXlLlF|T zHyzZxVL@Ths;C4g7&QQGd*s(LXPrP z%-hOLdnO@clZA;i*i~77QgF;v~@y{ zLs3{#Rd*Igxwf^^WUJzyLs!AK5Zy|fVu~8F1KCRDWHlX`e_X|Wv~dJy9*i+yPyR9${;XT-wQ<>~A3of64mvYH}v!Ah_=fYyS}RHy}A z?hSOrH%&*8zbfKkA4Z|Jbp$*gqM!75JWM7mthvx40^Kj@6Q8!IWk&HH>lSIH?w@(t zL8ZoH_zOWR-A$c^hE}bgv8((|lgeip@JWZh@Zf__vJ|=o&jybg&pfS>#)XRI*Bx*M zM^BqgaN3xLxNs8vDI#OHq-%ejAB;2|9O=GvPh0Q}x^QGVFnT5MD-DfSu-ut=9hV(% zb6e_*g=C@M^%MKNT`zje0h1t3PvBuUSyh@n#iGviBd6GrKgc$NFWt{kvhHm_=|tGNC++Y(|&=**wz)|xQb9^C6iQBDKPNE3_dO_`&a6V^_dTf%P_HCC|$%M zOT(`ofNxTz#uyr>fXHj!GB+AVI6YQVi0?{nG1a~I;WUc0YZ3Ek)jD+ zN=*y*&MT;q9ymYDdn?!7PItwWYhkLj>>DUjJ1!(DEPkFD(H4i%EZZBe8c?QlqVBrL zg_Gh1JhmlO@~b@F^aI*F>KehTMJU#w+Vohp7gf%h!aC|M<&+~l0=)uXO0;GXO;&Am z0t@L*t&W(uItdo%VUL8pwSS_DA5!h_AAG}748dQ{JJ3NwL(S&Q}T*mqb;Vl~Vt5+IS?vkj;b-v%6O2L$hQIl2XA$5G8@;;Qm zR==F@ufEUwXXhrUd)|Jo2Ft!>nrkRto!mS?1u#|}rm9cmS5{W?vQIMVKz$`Hi4b8u zg!!o`9fXDNyY(WmUxX-u6c-*nm7?Qm-n`4o*-I~mCEo~)i+4^KYCJ}SOB5Yo@PHG7 z&M}-ZBkNPY$;vH;rJ*K9Y;uP8USN3P(>~T6EiO~JI>bT*0{70&1;0r;#BzxQI=j{y zFfaBXvC^V*LIbT-m}-O?&Rj1!Ob%K#WjR;hRq3+_Mh>}FA7gYU`r-S@=41`wpXC02 zTR7vGkG-fdGwRMbCqV7jy*~O>@X9th?j`?RKL0K_H_Ym3OFnnJ#;8*EXS?$ireyCH z>sUe-2a7VFLf^0 zkd2?;)Sd|-SD`&R*JiYEmed*1R*_6x22X4c%xT>Uyk1e|Y^%kyG#hSc$k%j(5H|Dv zojTp4=1f{=N*3Px^GN z<3K8TG`CJ&dv}*kfnr{Ndnrn81F9Z^;WlZyyh{psJ8&1;f>(b7-LlaIbY+`+O%+-$ zCtRfKi-;>I2l!w9I{OQtB9Er?&rb8JR`2vkuGg5=kkvX!vQSC>;E1QfpqgL5EoGeW zNT+EoDzc|tp!Xr&EH^E9Jux_&PS9o3pIdic;q4!N^LMWvuu==56$`+7EqS*cT4EAt z#YyrX(=Y<>S81ZbKB-fBZ3}&*nt`;71$#6r)Qo$&*UfzZSd=0h>~ZC_&g7Q`RZ4IK z(Akku+O8k5ar{jh)%h(p34YorV#hz~*X)SpD2>&BBimX4U$&+82nn|KDXG)=AO=OZ zC;U)a#0p2-kX0dU5%sYaRfF659=I*~i+^2BSL@W#t0$U~^tUn~YEr{Nf)pUk&s+B6 zxmfC{Hpij4CRVm6;ipTG(&hLJGHSkuwAm3?yhmzGjZB0L{f+XYl(%QzDVN5c%_(N& z{x`>;T00em%G?Hpm2N2)KQSZ}R{W)wXc7REXV~L_#~Ep(HJF^)ND;MlP|?SYF;V%K zN76RF@60G6;wMF3S`HYpxQ+CVqngS~T1!*&A1rncjJWl?<@Q&qq>*#k6TkszhkXVt zd*mifu8Znhd0h6M#uhn6x1{;D4Mi|3hg|3;44ypr&qk`c+P-CbVWL^$K8qs2%h7X% z)0{v>(2A7VT^AB!!pv&$jMUf5tYWjgbXELZ{zSS_v>Bxdm@zk~zgZtL_mox9uadEo z&jhWUo?qkqc{UJ7f@FAN#P;x-Hpn5%N(01lmhpYbB3JZvm2Fq_C>;u1Vs7^Wiu1?j z(ky)6LtBRBH?}@jIJDO|(^9Y%79L8oE{0hmYXIc7&fg5DQFFL- z#;BW8`hYBbL{>L{9mq)SIeHPVs&cH$_B5DBIn)WvD1@hW#MvVMDR3_M^SAKd1}ry_ zO7hYU^g+GIjjm~K7MQE9pqJ;v(a!vO{l)y~t!|VIFTPRrb~h&*4pn;+t8DCF zq~#BeATy&w@FrAY({y&`co>B8n*{RYD>(pp_tIKWB`1Hb+s9S^`+U8Bd_AS>XI4fB zFX-*fV4@#OcR|6R;-2OfvfI2;g<^erU{H8wueW1CbKF>=4`NnV+9PRX1fyN1T=-ktshj z&KVa&;b2*-mKsykU@9zifRX((Ong&ZD*=_R@G=Mw4{ubV#EW*0! zGFk@uwyx_&$oXf#$Km)C8aJAkXy>;@utSjRq|fKnb=87Jygul+c~Rpw{N(g}IG*g@ zzI|zC1<`~SHGoY{#(uV0n*p_^i+CaX+A()r}hL>^R34Zf=f1D)Nw% z5pvFUtELoyjPTs6K@2_MF!*Y;J}_E<05yfo-0-U+g)&0rZl|rLptF$7l7lNG3)hL1 zK7;E!m1@ryDAte)S#cIRUD7R`xm9Dgm%Oq%Wrvw=owwWj`+hzDVy8d2c{mu-WcXUP zu75s%S@Z|*1qt9yA#ugJ1u}-WJV6#oq~HGk|$^*3_CX;>W5qHH8tI znfzaUT08Lb=JnYlN@UoS4W>3*Nl>QWrEADj33Sb&z0dRE>@4l6vXeOAo%lRuEJ9Db z+QH;@1+5Vjxj9b4CX-OBM?Pk?jNFOOe-->LjkSSAH0sC@U8$>FON>iRX0&NV=E0!4 z6wi7s;t4WXhCz3xV$T)p+-^6jLbKuS-fDkYZjM^xG}7fkJAg46Mn#d_iLb8sUYQ$0rLhtu_+(Zxq9FO`5qn$J0SWvSM zc`4~Y*Esz^MidBim+K6s6=u`a%vVSLuL3zh&G-C0#>}< zX*T=`+ndRFM$$y=eDNg@$cU*&Ys`*ObCo_yIH>a2tU5-e!EKe#6O$qz{}t*Lf^6el zA+A%awYH2tDL3uUk9q7AJxlIl9LB`V#SNpKOZIzrne*cjiIf6f6&EB64Hx+IlA$J? zux_LCe8jUmeCO~gGynawN_*BPb^>Z6dIXo>sNbTr7Y9&yyUSd~SI@c8E(=`zE>re` z8-ho&?ks1Vu4SQKi18k3f?4;E+nz#dqC6B`GCmHyqrmpz^kQ=AS`e?#UNcRN+u+Sm z>LCwuz`C9r|Ku?O_y)eES*c}wgB5tq~AKHd2)}|>j|q_$f{cO=yEdMOw0-l zsZv*)hNIM@v<7-%;+bBe2GY$Wq?VcVW)dY-e(bsLm8#C*`6bTPkt#q!sNc8!(a8@_ zi?7=}*Ir2&^vJKSs@?usXU%u_SA*ds3EV?`dpy%d<;sVX%sXceqL zH{E#PG^N8pI=@d{yA~Y2W_xBnJsRKhmy~qC5t)dT7Czwxj5diYzRx}YE3z(-at7~I zTaB{QR>8}yR4S+lR6rkKTPHetzFHDtU3G0*5+D}5Oq3(V^ZS&mT^8W&+we_!MN5X1 zF6#ra_XKC&Go~U*c^dQb4wa9zX|(AqO}rdobfrY&MS)h3S7RT<*0kcgQgmz`&4yYm zl`7O#tuyvCWz}kFZ?Z`r2T!amVk%hmV4 zluBP3bxs8;q5mqK;|9td=C3@%-O6i1l)y>w_{}jqbJ{#XIdLb4?6QNubVUH+WvS%s zOIB07bb!q}t~LZYyA`b?nU$yyuZh_*gMQ6A@`YfYwq$&W5JD~f^ou*sB~ez*J{4VD z%imI9<&thFqV4hmIx(IMkIcH)Ap@YZoS<4l>+S2Cb#Z~WArX&n&gNUdKRXwdy*hEGlqTq%-Y?^JIX*ne<08W2vcPR1_Hg@KLuG)3H~F?P}3|EhSz<#LBHKnKRjy94VNv(L~ZV^_;~VLS%^8XzbGrV zyRK|5U09Hm?yas<)=5{s*4sb-t*Qd+H`q1~Zha5e!NV*tv}T7<1Jxr2E=}v6mGU0D z#*ur%Fn4t)fb|gmbH4FI53LdnA4nrnNHS6^bjbzTJH8g9^QMMARR>er-BtUnv?geu z<)^P+`3+TfKGlY#MU9#2DQOaMC;8#IWh1K(4>&^0wwRcS3!-(y2~NOFzaU=u#|<}b zb#m=6`M${s_rf!?7irQ|%RFGiW-+{I!I^66Q|oVz2J0gKx75JW6J3~+k`jzv0eFi1 zJ0hxO#J`IAWz69Si+y*@6e4iz&ft1SX+{IzMSC;u9Bt4@HNO~5hPEDBXWU78Jk$cQ zV$O5y#M=0#QPI6$hwv9ygr`Oec?s^#Kx-999T>h^q?i^#BwZ~aanN!n{^`glm`{$mU<-QJOYsJb{mYq>*kTu_UZb!0df)F=m1^#iJE zF%N0NmsYjey*WY%la6HOZZgv9W;7M1G@=X!kl(fJKt6?%WRNNrN$Oa zwZU71T7{*6Ado%ke$QIci#_*AvJ@{2iyy!`*JuR#hll)Yz*gBvKSCKc9|p0=%_`0J z=-cwm(7qJ2|V$EonI` z6Zp2%)dof1H7u=a+bEPemY87}fTRqNr3|ofUJM5N21!06fIMH&cqlAbED)D5e z25eTM@wM~Q0CVFUOiNRUoPW`Ea3V9^guR)g_xHmXp@du=VEJWUW~j7mD*OHvPo1E%sL|qDyC|70A(B-}F-9a|dpg}I4jzkphleSgZJz%Q z$$`Uw#_LXUVX1N~jH5+Y!lW-n3#^|dst#)!bJV}->-U!qY6q!12&3t)E6O#RmVQf} zUlxNY9vSbS@4wc;hX@oPfN`9l{SH^f> z$j0O84Q_Z~y2Pjan>bRiGg+CrTN2~w!poW405;sjP6`gdq>Y@$VgfTegyXdjQ9^7KxHr1#=EX3G?CpN1t#&cJTE7zXo{m^70H$r$)AHWmEO z%0wh79&myz9Ju8qAzF+v&WE|jF$9W^HH6P7k%jPK8$+D9(cRn2{_cA>ZswQC)9Hgt zc!bNa8f#>#mmPq6=V`W;+1h4KJXnUe$VBiOb9`ZmeGgxW^8}oo$-_g5{&DDbr{C6; zd~HL0r)3IkH{rBldvyVa3o|3zL7K`Kc-pCuizVfiW;S;G&H9tGGc&a{24VmVHnM({ zTz-;9xl`oMK$uY#-C4wpUe9%Op)6A{25{VDjqiLHbeR)<2|s6?EWWDH`hJJt?PDAf za*AVPGaV65z|kiKYHI_vishuHJr}gp_DpAi=m#aiyYmgs!_~d`SiXbc<{4<*KH4HE zyB%7-voq%-k6L|dz>n6R_ZtnPVL<0!Lv}s#vkZ`|J(*eP5aWbCFdd%8(*z}nRqo2L zmPA{YYQumqnsXy?30d+Ip&I>fWfhvF)Z2mt+gh7g{**EZ{Nkjzor!KWY{jIlhs25W zeSg`d2j$yCwh;DmR^LC_4LMDxa%4kNO_F)+<@vTT#;g%Yq3~BdZh-K_#s=fH;*`Wr zb8Zw@C{I*81{|hybMDE6FM} zy0cy139V;)psdsK+J+w1eaYlD4s7JzwMncP-Hpbqkl7mcY|mK=D9O%j^nz~b{}@v& z1PL>IF%+Edo&)0f0Ruxu#ohs5Lbm6sNJ><^8?F>i4!7+iwtP&a#~75gP3K#(+{T}( zWoDPu(PKAfTQB1y6@r_ph+x$v?6A+OOaB9uKx@CP{wy9jolOGsCrnpVKXCo_$$Y=- zCmQUlFW@~fw=3F{;rRoBfK9-l3cK4kP`!pTxx6efjL&GH!5VA{r8vp2I4en1@^)T9 z;HE$8qdx58n(2*Bb?aqPw$@ZIssW0P;9#h)azzE1-;ytqs(($>nIJtjj(~$&=-uc#$%@cN&5tx^?s-1`E%W=F|iAb6Np`JErpXHZ4x2 zAm0sb-(44aY;E)LTy|bT2leg6pyGKic5~j@^>djidZXe54{TA9u;@Lv$@!j_5wSmlBXxoTP~ENsgRhvDEzEV~q>1!^V{P}zJDD<)WLVgI7%8d+ z(Wc`1+gKIVv>U6aruC|La%_l{_t8{IQjb%DDPRw|%~(B==Z%eKgAkZ7i4VSn8w#9_ zGQ^%;7~Gr;HLhi2tC+{=F5Y+}pM#9mOg`)R0|!26p_ujjs8~f)5{Zq6VGg{N$o*L8 z;bzxW`KFnKGXju($}HME13HpD!u?*s$2fmj^g*s4DgF@0d&Nh%-5w*l{-d)HhP$w) zmtMj|x;?goBA^lfOcgo)p!Mmu2OrN!P1Q9QmpiOhR4np>nqqinerS)fjT+BIi7q@R z$A?mLE31LlDUqDuwJ;~ghmf%blc4-*5A|hM-i#Z82kpx!F`gL7`Of|Yd1mh|`LehB zg>XT>?C*Z@nQRDjF#>h>j+Hm}tcmOb`XFmg;ntB&jOZ=1dR~sP}HLdoBC-ls@S<=h3iKbvh0}I=!H0-;1UFq z9=(Db!*`P)3s?&aduC=7R#3_$9kf8;JS9p}(tCyOB6!R9TvSOc_nCG-lCNA8xCuqWBChU)U#rT+zu7hM|f%(-o zwCN9yC6FE#OTatx@_|v&e@LQpQ~1K6FovI;0 zm9si-{7Hpvq-5kJ_IvDu?#{s4wPCzRtXYo<$uR~2wH}!Rt&X~QPe?lAR7*E;g4NTN zu9o54F%<^l49nrX0T|t^>3)vSnQCe0iv{8;jI`1hV?Nq}gz`U}n~E2%5x3Tlt;~I& zascA1Z-FDZE5;V4!|xNLYv1p$D~g|fJ0Tw=qUjIbp#>sD z&m?|b(-N=6GPe}MuGTw7k0Q($Owl@FO1Xn+-=jA0ZMPBZy*m&me}xPzhOZeliS4i?Pez#+^KFt#Oj5s18%lUyvX z&i5`%Xa-)thPgZANY}T!WWBwI7Bp)@aBmF%imAI~b8;Y-aMu*%3BpXQ?%RpI!wdbbK64yK-~XS7pBzJsPzGrcHO^omyAha6lXW^fS`?B}J5g3Y^BGs{M= zkAC{}vkw#22KV{+xx#*EIO*;ek(vu7b3h>O>3J-p42lX@&cSYgWjo)oq#a# zjff~rB%e3TI=_zLl#8Xi$3aXmG+V!mo%aI}lFXpw#QuhlY!R|1g{Cv0L1u{MX-OTZ zfX)^azy}g!dBgmAeiJuJ>nQa5M3t5hd? zyM%?~?oye^I1xuv|F)B;WD>dw3!}yY!OCcgVy`QSDJT_w>s?l(SH&_H<=~4~Gn$g< zf5bpEn1XUj{ztr~QszsVNN?oz-+el0VBI_pJ(q=I8aDYYEqRZTmAzwqEqSY~=pV-k zr|Xn#n1ZgHqp3#U5pkp-ctz<#7UD-GXudY9!B`Cm1)?d@!VhtNn8f~qHr}f^iKZi~ z-$>)Xwg2m8tSfe{i>AG`?MWl4C(CdYz41No4{6#XM(>s}zjyFBX!qX6LEcJ}Wo5;Sb!Sl;!w>0oqkdAC5ji!tCr7st4uK*)iW~49Z3fC(RUO7fA&moCzF7>`#saw4H6)Wk|b6(!UX%Ur*mXPKa zZ0QiZ&7k5nttvwpy$RDrCi=6qbeJfPHz_3UgjLW6QP=PzlY@CYqDIi>9E)A_p}P-e zje{QOQ3%rGf=Vf9lT3tE!r4fl5x98)lqn-_y35)N6SHO$#zPH1#N^Igojs8%FD~H% zfQc=9KmABXn#~+T|H!8%2Z!21v90d46uR9u2cVY)|owIHgHBlgWmDTjRVl|cC zM;ld@ZgoTSSCKaj6&L{BA)$MbI1jEXT0^>6yXE%=%Zqcw&@J%h?8e1tOW?V(V3ndG zHh#f3Q*z_aofOJ?drwm}A||941HwEe0s8bKR}qUYBb^5c?r%4zU>z`jeu%c7k~{s) zC9dyfq{Zx3JxaL(Bp7%5XUBhJfwaR9|Mfn>=)=F^2g?QK|L`L+1pFI*7~zcJtFTJP zFZK=6#Ke8X9u*Grs-oow`~`-l)F;vzqIz`GFhR?tmrzPCL|x(7i%S7(*49WF9Eo5@ zYqNyerOhiX+xah3@*uk8@0;-rza82XXr?+I0XpH2q4dzA>qe1F`Vfd1yHG&D>mbhh zH8+dulE@Y^ATQcV%NYr|AoMplq+8KyP2j!@462sKYDDCUUz1<>T#}s~y%Ht82JleY z#aBzks*02xlIDRW5L7FockP0$-KxyVf?kt<|4B~5gE(&qe^Td@DR`9DJ71~bq$5QD z)W3k@kgwNz)^FcwGEZQ-ZkvjpW3vt&mpiD~)-prsx9pcP+8V-^u(|(T=0=b=QcMyZY}8Y%3@@&{PWI1utB| zl!sl#SL6Ue6RDLOxDgHRqP4jZaotM5Y6>}ujQeh069~v)^)f|wI|(%vy}=l=iF0AI zzJW#Fs8;wMy*}DFJ#@!Iv6UNB>Bw<~=e9Peq6t0funTo~t2G@F95B18K#xZ%o8&>s zW+>Ul`hqq>kFcLg6Gmt{^H>1*6F>B|8N==PaZ2v+;l8^s!wmi?JSxOycvJ|4M}_G0 zJl|&6rHwLi_^9dzl8!A%kVB$t4w9sly|IL{#cIvVSbsE2Nss`=WX=e(W;_m4@wQk2 z%vlF`2)tK*WO=uF7&IMX%t6pfDrR`5AEkI^Vt)o};ak}8^dcX_+$*0X6=WOg_D@?T zrVwOc2Xav17#@R`6UD_)$u08NMmrW%0Abb~4)U=#Mk9=PCdFdv2rSy5K^yvGJwS#PsT?WZg=Lc7HV^ z`<>YCkwv#Md88fiS@2p3L3j;mP?6{LsuwWG7U;NJ|NZG&;CA7Qu_U&bLJ*t%(Wmcl;^U*BHgLN?b?|t@=Ay zr;4@pB`sr{8ZRaF?BK(3c6#(5+1ulH$7kI#fqoobQcw_YZG9=VNM7eRM6$1x)Xrw3 z^gLBmIN*o_PBTq_4r!Z!6RbB(R;-{gQz@yFKraKWf0*L4w-3bB&`{2tA#2R%w^>`8 z$Z<)3osy4cgJKAMNo}?)R+p@x-X&@!Uddv^jzKP_hWNB@*A$qqOAPJ-IlrX0!AP(U z#$`%ad85}&TY3Wb&iV>2_T@4q8vHrK>uVuvxURb4TE6dICMGv;U1qS>#(7HWl*p8b zJwNZU9#*(t3e(r;LXkL6QoR0x#C4M5j;~#lSSBez#I*MsYUd2)*;L7FMqW<)(^!&S zl9L@HJ@T5&NKT%UI;ggR`bIpJ+`EvWsMKU(Le$UsUkd?23(ST}W@E+TH68{EfU_wxGa)si-t?{>8YOfJW%| z&Hh>W+UYF9j8G{&X6;CH$L=+ed^pdDy_&;?UPB^qxrT%n*A(1Di;7E1XwBQ@%2m~j zI+t-N67r)Ny|CAFL33pAaLXDRz1j+6-aTsM3%&l`x#78gkQ5;q>@S;@Jufc#Vv&KX zPYgmQ4G$c!P4isf)!2A@{+=hdXbo~#k}pu?3uyTgr5Ic;rpOogfk6X$g7Sqa#3cD* znuM4X$ifHaB}u+O>tDzm1K{Bl3GPD4O@==E1Gg1jtjwMjgU@Vt@HPcaKdU!Os>r-m zn4OB7Y#v5Uift_;-V*XR5@U7y#xEXx=~{Ya6lPPZ2NO(c=GChi5-Y#~CI2gw#a?B5 zcrr-f2^I8iR0Y`KVxp1RGMtH!XTetI89Dm3MfnSU(XHY;SdK}e+S%%U2nP}PkG9kOWWSy(kq~c3gO@D9z zjkrc#p&L>VB#uiYHI~qhJI(mJ1xXiu{@k#@4%tt-2ylP<YlGG%=H7dVSgZ4D*ElcOoMpRL`;kp@H=4(6klzlI z(ODlz7aRu$Q@r^D6RctC_#mLrT*qtc5J4-{CKPus#}G8~>(56Aua8bYLe>NOdFM@) zUvY44!5onhrVzyZl#A%j(&l~_&3p)IaL*>>&&Jf@S;gky_bl}ut&W>)S<4>nK~U+yI`*Pt`*gN z;;(?PvmGDkMYM1Vhd;C?*RKW@ZP4s-5h&?mBx({Z=^khJHwy&pVJH@`l}SZh7u1fQ zV=(+L*$Uf5r||&bQ*s0b^}|~d29XKh4BStyVL{iqYDHdUdg~W!{xD$mw_I`NzYCf( zr9m5`H)_IU$(Br6cY9zpAHj2gc8!S%t})Su@c|G~{JDIQL@Crt?A2Y0L$*NJ^kN82 zXP$8RAN)xT|G&|y2U|B4UDI0O;>j1}iv{&1%(%|#D}G7I7kfyH+R*X45ARRU4o=Px z_H%Locj(Ek93JLcpty1Vk`gwd6T$^svKp=sDXFmP0Yol|D-#BxL9Q6RqEhdwB<4mI z6pj*5soK!g7jzNfYl^px1f6dLU2NQ#!{r9-E2N_elk~@v=YLG%&*k&vU%fp1Hd@NK zi=e&<1L5&QcTcGUZxZF#K^ph|b)`Z4Pm>bJRqtOu>dP?Qt@Fki38bG+z|ze)cvBfm z=~4|ECUL(yyo*+(-gRBw?H?9!(TWU?4tsW2YqV=c@Vai-=k>26?0p z4|f$QHQo9K$i8{e6z}f5bHR*e=rqrpG(_Ie$!g>DCjR)F855!>t8=&tGVy1U{aB3L zzgN{hFI7Wyi1DSbdm9(~=MkiN%Fq;YLT{8VJ~pqb^QyiWF?sfHZ0KKzp)sU&T*w5? zXj*HIb|M$Zr$K zvk-#lh2UB}=y==fh;)mhXq{hDFc#~T&s9YQ%B~}X-q`h~=kw`W=riv51w?*Be!h~r z9fn>)3%AhPRm(#3_QDo$-!0IgG;?il4QcF^a>TR6k+fY;XW&M&h`L*dEL!*7YKleg zEZqAjF%BT8mVhH(91NPP<)1x={Vi5gD%T@ls7aD#EI-l3KF%@uIibe^# zfsN<9?HJ8KfPvs{J&C4t4k4#-Aj@0yaD^(mIcO&}b|wF8Av8Vg zO@8QX5Bq!MIr-h5*<{aN?j;YIO`U{B&vT3Yzd{Fx0ZG2hS$GR- z=gI5ei}ZxsOg)bl&ry7sBE8oGG}~J9sVY`|wv*;D+9b-dPIwzew!OHt7xuZ8l|A1m_k_On>zg}Nmrnr5t z)s|+AE|2z2BVl`{f%G}Vi-Pj^nK;?}h+-VyNlmXathrLEan&uO%|gBlRb|73zv3eH zYfq1anvg~X2EahhSUC$iX4VWSZOpdDQwB*f^_kb3L_u5Kdq`5RpHP{HCsz9f@G|&y zLteq#IJxj!2sjhYaW^!HFlMBtHwt_Z;;v+GJA_NjI=_jr(vE33C3{I@UXBA($aca zIDR&+2Wg0WlKL2RV7n3NXlgeHNIJd0^|62YZRvpW*p2DLW)O#|!fH#A;Vd1yK|NI1 zQmi~8njXGWoscKpt4?sg^rS25iIqr?UsF#=|EhY@P0$h+eiW@+-N^xOjEjS2k+`~S zrX)fy;s~u&#%L_)5-tk}QqUT0;;Y7Hq@lVkehdy9+2%e>=8 z@Ly=5X1btx#@1$Ep<##o%v#&<`c%YqQ&af)X^mmo8U8oc;0XL>#pikTH28ghFfQP^ zXlJY|krGpD>g^xRZWkSK^QIBJ5ln9?V3%MQe>goQ=*#2adE4GdlQYun>b?eg*4V-` z7%<&pc#LU#Yc!RsIDpQMlWWP}qR@5w`0^pNQGi(4M&F1YB#O4nVS!OiBo7$>TJ`Igm}(M4h>7*w{?iOe3GS(XNgugHYHh)!(xcK8-@V4lOm=MkPt zMqvJqUcWy%ibmCTG-pJ(7~<~;rYAf{sHQ>laK;oxeIRDrIanALk?SiQv{Y2lyROmu zDR`vM?hL-$uJqY$L$@#PaV6-!>tEX!_hI`zO~@I0VIgu04zBU?0~Lo{&{)xRQ{@Vm z3RZ{rrP+^@bzZDkO)<7WwoSu@l9^l&ZFKuSU%L8vd8PM~*-u;4lg9=xu?ov66M~o*C=I zGG{O%=@wc(;kGa{tKr!#$fHNL77A6m@cdSn?&^8h=s{K}Od0htUr>`J%FV`uvg|G- zq5`uLHo1apr9R{eFYm=XYVZxR>`5wEUa_xk;sQ80q{@J18C?4G&=sSy)p#WnS7D={ zJX{}@mu2SR8M{p2`lI`h)eC3&nm(z8w^)PTDF)H{IF@dZh77s53>LEO2 zkyp7GLqjSNf!P{^p1}XW-FOu*1pi8FW2izFJB7nuV1?I^(8y@Oz6t{`PGZ8kFyzPFnr^E={drqmW~`P}cr(X90Nor?vJlxa1*w-{ z_s~{)BV^p&Ff&+nOBmcJTI7}9g|(6j&8tntXz8F;yj~iRi5ZZl;5H z*U`zxqi7T8BClv&=He;P`<(~yhX-$uPF^3JMvrmW8`w5^eD)g{Og6M&3sy`4OMInr zVJT_}2_I-(j=g@;2o{Q6(abCAl^V{JjpO#;LtI18f8%qr_?EzA9=U=Zg_q)>hd*Y7 zeW2hkd=9Qz4*@I9?Mz+~FtyW4<*wi(OmV_X`cb*-!sog4MjLYeObJ$#RN3Vax4Ulj zv#Ch%BUncuLKCDg$szN>L(5ySSo%wv9t@LZ)@5cL8QR4XM|FHOvj?ggj(?~+BnUPt zG&<6*P@DmKSO9iq=>?>AIC{y%I~EMab93{MuV^GGxyfF%kuIjJ)?yy(;qa!_Cw2%_ zq^?&RTJRIDa}nrf51{a*p23Z<-`+i_wh4a!nsWIV{6U3HknLIW27s7`GQ`^6NX!L^ zqJ->{$O4HT8KcmnaJ&yQoeHYEQr~vEmyAIVB2Ml0Nw=|K z7=k+o9~#_YhsUp99Bl%fCbfM34NYLG=|^WF7%TeVF&_+}S9#TfL2E^8a!s9wcPptm zhtBe*$qKy;HTDKnf}6-r`iv9|M~BM^t*@Bi^?CI3`}aS;J<1L~e0Y0&cyM<7{v zIpSY^{`ey4-arL(oqF~8`bF}0{TrPAUwuA5|6Ha!7cV|vB+0*`4a0VJlN*xlbmnk3 zi#`0V!&`-oe|?h+Wt@(5DbS_-6f~Z@5$;X`esr%bv)Li^Cdf4xm$?8R^Es{fHDRc) z{NHKSMqgRjb9+v)P~Y6Q+Z!f7yp{BotZs0-VLgu!ki9*{!Dn?z zeej)GhWQZ!Q55;*Bg4cyhHZDg{~-dmyzhSq@)K#+wFK|sm7GV&*pcV5tA(AeGU~ZH zNae7Iw4#iJzDJ^WnP6&@a|xU$SU~{sA??9?wp}dxbg#YpK>i8TH{b^3H}4;7!N(?G z-~i38!H{kJ3sF2xsqXH-=n=Z7Qh5c>eQ4gmZKyI<9F%=`2?vAwMNI$o>y81#U11bO zm}i7})y)v=p6Pd>jpnSDBzM=^@P)}!(zHHj8ic`<*6l64`*L!jE;ig1f&qWMY2^?qEDge_+Xnh^~-obC67Q8JazVM z65d2+GAp@Q=e5!WI_>ZA_7}nq&#-|95%Pyj`a`TK^zA_WdAx8A{b0A&@W;g@(k^t` z2my^&Scu8c!eg5l1vGF7z;_dC;0Zqxj7eRP9(Y?CbB(0L&2o%8ZgPn^i$g=<0JLd4 zEakJKx#j#iPROg-NZBE_q?gb1^EAUhsr*2?p)hO(dk{hotxPTn^4X*;Gw-9EpKjP4nTdKw=`xMfGKS|K| zkE789h^Sp&Zctku z7VzFGVIgdmQe$gN*si4NkyfMSB?50aui$Lh)a!N09)fP}R6n;T`ADk;saSofbpSgO z77iIv1|D2#+-}f9*nl8-J3pof_zey#PQo5+6+jiXcbHdtS6xN?s(wf^G6Nk-SRB~`gnB7RsWP098c zw%e(fi^2msu2T&*X(O1~RW`bF2pLl-ZlzGV?cDj*@1w9SxRM!T{x?uF#giYCJ+V7$ z8_p0D0lBB8w3mi}{D`$&ITMT;jDhKI&{KcVTo#P|aZMw-(-0u|F&FB7FcDpWb5p{U zBrm!w3-=cuaqtvbi^zPa|P^6Y4*Y~_&NgPbE?XC9Dq9zXd#vSXU+t`@cGb)|*~ zwqak8szGpf8pe8ilzyD8CgbcSGn0(PY8eK|gVY)fa6qWIV@P%%^%FLk29utX+mkA# z7G3hf;_Jil2;1;$Dl*LZ>RrMq?fI0WV{q@E*Co6SvoGx2Et={02F`vCcTGcN2K%NB zyY#`CLoJ`W?hqsJH*9OjrmYRsGdFkR6x(EkvG%4>7cULs$Hyk0?^54CD|uiS=F`>y z{uZM8%i#FvbiI7SUhuaPDDU2n@_Y$L%&VD1Jy*qN1S`OaJYjWFwI!XB!?*80y-vyD`*$DS z9-SSZ{G5{4rw7L;DS2~n{5B;Y-v9OJ^yqc=`u$%|(jGv9Uyo0IhR@+t0(2H_RbjKn z*samE#_qJY^RWe8C0p>tSiwwbdchf<>2j?10bE)@2a&O2EStwWD5~FK`6;I|<1zqNSh>+2J3^_TMzhCtCUa{O|GSk1vwHf0i$x z3--nKNPWFKr({Q#+v(?eJM!eReR34G(a6QROwDB?;IUzYjqSBU!>D5m1|s|h13E?O zgj9eqEk`+s)yR>jgggg0aEMher4}KAuK87W`Sl~E zOMYETNG;!Rp-d$HcvZ75AqY%`!%E#Jy(N>7##d5$#E!YgEKCiT18A;W6icVnrXXG%~oB@e0 zP4Seh^II~1kXM8G=Q?HVU#?I8P$zsYeykJMloZepV2ixICBL-gFKb$(?{)ynTV$TH zrRpVw@j45KovyFs)l38Y_AWz~%+3Y{DWvDTt;-y*0Jtk^6=YT$c7L<17>pO5lb*J< zVrzF&+DSfR3oiJwhE&#lWE-w%4L1s`nG7cV1oDlMaZPT&Q{$|uY*dLmHXnU6k2S4n zI*K*A^DP*y%3&NRUenBYeP%;4d-qRbh)?gGb*%n~5iGs0vRKiwtysMrd>JhdcLvJI zSgyBTN$?lxy|dmV-h2N9+=6rccWF{g2A>D0mU5Fw8aPeT+%^CientW)pMWvD<^<|eZfw%}N-aMp%jK@>)Ucw|Ts=h)F zUK><|E~PL=QA+Oa6SpTH?tgXdU%0NTVVIg0ir#SmZ*DlWaSfk4 z02i=*mElR%24=`0FrGIl@h`a}-gef)-x|m-qu;uTvO&omk!Zf5&Es4cu>joAW2+OR z*>dE}I1HanySM-^%-IiE#=j_vK2l0nN;Pu2yIWMeEp0N_g0F3w)ZLO7a#zp=6|^qs zuAoxwUhVICNXSj+CSBNRSJw167^gv;rWXk9)x|(!FL%#oIzft0wj3pojrT^=xIN66 zZX(ieCy;K&GQ!V-x6M1ezUc|{+E}$0alI*;n~l_+xxPL?!#)MGaN+((X!-3ut(a8! zvpjZa{L{Kl9_OFTGtqDC`2QJPQ~9glIPIP`wLk=IaDnNl)Cr>hW;wLz2TMIlr(umv zlgblAgG%GV$nuUKqf}z-(-Q~;PlM1tOSMgbhuF`~N5cA6_LqvFTi9aaTSg9feMN1wIA_=-);UmmBNru!ao~EW4CV_s8H`GQ7R~xEMqE{f(#(7b z4vHTNZycP@7yv=cVma!*1?kZwp7i%y52Yc~$Xi!UZ|^4rI&`ensmT0AOO& zltl0A>Xt}WFO6;unZz*P04+7TXg$k)f=2-|WQI7M?*$WU5a%Ga|K?-^#yP-`SLc7* zy?BuXiX(jYUYIiR?&Nv$$A8^U`s9r9a@*;Q^d0oah}_R-J8%{cz7jZVur~qv0u>jp z?Ym9w+;69Z-ZZq(lr)joOp_sBbB}N9`HUe$*6pAj~+~u|jrT;>lNBY-aSzYq$=mL_-2$budM6=Q9PliJ4 z(!%grEXrHO4+YH?EdzBC@+Al_l}%okM6XTLng9BHBY=zZy}pqJ&!vfv+Q4gQ3ji$>-3#@ zx(|DGXB)X6wm&KczC{dL*h4)$8EBIOnfBkNOj=b88FHwXo!a{dFM9PH56-&Bu!vtxVT+&Bg7z zdFnZL$|!^|w2qrr&!EveubH87+*@j(j+}aD9ASE}i(az-FpbZMtvf*DpJMLrfc*1J z*>;GSg1bn-{|pVw_A1n&&Ls=p7`wIKNiNzZCx!_!bxj5IQF1l)4LHXZmskUuwjm{FQm<7g{cO8*C z3KXqWe4RD9QX7Jnq0uPTOUfc&v+5Rtk9U$kVpRxFE}B-p`dpguL)qf~3vh{peXHD0mReaGhJMT&3 zWGEr*G!JE4xbMl=)Lv-e1WQPpLBz4;c!+YmgSE1|waZ(F8!EDpn+>;iq6oDDTd>c6 z-gtNiNC9?T7Geve^Ds@wGt#i)5`wuFOh~nx*S92S$=iCkun0@8g4YYf8R z;Evf?Xhm47KRjeMo~Umyj(~{XaKO$d9sq*xlR3`?tT1q+>AG?nKkupv<>QF z3Io{y5~>5CyKaNZn0yufohnz(aF)l&Fa}q(9LjZzaVXxP@x-%19-`@p9G`mkO_N zo>Nk?1(?Sb+%x8Zxn7}3!n$tIlEiefoRCW@YFZ%xm0wcAR5BrN1Ydi#^*%tbHLaCQ zNj~STBKyB1?|#x!f&Bav_{6nR!)&TnniAjd?U8psc~Iu9Mh=+L&3007a4+Qx^7BvI zazZ|V5$Zc#l_(Au$L0&#`ByFH-VAW^S+n8(-plXDhr{M{aTPL}iG zolZWejaq8xvOE^{^_Y0F+n-)y-3{^yT8A4)D~hO05TVFbfZtKL1vc5f^qLEGw9*wI zFuv#-sfM+KZO{@oSbA^eaEAMxcui{+VxQWa#nQ*h6VE-}GQdQR`F)(3x}#^x#M^;| zT5!MmEFeJu#8{x0Ueq%Rh0Vx?r*OX!#SKTAPr|s9PN-Cy^foN;rtrvaM%Rg0Z^{Bs z=5>h?lQt|EE+Mlp;fh+R%o@HX-i0x4>9njpinJ>QZOp=nGA!jU!(+-*+%KtB6TeT8 zdI`@Ex_e^5)v;^~YNhLfa|M@`|Iw>j(#k``)E%<&WZAT4Qn=16o7SUnIFBc?KMgx^ zF;P6j954gHWQ%m_QHJ)hU-;aYJoaa|1915e2_C;5-6)~AoWJlnF=mloHXqP^r-5xM z`Zqp5E@$?BPuogQg23z2Er6W{z3)fO3~yW;rgix@J}1YoJ1rKj&R9`E8oPIAak%e^ zy)o?1*qXv^TC@0%Q|QD_^2hjddH3=@nfCtQ)V(sll_^>AR>+jBS=}m_547NQDMJfb zxJkA7AT_(&Jdl0Spn=1_LqC8V(hn$o_WBzSzyV6b(_mv_ki&fpJ`c(myk{5$1kXOs z4-J>spU^7rZ}WP*1A!Uw`QJX*7Y{Meh%s^hAyiEHC!>M!AhF)dKFq~7MB&Mp_?4c9 z&ME!%gqKur{556BgMTS2XzRuiJ+G}SP z{(RW+=odcs9EBl8^1tzU;7sRT1t>?tgEWMwyDaXzy`lN`9FI17Lh~0sPmELL+}NCg z%LC)+bN#uFlJn`#{>8<{Ui`x6zWgn_h--Hw7WT*@@MyPKX(U~FADPT3=C130+>y_4 zT5h_PpvMj)$7%in;s}NC`ZmXzh6=+(ovO%6=4Dj<5F{rwjf7b>d8?XM*_4{#6kIYP zCsY-lC7{djn5{9UkculRGS+mB1##4&BT$&{G@m1ecXmaI)pPRXa5 z@%p@k#Ox;@-|4S-E3x?zoN)TLWlC3mT;Z<0Xa(0XG9$G#o*DLClt2vEzxoL;y*9nN zv#?aytlx%Dj4|>K=aK13Utu>DX6`9maNsI9ggyffsO5R&y&T*iBIuec+OOfi9aLFJ zbgt0_&2#^R1iQ!a+D<4Kya{h7*}&%Owq8ssvsh-i$3}QzPK=Zfm;&Sk6de|{tPs_U ze(VMrut8*wW@Ng`MVYc%i8QOJD(O`;YSJCaBn;6Em-P0V_@YRr1y776LM84#-x z{EA7&YgRAEDfguO30+Mf(gTfq0I;%K)pf;3kYWk5zv(!dH}v6ds`hE zOV25y_A*9cd#<1h`ODU9DQDhogw`(}XY9aJzHi0q`p_r7t!A$vwZHxFr-z2ZeSt$> zwuBQh1H&B>PlpT-ErpR%*0L5QQ}V3gQnGnE-NZI=vgTV=dIQk=$_N~r~#kd%({3F3zKMFYIGO%>!zW?t!eM-vcs77qeM(czSej zc6{=4v{B*(NdgB9;G~ibVKmn6Zt1VW>->p#XSDu11&EU5Rb>W7;;{;=ElgMTN`)mx#0ikjNo#;`AOz(&X$S(fqB6V46HqacsK(QRvMD}&oV-Od}YMd@x{ z0pn@K^D@H&ve;;9uUTQ%q3|RGZjh4sfVl+FLR~-C-%-9g9VK zU|pntblTQh2O;uSD!ML|p2L`+4iwBnRpmtDEeVE{qOIddwk3a3ozRxGp>3P_!b9F` zY!Tg6myKsq3XA=J0RRC1|4>T@2*jDpAT3t_07)$Y08mQ<1QY-W00;m80000000000 z000000000L0001YVRU0?Uu0!$WprgQ8TeIRC~B_2 zz45_^%O>fbv8i1&-u4zn;84;?;)NnvlCrB7`R<1yB|FZaqQISBBoajqA3ihl%y0td zO817>je|F*Z~h9uDOC%EtE=;95>4QW7bq>3(8&@FxCY@&v!cQAO9~&+Sgs_z$xb1m z3GwhLp8hSGK(9JzSr1Y<=q!S3xP^)f1pHc{cEBYRs?~xsSs?7VYy38Ykt~|PpMwdN zJH{n2D3tC&RS)L@a~?4L8t3%v_3PbkmoX1EQ>K0`!bJP}YH@zKyuLhnlb!k{e@KDW zg268xH&{a61JhdYg5?51upKA^tTu=xI7N7OhC43n6s)S;jv0(5P;%=G&pUTS)`*VV zqX|j^lMtU>!{R!IU(c==*J(6?PmA?GSAVSG)7gg)XUp~C@)}kj;C!{bSgaST zX}Kk5T41t_CLnmr9dmx<$)U0+s*GyE^_o`mRx9J+O!m?6p=6E~f?11p{JI}Xf!UH5 zE*d^`j*IbGsDg=&?sD7Z5FJMAn#(#8s)qUameuI4R9&M@0cr^O^E^>Zno`4={So1u?Lh?Mtws$Uz-l6kspFjwB*9C_}H%W<{etgYMezB zWlC=(YjFnF89(q%$p>#zmBZT$#?}Ncm@2V?&4x?vHk-tvsL}v_c0@(yS4`pL@35q; z{8IYSifrRxJ{);(Q+K)Gwi!$+Jp-crvvnpVKi#AUl+TzY#~BAo6Uj-eG#VcY>t16# zh4~!fz-Ii?5@IBtKK@F3M*G3mk~?2uIvGfwEB!g9hxjJ-Jo^D}emKOO{!AZJl8p?I zo97|olGfJKX%t1o(9@vUcn%ZK#}8a)&~-*WKO;HvTCnHCaKG&)?5lJl?)1zDe|DwR zU8hrMXD_kgG382$03nV8@$Zc)&?ni)cMy(pvUgVIrz|SikB}Bw$AtjXNuT#8P%6^r zhSKZF^ontTC7e*o0UDzW{683M)fPc%@R{z91A<%k>~^J+2q!Nudc42?vsS>iikI-E zL(>yZ>zM1Tmoj8ULx#94`IeU*6QWOjeew2GMO<)c9TgP|%m$kbtm;hRC4xdCg9{~J zxv+%?Mk+cigMR_9j^3=VVkEaHU9gzYTi3E&^aJ3vl(|J2KzhDUgsQEtlgtMCyh~EZJ4ffNl^ge_;-!39Xpu>q zg*^Mh;C^ze((t{_hD^Ut=|tay&3h`u$c!bfavTr&KvIS@ z^aq;&EACRz%As_eBH|Ih(TI+aWkWQPjIA#}yjv{K)|VIYe+!sfcwp^+4fJ}wdjEd0 z{Ou)}cUfSql4|2d!>1 z`Zvw+D;Ax@Bz4NF< z1>eOt@PBiNFILOT7d-dr`Pe#WHDT~FntWIK^Q&O@)y@6%r+vsPh`pS|Bqd581My#y zDqcOu1CbPR?+`{6m)tg3hK}7YpMl5bpI_bF56DnheE!fsy_f9@(xj^MsWNOln1AAk zqsqQ*$KtNEeNt>6H9gdB&W~Fz+vT<}Jj6_>;epKwyzhhnMLJDcN`kH`h;=6@LZOq6 z3&2*pK2D)kj&J=@XT$1WJ8?$~NYWpQZr;P~zB{-DgMy=_`$A>VHqtlx72G})V0)Vm zMjs6WGYA#BqVZRb0-@|EC#djyD^_%-wY@BEv%$i;L4%bt2)nU7P<^COataKd3`oZk zV!<`cLBlNMh!U7E$jTm;XXn3TNrC3?d!~~kM#~9^Yp@T=l@Dn>Mv}jzP#1r6m|H4M zj7sm8tIh&*j<^l5TU%S0llwBFKWW+5&4_RQm($bJDNKO=Q#vt|b7z?8L3V9UHyU4B zSo!bHQURsng;>%p>T$ddjTSNZU7(~!(oXNkX#M!^AR=}M&Qa6C2@j-kNeWgp=qnp< zwV}p{l2sPHUrCKfOY>-xL;4&EG5yN+2i@~8zqA>(In$#hBv$j|7Bq#jJxm%d=9i;b z9`E?Fj*hWGE*x8o?`_7*`!w`^T-Nb(SSWdYolR&HK^TU=q_(Lws7O<-QLK6p3PuqR zf<_c6iV>Sa#gh!Xo5@ez?5;bzCSF286j~38BKU`hda;OzZ6JqMw6TaFhy}4qK}w(^ zLcRD4YW-H5W_B5TbImi~@Xoh8JC}LCm4|P>tVEv`@0MaaFBGo^_MIJ~*WL`AKm6-- zC>JJ+N>3!(RCzQRs6X|g@MG$rng8C2d+h9NJQipygty-6!b|PjFf2G8^ZA>h85;5D z=w4)?uXhE&F5I*6FCHnnmc(7z3%%R&KD=fDz~n*zyiK1|5r;}aTp)FSeo&b(jEvQ{a@^0HEqpg2=}Nb)QCiKNb+^SP6ioIhD3KoFb{juw^VR zfG3pl$g&{7@AUwfRL6-?K>g<7zYA?U=!)svw6WaQZTgsLc$jB?R2knce;f`rKXT zZN_kUp8Jo{e8l%+t|~y^ST)u=W!M`n$}F$ld>*ZPjYpqXSuPUXi}T5eZoF&Uh5w7N z_2PVxh7+$?)AZ|V*rJ!=j|T2nP0RFr;D#6C-(AkBp{9J)Cq6^OWA%Th6oL>`@CgWZ I;OPYZ0Okmu$^ZZW literal 0 HcmV?d00001 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel.zip b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel.zip new file mode 100644 index 0000000000000000000000000000000000000000..5482eb34cdfba1ffb76279af28fa7b7eca38331a GIT binary patch literal 63549 zcmagFQ>-vd(51U=+qP}nwr$(CZQHhO+jzHaYre^3&c#X2f4c9wZaUq`lU1v#6r_Pc zPyitQ+yCzc``>|?rH!e+v7xbrDV>97i>j{uHXDlXdOb%%;4GLq9q6$jwl|R13X59(hG=qUxq~zYb^E@x6VWq6OsHW#mPu~aJg}hdJ zl7oN0OtK==Q13wr&WVMJ99ka{4a^a0u3p)DYz?y3A(?lqBZEY|{KfH?Nr)VKg(i3@ zFP8Xysslv`$pSL@5g|%PT`Y=>qY$+Z>M~BS$s`1RJmK>mbK$^1>qSAdG&ci2G@SlP zZ|1z2?Y0>WTDGP3oQeIp*<1C=u9x@QC7Cy_?jE(|+~Cq(Y(t9iejp|>sJXXfLQQC! zK`H6fp=(8Z>kzc6+@Z^%wizWRA^Mzlqy0BPjk4hOH_8~l5N$D3WFpT@3jqi6Vu*^k*v!&XiEM$f3x4`P4Kntd6;A$3y$xv}uimJkRA zC47c$Bp4`47ySh0zaj*jq{=2k(!!N^lT-y#m`$w?1{v_OM-vcAtHaEAc74`VQb|Ks z*!|1{TC3vPaUs{AR>Qs|Bmb_y)DJ4oUpLjGP@deF18~X73kp|FC{;40^bX$Krljv) zI_ah($)uPDlS^%Q=|n{>>5&vdE3+;J($@-6I64}8lIDrYgegf!K|0Js!zVN5@jWDN zg7sQ1Z?KHlV_wY_F8I13cg-Z_K-_wlYBxEyHoWlD)5da0y`O+qH5PfGx6idE3+&pm zz?WCHSicLD($!c_Tyn`FQfD~(&=S)ED}2(^ZAj8@J9amcXi7S!S2jlMNg`qG)+Fb zrFNMKzjWG09G-)`qr^$F2?)N-d1s%Z`HSQI7t8`TeL-H)Ii76rmU@aH^m{+N9>lb? z404}f35M&+g)})Y6t1dJq}KP~f$DiMBvFGLntuOU1)s$dsYi#6a%FE{Z z4Dk5qHgX3^8VuG`DwBTcw4knsf(Jnrm&IYM5f+p8rX-oEUw;7z@D zAOQdr{y)ir^dDKwOn+*?OguDONgqh{Wb@3-<7(|Ni-}+Q+8i@&;4L_ zN%vTr90E6OR1t!OCCO|Sk!DhpG0kf~y@X_%O1mbmjS~m{ywCmN`RvTSR64V3Qz%L^ zaMZAkc+qBm^j;ry5aN!M>rx6jTHV?DU+{kE4Cjy(2_g+AxW9=&v7qk&3zXO(1fN3C zkO5Sxgkmr)LRu6lXdAimd&pJ-iUaPf;Wf(8DMijVKgOV>nnDvYgk>QUL>mDxrXz@t zq=F^^U?dGHIT=z2@#avtWsb-oL>~*?@f;J<072p<2-DRSU^8CB7nJqttc@EF9`Wg* zn9>=u9(C4c`0CA%m$Sbs?riU5^MQrVzkTYZjG_GZAXx~I0DuN9pW;B(qZov z6Uk#`2Bv25GDZwRsWGc9RX2qiG~CR9@ zU9zDLw)@_wB%1gf;75Y~u9J2-1nA5n!K0SK>hiZ$jcbrt4cw@t|xs&4> zaZOkYz^A#6;LVQ6teiQl30SxnY;k)!n=q?e>{av;l%{``{Vbr$P@^a|gWskPx+bJU zoYLe=5HiFiDW(;Q8WuzX%7D&@lEt+x0oCn-_T`Hpf&!Q$jRvG+jnjNFM@fkhM-5|F z-nE8~x85x)o~lI+@xLXODz#{hEU}u-e0;?a^>tlsq04zXFK(bttq|>^^qY;D9oiL( zRzt$P$J^s)&VArx)cm)KN9AC{F1KE529N%5sg?~&r`9(ly^fb#!Sp8MPm%|q#NsBJ zqrE6dnMTC7TuD+ad&GSSjb&tZF8zCXgB%cdJ!;SxX!kGLz?TCYr2+fYW3kwQ zoGB9_mGoncbOAf^f?ZxXlZ|zu#Ex*>Ai-;!5`j0vg)DdyYGL^VqcI?5y8Kr{tAN^k zPCZ9Z2TFg2azw=(n~of?wATxS316i;+p1{WiLz2&ySy@-#;s68JC;N%(#i=V=g<_;RAG6kIc&-C7E>c z-MxWcp~|vfSy_9uEVS_~mUITUc!~!UV_>$?i!x-lxaX-B3TgT}F@<;R!QhgXNNzcy zvA47PdKdW(ayA;F(z{@*>mTgq5BC32=>JeFs&Yfs?LTTsLH++x%gWwJ-`Um3*3yN} z#?_F{#lfaTUE6+p49Rc39s_|RB8B*3gma8D0lBhZu_URJthqOb$9bNYI; zNui~`H$zu1ttbV$3LR-DGuPg%uhYWS(f#MvFSVzyFBqeU-=)gK)+NdD#hj(HQ+6-E zKM+GszmLoJrT?$r$CXQy#vquCu~inYT1Cq+?FH^W5?jJFEZP0c{cZAP_Unm#y`C^^ z6p{u_f!VajUl9n^$pUa^p2p`)-DFg$;c z?vBp>mfPFpq+eLQ0}PB*Fvat4v$KWPLkoaefoK_QMlC6ZvLPsl_ut0*@!@GYd^^5f z(sXyXw$$i96gY9NqfF>HA>&j4u=tu|Op29O6fE-Oov4*VV$&ejACh&+Fs@wg6iC55 zSpWbJxSLHU1jfU-i}ei5Xii9!Oz%^rKm*h{NLbJUGo1@ddpM>|-AU&pty9u%63o+? zG_Zdda$#Y>M=s89y{wyj+L(HB=gN0|E7K{fV^le1s%+q+ZF^TrWi=r|ku@R_ftImT zGMzeWO=5?k`IY9)3R+sq+RM`3H_V|l*?it}T8F33T2isgni2E*4d97PDlXHJld{%{ONiozn{F2e{+BgGULrjBbTHOD}dx=UHjvZFFuGQ}58j9(RgzZ6A+ zYu}||wmxyj<=$(T-=ANb_J4V~tn(IVWZMm}C+t>Ilw4Wn#G$oo6=m@?Ilk0~!9knR z)mP|{j%wSkqws12INzE^o@isaAOyd0{~rFy;$g))GND6vpq|8OWu>8J1x3d~O9iT^ ztcY-^KKyit_l3{s#G104=+vnNXV9ZL=p(4?3jFsW&KlalGTan^8^}XhOn{(lF9|Qcp3`pN z2RYvyInhAN^Kru!DJtPPI5I_nfd0;u?2)S#Ws9x%vMqOkzAW8KS-4~Fu6{qKhqq1v z_7O)LJD*@BhF;tl4LWdE=6C3z!1YN`k+eY;0A+1A5H^d89Vb{WgtiyuQiGRRqfA`? zYUtVCyQ;^@&fUMR!Y}#XG9s+pST8vr5_hH-3aTnDJ|Jg#y_G9b%5*gbBpR^J6qet` zpaQo(Y6X>Omv?bDiW@)K7{i$Vst+|?dyS>Fht(v#XgwkWMC7|X82z7U#ZvLhN7ZJ7-9bT8J_dqLN}KK58aH*WxRInC^B0(Xl+C3O0T5E5XP=c zyUvghJU$qZO7+5CjNO{=4;84|^4v2syVSsQ-%Wg|zg#*VMS+pH4K11#DVdjDG zy^`CG40eyZ5QzIa$12>z5<1C{_53zyIs`K|j@E5AyGDxaq@-5TW(vx0prjh=Wav7` zGYfRMAzMsOFWHyc(KyOJ?G_OOb!nZ0sY7(@8{$qmW`Q4mQN!inh&tro*l!}cJP@3% zoj1hTvKg^}PxKF>tP*}5dVmgakW)YNJbXdi+h~Vf~kMQyGHyw>)WtlK)UNYvL=IHnt0V7O?)?7WN z<2dYCOO5`G_STeNmrs)Mm`_rz4s`1lT6-E7&i{#L7@1@JWG=$z=3sSN-9PNEb&hms zb=A`aDRftPeM~wc!cf5v8|EJ?XHZT}u>qIj zrl1MZj)ulWh=rLTZ2$$<45~v1c7J2z z*oh%Zet-D*8vY^QesB2t`*Z#I`e+3B^ya|V@8|95&DpOv@Xei}`v=3Xr`L}Q2-PkI z{H}sER1XWGMC`P*3FHVbPzFq(ZA8VC7-T|KuMwAUCC(yMfg{Bd+F&9BE%R^=QbG}8 zJXgE=t)bfC@252dX7q<6qlAgfSyX2$iUXFjY9I5he2a^8q`;zG|gd z8|H4p!i^cPTuGgGya$`Cf_~JbwQ1#wFM0>04n0GlQ7)@sQkBZACBnstqCveTO zF~t)leyxnbBMs|SFnnO0nWkhf=`-op<&;c;)qm?o!#hvbUXX*D$p|1a%{0D z((nH}j@i?fyRVbiAL$4D{_}Lk?D2DJ)R^Jx-`L|Ljgmi4&=_|jbb`o2@`85PEbb=i z99l~i3R;UGqNp zY5`oF>Spax;mDf+KOhY3y+eyl((7$+BwA}97f&>*Xu=8sPN{65aNASNts8*?7V0O0 z5iB?gg%d!x=m8PwUNmzKv{OeS98t6~HYjsb^l5$)V&nH`)CzT+!KYj$y(DO(XeP|z6|6&I zH`monr015rl^+npB~|g)d512{xV(iE(Wj+dPZ$FKDqTb)*HW1Q9saEnQj4HeU-)nK z?_*3_t3ca&uZS!Ze5bOdSgX$WSTAE)OR#+YSfbRI46Lz}8}4w4X#?&61`D%QPPIuXp^xcS*XjSG54NW%Ca0_dNA;>?&%xtk4xa4^+qqu|w8 z{2xDE>@TRYIr97%9J2oi`13=HSv-U&X)Xn6kiwecU`vX`8$x1Rx^!WPHeGLWOi!_V zkAHHRFGsj~{iAhl!*W~ud6L_}iapP}(p@;^^CPAU`B_C;&^kE3vz32=YZ>>vW(J!} zq$?S5lgItPvy*6}kD&bCcf*f=yN{zr4A%iFnf!YrC^?WnAIdy9d$PYy{^u;42gS|# zb9`G=Uo0EqT58mk*Ii;;U|Wc(E9VB6PYOlPY_|gtTJTksqV<#9Nkd6pR^xG?`A~aR zW?Fyvu6FCT*CcK<>ztH*Q!`!SNM1FfdHFxD6&_kYv8bA#M%<+z->z;qr84C<*sf6t*%GSte{a zZc5&y<<`Bb($v(-BZ1TR3Dy-PjO}qtb`cArtxc7Pq^N21U{yDWzh@6yXJ)_)!KXt- z#c7LjL8N$ic;EhMc6&reN)$Q z;*@xc?hUC=KWx(M6Nce)`KAe5XPC9EwJRun#!E(bHA!2ZS6RcCa|Y31+ebj!^T?>I z#H`(&hZaJ11xaU~U@~pS;y1rr7Nw;VqEjRmTPK;TEO(2AM6!LgLg5P)C4mw0r3@R5 zHbz+w)Glzdv1vrRgDc^TE_K`lNEebJQ{b-3azOWrM+nqb#Oqy2I5^u1r>uSIrdHsk zXlm5Du1hRltlh6O@ec1f125O#;@8*f&(>h`68so3XkBd8&>?N#-n9^wOan=F2iNKQfub!9-Ya3j|BfDaXTN`Lu0P#?q76W6kLReXi_BKd zv6{rEJyB}z#-*&1sDZUlO$AMwX(b{2Rl-lZK`I{)rQjrG##-s-jDT@W2e{<-TT2KWDr<4w45W zR($f$!mNRUzcH6CQffu_wAVKoCfQv=tfR`Zv>x5~T%tggOb9*6o$g7m*FJ=Y6L|xY zrPx;1_q>fRC~EIjF~Uf;qFsS~M?jU%A;(oy#Wx2I3h^f~;Di3HTnXoFx?-O%vQ?&_ zSRvn~8Z1`p9xfUCW9Oo-g{nESy~jafvK-6bh7Ig}RG-Z%x0|q5l&~X$@P$hSMT?RD z^>tzJm(`p?4z1!5yfdIsxa5ZMBI}iUXwVc9CnYFx&lBxe+bT4an^0j;xke!86Ig1wy1a3U-6AC-WBj7PCA>u>I_E-fFcVmP_2=vOHFl z3)0;FbmK>mgat}282KFCx5a%Rame9y0FgDHi+r2E&ochpLtHncZbyQ-HH$}jWd~<{ zLbB_zRK4X)wN6ud-a;FGF^!lyYDzCwtX8m$>U=DH+Pevc$q(RJkn0+p}R|8}*Swt$mLv>W22k*bK28zK$h zFW9xNytvhmT__Vx9n+hdyC6`@-zi)?xFJ|MG%X?tg)&;S$|GoV5S)rIiO;zma=+sd4usXl6 zN0|yC!rXlPumMWUK~6aZteL_z4C@B!kmBePYMVUSJzDFb)KK+10?`%%Tl~-i{U8Ke zRala(02OqrU<0)s!5I)Bf2oG%0g{rcS`c$4QB9{VCmU=c2gcx6ScsO^7%78G&ye%<<22B)EMX>|IKJq!K3`FQzL^>VQKLm~HLIEn>~#dq~77PE$3 zfNJUir8X4E;93DogAlQq0qSJ;+K!W^%s|CXUFMhp2qwGQlc{Zc%&OG*>>3zppd>_m zoq-$bP(QxzY#rGk#NKS3fA{+l)*h~|9^P!ddAYiSBjh;xQ}X7^wUHkvdi|lrFW#>+ z8-FH%Q@0T8d*u>I|I)x@Oh)2t9YQ-@K3nki4~ z%_8#W9L!ApPCKIF80EbXRqU_87>$;mk$1@tFW@J?uwmwY1{ER0OH#Wxqj_t?NMou< zMbSZ%#WThND^Md8P$87Wmab#qs}3sYr3DTM`CT1aIr>@j^m6s`bi4R*{^%rr`Yklc z{W4)p!EsZYMG(O-4w`0dE@iRKL3yYX8q>1aHl&ZjH0XR;?uA`POFJ@x`|64&J)inz zUEP?&J+*^nIbf)lnze+11Owe>d1#&K_}tA=r?kL)uu!Ou$EZfwMV$}^l7lx*nIKS(_XBianEu2Fyps%lW3^l8}@pHJK2Zj zcLZBH8x~%4ZIcWQOqrA!8CJaaoDeEwdS}$of5n21Q_?);bKxnH$_5b!)}gu#l6AJG z#4wdn9WmkrfFh%qMF6X=Z4}; z<1rJ8Z1p`x4GihRNU4#r%bYpU!{|3~`l(>_&6nu?GWtCkeI3`+%Ms_cf(*FkR!#v@ zxLa;y-jLj8CY?3Wl<>K_k{Hht^3oBJQKzhtqTfxeP%iz^6lcIxY;s?JD7D829)<2j9)36JPxk9RGzrGHbvBL zVkz^gn}qK+1<$hfUNdbc9ydYb%Uy`)Efa5*&9l*7#t+@SZQ`f&p7O;NQaYVkdG*#F zXFX=+^#=Qx)%N|hpi?O>#7xGD+V~5A0u@Z9S%-dmID6oG{YAE)q3E-8qIT~cWyJH~ z$%0sZ>d0s*$UHkV8HiTA;d%!Q1rU^y>wp}6%gP&wP`0_XF-p8*5}EYHv@G)kg$^+g zaih<>{Njgqvrl>w=6P*n4*Hay^WfQ0xh(WX`4E+rv7%S9H$(}Z?EcwD@K&CVHxzuh zTkE{ecM-#T|42b$W&n)V*1y^78O^KuF_n-e>*&VzH5n}Fh%cGyF0Ww{YK3eI9B0Y0 z=p2<)HGGCv$XvTV`79C=q{zRc^fan%l91XWGCN&D>z6E+X#AfBgCy+sRCXa9V@*gi zM$1%dd>Wf46QAv*1{rjV`gd~~xb0RrZD&${cfw{;C;-r8NxQE7TNx{2E&0a)7l1~|mze$1X3Ku3qP{*J8K`8j#?KVL?ESi>E=8fvRcJrN0mX^3^|TZG%h;a>D4E74lU!24q(#{Q09TQ+`TC5E7@=;Y}0 zmyZko*m4GSG#OpQ+ff)c3v!gWEFpH(2oR|2e-RU3urv*hhzI0)ah|ArDxynNa1&ceaY-!&eX?A+H)g;?Lr;cM7Bwt zvrFJ@*FfFTyJH?1wz9C?AIm|#x?gPAV}I*~j4q$WV%8NpcXRYCPS`Di%!$`5LFHw+ zh&hF%9lN4PEduQ~zrxgtPF}+{NeN5mP~fiS!}p?ywquFvfm@4_kXrm^_LYxLRvYP3^;N zt2vOD(Wxl=4&8rpDY!WC_1r|8G;wEOQo35i^ehpFi$}2BG51K@x4ooK!rZ_B+DW3o z!H#*$^`3aK+``6$l6$e>Yj%?#UmP|@>eY7!vY-$DS=K(|LOzdK)acICtG}iXi1oQ_BlF#<61e9&#ELk&>xgrf{wC^Nb!FFiVZ8EEz~se`WMSva6Gwvkhj{ z&L;S+$L2etzqn)FHq6mZFwdG!7EE&S-MDb=`ro|#b>D&gZV5UVhu2QhUSG+_-c=q7 zuoRs#=MB+>vjQpET0w;-VZN=458S|mf%FSEKESWXP(>nS1S$s9wRCZbmzE9pQ=dZR zb!)l`G6xxx!pS7Y^V zole2Z68*L!C8uf0MKz4!>r870cPAAk3rzyi?b7bD(krf_5#<|Li{?z&+4h2;Adj~K zp-<0HxVN7m*wjJ3e94)htSmVRTA5FE2Z|NTt-FN+iZ(CF*yH#aXst3u2N!skFN2AH z8I}LYCL)LK7+DUPS)3z#))knr=_W=UpX5}$;{N0xG(8yMjLUj;r`s>i|BkbVQ4G9A zZ1*)lWHZ zII^a#-TlbnTU2&^`1j7kzp*|2UOPC4`pDb|7rrgKZns5pKW=ZVmbe6LG%UGiePq^%L_3{f%BT1dISUZ*D_>$=H3&Tih-Jx<7G1dgFkWp{^he zWF<>#!csE8M@q6ZfJx$`jEd5XGSItftqfaSr1{-lJg6R}42GekXA5Xvm}6jyp@!eO zw5s~!`}db`ENn9)+DK1JfYblwaWgvmBziOXeW}35fHV;a@3or5A-^_e|Udik0&3f z&}tS0K|WRC#~Dxs8u4VyOj?Kw#zd#7gc?aHr)=R!O@*Ui!GXI7!xU6S#D+p8xiXO1 z<~fj5S0NPj)3JfVgM|n?DmqjH1r-g+L%*~hN($;PdHQd*&U#eZY`vE%WFFFH8+FaM zyC^&CAnjgCvCciLibkz;7ts1WT6HVR#Dz&${}{5fD4774;YI8ta|n``)^ z?EarR_)Qx|Z(yr#xK7TTvO;-ujsl=FgExNh5dd+N%Ww1PgXM;9oA3^tjDwQ_^OcVm zfKr4C`4K;{Y2hDEDJU-=xBl<7lJ7J1Brf&KvxFevl>e@LvIvOB^M|9fkPyUI>MRl4 zA}+6|-{0f;_wV%cbTiuIBmL@azE3)Bey`W(i_g>h=a*X}qk`-fjD`VR1u@aNQIai# zNi-v_%eYglSW(OgHvlr#remsTGBuGz2*TW5WxN5#1-OFMjWG_+dRYr%NTrjN5P{Z8 zQ1ehY&KsB6wj@qvp;$cLz%!zii0k>egJgO4oWMDBd0Xf{%aoDF>v>{n`G?!fFy_g0 zdoewLuPJ^e=<1w7$hl~#b4(;zL;RLR@i;k(r#2FB70`&r3hhKOtcjV$DA_F>OpU%1a&9OoMoS@(lMdMpu^>HSur;;WX?B&NY4cR z5eLSLiRplaJyUgfP__6ss6vYYx)gkkr3Kig+@#$d$J4E4W}IRzD@fHO_{0J!nw++Pq9*&nL*>>9F;zX472v1Kt>JdB4;IlFk5j@NmO)gE>xlaXfdOZ#nxd3KOeVP zH5g|Z>EvhhX-;Q*25>YfyjpZ(-c9MbQL0pnI?zjHnoQ0jBiqa#|;)Np1=AWLr zjM25UE~j@2M?^(WZ5_>^x|li{!xyu{X-np*!k(IV>-c$RpoDWWq-aIx)eD7hx&@TJ zuHIH+!qOXp6*~=Dz_U{Ys$h8+vqJuQ`UFS9&6t zPe!yMZYRJK%k>(>Fb{9LIrz>qH8-x0L3_+T^TKfdlG0k3kF2I+%zOOWB1dTO=tJo( z*x9!_J)hkrSC>&AwBLUJzBc6A&E&4^(?ZOUDKQ;vA2-d@?` z8G`$3ed6q1s!=ZNekc8xK%2o#WQ{cM{uVMuB?eOR+Glsavcx(`udYwDv}75v-akt7|3UGF`fTfQYjB{bgn6}9sW(P_?1!u6?3uU{p*=tRSX0oAF0rF_j|G- zrj}p02~ze08Lf2U6F}SbkJ*P11Ra1hgj!yi^z##i_^#yURe)eM-G|7MmZdIinUDc) zVRA!oTNhS;yYl@%WmBk~GxtozZk?mzI^S)9w~$h@0cIIy z3juI1=DL}X;K&j8F;wa#6O}zL`?7v=EoT`Y*7=ED9saEPHs-8Z=;k}dZPqq*?|nOr z!}C$h&SFZ&l2^Wor608%re36&Y&)pQ#z7W?PMMXqr61x)NnTMpw)SV9qSy+M!P2-B zgY|}9apdy)MwK8nx!e{*X^+|}Emz45-n7h++^Yi!zS+Tmsah42;29TXv?SXz>Vm$` zS<)Gl>z#sEhA+x%t7Ea6dn!Vi$l63N2CNNGrR{bX9U7Y)Ezt~UF<3$tP?7}^YDGK@ zzTJ|_@t}j6l9&lM3qoK&lTXEHzZP%JYNF}EnPYJps*gjG%<01hDfrBTW%O}$#(!0HF#$4lPQ#hT^K;Sz*~YPF(Ymec zIDCCRk1J0RUCf1(aHm4FzRe7|n8Mjvs3S5#%#N}vwpv+{4;LVYTlO1)WF2bO^h_jvJKgOA`(=nhHl{r7}|RR-D_~+ zdaBrdSQhHZ4idmsRs6k?Z4Gu3pT=*DM=9#s$QoP`m1q*=uRN3)3gkEVL&cnLR#lh97nonM*p*#}P5w z$@kDRgNFJJ47RzQ0rOe$^SOz_YJRCH_-Rj@(OUE+?X65iB{Gf))-}SF+hicIY$Fjk!c* zu3Ncvt;!SLdehvN1Fo8n-r86{1d$7rPhiYC8%K0)_}GS%i9fDC zm#aW_dp*5+bp8H*rll=KHhvY5YB-G~LfobE;^P-L^#g17l~meLdeJ!|GO)wl+SoN+ zX&lW~M>7%-_-ppL6yO0hwVlW5^7*eR{2QcBt04T0z+Vz>NpyV-#5#tZc(HWXYbY~j zsRcR;t-2}Sd>%FGyZ;{kx^ZHEU%5Iv{eFxd!YOJ+D_LpjW?Kb?7{1^V?DKrJ|8djR?GUoFVE!D<&f2f-_j7PB;(x2r|9hiXNivu)g-a0*i^FkC;uQ9hk&PLJ>Q5Jj+ z+f1FX12=oq81QYzHvjYg{Ffq{?h_B^Pg@)IfU}q|&b$;; z=8Iflo^g53LKGbdWBNyWKDS^fwzVV&*?~EMPj$g&U)!F{T z?yPS(MLmJo6SPSnHVh`bvo5VUFmuB2jB5wptcA{K>z*5lQy^Ow!%sNOnBH&HR;RRq zB0{P{Cd(5(3UrrYMcJ*e0#gJ#3Jkct2Ad|M7kJslwA?yfQrmQ4Y5{~_4O`AqaOyE` zMz*IdRyl3F@A1}bLkNyGM-b_e+%^|}QUq>CJ`*}p3D*qa|MS zOTr z#O;D?8+Zz&ex_@o%9s^+EG+-QKgJ!)onOpf$ie^xctuIDrIg{>k26)sploK1+`Pfr zX=^EH@Q_j8YF5D`x^rcy?d~$)e2+{MF}6g}XGG z?_zB&@A!hIs3|9iO6G(;2-@N4A4>nOXv`4B;Mmf#-6*w0WRb(fmKs88(L4aXXJAf= zlpe=~WVs#D^cYtriE$gFDD6mbY1>KQNnws6Qw+8n!IiU{6@-s+kxWgZ(MVt%}Z2X<=|8R5aOY za9~tS?PZ5C(b|#cP%o8PJIX5L5h>-9NP^V7B|;?UojVm82j0|&H-ZLbP91v)E{=6D zwD+-~Ge8=!!|qnaB0@kKfI?+#BQH)NE-sD1*-xxHLKaY$6$h71NbjKj>cxLzu|X7B zpIF)j#Vi|XritVv8`eA$)QQ?z^-)z>xw)Musih>O(gI0)XW}<=1dDBIaSmx?-Ww^d z-Q%d3HTlW%eS^gB=bA}_E<`G*T8mBMm$Rb;zfq+IuT3Qb5aQ5t4ZU8-6 zz`bzug?Gc&4N~3RoYv#?)q~4Hc9z;>ts4fVm2mSg0eoF~@@giefdo~=m3dxo;@6Qn3?w8+ye(|h`3y)CFagj^RpRI$idOcPg&LP76iT^=Th{@-;Q0FLoQj* zrzuQ-v$_BJpoF%Q*vNXW7V}G=cL0C-L@HV> zU9+YVR>_w3mYMgiOhZZfz_WO?4u`DU@_w?nXc*C|OIVwDyl|c!g>c9-3*883HDWH1 z1DL9vCicZ&Qd#2$Z`jJpdN*`4PfBP%D|ub4xsApax<$ErVrCNQadJ=H_Rjsl%)E5vd2+5&fQy3dq?@3-X6nyGChuJym)dEJ* zkdpfg>Ga*|uAqq*wK_SPc?bpgx_ml}BKN#xQC69)!LM9jW`mt|vOyHSr97PRffs43pL>|PZu$KqI>p&vm85(w5IJ*z{!0)i@ zA?K=n4){-I+n?w0%hx*;&@e~icXsQVgxoB+#-DxAHaoZT{D<)OX!=*3jMTjL^$)n( zkmunqZIi(t&et~W7^GT@MO-sm8pV&FRY;=~7~5%Yba5;wEu_-mYN*pfc>1n#&*6o( zUegRto_`=fZZ)f=V*L_M$-TL=Zo)`MHN-SAeGSOhIuX5_u8|i@UT#64*`Hb>!`2>YYDb>gNDuCupSy`#wn$lUk zWyaEbbP#X+Al1b+siSkV40}(-$ufMYnhM0Y)G16^^zJ$5M@6}4K$ngI#jGcZSqGu) zn-iZ!K^`JKBEJ+5SfA#gNgu7X;fMHUcz3=#&*t2umzUN+Dh%mtqy#~6uLK%D1|atv zL=v)>cnHGjc+p?r7MAsV96td(@iW|YjsG`*LVB8u`1G<$gx|{i-Zo`Ouh)nlfAMYmkA;WQ4lbdcIjFfN64d&P z_`5+y#JY)WyxNG)5W{>c%S^AyTg$%j5ivYh-gi@8-gi-*-)IvjkI-rzy0Uq@Hq{5p z%-TBbrd0PruQ;)kLeb46GPe;O0i&t#3=~r5sey3?vd5s~pVQL02ln^7bC6v7T( zJ-=&i&9sbnHVp5KfOGrI(^Xv>ub&g;(V%n~H1h%qwZ+VEB4yIclux2_IcGEM{AJRJ z+Yu|q`$KeuEnu5|2QQ9QoZiDwo*4EiM3J!Abz^mPe2qJ1Bfpyrsbl5>SUYR{y)3RT zde@5Wsa`o-j`BpL)cTDz+c)E4QUX;X$qCK!=Ub~TsI~1&D-WJRrwNp$-aE&B!rP!l zy7%&JkQ#Wx=_ux<8A5(C5BXLy9V$ABw;2SCZZ-(}x3|rn3Hk}0>{DT1?WBEVR}k)| z{LMnEz;ntCSJ1}vebnIa8@k~Eobl{Lc~l8k1H;rI$X0gHlcvIrnFP4SQnoWS&bGXB z^ap_kll&LB(9AHjgrnw`@XA-Ys5 zv!f%;Y-DqhDo`$o)b)}Yv~%EU(#bL|D_ukcli@pwQ(N;Tv%{4>Y>;Le#vie^x5A>% zRtC3~Gqj0%b)E(`M;xuYxo$Yk@5*ojcGjB5+pAz?ot>Q-o#UvneJpQ#=VPY^!e$CW zA^<#qEj8WWn+otjd5NyQsvg}Wj|WLX_u~Gpg~e~Tvo2c|SN)qA_2>t2)c6xC`tR4< zkA{MtsV`XUx0tAlQNO*+-WnX&I3&77!Foq zF;KJUyF#-~^o<;hzhG%6ibEm|0ZG7sJ?J9;{Aoivtcnf15S9f?(54eCC_Umm=kOK) zdcx#?6Di1;L&)sf-ZA7tG{39K3D(0M0AQIlZv@eVI1I*^W&Ya6mnR<|%gf3`G|iDJ z*-uM=mrvEx@%i=k{Q0^h(9w^wDHW^~-`>Nhrwx0+)mKU8Ho8P9*N`PZZHf_`7?R(7 zly)5(BL$|0c^N~SI8Jr3|E|VjQdXVDUmyga41j{*+Z{Zy2lDxL^!CP@Ilp~8-|X4< zZx7Dk_51xg`ntb;pxN~gk7i%Wr@N=Kue3Wpn0-2aslUHYUdk~NvT3OX&a@E!lmN+D zjpVqxMR6DvB*qi!7>_{b3rvib8(@dT_==EpYzo9w9c9N>21NL>g(s=)G8(DDg-Dw< z3r#@H^pV->ttw%39Bs{*|3oiellR{6VVQ5thpBD&)18_Rjh6jvvGUwQp4%9ET+UW! zY^3|_;J+g?Yquv#Ca_8sGD<9DiWS-!WXjjZ7YuF(Fs5>_cpo%wX#(}7~^vbV%wWeZH3%~k8lB}KY zI=v$fuHglU_N|NmrCo%d!~l^&1j$5+lMp-DQncV8KDt+z+l7wIXk>2Lmo(wt1`{-x zpk#?bdMt{Csxk#gqgZM38DWS@SJcp}p@_#FU(HM=+AWlDqSa8w4lf&aR;R0NPo}Qc zp&FB9BnO}?++LDUvq{deg5HKC0b@IO{xKYX^liVo4BHj{lq>QUNqP1*kxLq;&`7J? z2DEI$6~69 zn*U7^$js&MY89YMv%_eUGt8CZFf>YK3@0_HEU!5x zA;tyW@5DfLN878}Qu?>H;lnRvA)#K?uAC_cL_giBr=jvZ+ES?$h)Pt@C3+U{AU(^% zr%6RBl45ffJ+KGbM`^=c1D{kt@)Ki3h6SuspIMMp-9iFN)d0pr>?$9UmsaseAx)QV z2;N@P+pfCmTJAd4e5vsS2_Bb+u}KgY2}I_s7~qBP&XWY;MwvMowZK z>fwbDcFUDx^Cn4>-QX@7EQ?FnwLApQ0hizd0BdrcH+79|^-{m$p4b zkkKF~+}nB(H=IWU8>p|~PZ>?z$9(wMb_GiM^~KZrC68^F~YM>C$h^Ou>^hDhh{ zYb%Q2R=8Pq)La-U0*fuz|U{&S}*RYv7qy#jf#J@AeE{S-f2~k@RW_BK#irlcF0y zbaXD83ZRDBjLtAX@sM~TBpuKQL(5-mdS^54qop-Kbl2q)*I$3P#>grhpV94dbC9&G z>vZMy_-C4VR+6{EA+chOF2ZIEt@p2v(9!LfB=PZUAb-R@W8i{gf>A?$B!8_xrb@9% z9F!*fBv9ia3ne0Ei{T{`VniKb;WWpZN9lHLgtjFlCf1g7z;zOHIRP3FtU57mm{7O^ z7Shes(eszp6p(YF=Gyiwdz=!{)ue8hR@7L5h|S}VwK*r=C1ZOW+gwBRG1Ue{`Af6K zWsLEV8wfR-Z;PTR>6AD0-7&yJ8}p6$)Ch4Qx7a1i+=E7N&LW9m4DNbU_W-vzn6)K1 zkpy$?JP%$e1|q7DH1&z@4>@-OW92h?)Z#S4>u6D7|M66)3`}_~gID%k!J(_QD-v*W zLIt-T+7;jWxFUm|QBQDc$`!Q_o+$@4&^{JWoW&>>gyne1X*plL+^OWby*}TE_}k$T z;8EAp?e*|}x|Lg_KMwi@&i8r%AXlWst*;4N0&khS;yMRXS$LoO<*QwPoP-&qYQs|g zvx9y0m zO2?9_!}4_q(*og#m*GkOlp)$YgHQ`jI5E|A*dfspk2PdbiHyzDdUTCgr-P+h#-W2% zYBahw#}gg>Yr_a=M)a?SeF!(3#X>LCG~(c3!2WTIqTl4qgvY|Iqp;C5$XN;hSCn%M4-U zRFoTh0A<{05 z2>t9a6f^n^V% zY=m#w@i`amrZSx7#v_P+V%W8zz7!d!$b8}#J;e$ps9S>F&#HZu-Vc6-M6xFN)nd3C z*ydANs_*Z&dw6}nw~f0mBgQJRYkIUN-XeP8Z$M-$?C(7Lm)|rzVDyZrRSh86)Df~< zuJoz z%8(Dk`j=ic+0ep!q0+BMYX&y-kS9A!R}3Nqrzyf?C}v_*abDl0rcxLB%%+*9mR|R? z9`+XIlg@c;Tm`k((lzJ&t*LIWr9`NPhI#O1O!P0t(a3)_@o;dTfMQ%lc;DX+Z}q9U z?wk>LT6E)nt6twJm&BW~0xam5f$K~lI)lic*KKZH_Vo@Cik@2oP= zgvwi0<8tNQREvQL3q^0&u!{hS6Vmjt*R!?@%N9SXOqIL6T6Q=xhZ?!LryIGm&-XN2 zu7)5Up0HWD5k6=pSSU@~D3{->l>0q}ObcTmCi&Mwq<*^ODW;RhcJAFRQe2M`ya*?= zt*N7mJNZ8a@=A#(2}asNdF|yk-e>*BU?U>*`0vw zo%ag{3BMiq(EOCve(J#COMOT^Kz~7LLfj7chh+l%dDwCDFviqJ2Wy95`^KUpI;pEQ zK2Nuc9>0bQJcSxkE&77wt^_*P%$mr{H{5?CvH|-KH<=&Xt46B-7b5#1VEuBkc5$?E z^058$9~8!O?Y7_Mu=A)sHQ$}KIo3pf8m7jpyWiODW$S7)%75iK>wp;vN?Akc4s@3B5?CIh07^aH|dW3EY&Kwl)+OvUfGHv$0EF_1yKYH`^smQU0dC~VyM5yu?#WDj4&QWT&E;5m_-Y)cp zc(1?!#TOmVzn0AXC>i}@)ZjN-Rj6jyuL+W+K>sW>;D!Pc0X$?SNeT)QYqexi8D!vl zAvwetAS5`~T{DnCSz{fAa6-dnV2z`hAPu7aGdn61){ULq*<-`#q0s2#_u?Fir=489 z>@3~vi%&Y9K2r835}{w@e$~)&N@+9%H5CmK|0E^zqL2GHhB(Pdz{XL!o%F^6kAx9u z67SOz`{9a0{&uA;H4xKC>k@JyV}zsyy19dus^fY4xx09>;#~B}m#*?>?dS~3hoz>5 znXQ(RtK0iRiXZ=INLEXa&D8C`(BNi$L-})ZaNz?`9Hlt^jz&_yVTK7s#1~1Kss}qm zjHQuDsE|-L5Ct%mc;Fs`Iz9<-(3-=c?llUPO`{+zI@f`iFj5=SI2Z?&+cQv@qV6LZ z<)|~F5Am5nCKvm>P+&CYN~auyOGDer6^p zjd&UnLZ(J156sAvI0!y=Ql%_5q^+@|*JrWK?H5Id5DW;P3{xL3%Au& zCcy0Z5FQ}RP?Ki1;S7M8C6HF(3=ry`ENjXp!YPv07K)tyA#T$`E6%CnTle@9u(%~j z8D}AiBPl1v{D+WVaAa&F!!I&nUr?38N-osv5e1(rg7D?vpys6}wWEtL~f7l+uxLhMjUsJO1QO7Sg&5tWArWe8vInhXDx60{YE-mzq9);s z-nZ>$KSD9zxU7#68f0pLBKwCihDZmuO7=~-u+?{DeFp+p|2_j0n?<*dbB{gwdz?g* z)}Qgp`h+4ht}@8fa{R6?5A&YcP7}#euQqS}&!IFY;$yUbZ+o_JyDvgy zu;RF8dtirB%`eE3+vsS(BSDf@EcLL3in`v7<7rk2>8q+}(Dy!KVoGnYLQ|7IF_KE! zAGXro3LP%0AZ}>N%td_xAuSsed(z_7s8ibg@!Cq^-q5N7nfgzv2;9Q!uhIAYskF%6;O7iO(B}4twi=K0F&}t(_NPur42-DZ#eKaHp-)z#)Ccz}tpehjhtb1ch ziGjerp~8Xx)?$|Q{;gW)InC)Pd|gg#BqKvC!e(i70w62v8kAJ?VLPHsw*#cl0x(!B z8##lWM&8}bu=o+mC_Snq3Z?|FLk72`4i#a^wYrFTD<`)t+iaylhILl|5~&dam1T}? z66!`{85+J=xdQ0=U$rPz3I<7Ma42!&%sks>0TDkizH zSlQ5_O1uJ`>KT#+EhhVVlK!?j;etZAgj(~8hAaY|X>vwSQ|4ytD^SVZ88XU}fi|X- z0#XIYB1j@{}8F-^fm3f(z(HIgwNEeBff zeaEUDY!?1(N$wCJQPD2VYV9i zpsog3%E{2l!@L1$bpD3p(($(a#+z5_jKnvz1@CGxNE!du$OEtbV#*l_8t{N$zkHN3 z*Zww++kL0}u&t7?CY%z1`!m*+Yg9#9z<8r}RcJCYuUn&~k1nm@>R-bnOGgQ|9_+mi zU5$WXmS9g&^h;=w7i{c&m5lu)D}u+qx~~JIl9#QD0cz0-fmg3}NA-a#4P12h1<~mj zmVQ2p*0g0QmN3_}fgFK}My%SyA3@~q)1njw2~u?D7e*a*)-ekjLoDdD$Zu>a(!Pv78I?mGw zQDjM025je}7{6V4N+OM~sLAnAl-YidQl)=`BsGmB)?zEuh)kNT!6(M@S70Yl2)Pi<;Uyl!tP9Cb z8dcwD!}}1eY=3G7ib#Byi#dQ?_g;>VL9$=vNy5KjP0Sd~AH16uJ+8mshT=G(y0Ozq(fa)gmaZJz zgHZNBreGsKWH!g*2F4MSBzh!K7_pV~H>rcw&a+8s`qWl`_Cn1nCq4GHaD?Bs5G-0` zu=4yqDk||$7QQo0ABJr)wCQjW$!xtHmFsNzgcaNQ(h3mCUSr@$LeaMCt zChPfLy3Z_|TE0mse~EUvG=xC-u6F_^8=POOJ9ne%8m6k=%m@pe>n4P0c87g~3%|(d zetyRY7-D8L7rk)%2y~U|!A=FO!Eav>BqaT*T_{%Uvz?*LRgTSwn~4E&^OLdzZYoqh z@)BTokJi*K{oSIYow7azyhy@!jNw*!sb#w`u`S-BFD>dC;P?_I%MchxHN zhl%~WqYD`e4QX{R-MeOmK58<^WKkX9{bZ@jWGQ#yyzIBMw?t*Rp&uhADL0$cYO$FU_%2pe8Ir-U|szqEAlk1G}wLld#1fC!witUw#$l zorLRc8C4{4R=)f%+y{zx7-TLf zh_Q$oUc%aRdB!ZiL4aUD)yMS;W^cC6o9ywR%P3orrNQyj%Q?wRVv7IhhFvfA=KpEa z)Jb9`@1yv5>32i`e!B=fS|dXU-E-P#->zhF7h=wm0d^u^1R0DlV3n6psc(>gBKl~^ zdo~I-k?L|8E%=a_yyY$7vwFUv(j=179<9^a3*R6C6FUtDU%m0aZK#I6H$sC{GL6p& z4%daAR2?uZG2*^^+f&d>kQt^jKc?!oJmi$@6km zqW>Xp9-O@L&`GhzL!9{>ljURQ*4S!Y|y*#Ol}n|Z;D-RAbd*N zhPeCu_xGRbS}`UV4GOeCT%&&)wcQ^qJQ8hq0IHX$947f#cF%V@gW6M0`nK!Rhy?{h zkZF58NHJgQ#Dp5LMnZCFZEe`DxG+J0R+$GJw4~_(d=knWb;rU3i;H&h4dg9PdHL!ay-C{ZE&u=C{o=vW#a!-AzqwG@E^atTD5g-{cKY zBbID>gwY5%@9QA7w`2G`gQ z(4Q&A?_5?tYaVu>Ats1RnxE~cm6;_k(08*;MLh7TFxf%0x|NwH z%4=UCv#^6=b%>t*qPbwoSx413lEcudF(-Ub#!vG(MoP z`L8J1-Cysp4`1a7qavnUlFRH-HrS{G2~*zvh0C*4>QJId)VrWQvWf|=gD+N}LEpLP zJ4&}(m){?nA0F3Ntw3-TY*7!L{G~_jd6XVQ;^+GPu3>rPW62yjEqw;WN#%Fiml#)C z5@J(dGGxjaG1zj0>0IYNy7pUBuzfhR_ZYo>hi2QdfN1ma`H}|z;y@<7>ftcR!iGFn zhj9g!lOFn=#eX-znTjmho5cm|w19x(HkG~A_rPs0H=BUE^tBp4rUvVy3_kc#)uxz0 zu|76-b8QgaVe$Dt{{DyBy7WbvmkCr8PXEAQzB#E~JjU{Fa6(os#WLCWPg_4*E~)wJ z!kXCBQ^F|Imn?hRA|Db9RDgJLKCrz(somY2_nLSrrl3l>Fane19-Xm+CV;TOmYdJb zI^S+Ywo(4==%NMfy}?EnZh7`O=W>5GxdQ$!R*fwaMloE@oVp%JxKVr9ZFe#9GPrUb z>2ZY&q*Hcz+>bQAExFd3dT{8&NYn} z!$3tyx}Iy@v|(v@hW$165RqJ5OF=|Uu+p&ENB2E!mg-c+?7rW|1j#l$%@Nmuq>MN3 zv_&#gsHNri^rY2cOHh;1>Q2+nlc+=N$X~PH-%Mf&B_vuTS3%llDp(2O%%714k!C?d za9&#(E|dP81`11LYBa-lvIupmWdl2WzLho=sTfXd@~T&*Lv5=^5f*u+Qfp!gMN1ZN+ue(YH03Wt|{$R zowVfq`V^ciZ5*ZAvX4r|{$f`8F%NcFc}(+h8Fx{2ug5#VJ>7}^Cbkt-Y3Ry3fy~{f zmAm54WChzt-<9DuZB%QBOs3I)D227zR5adryZk7lI`8Vk;Pk!wmKQBG|}-Q#1onM$RtG)}k~<*!#ONdz^;h!M-bpfq{Z1l-lUN))=*Qs}<^SbT#{bx=_1k{;YDh1r{wysnIh>2vqJ_Y>ibO~Z*ADP(u6z8`+{RpzHdqB(%e#64Yt%1HrxLhMXf1H4mUu&$t{ z7E6Z#7jK7>Mk(((5y|M+Rl~QTsCTGXvo{A7@GDz?sdwkDu+asuny^gW`uX++A};B1YQ_%*Y&%nfgm}juIeEarB)B=x}02Xsi$nxo`zzcF`aIX++kX?In@GbRQ#o! zp{EX+iOS*^6S!~ZQFZ#3JuQU4*uX3vpmjuUltWDo2eN(-;q(>6_wSltD2 zK|TX1?v7~%jE}TT4A4^t(sdFZTim7ZwPlgD%H%u>4S{oCZMhe@G_m_}ddFq#(kU^F z5bqx4G?RVKOsLjNn!RT4A0@l{6;{ei#7DaFrFQY?EgdV_4~^9(gh1MwYt`@tqDhrq z0Q2dJP+#5|->M&nmPD8sd#QDf7elk4n32&e? zGR!Ihe?!D~ysTb=RJcerg`u7NXoc&#SWg{lPa>wZU`Xo@Rg-a|i#1g5t7$xJVe0J1 z++qc}mLB~#S7L%3N5N62B19$Ch>0vTVbsB^!8P+fLZx=E@rU7( zX(jLzQqc({d-vZ+CLQ3UVvhJwp;btFm@l%LKIpF0-^ia@JCCOG)F5H0VQ}&lLA+Ki zNlN^r-sT5(@{iMD0HX*Jh!3snjDNAVn zN#0d!o}QrAjfFk%5vEloD-@ZY0Q^W5b5je9~s zX39cY(xHC^g6TVTB`>JK@ddA&QTc>rDPUsJv#_QVwYDtsdt%2h#fNkNV4L01du?iLBqG9avkc!C<<9xSs#V$ndiBvkLjY()2vr!5|>aU%$k4|6i+<|3r#7**QAv+u9kMIQ{HSs($QQ z0b{p*v+|67XYgq~9F=}&t{dM}|$dL2BH>5^Ol|G}JvRK1f(Hh-~uH%^fwvs)EJ?Ob%omwvmT(8&r z6YY!2M3GQOr|a2t8-~t3ROk;}^_XG+C88X5O7dE*$PF~rbg}>Htu%y0-*w8z&FnVJ zQ$t9`rJw{zwHO2GYTX_xFrVof#x5|ECOfJbk?@*>^2Q%7(t7rw9WGQy8w~V5g2ZXO z15=vcQyM|1`W7;cIyA1Gx>Hs4^(w;!j)9&I4qsmvjxKC|xV=rgKz;30l??E^R~ zUIwztx6#$q!I75&R0bhJQ~eSRZl!H|1>pjdG3$&W+c5t{SX+_FTGk$Y6!{Nf?PC!& ztJeGF*4d>evJ3OW)A#=LY!ddv)6>Vt)5FQPC9u-U-OuH9@8js>%n?i-fA8Ur{pIxI zP8f7dHUjFVUeUN$rEfeYEpb*Y92tad0ka5u>Lt`nfl;xkN`T@3EqS-VA6#{s6-*ff zxt%>Ia&?p8&khd=s#njo7dwi4HsvHz90)3MfLq(WXP^L_1p zOaLlLj}A?f6u?7A^Y~9@1)502%07NlhQ4hsv@IXKdW4g;a*IQXTOX$ zEQW z4AibW3@20SN^&Ym0~ymYC0QpAT$u?(hERp+PZMV)Nxd$bgkog>I|Zd$l`U1Yzn!%a zN63RR?2(3}|54yw-7^+N3a^MOJkZc6I5SA{W7lCzWrLdp?JvK=3L%rL5WHOt(1YsOayTia6l*=c=tXTb$pYT85~Pf25Q^G z)%A7$cB$vIjM_-YgJ~tcyxq26;HwTs3I0QuAS*c%DSz~8xav6mjwc=;n-Zpmu6O$v z=cjXX$E&-qL&}Grz=d?R#^#0a>|zBfLv3Et280^%#~@M?Ox@IB^D?o^w{+{d*f`AN zO^J=il}%Dr{{{BIR(J32hKT(##}cCVZn}eGdKJ}_&WQ7XI+b`xd-AR0rhgH-;Gj%- zs@ih(r#!Zc8~VTpE=-(s+!-dOv|lxmjdGn(i|E#>Kp{fE+6XRyn`%YpCgVF+PN$W1 z9I4$V`4D);+%+7y%G6jB5_Uhb2=uN%M_SZfm(D43I8OnYT4WDl$n|{s9&GKl0$A;G zO)rQ|2GgF829Zt2;?MGvR=c?t28yR9jcbE2`VXq9*Bk+p&{i3LKxlJNU{R9kC--1-uCY>6EsKFyGPqcG zs+n(!q$(XY!liT~`FA>|8P3T7%c<0?vpA?_%LtwZnr>%Zd9jvXrPS8lXr(#aR1#{1 z!+6nSRrD4&FgLHQ0*;T1uID`dGH^%y6h=GL&vF=;$pXuwmI*O7MQUD1WS%Bum;%yi z=fgGE3k`3!z`ubGXp*|PI*yV?3r zNg=rG@>iIoMb&z-KE`rK{yv3wT~N?yLRGfU)qc#-KEkHZOhqyFo@W{k?k9dSBdv7| zG2)ytkQk8{JTc-Aa+$2eNpo=l{Zs=WzOB_hiyKl>N>X!j-h4vP75B~6D9n=KDRAK8 zgA;?1xbEKs8n<@uvM70jXirko3E(Km8duF2Ub=d%YB$9Q1S2bQc@k}O8jMziNpgAy zvIPdNA2?#3^#LY;BxmglN)jEDdDiHvaa(8-%-~GjA{uaqEsjd0<lh*hdkX3 zrZT`q@vLcs1|!h5nLhJaw;L)%ae4B;hsIVc(z*&yf^o$%2d|80?{&~}-IS*t`#wLD zKC=DY*4JS{DP5Ws6O>vp(qFbNnxKo|kt$s~80O{DxqY8>o-5WtN`@Nz{Y!|MEflcR zmxy^b7CTPbX&Ibw`_*35cp%yvFC3<+`r5+tH)~N#>noPQmkEu1LLNGOd_jYBW0i3? zV&wT3YKvAqcQe125?&(QMqS5Oa7q}@@#xqZnB^oejuh+ePrlkioJ~PTFPjErlzV=} zm@TqNrWH&#=)M^JBi}@u2;rR3;^(B^K0Up{8-I}`VsY%>Ej`fBOa6_VU4JU&LtK;~ zY?I!&zQQa(cfL*SD-VeW<0MAQysyjr_OWZ$;TlP+nkkoi;Yu8!(0oOF=}6*!<1Rbx zl8VftL}@W*Ss$)I6;wM`DP;7LuvLjdG*6sdAtA1xJX5qq9B5Qn}S18Hh;ATi~c5rZ| zB*y=nUt!RV2K)ps`Yl$;|yu0_>%gxoW4!jaFVf^Q-WXZbw%5*jnIDU#NAl|GZt>Ao$*Q_g~Wq{l%&E&W@wSrD!F1wLOIB^t6>q=)i40FfZCw z+r%xFP{-`7XTjVN@AOzc^QahWV)Mt~}5smh>W;5b>n zc*ATBaSy|4HSM{nfA&Vi(!?~xx44Gp96MwbL6~I-CHUu-Btco6xdnxFCNZnB$XGmZ z$y3Elh%SA_%B#*Rv42~u1G`x?s&Yy$>X=$!1ypu{t6GY87_WIxyRyHS^BFFCUziNc z3o=gLC%KTzM0(7E@ndag_yj?^gzU*is3yMH_VE(pt&5!ZfH+dhy-k*uSjDGIi1?gY zvbc)03e!gLN{XUr*A&$`O{GxU@&{~i-(a08CA)=whMAY_IOPQX8{h#ud{4FMzM{MD za)m%)8;gFIm`A?484tKI-1ZUR!2gjvI>;@4?+nKHzmRmuSL<+3bJ}WXV_I!e?|j#K z{|i484{LLstJT4+&mDFHJ6E^ z)hFA)E2eCQZPuxQ0wyR;oQ@r7EJ+oMeXs?c@K(P%J})W*Yj*U06eeu)RNB6mZF_8K z*Qzv?tH?9UF7t5RdzjiQIuBD*ykWIsO-r&$Sv5(k_T3iH*?In9e!n<;n7e1E#D-%? zIe~+g6|}>YH=pM_LKSB^4|AmvF<=0eU8WGUMMQ%_+sbiad_%t!DhZT(2Q85$v}red zs$YP4Ve*@HE9fs8SCB#Ll)y9xa8FVh4FgcIHL96P5(KwtD_)7)zwqx9-LaTrnO`AF z6})3_3h-r*!CQ^ZYP!?Mom+IJIo5P8%*TTBQ`af0t(A|LucaRYncm1aG0-2qxQiP7 zrtm71Gg2LU5Ls6OmU%H^Ree&UZl&oa1>=2V8&_o`=HRHzYK^jHaS5z-e~u&sq4I$L z4P&L8V|;aJ(64H;1YJ*#?p*is4m=MJ4jMhSb?FX%g6Qnj+NQ)!zAs~4_4XM(f_(Mp zY#IAc*(!qcP+Y!9!%h{?r^2m-i-THXLSTlFR%$|KMV1u83K8cjLX%-BlxdslFv2P* ziZbt3nWSjZb3#(Z6=+SOUFdOItx%QIZZiI1)(H1mf-bW~Sq2Ytqst79Aj8_`o^-cA zPz%Z=aZEmFAXH)eXBjkNs@Im^vWeu~-_VF5qLxt2lF}$Rnpcp~arv;so_bfV4AP^z zRcEa!Y#B>G_&!6dIyO(_VH=^MIAj7cj3Ba|*PSRE#3fmHa{Cp}v-y>%Q;V1HKN%Gj zMG;9=)qf_Jv_xcx=JM~WZDk^_n7$|}yY4jEET$k3_#e;gpYN;3e1GD`CA)Ud2AIE{ zz9RF@Cj+uwqTnM+Hc^w?)m|E~Fns992Tg}Lyux~%AH=6(#6FR}^LsuDK3ZuEE7_a* zVi9??dI9xZA@{R2h`&@hM*Hw;ITxf zKBYI9bS0s?4dc|C&az1IOH`-g96bzSPj6_)%}cb+l3ByA^5bM?r-&l(3MlvlRR@9} zYEK?fi;C>V&t-%#C^uLGUuDofn|n!ZBY%jBu4%C*(wP+}i(?#3D=P8`U$u}fxf*=;8#Mb$E9sdJhL9n>@OP&qyycsh2&Unxx(4&;wu`r={Xv~LUwvTA#!F*x_3y@T&(eIs ztgq4~CP%=KYvjVlM93&rXK%O1?)Vh3^(8)J+7rR@dDM?Afms+*6T0G;EffN7z-2(n zmL(SqK_&NEqvw5}l_@bLHs(^tPS)v|-iDjHIt`74@xG2P*F1`iWIKx~xj+6ye@sV< z@Up@T>o5xr;8-on^I+9{YJ6zl^jsSOGWmEi4r=&9J?yb!@m^={>k#Ezdx%O|UYqxL zt?(BYFTZeAJ}zNC!baDV@*3Tp`R_0beB-9l&ii&z>==CUX$%P1U*>md+b=>Ts&L={ zzZ;(oN**Fdo87oZM_qns#fX&w2!8+qIefc#|D~_Ubf9ABr@7sMDInVRLShIN+=VM-9xG-0^!uNI1o(mz&(%+2%>K{J z;eYQEKh)1(&USXzPV{E>F0{@jPR?2Cx=tHXNdE~rRo@-%ssH8mJl-mS+TSp!2SW$i zo;e|__{V~zgvc~09WgDYvDe3n1PBTLbs>@VF^g}@r^Bo4bH&rB>rrHBvm&27q1kv| zXTQVelYSE#aTp~?aJE8|6(&u-y#PoFPXUs*^n`3EcX2vZx~ou$vEO0KdVdpRZJ7N1 zAADtKvqU|Cm3@9Z6%?--6k!zQNuxY6?8sh}C=&C4&riPI{gf=wLX)MtzCbv+EXMBu z`UF-HjTVr3%)N_rtL)7tt;*#JNv0NA!a}ty?TVGW9jCCIa~*cTdOBrXxzgVUrwJ0+ znN@e>a$?wZYp->+SU;9?yKrU6Un;rr+`{z-k<=+bP!{%{22A^LV`MbJ7 zO+4jDPn?Kg{RsPI@GJ}xjuf>B5hAnxN|ntD9TAi9NJ)wyMvx|F-9kVnLzl>~qC&t| zVHvd;WSm?~3o6J?Dya7Z5RaU?ajI1JP8M5!(aJUF z`c+}srcZze%6P2MwmZj5s(%uh&f>6nGTDf}CP?<9Ct1-%7^k<2ZDGrd!Z-TGZBK1} z%ZzM=rjwQcAmDeQZA}Z9j;pa$-i%FQ;^xNI4-ZTuQ0l|Eh+wcw2=Mv(_UfMFZM;9) zXoqTGt{Anv#G&P;c?g94Lkv7euR*g*%BS_PQb^=ZO}p%H>B66mRSubH;^z5)EE(=8 z;)2&scb^k~`_Y^2mO9ntw^a_5m+ko&Njee~n2MJFib4HEB{D@Yre@VQh5- z$$fUQfZ-lHNgZxPBs2|<2M#z05u%H@c{g_*7|BTr9E$QKGX+=&SUZ1(!P{;7e%K%C z_*u8gOBzz_7E2Npt?!5G2(Uh~!orPA*}vI;b38s_z}*BU!;|;awSWlBMu12B2`UvJ zp>=E+NkB}qNptfuwPtGvoD=H1Y4R89y#yj#G&u+j%`VBaPTx*Q_HORk=wY}Gx`B-F zC-0c`z>e~)ypNvdUCYX)`rrEV;nxD`<&k3f-GgJL!glp-GGSM^XxcSnp<{OG^g$}p z>q9*4s9yuCJ++5AOxzPhty^z}Y>``8x6F!6{ zY+RYrh{)4M8%sioFGE9XW5pW}PdR!|J$TB~Geye$>~C&YSgB%H`YIiLm2hXP`g?G8 zicKL>==#{Bd-P9c*0sOcqprapv#Wj5YVVwWm-y0|5|1?f#LnkX*6MGA5JF_e`MLR$j>VJA?GYAJJ(K>8hDvHk-QY&i;sc2RW zMF&w{F?9sAYy}A_K-BNUzSekDZ{u<>HH;vBY|?3#^I2g~eV^^+RtUhjPEMmzmPkma z+j!eiK|Did<~QthOeNgb(<2i)jum+Aq!wHeIb9%%O2Kb>u03L zc&G&@w@V;os~J|ZG0+vkhNxcwDrN)KI$jbnkNSz(Domb+q{g-4ETK;z74|-rZWC46 z5F&J&2Cv!F|BVX9@V&&ycD#RU**uZtKb=)OQlgD9vf4O#I_;g2V}iVXH;x4bl+)9? zp&T#_USSbEvpj$ht<~>QHi3*99(!BU6}zEU3Q-eHeMpH}Fh}UwZN3krsqM^#Lt~FN zVPwxZYR%WTLwD5pYQZ#gy#5&mkbi#hc5riZaN^ILaqtS81Y}mOe_lM9#sb{h+Fk>D zXKicb1i4lYev+)1eUJ0g#i-I6Ip=}=7hL>>gxl#{;0mpmDSLUX z$^c$M!!-6`8)$Em=+#WSv?ndrg5-;Fjpu-8cy)O7#?ICtet+-2 z(qfB902`al&~Fpy!LWK7uY+O>swqSv8Xas+@UO(K>yIy)Q^sB-QGjTNiL&^?M4Ej9 zOg;~xE!H$V+n6dcV0+7Hgt~ZrES?6CA^i!U6$6rEh#mJjHc?8LniI; zhLOI-oYv?ZX;PpI+o#|*p$ICVZ=d>(#slvSnA!#=2~w`*%d}Lc5`k)R9bAWEaG9^U z!3YJ8@<>>EhPJ`sd|t_7(GyBTstj8xL2CsyODLaI63S$3^e<@ukrHrd8k11Bj#9oV zNn-#VsfB3Z_qylEQBM`6-vKX;aD67V);WT3fwcE$q`Cw&OzaaU?k=E}y=#;V=HG_u zRL9ijHYkJ_y(RNHj(W>PN1b6JgT3~Nj^qZ;0$9Cs%CDqVIFS0eua^k&nP($I=U=WQ zDak_Efz5h# z8Y~52cGE0^`OaFpFUbmrxHm6J?$O}N+&cZR>&KMi>}H2V`Ip2;{buy|>nGwfkc%F0 z6Th)RI)Yt#14*jS@b{|GONMD;ZfU{NR|ZpM;?Q?{aMKX!l>`Sl14~ z^IY{6@8unYgfy(2=}ne zMGmcpM20#(d8s=3hQC1$D`wF-@VM*$@O4hnnFL!Dj&0k2tcjh;#I|kQ$;7s8+qR8~ zZQJI}eYmgpu^(!!uIgU9`6dSro*}LNOx^Lt*f5lX70E*=@*OP`$6DEmhO~TbB-LVW1404uT4k z0$kmH^wmcb6$dO?fw71hagMbQP07g)l}|{@MMmT*;kknt!+$F260XbwW|44iox)a`hJA`BRvpmbZ|q&(U2bU&ib`ZQmCPpv3Ni<33ujUv zg_AgOrdtYZawn=N(kUS6h>OlRC2}Hw&pieG$ z1>JBc(0+k!EU0`sAWH1-QBZA$MF=W$p3aQUD!(+zT%69B((d4S>fhW}CU#7Z8SvBB zO~=;u{f;|dg9tSJdjveu+ z)Riwsph^E*-JEOeS%kguqVkw^fK+e5ju`@1YJIQ&gD^uA=N>rn%2-M4Xt!y>J%ZEi zT+#CI?ajHip-~e5g^dlXwg|HRZ(Mmf8qM8?9M(kfwpPae(%LgH+44XlU z;-O@x(B3n8-tT_kP;8AzvKi(+R}Po5LGJ-qD@kvJNgalN`r-#-w9nT@FW+OebVuk8 zsKC;$3HW;nNAL2gxc00@U=%-x1|~btJvJ7z!p4^`wXZ#=cj7S#V}ft>#h92O*@0Q6z``xS;Y~-9oCZyp2(7`dR17S9}z(i z9MRB_vQ_Ilv{XBf{3p}B8_I$Ld*m)g$iLNTFyS;A2TIKSLAX^myDF*215{wK4@Kr@ z2l>(y2a;){ zz487}ltl%n!Bl=k`S$;c^3P8+kdvMBkNwcerER;<5r6%K-ZVb2<{JHD&y~>fv}(DU zFr;QX#l{f8`Tx!>s-FrIsu!m=w~hVwri_pOaSl&bv#fLRH7lu?*ek-<0xT+;MSGY3 zSZs(^6|p7owyxQMb}9f=C}|tCU}9-nxq2YzX{(!j2I1A9B?Cc{hBMV^$rE^Ah!$1u zABX!p23)U+UqcelQFiA`5XX#c>mhU8qWkrLn<#&L2qHY1WZPQxv;oY8U>CO(g8oFH z_}awDFTT})s>(bAO{BR$)=0Fs9eV%}?+JPJ9S{fgvUuh=QXabw`*kpqQzJxgSh9mT zCVF21`o-kH$Q@tx5ysDTwTfs2gr6`_g`-sIA|1j+f*HKOT%UFhF7EZ;^t?`#%lSfv zzxS8ZEot@Iz>Af}9Ghv8C7ys3Ks7A~{hda^dCY9u)c*C2CVA=MP{RDN+4?etm=S=Iw6f;OpSxZ43N|d9!=JI`R{o z_Rlos=9Pkj?;YNokN2Ge^1vkk=O$xruxDW!o&%$>L2{4)l;#wh3C6K|k26uK(z01Z zZNhWGKZ2EV0iE4Q3RDU$ftS*Ks7p3YJ8y2xsK^-Hr2;BLRl))`r?oCY{6*i$JEZwl zdHgXwqG?27(ng}&YGyiZlf_O`C5bL|Uq7Gbxg3%r@sEo};p;&p`%GsF_b=0Co- z!bD3omiS`J#+(C>C=ad-DW)}c*S-edk&-bohO}R3WumAK>ty15trSKT@TkJ#7Jwk# zEZysERU#ula`E^u(4NLg6hOtKpwQUW4 z|M1j@67O?=2!LAvQJLm_o)Hw*1_hs3gQ4Ylb&n?HRBv}o>(@iu8vk!P2%m52mv@4Z z@BK*Syv@;mUP?P~^o{{@o%A+#_%*3on3!HQ-jF*Pe9jzcesOoz3haw-sIkRYXNQG5 z`w&gF-bBp#JDI7Ub`>3(F6Zy@T&>^7BKY`bjAPGNO6fIA=7JODZ{)?ExcbhGHdl6@ zu&|?&%9_>SE9sWJ%tVhu9MH&s14 zTT~23K+?aW{A*Rh=Hj22AY%%nZq8406We4>AY=R--ejq6sLZJjk0DRhKJ*~hM@#ET?C3*iRFL|w(sQS{bzzAVAvcZZ0HE!Gv1!98d#$%U5{oqEa3 z=+veYo3D6OrE_%lS^l~{l*$;0=tmwCC@aTfBCTc6QERj?dGF1H+i{ z-<^q6LHVim`TOFjk%w7nic8>z$sJ~qhP?tMD%9%Zj~HL|zGQih)U<(CdpU?s3GDS3 zBJV)81wB|+t)Q*SseL4=KD2q84_F%A6&R~#ciCOmmS|; zNfT>9(sG8=<2)TTOkAA}B9HU6u9gw%V&f8$L2EYa8>m7KQ1Uv^KR_>?Y*K+mv@=h# zi0&X69MaCg6xAtB%I2AWaSav2%UePTbbSHoDE_TN2g_a4waFc?|c?n6FQ3XRZ&@JiC zO#i{AP;|}NIL19!tfXYeN>e5e zXqi9IWQzDL*23S|Q^ttmEDS)-dtANieR$RF4ay2{U!&49f86P{q_T+gP~f1LF;mjZ z>Xu-?j2*{3-s}d)*%ojPS;=i6A(g@%#IQw_`2-svg~W|j!v#AxLF`!`_Mg*Id#>Fj znFigwvXuq?yBMF3nu3!-E6F^fo7@Hxjx%~dX;3d<_F{A;7J$Bg_-u`()O5l@c=#f* z%+a{fyDni<`7dW1yYNTE$a2m7=S8K*P_U6#UKhPKnYsFTU;04MQe}1(QPqQ#W1STd%fwV?0{@)}NI< z#IXwJMT+)Mr>qZQWkK8|_D)zVO7}x!@}f`WY9zTgp|6qr<)P+BIA&zvB+w->PMM2A_5vc6 z9mM3nq!C-y#!gv?CU@;bzW2L_rP-Y5-bB3%n(kzZ$W!`%$Z`TzFyq1Sq!BV8d zZg8HjqPXUjkk%ZL68@tP`oXvLj~QI`r~Tf6gntF4A>(%Y)yN$(T|Dyt{yN9aO*~)F z7P&~~s5p>t@V1N}8}~V~n#i;o&v;`-=3{ch%i%CIHcybV3#IfJ89oug^mtcr~z}fR$kU;k}@k zl&dgm#QI2{iE%L?BiV$*RvoL!VesZ_8zmrgyLEb4LBh$FMq>NEZXH<3>c0$sj<>oz z3K5m>+<3}r{WI|+pVxDs8;4;9Dl>Tn%sO?f=zGPdd7Dri8X1VOUFOeS!q9c5=fsWc z<-hEUjm13yZIw#azp#U*C)RQbDlM?jJ+0w>Nf@Zl`~8p(*#%){=P1h+nQmX6$v(9m zenk=dgTQLqNd#r2F&b(W>n|GExn=p|dA2@U-^6@E!?ZPwJkPwKC)RkbBp8ZDGdw|l z%PTkx+9fk_ps;J%h!=Z0VSNXW;XF3PWwhr;vWyfN;#8x@e%}Jolr{2r1GWZPc8x=( z$S{zAkGz(I2_1M!=G{;VPQzTi#^q#cw>o1E_p$+qSTAukW(K;B?(io5>|5lwr{kS- ze8aI8AiO6(rtRNO$p~#-^X!V+p>Yl6T^9-L*j`;1fwtu(T??F!;Dl<#4qj&&zxqC) z?kDHdaAeanVT*sug8E$Zc8Xy^n>Cxf2ok!^)4PR(WDjXU&KuL@K$iEZn;&f>4)GKN zc=`dlHR_)?&k6)zQ#2am5bTzF43>1B8BRp_I>;kDSDF#6urwP{*!L@!wQr`X_6X3| zy4~%Jac)=8-Z$i$t|v@Rbt>x0gWq8`?lw|-+bSnOlA*%Z81;>lUMxH)-D;xLXm>)i z>6`%H)NKl^SoBLp*M_;vV8I%WaxE7g^c=t$Eq~L;w|zGmVcXDx$6e`j<8e;&?7Zf^ zSf62t3(=Q{Juvz^l!bzeT&W~C|AoiddWJ&kB$LAJu&;$+&y0>oM{#1Ein(n{_(7zK z^p>^}tEZd{WxaED3Yv3m1yi+jp;``1LcwMuga@NE87CIg=2F6)?%2unP#ck)gk<2q z;w6JO*t2cddTs!yGYzQVgrZ3^V>^EBW?7r63r+FRZg9(%>C;R~jD>HCAnlFyzcA|i z|k%=k{;H$a5)Q3 zkItz3lP|jcF66K`ceg@MzYSIex`;c9a~%5Oc)tG;lumJjnPUoy;JV@-T*_`rWTd>= zEN`;r&Fx?-xP?_=3n}7qG-R8)oQ$K`F{aW~#hPw@PUL8I%F^-b?sm+B|LBu*zJa&J zwlz;#jcqXnkxl#|M4|V{qPA17gGiCv6A!;Go}2Mj)vK+`FG6r;8&;r-x3*|jx^E%E z)tAotOpuaO6Y7Grf*ZxQU0E*79Zs^KgUx-Q}xJ8UEM3JH5I8;#Z7;7?D&K5 zjlNyMDN|lIi`aK}ZQ;wAO&Yhf>UXna`h=;tu}ZcPbarCUg&q?42Vbi|O4HnD4a1-2 zG}hb1I0#>K3^=M}<8#GrloNY^-pJbM3Ov>w0!`NRjxnzv2*)gB=*87;FPEJ7(cbSs z+ikq9==Ntq$BAqEv2Vox)r*WqBhw#08xg`8|6ddD$ASTJb~3m6DHu1pd|WqL<0#*H z_`s95-i6kbB=07FvMWlm%#7UgwKbo{9qwF=gG>@mY$8{{Ab}nS!mmtyAeDijVy(op zjkYO@4pnETv#PZ9jYPkFmo^zzQ#cH3X2|po8Fmu z5vPV3I$1*bs;l{Ll+JdHs|UcUT9}krik@K9M=fz8Er2VmCJobVL!G<~4p^3|%EbmX zs(+09C;{#}rZ#0A6YtB=?8o%xjHmgn_Fa`|QQXn7yu6%%7n@~Qw^3D_yW!>e*Ujnf z*6jGu2(kQcISn-c`C&O3S|k3IZY-nNK$x$Fvh2>XAABsue=eS~`nC&0s?H+~WeoF- z3)ATwn<`Doe@YV_X~JN{NX~betVZ1FCJxUha#R+QFowVQ=dN{Q{(XIX&11iw_nn^h zjr~NLir#%C;&t=8d;N9!_&OkU{|?5>{GfP!-cMMj%9E-pC3+}E#_^$tnl~yeDqa

CD$*CmoF<(j3m8~@B`NMlr42#cOJB;~rl`rV(M1Y=a~kpS8z^|%y;B~^82ah2=Z zs!X>k@40jp{E9HFwJE1*AUjd4RZdnjPy{Aa9LJhQap&=iOQFIsD>4_U^cDU&(#{`P z95%fPC}MLi?SrbzukDOly1PAnJ-$;RJ5AM4hAmhNmITsS(whmjV#vLLj{2qRC<;_Z zKJ3FP)U}O*7eEXEjwixp!oypNEF&@egFgxAid$zC@3C)@SLzKd$`7hE9wT0e+URfU zEj6_2{7u~DZ<iuD*>EQwU(!K2=HyFZE8Ne7-z^}BlS|M_0;`KZZ{4MQiFP4%;{?|{O z^A3F&ty9bBB2ExMrj>w)bFrq!DhtCFUJow`b1I#EjaI!djW~PqG}DPhveW~0_Tp>$FtS8%Sg~B9 z75-n2X_=TWQcus1D>c{K!>z5Yr6m<{Od~IKrrAM{IT>J`KQ20Ps5%)Ck0H=#;Tc+z zq@3KSbkfjW`U=J$DBk3nnwltKNl8+%*kz|t;m9DB{!u!8`vm<^zlOYpL>!sy^2vy; z1k#~FW9{pUaNR|yvs1`ws1=w5V22(TmIJEv#riFV#AR66FO)9gQKS)855PC6(_)QG zQbFW3Z&{m+B3&M9D8+Xrw^-`m`*E8pwpVI=&7QHn+igSn(vV|_Udqgh_RcG5kRP}| zEc&X}Jx+JUlY2L(>gCCDlLD;jB1O+YL@R!R1YfCyU=uBvY@J}il1|eSqJJXX-R|x`yt$4Md=_sV&9_= zne!r438bXx=&1|?U-RZ&R?bm+F+Anw&xCl_q>;vBWQ0WV0VW?fG3XrA87qoD&6}*; zVt6`QQsgFgMBfFbHv!#a{n6qwwYyUsL=bS_>|Ds3q*EM^NRX?0odN4&FETqF1~&}Q zN~M`bn9VXt+{be z<~b4CfbR9tr-FBm>2V*$=kob?;ki+EZ+ptQ^EGC*asbDjpD-0?k67mtiXe9);#o7P zR=3F_os<|+vQj>2T#^x>q<%}~AU#5IV>;&|13aWMuC>=`uAc0vz)^8o|nsgE;NcE{;Qi;hqu@ z=b?V~t+IXU6LGz}dG^jEkzn2nQ*3K{Cou=S6HPp3CGwObH{ZFy>_)1s2T9ExU#kI6 zKK=7ixf)LFl=5WgVG!NIVA#D>Bh-l_^cHOtk$Cb`k5a^bJKCnB-!h5>NYB}*DLtjK(K{+ts z68h|yEj2|9eLzmSe~o(A-;@T8Sq)jOgJesUln<^1YE0_+_1iM$$-f!2Eycx-bPEi= zfXxcC!q*dnvww-Y9Qt$Xt}FZlV{ZYTHG|e_p>$$__^+k!_QOjof7)=91I9Is!3R{D zX>m^Km0sJ!9;s&_9pb?S@58TMM6(W>xsXao1t$j-BG(Ctx(eDXAlohkX(=}#S z3R^~htVP%2wY>*zi$V*mtLbW;I(zrVFarpcZKJ0&oFo9Y!UFu|Kc0(~o@z@xx_eSp zs}ezm1b{x*cZgZ@J+$3{wDLVlV|sKlbof`aKb5>A>rRC<&TMW8Gq2&CKw8~&G#YC= z6n2J{T*BnAP`=F%*B-$P%96ELwk zRycIjIOUz#^fIvU>l?iSVbbp2cb-5n61k%PS0L$ut1UHl=c4v_mWpClIw|2$ji*egOEjKzeg3EBz-DSQFJDE`A4A8o<0A&~< zDW!uZ9n(iX`GT<6yLK|-H6{`z2H9bbu`_@FKh-@{sh#m!iX>(NtUxoi-h3aw5%}-| z>WjKb@THy+Efx-zwd!cFL=C3H(*~J2Kf}d0#kCUAC@gif(ghpSDS!5#f`RG>^X2B@ z2Z>jNgM~Os;Sn<%@&yd07sEePHJg5K zr!RBg3;+jT{Y;WOJ>F3G?u@IaDT?FG8FKuVq^8_mM++p5FAE=JA)Emat;A%H?d^rH`1)ITgQ^(&rrUTK=-Z~g2Qk;s zagWRSDJ*_0Kgq#=i)e=^&qe>YOZQbPHtG71|K>%l$Hq#73v6V9);Ou2&1 z91__S0?3oygDMBdOtVx)2a4;cZg7rr1Y9d;42Ob!H9%CMxuuIRt821Wg^<136%32K z;$juQ%V%hT7Ip|?+j@X$EHl?Jx8iDWhzyq%H2+TZDCgtxfK^rXf~b;fFYCTJB@nJ4=B!v z8bxsyCPUIAgSAa#w~wN#CUu9Eex1L^=lg!W;9_SWq-7)o(sblnwtir~U|IABxCCj- zpDv&mfdH0Jy9~|#%}3Di#n~RiM=j6=VmF9&cGldMIO@-?G(C+OmzDD0I;{ivdGq@0 z5j84&+73&bqck|n|I$75sT8L6(9!q#aCVmNRM|z`_MP-Rbv#l}yvE7&b_Kl&6s09z z!Y+$gt5-gDwVcA`cfcz6T{?Rsn`rcr5r$HCg_anPn#@@9ip+ySO&Pw;TI3UChzyhN zOy!ct-p%-N01AU>FVRHhp>yGCv> zATVI0^Ml;7P#Y4LXbH)B8tPFiB?vvj_4vtPCU`#e6J`fj+hak^ew3x;16`Ah0~t{u z&|RK0*f!WrGjl&3`CkeYwrVD~!7!Ujt&V?=y&aa5pF3Ptl{T&oVt$aV#E*N&|KJw} ztMC9>Efa3YUhy86aN*|&m>&eC9yhyg5}ikpMzyE`w${6KO^@3e3XjPQDM}|)f#r7c za#YNYpe&Ww&GM|w( zk-A>|C;~HM8_=6_V%6NGj}i~6|87)nnQA|LGd55CdxmJqn)M>3PV@%2Y z3lP9Mc8Qs#a5D*KVdde4)y^X)++F7Wcti$JBBK{ZN!Y?2^bGpmi6J<7TxZ$R{w6`-sq48E_s(Jf58jICtG)wGs)1h zR4>AO4>QH8H{^Asl$xvv!;nmf$LK7ye>lCEn!Xmq?|0NpSK~E!Gm?79hg|uXpk$+5 z7h?xZmeA?OTwkDFK|8yTtd({bF!XAw=*FukQRT5q90YBF_3;%aInAnok&|T8sJSzy zNx9TKxo*0Bn`P@w8W}~QTF<{8a^sulhFr`UdZ5oZyjL{kB zg^Oo-iyBC`kO8c+7|bO~sr@!X?yrU-$P#&nfA9IqU{uoIm!FwslJ!wy^`xoPwxUH|qqD_Gi>*q9rn+s$k+!@> zZOq|$OorRPqQu(6U?+gr4CO_jHLPMuW7oZN_*JJwpkCNFW^1|T9$KmFrAg=XPZi8B zrE|O>xx<2$XZTxrO^8x>2|oWhre|)uCny)*l+ayH@R#mLApC5V+CdFR!} zU{{agb!78W^^r9(dsfh|S?AxOSf{O-A0ot1i$C?^&U0zBb&GFhH_!676j+6%2dZd? z{2!fIFQ!LU-Rsan&{=L!Eur;}_09VDe#0$inSTnO-=A#I#Tf-y9C7y=G(gcVRQ3$a zu1hlsHlmb`0HLtd{?b7w1Ac&wIwPcS=qf_&=lrQA*{h$LM`nbY*%URU#X`xL$_N#4 zcsoR7b(rdDt->Y>DhVYo++p)<+f3^zaUpQVTjN7-zsDZY+UznBevOrGfnFNI=I=;62C=EJd~`jYBxQP4DmgJTbKBrG zq@c0b`hxpu!QSa+J@%fm%7NQ{9DgE|0Fxbm>pj*zVNFaA*t?1}y4nf?5!=Dqpkj8N z4z@i*k!kL+%4ZU(`@q7UV1LYr^R5&4tWlS4lW~^m_&9%ucy{=z2`nFS0ayr(ihxuU zMJ0rfY$HwcELeWekw*OvEB%P*H4i+FSW~sRx02(jZ)G9Ypn>A-xSsm*xeQ@J0R3Bi zm#mAfe4S4~!CQ4DHX+zHE?z?~&%whiFpOrWaU=C3CLV3up0)BGr^b(1bMM`=cbz)gEI{v3VCShb)8U52g!MrXoBdLqmcsdCP1{KUrOrb*Fr zK!+HbC(=u!m7)~yX0WXqq#lewElNy_D2h0@Sk`cLnw8i@9&7{VYfVc=huL0<#@_N} zai-1ak4$g^TsY7J@>@E@a3p7Y_+SID`hoM9(1n2b8|pRQp9BGgGe|lLV4#6&Y#;qD zAW6DwX@&$y>49TRFx|e<0jTgA_wyga}AxUG71>YVNdkaP_#^k+3nr_taDrrF`IL9N13LJ-Lw z^}J^<>BXJRYg(nPRpKCM$8RDZr585jm>qjcX6~H1DdswIYeOx!Q0eY(YFibyA ztamVmMKI>1tX;pRY0^=K7^qg-Hdl3aNE0j$LBj_=@{5E`mfCz9LR9BE>1TP5b%|Pw zRs2=a(X;RM2%V0_s>5cWbZ(80ka&MG4-?}<*(Q0)v~)~1Az#e9b<}8Q#m!JLbU?SB zVebq=!Ke4B!NTMP4R0{yrzEM%*4iAEmTZ~saRP`E-YM8sZUGhSEa2NNR~wZ5*Kl;I z?a*Yj5@Z%_tW_zh|J1oT(#-^m9N3;h0y7YBzdL4zG>E40Bv=%G@=)ge z^yHF*Y0I1LW3Gx4#)@^B@8!U0oG!8iZ1tQ1PQWM@h?yP^)KM)`t+yPR55Kh}uUz}(BPx>uwepw8*WOQO+e&AY% z03zrQ2^@N!P)%%b(efeU@0e&QkMdfNzI|b%1<)J8r<6A;0cFhhg&cgozL3TTmP-Pv zUrD2dJ5yCzyQQ(tZv5PNjbJ0qoB(iJEV`)qyf~3U9kD7w3u$vvTTM15#ggkPcX3nC z51+4_fDrD=00Lq`B>`)vjQ|AVWx=Z1ue2?-E-J+hI5FK>Pc&Y;li%>Y?N5V8KyDaQ zW#e59@2;-^01WMKZYEth&Dl13{XI;|=#e9jdDVH#JJTXYZs4hiByA}=HO(!@W9(Kn z0qp%`!)|VhEV_Gpfg`$#@k!qpRS8xq_w0xwsj>Uxl;yWE!?<)EK?OvPs%J%~LAH`% zp?!P`s?$3avc8Mw*ex^L`*dtc3MNjo#co#@Ze?+Yq?k1z+aITq4Ywq=>I7w7uUU`uLhehMwZu+5L-*AmZwm z0=2h;TE%uz)1C`nYJaA;MDmA{;NSTM=i}*Fe5}|(^zaHY=@@Gjl-&-i*x8x$l}D>N zH4s4W$R|X{Y#h`vY|Lpud6of^btE?r8)lx=2d2l@c$%alv(8%?(UNG-R&5*;{E)7{RGnBQm6Jd zD-`_5Y~)e}JP|ukfatG%7%qZU5zJjqe<^uo?A);K*oYbcN%h-W_-fjL%a#vV^epSh zT1V5t$Xz|0h^>akL!Ufyl_ISWPXJolS)_ugxqCFCQ%-AR`e7PZyyuh3W+CU($*J1p|J7kWABgb>L0&0pY2ZNwTh9PsRr66&pAEtuq-E&}q zfbHP0afwghmyrFrDzXwa|AsrIi_>k#sQqsiz%eFOUGw>tEU(F@YPtC(P0aYs+1AU% zXrQdDgEdghiG{`-Ll&TWH-RINT-=1>{oQn#E9o}CpG;0aFub_Uu z(!IGb9qa6JUBS$4owUCq|C#0W;aEWzag5b)(1$1}v2H{aTQ-Sv3*BBlo)foNs+`Gkq%rCV;P0ObAYQj}s1 zl9nYc;@SSGOG_rs{nhV)-m5_Tz*1;(Kh9`|cHLOhPhKlz){uGHS=cbjw!d2G%{xd} zM)zB2(RehqKp&TxTMAd!iSSVB1yYpps6>JD$gw0+%lu95YN)m?T~Gp%ZAxRGKrV%G)rXJ26FwS#Qn}pO zH==oaZ-c+Elc^_xMC+jX%;#pST^KdH>IK=gU?yBN#L_45Q2pbcnh@~*)2mBw11uvo zf0U&cqCpN-018IDqHTEVZEZ{sE9?7Pt)FKM z`FJ^$yXVf};<&Z?=$J<3QLqPwXeA$4d%WtRR+%>v&z{QIfDq>>M8GCF_L`l7T*vQE z>g*39w^e%aws`MtT@8J)5x1hW__5s0 z!74QIc~(hWAx6=kN)NGZN{u(}I4a5{04bF*46!>97bY``X)s$GYaI-Sab4avxlYq4StDpfXxTCf>MWYm6wW{4FnS(}kq`w(nhL)qs<_OZCdQx3 z*gA-W+z)faM`M+(NL)Y2%L&R~*;r$*(_i53Cf9VY^)2r~b^_fO*WR~jR!~mAK%AeW zlh~gt!!AJHh%9Axm*S)&Pv{yvbfW7H8Z(*&gWYcmOo3PBa+#e1!T&+i`7+nj{5DlrNTC`*G0C*Q^aUMECGgOFZcbOpZu(k zSiYm;OMf0?a8BldF?I%_$WxG@9-+eCBZ+%%F2<&Sz8n@od2TveWv(*Bgn+91P_H1| z#dmOsUgs|;BLVO#*s~QlSEHlE+>A=W;E_r|z-0$Mq_%fb1DDN*c#w6W(E;Ge^CIAC zx_7Nfzf-hQJqCL07X)wN!?W za3QW~X|$a~#ebL9sF3*xiC%;Qu+&BSI#m2 zs0CBfoK?IdT*I-!c{rU$gtl!II+PZ|tzid=snl%AWp2Z|bK_C2eLvN1+J?Z8oDG=* z3xdm*w{h=_Ew^+S zb#;Ycuvq@FvxWN(FY7RJbRbH$TShy8?w6{(S`X_QfP;>9ZrV#aEKWW?Yij^SD9)}h zZB!KL4k%V)?xIwB2v^w(!P6o(u|r0^I4%nJpXyLzo)D>t@qT+~Ktx@Fcz%63n0miD z4_BF@WSV>j40|D z5s+eJ(d>!HdIm*`pUgcN{0fe-LarwdDrU|VFbO?07soI4oyI`xg+wuOANT>lqW&~Y zF!`-kor@iBJkpul;DESmcDKX0>$o>g(UHIhX3)`q>$kR zW|;;-hf{?1A90v9{bRRQnP%rJI?d^vqs~^pIdW2Xa_kX?)hSZvRPG*Y6}2E%7xcU^ z`CCJyQz~UNI~Cbgpz;ia{PGQHLV!C@zR1iGUWE2TUI(H{Q~<$=Jk;F2zXX+8f>fYV z|EgFHa;rOVA{D7k`@|Y7YJ&;tupm7D;K8(37H=Fi1N|j#%xa#z)i5s+|PH5yI#V zcIv@kTN;ftJg}EPHQYD~t_G0ySq(Pmtnx(eNsABd2CPY2qPWUQHqL?%s&VY_=vY9T z#Ce`^9EjC48iMU=9f!{l6A_MsGjumc7sEqlk~?kndP(^tY~FqBiVpk#ve3poJY^MG zNyrN^LHl{gf!`iTRgoj;NwXlKuWZ>%^za#dyhEF_~^NSi6zC6}1v03{UWLgM(g!1n3C&<@tw@ zAK<~o3lwwM`5sR5i~3%>#q!ocSP+6 z*BQ6)J@{*AX2o0tz+AgAc@H!B=OS^EZ%Cn79df^W&xxkrSlJ`a;zu!dfq<6Yu;mVVTErbrjbrW=7wBobyyHEgK5I1sbL#mQP<>85VKZG^_k`hh6 z>!p9yX8GipzfOYBx42Ay0EaEKkHT>Rm0RKxw{F=^A*nFXLXYoWY|XOAlI^xRldN95 zO-6Vl*+VduBPnVh3jU-g|G#^n24K~}F~PsHuAG;4f$#A@)5vBy=aILlidDIwiE^(f zJ^^`hrK=m=9{W(4jV`!xUJgy_c)NzPqM_uNQG(e6XVB_P4ACA^Jpd9wEla{}MTWo1 zZ)!uf-<0UJfZ+^}e7Dg-g&Ng+WD<7ehGs?Y>cgrJbJTC>_7mPJRe2%k;#=e3bs7<@ zOTtw=9kmZSZe}aZhvq%2g>`nTV8utqW9%sZ9V1GeaUd{eLb*Ep=Fj#grZz_cRm6Uz-3a&WNmi5m0LeCMEx7rYhZD?qdLZh0 zLTwkKQe|3VMMJ55_j}c+~8Qd0VGq5hE;Z<>a?KK*vgF>|=DWaX-8-XkFmnOX(gAK2< zau|=p7hVRH#!0JG-A=~Q^w;w(SJhgwodVN`zvYA0ozu2VpM#WJiP`zq&s?TYcN@Y( zj!7uI%rOF2D(mQ+`_ds%%-4AjS_)nR+~@5evX{sLse~9QYb2z z{zoVc4F_Na)kHV@>4w!2G}KlaoWqu$r0TIQtLGG9>Q(Vn?kU@~&K*xEa%mu&iSS|q z21LS4O?AvdJ`T?*;u=n@bWLJ#;XLl9F74q7B8*+A0TZkX1|QB!j5Z}gU3^$c-?LV(;)Gk6+zat<)NeGP z1>E4x<>HPX%P|>8F-}@Fzduhk4x%0!YESByThSxgO-YD3xCiOd;<|RK2zWWhpKf6! z)S7-@m+{K>?6~Yk{Lp(-{~%?2&X!!;9pX>9O7gg_32Z|YMLjr1vShy{KX9>(23USq zazta!W*MKoqj`dct4d$7T)vGpUF1|!S{rB^d~nAf4n9cN) zR@=61yQkgL)8@2o+qTW=X&ci$ZQHhO+xD$}?mhSX&W`_a;7BofRMoNKD1zs#VWF_`pYIUDfO&M;(QL5fGYl_a(7QgB+Qw@s- zd_!)eW&{Lki*E{K>gO20D81k#WFTIc?dmBSg=Epu&fepCPec!<6;vF_RY8U zHi?lIM&ofu#)yL7=SajNrsM)$5>v7q$R!x9^2*6vWu<&e*aP)a+7gHrue|DiA{!nk z7>qJ-nJ6BB0elBn&wHYkwLP~Xvkle&UwmCf1!OxLADRXU-6O6awN=07Cru0P`Bap- zx5L5Y*kt_?Ei(Mk%DH4z%X3+jXiAfWn7}IhQg`c`koxDR6(%!Dtj3R;G_*2?9VGp- z65~vR4557_Z5ruSQ_6EUDJ&^;M-2O7g@KBX&-J!T=&4oG@RuOpx(b_mB&rbyO{CO4 z3YKKGJVBo(Utt*^4_GO1-@o;pS&>*ZSGMkGc3$aHQgYepm{u~0cR!Wybs1))|VU=>EwyhubR$;g=996frcDeU(Nri7lhsmdFjnqESQY0JqSK|Drf1# z)yQ0Z{#EZIt`h#Dlb95Q54_Xo^Swx8X^SAoxJn+T1i{Pe4LMFa1w*U$+y3f;Cnq|3 z>jZxg-?8isDe8?bYa5!E>wpLF--4;xpCNIgk?8l!H(N?!iF`Q2GJKd-hgWwFm5{zv=Sri4+3_mMDlR(?%T>4hgG zFFh)?cBxl#g9pU?H`Zp0P&)&ogD$G1`cyZ4VmU;euVM7B19AL6V* ztO`Du+upJc?I;`*)?W7cn|j+9?{3`=XL&2sXTQ;(X0)VpXO7tVmpVjGPwWU;d_Wo- z8eCpKKQw@^MmNQM-)ZP+g9l6qntmC&3lV&uC zB(U(IAeGu*z|hJKPM?W$r?rQy&X9%a<|)zLn1+zqZ_lR0p{nBK1ivqWy?GqTChIxa z+w|ZkpcEtbzyFFlo}4}O#hUPjwzQw3+@Bhp-_aO2+&%zc?Fu&v|>4*Afhg$kJfvjx8{Uuht`)M6>BYvOr^W%bFd|InoEaM%Tx&Q>Ih zSHNZ2YdrE>M#IfZpM$8!$=@|50;ZB>CiDgb`R99!cft>vd8zXSok)>Jq2+r+Xj!vt z`xl~jNrbPinZHn4To^iYnmfBASo={B4;16glc#jPk|S7N5>czuH*H8)@>7U%lgN)w zjsUmgkxG2RL(SY~htEV*&OEeU2(4P&N@SYS>Qj&(j1hEbka@zD*d6(bSAWS5#W2xh zQ58y8r{}SpQFg2%n_|GT?7(%b*HilrAkXOv3t*G(hmwx`WOk+pFUlRgx}?pP={5n^aM`Fm^Ui!2612tbL6^bC%D~}aHLNZ z5pXFTx_H{7R0}sCLX;WTFQ@<2IC;O&L$$}Ma_^q<^Em?tEuX5)sTr2_#w_teJ$k)M zZ9#xZMZ`J$rl`sb59{p+rO;z^8r|``-7iJ~&Z@Gzzn$1D9*=g`$9my;EQK!G!{8I< znpUx{q*%j$(#-aR81vf>KM_Zty`D)vKX{JZ)=oG*$1zx2x}cSU#Y=CB(Uha=pptRw zkOPlHwl(>W-Pg{-oul&2v-A|1G0&2xR74?0E6IX4mmi9c?oKq1*)Croe)x*`b^^ua zLv_EyHBGOojD5dQrj1;}1 zUw@-!K_o8u?N6qQ51(Fkki#GL+1&AlUO>(2lFV!tnoY)NPKq67UP11}XK0kQ;Leaz z4TTQa%r97j;38%yz>mpS;wQFSNzdB@Fq9G*B7caFseE>0;rhbx#g>q=U!z^132sf7t#U{4772W;E&M-)3FlKinJ9@DzF520=SW=_T7O7Ue)!p?s# z>w9dC^yany?PVi-#du2hk^(7BHY-?c$|>rj+jCX^I?gAo4PCpFod-Dy%oWGM^Hw%S z!PgV&Ya{7`*RzY?kuO3#DV-Xryzf)m`S9EghDNYvFwOuf`z1)}bViZ`M6z>S!yV4@ zXyj}w>OOc)hCdE->Zf-qX~z;6`2jJ?&7O?X(y%-T z42>`P{Ze#t=|Qv_u7^2o`w3iJz=i^rlOYmUa?C~T-0Pp9x<0`@64JDUj}CtJV(u<4 zDvmaRdqtyVBxA99Mr=JwB}bFE$;w_F!!gPyPhtqVtN`e%FOXb{Dhs!_fR&K8Eze~V z>!H6(+hUG076`bSp-;xAmWPw7hF6*a0W`ONXX`jHC1h8+g}gfcdLXbIf^a`2GdSN& zr2I*B2yu?UCNL6F;i1an?HPTeX(`|m{qi@oPJ~=u>C)BH+0Xu1EA;q0#c`)Zuwy3( zox_y$V^oJ3S`GY+R9;<$|J3MqF`t&1ut~LwZDhdKlld5JU8BdkB;NddAlB$>E^hXz zTKH`m%^3G6OJ?9z9B}`wKPvF{DUEyOJ=_p~OH#Iqt-M9?ufqaE!Nm7YI2Dt=;I3WL zM=SZb;PiMg2*_?BVCEtmkLe;jwas|rpc14c%R|~y#t&hw4KE4BsvL#1Fe=qIh-=no z7ZKf_HSU&cEog4ENe3?W2xim5VlG&3Qth02D-o-sQywMIQdZY<%f}r-HtUp5>sXIVDNhcC00LG=Uy`6&J*i_;qzPFn`ul zgWH$063QanJMk6UMZXQ@#N3h#N`j}O>!HT))#k+ib~)yx-xN8lwAWHP0jTQMCd{wA ztXR%RnoiiMloe04Q6^{Kbzew#8j&m!r>7(@@%7hpR(2W{(-*x$Yhwh&E;=g6G@-fG zx|sYs(=K#{@}N2WF)AY*wBj_b&2HJC={}q_B%-AP>k4Sp3$;p0Op?*-o>x><|5*Fp zeaE}H7)8HBx$Ypz|tD)+l6DLubf#2BpLd~z*rL%*909CJc0zhCF88uw zGjABWM_YJVd*OKp8z=`>u)OIGoQ_pi9fi+`GM%>+{8mHlEnocaKes*8)5lXg{ znEq$1ZkOPy?#f+A3$> zXxs~@F5x@ZD(_cpM4Ff`9Vm`F&-B)?o($gjWruOR3`iW|5G!wP!7#$Kr`2a%ln!81 zU{%AOidN#1Ty)FTWmy`98fE)J64SJiY@FW&^jcp%Jc^zY4|s#`LJylG%K$hP`(5j+ ztFLH-$^i1IDX&MU%%D7{mvLk48r_Auvz?Z%Tte~V_{Wl8Jq_eJ%uES(TTyJ+H*$o ze9$^n&Xz!{B7uEarmF!y%WH(%b4glt-I^-fT~KcB!cl)y9PQWpX^eu}1$PQGHG_g2 zK5e4p6qCapoHm*5*MhhB1yp*dzSgrwMS6e#1k;Y!O&1Sy zwlp&`RA7#FOj>2GbGS=0{gwrF9R_6=C#Or4DHi9jjJM3`;r)n*EyG!_52q>vBJEw| zXh35L14Cr46WifC3ym46Gc!(inv6U7D`xL;OGH~_%5Tr9?)i7OF=0y}VMxkuZkV8H zuutwblQ&|e1`-I}#D$D53UkAp)IsGonT$iP^y|--v)-eYdoeu6PZ=21B;}O@lM{$SS8mUdeqK!YK_{oX_$qU9nwAmW`UJ+}@5JM!t9=y>a9)wl13QcuXM*3?04Z1+4u{MTS))ZP}cs{O^I=c zi2UI5#9`0~LY#u>0{ixX?4_M1p>g9*Y=466D97PBxGdPeZHim2ttrvMNThNXkCII_ z2H1hTP(--v24O>0diXY=S(FXlUkI}Le^12CNU-T%w^@kb^VB>R0)?$atsG^u!rbUe zzlBZS7LUMoz;ir8DhFl^%e_XU!;6QF!ThUi;lbnn?(Ipii+>bRz}p>I5pEBx%qHjM zG(&HBIa$Ea01O7PV_zwPo9y1q7 zpX)4*&=(ZYpXe_KWPSa-p*C)vUnHPS?N@%#w`dlN4d7Qh$(iEA%LzxdWg%IWtfwtJ z5(m}c_4(1ko#Od!Y5JB)PB58N>cW5XaZpMCr*_?%9<;&-tJnLt0K8(2wQN#(M@wq(O zpl<#!WuR%tOCJBncp}SVb`(vwbwm^NJkcd~s)a;M{j8Fwy+=OgeCAd4Cr1f`@`VD0%uC0O= zA?EcatpSPkZ$%?3>E+78(*)Im0}4<(?y7C;<@qz05yC**OJ%_(suzheh*EW*QWpR( zDkdV%Pe?*-i*?)S@5}zg?!1D<(9Z~nbaK6fcH|zRpIC7mj{r1*Za*I+2qh!ZwU!nu zHWo%o5rN^TKvi@_YLiH=_T8~Hl$Zjg2TpZ8n`Xy2wuaB$K1s2qItrB*MN(gND%p`o z?8ddyl&D@Xo=PPIP7AWs6&21_CukfcYTF`(){FI*g{y*UhUP>Zqfagcwt*y~B>oy+ zE4)c$G?11*MfEA6{_o)p+TYNkaiarH?536crnNDnKOAJqTIZ07pINX@eVI{o0-Th3 ze~~k_(!)Jr5jtF>YeWruPejVpRw;*)r>6(`GU265GM9!P`VO5l^TW1l!KV{z$MZwY zf7z~uf)fUwQ}pG*vEU*~{xsse=eRvL|6_d+zZsNCrfWtR??fp0!e48Yep-pyaOzSN z3f*X+7ynn--VJxq)9aUSW(DQmIR#r%^nfJ}|VVO=~ z>IRo9W*P=B>znRsAap3St_Re6WGp;;aI#Mv%b)hyo}u$zrsvX}_``3;KJW91gmxfVlv*d2{;L~(8VJDn*!mnsPsZS+k*<}q%Nja$=OyuO4 z;8|W!_t%%}pca^Acdik_Xr@geUkAxW$rLbHyN=U8eLc4pfM{f=5ZHY|Xb?DoF#MpV z!jO7bqwVnqdOg0}A&xxw%CTKIR24HczRlBFD6Jz5bFGakYglCBa0N8fj5 z$?eYarGGy$w8{<7l<)*6({*P0L+GRU{N*s)?eCjoScWeasH)e+kz8<(l+%AR+h<Q`5e9gUTp zl-<~1XyvB3Dk)$D$M{?us=Vl9j8|!nzy({L22c3`ejAhEAZy#Dv{zngtsKG2N#w&8 z$5COy6Uotf$ADciYJg9zJ2&p`V=BVx^(1c?E`OEfQvMl^Lf=tUt3Fd(q27V-jl+QC z201#SUfHB8d5v|_QMXCH>GcJ^9_aMpW}zqpnFCdFa8py(8Q#4|2)eOfh?m}`EBz#? zIfwT^qBqOtD|se|r>XC4wrsOy$>B9!D2gbjTL5+0_q*yyC(2`|@qrVvT%`Ss%x+jF zC2N1ID0fVI=9VEzC!7>uf1|%Vf%BP??rmNtT9iogV3NWd&q(sY=ZIh%{-hN*>Y({-_T3wM~QV5>EvR(OOU!MrnXVow{p2CzPD zREAct;BOEE;h))q0^CULc z%s-xLOFC;u6>mGp+B+54G{M{6c>;64s@{O*&_R=)5i_6F(kUR+Cje1rzw;?>{&K-z zk*fS$$LDJPKG!Noc$72FHe+}G{Dq^f^isCXeUZ)CAOvn>A8=AK50n38`mO#LN}6Bl zk3o!V(kPlpkknr9@NFznfwJNG92zvZe9*Ljbc1X_*z(k*c<-y~V@hPM*0SNQmKdUk zk7!6DwuL(!Nx1!oMQ1T9g>!A@vneQ;+~s9o&W?7=+GtkbqSQVHJ!-CzA9GDqo8F1` zekA|8;q;`s>6L~48*^i&TP;U1BrMU`FossX!rGs z5cfQQbQo*?D(?!HT}1kz)|@qk+iaP9@tcAt85Tu^!?tG${2?4SPPCLC=EN(Ln6^sj z?6@}!Vk8l)9e(B%Hb8jXkpDWIh#M8T&-T*L4o+;>&MmE>2C>DrWjjIsceXpg0JtaVU4z6$F|U@)2?4*P$`k;J5BL zwIjc^4)+A`NHXsy-Bi|_%YBdAA-@f|*Zt@l$F-B_dxHC+@bxJ(UVBWWPE>BIYA%ph zPzoo9y+|zt|I&WD;$}Ly-PleO(-1)1SYN^t2UR_-04Qk!l*~6zDPE zqZ@(l4tqx{?#kcgk@1#;W&AEkI3V5_uG+L-X}%Za)_`Q0$T%9U4xk$Ujl#8{yS}oz zpN8g4YQY$}U=$5Ij!1N$W*180R$y<;k8}?XPpnk~>nja3m6k5TggO(6!k$q8z6j~>amzJAaMQ}p#cQ~Y&FPebSS>=-n01p3!m_K&)m z6Zz2ZXS2Q68uX)D14s}P7z|)8D7O^QOv$pkDL<-Y=hD7jy~+He?m9XXAg2EOa08*? z!TI8Ls(A!kVH?cZn#=|}0s7kJ`~8p*!@g0vFkXd5gQdOlnvKfUO^m70jH^4>^ZN;z zk$-ld<L8fR8lU zI#eBQA~BQ}N1YCL!~WsOj9j-vr)K07pYImD{YMHn*gFUYYY^sqPv|@G6)>1fz*!-) z1k`-FX!AJz<~B+l%v>Lvp*od2!&{Y>=diA2nH!twX@+m$SskTY`^6kH6<)zBw3lOe zdFlUtC$r)jo*LLD(ftLFF!o40pEpSXw~5`>uqDZ^WAs;9C%-N>2XE{-WbGJ&Ic(4c z4AX~_2Rw{)S(1^OeHh;CE1LB@vgh<0y=lv~c2EoVQW0aTrIzr^mSt(9jdjcP6{MQoXuU%)mIeL_tFpXnDqT)y`M#^y>o-oxV60 zBHU)E^^#lNwND7P0;!y^_xL1EX!#E)MdG>V9$hZ=jQh2cF0TbYYUbJpZ*?=-r2-Y8 zT{Q2N_ZF~kolWRV7 z1E^N@l;5ViJCp>7s}80iT;UzF>@eGG^)6~)3#N=jL&j1Qx%tSNXLB{QbYL}Y(V{91IktO+SsQPcgR!{;zZnqCej{ks(*YW@r;fnaXG1lufX;ResDkvZi|opGRx#lH9qqZlL9x&JuWCB#*40IVo1llnVTBlC>gu;fdG_KUY%rWM697hxYZXmhNDe*`&*5=ozKm&tD&$yWb5)x#0ZXTCW-eA zLJzs$cRN{f0wpn;mdCYxn2!&s?zL+=8TKsHSi@Nsodr5Uq$GdMlxfb6iuBJD)ZQAe)SD_cPts>B7T9yg zQPm0}@H(*k zXxisbfDP>{mw+In*`u%gnv{HC_t`9ie->oNmG}g_oi<*v%L|tP)ld~L17_u<`cEO; zHa9be=BtPfmh-9Q^j;gyy5~5h<9@l@p=T$Lr(1!v$Iq?#4%V(@8jTy(;a}w%8?>oN z_1i8qB5>->k_|7!9BpfS@jf%T0ezh!g8ulkNYp-$uXsq^AX0sE`bLKFeLnC4NTb;8 zX4v51Qv#Dn2q{ZB)3jH1cb@HF0XR26sevUdB$nxL7KbkqWtkVMf`A+1n$0nR@d3^s zJ0q#j$=RWkG#?MaDav;5945m-!7ymQeBlg((n(y!+TZbVhRQQ{z~wfOr_M!AZJOFT zC)br$t;{e#y_au}u>_v*Lr6q#yKX$TWsY;j=r7wk@?IYC<2XIP-(d;J zM=KqyFJ}ghf$beTeYC_T-knT-4h#(HzV{w=zB9CiTe*>AJ`R&!n>VRjkn4C5rHxY` zR-XB;CrsIE+O{U=4k3~aS$u-*LWf+xat@`1!uI{h`a3LjXrlaqOislf6Rg4EQI@W{ z@+*BCU7MAJN*N=YB6sXMwSuKMK9Y`>b_xRGfDTDY}(NO zyIfXQZsbq;&1-vb|2Y-W)=8B+UCj$hru}P^iIkeJ_D*GoIm8kRD_H}3HqN8YFCqA) zu2&eQ4$h9&x@zU+@~u$$XqH>Vql6PFJp(Qb(@Pp{`)yk8U;!}1k^_(Z4*Ydu(&Kcx z)vBE?0%Z*e=gJv~38$}M@V23wRXJ+xb{IRVmF~l{MqR}?Ud~&8BgZN+ZqzeI_YSp6 zJ?q^WDe13Gya~*St8*`&@BDlpq1NhCoA$!At>aNz`DI!;wJenW^FBe3=D-}bQ=K+$ zOqi1<9ON>T_ejY>D^;c;ny12d7#&&SS+E!{Ky|Iwg3EyTUP2$ejXV*<8A9_6%5*Zm zC!mwmON#(kdWt6hkyXsVt1=hvWo*%!#%$9jc#(A|uLr8++Vl5_x(rx*UVy%aZ^lkd}U-GM1KVN&)doA_4 zgBN60?SzG&8|ilNTl+2%590&$P$3TttTQnWtgX{z-!kKNBv#(=f_h+rxL?VBJyuNm zho4{KOMmx@EY@E>R)QKC_=cf5XvD`|=G6_X0GKRrVLCT*CLSY(p8&Iry9_8V*aBk} z$EwqCcK39)clEw?s{gqQAqffvzg?J!V8Fh3X?s%Dd-fjTW>){n3lW~UuVMlzwp#;R z#vr5ixc{*-^Xs~`bwMjHFfx60eX#n6);E$l+rO;4G-kM;&gqWA$Rk)8dAz%L3MALJ_m|%QXrt|{N(o6 zK9ad(T4z?eIRI9^S0d#0k5AJvVjd>pK;_~fH`z#WUwkJ87DR3PuFurSpQAhjrdG9B zg*OanHVL9cz4Ubo{hU`9HK9muRcTupe`WWj<|3kxe^)v9vC9Ax`YXR;Poj~+Pcduy zcSD{{#!Iirrj<3y1~8&(rU)DL;Ro{H3cLMjxcC$fiMj-fZ>G8I(Yu-klQ!lgv9v!A$+@QdkA)2*iny`2+Xp z{`}!=!#0HliBqZAm!bE^+pDF6Z%cb8pGQ*GD}|p#z+zwz69Jd{_f4=l&FjDeP~mC< zYOr$7a0+fwcehYp#xwNtzw5C$(Ub~~XYf~`Sd2i8DA)H^BB_9tL@Y*J0q5AL8;z2{h?q;qwF6`V zv9t6s9@M{~FVDXhjw_XQ3M7kvvIkIDIYh(%VQ0#JIQ`hgKSaJtrAwEM041<(ybwF# zj{K~F`lT%9Bl1lz1MyD+QJ{SNes9(Da^?LYf##THrYO3JlpVR11LPu(;I-@sl$~O7 z-x}QDA~;hjZB5`rsPrXgMt^?EEWp?Ghv=c2%+CH*bz#N3;_FPL^`5=}zK5NXTg!GM zjtnh7SpIvR3nhi%Mj_O%sZ@#~94INY5y^6PTq4*rT_e7b7yR9B)Aw0aL-K5aPbn|4 zx1PmBUOjyt9EohYk8j0c&y){*Zy=%W48bGggz zh-gzoNZO{;E@A?LWZ!NztB?BZyDlit%bZCs@5iD1T#M#){wSH7G1cs_97sg~NMgZm zu3~7XsaA&EkYgQd-Bb_%(96_6!BOb7REI!`(els-`GHi$ZkAUW zk~dN7RHsgBE!Aq@gFPLv+jnYLFqu^sJN~KPWB9^+rcMP97+*?^go&ck)VO@cWXJ^n z)gp=`}r-aXyGbd_N{R+ubZ{g_O4>UweQsu1VR5usJ zfhD<Gz2i%-W!&oq`<7`kE%g4{f&K|7#{`W zabJp3<(>05T}h$9D_2dt|3ZRlq*d7LmFNX6k=Smo~m@Q%!WatbLTup7NZT3!Ajump6+D`YW@lyy$^ z(AW*8oPeRj9KDToq+tbh@6Tz$j+6`7*}}hfrL5L`seZfR1*4rdL%WsrK8n^v9Q^aP zlW|&%YX^sZ8!TaI(eHRf&vSQ-oBembIx+TVyp|G>w9G&iV-1zu|Z?B=HIBrg65r5 zQ#LbuiMDfkpBIKskb!%GaI8uoCE3-{6Qhoh!4504W5 zHaqGvk_Fe1uIB{fT9D?0notDzp%x6ghvDp z0Ib6R06<~LasPk~4gg?dVWwwl zZQ}kPi14-zb*WeYK(#X9|C_@9<~CCR08S>(F8{j6{QFpY?cd#Gfb&~N3jkpMH{rj& zLxE;?|Haa?HF0*cbF|WPa<+3cF#At4o@?6|1OgnMxDEgS9Q!|jEe&kN+Qrev$-~y@ zKlD$FPYTKmoGVOV=l^?|m5hN*J4a_dTRUSDr~g2OoOR;61TNDakOyqxALz6NQeB+v zO>B+-1Mi>y)xh=m2lkzTetdRz)=o@j_AU(0CQi=(K3)GbTm6^A_5Xmt`@gyW@VowZ z`af*}|2Mrn;D6EoUw_trr~lI>;$Pl@{{!K$|8^4mcg{cWy}+&Z4>%Hh&%yGT(0 literal 0 HcmV?d00001 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl new file mode 100644 index 0000000000..ffeb167cfc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl @@ -0,0 +1,7 @@ +# cgroup.conf +# https://slurm.schedmd.com/cgroup.conf.html + +ConstrainCores=yes +ConstrainRamSpace=yes +ConstrainSwapSpace=no +ConstrainDevices=yes diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl new file mode 100644 index 0000000000..4951289842 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl @@ -0,0 +1,67 @@ +# slurm.conf +# https://slurm.schedmd.com/slurm.conf.html +# https://slurm.schedmd.com/configurator.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxNodeCount=64000 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +AccountingStoreFlags=job_comment +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/cgroup +SlurmctldDebug=info +SlurmdDebug=info +DebugFlags=Power + +# +# +# TIMERS +MessageTimeout=60 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=120 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl new file mode 100644 index 0000000000..8c90a9dfbe --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl @@ -0,0 +1,31 @@ +# slurmdbd.conf +# https://slurm.schedmd.com/slurmdbd.conf.html + +DebugLevel=info +PidFile=/var/run/slurm/slurmdbd.pid + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +AuthType=auth/{auth_key} +AuthAltTypes=auth/jwt +AuthAltParameters=jwt_key={state_save}/jwt_hs256.key + +DbdHost={control_host} + +LogFile={slurmlog}/slurmdbd.log + +SlurmUser=slurm + +StorageLoc={db_name} + +StorageType=accounting_storage/mysql +StorageHost={db_host} +StoragePort={db_port} +StorageUser={db_user} +StoragePass={db_pass} + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh new file mode 100644 index 0000000000..db514fc9e5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -x /opt/apps/adm/slurm/slurm_epilog ]]; then + exec /opt/apps/adm/slurm/slurm_epilog +fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh new file mode 100644 index 0000000000..37a91bb1ea --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -x /opt/apps/adm/slurm/slurm_prolog ]]; then + exec /opt/apps/adm/slurm/slurm_prolog +fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh new file mode 100644 index 0000000000..0877ff3b19 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SLURM_EXTERNAL_ROOT="/opt/apps/adm/slurm" +SLURM_MUX_FILE="slurm_mux" + +mkdir -p "${SLURM_EXTERNAL_ROOT}" +mkdir -p "${SLURM_EXTERNAL_ROOT}/logs" +mkdir -p "${SLURM_EXTERNAL_ROOT}/etc" + +# create common prolog / epilog "multiplex" script +if [ ! -f "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" ]; then + # indentation matters in EOT below; do not blindly edit! + cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CMD="${0##*/}" +# Locate script +BASE=$(readlink -f $0) +BASE=${BASE%/*} + +export CLUSTER_ADM_BASE=${BASE} + +# Source config file if it exists for extra DEBUG settings +# used below +SLURM_MUX_CONF=${CLUSTER_ADM_BASE}/etc/slurm_mux.conf +if [[ -r ${SLURM_MUX_CONF} ]]; then + source ${SLURM_MUX_CONF} +fi + +# Setup logging if configured and directory exists +LOGFILE="/dev/null" +if [[ -d ${DEBUG_SLURM_MUX_LOG_DIR} && ${DEBUG_SLURM_MUX_ENABLE_LOG} == "yes" ]]; then + LOGFILE="${DEBUG_SLURM_MUX_LOG_DIR}/${CMD}-${SLURM_SCRIPT_CONTEXT}-job-${SLURMD_NODENAME}.log" + exec >>${LOGFILE} 2>&1 +fi + +# Global scriptlets +for SCRIPTLET in ${BASE}/${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do + if [[ -x ${SCRIPTLET} ]]; then + echo "Running ${SCRIPTLET}" + ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 + echo "Running ${SCRIPTLET} returned $?" + fi +done + +# Per partition scriptlets +for SCRIPTLET in ${BASE}/partition-${SLURM_JOB_PARTITION}-${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do + if [[ -x ${SCRIPTLET} ]]; then + echo "Running ${SCRIPTLET}" + ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 + echo "Running ${SCRIPTLET} returned $?" + fi +done +EOT +fi + +# ensure proper permissions on slurm_mux script +chmod 0755 "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" + +# create default slurm_mux configuration file +if [ ! -f "${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" ]; then + cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" +# these settings are intended for temporary debugging purposes only; leaving +# them enabled will write files for each job to a shared NFS directory without +# any automated cleanup +DEBUG_SLURM_MUX_LOG_DIR=/opt/apps/adm/slurm/logs +DEBUG_SLURM_MUX_ENABLE_LOG=no +EOT +fi + +# create epilog symbolic link +if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_epilog" ]; then + cd ${SLURM_EXTERNAL_ROOT} + # delete existing file if necessary + rm -f slurm_epilog + ln -s ${SLURM_MUX_FILE} slurm_epilog + cd - >/dev/null +fi + +# create prolog symbolic link +if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_prolog" ]; then + cd ${SLURM_EXTERNAL_ROOT} + # delete existing file if necessary + rm -f slurm_prolog + ln -s ${SLURM_MUX_FILE} slurm_prolog + cd - >/dev/null +fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf new file mode 100644 index 0000000000..e63b2d1100 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf @@ -0,0 +1,406 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + scripts_dir = abspath("${path.module}/scripts") + + bucket_dir = coalesce(var.bucket_dir, format("%s-files", var.slurm_cluster_name)) +} + +######## +# DATA # +######## + +data "google_storage_bucket" "this" { + name = var.bucket_name +} + +########## +# RANDOM # +########## + +resource "random_uuid" "cluster_id" { +} + +################## +# CLUSTER CONFIG # +################## + +locals { + config = { + enable_bigquery_load = var.enable_bigquery_load + cloudsql_secret = var.cloudsql_secret + cluster_id = random_uuid.cluster_id.result + project = var.project_id + slurm_cluster_name = var.slurm_cluster_name + enable_slurm_auth = var.enable_slurm_auth + bucket_path = local.bucket_path + enable_debug_logging = var.enable_debug_logging + extra_logging_flags = var.extra_logging_flags + controller_state_disk = var.controller_state_disk + + # storage + disable_default_mounts = var.disable_default_mounts + network_storage = var.network_storage + + # timeouts + controller_startup_scripts_timeout = var.controller_startup_scripts_timeout + compute_startup_scripts_timeout = var.compute_startup_scripts_timeout + + munge_mount = local.munge_mount + slurm_key_mount = var.slurm_key_mount + + # slurm conf + prolog_scripts = [for k, v in google_storage_bucket_object.prolog_scripts : k] + epilog_scripts = [for k, v in google_storage_bucket_object.epilog_scripts : k] + task_prolog_scripts = [for k, v in google_storage_bucket_object.task_prolog_scripts : k] + task_epilog_scripts = [for k, v in google_storage_bucket_object.task_epilog_scripts : k] + cloud_parameters = var.cloud_parameters + + # hybrid + hybrid = var.enable_hybrid + google_app_cred_path = var.enable_hybrid ? local.google_app_cred_path : null + output_dir = var.enable_hybrid ? local.output_dir : null + install_dir = var.enable_hybrid ? local.install_dir : null + slurm_control_host = var.enable_hybrid ? var.slurm_control_host : null + slurm_control_host_port = var.enable_hybrid ? local.slurm_control_host_port : null + slurm_control_addr = var.enable_hybrid ? var.slurm_control_addr : null + slurm_bin_dir = var.enable_hybrid ? local.slurm_bin_dir : null + slurm_log_dir = var.enable_hybrid ? local.slurm_log_dir : null + controller_network_attachment = var.controller_network_attachment + + + # config files templates + slurmdbd_conf_tpl = file(coalesce(var.slurmdbd_conf_tpl, "${local.etc_dir}/slurmdbd.conf.tpl")) + slurm_conf_tpl = var.slurm_conf_template != null ? var.slurm_conf_template : file(coalesce(var.slurm_conf_tpl, "${local.etc_dir}/slurm.conf.tpl")) + cgroup_conf_tpl = file(coalesce(var.cgroup_conf_tpl, "${local.etc_dir}/cgroup.conf.tpl")) + + # Providers + endpoint_versions = var.endpoint_versions + } + + x_nodeset = toset(var.nodeset[*].nodeset_name) + x_nodeset_dyn = toset(var.nodeset_dyn[*].nodeset_name) + x_nodeset_tpu = toset(var.nodeset_tpu[*].nodeset.nodeset_name) + x_nodeset_overlap = setintersection([], local.x_nodeset, local.x_nodeset_dyn, local.x_nodeset_tpu) + + etc_dir = abspath("${path.module}/etc") + + bucket_path = format("%s/%s", data.google_storage_bucket.this.url, local.bucket_dir) + + slurm_control_host_port = coalesce(var.slurm_control_host_port, "6818") + + google_app_cred_path = var.google_app_cred_path != null ? abspath(var.google_app_cred_path) : null + slurm_bin_dir = var.slurm_bin_dir != null ? abspath(var.slurm_bin_dir) : null + slurm_log_dir = var.slurm_log_dir != null ? abspath(var.slurm_log_dir) : null + + munge_mount = var.enable_hybrid ? { + server_ip = lookup(var.munge_mount, "server_ip", coalesce(var.slurm_control_addr, var.slurm_control_host)) + remote_mount = lookup(var.munge_mount, "remote_mount", "/etc/munge/") + fs_type = lookup(var.munge_mount, "fs_type", "nfs") + mount_options = lookup(var.munge_mount, "mount_options", "") + } : null + + output_dir = can(coalesce(var.output_dir)) ? abspath(var.output_dir) : abspath(".") + install_dir = can(coalesce(var.install_dir)) ? abspath(var.install_dir) : local.output_dir +} + +resource "google_storage_bucket_object" "config" { + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/config.yaml" + content = yamlencode(local.config) + source_md5hash = md5(yamlencode(local.config)) + + # Take dependency on all other "config artifacts" so creation of `config.yaml` + # can be used as a signal for setup.py that "everything is ready". + # Some of following files, particularly mount scripts for new NFSes, can take a while to be created. + depends_on = [ + google_storage_bucket_object.controller_startup_scripts, + google_storage_bucket_object.nodeset_startup_scripts, + google_storage_bucket_object.prolog_scripts, + google_storage_bucket_object.epilog_scripts, + google_storage_bucket_object.task_prolog_scripts, + google_storage_bucket_object.task_epilog_scripts + ] +} + +resource "google_storage_bucket_object" "nodeset_config" { + for_each = { for ns in var.nodeset : ns.nodeset_name => merge(ns, { + instance_properties = jsondecode(ns.instance_properties_json) + }) } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +resource "google_storage_bucket_object" "nodeset_dyn_config" { + for_each = { for ns in var.nodeset_dyn : ns.nodeset_name => ns } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_dyn_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +resource "google_storage_bucket_object" "nodeset_tpu_config" { + for_each = { for n in var.nodeset_tpu[*].nodeset : n.nodeset_name => n } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_tpu_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +######### +# DEVEL # +######### + +locals { + build_dir = abspath("${path.module}/build") + + slurm_gcp_devel_controller_zip = "slurm-gcp-devel-controller.zip" + slurm_gcp_devel_compute_zip = "slurm-gcp-devel.zip" + slurm_gcp_devel_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_controller_zip) + slurm_gcp_devel_compute_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_compute_zip) + + controller_files = [ + "tools/gpu-test", + "tools/task-epilog", + "tools/task-prolog", + "conf.py", + "file_cache.py", + "get_tpu_vmcount.py", + "job_submit.lua.tpl", + "load_bq.py", + "local_pubsub.py", + "mig_flex.py", + "resume_wrapper.sh", + "resume.py", + "setup_network_storage.py", + "setup.py", + "slurmsync.py", + "sort_nodes.py", + "suspend_wrapper.sh", + "suspend.py", + "tpu.py", + "util.py", + "watch_delete_vm_op.py", + ] + + compute_files = [ + "tools/gpu-test", + "tools/task-epilog", + "tools/task-prolog", + "file_cache.py", + "get_tpu_vmcount.py", + "job_submit.lua.tpl", + "local_pubsub.py", + "mig_flex.py", + "setup_network_storage.py", + "setup.py", + "slurmsync.py", + "sort_nodes.py", + "suspend.py", + "tpu.py", + "util.py", + "watch_delete_vm_op.py", + ] +} + +data "archive_file" "slurm_gcp_devel_controller_zip" { + output_path = "${local.build_dir}/${local.slurm_gcp_devel_controller_zip}" + type = "zip" + + dynamic "source" { + for_each = local.controller_files + content { + content = file("${local.scripts_dir}/${source.value}") + filename = source.value + } + } +} + +data "archive_file" "slurm_gcp_devel_compute_zip" { + output_path = "${local.build_dir}/${local.slurm_gcp_devel_compute_zip}" + type = "zip" + + dynamic "source" { + for_each = local.compute_files + content { + content = file("${local.scripts_dir}/${source.value}") + filename = source.value + } + } +} + +resource "google_storage_bucket_object" "devel" { + bucket = var.bucket_name + name = local.slurm_gcp_devel_zip_bucket + source = data.archive_file.slurm_gcp_devel_controller_zip.output_path + source_md5hash = data.archive_file.slurm_gcp_devel_controller_zip.output_md5 +} + +resource "google_storage_bucket_object" "devel_compute" { + bucket = var.bucket_name + name = local.slurm_gcp_devel_compute_zip_bucket + source = data.archive_file.slurm_gcp_devel_compute_zip.output_path + source_md5hash = data.archive_file.slurm_gcp_devel_compute_zip.output_md5 +} + +########### +# SCRIPTS # +########### + +resource "google_storage_bucket_object" "controller_startup_scripts" { + for_each = { + for x in local.controller_startup_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-controller-script-%s", local.bucket_dir, each.key) + content = each.value.content + source_md5hash = md5(each.value.content) +} + +resource "google_storage_bucket_object" "nodeset_startup_scripts" { + for_each = { for x in flatten([ + for nodeset, scripts in var.nodeset_startup_scripts + : [for s in scripts + : { + content = s.content, + name = format("slurm-nodeset-%s-script-%s", nodeset, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_")) } + ]]) : x.name => x.content } + + bucket = var.bucket_name + name = format("%s/%s", local.bucket_dir, each.key) + content = each.value + source_md5hash = md5(each.value) +} + +resource "google_storage_bucket_object" "prolog_scripts" { + for_each = { + for x in local.prolog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-prolog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +resource "google_storage_bucket_object" "epilog_scripts" { + for_each = { + for x in local.epilog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-epilog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +resource "google_storage_bucket_object" "task_prolog_scripts" { + for_each = { + for x in local.task_prolog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-task_prolog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +resource "google_storage_bucket_object" "task_epilog_scripts" { + for_each = { + for x in local.task_epilog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-task_epilog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +############################ +# DATA: CHS GPU HEALTH CHECK +############################ + +data "local_file" "chs_gpu_health_check" { + filename = "${path.module}/scripts/tools/gpu-test" +} + +################################ +# DATA: EXTERNAL PROLOG/EPILOG # +################################ + +data "local_file" "external_epilog" { + filename = "${path.module}/files/external_epilog.sh" +} + +data "local_file" "external_prolog" { + filename = "${path.module}/files/external_prolog.sh" +} + +data "local_file" "setup_external" { + filename = "${path.module}/files/setup_external.sh" +} + +locals { + external_epilog = [{ + filename = "z_external_epilog.sh" + content = data.local_file.external_epilog.content + source = null + }] + external_prolog = [{ + filename = "z_external_prolog.sh" + content = data.local_file.external_prolog.content + source = null + }] + setup_external = [{ + filename = "z_setup_external.sh" + content = data.local_file.setup_external.content + }] + chs_gpu_health_check = [{ + filename = "a_chs_gpu_health_check.sh" + content = data.local_file.chs_gpu_health_check.content + source = null + }] + + chs_prolog = var.enable_chs_gpu_health_check_prolog ? local.chs_gpu_health_check : [] + ext_prolog = var.enable_external_prolog_epilog ? local.external_prolog : [] + prolog_scripts = concat(local.chs_prolog, local.ext_prolog, var.prolog_scripts) + task_prolog_scripts = var.task_prolog_scripts + + chs_epilog = var.enable_chs_gpu_health_check_epilog ? local.chs_gpu_health_check : [] + ext_epilog = var.enable_external_prolog_epilog ? local.external_epilog : [] + epilog_scripts = concat(local.chs_epilog, local.ext_epilog, var.epilog_scripts) + task_epilog_scripts = var.task_epilog_scripts + + controller_startup_scripts = var.enable_external_prolog_epilog ? concat(local.setup_external, var.controller_startup_scripts) : var.controller_startup_scripts + + +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf new file mode 100644 index 0000000000..111c997d62 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "slurm_bucket_path" { + description = "GCS Bucket URI of Slurm cluster file storage." + value = local.bucket_path +} + +output "bucket_name" { + description = "GCS Bucket name of Slurm cluster file storage." + value = data.google_storage_bucket.this.name +} + +output "bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + value = local.bucket_dir +} + +output "config" { + description = "Cluster configuration." + value = local.config + + precondition { + condition = var.enable_hybrid ? can(coalesce(var.slurm_control_host)) : true + error_message = "Input slurm_control_host is required." + } + + precondition { + condition = length(local.x_nodeset_overlap) == 0 + error_message = "All nodeset names must be unique among all nodeset types." + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py new file mode 100644 index 0000000000..89ceefa3df --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py @@ -0,0 +1,658 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Iterable, Dict, Set, Tuple +from itertools import chain +from collections import defaultdict +import json +from pathlib import Path +import util +from util import dirs, slurmdirs +import tpu +from addict import Dict as NSDict # type: ignore + +FILE_PREAMBLE = """ +# Warning: +# This file is managed by a script. Manual modifications will be overwritten. +""" + + + +def dict_to_conf(conf, delim=" ") -> str: + """convert dict to delimited slurm-style key-value pairs""" + + def filter_conf(pair): + k, v = pair + if isinstance(v, list): + v = ",".join(str(el) for el in v if el is not None) + return k, (v if bool(v) or v == 0 else None) + + return delim.join( + f"{k}={v}" for k, v in map(filter_conf, conf.items()) if v is not None + ) + + +TOPOLOGY_PLUGIN_TREE = "topology/tree" + +def topology_plugin(lkp: util.Lookup) -> str: + """ + Returns configured topology plugin, defaults to `topology/tree`. + """ + cp, key = lkp.cfg.cloud_parameters, "topology_plugin" + if key not in cp or cp[key] is None: + return TOPOLOGY_PLUGIN_TREE + return cp[key] + +def conflines(lkp: util.Lookup) -> str: + params = lkp.cfg.cloud_parameters + def get(key, default): + """ + Returns the value of the key in params if it exists and is not None, + otherwise returns supplied default. + We can't rely on the `dict.get` method because the value could be `None` as + well as empty NSDict, depending on type of the `cfg.cloud_parameters`. + TODO: Simplify once NSDict is removed from the codebase. + """ + if key not in params or params[key] is None: + return default + return params[key] + + no_comma_params = get("no_comma_params", False) + + any_gpus = any( + lkp.template_info(nodeset.instance_template).gpu + for nodeset in lkp.cfg.nodeset.values() + ) + + any_tpu = any( + tpu_nodeset is not None + for part in lkp.cfg.partitions.values() + for tpu_nodeset in part.partition_nodeset_tpu + ) + + any_gke = any( + lkp.nodeset_is_gke(nodeset) + for nodeset in lkp.cfg.nodeset.values() + ) + + any_dynamic = any(bool(p.partition_feature) for p in lkp.cfg.partitions.values()) + comma_params = { + "LaunchParameters": [ + "enable_nss_slurm", + "use_interactive_step", + ], + "SlurmctldParameters": [ + "cloud_reg_addrs" if any_dynamic or any_tpu or any_gke else "cloud_dns", + "enable_configless", + "idle_on_node_suspend", + ], + "GresTypes": [ + "gpu" if any_gpus else None, + ], + } + + scripts_dir = lkp.cfg.install_dir or dirs.scripts + prolog_path = Path(dirs.custom_scripts / "prolog.d") + epilog_path = Path(dirs.custom_scripts / "epilog.d") + task_prolog_path = Path(dirs.custom_scripts / "task_prolog.d") + task_epilog_path = Path(dirs.custom_scripts / "task_epilog.d") + default_tree_width = 65533 if any_dynamic else 128 + + conf_options = { + **(comma_params if not no_comma_params else {}), + "Prolog": f"{prolog_path}/*" if lkp.cfg.prolog_scripts else None, + "Epilog": f"{epilog_path}/*" if lkp.cfg.epilog_scripts else None, + "TaskProlog": f"{task_prolog_path}/task-prolog" if lkp.cfg.task_prolog_scripts else None, + "TaskEpilog": f"{task_epilog_path}/task-epilog" if lkp.cfg.task_epilog_scripts else None, + "PrologFlags": get("prolog_flags", None), + "SwitchType": get("switch_type", None), + "PrivateData": get("private_data", []), + "SchedulerParameters": get("scheduler_parameters", [ + "bf_continue", + "salloc_wait_nodes", + "ignore_prefer_validation", + ]), + "ResumeProgram": f"{scripts_dir}/resume_wrapper.sh", + "ResumeFailProgram": f"{scripts_dir}/suspend_wrapper.sh", + "ResumeRate": get("resume_rate", 0), + "ResumeTimeout": get("resume_timeout", 300), + "SuspendProgram": f"{scripts_dir}/suspend_wrapper.sh", + "SuspendRate": get("suspend_rate", 0), + "SuspendTimeout": get("suspend_timeout", 300), + "SlurmdTimeout": get("slurmd_timeout", 300), + "UnkillableStepTimeout": get("unkillable_step_timeout", 300), + "TreeWidth": get("tree_width", default_tree_width), + "JobSubmitPlugins": "lua" if any_tpu else None, + "TopologyPlugin": topology_plugin(lkp), + "TopologyParam": get("topology_param", "SwitchAsNodeRank"), + } + return dict_to_conf(conf_options, delim="\n") + + + + +def nodeset_lines(nodeset, lkp: util.Lookup) -> str: + template_info = lkp.template_info(nodeset.instance_template) + machine_conf = lkp.template_machine_conf(nodeset.instance_template) + + # follow https://slurm.schedmd.com/slurm.conf.html#OPT_Boards + # by setting Boards, SocketsPerBoard, CoresPerSocket, and ThreadsPerCore + gres = f"gpu:{template_info.gpu.count}" if template_info.gpu else None + node_conf = { + "RealMemory": machine_conf.memory, + "Boards": machine_conf.boards, + "SocketsPerBoard": machine_conf.sockets_per_board, + "CoresPerSocket": machine_conf.cores_per_socket, + "ThreadsPerCore": machine_conf.threads_per_core, + "CPUs": machine_conf.cpus, + "Gres": gres, + **nodeset.node_conf, + } + nodelist = lkp.nodelist(nodeset) + + return "\n".join( + map( + dict_to_conf, + [ + {"NodeName": nodelist, "State": "CLOUD", **node_conf}, + {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, + ], + ) + ) + + +def nodeset_tpu_lines(nodeset, lkp: util.Lookup) -> str: + nodelist = lkp.nodelist(nodeset) + return "\n".join( + map( + dict_to_conf, + [ + {"NodeName": nodelist, "State": "CLOUD", **nodeset.node_conf}, + {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, + ], + ) + ) + + +def nodeset_dyn_lines(nodeset): + """generate slurm NodeSet definition for dynamic nodeset""" + return dict_to_conf( + {"NodeSet": nodeset.nodeset_name, "Feature": nodeset.nodeset_feature} + ) + + +def partitionlines(partition, lkp: util.Lookup) -> str: + """Make a partition line for the slurm.conf""" + MIN_MEM_PER_CPU = 100 + + def defmempercpu(nodeset_name: str) -> int: + nodeset = lkp.cfg.nodeset.get(nodeset_name) + template = nodeset.instance_template + machine = lkp.template_machine_conf(template) + mem_spec_limit = int(nodeset.node_conf.get("MemSpecLimit", 0)) + return max(MIN_MEM_PER_CPU, (machine.memory - mem_spec_limit) // machine.cpus) + + defmem = min( + map(defmempercpu, partition.partition_nodeset), default=MIN_MEM_PER_CPU + ) + + nodesets = list( + chain( + partition.partition_nodeset, + partition.partition_nodeset_dyn, + partition.partition_nodeset_tpu, + ) + ) + + is_tpu = len(partition.partition_nodeset_tpu) > 0 + is_dyn = len(partition.partition_nodeset_dyn) > 0 + + oversub_exlusive = partition.enable_job_exclusive or is_tpu + power_down_on_idle = partition.enable_job_exclusive and not is_dyn + + line_elements = { + "PartitionName": partition.partition_name, + "Nodes": ",".join(nodesets), + "State": "UP", + "DefMemPerCPU": defmem, + "SuspendTime": 300, + "Oversubscribe": "Exclusive" if oversub_exlusive else None, + "PowerDownOnIdle": "YES" if power_down_on_idle else None, + **partition.partition_conf, + } + + return dict_to_conf(line_elements) + + +def suspend_exc_lines(lkp: util.Lookup) -> Iterable[str]: + static_nodelists = [] + for ns in lkp.power_managed_nodesets(): + if ns.node_count_static: + nodelist = lkp.nodelist_range(ns.nodeset_name, 0, ns.node_count_static) + static_nodelists.append(nodelist) + suspend_exc_nodes = {"SuspendExcNodes": static_nodelists} + + dyn_parts = [ + p.partition_name + for p in lkp.cfg.partitions.values() + if len(p.partition_nodeset_dyn) > 0 + ] + suspend_exc_parts = {"SuspendExcParts": [*dyn_parts]} + + return filter( + None, + [ + dict_to_conf(suspend_exc_nodes) if static_nodelists else None, + dict_to_conf(suspend_exc_parts), + ], + ) + + +def make_cloud_conf(lkp: util.Lookup) -> str: + """generate cloud.conf snippet""" + lines = [ + FILE_PREAMBLE, + conflines(lkp), + *(nodeset_lines(n, lkp) for n in lkp.cfg.nodeset.values()), + *(nodeset_dyn_lines(n) for n in lkp.cfg.nodeset_dyn.values()), + *(nodeset_tpu_lines(n, lkp) for n in lkp.cfg.nodeset_tpu.values()), + *(partitionlines(p, lkp) for p in lkp.cfg.partitions.values()), + *(suspend_exc_lines(lkp)), + ] + return "\n\n".join(filter(None, lines)) + + +def gen_cloud_conf(lkp: util.Lookup) -> None: + content = make_cloud_conf(lkp) + + conf_file = lkp.etc_dir / "cloud.conf" + conf_file.write_text(content) + util.chown_slurm(conf_file, mode=0o644) + + +def install_slurm_conf(lkp: util.Lookup) -> None: + """install slurm.conf""" + if lkp.cfg.ompi_version: + mpi_default = "pmi2" + else: + mpi_default = "none" + + conf_options = { + "name": lkp.cfg.slurm_cluster_name, + "control_addr": lkp.control_addr if lkp.control_addr else lkp.hostname_fqdn, + "control_host": lkp.control_host, + "accounting_storage_host": lkp.control_addr if lkp.cfg.controller_network_attachment else lkp.control_host, + "control_host_port": lkp.control_host_port, + "scripts": dirs.scripts, + "slurmlog": dirs.log, + "state_save": slurmdirs.state, + "mpi_default": mpi_default, + "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", + } + + conf = lkp.cfg.slurm_conf_tpl.format(**conf_options) + + conf_file = lkp.etc_dir / "slurm.conf" + conf_file.write_text(conf) + util.chown_slurm(conf_file, mode=0o644) + + +def install_slurmdbd_conf(lkp: util.Lookup) -> None: + """install slurmdbd.conf""" + conf_options = { + "control_host": lkp.control_host, + "slurmlog": dirs.log, + "state_save": slurmdirs.state, + "db_name": "slurm_acct_db", + "db_user": "slurm", + "db_pass": '""', + "db_host": "localhost", + "db_port": "3306", + "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", + } + + if lkp.cfg.cloudsql_secret: + secret_name = f"{lkp.cfg.slurm_cluster_name}-slurm-secret-cloudsql" + payload = json.loads(util.access_secret_version(lkp.project, secret_name)) + + if payload["db_name"] and payload["db_name"] != "": + conf_options["db_name"] = payload["db_name"] + if payload["user"] and payload["user"] != "": + conf_options["db_user"] = payload["user"] + if payload["password"] and payload["password"] != "": + conf_options["db_pass"] = payload["password"] + + db_host_str = payload["server_ip"].split(":") + if db_host_str[0]: + conf_options["db_host"] = db_host_str[0] + conf_options["db_port"] = ( + db_host_str[1] if len(db_host_str) >= 2 else "3306" + ) + + conf = lkp.cfg.slurmdbd_conf_tpl.format(**conf_options) + + conf_file = lkp.etc_dir / "slurmdbd.conf" + conf_file.write_text(conf) + util.chown_slurm(conf_file, 0o600) + + +def install_cgroup_conf(lkp: util.Lookup) -> None: + """install cgroup.conf""" + conf_file = lkp.etc_dir / "cgroup.conf" + conf_file.write_text(lkp.cfg.cgroup_conf_tpl) + util.chown_slurm(conf_file, mode=0o600) + + +def install_jobsubmit_lua(lkp: util.Lookup) -> None: + """install job_submit.lua if there are tpu nodes in the cluster""" + if not any( + tpu_nodeset is not None + for part in lkp.cfg.partitions.values() + for tpu_nodeset in part.partition_nodeset_tpu + ): + return # No TPU partitions, no need for job_submit.lua + + scripts_dir = lkp.cfg.slurm_scripts_dir or dirs.scripts + tpl = (scripts_dir / "job_submit.lua.tpl").read_text() + conf = tpl.format(scripts_dir=scripts_dir) + + conf_file = lkp.etc_dir / "job_submit.lua" + conf_file.write_text(conf) + util.chown_slurm(conf_file, 0o600) + + +def gen_cloud_gres_conf_lines(lkp: util.Lookup) -> str: + """generate cloud_gres.conf's content""" + + gpu_nodes = defaultdict(list) + for nodeset in lkp.cfg.nodeset.values(): + ti = lkp.template_info(nodeset.instance_template) + gpu_count = ti.gpu.count if ti.gpu else 0 + gpu_type = ti.gpu.type if ti.gpu else None + if gpu_count: + gpu_nodes[(gpu_count, gpu_type)].append(lkp.nodelist(nodeset)) + + lines = [ + dict_to_conf( + { + "NodeName": names, + "Name": "gpu", + "Type": gpu_type, + "File": "/dev/nvidia{}".format(f"[0-{gpu_count-1}]" if gpu_count > 1 else "0"), + } + ) + for (gpu_count, gpu_type), names in gpu_nodes.items() + ] + lines.append("\n") + return "\n".join(lines) + + +def gen_cloud_gres_conf(lkp: util.Lookup) -> None: + """create cloud_gres.conf file""" + + content = FILE_PREAMBLE + gen_cloud_gres_conf_lines(lkp) + + conf_file = lkp.etc_dir / "cloud_gres.conf" + conf_file.write_text(content) + util.chown_slurm(conf_file, mode=0o600) + + +def install_gres_conf(lkp: util.Lookup) -> None: + conf_file = lkp.etc_dir / "cloud_gres.conf" + gres_conf = lkp.etc_dir / "gres.conf" + if not gres_conf.exists(): + gres_conf.symlink_to(conf_file) + util.chown_slurm(gres_conf, mode=0o600) + + +class Switch: + """ + Represents a switch in the topology.conf file. + NOTE: It's class user job to make sure that there is no leaf-less Switches in the tree + """ + + def __init__( + self, + name: str, + nodes: Optional[Iterable[str]] = None, + switches: Optional[Dict[str, "Switch"]] = None, + ): + self.name = name + self.nodes = nodes or [] + self.switches = switches or {} + + def conf_line(self) -> str: + d = {"SwitchName": self.name} + if self.nodes: + d["Nodes"] = util.to_hostlist(self.nodes) + if self.switches: + d["Switches"] = util.to_hostlist(self.switches.keys()) + return dict_to_conf(d) + + def render_conf_lines(self) -> Iterable[str]: + yield self.conf_line() + for s in sorted(self.switches.values(), key=lambda s: s.name): + yield from s.render_conf_lines() + +class TopologySummary: + """ + Represents a summary of the topology, to make judgements about changes. + To be stored in JSON file along side of topology.conf to simplify parsing. + """ + def __init__( + self, + physical_host: Optional[Dict[str, str]] = None, + down_nodes: Optional[Iterable[str]] = None, + tpu_nodes: Optional[Iterable[str]] = None, + ) -> None: + self.physical_host = physical_host or {} + self.down_nodes = set(down_nodes or []) + self.tpu_nodes = set(tpu_nodes or []) + + + @classmethod + def path(cls, lkp: util.Lookup) -> Path: + return lkp.etc_dir / "cloud_topology.summary.json" + + @classmethod + def loads(cls, s: str) -> "TopologySummary": + d = json.loads(s) + return cls( + physical_host=d.get("physical_host"), + down_nodes=d.get("down_nodes"), + tpu_nodes=d.get("tpu_nodes"), + ) + + @classmethod + def load(cls, lkp: util.Lookup) -> "TopologySummary": + p = cls.path(lkp) + if not p.exists(): + return cls() # Return empty instance + return cls.loads(p.read_text()) + + def dumps(self) -> str: + return json.dumps( + { + "physical_host": self.physical_host, + "down_nodes": list(self.down_nodes), + "tpu_nodes": list(self.tpu_nodes), + }, + indent=2) + + def dump(self, lkp: util.Lookup) -> None: + TopologySummary.path(lkp).write_text(self.dumps()) + + def _nodenames(self) -> Set[str]: + return set(self.physical_host) | self.down_nodes | self.tpu_nodes + + def requires_reconfigure(self, prev: "TopologySummary") -> bool: + """ + Reconfigure IFF one of the following occurs: + * A node is added + * A node get a non-empty physicalHost + """ + if len(self._nodenames() - prev._nodenames()) > 0: + return True + for n, ph in self.physical_host.items(): + if ph and ph != prev.physical_host.get(n): + return True + return False + +class TopologyBuilder: + def __init__(self) -> None: + self._r = Switch("") # fake root, not part of the tree + self.summary = TopologySummary() + + def add(self, path: List[str], nodes: Iterable[str]) -> None: + n = self._r + assert path + for p in path: + n = n.switches.setdefault(p, Switch(p)) + n.nodes = [*n.nodes, *nodes] + + def render_conf_lines(self) -> Iterable[str]: + if not self._r.switches: + return [] # type: ignore + for s in sorted(self._r.switches.values(), key=lambda s: s.name): + yield from s.render_conf_lines() + + def compress(self) -> "TopologyBuilder": + compressed = TopologyBuilder() + compressed.summary = self.summary + def _walk( + u: Switch, c: Switch + ): # u: uncompressed node, c: its counterpart in compressed tree + pref = f"{c.name}_" if c != compressed._r else "s" + for i, us in enumerate(sorted(u.switches.values(), key=lambda s: s.name)): + cs = Switch(f"{pref}{i}", nodes=us.nodes) + c.switches[cs.name] = cs + _walk(us, cs) + + _walk(self._r, compressed._r) + return compressed + + +def add_tpu_nodeset_topology(nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup): + tpuobj = tpu.TPU.make(nodeset.nodeset_name, lkp) + static, dynamic = lkp.nodenames(nodeset) + + pref = ["tpu-root", f"ns_{nodeset.nodeset_name}"] + if tpuobj.vmcount == 1: # Put all nodes in one switch + all_nodes = list(chain(static, dynamic)) + bldr.add(pref, all_nodes) + bldr.summary.tpu_nodes.update(all_nodes) + return + + # Chunk nodes into sub-switches of size `vmcount` + chunk_num = 0 + for nodenames in (static, dynamic): + for nodeschunk in util.chunked(nodenames, n=tpuobj.vmcount): + chunk_name = f"{nodeset.nodeset_name}-{chunk_num}" + chunk_num += 1 + bldr.add([*pref, chunk_name], nodeschunk) + bldr.summary.tpu_nodes.update(nodeschunk) + +_SLURM_TOPO_ROOT = "slurm-root" + +def _make_physical_path(physical_host: str) -> List[str]: + assert physical_host.startswith("/"), f"Unexpected physicalHost: {physical_host}" + parts = physical_host[1:].split("/") + # Due to issues with Slurm's topology plugin, we can not use all components of `physicalHost`, + # trim it down to `cluster/rack`. + short_path = parts[:2] + return [_SLURM_TOPO_ROOT, *short_path] + +def add_nodeset_topology( + nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup +) -> None: + up_nodes = set() + default_path = [_SLURM_TOPO_ROOT, f"ns_{nodeset.nodeset_name}"] + + for inst in lkp.instances().values(): + try: + if lkp.node_nodeset_name(inst.name) != nodeset.nodeset_name: + continue + except Exception: + continue + + phys_host = inst.resource_status.physical_host or "" + bldr.summary.physical_host[inst.name] = phys_host + up_nodes.add(inst.name) + + if phys_host: + bldr.add(_make_physical_path(phys_host), [inst.name]) + else: + bldr.add(default_path, [inst.name]) + + down_nodes = [] + for node in chain(*lkp.nodenames(nodeset)): + if node not in up_nodes: + down_nodes.append(node) + if down_nodes: + bldr.add(default_path, down_nodes) + bldr.summary.down_nodes.update(down_nodes) + +def gen_topology(lkp: util.Lookup) -> TopologyBuilder: + bldr = TopologyBuilder() + for ns in lkp.cfg.nodeset_tpu.values(): + add_tpu_nodeset_topology(ns, bldr, lkp) + for ns in lkp.cfg.nodeset.values(): + add_nodeset_topology(ns, bldr, lkp) + return bldr + +def gen_topology_conf(lkp: util.Lookup) -> Tuple[bool, TopologySummary]: + """ + Generates slurm topology.conf. + Returns whether the topology.conf got updated. + """ + topo = gen_topology(lkp).compress() + conf_file = lkp.etc_dir / "cloud_topology.conf" + + with open(conf_file, "w") as f: + f.writelines(FILE_PREAMBLE + "\n") + for line in topo.render_conf_lines(): + f.write(line) + f.write("\n") + f.write("\n") + + prev_summary = TopologySummary.load(lkp) + return topo.summary.requires_reconfigure(prev_summary), topo.summary + +def install_topology_conf(lkp: util.Lookup) -> None: + conf_file = lkp.etc_dir / "cloud_topology.conf" + summary_file = lkp.etc_dir / "cloud_topology.summary.json" + topo_conf = lkp.etc_dir / "topology.conf" + + if not topo_conf.exists(): + topo_conf.symlink_to(conf_file) + + util.chown_slurm(conf_file, mode=0o600) + util.chown_slurm(summary_file, mode=0o600) + + +def gen_controller_configs(lkp: util.Lookup) -> None: + install_slurm_conf(lkp) + install_slurmdbd_conf(lkp) + gen_cloud_conf(lkp) + gen_cloud_gres_conf(lkp) + install_gres_conf(lkp) + install_cgroup_conf(lkp) + install_jobsubmit_lua(lkp) + + if topology_plugin(lkp) == TOPOLOGY_PLUGIN_TREE: + _, summary = gen_topology_conf(lkp) + summary.dump(lkp) + install_topology_conf(lkp) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py new file mode 100644 index 0000000000..cd2e41e5af --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py @@ -0,0 +1,80 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any +from pathlib import Path +import shutil +import pickle + +import logging +log = logging.getLogger() + +# Can't reuse tool from util.py to avoid circular dependencies +# TODO: break down util.py for better modularity. +def _chown_slurm(path: Path) -> None: + shutil.chown(path, user="slurm", group="slurm") + +class FileCache: + def __init__(self, path: Path): + self.path = path + + def get(self, key: str) -> Any | None: + p = self.path / key + if not p.exists(): + return None + + try: + with p.open("rb") as f: + return pickle.load(f) + + except Exception as e: + log.warning(f"Failed to read cached value at {p}: {e}") + return None + + def set(self, key: str, data: Any) -> None: + p = self.path / key + + try: + # Create & chown before writing to minimize chances + # of ending up with root-owned corrupted file that can't be cleaned up + # TODO: restrict usage of cache by root to avoid all this complexity + # or have a cache per user. + p.touch(exist_ok=True) + _chown_slurm(p) + with p.open("wb") as f: + pickle.dump(data, f) + + except Exception as e: + log.warning(f"Failed to write cached value at {p}: {e}") + + +class NoCache: + def get(self, key: str) -> Any: + log.warning("No cache used") + return None + + def set(self, key: str, data: Any) -> None: + log.warning("No cache used") + + +def cache(name: str) -> FileCache | NoCache: + try: + path = Path("/tmp/slurm_gcp_cache/") / name + if not path.exists(): + path.mkdir(exist_ok=True, parents=True) + _chown_slurm(path) + return FileCache(path) + except: + log.exception(f"Failed to create cache, fallback to NoCache") + return NoCache() diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py new file mode 100644 index 0000000000..df0fd8ebe0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py @@ -0,0 +1,76 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright 2024 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import util +import tpu + + +def get_vmcount_of_tpu_part(part): + res = 0 + lkp = util.lookup() + for ns in lkp.cfg.partitions[part].partition_nodeset_tpu: + tpu_obj = tpu.TPU.make(ns, lkp) + if res == 0: + res = tpu_obj.vmcount + else: + if res != tpu_obj.vmcount: + # this should not happen, that in the same partition there are different vmcount nodesets + return -1 + return res + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--partitions", + "-p", + help="The partition(s) to retrieve the TPU vmcount value for.", + ) + args = parser.parse_args() + if not args.partitions: + exit(0) + + # useful exit code + # partition does not exists in config.yaml, thus do not exist in slurm + PART_INVALID = -1 + # in the same partition there are nodesets with different vmcounts + DIFF_VMCOUNTS_SAME_PART = -2 + # partition is a list of partitions in which at least two of them have different vmcount + DIFF_PART_DIFFERENT_VMCOUNTS = -3 + vmcounts = [] + # valid equals to 0 means that we are ok, otherwise it will be set to one of the previously defined exit codes + valid = 0 + for part in args.partitions.split(","): + if part not in util.lookup().cfg.partitions: + valid = PART_INVALID + break + else: + if util.lookup().partition_is_tpu(part): + vmcount = get_vmcount_of_tpu_part(part) + if vmcount == -1: + valid = DIFF_VMCOUNTS_SAME_PART + break + vmcounts.append(vmcount) + else: + vmcounts.append(0) + # this means that there are different vmcounts for these partitions + if valid == 0 and len(set(vmcounts)) != 1: + valid = DIFF_PART_DIFFERENT_VMCOUNTS + if valid != 0: + print(f"VMCOUNT:{valid}") + else: + print(f"VMCOUNT:{vmcounts[0]}") diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl new file mode 100644 index 0000000000..810a0742b0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl @@ -0,0 +1,103 @@ +SCRIPTS_DIR = "{scripts_dir}" +NO_VAL = 4294967294 +-- get_tpu_vmcount.py exit code +PART_INVALID = -1 -- partition does not exists in config.yaml, thus do not exist in slurm +DIFF_VMCOUNTS_SAME_PART = -2 -- in the same partition there are nodesets with different vmcounts +DIFF_PART_DIFFERENT_VMCOUNTS = -3 -- partition is a list of partitions in which at least two of them have different vmcount +UNKWOWN_ERROR = -4 -- get_tpu_vmcount.py did not return a valid response + +function get_part(job_desc, part_list) + if job_desc.partition then + return job_desc.partition + end + for name, val in pairs(part_list) do + if val.flag_default == 1 then + return name + end + end + return nil +end + +function os.capture(cmd, raw) + local handle = assert(io.popen(cmd, 'r')) + local output = assert(handle:read('*a')) + handle:close() + return output +end + +function get_vmcount(part) + local cmd = SCRIPTS_DIR .. "/get_tpu_vmcount.py -p " .. part + local out = os.capture(cmd, true) + for line in out:gmatch("(.-)\r?\n") do + local tag, val = line:match("([^:]+):([^:]+)") + if tag == "VMCOUNT" then + return tonumber(val) + end + end + return UNKWOWN_ERROR +end + +function slurm_job_submit(job_desc, part_list, submit_uid) + local part = get_part(job_desc, part_list) + local vmcount = get_vmcount(part) + -- Only do something if the job is in a TPU partition, if vmcount is 0, it implies that the partition(s) specified are not TPU ones + if vmcount == 0 then + return slurm.SUCCESS + end + -- This is a TPU job, but as the vmcount is 1 it can he handled the same way + if vmcount == 1 then + return slurm.SUCCESS + end + -- Check for errors + if vmcount == PART_INVALID then + slurm.log_user("Invalid partition specified " .. part) + return slurm.FAILURE + end + if vmcount == DIFF_VMCOUNTS_SAME_PART then + slurm.log_user("In partition(s) " .. part .. + " there are more than one tpu nodeset vmcount, this should not happen.") + return slurm.ERROR + end + if vmcount == DIFF_PART_DIFFERENT_VMCOUNTS then + slurm.log_user("In partition list " .. part .. + " there are more than one TPU types, cannot determine which is the correct vmcount to use, please retry with only one partition.") + return slurm.FAILURE + end + if vmcount == UNKWOWN_ERROR then + slurm.log_user("Something went wrong while executing get_tpu_vmcount.py.") + return slurm.ERROR + end + -- This is surely a TPU node + if vmcount > 1 then + local min_nodes = job_desc.min_nodes + local max_nodes = job_desc.max_nodes + -- if not specified assume it is one, this should be improved taking into account the cpus, mem, and other factors + if min_nodes == NO_VAL then + min_nodes = 1 + max_nodes = 1 + end + -- as max_nodes can be higher than the nodes in the partition, we are not able to calculate with certainty the nodes that this job will have if this value is set to something + -- different than min_nodes + if min_nodes ~= max_nodes then + slurm.log_user("Max nodes cannot be set different than min nodes for the TPU partitions.") + return slurm.ERROR + end + -- Set the number of switches to the number of nodes originally requested by the job, as the job requests "TPU groups" + job_desc.req_switch = min_nodes + + -- Apply the node increase into the job description. + job_desc.min_nodes = min_nodes * vmcount + job_desc.max_nodes = max_nodes * vmcount + -- if job_desc.features then + -- slurm.log_user("Features: %s",job_desc.features) + -- end + end + + return slurm.SUCCESS +end + +function slurm_job_modify(job_desc, job_rec, part_list, modify_uid) + return slurm.SUCCESS +end + +return slurm.SUCCESS diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py new file mode 100644 index 0000000000..cabd6e3e9f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py @@ -0,0 +1,352 @@ +#!/slurm/python/venv/bin/python3.13 +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Callable, Any +import argparse +import os +import shelve +import uuid +from collections import namedtuple +from datetime import datetime, timedelta, timezone +from pathlib import Path +from pprint import pprint + +import util +from google.api_core import exceptions, retry +from google.cloud import bigquery as bq +from google.cloud.bigquery import SchemaField # type: ignore +from util import lookup, run + +SACCT = "sacct" +script = Path(__file__).resolve() + +DEFAULT_TIMESTAMP_FILE = script.parent / "bq_timestamp" +timestamp_file = Path(os.environ.get("TIMESTAMP_FILE", DEFAULT_TIMESTAMP_FILE)) +# The maximum request to insert_rows is 10MB, each sacct row is about 1200 bytes or ~ 8000 rows. +# Set to 5000 for a little wiggle room. +BQ_ROW_BATCH_SIZE = 5000 + +# cluster_id_file = script.parent / 'cluster_uuid' +# try: +# cluster_id = cluster_id_file.read_text().rstrip() +# except FileNotFoundError: +# cluster_id = uuid.uuid4().hex +# cluster_id_file.write_text(cluster_id) + +job_idx_cache_path = script.parent / "bq_job_idx_cache" + +SLURM_TIME_FORMAT = r"%Y-%m-%dT%H:%M:%S" + + +def make_datetime(time_string): + if time_string == "None": + return None + return datetime.strptime(time_string, SLURM_TIME_FORMAT).replace( + tzinfo=timezone.utc + ) + + +def make_time_interval(seconds): + sign = 1 + if seconds < 0: + sign = -1 + seconds = abs(seconds) + d, r = divmod(seconds, 60 * 60 * 24) + h, r = divmod(r, 60 * 60) + m, s = divmod(r, 60) + d *= sign + h *= sign + return f"{d}D {h:02}:{m:02}:{s}" + + +converters: Dict[str, Callable[[Any], Any]] = { + "DATETIME": make_datetime, + "INTERVAL": make_time_interval, + "STRING": str, + "INT64": lambda n: int(n or 0), +} + + +def schema_field(field_name, data_type, description, required=False): + return SchemaField( + field_name, + data_type, + description=description, + mode="REQUIRED" if required else "NULLABLE", + ) + + +schema_fields = [ + schema_field("cluster_name", "STRING", "cluster name", required=True), + schema_field("cluster_id", "STRING", "UUID for the cluster", required=True), + schema_field("entry_uuid", "STRING", "entry UUID for the job row", required=True), + schema_field( + "job_db_uuid", "STRING", "job db index from the slurm database", required=True + ), + schema_field("job_id_raw", "INT64", "raw job id", required=True), + schema_field("job_id", "STRING", "job id", required=True), + schema_field("state", "STRING", "final job state", required=True), + schema_field("job_name", "STRING", "job name"), + schema_field("partition", "STRING", "job partition"), + schema_field("submit_time", "DATETIME", "job submit time"), + schema_field("start_time", "DATETIME", "job start time"), + schema_field("end_time", "DATETIME", "job end time"), + schema_field("elapsed_raw", "INT64", "STRING", "job run time in seconds"), + # schema_field("elapsed_time", "INTERVAL", "STRING", "job run time interval"), + schema_field("timelimit_raw", "STRING", "job timelimit in minutes"), + schema_field("timelimit", "STRING", "job timelimit"), + # schema_field("num_tasks", "INT64", "number of allocated tasks in job"), + schema_field("nodelist", "STRING", "names of nodes allocated to job"), + schema_field("user", "STRING", "user responsible for job"), + schema_field("uid", "INT64", "uid of job user"), + schema_field("group", "STRING", "group of job user"), + schema_field("gid", "INT64", "gid of job user"), + schema_field("wckey", "STRING", "job wckey"), + schema_field("qos", "STRING", "job qos"), + schema_field("comment", "STRING", "job comment"), + schema_field("admin_comment", "STRING", "job admin comment"), + # extra will be added in 23.02 + # schema_field("extra", "STRING", "job extra field"), + schema_field("exitcode", "STRING", "job exit code"), + schema_field("alloc_cpus", "INT64", "count of allocated CPUs"), + schema_field("alloc_nodes", "INT64", "number of nodes allocated to job"), + schema_field("alloc_tres", "STRING", "allocated trackable resources (TRES)"), + # schema_field("system_cpu", "INTERVAL", "cpu time used by parent processes"), + # schema_field("cpu_time", "INTERVAL", "CPU time used (elapsed * cpu count)"), + schema_field("cpu_time_raw", "INT64", "CPU time used (elapsed * cpu count)"), + # schema_field("ave_cpu", "INT64", "Average CPU time of all tasks in job"), + # schema_field( + # "tres_usage_tot", + # "STRING", + # "Tres total usage by all tasks in job", + # ), +] + + +slurm_field_map = { + "job_db_uuid": "DBIndex", + "job_id_raw": "JobIDRaw", + "job_id": "JobID", + "state": "State", + "job_name": "JobName", + "partition": "Partition", + "submit_time": "Submit", + "start_time": "Start", + "end_time": "End", + "elapsed_raw": "ElapsedRaw", + "elapsed_time": "Elapsed", + "timelimit_raw": "TimelimitRaw", + "timelimit": "Timelimit", + "num_tasks": "NTasks", + "nodelist": "Nodelist", + "user": "User", + "uid": "Uid", + "group": "Group", + "gid": "Gid", + "wckey": "Wckey", + "qos": "Qos", + "comment": "Comment", + "admin_comment": "AdminComment", + # "extra": "Extra", + "exit_code": "ExitCode", + "alloc_cpus": "AllocCPUs", + "alloc_nodes": "AllocNodes", + "alloc_tres": "AllocTres", + "system_cpu": "SystemCPU", + "cpu_time": "CPUTime", + "cpu_time_raw": "CPUTimeRaw", + "ave_cpu": "AveCPU", + "tres_usage_tot": "TresUsageInTot", +} + +# new field name is the key for job_schema. Used to lookup the datatype when +# creating the job rows +job_schema = {field.name: field for field in schema_fields} +# Order is important here, as that is how they are parsed from sacct output +Job = namedtuple("Job", job_schema.keys()) # type: ignore +# ... see https://github.com/python/mypy/issues/848 + +client = bq.Client( + project=lookup().cfg.project, + credentials=util.default_credentials(), + client_options=util.create_client_options(util.ApiEndpoint.BQ), +) +dataset_id = f"{lookup().cfg.slurm_cluster_name}_job_data" +dataset = bq.DatasetReference(project=lookup().project, dataset_id=dataset_id) +table = bq.Table( + bq.TableReference(dataset, f"jobs_{lookup().cfg.slurm_cluster_name}"), schema_fields +) + + +class JobInsertionFailed(Exception): + pass + + +def make_job_row(job): + job_row = { + field_name: converters[field.field_type](job[field_name]) + for field_name, field in job_schema.items() + if field_name in job + } + job_row["entry_uuid"] = uuid.uuid4().hex + job_row["cluster_id"] = lookup().cfg.cluster_id + job_row["cluster_name"] = lookup().cfg.slurm_cluster_name + return job_row + + +def load_slurm_jobs(start, end): + states = ",".join( + ( + "BOOT_FAIL", + "CANCELLED", + "COMPLETED", + "DEADLINE", + "FAILED", + "NODE_FAIL", + "OUT_OF_MEMORY", + "PREEMPTED", + "REQUEUED", + "REVOKED", + "TIMEOUT", + ) + ) + start_iso = start.isoformat(timespec="seconds") + end_iso = end.isoformat(timespec="seconds") + # slurm_fields and bq_fields will be in matching order + slurm_fields = ",".join(slurm_field_map.values()) + bq_fields = slurm_field_map.keys() + cmd = ( + f"{SACCT} --start {start_iso} --end {end_iso} -X -D --format={slurm_fields} " + f"--state={states} --parsable2 --noheader --allusers --duplicates" + ) + text = run(cmd).stdout.splitlines() + # zip pairs bq_fields with the value from sacct + jobs = [dict(zip(bq_fields, line.split("|"))) for line in text] + + # The job index cache allows us to avoid sending duplicate jobs. This avoids a race condition with updating the database. + with shelve.open(str(job_idx_cache_path), flag="r") as job_idx_cache: + job_rows = [ + make_job_row(job) + for job in jobs + if str(job["job_db_uuid"]) not in job_idx_cache + ] + return job_rows + + +def init_table(): + global dataset + global table + dataset = client.create_dataset(dataset, exists_ok=True) # type: ignore + table = client.create_table(table, exists_ok=True) + until_found = retry.Retry(predicate=retry.if_exception_type(exceptions.NotFound)) + table = client.get_table(table, retry=until_found) + # cannot add required fields to an existing schema + table.schema = schema_fields + table = client.update_table(table, ["schema"]) + + +def purge_job_idx_cache(): + purge_time = datetime.now() - timedelta(minutes=30) + with shelve.open(str(job_idx_cache_path), writeback=True) as cache: + to_delete = [] + for idx, stamp in cache.items(): + if stamp < purge_time: + to_delete.append(idx) + for idx in to_delete: + del cache[idx] + + +def bq_submit(jobs): + try: + result = client.insert_rows(table, jobs) + except exceptions.NotFound as e: + print(f"failed to upload job data, table not yet found: {e}") + raise e + except Exception as e: + print(f"failed to upload job data: {e}") + raise e + if result: + pprint(jobs) + pprint(result) + raise JobInsertionFailed("failed to upload job data to big query") + print(f"successfully loaded {len(jobs)} jobs") + + +def get_time_window(): + if not timestamp_file.is_file(): + timestamp_file.touch() + try: + timestamp = datetime.strptime( + timestamp_file.read_text().rstrip(), SLURM_TIME_FORMAT + ) + # time window will overlap the previous by 10 minutes. Duplicates will be filtered out by the job_idx_cache + start = timestamp - timedelta(minutes=10) + except ValueError: + # timestamp 1 is 1 second after the epoch; timestamp 0 is special for sacct + start = datetime.fromtimestamp(1) + # end is now() truncated to the last second + end = datetime.now().replace(microsecond=0) + return start, end + + +def write_timestamp(time): + timestamp_file.write_text(time.isoformat(timespec="seconds")) + + +def update_job_idx_cache(jobs, timestamp): + with shelve.open(str(job_idx_cache_path), writeback=True) as job_idx_cache: + for job in jobs: + job_idx = str(job["job_db_uuid"]) + job_idx_cache[job_idx] = timestamp + + +def main(): + if not lookup().cfg.enable_bigquery_load: + print("bigquery load is not currently enabled") + exit(0) + init_table() + + start, end = get_time_window() + jobs = load_slurm_jobs(start, end) + # on failure, an exception will cause the timestamp not to be rewritten. So + # it will try again next time. If some writes succeed, we don't currently + # have a way to not submit duplicates next time. + if jobs: + num_batches = (len(jobs) - 1) // BQ_ROW_BATCH_SIZE + 1 + print( + f"loading {num_batches} batches of BigQuery data in batches of size : {BQ_ROW_BATCH_SIZE}" + ) + for batch_indx, job_indx in enumerate(range(0, len(jobs), BQ_ROW_BATCH_SIZE)): + print(f"loading BigQuery data batch {batch_indx} of {num_batches}") + bq_submit(jobs[job_indx : job_indx + BQ_ROW_BATCH_SIZE]) + write_timestamp(end) + update_job_idx_cache(jobs, end) + + +parser = argparse.ArgumentParser(description="submit slurm job data to big query") +parser.add_argument( + "timestamp_file", + nargs="?", + action="store", + type=Path, + help="specify timestamp file for reading and writing the time window start. Precedence over TIMESTAMP_FILE env var.", +) + +purge_job_idx_cache() +if __name__ == "__main__": + args = parser.parse_args() + if args.timestamp_file: + timestamp_file = args.timestamp_file.resolve() + main() diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py new file mode 100644 index 0000000000..d4a4477f83 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py @@ -0,0 +1,196 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Implementation of message queue that mimics interface of GCP (PubSub)[https://cloud.google.com/pubsub] + +Messages are stored on controller state disk (to survive controller re-creation) with following layout: + +// +├- +| └- +└- .staging + └- + └- + +One message is one immutable file, that will be deleted after acknowledgement. +NOTE: Implementation assumes that both `` and `.staging/` are on the same disk device, +so it can rely on atomic "move / rename" operation. +""" +from typing import Any +import util +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +import os +import uuid + +import logging +log = logging.getLogger() + + +@dataclass(frozen=True) +class Message: + id: str + created: datetime + data: Any + + def to_json(self) -> dict[str, str]: + return dict( + id=self.id, + created=self.created.isoformat(), + data=self.data) + + @classmethod + def from_json(cls, data: dict[str, str]) -> 'Message': + return cls( + id=data['id'], + created=datetime.fromisoformat(data['created']), + data=data['data']) + +class Topic: + """ + Acts as PubSub topic (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics). + We can have multiple instances of + """ + def __init__(self, path: Path, staging: Path) -> None: + self._path = path + self._staging = staging + + def _gen_id(self, created: datetime) -> str: + ts = created.strftime("%Y_%m_%d-%H_%M_%S") + suf = str(uuid.uuid4())[:8] + return f"{ts}-{suf}" + + def publish(self, data: Any) -> None: + created = util.now() + id = self._gen_id(created) + msg = Message(id=id, created=created, data=data) + + staged = self._staging / msg.id + dst = self._path / msg.id + + # Write to stagin area first then perform atomic move + # to prevent "reads of partial writes" + staged.write_text(json.dumps(msg.to_json())) + util.chown_slurm(staged) + staged.rename(dst) + + +class Subscription: + """ + Acts as PubSub subscription (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) + with following settings: + + ``` + ackDeadlineSeconds = +Inf # don't resend message that was already being delivered but not acked yet + retainAckedMessages = False # don't persist messages that were already acked + enableMessageOrdering = True # delivers messages in chronoligical order + messageRetentionDuration = +Inf # don't expire messages + deadLetterPolicy = None # "deadlettering" is disabled, subscriber should take care of any poisonous messages + retryPolicy = { # NACKed message will be re-delievered after some time + minimumBackoff = 30s # NOTE: Practically there is no timer, but Subscription instance will not try to re-deliver NACKed messages. + maximumBackoff = 30s # Assumes that slurmsync runs every 30+ sec. + } + ``` + + IMPORTANT: Should only be run as part of slurmsync, + this is our way to ensure that at most one instance exists at a time. + There is no concurancy safeguards in place, avoid multithreaded `pull`, + while multithreaded `ack` & `modify_ack_deadline` are OK. + """ + + def __init__(self, path: Path) -> None: + self._path: Path = path + # contains ALL messages pulled by this subscription instance + # both acked, nacked, and still being processed + # used to prevent double delivery within lifetime of subscription (slurmsync) + self._pulled: set[str] = set() + + def _delete(self, id: str) -> None: + log.debug(f"removing {id}") + try: + os.unlink(self._path / id) + except: + log.exception(f"Failed to remove message {id}") + + def _read_msg(self, id: str) -> Message | None: + try: + with open(self._path / id, 'r') as f: + content = json.loads(f.read()) + return Message.from_json(content) + except Exception: + log.exception(f"Failed to read message {id}") + self._delete(id) # delete message to reduce "deadlettering" + return None + + def pull(self, max_messages: int) -> list[Message]: + if not self._path.exists(): + log.warning(f"Topic {self._path} does not exist") + return [] + res = [] + ls = sorted(os.listdir(self._path)) + for name in ls: + msg = self._read_msg(name) + if msg is not None and msg.id not in self._pulled: + self._pulled.add(msg.id) + res.append(msg) + + if len(res) >= max_messages: + break + return res + + + def ack(self, ids: list[str]) -> None: + for id in ids: + self._delete(id) + + + def modify_ack_deadline(self, ids: list[str], deadline: int) -> None: + """ + Modifies the ack deadline for a specific message. + IMPORTANT: Only accepts deadline=0, which is a way to NACK + Any other values are also meaningless due to ackDeadlineSeconds==+Inf + """ + assert deadline == 0 # no op, next subscriber (slurmsync) will pick this up + + +# Topics and Subscriptions are singletons +# TODO: consider making thread-safe +_topics = {} +_subscriptions = {} + +def _make_path(name: str) -> Path: + p = util.slurmdirs.state / "pubsub" / name + p.mkdir(parents=True, exist_ok=True) + util.chown_slurm(p) + return p + +def _make_staging_path(name: str) -> Path: + p = util.slurmdirs.state / "pubsub" / ".staging" / name + p.mkdir(parents=True, exist_ok=True) + util.chown_slurm(p) + return p + +def topic(name: str) -> Topic: + if name not in _topics: + _topics[name] = Topic(_make_path(name), _make_staging_path(name)) + return _topics[name] + +def subscription(name: str) -> Subscription: + if name not in _subscriptions: + _subscriptions[name] = Subscription(_make_path(name)) + return _subscriptions[name] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py new file mode 100644 index 0000000000..8ea3d0657e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py @@ -0,0 +1,254 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional + +import util +import uuid +from addict import Dict as NSDict # type: ignore +from datetime import datetime, timedelta +from collections import defaultdict +import logging +from time import sleep + +log = logging.getLogger() + +DWS_EOL_RESERVATION_DURATION = 10 # minutes + +def _duration(flex_options: NSDict, job_id: Optional[int], lkp: util.Lookup) -> int: + dur = flex_options.max_run_duration + if not job_id or not flex_options.use_job_duration: + return dur + + job = lkp.job(job_id) + if not job or not job.duration: + return dur + + if timedelta(minutes=10) <= job.duration <= timedelta(weeks=1): + return int(job.duration.total_seconds()) + + log.info("Job TimeLimit cannot be less than 10 minutes or exceed one week") + return dur + +def _create_slurm_reservation(node_name: str, boot_time: datetime, run_duration: int, lkp: util.Lookup): + """ + Create a Slurm reservation starting at EOL - buffer time. + """ + eol = boot_time + timedelta(seconds=run_duration) + start_str = eol.strftime("%Y-%m-%dT%H:%M:%S") + reservation_name = f"dws-eol-{node_name}" + log.debug(f"creating slurm reservation for {node_name}") + try: + util.run(f"{lkp.scontrol} create reservation user=slurm starttime={start_str} duration={DWS_EOL_RESERVATION_DURATION} nodes={node_name} reservationname={reservation_name} flags=maint,ignore_jobs") + except Exception as e: + log.error(f"Failed to create reservation for {node_name}: {e}") + +def _delete_slurm_reservation(node_name: str, lkp: util.Lookup): + """ + Delete the Slurm reservation for the given node. + """ + reservation_name = f"dws-eol-{node_name}" + try: + util.run(f"{lkp.scontrol} delete reservation {reservation_name}") + log.debug(f"Deleted Slurm reservation {reservation_name} for {node_name}") + except Exception as e: + log.error(f"Failed to delete reservation for {node_name}: {e}") + +def resume_flex_chunk(nodes: List[str], job_id: Optional[int], lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + assert len(nodeset.zone_policy_allow) > 0 + region = lkp.node_region(model) + + assert nodeset.dws_flex.enabled + + uid = str(uuid.uuid4())[:8] + if job_id: + mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-job-{job_id}-{uid}" + else: + mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-{uid}" + + # Create MIG + req = lkp.compute.regionInstanceGroupManagers().insert( + project=lkp.project, + region=region, + body=dict( + name=mig_name, + versions=[dict(instanceTemplate=nodeset.instance_template)], + targetSize=0, + distributionPolicy=dict( + zones=[ + dict(zone=f"zones/{z}") for z in nodeset.zone_policy_allow + ], + targetShape="ANY_SINGLE_ZONE" ), + updatePolicy = dict(instanceRedistributionType = "NONE" ), + instanceLifecyclePolicy=dict(defaultActionOnFailure= "DO_NOTHING" ), # TODO(FLEX): Not supported yet, migrate once supported + ) + ) + util.log_api_request(req) + op = req.execute() + res = util.wait_for_operation(op) + assert "error" not in res, f"{res}" + + # Create resize request + duration_seconds = _duration(nodeset.dws_flex, job_id, lkp) + req = lkp.compute.regionInstanceGroupManagerResizeRequests().insert( + project=lkp.project, + region=region, + instanceGroupManager=mig_name, + body=dict( + name="initial-resize", + instances=[dict(name=n) for n in nodes], + requested_run_duration=dict( + seconds=duration_seconds + ) + ) + ) + util.log_api_request(req) + op = req.execute() + res = util.wait_for_operation(op) + + # Create Slurm reservations if use_job_duration is set + if nodeset.dws_flex.use_job_duration: + # Get run duration (seconds) + run_duration = duration_seconds + for node_name in nodes: + # Fetch instance creation time from GCP instance (via util.py) + instance = lkp.instance(node_name) + if(instance and instance.creation_timestamp): + log.debug("creating with creation_timestamp") + boot_time = instance.creation_timestamp # Already a datetime object + else: + boot_time = datetime.utcnow() + log.debug("creating with utcnow time: {boot_time}") + _create_slurm_reservation(node_name, boot_time, run_duration, lkp) + + assert "error" not in res, f"{res}" + +def _suspend_flex_mig(mig_self_link: str, nodes: List[str], lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + assert len(nodeset.zone_policy_allow) > 0 + region = lkp.node_region(model) + project=lkp.project + instanceGroupManager=util.trim_self_link(mig_self_link) + + links = [ + f"zones/{inst.zone}/instances/{inst.name}" + for inst in [ + lkp.instance(node) for node in nodes + ] if inst + ] + + target_mig=lkp.get_mig(lkp.project, region, instanceGroupManager) + assert target_mig + + # TODO(FLEX): This will not work if MIG didn't obtain capacity yet. + # The request will fail and MIG will continue provisioning. + # Instead whole MIG should be deleted. + # + All other instances in MIG are not provisioned also, safe to delete + # - Need to come up will clear test to differentiate non-provisioned MIG and single VM being down; + # Particularly CRITICAL due to ActionOnFailure=DO_NOTHING + # - Need to `down_nodes_notify_jobs` for all nodes in MIG, make sure that it doesn't interfere with Slurm suspend-flow. + + if target_mig["targetSize"] == len(nodes): #We can just delete the whole MIG in this case + req = lkp.compute.regionInstanceGroupManagers().delete( + project=project, + region=region, + instanceGroupManager=instanceGroupManager, + ) + else: + req = lkp.compute.regionInstanceGroupManagers().deleteInstances( + project=project, + region=region, + instanceGroupManager=instanceGroupManager, + body=dict( + instances=links, + skipInstancesOnValidationError=True, + ) + ) + + util.log_api_request(req) + op = req.execute() + + res = util.wait_for_operation(op) + + # Delete Slurm reservations for nodes being deprovisioned + for node_name in nodes: + log.info("delete dws reservation") + _delete_slurm_reservation(node_name, lkp) + + assert "error" not in res, f"{res}" + +def _suspend_provisioning_inst(nodes:List[str], node_template:str, lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + assert len(nodeset.zone_policy_allow) > 0 + region = lkp.node_region(model) + + mig_list=lkp.get_mig_list(lkp.project, region) + + # FLEX (#TODO): If we enter this conditional it's likely this was called so early that MIG creation hasn't started + # Consider potentially retrying? No natural mechanism for retry currently but we could + # perhaps use slurmsync and then try it again to ensure it wasn't a case of being too early. + # This is important since we're now enabling long ResumeTimeout (Slurm won't call suspend on node within reasonable timeframe) + # so until we do this is slurmsync this is a temporary workaround. + + if not mig_list or not mig_list.get("items"): + log.info("No matching MIG found to delete! Retrying...") + sleep(5) + mig_list=lkp.get_mig_list(lkp.project, region) + if not mig_list or not mig_list.get("items"): + return + + for mig in mig_list["items"]: + if mig["instanceTemplate"] == node_template: + if mig["currentActions"]["creating"] > 0 and mig["targetSize"] == mig["currentActions"]["creating"]: + req = lkp.compute.regionInstanceGroupManagers().delete( + project=lkp.project, + region=region, + instanceGroupManager=util.trim_self_link(mig["selfLink"]), + ) + + util.log_api_request(req) + op = req.execute() + + res = util.wait_for_operation(op) + assert "error" not in res, f"{res}" + return + + log.info("No matching MIG found to delete!") + +def suspend_flex_nodes(nodes: List[str], lkp: util.Lookup) -> None: + by_mig = defaultdict(list) + not_provisioned = defaultdict(list) + for node in nodes: + inst = lkp.instance(node) + if not inst: + not_provisioned[lkp.node_template(node)].append(node) + else: + mig = inst.metadata.get("created-by") + if not mig: + log.error(f"Can not suspend {node}, can not find associated MIG") + continue + by_mig[mig].append(node) + + for mig, nodes in by_mig.items(): + _suspend_flex_mig(mig, nodes, lkp) + + for node_template, nodes in not_provisioned.items(): + _suspend_provisioning_inst(nodes, node_template, lkp) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt new file mode 100644 index 0000000000..2ab3162ccf --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt @@ -0,0 +1,9 @@ +pytest +pytest-mock +pytest_unordered +mock + +types-mock +types-httplib2 +types-requests +types-PyYAML diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt new file mode 100644 index 0000000000..e923e53dbf --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt @@ -0,0 +1,18 @@ +addict==2.4.0 +google-api-core==2.19.0 +google-api-python-client==2.93.0 +google-auth==2.40.3 +google-auth-httplib2==0.1.0 +google-cloud-bigquery==3.11.3 +google-cloud-core==2.3.3 +google-cloud-secret-manager~=2.22 +google-cloud-storage==2.10.0 +google-cloud-tpu==1.10.0 +google-resumable-media==2.5.0 +googleapis-common-protos==1.59.1 +grpcio==1.60.0 +grpcio-status==1.60.0 +httplib2==0.22.0 +more-executors==2.11.4 +pyyaml==6.0.2 +requests==2.32.4 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py new file mode 100644 index 0000000000..ea0012a0b1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py @@ -0,0 +1,703 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Dict, Any +import argparse +from datetime import timedelta +import shlex +import json +import logging +import os +import yaml +import collections +from pathlib import Path +from dataclasses import dataclass +from addict import Dict as NSDict # type: ignore + +import util +from util import ( + chunked, + ensure_execute, + execute_with_futures, + log_api_request, + map_with_futures, + run, + separate, + to_hostlist, + trim_self_link, + wait_for_operation, +) +from util import lookup, ReservationDetails +import tpu +import mig_flex + +log = logging.getLogger() + +PLACEMENT_MAX_CNT = 1500 +# Placement group needs to be the same for an entire bulk_insert hence +# if placement is used the actual BULK_INSERT_LIMIT will be +# max([1000, PLACEMENT_MAX_CNT]) +BULK_INSERT_LIMIT = 5000 + +# https://cloud.google.com/compute/docs/instance-groups#types_of_managed_instance_groups +ZONAL_MIG_SIZE_LIMIT = 1000 + + +@dataclass(frozen=True) +class ResumeJobData: + job_id: int + partition: str + nodes_alloc: List[str] + +@dataclass(frozen=True) +class ResumeData: + jobs: List[ResumeJobData] + + +def get_resume_file_data() -> Optional[ResumeData]: + if not (path := os.getenv("SLURM_RESUME_FILE")): + log.error("SLURM_RESUME_FILE was not in environment. Cannot get detailed job, node, partition allocation data.") + return None + blob = Path(path).read_text() + log.debug(f"Resume data: {blob}") + data = json.loads(blob) + + jobs = [] + for jo in data.get("jobs", []): + job = ResumeJobData( + job_id = jo.get("job_id"), + partition = jo.get("partition"), + nodes_alloc = util.to_hostnames(jo.get("nodes_alloc")), + ) + jobs.append(job) + return ResumeData(jobs=jobs) + +def instance_properties(nodeset: NSDict, model:str, placement_group:Optional[str], labels:Optional[dict], job_id:Optional[int]): + props = NSDict() + + if labels: # merge in extra labels on instance and disks + template_link = lookup().node_template(model) + template_info = lookup().template_info(template_link) + + props.labels = {**template_info.labels, **labels} + + for disk in template_info.disks: + if disk.initializeParams.get("diskType", "local-ssd") == "local-ssd": + continue # do not label local ssd + disk.initializeParams.labels.update(labels) + props.disks = template_info.disks + + if placement_group: + props.resourcePolicies = [placement_group] + + if reservation := lookup().nodeset_reservation(nodeset): + update_reservation_props(reservation, props, placement_group, reservation.calendar) + + if (fr := lookup().future_reservation(nodeset)) and fr.specific: + assert fr.active_reservation + update_reservation_props(fr.active_reservation, props, placement_group, fr.calendar) + + if props.resourcePolicies: + props.scheduling.onHostMaintenance = "TERMINATE" + + if nodeset.maintenance_interval: + props.scheduling.maintenanceInterval = nodeset.maintenance_interval + + if nodeset.dws_flex.enabled and nodeset.dws_flex.use_bulk_insert: + update_props_dws(props, nodeset.dws_flex, job_id) + + # Override with properties explicit specified in the nodeset + props.update(nodeset.get("instance_properties") or {}) + return props + +def update_reservation_props(reservation:ReservationDetails, props:NSDict, placement_group:Optional[str], calendar_mode:bool) -> None: + props.reservationAffinity = { + "consumeReservationType": "SPECIFIC_RESERVATION", + "key": f"compute.{util.universe_domain()}/reservation-name", + "values": [reservation.bulk_insert_name], + } + + if reservation.dense or calendar_mode: + props.scheduling.provisioningModel = "RESERVATION_BOUND" + + # Figure out `resourcePolicies` + if reservation.policies: # use ones already attached to reservations + props.resourcePolicies = reservation.policies + elif reservation.dense and placement_group: # use once created by Slurm + props.resourcePolicies = [placement_group] + else: # vanilla reservations don't support external policies + props.resourcePolicies = [] + log.info( + f"reservation {reservation.bulk_insert_name} is being used with resourcePolicies: {props.resourcePolicies}") + +def update_props_dws(props: NSDict, dws_flex: NSDict, job_id: Optional[int]) -> None: + props.scheduling.onHostMaintenance = "TERMINATE" + props.scheduling.instanceTerminationAction = "DELETE" + props.reservationAffinity['consumeReservationType'] = "NO_RESERVATION" + props.scheduling.maxRunDuration['seconds'] = dws_flex_duration(dws_flex, job_id) + +def dws_flex_duration(dws_flex: NSDict, job_id: Optional[int]) -> int: + max_duration = dws_flex.max_run_duration + if dws_flex.use_job_duration and job_id is not None and (job := lookup().job(job_id)) and job.duration: + if timedelta(seconds=30) <= job.duration <= timedelta(weeks=1): + max_duration = int(job.duration.total_seconds()) + else: + log.info("Job TimeLimit cannot be less than 30 seconds or exceed one week") + return max_duration + +def create_instances_request(nodes: List[str], placement_group: Optional[str], excl_job_id: Optional[int]): + """Call regionInstances.bulkInsert to create instances""" + assert 0 < len(nodes) <= BULK_INSERT_LIMIT + + # model here indicates any node that can be used to describe the rest + model = next(iter(nodes)) + log.debug(f"create_instances_request: {model} placement: {placement_group}") + + nodeset = lookup().node_nodeset(model) + template = lookup().node_template(model) + labels = {"slurm_job_id": excl_job_id} if excl_job_id else None + + body = dict( + count = len(nodes), + sourceInstanceTemplate = template, + # key is instance name, value overwrites properties (no overwrites) + perInstanceProperties = {k: {} for k in nodes}, + instanceProperties = instance_properties( + nodeset, model, placement_group, labels, excl_job_id + ), + ) + + if placement_group and excl_job_id is not None: + pass # do not set minCount to force "all or nothing" behavior + else: + body["minCount"] = 1 + + zone_allow = nodeset.zone_policy_allow or [] + zone_deny = nodeset.zone_policy_deny or [] + + if len(zone_allow) == 1: # if only one zone is used, use zonal BulkInsert API, as less prone to errors + api_method = lookup().compute.instances().bulkInsert + method_args = {"zone": zone_allow[0]} + else: + api_method = lookup().compute.regionInstances().bulkInsert + method_args = {"region": lookup().node_region(model)} + + body["locationPolicy"] = dict( + locations = { + **{ f"zones/{z}": {"preference": "ALLOW"} for z in zone_allow }, + **{ f"zones/{z}": {"preference": "DENY"} for z in zone_deny }}, + targetShape = nodeset.zone_target_shape, + ) + + req = api_method( + project=lookup().project, + body=body, + **method_args) + log.debug(f"new request: endpoint={req.methodId} nodes={to_hostlist(nodes)}") + log_api_request(req) + return req + +@dataclass() +class PlacementAndNodes: + placement: Optional[str] + nodes: List[str] + +@dataclass(frozen=True) +class BulkChunk: + nodes: List[str] + prefix: str # - + chunk_idx: int + excl_job_id: Optional[int] + placement_group: Optional[str] = None + + @property + def name(self): + if self.placement_group is not None: + return f"{self.prefix}:job{self.excl_job_id}:{self.placement_group}:{self.chunk_idx}" + if self.excl_job_id is not None: + return f"{self.prefix}:job{self.excl_job_id}:{self.chunk_idx}" + return f"{self.prefix}:{self.chunk_idx}" + + +def group_nodes_bulk(nodes: List[str], resume_data: Optional[ResumeData], lkp: util.Lookup): + """group nodes by nodeset, placement_group, exclusive_job_id if any""" + if resume_data is None: # all nodes will be considered jobless + resume_data = ResumeData(jobs=[]) + + nodes_set = set(nodes) # turn into set to simplify intersection + non_excl = nodes_set.copy() + groups : Dict[Optional[int], List[PlacementAndNodes]] = {} # excl_job_id|none -> PlacementAndNodes + + # expand all exclusive job nodelists + for job in resume_data.jobs: + if not lkp.cfg.partitions[job.partition].enable_job_exclusive: + continue + + groups[job.job_id] = [] + # placement group assignment is based on all allocated nodes, ... + for pn in create_placements(job.nodes_alloc, job.job_id, lkp): + groups[job.job_id].append( + PlacementAndNodes( + placement=pn.placement, + #... but we only want to handle nodes in nodes_resume in this run. + nodes = sorted(set(pn.nodes) & nodes_set) + )) + non_excl.difference_update(job.nodes_alloc) + + groups[None] = create_placements(sorted(non_excl), excl_job_id=None, lkp=lkp) + + def chunk_nodes(nodes: List[str]): + if not nodes: + return [] + + model = nodes[0] + + if lkp.is_flex_node(model): + chunk_size = ZONAL_MIG_SIZE_LIMIT + elif lkp.node_is_tpu(model): + ns_name = lkp.node_nodeset_name(model) + chunk_size = tpu.TPU.make(ns_name, lkp).vmcount + else: + chunk_size = BULK_INSERT_LIMIT + + return chunked(nodes, n=chunk_size) + + chunks = [ + BulkChunk( + nodes=nodes_chunk, + prefix=lkp.node_prefix(nodes_chunk[0]), # - + excl_job_id = job_id, + placement_group=pn.placement, + chunk_idx=i) + + for job_id, placements in groups.items() + for pn in placements if pn.nodes + for i, nodes_chunk in enumerate(chunk_nodes(pn.nodes)) + ] + return {chunk.name: chunk for chunk in chunks} + + +def resume_nodes(nodes: List[str], resume_data: Optional[ResumeData]): + """resume nodes in nodelist""" + lkp = lookup() + # Prevent dormant nodes associated with a reservation from being resumed + nodes, dormant_res_nodes = util.separate(lkp.is_dormant_res_node, nodes) + + if dormant_res_nodes: + log.warning(f"Resume was unable to resume reservation nodes={dormant_res_nodes}") + down_nodes_notify_jobs(dormant_res_nodes, "Reservation is not active, nodes cannot be resumed", resume_data) + + nodes, flex_managed = util.separate(lkp.is_provisioning_flex_node, nodes) + if flex_managed: + log.warning(f"Resume was unable to resume nodes={flex_managed} already managed by MIGs") + down_nodes_notify_jobs(flex_managed, "VM is managed MIG, can not be resumed", resume_data) + + if not nodes: + log.info("No nodes to resume") + return + + nodes = sorted(nodes, key=lkp.node_prefix) + grouped_nodes = group_nodes_bulk(nodes, resume_data, lkp) + + if log.isEnabledFor(logging.DEBUG): + grouped_nodelists = { + group: to_hostlist(chunk.nodes) for group, chunk in grouped_nodes.items() + } + log.debug( + "node bulk groups: \n{}".format(yaml.safe_dump(grouped_nodelists).rstrip()) + ) + + tpu_chunks, flex_chunks = [], [] + bi_inserts = {} + + for group, chunk in grouped_nodes.items(): + model = chunk.nodes[0] + + if lkp.node_is_tpu(model): + tpu_chunks.append(chunk.nodes) + elif lkp.is_flex_node(model): + flex_chunks.append(chunk) + else: + bi_inserts[group] = create_instances_request( + chunk.nodes, chunk.placement_group, chunk.excl_job_id + ) + + for chunk in flex_chunks: + mig_flex.resume_flex_chunk(chunk.nodes, chunk.excl_job_id, lkp) + + # execute all bulkInsert requests with batch + bulk_ops = dict( + zip(bi_inserts.keys(), map_with_futures(ensure_execute, bi_inserts.values())) + ) + log.debug(f"bulk_ops={yaml.safe_dump(bulk_ops)}") + started = { + group: op for group, op in bulk_ops.items() if not isinstance(op, Exception) + } + failed = { + group: err for group, err in bulk_ops.items() if isinstance(err, Exception) + } + if failed: + failed_reqs = [str(e) for e in failed.items()] + log.error("bulkInsert API failures: {}".format("; ".join(failed_reqs))) + for ident, exc in failed.items(): + down_nodes_notify_jobs(grouped_nodes[ident].nodes, f"GCP Error: {exc._get_reason()}", resume_data) # type: ignore + + if log.isEnabledFor(logging.DEBUG): + for group, op in started.items(): + group_nodes = grouped_nodelists[group] + name = op["name"] + gid = op["operationGroupId"] + log.debug( + f"new bulkInsert operation started: group={group} nodes={group_nodes} name={name} operationGroupId={gid}" + ) + # wait for all bulkInserts to complete and log any errors + bulk_operations = {group: wait_for_operation(op) for group, op in started.items()} + + # Start TPU after regular nodes so that regular nodes are not affected by the slower TPU nodes + execute_with_futures(tpu.start_tpu, tpu_chunks) + + for group, op in bulk_operations.items(): + _handle_bulk_insert_op(op, grouped_nodes[group].nodes, resume_data) + + +def _get_failed_zonal_instance_inserts(bulk_op: Any, zone: str, lkp: util.Lookup) -> list[Any]: + group_id = bulk_op["operationGroupId"] + user = bulk_op["user"] + started = bulk_op["startTime"] + ended = bulk_op["endTime"] + + fltr = f'(user eq "{user}") AND (operationType eq "insert") AND (creationTimestamp > "{started}") AND (creationTimestamp < "{ended}")' + act = lkp.compute.zoneOperations() + req = act.list(project=lkp.project, zone=zone, filter=fltr) + ops = [] + while req is not None: + result = util.ensure_execute(req) + for op in result.get("items", []): + if op.get("operationGroupId") == group_id and "error" in op: + ops.append(op) + req = act.list_next(req, result) + return ops + + +def _get_failed_instance_inserts(bulk_op: Any, lkp: util.Lookup) -> list[Any]: + zones = set() # gather zones that had failed inserts + for loc, stat in bulk_op.get("instancesBulkInsertOperationMetadata", {}).get("perLocationStatus", {}).items(): + pref, zone = loc.split("/", 1) + if not pref == "zones": + log.error(f"Unexpected location: {loc} in operation {bulk_op['name']}") + continue + if stat.get("targetVmCount", 0) != stat.get("createdVmCount", 0): + zones.add(zone) + + res = [] + for zone in zones: + res.extend(_get_failed_zonal_instance_inserts(bulk_op, zone, lkp)) + return res + +def _handle_bulk_insert_op(op: Dict, nodes: List[str], resume_data: Optional[ResumeData]) -> None: + """ + Handles **DONE** BulkInsert operations + """ + assert op["operationType"] == "bulkInsert" and op["status"] == "DONE", f"unexpected op: {op}" + + group_id = op["operationGroupId"] + if "error" in op: + error = op["error"]["errors"][0] + log.error( + f"bulkInsert operation error: {error['code']} name={op['name']} operationGroupId={group_id} nodes={to_hostlist(nodes)}" + ) + + created = 0 + for status in op["instancesBulkInsertOperationMetadata"]["perLocationStatus"].values(): + created += status.get("createdVmCount", 0) + if created == len(nodes): + log.info(f"created {len(nodes)} instances: nodes={to_hostlist(nodes)}") + return # no need to gather status of insert-operations. + + # TODO: don't gather insert-operations per bulkInsert request, instead aggregate it + # across all bulkInserts (goes one level above this function) + failed = _get_failed_instance_inserts(op, util.lookup()) + + # Multiple errors are possible, group by all of them (joined string codes) + by_error_inserts = util.groupby_unsorted( + failed, + lambda op: "+".join(err["code"] for err in op["error"]["errors"]), + ) + for code, failed_ops in by_error_inserts: + failed_ops = list(failed_ops) + failed_nodes = [trim_self_link(op["targetLink"]) for op in failed_ops] + hostlist = util.to_hostlist(failed_nodes) + log.error( + f"{len(failed_nodes)} instances failed to start: {code} ({hostlist}) operationGroupId={group_id}" + ) + + msg = "; ".join( + f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" + for err in failed_ops[0]["error"]["errors"] + ) + if code != "RESOURCE_ALREADY_EXISTS": + down_nodes_notify_jobs(failed_nodes, f"GCP Error: {msg}", resume_data) + log.error( + f"errors from insert for node '{failed_nodes[0]}' ({failed_ops[0]['name']}): {msg}" + ) + + +def down_nodes_notify_jobs(nodes: List[str], reason: str, resume_data: Optional[ResumeData]) -> None: + """set nodes down with reason""" + nodes_set = set(nodes) # turn into set to speed up intersection + jobs = resume_data.jobs if resume_data else [] + reason_quoted = shlex.quote(reason) + + for job in jobs: + if not (set(job.nodes_alloc) & nodes_set): + continue + run(f"{lookup().scontrol} update jobid={job.job_id} admincomment={reason_quoted}", check=False) + run(f"{lookup().scontrol} notify {job.job_id} {reason_quoted}", check=False) + + nodelist = util.to_hostlist(nodes) + log.error(f"Marking nodes {nodelist} as DOWN, reason: {reason}") + run(f"{lookup().scontrol} update nodename={nodelist} state=down reason={reason_quoted}", check=False) + + + + +def create_placement_request(pg_name: str, region: str, max_distance: Optional[int], accelerator_topology: Optional[str]): + config = { + "name": pg_name, + "region": region, + "groupPlacementPolicy": { + "collocation": "COLLOCATED", + "maxDistance": max_distance, + "gpuTopology": accelerator_topology, + }, + } + + request = lookup().compute.resourcePolicies().insert( + project=lookup().project, region=region, body=config + ) + log_api_request(request) + return request + + +def create_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + nodeset_map = collections.defaultdict(list) + for node in nodes: # split nodes on nodesets + nodeset_map[lkp.node_nodeset_name(node)].append(node) + + placements = [] + for _, ns_nodes in nodeset_map.items(): + placements.extend(create_nodeset_placements(ns_nodes, excl_job_id, lkp)) + return placements + + +def _allocate_nodes_to_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + # canned result for no placement policies created + no_pp = [PlacementAndNodes(placement=None, nodes=nodes)] + + model = nodes[0] + nodeset = lkp.node_nodeset(model) + + is_slice = bool(getattr(nodeset, 'accelerator_topology', None)) + + excl_job_placement = (excl_job_id is not None) and (not is_slice) + + if excl_job_placement and len(nodes) < 2: + return no_pp # don't create placement_policy for just one node + + if lkp.is_flex_node(model): + return no_pp # TODO(FLEX): Add support for workload policies + if lkp.node_is_tpu(model): + return no_pp + if not (nodeset.enable_placement and valid_placement_node(model)): + return no_pp + + max_count = calculate_chunk_size(nodeset, lkp) + + name_prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-{nodeset.nodeset_name}" + + if excl_job_placement: # simply chunk given nodes by max size of placement + return [ + PlacementAndNodes(placement=f"{name_prefix}-{excl_job_id}-{i}", nodes=chunk) + for i, chunk in enumerate(chunked(nodes, n=max_count)) + ] + + # split whole nodeset (not only nodes to resume) into chunks of max size of placement + # create placements (most likely already exists) placements for requested nodes + chunks = collections.defaultdict(list) # chunk_id -> nodes + invalid = [] + + for node in nodes: + try: + chunk = lkp.node_index(node) // max_count + chunks[chunk].append(node) + except: + invalid.append(node) + + placements = [ + # NOTE: use 0 instead of job_id for consistency with previous SlurmGCP behavior + PlacementAndNodes(placement=f"{name_prefix}-0-{c_id}", nodes=c_nodes) + for c_id, c_nodes in chunks.items() + ] + + if invalid: + placements.append(PlacementAndNodes(placement=None, nodes=invalid)) + log.error(f"Could not find placement for nodes with unexpected names: {to_hostlist(invalid)}") + + return placements + +def calculate_hosts_per_topo(accelerator_topology: str, machine_type: NSDict) -> int: + # Calculate total number of hosts per topology (Assumes format: '1x72') + try: + top_split = [int(x) for x in accelerator_topology.split("x")] + except Exception as e: + log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") + raise e + + if len(machine_type.accelerators) == 0: + gpus_per_machine = 0 + else: + gpus_per_machine = machine_type.accelerators[0].count + + if len(top_split) != 2: + log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") + elif top_split[0] <= 0 or top_split[1] <= 0: + log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") + elif gpus_per_machine <= 0: + log.error(f"The machine type has no accelerators. Cannot use accelerator topology {accelerator_topology}.") + elif top_split[1] % gpus_per_machine: + log.error(f"The GPU count {gpus_per_machine} per node is not a factor of the accelerator topology {accelerator_topology}") + + return (top_split[0] * top_split[1]) // gpus_per_machine + +def calculate_chunk_size(nodeset: NSDict, lkp: util.Lookup) -> int: + # Calculates the chunk size based on max distance value received or accelerator topology + # Assuming nodeset is not tpu + machine_type = lkp.template_info(nodeset.instance_template).machine_type + max_distance = nodeset.placement_max_distance + accelerator_topology = nodeset.accelerator_topology + + # Look for accelerator topology first + if accelerator_topology: + hosts_per_topo = calculate_hosts_per_topo(accelerator_topology, machine_type) + return hosts_per_topo + + if max_distance == 1: + return 22 + elif max_distance == 2: + if machine_type.family.startswith("a3"): + return 256 + else: + return 150 + elif max_distance == 3: + return 1500 + else: + return PLACEMENT_MAX_CNT + +def create_nodeset_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + placements = _allocate_nodes_to_placements(nodes, excl_job_id, lkp) + region = lkp.node_region(nodes[0]) + max_distance = lkp.node_nodeset(nodes[0]).get('placement_max_distance') + accelerator_topology = lkp.nodeset_accelerator_topology(lkp.node_nodeset_name(nodes[0])) + + if log.isEnabledFor(logging.DEBUG): + debug_p = {p.placement: to_hostlist(p.nodes) for p in placements} + log.debug( + f"creating {len(placements)} placement groups: \n{yaml.safe_dump(debug_p).rstrip()}" + ) + + requests = { + p.placement: create_placement_request(p.placement, region, max_distance, accelerator_topology) for p in placements if p.placement + } + if not requests: + return placements + # TODO: aggregate all requests for whole resume and execute them at once (don't limit to nodeset/job) + ops = dict( + zip(requests.keys(), map_with_futures(ensure_execute, requests.values())) + ) + + def classify_result(item): + op = item[1] + if not isinstance(op, Exception): + return "submitted" + if all(e.get("reason") == "alreadyExists" for e in op.error_details): # type: ignore + return "redundant" + return "failed" + + grouped_ops = dict(util.groupby_unsorted(list(ops.items()), classify_result)) + submitted, redundant, failed = ( + dict(grouped_ops.get(key, {})) for key in ("submitted", "redundant", "failed") + ) + if redundant: + log.warning( + "placement policies already exist: {}".format(",".join(redundant.keys())) + ) + if failed: + reqs = [f"{e}" for _, e in failed.values()] + log.fatal("failed to create placement policies: {}".format("; ".join(reqs))) + operations = {group: wait_for_operation(op) for group, op in submitted.items()} + for group, op in operations.items(): + if "error" in op: + msg = "; ".join( + f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" + for err in op["error"]["errors"] + ) + log.error( + f"placement group failed to create: '{group}' ({op['name']}): {msg}" + ) + + log.info( + f"created {len(operations)} placement groups ({to_hostlist(operations.keys())})" + ) + return placements + + +def valid_placement_node(node: str) -> bool: + invalid_types = frozenset(["e2", "t2d", "n1", "t2a", "m1", "m2", "m3"]) + mt = lookup().node_template_info(node).machineType + if mt.split("-")[0] in invalid_types: + log.warn(f"Unsupported machine type for placement policy: {mt}.") + log.warn( + f"Please do not use any the following machine types with placement policy: ({','.join(invalid_types)})" + ) + return False + return True + + +def main(nodelist: str) -> None: + """main called when run as script""" + log.debug(f"ResumeProgram {nodelist}") + # Filter out nodes not in config.yaml + other_nodes, nodes = separate( + lookup().is_power_managed_node, util.to_hostnames(nodelist) + ) + if other_nodes: + log.error( + f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" + ) + + if not nodes: + log.info("No nodes to resume") + return + resume_data = get_resume_file_data() + log.info(f"resume {util.to_hostlist(nodes)}") + resume_nodes(nodes, resume_data) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("nodelist", help="list of nodes to resume") + args = util.init_log_and_parse(parser) + main(args.nodelist) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh new file mode 100644 index 0000000000..023d246f01 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PYTHON_SCRIPT="${SCRIPT_DIR}/resume.py" + +# Capture all arguments passed by Slurm (the nodelist). +ALL_ARGS=("$@") + +# This array will hold extra argument for resume.py, like the resume data file. +UNIQUE_RESUME_FILE="" + +# Handle SLURM_RESUME_FILE if provided +if [ -n "${SLURM_RESUME_FILE-}" ] && [ -f "$SLURM_RESUME_FILE" ]; then + SAFE_DIR="/tmp/slurm_resume_data" + mkdir -p "$SAFE_DIR" + + UNIQUE_RESUME_FILE="${SAFE_DIR}/resumedata.$$.json" + cp "$SLURM_RESUME_FILE" "$UNIQUE_RESUME_FILE" +fi + +SLURM_RESUME_FILE="${UNIQUE_RESUME_FILE}" +setsid "${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & + +exit 0 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py new file mode 100644 index 0000000000..846524adf2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py @@ -0,0 +1,660 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import shutil +import subprocess +import stat +import time +import yaml +from pathlib import Path +import functools + +import util +from util import ( + lookup, + dirs, + slurmdirs, + run, + install_custom_scripts, +) +import conf +import slurmsync + +from setup_network_storage import ( + setup_network_storage, + setup_nfs_exports, +) + + +log = logging.getLogger() + + +MOTD_HEADER = """ + SSSSSSS + SSSSSSSSS + SSSSSSSSS + SSSSSSSSS + SSSS SSSSSSS SSSS + SSSSSS SSSSSS + SSSSSS SSSSSSS SSSSSS + SSSS SSSSSSSSS SSSS + SSS SSSSSSSSS SSS + SSSSS SSSS SSSSSSSSS SSSS SSSSS + SSS SSSSSS SSSSSSSSS SSSSSS SSS + SSSSSS SSSSSSS SSSSSS + SSS SSSSSS SSSSSS SSS + SSSSS SSSS SSSSSSS SSSS SSSSS + S SSS SSSSSSSSS SSS S + SSS SSSS SSSSSSSSS SSSS SSS + S SSS SSSSSS SSSSSSSSS SSSSSS SSS S + SSSSS SSSSSS SSSSSSSSS SSSSSS SSSSS + S SSSSS SSSS SSSSSSS SSSS SSSSS S + S SSS SSS SSS SSS S + S S S S + SSS + SSS + SSS + SSS + SSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSS SSSSSSSSSSSSSSSSSSSS +SSSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSSS SSSSSSSSSSSSSSSSSSSSSS +SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSSS SSS SSSSSSSSSSSSSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSS SSS SSSSSSSSSSSSS SSSS SSSS SSSS SSSS + +""" +_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" + +def start_motd(): + """advise in motd that slurm is currently configuring""" + wall_msg = "*** Slurm is currently being configured in the background. ***" + motd_msg = MOTD_HEADER + wall_msg + "\n\n" + Path("/etc/motd").write_text(motd_msg) + util.run(f"wall -n '{wall_msg}'", timeout=30) + + +def end_motd(broadcast=True): + """modify motd to signal that setup is complete""" + Path("/etc/motd").write_text(MOTD_HEADER) + + if not broadcast: + return + + run( + "wall -n '*** Slurm {} setup complete ***'".format(lookup().instance_role), + timeout=30, + ) + if not lookup().is_controller: + run( + """wall -n ' +/home on the controller was mounted over the existing /home. +Log back in to ensure your home directory is correct. +'""", + timeout=30, + ) + + +def failed_motd(): + """modify motd to signal that setup is failed""" + wall_msg = f"*** Slurm setup failed! Please view log: {util.get_log_path()} ***" + motd_msg = MOTD_HEADER + wall_msg + "\n\n" + Path("/etc/motd").write_text(motd_msg) + util.run(f"wall -n '{wall_msg}'", timeout=30) + + +def _startup_script_timeout(lkp: util.Lookup) -> int: + if lkp.is_controller: + return lkp.cfg.get("controller_startup_scripts_timeout", 300) + elif lkp.instance_role == "compute": + return lkp.cfg.get("compute_startup_scripts_timeout", 300) + elif lkp.is_login_node: + return lkp.cfg.login_groups[util.instance_login_group()].get("startup_scripts_timeout", 300) + return 300 + + +def run_custom_scripts(): + """run custom scripts based on instance_role""" + custom_dir = dirs.custom_scripts + if lookup().is_controller: + # controller has all scripts, but only runs controller.d + custom_dirs = [custom_dir / "controller.d"] + elif lookup().instance_role == "compute": + # compute setup with nodeset.d + custom_dirs = [custom_dir / "nodeset.d"] + elif lookup().is_login_node: + # login setup with only login.d + custom_dirs = [custom_dir / "login.d"] + else: + # Unknown role: run nothing + custom_dirs = [] + + timeout = _startup_script_timeout(lookup()) + + custom_scripts = [ + p + for d in custom_dirs + for p in d.rglob("*") + if p.is_file() and not p.name.endswith(".disabled") + ] + print_scripts = ",".join(str(s.relative_to(custom_dir)) for s in custom_scripts) + log.debug(f"custom scripts to run: {custom_dir}/({print_scripts})") + + try: + for script in custom_scripts: + log.info(f"running script {script.name} with timeout={timeout}") + result = run(str(script), timeout=timeout, check=False, shell=True) + runlog = ( + f"{script.name} returncode={result.returncode}\n" + f"stdout={result.stdout}stderr={result.stderr}" + ) + log.info(runlog) + result.check_returncode() + except OSError as e: + log.error(f"script {script} is not executable") + raise e + except subprocess.TimeoutExpired as e: + log.error(f"script {script} did not complete within timeout={timeout}") + raise e + except Exception as e: + log.exception(f"script {script} encountered an exception") + raise e + +def mount_save_state_disk(): + disk_name = f"/dev/disk/by-id/google-{lookup().cfg.controller_state_disk.device_name}" + mount_point = util.slurmdirs.state + fs_type = "ext4" + + rdevice = util.run(f"realpath {disk_name}").stdout.strip() + file_output = util.run(f"file -s {rdevice}").stdout.strip() + if "filesystem" not in file_output: + util.run(f"mkfs -t {fs_type} -q {rdevice}") + + fstab_entry = f"{disk_name} {mount_point} {fs_type}" + with open("/etc/fstab", "r") as f: + fstab = f.readlines() + if fstab_entry not in fstab: + with open("/etc/fstab", "a") as f: + f.write(f"{fstab_entry} defaults 0 0\n") + + util.run(f"systemctl daemon-reload") + + os.makedirs(mount_point, exist_ok=True) + util.run(f"mount {mount_point}") + + util.chown_slurm(mount_point) + + +def setup_jwt_key(): + jwt_key = Path(slurmdirs.state / "jwt_hs256.key") + + if jwt_key.exists(): + log.info("JWT key already exists. Skipping key generation.") + else: + run("dd if=/dev/urandom bs=32 count=1 > " + str(jwt_key), shell=True) + + util.chown_slurm(jwt_key, mode=0o400) + + +def _generate_key(p: Path) -> None: + run(f"dd if=/dev/random of={p} bs=1024 count=1") + + +def setup_key(lkp: util.Lookup) -> None: + file_name = "munge.key" + dir = dirs.munge + + if lkp.cfg.enable_slurm_auth: + file_name = "slurm.key" + dir = slurmdirs.etc + + dst = Path(dir / file_name) + + if lkp.cfg.controller_state_disk.device_name: + # Copy key from persistent state disk + persist = slurmdirs.state / file_name + if not persist.exists(): + _generate_key(persist) + + shutil.copyfile(persist, dst) + if lkp.cfg.enable_slurm_auth: + util.chown_slurm(dst, mode=0o400) + util.chown_slurm(persist, mode=0o400) + else: + shutil.chown(dst, user="munge", group="munge") + os.chmod(dst, stat.S_IRUSR) + else: + if dst.exists(): + log.info("key already exists. Skipping key generation.") + else: + _generate_key(dst) + if lkp.cfg.enable_slurm_auth: + util.chown_slurm(dst, mode=0o400) + else: + shutil.chown(dst, user="munge", group="munge") + os.chmod(dst, stat.S_IRUSR) + + if lkp.cfg.enable_slurm_auth: + # Put key into shared volume for distribution + distributed = util.slurmdirs.key_distribution / file_name + shutil.copyfile(dst, distributed) + util.chown_slurm(distributed, mode=0o400) + # Munge is distributed from /etc/munge. + else: + run("systemctl restart munge", timeout=30) + + +def setup_nss_slurm(): + """install and configure nss_slurm""" + # setup nss_slurm + util.mkdirp(Path("/var/spool/slurmd")) + run( + "ln -s {}/lib/libnss_slurm.so.2 /usr/lib64/libnss_slurm.so.2".format( + slurmdirs.prefix + ), + check=False, + ) + run(r"sed -i 's/\(^\(passwd\|group\):\s\+\)/\1slurm /g' /etc/nsswitch.conf") + + +def setup_sudoers(): + content = """ +# Allow SlurmUser to manage the slurm daemons +slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmd.service +slurm ALL= NOPASSWD: /usr/bin/systemctl restart sackd.service +slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmctld.service +""" + sudoers_file = Path("/etc/sudoers.d/slurm") + sudoers_file.write_text(content) + sudoers_file.chmod(0o0440) + + +def setup_maintenance_script(): + perform_maintenance = """#!/bin/bash + +#SBATCH --priority=low +#SBATCH --time=180 + +VM_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") +ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" | cut -d '/' -f 4) + +gcloud compute instances perform-maintenance $VM_NAME \ + --zone=$ZONE +""" + + + with open(_MAINTENANCE_SBATCH_SCRIPT_PATH, "w") as f: + f.write(perform_maintenance) + + util.chown_slurm(_MAINTENANCE_SBATCH_SCRIPT_PATH, mode=0o755) + + +def update_system_config(file, content): + """Add system defaults options for service files""" + sysconfig = Path("/etc/sysconfig") + default = Path("/etc/default") + + if sysconfig.exists(): + conf_dir = sysconfig + elif default.exists(): + conf_dir = default + else: + raise Exception("Cannot determine system configuration directory.") + + slurmd_file = Path(conf_dir, file) + slurmd_file.write_text(content) + +def _symlink_mysql_datadir(lkp: util.Lookup) -> None: + """ Symlink /var/lib/mysql to controller state disk if needed. """ + if not lkp.cfg.controller_state_disk.device_name: + return + + datadir = Path("/var/lib/mysql") + dst = slurmdirs.state / "mysql" + + if dst.exists(): + run(f"rm -rf {datadir}") + else: + shutil.move(datadir, dst) + + datadir.symlink_to(dst, target_is_directory=True) + shutil.chown(datadir, user="mysql", group="mysql") + run(f"chown -R mysql:mysql {dst}") + +def configure_mysql(lkp: util.Lookup) -> None: + cnfdir = Path("/etc/my.cnf.d") + if not cnfdir.exists(): + cnfdir = Path("/etc/mysql/conf.d") + if not (cnfdir / "mysql_slurm.cnf").exists(): + (cnfdir / "mysql_slurm.cnf").write_text( + """ +[mysqld] +bind-address=127.0.0.1 +innodb_buffer_pool_size=1024M +innodb_log_file_size=64M +innodb_lock_wait_timeout=900 +""" + ) + + run("systemctl stop mariadb", timeout=30) + _symlink_mysql_datadir(lkp) + + run("systemctl enable mariadb", timeout=30) + run("systemctl restart mariadb", timeout=30) + + db_name = "slurm_acct_db" + + + cmd = "mysql -u root -e" + for host in ("localhost", lkp.control_host): + run(f"""{cmd} "drop user if exists 'slurm'@'{host}'";""", timeout=30) + run(f"""{cmd} "create user 'slurm'@'{host}'";""", timeout=30) + run(f"""{cmd} "grant all on {db_name}.* TO 'slurm'@'{host}'";""", timeout=30) + + +def configure_dirs(): + for p in dirs.values(): + util.mkdirp(p) + + for p in (dirs.slurm, dirs.scripts, dirs.custom_scripts): + util.chown_slurm(p) + + for p in slurmdirs.values(): + util.mkdirp(p) + util.chown_slurm(p) + + for sl, tgt in ( # create symlinks + (Path("/etc/slurm"), slurmdirs.etc), + (dirs.scripts / "etc", slurmdirs.etc), + (dirs.scripts / "log", dirs.log), + ): + if sl.exists() and sl.is_symlink(): + sl.unlink() + sl.symlink_to(tgt) + + # copy auxiliary scripts + for dst_folder, src_file in ((lookup().cfg.slurm_bin_dir, + Path("sort_nodes.py")), + (dirs.custom_scripts / "task_prolog.d", + Path("tools/task-prolog")), + (dirs.custom_scripts / "task_epilog.d", + Path("tools/task-epilog"))): + dst = Path(dst_folder) / src_file.name + util.mkdirp(dst.parent) + shutil.copyfile(util.scripts_dir / src_file, dst) + os.chmod(dst, 0o755) + + +def self_report_controller_address(lkp: util.Lookup) -> None: + if not lkp.cfg.controller_network_attachment: + return # only self report address if network attachment is used + data = { "slurm_control_addr": lkp.cfg.slurm_control_addr } + bucket, prefix = util._get_bucket_and_common_prefix() + blob = util.storage_client().bucket(bucket).blob(f"{prefix}/controller_addr.yaml") + with blob.open('w') as f: + f.write(yaml.dump(data)) + +def setup_controller(): + """Run controller setup""" + log.info("Setting up controller") + lkp = util.lookup() + util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) + install_custom_scripts() + conf.gen_controller_configs(lkp) + + if lkp.cfg.controller_state_disk.device_name != None: + mount_save_state_disk() + + setup_jwt_key() + setup_key(lkp) + + setup_sudoers() + setup_network_storage() + + run_custom_scripts() + + if not lkp.cfg.cloudsql_secret: + configure_mysql(lkp) + + run("systemctl enable slurmdbd", timeout=30) + run("systemctl restart slurmdbd", timeout=30) + + # Wait for slurmdbd to come up + time.sleep(5) + + sacctmgr = f"{slurmdirs.prefix}/bin/sacctmgr -i" + result = run( + f"{sacctmgr} add cluster {lkp.cfg.slurm_cluster_name}", timeout=30, check=False + ) + if "already exists" in result.stdout: + log.info(result.stdout) + elif result.returncode > 1: + result.check_returncode() # will raise error + + run("systemctl enable slurmctld", timeout=30) + run("systemctl restart slurmctld", timeout=30) + + run("systemctl enable slurmrestd", timeout=30) + run("systemctl restart slurmrestd", timeout=30) + + # Export at the end to signal that everything is up + run("systemctl enable nfs-server", timeout=30) + run("systemctl start nfs-server", timeout=30) + + setup_nfs_exports() + run("systemctl enable --now slurmcmd.timer", timeout=30) + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status slurmdbd", timeout=30) + run("systemctl status slurmctld", timeout=30) + run("systemctl status slurmrestd", timeout=30) + + try: + slurmsync.sync_instances() + except Exception: + log.exception("Failed to sync instances, will try next time.") + + run("systemctl enable slurm_load_bq.timer", timeout=30) + run("systemctl start slurm_load_bq.timer", timeout=30) + run("systemctl status slurm_load_bq.timer", timeout=30) + + # Add script to perform maintenance + setup_maintenance_script() + + self_report_controller_address(lkp) + + log.info("Done setting up controller") + pass + + +def setup_login(): + """run login node setup""" + log.info("Setting up login") + + lkp = lookup() + slurmctld_host = f"{lkp.control_host}" + if lkp.control_addr: + slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" + sackd_options = [ + f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', + ] + sysconf = f"""SACKD_OPTIONS='{" ".join(sackd_options)}'""" + update_system_config("sackd", sysconf) + install_custom_scripts() + + setup_network_storage() + setup_sudoers() + if not lkp.cfg.enable_slurm_auth: + run("systemctl restart munge", timeout=30) + run("systemctl enable sackd", timeout=30) + run("systemctl restart sackd", timeout=30) + run("systemctl enable --now slurmcmd.timer", timeout=30) + + run_custom_scripts() + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status sackd", timeout=30) + + log.info("Done setting up login") + + +def setup_compute(): + """run compute node setup""" + log.info("Setting up compute") + + lkp = lookup() + util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) + slurmctld_host = f"{lkp.control_host}" + if lkp.control_addr: + slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" + slurmd_options = [ + f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', + ] + + try: + slurmd_feature = util.instance_metadata("attributes/slurmd_feature", silent=True) + except util.MetadataNotFoundError: + slurmd_feature = None + + if slurmd_feature is not None: + slurmd_options.append(f'--conf="Feature={slurmd_feature}"') + slurmd_options.append("-Z") + + sysconf = f"""SLURMD_OPTIONS='{" ".join(slurmd_options)}'""" + update_system_config("slurmd", sysconf) + install_custom_scripts() + + setup_nss_slurm() + setup_network_storage() + + has_gpu = run("lspci | grep --ignore-case 'NVIDIA' | wc -l", shell=True).returncode + if has_gpu: + run("nvidia-smi") + + run_custom_scripts() + + setup_sudoers() + if not lkp.cfg.enable_slurm_auth: + run("systemctl restart munge", timeout=30) + run("systemctl enable slurmd", timeout=30) + run("systemctl restart slurmd", timeout=30) + run("systemctl enable --now slurmcmd.timer", timeout=30) + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status slurmd", timeout=30) + + log.info("Done setting up compute") + +def setup_cloud_ops() -> None: + """Add health checks, deployment info, and updated setup path to cloud ops config.""" + cloudOpsStatus = run( + "systemctl is-active --quiet google-cloud-ops-agent.service", check=False + ).returncode + + if cloudOpsStatus != 0: + return + + with open("/etc/google-cloud-ops-agent/config.yaml", "r") as f: + file = yaml.safe_load(f) + + # Update setup receiver path + file["logging"]["receivers"]["setup"]["include_paths"] = ["/var/log/slurm/setup.log"] + + cluster_info = { + 'type':'modify_fields', + 'fields': { + 'labels."cluster_name"':{ + 'static_value':f"{lookup().cfg.slurm_cluster_name}" + }, + 'labels."hostname"':{ + 'static_value': f"{lookup().hostname}" + } + } + } + + file["logging"]["processors"]["add_cluster_info"] = cluster_info + file["logging"]["service"]["pipelines"]["slurmlog_pipeline"]["processors"].append("add_cluster_info") + file["logging"]["service"]["pipelines"]["slurmlog2_pipeline"]["processors"].append("add_cluster_info") + + with open("/etc/google-cloud-ops-agent/config.yaml", "w") as f: + yaml.safe_dump(file, f, sort_keys=False) + + retries = 2 + for _ in range(retries): + try: + run("systemctl restart google-cloud-ops-agent.service", timeout=120) + break + except subprocess.TimeoutExpired: + log.error("google-cloud-ops-agent.service did not restart within 120s.") + result=run("cat /var/log/google-cloud-ops-agent/subagents/logging-module.log", timeout=120, shell=True) + if result.stdout: + log.error(f"Logs for google-cloud-ops-agent (logging-module.log file):\n{result.stdout}") + raise + + +def main(): + start_motd() + + log.info("Starting setup, fetching config") + sleep_seconds = 5 + while True: + try: + _, cfg = util.fetch_config() + util.update_config(cfg) + break + except util.DeffetiveStoredConfigError as e: + log.warning(f"config is not ready yet: {e}, sleeping for {sleep_seconds}s") + except Exception as e: + log.exception(f"unexpected error while fetching config, sleeping for {sleep_seconds}s") + time.sleep(sleep_seconds) + log.info("Config fetched") + setup_cloud_ops() + configure_dirs() + # call the setup function for the instance type + { + "controller": setup_controller, + "compute": setup_compute, + "login": setup_login, + }.get( + lookup().instance_role, + lambda: log.fatal(f"Unknown node role: {lookup().instance_role}"))() + + end_motd() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--slurmd-feature", dest="slurmd_feature", help="Unused, to be removed.") + _ = util.init_log_and_parse(parser) + + try: + main() + except Exception: + log.exception("Aborting setup...") + failed_motd() diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py new file mode 100644 index 0000000000..095f42e758 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py @@ -0,0 +1,327 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import os +import sys +import stat +import time +import logging +import uuid + +import shutil +from pathlib import Path +from concurrent.futures import as_completed +from addict import Dict as NSDict # type: ignore + +import util +from util import NSMount, lookup, run, dirs, separate +from more_executors import Executors, ExceptionRetryPolicy + + +log = logging.getLogger() + +def mounts_by_local(mounts: list[NSMount]) -> dict[str, NSMount]: + """convert list of mounts to dict of mounts, local_mount as key""" + return {str(m.local_mount.resolve()): m for m in mounts} + + +def _get_default_mounts(lkp: util.Lookup) -> list[NSMount]: + if lkp.cfg.disable_default_mounts: + return [] + return [ + NSMount( + server_ip=lkp.controller_mount_server_ip(), + remote_mount=path, + local_mount=path, + fs_type="nfs", + mount_options="defaults,hard,intr", + ) + for path in ( + dirs.home, + dirs.apps, + ) + ] + +def get_slurm_bucket_mount() -> NSMount: + bucket, path = util._get_bucket_and_common_prefix() + return NSMount( + fs_type="gcsfuse", + server_ip="", + remote_mount=Path(bucket), + local_mount=dirs.slurm_bucket_mount, + mount_options=f"defaults,_netdev,implicit_dirs,only_dir={path}", + ) + +def resolve_network_storage() -> List[NSMount]: + """Combine appropriate network_storage fields to a single list""" + lkp = lookup() + + # create dict of mounts, local_mount: mount_info + mounts = mounts_by_local(_get_default_mounts(lkp)) + + if lkp.is_controller and util.should_mount_slurm_bucket(): + mounts.update(mounts_by_local([get_slurm_bucket_mount()])) + + # On non-controller instances, entries in network_storage could overwrite + # default exports from the controller. Be careful, of course + common = [lkp.normalize_ns_mount(m) for m in lkp.cfg.network_storage] + mounts.update(mounts_by_local(common)) + + if lkp.is_login_node: + login_group = lkp.cfg.login_groups[util.instance_login_group()] + login_ns = [lkp.normalize_ns_mount(m) for m in login_group.network_storage] + mounts.update(mounts_by_local(login_ns)) + + if lkp.instance_role == "compute": + try: + nodeset = lkp.node_nodeset() + except Exception: + pass # external nodename, skip lookup + else: + nodeset_ns = [lkp.normalize_ns_mount(m) for m in nodeset.network_storage] + mounts.update(mounts_by_local(nodeset_ns)) + + return list(mounts.values()) + + +def is_controller_mount(mount) -> bool: + # NOTE: Valid Lustre server_ip can take the form of '@tcp' + server_ip = mount.server_ip.split("@")[0] + mount_addr = util.host_lookup(server_ip) + return mount_addr == lookup().control_host_addr + +def setup_network_storage(): + """prepare network fs mounts and add them to fstab""" + log.info("Set up network storage") + + all_mounts = resolve_network_storage() + if lookup().is_controller: + mounts, _ = separate(is_controller_mount, all_mounts) + else: + mounts = all_mounts + + # Determine fstab entries and write them out + fstab_entries = [] + for mount in mounts: + local_mount = mount.local_mount + fs_type = mount.fs_type + server_ip = mount.server_ip or "" + src = mount.remote_mount if fs_type == "gcsfuse" else f"{server_ip}:{mount.remote_mount}" + + log.info(f"Setting up mount ({fs_type}) {src} to {local_mount}") + util.mkdirp(local_mount) + + mount_options = mount.mount_options.split(",") if mount.mount_options else [] + if "_netdev" not in mount_options: + mount_options += ["_netdev"] + options_line = ",".join(mount_options) + + + fstab_entries.append(f"{src} {local_mount} {fs_type} {options_line} 0 0") + + fstab = Path("/etc/fstab") + if not Path(fstab.with_suffix(".bak")).is_file(): + shutil.copy2(fstab, fstab.with_suffix(".bak")) + shutil.copy2(fstab.with_suffix(".bak"), fstab) + with open(fstab, "a") as f: + f.write("\n") + for entry in fstab_entries: + f.write(entry) + f.write("\n") + + mount_fstab(mounts, log) + if lookup().cfg.enable_slurm_auth: + slurm_key_mount_handler() + else: + munge_mount_handler() + + +def mount_fstab(mounts: list[NSMount], log): + """Wait on each mount, then make sure all fstab is mounted""" + def mount_path(path: Path): + log.info(f"Waiting for '{path}' to be mounted...") + try: + run(f"mount {path}", timeout=120) + except Exception as e: + exc_type, _, _ = sys.exc_info() + log.error(f"mount of path '{path}' failed: {exc_type}: {e}") + raise e + log.info(f"Mount point '{path}' was mounted.") + + MAX_MOUNT_TIMEOUT = 60 * 5 + future_list = [] + retry_policy = ExceptionRetryPolicy( + max_attempts=120, exponent=1.6, sleep=1.0, max_sleep=16.0 + ) + with Executors.thread_pool().with_timeout(MAX_MOUNT_TIMEOUT).with_retry( + retry_policy=retry_policy + ) as exe: + for m in mounts: + future = exe.submit(mount_path, m.local_mount) + future_list.append(future) + + # Iterate over futures, checking for exceptions + for future in as_completed(future_list): + try: + future.result() + except Exception as e: + raise e + + +def munge_mount_handler(): + if lookup().is_controller: + return + mnt = lookup().munge_mount + + log.info(f"Mounting munge share to: {mnt.local_mount}") + mnt.local_mount.mkdir() + if mnt.fs_type == "gcsfuse": + cmd = [ + "gcsfuse", + f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, + mnt.server_ip, + str(mnt.local_mount), + ] + else: + cmd = [ + "mount", + f"--types={mnt.fs_type}", + f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, + f"{mnt.server_ip}:{mnt.remote_mount}", + str(mnt.local_mount), + ] + # wait max 240s for munge mount + timeout = 240 + for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): + try: + run(cmd, timeout=timeout) + break + except Exception as e: + log.error( + f"munge mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" + ) + time.sleep(wait) + err = e + continue + else: + raise err + + munge_key = Path(dirs.munge / "munge.key") + log.info(f"Copy munge.key from: {mnt.local_mount}") + shutil.copy2(Path(mnt.local_mount / "munge.key"), munge_key) + + log.info("Restrict permissions of munge.key") + shutil.chown(munge_key, user="munge", group="munge") + os.chmod(munge_key, stat.S_IRUSR) + + log.info(f"Unmount {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + run(f"fusermount -u {mnt.local_mount}", timeout=120) + else: + run(f"umount {mnt.local_mount}", timeout=120) + shutil.rmtree(mnt.local_mount) + +def slurm_key_mount_handler(): + if lookup().is_controller: + return + mnt = lookup().slurm_key_mount + + log.info(f"Mounting slurm_key share to: {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + cmd = [ + "gcsfuse", + f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, + mnt.server_ip, + str(mnt.local_mount), + ] + else: + cmd = [ + "mount", + f"--types={mnt.fs_type}", + f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, + f"{mnt.server_ip}:{mnt.remote_mount}", + str(mnt.local_mount), + ] + timeout = 120 # wait max 120s to mount + for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): + try: + run(cmd, timeout=timeout) + break + except Exception as e: + log.error( + f"slurm key mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" + ) + time.sleep(wait) + err = e + continue + else: + raise err + + file_name = "slurm.key" + dst = Path(util.slurmdirs.etc / file_name) + log.info(f"Copy slurm.key from: {mnt.local_mount}") + shutil.copy2(mnt.local_mount / file_name, dst) + + log.info("Restrict permissions of slurm.key") + util.chown_slurm(dst, mode=0o400) + + log.info(f"Unmount {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + run(f"fusermount -u {mnt.local_mount}", timeout=120) + else: + run(f"umount {mnt.local_mount}", timeout=120) + shutil.rmtree(mnt.local_mount) + + +def setup_nfs_exports(): + """nfs export all needed directories""" + lkp = util.lookup() + assert lkp.is_controller + + # The controller only needs to set up exports for cluster-internal mounts + exported_mounts = [m for m in resolve_network_storage() if is_controller_mount(m)] + + # key by remote mount path since that is what needs exporting + to_export = {m.remote_mount: "*(rw,no_subtree_check,no_root_squash)" for m in exported_mounts} + + key_mount = lkp.slurm_key_mount if lkp.cfg.enable_slurm_auth else lkp.munge_mount + if is_controller_mount(key_mount): + # Export key mount as read-only + to_export[key_mount.remote_mount] = "*(ro,no_subtree_check,no_root_squash)" + + if util.should_mount_slurm_bucket(): + mnt = get_slurm_bucket_mount() + # FSID is required for virtual filesystem that is not based on a device + # Also export it as read-only + fsid=str(uuid.uuid4()) + to_export[mnt.local_mount] = f"*(ro,no_subtree_check,no_root_squash,fsid={fsid})" + + # export path if corresponding selector boolean is True + lines = [] + for path,options in to_export.items(): + util.mkdirp(Path(path)) + run(rf"sed -i '\#{path}#d' /etc/exports", timeout=30) + lines.append(f"{path} {options}") + + exportsd = Path("/etc/exports.d") + util.mkdirp(exportsd) + with (exportsd / "slurm.exports").open("w") as f: + f.write("\n") + f.write("\n".join(lines)) + run("exportfs -a", timeout=30) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py new file mode 100644 index 0000000000..1bfdd5acce --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py @@ -0,0 +1,679 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import fcntl +import json +import logging +import re +import sys +import shlex +from datetime import datetime, timedelta +from itertools import chain +from pathlib import Path +from dataclasses import dataclass +from typing import Dict, Tuple, List, Optional, Protocol, Any +from functools import lru_cache + +import util +from util import ( + batch_execute, + ensure_execute, + execute_with_futures, + FutureReservation, + install_custom_scripts, + run, + separate, + to_hostlist, + NodeState, + chunked, + dirs, +) +from util import lookup +from suspend import delete_instances +import tpu +import conf +import watch_delete_vm_op + +log = logging.getLogger() + +TOT_REQ_CNT = 1000 +_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" + +class NodeAction(Protocol): + def apply(self, nodes:List[str]) -> None: + ... + + def __hash__(self): + ... + +@dataclass(frozen=True) +class NodeActionPowerUp(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to resume ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=power_up") + +@dataclass(frozen=True) +class NodeActionIdle(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} nodes to idle ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=resume") + +@dataclass(frozen=True) +class NodeActionPowerDown(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to power down ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=power_down") + + +@dataclass(frozen=True) +class NodeActionPowerDownForce(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to power down ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=power_down_force") + + +@dataclass(frozen=True) +class NodeActionDelete(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to delete ({hostlist})") + delete_instances(nodes) + +@dataclass(frozen=True) +class NodeActionPrempt(): + def apply(self, nodes:List[str]) -> None: + NodeActionDown(reason="Preempted instance").apply(nodes) + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances restarted ({hostlist})") + start_instances(nodes) + +@dataclass(frozen=True) +class NodeActionUnchanged(): + def apply(self, nodes:List[str]) -> None: + pass + +@dataclass(frozen=True) +class NodeActionDown(): + reason: str + + def apply(self, nodes: List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} nodes set down ({hostlist}) with reason={self.reason}") + run(f"{lookup().scontrol} update nodename={hostlist} state=down reason={shlex.quote(self.reason)}") + +@dataclass(frozen=True) +class NodeActionUnknown(): + slurm_state: Optional[NodeState] + instance_state: Optional[str] + + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.error(f"{len(nodes)} nodes have unexpected {self.slurm_state} and instance state:{self.instance_state}, ({hostlist})") + +def start_instance_op(node: str) -> Any: + inst = lookup().instance(node) + assert inst + + return lookup().compute.instances().start( + project=lookup().project, + zone=inst.zone, + instance=inst.name, + ) + + +def start_instances(node_list): + log.info("{} instances to start ({})".format(len(node_list), ",".join(node_list))) + lkp = lookup() + # TODO: use code from resume.py to assign proper placement + normal, tpu_nodes = separate(lkp.node_is_tpu, node_list) + ops = {node: start_instance_op(node) for node in normal} + + done, failed = batch_execute(ops) + + tpu_start_data = [] + for ns, nodes in util.groupby_unsorted(tpu_nodes, lkp.node_nodeset_name): + tpuobj = tpu.TPU.make(ns, lkp) + for snodes in chunked(nodes, n=tpuobj.vmcount): + tpu_start_data.append({"tpu": tpuobj, "node": snodes}) + execute_with_futures(tpu.start_tpu, tpu_start_data) + + +def _find_dynamic_node_status() -> NodeAction: + # TODO: cover more cases: + # * delete dead dynamic nodes + # * delete orhpaned instances + return NodeActionUnchanged() # don't touch dynamic nodes + +def get_fr_action(fr: FutureReservation, state:Optional[NodeState]) -> Optional[NodeAction]: + now = util.now() + if state is None: + return None # handle like any other node + if fr.start_time < now < fr.end_time: + return None # handle like any other node + + if state.base == "DOWN": + return NodeActionUnchanged() + if fr.start_time >= now: + msg = f"Waiting for reservation:{fr.name} to start at {fr.start_time}" + else: + msg = f"Reservation:{fr.name} is after its end-time" + return NodeActionDown(reason=msg) + +def _find_tpu_node_action(nodename, state) -> NodeAction: + lkp = lookup() + tpuobj = tpu.TPU.make(lkp.node_nodeset_name(nodename), lkp) + inst = tpuobj.get_node(nodename) + # If we do not find the node but it is from a Tpu that has multiple vms look for the master node + if inst is None and tpuobj.vmcount > 1: + # Get the tpu slurm nodelist of the nodes in the same tpu group as nodename + nodelist = run( + f"{lkp.scontrol} show topo {nodename}" + + " | awk -F'=' '/Level=0/ { print $NF }'", + shell=True, + ).stdout + l_nodelist = util.to_hostnames(nodelist) + group_names = set(l_nodelist) + # get the list of all the existing tpus in the nodeset + tpus_list = set(tpuobj.list_node_names()) + # In the intersection there must be only one node that is the master + tpus_int = list(group_names.intersection(tpus_list)) + if len(tpus_int) > 1: + log.error( + f"More than one cloud tpu node for tpu group {nodelist}, there should be only one that should be {l_nodelist[0]}, but we have found {tpus_int}" + ) + return NodeActionUnknown(slurm_state=state, instance_state=None) + if len(tpus_int) == 1: + inst = tpuobj.get_node(tpus_int[0]) + # if len(tpus_int ==0) this case is not relevant as this would be the case always that a TPU group is not running + if inst is None: + if state.base == "DOWN" and "POWERED_DOWN" in state.flags: + return NodeActionIdle() + if "POWERING_DOWN" in state.flags: + return NodeActionIdle() + if "COMPLETING" in state.flags: + return NodeActionDown(reason="Unbacked instance") + if state.base != "DOWN" and not ( + set(("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN")) + & state.flags + ): + return NodeActionDown(reason="Unbacked instance") + if lkp.is_static_node(nodename): + return NodeActionPowerUp() + elif ( + state is not None + and "POWERED_DOWN" not in state.flags + and "POWERING_DOWN" not in state.flags + and inst.state == tpu.TPU.State.STOPPED + ): + if tpuobj.preemptible: + return NodeActionPrempt() + if state.base != "DOWN": + return NodeActionDown(reason="Instance terminated") + elif ( + state is None or "POWERED_DOWN" in state.flags + ) and inst.state == tpu.TPU.State.READY: + return NodeActionDelete() + elif state is None: + # if state is None here, the instance exists but it's not in Slurm + return NodeActionUnknown(slurm_state=state, instance_state=inst.status) + + return NodeActionUnchanged() + +def get_node_action(nodename: str) -> NodeAction: + """Determine node/instance status that requires action""" + lkp = lookup() + state = lkp.node_state(nodename) + + if lkp.node_is_gke(nodename): + return NodeActionUnchanged() + + if lkp.node_is_fr(nodename): + fr = lkp.future_reservation(lkp.node_nodeset(nodename)) + assert fr + if action := get_fr_action(fr, state): + return action + + if lkp.node_is_dyn(nodename): + return _find_dynamic_node_status() + + if lkp.node_is_tpu(nodename): + return _find_tpu_node_action(nodename, state) + + # split below is workaround for VMs whose hostname is FQDN + inst = lkp.instance(nodename.split(".")[0]) + power_flags = frozenset( + ("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN") + ) & (state.flags if state is not None else set()) + + if (state is None) and (inst is None): + # Should never happen + return NodeActionUnknown(None, None) + if inst is None: + assert state is not None # to keep type-checker happy + if "POWERING_UP" in state.flags: + return NodeActionUnchanged() + if state.base == "DOWN" and "POWERED_DOWN" in state.flags: + return NodeActionIdle() + if "POWERING_DOWN" in state.flags: + return NodeActionIdle() + if "COMPLETING" in state.flags: + return NodeActionDown(reason="Unbacked instance") + if state.base != "DOWN" and not power_flags: + return NodeActionDown(reason="Unbacked instance") + if state.base == "DOWN" and not power_flags: + return NodeActionPowerDown() + if "NOT_RESPONDING" in state.flags: + return NodeActionPowerDown() + if "POWERED_DOWN" in state.flags and lkp.is_static_node(nodename): + return NodeActionPowerUp() + elif ( + state is not None + and "POWERED_DOWN" not in state.flags + and "POWERING_DOWN" not in state.flags + and inst.status == "TERMINATED" + ): + if inst.scheduling.preemptible: + return NodeActionPrempt() + if state.base != "DOWN": + return NodeActionDown(reason="Instance terminated") + elif (state is None or "POWERED_DOWN" in state.flags) and inst.status == "RUNNING": + log.info("%s is potential orphan node", nodename) + threshold = timedelta(seconds=90) + age = util.now() - inst.creation_timestamp + log.info(f"{nodename} state: {state}, age: {age}") + if age < threshold: + log.info(f"{nodename} not marked as orphan, it started less than {threshold.seconds}s ago ({age.seconds}s)") + return NodeActionUnchanged() + return NodeActionDelete() + elif state is None: + # if state is None here, the instance exists but it's not in Slurm + return NodeActionUnknown(slurm_state=state, instance_state=inst.status) + elif lkp.is_flex_node(nodename) and "POWERING_UP" in state.flags: + threshold = timedelta(seconds=int(lkp.cfg.compute_startup_scripts_timeout) * 2) #extra buffer for unexpectedly long startup scripts + if util.now() - inst.creation_timestamp > threshold: + log.info(f"{nodename} was unable to join the cluster after {threshold.seconds}s, potential failure on VM startup. Powering down...") + return NodeActionPowerDownForce() + return NodeActionUnchanged() + + +def delete_resource_policies(links: list[str], lkp: util.Lookup) -> None: + requests = {} + for link in links: + name = util.trim_self_link(link) + region = util.parse_self_link(link).region + requests[name] = lkp.compute.resourcePolicies().delete(project=lkp.project, region=region, resourcePolicy=name) + + def swallow_err(_: str) -> None: + pass + + done, failed = batch_execute(requests, log_err=swallow_err) + if failed: + # Filter out resourceInUseByAnotherResource errors , they are expected to happen + def ignore_err(e) -> bool: + return "resourceInUseByAnotherResource" in str(e) + + failures = [f"{n}: {e}" for n, (_, e) in failed.items() if not ignore_err(e)] + if failures: + log.error(f"some placement groups failed to delete: {failures}") + log.info( + f"deleted {len(done)} of {len(links)} placement groups ({to_hostlist(done.keys())})" + ) + + + +@lru_cache +def _get_resource_policies_in_region(lkp: util.Lookup, region: str) -> list[Any]: + res = [] + act = lkp.compute.resourcePolicies() + op = act.list(project=lkp.project, region=region) + prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-" + while op is not None: + result = ensure_execute(op) + res.extend([p for p in result.get("items", []) if p.get("name", "").startswith(prefix)]) + op = act.list_next(op, result) + return res + + +@lru_cache +def _get_resource_policies(lkp: util.Lookup) -> list[Any]: + res = [] + for region in lkp.cluster_regions(): + res.extend(_get_resource_policies_in_region(lkp, region)) + return res + +def sync_placement_groups(): + """Delete placement policies that are for jobs that have completed/terminated""" + keep_states = frozenset( + [ + "RUNNING", + "CONFIGURING", + "STOPPED", + "SUSPENDED", + "COMPLETING", + "PENDING", + ] + ) + + lkp = lookup() + keep_jobs = { + str(job.id) + for job in lkp.get_jobs() + if job.job_state in keep_states + } + keep_jobs.add("0") # Job 0 is a placeholder for static node placement + + to_delete = [] + pg_regex = re.compile( + rf"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-(?P[^\s\-]+)-(?P\d+)-(?P\d+)" + ) + + for pg in _get_resource_policies(lkp): + name = pg["name"] + + if (mtch := pg_regex.match(name)) is None: + log.warning(f"Unexpected resource policy {name=}") + continue + if mtch.group("job_id") not in keep_jobs: + to_delete.append(pg["selfLink"]) + + if to_delete: + delete_resource_policies(to_delete, lkp) + + +def sync_instances(): + compute_instances = { + name for name, inst in lookup().instances().items() if inst.role == "compute" + } + slurm_nodes = set(lookup().slurm_nodes().keys()) + log.debug(f"reconciling {len(compute_instances)} GCP instances and {len(slurm_nodes)} Slurm nodes.") + + for action, nodes in util.groupby_unsorted(list(compute_instances | slurm_nodes), get_node_action): + action.apply(list(nodes)) + + +def reconfigure_slurm(): + update_msg = "*** slurm configuration was updated ***" + if lookup().cfg.hybrid: + # terraform handles generating the config.yaml, don't do it here + return + + upd, cfg_new = util.fetch_config() + if not upd: + log.debug("No changes in config detected.") + return + log.debug("Changes in config detected. Reconfiguring Slurm now.") + util.update_config(cfg_new) + + if lookup().is_controller: + conf.gen_controller_configs(lookup()) + log.info("Restarting slurmctld to make changes take effect.") + try: + # TODO: consider removing "restart" since "reconfigure" should restart slurmctld as well + run("sudo systemctl restart slurmctld.service", check=False) + util.scontrol_reconfigure(lookup()) + except Exception: + log.exception("failed to reconfigure slurmctld") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + elif lookup().instance_role_safe == "compute": + log.info("Restarting slurmd to make changes take effect.") + run("systemctl restart slurmd") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + elif lookup().is_login_node: + log.info("Restarting sackd to make changes take effect.") + run("systemctl restart sackd") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + + +def update_topology(lkp: util.Lookup) -> None: + if conf.topology_plugin(lkp) != conf.TOPOLOGY_PLUGIN_TREE: + return + updated, summary = conf.gen_topology_conf(lkp) + if updated: + log.info("Topology configuration updated. Reconfiguring Slurm.") + util.scontrol_reconfigure(lkp) + # Safe summary only after Slurm got reconfigured, so summary reflects Slurm POV + summary.dump(lkp) + + +def delete_reservation(lkp: util.Lookup, reservation_name: str) -> None: + util.run(f"{lkp.scontrol} delete reservation {reservation_name}") + + +def create_reservation(lkp: util.Lookup, reservation_name: str, node: str, start_time: datetime) -> None: + # Format time to be compatible with slurm reservation. + formatted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S') + + util.run(f"{lkp.scontrol} create reservation user=slurm starttime={formatted_start_time} duration=180 nodes={node} reservationname={reservation_name} flags=maint,ignore_jobs") + + +def get_slurm_reservation_maintenance(lkp: util.Lookup) -> Dict[str, datetime]: + res = util.run(f"{lkp.scontrol} show reservation --json") + all_reservations = json.loads(res.stdout) + reservation_map = {} + + for reservation in all_reservations['reservations']: + name = reservation.get('name') + nodes = reservation.get('node_list') + time_epoch = reservation.get('start_time', {}).get('number') + + if name is None or nodes is None or time_epoch is None: + continue + + if reservation.get('node_count') != 1: + continue + + if name != f"{nodes}_maintenance": + continue + + reservation_map[name] = datetime.fromtimestamp(time_epoch) + + return reservation_map + +@lru_cache +def get_upcoming_maintenance(lkp: util.Lookup) -> Dict[str, Tuple[str, datetime]]: + upc_maint_map = {} + + for node, inst in lkp.instances().items(): + if inst.resource_status.upcoming_maintenance: + upc_maint_map[node + "_maintenance"] = (node, inst.resource_status.upcoming_maintenance.window_start_time) + + return upc_maint_map + + +def sync_maintenance_reservation(lkp: util.Lookup) -> None: + upc_maint_map = get_upcoming_maintenance(lkp) # map reservation_name -> (node_name, time) + log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") + + curr_reservation_map = get_slurm_reservation_maintenance(lkp) # map reservation_name -> time + log.debug(f"curr-reservation-map: {curr_reservation_map}") + + del_reservation = set(curr_reservation_map.keys() - upc_maint_map.keys()) + create_reservation_map = {} + + for res_name, (node, start_time) in upc_maint_map.items(): + try: + enabled = lkp.node_nodeset(node).enable_maintenance_reservation + except Exception: + enabled = False + + if not enabled: + if res_name in curr_reservation_map: + del_reservation.add(res_name) + continue + + if res_name in curr_reservation_map: + diff = curr_reservation_map[res_name] - start_time + if abs(diff) <= timedelta(seconds=1): + continue + else: + del_reservation.add(res_name) + create_reservation_map[res_name] = (node, start_time) + else: + create_reservation_map[res_name] = (node, start_time) + + log.debug(f"del-reservation: {del_reservation}") + for res_name in del_reservation: + delete_reservation(lkp, res_name) + + log.debug(f"create-reservation-map: {create_reservation_map}") + for res_name, (node, start_time) in create_reservation_map.items(): + create_reservation(lkp, res_name, node, start_time) + + +def delete_maintenance_job(job_name: str) -> None: + util.run(f"scancel --name={job_name}") + + +def create_maintenance_job(job_name: str, node: str) -> None: + util.run(f"sbatch --job-name={job_name} --nodelist={node} {_MAINTENANCE_SBATCH_SCRIPT_PATH}") + + +def get_slurm_maintenance_job(lkp: util.Lookup) -> Dict[str, str]: + jobs = {} + + for job in lkp.get_jobs(): + if job.name is None or job.required_nodes is None or job.job_state is None: + continue + + if job.name != f"{job.required_nodes}_maintenance": + continue + + if job.job_state != "PENDING": + continue + + jobs[job.name] = job.required_nodes + + return jobs + + +def sync_opportunistic_maintenance(lkp: util.Lookup) -> None: + upc_maint_map = get_upcoming_maintenance(lkp) # map job_name -> (node_name, time) + log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") + + curr_jobs = get_slurm_maintenance_job(lkp) # map job_name -> node. + log.debug(f"curr-maintenance-job-map: {curr_jobs}") + + del_jobs = set(curr_jobs.keys() - upc_maint_map.keys()) + create_jobs = {} + + for job_name, (node, _) in upc_maint_map.items(): + try: + enabled = lkp.node_nodeset(node).enable_opportunistic_maintenance + except Exception: + enabled = False + + if not enabled: + if job_name in curr_jobs: + del_jobs.add(job_name) + continue + + if job_name not in curr_jobs: + create_jobs[job_name] = node + + log.debug(f"del-maintenance-job: {del_jobs}") + for job_name in del_jobs: + delete_maintenance_job(job_name) + + log.debug(f"create-maintenance-job: {create_jobs}") + for job_name, node in create_jobs.items(): + create_maintenance_job(job_name, node) + + + +def sync_flex_migs(lkp: util.Lookup) -> None: + pass + + +def process_messages(lkp: util.Lookup) -> None: + try: + watch_delete_vm_op.watch_vm_delete_ops(lkp) + except: + log.exception("failed during watching delete VM operations") + + +def main(): + lkp = lookup() + if util.should_mount_slurm_bucket() and not lkp.is_controller: + return + try: + reconfigure_slurm() + except Exception: + log.exception("failed to reconfigure slurm") + if lkp.is_controller: + try: + process_messages(lkp) + except: + log.exception("failed to process messages") + + try: + sync_instances() + except Exception: + log.exception("failed to sync instances") + + try: + sync_flex_migs(lkp) + except Exception: + log.exception("failed to sync DWS Flex MIGs") + + try: + sync_placement_groups() + except Exception: + log.exception("failed to sync placement groups") + + try: + update_topology(lkp) + except Exception: + log.exception("failed to update topology") + + try: + sync_maintenance_reservation(lkp) + except Exception: + log.exception("failed to sync slurm reservation for scheduled maintenance") + + try: + sync_opportunistic_maintenance(lkp) + except Exception: + log.exception("failed to sync opportunistic reservation for scheduled maintenance") + + + try: + # TODO: it performs 1 to 4 GCS list requests, + # use cached version, combine with `_list_config_blobs` + install_custom_scripts(check_hash=True) + except Exception: + log.exception("failed to sync custom scripts") + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + _ = util.init_log_and_parse(parser) + + pid_file = (Path("/tmp") / Path(__file__).name).with_suffix(".pid") + with pid_file.open("w") as fp: + try: + fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) + main() + except BlockingIOError: + sys.exit(0) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py new file mode 100644 index 0000000000..ae36c54222 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py @@ -0,0 +1,171 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script sorts nodes based on their `physicalHost`. + +See https://cloud.google.com/compute/docs/instances/use-compact-placement-policies + +You can reduce latency in tightly coupled HPC workloads (including distributed ML training) +by deploying them to machines that are located close together. +For example, if you deploy your workload on a single physical rack, you can expect lower latency +than if your workload is spread across multiple racks. +Sending data across multiple rack requires sending data through additional network switches. + +Example usage: +``` my_sbatch.sh +#SBATCH --ntasks-per-node=8 +#SBATCH --nodes=64 + +export SLURM_HOSTFILE=$(sort_nodes.py) + +srun -l hostname | sort +``` +""" +import os +import subprocess +import uuid +from typing import List, Optional, Dict +from collections import OrderedDict + +def order(paths: List[List[str]]) -> List[str]: + """ + Orders the leaves of the tree in a way that minimizes the sum of distance in between + each pair of neighboring nodes in the resulting order. + The resulting order will always start from the first node in the input list. + The ordering is "stable" with respect to the input order of the leaves i.e. + given a choice between two nodes (identical in other ways) it will select "nodelist-smallest" one. + + Returns a list of nodenames, ordered as described above. + """ + if not paths: return [] + class Vert: + "Represents a vertex in a *network* tree." + def __init__(self, name: str, parent: Optional["Vert"]): + self.name = name + self.parent = parent + # Use `OrderedDict` to preserve insertion order + # TODO: once we move to Python 3.7+ use regular `dict` since it has the same guarantee + self.children: OrderedDict = OrderedDict() + + # build a tree, children are ordered by insertion order + root = Vert("", None) + for path in paths: + n = root + for v in path: + if v not in n.children: + n.children[v] = Vert(v, n) + n = n.children[v] + + # walk the tree in insertion order, gather leaves + result = [] + def gather_nodes(v: Vert) -> None: + if not v.children: # this is a Slurm node + result.append(v.name) + for u in v.children.values(): + gather_nodes(u) + gather_nodes(root) + return result + + +class Instance: + def __init__(self, name: str, zone: str, physical_host: Optional[str]): + self.name = name + self.zone = zone + self.physical_host = physical_host + + +def make_path(node_name: str, inst: Optional[Instance]) -> List[str]: + if not inst: # node with unknown instance (e.g. hybrid cluster) + return ["unknown", node_name] + zone = f"zone_{inst.zone}" + if not inst.physical_host: # node without physical host info (e.g. no placement policy) + return [zone, "unknown", node_name] + + assert inst.physical_host.startswith("/"), f"Unexpected physicalHost: {inst.physical_host}" + parts = inst.physical_host[1:].split("/") + if len(parts) >= 4: + return [*parts, node_name] + return [zone, *parts, node_name] + + +def to_hostnames(nodelist: str) -> List[str]: + cmd = ["scontrol", "show", "hostnames", nodelist] + out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout + return [n.decode("utf-8") for n in out.splitlines()] + + +def get_instances(node_names: List[str]) -> Dict[str, Optional[Instance]]: + fmt = ( + "--format=csv[no-heading,separator=','](zone,resourceStatus.physicalHost,name)" + ) + cmd = ["gcloud", "compute", "instances", "list", fmt] + + scp = os.path.commonprefix(node_names) + if scp: + cmd.append(f"--filter=name~'{scp}.*'") + out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout + d = {} + for line in out.splitlines(): + zone, physical_host, name = line.decode("utf-8").split(",") + d[name] = Instance(name, zone, physical_host) + return {n: d.get(n) for n in node_names} + + +def main(args) -> None: + nodelist = args.nodelist or os.getenv("SLURM_NODELIST") + if not nodelist: + raise ValueError("nodelist is not provided and SLURM_NODELIST is not set") + + if args.ntasks_per_node is None: + args.ntasks_per_node = int(os.getenv("SLURM_NTASKS_PER_NODE", "") or 1) + assert args.ntasks_per_node > 0 + + output = args.output or f"hosts.{uuid.uuid4()}" + + node_names = to_hostnames(nodelist) + instannces = get_instances(node_names) + paths = [make_path(n, instannces[n]) for n in node_names] + ordered = order(paths) + + with open(output, "w") as f: + for node in ordered: + for _ in range(args.ntasks_per_node): + f.write(node) + f.write("\n") + print(output) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--nodelist", + type=str, + help="Slurm 'hostlist expression' of nodes to sort, if not set the value of SLURM_NODELIST environment variable will be used", + ) + parser.add_argument( + "--ntasks-per-node", + type=int, + help="""Number of times to repeat each node in resulting sorted list. +If not set, the value of SLURM_NTASKS_PER_NODE environment variable will be used, +if neither is set, defaults to 1""", + ) + parser.add_argument( + "--output", type=str, help="Output file to write, defaults to 'hosts.'" + ) + args = parser.parse_args() + main(args) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py new file mode 100644 index 0000000000..ecef70f1cc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py @@ -0,0 +1,126 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Any +import argparse +import logging + +import util +from util import ( + log_api_request, + batch_execute, + to_hostlist, + separate, +) +from util import lookup +import tpu +import mig_flex +import watch_delete_vm_op + +log = logging.getLogger() + +TOT_REQ_CNT = 1000 + + +def truncate_iter(iterable, max_count): + end = "..." + _iter = iter(iterable) + for i, el in enumerate(_iter, start=1): + if i >= max_count: + yield end + break + yield el + + +def delete_instance_request(name: str) -> Any: + inst = lookup().instance(name) + assert inst + + request = lookup().compute.instances().delete( + project=lookup().project, + zone=inst.zone, + instance=name, + ) + log_api_request(request) + return request + + +def delete_instances(instances): + """delete instances individually""" + invalid, valid = separate(lambda inst: bool(lookup().instance(inst)), instances) + if len(invalid) > 0: + log.debug("instances do not exist: {}".format(",".join(invalid))) + if len(valid) == 0: + log.debug("No instances to delete") + return + + requests = {inst: delete_instance_request(inst) for inst in valid} + + log.info(f"to delete {len(valid)} instances ({to_hostlist(valid)})") + ops, failed = batch_execute(requests) + for node, (_, err) in failed.items(): + log.error(f"instance {node} failed to delete: {err}") + + log.info(f"deleting {len(ops)} instances {to_hostlist(ops.keys())}") + + topic = watch_delete_vm_op.watch_delete_vm_op_topic() + for node, op in ops.items(): + topic.publish(op, node) + + + + +def suspend_nodes(nodes: List[str]) -> None: + lkp = lookup() + other_nodes, tpu_nodes = util.separate(lkp.node_is_tpu, nodes) + bulk_nodes, flex_nodes = util.separate(lkp.is_flex_node, other_nodes) + + mig_flex.suspend_flex_nodes(flex_nodes, lkp) + delete_instances(bulk_nodes) + tpu.delete_tpu_instances(tpu_nodes) + + +def main(nodelist): + """main called when run as script""" + log.debug(f"SuspendProgram {nodelist}") + + # Filter out nodes not in config.yaml + other_nodes, pm_nodes = separate( + lookup().is_power_managed_node, util.to_hostnames(nodelist) + ) + if other_nodes: + log.debug( + f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" + ) + if pm_nodes: + log.debug(f"Suspending nodes '{to_hostlist(pm_nodes)}' from '{nodelist}'") + else: + log.debug("No cloud nodes to suspend") + return + + log.info(f"suspend {nodelist}") + suspend_nodes(pm_nodes) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("nodelist", help="list of nodes to suspend") + args = util.init_log_and_parse(parser) + + main(args.nodelist) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh new file mode 100644 index 0000000000..9079e4e4b0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PYTHON_SCRIPT="${SCRIPT_DIR}/suspend.py" + +# Capture all arguments passed by Slurm (the nodelist). +ALL_ARGS=("$@") + +"${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & +disown + +exit 0 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py new file mode 100644 index 0000000000..0ce7fb5ec4 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py @@ -0,0 +1,116 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Any +import sys +from dataclasses import dataclass, field +from datetime import datetime + +SCRIPTS_DIR = "community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts" +if SCRIPTS_DIR not in sys.path: + sys.path.append(SCRIPTS_DIR) # TODO: make this more robust + +import util + + +SOME_TS = datetime.fromisoformat("2018-09-03T20:56:35.450686+00:00") +# TODO: use "real" classes once they are defined (instead of NSDict) + +@dataclass +class Placeholder: + pass + +@dataclass +class TstNodeset: + nodeset_name: str = "cantor" + node_count_static: int = 0 + node_count_dynamic_max: int = 0 + node_conf: dict[str, Any] = field(default_factory=dict) + instance_template: Optional[str] = None + reservation_name: Optional[str] = "" + zone_policy_allow: Optional[list[str]] = field(default_factory=list) + enable_placement: bool = True + placement_max_distance: Optional[int] = None + accelerator_topology: Optional[str] = "" + future_reservation: Optional[str] = "" + +@dataclass +class TstPartition: + partition_name: str = "euler" + partition_nodeset: list[str] = field(default_factory=list) + partition_nodeset_tpu: list[str] = field(default_factory=list) + enable_job_exclusive: bool = False + +@dataclass +class TstCfg: + slurm_cluster_name: str = "m22" + cloud_parameters: dict[str, Any] = field(default_factory=dict) + + partitions: dict[str, TstPartition] = field(default_factory=dict) + nodeset: dict[str, TstNodeset] = field(default_factory=dict) + nodeset_tpu: dict[str, TstNodeset] = field(default_factory=dict) + nodeset_dyn: dict[str, TstNodeset] = field(default_factory=dict) + + install_dir: Optional[str] = None + output_dir: Optional[str] = None + + prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + task_prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + task_epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + + +@dataclass +class TstTPU: # to prevent client initialization durint "TPU.__init__" + vmcount: int + +@dataclass +class TstMachineConf: + cpus: int + memory: int + sockets: int + sockets_per_board: int + cores_per_socket: int + boards: int + threads_per_core: int + + +@dataclass +class TstTemplateInfo: + gpu: Optional[util.AcceleratorInfo] + +def tstInstance(name: str, physical_host: Optional[str] = None): + return util.Instance( + name=name, + zone="anorien", + status="RUNNING", + creation_timestamp=SOME_TS, + resource_status=util.InstanceResourceStatus( + physical_host=physical_host, + upcoming_maintenance=None, + ), + scheduling=util.NSDict(), + role="compute", + metadata={}, + ) + +def make_to_hostnames_mock(tbl: Optional[dict[str, list[str]]]): + tbl = tbl or {} + + def se(k: str) -> list[str]: + if k not in tbl: + raise AssertionError(f"to_hostnames mock: unexpected nodelist: '{k}'") + return tbl[k] + + return se diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py new file mode 100644 index 0000000000..6bd6762748 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py @@ -0,0 +1,226 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from mock import Mock +from common import TstNodeset, TstCfg, TstMachineConf, TstTemplateInfo, Placeholder + +import addict # type: ignore +import conf +import util + + +def test_nodeset_tpu_lines(): + nodeset = TstNodeset( + "turbo", + node_count_static=2, + node_count_dynamic_max=3, + node_conf={"red": "velvet"}, + ) + assert conf.nodeset_tpu_lines(nodeset, util.Lookup(TstCfg())) == "\n".join( + [ + "NodeName=m22-turbo-[0-4] State=CLOUD red=velvet", + "NodeSet=turbo Nodes=m22-turbo-[0-4]", + ] + ) + + +def test_nodeset_lines(): + nodeset = TstNodeset( + "turbo", + node_count_static=2, + node_count_dynamic_max=3, + node_conf={"red": "velvet", "CPUs": 55}, + ) + lkp = util.Lookup(TstCfg()) + lkp.template_info = Mock(return_value=TstTemplateInfo( + gpu=util.AcceleratorInfo(type="Popov", count=33) + )) + mc = TstMachineConf( + cpus=5, + memory=6, + sockets=7, + sockets_per_board=8, + boards=9, + threads_per_core=10, + cores_per_socket=11, + ) + lkp.template_machine_conf = Mock(return_value=mc) # type: ignore[method-assign] + assert conf.nodeset_lines(nodeset, lkp) == "\n".join( + [ + "NodeName=m22-turbo-[0-4] State=CLOUD RealMemory=6 Boards=9 SocketsPerBoard=8 CoresPerSocket=11 ThreadsPerCore=10 CPUs=55 Gres=gpu:33 red=velvet", + "NodeSet=turbo Nodes=m22-turbo-[0-4]", + ] + ) + + +@pytest.mark.parametrize( + "value,want", + [ + ({"a": 1}, "a=1"), + ({"a": "two"}, "a=two"), + ({"a": [3, 4]}, "a=3,4"), + ({"a": ["five", "six"]}, "a=five,six"), + ({"a": None}, ""), + ({"a": ["seven", None, 8]}, "a=seven,8"), + ({"a": 1, "b": "two"}, "a=1 b=two"), + ({"a": 1, "b": None, "c": "three"}, "a=1 c=three"), + ({"a": 0, "b": None, "c": 0.0, "e": ""}, "a=0 c=0.0"), + ({"a": [0, 0.0, None, "X", "", "Y"]}, "a=0,0.0,X,,Y"), + ]) +def test_dict_to_conf(value: dict, want: str): + assert conf.dict_to_conf(value) == want + + + +@pytest.mark.parametrize( + "cfg,want", + [ + (TstCfg( + install_dir="ukulele", + ), + """LaunchParameters=enable_nss_slurm,use_interactive_step +SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend +SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +SlurmdTimeout=300 +UnkillableStepTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + (TstCfg( + install_dir="ukulele", + cloud_parameters={ + "no_comma_params": True, + "private_data": None, + "scheduler_parameters": None, + "resume_rate": None, + "resume_timeout": None, + "suspend_rate": None, + "suspend_timeout": None, + "unkillable_step_timeout": None, + "slurmd_timeout": None, + "topology_plugin": None, + "topology_param": None, + "tree_width": None, + }, + ), + """SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +SlurmdTimeout=300 +UnkillableStepTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + (TstCfg( + install_dir="ukulele", + cloud_parameters={ + "no_comma_params": True, + "private_data": [ + "events", + "jobs", + ], + "scheduler_parameters": [ + "bf_busy_nodes", + "bf_continue", + "ignore_prefer_validation", + "nohold_on_prolog_fail", + ], + "resume_rate": 1, + "resume_timeout": 2, + "suspend_rate": 3, + "suspend_timeout": 4, + "slurmd_timeout": 5, + "unkillable_step_timeout": 6, + "tree_width": 7, + "topology_plugin": "guess", + "topology_param": "yellow", + }, + ), + """PrivateData=events,jobs +SchedulerParameters=bf_busy_nodes,bf_continue,ignore_prefer_validation,nohold_on_prolog_fail +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=1 +ResumeTimeout=2 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=3 +SuspendTimeout=4 +SlurmdTimeout=5 +UnkillableStepTimeout=6 +TreeWidth=7 +TopologyPlugin=guess +TopologyParam=yellow"""), + (TstCfg( + install_dir="ukulele", + task_prolog_scripts=[Placeholder()], + task_epilog_scripts=[Placeholder()], + ), + """LaunchParameters=enable_nss_slurm,use_interactive_step +SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend +TaskProlog=/slurm/custom_scripts/task_prolog.d/task-prolog +TaskEpilog=/slurm/custom_scripts/task_epilog.d/task-epilog +SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +SlurmdTimeout=300 +UnkillableStepTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + ]) +def test_conflines(cfg, want): + assert conf.conflines(util.Lookup(cfg)) == want + + cfg.cloud_parameters = addict.Dict(cfg.cloud_parameters) + assert conf.conflines(util.Lookup(cfg)) == want + + +@pytest.mark.parametrize( + "cfg,gputype,gpucount,want", + [ + (TstCfg(), + "", + 0, + "\n"), + (TstCfg( + nodeset={"turbo": TstNodeset("turbo")} + ), + "Popov", + 8, + "Name=gpu Type=Popov File=/dev/nvidia[0-7]\n\n"), + ]) +def test_gen_cloud_gres_conf_lines(cfg, gputype, gpucount, want): + lkp = util.Lookup(cfg) + lkp.template_info = Mock(return_value=TstTemplateInfo( + gpu=util.AcceleratorInfo(type=gputype, count=gpucount) + )) + assert conf.gen_cloud_gres_conf_lines(lkp) == want diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py new file mode 100644 index 0000000000..77f1229605 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py @@ -0,0 +1,175 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import os +import pytest +import unittest.mock +import unittest +import tempfile + +from common import TstCfg, TstNodeset, TstPartition, TstTPU # needed to import util +import util +import resume +from resume import ResumeData, ResumeJobData, BulkChunk, PlacementAndNodes + +def test_get_resume_file_data_no_env(): + with unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": ""}): + assert resume.get_resume_file_data() is None + + +def test_get_resume_file_data(): + with tempfile.NamedTemporaryFile() as f: + f.write(b"""{ + "jobs": [ + { + "extra": null, + "job_id": 1, + "features": null, + "nodes_alloc": "green-[0-2]", + "nodes_resume": "green-[0-1]", + "oversubscribe": "OK", + "partition": "red", + "reservation": null + } + ], + "all_nodes_resume": "green-[0-1]" +}""") + f.flush() + with ( + unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": f.name}), + unittest.mock.patch("util.to_hostnames") as mock_to_hostnames, + ): + mock_to_hostnames.return_value = ["green-0", "green-1", "green-2"] + assert resume.get_resume_file_data() == ResumeData(jobs=[ + ResumeJobData( + job_id = 1, + partition="red", + nodes_alloc=["green-0", "green-1", "green-2"], + ) + ]) + mock_to_hostnames.assert_called_once_with("green-[0-2]") + + +@unittest.mock.patch("tpu.TPU.make") +@unittest.mock.patch("resume.create_placements") +def test_group_nodes_bulk(mock_create_placements, mock_tpu): + cfg = TstCfg( + nodeset={ + "n": TstNodeset(nodeset_name="n"), + }, + nodeset_tpu={ + "t": TstNodeset(nodeset_name="t"), + }, + partitions={ + "p1": TstPartition( + partition_name="p1", + enable_job_exclusive=True, + ), + "p2": TstPartition( + partition_name="p2", + partition_nodeset_tpu=["t"], + enable_job_exclusive=True, + ) + } + ) + lkp = util.Lookup(cfg) + + def mock_create_placements_se(nodes, excl_job_id, lkp): + args = (set(nodes), excl_job_id) + if ({'c-n-1', 'c-n-2', 'c-t-8', 'c-t-9'}, None) == args: + return [ + PlacementAndNodes("g0", ["c-n-1", "c-n-2"]), + PlacementAndNodes(None, ['c-t-8', 'c-t-9']), + ] + if ({"c-n-0", "c-n-8"}, 1) == args: + return [ + PlacementAndNodes("g10", ["c-n-0"]), + PlacementAndNodes("g11", ["c-n-8"]), + ] + if ({'c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5'}, 2) == args: + return [ + PlacementAndNodes(None, ['c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5']) + ] + raise AssertionError(f"unexpected invocation: '{args}'") + mock_create_placements.side_effect = mock_create_placements_se + + def mock_tpu_se(ns: str, lkp) -> TstTPU: + if ns == "t": + return TstTPU(vmcount=2) + raise AssertionError(f"unexpected invocation: '{ns}'") + mock_tpu.side_effect = mock_tpu_se + + got = resume.group_nodes_bulk( + ["c-n-0", "c-n-1", "c-n-2", "c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-8", "c-t-9"], + ResumeData(jobs=[ + ResumeJobData(job_id=1, partition="p1", nodes_alloc=["c-n-0", "c-n-8"]), + ResumeJobData(job_id=2, partition="p2", nodes_alloc=["c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-4", "c-t-5"]), + ]), lkp) + mock_create_placements.assert_called() + assert got == { + "c-n:jobNone:g0:0": BulkChunk( + nodes=["c-n-1", "c-n-2"], prefix="c-n", chunk_idx=0, excl_job_id=None, placement_group="g0"), + "c-n:job1:g10:0": BulkChunk( + nodes=["c-n-0"], prefix="c-n", chunk_idx=0, excl_job_id=1, placement_group="g10"), + "c-t:0": BulkChunk( + nodes=["c-t-8", "c-t-9"], prefix="c-t", chunk_idx=0, excl_job_id=None, placement_group=None), + "c-t:job2:0": BulkChunk( + nodes=["c-t-0", "c-t-1"], prefix="c-t", chunk_idx=0, excl_job_id=2, placement_group=None), + "c-t:job2:1": BulkChunk( + nodes=["c-t-2", "c-t-3"], prefix="c-t", chunk_idx=1, excl_job_id=2, placement_group=None), + } + + +@pytest.mark.parametrize( + "nodes,excl_job_id,expected", + [ + ( # TPU - no placements + ["c-t-0", "c-t-2"], 4, [PlacementAndNodes(None, ["c-t-0", "c-t-2"])] + ), + ( # disabled placements - no placemens + ["c-x-0", "c-x-2"], 4, [PlacementAndNodes(None, ["c-x-0", "c-x-2"])] + ), + ( # excl_job + ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], 4, [ + PlacementAndNodes("c-slurmgcp-managed-n-4-0", ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"]) + ] + ), + ( # no excl_job + ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], None, [ + PlacementAndNodes("c-slurmgcp-managed-n-0-0", ["c-n-0", "c-n-2"]), + PlacementAndNodes('c-slurmgcp-managed-n-0-1', ['c-n-2011']), + PlacementAndNodes(None, ["c-n-uno"]), + ] + ), + ], +) +def test_allocate_nodes_to_placements(nodes: list[str], excl_job_id: Optional[int], expected: list[PlacementAndNodes]): + cfg = TstCfg( + slurm_cluster_name="c", + nodeset={ + "n": TstNodeset(nodeset_name="n", enable_placement=True), + "x": TstNodeset(nodeset_name="x", enable_placement=False) + }, + nodeset_tpu={ + "t": TstNodeset(nodeset_name="t") + }) + lkp = util.Lookup(cfg) + + with unittest.mock.patch("resume.valid_placement_node") as mock_valid_placement_node: + mock_valid_placement_node.return_value = True + lkp.template_info = unittest.mock.Mock(return_value=unittest.mock.Mock(machine_type=unittest.mock.Mock(family="n1"))) + + assert resume._allocate_nodes_to_placements(nodes, excl_job_id, lkp) == expected diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py new file mode 100644 index 0000000000..df9f3a0137 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py @@ -0,0 +1,215 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import json +import mock +from pytest_unordered import unordered +from common import TstCfg, TstNodeset, TstTPU, tstInstance +import sort_nodes + +import util +import conf +import tempfile + +PRELUDE = """ +# Warning: +# This file is managed by a script. Manual modifications will be overwritten. + +""" + +def test_gen_topology_conf_empty(): + out_dir = tempfile.mkdtemp() + cfg = TstCfg(output_dir=out_dir) + conf.gen_topology_conf(util.Lookup(cfg)) + assert open(out_dir + "/cloud_topology.conf").read() == PRELUDE + "\n" + + +@mock.patch("tpu.TPU.make") +def test_gen_topology_conf(tpu_mock): + output_dir = tempfile.mkdtemp() + cfg = TstCfg( + nodeset_tpu={ + "a": TstNodeset("bold", node_count_static=4, node_count_dynamic_max=5), + "b": TstNodeset("slim", node_count_dynamic_max=3), + }, + nodeset={ + "c": TstNodeset("green", node_count_static=2, node_count_dynamic_max=3), + "d": TstNodeset("blue", node_count_static=7), + "e": TstNodeset("pink", node_count_dynamic_max=4), + }, + output_dir=output_dir, + ) + + def tpu_se(ns: str, lkp) -> TstTPU: + if ns == "bold": + return TstTPU(vmcount=3) + if ns == "slim": + return TstTPU(vmcount=1) + raise AssertionError(f"unexpected TPU name: '{ns}'") + + tpu_mock.side_effect = tpu_se + + lkp = util.Lookup(cfg) + lkp.instances = lambda: { n.name: n for n in [ # type: ignore[assignment] + # nodeset blue + tstInstance("m22-blue-0"), # no physicalHost + tstInstance("m22-blue-0", physical_host="/a/a/a"), + tstInstance("m22-blue-1", physical_host="/a/a/b"), + tstInstance("m22-blue-2", physical_host="/a/b/a"), + tstInstance("m22-blue-3", physical_host="/b/a/a"), + # nodeset green + tstInstance("m22-green-3", physical_host="/a/a/c"), + ]} + + uncompressed = conf.gen_topology(lkp) + want_uncompressed = [ + #NOTE: the switch names are not unique, it's not valid content for topology.conf + # The uniquefication and compression of names are done in the compress() method + "SwitchName=slurm-root Switches=a,b,ns_blue,ns_green,ns_pink", + # "physical" topology + 'SwitchName=a Switches=a,b', + 'SwitchName=a Nodes=m22-blue-[0-1],m22-green-3', + 'SwitchName=b Nodes=m22-blue-2', + 'SwitchName=b Switches=a', + 'SwitchName=a Nodes=m22-blue-3', + # topology "by nodeset" + "SwitchName=ns_blue Nodes=m22-blue-[4-6]", + "SwitchName=ns_green Nodes=m22-green-[0-2,4]", + "SwitchName=ns_pink Nodes=m22-pink-[0-3]", + # TPU topology + "SwitchName=tpu-root Switches=ns_bold,ns_slim", + "SwitchName=ns_bold Switches=bold-[0-3]", + "SwitchName=bold-0 Nodes=m22-bold-[0-2]", + "SwitchName=bold-1 Nodes=m22-bold-3", + "SwitchName=bold-2 Nodes=m22-bold-[4-6]", + "SwitchName=bold-3 Nodes=m22-bold-[7-8]", + "SwitchName=ns_slim Nodes=m22-slim-[0-2]"] + assert list(uncompressed.render_conf_lines()) == want_uncompressed + + compressed = uncompressed.compress() + want_compressed = [ + "SwitchName=s0 Switches=s0_[0-4]", # root + # "physical" topology + 'SwitchName=s0_0 Switches=s0_0_[0-1]', # /a + 'SwitchName=s0_0_0 Nodes=m22-blue-[0-1],m22-green-3', # /a/a + 'SwitchName=s0_0_1 Nodes=m22-blue-2', # /a/b + 'SwitchName=s0_1 Switches=s0_1_0', # /b + 'SwitchName=s0_1_0 Nodes=m22-blue-3', # /b/a + # topology "by nodeset" + "SwitchName=s0_2 Nodes=m22-blue-[4-6]", + "SwitchName=s0_3 Nodes=m22-green-[0-2,4]", + "SwitchName=s0_4 Nodes=m22-pink-[0-3]", + # TPU topology + "SwitchName=s1 Switches=s1_[0-1]", + "SwitchName=s1_0 Switches=s1_0_[0-3]", + "SwitchName=s1_0_0 Nodes=m22-bold-[0-2]", + "SwitchName=s1_0_1 Nodes=m22-bold-3", + "SwitchName=s1_0_2 Nodes=m22-bold-[4-6]", + "SwitchName=s1_0_3 Nodes=m22-bold-[7-8]", + "SwitchName=s1_1 Nodes=m22-slim-[0-2]"] + assert list(compressed.render_conf_lines()) == want_compressed + + upd, summary = conf.gen_topology_conf(lkp) + assert upd == True + want_written = PRELUDE + "\n".join(want_compressed) + "\n\n" + assert open(output_dir + "/cloud_topology.conf").read() == want_written + + summary.dump(lkp) + summary_got = json.loads(open(output_dir + "/cloud_topology.summary.json").read()) + + assert summary_got == { + "down_nodes": unordered( + [f"m22-blue-{i}" for i in (4,5,6)] + + [f"m22-green-{i}" for i in (0,1,2,4)] + + [f"m22-pink-{i}" for i in range(4)]), + "tpu_nodes": unordered( + [f"m22-bold-{i}" for i in range(9)] + + [f"m22-slim-{i}" for i in range(3)]), + 'physical_host': { + 'm22-blue-0': '/a/a/a', + 'm22-blue-1': '/a/a/b', + 'm22-blue-2': '/a/b/a', + 'm22-blue-3': '/b/a/a', + 'm22-green-3': '/a/a/c'}, + } + + + +def test_gen_topology_conf_update(): + cfg = TstCfg( + nodeset={ + "c": TstNodeset("green", node_count_static=2), + }, + output_dir=tempfile.mkdtemp(), + ) + lkp = util.Lookup(cfg) + lkp.instances = lambda: { # type: ignore[assignment] + # no instances + } + + # initial generation - reconfigure + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # add node: node_count_static 2 -> 3 - reconfigure + lkp.cfg.nodeset["c"].node_count_static = 3 + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # remove node: node_count_static 3 -> 2 - no reconfigure + lkp.cfg.nodeset["c"].node_count_static = 2 + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + # set empty physicalHost - no reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + # set physicalHost - reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/c")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # change physicalHost - reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/z")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # shut down node - no reconfigure + lkp.instances = lambda: {} # type: ignore[assignment] + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + +@pytest.mark.parametrize( + "paths,expected", + [ + (["z/n-0", "z/n-1", "z/n-2", "z/n-3", "z/n-4", "z/n-10"], ['n-0', 'n-1', 'n-2', 'n-3', 'n-4', 'n-10']), + (["y/n-0", "z/n-1", "x/n-2", "x/n-3", "y/n-4", "g/n-10"], ['n-0', 'n-4', 'n-1', 'n-2', 'n-3', 'n-10']), + ]) +def test_sort_nodes_order(paths: list[str], expected: list[str]) -> None: + paths_expanded = [l.split("/") for l in paths] + assert sort_nodes.order(paths_expanded) == expected diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py new file mode 100644 index 0000000000..69617d0301 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py @@ -0,0 +1,668 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Type + +import pytest +from mock import Mock +from datetime import datetime, timezone, timedelta +import unittest + +from common import TstNodeset, TstCfg # needed to import util +import util +from util import NodeState, MachineType, AcceleratorInfo, UpcomingMaintenance, InstanceResourceStatus, FutureReservation, ReservationDetails +from google.api_core.client_options import ClientOptions # noqa: E402 +from addict import Dict as NSDict # type: ignore + +# Note: need to install pytest-mock + +@pytest.mark.parametrize( + "name,expected", + [ + ( + "az-buka-23", + { + "cluster": "az", + "nodeset": "buka", + "node": "23", + "prefix": "az-buka", + "range": None, + "suffix": "23", + }, + ), + ( + "az-buka-xyzf", + { + "cluster": "az", + "nodeset": "buka", + "node": "xyzf", + "prefix": "az-buka", + "range": None, + "suffix": "xyzf", + }, + ), + ( + "az-buka-[2-3]", + { + "cluster": "az", + "nodeset": "buka", + "node": "[2-3]", + "prefix": "az-buka", + "range": "[2-3]", + "suffix": None, + }, + ), + ], +) +def test_node_desc(name, expected): + assert util.lookup()._node_desc(name) == expected + + +@pytest.mark.parametrize( + "name,expected", + [ + ("az-buka-23", 23), + ("az-buka-0", 0), + ("az-buka", Exception), + ("az-buka-xyzf", ValueError), + ("az-buka-[2-3]", ValueError), + ], +) +def test_node_index(name, expected): + if type(expected) is type and issubclass(expected, Exception): + with pytest.raises(expected): + util.lookup().node_index(name) + else: + assert util.lookup().node_index(name) == expected + + +@pytest.mark.parametrize( + "name", + [ + "az-buka", + ], +) +def test_node_desc_fail(name): + with pytest.raises(Exception): + util.lookup()._node_desc(name) + + +@pytest.mark.parametrize( + "names,expected", + [ + ("pedro,pedro-1,pedro-2,pedro-01,pedro-02", "pedro,pedro-[1-2,01-02]"), + ("pedro,,pedro-1,,pedro-2", "pedro,pedro-[1-2]"), + ("pedro-8,pedro-9,pedro-10,pedro-11", "pedro-[8-9,10-11]"), + ("pedro-08,pedro-09,pedro-10,pedro-11", "pedro-[08-11]"), + ("pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-09]"), + ("pedro-10,pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-10]"), + ("pedro-8,pedro-9,juan-10,juan-11", "juan-[10-11],pedro-[8-9]"), + ("az,buki,vedi", "az,buki,vedi"), + ("a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12", "a[0-9,10-12]"), + ("a0,a2,a4,a6,a7,a8,a11,a12", "a[0,2,4,6-8,11-12]"), + ("seas7-0,seas7-1", "seas7-[0-1]"), + ], +) +def test_to_hostlist(names, expected): + assert util.to_hostlist(names.split(",")) == expected + + +@pytest.mark.parametrize( + "api,ep_ver,expected", + [ + ( + util.ApiEndpoint.BQ, + "v1", + ClientOptions(api_endpoint="https://bq.googleapis.com/v1/"), + ), + ( + util.ApiEndpoint.COMPUTE, + "staging_v1", + ClientOptions(api_endpoint="https://compute.googleapis.com/staging_v1/"), + ), + ( + util.ApiEndpoint.SECRET, + "v1", + ClientOptions(api_endpoint="https://secret_manager.googleapis.com/v1/"), + ), + ( + util.ApiEndpoint.STORAGE, + "beta", + ClientOptions(api_endpoint="https://storage.googleapis.com/beta/"), + ), + ( + util.ApiEndpoint.TPU, + "alpha", + ClientOptions(api_endpoint="https://tpu.googleapis.com/alpha/"), + ), + ], +) +def test_create_client_options( + api: util.ApiEndpoint, ep_ver: str, expected: ClientOptions, mocker +): + ud_mock = mocker.patch("util.universe_domain") + ep_mock = mocker.patch("util.endpoint_version") + ud_mock.return_value = "googleapis.com" + ep_mock.return_value = ep_ver + assert util.create_client_options(api).__repr__() == expected.__repr__() + + + +@pytest.mark.parametrize( + "nodeset,err", + [ + (TstNodeset(reservation_name="projects/x/reservations/y"), AssertionError), # no zones + (TstNodeset( + reservation_name="projects/x/reservations/y", + zone_policy_allow=["eine", "zwei"]), AssertionError), # multiples zones + (TstNodeset( + reservation_name="robin", + zone_policy_allow=["eine"]), ValueError), # invalid name + (TstNodeset( + reservation_name="projects/reservations/y", + zone_policy_allow=["eine"]), ValueError), # invalid name + (TstNodeset( + reservation_name="projects/x/zones/z/reservations/y", + zone_policy_allow=["eine"]), ValueError), # invalid name + ] +) +def test_nodeset_reservation_err(nodeset, err): + lkp = util.Lookup(TstCfg()) + lkp._get_reservation = Mock() + with pytest.raises(err): + lkp.nodeset_reservation(nodeset) + lkp._get_reservation.assert_not_called() # type: ignore + +@pytest.mark.parametrize( + "nodeset,policies,expected", + [ + (TstNodeset(), [], None), # no reservation + (TstNodeset( + reservation_name="projects/bobin/reservations/robin", + zone_policy_allow=["eine"]), + [], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=[], + deployment_type=None, + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/bobin/reservations/robin")), + (TstNodeset( + reservation_name="projects/bobin/reservations/robin", + zone_policy_allow=["eine"]), + ["seven/wanders", "five/red/apples", "yum"], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=["wanders", "apples", "yum"], + deployment_type=None, + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/bobin/reservations/robin")), + (TstNodeset( + reservation_name="projects/bobin/reservations/robin/snek/cheese-brie-6", + zone_policy_allow=["eine"]), + [], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=[], + deployment_type=None, + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/bobin/reservations/robin/snek/cheese-brie-6")), + + ]) + +def test_nodeset_reservation_ok(nodeset, policies, expected): + lkp = util.Lookup(TstCfg()) + lkp._get_reservation = Mock() + + if not expected: + assert lkp.nodeset_reservation(nodeset) is None + lkp._get_reservation.assert_not_called() # type: ignore + return + + lkp._get_reservation.return_value = { # type: ignore + "resourcePolicies": {i: p for i, p in enumerate(policies)}, + } + assert lkp.nodeset_reservation(nodeset) == expected + lkp._get_reservation.assert_called_once_with(expected.project, expected.zone, expected.name) # type: ignore + +@pytest.mark.parametrize( + "job_info,expected_job", + [ + ( + """JobId=123 + TimeLimit=02:00:00 + JobName=myjob + JobState=PENDING + ReqNodeList=node-[1-10]""", + util.Job( + id=123, + duration=timedelta(days=0, hours=2, minutes=0, seconds=0), + name="myjob", + job_state="PENDING", + required_nodes="node-[1-10]" + ), + ), + ( + """JobId=456 + JobName=anotherjob + JobState=PENDING + ReqNodeList=node-group1""", + util.Job( + id=456, + duration=None, + name="anotherjob", + job_state="PENDING", + required_nodes="node-group1" + ), + ), + ( + """JobId=789 + TimeLimit=00:30:00 + JobState=COMPLETED""", + util.Job( + id=789, + duration=timedelta(minutes=30), + name=None, + job_state="COMPLETED", + required_nodes=None + ), + ), + ( + """JobId=101112 + TimeLimit=1-00:30:00 + JobState=COMPLETED, + ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", + util.Job( + id=101112, + duration=timedelta(days=1, hours=0, minutes=30, seconds=0), + name=None, + job_state="COMPLETED", + required_nodes="node-[1-10],grob-pop-[2,1,44-77]" + ), + ), + ( + """JobId=131415 + TimeLimit=1-00:30:00 + JobName=mynode-1_maintenance + JobState=COMPLETED, + ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", + util.Job( + id=131415, + duration=timedelta(days=1, hours=0, minutes=30, seconds=0), + name="mynode-1_maintenance", + job_state="COMPLETED", + required_nodes="node-[1-10],grob-pop-[2,1,44-77]" + ), + ), + ], +) +def test_parse_job_info(job_info, expected_job): + lkp = util.Lookup(TstCfg()) + assert lkp._parse_job_info(job_info) == expected_job + + + +@pytest.mark.parametrize( + "node,state,want", + [ + ("c-n-2", NodeState("DOWN", frozenset([])), NodeState("DOWN", frozenset([]))), # happy scenario + ("c-d-vodoo", None, None), # dynamic nodeset + ("c-x-44", None, None), # unknown(removed) nodeset + ("c-n-7", None, None), # Out of bounds: c-n-[0-4] - downsized nodeset + ("c-t-7", None, None), # Out of bounds: c-t-[0-4] - downsized nodeset TPU + ("c-n-2", None, RuntimeError), # something is wrong + ("c-t-2", None, RuntimeError), # something is wrong, but TPU + + # Check boundaries match [0-5) + ("c-n-5", None, None), # out of boundaries + ("c-n-4", None, RuntimeError), # within boundaries + ]) +def test_node_state(node: str, state: Optional[NodeState], want: NodeState | None | Type[Exception]): + cfg = TstCfg( + slurm_cluster_name="c", + nodeset={ + "n": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, + nodeset_tpu={ + "t": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, + nodeset_dyn={ + "d": TstNodeset()}, + ) + lkp = util.Lookup(cfg) + lkp.slurm_nodes = lambda: {node: state} if state else {} # type: ignore[assignment] + # ... see https://github.com/python/typeshed/issues/6347 + + if type(want) is type and issubclass(want, Exception): + with pytest.raises(want): + lkp.node_state(node) + else: + assert lkp.node_state(node) == want + + + +@pytest.mark.parametrize( + "jo,want", + [ + ({ + "accelerators": [ { "guestAcceleratorCount": 1, "guestAcceleratorType": "nvidia-tesla-a100" } ], + "creationTimestamp": "1969-12-31T16:00:00.000-08:00", + "description": "Accelerator Optimized: 1 NVIDIA Tesla A100 GPU, 12 vCPUs, 85GB RAM", + "guestCpus": 12, + "id": "1000012", + "imageSpaceGb": 0, + "isSharedCpu": False, + "kind": "compute#machineType", + "maximumPersistentDisks": 128, + "maximumPersistentDisksSizeGb": "263168", + "memoryMb": 87040, + "name": "a2-highgpu-1g", + "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/us-central1-a/machineTypes/a2-highgpu-1g", + "zone": "us-central1-a" + }, MachineType( + name="a2-highgpu-1g", + guest_cpus=12, + memory_mb=87040, + accelerators=[ + AcceleratorInfo(type="nvidia-tesla-a100", count=1) + ] + )), + ({ + "architecture": "X86_64", + "creationTimestamp": "1969-12-31T16:00:00.000-08:00", + "description": "8 vCPUs, 32 GB RAM", + "guestCpus": 8, + "id": "1210008", + "imageSpaceGb": 0, + "isSharedCpu": False, + "kind": "compute#machineType", + "maximumPersistentDisks": 128, + "maximumPersistentDisksSizeGb": "263168", + "memoryMb": 32768, + "name": "t2d-standard-8", + "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/europe-north2-b/machineTypes/t2d-standard-8", + "zone": "europe-north2-b" + }, MachineType( + name="t2d-standard-8", + guest_cpus=8, + memory_mb=32768, + accelerators=[] + )), + ]) +def test_MachineType_from_json(jo: dict, want: MachineType): + assert MachineType.from_json(jo) == want + + +@pytest.mark.parametrize( + "template,expected", + [ + ( + NSDict({ + "machine_type": MachineType( + name="e2", + guest_cpus=12, + memory_mb=87040, + accelerators=[]), + }), + None + ), + ( + NSDict({ + "machine_type": MachineType( + name="tpu-machine", + guest_cpus=12, + memory_mb=87040, + accelerators=[ + AcceleratorInfo(type="tpu-v6", count=1) + ]), + }), + None + ), + ( + NSDict({ + "machine_type": MachineType( + name="a2-highgpu-1g", + guest_cpus=12, + memory_mb=87040, + accelerators=[AcceleratorInfo(type="nvidia-tesla-a100", count=1)] + ), + }), + AcceleratorInfo(type="nvidia-tesla-a100", count=1) + ), + ( + NSDict({ + "machine_type": MachineType( + name="a2-highgpu-1g", + guest_cpus=12, + memory_mb=87040, + accelerators=[]), + "guestAccelerators":[ { "acceleratorCount": 1, "acceleratorType": "nvidia-tesla-a100" } ], + }), + AcceleratorInfo(type="nvidia-tesla-a100", count=1) + ), + ], +) +def test_get_template_gpu(template, expected): + assert util.get_template_gpu(template) == expected + + +UTC, PST = timezone.utc, timezone(timedelta(hours=-8)) + +@pytest.mark.parametrize( + "got,want", + [ + # from instance.creationTimestamp: + ("2024-11-30T12:47:51.676-08:00", datetime(2024, 11, 30, 12, 47, 51, 676000, tzinfo=PST)), + # from futureReservation.creationTimestamp + ("2024-11-05T15:23:33.702-08:00", datetime(2024, 11, 5, 15, 23, 33, 702000, tzinfo=PST)), + # from futureReservation.timeWindow.endTime + ("2025-01-15T00:00:00Z", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), + # fallback to UTC if no tz is specified + ("2025-01-15T00:00:00", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), + ]) +def test_parse_gcp_timestamp(got: str, want: datetime): + assert util.parse_gcp_timestamp(got) == want + + +@pytest.mark.parametrize( + "got,want", + [ + (None, None), + (dict( + windowStartTime="2025-01-15T00:00:00Z", + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + (dict( + startTimeWindow=dict( + earliest="2025-01-15T00:00:00Z"), + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + (dict( + windowStartTime="2025-01-15T00:00:00Z", + startTimeWindow=dict( + earliest="2025-01-25T00:00:00Z"), # ignored + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + ]) +def tests_parse_UpcomingMaintenance_OK(got: dict, want: Optional[UpcomingMaintenance]): + assert UpcomingMaintenance.from_json(got) == want + + +@pytest.mark.parametrize( + "got", + [ + {}, + dict( + windowStartTime=dict( + earliest="2025-01-15T00:00:00Z")), + ]) +def tests_parse_UpcomingMaintenance_FAIL(got: dict): + with pytest.raises(ValueError): + UpcomingMaintenance.from_json(got) + + +@pytest.mark.parametrize( + "got,want", + [ + (None, InstanceResourceStatus( + physical_host=None, + upcoming_maintenance=None)), + ({}, InstanceResourceStatus( + physical_host=None, + upcoming_maintenance=None)), + (dict( + physicalHost="/aaa/bbb/ccc"), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=None)), + (dict( # invalid upcomingMaintenance field to be ignored + physicalHost="/aaa/bbb/ccc", + upcomingMaintenance="maintenance is upon us"), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=None)), + (dict( + physicalHost="/aaa/bbb/ccc", + upcomingMaintenance=dict(windowStartTime="2025-01-15T00:00:00Z")), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=UpcomingMaintenance( + window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC)))), + ]) +def test_parse_InstanceResourceStatus(got: dict, want: Optional[InstanceResourceStatus]): + assert InstanceResourceStatus.from_json(got) == want + + +@pytest.mark.parametrize( + "link,component_name,expected", + [ + ( + "mylink/regions/us-cental1/other", + "regions", + "us-cental1" + ), + ( + "mylink/global/other", + "regions", + None + ), + ], +) +def test_get_self_link_component(link, component_name, expected): + assert util.get_self_link_component(link, component_name) == expected + + +def test_future_reservation_none(): + lkp = util.Lookup(TstCfg()) + assert lkp.future_reservation(TstNodeset()) == None + + +def test_future_reservation_declined(): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-03T23:30:00Z" }, + status = {"procurementStatus": "DECLINED"}, + reservationMode = "CALENDAR", + specificReservationRequired = True, + )) + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 3, 23, 30, tzinfo=timezone.utc), + reservation_mode="CALENDAR", + active_reservation=None) + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + +@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 13, 0, 0, tzinfo=timezone.utc)) +def test_future_reservation_active(_): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, + status = { + "procurementStatus": "FULFILLED", + "autoCreatedReservations": [ + "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" + ], + }, + specificReservationRequired = True, + )) + lkp._get_reservation = Mock(return_value=dict()) + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), + reservation_mode=None, + active_reservation=ReservationDetails( + project='manhattan', + zone='danger', + name='melon', + policies=[], + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/manhattan/reservations/melon", + deployment_type=None)) + + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + lkp._get_reservation.assert_called_once_with("manhattan", "danger", "melon") + +@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 28, 0, 0, tzinfo=timezone.utc)) +def test_future_reservation_inactive(_): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, + status = { + "procurementStatus": "FULFILLED", + "autoCreatedReservations": [ + "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" + ], + }, + reservationMode = "DEFAULT", + specificReservationRequired = True, + )) + lkp._get_reservation = Mock() + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), + reservation_mode="DEFAULT", + active_reservation=None) + + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + lkp._get_reservation.assert_not_called() diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test new file mode 100644 index 0000000000..a583642015 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test @@ -0,0 +1,133 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +unset CUDA_VISIBLE_DEVICES + +LOG_FILE="/var/log/slurm/chs_health_check.log" +TMP_DCGM_OUT="/tmp/dcgm.out" +TMP_ECC_ERRORS_OUT="/tmp/ecc_errors.out" + +log_step() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" +} + +# Fail gracefully if nvidia-smi or dcgmi doesn't exist +if ! type -P nvidia-smi 1>/dev/null; then + log_step "nvidia-smi not found - this script requires nvidia-smi to function" + exit 0 +fi + +if ! type -P dcgmi 1>/dev/null; then + log_step "dcgmi not found - this script requires dcgmi to function" + exit 0 +fi + +if ! type -P nv-hostengine 1>/dev/null; then + log_step "nv-hostengine not found - this script requires nv-hostengine to function" + exit 0 +fi + +################################################### +# Disable running health checks +################################################### +# Check if the environment variable '$SLURM_JOB_EXTRA' is set and contains the +# substring 'healthchecks_prolog=off' +if [[ -n "$SLURM_JOB_EXTRA" ]]; then + log_step "Environment variable SLURM_JOB_EXTRA is set. Checking if it contains healthchecks_prolog=off." + # Check if the value of the variable matches the string "healthchecks_prolog=off" + if [[ "$SLURM_JOB_EXTRA" == *"healthchecks_prolog=off"* ]]; then + log_step "Environment variable SLURM_JOB_EXTRA matches substring healthchecks_prolog=off. Skipping health checks." + exit 0 + else + log_step "Environment variable SLURM_JOB_EXTRA does NOT match substring healthchecks_prolog=off. Attempting to run health checks." + fi +else + log_step "Environment variable SLURM_JOB_EXTRA is NOT set. Attempting to run health checks." +fi + +# Exit if GPU isn't H/B 100/200 +GPU_MODEL=$(nvidia-smi --query-gpu=name --format=csv,noheader) +if ! [[ "$GPU_MODEL" =~ [BH][1-2]00 ]]; then + log_step "No Supported GPU detected" + exit 0 +fi + +NUMGPUS=$(nvidia-smi -L | wc -l) + +# Check that all GPUs are healthy via DCGM and check for ECC errors +if [ $NUMGPUS -gt 0 ]; then + log_step "Execute DCGM health check, ECC error check, and NVLink error check for GPUs" + GPULIST=$(nvidia-smi --query-gpu=index --format=csv,noheader | tr '\n' ',' | sed 's/,$//') + rm -f $TMP_DCGM_OUT + rm -f $TMP_ECC_ERRORS_OUT + + # Run DCGM checks + START_HOSTENGINE=false + if ! pidof nv-hostengine > /dev/null; then + log_step "Starting nv-hostengine..." + nv-hostengine >> "$LOG_FILE" 2>&1 + sleep 1 # Give it a moment to start up + START_HOSTENGINE=true + fi + GROUPID=$(dcgmi group -c gpuinfo | awk '{print $NF}' | tr -d ' ') + dcgmi group -g $GROUPID -a $GPULIST >> "$LOG_FILE" 2>&1 + dcgmi diag -g $GROUPID -r 1 > "$TMP_DCGM_OUT" 2>&1 + cat "$TMP_DCGM_OUT" >> "$LOG_FILE" + dcgmi group -d $GROUPID >> "$LOG_FILE" 2>&1 + + # Terminate the host engine if it was manually started + if [ "$START_HOSTENGINE" = true ]; then + log_step "Terminating nv-hostengine..." + nv-hostengine -t >> "$LOG_FILE" 2>&1 + fi + + # Check for DCGM failures + DCGM_FAILED=0 + if grep -i fail "$TMP_DCGM_OUT" > /dev/null; then + DCGM_FAILED=1 + fi + + # Check for ECC errors + nvidia-smi --query-gpu=ecc.errors.uncorrected.volatile.total --format=csv,noheader > "$TMP_ECC_ERRORS_OUT" + cat "$TMP_ECC_ERRORS_OUT" >> "$LOG_FILE" + ECC_ERRORS=$(awk -F', ' '{sum += $2} END {print sum}' "$TMP_ECC_ERRORS_OUT") + log_step "ECC Errors: $ECC_ERRORS" + + # Check for NVLink errors + NVLINK_ERRORS=$(nvidia-smi nvlink -sc 0bz -i 0 2>/dev/null | grep -i "Error Count" | awk '{sum += $3} END {print sum}') + # Set to 0 if empty/null + NVLINK_ERRORS=${NVLINK_ERRORS:-0} + log_step "NVLink Errors: $NVLINK_ERRORS" + + if [ $DCGM_FAILED -eq 1 ] || \ + [ $ECC_ERRORS -gt 0 ] || \ + [ $NVLINK_ERRORS -gt 0 ]; then + REASON="GPU issues detected: " + if [ $DCGM_FAILED -eq 1 ]; then + REASON+="DCGM test failed, " + fi + if [ $ECC_ERRORS -gt 0 ]; then + REASON+="ECC errors found ($ECC_ERRORS double-bit errors), " + fi + if [ $NVLINK_ERRORS -gt 0 ]; then + REASON+="NVLink errors detected ($NVLINK_ERRORS errors), " + fi + REASON+="see $LOG_FILE" + log_step "$REASON" + exit 1 + fi +fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog new file mode 100644 index 0000000000..a22ddea9e5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Main TaskEpilog Script +# This script executes all *.sh scripts found in /slurm/custom_scripts/task_epilog.d/ +# +# slurm.conf configuration: +# TaskEpilog=/slurm/scripts/tools/task-epilog + +# Directory containing the individual task epilog scripts +EPILOG_D_DIR="/slurm/custom_scripts/task_epilog.d" + +# --- Output Handling for TaskEpilog --- +# The stdout and stderr of this script (and the sub-scripts it calls) +# are typically captured by Slurm and written to the job's output/error file +# or a separate Slurm log, depending on configuration. +# Unlike TaskProlog, stdout is not typically parsed for special commands +# like 'export' or 'print' to affect the (now finished) task's environment. +# +# --- Error Handling --- +# If any script in EPILOG_D_DIR exits with a non-zero status, +# this main script will also exit with a non-zero status. +# Slurm will log this. Depending on Slurm's configuration, +# frequent epilog failures might lead to node issues or alerts. +set -e # Exit immediately if a command exits with a non-zero status. + +# Check if the directory exists +if [[ ! -d "$EPILOG_D_DIR" ]]; then + # Log in task stdout and exit if the directory is missing. This likely indicates a configuration error. + echo "print TaskEpilog Error: Directory '$EPILOG_D_DIR' not found. Check Slurm configuration." + exit 1 +fi + +# Find and execute all *.sh scripts in the directory +# Scripts will be executed in reverse alphabetical order of their filenames. +find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do + if [[ -x "$script" ]]; then + # Execute the script. Its stdout will be captured by this wrapper. + # Its stderr will also be passed through. + # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. + "$script" + else + # Log in task stdout a warning if a *.sh file is found but is not executable + echo "print TaskEpilog Warning: Script '$script' is not executable and will be skipped." + fi +done + +# Check if any scripts were found and executed +if [[ $(find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then + # Log in task stdout if no scripts were found to execute + echo "print TaskEpilog Info: No executable *.sh scripts found in $EPILOG_D_DIR." +fi + +# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) +exit 0 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog new file mode 100644 index 0000000000..feddb23209 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Main TaskProlog Script +# This script executes all *.sh scripts found in /slurm/custom_scripts/task_prolog.d/ +# +# slurm.conf configuration: +# TaskProlog=/slurm/scripts/tools/task-prolog + +# Directory containing the individual task prolog scripts +PROLOG_D_DIR="/slurm/custom_scripts/task_prolog.d" + +# --- Output Handling for TaskProlog --- +# Slurm's TaskProlog can interpret specific stdout lines: +# - "export NAME=value" : Sets an environment variable for the task. +# - "unset NAME" : Unsets an environment variable for the task. +# - "print message" : Prints a message to the task's standard output. +# +# This wrapper script will concatenate the stdout of all sub-scripts. +# If sub-scripts need to set/unset environment variables or print messages +# for the task, they should output the appropriate "export", "unset", or "print" +# commands to their own stdout. + +# --- Error Handling --- +# If any script in PROLOG_D_DIR exits with a non-zero status, +# this main script will also exit with a non-zero status. +# This will typically cause the task to fail. +set -e # Exit immediately if a command exits with a non-zero status. + +# Check if the directory exists +if [[ ! -d "$PROLOG_D_DIR" ]]; then + # Log in task stdout and exit if the directory is missing. All jobs will be failed. + echo "print TaskProlog Error: Directory '$PROLOG_D_DIR' not found. Check Slurm configuration." + exit 1 +fi + +# Find and execute all *.sh scripts in the directory +# Scripts will be executed in reverse alphabetical order of their filenames. +find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do + if [[ -x "$script" ]]; then + # Execute the script. Its stdout will be captured by this wrapper. + # Its stderr will also be passed through. + # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. + "$script" + else + # Log a warning in task stdout if a *.sh file is found but is not executable + echo "print TaskProlog Warning: Script '$script' is not executable and will be skipped." + fi +done + +# Check if any scripts were found and executed +if [[ $(find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then + # Log in task stdout if no scripts were found to execute + echo "print TaskProlog Info: No executable *.sh scripts found in $PROLOG_D_DIR." +fi + +# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) +exit 0 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py new file mode 100644 index 0000000000..531f0348dc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py @@ -0,0 +1,331 @@ +# mypy: ignore-errors +# This implementation of TPU integration is to be deprecated + +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import socket +import logging +from pathlib import Path +import yaml + +import util +from util import create_client_options, ApiEndpoint + +from google.cloud import tpu_v2 as tpu # noqa: E402 +import google.api_core.exceptions as gExceptions # noqa: E402 + +log = logging.getLogger() + +_tpu_cache = {} + +class TPU: + """Class for handling the TPU-vm nodes""" + + State = tpu.types.cloud_tpu.Node.State + TPUS_PER_VM = 4 + __expected_states = { + "create": State.READY, + "start": State.READY, + "stop": State.STOPPED, + } + + __tpu_version_mapping = { + "V2": tpu.AcceleratorConfig().Type.V2, + "V3": tpu.AcceleratorConfig().Type.V3, + "V4": tpu.AcceleratorConfig().Type.V4, + } + + @classmethod + def make(cls, nodeset_name: str, lkp: util.Lookup) -> "TPU": + key = (id(lkp), nodeset_name) + if key not in _tpu_cache: + nodeset = lkp.cfg.nodeset_tpu[nodeset_name] + _tpu_cache[key] = cls(nodeset, lkp) + return _tpu_cache[key] + + + def __init__(self, nodeset: object, lkp: util.Lookup): + self._nodeset = nodeset + self.lkp = lkp + self._parent = f"projects/{lkp.project}/locations/{nodeset.zone}" + co = create_client_options(ApiEndpoint.TPU) + self._client = tpu.TpuClient(client_options=co) + self.data_disks = [] + for data_disk in nodeset.data_disks: + ad = tpu.AttachedDisk() + ad.source_disk = data_disk + ad.mode = tpu.AttachedDisk.DiskMode.DISK_MODE_UNSPECIFIED + self.data_disks.append(ad) + ns_ac = nodeset.accelerator_config + if ns_ac.topology != "" and ns_ac.version != "": + ac = tpu.AcceleratorConfig() + ac.topology = ns_ac.topology + ac.type_ = self.__tpu_version_mapping[ns_ac.version] + self.ac = ac + else: + req = tpu.GetAcceleratorTypeRequest( + name=f"{self._parent}/acceleratorTypes/{nodeset.node_type}" + ) + self.ac = self._client.get_accelerator_type(req).accelerator_configs[0] + self.vmcount = self.__calc_vm_from_topology(self.ac.topology) + + @property + def nodeset(self): + return self._nodeset + + @property + def preserve_tpu(self): + return self._nodeset.preserve_tpu + + @property + def node_type(self): + return self._nodeset.node_type + + @property + def tf_version(self): + return self._nodeset.tf_version + + @property + def enable_public_ip(self): + return self._nodeset.enable_public_ip + + @property + def preemptible(self): + return self._nodeset.preemptible + + @property + def reserved(self): + return self._nodeset.reserved + + @property + def service_account(self): + return self._nodeset.service_account + + @property + def zone(self): + return self._nodeset.zone + + def check_node_type(self): + if self.node_type is None: + return False + try: + request = tpu.GetAcceleratorTypeRequest( + name=f"{self._parent}/acceleratorTypes/{self.node_type}" + ) + return self._client.get_accelerator_type(request=request) is not None + except Exception: + return False + + def check_tf_version(self): + try: + request = tpu.GetRuntimeVersionRequest( + name=f"{self._parent}/runtimeVersions/{self.tf_version}" + ) + return self._client.get_runtime_version(request=request) is not None + except Exception: + return False + + def __calc_vm_from_topology(self, topology): + topo = topology.split("x") + tot = 1 + for num in topo: + tot = tot * int(num) + return tot // self.TPUS_PER_VM + + def __check_resp(self, response, op_name): + des_state = self.__expected_states.get(op_name) + # If the state is not in the table just print the response + if des_state is None: + return False + if response.__class__.__name__ != "Node": # If the response is not a node fail + return False + if response.state == des_state: + return True + return False + + def list_nodes(self): + try: + request = tpu.ListNodesRequest(parent=self._parent) + res = self._client.list_nodes(request=request) + except gExceptions.NotFound: + res = None + return res + + def list_node_names(self): + return [node.name.split("/")[-1] for node in self.list_nodes()] + + def start_node(self, nodename): + request = tpu.StartNodeRequest(name=f"{self._parent}/nodes/{nodename}") + resp = self._client.start_node(request=request).result() + return self.__check_resp(resp, "start") + + def stop_node(self, nodename): + request = tpu.StopNodeRequest(name=f"{self._parent}/nodes/{nodename}") + resp = self._client.stop_node(request=request).result() + return self.__check_resp(resp, "stop") + + def get_node(self, nodename): + try: + request = tpu.GetNodeRequest(name=f"{self._parent}/nodes/{nodename}") + res = self._client.get_node(request=request) + except gExceptions.NotFound: + res = None + return res + + def _register_node(self, nodename, ip_addr): + dns_name = socket.getnameinfo((ip_addr, 0), 0)[0] + util.run( + f"{self.lkp.scontrol} update nodename={nodename} nodeaddr={ip_addr} nodehostname={dns_name}" + ) + + def create_node(self, nodename): + if self.vmcount > 1 and not isinstance(nodename, list): + log.error( + f"Tried to create a {self.vmcount} node TPU on nodeset {self._nodeset.nodeset_name} but only received one nodename {nodename}" + ) + return False + if self.vmcount > 1 and ( + isinstance(nodename, list) and len(nodename) != self.vmcount + ): + log.error( + f"Expected to receive a list of {self.vmcount} nodenames for TPU node creation in nodeset {self._nodeset.nodeset_name}, but received this list {nodename}" + ) + return False + + node = tpu.Node() + node.accelerator_config = self.ac + node.runtime_version = f"tpu-vm-tf-{self.tf_version}" + startup_script = """ + #!/bin/bash + echo "startup script not found > /var/log/startup_error.log" + """ + with open( + Path(self.lkp.cfg.slurm_scripts_dir or util.dirs.scripts) / "startup.sh", "r" + ) as script: + startup_script = script.read() + if isinstance(nodename, list): + node_id = nodename[0] + slurm_names = [] + wid = 0 + for node_wid in nodename: + slurm_names.append(f"WORKER_{wid}:{node_wid}") + wid += 1 + else: + node_id = nodename + slurm_names = [f"WORKER_0:{nodename}"] + node.metadata = { + "slurm_docker_image": self.nodeset.docker_image, + "startup-script": startup_script, + "slurm_instance_role": "compute", + "slurm_cluster_name": self.lkp.cfg.slurm_cluster_name, + "slurm_bucket_path": self.lkp.cfg.bucket_path, + "slurm_names": ";".join(slurm_names), + "universe_domain": util.universe_domain(), + } + node.tags = [self.lkp.cfg.slurm_cluster_name] + if self.nodeset.service_account: + node.service_account.email = self.nodeset.service_account.email + node.service_account.scope = self.nodeset.service_account.scopes + node.scheduling_config.preemptible = self.preemptible + node.scheduling_config.reserved = self.reserved + node.network_config.subnetwork = self.nodeset.subnetwork + node.network_config.enable_external_ips = self.enable_public_ip + if self.data_disks: + node.data_disks = self.data_disks + + request = tpu.CreateNodeRequest(parent=self._parent, node=node, node_id=node_id) + resp = self._client.create_node(request=request).result() + if not self.__check_resp(resp, "create"): + return False + if isinstance(nodename, list): + for node_id, net_endpoint in zip(nodename, resp.network_endpoints): + self._register_node(node_id, net_endpoint.ip_address) + else: + ip_add = resp.network_endpoints[0].ip_address + self._register_node(nodename, ip_add) + return True + + def delete_node(self, nodename): + request = tpu.DeleteNodeRequest(name=f"{self._parent}/nodes/{nodename}") + try: + resp = self._client.delete_node(request=request).result() + if resp: + return self.get_node(nodename=nodename) is None + return False + except gExceptions.NotFound: + # log only error if vmcount is 1 as for other tpu vm count, this could be "phantom" nodes + if self.vmcount == 1: + log.error(f"Tpu single node {nodename} not found") + else: + # for the TPU nodes that consist in more than one vm, only the first node of the TPU a.k.a. the master node will + # exist as real TPU nodes, so the other ones are expected to not be found, check the hostname of the node that has + # not been found, and if it ends in 0, it means that is the master node and it should have been found, and in consequence + # log an error + nodehostname = yaml.safe_load( + util.run(f"{self.lkp.scontrol} --yaml show node {nodename}").stdout.rstrip() + )["nodes"][0]["hostname"] + if nodehostname.split("-")[-1] == "0": + log.error(f"TPU master node {nodename} not found") + else: + log.info(f"Deleted TPU 'phantom' node {nodename}") + # If the node is not found it is tecnichally deleted, so return success. + return True + +def _stop_tpu(node: str) -> None: + lkp = util.lookup() + tpuobj = TPU.make(lkp.node_nodeset_name(node), lkp) + if tpuobj.nodeset.preserve_tpu and tpuobj.vmcount == 1: + log.info(f"stopping node {node}") + if tpuobj.stop_node(node): + return + log.error("Error stopping node {node} will delete instead") + log.info(f"deleting node {node}") + if not tpuobj.delete_node(node): + log.error("Error deleting node {node}") + + +def delete_tpu_instances(instances: List[str]) -> None: + util.execute_with_futures(_stop_tpu, instances) + + +def start_tpu(node: List[str]): + lkp = util.lookup() + tpuobj = TPU.make(lkp.node_nodeset_name(node[0]), lkp) + + if len(node) == 1: + node = node[0] + log.debug( + f"Will create a TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node}" + ) + tpunode = tpuobj.get_node(node) + if tpunode is None: + if not tpuobj.create_node(nodename=node): + log.error("Error creating tpu node {node}") + else: + if tpuobj.preserve_tpu: + if not tpuobj.start_node(nodename=node): + log.error("Error starting tpu node {node}") + else: + log.info( + f"Tpu node {node} is already created, but will not start it because nodeset does not have preserve_tpu option active." + ) + else: + log.debug( + f"Will create a multi-vm TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node[0]}" + ) + if not tpuobj.create_node(nodename=node): + log.error("Error creating tpu node {node}") diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py new file mode 100644 index 0000000000..217fd0bca2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py @@ -0,0 +1,2224 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable, List, Tuple, Optional, Any, Dict, Sequence, Type, Callable, Union +import argparse +import base64 +from dataclasses import dataclass, field +from datetime import timedelta, datetime, timezone +import hashlib +import inspect +import json +import logging +import logging.config +import logging.handlers +import math +import os +import re +import shlex +import shutil +import socket +import subprocess +import sys +from enum import Enum +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager +from functools import lru_cache, reduce, wraps +from itertools import chain, islice +from pathlib import Path +from time import sleep, time + +# TODO: remove "type: ignore" once moved to newer version of libraries +from google.cloud import secretmanager +from google.cloud import storage # type: ignore + +import google.auth # type: ignore +from google.oauth2 import service_account # type: ignore +import googleapiclient.discovery # type: ignore +import google_auth_httplib2 # type: ignore +from googleapiclient.http import set_user_agent # type: ignore +from google.api_core.client_options import ClientOptions +import httplib2 + +import google.api_core.exceptions as gExceptions + +import requests as requests_lib + +import yaml +from addict import Dict as NSDict # type: ignore +import file_cache + +USER_AGENT = "Slurm_GCP_Scripts/1.5 (GPN:SchedMD)" +ENV_CONFIG_YAML = os.getenv("SLURM_CONFIG_YAML") +if ENV_CONFIG_YAML: + CONFIG_FILE = Path(ENV_CONFIG_YAML) +else: + CONFIG_FILE = Path(__file__).with_name("config.yaml") +API_REQ_LIMIT = 2000 + + +def mkdirp(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + + +scripts_dir = next( + p for p in (Path(__file__).parent, Path("/slurm/scripts")) if p.is_dir() +) + + +# load all directories as Paths into a dict-like namespace +dirs = NSDict( + home = Path("/home"), + apps = Path("/opt/apps"), + slurm = Path("/slurm"), + scripts = scripts_dir, + custom_scripts = Path("/slurm/custom_scripts"), + munge = Path("/etc/munge"), + secdisk = Path("/mnt/disks/sec"), + log = Path("/var/log/slurm"), + slurm_bucket_mount = Path("/slurm/bucket"), +) + +slurmdirs = NSDict( + prefix = Path("/usr/local"), + etc = Path("/usr/local/etc/slurm"), + state = Path("/var/spool/slurm"), + key_distribution = Path("/slurm/key_distribution"), +) + + +# TODO: Remove this hack (relies on undocumented behavior of PyYAML) +# No need to represent NSDict and Path once we move to properly typed & serializable config. +yaml.SafeDumper.yaml_representers[ + None # type: ignore +] = lambda self, data: yaml.representer.SafeRepresenter.represent_str(self, str(data)) # type: ignore + + +class ApiEndpoint(Enum): + COMPUTE = "compute" + BQ = "bq" + STORAGE = "storage" + TPU = "tpu" + SECRET = "secret_manager" + + +@dataclass(frozen=True) +class AcceleratorInfo: + type: str + count: int + + @classmethod + def from_json(cls, jo: dict) -> "AcceleratorInfo": + return cls( + type=jo["guestAcceleratorType"], + count=jo["guestAcceleratorCount"]) + +@dataclass(frozen=True) +class MachineType: + name: str + guest_cpus: int + memory_mb: int + accelerators: List[AcceleratorInfo] + + @classmethod + def from_json(cls, jo: dict) -> "MachineType": + return cls( + name=jo["name"], + guest_cpus=jo["guestCpus"], + memory_mb=jo["memoryMb"], + accelerators=[ + AcceleratorInfo.from_json(a) for a in jo.get("accelerators", [])], + ) + + @property + def family(self) -> str: + # TODO: doesn't work with N1 custom machine types + # See https://cloud.google.com/compute/docs/instances/creating-instance-with-custom-machine-type#create + return self.name.split("-")[0] + + @property + def supports_smt(self) -> bool: + # https://cloud.google.com/compute/docs/cpu-platforms + if self.family in ("t2a", "t2d", "h3", "c4a", "h4d",): + return False + if self.guest_cpus == 1: + return False + return True + + @property + def sockets(self) -> int: + return { + "h3": 2, + "h4d": 2, + "c2d": 2 if self.guest_cpus > 56 else 1, + "a3": 2, + "c2": 2 if self.guest_cpus > 30 else 1, + "c3": 2 if self.guest_cpus > 88 else 1, + "c3d": 2 if self.guest_cpus > 180 else 1, + "c4": 2 if self.guest_cpus > 96 else 1, + "c4d": 2 if self.guest_cpus > 192 else 1, + }.get( + self.family, + 1, # assume 1 socket for all other families + ) + + +@dataclass(frozen=True) +class UpcomingMaintenance: + window_start_time: datetime + + @classmethod + def from_json(cls, jo: Optional[dict]) -> Optional["UpcomingMaintenance"]: + if jo is None: + return None + try: + if "windowStartTime" in jo: + ts = parse_gcp_timestamp(jo["windowStartTime"]) + elif "startTimeWindow" in jo: + ts = parse_gcp_timestamp(jo["startTimeWindow"]["earliest"]) + else: + raise Exception("Neither windowStartTime nor startTimeWindow are found") + except BaseException as e: + raise ValueError(f"Unexpected format for upcomingMaintenance: {jo}") from e + return cls(window_start_time=ts) + +@dataclass(frozen=True) +class InstanceResourceStatus: + physical_host: Optional[str] + upcoming_maintenance: Optional[UpcomingMaintenance] + + @classmethod + def from_json(cls, jo: Optional[dict]) -> "InstanceResourceStatus": + if not jo: + return cls( + physical_host=None, + upcoming_maintenance=None, + ) + + try: + maint = UpcomingMaintenance.from_json(jo.get("upcomingMaintenance")) + except ValueError as e: + log.exception("Failed to parse upcomingMaintenance, ignoring") + maint = None # intentionally swallow exception + + return cls( + physical_host=jo.get("physicalHost"), + upcoming_maintenance=maint, + ) + + +@dataclass(frozen=True) +class Instance: + name: str + zone: str + status: str + creation_timestamp: datetime + role: Optional[str] + resource_status: InstanceResourceStatus + metadata: Dict[str, str] + # TODO: use proper InstanceScheduling class + scheduling: NSDict + + @classmethod + def from_json(cls, jo: dict) -> "Instance": + return cls( + name=jo["name"], + zone=trim_self_link(jo["zone"]), + status=jo["status"], + creation_timestamp=parse_gcp_timestamp(jo["creationTimestamp"]), + resource_status=InstanceResourceStatus.from_json(jo.get("resourceStatus")), + scheduling=NSDict(jo.get("scheduling")), + role = jo.get("labels", {}).get("slurm_instance_role"), + metadata = {k["key"]: k["value"] for k in jo.get("metadata", {}).get("items", [])} + ) + + +@dataclass(frozen=True) +class NSMount: + server_ip: str + local_mount: Path + remote_mount: Path + fs_type: str + mount_options: str + +@lru_cache(maxsize=1) +def default_credentials(): + return google.auth.default()[0] + + +@lru_cache(maxsize=1) +def authentication_project(): + return google.auth.default()[1] + + +DEFAULT_UNIVERSE_DOMAIN = "googleapis.com" + + +def now() -> datetime: + """ + Return current time as timezone-aware datetime. + + IMPORTANT: DO NOT use `datetime.now()`, unless you explicitly need to have tz-naive datetime. + Otherwise there is a risk of getting: "cannot compare naive and aware datetimes" error, + since all timetstamps we receive from GCP API are tz-aware. + + Another motivation for this function is to allow to mock time in tests. + """ + return datetime.now(timezone.utc) + +def parse_gcp_timestamp(s: str) -> datetime: + """ + Parse timestamp strings returned by GCP API into datetime. + Works with both Zulu and non-Zulu timestamps. + NOTE: It always return tz-aware datetime (fallbacks to UTC and logs error). + """ + # Requires Python >= 3.7 + # TODO: Remove this "hack" of trimming the Z from timestamps once we move to Python 3.11 + # (context: https://discuss.python.org/t/parse-z-timezone-suffix-in-datetime/2220/30) + ts = datetime.fromisoformat(s.replace('Z', '+00:00')) + if ts.tzinfo is None: # fallback to UTC + log.error(f"Received timestamp without timezone info: {s}") + ts = ts.replace(tzinfo=timezone.utc) + return ts + + +def universe_domain() -> str: + try: + return instance_metadata("attributes/universe_domain") + except MetadataNotFoundError: + return DEFAULT_UNIVERSE_DOMAIN + + +def endpoint_version(api: ApiEndpoint) -> Optional[str]: + return lookup().endpoint_versions.get(api.value, None) + + +@lru_cache(maxsize=1) +def get_credentials() -> Optional[service_account.Credentials]: + """Get credentials for service account""" + key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if key_path is not None: + credentials = service_account.Credentials.from_service_account_file( + key_path, scopes=[f"https://www.{universe_domain()}/auth/cloud-platform"] + ) + else: + credentials = default_credentials() + + return credentials + + +@lru_cache(maxsize=1) +def get_dev_key() -> Optional[str]: + """Get dev key for project (uses json or yaml format)""" + try: + with open("/etc/slurm/slurm_vars.yaml", 'r') as file: + data = yaml.safe_load(file) + return data['google_developer_key'] + except: + return None + + +def create_client_options(api: ApiEndpoint) -> ClientOptions: + """Create client options for cloud endpoints""" + ver = endpoint_version(api) + ud = universe_domain() + options = {} + if ud and ud != DEFAULT_UNIVERSE_DOMAIN: + options["universe_domain"] = ud + if ver: + options["api_endpoint"] = f"https://{api.value}.{ud}/{ver}/" + co = ClientOptions(**options) + log.debug(f"Using ClientOptions = {co} for API: {api.value}") + return co + +log = logging.getLogger() + + +def access_secret_version(project_id, secret_id, version_id="latest"): + """ + Access the payload for the given secret version if one exists. The version + can be a version number as a string (e.g. "5") or an alias (e.g. "latest"). + """ + co = create_client_options(ApiEndpoint.SECRET) + client = secretmanager.SecretManagerServiceClient(client_options=co) + name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" + try: + response = client.access_secret_version(request={"name": name}) + log.debug(f"Secret '{name}' was found.") + payload = response.payload.data.decode("UTF-8") + except gExceptions.NotFound: + log.debug(f"Secret '{name}' was not found!") + payload = None + + return payload + + +def parse_self_link(self_link: str): + """Parse a selfLink url, extracting all useful values + https://.../v1/projects//regions//... + {'project': , 'region': , ...} + can also extract zone, instance (name), image, etc + """ + link_patt = re.compile(r"(?P[^\/\s]+)s\/(?P[^\s\/]+)") + return NSDict(link_patt.findall(self_link)) + + +def parse_bucket_uri(uri: str): + """ + Parse a bucket url + E.g. gs:/// + """ + pattern = re.compile(r"gs://(?P[^/\s]+)/(?P([^/\s]+)(/[^/\s]+)*)") + matches = pattern.match(uri) + assert matches, f"Unexpected bucker URI: '{uri}'" + return matches.group("bucket"), matches.group("path") + + +def get_template_gpu(template): + """get gpu info from machine type or guest accelerators""" + gpu_keyword = "nvidia" + gpu = None + if template.machine_type.accelerators: + tma = template.machine_type.accelerators[0] + if gpu_keyword in tma.type.lower(): + gpu = tma + elif template.guestAccelerators: + tga = template.guestAccelerators[0] + if gpu_keyword in tga.acceleratorType.lower(): + gpu = AcceleratorInfo( + type=tga.acceleratorType, + count=tga.acceleratorCount) + return gpu + + +def trim_self_link(link: str): + """get resource name from self link url, eg. + https://.../v1/projects//regions/ + -> + """ + try: + return link[link.rindex("/") + 1 :] + except ValueError: + raise Exception(f"'/' not found, not a self link: '{link}' ") + + +def get_self_link_component(link: str, component_name: str): + """ + Extracts a component (e.g., 'region', 'project') from a self-link URL. + Args: + link: The self-link URL string. + component_name: The name of the component to extract (e.g., 'regions', 'projects'). + Returns: + The extracted component value (e.g., '', ''), + or None if the component is not found in the link. + """ + search_string = f"/{component_name}/" + start_index = link.rfind(search_string) + + if start_index == -1: + return None + + start_index += len(search_string) + end_index = link.find("/", start_index) + + if end_index == -1: + # If no further slash, the rest of the string is the component + return link[start_index:] + else: + return link[start_index:end_index] + + +def execute_with_futures(func, seq): + with ThreadPoolExecutor() as exe: + futures = [] + for i in seq: + future = exe.submit(func, i) + futures.append(future) + for future in as_completed(futures): + result = future.exception() + if result is not None: + raise result + + +def map_with_futures(func, seq): + with ThreadPoolExecutor() as exe: + futures = [] + for i in seq: + future = exe.submit(func, i) + futures.append(future) + for future in futures: + # Will be result or raise Exception + res = None + try: + res = future.result() + except Exception as e: + res = e + yield res + +def should_mount_slurm_bucket() -> bool: + try: + return instance_metadata("attributes/slurm_bucket_mount", silent=True).lower() == "true" + except MetadataNotFoundError: + return False + + +def _get_bucket_and_common_prefix() -> Tuple[str, str]: + uri = instance_metadata("attributes/slurm_bucket_path") + return parse_bucket_uri(uri) + +def blob_get(file): + bucket_name, path = _get_bucket_and_common_prefix() + blob_name = f"{path}/{file}" + return storage_client().get_bucket(bucket_name).blob(blob_name) + + +def blob_list(prefix="", delimiter=None): + bucket_name, path = _get_bucket_and_common_prefix() + blob_prefix = f"{path}/{prefix}" + # Note: The call returns a response only when the iterator is consumed. + blobs = storage_client().list_blobs( + bucket_name, prefix=blob_prefix, delimiter=delimiter + ) + return [blob for blob in blobs] + +def file_list(prefix="", subpath="") -> List[os.DirEntry]: + path = dirs.slurm_bucket_mount + file_prefix = f"{path}/{subpath}" + try: + files = os.scandir(file_prefix) + return [file for file in files if file.name.startswith(prefix)] + except: + return [] + # Not considering lack of file's existence as fatal (we may check for files we know don't exist). + # Responsibility of callee to determine if it is fatal or not, blob_list returns empty iterator in similar cases. + +def hash_file(fullpath: Path) -> str: + with open(fullpath, "rb") as f: + file_hash = hashlib.md5() + chunk = f.read(8192) + while chunk: + file_hash.update(chunk) + chunk = f.read(8192) + return base64.b64encode(file_hash.digest()).decode("utf-8") + + +def install_custom_scripts(check_hash:bool=False): + """download custom scripts from gcs bucket""" + role, tokens = lookup().instance_role, [] + + mounted_scripts=False + if should_mount_slurm_bucket() and role != "controller": + mounted_scripts=True + + all_prolog_tokens = ["prolog", "epilog", "task_prolog", "task_epilog"] + if role == "controller": + tokens = ["controller"] + all_prolog_tokens + elif role == "compute": + tokens = [f"nodeset-{lookup().node_nodeset_name()}"] + all_prolog_tokens + elif role == "login": + tokens = [f"login-{instance_login_group()}"] + + prefixes = [f"slurm-{tok}-script" for tok in tokens] + + # TODO: use single `blob_list`, to reduce ~4x number of GCS requests + if mounted_scripts: + source_collection = list(chain.from_iterable(file_list(prefix=p) for p in prefixes)) + else: + source_collection = list(chain.from_iterable(blob_list(prefix=p) for p in prefixes)) + + script_pattern = re.compile(r"^slurm-(?P\S+)-script-(?P\S+)") + for source in source_collection: + if mounted_scripts: + m = script_pattern.match(source.name) + else: + m = script_pattern.match(Path(source.name).name) + + if not m: + log.warning(f"found blob that doesn't match expected pattern: {source.name}") + continue + path_parts = m["path"].split("-") + path_parts[0] += ".d" + stem, _, ext = m["name"].rpartition("_") + filename = ".".join((stem, ext)) + + path = Path(*path_parts, filename) + fullpath = (dirs.custom_scripts / path).resolve() + mkdirp(fullpath.parent) + + for par in path.parents: + chown_slurm(dirs.custom_scripts / par) + need_update = True + + if check_hash and fullpath.exists() and isinstance(source,storage.Blob): + # TODO: MD5 reported by gcloud may differ from the one calculated here (e.g. if blob got gzipped), + # consider using gCRC32C + need_update = hash_file(fullpath) != source.md5_hash + + log.info(f"installing custom script: {path} from {source.name}") + + if isinstance(source,os.DirEntry): + shutil.copy(source.path, fullpath) #Needs to be copied since mounted nfs is read-only + chown_slurm(fullpath, mode=0o755) + + elif need_update: + with fullpath.open("wb") as f: + source.download_to_file(f) + chown_slurm(fullpath, mode=0o755) + +def compute_service(version="beta"): + """Make thread-safe compute service handle + creates a new Http for each request + """ + credentials = get_credentials() + dev_key = get_dev_key() + + def build_request(http, *args, **kwargs): + new_http = set_user_agent(httplib2.Http(), USER_AGENT) + if credentials is not None: + new_http = google_auth_httplib2.AuthorizedHttp(credentials, http=new_http) + return googleapiclient.http.HttpRequest(new_http, *args, **kwargs) + + ver = endpoint_version(ApiEndpoint.COMPUTE) + disc_url = googleapiclient.discovery.DISCOVERY_URI + if ver: + version = ver + disc_url = disc_url.replace(DEFAULT_UNIVERSE_DOMAIN, universe_domain()) + + log.debug(f"Using version={version} of Google Compute Engine API") + return googleapiclient.discovery.build( + "compute", + version, + requestBuilder=build_request, + credentials=credentials, + developerKey=dev_key, + discoveryServiceUrl=disc_url, + cache_discovery=False, # See https://github.com/googleapis/google-api-python-client/issues/299 + ) + +def storage_client() -> storage.Client: + """ + Config-independent storage client + """ + ud = universe_domain() + co = {} + if ud and ud != DEFAULT_UNIVERSE_DOMAIN: + co["universe_domain"] = ud + return storage.Client(client_options=ClientOptions(**co)) + + +class DeffetiveStoredConfigError(Exception): + """ + Raised when config can not be loaded and assembled from bucket + """ + pass + + +def _fill_cfg_defaults(cfg: NSDict) -> NSDict: + if not cfg.slurm_log_dir: + cfg.slurm_log_dir = dirs.log + if not cfg.slurm_bin_dir: + cfg.slurm_bin_dir = slurmdirs.prefix / "bin" + if not cfg.slurm_control_host: + try: + control_dns_name = instance_metadata("attributes/slurm_control_dns", silent=True) + cfg.slurm_control_host = control_dns_name + except MetadataNotFoundError: + cfg.slurm_control_host = f"{cfg.slurm_cluster_name}-controller" + if not cfg.slurm_control_host_port: + cfg.slurm_control_host_port = "6820-6830" + return cfg + +@dataclass +class _ConfigBlobs: + """ + "Private" class that represent a collection of GCS blobs for configuration + """ + core: storage.Blob + controller_addr: Optional[storage.Blob] + partition: List[storage.Blob] = field(default_factory=list) + nodeset: List[storage.Blob] = field(default_factory=list) + nodeset_dyn: List[storage.Blob] = field(default_factory=list) + nodeset_tpu: List[storage.Blob] = field(default_factory=list) + login_group: List[storage.Blob] = field(default_factory=list) + + @property + def hash(self) -> str: + h = hashlib.md5() + all = [self.core] + self.partition + self.nodeset + self.nodeset_dyn + self.nodeset_tpu + if self.controller_addr: + all.append(self.controller_addr) + + # sort blobs so hash is consistent + for blob in sorted(all, key=lambda b: b.name): + h.update(blob.md5_hash.encode("utf-8")) + return h.hexdigest() + +@dataclass +class _ConfigFiles: + """ + "Private" class that represent a collection of files for configuration + """ + core: Path + controller_addr: Optional[Path] + partition: List[Path] = field(default_factory=list) + nodeset: List[Path] = field(default_factory=list) + nodeset_dyn: List[Path] = field(default_factory=list) + nodeset_tpu: List[Path] = field(default_factory=list) + login_group: List[Path] = field(default_factory=list) + +def _list_config_blobs() -> _ConfigBlobs: + _, common_prefix = _get_bucket_and_common_prefix() + + core: Optional[storage.Blob] = None + controller_addr: Optional[storage.Blob] = None + rest: Dict[str, List[storage.Blob]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} + + is_controller = instance_role() == "controller" + + for blob in blob_list(prefix=""): + if blob.name == f"{common_prefix}/config.yaml": + core = blob + if blob.name == f"{common_prefix}/controller_addr.yaml" and not is_controller: + # Don't add this config blobs for controller to avoid "double reconfiguration": + # Initially this file doesn't exist and produce later by `setup_controller`; + # Appearance of this blob would trigger change in combined hash of config files; + # Ignore existence of this file for controller, assume that + # no other instance nodes will proceed with configuration until this file is created. + controller_addr = blob + for key in rest.keys(): + if blob.name.startswith(f"{common_prefix}/{key}_configs/"): + rest[key].append(blob) + + if core is None: + raise DeffetiveStoredConfigError(f"{common_prefix}/config.yaml not found in bucket") + + return _ConfigBlobs(core=core, controller_addr=controller_addr, **rest) + +def _list_config_files() -> _ConfigFiles: + file_dir = dirs.slurm_bucket_mount + core: Optional[Path] = None + controller_addr: Optional[Path] = None + rest: Dict[str, List[Path]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} + + if Path(f"{file_dir}/config.yaml").exists(): + core = Path(f"{file_dir}/config.yaml") + + for key in rest.keys(): + for f in file_list(subpath=f"{key}_configs"): + rest[key].append(f.path) + + if core is None: + raise Exception(f"config.yaml was not found in mounted folder: {dirs.slurm_bucket_mount}") #Intentionally not using DeffetiveStoredConfigError as this is considered a fatal error + + return _ConfigFiles(core=core, controller_addr=None, **rest) + +def _fetch_config(old_hash: Optional[str]) -> Optional[Tuple[NSDict, str]]: + """Fetch config from bucket, returns None if no changes are detected.""" + blobs = _list_config_blobs() + if old_hash == blobs.hash: + return None + + def _download(bs) -> List[Any]: + return [yaml.safe_load(b.download_as_text()) for b in bs] + + return _assemble_config( + core=_download([blobs.core])[0], + controller_addr=_download([blobs.controller_addr])[0] if blobs.controller_addr else None, + partitions=_download(blobs.partition), + nodesets=_download(blobs.nodeset), + nodesets_dyn=_download(blobs.nodeset_dyn), + nodesets_tpu=_download(blobs.nodeset_tpu), + login_groups=_download(blobs.login_group), + ), blobs.hash + +def _fetch_mounted_config() -> Optional[Tuple[NSDict, str]]: + if not dirs.slurm_bucket_mount.is_mount(): + raise Exception(f"{dirs.slurm_bucket_mount} is not mounted") + + files = _list_config_files() + + def _load(files) -> List[Any]: + file_yaml=[] + for file in files: + with open(file, "r") as f: + file_yaml.append(yaml.safe_load(f)) + return file_yaml + + return _assemble_config( + core=_load([files.core])[0], + controller_addr=None, + partitions=_load(files.partition), + nodesets=_load(files.nodeset), + nodesets_dyn=_load(files.nodeset_dyn), + nodesets_tpu=_load(files.nodeset_tpu), + login_groups=_load(files.login_group), + ) + +def controller_lookup_self_ip() -> str: + assert instance_role() == "controller" + # Get IP of LAST network-interface + # TODO: Consider change order of NICs definition, so right NIC is always @0. + idx = instance_metadata("network-interfaces").split()[-1] # either `0/` or `1/` + return instance_metadata(f"network-interfaces/{idx}ip") + +def _assemble_config( + core: Any, + controller_addr: Optional[Any], + partitions: List[Any], + nodesets: List[Any], + nodesets_dyn: List[Any], + nodesets_tpu: List[Any], + login_groups: List[Any], + ) -> NSDict: + cfg = NSDict(core) + + if cfg.controller_network_attachment: + # lookup controller address + if instance_role() == "controller": + # ignore stored value of `controller_addr`, it will be overwritten during `setup_controller` + cfg.slurm_control_addr = controller_lookup_self_ip() + else: + if not controller_addr: + raise DeffetiveStoredConfigError("controller_addr.yaml not found in bucket") + cfg.slurm_control_addr = controller_addr["slurm_control_addr"] + + # add partition configs + for p_yaml in partitions: + p_cfg = NSDict(p_yaml) + assert p_cfg.get("partition_name"), "partition_name is required" + p_name = p_cfg.partition_name + assert p_name not in cfg.partitions, f"partition {p_name} already defined" + cfg.partitions[p_name] = p_cfg + + # add nodeset configs + ns_names = set() + def _add_nodesets(yamls: List[Any], target: dict): + for ns_yaml in yamls: + ns_cfg = NSDict(ns_yaml) + assert ns_cfg.get("nodeset_name"), "nodeset_name is required" + ns_name = ns_cfg.nodeset_name + assert ns_name not in ns_names, f"nodeset {ns_name} already defined" + target[ns_name] = ns_cfg + ns_names.add(ns_name) + + _add_nodesets(nodesets, cfg.nodeset) + _add_nodesets(nodesets_dyn, cfg.nodeset_dyn) + _add_nodesets(nodesets_tpu, cfg.nodeset_tpu) + + # validate that configs for all referenced nodesets are present + for p in cfg.partitions.values(): + for ns_name in chain(p.partition_nodeset, p.partition_nodeset_dyn, p.partition_nodeset_tpu): + if ns_name not in ns_names: + raise DeffetiveStoredConfigError(f"nodeset {ns_name} not defined in config") + + for lg_yaml in login_groups: + lg_cfg = NSDict(lg_yaml) + assert lg_cfg.get("group_name"), "group_name is required" + lg_name = lg_cfg.group_name + assert lg_name not in cfg.login_groups + cfg.login_groups[lg_name] = lg_cfg + + if instance_role() == "login": + group = instance_login_group() + if group not in cfg.login_groups: + raise DeffetiveStoredConfigError(f"login group '{group}' does not exist in config") + + return _fill_cfg_defaults(cfg) + +def fetch_config() -> Tuple[bool, NSDict]: + """ + Fetches config from bucket and saves it locally + Returns True if new (updated) config was fetched + """ + hash_file = Path("/slurm/scripts/.config.hash") + old_hash = hash_file.read_text() if hash_file.exists() else None + + if should_mount_slurm_bucket() and instance_role() != "controller": + cfg = _fetch_mounted_config() + CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) + chown_slurm(CONFIG_FILE) + return False, cfg + + cfg_and_hash = _fetch_config(old_hash=old_hash) + + if not cfg_and_hash: + return False, _load_config() + + cfg, hash = cfg_and_hash + hash_file.write_text(hash) + chown_slurm(hash_file) + CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) + chown_slurm(CONFIG_FILE) + return True, cfg + +def owned_file_handler(filename): + """create file handler""" + chown_slurm(filename) + return logging.handlers.WatchedFileHandler(filename, delay=True) + +def get_log_path() -> Path: + """ + Returns path to log file for the current script. + e.g. resume.py -> /var/log/slurm/resume.log + """ + cfg_log_dir = lookup().cfg.slurm_log_dir + log_dir = Path(cfg_log_dir) if cfg_log_dir else dirs.log + return (log_dir / Path(sys.argv[0]).name).with_suffix(".log") + +def init_log_and_parse(parser: argparse.ArgumentParser) -> argparse.Namespace: + parser.add_argument( + "--debug", + "-d", + dest="loglevel", + action="store_const", + const=logging.DEBUG, + default=logging.INFO, + help="Enable debugging output", + ) + parser.add_argument( + "--trace-api", + "-t", + action="store_true", + help="Enable detailed api request output", + ) + args = parser.parse_args() + loglevel = args.loglevel + if lookup().cfg.enable_debug_logging: + loglevel = logging.DEBUG + if args.trace_api: + lookup().cfg.extra_logging_flags["trace_api"] = True + # Configure root logger + logging.config.dictConfig({ + "version": 1, + "disable_existing_loggers": True, + "formatters": { + "standard": { + "format": "%(levelname)s: %(message)s", + }, + "stamp": { + "format": "%(asctime)s %(levelname)s: %(message)s", + }, + }, + "handlers": { + "stdout_handler": { + "level": logging.DEBUG, + "formatter": "standard", + "class": "logging.StreamHandler", + "stream": sys.stdout, + }, + "file_handler": { + "()": owned_file_handler, + "level": logging.DEBUG, + "formatter": "stamp", + "filename": get_log_path(), + }, + }, + "root": { + "handlers": ["stdout_handler", "file_handler"], + "level": loglevel, + }, + }) + + sys.excepthook = _handle_exception + + return args + + +def log_api_request(request): + """log.trace info about a compute API request""" + if not lookup().cfg.extra_logging_flags.get("trace_api"): + return + # output the whole request object as pretty yaml + # the body is nested json, so load it as well + rep = json.loads(request.to_json()) + if rep.get("body", None) is not None: + rep["body"] = json.loads(rep["body"]) + pretty_req = yaml.safe_dump(rep).rstrip() + # label log message with the calling function + log.debug(f"{inspect.stack()[1].function}:\n{pretty_req}") + + +def _handle_exception(exc_type, exc_value, exc_trace): + """log exceptions other than KeyboardInterrupt""" + if not issubclass(exc_type, KeyboardInterrupt): + log.exception("Fatal exception", exc_info=(exc_type, exc_value, exc_trace)) + sys.__excepthook__(exc_type, exc_value, exc_trace) + + +def run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + timeout=None, + check=True, + universal_newlines=True, + **kwargs, +): + """Wrapper for subprocess.run() with convenient defaults""" + if isinstance(args, list): + args = list(filter(lambda x: x is not None, args)) + args = " ".join(args) + if not shell and isinstance(args, str): + args = shlex.split(args) + log.debug(f"run: {args}") + try: + result = subprocess.run( + args, + stdout=stdout, + stderr=stderr, + shell=shell, + timeout=timeout, + check=check, + universal_newlines=universal_newlines, + **kwargs, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + log_subprocess(e) + raise + log_subprocess(result) + return result + +def log_subprocess(subj: subprocess.CalledProcessError | subprocess.TimeoutExpired | subprocess.CompletedProcess) -> None: + match subj: + case subprocess.CompletedProcess(returncode=0): + # Do not log successful runs, to not overwhelm logs (e.g. scontrol show jobs --json) + # TODO: consider still doing it in DEBUG or trim output to few KBs. + return + case subprocess.CompletedProcess(): # non-zero returncode + log.error(f"Command '{subj.args}' returned exit status {subj.returncode}.") + case subprocess.CalledProcessError() | subprocess.TimeoutExpired(): + log.error(str(subj)) + + + def normalize(out: None | str | bytes) -> None | str: + """ + Turns stderr and stdout into string: + > A bytes sequence, or a string if run() was called with an encoding, errors, or text=True. None if was not captured. + """ + match out: + case None: + return None + case str(): + return out.strip() + case bytes(): + return out.decode().strip() + case _: + return repr(out) + + if stdout := normalize(subj.stdout): + log.error(f"stdout: {stdout}") + if stderr := normalize(subj.stderr): + log.error(f"stderr: {stderr}") + + +def chown_slurm(path: Path, mode=None) -> None: + if path.exists(): + if mode: + path.chmod(mode) + else: + mkdirp(path.parent) + if mode: + path.touch(mode=mode) + else: + path.touch() + try: + shutil.chown(path, user="slurm", group="slurm") + except LookupError: + log.warning(f"User 'slurm' does not exist. Cannot 'chown slurm:slurm {path}'.") + except PermissionError: + log.warning(f"Not authorized to 'chown slurm:slurm {path}'.") + except Exception as err: + log.error(err) + + +@contextmanager +def cd(path): + """Change working directory for context""" + prev = Path.cwd() + os.chdir(path) + try: + yield + finally: + os.chdir(prev) + + +def cached_property(f): + return property(lru_cache()(f)) + + +def retry(max_retries: int, init_wait_time: float, warn_msg: str, exc_type: Type[Exception]): + """Retries functions that raises the exception exc_type. + Retry time is increased by a factor of two for every iteration. + + Args: + max_retries (int): Maximum number of retries + init_wait_time (float): Initial wait time in secs + warn_msg (str): Message to print during retries + exc_type (Exception): Exception type to check for + """ + + if max_retries <= 0: + raise ValueError("Incorrect value for max_retries, must be >= 1") + if init_wait_time <= 0.0: + raise ValueError("Invalid value for init_wait_time, must be > 0.0") + + def decorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + retry = 0 + secs = init_wait_time + captured_exc: Optional[BaseException] = None + while retry < max_retries: + try: + return f(*args, **kwargs) + except exc_type as e: + captured_exc = e + log.warn(f"{warn_msg}, retrying in {secs}") + sleep(secs) + retry += 1 + secs *= 2 + assert captured_exc + raise captured_exc + + return wrapper + + return decorator + + +def separate(pred: Callable[[Any], bool], coll: Iterable[Any]) -> Tuple[List[Any], List[Any]]: + """filter into 2 lists based on pred returning True or False + returns ([False], [True]) + """ + res: Tuple[List[Any], List[Any]] = ([],[]) + for el in coll: + res[pred(el)].append(el) + return res + + +def chunked(iterable, n=API_REQ_LIMIT): + """group iterator into chunks of max size n""" + it = iter(iterable) + while True: + chunk = list(islice(it, n)) + if not chunk: + return + yield chunk + +def groupby_unsorted(seq: Sequence[Any], key): + indices = defaultdict(list) + for i, el in enumerate(seq): + indices[key(el)].append(i) + for k, idxs in indices.items(): + yield k, (seq[i] for i in idxs) + + +@lru_cache(maxsize=32) +def find_ratio(a, n, s, r0=None): + """given the start (a), count (n), and sum (s), find the ratio required""" + if n == 2: + return s / a - 1 + an = a * n + if n == 1 or s == an: + return 1 + if r0 is None: + # we only need to know which side of 1 to guess, and the iteration will work + r0 = 1.1 if an < s else 0.9 + + # geometric sum formula + def f(r): + return a * (1 - r**n) / (1 - r) - s + + # derivative of f + def df(r): + rm1 = r - 1 + rn = r**n + return (a * (rn * (n * rm1 - r) + r)) / (r * rm1**2) + + MIN_DR = 0.0001 # negligible change + r = r0 + # print(f"r(0)={r0}") + MAX_TRIES = 64 + for i in range(1, MAX_TRIES + 1): + try: + dr = f(r) / df(r) + except ZeroDivisionError: + log.error(f"Failed to find ratio due to zero division! Returning r={r0}") + return r0 + r = r - dr + # print(f"r({i})={r}") + # if the change in r is small, we are close enough + if abs(dr) < MIN_DR: + break + else: + log.error(f"Could not find ratio after {MAX_TRIES}! Returning r={r0}") + return r0 + return r + + +def backoff_delay(start, timeout=None, ratio=None, count: int = 0): + """generates `count` waits starting at `start` + sum of waits is `timeout` or each one is `ratio` bigger than the last + the last wait is always 0""" + # timeout or ratio must be set but not both + assert (timeout is None) ^ (ratio is None) + assert ratio is None or ratio > 0 + assert timeout is None or timeout >= start + assert (count > 1 or timeout is not None) and isinstance(count, int) + assert start > 0 + + if count == 0: + # Equation for auto-count is tuned to have a max of + # ~int(timeout) counts with a start wait of <0.01. + # Increasing start wait decreases count eg. + # backoff_delay(10, timeout=60) -> count = 5 + count = int( + (timeout / ((start + 0.05) ** (1 / 2)) + 2) // math.log(timeout + 2) + ) + + yield start + # if ratio is set: + # timeout = start * (1 - ratio**(count - 1)) / (1 - ratio) + if ratio is None: + ratio = find_ratio(start, count - 1, timeout) + + wait = start + # we have start and 0, so we only need to generate count - 2 + for _ in range(count - 2): + wait *= ratio + yield wait + yield 0 + return + + +ROOT_URL = "http://metadata.google.internal/computeMetadata/v1" + +class MetadataNotFoundError(Exception): + pass + +def get_metadata(path:str, silent=False) -> str: + """Get metadata relative to metadata/computeMetadata/v1""" + HEADERS = {"Metadata-Flavor": "Google"} + url = f"{ROOT_URL}/{path}" + try: + resp = requests_lib.get(url, headers=HEADERS) + resp.raise_for_status() + return resp.text + except requests_lib.exceptions.HTTPError: + if not silent: + log.warning(f"metadata not found ({url})") + raise MetadataNotFoundError(f"failed to get_metadata from {url}") + + +@lru_cache(maxsize=None) +def instance_metadata(path: str, silent:bool=False) -> str: + return get_metadata(f"instance/{path}", silent=silent) + +def instance_role(): + return instance_metadata("attributes/slurm_instance_role") + + +def instance_login_group(): + return instance_metadata("attributes/slurm_login_group") + + +def natural_sort(text): + def atoi(text): + return int(text) if text.isdigit() else text + + return [atoi(w) for w in re.split(r"(\d+)", text)] + + +def to_hostlist(names: Iterable[str]) -> str: + """ + Fast implementation of `hostlist` that doesn't invoke `scontrol` + IMPORTANT: + * Acts as `scontrol show hostlistsorted`, i.e. original order is not preserved + * Achieves worse compression than `scontrol show hostlist` for some cases + """ + pref = defaultdict(list) + tokenizer = re.compile(r"^(.*?)(\d*)$") + for name in filter(None, names): + matches = tokenizer.match(name) + assert matches, name + p, s = matches.groups() + pref[p].append(s) + + def _compress_suffixes(ss: List[str]) -> List[str]: + cur, res = None, [] + + def cur_repr(): + assert cur + nums, strs = cur + if nums[0] == nums[1]: + return strs[0] + return f"{strs[0]}-{strs[1]}" + + for s in sorted(ss, key=int): + n = int(s) + if cur is None: + cur = ((n, n), (s, s)) + continue + + nums, strs = cur + if n == nums[1] + 1: + cur = ((nums[0], n), (strs[0], s)) + else: + res.append(cur_repr()) + cur = ((n, n), (s, s)) + if cur: + res.append(cur_repr()) + return res + + res = [] + for p in sorted(pref.keys()): + sl = defaultdict(list) + for s in pref[p]: + sl[len(s)].append(s) + cs = [] + for ln in sorted(sl.keys()): + if ln == 0: + res.append(p) + else: + cs.extend(_compress_suffixes(sl[ln])) + if not cs: + continue + if len(cs) == 1 and "-" not in cs[0]: + res.append(f"{p}{cs[0]}") + else: + res.append(f"{p}[{','.join(cs)}]") + return ",".join(res) + +@lru_cache(maxsize=None) +def to_hostnames(nodelist: str) -> List[str]: + """make list of hostnames from hostlist expression""" + if not nodelist: + return [] # avoid degenerate invocation of scontrol + if isinstance(nodelist, str): + hostlist = nodelist + else: + hostlist = ",".join(nodelist) + hostnames = run(f"{lookup().scontrol} show hostnames {hostlist}").stdout.splitlines() + return hostnames + + +def retry_exception(exc) -> bool: + """return true for exceptions that should always be retried""" + msg = str(exc) + retry_errors = ( + "Rate Limit Exceeded", + "Quota Exceeded", + "Quota exceeded", + ) + return any(err in msg for err in retry_errors) + + +def ensure_execute(request): + """Handle rate limits and socket time outs""" + + for retry, wait in enumerate(backoff_delay(0.5, timeout=10 * 60, count=20)): + try: + return request.execute() + except googleapiclient.errors.HttpError as e: + if retry_exception(e): + log.error(f"retry:{retry} '{e}'") + sleep(wait) + continue + raise + + except socket.timeout as e: + # socket timed out, try again + log.debug(e) + + except Exception as e: + log.error(e, exc_info=True) + raise + + break + + +def batch_execute(requests, retry_cb=None, log_err=log.error): + """execute list or dict as batch requests + retry if retry_cb returns true + """ + BATCH_LIMIT = 1000 + if not isinstance(requests, dict): + requests = {str(k): v for k, v in enumerate(requests)} # rid generated here + done = {} + failed = {} + timestamps: List[float] = [] + rate_limited = False + + def batch_callback(rid, resp, exc): + nonlocal rate_limited + if exc is not None: + log_err(f"compute request exception {rid}: {exc}") + if retry_exception(exc): + rate_limited = True + else: + req = requests.pop(rid) + failed[rid] = (req, exc) + else: + # if retry_cb is set, don't move to done until it returns false + if retry_cb is None or not retry_cb(resp): + requests.pop(rid) + done[rid] = resp + + def batch_request(reqs): + batch = lookup().compute.new_batch_http_request(callback=batch_callback) + for rid, req in reqs: + batch.add(req, request_id=rid) + return batch + + while requests: + if timestamps: + timestamps = [stamp for stamp in timestamps if stamp > time()] + if rate_limited and timestamps: + stamp = next(iter(timestamps)) + sleep(max(stamp - time(), 0)) + rate_limited = False + # up to API_REQ_LIMIT (2000) requests + # in chunks of up to BATCH_LIMIT (1000) + batches = [ + batch_request(chunk) + for chunk in chunked(islice(requests.items(), API_REQ_LIMIT), BATCH_LIMIT) + ] + timestamps.append(time() + 100) + with ThreadPoolExecutor() as exe: + futures = [] + for batch in batches: + future = exe.submit(ensure_execute, batch) + futures.append(future) + for future in futures: + result = future.exception() + if result is not None: + raise result + + return done, failed + + +def get_operation_req(lkp: "Lookup", name: str, region: Optional[str]=None, zone: Optional[str]=None) -> Any: + if zone: + return lkp.compute.zoneOperations().get(project=lkp.project, zone=zone, operation=name) + elif region: + return lkp.compute.regionOperations().get(project=lkp.project, region=region, operation=name) + return lkp.compute.globalOperations().get(project=lkp.project, operation=name) + +def wait_request(operation, project: str): + """makes the appropriate wait request for a given operation""" + if "zone" in operation: + req = lookup().compute.zoneOperations().wait( + project=project, + zone=trim_self_link(operation["zone"]), + operation=operation["name"], + ) + elif "region" in operation: + req = lookup().compute.regionOperations().wait( + project=project, + region=trim_self_link(operation["region"]), + operation=operation["name"], + ) + else: + req = lookup().compute.globalOperations().wait( + project=project, operation=operation["name"] + ) + return req + + +def wait_for_operation(operation) -> Dict[str, Any]: + """wait for given operation""" + project = parse_self_link(operation["selfLink"]).project + wait_req = wait_request(operation, project=project) + + while True: + result = ensure_execute(wait_req) + if result["status"] == "DONE": + log_errors = " with errors" if "error" in result else "" + log.debug( + f"operation complete{log_errors}: type={result['operationType']}, name={result['name']}" + ) + return result + + + +def getThreadsPerCore(template) -> int: + if not template.machine_type.supports_smt: + return 1 + return template.advancedMachineFeatures.threadsPerCore or 2 + + +@retry( + max_retries=9, + init_wait_time=1, + warn_msg="Temporary failure in name resolution", + exc_type=socket.gaierror, +) +def host_lookup(host_name: str) -> str: + return socket.gethostbyname(host_name) + + +class Dumper(yaml.SafeDumper): + """Add representers for pathlib.Path and NSDict for yaml serialization""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.add_representer(NSDict, self.represent_nsdict) + self.add_multi_representer(Path, self.represent_path) + + @staticmethod + def represent_nsdict(dumper, data): + return dumper.represent_mapping("tag:yaml.org,2002:map", data.items()) + + @staticmethod + def represent_path(dumper, path): + return dumper.represent_scalar("tag:yaml.org,2002:str", str(path)) + + +@dataclass(frozen=True) +class ReservationDetails: + project: str + zone: str + name: str + policies: List[str] # names (not URLs) of resource policies + bulk_insert_name: str # name in format suitable for bulk insert (currently identical to user supplied name in long format) + deployment_type: Optional[str] + reservation_mode: Optional[str] + assured_count: int + delete_at_time: Optional[datetime] + + @property + def dense(self) -> bool: + return self.deployment_type == "DENSE" + + @property + def calendar(self) -> bool: + return self.reservation_mode == "CALENDAR" + +@dataclass(frozen=True) +class FutureReservation: + project: str + zone: str + name: str + specific: bool + start_time: datetime + end_time: datetime + reservation_mode: Optional[str] + active_reservation: Optional[ReservationDetails] + + @property + def calendar(self) -> bool: + return self.reservation_mode == "CALENDAR" + +@dataclass +class Job: + id: int + name: Optional[str] = None + required_nodes: Optional[str] = None + job_state: Optional[str] = None + duration: Optional[timedelta] = None + +@dataclass(frozen=True) +class NodeState: + base: str + flags: frozenset + +class Lookup: + """Wrapper class for cached data access""" + + def __init__(self, cfg): + self._cfg = cfg + + @property + def cfg(self): + return self._cfg + + @property + def project(self): + return self.cfg.project or authentication_project() + + @cached_property + def control_addr(self) -> Optional[str]: + return self.cfg.get("slurm_control_addr", None) + + @property + def control_host(self): + return self.cfg.slurm_control_host + + @cached_property + def control_host_addr(self): + return self.control_addr or host_lookup(self.cfg.slurm_control_host) + + @property + def control_host_port(self): + return self.cfg.slurm_control_host_port + + @property + def endpoint_versions(self): + return self.cfg.endpoint_versions + + @property + def scontrol(self): + return Path(self.cfg.slurm_bin_dir or "") / "scontrol" + + @cached_property + def instance_role(self): + return instance_role() + + @cached_property + def instance_role_safe(self): + try: + role = self.instance_role + except Exception as e: + log.error(e) + role = None + return role + + @property + def is_controller(self): + return self.instance_role_safe == "controller" + + @property + def is_login_node(self): + return self.instance_role_safe == "login" + + @cached_property + def compute(self): + # TODO evaluate when we need to use google_app_cred_path + if self.cfg.google_app_cred_path: + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.cfg.google_app_cred_path + return compute_service() + + @cached_property + def hostname(self): + return socket.gethostname() + + @cached_property + def hostname_fqdn(self): + return socket.getfqdn() + + @cached_property + def zone(self): + return instance_metadata("zone") + + node_desc_regex = re.compile( + r"^(?P(?P[^\s\-]+)-(?P\S+))-(?P(?P\w+)|(?P\[[\d,-]+\]))$" + ) + + @lru_cache(maxsize=None) + def _node_desc(self, node_name): + """Get parts from node name""" + if not node_name: + node_name = self.hostname + # workaround below is for VMs whose hostname is FQDN + node_name_short = node_name.split(".")[0] + m = self.node_desc_regex.match(node_name_short) + if not m: + raise Exception(f"node name {node_name} is not valid") + return m.groupdict() + + def node_prefix(self, node_name=None): + return self._node_desc(node_name)["prefix"] + + def node_index(self, node: str) -> int: + """ node_index("cluster-nodeset-45") == 45 """ + suff = self._node_desc(node)["suffix"] + + if suff is None: + raise ValueError(f"Node {node} name does not end with numeric index") + return int(suff) + + def node_nodeset_name(self, node_name=None): + return self._node_desc(node_name)["nodeset"] + + def node_nodeset(self, node_name=None): + nodeset_name = self.node_nodeset_name(node_name) + if nodeset_name in self.cfg.nodeset_tpu: + return self.cfg.nodeset_tpu[nodeset_name] + + return self.cfg.nodeset[nodeset_name] + + def partition_is_tpu(self, part: str) -> bool: + """check if partition with name part contains a nodeset of type tpu""" + return len(self.cfg.partitions[part].partition_nodeset_tpu) > 0 + + + def node_is_tpu(self, node_name=None): + nodeset_name = self.node_nodeset_name(node_name) + return self.cfg.nodeset_tpu.get(nodeset_name) is not None + + def nodeset_is_tpu(self, nodeset_name=None) -> bool: + return self.cfg.nodeset_tpu.get(nodeset_name) is not None + + def node_is_fr(self, node_name:str) -> bool: + return bool(self.node_nodeset(node_name).future_reservation) + + def is_dormant_res_node(self, node_name:str) -> bool: + fr = self.future_reservation(self.node_nodeset(node_name)) + res = self.nodeset_reservation(self.node_nodeset(node_name)) + + if fr is None and res is None: + return False + + if fr: + return fr.active_reservation is None + + if res: + if res.calendar: + # If reservation is calendar based, check if it is past the delete_at_time + if res.delete_at_time is not None and now() >= res.delete_at_time: + log.debug(f"DWS calendar reservation {res.bulk_insert_name} is past deletion time {res.delete_at_time}, skipping resume.") + return True + + # If assured_count is 0 do not resume nodes as they are not active yet + if res.delete_at_time is not None and res.assured_count <= 0: + log.debug(f"DWS calendar reservation {res.bulk_insert_name} is not active yet, skipping resume.") + return True + + return False + + def node_is_dyn(self, node_name=None) -> bool: + nodeset = self.node_nodeset_name(node_name) + return self.cfg.nodeset_dyn.get(nodeset) is not None + + def node_is_gke(self, node_name=None) -> bool: + return self.nodeset_is_gke(self.node_nodeset(node_name)) + + def nodeset_is_gke(self, nodeset=None) -> bool: + return "gke_nodepool" in nodeset + + def node_template(self, node_name=None) -> str: + """ Self link of nodeset template """ + return self.node_nodeset(node_name).instance_template + + def node_template_info(self, node_name=None): + return self.template_info(self.node_template(node_name)) + + def node_region(self, node_name=None): + nodeset = self.node_nodeset(node_name) + return parse_self_link(nodeset.subnetwork).region + + def nodeset_accelerator_topology(self, nodeset_name: str) -> Optional[str]: + if not self.nodeset_is_tpu(nodeset_name): + return getattr(self.cfg.nodeset[nodeset_name], 'accelerator_topology', None) + return None + + def nodeset_prefix(self, nodeset_name): + return f"{self.cfg.slurm_cluster_name}-{nodeset_name}" + + def nodelist_range(self, nodeset_name: str, start: int, count: int) -> str: + assert 0 <= start and 0 < count + pref = self.nodeset_prefix(nodeset_name) + if count == 1: + return f"{pref}-{start}" + return f"{pref}-[{start}-{start + count - 1}]" + + def static_dynamic_sizes(self, nodeset: NSDict) -> Tuple[int, int]: + return (nodeset.node_count_static or 0, nodeset.node_count_dynamic_max or 0) + + def nodelist(self, nodeset) -> str: + cnt = sum(self.static_dynamic_sizes(nodeset)) + if cnt == 0: + return "" + return self.nodelist_range(nodeset.nodeset_name, 0, cnt) + + def nodenames(self, nodeset) -> Tuple[Iterable[str], Iterable[str]]: + pref = self.nodeset_prefix(nodeset.nodeset_name) + s_count, d_count = self.static_dynamic_sizes(nodeset) + return ( + (f"{pref}-{i}" for i in range(s_count)), + (f"{pref}-{i}" for i in range(s_count, s_count + d_count)), + ) + + def power_managed_nodesets(self) -> Iterable[NSDict]: + return chain(self.cfg.nodeset.values(), self.cfg.nodeset_tpu.values()) + + def is_power_managed_node(self, node_name: str) -> bool: + try: + ns = self.node_nodeset(node_name) + if ns is None: + return False + idx = int(self._node_desc(node_name)["suffix"]) + return idx < sum(self.static_dynamic_sizes(ns)) + except Exception: + return False + + def is_static_node(self, node_name: str) -> bool: + if not self.is_power_managed_node(node_name): + return False + idx = int(self._node_desc(node_name)["suffix"]) + return idx < self.node_nodeset(node_name).node_count_static + + @lru_cache(maxsize=None) + def slurm_nodes(self) -> Dict[str, NodeState]: + def parse_line(node_line) -> Tuple[str, NodeState]: + """turn node,state line to (node, NodeState)""" + # state flags include: CLOUD, COMPLETING, DRAIN, FAIL, POWERED_DOWN, + # POWERING_DOWN + node, fullstate = node_line.split(",") + state = fullstate.split("+") + state_tuple = NodeState(base=state[0], flags=frozenset(state[1:])) + return (node, state_tuple) + + cmd = ( + f"{self.scontrol} show nodes | " + r"grep -oP '^NodeName=\K(\S+)|\s+State=\K(\S+)' | " + r"paste -sd',\n'" + ) + node_lines = run(cmd, shell=True).stdout.rstrip().splitlines() + nodes = { + node: state + for node, state in map(parse_line, node_lines) + if "CLOUD" in state.flags or "DYNAMIC_NORM" in state.flags + } + return nodes + + def node_state(self, nodename: str) -> Optional[NodeState]: + state = self.slurm_nodes().get(nodename) + if state is not None: + return state + + # state is None => Slurm doesn't know this node, + # there are two reasons: + # * happy: + # * node belongs to removed nodeset + # * node belongs to downsized portion of nodeset + # * dynamic node that didn't register itself + # * unhappy: + # * there is a drift in Slurm and SlurmGCP configurations + # * `slurm_nodes` function failed to handle `scontrol show nodes`, + # TODO: make `slurm_nodes` robust by using `scontrol show nodes --json` + # In either of "unhappy" cases it's too dangerous to proceed - abort slurmsync. + try: + ns = self.node_nodeset(nodename) + except: + log.info(f"Unknown node {nodename}, belongs to unknown nodeset") + return None # Can't find nodeset, may be belongs to removed nodeset + + if self.node_is_dyn(nodename): + log.info(f"Unknown node {nodename}, belongs to dynamic nodeset") + return None # we can't make any judjment for dynamic nodes + + cnt = sum(self.static_dynamic_sizes(ns)) + if self.node_index(nodename) >= cnt: + log.info(f"Unknown node {nodename}, out of nodeset size boundaries ({cnt})") + return None # node belongs to downsized nodeset + + raise RuntimeError(f"Slurm does not recognize node {nodename}, potential misconfiguration.") + + + @lru_cache(maxsize=1) + def instances(self) -> Dict[str, Instance]: + instance_information_fields = [ + "creationTimestamp", + "name", + "resourceStatus", + "scheduling", + "status", + "labels.slurm_instance_role", + "zone", + "metadata", + ] + + instance_fields = ",".join(sorted(instance_information_fields)) + fields = f"items.zones.instances({instance_fields}),nextPageToken" + flt = f"labels.slurm_cluster_name={self.cfg.slurm_cluster_name} AND name:{self.cfg.slurm_cluster_name}-*" + act = self.compute.instances() + op = act.aggregatedList(project=self.project, fields=fields, filter=flt) + + instances = {} + while op is not None: + result = ensure_execute(op) + for zone in result.get("items", {}).values(): + for jo in zone.get("instances", []): + inst = Instance.from_json(jo) + if inst.name in instances: + log.error(f"Duplicate VM name {inst.name} across multiple zones") + instances[inst.name] = inst + op = act.aggregatedList_next(op, result) + return instances + + def instance(self, instance_name: str) -> Optional[Instance]: + return self.instances().get(instance_name) + + @lru_cache() + def _get_reservation(self, project: str, zone: str, name: str) -> Any: + """See https://cloud.google.com/compute/docs/reference/rest/v1/reservations""" + return self.compute.reservations().get( + project=project, zone=zone, reservation=name).execute() + + @lru_cache() + def get_mig(self, project: str, region: str, self_link:str) -> Any: + """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" + return self.compute.regionInstanceGroupManagers().get(project=project, region=region, instanceGroupManager=self_link).execute() + + @lru_cache + def get_mig_instances(self, project: str, region: str, self_link:str) -> Any: + return self.compute.regionInstanceGroupManagers().listManagedInstances(project=project, region=region, instanceGroupManager=self_link).execute() + + @lru_cache() + def get_mig_list(self, project: str, region: str) -> Any: + """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" + return self.compute.regionInstanceGroupManagers().list(project=project, region=region).execute() + + @lru_cache() + def _get_future_reservation(self, project:str, zone:str, name: str) -> Any: + """See https://cloud.google.com/compute/docs/reference/rest/v1/futureReservations""" + return self.compute.futureReservations().get(project=project, zone=zone, futureReservation=name).execute() + + def get_reservation_details(self, project:str, zone:str, name:str, bulk_insert_name:str) -> ReservationDetails: + reservation = self._get_reservation(project, zone, name) + + # Converts policy URLs to names, e.g.: + # projects/111111/regions/us-central1/resourcePolicies/zebra -> zebra + policies = [u.split("/")[-1] for u in reservation.get("resourcePolicies", {}).values()] + + return ReservationDetails( + project=project, + zone=zone, + name=name, + policies=policies, + deployment_type=reservation.get("deploymentType"), + reservation_mode=reservation.get("reservationMode"), + assured_count=int(reservation.get("specificReservation", {}).get("assuredCount", 0)), + delete_at_time=parse_gcp_timestamp(reservation.get("deleteAtTime")) if reservation.get("deleteAtTime") else None, + bulk_insert_name=bulk_insert_name) + + def nodeset_reservation(self, nodeset: NSDict) -> Optional[ReservationDetails]: + if not nodeset.reservation_name: + return None + + zones = list(nodeset.zone_policy_allow or []) + assert len(zones) == 1, "Only single zone is supported if using a reservation" + zone = zones[0] + + regex = re.compile(r'^projects/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$') + if not (match := regex.match(nodeset.reservation_name)): + raise ValueError( + f"Invalid reservation name: '{nodeset.reservation_name}', expected format is 'projects/PROJECT/reservations/NAME'" + ) + + project, name = match.group("project", "reservation") + return self.get_reservation_details(project, zone, name, nodeset.reservation_name) + + def future_reservation(self, nodeset: NSDict) -> Optional[FutureReservation]: + if not nodeset.future_reservation: + return None + + active_reservation = None + match = re.search(r'^projects/(?P[^/]+)/zones/(?P[^/]+)/futureReservations/(?P[^/]+)(/.*)?$', nodeset.future_reservation) + assert match, f"Invalid future reservation name '{nodeset.future_reservation}'" + project, zone, name = match.group("project","zone","name") + fr = self._get_future_reservation(project,zone,name) + + start_time = parse_gcp_timestamp(fr["timeWindow"]["startTime"]) + end_time = parse_gcp_timestamp(fr["timeWindow"]["endTime"]) + + if "autoCreatedReservations" in fr["status"] and (res:=fr["status"]["autoCreatedReservations"][0]): + if start_time <= now() <=end_time: + match = re.search(r'projects/(?P[^/]+)/zones/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$',res) + assert match, f"Unexpected reservation name '{res}'" + res_name = match.group("name") + bulk_insert_name = f"projects/{project}/reservations/{res_name}" + active_reservation = self.get_reservation_details(project, zone, res_name, bulk_insert_name) + + return FutureReservation( + project=project, + zone=zone, + name=name, + specific=fr["specificReservationRequired"], + start_time=start_time, + end_time=end_time, + reservation_mode=fr.get("reservationMode"), + active_reservation=active_reservation + ) + + @lru_cache(maxsize=1) + def machine_types(self): + field_names = "name,zone,guestCpus,memoryMb,accelerators" + fields = f"items.zones.machineTypes({field_names}),nextPageToken" + + machines: Dict[str, Dict[str, Any]] = defaultdict(dict) + act = self.compute.machineTypes() + op = act.aggregatedList(project=self.project, fields=fields) + while op is not None: + result = ensure_execute(op) + machine_iter = chain.from_iterable( + scope.get("machineTypes", []) for scope in result["items"].values() + ) + for machine in machine_iter: + name = machine["name"] + zone = machine["zone"] + machines[name][zone] = machine + + op = act.aggregatedList_next(op, result) + return machines + + def machine_type(self, name: str) -> MachineType: + custom_patt = re.compile( + r"((?P\w+)-)?custom-(?P\d+)-(?P\d+)" + ) + if match := custom_patt.match(name): + return MachineType( + name=name, + guest_cpus=int(match.group("cpus")), + memory_mb=int(match.group("mem")), + accelerators=[], + ) + + machines = self.machine_types() + if name not in machines: + raise Exception(f"machine type {name} not found") + per_zone = machines[name] + assert per_zone + return MachineType.from_json( + next(iter(per_zone.values())) # pick the first/any zone + ) + + def template_machine_conf(self, template_link): + template = self.template_info(template_link) + machine = template.machine_type + + machine_conf = NSDict() + machine_conf.boards = 1 # No information, assume 1 + machine_conf.sockets = machine.sockets + # the value below for SocketsPerBoard must be type int + machine_conf.sockets_per_board = machine_conf.sockets // machine_conf.boards + machine_conf.threads_per_core = 1 + _div = 2 if getThreadsPerCore(template) == 1 else 1 + machine_conf.cpus = ( + int(machine.guest_cpus / _div) if machine.supports_smt else machine.guest_cpus + ) + machine_conf.cores_per_socket = int(machine_conf.cpus / machine_conf.sockets) + # Because the actual memory on the host will be different than + # what is configured (e.g. kernel will take it). From + # experiments, about 16 MB per GB are used (plus about 400 MB + # buffer for the first couple of GB's. Using 30 MB to be safe. + gb = machine.memory_mb // 1024 + machine_conf.memory = machine.memory_mb - (400 + (30 * gb)) + return machine_conf + + @lru_cache(maxsize=None) + def template_info(self, template_link): + template_name = trim_self_link(template_link) + cache = file_cache.cache("template_cache") + + if cached := cache.get(template_name): + return NSDict(cached) + + region = get_self_link_component(template_link, "regions") + + template = ensure_execute( + self.compute.instanceTemplates().get( + project=self.project, instanceTemplate=template_name + ) if region is None else + self.compute.regionInstanceTemplates().get( + project=self.project, region=region, instanceTemplate=template_name + ) + ).get("properties") + template = NSDict(template) + # name and link are not in properties, so stick them in + template.name = template_name + template.link = template_link + template.machine_type = self.machine_type(template.machineType) + # TODO delete metadata to reduce memory footprint? + # del template.metadata + + template.gpu = get_template_gpu(template) + + cache.set(template_name, template.to_dict()) + return template + + def _parse_job_info(self, job_info: str) -> Job: + """Extract job details""" + if match:= re.search(r"JobId=(\d+)", job_info): + job_id = int(match.group(1)) + else: + raise ValueError(f"Job ID not found in the job info: {job_info}") + + if match:= re.search(r"TimeLimit=(?:(\d+)-)?(\d{2}):(\d{2}):(\d{2})", job_info): + days, hours, minutes, seconds = match.groups() + duration = timedelta( + days=int(days) if days else 0, + hours=int(hours), + minutes=int(minutes), + seconds=int(seconds) + ) + else: + duration = None + + if match := re.search(r"JobName=([^\n]+)", job_info): + name = match.group(1) + else: + name = None + + if match := re.search(r"JobState=(\w+)", job_info): + job_state = match.group(1) + else: + job_state = None + + if match := re.search(r"ReqNodeList=([^ ]+)", job_info): + required_nodes = match.group(1) + else: + required_nodes = None + + return Job(id=job_id, duration=duration, name=name, job_state=job_state, required_nodes=required_nodes) + + @lru_cache + def get_jobs(self) -> List[Job]: + res = run(f"{self.scontrol} show jobs", timeout=30) + + return [self._parse_job_info(job) for job in res.stdout.split("\n\n")[:-1]] + + @lru_cache + def job(self, job_id: int) -> Optional[Job]: + job_info = run(f"{self.scontrol} show jobid {job_id}", check=False).stdout.rstrip() + if not job_info: + return None + + return self._parse_job_info(job_info=job_info) + + @property + def etc_dir(self) -> Path: + return Path(self.cfg.output_dir or slurmdirs.etc) + + def controller_mount_server_ip(self) -> str: + return self.control_addr or self.control_host + + def normalize_ns_mount(self, ns: Union[dict, NSMount]) -> NSMount: + if isinstance(ns, NSMount): + return ns + + server_ip = ns.get("server_ip") or "$controller" + if server_ip == "$controller": + server_ip = self.controller_mount_server_ip() + + return NSMount( + server_ip=server_ip, + local_mount=Path(ns["local_mount"]), + remote_mount=Path(ns["remote_mount"]), + fs_type=ns["fs_type"], + mount_options=ns["mount_options"], + ) + + @property + def munge_mount(self) -> NSMount: + if self.cfg.munge_mount: + mnt = self.cfg.munge_mount + mnt.local_mount = mnt.local_mount or "/mnt/munge" + return self.normalize_ns_mount(mnt) + else: + return NSMount( + server_ip=self.controller_mount_server_ip(), + local_mount=Path("/mnt/munge"), + remote_mount=dirs.munge, + fs_type="nfs", + mount_options="defaults,hard,intr,_netdev", + ) + + @property + def slurm_key_mount(self) -> NSMount: + if self.cfg.slurm_key_mount: + mnt = self.cfg.slurm_key_mount + mnt.local_mount = mnt.local_mount or slurmdirs.key_distribution + return self.normalize_ns_mount(mnt) + else: + return NSMount( + server_ip=self.controller_mount_server_ip(), + local_mount=slurmdirs.key_distribution, + remote_mount=slurmdirs.key_distribution, + fs_type="nfs", + mount_options="defaults,hard,intr,_netdev", + ) + + def is_flex_node(self, node: str) -> bool: + try: + nodeset = self.node_nodeset(node) + if nodeset.dws_flex.use_bulk_insert: + return False #For legacy flex support + return bool(nodeset.dws_flex.enabled) + except: + return False + + def is_provisioning_flex_node(self, node:str) -> bool: + if not self.is_flex_node(node): + return False + if self.instance(node) is not None: + return True + + nodeset = self.node_nodeset(node) + zones = nodeset.zone_policy_allow + assert len(zones) > 0 + region = self.node_region(node) + + potential_migs=[] + mig_list=self.get_mig_list(self.project, region) + + if not mig_list or not mig_list.get("items"): + return False + + for mig in mig_list["items"]: + if not mig.get("instanceTemplate"): #possibly an old MIG + return False + if mig["instanceTemplate"] == self.node_template(node) and mig["currentActions"]["creating"] > 0: + potential_migs.append(self.get_mig_instances(self.project, region, trim_self_link(mig["selfLink"]))) + + if not potential_migs: + return False + + for instance_collection in potential_migs[0]["managedInstances"]: + if node in instance_collection["name"] and instance_collection["currentAction"]=="CREATING": + return True + return False + + def cluster_regions(self) -> list[str]: + """ + Returns all regions used in cluster + NOTE: only concerned with normal nodesets, + neither TPU, nor dynamic, nor login node, nor controller node are considered + """ + res = set() + for nodeset in self.cfg.nodeset.values(): + res.add(parse_self_link(nodeset.subnetwork).region) + return list(res) + + + +_lkp: Optional[Lookup] = None + +def _load_config() -> NSDict: + return NSDict(yaml.safe_load(CONFIG_FILE.read_text())) + +def lookup() -> Lookup: + global _lkp + if _lkp is None: + try: + cfg = _load_config() + except FileNotFoundError: + log.error(f"config file not found: {CONFIG_FILE}") + cfg = NSDict() # TODO: fail here, once all code paths are covered (mainly init_logging) + _lkp = Lookup(cfg) + return _lkp + +def update_config(cfg: NSDict) -> None: + global _lkp + _lkp = Lookup(cfg) + +def scontrol_reconfigure(lkp: Lookup) -> None: + log.info("Running systemctl restart slurmctld.service") + run("sudo systemctl restart slurmctld.service", timeout=30) + log.info("Running scontrol reconfigure") + run(f"{lkp.scontrol} reconfigure") diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py new file mode 100644 index 0000000000..d1d77a1833 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py @@ -0,0 +1,124 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + + +from dataclasses import dataclass, asdict +import util +import local_pubsub + +import logging +log = logging.getLogger() + +# Name of the topic +TOPIC = "watch_delete_vm_op" + +@dataclass(frozen=True) +class WatchDeleteVmOp_Message: + op_name: str + zone: str + node: str + +class WatchDeleteVmOp_Topic: + def __init__(self, topic: local_pubsub.Topic) -> None: + self._t = topic + + def publish(self, op: dict[str, Any], node: str) -> None: + assert op.get("operationType") == "delete" + assert op.get("zone") + assert node + + msg = WatchDeleteVmOp_Message(op_name=op["name"], zone=op["zone"], node=node) + self._t.publish(data=asdict(msg)) + + +def watch_delete_vm_op_topic() -> WatchDeleteVmOp_Topic: + return WatchDeleteVmOp_Topic(local_pubsub.topic(TOPIC)) + + +def _watch_op(lkp: util.Lookup, m: WatchDeleteVmOp_Message) -> bool: + """ + Processes VM delete-operation. + If operation is still running - do nothing + If operation failed - log error & remove op from watch list + If operation is done - remove op from watch list do nothing + + To avoid querying status for each op individually, use list of VM instances as + a source of data. Don't query op for instance X if instance X is not present + (presumably deleted). + NOTE: This optimization can lead to false-positives - + absence of error-logs in case op failed, but VM got deleted by other means. + + Returns True if message should be marked as processed (ack). + """ + + inst = lkp.instance(m.node) + + if not inst: + log.debug(f"Stop watching op {m.op_name}, VM {m.node} appears to be deleted") + return True # ack, potentially false-positive + + if inst.status == "TERMINATED": + log.debug(f"Stop watching op {m.op_name}, VM {m.node} is TERMINATED") + return True # ack, potentially false-positive + + if inst.status == "STOPPING": + log.debug(f"Skipping op {m.op_name}, VM {m.node} is STOPPING") + return False # try later + + try: + op = util.get_operation_req(lkp, m.op_name, zone=m.zone).execute() + except: + # TODO: consider less conservative handling, but be careful not to cause deadlettering. + log.exception(f"Failed to get operation {m.op_name}, will not retry") + return True # ack (remove) + + if op["status"] != "DONE": + log.debug(f"Watching op {m.op_name} is still not done ({op['status']})") + return False # try later + + if "error" in op: + log.error(f"Operation {m.op_name} to delete {m.node} finished with error: {op['error']}") + else: + log.debug(f"Operation {m.op_name} to delete {m.node} successfully finished") + return True # ack + + +def watch_vm_delete_ops(lkp: util.Lookup) -> None: + sub = local_pubsub.subscription(TOPIC) + + # Pull once instead of "pulling until empty", motivation: + # Bulk of cases processed by `_watch_op` relies on freshness of `lkp.instances`, + # `lkp.instances` are fetched once during run of `slurmsync`. + # Therefore we shouldn't try to re-process messages that has been already NACKed in this run, + # since they will be handled with the same `lkp.instance` as a previous attempt. + msgs = sub.pull(max_messages=1000) # 1000 is arbitrary number to be adjusted if needed. + log.debug(f"Processing {len(msgs)} delete VM operations") + # TODO: handle messages in butches to improve latency + for m in msgs: + try: + dm = WatchDeleteVmOp_Message(**m.data) + ack = _watch_op(lkp, dm) + except Exception: + log.exception(f"Failed to process the message {m.id}, removing") + ack = True + if ack: + sub.ack([m.id]) + else: + sub.modify_ack_deadline([m.id], deadline=0) # NACK + + + + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf new file mode 100644 index 0000000000..71905a0342 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf @@ -0,0 +1,504 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "bucket_name" { + description = <<-EOD + Name of GCS bucket to use. + EOD + type = string +} + +variable "bucket_dir" { + description = "Bucket directory for cluster files to be put into." + type = string + default = null +} + +variable "enable_debug_logging" { + type = bool + description = "Enables debug logging mode. Not for production use." + default = false +} + +variable "extra_logging_flags" { + type = map(bool) + description = "The only available flag is `trace_api`" + default = {} +} + +variable "project_id" { + description = "The GCP project ID." + type = string +} + +variable "enable_slurm_auth" { + description = < x... } + nodeset_map = { for k, vs in local.nodeset_map_ell : k => vs[0] } + + nodeset_tpu_map_ell = { for x in var.nodeset_tpu : x.nodeset_name => x... } + nodeset_tpu_map = { for k, vs in local.nodeset_tpu_map_ell : k => vs[0] } + + nodeset_dyn_map_ell = { for x in var.nodeset_dyn : x.nodeset_name => x... } + nodeset_dyn_map = { for k, vs in local.nodeset_dyn_map_ell : k => vs[0] } + + + no_reservation_affinity = { type : "NO_RESERVATION" } +} + +# NODESET +module "slurm_nodeset_template" { + source = "../../internal/slurm-gcp/instance_template" + for_each = local.nodeset_map + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + slurm_instance_role = "compute" + slurm_bucket_path = module.slurm_files.slurm_bucket_path + + additional_disks = each.value.additional_disks + bandwidth_tier = each.value.bandwidth_tier + can_ip_forward = each.value.can_ip_forward + advanced_machine_features = each.value.advanced_machine_features + disk_auto_delete = each.value.disk_auto_delete + disk_labels = each.value.disk_labels + disk_resource_manager_tags = each.value.disk_resource_manager_tags + disk_size_gb = each.value.disk_size_gb + disk_type = each.value.disk_type + enable_confidential_vm = each.value.enable_confidential_vm + enable_oslogin = each.value.enable_oslogin + enable_shielded_vm = each.value.enable_shielded_vm + gpu = each.value.gpu + labels = merge(each.value.labels, { slurm_nodeset = each.value.nodeset_name }) + machine_type = each.value.machine_type + metadata = merge(each.value.metadata, local.universe_domain) + min_cpu_platform = each.value.min_cpu_platform + name_prefix = each.value.nodeset_name + on_host_maintenance = each.value.on_host_maintenance + preemptible = each.value.preemptible + region = each.value.region + resource_manager_tags = each.value.resource_manager_tags + spot = each.value.spot + termination_action = each.value.termination_action + service_account = each.value.service_account + shielded_instance_config = each.value.shielded_instance_config + source_image_family = each.value.source_image_family + source_image_project = each.value.source_image_project + source_image = each.value.source_image + subnetwork = each.value.subnetwork_self_link + additional_networks = each.value.additional_networks + access_config = each.value.access_config + tags = concat([local.slurm_cluster_name], each.value.tags) + + max_run_duration = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? each.value.dws_flex.max_run_duration : null + provisioning_model = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? "FLEX_START" : null + reservation_affinity = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? local.no_reservation_affinity : null +} + +module "nodeset_cleanup" { + source = "./modules/cleanup_compute" + for_each = local.nodeset_map + + nodeset = each.value + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + enable_cleanup_compute = var.enable_cleanup_compute + universe_domain = var.universe_domain + endpoint_versions = var.endpoint_versions + gcloud_path_override = var.gcloud_path_override + nodeset_template = module.slurm_nodeset_template[each.value.nodeset_name].self_link +} + +locals { + nodesets = [for name, ns in local.nodeset_map : { + nodeset_name = ns.nodeset_name + node_conf = ns.node_conf + dws_flex = ns.dws_flex + instance_template = module.slurm_nodeset_template[ns.nodeset_name].self_link + node_count_dynamic_max = ns.node_count_dynamic_max + node_count_static = ns.node_count_static + subnetwork = ns.subnetwork_self_link + reservation_name = ns.reservation_name + future_reservation = ns.future_reservation + maintenance_interval = ns.maintenance_interval + instance_properties_json = ns.instance_properties_json + enable_placement = ns.enable_placement + placement_max_distance = ns.placement_max_distance + network_storage = ns.network_storage + zone_target_shape = ns.zone_target_shape + zone_policy_allow = ns.zone_policy_allow + zone_policy_deny = ns.zone_policy_deny + enable_maintenance_reservation = ns.enable_maintenance_reservation + enable_opportunistic_maintenance = ns.enable_opportunistic_maintenance + accelerator_topology = ns.accelerator_topology + }] +} + +# NODESET TPU +module "slurm_nodeset_tpu" { + source = "../../internal/slurm-gcp/nodeset_tpu" + for_each = local.nodeset_tpu_map + + project_id = var.project_id + node_count_dynamic_max = each.value.node_count_dynamic_max + node_count_static = each.value.node_count_static + nodeset_name = each.value.nodeset_name + zone = each.value.zone + node_type = each.value.node_type + accelerator_config = each.value.accelerator_config + tf_version = each.value.tf_version + preemptible = each.value.preemptible + preserve_tpu = each.value.preserve_tpu + enable_public_ip = each.value.enable_public_ip + service_account = each.value.service_account + data_disks = each.value.data_disks + docker_image = each.value.docker_image + subnetwork = each.value.subnetwork +} + +module "nodeset_cleanup_tpu" { + source = "./modules/cleanup_tpu" + for_each = local.nodeset_tpu_map + + nodeset = { + nodeset_name = each.value.nodeset_name + zone = each.value.zone + } + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + enable_cleanup_compute = var.enable_cleanup_compute + universe_domain = var.universe_domain + endpoint_versions = var.endpoint_versions + gcloud_path_override = var.gcloud_path_override + + depends_on = [ + # Depend on controller network, as a best effort to avoid + # subnetwork resourceInUseByAnotherResource error + var.subnetwork_self_link + ] +} + +resource "google_storage_bucket_object" "parition_config" { + for_each = { for p in var.partitions : p.partition_name => p } + + bucket = module.slurm_files.bucket_name + name = "${module.slurm_files.bucket_dir}/partition_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +moved { + from = module.slurm_files.google_storage_bucket_object.parition_config + to = google_storage_bucket_object.parition_config +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf new file mode 100644 index 0000000000..218c36e392 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf @@ -0,0 +1,191 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# BUCKET + +locals { + synt_suffix = substr(md5("${local.controller_project_id}${var.deployment_name}"), 0, 5) + synth_bucket_name = "${local.slurm_cluster_name}${local.synt_suffix}" + + bucket_name = var.create_bucket ? module.bucket[0].name : var.bucket_name +} + +module "bucket" { + source = "terraform-google-modules/cloud-storage/google" + version = ">= 6.1" + + count = var.create_bucket ? 1 : 0 + + location = var.region + names = [local.synth_bucket_name] + prefix = "slurm" + project_id = local.controller_project_id + + force_destroy = { + (local.synth_bucket_name) = true + } + + labels = merge(local.labels, { + slurm_cluster_name = local.slurm_cluster_name + }) +} + +# BUCKET IAMs +locals { + compute_sa = toset(flatten([for x in module.slurm_nodeset_template : x.service_account])) + compute_tpu_sa = toset(flatten([for x in module.slurm_nodeset_tpu : x.service_account])) + login_sa = toset(flatten([for x in module.login : x.service_account])) + + viewers = toset(flatten([ + "serviceAccount:${module.slurm_controller_template.service_account.email}", + formatlist("serviceAccount:%s", [for x in local.compute_sa : x.email]), + formatlist("serviceAccount:%s", [for x in local.compute_tpu_sa : x.email if x.email != null]), + formatlist("serviceAccount:%s", [for x in local.login_sa : x.email]), + ])) +} + + +resource "google_storage_bucket_iam_member" "viewers" { + for_each = local.viewers + bucket = local.bucket_name + role = "roles/storage.objectViewer" + member = each.value +} + +resource "google_storage_bucket_iam_member" "legacy_readers" { + for_each = local.viewers + bucket = local.bucket_name + role = "roles/storage.legacyBucketReader" + member = each.value +} + +locals { + daos_ns = [ + for ns in var.network_storage : + ns if ns.fs_type == "daos" + ] + + daos_client_install_runners = [ + for ns in local.daos_ns : + ns.client_install_runner if ns.client_install_runner != null + ] + + daos_mount_runners = [ + for ns in local.daos_ns : + ns.mount_runner if ns.mount_runner != null + ] + + daos_network_storage_runners = concat( + local.daos_client_install_runners, + local.daos_mount_runners, + ) + + daos_install_mount_script = { + filename = "ghpc_daos_mount.sh" + content = length(local.daos_ns) > 0 ? module.daos_network_storage_scripts[0].startup_script : "" + } + + common_scripts = length(local.daos_ns) > 0 ? [local.daos_install_mount_script] : [] +} + +# SLURM FILES +locals { + ghpc_startup_script_controller = concat( + local.common_scripts, + [{ + filename = "ghpc_startup.sh" + content = var.controller_startup_script + }]) + + controller_state_disk = { + device_name : try(google_compute_disk.controller_disk[0].name, null) + } + + + nodeset_startup_scripts = { for k, v in local.nodeset_map : k => concat(local.common_scripts, v.startup_script) } +} + +module "daos_network_storage_scripts" { + count = length(local.daos_ns) > 0 ? 1 : 0 + + source = "../../../../modules/scripts/startup-script" + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.daos_network_storage_runners +} + +module "slurm_files" { + source = "./modules/slurm_files" + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + bucket_dir = var.bucket_dir + bucket_name = local.bucket_name + controller_network_attachment = var.controller_network_attachment + + slurmdbd_conf_tpl = var.slurmdbd_conf_tpl + slurm_conf_tpl = var.slurm_conf_tpl + slurm_conf_template = var.slurm_conf_template + cgroup_conf_tpl = var.cgroup_conf_tpl + cloud_parameters = var.cloud_parameters + cloudsql_secret = try( + one(google_secret_manager_secret_version.cloudsql_version[*].id), + null) + + controller_startup_scripts = local.ghpc_startup_script_controller + controller_startup_scripts_timeout = var.controller_startup_scripts_timeout + nodeset_startup_scripts = local.nodeset_startup_scripts + compute_startup_scripts_timeout = var.compute_startup_scripts_timeout + controller_state_disk = local.controller_state_disk + + enable_debug_logging = var.enable_debug_logging + extra_logging_flags = var.extra_logging_flags + + enable_slurm_auth = var.enable_slurm_auth + + enable_bigquery_load = var.enable_bigquery_load + enable_external_prolog_epilog = var.enable_external_prolog_epilog + enable_chs_gpu_health_check_prolog = var.enable_chs_gpu_health_check_prolog + enable_chs_gpu_health_check_epilog = var.enable_chs_gpu_health_check_epilog + epilog_scripts = var.epilog_scripts + prolog_scripts = var.prolog_scripts + task_epilog_scripts = var.task_epilog_scripts + task_prolog_scripts = var.task_prolog_scripts + + disable_default_mounts = !var.enable_default_mounts + network_storage = [ + for storage in var.network_storage : { + server_ip = storage.server_ip, + remote_mount = storage.remote_mount, + local_mount = storage.local_mount, + fs_type = storage.fs_type, + mount_options = storage.mount_options + } + if storage.fs_type != "daos" + ] + + nodeset = local.nodesets + nodeset_dyn = values(local.nodeset_dyn_map) + # Use legacy format for now + nodeset_tpu = values(module.slurm_nodeset_tpu)[*] + + + depends_on = [module.bucket] + + # Providers + endpoint_versions = var.endpoint_versions +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf new file mode 100644 index 0000000000..85ad10fa21 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf @@ -0,0 +1,814 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########### +# GENERAL # +########### + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "deployment_name" { + description = "Name of the deployment." + type = string +} + +variable "slurm_cluster_name" { + type = string + description = <<-EOD + Cluster name, used for resource naming and slurm accounting. + If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). + EOD + default = null + + validation { + condition = var.slurm_cluster_name == null || can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) + error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." + } +} + +variable "region" { + type = string + description = "The default region to place resources in." +} + +variable "zone" { + type = string + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 

list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disable\_login\_public\_ips](#input\_disable\_login\_public\_ips) | DEPRECATED: Use `enable_login_public_ips` instead. | `bool` | `null` | no | +| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_login\_public\_ips](#input\_enable\_login\_public\_ips) | If set to true. The login node will have a random public IP assigned to it. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for login nodes. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Unique name prefix for login nodes. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all login groups. | `string` | n/a | yes | +| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the login instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the login instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [login\_nodes](#output\_login\_nodes) | Slurm login instance definition. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf new file mode 100644 index 0000000000..6ebe5902dc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf @@ -0,0 +1,115 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-login", ghpc_role = "scheduler" }) +} + +module "instance_validation" { + source = "../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata + ) + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.disk_resource_manager_tags + } + ] + + public_access_config = [{ nat_ip = null, network_tier = null }] + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + # lower, replace `_` with `-`, and remove any non-alphanumeric characters + group_name = replace( + replace( + lower(var.name_prefix), + "_", "-"), + "/[^-a-z0-9]/", "") + + + login_node = { + group_name = local.group_name + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(var.disk_labels, local.labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + additional_networks = var.additional_networks + + can_ip_forward = var.can_ip_forward + advanced_machine_features = var.advanced_machine_features + + enable_confidential_vm = var.enable_confidential_vm + access_config = var.enable_login_public_ips ? local.public_access_config : [] + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + gpu = one(local.guest_accelerator) + labels = local.labels + machine_type = var.machine_type + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + num_instances = var.num_instances + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + region = var.region + resource_manager_tags = var.resource_manager_tags + zone = var.zone + + service_account = local.service_account + + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + static_ips = var.static_ips + bandwidth_tier = var.bandwidth_tier + + subnetwork = var.subnetwork_self_link + tags = var.tags + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml new file mode 100644 index 0000000000..47f003258e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name_prefix + has_to_be_used: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf new file mode 100644 index 0000000000..e700542794 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "login_nodes" { + description = "Slurm login instance definition." + value = [local.login_node] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf new file mode 100644 index 0000000000..7c1a2e06b5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf @@ -0,0 +1,419 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +variable "project_id" { # tflint-ignore: terraform_unused_declarations + type = string + description = "Project ID to create resources in." +} + +variable "region" { + type = string + description = "Region where the instances should be created." + default = null +} + +variable "zone" { + type = string + description = <<-EOD + Zone where the instances should be created. If not specified, instances will be + spread across available zones in the region. + EOD + default = null +} + +variable "name_prefix" { + type = string + description = <<-EOD + Unique name prefix for login nodes. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all login groups. + EOD +} + +variable "num_instances" { + type = number + description = "Number of instances to create. This value is ignored if static_ips is provided." + default = 1 +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} +} + +variable "disk_type" { + type = string + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + default = "pd-ssd" +} + +variable "disk_size_gb" { + type = number + description = "Boot disk size in GB." + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + type = list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string)) + auto_delete = optional(bool) + boot = optional(bool) + disk_resource_manager_tags = optional(map(string)) + })) + description = "List of maps of disks." + default = [] +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })) + nullable = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "disable_smt" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + type = bool + default = null + validation { + condition = var.disable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "static_ips" { + type = list(string) + description = "List of static IPs for VM instances." + default = [] +} + +variable "bandwidth_tier" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 6.16 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.16 | +| [helm](#provider\_helm) | ~> 2.17 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.cert_manager](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.prometheus](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.slurm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.slurm_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cert\_manager\_chart\_version](#input\_cert\_manager\_chart\_version) | Version of the Cert Manager chart to install. | `string` | `"v1.18.2"` | no | +| [cert\_manager\_values](#input\_cert\_manager\_values) | Value overrides for the Cert Manager release | `any` | 
{
  "crds": {
    "enabled": true
  }
}
| no | +| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | +| [install\_kube\_prometheus\_stack](#input\_install\_kube\_prometheus\_stack) | Install the Kube Prometheus Stack. | `bool` | `false` | no | +| [install\_slurm\_chart](#input\_install\_slurm\_chart) | Install slurm-operator chart. | `bool` | `true` | no | +| [install\_slurm\_operator\_chart](#input\_install\_slurm\_operator\_chart) | Install slurm-operator chart. | `bool` | `true` | no | +| [node\_pool\_names](#input\_node\_pool\_names) | Names of node pools, for use in node affinities (Slinky system components). | `list(string)` | `null` | no | +| [project\_id](#input\_project\_id) | The project ID that hosts the GKE cluster. | `string` | n/a | yes | +| [prometheus\_chart\_version](#input\_prometheus\_chart\_version) | Version of the Kube Prometheus Stack chart to install. | `string` | `"77.0.1"` | no | +| [prometheus\_values](#input\_prometheus\_values) | Value overrides for the Prometheus release | `any` | 
{
  "installCRDs": true
}
| no | +| [slurm\_chart\_version](#input\_slurm\_chart\_version) | Version of the Slurm chart to install. | `string` | `"0.3.1"` | no | +| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | +| [slurm\_operator\_chart\_version](#input\_slurm\_operator\_chart\_version) | Version of the Slurm Operator chart to install. | `string` | `"0.3.1"` | no | +| [slurm\_operator\_namespace](#input\_slurm\_operator\_namespace) | slurm namespace for charts | `string` | `"slinky"` | no | +| [slurm\_operator\_repository](#input\_slurm\_operator\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | +| [slurm\_operator\_values](#input\_slurm\_operator\_values) | Value overrides for the Slinky release | `any` | `{}` | no | +| [slurm\_repository](#input\_slurm\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | +| [slurm\_values](#input\_slurm\_values) | Value overrides for the Slurm release | `any` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [slurm\_namespace](#output\_slurm\_namespace) | namespace for the slurm chart | +| [slurm\_operator\_namespace](#output\_slurm\_operator\_namespace) | namespace for the slinky operator chart | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/main.tf new file mode 100644 index 0000000000..aff33b73a0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/main.tf @@ -0,0 +1,197 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] + project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] + + # Define affinity settings when node pools are specified + node_pool_affinity = var.node_pool_names != null ? { + nodeAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [{ + matchExpressions = [{ + key = "cloud.google.com/gke-nodepool" + operator = "In" + values = var.node_pool_names + }] + }] + } + } + } : {} +} + +data "google_client_config" "default" {} + +data "google_container_cluster" "gke_cluster" { + project = local.project_id + name = local.cluster_name + location = local.cluster_location +} + +resource "helm_release" "cert_manager" { + name = "cert-manager" + chart = "cert-manager" + repository = "https://charts.jetstack.io" + version = var.cert_manager_chart_version + namespace = "cert-manager" + create_namespace = true + + values = concat( + [yamlencode({ + affinity = local.node_pool_affinity + webhook = { + affinity = local.node_pool_affinity + } + cainjector = { + affinity = local.node_pool_affinity + } + startupapicheck = { + affinity = local.node_pool_affinity + } + })], + [yamlencode(var.cert_manager_values)] + ) +} + +resource "helm_release" "slurm_operator" { + count = var.install_slurm_operator_chart ? 1 : 0 + name = "slurm-operator" + chart = "slurm-operator" + repository = var.slurm_operator_repository + version = var.slurm_operator_chart_version + namespace = var.slurm_operator_namespace + create_namespace = true + + # The Cert Manager webhook deployment must be running to provision the Operator + depends_on = [ + helm_release.cert_manager + ] + + values = concat( + [yamlencode({ + operator = { + affinity = local.node_pool_affinity + } + webhook = { + affinity = local.node_pool_affinity + } + })], + [yamlencode(var.slurm_operator_values)] + ) +} + +resource "helm_release" "slurm" { + count = var.install_slurm_chart ? 1 : 0 + name = "slurm" + chart = "slurm" + repository = var.slurm_repository + version = var.slurm_chart_version + namespace = var.slurm_namespace + create_namespace = true + + # The Slurm Operator must be running to provision Slurm clusters/nodesets + depends_on = [ + helm_release.slurm_operator + ] + + values = concat( + [yamlencode({ + controller = { + affinity = local.node_pool_affinity + } + accounting = { + affinity = local.node_pool_affinity + } + mariadb = { + primary = { + affinity = local.node_pool_affinity + } + secondary = { + affinity = local.node_pool_affinity + } + } + restapi = { + affinity = local.node_pool_affinity + } + slurm-exporter = { + exporter = { + affinity = local.node_pool_affinity + } + } + })], + [yamlencode(var.slurm_values)] + ) +} + +resource "helm_release" "prometheus" { + count = var.install_kube_prometheus_stack ? 1 : 0 + name = "prometheus" + chart = "kube-prometheus-stack" + repository = "https://prometheus-community.github.io/helm-charts" + version = var.prometheus_chart_version + namespace = "prometheus" + create_namespace = true + + values = concat( + [yamlencode({ + crds = { + upgradeJob = { + affinity = local.node_pool_affinity + } + } + alertmanager = { + alertmanagerSpec = { + affinity = local.node_pool_affinity + } + } + prometheusOperator = { + admissionWebhooks = { + deployment = { + affinity = local.node_pool_affinity + } + patch = { + affinity = local.node_pool_affinity + } + } + affinity = local.node_pool_affinity + } + prometheus = { + prometheusSpec = { + affinity = local.node_pool_affinity + } + } + thanosRuler = { + thanosRulerSpec = { + affinity = local.node_pool_affinity + } + } + kube-state-metrics = { + affinity = local.node_pool_affinity + } + grafana = { + affinity = local.node_pool_affinity + imageRenderer = { + affinity = local.node_pool_affinity + } + } + prometheus-windows-exporter = { + affinity = local.node_pool_affinity + } + })], + [yamlencode(var.prometheus_values)] + ) +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/metadata.yaml new file mode 100644 index 0000000000..e18197e2b7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/outputs.tf new file mode 100644 index 0000000000..8ea6385905 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/outputs.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "slurm_namespace" { + description = "namespace for the slurm chart" + value = var.slurm_namespace +} + +output "slurm_operator_namespace" { + description = "namespace for the slinky operator chart" + value = var.slurm_operator_namespace +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/providers.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/providers.tf new file mode 100644 index 0000000000..313d6dc58e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/providers.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +provider "helm" { + kubernetes { + host = "https://${data.google_container_cluster.gke_cluster.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode( + data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate, + ) + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/variables.tf new file mode 100644 index 0000000000..8acaf78562 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/variables.tf @@ -0,0 +1,127 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The project ID that hosts the GKE cluster." + type = string +} + +variable "cluster_id" { + description = "An identifier for the GKE cluster resource with format projects//locations//clusters/." + type = string + nullable = false +} + +variable "node_pool_names" { + description = "Names of node pools, for use in node affinities (Slinky system components)." + type = list(string) + default = null +} + +variable "cert_manager_chart_version" { + description = "Version of the Cert Manager chart to install." + type = string + default = "v1.18.2" +} + +variable "cert_manager_values" { + description = "Value overrides for the Cert Manager release" + type = any + default = { + crds = { + enabled = true + } + } +} + +variable "slurm_operator_chart_version" { + description = "Version of the Slurm Operator chart to install." + type = string + default = "0.3.1" +} + +variable "slurm_operator_values" { + description = "Value overrides for the Slinky release" + type = any + default = {} +} + +variable "slurm_chart_version" { + description = "Version of the Slurm chart to install." + type = string + default = "0.3.1" +} + +variable "slurm_values" { + description = "Value overrides for the Slurm release" + type = any + default = {} +} + +variable "install_kube_prometheus_stack" { + # Components detailed at https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack + description = "Install the Kube Prometheus Stack." + type = bool + default = false +} + +variable "prometheus_chart_version" { + description = "Version of the Kube Prometheus Stack chart to install." + type = string + default = "77.0.1" +} + +variable "prometheus_values" { + description = "Value overrides for the Prometheus release" + type = any + default = { + installCRDs = true + } +} + +variable "slurm_namespace" { + description = "slurm namespace for charts" + type = string + default = "slurm" +} + +variable "slurm_operator_namespace" { + description = "slurm namespace for charts" + type = string + default = "slinky" +} + +variable "install_slurm_chart" { + description = "Install slurm-operator chart." + type = bool + default = true +} + +variable "install_slurm_operator_chart" { + description = "Install slurm-operator chart." + type = bool + default = true +} + +variable "slurm_repository" { + description = "Value overrides for the Slinky release" + type = string + default = "oci://ghcr.io/slinkyproject/charts" +} + +variable "slurm_operator_repository" { + description = "Value overrides for the Slinky release" + type = string + default = "oci://ghcr.io/slinkyproject/charts" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/versions.tf new file mode 100644 index 0000000000..ae4327aeef --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/versions.tf @@ -0,0 +1,28 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + google = { + source = "hashicorp/google" + version = ">= 6.16" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/README.md new file mode 100644 index 0000000000..71a862fd6c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/README.md @@ -0,0 +1,149 @@ +## Description + +This module creates a Toolkit runner that will install HTCondor on RedHat 7 or +8 and its derivative operating systems. These include the CentOS 7 and Rocky +Linux 8 releases of the [HPC VM Image][hpcvmimage]. It may also function on +RedHat 9 and derivatives, however it is not yet supported. Please report any +[issues] on these 3 distributions or open a [discussion] to request support on +Debian or Ubuntu distributions. + +[issues]: https://github.com/GoogleCloudPlatform/hpc-toolkit/issues +[discussion]: https://github.com/GoogleCloudPlatform/hpc-toolkit/discussions + +It also exports a list of Google Cloud APIs which must be enabled prior to +provisioning an HTCondor Pool. + +It is expected to be used with the [htcondor-setup] and +[htcondor-execute-point] modules. + +[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[htcondor-setup]: ../../scheduler/htcondor-setup/README.md +[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md + +### Example + +The following code snippet uses this module to create startup scripts that +install the HTCondor software into a custom VM image. + +```yaml +deployment_groups: +- group: primary + modules: + - id: network1 + source: modules/network/vpc + outputs: + - network_name + + - id: htcondor_install + source: community/modules/scripts/htcondor-install + + - id: htcondor_install_script + source: modules/scripts/startup-script + use: + - htcondor_install + +- group: packer + modules: + - id: custom-image + source: modules/packer/custom-image + kind: packer + use: + - network1 + - htcondor_install_script + settings: + disk_size: 50 + source_image_family: hpc-rocky-linux-8 + image_family: "htcondor-10x" +``` + +A full example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +## Important note + +All POSIX users and HTCondor jobs can act as the service account attached to +VMs within the pool. This enables the use of IAM restrictions via service +accounts but also allows users to access services to which system daemons need +access (e.g. to create Cloud Logging entries). If this is undesirable, one can +restrict access to the instance metadata server to the `root` and `condor` +users. This will allow system services to use the service account, but not +other POSIX users or HTCondor jobs. The firewall example below is appropriate +for CentOS 7. + +```shell +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 1 \ + -m owner --uid-owner root -p tcp -d metadata.google.internal --dport 80 -j ACCEPT +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 2 \ + -m owner --uid-owner condor -p tcp -d metadata.google.internal --dport 80 -j ACCEPT +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 3 \ + -p tcp -d metadata.google.internal --dport 80 -j DROP +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 4 \ + -p tcp -d metadata.google.internal --dport 8080 -j DROP +firewall-cmd --permanent --zone=public --add-port=9618/tcp +firewall-cmd --reload +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [condor\_version](#input\_condor\_version) | Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: "23.0.0","23.*")) | `string` | `"23.*"` | no | +| [enable\_docker](#input\_enable\_docker) | Install and enable docker daemon alongside HTCondor | `bool` | `true` | no | +| [http\_proxy](#input\_http\_proxy) | Set system default web (http and https) proxy for Windows HTCondor installation | `string` | `""` | no | +| [python\_windows\_installer\_url](#input\_python\_windows\_installer\_url) | URL of Python installer for Windows | `string` | `"https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcp\_service\_list](#output\_gcp\_service\_list) | Google Cloud APIs required by HTCondor | +| [runners](#output\_runners) | Runner to install HTCondor using startup-scripts | +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | Windows PowerShell script to install HTCondor | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py new file mode 100644 index 0000000000..77bafa0310 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py @@ -0,0 +1,417 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script for resizing managed instance group (MIG) cluster size based +# on the number of jobs in the Condor Queue. + +from absl import app +from absl import flags +from collections import OrderedDict +from datetime import datetime +from pprint import pprint +from googleapiclient import discovery +from oauth2client.client import GoogleCredentials + +import argparse +import os +import math +import time +import htcondor +import classad + +parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument("--p", required=True, help="Project id", type=str) +parser.add_argument( + "--z", + required=True, + help="Name of GCP zone where the managed instance group is located", + type=str, +) +parser.add_argument( + "--r", + required=True, + help="Name of GCP region where the managed instance group is located", + type=str, +) +parser.add_argument( + "--mz", + required=False, + help="Enabled multizone (regional) managed instance group", + action="store_true", +) +parser.add_argument( + "--g", required=True, help="Name of the managed instance group", type=str +) +parser.add_argument( + "--i", + default=0, + help="Minimum number of idle compute instances", + type=int +) +parser.add_argument( + "--c", required=True, help="Maximum number of compute instances", type=int +) +parser.add_argument( + "--v", + default=0, + help="Increase output verbosity. 1-show basic debug info. 2-show detail debug info", + type=int, + choices=[0, 1, 2], +) +parser.add_argument( + "--d", + default=0, + help="Dry Run, default=0, if 1, then no scaling actions", + type=int, + choices=[0, 1], +) + +args = parser.parse_args() + +class AutoScaler: + def __init__(self, multizone=False): + + self.multizone = multizone + # Obtain credentials + self.credentials = GoogleCredentials.get_application_default() + self.service = discovery.build("compute", "v1", credentials=self.credentials) + + if self.multizone: + self.instanceGroupManagers = self.service.regionInstanceGroupManagers() + else: + self.instanceGroupManagers = self.service.instanceGroupManagers() + + # Remove specified instances from MIG and decrease MIG size + def deleteFromMig(self, node_self_links): + requestDelInstance = self.instanceGroupManagers.deleteInstances( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + body={ "instances": node_self_links }, + ) + + # execute if not a dry-run + if not self.dryrun: + response = requestDelInstance.execute() + if self.debug > 0: + pprint(response) + return response + return "Dry Run" + + def getInstanceTemplateInfo(self): + requestTemplateName = self.instanceGroupManagers.get( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + fields="instanceTemplate", + ) + responseTemplateName = requestTemplateName.execute() + template_name = "" + + if self.debug > 1: + print("Request for the template name") + pprint(responseTemplateName) + + if len(responseTemplateName) > 0: + template_url = responseTemplateName.get("instanceTemplate") + template_url_partitioned = template_url.split("/") + template_name = template_url_partitioned[len(template_url_partitioned) - 1] + + requestInstanceTemplate = self.service.instanceTemplates().get( + project=self.project, instanceTemplate=template_name, fields="properties" + ) + responseInstanceTemplateInfo = requestInstanceTemplate.execute() + + if self.debug > 1: + print("Template information") + pprint(responseInstanceTemplateInfo["properties"]) + + machine_type = responseInstanceTemplateInfo["properties"]["machineType"] + is_spot = responseInstanceTemplateInfo["properties"]["scheduling"][ + "preemptible" + ] + if self.debug > 0: + print("Machine Type: " + machine_type) + print("Is spot: " + str(is_spot)) + request = self.service.machineTypes().get( + project=self.project, zone=self.zone, machineType=machine_type + ) + response = request.execute() + guest_cpus = response["guestCpus"] + if self.debug > 1: + print("Machine information") + pprint(responseInstanceTemplateInfo["properties"]) + if self.debug > 0: + print("Guest CPUs: " + str(guest_cpus)) + + instanceTemplateInfo = { + "machine_type": machine_type, + "is_spot": is_spot, + "guest_cpus": guest_cpus, + } + return instanceTemplateInfo + + def scale(self): + # diagnosis + if self.debug > 1: + print("Launching autoscaler.py with the following arguments:") + print("project_id: " + self.project) + print("zone: " + self.zone) + print("region: " + self.region) + print(f"multizone: {self.multizone}") + print("group_manager: " + self.instance_group_manager) + print("computeinstancelimit: " + str(self.compute_instance_limit)) + print("debuglevel: " + str(self.debug)) + + if self.multizone: + self.zoneargs = {"region": self.region} + else: + self.zoneargs = {"zone": self.zone} + + # Each HTCondor scheduler (SchedD), maintains a list of jobs under its + # stewardship. A full list of Job ClassAd attributes can be found at + # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html + schedd = htcondor.Schedd() + # encourage the job queue to start a new negotiation cycle; there are + # internal unconfigurable rate limits so not guaranteed; this is not + # strictly required for success, but may reduce latency of autoscaling + schedd.reschedule() + REQUEST_CPUS_ATTRIBUTE = "RequestCpus" + REQUEST_GPUS_ATTRIBUTE = "RequestGpus" + REQUEST_MEMORY_ATTRIBUTE = "RequestMemory" + job_attributes = [ + REQUEST_CPUS_ATTRIBUTE, + REQUEST_GPUS_ATTRIBUTE, + REQUEST_MEMORY_ATTRIBUTE, + ] + + instanceTemplateInfo = self.getInstanceTemplateInfo() + self.is_spot = instanceTemplateInfo["is_spot"] + self.cores_per_node = instanceTemplateInfo["guest_cpus"] + print(f"MIG is configured for Spot pricing: {self.is_spot}") + print("Number of CPU per compute node: " + str(self.cores_per_node)) + + # this query will constrain the search for jobs to those that either + # require spot VMs or do not require Spot VMs based on whether the + # VM instance template is configured for Spot pricing + spot_query = classad.ExprTree(f"RequireId == \"{self.instance_group_manager}\"") + + # For purpose of scaling a Managed Instance Group, count only jobs that + # are idle and likely participated in a negotiation cycle (there does + # not appear to be a single classad attribute for this). + # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html#JobStatus + LAST_CYCLE_ATTRIBUTE = "LastNegotiationCycleTime0" + coll = htcondor.Collector() + negotiator_ad = coll.query(htcondor.AdTypes.Negotiator, projection=[LAST_CYCLE_ATTRIBUTE]) + if len(negotiator_ad) != 1: + print(f"There should be exactly 1 negotiator in the pool. There is {len(negotiator_ad)}") + exit() + last_negotiation_cycle_time = negotiator_ad[0].get(LAST_CYCLE_ATTRIBUTE) + if not last_negotiation_cycle_time: + print(f"The negotiator has not yet started a match cycle. Exiting auto-scaling.") + exit() + + print(f"Last negotiation cycle occurred at: {datetime.fromtimestamp(last_negotiation_cycle_time)}") + idle_job_query = classad.ExprTree(f"JobStatus == 1 && QDate < {last_negotiation_cycle_time}") + idle_job_ads = schedd.query(constraint=idle_job_query.and_(spot_query), + projection=job_attributes) + + total_idle_request_cpus = sum(j[REQUEST_CPUS_ATTRIBUTE] for j in idle_job_ads) + print(f"Total CPUs requested by idle jobs: {total_idle_request_cpus}") + + if self.debug > 1: + print("Information about the compute instance template") + pprint(instanceTemplateInfo) + + # Calculate the minimum number of instances that, for fully packed + # execute points, could satisfy current job queue + min_hosts_for_idle_jobs = math.ceil(total_idle_request_cpus / self.cores_per_node) + if self.debug > 0: + print(f"Minimum hosts needed: {total_idle_request_cpus} / {self.cores_per_node} = {min_hosts_for_idle_jobs}") + + # Get current number of instances in the MIG + requestGroupInfo = self.instanceGroupManagers.get( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + ) + responseGroupInfo = requestGroupInfo.execute() + current_target = responseGroupInfo["targetSize"] + print(f"Current MIG target size: {current_target}") + + # Find instances that are being modified by the MIG (currentAction is + # any value other than "NONE"). A common reason an instance is modified + # is it because it has failed a health check. + reqModifyingInstances = self.instanceGroupManagers.listManagedInstances( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + filter="currentAction != \"NONE\"", + orderBy="creationTimestamp desc" + ) + respModifyingInstances = reqModifyingInstances.execute() + + # Find VMs that are idle (no dynamic slots created from partitionable + # slots) in the MIG handled by this autoscaler + filter_idle_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots==0") + filter_claimed_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots>0") + filter_mig = classad.ExprTree(f"regexp(\".*/{self.instance_group_manager}$\", CloudCreatedBy)") + # A full list of Machine (StartD) ClassAd attributes can be found at + # https://htcondor.readthedocs.io/en/latest/classad-attributes/machine-classad-attributes.html + idle_node_ads = coll.query(htcondor.AdTypes.Startd, + constraint=filter_idle_vms.and_(filter_mig), + projection=["Machine", "CloudZone"]) + + NODENAME_ATTRIBUTE = "Machine" + claimed_node_ads = coll.query(htcondor.AdTypes.Startd, + constraint=filter_claimed_vms.and_(filter_mig), + projection=[NODENAME_ATTRIBUTE]) + claimed_nodes = [ ad[NODENAME_ATTRIBUTE].split(".")[0] for ad in claimed_node_ads] + + # treat OrderedDict as a set by ignoring key values; this set will + # contain VMs we would consider deleting, in inverse order of + # their readiness to join pool (creating, unhealthy, healthy+idle) + idle_nodes = OrderedDict() + try: + modifyingInstances = respModifyingInstances["managedInstances"] + except KeyError: + modifyingInstances = [] + + print(f"There are {len(modifyingInstances)} VMs being modified by the managed instance group") + + # there is potential for nodes in MIG health check "VERIFYING" state + # to have already joined the pool and be running jobs + for instance in modifyingInstances: + self_link = instance["instance"] + node_name = self_link.rsplit("/", 1)[-1] + if node_name not in claimed_nodes: + idle_nodes[self_link] = "modifying" + + for ad in idle_node_ads: + node = ad["Machine"].split(".")[0] + zone = ad["CloudZone"] + self_link = "https://www.googleapis.com/compute/v1/projects/" + \ + self.project + "/zones/" + zone + "/instances/" + node + # there is potential for nodes in MIG health check "VERIFYING" state + # to have already joined the pool and be idle; delete them last + if self_link in idle_nodes: + idle_nodes.move_to_end(self_link) + idle_nodes[self_link] = "idle" + n_idle = len(idle_nodes) + + print(f"There are {n_idle} VMs being modified or idle in the pool") + if self.debug > 1: + print("Listing idle nodes:") + pprint(idle_nodes) + + # always keep size tending toward the minimum idle VMs requested + new_target = current_target + self.compute_instance_min_idle - n_idle + min_hosts_for_idle_jobs + if new_target > self.compute_instance_limit: + self.size = self.compute_instance_limit + print(f"MIG target size will be limited by {self.compute_instance_limit}") + else: + self.size = new_target + + print(f"New MIG target size: {self.size}") + + if self.debug > 1: + print("MIG Information:") + print(responseGroupInfo) + + if self.size == current_target: + if current_target == 0: + print("Queue is empty") + print("Running correct number of VMs to handle queue") + exit() + + if self.size < current_target: + print("Scaling down. Looking for nodes that can be shut down") + + if self.debug > 1: + print("Compute node busy status:") + for node in idle_nodes: + print(node) + + # Shut down idle nodes up to our calculated limit + nodes_to_delete = list(idle_nodes.keys())[0:current_target-self.size] + for node in nodes_to_delete: + print(f"Attempting to delete: {node.rsplit('/',1)[-1]}") + respDel = self.deleteFromMig(nodes_to_delete) + + if self.debug > 1: + print("Scaling down complete") + + if self.size > current_target: + print( + "Scaling up. Need to increase number of instances to " + str(self.size) + ) + # Request to resize + request = self.instanceGroupManagers.resize( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + size=self.size, + ) + response = request.execute() + if self.debug > 1: + print("Requesting to increase MIG size") + pprint(response) + print("Scaling up complete") + + +def main(): + + scaler = AutoScaler(args.mz) + + # Project ID + scaler.project = args.p # Ex:'slurm-var-demo' + + # Name of the zone where the managed instance group is located + scaler.zone = args.z # Ex: 'us-central1-f' + + # Name of the region where the managed instance group is located + scaler.region = args.r # Ex: 'us-central1' + + # The name of the managed instance group. + scaler.instance_group_manager = args.g # Ex: 'condor-compute-igm' + + # Default number of cores per instance, will be replaced with actual value + scaler.cores_per_node = 4 + + # Default number of running instances that the managed instance group should maintain at any given time. This number will go up and down based on the load (number of jobs in the queue) + scaler.size = 0 + + scaler.compute_instance_min_idle = args.i + + # Dry run: : 0, run scaling; 1, only provide info. + scaler.dryrun = args.d > 0 + + # Debug level: 1-print debug information, 2 - print detail debug information + scaler.debug = 0 + if args.v: + scaler.debug = args.v + + # Limit for the maximum number of compute instance. If zero (default setting), no limit will be enforced by the script + scaler.compute_instance_limit = 0 + if args.c: + scaler.compute_instance_limit = abs(args.c) + + scaler.scale() + + +if __name__ == "__main__": + main() diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml new file mode 100644 index 0000000000..db989f9d40 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml @@ -0,0 +1,46 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install but do not activate HTCondor autoscaler + become: true + hosts: localhost + tasks: + - name: Install Python 3 pip + ansible.builtin.package: + name: python3-pip + state: present + - name: Create virtual environment for HTCondor autoscaler + ansible.builtin.pip: + name: pip + version: 21.3.1 # last Python 3.6-compatible release + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv + - name: Install latest setuptools + ansible.builtin.pip: + name: setuptools + version: 59.6.0 # last Python 3.6-compatible release + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv + - name: Install HTCondor autoscaler dependencies + with_items: + - oauth2client + - google-api-python-client + - absl-py + - htcondor + ansible.builtin.pip: + name: "{{ item }}" + state: present # rely on pip resolver to pick latest compatible releases + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml new file mode 100644 index 0000000000..4d3abbbfd6 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml @@ -0,0 +1,94 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The instructions for installing HTCondor may change with time, although we +# anticipate that they will stay fixed for the 23.0 releases. Find up-to-date +# recommendations at: +## https://htcondor.readthedocs.io/en/latest/getting-htcondor/from-our-repositories.html + +--- +- name: Ensure HTCondor is installed + hosts: all + vars: + enable_docker: true + htcondor_key: https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-23.0-Key + docker_key: https://download.docker.com/linux/centos/gpg + become: true + module_defaults: + ansible.builtin.yum: + lock_timeout: 300 + tasks: + - name: Enable EPEL repository + ansible.builtin.yum: + name: + - epel-release + - name: Directly install RPM verification keys + ansible.builtin.rpm_key: + state: present + key: "{{ item }}" + loop: + - "{{ htcondor_key }}" + - "{{ docker_key }}" + register: key_install + retries: 10 + delay: 60 + until: key_install is success + - name: Enable HTCondor LTS Release repository + ansible.builtin.yum_repository: + name: htcondor-feature + description: HTCondor LTS Release (23.0) + file: htcondor + baseurl: https://research.cs.wisc.edu/htcondor/repo/23.0/el$releasever/$basearch/release + gpgkey: "{{ htcondor_key }}" + gpgcheck: true + repo_gpgcheck: true + priority: "90" + - name: Install HTCondor + ansible.builtin.yum: + name: condor-{{ condor_version | default("23.*") | string }} + state: present + - name: Ensure token directory + ansible.builtin.file: + path: /etc/condor/tokens.d + mode: 0700 + owner: root + group: root + - name: Install Docker and configure HTCondor to use it + when: enable_docker | bool # allows string to be passed at CLI + block: + - name: Setup Docker repo + ansible.builtin.yum_repository: + name: docker-ce-stable + description: Docker CE Stable - $basearch + baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable + enabled: yes + gpgcheck: yes + gpgkey: "{{ docker_key }}" + - name: Install Docker + ansible.builtin.yum: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin + - name: Enable Docker + ansible.builtin.service: + name: docker + state: started + enabled: true + - name: Add condor to docker group + ansible.builtin.user: + name: condor + groups: docker + append: yes diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/main.tf new file mode 100644 index 0000000000..0853e035f4 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + runners = [ + { + "type" = "ansible-local" + "source" = "${path.module}/files/install-htcondor.yaml" + "destination" = "install-htcondor.yaml" + "args" = join(" ", [ + "-e enable_docker=${var.enable_docker}", + "-e condor_version=${var.condor_version}", + ]) + }, + { + "type" = "ansible-local" + "content" = file("${path.module}/files/install-htcondor-autoscaler-deps.yml") + "destination" = "install-htcondor-autoscaler-deps.yml" + }, + { + "type" = "data" + "content" = file("${path.module}/files/autoscaler.py") + "destination" = "/usr/local/htcondor/bin/autoscaler.py" + }, + ] + + install_htcondor_ps1 = templatefile( + "${path.module}/templates/install-htcondor.ps1.tftpl", { + condor_version = var.condor_version, + http_proxy = var.http_proxy, + python_windows_installer_url = var.python_windows_installer_url, + }) + + required_apis = [ + "compute.googleapis.com", + "secretmanager.googleapis.com", + ] +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf new file mode 100644 index 0000000000..c7951737ff --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "runners" { + description = "Runner to install HTCondor using startup-scripts" + value = local.runners +} + +output "windows_startup_ps1" { + description = "Windows PowerShell script to install HTCondor" + value = local.install_htcondor_ps1 +} + +output "gcp_service_list" { + description = "Google Cloud APIs required by HTCondor" + value = local.required_apis +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl new file mode 100644 index 0000000000..7492da3c12 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl @@ -0,0 +1,59 @@ +#Requires -RunAsAdministrator + +# Windows 2016 needs forced upgrade to TLS 1.2 +[Net.ServicePointManager]::SecurityProtocol = 'Tls12' + +# important for catching exception in Invoke-WebRequest +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +%{ if http_proxy != "" ~} +[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") +%{ endif ~} + +# do not show progress bar when running Invoke-WebRequest +$ProgressPreference = 'SilentlyContinue' + +# download C Runtime DLL necessary for HTCondor installer +$runtime_installer = 'C:\vc_redist.x64.exe' +Invoke-WebRequest https://aka.ms/vs/17/release/vc_redist.x64.exe -OutFile "$runtime_installer" +Start-Process -FilePath "$runtime_installer" -Wait -ArgumentList "/norestart /quiet /log c:\vc_redist_log.txt" +Remove-Item "$runtime_installer" + +# download HTCondor installer +$htcondor_installer = 'C:\htcondor.msi' +%{ if condor_version == "23.*" } +Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/current/condor-Windows-x64.msi -OutFile "$htcondor_installer" +%{ else ~} +Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/${condor_version}/release/condor-${condor_version}-Windows-x64.msi -OutFile "$htcondor_installer" +%{ endif ~} +$args='/qn /l* condor-install-log.txt /i' +$args=$args + " $htcondor_installer" +$args=$args + ' NEWPOOL="N"' +$args=$args + ' RUNJOBS="N"' +$args=$args + ' SUBMITJOBS="N"' +$args=$args + ' INSTALLDIR="C:\Condor"' +Start-Process "msiexec.exe" -Wait -ArgumentList "$args" +Remove-Item "$htcondor_installer" + +# do not start HTCondor on boot by default. Allow startup script to download +# configuration first and then start HTCondor +Set-Service -StartupType Manual condor + +# remove settings from condor_config that we want to override in configuration step +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^CONDOR_HOST' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^INSTALL_USER' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^DAEMON_LIST' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^use SECURITY' -NotMatch) + +# install Python so that custom ClassAd hooks can execute +$python_installer = 'C:\python-installer.exe' +Invoke-WebRequest -Uri "${python_windows_installer_url}" -OutFile "$python_installer" +Start-Process -FilePath "$python_installer" -Wait -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1 Include_test=0' +%{ if http_proxy == "" ~} +Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --no-warn-script-location requests" +%{ else ~} +Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --proxy ${http_proxy} --no-warn-script-location requests" +%{ endif ~} +Invoke-WebRequest -Uri "https://raw.githubusercontent.com/htcondor/htcondor/main/src/condor_scripts/common-cloud-attributes-google.py" -OutFile "C:\Condor\bin\common-cloud-attributes-google.py" +Remove-Item "$python_installer" diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/variables.tf new file mode 100644 index 0000000000..1afdf4e0eb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "enable_docker" { + description = "Install and enable docker daemon alongside HTCondor" + type = bool + default = true +} + +variable "condor_version" { + description = "Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: \"23.0.0\",\"23.*\"))" + type = string + default = "23.*" + + validation { + error_message = "var.condor_version must be set to \"23.*\" for latest 23.0 release or to a specific \"23.0.y\" release." + condition = var.condor_version == "23.*" || ( + length(split(".", var.condor_version)) == 3 && alltrue([ + for v in split(".", var.condor_version) : can(tonumber(v)) + ]) && split(".", var.condor_version)[0] == "23" + && split(".", var.condor_version)[1] == "0" + ) + } +} + +variable "http_proxy" { + description = "Set system default web (http and https) proxy for Windows HTCondor installation" + type = string + default = "" + nullable = false +} + +variable "python_windows_installer_url" { + description = "URL of Python installer for Windows" + type = string + default = "https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe" + nullable = false +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/README.md new file mode 100644 index 0000000000..55c2fc7e4e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/README.md @@ -0,0 +1,116 @@ +## Description + +This module will create a startup-script runner that will execute Ramble commands. + +Ramble is a multi-platform experimentation framework capable of driving +software installation, acquiring input files, configuring experiments, and +extracting results. For more information about Ramble, see: +https://github.com/GoogleCloudPlatform/ramble + +This module outputs a startup script runner, which can be combined with other +startup script runners to execute a set of Ramble commands. + +Ramble makes extensive use of Spack. It must be installed with a Toolkit runner +generated by the [spack-setup module](../spack-setup/README.md) following the +[basic example](#basic-example) below. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +# Examples + +## Basic Example + +Below is a basic example of using this module. + +```yaml + - id: spack + source: community/modules/scripts/spack-setup + + - id: ramble-setup + source: community/modules/scripts/ramble-setup + + - id: ramble-execute + source: community/modules/scripts/ramble-execute + use: [spack, ramble-setup] + settings: + commands: + - ramble list +``` + +This example shows installing Spack and Ramble with their own modules +(spack-setup and ramble-setup respectively). Then the ramble-execute module +is added to simply list all applications Ramble knows about. + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | +| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [log\_file](#input\_log\_file) | Log file to write output from Ramble execute steps into | `string` | `"/var/log/ramble-execute.log"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by an instance of ramble-setup.
Can be defined explicitly, or by chaining an instance of a ramble-setup module
through a `use` setting. | `string` | n/a | yes | +| [ramble\_runner](#input\_ramble\_runner) | Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | +| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting.
Defaults to /etc/profile.d/spack.sh if not set. | `string` | `"/etc/profile.d/spack.sh"` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the ramble-setup module. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble startup script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for ramble, to be reused by ramble-execute module. | +| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | +| [ramble\_runner](#output\_ramble\_runner) | Runner to execute Ramble commands using an ansible playbook. The startup-script module
will automatically handle installation of ansible. | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to Spack profile script. | +| [startup\_script](#output\_startup\_script) | Ramble startup script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/main.tf new file mode 100644 index 0000000000..7ef0b029e3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/main.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ramble-execute", ghpc_role = "scripts" }) +} + +locals { + commands_content = var.commands == null ? "echo 'no ramble commands provided'" : indent(4, yamlencode(var.commands)) + + execute_contents = templatefile( + "${path.module}/templates/ramble_execute.yml.tpl", + { + pre_script = "if [ -f ${var.spack_profile_script_path} ]; then . ${var.spack_profile_script_path}; fi; . ${var.ramble_profile_script_path}" + log_file = var.log_file + commands = local.commands_content + system_user_name = var.system_user_name + } + ) + + data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] + + execute_md5 = substr(md5(local.execute_contents), 0, 4) + execute_runner = { + type = "ansible-local" + content = local.execute_contents + destination = "ramble_execute_${local.execute_md5}.yml" + } + + previous_runners = var.ramble_runner != null ? [var.ramble_runner] : [] + runners = concat(local.previous_runners, local.data_runners, [local.execute_runner]) + + # Destinations should be unique while also being known at time of apply + combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) + combined_md5 = substr(md5(local.combined_unique_string), 0, 4) + combined_runner = { + type = "shell" + content = module.startup_script.startup_script + destination = "combined_install_ramble_${local.combined_md5}.sh" + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = var.gcs_bucket_path +} + +resource "local_file" "debug_file_ansible_execute" { + content = local.execute_contents + filename = "${path.module}/debug_execute_${local.execute_md5}.yml" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf new file mode 100644 index 0000000000..4e6c3a44d8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Ramble startup script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Ramble startup script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "ramble_runner" { + description = <<-EOT + Runner to execute Ramble commands using an ansible playbook. The startup-script module + will automatically handle installation of ansible. + EOT + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for ramble, to be reused by ramble-execute module." + value = var.gcs_bucket_path +} + +output "spack_profile_script_path" { + description = "Path to Spack profile script." + value = var.spack_profile_script_path +} + +output "ramble_profile_script_path" { + description = "Path to Ramble profile script." + value = var.ramble_profile_script_path +} + +output "system_user_name" { + description = "The system user used to execute commands." + value = var.system_user_name +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl new file mode 100644 index 0000000000..0e98f3aa2c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Execute Commands + hosts: localhost + vars: + pre_script: ${pre_script} + log_file: ${log_file} + commands: ${commands} + system_user_name: ${system_user_name} + tasks: + - name: Execute command block + block: + - name: Print commands to be executed + ansible.builtin.debug: + msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" + + - name: Streaming log info + ansible.builtin.debug: + msg: | + Logs from commands will not be printed here until success (or failure) + Streaming logs can be found at {{ log_file }} + + - name: Ensure user can write to log file + ansible.builtin.file: + path: "{{ log_file }}" + state: touch + owner: "{{ system_user_name }}" + + - name: Execute commands + ansible.builtin.shell: | + set -eo pipefail + { + {{ pre_script }} + echo " === Starting commands ===" + {{ commands }} + echo " === Finished commands ===" + } 2>&1 | tee -a {{ log_file }} + args: + executable: /bin/bash + register: output + become: true + become_user: "{{ system_user_name }}" + + always: + - name: Print commands output + ansible.builtin.debug: + var: output.stdout_lines diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/variables.tf new file mode 100644 index 0000000000..ec67228df5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/variables.tf @@ -0,0 +1,114 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing spack scripts." + type = string +} + +variable "region" { + description = "Region to place bucket containing spack scripts." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "log_file" { + description = "Log file to write output from Ramble execute steps into" + default = "/var/log/ramble-execute.log" + type = string +} + +variable "data_files" { + description = <<-EOT + A list of files to be transferred prior to running commands. + It must specify one of 'source' (absolute local file path) or 'content' (string). + It must specify a 'destination' with absolute path where file should be placed. + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) + error_message = "All destinations must be absolute paths and start with '/'." + } + validation { + condition = alltrue([ + for r in var.data_files : + can(r["content"]) != can(r["source"]) + ]) + error_message = "A data_file must specify either 'content' or 'source', but never both." + } + validation { + condition = alltrue([ + for r in var.data_files : + lookup(r, "content", lookup(r, "source", null)) != null + ]) + error_message = "A data_file must specify a non-null 'content' or 'source'." + } +} + +variable "commands" { + description = "String of commands to run within this module" + default = null + type = string +} + +variable "ramble_runner" { + description = "Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module." + type = object({ + type = string + content = string + destination = string + }) +} + +variable "system_user_name" { + description = "Name of the system user used to execute commands. Generally passed from the ramble-setup module." + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string +} + +variable "spack_profile_script_path" { + description = <<-EOT + Path to the Spack profile.d script. + Can be defined explicitly, or by chaining an instance of a spack-setup module + through a `use` setting. + Defaults to /etc/profile.d/spack.sh if not set. + EOT + type = string + default = "/etc/profile.d/spack.sh" +} + +variable "ramble_profile_script_path" { + description = <<-EOT + Path to the Ramble profile.d script. Created by an instance of ramble-setup. + Can be defined explicitly, or by chaining an instance of a ramble-setup module + through a `use` setting. + EOT + type = string +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/versions.tf new file mode 100644 index 0000000000..9b23317323 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/versions.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/README.md new file mode 100644 index 0000000000..9891088105 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/README.md @@ -0,0 +1,128 @@ +## Description + +This module will create a set of startup-script runners that will setup Ramble, +and install Ramble’s dependencies. + +Ramble is a multi-platform experimentation framework capable of driving +software installation, acquiring input files, configuring experiments, and +extracting results. For more information about ramble, see: +https://github.com/GoogleCloudPlatform/ramble + +This module outputs two startup script runners, which can be added to startup +scripts to setup, ramble and its dependencies. + +For this module to be completely functional, it depends on a spack +installation. For more information, see Cluster-Toolkit’s Spack module. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +# Examples + +## Basic Example + +```yaml +- id: ramble-setup + source: community/modules/scripts/ramble-setup +``` + +This example simply installs ramble on a VM. + +## Full Example + +```yaml +- id: ramble-setup + source: community/modules/scripts/ramble-setup + settings: + install_dir: /ramble + ramble_url: https://github.com/GoogleCloudPlatform/ramble + ramble_ref: v0.2.1 + log_file: /var/log/ramble.log + chown_owner: “owner” + chgrp_group: “user_group” + chmod_mode: “a+r” +``` + +This example simply installs ramble into a VM at the location `/ramble`, checks +out the v0.2.1 tag, changes the owner and group to “owner” and “user_group”, +and chmod’s the clone to make it world readable. + +Also see a more complete [Ramble example blueprint](../../../examples/ramble.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 4.42 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [chmod\_mode](#input\_chmod\_mode) | Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify).
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | +| [install\_dir](#input\_install\_dir) | Destination directory of installation of Ramble. | `string` | `"/apps/ramble"` | no | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by this module | `string` | `"/etc/profile.d/ramble.sh"` | no | +| [ramble\_ref](#input\_ramble\_ref) | Git ref to checkout for Ramble. | `string` | `"develop"` | no | +| [ramble\_url](#input\_ramble\_url) | URL for Ramble repository to clone. | `string` | `"https://github.com/GoogleCloudPlatform/ramble"` | no | +| [ramble\_virtualenv\_path](#input\_ramble\_virtualenv\_path) | Virtual environment path in which to install Ramble Python interpreter and other dependencies | `string` | `"/usr/local/ramble-python"` | no | +| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | +| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Ramble. It will be created if it does not exist. | `string` | `"ramble"` | no | +| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble installation script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module. | +| [ramble\_path](#output\_ramble\_path) | Location ramble is installed into. | +| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | +| [ramble\_ref](#output\_ramble\_ref) | Git ref the ramble install is checked out to use | +| [ramble\_runner](#output\_ramble\_runner) | Runner to be used with startup-script module or passed to ramble-execute module.
- installs Ramble dependencies
- installs Ramble
- generates profile.d script to enable access to Ramble
This is safe to run in parallel by multiple machines. | +| [startup\_script](#output\_startup\_script) | Ramble installation script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/main.tf new file mode 100644 index 0000000000..4389af7d33 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/main.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ramble-setup", ghpc_role = "scripts" }) +} + +locals { + profile_script = <<-EOF + if [ -f ${var.install_dir}/share/ramble/setup-env.sh ]; then + test -t 1 && echo "** Ramble's python virtualenv (/usr/local/ramble-python) is activated. Call 'deactivate' to deactivate." + VIRTUAL_ENV_DISABLE_PROMPT=1 . ${var.ramble_virtualenv_path}/bin/activate + . ${var.install_dir}/share/ramble/setup-env.sh + fi + EOF + + script_content = templatefile( + "${path.module}/templates/ramble_setup.yml.tftpl", + { + sw_name = "ramble" + profile_script = indent(4, yamlencode(local.profile_script)) + install_dir = var.install_dir + git_url = var.ramble_url + git_ref = var.ramble_ref + chmod_mode = var.chmod_mode + system_user_name = var.system_user_name + system_user_uid = var.system_user_uid + system_user_gid = var.system_user_gid + finalize_setup_script = "echo 'no finalize setup script'" + profile_script_path = var.ramble_profile_script_path + } + ) + + install_ramble_deps_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/install_ramble_deps.yml" + "destination" = "install_ramble_deps.yml" + "args" = "-e virtualenv_path=${var.ramble_virtualenv_path}" + } + + python_reqs_content = templatefile( + "${path.module}/templates/install_ramble_python_deps.yml.tftpl", + { + install_dir = var.install_dir + virtualenv_path = var.ramble_virtualenv_path + } + ) + + python_reqs_runner = { + "type" = "ansible-local" + "content" = local.python_reqs_content + "destination" = "install_ramble_reqs.yml" + } + + install_ramble_runner = { + "type" = "ansible-local" + "content" = local.script_content + "destination" = "install_ramble.yml" + } + + bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}"), 0, 8) + # Max bucket name length is 63, so truncate deployment_name if necessary. + # The string "-ramble-scripts-" is 16 characters and bucket_md5 is 8 characters, + # leaving 63-16-8=39 chars for deployment_name. + bucket_name = "${substr(var.deployment_name, 0, 39)}-ramble-scripts-${local.bucket_md5}" + runners = [local.install_ramble_deps_runner, local.install_ramble_runner, local.python_reqs_runner] + + combined_runner = { + "type" = "shell" + "content" = module.startup_script.startup_script + "destination" = "ramble-install-and-setup.sh" + } + +} + +resource "google_storage_bucket" "bucket" { + project = var.project_id + name = local.bucket_name + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" +} + +resource "local_file" "debug_file_shell_install" { + content = local.script_content + filename = "${path.module}/debug_install.yml" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf new file mode 100644 index 0000000000..e587470eac --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Ramble installation script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Ramble installation script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "ramble_runner" { + description = <<-EOT + Runner to be used with startup-script module or passed to ramble-execute module. + - installs Ramble dependencies + - installs Ramble + - generates profile.d script to enable access to Ramble + This is safe to run in parallel by multiple machines. + EOT + value = local.combined_runner +} + +output "ramble_path" { + description = "Location ramble is installed into." + value = var.install_dir +} + +output "ramble_ref" { + description = "Git ref the ramble install is checked out to use" + value = var.ramble_ref +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module." + value = "gs://${google_storage_bucket.bucket.name}" +} + +output "ramble_profile_script_path" { + description = "Path to Ramble profile script." + value = var.ramble_profile_script_path +} + +output "system_user_name" { + description = "The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands." + value = var.system_user_name +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml new file mode 100644 index 0000000000..b7905bbe9e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml @@ -0,0 +1,50 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Create python virtual env for a tool + become: yes + hosts: localhost + vars: + virtualenv_path: ${virtualenv_path} + tasks: + - name: Install dependencies through system package manager + ansible.builtin.package: + name: + - python3 + - python3-pip + - git + register: package + changed_when: package.changed + retries: 5 + delay: 10 + until: package is success + + - name: Create virtualenv for tool + # Python 3.6 is minimum we wish to support due to ease of installation on + # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip + # supported by 3.6. Additionally, recent versions of pip are necessary for + # proper dependency resolution of real-world problems with google-cloud-* + # (and third-party) Python packages (20.3+ probably effective minimum). + ansible.builtin.pip: + name: pip>=21.3.1 + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv + + - name: Add google-cloud-storage to virtualenv + ansible.builtin.pip: + name: google-cloud-storage + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl new file mode 100644 index 0000000000..ea14780a58 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl @@ -0,0 +1,28 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Install Python Requirements + hosts: localhost + vars: + install_dir: ${install_dir} + virtualenv_path: ${virtualenv_path} + tasks: + + - name: Install dependencies + ansible.builtin.pip: + requirements: "{{ install_dir }}/requirements.txt" + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl new file mode 100644 index 0000000000..ca48a5afa0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl @@ -0,0 +1,157 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install Software + hosts: localhost + vars: + sw_name: ${sw_name} + profile_script: ${profile_script} + install_dir: ${install_dir} + git_url: ${git_url} + git_ref: ${git_ref} + chmod_mode: ${chmod_mode} + system_user_name: ${system_user_name} + system_user_uid: ${system_user_uid} + system_user_gid: ${system_user_gid} + finalize_setup_script: ${finalize_setup_script} + profile_script_path: ${profile_script_path} + tasks: + - name: Print software name + ansible.builtin.debug: + msg: "Running installation for software: {{ sw_name }}" + + - name: Add profile script for software + ansible.builtin.copy: + dest: "{{ profile_script_path }}" + mode: '0644' + content: "{{ profile_script }}" + when: profile_script + + - name: Look up user to use for install + block: + + - name: Check if user already exists + ansible.builtin.getent: + database: passwd + key: "{{ system_user_name }}" + + - name: Look up existing user details + ansible.builtin.user: + name: "{{ system_user_name }}" + register: system_user + + rescue: + - name: User did not exist, create group for system user + ansible.builtin.group: + name: "{{ system_user_name }}" + gid: "{{ system_user_gid }}" + system: true + register: system_group + + - name: Create system user + ansible.builtin.user: + name: "{{ system_user_name }}" + comment: "{{ sw_name }} installation" + uid: "{{ system_user_uid }}" + group: "{{ system_group.name }}" + system: true + register: system_user + + - name: Create parent of install directory + ansible.builtin.file: + path: "{{ install_dir | dirname }}" + state: directory + + - name: Set lock dir + ansible.builtin.set_fact: + lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" + + - name: Acquire lock + ansible.builtin.command: + mkdir "{{ lock_dir }}" + register: lock_out + changed_when: lock_out.rc == 0 + failed_when: false + + - name: Add hostname to lock_dir + ansible.builtin.file: + path: "{{ lock_dir }}/{{ ansible_hostname }}" + state: touch + when: lock_out.rc == 0 + + - name: Clone branch or tag into installation directory + ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} + failed_when: false + register: clone_res + when: lock_out.rc == 0 + + - name: Clone commit hash into installation directory + ansible.builtin.command: "{{ item }}" + with_items: + - git clone {{ git_url }} {{ install_dir }} + - git -C {{ install_dir }} checkout {{ git_ref }} + when: lock_out.rc == 0 and clone_res.rc != 0 + + - name: Transfer ownership to system user + ansible.builtin.file: + path: "{{ install_dir }}" + owner: "{{ system_user.name }}" + group: "{{ system_user.group }}" + recurse: true + follow: false + when: lock_out.rc == 0 + + - name: Finalize setup + ansible.builtin.shell: "{{ finalize_setup_script }}" + when: lock_out.rc == 0 and finalize_setup_script + become: true + become_user: "{{ system_user.name }}" + + - name: Apply chmod + ansible.builtin.file: + path: "{{ install_dir }}" + mode: "{{ chmod_mode | default(omit, true) }}" + recurse: true + follow: false + when: (lock_out.rc == 0) and (chmod_mode != None) + + - name: Release lock + ansible.builtin.file: + path: "{{ lock_dir }}/done" + state: touch + when: lock_out.rc == 0 + + - name: Wait for lock + block: + - name: Wait for lock + ansible.builtin.wait_for: + path: "{{ lock_dir }}/done" + state: present + timeout: 600 + sleep: 10 + when: lock_out.rc != 0 + + rescue: + - name: Timed out on waiting for lock, get lock directory contents + ansible.builtin.find: + paths: "{{ lock_dir }}" + register: lock_dir_contents + + - name: Print lock directory contents, it should contain name of host that is holding lock + ansible.builtin.debug: + msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" + + - name: Failed to get lock + ansible.builtin.fail: + msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/variables.tf new file mode 100644 index 0000000000..0d3a8eed05 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/variables.tf @@ -0,0 +1,97 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "install_dir" { + description = "Destination directory of installation of Ramble." + default = "/apps/ramble" + type = string +} + +variable "ramble_url" { + description = "URL for Ramble repository to clone." + default = "https://github.com/GoogleCloudPlatform/ramble" + type = string +} + +variable "ramble_ref" { + description = "Git ref to checkout for Ramble." + default = "develop" + type = string +} + +variable "chmod_mode" { + description = <<-EOT + Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify). + For usage information see: + https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode + EOT + default = "" + type = string + nullable = false +} + +variable "system_user_name" { + description = "Name of system user that will perform installation of Ramble. It will be created if it does not exist." + default = "ramble" + type = string + nullable = false +} + +variable "system_user_uid" { + description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." + default = 1104762904 + type = number + nullable = false +} + +variable "system_user_gid" { + description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." + default = 1104762904 + type = number + nullable = false +} + +variable "ramble_virtualenv_path" { + description = "Virtual environment path in which to install Ramble Python interpreter and other dependencies" + default = "/usr/local/ramble-python" + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing startup script." + type = string +} + +variable "region" { + description = "Region to place bucket containing startup script." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "ramble_profile_script_path" { + description = "Path to the Ramble profile.d script. Created by this module" + type = string + default = "/etc/profile.d/ramble.sh" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/versions.tf new file mode 100644 index 0000000000..936b4a5b80 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/README.md new file mode 100644 index 0000000000..8cbb75fb42 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/README.md @@ -0,0 +1,141 @@ +## Description + +This module creates a script that defines a software build using Spack and +performs any additional customization to a Spack installation. + +There are two main variable inputs that can be used to define a Spack build: +`data_files` and `commands`. + +- `data_files`: Any files specified will be transferred to the machine running + outputted script. Data file `content` can be defined inline in the blueprint + or can point to a `source`, an absolute local path of a file. This can be used + to transfer environment definition files, config definition files, GPG keys, + or software licenses. `data_files` are transferred before `commands` are run. +- `commands`: A script that is run. This can be used to perform actions such as + installation of compilers & packages, environment creation, adding a build + cache, and modifying the spack configuration. + +## Example + +The `spack-execute` module should `use` a `spack-setup` module. This will +prepend the installation of Spack and its dependencies to the build. Then +`spack-execute` can be used by a module that takes `startup-script` as an input. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: builder-vm + source: modules/compute/vm-instance + use: [network1, spack-build] +``` + +To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. + +[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml + +### Using with `startup-script` module + +The `spack-runner` output can be used by the `startup-script` module. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(spack-build.spack-runner) + - type: shell + destination: "my-script.sh" + content: echo 'hello world' + + - id: workstation + source: modules/compute/vm-instance + use: [network1, startup-script] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | +| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [log\_file](#input\_log\_file) | Defines the logfile that script output will be written to | `string` | `"/var/log/spack.log"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by an instance of spack-setup.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting. | `string` | n/a | yes | +| [spack\_runner](#input\_spack\_runner) | Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | +| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the spack-setup module. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack startup script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | +| [spack\_runner](#output\_spack\_runner) | Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules. | +| [startup\_script](#output\_startup\_script) | Spack startup script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/main.tf new file mode 100644 index 0000000000..04ebcf7d49 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "spack-execute", ghpc_role = "scripts" }) +} + +locals { + commands_content = var.commands == null ? "echo 'no spack commands provided'" : indent(4, yamlencode(var.commands)) + + execute_contents = templatefile( + "${path.module}/templates/execute_commands.yml.tpl", + { + pre_script = ". ${var.spack_profile_script_path}" + log_file = var.log_file + commands = local.commands_content + system_user_name = var.system_user_name + } + ) + + data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] + + execute_md5 = substr(md5(local.execute_contents), 0, 4) + execute_runner = { + type = "ansible-local" + content = local.execute_contents + destination = "spack_execute_${local.execute_md5}.yml" + } + + runners = concat([var.spack_runner], local.data_runners, [local.execute_runner]) + + # Destinations should be unique while also being known at time of apply + combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) + combined_md5 = substr(md5(local.combined_unique_string), 0, 4) + combined_runner = { + type = "shell" + content = module.startup_script.startup_script + destination = "combined_install_spack_${local.combined_md5}.sh" + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = var.gcs_bucket_path +} + +resource "local_file" "debug_file_ansible_execute" { + content = local.execute_contents + filename = "${path.module}/debug_execute_${local.execute_md5}.yml" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/outputs.tf new file mode 100644 index 0000000000..4a52532d51 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Spack startup script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Spack startup script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "spack_runner" { + description = "Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules." + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." + value = var.gcs_bucket_path +} + +output "spack_profile_script_path" { + description = "Path to the Spack profile.d script." + value = var.spack_profile_script_path +} + +output "system_user_name" { + description = "The system user used to execute commands." + value = var.system_user_name +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl new file mode 100644 index 0000000000..0e98f3aa2c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Execute Commands + hosts: localhost + vars: + pre_script: ${pre_script} + log_file: ${log_file} + commands: ${commands} + system_user_name: ${system_user_name} + tasks: + - name: Execute command block + block: + - name: Print commands to be executed + ansible.builtin.debug: + msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" + + - name: Streaming log info + ansible.builtin.debug: + msg: | + Logs from commands will not be printed here until success (or failure) + Streaming logs can be found at {{ log_file }} + + - name: Ensure user can write to log file + ansible.builtin.file: + path: "{{ log_file }}" + state: touch + owner: "{{ system_user_name }}" + + - name: Execute commands + ansible.builtin.shell: | + set -eo pipefail + { + {{ pre_script }} + echo " === Starting commands ===" + {{ commands }} + echo " === Finished commands ===" + } 2>&1 | tee -a {{ log_file }} + args: + executable: /bin/bash + register: output + become: true + become_user: "{{ system_user_name }}" + + always: + - name: Print commands output + ansible.builtin.debug: + var: output.stdout_lines diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/variables.tf new file mode 100644 index 0000000000..851cd1aed8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/variables.tf @@ -0,0 +1,103 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing spack scripts." + type = string +} + +variable "region" { + description = "Region to place bucket containing spack scripts." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "log_file" { + description = "Defines the logfile that script output will be written to" + default = "/var/log/spack.log" + type = string +} + +variable "data_files" { + description = <<-EOT + A list of files to be transferred prior to running commands. + It must specify one of 'source' (absolute local file path) or 'content' (string). + It must specify a 'destination' with absolute path where file should be placed. + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) + error_message = "All destinations must be absolute paths and start with '/'." + } + validation { + condition = alltrue([ + for r in var.data_files : + can(r["content"]) != can(r["source"]) + ]) + error_message = "A data_file must specify either 'content' or 'source', but never both." + } + validation { + condition = alltrue([ + for r in var.data_files : + lookup(r, "content", lookup(r, "source", null)) != null + ]) + error_message = "A data_file must specify a non-null 'content' or 'source'." + } +} + +variable "commands" { + description = "String of commands to run within this module" + type = string + default = null +} + +variable "spack_runner" { + description = "Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module." + type = object({ + type = string + content = string + destination = string + }) +} + +variable "system_user_name" { + description = "Name of the system user used to execute commands. Generally passed from the spack-setup module." + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string +} + +variable "spack_profile_script_path" { + description = <<-EOT + Path to the Spack profile.d script. Created by an instance of spack-setup. + Can be defined explicitly, or by chaining an instance of a spack-setup module + through a `use` setting. + EOT + type = string +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/versions.tf new file mode 100644 index 0000000000..09583c3d43 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/versions.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.0.0" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/README.md new file mode 100644 index 0000000000..01d3e6d389 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/README.md @@ -0,0 +1,382 @@ +## Description + +This module can be used to setup and install Spack on a VM. To actually run +Spack commands to install other software use the +[spack-execute](../spack-execute/) module. + +This module generates a script that performs the following: + +1. Install system dependencies needed for Spack +1. Clone Spack into a predefined directory +1. Check out a specific version of Spack + +There are several options on how to consume the outputs of this module: + +> [!IMPORTANT] +> Breaking changes between after v1.21.0. `spack-install` module replaced by +> `spack-setup` and `spack-execute` modules. +> [Details Below](#deprecations-and-breaking-changes) + +## Examples + +### `use` `spack-setup` with `spack-execute` + +This will prepend the `spack-setup` script to the `spack-execute` commands. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: builder + source: modules/compute/vm-instance + use: [network1, spack-build] +``` + +### `use` `spack-setup` with `vm-instance` or Slurm module + +This will run `spack-setup` scripts on the downstream compute resource. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-installer + source: modules/compute/vm-instance + use: [network1, spack-setup] +``` + +OR + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: [network1, partition1, spack-setup] +``` + +### Build `starup-script` with `spack-runner` output + +This will use the generated `spack-setup` script as one step in `startup-script`. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(spack-setup.spack-runner) + - type: shell + destination: "my-script.sh" + content: echo 'hello world' + + - id: workstation + source: modules/compute/vm-instance + use: [network1, startup-script] +``` + +To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. + +[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml + +## Environment Setup + +### Activating Spack + +[Spack installation] produces a setup script that adds `spack` to your `PATH` as +well as some other command-line integration tools. This script can be found at +`/share/spack/setup-env.sh`. This script will be automatically +added to bash startup by any machine that runs the `spack_runner`. + +If you have multiple machines that all want to use the same shared Spack +installation you can just have both machines run the `spack_runner`. + +[Spack installation]: https://spack-tutorial.readthedocs.io/en/latest/tutorial_basics.html#installing-spack + +### Managing Spack Python dependencies + +Spack is configured with [SPACK_PYTHON] to ensure that Spack itself uses a +Python virtual environment with a supported copy of Python with the package +`google-cloud-storage` pre-installed. This enables Spack to use mirrors and +[build caches][builds] on Google Cloud Storage. It does not configure Python +packages *inside* Spack virtual environments. If you need to add more Python +dependencies for Spack itself, use the `spack python` command: + +```shell +sudo -i spack python -m pip install package-name +``` + +[SPACK_PYTHON]: https://spack.readthedocs.io/en/latest/getting_started.html#shell-support +[builds]: https://spack.readthedocs.io/en/latest/binary_caches.html + +## Spack Permissions + +### System `spack` user is created - Default + +By default this module will create a `spack` linux user and group with +consistent UID and GID. This user and group will own the Spack installation. To +allow a user to manually add Spack packages to the system Spack installation, +you can add the user to the spack group: + +```sh +sudo usermod -a -G spack +``` + +Log out and back in so the group change will take effect, then `` will +be able to call `spack install `. + +> [!NOTE] +> A background persistent SSH connections may prevent the group change from +> taking effect. + +You can use the `system_user_name`, `system_user_uid`, and `system_user_gid` to +customize the name and ids of the system user. While unlikely, it is possible +that the default `system_user_uid` or `system_user_gid` could conflict with +existing UIDs. + +### Use and existing user + +Alternatively, if `system_user_name` is a user already on the system, then this +existing user will be used for Spack installation. + +#### OS Login User + +If OS Login is enabled (default for most Cluster Toolkit modules) then you can +provide an OS Login user name: + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + system_user_name: username_company_com +``` + +This will work even if the user has not yet logged onto the machine. When the +specified user does log on to the machine they will be able to call +`spack install` without any further configuration. + +#### Pre-configured user + +You can also use a startup script to configure a user: + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + system_user_name: special-user + + - id: startup + source: modules/scripts/startup-script + settings: + runners: + - type: shell + destination: "create_user.sh" + content: | + #!/bin/bash + sudo useradd -u 799 special-user + sudo groupadd -g 922 org-group + sudo usermod -g org-group special-user + - $(spack-setup.spack_runner) + + - id: spack-vms + source: modules/compute/vm-instance + use: [network1, startup] + settings: + name_prefix: spack-vm + machine_type: n2d-standard-2 + instance_count: 5 +``` + +### Chaining spack installations + +If there is a need to have a non-root user to install spack packages it is +recommended to create a separate installation for that user and chain Spack installations +([Spack docs](https://spack.readthedocs.io/en/latest/chain.html#chaining-spack-installations)). + +Steps to chain Spack installations: + +1. Get the version of the system Spack: + + ```sh + $ spack --version + + 0.20.0 (e493ab31c6f81a9e415a4b0e0e2263374c61e758) + # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + # Note commit hash and use in next step + ``` + +1. Clone a new spack installation: + + ```sh + git clone -c feature.manyFiles=true https://github.com/spack/spack.git /spack + git -C /spack checkout + ``` + +1. Point the new Spack installation to the system Spack installation. Create a + file at `/spack/etc/spack/upstreams.yaml` with the following + contents: + + ```yaml + upstreams: + spack-instance-1: + install_tree: /sw/spack/opt/spack/ + ``` + +1. Add the following line to your `.bashrc` to make sure the new `spack` is in + your `PATH`. + + ```sh + . /spack/share/spack/setup-env.sh + ``` + +## Deprecations and Breaking Changes + +The old `spack-install` module has been replaced by the `spack-setup` and +`spack-execute` modules. Generally this change strives to allow for a more +flexible definition of a Spack build by using native Spack commands. + +For every deprecated variable from `spack-install` there is documentation on how +to perform the equivalent action using `commands` and `data_files`. The +documentation can be found on the [inputs table](#inputs) below. + +Below is a simple example of the same functionality shown before and after the +breaking changes. + +```yaml + # Before + - id: spack-install + source: community/modules/scripts/spack-install + settings: + install_dir: /sw/spack + compilers: + - gcc@10.3.0 target=x86_64 + packages: + - intel-mpi@2018.4.274%gcc@10.3.0 + +- id: spack-startup + source: modules/scripts/startup-script + settings: + runners: + - $(spack.install_spack_deps_runner) + - $(spack.install_spack_runner) +``` + +```yaml + # After + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + install_dir: /sw/spack + + - id: spack-execute + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + spack load gcc@10.3.0 target=x86_64 + spack compiler find --scope site + spack install intel-mpi@2018.4.274%gcc@10.3.0 + +- id: spack-startup + source: modules/scripts/startup-script + settings: + runners: + - $(spack-execute.spack-runner) +``` + +Although the old `spack-install` module will no longer be maintained, it is +still possible to use the old module in a blueprint by referencing an old +version from GitHub. Note the source line in the following example. + +```yaml + - id: spack-install + source: github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/spack-install?ref=v1.22.1&depth=1 +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 4.42 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [chmod\_mode](#input\_chmod\_mode) | `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification.
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `"g+w"` | no | +| [configure\_for\_google](#input\_configure\_for\_google) | When true, the spack installation will be configured to pull from Google's Spack binary cache. | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | +| [install\_dir](#input\_install\_dir) | Directory to install spack into. | `string` | `"/sw/spack"` | no | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by this module | `string` | `"/etc/profile.d/spack.sh"` | no | +| [spack\_ref](#input\_spack\_ref) | Git ref to checkout for spack. | `string` | `"v0.20.0"` | no | +| [spack\_url](#input\_spack\_url) | URL to clone the spack repo from. | `string` | `"https://github.com/spack/spack"` | no | +| [spack\_virtualenv\_path](#input\_spack\_virtualenv\_path) | Virtual environment path in which to install Spack Python interpreter and other dependencies | `string` | `"/usr/local/spack-python"` | no | +| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Spack. It will be created if it does not exist. | `string` | `"spack"` | no | +| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack installation script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | +| [spack\_path](#output\_spack\_path) | Path to the root of the spack installation | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | +| [spack\_runner](#output\_spack\_runner) | Runner to be used with startup-script module or passed to spack-execute module.
- installs Spack dependencies
- installs Spack
- generates profile.d script to enable access to Spack
This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. | +| [startup\_script](#output\_startup\_script) | Spack installation script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Spack. It can be reused by spack-execute module to install spack packages. | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/main.tf new file mode 100644 index 0000000000..d45f5d1be3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/main.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "spack-setup", ghpc_role = "scripts" }) +} + +locals { + profile_script = <<-EOF + SPACK_PYTHON=${var.spack_virtualenv_path}/bin/python3 + if [ -f ${var.install_dir}/share/spack/setup-env.sh ]; then + test -t 1 && echo "Running Spack setup, this may take a moment on first login." + . ${var.install_dir}/share/spack/setup-env.sh + fi + EOF + + supported_cache_versions = ["v0.19.0", "v0.20.0"] + cache_version = contains(local.supported_cache_versions, var.spack_ref) ? var.spack_ref : "latest" + add_google_mirror_script = !var.configure_for_google ? "" : <<-EOF + if ! spack mirror list | grep -q google_binary_cache; then + spack mirror add --scope site google_binary_cache gs://spack/${local.cache_version} + spack buildcache keys --install --trust + fi + EOF + + finalize_setup_script = <<-EOF + set -e + . ${var.spack_profile_script_path} + spack config --scope site add 'packages:all:permissions:read:world' + spack config --scope site add 'packages:all:permissions:write:group' + spack gpg init + spack compiler find --scope site + ${local.add_google_mirror_script} + # perform fast install to make sure Spack is fully initialized + spack install xz + spack uninstall --yes-to-all xz + EOF + + script_content = templatefile( + "${path.module}/templates/spack_setup.yml.tftpl", + { + sw_name = "spack" + profile_script = indent(4, yamlencode(local.profile_script)) + install_dir = var.install_dir + git_url = var.spack_url + git_ref = var.spack_ref + chmod_mode = var.chmod_mode + system_user_name = var.system_user_name + system_user_uid = var.system_user_uid + system_user_gid = var.system_user_gid + finalize_setup_script = indent(4, yamlencode(local.finalize_setup_script)) + profile_script_path = var.spack_profile_script_path + } + ) + + install_spack_deps_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/install_spack_deps.yml" + "destination" = "install_spack_deps.yml" + "args" = "-e virtualenv_path=${var.spack_virtualenv_path}" + } + install_spack_runner = { + "type" = "ansible-local" + "content" = local.script_content + "destination" = "install_spack.yml" + } + + bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}.${local.script_content}"), 0, 8) + # Max bucket name length is 63, so truncate deployment_name if necessary. + # The string "-spack-scripts-" is 15 characters and bucket_md5 is 8 characters, + # leaving 63-15-8=40 chars for deployment_name. Using 39 so it has the same prefix as the + # ramble-setup module's GCS bucket. + bucket_name = "${substr(var.deployment_name, 0, 39)}-spack-scripts-${local.bucket_md5}" + runners = [local.install_spack_deps_runner, local.install_spack_runner] + + combined_runner = { + "type" = "shell" + "content" = module.startup_script.startup_script + "destination" = "spack-install-and-setup.sh" + } +} + +resource "google_storage_bucket" "bucket" { + project = var.project_id + name = local.bucket_name + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" +} + +resource "local_file" "debug_file_shell_install" { + content = local.script_content + filename = "${path.module}/debug_install.yml" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml new file mode 100644 index 0000000000..2ada34471f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/outputs.tf new file mode 100644 index 0000000000..d94b9757db --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/outputs.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Spack installation script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Spack installation script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "spack_path" { + description = "Path to the root of the spack installation" + value = var.install_dir +} + +output "spack_runner" { + description = <<-EOT + Runner to be used with startup-script module or passed to spack-execute module. + - installs Spack dependencies + - installs Spack + - generates profile.d script to enable access to Spack + This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. + EOT + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." + value = "gs://${google_storage_bucket.bucket.name}" +} + +output "spack_profile_script_path" { + description = "Path to the Spack profile.d script." + value = var.spack_profile_script_path +} + +output "system_user_name" { + description = "The system user used to install Spack. It can be reused by spack-execute module to install spack packages." + value = var.system_user_name +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml new file mode 100644 index 0000000000..b7905bbe9e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml @@ -0,0 +1,50 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Create python virtual env for a tool + become: yes + hosts: localhost + vars: + virtualenv_path: ${virtualenv_path} + tasks: + - name: Install dependencies through system package manager + ansible.builtin.package: + name: + - python3 + - python3-pip + - git + register: package + changed_when: package.changed + retries: 5 + delay: 10 + until: package is success + + - name: Create virtualenv for tool + # Python 3.6 is minimum we wish to support due to ease of installation on + # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip + # supported by 3.6. Additionally, recent versions of pip are necessary for + # proper dependency resolution of real-world problems with google-cloud-* + # (and third-party) Python packages (20.3+ probably effective minimum). + ansible.builtin.pip: + name: pip>=21.3.1 + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv + + - name: Add google-cloud-storage to virtualenv + ansible.builtin.pip: + name: google-cloud-storage + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl new file mode 100644 index 0000000000..ca48a5afa0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl @@ -0,0 +1,157 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install Software + hosts: localhost + vars: + sw_name: ${sw_name} + profile_script: ${profile_script} + install_dir: ${install_dir} + git_url: ${git_url} + git_ref: ${git_ref} + chmod_mode: ${chmod_mode} + system_user_name: ${system_user_name} + system_user_uid: ${system_user_uid} + system_user_gid: ${system_user_gid} + finalize_setup_script: ${finalize_setup_script} + profile_script_path: ${profile_script_path} + tasks: + - name: Print software name + ansible.builtin.debug: + msg: "Running installation for software: {{ sw_name }}" + + - name: Add profile script for software + ansible.builtin.copy: + dest: "{{ profile_script_path }}" + mode: '0644' + content: "{{ profile_script }}" + when: profile_script + + - name: Look up user to use for install + block: + + - name: Check if user already exists + ansible.builtin.getent: + database: passwd + key: "{{ system_user_name }}" + + - name: Look up existing user details + ansible.builtin.user: + name: "{{ system_user_name }}" + register: system_user + + rescue: + - name: User did not exist, create group for system user + ansible.builtin.group: + name: "{{ system_user_name }}" + gid: "{{ system_user_gid }}" + system: true + register: system_group + + - name: Create system user + ansible.builtin.user: + name: "{{ system_user_name }}" + comment: "{{ sw_name }} installation" + uid: "{{ system_user_uid }}" + group: "{{ system_group.name }}" + system: true + register: system_user + + - name: Create parent of install directory + ansible.builtin.file: + path: "{{ install_dir | dirname }}" + state: directory + + - name: Set lock dir + ansible.builtin.set_fact: + lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" + + - name: Acquire lock + ansible.builtin.command: + mkdir "{{ lock_dir }}" + register: lock_out + changed_when: lock_out.rc == 0 + failed_when: false + + - name: Add hostname to lock_dir + ansible.builtin.file: + path: "{{ lock_dir }}/{{ ansible_hostname }}" + state: touch + when: lock_out.rc == 0 + + - name: Clone branch or tag into installation directory + ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} + failed_when: false + register: clone_res + when: lock_out.rc == 0 + + - name: Clone commit hash into installation directory + ansible.builtin.command: "{{ item }}" + with_items: + - git clone {{ git_url }} {{ install_dir }} + - git -C {{ install_dir }} checkout {{ git_ref }} + when: lock_out.rc == 0 and clone_res.rc != 0 + + - name: Transfer ownership to system user + ansible.builtin.file: + path: "{{ install_dir }}" + owner: "{{ system_user.name }}" + group: "{{ system_user.group }}" + recurse: true + follow: false + when: lock_out.rc == 0 + + - name: Finalize setup + ansible.builtin.shell: "{{ finalize_setup_script }}" + when: lock_out.rc == 0 and finalize_setup_script + become: true + become_user: "{{ system_user.name }}" + + - name: Apply chmod + ansible.builtin.file: + path: "{{ install_dir }}" + mode: "{{ chmod_mode | default(omit, true) }}" + recurse: true + follow: false + when: (lock_out.rc == 0) and (chmod_mode != None) + + - name: Release lock + ansible.builtin.file: + path: "{{ lock_dir }}/done" + state: touch + when: lock_out.rc == 0 + + - name: Wait for lock + block: + - name: Wait for lock + ansible.builtin.wait_for: + path: "{{ lock_dir }}/done" + state: present + timeout: 600 + sleep: 10 + when: lock_out.rc != 0 + + rescue: + - name: Timed out on waiting for lock, get lock directory contents + ansible.builtin.find: + paths: "{{ lock_dir }}" + register: lock_dir_contents + + - name: Print lock directory contents, it should contain name of host that is holding lock + ansible.builtin.debug: + msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" + + - name: Failed to get lock + ansible.builtin.fail: + msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/variables.tf new file mode 100644 index 0000000000..85baeec401 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/variables.tf @@ -0,0 +1,106 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +# spack-setup variables + +variable "install_dir" { + description = "Directory to install spack into." + type = string + default = "/sw/spack" +} + +variable "spack_url" { + description = "URL to clone the spack repo from." + type = string + default = "https://github.com/spack/spack" +} + +variable "spack_ref" { + description = "Git ref to checkout for spack." + type = string + default = "v0.20.0" +} + +variable "configure_for_google" { + description = "When true, the spack installation will be configured to pull from Google's Spack binary cache." + type = bool + default = true +} + + +variable "chmod_mode" { + description = <<-EOT + `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification. + For usage information see: + https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode + EOT + default = "g+w" + type = string + nullable = false +} + +variable "system_user_name" { + description = "Name of system user that will perform installation of Spack. It will be created if it does not exist." + default = "spack" + type = string + nullable = false +} + +variable "system_user_uid" { + description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." + default = 1104762903 + type = number + nullable = false +} + +variable "system_user_gid" { + description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." + default = 1104762903 + type = number + nullable = false +} + +variable "spack_virtualenv_path" { + description = "Virtual environment path in which to install Spack Python interpreter and other dependencies" + default = "/usr/local/spack-python" + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing startup script." + type = string +} + +variable "region" { + description = "Region to place bucket containing startup script." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "spack_profile_script_path" { + description = "Path to the Spack profile.d script. Created by this module" + type = string + default = "/etc/profile.d/spack.sh" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/versions.tf new file mode 100644 index 0000000000..ff1180fc1b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/README.md new file mode 100644 index 0000000000..ee9c057c39 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/README.md @@ -0,0 +1,87 @@ +## Description + +This module will insert a dependency on the completion of the startup script +for one or more specified compute VMs and report back if it fails. This can be useful when running +post-boot installation scripts that require the startup script to finish setting up a node. + +> **_WARNING:_**: this module is experimental and not fully supported. + +### Additional Dependencies + +* [**gcloud**](https://cloud.google.com/sdk/gcloud) must be present in the path + of the machine where `terraform apply` is run. + +### Example + +```yaml +- id: workstation + source: modules/compute/vm-instance + use: + - network1 + - my-startup-script + settings: + instance_count: 4 + +# Wait for all instances of the above VM to finish running startup scripts. +- id: wait + source: community/modules/scripts/wait-for-startup + settings: + instance_names: $(workstation.name) +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.validate_instance_names](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.wait_for_startup](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | +| [instance\_name](#input\_instance\_name) | Name of the instance we are waiting for (can be null if 'instance\_names' is not empty) | `string` | `null` | no | +| [instance\_names](#input\_instance\_names) | A list of instance names we are waiting for, in addition to the one mentioned in 'instance\_name' (if any) | `list(string)` | `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [timeout](#input\_timeout) | Timeout in seconds | `number` | `1200` | no | +| [zone](#input\_zone) | The GCP zone where the instance is running | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/main.tf new file mode 100644 index 0000000000..3f6b416251 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + combined_instance_names = concat(var.instance_names, [var.instance_name]) +} + +resource "null_resource" "validate_instance_names" { + lifecycle { + precondition { + condition = var.instance_name != null || length(var.instance_names) > 0 + error_message = "At least one instance name must be provided" + } + } +} + +resource "null_resource" "wait_for_startup" { + count = length(local.combined_instance_names) + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/wait-for-startup-status.sh" + environment = { + INSTANCE_NAME = self.triggers.instance_name + ZONE = var.zone + PROJECT_ID = var.project_id + TIMEOUT = var.timeout + GCLOUD_PATH = var.gcloud_path_override + } + } + + triggers = { + instance_name = local.combined_instance_names[count.index] + } +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf new file mode 100644 index 0000000000..11a2ddf118 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf @@ -0,0 +1,15 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh new file mode 100644 index 0000000000..fae5833121 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z "${INSTANCE_NAME}" ]]; then + echo "INSTANCE_NAME is unset... exiting" + exit 0 +fi +if [[ -z "${ZONE}" ]]; then + echo "ZONE is unset" + exit 1 +fi +if [[ -z "${PROJECT_ID}" ]]; then + echo "PROJECT_ID is unset" + exit 1 +fi +if [[ -z "${TIMEOUT}" ]]; then + echo "TIMEOUT is unset" + exit 1 +fi + +if [[ -n "${GCLOUD_PATH}" ]]; then + export PATH="$GCLOUD_PATH:$PATH" +fi + +echo "Waiting for startup: instance_name='${INSTANCE_NAME}', zone='${ZONE}', project_id='${PROJECT_ID}', timeout_seconds='${TIMEOUT}'" + +# Wrapper around grep that swallows the error status code 1 +c1grep() { grep "$@" || test $? = 1; } + +now=$(date +%s) + +# If VM was created more than 30 days ago, serial port logs may no longer exist. +# Exit without errors if the instance is older than 30 days. +logsExpiryDays=30 +createdTimestampIso=$(gcloud compute instances describe "${INSTANCE_NAME}" --project "${PROJECT_ID}" --zone "${ZONE}" --format "value(creationTimestamp)") +earliestAllowedCreatedTimestamp=$(date -d "${createdTimestampIso} +${logsExpiryDays} day" +%s) +if [[ "$earliestAllowedCreatedTimestamp" -lt "$now" ]]; then + echo "Instance was created more than 30 days ago - serial port 1 logs are likely expired... exiting" + exit 0 +fi + +deadline=$((now + TIMEOUT)) +error_file=$(mktemp) +fetch_cmd="gcloud compute instances get-serial-port-output ${INSTANCE_NAME} --port 1 --zone ${ZONE} --project ${PROJECT_ID}" +# Match string for all finish types of the old guest agent and successful +# finishes on the new guest agent +FINISH_LINE="startup-script exit status" +# Match string for failures on the new guest agent +FINISH_LINE_ERR="Script \"startup-script\" failed with error:" + +# NEW: Accept also these finish lines as success. +STARTUP_SCRIPT_SUCCEEDED_LINE="google-startup-scripts.service: Succeeded." +STARTUP_SCRIPT_FINISHED_LINE="Finished Google Compute Engine Startup Scripts." +STARTUP_SCRIPT_SERVICE_FINISHED_LINE="Finished google-startup-scripts.service - Google Compute Engine Startup Scripts." + +NON_FATAL_ERRORS=( + "Internal error" +) + +until [[ now -gt deadline ]]; do + ser_log=$( + set -o pipefail + ${fetch_cmd} 2>"${error_file}" | + c1grep "${FINISH_LINE}\|${FINISH_LINE_ERR}\|${STARTUP_SCRIPT_SUCCEEDED_LINE}\|${STARTUP_SCRIPT_FINISHED_LINE}\|${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}" + ) || { + err=$(cat "${error_file}") + echo "$err" + fatal_error="true" + for e in "${NON_FATAL_ERRORS[@]}"; do + if [[ $err = *"$e"* ]]; then + fatal_error="false" + break + fi + done + + if [[ $fatal_error = "true" ]]; then + exit 1 + fi + } + if [[ -n "${ser_log}" ]]; then break; fi + sleep 5 + now=$(date +%s) +done + +# This line checks for an exit code - the assumption is that there is a number +# at the end of the line and it is an exit code. +# Modified to correctly extract the last numeric exit status from the relevant log line. +LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=Script \"startup-script\" failed with error: exit status )[0-9]+" | tail -n 1) +if [[ -z "${LAST_EXIT_STATUS}" ]]; then + LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=startup-script exit status )[0-9]+" | tail -n 1) +fi + +# This specific text is monitored for in tests, do not change. +INSPECT_OUTPUT_TEXT="To inspect the startup script output, please run:" + +# --- Prioritize explicit failure from the script itself --- +if [[ "${LAST_EXIT_STATUS}" == 1 ]]; then + echo "startup-script finished with errors, ${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" + exit 1 +# --- Then explicit success from the script itself --- +elif [[ "${LAST_EXIT_STATUS}" == 0 ]]; then + echo "startup-script finished successfully" + exit 0 +elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SUCCEEDED_LINE}"; then + echo "startup-script finished successfully (startup script succeeded line detected)" + exit 0 +elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_FINISHED_LINE}"; then + echo "startup-script finished successfully (startup script finished line detected)" + exit 0 +elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}"; then + echo "startup-script finished successfully (startup script service finished line detected)" + exit 0 +# --- If we reached deadline, it's a timeout --- +elif [[ now -ge deadline ]]; then + echo "startup-script timed out after ${TIMEOUT} seconds" + echo "${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" + exit 1 +# --- All other cases are considered failure or invalid state --- +else + echo "Invalid or undetermined startup script status. Last detected exit status: '${LAST_EXIT_STATUS}'" + echo "${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" + exit "${LAST_EXIT_STATUS}" +fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf new file mode 100644 index 0000000000..fe6410a920 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "instance_name" { + description = "Name of the instance we are waiting for (can be null if 'instance_names' is not empty)" + type = string + default = null +} + +variable "instance_names" { + description = "A list of instance names we are waiting for, in addition to the one mentioned in 'instance_name' (if any)" + type = list(string) + default = [] +} + +variable "zone" { + description = "The GCP zone where the instance is running" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "timeout" { + description = "Timeout in seconds" + type = number + default = 1200 + validation { + condition = var.timeout >= 0 + error_message = "The timeout should be non-negative" + } +} + +variable "gcloud_path_override" { + description = "Directory of the gcloud executable to be used during cleanup" + type = string + default = "" + nullable = false +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf new file mode 100644 index 0000000000..8cd43b944e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/README.md new file mode 100644 index 0000000000..fc25bc0a55 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/README.md @@ -0,0 +1,109 @@ +## Description + +This module contains a set of scripts to be used in customizing Windows VMs at +boot or during image building. Please note that the installation of NVIDIA GPU +drivers takes, at minimum, 30-60 minutes. It is therefore recommended to build +a custom image and reuse it as shown below, rather than install GPU drivers at +boot time. + +> NOTE: the output `windows_startup_ps1` must be passed explicitly as shown +> below when used with Packer modules. This is due to a limitation in the `use` +> keyword and inputs of type `list` in Packer modules; this does not impact +> Terraform modules + +### NVIDIA Drivers and CUDA Toolkit + +Many Google Cloud VM families include or can have NVIDIA GPUs attached to them. +This module supports GPU applications by enabling you to easily install +a compatible release of NVIDIA drivers and of the CUDA Toolkit. The script is +the [solution recommended by our documentation][docs] and is [directly sourced +from GitHub][script-src]. + +[docs]: https://cloud.google.com/compute/docs/gpus/install-drivers-gpu#windows +[script-src]: https://github.com/GoogleCloudPlatform/compute-gpu-installation/blob/24dac3004360e0696c49560f2da2cd60fcb80107/windows/install_gpu_driver.ps1 + +```yaml +- group: primary + modules: + - id: network1 + source: modules/network/vpc + settings: + enable_iap_rdp_ingress: true + enable_iap_winrm_ingress: true + + - id: windows_startup + source: community/modules/scripts/windows-startup-script + settings: + install_nvidia_driver: true + +- group: packer + modules: + - id: image + source: modules/packer/custom-image + kind: packer + use: + - network1 + - windows_startup + settings: + source_image_family: windows-2016 + machine_type: n1-standard-8 + accelerator_count: 1 + accelerator_type: nvidia-tesla-t4 + disk_size: 75 + disk_type: pd-ssd + omit_external_ip: false + state_timeout: 15m +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [http\_proxy](#input\_http\_proxy) | Set http and https proxy for use by Invoke-WebRequest commands | `string` | `""` | no | +| [http\_proxy\_set\_environment](#input\_http\_proxy\_set\_environment) | Set system default environment variables http\_proxy and https\_proxy for all commands | `bool` | `false` | no | +| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install\_nvidia\_driver\_script | `bool` | `false` | no | +| [install\_nvidia\_driver\_args](#input\_install\_nvidia\_driver\_args) | Arguments to supply to NVIDIA driver install script | `string` | `"/s /n"` | no | +| [install\_nvidia\_driver\_script](#input\_install\_nvidia\_driver\_script) | Install script for NVIDIA drivers specified by http/https URL | `string` | `"https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe"` | no | +| [no\_proxy](#input\_no\_proxy) | Environment variables no\_proxy (only used if var.http\_proxy\_set\_environment is enabled) | `string` | `"169.254.169.254,metadata,metadata.google.internal,.googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | A string list of scripts selected by this module | + diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/main.tf new file mode 100644 index 0000000000..5e6bc8b94d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/main.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + setx_http_proxy_ps1 = !var.http_proxy_set_environment ? [] : [ + templatefile("${path.module}/templates/setx_http_proxy.ps1", { + "http_proxy" : var.http_proxy, + "no_proxy" : var.no_proxy, + }) + ] + + nvidia_ps1 = !var.install_nvidia_driver ? [] : [ + templatefile("${path.module}/templates/install_gpu_driver.ps1.tftpl", { + "url" : var.install_nvidia_driver_script + "args" : var.install_nvidia_driver_args + "http_proxy" : var.http_proxy, + }) + ] + + startup_ps1 = concat(local.setx_http_proxy_ps1, local.nvidia_ps1) +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf new file mode 100644 index 0000000000..006ea312ad --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "windows_startup_ps1" { + description = "A string list of scripts selected by this module" + value = local.startup_ps1 +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl new file mode 100644 index 0000000000..55c4a2a3cd --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl @@ -0,0 +1,38 @@ +#Requires -RunAsAdministrator + +# Windows 2016 needs forced upgrade to TLS 1.2 +[Net.ServicePointManager]::SecurityProtocol = 'Tls12' + +# important for catching exception in Invoke-WebRequest +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +%{ if http_proxy != "" } +[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") +%{ endif } + +# Create the folder for the driver download +$file_dir = 'C:\NVIDIA-Driver\nvidia_installer_windows.exe' +if (!(Test-Path -Path 'C:\NVIDIA-Driver')) { + New-Item -Path 'C:\' -Name 'NVIDIA-Driver' -ItemType 'directory' | Out-Null +} + +# Download the file to a specified directory +Write-Output "Downloading ${url} to $file_dir" +# Disabling progress bar has surprising large (10-100x) impact on speed +$ProgressPreference = 'SilentlyContinue' +try { + Invoke-WebRequest -Uri "${url}" -OutFile "$file_dir" +} catch { + Write-Output "$_" + throw "Failed to download ${url}; exiting startup script" +} + +# Install the file with the specified path from earlier as well as the RunAs admin option +Write-Output "Executing $file_dir with arguments '${args}'" +try { + Start-Process -FilePath "$file_dir" -ArgumentList '${args}' -Wait +} catch { + Write-Output "$_" + throw "Could not install NVIDIA driver; exiting startup script" +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 new file mode 100644 index 0000000000..ca4d13f98b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 @@ -0,0 +1,21 @@ +<# + Copyright 2025 "Google LLC" + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#> + +#Requires -RunAsAdministrator + +setx http_proxy ${http_proxy} /m +setx https_proxy ${http_proxy} /m +setx no_proxy ${no_proxy} /m diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf new file mode 100644 index 0000000000..9e4fb9e67d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "install_nvidia_driver" { + description = "Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install_nvidia_driver_script" + type = bool + default = false +} + +variable "install_nvidia_driver_script" { + description = "Install script for NVIDIA drivers specified by http/https URL" + type = string + default = "https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe" +} + +variable "install_nvidia_driver_args" { + description = "Arguments to supply to NVIDIA driver install script" + type = string + default = "/s /n" +} + +variable "http_proxy" { + description = "Set http and https proxy for use by Invoke-WebRequest commands" + type = string + default = "" + nullable = false +} + +variable "http_proxy_set_environment" { + description = "Set system default environment variables http_proxy and https_proxy for all commands" + type = bool + default = false + nullable = false +} + +variable "no_proxy" { + description = "Environment variables no_proxy (only used if var.http_proxy_set_environment is enabled)" + type = string + default = "169.254.169.254,metadata,metadata.google.internal,.googleapis.com" + nullable = false +} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf new file mode 100644 index 0000000000..dfeeac34f8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/cluster/modules/embedded/modules/README.md b/deletion-test/cluster/modules/embedded/modules/README.md new file mode 100644 index 0000000000..6886b3f330 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/README.md @@ -0,0 +1,554 @@ +# Modules + +This directory contains a set of core modules built for the Cluster Toolkit. Modules +describe the building blocks of an AI/ML and HPC deployment. The expected fields in a +module are listed in more detail [below](#module-fields). Blueprints can be +extended in functionality by incorporating [modules from GitHub +repositories][ghmods]. + +[ghmods]: #github-modules + +## All Modules + +Modules from various sources are all listed here for visibility. Badges are used +to indicate the source and status of many of these resources. + +Modules listed below with the ![core-badge] badge are located in this +folder and are tested and maintained by the Cluster Toolkit team. + +Modules labeled with the ![community-badge] badge are contributed by +the community (including the Cluster Toolkit team, partners, etc.). Community modules +are located in the [community folder](../community/modules/README.md). + +Modules labeled with the ![deprecated-badge] badge are now deprecated and may be +removed in the future. Customers are advised to transition to alternatives. + +Modules that are still in development and less stable are labeled with the +![experimental-badge] badge. + +[core-badge]: https://img.shields.io/badge/-core-blue?style=plastic +[community-badge]: https://img.shields.io/badge/-community-%23b8def4?style=plastic +[stable-badge]: https://img.shields.io/badge/-stable-lightgrey?style=plastic +[experimental-badge]: https://img.shields.io/badge/-experimental-%23febfa2?style=plastic +[deprecated-badge]: https://img.shields.io/badge/-deprecated-%23fea2a2?style=plastic + +### Compute + +* **[vm-instance]** ![core-badge] : Creates one or more VM instances. +* **[schedmd-slurm-gcp-v6-partition]** ![core-badge] : + Creates a partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. +* **[schedmd-slurm-gcp-v6-nodeset]** ![core-badge] : + Creates a nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. +* **[schedmd-slurm-gcp-v6-nodeset-tpu]** ![core-badge] : + Creates a TPU nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. +* **[schedmd-slurm-gcp-v6-nodeset-dynamic]** ![core-badge] ![experimental-badge]: + Creates a dynamic nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module and instance template. +* **[gke-node-pool]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes node pool using GKE. +* **[resource-policy]** ![core-badge] ![experimental-badge] : Create a resource policy for compute engines that can be applied to gke-node-pool's nodes. +* **[gke-job-template]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes job file to be used with a [gke-node-pool]. +* **[htcondor-execute-point]** ![community-badge] ![experimental-badge] : + Manages a group of execute points for use in an [HTCondor + pool][htcondor-setup]. +* **[mig]** ![community-badge] ![experimental-badge] : Creates a Managed Instance Group. +* **[notebook]** ![community-badge] ![experimental-badge] : Creates a Vertex AI + Notebook. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[gke-nodeset]** ![community-badge] ![experimental-badge] : Create a slinky nodeset to be used by the [gke-partition] module. +* **[gke-partition]** ![community-badge] ![experimental-badge] : Creates a slinky partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. + +[vm-instance]: compute/vm-instance/README.md +[gke-node-pool]: ../modules/compute/gke-node-pool/README.md +[resource-policy]: ../modules/compute/resource-policy/README.md +[gke-job-template]: ../modules/compute/gke-job-template/README.md +[schedmd-slurm-gcp-v6-partition]: ../community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md +[schedmd-slurm-gcp-v6-nodeset]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md +[schedmd-slurm-gcp-v6-nodeset-tpu]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md +[schedmd-slurm-gcp-v6-nodeset-dynamic]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md +[htcondor-execute-point]: ../community/modules/compute/htcondor-execute-point/README.md +[mig]: ../community/modules/compute/mig/README.md +[notebook]: ../community/modules/compute/notebook/README.md +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +### Database + +* **[slurm-cloudsql-federation]** ![community-badge] ![experimental-badge] : + Creates a [Google SQL Instance](https://cloud.google.com/sql/) meant to be + integrated with a [slurm-controller][schedmd-slurm-gcp-v6-controller]. +* **[bigquery-dataset]** ![community-badge] ![experimental-badge] : Creates a BQ + dataset. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[bigquery-table]** ![community-badge] ![experimental-badge] : Creates a BQ + table. Primarily used for + [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. + +[slurm-cloudsql-federation]: ../community/modules/database/slurm-cloudsql-federation/README.md +[bigquery-dataset]: ../community/modules/database/bigquery-dataset/README.md +[bigquery-table]: ../community/modules/database/bigquery-table/README.md +[fsi-montecarlo-on-batch]: ../community/modules/files/fsi-montecarlo-on-batch/README.md + +### File System + +* **[filestore]** ![core-badge] : Creates a + [filestore](https://cloud.google.com/filestore) file system. +* **[parallelstore]** ![core-badge] ![experimental-badge]: Creates a + [parallelstore](https://cloud.google.com/parallelstore) file system. +* **[pre-existing-network-storage]** ![core-badge] : Specifies a + pre-existing file system that can be mounted on a VM. +* **[managed-lustre]** ![core-badge] ![experimental-badge]: Creates a + [managed-lustred](https://cloud.google.com/managed-lustre) file system. +* **[DDN-EXAScaler]** ![community-badge] ![deprecated-badge] : Creates + a [DDN EXAscaler lustre](https://www.ddn.com/partners/google-cloud-platform/) + file system. This module is deprecated and will be removed by July 1, 2025. Consider migrating to managed-lustre. +* **[cloud-storage-bucket]** ![core-badge] : Creates a Google Cloud Storage (GCS) bucket. +* **[gke-persistent-volume]** ![core-badge] ![experimental-badge] : Creates + persistent volumes and persistent volume claims for shared storage. +* **[nfs-server]** ![community-badge] ![experimental-badge] : Creates a VM and + configures an NFS server that can be mounted by other VM. +* **[weka-client]** ![community-badge] ![experimental-badge] : Installs client + and mounts [WEKA](https://www.weka.io/) filesystems. + +[filestore]: file-system/filestore/README.md +[parallelstore]: file-system/parallelstore/README.md +[pre-existing-network-storage]: file-system/pre-existing-network-storage/README.md +[managed-lustre]: file-system/managed-lustre/README.md +[ddn-exascaler]: ../community/modules/file-system/DDN-EXAScaler/README.md +[nfs-server]: ../community/modules/file-system/nfs-server/README.md +[cloud-storage-bucket]: file-system/cloud-storage-bucket/README.md +[gke-persistent-volume]: file-system/gke-persistent-volume/README.md +[weka-client]: ../community/modules/file-system/weka-client/README.md + +### Monitoring + +* **[dashboard]** ![core-badge] : Creates a + [monitoring dashboard](https://cloud.google.com/monitoring/dashboards) for + visually tracking a Cluster Toolkit deployment. + +[dashboard]: monitoring/dashboard/README.md + +### Network + +* **[vpc]** ![core-badge] : Creates a + [Virtual Private Cloud (VPC)](https://cloud.google.com/vpc) network with + regional subnetworks and firewall rules. +* **[multivpc]** ![core-badge] ![experimental-badge]: Creates a variable + number of VPC networks using the [vpc] module. +* **[pre-existing-vpc]** ![core-badge] : Used to connect newly + built components to a pre-existing VPC network. +* **[firewall-rules]** ![core-badge] ![experimental-badge] : Add custom firewall + rules to existing networks (commonly used with [pre-existing-vpc]). +* **[private-service-access]** ![community-badge] ![experimental-badge] : + Configures Private Services Access for a VPC network (commonly used with [filestore] and [slurm-cloudsql-federation]). + +[vpc]: network/vpc/README.md +[multivpc]: network/multivpc/README.md +[pre-existing-vpc]: network/pre-existing-vpc/README.md +[firewall-rules]: network/firewall-rules/README.md +[private-service-access]: ../community/modules/network/private-service-access/README.md + +### Packer + +* **[custom-image]** ![core-badge] : Creates a custom VM Image + based on the GCP HPC VM image. + +[custom-image]: packer/custom-image/README.md + +### Project + +* **[service-account]** ![community-badge] ![experimental-badge] : Creates [service + accounts](https://cloud.google.com/iam/docs/service-accounts) for a GCP + project. +* **[service-enablement]** ![community-badge] ![experimental-badge] : Allows enabling + various APIs for a Google Cloud Project. + +[service-account]: ../community/modules/project/service-account/README.md +[service-enablement]: ../community/modules/project/service-enablement/README.md + +### Pub/Sub + +* **[topic]** ![community-badge] ![experimental-badge] : Creates a +Pub/Sub topic. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[bigquery-sub]** ![community-badge] ![experimental-badge] : Creates a +Pub/Sub subscription. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. + +[topic]: ../community/modules/pubsub/topic/README.md +[bigquery-sub]: ../community/modules/pubsub/bigquery-sub/README.md + +### Remote Desktop + +* **[chrome-remote-desktop]** ![community-badge] ![experimental-badge] : Creates + a GPU accelerated Chrome Remote Desktop. + +[chrome-remote-desktop]: ../community/modules/remote-desktop/chrome-remote-desktop/README.md + +### Scheduler + +* **[batch-job-template]** ![core-badge] : Creates a Google Cloud Batch job + template that works with other Toolkit modules. +* **[batch-login-node]** ![core-badge] : Creates a VM that can be used for + submission of Google Cloud Batch jobs. +* **[gke-cluster]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes cluster using GKE. +* **[pre-existing-gke-cluster]** ![core-badge] ![experimental-badge] : Retrieves an existing GKE cluster. Substitute for ([gke-cluster]) module. +* **[schedmd-slurm-gcp-v6-controller]** ![core-badge] : + Creates a Slurm controller node. +* **[schedmd-slurm-gcp-v6-login]** ![core-badge] : + Creates a Slurm login node. +* **[htcondor-setup]** ![community-badge] ![experimental-badge] : Creates the + base infrastructure for an HTCondor pool (service accounts and Cloud Storage bucket). +* **[htcondor-pool-secrets]** ![community-badge] ![experimental-badge] : Creates + and manages access to the secrets necessary for secure operation of an + HTCondor pool. +* **[htcondor-access-point]** ![community-badge] ![experimental-badge] : Creates + a regional instance group managing a highly available HTCondor access point + (login node). + +[batch-job-template]: ../modules/scheduler/batch-job-template/README.md +[batch-login-node]: ../modules/scheduler/batch-login-node/README.md +[gke-cluster]: ../modules/scheduler/gke-cluster/README.md +[pre-existing-gke-cluster]: ../modules/scheduler/pre-existing-gke-cluster/README.md +[htcondor-setup]: ../community/modules/scheduler/htcondor-setup/README.md +[htcondor-pool-secrets]: ../community/modules/scheduler/htcondor-pool-secrets/README.md +[htcondor-access-point]: ../community/modules/scheduler/htcondor-access-point/README.md +[schedmd-slurm-gcp-v6-controller]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md +[schedmd-slurm-gcp-v6-login]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md + +### Scripts + +* **[startup-script]** ![core-badge] : Creates a customizable startup script + that can be fed into compute VMs. +* **[windows-startup-script]** ![community-badge] ![experimental-badge]: Creates + Windows PowerShell (PS1) scripts that can be used to customize Windows VMs + and VM images. +* **[htcondor-install]** ![community-badge] ![experimental-badge] : Creates + a startup script to install HTCondor and exports a list of required APIs +* **[ramble-execute]** ![community-badge] ![experimental-badge] : Creates a + startup script to execute + [Ramble](https://github.com/GoogleCloudPlatform/ramble) commands on a target + VM +* **[ramble-setup]** ![community-badge] ![experimental-badge] : Creates a + startup script to install + [Ramble](https://github.com/GoogleCloudPlatform/ramble) on an instance or a + slurm login or controller. +* **[spack-setup]** ![community-badge] ![experimental-badge] : Creates a startup + script to install [Spack](https://github.com/spack/spack) on an instance or a + slurm login or controller. +* **[spack-execute]** ![community-badge] ![experimental-badge] : Defines a + software build using [Spack](https://github.com/spack/spack). +* **[wait-for-startup]** ![community-badge] ![experimental-badge] : Waits for + successful completion of a startup script on a compute VM. + +[startup-script]: scripts/startup-script/README.md +[windows-startup-script]: ../community/modules/scripts/windows-startup-script/README.md +[htcondor-install]: ../community/modules/scripts/htcondor-install/README.md +[kubernetes-operations]: ../community/modules/scripts/kubernetes-operations/README.md +[ramble-execute]: ../community/modules/scripts/ramble-execute/README.md +[ramble-setup]: ../community/modules/scripts/ramble-setup/README.md +[spack-setup]: ../community/modules/scripts/spack-setup/README.md +[spack-execute]: ../community/modules/scripts/spack-execute/README.md +[wait-for-startup]: ../community/modules/scripts/wait-for-startup/README.md + +## Module Fields + +### ID (Required) + +The `id` field is used to uniquely identify and reference a defined module. +ID's are used in [variables](../examples/README.md#variables) and become the +name of each module when writing the terraform `main.tf` file. They are also +used in the [use](#use-optional) and [outputs](#outputs-optional) lists +described below. + +For terraform modules, the ID will be rendered into the terraform module label +at the top level main.tf file. + +### Source (Required) + +The source is a path or URL that points to the source files for Packer or +Terraform modules. A source can either be a filesystem path or a URL to a git +repository: + +* Filesystem paths + * modules embedded in the `gcluster` executable + * modules in the local filesystem +* Remote modules using [Terraform URL syntax](https://developer.hashicorp.com/terraform/language/modules/sources) + * Hosted on [GitHub](https://developer.hashicorp.com/terraform/language/modules/sources#github) + * Google Cloud Storage [Buckets](https://developer.hashicorp.com/terraform/language/modules/sources#gcs-bucket) + * Generic [git repositories](https://developer.hashicorp.com/terraform/language/modules/sources#generic-git-repository) + + when modules are in a subdirectory of the git repository, a special + double-slash `//` notation can be required as described below + +An important distinction is that those URLs are natively supported by Terraform so +they are not copied to your deployment directory. Packer does not have native +support for git-hosted modules so the Toolkit will copy these modules into the +deployment folder on your behalf. + +#### Embedded Modules + +Embedded modules are added to the gcluster binary during compilation and cannot +be edited. To refer to embedded modules, set the source path to +`modules/<>` or `community/modules/<>`. + +The paths match the modules in the repository structure for [core modules](./) +and [community modules](../community/modules/). Because the modules are embedded +during compilation, your local copies may differ unless you recompile gcluster. + +For example, this example snippet uses the embedded pre-existing-vpc module: + +```yaml + - id: network1 + source: modules/network/pre-existing-vpc +``` + +#### Local Modules + +Local modules point to a module in the file system and can easily be edited. +They are very useful during module development. To use a local module, set +the source to a path starting with `/`, `./`, or `../`. For instance, the +following module definition refers the local pre-existing-vpc modules. + +```yaml + - id: network1 + source: modules/network/pre-existing-vpc +``` + +> **_NOTE:_** Relative paths (beginning with `.` or `..` must be relative to the +> working directory from which `gcluster` is executed. This example would have to be +> run from a local copy of the Cluster Toolkit repository. An alternative is to use +> absolute paths to modules. + +#### GitHub-hosted Modules and Packages + +To use a Terraform module available on GitHub, set the source to a path starting +with `github.com` (HTTPS) or `git@github.com` (SSH). For instance, the following +module definition sources the Toolkit vpc module: + +```yaml + - id: network1 + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc +``` + +This example uses the [double-slash notation][tfsubdir] (`//`) to indicate that +the Toolkit is a "package" of multiple modules whose root directory is the root +of the git repository. The remainder of the path indicates the sub-directory of +the vpc module. + +The example above uses the default `main` branch of the Toolkit. Specific +[revisions][tfrev] can be selected with any valid [git reference][gitref]. +(git branch, commit hash or tag). If the git reference is a tag or branch, we +recommend setting `&depth=1` to reduce the data transferred over the network. +This option cannot be set when the reference is a commit hash. The following +examples select the vpc module on the active `develop` branch and also an older +release of the filestore module: + +```yaml + - id: network1 + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc?ref=develop + ... + - id: homefs + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/file-system/filestore?ref=v1.22.1&depth=1 +``` + +Because Terraform modules natively support this syntax, gcluster will not copy +GitHub-hosted modules into your deployment folder. Terraform will download them +into a hidden folder when you run `terraform init`. + +[tfrev]: https://www.terraform.io/language/modules/sources#selecting-a-revision +[gitref]: https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection#_single_revisions +[tfsubdir]: https://www.terraform.io/language/modules/sources#modules-in-package-sub-directories + +##### GitHub-hosted Packer modules + +Packer does not natively support GitHub-hosted modules so `gcluster create` will +copy modules into your deployment folder. + +If the module uses `//` package notation, `gcluster create` will copy the entire +repository to the module path: `deployment_name/group_name/module_id`. However, +when `gcluster deploy` is invoked, it will run Packer from the subdirectory +`deployment_name/group_name/module_id/subdirectory/after/double_slash`. + +If the module does not use `//` package notation, `gcluster create` will copy +only the final directory in the path to `deployment_name/group_name/module_id`. + +In all cases, `gcluster create` will remove the `.git` directory from the packer +module to ensure that you can manage the entire deployment directory with its +own git versioning. + +##### GitHub over SSH + +Get module from GitHub over SSH: + +```yaml + - id: network1 + source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc +``` + +Specific versions can be selected as for HTTPS: + +```yaml + - id: network1 + source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc?ref=v1.22.1&depth=1 +``` + +##### Generic Git Modules + +To use a Terraform module available in a non-GitHub git repository such as +gitlab, set the source to a path starting `git::`. Two Standard git protocols +are supported, `git::https://` for HTTPS or `git::git@github.com` for SSH. + +Additional formatting and features after `git::` are identical to that of the +[GitHub Modules](#github-modules) described above. + +#### Google Cloud Storage Modules + +To use a Terraform module available in a Google Cloud Storage bucket, set the source +to a URL with the special `gcs::` prefix, followed by a [GCS bucket object URL](https://cloud.google.com/storage/docs/request-endpoints#typical). + +For example: `gcs::https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH_TO_MODULE` + +### Kind (May be Required) + +`kind` refers to the way in which a module is deployed. Currently, `kind` can be +either `terraform` or `packer`. It must be specified for modules of type +`packer`. If omitted, it will default to `terraform`. + +### Settings (May Be Required) + +The settings field is a map that supplies any user-defined variables for each +module. Settings values can be simple strings, numbers or booleans, but can +also support complex data types like maps and lists of variable depth. These +settings will become the values for the variables defined in either the +`variables.tf` file for Terraform or `variable.pkr.hcl` file for Packer. + +For some modules, there are mandatory variables that must be set, +therefore `settings` is a required field in that case. In many situations, a +combination of sensible defaults, deployment variables and used modules can +populated all required settings and therefore the settings field can be omitted. + +### Use (Optional) + +The `use` field is a powerful way of linking a module to one or more other +modules. When a module "uses" another module, the outputs of the used +module are compared to the settings of the current module. If they have +matching names and the setting has no explicit value, then it will be set to +the used module's output. For example, see the following blueprint snippet: + +```yaml +modules: +- id: network1 + source: modules/network/vpc + +- id: workstation + source: modules/compute/vm-instance + use: [network1] + settings: + ... +``` + +In this snippet, the VM instance `workstation` uses the outputs of vpc +`network1`. + +In this case both `network_self_link` and `subnetwork_self_link` in the +[workstation settings](compute/vm-instance/README.md#Inputs) will be set +to `$(network1.network_self_link)` and `$(network1.subnetwork_self_link)` which +refer to the [network1 outputs](network/vpc/README#Outputs) +of the same names. + +The order of precedence that `gcluster` uses in determining when to infer a setting +value is in the following priority order: + +1. Explicitly set in the blueprint using the `settings` field +1. Output from a used module, taken in the order provided in the `use` list +1. Deployment variable (`vars`) of the same name +1. Default value for the setting + +> **_NOTE:_** See the +> [network storage documentation](./../docs/network_storage.md) for more +> information about mounting network storage file systems via the `use` field. + +### Outputs (Optional) + +The `outputs` field adds the output of individual Terraform modules to the +output of its deployment group. This enables the value to be available via +`terraform output`. This can useful for displaying the IP of a login node or +printing instructions on how to use a module, as we have in the +[monitoring dashboard module](monitoring/dashboard/README.md#Outputs). + +The outputs field is a lists that it can be in either of two formats: a string +equal to the name of the module output, or a map specifying the `name`, +`description`, and whether the value is `sensitive` and should be suppressed +from the standard output of Terraform commands. An example is shown below +that displays the internal and public IP addresses of a VM created by the +vm-instance module: + +```yaml + - id: vm + source: modules/compute/vm-instance + use: + - network1 + settings: + machine_type: e2-medium + outputs: + - internal_ip + - name: external_ip + description: "External IP of VM" + sensitive: true +``` + +The outputs shown after running Terraform apply will resemble: + +```text +Apply complete! Resources: 7 added, 0 changed, 0 destroyed. + +Outputs: + +external_ip_simplevm = +internal_ip_simplevm = [ + "10.128.0.19", +] +``` + +### Required Services (APIs) (optional) + +Each Toolkit module depends upon Google Cloud services ("APIs") being enabled +in the project used by the AI/ML and HPC environment. For example, the [creation of +VMs](compute/vm-instance/) requires the Compute Engine API +(compute.googleapis.com). The [startup-script](scripts/startup-script/) module +requires the Cloud Storage API (storage.googleapis.com) for storage of the +scripts themselves. Each module included in the Toolkit source code describes +its required APIs internally. The Toolkit will merge the requirements from all +modules and [automatically validate](../README.md#blueprint-validation) that all +APIs are enabled in the project specified by `$(vars.project_id)`. + +## Common Settings + +The following common naming conventions should be used to decrease the verbosity +needed to define a blueprint. This is intentional to allow multiple +modules to share inferred settings from deployment variables or from other +modules listed under the `use` field. + +For example, if all modules are to be created in a single region, that region +can be defined as a deployment variable named `region`, which is shared between +all modules without an explicit setting. Similarly, if many modules need to be +connected to the same VPC network, they all can add the vpc module ID to their +`use` list so that `network_self_link` would be inferred from that vpc module rather +than having to set it manually. + +* **project_id**: The GCP project ID in which to create the GCP resources. +* **deployment_name**: The name of the current deployment of a blueprint. This + can help to avoid naming conflicts of modules when multiple deployments are + created from the same blueprint. +* **region**: The GCP + [region](https://cloud.google.com/compute/docs/regions-zones) the module + will be created in. +* **zone**: The GCP [zone](https://cloud.google.com/compute/docs/regions-zones) + the module will be created in. +* **labels**: + [Labels](https://cloud.google.com/resource-manager/docs/creating-managing-labels) + added to the module. In order to include any module in advanced + monitoring, labels must be exposed. We strongly recommend that all modules + expose this variable. + +## Writing Custom Cluster Toolkit Modules + +Modules are flexible by design, however we define some [best practices](../docs/module-guidelines.md) when +creating a new module meant to be used with the Cluster Toolkit. diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/README.md b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/README.md new file mode 100644 index 0000000000..f807cd727e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/README.md @@ -0,0 +1,133 @@ +## Description + +This module is used to create a Kubernetes job template file. + +The job template file can be submitted as is or used as a template for further +customization. Add the `instructions` output to a blueprint (as shown below) to +get instructions on how to use `kubectl` to submit the job. + +This module is designed to `use` one or more `gke-node-pool` modules. The job +will be configured to run on any of the specified node pools. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a GKE job template file. + +```yaml + - id: job-template + source: modules/compute/gke-job-template + use: [compute_pool] + settings: + node_count: 3 + outputs: [instructions] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### Storage Options + +This module natively supports: + +* Filestore as a shared file system between pods/nodes. +* Pod level ephemeral storage options: + * memory backed emptyDir + * local SSD backed emptyDir + * SSD persistent disk backed ephemeral volume + * balanced persistent disk backed ephemeral volume + +See the [storage-gke.yaml blueprint](../../../examples/storage-gke.yaml) and the +associated [documentation](../../../../examples/README.md#storage-gkeyaml--) for +examples of how to use Filestore and ephemeral storage with this module. + +### Requested Resources + +When one or more `gke-node-pool` modules are referenced with the `use` field. +The requested resources will be populated to achieve a 1 pod per node packing +while still leaving some headroom for required system pods. + +This functionality can be overridden by specifying the desired cpu requirement +using the `requested_cpu_per_pod` setting. + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.2 | +| [local](#requirement\_local) | >= 2.0.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allocatable\_cpu\_per\_node](#input\_allocatable\_cpu\_per\_node) | The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | +| [allocatable\_gpu\_per\_node](#input\_allocatable\_gpu\_per\_node) | The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | +| [backoff\_limit](#input\_backoff\_limit) | Controls the number of retries before considering a Job as failed. Set to zero for shared fate. | `number` | `0` | no | +| [command](#input\_command) | The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes. | `list(string)` | 
[
  "hostname"
]
| no | +| [completion\_mode](#input\_completion\_mode) | Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information | `string` | `"Indexed"` | no | +| [ephemeral\_volumes](#input\_ephemeral\_volumes) | Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB. | 
list(object({
    type       = string
    mount_path = string
    size_gb    = number
  }))
| `[]` | no | +| [has\_gpu](#input\_has\_gpu) | Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module. | `list(bool)` | 
[
  false
]
| no | +| [image](#input\_image) | The container image the job should use. | `string` | `"debian"` | no | +| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account to run the job as. If null then no service account is specified. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the GKE job template. Key-value pairs. | `map(string)` | n/a | yes | +| [machine\_family](#input\_machine\_family) | The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria. | `string` | `null` | no | +| [name](#input\_name) | The name of the job. | `string` | `"my-job"` | no | +| [node\_count](#input\_node\_count) | How many nodes the job should run in parallel. | `number` | `1` | no | +| [node\_pool\_names](#input\_node\_pool\_names) | A list of node pool names on which to run the job. Can be populated via `use` field. | `list(string)` | `[]` | no | +| [node\_selectors](#input\_node\_selectors) | A list of node selectors to use to place the job. | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | +| [persistent\_volume\_claims](#input\_persistent\_volume\_claims) | A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module. | 
list(object({
    name          = string
    namespace     = string
    mount_path    = string
    mount_options = string
    storage_type  = string
  }))
| `[]` | no | +| [random\_name\_sufix](#input\_random\_name\_sufix) | Appends a random suffix to the job name to avoid clashes. | `bool` | `true` | no | +| [requested\_cpu\_per\_pod](#input\_requested\_cpu\_per\_pod) | The requested cpu per pod. If null, allocatable\_cpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_cpu\_per\_node. | `number` | `-1` | no | +| [requested\_gpu\_per\_pod](#input\_requested\_gpu\_per\_pod) | The requested gpu per pod. If null, allocatable\_gpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_gpu\_per\_node. | `number` | `-1` | no | +| [restart\_policy](#input\_restart\_policy) | Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed. | `string` | `"Never"` | no | +| [security\_context](#input\_security\_context) | The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | +| [tolerations](#input\_tolerations) | Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field. | 
list(object({
    key      = string
    operator = string
    value    = string
    effect   = string
  }))
| 
[
  {
    "effect": "NoSchedule",
    "key": "user-workload",
    "operator": "Equal",
    "value": "true"
  }
]
| no | +| [tpu\_accelerator\_type](#input\_tpu\_accelerator\_type) | The TPU accelerator type label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | +| [tpu\_chips\_per\_node](#input\_tpu\_chips\_per\_node) | The number of TPU chips per node. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | +| [tpu\_topology](#input\_tpu\_topology) | The TPU topology label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for submitting the GKE job. | + diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/main.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/main.tf new file mode 100644 index 0000000000..e84138bb3f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/main.tf @@ -0,0 +1,181 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-job-template", ghpc_role = "compute" }) +} + +locals { + tpu_accelerator_node_selector = var.tpu_accelerator_type[0] != null ? [{ + key = "cloud.google.com/gke-tpu-accelerator" + value = var.tpu_accelerator_type[0] + }] : [] + + tpu_topology_node_selector = var.tpu_topology[0] != null ? [{ + key = "cloud.google.com/gke-tpu-topology" + value = var.tpu_topology[0] + }] : [] +} + +locals { + # Start with the minimum cpu available of used node pools + min_allocatable_cpu = min(var.allocatable_cpu_per_node...) + full_node_cpu_request = ( + local.min_allocatable_cpu > 2 ? # if large enough + local.min_allocatable_cpu - 1 : # leave headroom for 1 cpu + local.min_allocatable_cpu / 2 + 0.1 # else take just over half + ) - (local.any_gcs ? 0.25 : 0) # save room for gcs side car + + cpu_request = ( + var.requested_cpu_per_pod >= 0 ? # if user supplied requested cpu + var.requested_cpu_per_pod : # then honor it + ( # else + local.min_allocatable_cpu >= 0 ? # if allocatable cpu was supplied + local.full_node_cpu_request : # then claim the full node + -1 # else do not set a limit + ) + ) + millicpu = floor(local.cpu_request * 1000) + cpu_request_string = local.millicpu >= 0 ? "${local.millicpu}m" : null + full_node_request = local.min_allocatable_cpu >= 0 && var.requested_cpu_per_pod < 0 + + memory_request_value = try(sum([for ed in var.ephemeral_volumes : + ed.size_gb + if ed.type == "memory" + ]), 0) + memory_request_string = local.memory_request_value > 0 ? "${local.memory_request_value}Gi" : null + + ephemeral_request_value = try(sum([for ed in var.ephemeral_volumes : + ed.size_gb + if ed.type == "local-ssd" + ]), 0) + ephemeral_request_string = local.ephemeral_request_value > 0 ? "${local.ephemeral_request_value}Gi" : null + + uses_local_ssd = anytrue([for ed in var.ephemeral_volumes : + ed.type == "local-ssd" + ]) + local_ssd_node_selector = local.uses_local_ssd ? [{ + key = "cloud.google.com/gke-ephemeral-storage-local-ssd" + value = "true" + }] : [] + + # Setup limit for GPUs per pod + min_allocatable_gpu = min(var.allocatable_gpu_per_node...) + min_allocatable_gpu_per_pod = local.min_allocatable_gpu > 0 ? local.min_allocatable_gpu : null + gpu_limit_per_pod = var.requested_gpu_per_pod > 0 ? var.requested_gpu_per_pod : local.min_allocatable_gpu_per_pod + gpu_limit_string = alltrue(var.has_gpu) ? tostring(local.gpu_limit_per_pod) : null + + empty_dir_volumes = [for ed in var.ephemeral_volumes : + { + name = replace(trim(ed.mount_path, "/"), "/", "-") + mount_path = ed.mount_path + size_limit = "${ed.size_gb}Gi" + in_memory = ed.type == "memory" + } + if contains(["memory", "local-ssd"], ed.type) + ] + + ephemeral_pd_volumes = [for pd in var.ephemeral_volumes : + { + name = replace(trim(pd.mount_path, "/"), "/", "-") + mount_path = pd.mount_path + storage_class_name = pd.type == "pd-ssd" ? "premium-rwo" : "standard-rwo" + storage = "${pd.size_gb}Gi" + } + if contains(["pd-balanced", "pd-ssd"], pd.type) + ] + + pvc_volumes = [for pvc in var.persistent_volume_claims : + { + name = replace(trim(pvc.mount_path, "/"), "/", "-") + mount_path = pvc.mount_path + claim_name = pvc.name + } + ] + + volume_mounts = [for v in concat(local.empty_dir_volumes, local.ephemeral_pd_volumes, local.pvc_volumes) : + { + name = v.name + mount_path = v.mount_path + } + ] + + suffix = var.random_name_sufix ? "-${random_id.resource_name_suffix.hex}" : "" + machine_family_node_selector = var.machine_family != null ? [{ + key = "cloud.google.com/machine-family" + value = var.machine_family + }] : [] + node_selectors = concat(local.machine_family_node_selector, local.local_ssd_node_selector, local.tpu_accelerator_node_selector, local.tpu_topology_node_selector, var.node_selectors) + + any_gcs = anytrue([for pvc in var.persistent_volume_claims : + pvc.storage_type == "gcs" + ]) + + job_template_contents = templatefile( + "${path.module}/templates/gke-job-base.yaml.tftpl", + { + name = var.name + suffix = local.suffix + image = var.image + command = var.command + node_count = var.node_count + completion_mode = var.completion_mode + k8s_service_account_name = var.k8s_service_account_name + node_pool_names = var.node_pool_names + node_selectors = local.node_selectors + tpu_limit = var.tpu_chips_per_node[0] + full_node_request = local.full_node_request + cpu_request = local.cpu_request_string + gpu_limit = local.gpu_limit_string + restart_policy = var.restart_policy + backoff_limit = var.backoff_limit + tolerations = distinct(var.tolerations) + security_context = var.security_context + labels = local.labels + + empty_dir_volumes = local.empty_dir_volumes + ephemeral_pd_volumes = local.ephemeral_pd_volumes + pvc_volumes = local.pvc_volumes + volume_mounts = local.volume_mounts + memory_request = local.memory_request_string + ephemeral_request = local.ephemeral_request_string + gcs_annotation = local.any_gcs + } + ) + + job_template_output_path = "${path.root}/${var.name}${local.suffix}.yaml" + +} + +resource "random_id" "resource_name_suffix" { + byte_length = 2 + keepers = { + timestamp = timestamp() + } +} + +resource "local_file" "job_template" { + content = local.job_template_contents + filename = local.job_template_output_path + + lifecycle { + precondition { + condition = local.any_gcs ? var.k8s_service_account_name != null : true + error_message = "When using GCS, a kubernetes service account with workload identity is required. gke-cluster module will perform this setup when var.configure_workload_identity_sa is set to true." + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/outputs.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/outputs.tf new file mode 100644 index 0000000000..adf78e936d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instructions" { + description = "Instructions for submitting the GKE job." + value = <<-EOT + A GKE job file has been created locally at: + ${abspath(local.job_template_output_path)} + + Use the following commands to: + Submit your job: + kubectl create -f ${abspath(local.job_template_output_path)} + EOT +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl new file mode 100644 index 0000000000..11df39ce2c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl @@ -0,0 +1,128 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ${name}${suffix} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + parallelism: ${node_count} + completions: ${node_count} + completionMode: ${completion_mode} + template: + %{~ if gcs_annotation ~} + metadata: + annotations: + gke-gcsfuse/volumes: "true" + %{~ endif ~} + spec: + %{~ if length(security_context) > 0 ~} + securityContext: + %{~ for context in security_context ~} + ${context.key}: ${context.value} + %{~ endfor ~} + %{~ endif ~} + %{~ if k8s_service_account_name != null ~} + serviceAccountName: ${k8s_service_account_name} + %{~ endif ~} + %{~ if length(node_pool_names) > 0 ~} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + %{~ for node_pool in node_pool_names ~} + - ${node_pool} + %{~ endfor ~} + %{~ endif ~} + %{~ if length(node_selectors) > 0 ~} + nodeSelector: + %{~ for selector in node_selectors ~} + ${selector.key}: "${selector.value}" + %{~ endfor ~} + %{~ endif ~} + tolerations: + %{~ for toleration in tolerations ~} + - key: ${toleration.key} + operator: ${toleration.operator} + value: "${toleration.value}" + effect: ${toleration.effect} + %{~ endfor ~} + containers: + - name: ${name}-container + image: ${image} + command: + %{for s in command}- ${indent(8, yamlencode(s))}%{~ endfor } + %{~ if gpu_limit != null || cpu_request != null || tpu_limit != null ~} + resources: + %{~ if gpu_limit != null || tpu_limit != null ~} + limits: + %{~ if gpu_limit != null ~} + # GPUs should only be specified as limits + # https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/ + nvidia.com/gpu: ${gpu_limit} + %{~ endif ~} + %{~ if tpu_limit != null ~} + google.com/tpu: ${tpu_limit} + %{~ endif ~} + %{~ endif ~} + %{~ if cpu_request != null || memory_request != null || ephemeral_request != null || tpu_limit != null ~} + requests: + %{~ if full_node_request ~} + # cpu request attempts full node per pod + %{~ endif ~} + %{~ if cpu_request != null ~} + cpu: ${cpu_request} + %{~ endif ~} + %{~ if tpu_limit != null ~} + google.com/tpu: ${tpu_limit} + %{~ endif ~} + %{~ if memory_request != null ~} + memory: ${memory_request} + %{~ endif ~} + %{~ if ephemeral_request != null ~} + ephemeral-storage: ${ephemeral_request} + %{~ endif ~} + %{~ endif ~} + %{~ endif ~} + %{~ if length(volume_mounts) > 0 ~} + volumeMounts: + %{~ for v in volume_mounts ~} + - name: ${v.name} + mountPath: ${v.mount_path} + %{~ endfor ~} + %{~ endif ~} + %{~ if length(volume_mounts) > 0 ~} + volumes: + %{~ for ed in empty_dir_volumes ~} + - name: ${ed.name} + emptyDir: + sizeLimit: ${ed.size_limit} + %{~ if ed.in_memory ~} + medium: "Memory" + %{~ endif ~} + %{~ endfor ~} + %{~ for pd in ephemeral_pd_volumes ~} + - name: ${pd.name} + ephemeral: + volumeClaimTemplate: + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: ${pd.storage_class_name} + resources: + requests: + storage: ${pd.storage} + %{~ endfor ~} + %{~ for pvc in pvc_volumes ~} + - name: ${pvc.name} + persistentVolumeClaim: + claimName: ${pvc.claim_name} + %{~ endfor ~} + %{~ endif ~} + restartPolicy: ${restart_policy} + backoffLimit: ${backoff_limit} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/variables.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/variables.tf new file mode 100644 index 0000000000..fd83f2b692 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/variables.tf @@ -0,0 +1,206 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "name" { + description = "The name of the job." + type = string + default = "my-job" +} + +variable "node_count" { + description = "How many nodes the job should run in parallel." + type = number + default = 1 +} + +variable "completion_mode" { + description = "Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information" + type = string + default = "Indexed" +} + +variable "command" { + description = "The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes." + type = list(string) + default = ["hostname"] +} + +variable "image" { + description = "The container image the job should use." + type = string + default = "debian" +} + +variable "k8s_service_account_name" { + description = "Kubernetes service account to run the job as. If null then no service account is specified." + type = string + default = null +} + +variable "node_pool_names" { + description = "A list of node pool names on which to run the job. Can be populated via `use` field." + type = list(string) + default = [] +} + +variable "allocatable_cpu_per_node" { + description = "The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." + type = list(number) + default = [-1] +} + +variable "has_gpu" { + description = "Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module." + type = list(bool) + default = [false] +} + +variable "requested_cpu_per_pod" { + description = "The requested cpu per pod. If null, allocatable_cpu_per_node will be used to claim whole nodes. If provided will override allocatable_cpu_per_node." + type = number + default = -1 +} + +variable "allocatable_gpu_per_node" { + description = "The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." + type = list(number) + default = [-1] +} + +variable "requested_gpu_per_pod" { + description = "The requested gpu per pod. If null, allocatable_gpu_per_node will be used to claim whole nodes. If provided will override allocatable_gpu_per_node." + type = number + default = -1 +} + +variable "tolerations" { + description = "Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field." + type = list(object({ + key = string + operator = string + value = string + effect = string + })) + default = [ + { + key = "user-workload" + operator = "Equal" + value = "true" + effect = "NoSchedule" + } + ] +} + +variable "security_context" { + description = "The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/" + type = list(object({ + key = string + value = string + })) + default = [] +} + +variable "machine_family" { + description = "The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria." + type = string + default = null +} + +variable "node_selectors" { + description = "A list of node selectors to use to place the job." + type = list(object({ + key = string + value = string + })) + default = [] +} + +variable "restart_policy" { + description = "Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed." + type = string + default = "Never" +} + +variable "backoff_limit" { + description = "Controls the number of retries before considering a Job as failed. Set to zero for shared fate." + type = number + default = 0 +} + +variable "random_name_sufix" { + description = "Appends a random suffix to the job name to avoid clashes." + type = bool + default = true +} + +variable "persistent_volume_claims" { + description = "A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module." + type = list(object({ + name = string + namespace = string + mount_path = string + mount_options = string + storage_type = string + })) + default = [] +} + +variable "ephemeral_volumes" { + description = "Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB." + type = list(object({ + type = string + mount_path = string + size_gb = number + })) + default = [] + validation { + condition = alltrue([ + for v in var.ephemeral_volumes : + contains(["pd-balanced", "pd-ssd", "memory", "local-ssd"], v.type) + ]) + error_message = "Type must be one of 'pd-balanced', 'pd-ssd', 'memory', 'local-ssd'." + } + validation { + condition = alltrue([ + for v in var.ephemeral_volumes : + substr(v.mount_path, 0, 1) == "/" + ]) + error_message = "Mount path must start with the '/' character." + } +} + +variable "labels" { + description = "Labels to add to the GKE job template. Key-value pairs." + type = map(string) +} + +variable "tpu_accelerator_type" { + description = "The TPU accelerator type label. Populated from gke-node-pool via `use` field." + type = list(string) + default = [null] +} + +variable "tpu_topology" { + description = "The TPU topology label. Populated from gke-node-pool via `use` field." + type = list(string) + default = [null] +} + +variable "tpu_chips_per_node" { + description = "The number of TPU chips per node. Populated from gke-node-pool via `use` field." + type = list(string) + default = [null] +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/versions.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/versions.tf new file mode 100644 index 0000000000..0f902ac8c5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/versions.tf @@ -0,0 +1,28 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.2" + + required_providers { + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/README.md b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/README.md new file mode 100644 index 0000000000..b25d905252 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/README.md @@ -0,0 +1,388 @@ +## Description + +This module creates a Google Kubernetes Engine +([GKE](https://cloud.google.com/kubernetes-engine)) node pool. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a GKE node group. + +```yaml + - id: compute_pool + source: modules/compute/gke-node-pool + use: [gke_cluster] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### Taints and Tolerations + +By default node pools created with this module will be tainted with +`user-workload=true:NoSchedule` to prevent system pods from being scheduled. +User jobs targeting the node pool should include this toleration. This behavior +can be overridden using the `taints` setting. See +[docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-taints) for +more info. + +### Local SSD Storage +GKE offers two options for managing locally attached SSDs. + +The first, and recommended, option is for GKE to manage the ephemeral storage +space on the node, which will then be automatically attached to pods which +request an `emptyDir` volume. This can be accomplished using the +[`local_ssd_count_ephemeral_storage`] variable. + +The second, more complex, option is for GCP to attach these nodes as raw block +storage. In this case, the cluster administrator is responsible for software +RAID settings, partitioning, formatting and mounting these disks on the host +OS. Still, this may be desired behavior in use cases which aren't supported +by an `emptyDir` volume (for example, a `ReadOnlyMany` or `ReadWriteMany` PV). +This can be accomplished using the [`local_ssd_count_nvme_block`] variable. + +The [`local_ssd_count_ephemeral_storage`] and [`local_ssd_count_nvme_block`] +variables are mutually exclusive and cannot be mixed together. + +Also, the number of SSDs which can be attached to a node depends on the +[machine type](https://cloud.google.com/compute/docs/disks#local_ssd_machine_type_restrictions). + +See [docs](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd) +for more info. + +[`local_ssd_count_ephemeral_storage`]: #input\_local\_ssd\_count\_ephemeral\_storage +[`local_ssd_count_nvme_block`]: #input\_local\_ssd\_count\_nvme\_block + +### Considerations with GPUs + +When a GPU is attached to a node an additional taint is automatically added: +`nvidia.com/gpu=present:NoSchedule`. For jobs to get placed on these nodes, the +equivalent toleration is required. The `gke-job-template` module will +automatically apply this toleration when using a node pool with GPUs. + +Nvidia GPU drivers must be installed. The recommended approach for GKE to install +GPU dirvers is by applying a DaemonSet to the cluster. See +[these instructions](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus#cos). + +However, in some cases it may be desired to compile a different driver (such as +a desire to install a newer version, compatibility with the +[Nvidia GPU-operator](https://github.com/NVIDIA/gpu-operator) or other +use-cases). In this case, ensure that you turn off the +[enable_secure_boot](#input\_enable\_secure\_boot) option to allow unsigned +kernel modules to be loaded. + +#### Maximize GPU network bandwidth with GPUDirect and multi-networking +For A3 Series machines to achieve optimal performance , GKE provide two networking stacks for remote direct memory access (RDMA): + +- A3 High machine types (a3-highgpu-8g): utilize GPUDirect-TCPX to reduce the overhead required to transfer packet payloads to and from GPUs, which significantly improves throughput at scale compared to GPUs that don't use GPUDirect. +- A3 Mega machine types (a3-megagpu-8g): utilize GPUDirect-TCPXO to improve GPU to GPU communication, and further improves GPU to VM communication. + +To achieve this, when creating nodepools with A3 Series machine type, pass in a multivpc module to the gke-node-pool module, and the gke-node-pool module would detect the eligible machine type and enable GPUDirect for it. More specifically, the below components will be installed in the nodepool for enabling GPUDirect. + +- Install NCCL plugin for GPUDirect [TCPX](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpx) or [TCPXO](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpxo) +- Install [NRI](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nri_device_injector) device injector plugin +- Provide support for injecting GPUDirect required components(annotations, volumes, rxdm sidecar etc.) into the user workload in the form of Kubernetes Job. + - Provide sample workload to showcase how it will be updated with the required components injected, and how it can be deployed. + - Allow user to use the provided script to update their own workload and deploy. + +The GPUDirect supports included in the Cluster Toolkit aim to automate the [GPUDirect User Guid](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#install-gpudirect-tcpx-nccl) and provide better usability. + +> **_NOTE:_** You must [enable multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) feature when creating the GKE cluster. When gke-cluster depends on multivpc (with the use keyword), multi networking will be automatically enabled on the cluster creation. +> When gke-cluster or pre-existing-gke-cluster depends on multivpc (with the use keyword), the [network objects](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment) required for multi networking will be created on the cluster. + +### GPUs Examples + +There are several ways to add GPUs to a GKE node pool. See +[docs](https://cloud.google.com/compute/docs/gpus) for more info on GPUs. + +The following is a node pool that uses `a2`, `a3` or `g2` machine types which has a +fixed number of attached GPUs, let's call these machine types as "pre-defined gpu machine families": + +```yaml + - id: simple-a2-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g +``` + +> **Note**: It is not necessary to define the [`guest_accelerator`] setting when +> using pre-defined gpu machine families as information about GPUs, such as type, count and +> `gpu_driver_installation_config`, is automatically inferred from the machine type. +> Optional fields such as `gpu_partition_size` need to be specified only if they have +> non-default values. + +The following scenarios require the [`guest_accelerator`] block is specified: + +- To partition an A100 GPU into multiple GPUs on an A2 family machine. +- To specify a time sharing configuration on a GPUs. +- To attach a GPU to an N1 family machine. + +The following is an example of +[partitioning](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus-multi) +an A100 GPU: + +> **Note**: In the following example, `type`, `count` and `gpu_driver_installation_config` are picked up automatically. + +```yaml + - id: multi-instance-gpu-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g + guest_accelerator: + - gpu_partition_size: 1g.5gb +``` + +[`guest_accelerator`]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_guest_accelerator + +The following is an example of +[GPU time sharing](https://cloud.google.com/kubernetes-engine/docs/concepts/timesharing-gpus) +(with partitioned GPUs): + +```yaml + - id: time-sharing-gpu-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g + guest_accelerator: + - gpu_partition_size: 1g.5gb + gpu_sharing_config: + gpu_sharing_strategy: TIME_SHARING + max_shared_clients_per_gpu: 3 +``` + +Following is an example of using a GPU attached to an `n1` machine: + +```yaml + - id: t4-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: n1-standard-16 + guest_accelerator: + - type: nvidia-tesla-t4 + count: 2 +``` + +The following is an example of using a GPU (with sharing config) attached to an `n1` machine: + +```yaml + - id: n1-t4-pool + source: community/modules/compute/gke-node-pool + use: [gke_cluster] + settings: + name: n1-t4-pool + machine_type: n1-standard-1 + guest_accelerator: + - type: nvidia-tesla-t4 + count: 2 + gpu_driver_installation_config: + gpu_driver_version: "LATEST" + gpu_sharing_config: + max_shared_clients_per_gpu: 2 + gpu_sharing_strategy: "TIME_SHARING" +``` + +Finally, the following is adding multivpc to a node pool: + +```yaml + - id: network + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: gke-cluster + source: modules/scheduler/gke-cluster + use: [network, multinetwork] + settings: + cluster_name: $(vars.deployment_name) + + - id: a3-megagpu_pool + source: modules/compute/gke-node-pool + use: [gke-cluster, multinetwork] + settings: + machine_type: a3-megagpu-8g + ... +``` + +## Using GCE Reservations +You can reserve Google Compute Engine instances in a specific zone to ensure resources are available for their workloads when needed. For more details on how to manage reservations, see [Reserving Compute Engine zonal resources](https://cloud.google.com/compute/docs/instances/reserving-zonal-resources). + +After creating a reservation, you can consume the reserved GCE VM instances in GKE. GKE clusters deployed using Cluster Toolkit support the same consumption modes as Compute Engine: NO_RESERVATION(default), ANY_RESERVATION, SPECIFIC_RESERVATION. + +This can be accomplished using [`reservation_affinity`](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/compute/gke-node-pool/README.md#input_reservation_affinity). + +```yaml +# Target any reservation +reservation_affinity: + consume_reservation_type: ANY_RESERVATION + +# Target a specific reservation +reservation_affinity: + consume_reservation_type: SPECIFIC_RESERVATION + specific_reservations: + - name: specific-reservation-1 +``` + +The following requirements need to be satisfied for the node pool nodes to be able to use a specific reservation: +1. A reservation with the name must exist in the specified project(`var.project_id`) and one of the specified zones(`var.zones`). +2. Its consumption type must be `specific`. +3. Its GCE VM Properties must match with those of the Node Pool; Machine type, Accelerators (GPU Type and count), Local SSD disk type and count. + +If you want to utilise a shared reservation, the owner project of the shared reservation needs to be explicitly specified like the following. Note that a shared reservation can be used by the project that hosts the reservation (owner project) and by the projects the reservation is shared with (consumer projects). See how to [create and use a shared reservation](https://cloud.google.com/compute/docs/instances/reservations-shared). + +```yaml +reservation_affinity: + consume_reservation_type: SPECIFIC_RESERVATION + specific_reservations: + - name: specific-reservation-shared + project: shared_reservation_owner_project_id +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 7.2 | +| [google-beta](#requirement\_google-beta) | >= 7.2 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 7.2 | +| [google-beta](#provider\_google-beta) | >= 7.2 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | +| [tpu](#module\_tpu) | ../../internal/tpu-definition | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_container_node_pool.node_pool](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | +| [null_resource.enable_tcpx_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.enable_tcpxo_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.install_dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_machine_types.machine_info](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | +| [google_compute_region_instance_template.instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_template) | data source | +| [google_compute_reservation.specific_reservations](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [auto\_repair](#input\_auto\_repair) | Whether the nodes will be automatically repaired. | `bool` | `true` | no | +| [auto\_upgrade](#input\_auto\_upgrade) | Whether the nodes will be automatically upgraded. | `bool` | `false` | no | +| [autoscaling\_total\_max\_nodes](#input\_autoscaling\_total\_max\_nodes) | Total maximum number of nodes in the NodePool. | `number` | `1000` | no | +| [autoscaling\_total\_min\_nodes](#input\_autoscaling\_total\_min\_nodes) | Total minimum number of nodes in the NodePool. | `number` | `0` | no | +| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | +| [compact\_placement](#input\_compact\_placement) | DEPRECATED: Use `placement_policy` | `bool` | `null` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for each node. | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Disk type for each node. | `string` | `null` | no | +| [enable\_flex\_start](#input\_enable\_flex\_start) | If true, start the node pool with Flex Start provisioning model.
To learn more about flex-start mode, please refer to
https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and
https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest | `bool` | `false` | no | +| [enable\_gcfs](#input\_enable\_gcfs) | Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config). | `bool` | `false` | no | +| [enable\_numa\_aware\_scheduling](#input\_enable\_numa\_aware\_scheduling) | Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling. | `bool` | `false` | no | +| [enable\_private\_nodes](#input\_enable\_private\_nodes) | Whether nodes have internal IP addresses only. | `bool` | `true` | no | +| [enable\_queued\_provisioning](#input\_enable\_queued\_provisioning) | If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool. | `bool` | `false` | no | +| [enable\_secure\_boot](#input\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | +| [gke\_version](#input\_gke\_version) | GKE version | `string` | n/a | yes | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = optional(string)
    count = optional(number, 0)
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | +| [host\_maintenance\_interval](#input\_host\_maintenance\_interval) | Specifies the frequency of planned maintenance events. | `string` | `""` | no | +| [image\_type](#input\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | +| [initial\_node\_count](#input\_initial\_node\_count) | The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static\_node\_count and must be set to a value between autoscaling\_total\_min\_nodes and autoscaling\_total\_max\_nodes. | `number` | `null` | no | +| [internal\_ghpc\_module\_id](#input\_internal\_ghpc\_module\_id) | DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide). | `string` | n/a | yes | +| [is\_reservation\_active](#input\_is\_reservation\_active) | Whether the specified reservation is already created. | `bool` | `true` | no | +| [kubernetes\_labels](#input\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_count\_ephemeral\_storage](#input\_local\_ssd\_count\_ephemeral\_storage) | The number of local SSDs to attach to each node to back ephemeral storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | +| [local\_ssd\_count\_nvme\_block](#input\_local\_ssd\_count\_nvme\_block) | The number of local SSDs to attach to each node to back block storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | +| [machine\_type](#input\_machine\_type) | The name of a Google Compute Engine machine type. | `string` | `"c2-standard-60"` | no | +| [max\_pods\_per\_node](#input\_max\_pods\_per\_node) | The maximum number of pods per node in this node pool. This will force replacement. | `number` | `null` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [name](#input\_name) | The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix.
If setting manually, ensure a unique value across all gke-node-pools. | `string` | `null` | no | +| [num\_node\_pools](#input\_num\_node\_pools) | Number of node pools to create. This is same as num\_slices. | `number` | `1` | no | +| [num\_slices](#input\_num\_slices) | Number of TPUs slices to create. This is same as num\_node\_pools. | `number` | `1` | no | +| [placement\_policy](#input\_placement\_policy) | Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| 
{
  "name": null,
  "tpu_topology": null,
  "type": null
}
| no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Reservation resource to consume. When targeting SPECIFIC\_RESERVATION, specific\_reservations needs be specified.
Even though specific\_reservations is a list, only one reservation is allowed by the NodePool API.
It is assumed that the specified reservation exists and has available capacity.
For a shared reservation, specify the project\_id as well in which it was created.
To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared | 
object({
    consume_reservation_type = string
    specific_reservations = optional(list(object({
      name    = string
      project = optional(string)
    })))
  })
| 
{
  "consume_reservation_type": "NO_RESERVATION",
  "specific_reservations": []
}
| no | +| [run\_workload\_script](#input\_run\_workload\_script) | Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only. | `bool` | `true` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [static\_node\_count](#input\_static\_node\_count) | The static number of nodes in the node pool. If set, autoscaling will be disabled. | `number` | `null` | no | +| [taints](#input\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | +| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | +| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | +| [total\_max\_nodes](#input\_total\_max\_nodes) | DEPRECATED: Use autoscaling\_total\_max\_nodes. | `number` | `null` | no | +| [total\_min\_nodes](#input\_total\_min\_nodes) | DEPRECATED: Use autoscaling\_total\_min\_nodes. | `number` | `null` | no | +| [upgrade\_settings](#input\_upgrade\_settings) | Defines node pool upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | +| [zones](#input\_zones) | A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable. | `list(string)` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [allocatable\_cpu\_per\_node](#output\_allocatable\_cpu\_per\_node) | Number of CPUs available for scheduling pods on each node. | +| [allocatable\_gpu\_per\_node](#output\_allocatable\_gpu\_per\_node) | Number of GPUs available for scheduling pods on each node. | +| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | +| [guest\_accelerator](#output\_guest\_accelerator) | The accelerator type of the nodes. | +| [has\_gpu](#output\_has\_gpu) | Boolean value indicating whether nodes in the pool are configured with GPUs. | +| [instance\_templates](#output\_instance\_templates) | The URLs of Instance Templates | +| [instructions](#output\_instructions) | Instructions for submitting the sample GPUDirect enabled job. | +| [machine\_type](#output\_machine\_type) | Machine Type | +| [node\_count\_static](#output\_node\_count\_static) | The number of static nodes in node-pool. | +| [node\_pool\_names](#output\_node\_pool\_names) | Names of the node pools. | +| [static\_gpu\_count](#output\_static\_gpu\_count) | Total number of GPUs in the node pool. Available only for static node pools. | +| [tolerations](#output\_tolerations) | Tolerations needed for a pod to be scheduled on this node pool. | +| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | +| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | +| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | + diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf new file mode 100644 index 0000000000..0c1c255255 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +## Required variables: +# local_ssd_count_ephemeral_storage +# local_ssd_count_nvme_block +# machine_type + +locals { + + local_ssd_machines = { + "a3-highgpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, + "a3-megagpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, + "a3-ultragpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, + "a4-highgpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, + } + + generated_local_ssd_config = lookup(local.local_ssd_machines, var.machine_type, { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = null }) + + # Select in priority order: + # (1) var.local_ssd_count_ephemeral_storage and var.local_ssd_count_nvme_block if any is not null + # (2) local.local_ssd_machines if not empty + # (3) default to null value for both local_ssd_count_ephemeral_storage and local_ssd_count_nvme_block + local_ssd_config = (var.local_ssd_count_ephemeral_storage == null && var.local_ssd_count_nvme_block == null) ? local.generated_local_ssd_config : { local_ssd_count_ephemeral_storage = var.local_ssd_count_ephemeral_storage, local_ssd_count_nvme_block = var.local_ssd_count_nvme_block } +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml new file mode 100644 index 0000000000..1106f63479 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml @@ -0,0 +1,50 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: my-sample-job +spec: + parallelism: 2 + completions: 2 + completionMode: Indexed + template: + spec: + containers: + - name: nccl-test + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx-dev:v3.1.9 + imagePullPolicy: Always + command: + - /bin/sh + - -c + - | + service ssh restart; + sleep infinity; + env: + - name: LD_LIBRARY_PATH + value: /usr/local/nvidia/lib64 + volumeMounts: + - name: config-volume + mountPath: /configs + resources: + limits: + nvidia.com/gpu: 8 + volumes: + - name: config-volume + configMap: + name: nccl-configmap + defaultMode: 0777 + restartPolicy: Never + backoffLimit: 0 diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml new file mode 100644 index 0000000000..bce6720681 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml @@ -0,0 +1,70 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: my-sample-job +spec: + parallelism: 2 + completions: 2 + completionMode: Indexed + template: + spec: + hostname: host1 + subdomain: nccl-host-1 + containers: + - name: nccl-test + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.14 + imagePullPolicy: Always + command: + - /bin/sh + - -c + - | + set -ex + chmod 755 /scripts/demo-run-nccl-test-tcpxo-via-mpi.sh + cat >/scripts/allgather.sh < 0: + container["env"].extend(env_vars) + container["volumeMounts"].extend(volume_mounts) + +if __name__ == "__main__": + main() diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py new file mode 100644 index 0000000000..db9fb3e7ff --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py @@ -0,0 +1,186 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +import argparse +import os + +def main(): + parser = argparse.ArgumentParser(description="TCPXO Job Manifest Generator") + parser.add_argument("-f", "--file", required=True, help="Path to your job template YAML file") + parser.add_argument("-r", "--rxdm", required=True, help="RxDM version") + + args = parser.parse_args() + + # Get the YAML file from the user + if not args.file: + args.file = input("Please provide the path to your job template YAML file: ") + + # Get component versions from user + if not args.rxdm: + args.rxdm = input("Enter the RxDM version: ") + + # Load and modify the YAML + with open(args.file, "r") as file: + job_manifest = yaml.load(file, Loader=yaml.BaseLoader) + + # Update annotations + add_annotations(job_manifest) + + # Update volumes + add_volumes(job_manifest) + + # Update tolerations + add_tolerations(job_manifest) + + # Add tcpxo-daemon container + add_tcpxo_daemon_container(job_manifest, args.rxdm) + + # Update environment variables and volumeMounts for GPU containers + update_gpu_containers(job_manifest) + + # Generate the new YAML file + updated_job = str(yaml.dump(job_manifest, default_flow_style=False, width=1000, default_style="|", sort_keys=False)).replace("|-", "") + + new_file_name = args.file.replace(".yaml", "-tcpxo.yaml") + with open(new_file_name, "w", encoding="utf-8") as file: + file.write(updated_job) + + # Step 7: Provide instructions to the user + print("\nA new manifest has been generated and updated to have TCPXO enabled based on the provided workload") + print("It can be found in {path}".format(path=os.path.abspath(new_file_name))) + print("You can use the following commands to submit the sample job:") + print(" kubectl create -f {path}".format(path=os.path.abspath(new_file_name))) + +def add_annotations(job_manifest): + annotations = { + 'devices.gke.io/container.tcpxo-daemon':"""|+ +- path: /dev/nvidia0 +- path: /dev/nvidia1 +- path: /dev/nvidia2 +- path: /dev/nvidia3 +- path: /dev/nvidia4 +- path: /dev/nvidia5 +- path: /dev/nvidia6 +- path: /dev/nvidia7 +- path: /dev/nvidiactl +- path: /dev/nvidia-uvm +- path: /dev/dmabuf_import_helper""", + "networking.gke.io/default-interface": "eth0", + "networking.gke.io/interfaces": """| +[ + {"interfaceName":"eth0","network":"default"}, + {"interfaceName":"eth1","network":"vpc1"}, + {"interfaceName":"eth2","network":"vpc2"}, + {"interfaceName":"eth3","network":"vpc3"}, + {"interfaceName":"eth4","network":"vpc4"}, + {"interfaceName":"eth5","network":"vpc5"}, + {"interfaceName":"eth6","network":"vpc6"}, + {"interfaceName":"eth7","network":"vpc7"}, + {"interfaceName":"eth8","network":"vpc8"} +]""", + } + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("metadata", {}) + + # Add/update annotations + pod_template_spec = job_manifest["spec"]["template"]["metadata"] + if "annotations" in pod_template_spec: + pod_template_spec["annotations"].update(annotations) + else: + pod_template_spec["annotations"] = annotations + +def add_tolerations(job_manifest): + tolerations = [ + {"key": "user-workload", "operator": "Equal", "value": """\"true\"""", "effect": "NoSchedule"}, + ] + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add tolerations + pod_spec = job_manifest["spec"]["template"]["spec"] + if "tolerations" in pod_spec: + pod_spec["tolerations"].extend(tolerations) + else: + pod_spec["tolerations"] = tolerations + +def add_volumes(job_manifest): + volumes = [ + {"name": "nvidia-install-dir-host", "hostPath": {"path": "/home/kubernetes/bin/nvidia"}}, + {"name": "sys", "hostPath": {"path": "/sys"}}, + {"name": "proc-sys", "hostPath": {"path": "/proc/sys"}}, + {"name": "aperture-devices", "hostPath": {"path": "/dev/aperture_devices"}}, + ] + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add volumes + pod_spec = job_manifest["spec"]["template"]["spec"] + if "volumes" in pod_spec: + pod_spec["volumes"].extend(volumes) + else: + pod_spec["volumes"] = volumes + + +def add_tcpxo_daemon_container(job_template, rxdm_version): + tcpxo_daemon_container = { + "name": "tcpxo-daemon", + "image": f"us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:{rxdm_version}", # Use provided RxDM version + "imagePullPolicy": "Always", + "command": ["/bin/sh", "-c"], + "args": [ + """| + set -ex + chmod 755 /fts/entrypoint_rxdm_container.sh + /fts/entrypoint_rxdm_container.sh --num_hops=2 --num_nics=8 --uid= --alsologtostderr""" + ], + "securityContext": { + "capabilities": {"add": ["NET_ADMIN", "NET_BIND_SERVICE"]} + }, + "volumeMounts": [ + {"name": "nvidia-install-dir-host", "mountPath": "/usr/local/nvidia"}, + {"name": "sys", "mountPath": "/hostsysfs"}, + {"name": "proc-sys", "mountPath": "/hostprocsysfs"}, + ], + "env": [{"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}], + } + + # Create path if it doesn't exist + job_template.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add container + pod_spec = job_template["spec"]["template"]["spec"] + pod_spec.setdefault("containers", []).insert(0, tcpxo_daemon_container) + +def update_gpu_containers(job_manifest): + env_vars = [ + {"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}, + {"name": "NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY", "value": "/dev/aperture_devices"}, + ] + volume_mounts = [{"name": "aperture-devices", "mountPath": "/dev/aperture_devices"}] + + pod_spec = job_manifest.get("spec", {}).get("template", {}).get("spec", {}) + for container in pod_spec.get("containers", []): + # Create path if it doesn't exist + container.setdefault("env", []) + container.setdefault("volumeMounts", []) + if int(container.get("resources", {}).get("limits", {}).get("nvidia.com/gpu", 0)) > 0: + container["env"].extend(env_vars) + container["volumeMounts"].extend(volume_mounts) + +if __name__ == "__main__": + main() diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf new file mode 100644 index 0000000000..d23d050986 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf @@ -0,0 +1,87 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# Enable GPUDirect for A3 and A3Mega VMs, this involve multiple kubectl steps to integrate with the created cluster +# 1. Install NCCL plugin daemonset +# 2. Install NRI plugin daemonset +# 3. Update provided workload to inject rxdm sidecar and other required annotation, volume etc. +locals { + workload_path_tcpx = "${path.module}/gpu-direct-workload/sample-tcpx-workload-job.yaml" + workload_path_tcpxo = "${path.module}/gpu-direct-workload/sample-tcpxo-workload-job.yaml" + + gpu_direct_settings = { + "a3-highgpu-8g" = { + # Manifest to be installed for enabling TCPX on a3-highgpu-8g machines + gpu_direct_manifests = [ + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-tcpx-installer.yaml", # nccl_plugin v3.1.9 for tcpx + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-config.yaml", # nccl_configmap + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/nri_device_injector/nri-device-injector.yaml", # nri_plugin + ] + updated_workload_path = replace(local.workload_path_tcpx, ".yaml", "-tcpx.yaml") + rxdm_version = "v2.0.12" # matching nccl-tcpx-installer version v3.1.9 + min_additional_networks = 4 + major_minor_version_acceptable_map = { + "1.27" = "1.27.7-gke.1121000" + "1.28" = "1.28.8-gke.1095000" + "1.29" = "1.29.3-gke.1093000" + "1.30" = "1.30.2-gke.1023000" + } + } + "a3-megagpu-8g" = { + # Manifest to be installed for enabling TCPXO on a3-megagpu-8g machines + gpu_direct_manifests = [ + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/gpudirect-tcpxo/nccl-tcpxo-installer.yaml", # nccl_plugin v1.0.14 for tcpxo + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/nri_device_injector/nri-device-injector.yaml", # nri_plugin + ] + updated_workload_path = replace(local.workload_path_tcpxo, ".yaml", "-tcpxo.yaml") + rxdm_version = "v1.0.20" # matching nccl-tcpxo-installer version v1.0.14 + min_additional_networks = 8 + major_minor_version_acceptable_map = { + "1.28" = "1.28.9-gke.1250000" + "1.29" = "1.29.4-gke.1542000" + "1.30" = "1.30.4-gke.1129000" + "1.31" = "1.31.1-gke.2008000" + "1.32" = "1.32.2-gke.1489001" + } + } + } + + min_additional_networks = try(local.gpu_direct_settings[var.machine_type].min_additional_networks, 0) + + gke_version_regex = "(\\d+\\.\\d+)\\.(\\d+)-gke\\.(\\d+)" # GKE version format: 1.X.Y-gke.Z , regex output: ["1.X" , "Y", "Z"] + + gke_version_parts = regex(local.gke_version_regex, var.gke_version) + gke_version_major = local.gke_version_parts[0] + + major_minor_version_acceptable_map = try(local.gpu_direct_setting[var.machine_type].major_minor_version_acceptable_map, null) + minor_version_acceptable = try(contains(keys(local.major_minor_version_acceptable_map), local.gke_version_major), false) ? local.major_minor_version_acceptable_map[local.gke_version_major] : "1.0.0-gke.0" + minor_version_acceptable_parts = regex(local.gke_version_regex, local.minor_version_acceptable) + gke_gpudirect_compatible = local.gke_version_parts[1] > local.minor_version_acceptable_parts[1] || (local.gke_version_parts[1] == local.minor_version_acceptable_parts[1] && local.gke_version_parts[2] >= local.minor_version_acceptable_parts[2]) +} + +check "gpu_direct_check_multi_vpc" { + assert { + condition = length(var.additional_networks) >= local.min_additional_networks + error_message = "To achieve optimal performance for ${var.machine_type} machine, at least ${local.min_additional_networks} additional vpc is recommended. You could configure it in the blueprint through modules/network/multivpc with network_count set as ${local.min_additional_networks}" + } +} + +check "gke_version_requirements" { + assert { + condition = local.gke_gpudirect_compatible + error_message = "GPUDirect is not supported on GKE version ${var.gke_version} for ${var.machine_type} machine. For supported version details visit https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#requirements" + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf new file mode 100644 index 0000000000..1ddc7ba8c3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +data "google_compute_machine_types" "machine_info" { + for_each = var.zones == null ? toset([]) : toset(var.zones) + + project = var.project_id + zone = each.key + filter = "name = \"${var.machine_type}\"" +} + +locals { + valid_machine_info = { + for zone, data in data.google_compute_machine_types.machine_info : + zone => data.machine_types if length(data.machine_types) > 0 + } + + guest_cpus = try(local.valid_machine_info[0].guest_cpus, 0) +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/main.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/main.tf new file mode 100644 index 0000000000..05314497fc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/main.tf @@ -0,0 +1,482 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-node-pool", ghpc_role = "compute" }) +} + +locals { + upgrade_settings = { + strategy = var.upgrade_settings.strategy + max_surge = coalesce(var.upgrade_settings.max_surge, 0) + max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) + } +} + +module "gpu" { + source = "../../internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + has_gpu = length(local.guest_accelerator) > 0 + allocatable_gpu_per_node = local.has_gpu ? max(local.guest_accelerator[*].count...) : -1 + is_static_node_pool_with_gpus = var.static_node_count != null && local.allocatable_gpu_per_node != -1 + static_gpu_count = local.is_static_node_pool_with_gpus ? var.static_node_count * local.allocatable_gpu_per_node : 0 + gpu_taint = local.has_gpu ? [{ + key = "nvidia.com/gpu" + value = "present" + effect = "NO_SCHEDULE" + }] : [] + + autoscale_set = var.autoscaling_total_min_nodes != 0 || var.autoscaling_total_max_nodes != 1000 + static_node_set = var.static_node_count != null + initial_node_set = try(var.initial_node_count > 0, false) + + module_unique_id = replace(lower(var.internal_ghpc_module_id), "/[^a-z0-9\\-]/", "") +} + + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] +} + +module "tpu" { + source = "../../internal/tpu-definition" + + machine_type = var.machine_type + placement_policy = var.placement_policy +} + + +data "google_container_cluster" "gke_cluster" { + name = local.cluster_name + location = local.cluster_location +} + +resource "google_container_node_pool" "node_pool" { + provider = google-beta + + count = max(var.num_node_pools, var.num_slices) + + name = (max(var.num_node_pools, var.num_slices) == 1) ? coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])) : join("-", [coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])), count.index]) + cluster = var.cluster_id + node_locations = var.zones + + node_count = var.static_node_count + dynamic "autoscaling" { + for_each = local.static_node_set ? [] : [1] + content { + total_min_node_count = var.autoscaling_total_min_nodes + total_max_node_count = var.autoscaling_total_max_nodes + location_policy = "ANY" + } + } + + initial_node_count = var.initial_node_count + + max_pods_per_node = var.max_pods_per_node + + management { + auto_repair = var.auto_repair + auto_upgrade = var.auto_upgrade + } + + upgrade_settings { + strategy = local.upgrade_settings.strategy + max_surge = local.upgrade_settings.max_surge + max_unavailable = local.upgrade_settings.max_unavailable + } + + dynamic "placement_policy" { + for_each = var.placement_policy.type != null ? [1] : [] + content { + type = var.placement_policy.type + policy_name = var.placement_policy.name + tpu_topology = module.tpu.is_tpu ? var.placement_policy.tpu_topology : null + } + } + + dynamic "queued_provisioning" { + for_each = var.enable_queued_provisioning ? [1] : [] + content { + enabled = true + } + } + + node_config { + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + resource_labels = local.labels + labels = var.kubernetes_labels + service_account = var.service_account_email + oauth_scopes = var.service_account_scopes + machine_type = var.machine_type + spot = var.spot + image_type = var.image_type + flex_start = var.enable_flex_start + max_run_duration = var.max_run_duration != null ? "${var.max_run_duration}s" : null + + dynamic "guest_accelerator" { + for_each = local.guest_accelerator + iterator = ga + content { + type = coalesce(ga.value.type, try(local.generated_guest_accelerator[0].type, "")) + count = coalesce(try(ga.value.count, 0) > 0 ? ga.value.count : try(local.generated_guest_accelerator[0].count, "0")) + + gpu_partition_size = try(ga.value.gpu_partition_size, null) + + dynamic "gpu_driver_installation_config" { + # in case user did not specify guest_accelerator settings, we need a try to default to [] + for_each = try([ga.value.gpu_driver_installation_config], [{ gpu_driver_version = "DEFAULT" }]) + iterator = gdic + content { + gpu_driver_version = gdic.value.gpu_driver_version + } + } + + dynamic "gpu_sharing_config" { + for_each = try(ga.value.gpu_sharing_config == null, true) ? [] : [ga.value.gpu_sharing_config] + iterator = gsc + content { + gpu_sharing_strategy = gsc.value.gpu_sharing_strategy + max_shared_clients_per_gpu = gsc.value.max_shared_clients_per_gpu + } + } + } + } + + dynamic "taint" { + for_each = concat(var.taints, local.gpu_taint, module.tpu.tpu_taint) + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } + + dynamic "ephemeral_storage_local_ssd_config" { + for_each = local.local_ssd_config.local_ssd_count_ephemeral_storage != null ? [1] : [] + content { + local_ssd_count = local.local_ssd_config.local_ssd_count_ephemeral_storage + } + } + + dynamic "local_nvme_ssd_block_config" { + for_each = local.local_ssd_config.local_ssd_count_nvme_block != null ? [1] : [] + content { + local_ssd_count = local.local_ssd_config.local_ssd_count_nvme_block + } + } + + shielded_instance_config { + enable_secure_boot = var.enable_secure_boot + enable_integrity_monitoring = true + } + + dynamic "gcfs_config" { + for_each = var.enable_gcfs ? [1] : [] + content { + enabled = true + } + } + + gvnic { + enabled = var.image_type == "COS_CONTAINERD" + } + + dynamic "advanced_machine_features" { + for_each = local.set_threads_per_core ? [1] : [] + content { + threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf + } + } + + # Implied by Workload Identity + workload_metadata_config { + mode = "GKE_METADATA" + } + # Implied by workload identity. + metadata = { + "disable-legacy-endpoints" = "true" + } + + linux_node_config { + sysctls = { + "net.ipv4.tcp_rmem" = "4096 87380 16777216" + "net.ipv4.tcp_wmem" = "4096 16384 16777216" + } + } + + reservation_affinity { + consume_reservation_type = var.reservation_affinity.consume_reservation_type + key = local.is_valid_reservation ? local.reservation_resource_api_label : null + values = local.is_valid_reservation ? (var.is_reservation_active ? local.active_reservation_values : local.default_reservation_values) : null + } + + dynamic "host_maintenance_policy" { + for_each = var.host_maintenance_interval != "" ? [1] : [] + content { + maintenance_interval = var.host_maintenance_interval + } + } + + kubelet_config { + cpu_manager_policy = var.enable_numa_aware_scheduling ? "static" : null + dynamic "topology_manager" { + for_each = var.enable_numa_aware_scheduling ? [1] : [] + content { + policy = "restricted" + } + } + dynamic "memory_manager" { + for_each = var.enable_numa_aware_scheduling ? [1] : [] + content { + policy = "Static" + } + } + } + } + + network_config { + dynamic "additional_node_network_configs" { + for_each = var.additional_networks + + content { + network = additional_node_network_configs.value.network + subnetwork = additional_node_network_configs.value.subnetwork + } + } + + enable_private_nodes = var.enable_private_nodes + } + + timeouts { + create = var.timeout_create + update = var.timeout_update + } + + lifecycle { + ignore_changes = [ + node_config[0].labels, + initial_node_count, + # Ignore local/ephemeral ssd configs as they are tied to machine types. + node_config[0].ephemeral_storage_local_ssd_config, + node_config[0].local_nvme_ssd_block_config, + ] + precondition { + condition = (var.max_pods_per_node == null) || (data.google_container_cluster.gke_cluster.networking_mode == "VPC_NATIVE") + error_message = "max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." + } + precondition { + condition = !local.static_node_set || !local.autoscale_set + error_message = "static_node_count cannot be set with either autoscaling_total_min_nodes or autoscaling_total_max_nodes." + } + precondition { + condition = !local.static_node_set || !local.initial_node_set + error_message = "initial_node_count cannot be set with static_node_count." + } + precondition { + condition = !local.initial_node_set || (coalesce(var.initial_node_count, 0) >= var.autoscaling_total_min_nodes && coalesce(var.initial_node_count, 0) <= var.autoscaling_total_max_nodes) + error_message = "initial_node_count must be between autoscaling_total_min_nodes and autoscaling_total_max_nodes included." + } + precondition { + condition = !(coalesce(local.local_ssd_config.local_ssd_count_ephemeral_storage, 0) > 0 && coalesce(local.local_ssd_config.local_ssd_count_nvme_block, 0) > 0) + error_message = "Only one of local_ssd_count_ephemeral_storage or local_ssd_count_nvme_block can be set to a non-zero value." + } + precondition { + condition = ( + (var.reservation_affinity.consume_reservation_type != "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 0) || + (var.reservation_affinity.consume_reservation_type == "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 1) + ) + error_message = <<-EOT + When using NO_RESERVATION or ANY_RESERVATION as the `consume_reservation_type`, `specific_reservations` cannot be set. + On the other hand, with SPECIFIC_RESERVATION you must set `specific_reservations`. + EOT + } + precondition { + condition = ( + (local.input_specific_reservations_count == 0) || + ((length(local.verified_specific_reservations) == 1 || !var.is_reservation_active) && + length(local.specific_reservation_requirement_violations) == 0) + ) + error_message = <<-EOT + Check if your reservation is configured correctly: + - A reservation with the name must exist in the specified project and one of the specified zones + + - Its consumption type must be "specific" + %{for property in local.specific_reservation_requirement_violations} + - ${local.specific_reservation_requirement_violation_messages[property]} + %{endfor} + EOT + } + precondition { + condition = ( + (local.input_specific_reservations_count == 0) || + (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) == 0) || + (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) > 0 && try(local.input_reservation_projects[0], var.project_id) == var.project_id) + ) + error_message = "Shared extended reservations are not supported by GKE." + } + precondition { + condition = contains(["SURGE"], local.upgrade_settings.strategy) + error_message = "Only SURGE strategy is supported" + } + precondition { + condition = local.upgrade_settings.max_unavailable >= 0 + error_message = "max_unavailable should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_surge >= 0 + error_message = "max_surge should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 + error_message = "At least one of max_unavailable or max_surge must greater than 0" + } + precondition { + condition = var.placement_policy.type != "COMPACT" || (var.zones != null ? (length(var.zones) == 1) : false) + error_message = "Compact placement is only available for node pools operating in a single zone." + } + precondition { + condition = var.placement_policy.type != "COMPACT" || local.upgrade_settings.strategy != "BLUE_GREEN" + error_message = "Compact placement is not supported with blue-green upgrades." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.placement_policy.type == "COMPACT") + error_message = "placement_policy cannot be COMPACT when enable_queued_provisioning is true." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.reservation_affinity.consume_reservation_type != "NO_RESERVATION") + error_message = "reservation_affinity should be NO_RESERVATION when enable_queued_provisioning is true." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.autoscaling_total_min_nodes != 0) + error_message = "autoscaling_total_min_nodes should be 0 when enable_queued_provisioning is true." + } + precondition { + condition = !(var.num_node_pools > 1 && var.num_slices > 1) + error_message = "num_node_pools is for CPUs and GPUS, and num_slices is for TPUs. Both cannot be set at the same time to create a group of identical nodepools / slices." + } + precondition { + condition = !(var.num_node_pools == 0 && var.num_slices == 0) + error_message = "Either num_node_pools (for CPUs and GPUS) or num_slices (for TPUs) should be set to a positive integer value." + } + precondition { + condition = !(var.num_node_pools < 0 || var.num_slices < 0) + error_message = "Negative integer value of num_node_pools or num_slices is not valid. Please use a positive integer value to set num_node_pools for CPUs and GPUS, and num_slices for TPUs." + } + precondition { + condition = var.enable_flex_start == true ? (var.auto_repair == false) : true + error_message = "enable_flex_start needs node auto_repair set to false." + } + precondition { + condition = var.enable_flex_start == true ? (var.static_node_count == null) : true + error_message = "enable_flex_start does not work with static_node_count. static_node_count should be set to null." + } + precondition { + condition = var.enable_flex_start == true ? (var.reservation_affinity.consume_reservation_type == "NO_RESERVATION") : true + error_message = "enable_flex_start only works with reservation_affinity consume_reservation_type NO_RESERVATION." + } + precondition { + condition = var.enable_flex_start == true ? (var.spot == false) : true + error_message = "Both enable_flex_start and spot consumption option cannot be set to true at the same time." + } + } +} + +locals { + supported_machine_types_for_install_dependencies = ["a3-highgpu-8g", "a3-megagpu-8g"] +} + +# Replicates GKE's naming logic for its instance templates. The full +# pattern is "gke-{cluster_name}-{nodepool_name}-{hash}". +# +# This code builds the "{cluster_name}-{nodepool_name}" prefix, which is +# capped at 32 characters plus a dash '-' in between, by truncating names if needed: +# - If both names > 16 chars, both are cut to 16. +# - If one name > 16, it's shortened so the combined name length is 32. +data "google_compute_region_instance_template" "instance_template" { + for_each = { for idx, np in google_container_node_pool.node_pool : idx => np } + project = var.project_id + filter = "name: gke-${ + (length(local.cluster_name) <= 16 && length(each.value.name) <= 16) ? "${local.cluster_name}-${each.value.name}" : + (length(local.cluster_name) > 16 && length(each.value.name) > 16) ? "${substr(local.cluster_name, 0, 16)}-${substr(each.value.name, 0, 16)}" : + (length(local.cluster_name) > 16) ? "${substr(local.cluster_name, 0, 32 - length(each.value.name))}-${each.value.name}" : + "${local.cluster_name}-${substr(each.value.name, 0, 32 - length(local.cluster_name))}" + }*" + most_recent = true +} + +resource "null_resource" "install_dependencies" { + count = var.run_workload_script && contains(local.supported_machine_types_for_install_dependencies, var.machine_type) ? 1 : 0 + provisioner "local-exec" { + command = "pip3 install pyyaml" + } +} + +locals { + gpu_direct_setting = lookup(local.gpu_direct_settings, var.machine_type, { gpu_direct_manifests = [], updated_workload_path = "", rxdm_version = "" }) +} + +# execute script to inject rxdm sidecar into workload to enable tcpx for a3-highgpu-8g VM workload +resource "null_resource" "enable_tcpx_in_workload" { + count = var.run_workload_script && var.machine_type == "a3-highgpu-8g" ? 1 : 0 + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpx-in-workload.py --file ${local.workload_path_tcpx} --rxdm ${local.gpu_direct_setting.rxdm_version}" + } + + depends_on = [null_resource.install_dependencies] +} + +# execute script to inject rxdm sidecar into workload to enable tcpxo for a3-megagpu-8g VM workload +resource "null_resource" "enable_tcpxo_in_workload" { + count = var.run_workload_script && var.machine_type == "a3-megagpu-8g" ? 1 : 0 + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py --file ${local.workload_path_tcpxo} --rxdm ${local.gpu_direct_setting.rxdm_version}" + } + + depends_on = [null_resource.install_dependencies] +} + +# apply manifest to enable tcpx +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + apply_manifests = flatten([ + for manifest in local.gpu_direct_setting.gpu_direct_manifests : [ + { + source = manifest + } + ] + ]) +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/metadata.yaml new file mode 100644 index 0000000000..e980d595a2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com +ghpc: + inject_module_id: internal_ghpc_module_id diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/outputs.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/outputs.tf new file mode 100644 index 0000000000..44e1c3d971 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/outputs.tf @@ -0,0 +1,152 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "node_pool_names" { + description = "Names of the node pools." + value = google_container_node_pool.node_pool[*].name +} + +locals { + # Shared core machines only have 1 cpu allocatable, even if they have 2 cpu capacity + vcpu = local.machine_shared_core ? 1 : local.guest_cpus + useable_cpu = local.set_threads_per_core ? local.threads_per_core * local.vcpu / 2 : local.vcpu + + # allocatable resource definition: https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes#cpu_reservations + second_core = local.useable_cpu > 1 ? 1 : 0 + third_fourth_core = local.useable_cpu == 3 ? 1 : local.useable_cpu > 3 ? 2 : 0 + cores_above_four = local.useable_cpu > 4 ? local.useable_cpu - 4 : 0 + + allocatable_cpu = 0.94 + (0.99 * local.second_core) + (0.995 * local.third_fourth_core) + (0.9975 * local.cores_above_four) +} + +output "allocatable_cpu_per_node" { + description = "Number of CPUs available for scheduling pods on each node." + value = local.allocatable_cpu +} + +output "has_gpu" { + description = "Boolean value indicating whether nodes in the pool are configured with GPUs." + value = local.has_gpu +} + +output "allocatable_gpu_per_node" { + description = "Number of GPUs available for scheduling pods on each node." + value = local.allocatable_gpu_per_node +} + +output "static_gpu_count" { + description = "Total number of GPUs in the node pool. Available only for static node pools." + value = local.static_gpu_count +} + +locals { + translate_toleration = { + PREFER_NO_SCHEDULE = "PreferNoSchedule" + NO_SCHEDULE = "NoSchedule" + NO_EXECUTE = "NoExecute" + } + taints = google_container_node_pool.node_pool[0].node_config[0].taint + tolerations = [for taint in local.taints : { + key = taint.key + operator = "Equal" + value = taint.value + effect = lookup(local.translate_toleration, taint.effect, null) + }] +} + +output "tolerations" { + description = "Tolerations needed for a pod to be scheduled on this node pool." + value = local.tolerations +} + +locals { + gpu_direct_enabled = var.machine_type == "a3-highgpu-8g" || var.machine_type == "a3-megagpu-8g" + script_path = { + a3-highgpu-8g = "enable-tcpx-in-workload.py", + a3-megagpu-8g = "enable-tcpxo-in-workload.py" + } + nccl_path = var.machine_type == "a3-highgpu-8g" ? "configs" : "scripts" + gpu_direct_instruction = <<-EOT + Since you are using ${var.machine_type} machine type that has GPUDirect support, your nodepool had been configured with the required plugins. + To fully utilize GPUDirect you will need to add some components into your workload manifest. Details below: + + A sample GKE job that has GPUDirect enabled and NCCL test included has been generated locally at: + ${abspath(local.gpu_direct_setting.updated_workload_path)} + + You can use the following commands to submit the sample job: + kubectl create -f ${abspath(local.gpu_direct_setting.updated_workload_path)} + After submitting the sample job, you can validate the GPU performance by initiating NCCL test included in the sample workload: + NCCL test can be initiated from any one of the sample job Pods and coordinate with the peer Pods: + export POD_NAME=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | head -n 1) + export PEER_POD_IPS=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.status.podIP}}{{" "}}{{end}}') + kubectl exec --stdin --tty --container=nccl-test $POD_NAME -- /${local.nccl_path}/allgather.sh $PEER_POD_IPS + + If you would like to enable GPUDirect for your own workload, please follow the below steps: + export WORKLOAD_PATH=<> + python3 ${abspath("${path.module}/gpu-direct-workload/scripts/${lookup(local.script_path, var.machine_type, "")}")} --file $WORKLOAD_PATH --rxdm ${local.gpu_direct_setting.rxdm_version} + **WARNING** + The "--rxdm" version is tied to the nccl-tcpx/o-installer that had been deployed to your cluster, changing it to other value might have impact on performance + **WARNING** + + Or you can also follow our GPUDirect user guide to update your workload + https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#add-gpudirect-manifests + EOT +} + +output "instructions" { + description = "Instructions for submitting the sample GPUDirect enabled job." + value = local.gpu_direct_enabled ? local.gpu_direct_instruction : null +} + +output "node_count_static" { + description = "The number of static nodes in node-pool." + value = coalesce(var.static_node_count, var.initial_node_count, 0) +} + +output "guest_accelerator" { + description = "The accelerator type of the nodes." + value = local.guest_accelerator +} + +output "cluster_id" { + description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." + value = var.cluster_id +} + +output "machine_type" { + description = "Machine Type" + value = var.machine_type +} + +output "instance_templates" { + description = "The URLs of Instance Templates" + value = [for key, template in data.google_compute_region_instance_template.instance_template : template.self_link] +} + +output "tpu_accelerator_type" { + description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." + value = module.tpu.is_tpu ? module.tpu.tpu_accelerator_type : null +} + +output "tpu_topology" { + description = "The topology of the TPU slice (e.g., '4x4')." + value = module.tpu.is_tpu ? module.tpu.tpu_topology : null +} + +output "tpu_chips_per_node" { + description = "The number of TPU chips on each node in the pool." + value = module.tpu.is_tpu ? module.tpu.tpu_chips_per_node : null +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf new file mode 100644 index 0000000000..7c29e3902a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf @@ -0,0 +1,107 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# Split the input into three different lists where the details of a given reservation are at the same index across these lists. +locals { + # Specific block of an extended reservation can be targeted with exr-one/reservationBlocks/exr-one-block-1 + # Data source needs to be queried with the reservation name only. So, we extract the reservation name + input_reservation_names = [for r in try(var.reservation_affinity.specific_reservations, []) : split("/", r.name)[0]] + input_reservation_projects = [for r in try(var.reservation_affinity.specific_reservations, []) : coalesce(r.project, var.project_id)] + # We, also, remember the suffix "/reservationBlocks/exr-one-block-1" for use elsewhere afterwards + input_reservation_suffixes = [for r in try(var.reservation_affinity.specific_reservations, []) : substr(r.name, length(split("/", r.name)[0]), -1)] + # Adding this variable to by-pass the machine-type validation for TPUs + is_tpu = var.placement_policy.tpu_topology != null +} + +data "google_compute_reservation" "specific_reservations" { + for_each = ( + local.input_specific_reservations_count == 0 ? + {} : + { + for pair in flatten([ + for zone in try(var.zones, []) : [ + for i, reservation_name in try(local.input_reservation_names, []) : { + key : "${local.input_reservation_projects[i]}/${zone}/${reservation_name}" + zone : zone + reservation_name : reservation_name + project : local.input_reservation_projects[i] + } + ] + ]) : + pair.key => pair + } + ) + name = each.value.reservation_name + zone = each.value.zone + project = each.value.project +} + +locals { + generated_guest_accelerator = module.gpu.machine_type_guest_accelerator + reservation_resource_api_label = "compute.googleapis.com/reservation-name" + input_specific_reservations_count = try(length(var.reservation_affinity.specific_reservations), 0) + + # Filter specific reservations + verified_specific_reservations = [for k, v in data.google_compute_reservation.specific_reservations : v if(v.specific_reservation != null && v.specific_reservation_required == true)] + + # Build two maps to be used to compare the VM properties between reservations and the node pool + # Validation of only machine-type for CPUs and and both machine-type and guest-accelerators for GPUs + # Skip this for TPUs ( returns an empty list to skip the machine-type validation for aggregate TPU reservations) + reservation_vm_properties = local.is_tpu ? [] : [for reservation in local.verified_specific_reservations : { + "machine_type" : try(reservation.specific_reservation[0].instance_properties[0].machine_type, "") + "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs + { for acc in try(reservation.specific_reservation[0].instance_properties[0].guest_accelerators, []) : acc.accelerator_type => acc.accelerator_count } + ) : {} # If no GPUs, it's an empty map {} + }] + + nodepool_vm_properties = { + "machine_type" : var.machine_type + "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs + { for acc in try(local.guest_accelerator, []) : coalesce(acc.type, try(local.generated_guest_accelerator[0].type, "")) => coalesce(acc.count, try(local.generated_guest_accelerator[0].count, 0)) } + ) : {} # If no GPUs, it's an empty map {} + } + + # Compare two maps by counting the keys that mismatch. + # Know that in map comparison the order of keys does not matter. That is {NVME: x, SCSI: y} and {SCSI: y, NVME: x} are equal + # As of this writing, there is only one reservation supported by the Node Pool API. So, directly accessing it from the list + specific_reservation_requirement_violations = length(local.reservation_vm_properties) == 0 ? [] : [for k, v in local.nodepool_vm_properties : k if v != local.reservation_vm_properties[0][k]] + + specific_reservation_requirement_violation_messages = { + "machine_type" : <<-EOT + The reservation has "${try(local.reservation_vm_properties[0].machine_type, "")}" machine type and the node pool has "${local.nodepool_vm_properties.machine_type}". Check the relevant node pool setting: "machine_type" + EOT + "guest_accelerators" : <<-EOT + The reservation has ${jsonencode(try(local.reservation_vm_properties[0].guest_accelerators, {}))} accelerators and the node pool has ${jsonencode(try(local.nodepool_vm_properties.guest_accelerators, {}))}. Check the relevant node pool setting: "guest_accelerator". When unspecified, for the machine_type=${var.machine_type}, the default is guest_accelerator=${jsonencode(try(local.generated_guest_accelerator, [{}]))}. + EOT + } +} + +locals { + # Check if reservation is valid, that is, if it exists, there should be only 1 verified specific reservation or the reservation doesn't exist + is_valid_reservation = length(local.verified_specific_reservations) == 1 || !var.is_reservation_active + + # Build the list of reservation names when var.is_reservation_active is true + active_reservation_values = [ + for i, r in local.verified_specific_reservations : + length(local.input_reservation_suffixes[i]) > 0 ? + format("%s%s", r.name, local.input_reservation_suffixes[i]) : + "projects/${r.project}/reservations/${r.name}" + ] + + # Define a default reservation value if no specific reservations are present + specific_reservation_name = length(local.input_reservation_names) > 0 ? local.input_reservation_names[0] : "" + default_reservation_values = ["projects/${var.project_id}/reservations/${local.specific_reservation_name}"] +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf new file mode 100644 index 0000000000..e582db33da --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' + +# "inputs": +# var.machine_type: Machine type for the instance being evaluated. +# var.threads_per_core : Sets the number of threads per physical core, where 0 +# has behavior described in description. + +# "outputs": +# local.set_threads_per_core: bool that tells if threads per core should be set, +# to be used with a dynamic block. +# local.threads_per_core: actual threads_per_core to be used. + +locals { + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + machine_shared_core = length(local.machine_vals) <= 2 + machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) + + smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) + smt_capable_vcpu = local.machine_vcpus >= 2 + + smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core + set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) + threads_per_core = var.threads_per_core == 2 ? 2 : 1 +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/variables.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/variables.tf new file mode 100644 index 0000000000..b44ea28d57 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/variables.tf @@ -0,0 +1,487 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" + type = string +} + +variable "zones" { + description = "A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable." + type = list(string) + default = null +} + +variable "name" { + description = <<-EOD + The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix. + If setting manually, ensure a unique value across all gke-node-pools. + EOD + type = string + default = null + + validation { + # Check if the variable is null OR if it matches the GCP resource naming regex. + condition = var.name == null || can(regex("^[a-z]([-a-z0-9]{0,34}[a-z0-9])?$", var.name)) + error_message = <<-EOD + If provided, the node pool name must be between 1 and 36 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. + Underscores are not allowed. A shorter length is enforced to accommodate a suffix when creating multiple node pools. + EOD + } +} + +variable "internal_ghpc_module_id" { + description = "DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide)." + type = string +} + +variable "machine_type" { + description = "The name of a Google Compute Engine machine type." + type = string + default = "c2-standard-60" +} + +variable "disk_size_gb" { + description = "Size of disk for each node." + type = number + default = 100 +} + +variable "disk_type" { + description = "Disk type for each node." + type = string + default = null +} + +variable "enable_gcfs" { + description = "Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config)." + type = bool + default = false +} + +variable "enable_secure_boot" { + description = "Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info." + type = bool + default = true +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = optional(string) + count = optional(number, 0) + gpu_driver_installation_config = optional(object({ + gpu_driver_version = string + }), { gpu_driver_version = "DEFAULT" }) + gpu_partition_size = optional(string) + gpu_sharing_config = optional(object({ + gpu_sharing_strategy = string + max_shared_clients_per_gpu = number + })) + })) + default = [] + nullable = false + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.count != null]) + error_message = "var.guest_accelerator[*].count cannot be null" + } + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.count >= 0]) + error_message = "var.guest_accelerator[*].count must never be negative" + } + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.gpu_driver_installation_config != null]) + error_message = "var.guest_accelerator[*].gpu_driver_installation_config must not be null; leave unset to enable GKE to select default GPU driver installation" + } +} + +variable "image_type" { + description = "The default image type used by NAP once a new node pool is being created. Use either COS_CONTAINERD or UBUNTU_CONTAINERD." + type = string + default = "COS_CONTAINERD" +} + +variable "local_ssd_count_ephemeral_storage" { + description = <<-EOT + The number of local SSDs to attach to each node to back ephemeral storage. + Uses NVMe interfaces. Must be supported by `machine_type`. + When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. + [See above](#local-ssd-storage) for more info. + EOT + type = number + default = null +} + +variable "local_ssd_count_nvme_block" { + description = <<-EOT + The number of local SSDs to attach to each node to back block storage. + Uses NVMe interfaces. Must be supported by `machine_type`. + When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. + [See above](#local-ssd-storage) for more info. + + EOT + type = number + default = null +} + +variable "autoscaling_total_min_nodes" { + description = "Total minimum number of nodes in the NodePool." + type = number + default = 0 +} + +variable "autoscaling_total_max_nodes" { + description = "Total maximum number of nodes in the NodePool." + type = number + default = 1000 +} + +variable "static_node_count" { + description = "The static number of nodes in the node pool. If set, autoscaling will be disabled." + type = number + default = null +} + +variable "is_reservation_active" { + description = "Whether the specified reservation is already created." + type = bool + default = true +} + +variable "auto_repair" { + description = "Whether the nodes will be automatically repaired." + type = bool + default = true +} + +variable "auto_upgrade" { + description = "Whether the nodes will be automatically upgraded." + type = bool + default = false +} + +variable "threads_per_core" { + description = <<-EOT + Sets the number of threads per physical core. By setting threads_per_core + to 2, Simultaneous Multithreading (SMT) is enabled extending the total number + of virtual cores. For example, a machine of type c2-standard-60 will have 60 + virtual cores with threads_per_core equal to 2. With threads_per_core equal + to 1 (SMT turned off), only the 30 physical cores will be available on the VM. + + The default value of \"0\" will turn off SMT for supported machine types, and + will fall back to GCE defaults for unsupported machine types (t2d, shared-core + instances, or instances with less than 2 vCPU). + + Disabling SMT can be more performant in many HPC workloads, therefore it is + disabled by default where compatible. + + null = SMT configuration will use the GCE defaults for the machine type + 0 = SMT will be disabled where compatible (default) + 1 = SMT will always be disabled (will fail on incompatible machine types) + 2 = SMT will always be enabled (will fail on incompatible machine types) + EOT + type = number + default = 0 + + validation { + condition = var.threads_per_core == null || try(var.threads_per_core >= 0, false) && try(var.threads_per_core <= 2, false) + error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." + } +} + +variable "spot" { + description = "Provision VMs using discounted Spot pricing, allowing for preemption" + type = bool + default = false +} + +# tflint-ignore: terraform_unused_declarations +variable "compact_placement" { + description = "DEPRECATED: Use `placement_policy`" + type = bool + default = null + validation { + condition = var.compact_placement == null + error_message = "`compact_placement` is deprecated. Use `placement_policy` instead" + } +} + +variable "placement_policy" { + description = <<-EOT + Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool. + It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + EOT + + type = object({ + type = string + name = optional(string) + tpu_topology = optional(string) + }) + default = { + type = null + name = null + tpu_topology = null + } + validation { + condition = var.placement_policy.type == null || try(contains(["COMPACT"], var.placement_policy.type), false) + error_message = "`COMPACT` is the only supported value for `placement_policy.type`." + } +} + +variable "service_account_email" { + description = "Service account e-mail address to use with the node pool" + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to to use with the node pool." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "taints" { + description = "Taints to be applied to the system node pool." + type = list(object({ + key = string + value = any + effect = string + })) + default = [] +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} + +variable "kubernetes_labels" { + description = <<-EOT + Kubernetes labels to be applied to each node in the node group. Key-value pairs. + (The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) + EOT + type = map(string) + default = null +} + +variable "timeout_create" { + description = "Timeout for creating a node pool" + type = string + default = null +} + +variable "timeout_update" { + description = "Timeout for updating a node pool" + type = string + default = null +} + +# Deprecated + +# tflint-ignore: terraform_unused_declarations +variable "total_min_nodes" { + description = "DEPRECATED: Use autoscaling_total_min_nodes." + type = number + default = null + validation { + condition = var.total_min_nodes == null + error_message = "total_min_nodes was renamed to autoscaling_total_min_nodes and is deprecated; use autoscaling_total_min_nodes" + } +} + +# tflint-ignore: terraform_unused_declarations +variable "total_max_nodes" { + description = "DEPRECATED: Use autoscaling_total_max_nodes." + type = number + default = null + validation { + condition = var.total_max_nodes == null + error_message = "total_max_nodes was renamed to autoscaling_total_max_nodes and is deprecated; use autoscaling_total_max_nodes" + } +} + +# tflint-ignore: terraform_unused_declarations +variable "service_account" { + description = "DEPRECATED: use service_account_email and scopes." + type = object({ + email = string, + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "service_account is deprecated and replaced with service_account_email and scopes." + } +} + +variable "additional_networks" { + description = "Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool" + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + })) + nullable = false +} + +variable "reservation_affinity" { + description = <<-EOT + Reservation resource to consume. When targeting SPECIFIC_RESERVATION, specific_reservations needs be specified. + Even though specific_reservations is a list, only one reservation is allowed by the NodePool API. + It is assumed that the specified reservation exists and has available capacity. + For a shared reservation, specify the project_id as well in which it was created. + To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared + EOT + type = object({ + consume_reservation_type = string + specific_reservations = optional(list(object({ + name = string + project = optional(string) + }))) + }) + default = { + consume_reservation_type = "NO_RESERVATION" + specific_reservations = [] + } + validation { + condition = contains(["NO_RESERVATION", "ANY_RESERVATION", "SPECIFIC_RESERVATION"], var.reservation_affinity.consume_reservation_type) + error_message = "Accepted values are: {NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION}" + } +} + +variable "host_maintenance_interval" { + description = "Specifies the frequency of planned maintenance events." + type = string + default = "" + nullable = false + validation { + condition = contains(["", "PERIODIC", "AS_NEEDED"], var.host_maintenance_interval) + error_message = "Invalid host_maintenance_interval value. Must be PERIODIC, AS_NEEDED or the empty string" + } +} + +variable "initial_node_count" { + description = "The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static_node_count and must be set to a value between autoscaling_total_min_nodes and autoscaling_total_max_nodes." + type = number + default = null +} + +variable "gke_version" { + description = "GKE version" + type = string +} + +variable "max_pods_per_node" { + description = "The maximum number of pods per node in this node pool. This will force replacement." + type = number + default = null +} + +variable "upgrade_settings" { + description = <<-EOT + Defines node pool upgrade settings. It is highly recommended that you define all max_surge and max_unavailable. + If max_surge is not specified, it would be set to a default value of 0. + If max_unavailable is not specified, it would be set to a default value of 1. + EOT + type = object({ + strategy = string + max_surge = optional(number) + max_unavailable = optional(number) + }) + default = { + strategy = "SURGE" + max_surge = 0 + max_unavailable = 1 + } +} + +variable "run_workload_script" { + description = "Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only." + type = bool + default = true +} + +variable "enable_queued_provisioning" { + description = "If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool." + type = bool + default = false +} + +variable "enable_flex_start" { + description = <<-EOT + If true, start the node pool with Flex Start provisioning model. + To learn more about flex-start mode, please refer to + https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and + https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest + EOT + type = bool + default = false +} + +variable "max_run_duration" { + description = "The duration (in whole seconds) of the instance. Instance will run and be terminated after then." + type = number + default = null +} + +variable "enable_private_nodes" { + description = "Whether nodes have internal IP addresses only." + type = bool + default = true +} + +variable "num_node_pools" { + description = "Number of node pools to create. This is same as num_slices." + type = number + default = 1 +} + +variable "num_slices" { + description = "Number of TPUs slices to create. This is same as num_node_pools." + type = number + default = 1 +} + +variable "enable_numa_aware_scheduling" { + description = "Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling." + type = bool + default = false +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/versions.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/versions.tf new file mode 100644 index 0000000000..f018d04fc5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/versions.tf @@ -0,0 +1,38 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 7.2" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 7.2" + } + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/README.md b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/README.md new file mode 100644 index 0000000000..3b769e8761 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/README.md @@ -0,0 +1,82 @@ +## Description + +This modules create a [resource policy for compute engines](https://cloud.google.com/compute/docs/instances/placement-policies-overview). This policy can be passed to a gke-node-pool module to apply the policy on the node-pool's nodes. + +Note: By default, you can't apply compact placement policies with a max distance value to A3 VMs. To request access to this feature, contact your [Technical Account Manager (TAM)](https://cloud.google.com/tam) or the [Sales team](https://cloud.google.com/contact). + +### Example + +The following example creates a group placement resource policy and applies it to a gke-node-pool. + +```yaml + - id: group_placement_1 + source: modules/compute/resource-policy + settings: + name: gp-np-1 + group_placement_max_distance: 2 + + - id: node_pool_1 + source: modules/compute/gke-node-pool + use: [group_placement_1] + settings: + machine_type: e2-standard-8 + outputs: [instructions] +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google-beta](#requirement\_google-beta) | >= 6.29.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google-beta](#provider\_google-beta) | >= 6.29.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_resource_policy.policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [group\_placement\_max\_distance](#input\_group\_placement\_max\_distance) | The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | `number` | `0` | no | +| [name](#input\_name) | The resource policy's name. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The project ID for the resource policy. | `string` | n/a | yes | +| [region](#input\_region) | The region for the the resource policy. | `string` | n/a | yes | +| [workload\_policy](#input\_workload\_policy) | Describes the workload policy | 
object({
    type                  = optional(string, null)
    max_topology_distance = optional(string, null)
    accelerator_topology  = optional(string, null)
  })
| 
{
  "accelerator_topology": null,
  "max_topology_distance": null,
  "type": null
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [placement\_policy](#output\_placement\_policy) | Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions.
The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. | + diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/main.tf b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/main.tf new file mode 100644 index 0000000000..906424ca7c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + name = "${var.name}-${random_id.resource_name_suffix.hex}" +} + +resource "google_compute_resource_policy" "policy" { + name = local.name + region = var.region + project = var.project_id + provider = google-beta + + dynamic "workload_policy" { + for_each = var.workload_policy.type != null ? [1] : [] + + content { + type = var.workload_policy.type + max_topology_distance = var.workload_policy.max_topology_distance + accelerator_topology = var.workload_policy.accelerator_topology + } + } + + dynamic "group_placement_policy" { + for_each = var.group_placement_max_distance > 0 ? [1] : [] + + content { + collocation = "COLLOCATED" + max_distance = var.group_placement_max_distance + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/outputs.tf b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/outputs.tf new file mode 100644 index 0000000000..c1dc65bcbb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "placement_policy" { + description = <<-EOT + Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. + It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. + EOT + + value = { + type = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? "COMPACT" : null + name = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? local.name : null + tpu_topology = (var.workload_policy.type != null) ? var.workload_policy.accelerator_topology : null + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/variables.tf b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/variables.tf new file mode 100644 index 0000000000..92434326ca --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/variables.tf @@ -0,0 +1,64 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID for the resource policy." + type = string +} + +variable "region" { + description = "The region for the the resource policy." + type = string +} + +variable "name" { + description = "The resource policy's name." + type = string + + validation { + # Check if the variable matches the GCP resource naming regex. + condition = can(regex("^[a-z]([-a-z0-9]{0,52}[a-z0-9])?$", var.name)) + error_message = <<-EOD + The resource policy name must be between 1 and 54 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. + Underscores are not allowed. A shorter length is enforced to accommodate a random suffix. + EOD + } +} + +variable "group_placement_max_distance" { + description = <<-EOT + The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + EOT + + type = number + default = 0 +} + +variable "workload_policy" { + description = "Describes the workload policy" + type = object({ + type = optional(string, null) + max_topology_distance = optional(string, null) + accelerator_topology = optional(string, null) + }) + default = { + type = null + max_topology_distance = null + accelerator_topology = null + } + nullable = false +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/versions.tf b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/versions.tf new file mode 100644 index 0000000000..f235fbade3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:resource-policy/v1.37.2" + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/README.md b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/README.md new file mode 100644 index 0000000000..0c4737e0d9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/README.md @@ -0,0 +1,257 @@ +## Description + +This module creates one or more +[compute VM instances](https://cloud.google.com/compute/docs/instances). + +### Example + +```yaml +- id: compute + source: modules/compute/vm-instance + use: [network1] + settings: + instance_count: 8 + name_prefix: compute + machine_type: c2-standard-60 +``` + +This creates a cluster of 8 compute VMs that are: + +* named `compute-[0-7]` +* on the network defined by the `network1` module +* of type c2-standard-60 + +> **_NOTE:_** Simultaneous Multithreading (SMT) is deactivated by default +> (threads_per_core=1), which means only the physical cores are visible on the +> VM. With SMT disabled, a machine of type c2-standard-60 will only have the 30 +> physical cores visible. To change this, set `threads_per_core=2` under +> settings. + +### VPC Networks + +There are two methods for adding network connectivity to the `vm-instance` +module. The first is shown in the example above, where a `vpc` module or +`pre-existing-vpc` module is used by the `vm-instance` module. When this +happens, the `network_self_link` and `subnetwork_self_link` outputs from the +network are provided as input to the `vm-instance` and a network interface is +defined based on that. This can also be done updating the `network_self_link` and +`subnetwork_self_link` settings directly. + +The alternative option can be used when more than one network needs to be added +to the `vm-instance` or further customization is needed beyond what is provided +via other variables. For this option, the `network_interfaces` variable can be +used to set up one or more network interfaces on the VM instance. The format is +consistent with the terraform `google_compute_instance` `network_interface` +block, and more information can be found in the +[terraform docs][network-interface-tf]. + +> **_NOTE:_** When supplying the `network_interfaces` variable, networks +> associated with the `vm-instance` via use will be ignored in favor of the +> networks added in `network_interfaces`. In addition, `bandwidth_tier` and +> `disable_public_ips` will not apply to networks defined in +> `network_interfaces`. + +[network-interface-tf]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + +### SSH key metadata + +This module will ignore all changes to the `ssh-keys` metadata field that are +typically set by [external Google Cloud tools that automate SSH access][gcpssh] +when not using OS Login. For example, clicking on the Google Cloud Console SSH +button next to VMs in the VM Instances list will temporarily modify VM metadata +to include a dynamically-generated SSH public key. + +[gcpssh]: https://cloud.google.com/compute/docs/connect/add-ssh-keys#metadata + +### Placement + +The `placement_policy` variable can be used to control where your VM instances +are physically located relative to each other within a zone. See the official +placement [guide][guide-link] and [api][api-link] documentation. + +[guide-link]: https://cloud.google.com/compute/docs/instances/define-instance-placement +[api-link]: https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement + +Use the following settings for compact placement: + +```yaml + ... + settings: + instance_count: 4 + machine_type: c2-standard-60 + placement_policy: + collocation: "COLLOCATED" +``` + +By default the above placement policy will always result in the most compact set +of VMs available. If you would like that provisioning failed if some level of +compactness is not obtainable, you can enforce this with the [`max_distance` +setting](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies): + +```yaml + ... + settings: + instance_count: 4 + machine_type: c2-standard-60 + placement_policy: + collocation: "COLLOCATED" + max_distance: 1 +``` + +Use the following settings for spread placement: + +```yaml + ... + settings: + instance_count: 4 + machine_type: n2-standard-4 + placement_policy: + availability_domain_count: 2 +``` + +When `vm_count` is not set, as shown in the examples above, then the VMs will be +added to the placement policy incrementally. This is the **recommended way** to +use placement policies. + +If `vm_count` is specified then VMs will stay in pending state until the +specified number of VMs are created. See the warning below if using this field. + +> [!WARNING] +> When creating a compact placement using `vm_count` with more than 10 VMs, you +> must add `-parallelism=` argument on apply. For example if you have 15 VMs +> in a placement group: `terraform apply -parallelism=15`. This is because +> terraform self limits to 10 parallel requests by default but the create +> instance requests will not succeed until all VMs in the placement group have +> been requested, forming a deadlock. + +### GPU Support + +More information on GPU support in `vm-instance` and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../docs/gpu-support.md) + +## Lifecycle + +The `vm-instance` module will be replaced when the `instance_image` variable is +changed and `terraform apply` is run on the deployment group folder or +`gcluster deploy` is run. However, it will not be automatically replaced if a new +image is created in a family. + +To selectively replace the vm-instance(s), consider running terraform +`apply -replace` such as: + +> See https://developer.hashicorp.com/terraform/cli/commands/plan#replace-address for precise syntax terraform apply -replace=ADDRESS + +```shell +terraform state list +# search for the module ID and resource +terraform apply -replace="address" +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 4.73.0 | +| [google-beta](#requirement\_google-beta) | >= 6.13.0 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.73.0 | +| [google-beta](#provider\_google-beta) | >= 6.13.0 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | +| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance) | resource | +| [google-beta_google_compute_resource_policy.placement_policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | +| [google_compute_address.compute_ip](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_disk.additional_disks](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.replace_vm_trigger_from_placement](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | +| [additional\_persistent\_disks](#input\_additional\_persistent\_disks) | Configurations of additional disks to be included on the partition nodes. | 
object({
    count = optional(number, 0)
    type  = optional(string, "pd-balanced")
    size  = optional(number, 200)
  })
| `{}` | no | +| [allocate\_ip](#input\_allocate\_ip) | If not null, allocate IPs with the given configuration. See details at
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address | 
object({
    address_type = optional(string, "INTERNAL")
    purpose      = optional(string),
    network_tier = optional(string),
    ip_version   = optional(string, "IPV4"),
  })
| `null` | no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | +| [automatic\_restart](#input\_automatic\_restart) | Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user). | `bool` | `null` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will optionally be used name resources according to `name_prefix` | `string` | n/a | yes | +| [disable\_public\_ips](#input\_disable\_public\_ips) | If set to true, instances will not have public IPs | `bool` | `false` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | +| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-standard"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | +| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_count](#input\_local\_ssd\_count) | The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd. | `number` | `0` | no | +| [local\_ssd\_interface](#input\_local\_ssd\_interface) | Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set. | `string` | `"NVME"` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface

**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply
to network interfaces defined in this variable.

Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "MRDMA", "IRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. Can use "default" for the default network. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `null` | no | +| [placement\_policy](#input\_placement\_policy) | Control where your VM instances are physically located relative to each other within a zone.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy#nested_group_placement_policy | `any` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [provisioning\_model](#input\_provisioning\_model) | Provisioning model for cloud instance. | `string` | `null` | no | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME
- RESERVATION\_NAME

Must be a "SPECIFIC\_RESERVATION"
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | +| [service\_account](#input\_service\_account) | DEPRECATED - Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [spot](#input\_spot) | DEPRECATED - Use `provisioning_model` instead. | `bool` | `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | +| [zone](#input\_zone) | Compute Platform zone | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [external\_ip](#output\_external\_ip) | External IP of the instances (if enabled) | +| [instructions](#output\_instructions) | Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions. | +| [internal\_ip](#output\_internal\_ip) | Internal IP of the instances | +| [name](#output\_name) | Names of instances created | +| [self\_link](#output\_self\_link) | The tuple URIs of the created instances | + diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/compute_image.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/main.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/main.tf new file mode 100644 index 0000000000..0a8c7d354e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/main.tf @@ -0,0 +1,334 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "vm-instance", ghpc_role = "compute" }) +} + +module "gpu" { + source = "../../internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + native_fstype = [] + startup_script = local.startup_from_network_storage != null ? ( + { startup-script = local.startup_from_network_storage }) : {} + network_storage = var.network_storage != null ? ( + { network_storage = jsonencode(var.network_storage) }) : {} + + prefix_optional_deployment_name = var.name_prefix != null ? var.name_prefix : var.deployment_name + prefix_always_deployment_name = var.name_prefix != null ? "${var.deployment_name}-${var.name_prefix}" : var.deployment_name + resource_prefix = var.add_deployment_name_before_prefix ? local.prefix_always_deployment_name : local.prefix_optional_deployment_name + + enable_gvnic = var.bandwidth_tier != "not_enabled" + enable_tier_1 = var.bandwidth_tier == "tier_1_enabled" + + provisioning_model = var.provisioning_model + + spot = var.provisioning_model == "SPOT" + + # compact_placement : true when placement policy is provided and collocation set; false if unset + compact_placement = try(var.placement_policy.collocation, null) != null + + gpu_attached = contains(["a2", "g2"], local.machine_family) || length(local.guest_accelerator) > 0 + + # both of these must be false if either compact placement or preemptible/spot instances are used + # automatic restart is tolerant of GPUs while on host maintenance is not + automatic_restart_default = local.compact_placement || local.spot ? false : null + on_host_maintenance_default = local.compact_placement || local.spot || local.gpu_attached ? "TERMINATE" : "MIGRATE" + + automatic_restart = ( + var.automatic_restart != null + ? var.automatic_restart + : local.automatic_restart_default + ) + + on_host_maintenance = ( + var.on_host_maintenance != null + ? var.on_host_maintenance + : local.on_host_maintenance_default + ) + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + # Network Interfaces + # Support for `use` input and base network parameters like `network_self_link` and `subnetwork_self_link` + empty_access_config = { + nat_ip = null, + public_ptr_domain_name = null, + network_tier = null + } + default_network_interface = { + network = var.network_self_link + subnetwork = var.subnetwork_self_link + subnetwork_project = null # will populate from subnetwork_self_link + network_ip = null + nic_type = local.enable_gvnic ? "GVNIC" : null + stack_type = null + queue_count = null + access_config = var.disable_public_ips ? [] : [local.empty_access_config] + ipv6_access_config = [] + alias_ip_range = [] + } + network_interfaces = coalescelist(var.network_interfaces, [local.default_network_interface]) + network_interfaces_with_ips = var.allocate_ip == null ? local.network_interfaces : [ + for i, interface in local.network_interfaces : + merge(interface, { + network_ip = google_compute_address.compute_ip[i].address + }) + ] +} + +resource "null_resource" "image" { + triggers = { + name = try(var.instance_image.name, null), + family = try(var.instance_image.family, null), + project = try(var.instance_image.project, null) + } +} + +resource "google_compute_disk" "additional_disks" { + project = var.project_id + + count = var.instance_count * var.additional_persistent_disks.count + + # NB: this resource array must be sliced accounting for var.instance_count + name = "${local.resource_prefix}-disk-${count.index}" + type = var.additional_persistent_disks.type + size = var.additional_persistent_disks.size + labels = local.labels + zone = var.zone +} + +resource "google_compute_resource_policy" "placement_policy" { + project = var.project_id + provider = google-beta + + count = var.placement_policy != null ? 1 : 0 + name = "${local.resource_prefix}-vm-instance-placement" + group_placement_policy { + vm_count = try(var.placement_policy.vm_count, null) + availability_domain_count = try(var.placement_policy.availability_domain_count, null) + collocation = try(var.placement_policy.collocation, null) + max_distance = try(var.placement_policy.max_distance, null) + } +} + +resource "null_resource" "replace_vm_trigger_from_placement" { + triggers = { + vm_count = try(tostring(var.placement_policy.vm_count), "") + availability_domain_count = try(tostring(var.placement_policy.availability_domain_count), "") + max_distance = try(tostring(var.placement_policy.max_distance), "") + collocation = try(var.placement_policy.collocation, "") + } +} + +resource "google_compute_address" "compute_ip" { + project = var.project_id + + count = var.allocate_ip != null ? length(local.network_interfaces) : 0 + + name = "${local.resource_prefix}-${count.index}" + + address = local.network_interfaces[count.index].network_ip + region = var.region + network = can(coalesce(local.network_interfaces[count.index].subnetwork)) ? null : local.network_interfaces[count.index].network + subnetwork = local.network_interfaces[count.index].subnetwork + address_type = var.allocate_ip.address_type + purpose = var.allocate_ip.purpose + network_tier = var.allocate_ip.network_tier + ip_version = var.allocate_ip.ip_version +} + +resource "google_compute_instance" "compute_vm" { + project = var.project_id + provider = google-beta + + count = var.instance_count + + depends_on = [var.network_self_link, var.network_storage] + + name = "${local.resource_prefix}-${count.index}" + min_cpu_platform = var.min_cpu_platform + machine_type = var.machine_type + zone = var.zone + + resource_policies = google_compute_resource_policy.placement_policy[*].self_link + + tags = var.tags + labels = local.labels + + boot_disk { + initialize_params { + image = data.google_compute_image.compute_image.self_link + size = var.disk_size_gb + type = var.disk_type + labels = local.labels + } + + device_name = "${local.resource_prefix}-boot-disk-${count.index}" + auto_delete = var.auto_delete_boot_disk + } + + dynamic "attached_disk" { + for_each = slice( + google_compute_disk.additional_disks, + var.additional_persistent_disks.count * count.index, + var.additional_persistent_disks.count * count.index + var.additional_persistent_disks.count, + ) + + content { + source = attached_disk.value.self_link + device_name = "additional-disk-${attached_disk.key}" + mode = "READ_WRITE" + } + } + + dynamic "scratch_disk" { + for_each = range(var.local_ssd_count) + content { + interface = var.local_ssd_interface + } + } + + dynamic "network_interface" { + for_each = local.network_interfaces_with_ips + + content { + network = network_interface.value.network + subnetwork = network_interface.value.subnetwork + subnetwork_project = network_interface.value.subnetwork_project + network_ip = network_interface.value.network_ip + nic_type = network_interface.value.nic_type + stack_type = network_interface.value.stack_type + queue_count = network_interface.value.queue_count + dynamic "access_config" { + for_each = network_interface.value.access_config + content { + nat_ip = access_config.value.nat_ip + public_ptr_domain_name = access_config.value.public_ptr_domain_name + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = network_interface.value.ipv6_access_config + content { + public_ptr_domain_name = ipv6_access_config.value.public_ptr_domain_name + network_tier = ipv6_access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = network_interface.value.alias_ip_range + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + } + } + + network_performance_config { + total_egress_bandwidth_tier = local.enable_tier_1 ? "TIER_1" : "DEFAULT" + } + + service_account { + email = var.service_account_email + scopes = var.service_account_scopes + } + + dynamic "guest_accelerator" { + for_each = local.guest_accelerator + content { + count = guest_accelerator.value.count + type = guest_accelerator.value.type + } + } + + scheduling { + on_host_maintenance = local.on_host_maintenance + automatic_restart = local.automatic_restart + preemptible = local.spot + provisioning_model = local.provisioning_model + } + + dynamic "advanced_machine_features" { + for_each = local.set_threads_per_core ? [1] : [] + content { + threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf + } + } + + dynamic "reservation_affinity" { + for_each = var.reservation_name == "" ? [] : [1] + content { + type = "SPECIFIC_RESERVATION" + specific_reservation { + key = "compute.googleapis.com/reservation-name" + values = [var.reservation_name] + } + } + } + + metadata = merge( + local.network_storage, + local.startup_script, + local.enable_oslogin, + local.disable_automatic_updates_metadata, + var.metadata + ) + + lifecycle { + ignore_changes = [ + metadata["ssh-keys"], + ] + + replace_triggered_by = [ + null_resource.replace_vm_trigger_from_placement + ] + + precondition { + condition = (length(var.network_interfaces) == 0) != (var.network_self_link == null && var.subnetwork_self_link == null) + error_message = "Exactly one of network_interfaces or network_self_link/subnetwork_self_link must be specified." + } + precondition { + condition = alltrue([for interface in var.network_interfaces : interface.network_ip == null]) || var.instance_count == 1 + error_message = <<-EOT + The network_ip cannot be statically set on vm-instance when the VM instance_count is greater than 1. + Either set the network_ip to null to allow it to be set dynamically for all instances, or create modules for each VM instance with its own network interface. + EOT + } + precondition { + condition = !contains([ + "c3-:pd-standard", + "h3-:pd-standard", + "h3-:pd-ssd", + ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") + error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/outputs.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/outputs.tf new file mode 100644 index 0000000000..eab8cb56bd --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "name" { + description = "Names of instances created" + value = google_compute_instance.compute_vm[*].name +} + +output "self_link" { + description = "The tuple URIs of the created instances" + value = google_compute_instance.compute_vm[*].self_link +} + +output "external_ip" { + description = "External IP of the instances (if enabled)" + value = try(google_compute_instance.compute_vm[*].network_interface[0].access_config[0].nat_ip, []) +} + +output "internal_ip" { + description = "Internal IP of the instances" + value = google_compute_instance.compute_vm[*].network_interface[0].network_ip +} + +locals { + first_instance_link = try(google_compute_instance.compute_vm[0].self_link, "no-instance") + ssh_instructions = <<-EOT + Use the following commands to SSH into the first VM created: + gcloud compute ssh ${local.first_instance_link} --project ${var.project_id} + If not accessible from the public internet, use an SSH tunnel through IAP: + gcloud compute ssh ${local.first_instance_link} --tunnel-through-iap --project ${var.project_id} + EOT +} + +output "instructions" { + description = "Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions." + value = var.instance_count > 0 ? local.ssh_instructions : "No instances were created." +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf new file mode 100644 index 0000000000..02bc58e4f7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "inputs": +# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. +# var.network_storage : to be passed into metadata somewhere else (not here) +# var.startup_script : to be changed into a more complete file system with all the fs runners + +# "outputs": +# local.startup_from_network_storage : A full startup script with all the runners that are not supported +# natively and were included in the network_storage structure + +locals { + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + # Pull out runners to include in startup script + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + startup_script_runner = [{ + content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" + destination = "passed_startup_script.sh" + type = "shell" + }] + + full_runner_list = concat( + local.storage_client_install_runners, + local.mount_runners, + local.startup_script_runner + ) + + startup_from_network_storage = module.netstorage_startup_script.startup_script +} + +module "netstorage_startup_script" { + source = "../../scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.full_runner_list +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf new file mode 100644 index 0000000000..e582db33da --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' + +# "inputs": +# var.machine_type: Machine type for the instance being evaluated. +# var.threads_per_core : Sets the number of threads per physical core, where 0 +# has behavior described in description. + +# "outputs": +# local.set_threads_per_core: bool that tells if threads per core should be set, +# to be used with a dynamic block. +# local.threads_per_core: actual threads_per_core to be used. + +locals { + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + machine_shared_core = length(local.machine_vals) <= 2 + machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) + + smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) + smt_capable_vcpu = local.machine_vcpus >= 2 + + smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core + set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) + threads_per_core = var.threads_per_core == 2 ? 2 : 1 +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/variables.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/variables.tf new file mode 100644 index 0000000000..5519b8cd40 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/variables.tf @@ -0,0 +1,452 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "instance_count" { + description = "Number of instances" + type = number + default = 1 +} + +variable "instance_image" { + description = "Instance Image" + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "disk_size_gb" { + description = "Size of disk for instances." + type = number + default = 200 +} + +variable "disk_type" { + description = "Disk type for instances." + type = string + default = "pd-standard" +} + +variable "auto_delete_boot_disk" { + description = "Controls if boot disk should be auto-deleted when instance is deleted." + type = bool + default = true +} + +variable "local_ssd_count" { + description = "The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd." + type = number + default = 0 +} + +variable "local_ssd_interface" { + description = "Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set." + type = string + default = "NVME" +} + +variable "additional_persistent_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = object({ + count = optional(number, 0) + type = optional(string, "pd-balanced") + size = optional(number, 200) + }) + default = {} +} + +variable "name_prefix" { + description = <<-EOT + An optional name for all VM and disk resources. + If not supplied, `deployment_name` will be used. + When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, + then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". + EOT + type = string + default = null +} + +variable "add_deployment_name_before_prefix" { + description = <<-EOT + If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. + See `name_prefix` for further details on resource naming behavior. + EOT + type = bool + default = false +} + +variable "disable_public_ips" { + description = "If set to true, instances will not have public IPs" + type = bool + default = false +} + +variable "machine_type" { + description = "Machine type to use for the instance creation" + type = string + default = "c2-standard-60" +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "deployment_name" { + description = "Name of the deployment, will optionally be used name resources according to `name_prefix`" + type = string +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} + +variable "service_account_email" { + description = "Service account e-mail address to use with the node pool" + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to to use with the node pool." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +# tflint-ignore: terraform_unused_declarations +variable "service_account" { + description = "DEPRECATED - Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string, + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "The 'service_account' setting is deprecated, please use 'var.service_account_email' and 'var.service_account_scopes' instead." + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the VM. Can use \"default\" for the default network." + type = string + default = null +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the VM." + type = string + default = null +} + +variable "network_interfaces" { + description = <<-EOT + A list of network interfaces. The options match that of the terraform + network_interface block of google_compute_instance. For descriptions of the + subfields or more information see the documentation: + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + + **_NOTE:_** If `network_interfaces` are set, `network_self_link` and + `subnetwork_self_link` will be ignored, even if they are provided through + the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply + to network interfaces defined in this variable. + + Subfields: + network (string, required if subnetwork is not supplied) + subnetwork (string, required if network is not supplied) + subnetwork_project (string, optional) + network_ip (string, optional) + nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "MRDMA", "IRDMA"]) + stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) + queue_count (number, optional) + access_config (object, optional) + ipv6_access_config (object, optional) + alias_ip_range (list(object), optional) + EOT + type = list(object({ + network = string, + subnetwork = string, + subnetwork_project = string, + network_ip = string, + nic_type = string, + stack_type = string, + queue_count = number, + access_config = list(object({ + nat_ip = string, + public_ptr_domain_name = string, + network_tier = string + })), + ipv6_access_config = list(object({ + public_ptr_domain_name = string, + network_tier = string + })), + alias_ip_range = list(object({ + ip_cidr_range = string, + subnetwork_range_name = string + })) + })) + default = [] + validation { + condition = alltrue([ + for ni in var.network_interfaces : (ni.network == null) != (ni.subnetwork == null) + ]) + error_message = "All additional network interfaces must define exactly one of \"network\" or \"subnetwork\"." + } + validation { + condition = alltrue([ + for ni in var.network_interfaces : ni.nic_type == "GVNIC" || ni.nic_type == "VIRTIO_NET" || ni.nic_type == "MRDMA" || ni.nic_type == "IRDMA" || ni.nic_type == null + ]) + error_message = "In the variable network_interfaces, field \"nic_type\" must be \"GVNIC\", \"VIRTIO_NET\", \"MRDMA\", \"IRDMA\", or null." + } + validation { + condition = alltrue([ + for ni in var.network_interfaces : ni.stack_type == "IPV4_ONLY" || ni.stack_type == "IPV4_IPV6" || ni.stack_type == null + ]) + error_message = "In the variable network_interfaces, field \"stack_type\" must be either \"IPV4_ONLY\", \"IPV4_IPV6\" or null." + } +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "zone" { + description = "Compute Platform zone" + type = string +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "startup_script" { + description = "Startup script used on the instance" + type = string + default = null +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false +} + +variable "automatic_restart" { + description = "Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user)." + type = bool + default = null +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" + type = string + default = null + validation { + condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) + error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." + } +} + +variable "bandwidth_tier" { + description = <= 0, false) && try(var.threads_per_core <= 2, false) + error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." + } + +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "allocate_ip" { + description = <<-EOT + If not null, allocate IPs with the given configuration. See details at + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address + EOT + type = object({ + address_type = optional(string, "INTERNAL") + purpose = optional(string), + network_tier = optional(string), + ip_version = optional(string, "IPV4"), + }) + default = null +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "reservation_name" { + description = <<-EOD + Name of the reservation to use for VM resources, should be in one of the following formats: + - projects/PROJECT_ID/reservations/RESERVATION_NAME + - RESERVATION_NAME + + Must be a "SPECIFIC_RESERVATION" + Set to empty string if using no reservation or automatically-consumed reservations + EOD + type = string + default = "" + nullable = false + + validation { + condition = length(regexall("^((projects/([a-z0-9-]+)/reservations/)?([a-z0-9-]+))?$", var.reservation_name)) > 0 + error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME', [...] is an optional part." + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/versions.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/versions.tf new file mode 100644 index 0000000000..0429782c6d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/versions.tf @@ -0,0 +1,41 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.73.0" + } + + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.13.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/README.md new file mode 100644 index 0000000000..285a20bde2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/README.md @@ -0,0 +1,170 @@ +## Description + +This module creates a [Google Cloud Storage (GCS) bucket](https://cloud.google.com/storage). + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +### Example + +The following example will create a bucket named `simulation-results-xxxxxxxx`, +where `xxxxxxxx` is a randomly generated id. + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: + name_prefix: simulation-results + random_suffix: true +``` + +> **_NOTE:_** Use of `random_suffix` may cause the following error when used +> with other modules: +> `value depends on resource attributes that cannot be determined until apply`. +> To resolve this set `random_suffix` to `false` (default). + + + +> **_NOTE:_** Bucket namespace is shared by all users of Google Cloud so it is +> possible to have a bucket name clash with an existing bucket that is not in +> your project. To resolve this try to use a more unique name, or set the +> `random_suffix` variable to `true`. + +## Naming of Bucket + +There are potentially three parts to the bucket name. Each of these parts are +configurable in the blueprint. + +1. A **custom prefix**, provided by the user in the blueprint \ +Provide the custom prefix using the `name_prefix` setting. + +1. The **deployment name**, included by default \ +The deployment name can be excluded by setting `use_deployment_name_in_bucket_name: false`. + +1. A **random id** suffix, excluded by default \ +The random id can be included by setting `random_suffix: true`. + +If none of these are provided (no `name_prefix`, +`use_deployment_name_in_bucket_name: false`, & `random_suffix: false`), then the +bucket name will default to `no-bucket-name-provided`. + +Since bucket namespace is shared by all users of Google Cloud, it is more likely +to experience naming clashes than with other resources. In many cases, adding +the `random_suffix` will resolve the naming clash issue. + +> **Warning**: If a bucket is created with a `random_suffix` and then used as +> the bucket for a startup script in the same deployment group this will cause a +> `not known at apply time` error in terraform. The solution is to either create +> the bucket in a separate deployment group or to remove the random suffix. + +## Mounting + +To mount the Cloud Storage bucket you must first ensure that the GCS Fuse client +has been installed and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. + +If mounting is not automatically handled as described above, the +`cloud-storage-bucket` module outputs runners that can be used with the +`startup-script` module to install the client and mount the file system. See the +following example: + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: {local_mount: /data} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(bucket.client_install_runner) + - $(bucket.mount_runner) +``` + +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | +| [google-beta](#requirement\_google-beta) | >= 6.9.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [google-beta](#provider\_google-beta) | >= 6.9.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource | +| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [autoclass](#input\_autoclass) | Configure bucket autoclass setup

The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern.

The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time.
Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE')

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/autoclass | 
object({
    enabled                = optional(bool, false)
    terminal_storage_class = optional(string, null)
  })
| 
{
  "enabled": false
}
| no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the GCS bucket. | `string` | n/a | yes | +| [enable\_hierarchical\_namespace](#input\_enable\_hierarchical\_namespace) | If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket. | `bool` | `false` | no | +| [enable\_object\_retention](#input\_enable\_object\_retention) | If true, enables retention policy at per object level for the bucket.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/object-lock | `bool` | `false` | no | +| [enable\_versioning](#input\_enable\_versioning) | If true, enables versioning for the bucket. | `bool` | `false` | no | +| [force\_destroy](#input\_force\_destroy) | If true will destroy bucket with all objects stored within. | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to the GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | +| [lifecycle\_rules](#input\_lifecycle\_rules) | List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule | 
list(object({
    # Object with keys:
    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.
    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.
    action = object({
      type          = string
      storage_class = optional(string)
    })

    # Object with keys:
    # - age - (Optional) Minimum age of an object in days to satisfy this condition.
    # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field.
    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.
    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".
    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY.
    # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition.
    # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition.
    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.
    # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition.
    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.
    # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object.
    # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent.
    condition = object({
      age                        = optional(number)
      send_age_if_zero           = optional(bool)
      created_before             = optional(string)
      with_state                 = optional(string)
      matches_storage_class      = optional(string)
      matches_prefix             = optional(string)
      matches_suffix             = optional(string)
      num_newer_versions         = optional(number)
      custom_time_before         = optional(string)
      days_since_custom_time     = optional(number)
      days_since_noncurrent_time = optional(number)
      noncurrent_time_before     = optional(string)
    })
  }))
| `[]` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | +| [mount\_options](#input\_mount\_options) | Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories) | `string` | `"defaults,_netdev,implicit_dirs"` | no | +| [name\_prefix](#input\_name\_prefix) | Name Prefix. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | +| [public\_access\_prevention](#input\_public\_access\_prevention) | Bucket public access can be controlled by setting a value of either `inherited` or `enforced`.
When set to `enforced`, public access to the bucket is blocked.
If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/public-access-prevention | `string` | `null` | no | +| [random\_suffix](#input\_random\_suffix) | If true, a random id will be appended to the suffix of the bucket name. | `bool` | `false` | no | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [retention\_policy\_period](#input\_retention\_policy\_period) | If defined, this will configure retention\_policy with retention\_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/bucket-lock | `number` | `null` | no | +| [soft\_delete\_retention\_duration](#input\_soft\_delete\_retention\_duration) | If defined, this will configure soft\_delete\_policy with retention\_duration\_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days).
Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/soft-delete | `number` | `null` | no | +| [storage\_class](#input\_storage\_class) | The storage class of the GCS bucket. | `string` | `"REGIONAL"` | no | +| [uniform\_bucket\_level\_access](#input\_uniform\_bucket\_level\_access) | Allow uniform control access to the bucket. | `bool` | `true` | no | +| [use\_deployment\_name\_in\_bucket\_name](#input\_use\_deployment\_name\_in\_bucket\_name) | If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments. | `bool` | `true` | no | +| [viewers](#input\_viewers) | A list of additional accounts that can read packages from this bucket | `set(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use gcs fuse. | +| [gcs\_bucket\_name](#output\_gcs\_bucket\_name) | Bucket name. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | The gsutil bucket path with format of `gs://`. | +| [mount\_runner](#output\_mount\_runner) | Runner that mounts the cloud storage bucket with gcs fuse. | +| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | + diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf new file mode 100644 index 0000000000..81ba0ca6a9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf @@ -0,0 +1,126 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "cloud-storage-bucket", ghpc_role = "file-system" }) +} + +locals { + prefix = var.name_prefix != null ? var.name_prefix : "" + deployment = var.use_deployment_name_in_bucket_name ? var.deployment_name : "" + suffix = var.random_suffix ? random_id.resource_name_suffix.hex : "" + first_dash = (local.prefix != "" && (local.deployment != "" || local.suffix != "")) ? "-" : "" + second_dash = local.deployment != "" && local.suffix != "" ? "-" : "" + composite_name = "${local.prefix}${local.first_dash}${local.deployment}${local.second_dash}${local.suffix}" + name = local.composite_name == "" ? "no-bucket-name-provided" : local.composite_name +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket" "bucket" { + provider = google-beta + project = var.project_id + name = local.name + uniform_bucket_level_access = var.uniform_bucket_level_access + location = var.region + storage_class = var.storage_class + labels = local.labels + force_destroy = var.force_destroy + public_access_prevention = var.public_access_prevention + enable_object_retention = var.enable_object_retention + hierarchical_namespace { + enabled = var.enable_hierarchical_namespace + } + + dynamic "autoclass" { + for_each = var.autoclass.enabled ? [1] : [] + content { + enabled = var.autoclass.enabled + terminal_storage_class = var.autoclass.terminal_storage_class + } + } + + dynamic "soft_delete_policy" { + for_each = var.soft_delete_retention_duration == null ? [] : [1] + content { + retention_duration_seconds = var.soft_delete_retention_duration + } + } + + dynamic "retention_policy" { + for_each = var.retention_policy_period == null ? [] : [1] + content { + retention_period = var.retention_policy_period + } + } + + dynamic "versioning" { + for_each = var.enable_versioning ? [1] : [] + content { + enabled = var.enable_versioning + } + } + + dynamic "lifecycle_rule" { + for_each = var.lifecycle_rules + content { + action { + type = lifecycle_rule.value.action.type + storage_class = lookup(lifecycle_rule.value.action, "storage_class", null) + } + condition { + age = lookup(lifecycle_rule.value.condition, "age", null) + send_age_if_zero = lookup(lifecycle_rule.value.condition, "send_age_if_zero", null) + created_before = lookup(lifecycle_rule.value.condition, "created_before", null) + with_state = lookup(lifecycle_rule.value.condition, "with_state", contains(keys(lifecycle_rule.value.condition), "is_live") ? (lifecycle_rule.value.condition["is_live"] ? "LIVE" : null) : null) + matches_storage_class = lifecycle_rule.value.condition["matches_storage_class"] != null ? split(",", lifecycle_rule.value.condition["matches_storage_class"]) : null + matches_prefix = lifecycle_rule.value.condition["matches_prefix"] != null ? split(",", lifecycle_rule.value.condition["matches_prefix"]) : null + matches_suffix = lifecycle_rule.value.condition["matches_suffix"] != null ? split(",", lifecycle_rule.value.condition["matches_suffix"]) : null + num_newer_versions = lookup(lifecycle_rule.value.condition, "num_newer_versions", null) + custom_time_before = lookup(lifecycle_rule.value.condition, "custom_time_before", null) + days_since_custom_time = lookup(lifecycle_rule.value.condition, "days_since_custom_time", null) + days_since_noncurrent_time = lookup(lifecycle_rule.value.condition, "days_since_noncurrent_time", null) + noncurrent_time_before = lookup(lifecycle_rule.value.condition, "noncurrent_time_before", null) + } + } + } + + lifecycle { + precondition { + condition = !var.autoclass.enabled || !var.enable_hierarchical_namespace + error_message = "Hierarchical namespace is not compatible with Autoclass enabled." + } + + precondition { + condition = !var.enable_hierarchical_namespace || var.uniform_bucket_level_access + error_message = "Hierarchical namespace is not compatible with Uniform bucket level access disabled." + } + + precondition { + condition = !var.enable_versioning || !var.enable_hierarchical_namespace + error_message = "Hierarchical namespace is not compatible with Object versioning enabled." + } + } +} + +resource "google_storage_bucket_iam_binding" "viewers" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.objectViewer" + members = var.viewers +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf new file mode 100644 index 0000000000..29ddfef2d2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a remote network storage to be mounted by fs-tab." + value = { + remote_mount = local.name + local_mount = var.local_mount + fs_type = "gcsfuse" + mount_options = var.mount_options + server_ip = "" + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } +} + +locals { + client_install_runner = { + "type" = "shell" + "content" = file("${path.module}/scripts/install-gcs-fuse.sh") + "destination" = "install-gcsfuse${replace(var.local_mount, "/", "_")}.sh" + } + + mount_runner = { + "type" = "shell" + "destination" = "mount_gcs${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"not-used\" \"${local.name}\" \"${var.local_mount}\" \"gcsfuse\" \"${var.mount_options}\"" + "content" = file("${path.module}/scripts/mount.sh") + } +} + +output "client_install_runner" { + description = "Runner that performs client installation needed to use gcs fuse." + value = local.client_install_runner +} + +output "mount_runner" { + description = "Runner that mounts the cloud storage bucket with gcs fuse." + value = local.mount_runner +} + +output "gcs_bucket_path" { + description = "The gsutil bucket path with format of `gs://`." + # cannot use resource attribute, will cause lookup failure in startup-script + value = "gs://${local.name}" + + # needed to make sure bucket contents are deleted before bucket + depends_on = [ + google_storage_bucket.bucket + ] +} + +output "gcs_bucket_name" { + description = "Bucket name." + value = google_storage_bucket.bucket.name +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh new file mode 100644 index 0000000000..f8a990260b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ ! "$(which gcsfuse)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then + tee /etc/yum.repos.d/gcsfuse.repo >/dev/null </dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf new file mode 100644 index 0000000000..9804e4b268 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf @@ -0,0 +1,254 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which GCS bucket will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment; used as part of name of the GCS bucket." + type = string +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "labels" { + description = "Labels to add to the GCS bucket. Key-value pairs." + type = map(string) +} + +variable "local_mount" { + description = "The mount point where the contents of the device may be accessed after mounting." + type = string + default = "/mnt" +} + +variable "mount_options" { + description = "Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories)" + type = string + default = "defaults,_netdev,implicit_dirs" +} + +variable "name_prefix" { + description = "Name Prefix." + type = string + default = null +} + +variable "use_deployment_name_in_bucket_name" { + description = "If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments." + type = bool + default = true +} + +variable "random_suffix" { + description = "If true, a random id will be appended to the suffix of the bucket name." + type = bool + default = false +} + +variable "force_destroy" { + description = "If true will destroy bucket with all objects stored within." + type = bool + default = false +} + +variable "viewers" { + description = "A list of additional accounts that can read packages from this bucket" + type = set(string) + default = [] + + validation { + error_message = "All bucket viewers must be in IAM style: user:user@example.com, serviceAccount:sa@example.com, or group:group@example.com." + condition = alltrue([ + for viewer in var.viewers : length(regexall("^(user|serviceAccount|group):", viewer)) > 0 + ]) + } +} + +variable "enable_hierarchical_namespace" { + description = "If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket." + type = bool + default = false +} + +variable "uniform_bucket_level_access" { + description = "Allow uniform control access to the bucket." + type = bool + default = true +} + +variable "storage_class" { + description = "The storage class of the GCS bucket." + type = string + default = "REGIONAL" + validation { + condition = contains([ + "STANDARD", + "MULTI_REGIONAL", + "REGIONAL", + "NEARLINE", + "COLDLINE", + "ARCHIVE" + ], var.storage_class) + error_message = "Allowed values for GCS storage_class are 'STANDARD', 'MULTI_REGIONAL', 'REGIONAL', 'NEARLINE', 'COLDLINE', 'ARCHIVE'.\nhttps://cloud.google.com/storage/docs/storage-classes" + } +} + +variable "autoclass" { + description = <<-EOT + Configure bucket autoclass setup + + The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern. + + The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time. + Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE') + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/autoclass + EOT + type = object({ + enabled = optional(bool, false) + terminal_storage_class = optional(string, null) + }) + default = { + enabled = false + } + nullable = false + validation { + condition = !can(coalesce(var.autoclass.terminal_storage_class)) || var.autoclass.enabled + error_message = "Cannot set bucket var.autoclass.terminal_storage_class unless var.autoclass.enabled is true" + } +} + +variable "public_access_prevention" { + description = <<-EOT + Bucket public access can be controlled by setting a value of either `inherited` or `enforced`. + When set to `enforced`, public access to the bucket is blocked. + If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/public-access-prevention + EOT + type = string + default = null + validation { + condition = var.public_access_prevention == null ? true : contains([ + "inherited", + "enforced" + ], var.public_access_prevention) + error_message = "Allowed values for public_access_prevention are 'inherited', 'enforced'.\n" + } +} + +variable "soft_delete_retention_duration" { + description = <<-EOT + If defined, this will configure soft_delete_policy with retention_duration_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days). + Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/soft-delete + EOT + type = number + default = null + validation { + condition = var.soft_delete_retention_duration == null ? true : var.soft_delete_retention_duration == 0 || var.soft_delete_retention_duration >= 604800 && var.soft_delete_retention_duration <= 7776000 + error_message = "var.soft_delete_retention_duration value can be 0 or in between 604800(7 days) and 7776000(90 days)." + } +} + +variable "enable_versioning" { + description = "If true, enables versioning for the bucket." + type = bool + default = false +} + +variable "lifecycle_rules" { + description = "List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule" + type = list(object({ + # Object with keys: + # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass. + # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule. + action = object({ + type = string + storage_class = optional(string) + }) + + # Object with keys: + # - age - (Optional) Minimum age of an object in days to satisfy this condition. + # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field. + # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition. + # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY". + # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY. + # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition. + # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition. + # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition. + # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition. + # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true. + # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object. + # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent. + condition = object({ + age = optional(number) + send_age_if_zero = optional(bool) + created_before = optional(string) + with_state = optional(string) + matches_storage_class = optional(string) + matches_prefix = optional(string) + matches_suffix = optional(string) + num_newer_versions = optional(number) + custom_time_before = optional(string) + days_since_custom_time = optional(number) + days_since_noncurrent_time = optional(number) + noncurrent_time_before = optional(string) + }) + })) + default = [] +} + +variable "retention_policy_period" { + description = <<-EOT + If defined, this will configure retention_policy with retention_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/bucket-lock + EOT + type = number + default = null + validation { + condition = var.retention_policy_period == null ? true : var.retention_policy_period > 0 && var.retention_policy_period <= 3155760000 + error_message = "var.soft_delete_policy_retention_duration value must be in between 1 and 3155760000(100 years) seconds." + } +} + +variable "enable_object_retention" { + description = <<-EOT + If true, enables retention policy at per object level for the bucket. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/object-lock + EOT + type = bool + default = false +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf new file mode 100644 index 0000000000..217ee2f3a2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.9.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" + } + required_version = ">= 0.14.0" +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/README.md new file mode 100644 index 0000000000..3bf251828e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/README.md @@ -0,0 +1,248 @@ +## Description + +This module creates a [filestore](https://cloud.google.com/filestore) +instance. Filestore is a high performance network file system that can be +mounted to one or more compute VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +### Deletion protection + +We recommend considering enabling [Filestore deletion protection][fdp]. Deletion +protection will prevent unintentional deletion of an entire Filestore instance. +It does not prevent deletion of files within the Filestore instance when mounted +by a VM. It is not available on some [tiers](#filestore-tiers), including the +default BASIC\_HDD tier or BASIC\_SSD tier. Follow the documentation link for +up to date details. + +Usage can be enabled in a blueprint with, for example: + +```yaml + - id: homefs + source: modules/file-system/filestore + use: [network] + settings: + deletion_protection: + enabled: true + reason: Avoid data loss + filestore_tier: ZONAL + local_mount: /home + size_gb: 1024 +``` + +[fdp]: https://cloud.google.com/filestore/docs/deletion-protection + +### Filestore tiers + +At the time of writing, Filestore supports 5 [tiers of service][tiers] that are +specified in the Toolkit using the following names: + +- Basic HDD: "BASIC\_HDD" ([preferred][tierapi]) or "STANDARD" (deprecated) +- Basic SSD: "BASIC\_SSD" ([preferred][tierapi]) or "PREMIUM" (deprecated) +- Zonal: "ZONAL" +- Enterprise: "ENTERPRISE" +- Regional: "REGIONAL" + +[tierapi]: https://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier + +**Please review the minimum storage requirements for each tier**. The Terraform +module can only enforce the minimum value of the `size_gb` parameter for the +lowest tier of service. If you supply a value that is too low, Filestore +creation will fail when you run `terraform apply`. + +[tiers]: https://cloud.google.com/filestore/docs/service-tiers + +### Filestore protocols and mount options +After Filestore instance is created, you can mount this to the compute node +using different mount options. Toolkit uses [default mount options](https://linux.die.net/man/8/mount) +for all tier services. Filestore has recommended mount options for different +service tiers which may overall improve performance. These can be found here: +[recommended mount options.](https://cloud.google.com/filestore/docs/mounting-fileshares) +While creating filestore module, you can overwrite these mount options as +mentioned below. + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /homefs + mount_options: defaults,hard,timeo=600,retrans=3,_netdev +``` + +Filestore supports NFS protocols `NFS_V3` (default) and `NFS_V4_1`. Protocol support depends on the selected tier: +- `NFS_V3`: Supported on all tiers (`BASIC_HDD`, `BASIC_SSD`, `HIGH_SCALE_SSD`, `ZONAL`, `ENTERPRISE`). +- `NFS_V4_1`: Supported only on `HIGH_SCALE_SSD`, `ZONAL`, `REGIONAL`, and `ENTERPRISE`. +This can be specified at creation time via the `protocol` variable. By default, `NFS_V3` is used for compatibility. +See the example below and [this page](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#protocol-1) for more information. + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /homefs + protocol: NFS_V4_1 + filestore_tier: ZONAL +``` + +### Filestore quota + +Your project must have unused quota for Cloud Filestore in the region you will +provision the storage. This can be found by browsing to the [Quota tab within IAM +& Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. +Please note that there are separate quota limits for HDD and SSD storage. + +All projects begin with 0 available quota for High Scale SSD tier. To use this +tier, [make a request and wait for it to be approved][hs-ssd-quota]. + +[hs-ssd-quota]: https://cloud.google.com/filestore/docs/high-scale + +### Example - Basic HDD + +The Filestore instance defined below will have the following attributes: + +- (default) `BASIC_HDD` tier +- (default) 1TiB capacity +- `homefs` module ID +- mount point at `/home` +- connected to the network defined in the `network1` module + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /home +``` + +### Example - High Scale SSD + +The Filestore instance defined below will have the following attributes: + +- `HIGH_SCALE_SSD` tier +- 10TiB capacity +- `highscale` module ID +- mount point at `/projects` +- connected to the VPC network defined in the `network1` module + +```yaml +- id: highscale + source: modules/file-system/filestore + use: [network1] + settings: + filestore_tier: HIGH_SCALE_SSD + size_gb: 10240 + local_mount: /projects +``` + +## Mounting + +To mount the Filestore instance you must first ensure that the NFS client has +been installed and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +See the [hpc-slurm](../../../examples/hpc-slurm.yaml) for +an example of using this module with Slurm. + +If mounting is not automatically handled as described above, the `filestore` +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + - id: filestore + source: modules/file-system/filestore + use: [network1] + settings: {local_mount: /scratch} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(filestore.install_nfs_client_runner) + - $(filestore.mount_runner) + +``` + +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 6.4 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.4 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_filestore_instance.filestore_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [connect\_mode](#input\_connect\_mode) | Used to select mode - supported values DIRECT\_PEERING and PRIVATE\_SERVICE\_ACCESS. | `string` | `"DIRECT_PEERING"` | no | +| [deletion\_protection](#input\_deletion\_protection) | Configure Filestore instance deletion protection | 
object({
    enabled = optional(bool, false)
    reason  = optional(string)
  })
| 
{
  "enabled": false
}
| no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the filestore instance if no name is specified. | `string` | n/a | yes | +| [description](#input\_description) | A description of the filestore instance. | `string` | `""` | no | +| [filestore\_share\_name](#input\_filestore\_share\_name) | Name of the file system share on the instance. | `string` | `"nfsshare"` | no | +| [filestore\_tier](#input\_filestore\_tier) | The service tier of the instance. | `string` | `"BASIC_HDD"` | no | +| [labels](#input\_labels) | Labels to add to the filestore instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mount](#input\_local\_mount) | Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md)). | `string` | `"/shared"` | no | +| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"defaults,_netdev"` | no | +| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [nfs\_export\_options](#input\_nfs\_export\_options) | Define NFS export options. | 
list(object({
    access_mode = optional(string)
    ip_ranges   = optional(list(string))
    squash_mode = optional(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | ID of project in which Filestore instance will be created. | `string` | n/a | yes | +| [protocol](#input\_protocol) | NFS protocol version. Default is NFS\_V3. NFS\_V4\_1 is only supported with HIGH\_SCALE\_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers. | `string` | `"NFS_V3"` | no | +| [region](#input\_region) | Location for Filestore instances at Enterprise tier. | `string` | n/a | yes | +| [reserved\_ip\_range](#input\_reserved\_ip\_range) | Reserved IP range for Filestore instance. Users are encouraged to set to null
for automatic selection. If supplied, it must be:

CIDR format when var.connect\_mode == "DIRECT\_PEERING"
Named IP Range when var.connect\_mode == "PRIVATE\_SERVICE\_ACCESS"

See Cloud documentation for more details:

https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range | `string` | `null` | no | +| [size\_gb](#input\_size\_gb) | Storage size of the filestore instance in GB. | `number` | `1024` | no | +| [zone](#input\_zone) | Location for Filestore instances below Enterprise tier. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | +| [filestore\_id](#output\_filestore\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | +| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | +| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | +| [network\_storage](#output\_network\_storage) | Describes a filestore instance. | + diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/main.tf new file mode 100644 index 0000000000..ce035dbb2b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/main.tf @@ -0,0 +1,116 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "filestore", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + is_high_capacity_tier = contains(["HIGH_SCALE_SSD", "ZONAL", "REGIONAL"], var.filestore_tier) && var.size_gb >= 10240 && var.size_gb <= 102400 + + timeouts = local.is_high_capacity_tier ? [1] : [] + server_ip = google_filestore_instance.filestore_instance.networks[0].ip_addresses[0] + remote_mount = format("/%s", google_filestore_instance.filestore_instance.file_shares[0].name) + fs_type = "nfs" + mount_options = var.mount_options + + install_nfs_client_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-nfs-client.sh" + "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" + } + mount_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" + } + + # id format: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network#id + split_network_id = split("/", var.network_id) + network_name = local.split_network_id[4] + network_project = local.split_network_id[1] + shared_vpc = local.network_project != var.project_id +} + +resource "google_filestore_instance" "filestore_instance" { + project = var.project_id + + name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + description = var.description + location = contains(["ENTERPRISE", "REGIONAL"], var.filestore_tier) ? var.region : var.zone + tier = var.filestore_tier + protocol = var.protocol + + deletion_protection_enabled = var.deletion_protection.enabled + deletion_protection_reason = var.deletion_protection.reason + + file_shares { + capacity_gb = var.size_gb + name = var.filestore_share_name + dynamic "nfs_export_options" { + for_each = var.nfs_export_options + content { + access_mode = nfs_export_options.value.access_mode + ip_ranges = nfs_export_options.value.ip_ranges + squash_mode = nfs_export_options.value.squash_mode + } + } + } + + labels = local.labels + + networks { + network = local.shared_vpc ? var.network_id : local.network_name + connect_mode = var.connect_mode + modes = ["MODE_IPV4"] + reserved_ip_range = var.reserved_ip_range + } + + dynamic "timeouts" { + for_each = local.timeouts + content { + create = "1h" + update = "1h" + delete = "1h" + } + } + + lifecycle { + precondition { + condition = ( + var.reserved_ip_range == null || + var.connect_mode == "PRIVATE_SERVICE_ACCESS" || + var.connect_mode == "DIRECT_PEERING" && can(cidrhost(var.reserved_ip_range, 0)) && contains(["24", "29"], try(split("/", var.reserved_ip_range)[1], "")) + ) + error_message = <<-EOT + If connect_mode is set to DIRECT_PEERING and reserved_ip_range is + specified then it must be a CIDR IP range with suffix range size 29 for + BASIC_HDD or BASIC_SSD tiers. Otherwise the range size must be 24. + EOT + } + + precondition { + condition = !startswith(var.filestore_tier, "BASIC") || var.protocol != "NFS_V4_1" + error_message = "NFS_V4_1 is not supported on BASIC Filestore tiers." + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/metadata.yaml new file mode 100644 index 0000000000..5298336f09 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - file.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/outputs.tf new file mode 100644 index 0000000000..9bdb3bdc7b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/outputs.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a filestore instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = local.mount_options + client_install_runner = local.install_nfs_client_runner + mount_runner = local.mount_runner + } +} + +output "install_nfs_client" { + description = "Script for installing NFS client" + value = file("${path.module}/scripts/install-nfs-client.sh") +} + +output "install_nfs_client_runner" { + description = "Runner to install NFS client using the startup-script module" + value = local.install_nfs_client_runner +} + +output "mount_runner" { + description = <<-EOT + Runner to mount the file-system using an ansible playbook. The startup-script + module will automatically handle installation of ansible. + - id: example-startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(your-fs-id.mount_runner) + ... + EOT + value = local.mount_runner +} + +output "filestore_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" + value = google_filestore_instance.filestore_instance.id +} + +output "capacity_gib" { + description = "File share capacity in GiB." + value = google_filestore_instance.filestore_instance.file_shares[0].capacity_gb +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/mount.sh b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/variables.tf new file mode 100644 index 0000000000..2d7e9258c0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/variables.tf @@ -0,0 +1,189 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which Filestore instance will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the filestore instance if no name is specified." + type = string +} + +variable "zone" { + description = "Location for Filestore instances below Enterprise tier." + type = string +} + +variable "region" { + description = "Location for Filestore instances at Enterprise tier." + type = string +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is connected given in the format: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "name" { + description = "The resource name of the instance." + type = string + default = null +} + +variable "filestore_share_name" { + description = "Name of the file system share on the instance." + type = string + default = "nfsshare" +} + +variable "local_mount" { + description = "Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md))." + type = string + default = "/shared" +} + +variable "size_gb" { + description = "Storage size of the filestore instance in GB." + type = number + default = 1024 + validation { + condition = var.size_gb >= 1024 + error_message = "No Filestore tier supports less than 1024GiB.\nSee https://cloud.google.com/filestore/docs/service-tiers." + } +} + +variable "filestore_tier" { + description = "The service tier of the instance." + type = string + default = "BASIC_HDD" + validation { + condition = var.filestore_tier != "STANDARD" + error_message = "The preferred name for STANDARD tier is now BASIC_HDD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." + } + validation { + condition = var.filestore_tier != "PREMIUM" + error_message = "The preferred name for PREMIUM tier is now BASIC_SSD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." + } + validation { + condition = contains([ + "BASIC_HDD", + "BASIC_SSD", + "HIGH_SCALE_SSD", + "ZONAL", + "REGIONAL", + "ENTERPRISE" + ], var.filestore_tier) + # Avoid adding the legacy tier name in error_message, for e.g. 'HIGH_SCALE_SSD', 'ENTERPRISE'. + # As we want to steer the customer to new one's, but also support the legacy ones for older customers. + error_message = "Allowed values for filestore_tier are 'BASIC_HDD','BASIC_SSD','ZONAL','REGIONAL'.\nhttps://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#tier\nhttps://cloud.google.com/filestore/docs/reference/rest/v1/Tier." + } +} + +variable "labels" { + description = "Labels to add to the filestore instance. Key-value pairs." + type = map(string) +} + +variable "connect_mode" { + description = "Used to select mode - supported values DIRECT_PEERING and PRIVATE_SERVICE_ACCESS." + type = string + default = "DIRECT_PEERING" + nullable = false + validation { + condition = contains(["DIRECT_PEERING", "PRIVATE_SERVICE_ACCESS"], var.connect_mode) + error_message = "Allowed values for connect_mode are \"DIRECT_PEERING\" or \"PRIVATE_SERVICE_ACCESS\"." + } +} + +variable "nfs_export_options" { + description = "Define NFS export options." + type = list(object({ + access_mode = optional(string) + ip_ranges = optional(list(string)) + squash_mode = optional(string) + })) + default = [] + nullable = false +} + +variable "reserved_ip_range" { + description = <<-EOT + Reserved IP range for Filestore instance. Users are encouraged to set to null + for automatic selection. If supplied, it must be: + + CIDR format when var.connect_mode == "DIRECT_PEERING" + Named IP Range when var.connect_mode == "PRIVATE_SERVICE_ACCESS" + + See Cloud documentation for more details: + + https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range + EOT + type = string + default = null + nullable = true +} + +variable "mount_options" { + description = "NFS mount options to mount file system." + type = string + default = "defaults,_netdev" +} + +variable "deletion_protection" { + description = "Configure Filestore instance deletion protection" + type = object({ + enabled = optional(bool, false) + reason = optional(string) + }) + default = { + enabled = false + } + nullable = false + + validation { + condition = !can(coalesce(var.deletion_protection.reason)) || var.deletion_protection.enabled + error_message = "Cannot set Filestore var.deletion_protection.reason unless var.deletion_protection.enabled is true" + } +} + +variable "protocol" { + description = "NFS protocol version. Default is NFS_V3. NFS_V4_1 is only supported with HIGH_SCALE_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers." + type = string + default = "NFS_V3" + validation { + condition = contains(["NFS_V3", "NFS_V4_1"], var.protocol) + error_message = "Allowed values for protocol are 'NFS_V3' or 'NFS_V4_1'." + } +} + +variable "description" { + description = "A description of the filestore instance." + type = string + default = "" + validation { + condition = length(var.description) <= 2048 + error_message = "Filestore description must be 2048 characters or fewer" + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/versions.tf new file mode 100644 index 0000000000..1ba0e7967e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.4" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/README.md new file mode 100644 index 0000000000..88ae4511e3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/README.md @@ -0,0 +1,200 @@ +## Description + +This module creates Kubernetes Persistent Volumes (PV) and Persistent Volume +Claims (PVC) that can be used by a [gke-job-template]. + +`gke-persistent-volume` works with Filestore, Google Cloud Storage and Managed Lustre. Each +`gke-persistent-volume` can only be used with a single file system so if multiple +shared file systems are used then multiple `gke-persistent-volume` modules are +needed in the blueprint. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a Filestore and then uses the +`gke-persistent-volume` module to use the Filestore as shared storage in a +`gke-job-template`. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] + settings: + master_authorized_networks: + - display_name: deployment-machine + cidr_block: /32 + + - id: datafs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /data + + - id: datafs-pv + source: modules/file-system/gke-persistent-volume + use: [datafs, gke_cluster] + + - id: job-template + source: modules/compute/gke-job-template + use: [datafs-pv, compute_pool, gke_cluster] +``` + +The following example creates a GCS bucket and then uses the +`gke-persistent-volume` module to use the bucket as shared storage in a +`gke-job-template`. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] + settings: + master_authorized_networks: + - display_name: deployment-machine + cidr_block: /32 + + - id: data-bucket + source: modules/file-system/cloud-storage-bucket + settings: + local_mount: /data + + - id: datagcs-pv + source: modules/file-system/gke-persistent-volume + use: [data-bucket, gke_cluster] + + - id: job-template + source: modules/compute/gke-job-template + use: [datagcs-pv, compute_pool, gke_cluster] +``` + +The following example creates a Managed Lustre and then uses the +`gke-persistent-volume` module to use the Lustre as shared storage in a +`gke-job-template`. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] + settings: + master_authorized_networks: + - display_name: deployment-machine + cidr_block: /32 + + - id: data-managedlustre + source: modules/file-system/managed-lustre + settings: + local_mount: /data + + - id: datalustre-pv + source: modules/file-system/gke-persistent-volume + use: [data-managedlustre, gke_cluster] + + - id: job-template + source: modules/compute/gke-job-template + use: [datalustre-pv, compute_pool, gke_cluster] +``` + +See example +[storage-gke.yaml](../../../../examples/README.md#storage-gkeyaml--) blueprint +for a complete example. + +### Authorized Network + +Since the `gke-persistent-volume` module is making calls to the Kubernetes API +to create Kubernetes entities, the machine performing the deployment must be +authorized to connect to the Kubernetes API. You can add the +`master_authorized_networks` settings block, as shown in the example above, with +the IP address of the machine performing the deployment. This will ensure that +the deploying machine can connect to the cluster. + +### Connecting Via Use + +The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit +modules. For example the `gke-persistent-volume` module can `use` a +`gke-cluster` module and a `filestore` module, as shown in the example above. + +```mermaid + graph TD; + vpc--> |OneToMany| gke-cluster; + gke-cluster--> |OneToMany| gke-node-pool; + gke-node-pool--> |ManyToMany| gke-job-template; + gke-cluster--> |OneToMany| gke-persistent-volume; + gke-persistent-volume--> |ManyToMany| gke-job-template; + vpc--> |OneToMany| filestore; + vpc--> |OneToMany| gcs; + vpc--> |OneToMany| managed-lustre; + filestore--> |OneToOne| gke-persistent-volume; + gcs--> |OneToOne| gke-persistent-volume; + managed-lustre--> |OneToOne| gke-persistent-volume; + ``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [kubectl](#provider\_kubectl) | >= 1.7.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubectl_manifest.pv](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.pvc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.pvc_namespace](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [capacity\_gib](#input\_capacity\_gib) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | +| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | +| [filestore\_id](#input\_filestore\_id) | An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | +| [gcs\_bucket\_name](#input\_gcs\_bucket\_name) | The gcs bucket to be used with the persistent volume. | `string` | `null` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [lustre\_id](#input\_lustre\_id) | An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | +| [network\_storage](#input\_network\_storage) | Network attached storage mount to be configured. | 
object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  })
| n/a | yes | +| [pv\_name](#input\_pv\_name) | The name for PV. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | +| [pvc\_name](#input\_pvc\_name) | The name for PVC. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object describing the Kubernetes PersistentVolumeClaim created by this module. | +| [pvc\_name](#output\_pvc\_name) | The name of the Kubernetes PVC created by this module. | + diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/main.tf new file mode 100644 index 0000000000..818ebaf595 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/main.tf @@ -0,0 +1,155 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-persistent-volume", ghpc_role = "file-system" }) +} + +locals { + # Flags indicating which storage type is active based on input variables. + storage_type_active = { + gcs = var.gcs_bucket_name != null + lustre = var.lustre_id != null + filestore = var.filestore_id != null + } + + # Determine the active storage type name. + active_types = [for type, is_active in local.storage_type_active : type if is_active] + + # The precondition in kubectl_manifest.pv ensures exactly one type is active. + storage_type = length(local.active_types) > 0 ? local.active_types[0] : "unknown" + + # Map containing the base name derivation logic for each storage type. + base_name_map = { + gcs = var.gcs_bucket_name + lustre = var.lustre_id != null ? split("/", var.lustre_id)[5] : null + filestore = var.filestore_id != null ? split("/", var.filestore_id)[5] : null + } + # Retrieve the base name for the active storage type. + base_name = local.base_name_map[local.storage_type] + + # PV and PVC names + pv_name = var.pv_name != null ? var.pv_name : "${local.base_name}-pv" + pvc_name = var.pvc_name != null ? var.pvc_name : "${local.base_name}-pvc" + + # Template file paths + pv_templates = { + gcs = "${path.module}/templates/gcs-pv.yaml.tftpl" + lustre = "${path.module}/templates/managed-lustre-pv.yaml.tftpl" + filestore = "${path.module}/templates/filestore-pv.yaml.tftpl" + } + pvc_templates = { + gcs = "${path.module}/templates/gcs-pvc.yaml.tftpl" + lustre = "${path.module}/templates/managed-lustre-pvc.yaml.tftpl" + filestore = "${path.module}/templates/filestore-pvc.yaml.tftpl" + } + + # Common variables for all PVC templates + common_pvc_vars = { + pv_name = local.pv_name + pvc_name = local.pvc_name + labels = local.labels + capacity = "${var.capacity_gib}Gi" + namespace = var.namespace + } + + # Common variables for all PV templates + common_pv_vars = { + pv_name = local.pv_name + capacity = "${var.capacity_gib}Gi" + labels = local.labels + } + + # Variables for PV templates, merging common vars with type-specific ones. + pv_template_vars = { + gcs = merge(local.common_pv_vars, { + mount_options = var.gcs_bucket_name != null ? split(",", var.network_storage.mount_options) : [] + bucket_name = var.gcs_bucket_name + namespace = var.namespace + pvc_name = local.pvc_name + }) + lustre = merge(local.common_pv_vars, { + location = var.lustre_id != null ? split("/", var.lustre_id)[3] : null + project = split("/", var.cluster_id)[1] + instance_name = local.base_name + server_ip = var.lustre_id != null ? split("@", var.network_storage.server_ip)[0] : null + filesystem_name = var.network_storage.remote_mount + pvc_name = local.pvc_name + namespace = var.namespace + }) + filestore = merge(local.common_pv_vars, { + location = var.filestore_id != null ? split("/", var.filestore_id)[3] : null + filestore_name = local.base_name + share_name = trimprefix(var.network_storage.remote_mount, "/") + ip_address = var.network_storage.server_ip + pvc_name = local.pvc_name + namespace = var.namespace + }) + } + + # Rendered YAML contents + pv_content = templatefile( + local.pv_templates[local.storage_type], + local.pv_template_vars[local.storage_type] + ) + pvc_content = templatefile( + local.pvc_templates[local.storage_type], + local.common_pvc_vars + ) + + # GKE Cluster details + cluster_name = split("/", var.cluster_id)[5] + cluster_location = split("/", var.cluster_id)[3] +} + +data "google_container_cluster" "gke_cluster" { + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +provider "kubectl" { + host = "https://${data.google_container_cluster.gke_cluster.endpoint}" + cluster_ca_certificate = base64decode(data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) + token = data.google_client_config.default.access_token + load_config_file = false +} + +resource "kubectl_manifest" "pvc_namespace" { + count = var.namespace != "default" ? 1 : 0 + + yaml_body = templatefile("${path.module}/templates/namespace.yaml.tftpl", { + namespace = var.namespace + }) +} + +resource "kubectl_manifest" "pv" { + yaml_body = local.pv_content + + lifecycle { + precondition { + condition = length(local.active_types) == 1 + error_message = "Exactly one of gcs_bucket_name, filestore_id, or lustre_id must be set." + } + } +} + +resource "kubectl_manifest" "pvc" { + yaml_body = local.pvc_content + depends_on = [kubectl_manifest.pv, kubectl_manifest.pvc_namespace] +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf new file mode 100644 index 0000000000..60cf2dbe0f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "persistent_volume_claims" { + description = "An object describing the Kubernetes PersistentVolumeClaim created by this module." + value = { + name = local.pvc_name + namespace = var.namespace + mount_path = var.network_storage.local_mount + mount_options = var.network_storage.mount_options + storage_type = local.storage_type + } +} + +output "pvc_name" { + description = "The name of the Kubernetes PVC created by this module." + value = local.pvc_name +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl new file mode 100644 index 0000000000..06a1276c1e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl @@ -0,0 +1,26 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${pv_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + storageClassName: "" + capacity: + storage: ${capacity} + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + volumeMode: Filesystem + csi: + driver: filestore.csi.storage.gke.io + volumeHandle: "modeInstance/${location}/${filestore_name}/${share_name}" + volumeAttributes: + ip: ${ip_address} + volume: ${share_name} + claimRef: + name: ${pvc_name} + namespace: ${namespace} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl new file mode 100644 index 0000000000..83cfb3bc8c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl @@ -0,0 +1,18 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: ${pv_name} + resources: + requests: + storage: ${capacity} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl new file mode 100644 index 0000000000..aa0e570a8b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${pv_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + storageClassName: "" + capacity: + storage: ${capacity} + accessModes: + - ReadWriteMany + %{~ if mount_options != null ~} + mountOptions: + %{~ for key in mount_options ~} + - ${key} + %{~ endfor ~} + %{~ endif ~} + csi: + driver: gcsfuse.csi.storage.gke.io + volumeHandle: ${bucket_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl new file mode 100644 index 0000000000..4d02c85629 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: ${pv_name} + resources: + requests: + storage: ${capacity} + claimRef: + name: ${pvc_name} + namespace: ${namespace} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl new file mode 100644 index 0000000000..2b3b5e7738 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl @@ -0,0 +1,26 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${pv_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + storageClassName: "" + capacity: + storage: ${capacity} + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + volumeMode: Filesystem + claimRef: + namespace: ${namespace} + name: ${pvc_name} + csi: + driver: lustre.csi.storage.gke.io + volumeHandle: "${project}/${location}/${instance_name}/default-pool/default-container" + volumeAttributes: + ip: ${server_ip} + filesystem: ${filesystem_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl new file mode 100644 index 0000000000..83cfb3bc8c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl @@ -0,0 +1,18 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: ${pv_name} + resources: + requests: + storage: ${capacity} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl new file mode 100644 index 0000000000..fa7647e33f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf new file mode 100644 index 0000000000..fd281756e7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf @@ -0,0 +1,93 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "cluster_id" { + description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" + type = string +} + +variable "network_storage" { + description = "Network attached storage mount to be configured." + type = object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + }) +} + +variable "filestore_id" { + description = "An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." + type = string + default = null + validation { + condition = ( + var.filestore_id == null || + try(length(split("/", var.filestore_id)), 0) == 6 + ) + error_message = "filestore_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." + } +} + +variable "lustre_id" { + description = "An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." + type = string + default = null + validation { + condition = ( + var.lustre_id == null || + try(length(split("/", var.lustre_id)), 0) == 6 + ) + error_message = "lustre_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." + } +} + +variable "gcs_bucket_name" { + description = "The gcs bucket to be used with the persistent volume." + type = string + default = null +} + +variable "capacity_gib" { + description = "The storage capacity with which to create the persistent volume." + type = number +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} + +variable "namespace" { + description = "Kubernetes namespace to deploy the storage PVC/PV" + type = string + default = "default" +} + +variable "pv_name" { + description = "The name for PV. IF not set, a name will be generated based on the storage name." + type = string + default = null +} + +variable "pvc_name" { + description = "The name for PVC. IF not set, a name will be generated based on the storage name." + type = string + default = null +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf new file mode 100644 index 0000000000..fa1c3e2b3f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf @@ -0,0 +1,30 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.7.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.74.0" + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/README.md new file mode 100644 index 0000000000..78ef5402aa --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/README.md @@ -0,0 +1,134 @@ +## Description + +This module creates Kubernetes Storage Class (SC) that can be used by a Persistent Volume Claim (PVC) +to dynamically provision GCP storage resources like Parallelstore. + +### Example + +The following example uses the `gke-storage` module to creates a Parallelstore Storage Class and Persistent Volume Claim, +then use them in a `gke-job-template` to dynamically provision the resource. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network] + settings: + enable_parallelstore_csi: true + + # Private Service Access (PSA) requires the compute.networkAdmin role which is + # included in the Owner role, but not Editor. + # PSA is required for all Parallelstore functionality. + # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: gke_storage + source: modules/file-system/gke-storage + use: [ gke_cluster, private_service_access ] + settings: + storage_type: Parallelstore + access_mode: ReadWriteMany + sc_volume_binding_mode: Immediate + sc_reclaim_policy: Delete + sc_topology_zones: [$(vars.zone)] + pvc_count: 2 + capacity_gb: 12000 + + - id: job_template + source: modules/compute/gke-job-template + use: [gke_storage, compute_pool] +``` + +See example +[gke-managed-parallelstore.yaml](../../../examples/README.md#gke-managed-parallelstoreyaml--) blueprint +for a complete example. + +### Authorized Network + +Since the `gke-storage` module is making calls to the Kubernetes API +to create Kubernetes entities, the machine performing the deployment must be +authorized to connect to the Kubernetes API. You can add the +`master_authorized_networks` settings block, as shown in the example above, with +the IP address of the machine performing the deployment. This will ensure that +the deploying machine can connect to the cluster. + +### Connecting Via Use + +The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit +modules. For example the `gke-storage` module can `use` a +`gke-cluster` module and a `private_service_access` module, as shown in the example above. + +```mermaid +graph TD; + vpc-->|OneToMany|gke-cluster; + gke-cluster-->|OneToMany|gke-node-pool; + gke-node-pool-->|ManyToMany|gke-job-template; + gke-cluster-->|OneToMany|gke-storage; + gke-storage-->|ManyToMany|gke-job-template; +``` + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_mode](#input\_access\_mode) | The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)
Valid access modes:
- ReadWriteOnce
- ReadOnlyMany
- ReadWriteMany
- ReadWriteOncePod | `string` | n/a | yes | +| [capacity\_gb](#input\_capacity\_gb) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | +| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [mount\_options](#input\_mount\_options) | Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class. | `string` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [pv\_mount\_path](#input\_pv\_mount\_path) | Path within the container at which the volume should be mounted. Must not contain ':'. | `string` | `"/data"` | no | +| [pvc\_count](#input\_pvc\_count) | How many PersistentVolumeClaims that will be created | `number` | `1` | no | +| [sc\_reclaim\_policy](#input\_sc\_reclaim\_policy) | Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted.
[More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming)
Supported value:
- Retain
- Delete | `string` | n/a | yes | +| [sc\_topology\_zones](#input\_sc\_topology\_zones) | Zone location that allow the volumes to be dynamically provisioned. | `list(string)` | `null` | no | +| [sc\_volume\_binding\_mode](#input\_sc\_volume\_binding\_mode) | Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound.
Supported value:
- Immediate
- WaitForFirstConsumer | `string` | `"WaitForFirstConsumer"` | no | +| [storage\_type](#input\_storage\_type) | The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview)
to used. This module currently support dynamic provisioning for the below storage options
- Parallelstore
- Hyperdisk-balanced
- Hyperdisk-throughput
- Hyperdisk-extreme | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object that describes a k8s PVC created by this module. | + diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/main.tf new file mode 100644 index 0000000000..9c9a641f79 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/main.tf @@ -0,0 +1,86 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-storage", ghpc_role = "file-system" }) +} + +locals { + storage_type = lower(var.storage_type) + storage_class_name = "${local.storage_type}-sc" + pvc_name_prefix = "${local.storage_type}-pvc" +} + +check "private_vpc_connection_peering" { + assert { + condition = lower(var.storage_type) != "parallelstore" ? true : var.private_vpc_connection_peering != null + error_message = <<-EOT + Parallelstore must be run within the same VPC as the GKE cluster and have private services access enabled. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access. + If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). + EOT + } +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + # count = var.pvc_count + apply_manifests = flatten( + [ + # create StorageClass in the cluster + { + content = templatefile( + "${path.module}/storage-class/${local.storage_class_name}.yaml.tftpl", + { + name = local.storage_class_name + labels = local.labels + volume_binding_mode = var.sc_volume_binding_mode + reclaim_policy = var.sc_reclaim_policy + topology_zones = var.sc_topology_zones + }) + }, + var.namespace != "default" ? [{ + content = templatefile( + "${path.module}/persistent-volume-claim/namespace.yaml.tftpl", + { + namespace = var.namespace + }) + }] : [], + # create PersistentVolumeClaim in the cluster + flatten([ + for idx in range(var.pvc_count) : [ + { + content = templatefile( + "${path.module}/persistent-volume-claim/${(local.pvc_name_prefix)}.yaml.tftpl", + { + pvc_name = "${local.pvc_name_prefix}-${idx}" + labels = local.labels + capacity = "${var.capacity_gb}Gi" + access_mode = var.access_mode + storage_class_name = local.storage_class_name + namespace = var.namespace + } + ) + } + ] + ]) + ]) +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/metadata.yaml new file mode 100644 index 0000000000..8722823274 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/outputs.tf new file mode 100644 index 0000000000..ce80cdb266 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/outputs.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "persistent_volume_claims" { + description = "An object that describes a k8s PVC created by this module." + value = flatten([ + for idx in range(var.pvc_count) : [{ + name = "${local.pvc_name_prefix}-${idx}" + namespace = var.namespace + mount_path = "${var.pv_mount_path}/${local.pvc_name_prefix}-${idx}" + mount_options = var.mount_options + storage_type = local.storage_type + }] + ]) +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl new file mode 100644 index 0000000000..fa7647e33f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl new file mode 100644 index 0000000000..46e1f023d3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl @@ -0,0 +1,25 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + type: hyperdisk-balanced + provisioned-throughput-on-create: "250Mi" + provisioned-iops-on-create: "7000" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl new file mode 100644 index 0000000000..445020d001 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl @@ -0,0 +1,24 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + %{~ endfor ~} + type: hyperdisk-extreme + provisioned-iops-on-create: "50000" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl new file mode 100644 index 0000000000..ec404aec45 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl @@ -0,0 +1,24 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + type: hyperdisk-throughput + provisioned-throughput-on-create: "250Mi" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl new file mode 100644 index 0000000000..e6b8ea8d3e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl @@ -0,0 +1,21 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: parallelstore.csi.storage.gke.io +parameters: +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/variables.tf new file mode 100644 index 0000000000..dba1c33b77 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/variables.tf @@ -0,0 +1,144 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" + type = string +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} + +variable "storage_type" { + description = <<-EOT + The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview) + to used. This module currently support dynamic provisioning for the below storage options + - Parallelstore + - Hyperdisk-balanced + - Hyperdisk-throughput + - Hyperdisk-extreme + EOT + type = string + nullable = false + validation { + condition = var.storage_type == null ? false : contains(["parallelstore", "hyperdisk-balanced", "hyperdisk-throughput", "hyperdisk-extreme"], lower(var.storage_type)) + error_message = "Allowed string values for var.storage_type are \"Parallelstore\", \"Hyperdisk-balanced\", \"Hyperdisk-throughput\", \"Hyperdisk-extreme\"." + } +} + +variable "access_mode" { + description = <<-EOT + The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) + Valid access modes: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + EOT + type = string + nullable = false + validation { + condition = var.access_mode == null ? false : contains(["readwriteonce", "readonlymany", "readwritemany", "readwriteoncepod"], lower(var.access_mode)) + error_message = "Allowed string values for var.access_mode are \"ReadWriteOnce\", \"ReadOnlyMany\", \"ReadWriteMany\", \"ReadWriteOncePod\"." + } +} + +variable "sc_volume_binding_mode" { + description = <<-EOT + Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound. + Supported value: + - Immediate + - WaitForFirstConsumer + EOT + type = string + default = "WaitForFirstConsumer" + validation { + condition = var.sc_volume_binding_mode == null ? true : contains(["immediate", "waitforfirstconsumer"], lower(var.sc_volume_binding_mode)) + error_message = "Allowed string values for var.sc_volume_binding_mode are \"Immediate\", \"WaitForFirstConsumer\"." + } +} + +variable "sc_reclaim_policy" { + description = <<-EOT + Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted. + [More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming) + Supported value: + - Retain + - Delete + EOT + type = string + nullable = false + validation { + condition = var.sc_reclaim_policy == null ? true : contains(["retain", "delete"], lower(var.sc_reclaim_policy)) + error_message = "Allowed string values for var.sc_reclaim_policy are \"Retain\", \"Delete\"." + } +} + +variable "sc_topology_zones" { + description = "Zone location that allow the volumes to be dynamically provisioned." + type = list(string) + default = null +} + +variable "pvc_count" { + description = "How many PersistentVolumeClaims that will be created" + type = number + default = 1 +} + +variable "pv_mount_path" { + description = "Path within the container at which the volume should be mounted. Must not contain ':'." + type = string + default = "/data" + validation { + condition = var.pv_mount_path == null ? true : !strcontains(var.pv_mount_path, ":") + error_message = "pv_mount_path must not contain ':', please correct it and retry" + } +} + +variable "mount_options" { + description = "Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class." + type = string + default = null +} + +variable "capacity_gb" { + description = "The storage capacity with which to create the persistent volume." + type = number +} + +variable "private_vpc_connection_peering" { + description = <<-EOT + The name of the VPC Network peering connection. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access and + If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). + EOT + type = string + default = null +} + +variable "namespace" { + description = "Kubernetes namespace to deploy the storage PVC/PV" + type = string + default = "default" +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/versions.tf new file mode 100644 index 0000000000..bcc803e41e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/versions.tf @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.74.0" + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/README.md new file mode 100644 index 0000000000..28530a379f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/README.md @@ -0,0 +1,289 @@ +## Description + +This module creates a [Managed Lustre](https://cloud.google.com/managed-lustre) +instance. Managed Lustre is a high performance network file system that can be +mounted to one or more VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +### Supported Operating Systems + +A Managed Lustre instance can be used with Slurm cluster or compute +VM running Ubuntu 20.04, 22.04 or Rocky Linux 8 (including the HPC flavor). + +### Managed Lustre Access + +Managed Lustre must be enabled for your project by Google staff. Please contact +your sales representative for further steps. + +### Example - New VPC + +For Managed Lustre instance, the snippet below creates new VPC and configures +private-service-access for this newly created network. Both items are required +to be passed to the Lustre module to ensure that they're built in order and +that the correct subnetwork has private service access. + +```yaml + - id: network + source: modules/network/vpc + + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] +``` + +### Example - Slurm + +When using Slurm you must take into consideration whether or not you are using +an official image from the `schedmd-slurm-public` project or building your own. +The Lustre client modules are pre-installed in the official images. With the +official images, Lustre can be used as follows: + +```yaml +- id: managed_lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + +# Other modules: nodesets, partitions, login, etc. + +- id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: + - network + - lustre_partition + - managed_lustre + - slurm_login + settings: + machine_type: n2-standard-4 + enable_controller_public_ips: true +``` + +For custom images you must install the modules during the image build as the +Slurm cluster will not run the installation script like it does for the +standard VMs. + +Assuming you have a startup script for the Slurm image building, you can add +this Ansible playbook to correctly install the Lustre drivers into the image +(for Slurm-GCP versions greater than 6.10.0): + +```yaml +- type: data + destination: /var/tmp/slurm_vars.json + content: | + { + "reboot": false, + "install_cuda": false, + "install_gcsfuse": true, + "install_lustre": false, + "install_managed_lustre": true, + "install_nvidia_repo": true, + "install_ompi": true, + "allow_kernel_upgrades": false, + "monitoring_agent": "cloud-ops", + } +``` + +The `install_managed_lustre: true` line specifies that slurm-gcp should install +the correct modules within the slurm image. This runner should be placed +ahead of the script that calls the ansible build of the slurm-gcp image. + +### Example - Existing VPC + +If you want to use existing network with private-service-access configured, you need +to manually provide `private_vpc_connection_peering` to the Managed Lustre module. +You can get this details from the Google Cloud Console UI in `VPC network peering` +section. Below is the example of using existing network and creating Managed Lustre. +If existing network is not configured with private-service-access, you can follow +[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) +to set it up. + +```yaml + - id: network + source: modules/network/pre-existing-vpc + settings: + network_name: // Add network name + subnetwork_name: // Add subnetwork name + + - id: lustre + source: modules/file-system/managed-lustre + use: [network] + settings: + private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" +``` + +### Example - GKE compatibility + +By default the Managed Lustre instance that is deployed is not compatible with +GKE. To enable the compatibility use the `gke_support_enabled: true` option. +This creates a file `/etc/modprobe/lnet.conf` that changes the listening port +to 6988. + +```yaml + - id: managed-lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + gke_support_enabled: true +``` + +> [!WARNING] +> +> 1. VMs cannot connect to both GKE compatible and GKE incompatible lustre +> instances at the same time as they connect to different ports. Lustre can +> only listen to one port at a time. +> +> 2. Setting `gke_support_enabled: true` will not affect Slurm nodes, GKE +> compatibility must be built into the Slurm image. + +### Example - Importing data from GSC Bucket + +One option with the Managed Lustre instance is to import data from a GSC bucket +upon the lustre instance creation. To do this, use the `import_gcs_bucket_uri` +variable to dictate the bucket to pull data from. The data will be imported +under the directory specified by `local_mount` (`/shared` if unspecified). + +> [!NOTE] +> +> 1. This is a one way operation. Once the data has been copied to the lustre +> instance it will not be updated with any changes made to the GCS bucket. +> +> 2. Once the lustre instance has been created in Terraform, the copy process +> will proceed in the background. Data may not be appear in the mounted +> directory for a period of time after the deployment has completed (see below). + +```yaml +- id: managed_lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + import_gcs_bucket_uri: gs:// +``` + +> [!WARNING] +> Please follow [this guide](https://cloud.google.com/managed-lustre/docs/transfer-data#required_permissions) +> to set up the correct IAM permissions for importing data from GCS to lustre. +> Without this, the copy process may fail silently leaving an empty lustre +> instance. + +If an import is requested, gcluster will output a json response similar to: + +```json +{ + "name": "projects//locations//operations/", + "metadata": { + "@type": "type.googleapis.com/google.cloud.lustre.v1.ImportDataMetadata", + "createTime": "", + "target": "projects//locations//instances/", + "requestedCancellation": false, + "apiVersion": "v1" + }, + "done": false +} +``` + +You can retrieve more information about the transfer using the following +command, substituting with values from the json response above: + +```bash +gcloud lustre operations describe --location --project +``` + +This will provide information on if the transfer is complete or if any errors +have occurred. See more at +[Get operation](https://cloud.google.com/managed-lustre/docs/transfer-data#get_operation). + +## License + + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 6.27.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.27.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_lustre_instance.lustre_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/lustre_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | +| [google_storage_bucket.lustre_import_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the Lustre instance if no name is specified. | `string` | n/a | yes | +| [description](#input\_description) | Description of the created Lustre instance. | `string` | `"Lustre Instance"` | no | +| [gke\_support\_enabled](#input\_gke\_support\_enabled) | Set to true to create Managed Lustre instance with GKE compatibility.
Note: This does not work with Slurm, the Slurm image must be built with
the correct compatibility. | `bool` | `false` | no | +| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to managed lustre. Data will
be imported to the local\_mount directory. Changing this value will not
trigger a redeployment, to prevent data deletion. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the Managed Lustre instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mount](#input\_local\_mount) | Local mount point for the Managed Lustre instance. | `string` | `"/shared"` | no | +| [mount\_options](#input\_mount\_options) | Mounting options for the file system. | `string` | `"defaults,_netdev"` | no | +| [name](#input\_name) | Name of the Lustre instance | `string` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | Network self-link this instance will be on, required for checking private service access | `string` | n/a | yes | +| [per\_unit\_storage\_throughput](#input\_per\_unit\_storage\_throughput) | Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000. | `number` | `500` | no | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | ID of project in which Lustre instance will be created. | `string` | n/a | yes | +| [remote\_mount](#input\_remote\_mount) | Remote mount point of the Managed Lustre instance | `string` | n/a | yes | +| [size\_gib](#input\_size\_gib) | Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations | `number` | `36000` | no | +| [zone](#input\_zone) | Location for the Lustre instance. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | +| [install\_managed\_lustre\_client](#output\_install\_managed\_lustre\_client) | Script for installing Managed Lustre client | +| [lustre\_id](#output\_lustre\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | +| [network\_storage](#output\_network\_storage) | Describes a Managed Lustre instance. | + diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/main.tf new file mode 100644 index 0000000000..a969c53673 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/main.tf @@ -0,0 +1,104 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "managed-lustre", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +data "google_compute_network_peering" "private_peering" { + name = var.private_vpc_connection_peering + network = var.network_self_link +} + +locals { + server_ip = split(":", google_lustre_instance.lustre_instance.mount_point)[0] + remote_mount = split(":", google_lustre_instance.lustre_instance.mount_point)[1] + fs_type = "lustre" + mount_options = var.mount_options + instance_id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + destination_path = "/" + + install_managed_lustre_client_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-managed-lustre-client.sh" + "destination" = "install-managed-lustre-client${replace(var.local_mount, "/", "_")}.sh" + "args" = var.gke_support_enabled ? "1" : "0" + } + mount_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" + } + + bucket_count = try(length(data.google_storage_bucket.lustre_import_bucket), 0) +} + +data "google_storage_bucket" "lustre_import_bucket" { + count = try(length(var.import_gcs_bucket_uri) > 0, false) ? 1 : 0 + + name = split("//", var.import_gcs_bucket_uri)[1] +} + +resource "google_lustre_instance" "lustre_instance" { + project = var.project_id + + description = var.description + instance_id = local.instance_id + location = var.zone + + filesystem = var.remote_mount + capacity_gib = var.size_gib + per_unit_storage_throughput = var.per_unit_storage_throughput + + labels = local.labels + network = var.network_id + + gke_support_enabled = var.gke_support_enabled + + timeouts { + create = "1h" + update = "1h" + delete = "1h" + } + + depends_on = [var.private_vpc_connection_peering, data.google_storage_bucket.lustre_import_bucket] + + lifecycle { + precondition { + condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" + error_message = "The subnetwork that the lustre instance is hosted on must have private service access." + } + } + + provisioner "local-exec" { + command = < 0 ]]; then + curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + -d '{"gcsPath": {"uri":"${coalesce(var.import_gcs_bucket_uri, "gs://")}"}, "lustrePath": {"path":"${local.destination_path}"}}' \ + https://lustre.googleapis.com/v1/projects/${var.project_id}/locations/${var.zone}/instances/${local.instance_id}:importData + fi + EOF + interpreter = ["bash", "-c"] + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/metadata.yaml new file mode 100644 index 0000000000..66da9827b6 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - lustre.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/outputs.tf new file mode 100644 index 0000000000..6de815524a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/outputs.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a Managed Lustre instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = local.mount_options + client_install_runner = local.install_managed_lustre_client_runner + mount_runner = local.mount_runner + } +} + +output "install_managed_lustre_client" { + description = "Script for installing Managed Lustre client" + value = file("${path.module}/scripts/install-managed-lustre-client.sh") +} + +output "lustre_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" + value = google_lustre_instance.lustre_instance.id +} + +output "capacity_gib" { + description = "File share capacity in GiB." + value = google_lustre_instance.lustre_instance.capacity_gib +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh new file mode 100644 index 0000000000..878130ab47 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install Managed Lustre client modules +# Based on these instructions: https://cloud.google.com/managed-lustre/docs/connect-from-compute-engine + +# The client modules currently only support Rocky 8, and Ubuntu 20.04/22.04 + +set -e + +GKE_ENABLED=$1 + +# Update lnet to enable GKE supported Lustre instance +if [[ $GKE_ENABLED == "1" ]]; then + if [[ -f "/etc/modprobe.d/lnet.conf" ]] && grep -Fq "options lnet accept_port" /etc/modprobe.d/lnet.conf; then + echo "Lnet accept port already set, continuing without updating /etc/modprobe.d/lnet.conf" + else + echo "options lnet accept_port=6988" >>/etc/modprobe.d/lnet.conf + fi +fi + +if grep -q lustre /proc/filesystems; then + echo "Skipping managed lustre client install as it is already supported" + exit 0 +fi + +# Get distro information +. /etc/os-release +DIST="NA" +if [[ $NAME == *"Ubuntu"* ]]; then + if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then + DIST="Ubuntu" + fi +elif [[ $NAME == *"Rocky"* ]]; then + if [[ $VERSION_ID == "8"* ]]; then + DIST="Rocky" + fi +fi + +if [[ ${DIST} == "Ubuntu" ]]; then + KEY_LOC=/etc/apt/keyrings + KEY_NAME=gcp-ar-repo.gpg + # Download new repo key + mkdir -p "${KEY_LOC}" + wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null + + # Set up apt repo + echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list + + # Install modules + apt update + apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) +elif [[ ${DIST} == "Rocky" ]]; then + # Set up yum repo + touch /etc/yum.repos.d/artifact-registry.repo + tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF + [lustre-client-rocky-8] + name=lustre-client-rocky-8 + baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + # Install modules + yum makecache + yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client +fi + +if [[ $DIST != "NA" ]]; then + # Load the new lustre client module + modprobe lustre +fi diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/variables.tf new file mode 100644 index 0000000000..65607af66d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/variables.tf @@ -0,0 +1,131 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which Lustre instance will be created." + type = string +} + +variable "description" { + description = "Description of the created Lustre instance." + type = string + default = "Lustre Instance" +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the Lustre instance if no name is specified." + type = string +} + +variable "zone" { + description = "Location for the Lustre instance." + type = string +} + +variable "name" { + description = "Name of the Lustre instance" + type = string +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is connected given in the format: + `projects//global/networks/`" + EOT + type = string + nullable = false + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "network_self_link" { + description = "Network self-link this instance will be on, required for checking private service access" + type = string + nullable = false +} + +variable "remote_mount" { + description = "Remote mount point of the Managed Lustre instance" + type = string + nullable = false +} + +variable "local_mount" { + description = "Local mount point for the Managed Lustre instance." + type = string + default = "/shared" +} + +variable "size_gib" { + description = "Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations" + type = number + default = 36000 +} + +variable "per_unit_storage_throughput" { + description = "Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000." + type = number + default = 500 +} + +variable "labels" { + description = "Labels to add to the Managed Lustre instance. Key-value pairs." + type = map(string) +} + +variable "mount_options" { + description = "Mounting options for the file system." + type = string + default = "defaults,_netdev" +} + +variable "private_vpc_connection_peering" { + description = <<-EOT + The name of the VPC Network peering connection. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access and + If using existing VPC with private-service-access enabled, set this manually." + EOT + type = string + nullable = false +} + +variable "gke_support_enabled" { + description = <<-EOT + Set to true to create Managed Lustre instance with GKE compatibility. + Note: This does not work with Slurm, the Slurm image must be built with + the correct compatibility. + EOT + type = bool + nullable = false + default = false +} + +variable "import_gcs_bucket_uri" { + description = <<-EOT + The name of the GCS bucket to import data from to managed lustre. Data will + be imported to the local_mount directory. Changing this value will not + trigger a redeployment, to prevent data deletion. + EOT + type = string + default = null + + validation { + condition = startswith(coalesce(var.import_gcs_bucket_uri, "gs://"), "gs://") + error_message = "The GCS bucket uri must start with 'gs://'" + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/versions.tf new file mode 100644 index 0000000000..2322c9a8fd --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.27.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/README.md new file mode 100644 index 0000000000..82332f3406 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/README.md @@ -0,0 +1,193 @@ +## Description + +This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) +storage pool. + +NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. +NetApp Volume provides: + +- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x +- a [rich feature set][service-levels] +- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) +- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data +- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost + +Support for NetApp Volumes is split into two modules. + +- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. +- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +### NetApp storage pool service levels + +The netapp-storage-pool module currently supports the following NetApp Volumes [service levels][service-levels]: + +- Standard: 16 KiBps throughput per provisioned KiB of volume capacity. +- Premium: 64 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. +- Extreme: 128 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. + +Check the [service level matrix][service-levels] for additional information on capability differences between service levels. Flex service levels are currently not supported, but you can connect to existing Flex volumes using the [pre-existing-network-storage module][pre-existing]. + +### On-boarding NetApp Volumes +NetApp Volumes uses [Private Service Access](https://cloud.google.com/vpc/docs/private-services-access) (PSA) to connect volumes to your network. Before you create a storage pool, make sure to [connect NetApp Volumes to your network](https://cloud.google.com/netapp/volumes/docs/get-started/configure-access/networking). + +Example of creating a storage pool using a new network: + +```yaml +deployment_groups: +- group: primary + modules: + - id: network + source: modules/network/vpc + settings: + region: $(vars.region) + + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + service_name: "netapp.servicenetworking.goog" + deletion_policy: "ABANDON" + + - id: netapp_pool + source: modules/file-system/netapp-storage-pool + use: [network, private_service_access] + settings: + pool_name: $(vars.deployment_name)-eda-pool + capacity_gib: 20000 + service_level: "EXTREME" + region: $(vars.region) +``` + +Example of creating a storage pool using an existing network which was already PSA-peered with NetApp Volume: + +```yaml +deployment_groups: + - group: primary + modules: + - id: network + source: modules/network/pre-existing-vpc + settings: + project_id: $(vars.project_id) + region: $(vars.region) + network_name: $(vars.network) + + - id: netapp_pool + source: modules/file-system/netapp-storage-pool + use: [network] + settings: + pool_name: "eda-pool" + capacity_gib: 20000 + service_level: "EXTREME" + region: $(vars.region) +``` + +### Storage pool example + +The following example shows all available parameters in use: + +```yaml + - id: netapp_pool + source: modules/file-system/netapp-storage-pool + use: [network, private_service_access] + settings: + pool_name: "mypool" + region: "us-west4" + capacity_gib: 2048 + service_level: "EXTREME" + active_directory_policy: "projects/myproject/locations/us-east4/activeDirectories/my-ad" + cmek_policy: "projects/myproject/locations/us-east4/kmsConfigs/my-cmek-policy" + ldap_enabled: false + allow_auto_tiering: false + description: "Demo storage pool" + labels: + owner: bob +``` + +### NetApp Volumes quota + +Your project must have unused quota for NetApp Volumes in the region you will +provision the storage pool. This can be found by browsing to the [Quota tab within IAM & Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. +Please note that there are separate quota limits for Standard and Premium/Extreme service levels. + +See also NetApp Volumes [default quotas](https://cloud.google.com/netapp/volumes/docs/quotas#netapp-volumes-default-quotas). + +[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels +[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering +[pre-existing]: ../pre-existing-network-storage/README.md +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [google](#requirement\_google) | >= 6.45.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.45.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_netapp_storage_pool.netapp_storage_pool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_storage_pool) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [active\_directory\_policy](#input\_active\_directory\_policy) | The ID of the Active Directory policy to apply to the storage pool in the format:
`projects//locations//activeDirectoryPolicies/` | `string` | `null` | no | +| [allow\_auto\_tiering](#input\_allow\_auto\_tiering) | Whether to allow automatic tiering for the storage pool. | `bool` | `false` | no | +| [capacity\_gib](#input\_capacity\_gib) | The capacity of the storage pool in GiB. | `number` | `2048` | no | +| [cmek\_policy](#input\_cmek\_policy) | The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format:
`projects//locations//kmsConfigs/` | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used as name of the NetApp storage pool if no name is specified. | `string` | n/a | yes | +| [description](#input\_description) | A description of the NetApp storage pool. | `string` | `""` | no | +| [labels](#input\_labels) | Labels to add to the NetApp storage pool. Key-value pairs. | `map(string)` | n/a | yes | +| [ldap\_enabled](#input\_ldap\_enabled) | Whether to enable LDAP for the storage pool. | `bool` | `false` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | Network self-link the pool will be on, required for checking private service access | `string` | n/a | yes | +| [pool\_name](#input\_pool\_name) | The name of the storage pool. Leave empty to generate name based on deployment name. | `string` | `null` | no | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the private VPC connection peering. | `string` | `"sn-netapp-prod"` | no | +| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | +| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | +| [service\_level](#input\_service\_level) | The service level of the storage pool. | `string` | `"PREMIUM"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gb](#output\_capacity\_gb) | Storage pool capacity in GiB. | +| [netapp\_storage\_pool\_id](#output\_netapp\_storage\_pool\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}` | + diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/main.tf new file mode 100644 index 0000000000..b9d63c11c3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/main.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "netapp-storage-pool", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +data "google_compute_network_peering" "private_peering" { + name = var.private_vpc_connection_peering + network = var.network_self_link +} + +resource "google_netapp_storage_pool" "netapp_storage_pool" { + project = var.project_id + + name = var.pool_name != null ? var.pool_name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + location = var.region + network = var.network_id + service_level = var.service_level + capacity_gib = var.capacity_gib + + active_directory = var.active_directory_policy + kms_config = var.cmek_policy + ldap_enabled = var.ldap_enabled + allow_auto_tiering = var.allow_auto_tiering + + description = var.description + labels = local.labels + + depends_on = [data.google_compute_network_peering.private_peering] + + lifecycle { + precondition { + condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" + error_message = "The network for the storage pool must have private service access." + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml new file mode 100644 index 0000000000..7a5291f9d5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - netapp.googleapis.com + - servicenetworking.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf new file mode 100644 index 0000000000..91379631c6 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "netapp_storage_pool_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}`" + value = google_netapp_storage_pool.netapp_storage_pool.id +} + +output "capacity_gb" { + description = "Storage pool capacity in GiB." + value = google_netapp_storage_pool.netapp_storage_pool.capacity_gib +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf new file mode 100644 index 0000000000..04f19fd3fb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf @@ -0,0 +1,133 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which the NetApp storage pool will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the deployment, used as name of the NetApp storage pool if no name is specified." + type = string +} + +variable "region" { + description = "Location for NetApp storage pool." + type = string +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "network_self_link" { + description = "Network self-link the pool will be on, required for checking private service access" + type = string + nullable = false +} + +variable "private_vpc_connection_peering" { + description = "The name of the private VPC connection peering." + type = string + default = "sn-netapp-prod" +} + +variable "pool_name" { + description = "The name of the storage pool. Leave empty to generate name based on deployment name." + type = string + default = null +} + +variable "service_level" { + description = "The service level of the storage pool." + type = string + default = "PREMIUM" + validation { + condition = contains(["STANDARD", "PREMIUM", "EXTREME"], var.service_level) + error_message = "Allowed values for service_level are 'STANDARD', 'PREMIUM', or 'EXTREME'." + } +} + +variable "capacity_gib" { + description = "The capacity of the storage pool in GiB." + type = number + default = 2048 + validation { + condition = var.capacity_gib >= 2048 + error_message = "The minimum capacity for the storage pool is 2048 GiB." + } +} + +variable "active_directory_policy" { + description = <<-EOT + The ID of the Active Directory policy to apply to the storage pool in the format: + `projects//locations//activeDirectoryPolicies/` + EOT + type = string + default = null + validation { + condition = var.active_directory_policy == null ? true : length(split("/", var.active_directory_policy)) == 6 + error_message = "The active directory policy must be provided in the following format: projects//locations//activeDirectoryPolicies/." + } +} + +variable "cmek_policy" { + description = <<-EOT + The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format: + `projects//locations//kmsConfigs/` + EOT + type = string + default = null + validation { + condition = var.cmek_policy == null ? true : length(split("/", var.cmek_policy)) == 6 + error_message = "The CMEK policy must be provided in the following format: projects//locations//kmsConfigs/." + } +} + +variable "ldap_enabled" { + description = "Whether to enable LDAP for the storage pool." + type = bool + default = false +} + +variable "allow_auto_tiering" { + description = "Whether to allow automatic tiering for the storage pool." + type = bool + default = false +} + +variable "description" { + description = "A description of the NetApp storage pool." + type = string + default = "" + validation { + condition = length(var.description) <= 2048 + error_message = "NetApp storage pool description must be 2048 characters or fewer" + } +} + +variable "labels" { + description = "Labels to add to the NetApp storage pool. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf new file mode 100644 index 0000000000..f6501116cd --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.45.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" + } + + required_version = ">= 1.5.7" +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/README.md new file mode 100644 index 0000000000..6aaaf0cb05 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/README.md @@ -0,0 +1,201 @@ +## Description + +This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) +volume. + +NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. +NetApp Volume provides: + +- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x +- a [rich feature set][service-levels] +- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) +- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data +- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost + +Support for NetApp Volumes is split into two modules. + +- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. +- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +## Deletion protection +The netapp-volume module currently doesn't implement volume deletion protection. If you create a volume with Cluster Toolkit by using this module, Cluster Toolkit will also delete it when you run `gcluster destroy`. All the data in the volume will be gone. If you want to retain the volume instead, it is advised to [use existing volumes not created by Cluster Toolkit](#using-existing-volumes-not-created-by-cluster-toolkit). + +## Volumes overview +Volumes are filesystem containers which can be shared using NFS or SMB filesharing protocols. Volumes *live* inside of [storage pools](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview), which can be provisioned using the [netapp-storage-pool] module. Volumes inherit fundamental settings from the pool. They *consume* capacity provided by the pool. You can create one or multiple volumes *inside* a pool. + +[netapp-storage-pool]: ../netapp-storage-pool/README.md +[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels +[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering +[pre-existing]: ../pre-existing-network-storage/README.md +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## Volume examples +The following examples show the use of netapp-volume. They builds on top of an storage pool which can be provisioned using the [netapp-storage-pool][netapp-storage-pool] module. + +### Example with minimal parameters + +```yaml + - id: home_volume + source: modules/file-system/netapp-volume + use: [netapp_pool] # Create this pool using the netapp-storage-pool module + settings: + volume_name: "eda-home" + capacity_gib: 1024 # Size up to available capacity in the pool + local_mount: "/eda-home" # Mount point at client when client uses USE directive + protocols: ["NFSV3"] + region: $(vars.region) + # Default export policy exports to "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" and no_root_squash +``` + +### Example with all parameters + +```yaml + - id: shared_volume + source: modules/file-system/netapp-volume + use: [netapp_pool] # Create this pool using the netapp-storage-pool module + settings: + volume_name: "eda-shared" + capacity_gib: 25000 # Size up to available capacity in the pool + large_capacity: true + local_mount: "/shared" # Mount point at client when client uses USE directive + mount_options: "rw" # Allows customizing mount options for special workloads + protocols: ["NFSV3","NFSV4"] # List of protocols. ["NFSV3], ["NFSv4] or ["NFSV3, "NFSV4"] + region: $(vars.region) + unix_permissions: "0777" # Specify default permissions for roo inode owned by root:root + # If no export policy is specified, a permissive default policy will be applied, which is: + # allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" # RFC1918 + # has_root_access = true # no_root_squash enabled + # access_type = "READ_WRITE" + export_policy: + - allowed_clients: "10.10.20.8,10.10.20.9" + has_root_access: true # no_root_squash enabled + access_type: "READ_WRITE" + nfsv3: false # allow only NFSv4 for these hosts + nfsv4: true + - allowed_clients: "10.0.0.0/8" + has_root_access: false # no_root_squash disabled + access_type: "READ_WRITE" + nfsv3: true # allow only NFSv3 for these hosts + nfsv4: false + tiering_policy: # Enable auto-tiering. Requires auto-tiering enabled storage pool + tier_action: "ENABLED" + cooling_threshold_days: 31 # tier data blocks which have not been touched for 31 days + + description: "Shared volume for EDA job" + labels: + owner: bob +``` + +## Protocol support +Since Cluster Toolkit is currently built to provision Linux-based compute clusters, this module supports NFSv3 and NFSv4.1 only. SMB is blocked. + +## Large volumes +Volumes larger than 15 TiB can be created as [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). Such volumes can grow up to 3 PiB and can scale read performance up to 29 GiBps. They provide six IP addresses to the volume. They are exported via the `server_ips` output. When connecting a large volume to a client using the USE directive, cluster toolkit currently uses the first IP only. This will be improved in the future. + +This feature is allow-listed GA. To request allow-listing, see [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). + +## Auto-tiering support +For auto-tiering enabled storage pools you can enable auto-tiering on the volume. For more information, see [manage auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering). + +## Using existing volumes not created by Cluster Toolkit +NetApp Volumes volumes are regular NFS exports. You can use the [pre-existing-network-storage] module to integrate them into Cluster Toolkit. + +Example code: + +```yaml +- id: homefs + source: modules/file-system/pre-existing-network-storage + settings: + server_ip: ## Set server IP here ## + remote_mount: nfsshare + local_mount: /home + fs_type: nfs +``` + +This creates a resource in Cluster Toolkit which references the specified NFS export, which will be mounted at `/home` by clients which mount if via USE directive. + +Note that the `server_ip` must be known before deployment and this module does not allow +to specify a list of IPs for large volumes. + +[pre-existing-network-storage]: ../pre-existing-network-storage/README.md + +## FlexCache support +NetApp FlexCache technology accelerates data access, reduces WAN latency and lowers WAN bandwidth costs for read-intensive workloads, especially where clients need to access the same data repeatedly. When you create a FlexCache volume, you create a remote cache of an already existing (origin) volume that contains only the actively accessed data (hot data) of the origin volume. + +The FlexCache support in Google Cloud NetApp Volumes allows you to provision a cache volume in your Google network to improve performance for hybrid cloud environments. A FlexCache volume can help you transition workloads to the hybrid cloud by caching data from an on-premises data center to cloud. + +Deploying FlexCache volumes requires manual steps on the ONTAP origin side, which are not automated. Therefore this module has no support to deploy FlexCache volumes today. Deploy them manually and use the [pre-existing-network-storage](#using-existing-volumes-not-created-by-cluster-toolkit) instead. + +## License + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [google](#requirement\_google) | >= 6.45.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.45.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_netapp_volume.netapp_volume](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_volume) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [capacity\_gib](#input\_capacity\_gib) | The capacity of the volume in GiB. | `number` | `1024` | no | +| [description](#input\_description) | A description of the NetApp volume. | `string` | `""` | no | +| [export\_policy\_rules](#input\_export\_policy\_rules) | Define NFS export policy. | 
list(object({
    allowed_clients = optional(string)
    has_root_access = optional(bool, false)
    access_type     = optional(string, "READ_WRITE")
    nfsv3           = optional(bool)
    nfsv4           = optional(bool)
  }))
| 
[
  {
    "access_type": "READ_WRITE",
    "allowed_clients": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
    "has_root_access": true
  }
]
| no | +| [labels](#input\_labels) | Labels to add to the NetApp volume. Key-value pairs. | `map(string)` | n/a | yes | +| [large\_capacity](#input\_large\_capacity) | If true, the volume will be created with large capacity.
Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. | `bool` | `false` | no | +| [local\_mount](#input\_local\_mount) | Mountpoint for this volume. | `string` | `"/shared"` | no | +| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"rw,hard,rsize=65536,wsize=65536,tcp"` | no | +| [netapp\_storage\_pool\_id](#input\_netapp\_storage\_pool\_id) | The ID of the NetApp storage pool to use for the volume. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | +| [protocols](#input\_protocols) | The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported. | `list(string)` | 
[
  "NFSV3"
]
| no | +| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | +| [tiering\_policy](#input\_tiering\_policy) | Define the tiering policy for the NetApp volume. | 
object({
    tier_action            = optional(string)
    cooling_threshold_days = optional(number)
  })
| `null` | no | +| [unix\_permissions](#input\_unix\_permissions) | UNIX permissions for root inode in the volume. | `string` | `"0777"` | no | +| [volume\_name](#input\_volume\_name) | The name of the volume. Needs to be unique within the storage pool. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gb](#output\_capacity\_gb) | Volume capacity in GiB. | +| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | +| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | +| [netapp\_volume\_id](#output\_netapp\_volume\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}` | +| [network\_storage](#output\_network\_storage) | Describes a NetApp Volumes volume. | +| [server\_ips](#output\_server\_ips) | List of IP addresses of the volume. | + diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/main.tf new file mode 100644 index 0000000000..d8345bf347 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/main.tf @@ -0,0 +1,92 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "netapp-volume", ghpc_role = "file-system" }) +} + +# resource "random_id" "resource_name_suffix" { +# byte_length = 4 +# } + +locals { + full_path = split(":", google_netapp_volume.netapp_volume.mount_options[0].export_full) + server_ip = local.full_path[0] + remote_mount = local.full_path[1] + # Large volumes will have 6 IPs + server_ips = [for ip in google_netapp_volume.netapp_volume.mount_options[*].export_full : split(":", ip)[0]] + fs_type = "nfs" + mount_options = var.mount_options + + install_nfs_client_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-nfs-client.sh" + "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" + } + mount_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${join(",", local.server_ips)}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" + } + + split_pool_id = split("/", var.netapp_storage_pool_id) + pool_name = local.split_pool_id[5] +} + +resource "google_netapp_volume" "netapp_volume" { + project = var.project_id + + name = var.volume_name + share_name = var.volume_name + location = var.region + protocols = var.protocols + capacity_gib = var.capacity_gib + large_capacity = var.large_capacity + multiple_endpoints = var.large_capacity == true ? true : null + storage_pool = local.pool_name + unix_permissions = var.unix_permissions + + dynamic "tiering_policy" { + for_each = var.tiering_policy == null ? [] : [0] + content { + cooling_threshold_days = lookup(var.tiering_policy, "cooling_threshold_days", null) + tier_action = lookup(var.tiering_policy, "tier_action", null) + } + } + + description = var.description + labels = local.labels + + dynamic "export_policy" { + for_each = var.export_policy_rules == null ? [] : [0] + content { + dynamic "rules" { + for_each = var.export_policy_rules + content { + access_type = rules.value.access_type + allowed_clients = rules.value.allowed_clients + has_root_access = rules.value.has_root_access + nfsv3 = rules.value.nfsv3 == null ? contains([for p in var.protocols : lower(p)], "nfsv3") : rules.value.nfsv3 + nfsv4 = rules.value.nfsv4 == null ? contains([for p in var.protocols : lower(p)], "nfsv4") : rules.value.nfsv4 + } + } + } + } + + depends_on = [var.netapp_storage_pool_id] +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/metadata.yaml new file mode 100644 index 0000000000..e4a7aaaa14 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - netapp.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/outputs.tf new file mode 100644 index 0000000000..641eae007a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/outputs.tf @@ -0,0 +1,66 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +output "network_storage" { + description = "Describes a NetApp Volumes volume." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = local.mount_options + client_install_runner = local.install_nfs_client_runner + mount_runner = local.mount_runner + } +} + +output "install_nfs_client" { + description = "Script for installing NFS client" + value = file("${path.module}/scripts/install-nfs-client.sh") +} + +output "install_nfs_client_runner" { + description = "Runner to install NFS client using the startup-script module" + value = local.install_nfs_client_runner +} + +output "mount_runner" { + description = <<-EOT + Runner to mount the file-system using an ansible playbook. The startup-script + module will automatically handle installation of ansible. + - id: example-startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(your-fs-id.mount_runner) + ... + EOT + value = local.mount_runner +} + +output "netapp_volume_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}`" + value = google_netapp_volume.netapp_volume.id +} + +output "capacity_gb" { + description = "Volume capacity in GiB." + value = google_netapp_volume.netapp_volume.capacity_gib +} + +output "server_ips" { + description = "List of IP addresses of the volume." + value = local.server_ips +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..1b1595e5a4 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsupported distribution' + return 1 + fi +fi diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh new file mode 100644 index 0000000000..8253d40a24 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IPS=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +# accept a list of colon-separated IPs and randomly pick one to enable load balancing +# In recent changes cluster toolkit doesn't seem to use this file anymore, +# which makes all mounts use the first IP in the list. Needs to be investigated in future. +IFS="," read -r -a arrIPS <<<"${SERVER_IPS}" +rand1=$(od -vAn -t d -N1 /dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/variables.tf new file mode 100644 index 0000000000..272558ff77 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/variables.tf @@ -0,0 +1,133 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which the NetApp storage pool will be created." + type = string +} + +variable "netapp_storage_pool_id" { + description = "The ID of the NetApp storage pool to use for the volume." + type = string + validation { + condition = length(split("/", var.netapp_storage_pool_id)) == 6 + error_message = "The storage pool id must be provided in the following format: projects//locations//storagePools/." + } +} + +variable "region" { + description = "Location for NetApp storage pool." + type = string +} + +variable "volume_name" { + description = "The name of the volume. Needs to be unique within the storage pool." + type = string + default = null +} + +variable "capacity_gib" { + description = "The capacity of the volume in GiB." + type = number + default = 1024 + validation { + condition = var.capacity_gib >= 100 + error_message = "The minimum capacity for the volume is 100 GiB." + } +} + +variable "protocols" { + description = "The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported." + type = list(string) + default = ["NFSV3"] + validation { + condition = alltrue([for p in var.protocols : contains(["NFSV3", "NFSV4"], p)]) + error_message = "Allowed values for protocols are 'NFSV3' or 'NFSV4'." + } +} + +variable "description" { + description = "A description of the NetApp volume." + type = string + default = "" + validation { + condition = length(var.description) <= 2048 + error_message = "NetApp volume description must be 2048 characters or fewer" + } +} + +variable "labels" { + description = "Labels to add to the NetApp volume. Key-value pairs." + type = map(string) +} + +variable "local_mount" { + description = "Mountpoint for this volume." + type = string + default = "/shared" +} + +variable "mount_options" { + description = "NFS mount options to mount file system." + type = string + default = "rw,hard,rsize=65536,wsize=65536,tcp" +} + +variable "large_capacity" { + description = <<-EOT + If true, the volume will be created with large capacity. + Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. + EOT + type = bool + default = false +} + +variable "unix_permissions" { + description = "UNIX permissions for root inode in the volume." + type = string + default = "0777" + validation { + condition = length(var.unix_permissions) <= 4 + error_message = "UNIX permissions must be a 4-digit octal number." + } +} + +variable "tiering_policy" { + description = "Define the tiering policy for the NetApp volume." + type = object({ + tier_action = optional(string) + cooling_threshold_days = optional(number) + }) + default = null +} + +variable "export_policy_rules" { + description = "Define NFS export policy." + type = list(object({ + allowed_clients = optional(string) + has_root_access = optional(bool, false) + access_type = optional(string, "READ_WRITE") + nfsv3 = optional(bool) + nfsv4 = optional(bool) + })) + # Permissive default if user does not specify nfs_export_options. Allow all RFC1918 CIDRS with no_root_squash + default = [{ + allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", + has_root_access = true, + access_type = "READ_WRITE", + }] + nullable = true +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/versions.tf new file mode 100644 index 0000000000..c624d5100b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/versions.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.45.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" + } + + required_version = ">= 1.5.7" +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/README.md new file mode 100644 index 0000000000..0b942f067f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/README.md @@ -0,0 +1,196 @@ +## Description + +This module creates [parallelstore](https://cloud.google.com/parallelstore) +instance. Parallelstore is Google Cloud's first party parallel file system +service based on [Intel DAOS](https://docs.daos.io/v2.2/) + +### Supported Operating Systems + +A parallelstore instance can be used with Slurm cluster or compute +VM running Ubuntu 22.04, debian 12 or HPC Rocky Linux 8. + +### Parallelstore Quota + +To get access to a private preview of Parallelstore APIs, your project needs to +be allowlisted. To set this up, please work with your account representative. + +### Parallelstore mount options + +After parallelstore instance is created, you can specify mount options depending +upon your workload. DAOS is configured to deliver the best user experience for +interactive workloads with aggressive caching. If you are running parallel +workloads concurrently accessing the sane files from multiple client nodes, it +is recommended to disable the writeback cache to avoid cross-client consistency +issues. You can specify different mount options as follows, + +```yaml + - id: parallelstore + source: modules/file-system/parallelstore + use: [network, ps_connect] + settings: + mount_options: "disable-wb-cache,thread-count=20,eq-count=8" +``` + +### Example - New VPC + +For parallelstore instance, Below snippet creates new VPC and configures private-service-access +for this newly created network. + +```yaml + - id: network + source: modules/network/vpc + + # Private Service Access (PSA) requires the compute.networkAdmin role which is + # included in the Owner role, but not Editor. + # PSA is required for all Parallelstore functionality. + # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: parallelstore + source: modules/file-system/parallelstore + use: [network, private_service_access] +``` + +### Example - Existing VPC + +If you want to use existing network with private-service-access configured, you need +to manually provide `private_vpc_connection_peering` to the parallelstore module. +You can get this details from the Google Cloud Console UI in `VPC network peering` +section. Below is the example of using existing network and creating parallelstore. +If existing network is not configured with private-service-access, you can follow +[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) +to set it up. + +```yaml + - id: network + source: modules/network/pre-existing-vpc + settings: + network_name: // Add network name + subnetwork_name: // Add subnetwork name + + - id: parallelstore + source: modules/file-system/parallelstore + use: [network] + settings: + private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" +``` + +### Import data from GCS bucket + +You can import data from your GCS bucket to parallelstore instance. Important to +note that data may not be available to the instance immediately. This depends on +latency and size of data. Below is the example of importing data from bucket. + +```yaml + - id: parallelstore + source: modules/file-system/parallelstore + use: [network] + settings: + import_gcs_bucket_uri: gs://gcs-bucket/folder-path + import_destination_path: /gcs/import/ +``` + +Here you can replace `import_gcs_bucket_uri` with the uri of sub folder within GCS +bucket and `import_destination_path` with local directory within parallelstore +instance. + +### Additional configuration for DAOS agent and dfuse +Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + daos_agent_config: | + credential_config: + cache_expiration: 1m +``` + +Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/parallelstore + settings: + dfuse_environment: + D_LOG_FILE: /tmp/client.log + D_APPEND_PID_TO_LOG: 1 + D_LOG_MASK: debug +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13 | +| [google](#requirement\_google) | >= 6.13.0 | +| [null](#requirement\_null) | ~> 3.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.13.0 | +| [null](#provider\_null) | ~> 3.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_parallelstore_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/parallelstore_instance) | resource | +| [null_resource.hydration](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [daos\_agent\_config](#input\_daos\_agent\_config) | Additional configuration to be added to daos\_config.yml | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. | `string` | n/a | yes | +| [dfuse\_environment](#input\_dfuse\_environment) | Additional environment variables for DFuse process | `map(string)` | `{}` | no | +| [directory\_stripe](#input\_directory\_stripe) | The parallelstore stripe level for directories. | `string` | `null` | no | +| [file\_stripe](#input\_file\_stripe) | The parallelstore stripe level for files. | `string` | `null` | no | +| [import\_destination\_path](#input\_import\_destination\_path) | The name of local path to import data on parallelstore instance from GCS bucket. | `string` | `null` | no | +| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to parallelstore. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to parallel store instance. | `map(string)` | `{}` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/parallelstore"` | no | +| [mount\_options](#input\_mount\_options) | Options describing various aspects of the parallelstore instance. | `string` | `"disable-wb-cache,thread-count=16,eq-count=8"` | no | +| [name](#input\_name) | Name of parallelstore instance. | `string` | `null` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [size\_gb](#input\_size\_gb) | Storage size of the parallelstore instance in GB. | `number` | `12000` | no | +| [zone](#input\_zone) | Location for parallelstore instance. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions to monitor import-data operation from GCS bucket to parallelstore. | +| [network\_storage](#output\_network\_storage) | Describes a parallelstore instance. | + diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/main.tf new file mode 100644 index 0000000000..acc2a0551e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/main.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "parallelstore", ghpc_role = "file-system" }) +} + +locals { + fs_type = "daos" + server_ip = "" + remote_mount = "" + id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + access_points = jsonencode(google_parallelstore_instance.instance.access_points) + destination_path = var.import_destination_path == null ? "/" : var.import_destination_path + + client_install_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-daos-client.sh" + "destination" = "install_daos_client.sh" + } + + mount_runner = { + "type" = "shell" + "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { + access_points = local.access_points + daos_agent_config = var.daos_agent_config + dfuse_environment = var.dfuse_environment + local_mount = var.local_mount + mount_options = join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) + }) + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + } +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_parallelstore_instance" "instance" { + project = var.project_id + instance_id = local.id + location = var.zone + capacity_gib = var.size_gb + network = var.network_id + file_stripe_level = var.file_stripe + directory_stripe_level = var.directory_stripe + + labels = local.labels + + depends_on = [var.private_vpc_connection_peering] +} + +resource "null_resource" "hydration" { + count = var.import_gcs_bucket_uri != null ? 1 : 0 + + depends_on = [resource.google_parallelstore_instance.instance] + provisioner "local-exec" { + command = "curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -d '{\"source_gcs_bucket\": {\"uri\":\"${var.import_gcs_bucket_uri}\"}, \"destination_parallelstore\": {\"path\":\"${local.destination_path}\"}}' https://parallelstore.googleapis.com/v1beta/projects/${var.project_id}/locations/${var.zone}/instances/${local.id}:importData" + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/metadata.yaml new file mode 100644 index 0000000000..c0994d15bb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - parallelstore.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/outputs.tf new file mode 100644 index 0000000000..f6e817ac8a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/outputs.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + operation_instructions = <<-EOT + Data is being imported from GCS bucket to parallelstore instance. It may + not be available immediately. + EOT +} + +output "network_storage" { + description = "Describes a parallelstore instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = var.mount_options + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } + + precondition { + condition = var.import_gcs_bucket_uri != null || var.import_destination_path == null + error_message = <<-EOD + Please specify import_gcs_bucket_uri to import data to parallelstore instance. + EOD + } +} + +output "instructions" { + description = "Instructions to monitor import-data operation from GCS bucket to parallelstore." + value = var.import_gcs_bucket_uri != null ? local.operation_instructions : "Data is not imported from GCS bucket." +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh new file mode 100644 index 0000000000..e96eadb56a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || + { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 +fi + +if [ -x /bin/daos ]; then + echo "DAOS already installed" + daos version +else + # Install the DAOS client library + # The following commands should be executed on each client vm. + ## For Rocky linux 8 / RedHat 8. + if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then + # 1) Add the Parallelstore package repository + cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF + [parallelstore-v2-6-el${OS_VERSION_MAJOR}] + name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 + baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + + ## TODO: Remove disable automatic update script after issue is fixed. + if [ -x /usr/bin/google_disable_automatic_updates ]; then + /usr/bin/google_disable_automatic_updates + fi + dnf clean all + dnf makecache + + # 2) Install daos-client + dnf install -y epel-release # needed for capstone + dnf install -y daos-client + + # 3) Upgrade libfabric + dnf upgrade -y libfabric + + # For Ubuntu 22.04 and debian 12, + elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then + # shellcheck disable=SC2034 + DEBIAN_FRONTEND=noninteractive + + # 1) Add the Parallelstore package repository + curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg + echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list + + apt-get update + + # 2) Install daos-client + apt-get install -y daos-client + + # 3) Create daos_agent.service (comes pre-installed with RedHat) + if ! getent passwd daos_agent >/dev/null 2>&1; then + useradd daos_agent + fi + cat >/etc/systemd/system/daos_agent.service <<-EOF + [Unit] + Description=DAOS Agent + StartLimitIntervalSec=60 + Wants=network-online.target + After=network-online.target + + [Service] + Type=notify + User=daos_agent + Group=daos_agent + RuntimeDirectory=daos_agent + RuntimeDirectoryMode=0755 + ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml + StandardOutput=journal + StandardError=journal + Restart=always + RestartSec=10 + LimitMEMLOCK=infinity + LimitCORE=infinity + StartLimitBurst=5 + + [Install] + WantedBy=multi-user.target + EOF + else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + fi +fi + +exit 0 diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl new file mode 100644 index 0000000000..c6f5d53660 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl @@ -0,0 +1,110 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; } && { [[ "$${OS_VERSION_MAJOR}" = "8" ]] || [[ "$${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "$${OS_ID}" = "ubuntu" ]] && [[ "$${OS_VERSION}" = "22.04" ]]; } || + { [[ "$${OS_ID}" = "debian" ]] && [[ "$${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system $${OS_ID} $${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + +fi + +# Edit agent config +daos_config=/etc/daos/daos_agent.yml + +# rewrite $daos_config from scratch +mv $${daos_config} $${daos_config}.orig + +exclude_fabric_ifaces="" +# Get names of network interfaces not in first PCI slot +# The first PCI slot is a standard network adapter while remaining interfaces +# are typically network cards dedicated to GPU or workload communication +if [[ "$${OS_ID}" == "debian" ]] || [[ "$${OS_ID}" = "ubuntu" ]]; then + extra_interfaces=$(find /sys/class/net/ -not -name 'enp0s*' -regextype posix-extended -regex '.*/enp[0-9]+s.*' -printf '"%f"\n' | paste -s -d ',') +elif [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; then + extra_interfaces=$(find /sys/class/net/ -not -name eth0 -regextype posix-extended -regex '.*/eth[0-9]+' -printf '"%f"\n' | paste -s -d ',') +fi + +cat > $daos_config </etc/systemd/system/"$${service_name}" </global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "import_gcs_bucket_uri" { + description = "The name of the GCS bucket to import data from to parallelstore." + type = string + default = null +} + +variable "import_destination_path" { + description = "The name of local path to import data on parallelstore instance from GCS bucket." + type = string + default = null +} + +variable "file_stripe" { + description = "The parallelstore stripe level for files." + type = string + default = null + validation { + condition = var.file_stripe == null ? true : contains([ + "FILE_STRIPE_LEVEL_UNSPECIFIED", + "FILE_STRIPE_LEVEL_MIN", + "FILE_STRIPE_LEVEL_BALANCED", + "FILE_STRIPE_LEVEL_MAX", + ], var.file_stripe) + error_message = "var.file_stripe must be set to \"FILE_STRIPE_LEVEL_UNSPECIFIED\", \"FILE_STRIPE_LEVEL_MIN\", \"FILE_STRIPE_LEVEL_BALANCED\", or \"FILE_STRIPE_LEVEL_MAX\"" + } +} + +variable "directory_stripe" { + description = "The parallelstore stripe level for directories." + type = string + default = null + validation { + condition = var.directory_stripe == null ? true : contains([ + "DIRECTORY_STRIPE_LEVEL_UNSPECIFIED", + "DIRECTORY_STRIPE_LEVEL_MIN", + "DIRECTORY_STRIPE_LEVEL_BALANCED", + "DIRECTORY_STRIPE_LEVEL_MAX", + ], var.directory_stripe) + error_message = "var.directory_stripe must be set to \"DIRECTORY_STRIPE_LEVEL_UNSPECIFIED\", \"DIRECTORY_STRIPE_LEVEL_MIN\", \"DIRECTORY_STRIPE_LEVEL_BALANCED\", or \"DIRECTORY_STRIPE_LEVEL_MAX\"" + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/versions.tf new file mode 100644 index 0000000000..174b5281e4 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.13.0" + } + + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/README.md new file mode 100644 index 0000000000..47cf1518a1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/README.md @@ -0,0 +1,192 @@ +## Description + +This module defines a file-system that already exists (i.e. it does not create +a new file system) in a way that can be shared with other modules. This allows +a compute VM to mount a filesystem that is not part of the current deployment +group. + +The pre-existing network storage can be referenced in the same way as any Cluster +Toolkit supported file-system such as [filestore](../filestore/README.md). + +For more information on network storage options in the Cluster Toolkit, see +the extended [Network Storage documentation](../../../docs/network_storage.md). + +### Example + +```yaml +- id: homefs + source: modules/file-system/pre-existing-network-storage + settings: + server_ip: ## Set server IP here ## + remote_mount: nfsshare + local_mount: /home + fs_type: nfs +``` + +This creates a pre-existing-network-storage module in terraform at the +provided IP in `server_ip` of type nfs that will be mounted at `/home`. Note +that the `server_ip` must be known before deployment. + +The following is an example of using `pre-existing-network-storage` with a GCS +bucket: + +```yaml +- id: data-bucket + source: modules/file-system/pre-existing-network-storage + settings: + remote_mount: my-bucket-name + local_mount: /data + fs_type: gcsfuse + mount_options: defaults,_netdev,implicit_dirs +``` + +The `implicit_dirs` mount option allows object paths to be treated as if they +were directories. This is important when working with files that were created by +another source, but there may have performance impacts. The `_netdev` mount option +denotes that the storage device requires network access. + +The following is an example of using `pre-existing-network-storage` with the `lustre` +filesystem: + +```yaml +- id: lustrefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: lustre + server_ip: 192.168.227.11@tcp + local_mount: /scratch + remote_mount: /exacloud +``` + +Note the use of the MGS NID (Network ID) in the `server_ip` field - in +particular, note the `@tcp` suffix. + +The following is an example of using `pre-existing-network-storage` with the +`managed_lustre` filesystem: + +```yaml +- id: lustrefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: managed_lustre + server_ip: 192.168.227.11@tcp + local_mount: /scratch + remote_mount: /mg_lustre +``` + +This is similar to the `lustre` filesystem, with the exception that it connects +with a managed Lustre instance hosted by GCP. Currently only Rocky 8 and +Ubuntu 20.04 and Ubuntu 22.04 are supported. + +The following is an example of using `pre-existing-network-storage` with the `daos` +filesystem. In order to use existing `parallelstore` instance, `fs_type` needs to be +explicitly mentioned in blueprint. The `remote_mount` option refers to `access_points` +for `parallelstore` instance. + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 +``` + +Parallelstore supports additional options for its mountpoints under `parallelstore_options` setting. +Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 + parallelstore_options: + daos_agent_config: | + credential_config: + cache_expiration: 1m +``` + +Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 + parallelstore_options: + dfuse_environment: + D_LOG_FILE: /tmp/client.log + D_APPEND_PID_TO_LOG: 1 + D_LOG_MASK: debug +``` + +### Mounting + +For the `fs_type` listed below, this module will provide `client_install_runner` +and `mount_runner` outputs. These can be used to create a startup script to +mount the network storage system. + +Supported `fs_type`: + +- nfs +- lustre +- managed_lustre +- gcsfuse +- daos + +[scripts/mount.sh](./scripts/mount.sh) is used as the contents of +`mount_runner`. This script will update `/etc/fstab` and mount the network +storage. This script will fail if the specified `local_mount` is already being +used by another entry in `/etc/fstab`. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. + +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## License + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [fs\_type](#input\_fs\_type) | Type of file system to be mounted (e.g., nfs, lustre) | `string` | `"nfs"` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | +| [managed\_lustre\_options](#input\_managed\_lustre\_options) | Managed Lustre specific options:
gke\_support\_enabled (bool, default = false)
Note: gke\_support\_enabled does not work with Slurm, the Slurm image must be built with
the correct compatibility. | 
object({
    gke_support_enabled = optional(bool, false)
  })
| `{}` | no | +| [mount\_options](#input\_mount\_options) | Options describing various aspects of the file system. Consider adding setting to 'defaults,\_netdev,implicit\_dirs' when using gcsfuse. | `string` | `"defaults,_netdev"` | no | +| [parallelstore\_options](#input\_parallelstore\_options) | Parallelstore specific options | 
object({
    daos_agent_config = optional(string, "")
    dfuse_environment = optional(map(string), {})
  })
| `{}` | no | +| [remote\_mount](#input\_remote\_mount) | Remote FS name or export. This is the exported directory for nfs, fs name for lustre, and bucket name (without gs://) for gcsfuse. | `string` | n/a | yes | +| [server\_ip](#input\_server\_ip) | The device name as supplied to fs-tab, excluding remote fs-name(for nfs, that is the server IP, for lustre [:]). This can be omitted for gcsfuse. | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use file system. | +| [mount\_runner](#output\_mount\_runner) | Runner that mounts the file system. | +| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | + diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf new file mode 100644 index 0000000000..203b6dfdac --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf @@ -0,0 +1,124 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a remote network storage to be mounted by fs-tab." + value = { + server_ip = var.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = var.mount_options + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } +} + +locals { + # Update remote mount to include a slash if the fs_type requires one to exist + remote_mount_with_slash = length(regexall("^/.*", var.remote_mount)) > 0 ? ( + var.remote_mount + ) : format("/%s", var.remote_mount) + remote_mount = contains(local.mount_vanilla_supported_fstype, local.fs_type) ? ( + local.remote_mount_with_slash + ) : var.remote_mount + + ml_gke_support_enabled = coalesce(try(var.managed_lustre_options.gke_support_enabled, false), false) + + # Collapse fs_type lustre and managed lustre for most uses, only needs to be + # different for client installation + fs_type = strcontains(var.fs_type, "lustre") ? "lustre" : var.fs_type + + # Client Install + ddn_lustre_client_install_script = templatefile( + "${path.module}/templates/ddn_exascaler_luster_client_install.tftpl", + { + server_ip = split("@", var.server_ip)[0] + remote_mount = local.remote_mount + local_mount = var.local_mount + } + ) + managed_lustre_client_install_script = file("${path.module}/scripts/install-managed-lustre-client.sh") + nfs_client_install_script = file("${path.module}/scripts/install-nfs-client.sh") + gcs_fuse_install_script = file("${path.module}/scripts/install-gcs-fuse.sh") + daos_client_install_script = file("${path.module}/scripts/install-daos-client.sh") + + install_scripts = { + "lustre" = local.ddn_lustre_client_install_script + "managed_lustre" = local.managed_lustre_client_install_script + "nfs" = local.nfs_client_install_script + "gcsfuse" = local.gcs_fuse_install_script + "daos" = local.daos_client_install_script + } + + client_install_runner = { + "type" = "shell" + "content" = lookup(local.install_scripts, var.fs_type, "echo 'skipping: client_install_runner not yet supported for ${var.fs_type}'") + "destination" = "install_filesystem_client${replace(var.local_mount, "/", "_")}.sh" + "args" = local.ml_gke_support_enabled ? "1" : "" + } + + mount_vanilla_supported_fstype = ["lustre", "nfs"] + mount_runner_vanilla = { + "type" = "shell" + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"${var.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" + "content" = ( + contains(local.mount_vanilla_supported_fstype, local.fs_type) ? + file("${path.module}/scripts/mount.sh") : + "echo 'skipping: mount_runner not yet supported for ${var.fs_type}'" + ) + } + gcsbucket = trimprefix(var.remote_mount, "gs://") + mount_runner_gcsfuse = { + "type" = "shell" + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"not-used\" \"${local.gcsbucket}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" + "content" = file("${path.module}/scripts/mount.sh") + } + + mount_runner_daos = { + "type" = "shell" + "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { + access_points = var.remote_mount + daos_agent_config = var.parallelstore_options.daos_agent_config + dfuse_environment = var.parallelstore_options.dfuse_environment + local_mount = var.local_mount + # avoid passing "--" as mount option to dfuse + mount_options = length(var.mount_options) == 0 ? "" : join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) + }) + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + } + + mount_scripts = { + "lustre" = local.mount_runner_vanilla + "nfs" = local.mount_runner_vanilla + "gcsfuse" = local.mount_runner_gcsfuse + "daos" = local.mount_runner_daos + } + + mount_runner = lookup(local.mount_scripts, local.fs_type, local.mount_runner_vanilla) +} + +output "client_install_runner" { + description = "Runner that performs client installation needed to use file system." + value = local.client_install_runner +} + +output "mount_runner" { + description = "Runner that mounts the file system." + value = local.mount_runner +} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh new file mode 100644 index 0000000000..e96eadb56a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || + { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 +fi + +if [ -x /bin/daos ]; then + echo "DAOS already installed" + daos version +else + # Install the DAOS client library + # The following commands should be executed on each client vm. + ## For Rocky linux 8 / RedHat 8. + if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then + # 1) Add the Parallelstore package repository + cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF + [parallelstore-v2-6-el${OS_VERSION_MAJOR}] + name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 + baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + + ## TODO: Remove disable automatic update script after issue is fixed. + if [ -x /usr/bin/google_disable_automatic_updates ]; then + /usr/bin/google_disable_automatic_updates + fi + dnf clean all + dnf makecache + + # 2) Install daos-client + dnf install -y epel-release # needed for capstone + dnf install -y daos-client + + # 3) Upgrade libfabric + dnf upgrade -y libfabric + + # For Ubuntu 22.04 and debian 12, + elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then + # shellcheck disable=SC2034 + DEBIAN_FRONTEND=noninteractive + + # 1) Add the Parallelstore package repository + curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg + echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list + + apt-get update + + # 2) Install daos-client + apt-get install -y daos-client + + # 3) Create daos_agent.service (comes pre-installed with RedHat) + if ! getent passwd daos_agent >/dev/null 2>&1; then + useradd daos_agent + fi + cat >/etc/systemd/system/daos_agent.service <<-EOF + [Unit] + Description=DAOS Agent + StartLimitIntervalSec=60 + Wants=network-online.target + After=network-online.target + + [Service] + Type=notify + User=daos_agent + Group=daos_agent + RuntimeDirectory=daos_agent + RuntimeDirectoryMode=0755 + ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml + StandardOutput=journal + StandardError=journal + Restart=always + RestartSec=10 + LimitMEMLOCK=infinity + LimitCORE=infinity + StartLimitBurst=5 + + [Install] + WantedBy=multi-user.target + EOF + else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + fi +fi + +exit 0 diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh new file mode 100644 index 0000000000..f8a990260b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ ! "$(which gcsfuse)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then + tee /etc/yum.repos.d/gcsfuse.repo >/dev/null <>/etc/modprobe.d/lnet.conf + fi +fi + +if grep -q lustre /proc/filesystems; then + echo "Skipping managed lustre client install as it is already supported" + exit 0 +fi + +# Get distro information +. /etc/os-release +DIST="NA" +if [[ $NAME == *"Ubuntu"* ]]; then + if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then + DIST="Ubuntu" + fi +elif [[ $NAME == *"Rocky"* ]]; then + if [[ $VERSION_ID == "8"* ]]; then + DIST="Rocky" + fi +fi + +if [[ ${DIST} == "Ubuntu" ]]; then + KEY_LOC=/etc/apt/keyrings + KEY_NAME=gcp-ar-repo.gpg + # Download new repo key + mkdir -p "${KEY_LOC}" + wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null + + # Set up apt repo + echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list + + # Install modules + apt update + apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) +elif [[ ${DIST} == "Rocky" ]]; then + # Set up yum repo + touch /etc/yum.repos.d/artifact-registry.repo + tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF + [lustre-client-rocky-8] + name=lustre-client-rocky-8 + baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + # Install modules + yum makecache + yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client +fi + +if [[ $DIST != "NA" ]]; then + # Load the new lustre client module + modprobe lustre +fi diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl new file mode 100644 index 0000000000..f5f0291e85 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl @@ -0,0 +1,50 @@ +#!/bin/sh + +# Copyright 2022 DataDirect Networks +# Modifications Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prior Art: https://github.com/DDNStorage/exascaler-cloud-terraform/blob/78deadbb2c1fa7e4603cf9605b0f7d1782117954/gcp/templates/client-script.tftpl + +# install new EXAScaler Cloud clients: +# all instances must be in the same zone +# and connected to the same network and subnet +# to set up EXAScaler Cloud filesystem on a new client instance, +# run the following commands on the client with root privileges: +set -e +if [[ ! -z $(cat /proc/filesystems | grep lustre) ]]; then + echo "Skipping lustre client install as it is already supported" + exit 0 +fi + +cat >/etc/esc-client.conf< $daos_config </etc/systemd/system/"$${service_name}" < +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string
    count = number
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [guest\_accelerator](#output\_guest\_accelerator) | Sanitized list of the type and count of accelerator cards attached to the instance. | +| [machine\_type\_guest\_accelerator](#output\_machine\_type\_guest\_accelerator) | List of the type and count of accelerator cards attached to the specified machine type. | + diff --git a/deletion-test/cluster/modules/embedded/modules/internal/gpu-definition/main.tf b/deletion-test/cluster/modules/embedded/modules/internal/gpu-definition/main.tf new file mode 100644 index 0000000000..f0861cddc9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/gpu-definition/main.tf @@ -0,0 +1,98 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "machine_type" { + description = "Machine type to use for the instance creation" + type = string +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string + count = number + gpu_driver_installation_config = optional(object({ + gpu_driver_version = string + }), { gpu_driver_version = "DEFAULT" }) + gpu_partition_size = optional(string) + gpu_sharing_config = optional(object({ + gpu_sharing_strategy = string + max_shared_clients_per_gpu = number + })) + })) + default = [] + nullable = false +} + +locals { + # example state; terraform will ignore diffs if last element of URL matches + # guest_accelerator = [ + # { + # count = 1 + # type = "https://www.googleapis.com/compute/beta/projects/PROJECT/zones/ZONE/acceleratorTypes/nvidia-tesla-a100" + # }, + # ] + accelerator_machines = { + "a2-highgpu-1g" = { type = "nvidia-tesla-a100", count = 1 }, + "a2-highgpu-2g" = { type = "nvidia-tesla-a100", count = 2 }, + "a2-highgpu-4g" = { type = "nvidia-tesla-a100", count = 4 }, + "a2-highgpu-8g" = { type = "nvidia-tesla-a100", count = 8 }, + "a2-megagpu-16g" = { type = "nvidia-tesla-a100", count = 16 }, + "a2-ultragpu-1g" = { type = "nvidia-a100-80gb", count = 1 }, + "a2-ultragpu-2g" = { type = "nvidia-a100-80gb", count = 2 }, + "a2-ultragpu-4g" = { type = "nvidia-a100-80gb", count = 4 }, + "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, + "a3-highgpu-1g" = { type = "nvidia-h100-80gb", count = 1 }, + "a3-highgpu-2g" = { type = "nvidia-h100-80gb", count = 2 }, + "a3-highgpu-4g" = { type = "nvidia-h100-80gb", count = 4 }, + "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, + "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, + "a4-highgpu-8g-lowmem" = { type = "nvidia-b200", count = 8 }, + "a4-highgpu-8g" = { type = "nvidia-b200", count = 8 }, + "a4x-highgpu-4g" = { type = "nvidia-gb200", count = 4 }, + "a4x-highgpu-4g-nolssd" = { type = "nvidia-gb200", count = 4 }, + "g2-standard-4" = { type = "nvidia-l4", count = 1 }, + "g2-standard-8" = { type = "nvidia-l4", count = 1 }, + "g2-standard-12" = { type = "nvidia-l4", count = 1 }, + "g2-standard-16" = { type = "nvidia-l4", count = 1 }, + "g2-standard-24" = { type = "nvidia-l4", count = 2 }, + "g2-standard-32" = { type = "nvidia-l4", count = 1 }, + "g2-standard-48" = { type = "nvidia-l4", count = 4 }, + "g2-standard-96" = { type = "nvidia-l4", count = 8 }, + } + generated_guest_accelerator = try([local.accelerator_machines[var.machine_type]], []) + + # Select in priority order: + # (1) var.guest_accelerator if not empty + # (2) local.generated_guest_accelerator if not empty + # (3) default to empty list if both are empty + guest_accelerator = try(coalescelist(var.guest_accelerator, local.generated_guest_accelerator), []) +} + +output "guest_accelerator" { + description = "Sanitized list of the type and count of accelerator cards attached to the instance." + value = local.guest_accelerator +} + +output "machine_type_guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the specified machine type." + value = local.generated_guest_accelerator +} + +terraform { + required_version = ">= 1.3" +} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/README.md b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/README.md new file mode 100644 index 0000000000..21746fe0d8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/README.md @@ -0,0 +1,30 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [disk\_type](#input\_disk\_type) | The disk type to validate. | `string` | n/a | yes | +| [machine\_type](#input\_machine\_type) | The machine type to validate. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/main.tf b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/main.tf new file mode 100644 index 0000000000..d89d7edfec --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/main.tf @@ -0,0 +1,52 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +check "disk_type_c4_compatibility" { + assert { + condition = !(can(regex("^c4-", var.machine_type)) && var.disk_type == "pd-ssd") + error_message = "The C4 machine series does not support pd-ssd. Please use hyperdisk-balanced or another compatible disk type." + } +} + + +check "disk_type_c2_compatibility" { + assert { + condition = !(can(regex("^c2-", var.machine_type)) && can(regex("hyperdisk", var.disk_type))) + error_message = "The C2 machine series does not support Hyperdisk as a boot disk. Please use a compatible disk type like pd-ssd, pd-standard, or pd-balanced." + } +} + + +check "disk_type_pd_extreme_compatibility" { + assert { + condition = var.disk_type != "pd-extreme" || can(regex("^(m1-|m2-|m3-|n2-|n2d-)", var.machine_type)) + error_message = "pd-extreme disks are only supported for M1, M2, M3, N2, and N2D machine series." + } +} + + +check "disk_type_hyperdisk_extreme_compatibility" { + assert { + condition = var.disk_type != "hyperdisk-extreme" || can(regex("^(c3-|m1-|m3-|n2-)", var.machine_type)) + error_message = "hyperdisk-extreme disks are only supported for C3, M1, M3, and N2 machine series." + } +} + + +check "disk_type_hyperdisk_throughput_compatibility" { + assert { + condition = var.disk_type != "hyperdisk-throughput" || can(regex("^(c3-|c3d-|n4-|n2-|n2d-|n1-|t2d-|m1-)", var.machine_type)) + error_message = "hyperdisk-throughput disks are only supported for C3, C3D, N4, N2, N2D, N1, T2D, and M1 machine series." + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/variables.tf b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/variables.tf new file mode 100644 index 0000000000..23478051b3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/variables.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "machine_type" { + type = string + description = "The machine type to validate." +} + +variable "disk_type" { + type = string + description = "The disk type to validate." +} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/versions.tf b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/versions.tf new file mode 100644 index 0000000000..4702005614 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/versions.tf @@ -0,0 +1,17 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 0.15.0" +} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/README.md b/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/README.md new file mode 100644 index 0000000000..8aa9270a0a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/README.md @@ -0,0 +1,54 @@ + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | +| [google-beta](#requirement\_google-beta) | >= 6.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google-beta](#provider\_google-beta) | >= 6.0.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_network_attachment.self](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_network_attachment) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [connection\_preference](#input\_connection\_preference) | The connection preference of service attachment. | `string` | `"ACCEPT_AUTOMATIC"` | no | +| [name](#input\_name) | Name of the resource. Provided by the client when the resource is created | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The ID of the project in which the resource belongs. | `string` | n/a | yes | +| [region](#input\_region) | Region where the network attachment resides | `string` | n/a | yes | +| [subnetwork\_self\_links](#input\_subnetwork\_self\_links) | An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment. | `list(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [self\_link](#output\_self\_link) | Server-defined URL for the resource. | + diff --git a/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/main.tf b/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/main.tf new file mode 100644 index 0000000000..bbbece7085 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +variable "connection_preference" { + type = string + description = "The connection preference of service attachment." + default = "ACCEPT_AUTOMATIC" +} + +variable "subnetwork_self_links" { + type = list(string) + description = " An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment." +} + +variable "name" { + type = string + description = "Name of the resource. Provided by the client when the resource is created" +} + +variable "project_id" { + type = string + description = "The ID of the project in which the resource belongs." +} + +variable "region" { + type = string + description = "Region where the network attachment resides" +} + + +resource "google_compute_network_attachment" "self" { + provider = google-beta + + project = var.project_id + region = var.region + name = var.name + connection_preference = var.connection_preference + subnetworks = var.subnetwork_self_links +} + + +output "self_link" { + value = google_compute_network_attachment.self.self_link + description = "Server-defined URL for the resource." +} + +terraform { + required_version = ">= 0.15.0" + + required_providers { + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.0.0" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/metadata.yaml new file mode 100644 index 0000000000..e80fc96b9c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/README.md b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/README.md new file mode 100644 index 0000000000..610d82c1b9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/README.md @@ -0,0 +1,85 @@ +## Description + +This is an internal helper module designed to encapsulate and centralize all hardware-specific logic for Google Cloud TPUs. It is intended to be called by parent modules like `gke-node-pool` to determine if a node pool is TPU-based and to retrieve its specific attributes. + +This module's primary responsibilities are: + +* Reliably detect if a node pool is for TPUs by checking its `placement_policy`. +* Determine the correct GKE `tpu-accelerator` label based on the machine type family. +* Determine the `number of chips per node` based on the specific machine type. +* Generate the standard **Kubernetes taint** that should be applied to TPU nodes. + +This follows the same design pattern as the `gpu-definition` internal module, promoting a clean separation of concerns within the gke-node-pool module. + +## Usage + +This module is not intended for direct use in a blueprint. It should be called from a parent module like `gke-node-pool`. + +```yaml +module "tpu" { + source = "../../internal/tpu-definition" + + # Pass the parent module's variables to this module + machine_type = var.machine_type + placement_policy = var.placement_policy +} + +# Example of consuming the module's outputs in the parent module +locals { + # The tpu_taint is then used in the node_config's dynamic "taint" block + tpu_taint = module.tpu.tpu_taint +} +``` + +## License + + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [machine\_type](#input\_machine\_type) | The machine type of the node pool. | `string` | n/a | yes | +| [placement\_policy](#input\_placement\_policy) | The placement policy for the node pool. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [is\_tpu](#output\_is\_tpu) | Boolean value indicating if the node pool is for TPUs. | +| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | +| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | +| [tpu\_taint](#output\_tpu\_taint) | A list containing the standard TPU taint object if the node pool is for TPUs. | +| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | + diff --git a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/main.tf b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/main.tf new file mode 100644 index 0000000000..c8ee417d71 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/main.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # Determine if this is a TPU node pool by checking if the machine_type exists in our authoritative map of TPU machine types. + is_tpu = contains(keys(local.tpu_chip_count_map), var.machine_type) + + tpu_taint = local.is_tpu ? [{ + key = "google.com/tpu" + value = "present" + effect = "NO_SCHEDULE" + }] : [] + + # Map of machine prefixes to GKE accelerator labels. + tpu_accelerator_map = { + "ct4p" = "tpu-v4-podslice" # TPU v4 + "ct5lp" = "tpu-v5-lite-podslice" # TPU v5e + "ct5p" = "tpu-v5p-slice" # TPU v5p + "ct6e" = "tpu-v6e-slice" # TPU v6e + "tpu7x" = "tpu7x" # TPU v7x + } + + # Map specific GCE machine types to the number of TPU chips per node (VM). + # The machine-type map must be updated to reflect new TPU releases with reference to public documentation: https://docs.cloud.google.com/tpu/docs/intro-to-tpu + tpu_chip_count_map = { + # v4 - ct4p + "ct4p-hightpu-4t" = 4 + + # v5e - ct5lp + "ct5lp-hightpu-1t" = 1 + "ct5lp-hightpu-4t" = 4 + "ct5lp-hightpu-8t" = 8 + + # v5p - ct5p + "ct5p-hightpu-1t" = 1 + "ct5p-hightpu-2t" = 2 + "ct5p-hightpu-4t" = 4 + + # v6e - ct6e + "ct6e-standard-1t" = 1 + "ct6e-standard-4t" = 4 + "ct6e-standard-8t" = 8 + + # v7x - tpu7x + "tpu7x-standard-4t" = 4 + } + + # Robustly extract the machine family prefix (e.g., "ct6e"). + tpu_machine_family = local.is_tpu ? element(split("-", var.machine_type), 0) : "" + tpu_accelerator_type = local.is_tpu ? lookup(local.tpu_accelerator_map, local.tpu_machine_family, null) : null + tpu_chips_per_node = local.is_tpu ? lookup(local.tpu_chip_count_map, var.machine_type, null) : null +} + +terraform { + required_version = ">= 1.3" +} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/outputs.tf b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/outputs.tf new file mode 100644 index 0000000000..fa3c21fa34 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "is_tpu" { + description = "Boolean value indicating if the node pool is for TPUs." + value = local.is_tpu +} + +output "tpu_accelerator_type" { + description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." + value = local.tpu_accelerator_type +} + +output "tpu_topology" { + description = "The topology of the TPU slice (e.g., '4x4')." + value = local.is_tpu ? var.placement_policy.tpu_topology : null +} + +output "tpu_chips_per_node" { + description = "The number of TPU chips on each node in the pool." + value = local.tpu_chips_per_node +} + +output "tpu_taint" { + description = "A list containing the standard TPU taint object if the node pool is for TPUs." + value = local.tpu_taint +} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/variables.tf b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/variables.tf new file mode 100644 index 0000000000..254488c02d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/variables.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "machine_type" { + description = "The machine type of the node pool." + type = string +} + +variable "placement_policy" { + description = "The placement policy for the node pool." + type = object({ + type = string + name = optional(string) + tpu_topology = optional(string) + }) +} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/README.md b/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/README.md new file mode 100644 index 0000000000..aefac9d187 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/README.md @@ -0,0 +1,56 @@ + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_network_peering.peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [export\_custom\_routes](#input\_export\_custom\_routes) | (Optional) Whether to export the custom routes to the peer network. Defaults to false. | `bool` | `null` | no | +| [import\_custom\_routes](#input\_import\_custom\_routes) | (Optional) Whether to import the custom routes from the peer network. Defaults to false. | `bool` | `null` | no | +| [import\_subnet\_routes\_with\_public\_ip](#input\_import\_subnet\_routes\_with\_public\_ip) | (Optional) Whether subnet routes with public IP range are imported. | `bool` | `null` | no | +| [name](#input\_name) | Name of the peering. | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | The primary network of the peering. | `string` | n/a | yes | +| [peer\_network\_self\_link](#input\_peer\_network\_self\_link) | The peer network in the peering. The peer network may belong to a different project. | `string` | n/a | yes | +| [stack\_type](#input\_stack\_type) | (Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [peering\_name](#output\_peering\_name) | Name of the peering. | + diff --git a/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/main.tf b/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/main.tf new file mode 100644 index 0000000000..386fa9377b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/main.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "name" { + type = string + description = "Name of the peering." +} + +variable "network_self_link" { + type = string + description = "The primary network of the peering." +} + +variable "peer_network_self_link" { + type = string + description = "The peer network in the peering. The peer network may belong to a different project." +} + +variable "export_custom_routes" { + type = bool + description = "(Optional) Whether to export the custom routes to the peer network. Defaults to false." + default = null +} + +variable "import_custom_routes" { + type = bool + description = "(Optional) Whether to import the custom routes from the peer network. Defaults to false." + default = null +} + +variable "import_subnet_routes_with_public_ip" { + type = bool + description = "(Optional) Whether subnet routes with public IP range are imported. " + default = null +} + +variable "stack_type" { + type = string + description = "(Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. " + default = null +} + +resource "google_compute_network_peering" "peering" { + name = var.name + network = var.network_self_link + peer_network = var.peer_network_self_link + export_custom_routes = var.export_custom_routes + import_custom_routes = var.import_custom_routes + import_subnet_routes_with_public_ip = var.import_subnet_routes_with_public_ip + stack_type = var.stack_type +} + +output "peering_name" { + value = google_compute_network_peering.peering.name + description = "Name of the peering." +} + +terraform { + required_version = ">= 0.15.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/metadata.yaml new file mode 100644 index 0000000000..e80fc96b9c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/README.md b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/README.md new file mode 100644 index 0000000000..d7054eb725 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/README.md @@ -0,0 +1,244 @@ +## Description + +This module simplifies the following functionality: + +* Applying Kubernetes manifests to GKE clusters: It provides flexible options for specifying manifests, allowing you to either directly embed them as strings content or reference them from URLs, files, templates, or entire .yaml and .tftpl files in directories. +* Deploying commonly used infrastructure like [Kueue](https://kueue.sigs.k8s.io/docs/) or [Jobset](https://jobset.sigs.k8s.io/docs/). + +> Note: Kueue can work with a variety of frameworks out of the box, find them [here](https://kueue.sigs.k8s.io/docs/tasks/run/) + +### Explanation + +* **Manifest:** + * **Raw String:** Specify manifests directly within the module configuration using the `content: manifest_body` format. + * **File/Template/Directory Reference:** Set `source` to the path to: + * A single URL to a manifest file. Ex.: `https://github.com/.../myrepo/manifest.yaml`. + + > **Note:** Applying from a URL has important limitations. Please review the [Considerations & Callouts for Applying from URLs](#applying-manifests-from-urls-considerations--callouts) section below. + * A single local YAML manifest file (`.yaml`). Ex.: `./manifest.yaml`. + * A template file (`.tftpl`) to generate a manifest. Ex.: `./template.yaml.tftpl`. You can pass the variables to format the template file in `template_vars`. + * A directory containing multiple YAML or template files. Ex: `./manifests/`. You can pass the variables to format the template files in `template_vars`. + +#### Manifest Example + +```yaml +- id: existing-gke-cluster + source: modules/scheduler/pre-existing-gke-cluster + settings: + project_id: $(vars.project_id) + cluster_name: my-gke-cluster + region: us-central1 + +- id: kubectl-apply + source: modules/management/kubectl-apply + use: [existing-gke-cluster] + settings: + - content: | + apiVersion: v1 + kind: Namespace + metadata: + name: my-namespace + - source: "https://github.com/kubernetes-sigs/jobset/releases/download/v0.6.0/manifests.yaml" + - source: $(ghpc_stage("manifests/configmap1.yaml")) + - source: $(ghpc_stage("manifests/configmap2.yaml.tftpl")) + template_vars: {name: "dev-config", public: "false"} + - source: $(ghpc_stage("manifests"))/ + template_vars: {name: "dev-config", public: "false"} +``` + +#### Pre-build infrastructure Example + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml")) + jobset: + install: true +``` + +The `config_path` field in `kueue` installation accepts a template file, too. You will need to provide variables for the template using `config_template_vars` field. + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) + config_template_vars: {name: "dev-config", public: "false"} + jobset: + install: true +``` + +You can specify a particular kueue version that you would like to use using the `version` flag. By default, we recommend customers to [use v0.10.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L68). You can find the list of supported kueue versions [here](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L18). + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + version: v0.10.0 + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) + config_template_vars: {name: "dev-config", public: "false"} + jobset: + install: true +``` + +> **_NOTE:_** +> +> The `project_id` and `region` settings would be inferred from the deployment variables of the same name, but they are included here for clarity. +> +> Terraform may apply resources in parallel, leading to potential dependency issues. If a resource's dependencies aren't ready, it will be applied again up to 15 times. + +## Callouts + +### Applying Manifests from URLs: Considerations & Callouts + +While this module supports applying manifests directly from remote `http://` or `https://` URLs, this method introduces complexities not present when using local files. For production environments, we recommend sourcing manifests from local paths or a version-controlled Git repository. Moreover, this method will be deprecated soon. Hence we recommend to use other methods to source manifests. + +If you choose to use the URL method, be aware of the following potential issues and their solutions. + +#### **1. Apply Order and Race Conditions** + +The module applies manifests from the `apply_manifests` list in parallel. This can create a **race condition** if one manifest depends on another. The most common example is applying a manifest with custom resources (like a `ClusterQueue`) at the same time as the manifest that defines it (the `CustomResourceDefinition` or CRD). + +There is **no guarantee** that the CRD will be applied before the resource that uses it. This can lead to non-deterministic deployment failures with errors like: + +```Error: resource [kueue.x-k8s.io/v1beta1/ClusterQueue] isn't valid for cluster``` + +##### **Recommended Workaround: Two-Stage Apply** + +To ensure a reliable deployment, you must manually enforce the correct order of operations. + +1. **Initial Deployment:** In your blueprint, include **only** the manifest(s) containing the `CustomResourceDefinition` (CRD) resources in the `apply_manifests` list. + + *Example `settings` for the first run:* + + ```yaml + settings: + apply_manifests: + # This manifest contains the CRDs for Kueue + - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" + server_side_apply: true + ``` + +2. **Run the deployment** (`gcluster deploy` or `terraform apply`). + +3. **Second Deployment:** Once the first apply is successful, **add** the manifests containing your custom resources (like `ClusterQueue`, `LocalQueue`) to the list. + + *Example `settings` for the second run:* + + ```yaml + settings: + apply_manifests: + # The CRD manifest is still present + - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" + server_side_apply: true + + # Now, add your configuration manifest + - source: "https://gist.githubusercontent.com/YourUser/..." # Your configuration URL + server_side_apply: true + ``` + +4. **Run the deployment command again.** Since the CRDs are now guaranteed to exist in the cluster, this second apply will succeed reliably. + +#### **2. Large Manifests (CRDs)** + +* **Issue:** Applying very large manifests can fail with a `metadata.annotations: Too long` error. +* **Solution:** Enable Server-Side Apply by setting `server_side_apply: true` for the manifest entry. + +#### **3. Conflicts on Re-application** + +* **Issue:** Re-running a deployment after a partial failure can cause server-side apply field manager `conflicts`. +* **Solution:** Forcibly take ownership of the resource fields by setting `force_conflicts: true`. + +#### **4. Terraform Template Files (`.tftpl`)** + +* **Limitation:** This module **cannot** render a template file (`.tftpl`) when sourced from a remote URL. +* **Workaround:** You must render the template into a pure YAML file locally, host that rendered file at a URL, and provide the URL of the rendered file in your blueprint. + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 7.2 | +| [helm](#requirement\_helm) | ~> 2.17 | +| [http](#requirement\_http) | ~> 3.0 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 7.2 | +| [http](#provider\_http) | ~> 3.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [configure\_kueue](#module\_configure\_kueue) | ./kubectl | n/a | +| [install\_gib](#module\_install\_gib) | ./kubectl | n/a | +| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | +| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | +| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | +| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | +| [kubectl\_apply\_manifests](#module\_kubectl\_apply\_manifests) | ./kubectl | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.gib_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.initial_gib_version](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.jobset_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.kueue_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | +| [http_http.manifest_from_url](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [apply\_manifests](#input\_apply\_manifests) | A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md).
NOTE: The `enable` input acts as a FF to apply a manifest or not. By default it is always set to `true`. | 
list(object({
    enable            = optional(bool, true)
    content           = optional(string, null)
    source            = optional(string, null)
    template_vars     = optional(map(any), null)
    server_side_apply = optional(bool, false)
    wait_for_rollout  = optional(bool, true)
  }))
| `[]` | no | +| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | +| [gib](#input\_gib) | Install the NCCL gIB plugin | 
object({
    install = bool
    path    = string
    template_vars = object({
      image   = optional(string, "us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib")
      version = string
      node_affinity = optional(any, {
        requiredDuringSchedulingIgnoredDuringExecution = {
          nodeSelectorTerms = [{
            matchExpressions = [{
              key      = "cloud.google.com/gke-gpu",
              operator = "In",
              values   = ["true"]
            }]
          }]
        }
      })
      accelerator_count = number
      max_unavailable   = optional(string, "50%")
    })
  })
| 
{
  "install": false,
  "path": "",
  "template_vars": {
    "accelerator_count": 0,
    "version": ""
  }
}
| no | +| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | +| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "0.10.1")
  })
| `{}` | no | +| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "0.13.3")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | +| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | +| [target\_architecture](#input\_target\_architecture) | The target architecture for the GKE nodes and gIB plugin (e.g., 'x86\_64' or 'arm64'). | `string` | `"x86_64"` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/README.md b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/README.md new file mode 100644 index 0000000000..1957899617 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/README.md @@ -0,0 +1,64 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [helm](#provider\_helm) | ~> 2.17 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | +| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | +| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | +| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | +| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | +| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | +| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | +| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | +| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | +| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | +| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | +| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | +| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | +| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | +| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | +| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | +| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | +| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | +| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | +| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | +| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | +| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | +| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | +| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | +| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | +| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | +| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | +| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | +| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | +| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf new file mode 100644 index 0000000000..8cc09bd3e2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf @@ -0,0 +1,79 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "helm_release" "apply_chart" { + # Required Identification + name = var.release_name + chart = var.chart_name + + # Chart Source & Version + repository = var.chart_repository + version = var.chart_version + devel = var.devel + + # Target Namespace + namespace = var.namespace + create_namespace = var.create_namespace + + # Values Configuration + values = var.values_yaml + + dynamic "set" { + for_each = var.set_values + content { + name = set.value.name + value = set.value.value + type = set.value.type + } + } + + # Installation/Upgrade Behavior + description = var.description + atomic = var.atomic + cleanup_on_fail = var.cleanup_on_fail + dependency_update = var.dependency_update + disable_crd_hooks = var.disable_crd_hooks + disable_openapi_validation = var.disable_openapi_validation + disable_webhooks = var.disable_webhooks + force_update = var.force_update + lint = var.lint + max_history = var.max_history + recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI + render_subchart_notes = var.render_subchart_notes + reset_values = var.reset_values + reuse_values = var.reuse_values + skip_crds = var.skip_crds + timeout = var.timeout + wait = var.wait + wait_for_jobs = var.wait_for_jobs + + # Verification & Credentials + keyring = var.keyring + pass_credentials = var.pass_credentials + verify = var.verify + + # Post Rendering + dynamic "postrender" { + # Only include the block if var.postrender is not null + for_each = var.postrender == null ? [] : [var.postrender] + content { + binary_path = postrender.value.binary_path + } + } + + # Lifecycle block (optional - generally avoid complex lifecycle in generic modules) + # lifecycle { + # ignore_changes = [] + # } +} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf new file mode 100644 index 0000000000..04e8e214fc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf @@ -0,0 +1,212 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +# --- Required --- +variable "release_name" { + description = "Name of the Helm release." + type = string +} + +variable "chart_name" { + description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." + type = string +} + +# --- Chart Location & Version --- +variable "chart_repository" { + description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." + type = string + default = null +} + +variable "chart_version" { + description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." + type = string + default = null +} + +variable "devel" { + description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." + type = bool + default = false +} + +# --- Namespace --- +variable "namespace" { + description = "Kubernetes namespace to install the Helm release into." + type = string + default = "default" +} + +variable "create_namespace" { + description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." + type = bool + default = true # Common convenience setting +} + +# --- Values Customization --- +variable "values_yaml" { + description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." + type = list(string) + default = [] +} + +variable "set_values" { + description = "List of objects defining values to set ('helm install --set')." + type = list(object({ + name = string # Path to the value (e.g., 'service.type', 'replicaCount') + value = string # The value to set + type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') + })) + default = [] +} + +# --- Installation/Upgrade Behavior --- +variable "description" { + description = "Set an optional description for the Helm release." + type = string + default = null +} + +variable "atomic" { + description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." + type = bool + default = false +} + +variable "wait" { + description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." + type = bool + default = true # Often a good default for dependencies +} + +variable "wait_for_jobs" { + description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." + type = bool + default = false # Helm CLI default is false +} + +variable "timeout" { + description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." + type = number + default = 300 # 5 minutes (Helm CLI default) +} + +variable "cleanup_on_fail" { + description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." + type = bool + default = false +} + +variable "dependency_update" { + description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." + type = bool + default = false +} + +variable "disable_crd_hooks" { + description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." + type = bool + default = false +} + +variable "disable_openapi_validation" { + description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." + type = bool + default = false +} + +variable "disable_webhooks" { + description = "Prevent hooks from running ('helm install --no-hooks')." + type = bool + default = false +} + +variable "force_update" { + description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." + type = bool + default = false +} + +variable "lint" { + description = "Run the helm chart linter during the plan ('helm lint')." + type = bool + default = false +} + +variable "max_history" { + description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." + type = number + default = null # Terraform provider defaults to Helm's default (usually 10) +} + +variable "recreate_pods" { + description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." + type = bool + default = false +} + +variable "render_subchart_notes" { + description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." + type = bool + default = false +} + +variable "reset_values" { + description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." + type = bool + default = false +} + +variable "reuse_values" { + description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." + type = bool + default = false # Helm CLI default is false +} + +variable "skip_crds" { + description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." + type = bool + default = false +} + +# --- Verification & Credentials --- +variable "keyring" { + description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." + type = string + default = null # Defaults to Helm's default keyring location +} + +variable "pass_credentials" { + description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." + type = bool + default = false +} + +variable "verify" { + description = "Verify the package before installing it ('helm install --verify')." + type = bool + default = false +} + +# --- Advanced Rendering --- +variable "postrender" { + description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." + type = object({ + binary_path = string # Path to the post-renderer executable + }) + default = null # Disabled by default +} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf new file mode 100644 index 0000000000..09d912e2c9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml new file mode 100644 index 0000000000..92fc1bca22 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml @@ -0,0 +1,25 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For referencing the original jobset helm chart values, pull the latest jobset chart version +# `helm pull oci://registry.k8s.io/jobset/charts/jobset --version=0.10.1` (latest helm chart version) + +controller: + # It ensures the Jobset pod(s) can be scheduled on GKE clusters where the + # system node pool uses the default "gke-managed-components" taint. + tolerations: + - key: "components.gke.io/gke-managed-components" + operator: "Equal" + value: "true" + effect: "NoSchedule" diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/README.md b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/README.md new file mode 100644 index 0000000000..691f4dc34a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/README.md @@ -0,0 +1,55 @@ + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | + +## Providers + +| Name | Version | +|------|---------| +| [kubectl](#provider\_kubectl) | >= 1.7.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubectl_manifest.apply_doc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_path_documents.templates](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | +| [kubectl_path_documents.yamls](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | +| [force\_conflicts](#input\_force\_conflicts) | The force\_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller) | `bool` | `false` | no | +| [server\_side\_apply](#input\_server\_side\_apply) | Allow using kubectl server-side apply method. | `bool` | `false` | no | +| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `null` | no | +| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | +| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf new file mode 100644 index 0000000000..acf1d3c908 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf @@ -0,0 +1,92 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + yaml_separator = "\n---" + + # This locals block processes manifest inputs from one of four methods, + # evaluated in order of precedence using coalesce. + + # --- METHOD 1: Direct Content Input --- + # Used when manifest content is passed directly as a string. + content_yaml_body = var.content + + # Fallback for safe path checking in subsequent methods. + null_safe_source = coalesce(var.source_path, " ") + + # --- METHOD 2: Single Local YAML File --- + # Used when var.source_path points to a local .yaml file. + yaml_file = length(regexall("\\.yaml(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null + yaml_file_content = local.yaml_file != null ? file(local.yaml_file) : null + + # --- METHOD 3: Single Local Template File --- + # Used when var.source_path points to a local .tftpl file. + template_file = length(regexall("\\.tftpl(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null + template_file_content = local.template_file != null ? templatefile(local.template_file, var.template_vars) : null + + # --- CONSOLIDATE & PROCESS --- + # Coalesce finds the first non-null content from the methods above. + yaml_body = coalesce(local.content_yaml_body, local.yaml_file_content, local.template_file_content, " ") + # Ensure only valid YAML is processed + # It explicitly tests if the content can be decoded before including it. + yaml_body_docs = compact(flatten([ + for doc in split(local.yaml_separator, local.yaml_body) : [ + for content in [trimspace(doc)] : ( + # Use a temporary local variable and can() to test for successful YAML decoding. + # This handles malformed documents (like comment blocks) which cause yamldecode() to fail. + can(yamldecode(content)) && length(yamldecode(content)) > 0 ? content : null + ) + ] + ])) + + # --- METHOD 4: Directory of Files --- + # If no content was found via the methods above AND the source path looks like a directory, + # we assume this is the desired method. The data blocks below will handle it. + directory = length(local.yaml_body_docs) == 0 && endswith(local.null_safe_source, "/") ? abspath(var.source_path) : null + + # --- FINAL AGGREGATION --- + # Combine documents from single-source methods and directory-scan methods into one list. + docs_list = concat(try(local.yaml_body_docs, []), try(data.kubectl_path_documents.yamls[0].documents, []), try(data.kubectl_path_documents.templates[0].documents, [])) + docs_map = tomap({ + for index, doc in local.docs_list : index => doc + }) +} + +data "kubectl_path_documents" "yamls" { + count = local.directory != null ? 1 : 0 + pattern = "${local.directory}/*.yaml" +} + +data "kubectl_path_documents" "templates" { + count = local.directory != null ? 1 : 0 + pattern = "${local.directory}/*.tftpl" + vars = var.template_vars +} + +resource "kubectl_manifest" "apply_doc" { + for_each = local.docs_map + yaml_body = each.value + server_side_apply = var.server_side_apply + wait_for_rollout = var.wait_for_rollout + force_conflicts = var.force_conflicts + + lifecycle { + precondition { + condition = !var.force_conflicts || var.server_side_apply + error_message = "The 'force_conflicts' variable can only be set to true when 'server_side_apply' is also true." + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf new file mode 100644 index 0000000000..7bf34e089c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "content" { + description = "The YAML body to apply to gke cluster." + type = string + default = null +} + +variable "source_path" { + description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." + type = string + default = null +} + +variable "template_vars" { + description = "The values to populate template file(s) with." + type = any + default = null +} + +variable "server_side_apply" { + description = "Allow using kubectl server-side apply method." + type = bool + default = false +} + +variable "wait_for_rollout" { + description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." + type = bool + default = true +} + +variable "force_conflicts" { + description = "The force_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller)" + type = bool + default = false +} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf new file mode 100644 index 0000000000..cce452239f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.7.0" + } + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml new file mode 100644 index 0000000000..7c0bef7013 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml @@ -0,0 +1,30 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For referencing the original Kueue helm chart values, pull the latest helm chart version +# `helm pull oci://registry.k8s.io/kueue/charts/kueue --version=0.13.3` (latest helm chart version) + +controllerManager: + # -- Enables the Topology-Aware Scheduling feature gate. + featureGates: + - name: TopologyAwareScheduling + enabled: true + + # It ensures the Kueue pod can schedule on GKE clusters where the + # system node pool uses the default "gke-managed-components" taint. + tolerations: + - key: "components.gke.io/gke-managed-components" + operator: "Equal" + value: "true" + effect: "NoSchedule" diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/main.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/main.tf new file mode 100644 index 0000000000..73a15ad1ab --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/main.tf @@ -0,0 +1,271 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] + project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] + + # 1. First, Identify manifests that are explicitly enabled. + enabled_manifests = { + for index, manifest in var.apply_manifests : index => manifest + if try(manifest.enable, true) + } + + # 2. Identify URL-based manifests + url_manifests = { + for index, manifest in local.enabled_manifests : index => manifest + if try(manifest.source, null) != null && (startswith(manifest.source, "http://") || startswith(manifest.source, "https://")) + } + + # 3. Rebuild the map by populating the 'content' field for URLs based manifest + processed_apply_manifests_map = tomap({ + for index, manifest in local.enabled_manifests : tostring(index) => { + # If this manifest was a URL, its content is the body from the HTTP call. + content = contains(keys(local.url_manifests), tostring(index)) ? data.http.manifest_from_url[tostring(index)].body : manifest.content + + # If this was a URL, its source path is now null. Otherwise, use original. + source = contains(keys(local.url_manifests), tostring(index)) ? null : manifest.source + + # Pass other vars + template_vars = manifest.template_vars + server_side_apply = manifest.server_side_apply + wait_for_rollout = manifest.wait_for_rollout + } + }) + + install_kueue = try(var.kueue.install, false) + install_jobset = try(var.jobset.install, false) + install_gpu_operator = try(var.gpu_operator.install, false) + install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) + install_gib = try(var.gib.install, false) +} + +data "http" "manifest_from_url" { + for_each = local.url_manifests + url = each.value.source +} + +data "google_container_cluster" "gke_cluster" { + project = local.project_id + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +module "kubectl_apply_manifests" { + for_each = local.processed_apply_manifests_map + source = "./kubectl" + depends_on = [var.gke_cluster_exists] + + content = each.value.content + source_path = each.value.source + template_vars = each.value.template_vars + server_side_apply = each.value.server_side_apply + wait_for_rollout = each.value.wait_for_rollout + + providers = { + kubectl = kubectl + } +} + +module "install_kueue" { + source = "./helm_install" + count = local.install_kueue ? 1 : 0 + wait = false + timeout = 1200 + release_name = "kueue" + chart_repository = "oci://registry.k8s.io/kueue/charts" + chart_name = "kueue" + chart_version = var.kueue.version + namespace = "kueue-system" + create_namespace = true + values_yaml = [ + file("${path.module}/kueue/kueue-helm-values.yaml") + ] + + depends_on = [var.gke_cluster_exists] +} + +module "configure_kueue" { + source = "./kubectl" + source_path = local.install_kueue ? try(var.kueue.config_path, "") : null + template_vars = local.install_kueue ? try(var.kueue.config_template_vars, null) : null + depends_on = [module.install_kueue] + + server_side_apply = true + wait_for_rollout = true + + providers = { + kubectl = kubectl + } +} + +module "install_jobset" { + source = "./helm_install" + count = local.install_jobset ? 1 : 0 + wait = false + timeout = 1200 + release_name = "jobset" + chart_repository = "oci://registry.k8s.io/jobset/charts" + chart_name = "jobset" + chart_version = var.jobset.version + namespace = "jobset-system" + create_namespace = true + values_yaml = [ + file("${path.module}/jobset/jobset-helm-values.yaml") + ] + depends_on = [var.gke_cluster_exists, module.configure_kueue] +} + +module "install_nvidia_dra_driver" { + count = local.install_nvidia_dra_driver ? 1 : 0 + depends_on = [module.kubectl_apply_manifests, var.gke_cluster_exists, module.configure_kueue] + source = "./helm_install" + + release_name = "nvidia-dra-driver-gpu" # The release name + chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts + chart_name = "nvidia-dra-driver-gpu" # The chart name + chart_version = var.nvidia_dra_driver.version # The chart version + namespace = "nvidia-dra-driver-gpu" # The target namespace + create_namespace = true # Equivalent to --create-namespace + + # Use the 'values' argument to pass the YAML content + # This corresponds to the -f <(cat < +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_monitoring_dashboard.dashboard](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_dashboard) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [base\_dashboard](#input\_base\_dashboard) | Baseline dashboard template, select from HPC or Empty | `string` | `"HPC"` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the monitoring dashboard instance. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [title](#input\_title) | Title of the created dashboard | `string` | `"Cluster Toolkit Dashboard"` | no | +| [widgets](#input\_widgets) | List of additional widgets to add to the base dashboard. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for accessing the monitoring dashboard | + diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl new file mode 100644 index 0000000000..f25cbbd2c6 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl @@ -0,0 +1,17 @@ +{ + "displayName": "${title}: ${deployment_name}", + "gridLayout": { + "columns": 2, + "widgets": [ + { + "text": { + "content": "Metrics from the ${deployment_name} deployment of the Cluster Toolkit.", + "format": "MARKDOWN" + }, + "title": "${title}" + }%{ for widget in widgets ~}, + ${widget} + %{endfor ~} + ] + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl new file mode 100644 index 0000000000..5b20435a9a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl @@ -0,0 +1,595 @@ +{ + "displayName": "${title}: ${deployment_name}", + "labels": ${jsonencode(labels)}, + "gridLayout": { + "columns": 2, + "widgets": [ + { + "text": { + "content": "HPC metrics from the ${deployment_name} deployment of the Cluster Toolkit.", + "format": "MARKDOWN" + }, + "title": "${title}" + }, + { + "title": "VM Instance - Memory utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/memory/percent_used\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - CPU Utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", + "pickTimeSeriesFilter": { + "direction": "TOP", + "numTimeSeries": 20, + "rankingMethod": "METHOD_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - CPU utilization (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + }, + "unitOverride": "%" + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk read operations", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/read_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk write operations", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/write_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk Read Bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/disk/read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk Write Bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/disk/write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Throttled read bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Throttled write bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Received packets", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/received_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - Sent packets", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - Received bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/received_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Sent bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_MEAN", + "groupByFields": [ + "metric.label.\"instance_name\"", + "metric.label.\"loadbalanced\"", + "resource.label.\"project_id\"", + "resource.label.\"instance_id\"", + "resource.label.\"zone\"" + ], + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Network Traffic Bytes (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/interface/traffic\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Network Packets (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/interface/packets\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "TCP connections", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/network/tcp_connections\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + }, + "unitOverride": "1" + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - CPU utilization for steal", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "STACKED_BAR", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MAX" + }, + "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\" metric.label.\"cpu_state\"=\"steal\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - CPU utilization [MEAN]", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }%{ for widget in widgets ~}, + ${widget} + %{endfor ~} + ] + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/main.tf b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/main.tf new file mode 100644 index 0000000000..df3c5c36b0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/main.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "dashboard", ghpc_role = "monitoring" }) +} + +locals { + dash_path = "${path.module}/dashboards/${var.base_dashboard}.json.tpl" +} + +resource "google_monitoring_dashboard" "dashboard" { + dashboard_json = templatefile(local.dash_path, { + widgets = var.widgets + deployment_name = var.deployment_name + title = var.title + labels = local.labels + } + ) + project = var.project_id +} diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/metadata.yaml new file mode 100644 index 0000000000..de1a10f57d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - stackdriver.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/outputs.tf b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/outputs.tf new file mode 100644 index 0000000000..b7ff35fb0e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/outputs.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instructions" { + description = "Instructions for accessing the monitoring dashboard" + value = <<-EOT + A monitoring dashboard has been created. To view, navigate to the following URL: + https://console.cloud.google.com/monitoring/dashboards/builder${regex("/[0-9a-z-]*$", google_monitoring_dashboard.dashboard.id)} + EOT +} diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/variables.tf b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/variables.tf new file mode 100644 index 0000000000..8194f8b73a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/variables.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "base_dashboard" { + description = "Baseline dashboard template, select from HPC or Empty" + type = string + default = "HPC" + validation { + condition = contains(["HPC", "Empty"], var.base_dashboard) + error_message = "Must set var.base_dashboard to either \"HPC\" or \"Empty\"." + } +} + +variable "title" { + description = "Title of the created dashboard" + type = string + default = "Cluster Toolkit Dashboard" +} + +variable "widgets" { + description = "List of additional widgets to add to the base dashboard." + type = list(string) + default = [] +} + +variable "labels" { + description = "Labels to add to the monitoring dashboard instance. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/versions.tf b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/versions.tf new file mode 100644 index 0000000000..2717fe79f6 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/README.md b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/README.md new file mode 100644 index 0000000000..057f4b649d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/README.md @@ -0,0 +1,111 @@ +## Description + +This module facilitates the creation of custom firewall rules for existing +networks. + +## Example usage + +This module can be used by other Toolkit modules to create application-specific +firewall rules or in conjunction with the [pre-existing-vpc] module to enable +traffic in existing networks. The snippet below is drawn from the +[ml-slurm.yaml] example: + +```yaml +- group: primary + modules: + - id: network + source: modules/network/pre-existing-vpc + + # this example anticipates that the VPC default network has internal traffic + # allowed and IAP tunneling for SSH connections + - id: firewall_rule + source: modules/network/firewall-rules + use: + - network + settings: + ingress_rules: + - name: $(vars.deployment_name)-allow-internal-traffic + description: Allow internal traffic + destination_ranges: + - $(network.subnetwork_address) + source_ranges: + - $(network.subnetwork_address) + allow: + - protocol: tcp + ports: + - 0-65535 + - protocol: udp + ports: + - 0-65535 + - protocol: icmp + - name: $(vars.deployment_name)-allow-iap-ssh + description: Allow IAP-tunneled SSH connections + destination_ranges: + - $(network.subnetwork_address) + source_ranges: + - 35.235.240.0/20 + allow: + - protocol: tcp + ports: + - 22 +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [firewall\_rule](#module\_firewall\_rule) | terraform-google-modules/network/google//modules/firewall-rules | ~> 12.0 | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.pga_check](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [google_compute_subnetwork.subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [egress\_rules](#input\_egress\_rules) | List of egress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | +| [ingress\_rules](#input\_ingress\_rules) | List of ingress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | +| [network\_name](#input\_network\_name) | The name of the network to create firewall rules in | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The project ID to host the network in | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork whose global network firewall rules will be modified. | `string` | n/a | yes | + +## Outputs + +No outputs. + + +[pre-existing-vpc]: ../pre-existing-vpc/README.md +[ml-slurm.yaml]: ../../../examples/ml-slurm.yaml diff --git a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/main.tf b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/main.tf new file mode 100644 index 0000000000..05241278ad --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/main.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + use_subnetwork_data = (var.project_id == null || var.network_name == null) && var.subnetwork_self_link != null +} + +# the google_compute_network data source does not allow identification by +# self_link, which uniquely identifies subnet, project, and network +data "google_compute_subnetwork" "subnetwork" { + # Only instantiate this data source if needed + count = local.use_subnetwork_data ? 1 : 0 + self_link = var.subnetwork_self_link +} + +locals { + # Derived values from data source, null if data source is not used + derived_project_id = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].project : null + derived_network_name = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].network : null + + # Effective values: Use var if provided, otherwise use derived value + effective_project_id = coalesce(var.project_id, local.derived_project_id) + effective_network_name = coalesce(var.network_name, local.derived_network_name) +} + +# Module-level check for Private Google Access on the subnetwork. +# This check is only relevant if subnetwork_self_link was provided and used. +resource "terraform_data" "pga_check" { + count = local.use_subnetwork_data ? 1 : 0 + + lifecycle { + precondition { + condition = data.google_compute_subnetwork.subnetwork[0].private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.subnetwork[0].name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } + } +} + +module "firewall_rule" { + source = "terraform-google-modules/network/google//modules/firewall-rules" + version = "~> 12.0" + project_id = local.effective_project_id + network_name = local.effective_network_name + + ingress_rules = var.ingress_rules + egress_rules = var.egress_rules +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/variables.tf b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/variables.tf new file mode 100644 index 0000000000..05e9be4425 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/variables.tf @@ -0,0 +1,88 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork whose global network firewall rules will be modified." + type = string +} + +variable "project_id" { + description = "The project ID to host the network in" + type = string + default = null +} + +variable "network_name" { + description = "The name of the network to create firewall rules in" + type = string + default = null +} + +variable "ingress_rules" { + description = "List of ingress rules" + default = [] + type = list(object({ + name = string + description = optional(string, null) + disabled = optional(bool, null) + priority = optional(number, null) + destination_ranges = optional(list(string), []) + source_ranges = optional(list(string), []) + source_tags = optional(list(string)) + source_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + target_service_accounts = optional(list(string)) + + allow = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + deny = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + log_config = optional(object({ + metadata = string + })) + })) +} + +variable "egress_rules" { + description = "List of egress rules" + default = [] + type = list(object({ + name = string + description = optional(string, null) + disabled = optional(bool, null) + priority = optional(number, null) + destination_ranges = optional(list(string), []) + source_ranges = optional(list(string), []) + source_tags = optional(list(string)) + source_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + target_service_accounts = optional(list(string)) + + allow = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + deny = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + log_config = optional(object({ + metadata = string + })) + })) +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/versions.tf new file mode 100644 index 0000000000..9061dd3ae5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.74.0" + } + + required_version = ">= 1.5" +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/README.md b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/README.md new file mode 100644 index 0000000000..abbfe3b97b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/README.md @@ -0,0 +1,143 @@ +## Description + +This module accomplishes the following: + +* Creates one [VPC network][cft-network] + * Each VPC contains a variable number of subnetworks as specified in the + `subnetworks_template` variable + * Each subnetwork contains distinct IP address ranges +* Outputs the following unique parameters + * `subnetwork_interfaces` which is compatible with Slurm and vm-instance + modules + * `subnetwork_interfaces_gke` which is compatible with GKE modules + +This module is a simplified version of the VPC module and its main difference +is the variable `subnetwork_template` which is the template for all subnetworks +created within the network. This template contains the following values: + +1. `count`: The number of subnetworks to be created +1. `name_prefix`: The prefix for the subnetwork names +1. `ip_range`: [CIDR-formatted IP range][cidr] +1. `region`: The region where the subnetwork will be deployed + +> [!WARNING] +> The `ip_range` should be always be large enough to split into `count` +> subnetworks and the number of required connections within. + +[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v10.0.0 +[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation + +### Example + +This snippet uses the gpu-vpc module to create a new VPC network named +`test-rdma-net` with 8 subnetworks named `test-mrdma-sub-#` where # ranges from +0 to 7. The subnetworks will split the `ip_range` evenly, starting from bit 16 +(0 indexed). The networks are ingested by the Slurm nodeset within the +`additional_networks` setting. + +```yaml + - id: rdma-net + source: modules/network/gpu-rdma-vpc + settings: + network_name: test-rdma-net + network_profile: https://www.googleapis.com/compute/beta/projects/$(vars.project_id)/global/networkProfiles/$(vars.zone)-vpc-roce + network_routing_mode: REGIONAL + subnetworks_template: + name_prefix: test-mrdma-sub + count: 8 + ip_range: 192.168.0.0/16 + region: $(vars.region) + + - id: a3_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network0] + settings: + machine_type: a3-ultragpu-8g + additional_networks: + $(concat( + [{ + network=null, + subnetwork=network1.subnetwork_self_link, + subnetwork_project=vars.project_id, + nic_type="GVNIC", + queue_count=null, + network_ip="", + stack_type=null, + access_config=[], + ipv6_access_config=[], + alias_ip_range=[] + }], + rdma-net.subnetwork_interfaces + )) + ... +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | DEPRECATED: enable\_internal\_traffic can not be specified for gpu-rdma-vpc. | `bool` | `null` | no | +| [firewall\_log\_config](#input\_firewall\_log\_config) | DEPRECATED: firewall\_log\_config can not be specified for gpu-rdma-vpc. | `string` | `null` | no | +| [firewall\_rules](#input\_firewall\_rules) | DEPRECATED: firewall\_rules can not be specified for gpu-rdma-vpc. | `any` | `null` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}} | `string` | n/a | yes | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "REGIONAL") | `string` | `"REGIONAL"` | no | +| [nic\_type](#input\_nic\_type) | NIC type for use in modules that use the output | `string` | `"MRDMA"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | +| [subnetworks\_template](#input\_subnetworks\_template) | Specifications for the subnetworks that will be created within this VPC.

count (number, required, number of subnets to create, default is 8)
name\_prefix (string, required, subnet name prefix, default is deployment name)
ip\_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16)
region (string, optional, region to deploy subnets to, defaults to vars.region) | 
object({
    count       = number
    name_prefix = string
    ip_range    = string
    region      = optional(string)
  })
| 
{
  "count": 8,
  "ip_range": "192.168.0.0/16",
  "name_prefix": null,
  "region": null
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [network\_id](#output\_network\_id) | ID of the new VPC network | +| [network\_name](#output\_network\_name) | Name of the new VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | +| [subnetwork\_interfaces](#output\_subnetwork\_interfaces) | Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules) | +| [subnetwork\_interfaces\_gke](#output\_subnetwork\_interfaces\_gke) | Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool) | +| [subnetwork\_name\_prefix](#output\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | +| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | + diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/main.tf b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/main.tf new file mode 100644 index 0000000000..e37db01976 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/main.tf @@ -0,0 +1,79 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + autoname = replace(var.deployment_name, "_", "-") + network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name + subnet_prefix = var.subnetworks_template.name_prefix == null ? "${local.autoname}-subnet" : var.subnetworks_template.name_prefix + + new_bits = ceil(log(var.subnetworks_template.count, 2)) + template_subnetworks = [for i in range(var.subnetworks_template.count) : + { + subnet_name = "${local.subnet_prefix}-${i}" + subnet_region = try(var.subnetworks_template.region, var.region) + subnet_ip = cidrsubnet(var.subnetworks_template.ip_range, local.new_bits, i) + } + ] + + firewall_rules = [] + + output_subnets = [ + for subnet in module.vpc.subnets : { + network = null + subnetwork = subnet.self_link + subnetwork_project = null # will populate from subnetwork_self_link + network_ip = null + nic_type = var.nic_type + stack_type = null + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } + ] + + output_subnets_gke = [ + for i in range(length(module.vpc.subnets)) : { + network = local.network_name + subnetwork = local.template_subnetworks[i].subnet_name + subnetwork_project = var.project_id + network_ip = null + nic_type = var.nic_type + stack_type = null + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } + ] +} + +module "vpc" { + source = "terraform-google-modules/network/google" + version = "~> 12.0" + + network_name = local.network_name + project_id = var.project_id + auto_create_subnetworks = false + subnets = local.template_subnetworks + routing_mode = var.network_routing_mode + mtu = var.mtu + description = var.network_description + shared_vpc_host = var.shared_vpc_host + delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes + firewall_rules = local.firewall_rules + network_profile = var.network_profile +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf new file mode 100644 index 0000000000..0a21f1d3f2 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the new VPC network" + value = module.vpc.network_name + depends_on = [module.vpc] +} + +output "network_id" { + description = "ID of the new VPC network" + value = module.vpc.network_id + depends_on = [module.vpc] +} + +output "network_self_link" { + description = "Self link of the new VPC network" + value = module.vpc.network_self_link + depends_on = [module.vpc] +} + +output "subnetworks" { + description = "Full list of subnetwork objects belonging to the new VPC network" + value = module.vpc.subnets + depends_on = [module.vpc] +} + +output "subnetwork_interfaces" { + description = "Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules)" + value = local.output_subnets + depends_on = [module.vpc] +} + +# The output subnetwork_interfaces is compatible with vm-instance module but not with gke-node-pool +# See https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/99493df21cecf6a092c45298bf7a45e0343cf622/modules/compute/vm-instance/variables.tf#L220 +# So, we need a separate output that makes the network and subnetwork names available +output "subnetwork_interfaces_gke" { + description = "Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool)" + value = local.output_subnets_gke + depends_on = [module.vpc] +} + +output "subnetwork_name_prefix" { + description = "Prefix of the RDMA subnetwork names" + value = var.subnetworks_template.name_prefix +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf new file mode 100644 index 0000000000..a30fb50e7d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf @@ -0,0 +1,164 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "network_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" + type = string + default = null +} + +variable "region" { + description = "The default region for Cloud resources" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "subnetworks_template" { + description = <<-EOT + Specifications for the subnetworks that will be created within this VPC. + + count (number, required, number of subnets to create, default is 8) + name_prefix (string, required, subnet name prefix, default is deployment name) + ip_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16) + region (string, optional, region to deploy subnets to, defaults to vars.region) + EOT + nullable = false + type = object({ + count = number + name_prefix = string + ip_range = string + region = optional(string) + }) + default = { + count = 8 + name_prefix = null + ip_range = "192.168.0.0/16" + region = null + } + + validation { + condition = var.subnetworks_template.count > 0 + error_message = "Number of subnetworks must be greater than 0" + } + + validation { + condition = can(cidrhost(var.subnetworks_template.ip_range, 0)) + error_message = "IP address range must be in CIDR format." + } +} + +variable "network_routing_mode" { + type = string + default = "REGIONAL" + description = "The network routing mode (default \"REGIONAL\")" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "shared_vpc_host" { + type = bool + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + default = false +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_internal_traffic" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." + type = bool + default = null + validation { + condition = var.enable_internal_traffic == null + error_message = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." + } +} + +variable "firewall_rules" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." + type = any + default = null + validation { + condition = var.firewall_rules == null + error_message = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." + } +} + +variable "firewall_log_config" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." + type = string + default = null + validation { + condition = var.firewall_log_config == null + error_message = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." + } +} + +variable "network_profile" { + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + EOT + type = string + nullable = false + + validation { + condition = can(coalesce(var.network_profile)) + error_message = "var.network_profile must be specified and not an empty string" + } +} + +variable "nic_type" { + description = "NIC type for use in modules that use the output" + type = string + nullable = true + default = "MRDMA" + + validation { + condition = contains(["MRDMA"], var.nic_type) + error_message = "The nic_type must be \"MRDMA\"." + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf new file mode 100644 index 0000000000..71b7106734 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.15.0" +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/multivpc/README.md b/deletion-test/cluster/modules/embedded/modules/network/multivpc/README.md new file mode 100644 index 0000000000..973e6b32c9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/multivpc/README.md @@ -0,0 +1,136 @@ +## Description + +This module accomplishes the following: + +* Creates 2 to 8 [VPC networks][vpc] + * Each VPC contains exactly 1 subnetwork + * Each subnetwork contains distinct IP address ranges +* Outputs the `additional_networks` parameter, which is compatible with Slurm + modules + +There are 4 variables that differentiate this module from the standard VPC +module. + +1. `network_prefix`: The name prefix of the VPCs to be created. All + networks and subnetworks will start with this and end with a unique number. +1. `network_count`: The number of VPCs to be created. +1. `global_ip_address_range`: [CIDR-formatted IP range][cidr] +1. `network_cidr_suffix`: The CIDR suffix that defines the address + space that the individual VPCs will cover. + +> [!WARNING] +> The `network_cidr_suffix` should be always be larger than the CIDR suffix on +> `global_ip_address_range`. The difference between these two suffixes should +> be large enough to accommodate the number of VPCs that are being deployed +> (e.g. CIDR suffix bit difference <= `ceil(log2(network_count)))`). + +> [!NOTE] +> For deployments that need multiple VPCs that do not meet this use-case, users +> should deploy multiple individual VPC modules. + +[vpc]: ../vpc/README.md +[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation + +### Example + +This snippet uses the multivpc module to create 8 new VPC networks named +`multivpc-net-#` where # ranges from 0 to 7. Additionally, it creates 1 +subnetwork in each VPC. + +```yaml + - id: network + source: modules/network/vpc + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: a3_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network, multinetwork] + settings: + machine_type: a3-highgpu-8g + ... +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [vpcs](#module\_vpcs) | ../vpc | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.global_ip_cidr_suffix](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | +| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | +| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | +| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | +| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | +| [global\_ip\_address\_range](#input\_global\_ip\_address\_range) | IP address range (CIDR) that will span entire set of VPC networks | `string` | `"172.16.0.0/12"` | no | +| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT) | `number` | `2` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_count](#input\_network\_count) | The number of vpc nettworks to create | `number` | `4` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_interface\_defaults](#input\_network\_interface\_defaults) | The template of the network settings to be used on all vpcs. | 
object({
    network            = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string, "GVNIC")
    stack_type         = optional(string, "IPV4_ONLY")
    queue_count        = optional(string)
    access_config = optional(list(object({
      nat_ip                 = string
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  })
| 
{
  "access_config": [],
  "alias_ip_range": [],
  "ipv6_access_config": [],
  "network": null,
  "network_ip": "",
  "nic_type": "GVNIC",
  "queue_count": null,
  "stack_type": "IPV4_ONLY",
  "subnetwork": null,
  "subnetwork_project": null
}
| no | +| [network\_name\_prefix](#input\_network\_name\_prefix) | The base name of the vpcs and their subnets, will be appended with a sequence number | `string` | `""` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network dynamic routing mode | `string` | `"REGIONAL"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [subnetwork\_cidr\_suffix](#input\_subnetwork\_cidr\_suffix) | The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network. | `number` | `16` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [additional\_networks](#output\_additional\_networks) | Network interfaces for each subnetwork created by this module | +| [network\_ids](#output\_network\_ids) | IDs of the new VPC network | +| [network\_names](#output\_network\_names) | Names of the new VPC networks | +| [network\_self\_links](#output\_network\_self\_links) | Self link of the new VPC network | +| [subnetwork\_addresses](#output\_subnetwork\_addresses) | IP address range of the primary subnetwork | +| [subnetwork\_names](#output\_subnetwork\_names) | Names of the subnetwork created in each network | +| [subnetwork\_self\_links](#output\_subnetwork\_self\_links) | Self link of the primary subnetwork | + diff --git a/deletion-test/cluster/modules/embedded/modules/network/multivpc/main.tf b/deletion-test/cluster/modules/embedded/modules/network/multivpc/main.tf new file mode 100644 index 0000000000..ad06e793c1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/multivpc/main.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # this input variable is validated to be in CIDR format + network_name = coalesce(replace(var.network_name_prefix, "_", "-"), replace(var.deployment_name, "_", "-")) + global_ip_cidr_prefix = split("/", var.global_ip_address_range)[0] + global_ip_cidr_suffix = split("/", var.global_ip_address_range)[1] + global_ip_cidr_valid = "${local.global_ip_cidr_prefix}/${terraform_data.global_ip_cidr_suffix.output}" + subnetwork_new_bits = var.subnetwork_cidr_suffix - local.global_ip_cidr_suffix + maximum_subnetworks = pow(2, local.subnetwork_new_bits) + additional_networks = [ + for vpc in module.vpcs : + merge(var.network_interface_defaults, { + network = vpc.network_name + subnetwork = vpc.subnetwork_name + subnetwork_project = var.project_id + }) + ] +} + +resource "terraform_data" "global_ip_cidr_suffix" { + input = local.global_ip_cidr_suffix + lifecycle { + precondition { + condition = local.maximum_subnetworks >= var.network_count + error_message = < 1 + error_message = "The minimum VPCs able to be created by this module is 2. Use the standard Toolkit module at modules/network/vpc for count = 1" + } + validation { + condition = var.network_count <= 8 + error_message = "The maximum VPCs able to be created by this module is 8" + } +} + +variable "global_ip_address_range" { + description = "IP address range (CIDR) that will span entire set of VPC networks" + type = string + default = "172.16.0.0/12" + + validation { + condition = can(cidrhost(var.global_ip_address_range, 0)) + error_message = "var.global_ip_address_range must be an IPv4 CIDR range (e.g. \"172.16.0.0/12\")." + } +} + +variable "subnetwork_cidr_suffix" { + description = "The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network." + type = number + default = 16 +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "network_routing_mode" { + type = string + default = "REGIONAL" + description = "The network dynamic routing mode" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "ips_per_nat" { + type = number + description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT)" + default = 2 +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_iap_ssh_ingress" { + type = bool + description = "Enable a firewall rule to allow SSH access using IAP tunnels" + default = true +} + +variable "enable_iap_rdp_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" + default = false +} + +variable "enable_iap_winrm_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" + default = false +} + +variable "enable_internal_traffic" { + type = bool + description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" + default = true +} + +variable "extra_iap_ports" { + type = list(string) + description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" + default = [] +} + +variable "allowed_ssh_ip_ranges" { + type = list(string) + description = "A list of CIDR IP ranges from which to allow ssh access" + default = [] + + validation { + condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) + error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." + } +} + +variable "firewall_rules" { + type = any + description = "List of firewall rules" + default = [] +} + +variable "network_interface_defaults" { + type = object({ + network = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + network_ip = optional(string, "") + nic_type = optional(string, "GVNIC") + stack_type = optional(string, "IPV4_ONLY") + queue_count = optional(string) + access_config = optional(list(object({ + nat_ip = string + network_tier = string + public_ptr_domain_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + public_ptr_domain_name = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + }) + description = "The template of the network settings to be used on all vpcs." + default = { + network = null + subnetwork = null + subnetwork_project = null + network_ip = "" + nic_type = "GVNIC" + stack_type = "IPV4_ONLY" + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } +} + +variable "network_profile" { + type = string + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown + EOT + default = null +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/multivpc/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/multivpc/versions.tf new file mode 100644 index 0000000000..e75a67f7b6 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/multivpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.4.0" +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/README.md b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/README.md new file mode 100644 index 0000000000..4d63b17091 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/README.md @@ -0,0 +1,94 @@ +## Description + +This module discovers a subnetwork that already exists in Google Cloud and +outputs subnetwork attributes that uniquely identify it for use by other modules. + +For example, the blueprint below discovers the referred to subnetwork. +With the `use` keyword, the [vm-instance] module accepts the `subnetwork_self_link` +input variables that uniquely identify the subnetwork in which the VM will be created. + +[vpc]: ../vpc/README.md +[vm-instance]: ../../compute/vm-instance/README.md + +> **_NOTE:_** Additional IAM work is needed for this to work correctly. + +### Example + +```yaml +- id: network + source: modules/network/pre-existing-subnetwork + settings: + subnetwork_self_link: https://www.googleapis.com/compute/v1/projects/name-of-host-project/regions/REGION/subnetworks/SUBNETNAME + +- id: example_vm + source: modules/compute/vm-instance + use: + - network + settings: + name_prefix: example + machine_type: c2-standard-4 +``` + +As described in documentation: +[https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork] + +If subnetwork_self_link is provided then name,region,project is ignored. + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [project](#input\_project) | Name of the project that owns the subnetwork | `string` | `null` | no | +| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | `null` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self-link of the subnet in the VPC | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | +| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | + diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/main.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/main.tf new file mode 100644 index 0000000000..9fb206f969 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/main.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +data "google_compute_subnetwork" "primary_subnetwork" { + name = var.subnetwork_name + region = var.region + project = var.project + self_link = var.subnetwork_self_link + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The subnetwork: ${coalesce(var.subnetwork_name, var.subnetwork_self_link)} could not be found." + } + } +} + +# Module-level check for Private Google Access on the subnetwork +check "private_google_access_enabled_subnetwork" { + assert { + condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml new file mode 100644 index 0000000000..6a6f1e5757 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com +ghpc: + has_to_be_used: true diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf new file mode 100644 index 0000000000..868708dc6b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "subnetwork" { + description = "Full subnetwork object in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork +} + +output "subnetwork_name" { + description = "Name of the subnetwork in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.name +} + +output "subnetwork_self_link" { + description = "Subnetwork self-link in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.self_link +} + +output "subnetwork_address" { + description = "Subnetwork IP range in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf new file mode 100644 index 0000000000..d5191843e8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "subnetwork_self_link" { + description = "Self-link of the subnet in the VPC" + type = string + default = null +} + +variable "project" { + description = "Name of the project that owns the subnetwork" + type = string + default = null +} + +variable "subnetwork_name" { + description = "Name of the pre-existing VPC subnetwork" + type = string + default = null +} + +variable "region" { + description = "Region in which to search for primary subnetwork" + type = string + default = null +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf new file mode 100644 index 0000000000..917d948433 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.74.0" + } + + required_version = ">= 1.5" +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/README.md b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/README.md new file mode 100644 index 0000000000..38a1840c2d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/README.md @@ -0,0 +1,110 @@ +## Description + +This module discovers a VPC network that already exists in Google Cloud and +outputs network attributes that uniquely identify it for use by other modules. +The module outputs are aligned with the [vpc module][vpc] so that it can be used +as a drop-in substitute when a VPC already exists. + +For example, the blueprint below discovers the "default" global network and the +"default" regional subnetwork in us-central1. With the `use` keyword, the +[vm-instance] module accepts the `network_self_link` and `subnetwork_self_link` +input variables that uniquely identify the network and subnetwork in which the +VM will be created. + +[vpc]: ../vpc/README.md +[vm-instance]: ../../compute/vm-instance/README.md + +### Example + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + settings: + project_id: $(vars.project_id) + region: us-central1 + +- id: example_vm + source: modules/compute/vm-instance + use: + - network1 + settings: + name_prefix: example + machine_type: c2-standard-4 +``` + +> **_NOTE:_** The `project_id` and `region` settings would be inferred from the +> deployment variables of the same name, but they are included here for clarity. + +### Use shared-vpc + +If a network is created in different project, this module can be used to +reference the network. To use a network from a different project first make sure +you have a [cloud nat][cloudnat] and [IAP][iap] forwarding. For more details, +refer [shared-vpc][shared-vpc-doc] + +[cloudnat]: https://cloud.google.com/nat/docs/overview +[iap]: https://cloud.google.com/iap/docs/using-tcp-forwarding +[shared-vpc-doc]: ../../../examples/README.md#hpc-slurm-sharedvpcyaml-community-badge-experimental-badge + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_network.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network) | data source | +| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [network\_name](#input\_network\_name) | Name of the existing VPC network | `string` | `"default"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | n/a | yes | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork; defaults to var.network\_name if set to null. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [network\_id](#output\_network\_id) | ID of the existing VPC network | +| [network\_name](#output\_network\_name) | Name of the existing VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the existing VPC network | +| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | +| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | + diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/main.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/main.tf new file mode 100644 index 0000000000..ed332bab72 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/main.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +data "google_compute_network" "vpc" { + name = var.network_name + project = var.project_id + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The network: ${var.network_name} could not be found in project: ${var.project_id}." + } + } +} + +locals { + subnetwork_name = var.subnetwork_name != null ? var.subnetwork_name : var.network_name +} + +data "google_compute_subnetwork" "primary_subnetwork" { + name = local.subnetwork_name + region = var.region + project = var.project_id + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The subnetwork: ${local.subnetwork_name} could not be found in project: ${var.project_id} and region: ${var.region}." + } + } +} + +# Module-level check for Private Google Access on the subnetwork +check "private_google_access_enabled_subnetwork" { + assert { + condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/outputs.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/outputs.tf new file mode 100644 index 0000000000..00861af5ca --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the existing VPC network" + value = data.google_compute_network.vpc.name +} + +output "network_id" { + description = "ID of the existing VPC network" + value = data.google_compute_network.vpc.id +} + +output "network_self_link" { + description = "Self link of the existing VPC network" + value = data.google_compute_network.vpc.self_link +} + +output "subnetwork" { + description = "Full subnetwork object in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork +} + +output "subnetwork_name" { + description = "Name of the subnetwork in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.name +} + +output "subnetwork_self_link" { + description = "Subnetwork self-link in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.self_link +} + +output "subnetwork_address" { + description = "Subnetwork IP range in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/variables.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/variables.tf new file mode 100644 index 0000000000..291a81604a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "network_name" { + description = "Name of the existing VPC network" + type = string + default = "default" +} + +variable "subnetwork_name" { + description = "Name of the pre-existing VPC subnetwork; defaults to var.network_name if set to null." + type = string + default = null +} + +variable "region" { + description = "Region in which to search for primary subnetwork" + type = string +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/versions.tf new file mode 100644 index 0000000000..81fe5aeff3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.74.0" + } + + required_version = ">= 1.5" +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/README.md b/deletion-test/cluster/modules/embedded/modules/network/vpc/README.md new file mode 100644 index 0000000000..2c2b1aa1a3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/vpc/README.md @@ -0,0 +1,237 @@ +## Description + +This module creates a new [VPC network][vpc] with 1 or more subnetworks and +a [Cloud Router][router] for every region with a subnetwork. By default, it will +create: + +* A [Cloud NAT][nat] to enable outbound access to the public internet for VMs + without public IP addresses; VMs with public IP addresses bypass the NAT to + directly access the public internet +* A firewall rule that enables inbound SSH access from [Identity-Aware + Proxy][iap] +* A firewall rule that enables all traffic internal to the network + +This behavior is optional and can be configured as [described below](#inputs). +This module is based on networking support in the [Cloud Foundation +Toolkit][cft]. We recommend following the [documentation for the network +module][cft-network] and [submodules][cft-network-submodules] for more details. +In particular, the detailed structure of input variables can be found for: + +* [var.firewall\_rules](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/firewall-rules#inputs) +* [var.secondary\_ranges](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets#inputs) + +[vpc]: https://cloud.google.com/vpc +[router]: https://github.com/terraform-google-modules/terraform-google-cloud-router +[nat]: https://github.com/terraform-google-modules/terraform-google-cloud-nat +[iap]: https://cloud.google.com/iap +[cft]: https://cloud.google.com/foundation-toolkit +[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0 +[cft-network-submodules]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules + +Additionally, [Google Private Access][gpa] is enabled by default on all +subnetworks unless it is explicitly disabled. This setting ensures that all VMs +can use Google services such as [Cloud Storage][gcs] even if they do not have +public IP addresses or Cloud NAT is disabled. + +[gpa]: https://cloud.google.com/vpc/docs/private-google-access +[gcs]: https://cloud.google.com/storage + +### Example + +This creates a new VPC network named `cluster-net`. + +```yaml + - id: network1 + source: modules/network/vpc + settings: + network_name: cluster-net +``` + +### Deprecation warning + +The variables listed below have been deprecated and will be removed in a future +release. Until they are removed,You may continue to use them in Toolkit +blueprints with the same functionality as documented in the [Toolkit 1.0 +release][vpc1.0]. + +* Deprecated variables + * `var.primary_subnetwork` + * `var.additional_subnetworks` + * `var.subnetwork_size` + +[vpc1.0]: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/v1.0.0/modules/network/vpc/README.md + +The following variables have been added to support explicit IP ranges for +subnetworks while retaining existing functionality. We advise adopting them even +if not using explicit IP ranges . The Toolkit ***does not support*** mixing +deprecated variables with the new replacements. The new functionality is +described in [more detail below](#subnetworks). + +* New variables to adopt + * `var.subnetworks` + * A value for this can be generated by merging `var.primary_subnetwork` and + `var.additional_subnetworks` into a single list + * `var.default_primary_subnetwork_size` + * This variable has been renamed for clarity; its value can be directly + copied from an explicit setting for `var.subnetwork_size`; if your blueprint + does not have an explicit setting, the default values are the same + +### Subnetworks + +This module will always provision at least 1 "primary" subnetwork in which most +resources are expected to be provisioned. This primary subnetwork is determined +by + +1. The first element of [var.subnetworks](#input_subnetworks) if it is not the + empty list +2. A default subnetwork automatically calculated from + * [var.subnetwork_name](#input_subnetwork_name) + * [var.region](#input_region) + * [var.network_address_range](#input_network_address_range) + * [var.default_primary_subnetwork_size](#input_default_primary_subnetwork_size) + +If `var.subnetworks` is provided then the primary subnetwork name is taken +explicitly from it and `var.subnetwork_name` is ignored. + +`var.subnetworks` behaves identically to the [Cloud Foundation Toolkit subnets +module][cftsubnets] with the lone exception that one can provide ***one*** of +the following settings for each subnetwork: + +* `new_bits` +* `subnet_ip` + +If each subnetwork defines `subnet_ip` then these are taken to be their explicit +CIDR IP ranges. If each subnetwork defines `new_bits`, then these are taken to +be the size of the CIDR subnetwork (in bits). IP ranges for each subnetwork are +calculated using `var.network_address_range` as the base IP, producing the most +compact set of subnetworks possible. + +> **_NOTE:_** we do not presently support the modification of individual subnetworks +> when using this module to provision more than 1 subnetwork using automatically +> calculated IP ranges based upon `new_bits`. Doing so will cause IP ranges to be +> recalculated for each subnetwork. We advise appending new subnetworks to the end +> of `var.subnetworks`. + +[cftsubnets]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets + +### SSH Access + +By default a firewall rule is created to allow inbound SSH access from +[Identity-Aware Proxy][iap]. A user must have the `IAP-Secured Tunnel User` +(`roles/iap.tunnelResourceAccessor`) IAM role to be able to SSH over IAP. + +To allow regular SSH access from a known IP address you can add the following +`firewall_rules` setting to the `vpc` module: + +```yaml + - id: network1 + source: modules/network/vpc + settings: + firewall_rules: + - name: ssh-my-machine + direction: INGRESS + ranges: [/32] + allow: + - protocol: tcp + ports: [22] +``` + +> **Note**: You must populate the above example with the source IP address from +> which you plan to SSH from. You can use a service like +> [whatismyip.com](https://whatismyip.com) to determine your IP address. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +| Name | Version | +|------|---------| +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 7.3 | +| [nat\_ip\_addresses](#module\_nat\_ip\_addresses) | terraform-google-modules/address/google | ~> 4.1 | +| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.cloud_nat_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.network_profile_firewall_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.secondary_ranges_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_subnetworks](#input\_additional\_subnetworks) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `list(map(string))` | `null` | no | +| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | +| [default\_primary\_subnetwork\_size](#input\_default\_primary\_subnetwork\_size) | The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks | `number` | `15` | no | +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_cloud\_nat](#input\_enable\_cloud\_nat) | Enable the creation of Cloud NATs. | `bool` | `true` | no | +| [enable\_cloud\_router](#input\_enable\_cloud\_router) | Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview | `bool` | `true` | no | +| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | +| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | +| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | +| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | +| [firewall\_log\_config](#input\_firewall\_log\_config) | Firewall log configuration for Toolkit firewall rules (var.enable\_iap\_ssh\_ingress and others) | `string` | `"DISABLE_LOGGING"` | no | +| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | +| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples | `number` | `2` | no | +| [labels](#input\_labels) | Labels to add to network resources that support labels. Key-value pairs of strings. | `map(string)` | `{}` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_address\_range](#input\_network\_address\_range) | IP address range (CIDR) for global network | `string` | `"10.0.0.0/9"` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "GLOBAL") | `string` | `"GLOBAL"` | no | +| [primary\_subnetwork](#input\_primary\_subnetwork) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `map(string)` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [secondary\_ranges](#input\_secondary\_ranges) | "Secondary ranges associated with the subnets.
This will be deprecated in favour of secondary\_ranges\_list at a later date.
Please migrate to using the same." | `map(list(object({ range_name = string, ip_cidr_range = string })))` | `{}` | no | +| [secondary\_ranges\_list](#input\_secondary\_ranges\_list) | "List of secondary ranges associated with the subnetworks.
Each subnetwork must be specified at most once in this list." | 
list(object({
    subnetwork_name = string,
    ranges = list(object({
      range_name    = string,
      ip_cidr_range = string
    }))
  }))
| `[]` | no | +| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-primary-subnet") | `string` | `null` | no | +| [subnetwork\_size](#input\_subnetwork\_size) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `number` | `null` | no | +| [subnetworks](#input\_subnetworks) | List of subnetworks to create within the VPC. If left empty, it will be
replaced by a single, default subnetwork constructed from other parameters
(e.g. var.region). In all cases, the first subnetwork in the list is identified
by outputs as a "primary" subnetwork.

subnet\_name (string, required, name of subnet)
subnet\_region (string, required, region of subnet)
subnet\_ip (string, mutually exclusive with new\_bits, CIDR-formatted IP range for subnetwork)
new\_bits (number, mutually exclusive with subnet\_ip, CIDR bits used to calculate subnetwork range)
subnet\_private\_access (bool, optional, Enable Private Access on subnetwork)
subnet\_flow\_logs (map(string), optional, Configure Flow Logs see terraform-google-network module)
description (string, optional, Description of Network)
purpose (string, optional, related to Load Balancing)
role (string, optional, related to Load Balancing) | `list(map(string))` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nat\_ips](#output\_nat\_ips) | External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used) | +| [network\_id](#output\_network\_id) | ID of the new VPC network | +| [network\_name](#output\_network\_name) | Name of the new VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | +| [subnetwork](#output\_subnetwork) | Primary subnetwork object | +| [subnetwork\_address](#output\_subnetwork\_address) | IP address range of the primary subnetwork | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the primary subnetwork | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Self link of the primary subnetwork | +| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | + diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/main.tf b/deletion-test/cluster/modules/embedded/modules/network/vpc/main.tf new file mode 100644 index 0000000000..24c8eb22bd --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/vpc/main.tf @@ -0,0 +1,256 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +resource "terraform_data" "secondary_ranges_validation" { + lifecycle { + precondition { + condition = !(length(var.secondary_ranges) > 0 && length(var.secondary_ranges_list) > 0) + error_message = "Only one of var.secondary_ranges or var.secondary_ranges_list should be specified" + } + } +} + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "vpc", ghpc_role = "network" }) +} + +locals { + autoname = replace(var.deployment_name, "_", "-") + network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name + subnetwork_name = var.subnetwork_name == null ? "${local.autoname}-primary-subnet" : var.subnetwork_name + + # define a default subnetwork for cases in which no explicit subnetworks are + # defined in var.subnetworks + default_primary_subnetwork_cidr_block = cidrsubnet(var.network_address_range, var.default_primary_subnetwork_size, 0) + default_primary_subnetwork = { + subnet_name = local.subnetwork_name + subnet_ip = local.default_primary_subnetwork_cidr_block + subnet_region = var.region + subnet_private_access = true + subnet_flow_logs = false + description = "primary subnetwork in ${local.network_name}" + purpose = null + role = null + } + + # Identify user-supplied primary subnetwork + # (1) explicit var.subnetworks[0] + # (2) implicit local default subnetwork + input_primary_subnetwork = coalesce(try(var.subnetworks[0], null), local.default_primary_subnetwork) + + # Identify user-supplied additional subnetworks + # (1) explicit var.subnetworks[1:end] + # (2) empty list + input_additional_subnetworks = try(slice(var.subnetworks, 1, length(var.subnetworks)), []) + + # at this point we have constructed a list of subnetworks but need to extract + # user-provided CIDR blocks or calculate them from user-provided new_bits + # after we complete deprecation, local.all_subnetworks can be replaced with + # var.subnetworks (or local.default_primary_subnetwork if that is null) + input_subnetworks = concat([local.input_primary_subnetwork], local.input_additional_subnetworks) + subnetworks_cidr_blocks = try( + local.input_subnetworks[*]["subnet_ip"], + cidrsubnets(var.network_address_range, local.input_subnetworks[*]["new_bits"]...) + ) + + # merge in the CIDR blocks (even when already there) and remove new_bits + subnetworks = [for i, subnet in local.input_subnetworks : + merge({ for k, v in subnet : k => v if k != "new_bits" }, { "subnet_ip" = local.subnetworks_cidr_blocks[i] }) + ] + + # gather the unique regions for purposes of creating Router/NAT + cloud_router_regions = var.enable_cloud_router ? distinct([for subnet in local.subnetworks : subnet.subnet_region]) : [] + cloud_nat_regions = var.enable_cloud_nat ? local.cloud_router_regions : [] + + # this comprehension should have 1 and only 1 match + output_primary_subnetwork = one([for k, v in module.vpc.subnets : v if k == "${local.subnetworks[0].subnet_region}/${local.subnetworks[0].subnet_name}"]) + output_primary_subnetwork_name = local.output_primary_subnetwork.name + output_primary_subnetwork_self_link = local.output_primary_subnetwork.self_link + output_primary_subnetwork_ip_cidr_range = local.output_primary_subnetwork.ip_cidr_range + + iap_ports = distinct(concat(compact([ + var.enable_iap_rdp_ingress ? "3389" : "", + var.enable_iap_ssh_ingress ? "22" : "", + var.enable_iap_winrm_ingress ? "5986" : "", + ]), var.extra_iap_ports)) + + firewall_log_api_values = { + "DISABLE_LOGGING" = null + "INCLUDE_ALL_METADATA" = { metadata = "INCLUDE_ALL_METADATA" }, + "EXCLUDE_ALL_METADATA" = { metadata = "EXCLUDE_ALL_METADATA" }, + } + firewall_log_config = lookup(local.firewall_log_api_values, var.firewall_log_config, null) + + allow_iap_ingress = { + name = "${local.network_name}-fw-allow-iap-ingress" + description = "allow TCP access via Identity-Aware Proxy" + direction = "INGRESS" + priority = null + ranges = ["35.235.240.0/20"] + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = local.iap_ports + }] + deny = [] + log_config = local.firewall_log_config + } + + allow_ssh_ingress = { + name = "${local.network_name}-fw-allow-ssh-ingress" + description = "allow SSH access" + direction = "INGRESS" + priority = null + ranges = var.allowed_ssh_ip_ranges + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = ["22"] + }] + deny = [] + log_config = local.firewall_log_config + } + + allow_internal_traffic = { + name = "${local.network_name}-fw-allow-internal-traffic" + priority = null + description = "allow traffic between nodes of this VPC" + direction = "INGRESS" + ranges = [var.network_address_range] + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = ["0-65535"] + }, { + protocol = "udp" + ports = ["0-65535"] + }, { + protocol = "icmp" + ports = null + }, + ] + deny = [] + log_config = local.firewall_log_config + } + + firewall_rules = concat( + var.firewall_rules, + length(var.allowed_ssh_ip_ranges) > 0 ? [local.allow_ssh_ingress] : [], + var.enable_internal_traffic ? [local.allow_internal_traffic] : [], + length(local.iap_ports) > 0 ? [local.allow_iap_ingress] : [] + ) + + secondary_ranges_map = { + for secondary_range in var.secondary_ranges_list : + secondary_range.subnetwork_name => secondary_range.ranges + } +} + +resource "terraform_data" "network_profile_firewall_validation" { + lifecycle { + precondition { + condition = !(try(strcontains(var.network_profile, "roce"), false) && length(local.firewall_rules) > 0) + error_message = "If var.network_profile contains 'roce', var.firewall_rules must be empty and var.enable_internal_traffic must be false, please see: https://cloud.google.com/vpc/docs/rdma-network-profiles#additional_features_that_dont_apply_to_traffic_from_rdma_nics" + } + } +} + +module "vpc" { + source = "terraform-google-modules/network/google" + version = "~> 12.0" + + depends_on = [terraform_data.network_profile_firewall_validation] + + network_name = local.network_name + project_id = var.project_id + auto_create_subnetworks = false + subnets = local.subnetworks + secondary_ranges = length(local.secondary_ranges_map) > 0 ? local.secondary_ranges_map : var.secondary_ranges + routing_mode = var.network_routing_mode + mtu = var.mtu + description = var.network_description + shared_vpc_host = var.shared_vpc_host + delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes + firewall_rules = local.firewall_rules + network_profile = var.network_profile +} + +resource "terraform_data" "cloud_nat_validation" { + lifecycle { + precondition { + condition = var.enable_cloud_router == true || var.enable_cloud_nat == false + error_message = <<-EOD + "Cannot have Cloud NAT without a Cloud Router. If you desire Cloud NAT functionality please set `enable_cloud_router` to true." + EOD + } + } +} + +# This use of the module may appear odd when var.ips_per_nat = 0. The module +# will be called for all regions with subnetworks but names will be set to the +# empty list. This is a perfectly valid value (the default!). In this scenario, +# no IP addresses are created and all module outputs are empty lists. +# +# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/variables.tf#L27 +# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/outputs.tf +module "nat_ip_addresses" { + source = "terraform-google-modules/address/google" + version = "~> 4.1" + + depends_on = [terraform_data.cloud_nat_validation] + + for_each = toset(local.cloud_nat_regions) + + project_id = var.project_id + region = each.value + # an external, regional (not global) IP address is suited for a regional NAT + address_type = "EXTERNAL" + global = false + labels = local.labels + names = [for idx in range(var.ips_per_nat) : "${local.network_name}-nat-ips-${each.value}-${idx}"] +} + +module "cloud_router" { + source = "terraform-google-modules/cloud-router/google" + version = "~> 7.3" + + depends_on = [terraform_data.cloud_nat_validation] + + for_each = toset(local.cloud_router_regions) + + project = var.project_id + name = "${local.network_name}-router" + region = each.value + network = module.vpc.network_name + # in scenario with no NAT IPs, no NAT is created even if router is created + # https://github.com/terraform-google-modules/terraform-google-cloud-router/blob/v2.0.0/nat.tf#L18-L20 + nats = length(module.nat_ip_addresses[each.value].self_links) == 0 ? [] : [ + { + name : "cloud-nat-${each.value}", + nat_ips : module.nat_ip_addresses[each.value].self_links + }, + ] +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/network/vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/outputs.tf b/deletion-test/cluster/modules/embedded/modules/network/vpc/outputs.tf new file mode 100644 index 0000000000..c2ee6bdf6b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/vpc/outputs.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the new VPC network" + value = module.vpc.network_name + depends_on = [module.vpc, module.cloud_router] +} + +output "network_id" { + description = "ID of the new VPC network" + value = module.vpc.network_id + depends_on = [module.vpc, module.cloud_router] +} + +output "network_self_link" { + description = "Self link of the new VPC network" + value = module.vpc.network_self_link + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetworks" { + description = "Full list of subnetwork objects belonging to the new VPC network" + value = module.vpc.subnets + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork" { + description = "Primary subnetwork object" + value = local.output_primary_subnetwork + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_name" { + description = "Name of the primary subnetwork" + value = local.output_primary_subnetwork_name + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_self_link" { + description = "Self link of the primary subnetwork" + value = local.output_primary_subnetwork_self_link + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_address" { + description = "IP address range of the primary subnetwork" + value = local.output_primary_subnetwork_ip_cidr_range + depends_on = [module.vpc, module.cloud_router] +} + +output "nat_ips" { + description = "External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used)" + value = flatten([for ipmod in module.nat_ip_addresses : ipmod.addresses]) +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/variables.tf b/deletion-test/cluster/modules/embedded/modules/network/vpc/variables.tf new file mode 100644 index 0000000000..e036189404 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/vpc/variables.tf @@ -0,0 +1,301 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "labels" { + description = "Labels to add to network resources that support labels. Key-value pairs of strings." + type = map(string) + default = {} + nullable = false +} + +variable "network_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" + type = string + default = null +} + +variable "subnetwork_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-primary-subnet\")" + type = string + default = null +} + +# tflint-ignore: terraform_unused_declarations +variable "subnetwork_size" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = number + default = null + validation { + condition = var.subnetwork_size == null + error_message = "subnetwork_size is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +variable "default_primary_subnetwork_size" { + description = "The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks" + type = number + default = 15 +} + +variable "region" { + description = "The default region for Cloud resources" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "network_address_range" { + description = "IP address range (CIDR) for global network" + type = string + default = "10.0.0.0/9" + + validation { + condition = can(cidrhost(var.network_address_range, 0)) + error_message = "IP address range must be in CIDR format." + } +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "subnetworks" { + description = <<-EOT + List of subnetworks to create within the VPC. If left empty, it will be + replaced by a single, default subnetwork constructed from other parameters + (e.g. var.region). In all cases, the first subnetwork in the list is identified + by outputs as a "primary" subnetwork. + + subnet_name (string, required, name of subnet) + subnet_region (string, required, region of subnet) + subnet_ip (string, mutually exclusive with new_bits, CIDR-formatted IP range for subnetwork) + new_bits (number, mutually exclusive with subnet_ip, CIDR bits used to calculate subnetwork range) + subnet_private_access (bool, optional, Enable Private Access on subnetwork) + subnet_flow_logs (map(string), optional, Configure Flow Logs see terraform-google-network module) + description (string, optional, Description of Network) + purpose (string, optional, related to Load Balancing) + role (string, optional, related to Load Balancing) + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_name"]) + ]) + error_message = "All subnetworks must define \"subnet_name\"." + } + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_region"]) + ]) + error_message = "All subnetworks must define \"subnet_region\"." + } + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_ip"]) != can(s["new_bits"]) + ]) + error_message = "All subnetworks must define exactly one of \"subnet_ip\" or \"new_bits\"." + } + validation { + condition = alltrue([for s in var.subnetworks : can(s["subnet_ip"])]) || alltrue([for s in var.subnetworks : can(s["new_bits"])]) + error_message = "All subnetworks must make same choice of \"subnet_ip\" or \"new_bits\"." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "primary_subnetwork" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = map(string) + default = null + validation { + condition = var.primary_subnetwork == null + error_message = "primary_subnetwork is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "additional_subnetworks" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = list(map(string)) + default = null + validation { + condition = var.additional_subnetworks == null + error_message = "additional_subnetworks is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +variable "secondary_ranges" { + type = map(list(object({ range_name = string, ip_cidr_range = string }))) + description = <<-EOT + "Secondary ranges associated with the subnets. + This will be deprecated in favour of secondary_ranges_list at a later date. + Please migrate to using the same." + EOT + default = {} +} + +variable "secondary_ranges_list" { + type = list(object({ + subnetwork_name = string, + ranges = list(object({ + range_name = string, + ip_cidr_range = string + })) + })) + description = <<-EOT + "List of secondary ranges associated with the subnetworks. + Each subnetwork must be specified at most once in this list." + EOT + default = [] + validation { + condition = (length(var.secondary_ranges_list[*].subnetwork_name) == + length(distinct(var.secondary_ranges_list[*].subnetwork_name))) + error_message = "Each subnetwork should be specified at most once in this list. Remove any duplicates." + } +} + +variable "network_routing_mode" { + type = string + default = "GLOBAL" + description = "The network routing mode (default \"GLOBAL\")" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "ips_per_nat" { + type = number + description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples" + default = 2 +} + +variable "shared_vpc_host" { + type = bool + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + default = false +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_iap_ssh_ingress" { + type = bool + description = "Enable a firewall rule to allow SSH access using IAP tunnels" + default = true +} + +variable "enable_iap_rdp_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" + default = false +} + +variable "enable_iap_winrm_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" + default = false +} + +variable "enable_internal_traffic" { + type = bool + description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" + default = true +} + +variable "enable_cloud_router" { + type = bool + description = "Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview" + default = true +} + +variable "enable_cloud_nat" { + type = bool + description = "Enable the creation of Cloud NATs." + default = true +} + +variable "extra_iap_ports" { + type = list(string) + description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" + default = [] +} + +variable "allowed_ssh_ip_ranges" { + type = list(string) + description = "A list of CIDR IP ranges from which to allow ssh access" + default = [] + + validation { + condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) + error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." + } +} + +variable "firewall_rules" { + type = any + description = "List of firewall rules" + default = [] +} + +variable "firewall_log_config" { + type = string + description = "Firewall log configuration for Toolkit firewall rules (var.enable_iap_ssh_ingress and others)" + default = "DISABLE_LOGGING" + nullable = false + + validation { + condition = contains([ + "INCLUDE_ALL_METADATA", + "EXCLUDE_ALL_METADATA", + "DISABLE_LOGGING", + ], var.firewall_log_config) + error_message = "var.firewall_log_config must be set to \"DISABLE_LOGGING\", or enable logging with \"INCLUDE_ALL_METADATA\" or \"EXCLUDE_ALL_METADATA\"" + } +} + +variable "network_profile" { + type = string + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown + EOT + default = null +} diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/vpc/versions.tf new file mode 100644 index 0000000000..71b7106734 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/network/vpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.15.0" +} diff --git a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/README.md b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/README.md new file mode 100644 index 0000000000..192d7575a5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/README.md @@ -0,0 +1,320 @@ +# Custom Images in the Cluster Toolkit (formerly HPC Toolkit) + +Please review the +[introduction to image building](../../../docs/image-building.md) for general +information on building custom images using the Toolkit. + +## Introduction + +This module uses [Packer](https://www.packer.io/) to create an image within an +Cluster Toolkit deployment. Packer operates by provisioning a short-lived VM in +Google Cloud on which it executes scripts to customize the boot disk for +repeated use. The VM's boot disk is specified from a source image that defaults +to the [HPC VM Image][hpcimage]. This Packer "template" supports customization +by the following approaches following a [recommended use](#recommended-use): + +- [startup-script metadata][startup-metadata] from [raw string][sss] or + [file][ssf] +- [Shell scripts][shell] uploaded from the Packer execution environment to the + VM +- [Ansible playbooks][ansible] uploaded from the Packer execution environment to + the VM + +They can be specified independently of one another, so that anywhere from 1 to 3 +solutions can be used simultaneously. In the case that 0 scripts are supplied, +the source boot disk is effectively copied to your project without +customization. This can be useful in scenarios where increased control over the +image maintenance lifecycle is desired or when policies restrict the use of +images to internal projects. + +## Minimum requirements + +### Outbound internet access + +Most customization scripts require access to resources on the public internet. +This can be achieved by one of the following 2 approaches: + +1. Using a public IP address on the VM + +- Set [var.omit_external_ip](#input_omit_external_ip) to `false` + +1. Configuring a VPC with a Cloud NAT in the region of the VM + +- Use the [vpc] module which automates NAT creation + +### Inbound internet access + +Read [order of execution](#order-of-execution) below for a discussion of VM +customization solutions and their requirements for inbound SSH access. +[Environments without SSH access](#environments-without-ssh-access) should use +the metadata-based startup-script solution. + +A simple way to enable inbound SSH access is to use the VPC module with +`allowed_ssh_ip_ranges` set to `0.0.0.0/0`. + +### User or service account executing Packer at command line + +The user or service account running Packer must have the permission to create +VMs in the selected VPC network and, if [use\_iap](#input_use_iap) is set, must +have the "IAP-Secured Tunnel User" role. Recommended roles are: + +- `roles/compute.instanceAdmin.v1` +- `roles/iap.tunnelResourceAccessor` + +### VM service account roles + +The service account attached to the temporary build VM created by Packer should +have the ability to write Cloud Logging entries so that you may inspect and +debug build logs. When using the metadata startup-script customization solution, +the service account attached to the temporary build VM created by Packer must +have the permission to modify its own metadata and to read from Cloud Storage +buckets. Recommended roles are: + +- `roles/compute.instanceAdmin.v1` +- `roles/iam.serviceAccountUser` +- `roles/logging.logWriter` +- `roles/monitoring.metricWriter` +- `roles/storage.objectViewer` + +It is recommended to create this service account as a separate step outside a +blueprint due to known delay in [IAM bindings propagation][iamprop]. + +## Example blueprints + +A recommended pattern for building images with this module is to use the +terraform based [startup-script] module along with this packer custom-image +module. Below you can find links to several examples of this pattern, including +usage instructions. + +### [Image Builder] + +The [Image Builder] blueprint demonstrates a solution that builds an image +using: + +- The [HPC VM Image][hpcimage] as a base upon which to customize +- A VPC network with firewall rules that allow IAP-based SSH tunnels +- A Toolkit runner that installs a custom script + +Please review the [examples README] for usage instructions. + +## Order of execution + +The startup script specified in metadata executes in parallel with the other +supported methods. However, the remaining methods execute in a well-defined +order relative to one another. + +1. All shell scripts will execute in the configured order +1. After shell scripts complete, all Ansible playbooks will execute in the + configured order + +> **_NOTE:_** if both [startup_script][sss] and [startup_script_file][ssf] are +> specified, then [startup_script_file][ssf] takes precedence. + +## Recommended use + +Because the [metadata startup script executes in parallel](#order-of-execution) +with the other solutions, conflicts can arise, especially when package managers +(`yum` or `apt`) lock their databases during package installation. Therefore, it +is recommended to choose one of the following approaches: + +1. Specify _either_ [startup_script][sss] _or_ [startup_script_file][ssf] and do + not specify [shell_scripts][shell] or [ansible_playbooks][ansible]. + - This can be especially useful in + [environments that restrict SSH access](#environments-without-ssh-access) +1. Specify any combination of [shell_scripts][shell] and + [ansible_playbooks][ansible] and do not specify [startup_script][sss] or + [startup_script_file][ssf]. + +If any of the startup script approaches fail by returning a code other than 0, +Packer will determine that the build has failed and refuse to save the image. + +## External access with SSH + +The [shell scripts][shell] and [Ansible playbooks][ansible] customization +solutions both require SSH access to the VM from the Packer execution +environment. SSH access can be enabled one of 2 ways: + +1. The VM is created without a public IP address and SSH tunnels are created + using [Identity-Aware Proxy (IAP)][iaptunnel]. + - Allow [use_iap](#input_use_iap) to take on its default value of `true` +1. The VM is created with an IP address on the public internet and firewall + rules allow SSH access from the Packer execution environment. + - Set `omit_external_ip = false` (or `omit_external_ip: false` in a + blueprint) + - Add firewall rules that open SSH to the VM + +The Packer template defaults to using to the 1st IAP-based solution because it +is more secure (no exposure to public internet) and because the [vpc] module +automatically sets up all necessary firewall rules for SSH tunneling and +outbound-only access to the internet through [Cloud NAT][cloudnat]. + +In either SSH solution, customization scripts should be supplied as files in the +[shell_scripts][shell] and [ansible_playbooks][ansible] settings. + +## Environments without SSH access + +Many network environments disallow SSH access to VMs. In these environments, the +[metadata-based startup scripts][startup-metadata] are appropriate because they +execute entirely independently of the Packer execution environment. + +In this scenario, a single scripts should be supplied in the form of a string to +the [startup_script][sss] input variable. This solution integrates well with +Toolkit runners. Runners operate by using a single startup script whose behavior +is extended by downloading and executing a customizable set of runners from +Cloud Storage at startup. + +> **_NOTE:_** Packer will attempt to use SSH if either [shell_scripts][shell] or +> [ansible_playbooks][ansible] are set to non-empty values. Leave them at their +> default, empty values to ensure access by SSH is disabled. + +## Supplying startup script as a string + +The [startup_script][sss] parameter accepts scripts formatted as strings. In +Packer and Terraform, multi-line strings can be specified using +[heredoc syntax](https://www.terraform.io/language/expressions/strings#heredoc-strings) +in an input [Packer variables file][pkrvars] (`*.pkrvars.hcl`) For example, the +following snippet defines a multi-line bash script followed by an integer +representing the size, in GiB, of the resulting image: + +```hcl +startup_script = <<-EOT + #!/bin/bash + yum install -y epel-release + yum install -y jq + EOT + +disk_size = 100 +``` + +In a blueprint, the equivalent syntax is: + +```yaml +... + settings: + startup_script: | + #!/bin/bash + yum install -y epel-release + yum install -y jq + disk_size: 100 +... +``` + +## Monitoring startup script execution + +When using startup script customization, Packer will print very limited output +to the console. For example: + +```text +==> example.googlecompute.toolkit_image: Waiting for any running startup script to finish... +==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... +==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... +==> example.googlecompute.toolkit_image: Startup script, if any, has finished running. +``` + +### Debugging startup-script failures + +> [!NOTE] +> There can be a delay in the propagation of the logs from the instance to +> Cloud Logging, so it may require waiting a few minutes to see the full logs. + +If the Packer image build fails, the module will output a `gcloud` command +that can be used directly to review startup-script execution. + +## License + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + +```text + http://www.apache.org/licenses/LICENSE-2.0 +``` + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_count](#input\_accelerator\_count) | Number of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `number` | `null` | no | +| [accelerator\_type](#input\_accelerator\_type) | Type of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `string` | `null` | no | +| [ansible\_playbooks](#input\_ansible\_playbooks) | A list of Ansible playbook configurations that will be uploaded to customize the VM image | 
list(object({
    playbook_file   = string
    galaxy_file     = string
    extra_arguments = list(string)
  }))
| `[]` | no | +| [communicator](#input\_communicator) | Communicator to use for provisioners that require access to VM ("ssh" or "winrm") | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name | `string` | n/a | yes | +| [disk\_size](#input\_disk\_size) | Size of disk image in GB | `number` | `null` | no | +| [disk\_type](#input\_disk\_type) | Type of persistent disk to provision | `string` | `"pd-balanced"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [image\_family](#input\_image\_family) | The family name of the image to be built. Defaults to `deployment_name` | `string` | `null` | no | +| [image\_name](#input\_image\_name) | The name of the image to be built. If not supplied, it will be set to image\_family-$ISO\_TIMESTAMP | `string` | `null` | no | +| [image\_storage\_locations](#input\_image\_storage\_locations) | Storage location, either regional or multi-regional, where snapshot content is to be stored and only accepts 1 value.
See https://developer.hashicorp.com/packer/plugins/builders/googlecompute#image_storage_locations | `list(string)` | `null` | no | +| [labels](#input\_labels) | Labels to apply to the short-lived VM | `map(string)` | `null` | no | +| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2-standard-4"` | no | +| [manifest\_file](#input\_manifest\_file) | File to which to write Packer build manifest | `string` | `"packer-manifest.json"` | no | +| [metadata](#input\_metadata) | Instance metadata for the builder VM (use var.startup\_script or var.startup\_script\_file to set startup-script metadata) | `map(string)` | `{}` | no | +| [network\_project\_id](#input\_network\_project\_id) | Project ID of Shared VPC network | `string` | `null` | no | +| [omit\_external\_ip](#input\_omit\_external\_ip) | Provision the image building VM without a public IP address | `bool` | `true` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which to create VM and image | `string` | n/a | yes | +| [scopes](#input\_scopes) | DEPRECATED: use var.service\_account\_scopes | `set(string)` | `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | The service account email to use. If null or 'default', then the default Compute Engine service account will be used. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Service account scopes to attach to the instance. See
https://cloud.google.com/compute/docs/access/service-accounts. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shell\_scripts](#input\_shell\_scripts) | A list of paths to local shell scripts which will be uploaded to customize the VM image | `list(string)` | `[]` | no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [source\_image](#input\_source\_image) | Source OS image to build from | `string` | `null` | no | +| [source\_image\_family](#input\_source\_image\_family) | Alternative to source\_image. Specify image family to build from latest image in family | `string` | `"hpc-rocky-linux-8"` | no | +| [source\_image\_project\_id](#input\_source\_image\_project\_id) | A list of project IDs to search for the source image. Packer will search the
first project ID in the list first, and fall back to the next in the list,
until it finds the source image. | `list(string)` | `null` | no | +| [ssh\_username](#input\_ssh\_username) | Username to use for SSH access to VM | `string` | `"hpc-toolkit-packer"` | no | +| [startup\_script](#input\_startup\_script) | Startup script (as raw string) used to build the custom Linux VM image (overridden by var.startup\_script\_file if both are set) | `string` | `null` | no | +| [startup\_script\_file](#input\_startup\_script\_file) | File path to local shell script that will be used to customize the Linux VM image (overrides var.startup\_script) | `string` | `null` | no | +| [state\_timeout](#input\_state\_timeout) | The time to wait for instance state changes, including image creation | `string` | `"10m"` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of subnetwork in which to provision image building VM | `string` | n/a | yes | +| [tags](#input\_tags) | Assign network tags to apply firewall rules to VM instance | `list(string)` | `null` | no | +| [use\_iap](#input\_use\_iap) | Use IAP proxy when connecting by SSH | `bool` | `true` | no | +| [use\_os\_login](#input\_use\_os\_login) | Use OS Login when connecting by SSH | `bool` | `false` | no | +| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | A list of strings containing PowerShell scripts which will customize a Windows VM image (requires WinRM communicator) | `list(string)` | `[]` | no | +| [wrap\_startup\_script](#input\_wrap\_startup\_script) | Wrap startup script with Packer-generated wrapper | `bool` | `true` | no | +| [zone](#input\_zone) | Cloud zone in which to provision image building VM | `string` | n/a | yes | + +## Outputs + +No outputs. + + +[ansible]: #input_ansible_playbooks +[cloudnat]: https://cloud.google.com/nat/docs/overview +[examples readme]: ../../../examples/README.md#image-builderyaml- +[hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[iamprop]: https://cloud.google.com/iam/docs/access-change-propagation +[iaptunnel]: https://cloud.google.com/iap/docs/using-tcp-forwarding +[image builder]: ../../../examples/image-builder.yaml +[logging-console]: https://console.cloud.google.com/logs/ +[logging-read-docs]: https://cloud.google.com/sdk/gcloud/reference/logging/read +[pkrvars]: https://www.packer.io/guides/hcl/variables#from-a-file +[shell]: #input_shell_scripts +[ssf]: #input_startup_script_file +[sss]: #input_startup_script +[startup-metadata]: https://cloud.google.com/compute/docs/instances/startup-scripts/linux +[startup-script]: ../../../modules/scripts/startup-script +[vpc]: ../../network/vpc/README.md diff --git a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/image.pkr.hcl b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/image.pkr.hcl new file mode 100644 index 0000000000..9282cf7433 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/image.pkr.hcl @@ -0,0 +1,216 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "custom-image", ghpc_role = "packer" }) + + # construct a unique image name from the image family + image_family = var.image_family != null ? var.image_family : var.deployment_name + image_name_default = "${local.image_family}-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" + image_name = var.image_name != null ? var.image_name : local.image_name_default + + # construct vm image name for use when getting logs + instance_name = "packer-${substr(uuidv4(), 0, 6)}" + + # default to explicit var.communicator, otherwise in-order: ssh/winrm/none + shell_script_communicator = length(var.shell_scripts) > 0 ? "ssh" : "" + ansible_playbook_communicator = length(var.ansible_playbooks) > 0 ? "ssh" : "" + powershell_script_communicator = length(var.windows_startup_ps1) > 0 ? "winrm" : "" + communicator = coalesce( + var.communicator, + local.shell_script_communicator, + local.ansible_playbook_communicator, + local.powershell_script_communicator, + "none" + ) + + # must not enable IAP when no communicator is in use + use_iap = local.communicator == "none" ? false : var.use_iap + + # construct metadata from startup_script and metadata variables + startup_script_metadata = var.startup_script == null ? {} : { startup-script = var.startup_script } + + linux_user_metadata = { + block-project-ssh-keys = "TRUE" + shutdown-script = <<-EOT + #!/bin/bash + userdel -r ${var.ssh_username} + sed -i '/${var.ssh_username}/d' /var/lib/google/google_users + EOT + } + windows_packer_user = "packer_user" + windows_user_metadata = { + sysprep-specialize-script-cmd = "winrm quickconfig -quiet & net user /add ${local.windows_packer_user} & net localgroup administrators ${local.windows_packer_user} /add & winrm set winrm/config/service/auth @{Basic=\\\"true\\\"}" + windows-shutdown-script-cmd = <<-EOT + net user /delete ${local.windows_packer_user} + EOT + } + user_metadata = local.communicator == "winrm" ? local.windows_user_metadata : local.linux_user_metadata + + # merge metadata such that var.metadata always overrides user management + # metadata but always allow var.startup_script to override var.metadata + metadata = merge( + local.user_metadata, + var.metadata, + local.startup_script_metadata, + ) + + # determine best value for on_host_maintenance if not supplied by user + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + gpu_attached = contains(["a2", "g2"], local.machine_family) || var.accelerator_type != null + on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" + on_host_maintenance = ( + var.on_host_maintenance != null + ? var.on_host_maintenance + : local.on_host_maintenance_default + ) + + accelerator_type = var.accelerator_type == null ? null : "projects/${var.project_id}/zones/${var.zone}/acceleratorTypes/${var.accelerator_type}" + + winrm_username = local.communicator == "winrm" ? "packer_user" : null + winrm_insecure = local.communicator == "winrm" ? true : null + winrm_use_ssl = local.communicator == "winrm" ? true : null + + enable_integrity_monitoring = var.enable_shielded_vm && var.shielded_instance_config.enable_integrity_monitoring + enable_secure_boot = var.enable_shielded_vm && var.shielded_instance_config.enable_secure_boot + enable_vtpm = var.enable_shielded_vm && var.shielded_instance_config.enable_vtpm + + image_licenses = [ + "projects/click-to-deploy-images/global/licenses/hpc-toolkit-vm-image" + ] +} + +source "googlecompute" "toolkit_image" { + communicator = local.communicator + project_id = var.project_id + image_name = local.image_name + image_family = local.image_family + image_labels = local.labels + instance_name = local.instance_name + machine_type = var.machine_type + accelerator_type = local.accelerator_type + accelerator_count = var.accelerator_count + on_host_maintenance = local.on_host_maintenance + disk_size = var.disk_size + disk_type = var.disk_type + omit_external_ip = var.omit_external_ip + use_internal_ip = var.omit_external_ip + subnetwork = var.subnetwork_name + network_project_id = var.network_project_id + service_account_email = var.service_account_email + scopes = var.service_account_scopes + source_image = var.source_image + source_image_family = var.source_image_family + source_image_project_id = var.source_image_project_id + ssh_username = var.ssh_username + tags = var.tags + use_iap = local.use_iap + use_os_login = var.use_os_login + winrm_username = local.winrm_username + winrm_insecure = local.winrm_insecure + winrm_use_ssl = local.winrm_use_ssl + zone = var.zone + labels = local.labels + metadata = local.metadata + startup_script_file = var.startup_script_file + wrap_startup_script = var.wrap_startup_script + state_timeout = var.state_timeout + image_storage_locations = var.image_storage_locations + enable_secure_boot = local.enable_secure_boot + enable_vtpm = local.enable_vtpm + enable_integrity_monitoring = local.enable_integrity_monitoring + image_licenses = local.image_licenses +} + +build { + name = var.deployment_name + sources = ["sources.googlecompute.toolkit_image"] + + # using dynamic blocks to create provisioners ensures that there are no + # provisioner blocks when none are provided and we can use the none + # communicator when using startup-script + + # provisioner "shell" blocks + dynamic "provisioner" { + labels = ["shell"] + for_each = var.shell_scripts + content { + execute_command = "sudo -H sh -c '{{ .Vars }} {{ .Path }}'" + script = provisioner.value + } + } + + # provisioner "powershell" blocks + dynamic "provisioner" { + labels = ["powershell"] + for_each = var.windows_startup_ps1 + content { + inline = split("\n", provisioner.value) + } + } + + dynamic "provisioner" { + labels = ["powershell"] + for_each = length(var.windows_startup_ps1) > 0 ? [1] : [] + content { + inline = [ + "GCESysprep -no_shutdown" + ] + } + } + + # provisioner "ansible-local" blocks + # this installs custom roles/collections from ansible-galaxy in /home/packer + # which will be removed at the end; consider modifying /etc/ansible/ansible.cfg + dynamic "provisioner" { + labels = ["ansible-local"] + for_each = var.ansible_playbooks + content { + playbook_file = provisioner.value.playbook_file + galaxy_file = provisioner.value.galaxy_file + extra_arguments = provisioner.value.extra_arguments + } + } + + post-processor "manifest" { + output = var.manifest_file + strip_path = true + custom_data = { + built-by = "cloud-hpc-toolkit" + } + } + + # If there is an error during image creation, print out command for getting packer VM logs + error-cleanup-provisioner "shell-local" { + environment_vars = [ + "PRJ_ID=${var.project_id}", + "INST_NAME=${local.instance_name}", + "ZONE=${var.zone}", + ] + inline_shebang = "/bin/bash -e" + inline = [ + "type -P gcloud > /dev/null || exit 0", + "INST_ID=$(gcloud compute instances describe $INST_NAME --project $PRJ_ID --format=\"value(id)\" --zone=$ZONE)", + "echo 'Error building image try checking logs:'", + join(" ", ["echo \"gcloud logging --project $PRJ_ID read", + "'logName=(\\\"projects/$PRJ_ID/logs/GCEMetadataScripts\\\" OR \\\"projects/$PRJ_ID/logs/google_metadata_script_runner\\\") AND resource.labels.instance_id=$INST_ID'", + "--format=\\\"table(timestamp, resource.labels.instance_id, jsonPayload.message)\\\"", + "--order=asc\"" + ] + ) + ] + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/metadata.yaml new file mode 100644 index 0000000000..23108c4e17 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - logging.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/variables.pkr.hcl b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/variables.pkr.hcl new file mode 100644 index 0000000000..3cede102ce --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/variables.pkr.hcl @@ -0,0 +1,276 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "deployment_name" { + description = "Cluster Toolkit deployment name" + type = string +} + +variable "project_id" { + description = "Project in which to create VM and image" + type = string +} + +variable "machine_type" { + description = "VM machine type on which to build new image" + type = string + default = "n2-standard-4" +} + +variable "disk_size" { + description = "Size of disk image in GB" + type = number + default = null +} + +variable "disk_type" { + description = "Type of persistent disk to provision" + type = string + default = "pd-balanced" +} + +variable "zone" { + description = "Cloud zone in which to provision image building VM" + type = string +} + +variable "network_project_id" { + description = "Project ID of Shared VPC network" + type = string + default = null +} + +variable "subnetwork_name" { + description = "Name of subnetwork in which to provision image building VM" + type = string +} + +variable "omit_external_ip" { + description = "Provision the image building VM without a public IP address" + type = bool + default = true +} + +variable "tags" { + description = "Assign network tags to apply firewall rules to VM instance" + type = list(string) + default = null +} + +variable "image_family" { + description = "The family name of the image to be built. Defaults to `deployment_name`" + type = string + default = null +} + +variable "image_name" { + description = "The name of the image to be built. If not supplied, it will be set to image_family-$ISO_TIMESTAMP" + type = string + default = null +} + +variable "source_image_project_id" { + description = < +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 4.0 | +| [local](#requirement\_local) | >= 2.0.0 | +| [null](#requirement\_null) | ~> 3.0 | +| [random](#requirement\_random) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.0 | +| [local](#provider\_local) | >= 2.0.0 | +| [null](#provider\_null) | ~> 3.0 | +| [random](#provider\_random) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_template](#module\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.submit_script](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [null_resource.submit_job](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.submit_job_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used for the job\_id | `string` | n/a | yes | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs | `bool` | `true` | no | +| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions. Valid inputs are `"alpha"`, `"beta"` and "" (empty string for default version) | `string` | `""` | no | +| [image](#input\_image) | DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided. | `any` | `null` | no | +| [instance\_image](#input\_instance\_image) | Google Cloud Batch compute node image. Ignored if `instance_template` is provided.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [instance\_template](#input\_instance\_template) | Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions. | `string` | `null` | no | +| [job\_filename](#input\_job\_filename) | The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified | `string` | `null` | no | +| [job\_id](#input\_job\_id) | An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs. | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided. | `map(string)` | n/a | yes | +| [log\_policy](#input\_log\_policy) | Create a block to define log policy.
When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging.
When set to `PATH`, path must be added to generated template.
When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. | `string` | `"CLOUD_LOGGING"` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided. | `string` | `"n2-standard-4"` | no | +| [mpi\_mode](#input\_mpi\_mode) | Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`. | `bool` | `false` | no | +| [native\_batch\_mounting](#input\_native\_batch\_mounting) | Batch can mount some fs\_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts. | `bool` | `true` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which to run the Google Cloud Batch job | `string` | n/a | yes | +| [runnable](#input\_runnable) | A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`. | `string` | `null` | no | +| [runnables](#input\_runnables) | A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template. | 
list(object({
    script = string
  }))
| `null` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/devstorage.read_only",
    "https://www.googleapis.com/auth/logging.write",
    "https://www.googleapis.com/auth/monitoring.write",
    "https://www.googleapis.com/auth/servicecontrol",
    "https://www.googleapis.com/auth/service.management.readonly",
    "https://www.googleapis.com/auth/trace.append"
  ]
}
| no | +| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided. | `string` | `null` | no | +| [submit](#input\_submit) | When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply. | `bool` | `false` | no | +| [subnetwork](#input\_subnetwork) | The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided. | `any` | `null` | no | +| [task\_count](#input\_task\_count) | Number of parallel tasks | `number` | `1` | no | +| [task\_count\_per\_node](#input\_task\_count\_per\_node) | Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value. | `number` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcloud\_version](#output\_gcloud\_version) | The version of gcloud to be used. | +| [instance\_template](#output\_instance\_template) | Instance template used by the Batch job. | +| [instructions](#output\_instructions) | Instructions for submitting the Batch job. | +| [job\_data](#output\_job\_data) | All data associated with the defined job, typically provided as input to clout-batch-login-node. | +| [network\_storage](#output\_network\_storage) | An array of network attached storage mounts used by the Batch job. | +| [startup\_script](#output\_startup\_script) | Startup script run before Google Cloud Batch job starts. | + diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/main.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/main.tf new file mode 100644 index 0000000000..0d681536c9 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/main.tf @@ -0,0 +1,149 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "batch-job-template", ghpc_role = "scheduler" }) +} + +locals { + instance_template = coalesce(var.instance_template, module.instance_template.self_link) + + tasks_per_node = var.task_count_per_node != null ? var.task_count_per_node : (var.mpi_mode ? 1 : null) + + one_line_runnable = coalesce(var.runnable, "## Add your workload here ##") + runnables = coalesce(var.runnables, [{ script = local.one_line_runnable }]) + + job_template_contents = templatefile( + "${path.module}/templates/batch-job-base.yaml.tftpl", + { + synchronized = var.mpi_mode + runnables = local.runnables + task_count = var.task_count + tasks_per_node = local.tasks_per_node + require_hosts_file = var.mpi_mode + permissive_ssh = var.mpi_mode + log_policy = var.log_policy + instance_template = local.instance_template + nfs_volumes = local.native_batch_network_storage + labels = local.labels + } + ) + + submit_job_id = "${var.job_id}-${random_id.submit_job_suffix.hex}" + job_filename = coalesce(var.job_filename, "${var.job_id}.yaml") + job_template_output_path = "${path.root}/${local.job_filename}" + + submit_script_contents = templatefile( + "${path.module}/templates/batch-submit.sh.tftpl", + { + project = var.project_id + location = var.region + config = local_file.job_template.filename + submit_job_id = local.submit_job_id + } + ) + submit_script_output_path = "${path.root}/submit-${var.job_id}.sh" + + subnetwork_name = var.subnetwork != null ? var.subnetwork.name : "default" + subnetwork_project = var.subnetwork != null ? var.subnetwork.project : var.project_id + + # Filter network_storage for native Batch support + native_fstype = var.native_batch_mounting ? ["nfs"] : [] + native_batch_network_storage = [ + for ns in var.network_storage : + ns if contains(local.native_fstype, ns.fs_type) + ] + # other processing happens in startup_from_network_storage.tf + + # this code is similar to code in Packer and vm-instance modules + # it differs in that this module does not (yet) expose var.guest_acclerator + # for attaching GPUs to N1 VMs. For now, identify only A2 types. + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + gpu_attached = contains(["a2", "g2"], local.machine_family) + on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" + + on_host_maintenance = coalesce(var.on_host_maintenance, local.on_host_maintenance_default) + + network_storage_metadata = var.network_storage != null ? ({ network_storage = jsonencode(var.network_storage) }) : {} + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.network_storage_metadata, + local.disable_automatic_updates_metadata + ) +} + +module "instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = var.instance_template == null ? "${var.job_id}-instance-template" : "unused-template" + project_id = var.project_id + subnetwork = local.subnetwork_name + subnetwork_project = local.subnetwork_project + service_account = var.service_account + access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + labels = local.labels + + machine_type = var.machine_type + startup_script = local.startup_from_network_storage + metadata = local.metadata + source_image_family = data.google_compute_image.compute_image.family + source_image = data.google_compute_image.compute_image.name + source_image_project = data.google_compute_image.compute_image.project + on_host_maintenance = local.on_host_maintenance +} + +resource "local_file" "job_template" { + content = local.job_template_contents + filename = local.job_template_output_path + + lifecycle { + precondition { + condition = var.runnable == null || var.runnables == null + error_message = "var.runnable and var.runnables (plural) cannot both be set." + } + } +} + +resource "random_id" "submit_job_suffix" { + byte_length = 4 + keepers = { + always_run = timestamp() + } +} + +resource "local_file" "submit_script" { + content = local.submit_script_contents + filename = local.submit_script_output_path +} + +resource "null_resource" "submit_job" { + depends_on = [local_file.job_template, local_file.submit_script] + count = var.submit ? 1 : 0 + + # A new deployment should always submit a new job. Old finished jobs aren't persistent parts of + # Cloud infrastructure. + triggers = { + always_run = timestamp() + } + + provisioner "local-exec" { + command = local.submit_script_output_path + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml new file mode 100644 index 0000000000..387e810962 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - batch.googleapis.com + - compute.googleapis.com +ghpc: + inject_module_id: job_id diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/outputs.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/outputs.tf new file mode 100644 index 0000000000..0b1295975a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/outputs.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + provided_instance_tpl_msg = "The Batch job template uses the existing VM instance template:" + generated_instance_tpl_msg = "The Batch job template uses a new VM instance template created matching the provided settings:" + submit_msg = <<-EOT + + The job has been submitted. See job status at: + https://console.cloud.google.com/batch/jobsDetail/regions/${var.region}/jobs/${local.submit_job_id}?project=${var.project_id} + EOT +} + +output "instructions" { + description = "Instructions for submitting the Batch job." + value = <<-EOT + + A Batch job template file has been created locally at: + ${abspath(local.job_template_output_path)} + + ${var.instance_template == null ? local.generated_instance_tpl_msg : local.provided_instance_tpl_msg} + ${local.instance_template} + ${var.submit ? local.submit_msg : ""} + + Use the following commands to: + Submit your job${var.submit ? " (Note: job has already been submitted)" : ""}: + gcloud ${var.gcloud_version} batch jobs submit ${local.submit_job_id} --config=${abspath(local.job_template_output_path)} --location=${var.region} --project=${var.project_id} + + Check status: + gcloud ${var.gcloud_version} batch jobs describe ${local.submit_job_id} --location=${var.region} --project=${var.project_id} | grep state: + + Delete job: + gcloud ${var.gcloud_version} batch jobs delete ${local.submit_job_id} --location=${var.region} --project=${var.project_id} + + List all jobs: + gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} + EOT +} + +output "job_data" { + description = "All data associated with the defined job, typically provided as input to clout-batch-login-node." + value = { + template_contents = local.job_template_contents, + filename = local.job_filename, + id = local.submit_job_id + } +} + +output "instance_template" { + description = "Instance template used by the Batch job." + value = local.instance_template +} + +output "network_storage" { + description = "An array of network attached storage mounts used by the Batch job." + value = var.network_storage +} + +output "startup_script" { + description = "Startup script run before Google Cloud Batch job starts." + value = var.startup_script +} + +output "gcloud_version" { + description = "The version of gcloud to be used." + value = var.gcloud_version +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf new file mode 100644 index 0000000000..02bc58e4f7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "inputs": +# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. +# var.network_storage : to be passed into metadata somewhere else (not here) +# var.startup_script : to be changed into a more complete file system with all the fs runners + +# "outputs": +# local.startup_from_network_storage : A full startup script with all the runners that are not supported +# natively and were included in the network_storage structure + +locals { + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + # Pull out runners to include in startup script + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + startup_script_runner = [{ + content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" + destination = "passed_startup_script.sh" + type = "shell" + }] + + full_runner_list = concat( + local.storage_client_install_runners, + local.mount_runners, + local.startup_script_runner + ) + + startup_from_network_storage = module.netstorage_startup_script.startup_script +} + +module "netstorage_startup_script" { + source = "../../scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.full_runner_list +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl new file mode 100644 index 0000000000..83fccde53b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl @@ -0,0 +1,53 @@ +taskGroups: + - taskSpec: + runnables: + %{~ if synchronized ~} + - barrier: + name: "wait-for-node-startup" + %{~ endif ~} + %{~ for runnable in runnables ~} + - script: + text: ${indent(12, chomp(yamlencode(runnable.script)))} + %{~ if synchronized ~} + - barrier: + name: "wait-for-script-to-complete" + %{~ endif ~} + %{~ endfor ~} + %{~ if length(nfs_volumes) > 0 ~} + volumes: + %{~ for index, vol in nfs_volumes ~} + - nfs: + server: "${vol.server_ip}" + remotePath: "${vol.remote_mount}" + %{~ if vol.mount_options != "" && vol.mount_options != null ~} + mountOptions: "${vol.mount_options}" + %{~ endif ~} + mountPath: "${vol.local_mount}" + %{~ endfor ~} + %{~ endif ~} + taskCount: ${task_count} + %{~ if tasks_per_node != null ~} + taskCountPerNode: ${tasks_per_node} + %{~ endif ~} + requireHostsFile: ${require_hosts_file} + permissiveSsh: ${permissive_ssh} +%{~ if instance_template != null } +allocationPolicy: + instances: + - instanceTemplate: "${instance_template}" +%{~ endif } +%{~ if log_policy == "CLOUD_LOGGING" } +logsPolicy: + destination: "CLOUD_LOGGING" +%{ endif } +%{~ if log_policy == "PATH" } +logsPolicy: + destination: "PATH" + logsPath: ## Add logging path here +%{ endif } +%{~ if length(labels) > 0 ~} +labels: +%{ for k, v in labels ~} + ${k}: "${v}" +%{ endfor } +%{~ endif ~} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl new file mode 100644 index 0000000000..25f89c3ceb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl @@ -0,0 +1,10 @@ +#!/bin/bash +set -e -o pipefail +GCLOUD_MAJOR_VERSION=$(gcloud --version | head -n 1 | awk '{print $NF}' | cut -f1 --delimiter=.) +if [ $((GCLOUD_MAJOR_VERSION >= 461)) ]; then + gcloud batch jobs submit ${submit_job_id} --project=${project} --location=${location} --config=${config} + echo "batch job ${submit_job_id} successfully submitted" +else + echo "gcloud must be updated to version 461.0.0 or later." + exit 1 +fi diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/variables.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/variables.tf new file mode 100644 index 0000000000..f65fbd111e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/variables.tf @@ -0,0 +1,240 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "region" { + description = "The region in which to run the Google Cloud Batch job" + type = string +} + +variable "deployment_name" { + description = "Name of the deployment, used for the job_id" + type = string +} + +variable "labels" { + description = "Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided." + type = map(string) +} + +variable "job_id" { + description = "An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs." + type = string +} + +variable "job_filename" { + description = "The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified" + type = string + default = null +} + +variable "gcloud_version" { + description = "The version of the gcloud cli being used. Used for output instructions. Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default version)" + type = string + default = "" + + validation { + condition = contains(["alpha", "beta", ""], var.gcloud_version) + error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." + } +} + +variable "task_count" { + description = "Number of parallel tasks" + type = number + default = 1 +} + +variable "task_count_per_node" { + description = "Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value." + type = number + default = null +} + +variable "mpi_mode" { + description = "Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`." + type = bool + default = false +} + +variable "log_policy" { + description = <<-EOT + Create a block to define log policy. + When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging. + When set to `PATH`, path must be added to generated template. + When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. + EOT + type = string + default = "CLOUD_LOGGING" + + validation { + condition = contains(["CLOUD_LOGGING", "PATH", "DESTINATION_UNSPECIFIED"], var.log_policy) + error_message = "Allowed values for log_policy are 'CLOUD_LOGGING', 'PATH', or 'DESTINATION_UNSPECIFIED'." + } +} + +variable "runnables" { + description = "A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template." + type = list(object({ + script = string + })) + default = null +} + +variable "runnable" { + description = "A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`." + type = string + default = null +} + +variable "instance_template" { + description = "Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions." + type = string + default = null +} + +variable "subnetwork" { + description = "The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided." + type = any + default = null +} + +variable "enable_public_ips" { + description = "If set to true, instances will have public IPs" + type = bool + default = true +} + +variable "service_account" { + description = "Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided." + type = object({ + email = string, + scopes = set(string) + }) + default = { + email = null + scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/trace.append" + ] + } +} + +variable "machine_type" { + description = "Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided." + type = string + default = "n2-standard-4" +} + +variable "startup_script" { + description = "Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided." + type = string + default = null +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided." + type = list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "native_batch_mounting" { + description = "Batch can mount some fs_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts." + type = bool + default = true +} + +# Deprecated, replaced by instance_image +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided." + type = any + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + Google Cloud Batch compute node image. Ignored if `instance_template` is provided. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE`" + type = string + default = null + validation { + condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) + error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." + } +} + +variable "submit" { + description = "When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply." + type = bool + default = false +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/versions.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/versions.tf new file mode 100644 index 0000000000..a1161e1354 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + google = { + source = "hashicorp/google" + version = ">= 4.0" + } + } + required_version = ">= 1.1" +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/README.md b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/README.md new file mode 100644 index 0000000000..c20ca7dbeb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/README.md @@ -0,0 +1,127 @@ +# Description + +This module creates a VM that acts as a login node to test and submit Google +Cloud Batch jobs. It is intended to be used along with the `batch-job-template` +module. + +This login node: + +- Uses the same VM settings as the first provided `batch-job-template`, such as + image, machine type, etc... +- Runs the same `startup-script` as the first provided `batch-job-template`. +- Has the same mounted file systems as the provided `batch-job-template`. +- Contains a folder with job templates generated by `batch-job-template` modules. + +Since the login node has the same mounted storage and is a homogeneous machine +to the Google Cloud Batch compute VMs, it can be used to inspect shared file +systems and test installed software before submitting a Google Cloud Batch job. + +## Example + +```yaml +- id: batch-job + source: modules/scheduler/batch-job-template + ... + +- id: batch-login + source: modules/scheduler/batch-login-node + use: [batch-job] + outputs: [instructions] +``` + +## Authentication + +To submit jobs from the login node, the service account attached to the VM needs +the `Batch Job Administrator` role. In most cases this service account will be +the Compute Engine default service account and will not be granted this role by +default. + +You can grant this role either by adding the `Batch Job Administrator` role to +the service account in the IAM page in the Google Cloud Console, or by running +the following command line: + +```bash +gcloud projects add-iam-policy-binding \ + --member=serviceAccount: \ + --role=roles/batch.jobsAdmin +``` + +## gcloud Batch Access + +Until the Google Cloud Batch API is generally available (GA), it may not be +available in all versions of the `gcloud` cli. You can test if the Google Cloud +Batch commands are available by running `gcloud [alpha|beta|] batch -h`. If the +Google Cloud Batch cli is not available it can generally be mitigated by either +updating `gcloud` by running `gcloud components update`, or using an image that +contains a more recent version of `gcloud`. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [login\_startup\_script](#module\_login\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_from_template.batch_login](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | +| [google_compute_instance_template.batch_instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [batch\_job\_directory](#input\_batch\_job\_directory) | The path of the directory on the login node in which to place the Google Cloud Batch job template | `string` | `"/home/batch-jobs"` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, also used for the job\_id | `string` | n/a | yes | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions.
Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default
version). Typically supplied by a batch-job-template module. If multiple
batch-job-template modules supply the gcloud\_version, only the first will be used. | `string` | `""` | no | +| [instance\_template](#input\_instance\_template) | Login VM instance template self-link. Typically supplied by a
batch-job-template module. If multiple batch-job-template modules supply the
instance\_template, the first will be used. | `string` | n/a | yes | +| [job\_data](#input\_job\_data) | List of jobs and supporting data for each, typically provided via "use" from the batch-job-template module. | 
list(object({
    template_contents = string,
    filename          = string,
    id                = string
  }))
| n/a | yes | +| [job\_filename](#input\_job\_filename) | Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [job\_id](#input\_job\_id) | Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions. | `string` | `null` | no | +| [job\_template\_contents](#input\_job\_template\_contents) | Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the login node. Key-value pairs | `map(string)` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which to create the login node | `string` | n/a | yes | +| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [zone](#input\_zone) | The zone in which to create the login node | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for accessing the login node and submitting Google Cloud Batch jobs | +| [login\_node\_name](#output\_login\_node\_name) | Name of the created VM | + diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/main.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/main.tf new file mode 100644 index 0000000000..6f539af122 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/main.tf @@ -0,0 +1,127 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "batch-login-node", ghpc_role = "scheduler" }) +} + +data "google_compute_instance_template" "batch_instance_template" { + name = var.instance_template +} + +locals { + job_template_runners = [for job in var.job_data : { + content = job.template_contents + destination = "${var.batch_job_directory}/${job.filename}" + type = "data" + }] + + instance_template_metadata = data.google_compute_instance_template.batch_instance_template.metadata + startup_metadata = { startup-script = module.login_startup_script.startup_script } + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + login_metadata = merge(local.instance_template_metadata, local.startup_metadata, local.oslogin_metadata) + + batch_command_instructions = join("\n", [for job in var.job_data : <<-EOT + ## For job: ${job.id} ## + + Submit your job from login node: + gcloud ${var.gcloud_version} batch jobs submit ${job.id} --config=${var.batch_job_directory}/${job.filename} --location=${var.region} --project=${var.project_id} + + Check status: + gcloud ${var.gcloud_version} batch jobs describe ${job.id} --location=${var.region} --project=${var.project_id} | grep state: + + Delete job: + gcloud ${var.gcloud_version} batch jobs delete ${job.id} --location=${var.region} --project=${var.project_id} + + EOT + ]) + + list_all_jobs = <<-EOT + List all jobs: + gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} + EOT + + readme_contents = <<-EOT + # Batch Job Templates + + This folder contains Batch job templates created by the Cluster Toolkit. + These templates can be edited before submitting to Batch to capture more + complex workloads. + + Use the following commands to: + ${local.list_all_jobs} + + ${local.batch_command_instructions} + EOT + + # Construct startup script for network storage + storage_client_install_runners = [ + for i, ns in var.network_storage : merge(ns.client_install_runner, { + destination = "${i}-${ns.client_install_runner.destination}" + }) if ns.client_install_runner != null + ] + mount_runners = [ + for i, ns in var.network_storage : merge(ns.mount_runner, { + destination = "${i}-${ns.mount_runner.destination}" + }) if ns.mount_runner != null + ] + + startup_script_runner = { + content = var.startup_script != null ? var.startup_script : "echo 'Batch job template had no startup script'" + destination = "passed_startup_script.sh" + type = "shell" + } +} + +module "login_startup_script" { + source = "../../scripts/startup-script" + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = concat( + local.storage_client_install_runners, + local.mount_runners, + [local.startup_script_runner], + local.job_template_runners, + [ + { + content = local.readme_contents + destination = "${var.batch_job_directory}/README.md" + type = "data" + } + ] + ) +} + +resource "google_compute_instance_from_template" "batch_login" { + name = "${var.deployment_name}-batch-login" + source_instance_template = var.instance_template + project = var.project_id + zone = var.zone + metadata = local.login_metadata + + service_account { + scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml new file mode 100644 index 0000000000..9af2319b4a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - batch.googleapis.com + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/outputs.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/outputs.tf new file mode 100644 index 0000000000..ea8eccf8d5 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/outputs.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "login_node_name" { + description = "Name of the created VM" + value = google_compute_instance_from_template.batch_login.name +} + +output "instructions" { + description = "Instructions for accessing the login node and submitting Google Cloud Batch jobs" + value = <<-EOT + + Batch job template files will be placed on the Batch login node in the following directory: + ${var.batch_job_directory} + + Use the following commands to: + SSH into the login node: + gcloud compute ssh --zone ${google_compute_instance_from_template.batch_login.zone} ${google_compute_instance_from_template.batch_login.name} --project ${google_compute_instance_from_template.batch_login.project} + + ${local.list_all_jobs} + + ${local.batch_command_instructions} + EOT +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/variables.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/variables.tf new file mode 100644 index 0000000000..3b9caa7001 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/variables.tf @@ -0,0 +1,151 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "deployment_name" { + description = "Name of the deployment, also used for the job_id" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "region" { + description = "The region in which to create the login node" + type = string +} + +variable "zone" { + description = "The zone in which to create the login node" + type = string +} + +variable "labels" { + description = "Labels to add to the login node. Key-value pairs" + type = map(string) +} + +variable "instance_template" { + description = <<-EOT + Login VM instance template self-link. Typically supplied by a + batch-job-template module. If multiple batch-job-template modules supply the + instance_template, the first will be used. + EOT + type = string +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module." + type = list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "startup_script" { + description = "Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module." + type = string + default = null +} + +variable "job_data" { + description = "List of jobs and supporting data for each, typically provided via \"use\" from the batch-job-template module." + type = list(object({ + template_contents = string, + filename = string, + id = string + })) + validation { + condition = length(distinct([for job in var.job_data : job.filename])) == length(var.job_data) + error_message = "All filenames in var.job_data must be unique." + } + validation { + condition = length(distinct([for job in var.job_data : job.id])) == length(var.job_data) + error_message = "All job IDs in var.job_data must be unique." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_template_contents" { + description = "Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module." + type = string + default = null + validation { + condition = var.job_template_contents == null + error_message = "job_template_contents is deprecated. Please use `job_data` instead." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_filename" { + description = "Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module." + type = string + default = null + validation { + condition = var.job_filename == null + error_message = "job_filename is deprecated. Please use `job_data` instead." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_id" { + description = "Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions." + type = string + default = null + validation { + condition = var.job_id == null + error_message = "job_id is deprecated. Please use `job_data` instead." + } +} + +variable "gcloud_version" { + description = <<-EOT + The version of the gcloud cli being used. Used for output instructions. + Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default + version). Typically supplied by a batch-job-template module. If multiple + batch-job-template modules supply the gcloud_version, only the first will be used. + EOT + type = string + default = "" + + validation { + condition = contains(["alpha", "beta", ""], var.gcloud_version) + error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." + } +} + +variable "batch_job_directory" { + description = "The path of the directory on the login node in which to place the Google Cloud Batch job template" + type = string + default = "/home/batch-jobs" +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/versions.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/versions.tf new file mode 100644 index 0000000000..15337a1d7b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/README.md b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/README.md new file mode 100644 index 0000000000..dd4f7fdaa7 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/README.md @@ -0,0 +1,220 @@ +## Description + +This module creates a Google Kubernetes Engine +([GKE](https://cloud.google.com/kubernetes-engine)) cluster. + +### Example + +The following example creates a GKE cluster and a VPC designed to work with GKE. +See [VPC Network](#vpc-network) section for more information about network +requirements. + +```yaml + - id: network1 + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### VPC Network + +This module is configured to create a +[VPC-native cluster](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips). +This means that alias IPs are used and that the subnetwork requires secondary +ranges for pods and services. In the example shown above these secondary ranges +are created in the VPC module. By default the `gke-cluster` module will look for +ranges with the names `pods` and `services`. These names can be configured using +the `pods_ip_range_name` and `services_ip_range_name` settings. + +### Multi-networking + +To [enable Multi-networking](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment), pass multivpc module to gke-cluster module as described in example below. Passing a multivpc module enables multi networking and [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en) on the cluster. + +```yaml + - id: network + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: gke-cluster + source: modules/scheduler/gke-cluster + use: [network, multinetwork] ## enables multi networking and Dataplane V2 on cluster + settings: + cluster_name: $(vars.deployment_name) +``` + +Find an example of multi networking in GKE [here](../../../examples/gke-a3-megagpu.yaml). + +### Cluster Limitations + +The current implementations has the following limitations: + +- Autopilot is disabled +- Auto-provisioning of new node pools is disabled +- Network policies are not supported +- General addon configuration is not supported +- Only regional cluster is supported + +### GKE Inference Gateway + +Setting `enable_inference_gateway` to `true` will enable the `HttpLoadBalancing` +addon and deploy the Inference Gateway CRDs. This feature requires a subnet with +`purpose` set to `REGIONAL_MANAGED_PROXY` in the VPC. For more information, see +the [GKE Inference Gateway documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/serve-with-gke-inference-gateway). + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 7.2 | +| [google-beta](#requirement\_google-beta) | >= 7.2 | +| [kubernetes](#requirement\_kubernetes) | >= 2.36 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 7.2 | +| [google-beta](#provider\_google-beta) | >= 7.2 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | +| [workload\_identity](#module\_workload\_identity) | terraform-google-modules/kubernetes-engine/google//modules/workload-identity | >= 40.0 | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_cluster) | resource | +| [google-beta_google_container_node_pool.system_node_pools](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | +| [google-beta_google_container_engine_versions.version_prefix_filter](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/data-sources/google_container_engine_versions) | data source | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks enables multi networking and creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [authenticator\_security\_group](#input\_authenticator\_security\_group) | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| [autoscaling\_profile](#input\_autoscaling\_profile) | (Beta) Optimize for utilization or availability when deciding to remove nodes. Can be BALANCED or OPTIMIZE\_UTILIZATION. | `string` | `"OPTIMIZE_UTILIZATION"` | no | +| [cloud\_dns\_config](#input\_cloud\_dns\_config) | Configuration for Using Cloud DNS for GKE.

additive\_vpc\_scope\_dns\_domain: This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = "CLOUD\_DNS" and cluster\_dns\_scope = "CLUSTER\_SCOPE" must both be set as well.
cluster\_dns: Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS.
cluster\_dns\_scope: The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE.
cluster\_dns\_domain: The suffix used for all cluster service records. | 
object({
    additive_vpc_scope_dns_domain = optional(string)
    cluster_dns                   = optional(string, "PROVIDER_UNSPECIFIED")
    cluster_dns_scope             = optional(string, "DNS_SCOPE_UNSPECIFIED")
    cluster_dns_domain            = optional(string)
  })
| 
{
  "additive_vpc_scope_dns_domain": null,
  "cluster_dns": "PROVIDER_UNSPECIFIED",
  "cluster_dns_domain": null,
  "cluster_dns_scope": "DNS_SCOPE_UNSPECIFIED"
}
| no | +| [cluster\_availability\_type](#input\_cluster\_availability\_type) | Type of cluster availability. Possible values are: {REGIONAL, ZONAL} | `string` | `"REGIONAL"` | no | +| [cluster\_reference\_type](#input\_cluster\_reference\_type) | How the google\_container\_node\_pool.system\_node\_pools refers to the cluster. Possible values are: {SELF\_LINK, NAME} | `string` | `"SELF_LINK"` | no | +| [configure\_workload\_identity\_sa](#input\_configure\_workload\_identity\_sa) | When true, a kubernetes service account will be created and bound using workload identity to the service account used to create the cluster. | `bool` | `false` | no | +| [default\_max\_pods\_per\_node](#input\_default\_max\_pods\_per\_node) | The default maximum number of pods per node in this cluster. | `number` | `null` | no | +| [deletion\_protection](#input\_deletion\_protection) | "Determines if the cluster can be deleted by gcluster commands or not".
To delete a cluster provisioned with deletion\_protection set to true, you must first set it to false and apply the changes.
Then proceed with deletion as usual. | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`. | `string` | n/a | yes | +| [enable\_dataplane\_v2](#input\_enable\_dataplane\_v2) | Enables [Dataplane v2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2). This setting is immutable on clusters. If null, will default to false unless using multi-networking, in which case it will default to true | `bool` | `null` | no | +| [enable\_dcgm\_monitoring](#input\_enable\_dcgm\_monitoring) | Enable GKE to collect DCGM metrics | `bool` | `false` | no | +| [enable\_external\_dns\_endpoint](#input\_enable\_external\_dns\_endpoint) | Allow [DNS-based approach](https://cloud.google.com/kubernetes-engine/docs/concepts/network-isolation#dns-based_endpoint) for accessing the GKE control plane.
Refer this [dedicated blog](https://cloud.google.com/blog/products/containers-kubernetes/new-dns-based-endpoint-for-the-gke-control-plane) for more details. | `bool` | `false` | no | +| [enable\_filestore\_csi](#input\_enable\_filestore\_csi) | The status of the Filestore Container Storage Interface (CSI) driver addon, which allows the usage of filestore instance as volumes. | `bool` | `false` | no | +| [enable\_gcsfuse\_csi](#input\_enable\_gcsfuse\_csi) | The status of the GCSFuse Container Storage Interface (CSI) driver addon, which allows the usage of a GCS bucket as volumes. | `bool` | `false` | no | +| [enable\_inference\_gateway](#input\_enable\_inference\_gateway) | If true, enables GKE features required for Inference Gateway, including the HttpLoadBalancing addon, and installs required CRDs. | `bool` | `false` | no | +| [enable\_k8s\_beta\_apis](#input\_enable\_k8s\_beta\_apis) | List of Enabled Kubernetes Beta APIs. | `list(string)` | `null` | no | +| [enable\_managed\_lustre\_csi](#input\_enable\_managed\_lustre\_csi) | The status of the Google Compute Engine Managed Lustre Container Storage Interface (CSI) driver addon, which allows the usage of a lustre as volumes. | `bool` | `false` | no | +| [enable\_master\_global\_access](#input\_enable\_master\_global\_access) | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `false` | no | +| [enable\_multi\_networking](#input\_enable\_multi\_networking) | Enables [multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) (Requires GKE Enterprise). This setting is immutable on clusters and enables [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en). If null, will determine state based on if additional\_networks are passed in. | `bool` | `null` | no | +| [enable\_node\_local\_dns\_cache](#input\_enable\_node\_local\_dns\_cache) | Enable GKE NodeLocal DNSCache addon to improve DNS lookup latency | `bool` | `false` | no | +| [enable\_parallelstore\_csi](#input\_enable\_parallelstore\_csi) | The status of the Google Compute Engine Parallelstore Container Storage Interface (CSI) driver addon, which allows the usage of a parallelstore as volumes. | `bool` | `false` | no | +| [enable\_persistent\_disk\_csi](#input\_enable\_persistent\_disk\_csi) | The status of the Google Compute Engine Persistent Disk Container Storage Interface (CSI) driver addon, which allows the usage of a PD as volumes. | `bool` | `true` | no | +| [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | (Beta) Whether the master's internal IP address is used as the cluster endpoint. | `bool` | `true` | no | +| [enable\_private\_ipv6\_google\_access](#input\_enable\_private\_ipv6\_google\_access) | The private IPv6 google access type for the VMs in this subnet. | `bool` | `true` | no | +| [enable\_private\_nodes](#input\_enable\_private\_nodes) | (Beta) Whether nodes have internal IP addresses only. | `bool` | `true` | no | +| [enable\_ray\_operator](#input\_enable\_ray\_operator) | The status of the Ray operator addon, This feature enables Kubernetes APIs for managing and scaling Ray clusters and jobs. You control and are responsible for managing ray.io custom resources in your cluster. This feature is not compatible with GKE clusters that already have another Ray operator installed. Supports clusters on Kubernetes version 1.29.8-gke.1054000 or later. | `bool` | `false` | no | +| [gcp\_public\_cidrs\_access\_enabled](#input\_gcp\_public\_cidrs\_access\_enabled) | Whether the cluster master is accessible via all the Google Compute Engine Public IPs. To view this list of IP addresses look here https://cloud.google.com/compute/docs/faq#find_ip_range | `bool` | `false` | no | +| [k8s\_network\_names](#input\_k8s\_network\_names) | Kubernetes network names details for GKE. If starting index is not specified for gvnic or rdma, it would be set to the default values. | 
object({
    gvnic_prefix      = optional(string, "")
    gvnic_start_index = optional(number, 1)
    gvnic_postfix     = optional(string, "")
    rdma_prefix       = optional(string, "")
    rdma_start_index  = optional(number, 0)
    rdma_postfix      = optional(string, "")
  })
| 
{
  "gvnic_postfix": "",
  "gvnic_prefix": "gvnic-",
  "gvnic_start_index": 1,
  "rdma_postfix": "",
  "rdma_prefix": "rdma-",
  "rdma_start_index": 0
}
| no | +| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account name to use with the gke cluster | `string` | `"workload-identity-k8s-sa"` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [maintenance\_exclusions](#input\_maintenance\_exclusions) | List of maintenance exclusions. A cluster can have up to three. | 
list(object({
    name            = string
    start_time      = string
    end_time        = string
    exclusion_scope = string
  }))
| `[]` | no | +| [maintenance\_start\_time](#input\_maintenance\_start\_time) | Start time for daily maintenance operations. Specified in GMT with `HH:MM` format. | `string` | `"09:00"` | no | +| [master\_authorized\_networks](#input\_master\_authorized\_networks) | External network that can access Kubernetes master through HTTPS. Must be specified in CIDR notation. | 
list(object({
    cidr_block   = string
    display_name = string
  }))
| `[]` | no | +| [master\_ipv4\_cidr\_block](#input\_master\_ipv4\_cidr\_block) | (Beta) The IP range in CIDR notation to use for the hosted master network. | `string` | `"172.16.0.32/28"` | no | +| [min\_master\_version](#input\_min\_master\_version) | The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release. | `string` | `null` | no | +| [name\_suffix](#input\_name\_suffix) | Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`. | `string` | `""` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`. | `string` | n/a | yes | +| [networking\_mode](#input\_networking\_mode) | Determines whether alias IPs or routes will be used for pod IPs in the cluster. Options are VPC\_NATIVE or ROUTES. VPC\_NATIVE enables IP aliasing. The default is VPC\_NATIVE. | `string` | `"VPC_NATIVE"` | no | +| [pods\_ip\_range\_name](#input\_pods\_ip\_range\_name) | The name of the secondary subnet ip range to use for pods. | `string` | `"pods"` | no | +| [prefix\_with\_deployment\_name](#input\_prefix\_with\_deployment\_name) | If true, cluster name will be prefixed by `deployment_name` (ex: -). | `bool` | `true` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [region](#input\_region) | The region to host the cluster in. | `string` | n/a | yes | +| [release\_channel](#input\_release\_channel) | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. | `string` | `"UNSPECIFIED"` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the system node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the system node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [services\_ip\_range\_name](#input\_services\_ip\_range\_name) | The name of the secondary subnet range to use for services. | `string` | `"services"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to host the cluster in. | `string` | n/a | yes | +| [system\_node\_pool\_disk\_size\_gb](#input\_system\_node\_pool\_disk\_size\_gb) | Size of disk for each node of the system node pool. | `number` | `100` | no | +| [system\_node\_pool\_disk\_type](#input\_system\_node\_pool\_disk\_type) | Disk type for each node of the system node pool. | `string` | `null` | no | +| [system\_node\_pool\_enable\_secure\_boot](#input\_system\_node\_pool\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | +| [system\_node\_pool\_enabled](#input\_system\_node\_pool\_enabled) | Create a system node pool. | `bool` | `true` | no | +| [system\_node\_pool\_image\_type](#input\_system\_node\_pool\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | +| [system\_node\_pool\_kubernetes\_labels](#input\_system\_node\_pool\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | +| [system\_node\_pool\_machine\_type](#input\_system\_node\_pool\_machine\_type) | Machine type for the system node pool. | `string` | `"e2-standard-4"` | no | +| [system\_node\_pool\_name](#input\_system\_node\_pool\_name) | Name of the system node pool. | `string` | `"system"` | no | +| [system\_node\_pool\_node\_count](#input\_system\_node\_pool\_node\_count) | The total min and max nodes to be maintained in the system node pool. | 
object({
    total_min_nodes = number
    total_max_nodes = number
  })
| 
{
  "total_max_nodes": 10,
  "total_min_nodes": 2
}
| no | +| [system\_node\_pool\_taints](#input\_system\_node\_pool\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "components.gke.io/gke-managed-components",
    "value": true
  }
]
| no | +| [system\_node\_pool\_zones](#input\_system\_node\_pool\_zones) | The zones to use for the system node pool. If not specified, the cluster default node zone(s) will be used. | `list(string)` | `null` | no | +| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | +| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | +| [upgrade\_settings](#input\_upgrade\_settings) | Defines gke cluster upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | +| [version\_prefix](#input\_version\_prefix) | If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally. | `string` | `"1.31."` | no | +| [zone](#input\_zone) | Zone for a zonal cluster. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | An identifier for the resource with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | +| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | +| [gke\_version](#output\_gke\_version) | GKE cluster's version. | +| [instructions](#output\_instructions) | Instructions on how to connect to the created cluster. | +| [k8s\_service\_account\_name](#output\_k8s\_service\_account\_name) | Name of k8s service account. | + diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/main.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/main.tf new file mode 100644 index 0000000000..6106f8d90f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/main.tf @@ -0,0 +1,470 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-cluster", ghpc_role = "scheduler" }) +} + +locals { + upgrade_settings = { + strategy = var.upgrade_settings.strategy + max_surge = coalesce(var.upgrade_settings.max_surge, 0) + max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) + } +} + +locals { + dash = var.prefix_with_deployment_name && var.name_suffix != "" ? "-" : "" + prefix = var.prefix_with_deployment_name ? var.deployment_name : "" + name_maybe_empty = "${local.prefix}${local.dash}${var.name_suffix}" + name = local.name_maybe_empty != "" ? local.name_maybe_empty : "NO-NAME-GIVEN" + + cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ + security_group = var.authenticator_security_group + }] + + default_sa_email = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" + sa_email = coalesce(var.service_account_email, local.default_sa_email) + + # additional VPCs enable multi networking + derived_enable_multi_networking = coalesce(var.enable_multi_networking, length(var.additional_networks) > 0) + + # multi networking needs enabled Dataplane v2 + derived_enable_dataplane_v2 = coalesce(var.enable_dataplane_v2, local.derived_enable_multi_networking) + + default_monitoring_component = [ + "SYSTEM_COMPONENTS", + "POD", + "DAEMONSET", + "DEPLOYMENT", + "STATEFULSET", + "STORAGE", + "HPA", + "CADVISOR", + "KUBELET" + ] + + default_logging_component = [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] +} + +data "google_project" "project" { + project_id = var.project_id +} + +data "google_container_engine_versions" "version_prefix_filter" { + provider = google-beta + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + version_prefix = var.version_prefix +} + +locals { + master_version = var.min_master_version != null ? var.min_master_version : data.google_container_engine_versions.version_prefix_filter.latest_master_version +} + +resource "google_container_cluster" "gke_cluster" { + provider = google-beta + + project = var.project_id + name = local.name + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + resource_labels = local.labels + networking_mode = var.networking_mode + # decouple node pool lifecycle from cluster life cycle + remove_default_node_pool = true + initial_node_count = 1 # must be set when remove_default_node_pool is set + node_locations = var.system_node_pool_zones + + deletion_protection = var.deletion_protection + + dynamic "enable_k8s_beta_apis" { + for_each = var.enable_k8s_beta_apis != null ? [1] : [] + content { + enabled_apis = var.enable_k8s_beta_apis + } + } + + network = var.network_id + subnetwork = var.subnetwork_self_link + + # Note: the existence of the "master_authorized_networks_config" block enables + # the master authorized networks even if it's empty. + master_authorized_networks_config { + dynamic "cidr_blocks" { + for_each = var.master_authorized_networks + content { + cidr_block = cidr_blocks.value.cidr_block + display_name = cidr_blocks.value.display_name + } + } + gcp_public_cidrs_access_enabled = var.gcp_public_cidrs_access_enabled + } + + private_ipv6_google_access = var.enable_private_ipv6_google_access ? "PRIVATE_IPV6_GOOGLE_ACCESS_TO_GOOGLE" : null + default_max_pods_per_node = var.default_max_pods_per_node + master_auth { + client_certificate_config { + issue_client_certificate = false + } + } + + enable_shielded_nodes = true + + cluster_autoscaling { + # Controls auto provisioning of node-pools + enabled = false + + # Controls autoscaling algorithm of node-pools + autoscaling_profile = var.autoscaling_profile + } + + datapath_provider = local.derived_enable_dataplane_v2 ? "ADVANCED_DATAPATH" : "LEGACY_DATAPATH" + + enable_multi_networking = local.derived_enable_multi_networking + + network_policy { + # Enabling NetworkPolicy for clusters with DatapathProvider=ADVANCED_DATAPATH + # is not allowed. Dataplane V2 will take care of network policy enforcement + # instead. + enabled = false + # GKE Dataplane V2 support. This must be set to PROVIDER_UNSPECIFIED in + # order to let the datapath_provider take effect. + # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/656#issuecomment-720398658 + provider = "PROVIDER_UNSPECIFIED" + } + + private_cluster_config { + enable_private_nodes = var.enable_private_nodes + enable_private_endpoint = var.enable_private_endpoint + master_ipv4_cidr_block = var.master_ipv4_cidr_block + master_global_access_config { + enabled = var.enable_master_global_access + } + } + + ip_allocation_policy { + cluster_secondary_range_name = var.pods_ip_range_name + services_secondary_range_name = var.services_ip_range_name + } + + workload_identity_config { + workload_pool = "${var.project_id}.svc.id.goog" + } + + dynamic "gateway_api_config" { + for_each = var.enable_inference_gateway ? [1] : [] + content { + channel = "CHANNEL_STANDARD" + } + } + + dynamic "authenticator_groups_config" { + for_each = local.cluster_authenticator_security_group + content { + security_group = authenticator_groups_config.value.security_group + } + } + + release_channel { + channel = var.release_channel + } + min_master_version = local.master_version + + maintenance_policy { + daily_maintenance_window { + start_time = var.maintenance_start_time + } + + dynamic "maintenance_exclusion" { + for_each = var.maintenance_exclusions + content { + exclusion_name = maintenance_exclusion.value.name + start_time = maintenance_exclusion.value.start_time + end_time = maintenance_exclusion.value.end_time + exclusion_options { + scope = maintenance_exclusion.value.exclusion_scope + } + } + } + } + + dynamic "dns_config" { + for_each = var.cloud_dns_config != null ? [1] : [] + content { + additive_vpc_scope_dns_domain = var.cloud_dns_config.additive_vpc_scope_dns_domain + cluster_dns = var.cloud_dns_config.cluster_dns + cluster_dns_scope = var.cloud_dns_config.cluster_dns_scope + cluster_dns_domain = var.cloud_dns_config.cluster_dns_domain + } + } + + addons_config { + gcp_filestore_csi_driver_config { + enabled = var.enable_filestore_csi + } + gcs_fuse_csi_driver_config { + enabled = var.enable_gcsfuse_csi + } + gce_persistent_disk_csi_driver_config { + enabled = var.enable_persistent_disk_csi + } + dns_cache_config { + enabled = var.enable_node_local_dns_cache + } + parallelstore_csi_driver_config { + enabled = var.enable_parallelstore_csi + } + ray_operator_config { + enabled = var.enable_ray_operator + } + lustre_csi_driver_config { + enabled = var.enable_managed_lustre_csi + } + dynamic "http_load_balancing" { + for_each = var.enable_inference_gateway ? [1] : [] + content { + disabled = false + } + } + } + + timeouts { + create = var.timeout_create + update = var.timeout_update + } + + node_config { + shielded_instance_config { + enable_secure_boot = var.system_node_pool_enable_secure_boot + enable_integrity_monitoring = true + } + } + + control_plane_endpoints_config { + dns_endpoint_config { + allow_external_traffic = var.enable_external_dns_endpoint + } + } + + lifecycle { + # Ignore all changes to the default node pool. It's being removed after creation. + ignore_changes = [ + node_config, + min_master_version, + ] + precondition { + condition = var.default_max_pods_per_node == null || var.networking_mode == "VPC_NATIVE" + error_message = "default_max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." + } + precondition { + condition = coalesce(var.enable_dataplane_v2, true) || !local.derived_enable_multi_networking + error_message = "'enable_dataplane_v2' cannot be false when enabling multi networking." + } + precondition { + condition = coalesce(var.enable_multi_networking, true) || length(var.additional_networks) == 0 + error_message = "'enable_multi_networking' cannot be false when using multivpc module, which passes additional_networks." + } + } + + monitoring_config { + enable_components = var.enable_dcgm_monitoring ? concat(local.default_monitoring_component, ["DCGM"]) : local.default_monitoring_component + managed_prometheus { + enabled = true + } + } + + logging_config { + enable_components = local.default_logging_component + } +} + +# We define explicit node pools, so that it can be modified without +# having to destroy the entire cluster. +resource "google_container_node_pool" "system_node_pools" { + provider = google-beta + count = var.system_node_pool_enabled ? 1 : 0 + + project = var.project_id + name = var.system_node_pool_name + cluster = var.cluster_reference_type == "NAME" ? google_container_cluster.gke_cluster.name : google_container_cluster.gke_cluster.self_link + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + node_locations = var.system_node_pool_zones + version = local.master_version + + autoscaling { + total_min_node_count = var.system_node_pool_node_count.total_min_nodes + total_max_node_count = var.system_node_pool_node_count.total_max_nodes + } + + upgrade_settings { + strategy = local.upgrade_settings.strategy + max_surge = local.upgrade_settings.max_surge + max_unavailable = local.upgrade_settings.max_unavailable + } + + management { + auto_repair = true + auto_upgrade = true + } + + node_config { + labels = var.system_node_pool_kubernetes_labels + resource_labels = local.labels + service_account = var.service_account_email + oauth_scopes = var.service_account_scopes + machine_type = var.system_node_pool_machine_type + disk_size_gb = var.system_node_pool_disk_size_gb + disk_type = var.system_node_pool_disk_type + + dynamic "taint" { + for_each = var.system_node_pool_taints + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } + + # Forcing the use of the Container-optimized image, as it is the only + # image with the proper logging daemon installed. + # + # cos images use Shielded VMs since v1.13.6-gke.0. + # https://cloud.google.com/kubernetes-engine/docs/how-to/node-images + # + # We use COS_CONTAINERD to be compatible with (optional) gVisor. + # https://cloud.google.com/kubernetes-engine/docs/how-to/sandbox-pods + image_type = var.system_node_pool_image_type + + shielded_instance_config { + enable_secure_boot = var.system_node_pool_enable_secure_boot + enable_integrity_monitoring = true + } + + gvnic { + enabled = var.system_node_pool_image_type == "COS_CONTAINERD" + } + + # Implied by Workload Identity + workload_metadata_config { + mode = "GKE_METADATA" + } + # Implied by workload identity. + metadata = { + "disable-legacy-endpoints" = "true" + } + } + + lifecycle { + ignore_changes = [ + node_config[0].labels, + node_config[0].taint, + version, + ] + precondition { + condition = contains(["SURGE"], local.upgrade_settings.strategy) + error_message = "Only SURGE strategy is supported" + } + precondition { + condition = local.upgrade_settings.max_unavailable >= 0 + error_message = "max_unavailable should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_surge >= 0 + error_message = "max_surge should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 + error_message = "At least one of max_unavailable or max_surge must greater than 0" + } + } +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${google_container_cluster.gke_cluster.endpoint}" + cluster_ca_certificate = base64decode(google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) + token = data.google_client_config.default.access_token +} + +module "workload_identity" { + count = var.configure_workload_identity_sa ? 1 : 0 + source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" + version = ">= 40.0" + + use_existing_gcp_sa = true + name = var.k8s_service_account_name + gcp_sa_name = local.sa_email + project_id = var.project_id + + # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1059 + depends_on = [ + data.google_project.project, + google_container_cluster.gke_cluster + ] +} + +locals { + k8s_service_account_name = one(module.workload_identity[*].k8s_service_account_name) +} + +locals { + # Separate gvnic and rdma networks and assign indexes + gvnic_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "GVNIC")] : + merge(net, { name = "${var.k8s_network_names.gvnic_prefix}${idx + var.k8s_network_names.gvnic_start_index}${var.k8s_network_names.gvnic_postfix}" }) + ] + + rdma_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "RDMA")] : + merge(net, { name = "${var.k8s_network_names.rdma_prefix}${idx + var.k8s_network_names.rdma_start_index}${var.k8s_network_names.rdma_postfix}" }) + ] + + all_networks = concat(local.gvnic_networks, local.rdma_networks) +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = google_container_cluster.gke_cluster.id + project_id = var.project_id + + apply_manifests = concat(flatten([ + for idx, network_info in local.all_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = network_info.name, + network_name = network_info.network + subnetwork_name = network_info.subnetwork, + device_mode = strcontains(upper(network_info.nic_type), "RDMA") ? "RDMA" : "NetDevice" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = network_info.name } + } + ] + ]), + var.enable_inference_gateway ? [ + { + source = "https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.0.0/manifests.yaml", + template_vars = {} + } + ] : [] + ) +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml new file mode 100644 index 0000000000..bd1517ce8f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/outputs.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/outputs.tf new file mode 100644 index 0000000000..3326a5468e --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/outputs.tf @@ -0,0 +1,104 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_id" { + description = "An identifier for the resource with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." + value = google_container_cluster.gke_cluster.id +} + +output "gke_cluster_exists" { + description = "A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations." + value = true + depends_on = [ + google_container_cluster.gke_cluster + ] +} + +locals { + private_endpoint_message = trimspace( + <<-EOT + This cluster was created with 'enable_private_endpoint: true'. + It cannot be accessed from a public IP addresses. + One way to access this cluster is from a VM created in the GKE cluster subnet. + EOT + ) + master_authorized_networks_message = length(var.master_authorized_networks) == 0 ? "" : trimspace( + <<-EOT + The following networks have been authorized to access this cluster: + ${join("\n", [for x in var.master_authorized_networks : " ${x.display_name}: ${x.cidr_block}"])}" + EOT + ) + public_endpoint_message = trimspace( + <<-EOT + To add authorized networks you can allowlist your IP with this command: + gcloud container clusters update ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} \ + --enable-master-authorized-networks \ + --master-authorized-networks /32 + EOT + ) + allowlist_your_ip_message = var.enable_private_endpoint ? local.private_endpoint_message : local.public_endpoint_message + kubernetes_service_account_message = local.k8s_service_account_name == null ? "" : trimspace( + <<-EOT + Use the following Kubernetes Service Account in the default namespace to run your workloads: + ${local.k8s_service_account_name} + The GCP Service Account mapped to this Kubernetes Service Account is: + ${local.sa_email} + EOT + ) + kubernetes_cluster_fetch_credential_message = var.enable_external_dns_endpoint ? trimspace( + <<-EOT + Use the following command to fetch credentials for the created cluster: + gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} \ + --dns-endpoint + EOT + ) : trimspace( + <<-EOT + Use the following command to fetch credentials for the created cluster: + gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} + EOT + ) +} + +output "instructions" { + description = "Instructions on how to connect to the created cluster." + value = trimspace( + <<-EOT + ${local.master_authorized_networks_message} + + ${local.allowlist_your_ip_message} + + ${local.kubernetes_cluster_fetch_credential_message} + + ${local.kubernetes_service_account_message} + EOT + ) +} + +output "k8s_service_account_name" { + description = "Name of k8s service account." + value = local.k8s_service_account_name +} + +output "gke_version" { + description = "GKE cluster's version." + value = google_container_cluster.gke_cluster.master_version +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl new file mode 100644 index 0000000000..d376a1a760 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl @@ -0,0 +1,9 @@ +--- +apiVersion: networking.gke.io/v1 +kind: GKENetworkParamSet +metadata: + name: ${name} +spec: + vpc: ${network_name} + vpcSubnet: ${subnetwork_name} + deviceMode: ${device_mode} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl new file mode 100644 index 0000000000..1571a92692 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl @@ -0,0 +1,11 @@ +--- +apiVersion: networking.gke.io/v1 +kind: Network +metadata: + name: ${name} +spec: + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: ${name} + type: Device diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/variables.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/variables.tf new file mode 100644 index 0000000000..8d863b1730 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/variables.tf @@ -0,0 +1,533 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "name_suffix" { + description = "Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`." + type = string + default = "" +} + +variable "deployment_name" { + description = "Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`." + type = string +} + +variable "prefix_with_deployment_name" { + description = "If true, cluster name will be prefixed by `deployment_name` (ex: -)." + type = bool + default = true +} + +variable "region" { + description = "The region to host the cluster in." + type = string +} + +variable "zone" { + description = "Zone for a zonal cluster." + default = null + type = string +} + +variable "network_id" { + description = "The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`." + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to host the cluster in." + type = string +} + +variable "pods_ip_range_name" { + description = "The name of the secondary subnet ip range to use for pods." + type = string + default = "pods" +} + +variable "services_ip_range_name" { + description = "The name of the secondary subnet range to use for services." + type = string + default = "services" +} + +variable "enable_private_ipv6_google_access" { + description = "The private IPv6 google access type for the VMs in this subnet." + type = bool + default = true +} + +variable "release_channel" { + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`." + type = string + default = "UNSPECIFIED" +} + +variable "min_master_version" { + description = "The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release." + type = string + default = null +} + +variable "version_prefix" { + description = "If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally." + type = string + default = "1.31." +} + +variable "maintenance_start_time" { + description = "Start time for daily maintenance operations. Specified in GMT with `HH:MM` format." + type = string + default = "09:00" +} + +variable "maintenance_exclusions" { + description = "List of maintenance exclusions. A cluster can have up to three." + type = list(object({ + name = string + start_time = string + end_time = string + exclusion_scope = string + })) + default = [] + validation { + condition = alltrue([ + for x in var.maintenance_exclusions : + contains(["NO_UPGRADES", "NO_MINOR_UPGRADES", "NO_MINOR_OR_NODE_UPGRADES"], x.exclusion_scope) + ]) + error_message = "`exclusion_scope` must be set to `NO_UPGRADES` OR `NO_MINOR_UPGRADES` OR `NO_MINOR_OR_NODE_UPGRADES`." + } +} + +variable "cloud_dns_config" { + description = < **_NOTE:_** The `project_id` and `region` settings would be inferred from the +> deployment variables of the same name, but they are included here for clarity. + +### Multi-networking + +To create network objects in GKE cluster, you can pass a multivpc module to a pre-existing-gke-cluster module instead of [applying a manifest manually](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment). + +```yaml + - id: network + source: modules/network/vpc + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: existing-gke-cluster ## multinetworking must be enabled in advance when cluster creation + source: modules/scheduler/pre-existing-gke-cluster + use: [multinetwork] + settings: + cluster_name: $(vars.deployment_name) +``` + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_container_cluster.existing_gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the existing cluster | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project that hosts the existing cluster | `string` | n/a | yes | +| [rdma\_subnetwork\_name\_prefix](#input\_rdma\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | `string` | `null` | no | +| [region](#input\_region) | Region in which to search for the cluster | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | +| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster exists. | +| [gke\_version](#output\_gke\_version) | GKE cluster's version. | + diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf new file mode 100644 index 0000000000..926d2be100 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_container_cluster" "existing_gke_cluster" { + name = var.cluster_name + project = var.project_id + location = var.region +} + +locals { + rdma_networks = [for network_info in var.additional_networks : network_info if strcontains(upper(network_info.nic_type), "RDMA")] + non_rdma_networks = [for network_info in var.additional_networks : network_info if !strcontains(upper(network_info.nic_type), "RDMA")] + apply_manifests_rdma_networks = flatten([ + for idx, network_info in local.rdma_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = "${var.rdma_subnetwork_name_prefix}-${idx}", + network_name = network_info.network + subnetwork_name = "${var.rdma_subnetwork_name_prefix}-${idx}", + device_mode = "RDMA" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = "${var.rdma_subnetwork_name_prefix}-${idx}" } + } + ] + ]) + + apply_manifests_non_rdma_networks = flatten([ + for idx, network_info in local.non_rdma_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = network_info.subnetwork + network_name = network_info.network + subnetwork_name = network_info.subnetwork + device_mode = "NetDevice" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = network_info.subnetwork } + } + ] + ]) +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = data.google_container_cluster.existing_gke_cluster.id + project_id = var.project_id + + apply_manifests = concat(local.apply_manifests_non_rdma_networks, local.apply_manifests_rdma_networks) +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf new file mode 100644 index 0000000000..8884ee30b0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_id" { + description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." + value = data.google_container_cluster.existing_gke_cluster.id +} + +output "gke_cluster_exists" { + description = "A static flag that signals to downstream modules that a cluster exists." + value = true + depends_on = [ + data.google_container_cluster.existing_gke_cluster + ] +} + +output "gke_version" { + description = "GKE cluster's version." + value = data.google_container_cluster.existing_gke_cluster.master_version +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl new file mode 100644 index 0000000000..d376a1a760 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl @@ -0,0 +1,9 @@ +--- +apiVersion: networking.gke.io/v1 +kind: GKENetworkParamSet +metadata: + name: ${name} +spec: + vpc: ${network_name} + vpcSubnet: ${subnetwork_name} + deviceMode: ${device_mode} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl new file mode 100644 index 0000000000..1571a92692 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl @@ -0,0 +1,11 @@ +--- +apiVersion: networking.gke.io/v1 +kind: Network +metadata: + name: ${name} +spec: + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: ${name} + type: Device diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf new file mode 100644 index 0000000000..9e9ed98ed3 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project that hosts the existing cluster" + type = string +} + +variable "cluster_name" { + description = "Name of the existing cluster" + type = string +} + +variable "region" { + description = "Region in which to search for the cluster" + type = string +} + +variable "additional_networks" { + description = "Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster." + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + })) +} + +variable "rdma_subnetwork_name_prefix" { + description = "Prefix of the RDMA subnetwork names" + default = null + type = string +} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf new file mode 100644 index 0000000000..562d8647b1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "> 5.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.74.0" + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/README.md b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/README.md new file mode 100644 index 0000000000..db9094909b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/README.md @@ -0,0 +1,355 @@ +## Description + +This module creates a startup script that will execute a list of runners in the +order they are specified. The runners are copied to a GCS bucket at deployment +time and then copied into the VM as they are executed after startup. + +Each runner receives the following attributes: + +- `destination`: (Required) The name of the file at the destination VM. If an + absolute path is provided, the file will be copied to that path, otherwise + the file will be created in a temporary folder and deleted once the startup + script runs. +- `type`: (Required) The type of the runner, one of the following: + - `shell`: The runner is a shell script and will be executed once copied to + the destination VM. + - `ansible-local`: The runner is an ansible playbook and will run on the VM + with the following command line flags: + + ```shell + ansible-playbook --connection=local --inventory=localhost, \ + --limit localhost <> + ``` + + - `data`: The data or file specified will be copied to `<>`. No + action will be performed after the data is staged. This data can be used by + subsequent runners or simply made available on the VM for later use. +- `content`: (Optional) Content to be uploaded and, if `type` is + either `shell` or `ansible-local`, executed. Must be defined if `source` is + not. +- `source`: (Optional) A path to the file or data you want to upload. Must be + defined if `content` is not. The source path is relative to the deployment + group directory. To ensure correctness of path use `ghpc_stage` function, that + would copy referenced file to the deployment group directory. For example: + + ```yaml + source: $(ghpc_stage("path/to/file")) + ``` + + For more examples with context, see the + [example blueprint snippet](#example). To reference any other source file, an + absolute path must be used. + +- `args`: (Optional) Arguments to be passed to `shell` or `ansible-local` + runners. For `shell` runners, these will be passed as arguments to the script + when it is executed. For `ansible-local` runners, they will be appended to + a list of default arguments that invoke `ansible-playbook` on the localhost. + Therefore`args` should not include any arguments that alter this behavior, + such as `--connection`, `--inventory`, or `--limit`. + +### Runner dependencies + +`ansible-local` runners require Ansible to be installed in the VM before +running. To support other playbook runners in the Cluster Toolkit, we install +version 2.11 of `ansible-core` as well as the larger package of collections +found in `ansible` version 4.10.0. + +If an `ansible-local` runner is found in the list supplied to this module, +a script to install Ansible will be prepended to the list of runners. This +behavior can be disabled by setting `var.prepend_ansible_installer` to `false`. +This script will do the following at VM startup: + +- Install system-wide python3 if not already installed using system package + managers (yum, apt-get, etc) +- Install `python3-distutils` system-wide in debian and ubuntu based + environments. This can be a missing dependency on system installations of + python3 for installing and upgrading pip. +- Install system-wide pip3 if not already installed and upgrade pip3 if the + version is not at least 18.0. +- Install and create a virtual environment located at `/usr/local/ghpc-venv`. +- Install ansible into this virtual environment if the current version of + ansible is not version 2.11 or higher. + +To use the virtual environment created by this script, you can activate it by +running the following command on the VM: + +```shell +source /usr/local/ghpc-venv/bin/activate +``` + +You may also need to provide the correct python interpreter as the python3 +binary in the virtual environment. This can be done by adding the following flag +when calling `ansible-playbook`: + +```shell +-e ansible_python_interpreter=/usr/local/ghpc-venv/bin/activate +``` + +> **_NOTE:_** ansible-playbook and other ansible command line tools will only be +> accessible from the command line (and in your PATH variable) after activating +> this environment. + +### Staging the runners + +Runners will be uploaded to a +[GCS bucket](https://cloud.google.com/storage/docs/creating-buckets). This +bucket will be created by this module and named as +`${var.deployment_name}-startup-scripts-${random_id}`. VMs using the startup +script created by this module will pull the runners content from a GCS bucket +and therefore must have access to GCS. + +> **_NOTE:_** To ensure access to GCS, set the following OAuth scope on the +> instance using the startup scripts: +> `https://www.googleapis.com/auth/devstorage.read_only`. +> +> This is set as a default scope in the [vm-instance], +> [schedMD-slurm-on-gcp-login-node] and [schedMD-slurm-on-gcp-controller] +> modules + +[vm-instance]: ../../compute/vm-instance/README.md +[schedMD-slurm-on-gcp-login-node]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md +[schedMD-slurm-on-gcp-controller]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md + +### Tracking startup script execution + +For more information on how to use startup scripts on Google Cloud Platform, +please refer to +[this document](https://cloud.google.com/compute/docs/instances/startup-scripts/linux). + +To debug startup scripts from a Linux VM created with startup script generated +by this module: + +```shell +sudo DEBUG=1 google_metadata_script_runner startup +``` + +To view outputs from a Linux startup script, run: + +```shell +sudo journalctl -u google-startup-scripts.service +``` + +### Monitoring Agent Installation + +This `startup-script` module has several options for installing a Google +monitoring agent. There are two relevant settings: `install_stackdriver_agent` +and `install_cloud_ops_agent`. + +The _Stackdriver Agent_ also called the _Legacy Cloud Monitoring Agent_ provides +better performance under some HPC workloads. While official documentation +recommends using the _Cloud Ops Agent_, it is recommended to use +`install_stackdriver_agent` when performance is important. + +#### Stackdriver Agent Installation + +If an image or machine already has Cloud Ops Agent installed and you would like +to instead use the Stackdriver Agent, the following script will remove the Cloud +Ops Agent and install the Stackdriver Agent. + +```bash +# Remove Cloud Ops Agent +sudo systemctl stop google-cloud-ops-agent.service +sudo systemctl disable google-cloud-ops-agent.service +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +sudo bash add-google-cloud-ops-agent-repo.sh --uninstall +sudo bash add-google-cloud-ops-agent-repo.sh --remove-repo + +# Install Stackdriver Agent +curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh +sudo bash add-monitoring-agent-repo.sh --also-install +curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh +sudo bash add-logging-agent-repo.sh --also-install +sudo service stackdriver-agent start +sudo service google-fluentd restart +``` + +#### Cloud Ops Agent Installation + +If an image or machine already has the Stackdriver Agent installed and you would +like to instead use the Cloud Ops Agent, the following script will remove the +Stackdriver Agent and install the Cloud Ops Agent. + +```bash +# UnInstall Stackdriver Agent + +sudo systemctl stop stackdriver-agent.service +sudo systemctl disable stackdriver-agent.service +curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh +sudo dpkg --configure -a +sudo bash add-monitoring-agent-repo.sh --uninstall +sudo bash add-monitoring-agent-repo.sh --remove-repo +sudo systemctl stop google-fluentd.service +sudo systemctl disable google-fluentd.service +sudo dpkg --configure -a +curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh +sudo bash add-logging-agent-repo.sh --uninstall +sudo bash add-logging-agent-repo.sh --remove-repo + +# Install ops-agent + +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +sudo bash add-google-cloud-ops-agent-repo.sh --also-install +sudo service google-cloud-ops-agent start +``` + +As a reminder, this should be in a startup script, which should run on all +Compute nodes via the `compute_startup_script` on the controller. + +#### Testing Installation + +You can test if one of the agents is running using the following commands: + +```bash +# For Cloud Ops Agent +$ sudo systemctl is-active google-cloud-ops-agent"*" +active +active +active +active + +# For Legacy Monitoring and Logging Agents +$ sudo service stackdriver-agent status +stackdriver-agent is running [ OK ] +$ sudo service google-fluentd status +google-fluentd is running [ OK ] +``` + +For official documentation see troubleshooting docs: + +- [Cloud Ops Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-install-startup) +- [Legacy Monitoring Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/monitoring/troubleshooting) +- [Legacy Logging Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/logging/troubleshooting) + +### Example + +```yaml +- id: startup + source: modules/scripts/startup-script + settings: + runners: + # Some modules such as filestore have runners as outputs for convenience: + - $(homefs.install_nfs_client_runner) + # These runners can still be created manually: + # - type: shell + # destination: "modules/filestore/scripts/install_nfs_client.sh" + # source: "modules/filestore/scripts/install_nfs_client.sh" + - type: ansible-local + destination: "modules/filestore/scripts/mount.yaml" + source: "modules/filestore/scripts/mount.yaml" + - type: data + source: /tmp/foo.tgz + destination: /tmp/bar.tgz + - type: shell + destination: "decompress.sh" + content: | + #!/bin/sh + echo $2 + tar zxvf /tmp/$1 -C / + args: "bar.tgz 'Expanding file'" + +- id: compute-cluster + source: modules/compute/vm-instance + use: [homefs, startup] +``` + +In the above example, a new GCS bucket is created to upload the startup-scripts. +But in the case where the user wants to reuse existing GCS bucket or folder, +they are able to do so by using the `gcs_bucket_path` as shown in the below example + +```yaml +- id: startup + source: modules/scripts/startup-script + settings: + gcs_bucket_path: gs://user-test-bucket/folder1/folder2 + install_stackdriver_agent: true + +- id: compute-cluster + source: modules/compute/vm-instance + use: [startup] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 6.41 | +| [local](#requirement\_local) | >= 2.0.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.41 | +| [local](#provider\_local) | >= 2.0.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.configs_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | +| [google_storage_bucket_object.scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [local_file.debug_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [ansible\_virtualenv\_path](#input\_ansible\_virtualenv\_path) | Virtual environment path in which to install Ansible | `string` | `"/usr/local/ghpc-venv"` | no | +| [bucket\_viewers](#input\_bucket\_viewers) | Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account) | `list(string)` | `[]` | no | +| [configure\_ssh\_host\_patterns](#input\_configure\_ssh\_host\_patterns) | If specified, it will automate ssh configuration by:
- Defining a Host block for every element of this variable and setting StrictHostKeyChecking to 'No'.
Ex: "hpc*", "hpc01*", "ml*"
- The first time users log-in, it will create ssh keys that are added to the authorized keys list
This requires a shared /home filesystem and relies on specifying the right prefix. | `list(string)` | `[]` | no | +| [debug\_file](#input\_debug\_file) | Path to an optional local to be written with 'startup\_script'. | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used to name GCS bucket for startup scripts. | `string` | n/a | yes | +| [docker](#input\_docker) | Install and configure Docker | 
object({
    enabled        = optional(bool, false)
    world_writable = optional(bool, false)
    daemon_config  = optional(string, "")
  })
| 
{
  "enabled": false
}
| no | +| [enable\_docker\_world\_writable](#input\_enable\_docker\_world\_writable) | DEPRECATED: use var.docker | `bool` | `null` | no | +| [enable\_gpu\_network\_wait\_online](#input\_enable\_gpu\_network\_wait\_online) | Enable a SystemD unit that blocks execution of startup-scripts until after all network interfaces are online. (Works on reboots or boots of an image built using this solution) | `bool` | `false` | no | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | `null` | no | +| [http\_no\_proxy](#input\_http\_no\_proxy) | Domains for which to disable http\_proxy behavior. Honored only if var.http\_proxy is set | `string` | `".google.com,.googleapis.com,metadata.google.internal,localhost,127.0.0.1"` | no | +| [http\_proxy](#input\_http\_proxy) | Web (http and https) proxy configuration for pip, apt, and yum/dnf and interactive shells | `string` | `""` | no | +| [install\_ansible](#input\_install\_ansible) | Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used. | `bool` | `null` | no | +| [install\_cloud\_ops\_agent](#input\_install\_cloud\_ops\_agent) | Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true. | `bool` | `false` | no | +| [install\_cloud\_rdma\_drivers](#input\_install\_cloud\_rdma\_drivers) | If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8. Should not be enabled if using the HPC VM Image. | `bool` | `false` | no | +| [install\_docker](#input\_install\_docker) | DEPRECATED: use var.docker. | `bool` | `null` | no | +| [install\_stackdriver\_agent](#input\_install\_stackdriver\_agent) | Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance. | `bool` | `false` | no | +| [labels](#input\_labels) | Labels for the created GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_filesystem](#input\_local\_ssd\_filesystem) | Create and mount a filesystem from local SSD disks (data will be lost if VMs are powered down without enabling migration); enable by setting mountpoint field to a valid directory path. | 
object({
    fs_type     = optional(string, "ext4")
    mountpoint  = optional(string, "")
    permissions = optional(string, "0755")
  })
| 
{
  "fs_type": "ext4",
  "mountpoint": "",
  "permissions": "0755"
}
| no | +| [managed\_lustre](#input\_managed\_lustre) | Configure Managed Lustre (assumes driver already installed) | 
object({
    enabled = optional(bool, false)
    port    = optional(number, 988)
  })
| 
{
  "enabled": false,
  "port": 988
}
| no | +| [prepend\_ansible\_installer](#input\_prepend\_ansible\_installer) | DEPRECATED. Use `install_ansible=false` to prevent ansible installation. | `bool` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [runners](#input\_runners) | List of runners to run on remote VM.
Runners can be of type ansible-local, shell or data.
A runner must specify one of 'source' or 'content'.
All runners must specify 'destination'. If 'destination' does not include a
path, it will be copied in a temporary folder and deleted after running.
Runners may also pass 'args', which will be passed as argument to shell runners only. | `list(map(string))` | `[]` | no | +| [set\_ofi\_cloud\_rdma\_tunables](#input\_set\_ofi\_cloud\_rdma\_tunables) | Controls whether to enable specific OFI environment variables for workloads using Cloud RDMA networking. Should be false for non-RDMA workloads. | `bool` | `false` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [compute\_startup\_script](#output\_compute\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | +| [controller\_startup\_script](#output\_controller\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | +| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | + diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml new file mode 100644 index 0000000000..02c449c7cb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml @@ -0,0 +1,37 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Configure ssh between nodes + become: true + hosts: localhost + vars: + ssh_config_path: "/etc/ssh/ssh_config" + bashrc: "{{ '/etc/bashrc' if ansible_facts['os_family'] == 'RedHat' else '/etc/bash.bashrc' }}" + setup_ssh_script: "/bin/bash /usr/local/ghpc/setup-ssh-keys.sh" + tasks: + - name: "Set StrictHostKeyChecking to no" + ansible.builtin.blockinfile: + path: "{{ ssh_config_path }}" + block: | + Host "{{ item }}" + StrictHostKeyChecking no + marker: "# {mark} ANSIBLE MANAGED BLOCK {{item}}" + loop: "{{ host_name_prefix }}" + - name: "Create ssh keys in .bashrc if not already done" + ansible.builtin.lineinfile: + path: "{{ bashrc }}" + regexp: '^{{ setup_ssh_script }}' + line: "{{ setup_ssh_script }}" diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh new file mode 100644 index 0000000000..38c7ff9b5c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +web_proxy="${1:-}" +if [ -z "$web_proxy" ]; then + echo "Error: must provide 1 argument identifying http/https proxy" + exit 1 +fi + +# configure pip to use proxy +PIP_CONF=/etc/pip.conf +if [ ! -f "$PIP_CONF" ]; then + cat <<-EOF >"$PIP_CONF" + [global] + proxy=$web_proxy + EOF +fi + +# configure yum or dnf to use proxy +if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + YUM_CONF="/etc/yum.conf" + if ! grep -q '^proxy=.*' "$YUM_CONF"; then + sed --follow-symlinks -i.bak "/^\[main]/a proxy=$web_proxy" "$YUM_CONF" + else + sed --follow-symlinks -i.bak "s,proxy=.*,proxy=$web_proxy," "$YUM_CONF" + fi +fi + +# configure apt to use proxy +if [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release 2>/dev/null || + grep -qi ubuntu /etc/os-release 2>/dev/null; then + APT_CONF_PROXY="/etc/apt/apt.conf.d/99proxy.conf" + if [ ! -f "$APT_CONF_PROXY" ]; then + cat <<-EOF >"$APT_CONF_PROXY" + Acquire::http::Proxy "$web_proxy"; + Acquire::https::Proxy "$web_proxy"; + EOF + fi +fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh new file mode 100644 index 0000000000..682e1352a1 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script applies fixes to VMs that must occur early in boot. For example, +# when yum or apt repositories are misconfigured, preventing most package +# operations from completing successfully. + +source /etc/os-release + +if [[ "$PRETTY_NAME" == "CentOS Linux 7 (Core)" ]]; then + echo "Applying hotfixes for CentOS 7" + if grep -q '^mirrorlist' /etc/yum.repos.d/CentOS-Base.repo; then + echo "Removing mirrorlist from default CentOS 7 repositories" + sed -i '/^mirrorlist/d' /etc/yum.repos.d/CentOS-Base.repo + fi + if grep -q '^#baseurl=http://mirror.centos.org' /etc/yum.repos.d/CentOS-Base.repo; then + echo "Reconfiguring default CentOS 7 repositories to use CentOS Vault" + sed -i 's,^#baseurl=http://mirror.centos.org/,baseurl=http://vault.centos.org/,' /etc/yum.repos.d/CentOS-Base.repo + fi +fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh new file mode 100644 index 0000000000..3a29ae808f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh @@ -0,0 +1,73 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Given a url and filename, download an object to the vardir. When the installed +# version of gcloud is >=402.0.0 (Sept. 2022), then gcloud storage is used to +# fetch from the bucket. Otherwise gsutil is used. Note, the service account for +# the instance must be properly configured with a role having authorization to +# get objects from the bucket. +# +# This function is intended for single file downloads and no attempt is made to +# verify the checksum other than the default behavior of gcloud or gsutil. +# +# This function has no other platform dependencies other than gcloud / gsutil. + +# This code originated from: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 +stdlib::get_from_bucket() { + local OPTIND opt url fname dir="${VARDIR:-/var/lib/startup}" + while getopts ":u:f:d:" opt; do + case "${opt}" in + u) url="${OPTARG}" ;; + f) fname="${OPTARG}" ;; + d) dir="${OPTARG}" ;; + :) + stdlib::mandatory_argument -n stdlib::get_from_bucket -f "$OPTARG" + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error 'Usage: stdlib::get_from_bucket -u -f -d ' + stdlib::info 'For example: stdlib::get_from_bucket -u gs://mybucket/foo.tgz -d /var/tmp' + return "${E_UNKNOWN_ARG}" + ;; + esac + done + # Trivially compute the filename from the URL if unspecified. + if [[ -z ${fname} ]]; then + fname=${url##*/} + stdlib::debug "Computed filename='${fname}' given URL." + fi + [[ -d ${dir} ]] || mkdir "${dir}" + local attempt=0 + local max_retries=7 + # store gcs command as array and then split when called by stdlib::cmd + if stdlib::cmd gcloud help storage cp &>/dev/null; then + gcs_command=(gcloud storage cp --no-user-output-enabled) + else + gcs_command=(gsutil -q cp) + fi + while [[ $attempt -le $max_retries ]]; do + if [[ $attempt -gt 0 ]]; then + local wait=$((2 ** attempt)) + stdlib::error "Retry attempt ${attempt} of ${max_retries} with exponential backoff: ${wait} seconds." + sleep $wait + fi + if stdlib::cmd "${gcs_command[@]}" "${url}" "${dir}/${fname}"; then + break + else + stdlib::error "${gcs_command[*]} reported non-zero exit code fetching ${url}." + ((attempt++)) + fi + done +} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh new file mode 100644 index 0000000000..eac2b2e32a --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh @@ -0,0 +1,247 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +REQ_ANSIBLE_VERSION=2.15 +REQ_ANSIBLE_PIP_VERSION=8.7.0 +REQ_PIP_WHEEL_VERSION=0.45.1 +REQ_PIP_SETUPTOOLS_VERSION=80.8.0 +REQ_PIP_MAJOR_VERSION=25 +REQ_PYTHON3_VERSION=9 + +apt_wait() { + while fuser /var/lib/apt/lists/lock >/dev/null 2>&1; do + echo "Sleeping for apt lists lock" + sleep 3 + done +} + +# Installs any dependencies needed for python based on the OS +install_python_deps() { + # this file is present on both Debian and Ubuntu OSes + if [ -f /etc/debian_version ]; then + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3-setuptools python3-venv + fi +} + +# Gets the name of the python executable for python starting with python3, then +# checking python. Sets the variable to an empty string if neither are found. +get_python_path() { + python_path="" + if command -v python3 1>/dev/null; then + python_path=$(command -v python3) + elif command -v python 1>/dev/null; then + python_path=$(command -v python) + fi +} + +# Returns the python major version. If provided, it will use the first argument +# as the python executable, otherwise it will default to simply "python". +get_python_major_version() { + python_path=${1:-python} + python_major_version=$(${python_path} -c "import sys; print(sys.version_info.major)") +} + +# Returns the python minor version. If provided, it will use the first argument +# as the python executable, otherwise it will default to simply "python". +get_python_minor_version() { + python_path=${1:-python} + python_minor_version=$(${python_path} -c "import sys; print(sys.version_info.minor)") +} + +# Install python3 with the yum package manager. Updates python_path to the +# newly installed packaged. +install_python3_dnf() { + major_version=$(rpm -E "%{rhel}") + set -- "--disablerepo=*" "--enablerepo=baseos,appstream" + if grep -qi 'ID="rhel"' /etc/os-release; then + # Do not set --disablerepo / --enablerepo on RedHat, due to + # complex repo names; clear array + set -- + fi + # On Rocky Linux 9, Python 3.9 is installed by default but this + # has already been dropped by ansible-core for control nodes. + # https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix + # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where + # it is available as "python3*" but must be named explicitly on + # older releases. It also ensures longer support for Ansible. + if [ "${major_version}" -lt "10" ]; then + dnf install "$@" -y python3.12 python3.12-pip + python_path=$(command -v python3.12) + else + dnf install "$@" -y python3 python3-pip + python_path=$(command -v python3) + fi +} + +# Install python3 with the apt package manager. Updates python_path to the +# newly installed packaged. +install_python3_apt() { + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3 python3-setuptools python3-pip python3-venv + python_path=$(command -v python3) +} + +install_python3() { + if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || + [ -f /etc/system-release ]; then + install_python3_dnf + elif [ -f /etc/debian_version ]; then + install_python3_apt + else + echo "Error: Unsupported Distribution" + return 1 + fi +} + +# Install pip3 with the dnf package manager. Updates python_path to the +# newly installed packaged. +install_pip3_dnf() { + major_version=$(rpm -E "%{rhel}") + set -- "--disablerepo=*" "--enablerepo=baseos,appstream" + if grep -qi 'ID="rhel"' /etc/os-release; then + # Do not set --disablerepo / --enablerepo on RedHat, due to complex repo names + # clear array + set -- + fi + # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where + # it is available as "python3*" but must be named explicitly on + # older releases. It also ensures longer support for Ansible. + if [ "${major_version}" -lt "10" ]; then + dnf install "$@" -y python3.12-pip + else + dnf install "$@" -y python3-pip + fi +} + +# Install pip3 with the apt package manager. Updates python_path to the +# newly installed packaged. +install_pip3_apt() { + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3-pip +} + +install_pip3() { + if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || + [ -f /etc/system-release ]; then + install_pip3_dnf + elif [ -f /etc/debian_version ]; then + install_pip3_apt + else + echo "Error: Unsupported Distribution" + return 1 + fi +} + +main() { + if [ $# -gt 1 ]; then + echo "Error: provide only 1 optional argument identifying virtual environment path for Ansible" + return 1 + fi + + venv_path="${1:-/usr/local/ghpc-venv}" + + # Get the python3 executable, or install it if not found + get_python_path + get_python_major_version "${python_path}" + get_python_minor_version "${python_path}" + if [ "${python_path}" = "" ] || [ "${python_major_version}" = "2" ] || [ "${python_minor_version}" -lt "${REQ_PYTHON3_VERSION}" ]; then + if ! install_python3; then + return 1 + fi + get_python_major_version "${python_path}" + get_python_minor_version "${python_path}" + else + install_python_deps + fi + + # Install OS-packaged pip + if ! ${python_path} -m pip --version 2>/dev/null; then + if ! install_pip3; then + return 1 + fi + fi + + # Create pip virtual environment for Cluster Toolkit + ${python_path} -m venv "${venv_path}" --copies + venv_python_path=${venv_path}/bin/python3 + + # Upgrade pip if necessary + pip_version=$(${venv_python_path} -m pip --version | sed -nr 's/^pip ([0-9]+\.[0-9]+).*$/\1/p') + pip_major_version=$(echo "${pip_version}" | cut -d '.' -f 1) + if [ "${pip_major_version}" -lt "${REQ_PIP_MAJOR_VERSION}" ]; then + ${venv_python_path} -m pip install --upgrade pip + fi + + # upgrade wheel if necessary + wheel_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^wheel" || true) + if [ "$wheel_pkg" != "wheel==${REQ_PIP_WHEEL_VERSION}" ]; then + ${venv_python_path} -m pip install -U wheel==${REQ_PIP_WHEEL_VERSION} + fi + + # upgrade setuptools if necessary + setuptools_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^setuptools" || true) + if [ "$setuptools_pkg" != "setuptools==${REQ_PIP_SETUPTOOLS_VERSION}" ]; then + ${venv_python_path} -m pip install -U setuptools==${REQ_PIP_SETUPTOOLS_VERSION} + fi + + # configure ansible to always use correct Python binary + if [ ! -f /etc/ansible/ansible.cfg ]; then + mkdir /etc/ansible + cat <<-EOF >/etc/ansible/ansible.cfg + [defaults] + interpreter_python=${venv_python_path} + stdout_callback=debug + stderr_callback=debug + EOF + fi + + # Install ansible + ansible_version="" + if command -v ansible-playbook 1>/dev/null; then + ansible_version=$(ansible-playbook --version 2>/dev/null | sed -nr 's/^ansible-playbook.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p') + ansible_major_vers=$(echo "${ansible_version}" | cut -d '.' -f 1) + ansible_minor_vers=$(echo "${ansible_version}" | cut -d '.' -f 2) + ansible_req_major_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 1) + ansible_req_minor_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 2) + fi + if [ -z "${ansible_version}" ] || [ "${ansible_major_vers}" -ne "${ansible_req_major_vers}" ] || + [ "${ansible_minor_vers}" -lt "${ansible_req_minor_vers}" ]; then + ${venv_python_path} -m pip install ansible=="${REQ_ANSIBLE_PIP_VERSION}" + fi + while read -r cmd; do + if ! [ -L "/usr/bin/${cmd}" ]; then + ln -s "${venv_path}/bin/${cmd}" "/usr/bin/${cmd}" + fi + done <<-EOF + ansible + ansible-config + ansible-connection + ansible-console + ansible-doc + ansible-galaxy + ansible-inventory + ansible-playbook + ansible-pull + ansible-test + ansible-vault + EOF +} + +main "$@" diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh new file mode 100644 index 0000000000..375792459b --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +OS_ID="$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g')" +OS_VERSION="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g')" +OS_VERSION_MAJOR="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//')" +REBOOT_FILE="/etc/.rdma_reboot" + +if { [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; } && { [ "${OS_VERSION_MAJOR}" = "8" ]; }; then + KMOD_VERSION="$(dnf list installed | awk '$1 ~ /^kmod-idpf-irdma(\.|$)/ {print $2}')" + + # For images that do not already have Cloud RDMA drivers installed + if [ -z "${KMOD_VERSION}" ] && [ -z "${REBOOT_FILE}" ]; then + sudo dnf update -y + sudo dnf install https://depot.ciq.com/public/files/gce-accelerator/irdma-kernel-modules-el8-x86_64/irdma-repos.rpm -y + sudo dnf install kmod-idpf-irdma rdma-core libibverbs-utils librdmacm-utils infiniband-diags perftest -y + sudo touch "${REBOOT_FILE}" + reboot + fi + echo "This image has IRDMA packages already installed, exiting." + exit 0 +else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. Cloud RDMA Drivers are only supported on Rocky Linux 8." + exit 1 +fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_docker.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_docker.yml new file mode 100644 index 0000000000..f9b0abeb14 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_docker.yml @@ -0,0 +1,113 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install and configure Docker + hosts: all + become: true + vars: + docker_data_root: '' + docker_daemon_config: '' + enable_docker_world_writable: false + tasks: + - name: Check if docker is installed + ansible.builtin.stat: + path: /usr/bin/docker + register: docker_binary + - name: Download Docker Installer + ansible.builtin.get_url: + url: https://get.docker.com + dest: /tmp/get-docker.sh + owner: root + group: root + mode: '0644' + when: not docker_binary.stat.exists + - name: Install Docker + ansible.builtin.command: sh /tmp/get-docker.sh + register: docker_installed + changed_when: docker_installed.rc != 0 + when: not docker_binary.stat.exists + - name: Create Docker daemon configuration + ansible.builtin.copy: + dest: /etc/docker/daemon.json + mode: '0644' + content: '{{ docker_daemon_config }}' + validate: /usr/bin/dockerd --validate --config-file %s + when: docker_daemon_config + notify: + - Restart Docker + - name: Create Docker service override directory + ansible.builtin.file: + path: /etc/systemd/system/docker.service.d + state: directory + owner: root + group: root + mode: '0755' + - name: Create Docker service override configuration + ansible.builtin.copy: + dest: /etc/systemd/system/docker.service.d/data-root.conf + mode: '0644' + content: | + [Unit] + {% if docker_data_root %} + RequiresMountsFor={{ docker_data_root }} + {% endif %} + After=mount-localssd-raid.service + - name: Create Docker socket override directory + ansible.builtin.file: + path: /etc/systemd/system/docker.socket.d + state: directory + owner: root + group: root + mode: '0755' + when: enable_docker_world_writable + - name: Create Docker socket override configuration + ansible.builtin.copy: + dest: /etc/systemd/system/docker.socket.d/world-writable.conf + mode: '0644' + content: | + [Socket] + SocketMode=0666 + when: enable_docker_world_writable + notify: + - Reload SystemD + - Recreate Docker socket + - name: Delete Docker socket override configuration + ansible.builtin.file: + path: /etc/systemd/system/docker.socket.d/world-writable.conf + state: absent + when: not enable_docker_world_writable + notify: + - Reload SystemD + - Recreate Docker socket + + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + - name: Recreate Docker socket + ansible.builtin.service: + name: docker.socket + state: restarted + - name: Restart Docker + ansible.builtin.service: + name: docker.service + state: restarted + + post_tasks: + - name: Start Docker + ansible.builtin.service: + name: docker.service + state: started + enabled: true diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml new file mode 100644 index 0000000000..9d295dfc7d --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml @@ -0,0 +1,56 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install network wait service for A3/A4 variants + hosts: all + become: true + tasks: + + - name: Create universal SystemD service for GPU networking delay + when: ansible_os_family == "Debian" + ansible.builtin.copy: + dest: /etc/systemd/system/delay-gpu-network.service + owner: root + group: root + mode: "0644" + content: | + [Unit] + Description=Delay boot on multi-NIC VMs until networks are routable + After=network-online.target + Wants=network-online.target + Before=google-startup-scripts.service + + [Service] + # This condition checks if the machine type is one of the supported A3/A4 variants. + # The service will only run if the machine type matches. + ExecCondition=/bin/bash -c "/usr/bin/curl -s -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/machine-type | grep -qE '(/a3-highgpu-8g|/a3-megagpu-8g|/a3-ultragpu-8g|/a4-highgpu-8g|/a4x-highgpu-4g)$'" + ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -o routable --timeout=180 + ExecStartPost=/bin/sleep 30 + + [Install] + WantedBy=multi-user.target + notify: + - Reload SystemD + + - name: Enable universal GPU network delay service + when: ansible_os_family == "Debian" + ansible.builtin.systemd_service: + name: delay-gpu-network.service + enabled: true + + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml new file mode 100644 index 0000000000..94699471bb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml @@ -0,0 +1,33 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure Managed Lustre (assumes driver already installed) + hosts: all + become: true + vars: + default_lustre_port: 988 + managed_lustre_port: "{{ default_lustre_port }}" + tasks: + # Ideally changes to this file would also trigger an execution of lnetctl + # command to update accept_port but it is unclear if lnetctl supports this. + - name: Update lnet to use non-default port + when: managed_lustre_port | int != {{ default_lustre_port }} + ansible.builtin.copy: + owner: root + group: root + mode: '0644' + dest: /etc/modprobe.d/lnet.conf + content: | + options lnet accept_port={{ managed_lustre_port | int }} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh new file mode 100644 index 0000000000..eb4bf899b8 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +LEGACY_MONITORING_PACKAGE='stackdriver-agent' +LEGACY_MONITORING_SCRIPT_URL='https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh' +LEGACY_LOGGING_PACKAGE='google-fluentd' +LEGACY_LOGGING_SCRIPT_URL='https://dl.google.com/cloudagents/add-logging-agent-repo.sh' + +OPSAGENT_PACKAGE='google-cloud-ops-agent' +OPSAGENT_SCRIPT_URL='https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh' + +ops_or_legacy="${1:-legacy}" + +fail() { + echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" + exit 1 +} + +handle_debian() { + is_legacy_monitoring_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_MONITORING_PACKAGE} | + grep "${LEGACY_MONITORING_PACKAGE} is installed" + } + + is_legacy_logging_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_LOGGING_PACKAGE} | + grep "${LEGACY_LOGGING_PACKAGE} is installed" + } + + is_legacy_installed() { + is_legacy_monitoring_installed || is_legacy_logging_installed + } + + is_opsagent_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${OPSAGENT_PACKAGE} | + grep "${OPSAGENT_PACKAGE} is installed" + } + + install_with_retry() { + MAX_RETRY=50 + RETRY=0 + until [ ${RETRY} -eq ${MAX_RETRY} ] || curl -s "${1}" | bash -s -- --also-install; do + RETRY=$((RETRY + 1)) + echo "WARNING: Installation of ${1} failed on try ${RETRY} of ${MAX_RETRY}" + sleep 5 + done + if [ $RETRY -eq $MAX_RETRY ]; then + echo "ERROR: Installation of ${1} was not successful after ${MAX_RETRY} attempts." + exit 1 + fi + } + + install_opsagent() { + install_with_retry "${OPSAGENT_SCRIPT_URL}" + } + + install_stackdriver_agent() { + install_with_retry "${LEGACY_MONITORING_SCRIPT_URL}" + install_with_retry "${LEGACY_LOGGING_SCRIPT_URL}" + service stackdriver-agent start + service google-fluentd start + } +} + +handle_redhat() { + is_legacy_monitoring_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_MONITORING_PACKAGE} | + grep "${LEGACY_MONITORING_PACKAGE} is installed" + } + + is_legacy_logging_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_LOGGING_PACKAGE} | + grep "${LEGACY_LOGGING_PACKAGE} is installed" + } + + is_legacy_installed() { + is_legacy_monitoring_installed || is_legacy_logging_installed + } + + is_opsagent_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${OPSAGENT_PACKAGE} | + grep "${OPSAGENT_PACKAGE} is installed" + } + + install_opsagent() { + curl -s "${OPSAGENT_SCRIPT_URL}" | bash -s -- --also-install + } + + install_stackdriver_agent() { + curl -sS "${LEGACY_MONITORING_SCRIPT_URL}" | bash -s -- --also-install + curl -sS "${LEGACY_LOGGING_SCRIPT_URL}" | bash -s -- --also-install + service stackdriver-agent start + service google-fluentd start + } +} + +main() { + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + handle_redhat + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + handle_debian + else + fail "Unsupported platform." + fi + + # Handle cases that agent is already installed + if [[ -z "$(is_legacy_monitoring_installed)" && -n $(is_legacy_logging_installed) ]] || + [[ -n "$(is_legacy_monitoring_installed)" && -z $(is_legacy_logging_installed) ]]; then + fail "Bad state: legacy agent is partially installed" + elif [[ "${ops_or_legacy}" == "legacy" ]] && is_legacy_installed; then + echo "Legacy agent is already installed" + exit 0 + elif [[ "${ops_or_legacy}" != "legacy" ]] && is_opsagent_installed; then + echo "Ops agent is already installed" + exit 0 + elif is_legacy_installed || is_opsagent_installed; then + fail "Agent is already installed but does not match requested agent of ${ops_or_legacy}" + fi + + # install agent + if [[ "${ops_or_legacy}" == "legacy" ]]; then + echo "Installing legacy monitoring agent (stackdriver)" + install_stackdriver_agent + else + echo "Installing cloud ops agent" + echo "WARNING: cloud ops agent may have a performance impact. Consider using legacy monitoring agent (stackdriver)." + install_opsagent + fi +} + +main diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh new file mode 100644 index 0000000000..738181aafb --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_COMPLETE_FILE="/run/startup_script_msg" + +# Ensure we're in an interactive terminal and not root +if [ -t 1 ] && [ "$(id -u)" -ne 0 ]; then + # Check if the file has contents otherwise skip + if [ -s "$SCRIPT_COMPLETE_FILE" ]; then + echo + cat "$SCRIPT_COMPLETE_FILE" + echo + fi +fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml new file mode 100644 index 0000000000..d94aac81fd --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml @@ -0,0 +1,100 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Configure local SSDs + become: true + hosts: localhost + vars: + raid_name: localssd + array_dev: /dev/md/{{ raid_name }} + fstype: ext4 + interface: nvme + mode: '0755' + mountpoint: /mnt/{{ raid_name }} + tasks: + - name: Get local SSD devices + ansible.builtin.find: + file_type: link + path: /dev/disk/by-id + patterns: google-local-{{ "nvme-" if interface == "nvme" else "" }}ssd-* + register: local_ssd_devices + + - name: Exit if zero local ssd found + ansible.builtin.meta: end_play + when: local_ssd_devices.files | length == 0 + + - name: Install mdadm + ansible.builtin.package: + name: mdadm + state: present + + # this service will act during the play and upon reboots to ensure that local + # SSD volumes are always assembled into a RAID and re-formatted if necessary; + # there are many scenarios where a VM can be stopped or migrated during + # maintenance and the contents of local SSD will be discarded + - name: Install service to create local SSD RAID and format it + ansible.builtin.copy: + dest: /etc/systemd/system/create-localssd-raid.service + mode: 0644 + content: | + [Unit] + After=local-fs.target + Before=slurmd.service docker.service + ConditionPathExists=!{{ array_dev }} + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/bash -c "/usr/sbin/mdadm --create {{ array_dev }} --name={{ raid_name }} --homehost=any --level=0 --raid-devices={{ local_ssd_devices.files | length }} /dev/disk/by-id/google-local-nvme-ssd-*{{ " --force" if local_ssd_devices.files | length == 1 else "" }}" + ExecStartPost=/usr/sbin/mkfs -t {{ fstype }}{{ " -m 0" if fstype == "ext4" else "" }} {{ array_dev }} + + [Install] + WantedBy=slurmd.service docker.service + + - name: Create RAID array and format + ansible.builtin.systemd: + name: create-localssd-raid.service + state: started + enabled: true + daemon_reload: true + + - name: Install service to mount local SSD array + ansible.builtin.copy: + dest: /etc/systemd/system/mount-localssd-raid.service + mode: 0644 + content: | + [Unit] + After=local-fs.target create-localssd-raid.service + Before=slurmd.service docker.service + Wants=create-localssd-raid.service + ConditionPathIsMountPoint=!{{ mountpoint }} + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/systemd-mount -t {{ fstype }} -o discard,defaults,nofail {{ array_dev }} {{ mountpoint }} + ExecStartPost=/usr/bin/chmod {{ mode }} {{ mountpoint }} + ExecStop=/usr/bin/systemd-umount {{ mountpoint }} + + [Install] + WantedBy=slurmd.service docker.service + + - name: Mount RAID array and set permissions + ansible.builtin.systemd: + name: mount-localssd-raid.service + state: started + enabled: true + daemon_reload: true diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh new file mode 100644 index 0000000000..1c8018fb01 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! -d ~/.ssh/ ]; then + source /usr/local/ghpc-venv/bin/activate + ansible-playbook /usr/local/ghpc/setup-ssh-keys.yml +fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml new file mode 100644 index 0000000000..692896bb9c --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml @@ -0,0 +1,40 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Setup SSH Keys for user + become: false + hosts: localhost + vars: + pub_key_path: "{{ ansible_env.HOME }}/.ssh" + pub_key_file: "{{ pub_key_path }}/id_rsa" + auth_key_file: "{{ pub_key_path }}/authorized_keys" + tasks: + - name: "Create .ssh folder" + ansible.builtin.file: + path: "{{ pub_key_path }}" + state: directory + mode: 0700 + owner: "{{ ansible_user_id }}" + - name: Create keys + community.crypto.openssh_keypair: + path: "{{ pub_key_file }}" + owner: "{{ ansible_user_id }}" + - name: Copy public key to authorized keys + ansible.builtin.copy: + src: "{{ pub_key_file }}.pub" + dest: "{{ auth_key_file }}" + owner: "{{ ansible_user_id }}" + mode: 0644 diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh new file mode 100644 index 0000000000..8ca40bc73f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh @@ -0,0 +1,39 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code contains minor changes from the original: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 + +stdlib::main() { + DELETE_AT_EXIT="$(mktemp -d)" + readonly DELETE_AT_EXIT + + # Initialize state required by other functions, e.g. debug() + stdlib::init + stdlib::debug "Loaded startup-script-stdlib as an executable." + + stdlib::load_config_values + + stdlib::load_runners +} + +# if script is being executed and not sourced. +if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then + stdlib::finish() { + [[ -d ${DELETE_AT_EXIT:-} ]] && rm -rf "${DELETE_AT_EXIT}" + } + trap stdlib::finish EXIT + + stdlib::main "$@" +fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh new file mode 100644 index 0000000000..589a3215ab --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh @@ -0,0 +1,266 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code contains minor changes from the original in: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 + +# Standard library of functions useful for startup scripts. + +# These are outside init_global_vars so logging functions work with the most +# basic case of `source startup-script-stdlib.sh` +readonly SYSLOG_DEBUG_PRIORITY="${SYSLOG_DEBUG_PRIORITY:-syslog.debug}" +readonly SYSLOG_INFO_PRIORITY="${SYSLOG_INFO_PRIORITY:-syslog.info}" +readonly SYSLOG_ERROR_PRIORITY="${SYSLOG_ERROR_PRIORITY:-syslog.error}" +# Global counter of how many times stdlib::init() has been called. +STARTUP_SCRIPT_STDLIB_INITIALIZED=0 + +# Error codes +readonly E_RUN_OR_DIE=5 +readonly E_MISSING_MANDATORY_ARG=9 +readonly E_UNKNOWN_ARG=10 + +SCRIPT_COMPLETE_FILE="/run/startup_script_msg" +SUCCESS_MESSAGE="* NOTICE **: The Cluster Toolkit startup scripts have finished running successfully." +readonly SUCCESS_MESSAGE +ERROR_MESSAGE="** ERROR **: The Cluster Toolkit startup scripts have finished running, but produced an error." +readonly ERROR_MESSAGE +WARNING_MESSAGE="** WARNING **: The Cluster Toolkit startup scripts are currently running." +readonly WARNING_MESSAGE + +stdlib::debug() { + [[ -z ${DEBUG:-} ]] && return 0 + local ds msg + msg="$*" + logger -p "${SYSLOG_DEBUG_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + [[ -n ${QUIET:-} ]] && return 0 + ds="$(date +"${DATE_FMT}") " + echo -e "${BLUE}${ds}Debug [$$]: ${msg}${NC}" >&2 +} + +stdlib::info() { + local ds msg + msg="$*" + logger -p "${SYSLOG_INFO_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + [[ -n ${QUIET:-} ]] && return 0 + ds="$(date +"${DATE_FMT}") " + echo -e "${GREEN}${ds}Info [$$]: ${msg}${NC}" >&2 +} + +stdlib::error() { + local ds msg + msg="$*" + ds="$(date +"${DATE_FMT}") " + logger -p "${SYSLOG_ERROR_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + echo -e "${RED}${ds}Error [$$]: ${msg}${NC}" >&2 +} + +stdlib::announce_runners_start() { + if [ -z "$recursive_proc" ]; then + wall -n "$WARNING_MESSAGE" + echo "$WARNING_MESSAGE" >"$SCRIPT_COMPLETE_FILE" + fi + export recursive_proc=$((${recursive_proc:=0} + 1)) +} + +stdlib::announce_runners_end() { + exit_code=$1 + export recursive_proc=$((${recursive_proc:=0} - 1)) + if [ "$recursive_proc" -le "0" ]; then + if [ "$exit_code" -ne "0" ]; then + wall -n "$ERROR_MESSAGE" + echo "$ERROR_MESSAGE" >"$SCRIPT_COMPLETE_FILE" + else + wall -n "$SUCCESS_MESSAGE" + echo -n "" >"$SCRIPT_COMPLETE_FILE" + fi + fi +} + +# The main initialization function of this library. This should be kept to the +# minimum amount of work required for all functions to operate cleanly. +stdlib::init() { + if [[ ${STARTUP_SCRIPT_STDLIB_INITIALIZED} -gt 0 ]]; then + stdlib::info 'stdlib::init()'" already initialized, no action taken." + return 0 + fi + ((STARTUP_SCRIPT_STDLIB_INITIALIZED++)) || true + stdlib::init_global_vars + stdlib::init_directories + stdlib::debug "stdlib::init(): startup-script-stdlib.sh initialized and ready" +} + +# Initialize global variables. +stdlib::init_global_vars() { + # The program name, used for logging. + readonly PROG="${PROG:-startup-script-stdlib}" + # Date format used for stderr logging. Passed to date + command. + readonly DATE_FMT="${DATE_FMT:-"%a %b %d %H:%M:%S %z %Y"}" + # var directory + readonly VARDIR="${VARDIR:-/var/lib/startup}" + # Override this with file://localhost/tmp/foo/bar in spec test context + readonly METADATA_BASE="${METADATA_BASE:-http://metadata.google.internal}" + + # Color variables + if [[ -n ${COLOR:-} ]]; then + readonly NC='\033[0m' # no color + readonly RED='\033[0;31m' # error + readonly GREEN='\033[0;32m' # info + readonly BLUE='\033[0;34m' # debug + else + readonly NC='' + readonly RED='' + readonly GREEN='' + readonly BLUE='' + fi + + return 0 +} + +stdlib::init_directories() { + if ! [[ -e ${VARDIR} ]]; then + install -d -m 0755 -o 0 -g 0 "${VARDIR}" + fi +} + +## +# Get a metadata key. When used without -o, this function is guaranteed to +# produce no output on STDOUT other than the retrieved value. This is intended +# to support the use case of +# FOO="$(stdlib::metadata_get -k instance/attributes/foo)" +# +# If the requested key does not exist, the error code will be 22 and zero bytes +# written to STDOUT. +stdlib::metadata_get() { + local OPTIND opt key outfile + local metadata="${METADATA_BASE%/}/computeMetadata/v1" + local exit_code + while getopts ":k:o:" opt; do + case "${opt}" in + k) key="${OPTARG}" ;; + o) outfile="${OPTARG}" ;; + :) + stdlib::error "Invalid option: -${OPTARG} requires an argument" + stdlib::metadata_get_usage + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error "Unknown option: -${opt}" + stdlib::metadata_get_usage + return "${E_UNKNOWN_ARG}" + ;; + esac + done + local url="${metadata}/${key#/}" + + stdlib::debug "Getting metadata resource url=${url}" + if [[ -z ${outfile:-} ]]; then + curl --location --silent --connect-timeout 1 --fail \ + -H 'Metadata-Flavor: Google' "$url" 2>/dev/null + exit_code=$? + else + stdlib::cmd curl --location \ + --silent \ + --connect-timeout 1 \ + --fail \ + --output "${outfile}" \ + -H 'Metadata-Flavor: Google' \ + "$url" + exit_code=$? + fi + case "${exit_code}" in + 22 | 37) + stdlib::debug "curl exit_code=${exit_code} for url=${url}" \ + "(Does not exist)" + ;; + esac + return "${exit_code}" +} + +stdlib::metadata_get_usage() { + stdlib::info 'Usage: stdlib::metadata_get -k ' + stdlib::info 'For example: stdlib::metadata_get -k instance/attributes/startup-config' +} + +# Load configuration values in the spirit of /etc/sysconfig defaults, but from +# metadata instead of the filesystem. +stdlib::load_config_values() { + local config_file + local key="instance/attributes/startup-script-config" + # shellcheck disable=SC2119 + config_file="$(stdlib::mktemp)" + stdlib::metadata_get -k "${key}" -o "${config_file}" + local status=$? + case "$status" in + 0) + stdlib::debug "SUCCESS: Configuration data sourced from $key" + ;; + 22 | 37) + stdlib::debug "no configuration data loaded from $key" + ;; + *) + stdlib::error "metadata_get -k $key returned unknown status=${status}" + ;; + esac + # shellcheck source=/dev/null + source "${config_file}" +} + +# Run a command logging the entry and exit. Intended for system level commands +# and operational debugging. Not intended for use with redirection. This is +# not named run() because bats uses a run() function. +stdlib::cmd() { + local exit_code argv=("$@") + stdlib::debug "BEGIN: stdlib::cmd() command=[${argv[*]}]" + "${argv[@]}" + exit_code=$? + stdlib::debug "END: stdlib::cmd() command=[${argv[*]}] exit_code=${exit_code}" + return $exit_code +} + +# Run a command successfully or exit the program with an error. +stdlib::run_or_die() { + if ! stdlib::cmd "$@"; then + stdlib::error "stdlib::run_or_die(): exiting with exit code ${E_RUN_OR_DIE}." + exit "${E_RUN_OR_DIE}" + fi +} + +# Intended to take advantage of automatic cleanup of startup script library +# temporary files without exporting a modified TMPDIR to child processes, which +# would cause the children to have their TMPDIR deleted out from under them. +# shellcheck disable=SC2120 +stdlib::mktemp() { + TMPDIR="${DELETE_AT_EXIT:-${TMPDIR}}" mktemp "$@" +} + +# Return a nice error message if a mandatory argument is missing. +stdlib::mandatory_argument() { + local OPTIND opt name flag + while getopts ":n:f:" opt; do + case "$opt" in + n) name="${OPTARG}" ;; + f) flag="${OPTARG}" ;; + :) + stdlib::error "Invalid argument: -${OPTARG} requires an argument to stdlib::mandatory_argument()" + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error "Unknown argument: -${OPTARG}" + stdlib::info "Usage: stdlib::mandatory_argument -n -f " + return "${E_UNKNOWN_ARG}" + ;; + esac + done + stdlib::error "Invalid argument: -${flag} requires an argument to ${name}()." +} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/main.tf b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/main.tf new file mode 100644 index 0000000000..02124eeddc --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/main.tf @@ -0,0 +1,306 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "startup-script", ghpc_role = "scripts" }) +} + +locals { + monitoring_agent_installer = ( + var.install_cloud_ops_agent || var.install_stackdriver_agent ? + [{ + type = "shell" + source = "${path.module}/files/install_monitoring_agent.sh" + destination = "install_monitoring_agent_automatic.sh" + args = var.install_cloud_ops_agent ? "ops" : "legacy" # install legacy (stackdriver) + }] : + [] + ) + + warnings = [ + { + type = "data" + content = file("${path.module}/files/running-script-warning.sh") + destination = "/etc/profile.d/99-running-script-warning.sh" + } + ] + + configure_ssh = length(var.configure_ssh_host_patterns) > 0 + host_args = { + host_name_prefix = var.configure_ssh_host_patterns + } + + prefix_file = "/tmp/prefix_file.json" + ansible_docker_settings_file = "/tmp/ansible_docker_settings.json" + + docker_config = try(jsondecode(var.docker.daemon_config), {}) + docker_data_root = try(local.docker_config.data-root, null) + + configure_ssh_runners = local.configure_ssh ? [ + { + type = "data" + source = "${path.module}/files/setup-ssh-keys.sh" + destination = "/usr/local/ghpc/setup-ssh-keys.sh" + }, + { + type = "data" + source = "${path.module}/files/setup-ssh-keys.yml" + destination = "/usr/local/ghpc/setup-ssh-keys.yml" + }, + { + type = "data" + content = jsonencode(local.host_args) + destination = local.prefix_file + }, + { + type = "ansible-local" + content = file("${path.module}/files/configure-ssh.yml") + destination = "configure-ssh.yml" + args = "-e @${local.prefix_file}" + } + ] : [] + + proxy_runner = var.http_proxy == "" ? [] : [ + { + type = "data" + destination = "/etc/profile.d/http_proxy.sh" + content = <<-EOT + #!/bin/bash + export http_proxy=${var.http_proxy} + export https_proxy=${var.http_proxy} + export NO_PROXY=${var.http_no_proxy} + EOT + }, + { + type = "shell" + source = "${path.module}/files/configure_proxy.sh" + destination = "configure_proxy.sh" + args = var.http_proxy + } + ] + + ofi_runner = !var.set_ofi_cloud_rdma_tunables ? [] : [ + { + type = "data" + destination = "/etc/profile.d/set_ofi_cloud_rdma_tunables.sh" + content = <<-EOT + #!/bin/bash + export FI_PROVIDER="verbs;ofi_rxm" + export FI_OFI_RXM_USE_RNDV_WRITE=0 + export FI_VERBS_INLINE_SIZE=39 + export I_MPI_FABRICS="shm:ofi" + export FI_UNIVERSE_SIZE=1024 + export I_MPI_ADJUST_ALLTOALL=1 + export I_MPI_ADJUST_IALLTOALL=1 + export I_MPI_ADJUST_BCAST=4 + export I_MPI_ADJUST_IBCAST=1 + EOT + }, + ] + + rdma_runner = !var.install_cloud_rdma_drivers ? [] : [ + { + type = "shell" + source = "${path.module}/files/install_cloud_rdma_drivers.sh" + destination = "install_cloud_rdma_drivers.sh" + } + ] + + docker_runner = !var.docker.enabled ? [] : [ + { + type = "data" + destination = local.ansible_docker_settings_file + content = jsonencode({ + enable_docker_world_writable = var.docker.world_writable + docker_daemon_config = var.docker.daemon_config + docker_data_root = local.docker_data_root + }) + }, + { + type = "ansible-local" + destination = "install_docker.yml" + content = file("${path.module}/files/install_docker.yml") + args = "-e \"@${local.ansible_docker_settings_file}\"" + }, + ] + + managed_lustre_runner = !var.managed_lustre.enabled ? [] : [ + { + type = "ansible-local" + destination = "install_managed_lustre.yml" + content = file("${path.module}/files/install_managed_lustre.yml") + args = "-e managed_lustre_port=${var.managed_lustre.port}" + }, + ] + + gpu_network_wait_online_runner = !var.enable_gpu_network_wait_online ? [] : [ + { + type = "ansible-local" + destination = "install_gpu_network_wait_online.yml" + content = file("${path.module}/files/install_gpu_network_wait_online.yml") + args = "" + }, + ] + + local_ssd_filesystem_enabled = can(coalesce(var.local_ssd_filesystem.mountpoint)) + raid_setup = !local.local_ssd_filesystem_enabled ? [] : [ + { + type = "ansible-local" + destination = "setup-raid.yml" + content = file("${path.module}/files/setup-raid.yml") + args = join(" ", [ + "-e mountpoint=${var.local_ssd_filesystem.mountpoint}", + "-e fs_type=${var.local_ssd_filesystem.fs_type}", + "-e mode=${var.local_ssd_filesystem.permissions}", + ]) + }, + ] + + supplied_ansible_runners = anytrue([for r in var.runners : r.type == "ansible-local"]) + has_ansible_runners = anytrue([ + local.supplied_ansible_runners, + local.configure_ssh, + var.docker.enabled, + var.managed_lustre.enabled, + var.enable_gpu_network_wait_online, + local.local_ssd_filesystem_enabled + ]) + + install_ansible = coalesce(var.install_ansible, local.has_ansible_runners) + ansible_installer = local.install_ansible ? [{ + type = "shell" + source = "${path.module}/files/install_ansible.sh" + destination = "install_ansible_automatic.sh" + args = var.ansible_virtualenv_path + }] : [] + + hotfix_runner = [{ + type = "shell" + source = "${path.module}/files/early_run_hotfixes.sh" + destination = "early_run_hotfixes.sh" + }] + + runners = concat( + local.warnings, + local.hotfix_runner, + local.proxy_runner, + local.ofi_runner, + local.rdma_runner, + local.monitoring_agent_installer, + local.ansible_installer, + local.raid_setup, # order RAID early to ensure filesystem is ready for subsequent runners + local.managed_lustre_runner, + local.configure_ssh_runners, + local.docker_runner, + local.gpu_network_wait_online_runner, + var.runners + ) + + bucket_regex = "^gs://([^/]*)/*(.*)" + gcs_bucket_path_trimmed = var.gcs_bucket_path == null ? null : trimsuffix(var.gcs_bucket_path, "/") + storage_folder_path = local.gcs_bucket_path_trimmed == null ? null : regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[1] + storage_folder_path_prefix = local.storage_folder_path == null || local.storage_folder_path == "" ? "" : "${local.storage_folder_path}/" + + user_provided_bucket_name = try(regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[0], null) + storage_bucket_name = coalesce(one(google_storage_bucket.configs_bucket[*].name), local.user_provided_bucket_name) + + load_runners = templatefile( + "${path.module}/templates/startup-script-custom.tftpl", + { + bucket = local.storage_bucket_name, + http_proxy = var.http_proxy, + no_proxy = var.http_no_proxy, + runners = [ + for runner in local.runners : { + object = google_storage_bucket_object.scripts[basename(runner["destination"])].output_name + type = runner["type"] + destination = runner["destination"] + args = contains(keys(runner), "args") ? runner["args"] : "" + } + ] + } + ) + + stdlib_head = file("${path.module}/files/startup-script-stdlib-head.sh") + get_from_bucket = file("${path.module}/files/get_from_bucket.sh") + stdlib_body = file("${path.module}/files/startup-script-stdlib-body.sh") + + # List representing complete content, to be concatenated together. + stdlib_list = [ + local.stdlib_head, + local.get_from_bucket, + local.load_runners, + local.stdlib_body, + ] + + # Final content output to the user + stdlib = join("", local.stdlib_list) + + runners_map = { for runner in local.runners : + basename(runner["destination"]) => { + content = lookup(runner, "content", null) + source = lookup(runner, "source", null) + } + } +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket" "configs_bucket" { + count = var.gcs_bucket_path == null ? 1 : 0 + project = var.project_id + name = "${var.deployment_name}-startup-scripts-${random_id.resource_name_suffix.hex}" + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +resource "google_storage_bucket_iam_binding" "viewers" { + bucket = local.storage_bucket_name + role = "roles/storage.objectViewer" + members = var.bucket_viewers +} + +resource "google_storage_bucket_object" "scripts" { + # this writes all scripts exactly once into GCS + for_each = local.runners_map + name = "${local.storage_folder_path_prefix}${each.key}-${substr(try(md5(each.value.content), filemd5(each.value.source)), 0, 4)}" + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) + bucket = local.storage_bucket_name + timeouts { + create = "10m" + update = "10m" + } + + lifecycle { + precondition { + condition = !(var.install_cloud_ops_agent && var.install_stackdriver_agent) + error_message = "Only one of var.install_stackdriver_agent or var.install_cloud_ops_agent can be set. Stackdriver is recommended for best performance." + } + } +} + +resource "local_file" "debug_file" { + for_each = toset(var.debug_file != null ? [var.debug_file] : []) + filename = var.debug_file + content = local.stdlib +} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/metadata.yaml new file mode 100644 index 0000000000..2ada34471f --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/outputs.tf b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/outputs.tf new file mode 100644 index 0000000000..6a15082814 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/outputs.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "script to load and run all runners, as a string value." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} + +output "compute_startup_script" { + description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} + +output "controller_startup_script" { + description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl new file mode 100644 index 0000000000..3c894b00b0 --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl @@ -0,0 +1,65 @@ + + +stdlib::run_playbook() { + if [ ! "$(which ansible-playbook)" ]; then + stdlib::error "ansible-playbook not found"\ + "Please install ansible before running ansible-local runners." + exit 1 + fi + ansible-playbook --connection=local --inventory=localhost, --limit localhost $1 $2 + ret_code=$? + return $${ret_code} +} + +stdlib::runner() { + + type=$1 + object=$2 + destination=$3 + tmpdir=$4 + args=$5 + + destpath="$(dirname $destination)" + filename="$(basename $destination)" + + if [ "$destpath" = "." ]; then + destpath=$tmpdir + fi + + stdlib::get_from_bucket -u "gs://${bucket}/$object" -d "$destpath" -f "$filename" + + stdlib::info "=== start executing runner: $object ===" + case "$1" in + ansible-local) stdlib::run_playbook "$destpath/$filename" "$args";; + shell) chmod u+x /$destpath/$filename && $destpath/$filename $args;; + esac + + exit_code=$? + stdlib::info "=== $object finished with exit_code=$exit_code ===" + if [ "$exit_code" -ne "0" ] ; then + stdlib::error "=== execution of $object failed, exiting ===" + stdlib::announce_runners_end "$exit_code" + exit $exit_code + fi +} + +stdlib::load_runners(){ + tmpdir="$(mktemp -d)" + + stdlib::debug "=== BEGIN Running runners ===" + stdlib::announce_runners_start + + %{if http_proxy != "" ~} + stdlib::info "=== Setting HTTP_PROXY,HTTPS_PROXY to ${http_proxy} ===" + export http_proxy=${http_proxy} + export https_proxy=${http_proxy} + export NO_PROXY=${no_proxy} + %{endif ~} + + %{for r in runners ~} + stdlib::runner "${r.type}" "${r.object}" "${r.destination}" $${tmpdir} "${r.args}" + %{endfor ~} + + stdlib::announce_runners_end "0" + stdlib::debug "=== END Running runners ===" +} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/variables.tf b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/variables.tf new file mode 100644 index 0000000000..7080085ece --- /dev/null +++ b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/variables.tf @@ -0,0 +1,298 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used to name GCS bucket for startup scripts." + type = string +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string + default = null +} + +variable "bucket_viewers" { + description = "Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account)" + type = list(string) + default = [] + + validation { + condition = alltrue([ + for u in var.bucket_viewers : length(regexall("^(allUsers$|allAuthenticatedUsers$|user:|group:|serviceAccount:|domain:)", u)) > 0 + ]) + error_message = "Bucket viewer members must begin with user/group/serviceAccount/domain following https://cloud.google.com/iam/docs/reference/rest/v1/Policy#Binding" + } +} + +variable "debug_file" { + description = "Path to an optional local to be written with 'startup_script'." + type = string + default = null +} + +variable "labels" { + description = "Labels for the created GCS bucket. Key-value pairs." + type = map(string) +} + +variable "runners" { + description = < 0 + error_message = "The POSIX permissions for the mountpoint must be represented as a 3 or 4-digit octal" + } + + default = { + fs_type = "ext4" + mountpoint = "" + permissions = "0755" + } + + nullable = false +} + +variable "install_cloud_ops_agent" { + description = "Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true." + type = bool + default = false +} + +variable "install_stackdriver_agent" { + description = "Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance." + type = bool + default = false +} + +variable "install_ansible" { + description = "Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used." + type = bool + default = null +} + +variable "configure_ssh_host_patterns" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 4.84 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [home\_pv](#module\_home\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | +| [kubectl\_apply](#module\_kubectl\_apply) | ../../../../modules/management/kubectl-apply | n/a | +| [slurm\_key\_pv](#module\_slurm\_key\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.gke_nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | +| [filestore\_id](#input\_filestore\_id) | An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `list(string)` | n/a | yes | +| [image](#input\_image) | The image for slurm daemon | `string` | n/a | yes | +| [instance\_templates](#input\_instance\_templates) | The URLs of Instance Templates | `list(string)` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| n/a | yes | +| [node\_count\_static](#input\_node\_count\_static) | The number of static nodes in node-pool | `number` | n/a | yes | +| [node\_pool\_names](#input\_node\_pool\_names) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `list(string)` | n/a | yes | +| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used in slurm controller | `string` | n/a | yes | +| [slurm\_controller\_instance](#input\_slurm\_controller\_instance) | Slurm cluster controller instance | `any` | n/a | yes | +| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | +| [subnetwork](#input\_subnetwork) | Primary subnetwork object | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset\_name](#output\_nodeset\_name) | Name of the new Slinky nodset | + diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/main.tf new file mode 100644 index 0000000000..8b2f1deeac --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/main.tf @@ -0,0 +1,64 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +### GKE NodeSet +locals { + manifest_path = "${path.module}/templates/nodeset-general.yaml.tftpl" +} + +module "kubectl_apply" { + source = "../../../../modules/management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + apply_manifests = [{ + source = local.manifest_path, + template_vars = { + slurm_namespace = var.slurm_namespace, + nodeset_name = "${var.slurm_cluster_name}-${var.nodeset_name}", + nodeset_cr_name = "${var.slurm_cluster_name}-${var.nodeset_name}", + controller_name = "${var.slurm_cluster_name}-controller", + node_pool_name = var.node_pool_names[0], + node_count = var.node_count_static, + image = var.image, + home_pvc = module.home_pv.pvc_name + slurm_key_pvc = module.slurm_key_pv.pvc_name + } + }] +} + +data "google_storage_bucket" "this" { + name = var.slurm_bucket[0].name + + depends_on = [var.slurm_bucket] +} + +### Slurm NodeSet +locals { + nodeset = { + gke_nodepool = var.node_pool_names[0] + nodeset_name = var.nodeset_name + node_count_static = var.node_count_static + subnetwork = "https://www.googleapis.com/compute/v1/projects/${var.project_id}/regions/${var.subnetwork.region}/subnetworks/${var.subnetwork.name}" + instance_template = var.instance_templates[0] + } +} + +resource "google_storage_bucket_object" "gke_nodeset_config" { + bucket = data.google_storage_bucket.this.name + name = "${var.slurm_bucket_dir}/nodeset_configs/${var.nodeset_name}.yaml" + content = yamlencode(local.nodeset) +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml new file mode 100644 index 0000000000..ea2cfc221e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/output.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/output.tf new file mode 100644 index 0000000000..15970ff0b7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/output.tf @@ -0,0 +1,18 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_name" { + description = "Name of the new Slinky nodset" + value = local.nodeset.nodeset_name +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf new file mode 100644 index 0000000000..8a190c4019 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf @@ -0,0 +1,50 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + slurm_key_storage = { + server_ip = var.slurm_controller_instance.network_interface[0].network_ip + remote_mount = "/slurm/key_distribution" # defined in /community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py + client_install_runner = {} + mount_runner = {} + fs_type = "" + local_mount = "" + mount_options = "" + } +} + +module "slurm_key_pv" { + source = "../../../../modules/file-system/gke-persistent-volume" + labels = {} + capacity_gib = 1 + cluster_id = var.cluster_id + filestore_id = "projects/empty/locations/empty/instances/empty" # this does not apply since this NFS is not a filestore + namespace = var.slurm_namespace + network_storage = local.slurm_key_storage + pv_name = "slurm-key-pv" + pvc_name = "slurm-key-pvc" +} + +# Assume the var.network_storage[0] will be home and only one home pv is accepted for now. +module "home_pv" { + source = "../../../../modules/file-system/gke-persistent-volume" + labels = {} + capacity_gib = 1024 + cluster_id = var.cluster_id + filestore_id = var.filestore_id[0] + network_storage = var.network_storage[0] + namespace = var.slurm_namespace + pv_name = "home-pv" + pvc_name = "home-pvc" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl new file mode 100644 index 0000000000..a5a4a5e7ac --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl @@ -0,0 +1,203 @@ +apiVersion: slinky.slurm.net/v1alpha1 +kind: NodeSet +metadata: + annotations: + meta.helm.sh/release-name: slurm + meta.helm.sh/release-namespace: ${slurm_namespace} + labels: + app.kubernetes.io/component: compute + app.kubernetes.io/instance: slurm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: slurmd + app.kubernetes.io/part-of: slurm + app.kubernetes.io/version: "24.11" + helm.sh/chart: slurm-0.3.0 + nodeset.slinky.slurm.net/name: ${nodeset_name} + name: ${nodeset_name} + namespace: ${slurm_namespace} +spec: + clusterName: slurm + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + replicas: ${node_count} + revisionHistoryLimit: 0 + selector: + matchLabels: + app.kubernetes.io/instance: slurm + app.kubernetes.io/name: slurmd + nodeset.slinky.slurm.net/name: ${nodeset_name} + serviceName: slurm-compute + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: slurmd + labels: + app.kubernetes.io/component: compute + app.kubernetes.io/instance: slurm + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: slurmd + app.kubernetes.io/part-of: slurm + app.kubernetes.io/version: "24.11" + helm.sh/chart: slurm-0.3.0 + nodeset.slinky.slurm.net/name: ${nodeset_name} + spec: + automountServiceAccountToken: false + containers: + - args: + - -g + - -- + - bash + - -c + - | + mkdir -p /usr/local/lib/slurm + ln -s /usr/lib/x86_64-linux-gnu/slurm/spank_pyxis.so /usr/local/lib/slurm/spank_pyxis.so + /usr/local/bin/entrypoint.sh -Z --conf-server ${controller_name}:6825 -N $NODE_NAME + command: + - tini + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_CPUS + value: "0" + - name: POD_MEMORY + value: "0" + image: ${image} + imagePullPolicy: IfNotPresent + name: slurmd + ports: + - containerPort: 6818 + name: slurmd + protocol: TCP + readinessProbe: + exec: + command: + - scontrol + - show + - slurmd + resources: {} + securityContext: + capabilities: + add: + - BPF + - NET_ADMIN + - SYS_ADMIN + - SYS_NICE + privileged: true + volumeMounts: + - mountPath: /etc/slurm + name: etc-slurm + - mountPath: /run + name: run + - mountPath: /var/spool/slurmd + name: slurm-spool + - mountPath: /var/log/slurm + name: slurm-log + - mountPath: /home + name: home-pvc + dnsConfig: + searches: + - ${controller_name} + hostNetwork: true + initContainers: + - command: + - tini + - -g + - -- + - bash + - -c + - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# + SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env + contains:\n# SLURM_USER - username or UID\n\nfunction init::common() {\n\tlocal + dir\n\n\tdir=/var/spool/slurmd\n\tmkdir -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" + \"$dir\"\n\tchmod -v 700 \"$dir\"\n\n\tdir=/var/spool/slurmctld\n\tmkdir + -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" \"$dir\"\n\tchmod + -v 700 \"$dir\"\n}\n\nfunction init::slurm() {\n\tSLURM_MOUNT=/mnt/slurm\n\tSLURM_DIR=/mnt/etc/slurm\n\n\t# + Workaround to ephemeral volumes not supporting securityContext\n\t# https://github.com/kubernetes/kubernetes/issues/81089\n\n\t# + Copy Slurm config files, secrets, and scripts\n\tmkdir -p \"$SLURM_DIR\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -name \"*.conf\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -name \"*.key\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind + \"$${SLURM_MOUNT}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" + -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind \"$${SLURM_MOUNT}\" -type + f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 | xargs + -0r cp -vt \"$${SLURM_DIR}\"\n\n\t# Set general permissions and ownership\n\tfind + \"$${SLURM_DIR}\" -type f -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind + \"$${SLURM_DIR}\" -type f -name \"*.conf\" -print0 | xargs -0r chmod -v 644\n\tfind + \"$${SLURM_DIR}\" -type f -name \"*.key\" -print0 | xargs -0r chmod -v 600\n\tfind + \"$${SLURM_DIR}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" + -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind \"$${SLURM_DIR}\" + -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 + | xargs -0r chmod -v 755\n\n\t# Inject secrets into certain config files\n\tlocal + dbd_conf=\"slurmdbd.conf\"\n\tif [[ -f \"$${SLURM_MOUNT}/$${dbd_conf}\" ]]; + then\n\t\techo \"Injecting secrets from environment into: $${dbd_conf}\"\n\t\trm + -f \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tenvsubst <\"$${SLURM_MOUNT}/$${dbd_conf}\" + >\"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" + \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchmod -v 600 \"$${SLURM_DIR}/$${dbd_conf}\"\n\tfi\n\n\t# + Display Slurm directory files\n\tls -lAF \"$${SLURM_DIR}\"\n}\n\nfunction + main() {\n\tinit::common\n\tinit::slurm\n}\nmain\n" + env: + - name: SLURM_USER + value: slurm + image: ${image} + imagePullPolicy: IfNotPresent + name: init + resources: {} + volumeMounts: + - mountPath: /mnt/slurm + name: slurm-config + - mountPath: /mnt/etc/slurm + name: etc-slurm + - command: + - tini + - -g + - -- + - bash + - -c + - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# + SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env + contains:\n# SOCKET - Named socket to read from\n\nmkdir -v -p \"$(dirname + \"$SOCKET\")\"\nrm -f \"$SOCKET\"\nif ! [ -f \"$SOCKET\" ]; then\n\tmkfifo + -m 777 \"$SOCKET\"\nfi\nwhile IFS=\"\" read data; do\n\techo $data\ndone + <\"$SOCKET\"\n" + env: + - name: SOCKET + value: /var/log/slurm/slurmd.log + image: ghcr.io/slinkyproject/sackd:24.11-ubuntu24.04 + imagePullPolicy: IfNotPresent + name: logfile + resources: {} + restartPolicy: Always + volumeMounts: + - mountPath: /var/log/slurm + name: slurm-log + nodeSelector: + cloud.google.com/gke-nodepool: ${node_pool_name} + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Equal + value: present + volumes: + - emptyDir: + medium: Memory + name: etc-slurm + - emptyDir: {} + name: run + - name: slurm-config + persistentVolumeClaim: + claimName: ${slurm_key_pvc} + - emptyDir: + medium: Memory + name: slurm-spool + - emptyDir: + medium: Memory + name: slurm-log + - name: home-pvc + persistentVolumeClaim: + claimName: ${home_pvc} + updateStrategy: + rollingUpdate: + maxUnavailable: 20% + type: RollingUpdate diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/variables.tf new file mode 100644 index 0000000000..c091a0da86 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/variables.tf @@ -0,0 +1,118 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" + type = string +} + +variable "slurm_cluster_name" { + type = string + description = "Cluster name, used in slurm controller" + + validation { + condition = var.slurm_cluster_name != null && can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) + error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." + } +} + +variable "slurm_controller_instance" { + type = any + description = "Slurm cluster controller instance" +} + +variable "image" { + description = "The image for slurm daemon" + type = string + nullable = false +} + +variable "node_pool_names" { + description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." + type = list(string) + nullable = false +} + +variable "node_count_static" { + description = "The number of static nodes in node-pool" + type = number +} + +variable "subnetwork" { + description = "Primary subnetwork object" + type = any +} + +variable "slurm_namespace" { + description = "slurm namespace for charts" + type = string + default = "slurm" +} + +variable "nodeset_name" { + description = "The nodeset name" + type = string + default = "gkenodeset" +} + +variable "slurm_bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + type = string + nullable = false +} + +variable "slurm_bucket" { + description = "GCS Bucket of Slurm cluster file storage." + type = any + nullable = true +} + +variable "instance_templates" { + description = "The URLs of Instance Templates" + type = list(string) + nullable = false +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + + validation { + condition = length(var.network_storage) == 1 && var.network_storage[0].local_mount == "/home" + error_message = "The 'network_storage' variable must contain exactly one element, and that element's 'local_mount' attribute must be \"/home\"." + } +} + +variable "filestore_id" { + description = "An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." + type = list(string) + + validation { + condition = length(var.filestore_id) == 1 + error_message = "The 'filestore_id' variable must contain exactly one element." + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/versions.tf new file mode 100644 index 0000000000..3d7237cb92 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-nodeset/v1.51.0" + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/README.md new file mode 100644 index 0000000000..2a7c363a87 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/README.md @@ -0,0 +1,39 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 4.84 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [has\_tpu](#input\_has\_tpu) | If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu. | `bool` | `false` | no | +| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | +| [partition\_name](#input\_partition\_name) | The partition name | `string` | `"gke"` | no | +| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/main.tf new file mode 100644 index 0000000000..2949fd6594 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/main.tf @@ -0,0 +1,47 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "google_storage_bucket" "this" { + name = var.slurm_bucket[0].name + + depends_on = [var.slurm_bucket] +} + +### Slurm Partition +locals { + partition_conf = { + "PowerDownOnIdle" = "NO" + "SuspendTime" = "INFINITE" + "SuspendTimeout" = var.has_tpu ? 240 : 120 + "ResumeTimeout" = var.has_tpu ? 600 : 300 + } + + partition = { + partition_name = var.partition_name + partition_conf = local.partition_conf + + partition_nodeset = [var.nodeset_name] + partition_nodeset_tpu = [] + partition_nodeset_dyn = [] + # Options + enable_job_exclusive = true + power_down_on_idle = false + } +} + +resource "google_storage_bucket_object" "parition_config" { + bucket = data.google_storage_bucket.this.name + name = "${var.slurm_bucket_dir}/partition_configs/${var.partition_name}.yaml" + content = yamlencode(local.partition) +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/metadata.yaml new file mode 100644 index 0000000000..557e1fc2ae --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/variables.tf new file mode 100644 index 0000000000..3aeed2e59a --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/variables.tf @@ -0,0 +1,43 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "has_tpu" { + description = "If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu." + type = bool + default = false +} + +variable "nodeset_name" { + description = "The nodeset name" + type = string + default = "gkenodeset" +} + +variable "partition_name" { + description = "The partition name" + type = string + default = "gke" +} + +variable "slurm_bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + type = string + nullable = false +} + +variable "slurm_bucket" { + description = "GCS Bucket of Slurm cluster file storage." + type = any + nullable = true +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/versions.tf new file mode 100644 index 0000000000..aede55263c --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-partition/v1.51.0" + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md new file mode 100644 index 0000000000..4f65411ddf --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md @@ -0,0 +1,271 @@ +## Description + +This module performs the following tasks: + +- create an instance template from which execute points will be created +- create a managed instance group ([MIG][mig]) for execute points +- create a Toolkit runner to configure the autoscaler to scale the MIG + +It is expected to be used with the [htcondor-install] and [htcondor-setup] +modules. + +[htcondor-install]: ../../scripts/htcondor-install/README.md +[htcondor-setup]: ../../scheduler/htcondor-setup/README.md +[mig]: https://cloud.google.com/compute/docs/instance-groups/ + +### Known limitations + +This module may be used multiple times in a blueprint to create sets of +execute points in an HTCondor pool. If used more than 1 time, the setting +[name_prefix](#input_name_prefix) must be set to a value that is unique across +all uses of the htcondor-execute-point module. If you do not follow this +constraint, you will likely receive an error while running `terraform apply` +similar to that shown below. + +```text +Error: Invalid value for variable + + on modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf line 136, in module "startup_script": + 136: runners = local.all_runners + ├──────────────── + │ var.runners is list of map of string with 5 elements + +All startup-script runners must have a unique destination. +``` + +### How to configure jobs to select execute points + +HTCondor access points provisioned by the Toolkit are specially configured to +honor an attribute named `RequireId` in each [Job ClassAd][jobad]. This value +must be set to the ID of a MIG created by an instance of this module. The +[htcondor-access-point] module includes a setting `var.default_mig_id` that will +set this value automatically to the MIG ID corresponding to the module's +execute points. If this setting is left unset each job must specify `+RequireId` +explicitly. In all cases, the default value can be overridden explicitly as shown +below: + +```text +universe = vanilla +executable = /bin/echo +arguments = "Hello, World!" +output = out.$(ClusterId).$(ProcId) +error = err.$(ClusterId).$(ProcId) +log = log.$(ClusterId).$(ProcId) +request_cpus = 1 +request_memory = 100MB ++RequireId = "htcondor-pool-ep-mig" +queue +``` + +[htcondor-access-point]: ../../scheduler/htcondor-access-point/README.md +[jobad]: https://htcondor.readthedocs.io/en/latest/users-manual/matchmaking-with-classads.html + +### Example + +A full example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +The following code snippet creates a pool with 2 sets of HTCondor execute +points, one using On-demand pricing and the other using Spot pricing. They use +a startup script and network created in previous steps. + +```yaml +- id: htcondor_execute_point + source: community/modules/compute/htcondor-execute-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + min_idle: 2 + +- id: htcondor_execute_point_spot + source: community/modules/compute/htcondor-execute-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + spot: true + +- id: htcondor_access + source: community/modules/scheduler/htcondor-access-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + - htcondor_execute_point + - htcondor_execute_point_spot + settings: + default_mig_id: $(htcondor_execute_point.mig_id) + enable_public_ips: true + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - access_point_ips + - access_point_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## Behavior of Managed Instance Group (MIG) + +Regional [MIGs][mig] are used to provision Execute Points. By default, VMs +will be provisioned in any of the zones available in that region, however, it +can be constrained to run in fewer zones (or a single zone) using +[var.zones](#input_zones). + +When the configuration of an Execute Point is changed, the MIG can be configured +to [replace the VM][replacement] using a "proactive" or "opportunistic" policy. +By default, the policy is set to opportunistic. In practice, this means that +Execute Points will _NOT_ be automatically replaced by Terraform when changes to +the instance template / HTCondor configuration are made. We recommend leaving +this at the default value as it will allow the HTCondor autoscaler to replace +VMs when they become idle without disrupting running jobs. + +However, if it is desired [var.update_policy](#input_update_policy) can be set +to "PROACTIVE" to enable automatic replacement. This will disrupt running jobs +and send them back to the queue. Alternatively, one can leave the setting at +the default value of "OPPORTUNISTIC" and update: + +- intentionally by issuing an update via Cloud Console or using gcloud (below) +- VMs becomes unhealthy or are otherwise automatically replaced (e.g. regular + Google Cloud maintenance) + +For example, to manually update all instances in a MIG: + +```text +gcloud compute instance-groups managed update-instances \ + <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + +## Known Issues + +When using OS Login with "external users" (outside of the Google Cloud +organization), then Docker universe jobs will fail and cause the Docker daemon +to crash. This stems from the use of POSIX user ids (uid) outside the range +supported by Docker. Please consider disabling OS Login if this atypical +situation applies. + +```yaml +vars: + # add setting below to existing deployment variables + enable_oslogin: DISABLE +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 4.0 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.0 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [execute\_point\_instance\_template](#module\_execute\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [mig](#module\_mig) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.execute_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.execute_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Disk type for template | `string` | `"pd-balanced"` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape across zones for instance group managing execute points | `string` | `"ANY"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [execute\_point\_runner](#input\_execute\_point\_runner) | A list of Toolkit runners for configuring an HTCondor execute point | `list(map(string))` | `[]` | no | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account for HTCondor execute point (e-mail format) | `string` | n/a | yes | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | HTCondor execute point VM image

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to HTConodr execute points | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor execute points | `string` | `"n2-standard-4"` | no | +| [max\_size](#input\_max\_size) | Maximum size of the HTCondor execute point pool. | `number` | `5` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor execute points | `map(string)` | `{}` | no | +| [min\_idle](#input\_min\_idle) | Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max\_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly. | `number` | `0` | no | +| [name\_prefix](#input\_name\_prefix) | Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network HTCondor execute points will join | `string` | `"default"` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HTCondor execute points will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which HTCondor execute points will be created | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork HTCondor execute points will join | `string` | `null` | no | +| [target\_size](#input\_target\_size) | Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size. | `number` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | +| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | Startup script to run at boot-time for Windows-based HTCondor execute points | `list(string)` | `[]` | no | +| [zones](#input\_zones) | Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [autoscaler\_runner](#output\_autoscaler\_runner) | Toolkit runner to configure the HTCondor autoscaler | +| [mig\_id](#output\_mig\_id) | ID of the managed instance group containing the execute points | + diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml new file mode 100644 index 0000000000..375ae036cd --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml @@ -0,0 +1,74 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Role + hosts: localhost + become: true + vars: + spool_dir: /var/lib/condor/spool + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + handlers: + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml new file mode 100644 index 0000000000..a85158fdfc --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml @@ -0,0 +1,98 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This playbook makes the assumption that a virtual environment has been created +# with the autoscaler and its dependencies previously installed. A runner that +# does this is provided as an output of the htcondor-install module within the +# Cluster Toolkit at community/modules/scripts/htcondor-install. + +--- +- name: Configure HTCondor Autoscaler + hosts: all + vars: + python: /usr/local/htcondor/bin/python3 + autoscaler: /usr/local/htcondor/bin/autoscaler.py + systemd_override_path: /etc/systemd/system + become: true + tasks: + - name: User must supply HTCondor role + ansible.builtin.assert: + that: + - project_id is defined + - region is defined + - zone is defined + - mig_id is defined + - max_size is defined + - name: Create SystemD service for HTCondor autoscaler + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@.service" + mode: 0644 + content: | + [Unit] + Description=HTCondor Autoscaler MIG: %i + + [Service] + User=condor + Type=oneshot + ExecStart={{ python }} {{ autoscaler }} --p $PROJECT_ID --r $REGION --z $ZONE --mz --g %i --c $MAX_SIZE --i $MIN_IDLE + notify: + - Reload SystemD + - name: Create SystemD override directory for autoscaler configuration + ansible.builtin.file: + path: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d" + state: directory + owner: root + group: root + mode: 0755 + - name: Create autoscaler configuration + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d/miglimit.conf" + mode: 0644 + content: | + [Service] + Environment=PROJECT_ID={{ project_id }} + Environment=REGION={{ region }} + Environment=ZONE={{ zone }} + Environment=MAX_SIZE={{ max_size }} + Environment=MIN_IDLE={{ min_idle }} + notify: + - Reload SystemD + - name: Create SystemD timer for HTCondor autoscaler + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@.timer" + mode: 0644 + content: | + [Unit] + Description=Run HTCondor Autoscaler Periodically + + [Timer] + OnCalendar=minutely + AccuracySec=1us + RandomizedDelaySec=30 + # the directive below is ignored harmlessly on CentOS 7; this has impact + # that timing averages to 1 minute but is not precisely 1 minute; still + # useful to ensure that timers for different MIGs do not overlap + FixedRandomDelay=true + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + post_tasks: + - name: Activate HTCondor Autoscaler timer + ansible.builtin.systemd: + name: htcondor-autoscaler@{{ mig_id }}.timer + enabled: true + state: started diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf new file mode 100644 index 0000000000..7b0df94987 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf @@ -0,0 +1,218 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-execute-point", ghpc_role = "compute" }) +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + zones = coalescelist(var.zones, data.google_compute_zones.available.names) + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + windows_startup_ps1 = join("\n\n", flatten([var.windows_startup_ps1, local.execute_config_windows_startup_ps1])) + + is_windows_image = anytrue([for l in data.google_compute_image.compute_image.licenses : length(regexall("windows-cloud", l)) > 0]) + windows_startup_metadata = local.is_windows_image && local.windows_startup_ps1 != "" ? { + windows-startup-script-ps1 = local.windows_startup_ps1 + } : {} + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.windows_startup_metadata, + local.network_storage_metadata, + local.enable_oslogin, + local.disable_automatic_updates_metadata, + var.metadata + ) + + autoscaler_runner = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_configure_autoscaler.yml") + "destination" = "htcondor_configure_autoscaler_${module.mig.instance_group_manager.name}.yml" + "args" = join(" ", [ + "-e project_id=${var.project_id}", + "-e region=${var.region}", + "-e zone=${local.zones[0]}", # this value is required, but ignored by regional MIG autoscaler + "-e mig_id=${module.mig.instance_group_manager.name}", + "-e max_size=${var.max_size}", + "-e min_idle=${var.min_idle}", + ]) + } + + execute_config = templatefile("${path.module}/templates/condor_config.tftpl", { + htcondor_role = "get_htcondor_execute", + central_manager_ips = var.central_manager_ips, + guest_accelerator = local.guest_accelerator, + }) + + execute_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.execute_config.output_name}" + execute_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e htcondor_role=get_htcondor_execute", + "-e config_object=${local.execute_object}", + ]) + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.execute_point_runner, + [local.execute_runner], + ) + + execute_config_windows_startup_ps1 = templatefile( + "${path.module}/templates/download-condor-config.ps1.tftpl", + { + config_object = local.execute_object, + } + ) + + name_prefix = "${var.deployment_name}-${var.name_prefix}-ep" +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +resource "null_resource" "execute_config" { + triggers = { + config = local.execute_config + } +} + +resource "google_storage_bucket_object" "execute_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.execute_config.id), 0, 4)}" + content = local.execute_config + bucket = var.htcondor_bucket_name +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +module "execute_point_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.execute_point_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + gpu = one(local.guest_accelerator) + preemptible = var.spot + startup_script = local.is_windows_image ? null : module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.compute_image.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config +} + +module "mig" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = var.target_size + hostname = local.name_prefix + mig_name = local.name_prefix + instance_template = module.execute_point_instance_template.self_link + + health_check_name = "health-htcondor-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "SUBSTITUTE" + max_surge_fixed = length(local.zones) + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf new file mode 100644 index 0000000000..b31f40130f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "autoscaler_runner" { + value = local.autoscaler_runner + description = "Toolkit runner to configure the HTCondor autoscaler" +} + +output "mig_id" { + value = module.mig.instance_group_manager.name + description = "ID of the managed instance group containing the execute points" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl new file mode 100644 index 0000000000..c8f5ce31a8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl @@ -0,0 +1,31 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:${htcondor_role} +CONDOR_HOST = ${join(",", central_manager_ips)} + +# StartD configuration settings +%{ if length(guest_accelerator) > 0 ~} +use feature:GPUs +%{ endif ~} +use feature:PartitionableSlot +use feature:CommonCloudAttributesGoogle("-c created-by") +UPDATE_INTERVAL = 30 +TRUST_UID_DOMAIN = True +STARTER_ALLOW_RUNAS_OWNER = True +RUNBENCHMARKS = False diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl new file mode 100644 index 0000000000..19789f122e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl @@ -0,0 +1,34 @@ +# create directory for local condor_config customizations +$config_dir = 'C:\Condor\config' +if(!(test-path -PathType container -Path $config_dir)) +{ + New-Item -ItemType Directory -Path $config_dir +} + +# update local condor_config if blueprint has changed +$config_file = "$config_dir\50-ghpc-managed" +if (Test-Path -Path $config_file -PathType Leaf) +{ + $local_hash = gcloud --format="value(md5_hash)" storage hash $config_file +} +else +{ + $local_hash = "INVALID-HASH" +} + +$remote_hash = gcloud --format="value(md5_hash)" storage hash ${config_object} +if ($local_hash -cne $remote_hash) +{ + Write-Output "Updating condor configuration" + gcloud storage cp ${config_object} $config_file + if ($LASTEXITCODE -ne 0) + { + throw "Could not download HTCondor configuration; exiting startup script" + } + Restart-Service condor +} + +# ignored if service is already running; must be here to handle case where +# machine is rebooted, but configuration has previously been downloaded +# and service is disabled from automatic start +Start-Service condor diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf new file mode 100644 index 0000000000..aab8a54c2d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf @@ -0,0 +1,265 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HTCondor execute points will be created" + type = string +} + +variable "region" { + description = "The region in which HTCondor execute points will be created" + type = string +} + +variable "zones" { + description = "Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region." + type = list(string) + default = [] + nullable = false +} + +variable "distribution_policy_target_shape" { + description = "Target shape across zones for instance group managing execute points" + type = string + default = "ANY" +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to HTConodr execute points" + type = map(string) +} + +variable "machine_type" { + description = "Machine type to use for HTCondor execute points" + type = string + default = "n2-standard-4" +} + +variable "execute_point_runner" { + description = "A list of Toolkit runners for configuring an HTCondor execute point" + type = list(map(string)) + default = [] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "instance_image" { + description = <<-EOD + HTCondor execute point VM image + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "execute_point_service_account_email" { + description = "Service account for HTCondor execute point (e-mail format)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_self_link" { + description = "The self link of the network HTCondor execute points will join" + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork HTCondor execute points will join" + type = string + default = null +} + +variable "target_size" { + description = "Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size." + type = number + default = null +} + +variable "max_size" { + description = "Maximum size of the HTCondor execute point pool." + type = number + default = 5 +} + +variable "min_idle" { + description = "Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly." + type = number + default = 0 +} + +variable "metadata" { + description = "Metadata to add to HTCondor execute points" + type = map(string) + default = {} +} + +# this default is deliberately the opposite of vm-instance because of observed +# issues running HTCondor docker universe jobs with OS Login enabled and running +# jobs as a user with uid>2^31; these uids occur when users outside the GCP +# organization login to a VM and OS Login is enabled. +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "spot" { + description = "Provision VMs using discounted Spot pricing, allowing for preemption" + type = bool + default = false +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 100 +} + +variable "disk_type" { + description = "Disk type for template" + type = string + default = "pd-balanced" +} + +variable "windows_startup_ps1" { + description = "Startup script to run at boot-time for Windows-based HTCondor execute points" + type = list(string) + default = [] + nullable = false +} + +variable "central_manager_ips" { + description = "List of IP addresses of HTCondor Central Managers" + type = list(string) +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The HTCondor module supports 0 or 1 models of accelerator card on each execute point" + } +} + +variable "name_prefix" { + description = "Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module" + type = string + nullable = false + validation { + condition = length(var.name_prefix) > 0 + error_message = "var.name_prefix must be a set to a non-empty string and must also be unique across all instances of htcondor-execute-point" + } +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" + type = string + default = "OPPORTUNISTIC" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf new file mode 100644 index 0000000000..729dc3cda5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.1" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.74.0" + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/mig/README.md new file mode 100644 index 0000000000..278207b04a --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/mig/README.md @@ -0,0 +1,45 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_group_manager.mig](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_group_manager) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [base\_instance\_name](#input\_base\_instance\_name) | Base name for the instances in the MIG | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will be used to name MIG if `var.name` is not provided | `string` | n/a | yes | +| [ghpc\_module\_id](#input\_ghpc\_module\_id) | Internal GHPC field, do not set this value | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the MIG | `map(string)` | n/a | yes | +| [name](#input\_name) | Name of the MIG. If not provided, will be generated from `var.deployment_name` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the MIG will be created | `string` | n/a | yes | +| [target\_size](#input\_target\_size) | Target number of instances in the MIG | `number` | `0` | no | +| [versions](#input\_versions) | Application versions managed by this instance group. Each version deals with a specific instance template | 
list(object({
    name              = string
    instance_template = string
    target_size = optional(object({
      fixed   = optional(number)
      percent = optional(number)
    }))
  }))
| n/a | yes | +| [wait\_for\_instances](#input\_wait\_for\_instances) | Whether to wait for all instances to be created/updated before returning | `bool` | `false` | no | +| [zone](#input\_zone) | Compute Platform zone. Required, currently only zonal MIGs are supported | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [self\_link](#output\_self\_link) | The URL of the created MIG | + diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/mig/main.tf new file mode 100644 index 0000000000..0e7cf186c2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/mig/main.tf @@ -0,0 +1,85 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "mig", ghpc_role = "compute" }) +} + +locals { + sanitized_deploy_name = try(replace(lower(var.deployment_name), "/[^a-z0-9]/", ""), null) + sanitized_module_id = try(replace(lower(var.ghpc_module_id), "/[^a-z0-9]/", ""), null) + synth_mig_name = try("${local.sanitized_deploy_name}-${local.sanitized_module_id}", null) + + mig_name = var.name == null ? local.synth_mig_name : var.name + base_instance_name = var.base_instance_name == null ? local.mig_name : var.base_instance_name +} + +resource "google_compute_instance_group_manager" "mig" { + # REQUIRED + name = local.mig_name + base_instance_name = local.base_instance_name + zone = var.zone + + dynamic "version" { + for_each = var.versions + content { + name = version.value.name + instance_template = version.value.instance_template + dynamic "target_size" { + for_each = version.value.target_size != null ? [version.value.target_size] : [] + content { + fixed = target_size.value.fixed + percent = target_size.value.percent + } + } + } + } + + # OPTIONAL + project = var.project_id + target_size = var.target_size + wait_for_instances = var.wait_for_instances + + all_instances_config { + # TODO: validate that template metadata not getting wiped out + # TODO: validate that template labels not getting wiped out + labels = local.labels + } + + # OMITTED: + # * description + # * named_port + # * list_managed_instances_results + # * target_pools - specific for Load Balancers usage + # * wait_for_instances_status + # * auto_healing_policies + # * stateful_disk + # * stateful_internal_ip + # * update_policy + # * params + + + lifecycle { + precondition { + condition = local.mig_name != null + error_message = "Could not come up with a name for the MIG, specify `var.name`" + } + + precondition { + condition = local.base_instance_name != null + error_message = "Could not come up with a base_instance_name, specify `var.base_instance_name`" + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/mig/metadata.yaml new file mode 100644 index 0000000000..97a4fa9a89 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/mig/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com +ghpc: + inject_module_id: ghpc_module_id diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/mig/outputs.tf new file mode 100644 index 0000000000..23c66a3535 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/mig/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "self_link" { + description = "The URL of the created MIG" + value = google_compute_instance_group_manager.mig.self_link +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/mig/variables.tf new file mode 100644 index 0000000000..b6c3c0e78a --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/mig/variables.tf @@ -0,0 +1,86 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Project in which the MIG will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the deployment, will be used to name MIG if `var.name` is not provided" + type = string +} + +variable "labels" { + description = "Labels to add to the MIG" + type = map(string) +} + +variable "zone" { + description = "Compute Platform zone. Required, currently only zonal MIGs are supported" + type = string +} + + +variable "versions" { + description = <<-EOD + Application versions managed by this instance group. Each version deals with a specific instance template + EOD + type = list(object({ + name = string + instance_template = string + target_size = optional(object({ + fixed = optional(number) + percent = optional(number) + })) + })) + + validation { + condition = length(var.versions) > 0 + error_message = "At least one version must be provided" + } + +} + + +variable "ghpc_module_id" { + description = "Internal GHPC field, do not set this value" + type = string + default = null +} + +variable "name" { + description = "Name of the MIG. If not provided, will be generated from `var.deployment_name`" + type = string + default = null +} + +variable "base_instance_name" { + description = "Base name for the instances in the MIG" + type = string + default = null +} + + +variable "target_size" { + description = "Target number of instances in the MIG" + type = number + default = 0 +} + +variable "wait_for_instances" { + description = "Whether to wait for all instances to be created/updated before returning" + type = bool + default = false +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/mig/versions.tf new file mode 100644 index 0000000000..4147447b44 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/mig/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = "> 5.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:mig/v1.74.0" + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/README.md new file mode 100644 index 0000000000..1dcacc57e9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/README.md @@ -0,0 +1,112 @@ +# Description + +This module creates the Vertex AI Notebook, to be used in tutorials. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage + +This is a simple usage, using the default network: + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: + name_prefix: my-bucket + local_mount: /home/jupyter/my-bucket + + - id: notebook + source: community/modules/compute/notebook + use: [bucket] + settings: + name_prefix: notebook + machine_type: n1-standard-4 + +``` + +If the user wants do specify a custom subnetwork, or specific external IP restrictions, they can use the `network_interfaces` variable, here is an example on how to use a Shared VPC Subnet with an ephemeral external IP: + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: + name_prefix: my-bucket + local_mount: /home/jupyter/my-bucket + + - id: notebook + source: community/modules/compute/notebook + use: [bucket] + settings: + name_prefix: notebook + machine_type: n1-standard-4 + network_interfaces: + - network: "projects/HOST_PROJECT_ID/global/networks/SHARED_VPC_NAME" + subnet: "projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME" + nic_type: "VIRTIO_NET" +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 5.34 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 5.34 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.mount_script](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_workbench_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/workbench_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the notebook. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name, can be provided from the google-cloud-storage module | `string` | `null` | no | +| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "tf-latest-cpu",
  "name": null,
  "project": "deeplearning-platform-release"
}
| no | +| [labels](#input\_labels) | Labels to add to the resource Key-value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | The machine type to employ | `string` | n/a | yes | +| [mount\_runner](#input\_mount\_runner) | mount content from the google-cloud-storage module | `map(string)` | n/a | yes | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces for the VM instance. Each network interface is represented by an object with the following fields:

- network: (Optional) The name of the Virtual Private Cloud (VPC) network that this VM instance is connected to.

- subnet: (Optional) The name of the subnetwork within the specified VPC that this VM instance is connected to.

- nic\_type: (Optional) The type of vNIC to be used on this interface. Possible values are: `VIRTIO_NET`, `GVNIC`.

- access\_configs: (Optional) An array of access configurations for this network interface. The access\_config object contains:
* external\_ip: (Required) An external IP address associated with this instance. Specify an unused static external IP address available to the project or leave this field undefined to use an IP from a shared ephemeral IP address pool. If you specify a static external IP address, it must live in the same region as the zone of the instance. | 
list(object({
    network  = optional(string)
    subnet   = optional(string)
    nic_type = optional(string)
    access_configs = optional(list(object({
      external_ip = optional(string)
    })))
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | ID of project in which the notebook will be created. | `string` | n/a | yes | +| [service\_account\_email](#input\_service\_account\_email) | If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account | `string` | `null` | no | +| [zone](#input\_zone) | The zone to deploy to | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/main.tf new file mode 100644 index 0000000000..cd3ce3b4ea --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/main.tf @@ -0,0 +1,96 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "notebook", ghpc_role = "compute" }) +} + +locals { + suffix = random_id.resource_name_suffix.hex + #name = "thenotebook" + name = "notebook-${var.deployment_name}-${local.suffix}" + bucket = replace(var.gcs_bucket_path, "gs://", "") + post_script_filename = "mount-${local.suffix}.sh" + + # mount_runner_args is defined in the file: cluster-toolkit/modules/file-system/cloud-storage-bucket/outputs.tf + mount_args = split(" ", var.mount_runner.args) + + unused = local.mount_args[0] + remote_mount = local.mount_args[1] + local_mount = local.mount_args[2] + fs_type = local.mount_args[3] + # These options provide a "rw" mount of the GCS bucket + mount_options = "defaults,_netdev,allow_other,implicit_dirs,gid=1000,uid=1000" + + content0 = var.mount_runner.content + content1 = replace(local.content0, "$1", local.unused) + content2 = replace(local.content1, "$2", local.remote_mount) + content3 = replace(local.content2, "$3", local.local_mount) + content4 = replace(local.content3, "$4", local.fs_type) + content5 = replace(local.content4, "$5", local.mount_options) + +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket_object" "mount_script" { + name = local.post_script_filename + content = local.content5 + bucket = local.bucket +} + +resource "google_workbench_instance" "instance" { + name = local.name + location = var.zone + project = var.project_id + labels = local.labels + gce_setup { + machine_type = var.machine_type + metadata = { + post-startup-script = "${var.gcs_bucket_path}/${google_storage_bucket_object.mount_script.name}" + } + vm_image { + project = var.instance_image.project + family = var.instance_image.family + } + + dynamic "service_accounts" { + for_each = var.service_account_email == null ? [] : [1] + content { + email = var.service_account_email + } + } + + dynamic "network_interfaces" { + for_each = var.network_interfaces + content { + network = network_interfaces.value.network + subnet = network_interfaces.value.subnet + nic_type = network_interfaces.value.nic_type + + dynamic "access_configs" { + for_each = network_interfaces.value.access_configs != null ? network_interfaces.value.access_configs : [] + content { + external_ip = access_configs.value.external_ip + } + } + } + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml new file mode 100644 index 0000000000..4a7d5397ca --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - notebooks.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/variables.tf new file mode 100644 index 0000000000..4359de8c10 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/variables.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which the notebook will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment; used as part of name of the notebook." + type = string + # notebook name can have: lowercase letters, numbers, or hyphens (-) and cannot end with a hyphen + validation { + error_message = "The notebook name uses 'deployment_name' -- can only have: lowercase letters, numbers, or hyphens" + condition = can(regex("^[a-z0-9]+(?:-[a-z0-9]+)*$", var.deployment_name)) + } +} + +variable "zone" { + description = "The zone to deploy to" + type = string +} + +variable "machine_type" { + description = "The machine type to employ" + type = string +} + +variable "labels" { + description = "Labels to add to the resource Key-value pairs." + type = map(string) +} + +variable "instance_image" { + description = "Instance Image" + type = map(string) + default = { + project = "deeplearning-platform-release" + family = "tf-latest-cpu" + name = null + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "gcs_bucket_path" { + description = "Bucket name, can be provided from the google-cloud-storage module" + type = string + default = null +} + +variable "mount_runner" { + description = "mount content from the google-cloud-storage module" + type = map(string) + + validation { + condition = (length(split(" ", var.mount_runner.args)) == 5) + error_message = "There must be 5 elements in the Mount Runner Arguments: ${var.mount_runner.args} \n " + } +} + +variable "service_account_email" { + description = "If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account" + type = string + default = null +} + +variable "network_interfaces" { + type = list(object({ + network = optional(string) + subnet = optional(string) + nic_type = optional(string) + access_configs = optional(list(object({ + external_ip = optional(string) + }))) + })) + default = [] + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | +| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name    = string
    device_name  = string
    disk_size_gb = number
    disk_type    = string
    disk_labels  = map(string)
    auto_delete  = bool
    boot         = bool
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | +| [feature](#input\_feature) | The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used. | `string` | `null` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | Path to the Slurm bucket. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster. | `string` | n/a | yes | +| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_template\_self\_link](#output\_instance\_template\_self\_link) | The URI of the template. | +| [node\_name\_prefix](#output\_node\_name\_prefix) | The prefix to be used for the node names.

Make sure that nodes are named `-`
This temporary required for proper functioning of the nodes.
While Slurm scheduler uses "features" to bind node and nodeset,
the SlurmGCP relies on node names for this (to be switched to features as well). | +| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf new file mode 100644 index 0000000000..31d9f14ae7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf @@ -0,0 +1,128 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset-dynamic", ghpc_role = "compute" }) +} + +module "instance_validation" { + source = "../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + nodeset_name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + feature = coalesce(var.feature, local.nodeset_name) + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + universe_domain = { "universe_domain" = var.universe_domain } + + metadata = merge( + local.disable_automatic_updates_metadata, + local.universe_domain, + { slurmd_feature = local.feature }, + var.metadata + ) + + nodeset = { + nodeset_name = local.nodeset_name + nodeset_feature : local.feature + startup_script = local.ghpc_startup_script + network_storage = var.network_storage + } + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + } + ] + + public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + ghpc_startup_script = [{ + filename = "ghpc_nodeset_startup.sh" + content = var.startup_script + }] + +} + +module "slurm_nodeset_template" { + source = "../../internal/slurm-gcp/instance_template" + + project_id = var.project_id + region = var.region + name_prefix = local.nodeset_name + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = "compute" + slurm_bucket_path = var.slurm_bucket_path + metadata = local.metadata + + additional_disks = local.additional_disks + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(local.labels, var.disk_labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + + bandwidth_tier = var.bandwidth_tier + can_ip_forward = var.can_ip_forward + + advanced_machine_features = var.advanced_machine_features + enable_confidential_vm = var.enable_confidential_vm + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + labels = local.labels + machine_type = var.machine_type + + min_cpu_platform = var.min_cpu_platform + on_host_maintenance = var.on_host_maintenance + termination_action = try(var.spot_instance_config.termination_action, null) + preemptible = var.preemptible + spot = var.enable_spot_vm + service_account = local.service_account + gpu = one(local.guest_accelerator) # requires gpu_definition.tf + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + subnetwork = var.subnetwork_self_link + additional_networks = var.additional_networks + access_config = local.access_config + tags = concat([var.slurm_cluster_name], var.tags) +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml new file mode 100644 index 0000000000..a99e59d09f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [compute.googleapis.com] +ghpc: + inject_module_id: name diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf new file mode 100644 index 0000000000..2d2d1415cf --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_dyn" { + description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset +} + +output "instance_template_self_link" { + description = "The URI of the template." + value = module.slurm_nodeset_template.self_link +} + +output "node_name_prefix" { + description = <<-EOD + The prefix to be used for the node names. + + Make sure that nodes are named `-` + This temporary required for proper functioning of the nodes. + While Slurm scheduler uses "features" to bind node and nodeset, + the SlurmGCP relies on node names for this (to be switched to features as well). + EOD + value = "${var.slurm_cluster_name}-${local.nodeset_name}" + +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf new file mode 100644 index 0000000000..ec6206e317 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf @@ -0,0 +1,402 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "feature" { + type = string + description = "The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used." + default = null +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "slurm_cluster_name" { + description = "Name of the Slurm cluster." + type = string +} + +variable "slurm_bucket_path" { + description = "Path to the Slurm bucket." + type = string +} + + +variable "machine_type" { + description = "Compute Platform machine type to use for this partition compute nodes." + type = string + default = "c2-standard-60" +} + +variable "metadata" { + type = map(string) + description = "Metadata, provided as a map." + default = {} +} + +variable "instance_image" { + description = <<-EOD + Defines the image that will be used in the Slurm node group VM instances. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + + For more information on creating custom images that comply with Slurm on GCP + see the "Slurm on GCP Custom Images" section in docs/vm-images.md. + EOD + type = map(string) + default = { + family = "slurm-gcp-6-11-hpc-rocky-linux-8" + project = "schedmd-slurm-public" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations + description = <<-EOD + A flag that designates that the user is aware that they are requesting + to use a custom and potentially incompatible image for this Slurm on + GCP module. + + If the field is set to false, only the compatible families and project + names will be accepted. The deployment will fail with any other image + family or name. If set to true, no checks will be done. + + See: https://goo.gle/hpc-slurm-images + EOD + type = bool + default = false +} + + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "tags" { + type = list(string) + description = "Network tag list." + default = [] +} + +variable "disk_type" { + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + type = string + default = "pd-standard" +} + +variable "disk_size_gb" { + description = "Size of boot disk to create for the partition compute nodes." + type = number + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "additional_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = list(object({ + disk_name = string + device_name = string + disk_size_gb = number + disk_type = string + disk_labels = map(string) + auto_delete = bool + boot = bool + })) + default = [] +} + +variable "enable_confidential_vm" { + type = bool + description = "Enable the Confidential VM configuration. Note: the instance image must support option." + default = false +} + +variable "enable_shielded_vm" { + type = bool + description = "Enable the Shielded VM configuration. Note: the instance image must support option." + default = false +} + +variable "shielded_instance_config" { + type = object({ + enable_integrity_monitoring = bool + enable_secure_boot = bool + enable_vtpm = bool + }) + description = <<-EOD + Shielded VM configuration for the instance. Note: not used unless + enable_shielded_vm is 'true'. + - enable_integrity_monitoring : Compare the most recent boot measurements to the + integrity policy baseline and return a pair of pass/fail results depending on + whether they match or not. + - enable_secure_boot : Verify the digital signature of all boot components, and + halt the boot process if signature verification fails. + - enable_vtpm : Use a virtualized trusted platform module, which is a + specialized computer chip you can use to encrypt objects like keys and + certificates. + EOD + default = { + enable_integrity_monitoring = true + enable_secure_boot = true + enable_vtpm = true + } +} + + +variable "enable_oslogin" { + type = bool + description = <<-EOD + Enables Google Cloud os-login for user login and authentication for VMs. + See https://cloud.google.com/compute/docs/oslogin + EOD + default = true +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example." + type = bool + default = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "labels" { + description = "Labels to add to partition compute instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "The name of the minimum CPU platform that you want the instance to use." + type = string + default = null +} + +variable "on_host_maintenance" { + type = string + description = <<-EOD + Instance availability Policy. + + Note: Placement groups are not supported when on_host_maintenance is set to + "MIGRATE" and will be deactivated regardless of the value of + enable_placement. To support enable_placement, ensure on_host_maintenance is + set to "TERMINATE". + EOD + default = "TERMINATE" +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." + } +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + + +variable "service_account_email" { + description = "Service account e-mail address to attach to the compute instances." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the compute instances." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "enable_spot_vm" { + description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." + type = bool + default = false +} + +variable "spot_instance_config" { + description = "Configuration for spot VMs." + type = object({ + termination_action = string + }) + default = null +} + +variable "bandwidth_tier" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | +| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | +| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | +| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `null` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling worker nodes allowed in this partition.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of worker nodes to be statically created.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | +| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. | `string` | `""` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the TPU-vm. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the TPU-vm. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | +| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | `"2.14.0"` | no | +| [zone](#input\_zone) | Zone in which to create compute VMs. TPU partitions can only specify a single zone. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf new file mode 100644 index 0000000000..ac9b119702 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# locals { +# # This label allows for billing report tracking based on module. +# labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) +# } + +locals { + name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + nodeset_tpu = { + node_count_static = var.node_count_static + node_count_dynamic_max = var.node_count_dynamic_max + nodeset_name = local.name + node_type = var.node_type + + accelerator_config = var.accelerator_config + tf_version = var.tf_version + preemptible = var.preemptible + preserve_tpu = var.preserve_tpu + + data_disks = var.data_disks + docker_image = var.docker_image + + enable_public_ip = var.enable_public_ips + # TODO: rename to subnetwork_self_link, requires changes to the scripts + subnetwork = var.subnetwork_self_link + service_account = local.service_account + zone = var.zone + + project_id = var.project_id + reserved = var.reserved + network_storage = var.network_storage + } + + node_type_core_count = var.node_type == "" ? 0 : tonumber(regex("-(.*)", var.node_type)[0]) + + accelerator_core_list = var.accelerator_config.topology == "" ? [0, 0] : regexall("\\d+", var.accelerator_config.topology) + accelerator_core_count = length(local.accelerator_core_list) > 2 ? (local.accelerator_core_list[0] * local.accelerator_core_list[1] * local.accelerator_core_list[2]) * 2 : (local.accelerator_core_list[0] * local.accelerator_core_list[1]) * 2 + + tpu_core_count = local.accelerator_core_count == 0 ? local.node_type_core_count : local.accelerator_core_count +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml new file mode 100644 index 0000000000..95b6d1c730 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name + has_to_be_used: true diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf new file mode 100644 index 0000000000..8cb7b8663e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf @@ -0,0 +1,39 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_tpu" { + description = "Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset_tpu + + precondition { + condition = (var.node_type == "") != (var.accelerator_config == { topology : "", version : "" }) + error_message = "Either a node_type or an accelerator_config must be provided." + } + + precondition { + condition = ((local.tpu_core_count / 8) <= var.node_count_dynamic_max) || ((local.tpu_core_count / 8) <= var.node_count_static) + error_message = <<-EOD + When using TPUs there should be at least one node per every 8 cores. + Currently there are ${local.tpu_core_count} cores but only ${var.node_count_static} static nodes and ${var.node_count_dynamic_max} dynamic nodes. + EOD + } + + precondition { + condition = (var.node_count_dynamic_max % (local.tpu_core_count / 8) == 0) && (var.node_count_static % (local.tpu_core_count / 8) == 0) + error_message = <<-EOD + The number of worker nodes should be a multiple of ${local.tpu_core_count / 8}. + This is to ensure each node has a TPU machine for job scheduling. + EOD + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf new file mode 100644 index 0000000000..367b0bee09 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf @@ -0,0 +1,171 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "node_count_static" { + description = <<-EOD + Number of worker nodes to be statically created. + For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). + See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. + EOD + type = number + default = 0 +} + +variable "node_count_dynamic_max" { + description = <<-EOD + Maximum number of auto-scaling worker nodes allowed in this partition. + For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). + See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. + EOD + type = number + default = 0 +} + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "enable_public_ips" { + description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." + type = bool + default = false +} + +variable "disable_public_ips" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `enable_public_ips` instead." + type = bool + default = null + validation { + condition = var.disable_public_ips == null + error_message = "DEPRECATED: Use `enable_public_ips` instead." + } +} + +variable "node_type" { + description = "Specify a node type to base the vm configuration upon it." + type = string + default = "" +} + +variable "accelerator_config" { + description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." + type = object({ + topology = string + version = string + }) + default = { + topology = "" + version = "" + } + validation { + condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], var.accelerator_config.version) + error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" + } + validation { + condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) + error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." + } +} + +variable "tf_version" { + description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." + type = string + default = "2.14.0" +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + +variable "preserve_tpu" { + description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" + type = bool + default = false +} + +variable "zone" { + description = "Zone in which to create compute VMs. TPU partitions can only specify a single zone." + type = string +} + +variable "data_disks" { + description = "The data disks to include in the TPU node" + type = list(string) + default = [] +} + +variable "docker_image" { + description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" + type = string + default = null +} + +variable "subnetwork_self_link" { + type = string + description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." +} + +variable "service_account_email" { + description = "Service account e-mail address to attach to the TPU-vm." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the TPU-vm." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "service_account" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + } +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "reserved" { + description = "Specify whether TPU-vms in this nodeset are created under a reservation." + type = bool + default = false +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + })) + default = [] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf new file mode 100644 index 0000000000..398eeffdda --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.3" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.74.0" + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md new file mode 100644 index 0000000000..7c9e32debf --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md @@ -0,0 +1,227 @@ +## Description + +This module creates a nodeset data structure intended to be input to the +[schedmd-slurm-gcp-v6-partition](../schedmd-slurm-gcp-v6-partition/) module. + +Nodesets allow adding heterogeneous node types to a partition, and hence +running jobs that mix multiple node characteristics. See the [heterogeneous jobs +section][hetjobs] of the SchedMD documentation for more information. + +To specify nodes from a specific nodesets in a partition, the [`--nodelist`] +(or `-w`) flag can be used, for example: + +```bash +srun -N 3 -p compute --nodelist cluster-compute-group-[0-2] hostname +``` + +Where the 3 nodes will be selected from the nodes `cluster-compute-group-[0-2]` +in the compute partition. + +Additionally, depending on how the nodes differ, a constraint can be added via +the [`--constraint`] (or `-C`) flag or other flags such as `--mincpus` can be +used to specify nodes with the desired characteristics. + +[`--nodelist`]: https://slurm.schedmd.com/srun.html#OPT_nodelist +[`--constraint`]: https://slurm.schedmd.com/srun.html#OPT_constraint +[hetjobs]: https://slurm.schedmd.com/heterogeneous_jobs.html + +### Example + +The following code snippet creates a partition module using the `nodeset` +module as input with: + +* a max node count of 200 +* VM machine type of `c2-standard-30` +* partition name of "compute" +* default nodeset name of "ghpc" +* connected to the `network` module via `use` +* nodes mounted to homefs via `use` + +```yaml +- id: nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + node_count_dynamic_max: 200 + machine_type: c2-standard-30 + +- id: compute_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + use: + - homefs + - nodeset + settings: + partition_name: compute +``` + +## Custom Images + +For more information on creating valid custom images for the node group VM +instances or for custom instance templates, see our [vm-images.md] documentation +page. + +[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images + +## GPU Support + +More information on GPU support in Slurm on GCP and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) + +### Compute VM Zone Policies + +The Slurm on GCP nodeset module allows you to specify additional zones in +which to create VMs through [bulk creation][bulk]. This is valuable when +configuring partitions with popular VM families and you desire access to +more compute resources across zones. + +[bulk]: https://cloud.google.com/compute/docs/instances/multiple/about-bulk-creation +[networkpricing]: https://cloud.google.com/vpc/network-pricing + +> **_WARNING:_** Lenient zone policies can lead to additional egress costs when +> moving large amounts of data between zones in the same region. For example, +> traffic between VMs and traffic from VMs to shared filesystems such as +> Filestore. For more information on egress fees, see the +> [Network Pricing][networkpricing] Google Cloud documentation. +> +> To avoid egress charges, ensure your compute nodes are created in a single +> zone by setting var.zone and leaving var.zones to its default value of the +> empty list. +> +> **_NOTE:_** If a new zone is added to the region while the cluster is active, +> nodes in the partition may be created in that zone. In this case, the +> partition may need to be redeployed to ensure the newly added zone is denied. + +In the zonal example below, the nodeset's zone implicitly defaults to the +deployment variable `vars.zone`: + +```yaml +vars: + zone: us-central1-f + +- id: zonal-nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset +``` + +In the example below, we enable creation in additional zones: + +```yaml +vars: + zone: us-central1-f + +- id: multi-zonal-nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + settings: + zones: + - us-central1-a + - us-central1-b +``` + +## Support +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4 | +| [google](#requirement\_google) | >= 5.11 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 5.11 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.machine_type_zone_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [google_compute_machine_types.machine_types_by_zone](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | +| [google_compute_reservation.reservation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_topology](#input\_accelerator\_topology) | Specifies the shape of the Accelerator (GPU/TPU) slice. | `string` | `null` | no | +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = optional(string)
    subnetwork         = string
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    stack_type         = optional(string)
    queue_count        = optional(number)
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | +| [dws\_flex](#input\_dws\_flex) | If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes.
See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler
Options:
- enable: Enable DWS Flex Start
- max\_run\_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week).
- use\_job\_duration: Use the job duration to determine the max\_run\_duration, if job duration is not set, max\_run\_duration will be used.
- use\_bulk\_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances

Limitations:
- CAN NOT be used with reservations;
- CAN NOT be used with placement groups; | 
object({
    enabled          = optional(bool, true)
    max_run_duration = optional(number, 604800) # one week
    use_job_duration = optional(bool, false)
    use_bulk_insert  = optional(bool, false)
  })
| 
{
  "enabled": false
}
| no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_maintenance\_reservation](#input\_enable\_maintenance\_reservation) | Enables slurm reservation for scheduled maintenance. | `bool` | `false` | no | +| [enable\_opportunistic\_maintenance](#input\_enable\_opportunistic\_maintenance) | On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_placement](#input\_enable\_placement) | Use placement policy for VMs in this nodeset.
See: https://cloud.google.com/compute/docs/instances/placement-policies-overview
To set max\_distance of used policy, use `placement_max_distance` variable.

Enabled by default, reasons for users to disable it:
- If non-dense reservation is used, user can avoid extra-cost of creating placement policies;
- If user wants to avoid "all or nothing" VM provisioning behaviour;
- If user wants to intentionally have "spread" VMs (e.g. for reliability reasons) | `bool` | `true` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | +| [future\_reservation](#input\_future\_reservation) | If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT\_ID/zones/ZONE/futureReservations/FUTURE\_RESERVATION\_NAME'.
See https://cloud.google.com/compute/docs/instances/future-reservations-overview | `string` | `""` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_properties](#input\_instance\_properties) | Override the instance properties. Used to test features not supported by Slurm GCP,
recommended for advanced usage only.
See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert
If any sub-field (e.g. scheduling) is set, it will override the values computed by
SlurmGCP and ignoring values of provided vars. | `any` | `null` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for compute nodes. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | +| [maintenance\_interval](#input\_maintenance\_interval) | Sets the maintenance interval for instances in this nodeset.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. | `string` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_conf](#input\_node\_conf) | Map of Slurm node line configuration. | `map(any)` | `{}` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling nodes allowed in this partition. | `number` | `10` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | +| [placement\_max\_distance](#input\_placement\_max\_distance) | Maximum distance between nodes in the placement group. Requires enable\_placement to be true. Values must be supported by the chosen machine type. | `number` | `null` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | +| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]
- RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]

Must be a "SPECIFIC" reservation
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [zone](#input\_zone) | Zone in which to create compute VMs. Additional zones in the same region can be specified in var.zones. | `string` | n/a | yes | +| [zone\_target\_shape](#input\_zone\_target\_shape) | Strategy for distributing VMs across zones in a region.
ANY
GCE picks zones for creating VM instances to fulfill the requested number of VMs
within present resource constraints and to maximize utilization of unused zonal
reservations.
ANY\_SINGLE\_ZONE (default)
GCE always selects a single zone for all the VMs, optimizing for resource quotas,
available reservations and general capacity.
BALANCED
GCE prioritizes acquisition of resources, scheduling VMs in zones where resources
are available while distributing VMs as evenly as possible across allowed zones
to minimize the impact of zonal failure. | `string` | `"ANY_SINGLE_ZONE"` | no | +| [zones](#input\_zones) | Additional zones in which to allow creation of partition nodes. Google Cloud
will find zone based on availability, quota and reservations.
Should not be set if SPECIFIC reservation is used. | `set(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf new file mode 100644 index 0000000000..da6aae33ee --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf @@ -0,0 +1,232 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) +} + +module "instance_validation" { + source = "../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata + ) + + name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.disk_resource_manager_tags + } + ] + + public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + ghpc_startup_script = [{ + filename = "ghpc_nodeset_startup.sh" + content = var.startup_script + }] + + termination_action = (var.dws_flex.enabled && !var.dws_flex.use_bulk_insert) ? "DELETE" : try(var.spot_instance_config.termination_action, null) + + nodeset = { + node_count_static = var.node_count_static + node_count_dynamic_max = var.node_count_dynamic_max + node_conf = var.node_conf + nodeset_name = local.name + dws_flex = var.dws_flex + + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(local.labels, var.disk_labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + + bandwidth_tier = var.bandwidth_tier + can_ip_forward = var.can_ip_forward + + enable_confidential_vm = var.enable_confidential_vm + enable_placement = var.enable_placement + placement_max_distance = var.placement_max_distance + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + gpu = one(local.guest_accelerator) + accelerator_topology = var.accelerator_topology + + labels = local.labels + machine_type = terraform_data.machine_type_zone_validation.output + advanced_machine_features = var.advanced_machine_features + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + region = var.region + resource_manager_tags = var.resource_manager_tags + service_account = local.service_account + shielded_instance_config = var.shielded_instance_config + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + subnetwork_self_link = var.subnetwork_self_link + additional_networks = var.additional_networks + access_config = local.access_config + tags = var.tags + spot = var.enable_spot_vm + termination_action = local.termination_action + reservation_name = local.reservation_name + future_reservation = local.future_reservation + maintenance_interval = var.maintenance_interval + instance_properties_json = jsonencode(var.instance_properties) + + zone_target_shape = var.zone_target_shape + zone_policy_allow = local.zones + zone_policy_deny = local.zones_deny + + startup_script = local.ghpc_startup_script + network_storage = var.network_storage + + enable_maintenance_reservation = var.enable_maintenance_reservation + enable_opportunistic_maintenance = var.enable_opportunistic_maintenance + } +} + +locals { + zones = setunion(var.zones, [var.zone]) + zones_deny = setsubtract(data.google_compute_zones.available.names, local.zones) +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region + + lifecycle { + postcondition { + condition = length(setsubtract(local.zones, self.names)) == 0 + error_message = <<-EOD + Invalid zones=${jsonencode(setsubtract(local.zones, self.names))} + Available zones=${jsonencode(self.names)} + EOD + } + } +} + +locals { + res_match = regex("^(?P(?Pprojects/(?P[a-z0-9-]+)/reservations/)?(?P[a-z0-9-]+)(?P/reservationBlocks/[a-z0-9-]+)?)?$", var.reservation_name) + + res_short_name = local.res_match.name + res_project = coalesce(local.res_match.project, var.project_id) + res_prefix = coalesce(local.res_match.prefix, "projects/${local.res_project}/reservations/") + res_suffix = local.res_match.suffix == null ? "" : local.res_match.suffix + + reservation_name = local.res_match.whole == null ? "" : "${local.res_prefix}${local.res_short_name}${local.res_suffix}" +} + +locals { + fr_match = regex("^(?Pprojects/(?P[a-z0-9-]+)/zones/(?P[a-z0-9-]+)/futureReservations/)?(?P[a-z0-9-]+)?$", var.future_reservation) + + fr_name = local.fr_match.name + fr_project = coalesce(local.fr_match.project, var.project_id) + fr_zone = coalesce(local.fr_match.zone, var.zone) + + future_reservation = var.future_reservation == "" ? "" : "projects/${local.fr_project}/zones/${local.fr_zone}/futureReservations/${local.fr_name}" +} + + +# tflint-ignore: terraform_unused_declarations +data "google_compute_reservation" "reservation" { + count = length(local.reservation_name) > 0 ? 1 : 0 + + name = local.res_short_name + project = local.res_project + zone = var.zone + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "Couldn't find the reservation ${var.reservation_name}" + } + + postcondition { + condition = coalesce(self.specific_reservation_required, true) + error_message = < 0] +} + +resource "terraform_data" "machine_type_zone_validation" { + input = var.machine_type + lifecycle { + precondition { + condition = length(local.zones_with_machine_type) > 0 + error_message = <<-EOT + machine type ${var.machine_type} is not available in any of the zones ${jsonencode(local.zones)}". To list zones in which it is available, run: + + gcloud compute machine-types list --filter="name=${var.machine_type}" + EOT + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml new file mode 100644 index 0000000000..95b6d1c730 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name + has_to_be_used: true diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf new file mode 100644 index 0000000000..18ed74e2d5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf @@ -0,0 +1,112 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset" { + description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset + + precondition { + condition = !contains([ + "c3-:pd-standard", + "h3-:pd-standard", + "h3-:pd-ssd", + ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") + error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." + } + + precondition { + condition = var.reservation_name == "" || length(var.zones) == 0 + error_message = <<-EOD + If a reservation is specified, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.accelerator_topology == null || var.enable_placement + error_message = "accelerator_topology requires enable_placement to be set to true." + } + + precondition { + condition = (var.accelerator_topology == null) || try(tonumber(split("x", var.accelerator_topology)[1]) % local.guest_accelerator[0].count == 0, false) + error_message = "accelerator_topology must be divisible by number of gpus in machine." + } + + precondition { + condition = var.placement_max_distance == null || var.enable_placement + error_message = "placement_max_distance requires enable_placement to be set to true." + } + + precondition { + condition = !(startswith(var.machine_type, "a3-") && var.placement_max_distance == 1) + error_message = "A3 machines do not support a placement_max_distance of 1." + } + + precondition { + condition = var.reservation_name == "" || !var.dws_flex.enabled + error_message = "Cannot use reservations with DWS Flex." + } + + precondition { + condition = !var.enable_placement || !var.dws_flex.enabled + error_message = "Cannot use DWS Flex with `enable_placement`." + } + + precondition { + condition = length(var.zones) == 0 || !var.dws_flex.enabled + error_message = <<-EOD + If a DWS Flex is enabled, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.on_host_maintenance == "TERMINATE" || !var.dws_flex.enabled + error_message = "If DWS Flex is used, `on_host_maintenance` should be set to 'TERMINATE'" + } + + precondition { + condition = !var.enable_spot_vm || !var.dws_flex.enabled + error_message = "Cannot use both Flex-Start and Spot VMs for provisioning." + } + + precondition { + condition = var.reservation_name == "" || var.future_reservation == "" + error_message = "Cannot use reservations and future reservations in the same nodeset" + } + + precondition { + condition = !var.enable_placement || var.future_reservation == "" + error_message = "Cannot use `enable_placement` with future reservations." + } + + precondition { + condition = var.future_reservation == "" || length(var.zones) == 0 + error_message = <<-EOD + If a future reservation is specified, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.future_reservation == "" || local.fr_zone == var.zone + error_message = <<-EOD + The zone of the deployment must match that of the future reservation + EOD + } + + precondition { + condition = var.node_count_dynamic_max > 0 || var.node_count_static > 0 + error_message = <<-EOD + This nodeset contains zero nodes, there should be at least one static or dynamic node + EOD + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf new file mode 100644 index 0000000000..06ef5aac6f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf @@ -0,0 +1,641 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "node_conf" { + description = "Map of Slurm node line configuration." + type = map(any) + default = {} + validation { + condition = lookup(var.node_conf, "Sockets", null) == null + error_message = <<-EOD + `Sockets` field is in conflict with `SocketsPerBoard` which is automatically generated by SlurmGCP. + Instead, you can override the following fields: `Boards`, `SocketsPerBoard`, `CoresPerSocket`, and `ThreadsPerCore`. + See: https://slurm.schedmd.com/slurm.conf.html#OPT_Boards and https://slurm.schedmd.com/slurm.conf.html#OPT_Sockets_1 + EOD + } +} + +variable "node_count_static" { + description = "Number of nodes to be statically created." + type = number + default = 0 +} + +variable "node_count_dynamic_max" { + description = "Maximum number of auto-scaling nodes allowed in this partition." + type = number + default = 10 +} + +## VM Definition +variable "instance_template" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Instance template can not be specified for compute nodes." + type = string + default = null + validation { + condition = var.instance_template == null + error_message = "DEPRECATED: Instance template can not be specified for compute nodes." + } +} + +variable "machine_type" { + description = "Compute Platform machine type to use for this partition compute nodes." + type = string + default = "c2-standard-60" +} + +variable "metadata" { + type = map(string) + description = "Metadata, provided as a map." + default = {} +} + +variable "instance_image" { + description = <<-EOD + Defines the image that will be used in the Slurm node group VM instances. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + + For more information on creating custom images that comply with Slurm on GCP + see the "Slurm on GCP Custom Images" section in docs/vm-images.md. + EOD + type = map(string) + default = { + family = "slurm-gcp-6-11-hpc-rocky-linux-8" + project = "schedmd-slurm-public" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations + description = <<-EOD + A flag that designates that the user is aware that they are requesting + to use a custom and potentially incompatible image for this Slurm on + GCP module. + + If the field is set to false, only the compatible families and project + names will be accepted. The deployment will fail with any other image + family or name. If set to true, no checks will be done. + + See: https://goo.gle/hpc-slurm-images + EOD + type = bool + default = false +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "tags" { + type = list(string) + description = "Network tag list." + default = [] +} + +variable "disk_type" { + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + type = string + default = "pd-standard" +} + +variable "disk_size_gb" { + description = "Size of boot disk to create for the partition compute nodes." + type = number + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string)) + auto_delete = optional(bool) + boot = optional(bool) + disk_resource_manager_tags = optional(map(string)) + })) + default = [] +} + +variable "enable_confidential_vm" { + type = bool + description = "Enable the Confidential VM configuration. Note: the instance image must support option." + default = false +} + +variable "enable_shielded_vm" { + type = bool + description = "Enable the Shielded VM configuration. Note: the instance image must support option." + default = false +} + +variable "shielded_instance_config" { + type = object({ + enable_integrity_monitoring = bool + enable_secure_boot = bool + enable_vtpm = bool + }) + description = <<-EOD + Shielded VM configuration for the instance. Note: not used unless + enable_shielded_vm is 'true'. + - enable_integrity_monitoring : Compare the most recent boot measurements to the + integrity policy baseline and return a pair of pass/fail results depending on + whether they match or not. + - enable_secure_boot : Verify the digital signature of all boot components, and + halt the boot process if signature verification fails. + - enable_vtpm : Use a virtualized trusted platform module, which is a + specialized computer chip you can use to encrypt objects like keys and + certificates. + EOD + default = { + enable_integrity_monitoring = true + enable_secure_boot = true + enable_vtpm = true + } +} + + +variable "enable_oslogin" { + type = bool + description = <<-EOD + Enables Google Cloud os-login for user login and authentication for VMs. + See https://cloud.google.com/compute/docs/oslogin + EOD + default = true +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example." + type = bool + default = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "labels" { + description = "Labels to add to partition compute instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "The name of the minimum CPU platform that you want the instance to use." + type = string + default = null +} + +variable "on_host_maintenance" { + type = string + description = <<-EOD + Instance availability Policy. + + Note: Placement groups are not supported when on_host_maintenance is set to + "MIGRATE" and will be deactivated regardless of the value of + enable_placement. To support enable_placement, ensure on_host_maintenance is + set to "TERMINATE". + EOD + default = "TERMINATE" +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." + } +} + +variable "accelerator_topology" { + type = string + description = "Specifies the shape of the Accelerator (GPU/TPU) slice." + nullable = true + default = null +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + + +variable "service_account_email" { + description = "Service account e-mail address to attach to the compute instances." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the compute instances." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "service_account" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + } +} + +variable "enable_spot_vm" { + description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." + type = bool + default = false +} + +variable "spot_instance_config" { + description = "Configuration for spot VMs." + type = object({ + termination_action = string + }) + default = null +} + +variable "bandwidth_tier" { + description = < 0 + error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME[/reservationBlocks/BLOCK_ID]', [...] are optional parts." + } +} + +variable "future_reservation" { + description = <<-EOD + If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zones/ZONE/futureReservations/FUTURE_RESERVATION_NAME'. + See https://cloud.google.com/compute/docs/instances/future-reservations-overview + EOD + type = string + default = "" + nullable = false + + validation { + condition = length(regexall("^(projects/([a-z0-9-]+)/zones/([a-z0-9-]+)/futureReservations/([a-z0-9-]+))?$", var.future_reservation)) > 0 || length(regexall("^([a-z0-9-]+)$", var.future_reservation)) > 0 + error_message = "Future reservation must be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zone/ZONE/futureReservations/FUTURE_RESERVATION_NAME'." + } +} + +variable "maintenance_interval" { + description = <<-EOD + Sets the maintenance interval for instances in this nodeset. + See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. + EOD + type = string + default = null +} + +variable "startup_script" { + description = "Startup script used by VMs in this nodeset" + type = string + default = "# no-op" +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + })) + default = [] +} + + +variable "instance_properties" { + description = <<-EOD + Override the instance properties. Used to test features not supported by Slurm GCP, + recommended for advanced usage only. + See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert + If any sub-field (e.g. scheduling) is set, it will override the values computed by + SlurmGCP and ignoring values of provided vars. + EOD + type = any + default = null +} + + +variable "enable_maintenance_reservation" { + type = bool + description = "Enables slurm reservation for scheduled maintenance." + default = false +} + + +variable "enable_opportunistic_maintenance" { + type = bool + description = "On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle." + default = false +} + + +variable "dws_flex" { + description = <<-EOD + If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes. + See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler + Options: + - enable: Enable DWS Flex Start + - max_run_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week). + - use_job_duration: Use the job duration to determine the max_run_duration, if job duration is not set, max_run_duration will be used. + - use_bulk_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances + + Limitations: + - CAN NOT be used with reservations; + - CAN NOT be used with placement groups; + + EOD + + type = object({ + enabled = optional(bool, true) + max_run_duration = optional(number, 604800) # one week + use_job_duration = optional(bool, false) + use_bulk_insert = optional(bool, false) + }) + default = { + enabled = false + } + validation { + condition = var.dws_flex.max_run_duration >= 600 && var.dws_flex.max_run_duration <= 604800 + error_message = "Max duration must be at least than 10 minutes, and cannot be more than one week." + } +} + +variable "placement_max_distance" { + type = number + description = "Maximum distance between nodes in the placement group. Requires enable_placement to be true. Values must be supported by the chosen machine type." + nullable = true + default = null + + validation { + condition = coalesce(var.placement_max_distance, 1) >= 1 && coalesce(var.placement_max_distance, 3) <= 3 + error_message = "Invalid value for placement_max_distance. Valid values are null, 1, 2, or 3." + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf new file mode 100644 index 0000000000..e014c318e4 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.4" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.11" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.74.0" + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md new file mode 100644 index 0000000000..d3dbcd959e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md @@ -0,0 +1,105 @@ +## Description + +This module creates a compute partition that can be used as input to the +[schedmd-slurm-gcp-v6-controller](../../scheduler/schedmd-slurm-gcp-v6-controller/README.md). + +The partition module is designed to work alongside the +[schedmd-slurm-gcp-v6-nodeset](../schedmd-slurm-gcp-v6-nodeset/README.md) +module. A partition can be made up of one or +more nodesets, provided either through `use` (preferred) or defined manually +in the `nodeset` variable. + +### Example + +The following code snippet creates a partition module with: + +* 2 nodesets added via `use`. + * The first nodeset is made up of machines of type `c2-standard-30`. + * The second nodeset is made up of machines of type `c2-standard-60`. + * Both nodesets have a maximum count of 200 dynamically created nodes. +* partition name of "compute". +* connected to the `network` module via `use`. +* nodes mounted to homefs via `use`. + +```yaml +- id: nodeset_1 + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + name: c30 + node_count_dynamic_max: 200 + machine_type: c2-standard-30 + +- id: nodeset_2 + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + name: c60 + node_count_dynamic_max: 200 + machine_type: c2-standard-60 + +- id: compute_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + use: + - homefs + - nodeset_1 + - nodeset_2 + settings: + partition_name: compute +``` + +## Support + +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [exclusive](#input\_exclusive) | Exclusive job access to nodes. When set to true nodes execute single job and are deleted
after job exits. If set to false, multiple jobs can be scheduled on one node. | `bool` | `true` | no | +| [is\_default](#input\_is\_default) | Sets this partition as the default partition by updating the partition\_conf.
If "Default" is already set in partition\_conf, this variable will have no effect. | `bool` | `false` | no | +| [network\_storage](#input\_network\_storage) | DEPRECATED | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | A list of nodesets.
For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip     = string
      remote_mount  = string
      local_mount   = string
      fs_type       = string
      mount_options = string
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | +| [partition\_conf](#input\_partition\_conf) | Slurm partition configuration as a map.
See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION | `map(string)` | `{}` | no | +| [partition\_name](#input\_partition\_name) | The name of the slurm partition. | `string` | n/a | yes | +| [resume\_timeout](#input\_resume\_timeout) | Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'ResumeTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. | `number` | `null` | no | +| [suspend\_time](#input\_suspend\_time) | Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram.
This sets 'SuspendTime' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details.
NOTE: use value -1 to exclude partition from suspend.
NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. | `number` | `300` | no | +| [suspend\_timeout](#input\_suspend\_timeout) | Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'SuspendTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. | `number` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Details of a nodesets in this partition | +| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of a dynamic nodesets in this partition | +| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of a TPU nodesets in this partition | +| [partitions](#output\_partitions) | Details of a slurm partition | + diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf new file mode 100644 index 0000000000..1618c64280 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf @@ -0,0 +1,41 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + use_static = [for ns in concat(var.nodeset, var.nodeset_tpu) : ns.nodeset_name if ns.node_count_static > 0] + + has_node = length(var.nodeset) > 0 + has_dyn = length(var.nodeset_dyn) > 0 + has_tpu = length(var.nodeset_tpu) > 0 + has_flex = length([for ns in var.nodeset : ns.dws_flex.enabled if ns.dws_flex.enabled]) > 0 +} + +locals { + partition_conf = merge({ + "Default" = var.is_default ? "YES" : null + "SuspendTime" = var.suspend_time < 0 ? "INFINITE" : var.suspend_time + "SuspendTimeout" = var.suspend_timeout != null ? var.suspend_timeout : (local.has_tpu ? 240 : 120) + }, var.partition_conf, { "ResumeTimeout" = local.has_flex ? 65535 : try(var.partition_conf["ResumeTimeout"], coalesce(var.resume_timeout, (local.has_tpu ? 600 : 300))) }) + + partition = { + partition_name = var.partition_name + partition_conf = local.partition_conf + + partition_nodeset = [for ns in var.nodeset : ns.nodeset_name] + partition_nodeset_tpu = [for ns in var.nodeset_tpu : ns.nodeset_name] + partition_nodeset_dyn = [for ns in var.nodeset_dyn : ns.nodeset_name] + # Options + enable_job_exclusive = var.exclusive + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml new file mode 100644 index 0000000000..13ea127b3c --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + has_to_be_used: true diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf new file mode 100644 index 0000000000..35dece64fb --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf @@ -0,0 +1,54 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "partitions" { + description = "Details of a slurm partition" + + value = [local.partition] + + precondition { + condition = (length(local.use_static) == 0) || !var.exclusive + error_message = <<-EOD + Can't use static nodes within partition with `var.exclusive` set to `true`. + NOTE: Partition's `var.exclusive` is set to `true` by default. Set it to `false` explicitly to use static nodes. + EOD + } + + precondition { + # Can not mix TPU with other non-TPU nodesets due to SlurmGCP specific limitations; + # Can not mix dynamic with non-dynamic nodesets due to Slurms inability to + # turn off "power management" at nodeset level (can only do it at partition or node level). + condition = sum([for b in [local.has_node, local.has_dyn, local.has_tpu] : b ? 1 : 0]) == 1 + error_message = "Partition must contain exactly one type of nodeset." + } +} + +output "nodeset" { + description = "Details of a nodesets in this partition" + + value = var.nodeset +} + +output "nodeset_tpu" { + description = "Details of a TPU nodesets in this partition" + + value = var.nodeset_tpu +} + + +output "nodeset_dyn" { + description = "Details of a dynamic nodesets in this partition" + + value = var.nodeset_dyn +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf new file mode 100644 index 0000000000..a1c85adb90 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf @@ -0,0 +1,311 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "partition_name" { + description = "The name of the slurm partition." + type = string + + validation { + condition = can(regex("^[a-z](?:[a-z0-9]*)$", var.partition_name)) + error_message = "Variable 'partition_name' must be a match of regex '^[a-z](?:[a-z0-9]*)$'." + } +} + +variable "partition_conf" { + description = <<-EOD + Slurm partition configuration as a map. + See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION + EOD + type = map(string) + default = {} +} + +variable "is_default" { + description = <<-EOD + Sets this partition as the default partition by updating the partition_conf. + If "Default" is already set in partition_conf, this variable will have no effect. + EOD + type = bool + default = false +} + +variable "exclusive" { + description = <<-EOD + Exclusive job access to nodes. When set to true nodes execute single job and are deleted + after job exits. If set to false, multiple jobs can be scheduled on one node. + EOD + type = bool + default = true +} + +variable "nodeset" { + description = <<-EOD + A list of nodesets. + For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset + EOD + type = list(object({ + node_count_static = optional(number, 0) + node_count_dynamic_max = optional(number, 1) + node_conf = optional(map(string), {}) + nodeset_name = string + additional_disks = optional(list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string), {}) + auto_delete = optional(bool, true) + boot = optional(bool, false) + disk_resource_manager_tags = optional(map(string), {}) + })), []) + bandwidth_tier = optional(string, "platform_default") + can_ip_forward = optional(bool, false) + disk_auto_delete = optional(bool, true) + disk_labels = optional(map(string), {}) + disk_resource_manager_tags = optional(map(string), {}) + disk_size_gb = optional(number) + disk_type = optional(string) + enable_confidential_vm = optional(bool, false) + enable_placement = optional(bool, false) + placement_max_distance = optional(number, null) + enable_oslogin = optional(bool, true) + enable_shielded_vm = optional(bool, false) + enable_maintenance_reservation = optional(bool, false) + enable_opportunistic_maintenance = optional(bool, false) + gpu = optional(object({ + count = number + type = string + })) + accelerator_topology = optional(string, null) + dws_flex = object({ + enabled = bool + max_run_duration = number + use_job_duration = bool + use_bulk_insert = bool + }) + labels = optional(map(string), {}) + machine_type = optional(string) + advanced_machine_features = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + maintenance_interval = optional(string) + instance_properties_json = string + metadata = optional(map(string), {}) + min_cpu_platform = optional(string) + network_tier = optional(string, "STANDARD") + network_storage = optional(list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = optional(map(string)) + mount_runner = optional(map(string)) + })), []) + on_host_maintenance = optional(string) + preemptible = optional(bool, false) + region = optional(string) + resource_manager_tags = optional(map(string), {}) + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + shielded_instance_config = optional(object({ + enable_integrity_monitoring = optional(bool, true) + enable_secure_boot = optional(bool, true) + enable_vtpm = optional(bool, true) + })) + source_image_family = optional(string) + source_image_project = optional(string) + source_image = optional(string) + subnetwork_self_link = string + additional_networks = optional(list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + }))) + access_config = optional(list(object({ + nat_ip = string + network_tier = string + }))) + spot = optional(bool, false) + tags = optional(list(string), []) + termination_action = optional(string) + reservation_name = optional(string) + future_reservation = string + startup_script = optional(list(object({ + filename = string + content = string })), []) + + zone_target_shape = string + zone_policy_allow = set(string) + zone_policy_deny = set(string) + })) + default = [] + + validation { + condition = length(distinct(var.nodeset[*].nodeset_name)) == length(var.nodeset) + error_message = "All nodesets must have a unique name." + } +} + +variable "nodeset_tpu" { + description = "Define TPU nodesets, as a list." + type = list(object({ + node_count_static = optional(number, 0) + node_count_dynamic_max = optional(number, 5) + nodeset_name = string + enable_public_ip = optional(bool, false) + node_type = string + accelerator_config = optional(object({ + topology = string + version = string + }), { + topology = "" + version = "" + }) + tf_version = string + preemptible = optional(bool, false) + preserve_tpu = optional(bool, false) + zone = string + data_disks = optional(list(string), []) + docker_image = optional(string, "") + network_storage = optional(list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + })), []) + subnetwork = string + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + project_id = string + reserved = optional(string, false) + })) + default = [] + + validation { + condition = length(distinct([for x in var.nodeset_tpu : x.nodeset_name])) == length(var.nodeset_tpu) + error_message = "All TPU nodesets must have a unique name." + } +} + +variable "nodeset_dyn" { + description = "Defines dynamic nodesets, as a list." + type = list(object({ + nodeset_name = string + nodeset_feature = string + })) + default = [] + + validation { + condition = length(distinct([for x in var.nodeset_dyn : x.nodeset_name])) == length(var.nodeset_dyn) + error_message = "All dynamic nodesets must have a unique name." + } +} + +variable "resume_timeout" { + description = <<-EOD + Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use. + If null is given, then a smart default will be chosen depending on nodesets in partition. + This sets 'ResumeTimeout' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. + EOD + type = number + default = null + + validation { + condition = var.resume_timeout == null ? true : var.resume_timeout > 0 && var.resume_timeout < 65536 + error_message = "Value must be > 0 and < 65536" + } +} + +variable "suspend_time" { + description = <<-EOD + Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram. + This sets 'SuspendTime' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details. + NOTE: use value -1 to exclude partition from suspend. + NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. + EOD + type = number + default = 300 + + validation { + condition = var.suspend_time >= -1 + error_message = "Value must be >= -1." + } +} + +variable "suspend_timeout" { + description = <<-EOD + Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown. + If null is given, then a smart default will be chosen depending on nodesets in partition. + This sets 'SuspendTimeout' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. + EOD + type = number + default = null + + validation { + condition = var.suspend_timeout == null ? true : var.suspend_timeout > 0 + error_message = "Value must be > 0." + } +} + + +# tflint-ignore: terraform_unused_declarations +variable "network_storage" { + description = "DEPRECATED" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] + validation { + condition = length(var.network_storage) == 0 + error_message = <<-EOD + network_storage in partition module is deprecated and should not be set. + To add network storage to compute nodes, use network_storage of nodeset module instead. + EOD + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf new file mode 100644 index 0000000000..d388f4bfdd --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.3" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.74.0" + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/README.md b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/README.md new file mode 100644 index 0000000000..994f1500ba --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/README.md @@ -0,0 +1,157 @@ +## Description + +This module provides ways to create and manage Google Cloud Artifact Registry repositories. + +Currently this module is built to support repositories in Docker format although there are placeholder variables for other types which may work too. Remote repositories with pull-through cache functionality integrated with Google Secret Manager is currently supported. The aim of this module is to eventually offer feature parity with this [Terraform module](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository#nested_remote_repository_config), allowing creation of repositories in various formats, including Docker, Maven, NPM, Python, APT, YUM, and COMMON. + +This module is best suited for managing artifact repositories in HPC/AI containerized environments where artifacts need to be shared across distributed systems. It includes IAM role configurations and secret access handling for seamless integration with CI/CD pipelines and other services too. + +It is designed to help facilitate containerized workloads running in the Cluster Toolkit with SLURM leveraging [Enroot](https://github.com/NVIDIA/enroot) and [Pyxis](https://github.com/NVIDIA/pyxis). Docker repositories can store container images that are used in job submissions, enabling efficient and scalable execution of containerized HPC or AI based workloads. + +## Usage + +### Service Account / APIs + +You will need to enable the relevant APIs and create a Service Account for your cluster with the following Artifact Registry permissions. + +```yaml + - id: services-api + source: community/modules/project/service-enablement + settings: + gcp_service_list: + - secretmanager.googleapis.com + - cloudbuild.googleapis.com + - artifactregistry.googleapis.com + + - source: community/modules/project/service-account + kind: terraform + id: hpc_service_account + settings: + project_id: project_name + name: service_account_name + project_roles: + - artifactregistry.reader + - artifactregistry.writer + - secretmanager.secretAccessor +``` + +### Deployment + +Create a standard Docker repository. + +```yaml +- id: registry + source: community/modules/container/artifact-registry + settings: + repo_mode: STANDARD_REPOSITORY + format: DOCKER +``` + +Mirror of public Docker Hub repository. + +```yaml +- id: dockerhub_registry + source: community/modules/container/artifact-registry + settings: + repo_mode: REMOTE_REPOSITORY + format: DOCKER + repo_public_repository: DOCKER_HUB +``` + +Mirror of NVIDIA's [NGC Catalog](https://catalog.ngc.nvidia.com/containers). [API key](https://org.ngc.nvidia.com/setup/api-key) used in blueprint is stored in Secret Manager. + +```yaml +- id: ngc_registry + source: community/modules/container/artifact-registry + settings: + repo_mode: REMOTE_REPOSITORY + format: DOCKER + repo_mirror_url: "https://nvcr.io" + repo_username: $oauthtoken + repo_password: api_key_here + use_upstream_credentials: True +``` + +### Container Operations + +Retrieve `$REPOSITORY_NAME` from [Artifact Registry](https://console.cloud.google.com/artifacts) or by using `gcloud`. + +```yaml +gcloud artifacts repositories list --project="${PROJECT_ID}" +``` + +Pulling containers from your mirrored internal Artifact Repositories. + +Pull [Ubuntu](https://hub.docker.com/_/ubuntu) from Docker Hub mirror. + +```yaml +docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/library/ubuntu:latest +``` + +Pull [Pytorch](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch) from NGC Catalog mirror. + +```yaml +docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/nvidia/pytorch:24.11-py3 +``` + +Alternatively, proceed with running SLURM's [NVIDIA/pyxis](https://github.com/NVIDIA/pyxis) plugin, which will now be able to pull and use these containers directly from the mirrored repositories. + +Note: only Docker registries have been tested so far. Placeholders do exist for other registry types which may or may not work. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_artifact_registry_repository.artifact_registry](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository) | resource | +| [google_secret_manager_secret.repo_password_secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_version.repo_password_secret_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [random_password.repo_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | +| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment. | `string` | n/a | yes | +| [format](#input\_format) | Artifact Registry format (e.g., DOCKER). | `string` | `"DOCKER"` | no | +| [labels](#input\_labels) | Labels to add to the artifact registry. Key-value pairs. | `map(string)` | `{}` | no | +| [project\_id](#input\_project\_id) | Project ID where the artifact registry and secret are created. | `string` | n/a | yes | +| [region](#input\_region) | Region for the artifact registry. | `string` | n/a | yes | +| [repo\_mirror\_url](#input\_repo\_mirror\_url) | For REMOTE\_REPOSITORY, URL for a custom or common mirror. | `string` | `null` | no | +| [repo\_mode](#input\_repo\_mode) | Artifact Registry mode (STANDARD\_REPOSITORY, REMOTE\_REPOSITORY, etc.). | `string` | `"STANDARD_REPOSITORY"` | no | +| [repo\_password](#input\_repo\_password) | Optional password/API key. If null, one will be randomly generated. | `string` | `null` | no | +| [repo\_public\_repository](#input\_repo\_public\_repository) | For REMOTE\_REPOSITORY, name of a known public repo as per the Terraform module
(e.g., DOCKER\_HUB) or null for custom repo. | `string` | `null` | no | +| [repo\_username](#input\_repo\_username) | Username for external repository. | `string` | `null` | no | +| [repository\_base](#input\_repository\_base) | For APT/YUM public repos, repository\_base (e.g., 'DEBIAN', 'UBUNTU'). | `string` | `null` | no | +| [repository\_path](#input\_repository\_path) | For APT/YUM public repos, repository\_path (e.g., 'debian/dists/buster'). | `string` | `null` | no | +| [use\_upstream\_credentials](#input\_use\_upstream\_credentials) | Configure Service Account to use upstream credentials for REMOTE\_REPOSITORY:
If true, a username/password is used for the REMOTE\_REPOSITORY mirror.
If false (or if repo\_password == null), no password is created at all.
Note: Blueprint credentials will be stored in Secrets Manager. | `bool` | `false` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | (Optional) A list of objects to enable user-managed replication.
Each object can have:
location = string
kms\_key\_name = optional(string)
If empty, auto replication is used. | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [registry\_url](#output\_registry\_url) | The URL of the created artifact registry. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/main.tf b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/main.tf new file mode 100644 index 0000000000..c3406af607 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/main.tf @@ -0,0 +1,268 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "artifact-registry", ghpc_role = "container" }) +} + +locals { + # Auto (i.e., empty) vs user-managed replication + auto = length(var.user_managed_replication) == 0 ? true : false + + # For remote custom repositories, parse out host to create a base_component name + mirror_url_no_proto = var.repo_mirror_url != null ? replace(replace(var.repo_mirror_url, "https://", ""), "http://", "") : "" + mirror_host = local.mirror_url_no_proto != "" ? split("/", local.mirror_url_no_proto)[0] : "" + + base_component = replace( + replace( + replace( + lower( + local.mirror_host != "" + ? "${var.format}-${var.repo_mode}-${local.mirror_host}" + : "${var.format}-${var.repo_mode}-nohost" + ), + "\\.", "-" + ), + "/", "-" + ), + "_", "-" + ) + + repository_suffix = random_id.resource_name_suffix.hex + + # The final name for the artifact registry repository + repository_name = replace( + replace( + lower( + format("%s-%s", local.base_component, local.repository_suffix) + ), + ".", "-" + ), + "/", "-" + ) + + # The secret name is derived from the repository name + # with a suffix like "-secret". + derived_secret_name = format("%s-secret", local.repository_name) +} + +############################## +# PASSWORD / SECRET +############################## + +# Only create a random password if user didn't supply one +resource "random_password" "repo_password" { + count = var.use_upstream_credentials && var.repo_password == null ? 1 : 0 + length = 24 + special = true + override_special = "_-#=." +} + +resource "google_secret_manager_secret" "repo_password_secret" { + count = var.use_upstream_credentials ? 1 : 0 + project = var.project_id + + # Derive the secret ID from the repository name + secret_id = local.derived_secret_name + + labels = local.labels + + replication { + dynamic "auto" { + for_each = local.auto ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = local.auto ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = replicas.value.kms_key_name != null ? [1] : [] + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_version" "repo_password_secret_version" { + count = var.use_upstream_credentials ? 1 : 0 + secret = google_secret_manager_secret.repo_password_secret[0].id + + # If user provided a password, use it. Otherwise use the random password. + secret_data = var.repo_password != null ? var.repo_password : random_password.repo_password[0].result +} + +############################## +# IAM BINDINGS +############################## + +############################## +# ARTIFACT REGISTRY +############################## + +resource "random_id" "resource_name_suffix" { + byte_length = 2 +} + +resource "google_artifact_registry_repository" "artifact_registry" { + project = var.project_id + location = var.region + format = var.format + mode = var.repo_mode + description = var.deployment_name + labels = local.labels + repository_id = local.repository_name + + # Only create remote_repository_config if REMOTE_REPOSITORY + dynamic "remote_repository_config" { + for_each = var.repo_mode == "REMOTE_REPOSITORY" ? [1] : [] + content { + description = "Pull-through cache" + + dynamic "docker_repository" { + for_each = var.format == "DOCKER" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "docker_repository" { + for_each = var.format == "DOCKER" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "maven_repository" { + for_each = var.format == "MAVEN" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "maven_repository" { + for_each = var.format == "MAVEN" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "npm_repository" { + for_each = var.format == "NPM" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "npm_repository" { + for_each = var.format == "NPM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "python_repository" { + for_each = var.format == "PYTHON" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "python_repository" { + for_each = var.format == "PYTHON" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "apt_repository" { + for_each = var.format == "APT" && var.repo_public_repository != null ? [1] : [] + content { + public_repository { + repository_base = var.repository_base + repository_path = var.repository_path + } + } + } + + dynamic "apt_repository" { + for_each = var.format == "APT" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "yum_repository" { + for_each = var.format == "YUM" && var.repo_public_repository != null ? [1] : [] + content { + public_repository { + repository_base = var.repository_base + repository_path = var.repository_path + } + } + } + + dynamic "yum_repository" { + for_each = var.format == "YUM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "common_repository" { + for_each = var.format == "COMMON" ? [1] : [] + content { + uri = var.repo_mirror_url + } + } + + # Only enable upstream credentials if user wants it + dynamic "upstream_credentials" { + for_each = var.use_upstream_credentials ? [1] : [] + content { + username_password_credentials { + username = var.repo_username + password_secret_version = google_secret_manager_secret_version.repo_password_secret_version[0].name + } + } + } + } + } + + depends_on = [ + google_secret_manager_secret.repo_password_secret, + google_secret_manager_secret_version.repo_password_secret_version, + ] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml new file mode 100644 index 0000000000..6b68c98a54 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - secretmanager.googleapis.com + - artifactregistry.googleapis.com + - cloudbuild.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf new file mode 100644 index 0000000000..92b6dbb165 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "registry_url" { + description = "The URL of the created artifact registry." + value = "${var.region}-docker.pkg.dev/${var.project_id}/${var.deployment_name}" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf new file mode 100644 index 0000000000..a795060fb7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf @@ -0,0 +1,49 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "terraform_data" "input_validation" { + lifecycle { + precondition { + condition = ( + var.repo_password == null || + (var.use_upstream_credentials && var.repo_mode == "REMOTE_REPOSITORY") + ) + error_message = "repo_password may be set only when repo_mode=REMOTE_REPOSITORY and use_upstream_credentials=true." + } + + precondition { + condition = ( + !var.use_upstream_credentials || + var.repo_mode == "REMOTE_REPOSITORY" + ) + error_message = "use_upstream_credentials is allowed only when repo_mode is REMOTE_REPOSITORY." + } + + precondition { + condition = ( + var.repo_mode != "REMOTE_REPOSITORY" || + (var.repo_public_repository != null || var.repo_mirror_url != null) + ) + error_message = "For a REMOTE_REPOSITORY you must set repo_public_repository or repo_mirror_url." + } + + precondition { + condition = ( + !contains(["APT", "YUM"], var.format) || + (var.repository_base != null && var.repository_path != null) + ) + error_message = "APT/YUM formats require repository_base and repository_path." + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf new file mode 100644 index 0000000000..9a4eecb921 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf @@ -0,0 +1,122 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Project ID where the artifact registry and secret are created." + type = string +} + +variable "region" { + description = "Region for the artifact registry." + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment." + type = string +} + +variable "labels" { + description = "Labels to add to the artifact registry. Key-value pairs." + type = map(string) + default = {} +} + +variable "repo_password" { + description = "Optional password/API key. If null, one will be randomly generated." + type = string + default = null +} + +variable "user_managed_replication" { + description = <<-DOC + (Optional) A list of objects to enable user-managed replication. + Each object can have: + location = string + kms_key_name = optional(string) + If empty, auto replication is used. + DOC + type = list(object({ + location = string + kms_key_name = optional(string) + })) + default = [] +} + +variable "format" { + description = "Artifact Registry format (e.g., DOCKER)." + type = string + default = "DOCKER" +} + +variable "repo_mode" { + description = "Artifact Registry mode (STANDARD_REPOSITORY, REMOTE_REPOSITORY, etc.)." + type = string + default = "STANDARD_REPOSITORY" + + validation { + condition = can(regex("^(STANDARD_REPOSITORY|REMOTE_REPOSITORY|VIRTUAL_REPOSITORY)$", var.repo_mode)) + error_message = "repo_mode must be one of STANDARD_REPOSITORY, REMOTE_REPOSITORY, or VIRTUAL_REPOSITORY." + } +} + +variable "repo_public_repository" { + description = <<-DOC + For REMOTE_REPOSITORY, name of a known public repo as per the Terraform module + (e.g., DOCKER_HUB) or null for custom repo. + DOC + type = string + default = null + + # To Do: implement validation + # validation { + # condition = ((var.repo_mode != "REMOTE_REPOSITORY" && var.repo_public_repository == null) || (var.repo_mode == "REMOTE_REPOSITORY" && (var.repo_public_repository != null || var.repo_mirror_url != null))) + # error_message = "If repo_mode is REMOTE_REPOSITORY, you must set either repo_public_repository or repo_mirror_url. Otherwise, leave them null." + # } +} + +variable "repo_mirror_url" { + description = "For REMOTE_REPOSITORY, URL for a custom or common mirror." + type = string + default = null +} + +variable "use_upstream_credentials" { + description = <<-DOC + Configure Service Account to use upstream credentials for REMOTE_REPOSITORY: + If true, a username/password is used for the REMOTE_REPOSITORY mirror. + If false (or if repo_password == null), no password is created at all. + Note: Blueprint credentials will be stored in Secrets Manager. + DOC + type = bool + default = false +} + +variable "repo_username" { + description = "Username for external repository." + type = string + default = null +} + +variable "repository_base" { + description = "For APT/YUM public repos, repository_base (e.g., 'DEBIAN', 'UBUNTU')." + type = string + default = null +} + +variable "repository_path" { + description = "For APT/YUM public repos, repository_path (e.g., 'debian/dists/buster')." + type = string + default = null +} diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf new file mode 100644 index 0000000000..392a7131d2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md new file mode 100644 index 0000000000..23bf87398a --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md @@ -0,0 +1,76 @@ +## Description + +Creates a BigQuery dataset. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage +This is a simple usage. + +```yaml + - id: bq-dataset + source: community/modules/database/bigquery-dataset + settings: + dataset_id: my_dataset +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_dataset.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | The name of the dataset to be created | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the dataset. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [dataset\_id](#output\_dataset\_id) | Name of the dataset that was created. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf new file mode 100644 index 0000000000..1a9c4bba60 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-dataset", ghpc_role = "database" }) +} +locals { + dataset_id = var.dataset_id != null ? var.dataset_id : replace("${var.deployment_name}_dataset_${random_id.resource_name_suffix.hex}", "-", "_") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_bigquery_dataset" "pbsb" { + dataset_id = local.dataset_id + project = var.project_id + labels = local.labels +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml new file mode 100644 index 0000000000..87ff9357e4 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigquery.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf new file mode 100644 index 0000000000..9cd8e5df31 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "dataset_id" { + description = "Name of the dataset that was created." + value = google_bigquery_dataset.pbsb.dataset_id +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf new file mode 100644 index 0000000000..90c229af6b --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "dataset_id" { + description = "The name of the dataset to be created" + type = string + default = null +} + +variable "labels" { + description = "Labels to add to the dataset. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf new file mode 100644 index 0000000000..12ddbe842d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/README.md b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/README.md new file mode 100644 index 0000000000..ef67cfef01 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/README.md @@ -0,0 +1,87 @@ +## Description + +Creates a BigQuery table with a specified schema. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage + +```yaml +id: bq-table + source: community/modules/database/bigquery-table + use: [bq-dataset] + settings: + table_schema: + ' + [ + { + "name": "id", "type": "STRING" + } + ] + ' +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_table.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Dataset name to be used to create the new BQ Table | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the tables. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [table\_id](#input\_table\_id) | Table name to be used to create the new BQ Table | `string` | `null` | no | +| [table\_schema](#input\_table\_schema) | Schema used to create the new BQ Table | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [dataset\_id](#output\_dataset\_id) | ID of BQ dataset | +| [table\_id](#output\_table\_id) | ID of created BQ table | +| [table\_name](#output\_table\_name) | Name of created BQ table | + diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/main.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/main.tf new file mode 100644 index 0000000000..73f3923e00 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/main.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-table", ghpc_role = "database" }) +} + +locals { + table_id = var.table_id != null ? var.table_id : replace("${var.deployment_name}_table_${random_id.resource_name_suffix.hex}", "-", "_") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_bigquery_table" "pbsb" { + deletion_protection = false + project = var.project_id + table_id = local.table_id + dataset_id = var.dataset_id + schema = var.table_schema + labels = local.labels +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml new file mode 100644 index 0000000000..87ff9357e4 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigquery.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf new file mode 100644 index 0000000000..4220ec1390 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "table_name" { + description = "Name of created BQ table" + value = google_bigquery_table.pbsb.friendly_name +} +output "table_id" { + description = "ID of created BQ table" + value = google_bigquery_table.pbsb.table_id +} +output "dataset_id" { + description = "ID of BQ dataset" + value = google_bigquery_table.pbsb.dataset_id +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf new file mode 100644 index 0000000000..ec474b4e64 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "labels" { + description = "Labels to add to the tables. Key-value pairs." + type = map(string) +} + +variable "table_id" { + description = "Table name to be used to create the new BQ Table" + type = string + default = null +} + +variable "dataset_id" { + description = "Dataset name to be used to create the new BQ Table" + type = string +} + +variable "table_schema" { + description = "Schema used to create the new BQ Table" + type = string +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf new file mode 100644 index 0000000000..12ddbe842d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md new file mode 100644 index 0000000000..08364c175b --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md @@ -0,0 +1,107 @@ +## Description + +terraform-google-sql makes it easy to create a Google CloudSQL instance and +implement high availability settings. This module is meant for use with +Terraform 0.13+ and tested using Terraform 1.0+. + +The cloudsql created here is used to integrate with the slurm cluster to enable +accounting data storage. + +### Example + +```yaml +- id: cloudsql + source: community/modules/database/slurm-cloudsql-federation + use: [network] + settings: + sql_instance_name: slurm-sql6-demo + tier: "db-f1-micro" +``` + +This creates a cloud sql instance, including a database, user that would allow +the slurm cluster to use as an external DB. In addition, it will allow BigQuery +to run federated query through it. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | +| [google](#requirement\_google) | >= 3.83 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_connection.connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_connection) | resource | +| [google_compute_address.psc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_forwarding_rule.psc_consumer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_forwarding_rule) | resource | +| [google_sql_database.database](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database) | resource | +| [google_sql_database_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance) | resource | +| [google_sql_user.users](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [random_password.password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [authorized\_networks](#input\_authorized\_networks) | IP address ranges as authorized networks of the Cloud SQL for MySQL instances | `list(string)` | `[]` | no | +| [data\_cache\_enabled](#input\_data\_cache\_enabled) | Whether data cache is enabled for the instance. Can be used with ENTERPRISE\_PLUS edition. | `bool` | `false` | no | +| [database\_flags](#input\_database\_flags) | Database flags to set on instance. | `map(string)` | `{}` | no | +| [database\_version](#input\_database\_version) | The version of the database to be created. | `string` | `"MYSQL_8_0"` | no | +| [deletion\_protection](#input\_deletion\_protection) | Whether or not to allow Terraform to destroy the instance. | `string` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [disk\_autoresize](#input\_disk\_autoresize) | Set to false to disable automatic disk grow. | `bool` | `true` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of the database disk in GiB. | `number` | `null` | no | +| [edition](#input\_edition) | value | `string` | `"ENTERPRISE"` | no | +| [enable\_backups](#input\_enable\_backups) | Set true to enable backups | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is going to be created in.:
`projects//global/networks/`" | `string` | n/a | yes | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [query\_insights](#input\_query\_insights) | Query insights configuration. | 
object({
    enabled                 = optional(bool, false)
    query_plans_per_minute  = optional(number)
    query_string_length     = optional(number)
    record_application_tags = optional(bool)
    record_client_address   = optional(bool)
  })
| `{}` | no | +| [region](#input\_region) | The region where SQL instance will be configured | `string` | n/a | yes | +| [sql\_instance\_name](#input\_sql\_instance\_name) | name given to the sql instance for ease of identificaion | `string` | n/a | yes | +| [sql\_password](#input\_sql\_password) | Password for the SQL database. | `any` | `null` | no | +| [sql\_username](#input\_sql\_username) | Username for the SQL database | `string` | `"slurm"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self link of the network where Cloud SQL instance PSC endpoint will be created | `string` | `null` | no | +| [tier](#input\_tier) | The machine type to use for the SQL instance | `string` | n/a | yes | +| [use\_psc\_connection](#input\_use\_psc\_connection) | Create Private Service Connection instead of using Private Service Access peering | `bool` | `false` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cloudsql](#output\_cloudsql) | Describes the cloudsql instance. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf new file mode 100644 index 0000000000..9b518a1b5f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf @@ -0,0 +1,165 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "slurm-cloudsql-federation", ghpc_role = "database" }) +} + +locals { + user_managed_replication = var.user_managed_replication +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "random_password" "password" { + length = 12 + special = false +} + +locals { + sql_instance_name = var.sql_instance_name == null ? "${var.deployment_name}-sql-${random_id.resource_name_suffix.hex}" : var.sql_instance_name + sql_password = var.sql_password == null ? random_password.password.result : var.sql_password +} + + +resource "google_sql_database_instance" "instance" { + project = var.project_id + depends_on = [var.private_vpc_connection_peering] + name = local.sql_instance_name + region = var.region + deletion_protection = var.deletion_protection + database_version = var.database_version + + settings { + disk_size = var.disk_size_gb + disk_autoresize = var.disk_autoresize + edition = var.edition + tier = var.tier + user_labels = local.labels + + dynamic "data_cache_config" { + for_each = var.edition == "ENTERPRISE_PLUS" ? [""] : [] + content { + data_cache_enabled = var.data_cache_enabled + } + } + + dynamic "database_flags" { + for_each = var.database_flags + content { + name = database_flags.key + value = database_flags.value + } + } + + insights_config { + query_insights_enabled = var.query_insights.enabled + query_plans_per_minute = var.query_insights.query_plans_per_minute + query_string_length = var.query_insights.query_string_length + record_application_tags = var.query_insights.record_application_tags + record_client_address = var.query_insights.record_client_address + } + + ip_configuration { + ipv4_enabled = false + private_network = var.use_psc_connection ? null : var.network_id + enable_private_path_for_google_cloud_services = true + + dynamic "authorized_networks" { + for_each = var.use_psc_connection ? [] : var.authorized_networks + iterator = ip_range + + content { + value = ip_range.value + } + } + dynamic "psc_config" { + for_each = var.use_psc_connection ? [""] : [] + content { + psc_enabled = true + allowed_consumer_projects = [var.project_id] + } + } + } + + backup_configuration { + enabled = var.enable_backups + # to allow easy switching between ENTERPRISE and ENTERPRISE_PLUS + transaction_log_retention_days = 7 + } + } + lifecycle { + precondition { + condition = var.disk_autoresize && var.disk_size_gb == null || !var.disk_autoresize + error_message = "If setting disk_size_gb set disk_autorize to false to prevent re-provisioning of the instance after disk auto-expansion." + } + } +} + + + +resource "google_compute_address" "psc" { + count = var.use_psc_connection ? 1 : 0 + project = var.project_id + name = local.sql_instance_name + address_type = "INTERNAL" + region = var.region + subnetwork = var.subnetwork_self_link + labels = local.labels +} + +resource "google_compute_forwarding_rule" "psc_consumer" { + count = var.use_psc_connection ? 1 : 0 + name = local.sql_instance_name + project = var.project_id + region = var.region + subnetwork = var.subnetwork_self_link + ip_address = google_compute_address.psc[0].self_link + load_balancing_scheme = "" + recreate_closed_psc = true + target = google_sql_database_instance.instance.psc_service_attachment_link +} + +resource "google_sql_database" "database" { + project = var.project_id + name = "slurm_accounting" + instance = google_sql_database_instance.instance.name +} + +resource "google_sql_user" "users" { + project = var.project_id + name = var.sql_username + instance = google_sql_database_instance.instance.name + password = local.sql_password +} + +resource "google_bigquery_connection" "connection" { + provider = google + project = var.project_id + location = var.region + cloud_sql { + instance_id = google_sql_database_instance.instance.connection_name + database = google_sql_database.database.name + type = "MYSQL" + credential { + username = google_sql_user.users.name + password = google_sql_user.users.password + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml new file mode 100644 index 0000000000..fc0cae0859 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigqueryconnection.googleapis.com + - sqladmin.googleapis.com + - servicenetworking.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf new file mode 100644 index 0000000000..0d05221cd8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "cloudsql" { + description = "Describes the cloudsql instance." + sensitive = true + value = { + server_ip = var.use_psc_connection ? google_compute_address.psc[0].address : google_sql_database_instance.instance.ip_address[0].ip_address + user = google_sql_user.users.name + password = google_sql_user.users.password + db_name = google_sql_database.database.name + user_managed_replication = local.user_managed_replication + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf new file mode 100644 index 0000000000..a2f150419e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf @@ -0,0 +1,173 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "authorized_networks" { + description = "IP address ranges as authorized networks of the Cloud SQL for MySQL instances" + type = list(string) + default = [] + nullable = false +} + +variable "database_version" { + description = "The version of the database to be created." + type = string + default = "MYSQL_8_0" + validation { + condition = contains(["MYSQL_5_7", "MYSQL_8_0", "MYSQL_8_4"], var.database_version) + error_message = "The database version must be either MYSQL_5_7, MYSQL_8_0 or MYSQL_8_4." + } +} + +variable "data_cache_enabled" { + description = "Whether data cache is enabled for the instance. Can be used with ENTERPRISE_PLUS edition." + type = bool + default = false +} + +variable "database_flags" { + description = "Database flags to set on instance." + type = map(string) + default = {} + nullable = false +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "disk_autoresize" { + description = "Set to false to disable automatic disk grow." + type = bool + default = true +} + +variable "disk_size_gb" { + description = "Size of the database disk in GiB." + type = number + default = null +} + +variable "edition" { + description = "value" + type = string + validation { + condition = contains(["ENTERPRISE", "ENTERPRISE_PLUS"], var.edition) + error_message = "The database edition must be either ENTERPRISE or ENTERPRISE_PLUS" + } + default = "ENTERPRISE" +} + +variable "enable_backups" { + description = "Set true to enable backups" + type = bool + default = false +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "query_insights" { + description = "Query insights configuration." + nullable = false + default = {} + type = object({ + enabled = optional(bool, false) + query_plans_per_minute = optional(number) + query_string_length = optional(number) + record_application_tags = optional(bool) + record_client_address = optional(bool) + }) +} + +variable "region" { + description = "The region where SQL instance will be configured" + type = string +} + +variable "tier" { + description = "The machine type to use for the SQL instance" + type = string +} + +variable "sql_instance_name" { + description = "name given to the sql instance for ease of identificaion" + type = string +} + +variable "deletion_protection" { + description = "Whether or not to allow Terraform to destroy the instance." + type = string + default = false +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} + +variable "sql_username" { + description = "Username for the SQL database" + type = string + default = "slurm" +} + +variable "sql_password" { + description = "Password for the SQL database." + type = any + default = null +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is going to be created in.: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "private_vpc_connection_peering" { + description = "The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation." + type = string + default = null +} + +variable "subnetwork_self_link" { + description = "Self link of the network where Cloud SQL instance PSC endpoint will be created" + type = string + default = null +} + +variable "user_managed_replication" { + type = list(object({ + location = string + kms_key_name = optional(string) + })) + description = "Replication parameters that will be used for defined secrets" + default = [] +} + +variable "use_psc_connection" { + description = "Create Private Service Connection instead of using Private Service Access peering" + type = bool + default = false +} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf new file mode 100644 index 0000000000..7e672858b6 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" + } + + required_version = ">= 0.13.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md new file mode 100644 index 0000000000..d39a58afe1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md @@ -0,0 +1,158 @@ +> [!WARNING] +> This module is deprecated and will be removed on July 1, 2025. The +> recommended replacement is the +> [GCP Managed Lustre module](../../../../modules/file-system/managed-lustre/README.md) + +## Description +This module creates a DDN EXAScaler Cloud Lustre file system using code based on DDN's +[exascaler-cloud-terraform](https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp) (`scripts/2.2.2` is last release with GCP-specific module). + +More information about the architecture can be found at +[Overview of Lustre and EXAScaler Cloud][architecture]. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +> **Warning**: This file system has a license cost as described in the pricing +> section of the [DDN EXAScaler Cloud Marketplace Solution][marketplace]. +> +> **Note**: By default security.public_key is set to `null`, therefore the +> admin user is not created. To ensure the admin user is created, provide a +> public key via the security setting. +> +> **Note**: This module's instances require access to Google APIs and +> therefore, instances must have public IP address or it must be used in a +> subnetwork where [Private Google Access][private-google-access] is enabled. + +[private-google-access]: https://cloud.google.com/vpc/docs/configure-private-google-access +[marketplace]: https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud +[architecture]: https://cloud.google.com/architecture/parallel-file-systems-for-hpc#overview_of_lustre_and_exascaler_cloud + +## Mounting + +To mount the DDN EXAScaler Lustre file system you must first install the DDN +Lustre client and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +the [hpc-enterprise-slurm.yaml](../../../../examples/hpc-enterprise-slurm.yaml) for an +example of using this module with Slurm. + +If mounting is not automatically handled as described above, the DDN-EXAScaler +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + # This file system has an associated license cost. + # https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud + - id: lustrefs + source: community/modules/file-system/DDN-EXAScaler + use: [network1] + settings: {local_mount: /scratch} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(lustrefs.install_ddn_lustre_client_runner) + - $(lustrefs.mount_runner) + +``` + +See [additional documentation][ddn-install-docs] from DDN EXAScaler. + +[ddn-install-docs]: https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp#install-new-exascaler-cloud-clients +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## Support + +EXAScaler Cloud includes self-help support with access to publicly available +documents and videos. Premium support includes 24x7x365 access to DDN's experts, +along with support community access, automated notifications of updates and +other premium support features. For more information, visit +[EXAscaler Cloud on GCP][exa-gcp]. + +[exa-gcp]: https://console.cloud.google.com/marketplace/product/ddnstorage/exascaler-cloud + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [ddn\_exascaler](#module\_ddn\_exascaler) | github.com/DDNStorage/exascaler-cloud-terraform//gcp | a3355d50deebe45c0556b45bd599059b7c06988d | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [boot](#input\_boot) | Boot disk properties | 
object({
    disk_type   = string
    auto_delete = bool
    script_url  = string
  })
| 
{
  "auto_delete": true,
  "disk_type": "pd-standard",
  "script_url": null
}
| no | +| [cls](#input\_cls) | Compute client properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 0,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-2",
  "public_ip": true
}
| no | +| [clt](#input\_clt) | Compute client target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 0,
  "disk_size": 256,
  "disk_type": "pd-standard"
}
| no | +| [fsname](#input\_fsname) | EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long | `string` | `"exacloud"` | no | +| [image](#input\_image) | DEPRECATED: Source image properties | `any` | `null` | no | +| [instance\_image](#input\_instance\_image) | Source image properties

Expected Fields:
name: Unavailable with this module.
family: The image family to use.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp",
  "project": "ddn-public"
}
| no | +| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. Key-value pairs. | `map(string)` | `{}` | no | +| [local\_mount](#input\_local\_mount) | Mountpoint (at the client instances) for this EXAScaler system | `string` | `"/shared"` | no | +| [mds](#input\_mds) | Metadata server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | +| [mdt](#input\_mdt) | Metadata target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | +| [mgs](#input\_mgs) | Management server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | +| [mgt](#input\_mgt) | Management target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | +| [mnt](#input\_mnt) | Monitoring target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | +| [network\_properties](#input\_network\_properties) | Network options. 'network\_self\_link' or 'network\_properties' must be provided. | 
object({
    routing = string
    tier    = string
    id      = string
    auto    = bool
    mtu     = number
    new     = bool
    nat     = bool
  })
| `null` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self-link of the VPC network to where the system is connected. Ignored if 'network\_properties' is provided. 'network\_self\_link' or 'network\_properties' must be provided. | `string` | `null` | no | +| [oss](#input\_oss) | Object Storage server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 3,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-16",
  "public_ip": true
}
| no | +| [ost](#input\_ost) | Object Storage target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | +| [prefix](#input\_prefix) | EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud') | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Compute Platform project that will host the EXAScaler filesystem | `string` | n/a | yes | +| [security](#input\_security) | Security options | 
object({
    admin              = string
    public_key         = string
    block_project_keys = bool
    enable_os_login    = bool
    enable_local       = bool
    enable_ssh         = bool
    enable_http        = bool
    ssh_source_ranges  = list(string)
    http_source_ranges = list(string)
  })
| 
{
  "admin": "stack",
  "block_project_keys": false,
  "enable_http": false,
  "enable_local": false,
  "enable_os_login": true,
  "enable_ssh": false,
  "http_source_ranges": [
    "0.0.0.0/0"
  ],
  "public_key": null,
  "ssh_source_ranges": [
    "0.0.0.0/0"
  ]
}
| no | +| [service\_account](#input\_service\_account) | Service account name used by deploy application | 
object({
    new   = bool
    email = string
  })
| 
{
  "email": null,
  "new": false
}
| no | +| [subnetwork\_address](#input\_subnetwork\_address) | The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork\_properties' is provided. | `string` | `null` | no | +| [subnetwork\_properties](#input\_subnetwork\_properties) | Subnetwork properties. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | 
object({
    address = string
    private = bool
    id      = string
    new     = bool
  })
| `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork\_properties' is provided. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | `string` | `null` | no | +| [waiter](#input\_waiter) | Waiter to check progress and result for deployment. | `string` | `null` | no | +| [zone](#input\_zone) | Compute Platform zone where the servers will be located | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_config\_script](#output\_client\_config\_script) | Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler. | +| [http\_console](#output\_http\_console) | HTTP address to access the system web console. | +| [install\_ddn\_lustre\_client\_runner](#output\_install\_ddn\_lustre\_client\_runner) | Runner that encapsulates the `client_config_script` output on this module. | +| [mount\_command](#output\_mount\_command) | Command to mount the file system. `client_config_script` must be run first. | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the DDN EXAScaler Lustre file system | +| [network\_storage](#output\_network\_storage) | Describes a EXAScaler system to be mounted by other systems. | +| [private\_addresses](#output\_private\_addresses) | Private IP addresses for all instances. | +| [ssh\_console](#output\_ssh\_console) | Instructions to ssh into the instances. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf new file mode 100644 index 0000000000..6a2fc4b702 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ddn-exascaler", ghpc_role = "file-system" }) +} + +locals { + + network_id = var.network_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.network_self_link)[0] : null + named_net = { + routing = "REGIONAL" + tier = "STANDARD" + id = local.network_id + auto = false + mtu = 1500 + new = false + nat = false + } + + subnetwork_id = var.subnetwork_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.subnetwork_self_link)[0] : null + named_subnet = { + address = var.subnetwork_address + private = true + id = local.subnetwork_id + new = false + } +} + +module "ddn_exascaler" { + source = "github.com/DDNStorage/exascaler-cloud-terraform//gcp?ref=a3355d50deebe45c0556b45bd599059b7c06988d" + fsname = var.fsname + zone = var.zone + project = var.project_id + prefix = var.prefix + labels = local.labels + security = var.security + service_account = var.service_account + waiter = var.waiter + network = var.network_properties == null ? local.named_net : var.network_properties + subnetwork = var.subnetwork_properties == null ? local.named_subnet : var.subnetwork_properties + boot = var.boot + image = var.instance_image + mgs = var.mgs + mgt = var.mgt + mnt = var.mnt + mds = var.mds + mdt = var.mdt + oss = var.oss + ost = var.ost + cls = var.cls + clt = var.clt +} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml new file mode 100644 index 0000000000..b995bd4358 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - deploymentmanager.googleapis.com + - iam.googleapis.com + - runtimeconfig.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf new file mode 100644 index 0000000000..2e9ae732ae --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf @@ -0,0 +1,90 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +output "private_addresses" { + description = "Private IP addresses for all instances." + value = module.ddn_exascaler.private_addresses +} + +output "ssh_console" { + description = "Instructions to ssh into the instances." + value = module.ddn_exascaler.ssh_console +} + +output "client_config_script" { + description = "Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler." + value = module.ddn_exascaler.client_config +} + +output "install_ddn_lustre_client_runner" { + description = "Runner that encapsulates the `client_config_script` output on this module." + value = local.client_install_runner +} + +locals { + client_install_runner = { + "type" = "shell" + "content" = module.ddn_exascaler.client_config + "destination" = "install_ddn_lustre_client.sh" + } + + # Mount command provided by DDN does not support custom local mount + split_mount_cmd = split(" ", module.ddn_exascaler.mount_command) + split_mount_cmd_wo_mountpoint = slice(local.split_mount_cmd, 0, length(local.split_mount_cmd) - 1) + mount_cmd = "${join(" ", local.split_mount_cmd_wo_mountpoint)} ${var.local_mount}" + mount_cmd_w_mkdir = "mkdir -p ${var.local_mount} && ${local.mount_cmd}" + mount_runner = { + "type" = "shell" + "content" = local.mount_cmd_w_mkdir + "destination" = "mount-ddn-lustre.sh" + } +} + +output "mount_command" { + description = "Command to mount the file system. `client_config_script` must be run first." + value = local.mount_cmd_w_mkdir +} + +output "mount_runner" { + description = "Runner to mount the DDN EXAScaler Lustre file system" + value = local.mount_runner +} + +output "http_console" { + description = "HTTP address to access the system web console." + value = module.ddn_exascaler.http_console +} + +output "network_storage" { + description = "Describes a EXAScaler system to be mounted by other systems." + value = { + server_ip = split(":", split(" ", module.ddn_exascaler.mount_command)[3])[0] + remote_mount = length(regexall("^/.*", var.fsname)) > 0 ? var.fsname : format("/%s", var.fsname) + local_mount = var.local_mount != null ? var.local_mount : format("/mnt/%s", var.fsname) + fs_type = "lustre" + mount_options = "" + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } + depends_on = [ + module.ddn_exascaler + ] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf new file mode 100644 index 0000000000..68bcc8a8ba --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf @@ -0,0 +1,502 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +# EXAScaler filesystem name +# only alphanumeric characters are allowed, +# and the value must be 1-8 characters long +variable "fsname" { + description = "EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long" + type = string + default = "exacloud" +} + +# Project ID to manage resources +# https://cloud.google.com/resource-manager/docs/creating-managing-projects +variable "project_id" { + description = "Compute Platform project that will host the EXAScaler filesystem" + type = string +} + +# Zone name to manage resources +# https://cloud.google.com/compute/docs/regions-zones +variable "zone" { + description = "Compute Platform zone where the servers will be located" + type = string +} + +# Service account name used by deploy application +# https://cloud.google.com/iam/docs/service-accounts +# new: create a new custom service account or use an existing one: true or false +# email: existing service account email address, will be using if new is false +# set email = null to use the default compute service account +variable "service_account" { + description = "Service account name used by deploy application" + type = object({ + new = bool + email = string + }) + default = { + new = false + email = null + } +} + +# Waiter to check progress and result for deployment. +# To use Google Deployment Manager: +# waiter = "deploymentmanager" +# To use generic Google Cloud SDK command line: +# waiter = "sdk" +# If you don’t want to wait until the deployment is complete: +# waiter = null +# https://cloud.google.com/deployment-manager/runtime-configurator/creating-a-waiter +variable "waiter" { + description = "Waiter to check progress and result for deployment." + type = string + default = null +} + +# Security options +# admin: optional user name for remote SSH access +# Set admin = null to disable creation admin user +# public_key: path to the SSH public key on the local host +# Set public_key = null to disable creation admin user +# block_project_keys: true or false +# Block project-wide public SSH keys if you want to restrict +# deployment to only user with deployment-level public SSH key. +# https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys +# enable_os_login: true or false +# Enable or disable OS Login feature. +# Please note, enabling this option disables other security options: +# admin, public_key and block_project_keys. +# https://cloud.google.com/compute/docs/instances/managing-instance-access#enable_oslogin +# enable_local: true or false, enable or disable firewall rules for local access +# enable_ssh: true or false, enable or disable remote SSH access +# ssh_source_ranges: source IP ranges for remote SSH access in CIDR notation +# enable_http: true or false, enable or disable remote HTTP access +# http_source_ranges: source IP ranges for remote HTTP access in CIDR notation +variable "security" { + description = "Security options" + type = object({ + admin = string + public_key = string + block_project_keys = bool + enable_os_login = bool + enable_local = bool + enable_ssh = bool + enable_http = bool + ssh_source_ranges = list(string) + http_source_ranges = list(string) + }) + + default = { + admin = "stack" + public_key = null + block_project_keys = false + enable_os_login = true + enable_local = false + enable_ssh = false + enable_http = false + ssh_source_ranges = [ + "0.0.0.0/0" + ] + http_source_ranges = [ + "0.0.0.0/0" + ] + } +} + +variable "network_self_link" { + description = "The self-link of the VPC network to where the system is connected. Ignored if 'network_properties' is provided. 'network_self_link' or 'network_properties' must be provided." + type = string + default = null +} + +# Network properties +# https://cloud.google.com/vpc/docs/vpc +# routing: network-wide routing mode: REGIONAL or GLOBAL +# tier: networking tier for VM interfaces: STANDARD or PREMIUM +# id: existing network id, will be using if new is false +# auto: create subnets in each region automatically: false or true +# mtu: maximum transmission unit in bytes: 1460 - 1500 +# new: create a new network or use an existing one: true or false +# nat: allow instances without external IP to communicate with the outside world: true or false +variable "network_properties" { + description = "Network options. 'network_self_link' or 'network_properties' must be provided." + type = object({ + routing = string + tier = string + id = string + auto = bool + mtu = number + new = bool + nat = bool + }) + + default = null +} + +variable "subnetwork_self_link" { + description = "The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork_properties' is provided. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." + type = string + default = null +} + +variable "subnetwork_address" { + description = "The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork_properties' is provided." + type = string + default = null +} + +# Subnetwork properties +# https://cloud.google.com/vpc/docs/vpc +# address: IP range of internal addresses for a new subnetwork +# private: when enabled VMs in this subnetwork without external +# IP addresses can access Google APIs and services by using +# Private Google Access: true or false +# https://cloud.google.com/vpc/docs/private-access-options +# id: existing subnetwork id, will be using if new is false +# new: create a new subnetwork or use an existing one: true or false +variable "subnetwork_properties" { + description = "Subnetwork properties. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." + type = object({ + address = string + private = bool + id = string + new = bool + }) + default = null +} +# Boot disk properties +# disk_type: pd-standard, pd-ssd or pd-balanced +# auto_delete: true or false +# whether the disk will be auto-deleted when the instance is deleted +variable "boot" { + description = "Boot disk properties" + type = object({ + disk_type = string + auto_delete = bool + script_url = string + }) + default = { + disk_type = "pd-standard" + auto_delete = true + script_url = null + } +} + +# Source image properties +# project: project name +# family: image family name +# name: !!DEPRECATED!! - image name +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: Source image properties" + type = any + # Omitting type checking so validation can provide more useful error message + # type = object({ + # project = string + # family = string + # }) + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + Source image properties + + Expected Fields: + name: Unavailable with this module. + family: The image family to use. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "ddn-public" + family = "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp" + } + + validation { + condition = !can(coalesce(var.instance_image.name)) + error_message = "In var.instance_image, the \"name\" field is not used, please use the \"family\" setting." + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, the \"family\" field must be a string set to the image family." + } +} + +# Management server properties +# node_type: type of management server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of management servers +variable "mgs" { + description = "Management server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-32" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 1 + } +} + +# Management target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of management target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of management target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of management target in GB (scratch disk size must be exactly 375) +# disk_count: number of management targets +# disk_raid: create striped management target, true or false +variable "mgt" { + description = "Management target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 128 + disk_count = 1 + disk_raid = false + } +} + + +# Monitoring target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of monitoring target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of monitoring target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of monitoring target in GB (scratch disk size must be exactly 375) +# disk_count: number of monitoring targets +# disk_raid: create striped monitoring target, true or false +variable "mnt" { + description = "Monitoring target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 128 + disk_count = 1 + disk_raid = false + } +} + +# Metadata server properties +# node_type: type of metadata server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of metadata servers +variable "mds" { + description = "Metadata server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-32" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 1 + } +} + +# Metadata target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of metadata target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of metadata target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of metadata target in GB (scratch disk size must be exactly 375) +# disk_count: number of metadata targets +# disk_raid: create striped metadata target, true or false +variable "mdt" { + description = "Metadata target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-ssd" + disk_size = 3500 + disk_count = 1 + disk_raid = false + } +} + +# Object Storage server properties +# node_type: type of storage server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of storage servers +variable "oss" { + description = "Object Storage server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-16" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 3 + } +} + +# Object Storage target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of storage target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of storage target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of storage target in GB (scratch disk size must be exactly 375) +# disk_count: number of storage targets +# disk_raid: create striped storage target, true or false +variable "ost" { + description = "Object Storage target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-ssd" + disk_size = 3500 + disk_count = 1 + disk_raid = false + } +} + +# Compute client properties +# node_type: type of compute client +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of compute clients +variable "cls" { + description = "Compute client properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-2" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 0 + } +} +# Compute client target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of compute target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of compute target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of compute target in GB (scratch disk size must be exactly 375) +# disk_count: number of compute targets +variable "clt" { + description = "Compute client target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 256 + disk_count = 0 + } +} +variable "local_mount" { + description = "Mountpoint (at the client instances) for this EXAScaler system" + type = string + default = "/shared" +} + +variable "prefix" { + description = "EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud')" + type = string + default = null +} + +variable "labels" { + description = "Labels to add to EXAScaler Cloud deployment. Key-value pairs." + type = map(string) + default = {} +} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf new file mode 100644 index 0000000000..2981b4dd75 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md b/deletion-test/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md new file mode 100644 index 0000000000..04db0acb8c --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md @@ -0,0 +1 @@ +> **_NOTE:_** Cluster Toolkit is dropping support for the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository. The DAOS example blueprints (`hpc-slurm-daos.yaml` and `pfs-daos.yaml`) have been removed from the Cluster Toolkit. We recommend migrating to the first-party [Parallelstore](../../../../modules/file-system/parallelstore/) module for similar functionality. To help with this transition, see the Parallelstore example blueprints ([pfs-parallelstore.yaml](../../../../examples/pfs-parallelstore.yaml) and [ps-slurm.yaml](../../../../examples/ps-slurm.yaml)). If the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository is necessary, we recommend using the last Cluster Toolkit [v1.41.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/releases/tag/v1.41.0). diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/README.md b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/README.md new file mode 100644 index 0000000000..66aaaa46af --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/README.md @@ -0,0 +1,152 @@ +## Description + +This module creates a Network File Sharing (NFS) file system based on a VM +instance and [compute disk][disk]. This file system can share directories and +files with other clients over a network. `nfs-server` can be used by +[vm-instance](../../../../modules/compute/vm-instance/README.md) and SchedMD +community modules that create compute VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +If you are using Hyperdisk storage, check the possible disk size, IOPS, and throughput values for each disk type in the [Hyperdisk limits documentation](https://cloud.google.com/compute/docs/disks/hyperdisks#limits-disk). + +> **_WARNING:_** This module has only been tested against the HPC centos7 OS +> disk image (the default). Using other images may work, but have not been +> verified. + +[disk]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk + +### Example + +```yaml +- id: homefs + source: community/modules/file-system/nfs-server + use: [network1] +``` + +This creates a NFS on a virtual machine which allow other VMs to mount the +volume as an external file system. + +> **_NOTE:_** All disks are destroyed along with the instance, during a `gcluster destroy`/`terraform destroy` event. However, you can setup data retention with `create_boot_snapshot_before_destroy` (boot disk) and `create_snapshot_before_destroy` (data disk). + +## Mounting + +To mount the NFS Server you must first ensure that the NFS client has been +installed the and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +See the [hpc-centos-ss.yaml] test config for an example of using this module +with a `vm-instance` module. + +If mounting is not automatically handled as described above, the `nfs-server` +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + - id: nfs + source: community/modules/file-system/nfs-server + use: [network1] + settings: {local_mounts: [/mnt1]} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(nfs.install_nfs_client_runner) + - $(nfs.mount_runner) + +``` + +[hpc-centos-ss.yaml]: ../../../../tools/validate_configs/test_configs/hpc-centos-ss.yaml +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 6.14 | +| [null](#requirement\_null) | >= 3.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.14 | +| [null](#provider\_null) | >= 3.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_disk.attached_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_instance.compute_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource | +| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_default_service_account.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_default_service_account) | data source | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [auto\_delete\_disk](#input\_auto\_delete\_disk) | DEPRECATED: Whether or not the NFS disk should be auto-deleted | `string` | `null` | no | +| [boot\_disk\_size](#input\_boot\_disk\_size) | Storage size in GB for the boot disk | `number` | `null` | no | +| [boot\_disk\_type](#input\_boot\_disk\_type) | Storage type for the boot disk | `string` | `null` | no | +| [create\_boot\_snapshot\_before\_destroy](#input\_create\_boot\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the boot disk | `bool` | `false` | no | +| [create\_snapshot\_before\_destroy](#input\_create\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the NFS data disk | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the NFS instance if no name is specified. | `string` | n/a | yes | +| [disk\_size](#input\_disk\_size) | Storage size in GB for the NFS data disk | `number` | `"100"` | no | +| [image](#input\_image) | DEPRECATED: The VM image used by the NFS server | `string` | `null` | no | +| [instance\_image](#input\_instance\_image) | The VM image used by the NFS server.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to the NFS instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mounts](#input\_local\_mounts) | Mountpoint for this NFS compute instance | `list(string)` | 
[
  "/data"
]
| no | +| [machine\_type](#input\_machine\_type) | Type of the VM instance to use | `string` | `"n2d-standard-2"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the NFS VM. | `string` | `"default"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [provisioned\_iops](#input\_provisioned\_iops) | Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput | `number` | `null` | no | +| [provisioned\_throughput](#input\_provisioned\_throughput) | Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme | `number` | `null` | no | +| [scopes](#input\_scopes) | Scopes to apply to the controller | `list(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [service\_account](#input\_service\_account) | Service Account for the NFS server | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the NFS VM. | `string` | `null` | no | +| [type](#input\_type) | Storage type for the NFS data disk | `string` | `"pd-ssd"` | no | +| [zone](#input\_zone) | The zone name where the NFS instance located in. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | +| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | +| [network\_storage](#output\_network\_storage) | export of all desired folder directories | + diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf new file mode 100644 index 0000000000..a00d2681ba --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf @@ -0,0 +1,131 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "nfs-server", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + server_ip = google_compute_instance.compute_instance.network_interface[0].network_ip + fs_type = "nfs" + mount_options = "defaults,hard,intr" + install_nfs_client_runners = [for mount in var.local_mounts : + { + "type" = "shell" + "source" = "${path.module}/scripts/install-nfs-client.sh" + "destination" = "install-nfs${replace(mount, "/", "_")}.sh" + } + ] + mount_runners = [for mount in var.local_mounts : + { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"/exports${mount}\" \"${mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(mount, "/", "_")}.sh" + } + ] + ansible_mount_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/mount.yaml" + "destination" = "mount.yaml" + } +} + +data "google_compute_default_service_account" "default" {} + +resource "google_compute_disk" "attached_disk" { + project = var.project_id + name = "${local.name}-nfs-instance-disk" + size = var.disk_size + type = var.type + zone = var.zone + labels = local.labels + provisioned_iops = var.provisioned_iops + provisioned_throughput = var.provisioned_throughput + create_snapshot_before_destroy = var.create_snapshot_before_destroy +} + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project +} + +resource "null_resource" "image" { + triggers = { + name = try(var.instance_image.name, null), + family = try(var.instance_image.family, null), + project = var.instance_image.project + } +} + +resource "google_compute_disk" "boot_disk" { + project = var.project_id + + name = "${local.name}-boot-disk" + size = var.boot_disk_size + type = var.boot_disk_type + image = data.google_compute_image.compute_image.self_link + labels = local.labels + zone = var.zone + create_snapshot_before_destroy = var.create_boot_snapshot_before_destroy + + lifecycle { + replace_triggered_by = [null_resource.image] + ignore_changes = [ + image + ] + } +} + +resource "google_compute_instance" "compute_instance" { + project = var.project_id + name = "${local.name}-nfs-instance" + zone = var.zone + machine_type = var.machine_type + + boot_disk { + auto_delete = false + source = google_compute_disk.boot_disk.self_link + device_name = google_compute_disk.boot_disk.name + } + + attached_disk { + source = google_compute_disk.attached_disk.id + device_name = "attached_disk" + } + + network_interface { + network = var.network_self_link + subnetwork = var.subnetwork_self_link + } + + service_account { + email = var.service_account == null ? data.google_compute_default_service_account.default.email : var.service_account + scopes = var.scopes + } + + metadata = var.metadata + metadata_startup_script = templatefile("${path.module}/scripts/install-nfs-server.sh.tpl", { local_mounts = var.local_mounts }) + + labels = local.labels +} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf new file mode 100644 index 0000000000..e23b94e2b2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +# render the content for each folder +output "network_storage" { + description = "export of all desired folder directories" + value = [for i, mount in var.local_mounts : { + remote_mount = "/exports${mount}" + local_mount = mount + fs_type = local.fs_type + mount_options = local.mount_options + server_ip = local.server_ip + client_install_runner = local.install_nfs_client_runners[i] + mount_runner = local.mount_runners[i] + } + ] +} + +output "install_nfs_client" { + description = "Script for installing NFS client" + value = file("${path.module}/scripts/install-nfs-client.sh") +} + +output "install_nfs_client_runner" { + description = "Runner to install NFS client using the startup-script module" + value = local.install_nfs_client_runners[0] +} + +output "mount_runner" { + description = <<-EOT + Runner to mount the file-system using an ansible playbook. The startup-script + module will automatically handle installation of ansible. + - id: example-startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(your-fs-id.mount_runner) + ... + EOT + value = local.ansible_mount_runner +} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl new file mode 100644 index 0000000000..1b06a5f032 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl @@ -0,0 +1,35 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -ex + +if [ ! -d "/exports" ]; then # first load, format and mount the disk + # See https://cloud.google.com/compute/docs/disks/add-persistent-disk + uuid=$(uuidgen) + mkfs.ext4 -F -m 0 -U "$uuid" -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/disk/by-id/google-attached_disk + + mkdir /exports + echo "UUID=$uuid /exports ext4 discard,defaults 0 0" >> /etc/fstab + mount --target /exports/ + + %{ for mount in local_mounts ~} + mkdir -p /exports${mount} + chmod 755 /exports${mount} + echo '/exports${mount} *(rw,sync,no_root_squash)' >> "/etc/exports" + %{ endfor ~} +fi + +systemctl start nfs-server rpcbind +systemctl enable nfs-server +exportfs -r diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml new file mode 100644 index 0000000000..f7fbe58d5e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml @@ -0,0 +1,39 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Mounts the file systems specified in the metadata network_storage key + hosts: localhost + become: true + vars: + meta_key: "network_storage" + url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" + tasks: + - name: Read metadata network_storage information + ansible.builtin.uri: + url: "{{ url }}/{{ meta_key }}" + method: GET + headers: + Metadata-Flavor: "Google" + register: storage + - name: Mount file systems + ansible.posix.mount: + src: "{{ item.server_ip }}:/{{ item.remote_mount }}" + path: "{{ item.local_mount }}" + opts: "{{ item.mount_options }}" + boot: true + fstype: "{{ item.fs_type }}" + state: "mounted" + loop: "{{ storage.json }}" diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf new file mode 100644 index 0000000000..9a58da641e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf @@ -0,0 +1,194 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the NFS instance if no name is specified." + type = string +} + +variable "name" { + description = "The resource name of the instance." + type = string + default = null +} + +variable "zone" { + description = "The zone name where the NFS instance located in." + type = string +} + +variable "boot_disk_size" { + description = "Storage size in GB for the boot disk" + type = number + default = null +} + +variable "boot_disk_type" { + description = "Storage type for the boot disk" + type = string + default = null +} + +variable "create_boot_snapshot_before_destroy" { + description = "Whether to create a snapshot before destroying the boot disk" + type = bool + default = false +} + +variable "disk_size" { + description = "Storage size in GB for the NFS data disk" + type = number + default = "100" +} + +variable "type" { + description = "Storage type for the NFS data disk" + type = string + default = "pd-ssd" +} + +variable "create_snapshot_before_destroy" { + description = "Whether to create a snapshot before destroying the NFS data disk" + type = bool + default = false +} + +variable "provisioned_iops" { + description = "Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput" + type = number + default = null +} + +variable "provisioned_throughput" { + description = "Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme" + type = number + default = null +} + +# Deprecated, replaced by instance_image +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: The VM image used by the NFS server" + type = string + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + The VM image used by the NFS server. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +# Deprecated, replaced by create_snapshot_before_destroy and create_boot_snapshot_before_destroy +# tflint-ignore: terraform_unused_declarations +variable "auto_delete_disk" { + description = "DEPRECATED: Whether or not the NFS disk should be auto-deleted" + type = string + default = null + + validation { + condition = var.auto_delete_disk == null + error_message = "The 'var.auto_delete_disk' setting is broken in Cluster Toolkit versions >1.25.0 and deprecated in versions >1.48.0, please use 'var.create_snapshot_before_destroy' and 'var.create_boot_snapshot_before_destroy' instead." + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the NFS VM." + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the NFS VM." + type = string + default = null +} + +variable "machine_type" { + description = "Type of the VM instance to use" + type = string + default = "n2d-standard-2" +} + +variable "labels" { + description = "Labels to add to the NFS instance. Key-value pairs." + type = map(string) +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "service_account" { + description = "Service Account for the NFS server" + type = string + default = null +} + +variable "scopes" { + description = "Scopes to apply to the controller" + type = list(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "local_mounts" { + description = "Mountpoint for this NFS compute instance" + type = list(string) + default = ["/data"] + + validation { + condition = alltrue([ + for m in var.local_mounts : substr(m, 0, 1) == "/" + ]) + error_message = "Local mountpoints have to start with '/'." + } + validation { + condition = length(var.local_mounts) > 0 + error_message = "At least one local mount must be specified in var.local_mounts." + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf new file mode 100644 index 0000000000..63443806b8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.14" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/sycomp-scale/README.md b/deletion-test/primary/modules/embedded/community/modules/file-system/sycomp-scale/README.md new file mode 100644 index 0000000000..79ff12bc18 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/sycomp-scale/README.md @@ -0,0 +1,35 @@ +## Description + +This document provides information on how to deploy an instance of [Sycomp Intelligent Data Storage Platform](https://sycomp.com/solution/hpc/storage/) on Google Cloud Platform ([GCP](https://cloud.google.com/)) using the Google Cluster Toolkit. + +> **_NOTE:_** +> Sycomp Storage on GCP does not require an HPC Toolkit wrapper. +> Terraform modules are sourced directly from GitLab. + +Terraform modules for Sycomp Intelligent Data Storage Platform are downloaded on deployment using the Google Cloud Toolkit. + +The Terraform module parameters are documented in the `README.md` files in the respective module directories of the source GitLab repository. The main modules are: + +- `sycomp-scale` +- `sycomp-scale-expansion` + +## Examples + +The community examples folder (community/examples/sycomp/) contains four example blueprints that you can use to deploy or expand a Sycomp Storage cluster. + +- [community/examples/sycomp/sycomp-storage.yaml][sycomp-storage-yaml] - + Blueprint for deploying a Sycomp Storage cluster consisting of 3 storage servers. + +- [community/examples/sycomp/sycomp-storage-expansion.yaml][sycomp-storage-expansion-yaml] - + Blueprint for expanding the above created cluster from 3 to 4 storage servers. + +- [community/examples/sycomp/sycomp-storage-ece.yaml][sycomp-storage-ece-yaml] - + Blueprint for deploying a Sycomp Storage cluster consisting of 7 storage servers with ECE (Erasure Code Edition) software RAID. + +- [community/examples/sycomp/sycomp-storage-slurm.yaml][sycomp-storage-slurm-yaml] - + Blueprint for deploying a Slurm cluster and Sycomp Storage cluster with 3 servers. The Slurm compute nodes are configured as NFS clients and have the ability to use the Sycomp Storage filesystem. + +[sycomp-storage-yaml]: ../../../examples/sycomp/sycomp-storage.yaml +[sycomp-storage-expansion-yaml]: ../../../examples/sycomp/sycomp-storage-expansion.yaml +[sycomp-storage-ece-yaml]: ../../../examples/sycomp/sycomp-storage-ece.yaml +[sycomp-storage-slurm-yaml]: ../../../examples/sycomp/sycomp-storage-slurm.yaml diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/README.md b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/README.md new file mode 100644 index 0000000000..0e2a936167 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/README.md @@ -0,0 +1,182 @@ +## Description + +This module provides scripts for client installation and mounting [WEKA] +filesystems. Client supports both UDP and DPDK modes and allows customization of +mount parameters using Compute VM instance metadata. + +For deploying Weka cluster please consult [WEKA installation on GCP]. + +[WEKA]: https://www.weka.io/ +[WEKA installation on GCP]: https://docs.weka.io/planning-and-installation/weka-installation-on-gcp + +## Prerequisites + +* up and running Weka cluster +* running on a [supported OS](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#operating-system) +* [open firewall](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#required-ports) + between WEKA backend servers and clients +* VPC peering configuration: + * if clients share VPCs created for WEKA cluster, no additional configuration + is necessary + * if dedicated VPCs are in use for clients, then WEKA VPCs needs to be peered + with VPCs that are used as: + * primary interface on client + * interfaces dedicated for DPDK client + * if dedicated VPCs are in use for clients, then those VPCs needs to be peered + with each other + +## Mounting +This example creates mount scripts that will mount `default` filesystem from +`10.0.0.3` WEKA backend: + +```yaml + - id: wekafs + source: community/modules/file-system/weka-client + settings: + local_mount: /scratch + server_ip: 10.0.0.3 + remote_mount: default + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: $(wekafs.runners) +``` + +If you need to add mount script along other runners, remember to add all 4 +runners provided by this script as shown in this example: + +```yaml + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(wekafs.client_install_runner) + - $(wekafs.mount_runner) + - type: shell + content: | + #!/bin/bash + + echo Sample + destination: sample-script.sh +``` + +To use the client within Slurm partition, with DPDK, remember to set additional +networks, and configure metadata. In this example, all four additional interfaces +are dedicated to WEKA DPDK + +```yaml + - id: c2_60_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + - mount-at-startup # as defined in previous examples + settings: + bandwidth_tier: virtio_enabled # Weka requires VirtIO, from WEKA 4.4.1, DPDK is also supported on gVNIC + additional_networks: + - subnetwork: weka-client-1 + nic_type: VIRTIO_NET + - subnetwork: weka-client-2 + nic_type: VIRTIO_NET + - subnetwork: weka-client-3 + nic_type: VIRTIO_NET + - subnetwork: weka-client-4 + nic_type: VIRTIO_NET + machine_type: c2-standard-60 + metadata: + weka-data_interfaces: 1,2,3,4 # allocate interfaces 1, 2, 3 and 4 to DPDK + weka-mode: dpdk + weka-options: num_cores=4,dpdk_base_memory_mb=16 + node_conf: + # From https://docs.weka.io/planning-and-installation/bare-metal/planning-a-weka-system-installation + # do not set RealMem as this is set automatically by Cluster Toolkit + CoreSpecCount: 4 + MemSpecLimit: 5120 +``` + +Due to the fact, that client installation takes ~6-7 minutes, if you use WEKA together with Slurm and do not bundle +client in the instance image, you may need to increase the timeout for startups scripts. + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + settings: + compute_startup_scripts_timeout: 600 + login_startup_scripts_timeout: 600 + ... + - id: compute_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + settings: + resume_timeout: 600 + ... +``` + +## Supported VM metadata options +Client scripts do support following metadata keys: +* `weka-mode` - one of `udp` or `dpdk`. Defaults to `udp`. Sets client mode. +* `weka-data_interfaces` - comma separated list of interface identifiers, + specifying which interfaces are dedicated for data plane. Set to `1` to + dedicate second interface of instance for WEKA DPDK. Set to `2,5` to dedicate + third and sixth interface of instance for WEKA DPDK. +* `weka-mgmt_interface` - identifier of management interface, defaults to `0`, + which means to use primary interface as management interface. +* `weka-options` - additional [mount command options](https://docs.weka.io/weka-filesystems-and-object-stores/mounting-filesystems#mount-command-options) + to pass to `mount` command + +## Adding client to the OS image +To save time during the mount command install and precompile DPDK drivers in the +OS image. Following scripts compiles DPDK driver for currently running kernel. + +```shell +#!/bin/bash + +set -e -o pipefail + +echo Downloading and installing Weka client +curl --max-time 10 "{{ weka backend endpoint }}/dist/v1/install" | sh +WEKA_VERSION=$(weka -v | sed -e 's/^[^0-9]*//') +echo Installing Weka version: ${WEKA_VERSION} +weka version get "${WEKA_VERSION}" +weka version set "${WEKA_VERSION}" +# run setup for the second time, if it fails for the first time +weka local setup weka || weka local setup weka +weka version prepare "${WEKA_VERSION}" +weka local stop +weka local rm -f --all +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | +| [mount\_options](#input\_mount\_options) | Mount options for filesystem shared by all clients. | `string` | `""` | no | +| [remote\_mount](#input\_remote\_mount) | Weka filesystem name. | `string` | n/a | yes | +| [server\_ip](#input\_server\_ip) | Weka backend IP address used for bootstrapping. | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_install\_runner](#output\_client\_install\_runner) | Ansible runner that performs client installation needed to use file system. | +| [mount\_runner](#output\_mount\_runner) | Ansible runner that mounts the file system. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/metadata.yaml new file mode 100644 index 0000000000..419bc3fe46 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/outputs.tf new file mode 100644 index 0000000000..0bd9098d80 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/outputs.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + template_args = { + local_mount = var.local_mount + mount_options = var.mount_options == "" ? "" : "-o ${var.mount_options}" + remote_mount = var.remote_mount + server_ip = var.server_ip + service_name = "weka-mount${replace(var.local_mount, "/", "-")}" + } + mount_script = templatefile("${path.module}/templates/mount-weka.sh.tftpl", local.template_args) + + mount_runner_ansible = { + type = "ansible-local" + content = templatefile( + "${path.module}/templates/mount-weka.yaml.tftpl", + merge( + local.template_args, + { mount_weka_script = local.mount_script } + ) + ) + destination = "mount_filesystem${replace(var.local_mount, "/", "_")}.yaml" + } + + client_install_runner = { + type = "ansible-local" + content = templatefile("${path.module}/templates/install-weka-client.yaml.tftpl", local.template_args) + destination = "install_filesystem${replace(var.local_mount, "/", "_")}.yaml" + } +} + +# currently WEKA mounts are not compatible with network_storage logic, as WEKA volumes needs to be mounted by +# systemd script and not /etc/fstab entry, as the mount command needs to have network configuration which may change +# between restarts +# +#output "network_storage" { +# description = "Describes a remote network storage to be mounted by fs-tab." +# value = { +# server_ip = var.server_ip +# remote_mount = var.remote_mount +# local_mount = var.local_mount +# fs_type = var.fs_type +# mount_options = var.mount_options +# client_install_runner = local.client_install_runner +# mount_runner = local.mount_runner +# } +#} +# +output "client_install_runner" { + description = "Ansible runner that performs client installation needed to use file system." + value = local.client_install_runner +} + +output "mount_runner" { + description = "Ansible runner that mounts the file system." + value = local.mount_runner_ansible +} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl new file mode 100644 index 0000000000..ddc3acdb5d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl @@ -0,0 +1,133 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Mounts the file systems specified in the metadata network_storage key + hosts: localhost + become: true + vars: + meta_key: "network_storage" + url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" + tasks: + - name: Check if weka is installed + ansible.builtin.stat: + path: /usr/bin/weka + register: weka_binary + + - name: Create temporary location for installation script + ansible.builtin.tempfile: + state: file + register: + install_script + when: not weka_binary.stat.exists + + - name: Download WEKA client + ansible.builtin.get_url: + url: http://${server_ip}:14000/dist/v1/install + dest: "{{ install_script.path }}" + mode: "700" + when: not weka_binary.stat.exists + + - name: Run WEKA installation script + ansible.builtin.shell: + cmd: "{{ install_script.path }}" + when: not weka_binary.stat.exists + register: weka_install_result + changed_when: weka_install_result.rc == 0 + + - name: Read metadata network_storage information + ansible.builtin.uri: + url: "{{ url }}/weka-version" + method: GET + headers: + Metadata-Flavor: "Google" + status_code: + - 200 + - 404 + register: get_weka_version + + - name: Set WEKA version from metadata server + ansible.builtin.set_fact: + weka_version: "{{ get_weka_version.body }}" + when: get_weka_version.status == 200 + + - name: Get version of WEKA installation client + ansible.builtin.shell: + cmd: weka -v | sed -e 's/^[^0-9.]*\([0-9.]*\)[^0-9.]*$/\1/' + register: get_weka_client_version + changed_when: get_weka_client_version.rc == 0 + + - name: Set WEKA version from WEKA installation client + ansible.builtin.set_fact: + weka_version: "{{ get_weka_client_version.stdout }}" + when: get_weka_version.status == 404 + + - name: Download user-defined WEKA version + ansible.builtin.shell: + cmd: weka version get {{ weka_version }} + register: result + changed_when: result.rc == 0 + + - name: Set user-defined WEKA version + ansible.builtin.shell: + cmd: weka version set {{ weka_version }} + register: result + changed_when: result.rc == 0 + + - name: Setup WEKA client + ansible.builtin.shell: + cmd: weka local setup weka + register: setup_1_result + changed_when: setup_1_result.rc == 0 + failed_when: false # ignore errors + + - name: Setup WEKA client (2nd try) + ansible.builtin.shell: + cmd: weka local setup weka + register: result + changed_when: result.rc == 0 + when: setup_1_result.rc != 0 + + - name: Prepare WEKA version + ansible.builtin.shell: + cmd: weka version prepare {{ weka_version }} + register: result + changed_when: result.rc == 0 + + - name: Stop WEKA client + ansible.builtin.shell: + cmd: weka local stop + async: 30 + poll: 10 + register: weka_stop + changed_when: weka_stop.get("rc") == 0 # when killed by async, rc is not defined + failed_when: false # ignore errors + + - name: Stop WEKA client (2nd try) + ansible.builtin.shell: + cmd: weka local stop + async: 30 + poll: 10 + register: result + changed_when: result.rc == 0 + failed_when: false # ignore errors + when: weka_stop.get("rc") != 0 + + - name: Remove WEKA containers + ansible.builtin.shell: + cmd: weka local rm -f --all + register: result + changed_when: result.rc == 0 + failed_when: false # ignore errors diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl new file mode 100644 index 0000000000..19c6dc1fdc --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl @@ -0,0 +1,101 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# shellcheck disable=SC2034 +METADATA_BASE_URL="http://metadata.google.internal/computeMetadata/v1/instance" +# shellcheck disable=SC2034 +ATTR_URL="$${METADATA_BASE_URL}/attributes/weka-" +NET_URL="$${METADATA_BASE_URL}/network-interfaces" + +# shellcheck disable=SC1083 +WEKA_MODE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mode || echo -n udp) +# shellcheck disable=SC1083 +WEKA_DATA_INTERFACES=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}data_interfaces || exit 0) +# shellcheck disable=SC1083 +WEKA_MGMT_INTERFACE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mgmt_interface || echo -n 0) +# shellcheck disable=SC1083 +WEKA_OPTIONS=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}options || exit 0) + +WEKA_OPTIONS="$${WEKA_OPTIONS:+-o $WEKA_OPTIONS}" + +netmask_to_cidr () { + c=0 + # shellcheck disable=SC2086,SC1083 + x=0$( printf '%o' $${1//./ } ) + while [ "$x" -gt 0 ]; do + c=$(( c + x%2 )) + x=$(( x >> 1)) + done + echo $c ; +} + +# detect network interface naming scheme +if [[ -e /sys/class/net/eth0 ]] ; then + DEVICE_NAME="eth" + DEVICE_INDEX_BASE=0 +elif [[ -e /sys/class/net/ens4 ]] ; then + DEVICE_NAME="ens" + DEVICE_INDEX_BASE=4 +else + echo "Can't detect device names. Both /sys/class/net/eth0 and /sys/class/net/ens4 do not exists" + exit 1 +fi + +# ensure that /etc/hosts contains entry for hostname pointing to primary interface +NEW_IP=$(ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) | head -n 1 | sed -e 's/^.*inet \([0-9\.]\+\)\/.*$/\1/') +if [ -n "$NEW_IP" ] ; then + HOSTNAME=$(hostname) + sed -i -e "/$HOSTNAME/s/^[0-9\.]\+ $HOSTNAME/$NEW_IP $HOSTNAME/" /etc/hosts +else + echo "Failed to find primary interface address" + ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) + exit 1 +fi + +# shellcheck disable=SC2154 +echo "Mounting Weka ${server_ip}/${remote_mount} to ${local_mount}" +mkdir -p "${local_mount}" +service weka-agent start +if [[ $WEKA_MODE == "udp" ]] ; then + # shellcheck disable=SC2086,SC2154,SC2086 + mount -t wekafs ${mount_options} -o net=udp $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" + +elif [[ $WEKA_MODE == "dpdk" ]] ; then + declare -a DATA_INTERFACES + # split WEKA_DATA_INTERFACES by comma into array + # shellcheck disable=SC2034 + IFS=',' read -r -a DATA_INTERFACES <<< "$WEKA_DATA_INTERFACES" + + DATA_OPTIONS="" + # shellcheck disable=SC2066 + for interface in "$${DATA_INTERFACES[@]}" ; do + INTERFACE_IP=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/ip") + INTERFACE_MASK=$(netmask_to_cidr "$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/subnetmask")") + INTERFACE_GATEWAY=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/gateway") + + DATA_OPTIONS+="-o net=$DEVICE_NAME$((DEVICE_INDEX_BASE + interface))/$INTERFACE_IP/$INTERFACE_MASK/$INTERFACE_GATEWAY " + done + + # shellcheck disable=SC2086 + mount -t wekafs \ + ${mount_options} \ + -o mgmt_ip="$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$WEKA_MGMT_INTERFACE/ip")" \ + $DATA_OPTIONS $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" +else + echo "Unknown weka:mode metadata value: $${WEKA_MODE}. Allowed values: udp and dpdk" + exit 1 +fi diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl new file mode 100644 index 0000000000..84587103a9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl @@ -0,0 +1,54 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Mount the WEKA file systems + hosts: localhost + become: true + vars: + local_mount: "${local_mount}" + remote_mount: "${remote_mount}" + server_ip: "${server_ip}" + service_name: "weka-mount-${replace(local_mount, "/", "_")}" + tasks: + - name: Create mount script + ansible.builtin.copy: + dest: "/etc/{{ service_name }}.sh" + mode: "0755" + content: | + ${indent(8, mount_weka_script)} + + - name: Create systemd service for weka mount + ansible.builtin.copy: + dest: "/etc/systemd/system/{{ service_name }}.service" + mode: "0644" + content: | + [Install] + WantedBy=multi-user.target + [Unit] + Description=Mount Weka {{ server_ip }}/{{ remote_mount }} at {{ local_mount }} + After=network-online.target + Wants=network-online.target + [Service] + RemainAfterExit=true + Type=oneshot + ExecStart=/bin/bash -c "/etc/{{ service_name }}.sh" + + - name: Enable and start weka mount service + ansible.builtin.systemd: + name: "{{ service_name }}" + daemon_reload: true + enabled: true + state: started diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/variables.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/variables.tf new file mode 100644 index 0000000000..f07961d64c --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/variables.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "local_mount" { + description = "The mount point where the contents of the device may be accessed after mounting." + type = string + default = "/mnt" +} + +variable "mount_options" { + description = "Mount options for filesystem shared by all clients." + type = string + default = "" + nullable = false +} + +variable "remote_mount" { + description = "Weka filesystem name." + type = string +} + +variable "server_ip" { + description = "Weka backend IP address used for bootstrapping." + type = string + default = "" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/versions.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/versions.tf new file mode 100644 index 0000000000..9e6af1fa7f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.14.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb new file mode 100644 index 0000000000..f13726f691 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "project_id = \"${project_id}\"\n", + "dataset_id = \"${dataset_id}\"\n", + "table_id = \"${table_id}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ONI1Xo0-KtAD", + "outputId": "fb9ca475-e4ec-4cd0-e0e6-14f409eefd7a" + }, + "outputs": [], + "source": [ + "from google.cloud import bigquery\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "\n", + "client = bigquery.Client(project=project_id)\n", + "\n", + "df = client.query(f'''\n", + "SELECT ticker, cast(price AS FLOAT64) AS price, CAST(OFFSET as INTEGER) AS offset, start_date, end_date, iteration\n", + "FROM `{project_id}.{dataset_id}.{table_id}`,\n", + "UNNEST(simulation_results) as NUMERIC with OFFSET\n", + "WHERE epoch_time IN\n", + " # Get the latest simulation runs for each Ticker Symbol\n", + "(SELECT MAX(epoch_time) FROM `{project_id}.{dataset_id}.{table_id}` GROUP BY ticker)\n", + "'''\n", + ").to_dataframe()\n", + "# Display the data\n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define a function to plot the data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def plot_ticker(t,df):\n", + "\n", + " dtf = df[(df.ticker==t) &(df.offset == 250)].price.describe(include=[np.float64], percentiles=[.05, .01, .001])\n", + " cellText = []\n", + " for v in dtf.values:\n", + " cellText.append([v])\n", + " \n", + " pltf = df[df.ticker==t].pivot(index='offset', columns='iteration', values='price')\n", + " \n", + " fig = plt.figure(figsize=(10,5))\n", + " ax1 = fig.add_subplot(122)\n", + " pltf.plot(legend=False, ax=ax1, xlabel='Time(days)', ylabel='US$', title=f\"{ df[(df.ticker == t) & (df.offset == 0) & (df.iteration == 4)]}\")\n", + " ax2 = fig.add_subplot(121)\n", + " font_size=10\n", + " bbox=[0, 0, .5, 1]\n", + " ax2.axis('off')\n", + " mpl_table = ax2.table(cellText = cellText, rowLabels=dtf.index.values, bbox=bbox)\n", + " mpl_table.auto_set_font_size(False)\n", + " mpl_table.set_fontsize(font_size)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 808 + }, + "id": "jvBmb_KceX7z", + "outputId": "42a3ba9f-b68f-4c7b-d928-0fedeed9216c" + }, + "outputs": [], + "source": [ + "ticker_list = df.ticker.unique()\n", + "for t in ticker_list:\n", + " plot_ticker(t,df)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md new file mode 100644 index 0000000000..e54893a1bb --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md @@ -0,0 +1,97 @@ +## Description + +Copy files to a target GCS bucket. + +Primarily used for FSI - MonteCarlo Tutorial **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: +../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage +This copies the module files to the specified GCS bucket. It is expected that +the bucket will be mounted on the target VM. + +Some of the files are templates, and `main.tf` translates the files with the +passed variable values. This way the user does not have to change things like +pointing to the correct bigquery table or adding in the project_id. + +```yaml + - id: fsi_tutorial_files + source: community/modules/files/fsi-montecarlo-on-batch + use: [bq-dataset, bq-table, fsi_bucket, pubsub_topic] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 3.83 | +| [http](#requirement\_http) | ~> 3.0 | +| [random](#requirement\_random) | ~> 3.0 | +| [template](#requirement\_template) | ~> 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [http](#provider\_http) | ~> 3.0 | +| [random](#provider\_random) | ~> 3.0 | +| [template](#provider\_template) | ~> 2.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.get_iteration_sh](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.get_mc_reqs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.get_requirements](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.ipynb_obj_fsi](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.mc_obj_yaml](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.mc_run](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.run_batch_py](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [http_http.batch_py](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | +| [http_http.batch_requirements](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | +| [template_file.ipynb_fsi](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | +| [template_file.mc_run_py](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | +| [template_file.mc_run_yaml](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Bigquery dataset id | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to run project | `string` | n/a | yes | +| [table\_id](#input\_table\_id) | Bigquery table id | `string` | n/a | yes | +| [topic\_id](#input\_topic\_id) | Pubsub Topic Name | `string` | n/a | yes | +| [topic\_schema](#input\_topic\_schema) | Pubsub Topic schema | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh new file mode 100644 index 0000000000..50aa865a31 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ticker=("GOOG" "AMZN" "MSFT" "NVDA" "META" "TSLA" "PEP" "COST") +echo "BI: $BATCH_TASK_INDEX" +echo "TI: ${ticker[$BATCH_TASK_INDEX]}" +python3 -m pip install -r /mnt/disks/fsi/mc_run_reqs.txt +python3 /mnt/disks/fsi/mc_run.py \ + --ticker "${ticker[$BATCH_TASK_INDEX]}" \ + --iterations 500 \ + --start_date 2022-01-01 diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf new file mode 100644 index 0000000000..83dc7fe9cf --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf @@ -0,0 +1,102 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + bucket = replace(var.gcs_bucket_path, "gs://", "") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +data "template_file" "mc_run_py" { + template = file("${path.module}/mc_run.tpl.py") + vars = { + project_id = var.project_id + topic_id = var.topic_id + topic_schema = var.topic_schema + dataset_id = var.dataset_id + table_id = var.table_id + } +} + +resource "google_storage_bucket_object" "mc_run" { + name = "mc_run.py" + content = data.template_file.mc_run_py.rendered + bucket = local.bucket +} + +data "template_file" "mc_run_yaml" { + template = file("${path.module}/mc_run.tpl.yaml") + vars = { + project_id = var.project_id + bucket_name = local.bucket + region = var.region + } +} + +resource "google_storage_bucket_object" "mc_obj_yaml" { + name = "mc_run.yaml" + content = data.template_file.mc_run_yaml.rendered + bucket = local.bucket +} + +data "template_file" "ipynb_fsi" { + template = file("${path.module}/FSI_MonteCarlo.ipynb") + vars = { + project_id = var.project_id + dataset_id = var.dataset_id + table_id = var.table_id + } +} +resource "google_storage_bucket_object" "ipynb_obj_fsi" { + name = "FSI_MonteCarlo.ipynb" + content = data.template_file.ipynb_fsi.rendered + bucket = local.bucket +} + +data "http" "batch_py" { + url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/batch.py" +} + +resource "google_storage_bucket_object" "run_batch_py" { + name = "batch.py" + content = data.http.batch_py.response_body + bucket = local.bucket +} + +data "http" "batch_requirements" { + url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/requirements.txt" +} + +resource "google_storage_bucket_object" "get_requirements" { + name = "requirements.txt" + content = data.http.batch_requirements.response_body + bucket = local.bucket +} + +resource "google_storage_bucket_object" "get_iteration_sh" { + name = "iteration.sh" + content = file("${path.module}/iteration.sh") + bucket = local.bucket +} + +resource "google_storage_bucket_object" "get_mc_reqs" { + name = "mc_run_reqs.txt" + content = file("${path.module}/mc_run_reqs.txt") + bucket = local.bucket +} diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py new file mode 100644 index 0000000000..4e0a64e363 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Run MC simulation for VaR portfolio risk +""" + +import avro.schema +import io +import google.auth +import numpy +import time +import yfinance as yf + +from absl import app +from absl import flags +from avro.io import DatumWriter, BinaryEncoder, BinaryDecoder, DatumReader +from datetime import datetime +from datetime import timedelta +from google.cloud import pubsub_v1, bigquery +from google.cloud.pubsub import SchemaServiceClient + +PROJECT_ID = '${project_id}' +INCOMING_TOPIC_ID = '${topic_id}' +INCOMING_TOPIC_SCHEMA = '${topic_schema}' +DATASET_ID = '${dataset_id}' +TABLE_ID = '${table_id}' + + +FLAGS = flags.FLAGS + +flags.DEFINE_string("ticker", 'GOOG', "Nasdaq Stock Ticker to run, default GOOG") +flags.DEFINE_string("start_date", '2022-01-01' , "Start data for data query, default 2022-01-01") +flags.DEFINE_integer("calendar_days", 365 , "How many calendar days to include in the calculation") +flags.DEFINE_integer("epoch_time", f'{int(time.time())}' , "Epoch time, number of seconds since January 1st, 1970 at 00:00:00 UTC.") +flags.DEFINE_integer("iterations", 100 , "Number of iterations to run.") +flags.DEFINE_boolean("print_raw", False, "Dump raw data.") + +class VaRSimulator: + + def __init__(self): + pass + + def get_data(self): + self.get_historical_data_yahoo() + + def get_historical_data_yahoo(self): + + # get historical market data: https://pypi.org/project/yfinance/ + + self.raw_data = yf.Ticker(self.ticker).history(start=self.start_date, end=self.end_date ) + self.data = self.raw_data.Close + + def print_raw(self): + print(self.get_stats()) + print(type(self.raw_data)) + print(self.raw_data) + + def get_stats(self): + close = self.data + self.first = close[0] + self.last = close[-1] + self.trading_days = len(close) + self.cagr = (self.last / self.first) ** (365.0/self.calendar_days) -1.0 + self.volatility = self.data.pct_change().std() + return(self.first, self.last, self.trading_days, self.cagr, self.volatility) + + def run_simulation(self): + + returns = numpy.random.normal(self.cagr/self.trading_days, self.volatility, self.trading_days) + 1 + returns = numpy.insert(returns,0,1.0) + self.simulation_results = self.last * returns.cumprod() + return(self.simulation_results) + + def create_object(self): + self.object = { + "ticker": self.ticker, + "epoch_time": self.epoch_time, + "iteration": self.iteration, + "start_date": self.start_date, + "end_date": self.end_date, + "simulation_results": list(map(lambda x: {"price":x}, self.simulation_results)) + } + return(self.object) + + +class PubsubToBiquery: + + def __init__(self): + + the_time = int(time.time()) + + self.project_id = PROJECT_ID + + self.publisher_client = pubsub_v1.PublisherClient() + self.topic_path = self.publisher_client.topic_path(self.project_id, INCOMING_TOPIC_ID) + + self.schema_client = SchemaServiceClient() + self.schema_path = self.schema_client.schema_path(self.project_id, INCOMING_TOPIC_SCHEMA) + + pubsub_schema = self.schema_client.get_schema(request={"name": self.schema_path}) + avro_schema = avro.schema.parse(pubsub_schema.definition) + + self.writer = DatumWriter(avro_schema) + + + def publish_record(self,record): + + byte_stream = io.BytesIO() + encoder = BinaryEncoder(byte_stream) + self.writer.write(record, encoder) + data = byte_stream.getvalue() + byte_stream.flush() + future = self.publisher_client.publish(self.topic_path, data) + if(FLAGS.print_raw): + print(f"Published message ID: {future.result()}") + + +def main(argv): + + vr = VaRSimulator() + pbbq = PubsubToBiquery() + + vr.ticker =FLAGS.ticker + vr.start_date =FLAGS.start_date + vr.end_date =f'{(datetime.strptime(FLAGS.start_date,"%Y-%m-%d") + timedelta(days = FLAGS.calendar_days)).date()}' + vr.calendar_days = FLAGS.calendar_days + vr.epoch_time = FLAGS.epoch_time + vr.iteration = 1 + + vr.get_data() + vr.get_stats() + + for i in range(FLAGS.iterations): + vr.iteration = i + vr.run_simulation() + pbbq.publish_record(vr.create_object()) + + if(FLAGS.print_raw): + vr.print_raw() + + +if __name__ == "__main__": + """ This is executed when run from the command line """ + app.run(main) diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml new file mode 100644 index 0000000000..7f7de4840b --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +project_id: "${project_id}" +region: "${region}" + +job_prefix: 'fsi-' +machine_type: "n2-standard-2" +volumes: +- {bucket_name: "${bucket_name}", gcs_path: "/mnt/disks/fsi"} + +container: + image_uri: "python" + entry_point: "/bin/bash" + commands: ["/mnt/disks/fsi/iteration.sh", "$BATCH_TASK_INDEX"] + +task_count: 8 #optional +parallelism: 4 #optional +task_count_per_node: 2 #optional +cpu_milli: 1000 #optional +memory_mib: 102400 #optional + + +labels: + env: "monte" + type: "carlo" diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt new file mode 100644 index 0000000000..105ed70ad2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt @@ -0,0 +1,9 @@ +absl-py +avro +google-auth +google-cloud +google-cloud-batch +google-cloud-pubsub +google-cloud-bigquery +yfinance +PyYAML diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml new file mode 100644 index 0000000000..268c8faa9a --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [storage.googleapis.com] diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf new file mode 100644 index 0000000000..eddf3c9478 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which GCS bucket will be created." + type = string +} + +variable "gcs_bucket_path" { + description = "Bucket name" + type = string + default = null +} + +variable "topic_id" { + description = "Pubsub Topic Name" + type = string +} + +variable "topic_schema" { + description = "Pubsub Topic schema" + type = string +} + +variable "dataset_id" { + description = "Bigquery dataset id" + type = string +} + +variable "table_id" { + description = "Bigquery table id" + type = string +} + +variable "region" { + description = "Region to run project" + type = string +} diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf new file mode 100644 index 0000000000..86dcb4dc52 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + http = { + source = "hashicorp/http" + version = "~> 3.0" + } + template = { + source = "hashicorp/template" + version = "~> 2.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md new file mode 100644 index 0000000000..ae8462d763 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md @@ -0,0 +1,100 @@ +# Module: Slurm Instance + + + +- [Module: Slurm Instance](#module-slurm-instance) + - [Overview](#overview) + - [Module API](#module-api) + + + +## Overview + +This module creates a [compute instance](../../../../docs/glossary.md#vm) from +[instance template](../../../../docs/glossary.md#instance-template) for a +[Slurm cluster](../slurm_cluster/README.md). + +> **NOTE:** This module is only intended to be used by Slurm modules. For +> general usage, please consider using: +> +> - [terraform-google-modules/vm/google//modules/compute_instance](https://registry.terraform.io/modules/terraform-google-modules/vm/google/latest/submodules/compute_instance). +> **WARNING:** The source image is not modified. Make sure to use a compatible +> source image. + +## Module API + +For the terraform module API reference, please see +[README_TF.md](./README_TF.md). + + +Copyright (C) SchedMD LLC. +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.0 | +| [google](#requirement\_google) | >= 3.43 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.43 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_from_template.slurm_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | +| [null_resource.replace_trigger](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_instance_template.base](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | +| [hostname](#input\_hostname) | Hostname of instances | `string` | n/a | yes | +| [instance\_template](#input\_instance\_template) | Instance template self\_link used to create compute instances | `string` | n/a | yes | +| [network](#input\_network) | Network to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | +| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | +| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | +| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | +| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances | `list(string)` | `[]` | no | +| [subnetwork](#input\_subnetwork) | Subnet to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The project that subnetwork belongs to | `string` | `null` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [available\_zones](#output\_available\_zones) | List of available zones in region | +| [instances\_details](#output\_instances\_details) | List of all details for compute instances | +| [instances\_self\_links](#output\_instances\_self\_links) | List of self-links for compute instances | +| [names](#output\_names) | List of available zones in region | +| [slurm\_instances](#output\_slurm\_instances) | List of all resource objects for compute instances | + diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf new file mode 100644 index 0000000000..2af9008a0e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf @@ -0,0 +1,126 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########## +# LOCALS # +########## + +locals { + num_instances = length(var.static_ips) == 0 ? var.num_instances : length(var.static_ips) + + # local.static_ips is the same as var.static_ips with a dummy element appended + # at the end of the list to work around "list does not have any elements so cannot + # determine type" error when var.static_ips is empty + static_ips = concat(var.static_ips, ["NOT_AN_IP"]) + + network_interfaces = [for index in range(local.num_instances) : + concat([ + { + access_config = var.access_config + alias_ip_range = [] + ipv6_access_config = [] + network = var.network + network_ip = length(var.static_ips) == 0 ? "" : element(local.static_ips, index) + nic_type = null + queue_count = null + stack_type = null + subnetwork = var.subnetwork + subnetwork_project = var.subnetwork_project + } + ], + var.additional_networks + ) + ] +} + +################ +# DATA SOURCES # +################ + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +data "google_compute_instance_template" "base" { + project = var.project_id + name = var.instance_template +} + +############# +# INSTANCES # +############# +resource "null_resource" "replace_trigger" { + triggers = { + trigger = var.replace_trigger + } +} + +# TODO: `internal/slurm-gcp/login` is ONLY user of `internal/slurm-gcp/instance` +# Remove this module, add functionality (+ prune generality) to the login module directly. +resource "google_compute_instance_from_template" "slurm_instance" { + count = local.num_instances + name = format("%s-%s", var.hostname, format("%03d", count.index + 1)) + project = var.project_id + zone = var.zone == null ? data.google_compute_zones.available.names[count.index % length(data.google_compute_zones.available.names)] : var.zone + + allow_stopping_for_update = true + + dynamic "network_interface" { + for_each = local.network_interfaces[count.index] + iterator = nic + content { + dynamic "access_config" { + for_each = nic.value.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = nic.value.alias_ip_range + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + dynamic "ipv6_access_config" { + for_each = nic.value.ipv6_access_config + iterator = access_config + content { + network_tier = access_config.value.network_tier + } + } + network = nic.value.network + network_ip = nic.value.network_ip + nic_type = nic.value.nic_type + queue_count = nic.value.queue_count + subnetwork = nic.value.subnetwork + subnetwork_project = nic.value.subnetwork_project + } + } + + source_instance_template = data.google_compute_instance_template.base.self_link + # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 + # we have to explicitly override instance labels instead of inheriting them from template. + labels = data.google_compute_instance_template.base.labels + + + lifecycle { + replace_triggered_by = [null_resource.replace_trigger.id] + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf new file mode 100644 index 0000000000..4eba78a7e8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf @@ -0,0 +1,41 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "slurm_instances" { + description = "List of all resource objects for compute instances" + value = google_compute_instance_from_template.slurm_instance +} + +output "instances_self_links" { + description = "List of self-links for compute instances" + value = google_compute_instance_from_template.slurm_instance[*].self_link +} + +output "instances_details" { + description = "List of all details for compute instances" + value = google_compute_instance_from_template.slurm_instance[*] +} + +output "available_zones" { + description = "List of available zones in region" + value = data.google_compute_zones.available.names +} + +output "names" { + description = "List of available zones in region" + value = google_compute_instance_from_template.slurm_instance[*].name +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf new file mode 100644 index 0000000000..11111a2c05 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf @@ -0,0 +1,119 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The GCP project ID" + default = null +} + +variable "network" { + description = "Network to deploy to. Only one of network or subnetwork should be specified." + type = string + default = "" +} + +variable "subnetwork" { + description = "Subnet to deploy to. Only one of network or subnetwork should be specified." + type = string + default = "" +} + +variable "subnetwork_project" { + description = "The project that subnetwork belongs to" + type = string + default = null +} + +variable "hostname" { + description = "Hostname of instances" + type = string +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })) + nullable = false +} + +variable "static_ips" { + description = "List of static IPs for VM instances" + type = list(string) + default = [] +} + +variable "access_config" { + description = "Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet." + type = list(object({ + nat_ip = string + network_tier = string + })) + default = [] +} + +variable "num_instances" { + description = "Number of instances to create. This value is ignored if static_ips is provided." + type = number + default = 1 +} + +variable "instance_template" { + description = "Instance template self_link used to create compute instances" + type = string +} + +variable "region" { + description = "Region where the instances should be created." + type = string + default = null +} + +variable "zone" { + description = "Zone where the instances should be created. If not specified, instances will be spread across available zones in the region." + type = string + default = null +} + +######### +# SLURM # +######### + +variable "replace_trigger" { + description = "Trigger value to replace the instances." + type = string + default = "" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf new file mode 100644 index 0000000000..a3e84c09bf --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf @@ -0,0 +1,31 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 1.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.43" + } + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md new file mode 100644 index 0000000000..87394bef6a --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md @@ -0,0 +1,87 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.0 | +| [local](#requirement\_local) | ~> 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | ~> 2.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_template](#module\_instance\_template) | ../internal_instance_template | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.startup](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    disk_type                  = optional(string)
    disk_size_gb               = optional(number)
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `virtio_enabled` setting will only enable VirtioNet and will not enable TIER\_1.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard. | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See
https://cloud.google.com/compute/docs/gpus more details.
- type : the GPU type
- count : number of GPUs | 
object({
    type  = string
    count = number
  })
| `null` | no | +| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"n1-standard-1"` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Prefix for template resource. | `string` | `"default"` | no | +| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network
attribute for Legacy or Auto subnetted networks and subnetwork for custom
subnetted networks. | `string` | `null` | no | +| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | +| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the instances. See
'main.tf:local.service\_account' for the default. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming. | `string` | n/a | yes | +| [slurm\_instance\_role](#input\_slurm\_instance\_role) | Slurm instance type. Must be one of: controller; login; compute; or null. | `string` | n/a | yes | +| [source\_image](#input\_source\_image) | Source disk image. | `string` | `""` | no | +| [source\_image\_family](#input\_source\_image\_family) | Source image family. | `string` | `""` | no | +| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. If it is not provided, the provider project is used. | `string` | `""` | no | +| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must
exist in the same region this instance will be created in. Either network or
subnetwork must be provided. | `string` | `null` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [termination\_action](#input\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_template](#output\_instance\_template) | Instance template details | +| [labels](#output\_labels) | Labels attached to the instance template | +| [name](#output\_name) | Name of instance template | +| [self\_link](#output\_self\_link) | Self\_link of instance template | +| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | +| [tags](#output\_tags) | Tags that will be associated with instance(s) | + diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted new file mode 100644 index 0000000000..2edaa942d2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted @@ -0,0 +1,169 @@ +#!/bin/bash +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SLURM_DIR=/slurm +FLAGFILE=$SLURM_DIR/slurm_configured_do_not_remove +SCRIPTS_DIR=$SLURM_DIR/scripts +if [[ -z "$HOME" ]]; then + # google-startup-scripts.service lacks environment variables + HOME="$(getent passwd "$(whoami)" | cut -d: -f6)" +fi + +# Temporary workaround for transition period when some of older images +# don't have "baked in" python yet. +# TODO: Remove +SLURM_PY="/slurm/python/venv/bin/python3.13" +SYSTEM_PY="/usr/bin/python3" +if [[ ! -e "$SLURM_PY" ]]; then + echo "Symlink $SLURM_PY does not exist. Creating symlink to $SYSTEM_PY" + mkdir -p /slurm/python/venv/bin + ln -s "$SYSTEM_PY" "$SLURM_PY" +fi + +METADATA_SERVER="metadata.google.internal" +URL="http://$METADATA_SERVER/computeMetadata/v1" +CURL="curl -sS --fail --header Metadata-Flavor:Google" + +PING_METADATA="ping -q -w1 -c1 $METADATA_SERVER" +echo "INFO: $PING_METADATA" +for i in $(seq 10); do + [ $i -gt 1 ] && sleep 5; + $PING_METADATA > /dev/null && s=0 && break || s=$?; + echo "ERROR: Failed to contact metadata server, will retry" +done +if [ $s -ne 0 ]; then + echo "ERROR: Unable to contact metadata server, aborting" + wall -n '*** Slurm setup failed in the startup script! see `journalctl -u google-startup-scripts` ***' + exit 1 +else + echo "INFO: Successfully contacted metadata server" +fi + +PING_GOOGLE="ping -q -w1 -c1 8.8.8.8" +echo "INFO: $PING_GOOGLE" +for i in $(seq 5); do + [ $i -gt 1 ] && sleep 2; + $PING_GOOGLE > /dev/null && s=0 && break || s=$?; + echo "failed to ping Google DNS, will retry" +done +if [ $s -ne 0 ]; then + echo "WARNING: No internet access detected" +else + echo "INFO: Internet access detected" +fi + +mkdir -p $SCRIPTS_DIR +UNIVERSE_DOMAIN="$($CURL $URL/instance/attributes/universe_domain)" +BUCKET="$($CURL $URL/instance/attributes/slurm_bucket_path)" +if [[ -z $BUCKET ]]; then + echo "ERROR: No bucket path detected." + exit 1 +fi + +SCRIPTS_ZIP="$HOME/slurm-gcp-scripts.zip" +export CLOUDSDK_CORE_UNIVERSE_DOMAIN="$UNIVERSE_DOMAIN" + +INSTANCE_ROLE="$($CURL $URL/instance/attributes/slurm_instance_role)" + +if [ "$INSTANCE_ROLE" == "controller" ]; then + DEVEL_ZIP="slurm-gcp-devel-controller.zip" +else + DEVEL_ZIP="slurm-gcp-devel.zip" +fi +until gcloud storage cp "$BUCKET/$DEVEL_ZIP" "$SCRIPTS_ZIP"; do + echo "WARN: Could not download SlurmGCP scripts, retrying in 5 seconds." + # Remove marker used to determine if gcloud is being used in a GCE VM. + # This can get mistakenly set to False in some cases. + rm -f /root/.config/gcloud/gce + sleep 5 +done +unzip -o "$SCRIPTS_ZIP" -d "$SCRIPTS_DIR" +rm -rf "$SCRIPTS_ZIP" + +#temporary hack to not make the script fail on TPU vm +chown slurm:slurm -R "$SCRIPTS_DIR" || true +chmod 700 -R "$SCRIPTS_DIR" + + +if [ -f $FLAGFILE ]; then + echo "WARNING: Slurm was previously configured, quitting" + exit 0 +fi +touch $FLAGFILE + +function tpu_setup { + #allow the following command to fail, as this attribute does not exist for regular nodes + docker_image=$($CURL $URL/instance/attributes/slurm_docker_image 2> /dev/null || true) + if [ -z $docker_image ]; then #Not a tpu node, do not do anything + return + fi + if [ "$OS_ENV" == "slurm_container" ]; then #Already inside the slurm container, we should continue starting + return + fi + + #given a input_string like "WORKER_0:Joseph;WORKER_1:richard;WORKER_2:edward;WORKER_3:john" and a number 1, this function will print richard + parse_metadata() { + local number=$1 + local input_string=$2 + local word=$(echo "$input_string" | awk -v n="$number" -F ':|;' '{ for (i = 1; i <= NF; i+=2) if ($(i) == "WORKER_"n) print $(i+1) }') + echo "$word" + } + + input_string=$($CURL $URL/instance/attributes/slurm_names) + worker_id=$($CURL $URL/instance/attributes/tpu-env | awk '/WORKER_ID/ {print $2}' | tr -d \') + real_name=$(parse_metadata $worker_id $input_string) + + #Prepare to docker pull with gcloud + mkdir -p /root/.docker + cat << EOF > /root/.docker/config.json +{ + "credHelpers": { + "gcr.io": "gcloud", + "us-docker.pkg.dev": "gcloud" + } +} +EOF + #cgroup detection + CGV=1 + CGROUP_FLAGS="-v /sys/fs/cgroup:/sys/fs/cgroup:rw" + if [ -f /sys/fs/cgroup/cgroup.controllers ]; then #CGV2 + CGV=2 + fi + if [ $CGV == 2 ]; then + CGROUP_FLAGS="--cgroup-parent=docker.slice --cgroupns=private --tmpfs /run --tmpfs /run/lock --tmpfs /tmp" + if [ ! -f /etc/systemd/system/docker.slice ]; then #In case that there is no slice prepared for hosting the containers create it + printf "[Unit]\nDescription=docker slice\nBefore=slices.target\n[Slice]\nCPUAccounting=true\nMemoryAccounting=true" > /etc/systemd/system/docker.slice + systemctl start docker.slice + fi + fi + #for the moment always use --privileged, as systemd might not work properly otherwise + TPU_FLAGS="--privileged" + # TPU_FLAGS="--cap-add SYS_RESOURCE --device /dev/accel0 --device /dev/accel1 --device /dev/accel2 --device /dev/accel3" + # if [ $CGV == 2 ]; then #In case that we are in CGV2 for systemd to work correctly for the moment we go with privileged + # TPU_FLAGS="--privileged" + # fi + + docker run -d $CGROUP_FLAGS $TPU_FLAGS --net=host --name=slurmd --hostname=$real_name --entrypoint=/usr/bin/systemd --restart unless-stopped $docker_image + exit 0 +} + +tpu_setup #will do nothing for normal nodes or the container spawned inside TPU + +echo "INFO: Running python cluster setup script" +SETUP_SCRIPT_FILE=$SCRIPTS_DIR/setup.py +chmod +x $SETUP_SCRIPT_FILE +exec $SETUP_SCRIPT_FILE diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf new file mode 100644 index 0000000000..c91bbc4fd1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf @@ -0,0 +1,171 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "instance_validation" { + source = "../../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +########## +# LOCALS # +########## + +locals { + additional_disks = [ + for disk in var.additional_disks : { + disk_name = disk.disk_name + device_name = disk.device_name + auto_delete = disk.auto_delete + source = disk.source + boot = disk.boot + disk_size_gb = disk.disk_size_gb + disk_type = disk.disk_type + disk_labels = merge( + disk.disk_labels, + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) + disk_resource_manager_tags = disk.disk_resource_manager_tags + } + ] + + service_account = { + email = try(var.service_account.email, null) + scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) + } + + source_image_family = ( + var.source_image_family != "" && var.source_image_family != null + ? var.source_image_family + : "slurm-gcp-6-11-hpc-rocky-linux-8" + ) + source_image_project = ( + var.source_image_project != "" && var.source_image_project != null + ? var.source_image_project + : "projects/schedmd-slurm-public/global/images/family" + ) + + source_image = ( + var.source_image != null + ? var.source_image + : "" + ) + + + name_prefix = "${var.slurm_cluster_name}-${var.slurm_instance_role}-${var.name_prefix}" + + total_egress_bandwidth_tier = var.bandwidth_tier == "tier_1_enabled" ? "TIER_1" : "DEFAULT" + + nic_type_map = { + platform_default = null + virtio_enabled = "VIRTIO_NET" + gvnic_enabled = "GVNIC" + tier_1_enabled = "GVNIC" + } + nic_type = lookup(local.nic_type_map, var.bandwidth_tier, null) + + labels = merge(var.labels, + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) +} + +######## +# DATA # +######## + +data "local_file" "startup" { + filename = "${path.module}/files/startup_sh_unlinted" +} + +############ +# TEMPLATE # +############ + +module "instance_template" { + source = "../internal_instance_template" + + project_id = var.project_id + + # Network + can_ip_forward = var.can_ip_forward + network_ip = var.network_ip + network = var.network + nic_type = local.nic_type + region = var.region + subnetwork_project = var.subnetwork_project + subnetwork = var.subnetwork + tags = var.tags + total_egress_bandwidth_tier = local.total_egress_bandwidth_tier + additional_networks = var.additional_networks + access_config = var.access_config + + # Instance + machine_type = var.machine_type + min_cpu_platform = var.min_cpu_platform + name_prefix = local.name_prefix + gpu = var.gpu + service_account = local.service_account + shielded_instance_config = var.shielded_instance_config + advanced_machine_features = var.advanced_machine_features + enable_confidential_vm = var.enable_confidential_vm + enable_shielded_vm = var.enable_shielded_vm + preemptible = var.preemptible + spot = var.spot + on_host_maintenance = var.on_host_maintenance + labels = local.labels + instance_termination_action = var.termination_action + resource_manager_tags = var.resource_manager_tags + + # Metadata + startup_script = coalesce(var.internal_startup_script, data.local_file.startup.content) + metadata = merge( + var.metadata, + { + enable-oslogin = upper(var.enable_oslogin) + slurm_bucket_path = var.slurm_bucket_path + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) + + # Image + source_image_project = local.source_image_project + source_image_family = local.source_image_family + source_image = local.source_image + + # Disk + disk_type = var.disk_type + disk_size_gb = var.disk_size_gb + auto_delete = var.disk_auto_delete + disk_labels = merge( + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + var.disk_labels, + ) + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + + max_run_duration = var.max_run_duration + provisioning_model = var.provisioning_model + reservation_affinity = var.reservation_affinity +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf new file mode 100644 index 0000000000..65da41052e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "instance_template" { + description = "Instance template details" + value = module.instance_template +} + +output "self_link" { + description = "Self_link of instance template" + value = module.instance_template.self_link +} + +output "name" { + description = "Name of instance template" + value = module.instance_template.name +} + +output "tags" { + description = "Tags that will be associated with instance(s)" + value = module.instance_template.tags +} + +output "service_account" { + description = "Service account object, includes email and scopes." + value = module.instance_template.service_account +} + +output "labels" { + description = "Labels attached to the instance template" + value = local.labels +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf new file mode 100644 index 0000000000..35dd9c376f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf @@ -0,0 +1,431 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +########### +# GENERAL # +########### + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "on_host_maintenance" { + type = string + description = "Instance availability Policy" + default = "MIGRATE" +} + +variable "labels" { + type = map(string) + description = "Labels, provided as a map" + default = {} +} + +variable "enable_oslogin" { + type = bool + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >=0.13.0 | +| [google](#requirement\_google) | >= 3.88 | +| [google-beta](#requirement\_google-beta) | >= 6.13.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.88 | +| [google-beta](#provider\_google-beta) | >= 6.13.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance_template.tpl](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_template) | resource | +| [google_project.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    auto_delete                = bool
    boot                       = bool
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = map(string)
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | +| [alias\_ip\_range](#input\_alias\_ip\_range) | An array of alias IP ranges for this network interface. Can only be specified for network interfaces on subnet-mode networks.
ip\_cidr\_range: The IP CIDR range represented by this alias IP range. This IP CIDR range must belong to the specified subnetwork and cannot contain IP addresses reserved by system or used by other network interfaces. At the time of writing only a netmask (e.g. /24) may be supplied, with a CIDR format resulting in an API error.
subnetwork\_range\_name: The subnetwork secondary range name specifying the secondary range from which to allocate the IP CIDR range for this alias IP range. If left unspecified, the primary range of the subnetwork will be used. | 
object({
    ip_cidr_range         = string
    subnetwork_range_name = string
  })
| `null` | no | +| [auto\_delete](#input\_auto\_delete) | Whether or not the boot disk should be auto-deleted | `string` | `"true"` | no | +| [automatic\_restart](#input\_automatic\_restart) | (Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user). | `bool` | `true` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example | `string` | `"false"` | no | +| [disk\_encryption\_key](#input\_disk\_encryption\_key) | The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance | `string` | `null` | no | +| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `string` | `"100"` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Whether to enable the Confidential VM configuration on the instance. Note that the instance image must support Confidential VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Whether to enable the Shielded VM configuration on the instance. Note that the instance image must support Shielded VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | +| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See https://cloud.google.com/compute/docs/gpus more details | 
object({
    type  = string
    count = number
  })
| `null` | no | +| [instance\_termination\_action](#input\_instance\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | +| [ipv6\_access\_config](#input\_ipv6\_access\_config) | IPv6 access configurations. Currently a max of 1 IPv6 access configuration is supported. If not specified, the instance will have no external IPv6 Internet access. | 
list(object({
    network_tier = string
  }))
| `[]` | no | +| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create, e.g. n1-standard-1 | `string` | `"n1-standard-1"` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Name prefix for the instance template | `string` | n/a | yes | +| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks. | `string` | `""` | no | +| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | +| [nic\_type](#input\_nic\_type) | The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO\_NET. | `string` | `null` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | +| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | +| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#service_account. | 
object({
    email  = optional(string)
    scopes = set(string)
  })
| n/a | yes | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Not used unless enable\_shielded\_vm is true. Shielded VM configuration for the instance. | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [source\_image](#input\_source\_image) | Source disk image. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `""` | no | +| [source\_image\_family](#input\_source\_image\_family) | Source image family. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `"centos-7"` | no | +| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. The default project contains CentOS images. | `string` | `"centos-cloud"` | no | +| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | +| [stack\_type](#input\_stack\_type) | The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4\_ONLY. | `string` | `null` | no | +| [startup\_script](#input\_startup\_script) | User startup script to run when instances spin up | `string` | `""` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided. | `string` | `""` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [total\_egress\_bandwidth\_tier](#input\_total\_egress\_bandwidth\_tier) | Network bandwidth tier. Note: machine\_type must be a supported type. Values are 'TIER\_1' or 'DEFAULT'.
See https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration for details. | `string` | `"DEFAULT"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [name](#output\_name) | Name of instance template | +| [self\_link](#output\_self\_link) | Self-link of instance template | +| [service\_account](#output\_service\_account) | value | +| [tags](#output\_tags) | Tags that will be associated with instance(s) | + diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf new file mode 100644 index 0000000000..f8d2813ece --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf @@ -0,0 +1,234 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "instance_validation" { + source = "../../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +######### +# Locals +######### + +locals { + source_image = var.source_image != "" ? var.source_image : "centos-7-v20201112" + source_image_family = var.source_image_family != "" ? var.source_image_family : "centos-7" + source_image_project = var.source_image_project != "" ? var.source_image_project : "centos-cloud" + + boot_disk = [ + { + source_image = var.source_image != "" ? format("${local.source_image_project}/${local.source_image}") : format("${local.source_image_project}/${local.source_image_family}") + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_labels = var.disk_labels + auto_delete = var.auto_delete + disk_resource_manager_tags = var.disk_resource_manager_tags + boot = "true" + }, + ] + + all_disks = concat(local.boot_disk, var.additional_disks) + + # NOTE: Even if all the shielded_instance_config or confidential_instance_config + # values are false, if the config block exists and an unsupported image is chosen, + # the apply will fail so we use a single-value array with the default value to + # initialize the block only if it is enabled. + shielded_vm_configs = var.enable_shielded_vm ? [true] : [] + + gpu_enabled = var.gpu != null + alias_ip_range_enabled = var.alias_ip_range != null + preemptible = var.preemptible || var.spot + on_host_maintenance = ( + local.preemptible || var.enable_confidential_vm || local.gpu_enabled + ? "TERMINATE" + : var.on_host_maintenance + ) + automatic_restart = ( + # must be false when preemptible is true + local.preemptible ? false : var.automatic_restart + ) + + nic_type = var.total_egress_bandwidth_tier == "TIER_1" ? "GVNIC" : var.nic_type + + + provisioning_model = coalesce(var.provisioning_model, local.preemptible ? "SPOT" : "STANDARD") +} + +data "google_project" "this" { + project_id = var.project_id +} + +#################### +# Instance Template +#################### +resource "google_compute_instance_template" "tpl" { + provider = google-beta + name_prefix = "${var.name_prefix}-" + project = var.project_id + machine_type = var.machine_type + labels = var.labels + metadata = var.metadata + tags = var.tags + can_ip_forward = var.can_ip_forward + metadata_startup_script = var.startup_script + region = var.region + min_cpu_platform = var.min_cpu_platform + resource_manager_tags = var.resource_manager_tags + + service_account { + email = coalesce(var.service_account.email, "${data.google_project.this.number}-compute@developer.gserviceaccount.com") + scopes = lookup(var.service_account, "scopes", null) + } + + dynamic "disk" { + for_each = local.all_disks + content { + auto_delete = lookup(disk.value, "auto_delete", null) + boot = lookup(disk.value, "boot", null) + device_name = lookup(disk.value, "device_name", null) + disk_name = lookup(disk.value, "disk_name", null) + disk_size_gb = lookup(disk.value, "disk_size_gb", lookup(disk.value, "disk_type", null) == "local-ssd" ? "375" : null) + disk_type = lookup(disk.value, "disk_type", null) + interface = lookup(disk.value, "interface", lookup(disk.value, "disk_type", null) == "local-ssd" ? "NVME" : null) + mode = lookup(disk.value, "mode", null) + source = lookup(disk.value, "source", null) + source_image = lookup(disk.value, "source_image", null) + type = lookup(disk.value, "disk_type", null) == "local-ssd" ? "SCRATCH" : "PERSISTENT" + labels = (lookup(disk.value, "source", null) != null || lookup(disk.value, "disk_type", null) == "local-ssd") ? null : lookup(disk.value, "disk_labels", null) + resource_manager_tags = lookup(disk.value, "disk_resource_manager_tags", {}) + + dynamic "disk_encryption_key" { + for_each = compact([var.disk_encryption_key == null ? null : 1]) + content { + kms_key_self_link = var.disk_encryption_key + } + } + } + } + + network_interface { + network = var.network + subnetwork = var.subnetwork + subnetwork_project = var.subnetwork_project + network_ip = try(coalesce(var.network_ip), null) + nic_type = local.nic_type + stack_type = var.stack_type + dynamic "access_config" { + for_each = var.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = var.ipv6_access_config + content { + network_tier = ipv6_access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = local.alias_ip_range_enabled ? [var.alias_ip_range] : [] + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + } + + dynamic "network_interface" { + for_each = var.additional_networks + content { + network = network_interface.value.network + subnetwork = network_interface.value.subnetwork + subnetwork_project = network_interface.value.subnetwork_project + network_ip = try(coalesce(network_interface.value.network_ip), null) + nic_type = try(coalesce(network_interface.value.nic_type), null) + dynamic "access_config" { + for_each = network_interface.value.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = network_interface.value.ipv6_access_config + content { + network_tier = ipv6_access_config.value.network_tier + } + } + } + } + + network_performance_config { + total_egress_bandwidth_tier = coalesce(var.total_egress_bandwidth_tier, "DEFAULT") + } + + lifecycle { + create_before_destroy = "true" + } + + scheduling { + preemptible = local.preemptible + provisioning_model = local.provisioning_model + automatic_restart = local.automatic_restart + on_host_maintenance = local.on_host_maintenance + instance_termination_action = var.instance_termination_action + + dynamic "max_run_duration" { + for_each = var.max_run_duration != null ? [var.max_run_duration] : [] + content { + seconds = max_run_duration.value + } + } + } + + dynamic "reservation_affinity" { + for_each = var.reservation_affinity != null ? [var.reservation_affinity] : [] + content { + type = reservation_affinity.value.type + } + } + + advanced_machine_features { + enable_nested_virtualization = var.advanced_machine_features.enable_nested_virtualization + threads_per_core = var.advanced_machine_features.threads_per_core + turbo_mode = var.advanced_machine_features.turbo_mode + visible_core_count = var.advanced_machine_features.visible_core_count + performance_monitoring_unit = var.advanced_machine_features.performance_monitoring_unit + enable_uefi_networking = var.advanced_machine_features.enable_uefi_networking + } + + dynamic "shielded_instance_config" { + for_each = local.shielded_vm_configs + content { + enable_secure_boot = lookup(var.shielded_instance_config, "enable_secure_boot", shielded_instance_config.value) + enable_vtpm = lookup(var.shielded_instance_config, "enable_vtpm", shielded_instance_config.value) + enable_integrity_monitoring = lookup(var.shielded_instance_config, "enable_integrity_monitoring", shielded_instance_config.value) + } + } + + confidential_instance_config { + enable_confidential_compute = var.enable_confidential_vm + } + + dynamic "guest_accelerator" { + for_each = local.gpu_enabled ? [var.gpu] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf new file mode 100644 index 0000000000..69f8d3b98c --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "self_link" { + description = "Self-link of instance template" + value = google_compute_instance_template.tpl.self_link +} + +output "name" { + description = "Name of instance template" + value = google_compute_instance_template.tpl.name +} + +output "tags" { + description = "Tags that will be associated with instance(s)" + value = google_compute_instance_template.tpl.tags +} + +output "service_account" { + description = "value" + value = google_compute_instance_template.tpl.service_account[0] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf new file mode 100644 index 0000000000..c285c3fea5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf @@ -0,0 +1,398 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + type = string + description = "The GCP project ID" + default = null +} + +variable "name_prefix" { + description = "Name prefix for the instance template" + type = string +} + +variable "machine_type" { + description = "Machine type to create, e.g. n1-standard-1" + type = string + default = "n1-standard-1" +} + +variable "min_cpu_platform" { + description = "Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform" + type = string + default = null +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example" + type = string + default = "false" +} + +variable "tags" { + type = list(string) + description = "Network tags, provided as a list" + default = [] +} + +variable "labels" { + type = map(string) + description = "Labels, provided as a map" + default = {} +} + +variable "preemptible" { + type = bool + description = "Allow the instance to be preempted" + default = false +} + +variable "spot" { + description = <<-EOD + Provision as a SPOT preemptible instance. + See https://cloud.google.com/compute/docs/instances/spot for more details. + EOD + type = bool + default = false +} + +variable "instance_termination_action" { + description = <<-EOD + Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'. + See https://cloud.google.com/compute/docs/instances/spot for more details. + EOD + type = string + default = null +} + +variable "automatic_restart" { + type = bool + description = "(Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user)." + default = true +} + +variable "on_host_maintenance" { + type = string + description = "Instance availability Policy" + default = "MIGRATE" +} + +variable "region" { + type = string + description = "Region where the instance template should be created." + nullable = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +####### +# disk +####### +variable "source_image" { + description = "Source disk image. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." + type = string + default = "" +} + +variable "source_image_family" { + description = "Source image family. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." + type = string + default = "centos-7" +} + +variable "source_image_project" { + description = "Project where the source image comes from. The default project contains CentOS images." + type = string + default = "centos-cloud" +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = string + default = "100" +} + +variable "disk_type" { + description = "Boot disk type, can be either pd-ssd, local-ssd, or pd-standard" + type = string + default = "pd-standard" +} + +variable "disk_labels" { + description = "Labels to be assigned to boot disk, provided as a map" + type = map(string) + default = {} +} + +variable "disk_encryption_key" { + description = "The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance" + type = string + default = null +} + +variable "auto_delete" { + description = "Whether or not the boot disk should be auto-deleted" + type = string + default = "true" +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + description = "List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name" + type = list(object({ + source = optional(string) + disk_name = optional(string) + device_name = string + auto_delete = bool + boot = bool + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = map(string) + disk_resource_manager_tags = map(string) + })) + default = [] +} + +#################### +# network_interface +#################### +variable "network" { + description = "The name or self_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks." + type = string + default = "" +} + +variable "nic_type" { + description = "The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO_NET." + type = string + default = null +} + +variable "subnetwork" { + description = "The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided." + type = string + default = "" +} + +variable "subnetwork_project" { + description = "The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used." + type = string + default = null +} + +variable "network_ip" { + description = "Private IP address to assign to the instance if desired." + type = string + default = "" +} + +variable "stack_type" { + description = "The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4_ONLY." + type = string + default = null +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + })) +} + +variable "total_egress_bandwidth_tier" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 6.41 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.41 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance](#module\_instance) | ../instance | n/a | +| [template](#module\_template) | ../instance_template | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | +| [login\_nodes](#input\_login\_nodes) | Slurm login instance definitions. | 
object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  })
| n/a | yes | +| [network\_storage](#input\_network\_storage) | Storage to mounted on login instances
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path to directory in the bucket for configs | `string` | n/a | yes | +| [slurm\_bucket\_name](#input\_slurm\_bucket\_name) | Name of the bucket for configs | `string` | n/a | yes | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name | `string` | n/a | yes | +| [startup\_scripts](#input\_startup\_scripts) | List of scripts to be ran on login VMs startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | +| [startup\_scripts\_timeout](#input\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script. If any script exceeds this timeout,
then the instance setup process is considered failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instances](#output\_instances) | VM instances of login nodes | +| [service\_account](#output\_service\_account) | Service Account used by login VMs | + diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf new file mode 100644 index 0000000000..605461f7e6 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf @@ -0,0 +1,112 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "template" { + source = "../instance_template" + + project_id = var.project_id + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = "login" + slurm_bucket_path = var.slurm_bucket_path + name_prefix = local.name + + additional_disks = var.login_nodes.additional_disks + bandwidth_tier = var.login_nodes.bandwidth_tier + can_ip_forward = var.login_nodes.can_ip_forward + advanced_machine_features = var.login_nodes.advanced_machine_features + disk_auto_delete = var.login_nodes.disk_auto_delete + disk_labels = var.login_nodes.disk_labels + disk_resource_manager_tags = var.login_nodes.disk_resource_manager_tags + disk_size_gb = var.login_nodes.disk_size_gb + disk_type = var.login_nodes.disk_type + enable_confidential_vm = var.login_nodes.enable_confidential_vm + enable_oslogin = var.login_nodes.enable_oslogin + enable_shielded_vm = var.login_nodes.enable_shielded_vm + gpu = var.login_nodes.gpu + labels = var.login_nodes.labels + machine_type = var.login_nodes.machine_type + metadata = merge(var.login_nodes.metadata, { + "universe_domain" = var.universe_domain, + "slurm_login_group" = local.name + }) + min_cpu_platform = var.login_nodes.min_cpu_platform + on_host_maintenance = var.login_nodes.on_host_maintenance + preemptible = var.login_nodes.preemptible + region = var.login_nodes.region + resource_manager_tags = var.login_nodes.resource_manager_tags + service_account = var.login_nodes.service_account + shielded_instance_config = var.login_nodes.shielded_instance_config + source_image_family = var.login_nodes.source_image_family + source_image_project = var.login_nodes.source_image_project + source_image = var.login_nodes.source_image + spot = var.login_nodes.spot + subnetwork = var.login_nodes.subnetwork + tags = concat([var.slurm_cluster_name], var.login_nodes.tags) + termination_action = var.login_nodes.termination_action + + internal_startup_script = var.internal_startup_script +} + +module "instance" { + source = "../instance" + + access_config = var.login_nodes.access_config + hostname = "${var.slurm_cluster_name}-${local.name}" + + project_id = var.project_id + + instance_template = module.template.self_link + num_instances = var.login_nodes.num_instances + + additional_networks = var.login_nodes.additional_networks + region = var.login_nodes.region + static_ips = var.login_nodes.static_ips + subnetwork = var.login_nodes.subnetwork + zone = var.login_nodes.zone + + replace_trigger = var.replace_trigger +} + +resource "google_storage_bucket_object" "startup_scripts" { + for_each = { + for s in var.startup_scripts : format( + "slurm-login-%s-script-%s", local.name, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_") + ) => s.content + } + + bucket = var.slurm_bucket_name + name = "${var.slurm_bucket_dir}/${each.key}" + content = each.value + source_md5hash = md5(each.value) +} + +locals { + name = var.login_nodes.group_name # short hand + + config = { + group_name = local.name + startup_scripts_timeout = var.startup_scripts_timeout + network_storage = var.network_storage + } +} + +resource "google_storage_bucket_object" "config" { + bucket = var.slurm_bucket_name + name = "${var.slurm_bucket_dir}/login_group_configs/${local.name}.yaml" + content = yamlencode(local.config) + source_md5hash = md5(yamlencode(local.config)) + + # To ensure that login group "is not ready" until all startup scripts are written down + depends_on = [google_storage_bucket_object.startup_scripts] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf new file mode 100644 index 0000000000..04de18a188 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "service_account" { + value = module.template.service_account + description = "Service Account used by login VMs" +} + +output "instances" { + value = module.instance.slurm_instances + description = "VM instances of login nodes" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf new file mode 100644 index 0000000000..3efd862942 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf @@ -0,0 +1,188 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "slurm_cluster_name" { + type = string + description = "Cluster name" +} + +variable "slurm_bucket_path" { + type = string + description = "GCS Bucket URI of Slurm cluster file storage." +} + + +variable "slurm_bucket_name" { + type = string + description = "Name of the bucket for configs" +} + +variable "slurm_bucket_dir" { + type = string + description = "Path to directory in the bucket for configs" +} + + +variable "universe_domain" { + description = "Domain address for alternate API universe" + type = string + default = "googleapis.com" +} + +variable "login_nodes" { + description = "Slurm login instance definitions." + type = object({ + group_name = string + access_config = optional(list(object({ + nat_ip = string + network_tier = string + }))) + additional_disks = optional(list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string), {}) + auto_delete = optional(bool, true) + boot = optional(bool, false) + disk_resource_manager_tags = optional(map(string), {}) + })), []) + additional_networks = optional(list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })), []) + bandwidth_tier = optional(string, "platform_default") + can_ip_forward = optional(bool, false) + disk_auto_delete = optional(bool, true) + disk_labels = optional(map(string), {}) + disk_resource_manager_tags = optional(map(string), {}) + disk_size_gb = optional(number) + disk_type = optional(string, "n1-standard-1") + enable_confidential_vm = optional(bool, false) + enable_oslogin = optional(bool, true) + enable_shielded_vm = optional(bool, false) + gpu = optional(object({ + count = number + type = string + })) + labels = optional(map(string), {}) + machine_type = optional(string) + advanced_machine_features = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + metadata = optional(map(string), {}) + min_cpu_platform = optional(string) + num_instances = optional(number, 1) + on_host_maintenance = optional(string) + preemptible = optional(bool, false) + region = optional(string) + resource_manager_tags = optional(map(string), {}) + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + shielded_instance_config = optional(object({ + enable_integrity_monitoring = optional(bool, true) + enable_secure_boot = optional(bool, true) + enable_vtpm = optional(bool, true) + })) + source_image_family = optional(string) + source_image_project = optional(string) + source_image = optional(string) + static_ips = optional(list(string), []) + subnetwork = string + spot = optional(bool, false) + tags = optional(list(string), []) + zone = optional(string) + termination_action = optional(string) + }) +} + + +variable "startup_scripts" { + description = "List of scripts to be ran on login VMs startup." + type = list(object({ + filename = string + content = string + })) + default = [] +} + +variable "startup_scripts_timeout" { + description = < + +- [Module: Slurm Nodeset (TPU)](#module-slurm-nodeset-tpu) + - [Overview](#overview) + - [Module API](#module-api) + + + +## Overview + +This is a submodule of [slurm_cluster](../../../slurm_cluster/README.md). It +creates a Slurm TPU nodeset for [slurm_partition](../slurm_partition/README.md). + +## Module API + +For the terraform module API reference, please see +[README_TF.md](./README_TF.md). + + +Copyright (C) SchedMD LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.2 | +| [google](#requirement\_google) | >= 3.53 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.53 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.nodeset_tpu](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_subnetwork.nodeset_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | +| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | +| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `""` | no | +| [enable\_public\_ip](#input\_enable\_public\_ip) | Enables IP address to access the Internet. | `bool` | `false` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of nodes allowed in this partition to be created dynamically. | `number` | `0` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | +| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. Not needed if you use accelerator\_config | `string` | `null` | no | +| [nodeset\_name](#input\_nodeset\_name) | Name of Slurm nodeset. | `string` | n/a | yes | +| [preemptible](#input\_preemptible) | Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details. | `bool` | `false` | no | +| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `true` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the TPU-vm.
If none is given, the default service account and scopes will be used. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | +| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | n/a | yes | +| [zone](#input\_zone) | Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Nodeset details. | +| [nodeset\_name](#output\_nodeset\_name) | Nodeset name. | +| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf new file mode 100644 index 0000000000..1a6a9cfba1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf @@ -0,0 +1,121 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########### +# NODESET # +########### + +locals { + node_conf_hw = { + Mem334CPU96 = { + CPUs = 96 + Boards = 1 + Sockets = 2 + CoresPerSocket = 24 + ThreadsPerCore = 2 + RealMemory = 307200 + } + Mem400CPU240 = { + CPUs = 240 + Boards = 1 + Sockets = 2 + CoresPerSocket = 60 + ThreadsPerCore = 2 + RealMemory = 400000 + } + } + node_conf_mappings = { + "v2" = local.node_conf_hw.Mem334CPU96 + "v3" = local.node_conf_hw.Mem334CPU96 + "v4" = local.node_conf_hw.Mem400CPU240 + } + simple_nodes = ["v2-8", "v3-8", "v4-8"] +} + +locals { + snetwork = data.google_compute_subnetwork.nodeset_subnetwork.name + region = join("-", slice(split("-", var.zone), 0, 2)) + tpu_fam = var.accelerator_config.version != "" ? lower(var.accelerator_config.version) : split("-", var.node_type)[0] + #If subnetwork is specified and it does not have private_ip_google_access, we need to have public IPs on the TPU + #if no subnetwork is specified, the default one will be used, this does not have private_ip_google_access so we need public IPs too + pub_need = !data.google_compute_subnetwork.nodeset_subnetwork.private_ip_google_access + can_preempt = var.node_type != null ? contains(local.simple_nodes, var.node_type) : false + nodeset_tpu = { + nodeset_name = var.nodeset_name + node_conf = local.node_conf_mappings[local.tpu_fam] + node_type = var.node_type + accelerator_config = var.accelerator_config + tf_version = var.tf_version + preemptible = local.can_preempt ? var.preemptible : false + reserved = var.reserved + node_count_dynamic_max = var.node_count_dynamic_max + node_count_static = var.node_count_static + enable_public_ip = var.enable_public_ip + zone = var.zone + service_account = var.service_account != null ? var.service_account : local.service_account + preserve_tpu = local.can_preempt ? var.preserve_tpu : false + data_disks = var.data_disks + docker_image = var.docker_image != "" ? var.docker_image : "us-docker.pkg.dev/schedmd-slurm-public/tpu/slurm-gcp-6-9:tf-${var.tf_version}" + subnetwork = local.snetwork + network_storage = var.network_storage + } + + service_account = { + email = try(var.service_account.email, null) + scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) + } +} + +data "google_compute_subnetwork" "nodeset_subnetwork" { + name = var.subnetwork + region = local.region + project = var.project_id + + self_link = ( + length(regexall("/projects/([^/]*)", var.subnetwork)) > 0 + && length(regexall("/regions/([^/]*)", var.subnetwork)) > 0 + ? var.subnetwork + : null + ) +} + +resource "null_resource" "nodeset_tpu" { + triggers = { + nodeset = sha256(jsonencode(local.nodeset_tpu)) + } + lifecycle { + precondition { + condition = sum([var.node_count_dynamic_max, var.node_count_static]) > 0 + error_message = "Sum of node_count_dynamic_max and node_count_static must be > 0." + } + precondition { + condition = !(var.preemptible && var.reserved) + error_message = "Nodeset cannot be preemptible and reserved at the same time." + } + precondition { + condition = !(var.subnetwork == null && !var.enable_public_ip) + error_message = "Using the default subnetwork for the TPU nodeset requires enable_public_ip set to true." + } + precondition { + condition = !(var.subnetwork != null && (local.pub_need && !var.enable_public_ip)) + error_message = "The subnetwork specified does not have Private Google Access enabled. This is required when enable_public_ip is set to false." + } + precondition { + condition = !(var.node_type == null && (var.accelerator_config.topology == "" && var.accelerator_config.version == "")) + error_message = "Either a node type or an accelerator_config must be provided." + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf new file mode 100644 index 0000000000..fce700d567 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "nodeset_name" { + description = "Nodeset name." + value = local.nodeset_tpu.nodeset_name +} + +output "nodeset" { + description = "Nodeset details." + value = local.nodeset_tpu +} + +output "service_account" { + description = "Service account object, includes email and scopes." + value = local.service_account +} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf new file mode 100644 index 0000000000..a8c470dec9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf @@ -0,0 +1,158 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "nodeset_name" { + description = "Name of Slurm nodeset." + type = string + + validation { + condition = can(regex("^[a-z](?:[a-z0-9]{0,14})$", var.nodeset_name)) + error_message = "Variable 'nodeset_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,14})$'." + } +} + +variable "node_type" { + description = "Specify a node type to base the vm configuration upon it. Not needed if you use accelerator_config" + type = string + default = null +} + +variable "accelerator_config" { + description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." + type = object({ + topology = string + version = string + }) + default = { + topology = "" + version = "" + } + validation { + condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], upper(var.accelerator_config.version)) + error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" + } + validation { + condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) + error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." + } +} + +variable "docker_image" { + description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" + type = string + default = "" +} + +variable "tf_version" { + description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." + type = string +} + +variable "zone" { + description = "Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it." + type = string + + validation { + condition = can(coalesce(var.zone)) + error_message = "Zone cannot be null or empty." + } +} + +variable "preemptible" { + description = "Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details." + type = bool + default = false +} + +variable "reserved" { + description = "Specify whether TPU-vms in this nodeset are created under a reservation." + type = bool + default = false +} + +variable "preserve_tpu" { + description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" + type = bool + default = true +} + +variable "node_count_static" { + description = "Number of nodes to be statically created." + type = number + default = 0 + + validation { + condition = var.node_count_static >= 0 + error_message = "Value must be >= 0." + } +} + +variable "node_count_dynamic_max" { + description = "Maximum number of nodes allowed in this partition to be created dynamically." + type = number + default = 0 + + validation { + condition = var.node_count_dynamic_max >= 0 + error_message = "Value must be >= 0." + } +} + +variable "enable_public_ip" { + description = "Enables IP address to access the Internet." + type = bool + default = false +} + +variable "data_disks" { + type = list(string) + description = "The data disks to include in the TPU node" + default = [] +} + +variable "subnetwork" { + description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." + type = string +} + +variable "service_account" { + type = object({ + email = string + scopes = set(string) + }) + description = < +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | +| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | +| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | +| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | +| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | +| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "v0.7.2")
  })
| `{}` | no | +| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "v0.11.4")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | +| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md new file mode 100644 index 0000000000..1957899617 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md @@ -0,0 +1,64 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [helm](#provider\_helm) | ~> 2.17 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | +| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | +| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | +| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | +| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | +| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | +| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | +| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | +| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | +| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | +| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | +| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | +| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | +| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | +| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | +| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | +| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | +| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | +| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | +| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | +| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | +| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | +| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | +| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | +| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | +| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | +| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | +| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | +| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | +| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf new file mode 100644 index 0000000000..bd2383b772 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf @@ -0,0 +1,75 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "helm_release" "apply_chart" { + # Required Identification + name = var.release_name + chart = var.chart_name + + # Chart Source & Version + repository = var.chart_repository + version = var.chart_version + devel = var.devel + + # Target Namespace + namespace = var.namespace + create_namespace = var.create_namespace + + # Values Configuration + values = var.values_yaml + + dynamic "set" { + for_each = var.set_values + content { + name = set.value.name + value = set.value.value + type = set.value.type + } + } + + # Installation/Upgrade Behavior + description = var.description + atomic = var.atomic + cleanup_on_fail = var.cleanup_on_fail + dependency_update = var.dependency_update + disable_crd_hooks = var.disable_crd_hooks + disable_openapi_validation = var.disable_openapi_validation + disable_webhooks = var.disable_webhooks + force_update = var.force_update + lint = var.lint + max_history = var.max_history + recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI + render_subchart_notes = var.render_subchart_notes + reset_values = var.reset_values + reuse_values = var.reuse_values + skip_crds = var.skip_crds + timeout = var.timeout + wait = var.wait + wait_for_jobs = var.wait_for_jobs + + # Verification & Credentials + keyring = var.keyring + pass_credentials = var.pass_credentials + verify = var.verify + + # Post Rendering + dynamic "postrender" { + # Only include the block if var.postrender is not null + for_each = var.postrender == null ? [] : [var.postrender] + content { + binary_path = postrender.value.binary_path + } + } + +} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml new file mode 100644 index 0000000000..e18197e2b7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf new file mode 100644 index 0000000000..04e8e214fc --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf @@ -0,0 +1,212 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +# --- Required --- +variable "release_name" { + description = "Name of the Helm release." + type = string +} + +variable "chart_name" { + description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." + type = string +} + +# --- Chart Location & Version --- +variable "chart_repository" { + description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." + type = string + default = null +} + +variable "chart_version" { + description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." + type = string + default = null +} + +variable "devel" { + description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." + type = bool + default = false +} + +# --- Namespace --- +variable "namespace" { + description = "Kubernetes namespace to install the Helm release into." + type = string + default = "default" +} + +variable "create_namespace" { + description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." + type = bool + default = true # Common convenience setting +} + +# --- Values Customization --- +variable "values_yaml" { + description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." + type = list(string) + default = [] +} + +variable "set_values" { + description = "List of objects defining values to set ('helm install --set')." + type = list(object({ + name = string # Path to the value (e.g., 'service.type', 'replicaCount') + value = string # The value to set + type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') + })) + default = [] +} + +# --- Installation/Upgrade Behavior --- +variable "description" { + description = "Set an optional description for the Helm release." + type = string + default = null +} + +variable "atomic" { + description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." + type = bool + default = false +} + +variable "wait" { + description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." + type = bool + default = true # Often a good default for dependencies +} + +variable "wait_for_jobs" { + description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." + type = bool + default = false # Helm CLI default is false +} + +variable "timeout" { + description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." + type = number + default = 300 # 5 minutes (Helm CLI default) +} + +variable "cleanup_on_fail" { + description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." + type = bool + default = false +} + +variable "dependency_update" { + description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." + type = bool + default = false +} + +variable "disable_crd_hooks" { + description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." + type = bool + default = false +} + +variable "disable_openapi_validation" { + description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." + type = bool + default = false +} + +variable "disable_webhooks" { + description = "Prevent hooks from running ('helm install --no-hooks')." + type = bool + default = false +} + +variable "force_update" { + description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." + type = bool + default = false +} + +variable "lint" { + description = "Run the helm chart linter during the plan ('helm lint')." + type = bool + default = false +} + +variable "max_history" { + description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." + type = number + default = null # Terraform provider defaults to Helm's default (usually 10) +} + +variable "recreate_pods" { + description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." + type = bool + default = false +} + +variable "render_subchart_notes" { + description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." + type = bool + default = false +} + +variable "reset_values" { + description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." + type = bool + default = false +} + +variable "reuse_values" { + description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." + type = bool + default = false # Helm CLI default is false +} + +variable "skip_crds" { + description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." + type = bool + default = false +} + +# --- Verification & Credentials --- +variable "keyring" { + description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." + type = string + default = null # Defaults to Helm's default keyring location +} + +variable "pass_credentials" { + description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." + type = bool + default = false +} + +variable "verify" { + description = "Verify the package before installing it ('helm install --verify')." + type = bool + default = false +} + +# --- Advanced Rendering --- +variable "postrender" { + description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." + type = object({ + binary_path = string # Path to the post-renderer executable + }) + default = null # Disabled by default +} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf new file mode 100644 index 0000000000..09d912e2c9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md new file mode 100644 index 0000000000..46bfe51a32 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md @@ -0,0 +1,40 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.23 | + +## Providers + +| Name | Version | +|------|---------| +| [kubernetes](#provider\_kubernetes) | ~> 2.23 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubernetes_manifest.apply_manifests](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | +| [field\_manager](#input\_field\_manager) | (Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts. | 
object({
    name            = optional(string, null)
    force_conflicts = optional(bool, false)
  })
| `null` | no | +| [resource\_timeouts](#input\_resource\_timeouts) | (Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met. | 
object({
    create = optional(string, null)
    update = optional(string, null)
    delete = optional(string, null)
  })
| 
{
  "create": "15m",
  "delete": "5m",
  "update": "10m"
}
| no | +| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `""` | no | +| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | +| [wait\_for\_fields](#input\_wait\_for\_fields) | (Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern. | `map(string)` | `{}` | no | +| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf new file mode 100644 index 0000000000..f97f26038d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf @@ -0,0 +1,104 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + yaml_separator = "\n---" + + # --- 1. Determine the primary source of YAML content --- + # Prioritize 'content' variable if provided + primary_content_body = var.content != "" ? var.content : null + + # --- 2. Handle 'source_path' based on its type (File vs. Directory) --- + + # Check if source_path is a directory (indicated by trailing slash) + is_directory = endswith(var.source_path, "/") + directory_absolute_path = local.is_directory ? abspath(var.source_path) : null + + # Check if source_path is a single yaml or tftpl file (only if not a directory) + is_single_file = !local.is_directory && ( + length(regexall("\\.yaml$", lower(var.source_path))) > 0 || + length(regexall("\\.tftpl$", lower(var.source_path))) > 0 + ) + single_file_raw_content = local.is_single_file ? ( + length(regexall("\\.tftpl$", lower(var.source_path))) > 0 ? + templatefile(abspath(var.source_path), var.template_vars) : + file(abspath(var.source_path)) + ) : null + + # Docs from primary_content_body + docs_from_primary_source = [ + for doc in split(local.yaml_separator, coalesce(local.primary_content_body, local.single_file_raw_content, "")) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + + # Docs from .yaml files in a directory + directory_yaml_files = local.is_directory ? fileset(local.directory_absolute_path, "*.yaml") : [] + docs_from_directory_yamls = flatten([ + for file_name in local.directory_yaml_files : + [ + for doc in split(local.yaml_separator, file(format("%s/%s", local.directory_absolute_path, file_name))) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + ]) + + # Docs from .tftpl files in a directory + directory_template_files = local.is_directory ? fileset(local.directory_absolute_path, "*.tftpl") : [] + docs_from_directory_templates = flatten([ + for file_name in local.directory_template_files : + [ + for doc in split(local.yaml_separator, templatefile(format("%s/%s", local.directory_absolute_path, file_name), var.template_vars)) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + ]) + + all_parsed_docs = concat( + local.docs_from_primary_source, + local.docs_from_directory_yamls, + local.docs_from_directory_templates + ) + + # --- 5. Create the final map for `for_each` (keys must be unique strings) --- + docs_map = tomap({ + for index, doc in local.all_parsed_docs : index => doc + if length(trimspace(doc)) > 0 + }) +} + +# Apply all manifest files dynamically +resource "kubernetes_manifest" "apply_manifests" { + for_each = local.docs_map + manifest = yamldecode(each.value) + timeouts { + create = var.resource_timeouts.create + update = var.resource_timeouts.update + delete = var.resource_timeouts.delete + } + + dynamic "wait" { + for_each = var.wait_for_rollout ? [1] : [] + content { + rollout = var.wait_for_rollout + fields = var.wait_for_fields + } + } + + # Configure the 'field_manager' block dynamically + dynamic "field_manager" { + for_each = var.field_manager != null ? [var.field_manager] : [] + content { + name = field_manager.value.name + force_conflicts = field_manager.value.force_conflicts + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml new file mode 100644 index 0000000000..e18197e2b7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf new file mode 100644 index 0000000000..0b846189ea --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf @@ -0,0 +1,69 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +variable "content" { + description = "The YAML body to apply to gke cluster." + type = string + default = null +} + +variable "source_path" { + description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." + type = string + default = "" +} + +variable "template_vars" { + description = "The values to populate template file(s) with." + type = any + default = null +} + +variable "wait_for_rollout" { + description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." + type = bool + default = true +} + + +variable "wait_for_fields" { + description = "(Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern." + type = map(string) + default = {} +} + +variable "resource_timeouts" { + description = "(Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met." + type = object({ + create = optional(string, null) + update = optional(string, null) + delete = optional(string, null) + }) + default = { + create = "15m" # Default create timeout, also covers waiting for initial conditions + update = "10m" # Default update timeout, also covers waiting for update conditions + delete = "5m" # Default delete timeout + } +} + +variable "field_manager" { + description = "(Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts." + type = object({ + name = optional(string, null) + force_conflicts = optional(bool, false) + }) + default = null +} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf new file mode 100644 index 0000000000..61786b06de --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + # Defines the providers that this module depends on and their versions. + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.23" + } + } + required_version = ">= 1.3" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf new file mode 100644 index 0000000000..8db4870452 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf @@ -0,0 +1,183 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] + project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] + + install_gpu_operator = try(var.gpu_operator.install, false) + install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) +} + +data "google_container_cluster" "gke_cluster" { + project = local.project_id + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +module "install_kueue" { + source = "./helm_install" + depends_on = [var.gke_cluster_exists] + + release_name = "kueue" + + chart_name = "oci://registry.k8s.io/kueue/charts/kueue" + chart_version = var.kueue.version # Specify your desired Kueue version + + create_namespace = true # Helm can also create the namespace + wait = true + timeout = 600 # seconds +} + +module "install_jobset" { + source = "./helm_install" + depends_on = [var.gke_cluster_exists, module.install_kueue] + release_name = "jobset-controller" # The release name for your JobSet installation + chart_name = "oci://registry.k8s.io/jobset/charts/jobset" # The Helm repository URL for nvidia charts + chart_version = var.jobset.version + create_namespace = true + namespace = "jobset-system" +} + +module "install_nvidia_dra_driver" { + count = local.install_nvidia_dra_driver ? 1 : 0 + depends_on = [var.gke_cluster_exists] + source = "./helm_install" + + release_name = "nvidia-dra-driver-gpu" # The release name + chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts + chart_name = "nvidia-dra-driver-gpu" # The chart name + chart_version = var.nvidia_dra_driver.version # The chart version + namespace = "nvidia-dra-driver-gpu" # The target namespace + create_namespace = true # Equivalent to --create-namespace + + # Use the 'values' argument to pass the YAML content + # This corresponds to the -f <(cat < +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.2 | +| [google](#requirement\_google) | >= 6.40 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.40 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_global_address.private_ip_alloc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_global_address) | resource | +| [google_compute_network_peering_routes_config.private_vpc_peering_routes_gcnv](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering_routes_config) | resource | +| [google_service_networking_connection.private_vpc_connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_networking_connection) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [address](#input\_address) | The IP address or beginning of the address range allocated for the Private Service Access. | `string` | `null` | no | +| [deletion\_policy](#input\_deletion\_policy) | The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to supporting resources. Key-value pairs. | `map(string)` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to configure Private Service Access:
`projects//global/networks/`" | `string` | n/a | yes | +| [prefix\_length](#input\_prefix\_length) | The prefix length of the IP range allocated for the Private Service Access. | `number` | `16` | no | +| [project\_id](#input\_project\_id) | ID of project in which Private Service Access will be created. | `string` | n/a | yes | +| [service\_name](#input\_service\_name) | The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'. | `string` | `"servicenetworking.googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cidr\_range](#output\_cidr\_range) | CIDR range of the created google\_compute\_global\_address | +| [connect\_mode](#output\_connect\_mode) | Services that use Private Service Access typically specify connect\_mode
"PRIVATE\_SERVICE\_ACCESS". This output value sets connect\_mode and additionally
blocks terraform actions until the VPC connection has been created. | +| [private\_vpc\_connection\_peering](#output\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection that was created by the service provider. | +| [reserved\_ip\_range](#output\_reserved\_ip\_range) | Named IP range to be used by services connected with Private Service Access. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/main.tf b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/main.tf new file mode 100644 index 0000000000..429e4d93f0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/main.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "private-service-access", ghpc_role = "network" }) +} + +locals { + split_network_id = split("/", var.network_id) + network_name = local.split_network_id[4] + network_project = local.split_network_id[1] +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_compute_global_address" "private_ip_alloc" { + provider = google + name = "global-psconnect-ip-${random_id.resource_name_suffix.hex}" + project = var.project_id + purpose = "VPC_PEERING" + address_type = "INTERNAL" + network = var.network_id + prefix_length = var.prefix_length + labels = local.labels + address = var.address +} + +resource "google_service_networking_connection" "private_vpc_connection" { + network = var.network_id + service = var.service_name + reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name] + deletion_policy = var.deletion_policy + update_on_creation_fail = var.deletion_policy == "ABANDON" ? true : null +} + +# Google Cloud NetApp Volumes need enablement of custom_route import and export +resource "google_compute_network_peering_routes_config" "private_vpc_peering_routes_gcnv" { + count = var.service_name == "netapp.servicenetworking.goog" ? 1 : 0 + project = local.network_project + network = local.network_name + peering = google_service_networking_connection.private_vpc_connection.peering + + export_custom_routes = true + import_custom_routes = true +} diff --git a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml new file mode 100644 index 0000000000..93e8b3970e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - servicenetworking.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf new file mode 100644 index 0000000000..296f2e9140 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "private_vpc_connection_peering" { + description = "The name of the VPC Network peering connection that was created by the service provider." + sensitive = true + value = google_service_networking_connection.private_vpc_connection.peering +} + +output "connect_mode" { + description = <<-EOT + Services that use Private Service Access typically specify connect_mode + "PRIVATE_SERVICE_ACCESS". This output value sets connect_mode and additionally + blocks terraform actions until the VPC connection has been created. + EOT + value = "PRIVATE_SERVICE_ACCESS" + depends_on = [ + google_service_networking_connection.private_vpc_connection, + ] +} + +output "reserved_ip_range" { + description = "Named IP range to be used by services connected with Private Service Access." + value = google_compute_global_address.private_ip_alloc.name +} + +output "cidr_range" { + description = "CIDR range of the created google_compute_global_address" + value = "${google_compute_global_address.private_ip_alloc.address}/${google_compute_global_address.private_ip_alloc.prefix_length}" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/variables.tf b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/variables.tf new file mode 100644 index 0000000000..4b0a3e796f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/variables.tf @@ -0,0 +1,59 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "address" { + description = "The IP address or beginning of the address range allocated for the Private Service Access." + type = string + default = null +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to configure Private Service Access: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "labels" { + description = "Labels to add to supporting resources. Key-value pairs." + type = map(string) +} + +variable "prefix_length" { + description = "The prefix length of the IP range allocated for the Private Service Access." + type = number + default = 16 +} + +variable "project_id" { + description = "ID of project in which Private Service Access will be created." + type = string +} + +variable "service_name" { + description = "The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'." + type = string + default = "servicenetworking.googleapis.com" +} + +variable "deletion_policy" { + description = "The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON." + type = string + default = null +} diff --git a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/versions.tf b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/versions.tf new file mode 100644 index 0000000000..df2914cdb9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.40" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" + } + + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" + } + + required_version = ">= 1.2" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/new-project/README.md b/deletion-test/primary/modules/embedded/community/modules/project/new-project/README.md new file mode 100644 index 0000000000..5e5cabe9d5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/new-project/README.md @@ -0,0 +1,128 @@ +## Description + +This module allows you to create opinionated Google Cloud Platform projects. It +creates projects and configures aspects like Shared VPC connectivity, IAM +access, Service Accounts, and API enablement to follow best practices. + +This module is meant for use with Terraform 0.13. + +**Note:** This module has been removed from the Cluster Toolkit. The upstream module (`terraform-google-project-factory`) is now the recommended way to create and manage GCP projects. + +### Example + +```yaml +- id: project + source: github.com/terraform-google-modules/terraform-google-project-factory?rev=v17.0.0&depth=1 +``` + +This creates a new project with pre-defined project ID, a designated folder and +organization and associated billing account which will be used to pay for +services consumed. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [project\_factory](#module\_project\_factory) | terraform-google-modules/project-factory/google | ~> 11.3 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [activate\_api\_identities](#input\_activate\_api\_identities) | The list of service identities (Google Managed service account for the API) to force-create for the project (e.g. in order to grant additional roles).
APIs in this list will automatically be appended to `activate_apis`.
Not including the API in this list will follow the default behaviour for identity creation (which is usually when the first resource using the API is created).
Any roles (e.g. service agent role) must be explicitly listed. See https://cloud.google.com/iam/docs/understanding-roles#service-agent-roles-roles for a list of related roles. | 
list(object({
    api   = string
    roles = list(string)
  }))
| `[]` | no | +| [activate\_apis](#input\_activate\_apis) | The list of apis to activate within the project | `list(string)` | 
[
  "compute.googleapis.com",
  "serviceusage.googleapis.com",
  "storage.googleapis.com"
]
| no | +| [auto\_create\_network](#input\_auto\_create\_network) | Create the default network | `bool` | `false` | no | +| [billing\_account](#input\_billing\_account) | The ID of the billing account to associate this project with | `string` | n/a | yes | +| [bucket\_force\_destroy](#input\_bucket\_force\_destroy) | Force the deletion of all objects within the GCS bucket when deleting the bucket (optional) | `bool` | `false` | no | +| [bucket\_labels](#input\_bucket\_labels) | A map of key/value label pairs to assign to the bucket (optional) | `map(string)` | `{}` | no | +| [bucket\_location](#input\_bucket\_location) | The location for a GCS bucket to create (optional) | `string` | `"US"` | no | +| [bucket\_name](#input\_bucket\_name) | A name for a GCS bucket to create (in the bucket\_project project), useful for Terraform state (optional) | `string` | `""` | no | +| [bucket\_project](#input\_bucket\_project) | A project to create a GCS bucket (bucket\_name) in, useful for Terraform state (optional) | `string` | `""` | no | +| [bucket\_ula](#input\_bucket\_ula) | Enable Uniform Bucket Level Access | `bool` | `true` | no | +| [bucket\_versioning](#input\_bucket\_versioning) | Enable versioning for a GCS bucket to create (optional) | `bool` | `false` | no | +| [budget\_alert\_pubsub\_topic](#input\_budget\_alert\_pubsub\_topic) | The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}` | `string` | `null` | no | +| [budget\_alert\_spent\_percents](#input\_budget\_alert\_spent\_percents) | A list of percentages of the budget to alert on when threshold is exceeded | `list(number)` | 
[
  0.5,
  0.7,
  1
]
| no | +| [budget\_amount](#input\_budget\_amount) | The amount to use for a budget alert | `number` | `null` | no | +| [budget\_display\_name](#input\_budget\_display\_name) | The display name of the budget. If not set defaults to `Budget For ` | `string` | `null` | no | +| [budget\_monitoring\_notification\_channels](#input\_budget\_monitoring\_notification\_channels) | A list of monitoring notification channels in the form `[projects/{project_id}/notificationChannels/{channel_id}]`. A maximum of 5 channels are allowed. | `list(string)` | `[]` | no | +| [consumer\_quotas](#input\_consumer\_quotas) | The quotas configuration you want to override for the project. | 
list(object({
    service = string,
    metric  = string,
    limit   = string,
    value   = string,
  }))
| `[]` | no | +| [create\_project\_sa](#input\_create\_project\_sa) | Whether the default service account for the project shall be created | `bool` | `true` | no | +| [default\_network\_tier](#input\_default\_network\_tier) | Default Network Service Tier for resources created in this project. If unset, the value will not be modified. See https://cloud.google.com/network-tiers/docs/using-network-service-tiers and https://cloud.google.com/network-tiers. | `string` | `""` | no | +| [default\_service\_account](#input\_default\_service\_account) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | `string` | `"keep"` | no | +| [disable\_dependent\_services](#input\_disable\_dependent\_services) | Whether services that are enabled and which depend on this service should also be disabled when this service is destroyed. | `bool` | `true` | no | +| [disable\_services\_on\_destroy](#input\_disable\_services\_on\_destroy) | Whether project services will be disabled when the resources are destroyed | `bool` | `true` | no | +| [domain](#input\_domain) | The domain name (optional). | `string` | `""` | no | +| [enable\_shared\_vpc\_host\_project](#input\_enable\_shared\_vpc\_host\_project) | If this project is a shared VPC host project. If true, you must *not* set svpc\_host\_project\_id variable. Default is false. | `bool` | `false` | no | +| [folder\_id](#input\_folder\_id) | The ID of a folder to host this project | `string` | `""` | no | +| [grant\_services\_network\_role](#input\_grant\_services\_network\_role) | Whether or not to grant service agents the network roles on the host project | `bool` | `true` | no | +| [grant\_services\_security\_admin\_role](#input\_grant\_services\_security\_admin\_role) | Whether or not to grant Kubernetes Engine Service Agent the Security Admin role on the host project so it can manage firewall rules | `bool` | `false` | no | +| [group\_name](#input\_group\_name) | A group to control the project by being assigned group\_role (defaults to project editor) | `string` | `""` | no | +| [group\_role](#input\_group\_role) | The role to give the controlling group (group\_name) over the project (defaults to project editor) | `string` | `"roles/editor"` | no | +| [labels](#input\_labels) | Map of labels for project | `map(string)` | `{}` | no | +| [lien](#input\_lien) | Add a lien on the project to prevent accidental deletion | `bool` | `false` | no | +| [name](#input\_name) | The name for the project | `string` | `null` | no | +| [org\_id](#input\_org\_id) | The organization ID. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The ID to give the project. If not provided, the `name` will be used. | `string` | `""` | no | +| [project\_sa\_name](#input\_project\_sa\_name) | Default service account name for the project. | `string` | `"project-service-account"` | no | +| [random\_project\_id](#input\_random\_project\_id) | Adds a suffix of 4 random characters to the `project_id` | `bool` | `false` | no | +| [sa\_role](#input\_sa\_role) | A role to give the default Service Account for the project (defaults to none) | `string` | `""` | no | +| [shared\_vpc\_subnets](#input\_shared\_vpc\_subnets) | List of subnets fully qualified subnet IDs (ie. projects/$project\_id/regions/$region/subnetworks/$subnet\_id) | `list(string)` | `[]` | no | +| [svpc\_host\_project\_id](#input\_svpc\_host\_project\_id) | The ID of the host project which hosts the shared VPC | `string` | `""` | no | +| [usage\_bucket\_name](#input\_usage\_bucket\_name) | Name of a GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | +| [usage\_bucket\_prefix](#input\_usage\_bucket\_prefix) | Prefix in the GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | +| [vpc\_service\_control\_attach\_enabled](#input\_vpc\_service\_control\_attach\_enabled) | Whether the project will be attached to a VPC Service Control Perimeter | `bool` | `false` | no | +| [vpc\_service\_control\_perimeter\_name](#input\_vpc\_service\_control\_perimeter\_name) | The name of a VPC Service Control Perimeter to add the created project to | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [api\_s\_account](#output\_api\_s\_account) | API service account email | +| [api\_s\_account\_fmt](#output\_api\_s\_account\_fmt) | API service account email formatted for terraform use | +| [budget\_name](#output\_budget\_name) | The name of the budget if created | +| [domain](#output\_domain) | The organization's domain | +| [enabled\_api\_identities](#output\_enabled\_api\_identities) | Enabled API identities in the project | +| [enabled\_apis](#output\_enabled\_apis) | Enabled APIs in the project | +| [group\_email](#output\_group\_email) | The email of the G Suite group with group\_name | +| [project\_bucket\_self\_link](#output\_project\_bucket\_self\_link) | Project's bucket selfLink | +| [project\_bucket\_url](#output\_project\_bucket\_url) | Project's bucket url | +| [project\_id](#output\_project\_id) | ID of the project that was created | +| [project\_name](#output\_project\_name) | Name of the project that was created | +| [project\_number](#output\_project\_number) | Number of the project that was created | +| [service\_account\_display\_name](#output\_service\_account\_display\_name) | The display name of the default service account | +| [service\_account\_email](#output\_service\_account\_email) | The email of the default service account | +| [service\_account\_id](#output\_service\_account\_id) | The id of the default service account | +| [service\_account\_name](#output\_service\_account\_name) | The fully-qualified name of the default service account | +| [service\_account\_unique\_id](#output\_service\_account\_unique\_id) | The unique id of the default service account | + diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/README.md b/deletion-test/primary/modules/embedded/community/modules/project/service-account/README.md new file mode 100644 index 0000000000..0f5c10c7e4 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/service-account/README.md @@ -0,0 +1,111 @@ +## Description + +Allows creation of service accounts for a Google Cloud Platform project. + +### Example + +```yaml +- id: service_acct + source: community/modules/project/service-account + settings: + project_id: $(vars.project_id) + name: instance_acct + project_roles: + - logging.logWriter + - monitoring.metricWriter + - storage.objectViewer +``` + +This creates a service account in GCP project "project_id" with the name +"instance_acct". It will have the 3 roles listed for all resources within the +project. + +### Usage with startup-script module + +When this module is used in conjunction with the [startup-script] module, the +service account must be granted (at least) read access to the bucket. This can +be achieved by granting project-wide access as shown above or by specifying the +service account as a bucket viewer in the startup-script module: + +```yaml +- id: service_acct + source: community/modules/project/service-account + settings: + project_id: $(vars.project_id) + name: instance_acct + project_roles: + - logging.logWriter + - monitoring.metricWriter +- id: script + source: modules/scripts/startup-script + settings: + bucket_viewers: + - $(service_acct.service_account_iam_email) +``` + +[startup-script]: ../../../../modules/scripts/startup-script/README.md + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [service\_account](#module\_service\_account) | terraform-google-modules/service-accounts/google | ~> 4.2 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [billing\_account\_id](#input\_billing\_account\_id) | If assigning billing role, specify a billing account (default is to assign at the organizational level). | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment (will be prepended to service account name) | `string` | n/a | yes | +| [description](#input\_description) | Description of the created service account. | `string` | `"Service Account"` | no | +| [descriptions](#input\_descriptions) | Deprecated; create single service accounts using var.description. | `list(string)` | `null` | no | +| [display\_name](#input\_display\_name) | Display name of the created service account. | `string` | `"Service Account"` | no | +| [generate\_keys](#input\_generate\_keys) | Generate keys for service account. | `bool` | `false` | no | +| [grant\_billing\_role](#input\_grant\_billing\_role) | Grant billing user role. | `bool` | `false` | no | +| [grant\_xpn\_roles](#input\_grant\_xpn\_roles) | Grant roles for shared VPC management. | `bool` | `true` | no | +| [name](#input\_name) | Name of the service account to create. | `string` | n/a | yes | +| [names](#input\_names) | Deprecated; create single service accounts using var.name. | `list(string)` | `null` | no | +| [org\_id](#input\_org\_id) | Id of the organization for org-level roles. | `string` | `""` | no | +| [prefix](#input\_prefix) | Deprecated; prefix now set using var.deployment\_name | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | +| [project\_roles](#input\_project\_roles) | List of roles to grant to service account (e.g. "storage.objectViewer" or "compute.instanceAdmin.v1" | `list(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [key](#output\_key) | Service account key (if creation was requested) | +| [service\_account\_email](#output\_service\_account\_email) | Service account e-mail address | +| [service\_account\_iam\_email](#output\_service\_account\_iam\_email) | Service account IAM binding format (serviceAccount:name@example.com) | + diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/main.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-account/main.tf new file mode 100644 index 0000000000..e8a69be642 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/service-account/main.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + display_name = "${var.display_name} (${var.deployment_name})" + description = "${var.description} (${var.deployment_name})" +} + +module "service_account" { + source = "terraform-google-modules/service-accounts/google" + version = "~> 4.2" + + billing_account_id = var.billing_account_id + description = local.description + display_name = local.display_name + generate_keys = var.generate_keys + grant_billing_role = var.grant_billing_role + grant_xpn_roles = var.grant_xpn_roles + names = [var.name] + org_id = var.org_id + prefix = var.deployment_name + project_id = var.project_id + project_roles = [for role in var.project_roles : "${var.project_id}=>roles/${role}"] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/project/service-account/metadata.yaml new file mode 100644 index 0000000000..c4dcdffdf4 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/service-account/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-account/outputs.tf new file mode 100644 index 0000000000..f9c9be05c8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/service-account/outputs.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "key" { + description = "Service account key (if creation was requested)" + value = module.service_account.key +} + +output "service_account_email" { + description = "Service account e-mail address" + value = module.service_account.email + depends_on = [ + module.service_account, + ] +} + +output "service_account_iam_email" { + description = "Service account IAM binding format (serviceAccount:name@example.com)" + value = module.service_account.iam_email + depends_on = [ + module.service_account, + ] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/variables.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-account/variables.tf new file mode 100644 index 0000000000..53267f47e7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/service-account/variables.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "billing_account_id" { + description = "If assigning billing role, specify a billing account (default is to assign at the organizational level)." + type = string + default = "" +} + +variable "deployment_name" { + description = "Name of the deployment (will be prepended to service account name)" + type = string +} + +variable "description" { + description = "Description of the created service account." + type = string + default = "Service Account" +} + +# tflint-ignore: terraform_unused_declarations +variable "descriptions" { + description = "Deprecated; create single service accounts using var.description." + type = list(string) + default = null + + validation { + condition = var.descriptions == null + error_message = "var.descriptions has been deprecated in favor of creating single accounts with var.description" + } +} + +variable "display_name" { + description = "Display name of the created service account." + type = string + default = "Service Account" +} + +variable "generate_keys" { + description = "Generate keys for service account." + type = bool + default = false +} + +variable "grant_billing_role" { + description = "Grant billing user role." + type = bool + default = false +} + +variable "grant_xpn_roles" { + description = "Grant roles for shared VPC management." + type = bool + default = true +} + +variable "name" { + description = "Name of the service account to create." + type = string +} + +# tflint-ignore: terraform_unused_declarations +variable "names" { + description = "Deprecated; create single service accounts using var.name." + type = list(string) + default = null + + validation { + condition = var.names == null + error_message = "var.names has been deprecated in favor of creating single accounts with var.name" + } +} + +variable "org_id" { + description = "Id of the organization for org-level roles." + type = string + default = "" +} + +# tflint-ignore: terraform_unused_declarations +variable "prefix" { + description = "Deprecated; prefix now set using var.deployment_name" + type = string + default = null + + validation { + condition = var.prefix == null + error_message = "var.prefix has been deprecated in favor of setting prefix with var.deployment_name" + } +} + +variable "project_id" { + description = "ID of the project" + type = string +} + +variable "project_roles" { + description = "List of roles to grant to service account (e.g. \"storage.objectViewer\" or \"compute.instanceAdmin.v1\"" + type = list(string) +} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/versions.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-account/versions.tf new file mode 100644 index 0000000000..38e6e71945 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/service-account/versions.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/README.md b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/README.md new file mode 100644 index 0000000000..266eac26ec --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/README.md @@ -0,0 +1,70 @@ +## Description + +Allows management of multiple API services for a Google Cloud Platform project. + +### Example + +```yaml +- id: services-api + source: community/modules/project/service-enablement + settings: + gcp_service_list: [ + "file.googleapis.com", + "compute.googleapis.com" + ] +``` + +This allows the project to enable both the filestore API as well as the compute API. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_project_service.gcp_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [disable\_on\_destroy](#input\_disable\_on\_destroy) | Disable services on destroy if they were enabled (or already enabled) during apply (default: false) | `bool` | `false` | no | +| [gcp\_service\_list](#input\_gcp\_service\_list) | list of APIs to be enabled for the project | `list(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/main.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/main.tf new file mode 100644 index 0000000000..965e93c549 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/main.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +resource "google_project_service" "gcp_services" { + count = length(var.gcp_service_list) + project = var.project_id + service = var.gcp_service_list[count.index] + timeouts { + create = "30m" + update = "40m" + } + + disable_dependent_services = true + disable_on_destroy = var.disable_on_destroy +} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml new file mode 100644 index 0000000000..c594c8f819 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - serviceusage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/variables.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/variables.tf new file mode 100644 index 0000000000..08f13999fe --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "ID of the project" + type = string +} + +variable "gcp_service_list" { + description = "list of APIs to be enabled for the project" + type = list(string) +} + +variable "disable_on_destroy" { + description = "Disable services on destroy if they were enabled (or already enabled) during apply (default: false)" + type = bool + default = false +} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/versions.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/versions.tf new file mode 100644 index 0000000000..07f25fb045 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md new file mode 100644 index 0000000000..052e6aee23 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md @@ -0,0 +1,87 @@ +# Description + +This module creates a Bigquery Pub/Sub Subscription. + +Primarily used for FSI - MonteCarlo Tutorial: +**[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Example + +The following example creates a Bigquery subscription using a Bigquery table and +Pub/Sub topic. + +```yaml + - id: bq_subscription + source: community/modules/pubsub/bigquery-sub + use: [bq-table, pubsub_topic] +``` + +Also see usages in this +[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_project_iam_member.editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_project_iam_member.viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_pubsub_subscription.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Name of the dataset that was created. Can be provided by the bigquery-table module | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [subscription\_id](#input\_subscription\_id) | The name of the pubsub subscription to be created | `string` | `null` | no | +| [table\_id](#input\_table\_id) | ID of created BQ table. Can be provided by the bigquery-table module | `string` | n/a | yes | +| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [subscription\_id](#output\_subscription\_id) | Name of the subscription that was created. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf new file mode 100644 index 0000000000..8edbc6b24e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf @@ -0,0 +1,57 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-sub", ghpc_role = "pubsub" }) +} + +locals { + subscription_id = var.subscription_id != null ? var.subscription_id : "${var.deployment_name}_subscription_${random_id.resource_name_suffix.hex}" +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_project_iam_member" "viewer" { + project = data.google_project.project.project_id + role = "roles/bigquery.metadataViewer" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +resource "google_project_iam_member" "editor" { + project = data.google_project.project.project_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +resource "google_pubsub_subscription" "example" { + depends_on = [google_project_iam_member.editor, google_project_iam_member.viewer] + name = local.subscription_id + topic = var.topic_id + project = var.project_id + labels = local.labels + bigquery_config { + table = "${var.project_id}.${var.dataset_id}.${var.table_id}" + use_topic_schema = true + write_metadata = true + } + +} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml new file mode 100644 index 0000000000..9aedef48dc --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - pubsub.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf new file mode 100644 index 0000000000..fc81859503 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "subscription_id" { + description = "Name of the subscription that was created." + value = google_pubsub_subscription.example.name +} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf new file mode 100644 index 0000000000..ee4dbbed8e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "topic_id" { + description = "The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module" + type = string +} + +variable "subscription_id" { + description = "The name of the pubsub subscription to be created" + type = string + default = null +} + +variable "dataset_id" { + description = "Name of the dataset that was created. Can be provided by the bigquery-table module" + type = string +} + +variable "table_id" { + description = "ID of created BQ table. Can be provided by the bigquery-table module" + type = string +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf new file mode 100644 index 0000000000..46ad6e17c8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" + } + required_version = ">= 1.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/README.md b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/README.md new file mode 100644 index 0000000000..177f799dc6 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/README.md @@ -0,0 +1,82 @@ +## Description + +Creates a Pub/Sub topic + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +### Example + +The following example creates a Pub/Sub topic. + +```yaml + - id: pubsub_topic + source: community/modules/pubsub/topic +``` + +Also see usages in this +[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_pubsub_schema.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_schema) | resource | +| [google_pubsub_topic.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [schema\_id](#input\_schema\_id) | The name of the pubsub schema to be created | `string` | `null` | no | +| [schema\_json](#input\_schema\_json) | The JSON definition of the pubsub topic schema | `string` | `"{ \n \"name\" : \"Avro\", \n \"type\" : \"record\", \n \"fields\" : \n [\n {\"name\" : \"ticker\", \"type\" : \"string\"},\n {\"name\" : \"epoch_time\", \"type\" : \"int\"},\n {\"name\" : \"iteration\", \"type\" : \"int\"},\n {\"name\" : \"start_date\", \"type\" : \"string\"},\n {\"name\" : \"end_date\", \"type\" : \"string\"},\n {\n \"name\":\"simulation_results\",\n \"type\":{\n \"type\": \"array\", \n \"items\":{\n \"name\":\"Child\",\n \"type\":\"record\",\n \"fields\":[\n {\"name\":\"price\", \"type\":\"double\"}\n ]\n }\n }\n }\n ]\n }\n"` | no | +| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to be created | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [topic\_id](#output\_topic\_id) | Name of the topic that was created. | +| [topic\_schema](#output\_topic\_schema) | Name of the topic schema that was created. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/main.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/main.tf new file mode 100644 index 0000000000..4ba68fb5d0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "topic", ghpc_role = "pubsub" }) +} + +locals { + topic_id = var.topic_id != null ? var.topic_id : "${var.deployment_name}_topic_${random_id.resource_name_suffix.hex}" + schema_id = var.schema_id != null ? var.schema_id : "${var.deployment_name}_schema_${random_id.resource_name_suffix.hex}" +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_pubsub_topic" "example" { + name = local.topic_id + depends_on = [google_pubsub_schema.example] + project = var.project_id + labels = local.labels + schema_settings { + schema = "projects/${var.project_id}/schemas/${local.schema_id}" + encoding = "BINARY" + } +} + +resource "google_pubsub_schema" "example" { + name = local.schema_id + project = var.project_id + type = "AVRO" + + definition = var.schema_json +} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml new file mode 100644 index 0000000000..9aedef48dc --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - pubsub.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf new file mode 100644 index 0000000000..3ea9d951b2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "topic_id" { + description = "Name of the topic that was created." + value = google_pubsub_topic.example.name +} + + +output "topic_schema" { + description = "Name of the topic schema that was created." + value = local.schema_id +} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/variables.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/variables.tf new file mode 100644 index 0000000000..dca575d21d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/variables.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "topic_id" { + description = "The name of the pubsub topic to be created" + type = string + default = null +} + +variable "schema_id" { + description = "The name of the pubsub schema to be created" + type = string + default = null +} + +variable "schema_json" { + description = "The JSON definition of the pubsub topic schema" + type = string + default = < **Note**: This is an experimental module. This module has only been tested in +> limited capacity with the Cluster Toolkit. The module interface may have undergo +> breaking changes in the future. + +### Example + +The following example will create a single GPU accelerated remote desktop. + +```yaml + - id: remote-desktop + source: community/modules/remote-desktop/chrome-remote-desktop + use: [network1] + settings: + install_nvidia_driver: true +``` + +### Setting up the Remote Desktop + +1. Once the remote desktop has been deployed, navigate to https://remotedesktop.google.com/headless. +1. Click through `Begin`, `Next`, & `Authorize`. +1. Copy the code snippet for `Debian Linux`. +1. SSH into the remote desktop machine. It will be listed under + [VM Instances](https://console.cloud.google.com/compute/instances) in the + Google Cloud web console. +1. Run the copied command and follow instructions to set up a PIN. +1. You should now see your machine listed on the + [Chrome Remote Desktop page](https://remotedesktop.google.com/access) under `Remote devices`. +1. Click on your machine and enter PIN if prompted. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.12.31 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [client\_startup\_script](#module\_client\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | +| [instances](#module\_instances) | ../../../../modules/compute/vm-instance | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | +| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image th
at supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-v
m-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. Cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | +| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-balanced"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs on the internet. | `bool` | `true` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required | 
list(object({
    type  = string,
    count = number
  }))
| 
[
  {
    "count": 1,
    "type": "nvidia-tesla-t4-vws"
  }
]
| no | +| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Installs the nvidia driver (true/false). For details, see https://cloud.google.com/compute/docs/gpus/install-drivers-gpu | `bool` | n/a | yes | +| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | +| [instance\_image](#input\_instance\_image) | Image used to build chrome remote desktop node. The default image is
name="debian-12-bookworm-v20250610" and project="debian-cloud".
NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues.

An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud".

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "name": "debian-12-bookworm-v20250610",
  "project": "debian-cloud"
}
| no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation. Must be N1 family if GPU is used. | `string` | `"n1-standard-8"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface
**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.
Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "RDMA", "IRDMA", "MRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. | `string` | `"default"` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `"TERMINATE"` | no | +| [project\_id](#input\_project\_id) | Project in which Google Cloud resources will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/cloud-platform"
  ]
}
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core | `number` | `2` | no | +| [zone](#input\_zone) | Default zone for creating resources | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_name](#output\_instance\_name) | Name of the first instance created, if any. | +| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf new file mode 100644 index 0000000000..a5cf7c5d37 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "chrome-remote-desktop", ghpc_role = "remote-desktop" }) +} + +locals { + + user_startup_script_runners = var.startup_script == null ? [] : [ + { + type = "shell" + content = var.startup_script + destination = "user_startup_script.sh" + } + ] + + configure_nvidia_driver_runners = var.install_nvidia_driver == false ? [] : [ + { + type = "ansible-local" + content = file("${path.module}/scripts/configure-grid-drivers.yml") + destination = "/usr/local/ghpc/configure-grid-drivers.yml" + } + ] + + configure_chrome_remote_desktop_runners = [ + { + type = "ansible-local" + content = file("${path.module}/scripts/configure-chrome-desktop.yml") + destination = "/usr/local/ghpc/configure-chrome-desktop.yml" + } + ] + + disable_sleep = [ + { + type = "ansible-local" + content = file("${path.module}/scripts/disable-sleep.yml") + destination = "/usr/local/ghpc/disable-sleep.yml" + } + ] +} + +module "client_startup_script" { + source = "../../../../modules/scripts/startup-script" + + deployment_name = var.deployment_name + project_id = var.project_id + region = var.region + labels = local.labels + + runners = flatten([ + local.user_startup_script_runners, + local.configure_nvidia_driver_runners, + local.configure_chrome_remote_desktop_runners, + local.disable_sleep + ]) +} + +module "instances" { + source = "../../../../modules/compute/vm-instance" + + instance_count = var.instance_count + name_prefix = var.name_prefix + add_deployment_name_before_prefix = var.add_deployment_name_before_prefix + provisioning_model = var.spot ? "SPOT" : null + + deployment_name = var.deployment_name + project_id = var.project_id + region = var.region + zone = var.zone + labels = local.labels + + machine_type = var.machine_type + service_account_email = var.service_account.email + metadata = var.metadata + startup_script = module.client_startup_script.startup_script + enable_oslogin = var.enable_oslogin + + instance_image = var.instance_image + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + auto_delete_boot_disk = var.auto_delete_boot_disk + + disable_public_ips = !var.enable_public_ips + network_self_link = var.network_self_link + subnetwork_self_link = var.subnetwork_self_link + network_interfaces = var.network_interfaces + bandwidth_tier = var.bandwidth_tier + tags = var.tags + + threads_per_core = var.threads_per_core + guest_accelerator = var.guest_accelerator + on_host_maintenance = var.on_host_maintenance + + network_storage = var.network_storage + +} diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf new file mode 100644 index 0000000000..bcf8ece52d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "script to load and run all runners, as a string value." + value = module.client_startup_script.startup_script +} + +output "instance_name" { + description = "Name of the first instance created, if any." + value = var.instance_count > 0 ? module.instances.name[0] : null +} diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml new file mode 100644 index 0000000000..391aa86433 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml @@ -0,0 +1,61 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Ensure Desktop OS and Chrome Remote Desktop is installed + hosts: localhost + become: true + module_defaults: + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + tasks: + - name: Install desktop packages + ansible.builtin.apt: + name: + - xfce4 + - xfce4-goodies + state: present + register: apt_result + retries: 10 + delay: 30 + until: apt_result is success + + - name: Download and configure CRD + ansible.builtin.get_url: + url: https://dl.google.com/linux/direct/chrome-remote-desktop_current_amd64.deb + dest: /tmp/chrome-remote-desktop_current_amd64.deb + mode: "0755" + timeout: 30 + + - name: Install CRD + ansible.builtin.apt: + deb: /tmp/chrome-remote-desktop_current_amd64.deb + environment: + DEBIAN_FRONTEND: noninteractive + register: apt_result + retries: 10 + delay: 30 + until: apt_result is success + + - name: Configure CRD to use Xfce by default + ansible.builtin.copy: + dest: /etc/chrome-remote-desktop-session + content: "exec /etc/X11/Xsession /usr/bin/xfce4-session" + mode: 0644 + + - name: Start Chrome remote desktop + ansible.builtin.command: /etc/init.d/chrome-remote-desktop start + register: result + changed_when: result.rc == 0 diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml new file mode 100644 index 0000000000..daae08176d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml @@ -0,0 +1,163 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Ensure nvidia grid drivers and other binaries are installed + hosts: localhost + become: true + vars: + dist_settings: + bullseye: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-510.85.02-grid.run + grid_ver: vGPU14.2 + bookworm: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-550.54.15-grid.run + grid_ver: vGPU17.1 + jammy: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - gcc-12 # must match compiler used to build kernel on latest Ubuntu 22 + - pkg-config # observed to be necessary for GRID driver installation on latest Ubuntu 22 + - libglvnd-dev # observed to be necessary for GRID driver installation on latest Ubuntu 22 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-525.125.06-grid.run + grid_ver: vGPU15.3 + tasks: + - name: Fail if using wrong OS + ansible.builtin.assert: + that: + - ansible_os_family in ["Debian", "Ubuntu"] + - ansible_distribution_release in dist_settings.keys() | list + fail_msg: "ansible_os_family: {{ ansible_os_family }} or ansible_distribution_release: {{ansible_distribution_release}} was not acceptable." + + - name: Check if GRID driver installed + ansible.builtin.command: which nvidia-smi + register: nvidiasmi_result + ignore_errors: true + changed_when: false + + - name: Install binaries for GRID drivers + ansible.builtin.apt: + name: '{{ dist_settings[ansible_distribution_release]["packages"] }}' + state: present + update_cache: true + register: apt_result + retries: 6 + delay: 10 + until: apt_result is success + + - name: Install GRID driver if not existing + when: nvidiasmi_result is failed + block: + - name: Download GPU driver + ansible.builtin.get_url: + url: https://storage.googleapis.com/nvidia-drivers-us-public/GRID/{{ dist_settings[ansible_distribution_release]["grid_ver"] }}/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} + dest: /tmp/ + mode: "0755" + timeout: 30 + + - name: Stop gdm service + ansible.builtin.systemd: + name: gdm + state: stopped + + - name: Install GPU driver + ansible.builtin.shell: | + #jinja2: trim_blocks: "True" + {% if ansible_distribution_release == "jammy" %} + CC=gcc-12 /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent + {% else %} + /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent + {% endif %} + register: result + changed_when: result.rc == 0 + + - name: Download VirtualGL driver + ansible.builtin.get_url: + url: https://sourceforge.net/projects/virtualgl/files/3.0.2/virtualgl_3.0.2_amd64.deb/download + dest: /tmp/virtualgl_3.0.2_amd64.deb + mode: "0755" + timeout: 30 + + - name: Install VirtualGL + ansible.builtin.command: gdebi /tmp/virtualgl_3.0.2_amd64.deb --non-interactive + register: result + changed_when: result.rc == 0 + + - name: Fix headless Nvidia issue + block: + - name: Lookup gpu info + ansible.builtin.command: nvidia-xconfig --query-gpu-info + register: gpu_info + failed_when: gpu_info.rc != 0 + changed_when: false + + - name: Extract PCI ID + ansible.builtin.shell: | + set -o pipefail + echo "{{ gpu_info.stdout }}" | grep "PCI BusID " | head -n 1 | cut -d':' -f2-99 | xargs + args: + executable: /bin/bash + register: pci_id + changed_when: false + + - name: Configure nvidia-xconfig + ansible.builtin.command: nvidia-xconfig -a --allow-empty-initial-configuration --enable-all-gpus --virtual=1920x1200 --busid={{ pci_id.stdout }} + register: result + changed_when: result.rc == 0 + + - name: Set HardDPMS to false + ansible.builtin.replace: + path: /etc/X11/xorg.conf + regexp: "Section \"Device\"" + replace: "Section \"Device\"\n Option \"HardDPMS\" \"false\"" + + - name: Configure VirtualGL for X + ansible.builtin.command: vglserver_config +glx +s +f -t + register: result + changed_when: result.rc == 0 + + - name: Configure gdm for X + block: + - name: Configure default display manager + ansible.builtin.copy: + dest: /etc/X11/default-display-manager + content: "/usr/sbin/gdm3" + mode: 0644 + + - name: Switch boot target to gui + ansible.builtin.command: systemctl set-default graphical.target + register: result + changed_when: result.rc == 0 + + - name: Start gdm service + ansible.builtin.systemd: + name: gdm + daemon_reload: true + state: started diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml new file mode 100644 index 0000000000..6767b05fb2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml @@ -0,0 +1,39 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Mask sleep, suspend, hibernate, and hybrid-sleep targets + hosts: localhost + become: true + tasks: + + - name: Mask sleep target + ansible.builtin.systemd: + name: sleep.target + masked: true + + - name: Mask suspend target + ansible.builtin.systemd: + name: suspend.target + masked: true + + - name: Mask hibernate target + ansible.builtin.systemd: + name: hibernate.target + masked: true + + - name: Mask hybrid-sleep target + ansible.builtin.systemd: + name: hybrid-sleep.target + masked: true diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf new file mode 100644 index 0000000000..ac4c3b1869 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf @@ -0,0 +1,277 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which Google Cloud resources will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. Cloud resource names will include this value." + type = string + #default = "chrome-remote-desktop" +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zone" { + description = "Default zone for creating resources" + type = string +} + +variable "instance_count" { + description = "Number of instances" + type = number + default = 1 +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "instance_image" { + description = <<-EOD + Image used to build chrome remote desktop node. The default image is + name="debian-12-bookworm-v20250610" and project="debian-cloud". + NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues. + + An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud". + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "debian-cloud" + name = "debian-12-bookworm-v20250610" + } +} + +variable "disk_size_gb" { + description = "Size of disk for instances." + type = number + default = 200 +} + +variable "disk_type" { + description = "Disk type for instances." + type = string + default = "pd-balanced" +} + +variable "auto_delete_boot_disk" { + description = "Controls if boot disk should be auto-deleted when instance is deleted." + type = bool + default = true +} + +variable "name_prefix" { + description = <<-EOT + An optional name for all VM and disk resources. + If not supplied, `deployment_name` will be used. + When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, + then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". + EOT + type = string + default = null +} + +variable "add_deployment_name_before_prefix" { + description = <<-EOT + If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. + See `name_prefix` for further details on resource naming behavior. + EOT + type = bool + default = false +} + +variable "enable_public_ips" { + description = "If set to true, instances will have public IPs on the internet." + type = bool + default = true +} + +variable "machine_type" { + description = "Machine type to use for the instance creation. Must be N1 family if GPU is used." + type = string + default = "n1-standard-8" +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "service_account" { + description = "Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account." + type = object({ + email = string, + scopes = set(string) + }) + default = { + email = null + scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the VM." + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the VM." + type = string + default = null +} + +variable "network_interfaces" { + description = <<-EOT + A list of network interfaces. The options match that of the terraform + network_interface block of google_compute_instance. For descriptions of the + subfields or more information see the documentation: + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + **_NOTE:_** If `network_interfaces` are set, `network_self_link` and + `subnetwork_self_link` will be ignored, even if they are provided through + the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply + to network interfaces defined in this variable. + Subfields: + network (string, required if subnetwork is not supplied) + subnetwork (string, required if network is not supplied) + subnetwork_project (string, optional) + network_ip (string, optional) + nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "RDMA", "IRDMA", "MRDMA"]) + stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) + queue_count (number, optional) + access_config (object, optional) + ipv6_access_config (object, optional) + alias_ip_range (list(object), optional) + EOT + type = list(object({ + network = string, + subnetwork = string, + subnetwork_project = string, + network_ip = string, + nic_type = string, + stack_type = string, + queue_count = number, + access_config = list(object({ + nat_ip = string, + public_ptr_domain_name = string, + network_tier = string + })), + ipv6_access_config = list(object({ + public_ptr_domain_name = string, + network_tier = string + })), + alias_ip_range = list(object({ + ip_cidr_range = string, + subnetwork_range_name = string + })) + })) + default = [] +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "startup_script" { + description = "Startup script used on the instance" + type = string + default = null +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required" + type = list(object({ + type = string, + count = number + })) + default = [{ + type = "nvidia-tesla-t4-vws" + count = 1 + }] +} + +variable "threads_per_core" { + description = "Sets the number of threads per physical core" + type = number + default = 2 +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" + type = string + default = "TERMINATE" +} + +variable "bandwidth_tier" { + description = <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +This mode can be switched to proactive (automatic) replacement by setting +[var.update_policy](#input_update_policy) to "PROACTIVE". In this case we +recommend the use of Filestore to store the job queue state ("spool") and +setting [var.spool_parent_dir][#input_spool_parent_dir] to its mount point: + +```yaml + - id: spoolfs + source: modules/file-system/filestore + use: + - network1 + settings: + filestore_tier: ENTERPRISE + local_mount: /shared + +... + + - id: htcondor_access + source: community/modules/scheduler/htcondor-access-point + use: + - network1 + - spoolfs + - htcondor_secrets + - htcondor_setup + - htcondor_cm + - htcondor_execute_point_group + settings: + spool_parent_dir: /shared +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 3.83 | +| [null](#requirement\_null) | >= 3.0 | +| [random](#requirement\_random) | ~> 3.6 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [null](#provider\_null) | >= 3.0 | +| [random](#provider\_random) | ~> 3.6 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [access\_point\_instance\_template](#module\_access\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [htcondor\_ap](#module\_htcondor\_ap) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_address.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_region_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_region_disk) | resource | +| [google_storage_bucket_object.ap_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.ap_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_shuffle.zones](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/shuffle) | resource | +| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_instance.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | +| [google_compute_region_instance_group.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_runner](#input\_access\_point\_runner) | A list of Toolkit runners for configuring an HTCondor access point | `list(map(string))` | `[]` | no | +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account for access point (e-mail format) | `string` | n/a | yes | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [autoscaler\_runner](#input\_autoscaler\_runner) | A list of Toolkit runners for configuring autoscaling daemons | `list(map(string))` | `[]` | no | +| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | +| [default\_mig\_id](#input\_default\_mig\_id) | Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | +| [disk\_type](#input\_disk\_type) | Boot disk size in GB | `string` | `"pd-balanced"` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape acoss zones for instance group managing high availability of access point | `string` | `"ANY_SINGLE_ZONE"` | no | +| [enable\_high\_availability](#input\_enable\_high\_availability) | Provision HTCondor access point in high availability mode | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | Enable Public IPs on the access points | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor and Toolkit support installed."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | +| [mig\_id](#input\_mig\_id) | List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module) | `list(string)` | `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spool\_disk\_size\_gb](#input\_spool\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | +| [spool\_disk\_type](#input\_spool\_disk\_type) | Boot disk size in GB | `string` | `"pd-ssd"` | no | +| [spool\_parent\_dir](#input\_spool\_parent\_dir) | HTCondor access point configuration SPOOL will be set to subdirectory named "spool" | `string` | `"/var/lib/condor"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | +| [zones](#input\_zones) | Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_ips](#output\_access\_point\_ips) | IP addresses of the access points provisioned by this module | +| [access\_point\_name](#output\_access\_point\_name) | Name of the access point provisioned by this module | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml new file mode 100644 index 0000000000..6a2f50c831 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml @@ -0,0 +1,120 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Access Point + hosts: localhost + become: true + vars: + spool_dir: /var/lib/condor/spool + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + htcondor_spool_disk_device: /dev/disk/by-id/google-htcondor-spool-disk + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + - name: Configure HTCondor SchedD + when: htcondor_role == 'get_htcondor_submit' + block: + - name: Format spool disk + community.general.filesystem: + fstype: ext4 + state: present + dev: "{{ htcondor_spool_disk_device }}" + # RUN TUNE2FS + - name: Mount spool (creates mount point) + ansible.posix.mount: + path: "{{ spool_dir }}" + src: "{{ htcondor_spool_disk_device }}" + fstype: ext4 + opts: defaults + state: mounted + - name: Ensure spool free space + ansible.builtin.command: tune2fs -r 0 {{ htcondor_spool_disk_device }} + - name: Setup spool directory + ansible.builtin.file: + path: "{{ spool_dir }}" + state: directory + owner: condor + group: condor + mode: 0755 + recurse: true + - name: Create SystemD override directory for HTCondor + ansible.builtin.file: + path: /etc/systemd/system/condor.service.d + state: directory + owner: root + group: root + mode: 0755 + - name: Ensure HTCondor starts after shared filesystem is mounted + ansible.builtin.copy: + dest: /etc/systemd/system/condor.service.d/mount-spool.conf + mode: 0644 + content: | + [Unit] + RequiresMountsFor={{ spool_dir }} + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor configuration complete; startup-script may still be executing ********" diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf new file mode 100644 index 0000000000..fdbcf5c32f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf @@ -0,0 +1,338 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-access-point", ghpc_role = "scheduler" }) +} + +locals { + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + metadata = merge( + local.network_storage_metadata, + local.enable_oslogin_metadata, + local.disable_automatic_updates_metadata, + var.metadata + ) + + host_count = 1 + name_prefix = "${var.deployment_name}-ap" + + example_runner = { + type = "data" + destination = "/var/tmp/helloworld.sub" + content = <<-EOT + universe = vanilla + executable = /bin/sleep + arguments = 1000 + output = out.$(ClusterId).$(ProcId) + error = err.$(ClusterId).$(ProcId) + log = log.$(ClusterId).$(ProcId) + request_cpus = 1 + request_memory = 100MB + queue + EOT + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.access_point_runner, + [local.schedd_runner], + var.autoscaler_runner, + [local.example_runner] + ) + + ap_config = templatefile("${path.module}/templates/condor_config.tftpl", { + htcondor_role = "get_htcondor_submit", + central_manager_ips = var.central_manager_ips + spool_dir = "${var.spool_parent_dir}/spool", + mig_ids = var.mig_id, + default_mig_id = var.default_mig_id + }) + + ap_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.ap_config.output_name}" + schedd_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e htcondor_role=get_htcondor_submit", + "-e config_object=${local.ap_object}", + "-e spool_dir=${var.spool_parent_dir}/spool", + "-e htcondor_spool_disk_device=/dev/disk/by-id/google-${local.spool_disk_device_name}", + ]) + } + + access_point_ips = google_compute_address.ap.address + access_point_name = data.google_compute_instance.ap.name + + spool_disk_resource_name = "${var.deployment_name}-spool-disk" + spool_disk_device_name = "htcondor-spool-disk" + spool_disk_source = try(google_compute_disk.spool[0].name, google_compute_region_disk.spool[0].self_link) + + zones = coalescelist(var.zones, random_shuffle.zones.result) + + vm_family = split("-", var.machine_type)[0] + regional_pd_families = ["e2", "n1", "n2", "n2d"] +} + +data "google_compute_image" "htcondor" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + postcondition { + condition = self.disk_size_gb <= var.disk_size_gb + error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region + + lifecycle { + postcondition { + condition = alltrue([ + for z in var.zones : contains(self.names, z) + ]) + error_message = "Each entry in var.zones must be a zone in var.region: ${var.region}" + } + } +} + +resource "random_shuffle" "zones" { + input = data.google_compute_zones.available.names + result_count = var.enable_high_availability ? 2 : 1 +} + +data "google_compute_region_instance_group" "ap" { + self_link = module.htcondor_ap.self_link + lifecycle { + postcondition { + condition = length(self.instances) == local.host_count + error_message = "There should be ${local.host_count} access points found" + } + } +} + +data "google_compute_instance" "ap" { + self_link = data.google_compute_region_instance_group.ap.instances[0].instance +} + +resource "null_resource" "ap_config" { + triggers = { + config = local.ap_config + } +} + +resource "google_storage_bucket_object" "ap_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.ap_config.id), 0, 4)}" + content = local.ap_config + bucket = var.htcondor_bucket_name + + lifecycle { + precondition { + condition = var.default_mig_id == "" || contains(var.mig_id, var.default_mig_id) + error_message = "If set, var.default_mig_id must be an element in var.mig_id" + } + + # by construction, this precondition only fails when the user has set + # var.zones to a non-empty list of length not equal to 2 + precondition { + condition = !var.enable_high_availability || length(local.zones) == 2 + error_message = "When using HTCondor access point high availability, var.zones must be of length 2." + } + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +resource "google_compute_region_disk" "spool" { + count = var.enable_high_availability ? 1 : 0 + name = local.spool_disk_resource_name + labels = local.labels + type = var.spool_disk_type + region = var.region + size = var.spool_disk_size_gb + + replica_zones = local.zones + + lifecycle { + precondition { + condition = var.spool_disk_size_gb >= 200 + error_message = "When using HTCondor access point high availability, var.spool_disk_size_gb must be set to 200 or greater." + } + + precondition { + condition = contains(local.regional_pd_families, local.vm_family) + error_message = "When using HTCondor access point high availability, var.machine_type must be one of ${jsonencode(local.regional_pd_families)}." + } + } +} + +resource "google_compute_disk" "spool" { + count = var.enable_high_availability ? 0 : 1 + name = local.spool_disk_resource_name + labels = local.labels + type = var.spool_disk_type + zone = local.zones[0] + size = var.spool_disk_size_gb +} + +resource "google_compute_address" "ap" { + project = var.project_id + name = local.name_prefix + region = var.region + subnetwork = var.subnetwork_self_link + address_type = "INTERNAL" + purpose = "GCE_ENDPOINT" +} + +module "access_point_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.access_point_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + preemptible = false + startup_script = module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.htcondor.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + network_ip = google_compute_address.ap.id + + # spool disk + additional_disks = [ + { + source = local.spool_disk_source + device_name = local.spool_disk_device_name + } + ] +} + +module "htcondor_ap" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = local.host_count + hostname = local.name_prefix + instance_template = module.access_point_instance_template.self_link + + health_check_name = "health-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) + max_surge_fixed = 0 # must be 0 to preserve hostnames + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + + stateful_disks = [{ + device_name = local.spool_disk_device_name + delete_rule = "ON_PERMANENT_INSTANCE_DELETION" + }] + stateful_ips = var.enable_public_ips ? [{ + interface_name = "nic0" + delete_rule = "ON_PERMANENT_INSTANCE_DELETION" + is_external = true + }] : [] + + # the timeouts below are default for resource + wait_for_instances = true + mig_timeouts = { + create = "15m" + delete = "15m" + update = "15m" + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf new file mode 100644 index 0000000000..f7424c6d5d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "access_point_ips" { + description = "IP addresses of the access points provisioned by this module" + value = local.access_point_ips +} + +output "access_point_name" { + description = "Name of the access point provisioned by this module" + value = local.access_point_name +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl new file mode 100644 index 0000000000..214fbc726f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl @@ -0,0 +1,70 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:${htcondor_role} +CONDOR_HOST = ${join(",", central_manager_ips)} + +SPOOL = ${spool_dir} +SCHEDD_INTERVAL = 30 +TRUST_UID_DOMAIN = True +SUBMIT_ATTRS = RunAsOwner +RunAsOwner = True + +# When a job matches to a machine, add machine attributes to the job for +# condor_history (e.g. VM Instance ID) +use feature:JobsHaveInstanceIDs +SYSTEM_JOB_MACHINE_ATTRS = $(SYSTEM_JOB_MACHINE_ATTRS) \ + CloudVMType CloudZone CloudInterruptible +SYSTEM_JOB_MACHINE_ATTRS_HISTORY_LENGTH = 10 + +# Add Cloud attributes to SchedD ClassAd +use feature:ScheddCronOneShot(cloud, $(LIBEXEC)/common-cloud-attributes-google.py) +SCHEDD_CRON_cloud_PREFIX = Cloud + +# aid the user by automatically using RequireSpot in their Requirements, unless +# the user has explicitly used CloudInterruptible +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SPOT +JOB_TRANSFORM_SPOT @=end + REQUIREMENTS ! isUndefined(RequireSpot) && ! unresolved(Requirements, "^CloudInterruptible$") + SET Requirements ($(MY.Requirements)) && (CloudInterruptible is My.RequireSpot) +@end + +# help the user by enforcing that RequireSpot is undefined or a boolean +SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) SPOT +SUBMIT_REQUIREMENT_SPOT = isUndefined(RequireSpot) || isBoolean(RequireSpot) +SUBMIT_REQUIREMENT_SPOT_REASON = "If +RequireSpot is defined, it must be either True or False" + +%{ if length(mig_ids) > 0 ~} +MIG_IDS = "${join(" ", mig_ids)}" +MIG_ID_LIST = split($(MIG_IDS)) +%{ if default_mig_id != "" ~} +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) ID_DEFAULT +JOB_TRANSFORM_ID_DEFAULT @=end + DEFAULT RequireId "${default_mig_id}" +@end +%{ endif ~} +SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) MIGID +SUBMIT_REQUIREMENT_MIGID = !isUndefined(RequireId) && member(RequireId, $(MIG_ID_LIST)) +SUBMIT_REQUIREMENT_MIGID_REASON = strcat("Jobs must set +RequireId to one of following values surrounded by quotation marks:\n", $(MIG_IDS)) + +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) MIGID +JOB_TRANSFORM_MIGID @=end + REQUIREMENTS ! isUndefined(RequireId) && ! unresolved(Requirements, "^CloudCreatedBy$") + SET Requirements ($(MY.Requirements)) && regexp(strcat("/", My.RequireId, "$"), CloudCreatedBy) +@end +%{ endif ~} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf new file mode 100644 index 0000000000..f54a88ac2e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf @@ -0,0 +1,266 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zones" { + description = "Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region." + type = list(string) + default = [] + nullable = false + + validation { + condition = length(var.zones) <= 2 + error_message = "Set var.zones to the empty list or up to 2 zones in var.region" + } +} + +variable "distribution_policy_target_shape" { + description = "Target shape acoss zones for instance group managing high availability of access point" + type = string + default = "ANY_SINGLE_ZONE" +} + +variable "network_self_link" { + description = "The self link of the network in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "access_point_service_account_email" { + description = "Service account for access point (e-mail format)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 32 + nullable = false +} + +variable "disk_type" { + description = "Boot disk size in GB" + type = string + default = "pd-balanced" + nullable = false +} + +variable "spool_disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 32 + nullable = false +} + +variable "spool_disk_type" { + description = "Boot disk size in GB" + type = string + default = "pd-ssd" + nullable = false +} + +variable "metadata" { + description = "Metadata to add to HTCondor central managers" + type = map(string) + default = {} +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + nullable = false + validation { + condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "enable_high_availability" { + description = "Provision HTCondor access point in high availability mode" + type = bool + default = false +} + +variable "instance_image" { + description = <<-EOD + Custom VM image with HTCondor and Toolkit support installed." + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "machine_type" { + description = "Machine type to use for HTCondor central managers" + type = string + default = "n2-standard-4" +} + +variable "access_point_runner" { + description = "A list of Toolkit runners for configuring an HTCondor access point" + type = list(map(string)) + default = [] +} + +variable "autoscaler_runner" { + description = "A list of Toolkit runners for configuring autoscaling daemons" + type = list(map(string)) + default = [] +} + +variable "spool_parent_dir" { + description = "HTCondor access point configuration SPOOL will be set to subdirectory named \"spool\"" + type = string + default = "/var/lib/condor" +} + +variable "central_manager_ips" { + description = "List of IP addresses of HTCondor Central Managers" + type = list(string) +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "enable_public_ips" { + description = "Enable Public IPs on the access points" + type = bool + default = false +} + +variable "mig_id" { + description = "List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module)" + type = list(string) + default = [] + nullable = false + + validation { + condition = length(var.mig_id) > 0 + error_message = "At least 1 MIG containing execute points must be provided to this module" + } +} + +variable "default_mig_id" { + description = "Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id" + type = string + default = "" + nullable = false +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" + type = string + default = "OPPORTUNISTIC" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf new file mode 100644 index 0000000000..0d07e7abf1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.74.0" + } + + required_version = ">= 1.1" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md new file mode 100644 index 0000000000..dfab563a55 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md @@ -0,0 +1,159 @@ +## Description + +This module provisions a highly available HTCondor central manager using a [Managed +Instance Group (MIG)][mig] with auto-healing. + +[mig]: https://cloud.google.com/compute/docs/instance-groups + +## Usage + +This module provisions an HTCondor central manager with a standard +configuration. For the node to function correctly, you must supply the input +variable described below: + +- [var.central_manager_runner](#input_central_manager_runner) + - Runner must download a POOL password / signing key and create an [IDTOKEN] + with no scopes (full authorization). + +A reference implementation is included in the Toolkit module +[htcondor-pool-secrets]. You may substitute implementations so long as they +duplicate the functionality in the references. Usage is demonstrated in the +[HTCondor example][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- +[htcondor-pool-secrets]: ../htcondor-pool-secrets/README.md +[IDTOKEN]: https://htcondor.readthedocs.io/en/latest/admin-manual/security.html#introducing-idtokens + +## Behavior of Managed Instance Group (MIG) + +A regional [MIG][mig] is used to provision the central manager, although only +1 node will ever be active at a time. By default, the node will be provisioned +in any of the zones available in that region, however, it can be constrained to +run in fewer zones (or a single zone) using [var.zones](#input_zones). + +When the configuration of the Central Manager is changed, the MIG can be +configured to [replace the VM][replacement] using a "proactive" or +"opportunistic" policy. By default, the Central Manager replacement policy is +set to proactive. In practice, this means that the Central Manager will be +replaced by Terraform when changes to the instance template / HTCondor +configuration are made. The Central Manager is safe to replace automatically as +it gathers its state information from periodic messages exchanged with the rest +of the HTCondor pool. + +This mode can be configured by setting [var.update_policy](#input_update_policy) +to either "PROACTIVE" (default) or "OPPORTUNISTIC". If set to opportunistic +replacement, the Central Manager will be replaced only when: + +- intentionally by issuing an update via Cloud Console or using gcloud (below) +- the VM becomes unhealthy or is otherwise automatically replaced (e.g. regular + Google Cloud maintenance) + +For example, to manually update all instances in a MIG: + +```text +gcloud compute instance-groups managed update-instances \ + <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + +## Limiting inter-zone egress + +Because all the elements of the HTCondor pool use regional MIGs, they may be +subject to [interzone egress fees][network-pricing]. The primary traffic between +nodes of an HTCondor pool running embarrassingly parallel jobs is expected to +be limited to API traffic for job scheduling and monitoring. Please review the +[network pricing][network-pricing] documentation and determine if this cost is +a concern. If it is, use [var.zones](#input_zones) to constrain each node within +your HTCondor pool to operate within a single zone. + +[network-pricing]: https://cloud.google.com/vpc/network-pricing + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1.0 | +| [google](#requirement\_google) | >= 3.83 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [central\_manager\_instance\_template](#module\_central\_manager\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [htcondor\_cm](#module\_htcondor\_cm) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_address.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_storage_bucket_object.cm_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.cm_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_instance.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | +| [google_compute_region_instance_group.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [central\_manager\_runner](#input\_central\_manager\_runner) | A list of Toolkit runners for configuring an HTCondor central manager | `list(map(string))` | `[]` | no | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for central manager (can be supplied by htcondor-setup module) | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `20` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape for instance group managing high availability of central manager | `string` | `"ANY_SINGLE_ZONE"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor installed using the htcondor-install module."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor central manager will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Central Manager ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle). | `string` | `"PROACTIVE"` | no | +| [zones](#input\_zones) | Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [central\_manager\_ips](#output\_central\_manager\_ips) | IP addresses of the central managers provisioned by this module | +| [central\_manager\_name](#output\_central\_manager\_name) | Name of the central managers provisioned by this module | +| [list\_instances\_command](#output\_list\_instances\_command) | Command to list central managers provisioned by this module | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml new file mode 100644 index 0000000000..7408af6370 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml @@ -0,0 +1,72 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor central manager + hosts: localhost + become: true + vars: + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + handlers: + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf new file mode 100644 index 0000000000..d288a91144 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf @@ -0,0 +1,226 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-central-manager", ghpc_role = "scheduler" }) +} + +locals { + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + metadata = merge( + local.network_storage_metadata, + local.enable_oslogin_metadata, + local.disable_automatic_updates_metadata, + var.metadata + ) + + name_prefix = "${var.deployment_name}-cm" + + cm_config = templatefile("${path.module}/templates/condor_config.tftpl", {}) + + cm_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.cm_config.output_name}" + schedd_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e config_object=${local.cm_object}", + ]) + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.central_manager_runner, + [local.schedd_runner] + ) + + central_manager_ips = google_compute_address.cm.address + central_manager_name = data.google_compute_instance.cm.name + + list_instances_command = "gcloud compute instance-groups list-instances ${data.google_compute_region_instance_group.cm.name} --region ${var.region} --project ${var.project_id}" + + zones = coalescelist(var.zones, data.google_compute_zones.available.names) +} + +data "google_compute_image" "htcondor" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + postcondition { + condition = self.disk_size_gb <= var.disk_size_gb + error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +data "google_compute_region_instance_group" "cm" { + self_link = module.htcondor_cm.self_link + lifecycle { + postcondition { + condition = length(self.instances) == 1 + error_message = "There should only be 1 central manager found" + } + } +} + +data "google_compute_instance" "cm" { + self_link = data.google_compute_region_instance_group.cm.instances[0].instance +} + +resource "null_resource" "cm_config" { + triggers = { + config = local.cm_config + } +} + +resource "google_storage_bucket_object" "cm_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.cm_config.id), 0, 4)}" + content = local.cm_config + bucket = var.htcondor_bucket_name +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +resource "google_compute_address" "cm" { + project = var.project_id + name = local.name_prefix + region = var.region + subnetwork = var.subnetwork_self_link + address_type = "INTERNAL" + purpose = "GCE_ENDPOINT" +} + +module "central_manager_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.central_manager_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + preemptible = false + startup_script = module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.htcondor.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + network_ip = google_compute_address.cm.id +} + +module "htcondor_cm" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = 1 + hostname = local.name_prefix + instance_template = module.central_manager_instance_template.self_link + + health_check_name = "health-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) + max_surge_fixed = 0 # must be 0 to preserve hostnames + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + + # the timeouts below are default for resource + wait_for_instances = true + mig_timeouts = { + create = "15m" + delete = "15m" + update = "15m" + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf new file mode 100644 index 0000000000..a6272e7ca2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "list_instances_command" { + description = "Command to list central managers provisioned by this module" + value = local.list_instances_command +} + +output "central_manager_ips" { + description = "IP addresses of the central managers provisioned by this module" + value = local.central_manager_ips +} + +output "central_manager_name" { + description = "Name of the central managers provisioned by this module" + value = local.central_manager_name +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl new file mode 100644 index 0000000000..5b9676457e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl @@ -0,0 +1,31 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:get_htcondor_central_manager +CONDOR_HOST = $(IPV4_ADDRESS) + +# Central Manager configuration settings +# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-collector-configuration-file-entries +# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-negotiator-configuration-file-entries +# set classad lifetime (expiration) to ~5x the update interval for all daemons +# defaults to 900s +CLASSAD_LIFETIME = 180 +COLLECTOR_UPDATE_INTERVAL = 30 +NEGOTIATOR_UPDATE_INTERVAL = 30 +NEGOTIATOR_DEPTH_FIRST = True +NEGOTIATOR_UPDATE_AFTER_CYCLE = True diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf new file mode 100644 index 0000000000..7f85861c3f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf @@ -0,0 +1,192 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor central manager will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zones" { + description = "Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region." + type = list(string) + default = [] + nullable = false +} + +variable "distribution_policy_target_shape" { + description = "Target shape for instance group managing high availability of central manager" + type = string + default = "ANY_SINGLE_ZONE" +} + +variable "network_self_link" { + description = "The self link of the network in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "central_manager_service_account_email" { + description = "Service account e-mail for central manager (can be supplied by htcondor-setup module)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 20 + nullable = false +} + +variable "metadata" { + description = "Metadata to add to HTCondor central managers" + type = map(string) + default = {} +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + nullable = false + validation { + condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "instance_image" { + description = <<-EOD + Custom VM image with HTCondor installed using the htcondor-install module." + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "machine_type" { + description = "Machine type to use for HTCondor central managers" + type = string + default = "n2-standard-4" +} + +variable "central_manager_runner" { + description = "A list of Toolkit runners for configuring an HTCondor central manager" + type = list(map(string)) + default = [] +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Central Manager (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)." + type = string + default = "PROACTIVE" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf new file mode 100644 index 0000000000..4dee3adac7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.74.0" + } + + required_version = ">= 1.1.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md new file mode 100644 index 0000000000..7158e7bac6 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md @@ -0,0 +1,172 @@ +## Description + +This module is responsible for the following actions: + +- store an HTCondor Pool password in Google Cloud Secret Manager + - will generate a new password if one is not supplied +- create a secret in Google Cloud Secret Manager in which the HTCondor central + manager can place IDTOKENs (JWT Authorizations) for execute points to download +- create a Toolkit runner for the central manager + - download the POOL password / signing key + - create a local IDTOKEN for itself + - upload the execute point IDTOKEN secret +- create a Toolkit runner for access points + - download the POOL password / signing key + - create a local IDTOKEN for itself +- create a Toolkit runner for execute points + - Fetch the IDTOKEN secret generated by the central manager + +It is expected to be used with the [htcondor-install] and +[htcondor-execute-point] modules. + +[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[htcondor-install]: ../../scripts/htcondor-setup/README.md +[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md + +[htcrole]: https://htcondor.readthedocs.io/en/latest/getting-htcondor/admin-quick-start.html#what-get-htcondor-does-to-configure-a-role + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_setup + + - id: htcondor_startup_central_manager + source: modules/scripts/startup-script + settings: + runners: + - $(htcondor_install.install_htcondor_runner) + - $(htcondor_secrets.central_manager_runner) + - $(htcondor_setup.central_manager_runner) + +- id: htcondor_cm + source: modules/compute/vm-instance + use: + - network1 + - htcondor_startup_central_manager + settings: + name_prefix: cm0 + machine_type: c2-standard-4 + disable_public_ips: true + service_account: + email: $(htcondor_setup.central_manager_service_account) + scopes: + - cloud-platform + network_interfaces: + - network: null + subnetwork: $(network1.subnetwork_self_link) + subnetwork_project: $(vars.project_id) + network_ip: $(htcondor_setup.central_manager_internal_ip) + stack_type: null + access_config: [] + ipv6_access_config: [] + alias_ip_range: [] + nic_type: VIRTIO_NET + queue_count: null + outputs: + - internal_ip +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 4.84 | +| [random](#requirement\_random) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | +| [random](#provider\_random) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_secret_manager_secret.execute_point_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_iam_member.access_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.central_manager_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.central_manager_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.execute_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_version.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [random_password.pool](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [pool\_password](#input\_pool\_password) | HTCondor Pool Password | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [trust\_domain](#input\_trust\_domain) | Trust domain for HTCondor pool (if not supplied, will be set based on project\_id) | `string` | `""` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_runner](#output\_access\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor access point | +| [central\_manager\_runner](#output\_central\_manager\_runner) | Toolkit Runner to download pool secrets to an HTCondor central manager | +| [execute\_point\_runner](#output\_execute\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor execute point | +| [pool\_password\_secret\_id](#output\_pool\_password\_secret\_id) | Google Cloud Secret Manager ID containing HTCondor Pool Password | +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | PowerShell script to download pool secrets to an HTCondor execute point | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml new file mode 100644 index 0000000000..538c809c2a --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml @@ -0,0 +1,102 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Secrets + hosts: localhost + become: true + vars: + condor_config_root: /etc/condor + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - password_id is defined + - trust_domain is defined + - name: Set Pool Trust Domain + ansible.builtin.copy: + dest: "{{ condor_config_root }}/config.d/51-ghpc-trust-domain" + mode: 0644 + content: | + # these lines must appear AFTER any "use role:" settings + UID_DOMAIN = {{ trust_domain }} + TRUST_DOMAIN = {{ trust_domain }} + - name: Get HTCondor Pool password (token signing key) + when: htcondor_role != 'get_htcondor_execute' + ansible.builtin.shell: | + set -e -o pipefail +o history + POOL_PASSWORD=$(gcloud secrets versions access latest --secret={{ password_id }}) + echo -n "$POOL_PASSWORD" | sh -c "condor_store_cred add -c -i -" + args: + creates: "{{ condor_config_root }}/passwords.d/POOL" + executable: /bin/bash + - name: Configure HTCondor Central Manager + when: htcondor_role == 'get_htcondor_central_manager' + block: + - name: Create IDTOKEN for Central Manager + ansible.builtin.shell: | + umask 0077 + condor_token_create -identity condor@{{ trust_domain }} \ + -token condor@{{ trust_domain }} + args: + creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" + - name: Create IDTOKEN secret for Execute Points + when: xp_idtoken_secret_id | length > 0 + changed_when: true + ansible.builtin.shell: | + umask 0077 + TMPFILE=$(mktemp) + condor_token_create -authz READ -authz ADVERTISE_MASTER \ + -authz ADVERTISE_STARTD -identity condor@{{ trust_domain }} > "$TMPFILE" + gcloud secrets versions add --data-file "$TMPFILE" {{ xp_idtoken_secret_id }} + rm -f "$TMPFILE" + - name: Configure HTCondor SchedD + when: htcondor_role == 'get_htcondor_submit' + block: + - name: Create IDTOKEN to advertise access point + ansible.builtin.shell: | + umask 0077 + # DAEMON authorization can likely be removed in future when scopes + # needed to trigger a negotiation cycle are changed. Suggest review + # https://opensciencegrid.atlassian.net/jira/software/c/projects/HTCONDOR/issues/?filter=allissues + condor_token_create -authz READ -authz ADVERTISE_MASTER \ + -authz ADVERTISE_SCHEDD -authz DAEMON -identity condor@{{ trust_domain }} \ + -token condor@{{ trust_domain }} + args: + creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" + - name: Configure HTCondor StartD + when: htcondor_role == 'get_htcondor_execute' + block: + - name: Create SystemD override directory for HTCondor Execute Point + ansible.builtin.file: + path: /etc/systemd/system/condor.service.d + state: directory + owner: root + group: root + mode: 0755 + - name: Fetch IDTOKEN to advertise execute point + ansible.builtin.copy: + dest: "/etc/systemd/system/condor.service.d/htcondor-token-fetcher.conf" + mode: 0644 + content: | + [Service] + ExecStartPre=gcloud secrets versions access latest --secret {{ xp_idtoken_secret_id }} \ + --out-file {{ condor_config_root }}/tokens.d/condor@{{ trust_domain }} + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf new file mode 100644 index 0000000000..1a7c761760 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf @@ -0,0 +1,168 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-pool-secrets", ghpc_role = "scheduler" }) +} + +locals { + pool_password = coalesce(var.pool_password, random_password.pool.result) + auto = length(var.user_managed_replication) == 0 ? "" : "-user" + access_point_service_account_iam_email = "serviceAccount:${var.access_point_service_account_email}" + central_manager_service_account_iam_email = "serviceAccount:${var.central_manager_service_account_email}" + execute_point_service_account_iam_email = "serviceAccount:${var.execute_point_service_account_email}" + + trust_domain = coalesce(var.trust_domain, "c.${var.project_id}.internal") + + runner_cm = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_central_manager", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + + runner_access = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_submit", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + + runner_execute = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_execute", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + windows_startup_ps1 = templatefile( + "${path.module}/templates/fetch-idtoken.ps1.tftpl", + { + trust_domain = local.trust_domain, + xp_idtoken_secret_id = google_secret_manager_secret.execute_point_idtoken.secret_id, + } + ) +} + +resource "random_password" "pool" { + length = 24 + special = true + override_special = "_-#=." +} + +resource "google_secret_manager_secret" "pool_password" { + secret_id = "${var.deployment_name}-pool-password${local.auto}" + + labels = local.labels + + replication { + dynamic "auto" { + for_each = length(var.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_version" "pool_password" { + secret = google_secret_manager_secret.pool_password.id + secret_data = local.pool_password +} + +# this secret will be populated by the Central Manager +resource "google_secret_manager_secret" "execute_point_idtoken" { + secret_id = "${var.deployment_name}-execute-point-idtoken${local.auto}" + + labels = local.labels + + replication { + dynamic "auto" { + for_each = length(var.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_iam_member" "central_manager_password" { + secret_id = google_secret_manager_secret.pool_password.id + role = "roles/secretmanager.secretAccessor" + member = local.central_manager_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "central_manager_idtoken" { + secret_id = google_secret_manager_secret.execute_point_idtoken.id + role = "roles/secretmanager.secretVersionManager" + member = local.central_manager_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "access_point" { + secret_id = google_secret_manager_secret.pool_password.id + role = "roles/secretmanager.secretAccessor" + member = local.access_point_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "execute_point" { + secret_id = google_secret_manager_secret.execute_point_idtoken.id + role = "roles/secretmanager.secretAccessor" + member = local.execute_point_service_account_iam_email +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml new file mode 100644 index 0000000000..4b0bdbd616 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com + - secretmanager.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf new file mode 100644 index 0000000000..81c4986b16 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "pool_password_secret_id" { + description = "Google Cloud Secret Manager ID containing HTCondor Pool Password" + value = google_secret_manager_secret.pool_password.secret_id + sensitive = true +} + +output "central_manager_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor central manager" + value = local.runner_cm + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "access_point_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor access point" + value = local.runner_access + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "execute_point_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor execute point" + value = local.runner_execute + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "windows_startup_ps1" { + description = "PowerShell script to download pool secrets to an HTCondor execute point" + value = local.windows_startup_ps1 +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl new file mode 100644 index 0000000000..04c96291ee --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl @@ -0,0 +1,26 @@ +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +$config_dir = 'C:\Condor\config' +if(!(test-path -PathType container -Path $config_dir)) +{ + New-Item -ItemType Directory -Path $config_dir +} +$config_file = "$config_dir\51-ghpc-trust-domain" + +$config_string = @' +# these lines must appear AFTER any "use role:" settings +UID_DOMAIN = ${trust_domain} +TRUST_DOMAIN = ${trust_domain} +'@ + +Set-Content -Path "$config_file" -Value "$config_string" + +# obtain IDTOKEN for authentication by StartD to Central Manager +gcloud secrets versions access latest --secret ${xp_idtoken_secret_id} ` + --out-file C:\condor\tokens.d\condor@${trust_domain} + +if ($LASTEXITCODE -ne 0) +{ + throw "Could not download HTCondor IDTOKEN; exiting startup script" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf new file mode 100644 index 0000000000..22ef3644e8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "access_point_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "central_manager_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "execute_point_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "pool_password" { + description = "HTCondor Pool Password" + type = string + sensitive = true + default = null +} + +variable "trust_domain" { + description = "Trust domain for HTCondor pool (if not supplied, will be set based on project_id)" + type = string + default = "" +} + +variable "user_managed_replication" { + type = list(object({ + location = string + kms_key_name = optional(string) + })) + description = "Replication parameters that will be used for defined secrets" + default = [] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf new file mode 100644 index 0000000000..d8a1d96f5f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md new file mode 100644 index 0000000000..5a403c0a38 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md @@ -0,0 +1,128 @@ +## Description + +This module creates the service accounts for use by the primary elements of an +[HTCondor pool][pool]: + +- Central Managers +- Access Points +- Execute Points + +Each service account is assigned common roles necessary for the VM to function +properly. In particular, nearly every VM requires the ability to read from Cloud +Storage buckets and write Cloud Logging entries. These roles are configurable +as described below. + +[pool]: https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-admin-manual.html#the-different-roles-a-machine-can-play + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_service_accounts + source: community/modules/scheduler/htcondor-service-accounts + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + - htcondor_service_accounts + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_service_accounts + +- id: htcondor_cm + source: community/modules/scheduler/htcondor-central-manager + use: + - network1 + - htcondor_secrets + - htcondor_service_accounts + - htcondor_setup + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - central_manager_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [access\_point\_service\_account](#module\_access\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | +| [central\_manager\_service\_account](#module\_central\_manager\_service\_account) | ../../../../community/modules/project/service-account | n/a | +| [execute\_point\_service\_account](#module\_execute\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_roles](#input\_access\_point\_roles) | Project-wide roles for HTCondor Access Point service account | `list(string)` | 
[
  "compute.instanceAdmin.v1",
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [central\_manager\_roles](#input\_central\_manager\_roles) | Project-wide roles for HTCondor Central Manager service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_roles](#input\_execute\_point\_roles) | Project-wide roles for HTCondor Execute Point service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_service\_account\_email](#output\_access\_point\_service\_account\_email) | HTCondor Access Point Service Account (e-mail format) | +| [central\_manager\_service\_account\_email](#output\_central\_manager\_service\_account\_email) | HTCondor Central Manager Service Account (e-mail format) | +| [execute\_point\_service\_account\_email](#output\_execute\_point\_service\_account\_email) | HTCondor Execute Point Service Account (e-mail format) | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf new file mode 100644 index 0000000000..9d97b18642 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# NB: the community/modules/project/service-account module will not output the +# service account e-mail address until all IAM bindings have been created; if +# underlying implementation changes, this module should declare explicit +# depends_on the IAM bindings to prevent race conditions for services that +# require them + +module "access_point_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Access Point" + deployment_name = var.deployment_name + name = "access" + project_roles = var.access_point_roles +} + +module "execute_point_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Execute Point" + deployment_name = var.deployment_name + name = "execute" + project_roles = var.execute_point_roles +} + +module "central_manager_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Central Manager" + deployment_name = var.deployment_name + name = "cm" + project_roles = var.central_manager_roles +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml new file mode 100644 index 0000000000..c4dcdffdf4 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf new file mode 100644 index 0000000000..28f3a79457 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "access_point_service_account_email" { + description = "HTCondor Access Point Service Account (e-mail format)" + value = module.access_point_service_account.service_account_email +} + +output "central_manager_service_account_email" { + description = "HTCondor Central Manager Service Account (e-mail format)" + value = module.central_manager_service_account.service_account_email +} + +output "execute_point_service_account_email" { + description = "HTCondor Execute Point Service Account (e-mail format)" + value = module.execute_point_service_account.service_account_email +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf new file mode 100644 index 0000000000..ee186e0971 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "access_point_roles" { + description = "Project-wide roles for HTCondor Access Point service account" + type = list(string) + default = [ + "compute.instanceAdmin.v1", + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} + +variable "central_manager_roles" { + description = "Project-wide roles for HTCondor Central Manager service account" + type = list(string) + default = [ + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} + +variable "execute_point_roles" { + description = "Project-wide roles for HTCondor Execute Point service account" + type = list(string) + default = [ + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md new file mode 100644 index 0000000000..1722702ceb --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md @@ -0,0 +1,118 @@ +## Description + +This module creates a bucket in which to store HTCondor configurations and +a firewall rule that allows Managed Instance Group health checks to probe the +health of HTCondor VMs. + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_service_accounts + source: community/modules/scheduler/htcondor-service-accounts + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + - htcondor_service_accounts + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_service_accounts + +- id: htcondor_cm + source: community/modules/scheduler/htcondor-central-manager + use: + - network1 + - htcondor_secrets + - htcondor_service_accounts + - htcondor_setup + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - central_manager_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [health\_check\_firewall\_rule](#module\_health\_check\_firewall\_rule) | ../../../../modules/network/firewall-rules | n/a | +| [htcondor\_bucket](#module\_htcondor\_bucket) | ../../../../modules/file-system/cloud-storage-bucket | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account e-mail for HTCondor Access Point | `string` | n/a | yes | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for HTCondor Central Manager | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account e-mail for HTCondor Execute Points | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which Central Managers will be placed. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [htcondor\_bucket\_name](#output\_htcondor\_bucket\_name) | Name of the HTCondor configuration bucket | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf new file mode 100644 index 0000000000..e048362663 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-setup", ghpc_role = "scheduler" }) +} + +locals { + service_account_iam_email = [ + "serviceAccount:${var.access_point_service_account_email}", + "serviceAccount:${var.central_manager_service_account_email}", + "serviceAccount:${var.execute_point_service_account_email}", + ] + service_account_email = [ + var.access_point_service_account_email, + var.central_manager_service_account_email, + var.execute_point_service_account_email, + ] +} + +module "health_check_firewall_rule" { + source = "../../../../modules/network/firewall-rules" + + subnetwork_self_link = var.subnetwork_self_link + + ingress_rules = [{ + name = "allow-health-check-${var.deployment_name}" + description = "Allow Managed Instance Group Health Checks for HTCondor VMs" + direction = "INGRESS" + source_ranges = [ + "130.211.0.0/22", + "35.191.0.0/16", + ] + target_service_accounts = local.service_account_email + allow = [{ + protocol = "tcp" + ports = ["9618"] + }] + }] +} + +module "htcondor_bucket" { + source = "../../../../modules/file-system/cloud-storage-bucket" + + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + name_prefix = "${var.deployment_name}-htcondor-config" + random_suffix = true + labels = local.labels + viewers = local.service_account_iam_email + + use_deployment_name_in_bucket_name = false +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml new file mode 100644 index 0000000000..7b4918b962 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - iam.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf new file mode 100644 index 0000000000..a44223faee --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "htcondor_bucket_name" { + description = "Name of the HTCondor configuration bucket" + value = module.htcondor_bucket.gcs_bucket_name + + # ensure that all IAM bindings to the bucket and firewall rules are active + # before this modules output is allowed to propagate + depends_on = [ + module.htcondor_bucket, + module.health_check_firewall_rule + ] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf new file mode 100644 index 0000000000..147a2ca88d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which Central Managers will be placed." + type = string +} + +variable "access_point_service_account_email" { + description = "Service account e-mail for HTCondor Access Point" + type = string +} + +variable "central_manager_service_account_email" { + description = "Service account e-mail for HTCondor Central Manager" + type = string +} + +variable "execute_point_service_account_email" { + description = "Service account e-mail for HTCondor Execute Points" + type = string +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md new file mode 100644 index 0000000000..43254cbfa8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md @@ -0,0 +1,405 @@ +## Description + +This module creates a slurm controller node via the internal +[slurm\_instance\_template] module. + +More information about Slurm On GCP can be found at the +[project's GitHub page][slurm-gcp] and in the +[Slurm on Google Cloud User Guide][slurm-ug]. + +The [user guide][slurm-ug] provides detailed instructions on customizing and +enhancing the Slurm on GCP cluster as well as recommendations on configuring the +controller for optimal performance at different scales. + +[slurm\_instance\_template]: /community/modules/internal/slurm-gcp/instance_template/README.md +[slurm-ug]: https://goo.gle/slurm-gcp-user-guide. +[enable\_cleanup\_compute]: #input\_enable\_cleanup\_compute +[enable\_cleanup\_subscriptions]: #input\_enable\_cleanup\_subscriptions +[enable\_reconfigure]: #input\_enable\_reconfigure + +### Example + +```yaml +- id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: + - network + - homefs + - compute_partition + settings: + machine_type: c2-standard-8 +``` + +This creates a controller node with the following attributes: + +* connected to the primary subnetwork of `network` +* the filesystem with the ID `homefs` (defined elsewhere in the blueprint) + mounted +* One partition with the ID `compute_partition` (defined elsewhere in the + blueprint) +* machine type upgraded from the default `c2-standard-4` to `c2-standard-8` + +### Live Cluster Reconfiguration + +The `schedmd-slurm-gcp-v6-controller` module supports the reconfiguration of +partitions and slurm configuration in a running, active cluster. + +To reconfigure a running cluster: + +1. Edit the blueprint with the desired configuration changes +2. Call `gcluster create -w` to overwrite the deployment directory +3. Follow instructions in terminal to deploy + +The following are examples of updates that can be made to a running cluster: + +* Add or remove a partition to the cluster +* Resize an existing partition +* Attach new network storage to an existing partition + +> **NOTE**: Changing the VM `machine_type` of a partition may not work. +> It is better to create a new partition and delete the old one. + +## Custom Images + +For more information on creating valid custom images for the controller VM +instance or for custom instance templates, see our [vm-images.md] documentation +page. + +[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images + +## GPU Support + +More information on GPU support in Slurm on GCP and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) + +## Reservation for Scheduled Maintenance + +A [maintenance event](https://cloud.google.com/compute/docs/instances/host-maintenance-overview#maintenanceevents) is when a compute engine stops a VM to perform a hardware or +software update which is determined by the host maintenance policy. This can +also affect the running jobs if the maintenance kicks in. Now, Customers can +protect jobs from getting terminated due to maintenance using the cluster +toolkit. You can enable creation of reservation for scheduled maintenance for +your compute nodeset and Slurm will reserve your node for maintenance during the +maintenance window. If you try to schedule any jobs which overlap with the +maintenance reservation, Slurm would not schedule any job. + +You can specify in your blueprint like + +```yaml + - id: compute_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network] + settings: + enable_maintenance_reservation: true +``` + +To enable creation of reservation for maintenance. + +While running job on slurm cluster, you can specify total run time of the job +using [-t flag](https://slurm.schedmd.com/srun.html#OPT_time).This would only +run the job outside of the maintenance window. + +```shell +srun -n1 -pcompute -t 10:00 +``` + +Currently upcoming maintenance notification is supported in ALPHA version of +compute API. You can update the API version from your blueprint, + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + settings: + endpoint_versions: + compute: "alpha" +``` + +## Opportunistic GCP maintenance in Slurm + +Customers can also enable running GCP maintenance as Slurm job opportunistically +to perform early maintenance. If a node is detected for maintenance, Slurm will +create a job to perform maintenance and put it in the job queue. + +If [backfill](https://slurm.schedmd.com/sched_config.html#backfill) scheduler is +used, Slurm will backfill maintenance job if it can find any empty time window. + +Customer can also choose builtin scheduler type. In this case, Slurm would run +maintenance job in strictly priority order. If the maintenance job doesn't kick +in, then forced maintenance will take place at scheduled window. + +Customer can enable this feature at nodeset level by, + +```yaml + - id: debug_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network] + settings: + enable_opportunistic_maintenance: true +``` + +## Placement Max Distance + +When using +[enable_placement](../../../../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md#input_enable_placement) +with Slurm, Google Compute Engine will attempt to place VMs as physically close +together as possible. Capacity constraints at the time of VM creation may still +force VMs to be spread across multiple racks. Google provides the `max-distance` +flag which can used to control the maximum spreading allowed. Read more about +`max-distance` in the +[official docs](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies +). + +You can use the `placement_max_distance` setting on the nodeset module to control the `max-distance` behavior. See the following example: + +```yaml + - id: nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [ network ] + settings: + machine_type: c2-standard-4 + node_count_dynamic_max: 30 + enable_placement: true + placement_max_distance: 1 + +> [!NOTE] +> `schedmd-slurm-gcp-v6-nodeset.settings.enable_placement: true` must also be +> set for placement_max_distance to take effect. + +In the above case using a value of 1 will restrict VM to be placed on the same +rack. You can confirm that the `max-distance` was applied by calling the +following command while jobs are running: + +```shell +gcloud beta compute resource-policies list \ + --format='yaml(name,groupPlacementPolicy.maxDistance)' +``` + +> [!WARNING] +> If a zone lacks capacity, using a lower `max-distance` value (such as 1) is +> more likely to cause VMs creation to fail. + +## TreeWidth and Node Communication + +Slurm uses a fan out mechanism to communicate large groups of nodes. The shape +of this fan out tree is determined by the +[TreeWidth](https://slurm.schedmd.com/slurm.conf.html#OPT_TreeWidth) +configuration variable. + +In the cloud, this fan out mechanism can become unstable when nodes restart with +new IP addresses. You can enforce that all nodes communicate directly with the +controller by setting TreeWidth to a value >= largest partition. + +If the largest partition was 200 nodes, configure the blueprint as follows: + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + ... + settings: + cloud_parameters: + tree_width: 200 +``` + +The default has been set to 128. Values above this have not been fully tested +and may cause congestion on the controller. A more scalable solution is under +way. + +## ResumeRate and Node Resumption + +The `ResumeRate` parameter in `slurm.conf` controls the maximum number of nodes +that Slurm attempts to resume (power up) per minute. This is particularly +important in cloud environments where auto-scaling can lead to a large number of +nodes starting concurrently. + +When many nodes start simultaneously, they can place a heavy load on shared +resources, especially shared filesystems, as they all try to mount filesystems +and access configuration files at the same time. By limiting the `ResumeRate`, +you can stagger the node startup process, reducing the peak load on these shared +resources and improving overall cluster stability during scaling events. + +For example, to limit the node resumption rate to 100 nodes per minute, +configure the blueprint as follows: + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + ... + settings: + cloud_parameters: + resume_rate: 100 +``` + +Adjust this value based on the capabilities of your shared filesystem and the +expected scaling behavior of your cluster. + +## Support +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 6.41 | +| [google-beta](#requirement\_google-beta) | >= 6.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.41 | +| [google-beta](#provider\_google-beta) | >= 6.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [bucket](#module\_bucket) | terraform-google-modules/cloud-storage/google | >= 6.1 | +| [daos\_network\_storage\_scripts](#module\_daos\_network\_storage\_scripts) | ../../../../modules/scripts/startup-script | n/a | +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [login](#module\_login) | ../../internal/slurm-gcp/login | n/a | +| [nodeset\_cleanup](#module\_nodeset\_cleanup) | ./modules/cleanup_compute | n/a | +| [nodeset\_cleanup\_tpu](#module\_nodeset\_cleanup\_tpu) | ./modules/cleanup_tpu | n/a | +| [slurm\_controller\_template](#module\_slurm\_controller\_template) | ../../internal/slurm-gcp/instance_template | n/a | +| [slurm\_files](#module\_slurm\_files) | ./modules/slurm_files | n/a | +| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | +| [slurm\_nodeset\_tpu](#module\_slurm\_nodeset\_tpu) | ../../internal/slurm-gcp/nodeset_tpu | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance_from_template.controller](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_from_template) | resource | +| [google_compute_disk.controller_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_secret_manager_secret.cloudsql](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_iam_member.cloudsql_secret_accessor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_version.cloudsql_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [google_storage_bucket_iam_member.legacy_readers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | +| [google_storage_bucket_iam_member.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | +| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_project.controller_project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = string
    device_name                = string
    disk_type                  = string
    disk_size_gb               = number
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. If not specified, then one will be chosen based on slurm\_cluster\_name. | `string` | `null` | no | +| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket.
Ignored when 'create\_bucket' is true. | `string` | `null` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | +| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Defaults inherited from [Slurm GCP repo](https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/terraform/slurm_cluster/modules/slurm_files/README_TF.md#input_cloud_parameters) | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | +| [cloudsql](#input\_cloudsql) | Use this database instead of the one on the controller.
server\_ip : Address of the database server.
user : The user to access the database as.
password : The password, given the user, to access the given database. (sensitive)
db\_name : The database to access.
user\_managed\_replication : The list of location and (optional) kms\_key\_name for secret | 
object({
    server_ip = string
    user      = string
    password  = string # sensitive
    db_name   = string
    user_managed_replication = optional(list(object({
      location     = string
      kms_key_name = optional(string)
    })), [])
  })
| `null` | no | +| [compute\_startup\_script](#input\_compute\_startup\_script) | DEPRECATED: `compute_startup_script` has been deprecated.
Use `startup_script` of nodeset module instead. | `any` | `null` | no | +| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script in compute nodes. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | +| [controller\_project\_id](#input\_controller\_project\_id) | Optionally. Provision controller and config bucket in the different project | `string` | `null` | no | +| [controller\_startup\_script](#input\_controller\_startup\_script) | Startup script used by the controller VM. | `string` | `"# no-op"` | no | +| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    type = string
    size = number
  })
| 
{
  "size": 50,
  "type": "pd-ssd"
}
| no | +| [create\_bucket](#input\_create\_bucket) | Create GCS bucket instead of using an existing one. | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment. | `string` | n/a | yes | +| [disable\_controller\_public\_ips](#input\_disable\_controller\_public\_ips) | DEPRECATED: Use `enable_controller_public_ips` instead. | `bool` | `null` | no | +| [disable\_default\_mounts](#input\_disable\_default\_mounts) | DEPRECATED: Use `enable_default_mounts` instead. | `bool` | `null` | no | +| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | +| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | `true` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_controller\_public\_ips](#input\_enable\_controller\_public\_ips) | If set to true. The controller will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. | `bool` | `false` | no | +| [enable\_default\_mounts](#input\_enable\_default\_mounts) | Enable default global network storage from the controller
- /home
- /opt/apps | `bool` | `true` | no | +| [enable\_devel](#input\_enable\_devel) | DEPRECATED: `enable_devel` is always on. | `bool` | `null` | no | +| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/tools/prologs-epilogs/README.md | `bool` | `null` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | +| [enable\_slurm\_gcp\_plugins](#input\_enable\_slurm\_gcp\_plugins) | DEPRECATED: Slurm GCP plugins have been deprecated.
Instead of 'max\_hops' plugin please use the 'placement\_max\_distance' nodeset property.
Instead of 'enable\_vpmu' plugin please use 'advanced\_machine\_features.performance\_monitoring\_unit' nodeset property. | `any` | `null` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": "beta"
}
| no | +| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for controller. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | +| [login\_network\_storage](#input\_login\_network\_storage) | An array of network attached storage mounts to be configured on all login nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [login\_nodes](#input\_login\_nodes) | List of slurm login instance definitions. | 
list(object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  }))
| `[]` | no | +| [login\_startup\_script](#input\_login\_startup\_script) | Startup script used by the login VMs. | `string` | `"# no-op"` | no | +| [login\_startup\_scripts\_timeout](#input\_login\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in login\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = optional(map(string))
    mount_runner          = optional(map(string))
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | Define nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | +| [partitions](#input\_partitions) | Cluster partitions as a list. See module slurm\_partition. | 
list(object({
    partition_name        = string
    partition_conf        = optional(map(string), {})
    partition_nodeset     = optional(list(string), [])
    partition_nodeset_dyn = optional(list(string), [])
    partition_nodeset_tpu = optional(list(string), [])
    enable_job_exclusive  = optional(bool, false)
  }))
| `[]` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [region](#input\_region) | The default region to place resources in. | `string` | n/a | yes | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the controller instance. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the controller instance. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming and slurm accounting.
If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). | `string` | `null` | no | +| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | +| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | +| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Post deployment instructions. | +| [slurm\_bucket](#output\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | +| [slurm\_bucket\_dir](#output\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | +| [slurm\_bucket\_name](#output\_slurm\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | +| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | Bucket path used by cluster. | +| [slurm\_cluster\_name](#output\_slurm\_cluster\_name) | Slurm cluster name. | +| [slurm\_controller\_instance](#output\_slurm\_controller\_instance) | Compute instance of controller node | +| [slurm\_login\_instances](#output\_slurm\_login\_instances) | Compute instances of login nodes | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf new file mode 100644 index 0000000000..4a887b99cf --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf @@ -0,0 +1,213 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.disk_resource_manager_tags + } + ] + + state_disk = var.controller_state_disk != null ? [{ + source = google_compute_disk.controller_disk[0].name + device_name = google_compute_disk.controller_disk[0].name + disk_labels = null + auto_delete = false + boot = false + }] : [] + + synth_def_sa_email = "${data.google_project.controller_project.number}-compute@developer.gserviceaccount.com" + + service_account = { + email = coalesce(var.service_account_email, local.synth_def_sa_email) + scopes = var.service_account_scopes + } + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata, + local.universe_domain + ) + + controller_project_id = coalesce(var.controller_project_id, var.project_id) +} + +data "google_project" "controller_project" { + project_id = local.controller_project_id +} + +resource "google_compute_disk" "controller_disk" { + count = var.controller_state_disk != null ? 1 : 0 + + project = local.controller_project_id + name = "${local.slurm_cluster_name}-controller-save" + type = var.controller_state_disk.type + size = var.controller_state_disk.size + zone = var.zone +} + +# INSTANCE TEMPLATE +module "slurm_controller_template" { + source = "../../internal/slurm-gcp/instance_template" + + project_id = local.controller_project_id + region = var.region + slurm_instance_role = "controller" + slurm_cluster_name = local.slurm_cluster_name + labels = local.labels + + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(var.disk_labels, local.labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = concat(local.additional_disks, local.state_disk) + + bandwidth_tier = var.bandwidth_tier + slurm_bucket_path = module.slurm_files.slurm_bucket_path + can_ip_forward = var.can_ip_forward + advanced_machine_features = var.advanced_machine_features + resource_manager_tags = var.resource_manager_tags + + enable_confidential_vm = var.enable_confidential_vm + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + gpu = one(module.gpu.guest_accelerator) + + machine_type = var.machine_type + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + service_account = local.service_account + + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + subnetwork = var.subnetwork_self_link + + tags = concat([local.slurm_cluster_name], var.tags) + # termination_action = TODO: add support for termination_action (?) +} + +# INSTANCE +resource "google_compute_instance_from_template" "controller" { + provider = google-beta + + name = "${local.slurm_cluster_name}-controller" + project = local.controller_project_id + zone = var.zone + source_instance_template = module.slurm_controller_template.self_link + # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 + # we have to explicitly override instance labels instead of inheriting them from template. + labels = module.slurm_controller_template.labels + + allow_stopping_for_update = true + + # Can't rely on template to specify nics due to usage of static_ip + network_interface { + dynamic "access_config" { + for_each = var.enable_controller_public_ips ? ["unit"] : [] + content { + nat_ip = null + network_tier = null + } + } + network_ip = length(var.static_ips) == 0 ? "" : var.static_ips[0] + subnetwork = var.subnetwork_self_link + } + + dynamic "network_interface" { + for_each = var.controller_network_attachment != null ? [1] : [] + content { + network_attachment = var.controller_network_attachment + } + } +} + +moved { + from = module.slurm_controller_instance.google_compute_instance_from_template.slurm_instance[0] + to = google_compute_instance_from_template.controller +} + +# SECRETS: CLOUDSQL +resource "google_secret_manager_secret" "cloudsql" { + count = var.cloudsql != null ? 1 : 0 + + secret_id = "${local.slurm_cluster_name}-slurm-secret-cloudsql" + project = var.project_id + + replication { + dynamic "auto" { + for_each = length(var.cloudsql.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.cloudsql.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = nonsensitive(var.cloudsql.user_managed_replication) + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } + + labels = { + slurm_cluster_name = local.slurm_cluster_name + } +} + +resource "google_secret_manager_secret_version" "cloudsql_version" { + count = var.cloudsql != null ? 1 : 0 + + secret = google_secret_manager_secret.cloudsql[0].id + secret_data = jsonencode(var.cloudsql) +} + +resource "google_secret_manager_secret_iam_member" "cloudsql_secret_accessor" { + count = var.cloudsql != null ? 1 : 0 + + secret_id = google_secret_manager_secret.cloudsql[0].id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${local.service_account.email}" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl new file mode 100644 index 0000000000..219bdc5227 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl @@ -0,0 +1,65 @@ +# slurm.conf +# https://slurm.schedmd.com/high_throughput.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxArraySize=10001 +MaxJobCount=500000 +MaxNodeCount=65536 +MinJobAge=60 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +SlurmctldDebug=error +SlurmdDebug=error + +# +# +# TIMERS +MessageTimeout=60 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=120 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl new file mode 100644 index 0000000000..93ac47e341 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl @@ -0,0 +1,34 @@ +# slurmdbd.conf +# https://slurm.schedmd.com/slurmdbd.conf.html + +DebugLevel=info +PidFile=/var/run/slurm/slurmdbd.pid + +# https://slurm.schedmd.com/slurmdbd.conf.html#OPT_CommitDelay +CommitDelay=1 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +AuthType=auth/{auth_key} +AuthAltTypes=auth/jwt +AuthAltParameters=jwt_key={state_save}/jwt_hs256.key + +DbdHost={control_host} + +LogFile={slurmlog}/slurmdbd.log + +SlurmUser=slurm + +StorageLoc={db_name} + +StorageType=accounting_storage/mysql +StorageHost={db_host} +StoragePort={db_port} +StorageUser={db_user} +StoragePass={db_pass} + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl new file mode 100644 index 0000000000..d3f2615a68 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl @@ -0,0 +1,71 @@ +# slurm.conf +# https://slurm.schedmd.com/slurm.conf.html +# https://slurm.schedmd.com/configurator.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxNodeCount=64000 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +AccountingStoreFlags=job_comment +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/cgroup +SlurmctldDebug=info +SlurmdDebug=info +DebugFlags=Power + +# +# +# TIMERS +MessageTimeout=600 +BatchStartTimeout=600 +PrologEpilogTimeout=600 +PrologFlags=Contain + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=600 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf new file mode 100644 index 0000000000..21e915a125 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf @@ -0,0 +1,50 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +locals { + # TODO: deprecate `var.login_[ startup_script, startup_scripts_timeout, network_storage]` + # in favour of vars defined in user-facing login module + ghpc_startup_login = [{ + filename = "ghpc_startup.sh" + content = var.login_startup_script + }] + + login_startup_scripts = concat(local.common_scripts, local.ghpc_startup_login) +} + +module "login" { + source = "../../internal/slurm-gcp/login" + for_each = { for x in var.login_nodes : x.group_name => x } + + project_id = var.project_id + + slurm_cluster_name = local.slurm_cluster_name + slurm_bucket_path = module.slurm_files.slurm_bucket_path + slurm_bucket_name = module.slurm_files.bucket_name + slurm_bucket_dir = module.slurm_files.bucket_dir + + login_nodes = each.value + + startup_scripts = local.login_startup_scripts + startup_scripts_timeout = var.login_startup_scripts_timeout + + network_storage = var.login_network_storage + + universe_domain = var.universe_domain + + # trigger replacement of login nodes when the controller instance is replaced + # Needed for re-mounting volumes hosted on controller + replace_trigger = google_compute_instance_from_template.controller.self_link +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf new file mode 100644 index 0000000000..7622bdffef --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-controller", ghpc_role = "scheduler" }) +} + +locals { + # Since deployment name may be used to create a cluster name, we remove any invalid character from the beginning + # Also, slurm imposed a lot of restrictions to this name, so we format it to an acceptable string + tmp_cluster_name = substr(replace(lower(var.deployment_name), "/^[^a-z]*|[^a-z0-9]/", ""), 0, 10) + slurm_cluster_name = coalesce(var.slurm_cluster_name, local.tmp_cluster_name) + + universe_domain = { "universe_domain" = var.universe_domain } +} + +# See +# * slurm_files.tf +# * controller.tf +# * partition.tf +# * login.tf diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml new file mode 100644 index 0000000000..7b4918b962 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - iam.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md new file mode 100644 index 0000000000..002bf14145 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md @@ -0,0 +1,42 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name         = string
    subnetwork_self_link = string
    additional_networks = list(object({
      subnetwork = string
    }))
  })
| n/a | yes | +| [nodeset\_template](#input\_nodeset\_template) | Self link of the nodeset template | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf new file mode 100644 index 0000000000..bd8773cf84 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf @@ -0,0 +1,46 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + cleanup_dependencies_agg = flatten([ + var.nodeset.subnetwork_self_link, + var.nodeset.additional_networks[*].subnetwork, + var.nodeset_template]) +} + +# Can not use variadic list in `depends_on`, wrap it into a collection of `null_resource` +resource "null_resource" "dependencies" { + count = length(local.cleanup_dependencies_agg) +} + +resource "null_resource" "script" { + count = var.enable_cleanup_compute ? 1 : 0 + + triggers = { + project_id = var.project_id + cluster_name = var.slurm_cluster_name + nodeset_name = var.nodeset.nodeset_name + universe_domain = var.universe_domain + compute_endpoint_version = var.endpoint_versions.compute + gcloud_path_override = var.gcloud_path_override + } + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/cleanup_compute.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" + when = destroy + } + + # Ensure that clean up is done before attempt to delete the networks + depends_on = [null_resource.dependencies] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh new file mode 100644 index 0000000000..a98243d464 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +project="$1" +cluster_name="$2" +nodeset_name="$3" +universe_domain="$4" +compute_endpoint_version="$5" +gcloud_dir="$6" +MAX_ATTEMPTS=3 + +if [[ $# -ne 5 ]] && [[ $# -ne 6 ]]; then + echo "Usage: $0 []" + exit 1 +fi + +if [[ -n "${gcloud_dir}" ]]; then + export PATH="$gcloud_dir:$PATH" +fi + +export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" +export CLOUDSDK_CORE_PROJECT="${project}" + +if ! type -P gcloud 1>/dev/null; then + echo "gcloud is not available and your compute resources are not being cleaned up" + echo "https://console.cloud.google.com/compute/instances?project=${project}" + exit 1 +fi + +tmpfile=$(mktemp) # have to use a temp file, since `< <(gcloud ...)` doesn't work nicely with `head` +trap 'rm -f "$tmpfile"' EXIT + +echo "Deleting managed instance groups" +mig_filter="name:${cluster_name}-${nodeset_name}-*" +gcloud compute instance-groups managed list --format="value(self_link)" --filter="${mig_filter}" >"$tmpfile" +while batch="$(head -n 5)" && [[ ${#batch} -gt 0 ]]; do + groups=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line + # The lack of quotes around ${groups} is intentional and causes each new space-separated "word" to + # be treated as independent arguments. See PR#2523 + # shellcheck disable=SC2086 + for _ in $( #occasionally MIGs will fail to delete due to some active transformation happening, so let's retry + seq 1 $MAX_ATTEMPTS + ); do + if gcloud compute instance-groups managed delete --quiet ${groups}; then + break + fi + echo "MIG deletion failed, retrying" + done +done <"$tmpfile" +true >"$tmpfile" # Wipe contents of tmp file + +echo "Deleting compute nodes" +node_filter="name:${cluster_name}-${nodeset_name}-* labels.slurm_cluster_name=${cluster_name} AND labels.slurm_instance_role=compute" + +running_nodes_filter="${node_filter} AND status!=STOPPING" +# List all currently running instances and attempt to delete them +gcloud compute instances list --format="value(selfLink)" --filter="${running_nodes_filter}" >"$tmpfile" +# Do 500 instances at a time +while batch="$(head -n 500)" && [[ ${#batch} -gt 0 ]]; do + nodes=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line + # The lack of quotes around ${nodes} is intentional and causes each new space-separated "word" to + # be treated as independent arguments. See PR#2523 + # shellcheck disable=SC2086 + gcloud compute instances delete --quiet ${nodes} || echo "Failed to delete some instances" +done <"$tmpfile" + +# In case if controller tries to delete the nodes as well, +# wait until nodes in STOPPING state are deleted, before deleting the resource policies +stopping_nodes_filter="${node_filter} AND status=STOPPING" +while true; do + node=$(gcloud compute instances list --format="value(name)" --filter="${stopping_nodes_filter}" --limit=1) + if [[ -z "${node}" ]]; then + break + fi + echo "Waiting for instances to be deleted: ${node}" + sleep 5 +done + +echo "Deleting resource policies" +policies_filter="name:${cluster_name}-slurmgcp-managed-${nodeset_name}-*" +gcloud compute resource-policies list --format="value(selfLink)" --filter="${policies_filter}" | while read -r line; do + echo "Deleting resource policy: $line" + gcloud compute resource-policies delete --quiet "${line}" || { + echo "Failed to delete resource policy: $line" + } +done diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf new file mode 100644 index 0000000000..b6da69931c --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf @@ -0,0 +1,71 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "Project ID" +} + + +variable "slurm_cluster_name" { + type = string + description = "Name of the Slurm cluster" +} + +variable "enable_cleanup_compute" { + description = < [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | 3.2.3 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf new file mode 100644 index 0000000000..ec86a03a24 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "null_resource" "script" { + count = var.enable_cleanup_compute ? 1 : 0 + + triggers = { + project_id = var.project_id + cluster_name = var.slurm_cluster_name + nodeset_name = var.nodeset.nodeset_name + zone = var.nodeset.zone + universe_domain = var.universe_domain + compute_endpoint_version = var.endpoint_versions.compute + gcloud_path_override = var.gcloud_path_override + } + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/cleanup_tpu.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.zone} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" + when = destroy + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh new file mode 100644 index 0000000000..c724e342c3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +project="$1" +cluster_name="$2" +nodeset_name="$3" +zone="$4" +universe_domain="$5" +compute_endpoint_version="$6" +gcloud_dir="$7" + +if [[ $# -ne 6 ]] && [[ $# -ne 7 ]]; then + echo "Usage: $0 []" + exit 1 +fi + +if [[ -n "${gcloud_dir}" ]]; then + export PATH="$gcloud_dir:$PATH" +fi + +export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" +export CLOUDSDK_CORE_PROJECT="${project}" + +if ! type -P gcloud 1>/dev/null; then + echo "gcloud is not available and your compute resources are not being cleaned up" + echo "https://console.cloud.google.com/compute/instances?project=${project}" + exit 1 +fi + +echo "Deleting TPU nodes" +node_filter="name~${cluster_name}-${nodeset_name}" +running_nodes_filter="${node_filter} AND state!=DELETING" + +# List all currently running nodes and attempt to delete them +gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${running_nodes_filter}" | while read -r name; do + echo "Deleting TPU node: $name" + gcloud compute tpus tpu-vm delete --async --zone="${zone}" --quiet "${name}" || echo "Failed to delete $name" +done + +# Wait until nodes in DELETING state are deleted, before deleting the resource policies +deleting_nodes_filter="${node_filter} AND state=DELETING" +while true; do + node=$(gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${deleting_nodes_filter}" --limit=1) + if [[ -z "${node}" ]]; then + break + fi + echo "Waiting for nodes to be deleted: ${node}" + sleep 5 +done diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf new file mode 100644 index 0000000000..1ac6f64b75 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf @@ -0,0 +1,60 @@ +/** + * Copyright (C) Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "Project ID" +} + +variable "slurm_cluster_name" { + type = string + description = "Name of the Slurm cluster" +} + +variable "enable_cleanup_compute" { + description = < +Copyright (C) SchedMD LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.3 | +| [archive](#requirement\_archive) | ~> 2.0 | +| [google](#requirement\_google) | >= 6.41 | +| [local](#requirement\_local) | ~> 2.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [archive](#provider\_archive) | ~> 2.0 | +| [google](#provider\_google) | >= 6.41 | +| [local](#provider\_local) | ~> 2.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.controller_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.devel](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.devel_compute](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_dyn_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_tpu_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.task_epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.task_prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [random_uuid.cluster_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/uuid) | resource | +| [archive_file.slurm_gcp_devel_compute_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | +| [archive_file.slurm_gcp_devel_controller_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | +| [local_file.chs_gpu_health_check](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.external_epilog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.external_prolog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.setup_external](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. | `string` | `null` | no | +| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket to use. | `string` | n/a | yes | +| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | +| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Default behavior defined in scripts/conf.py | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | +| [cloudsql\_secret](#input\_cloudsql\_secret) | Secret URI to cloudsql secret. | `string` | `null` | no | +| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in compute\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | +| [controller\_startup\_scripts](#input\_controller\_startup\_scripts) | List of scripts to be ran on controller VM startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | +| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    device_name = string
  })
| 
{
  "device_name": null
}
| no | +| [disable\_default\_mounts](#input\_disable\_default\_mounts) | Disable default global network storage from the controller
- /home
- /apps | `bool` | `false` | no | +| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. Not for production use. | `bool` | `false` | no | +| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/v5/tools/prologs-epilogs/README.md | `bool` | `false` | no | +| [enable\_hybrid](#input\_enable\_hybrid) | Enables use of hybrid controller mode. When true, controller\_hybrid\_config will
be used instead of controller\_instance\_config and will disable login instances. | `bool` | `false` | no | +| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": null
}
| no | +| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | +| [google\_app\_cred\_path](#input\_google\_app\_cred\_path) | Path to Google Application Credentials. | `string` | `null` | no | +| [install\_dir](#input\_install\_dir) | Directory where the hybrid configuration directory will be installed on the
on-premise controller (e.g. /etc/slurm/hybrid). This updates the prefix path
for the resume and suspend scripts in the generated `cloud.conf` file.

This variable should be used when the TerraformHost and the SlurmctldHost
are different.

This will default to var.output\_dir if null. | `string` | `null` | no | +| [munge\_mount](#input\_munge\_mount) | Remote munge mount for compute and login nodes to acquire the munge.key.
By default, the munge mount server will be assumed to be the
`var.slurm_control_host` (or `var.slurm_control_addr` if non-null) when
`server_ip=null`. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| 
{
  "fs_type": "nfs",
  "mount_options": "",
  "remote_mount": "/etc/munge/",
  "server_ip": null
}
| no | +| [network\_storage](#input\_network\_storage) | Storage to mounted on all instances.
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | Cluster nodenets, as a list. | `list(any)` | `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Cluster nodenets (dynamic), as a list. | `list(any)` | `[]` | no | +| [nodeset\_startup\_scripts](#input\_nodeset\_startup\_scripts) | List of scripts to be ran on compute VM startup in the specific nodeset. | 
map(list(object({
    filename = string
    content  = string
  })))
| `{}` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Cluster nodenets (TPU), as a list. | `list(any)` | `[]` | no | +| [output\_dir](#input\_output\_dir) | Directory where this module will write its files to. These files include:
cloud.conf; cloud\_gres.conf; config.yaml; resume.py; suspend.py; and util.py. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The GCP project ID. | `string` | n/a | yes | +| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [slurm\_bin\_dir](#input\_slurm\_bin\_dir) | Path to directory of Slurm binary commands (e.g. scontrol, sinfo). If 'null',
then it will be assumed that binaries are in $PATH. | `string` | `null` | no | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | The cluster name, used for resource naming and slurm accounting. | `string` | n/a | yes | +| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | +| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | +| [slurm\_control\_addr](#input\_slurm\_control\_addr) | The IP address or a name by which the address can be identified.

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | +| [slurm\_control\_host](#input\_slurm\_control\_host) | The short, or long, hostname of the machine where Slurm control daemon is
executed (i.e. the name returned by the command "hostname -s").

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | +| [slurm\_control\_host\_port](#input\_slurm\_control\_host\_port) | The port number that the Slurm controller, slurmctld, listens to for work.

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldPort | `string` | `"6818"` | no | +| [slurm\_key\_mount](#input\_slurm\_key\_mount) | Remote mount for compute and login nodes to acquire the slurm.key. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| `null` | no | +| [slurm\_log\_dir](#input\_slurm\_log\_dir) | Directory where Slurm logs to. | `string` | `"/var/log/slurm"` | no | +| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | +| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [bucket\_dir](#output\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | +| [bucket\_name](#output\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | +| [config](#output\_config) | Cluster configuration. | +| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl new file mode 100644 index 0000000000..ffeb167cfc --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl @@ -0,0 +1,7 @@ +# cgroup.conf +# https://slurm.schedmd.com/cgroup.conf.html + +ConstrainCores=yes +ConstrainRamSpace=yes +ConstrainSwapSpace=no +ConstrainDevices=yes diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl new file mode 100644 index 0000000000..4951289842 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl @@ -0,0 +1,67 @@ +# slurm.conf +# https://slurm.schedmd.com/slurm.conf.html +# https://slurm.schedmd.com/configurator.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxNodeCount=64000 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +AccountingStoreFlags=job_comment +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/cgroup +SlurmctldDebug=info +SlurmdDebug=info +DebugFlags=Power + +# +# +# TIMERS +MessageTimeout=60 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=120 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl new file mode 100644 index 0000000000..8c90a9dfbe --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl @@ -0,0 +1,31 @@ +# slurmdbd.conf +# https://slurm.schedmd.com/slurmdbd.conf.html + +DebugLevel=info +PidFile=/var/run/slurm/slurmdbd.pid + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +AuthType=auth/{auth_key} +AuthAltTypes=auth/jwt +AuthAltParameters=jwt_key={state_save}/jwt_hs256.key + +DbdHost={control_host} + +LogFile={slurmlog}/slurmdbd.log + +SlurmUser=slurm + +StorageLoc={db_name} + +StorageType=accounting_storage/mysql +StorageHost={db_host} +StoragePort={db_port} +StorageUser={db_user} +StoragePass={db_pass} + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh new file mode 100644 index 0000000000..db514fc9e5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -x /opt/apps/adm/slurm/slurm_epilog ]]; then + exec /opt/apps/adm/slurm/slurm_epilog +fi diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh new file mode 100644 index 0000000000..37a91bb1ea --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -x /opt/apps/adm/slurm/slurm_prolog ]]; then + exec /opt/apps/adm/slurm/slurm_prolog +fi diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh new file mode 100644 index 0000000000..0877ff3b19 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SLURM_EXTERNAL_ROOT="/opt/apps/adm/slurm" +SLURM_MUX_FILE="slurm_mux" + +mkdir -p "${SLURM_EXTERNAL_ROOT}" +mkdir -p "${SLURM_EXTERNAL_ROOT}/logs" +mkdir -p "${SLURM_EXTERNAL_ROOT}/etc" + +# create common prolog / epilog "multiplex" script +if [ ! -f "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" ]; then + # indentation matters in EOT below; do not blindly edit! + cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CMD="${0##*/}" +# Locate script +BASE=$(readlink -f $0) +BASE=${BASE%/*} + +export CLUSTER_ADM_BASE=${BASE} + +# Source config file if it exists for extra DEBUG settings +# used below +SLURM_MUX_CONF=${CLUSTER_ADM_BASE}/etc/slurm_mux.conf +if [[ -r ${SLURM_MUX_CONF} ]]; then + source ${SLURM_MUX_CONF} +fi + +# Setup logging if configured and directory exists +LOGFILE="/dev/null" +if [[ -d ${DEBUG_SLURM_MUX_LOG_DIR} && ${DEBUG_SLURM_MUX_ENABLE_LOG} == "yes" ]]; then + LOGFILE="${DEBUG_SLURM_MUX_LOG_DIR}/${CMD}-${SLURM_SCRIPT_CONTEXT}-job-${SLURMD_NODENAME}.log" + exec >>${LOGFILE} 2>&1 +fi + +# Global scriptlets +for SCRIPTLET in ${BASE}/${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do + if [[ -x ${SCRIPTLET} ]]; then + echo "Running ${SCRIPTLET}" + ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 + echo "Running ${SCRIPTLET} returned $?" + fi +done + +# Per partition scriptlets +for SCRIPTLET in ${BASE}/partition-${SLURM_JOB_PARTITION}-${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do + if [[ -x ${SCRIPTLET} ]]; then + echo "Running ${SCRIPTLET}" + ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 + echo "Running ${SCRIPTLET} returned $?" + fi +done +EOT +fi + +# ensure proper permissions on slurm_mux script +chmod 0755 "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" + +# create default slurm_mux configuration file +if [ ! -f "${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" ]; then + cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" +# these settings are intended for temporary debugging purposes only; leaving +# them enabled will write files for each job to a shared NFS directory without +# any automated cleanup +DEBUG_SLURM_MUX_LOG_DIR=/opt/apps/adm/slurm/logs +DEBUG_SLURM_MUX_ENABLE_LOG=no +EOT +fi + +# create epilog symbolic link +if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_epilog" ]; then + cd ${SLURM_EXTERNAL_ROOT} + # delete existing file if necessary + rm -f slurm_epilog + ln -s ${SLURM_MUX_FILE} slurm_epilog + cd - >/dev/null +fi + +# create prolog symbolic link +if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_prolog" ]; then + cd ${SLURM_EXTERNAL_ROOT} + # delete existing file if necessary + rm -f slurm_prolog + ln -s ${SLURM_MUX_FILE} slurm_prolog + cd - >/dev/null +fi diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf new file mode 100644 index 0000000000..e63b2d1100 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf @@ -0,0 +1,406 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + scripts_dir = abspath("${path.module}/scripts") + + bucket_dir = coalesce(var.bucket_dir, format("%s-files", var.slurm_cluster_name)) +} + +######## +# DATA # +######## + +data "google_storage_bucket" "this" { + name = var.bucket_name +} + +########## +# RANDOM # +########## + +resource "random_uuid" "cluster_id" { +} + +################## +# CLUSTER CONFIG # +################## + +locals { + config = { + enable_bigquery_load = var.enable_bigquery_load + cloudsql_secret = var.cloudsql_secret + cluster_id = random_uuid.cluster_id.result + project = var.project_id + slurm_cluster_name = var.slurm_cluster_name + enable_slurm_auth = var.enable_slurm_auth + bucket_path = local.bucket_path + enable_debug_logging = var.enable_debug_logging + extra_logging_flags = var.extra_logging_flags + controller_state_disk = var.controller_state_disk + + # storage + disable_default_mounts = var.disable_default_mounts + network_storage = var.network_storage + + # timeouts + controller_startup_scripts_timeout = var.controller_startup_scripts_timeout + compute_startup_scripts_timeout = var.compute_startup_scripts_timeout + + munge_mount = local.munge_mount + slurm_key_mount = var.slurm_key_mount + + # slurm conf + prolog_scripts = [for k, v in google_storage_bucket_object.prolog_scripts : k] + epilog_scripts = [for k, v in google_storage_bucket_object.epilog_scripts : k] + task_prolog_scripts = [for k, v in google_storage_bucket_object.task_prolog_scripts : k] + task_epilog_scripts = [for k, v in google_storage_bucket_object.task_epilog_scripts : k] + cloud_parameters = var.cloud_parameters + + # hybrid + hybrid = var.enable_hybrid + google_app_cred_path = var.enable_hybrid ? local.google_app_cred_path : null + output_dir = var.enable_hybrid ? local.output_dir : null + install_dir = var.enable_hybrid ? local.install_dir : null + slurm_control_host = var.enable_hybrid ? var.slurm_control_host : null + slurm_control_host_port = var.enable_hybrid ? local.slurm_control_host_port : null + slurm_control_addr = var.enable_hybrid ? var.slurm_control_addr : null + slurm_bin_dir = var.enable_hybrid ? local.slurm_bin_dir : null + slurm_log_dir = var.enable_hybrid ? local.slurm_log_dir : null + controller_network_attachment = var.controller_network_attachment + + + # config files templates + slurmdbd_conf_tpl = file(coalesce(var.slurmdbd_conf_tpl, "${local.etc_dir}/slurmdbd.conf.tpl")) + slurm_conf_tpl = var.slurm_conf_template != null ? var.slurm_conf_template : file(coalesce(var.slurm_conf_tpl, "${local.etc_dir}/slurm.conf.tpl")) + cgroup_conf_tpl = file(coalesce(var.cgroup_conf_tpl, "${local.etc_dir}/cgroup.conf.tpl")) + + # Providers + endpoint_versions = var.endpoint_versions + } + + x_nodeset = toset(var.nodeset[*].nodeset_name) + x_nodeset_dyn = toset(var.nodeset_dyn[*].nodeset_name) + x_nodeset_tpu = toset(var.nodeset_tpu[*].nodeset.nodeset_name) + x_nodeset_overlap = setintersection([], local.x_nodeset, local.x_nodeset_dyn, local.x_nodeset_tpu) + + etc_dir = abspath("${path.module}/etc") + + bucket_path = format("%s/%s", data.google_storage_bucket.this.url, local.bucket_dir) + + slurm_control_host_port = coalesce(var.slurm_control_host_port, "6818") + + google_app_cred_path = var.google_app_cred_path != null ? abspath(var.google_app_cred_path) : null + slurm_bin_dir = var.slurm_bin_dir != null ? abspath(var.slurm_bin_dir) : null + slurm_log_dir = var.slurm_log_dir != null ? abspath(var.slurm_log_dir) : null + + munge_mount = var.enable_hybrid ? { + server_ip = lookup(var.munge_mount, "server_ip", coalesce(var.slurm_control_addr, var.slurm_control_host)) + remote_mount = lookup(var.munge_mount, "remote_mount", "/etc/munge/") + fs_type = lookup(var.munge_mount, "fs_type", "nfs") + mount_options = lookup(var.munge_mount, "mount_options", "") + } : null + + output_dir = can(coalesce(var.output_dir)) ? abspath(var.output_dir) : abspath(".") + install_dir = can(coalesce(var.install_dir)) ? abspath(var.install_dir) : local.output_dir +} + +resource "google_storage_bucket_object" "config" { + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/config.yaml" + content = yamlencode(local.config) + source_md5hash = md5(yamlencode(local.config)) + + # Take dependency on all other "config artifacts" so creation of `config.yaml` + # can be used as a signal for setup.py that "everything is ready". + # Some of following files, particularly mount scripts for new NFSes, can take a while to be created. + depends_on = [ + google_storage_bucket_object.controller_startup_scripts, + google_storage_bucket_object.nodeset_startup_scripts, + google_storage_bucket_object.prolog_scripts, + google_storage_bucket_object.epilog_scripts, + google_storage_bucket_object.task_prolog_scripts, + google_storage_bucket_object.task_epilog_scripts + ] +} + +resource "google_storage_bucket_object" "nodeset_config" { + for_each = { for ns in var.nodeset : ns.nodeset_name => merge(ns, { + instance_properties = jsondecode(ns.instance_properties_json) + }) } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +resource "google_storage_bucket_object" "nodeset_dyn_config" { + for_each = { for ns in var.nodeset_dyn : ns.nodeset_name => ns } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_dyn_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +resource "google_storage_bucket_object" "nodeset_tpu_config" { + for_each = { for n in var.nodeset_tpu[*].nodeset : n.nodeset_name => n } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_tpu_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +######### +# DEVEL # +######### + +locals { + build_dir = abspath("${path.module}/build") + + slurm_gcp_devel_controller_zip = "slurm-gcp-devel-controller.zip" + slurm_gcp_devel_compute_zip = "slurm-gcp-devel.zip" + slurm_gcp_devel_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_controller_zip) + slurm_gcp_devel_compute_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_compute_zip) + + controller_files = [ + "tools/gpu-test", + "tools/task-epilog", + "tools/task-prolog", + "conf.py", + "file_cache.py", + "get_tpu_vmcount.py", + "job_submit.lua.tpl", + "load_bq.py", + "local_pubsub.py", + "mig_flex.py", + "resume_wrapper.sh", + "resume.py", + "setup_network_storage.py", + "setup.py", + "slurmsync.py", + "sort_nodes.py", + "suspend_wrapper.sh", + "suspend.py", + "tpu.py", + "util.py", + "watch_delete_vm_op.py", + ] + + compute_files = [ + "tools/gpu-test", + "tools/task-epilog", + "tools/task-prolog", + "file_cache.py", + "get_tpu_vmcount.py", + "job_submit.lua.tpl", + "local_pubsub.py", + "mig_flex.py", + "setup_network_storage.py", + "setup.py", + "slurmsync.py", + "sort_nodes.py", + "suspend.py", + "tpu.py", + "util.py", + "watch_delete_vm_op.py", + ] +} + +data "archive_file" "slurm_gcp_devel_controller_zip" { + output_path = "${local.build_dir}/${local.slurm_gcp_devel_controller_zip}" + type = "zip" + + dynamic "source" { + for_each = local.controller_files + content { + content = file("${local.scripts_dir}/${source.value}") + filename = source.value + } + } +} + +data "archive_file" "slurm_gcp_devel_compute_zip" { + output_path = "${local.build_dir}/${local.slurm_gcp_devel_compute_zip}" + type = "zip" + + dynamic "source" { + for_each = local.compute_files + content { + content = file("${local.scripts_dir}/${source.value}") + filename = source.value + } + } +} + +resource "google_storage_bucket_object" "devel" { + bucket = var.bucket_name + name = local.slurm_gcp_devel_zip_bucket + source = data.archive_file.slurm_gcp_devel_controller_zip.output_path + source_md5hash = data.archive_file.slurm_gcp_devel_controller_zip.output_md5 +} + +resource "google_storage_bucket_object" "devel_compute" { + bucket = var.bucket_name + name = local.slurm_gcp_devel_compute_zip_bucket + source = data.archive_file.slurm_gcp_devel_compute_zip.output_path + source_md5hash = data.archive_file.slurm_gcp_devel_compute_zip.output_md5 +} + +########### +# SCRIPTS # +########### + +resource "google_storage_bucket_object" "controller_startup_scripts" { + for_each = { + for x in local.controller_startup_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-controller-script-%s", local.bucket_dir, each.key) + content = each.value.content + source_md5hash = md5(each.value.content) +} + +resource "google_storage_bucket_object" "nodeset_startup_scripts" { + for_each = { for x in flatten([ + for nodeset, scripts in var.nodeset_startup_scripts + : [for s in scripts + : { + content = s.content, + name = format("slurm-nodeset-%s-script-%s", nodeset, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_")) } + ]]) : x.name => x.content } + + bucket = var.bucket_name + name = format("%s/%s", local.bucket_dir, each.key) + content = each.value + source_md5hash = md5(each.value) +} + +resource "google_storage_bucket_object" "prolog_scripts" { + for_each = { + for x in local.prolog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-prolog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +resource "google_storage_bucket_object" "epilog_scripts" { + for_each = { + for x in local.epilog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-epilog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +resource "google_storage_bucket_object" "task_prolog_scripts" { + for_each = { + for x in local.task_prolog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-task_prolog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +resource "google_storage_bucket_object" "task_epilog_scripts" { + for_each = { + for x in local.task_epilog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-task_epilog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) +} + +############################ +# DATA: CHS GPU HEALTH CHECK +############################ + +data "local_file" "chs_gpu_health_check" { + filename = "${path.module}/scripts/tools/gpu-test" +} + +################################ +# DATA: EXTERNAL PROLOG/EPILOG # +################################ + +data "local_file" "external_epilog" { + filename = "${path.module}/files/external_epilog.sh" +} + +data "local_file" "external_prolog" { + filename = "${path.module}/files/external_prolog.sh" +} + +data "local_file" "setup_external" { + filename = "${path.module}/files/setup_external.sh" +} + +locals { + external_epilog = [{ + filename = "z_external_epilog.sh" + content = data.local_file.external_epilog.content + source = null + }] + external_prolog = [{ + filename = "z_external_prolog.sh" + content = data.local_file.external_prolog.content + source = null + }] + setup_external = [{ + filename = "z_setup_external.sh" + content = data.local_file.setup_external.content + }] + chs_gpu_health_check = [{ + filename = "a_chs_gpu_health_check.sh" + content = data.local_file.chs_gpu_health_check.content + source = null + }] + + chs_prolog = var.enable_chs_gpu_health_check_prolog ? local.chs_gpu_health_check : [] + ext_prolog = var.enable_external_prolog_epilog ? local.external_prolog : [] + prolog_scripts = concat(local.chs_prolog, local.ext_prolog, var.prolog_scripts) + task_prolog_scripts = var.task_prolog_scripts + + chs_epilog = var.enable_chs_gpu_health_check_epilog ? local.chs_gpu_health_check : [] + ext_epilog = var.enable_external_prolog_epilog ? local.external_epilog : [] + epilog_scripts = concat(local.chs_epilog, local.ext_epilog, var.epilog_scripts) + task_epilog_scripts = var.task_epilog_scripts + + controller_startup_scripts = var.enable_external_prolog_epilog ? concat(local.setup_external, var.controller_startup_scripts) : var.controller_startup_scripts + + +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf new file mode 100644 index 0000000000..111c997d62 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "slurm_bucket_path" { + description = "GCS Bucket URI of Slurm cluster file storage." + value = local.bucket_path +} + +output "bucket_name" { + description = "GCS Bucket name of Slurm cluster file storage." + value = data.google_storage_bucket.this.name +} + +output "bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + value = local.bucket_dir +} + +output "config" { + description = "Cluster configuration." + value = local.config + + precondition { + condition = var.enable_hybrid ? can(coalesce(var.slurm_control_host)) : true + error_message = "Input slurm_control_host is required." + } + + precondition { + condition = length(local.x_nodeset_overlap) == 0 + error_message = "All nodeset names must be unique among all nodeset types." + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py new file mode 100644 index 0000000000..89ceefa3df --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py @@ -0,0 +1,658 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Iterable, Dict, Set, Tuple +from itertools import chain +from collections import defaultdict +import json +from pathlib import Path +import util +from util import dirs, slurmdirs +import tpu +from addict import Dict as NSDict # type: ignore + +FILE_PREAMBLE = """ +# Warning: +# This file is managed by a script. Manual modifications will be overwritten. +""" + + + +def dict_to_conf(conf, delim=" ") -> str: + """convert dict to delimited slurm-style key-value pairs""" + + def filter_conf(pair): + k, v = pair + if isinstance(v, list): + v = ",".join(str(el) for el in v if el is not None) + return k, (v if bool(v) or v == 0 else None) + + return delim.join( + f"{k}={v}" for k, v in map(filter_conf, conf.items()) if v is not None + ) + + +TOPOLOGY_PLUGIN_TREE = "topology/tree" + +def topology_plugin(lkp: util.Lookup) -> str: + """ + Returns configured topology plugin, defaults to `topology/tree`. + """ + cp, key = lkp.cfg.cloud_parameters, "topology_plugin" + if key not in cp or cp[key] is None: + return TOPOLOGY_PLUGIN_TREE + return cp[key] + +def conflines(lkp: util.Lookup) -> str: + params = lkp.cfg.cloud_parameters + def get(key, default): + """ + Returns the value of the key in params if it exists and is not None, + otherwise returns supplied default. + We can't rely on the `dict.get` method because the value could be `None` as + well as empty NSDict, depending on type of the `cfg.cloud_parameters`. + TODO: Simplify once NSDict is removed from the codebase. + """ + if key not in params or params[key] is None: + return default + return params[key] + + no_comma_params = get("no_comma_params", False) + + any_gpus = any( + lkp.template_info(nodeset.instance_template).gpu + for nodeset in lkp.cfg.nodeset.values() + ) + + any_tpu = any( + tpu_nodeset is not None + for part in lkp.cfg.partitions.values() + for tpu_nodeset in part.partition_nodeset_tpu + ) + + any_gke = any( + lkp.nodeset_is_gke(nodeset) + for nodeset in lkp.cfg.nodeset.values() + ) + + any_dynamic = any(bool(p.partition_feature) for p in lkp.cfg.partitions.values()) + comma_params = { + "LaunchParameters": [ + "enable_nss_slurm", + "use_interactive_step", + ], + "SlurmctldParameters": [ + "cloud_reg_addrs" if any_dynamic or any_tpu or any_gke else "cloud_dns", + "enable_configless", + "idle_on_node_suspend", + ], + "GresTypes": [ + "gpu" if any_gpus else None, + ], + } + + scripts_dir = lkp.cfg.install_dir or dirs.scripts + prolog_path = Path(dirs.custom_scripts / "prolog.d") + epilog_path = Path(dirs.custom_scripts / "epilog.d") + task_prolog_path = Path(dirs.custom_scripts / "task_prolog.d") + task_epilog_path = Path(dirs.custom_scripts / "task_epilog.d") + default_tree_width = 65533 if any_dynamic else 128 + + conf_options = { + **(comma_params if not no_comma_params else {}), + "Prolog": f"{prolog_path}/*" if lkp.cfg.prolog_scripts else None, + "Epilog": f"{epilog_path}/*" if lkp.cfg.epilog_scripts else None, + "TaskProlog": f"{task_prolog_path}/task-prolog" if lkp.cfg.task_prolog_scripts else None, + "TaskEpilog": f"{task_epilog_path}/task-epilog" if lkp.cfg.task_epilog_scripts else None, + "PrologFlags": get("prolog_flags", None), + "SwitchType": get("switch_type", None), + "PrivateData": get("private_data", []), + "SchedulerParameters": get("scheduler_parameters", [ + "bf_continue", + "salloc_wait_nodes", + "ignore_prefer_validation", + ]), + "ResumeProgram": f"{scripts_dir}/resume_wrapper.sh", + "ResumeFailProgram": f"{scripts_dir}/suspend_wrapper.sh", + "ResumeRate": get("resume_rate", 0), + "ResumeTimeout": get("resume_timeout", 300), + "SuspendProgram": f"{scripts_dir}/suspend_wrapper.sh", + "SuspendRate": get("suspend_rate", 0), + "SuspendTimeout": get("suspend_timeout", 300), + "SlurmdTimeout": get("slurmd_timeout", 300), + "UnkillableStepTimeout": get("unkillable_step_timeout", 300), + "TreeWidth": get("tree_width", default_tree_width), + "JobSubmitPlugins": "lua" if any_tpu else None, + "TopologyPlugin": topology_plugin(lkp), + "TopologyParam": get("topology_param", "SwitchAsNodeRank"), + } + return dict_to_conf(conf_options, delim="\n") + + + + +def nodeset_lines(nodeset, lkp: util.Lookup) -> str: + template_info = lkp.template_info(nodeset.instance_template) + machine_conf = lkp.template_machine_conf(nodeset.instance_template) + + # follow https://slurm.schedmd.com/slurm.conf.html#OPT_Boards + # by setting Boards, SocketsPerBoard, CoresPerSocket, and ThreadsPerCore + gres = f"gpu:{template_info.gpu.count}" if template_info.gpu else None + node_conf = { + "RealMemory": machine_conf.memory, + "Boards": machine_conf.boards, + "SocketsPerBoard": machine_conf.sockets_per_board, + "CoresPerSocket": machine_conf.cores_per_socket, + "ThreadsPerCore": machine_conf.threads_per_core, + "CPUs": machine_conf.cpus, + "Gres": gres, + **nodeset.node_conf, + } + nodelist = lkp.nodelist(nodeset) + + return "\n".join( + map( + dict_to_conf, + [ + {"NodeName": nodelist, "State": "CLOUD", **node_conf}, + {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, + ], + ) + ) + + +def nodeset_tpu_lines(nodeset, lkp: util.Lookup) -> str: + nodelist = lkp.nodelist(nodeset) + return "\n".join( + map( + dict_to_conf, + [ + {"NodeName": nodelist, "State": "CLOUD", **nodeset.node_conf}, + {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, + ], + ) + ) + + +def nodeset_dyn_lines(nodeset): + """generate slurm NodeSet definition for dynamic nodeset""" + return dict_to_conf( + {"NodeSet": nodeset.nodeset_name, "Feature": nodeset.nodeset_feature} + ) + + +def partitionlines(partition, lkp: util.Lookup) -> str: + """Make a partition line for the slurm.conf""" + MIN_MEM_PER_CPU = 100 + + def defmempercpu(nodeset_name: str) -> int: + nodeset = lkp.cfg.nodeset.get(nodeset_name) + template = nodeset.instance_template + machine = lkp.template_machine_conf(template) + mem_spec_limit = int(nodeset.node_conf.get("MemSpecLimit", 0)) + return max(MIN_MEM_PER_CPU, (machine.memory - mem_spec_limit) // machine.cpus) + + defmem = min( + map(defmempercpu, partition.partition_nodeset), default=MIN_MEM_PER_CPU + ) + + nodesets = list( + chain( + partition.partition_nodeset, + partition.partition_nodeset_dyn, + partition.partition_nodeset_tpu, + ) + ) + + is_tpu = len(partition.partition_nodeset_tpu) > 0 + is_dyn = len(partition.partition_nodeset_dyn) > 0 + + oversub_exlusive = partition.enable_job_exclusive or is_tpu + power_down_on_idle = partition.enable_job_exclusive and not is_dyn + + line_elements = { + "PartitionName": partition.partition_name, + "Nodes": ",".join(nodesets), + "State": "UP", + "DefMemPerCPU": defmem, + "SuspendTime": 300, + "Oversubscribe": "Exclusive" if oversub_exlusive else None, + "PowerDownOnIdle": "YES" if power_down_on_idle else None, + **partition.partition_conf, + } + + return dict_to_conf(line_elements) + + +def suspend_exc_lines(lkp: util.Lookup) -> Iterable[str]: + static_nodelists = [] + for ns in lkp.power_managed_nodesets(): + if ns.node_count_static: + nodelist = lkp.nodelist_range(ns.nodeset_name, 0, ns.node_count_static) + static_nodelists.append(nodelist) + suspend_exc_nodes = {"SuspendExcNodes": static_nodelists} + + dyn_parts = [ + p.partition_name + for p in lkp.cfg.partitions.values() + if len(p.partition_nodeset_dyn) > 0 + ] + suspend_exc_parts = {"SuspendExcParts": [*dyn_parts]} + + return filter( + None, + [ + dict_to_conf(suspend_exc_nodes) if static_nodelists else None, + dict_to_conf(suspend_exc_parts), + ], + ) + + +def make_cloud_conf(lkp: util.Lookup) -> str: + """generate cloud.conf snippet""" + lines = [ + FILE_PREAMBLE, + conflines(lkp), + *(nodeset_lines(n, lkp) for n in lkp.cfg.nodeset.values()), + *(nodeset_dyn_lines(n) for n in lkp.cfg.nodeset_dyn.values()), + *(nodeset_tpu_lines(n, lkp) for n in lkp.cfg.nodeset_tpu.values()), + *(partitionlines(p, lkp) for p in lkp.cfg.partitions.values()), + *(suspend_exc_lines(lkp)), + ] + return "\n\n".join(filter(None, lines)) + + +def gen_cloud_conf(lkp: util.Lookup) -> None: + content = make_cloud_conf(lkp) + + conf_file = lkp.etc_dir / "cloud.conf" + conf_file.write_text(content) + util.chown_slurm(conf_file, mode=0o644) + + +def install_slurm_conf(lkp: util.Lookup) -> None: + """install slurm.conf""" + if lkp.cfg.ompi_version: + mpi_default = "pmi2" + else: + mpi_default = "none" + + conf_options = { + "name": lkp.cfg.slurm_cluster_name, + "control_addr": lkp.control_addr if lkp.control_addr else lkp.hostname_fqdn, + "control_host": lkp.control_host, + "accounting_storage_host": lkp.control_addr if lkp.cfg.controller_network_attachment else lkp.control_host, + "control_host_port": lkp.control_host_port, + "scripts": dirs.scripts, + "slurmlog": dirs.log, + "state_save": slurmdirs.state, + "mpi_default": mpi_default, + "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", + } + + conf = lkp.cfg.slurm_conf_tpl.format(**conf_options) + + conf_file = lkp.etc_dir / "slurm.conf" + conf_file.write_text(conf) + util.chown_slurm(conf_file, mode=0o644) + + +def install_slurmdbd_conf(lkp: util.Lookup) -> None: + """install slurmdbd.conf""" + conf_options = { + "control_host": lkp.control_host, + "slurmlog": dirs.log, + "state_save": slurmdirs.state, + "db_name": "slurm_acct_db", + "db_user": "slurm", + "db_pass": '""', + "db_host": "localhost", + "db_port": "3306", + "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", + } + + if lkp.cfg.cloudsql_secret: + secret_name = f"{lkp.cfg.slurm_cluster_name}-slurm-secret-cloudsql" + payload = json.loads(util.access_secret_version(lkp.project, secret_name)) + + if payload["db_name"] and payload["db_name"] != "": + conf_options["db_name"] = payload["db_name"] + if payload["user"] and payload["user"] != "": + conf_options["db_user"] = payload["user"] + if payload["password"] and payload["password"] != "": + conf_options["db_pass"] = payload["password"] + + db_host_str = payload["server_ip"].split(":") + if db_host_str[0]: + conf_options["db_host"] = db_host_str[0] + conf_options["db_port"] = ( + db_host_str[1] if len(db_host_str) >= 2 else "3306" + ) + + conf = lkp.cfg.slurmdbd_conf_tpl.format(**conf_options) + + conf_file = lkp.etc_dir / "slurmdbd.conf" + conf_file.write_text(conf) + util.chown_slurm(conf_file, 0o600) + + +def install_cgroup_conf(lkp: util.Lookup) -> None: + """install cgroup.conf""" + conf_file = lkp.etc_dir / "cgroup.conf" + conf_file.write_text(lkp.cfg.cgroup_conf_tpl) + util.chown_slurm(conf_file, mode=0o600) + + +def install_jobsubmit_lua(lkp: util.Lookup) -> None: + """install job_submit.lua if there are tpu nodes in the cluster""" + if not any( + tpu_nodeset is not None + for part in lkp.cfg.partitions.values() + for tpu_nodeset in part.partition_nodeset_tpu + ): + return # No TPU partitions, no need for job_submit.lua + + scripts_dir = lkp.cfg.slurm_scripts_dir or dirs.scripts + tpl = (scripts_dir / "job_submit.lua.tpl").read_text() + conf = tpl.format(scripts_dir=scripts_dir) + + conf_file = lkp.etc_dir / "job_submit.lua" + conf_file.write_text(conf) + util.chown_slurm(conf_file, 0o600) + + +def gen_cloud_gres_conf_lines(lkp: util.Lookup) -> str: + """generate cloud_gres.conf's content""" + + gpu_nodes = defaultdict(list) + for nodeset in lkp.cfg.nodeset.values(): + ti = lkp.template_info(nodeset.instance_template) + gpu_count = ti.gpu.count if ti.gpu else 0 + gpu_type = ti.gpu.type if ti.gpu else None + if gpu_count: + gpu_nodes[(gpu_count, gpu_type)].append(lkp.nodelist(nodeset)) + + lines = [ + dict_to_conf( + { + "NodeName": names, + "Name": "gpu", + "Type": gpu_type, + "File": "/dev/nvidia{}".format(f"[0-{gpu_count-1}]" if gpu_count > 1 else "0"), + } + ) + for (gpu_count, gpu_type), names in gpu_nodes.items() + ] + lines.append("\n") + return "\n".join(lines) + + +def gen_cloud_gres_conf(lkp: util.Lookup) -> None: + """create cloud_gres.conf file""" + + content = FILE_PREAMBLE + gen_cloud_gres_conf_lines(lkp) + + conf_file = lkp.etc_dir / "cloud_gres.conf" + conf_file.write_text(content) + util.chown_slurm(conf_file, mode=0o600) + + +def install_gres_conf(lkp: util.Lookup) -> None: + conf_file = lkp.etc_dir / "cloud_gres.conf" + gres_conf = lkp.etc_dir / "gres.conf" + if not gres_conf.exists(): + gres_conf.symlink_to(conf_file) + util.chown_slurm(gres_conf, mode=0o600) + + +class Switch: + """ + Represents a switch in the topology.conf file. + NOTE: It's class user job to make sure that there is no leaf-less Switches in the tree + """ + + def __init__( + self, + name: str, + nodes: Optional[Iterable[str]] = None, + switches: Optional[Dict[str, "Switch"]] = None, + ): + self.name = name + self.nodes = nodes or [] + self.switches = switches or {} + + def conf_line(self) -> str: + d = {"SwitchName": self.name} + if self.nodes: + d["Nodes"] = util.to_hostlist(self.nodes) + if self.switches: + d["Switches"] = util.to_hostlist(self.switches.keys()) + return dict_to_conf(d) + + def render_conf_lines(self) -> Iterable[str]: + yield self.conf_line() + for s in sorted(self.switches.values(), key=lambda s: s.name): + yield from s.render_conf_lines() + +class TopologySummary: + """ + Represents a summary of the topology, to make judgements about changes. + To be stored in JSON file along side of topology.conf to simplify parsing. + """ + def __init__( + self, + physical_host: Optional[Dict[str, str]] = None, + down_nodes: Optional[Iterable[str]] = None, + tpu_nodes: Optional[Iterable[str]] = None, + ) -> None: + self.physical_host = physical_host or {} + self.down_nodes = set(down_nodes or []) + self.tpu_nodes = set(tpu_nodes or []) + + + @classmethod + def path(cls, lkp: util.Lookup) -> Path: + return lkp.etc_dir / "cloud_topology.summary.json" + + @classmethod + def loads(cls, s: str) -> "TopologySummary": + d = json.loads(s) + return cls( + physical_host=d.get("physical_host"), + down_nodes=d.get("down_nodes"), + tpu_nodes=d.get("tpu_nodes"), + ) + + @classmethod + def load(cls, lkp: util.Lookup) -> "TopologySummary": + p = cls.path(lkp) + if not p.exists(): + return cls() # Return empty instance + return cls.loads(p.read_text()) + + def dumps(self) -> str: + return json.dumps( + { + "physical_host": self.physical_host, + "down_nodes": list(self.down_nodes), + "tpu_nodes": list(self.tpu_nodes), + }, + indent=2) + + def dump(self, lkp: util.Lookup) -> None: + TopologySummary.path(lkp).write_text(self.dumps()) + + def _nodenames(self) -> Set[str]: + return set(self.physical_host) | self.down_nodes | self.tpu_nodes + + def requires_reconfigure(self, prev: "TopologySummary") -> bool: + """ + Reconfigure IFF one of the following occurs: + * A node is added + * A node get a non-empty physicalHost + """ + if len(self._nodenames() - prev._nodenames()) > 0: + return True + for n, ph in self.physical_host.items(): + if ph and ph != prev.physical_host.get(n): + return True + return False + +class TopologyBuilder: + def __init__(self) -> None: + self._r = Switch("") # fake root, not part of the tree + self.summary = TopologySummary() + + def add(self, path: List[str], nodes: Iterable[str]) -> None: + n = self._r + assert path + for p in path: + n = n.switches.setdefault(p, Switch(p)) + n.nodes = [*n.nodes, *nodes] + + def render_conf_lines(self) -> Iterable[str]: + if not self._r.switches: + return [] # type: ignore + for s in sorted(self._r.switches.values(), key=lambda s: s.name): + yield from s.render_conf_lines() + + def compress(self) -> "TopologyBuilder": + compressed = TopologyBuilder() + compressed.summary = self.summary + def _walk( + u: Switch, c: Switch + ): # u: uncompressed node, c: its counterpart in compressed tree + pref = f"{c.name}_" if c != compressed._r else "s" + for i, us in enumerate(sorted(u.switches.values(), key=lambda s: s.name)): + cs = Switch(f"{pref}{i}", nodes=us.nodes) + c.switches[cs.name] = cs + _walk(us, cs) + + _walk(self._r, compressed._r) + return compressed + + +def add_tpu_nodeset_topology(nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup): + tpuobj = tpu.TPU.make(nodeset.nodeset_name, lkp) + static, dynamic = lkp.nodenames(nodeset) + + pref = ["tpu-root", f"ns_{nodeset.nodeset_name}"] + if tpuobj.vmcount == 1: # Put all nodes in one switch + all_nodes = list(chain(static, dynamic)) + bldr.add(pref, all_nodes) + bldr.summary.tpu_nodes.update(all_nodes) + return + + # Chunk nodes into sub-switches of size `vmcount` + chunk_num = 0 + for nodenames in (static, dynamic): + for nodeschunk in util.chunked(nodenames, n=tpuobj.vmcount): + chunk_name = f"{nodeset.nodeset_name}-{chunk_num}" + chunk_num += 1 + bldr.add([*pref, chunk_name], nodeschunk) + bldr.summary.tpu_nodes.update(nodeschunk) + +_SLURM_TOPO_ROOT = "slurm-root" + +def _make_physical_path(physical_host: str) -> List[str]: + assert physical_host.startswith("/"), f"Unexpected physicalHost: {physical_host}" + parts = physical_host[1:].split("/") + # Due to issues with Slurm's topology plugin, we can not use all components of `physicalHost`, + # trim it down to `cluster/rack`. + short_path = parts[:2] + return [_SLURM_TOPO_ROOT, *short_path] + +def add_nodeset_topology( + nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup +) -> None: + up_nodes = set() + default_path = [_SLURM_TOPO_ROOT, f"ns_{nodeset.nodeset_name}"] + + for inst in lkp.instances().values(): + try: + if lkp.node_nodeset_name(inst.name) != nodeset.nodeset_name: + continue + except Exception: + continue + + phys_host = inst.resource_status.physical_host or "" + bldr.summary.physical_host[inst.name] = phys_host + up_nodes.add(inst.name) + + if phys_host: + bldr.add(_make_physical_path(phys_host), [inst.name]) + else: + bldr.add(default_path, [inst.name]) + + down_nodes = [] + for node in chain(*lkp.nodenames(nodeset)): + if node not in up_nodes: + down_nodes.append(node) + if down_nodes: + bldr.add(default_path, down_nodes) + bldr.summary.down_nodes.update(down_nodes) + +def gen_topology(lkp: util.Lookup) -> TopologyBuilder: + bldr = TopologyBuilder() + for ns in lkp.cfg.nodeset_tpu.values(): + add_tpu_nodeset_topology(ns, bldr, lkp) + for ns in lkp.cfg.nodeset.values(): + add_nodeset_topology(ns, bldr, lkp) + return bldr + +def gen_topology_conf(lkp: util.Lookup) -> Tuple[bool, TopologySummary]: + """ + Generates slurm topology.conf. + Returns whether the topology.conf got updated. + """ + topo = gen_topology(lkp).compress() + conf_file = lkp.etc_dir / "cloud_topology.conf" + + with open(conf_file, "w") as f: + f.writelines(FILE_PREAMBLE + "\n") + for line in topo.render_conf_lines(): + f.write(line) + f.write("\n") + f.write("\n") + + prev_summary = TopologySummary.load(lkp) + return topo.summary.requires_reconfigure(prev_summary), topo.summary + +def install_topology_conf(lkp: util.Lookup) -> None: + conf_file = lkp.etc_dir / "cloud_topology.conf" + summary_file = lkp.etc_dir / "cloud_topology.summary.json" + topo_conf = lkp.etc_dir / "topology.conf" + + if not topo_conf.exists(): + topo_conf.symlink_to(conf_file) + + util.chown_slurm(conf_file, mode=0o600) + util.chown_slurm(summary_file, mode=0o600) + + +def gen_controller_configs(lkp: util.Lookup) -> None: + install_slurm_conf(lkp) + install_slurmdbd_conf(lkp) + gen_cloud_conf(lkp) + gen_cloud_gres_conf(lkp) + install_gres_conf(lkp) + install_cgroup_conf(lkp) + install_jobsubmit_lua(lkp) + + if topology_plugin(lkp) == TOPOLOGY_PLUGIN_TREE: + _, summary = gen_topology_conf(lkp) + summary.dump(lkp) + install_topology_conf(lkp) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py new file mode 100644 index 0000000000..cd2e41e5af --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py @@ -0,0 +1,80 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any +from pathlib import Path +import shutil +import pickle + +import logging +log = logging.getLogger() + +# Can't reuse tool from util.py to avoid circular dependencies +# TODO: break down util.py for better modularity. +def _chown_slurm(path: Path) -> None: + shutil.chown(path, user="slurm", group="slurm") + +class FileCache: + def __init__(self, path: Path): + self.path = path + + def get(self, key: str) -> Any | None: + p = self.path / key + if not p.exists(): + return None + + try: + with p.open("rb") as f: + return pickle.load(f) + + except Exception as e: + log.warning(f"Failed to read cached value at {p}: {e}") + return None + + def set(self, key: str, data: Any) -> None: + p = self.path / key + + try: + # Create & chown before writing to minimize chances + # of ending up with root-owned corrupted file that can't be cleaned up + # TODO: restrict usage of cache by root to avoid all this complexity + # or have a cache per user. + p.touch(exist_ok=True) + _chown_slurm(p) + with p.open("wb") as f: + pickle.dump(data, f) + + except Exception as e: + log.warning(f"Failed to write cached value at {p}: {e}") + + +class NoCache: + def get(self, key: str) -> Any: + log.warning("No cache used") + return None + + def set(self, key: str, data: Any) -> None: + log.warning("No cache used") + + +def cache(name: str) -> FileCache | NoCache: + try: + path = Path("/tmp/slurm_gcp_cache/") / name + if not path.exists(): + path.mkdir(exist_ok=True, parents=True) + _chown_slurm(path) + return FileCache(path) + except: + log.exception(f"Failed to create cache, fallback to NoCache") + return NoCache() diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py new file mode 100644 index 0000000000..df0fd8ebe0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py @@ -0,0 +1,76 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright 2024 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import util +import tpu + + +def get_vmcount_of_tpu_part(part): + res = 0 + lkp = util.lookup() + for ns in lkp.cfg.partitions[part].partition_nodeset_tpu: + tpu_obj = tpu.TPU.make(ns, lkp) + if res == 0: + res = tpu_obj.vmcount + else: + if res != tpu_obj.vmcount: + # this should not happen, that in the same partition there are different vmcount nodesets + return -1 + return res + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--partitions", + "-p", + help="The partition(s) to retrieve the TPU vmcount value for.", + ) + args = parser.parse_args() + if not args.partitions: + exit(0) + + # useful exit code + # partition does not exists in config.yaml, thus do not exist in slurm + PART_INVALID = -1 + # in the same partition there are nodesets with different vmcounts + DIFF_VMCOUNTS_SAME_PART = -2 + # partition is a list of partitions in which at least two of them have different vmcount + DIFF_PART_DIFFERENT_VMCOUNTS = -3 + vmcounts = [] + # valid equals to 0 means that we are ok, otherwise it will be set to one of the previously defined exit codes + valid = 0 + for part in args.partitions.split(","): + if part not in util.lookup().cfg.partitions: + valid = PART_INVALID + break + else: + if util.lookup().partition_is_tpu(part): + vmcount = get_vmcount_of_tpu_part(part) + if vmcount == -1: + valid = DIFF_VMCOUNTS_SAME_PART + break + vmcounts.append(vmcount) + else: + vmcounts.append(0) + # this means that there are different vmcounts for these partitions + if valid == 0 and len(set(vmcounts)) != 1: + valid = DIFF_PART_DIFFERENT_VMCOUNTS + if valid != 0: + print(f"VMCOUNT:{valid}") + else: + print(f"VMCOUNT:{vmcounts[0]}") diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl new file mode 100644 index 0000000000..810a0742b0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl @@ -0,0 +1,103 @@ +SCRIPTS_DIR = "{scripts_dir}" +NO_VAL = 4294967294 +-- get_tpu_vmcount.py exit code +PART_INVALID = -1 -- partition does not exists in config.yaml, thus do not exist in slurm +DIFF_VMCOUNTS_SAME_PART = -2 -- in the same partition there are nodesets with different vmcounts +DIFF_PART_DIFFERENT_VMCOUNTS = -3 -- partition is a list of partitions in which at least two of them have different vmcount +UNKWOWN_ERROR = -4 -- get_tpu_vmcount.py did not return a valid response + +function get_part(job_desc, part_list) + if job_desc.partition then + return job_desc.partition + end + for name, val in pairs(part_list) do + if val.flag_default == 1 then + return name + end + end + return nil +end + +function os.capture(cmd, raw) + local handle = assert(io.popen(cmd, 'r')) + local output = assert(handle:read('*a')) + handle:close() + return output +end + +function get_vmcount(part) + local cmd = SCRIPTS_DIR .. "/get_tpu_vmcount.py -p " .. part + local out = os.capture(cmd, true) + for line in out:gmatch("(.-)\r?\n") do + local tag, val = line:match("([^:]+):([^:]+)") + if tag == "VMCOUNT" then + return tonumber(val) + end + end + return UNKWOWN_ERROR +end + +function slurm_job_submit(job_desc, part_list, submit_uid) + local part = get_part(job_desc, part_list) + local vmcount = get_vmcount(part) + -- Only do something if the job is in a TPU partition, if vmcount is 0, it implies that the partition(s) specified are not TPU ones + if vmcount == 0 then + return slurm.SUCCESS + end + -- This is a TPU job, but as the vmcount is 1 it can he handled the same way + if vmcount == 1 then + return slurm.SUCCESS + end + -- Check for errors + if vmcount == PART_INVALID then + slurm.log_user("Invalid partition specified " .. part) + return slurm.FAILURE + end + if vmcount == DIFF_VMCOUNTS_SAME_PART then + slurm.log_user("In partition(s) " .. part .. + " there are more than one tpu nodeset vmcount, this should not happen.") + return slurm.ERROR + end + if vmcount == DIFF_PART_DIFFERENT_VMCOUNTS then + slurm.log_user("In partition list " .. part .. + " there are more than one TPU types, cannot determine which is the correct vmcount to use, please retry with only one partition.") + return slurm.FAILURE + end + if vmcount == UNKWOWN_ERROR then + slurm.log_user("Something went wrong while executing get_tpu_vmcount.py.") + return slurm.ERROR + end + -- This is surely a TPU node + if vmcount > 1 then + local min_nodes = job_desc.min_nodes + local max_nodes = job_desc.max_nodes + -- if not specified assume it is one, this should be improved taking into account the cpus, mem, and other factors + if min_nodes == NO_VAL then + min_nodes = 1 + max_nodes = 1 + end + -- as max_nodes can be higher than the nodes in the partition, we are not able to calculate with certainty the nodes that this job will have if this value is set to something + -- different than min_nodes + if min_nodes ~= max_nodes then + slurm.log_user("Max nodes cannot be set different than min nodes for the TPU partitions.") + return slurm.ERROR + end + -- Set the number of switches to the number of nodes originally requested by the job, as the job requests "TPU groups" + job_desc.req_switch = min_nodes + + -- Apply the node increase into the job description. + job_desc.min_nodes = min_nodes * vmcount + job_desc.max_nodes = max_nodes * vmcount + -- if job_desc.features then + -- slurm.log_user("Features: %s",job_desc.features) + -- end + end + + return slurm.SUCCESS +end + +function slurm_job_modify(job_desc, job_rec, part_list, modify_uid) + return slurm.SUCCESS +end + +return slurm.SUCCESS diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py new file mode 100644 index 0000000000..cabd6e3e9f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py @@ -0,0 +1,352 @@ +#!/slurm/python/venv/bin/python3.13 +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Callable, Any +import argparse +import os +import shelve +import uuid +from collections import namedtuple +from datetime import datetime, timedelta, timezone +from pathlib import Path +from pprint import pprint + +import util +from google.api_core import exceptions, retry +from google.cloud import bigquery as bq +from google.cloud.bigquery import SchemaField # type: ignore +from util import lookup, run + +SACCT = "sacct" +script = Path(__file__).resolve() + +DEFAULT_TIMESTAMP_FILE = script.parent / "bq_timestamp" +timestamp_file = Path(os.environ.get("TIMESTAMP_FILE", DEFAULT_TIMESTAMP_FILE)) +# The maximum request to insert_rows is 10MB, each sacct row is about 1200 bytes or ~ 8000 rows. +# Set to 5000 for a little wiggle room. +BQ_ROW_BATCH_SIZE = 5000 + +# cluster_id_file = script.parent / 'cluster_uuid' +# try: +# cluster_id = cluster_id_file.read_text().rstrip() +# except FileNotFoundError: +# cluster_id = uuid.uuid4().hex +# cluster_id_file.write_text(cluster_id) + +job_idx_cache_path = script.parent / "bq_job_idx_cache" + +SLURM_TIME_FORMAT = r"%Y-%m-%dT%H:%M:%S" + + +def make_datetime(time_string): + if time_string == "None": + return None + return datetime.strptime(time_string, SLURM_TIME_FORMAT).replace( + tzinfo=timezone.utc + ) + + +def make_time_interval(seconds): + sign = 1 + if seconds < 0: + sign = -1 + seconds = abs(seconds) + d, r = divmod(seconds, 60 * 60 * 24) + h, r = divmod(r, 60 * 60) + m, s = divmod(r, 60) + d *= sign + h *= sign + return f"{d}D {h:02}:{m:02}:{s}" + + +converters: Dict[str, Callable[[Any], Any]] = { + "DATETIME": make_datetime, + "INTERVAL": make_time_interval, + "STRING": str, + "INT64": lambda n: int(n or 0), +} + + +def schema_field(field_name, data_type, description, required=False): + return SchemaField( + field_name, + data_type, + description=description, + mode="REQUIRED" if required else "NULLABLE", + ) + + +schema_fields = [ + schema_field("cluster_name", "STRING", "cluster name", required=True), + schema_field("cluster_id", "STRING", "UUID for the cluster", required=True), + schema_field("entry_uuid", "STRING", "entry UUID for the job row", required=True), + schema_field( + "job_db_uuid", "STRING", "job db index from the slurm database", required=True + ), + schema_field("job_id_raw", "INT64", "raw job id", required=True), + schema_field("job_id", "STRING", "job id", required=True), + schema_field("state", "STRING", "final job state", required=True), + schema_field("job_name", "STRING", "job name"), + schema_field("partition", "STRING", "job partition"), + schema_field("submit_time", "DATETIME", "job submit time"), + schema_field("start_time", "DATETIME", "job start time"), + schema_field("end_time", "DATETIME", "job end time"), + schema_field("elapsed_raw", "INT64", "STRING", "job run time in seconds"), + # schema_field("elapsed_time", "INTERVAL", "STRING", "job run time interval"), + schema_field("timelimit_raw", "STRING", "job timelimit in minutes"), + schema_field("timelimit", "STRING", "job timelimit"), + # schema_field("num_tasks", "INT64", "number of allocated tasks in job"), + schema_field("nodelist", "STRING", "names of nodes allocated to job"), + schema_field("user", "STRING", "user responsible for job"), + schema_field("uid", "INT64", "uid of job user"), + schema_field("group", "STRING", "group of job user"), + schema_field("gid", "INT64", "gid of job user"), + schema_field("wckey", "STRING", "job wckey"), + schema_field("qos", "STRING", "job qos"), + schema_field("comment", "STRING", "job comment"), + schema_field("admin_comment", "STRING", "job admin comment"), + # extra will be added in 23.02 + # schema_field("extra", "STRING", "job extra field"), + schema_field("exitcode", "STRING", "job exit code"), + schema_field("alloc_cpus", "INT64", "count of allocated CPUs"), + schema_field("alloc_nodes", "INT64", "number of nodes allocated to job"), + schema_field("alloc_tres", "STRING", "allocated trackable resources (TRES)"), + # schema_field("system_cpu", "INTERVAL", "cpu time used by parent processes"), + # schema_field("cpu_time", "INTERVAL", "CPU time used (elapsed * cpu count)"), + schema_field("cpu_time_raw", "INT64", "CPU time used (elapsed * cpu count)"), + # schema_field("ave_cpu", "INT64", "Average CPU time of all tasks in job"), + # schema_field( + # "tres_usage_tot", + # "STRING", + # "Tres total usage by all tasks in job", + # ), +] + + +slurm_field_map = { + "job_db_uuid": "DBIndex", + "job_id_raw": "JobIDRaw", + "job_id": "JobID", + "state": "State", + "job_name": "JobName", + "partition": "Partition", + "submit_time": "Submit", + "start_time": "Start", + "end_time": "End", + "elapsed_raw": "ElapsedRaw", + "elapsed_time": "Elapsed", + "timelimit_raw": "TimelimitRaw", + "timelimit": "Timelimit", + "num_tasks": "NTasks", + "nodelist": "Nodelist", + "user": "User", + "uid": "Uid", + "group": "Group", + "gid": "Gid", + "wckey": "Wckey", + "qos": "Qos", + "comment": "Comment", + "admin_comment": "AdminComment", + # "extra": "Extra", + "exit_code": "ExitCode", + "alloc_cpus": "AllocCPUs", + "alloc_nodes": "AllocNodes", + "alloc_tres": "AllocTres", + "system_cpu": "SystemCPU", + "cpu_time": "CPUTime", + "cpu_time_raw": "CPUTimeRaw", + "ave_cpu": "AveCPU", + "tres_usage_tot": "TresUsageInTot", +} + +# new field name is the key for job_schema. Used to lookup the datatype when +# creating the job rows +job_schema = {field.name: field for field in schema_fields} +# Order is important here, as that is how they are parsed from sacct output +Job = namedtuple("Job", job_schema.keys()) # type: ignore +# ... see https://github.com/python/mypy/issues/848 + +client = bq.Client( + project=lookup().cfg.project, + credentials=util.default_credentials(), + client_options=util.create_client_options(util.ApiEndpoint.BQ), +) +dataset_id = f"{lookup().cfg.slurm_cluster_name}_job_data" +dataset = bq.DatasetReference(project=lookup().project, dataset_id=dataset_id) +table = bq.Table( + bq.TableReference(dataset, f"jobs_{lookup().cfg.slurm_cluster_name}"), schema_fields +) + + +class JobInsertionFailed(Exception): + pass + + +def make_job_row(job): + job_row = { + field_name: converters[field.field_type](job[field_name]) + for field_name, field in job_schema.items() + if field_name in job + } + job_row["entry_uuid"] = uuid.uuid4().hex + job_row["cluster_id"] = lookup().cfg.cluster_id + job_row["cluster_name"] = lookup().cfg.slurm_cluster_name + return job_row + + +def load_slurm_jobs(start, end): + states = ",".join( + ( + "BOOT_FAIL", + "CANCELLED", + "COMPLETED", + "DEADLINE", + "FAILED", + "NODE_FAIL", + "OUT_OF_MEMORY", + "PREEMPTED", + "REQUEUED", + "REVOKED", + "TIMEOUT", + ) + ) + start_iso = start.isoformat(timespec="seconds") + end_iso = end.isoformat(timespec="seconds") + # slurm_fields and bq_fields will be in matching order + slurm_fields = ",".join(slurm_field_map.values()) + bq_fields = slurm_field_map.keys() + cmd = ( + f"{SACCT} --start {start_iso} --end {end_iso} -X -D --format={slurm_fields} " + f"--state={states} --parsable2 --noheader --allusers --duplicates" + ) + text = run(cmd).stdout.splitlines() + # zip pairs bq_fields with the value from sacct + jobs = [dict(zip(bq_fields, line.split("|"))) for line in text] + + # The job index cache allows us to avoid sending duplicate jobs. This avoids a race condition with updating the database. + with shelve.open(str(job_idx_cache_path), flag="r") as job_idx_cache: + job_rows = [ + make_job_row(job) + for job in jobs + if str(job["job_db_uuid"]) not in job_idx_cache + ] + return job_rows + + +def init_table(): + global dataset + global table + dataset = client.create_dataset(dataset, exists_ok=True) # type: ignore + table = client.create_table(table, exists_ok=True) + until_found = retry.Retry(predicate=retry.if_exception_type(exceptions.NotFound)) + table = client.get_table(table, retry=until_found) + # cannot add required fields to an existing schema + table.schema = schema_fields + table = client.update_table(table, ["schema"]) + + +def purge_job_idx_cache(): + purge_time = datetime.now() - timedelta(minutes=30) + with shelve.open(str(job_idx_cache_path), writeback=True) as cache: + to_delete = [] + for idx, stamp in cache.items(): + if stamp < purge_time: + to_delete.append(idx) + for idx in to_delete: + del cache[idx] + + +def bq_submit(jobs): + try: + result = client.insert_rows(table, jobs) + except exceptions.NotFound as e: + print(f"failed to upload job data, table not yet found: {e}") + raise e + except Exception as e: + print(f"failed to upload job data: {e}") + raise e + if result: + pprint(jobs) + pprint(result) + raise JobInsertionFailed("failed to upload job data to big query") + print(f"successfully loaded {len(jobs)} jobs") + + +def get_time_window(): + if not timestamp_file.is_file(): + timestamp_file.touch() + try: + timestamp = datetime.strptime( + timestamp_file.read_text().rstrip(), SLURM_TIME_FORMAT + ) + # time window will overlap the previous by 10 minutes. Duplicates will be filtered out by the job_idx_cache + start = timestamp - timedelta(minutes=10) + except ValueError: + # timestamp 1 is 1 second after the epoch; timestamp 0 is special for sacct + start = datetime.fromtimestamp(1) + # end is now() truncated to the last second + end = datetime.now().replace(microsecond=0) + return start, end + + +def write_timestamp(time): + timestamp_file.write_text(time.isoformat(timespec="seconds")) + + +def update_job_idx_cache(jobs, timestamp): + with shelve.open(str(job_idx_cache_path), writeback=True) as job_idx_cache: + for job in jobs: + job_idx = str(job["job_db_uuid"]) + job_idx_cache[job_idx] = timestamp + + +def main(): + if not lookup().cfg.enable_bigquery_load: + print("bigquery load is not currently enabled") + exit(0) + init_table() + + start, end = get_time_window() + jobs = load_slurm_jobs(start, end) + # on failure, an exception will cause the timestamp not to be rewritten. So + # it will try again next time. If some writes succeed, we don't currently + # have a way to not submit duplicates next time. + if jobs: + num_batches = (len(jobs) - 1) // BQ_ROW_BATCH_SIZE + 1 + print( + f"loading {num_batches} batches of BigQuery data in batches of size : {BQ_ROW_BATCH_SIZE}" + ) + for batch_indx, job_indx in enumerate(range(0, len(jobs), BQ_ROW_BATCH_SIZE)): + print(f"loading BigQuery data batch {batch_indx} of {num_batches}") + bq_submit(jobs[job_indx : job_indx + BQ_ROW_BATCH_SIZE]) + write_timestamp(end) + update_job_idx_cache(jobs, end) + + +parser = argparse.ArgumentParser(description="submit slurm job data to big query") +parser.add_argument( + "timestamp_file", + nargs="?", + action="store", + type=Path, + help="specify timestamp file for reading and writing the time window start. Precedence over TIMESTAMP_FILE env var.", +) + +purge_job_idx_cache() +if __name__ == "__main__": + args = parser.parse_args() + if args.timestamp_file: + timestamp_file = args.timestamp_file.resolve() + main() diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py new file mode 100644 index 0000000000..d4a4477f83 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py @@ -0,0 +1,196 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Implementation of message queue that mimics interface of GCP (PubSub)[https://cloud.google.com/pubsub] + +Messages are stored on controller state disk (to survive controller re-creation) with following layout: + +// +├- +| └- +└- .staging + └- + └- + +One message is one immutable file, that will be deleted after acknowledgement. +NOTE: Implementation assumes that both `` and `.staging/` are on the same disk device, +so it can rely on atomic "move / rename" operation. +""" +from typing import Any +import util +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +import os +import uuid + +import logging +log = logging.getLogger() + + +@dataclass(frozen=True) +class Message: + id: str + created: datetime + data: Any + + def to_json(self) -> dict[str, str]: + return dict( + id=self.id, + created=self.created.isoformat(), + data=self.data) + + @classmethod + def from_json(cls, data: dict[str, str]) -> 'Message': + return cls( + id=data['id'], + created=datetime.fromisoformat(data['created']), + data=data['data']) + +class Topic: + """ + Acts as PubSub topic (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics). + We can have multiple instances of + """ + def __init__(self, path: Path, staging: Path) -> None: + self._path = path + self._staging = staging + + def _gen_id(self, created: datetime) -> str: + ts = created.strftime("%Y_%m_%d-%H_%M_%S") + suf = str(uuid.uuid4())[:8] + return f"{ts}-{suf}" + + def publish(self, data: Any) -> None: + created = util.now() + id = self._gen_id(created) + msg = Message(id=id, created=created, data=data) + + staged = self._staging / msg.id + dst = self._path / msg.id + + # Write to stagin area first then perform atomic move + # to prevent "reads of partial writes" + staged.write_text(json.dumps(msg.to_json())) + util.chown_slurm(staged) + staged.rename(dst) + + +class Subscription: + """ + Acts as PubSub subscription (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) + with following settings: + + ``` + ackDeadlineSeconds = +Inf # don't resend message that was already being delivered but not acked yet + retainAckedMessages = False # don't persist messages that were already acked + enableMessageOrdering = True # delivers messages in chronoligical order + messageRetentionDuration = +Inf # don't expire messages + deadLetterPolicy = None # "deadlettering" is disabled, subscriber should take care of any poisonous messages + retryPolicy = { # NACKed message will be re-delievered after some time + minimumBackoff = 30s # NOTE: Practically there is no timer, but Subscription instance will not try to re-deliver NACKed messages. + maximumBackoff = 30s # Assumes that slurmsync runs every 30+ sec. + } + ``` + + IMPORTANT: Should only be run as part of slurmsync, + this is our way to ensure that at most one instance exists at a time. + There is no concurancy safeguards in place, avoid multithreaded `pull`, + while multithreaded `ack` & `modify_ack_deadline` are OK. + """ + + def __init__(self, path: Path) -> None: + self._path: Path = path + # contains ALL messages pulled by this subscription instance + # both acked, nacked, and still being processed + # used to prevent double delivery within lifetime of subscription (slurmsync) + self._pulled: set[str] = set() + + def _delete(self, id: str) -> None: + log.debug(f"removing {id}") + try: + os.unlink(self._path / id) + except: + log.exception(f"Failed to remove message {id}") + + def _read_msg(self, id: str) -> Message | None: + try: + with open(self._path / id, 'r') as f: + content = json.loads(f.read()) + return Message.from_json(content) + except Exception: + log.exception(f"Failed to read message {id}") + self._delete(id) # delete message to reduce "deadlettering" + return None + + def pull(self, max_messages: int) -> list[Message]: + if not self._path.exists(): + log.warning(f"Topic {self._path} does not exist") + return [] + res = [] + ls = sorted(os.listdir(self._path)) + for name in ls: + msg = self._read_msg(name) + if msg is not None and msg.id not in self._pulled: + self._pulled.add(msg.id) + res.append(msg) + + if len(res) >= max_messages: + break + return res + + + def ack(self, ids: list[str]) -> None: + for id in ids: + self._delete(id) + + + def modify_ack_deadline(self, ids: list[str], deadline: int) -> None: + """ + Modifies the ack deadline for a specific message. + IMPORTANT: Only accepts deadline=0, which is a way to NACK + Any other values are also meaningless due to ackDeadlineSeconds==+Inf + """ + assert deadline == 0 # no op, next subscriber (slurmsync) will pick this up + + +# Topics and Subscriptions are singletons +# TODO: consider making thread-safe +_topics = {} +_subscriptions = {} + +def _make_path(name: str) -> Path: + p = util.slurmdirs.state / "pubsub" / name + p.mkdir(parents=True, exist_ok=True) + util.chown_slurm(p) + return p + +def _make_staging_path(name: str) -> Path: + p = util.slurmdirs.state / "pubsub" / ".staging" / name + p.mkdir(parents=True, exist_ok=True) + util.chown_slurm(p) + return p + +def topic(name: str) -> Topic: + if name not in _topics: + _topics[name] = Topic(_make_path(name), _make_staging_path(name)) + return _topics[name] + +def subscription(name: str) -> Subscription: + if name not in _subscriptions: + _subscriptions[name] = Subscription(_make_path(name)) + return _subscriptions[name] diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py new file mode 100644 index 0000000000..8ea3d0657e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py @@ -0,0 +1,254 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional + +import util +import uuid +from addict import Dict as NSDict # type: ignore +from datetime import datetime, timedelta +from collections import defaultdict +import logging +from time import sleep + +log = logging.getLogger() + +DWS_EOL_RESERVATION_DURATION = 10 # minutes + +def _duration(flex_options: NSDict, job_id: Optional[int], lkp: util.Lookup) -> int: + dur = flex_options.max_run_duration + if not job_id or not flex_options.use_job_duration: + return dur + + job = lkp.job(job_id) + if not job or not job.duration: + return dur + + if timedelta(minutes=10) <= job.duration <= timedelta(weeks=1): + return int(job.duration.total_seconds()) + + log.info("Job TimeLimit cannot be less than 10 minutes or exceed one week") + return dur + +def _create_slurm_reservation(node_name: str, boot_time: datetime, run_duration: int, lkp: util.Lookup): + """ + Create a Slurm reservation starting at EOL - buffer time. + """ + eol = boot_time + timedelta(seconds=run_duration) + start_str = eol.strftime("%Y-%m-%dT%H:%M:%S") + reservation_name = f"dws-eol-{node_name}" + log.debug(f"creating slurm reservation for {node_name}") + try: + util.run(f"{lkp.scontrol} create reservation user=slurm starttime={start_str} duration={DWS_EOL_RESERVATION_DURATION} nodes={node_name} reservationname={reservation_name} flags=maint,ignore_jobs") + except Exception as e: + log.error(f"Failed to create reservation for {node_name}: {e}") + +def _delete_slurm_reservation(node_name: str, lkp: util.Lookup): + """ + Delete the Slurm reservation for the given node. + """ + reservation_name = f"dws-eol-{node_name}" + try: + util.run(f"{lkp.scontrol} delete reservation {reservation_name}") + log.debug(f"Deleted Slurm reservation {reservation_name} for {node_name}") + except Exception as e: + log.error(f"Failed to delete reservation for {node_name}: {e}") + +def resume_flex_chunk(nodes: List[str], job_id: Optional[int], lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + assert len(nodeset.zone_policy_allow) > 0 + region = lkp.node_region(model) + + assert nodeset.dws_flex.enabled + + uid = str(uuid.uuid4())[:8] + if job_id: + mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-job-{job_id}-{uid}" + else: + mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-{uid}" + + # Create MIG + req = lkp.compute.regionInstanceGroupManagers().insert( + project=lkp.project, + region=region, + body=dict( + name=mig_name, + versions=[dict(instanceTemplate=nodeset.instance_template)], + targetSize=0, + distributionPolicy=dict( + zones=[ + dict(zone=f"zones/{z}") for z in nodeset.zone_policy_allow + ], + targetShape="ANY_SINGLE_ZONE" ), + updatePolicy = dict(instanceRedistributionType = "NONE" ), + instanceLifecyclePolicy=dict(defaultActionOnFailure= "DO_NOTHING" ), # TODO(FLEX): Not supported yet, migrate once supported + ) + ) + util.log_api_request(req) + op = req.execute() + res = util.wait_for_operation(op) + assert "error" not in res, f"{res}" + + # Create resize request + duration_seconds = _duration(nodeset.dws_flex, job_id, lkp) + req = lkp.compute.regionInstanceGroupManagerResizeRequests().insert( + project=lkp.project, + region=region, + instanceGroupManager=mig_name, + body=dict( + name="initial-resize", + instances=[dict(name=n) for n in nodes], + requested_run_duration=dict( + seconds=duration_seconds + ) + ) + ) + util.log_api_request(req) + op = req.execute() + res = util.wait_for_operation(op) + + # Create Slurm reservations if use_job_duration is set + if nodeset.dws_flex.use_job_duration: + # Get run duration (seconds) + run_duration = duration_seconds + for node_name in nodes: + # Fetch instance creation time from GCP instance (via util.py) + instance = lkp.instance(node_name) + if(instance and instance.creation_timestamp): + log.debug("creating with creation_timestamp") + boot_time = instance.creation_timestamp # Already a datetime object + else: + boot_time = datetime.utcnow() + log.debug("creating with utcnow time: {boot_time}") + _create_slurm_reservation(node_name, boot_time, run_duration, lkp) + + assert "error" not in res, f"{res}" + +def _suspend_flex_mig(mig_self_link: str, nodes: List[str], lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + assert len(nodeset.zone_policy_allow) > 0 + region = lkp.node_region(model) + project=lkp.project + instanceGroupManager=util.trim_self_link(mig_self_link) + + links = [ + f"zones/{inst.zone}/instances/{inst.name}" + for inst in [ + lkp.instance(node) for node in nodes + ] if inst + ] + + target_mig=lkp.get_mig(lkp.project, region, instanceGroupManager) + assert target_mig + + # TODO(FLEX): This will not work if MIG didn't obtain capacity yet. + # The request will fail and MIG will continue provisioning. + # Instead whole MIG should be deleted. + # + All other instances in MIG are not provisioned also, safe to delete + # - Need to come up will clear test to differentiate non-provisioned MIG and single VM being down; + # Particularly CRITICAL due to ActionOnFailure=DO_NOTHING + # - Need to `down_nodes_notify_jobs` for all nodes in MIG, make sure that it doesn't interfere with Slurm suspend-flow. + + if target_mig["targetSize"] == len(nodes): #We can just delete the whole MIG in this case + req = lkp.compute.regionInstanceGroupManagers().delete( + project=project, + region=region, + instanceGroupManager=instanceGroupManager, + ) + else: + req = lkp.compute.regionInstanceGroupManagers().deleteInstances( + project=project, + region=region, + instanceGroupManager=instanceGroupManager, + body=dict( + instances=links, + skipInstancesOnValidationError=True, + ) + ) + + util.log_api_request(req) + op = req.execute() + + res = util.wait_for_operation(op) + + # Delete Slurm reservations for nodes being deprovisioned + for node_name in nodes: + log.info("delete dws reservation") + _delete_slurm_reservation(node_name, lkp) + + assert "error" not in res, f"{res}" + +def _suspend_provisioning_inst(nodes:List[str], node_template:str, lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + assert len(nodeset.zone_policy_allow) > 0 + region = lkp.node_region(model) + + mig_list=lkp.get_mig_list(lkp.project, region) + + # FLEX (#TODO): If we enter this conditional it's likely this was called so early that MIG creation hasn't started + # Consider potentially retrying? No natural mechanism for retry currently but we could + # perhaps use slurmsync and then try it again to ensure it wasn't a case of being too early. + # This is important since we're now enabling long ResumeTimeout (Slurm won't call suspend on node within reasonable timeframe) + # so until we do this is slurmsync this is a temporary workaround. + + if not mig_list or not mig_list.get("items"): + log.info("No matching MIG found to delete! Retrying...") + sleep(5) + mig_list=lkp.get_mig_list(lkp.project, region) + if not mig_list or not mig_list.get("items"): + return + + for mig in mig_list["items"]: + if mig["instanceTemplate"] == node_template: + if mig["currentActions"]["creating"] > 0 and mig["targetSize"] == mig["currentActions"]["creating"]: + req = lkp.compute.regionInstanceGroupManagers().delete( + project=lkp.project, + region=region, + instanceGroupManager=util.trim_self_link(mig["selfLink"]), + ) + + util.log_api_request(req) + op = req.execute() + + res = util.wait_for_operation(op) + assert "error" not in res, f"{res}" + return + + log.info("No matching MIG found to delete!") + +def suspend_flex_nodes(nodes: List[str], lkp: util.Lookup) -> None: + by_mig = defaultdict(list) + not_provisioned = defaultdict(list) + for node in nodes: + inst = lkp.instance(node) + if not inst: + not_provisioned[lkp.node_template(node)].append(node) + else: + mig = inst.metadata.get("created-by") + if not mig: + log.error(f"Can not suspend {node}, can not find associated MIG") + continue + by_mig[mig].append(node) + + for mig, nodes in by_mig.items(): + _suspend_flex_mig(mig, nodes, lkp) + + for node_template, nodes in not_provisioned.items(): + _suspend_provisioning_inst(nodes, node_template, lkp) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt new file mode 100644 index 0000000000..2ab3162ccf --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt @@ -0,0 +1,9 @@ +pytest +pytest-mock +pytest_unordered +mock + +types-mock +types-httplib2 +types-requests +types-PyYAML diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt new file mode 100644 index 0000000000..e923e53dbf --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt @@ -0,0 +1,18 @@ +addict==2.4.0 +google-api-core==2.19.0 +google-api-python-client==2.93.0 +google-auth==2.40.3 +google-auth-httplib2==0.1.0 +google-cloud-bigquery==3.11.3 +google-cloud-core==2.3.3 +google-cloud-secret-manager~=2.22 +google-cloud-storage==2.10.0 +google-cloud-tpu==1.10.0 +google-resumable-media==2.5.0 +googleapis-common-protos==1.59.1 +grpcio==1.60.0 +grpcio-status==1.60.0 +httplib2==0.22.0 +more-executors==2.11.4 +pyyaml==6.0.2 +requests==2.32.4 diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py new file mode 100644 index 0000000000..ea0012a0b1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py @@ -0,0 +1,703 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Dict, Any +import argparse +from datetime import timedelta +import shlex +import json +import logging +import os +import yaml +import collections +from pathlib import Path +from dataclasses import dataclass +from addict import Dict as NSDict # type: ignore + +import util +from util import ( + chunked, + ensure_execute, + execute_with_futures, + log_api_request, + map_with_futures, + run, + separate, + to_hostlist, + trim_self_link, + wait_for_operation, +) +from util import lookup, ReservationDetails +import tpu +import mig_flex + +log = logging.getLogger() + +PLACEMENT_MAX_CNT = 1500 +# Placement group needs to be the same for an entire bulk_insert hence +# if placement is used the actual BULK_INSERT_LIMIT will be +# max([1000, PLACEMENT_MAX_CNT]) +BULK_INSERT_LIMIT = 5000 + +# https://cloud.google.com/compute/docs/instance-groups#types_of_managed_instance_groups +ZONAL_MIG_SIZE_LIMIT = 1000 + + +@dataclass(frozen=True) +class ResumeJobData: + job_id: int + partition: str + nodes_alloc: List[str] + +@dataclass(frozen=True) +class ResumeData: + jobs: List[ResumeJobData] + + +def get_resume_file_data() -> Optional[ResumeData]: + if not (path := os.getenv("SLURM_RESUME_FILE")): + log.error("SLURM_RESUME_FILE was not in environment. Cannot get detailed job, node, partition allocation data.") + return None + blob = Path(path).read_text() + log.debug(f"Resume data: {blob}") + data = json.loads(blob) + + jobs = [] + for jo in data.get("jobs", []): + job = ResumeJobData( + job_id = jo.get("job_id"), + partition = jo.get("partition"), + nodes_alloc = util.to_hostnames(jo.get("nodes_alloc")), + ) + jobs.append(job) + return ResumeData(jobs=jobs) + +def instance_properties(nodeset: NSDict, model:str, placement_group:Optional[str], labels:Optional[dict], job_id:Optional[int]): + props = NSDict() + + if labels: # merge in extra labels on instance and disks + template_link = lookup().node_template(model) + template_info = lookup().template_info(template_link) + + props.labels = {**template_info.labels, **labels} + + for disk in template_info.disks: + if disk.initializeParams.get("diskType", "local-ssd") == "local-ssd": + continue # do not label local ssd + disk.initializeParams.labels.update(labels) + props.disks = template_info.disks + + if placement_group: + props.resourcePolicies = [placement_group] + + if reservation := lookup().nodeset_reservation(nodeset): + update_reservation_props(reservation, props, placement_group, reservation.calendar) + + if (fr := lookup().future_reservation(nodeset)) and fr.specific: + assert fr.active_reservation + update_reservation_props(fr.active_reservation, props, placement_group, fr.calendar) + + if props.resourcePolicies: + props.scheduling.onHostMaintenance = "TERMINATE" + + if nodeset.maintenance_interval: + props.scheduling.maintenanceInterval = nodeset.maintenance_interval + + if nodeset.dws_flex.enabled and nodeset.dws_flex.use_bulk_insert: + update_props_dws(props, nodeset.dws_flex, job_id) + + # Override with properties explicit specified in the nodeset + props.update(nodeset.get("instance_properties") or {}) + return props + +def update_reservation_props(reservation:ReservationDetails, props:NSDict, placement_group:Optional[str], calendar_mode:bool) -> None: + props.reservationAffinity = { + "consumeReservationType": "SPECIFIC_RESERVATION", + "key": f"compute.{util.universe_domain()}/reservation-name", + "values": [reservation.bulk_insert_name], + } + + if reservation.dense or calendar_mode: + props.scheduling.provisioningModel = "RESERVATION_BOUND" + + # Figure out `resourcePolicies` + if reservation.policies: # use ones already attached to reservations + props.resourcePolicies = reservation.policies + elif reservation.dense and placement_group: # use once created by Slurm + props.resourcePolicies = [placement_group] + else: # vanilla reservations don't support external policies + props.resourcePolicies = [] + log.info( + f"reservation {reservation.bulk_insert_name} is being used with resourcePolicies: {props.resourcePolicies}") + +def update_props_dws(props: NSDict, dws_flex: NSDict, job_id: Optional[int]) -> None: + props.scheduling.onHostMaintenance = "TERMINATE" + props.scheduling.instanceTerminationAction = "DELETE" + props.reservationAffinity['consumeReservationType'] = "NO_RESERVATION" + props.scheduling.maxRunDuration['seconds'] = dws_flex_duration(dws_flex, job_id) + +def dws_flex_duration(dws_flex: NSDict, job_id: Optional[int]) -> int: + max_duration = dws_flex.max_run_duration + if dws_flex.use_job_duration and job_id is not None and (job := lookup().job(job_id)) and job.duration: + if timedelta(seconds=30) <= job.duration <= timedelta(weeks=1): + max_duration = int(job.duration.total_seconds()) + else: + log.info("Job TimeLimit cannot be less than 30 seconds or exceed one week") + return max_duration + +def create_instances_request(nodes: List[str], placement_group: Optional[str], excl_job_id: Optional[int]): + """Call regionInstances.bulkInsert to create instances""" + assert 0 < len(nodes) <= BULK_INSERT_LIMIT + + # model here indicates any node that can be used to describe the rest + model = next(iter(nodes)) + log.debug(f"create_instances_request: {model} placement: {placement_group}") + + nodeset = lookup().node_nodeset(model) + template = lookup().node_template(model) + labels = {"slurm_job_id": excl_job_id} if excl_job_id else None + + body = dict( + count = len(nodes), + sourceInstanceTemplate = template, + # key is instance name, value overwrites properties (no overwrites) + perInstanceProperties = {k: {} for k in nodes}, + instanceProperties = instance_properties( + nodeset, model, placement_group, labels, excl_job_id + ), + ) + + if placement_group and excl_job_id is not None: + pass # do not set minCount to force "all or nothing" behavior + else: + body["minCount"] = 1 + + zone_allow = nodeset.zone_policy_allow or [] + zone_deny = nodeset.zone_policy_deny or [] + + if len(zone_allow) == 1: # if only one zone is used, use zonal BulkInsert API, as less prone to errors + api_method = lookup().compute.instances().bulkInsert + method_args = {"zone": zone_allow[0]} + else: + api_method = lookup().compute.regionInstances().bulkInsert + method_args = {"region": lookup().node_region(model)} + + body["locationPolicy"] = dict( + locations = { + **{ f"zones/{z}": {"preference": "ALLOW"} for z in zone_allow }, + **{ f"zones/{z}": {"preference": "DENY"} for z in zone_deny }}, + targetShape = nodeset.zone_target_shape, + ) + + req = api_method( + project=lookup().project, + body=body, + **method_args) + log.debug(f"new request: endpoint={req.methodId} nodes={to_hostlist(nodes)}") + log_api_request(req) + return req + +@dataclass() +class PlacementAndNodes: + placement: Optional[str] + nodes: List[str] + +@dataclass(frozen=True) +class BulkChunk: + nodes: List[str] + prefix: str # - + chunk_idx: int + excl_job_id: Optional[int] + placement_group: Optional[str] = None + + @property + def name(self): + if self.placement_group is not None: + return f"{self.prefix}:job{self.excl_job_id}:{self.placement_group}:{self.chunk_idx}" + if self.excl_job_id is not None: + return f"{self.prefix}:job{self.excl_job_id}:{self.chunk_idx}" + return f"{self.prefix}:{self.chunk_idx}" + + +def group_nodes_bulk(nodes: List[str], resume_data: Optional[ResumeData], lkp: util.Lookup): + """group nodes by nodeset, placement_group, exclusive_job_id if any""" + if resume_data is None: # all nodes will be considered jobless + resume_data = ResumeData(jobs=[]) + + nodes_set = set(nodes) # turn into set to simplify intersection + non_excl = nodes_set.copy() + groups : Dict[Optional[int], List[PlacementAndNodes]] = {} # excl_job_id|none -> PlacementAndNodes + + # expand all exclusive job nodelists + for job in resume_data.jobs: + if not lkp.cfg.partitions[job.partition].enable_job_exclusive: + continue + + groups[job.job_id] = [] + # placement group assignment is based on all allocated nodes, ... + for pn in create_placements(job.nodes_alloc, job.job_id, lkp): + groups[job.job_id].append( + PlacementAndNodes( + placement=pn.placement, + #... but we only want to handle nodes in nodes_resume in this run. + nodes = sorted(set(pn.nodes) & nodes_set) + )) + non_excl.difference_update(job.nodes_alloc) + + groups[None] = create_placements(sorted(non_excl), excl_job_id=None, lkp=lkp) + + def chunk_nodes(nodes: List[str]): + if not nodes: + return [] + + model = nodes[0] + + if lkp.is_flex_node(model): + chunk_size = ZONAL_MIG_SIZE_LIMIT + elif lkp.node_is_tpu(model): + ns_name = lkp.node_nodeset_name(model) + chunk_size = tpu.TPU.make(ns_name, lkp).vmcount + else: + chunk_size = BULK_INSERT_LIMIT + + return chunked(nodes, n=chunk_size) + + chunks = [ + BulkChunk( + nodes=nodes_chunk, + prefix=lkp.node_prefix(nodes_chunk[0]), # - + excl_job_id = job_id, + placement_group=pn.placement, + chunk_idx=i) + + for job_id, placements in groups.items() + for pn in placements if pn.nodes + for i, nodes_chunk in enumerate(chunk_nodes(pn.nodes)) + ] + return {chunk.name: chunk for chunk in chunks} + + +def resume_nodes(nodes: List[str], resume_data: Optional[ResumeData]): + """resume nodes in nodelist""" + lkp = lookup() + # Prevent dormant nodes associated with a reservation from being resumed + nodes, dormant_res_nodes = util.separate(lkp.is_dormant_res_node, nodes) + + if dormant_res_nodes: + log.warning(f"Resume was unable to resume reservation nodes={dormant_res_nodes}") + down_nodes_notify_jobs(dormant_res_nodes, "Reservation is not active, nodes cannot be resumed", resume_data) + + nodes, flex_managed = util.separate(lkp.is_provisioning_flex_node, nodes) + if flex_managed: + log.warning(f"Resume was unable to resume nodes={flex_managed} already managed by MIGs") + down_nodes_notify_jobs(flex_managed, "VM is managed MIG, can not be resumed", resume_data) + + if not nodes: + log.info("No nodes to resume") + return + + nodes = sorted(nodes, key=lkp.node_prefix) + grouped_nodes = group_nodes_bulk(nodes, resume_data, lkp) + + if log.isEnabledFor(logging.DEBUG): + grouped_nodelists = { + group: to_hostlist(chunk.nodes) for group, chunk in grouped_nodes.items() + } + log.debug( + "node bulk groups: \n{}".format(yaml.safe_dump(grouped_nodelists).rstrip()) + ) + + tpu_chunks, flex_chunks = [], [] + bi_inserts = {} + + for group, chunk in grouped_nodes.items(): + model = chunk.nodes[0] + + if lkp.node_is_tpu(model): + tpu_chunks.append(chunk.nodes) + elif lkp.is_flex_node(model): + flex_chunks.append(chunk) + else: + bi_inserts[group] = create_instances_request( + chunk.nodes, chunk.placement_group, chunk.excl_job_id + ) + + for chunk in flex_chunks: + mig_flex.resume_flex_chunk(chunk.nodes, chunk.excl_job_id, lkp) + + # execute all bulkInsert requests with batch + bulk_ops = dict( + zip(bi_inserts.keys(), map_with_futures(ensure_execute, bi_inserts.values())) + ) + log.debug(f"bulk_ops={yaml.safe_dump(bulk_ops)}") + started = { + group: op for group, op in bulk_ops.items() if not isinstance(op, Exception) + } + failed = { + group: err for group, err in bulk_ops.items() if isinstance(err, Exception) + } + if failed: + failed_reqs = [str(e) for e in failed.items()] + log.error("bulkInsert API failures: {}".format("; ".join(failed_reqs))) + for ident, exc in failed.items(): + down_nodes_notify_jobs(grouped_nodes[ident].nodes, f"GCP Error: {exc._get_reason()}", resume_data) # type: ignore + + if log.isEnabledFor(logging.DEBUG): + for group, op in started.items(): + group_nodes = grouped_nodelists[group] + name = op["name"] + gid = op["operationGroupId"] + log.debug( + f"new bulkInsert operation started: group={group} nodes={group_nodes} name={name} operationGroupId={gid}" + ) + # wait for all bulkInserts to complete and log any errors + bulk_operations = {group: wait_for_operation(op) for group, op in started.items()} + + # Start TPU after regular nodes so that regular nodes are not affected by the slower TPU nodes + execute_with_futures(tpu.start_tpu, tpu_chunks) + + for group, op in bulk_operations.items(): + _handle_bulk_insert_op(op, grouped_nodes[group].nodes, resume_data) + + +def _get_failed_zonal_instance_inserts(bulk_op: Any, zone: str, lkp: util.Lookup) -> list[Any]: + group_id = bulk_op["operationGroupId"] + user = bulk_op["user"] + started = bulk_op["startTime"] + ended = bulk_op["endTime"] + + fltr = f'(user eq "{user}") AND (operationType eq "insert") AND (creationTimestamp > "{started}") AND (creationTimestamp < "{ended}")' + act = lkp.compute.zoneOperations() + req = act.list(project=lkp.project, zone=zone, filter=fltr) + ops = [] + while req is not None: + result = util.ensure_execute(req) + for op in result.get("items", []): + if op.get("operationGroupId") == group_id and "error" in op: + ops.append(op) + req = act.list_next(req, result) + return ops + + +def _get_failed_instance_inserts(bulk_op: Any, lkp: util.Lookup) -> list[Any]: + zones = set() # gather zones that had failed inserts + for loc, stat in bulk_op.get("instancesBulkInsertOperationMetadata", {}).get("perLocationStatus", {}).items(): + pref, zone = loc.split("/", 1) + if not pref == "zones": + log.error(f"Unexpected location: {loc} in operation {bulk_op['name']}") + continue + if stat.get("targetVmCount", 0) != stat.get("createdVmCount", 0): + zones.add(zone) + + res = [] + for zone in zones: + res.extend(_get_failed_zonal_instance_inserts(bulk_op, zone, lkp)) + return res + +def _handle_bulk_insert_op(op: Dict, nodes: List[str], resume_data: Optional[ResumeData]) -> None: + """ + Handles **DONE** BulkInsert operations + """ + assert op["operationType"] == "bulkInsert" and op["status"] == "DONE", f"unexpected op: {op}" + + group_id = op["operationGroupId"] + if "error" in op: + error = op["error"]["errors"][0] + log.error( + f"bulkInsert operation error: {error['code']} name={op['name']} operationGroupId={group_id} nodes={to_hostlist(nodes)}" + ) + + created = 0 + for status in op["instancesBulkInsertOperationMetadata"]["perLocationStatus"].values(): + created += status.get("createdVmCount", 0) + if created == len(nodes): + log.info(f"created {len(nodes)} instances: nodes={to_hostlist(nodes)}") + return # no need to gather status of insert-operations. + + # TODO: don't gather insert-operations per bulkInsert request, instead aggregate it + # across all bulkInserts (goes one level above this function) + failed = _get_failed_instance_inserts(op, util.lookup()) + + # Multiple errors are possible, group by all of them (joined string codes) + by_error_inserts = util.groupby_unsorted( + failed, + lambda op: "+".join(err["code"] for err in op["error"]["errors"]), + ) + for code, failed_ops in by_error_inserts: + failed_ops = list(failed_ops) + failed_nodes = [trim_self_link(op["targetLink"]) for op in failed_ops] + hostlist = util.to_hostlist(failed_nodes) + log.error( + f"{len(failed_nodes)} instances failed to start: {code} ({hostlist}) operationGroupId={group_id}" + ) + + msg = "; ".join( + f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" + for err in failed_ops[0]["error"]["errors"] + ) + if code != "RESOURCE_ALREADY_EXISTS": + down_nodes_notify_jobs(failed_nodes, f"GCP Error: {msg}", resume_data) + log.error( + f"errors from insert for node '{failed_nodes[0]}' ({failed_ops[0]['name']}): {msg}" + ) + + +def down_nodes_notify_jobs(nodes: List[str], reason: str, resume_data: Optional[ResumeData]) -> None: + """set nodes down with reason""" + nodes_set = set(nodes) # turn into set to speed up intersection + jobs = resume_data.jobs if resume_data else [] + reason_quoted = shlex.quote(reason) + + for job in jobs: + if not (set(job.nodes_alloc) & nodes_set): + continue + run(f"{lookup().scontrol} update jobid={job.job_id} admincomment={reason_quoted}", check=False) + run(f"{lookup().scontrol} notify {job.job_id} {reason_quoted}", check=False) + + nodelist = util.to_hostlist(nodes) + log.error(f"Marking nodes {nodelist} as DOWN, reason: {reason}") + run(f"{lookup().scontrol} update nodename={nodelist} state=down reason={reason_quoted}", check=False) + + + + +def create_placement_request(pg_name: str, region: str, max_distance: Optional[int], accelerator_topology: Optional[str]): + config = { + "name": pg_name, + "region": region, + "groupPlacementPolicy": { + "collocation": "COLLOCATED", + "maxDistance": max_distance, + "gpuTopology": accelerator_topology, + }, + } + + request = lookup().compute.resourcePolicies().insert( + project=lookup().project, region=region, body=config + ) + log_api_request(request) + return request + + +def create_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + nodeset_map = collections.defaultdict(list) + for node in nodes: # split nodes on nodesets + nodeset_map[lkp.node_nodeset_name(node)].append(node) + + placements = [] + for _, ns_nodes in nodeset_map.items(): + placements.extend(create_nodeset_placements(ns_nodes, excl_job_id, lkp)) + return placements + + +def _allocate_nodes_to_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + # canned result for no placement policies created + no_pp = [PlacementAndNodes(placement=None, nodes=nodes)] + + model = nodes[0] + nodeset = lkp.node_nodeset(model) + + is_slice = bool(getattr(nodeset, 'accelerator_topology', None)) + + excl_job_placement = (excl_job_id is not None) and (not is_slice) + + if excl_job_placement and len(nodes) < 2: + return no_pp # don't create placement_policy for just one node + + if lkp.is_flex_node(model): + return no_pp # TODO(FLEX): Add support for workload policies + if lkp.node_is_tpu(model): + return no_pp + if not (nodeset.enable_placement and valid_placement_node(model)): + return no_pp + + max_count = calculate_chunk_size(nodeset, lkp) + + name_prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-{nodeset.nodeset_name}" + + if excl_job_placement: # simply chunk given nodes by max size of placement + return [ + PlacementAndNodes(placement=f"{name_prefix}-{excl_job_id}-{i}", nodes=chunk) + for i, chunk in enumerate(chunked(nodes, n=max_count)) + ] + + # split whole nodeset (not only nodes to resume) into chunks of max size of placement + # create placements (most likely already exists) placements for requested nodes + chunks = collections.defaultdict(list) # chunk_id -> nodes + invalid = [] + + for node in nodes: + try: + chunk = lkp.node_index(node) // max_count + chunks[chunk].append(node) + except: + invalid.append(node) + + placements = [ + # NOTE: use 0 instead of job_id for consistency with previous SlurmGCP behavior + PlacementAndNodes(placement=f"{name_prefix}-0-{c_id}", nodes=c_nodes) + for c_id, c_nodes in chunks.items() + ] + + if invalid: + placements.append(PlacementAndNodes(placement=None, nodes=invalid)) + log.error(f"Could not find placement for nodes with unexpected names: {to_hostlist(invalid)}") + + return placements + +def calculate_hosts_per_topo(accelerator_topology: str, machine_type: NSDict) -> int: + # Calculate total number of hosts per topology (Assumes format: '1x72') + try: + top_split = [int(x) for x in accelerator_topology.split("x")] + except Exception as e: + log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") + raise e + + if len(machine_type.accelerators) == 0: + gpus_per_machine = 0 + else: + gpus_per_machine = machine_type.accelerators[0].count + + if len(top_split) != 2: + log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") + elif top_split[0] <= 0 or top_split[1] <= 0: + log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") + elif gpus_per_machine <= 0: + log.error(f"The machine type has no accelerators. Cannot use accelerator topology {accelerator_topology}.") + elif top_split[1] % gpus_per_machine: + log.error(f"The GPU count {gpus_per_machine} per node is not a factor of the accelerator topology {accelerator_topology}") + + return (top_split[0] * top_split[1]) // gpus_per_machine + +def calculate_chunk_size(nodeset: NSDict, lkp: util.Lookup) -> int: + # Calculates the chunk size based on max distance value received or accelerator topology + # Assuming nodeset is not tpu + machine_type = lkp.template_info(nodeset.instance_template).machine_type + max_distance = nodeset.placement_max_distance + accelerator_topology = nodeset.accelerator_topology + + # Look for accelerator topology first + if accelerator_topology: + hosts_per_topo = calculate_hosts_per_topo(accelerator_topology, machine_type) + return hosts_per_topo + + if max_distance == 1: + return 22 + elif max_distance == 2: + if machine_type.family.startswith("a3"): + return 256 + else: + return 150 + elif max_distance == 3: + return 1500 + else: + return PLACEMENT_MAX_CNT + +def create_nodeset_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + placements = _allocate_nodes_to_placements(nodes, excl_job_id, lkp) + region = lkp.node_region(nodes[0]) + max_distance = lkp.node_nodeset(nodes[0]).get('placement_max_distance') + accelerator_topology = lkp.nodeset_accelerator_topology(lkp.node_nodeset_name(nodes[0])) + + if log.isEnabledFor(logging.DEBUG): + debug_p = {p.placement: to_hostlist(p.nodes) for p in placements} + log.debug( + f"creating {len(placements)} placement groups: \n{yaml.safe_dump(debug_p).rstrip()}" + ) + + requests = { + p.placement: create_placement_request(p.placement, region, max_distance, accelerator_topology) for p in placements if p.placement + } + if not requests: + return placements + # TODO: aggregate all requests for whole resume and execute them at once (don't limit to nodeset/job) + ops = dict( + zip(requests.keys(), map_with_futures(ensure_execute, requests.values())) + ) + + def classify_result(item): + op = item[1] + if not isinstance(op, Exception): + return "submitted" + if all(e.get("reason") == "alreadyExists" for e in op.error_details): # type: ignore + return "redundant" + return "failed" + + grouped_ops = dict(util.groupby_unsorted(list(ops.items()), classify_result)) + submitted, redundant, failed = ( + dict(grouped_ops.get(key, {})) for key in ("submitted", "redundant", "failed") + ) + if redundant: + log.warning( + "placement policies already exist: {}".format(",".join(redundant.keys())) + ) + if failed: + reqs = [f"{e}" for _, e in failed.values()] + log.fatal("failed to create placement policies: {}".format("; ".join(reqs))) + operations = {group: wait_for_operation(op) for group, op in submitted.items()} + for group, op in operations.items(): + if "error" in op: + msg = "; ".join( + f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" + for err in op["error"]["errors"] + ) + log.error( + f"placement group failed to create: '{group}' ({op['name']}): {msg}" + ) + + log.info( + f"created {len(operations)} placement groups ({to_hostlist(operations.keys())})" + ) + return placements + + +def valid_placement_node(node: str) -> bool: + invalid_types = frozenset(["e2", "t2d", "n1", "t2a", "m1", "m2", "m3"]) + mt = lookup().node_template_info(node).machineType + if mt.split("-")[0] in invalid_types: + log.warn(f"Unsupported machine type for placement policy: {mt}.") + log.warn( + f"Please do not use any the following machine types with placement policy: ({','.join(invalid_types)})" + ) + return False + return True + + +def main(nodelist: str) -> None: + """main called when run as script""" + log.debug(f"ResumeProgram {nodelist}") + # Filter out nodes not in config.yaml + other_nodes, nodes = separate( + lookup().is_power_managed_node, util.to_hostnames(nodelist) + ) + if other_nodes: + log.error( + f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" + ) + + if not nodes: + log.info("No nodes to resume") + return + resume_data = get_resume_file_data() + log.info(f"resume {util.to_hostlist(nodes)}") + resume_nodes(nodes, resume_data) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("nodelist", help="list of nodes to resume") + args = util.init_log_and_parse(parser) + main(args.nodelist) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh new file mode 100644 index 0000000000..023d246f01 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PYTHON_SCRIPT="${SCRIPT_DIR}/resume.py" + +# Capture all arguments passed by Slurm (the nodelist). +ALL_ARGS=("$@") + +# This array will hold extra argument for resume.py, like the resume data file. +UNIQUE_RESUME_FILE="" + +# Handle SLURM_RESUME_FILE if provided +if [ -n "${SLURM_RESUME_FILE-}" ] && [ -f "$SLURM_RESUME_FILE" ]; then + SAFE_DIR="/tmp/slurm_resume_data" + mkdir -p "$SAFE_DIR" + + UNIQUE_RESUME_FILE="${SAFE_DIR}/resumedata.$$.json" + cp "$SLURM_RESUME_FILE" "$UNIQUE_RESUME_FILE" +fi + +SLURM_RESUME_FILE="${UNIQUE_RESUME_FILE}" +setsid "${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & + +exit 0 diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py new file mode 100644 index 0000000000..846524adf2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py @@ -0,0 +1,660 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import shutil +import subprocess +import stat +import time +import yaml +from pathlib import Path +import functools + +import util +from util import ( + lookup, + dirs, + slurmdirs, + run, + install_custom_scripts, +) +import conf +import slurmsync + +from setup_network_storage import ( + setup_network_storage, + setup_nfs_exports, +) + + +log = logging.getLogger() + + +MOTD_HEADER = """ + SSSSSSS + SSSSSSSSS + SSSSSSSSS + SSSSSSSSS + SSSS SSSSSSS SSSS + SSSSSS SSSSSS + SSSSSS SSSSSSS SSSSSS + SSSS SSSSSSSSS SSSS + SSS SSSSSSSSS SSS + SSSSS SSSS SSSSSSSSS SSSS SSSSS + SSS SSSSSS SSSSSSSSS SSSSSS SSS + SSSSSS SSSSSSS SSSSSS + SSS SSSSSS SSSSSS SSS + SSSSS SSSS SSSSSSS SSSS SSSSS + S SSS SSSSSSSSS SSS S + SSS SSSS SSSSSSSSS SSSS SSS + S SSS SSSSSS SSSSSSSSS SSSSSS SSS S + SSSSS SSSSSS SSSSSSSSS SSSSSS SSSSS + S SSSSS SSSS SSSSSSS SSSS SSSSS S + S SSS SSS SSS SSS S + S S S S + SSS + SSS + SSS + SSS + SSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSS SSSSSSSSSSSSSSSSSSSS +SSSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSSS SSSSSSSSSSSSSSSSSSSSSS +SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSSS SSS SSSSSSSSSSSSSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSS SSS SSSSSSSSSSSSS SSSS SSSS SSSS SSSS + +""" +_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" + +def start_motd(): + """advise in motd that slurm is currently configuring""" + wall_msg = "*** Slurm is currently being configured in the background. ***" + motd_msg = MOTD_HEADER + wall_msg + "\n\n" + Path("/etc/motd").write_text(motd_msg) + util.run(f"wall -n '{wall_msg}'", timeout=30) + + +def end_motd(broadcast=True): + """modify motd to signal that setup is complete""" + Path("/etc/motd").write_text(MOTD_HEADER) + + if not broadcast: + return + + run( + "wall -n '*** Slurm {} setup complete ***'".format(lookup().instance_role), + timeout=30, + ) + if not lookup().is_controller: + run( + """wall -n ' +/home on the controller was mounted over the existing /home. +Log back in to ensure your home directory is correct. +'""", + timeout=30, + ) + + +def failed_motd(): + """modify motd to signal that setup is failed""" + wall_msg = f"*** Slurm setup failed! Please view log: {util.get_log_path()} ***" + motd_msg = MOTD_HEADER + wall_msg + "\n\n" + Path("/etc/motd").write_text(motd_msg) + util.run(f"wall -n '{wall_msg}'", timeout=30) + + +def _startup_script_timeout(lkp: util.Lookup) -> int: + if lkp.is_controller: + return lkp.cfg.get("controller_startup_scripts_timeout", 300) + elif lkp.instance_role == "compute": + return lkp.cfg.get("compute_startup_scripts_timeout", 300) + elif lkp.is_login_node: + return lkp.cfg.login_groups[util.instance_login_group()].get("startup_scripts_timeout", 300) + return 300 + + +def run_custom_scripts(): + """run custom scripts based on instance_role""" + custom_dir = dirs.custom_scripts + if lookup().is_controller: + # controller has all scripts, but only runs controller.d + custom_dirs = [custom_dir / "controller.d"] + elif lookup().instance_role == "compute": + # compute setup with nodeset.d + custom_dirs = [custom_dir / "nodeset.d"] + elif lookup().is_login_node: + # login setup with only login.d + custom_dirs = [custom_dir / "login.d"] + else: + # Unknown role: run nothing + custom_dirs = [] + + timeout = _startup_script_timeout(lookup()) + + custom_scripts = [ + p + for d in custom_dirs + for p in d.rglob("*") + if p.is_file() and not p.name.endswith(".disabled") + ] + print_scripts = ",".join(str(s.relative_to(custom_dir)) for s in custom_scripts) + log.debug(f"custom scripts to run: {custom_dir}/({print_scripts})") + + try: + for script in custom_scripts: + log.info(f"running script {script.name} with timeout={timeout}") + result = run(str(script), timeout=timeout, check=False, shell=True) + runlog = ( + f"{script.name} returncode={result.returncode}\n" + f"stdout={result.stdout}stderr={result.stderr}" + ) + log.info(runlog) + result.check_returncode() + except OSError as e: + log.error(f"script {script} is not executable") + raise e + except subprocess.TimeoutExpired as e: + log.error(f"script {script} did not complete within timeout={timeout}") + raise e + except Exception as e: + log.exception(f"script {script} encountered an exception") + raise e + +def mount_save_state_disk(): + disk_name = f"/dev/disk/by-id/google-{lookup().cfg.controller_state_disk.device_name}" + mount_point = util.slurmdirs.state + fs_type = "ext4" + + rdevice = util.run(f"realpath {disk_name}").stdout.strip() + file_output = util.run(f"file -s {rdevice}").stdout.strip() + if "filesystem" not in file_output: + util.run(f"mkfs -t {fs_type} -q {rdevice}") + + fstab_entry = f"{disk_name} {mount_point} {fs_type}" + with open("/etc/fstab", "r") as f: + fstab = f.readlines() + if fstab_entry not in fstab: + with open("/etc/fstab", "a") as f: + f.write(f"{fstab_entry} defaults 0 0\n") + + util.run(f"systemctl daemon-reload") + + os.makedirs(mount_point, exist_ok=True) + util.run(f"mount {mount_point}") + + util.chown_slurm(mount_point) + + +def setup_jwt_key(): + jwt_key = Path(slurmdirs.state / "jwt_hs256.key") + + if jwt_key.exists(): + log.info("JWT key already exists. Skipping key generation.") + else: + run("dd if=/dev/urandom bs=32 count=1 > " + str(jwt_key), shell=True) + + util.chown_slurm(jwt_key, mode=0o400) + + +def _generate_key(p: Path) -> None: + run(f"dd if=/dev/random of={p} bs=1024 count=1") + + +def setup_key(lkp: util.Lookup) -> None: + file_name = "munge.key" + dir = dirs.munge + + if lkp.cfg.enable_slurm_auth: + file_name = "slurm.key" + dir = slurmdirs.etc + + dst = Path(dir / file_name) + + if lkp.cfg.controller_state_disk.device_name: + # Copy key from persistent state disk + persist = slurmdirs.state / file_name + if not persist.exists(): + _generate_key(persist) + + shutil.copyfile(persist, dst) + if lkp.cfg.enable_slurm_auth: + util.chown_slurm(dst, mode=0o400) + util.chown_slurm(persist, mode=0o400) + else: + shutil.chown(dst, user="munge", group="munge") + os.chmod(dst, stat.S_IRUSR) + else: + if dst.exists(): + log.info("key already exists. Skipping key generation.") + else: + _generate_key(dst) + if lkp.cfg.enable_slurm_auth: + util.chown_slurm(dst, mode=0o400) + else: + shutil.chown(dst, user="munge", group="munge") + os.chmod(dst, stat.S_IRUSR) + + if lkp.cfg.enable_slurm_auth: + # Put key into shared volume for distribution + distributed = util.slurmdirs.key_distribution / file_name + shutil.copyfile(dst, distributed) + util.chown_slurm(distributed, mode=0o400) + # Munge is distributed from /etc/munge. + else: + run("systemctl restart munge", timeout=30) + + +def setup_nss_slurm(): + """install and configure nss_slurm""" + # setup nss_slurm + util.mkdirp(Path("/var/spool/slurmd")) + run( + "ln -s {}/lib/libnss_slurm.so.2 /usr/lib64/libnss_slurm.so.2".format( + slurmdirs.prefix + ), + check=False, + ) + run(r"sed -i 's/\(^\(passwd\|group\):\s\+\)/\1slurm /g' /etc/nsswitch.conf") + + +def setup_sudoers(): + content = """ +# Allow SlurmUser to manage the slurm daemons +slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmd.service +slurm ALL= NOPASSWD: /usr/bin/systemctl restart sackd.service +slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmctld.service +""" + sudoers_file = Path("/etc/sudoers.d/slurm") + sudoers_file.write_text(content) + sudoers_file.chmod(0o0440) + + +def setup_maintenance_script(): + perform_maintenance = """#!/bin/bash + +#SBATCH --priority=low +#SBATCH --time=180 + +VM_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") +ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" | cut -d '/' -f 4) + +gcloud compute instances perform-maintenance $VM_NAME \ + --zone=$ZONE +""" + + + with open(_MAINTENANCE_SBATCH_SCRIPT_PATH, "w") as f: + f.write(perform_maintenance) + + util.chown_slurm(_MAINTENANCE_SBATCH_SCRIPT_PATH, mode=0o755) + + +def update_system_config(file, content): + """Add system defaults options for service files""" + sysconfig = Path("/etc/sysconfig") + default = Path("/etc/default") + + if sysconfig.exists(): + conf_dir = sysconfig + elif default.exists(): + conf_dir = default + else: + raise Exception("Cannot determine system configuration directory.") + + slurmd_file = Path(conf_dir, file) + slurmd_file.write_text(content) + +def _symlink_mysql_datadir(lkp: util.Lookup) -> None: + """ Symlink /var/lib/mysql to controller state disk if needed. """ + if not lkp.cfg.controller_state_disk.device_name: + return + + datadir = Path("/var/lib/mysql") + dst = slurmdirs.state / "mysql" + + if dst.exists(): + run(f"rm -rf {datadir}") + else: + shutil.move(datadir, dst) + + datadir.symlink_to(dst, target_is_directory=True) + shutil.chown(datadir, user="mysql", group="mysql") + run(f"chown -R mysql:mysql {dst}") + +def configure_mysql(lkp: util.Lookup) -> None: + cnfdir = Path("/etc/my.cnf.d") + if not cnfdir.exists(): + cnfdir = Path("/etc/mysql/conf.d") + if not (cnfdir / "mysql_slurm.cnf").exists(): + (cnfdir / "mysql_slurm.cnf").write_text( + """ +[mysqld] +bind-address=127.0.0.1 +innodb_buffer_pool_size=1024M +innodb_log_file_size=64M +innodb_lock_wait_timeout=900 +""" + ) + + run("systemctl stop mariadb", timeout=30) + _symlink_mysql_datadir(lkp) + + run("systemctl enable mariadb", timeout=30) + run("systemctl restart mariadb", timeout=30) + + db_name = "slurm_acct_db" + + + cmd = "mysql -u root -e" + for host in ("localhost", lkp.control_host): + run(f"""{cmd} "drop user if exists 'slurm'@'{host}'";""", timeout=30) + run(f"""{cmd} "create user 'slurm'@'{host}'";""", timeout=30) + run(f"""{cmd} "grant all on {db_name}.* TO 'slurm'@'{host}'";""", timeout=30) + + +def configure_dirs(): + for p in dirs.values(): + util.mkdirp(p) + + for p in (dirs.slurm, dirs.scripts, dirs.custom_scripts): + util.chown_slurm(p) + + for p in slurmdirs.values(): + util.mkdirp(p) + util.chown_slurm(p) + + for sl, tgt in ( # create symlinks + (Path("/etc/slurm"), slurmdirs.etc), + (dirs.scripts / "etc", slurmdirs.etc), + (dirs.scripts / "log", dirs.log), + ): + if sl.exists() and sl.is_symlink(): + sl.unlink() + sl.symlink_to(tgt) + + # copy auxiliary scripts + for dst_folder, src_file in ((lookup().cfg.slurm_bin_dir, + Path("sort_nodes.py")), + (dirs.custom_scripts / "task_prolog.d", + Path("tools/task-prolog")), + (dirs.custom_scripts / "task_epilog.d", + Path("tools/task-epilog"))): + dst = Path(dst_folder) / src_file.name + util.mkdirp(dst.parent) + shutil.copyfile(util.scripts_dir / src_file, dst) + os.chmod(dst, 0o755) + + +def self_report_controller_address(lkp: util.Lookup) -> None: + if not lkp.cfg.controller_network_attachment: + return # only self report address if network attachment is used + data = { "slurm_control_addr": lkp.cfg.slurm_control_addr } + bucket, prefix = util._get_bucket_and_common_prefix() + blob = util.storage_client().bucket(bucket).blob(f"{prefix}/controller_addr.yaml") + with blob.open('w') as f: + f.write(yaml.dump(data)) + +def setup_controller(): + """Run controller setup""" + log.info("Setting up controller") + lkp = util.lookup() + util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) + install_custom_scripts() + conf.gen_controller_configs(lkp) + + if lkp.cfg.controller_state_disk.device_name != None: + mount_save_state_disk() + + setup_jwt_key() + setup_key(lkp) + + setup_sudoers() + setup_network_storage() + + run_custom_scripts() + + if not lkp.cfg.cloudsql_secret: + configure_mysql(lkp) + + run("systemctl enable slurmdbd", timeout=30) + run("systemctl restart slurmdbd", timeout=30) + + # Wait for slurmdbd to come up + time.sleep(5) + + sacctmgr = f"{slurmdirs.prefix}/bin/sacctmgr -i" + result = run( + f"{sacctmgr} add cluster {lkp.cfg.slurm_cluster_name}", timeout=30, check=False + ) + if "already exists" in result.stdout: + log.info(result.stdout) + elif result.returncode > 1: + result.check_returncode() # will raise error + + run("systemctl enable slurmctld", timeout=30) + run("systemctl restart slurmctld", timeout=30) + + run("systemctl enable slurmrestd", timeout=30) + run("systemctl restart slurmrestd", timeout=30) + + # Export at the end to signal that everything is up + run("systemctl enable nfs-server", timeout=30) + run("systemctl start nfs-server", timeout=30) + + setup_nfs_exports() + run("systemctl enable --now slurmcmd.timer", timeout=30) + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status slurmdbd", timeout=30) + run("systemctl status slurmctld", timeout=30) + run("systemctl status slurmrestd", timeout=30) + + try: + slurmsync.sync_instances() + except Exception: + log.exception("Failed to sync instances, will try next time.") + + run("systemctl enable slurm_load_bq.timer", timeout=30) + run("systemctl start slurm_load_bq.timer", timeout=30) + run("systemctl status slurm_load_bq.timer", timeout=30) + + # Add script to perform maintenance + setup_maintenance_script() + + self_report_controller_address(lkp) + + log.info("Done setting up controller") + pass + + +def setup_login(): + """run login node setup""" + log.info("Setting up login") + + lkp = lookup() + slurmctld_host = f"{lkp.control_host}" + if lkp.control_addr: + slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" + sackd_options = [ + f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', + ] + sysconf = f"""SACKD_OPTIONS='{" ".join(sackd_options)}'""" + update_system_config("sackd", sysconf) + install_custom_scripts() + + setup_network_storage() + setup_sudoers() + if not lkp.cfg.enable_slurm_auth: + run("systemctl restart munge", timeout=30) + run("systemctl enable sackd", timeout=30) + run("systemctl restart sackd", timeout=30) + run("systemctl enable --now slurmcmd.timer", timeout=30) + + run_custom_scripts() + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status sackd", timeout=30) + + log.info("Done setting up login") + + +def setup_compute(): + """run compute node setup""" + log.info("Setting up compute") + + lkp = lookup() + util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) + slurmctld_host = f"{lkp.control_host}" + if lkp.control_addr: + slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" + slurmd_options = [ + f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', + ] + + try: + slurmd_feature = util.instance_metadata("attributes/slurmd_feature", silent=True) + except util.MetadataNotFoundError: + slurmd_feature = None + + if slurmd_feature is not None: + slurmd_options.append(f'--conf="Feature={slurmd_feature}"') + slurmd_options.append("-Z") + + sysconf = f"""SLURMD_OPTIONS='{" ".join(slurmd_options)}'""" + update_system_config("slurmd", sysconf) + install_custom_scripts() + + setup_nss_slurm() + setup_network_storage() + + has_gpu = run("lspci | grep --ignore-case 'NVIDIA' | wc -l", shell=True).returncode + if has_gpu: + run("nvidia-smi") + + run_custom_scripts() + + setup_sudoers() + if not lkp.cfg.enable_slurm_auth: + run("systemctl restart munge", timeout=30) + run("systemctl enable slurmd", timeout=30) + run("systemctl restart slurmd", timeout=30) + run("systemctl enable --now slurmcmd.timer", timeout=30) + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status slurmd", timeout=30) + + log.info("Done setting up compute") + +def setup_cloud_ops() -> None: + """Add health checks, deployment info, and updated setup path to cloud ops config.""" + cloudOpsStatus = run( + "systemctl is-active --quiet google-cloud-ops-agent.service", check=False + ).returncode + + if cloudOpsStatus != 0: + return + + with open("/etc/google-cloud-ops-agent/config.yaml", "r") as f: + file = yaml.safe_load(f) + + # Update setup receiver path + file["logging"]["receivers"]["setup"]["include_paths"] = ["/var/log/slurm/setup.log"] + + cluster_info = { + 'type':'modify_fields', + 'fields': { + 'labels."cluster_name"':{ + 'static_value':f"{lookup().cfg.slurm_cluster_name}" + }, + 'labels."hostname"':{ + 'static_value': f"{lookup().hostname}" + } + } + } + + file["logging"]["processors"]["add_cluster_info"] = cluster_info + file["logging"]["service"]["pipelines"]["slurmlog_pipeline"]["processors"].append("add_cluster_info") + file["logging"]["service"]["pipelines"]["slurmlog2_pipeline"]["processors"].append("add_cluster_info") + + with open("/etc/google-cloud-ops-agent/config.yaml", "w") as f: + yaml.safe_dump(file, f, sort_keys=False) + + retries = 2 + for _ in range(retries): + try: + run("systemctl restart google-cloud-ops-agent.service", timeout=120) + break + except subprocess.TimeoutExpired: + log.error("google-cloud-ops-agent.service did not restart within 120s.") + result=run("cat /var/log/google-cloud-ops-agent/subagents/logging-module.log", timeout=120, shell=True) + if result.stdout: + log.error(f"Logs for google-cloud-ops-agent (logging-module.log file):\n{result.stdout}") + raise + + +def main(): + start_motd() + + log.info("Starting setup, fetching config") + sleep_seconds = 5 + while True: + try: + _, cfg = util.fetch_config() + util.update_config(cfg) + break + except util.DeffetiveStoredConfigError as e: + log.warning(f"config is not ready yet: {e}, sleeping for {sleep_seconds}s") + except Exception as e: + log.exception(f"unexpected error while fetching config, sleeping for {sleep_seconds}s") + time.sleep(sleep_seconds) + log.info("Config fetched") + setup_cloud_ops() + configure_dirs() + # call the setup function for the instance type + { + "controller": setup_controller, + "compute": setup_compute, + "login": setup_login, + }.get( + lookup().instance_role, + lambda: log.fatal(f"Unknown node role: {lookup().instance_role}"))() + + end_motd() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--slurmd-feature", dest="slurmd_feature", help="Unused, to be removed.") + _ = util.init_log_and_parse(parser) + + try: + main() + except Exception: + log.exception("Aborting setup...") + failed_motd() diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py new file mode 100644 index 0000000000..095f42e758 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py @@ -0,0 +1,327 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import os +import sys +import stat +import time +import logging +import uuid + +import shutil +from pathlib import Path +from concurrent.futures import as_completed +from addict import Dict as NSDict # type: ignore + +import util +from util import NSMount, lookup, run, dirs, separate +from more_executors import Executors, ExceptionRetryPolicy + + +log = logging.getLogger() + +def mounts_by_local(mounts: list[NSMount]) -> dict[str, NSMount]: + """convert list of mounts to dict of mounts, local_mount as key""" + return {str(m.local_mount.resolve()): m for m in mounts} + + +def _get_default_mounts(lkp: util.Lookup) -> list[NSMount]: + if lkp.cfg.disable_default_mounts: + return [] + return [ + NSMount( + server_ip=lkp.controller_mount_server_ip(), + remote_mount=path, + local_mount=path, + fs_type="nfs", + mount_options="defaults,hard,intr", + ) + for path in ( + dirs.home, + dirs.apps, + ) + ] + +def get_slurm_bucket_mount() -> NSMount: + bucket, path = util._get_bucket_and_common_prefix() + return NSMount( + fs_type="gcsfuse", + server_ip="", + remote_mount=Path(bucket), + local_mount=dirs.slurm_bucket_mount, + mount_options=f"defaults,_netdev,implicit_dirs,only_dir={path}", + ) + +def resolve_network_storage() -> List[NSMount]: + """Combine appropriate network_storage fields to a single list""" + lkp = lookup() + + # create dict of mounts, local_mount: mount_info + mounts = mounts_by_local(_get_default_mounts(lkp)) + + if lkp.is_controller and util.should_mount_slurm_bucket(): + mounts.update(mounts_by_local([get_slurm_bucket_mount()])) + + # On non-controller instances, entries in network_storage could overwrite + # default exports from the controller. Be careful, of course + common = [lkp.normalize_ns_mount(m) for m in lkp.cfg.network_storage] + mounts.update(mounts_by_local(common)) + + if lkp.is_login_node: + login_group = lkp.cfg.login_groups[util.instance_login_group()] + login_ns = [lkp.normalize_ns_mount(m) for m in login_group.network_storage] + mounts.update(mounts_by_local(login_ns)) + + if lkp.instance_role == "compute": + try: + nodeset = lkp.node_nodeset() + except Exception: + pass # external nodename, skip lookup + else: + nodeset_ns = [lkp.normalize_ns_mount(m) for m in nodeset.network_storage] + mounts.update(mounts_by_local(nodeset_ns)) + + return list(mounts.values()) + + +def is_controller_mount(mount) -> bool: + # NOTE: Valid Lustre server_ip can take the form of '@tcp' + server_ip = mount.server_ip.split("@")[0] + mount_addr = util.host_lookup(server_ip) + return mount_addr == lookup().control_host_addr + +def setup_network_storage(): + """prepare network fs mounts and add them to fstab""" + log.info("Set up network storage") + + all_mounts = resolve_network_storage() + if lookup().is_controller: + mounts, _ = separate(is_controller_mount, all_mounts) + else: + mounts = all_mounts + + # Determine fstab entries and write them out + fstab_entries = [] + for mount in mounts: + local_mount = mount.local_mount + fs_type = mount.fs_type + server_ip = mount.server_ip or "" + src = mount.remote_mount if fs_type == "gcsfuse" else f"{server_ip}:{mount.remote_mount}" + + log.info(f"Setting up mount ({fs_type}) {src} to {local_mount}") + util.mkdirp(local_mount) + + mount_options = mount.mount_options.split(",") if mount.mount_options else [] + if "_netdev" not in mount_options: + mount_options += ["_netdev"] + options_line = ",".join(mount_options) + + + fstab_entries.append(f"{src} {local_mount} {fs_type} {options_line} 0 0") + + fstab = Path("/etc/fstab") + if not Path(fstab.with_suffix(".bak")).is_file(): + shutil.copy2(fstab, fstab.with_suffix(".bak")) + shutil.copy2(fstab.with_suffix(".bak"), fstab) + with open(fstab, "a") as f: + f.write("\n") + for entry in fstab_entries: + f.write(entry) + f.write("\n") + + mount_fstab(mounts, log) + if lookup().cfg.enable_slurm_auth: + slurm_key_mount_handler() + else: + munge_mount_handler() + + +def mount_fstab(mounts: list[NSMount], log): + """Wait on each mount, then make sure all fstab is mounted""" + def mount_path(path: Path): + log.info(f"Waiting for '{path}' to be mounted...") + try: + run(f"mount {path}", timeout=120) + except Exception as e: + exc_type, _, _ = sys.exc_info() + log.error(f"mount of path '{path}' failed: {exc_type}: {e}") + raise e + log.info(f"Mount point '{path}' was mounted.") + + MAX_MOUNT_TIMEOUT = 60 * 5 + future_list = [] + retry_policy = ExceptionRetryPolicy( + max_attempts=120, exponent=1.6, sleep=1.0, max_sleep=16.0 + ) + with Executors.thread_pool().with_timeout(MAX_MOUNT_TIMEOUT).with_retry( + retry_policy=retry_policy + ) as exe: + for m in mounts: + future = exe.submit(mount_path, m.local_mount) + future_list.append(future) + + # Iterate over futures, checking for exceptions + for future in as_completed(future_list): + try: + future.result() + except Exception as e: + raise e + + +def munge_mount_handler(): + if lookup().is_controller: + return + mnt = lookup().munge_mount + + log.info(f"Mounting munge share to: {mnt.local_mount}") + mnt.local_mount.mkdir() + if mnt.fs_type == "gcsfuse": + cmd = [ + "gcsfuse", + f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, + mnt.server_ip, + str(mnt.local_mount), + ] + else: + cmd = [ + "mount", + f"--types={mnt.fs_type}", + f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, + f"{mnt.server_ip}:{mnt.remote_mount}", + str(mnt.local_mount), + ] + # wait max 240s for munge mount + timeout = 240 + for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): + try: + run(cmd, timeout=timeout) + break + except Exception as e: + log.error( + f"munge mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" + ) + time.sleep(wait) + err = e + continue + else: + raise err + + munge_key = Path(dirs.munge / "munge.key") + log.info(f"Copy munge.key from: {mnt.local_mount}") + shutil.copy2(Path(mnt.local_mount / "munge.key"), munge_key) + + log.info("Restrict permissions of munge.key") + shutil.chown(munge_key, user="munge", group="munge") + os.chmod(munge_key, stat.S_IRUSR) + + log.info(f"Unmount {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + run(f"fusermount -u {mnt.local_mount}", timeout=120) + else: + run(f"umount {mnt.local_mount}", timeout=120) + shutil.rmtree(mnt.local_mount) + +def slurm_key_mount_handler(): + if lookup().is_controller: + return + mnt = lookup().slurm_key_mount + + log.info(f"Mounting slurm_key share to: {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + cmd = [ + "gcsfuse", + f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, + mnt.server_ip, + str(mnt.local_mount), + ] + else: + cmd = [ + "mount", + f"--types={mnt.fs_type}", + f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, + f"{mnt.server_ip}:{mnt.remote_mount}", + str(mnt.local_mount), + ] + timeout = 120 # wait max 120s to mount + for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): + try: + run(cmd, timeout=timeout) + break + except Exception as e: + log.error( + f"slurm key mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" + ) + time.sleep(wait) + err = e + continue + else: + raise err + + file_name = "slurm.key" + dst = Path(util.slurmdirs.etc / file_name) + log.info(f"Copy slurm.key from: {mnt.local_mount}") + shutil.copy2(mnt.local_mount / file_name, dst) + + log.info("Restrict permissions of slurm.key") + util.chown_slurm(dst, mode=0o400) + + log.info(f"Unmount {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + run(f"fusermount -u {mnt.local_mount}", timeout=120) + else: + run(f"umount {mnt.local_mount}", timeout=120) + shutil.rmtree(mnt.local_mount) + + +def setup_nfs_exports(): + """nfs export all needed directories""" + lkp = util.lookup() + assert lkp.is_controller + + # The controller only needs to set up exports for cluster-internal mounts + exported_mounts = [m for m in resolve_network_storage() if is_controller_mount(m)] + + # key by remote mount path since that is what needs exporting + to_export = {m.remote_mount: "*(rw,no_subtree_check,no_root_squash)" for m in exported_mounts} + + key_mount = lkp.slurm_key_mount if lkp.cfg.enable_slurm_auth else lkp.munge_mount + if is_controller_mount(key_mount): + # Export key mount as read-only + to_export[key_mount.remote_mount] = "*(ro,no_subtree_check,no_root_squash)" + + if util.should_mount_slurm_bucket(): + mnt = get_slurm_bucket_mount() + # FSID is required for virtual filesystem that is not based on a device + # Also export it as read-only + fsid=str(uuid.uuid4()) + to_export[mnt.local_mount] = f"*(ro,no_subtree_check,no_root_squash,fsid={fsid})" + + # export path if corresponding selector boolean is True + lines = [] + for path,options in to_export.items(): + util.mkdirp(Path(path)) + run(rf"sed -i '\#{path}#d' /etc/exports", timeout=30) + lines.append(f"{path} {options}") + + exportsd = Path("/etc/exports.d") + util.mkdirp(exportsd) + with (exportsd / "slurm.exports").open("w") as f: + f.write("\n") + f.write("\n".join(lines)) + run("exportfs -a", timeout=30) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py new file mode 100644 index 0000000000..1bfdd5acce --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py @@ -0,0 +1,679 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import fcntl +import json +import logging +import re +import sys +import shlex +from datetime import datetime, timedelta +from itertools import chain +from pathlib import Path +from dataclasses import dataclass +from typing import Dict, Tuple, List, Optional, Protocol, Any +from functools import lru_cache + +import util +from util import ( + batch_execute, + ensure_execute, + execute_with_futures, + FutureReservation, + install_custom_scripts, + run, + separate, + to_hostlist, + NodeState, + chunked, + dirs, +) +from util import lookup +from suspend import delete_instances +import tpu +import conf +import watch_delete_vm_op + +log = logging.getLogger() + +TOT_REQ_CNT = 1000 +_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" + +class NodeAction(Protocol): + def apply(self, nodes:List[str]) -> None: + ... + + def __hash__(self): + ... + +@dataclass(frozen=True) +class NodeActionPowerUp(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to resume ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=power_up") + +@dataclass(frozen=True) +class NodeActionIdle(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} nodes to idle ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=resume") + +@dataclass(frozen=True) +class NodeActionPowerDown(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to power down ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=power_down") + + +@dataclass(frozen=True) +class NodeActionPowerDownForce(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to power down ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=power_down_force") + + +@dataclass(frozen=True) +class NodeActionDelete(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to delete ({hostlist})") + delete_instances(nodes) + +@dataclass(frozen=True) +class NodeActionPrempt(): + def apply(self, nodes:List[str]) -> None: + NodeActionDown(reason="Preempted instance").apply(nodes) + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances restarted ({hostlist})") + start_instances(nodes) + +@dataclass(frozen=True) +class NodeActionUnchanged(): + def apply(self, nodes:List[str]) -> None: + pass + +@dataclass(frozen=True) +class NodeActionDown(): + reason: str + + def apply(self, nodes: List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} nodes set down ({hostlist}) with reason={self.reason}") + run(f"{lookup().scontrol} update nodename={hostlist} state=down reason={shlex.quote(self.reason)}") + +@dataclass(frozen=True) +class NodeActionUnknown(): + slurm_state: Optional[NodeState] + instance_state: Optional[str] + + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.error(f"{len(nodes)} nodes have unexpected {self.slurm_state} and instance state:{self.instance_state}, ({hostlist})") + +def start_instance_op(node: str) -> Any: + inst = lookup().instance(node) + assert inst + + return lookup().compute.instances().start( + project=lookup().project, + zone=inst.zone, + instance=inst.name, + ) + + +def start_instances(node_list): + log.info("{} instances to start ({})".format(len(node_list), ",".join(node_list))) + lkp = lookup() + # TODO: use code from resume.py to assign proper placement + normal, tpu_nodes = separate(lkp.node_is_tpu, node_list) + ops = {node: start_instance_op(node) for node in normal} + + done, failed = batch_execute(ops) + + tpu_start_data = [] + for ns, nodes in util.groupby_unsorted(tpu_nodes, lkp.node_nodeset_name): + tpuobj = tpu.TPU.make(ns, lkp) + for snodes in chunked(nodes, n=tpuobj.vmcount): + tpu_start_data.append({"tpu": tpuobj, "node": snodes}) + execute_with_futures(tpu.start_tpu, tpu_start_data) + + +def _find_dynamic_node_status() -> NodeAction: + # TODO: cover more cases: + # * delete dead dynamic nodes + # * delete orhpaned instances + return NodeActionUnchanged() # don't touch dynamic nodes + +def get_fr_action(fr: FutureReservation, state:Optional[NodeState]) -> Optional[NodeAction]: + now = util.now() + if state is None: + return None # handle like any other node + if fr.start_time < now < fr.end_time: + return None # handle like any other node + + if state.base == "DOWN": + return NodeActionUnchanged() + if fr.start_time >= now: + msg = f"Waiting for reservation:{fr.name} to start at {fr.start_time}" + else: + msg = f"Reservation:{fr.name} is after its end-time" + return NodeActionDown(reason=msg) + +def _find_tpu_node_action(nodename, state) -> NodeAction: + lkp = lookup() + tpuobj = tpu.TPU.make(lkp.node_nodeset_name(nodename), lkp) + inst = tpuobj.get_node(nodename) + # If we do not find the node but it is from a Tpu that has multiple vms look for the master node + if inst is None and tpuobj.vmcount > 1: + # Get the tpu slurm nodelist of the nodes in the same tpu group as nodename + nodelist = run( + f"{lkp.scontrol} show topo {nodename}" + + " | awk -F'=' '/Level=0/ { print $NF }'", + shell=True, + ).stdout + l_nodelist = util.to_hostnames(nodelist) + group_names = set(l_nodelist) + # get the list of all the existing tpus in the nodeset + tpus_list = set(tpuobj.list_node_names()) + # In the intersection there must be only one node that is the master + tpus_int = list(group_names.intersection(tpus_list)) + if len(tpus_int) > 1: + log.error( + f"More than one cloud tpu node for tpu group {nodelist}, there should be only one that should be {l_nodelist[0]}, but we have found {tpus_int}" + ) + return NodeActionUnknown(slurm_state=state, instance_state=None) + if len(tpus_int) == 1: + inst = tpuobj.get_node(tpus_int[0]) + # if len(tpus_int ==0) this case is not relevant as this would be the case always that a TPU group is not running + if inst is None: + if state.base == "DOWN" and "POWERED_DOWN" in state.flags: + return NodeActionIdle() + if "POWERING_DOWN" in state.flags: + return NodeActionIdle() + if "COMPLETING" in state.flags: + return NodeActionDown(reason="Unbacked instance") + if state.base != "DOWN" and not ( + set(("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN")) + & state.flags + ): + return NodeActionDown(reason="Unbacked instance") + if lkp.is_static_node(nodename): + return NodeActionPowerUp() + elif ( + state is not None + and "POWERED_DOWN" not in state.flags + and "POWERING_DOWN" not in state.flags + and inst.state == tpu.TPU.State.STOPPED + ): + if tpuobj.preemptible: + return NodeActionPrempt() + if state.base != "DOWN": + return NodeActionDown(reason="Instance terminated") + elif ( + state is None or "POWERED_DOWN" in state.flags + ) and inst.state == tpu.TPU.State.READY: + return NodeActionDelete() + elif state is None: + # if state is None here, the instance exists but it's not in Slurm + return NodeActionUnknown(slurm_state=state, instance_state=inst.status) + + return NodeActionUnchanged() + +def get_node_action(nodename: str) -> NodeAction: + """Determine node/instance status that requires action""" + lkp = lookup() + state = lkp.node_state(nodename) + + if lkp.node_is_gke(nodename): + return NodeActionUnchanged() + + if lkp.node_is_fr(nodename): + fr = lkp.future_reservation(lkp.node_nodeset(nodename)) + assert fr + if action := get_fr_action(fr, state): + return action + + if lkp.node_is_dyn(nodename): + return _find_dynamic_node_status() + + if lkp.node_is_tpu(nodename): + return _find_tpu_node_action(nodename, state) + + # split below is workaround for VMs whose hostname is FQDN + inst = lkp.instance(nodename.split(".")[0]) + power_flags = frozenset( + ("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN") + ) & (state.flags if state is not None else set()) + + if (state is None) and (inst is None): + # Should never happen + return NodeActionUnknown(None, None) + if inst is None: + assert state is not None # to keep type-checker happy + if "POWERING_UP" in state.flags: + return NodeActionUnchanged() + if state.base == "DOWN" and "POWERED_DOWN" in state.flags: + return NodeActionIdle() + if "POWERING_DOWN" in state.flags: + return NodeActionIdle() + if "COMPLETING" in state.flags: + return NodeActionDown(reason="Unbacked instance") + if state.base != "DOWN" and not power_flags: + return NodeActionDown(reason="Unbacked instance") + if state.base == "DOWN" and not power_flags: + return NodeActionPowerDown() + if "NOT_RESPONDING" in state.flags: + return NodeActionPowerDown() + if "POWERED_DOWN" in state.flags and lkp.is_static_node(nodename): + return NodeActionPowerUp() + elif ( + state is not None + and "POWERED_DOWN" not in state.flags + and "POWERING_DOWN" not in state.flags + and inst.status == "TERMINATED" + ): + if inst.scheduling.preemptible: + return NodeActionPrempt() + if state.base != "DOWN": + return NodeActionDown(reason="Instance terminated") + elif (state is None or "POWERED_DOWN" in state.flags) and inst.status == "RUNNING": + log.info("%s is potential orphan node", nodename) + threshold = timedelta(seconds=90) + age = util.now() - inst.creation_timestamp + log.info(f"{nodename} state: {state}, age: {age}") + if age < threshold: + log.info(f"{nodename} not marked as orphan, it started less than {threshold.seconds}s ago ({age.seconds}s)") + return NodeActionUnchanged() + return NodeActionDelete() + elif state is None: + # if state is None here, the instance exists but it's not in Slurm + return NodeActionUnknown(slurm_state=state, instance_state=inst.status) + elif lkp.is_flex_node(nodename) and "POWERING_UP" in state.flags: + threshold = timedelta(seconds=int(lkp.cfg.compute_startup_scripts_timeout) * 2) #extra buffer for unexpectedly long startup scripts + if util.now() - inst.creation_timestamp > threshold: + log.info(f"{nodename} was unable to join the cluster after {threshold.seconds}s, potential failure on VM startup. Powering down...") + return NodeActionPowerDownForce() + return NodeActionUnchanged() + + +def delete_resource_policies(links: list[str], lkp: util.Lookup) -> None: + requests = {} + for link in links: + name = util.trim_self_link(link) + region = util.parse_self_link(link).region + requests[name] = lkp.compute.resourcePolicies().delete(project=lkp.project, region=region, resourcePolicy=name) + + def swallow_err(_: str) -> None: + pass + + done, failed = batch_execute(requests, log_err=swallow_err) + if failed: + # Filter out resourceInUseByAnotherResource errors , they are expected to happen + def ignore_err(e) -> bool: + return "resourceInUseByAnotherResource" in str(e) + + failures = [f"{n}: {e}" for n, (_, e) in failed.items() if not ignore_err(e)] + if failures: + log.error(f"some placement groups failed to delete: {failures}") + log.info( + f"deleted {len(done)} of {len(links)} placement groups ({to_hostlist(done.keys())})" + ) + + + +@lru_cache +def _get_resource_policies_in_region(lkp: util.Lookup, region: str) -> list[Any]: + res = [] + act = lkp.compute.resourcePolicies() + op = act.list(project=lkp.project, region=region) + prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-" + while op is not None: + result = ensure_execute(op) + res.extend([p for p in result.get("items", []) if p.get("name", "").startswith(prefix)]) + op = act.list_next(op, result) + return res + + +@lru_cache +def _get_resource_policies(lkp: util.Lookup) -> list[Any]: + res = [] + for region in lkp.cluster_regions(): + res.extend(_get_resource_policies_in_region(lkp, region)) + return res + +def sync_placement_groups(): + """Delete placement policies that are for jobs that have completed/terminated""" + keep_states = frozenset( + [ + "RUNNING", + "CONFIGURING", + "STOPPED", + "SUSPENDED", + "COMPLETING", + "PENDING", + ] + ) + + lkp = lookup() + keep_jobs = { + str(job.id) + for job in lkp.get_jobs() + if job.job_state in keep_states + } + keep_jobs.add("0") # Job 0 is a placeholder for static node placement + + to_delete = [] + pg_regex = re.compile( + rf"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-(?P[^\s\-]+)-(?P\d+)-(?P\d+)" + ) + + for pg in _get_resource_policies(lkp): + name = pg["name"] + + if (mtch := pg_regex.match(name)) is None: + log.warning(f"Unexpected resource policy {name=}") + continue + if mtch.group("job_id") not in keep_jobs: + to_delete.append(pg["selfLink"]) + + if to_delete: + delete_resource_policies(to_delete, lkp) + + +def sync_instances(): + compute_instances = { + name for name, inst in lookup().instances().items() if inst.role == "compute" + } + slurm_nodes = set(lookup().slurm_nodes().keys()) + log.debug(f"reconciling {len(compute_instances)} GCP instances and {len(slurm_nodes)} Slurm nodes.") + + for action, nodes in util.groupby_unsorted(list(compute_instances | slurm_nodes), get_node_action): + action.apply(list(nodes)) + + +def reconfigure_slurm(): + update_msg = "*** slurm configuration was updated ***" + if lookup().cfg.hybrid: + # terraform handles generating the config.yaml, don't do it here + return + + upd, cfg_new = util.fetch_config() + if not upd: + log.debug("No changes in config detected.") + return + log.debug("Changes in config detected. Reconfiguring Slurm now.") + util.update_config(cfg_new) + + if lookup().is_controller: + conf.gen_controller_configs(lookup()) + log.info("Restarting slurmctld to make changes take effect.") + try: + # TODO: consider removing "restart" since "reconfigure" should restart slurmctld as well + run("sudo systemctl restart slurmctld.service", check=False) + util.scontrol_reconfigure(lookup()) + except Exception: + log.exception("failed to reconfigure slurmctld") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + elif lookup().instance_role_safe == "compute": + log.info("Restarting slurmd to make changes take effect.") + run("systemctl restart slurmd") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + elif lookup().is_login_node: + log.info("Restarting sackd to make changes take effect.") + run("systemctl restart sackd") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + + +def update_topology(lkp: util.Lookup) -> None: + if conf.topology_plugin(lkp) != conf.TOPOLOGY_PLUGIN_TREE: + return + updated, summary = conf.gen_topology_conf(lkp) + if updated: + log.info("Topology configuration updated. Reconfiguring Slurm.") + util.scontrol_reconfigure(lkp) + # Safe summary only after Slurm got reconfigured, so summary reflects Slurm POV + summary.dump(lkp) + + +def delete_reservation(lkp: util.Lookup, reservation_name: str) -> None: + util.run(f"{lkp.scontrol} delete reservation {reservation_name}") + + +def create_reservation(lkp: util.Lookup, reservation_name: str, node: str, start_time: datetime) -> None: + # Format time to be compatible with slurm reservation. + formatted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S') + + util.run(f"{lkp.scontrol} create reservation user=slurm starttime={formatted_start_time} duration=180 nodes={node} reservationname={reservation_name} flags=maint,ignore_jobs") + + +def get_slurm_reservation_maintenance(lkp: util.Lookup) -> Dict[str, datetime]: + res = util.run(f"{lkp.scontrol} show reservation --json") + all_reservations = json.loads(res.stdout) + reservation_map = {} + + for reservation in all_reservations['reservations']: + name = reservation.get('name') + nodes = reservation.get('node_list') + time_epoch = reservation.get('start_time', {}).get('number') + + if name is None or nodes is None or time_epoch is None: + continue + + if reservation.get('node_count') != 1: + continue + + if name != f"{nodes}_maintenance": + continue + + reservation_map[name] = datetime.fromtimestamp(time_epoch) + + return reservation_map + +@lru_cache +def get_upcoming_maintenance(lkp: util.Lookup) -> Dict[str, Tuple[str, datetime]]: + upc_maint_map = {} + + for node, inst in lkp.instances().items(): + if inst.resource_status.upcoming_maintenance: + upc_maint_map[node + "_maintenance"] = (node, inst.resource_status.upcoming_maintenance.window_start_time) + + return upc_maint_map + + +def sync_maintenance_reservation(lkp: util.Lookup) -> None: + upc_maint_map = get_upcoming_maintenance(lkp) # map reservation_name -> (node_name, time) + log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") + + curr_reservation_map = get_slurm_reservation_maintenance(lkp) # map reservation_name -> time + log.debug(f"curr-reservation-map: {curr_reservation_map}") + + del_reservation = set(curr_reservation_map.keys() - upc_maint_map.keys()) + create_reservation_map = {} + + for res_name, (node, start_time) in upc_maint_map.items(): + try: + enabled = lkp.node_nodeset(node).enable_maintenance_reservation + except Exception: + enabled = False + + if not enabled: + if res_name in curr_reservation_map: + del_reservation.add(res_name) + continue + + if res_name in curr_reservation_map: + diff = curr_reservation_map[res_name] - start_time + if abs(diff) <= timedelta(seconds=1): + continue + else: + del_reservation.add(res_name) + create_reservation_map[res_name] = (node, start_time) + else: + create_reservation_map[res_name] = (node, start_time) + + log.debug(f"del-reservation: {del_reservation}") + for res_name in del_reservation: + delete_reservation(lkp, res_name) + + log.debug(f"create-reservation-map: {create_reservation_map}") + for res_name, (node, start_time) in create_reservation_map.items(): + create_reservation(lkp, res_name, node, start_time) + + +def delete_maintenance_job(job_name: str) -> None: + util.run(f"scancel --name={job_name}") + + +def create_maintenance_job(job_name: str, node: str) -> None: + util.run(f"sbatch --job-name={job_name} --nodelist={node} {_MAINTENANCE_SBATCH_SCRIPT_PATH}") + + +def get_slurm_maintenance_job(lkp: util.Lookup) -> Dict[str, str]: + jobs = {} + + for job in lkp.get_jobs(): + if job.name is None or job.required_nodes is None or job.job_state is None: + continue + + if job.name != f"{job.required_nodes}_maintenance": + continue + + if job.job_state != "PENDING": + continue + + jobs[job.name] = job.required_nodes + + return jobs + + +def sync_opportunistic_maintenance(lkp: util.Lookup) -> None: + upc_maint_map = get_upcoming_maintenance(lkp) # map job_name -> (node_name, time) + log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") + + curr_jobs = get_slurm_maintenance_job(lkp) # map job_name -> node. + log.debug(f"curr-maintenance-job-map: {curr_jobs}") + + del_jobs = set(curr_jobs.keys() - upc_maint_map.keys()) + create_jobs = {} + + for job_name, (node, _) in upc_maint_map.items(): + try: + enabled = lkp.node_nodeset(node).enable_opportunistic_maintenance + except Exception: + enabled = False + + if not enabled: + if job_name in curr_jobs: + del_jobs.add(job_name) + continue + + if job_name not in curr_jobs: + create_jobs[job_name] = node + + log.debug(f"del-maintenance-job: {del_jobs}") + for job_name in del_jobs: + delete_maintenance_job(job_name) + + log.debug(f"create-maintenance-job: {create_jobs}") + for job_name, node in create_jobs.items(): + create_maintenance_job(job_name, node) + + + +def sync_flex_migs(lkp: util.Lookup) -> None: + pass + + +def process_messages(lkp: util.Lookup) -> None: + try: + watch_delete_vm_op.watch_vm_delete_ops(lkp) + except: + log.exception("failed during watching delete VM operations") + + +def main(): + lkp = lookup() + if util.should_mount_slurm_bucket() and not lkp.is_controller: + return + try: + reconfigure_slurm() + except Exception: + log.exception("failed to reconfigure slurm") + if lkp.is_controller: + try: + process_messages(lkp) + except: + log.exception("failed to process messages") + + try: + sync_instances() + except Exception: + log.exception("failed to sync instances") + + try: + sync_flex_migs(lkp) + except Exception: + log.exception("failed to sync DWS Flex MIGs") + + try: + sync_placement_groups() + except Exception: + log.exception("failed to sync placement groups") + + try: + update_topology(lkp) + except Exception: + log.exception("failed to update topology") + + try: + sync_maintenance_reservation(lkp) + except Exception: + log.exception("failed to sync slurm reservation for scheduled maintenance") + + try: + sync_opportunistic_maintenance(lkp) + except Exception: + log.exception("failed to sync opportunistic reservation for scheduled maintenance") + + + try: + # TODO: it performs 1 to 4 GCS list requests, + # use cached version, combine with `_list_config_blobs` + install_custom_scripts(check_hash=True) + except Exception: + log.exception("failed to sync custom scripts") + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + _ = util.init_log_and_parse(parser) + + pid_file = (Path("/tmp") / Path(__file__).name).with_suffix(".pid") + with pid_file.open("w") as fp: + try: + fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) + main() + except BlockingIOError: + sys.exit(0) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py new file mode 100644 index 0000000000..ae36c54222 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py @@ -0,0 +1,171 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script sorts nodes based on their `physicalHost`. + +See https://cloud.google.com/compute/docs/instances/use-compact-placement-policies + +You can reduce latency in tightly coupled HPC workloads (including distributed ML training) +by deploying them to machines that are located close together. +For example, if you deploy your workload on a single physical rack, you can expect lower latency +than if your workload is spread across multiple racks. +Sending data across multiple rack requires sending data through additional network switches. + +Example usage: +``` my_sbatch.sh +#SBATCH --ntasks-per-node=8 +#SBATCH --nodes=64 + +export SLURM_HOSTFILE=$(sort_nodes.py) + +srun -l hostname | sort +``` +""" +import os +import subprocess +import uuid +from typing import List, Optional, Dict +from collections import OrderedDict + +def order(paths: List[List[str]]) -> List[str]: + """ + Orders the leaves of the tree in a way that minimizes the sum of distance in between + each pair of neighboring nodes in the resulting order. + The resulting order will always start from the first node in the input list. + The ordering is "stable" with respect to the input order of the leaves i.e. + given a choice between two nodes (identical in other ways) it will select "nodelist-smallest" one. + + Returns a list of nodenames, ordered as described above. + """ + if not paths: return [] + class Vert: + "Represents a vertex in a *network* tree." + def __init__(self, name: str, parent: Optional["Vert"]): + self.name = name + self.parent = parent + # Use `OrderedDict` to preserve insertion order + # TODO: once we move to Python 3.7+ use regular `dict` since it has the same guarantee + self.children: OrderedDict = OrderedDict() + + # build a tree, children are ordered by insertion order + root = Vert("", None) + for path in paths: + n = root + for v in path: + if v not in n.children: + n.children[v] = Vert(v, n) + n = n.children[v] + + # walk the tree in insertion order, gather leaves + result = [] + def gather_nodes(v: Vert) -> None: + if not v.children: # this is a Slurm node + result.append(v.name) + for u in v.children.values(): + gather_nodes(u) + gather_nodes(root) + return result + + +class Instance: + def __init__(self, name: str, zone: str, physical_host: Optional[str]): + self.name = name + self.zone = zone + self.physical_host = physical_host + + +def make_path(node_name: str, inst: Optional[Instance]) -> List[str]: + if not inst: # node with unknown instance (e.g. hybrid cluster) + return ["unknown", node_name] + zone = f"zone_{inst.zone}" + if not inst.physical_host: # node without physical host info (e.g. no placement policy) + return [zone, "unknown", node_name] + + assert inst.physical_host.startswith("/"), f"Unexpected physicalHost: {inst.physical_host}" + parts = inst.physical_host[1:].split("/") + if len(parts) >= 4: + return [*parts, node_name] + return [zone, *parts, node_name] + + +def to_hostnames(nodelist: str) -> List[str]: + cmd = ["scontrol", "show", "hostnames", nodelist] + out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout + return [n.decode("utf-8") for n in out.splitlines()] + + +def get_instances(node_names: List[str]) -> Dict[str, Optional[Instance]]: + fmt = ( + "--format=csv[no-heading,separator=','](zone,resourceStatus.physicalHost,name)" + ) + cmd = ["gcloud", "compute", "instances", "list", fmt] + + scp = os.path.commonprefix(node_names) + if scp: + cmd.append(f"--filter=name~'{scp}.*'") + out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout + d = {} + for line in out.splitlines(): + zone, physical_host, name = line.decode("utf-8").split(",") + d[name] = Instance(name, zone, physical_host) + return {n: d.get(n) for n in node_names} + + +def main(args) -> None: + nodelist = args.nodelist or os.getenv("SLURM_NODELIST") + if not nodelist: + raise ValueError("nodelist is not provided and SLURM_NODELIST is not set") + + if args.ntasks_per_node is None: + args.ntasks_per_node = int(os.getenv("SLURM_NTASKS_PER_NODE", "") or 1) + assert args.ntasks_per_node > 0 + + output = args.output or f"hosts.{uuid.uuid4()}" + + node_names = to_hostnames(nodelist) + instannces = get_instances(node_names) + paths = [make_path(n, instannces[n]) for n in node_names] + ordered = order(paths) + + with open(output, "w") as f: + for node in ordered: + for _ in range(args.ntasks_per_node): + f.write(node) + f.write("\n") + print(output) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--nodelist", + type=str, + help="Slurm 'hostlist expression' of nodes to sort, if not set the value of SLURM_NODELIST environment variable will be used", + ) + parser.add_argument( + "--ntasks-per-node", + type=int, + help="""Number of times to repeat each node in resulting sorted list. +If not set, the value of SLURM_NTASKS_PER_NODE environment variable will be used, +if neither is set, defaults to 1""", + ) + parser.add_argument( + "--output", type=str, help="Output file to write, defaults to 'hosts.'" + ) + args = parser.parse_args() + main(args) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py new file mode 100644 index 0000000000..ecef70f1cc --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py @@ -0,0 +1,126 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Any +import argparse +import logging + +import util +from util import ( + log_api_request, + batch_execute, + to_hostlist, + separate, +) +from util import lookup +import tpu +import mig_flex +import watch_delete_vm_op + +log = logging.getLogger() + +TOT_REQ_CNT = 1000 + + +def truncate_iter(iterable, max_count): + end = "..." + _iter = iter(iterable) + for i, el in enumerate(_iter, start=1): + if i >= max_count: + yield end + break + yield el + + +def delete_instance_request(name: str) -> Any: + inst = lookup().instance(name) + assert inst + + request = lookup().compute.instances().delete( + project=lookup().project, + zone=inst.zone, + instance=name, + ) + log_api_request(request) + return request + + +def delete_instances(instances): + """delete instances individually""" + invalid, valid = separate(lambda inst: bool(lookup().instance(inst)), instances) + if len(invalid) > 0: + log.debug("instances do not exist: {}".format(",".join(invalid))) + if len(valid) == 0: + log.debug("No instances to delete") + return + + requests = {inst: delete_instance_request(inst) for inst in valid} + + log.info(f"to delete {len(valid)} instances ({to_hostlist(valid)})") + ops, failed = batch_execute(requests) + for node, (_, err) in failed.items(): + log.error(f"instance {node} failed to delete: {err}") + + log.info(f"deleting {len(ops)} instances {to_hostlist(ops.keys())}") + + topic = watch_delete_vm_op.watch_delete_vm_op_topic() + for node, op in ops.items(): + topic.publish(op, node) + + + + +def suspend_nodes(nodes: List[str]) -> None: + lkp = lookup() + other_nodes, tpu_nodes = util.separate(lkp.node_is_tpu, nodes) + bulk_nodes, flex_nodes = util.separate(lkp.is_flex_node, other_nodes) + + mig_flex.suspend_flex_nodes(flex_nodes, lkp) + delete_instances(bulk_nodes) + tpu.delete_tpu_instances(tpu_nodes) + + +def main(nodelist): + """main called when run as script""" + log.debug(f"SuspendProgram {nodelist}") + + # Filter out nodes not in config.yaml + other_nodes, pm_nodes = separate( + lookup().is_power_managed_node, util.to_hostnames(nodelist) + ) + if other_nodes: + log.debug( + f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" + ) + if pm_nodes: + log.debug(f"Suspending nodes '{to_hostlist(pm_nodes)}' from '{nodelist}'") + else: + log.debug("No cloud nodes to suspend") + return + + log.info(f"suspend {nodelist}") + suspend_nodes(pm_nodes) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("nodelist", help="list of nodes to suspend") + args = util.init_log_and_parse(parser) + + main(args.nodelist) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh new file mode 100644 index 0000000000..9079e4e4b0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PYTHON_SCRIPT="${SCRIPT_DIR}/suspend.py" + +# Capture all arguments passed by Slurm (the nodelist). +ALL_ARGS=("$@") + +"${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & +disown + +exit 0 diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py new file mode 100644 index 0000000000..0ce7fb5ec4 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py @@ -0,0 +1,116 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Any +import sys +from dataclasses import dataclass, field +from datetime import datetime + +SCRIPTS_DIR = "community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts" +if SCRIPTS_DIR not in sys.path: + sys.path.append(SCRIPTS_DIR) # TODO: make this more robust + +import util + + +SOME_TS = datetime.fromisoformat("2018-09-03T20:56:35.450686+00:00") +# TODO: use "real" classes once they are defined (instead of NSDict) + +@dataclass +class Placeholder: + pass + +@dataclass +class TstNodeset: + nodeset_name: str = "cantor" + node_count_static: int = 0 + node_count_dynamic_max: int = 0 + node_conf: dict[str, Any] = field(default_factory=dict) + instance_template: Optional[str] = None + reservation_name: Optional[str] = "" + zone_policy_allow: Optional[list[str]] = field(default_factory=list) + enable_placement: bool = True + placement_max_distance: Optional[int] = None + accelerator_topology: Optional[str] = "" + future_reservation: Optional[str] = "" + +@dataclass +class TstPartition: + partition_name: str = "euler" + partition_nodeset: list[str] = field(default_factory=list) + partition_nodeset_tpu: list[str] = field(default_factory=list) + enable_job_exclusive: bool = False + +@dataclass +class TstCfg: + slurm_cluster_name: str = "m22" + cloud_parameters: dict[str, Any] = field(default_factory=dict) + + partitions: dict[str, TstPartition] = field(default_factory=dict) + nodeset: dict[str, TstNodeset] = field(default_factory=dict) + nodeset_tpu: dict[str, TstNodeset] = field(default_factory=dict) + nodeset_dyn: dict[str, TstNodeset] = field(default_factory=dict) + + install_dir: Optional[str] = None + output_dir: Optional[str] = None + + prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + task_prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + task_epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + + +@dataclass +class TstTPU: # to prevent client initialization durint "TPU.__init__" + vmcount: int + +@dataclass +class TstMachineConf: + cpus: int + memory: int + sockets: int + sockets_per_board: int + cores_per_socket: int + boards: int + threads_per_core: int + + +@dataclass +class TstTemplateInfo: + gpu: Optional[util.AcceleratorInfo] + +def tstInstance(name: str, physical_host: Optional[str] = None): + return util.Instance( + name=name, + zone="anorien", + status="RUNNING", + creation_timestamp=SOME_TS, + resource_status=util.InstanceResourceStatus( + physical_host=physical_host, + upcoming_maintenance=None, + ), + scheduling=util.NSDict(), + role="compute", + metadata={}, + ) + +def make_to_hostnames_mock(tbl: Optional[dict[str, list[str]]]): + tbl = tbl or {} + + def se(k: str) -> list[str]: + if k not in tbl: + raise AssertionError(f"to_hostnames mock: unexpected nodelist: '{k}'") + return tbl[k] + + return se diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py new file mode 100644 index 0000000000..6bd6762748 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py @@ -0,0 +1,226 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from mock import Mock +from common import TstNodeset, TstCfg, TstMachineConf, TstTemplateInfo, Placeholder + +import addict # type: ignore +import conf +import util + + +def test_nodeset_tpu_lines(): + nodeset = TstNodeset( + "turbo", + node_count_static=2, + node_count_dynamic_max=3, + node_conf={"red": "velvet"}, + ) + assert conf.nodeset_tpu_lines(nodeset, util.Lookup(TstCfg())) == "\n".join( + [ + "NodeName=m22-turbo-[0-4] State=CLOUD red=velvet", + "NodeSet=turbo Nodes=m22-turbo-[0-4]", + ] + ) + + +def test_nodeset_lines(): + nodeset = TstNodeset( + "turbo", + node_count_static=2, + node_count_dynamic_max=3, + node_conf={"red": "velvet", "CPUs": 55}, + ) + lkp = util.Lookup(TstCfg()) + lkp.template_info = Mock(return_value=TstTemplateInfo( + gpu=util.AcceleratorInfo(type="Popov", count=33) + )) + mc = TstMachineConf( + cpus=5, + memory=6, + sockets=7, + sockets_per_board=8, + boards=9, + threads_per_core=10, + cores_per_socket=11, + ) + lkp.template_machine_conf = Mock(return_value=mc) # type: ignore[method-assign] + assert conf.nodeset_lines(nodeset, lkp) == "\n".join( + [ + "NodeName=m22-turbo-[0-4] State=CLOUD RealMemory=6 Boards=9 SocketsPerBoard=8 CoresPerSocket=11 ThreadsPerCore=10 CPUs=55 Gres=gpu:33 red=velvet", + "NodeSet=turbo Nodes=m22-turbo-[0-4]", + ] + ) + + +@pytest.mark.parametrize( + "value,want", + [ + ({"a": 1}, "a=1"), + ({"a": "two"}, "a=two"), + ({"a": [3, 4]}, "a=3,4"), + ({"a": ["five", "six"]}, "a=five,six"), + ({"a": None}, ""), + ({"a": ["seven", None, 8]}, "a=seven,8"), + ({"a": 1, "b": "two"}, "a=1 b=two"), + ({"a": 1, "b": None, "c": "three"}, "a=1 c=three"), + ({"a": 0, "b": None, "c": 0.0, "e": ""}, "a=0 c=0.0"), + ({"a": [0, 0.0, None, "X", "", "Y"]}, "a=0,0.0,X,,Y"), + ]) +def test_dict_to_conf(value: dict, want: str): + assert conf.dict_to_conf(value) == want + + + +@pytest.mark.parametrize( + "cfg,want", + [ + (TstCfg( + install_dir="ukulele", + ), + """LaunchParameters=enable_nss_slurm,use_interactive_step +SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend +SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +SlurmdTimeout=300 +UnkillableStepTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + (TstCfg( + install_dir="ukulele", + cloud_parameters={ + "no_comma_params": True, + "private_data": None, + "scheduler_parameters": None, + "resume_rate": None, + "resume_timeout": None, + "suspend_rate": None, + "suspend_timeout": None, + "unkillable_step_timeout": None, + "slurmd_timeout": None, + "topology_plugin": None, + "topology_param": None, + "tree_width": None, + }, + ), + """SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +SlurmdTimeout=300 +UnkillableStepTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + (TstCfg( + install_dir="ukulele", + cloud_parameters={ + "no_comma_params": True, + "private_data": [ + "events", + "jobs", + ], + "scheduler_parameters": [ + "bf_busy_nodes", + "bf_continue", + "ignore_prefer_validation", + "nohold_on_prolog_fail", + ], + "resume_rate": 1, + "resume_timeout": 2, + "suspend_rate": 3, + "suspend_timeout": 4, + "slurmd_timeout": 5, + "unkillable_step_timeout": 6, + "tree_width": 7, + "topology_plugin": "guess", + "topology_param": "yellow", + }, + ), + """PrivateData=events,jobs +SchedulerParameters=bf_busy_nodes,bf_continue,ignore_prefer_validation,nohold_on_prolog_fail +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=1 +ResumeTimeout=2 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=3 +SuspendTimeout=4 +SlurmdTimeout=5 +UnkillableStepTimeout=6 +TreeWidth=7 +TopologyPlugin=guess +TopologyParam=yellow"""), + (TstCfg( + install_dir="ukulele", + task_prolog_scripts=[Placeholder()], + task_epilog_scripts=[Placeholder()], + ), + """LaunchParameters=enable_nss_slurm,use_interactive_step +SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend +TaskProlog=/slurm/custom_scripts/task_prolog.d/task-prolog +TaskEpilog=/slurm/custom_scripts/task_epilog.d/task-epilog +SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +SlurmdTimeout=300 +UnkillableStepTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + ]) +def test_conflines(cfg, want): + assert conf.conflines(util.Lookup(cfg)) == want + + cfg.cloud_parameters = addict.Dict(cfg.cloud_parameters) + assert conf.conflines(util.Lookup(cfg)) == want + + +@pytest.mark.parametrize( + "cfg,gputype,gpucount,want", + [ + (TstCfg(), + "", + 0, + "\n"), + (TstCfg( + nodeset={"turbo": TstNodeset("turbo")} + ), + "Popov", + 8, + "Name=gpu Type=Popov File=/dev/nvidia[0-7]\n\n"), + ]) +def test_gen_cloud_gres_conf_lines(cfg, gputype, gpucount, want): + lkp = util.Lookup(cfg) + lkp.template_info = Mock(return_value=TstTemplateInfo( + gpu=util.AcceleratorInfo(type=gputype, count=gpucount) + )) + assert conf.gen_cloud_gres_conf_lines(lkp) == want diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py new file mode 100644 index 0000000000..77f1229605 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py @@ -0,0 +1,175 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import os +import pytest +import unittest.mock +import unittest +import tempfile + +from common import TstCfg, TstNodeset, TstPartition, TstTPU # needed to import util +import util +import resume +from resume import ResumeData, ResumeJobData, BulkChunk, PlacementAndNodes + +def test_get_resume_file_data_no_env(): + with unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": ""}): + assert resume.get_resume_file_data() is None + + +def test_get_resume_file_data(): + with tempfile.NamedTemporaryFile() as f: + f.write(b"""{ + "jobs": [ + { + "extra": null, + "job_id": 1, + "features": null, + "nodes_alloc": "green-[0-2]", + "nodes_resume": "green-[0-1]", + "oversubscribe": "OK", + "partition": "red", + "reservation": null + } + ], + "all_nodes_resume": "green-[0-1]" +}""") + f.flush() + with ( + unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": f.name}), + unittest.mock.patch("util.to_hostnames") as mock_to_hostnames, + ): + mock_to_hostnames.return_value = ["green-0", "green-1", "green-2"] + assert resume.get_resume_file_data() == ResumeData(jobs=[ + ResumeJobData( + job_id = 1, + partition="red", + nodes_alloc=["green-0", "green-1", "green-2"], + ) + ]) + mock_to_hostnames.assert_called_once_with("green-[0-2]") + + +@unittest.mock.patch("tpu.TPU.make") +@unittest.mock.patch("resume.create_placements") +def test_group_nodes_bulk(mock_create_placements, mock_tpu): + cfg = TstCfg( + nodeset={ + "n": TstNodeset(nodeset_name="n"), + }, + nodeset_tpu={ + "t": TstNodeset(nodeset_name="t"), + }, + partitions={ + "p1": TstPartition( + partition_name="p1", + enable_job_exclusive=True, + ), + "p2": TstPartition( + partition_name="p2", + partition_nodeset_tpu=["t"], + enable_job_exclusive=True, + ) + } + ) + lkp = util.Lookup(cfg) + + def mock_create_placements_se(nodes, excl_job_id, lkp): + args = (set(nodes), excl_job_id) + if ({'c-n-1', 'c-n-2', 'c-t-8', 'c-t-9'}, None) == args: + return [ + PlacementAndNodes("g0", ["c-n-1", "c-n-2"]), + PlacementAndNodes(None, ['c-t-8', 'c-t-9']), + ] + if ({"c-n-0", "c-n-8"}, 1) == args: + return [ + PlacementAndNodes("g10", ["c-n-0"]), + PlacementAndNodes("g11", ["c-n-8"]), + ] + if ({'c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5'}, 2) == args: + return [ + PlacementAndNodes(None, ['c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5']) + ] + raise AssertionError(f"unexpected invocation: '{args}'") + mock_create_placements.side_effect = mock_create_placements_se + + def mock_tpu_se(ns: str, lkp) -> TstTPU: + if ns == "t": + return TstTPU(vmcount=2) + raise AssertionError(f"unexpected invocation: '{ns}'") + mock_tpu.side_effect = mock_tpu_se + + got = resume.group_nodes_bulk( + ["c-n-0", "c-n-1", "c-n-2", "c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-8", "c-t-9"], + ResumeData(jobs=[ + ResumeJobData(job_id=1, partition="p1", nodes_alloc=["c-n-0", "c-n-8"]), + ResumeJobData(job_id=2, partition="p2", nodes_alloc=["c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-4", "c-t-5"]), + ]), lkp) + mock_create_placements.assert_called() + assert got == { + "c-n:jobNone:g0:0": BulkChunk( + nodes=["c-n-1", "c-n-2"], prefix="c-n", chunk_idx=0, excl_job_id=None, placement_group="g0"), + "c-n:job1:g10:0": BulkChunk( + nodes=["c-n-0"], prefix="c-n", chunk_idx=0, excl_job_id=1, placement_group="g10"), + "c-t:0": BulkChunk( + nodes=["c-t-8", "c-t-9"], prefix="c-t", chunk_idx=0, excl_job_id=None, placement_group=None), + "c-t:job2:0": BulkChunk( + nodes=["c-t-0", "c-t-1"], prefix="c-t", chunk_idx=0, excl_job_id=2, placement_group=None), + "c-t:job2:1": BulkChunk( + nodes=["c-t-2", "c-t-3"], prefix="c-t", chunk_idx=1, excl_job_id=2, placement_group=None), + } + + +@pytest.mark.parametrize( + "nodes,excl_job_id,expected", + [ + ( # TPU - no placements + ["c-t-0", "c-t-2"], 4, [PlacementAndNodes(None, ["c-t-0", "c-t-2"])] + ), + ( # disabled placements - no placemens + ["c-x-0", "c-x-2"], 4, [PlacementAndNodes(None, ["c-x-0", "c-x-2"])] + ), + ( # excl_job + ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], 4, [ + PlacementAndNodes("c-slurmgcp-managed-n-4-0", ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"]) + ] + ), + ( # no excl_job + ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], None, [ + PlacementAndNodes("c-slurmgcp-managed-n-0-0", ["c-n-0", "c-n-2"]), + PlacementAndNodes('c-slurmgcp-managed-n-0-1', ['c-n-2011']), + PlacementAndNodes(None, ["c-n-uno"]), + ] + ), + ], +) +def test_allocate_nodes_to_placements(nodes: list[str], excl_job_id: Optional[int], expected: list[PlacementAndNodes]): + cfg = TstCfg( + slurm_cluster_name="c", + nodeset={ + "n": TstNodeset(nodeset_name="n", enable_placement=True), + "x": TstNodeset(nodeset_name="x", enable_placement=False) + }, + nodeset_tpu={ + "t": TstNodeset(nodeset_name="t") + }) + lkp = util.Lookup(cfg) + + with unittest.mock.patch("resume.valid_placement_node") as mock_valid_placement_node: + mock_valid_placement_node.return_value = True + lkp.template_info = unittest.mock.Mock(return_value=unittest.mock.Mock(machine_type=unittest.mock.Mock(family="n1"))) + + assert resume._allocate_nodes_to_placements(nodes, excl_job_id, lkp) == expected diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py new file mode 100644 index 0000000000..df9f3a0137 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py @@ -0,0 +1,215 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import json +import mock +from pytest_unordered import unordered +from common import TstCfg, TstNodeset, TstTPU, tstInstance +import sort_nodes + +import util +import conf +import tempfile + +PRELUDE = """ +# Warning: +# This file is managed by a script. Manual modifications will be overwritten. + +""" + +def test_gen_topology_conf_empty(): + out_dir = tempfile.mkdtemp() + cfg = TstCfg(output_dir=out_dir) + conf.gen_topology_conf(util.Lookup(cfg)) + assert open(out_dir + "/cloud_topology.conf").read() == PRELUDE + "\n" + + +@mock.patch("tpu.TPU.make") +def test_gen_topology_conf(tpu_mock): + output_dir = tempfile.mkdtemp() + cfg = TstCfg( + nodeset_tpu={ + "a": TstNodeset("bold", node_count_static=4, node_count_dynamic_max=5), + "b": TstNodeset("slim", node_count_dynamic_max=3), + }, + nodeset={ + "c": TstNodeset("green", node_count_static=2, node_count_dynamic_max=3), + "d": TstNodeset("blue", node_count_static=7), + "e": TstNodeset("pink", node_count_dynamic_max=4), + }, + output_dir=output_dir, + ) + + def tpu_se(ns: str, lkp) -> TstTPU: + if ns == "bold": + return TstTPU(vmcount=3) + if ns == "slim": + return TstTPU(vmcount=1) + raise AssertionError(f"unexpected TPU name: '{ns}'") + + tpu_mock.side_effect = tpu_se + + lkp = util.Lookup(cfg) + lkp.instances = lambda: { n.name: n for n in [ # type: ignore[assignment] + # nodeset blue + tstInstance("m22-blue-0"), # no physicalHost + tstInstance("m22-blue-0", physical_host="/a/a/a"), + tstInstance("m22-blue-1", physical_host="/a/a/b"), + tstInstance("m22-blue-2", physical_host="/a/b/a"), + tstInstance("m22-blue-3", physical_host="/b/a/a"), + # nodeset green + tstInstance("m22-green-3", physical_host="/a/a/c"), + ]} + + uncompressed = conf.gen_topology(lkp) + want_uncompressed = [ + #NOTE: the switch names are not unique, it's not valid content for topology.conf + # The uniquefication and compression of names are done in the compress() method + "SwitchName=slurm-root Switches=a,b,ns_blue,ns_green,ns_pink", + # "physical" topology + 'SwitchName=a Switches=a,b', + 'SwitchName=a Nodes=m22-blue-[0-1],m22-green-3', + 'SwitchName=b Nodes=m22-blue-2', + 'SwitchName=b Switches=a', + 'SwitchName=a Nodes=m22-blue-3', + # topology "by nodeset" + "SwitchName=ns_blue Nodes=m22-blue-[4-6]", + "SwitchName=ns_green Nodes=m22-green-[0-2,4]", + "SwitchName=ns_pink Nodes=m22-pink-[0-3]", + # TPU topology + "SwitchName=tpu-root Switches=ns_bold,ns_slim", + "SwitchName=ns_bold Switches=bold-[0-3]", + "SwitchName=bold-0 Nodes=m22-bold-[0-2]", + "SwitchName=bold-1 Nodes=m22-bold-3", + "SwitchName=bold-2 Nodes=m22-bold-[4-6]", + "SwitchName=bold-3 Nodes=m22-bold-[7-8]", + "SwitchName=ns_slim Nodes=m22-slim-[0-2]"] + assert list(uncompressed.render_conf_lines()) == want_uncompressed + + compressed = uncompressed.compress() + want_compressed = [ + "SwitchName=s0 Switches=s0_[0-4]", # root + # "physical" topology + 'SwitchName=s0_0 Switches=s0_0_[0-1]', # /a + 'SwitchName=s0_0_0 Nodes=m22-blue-[0-1],m22-green-3', # /a/a + 'SwitchName=s0_0_1 Nodes=m22-blue-2', # /a/b + 'SwitchName=s0_1 Switches=s0_1_0', # /b + 'SwitchName=s0_1_0 Nodes=m22-blue-3', # /b/a + # topology "by nodeset" + "SwitchName=s0_2 Nodes=m22-blue-[4-6]", + "SwitchName=s0_3 Nodes=m22-green-[0-2,4]", + "SwitchName=s0_4 Nodes=m22-pink-[0-3]", + # TPU topology + "SwitchName=s1 Switches=s1_[0-1]", + "SwitchName=s1_0 Switches=s1_0_[0-3]", + "SwitchName=s1_0_0 Nodes=m22-bold-[0-2]", + "SwitchName=s1_0_1 Nodes=m22-bold-3", + "SwitchName=s1_0_2 Nodes=m22-bold-[4-6]", + "SwitchName=s1_0_3 Nodes=m22-bold-[7-8]", + "SwitchName=s1_1 Nodes=m22-slim-[0-2]"] + assert list(compressed.render_conf_lines()) == want_compressed + + upd, summary = conf.gen_topology_conf(lkp) + assert upd == True + want_written = PRELUDE + "\n".join(want_compressed) + "\n\n" + assert open(output_dir + "/cloud_topology.conf").read() == want_written + + summary.dump(lkp) + summary_got = json.loads(open(output_dir + "/cloud_topology.summary.json").read()) + + assert summary_got == { + "down_nodes": unordered( + [f"m22-blue-{i}" for i in (4,5,6)] + + [f"m22-green-{i}" for i in (0,1,2,4)] + + [f"m22-pink-{i}" for i in range(4)]), + "tpu_nodes": unordered( + [f"m22-bold-{i}" for i in range(9)] + + [f"m22-slim-{i}" for i in range(3)]), + 'physical_host': { + 'm22-blue-0': '/a/a/a', + 'm22-blue-1': '/a/a/b', + 'm22-blue-2': '/a/b/a', + 'm22-blue-3': '/b/a/a', + 'm22-green-3': '/a/a/c'}, + } + + + +def test_gen_topology_conf_update(): + cfg = TstCfg( + nodeset={ + "c": TstNodeset("green", node_count_static=2), + }, + output_dir=tempfile.mkdtemp(), + ) + lkp = util.Lookup(cfg) + lkp.instances = lambda: { # type: ignore[assignment] + # no instances + } + + # initial generation - reconfigure + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # add node: node_count_static 2 -> 3 - reconfigure + lkp.cfg.nodeset["c"].node_count_static = 3 + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # remove node: node_count_static 3 -> 2 - no reconfigure + lkp.cfg.nodeset["c"].node_count_static = 2 + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + # set empty physicalHost - no reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + # set physicalHost - reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/c")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # change physicalHost - reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/z")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # shut down node - no reconfigure + lkp.instances = lambda: {} # type: ignore[assignment] + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + +@pytest.mark.parametrize( + "paths,expected", + [ + (["z/n-0", "z/n-1", "z/n-2", "z/n-3", "z/n-4", "z/n-10"], ['n-0', 'n-1', 'n-2', 'n-3', 'n-4', 'n-10']), + (["y/n-0", "z/n-1", "x/n-2", "x/n-3", "y/n-4", "g/n-10"], ['n-0', 'n-4', 'n-1', 'n-2', 'n-3', 'n-10']), + ]) +def test_sort_nodes_order(paths: list[str], expected: list[str]) -> None: + paths_expanded = [l.split("/") for l in paths] + assert sort_nodes.order(paths_expanded) == expected diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py new file mode 100644 index 0000000000..69617d0301 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py @@ -0,0 +1,668 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Type + +import pytest +from mock import Mock +from datetime import datetime, timezone, timedelta +import unittest + +from common import TstNodeset, TstCfg # needed to import util +import util +from util import NodeState, MachineType, AcceleratorInfo, UpcomingMaintenance, InstanceResourceStatus, FutureReservation, ReservationDetails +from google.api_core.client_options import ClientOptions # noqa: E402 +from addict import Dict as NSDict # type: ignore + +# Note: need to install pytest-mock + +@pytest.mark.parametrize( + "name,expected", + [ + ( + "az-buka-23", + { + "cluster": "az", + "nodeset": "buka", + "node": "23", + "prefix": "az-buka", + "range": None, + "suffix": "23", + }, + ), + ( + "az-buka-xyzf", + { + "cluster": "az", + "nodeset": "buka", + "node": "xyzf", + "prefix": "az-buka", + "range": None, + "suffix": "xyzf", + }, + ), + ( + "az-buka-[2-3]", + { + "cluster": "az", + "nodeset": "buka", + "node": "[2-3]", + "prefix": "az-buka", + "range": "[2-3]", + "suffix": None, + }, + ), + ], +) +def test_node_desc(name, expected): + assert util.lookup()._node_desc(name) == expected + + +@pytest.mark.parametrize( + "name,expected", + [ + ("az-buka-23", 23), + ("az-buka-0", 0), + ("az-buka", Exception), + ("az-buka-xyzf", ValueError), + ("az-buka-[2-3]", ValueError), + ], +) +def test_node_index(name, expected): + if type(expected) is type and issubclass(expected, Exception): + with pytest.raises(expected): + util.lookup().node_index(name) + else: + assert util.lookup().node_index(name) == expected + + +@pytest.mark.parametrize( + "name", + [ + "az-buka", + ], +) +def test_node_desc_fail(name): + with pytest.raises(Exception): + util.lookup()._node_desc(name) + + +@pytest.mark.parametrize( + "names,expected", + [ + ("pedro,pedro-1,pedro-2,pedro-01,pedro-02", "pedro,pedro-[1-2,01-02]"), + ("pedro,,pedro-1,,pedro-2", "pedro,pedro-[1-2]"), + ("pedro-8,pedro-9,pedro-10,pedro-11", "pedro-[8-9,10-11]"), + ("pedro-08,pedro-09,pedro-10,pedro-11", "pedro-[08-11]"), + ("pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-09]"), + ("pedro-10,pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-10]"), + ("pedro-8,pedro-9,juan-10,juan-11", "juan-[10-11],pedro-[8-9]"), + ("az,buki,vedi", "az,buki,vedi"), + ("a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12", "a[0-9,10-12]"), + ("a0,a2,a4,a6,a7,a8,a11,a12", "a[0,2,4,6-8,11-12]"), + ("seas7-0,seas7-1", "seas7-[0-1]"), + ], +) +def test_to_hostlist(names, expected): + assert util.to_hostlist(names.split(",")) == expected + + +@pytest.mark.parametrize( + "api,ep_ver,expected", + [ + ( + util.ApiEndpoint.BQ, + "v1", + ClientOptions(api_endpoint="https://bq.googleapis.com/v1/"), + ), + ( + util.ApiEndpoint.COMPUTE, + "staging_v1", + ClientOptions(api_endpoint="https://compute.googleapis.com/staging_v1/"), + ), + ( + util.ApiEndpoint.SECRET, + "v1", + ClientOptions(api_endpoint="https://secret_manager.googleapis.com/v1/"), + ), + ( + util.ApiEndpoint.STORAGE, + "beta", + ClientOptions(api_endpoint="https://storage.googleapis.com/beta/"), + ), + ( + util.ApiEndpoint.TPU, + "alpha", + ClientOptions(api_endpoint="https://tpu.googleapis.com/alpha/"), + ), + ], +) +def test_create_client_options( + api: util.ApiEndpoint, ep_ver: str, expected: ClientOptions, mocker +): + ud_mock = mocker.patch("util.universe_domain") + ep_mock = mocker.patch("util.endpoint_version") + ud_mock.return_value = "googleapis.com" + ep_mock.return_value = ep_ver + assert util.create_client_options(api).__repr__() == expected.__repr__() + + + +@pytest.mark.parametrize( + "nodeset,err", + [ + (TstNodeset(reservation_name="projects/x/reservations/y"), AssertionError), # no zones + (TstNodeset( + reservation_name="projects/x/reservations/y", + zone_policy_allow=["eine", "zwei"]), AssertionError), # multiples zones + (TstNodeset( + reservation_name="robin", + zone_policy_allow=["eine"]), ValueError), # invalid name + (TstNodeset( + reservation_name="projects/reservations/y", + zone_policy_allow=["eine"]), ValueError), # invalid name + (TstNodeset( + reservation_name="projects/x/zones/z/reservations/y", + zone_policy_allow=["eine"]), ValueError), # invalid name + ] +) +def test_nodeset_reservation_err(nodeset, err): + lkp = util.Lookup(TstCfg()) + lkp._get_reservation = Mock() + with pytest.raises(err): + lkp.nodeset_reservation(nodeset) + lkp._get_reservation.assert_not_called() # type: ignore + +@pytest.mark.parametrize( + "nodeset,policies,expected", + [ + (TstNodeset(), [], None), # no reservation + (TstNodeset( + reservation_name="projects/bobin/reservations/robin", + zone_policy_allow=["eine"]), + [], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=[], + deployment_type=None, + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/bobin/reservations/robin")), + (TstNodeset( + reservation_name="projects/bobin/reservations/robin", + zone_policy_allow=["eine"]), + ["seven/wanders", "five/red/apples", "yum"], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=["wanders", "apples", "yum"], + deployment_type=None, + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/bobin/reservations/robin")), + (TstNodeset( + reservation_name="projects/bobin/reservations/robin/snek/cheese-brie-6", + zone_policy_allow=["eine"]), + [], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=[], + deployment_type=None, + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/bobin/reservations/robin/snek/cheese-brie-6")), + + ]) + +def test_nodeset_reservation_ok(nodeset, policies, expected): + lkp = util.Lookup(TstCfg()) + lkp._get_reservation = Mock() + + if not expected: + assert lkp.nodeset_reservation(nodeset) is None + lkp._get_reservation.assert_not_called() # type: ignore + return + + lkp._get_reservation.return_value = { # type: ignore + "resourcePolicies": {i: p for i, p in enumerate(policies)}, + } + assert lkp.nodeset_reservation(nodeset) == expected + lkp._get_reservation.assert_called_once_with(expected.project, expected.zone, expected.name) # type: ignore + +@pytest.mark.parametrize( + "job_info,expected_job", + [ + ( + """JobId=123 + TimeLimit=02:00:00 + JobName=myjob + JobState=PENDING + ReqNodeList=node-[1-10]""", + util.Job( + id=123, + duration=timedelta(days=0, hours=2, minutes=0, seconds=0), + name="myjob", + job_state="PENDING", + required_nodes="node-[1-10]" + ), + ), + ( + """JobId=456 + JobName=anotherjob + JobState=PENDING + ReqNodeList=node-group1""", + util.Job( + id=456, + duration=None, + name="anotherjob", + job_state="PENDING", + required_nodes="node-group1" + ), + ), + ( + """JobId=789 + TimeLimit=00:30:00 + JobState=COMPLETED""", + util.Job( + id=789, + duration=timedelta(minutes=30), + name=None, + job_state="COMPLETED", + required_nodes=None + ), + ), + ( + """JobId=101112 + TimeLimit=1-00:30:00 + JobState=COMPLETED, + ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", + util.Job( + id=101112, + duration=timedelta(days=1, hours=0, minutes=30, seconds=0), + name=None, + job_state="COMPLETED", + required_nodes="node-[1-10],grob-pop-[2,1,44-77]" + ), + ), + ( + """JobId=131415 + TimeLimit=1-00:30:00 + JobName=mynode-1_maintenance + JobState=COMPLETED, + ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", + util.Job( + id=131415, + duration=timedelta(days=1, hours=0, minutes=30, seconds=0), + name="mynode-1_maintenance", + job_state="COMPLETED", + required_nodes="node-[1-10],grob-pop-[2,1,44-77]" + ), + ), + ], +) +def test_parse_job_info(job_info, expected_job): + lkp = util.Lookup(TstCfg()) + assert lkp._parse_job_info(job_info) == expected_job + + + +@pytest.mark.parametrize( + "node,state,want", + [ + ("c-n-2", NodeState("DOWN", frozenset([])), NodeState("DOWN", frozenset([]))), # happy scenario + ("c-d-vodoo", None, None), # dynamic nodeset + ("c-x-44", None, None), # unknown(removed) nodeset + ("c-n-7", None, None), # Out of bounds: c-n-[0-4] - downsized nodeset + ("c-t-7", None, None), # Out of bounds: c-t-[0-4] - downsized nodeset TPU + ("c-n-2", None, RuntimeError), # something is wrong + ("c-t-2", None, RuntimeError), # something is wrong, but TPU + + # Check boundaries match [0-5) + ("c-n-5", None, None), # out of boundaries + ("c-n-4", None, RuntimeError), # within boundaries + ]) +def test_node_state(node: str, state: Optional[NodeState], want: NodeState | None | Type[Exception]): + cfg = TstCfg( + slurm_cluster_name="c", + nodeset={ + "n": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, + nodeset_tpu={ + "t": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, + nodeset_dyn={ + "d": TstNodeset()}, + ) + lkp = util.Lookup(cfg) + lkp.slurm_nodes = lambda: {node: state} if state else {} # type: ignore[assignment] + # ... see https://github.com/python/typeshed/issues/6347 + + if type(want) is type and issubclass(want, Exception): + with pytest.raises(want): + lkp.node_state(node) + else: + assert lkp.node_state(node) == want + + + +@pytest.mark.parametrize( + "jo,want", + [ + ({ + "accelerators": [ { "guestAcceleratorCount": 1, "guestAcceleratorType": "nvidia-tesla-a100" } ], + "creationTimestamp": "1969-12-31T16:00:00.000-08:00", + "description": "Accelerator Optimized: 1 NVIDIA Tesla A100 GPU, 12 vCPUs, 85GB RAM", + "guestCpus": 12, + "id": "1000012", + "imageSpaceGb": 0, + "isSharedCpu": False, + "kind": "compute#machineType", + "maximumPersistentDisks": 128, + "maximumPersistentDisksSizeGb": "263168", + "memoryMb": 87040, + "name": "a2-highgpu-1g", + "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/us-central1-a/machineTypes/a2-highgpu-1g", + "zone": "us-central1-a" + }, MachineType( + name="a2-highgpu-1g", + guest_cpus=12, + memory_mb=87040, + accelerators=[ + AcceleratorInfo(type="nvidia-tesla-a100", count=1) + ] + )), + ({ + "architecture": "X86_64", + "creationTimestamp": "1969-12-31T16:00:00.000-08:00", + "description": "8 vCPUs, 32 GB RAM", + "guestCpus": 8, + "id": "1210008", + "imageSpaceGb": 0, + "isSharedCpu": False, + "kind": "compute#machineType", + "maximumPersistentDisks": 128, + "maximumPersistentDisksSizeGb": "263168", + "memoryMb": 32768, + "name": "t2d-standard-8", + "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/europe-north2-b/machineTypes/t2d-standard-8", + "zone": "europe-north2-b" + }, MachineType( + name="t2d-standard-8", + guest_cpus=8, + memory_mb=32768, + accelerators=[] + )), + ]) +def test_MachineType_from_json(jo: dict, want: MachineType): + assert MachineType.from_json(jo) == want + + +@pytest.mark.parametrize( + "template,expected", + [ + ( + NSDict({ + "machine_type": MachineType( + name="e2", + guest_cpus=12, + memory_mb=87040, + accelerators=[]), + }), + None + ), + ( + NSDict({ + "machine_type": MachineType( + name="tpu-machine", + guest_cpus=12, + memory_mb=87040, + accelerators=[ + AcceleratorInfo(type="tpu-v6", count=1) + ]), + }), + None + ), + ( + NSDict({ + "machine_type": MachineType( + name="a2-highgpu-1g", + guest_cpus=12, + memory_mb=87040, + accelerators=[AcceleratorInfo(type="nvidia-tesla-a100", count=1)] + ), + }), + AcceleratorInfo(type="nvidia-tesla-a100", count=1) + ), + ( + NSDict({ + "machine_type": MachineType( + name="a2-highgpu-1g", + guest_cpus=12, + memory_mb=87040, + accelerators=[]), + "guestAccelerators":[ { "acceleratorCount": 1, "acceleratorType": "nvidia-tesla-a100" } ], + }), + AcceleratorInfo(type="nvidia-tesla-a100", count=1) + ), + ], +) +def test_get_template_gpu(template, expected): + assert util.get_template_gpu(template) == expected + + +UTC, PST = timezone.utc, timezone(timedelta(hours=-8)) + +@pytest.mark.parametrize( + "got,want", + [ + # from instance.creationTimestamp: + ("2024-11-30T12:47:51.676-08:00", datetime(2024, 11, 30, 12, 47, 51, 676000, tzinfo=PST)), + # from futureReservation.creationTimestamp + ("2024-11-05T15:23:33.702-08:00", datetime(2024, 11, 5, 15, 23, 33, 702000, tzinfo=PST)), + # from futureReservation.timeWindow.endTime + ("2025-01-15T00:00:00Z", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), + # fallback to UTC if no tz is specified + ("2025-01-15T00:00:00", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), + ]) +def test_parse_gcp_timestamp(got: str, want: datetime): + assert util.parse_gcp_timestamp(got) == want + + +@pytest.mark.parametrize( + "got,want", + [ + (None, None), + (dict( + windowStartTime="2025-01-15T00:00:00Z", + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + (dict( + startTimeWindow=dict( + earliest="2025-01-15T00:00:00Z"), + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + (dict( + windowStartTime="2025-01-15T00:00:00Z", + startTimeWindow=dict( + earliest="2025-01-25T00:00:00Z"), # ignored + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + ]) +def tests_parse_UpcomingMaintenance_OK(got: dict, want: Optional[UpcomingMaintenance]): + assert UpcomingMaintenance.from_json(got) == want + + +@pytest.mark.parametrize( + "got", + [ + {}, + dict( + windowStartTime=dict( + earliest="2025-01-15T00:00:00Z")), + ]) +def tests_parse_UpcomingMaintenance_FAIL(got: dict): + with pytest.raises(ValueError): + UpcomingMaintenance.from_json(got) + + +@pytest.mark.parametrize( + "got,want", + [ + (None, InstanceResourceStatus( + physical_host=None, + upcoming_maintenance=None)), + ({}, InstanceResourceStatus( + physical_host=None, + upcoming_maintenance=None)), + (dict( + physicalHost="/aaa/bbb/ccc"), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=None)), + (dict( # invalid upcomingMaintenance field to be ignored + physicalHost="/aaa/bbb/ccc", + upcomingMaintenance="maintenance is upon us"), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=None)), + (dict( + physicalHost="/aaa/bbb/ccc", + upcomingMaintenance=dict(windowStartTime="2025-01-15T00:00:00Z")), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=UpcomingMaintenance( + window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC)))), + ]) +def test_parse_InstanceResourceStatus(got: dict, want: Optional[InstanceResourceStatus]): + assert InstanceResourceStatus.from_json(got) == want + + +@pytest.mark.parametrize( + "link,component_name,expected", + [ + ( + "mylink/regions/us-cental1/other", + "regions", + "us-cental1" + ), + ( + "mylink/global/other", + "regions", + None + ), + ], +) +def test_get_self_link_component(link, component_name, expected): + assert util.get_self_link_component(link, component_name) == expected + + +def test_future_reservation_none(): + lkp = util.Lookup(TstCfg()) + assert lkp.future_reservation(TstNodeset()) == None + + +def test_future_reservation_declined(): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-03T23:30:00Z" }, + status = {"procurementStatus": "DECLINED"}, + reservationMode = "CALENDAR", + specificReservationRequired = True, + )) + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 3, 23, 30, tzinfo=timezone.utc), + reservation_mode="CALENDAR", + active_reservation=None) + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + +@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 13, 0, 0, tzinfo=timezone.utc)) +def test_future_reservation_active(_): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, + status = { + "procurementStatus": "FULFILLED", + "autoCreatedReservations": [ + "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" + ], + }, + specificReservationRequired = True, + )) + lkp._get_reservation = Mock(return_value=dict()) + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), + reservation_mode=None, + active_reservation=ReservationDetails( + project='manhattan', + zone='danger', + name='melon', + policies=[], + reservation_mode=None, + assured_count=0, + delete_at_time=None, + bulk_insert_name="projects/manhattan/reservations/melon", + deployment_type=None)) + + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + lkp._get_reservation.assert_called_once_with("manhattan", "danger", "melon") + +@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 28, 0, 0, tzinfo=timezone.utc)) +def test_future_reservation_inactive(_): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, + status = { + "procurementStatus": "FULFILLED", + "autoCreatedReservations": [ + "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" + ], + }, + reservationMode = "DEFAULT", + specificReservationRequired = True, + )) + lkp._get_reservation = Mock() + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), + reservation_mode="DEFAULT", + active_reservation=None) + + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + lkp._get_reservation.assert_not_called() diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test new file mode 100644 index 0000000000..a583642015 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test @@ -0,0 +1,133 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +unset CUDA_VISIBLE_DEVICES + +LOG_FILE="/var/log/slurm/chs_health_check.log" +TMP_DCGM_OUT="/tmp/dcgm.out" +TMP_ECC_ERRORS_OUT="/tmp/ecc_errors.out" + +log_step() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" +} + +# Fail gracefully if nvidia-smi or dcgmi doesn't exist +if ! type -P nvidia-smi 1>/dev/null; then + log_step "nvidia-smi not found - this script requires nvidia-smi to function" + exit 0 +fi + +if ! type -P dcgmi 1>/dev/null; then + log_step "dcgmi not found - this script requires dcgmi to function" + exit 0 +fi + +if ! type -P nv-hostengine 1>/dev/null; then + log_step "nv-hostengine not found - this script requires nv-hostengine to function" + exit 0 +fi + +################################################### +# Disable running health checks +################################################### +# Check if the environment variable '$SLURM_JOB_EXTRA' is set and contains the +# substring 'healthchecks_prolog=off' +if [[ -n "$SLURM_JOB_EXTRA" ]]; then + log_step "Environment variable SLURM_JOB_EXTRA is set. Checking if it contains healthchecks_prolog=off." + # Check if the value of the variable matches the string "healthchecks_prolog=off" + if [[ "$SLURM_JOB_EXTRA" == *"healthchecks_prolog=off"* ]]; then + log_step "Environment variable SLURM_JOB_EXTRA matches substring healthchecks_prolog=off. Skipping health checks." + exit 0 + else + log_step "Environment variable SLURM_JOB_EXTRA does NOT match substring healthchecks_prolog=off. Attempting to run health checks." + fi +else + log_step "Environment variable SLURM_JOB_EXTRA is NOT set. Attempting to run health checks." +fi + +# Exit if GPU isn't H/B 100/200 +GPU_MODEL=$(nvidia-smi --query-gpu=name --format=csv,noheader) +if ! [[ "$GPU_MODEL" =~ [BH][1-2]00 ]]; then + log_step "No Supported GPU detected" + exit 0 +fi + +NUMGPUS=$(nvidia-smi -L | wc -l) + +# Check that all GPUs are healthy via DCGM and check for ECC errors +if [ $NUMGPUS -gt 0 ]; then + log_step "Execute DCGM health check, ECC error check, and NVLink error check for GPUs" + GPULIST=$(nvidia-smi --query-gpu=index --format=csv,noheader | tr '\n' ',' | sed 's/,$//') + rm -f $TMP_DCGM_OUT + rm -f $TMP_ECC_ERRORS_OUT + + # Run DCGM checks + START_HOSTENGINE=false + if ! pidof nv-hostengine > /dev/null; then + log_step "Starting nv-hostengine..." + nv-hostengine >> "$LOG_FILE" 2>&1 + sleep 1 # Give it a moment to start up + START_HOSTENGINE=true + fi + GROUPID=$(dcgmi group -c gpuinfo | awk '{print $NF}' | tr -d ' ') + dcgmi group -g $GROUPID -a $GPULIST >> "$LOG_FILE" 2>&1 + dcgmi diag -g $GROUPID -r 1 > "$TMP_DCGM_OUT" 2>&1 + cat "$TMP_DCGM_OUT" >> "$LOG_FILE" + dcgmi group -d $GROUPID >> "$LOG_FILE" 2>&1 + + # Terminate the host engine if it was manually started + if [ "$START_HOSTENGINE" = true ]; then + log_step "Terminating nv-hostengine..." + nv-hostengine -t >> "$LOG_FILE" 2>&1 + fi + + # Check for DCGM failures + DCGM_FAILED=0 + if grep -i fail "$TMP_DCGM_OUT" > /dev/null; then + DCGM_FAILED=1 + fi + + # Check for ECC errors + nvidia-smi --query-gpu=ecc.errors.uncorrected.volatile.total --format=csv,noheader > "$TMP_ECC_ERRORS_OUT" + cat "$TMP_ECC_ERRORS_OUT" >> "$LOG_FILE" + ECC_ERRORS=$(awk -F', ' '{sum += $2} END {print sum}' "$TMP_ECC_ERRORS_OUT") + log_step "ECC Errors: $ECC_ERRORS" + + # Check for NVLink errors + NVLINK_ERRORS=$(nvidia-smi nvlink -sc 0bz -i 0 2>/dev/null | grep -i "Error Count" | awk '{sum += $3} END {print sum}') + # Set to 0 if empty/null + NVLINK_ERRORS=${NVLINK_ERRORS:-0} + log_step "NVLink Errors: $NVLINK_ERRORS" + + if [ $DCGM_FAILED -eq 1 ] || \ + [ $ECC_ERRORS -gt 0 ] || \ + [ $NVLINK_ERRORS -gt 0 ]; then + REASON="GPU issues detected: " + if [ $DCGM_FAILED -eq 1 ]; then + REASON+="DCGM test failed, " + fi + if [ $ECC_ERRORS -gt 0 ]; then + REASON+="ECC errors found ($ECC_ERRORS double-bit errors), " + fi + if [ $NVLINK_ERRORS -gt 0 ]; then + REASON+="NVLink errors detected ($NVLINK_ERRORS errors), " + fi + REASON+="see $LOG_FILE" + log_step "$REASON" + exit 1 + fi +fi diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog new file mode 100644 index 0000000000..a22ddea9e5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Main TaskEpilog Script +# This script executes all *.sh scripts found in /slurm/custom_scripts/task_epilog.d/ +# +# slurm.conf configuration: +# TaskEpilog=/slurm/scripts/tools/task-epilog + +# Directory containing the individual task epilog scripts +EPILOG_D_DIR="/slurm/custom_scripts/task_epilog.d" + +# --- Output Handling for TaskEpilog --- +# The stdout and stderr of this script (and the sub-scripts it calls) +# are typically captured by Slurm and written to the job's output/error file +# or a separate Slurm log, depending on configuration. +# Unlike TaskProlog, stdout is not typically parsed for special commands +# like 'export' or 'print' to affect the (now finished) task's environment. +# +# --- Error Handling --- +# If any script in EPILOG_D_DIR exits with a non-zero status, +# this main script will also exit with a non-zero status. +# Slurm will log this. Depending on Slurm's configuration, +# frequent epilog failures might lead to node issues or alerts. +set -e # Exit immediately if a command exits with a non-zero status. + +# Check if the directory exists +if [[ ! -d "$EPILOG_D_DIR" ]]; then + # Log in task stdout and exit if the directory is missing. This likely indicates a configuration error. + echo "print TaskEpilog Error: Directory '$EPILOG_D_DIR' not found. Check Slurm configuration." + exit 1 +fi + +# Find and execute all *.sh scripts in the directory +# Scripts will be executed in reverse alphabetical order of their filenames. +find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do + if [[ -x "$script" ]]; then + # Execute the script. Its stdout will be captured by this wrapper. + # Its stderr will also be passed through. + # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. + "$script" + else + # Log in task stdout a warning if a *.sh file is found but is not executable + echo "print TaskEpilog Warning: Script '$script' is not executable and will be skipped." + fi +done + +# Check if any scripts were found and executed +if [[ $(find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then + # Log in task stdout if no scripts were found to execute + echo "print TaskEpilog Info: No executable *.sh scripts found in $EPILOG_D_DIR." +fi + +# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) +exit 0 diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog new file mode 100644 index 0000000000..feddb23209 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Main TaskProlog Script +# This script executes all *.sh scripts found in /slurm/custom_scripts/task_prolog.d/ +# +# slurm.conf configuration: +# TaskProlog=/slurm/scripts/tools/task-prolog + +# Directory containing the individual task prolog scripts +PROLOG_D_DIR="/slurm/custom_scripts/task_prolog.d" + +# --- Output Handling for TaskProlog --- +# Slurm's TaskProlog can interpret specific stdout lines: +# - "export NAME=value" : Sets an environment variable for the task. +# - "unset NAME" : Unsets an environment variable for the task. +# - "print message" : Prints a message to the task's standard output. +# +# This wrapper script will concatenate the stdout of all sub-scripts. +# If sub-scripts need to set/unset environment variables or print messages +# for the task, they should output the appropriate "export", "unset", or "print" +# commands to their own stdout. + +# --- Error Handling --- +# If any script in PROLOG_D_DIR exits with a non-zero status, +# this main script will also exit with a non-zero status. +# This will typically cause the task to fail. +set -e # Exit immediately if a command exits with a non-zero status. + +# Check if the directory exists +if [[ ! -d "$PROLOG_D_DIR" ]]; then + # Log in task stdout and exit if the directory is missing. All jobs will be failed. + echo "print TaskProlog Error: Directory '$PROLOG_D_DIR' not found. Check Slurm configuration." + exit 1 +fi + +# Find and execute all *.sh scripts in the directory +# Scripts will be executed in reverse alphabetical order of their filenames. +find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do + if [[ -x "$script" ]]; then + # Execute the script. Its stdout will be captured by this wrapper. + # Its stderr will also be passed through. + # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. + "$script" + else + # Log a warning in task stdout if a *.sh file is found but is not executable + echo "print TaskProlog Warning: Script '$script' is not executable and will be skipped." + fi +done + +# Check if any scripts were found and executed +if [[ $(find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then + # Log in task stdout if no scripts were found to execute + echo "print TaskProlog Info: No executable *.sh scripts found in $PROLOG_D_DIR." +fi + +# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) +exit 0 diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py new file mode 100644 index 0000000000..531f0348dc --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py @@ -0,0 +1,331 @@ +# mypy: ignore-errors +# This implementation of TPU integration is to be deprecated + +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import socket +import logging +from pathlib import Path +import yaml + +import util +from util import create_client_options, ApiEndpoint + +from google.cloud import tpu_v2 as tpu # noqa: E402 +import google.api_core.exceptions as gExceptions # noqa: E402 + +log = logging.getLogger() + +_tpu_cache = {} + +class TPU: + """Class for handling the TPU-vm nodes""" + + State = tpu.types.cloud_tpu.Node.State + TPUS_PER_VM = 4 + __expected_states = { + "create": State.READY, + "start": State.READY, + "stop": State.STOPPED, + } + + __tpu_version_mapping = { + "V2": tpu.AcceleratorConfig().Type.V2, + "V3": tpu.AcceleratorConfig().Type.V3, + "V4": tpu.AcceleratorConfig().Type.V4, + } + + @classmethod + def make(cls, nodeset_name: str, lkp: util.Lookup) -> "TPU": + key = (id(lkp), nodeset_name) + if key not in _tpu_cache: + nodeset = lkp.cfg.nodeset_tpu[nodeset_name] + _tpu_cache[key] = cls(nodeset, lkp) + return _tpu_cache[key] + + + def __init__(self, nodeset: object, lkp: util.Lookup): + self._nodeset = nodeset + self.lkp = lkp + self._parent = f"projects/{lkp.project}/locations/{nodeset.zone}" + co = create_client_options(ApiEndpoint.TPU) + self._client = tpu.TpuClient(client_options=co) + self.data_disks = [] + for data_disk in nodeset.data_disks: + ad = tpu.AttachedDisk() + ad.source_disk = data_disk + ad.mode = tpu.AttachedDisk.DiskMode.DISK_MODE_UNSPECIFIED + self.data_disks.append(ad) + ns_ac = nodeset.accelerator_config + if ns_ac.topology != "" and ns_ac.version != "": + ac = tpu.AcceleratorConfig() + ac.topology = ns_ac.topology + ac.type_ = self.__tpu_version_mapping[ns_ac.version] + self.ac = ac + else: + req = tpu.GetAcceleratorTypeRequest( + name=f"{self._parent}/acceleratorTypes/{nodeset.node_type}" + ) + self.ac = self._client.get_accelerator_type(req).accelerator_configs[0] + self.vmcount = self.__calc_vm_from_topology(self.ac.topology) + + @property + def nodeset(self): + return self._nodeset + + @property + def preserve_tpu(self): + return self._nodeset.preserve_tpu + + @property + def node_type(self): + return self._nodeset.node_type + + @property + def tf_version(self): + return self._nodeset.tf_version + + @property + def enable_public_ip(self): + return self._nodeset.enable_public_ip + + @property + def preemptible(self): + return self._nodeset.preemptible + + @property + def reserved(self): + return self._nodeset.reserved + + @property + def service_account(self): + return self._nodeset.service_account + + @property + def zone(self): + return self._nodeset.zone + + def check_node_type(self): + if self.node_type is None: + return False + try: + request = tpu.GetAcceleratorTypeRequest( + name=f"{self._parent}/acceleratorTypes/{self.node_type}" + ) + return self._client.get_accelerator_type(request=request) is not None + except Exception: + return False + + def check_tf_version(self): + try: + request = tpu.GetRuntimeVersionRequest( + name=f"{self._parent}/runtimeVersions/{self.tf_version}" + ) + return self._client.get_runtime_version(request=request) is not None + except Exception: + return False + + def __calc_vm_from_topology(self, topology): + topo = topology.split("x") + tot = 1 + for num in topo: + tot = tot * int(num) + return tot // self.TPUS_PER_VM + + def __check_resp(self, response, op_name): + des_state = self.__expected_states.get(op_name) + # If the state is not in the table just print the response + if des_state is None: + return False + if response.__class__.__name__ != "Node": # If the response is not a node fail + return False + if response.state == des_state: + return True + return False + + def list_nodes(self): + try: + request = tpu.ListNodesRequest(parent=self._parent) + res = self._client.list_nodes(request=request) + except gExceptions.NotFound: + res = None + return res + + def list_node_names(self): + return [node.name.split("/")[-1] for node in self.list_nodes()] + + def start_node(self, nodename): + request = tpu.StartNodeRequest(name=f"{self._parent}/nodes/{nodename}") + resp = self._client.start_node(request=request).result() + return self.__check_resp(resp, "start") + + def stop_node(self, nodename): + request = tpu.StopNodeRequest(name=f"{self._parent}/nodes/{nodename}") + resp = self._client.stop_node(request=request).result() + return self.__check_resp(resp, "stop") + + def get_node(self, nodename): + try: + request = tpu.GetNodeRequest(name=f"{self._parent}/nodes/{nodename}") + res = self._client.get_node(request=request) + except gExceptions.NotFound: + res = None + return res + + def _register_node(self, nodename, ip_addr): + dns_name = socket.getnameinfo((ip_addr, 0), 0)[0] + util.run( + f"{self.lkp.scontrol} update nodename={nodename} nodeaddr={ip_addr} nodehostname={dns_name}" + ) + + def create_node(self, nodename): + if self.vmcount > 1 and not isinstance(nodename, list): + log.error( + f"Tried to create a {self.vmcount} node TPU on nodeset {self._nodeset.nodeset_name} but only received one nodename {nodename}" + ) + return False + if self.vmcount > 1 and ( + isinstance(nodename, list) and len(nodename) != self.vmcount + ): + log.error( + f"Expected to receive a list of {self.vmcount} nodenames for TPU node creation in nodeset {self._nodeset.nodeset_name}, but received this list {nodename}" + ) + return False + + node = tpu.Node() + node.accelerator_config = self.ac + node.runtime_version = f"tpu-vm-tf-{self.tf_version}" + startup_script = """ + #!/bin/bash + echo "startup script not found > /var/log/startup_error.log" + """ + with open( + Path(self.lkp.cfg.slurm_scripts_dir or util.dirs.scripts) / "startup.sh", "r" + ) as script: + startup_script = script.read() + if isinstance(nodename, list): + node_id = nodename[0] + slurm_names = [] + wid = 0 + for node_wid in nodename: + slurm_names.append(f"WORKER_{wid}:{node_wid}") + wid += 1 + else: + node_id = nodename + slurm_names = [f"WORKER_0:{nodename}"] + node.metadata = { + "slurm_docker_image": self.nodeset.docker_image, + "startup-script": startup_script, + "slurm_instance_role": "compute", + "slurm_cluster_name": self.lkp.cfg.slurm_cluster_name, + "slurm_bucket_path": self.lkp.cfg.bucket_path, + "slurm_names": ";".join(slurm_names), + "universe_domain": util.universe_domain(), + } + node.tags = [self.lkp.cfg.slurm_cluster_name] + if self.nodeset.service_account: + node.service_account.email = self.nodeset.service_account.email + node.service_account.scope = self.nodeset.service_account.scopes + node.scheduling_config.preemptible = self.preemptible + node.scheduling_config.reserved = self.reserved + node.network_config.subnetwork = self.nodeset.subnetwork + node.network_config.enable_external_ips = self.enable_public_ip + if self.data_disks: + node.data_disks = self.data_disks + + request = tpu.CreateNodeRequest(parent=self._parent, node=node, node_id=node_id) + resp = self._client.create_node(request=request).result() + if not self.__check_resp(resp, "create"): + return False + if isinstance(nodename, list): + for node_id, net_endpoint in zip(nodename, resp.network_endpoints): + self._register_node(node_id, net_endpoint.ip_address) + else: + ip_add = resp.network_endpoints[0].ip_address + self._register_node(nodename, ip_add) + return True + + def delete_node(self, nodename): + request = tpu.DeleteNodeRequest(name=f"{self._parent}/nodes/{nodename}") + try: + resp = self._client.delete_node(request=request).result() + if resp: + return self.get_node(nodename=nodename) is None + return False + except gExceptions.NotFound: + # log only error if vmcount is 1 as for other tpu vm count, this could be "phantom" nodes + if self.vmcount == 1: + log.error(f"Tpu single node {nodename} not found") + else: + # for the TPU nodes that consist in more than one vm, only the first node of the TPU a.k.a. the master node will + # exist as real TPU nodes, so the other ones are expected to not be found, check the hostname of the node that has + # not been found, and if it ends in 0, it means that is the master node and it should have been found, and in consequence + # log an error + nodehostname = yaml.safe_load( + util.run(f"{self.lkp.scontrol} --yaml show node {nodename}").stdout.rstrip() + )["nodes"][0]["hostname"] + if nodehostname.split("-")[-1] == "0": + log.error(f"TPU master node {nodename} not found") + else: + log.info(f"Deleted TPU 'phantom' node {nodename}") + # If the node is not found it is tecnichally deleted, so return success. + return True + +def _stop_tpu(node: str) -> None: + lkp = util.lookup() + tpuobj = TPU.make(lkp.node_nodeset_name(node), lkp) + if tpuobj.nodeset.preserve_tpu and tpuobj.vmcount == 1: + log.info(f"stopping node {node}") + if tpuobj.stop_node(node): + return + log.error("Error stopping node {node} will delete instead") + log.info(f"deleting node {node}") + if not tpuobj.delete_node(node): + log.error("Error deleting node {node}") + + +def delete_tpu_instances(instances: List[str]) -> None: + util.execute_with_futures(_stop_tpu, instances) + + +def start_tpu(node: List[str]): + lkp = util.lookup() + tpuobj = TPU.make(lkp.node_nodeset_name(node[0]), lkp) + + if len(node) == 1: + node = node[0] + log.debug( + f"Will create a TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node}" + ) + tpunode = tpuobj.get_node(node) + if tpunode is None: + if not tpuobj.create_node(nodename=node): + log.error("Error creating tpu node {node}") + else: + if tpuobj.preserve_tpu: + if not tpuobj.start_node(nodename=node): + log.error("Error starting tpu node {node}") + else: + log.info( + f"Tpu node {node} is already created, but will not start it because nodeset does not have preserve_tpu option active." + ) + else: + log.debug( + f"Will create a multi-vm TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node[0]}" + ) + if not tpuobj.create_node(nodename=node): + log.error("Error creating tpu node {node}") diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py new file mode 100644 index 0000000000..217fd0bca2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py @@ -0,0 +1,2224 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable, List, Tuple, Optional, Any, Dict, Sequence, Type, Callable, Union +import argparse +import base64 +from dataclasses import dataclass, field +from datetime import timedelta, datetime, timezone +import hashlib +import inspect +import json +import logging +import logging.config +import logging.handlers +import math +import os +import re +import shlex +import shutil +import socket +import subprocess +import sys +from enum import Enum +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager +from functools import lru_cache, reduce, wraps +from itertools import chain, islice +from pathlib import Path +from time import sleep, time + +# TODO: remove "type: ignore" once moved to newer version of libraries +from google.cloud import secretmanager +from google.cloud import storage # type: ignore + +import google.auth # type: ignore +from google.oauth2 import service_account # type: ignore +import googleapiclient.discovery # type: ignore +import google_auth_httplib2 # type: ignore +from googleapiclient.http import set_user_agent # type: ignore +from google.api_core.client_options import ClientOptions +import httplib2 + +import google.api_core.exceptions as gExceptions + +import requests as requests_lib + +import yaml +from addict import Dict as NSDict # type: ignore +import file_cache + +USER_AGENT = "Slurm_GCP_Scripts/1.5 (GPN:SchedMD)" +ENV_CONFIG_YAML = os.getenv("SLURM_CONFIG_YAML") +if ENV_CONFIG_YAML: + CONFIG_FILE = Path(ENV_CONFIG_YAML) +else: + CONFIG_FILE = Path(__file__).with_name("config.yaml") +API_REQ_LIMIT = 2000 + + +def mkdirp(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + + +scripts_dir = next( + p for p in (Path(__file__).parent, Path("/slurm/scripts")) if p.is_dir() +) + + +# load all directories as Paths into a dict-like namespace +dirs = NSDict( + home = Path("/home"), + apps = Path("/opt/apps"), + slurm = Path("/slurm"), + scripts = scripts_dir, + custom_scripts = Path("/slurm/custom_scripts"), + munge = Path("/etc/munge"), + secdisk = Path("/mnt/disks/sec"), + log = Path("/var/log/slurm"), + slurm_bucket_mount = Path("/slurm/bucket"), +) + +slurmdirs = NSDict( + prefix = Path("/usr/local"), + etc = Path("/usr/local/etc/slurm"), + state = Path("/var/spool/slurm"), + key_distribution = Path("/slurm/key_distribution"), +) + + +# TODO: Remove this hack (relies on undocumented behavior of PyYAML) +# No need to represent NSDict and Path once we move to properly typed & serializable config. +yaml.SafeDumper.yaml_representers[ + None # type: ignore +] = lambda self, data: yaml.representer.SafeRepresenter.represent_str(self, str(data)) # type: ignore + + +class ApiEndpoint(Enum): + COMPUTE = "compute" + BQ = "bq" + STORAGE = "storage" + TPU = "tpu" + SECRET = "secret_manager" + + +@dataclass(frozen=True) +class AcceleratorInfo: + type: str + count: int + + @classmethod + def from_json(cls, jo: dict) -> "AcceleratorInfo": + return cls( + type=jo["guestAcceleratorType"], + count=jo["guestAcceleratorCount"]) + +@dataclass(frozen=True) +class MachineType: + name: str + guest_cpus: int + memory_mb: int + accelerators: List[AcceleratorInfo] + + @classmethod + def from_json(cls, jo: dict) -> "MachineType": + return cls( + name=jo["name"], + guest_cpus=jo["guestCpus"], + memory_mb=jo["memoryMb"], + accelerators=[ + AcceleratorInfo.from_json(a) for a in jo.get("accelerators", [])], + ) + + @property + def family(self) -> str: + # TODO: doesn't work with N1 custom machine types + # See https://cloud.google.com/compute/docs/instances/creating-instance-with-custom-machine-type#create + return self.name.split("-")[0] + + @property + def supports_smt(self) -> bool: + # https://cloud.google.com/compute/docs/cpu-platforms + if self.family in ("t2a", "t2d", "h3", "c4a", "h4d",): + return False + if self.guest_cpus == 1: + return False + return True + + @property + def sockets(self) -> int: + return { + "h3": 2, + "h4d": 2, + "c2d": 2 if self.guest_cpus > 56 else 1, + "a3": 2, + "c2": 2 if self.guest_cpus > 30 else 1, + "c3": 2 if self.guest_cpus > 88 else 1, + "c3d": 2 if self.guest_cpus > 180 else 1, + "c4": 2 if self.guest_cpus > 96 else 1, + "c4d": 2 if self.guest_cpus > 192 else 1, + }.get( + self.family, + 1, # assume 1 socket for all other families + ) + + +@dataclass(frozen=True) +class UpcomingMaintenance: + window_start_time: datetime + + @classmethod + def from_json(cls, jo: Optional[dict]) -> Optional["UpcomingMaintenance"]: + if jo is None: + return None + try: + if "windowStartTime" in jo: + ts = parse_gcp_timestamp(jo["windowStartTime"]) + elif "startTimeWindow" in jo: + ts = parse_gcp_timestamp(jo["startTimeWindow"]["earliest"]) + else: + raise Exception("Neither windowStartTime nor startTimeWindow are found") + except BaseException as e: + raise ValueError(f"Unexpected format for upcomingMaintenance: {jo}") from e + return cls(window_start_time=ts) + +@dataclass(frozen=True) +class InstanceResourceStatus: + physical_host: Optional[str] + upcoming_maintenance: Optional[UpcomingMaintenance] + + @classmethod + def from_json(cls, jo: Optional[dict]) -> "InstanceResourceStatus": + if not jo: + return cls( + physical_host=None, + upcoming_maintenance=None, + ) + + try: + maint = UpcomingMaintenance.from_json(jo.get("upcomingMaintenance")) + except ValueError as e: + log.exception("Failed to parse upcomingMaintenance, ignoring") + maint = None # intentionally swallow exception + + return cls( + physical_host=jo.get("physicalHost"), + upcoming_maintenance=maint, + ) + + +@dataclass(frozen=True) +class Instance: + name: str + zone: str + status: str + creation_timestamp: datetime + role: Optional[str] + resource_status: InstanceResourceStatus + metadata: Dict[str, str] + # TODO: use proper InstanceScheduling class + scheduling: NSDict + + @classmethod + def from_json(cls, jo: dict) -> "Instance": + return cls( + name=jo["name"], + zone=trim_self_link(jo["zone"]), + status=jo["status"], + creation_timestamp=parse_gcp_timestamp(jo["creationTimestamp"]), + resource_status=InstanceResourceStatus.from_json(jo.get("resourceStatus")), + scheduling=NSDict(jo.get("scheduling")), + role = jo.get("labels", {}).get("slurm_instance_role"), + metadata = {k["key"]: k["value"] for k in jo.get("metadata", {}).get("items", [])} + ) + + +@dataclass(frozen=True) +class NSMount: + server_ip: str + local_mount: Path + remote_mount: Path + fs_type: str + mount_options: str + +@lru_cache(maxsize=1) +def default_credentials(): + return google.auth.default()[0] + + +@lru_cache(maxsize=1) +def authentication_project(): + return google.auth.default()[1] + + +DEFAULT_UNIVERSE_DOMAIN = "googleapis.com" + + +def now() -> datetime: + """ + Return current time as timezone-aware datetime. + + IMPORTANT: DO NOT use `datetime.now()`, unless you explicitly need to have tz-naive datetime. + Otherwise there is a risk of getting: "cannot compare naive and aware datetimes" error, + since all timetstamps we receive from GCP API are tz-aware. + + Another motivation for this function is to allow to mock time in tests. + """ + return datetime.now(timezone.utc) + +def parse_gcp_timestamp(s: str) -> datetime: + """ + Parse timestamp strings returned by GCP API into datetime. + Works with both Zulu and non-Zulu timestamps. + NOTE: It always return tz-aware datetime (fallbacks to UTC and logs error). + """ + # Requires Python >= 3.7 + # TODO: Remove this "hack" of trimming the Z from timestamps once we move to Python 3.11 + # (context: https://discuss.python.org/t/parse-z-timezone-suffix-in-datetime/2220/30) + ts = datetime.fromisoformat(s.replace('Z', '+00:00')) + if ts.tzinfo is None: # fallback to UTC + log.error(f"Received timestamp without timezone info: {s}") + ts = ts.replace(tzinfo=timezone.utc) + return ts + + +def universe_domain() -> str: + try: + return instance_metadata("attributes/universe_domain") + except MetadataNotFoundError: + return DEFAULT_UNIVERSE_DOMAIN + + +def endpoint_version(api: ApiEndpoint) -> Optional[str]: + return lookup().endpoint_versions.get(api.value, None) + + +@lru_cache(maxsize=1) +def get_credentials() -> Optional[service_account.Credentials]: + """Get credentials for service account""" + key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if key_path is not None: + credentials = service_account.Credentials.from_service_account_file( + key_path, scopes=[f"https://www.{universe_domain()}/auth/cloud-platform"] + ) + else: + credentials = default_credentials() + + return credentials + + +@lru_cache(maxsize=1) +def get_dev_key() -> Optional[str]: + """Get dev key for project (uses json or yaml format)""" + try: + with open("/etc/slurm/slurm_vars.yaml", 'r') as file: + data = yaml.safe_load(file) + return data['google_developer_key'] + except: + return None + + +def create_client_options(api: ApiEndpoint) -> ClientOptions: + """Create client options for cloud endpoints""" + ver = endpoint_version(api) + ud = universe_domain() + options = {} + if ud and ud != DEFAULT_UNIVERSE_DOMAIN: + options["universe_domain"] = ud + if ver: + options["api_endpoint"] = f"https://{api.value}.{ud}/{ver}/" + co = ClientOptions(**options) + log.debug(f"Using ClientOptions = {co} for API: {api.value}") + return co + +log = logging.getLogger() + + +def access_secret_version(project_id, secret_id, version_id="latest"): + """ + Access the payload for the given secret version if one exists. The version + can be a version number as a string (e.g. "5") or an alias (e.g. "latest"). + """ + co = create_client_options(ApiEndpoint.SECRET) + client = secretmanager.SecretManagerServiceClient(client_options=co) + name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" + try: + response = client.access_secret_version(request={"name": name}) + log.debug(f"Secret '{name}' was found.") + payload = response.payload.data.decode("UTF-8") + except gExceptions.NotFound: + log.debug(f"Secret '{name}' was not found!") + payload = None + + return payload + + +def parse_self_link(self_link: str): + """Parse a selfLink url, extracting all useful values + https://.../v1/projects//regions//... + {'project': , 'region': , ...} + can also extract zone, instance (name), image, etc + """ + link_patt = re.compile(r"(?P[^\/\s]+)s\/(?P[^\s\/]+)") + return NSDict(link_patt.findall(self_link)) + + +def parse_bucket_uri(uri: str): + """ + Parse a bucket url + E.g. gs:/// + """ + pattern = re.compile(r"gs://(?P[^/\s]+)/(?P([^/\s]+)(/[^/\s]+)*)") + matches = pattern.match(uri) + assert matches, f"Unexpected bucker URI: '{uri}'" + return matches.group("bucket"), matches.group("path") + + +def get_template_gpu(template): + """get gpu info from machine type or guest accelerators""" + gpu_keyword = "nvidia" + gpu = None + if template.machine_type.accelerators: + tma = template.machine_type.accelerators[0] + if gpu_keyword in tma.type.lower(): + gpu = tma + elif template.guestAccelerators: + tga = template.guestAccelerators[0] + if gpu_keyword in tga.acceleratorType.lower(): + gpu = AcceleratorInfo( + type=tga.acceleratorType, + count=tga.acceleratorCount) + return gpu + + +def trim_self_link(link: str): + """get resource name from self link url, eg. + https://.../v1/projects//regions/ + -> + """ + try: + return link[link.rindex("/") + 1 :] + except ValueError: + raise Exception(f"'/' not found, not a self link: '{link}' ") + + +def get_self_link_component(link: str, component_name: str): + """ + Extracts a component (e.g., 'region', 'project') from a self-link URL. + Args: + link: The self-link URL string. + component_name: The name of the component to extract (e.g., 'regions', 'projects'). + Returns: + The extracted component value (e.g., '', ''), + or None if the component is not found in the link. + """ + search_string = f"/{component_name}/" + start_index = link.rfind(search_string) + + if start_index == -1: + return None + + start_index += len(search_string) + end_index = link.find("/", start_index) + + if end_index == -1: + # If no further slash, the rest of the string is the component + return link[start_index:] + else: + return link[start_index:end_index] + + +def execute_with_futures(func, seq): + with ThreadPoolExecutor() as exe: + futures = [] + for i in seq: + future = exe.submit(func, i) + futures.append(future) + for future in as_completed(futures): + result = future.exception() + if result is not None: + raise result + + +def map_with_futures(func, seq): + with ThreadPoolExecutor() as exe: + futures = [] + for i in seq: + future = exe.submit(func, i) + futures.append(future) + for future in futures: + # Will be result or raise Exception + res = None + try: + res = future.result() + except Exception as e: + res = e + yield res + +def should_mount_slurm_bucket() -> bool: + try: + return instance_metadata("attributes/slurm_bucket_mount", silent=True).lower() == "true" + except MetadataNotFoundError: + return False + + +def _get_bucket_and_common_prefix() -> Tuple[str, str]: + uri = instance_metadata("attributes/slurm_bucket_path") + return parse_bucket_uri(uri) + +def blob_get(file): + bucket_name, path = _get_bucket_and_common_prefix() + blob_name = f"{path}/{file}" + return storage_client().get_bucket(bucket_name).blob(blob_name) + + +def blob_list(prefix="", delimiter=None): + bucket_name, path = _get_bucket_and_common_prefix() + blob_prefix = f"{path}/{prefix}" + # Note: The call returns a response only when the iterator is consumed. + blobs = storage_client().list_blobs( + bucket_name, prefix=blob_prefix, delimiter=delimiter + ) + return [blob for blob in blobs] + +def file_list(prefix="", subpath="") -> List[os.DirEntry]: + path = dirs.slurm_bucket_mount + file_prefix = f"{path}/{subpath}" + try: + files = os.scandir(file_prefix) + return [file for file in files if file.name.startswith(prefix)] + except: + return [] + # Not considering lack of file's existence as fatal (we may check for files we know don't exist). + # Responsibility of callee to determine if it is fatal or not, blob_list returns empty iterator in similar cases. + +def hash_file(fullpath: Path) -> str: + with open(fullpath, "rb") as f: + file_hash = hashlib.md5() + chunk = f.read(8192) + while chunk: + file_hash.update(chunk) + chunk = f.read(8192) + return base64.b64encode(file_hash.digest()).decode("utf-8") + + +def install_custom_scripts(check_hash:bool=False): + """download custom scripts from gcs bucket""" + role, tokens = lookup().instance_role, [] + + mounted_scripts=False + if should_mount_slurm_bucket() and role != "controller": + mounted_scripts=True + + all_prolog_tokens = ["prolog", "epilog", "task_prolog", "task_epilog"] + if role == "controller": + tokens = ["controller"] + all_prolog_tokens + elif role == "compute": + tokens = [f"nodeset-{lookup().node_nodeset_name()}"] + all_prolog_tokens + elif role == "login": + tokens = [f"login-{instance_login_group()}"] + + prefixes = [f"slurm-{tok}-script" for tok in tokens] + + # TODO: use single `blob_list`, to reduce ~4x number of GCS requests + if mounted_scripts: + source_collection = list(chain.from_iterable(file_list(prefix=p) for p in prefixes)) + else: + source_collection = list(chain.from_iterable(blob_list(prefix=p) for p in prefixes)) + + script_pattern = re.compile(r"^slurm-(?P\S+)-script-(?P\S+)") + for source in source_collection: + if mounted_scripts: + m = script_pattern.match(source.name) + else: + m = script_pattern.match(Path(source.name).name) + + if not m: + log.warning(f"found blob that doesn't match expected pattern: {source.name}") + continue + path_parts = m["path"].split("-") + path_parts[0] += ".d" + stem, _, ext = m["name"].rpartition("_") + filename = ".".join((stem, ext)) + + path = Path(*path_parts, filename) + fullpath = (dirs.custom_scripts / path).resolve() + mkdirp(fullpath.parent) + + for par in path.parents: + chown_slurm(dirs.custom_scripts / par) + need_update = True + + if check_hash and fullpath.exists() and isinstance(source,storage.Blob): + # TODO: MD5 reported by gcloud may differ from the one calculated here (e.g. if blob got gzipped), + # consider using gCRC32C + need_update = hash_file(fullpath) != source.md5_hash + + log.info(f"installing custom script: {path} from {source.name}") + + if isinstance(source,os.DirEntry): + shutil.copy(source.path, fullpath) #Needs to be copied since mounted nfs is read-only + chown_slurm(fullpath, mode=0o755) + + elif need_update: + with fullpath.open("wb") as f: + source.download_to_file(f) + chown_slurm(fullpath, mode=0o755) + +def compute_service(version="beta"): + """Make thread-safe compute service handle + creates a new Http for each request + """ + credentials = get_credentials() + dev_key = get_dev_key() + + def build_request(http, *args, **kwargs): + new_http = set_user_agent(httplib2.Http(), USER_AGENT) + if credentials is not None: + new_http = google_auth_httplib2.AuthorizedHttp(credentials, http=new_http) + return googleapiclient.http.HttpRequest(new_http, *args, **kwargs) + + ver = endpoint_version(ApiEndpoint.COMPUTE) + disc_url = googleapiclient.discovery.DISCOVERY_URI + if ver: + version = ver + disc_url = disc_url.replace(DEFAULT_UNIVERSE_DOMAIN, universe_domain()) + + log.debug(f"Using version={version} of Google Compute Engine API") + return googleapiclient.discovery.build( + "compute", + version, + requestBuilder=build_request, + credentials=credentials, + developerKey=dev_key, + discoveryServiceUrl=disc_url, + cache_discovery=False, # See https://github.com/googleapis/google-api-python-client/issues/299 + ) + +def storage_client() -> storage.Client: + """ + Config-independent storage client + """ + ud = universe_domain() + co = {} + if ud and ud != DEFAULT_UNIVERSE_DOMAIN: + co["universe_domain"] = ud + return storage.Client(client_options=ClientOptions(**co)) + + +class DeffetiveStoredConfigError(Exception): + """ + Raised when config can not be loaded and assembled from bucket + """ + pass + + +def _fill_cfg_defaults(cfg: NSDict) -> NSDict: + if not cfg.slurm_log_dir: + cfg.slurm_log_dir = dirs.log + if not cfg.slurm_bin_dir: + cfg.slurm_bin_dir = slurmdirs.prefix / "bin" + if not cfg.slurm_control_host: + try: + control_dns_name = instance_metadata("attributes/slurm_control_dns", silent=True) + cfg.slurm_control_host = control_dns_name + except MetadataNotFoundError: + cfg.slurm_control_host = f"{cfg.slurm_cluster_name}-controller" + if not cfg.slurm_control_host_port: + cfg.slurm_control_host_port = "6820-6830" + return cfg + +@dataclass +class _ConfigBlobs: + """ + "Private" class that represent a collection of GCS blobs for configuration + """ + core: storage.Blob + controller_addr: Optional[storage.Blob] + partition: List[storage.Blob] = field(default_factory=list) + nodeset: List[storage.Blob] = field(default_factory=list) + nodeset_dyn: List[storage.Blob] = field(default_factory=list) + nodeset_tpu: List[storage.Blob] = field(default_factory=list) + login_group: List[storage.Blob] = field(default_factory=list) + + @property + def hash(self) -> str: + h = hashlib.md5() + all = [self.core] + self.partition + self.nodeset + self.nodeset_dyn + self.nodeset_tpu + if self.controller_addr: + all.append(self.controller_addr) + + # sort blobs so hash is consistent + for blob in sorted(all, key=lambda b: b.name): + h.update(blob.md5_hash.encode("utf-8")) + return h.hexdigest() + +@dataclass +class _ConfigFiles: + """ + "Private" class that represent a collection of files for configuration + """ + core: Path + controller_addr: Optional[Path] + partition: List[Path] = field(default_factory=list) + nodeset: List[Path] = field(default_factory=list) + nodeset_dyn: List[Path] = field(default_factory=list) + nodeset_tpu: List[Path] = field(default_factory=list) + login_group: List[Path] = field(default_factory=list) + +def _list_config_blobs() -> _ConfigBlobs: + _, common_prefix = _get_bucket_and_common_prefix() + + core: Optional[storage.Blob] = None + controller_addr: Optional[storage.Blob] = None + rest: Dict[str, List[storage.Blob]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} + + is_controller = instance_role() == "controller" + + for blob in blob_list(prefix=""): + if blob.name == f"{common_prefix}/config.yaml": + core = blob + if blob.name == f"{common_prefix}/controller_addr.yaml" and not is_controller: + # Don't add this config blobs for controller to avoid "double reconfiguration": + # Initially this file doesn't exist and produce later by `setup_controller`; + # Appearance of this blob would trigger change in combined hash of config files; + # Ignore existence of this file for controller, assume that + # no other instance nodes will proceed with configuration until this file is created. + controller_addr = blob + for key in rest.keys(): + if blob.name.startswith(f"{common_prefix}/{key}_configs/"): + rest[key].append(blob) + + if core is None: + raise DeffetiveStoredConfigError(f"{common_prefix}/config.yaml not found in bucket") + + return _ConfigBlobs(core=core, controller_addr=controller_addr, **rest) + +def _list_config_files() -> _ConfigFiles: + file_dir = dirs.slurm_bucket_mount + core: Optional[Path] = None + controller_addr: Optional[Path] = None + rest: Dict[str, List[Path]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} + + if Path(f"{file_dir}/config.yaml").exists(): + core = Path(f"{file_dir}/config.yaml") + + for key in rest.keys(): + for f in file_list(subpath=f"{key}_configs"): + rest[key].append(f.path) + + if core is None: + raise Exception(f"config.yaml was not found in mounted folder: {dirs.slurm_bucket_mount}") #Intentionally not using DeffetiveStoredConfigError as this is considered a fatal error + + return _ConfigFiles(core=core, controller_addr=None, **rest) + +def _fetch_config(old_hash: Optional[str]) -> Optional[Tuple[NSDict, str]]: + """Fetch config from bucket, returns None if no changes are detected.""" + blobs = _list_config_blobs() + if old_hash == blobs.hash: + return None + + def _download(bs) -> List[Any]: + return [yaml.safe_load(b.download_as_text()) for b in bs] + + return _assemble_config( + core=_download([blobs.core])[0], + controller_addr=_download([blobs.controller_addr])[0] if blobs.controller_addr else None, + partitions=_download(blobs.partition), + nodesets=_download(blobs.nodeset), + nodesets_dyn=_download(blobs.nodeset_dyn), + nodesets_tpu=_download(blobs.nodeset_tpu), + login_groups=_download(blobs.login_group), + ), blobs.hash + +def _fetch_mounted_config() -> Optional[Tuple[NSDict, str]]: + if not dirs.slurm_bucket_mount.is_mount(): + raise Exception(f"{dirs.slurm_bucket_mount} is not mounted") + + files = _list_config_files() + + def _load(files) -> List[Any]: + file_yaml=[] + for file in files: + with open(file, "r") as f: + file_yaml.append(yaml.safe_load(f)) + return file_yaml + + return _assemble_config( + core=_load([files.core])[0], + controller_addr=None, + partitions=_load(files.partition), + nodesets=_load(files.nodeset), + nodesets_dyn=_load(files.nodeset_dyn), + nodesets_tpu=_load(files.nodeset_tpu), + login_groups=_load(files.login_group), + ) + +def controller_lookup_self_ip() -> str: + assert instance_role() == "controller" + # Get IP of LAST network-interface + # TODO: Consider change order of NICs definition, so right NIC is always @0. + idx = instance_metadata("network-interfaces").split()[-1] # either `0/` or `1/` + return instance_metadata(f"network-interfaces/{idx}ip") + +def _assemble_config( + core: Any, + controller_addr: Optional[Any], + partitions: List[Any], + nodesets: List[Any], + nodesets_dyn: List[Any], + nodesets_tpu: List[Any], + login_groups: List[Any], + ) -> NSDict: + cfg = NSDict(core) + + if cfg.controller_network_attachment: + # lookup controller address + if instance_role() == "controller": + # ignore stored value of `controller_addr`, it will be overwritten during `setup_controller` + cfg.slurm_control_addr = controller_lookup_self_ip() + else: + if not controller_addr: + raise DeffetiveStoredConfigError("controller_addr.yaml not found in bucket") + cfg.slurm_control_addr = controller_addr["slurm_control_addr"] + + # add partition configs + for p_yaml in partitions: + p_cfg = NSDict(p_yaml) + assert p_cfg.get("partition_name"), "partition_name is required" + p_name = p_cfg.partition_name + assert p_name not in cfg.partitions, f"partition {p_name} already defined" + cfg.partitions[p_name] = p_cfg + + # add nodeset configs + ns_names = set() + def _add_nodesets(yamls: List[Any], target: dict): + for ns_yaml in yamls: + ns_cfg = NSDict(ns_yaml) + assert ns_cfg.get("nodeset_name"), "nodeset_name is required" + ns_name = ns_cfg.nodeset_name + assert ns_name not in ns_names, f"nodeset {ns_name} already defined" + target[ns_name] = ns_cfg + ns_names.add(ns_name) + + _add_nodesets(nodesets, cfg.nodeset) + _add_nodesets(nodesets_dyn, cfg.nodeset_dyn) + _add_nodesets(nodesets_tpu, cfg.nodeset_tpu) + + # validate that configs for all referenced nodesets are present + for p in cfg.partitions.values(): + for ns_name in chain(p.partition_nodeset, p.partition_nodeset_dyn, p.partition_nodeset_tpu): + if ns_name not in ns_names: + raise DeffetiveStoredConfigError(f"nodeset {ns_name} not defined in config") + + for lg_yaml in login_groups: + lg_cfg = NSDict(lg_yaml) + assert lg_cfg.get("group_name"), "group_name is required" + lg_name = lg_cfg.group_name + assert lg_name not in cfg.login_groups + cfg.login_groups[lg_name] = lg_cfg + + if instance_role() == "login": + group = instance_login_group() + if group not in cfg.login_groups: + raise DeffetiveStoredConfigError(f"login group '{group}' does not exist in config") + + return _fill_cfg_defaults(cfg) + +def fetch_config() -> Tuple[bool, NSDict]: + """ + Fetches config from bucket and saves it locally + Returns True if new (updated) config was fetched + """ + hash_file = Path("/slurm/scripts/.config.hash") + old_hash = hash_file.read_text() if hash_file.exists() else None + + if should_mount_slurm_bucket() and instance_role() != "controller": + cfg = _fetch_mounted_config() + CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) + chown_slurm(CONFIG_FILE) + return False, cfg + + cfg_and_hash = _fetch_config(old_hash=old_hash) + + if not cfg_and_hash: + return False, _load_config() + + cfg, hash = cfg_and_hash + hash_file.write_text(hash) + chown_slurm(hash_file) + CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) + chown_slurm(CONFIG_FILE) + return True, cfg + +def owned_file_handler(filename): + """create file handler""" + chown_slurm(filename) + return logging.handlers.WatchedFileHandler(filename, delay=True) + +def get_log_path() -> Path: + """ + Returns path to log file for the current script. + e.g. resume.py -> /var/log/slurm/resume.log + """ + cfg_log_dir = lookup().cfg.slurm_log_dir + log_dir = Path(cfg_log_dir) if cfg_log_dir else dirs.log + return (log_dir / Path(sys.argv[0]).name).with_suffix(".log") + +def init_log_and_parse(parser: argparse.ArgumentParser) -> argparse.Namespace: + parser.add_argument( + "--debug", + "-d", + dest="loglevel", + action="store_const", + const=logging.DEBUG, + default=logging.INFO, + help="Enable debugging output", + ) + parser.add_argument( + "--trace-api", + "-t", + action="store_true", + help="Enable detailed api request output", + ) + args = parser.parse_args() + loglevel = args.loglevel + if lookup().cfg.enable_debug_logging: + loglevel = logging.DEBUG + if args.trace_api: + lookup().cfg.extra_logging_flags["trace_api"] = True + # Configure root logger + logging.config.dictConfig({ + "version": 1, + "disable_existing_loggers": True, + "formatters": { + "standard": { + "format": "%(levelname)s: %(message)s", + }, + "stamp": { + "format": "%(asctime)s %(levelname)s: %(message)s", + }, + }, + "handlers": { + "stdout_handler": { + "level": logging.DEBUG, + "formatter": "standard", + "class": "logging.StreamHandler", + "stream": sys.stdout, + }, + "file_handler": { + "()": owned_file_handler, + "level": logging.DEBUG, + "formatter": "stamp", + "filename": get_log_path(), + }, + }, + "root": { + "handlers": ["stdout_handler", "file_handler"], + "level": loglevel, + }, + }) + + sys.excepthook = _handle_exception + + return args + + +def log_api_request(request): + """log.trace info about a compute API request""" + if not lookup().cfg.extra_logging_flags.get("trace_api"): + return + # output the whole request object as pretty yaml + # the body is nested json, so load it as well + rep = json.loads(request.to_json()) + if rep.get("body", None) is not None: + rep["body"] = json.loads(rep["body"]) + pretty_req = yaml.safe_dump(rep).rstrip() + # label log message with the calling function + log.debug(f"{inspect.stack()[1].function}:\n{pretty_req}") + + +def _handle_exception(exc_type, exc_value, exc_trace): + """log exceptions other than KeyboardInterrupt""" + if not issubclass(exc_type, KeyboardInterrupt): + log.exception("Fatal exception", exc_info=(exc_type, exc_value, exc_trace)) + sys.__excepthook__(exc_type, exc_value, exc_trace) + + +def run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + timeout=None, + check=True, + universal_newlines=True, + **kwargs, +): + """Wrapper for subprocess.run() with convenient defaults""" + if isinstance(args, list): + args = list(filter(lambda x: x is not None, args)) + args = " ".join(args) + if not shell and isinstance(args, str): + args = shlex.split(args) + log.debug(f"run: {args}") + try: + result = subprocess.run( + args, + stdout=stdout, + stderr=stderr, + shell=shell, + timeout=timeout, + check=check, + universal_newlines=universal_newlines, + **kwargs, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + log_subprocess(e) + raise + log_subprocess(result) + return result + +def log_subprocess(subj: subprocess.CalledProcessError | subprocess.TimeoutExpired | subprocess.CompletedProcess) -> None: + match subj: + case subprocess.CompletedProcess(returncode=0): + # Do not log successful runs, to not overwhelm logs (e.g. scontrol show jobs --json) + # TODO: consider still doing it in DEBUG or trim output to few KBs. + return + case subprocess.CompletedProcess(): # non-zero returncode + log.error(f"Command '{subj.args}' returned exit status {subj.returncode}.") + case subprocess.CalledProcessError() | subprocess.TimeoutExpired(): + log.error(str(subj)) + + + def normalize(out: None | str | bytes) -> None | str: + """ + Turns stderr and stdout into string: + > A bytes sequence, or a string if run() was called with an encoding, errors, or text=True. None if was not captured. + """ + match out: + case None: + return None + case str(): + return out.strip() + case bytes(): + return out.decode().strip() + case _: + return repr(out) + + if stdout := normalize(subj.stdout): + log.error(f"stdout: {stdout}") + if stderr := normalize(subj.stderr): + log.error(f"stderr: {stderr}") + + +def chown_slurm(path: Path, mode=None) -> None: + if path.exists(): + if mode: + path.chmod(mode) + else: + mkdirp(path.parent) + if mode: + path.touch(mode=mode) + else: + path.touch() + try: + shutil.chown(path, user="slurm", group="slurm") + except LookupError: + log.warning(f"User 'slurm' does not exist. Cannot 'chown slurm:slurm {path}'.") + except PermissionError: + log.warning(f"Not authorized to 'chown slurm:slurm {path}'.") + except Exception as err: + log.error(err) + + +@contextmanager +def cd(path): + """Change working directory for context""" + prev = Path.cwd() + os.chdir(path) + try: + yield + finally: + os.chdir(prev) + + +def cached_property(f): + return property(lru_cache()(f)) + + +def retry(max_retries: int, init_wait_time: float, warn_msg: str, exc_type: Type[Exception]): + """Retries functions that raises the exception exc_type. + Retry time is increased by a factor of two for every iteration. + + Args: + max_retries (int): Maximum number of retries + init_wait_time (float): Initial wait time in secs + warn_msg (str): Message to print during retries + exc_type (Exception): Exception type to check for + """ + + if max_retries <= 0: + raise ValueError("Incorrect value for max_retries, must be >= 1") + if init_wait_time <= 0.0: + raise ValueError("Invalid value for init_wait_time, must be > 0.0") + + def decorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + retry = 0 + secs = init_wait_time + captured_exc: Optional[BaseException] = None + while retry < max_retries: + try: + return f(*args, **kwargs) + except exc_type as e: + captured_exc = e + log.warn(f"{warn_msg}, retrying in {secs}") + sleep(secs) + retry += 1 + secs *= 2 + assert captured_exc + raise captured_exc + + return wrapper + + return decorator + + +def separate(pred: Callable[[Any], bool], coll: Iterable[Any]) -> Tuple[List[Any], List[Any]]: + """filter into 2 lists based on pred returning True or False + returns ([False], [True]) + """ + res: Tuple[List[Any], List[Any]] = ([],[]) + for el in coll: + res[pred(el)].append(el) + return res + + +def chunked(iterable, n=API_REQ_LIMIT): + """group iterator into chunks of max size n""" + it = iter(iterable) + while True: + chunk = list(islice(it, n)) + if not chunk: + return + yield chunk + +def groupby_unsorted(seq: Sequence[Any], key): + indices = defaultdict(list) + for i, el in enumerate(seq): + indices[key(el)].append(i) + for k, idxs in indices.items(): + yield k, (seq[i] for i in idxs) + + +@lru_cache(maxsize=32) +def find_ratio(a, n, s, r0=None): + """given the start (a), count (n), and sum (s), find the ratio required""" + if n == 2: + return s / a - 1 + an = a * n + if n == 1 or s == an: + return 1 + if r0 is None: + # we only need to know which side of 1 to guess, and the iteration will work + r0 = 1.1 if an < s else 0.9 + + # geometric sum formula + def f(r): + return a * (1 - r**n) / (1 - r) - s + + # derivative of f + def df(r): + rm1 = r - 1 + rn = r**n + return (a * (rn * (n * rm1 - r) + r)) / (r * rm1**2) + + MIN_DR = 0.0001 # negligible change + r = r0 + # print(f"r(0)={r0}") + MAX_TRIES = 64 + for i in range(1, MAX_TRIES + 1): + try: + dr = f(r) / df(r) + except ZeroDivisionError: + log.error(f"Failed to find ratio due to zero division! Returning r={r0}") + return r0 + r = r - dr + # print(f"r({i})={r}") + # if the change in r is small, we are close enough + if abs(dr) < MIN_DR: + break + else: + log.error(f"Could not find ratio after {MAX_TRIES}! Returning r={r0}") + return r0 + return r + + +def backoff_delay(start, timeout=None, ratio=None, count: int = 0): + """generates `count` waits starting at `start` + sum of waits is `timeout` or each one is `ratio` bigger than the last + the last wait is always 0""" + # timeout or ratio must be set but not both + assert (timeout is None) ^ (ratio is None) + assert ratio is None or ratio > 0 + assert timeout is None or timeout >= start + assert (count > 1 or timeout is not None) and isinstance(count, int) + assert start > 0 + + if count == 0: + # Equation for auto-count is tuned to have a max of + # ~int(timeout) counts with a start wait of <0.01. + # Increasing start wait decreases count eg. + # backoff_delay(10, timeout=60) -> count = 5 + count = int( + (timeout / ((start + 0.05) ** (1 / 2)) + 2) // math.log(timeout + 2) + ) + + yield start + # if ratio is set: + # timeout = start * (1 - ratio**(count - 1)) / (1 - ratio) + if ratio is None: + ratio = find_ratio(start, count - 1, timeout) + + wait = start + # we have start and 0, so we only need to generate count - 2 + for _ in range(count - 2): + wait *= ratio + yield wait + yield 0 + return + + +ROOT_URL = "http://metadata.google.internal/computeMetadata/v1" + +class MetadataNotFoundError(Exception): + pass + +def get_metadata(path:str, silent=False) -> str: + """Get metadata relative to metadata/computeMetadata/v1""" + HEADERS = {"Metadata-Flavor": "Google"} + url = f"{ROOT_URL}/{path}" + try: + resp = requests_lib.get(url, headers=HEADERS) + resp.raise_for_status() + return resp.text + except requests_lib.exceptions.HTTPError: + if not silent: + log.warning(f"metadata not found ({url})") + raise MetadataNotFoundError(f"failed to get_metadata from {url}") + + +@lru_cache(maxsize=None) +def instance_metadata(path: str, silent:bool=False) -> str: + return get_metadata(f"instance/{path}", silent=silent) + +def instance_role(): + return instance_metadata("attributes/slurm_instance_role") + + +def instance_login_group(): + return instance_metadata("attributes/slurm_login_group") + + +def natural_sort(text): + def atoi(text): + return int(text) if text.isdigit() else text + + return [atoi(w) for w in re.split(r"(\d+)", text)] + + +def to_hostlist(names: Iterable[str]) -> str: + """ + Fast implementation of `hostlist` that doesn't invoke `scontrol` + IMPORTANT: + * Acts as `scontrol show hostlistsorted`, i.e. original order is not preserved + * Achieves worse compression than `scontrol show hostlist` for some cases + """ + pref = defaultdict(list) + tokenizer = re.compile(r"^(.*?)(\d*)$") + for name in filter(None, names): + matches = tokenizer.match(name) + assert matches, name + p, s = matches.groups() + pref[p].append(s) + + def _compress_suffixes(ss: List[str]) -> List[str]: + cur, res = None, [] + + def cur_repr(): + assert cur + nums, strs = cur + if nums[0] == nums[1]: + return strs[0] + return f"{strs[0]}-{strs[1]}" + + for s in sorted(ss, key=int): + n = int(s) + if cur is None: + cur = ((n, n), (s, s)) + continue + + nums, strs = cur + if n == nums[1] + 1: + cur = ((nums[0], n), (strs[0], s)) + else: + res.append(cur_repr()) + cur = ((n, n), (s, s)) + if cur: + res.append(cur_repr()) + return res + + res = [] + for p in sorted(pref.keys()): + sl = defaultdict(list) + for s in pref[p]: + sl[len(s)].append(s) + cs = [] + for ln in sorted(sl.keys()): + if ln == 0: + res.append(p) + else: + cs.extend(_compress_suffixes(sl[ln])) + if not cs: + continue + if len(cs) == 1 and "-" not in cs[0]: + res.append(f"{p}{cs[0]}") + else: + res.append(f"{p}[{','.join(cs)}]") + return ",".join(res) + +@lru_cache(maxsize=None) +def to_hostnames(nodelist: str) -> List[str]: + """make list of hostnames from hostlist expression""" + if not nodelist: + return [] # avoid degenerate invocation of scontrol + if isinstance(nodelist, str): + hostlist = nodelist + else: + hostlist = ",".join(nodelist) + hostnames = run(f"{lookup().scontrol} show hostnames {hostlist}").stdout.splitlines() + return hostnames + + +def retry_exception(exc) -> bool: + """return true for exceptions that should always be retried""" + msg = str(exc) + retry_errors = ( + "Rate Limit Exceeded", + "Quota Exceeded", + "Quota exceeded", + ) + return any(err in msg for err in retry_errors) + + +def ensure_execute(request): + """Handle rate limits and socket time outs""" + + for retry, wait in enumerate(backoff_delay(0.5, timeout=10 * 60, count=20)): + try: + return request.execute() + except googleapiclient.errors.HttpError as e: + if retry_exception(e): + log.error(f"retry:{retry} '{e}'") + sleep(wait) + continue + raise + + except socket.timeout as e: + # socket timed out, try again + log.debug(e) + + except Exception as e: + log.error(e, exc_info=True) + raise + + break + + +def batch_execute(requests, retry_cb=None, log_err=log.error): + """execute list or dict as batch requests + retry if retry_cb returns true + """ + BATCH_LIMIT = 1000 + if not isinstance(requests, dict): + requests = {str(k): v for k, v in enumerate(requests)} # rid generated here + done = {} + failed = {} + timestamps: List[float] = [] + rate_limited = False + + def batch_callback(rid, resp, exc): + nonlocal rate_limited + if exc is not None: + log_err(f"compute request exception {rid}: {exc}") + if retry_exception(exc): + rate_limited = True + else: + req = requests.pop(rid) + failed[rid] = (req, exc) + else: + # if retry_cb is set, don't move to done until it returns false + if retry_cb is None or not retry_cb(resp): + requests.pop(rid) + done[rid] = resp + + def batch_request(reqs): + batch = lookup().compute.new_batch_http_request(callback=batch_callback) + for rid, req in reqs: + batch.add(req, request_id=rid) + return batch + + while requests: + if timestamps: + timestamps = [stamp for stamp in timestamps if stamp > time()] + if rate_limited and timestamps: + stamp = next(iter(timestamps)) + sleep(max(stamp - time(), 0)) + rate_limited = False + # up to API_REQ_LIMIT (2000) requests + # in chunks of up to BATCH_LIMIT (1000) + batches = [ + batch_request(chunk) + for chunk in chunked(islice(requests.items(), API_REQ_LIMIT), BATCH_LIMIT) + ] + timestamps.append(time() + 100) + with ThreadPoolExecutor() as exe: + futures = [] + for batch in batches: + future = exe.submit(ensure_execute, batch) + futures.append(future) + for future in futures: + result = future.exception() + if result is not None: + raise result + + return done, failed + + +def get_operation_req(lkp: "Lookup", name: str, region: Optional[str]=None, zone: Optional[str]=None) -> Any: + if zone: + return lkp.compute.zoneOperations().get(project=lkp.project, zone=zone, operation=name) + elif region: + return lkp.compute.regionOperations().get(project=lkp.project, region=region, operation=name) + return lkp.compute.globalOperations().get(project=lkp.project, operation=name) + +def wait_request(operation, project: str): + """makes the appropriate wait request for a given operation""" + if "zone" in operation: + req = lookup().compute.zoneOperations().wait( + project=project, + zone=trim_self_link(operation["zone"]), + operation=operation["name"], + ) + elif "region" in operation: + req = lookup().compute.regionOperations().wait( + project=project, + region=trim_self_link(operation["region"]), + operation=operation["name"], + ) + else: + req = lookup().compute.globalOperations().wait( + project=project, operation=operation["name"] + ) + return req + + +def wait_for_operation(operation) -> Dict[str, Any]: + """wait for given operation""" + project = parse_self_link(operation["selfLink"]).project + wait_req = wait_request(operation, project=project) + + while True: + result = ensure_execute(wait_req) + if result["status"] == "DONE": + log_errors = " with errors" if "error" in result else "" + log.debug( + f"operation complete{log_errors}: type={result['operationType']}, name={result['name']}" + ) + return result + + + +def getThreadsPerCore(template) -> int: + if not template.machine_type.supports_smt: + return 1 + return template.advancedMachineFeatures.threadsPerCore or 2 + + +@retry( + max_retries=9, + init_wait_time=1, + warn_msg="Temporary failure in name resolution", + exc_type=socket.gaierror, +) +def host_lookup(host_name: str) -> str: + return socket.gethostbyname(host_name) + + +class Dumper(yaml.SafeDumper): + """Add representers for pathlib.Path and NSDict for yaml serialization""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.add_representer(NSDict, self.represent_nsdict) + self.add_multi_representer(Path, self.represent_path) + + @staticmethod + def represent_nsdict(dumper, data): + return dumper.represent_mapping("tag:yaml.org,2002:map", data.items()) + + @staticmethod + def represent_path(dumper, path): + return dumper.represent_scalar("tag:yaml.org,2002:str", str(path)) + + +@dataclass(frozen=True) +class ReservationDetails: + project: str + zone: str + name: str + policies: List[str] # names (not URLs) of resource policies + bulk_insert_name: str # name in format suitable for bulk insert (currently identical to user supplied name in long format) + deployment_type: Optional[str] + reservation_mode: Optional[str] + assured_count: int + delete_at_time: Optional[datetime] + + @property + def dense(self) -> bool: + return self.deployment_type == "DENSE" + + @property + def calendar(self) -> bool: + return self.reservation_mode == "CALENDAR" + +@dataclass(frozen=True) +class FutureReservation: + project: str + zone: str + name: str + specific: bool + start_time: datetime + end_time: datetime + reservation_mode: Optional[str] + active_reservation: Optional[ReservationDetails] + + @property + def calendar(self) -> bool: + return self.reservation_mode == "CALENDAR" + +@dataclass +class Job: + id: int + name: Optional[str] = None + required_nodes: Optional[str] = None + job_state: Optional[str] = None + duration: Optional[timedelta] = None + +@dataclass(frozen=True) +class NodeState: + base: str + flags: frozenset + +class Lookup: + """Wrapper class for cached data access""" + + def __init__(self, cfg): + self._cfg = cfg + + @property + def cfg(self): + return self._cfg + + @property + def project(self): + return self.cfg.project or authentication_project() + + @cached_property + def control_addr(self) -> Optional[str]: + return self.cfg.get("slurm_control_addr", None) + + @property + def control_host(self): + return self.cfg.slurm_control_host + + @cached_property + def control_host_addr(self): + return self.control_addr or host_lookup(self.cfg.slurm_control_host) + + @property + def control_host_port(self): + return self.cfg.slurm_control_host_port + + @property + def endpoint_versions(self): + return self.cfg.endpoint_versions + + @property + def scontrol(self): + return Path(self.cfg.slurm_bin_dir or "") / "scontrol" + + @cached_property + def instance_role(self): + return instance_role() + + @cached_property + def instance_role_safe(self): + try: + role = self.instance_role + except Exception as e: + log.error(e) + role = None + return role + + @property + def is_controller(self): + return self.instance_role_safe == "controller" + + @property + def is_login_node(self): + return self.instance_role_safe == "login" + + @cached_property + def compute(self): + # TODO evaluate when we need to use google_app_cred_path + if self.cfg.google_app_cred_path: + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.cfg.google_app_cred_path + return compute_service() + + @cached_property + def hostname(self): + return socket.gethostname() + + @cached_property + def hostname_fqdn(self): + return socket.getfqdn() + + @cached_property + def zone(self): + return instance_metadata("zone") + + node_desc_regex = re.compile( + r"^(?P(?P[^\s\-]+)-(?P\S+))-(?P(?P\w+)|(?P\[[\d,-]+\]))$" + ) + + @lru_cache(maxsize=None) + def _node_desc(self, node_name): + """Get parts from node name""" + if not node_name: + node_name = self.hostname + # workaround below is for VMs whose hostname is FQDN + node_name_short = node_name.split(".")[0] + m = self.node_desc_regex.match(node_name_short) + if not m: + raise Exception(f"node name {node_name} is not valid") + return m.groupdict() + + def node_prefix(self, node_name=None): + return self._node_desc(node_name)["prefix"] + + def node_index(self, node: str) -> int: + """ node_index("cluster-nodeset-45") == 45 """ + suff = self._node_desc(node)["suffix"] + + if suff is None: + raise ValueError(f"Node {node} name does not end with numeric index") + return int(suff) + + def node_nodeset_name(self, node_name=None): + return self._node_desc(node_name)["nodeset"] + + def node_nodeset(self, node_name=None): + nodeset_name = self.node_nodeset_name(node_name) + if nodeset_name in self.cfg.nodeset_tpu: + return self.cfg.nodeset_tpu[nodeset_name] + + return self.cfg.nodeset[nodeset_name] + + def partition_is_tpu(self, part: str) -> bool: + """check if partition with name part contains a nodeset of type tpu""" + return len(self.cfg.partitions[part].partition_nodeset_tpu) > 0 + + + def node_is_tpu(self, node_name=None): + nodeset_name = self.node_nodeset_name(node_name) + return self.cfg.nodeset_tpu.get(nodeset_name) is not None + + def nodeset_is_tpu(self, nodeset_name=None) -> bool: + return self.cfg.nodeset_tpu.get(nodeset_name) is not None + + def node_is_fr(self, node_name:str) -> bool: + return bool(self.node_nodeset(node_name).future_reservation) + + def is_dormant_res_node(self, node_name:str) -> bool: + fr = self.future_reservation(self.node_nodeset(node_name)) + res = self.nodeset_reservation(self.node_nodeset(node_name)) + + if fr is None and res is None: + return False + + if fr: + return fr.active_reservation is None + + if res: + if res.calendar: + # If reservation is calendar based, check if it is past the delete_at_time + if res.delete_at_time is not None and now() >= res.delete_at_time: + log.debug(f"DWS calendar reservation {res.bulk_insert_name} is past deletion time {res.delete_at_time}, skipping resume.") + return True + + # If assured_count is 0 do not resume nodes as they are not active yet + if res.delete_at_time is not None and res.assured_count <= 0: + log.debug(f"DWS calendar reservation {res.bulk_insert_name} is not active yet, skipping resume.") + return True + + return False + + def node_is_dyn(self, node_name=None) -> bool: + nodeset = self.node_nodeset_name(node_name) + return self.cfg.nodeset_dyn.get(nodeset) is not None + + def node_is_gke(self, node_name=None) -> bool: + return self.nodeset_is_gke(self.node_nodeset(node_name)) + + def nodeset_is_gke(self, nodeset=None) -> bool: + return "gke_nodepool" in nodeset + + def node_template(self, node_name=None) -> str: + """ Self link of nodeset template """ + return self.node_nodeset(node_name).instance_template + + def node_template_info(self, node_name=None): + return self.template_info(self.node_template(node_name)) + + def node_region(self, node_name=None): + nodeset = self.node_nodeset(node_name) + return parse_self_link(nodeset.subnetwork).region + + def nodeset_accelerator_topology(self, nodeset_name: str) -> Optional[str]: + if not self.nodeset_is_tpu(nodeset_name): + return getattr(self.cfg.nodeset[nodeset_name], 'accelerator_topology', None) + return None + + def nodeset_prefix(self, nodeset_name): + return f"{self.cfg.slurm_cluster_name}-{nodeset_name}" + + def nodelist_range(self, nodeset_name: str, start: int, count: int) -> str: + assert 0 <= start and 0 < count + pref = self.nodeset_prefix(nodeset_name) + if count == 1: + return f"{pref}-{start}" + return f"{pref}-[{start}-{start + count - 1}]" + + def static_dynamic_sizes(self, nodeset: NSDict) -> Tuple[int, int]: + return (nodeset.node_count_static or 0, nodeset.node_count_dynamic_max or 0) + + def nodelist(self, nodeset) -> str: + cnt = sum(self.static_dynamic_sizes(nodeset)) + if cnt == 0: + return "" + return self.nodelist_range(nodeset.nodeset_name, 0, cnt) + + def nodenames(self, nodeset) -> Tuple[Iterable[str], Iterable[str]]: + pref = self.nodeset_prefix(nodeset.nodeset_name) + s_count, d_count = self.static_dynamic_sizes(nodeset) + return ( + (f"{pref}-{i}" for i in range(s_count)), + (f"{pref}-{i}" for i in range(s_count, s_count + d_count)), + ) + + def power_managed_nodesets(self) -> Iterable[NSDict]: + return chain(self.cfg.nodeset.values(), self.cfg.nodeset_tpu.values()) + + def is_power_managed_node(self, node_name: str) -> bool: + try: + ns = self.node_nodeset(node_name) + if ns is None: + return False + idx = int(self._node_desc(node_name)["suffix"]) + return idx < sum(self.static_dynamic_sizes(ns)) + except Exception: + return False + + def is_static_node(self, node_name: str) -> bool: + if not self.is_power_managed_node(node_name): + return False + idx = int(self._node_desc(node_name)["suffix"]) + return idx < self.node_nodeset(node_name).node_count_static + + @lru_cache(maxsize=None) + def slurm_nodes(self) -> Dict[str, NodeState]: + def parse_line(node_line) -> Tuple[str, NodeState]: + """turn node,state line to (node, NodeState)""" + # state flags include: CLOUD, COMPLETING, DRAIN, FAIL, POWERED_DOWN, + # POWERING_DOWN + node, fullstate = node_line.split(",") + state = fullstate.split("+") + state_tuple = NodeState(base=state[0], flags=frozenset(state[1:])) + return (node, state_tuple) + + cmd = ( + f"{self.scontrol} show nodes | " + r"grep -oP '^NodeName=\K(\S+)|\s+State=\K(\S+)' | " + r"paste -sd',\n'" + ) + node_lines = run(cmd, shell=True).stdout.rstrip().splitlines() + nodes = { + node: state + for node, state in map(parse_line, node_lines) + if "CLOUD" in state.flags or "DYNAMIC_NORM" in state.flags + } + return nodes + + def node_state(self, nodename: str) -> Optional[NodeState]: + state = self.slurm_nodes().get(nodename) + if state is not None: + return state + + # state is None => Slurm doesn't know this node, + # there are two reasons: + # * happy: + # * node belongs to removed nodeset + # * node belongs to downsized portion of nodeset + # * dynamic node that didn't register itself + # * unhappy: + # * there is a drift in Slurm and SlurmGCP configurations + # * `slurm_nodes` function failed to handle `scontrol show nodes`, + # TODO: make `slurm_nodes` robust by using `scontrol show nodes --json` + # In either of "unhappy" cases it's too dangerous to proceed - abort slurmsync. + try: + ns = self.node_nodeset(nodename) + except: + log.info(f"Unknown node {nodename}, belongs to unknown nodeset") + return None # Can't find nodeset, may be belongs to removed nodeset + + if self.node_is_dyn(nodename): + log.info(f"Unknown node {nodename}, belongs to dynamic nodeset") + return None # we can't make any judjment for dynamic nodes + + cnt = sum(self.static_dynamic_sizes(ns)) + if self.node_index(nodename) >= cnt: + log.info(f"Unknown node {nodename}, out of nodeset size boundaries ({cnt})") + return None # node belongs to downsized nodeset + + raise RuntimeError(f"Slurm does not recognize node {nodename}, potential misconfiguration.") + + + @lru_cache(maxsize=1) + def instances(self) -> Dict[str, Instance]: + instance_information_fields = [ + "creationTimestamp", + "name", + "resourceStatus", + "scheduling", + "status", + "labels.slurm_instance_role", + "zone", + "metadata", + ] + + instance_fields = ",".join(sorted(instance_information_fields)) + fields = f"items.zones.instances({instance_fields}),nextPageToken" + flt = f"labels.slurm_cluster_name={self.cfg.slurm_cluster_name} AND name:{self.cfg.slurm_cluster_name}-*" + act = self.compute.instances() + op = act.aggregatedList(project=self.project, fields=fields, filter=flt) + + instances = {} + while op is not None: + result = ensure_execute(op) + for zone in result.get("items", {}).values(): + for jo in zone.get("instances", []): + inst = Instance.from_json(jo) + if inst.name in instances: + log.error(f"Duplicate VM name {inst.name} across multiple zones") + instances[inst.name] = inst + op = act.aggregatedList_next(op, result) + return instances + + def instance(self, instance_name: str) -> Optional[Instance]: + return self.instances().get(instance_name) + + @lru_cache() + def _get_reservation(self, project: str, zone: str, name: str) -> Any: + """See https://cloud.google.com/compute/docs/reference/rest/v1/reservations""" + return self.compute.reservations().get( + project=project, zone=zone, reservation=name).execute() + + @lru_cache() + def get_mig(self, project: str, region: str, self_link:str) -> Any: + """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" + return self.compute.regionInstanceGroupManagers().get(project=project, region=region, instanceGroupManager=self_link).execute() + + @lru_cache + def get_mig_instances(self, project: str, region: str, self_link:str) -> Any: + return self.compute.regionInstanceGroupManagers().listManagedInstances(project=project, region=region, instanceGroupManager=self_link).execute() + + @lru_cache() + def get_mig_list(self, project: str, region: str) -> Any: + """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" + return self.compute.regionInstanceGroupManagers().list(project=project, region=region).execute() + + @lru_cache() + def _get_future_reservation(self, project:str, zone:str, name: str) -> Any: + """See https://cloud.google.com/compute/docs/reference/rest/v1/futureReservations""" + return self.compute.futureReservations().get(project=project, zone=zone, futureReservation=name).execute() + + def get_reservation_details(self, project:str, zone:str, name:str, bulk_insert_name:str) -> ReservationDetails: + reservation = self._get_reservation(project, zone, name) + + # Converts policy URLs to names, e.g.: + # projects/111111/regions/us-central1/resourcePolicies/zebra -> zebra + policies = [u.split("/")[-1] for u in reservation.get("resourcePolicies", {}).values()] + + return ReservationDetails( + project=project, + zone=zone, + name=name, + policies=policies, + deployment_type=reservation.get("deploymentType"), + reservation_mode=reservation.get("reservationMode"), + assured_count=int(reservation.get("specificReservation", {}).get("assuredCount", 0)), + delete_at_time=parse_gcp_timestamp(reservation.get("deleteAtTime")) if reservation.get("deleteAtTime") else None, + bulk_insert_name=bulk_insert_name) + + def nodeset_reservation(self, nodeset: NSDict) -> Optional[ReservationDetails]: + if not nodeset.reservation_name: + return None + + zones = list(nodeset.zone_policy_allow or []) + assert len(zones) == 1, "Only single zone is supported if using a reservation" + zone = zones[0] + + regex = re.compile(r'^projects/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$') + if not (match := regex.match(nodeset.reservation_name)): + raise ValueError( + f"Invalid reservation name: '{nodeset.reservation_name}', expected format is 'projects/PROJECT/reservations/NAME'" + ) + + project, name = match.group("project", "reservation") + return self.get_reservation_details(project, zone, name, nodeset.reservation_name) + + def future_reservation(self, nodeset: NSDict) -> Optional[FutureReservation]: + if not nodeset.future_reservation: + return None + + active_reservation = None + match = re.search(r'^projects/(?P[^/]+)/zones/(?P[^/]+)/futureReservations/(?P[^/]+)(/.*)?$', nodeset.future_reservation) + assert match, f"Invalid future reservation name '{nodeset.future_reservation}'" + project, zone, name = match.group("project","zone","name") + fr = self._get_future_reservation(project,zone,name) + + start_time = parse_gcp_timestamp(fr["timeWindow"]["startTime"]) + end_time = parse_gcp_timestamp(fr["timeWindow"]["endTime"]) + + if "autoCreatedReservations" in fr["status"] and (res:=fr["status"]["autoCreatedReservations"][0]): + if start_time <= now() <=end_time: + match = re.search(r'projects/(?P[^/]+)/zones/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$',res) + assert match, f"Unexpected reservation name '{res}'" + res_name = match.group("name") + bulk_insert_name = f"projects/{project}/reservations/{res_name}" + active_reservation = self.get_reservation_details(project, zone, res_name, bulk_insert_name) + + return FutureReservation( + project=project, + zone=zone, + name=name, + specific=fr["specificReservationRequired"], + start_time=start_time, + end_time=end_time, + reservation_mode=fr.get("reservationMode"), + active_reservation=active_reservation + ) + + @lru_cache(maxsize=1) + def machine_types(self): + field_names = "name,zone,guestCpus,memoryMb,accelerators" + fields = f"items.zones.machineTypes({field_names}),nextPageToken" + + machines: Dict[str, Dict[str, Any]] = defaultdict(dict) + act = self.compute.machineTypes() + op = act.aggregatedList(project=self.project, fields=fields) + while op is not None: + result = ensure_execute(op) + machine_iter = chain.from_iterable( + scope.get("machineTypes", []) for scope in result["items"].values() + ) + for machine in machine_iter: + name = machine["name"] + zone = machine["zone"] + machines[name][zone] = machine + + op = act.aggregatedList_next(op, result) + return machines + + def machine_type(self, name: str) -> MachineType: + custom_patt = re.compile( + r"((?P\w+)-)?custom-(?P\d+)-(?P\d+)" + ) + if match := custom_patt.match(name): + return MachineType( + name=name, + guest_cpus=int(match.group("cpus")), + memory_mb=int(match.group("mem")), + accelerators=[], + ) + + machines = self.machine_types() + if name not in machines: + raise Exception(f"machine type {name} not found") + per_zone = machines[name] + assert per_zone + return MachineType.from_json( + next(iter(per_zone.values())) # pick the first/any zone + ) + + def template_machine_conf(self, template_link): + template = self.template_info(template_link) + machine = template.machine_type + + machine_conf = NSDict() + machine_conf.boards = 1 # No information, assume 1 + machine_conf.sockets = machine.sockets + # the value below for SocketsPerBoard must be type int + machine_conf.sockets_per_board = machine_conf.sockets // machine_conf.boards + machine_conf.threads_per_core = 1 + _div = 2 if getThreadsPerCore(template) == 1 else 1 + machine_conf.cpus = ( + int(machine.guest_cpus / _div) if machine.supports_smt else machine.guest_cpus + ) + machine_conf.cores_per_socket = int(machine_conf.cpus / machine_conf.sockets) + # Because the actual memory on the host will be different than + # what is configured (e.g. kernel will take it). From + # experiments, about 16 MB per GB are used (plus about 400 MB + # buffer for the first couple of GB's. Using 30 MB to be safe. + gb = machine.memory_mb // 1024 + machine_conf.memory = machine.memory_mb - (400 + (30 * gb)) + return machine_conf + + @lru_cache(maxsize=None) + def template_info(self, template_link): + template_name = trim_self_link(template_link) + cache = file_cache.cache("template_cache") + + if cached := cache.get(template_name): + return NSDict(cached) + + region = get_self_link_component(template_link, "regions") + + template = ensure_execute( + self.compute.instanceTemplates().get( + project=self.project, instanceTemplate=template_name + ) if region is None else + self.compute.regionInstanceTemplates().get( + project=self.project, region=region, instanceTemplate=template_name + ) + ).get("properties") + template = NSDict(template) + # name and link are not in properties, so stick them in + template.name = template_name + template.link = template_link + template.machine_type = self.machine_type(template.machineType) + # TODO delete metadata to reduce memory footprint? + # del template.metadata + + template.gpu = get_template_gpu(template) + + cache.set(template_name, template.to_dict()) + return template + + def _parse_job_info(self, job_info: str) -> Job: + """Extract job details""" + if match:= re.search(r"JobId=(\d+)", job_info): + job_id = int(match.group(1)) + else: + raise ValueError(f"Job ID not found in the job info: {job_info}") + + if match:= re.search(r"TimeLimit=(?:(\d+)-)?(\d{2}):(\d{2}):(\d{2})", job_info): + days, hours, minutes, seconds = match.groups() + duration = timedelta( + days=int(days) if days else 0, + hours=int(hours), + minutes=int(minutes), + seconds=int(seconds) + ) + else: + duration = None + + if match := re.search(r"JobName=([^\n]+)", job_info): + name = match.group(1) + else: + name = None + + if match := re.search(r"JobState=(\w+)", job_info): + job_state = match.group(1) + else: + job_state = None + + if match := re.search(r"ReqNodeList=([^ ]+)", job_info): + required_nodes = match.group(1) + else: + required_nodes = None + + return Job(id=job_id, duration=duration, name=name, job_state=job_state, required_nodes=required_nodes) + + @lru_cache + def get_jobs(self) -> List[Job]: + res = run(f"{self.scontrol} show jobs", timeout=30) + + return [self._parse_job_info(job) for job in res.stdout.split("\n\n")[:-1]] + + @lru_cache + def job(self, job_id: int) -> Optional[Job]: + job_info = run(f"{self.scontrol} show jobid {job_id}", check=False).stdout.rstrip() + if not job_info: + return None + + return self._parse_job_info(job_info=job_info) + + @property + def etc_dir(self) -> Path: + return Path(self.cfg.output_dir or slurmdirs.etc) + + def controller_mount_server_ip(self) -> str: + return self.control_addr or self.control_host + + def normalize_ns_mount(self, ns: Union[dict, NSMount]) -> NSMount: + if isinstance(ns, NSMount): + return ns + + server_ip = ns.get("server_ip") or "$controller" + if server_ip == "$controller": + server_ip = self.controller_mount_server_ip() + + return NSMount( + server_ip=server_ip, + local_mount=Path(ns["local_mount"]), + remote_mount=Path(ns["remote_mount"]), + fs_type=ns["fs_type"], + mount_options=ns["mount_options"], + ) + + @property + def munge_mount(self) -> NSMount: + if self.cfg.munge_mount: + mnt = self.cfg.munge_mount + mnt.local_mount = mnt.local_mount or "/mnt/munge" + return self.normalize_ns_mount(mnt) + else: + return NSMount( + server_ip=self.controller_mount_server_ip(), + local_mount=Path("/mnt/munge"), + remote_mount=dirs.munge, + fs_type="nfs", + mount_options="defaults,hard,intr,_netdev", + ) + + @property + def slurm_key_mount(self) -> NSMount: + if self.cfg.slurm_key_mount: + mnt = self.cfg.slurm_key_mount + mnt.local_mount = mnt.local_mount or slurmdirs.key_distribution + return self.normalize_ns_mount(mnt) + else: + return NSMount( + server_ip=self.controller_mount_server_ip(), + local_mount=slurmdirs.key_distribution, + remote_mount=slurmdirs.key_distribution, + fs_type="nfs", + mount_options="defaults,hard,intr,_netdev", + ) + + def is_flex_node(self, node: str) -> bool: + try: + nodeset = self.node_nodeset(node) + if nodeset.dws_flex.use_bulk_insert: + return False #For legacy flex support + return bool(nodeset.dws_flex.enabled) + except: + return False + + def is_provisioning_flex_node(self, node:str) -> bool: + if not self.is_flex_node(node): + return False + if self.instance(node) is not None: + return True + + nodeset = self.node_nodeset(node) + zones = nodeset.zone_policy_allow + assert len(zones) > 0 + region = self.node_region(node) + + potential_migs=[] + mig_list=self.get_mig_list(self.project, region) + + if not mig_list or not mig_list.get("items"): + return False + + for mig in mig_list["items"]: + if not mig.get("instanceTemplate"): #possibly an old MIG + return False + if mig["instanceTemplate"] == self.node_template(node) and mig["currentActions"]["creating"] > 0: + potential_migs.append(self.get_mig_instances(self.project, region, trim_self_link(mig["selfLink"]))) + + if not potential_migs: + return False + + for instance_collection in potential_migs[0]["managedInstances"]: + if node in instance_collection["name"] and instance_collection["currentAction"]=="CREATING": + return True + return False + + def cluster_regions(self) -> list[str]: + """ + Returns all regions used in cluster + NOTE: only concerned with normal nodesets, + neither TPU, nor dynamic, nor login node, nor controller node are considered + """ + res = set() + for nodeset in self.cfg.nodeset.values(): + res.add(parse_self_link(nodeset.subnetwork).region) + return list(res) + + + +_lkp: Optional[Lookup] = None + +def _load_config() -> NSDict: + return NSDict(yaml.safe_load(CONFIG_FILE.read_text())) + +def lookup() -> Lookup: + global _lkp + if _lkp is None: + try: + cfg = _load_config() + except FileNotFoundError: + log.error(f"config file not found: {CONFIG_FILE}") + cfg = NSDict() # TODO: fail here, once all code paths are covered (mainly init_logging) + _lkp = Lookup(cfg) + return _lkp + +def update_config(cfg: NSDict) -> None: + global _lkp + _lkp = Lookup(cfg) + +def scontrol_reconfigure(lkp: Lookup) -> None: + log.info("Running systemctl restart slurmctld.service") + run("sudo systemctl restart slurmctld.service", timeout=30) + log.info("Running scontrol reconfigure") + run(f"{lkp.scontrol} reconfigure") diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py new file mode 100644 index 0000000000..d1d77a1833 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py @@ -0,0 +1,124 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + + +from dataclasses import dataclass, asdict +import util +import local_pubsub + +import logging +log = logging.getLogger() + +# Name of the topic +TOPIC = "watch_delete_vm_op" + +@dataclass(frozen=True) +class WatchDeleteVmOp_Message: + op_name: str + zone: str + node: str + +class WatchDeleteVmOp_Topic: + def __init__(self, topic: local_pubsub.Topic) -> None: + self._t = topic + + def publish(self, op: dict[str, Any], node: str) -> None: + assert op.get("operationType") == "delete" + assert op.get("zone") + assert node + + msg = WatchDeleteVmOp_Message(op_name=op["name"], zone=op["zone"], node=node) + self._t.publish(data=asdict(msg)) + + +def watch_delete_vm_op_topic() -> WatchDeleteVmOp_Topic: + return WatchDeleteVmOp_Topic(local_pubsub.topic(TOPIC)) + + +def _watch_op(lkp: util.Lookup, m: WatchDeleteVmOp_Message) -> bool: + """ + Processes VM delete-operation. + If operation is still running - do nothing + If operation failed - log error & remove op from watch list + If operation is done - remove op from watch list do nothing + + To avoid querying status for each op individually, use list of VM instances as + a source of data. Don't query op for instance X if instance X is not present + (presumably deleted). + NOTE: This optimization can lead to false-positives - + absence of error-logs in case op failed, but VM got deleted by other means. + + Returns True if message should be marked as processed (ack). + """ + + inst = lkp.instance(m.node) + + if not inst: + log.debug(f"Stop watching op {m.op_name}, VM {m.node} appears to be deleted") + return True # ack, potentially false-positive + + if inst.status == "TERMINATED": + log.debug(f"Stop watching op {m.op_name}, VM {m.node} is TERMINATED") + return True # ack, potentially false-positive + + if inst.status == "STOPPING": + log.debug(f"Skipping op {m.op_name}, VM {m.node} is STOPPING") + return False # try later + + try: + op = util.get_operation_req(lkp, m.op_name, zone=m.zone).execute() + except: + # TODO: consider less conservative handling, but be careful not to cause deadlettering. + log.exception(f"Failed to get operation {m.op_name}, will not retry") + return True # ack (remove) + + if op["status"] != "DONE": + log.debug(f"Watching op {m.op_name} is still not done ({op['status']})") + return False # try later + + if "error" in op: + log.error(f"Operation {m.op_name} to delete {m.node} finished with error: {op['error']}") + else: + log.debug(f"Operation {m.op_name} to delete {m.node} successfully finished") + return True # ack + + +def watch_vm_delete_ops(lkp: util.Lookup) -> None: + sub = local_pubsub.subscription(TOPIC) + + # Pull once instead of "pulling until empty", motivation: + # Bulk of cases processed by `_watch_op` relies on freshness of `lkp.instances`, + # `lkp.instances` are fetched once during run of `slurmsync`. + # Therefore we shouldn't try to re-process messages that has been already NACKed in this run, + # since they will be handled with the same `lkp.instance` as a previous attempt. + msgs = sub.pull(max_messages=1000) # 1000 is arbitrary number to be adjusted if needed. + log.debug(f"Processing {len(msgs)} delete VM operations") + # TODO: handle messages in butches to improve latency + for m in msgs: + try: + dm = WatchDeleteVmOp_Message(**m.data) + ack = _watch_op(lkp, dm) + except Exception: + log.exception(f"Failed to process the message {m.id}, removing") + ack = True + if ack: + sub.ack([m.id]) + else: + sub.modify_ack_deadline([m.id], deadline=0) # NACK + + + + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf new file mode 100644 index 0000000000..71905a0342 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf @@ -0,0 +1,504 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "bucket_name" { + description = <<-EOD + Name of GCS bucket to use. + EOD + type = string +} + +variable "bucket_dir" { + description = "Bucket directory for cluster files to be put into." + type = string + default = null +} + +variable "enable_debug_logging" { + type = bool + description = "Enables debug logging mode. Not for production use." + default = false +} + +variable "extra_logging_flags" { + type = map(bool) + description = "The only available flag is `trace_api`" + default = {} +} + +variable "project_id" { + description = "The GCP project ID." + type = string +} + +variable "enable_slurm_auth" { + description = < x... } + nodeset_map = { for k, vs in local.nodeset_map_ell : k => vs[0] } + + nodeset_tpu_map_ell = { for x in var.nodeset_tpu : x.nodeset_name => x... } + nodeset_tpu_map = { for k, vs in local.nodeset_tpu_map_ell : k => vs[0] } + + nodeset_dyn_map_ell = { for x in var.nodeset_dyn : x.nodeset_name => x... } + nodeset_dyn_map = { for k, vs in local.nodeset_dyn_map_ell : k => vs[0] } + + + no_reservation_affinity = { type : "NO_RESERVATION" } +} + +# NODESET +module "slurm_nodeset_template" { + source = "../../internal/slurm-gcp/instance_template" + for_each = local.nodeset_map + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + slurm_instance_role = "compute" + slurm_bucket_path = module.slurm_files.slurm_bucket_path + + additional_disks = each.value.additional_disks + bandwidth_tier = each.value.bandwidth_tier + can_ip_forward = each.value.can_ip_forward + advanced_machine_features = each.value.advanced_machine_features + disk_auto_delete = each.value.disk_auto_delete + disk_labels = each.value.disk_labels + disk_resource_manager_tags = each.value.disk_resource_manager_tags + disk_size_gb = each.value.disk_size_gb + disk_type = each.value.disk_type + enable_confidential_vm = each.value.enable_confidential_vm + enable_oslogin = each.value.enable_oslogin + enable_shielded_vm = each.value.enable_shielded_vm + gpu = each.value.gpu + labels = merge(each.value.labels, { slurm_nodeset = each.value.nodeset_name }) + machine_type = each.value.machine_type + metadata = merge(each.value.metadata, local.universe_domain) + min_cpu_platform = each.value.min_cpu_platform + name_prefix = each.value.nodeset_name + on_host_maintenance = each.value.on_host_maintenance + preemptible = each.value.preemptible + region = each.value.region + resource_manager_tags = each.value.resource_manager_tags + spot = each.value.spot + termination_action = each.value.termination_action + service_account = each.value.service_account + shielded_instance_config = each.value.shielded_instance_config + source_image_family = each.value.source_image_family + source_image_project = each.value.source_image_project + source_image = each.value.source_image + subnetwork = each.value.subnetwork_self_link + additional_networks = each.value.additional_networks + access_config = each.value.access_config + tags = concat([local.slurm_cluster_name], each.value.tags) + + max_run_duration = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? each.value.dws_flex.max_run_duration : null + provisioning_model = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? "FLEX_START" : null + reservation_affinity = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? local.no_reservation_affinity : null +} + +module "nodeset_cleanup" { + source = "./modules/cleanup_compute" + for_each = local.nodeset_map + + nodeset = each.value + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + enable_cleanup_compute = var.enable_cleanup_compute + universe_domain = var.universe_domain + endpoint_versions = var.endpoint_versions + gcloud_path_override = var.gcloud_path_override + nodeset_template = module.slurm_nodeset_template[each.value.nodeset_name].self_link +} + +locals { + nodesets = [for name, ns in local.nodeset_map : { + nodeset_name = ns.nodeset_name + node_conf = ns.node_conf + dws_flex = ns.dws_flex + instance_template = module.slurm_nodeset_template[ns.nodeset_name].self_link + node_count_dynamic_max = ns.node_count_dynamic_max + node_count_static = ns.node_count_static + subnetwork = ns.subnetwork_self_link + reservation_name = ns.reservation_name + future_reservation = ns.future_reservation + maintenance_interval = ns.maintenance_interval + instance_properties_json = ns.instance_properties_json + enable_placement = ns.enable_placement + placement_max_distance = ns.placement_max_distance + network_storage = ns.network_storage + zone_target_shape = ns.zone_target_shape + zone_policy_allow = ns.zone_policy_allow + zone_policy_deny = ns.zone_policy_deny + enable_maintenance_reservation = ns.enable_maintenance_reservation + enable_opportunistic_maintenance = ns.enable_opportunistic_maintenance + accelerator_topology = ns.accelerator_topology + }] +} + +# NODESET TPU +module "slurm_nodeset_tpu" { + source = "../../internal/slurm-gcp/nodeset_tpu" + for_each = local.nodeset_tpu_map + + project_id = var.project_id + node_count_dynamic_max = each.value.node_count_dynamic_max + node_count_static = each.value.node_count_static + nodeset_name = each.value.nodeset_name + zone = each.value.zone + node_type = each.value.node_type + accelerator_config = each.value.accelerator_config + tf_version = each.value.tf_version + preemptible = each.value.preemptible + preserve_tpu = each.value.preserve_tpu + enable_public_ip = each.value.enable_public_ip + service_account = each.value.service_account + data_disks = each.value.data_disks + docker_image = each.value.docker_image + subnetwork = each.value.subnetwork +} + +module "nodeset_cleanup_tpu" { + source = "./modules/cleanup_tpu" + for_each = local.nodeset_tpu_map + + nodeset = { + nodeset_name = each.value.nodeset_name + zone = each.value.zone + } + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + enable_cleanup_compute = var.enable_cleanup_compute + universe_domain = var.universe_domain + endpoint_versions = var.endpoint_versions + gcloud_path_override = var.gcloud_path_override + + depends_on = [ + # Depend on controller network, as a best effort to avoid + # subnetwork resourceInUseByAnotherResource error + var.subnetwork_self_link + ] +} + +resource "google_storage_bucket_object" "parition_config" { + for_each = { for p in var.partitions : p.partition_name => p } + + bucket = module.slurm_files.bucket_name + name = "${module.slurm_files.bucket_dir}/partition_configs/${each.key}.yaml" + content = yamlencode(each.value) + source_md5hash = md5(yamlencode(each.value)) +} + +moved { + from = module.slurm_files.google_storage_bucket_object.parition_config + to = google_storage_bucket_object.parition_config +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf new file mode 100644 index 0000000000..218c36e392 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf @@ -0,0 +1,191 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# BUCKET + +locals { + synt_suffix = substr(md5("${local.controller_project_id}${var.deployment_name}"), 0, 5) + synth_bucket_name = "${local.slurm_cluster_name}${local.synt_suffix}" + + bucket_name = var.create_bucket ? module.bucket[0].name : var.bucket_name +} + +module "bucket" { + source = "terraform-google-modules/cloud-storage/google" + version = ">= 6.1" + + count = var.create_bucket ? 1 : 0 + + location = var.region + names = [local.synth_bucket_name] + prefix = "slurm" + project_id = local.controller_project_id + + force_destroy = { + (local.synth_bucket_name) = true + } + + labels = merge(local.labels, { + slurm_cluster_name = local.slurm_cluster_name + }) +} + +# BUCKET IAMs +locals { + compute_sa = toset(flatten([for x in module.slurm_nodeset_template : x.service_account])) + compute_tpu_sa = toset(flatten([for x in module.slurm_nodeset_tpu : x.service_account])) + login_sa = toset(flatten([for x in module.login : x.service_account])) + + viewers = toset(flatten([ + "serviceAccount:${module.slurm_controller_template.service_account.email}", + formatlist("serviceAccount:%s", [for x in local.compute_sa : x.email]), + formatlist("serviceAccount:%s", [for x in local.compute_tpu_sa : x.email if x.email != null]), + formatlist("serviceAccount:%s", [for x in local.login_sa : x.email]), + ])) +} + + +resource "google_storage_bucket_iam_member" "viewers" { + for_each = local.viewers + bucket = local.bucket_name + role = "roles/storage.objectViewer" + member = each.value +} + +resource "google_storage_bucket_iam_member" "legacy_readers" { + for_each = local.viewers + bucket = local.bucket_name + role = "roles/storage.legacyBucketReader" + member = each.value +} + +locals { + daos_ns = [ + for ns in var.network_storage : + ns if ns.fs_type == "daos" + ] + + daos_client_install_runners = [ + for ns in local.daos_ns : + ns.client_install_runner if ns.client_install_runner != null + ] + + daos_mount_runners = [ + for ns in local.daos_ns : + ns.mount_runner if ns.mount_runner != null + ] + + daos_network_storage_runners = concat( + local.daos_client_install_runners, + local.daos_mount_runners, + ) + + daos_install_mount_script = { + filename = "ghpc_daos_mount.sh" + content = length(local.daos_ns) > 0 ? module.daos_network_storage_scripts[0].startup_script : "" + } + + common_scripts = length(local.daos_ns) > 0 ? [local.daos_install_mount_script] : [] +} + +# SLURM FILES +locals { + ghpc_startup_script_controller = concat( + local.common_scripts, + [{ + filename = "ghpc_startup.sh" + content = var.controller_startup_script + }]) + + controller_state_disk = { + device_name : try(google_compute_disk.controller_disk[0].name, null) + } + + + nodeset_startup_scripts = { for k, v in local.nodeset_map : k => concat(local.common_scripts, v.startup_script) } +} + +module "daos_network_storage_scripts" { + count = length(local.daos_ns) > 0 ? 1 : 0 + + source = "../../../../modules/scripts/startup-script" + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.daos_network_storage_runners +} + +module "slurm_files" { + source = "./modules/slurm_files" + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + bucket_dir = var.bucket_dir + bucket_name = local.bucket_name + controller_network_attachment = var.controller_network_attachment + + slurmdbd_conf_tpl = var.slurmdbd_conf_tpl + slurm_conf_tpl = var.slurm_conf_tpl + slurm_conf_template = var.slurm_conf_template + cgroup_conf_tpl = var.cgroup_conf_tpl + cloud_parameters = var.cloud_parameters + cloudsql_secret = try( + one(google_secret_manager_secret_version.cloudsql_version[*].id), + null) + + controller_startup_scripts = local.ghpc_startup_script_controller + controller_startup_scripts_timeout = var.controller_startup_scripts_timeout + nodeset_startup_scripts = local.nodeset_startup_scripts + compute_startup_scripts_timeout = var.compute_startup_scripts_timeout + controller_state_disk = local.controller_state_disk + + enable_debug_logging = var.enable_debug_logging + extra_logging_flags = var.extra_logging_flags + + enable_slurm_auth = var.enable_slurm_auth + + enable_bigquery_load = var.enable_bigquery_load + enable_external_prolog_epilog = var.enable_external_prolog_epilog + enable_chs_gpu_health_check_prolog = var.enable_chs_gpu_health_check_prolog + enable_chs_gpu_health_check_epilog = var.enable_chs_gpu_health_check_epilog + epilog_scripts = var.epilog_scripts + prolog_scripts = var.prolog_scripts + task_epilog_scripts = var.task_epilog_scripts + task_prolog_scripts = var.task_prolog_scripts + + disable_default_mounts = !var.enable_default_mounts + network_storage = [ + for storage in var.network_storage : { + server_ip = storage.server_ip, + remote_mount = storage.remote_mount, + local_mount = storage.local_mount, + fs_type = storage.fs_type, + mount_options = storage.mount_options + } + if storage.fs_type != "daos" + ] + + nodeset = local.nodesets + nodeset_dyn = values(local.nodeset_dyn_map) + # Use legacy format for now + nodeset_tpu = values(module.slurm_nodeset_tpu)[*] + + + depends_on = [module.bucket] + + # Providers + endpoint_versions = var.endpoint_versions +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf new file mode 100644 index 0000000000..85ad10fa21 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf @@ -0,0 +1,814 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########### +# GENERAL # +########### + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "deployment_name" { + description = "Name of the deployment." + type = string +} + +variable "slurm_cluster_name" { + type = string + description = <<-EOD + Cluster name, used for resource naming and slurm accounting. + If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). + EOD + default = null + + validation { + condition = var.slurm_cluster_name == null || can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) + error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." + } +} + +variable "region" { + type = string + description = "The default region to place resources in." +} + +variable "zone" { + type = string + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disable\_login\_public\_ips](#input\_disable\_login\_public\_ips) | DEPRECATED: Use `enable_login_public_ips` instead. | `bool` | `null` | no | +| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_login\_public\_ips](#input\_enable\_login\_public\_ips) | If set to true. The login node will have a random public IP assigned to it. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for login nodes. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Unique name prefix for login nodes. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all login groups. | `string` | n/a | yes | +| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the login instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the login instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [login\_nodes](#output\_login\_nodes) | Slurm login instance definition. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf new file mode 100644 index 0000000000..6ebe5902dc --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf @@ -0,0 +1,115 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-login", ghpc_role = "scheduler" }) +} + +module "instance_validation" { + source = "../../../../modules/internal/instance_validations" + + machine_type = var.machine_type + disk_type = var.disk_type +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata + ) + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.disk_resource_manager_tags + } + ] + + public_access_config = [{ nat_ip = null, network_tier = null }] + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + # lower, replace `_` with `-`, and remove any non-alphanumeric characters + group_name = replace( + replace( + lower(var.name_prefix), + "_", "-"), + "/[^-a-z0-9]/", "") + + + login_node = { + group_name = local.group_name + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(var.disk_labels, local.labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + additional_networks = var.additional_networks + + can_ip_forward = var.can_ip_forward + advanced_machine_features = var.advanced_machine_features + + enable_confidential_vm = var.enable_confidential_vm + access_config = var.enable_login_public_ips ? local.public_access_config : [] + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + gpu = one(local.guest_accelerator) + labels = local.labels + machine_type = var.machine_type + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + num_instances = var.num_instances + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + region = var.region + resource_manager_tags = var.resource_manager_tags + zone = var.zone + + service_account = local.service_account + + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + static_ips = var.static_ips + bandwidth_tier = var.bandwidth_tier + + subnetwork = var.subnetwork_self_link + tags = var.tags + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml new file mode 100644 index 0000000000..47f003258e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name_prefix + has_to_be_used: true diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf new file mode 100644 index 0000000000..e700542794 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "login_nodes" { + description = "Slurm login instance definition." + value = [local.login_node] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf new file mode 100644 index 0000000000..db6cfc1318 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${var.instance_image.project}/global/images/family" : + "projects/${var.instance_image.project}/global/images" + ) + source_image_family = try(var.instance_image.family, "") + source_image = try(var.instance_image.name, "") +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf new file mode 100644 index 0000000000..7c1a2e06b5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf @@ -0,0 +1,419 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +variable "project_id" { # tflint-ignore: terraform_unused_declarations + type = string + description = "Project ID to create resources in." +} + +variable "region" { + type = string + description = "Region where the instances should be created." + default = null +} + +variable "zone" { + type = string + description = <<-EOD + Zone where the instances should be created. If not specified, instances will be + spread across available zones in the region. + EOD + default = null +} + +variable "name_prefix" { + type = string + description = <<-EOD + Unique name prefix for login nodes. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all login groups. + EOD +} + +variable "num_instances" { + type = number + description = "Number of instances to create. This value is ignored if static_ips is provided." + default = 1 +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} +} + +variable "disk_type" { + type = string + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + default = "pd-ssd" +} + +variable "disk_size_gb" { + type = number + description = "Boot disk size in GB." + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + type = list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string)) + auto_delete = optional(bool) + boot = optional(bool) + disk_resource_manager_tags = optional(map(string)) + })) + description = "List of maps of disks." + default = [] +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })) + nullable = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "disable_smt" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + type = bool + default = null + validation { + condition = var.disable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "static_ips" { + type = list(string) + description = "List of static IPs for VM instances." + default = [] +} + +variable "bandwidth_tier" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 6.16 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.16 | +| [helm](#provider\_helm) | ~> 2.17 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.cert_manager](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.prometheus](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.slurm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.slurm_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cert\_manager\_chart\_version](#input\_cert\_manager\_chart\_version) | Version of the Cert Manager chart to install. | `string` | `"v1.18.2"` | no | +| [cert\_manager\_values](#input\_cert\_manager\_values) | Value overrides for the Cert Manager release | `any` | 
{
  "crds": {
    "enabled": true
  }
}
| no | +| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | +| [install\_kube\_prometheus\_stack](#input\_install\_kube\_prometheus\_stack) | Install the Kube Prometheus Stack. | `bool` | `false` | no | +| [install\_slurm\_chart](#input\_install\_slurm\_chart) | Install slurm-operator chart. | `bool` | `true` | no | +| [install\_slurm\_operator\_chart](#input\_install\_slurm\_operator\_chart) | Install slurm-operator chart. | `bool` | `true` | no | +| [node\_pool\_names](#input\_node\_pool\_names) | Names of node pools, for use in node affinities (Slinky system components). | `list(string)` | `null` | no | +| [project\_id](#input\_project\_id) | The project ID that hosts the GKE cluster. | `string` | n/a | yes | +| [prometheus\_chart\_version](#input\_prometheus\_chart\_version) | Version of the Kube Prometheus Stack chart to install. | `string` | `"77.0.1"` | no | +| [prometheus\_values](#input\_prometheus\_values) | Value overrides for the Prometheus release | `any` | 
{
  "installCRDs": true
}
| no | +| [slurm\_chart\_version](#input\_slurm\_chart\_version) | Version of the Slurm chart to install. | `string` | `"0.3.1"` | no | +| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | +| [slurm\_operator\_chart\_version](#input\_slurm\_operator\_chart\_version) | Version of the Slurm Operator chart to install. | `string` | `"0.3.1"` | no | +| [slurm\_operator\_namespace](#input\_slurm\_operator\_namespace) | slurm namespace for charts | `string` | `"slinky"` | no | +| [slurm\_operator\_repository](#input\_slurm\_operator\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | +| [slurm\_operator\_values](#input\_slurm\_operator\_values) | Value overrides for the Slinky release | `any` | `{}` | no | +| [slurm\_repository](#input\_slurm\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | +| [slurm\_values](#input\_slurm\_values) | Value overrides for the Slurm release | `any` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [slurm\_namespace](#output\_slurm\_namespace) | namespace for the slurm chart | +| [slurm\_operator\_namespace](#output\_slurm\_operator\_namespace) | namespace for the slinky operator chart | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/main.tf new file mode 100644 index 0000000000..aff33b73a0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/main.tf @@ -0,0 +1,197 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] + project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] + + # Define affinity settings when node pools are specified + node_pool_affinity = var.node_pool_names != null ? { + nodeAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [{ + matchExpressions = [{ + key = "cloud.google.com/gke-nodepool" + operator = "In" + values = var.node_pool_names + }] + }] + } + } + } : {} +} + +data "google_client_config" "default" {} + +data "google_container_cluster" "gke_cluster" { + project = local.project_id + name = local.cluster_name + location = local.cluster_location +} + +resource "helm_release" "cert_manager" { + name = "cert-manager" + chart = "cert-manager" + repository = "https://charts.jetstack.io" + version = var.cert_manager_chart_version + namespace = "cert-manager" + create_namespace = true + + values = concat( + [yamlencode({ + affinity = local.node_pool_affinity + webhook = { + affinity = local.node_pool_affinity + } + cainjector = { + affinity = local.node_pool_affinity + } + startupapicheck = { + affinity = local.node_pool_affinity + } + })], + [yamlencode(var.cert_manager_values)] + ) +} + +resource "helm_release" "slurm_operator" { + count = var.install_slurm_operator_chart ? 1 : 0 + name = "slurm-operator" + chart = "slurm-operator" + repository = var.slurm_operator_repository + version = var.slurm_operator_chart_version + namespace = var.slurm_operator_namespace + create_namespace = true + + # The Cert Manager webhook deployment must be running to provision the Operator + depends_on = [ + helm_release.cert_manager + ] + + values = concat( + [yamlencode({ + operator = { + affinity = local.node_pool_affinity + } + webhook = { + affinity = local.node_pool_affinity + } + })], + [yamlencode(var.slurm_operator_values)] + ) +} + +resource "helm_release" "slurm" { + count = var.install_slurm_chart ? 1 : 0 + name = "slurm" + chart = "slurm" + repository = var.slurm_repository + version = var.slurm_chart_version + namespace = var.slurm_namespace + create_namespace = true + + # The Slurm Operator must be running to provision Slurm clusters/nodesets + depends_on = [ + helm_release.slurm_operator + ] + + values = concat( + [yamlencode({ + controller = { + affinity = local.node_pool_affinity + } + accounting = { + affinity = local.node_pool_affinity + } + mariadb = { + primary = { + affinity = local.node_pool_affinity + } + secondary = { + affinity = local.node_pool_affinity + } + } + restapi = { + affinity = local.node_pool_affinity + } + slurm-exporter = { + exporter = { + affinity = local.node_pool_affinity + } + } + })], + [yamlencode(var.slurm_values)] + ) +} + +resource "helm_release" "prometheus" { + count = var.install_kube_prometheus_stack ? 1 : 0 + name = "prometheus" + chart = "kube-prometheus-stack" + repository = "https://prometheus-community.github.io/helm-charts" + version = var.prometheus_chart_version + namespace = "prometheus" + create_namespace = true + + values = concat( + [yamlencode({ + crds = { + upgradeJob = { + affinity = local.node_pool_affinity + } + } + alertmanager = { + alertmanagerSpec = { + affinity = local.node_pool_affinity + } + } + prometheusOperator = { + admissionWebhooks = { + deployment = { + affinity = local.node_pool_affinity + } + patch = { + affinity = local.node_pool_affinity + } + } + affinity = local.node_pool_affinity + } + prometheus = { + prometheusSpec = { + affinity = local.node_pool_affinity + } + } + thanosRuler = { + thanosRulerSpec = { + affinity = local.node_pool_affinity + } + } + kube-state-metrics = { + affinity = local.node_pool_affinity + } + grafana = { + affinity = local.node_pool_affinity + imageRenderer = { + affinity = local.node_pool_affinity + } + } + prometheus-windows-exporter = { + affinity = local.node_pool_affinity + } + })], + [yamlencode(var.prometheus_values)] + ) +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/metadata.yaml new file mode 100644 index 0000000000..e18197e2b7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/outputs.tf new file mode 100644 index 0000000000..8ea6385905 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/outputs.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "slurm_namespace" { + description = "namespace for the slurm chart" + value = var.slurm_namespace +} + +output "slurm_operator_namespace" { + description = "namespace for the slinky operator chart" + value = var.slurm_operator_namespace +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/providers.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/providers.tf new file mode 100644 index 0000000000..313d6dc58e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/providers.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +provider "helm" { + kubernetes { + host = "https://${data.google_container_cluster.gke_cluster.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode( + data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate, + ) + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/variables.tf new file mode 100644 index 0000000000..8acaf78562 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/variables.tf @@ -0,0 +1,127 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "The project ID that hosts the GKE cluster." + type = string +} + +variable "cluster_id" { + description = "An identifier for the GKE cluster resource with format projects//locations//clusters/." + type = string + nullable = false +} + +variable "node_pool_names" { + description = "Names of node pools, for use in node affinities (Slinky system components)." + type = list(string) + default = null +} + +variable "cert_manager_chart_version" { + description = "Version of the Cert Manager chart to install." + type = string + default = "v1.18.2" +} + +variable "cert_manager_values" { + description = "Value overrides for the Cert Manager release" + type = any + default = { + crds = { + enabled = true + } + } +} + +variable "slurm_operator_chart_version" { + description = "Version of the Slurm Operator chart to install." + type = string + default = "0.3.1" +} + +variable "slurm_operator_values" { + description = "Value overrides for the Slinky release" + type = any + default = {} +} + +variable "slurm_chart_version" { + description = "Version of the Slurm chart to install." + type = string + default = "0.3.1" +} + +variable "slurm_values" { + description = "Value overrides for the Slurm release" + type = any + default = {} +} + +variable "install_kube_prometheus_stack" { + # Components detailed at https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack + description = "Install the Kube Prometheus Stack." + type = bool + default = false +} + +variable "prometheus_chart_version" { + description = "Version of the Kube Prometheus Stack chart to install." + type = string + default = "77.0.1" +} + +variable "prometheus_values" { + description = "Value overrides for the Prometheus release" + type = any + default = { + installCRDs = true + } +} + +variable "slurm_namespace" { + description = "slurm namespace for charts" + type = string + default = "slurm" +} + +variable "slurm_operator_namespace" { + description = "slurm namespace for charts" + type = string + default = "slinky" +} + +variable "install_slurm_chart" { + description = "Install slurm-operator chart." + type = bool + default = true +} + +variable "install_slurm_operator_chart" { + description = "Install slurm-operator chart." + type = bool + default = true +} + +variable "slurm_repository" { + description = "Value overrides for the Slinky release" + type = string + default = "oci://ghcr.io/slinkyproject/charts" +} + +variable "slurm_operator_repository" { + description = "Value overrides for the Slinky release" + type = string + default = "oci://ghcr.io/slinkyproject/charts" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/versions.tf new file mode 100644 index 0000000000..ae4327aeef --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/versions.tf @@ -0,0 +1,28 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + google = { + source = "hashicorp/google" + version = ">= 6.16" + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/README.md new file mode 100644 index 0000000000..71a862fd6c --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/README.md @@ -0,0 +1,149 @@ +## Description + +This module creates a Toolkit runner that will install HTCondor on RedHat 7 or +8 and its derivative operating systems. These include the CentOS 7 and Rocky +Linux 8 releases of the [HPC VM Image][hpcvmimage]. It may also function on +RedHat 9 and derivatives, however it is not yet supported. Please report any +[issues] on these 3 distributions or open a [discussion] to request support on +Debian or Ubuntu distributions. + +[issues]: https://github.com/GoogleCloudPlatform/hpc-toolkit/issues +[discussion]: https://github.com/GoogleCloudPlatform/hpc-toolkit/discussions + +It also exports a list of Google Cloud APIs which must be enabled prior to +provisioning an HTCondor Pool. + +It is expected to be used with the [htcondor-setup] and +[htcondor-execute-point] modules. + +[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[htcondor-setup]: ../../scheduler/htcondor-setup/README.md +[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md + +### Example + +The following code snippet uses this module to create startup scripts that +install the HTCondor software into a custom VM image. + +```yaml +deployment_groups: +- group: primary + modules: + - id: network1 + source: modules/network/vpc + outputs: + - network_name + + - id: htcondor_install + source: community/modules/scripts/htcondor-install + + - id: htcondor_install_script + source: modules/scripts/startup-script + use: + - htcondor_install + +- group: packer + modules: + - id: custom-image + source: modules/packer/custom-image + kind: packer + use: + - network1 + - htcondor_install_script + settings: + disk_size: 50 + source_image_family: hpc-rocky-linux-8 + image_family: "htcondor-10x" +``` + +A full example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +## Important note + +All POSIX users and HTCondor jobs can act as the service account attached to +VMs within the pool. This enables the use of IAM restrictions via service +accounts but also allows users to access services to which system daemons need +access (e.g. to create Cloud Logging entries). If this is undesirable, one can +restrict access to the instance metadata server to the `root` and `condor` +users. This will allow system services to use the service account, but not +other POSIX users or HTCondor jobs. The firewall example below is appropriate +for CentOS 7. + +```shell +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 1 \ + -m owner --uid-owner root -p tcp -d metadata.google.internal --dport 80 -j ACCEPT +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 2 \ + -m owner --uid-owner condor -p tcp -d metadata.google.internal --dport 80 -j ACCEPT +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 3 \ + -p tcp -d metadata.google.internal --dport 80 -j DROP +firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 4 \ + -p tcp -d metadata.google.internal --dport 8080 -j DROP +firewall-cmd --permanent --zone=public --add-port=9618/tcp +firewall-cmd --reload +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [condor\_version](#input\_condor\_version) | Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: "23.0.0","23.*")) | `string` | `"23.*"` | no | +| [enable\_docker](#input\_enable\_docker) | Install and enable docker daemon alongside HTCondor | `bool` | `true` | no | +| [http\_proxy](#input\_http\_proxy) | Set system default web (http and https) proxy for Windows HTCondor installation | `string` | `""` | no | +| [python\_windows\_installer\_url](#input\_python\_windows\_installer\_url) | URL of Python installer for Windows | `string` | `"https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcp\_service\_list](#output\_gcp\_service\_list) | Google Cloud APIs required by HTCondor | +| [runners](#output\_runners) | Runner to install HTCondor using startup-scripts | +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | Windows PowerShell script to install HTCondor | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py new file mode 100644 index 0000000000..77bafa0310 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py @@ -0,0 +1,417 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script for resizing managed instance group (MIG) cluster size based +# on the number of jobs in the Condor Queue. + +from absl import app +from absl import flags +from collections import OrderedDict +from datetime import datetime +from pprint import pprint +from googleapiclient import discovery +from oauth2client.client import GoogleCredentials + +import argparse +import os +import math +import time +import htcondor +import classad + +parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument("--p", required=True, help="Project id", type=str) +parser.add_argument( + "--z", + required=True, + help="Name of GCP zone where the managed instance group is located", + type=str, +) +parser.add_argument( + "--r", + required=True, + help="Name of GCP region where the managed instance group is located", + type=str, +) +parser.add_argument( + "--mz", + required=False, + help="Enabled multizone (regional) managed instance group", + action="store_true", +) +parser.add_argument( + "--g", required=True, help="Name of the managed instance group", type=str +) +parser.add_argument( + "--i", + default=0, + help="Minimum number of idle compute instances", + type=int +) +parser.add_argument( + "--c", required=True, help="Maximum number of compute instances", type=int +) +parser.add_argument( + "--v", + default=0, + help="Increase output verbosity. 1-show basic debug info. 2-show detail debug info", + type=int, + choices=[0, 1, 2], +) +parser.add_argument( + "--d", + default=0, + help="Dry Run, default=0, if 1, then no scaling actions", + type=int, + choices=[0, 1], +) + +args = parser.parse_args() + +class AutoScaler: + def __init__(self, multizone=False): + + self.multizone = multizone + # Obtain credentials + self.credentials = GoogleCredentials.get_application_default() + self.service = discovery.build("compute", "v1", credentials=self.credentials) + + if self.multizone: + self.instanceGroupManagers = self.service.regionInstanceGroupManagers() + else: + self.instanceGroupManagers = self.service.instanceGroupManagers() + + # Remove specified instances from MIG and decrease MIG size + def deleteFromMig(self, node_self_links): + requestDelInstance = self.instanceGroupManagers.deleteInstances( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + body={ "instances": node_self_links }, + ) + + # execute if not a dry-run + if not self.dryrun: + response = requestDelInstance.execute() + if self.debug > 0: + pprint(response) + return response + return "Dry Run" + + def getInstanceTemplateInfo(self): + requestTemplateName = self.instanceGroupManagers.get( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + fields="instanceTemplate", + ) + responseTemplateName = requestTemplateName.execute() + template_name = "" + + if self.debug > 1: + print("Request for the template name") + pprint(responseTemplateName) + + if len(responseTemplateName) > 0: + template_url = responseTemplateName.get("instanceTemplate") + template_url_partitioned = template_url.split("/") + template_name = template_url_partitioned[len(template_url_partitioned) - 1] + + requestInstanceTemplate = self.service.instanceTemplates().get( + project=self.project, instanceTemplate=template_name, fields="properties" + ) + responseInstanceTemplateInfo = requestInstanceTemplate.execute() + + if self.debug > 1: + print("Template information") + pprint(responseInstanceTemplateInfo["properties"]) + + machine_type = responseInstanceTemplateInfo["properties"]["machineType"] + is_spot = responseInstanceTemplateInfo["properties"]["scheduling"][ + "preemptible" + ] + if self.debug > 0: + print("Machine Type: " + machine_type) + print("Is spot: " + str(is_spot)) + request = self.service.machineTypes().get( + project=self.project, zone=self.zone, machineType=machine_type + ) + response = request.execute() + guest_cpus = response["guestCpus"] + if self.debug > 1: + print("Machine information") + pprint(responseInstanceTemplateInfo["properties"]) + if self.debug > 0: + print("Guest CPUs: " + str(guest_cpus)) + + instanceTemplateInfo = { + "machine_type": machine_type, + "is_spot": is_spot, + "guest_cpus": guest_cpus, + } + return instanceTemplateInfo + + def scale(self): + # diagnosis + if self.debug > 1: + print("Launching autoscaler.py with the following arguments:") + print("project_id: " + self.project) + print("zone: " + self.zone) + print("region: " + self.region) + print(f"multizone: {self.multizone}") + print("group_manager: " + self.instance_group_manager) + print("computeinstancelimit: " + str(self.compute_instance_limit)) + print("debuglevel: " + str(self.debug)) + + if self.multizone: + self.zoneargs = {"region": self.region} + else: + self.zoneargs = {"zone": self.zone} + + # Each HTCondor scheduler (SchedD), maintains a list of jobs under its + # stewardship. A full list of Job ClassAd attributes can be found at + # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html + schedd = htcondor.Schedd() + # encourage the job queue to start a new negotiation cycle; there are + # internal unconfigurable rate limits so not guaranteed; this is not + # strictly required for success, but may reduce latency of autoscaling + schedd.reschedule() + REQUEST_CPUS_ATTRIBUTE = "RequestCpus" + REQUEST_GPUS_ATTRIBUTE = "RequestGpus" + REQUEST_MEMORY_ATTRIBUTE = "RequestMemory" + job_attributes = [ + REQUEST_CPUS_ATTRIBUTE, + REQUEST_GPUS_ATTRIBUTE, + REQUEST_MEMORY_ATTRIBUTE, + ] + + instanceTemplateInfo = self.getInstanceTemplateInfo() + self.is_spot = instanceTemplateInfo["is_spot"] + self.cores_per_node = instanceTemplateInfo["guest_cpus"] + print(f"MIG is configured for Spot pricing: {self.is_spot}") + print("Number of CPU per compute node: " + str(self.cores_per_node)) + + # this query will constrain the search for jobs to those that either + # require spot VMs or do not require Spot VMs based on whether the + # VM instance template is configured for Spot pricing + spot_query = classad.ExprTree(f"RequireId == \"{self.instance_group_manager}\"") + + # For purpose of scaling a Managed Instance Group, count only jobs that + # are idle and likely participated in a negotiation cycle (there does + # not appear to be a single classad attribute for this). + # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html#JobStatus + LAST_CYCLE_ATTRIBUTE = "LastNegotiationCycleTime0" + coll = htcondor.Collector() + negotiator_ad = coll.query(htcondor.AdTypes.Negotiator, projection=[LAST_CYCLE_ATTRIBUTE]) + if len(negotiator_ad) != 1: + print(f"There should be exactly 1 negotiator in the pool. There is {len(negotiator_ad)}") + exit() + last_negotiation_cycle_time = negotiator_ad[0].get(LAST_CYCLE_ATTRIBUTE) + if not last_negotiation_cycle_time: + print(f"The negotiator has not yet started a match cycle. Exiting auto-scaling.") + exit() + + print(f"Last negotiation cycle occurred at: {datetime.fromtimestamp(last_negotiation_cycle_time)}") + idle_job_query = classad.ExprTree(f"JobStatus == 1 && QDate < {last_negotiation_cycle_time}") + idle_job_ads = schedd.query(constraint=idle_job_query.and_(spot_query), + projection=job_attributes) + + total_idle_request_cpus = sum(j[REQUEST_CPUS_ATTRIBUTE] for j in idle_job_ads) + print(f"Total CPUs requested by idle jobs: {total_idle_request_cpus}") + + if self.debug > 1: + print("Information about the compute instance template") + pprint(instanceTemplateInfo) + + # Calculate the minimum number of instances that, for fully packed + # execute points, could satisfy current job queue + min_hosts_for_idle_jobs = math.ceil(total_idle_request_cpus / self.cores_per_node) + if self.debug > 0: + print(f"Minimum hosts needed: {total_idle_request_cpus} / {self.cores_per_node} = {min_hosts_for_idle_jobs}") + + # Get current number of instances in the MIG + requestGroupInfo = self.instanceGroupManagers.get( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + ) + responseGroupInfo = requestGroupInfo.execute() + current_target = responseGroupInfo["targetSize"] + print(f"Current MIG target size: {current_target}") + + # Find instances that are being modified by the MIG (currentAction is + # any value other than "NONE"). A common reason an instance is modified + # is it because it has failed a health check. + reqModifyingInstances = self.instanceGroupManagers.listManagedInstances( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + filter="currentAction != \"NONE\"", + orderBy="creationTimestamp desc" + ) + respModifyingInstances = reqModifyingInstances.execute() + + # Find VMs that are idle (no dynamic slots created from partitionable + # slots) in the MIG handled by this autoscaler + filter_idle_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots==0") + filter_claimed_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots>0") + filter_mig = classad.ExprTree(f"regexp(\".*/{self.instance_group_manager}$\", CloudCreatedBy)") + # A full list of Machine (StartD) ClassAd attributes can be found at + # https://htcondor.readthedocs.io/en/latest/classad-attributes/machine-classad-attributes.html + idle_node_ads = coll.query(htcondor.AdTypes.Startd, + constraint=filter_idle_vms.and_(filter_mig), + projection=["Machine", "CloudZone"]) + + NODENAME_ATTRIBUTE = "Machine" + claimed_node_ads = coll.query(htcondor.AdTypes.Startd, + constraint=filter_claimed_vms.and_(filter_mig), + projection=[NODENAME_ATTRIBUTE]) + claimed_nodes = [ ad[NODENAME_ATTRIBUTE].split(".")[0] for ad in claimed_node_ads] + + # treat OrderedDict as a set by ignoring key values; this set will + # contain VMs we would consider deleting, in inverse order of + # their readiness to join pool (creating, unhealthy, healthy+idle) + idle_nodes = OrderedDict() + try: + modifyingInstances = respModifyingInstances["managedInstances"] + except KeyError: + modifyingInstances = [] + + print(f"There are {len(modifyingInstances)} VMs being modified by the managed instance group") + + # there is potential for nodes in MIG health check "VERIFYING" state + # to have already joined the pool and be running jobs + for instance in modifyingInstances: + self_link = instance["instance"] + node_name = self_link.rsplit("/", 1)[-1] + if node_name not in claimed_nodes: + idle_nodes[self_link] = "modifying" + + for ad in idle_node_ads: + node = ad["Machine"].split(".")[0] + zone = ad["CloudZone"] + self_link = "https://www.googleapis.com/compute/v1/projects/" + \ + self.project + "/zones/" + zone + "/instances/" + node + # there is potential for nodes in MIG health check "VERIFYING" state + # to have already joined the pool and be idle; delete them last + if self_link in idle_nodes: + idle_nodes.move_to_end(self_link) + idle_nodes[self_link] = "idle" + n_idle = len(idle_nodes) + + print(f"There are {n_idle} VMs being modified or idle in the pool") + if self.debug > 1: + print("Listing idle nodes:") + pprint(idle_nodes) + + # always keep size tending toward the minimum idle VMs requested + new_target = current_target + self.compute_instance_min_idle - n_idle + min_hosts_for_idle_jobs + if new_target > self.compute_instance_limit: + self.size = self.compute_instance_limit + print(f"MIG target size will be limited by {self.compute_instance_limit}") + else: + self.size = new_target + + print(f"New MIG target size: {self.size}") + + if self.debug > 1: + print("MIG Information:") + print(responseGroupInfo) + + if self.size == current_target: + if current_target == 0: + print("Queue is empty") + print("Running correct number of VMs to handle queue") + exit() + + if self.size < current_target: + print("Scaling down. Looking for nodes that can be shut down") + + if self.debug > 1: + print("Compute node busy status:") + for node in idle_nodes: + print(node) + + # Shut down idle nodes up to our calculated limit + nodes_to_delete = list(idle_nodes.keys())[0:current_target-self.size] + for node in nodes_to_delete: + print(f"Attempting to delete: {node.rsplit('/',1)[-1]}") + respDel = self.deleteFromMig(nodes_to_delete) + + if self.debug > 1: + print("Scaling down complete") + + if self.size > current_target: + print( + "Scaling up. Need to increase number of instances to " + str(self.size) + ) + # Request to resize + request = self.instanceGroupManagers.resize( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + size=self.size, + ) + response = request.execute() + if self.debug > 1: + print("Requesting to increase MIG size") + pprint(response) + print("Scaling up complete") + + +def main(): + + scaler = AutoScaler(args.mz) + + # Project ID + scaler.project = args.p # Ex:'slurm-var-demo' + + # Name of the zone where the managed instance group is located + scaler.zone = args.z # Ex: 'us-central1-f' + + # Name of the region where the managed instance group is located + scaler.region = args.r # Ex: 'us-central1' + + # The name of the managed instance group. + scaler.instance_group_manager = args.g # Ex: 'condor-compute-igm' + + # Default number of cores per instance, will be replaced with actual value + scaler.cores_per_node = 4 + + # Default number of running instances that the managed instance group should maintain at any given time. This number will go up and down based on the load (number of jobs in the queue) + scaler.size = 0 + + scaler.compute_instance_min_idle = args.i + + # Dry run: : 0, run scaling; 1, only provide info. + scaler.dryrun = args.d > 0 + + # Debug level: 1-print debug information, 2 - print detail debug information + scaler.debug = 0 + if args.v: + scaler.debug = args.v + + # Limit for the maximum number of compute instance. If zero (default setting), no limit will be enforced by the script + scaler.compute_instance_limit = 0 + if args.c: + scaler.compute_instance_limit = abs(args.c) + + scaler.scale() + + +if __name__ == "__main__": + main() diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml new file mode 100644 index 0000000000..db989f9d40 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml @@ -0,0 +1,46 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install but do not activate HTCondor autoscaler + become: true + hosts: localhost + tasks: + - name: Install Python 3 pip + ansible.builtin.package: + name: python3-pip + state: present + - name: Create virtual environment for HTCondor autoscaler + ansible.builtin.pip: + name: pip + version: 21.3.1 # last Python 3.6-compatible release + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv + - name: Install latest setuptools + ansible.builtin.pip: + name: setuptools + version: 59.6.0 # last Python 3.6-compatible release + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv + - name: Install HTCondor autoscaler dependencies + with_items: + - oauth2client + - google-api-python-client + - absl-py + - htcondor + ansible.builtin.pip: + name: "{{ item }}" + state: present # rely on pip resolver to pick latest compatible releases + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml new file mode 100644 index 0000000000..4d3abbbfd6 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml @@ -0,0 +1,94 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The instructions for installing HTCondor may change with time, although we +# anticipate that they will stay fixed for the 23.0 releases. Find up-to-date +# recommendations at: +## https://htcondor.readthedocs.io/en/latest/getting-htcondor/from-our-repositories.html + +--- +- name: Ensure HTCondor is installed + hosts: all + vars: + enable_docker: true + htcondor_key: https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-23.0-Key + docker_key: https://download.docker.com/linux/centos/gpg + become: true + module_defaults: + ansible.builtin.yum: + lock_timeout: 300 + tasks: + - name: Enable EPEL repository + ansible.builtin.yum: + name: + - epel-release + - name: Directly install RPM verification keys + ansible.builtin.rpm_key: + state: present + key: "{{ item }}" + loop: + - "{{ htcondor_key }}" + - "{{ docker_key }}" + register: key_install + retries: 10 + delay: 60 + until: key_install is success + - name: Enable HTCondor LTS Release repository + ansible.builtin.yum_repository: + name: htcondor-feature + description: HTCondor LTS Release (23.0) + file: htcondor + baseurl: https://research.cs.wisc.edu/htcondor/repo/23.0/el$releasever/$basearch/release + gpgkey: "{{ htcondor_key }}" + gpgcheck: true + repo_gpgcheck: true + priority: "90" + - name: Install HTCondor + ansible.builtin.yum: + name: condor-{{ condor_version | default("23.*") | string }} + state: present + - name: Ensure token directory + ansible.builtin.file: + path: /etc/condor/tokens.d + mode: 0700 + owner: root + group: root + - name: Install Docker and configure HTCondor to use it + when: enable_docker | bool # allows string to be passed at CLI + block: + - name: Setup Docker repo + ansible.builtin.yum_repository: + name: docker-ce-stable + description: Docker CE Stable - $basearch + baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable + enabled: yes + gpgcheck: yes + gpgkey: "{{ docker_key }}" + - name: Install Docker + ansible.builtin.yum: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin + - name: Enable Docker + ansible.builtin.service: + name: docker + state: started + enabled: true + - name: Add condor to docker group + ansible.builtin.user: + name: condor + groups: docker + append: yes diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf new file mode 100644 index 0000000000..0853e035f4 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + runners = [ + { + "type" = "ansible-local" + "source" = "${path.module}/files/install-htcondor.yaml" + "destination" = "install-htcondor.yaml" + "args" = join(" ", [ + "-e enable_docker=${var.enable_docker}", + "-e condor_version=${var.condor_version}", + ]) + }, + { + "type" = "ansible-local" + "content" = file("${path.module}/files/install-htcondor-autoscaler-deps.yml") + "destination" = "install-htcondor-autoscaler-deps.yml" + }, + { + "type" = "data" + "content" = file("${path.module}/files/autoscaler.py") + "destination" = "/usr/local/htcondor/bin/autoscaler.py" + }, + ] + + install_htcondor_ps1 = templatefile( + "${path.module}/templates/install-htcondor.ps1.tftpl", { + condor_version = var.condor_version, + http_proxy = var.http_proxy, + python_windows_installer_url = var.python_windows_installer_url, + }) + + required_apis = [ + "compute.googleapis.com", + "secretmanager.googleapis.com", + ] +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf new file mode 100644 index 0000000000..c7951737ff --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "runners" { + description = "Runner to install HTCondor using startup-scripts" + value = local.runners +} + +output "windows_startup_ps1" { + description = "Windows PowerShell script to install HTCondor" + value = local.install_htcondor_ps1 +} + +output "gcp_service_list" { + description = "Google Cloud APIs required by HTCondor" + value = local.required_apis +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl new file mode 100644 index 0000000000..7492da3c12 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl @@ -0,0 +1,59 @@ +#Requires -RunAsAdministrator + +# Windows 2016 needs forced upgrade to TLS 1.2 +[Net.ServicePointManager]::SecurityProtocol = 'Tls12' + +# important for catching exception in Invoke-WebRequest +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +%{ if http_proxy != "" ~} +[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") +%{ endif ~} + +# do not show progress bar when running Invoke-WebRequest +$ProgressPreference = 'SilentlyContinue' + +# download C Runtime DLL necessary for HTCondor installer +$runtime_installer = 'C:\vc_redist.x64.exe' +Invoke-WebRequest https://aka.ms/vs/17/release/vc_redist.x64.exe -OutFile "$runtime_installer" +Start-Process -FilePath "$runtime_installer" -Wait -ArgumentList "/norestart /quiet /log c:\vc_redist_log.txt" +Remove-Item "$runtime_installer" + +# download HTCondor installer +$htcondor_installer = 'C:\htcondor.msi' +%{ if condor_version == "23.*" } +Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/current/condor-Windows-x64.msi -OutFile "$htcondor_installer" +%{ else ~} +Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/${condor_version}/release/condor-${condor_version}-Windows-x64.msi -OutFile "$htcondor_installer" +%{ endif ~} +$args='/qn /l* condor-install-log.txt /i' +$args=$args + " $htcondor_installer" +$args=$args + ' NEWPOOL="N"' +$args=$args + ' RUNJOBS="N"' +$args=$args + ' SUBMITJOBS="N"' +$args=$args + ' INSTALLDIR="C:\Condor"' +Start-Process "msiexec.exe" -Wait -ArgumentList "$args" +Remove-Item "$htcondor_installer" + +# do not start HTCondor on boot by default. Allow startup script to download +# configuration first and then start HTCondor +Set-Service -StartupType Manual condor + +# remove settings from condor_config that we want to override in configuration step +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^CONDOR_HOST' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^INSTALL_USER' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^DAEMON_LIST' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^use SECURITY' -NotMatch) + +# install Python so that custom ClassAd hooks can execute +$python_installer = 'C:\python-installer.exe' +Invoke-WebRequest -Uri "${python_windows_installer_url}" -OutFile "$python_installer" +Start-Process -FilePath "$python_installer" -Wait -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1 Include_test=0' +%{ if http_proxy == "" ~} +Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --no-warn-script-location requests" +%{ else ~} +Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --proxy ${http_proxy} --no-warn-script-location requests" +%{ endif ~} +Invoke-WebRequest -Uri "https://raw.githubusercontent.com/htcondor/htcondor/main/src/condor_scripts/common-cloud-attributes-google.py" -OutFile "C:\Condor\bin\common-cloud-attributes-google.py" +Remove-Item "$python_installer" diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf new file mode 100644 index 0000000000..1afdf4e0eb --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "enable_docker" { + description = "Install and enable docker daemon alongside HTCondor" + type = bool + default = true +} + +variable "condor_version" { + description = "Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: \"23.0.0\",\"23.*\"))" + type = string + default = "23.*" + + validation { + error_message = "var.condor_version must be set to \"23.*\" for latest 23.0 release or to a specific \"23.0.y\" release." + condition = var.condor_version == "23.*" || ( + length(split(".", var.condor_version)) == 3 && alltrue([ + for v in split(".", var.condor_version) : can(tonumber(v)) + ]) && split(".", var.condor_version)[0] == "23" + && split(".", var.condor_version)[1] == "0" + ) + } +} + +variable "http_proxy" { + description = "Set system default web (http and https) proxy for Windows HTCondor installation" + type = string + default = "" + nullable = false +} + +variable "python_windows_installer_url" { + description = "URL of Python installer for Windows" + type = string + default = "https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe" + nullable = false +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md new file mode 100644 index 0000000000..55c2fc7e4e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md @@ -0,0 +1,116 @@ +## Description + +This module will create a startup-script runner that will execute Ramble commands. + +Ramble is a multi-platform experimentation framework capable of driving +software installation, acquiring input files, configuring experiments, and +extracting results. For more information about Ramble, see: +https://github.com/GoogleCloudPlatform/ramble + +This module outputs a startup script runner, which can be combined with other +startup script runners to execute a set of Ramble commands. + +Ramble makes extensive use of Spack. It must be installed with a Toolkit runner +generated by the [spack-setup module](../spack-setup/README.md) following the +[basic example](#basic-example) below. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +# Examples + +## Basic Example + +Below is a basic example of using this module. + +```yaml + - id: spack + source: community/modules/scripts/spack-setup + + - id: ramble-setup + source: community/modules/scripts/ramble-setup + + - id: ramble-execute + source: community/modules/scripts/ramble-execute + use: [spack, ramble-setup] + settings: + commands: + - ramble list +``` + +This example shows installing Spack and Ramble with their own modules +(spack-setup and ramble-setup respectively). Then the ramble-execute module +is added to simply list all applications Ramble knows about. + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | +| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [log\_file](#input\_log\_file) | Log file to write output from Ramble execute steps into | `string` | `"/var/log/ramble-execute.log"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by an instance of ramble-setup.
Can be defined explicitly, or by chaining an instance of a ramble-setup module
through a `use` setting. | `string` | n/a | yes | +| [ramble\_runner](#input\_ramble\_runner) | Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | +| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting.
Defaults to /etc/profile.d/spack.sh if not set. | `string` | `"/etc/profile.d/spack.sh"` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the ramble-setup module. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble startup script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for ramble, to be reused by ramble-execute module. | +| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | +| [ramble\_runner](#output\_ramble\_runner) | Runner to execute Ramble commands using an ansible playbook. The startup-script module
will automatically handle installation of ansible. | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to Spack profile script. | +| [startup\_script](#output\_startup\_script) | Ramble startup script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf new file mode 100644 index 0000000000..7ef0b029e3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ramble-execute", ghpc_role = "scripts" }) +} + +locals { + commands_content = var.commands == null ? "echo 'no ramble commands provided'" : indent(4, yamlencode(var.commands)) + + execute_contents = templatefile( + "${path.module}/templates/ramble_execute.yml.tpl", + { + pre_script = "if [ -f ${var.spack_profile_script_path} ]; then . ${var.spack_profile_script_path}; fi; . ${var.ramble_profile_script_path}" + log_file = var.log_file + commands = local.commands_content + system_user_name = var.system_user_name + } + ) + + data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] + + execute_md5 = substr(md5(local.execute_contents), 0, 4) + execute_runner = { + type = "ansible-local" + content = local.execute_contents + destination = "ramble_execute_${local.execute_md5}.yml" + } + + previous_runners = var.ramble_runner != null ? [var.ramble_runner] : [] + runners = concat(local.previous_runners, local.data_runners, [local.execute_runner]) + + # Destinations should be unique while also being known at time of apply + combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) + combined_md5 = substr(md5(local.combined_unique_string), 0, 4) + combined_runner = { + type = "shell" + content = module.startup_script.startup_script + destination = "combined_install_ramble_${local.combined_md5}.sh" + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = var.gcs_bucket_path +} + +resource "local_file" "debug_file_ansible_execute" { + content = local.execute_contents + filename = "${path.module}/debug_execute_${local.execute_md5}.yml" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf new file mode 100644 index 0000000000..4e6c3a44d8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Ramble startup script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Ramble startup script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "ramble_runner" { + description = <<-EOT + Runner to execute Ramble commands using an ansible playbook. The startup-script module + will automatically handle installation of ansible. + EOT + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for ramble, to be reused by ramble-execute module." + value = var.gcs_bucket_path +} + +output "spack_profile_script_path" { + description = "Path to Spack profile script." + value = var.spack_profile_script_path +} + +output "ramble_profile_script_path" { + description = "Path to Ramble profile script." + value = var.ramble_profile_script_path +} + +output "system_user_name" { + description = "The system user used to execute commands." + value = var.system_user_name +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl new file mode 100644 index 0000000000..0e98f3aa2c --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Execute Commands + hosts: localhost + vars: + pre_script: ${pre_script} + log_file: ${log_file} + commands: ${commands} + system_user_name: ${system_user_name} + tasks: + - name: Execute command block + block: + - name: Print commands to be executed + ansible.builtin.debug: + msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" + + - name: Streaming log info + ansible.builtin.debug: + msg: | + Logs from commands will not be printed here until success (or failure) + Streaming logs can be found at {{ log_file }} + + - name: Ensure user can write to log file + ansible.builtin.file: + path: "{{ log_file }}" + state: touch + owner: "{{ system_user_name }}" + + - name: Execute commands + ansible.builtin.shell: | + set -eo pipefail + { + {{ pre_script }} + echo " === Starting commands ===" + {{ commands }} + echo " === Finished commands ===" + } 2>&1 | tee -a {{ log_file }} + args: + executable: /bin/bash + register: output + become: true + become_user: "{{ system_user_name }}" + + always: + - name: Print commands output + ansible.builtin.debug: + var: output.stdout_lines diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf new file mode 100644 index 0000000000..ec67228df5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf @@ -0,0 +1,114 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing spack scripts." + type = string +} + +variable "region" { + description = "Region to place bucket containing spack scripts." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "log_file" { + description = "Log file to write output from Ramble execute steps into" + default = "/var/log/ramble-execute.log" + type = string +} + +variable "data_files" { + description = <<-EOT + A list of files to be transferred prior to running commands. + It must specify one of 'source' (absolute local file path) or 'content' (string). + It must specify a 'destination' with absolute path where file should be placed. + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) + error_message = "All destinations must be absolute paths and start with '/'." + } + validation { + condition = alltrue([ + for r in var.data_files : + can(r["content"]) != can(r["source"]) + ]) + error_message = "A data_file must specify either 'content' or 'source', but never both." + } + validation { + condition = alltrue([ + for r in var.data_files : + lookup(r, "content", lookup(r, "source", null)) != null + ]) + error_message = "A data_file must specify a non-null 'content' or 'source'." + } +} + +variable "commands" { + description = "String of commands to run within this module" + default = null + type = string +} + +variable "ramble_runner" { + description = "Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module." + type = object({ + type = string + content = string + destination = string + }) +} + +variable "system_user_name" { + description = "Name of the system user used to execute commands. Generally passed from the ramble-setup module." + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string +} + +variable "spack_profile_script_path" { + description = <<-EOT + Path to the Spack profile.d script. + Can be defined explicitly, or by chaining an instance of a spack-setup module + through a `use` setting. + Defaults to /etc/profile.d/spack.sh if not set. + EOT + type = string + default = "/etc/profile.d/spack.sh" +} + +variable "ramble_profile_script_path" { + description = <<-EOT + Path to the Ramble profile.d script. Created by an instance of ramble-setup. + Can be defined explicitly, or by chaining an instance of a ramble-setup module + through a `use` setting. + EOT + type = string +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf new file mode 100644 index 0000000000..9b23317323 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md new file mode 100644 index 0000000000..9891088105 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md @@ -0,0 +1,128 @@ +## Description + +This module will create a set of startup-script runners that will setup Ramble, +and install Ramble’s dependencies. + +Ramble is a multi-platform experimentation framework capable of driving +software installation, acquiring input files, configuring experiments, and +extracting results. For more information about ramble, see: +https://github.com/GoogleCloudPlatform/ramble + +This module outputs two startup script runners, which can be added to startup +scripts to setup, ramble and its dependencies. + +For this module to be completely functional, it depends on a spack +installation. For more information, see Cluster-Toolkit’s Spack module. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +# Examples + +## Basic Example + +```yaml +- id: ramble-setup + source: community/modules/scripts/ramble-setup +``` + +This example simply installs ramble on a VM. + +## Full Example + +```yaml +- id: ramble-setup + source: community/modules/scripts/ramble-setup + settings: + install_dir: /ramble + ramble_url: https://github.com/GoogleCloudPlatform/ramble + ramble_ref: v0.2.1 + log_file: /var/log/ramble.log + chown_owner: “owner” + chgrp_group: “user_group” + chmod_mode: “a+r” +``` + +This example simply installs ramble into a VM at the location `/ramble`, checks +out the v0.2.1 tag, changes the owner and group to “owner” and “user_group”, +and chmod’s the clone to make it world readable. + +Also see a more complete [Ramble example blueprint](../../../examples/ramble.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 4.42 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [chmod\_mode](#input\_chmod\_mode) | Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify).
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | +| [install\_dir](#input\_install\_dir) | Destination directory of installation of Ramble. | `string` | `"/apps/ramble"` | no | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by this module | `string` | `"/etc/profile.d/ramble.sh"` | no | +| [ramble\_ref](#input\_ramble\_ref) | Git ref to checkout for Ramble. | `string` | `"develop"` | no | +| [ramble\_url](#input\_ramble\_url) | URL for Ramble repository to clone. | `string` | `"https://github.com/GoogleCloudPlatform/ramble"` | no | +| [ramble\_virtualenv\_path](#input\_ramble\_virtualenv\_path) | Virtual environment path in which to install Ramble Python interpreter and other dependencies | `string` | `"/usr/local/ramble-python"` | no | +| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | +| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Ramble. It will be created if it does not exist. | `string` | `"ramble"` | no | +| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble installation script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module. | +| [ramble\_path](#output\_ramble\_path) | Location ramble is installed into. | +| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | +| [ramble\_ref](#output\_ramble\_ref) | Git ref the ramble install is checked out to use | +| [ramble\_runner](#output\_ramble\_runner) | Runner to be used with startup-script module or passed to ramble-execute module.
- installs Ramble dependencies
- installs Ramble
- generates profile.d script to enable access to Ramble
This is safe to run in parallel by multiple machines. | +| [startup\_script](#output\_startup\_script) | Ramble installation script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf new file mode 100644 index 0000000000..4389af7d33 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ramble-setup", ghpc_role = "scripts" }) +} + +locals { + profile_script = <<-EOF + if [ -f ${var.install_dir}/share/ramble/setup-env.sh ]; then + test -t 1 && echo "** Ramble's python virtualenv (/usr/local/ramble-python) is activated. Call 'deactivate' to deactivate." + VIRTUAL_ENV_DISABLE_PROMPT=1 . ${var.ramble_virtualenv_path}/bin/activate + . ${var.install_dir}/share/ramble/setup-env.sh + fi + EOF + + script_content = templatefile( + "${path.module}/templates/ramble_setup.yml.tftpl", + { + sw_name = "ramble" + profile_script = indent(4, yamlencode(local.profile_script)) + install_dir = var.install_dir + git_url = var.ramble_url + git_ref = var.ramble_ref + chmod_mode = var.chmod_mode + system_user_name = var.system_user_name + system_user_uid = var.system_user_uid + system_user_gid = var.system_user_gid + finalize_setup_script = "echo 'no finalize setup script'" + profile_script_path = var.ramble_profile_script_path + } + ) + + install_ramble_deps_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/install_ramble_deps.yml" + "destination" = "install_ramble_deps.yml" + "args" = "-e virtualenv_path=${var.ramble_virtualenv_path}" + } + + python_reqs_content = templatefile( + "${path.module}/templates/install_ramble_python_deps.yml.tftpl", + { + install_dir = var.install_dir + virtualenv_path = var.ramble_virtualenv_path + } + ) + + python_reqs_runner = { + "type" = "ansible-local" + "content" = local.python_reqs_content + "destination" = "install_ramble_reqs.yml" + } + + install_ramble_runner = { + "type" = "ansible-local" + "content" = local.script_content + "destination" = "install_ramble.yml" + } + + bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}"), 0, 8) + # Max bucket name length is 63, so truncate deployment_name if necessary. + # The string "-ramble-scripts-" is 16 characters and bucket_md5 is 8 characters, + # leaving 63-16-8=39 chars for deployment_name. + bucket_name = "${substr(var.deployment_name, 0, 39)}-ramble-scripts-${local.bucket_md5}" + runners = [local.install_ramble_deps_runner, local.install_ramble_runner, local.python_reqs_runner] + + combined_runner = { + "type" = "shell" + "content" = module.startup_script.startup_script + "destination" = "ramble-install-and-setup.sh" + } + +} + +resource "google_storage_bucket" "bucket" { + project = var.project_id + name = local.bucket_name + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" +} + +resource "local_file" "debug_file_shell_install" { + content = local.script_content + filename = "${path.module}/debug_install.yml" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf new file mode 100644 index 0000000000..e587470eac --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Ramble installation script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Ramble installation script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "ramble_runner" { + description = <<-EOT + Runner to be used with startup-script module or passed to ramble-execute module. + - installs Ramble dependencies + - installs Ramble + - generates profile.d script to enable access to Ramble + This is safe to run in parallel by multiple machines. + EOT + value = local.combined_runner +} + +output "ramble_path" { + description = "Location ramble is installed into." + value = var.install_dir +} + +output "ramble_ref" { + description = "Git ref the ramble install is checked out to use" + value = var.ramble_ref +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module." + value = "gs://${google_storage_bucket.bucket.name}" +} + +output "ramble_profile_script_path" { + description = "Path to Ramble profile script." + value = var.ramble_profile_script_path +} + +output "system_user_name" { + description = "The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands." + value = var.system_user_name +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml new file mode 100644 index 0000000000..b7905bbe9e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml @@ -0,0 +1,50 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Create python virtual env for a tool + become: yes + hosts: localhost + vars: + virtualenv_path: ${virtualenv_path} + tasks: + - name: Install dependencies through system package manager + ansible.builtin.package: + name: + - python3 + - python3-pip + - git + register: package + changed_when: package.changed + retries: 5 + delay: 10 + until: package is success + + - name: Create virtualenv for tool + # Python 3.6 is minimum we wish to support due to ease of installation on + # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip + # supported by 3.6. Additionally, recent versions of pip are necessary for + # proper dependency resolution of real-world problems with google-cloud-* + # (and third-party) Python packages (20.3+ probably effective minimum). + ansible.builtin.pip: + name: pip>=21.3.1 + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv + + - name: Add google-cloud-storage to virtualenv + ansible.builtin.pip: + name: google-cloud-storage + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl new file mode 100644 index 0000000000..ea14780a58 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl @@ -0,0 +1,28 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Install Python Requirements + hosts: localhost + vars: + install_dir: ${install_dir} + virtualenv_path: ${virtualenv_path} + tasks: + + - name: Install dependencies + ansible.builtin.pip: + requirements: "{{ install_dir }}/requirements.txt" + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl new file mode 100644 index 0000000000..ca48a5afa0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl @@ -0,0 +1,157 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install Software + hosts: localhost + vars: + sw_name: ${sw_name} + profile_script: ${profile_script} + install_dir: ${install_dir} + git_url: ${git_url} + git_ref: ${git_ref} + chmod_mode: ${chmod_mode} + system_user_name: ${system_user_name} + system_user_uid: ${system_user_uid} + system_user_gid: ${system_user_gid} + finalize_setup_script: ${finalize_setup_script} + profile_script_path: ${profile_script_path} + tasks: + - name: Print software name + ansible.builtin.debug: + msg: "Running installation for software: {{ sw_name }}" + + - name: Add profile script for software + ansible.builtin.copy: + dest: "{{ profile_script_path }}" + mode: '0644' + content: "{{ profile_script }}" + when: profile_script + + - name: Look up user to use for install + block: + + - name: Check if user already exists + ansible.builtin.getent: + database: passwd + key: "{{ system_user_name }}" + + - name: Look up existing user details + ansible.builtin.user: + name: "{{ system_user_name }}" + register: system_user + + rescue: + - name: User did not exist, create group for system user + ansible.builtin.group: + name: "{{ system_user_name }}" + gid: "{{ system_user_gid }}" + system: true + register: system_group + + - name: Create system user + ansible.builtin.user: + name: "{{ system_user_name }}" + comment: "{{ sw_name }} installation" + uid: "{{ system_user_uid }}" + group: "{{ system_group.name }}" + system: true + register: system_user + + - name: Create parent of install directory + ansible.builtin.file: + path: "{{ install_dir | dirname }}" + state: directory + + - name: Set lock dir + ansible.builtin.set_fact: + lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" + + - name: Acquire lock + ansible.builtin.command: + mkdir "{{ lock_dir }}" + register: lock_out + changed_when: lock_out.rc == 0 + failed_when: false + + - name: Add hostname to lock_dir + ansible.builtin.file: + path: "{{ lock_dir }}/{{ ansible_hostname }}" + state: touch + when: lock_out.rc == 0 + + - name: Clone branch or tag into installation directory + ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} + failed_when: false + register: clone_res + when: lock_out.rc == 0 + + - name: Clone commit hash into installation directory + ansible.builtin.command: "{{ item }}" + with_items: + - git clone {{ git_url }} {{ install_dir }} + - git -C {{ install_dir }} checkout {{ git_ref }} + when: lock_out.rc == 0 and clone_res.rc != 0 + + - name: Transfer ownership to system user + ansible.builtin.file: + path: "{{ install_dir }}" + owner: "{{ system_user.name }}" + group: "{{ system_user.group }}" + recurse: true + follow: false + when: lock_out.rc == 0 + + - name: Finalize setup + ansible.builtin.shell: "{{ finalize_setup_script }}" + when: lock_out.rc == 0 and finalize_setup_script + become: true + become_user: "{{ system_user.name }}" + + - name: Apply chmod + ansible.builtin.file: + path: "{{ install_dir }}" + mode: "{{ chmod_mode | default(omit, true) }}" + recurse: true + follow: false + when: (lock_out.rc == 0) and (chmod_mode != None) + + - name: Release lock + ansible.builtin.file: + path: "{{ lock_dir }}/done" + state: touch + when: lock_out.rc == 0 + + - name: Wait for lock + block: + - name: Wait for lock + ansible.builtin.wait_for: + path: "{{ lock_dir }}/done" + state: present + timeout: 600 + sleep: 10 + when: lock_out.rc != 0 + + rescue: + - name: Timed out on waiting for lock, get lock directory contents + ansible.builtin.find: + paths: "{{ lock_dir }}" + register: lock_dir_contents + + - name: Print lock directory contents, it should contain name of host that is holding lock + ansible.builtin.debug: + msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" + + - name: Failed to get lock + ansible.builtin.fail: + msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf new file mode 100644 index 0000000000..0d3a8eed05 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf @@ -0,0 +1,97 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "install_dir" { + description = "Destination directory of installation of Ramble." + default = "/apps/ramble" + type = string +} + +variable "ramble_url" { + description = "URL for Ramble repository to clone." + default = "https://github.com/GoogleCloudPlatform/ramble" + type = string +} + +variable "ramble_ref" { + description = "Git ref to checkout for Ramble." + default = "develop" + type = string +} + +variable "chmod_mode" { + description = <<-EOT + Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify). + For usage information see: + https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode + EOT + default = "" + type = string + nullable = false +} + +variable "system_user_name" { + description = "Name of system user that will perform installation of Ramble. It will be created if it does not exist." + default = "ramble" + type = string + nullable = false +} + +variable "system_user_uid" { + description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." + default = 1104762904 + type = number + nullable = false +} + +variable "system_user_gid" { + description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." + default = 1104762904 + type = number + nullable = false +} + +variable "ramble_virtualenv_path" { + description = "Virtual environment path in which to install Ramble Python interpreter and other dependencies" + default = "/usr/local/ramble-python" + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing startup script." + type = string +} + +variable "region" { + description = "Region to place bucket containing startup script." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "ramble_profile_script_path" { + description = "Path to the Ramble profile.d script. Created by this module" + type = string + default = "/etc/profile.d/ramble.sh" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf new file mode 100644 index 0000000000..936b4a5b80 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/README.md new file mode 100644 index 0000000000..8cbb75fb42 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/README.md @@ -0,0 +1,141 @@ +## Description + +This module creates a script that defines a software build using Spack and +performs any additional customization to a Spack installation. + +There are two main variable inputs that can be used to define a Spack build: +`data_files` and `commands`. + +- `data_files`: Any files specified will be transferred to the machine running + outputted script. Data file `content` can be defined inline in the blueprint + or can point to a `source`, an absolute local path of a file. This can be used + to transfer environment definition files, config definition files, GPG keys, + or software licenses. `data_files` are transferred before `commands` are run. +- `commands`: A script that is run. This can be used to perform actions such as + installation of compilers & packages, environment creation, adding a build + cache, and modifying the spack configuration. + +## Example + +The `spack-execute` module should `use` a `spack-setup` module. This will +prepend the installation of Spack and its dependencies to the build. Then +`spack-execute` can be used by a module that takes `startup-script` as an input. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: builder-vm + source: modules/compute/vm-instance + use: [network1, spack-build] +``` + +To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. + +[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml + +### Using with `startup-script` module + +The `spack-runner` output can be used by the `startup-script` module. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(spack-build.spack-runner) + - type: shell + destination: "my-script.sh" + content: echo 'hello world' + + - id: workstation + source: modules/compute/vm-instance + use: [network1, startup-script] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | +| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [log\_file](#input\_log\_file) | Defines the logfile that script output will be written to | `string` | `"/var/log/spack.log"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by an instance of spack-setup.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting. | `string` | n/a | yes | +| [spack\_runner](#input\_spack\_runner) | Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | +| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the spack-setup module. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack startup script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | +| [spack\_runner](#output\_spack\_runner) | Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules. | +| [startup\_script](#output\_startup\_script) | Spack startup script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf new file mode 100644 index 0000000000..04ebcf7d49 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "spack-execute", ghpc_role = "scripts" }) +} + +locals { + commands_content = var.commands == null ? "echo 'no spack commands provided'" : indent(4, yamlencode(var.commands)) + + execute_contents = templatefile( + "${path.module}/templates/execute_commands.yml.tpl", + { + pre_script = ". ${var.spack_profile_script_path}" + log_file = var.log_file + commands = local.commands_content + system_user_name = var.system_user_name + } + ) + + data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] + + execute_md5 = substr(md5(local.execute_contents), 0, 4) + execute_runner = { + type = "ansible-local" + content = local.execute_contents + destination = "spack_execute_${local.execute_md5}.yml" + } + + runners = concat([var.spack_runner], local.data_runners, [local.execute_runner]) + + # Destinations should be unique while also being known at time of apply + combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) + combined_md5 = substr(md5(local.combined_unique_string), 0, 4) + combined_runner = { + type = "shell" + content = module.startup_script.startup_script + destination = "combined_install_spack_${local.combined_md5}.sh" + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = var.gcs_bucket_path +} + +resource "local_file" "debug_file_ansible_execute" { + content = local.execute_contents + filename = "${path.module}/debug_execute_${local.execute_md5}.yml" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf new file mode 100644 index 0000000000..4a52532d51 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Spack startup script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Spack startup script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "spack_runner" { + description = "Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules." + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." + value = var.gcs_bucket_path +} + +output "spack_profile_script_path" { + description = "Path to the Spack profile.d script." + value = var.spack_profile_script_path +} + +output "system_user_name" { + description = "The system user used to execute commands." + value = var.system_user_name +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl new file mode 100644 index 0000000000..0e98f3aa2c --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Execute Commands + hosts: localhost + vars: + pre_script: ${pre_script} + log_file: ${log_file} + commands: ${commands} + system_user_name: ${system_user_name} + tasks: + - name: Execute command block + block: + - name: Print commands to be executed + ansible.builtin.debug: + msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" + + - name: Streaming log info + ansible.builtin.debug: + msg: | + Logs from commands will not be printed here until success (or failure) + Streaming logs can be found at {{ log_file }} + + - name: Ensure user can write to log file + ansible.builtin.file: + path: "{{ log_file }}" + state: touch + owner: "{{ system_user_name }}" + + - name: Execute commands + ansible.builtin.shell: | + set -eo pipefail + { + {{ pre_script }} + echo " === Starting commands ===" + {{ commands }} + echo " === Finished commands ===" + } 2>&1 | tee -a {{ log_file }} + args: + executable: /bin/bash + register: output + become: true + become_user: "{{ system_user_name }}" + + always: + - name: Print commands output + ansible.builtin.debug: + var: output.stdout_lines diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf new file mode 100644 index 0000000000..851cd1aed8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf @@ -0,0 +1,103 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing spack scripts." + type = string +} + +variable "region" { + description = "Region to place bucket containing spack scripts." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "log_file" { + description = "Defines the logfile that script output will be written to" + default = "/var/log/spack.log" + type = string +} + +variable "data_files" { + description = <<-EOT + A list of files to be transferred prior to running commands. + It must specify one of 'source' (absolute local file path) or 'content' (string). + It must specify a 'destination' with absolute path where file should be placed. + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) + error_message = "All destinations must be absolute paths and start with '/'." + } + validation { + condition = alltrue([ + for r in var.data_files : + can(r["content"]) != can(r["source"]) + ]) + error_message = "A data_file must specify either 'content' or 'source', but never both." + } + validation { + condition = alltrue([ + for r in var.data_files : + lookup(r, "content", lookup(r, "source", null)) != null + ]) + error_message = "A data_file must specify a non-null 'content' or 'source'." + } +} + +variable "commands" { + description = "String of commands to run within this module" + type = string + default = null +} + +variable "spack_runner" { + description = "Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module." + type = object({ + type = string + content = string + destination = string + }) +} + +variable "system_user_name" { + description = "Name of the system user used to execute commands. Generally passed from the spack-setup module." + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string +} + +variable "spack_profile_script_path" { + description = <<-EOT + Path to the Spack profile.d script. Created by an instance of spack-setup. + Can be defined explicitly, or by chaining an instance of a spack-setup module + through a `use` setting. + EOT + type = string +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf new file mode 100644 index 0000000000..09583c3d43 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.0.0" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/README.md new file mode 100644 index 0000000000..01d3e6d389 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/README.md @@ -0,0 +1,382 @@ +## Description + +This module can be used to setup and install Spack on a VM. To actually run +Spack commands to install other software use the +[spack-execute](../spack-execute/) module. + +This module generates a script that performs the following: + +1. Install system dependencies needed for Spack +1. Clone Spack into a predefined directory +1. Check out a specific version of Spack + +There are several options on how to consume the outputs of this module: + +> [!IMPORTANT] +> Breaking changes between after v1.21.0. `spack-install` module replaced by +> `spack-setup` and `spack-execute` modules. +> [Details Below](#deprecations-and-breaking-changes) + +## Examples + +### `use` `spack-setup` with `spack-execute` + +This will prepend the `spack-setup` script to the `spack-execute` commands. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: builder + source: modules/compute/vm-instance + use: [network1, spack-build] +``` + +### `use` `spack-setup` with `vm-instance` or Slurm module + +This will run `spack-setup` scripts on the downstream compute resource. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-installer + source: modules/compute/vm-instance + use: [network1, spack-setup] +``` + +OR + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: [network1, partition1, spack-setup] +``` + +### Build `starup-script` with `spack-runner` output + +This will use the generated `spack-setup` script as one step in `startup-script`. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(spack-setup.spack-runner) + - type: shell + destination: "my-script.sh" + content: echo 'hello world' + + - id: workstation + source: modules/compute/vm-instance + use: [network1, startup-script] +``` + +To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. + +[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml + +## Environment Setup + +### Activating Spack + +[Spack installation] produces a setup script that adds `spack` to your `PATH` as +well as some other command-line integration tools. This script can be found at +`/share/spack/setup-env.sh`. This script will be automatically +added to bash startup by any machine that runs the `spack_runner`. + +If you have multiple machines that all want to use the same shared Spack +installation you can just have both machines run the `spack_runner`. + +[Spack installation]: https://spack-tutorial.readthedocs.io/en/latest/tutorial_basics.html#installing-spack + +### Managing Spack Python dependencies + +Spack is configured with [SPACK_PYTHON] to ensure that Spack itself uses a +Python virtual environment with a supported copy of Python with the package +`google-cloud-storage` pre-installed. This enables Spack to use mirrors and +[build caches][builds] on Google Cloud Storage. It does not configure Python +packages *inside* Spack virtual environments. If you need to add more Python +dependencies for Spack itself, use the `spack python` command: + +```shell +sudo -i spack python -m pip install package-name +``` + +[SPACK_PYTHON]: https://spack.readthedocs.io/en/latest/getting_started.html#shell-support +[builds]: https://spack.readthedocs.io/en/latest/binary_caches.html + +## Spack Permissions + +### System `spack` user is created - Default + +By default this module will create a `spack` linux user and group with +consistent UID and GID. This user and group will own the Spack installation. To +allow a user to manually add Spack packages to the system Spack installation, +you can add the user to the spack group: + +```sh +sudo usermod -a -G spack +``` + +Log out and back in so the group change will take effect, then `` will +be able to call `spack install `. + +> [!NOTE] +> A background persistent SSH connections may prevent the group change from +> taking effect. + +You can use the `system_user_name`, `system_user_uid`, and `system_user_gid` to +customize the name and ids of the system user. While unlikely, it is possible +that the default `system_user_uid` or `system_user_gid` could conflict with +existing UIDs. + +### Use and existing user + +Alternatively, if `system_user_name` is a user already on the system, then this +existing user will be used for Spack installation. + +#### OS Login User + +If OS Login is enabled (default for most Cluster Toolkit modules) then you can +provide an OS Login user name: + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + system_user_name: username_company_com +``` + +This will work even if the user has not yet logged onto the machine. When the +specified user does log on to the machine they will be able to call +`spack install` without any further configuration. + +#### Pre-configured user + +You can also use a startup script to configure a user: + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + system_user_name: special-user + + - id: startup + source: modules/scripts/startup-script + settings: + runners: + - type: shell + destination: "create_user.sh" + content: | + #!/bin/bash + sudo useradd -u 799 special-user + sudo groupadd -g 922 org-group + sudo usermod -g org-group special-user + - $(spack-setup.spack_runner) + + - id: spack-vms + source: modules/compute/vm-instance + use: [network1, startup] + settings: + name_prefix: spack-vm + machine_type: n2d-standard-2 + instance_count: 5 +``` + +### Chaining spack installations + +If there is a need to have a non-root user to install spack packages it is +recommended to create a separate installation for that user and chain Spack installations +([Spack docs](https://spack.readthedocs.io/en/latest/chain.html#chaining-spack-installations)). + +Steps to chain Spack installations: + +1. Get the version of the system Spack: + + ```sh + $ spack --version + + 0.20.0 (e493ab31c6f81a9e415a4b0e0e2263374c61e758) + # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + # Note commit hash and use in next step + ``` + +1. Clone a new spack installation: + + ```sh + git clone -c feature.manyFiles=true https://github.com/spack/spack.git /spack + git -C /spack checkout + ``` + +1. Point the new Spack installation to the system Spack installation. Create a + file at `/spack/etc/spack/upstreams.yaml` with the following + contents: + + ```yaml + upstreams: + spack-instance-1: + install_tree: /sw/spack/opt/spack/ + ``` + +1. Add the following line to your `.bashrc` to make sure the new `spack` is in + your `PATH`. + + ```sh + . /spack/share/spack/setup-env.sh + ``` + +## Deprecations and Breaking Changes + +The old `spack-install` module has been replaced by the `spack-setup` and +`spack-execute` modules. Generally this change strives to allow for a more +flexible definition of a Spack build by using native Spack commands. + +For every deprecated variable from `spack-install` there is documentation on how +to perform the equivalent action using `commands` and `data_files`. The +documentation can be found on the [inputs table](#inputs) below. + +Below is a simple example of the same functionality shown before and after the +breaking changes. + +```yaml + # Before + - id: spack-install + source: community/modules/scripts/spack-install + settings: + install_dir: /sw/spack + compilers: + - gcc@10.3.0 target=x86_64 + packages: + - intel-mpi@2018.4.274%gcc@10.3.0 + +- id: spack-startup + source: modules/scripts/startup-script + settings: + runners: + - $(spack.install_spack_deps_runner) + - $(spack.install_spack_runner) +``` + +```yaml + # After + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + install_dir: /sw/spack + + - id: spack-execute + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + spack load gcc@10.3.0 target=x86_64 + spack compiler find --scope site + spack install intel-mpi@2018.4.274%gcc@10.3.0 + +- id: spack-startup + source: modules/scripts/startup-script + settings: + runners: + - $(spack-execute.spack-runner) +``` + +Although the old `spack-install` module will no longer be maintained, it is +still possible to use the old module in a blueprint by referencing an old +version from GitHub. Note the source line in the following example. + +```yaml + - id: spack-install + source: github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/spack-install?ref=v1.22.1&depth=1 +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 4.42 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [chmod\_mode](#input\_chmod\_mode) | `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification.
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `"g+w"` | no | +| [configure\_for\_google](#input\_configure\_for\_google) | When true, the spack installation will be configured to pull from Google's Spack binary cache. | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | +| [install\_dir](#input\_install\_dir) | Directory to install spack into. | `string` | `"/sw/spack"` | no | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by this module | `string` | `"/etc/profile.d/spack.sh"` | no | +| [spack\_ref](#input\_spack\_ref) | Git ref to checkout for spack. | `string` | `"v0.20.0"` | no | +| [spack\_url](#input\_spack\_url) | URL to clone the spack repo from. | `string` | `"https://github.com/spack/spack"` | no | +| [spack\_virtualenv\_path](#input\_spack\_virtualenv\_path) | Virtual environment path in which to install Spack Python interpreter and other dependencies | `string` | `"/usr/local/spack-python"` | no | +| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Spack. It will be created if it does not exist. | `string` | `"spack"` | no | +| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack installation script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | +| [spack\_path](#output\_spack\_path) | Path to the root of the spack installation | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | +| [spack\_runner](#output\_spack\_runner) | Runner to be used with startup-script module or passed to spack-execute module.
- installs Spack dependencies
- installs Spack
- generates profile.d script to enable access to Spack
This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. | +| [startup\_script](#output\_startup\_script) | Spack installation script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Spack. It can be reused by spack-execute module to install spack packages. | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf new file mode 100644 index 0000000000..d45f5d1be3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "spack-setup", ghpc_role = "scripts" }) +} + +locals { + profile_script = <<-EOF + SPACK_PYTHON=${var.spack_virtualenv_path}/bin/python3 + if [ -f ${var.install_dir}/share/spack/setup-env.sh ]; then + test -t 1 && echo "Running Spack setup, this may take a moment on first login." + . ${var.install_dir}/share/spack/setup-env.sh + fi + EOF + + supported_cache_versions = ["v0.19.0", "v0.20.0"] + cache_version = contains(local.supported_cache_versions, var.spack_ref) ? var.spack_ref : "latest" + add_google_mirror_script = !var.configure_for_google ? "" : <<-EOF + if ! spack mirror list | grep -q google_binary_cache; then + spack mirror add --scope site google_binary_cache gs://spack/${local.cache_version} + spack buildcache keys --install --trust + fi + EOF + + finalize_setup_script = <<-EOF + set -e + . ${var.spack_profile_script_path} + spack config --scope site add 'packages:all:permissions:read:world' + spack config --scope site add 'packages:all:permissions:write:group' + spack gpg init + spack compiler find --scope site + ${local.add_google_mirror_script} + # perform fast install to make sure Spack is fully initialized + spack install xz + spack uninstall --yes-to-all xz + EOF + + script_content = templatefile( + "${path.module}/templates/spack_setup.yml.tftpl", + { + sw_name = "spack" + profile_script = indent(4, yamlencode(local.profile_script)) + install_dir = var.install_dir + git_url = var.spack_url + git_ref = var.spack_ref + chmod_mode = var.chmod_mode + system_user_name = var.system_user_name + system_user_uid = var.system_user_uid + system_user_gid = var.system_user_gid + finalize_setup_script = indent(4, yamlencode(local.finalize_setup_script)) + profile_script_path = var.spack_profile_script_path + } + ) + + install_spack_deps_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/install_spack_deps.yml" + "destination" = "install_spack_deps.yml" + "args" = "-e virtualenv_path=${var.spack_virtualenv_path}" + } + install_spack_runner = { + "type" = "ansible-local" + "content" = local.script_content + "destination" = "install_spack.yml" + } + + bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}.${local.script_content}"), 0, 8) + # Max bucket name length is 63, so truncate deployment_name if necessary. + # The string "-spack-scripts-" is 15 characters and bucket_md5 is 8 characters, + # leaving 63-15-8=40 chars for deployment_name. Using 39 so it has the same prefix as the + # ramble-setup module's GCS bucket. + bucket_name = "${substr(var.deployment_name, 0, 39)}-spack-scripts-${local.bucket_md5}" + runners = [local.install_spack_deps_runner, local.install_spack_runner] + + combined_runner = { + "type" = "shell" + "content" = module.startup_script.startup_script + "destination" = "spack-install-and-setup.sh" + } +} + +resource "google_storage_bucket" "bucket" { + project = var.project_id + name = local.bucket_name + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" +} + +resource "local_file" "debug_file_shell_install" { + content = local.script_content + filename = "${path.module}/debug_install.yml" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml new file mode 100644 index 0000000000..2ada34471f --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf new file mode 100644 index 0000000000..d94b9757db --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Spack installation script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Spack installation script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "spack_path" { + description = "Path to the root of the spack installation" + value = var.install_dir +} + +output "spack_runner" { + description = <<-EOT + Runner to be used with startup-script module or passed to spack-execute module. + - installs Spack dependencies + - installs Spack + - generates profile.d script to enable access to Spack + This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. + EOT + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." + value = "gs://${google_storage_bucket.bucket.name}" +} + +output "spack_profile_script_path" { + description = "Path to the Spack profile.d script." + value = var.spack_profile_script_path +} + +output "system_user_name" { + description = "The system user used to install Spack. It can be reused by spack-execute module to install spack packages." + value = var.system_user_name +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml new file mode 100644 index 0000000000..b7905bbe9e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml @@ -0,0 +1,50 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Create python virtual env for a tool + become: yes + hosts: localhost + vars: + virtualenv_path: ${virtualenv_path} + tasks: + - name: Install dependencies through system package manager + ansible.builtin.package: + name: + - python3 + - python3-pip + - git + register: package + changed_when: package.changed + retries: 5 + delay: 10 + until: package is success + + - name: Create virtualenv for tool + # Python 3.6 is minimum we wish to support due to ease of installation on + # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip + # supported by 3.6. Additionally, recent versions of pip are necessary for + # proper dependency resolution of real-world problems with google-cloud-* + # (and third-party) Python packages (20.3+ probably effective minimum). + ansible.builtin.pip: + name: pip>=21.3.1 + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv + + - name: Add google-cloud-storage to virtualenv + ansible.builtin.pip: + name: google-cloud-storage + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl new file mode 100644 index 0000000000..ca48a5afa0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl @@ -0,0 +1,157 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install Software + hosts: localhost + vars: + sw_name: ${sw_name} + profile_script: ${profile_script} + install_dir: ${install_dir} + git_url: ${git_url} + git_ref: ${git_ref} + chmod_mode: ${chmod_mode} + system_user_name: ${system_user_name} + system_user_uid: ${system_user_uid} + system_user_gid: ${system_user_gid} + finalize_setup_script: ${finalize_setup_script} + profile_script_path: ${profile_script_path} + tasks: + - name: Print software name + ansible.builtin.debug: + msg: "Running installation for software: {{ sw_name }}" + + - name: Add profile script for software + ansible.builtin.copy: + dest: "{{ profile_script_path }}" + mode: '0644' + content: "{{ profile_script }}" + when: profile_script + + - name: Look up user to use for install + block: + + - name: Check if user already exists + ansible.builtin.getent: + database: passwd + key: "{{ system_user_name }}" + + - name: Look up existing user details + ansible.builtin.user: + name: "{{ system_user_name }}" + register: system_user + + rescue: + - name: User did not exist, create group for system user + ansible.builtin.group: + name: "{{ system_user_name }}" + gid: "{{ system_user_gid }}" + system: true + register: system_group + + - name: Create system user + ansible.builtin.user: + name: "{{ system_user_name }}" + comment: "{{ sw_name }} installation" + uid: "{{ system_user_uid }}" + group: "{{ system_group.name }}" + system: true + register: system_user + + - name: Create parent of install directory + ansible.builtin.file: + path: "{{ install_dir | dirname }}" + state: directory + + - name: Set lock dir + ansible.builtin.set_fact: + lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" + + - name: Acquire lock + ansible.builtin.command: + mkdir "{{ lock_dir }}" + register: lock_out + changed_when: lock_out.rc == 0 + failed_when: false + + - name: Add hostname to lock_dir + ansible.builtin.file: + path: "{{ lock_dir }}/{{ ansible_hostname }}" + state: touch + when: lock_out.rc == 0 + + - name: Clone branch or tag into installation directory + ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} + failed_when: false + register: clone_res + when: lock_out.rc == 0 + + - name: Clone commit hash into installation directory + ansible.builtin.command: "{{ item }}" + with_items: + - git clone {{ git_url }} {{ install_dir }} + - git -C {{ install_dir }} checkout {{ git_ref }} + when: lock_out.rc == 0 and clone_res.rc != 0 + + - name: Transfer ownership to system user + ansible.builtin.file: + path: "{{ install_dir }}" + owner: "{{ system_user.name }}" + group: "{{ system_user.group }}" + recurse: true + follow: false + when: lock_out.rc == 0 + + - name: Finalize setup + ansible.builtin.shell: "{{ finalize_setup_script }}" + when: lock_out.rc == 0 and finalize_setup_script + become: true + become_user: "{{ system_user.name }}" + + - name: Apply chmod + ansible.builtin.file: + path: "{{ install_dir }}" + mode: "{{ chmod_mode | default(omit, true) }}" + recurse: true + follow: false + when: (lock_out.rc == 0) and (chmod_mode != None) + + - name: Release lock + ansible.builtin.file: + path: "{{ lock_dir }}/done" + state: touch + when: lock_out.rc == 0 + + - name: Wait for lock + block: + - name: Wait for lock + ansible.builtin.wait_for: + path: "{{ lock_dir }}/done" + state: present + timeout: 600 + sleep: 10 + when: lock_out.rc != 0 + + rescue: + - name: Timed out on waiting for lock, get lock directory contents + ansible.builtin.find: + paths: "{{ lock_dir }}" + register: lock_dir_contents + + - name: Print lock directory contents, it should contain name of host that is holding lock + ansible.builtin.debug: + msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" + + - name: Failed to get lock + ansible.builtin.fail: + msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf new file mode 100644 index 0000000000..85baeec401 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf @@ -0,0 +1,106 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +# spack-setup variables + +variable "install_dir" { + description = "Directory to install spack into." + type = string + default = "/sw/spack" +} + +variable "spack_url" { + description = "URL to clone the spack repo from." + type = string + default = "https://github.com/spack/spack" +} + +variable "spack_ref" { + description = "Git ref to checkout for spack." + type = string + default = "v0.20.0" +} + +variable "configure_for_google" { + description = "When true, the spack installation will be configured to pull from Google's Spack binary cache." + type = bool + default = true +} + + +variable "chmod_mode" { + description = <<-EOT + `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification. + For usage information see: + https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode + EOT + default = "g+w" + type = string + nullable = false +} + +variable "system_user_name" { + description = "Name of system user that will perform installation of Spack. It will be created if it does not exist." + default = "spack" + type = string + nullable = false +} + +variable "system_user_uid" { + description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." + default = 1104762903 + type = number + nullable = false +} + +variable "system_user_gid" { + description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." + default = 1104762903 + type = number + nullable = false +} + +variable "spack_virtualenv_path" { + description = "Virtual environment path in which to install Spack Python interpreter and other dependencies" + default = "/usr/local/spack-python" + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing startup script." + type = string +} + +variable "region" { + description = "Region to place bucket containing startup script." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "spack_profile_script_path" { + description = "Path to the Spack profile.d script. Created by this module" + type = string + default = "/etc/profile.d/spack.sh" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf new file mode 100644 index 0000000000..ff1180fc1b --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md new file mode 100644 index 0000000000..ee9c057c39 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md @@ -0,0 +1,87 @@ +## Description + +This module will insert a dependency on the completion of the startup script +for one or more specified compute VMs and report back if it fails. This can be useful when running +post-boot installation scripts that require the startup script to finish setting up a node. + +> **_WARNING:_**: this module is experimental and not fully supported. + +### Additional Dependencies + +* [**gcloud**](https://cloud.google.com/sdk/gcloud) must be present in the path + of the machine where `terraform apply` is run. + +### Example + +```yaml +- id: workstation + source: modules/compute/vm-instance + use: + - network1 + - my-startup-script + settings: + instance_count: 4 + +# Wait for all instances of the above VM to finish running startup scripts. +- id: wait + source: community/modules/scripts/wait-for-startup + settings: + instance_names: $(workstation.name) +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.validate_instance_names](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.wait_for_startup](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | +| [instance\_name](#input\_instance\_name) | Name of the instance we are waiting for (can be null if 'instance\_names' is not empty) | `string` | `null` | no | +| [instance\_names](#input\_instance\_names) | A list of instance names we are waiting for, in addition to the one mentioned in 'instance\_name' (if any) | `list(string)` | `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [timeout](#input\_timeout) | Timeout in seconds | `number` | `1200` | no | +| [zone](#input\_zone) | The GCP zone where the instance is running | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf new file mode 100644 index 0000000000..3f6b416251 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + combined_instance_names = concat(var.instance_names, [var.instance_name]) +} + +resource "null_resource" "validate_instance_names" { + lifecycle { + precondition { + condition = var.instance_name != null || length(var.instance_names) > 0 + error_message = "At least one instance name must be provided" + } + } +} + +resource "null_resource" "wait_for_startup" { + count = length(local.combined_instance_names) + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/wait-for-startup-status.sh" + environment = { + INSTANCE_NAME = self.triggers.instance_name + ZONE = var.zone + PROJECT_ID = var.project_id + TIMEOUT = var.timeout + GCLOUD_PATH = var.gcloud_path_override + } + } + + triggers = { + instance_name = local.combined_instance_names[count.index] + } +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf new file mode 100644 index 0000000000..11a2ddf118 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf @@ -0,0 +1,15 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh new file mode 100644 index 0000000000..fae5833121 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z "${INSTANCE_NAME}" ]]; then + echo "INSTANCE_NAME is unset... exiting" + exit 0 +fi +if [[ -z "${ZONE}" ]]; then + echo "ZONE is unset" + exit 1 +fi +if [[ -z "${PROJECT_ID}" ]]; then + echo "PROJECT_ID is unset" + exit 1 +fi +if [[ -z "${TIMEOUT}" ]]; then + echo "TIMEOUT is unset" + exit 1 +fi + +if [[ -n "${GCLOUD_PATH}" ]]; then + export PATH="$GCLOUD_PATH:$PATH" +fi + +echo "Waiting for startup: instance_name='${INSTANCE_NAME}', zone='${ZONE}', project_id='${PROJECT_ID}', timeout_seconds='${TIMEOUT}'" + +# Wrapper around grep that swallows the error status code 1 +c1grep() { grep "$@" || test $? = 1; } + +now=$(date +%s) + +# If VM was created more than 30 days ago, serial port logs may no longer exist. +# Exit without errors if the instance is older than 30 days. +logsExpiryDays=30 +createdTimestampIso=$(gcloud compute instances describe "${INSTANCE_NAME}" --project "${PROJECT_ID}" --zone "${ZONE}" --format "value(creationTimestamp)") +earliestAllowedCreatedTimestamp=$(date -d "${createdTimestampIso} +${logsExpiryDays} day" +%s) +if [[ "$earliestAllowedCreatedTimestamp" -lt "$now" ]]; then + echo "Instance was created more than 30 days ago - serial port 1 logs are likely expired... exiting" + exit 0 +fi + +deadline=$((now + TIMEOUT)) +error_file=$(mktemp) +fetch_cmd="gcloud compute instances get-serial-port-output ${INSTANCE_NAME} --port 1 --zone ${ZONE} --project ${PROJECT_ID}" +# Match string for all finish types of the old guest agent and successful +# finishes on the new guest agent +FINISH_LINE="startup-script exit status" +# Match string for failures on the new guest agent +FINISH_LINE_ERR="Script \"startup-script\" failed with error:" + +# NEW: Accept also these finish lines as success. +STARTUP_SCRIPT_SUCCEEDED_LINE="google-startup-scripts.service: Succeeded." +STARTUP_SCRIPT_FINISHED_LINE="Finished Google Compute Engine Startup Scripts." +STARTUP_SCRIPT_SERVICE_FINISHED_LINE="Finished google-startup-scripts.service - Google Compute Engine Startup Scripts." + +NON_FATAL_ERRORS=( + "Internal error" +) + +until [[ now -gt deadline ]]; do + ser_log=$( + set -o pipefail + ${fetch_cmd} 2>"${error_file}" | + c1grep "${FINISH_LINE}\|${FINISH_LINE_ERR}\|${STARTUP_SCRIPT_SUCCEEDED_LINE}\|${STARTUP_SCRIPT_FINISHED_LINE}\|${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}" + ) || { + err=$(cat "${error_file}") + echo "$err" + fatal_error="true" + for e in "${NON_FATAL_ERRORS[@]}"; do + if [[ $err = *"$e"* ]]; then + fatal_error="false" + break + fi + done + + if [[ $fatal_error = "true" ]]; then + exit 1 + fi + } + if [[ -n "${ser_log}" ]]; then break; fi + sleep 5 + now=$(date +%s) +done + +# This line checks for an exit code - the assumption is that there is a number +# at the end of the line and it is an exit code. +# Modified to correctly extract the last numeric exit status from the relevant log line. +LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=Script \"startup-script\" failed with error: exit status )[0-9]+" | tail -n 1) +if [[ -z "${LAST_EXIT_STATUS}" ]]; then + LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=startup-script exit status )[0-9]+" | tail -n 1) +fi + +# This specific text is monitored for in tests, do not change. +INSPECT_OUTPUT_TEXT="To inspect the startup script output, please run:" + +# --- Prioritize explicit failure from the script itself --- +if [[ "${LAST_EXIT_STATUS}" == 1 ]]; then + echo "startup-script finished with errors, ${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" + exit 1 +# --- Then explicit success from the script itself --- +elif [[ "${LAST_EXIT_STATUS}" == 0 ]]; then + echo "startup-script finished successfully" + exit 0 +elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SUCCEEDED_LINE}"; then + echo "startup-script finished successfully (startup script succeeded line detected)" + exit 0 +elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_FINISHED_LINE}"; then + echo "startup-script finished successfully (startup script finished line detected)" + exit 0 +elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}"; then + echo "startup-script finished successfully (startup script service finished line detected)" + exit 0 +# --- If we reached deadline, it's a timeout --- +elif [[ now -ge deadline ]]; then + echo "startup-script timed out after ${TIMEOUT} seconds" + echo "${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" + exit 1 +# --- All other cases are considered failure or invalid state --- +else + echo "Invalid or undetermined startup script status. Last detected exit status: '${LAST_EXIT_STATUS}'" + echo "${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" + exit "${LAST_EXIT_STATUS}" +fi diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf new file mode 100644 index 0000000000..fe6410a920 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "instance_name" { + description = "Name of the instance we are waiting for (can be null if 'instance_names' is not empty)" + type = string + default = null +} + +variable "instance_names" { + description = "A list of instance names we are waiting for, in addition to the one mentioned in 'instance_name' (if any)" + type = list(string) + default = [] +} + +variable "zone" { + description = "The GCP zone where the instance is running" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "timeout" { + description = "Timeout in seconds" + type = number + default = 1200 + validation { + condition = var.timeout >= 0 + error_message = "The timeout should be non-negative" + } +} + +variable "gcloud_path_override" { + description = "Directory of the gcloud executable to be used during cleanup" + type = string + default = "" + nullable = false +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf new file mode 100644 index 0000000000..8cd43b944e --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md new file mode 100644 index 0000000000..fc25bc0a55 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md @@ -0,0 +1,109 @@ +## Description + +This module contains a set of scripts to be used in customizing Windows VMs at +boot or during image building. Please note that the installation of NVIDIA GPU +drivers takes, at minimum, 30-60 minutes. It is therefore recommended to build +a custom image and reuse it as shown below, rather than install GPU drivers at +boot time. + +> NOTE: the output `windows_startup_ps1` must be passed explicitly as shown +> below when used with Packer modules. This is due to a limitation in the `use` +> keyword and inputs of type `list` in Packer modules; this does not impact +> Terraform modules + +### NVIDIA Drivers and CUDA Toolkit + +Many Google Cloud VM families include or can have NVIDIA GPUs attached to them. +This module supports GPU applications by enabling you to easily install +a compatible release of NVIDIA drivers and of the CUDA Toolkit. The script is +the [solution recommended by our documentation][docs] and is [directly sourced +from GitHub][script-src]. + +[docs]: https://cloud.google.com/compute/docs/gpus/install-drivers-gpu#windows +[script-src]: https://github.com/GoogleCloudPlatform/compute-gpu-installation/blob/24dac3004360e0696c49560f2da2cd60fcb80107/windows/install_gpu_driver.ps1 + +```yaml +- group: primary + modules: + - id: network1 + source: modules/network/vpc + settings: + enable_iap_rdp_ingress: true + enable_iap_winrm_ingress: true + + - id: windows_startup + source: community/modules/scripts/windows-startup-script + settings: + install_nvidia_driver: true + +- group: packer + modules: + - id: image + source: modules/packer/custom-image + kind: packer + use: + - network1 + - windows_startup + settings: + source_image_family: windows-2016 + machine_type: n1-standard-8 + accelerator_count: 1 + accelerator_type: nvidia-tesla-t4 + disk_size: 75 + disk_type: pd-ssd + omit_external_ip: false + state_timeout: 15m +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [http\_proxy](#input\_http\_proxy) | Set http and https proxy for use by Invoke-WebRequest commands | `string` | `""` | no | +| [http\_proxy\_set\_environment](#input\_http\_proxy\_set\_environment) | Set system default environment variables http\_proxy and https\_proxy for all commands | `bool` | `false` | no | +| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install\_nvidia\_driver\_script | `bool` | `false` | no | +| [install\_nvidia\_driver\_args](#input\_install\_nvidia\_driver\_args) | Arguments to supply to NVIDIA driver install script | `string` | `"/s /n"` | no | +| [install\_nvidia\_driver\_script](#input\_install\_nvidia\_driver\_script) | Install script for NVIDIA drivers specified by http/https URL | `string` | `"https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe"` | no | +| [no\_proxy](#input\_no\_proxy) | Environment variables no\_proxy (only used if var.http\_proxy\_set\_environment is enabled) | `string` | `"169.254.169.254,metadata,metadata.google.internal,.googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | A string list of scripts selected by this module | + diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf new file mode 100644 index 0000000000..5e6bc8b94d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + setx_http_proxy_ps1 = !var.http_proxy_set_environment ? [] : [ + templatefile("${path.module}/templates/setx_http_proxy.ps1", { + "http_proxy" : var.http_proxy, + "no_proxy" : var.no_proxy, + }) + ] + + nvidia_ps1 = !var.install_nvidia_driver ? [] : [ + templatefile("${path.module}/templates/install_gpu_driver.ps1.tftpl", { + "url" : var.install_nvidia_driver_script + "args" : var.install_nvidia_driver_args + "http_proxy" : var.http_proxy, + }) + ] + + startup_ps1 = concat(local.setx_http_proxy_ps1, local.nvidia_ps1) +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf new file mode 100644 index 0000000000..006ea312ad --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "windows_startup_ps1" { + description = "A string list of scripts selected by this module" + value = local.startup_ps1 +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl new file mode 100644 index 0000000000..55c4a2a3cd --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl @@ -0,0 +1,38 @@ +#Requires -RunAsAdministrator + +# Windows 2016 needs forced upgrade to TLS 1.2 +[Net.ServicePointManager]::SecurityProtocol = 'Tls12' + +# important for catching exception in Invoke-WebRequest +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +%{ if http_proxy != "" } +[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") +%{ endif } + +# Create the folder for the driver download +$file_dir = 'C:\NVIDIA-Driver\nvidia_installer_windows.exe' +if (!(Test-Path -Path 'C:\NVIDIA-Driver')) { + New-Item -Path 'C:\' -Name 'NVIDIA-Driver' -ItemType 'directory' | Out-Null +} + +# Download the file to a specified directory +Write-Output "Downloading ${url} to $file_dir" +# Disabling progress bar has surprising large (10-100x) impact on speed +$ProgressPreference = 'SilentlyContinue' +try { + Invoke-WebRequest -Uri "${url}" -OutFile "$file_dir" +} catch { + Write-Output "$_" + throw "Failed to download ${url}; exiting startup script" +} + +# Install the file with the specified path from earlier as well as the RunAs admin option +Write-Output "Executing $file_dir with arguments '${args}'" +try { + Start-Process -FilePath "$file_dir" -ArgumentList '${args}' -Wait +} catch { + Write-Output "$_" + throw "Could not install NVIDIA driver; exiting startup script" +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 new file mode 100644 index 0000000000..ca4d13f98b --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 @@ -0,0 +1,21 @@ +<# + Copyright 2025 "Google LLC" + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#> + +#Requires -RunAsAdministrator + +setx http_proxy ${http_proxy} /m +setx https_proxy ${http_proxy} /m +setx no_proxy ${no_proxy} /m diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf new file mode 100644 index 0000000000..9e4fb9e67d --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "install_nvidia_driver" { + description = "Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install_nvidia_driver_script" + type = bool + default = false +} + +variable "install_nvidia_driver_script" { + description = "Install script for NVIDIA drivers specified by http/https URL" + type = string + default = "https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe" +} + +variable "install_nvidia_driver_args" { + description = "Arguments to supply to NVIDIA driver install script" + type = string + default = "/s /n" +} + +variable "http_proxy" { + description = "Set http and https proxy for use by Invoke-WebRequest commands" + type = string + default = "" + nullable = false +} + +variable "http_proxy_set_environment" { + description = "Set system default environment variables http_proxy and https_proxy for all commands" + type = bool + default = false + nullable = false +} + +variable "no_proxy" { + description = "Environment variables no_proxy (only used if var.http_proxy_set_environment is enabled)" + type = string + default = "169.254.169.254,metadata,metadata.google.internal,.googleapis.com" + nullable = false +} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf new file mode 100644 index 0000000000..dfeeac34f8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/primary/modules/embedded/modules/README.md b/deletion-test/primary/modules/embedded/modules/README.md new file mode 100644 index 0000000000..6886b3f330 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/README.md @@ -0,0 +1,554 @@ +# Modules + +This directory contains a set of core modules built for the Cluster Toolkit. Modules +describe the building blocks of an AI/ML and HPC deployment. The expected fields in a +module are listed in more detail [below](#module-fields). Blueprints can be +extended in functionality by incorporating [modules from GitHub +repositories][ghmods]. + +[ghmods]: #github-modules + +## All Modules + +Modules from various sources are all listed here for visibility. Badges are used +to indicate the source and status of many of these resources. + +Modules listed below with the ![core-badge] badge are located in this +folder and are tested and maintained by the Cluster Toolkit team. + +Modules labeled with the ![community-badge] badge are contributed by +the community (including the Cluster Toolkit team, partners, etc.). Community modules +are located in the [community folder](../community/modules/README.md). + +Modules labeled with the ![deprecated-badge] badge are now deprecated and may be +removed in the future. Customers are advised to transition to alternatives. + +Modules that are still in development and less stable are labeled with the +![experimental-badge] badge. + +[core-badge]: https://img.shields.io/badge/-core-blue?style=plastic +[community-badge]: https://img.shields.io/badge/-community-%23b8def4?style=plastic +[stable-badge]: https://img.shields.io/badge/-stable-lightgrey?style=plastic +[experimental-badge]: https://img.shields.io/badge/-experimental-%23febfa2?style=plastic +[deprecated-badge]: https://img.shields.io/badge/-deprecated-%23fea2a2?style=plastic + +### Compute + +* **[vm-instance]** ![core-badge] : Creates one or more VM instances. +* **[schedmd-slurm-gcp-v6-partition]** ![core-badge] : + Creates a partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. +* **[schedmd-slurm-gcp-v6-nodeset]** ![core-badge] : + Creates a nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. +* **[schedmd-slurm-gcp-v6-nodeset-tpu]** ![core-badge] : + Creates a TPU nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. +* **[schedmd-slurm-gcp-v6-nodeset-dynamic]** ![core-badge] ![experimental-badge]: + Creates a dynamic nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module and instance template. +* **[gke-node-pool]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes node pool using GKE. +* **[resource-policy]** ![core-badge] ![experimental-badge] : Create a resource policy for compute engines that can be applied to gke-node-pool's nodes. +* **[gke-job-template]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes job file to be used with a [gke-node-pool]. +* **[htcondor-execute-point]** ![community-badge] ![experimental-badge] : + Manages a group of execute points for use in an [HTCondor + pool][htcondor-setup]. +* **[mig]** ![community-badge] ![experimental-badge] : Creates a Managed Instance Group. +* **[notebook]** ![community-badge] ![experimental-badge] : Creates a Vertex AI + Notebook. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[gke-nodeset]** ![community-badge] ![experimental-badge] : Create a slinky nodeset to be used by the [gke-partition] module. +* **[gke-partition]** ![community-badge] ![experimental-badge] : Creates a slinky partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. + +[vm-instance]: compute/vm-instance/README.md +[gke-node-pool]: ../modules/compute/gke-node-pool/README.md +[resource-policy]: ../modules/compute/resource-policy/README.md +[gke-job-template]: ../modules/compute/gke-job-template/README.md +[schedmd-slurm-gcp-v6-partition]: ../community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md +[schedmd-slurm-gcp-v6-nodeset]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md +[schedmd-slurm-gcp-v6-nodeset-tpu]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md +[schedmd-slurm-gcp-v6-nodeset-dynamic]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md +[htcondor-execute-point]: ../community/modules/compute/htcondor-execute-point/README.md +[mig]: ../community/modules/compute/mig/README.md +[notebook]: ../community/modules/compute/notebook/README.md +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +### Database + +* **[slurm-cloudsql-federation]** ![community-badge] ![experimental-badge] : + Creates a [Google SQL Instance](https://cloud.google.com/sql/) meant to be + integrated with a [slurm-controller][schedmd-slurm-gcp-v6-controller]. +* **[bigquery-dataset]** ![community-badge] ![experimental-badge] : Creates a BQ + dataset. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[bigquery-table]** ![community-badge] ![experimental-badge] : Creates a BQ + table. Primarily used for + [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. + +[slurm-cloudsql-federation]: ../community/modules/database/slurm-cloudsql-federation/README.md +[bigquery-dataset]: ../community/modules/database/bigquery-dataset/README.md +[bigquery-table]: ../community/modules/database/bigquery-table/README.md +[fsi-montecarlo-on-batch]: ../community/modules/files/fsi-montecarlo-on-batch/README.md + +### File System + +* **[filestore]** ![core-badge] : Creates a + [filestore](https://cloud.google.com/filestore) file system. +* **[parallelstore]** ![core-badge] ![experimental-badge]: Creates a + [parallelstore](https://cloud.google.com/parallelstore) file system. +* **[pre-existing-network-storage]** ![core-badge] : Specifies a + pre-existing file system that can be mounted on a VM. +* **[managed-lustre]** ![core-badge] ![experimental-badge]: Creates a + [managed-lustred](https://cloud.google.com/managed-lustre) file system. +* **[DDN-EXAScaler]** ![community-badge] ![deprecated-badge] : Creates + a [DDN EXAscaler lustre](https://www.ddn.com/partners/google-cloud-platform/) + file system. This module is deprecated and will be removed by July 1, 2025. Consider migrating to managed-lustre. +* **[cloud-storage-bucket]** ![core-badge] : Creates a Google Cloud Storage (GCS) bucket. +* **[gke-persistent-volume]** ![core-badge] ![experimental-badge] : Creates + persistent volumes and persistent volume claims for shared storage. +* **[nfs-server]** ![community-badge] ![experimental-badge] : Creates a VM and + configures an NFS server that can be mounted by other VM. +* **[weka-client]** ![community-badge] ![experimental-badge] : Installs client + and mounts [WEKA](https://www.weka.io/) filesystems. + +[filestore]: file-system/filestore/README.md +[parallelstore]: file-system/parallelstore/README.md +[pre-existing-network-storage]: file-system/pre-existing-network-storage/README.md +[managed-lustre]: file-system/managed-lustre/README.md +[ddn-exascaler]: ../community/modules/file-system/DDN-EXAScaler/README.md +[nfs-server]: ../community/modules/file-system/nfs-server/README.md +[cloud-storage-bucket]: file-system/cloud-storage-bucket/README.md +[gke-persistent-volume]: file-system/gke-persistent-volume/README.md +[weka-client]: ../community/modules/file-system/weka-client/README.md + +### Monitoring + +* **[dashboard]** ![core-badge] : Creates a + [monitoring dashboard](https://cloud.google.com/monitoring/dashboards) for + visually tracking a Cluster Toolkit deployment. + +[dashboard]: monitoring/dashboard/README.md + +### Network + +* **[vpc]** ![core-badge] : Creates a + [Virtual Private Cloud (VPC)](https://cloud.google.com/vpc) network with + regional subnetworks and firewall rules. +* **[multivpc]** ![core-badge] ![experimental-badge]: Creates a variable + number of VPC networks using the [vpc] module. +* **[pre-existing-vpc]** ![core-badge] : Used to connect newly + built components to a pre-existing VPC network. +* **[firewall-rules]** ![core-badge] ![experimental-badge] : Add custom firewall + rules to existing networks (commonly used with [pre-existing-vpc]). +* **[private-service-access]** ![community-badge] ![experimental-badge] : + Configures Private Services Access for a VPC network (commonly used with [filestore] and [slurm-cloudsql-federation]). + +[vpc]: network/vpc/README.md +[multivpc]: network/multivpc/README.md +[pre-existing-vpc]: network/pre-existing-vpc/README.md +[firewall-rules]: network/firewall-rules/README.md +[private-service-access]: ../community/modules/network/private-service-access/README.md + +### Packer + +* **[custom-image]** ![core-badge] : Creates a custom VM Image + based on the GCP HPC VM image. + +[custom-image]: packer/custom-image/README.md + +### Project + +* **[service-account]** ![community-badge] ![experimental-badge] : Creates [service + accounts](https://cloud.google.com/iam/docs/service-accounts) for a GCP + project. +* **[service-enablement]** ![community-badge] ![experimental-badge] : Allows enabling + various APIs for a Google Cloud Project. + +[service-account]: ../community/modules/project/service-account/README.md +[service-enablement]: ../community/modules/project/service-enablement/README.md + +### Pub/Sub + +* **[topic]** ![community-badge] ![experimental-badge] : Creates a +Pub/Sub topic. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[bigquery-sub]** ![community-badge] ![experimental-badge] : Creates a +Pub/Sub subscription. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. + +[topic]: ../community/modules/pubsub/topic/README.md +[bigquery-sub]: ../community/modules/pubsub/bigquery-sub/README.md + +### Remote Desktop + +* **[chrome-remote-desktop]** ![community-badge] ![experimental-badge] : Creates + a GPU accelerated Chrome Remote Desktop. + +[chrome-remote-desktop]: ../community/modules/remote-desktop/chrome-remote-desktop/README.md + +### Scheduler + +* **[batch-job-template]** ![core-badge] : Creates a Google Cloud Batch job + template that works with other Toolkit modules. +* **[batch-login-node]** ![core-badge] : Creates a VM that can be used for + submission of Google Cloud Batch jobs. +* **[gke-cluster]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes cluster using GKE. +* **[pre-existing-gke-cluster]** ![core-badge] ![experimental-badge] : Retrieves an existing GKE cluster. Substitute for ([gke-cluster]) module. +* **[schedmd-slurm-gcp-v6-controller]** ![core-badge] : + Creates a Slurm controller node. +* **[schedmd-slurm-gcp-v6-login]** ![core-badge] : + Creates a Slurm login node. +* **[htcondor-setup]** ![community-badge] ![experimental-badge] : Creates the + base infrastructure for an HTCondor pool (service accounts and Cloud Storage bucket). +* **[htcondor-pool-secrets]** ![community-badge] ![experimental-badge] : Creates + and manages access to the secrets necessary for secure operation of an + HTCondor pool. +* **[htcondor-access-point]** ![community-badge] ![experimental-badge] : Creates + a regional instance group managing a highly available HTCondor access point + (login node). + +[batch-job-template]: ../modules/scheduler/batch-job-template/README.md +[batch-login-node]: ../modules/scheduler/batch-login-node/README.md +[gke-cluster]: ../modules/scheduler/gke-cluster/README.md +[pre-existing-gke-cluster]: ../modules/scheduler/pre-existing-gke-cluster/README.md +[htcondor-setup]: ../community/modules/scheduler/htcondor-setup/README.md +[htcondor-pool-secrets]: ../community/modules/scheduler/htcondor-pool-secrets/README.md +[htcondor-access-point]: ../community/modules/scheduler/htcondor-access-point/README.md +[schedmd-slurm-gcp-v6-controller]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md +[schedmd-slurm-gcp-v6-login]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md + +### Scripts + +* **[startup-script]** ![core-badge] : Creates a customizable startup script + that can be fed into compute VMs. +* **[windows-startup-script]** ![community-badge] ![experimental-badge]: Creates + Windows PowerShell (PS1) scripts that can be used to customize Windows VMs + and VM images. +* **[htcondor-install]** ![community-badge] ![experimental-badge] : Creates + a startup script to install HTCondor and exports a list of required APIs +* **[ramble-execute]** ![community-badge] ![experimental-badge] : Creates a + startup script to execute + [Ramble](https://github.com/GoogleCloudPlatform/ramble) commands on a target + VM +* **[ramble-setup]** ![community-badge] ![experimental-badge] : Creates a + startup script to install + [Ramble](https://github.com/GoogleCloudPlatform/ramble) on an instance or a + slurm login or controller. +* **[spack-setup]** ![community-badge] ![experimental-badge] : Creates a startup + script to install [Spack](https://github.com/spack/spack) on an instance or a + slurm login or controller. +* **[spack-execute]** ![community-badge] ![experimental-badge] : Defines a + software build using [Spack](https://github.com/spack/spack). +* **[wait-for-startup]** ![community-badge] ![experimental-badge] : Waits for + successful completion of a startup script on a compute VM. + +[startup-script]: scripts/startup-script/README.md +[windows-startup-script]: ../community/modules/scripts/windows-startup-script/README.md +[htcondor-install]: ../community/modules/scripts/htcondor-install/README.md +[kubernetes-operations]: ../community/modules/scripts/kubernetes-operations/README.md +[ramble-execute]: ../community/modules/scripts/ramble-execute/README.md +[ramble-setup]: ../community/modules/scripts/ramble-setup/README.md +[spack-setup]: ../community/modules/scripts/spack-setup/README.md +[spack-execute]: ../community/modules/scripts/spack-execute/README.md +[wait-for-startup]: ../community/modules/scripts/wait-for-startup/README.md + +## Module Fields + +### ID (Required) + +The `id` field is used to uniquely identify and reference a defined module. +ID's are used in [variables](../examples/README.md#variables) and become the +name of each module when writing the terraform `main.tf` file. They are also +used in the [use](#use-optional) and [outputs](#outputs-optional) lists +described below. + +For terraform modules, the ID will be rendered into the terraform module label +at the top level main.tf file. + +### Source (Required) + +The source is a path or URL that points to the source files for Packer or +Terraform modules. A source can either be a filesystem path or a URL to a git +repository: + +* Filesystem paths + * modules embedded in the `gcluster` executable + * modules in the local filesystem +* Remote modules using [Terraform URL syntax](https://developer.hashicorp.com/terraform/language/modules/sources) + * Hosted on [GitHub](https://developer.hashicorp.com/terraform/language/modules/sources#github) + * Google Cloud Storage [Buckets](https://developer.hashicorp.com/terraform/language/modules/sources#gcs-bucket) + * Generic [git repositories](https://developer.hashicorp.com/terraform/language/modules/sources#generic-git-repository) + + when modules are in a subdirectory of the git repository, a special + double-slash `//` notation can be required as described below + +An important distinction is that those URLs are natively supported by Terraform so +they are not copied to your deployment directory. Packer does not have native +support for git-hosted modules so the Toolkit will copy these modules into the +deployment folder on your behalf. + +#### Embedded Modules + +Embedded modules are added to the gcluster binary during compilation and cannot +be edited. To refer to embedded modules, set the source path to +`modules/<>` or `community/modules/<>`. + +The paths match the modules in the repository structure for [core modules](./) +and [community modules](../community/modules/). Because the modules are embedded +during compilation, your local copies may differ unless you recompile gcluster. + +For example, this example snippet uses the embedded pre-existing-vpc module: + +```yaml + - id: network1 + source: modules/network/pre-existing-vpc +``` + +#### Local Modules + +Local modules point to a module in the file system and can easily be edited. +They are very useful during module development. To use a local module, set +the source to a path starting with `/`, `./`, or `../`. For instance, the +following module definition refers the local pre-existing-vpc modules. + +```yaml + - id: network1 + source: modules/network/pre-existing-vpc +``` + +> **_NOTE:_** Relative paths (beginning with `.` or `..` must be relative to the +> working directory from which `gcluster` is executed. This example would have to be +> run from a local copy of the Cluster Toolkit repository. An alternative is to use +> absolute paths to modules. + +#### GitHub-hosted Modules and Packages + +To use a Terraform module available on GitHub, set the source to a path starting +with `github.com` (HTTPS) or `git@github.com` (SSH). For instance, the following +module definition sources the Toolkit vpc module: + +```yaml + - id: network1 + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc +``` + +This example uses the [double-slash notation][tfsubdir] (`//`) to indicate that +the Toolkit is a "package" of multiple modules whose root directory is the root +of the git repository. The remainder of the path indicates the sub-directory of +the vpc module. + +The example above uses the default `main` branch of the Toolkit. Specific +[revisions][tfrev] can be selected with any valid [git reference][gitref]. +(git branch, commit hash or tag). If the git reference is a tag or branch, we +recommend setting `&depth=1` to reduce the data transferred over the network. +This option cannot be set when the reference is a commit hash. The following +examples select the vpc module on the active `develop` branch and also an older +release of the filestore module: + +```yaml + - id: network1 + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc?ref=develop + ... + - id: homefs + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/file-system/filestore?ref=v1.22.1&depth=1 +``` + +Because Terraform modules natively support this syntax, gcluster will not copy +GitHub-hosted modules into your deployment folder. Terraform will download them +into a hidden folder when you run `terraform init`. + +[tfrev]: https://www.terraform.io/language/modules/sources#selecting-a-revision +[gitref]: https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection#_single_revisions +[tfsubdir]: https://www.terraform.io/language/modules/sources#modules-in-package-sub-directories + +##### GitHub-hosted Packer modules + +Packer does not natively support GitHub-hosted modules so `gcluster create` will +copy modules into your deployment folder. + +If the module uses `//` package notation, `gcluster create` will copy the entire +repository to the module path: `deployment_name/group_name/module_id`. However, +when `gcluster deploy` is invoked, it will run Packer from the subdirectory +`deployment_name/group_name/module_id/subdirectory/after/double_slash`. + +If the module does not use `//` package notation, `gcluster create` will copy +only the final directory in the path to `deployment_name/group_name/module_id`. + +In all cases, `gcluster create` will remove the `.git` directory from the packer +module to ensure that you can manage the entire deployment directory with its +own git versioning. + +##### GitHub over SSH + +Get module from GitHub over SSH: + +```yaml + - id: network1 + source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc +``` + +Specific versions can be selected as for HTTPS: + +```yaml + - id: network1 + source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc?ref=v1.22.1&depth=1 +``` + +##### Generic Git Modules + +To use a Terraform module available in a non-GitHub git repository such as +gitlab, set the source to a path starting `git::`. Two Standard git protocols +are supported, `git::https://` for HTTPS or `git::git@github.com` for SSH. + +Additional formatting and features after `git::` are identical to that of the +[GitHub Modules](#github-modules) described above. + +#### Google Cloud Storage Modules + +To use a Terraform module available in a Google Cloud Storage bucket, set the source +to a URL with the special `gcs::` prefix, followed by a [GCS bucket object URL](https://cloud.google.com/storage/docs/request-endpoints#typical). + +For example: `gcs::https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH_TO_MODULE` + +### Kind (May be Required) + +`kind` refers to the way in which a module is deployed. Currently, `kind` can be +either `terraform` or `packer`. It must be specified for modules of type +`packer`. If omitted, it will default to `terraform`. + +### Settings (May Be Required) + +The settings field is a map that supplies any user-defined variables for each +module. Settings values can be simple strings, numbers or booleans, but can +also support complex data types like maps and lists of variable depth. These +settings will become the values for the variables defined in either the +`variables.tf` file for Terraform or `variable.pkr.hcl` file for Packer. + +For some modules, there are mandatory variables that must be set, +therefore `settings` is a required field in that case. In many situations, a +combination of sensible defaults, deployment variables and used modules can +populated all required settings and therefore the settings field can be omitted. + +### Use (Optional) + +The `use` field is a powerful way of linking a module to one or more other +modules. When a module "uses" another module, the outputs of the used +module are compared to the settings of the current module. If they have +matching names and the setting has no explicit value, then it will be set to +the used module's output. For example, see the following blueprint snippet: + +```yaml +modules: +- id: network1 + source: modules/network/vpc + +- id: workstation + source: modules/compute/vm-instance + use: [network1] + settings: + ... +``` + +In this snippet, the VM instance `workstation` uses the outputs of vpc +`network1`. + +In this case both `network_self_link` and `subnetwork_self_link` in the +[workstation settings](compute/vm-instance/README.md#Inputs) will be set +to `$(network1.network_self_link)` and `$(network1.subnetwork_self_link)` which +refer to the [network1 outputs](network/vpc/README#Outputs) +of the same names. + +The order of precedence that `gcluster` uses in determining when to infer a setting +value is in the following priority order: + +1. Explicitly set in the blueprint using the `settings` field +1. Output from a used module, taken in the order provided in the `use` list +1. Deployment variable (`vars`) of the same name +1. Default value for the setting + +> **_NOTE:_** See the +> [network storage documentation](./../docs/network_storage.md) for more +> information about mounting network storage file systems via the `use` field. + +### Outputs (Optional) + +The `outputs` field adds the output of individual Terraform modules to the +output of its deployment group. This enables the value to be available via +`terraform output`. This can useful for displaying the IP of a login node or +printing instructions on how to use a module, as we have in the +[monitoring dashboard module](monitoring/dashboard/README.md#Outputs). + +The outputs field is a lists that it can be in either of two formats: a string +equal to the name of the module output, or a map specifying the `name`, +`description`, and whether the value is `sensitive` and should be suppressed +from the standard output of Terraform commands. An example is shown below +that displays the internal and public IP addresses of a VM created by the +vm-instance module: + +```yaml + - id: vm + source: modules/compute/vm-instance + use: + - network1 + settings: + machine_type: e2-medium + outputs: + - internal_ip + - name: external_ip + description: "External IP of VM" + sensitive: true +``` + +The outputs shown after running Terraform apply will resemble: + +```text +Apply complete! Resources: 7 added, 0 changed, 0 destroyed. + +Outputs: + +external_ip_simplevm = +internal_ip_simplevm = [ + "10.128.0.19", +] +``` + +### Required Services (APIs) (optional) + +Each Toolkit module depends upon Google Cloud services ("APIs") being enabled +in the project used by the AI/ML and HPC environment. For example, the [creation of +VMs](compute/vm-instance/) requires the Compute Engine API +(compute.googleapis.com). The [startup-script](scripts/startup-script/) module +requires the Cloud Storage API (storage.googleapis.com) for storage of the +scripts themselves. Each module included in the Toolkit source code describes +its required APIs internally. The Toolkit will merge the requirements from all +modules and [automatically validate](../README.md#blueprint-validation) that all +APIs are enabled in the project specified by `$(vars.project_id)`. + +## Common Settings + +The following common naming conventions should be used to decrease the verbosity +needed to define a blueprint. This is intentional to allow multiple +modules to share inferred settings from deployment variables or from other +modules listed under the `use` field. + +For example, if all modules are to be created in a single region, that region +can be defined as a deployment variable named `region`, which is shared between +all modules without an explicit setting. Similarly, if many modules need to be +connected to the same VPC network, they all can add the vpc module ID to their +`use` list so that `network_self_link` would be inferred from that vpc module rather +than having to set it manually. + +* **project_id**: The GCP project ID in which to create the GCP resources. +* **deployment_name**: The name of the current deployment of a blueprint. This + can help to avoid naming conflicts of modules when multiple deployments are + created from the same blueprint. +* **region**: The GCP + [region](https://cloud.google.com/compute/docs/regions-zones) the module + will be created in. +* **zone**: The GCP [zone](https://cloud.google.com/compute/docs/regions-zones) + the module will be created in. +* **labels**: + [Labels](https://cloud.google.com/resource-manager/docs/creating-managing-labels) + added to the module. In order to include any module in advanced + monitoring, labels must be exposed. We strongly recommend that all modules + expose this variable. + +## Writing Custom Cluster Toolkit Modules + +Modules are flexible by design, however we define some [best practices](../docs/module-guidelines.md) when +creating a new module meant to be used with the Cluster Toolkit. diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/README.md b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/README.md new file mode 100644 index 0000000000..f807cd727e --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/README.md @@ -0,0 +1,133 @@ +## Description + +This module is used to create a Kubernetes job template file. + +The job template file can be submitted as is or used as a template for further +customization. Add the `instructions` output to a blueprint (as shown below) to +get instructions on how to use `kubectl` to submit the job. + +This module is designed to `use` one or more `gke-node-pool` modules. The job +will be configured to run on any of the specified node pools. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a GKE job template file. + +```yaml + - id: job-template + source: modules/compute/gke-job-template + use: [compute_pool] + settings: + node_count: 3 + outputs: [instructions] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### Storage Options + +This module natively supports: + +* Filestore as a shared file system between pods/nodes. +* Pod level ephemeral storage options: + * memory backed emptyDir + * local SSD backed emptyDir + * SSD persistent disk backed ephemeral volume + * balanced persistent disk backed ephemeral volume + +See the [storage-gke.yaml blueprint](../../../examples/storage-gke.yaml) and the +associated [documentation](../../../../examples/README.md#storage-gkeyaml--) for +examples of how to use Filestore and ephemeral storage with this module. + +### Requested Resources + +When one or more `gke-node-pool` modules are referenced with the `use` field. +The requested resources will be populated to achieve a 1 pod per node packing +while still leaving some headroom for required system pods. + +This functionality can be overridden by specifying the desired cpu requirement +using the `requested_cpu_per_pod` setting. + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.2 | +| [local](#requirement\_local) | >= 2.0.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allocatable\_cpu\_per\_node](#input\_allocatable\_cpu\_per\_node) | The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | +| [allocatable\_gpu\_per\_node](#input\_allocatable\_gpu\_per\_node) | The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | +| [backoff\_limit](#input\_backoff\_limit) | Controls the number of retries before considering a Job as failed. Set to zero for shared fate. | `number` | `0` | no | +| [command](#input\_command) | The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes. | `list(string)` | 
[
  "hostname"
]
| no | +| [completion\_mode](#input\_completion\_mode) | Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information | `string` | `"Indexed"` | no | +| [ephemeral\_volumes](#input\_ephemeral\_volumes) | Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB. | 
list(object({
    type       = string
    mount_path = string
    size_gb    = number
  }))
| `[]` | no | +| [has\_gpu](#input\_has\_gpu) | Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module. | `list(bool)` | 
[
  false
]
| no | +| [image](#input\_image) | The container image the job should use. | `string` | `"debian"` | no | +| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account to run the job as. If null then no service account is specified. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the GKE job template. Key-value pairs. | `map(string)` | n/a | yes | +| [machine\_family](#input\_machine\_family) | The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria. | `string` | `null` | no | +| [name](#input\_name) | The name of the job. | `string` | `"my-job"` | no | +| [node\_count](#input\_node\_count) | How many nodes the job should run in parallel. | `number` | `1` | no | +| [node\_pool\_names](#input\_node\_pool\_names) | A list of node pool names on which to run the job. Can be populated via `use` field. | `list(string)` | `[]` | no | +| [node\_selectors](#input\_node\_selectors) | A list of node selectors to use to place the job. | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | +| [persistent\_volume\_claims](#input\_persistent\_volume\_claims) | A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module. | 
list(object({
    name          = string
    namespace     = string
    mount_path    = string
    mount_options = string
    storage_type  = string
  }))
| `[]` | no | +| [random\_name\_sufix](#input\_random\_name\_sufix) | Appends a random suffix to the job name to avoid clashes. | `bool` | `true` | no | +| [requested\_cpu\_per\_pod](#input\_requested\_cpu\_per\_pod) | The requested cpu per pod. If null, allocatable\_cpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_cpu\_per\_node. | `number` | `-1` | no | +| [requested\_gpu\_per\_pod](#input\_requested\_gpu\_per\_pod) | The requested gpu per pod. If null, allocatable\_gpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_gpu\_per\_node. | `number` | `-1` | no | +| [restart\_policy](#input\_restart\_policy) | Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed. | `string` | `"Never"` | no | +| [security\_context](#input\_security\_context) | The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | +| [tolerations](#input\_tolerations) | Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field. | 
list(object({
    key      = string
    operator = string
    value    = string
    effect   = string
  }))
| 
[
  {
    "effect": "NoSchedule",
    "key": "user-workload",
    "operator": "Equal",
    "value": "true"
  }
]
| no | +| [tpu\_accelerator\_type](#input\_tpu\_accelerator\_type) | The TPU accelerator type label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | +| [tpu\_chips\_per\_node](#input\_tpu\_chips\_per\_node) | The number of TPU chips per node. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | +| [tpu\_topology](#input\_tpu\_topology) | The TPU topology label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for submitting the GKE job. | + diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/main.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/main.tf new file mode 100644 index 0000000000..e84138bb3f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/main.tf @@ -0,0 +1,181 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-job-template", ghpc_role = "compute" }) +} + +locals { + tpu_accelerator_node_selector = var.tpu_accelerator_type[0] != null ? [{ + key = "cloud.google.com/gke-tpu-accelerator" + value = var.tpu_accelerator_type[0] + }] : [] + + tpu_topology_node_selector = var.tpu_topology[0] != null ? [{ + key = "cloud.google.com/gke-tpu-topology" + value = var.tpu_topology[0] + }] : [] +} + +locals { + # Start with the minimum cpu available of used node pools + min_allocatable_cpu = min(var.allocatable_cpu_per_node...) + full_node_cpu_request = ( + local.min_allocatable_cpu > 2 ? # if large enough + local.min_allocatable_cpu - 1 : # leave headroom for 1 cpu + local.min_allocatable_cpu / 2 + 0.1 # else take just over half + ) - (local.any_gcs ? 0.25 : 0) # save room for gcs side car + + cpu_request = ( + var.requested_cpu_per_pod >= 0 ? # if user supplied requested cpu + var.requested_cpu_per_pod : # then honor it + ( # else + local.min_allocatable_cpu >= 0 ? # if allocatable cpu was supplied + local.full_node_cpu_request : # then claim the full node + -1 # else do not set a limit + ) + ) + millicpu = floor(local.cpu_request * 1000) + cpu_request_string = local.millicpu >= 0 ? "${local.millicpu}m" : null + full_node_request = local.min_allocatable_cpu >= 0 && var.requested_cpu_per_pod < 0 + + memory_request_value = try(sum([for ed in var.ephemeral_volumes : + ed.size_gb + if ed.type == "memory" + ]), 0) + memory_request_string = local.memory_request_value > 0 ? "${local.memory_request_value}Gi" : null + + ephemeral_request_value = try(sum([for ed in var.ephemeral_volumes : + ed.size_gb + if ed.type == "local-ssd" + ]), 0) + ephemeral_request_string = local.ephemeral_request_value > 0 ? "${local.ephemeral_request_value}Gi" : null + + uses_local_ssd = anytrue([for ed in var.ephemeral_volumes : + ed.type == "local-ssd" + ]) + local_ssd_node_selector = local.uses_local_ssd ? [{ + key = "cloud.google.com/gke-ephemeral-storage-local-ssd" + value = "true" + }] : [] + + # Setup limit for GPUs per pod + min_allocatable_gpu = min(var.allocatable_gpu_per_node...) + min_allocatable_gpu_per_pod = local.min_allocatable_gpu > 0 ? local.min_allocatable_gpu : null + gpu_limit_per_pod = var.requested_gpu_per_pod > 0 ? var.requested_gpu_per_pod : local.min_allocatable_gpu_per_pod + gpu_limit_string = alltrue(var.has_gpu) ? tostring(local.gpu_limit_per_pod) : null + + empty_dir_volumes = [for ed in var.ephemeral_volumes : + { + name = replace(trim(ed.mount_path, "/"), "/", "-") + mount_path = ed.mount_path + size_limit = "${ed.size_gb}Gi" + in_memory = ed.type == "memory" + } + if contains(["memory", "local-ssd"], ed.type) + ] + + ephemeral_pd_volumes = [for pd in var.ephemeral_volumes : + { + name = replace(trim(pd.mount_path, "/"), "/", "-") + mount_path = pd.mount_path + storage_class_name = pd.type == "pd-ssd" ? "premium-rwo" : "standard-rwo" + storage = "${pd.size_gb}Gi" + } + if contains(["pd-balanced", "pd-ssd"], pd.type) + ] + + pvc_volumes = [for pvc in var.persistent_volume_claims : + { + name = replace(trim(pvc.mount_path, "/"), "/", "-") + mount_path = pvc.mount_path + claim_name = pvc.name + } + ] + + volume_mounts = [for v in concat(local.empty_dir_volumes, local.ephemeral_pd_volumes, local.pvc_volumes) : + { + name = v.name + mount_path = v.mount_path + } + ] + + suffix = var.random_name_sufix ? "-${random_id.resource_name_suffix.hex}" : "" + machine_family_node_selector = var.machine_family != null ? [{ + key = "cloud.google.com/machine-family" + value = var.machine_family + }] : [] + node_selectors = concat(local.machine_family_node_selector, local.local_ssd_node_selector, local.tpu_accelerator_node_selector, local.tpu_topology_node_selector, var.node_selectors) + + any_gcs = anytrue([for pvc in var.persistent_volume_claims : + pvc.storage_type == "gcs" + ]) + + job_template_contents = templatefile( + "${path.module}/templates/gke-job-base.yaml.tftpl", + { + name = var.name + suffix = local.suffix + image = var.image + command = var.command + node_count = var.node_count + completion_mode = var.completion_mode + k8s_service_account_name = var.k8s_service_account_name + node_pool_names = var.node_pool_names + node_selectors = local.node_selectors + tpu_limit = var.tpu_chips_per_node[0] + full_node_request = local.full_node_request + cpu_request = local.cpu_request_string + gpu_limit = local.gpu_limit_string + restart_policy = var.restart_policy + backoff_limit = var.backoff_limit + tolerations = distinct(var.tolerations) + security_context = var.security_context + labels = local.labels + + empty_dir_volumes = local.empty_dir_volumes + ephemeral_pd_volumes = local.ephemeral_pd_volumes + pvc_volumes = local.pvc_volumes + volume_mounts = local.volume_mounts + memory_request = local.memory_request_string + ephemeral_request = local.ephemeral_request_string + gcs_annotation = local.any_gcs + } + ) + + job_template_output_path = "${path.root}/${var.name}${local.suffix}.yaml" + +} + +resource "random_id" "resource_name_suffix" { + byte_length = 2 + keepers = { + timestamp = timestamp() + } +} + +resource "local_file" "job_template" { + content = local.job_template_contents + filename = local.job_template_output_path + + lifecycle { + precondition { + condition = local.any_gcs ? var.k8s_service_account_name != null : true + error_message = "When using GCS, a kubernetes service account with workload identity is required. gke-cluster module will perform this setup when var.configure_workload_identity_sa is set to true." + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf new file mode 100644 index 0000000000..adf78e936d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instructions" { + description = "Instructions for submitting the GKE job." + value = <<-EOT + A GKE job file has been created locally at: + ${abspath(local.job_template_output_path)} + + Use the following commands to: + Submit your job: + kubectl create -f ${abspath(local.job_template_output_path)} + EOT +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl new file mode 100644 index 0000000000..11df39ce2c --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl @@ -0,0 +1,128 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ${name}${suffix} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + parallelism: ${node_count} + completions: ${node_count} + completionMode: ${completion_mode} + template: + %{~ if gcs_annotation ~} + metadata: + annotations: + gke-gcsfuse/volumes: "true" + %{~ endif ~} + spec: + %{~ if length(security_context) > 0 ~} + securityContext: + %{~ for context in security_context ~} + ${context.key}: ${context.value} + %{~ endfor ~} + %{~ endif ~} + %{~ if k8s_service_account_name != null ~} + serviceAccountName: ${k8s_service_account_name} + %{~ endif ~} + %{~ if length(node_pool_names) > 0 ~} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + %{~ for node_pool in node_pool_names ~} + - ${node_pool} + %{~ endfor ~} + %{~ endif ~} + %{~ if length(node_selectors) > 0 ~} + nodeSelector: + %{~ for selector in node_selectors ~} + ${selector.key}: "${selector.value}" + %{~ endfor ~} + %{~ endif ~} + tolerations: + %{~ for toleration in tolerations ~} + - key: ${toleration.key} + operator: ${toleration.operator} + value: "${toleration.value}" + effect: ${toleration.effect} + %{~ endfor ~} + containers: + - name: ${name}-container + image: ${image} + command: + %{for s in command}- ${indent(8, yamlencode(s))}%{~ endfor } + %{~ if gpu_limit != null || cpu_request != null || tpu_limit != null ~} + resources: + %{~ if gpu_limit != null || tpu_limit != null ~} + limits: + %{~ if gpu_limit != null ~} + # GPUs should only be specified as limits + # https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/ + nvidia.com/gpu: ${gpu_limit} + %{~ endif ~} + %{~ if tpu_limit != null ~} + google.com/tpu: ${tpu_limit} + %{~ endif ~} + %{~ endif ~} + %{~ if cpu_request != null || memory_request != null || ephemeral_request != null || tpu_limit != null ~} + requests: + %{~ if full_node_request ~} + # cpu request attempts full node per pod + %{~ endif ~} + %{~ if cpu_request != null ~} + cpu: ${cpu_request} + %{~ endif ~} + %{~ if tpu_limit != null ~} + google.com/tpu: ${tpu_limit} + %{~ endif ~} + %{~ if memory_request != null ~} + memory: ${memory_request} + %{~ endif ~} + %{~ if ephemeral_request != null ~} + ephemeral-storage: ${ephemeral_request} + %{~ endif ~} + %{~ endif ~} + %{~ endif ~} + %{~ if length(volume_mounts) > 0 ~} + volumeMounts: + %{~ for v in volume_mounts ~} + - name: ${v.name} + mountPath: ${v.mount_path} + %{~ endfor ~} + %{~ endif ~} + %{~ if length(volume_mounts) > 0 ~} + volumes: + %{~ for ed in empty_dir_volumes ~} + - name: ${ed.name} + emptyDir: + sizeLimit: ${ed.size_limit} + %{~ if ed.in_memory ~} + medium: "Memory" + %{~ endif ~} + %{~ endfor ~} + %{~ for pd in ephemeral_pd_volumes ~} + - name: ${pd.name} + ephemeral: + volumeClaimTemplate: + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: ${pd.storage_class_name} + resources: + requests: + storage: ${pd.storage} + %{~ endfor ~} + %{~ for pvc in pvc_volumes ~} + - name: ${pvc.name} + persistentVolumeClaim: + claimName: ${pvc.claim_name} + %{~ endfor ~} + %{~ endif ~} + restartPolicy: ${restart_policy} + backoffLimit: ${backoff_limit} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/variables.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/variables.tf new file mode 100644 index 0000000000..fd83f2b692 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/variables.tf @@ -0,0 +1,206 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "name" { + description = "The name of the job." + type = string + default = "my-job" +} + +variable "node_count" { + description = "How many nodes the job should run in parallel." + type = number + default = 1 +} + +variable "completion_mode" { + description = "Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information" + type = string + default = "Indexed" +} + +variable "command" { + description = "The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes." + type = list(string) + default = ["hostname"] +} + +variable "image" { + description = "The container image the job should use." + type = string + default = "debian" +} + +variable "k8s_service_account_name" { + description = "Kubernetes service account to run the job as. If null then no service account is specified." + type = string + default = null +} + +variable "node_pool_names" { + description = "A list of node pool names on which to run the job. Can be populated via `use` field." + type = list(string) + default = [] +} + +variable "allocatable_cpu_per_node" { + description = "The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." + type = list(number) + default = [-1] +} + +variable "has_gpu" { + description = "Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module." + type = list(bool) + default = [false] +} + +variable "requested_cpu_per_pod" { + description = "The requested cpu per pod. If null, allocatable_cpu_per_node will be used to claim whole nodes. If provided will override allocatable_cpu_per_node." + type = number + default = -1 +} + +variable "allocatable_gpu_per_node" { + description = "The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." + type = list(number) + default = [-1] +} + +variable "requested_gpu_per_pod" { + description = "The requested gpu per pod. If null, allocatable_gpu_per_node will be used to claim whole nodes. If provided will override allocatable_gpu_per_node." + type = number + default = -1 +} + +variable "tolerations" { + description = "Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field." + type = list(object({ + key = string + operator = string + value = string + effect = string + })) + default = [ + { + key = "user-workload" + operator = "Equal" + value = "true" + effect = "NoSchedule" + } + ] +} + +variable "security_context" { + description = "The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/" + type = list(object({ + key = string + value = string + })) + default = [] +} + +variable "machine_family" { + description = "The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria." + type = string + default = null +} + +variable "node_selectors" { + description = "A list of node selectors to use to place the job." + type = list(object({ + key = string + value = string + })) + default = [] +} + +variable "restart_policy" { + description = "Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed." + type = string + default = "Never" +} + +variable "backoff_limit" { + description = "Controls the number of retries before considering a Job as failed. Set to zero for shared fate." + type = number + default = 0 +} + +variable "random_name_sufix" { + description = "Appends a random suffix to the job name to avoid clashes." + type = bool + default = true +} + +variable "persistent_volume_claims" { + description = "A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module." + type = list(object({ + name = string + namespace = string + mount_path = string + mount_options = string + storage_type = string + })) + default = [] +} + +variable "ephemeral_volumes" { + description = "Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB." + type = list(object({ + type = string + mount_path = string + size_gb = number + })) + default = [] + validation { + condition = alltrue([ + for v in var.ephemeral_volumes : + contains(["pd-balanced", "pd-ssd", "memory", "local-ssd"], v.type) + ]) + error_message = "Type must be one of 'pd-balanced', 'pd-ssd', 'memory', 'local-ssd'." + } + validation { + condition = alltrue([ + for v in var.ephemeral_volumes : + substr(v.mount_path, 0, 1) == "/" + ]) + error_message = "Mount path must start with the '/' character." + } +} + +variable "labels" { + description = "Labels to add to the GKE job template. Key-value pairs." + type = map(string) +} + +variable "tpu_accelerator_type" { + description = "The TPU accelerator type label. Populated from gke-node-pool via `use` field." + type = list(string) + default = [null] +} + +variable "tpu_topology" { + description = "The TPU topology label. Populated from gke-node-pool via `use` field." + type = list(string) + default = [null] +} + +variable "tpu_chips_per_node" { + description = "The number of TPU chips per node. Populated from gke-node-pool via `use` field." + type = list(string) + default = [null] +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/versions.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/versions.tf new file mode 100644 index 0000000000..0f902ac8c5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/versions.tf @@ -0,0 +1,28 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.2" + + required_providers { + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/README.md b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/README.md new file mode 100644 index 0000000000..b25d905252 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/README.md @@ -0,0 +1,388 @@ +## Description + +This module creates a Google Kubernetes Engine +([GKE](https://cloud.google.com/kubernetes-engine)) node pool. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a GKE node group. + +```yaml + - id: compute_pool + source: modules/compute/gke-node-pool + use: [gke_cluster] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### Taints and Tolerations + +By default node pools created with this module will be tainted with +`user-workload=true:NoSchedule` to prevent system pods from being scheduled. +User jobs targeting the node pool should include this toleration. This behavior +can be overridden using the `taints` setting. See +[docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-taints) for +more info. + +### Local SSD Storage +GKE offers two options for managing locally attached SSDs. + +The first, and recommended, option is for GKE to manage the ephemeral storage +space on the node, which will then be automatically attached to pods which +request an `emptyDir` volume. This can be accomplished using the +[`local_ssd_count_ephemeral_storage`] variable. + +The second, more complex, option is for GCP to attach these nodes as raw block +storage. In this case, the cluster administrator is responsible for software +RAID settings, partitioning, formatting and mounting these disks on the host +OS. Still, this may be desired behavior in use cases which aren't supported +by an `emptyDir` volume (for example, a `ReadOnlyMany` or `ReadWriteMany` PV). +This can be accomplished using the [`local_ssd_count_nvme_block`] variable. + +The [`local_ssd_count_ephemeral_storage`] and [`local_ssd_count_nvme_block`] +variables are mutually exclusive and cannot be mixed together. + +Also, the number of SSDs which can be attached to a node depends on the +[machine type](https://cloud.google.com/compute/docs/disks#local_ssd_machine_type_restrictions). + +See [docs](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd) +for more info. + +[`local_ssd_count_ephemeral_storage`]: #input\_local\_ssd\_count\_ephemeral\_storage +[`local_ssd_count_nvme_block`]: #input\_local\_ssd\_count\_nvme\_block + +### Considerations with GPUs + +When a GPU is attached to a node an additional taint is automatically added: +`nvidia.com/gpu=present:NoSchedule`. For jobs to get placed on these nodes, the +equivalent toleration is required. The `gke-job-template` module will +automatically apply this toleration when using a node pool with GPUs. + +Nvidia GPU drivers must be installed. The recommended approach for GKE to install +GPU dirvers is by applying a DaemonSet to the cluster. See +[these instructions](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus#cos). + +However, in some cases it may be desired to compile a different driver (such as +a desire to install a newer version, compatibility with the +[Nvidia GPU-operator](https://github.com/NVIDIA/gpu-operator) or other +use-cases). In this case, ensure that you turn off the +[enable_secure_boot](#input\_enable\_secure\_boot) option to allow unsigned +kernel modules to be loaded. + +#### Maximize GPU network bandwidth with GPUDirect and multi-networking +For A3 Series machines to achieve optimal performance , GKE provide two networking stacks for remote direct memory access (RDMA): + +- A3 High machine types (a3-highgpu-8g): utilize GPUDirect-TCPX to reduce the overhead required to transfer packet payloads to and from GPUs, which significantly improves throughput at scale compared to GPUs that don't use GPUDirect. +- A3 Mega machine types (a3-megagpu-8g): utilize GPUDirect-TCPXO to improve GPU to GPU communication, and further improves GPU to VM communication. + +To achieve this, when creating nodepools with A3 Series machine type, pass in a multivpc module to the gke-node-pool module, and the gke-node-pool module would detect the eligible machine type and enable GPUDirect for it. More specifically, the below components will be installed in the nodepool for enabling GPUDirect. + +- Install NCCL plugin for GPUDirect [TCPX](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpx) or [TCPXO](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpxo) +- Install [NRI](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nri_device_injector) device injector plugin +- Provide support for injecting GPUDirect required components(annotations, volumes, rxdm sidecar etc.) into the user workload in the form of Kubernetes Job. + - Provide sample workload to showcase how it will be updated with the required components injected, and how it can be deployed. + - Allow user to use the provided script to update their own workload and deploy. + +The GPUDirect supports included in the Cluster Toolkit aim to automate the [GPUDirect User Guid](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#install-gpudirect-tcpx-nccl) and provide better usability. + +> **_NOTE:_** You must [enable multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) feature when creating the GKE cluster. When gke-cluster depends on multivpc (with the use keyword), multi networking will be automatically enabled on the cluster creation. +> When gke-cluster or pre-existing-gke-cluster depends on multivpc (with the use keyword), the [network objects](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment) required for multi networking will be created on the cluster. + +### GPUs Examples + +There are several ways to add GPUs to a GKE node pool. See +[docs](https://cloud.google.com/compute/docs/gpus) for more info on GPUs. + +The following is a node pool that uses `a2`, `a3` or `g2` machine types which has a +fixed number of attached GPUs, let's call these machine types as "pre-defined gpu machine families": + +```yaml + - id: simple-a2-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g +``` + +> **Note**: It is not necessary to define the [`guest_accelerator`] setting when +> using pre-defined gpu machine families as information about GPUs, such as type, count and +> `gpu_driver_installation_config`, is automatically inferred from the machine type. +> Optional fields such as `gpu_partition_size` need to be specified only if they have +> non-default values. + +The following scenarios require the [`guest_accelerator`] block is specified: + +- To partition an A100 GPU into multiple GPUs on an A2 family machine. +- To specify a time sharing configuration on a GPUs. +- To attach a GPU to an N1 family machine. + +The following is an example of +[partitioning](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus-multi) +an A100 GPU: + +> **Note**: In the following example, `type`, `count` and `gpu_driver_installation_config` are picked up automatically. + +```yaml + - id: multi-instance-gpu-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g + guest_accelerator: + - gpu_partition_size: 1g.5gb +``` + +[`guest_accelerator`]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_guest_accelerator + +The following is an example of +[GPU time sharing](https://cloud.google.com/kubernetes-engine/docs/concepts/timesharing-gpus) +(with partitioned GPUs): + +```yaml + - id: time-sharing-gpu-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g + guest_accelerator: + - gpu_partition_size: 1g.5gb + gpu_sharing_config: + gpu_sharing_strategy: TIME_SHARING + max_shared_clients_per_gpu: 3 +``` + +Following is an example of using a GPU attached to an `n1` machine: + +```yaml + - id: t4-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: n1-standard-16 + guest_accelerator: + - type: nvidia-tesla-t4 + count: 2 +``` + +The following is an example of using a GPU (with sharing config) attached to an `n1` machine: + +```yaml + - id: n1-t4-pool + source: community/modules/compute/gke-node-pool + use: [gke_cluster] + settings: + name: n1-t4-pool + machine_type: n1-standard-1 + guest_accelerator: + - type: nvidia-tesla-t4 + count: 2 + gpu_driver_installation_config: + gpu_driver_version: "LATEST" + gpu_sharing_config: + max_shared_clients_per_gpu: 2 + gpu_sharing_strategy: "TIME_SHARING" +``` + +Finally, the following is adding multivpc to a node pool: + +```yaml + - id: network + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: gke-cluster + source: modules/scheduler/gke-cluster + use: [network, multinetwork] + settings: + cluster_name: $(vars.deployment_name) + + - id: a3-megagpu_pool + source: modules/compute/gke-node-pool + use: [gke-cluster, multinetwork] + settings: + machine_type: a3-megagpu-8g + ... +``` + +## Using GCE Reservations +You can reserve Google Compute Engine instances in a specific zone to ensure resources are available for their workloads when needed. For more details on how to manage reservations, see [Reserving Compute Engine zonal resources](https://cloud.google.com/compute/docs/instances/reserving-zonal-resources). + +After creating a reservation, you can consume the reserved GCE VM instances in GKE. GKE clusters deployed using Cluster Toolkit support the same consumption modes as Compute Engine: NO_RESERVATION(default), ANY_RESERVATION, SPECIFIC_RESERVATION. + +This can be accomplished using [`reservation_affinity`](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/compute/gke-node-pool/README.md#input_reservation_affinity). + +```yaml +# Target any reservation +reservation_affinity: + consume_reservation_type: ANY_RESERVATION + +# Target a specific reservation +reservation_affinity: + consume_reservation_type: SPECIFIC_RESERVATION + specific_reservations: + - name: specific-reservation-1 +``` + +The following requirements need to be satisfied for the node pool nodes to be able to use a specific reservation: +1. A reservation with the name must exist in the specified project(`var.project_id`) and one of the specified zones(`var.zones`). +2. Its consumption type must be `specific`. +3. Its GCE VM Properties must match with those of the Node Pool; Machine type, Accelerators (GPU Type and count), Local SSD disk type and count. + +If you want to utilise a shared reservation, the owner project of the shared reservation needs to be explicitly specified like the following. Note that a shared reservation can be used by the project that hosts the reservation (owner project) and by the projects the reservation is shared with (consumer projects). See how to [create and use a shared reservation](https://cloud.google.com/compute/docs/instances/reservations-shared). + +```yaml +reservation_affinity: + consume_reservation_type: SPECIFIC_RESERVATION + specific_reservations: + - name: specific-reservation-shared + project: shared_reservation_owner_project_id +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 7.2 | +| [google-beta](#requirement\_google-beta) | >= 7.2 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 7.2 | +| [google-beta](#provider\_google-beta) | >= 7.2 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | +| [tpu](#module\_tpu) | ../../internal/tpu-definition | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_container_node_pool.node_pool](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | +| [null_resource.enable_tcpx_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.enable_tcpxo_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.install_dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_machine_types.machine_info](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | +| [google_compute_region_instance_template.instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_template) | data source | +| [google_compute_reservation.specific_reservations](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [auto\_repair](#input\_auto\_repair) | Whether the nodes will be automatically repaired. | `bool` | `true` | no | +| [auto\_upgrade](#input\_auto\_upgrade) | Whether the nodes will be automatically upgraded. | `bool` | `false` | no | +| [autoscaling\_total\_max\_nodes](#input\_autoscaling\_total\_max\_nodes) | Total maximum number of nodes in the NodePool. | `number` | `1000` | no | +| [autoscaling\_total\_min\_nodes](#input\_autoscaling\_total\_min\_nodes) | Total minimum number of nodes in the NodePool. | `number` | `0` | no | +| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | +| [compact\_placement](#input\_compact\_placement) | DEPRECATED: Use `placement_policy` | `bool` | `null` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for each node. | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Disk type for each node. | `string` | `null` | no | +| [enable\_flex\_start](#input\_enable\_flex\_start) | If true, start the node pool with Flex Start provisioning model.
To learn more about flex-start mode, please refer to
https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and
https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest | `bool` | `false` | no | +| [enable\_gcfs](#input\_enable\_gcfs) | Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config). | `bool` | `false` | no | +| [enable\_numa\_aware\_scheduling](#input\_enable\_numa\_aware\_scheduling) | Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling. | `bool` | `false` | no | +| [enable\_private\_nodes](#input\_enable\_private\_nodes) | Whether nodes have internal IP addresses only. | `bool` | `true` | no | +| [enable\_queued\_provisioning](#input\_enable\_queued\_provisioning) | If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool. | `bool` | `false` | no | +| [enable\_secure\_boot](#input\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | +| [gke\_version](#input\_gke\_version) | GKE version | `string` | n/a | yes | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = optional(string)
    count = optional(number, 0)
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | +| [host\_maintenance\_interval](#input\_host\_maintenance\_interval) | Specifies the frequency of planned maintenance events. | `string` | `""` | no | +| [image\_type](#input\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | +| [initial\_node\_count](#input\_initial\_node\_count) | The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static\_node\_count and must be set to a value between autoscaling\_total\_min\_nodes and autoscaling\_total\_max\_nodes. | `number` | `null` | no | +| [internal\_ghpc\_module\_id](#input\_internal\_ghpc\_module\_id) | DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide). | `string` | n/a | yes | +| [is\_reservation\_active](#input\_is\_reservation\_active) | Whether the specified reservation is already created. | `bool` | `true` | no | +| [kubernetes\_labels](#input\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_count\_ephemeral\_storage](#input\_local\_ssd\_count\_ephemeral\_storage) | The number of local SSDs to attach to each node to back ephemeral storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | +| [local\_ssd\_count\_nvme\_block](#input\_local\_ssd\_count\_nvme\_block) | The number of local SSDs to attach to each node to back block storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | +| [machine\_type](#input\_machine\_type) | The name of a Google Compute Engine machine type. | `string` | `"c2-standard-60"` | no | +| [max\_pods\_per\_node](#input\_max\_pods\_per\_node) | The maximum number of pods per node in this node pool. This will force replacement. | `number` | `null` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [name](#input\_name) | The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix.
If setting manually, ensure a unique value across all gke-node-pools. | `string` | `null` | no | +| [num\_node\_pools](#input\_num\_node\_pools) | Number of node pools to create. This is same as num\_slices. | `number` | `1` | no | +| [num\_slices](#input\_num\_slices) | Number of TPUs slices to create. This is same as num\_node\_pools. | `number` | `1` | no | +| [placement\_policy](#input\_placement\_policy) | Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| 
{
  "name": null,
  "tpu_topology": null,
  "type": null
}
| no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Reservation resource to consume. When targeting SPECIFIC\_RESERVATION, specific\_reservations needs be specified.
Even though specific\_reservations is a list, only one reservation is allowed by the NodePool API.
It is assumed that the specified reservation exists and has available capacity.
For a shared reservation, specify the project\_id as well in which it was created.
To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared | 
object({
    consume_reservation_type = string
    specific_reservations = optional(list(object({
      name    = string
      project = optional(string)
    })))
  })
| 
{
  "consume_reservation_type": "NO_RESERVATION",
  "specific_reservations": []
}
| no | +| [run\_workload\_script](#input\_run\_workload\_script) | Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only. | `bool` | `true` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [static\_node\_count](#input\_static\_node\_count) | The static number of nodes in the node pool. If set, autoscaling will be disabled. | `number` | `null` | no | +| [taints](#input\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | +| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | +| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | +| [total\_max\_nodes](#input\_total\_max\_nodes) | DEPRECATED: Use autoscaling\_total\_max\_nodes. | `number` | `null` | no | +| [total\_min\_nodes](#input\_total\_min\_nodes) | DEPRECATED: Use autoscaling\_total\_min\_nodes. | `number` | `null` | no | +| [upgrade\_settings](#input\_upgrade\_settings) | Defines node pool upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | +| [zones](#input\_zones) | A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable. | `list(string)` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [allocatable\_cpu\_per\_node](#output\_allocatable\_cpu\_per\_node) | Number of CPUs available for scheduling pods on each node. | +| [allocatable\_gpu\_per\_node](#output\_allocatable\_gpu\_per\_node) | Number of GPUs available for scheduling pods on each node. | +| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | +| [guest\_accelerator](#output\_guest\_accelerator) | The accelerator type of the nodes. | +| [has\_gpu](#output\_has\_gpu) | Boolean value indicating whether nodes in the pool are configured with GPUs. | +| [instance\_templates](#output\_instance\_templates) | The URLs of Instance Templates | +| [instructions](#output\_instructions) | Instructions for submitting the sample GPUDirect enabled job. | +| [machine\_type](#output\_machine\_type) | Machine Type | +| [node\_count\_static](#output\_node\_count\_static) | The number of static nodes in node-pool. | +| [node\_pool\_names](#output\_node\_pool\_names) | Names of the node pools. | +| [static\_gpu\_count](#output\_static\_gpu\_count) | Total number of GPUs in the node pool. Available only for static node pools. | +| [tolerations](#output\_tolerations) | Tolerations needed for a pod to be scheduled on this node pool. | +| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | +| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | +| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | + diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf new file mode 100644 index 0000000000..0c1c255255 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +## Required variables: +# local_ssd_count_ephemeral_storage +# local_ssd_count_nvme_block +# machine_type + +locals { + + local_ssd_machines = { + "a3-highgpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, + "a3-megagpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, + "a3-ultragpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, + "a4-highgpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, + } + + generated_local_ssd_config = lookup(local.local_ssd_machines, var.machine_type, { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = null }) + + # Select in priority order: + # (1) var.local_ssd_count_ephemeral_storage and var.local_ssd_count_nvme_block if any is not null + # (2) local.local_ssd_machines if not empty + # (3) default to null value for both local_ssd_count_ephemeral_storage and local_ssd_count_nvme_block + local_ssd_config = (var.local_ssd_count_ephemeral_storage == null && var.local_ssd_count_nvme_block == null) ? local.generated_local_ssd_config : { local_ssd_count_ephemeral_storage = var.local_ssd_count_ephemeral_storage, local_ssd_count_nvme_block = var.local_ssd_count_nvme_block } +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml new file mode 100644 index 0000000000..1106f63479 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml @@ -0,0 +1,50 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: my-sample-job +spec: + parallelism: 2 + completions: 2 + completionMode: Indexed + template: + spec: + containers: + - name: nccl-test + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx-dev:v3.1.9 + imagePullPolicy: Always + command: + - /bin/sh + - -c + - | + service ssh restart; + sleep infinity; + env: + - name: LD_LIBRARY_PATH + value: /usr/local/nvidia/lib64 + volumeMounts: + - name: config-volume + mountPath: /configs + resources: + limits: + nvidia.com/gpu: 8 + volumes: + - name: config-volume + configMap: + name: nccl-configmap + defaultMode: 0777 + restartPolicy: Never + backoffLimit: 0 diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml new file mode 100644 index 0000000000..bce6720681 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml @@ -0,0 +1,70 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: my-sample-job +spec: + parallelism: 2 + completions: 2 + completionMode: Indexed + template: + spec: + hostname: host1 + subdomain: nccl-host-1 + containers: + - name: nccl-test + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.14 + imagePullPolicy: Always + command: + - /bin/sh + - -c + - | + set -ex + chmod 755 /scripts/demo-run-nccl-test-tcpxo-via-mpi.sh + cat >/scripts/allgather.sh < 0: + container["env"].extend(env_vars) + container["volumeMounts"].extend(volume_mounts) + +if __name__ == "__main__": + main() diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py new file mode 100644 index 0000000000..db9fb3e7ff --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py @@ -0,0 +1,186 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +import argparse +import os + +def main(): + parser = argparse.ArgumentParser(description="TCPXO Job Manifest Generator") + parser.add_argument("-f", "--file", required=True, help="Path to your job template YAML file") + parser.add_argument("-r", "--rxdm", required=True, help="RxDM version") + + args = parser.parse_args() + + # Get the YAML file from the user + if not args.file: + args.file = input("Please provide the path to your job template YAML file: ") + + # Get component versions from user + if not args.rxdm: + args.rxdm = input("Enter the RxDM version: ") + + # Load and modify the YAML + with open(args.file, "r") as file: + job_manifest = yaml.load(file, Loader=yaml.BaseLoader) + + # Update annotations + add_annotations(job_manifest) + + # Update volumes + add_volumes(job_manifest) + + # Update tolerations + add_tolerations(job_manifest) + + # Add tcpxo-daemon container + add_tcpxo_daemon_container(job_manifest, args.rxdm) + + # Update environment variables and volumeMounts for GPU containers + update_gpu_containers(job_manifest) + + # Generate the new YAML file + updated_job = str(yaml.dump(job_manifest, default_flow_style=False, width=1000, default_style="|", sort_keys=False)).replace("|-", "") + + new_file_name = args.file.replace(".yaml", "-tcpxo.yaml") + with open(new_file_name, "w", encoding="utf-8") as file: + file.write(updated_job) + + # Step 7: Provide instructions to the user + print("\nA new manifest has been generated and updated to have TCPXO enabled based on the provided workload") + print("It can be found in {path}".format(path=os.path.abspath(new_file_name))) + print("You can use the following commands to submit the sample job:") + print(" kubectl create -f {path}".format(path=os.path.abspath(new_file_name))) + +def add_annotations(job_manifest): + annotations = { + 'devices.gke.io/container.tcpxo-daemon':"""|+ +- path: /dev/nvidia0 +- path: /dev/nvidia1 +- path: /dev/nvidia2 +- path: /dev/nvidia3 +- path: /dev/nvidia4 +- path: /dev/nvidia5 +- path: /dev/nvidia6 +- path: /dev/nvidia7 +- path: /dev/nvidiactl +- path: /dev/nvidia-uvm +- path: /dev/dmabuf_import_helper""", + "networking.gke.io/default-interface": "eth0", + "networking.gke.io/interfaces": """| +[ + {"interfaceName":"eth0","network":"default"}, + {"interfaceName":"eth1","network":"vpc1"}, + {"interfaceName":"eth2","network":"vpc2"}, + {"interfaceName":"eth3","network":"vpc3"}, + {"interfaceName":"eth4","network":"vpc4"}, + {"interfaceName":"eth5","network":"vpc5"}, + {"interfaceName":"eth6","network":"vpc6"}, + {"interfaceName":"eth7","network":"vpc7"}, + {"interfaceName":"eth8","network":"vpc8"} +]""", + } + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("metadata", {}) + + # Add/update annotations + pod_template_spec = job_manifest["spec"]["template"]["metadata"] + if "annotations" in pod_template_spec: + pod_template_spec["annotations"].update(annotations) + else: + pod_template_spec["annotations"] = annotations + +def add_tolerations(job_manifest): + tolerations = [ + {"key": "user-workload", "operator": "Equal", "value": """\"true\"""", "effect": "NoSchedule"}, + ] + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add tolerations + pod_spec = job_manifest["spec"]["template"]["spec"] + if "tolerations" in pod_spec: + pod_spec["tolerations"].extend(tolerations) + else: + pod_spec["tolerations"] = tolerations + +def add_volumes(job_manifest): + volumes = [ + {"name": "nvidia-install-dir-host", "hostPath": {"path": "/home/kubernetes/bin/nvidia"}}, + {"name": "sys", "hostPath": {"path": "/sys"}}, + {"name": "proc-sys", "hostPath": {"path": "/proc/sys"}}, + {"name": "aperture-devices", "hostPath": {"path": "/dev/aperture_devices"}}, + ] + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add volumes + pod_spec = job_manifest["spec"]["template"]["spec"] + if "volumes" in pod_spec: + pod_spec["volumes"].extend(volumes) + else: + pod_spec["volumes"] = volumes + + +def add_tcpxo_daemon_container(job_template, rxdm_version): + tcpxo_daemon_container = { + "name": "tcpxo-daemon", + "image": f"us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:{rxdm_version}", # Use provided RxDM version + "imagePullPolicy": "Always", + "command": ["/bin/sh", "-c"], + "args": [ + """| + set -ex + chmod 755 /fts/entrypoint_rxdm_container.sh + /fts/entrypoint_rxdm_container.sh --num_hops=2 --num_nics=8 --uid= --alsologtostderr""" + ], + "securityContext": { + "capabilities": {"add": ["NET_ADMIN", "NET_BIND_SERVICE"]} + }, + "volumeMounts": [ + {"name": "nvidia-install-dir-host", "mountPath": "/usr/local/nvidia"}, + {"name": "sys", "mountPath": "/hostsysfs"}, + {"name": "proc-sys", "mountPath": "/hostprocsysfs"}, + ], + "env": [{"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}], + } + + # Create path if it doesn't exist + job_template.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add container + pod_spec = job_template["spec"]["template"]["spec"] + pod_spec.setdefault("containers", []).insert(0, tcpxo_daemon_container) + +def update_gpu_containers(job_manifest): + env_vars = [ + {"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}, + {"name": "NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY", "value": "/dev/aperture_devices"}, + ] + volume_mounts = [{"name": "aperture-devices", "mountPath": "/dev/aperture_devices"}] + + pod_spec = job_manifest.get("spec", {}).get("template", {}).get("spec", {}) + for container in pod_spec.get("containers", []): + # Create path if it doesn't exist + container.setdefault("env", []) + container.setdefault("volumeMounts", []) + if int(container.get("resources", {}).get("limits", {}).get("nvidia.com/gpu", 0)) > 0: + container["env"].extend(env_vars) + container["volumeMounts"].extend(volume_mounts) + +if __name__ == "__main__": + main() diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf new file mode 100644 index 0000000000..d23d050986 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf @@ -0,0 +1,87 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# Enable GPUDirect for A3 and A3Mega VMs, this involve multiple kubectl steps to integrate with the created cluster +# 1. Install NCCL plugin daemonset +# 2. Install NRI plugin daemonset +# 3. Update provided workload to inject rxdm sidecar and other required annotation, volume etc. +locals { + workload_path_tcpx = "${path.module}/gpu-direct-workload/sample-tcpx-workload-job.yaml" + workload_path_tcpxo = "${path.module}/gpu-direct-workload/sample-tcpxo-workload-job.yaml" + + gpu_direct_settings = { + "a3-highgpu-8g" = { + # Manifest to be installed for enabling TCPX on a3-highgpu-8g machines + gpu_direct_manifests = [ + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-tcpx-installer.yaml", # nccl_plugin v3.1.9 for tcpx + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-config.yaml", # nccl_configmap + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/nri_device_injector/nri-device-injector.yaml", # nri_plugin + ] + updated_workload_path = replace(local.workload_path_tcpx, ".yaml", "-tcpx.yaml") + rxdm_version = "v2.0.12" # matching nccl-tcpx-installer version v3.1.9 + min_additional_networks = 4 + major_minor_version_acceptable_map = { + "1.27" = "1.27.7-gke.1121000" + "1.28" = "1.28.8-gke.1095000" + "1.29" = "1.29.3-gke.1093000" + "1.30" = "1.30.2-gke.1023000" + } + } + "a3-megagpu-8g" = { + # Manifest to be installed for enabling TCPXO on a3-megagpu-8g machines + gpu_direct_manifests = [ + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/gpudirect-tcpxo/nccl-tcpxo-installer.yaml", # nccl_plugin v1.0.14 for tcpxo + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/nri_device_injector/nri-device-injector.yaml", # nri_plugin + ] + updated_workload_path = replace(local.workload_path_tcpxo, ".yaml", "-tcpxo.yaml") + rxdm_version = "v1.0.20" # matching nccl-tcpxo-installer version v1.0.14 + min_additional_networks = 8 + major_minor_version_acceptable_map = { + "1.28" = "1.28.9-gke.1250000" + "1.29" = "1.29.4-gke.1542000" + "1.30" = "1.30.4-gke.1129000" + "1.31" = "1.31.1-gke.2008000" + "1.32" = "1.32.2-gke.1489001" + } + } + } + + min_additional_networks = try(local.gpu_direct_settings[var.machine_type].min_additional_networks, 0) + + gke_version_regex = "(\\d+\\.\\d+)\\.(\\d+)-gke\\.(\\d+)" # GKE version format: 1.X.Y-gke.Z , regex output: ["1.X" , "Y", "Z"] + + gke_version_parts = regex(local.gke_version_regex, var.gke_version) + gke_version_major = local.gke_version_parts[0] + + major_minor_version_acceptable_map = try(local.gpu_direct_setting[var.machine_type].major_minor_version_acceptable_map, null) + minor_version_acceptable = try(contains(keys(local.major_minor_version_acceptable_map), local.gke_version_major), false) ? local.major_minor_version_acceptable_map[local.gke_version_major] : "1.0.0-gke.0" + minor_version_acceptable_parts = regex(local.gke_version_regex, local.minor_version_acceptable) + gke_gpudirect_compatible = local.gke_version_parts[1] > local.minor_version_acceptable_parts[1] || (local.gke_version_parts[1] == local.minor_version_acceptable_parts[1] && local.gke_version_parts[2] >= local.minor_version_acceptable_parts[2]) +} + +check "gpu_direct_check_multi_vpc" { + assert { + condition = length(var.additional_networks) >= local.min_additional_networks + error_message = "To achieve optimal performance for ${var.machine_type} machine, at least ${local.min_additional_networks} additional vpc is recommended. You could configure it in the blueprint through modules/network/multivpc with network_count set as ${local.min_additional_networks}" + } +} + +check "gke_version_requirements" { + assert { + condition = local.gke_gpudirect_compatible + error_message = "GPUDirect is not supported on GKE version ${var.gke_version} for ${var.machine_type} machine. For supported version details visit https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#requirements" + } +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf new file mode 100644 index 0000000000..1ddc7ba8c3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +data "google_compute_machine_types" "machine_info" { + for_each = var.zones == null ? toset([]) : toset(var.zones) + + project = var.project_id + zone = each.key + filter = "name = \"${var.machine_type}\"" +} + +locals { + valid_machine_info = { + for zone, data in data.google_compute_machine_types.machine_info : + zone => data.machine_types if length(data.machine_types) > 0 + } + + guest_cpus = try(local.valid_machine_info[0].guest_cpus, 0) +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/main.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/main.tf new file mode 100644 index 0000000000..05314497fc --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/main.tf @@ -0,0 +1,482 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-node-pool", ghpc_role = "compute" }) +} + +locals { + upgrade_settings = { + strategy = var.upgrade_settings.strategy + max_surge = coalesce(var.upgrade_settings.max_surge, 0) + max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) + } +} + +module "gpu" { + source = "../../internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + has_gpu = length(local.guest_accelerator) > 0 + allocatable_gpu_per_node = local.has_gpu ? max(local.guest_accelerator[*].count...) : -1 + is_static_node_pool_with_gpus = var.static_node_count != null && local.allocatable_gpu_per_node != -1 + static_gpu_count = local.is_static_node_pool_with_gpus ? var.static_node_count * local.allocatable_gpu_per_node : 0 + gpu_taint = local.has_gpu ? [{ + key = "nvidia.com/gpu" + value = "present" + effect = "NO_SCHEDULE" + }] : [] + + autoscale_set = var.autoscaling_total_min_nodes != 0 || var.autoscaling_total_max_nodes != 1000 + static_node_set = var.static_node_count != null + initial_node_set = try(var.initial_node_count > 0, false) + + module_unique_id = replace(lower(var.internal_ghpc_module_id), "/[^a-z0-9\\-]/", "") +} + + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] +} + +module "tpu" { + source = "../../internal/tpu-definition" + + machine_type = var.machine_type + placement_policy = var.placement_policy +} + + +data "google_container_cluster" "gke_cluster" { + name = local.cluster_name + location = local.cluster_location +} + +resource "google_container_node_pool" "node_pool" { + provider = google-beta + + count = max(var.num_node_pools, var.num_slices) + + name = (max(var.num_node_pools, var.num_slices) == 1) ? coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])) : join("-", [coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])), count.index]) + cluster = var.cluster_id + node_locations = var.zones + + node_count = var.static_node_count + dynamic "autoscaling" { + for_each = local.static_node_set ? [] : [1] + content { + total_min_node_count = var.autoscaling_total_min_nodes + total_max_node_count = var.autoscaling_total_max_nodes + location_policy = "ANY" + } + } + + initial_node_count = var.initial_node_count + + max_pods_per_node = var.max_pods_per_node + + management { + auto_repair = var.auto_repair + auto_upgrade = var.auto_upgrade + } + + upgrade_settings { + strategy = local.upgrade_settings.strategy + max_surge = local.upgrade_settings.max_surge + max_unavailable = local.upgrade_settings.max_unavailable + } + + dynamic "placement_policy" { + for_each = var.placement_policy.type != null ? [1] : [] + content { + type = var.placement_policy.type + policy_name = var.placement_policy.name + tpu_topology = module.tpu.is_tpu ? var.placement_policy.tpu_topology : null + } + } + + dynamic "queued_provisioning" { + for_each = var.enable_queued_provisioning ? [1] : [] + content { + enabled = true + } + } + + node_config { + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + resource_labels = local.labels + labels = var.kubernetes_labels + service_account = var.service_account_email + oauth_scopes = var.service_account_scopes + machine_type = var.machine_type + spot = var.spot + image_type = var.image_type + flex_start = var.enable_flex_start + max_run_duration = var.max_run_duration != null ? "${var.max_run_duration}s" : null + + dynamic "guest_accelerator" { + for_each = local.guest_accelerator + iterator = ga + content { + type = coalesce(ga.value.type, try(local.generated_guest_accelerator[0].type, "")) + count = coalesce(try(ga.value.count, 0) > 0 ? ga.value.count : try(local.generated_guest_accelerator[0].count, "0")) + + gpu_partition_size = try(ga.value.gpu_partition_size, null) + + dynamic "gpu_driver_installation_config" { + # in case user did not specify guest_accelerator settings, we need a try to default to [] + for_each = try([ga.value.gpu_driver_installation_config], [{ gpu_driver_version = "DEFAULT" }]) + iterator = gdic + content { + gpu_driver_version = gdic.value.gpu_driver_version + } + } + + dynamic "gpu_sharing_config" { + for_each = try(ga.value.gpu_sharing_config == null, true) ? [] : [ga.value.gpu_sharing_config] + iterator = gsc + content { + gpu_sharing_strategy = gsc.value.gpu_sharing_strategy + max_shared_clients_per_gpu = gsc.value.max_shared_clients_per_gpu + } + } + } + } + + dynamic "taint" { + for_each = concat(var.taints, local.gpu_taint, module.tpu.tpu_taint) + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } + + dynamic "ephemeral_storage_local_ssd_config" { + for_each = local.local_ssd_config.local_ssd_count_ephemeral_storage != null ? [1] : [] + content { + local_ssd_count = local.local_ssd_config.local_ssd_count_ephemeral_storage + } + } + + dynamic "local_nvme_ssd_block_config" { + for_each = local.local_ssd_config.local_ssd_count_nvme_block != null ? [1] : [] + content { + local_ssd_count = local.local_ssd_config.local_ssd_count_nvme_block + } + } + + shielded_instance_config { + enable_secure_boot = var.enable_secure_boot + enable_integrity_monitoring = true + } + + dynamic "gcfs_config" { + for_each = var.enable_gcfs ? [1] : [] + content { + enabled = true + } + } + + gvnic { + enabled = var.image_type == "COS_CONTAINERD" + } + + dynamic "advanced_machine_features" { + for_each = local.set_threads_per_core ? [1] : [] + content { + threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf + } + } + + # Implied by Workload Identity + workload_metadata_config { + mode = "GKE_METADATA" + } + # Implied by workload identity. + metadata = { + "disable-legacy-endpoints" = "true" + } + + linux_node_config { + sysctls = { + "net.ipv4.tcp_rmem" = "4096 87380 16777216" + "net.ipv4.tcp_wmem" = "4096 16384 16777216" + } + } + + reservation_affinity { + consume_reservation_type = var.reservation_affinity.consume_reservation_type + key = local.is_valid_reservation ? local.reservation_resource_api_label : null + values = local.is_valid_reservation ? (var.is_reservation_active ? local.active_reservation_values : local.default_reservation_values) : null + } + + dynamic "host_maintenance_policy" { + for_each = var.host_maintenance_interval != "" ? [1] : [] + content { + maintenance_interval = var.host_maintenance_interval + } + } + + kubelet_config { + cpu_manager_policy = var.enable_numa_aware_scheduling ? "static" : null + dynamic "topology_manager" { + for_each = var.enable_numa_aware_scheduling ? [1] : [] + content { + policy = "restricted" + } + } + dynamic "memory_manager" { + for_each = var.enable_numa_aware_scheduling ? [1] : [] + content { + policy = "Static" + } + } + } + } + + network_config { + dynamic "additional_node_network_configs" { + for_each = var.additional_networks + + content { + network = additional_node_network_configs.value.network + subnetwork = additional_node_network_configs.value.subnetwork + } + } + + enable_private_nodes = var.enable_private_nodes + } + + timeouts { + create = var.timeout_create + update = var.timeout_update + } + + lifecycle { + ignore_changes = [ + node_config[0].labels, + initial_node_count, + # Ignore local/ephemeral ssd configs as they are tied to machine types. + node_config[0].ephemeral_storage_local_ssd_config, + node_config[0].local_nvme_ssd_block_config, + ] + precondition { + condition = (var.max_pods_per_node == null) || (data.google_container_cluster.gke_cluster.networking_mode == "VPC_NATIVE") + error_message = "max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." + } + precondition { + condition = !local.static_node_set || !local.autoscale_set + error_message = "static_node_count cannot be set with either autoscaling_total_min_nodes or autoscaling_total_max_nodes." + } + precondition { + condition = !local.static_node_set || !local.initial_node_set + error_message = "initial_node_count cannot be set with static_node_count." + } + precondition { + condition = !local.initial_node_set || (coalesce(var.initial_node_count, 0) >= var.autoscaling_total_min_nodes && coalesce(var.initial_node_count, 0) <= var.autoscaling_total_max_nodes) + error_message = "initial_node_count must be between autoscaling_total_min_nodes and autoscaling_total_max_nodes included." + } + precondition { + condition = !(coalesce(local.local_ssd_config.local_ssd_count_ephemeral_storage, 0) > 0 && coalesce(local.local_ssd_config.local_ssd_count_nvme_block, 0) > 0) + error_message = "Only one of local_ssd_count_ephemeral_storage or local_ssd_count_nvme_block can be set to a non-zero value." + } + precondition { + condition = ( + (var.reservation_affinity.consume_reservation_type != "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 0) || + (var.reservation_affinity.consume_reservation_type == "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 1) + ) + error_message = <<-EOT + When using NO_RESERVATION or ANY_RESERVATION as the `consume_reservation_type`, `specific_reservations` cannot be set. + On the other hand, with SPECIFIC_RESERVATION you must set `specific_reservations`. + EOT + } + precondition { + condition = ( + (local.input_specific_reservations_count == 0) || + ((length(local.verified_specific_reservations) == 1 || !var.is_reservation_active) && + length(local.specific_reservation_requirement_violations) == 0) + ) + error_message = <<-EOT + Check if your reservation is configured correctly: + - A reservation with the name must exist in the specified project and one of the specified zones + + - Its consumption type must be "specific" + %{for property in local.specific_reservation_requirement_violations} + - ${local.specific_reservation_requirement_violation_messages[property]} + %{endfor} + EOT + } + precondition { + condition = ( + (local.input_specific_reservations_count == 0) || + (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) == 0) || + (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) > 0 && try(local.input_reservation_projects[0], var.project_id) == var.project_id) + ) + error_message = "Shared extended reservations are not supported by GKE." + } + precondition { + condition = contains(["SURGE"], local.upgrade_settings.strategy) + error_message = "Only SURGE strategy is supported" + } + precondition { + condition = local.upgrade_settings.max_unavailable >= 0 + error_message = "max_unavailable should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_surge >= 0 + error_message = "max_surge should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 + error_message = "At least one of max_unavailable or max_surge must greater than 0" + } + precondition { + condition = var.placement_policy.type != "COMPACT" || (var.zones != null ? (length(var.zones) == 1) : false) + error_message = "Compact placement is only available for node pools operating in a single zone." + } + precondition { + condition = var.placement_policy.type != "COMPACT" || local.upgrade_settings.strategy != "BLUE_GREEN" + error_message = "Compact placement is not supported with blue-green upgrades." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.placement_policy.type == "COMPACT") + error_message = "placement_policy cannot be COMPACT when enable_queued_provisioning is true." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.reservation_affinity.consume_reservation_type != "NO_RESERVATION") + error_message = "reservation_affinity should be NO_RESERVATION when enable_queued_provisioning is true." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.autoscaling_total_min_nodes != 0) + error_message = "autoscaling_total_min_nodes should be 0 when enable_queued_provisioning is true." + } + precondition { + condition = !(var.num_node_pools > 1 && var.num_slices > 1) + error_message = "num_node_pools is for CPUs and GPUS, and num_slices is for TPUs. Both cannot be set at the same time to create a group of identical nodepools / slices." + } + precondition { + condition = !(var.num_node_pools == 0 && var.num_slices == 0) + error_message = "Either num_node_pools (for CPUs and GPUS) or num_slices (for TPUs) should be set to a positive integer value." + } + precondition { + condition = !(var.num_node_pools < 0 || var.num_slices < 0) + error_message = "Negative integer value of num_node_pools or num_slices is not valid. Please use a positive integer value to set num_node_pools for CPUs and GPUS, and num_slices for TPUs." + } + precondition { + condition = var.enable_flex_start == true ? (var.auto_repair == false) : true + error_message = "enable_flex_start needs node auto_repair set to false." + } + precondition { + condition = var.enable_flex_start == true ? (var.static_node_count == null) : true + error_message = "enable_flex_start does not work with static_node_count. static_node_count should be set to null." + } + precondition { + condition = var.enable_flex_start == true ? (var.reservation_affinity.consume_reservation_type == "NO_RESERVATION") : true + error_message = "enable_flex_start only works with reservation_affinity consume_reservation_type NO_RESERVATION." + } + precondition { + condition = var.enable_flex_start == true ? (var.spot == false) : true + error_message = "Both enable_flex_start and spot consumption option cannot be set to true at the same time." + } + } +} + +locals { + supported_machine_types_for_install_dependencies = ["a3-highgpu-8g", "a3-megagpu-8g"] +} + +# Replicates GKE's naming logic for its instance templates. The full +# pattern is "gke-{cluster_name}-{nodepool_name}-{hash}". +# +# This code builds the "{cluster_name}-{nodepool_name}" prefix, which is +# capped at 32 characters plus a dash '-' in between, by truncating names if needed: +# - If both names > 16 chars, both are cut to 16. +# - If one name > 16, it's shortened so the combined name length is 32. +data "google_compute_region_instance_template" "instance_template" { + for_each = { for idx, np in google_container_node_pool.node_pool : idx => np } + project = var.project_id + filter = "name: gke-${ + (length(local.cluster_name) <= 16 && length(each.value.name) <= 16) ? "${local.cluster_name}-${each.value.name}" : + (length(local.cluster_name) > 16 && length(each.value.name) > 16) ? "${substr(local.cluster_name, 0, 16)}-${substr(each.value.name, 0, 16)}" : + (length(local.cluster_name) > 16) ? "${substr(local.cluster_name, 0, 32 - length(each.value.name))}-${each.value.name}" : + "${local.cluster_name}-${substr(each.value.name, 0, 32 - length(local.cluster_name))}" + }*" + most_recent = true +} + +resource "null_resource" "install_dependencies" { + count = var.run_workload_script && contains(local.supported_machine_types_for_install_dependencies, var.machine_type) ? 1 : 0 + provisioner "local-exec" { + command = "pip3 install pyyaml" + } +} + +locals { + gpu_direct_setting = lookup(local.gpu_direct_settings, var.machine_type, { gpu_direct_manifests = [], updated_workload_path = "", rxdm_version = "" }) +} + +# execute script to inject rxdm sidecar into workload to enable tcpx for a3-highgpu-8g VM workload +resource "null_resource" "enable_tcpx_in_workload" { + count = var.run_workload_script && var.machine_type == "a3-highgpu-8g" ? 1 : 0 + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpx-in-workload.py --file ${local.workload_path_tcpx} --rxdm ${local.gpu_direct_setting.rxdm_version}" + } + + depends_on = [null_resource.install_dependencies] +} + +# execute script to inject rxdm sidecar into workload to enable tcpxo for a3-megagpu-8g VM workload +resource "null_resource" "enable_tcpxo_in_workload" { + count = var.run_workload_script && var.machine_type == "a3-megagpu-8g" ? 1 : 0 + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py --file ${local.workload_path_tcpxo} --rxdm ${local.gpu_direct_setting.rxdm_version}" + } + + depends_on = [null_resource.install_dependencies] +} + +# apply manifest to enable tcpx +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + apply_manifests = flatten([ + for manifest in local.gpu_direct_setting.gpu_direct_manifests : [ + { + source = manifest + } + ] + ]) +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml new file mode 100644 index 0000000000..e980d595a2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com +ghpc: + inject_module_id: internal_ghpc_module_id diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf new file mode 100644 index 0000000000..44e1c3d971 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf @@ -0,0 +1,152 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "node_pool_names" { + description = "Names of the node pools." + value = google_container_node_pool.node_pool[*].name +} + +locals { + # Shared core machines only have 1 cpu allocatable, even if they have 2 cpu capacity + vcpu = local.machine_shared_core ? 1 : local.guest_cpus + useable_cpu = local.set_threads_per_core ? local.threads_per_core * local.vcpu / 2 : local.vcpu + + # allocatable resource definition: https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes#cpu_reservations + second_core = local.useable_cpu > 1 ? 1 : 0 + third_fourth_core = local.useable_cpu == 3 ? 1 : local.useable_cpu > 3 ? 2 : 0 + cores_above_four = local.useable_cpu > 4 ? local.useable_cpu - 4 : 0 + + allocatable_cpu = 0.94 + (0.99 * local.second_core) + (0.995 * local.third_fourth_core) + (0.9975 * local.cores_above_four) +} + +output "allocatable_cpu_per_node" { + description = "Number of CPUs available for scheduling pods on each node." + value = local.allocatable_cpu +} + +output "has_gpu" { + description = "Boolean value indicating whether nodes in the pool are configured with GPUs." + value = local.has_gpu +} + +output "allocatable_gpu_per_node" { + description = "Number of GPUs available for scheduling pods on each node." + value = local.allocatable_gpu_per_node +} + +output "static_gpu_count" { + description = "Total number of GPUs in the node pool. Available only for static node pools." + value = local.static_gpu_count +} + +locals { + translate_toleration = { + PREFER_NO_SCHEDULE = "PreferNoSchedule" + NO_SCHEDULE = "NoSchedule" + NO_EXECUTE = "NoExecute" + } + taints = google_container_node_pool.node_pool[0].node_config[0].taint + tolerations = [for taint in local.taints : { + key = taint.key + operator = "Equal" + value = taint.value + effect = lookup(local.translate_toleration, taint.effect, null) + }] +} + +output "tolerations" { + description = "Tolerations needed for a pod to be scheduled on this node pool." + value = local.tolerations +} + +locals { + gpu_direct_enabled = var.machine_type == "a3-highgpu-8g" || var.machine_type == "a3-megagpu-8g" + script_path = { + a3-highgpu-8g = "enable-tcpx-in-workload.py", + a3-megagpu-8g = "enable-tcpxo-in-workload.py" + } + nccl_path = var.machine_type == "a3-highgpu-8g" ? "configs" : "scripts" + gpu_direct_instruction = <<-EOT + Since you are using ${var.machine_type} machine type that has GPUDirect support, your nodepool had been configured with the required plugins. + To fully utilize GPUDirect you will need to add some components into your workload manifest. Details below: + + A sample GKE job that has GPUDirect enabled and NCCL test included has been generated locally at: + ${abspath(local.gpu_direct_setting.updated_workload_path)} + + You can use the following commands to submit the sample job: + kubectl create -f ${abspath(local.gpu_direct_setting.updated_workload_path)} + After submitting the sample job, you can validate the GPU performance by initiating NCCL test included in the sample workload: + NCCL test can be initiated from any one of the sample job Pods and coordinate with the peer Pods: + export POD_NAME=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | head -n 1) + export PEER_POD_IPS=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.status.podIP}}{{" "}}{{end}}') + kubectl exec --stdin --tty --container=nccl-test $POD_NAME -- /${local.nccl_path}/allgather.sh $PEER_POD_IPS + + If you would like to enable GPUDirect for your own workload, please follow the below steps: + export WORKLOAD_PATH=<> + python3 ${abspath("${path.module}/gpu-direct-workload/scripts/${lookup(local.script_path, var.machine_type, "")}")} --file $WORKLOAD_PATH --rxdm ${local.gpu_direct_setting.rxdm_version} + **WARNING** + The "--rxdm" version is tied to the nccl-tcpx/o-installer that had been deployed to your cluster, changing it to other value might have impact on performance + **WARNING** + + Or you can also follow our GPUDirect user guide to update your workload + https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#add-gpudirect-manifests + EOT +} + +output "instructions" { + description = "Instructions for submitting the sample GPUDirect enabled job." + value = local.gpu_direct_enabled ? local.gpu_direct_instruction : null +} + +output "node_count_static" { + description = "The number of static nodes in node-pool." + value = coalesce(var.static_node_count, var.initial_node_count, 0) +} + +output "guest_accelerator" { + description = "The accelerator type of the nodes." + value = local.guest_accelerator +} + +output "cluster_id" { + description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." + value = var.cluster_id +} + +output "machine_type" { + description = "Machine Type" + value = var.machine_type +} + +output "instance_templates" { + description = "The URLs of Instance Templates" + value = [for key, template in data.google_compute_region_instance_template.instance_template : template.self_link] +} + +output "tpu_accelerator_type" { + description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." + value = module.tpu.is_tpu ? module.tpu.tpu_accelerator_type : null +} + +output "tpu_topology" { + description = "The topology of the TPU slice (e.g., '4x4')." + value = module.tpu.is_tpu ? module.tpu.tpu_topology : null +} + +output "tpu_chips_per_node" { + description = "The number of TPU chips on each node in the pool." + value = module.tpu.is_tpu ? module.tpu.tpu_chips_per_node : null +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf new file mode 100644 index 0000000000..7c29e3902a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf @@ -0,0 +1,107 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# Split the input into three different lists where the details of a given reservation are at the same index across these lists. +locals { + # Specific block of an extended reservation can be targeted with exr-one/reservationBlocks/exr-one-block-1 + # Data source needs to be queried with the reservation name only. So, we extract the reservation name + input_reservation_names = [for r in try(var.reservation_affinity.specific_reservations, []) : split("/", r.name)[0]] + input_reservation_projects = [for r in try(var.reservation_affinity.specific_reservations, []) : coalesce(r.project, var.project_id)] + # We, also, remember the suffix "/reservationBlocks/exr-one-block-1" for use elsewhere afterwards + input_reservation_suffixes = [for r in try(var.reservation_affinity.specific_reservations, []) : substr(r.name, length(split("/", r.name)[0]), -1)] + # Adding this variable to by-pass the machine-type validation for TPUs + is_tpu = var.placement_policy.tpu_topology != null +} + +data "google_compute_reservation" "specific_reservations" { + for_each = ( + local.input_specific_reservations_count == 0 ? + {} : + { + for pair in flatten([ + for zone in try(var.zones, []) : [ + for i, reservation_name in try(local.input_reservation_names, []) : { + key : "${local.input_reservation_projects[i]}/${zone}/${reservation_name}" + zone : zone + reservation_name : reservation_name + project : local.input_reservation_projects[i] + } + ] + ]) : + pair.key => pair + } + ) + name = each.value.reservation_name + zone = each.value.zone + project = each.value.project +} + +locals { + generated_guest_accelerator = module.gpu.machine_type_guest_accelerator + reservation_resource_api_label = "compute.googleapis.com/reservation-name" + input_specific_reservations_count = try(length(var.reservation_affinity.specific_reservations), 0) + + # Filter specific reservations + verified_specific_reservations = [for k, v in data.google_compute_reservation.specific_reservations : v if(v.specific_reservation != null && v.specific_reservation_required == true)] + + # Build two maps to be used to compare the VM properties between reservations and the node pool + # Validation of only machine-type for CPUs and and both machine-type and guest-accelerators for GPUs + # Skip this for TPUs ( returns an empty list to skip the machine-type validation for aggregate TPU reservations) + reservation_vm_properties = local.is_tpu ? [] : [for reservation in local.verified_specific_reservations : { + "machine_type" : try(reservation.specific_reservation[0].instance_properties[0].machine_type, "") + "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs + { for acc in try(reservation.specific_reservation[0].instance_properties[0].guest_accelerators, []) : acc.accelerator_type => acc.accelerator_count } + ) : {} # If no GPUs, it's an empty map {} + }] + + nodepool_vm_properties = { + "machine_type" : var.machine_type + "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs + { for acc in try(local.guest_accelerator, []) : coalesce(acc.type, try(local.generated_guest_accelerator[0].type, "")) => coalesce(acc.count, try(local.generated_guest_accelerator[0].count, 0)) } + ) : {} # If no GPUs, it's an empty map {} + } + + # Compare two maps by counting the keys that mismatch. + # Know that in map comparison the order of keys does not matter. That is {NVME: x, SCSI: y} and {SCSI: y, NVME: x} are equal + # As of this writing, there is only one reservation supported by the Node Pool API. So, directly accessing it from the list + specific_reservation_requirement_violations = length(local.reservation_vm_properties) == 0 ? [] : [for k, v in local.nodepool_vm_properties : k if v != local.reservation_vm_properties[0][k]] + + specific_reservation_requirement_violation_messages = { + "machine_type" : <<-EOT + The reservation has "${try(local.reservation_vm_properties[0].machine_type, "")}" machine type and the node pool has "${local.nodepool_vm_properties.machine_type}". Check the relevant node pool setting: "machine_type" + EOT + "guest_accelerators" : <<-EOT + The reservation has ${jsonencode(try(local.reservation_vm_properties[0].guest_accelerators, {}))} accelerators and the node pool has ${jsonencode(try(local.nodepool_vm_properties.guest_accelerators, {}))}. Check the relevant node pool setting: "guest_accelerator". When unspecified, for the machine_type=${var.machine_type}, the default is guest_accelerator=${jsonencode(try(local.generated_guest_accelerator, [{}]))}. + EOT + } +} + +locals { + # Check if reservation is valid, that is, if it exists, there should be only 1 verified specific reservation or the reservation doesn't exist + is_valid_reservation = length(local.verified_specific_reservations) == 1 || !var.is_reservation_active + + # Build the list of reservation names when var.is_reservation_active is true + active_reservation_values = [ + for i, r in local.verified_specific_reservations : + length(local.input_reservation_suffixes[i]) > 0 ? + format("%s%s", r.name, local.input_reservation_suffixes[i]) : + "projects/${r.project}/reservations/${r.name}" + ] + + # Define a default reservation value if no specific reservations are present + specific_reservation_name = length(local.input_reservation_names) > 0 ? local.input_reservation_names[0] : "" + default_reservation_values = ["projects/${var.project_id}/reservations/${local.specific_reservation_name}"] +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf new file mode 100644 index 0000000000..e582db33da --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' + +# "inputs": +# var.machine_type: Machine type for the instance being evaluated. +# var.threads_per_core : Sets the number of threads per physical core, where 0 +# has behavior described in description. + +# "outputs": +# local.set_threads_per_core: bool that tells if threads per core should be set, +# to be used with a dynamic block. +# local.threads_per_core: actual threads_per_core to be used. + +locals { + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + machine_shared_core = length(local.machine_vals) <= 2 + machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) + + smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) + smt_capable_vcpu = local.machine_vcpus >= 2 + + smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core + set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) + threads_per_core = var.threads_per_core == 2 ? 2 : 1 +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf new file mode 100644 index 0000000000..b44ea28d57 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf @@ -0,0 +1,487 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" + type = string +} + +variable "zones" { + description = "A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable." + type = list(string) + default = null +} + +variable "name" { + description = <<-EOD + The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix. + If setting manually, ensure a unique value across all gke-node-pools. + EOD + type = string + default = null + + validation { + # Check if the variable is null OR if it matches the GCP resource naming regex. + condition = var.name == null || can(regex("^[a-z]([-a-z0-9]{0,34}[a-z0-9])?$", var.name)) + error_message = <<-EOD + If provided, the node pool name must be between 1 and 36 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. + Underscores are not allowed. A shorter length is enforced to accommodate a suffix when creating multiple node pools. + EOD + } +} + +variable "internal_ghpc_module_id" { + description = "DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide)." + type = string +} + +variable "machine_type" { + description = "The name of a Google Compute Engine machine type." + type = string + default = "c2-standard-60" +} + +variable "disk_size_gb" { + description = "Size of disk for each node." + type = number + default = 100 +} + +variable "disk_type" { + description = "Disk type for each node." + type = string + default = null +} + +variable "enable_gcfs" { + description = "Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config)." + type = bool + default = false +} + +variable "enable_secure_boot" { + description = "Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info." + type = bool + default = true +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = optional(string) + count = optional(number, 0) + gpu_driver_installation_config = optional(object({ + gpu_driver_version = string + }), { gpu_driver_version = "DEFAULT" }) + gpu_partition_size = optional(string) + gpu_sharing_config = optional(object({ + gpu_sharing_strategy = string + max_shared_clients_per_gpu = number + })) + })) + default = [] + nullable = false + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.count != null]) + error_message = "var.guest_accelerator[*].count cannot be null" + } + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.count >= 0]) + error_message = "var.guest_accelerator[*].count must never be negative" + } + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.gpu_driver_installation_config != null]) + error_message = "var.guest_accelerator[*].gpu_driver_installation_config must not be null; leave unset to enable GKE to select default GPU driver installation" + } +} + +variable "image_type" { + description = "The default image type used by NAP once a new node pool is being created. Use either COS_CONTAINERD or UBUNTU_CONTAINERD." + type = string + default = "COS_CONTAINERD" +} + +variable "local_ssd_count_ephemeral_storage" { + description = <<-EOT + The number of local SSDs to attach to each node to back ephemeral storage. + Uses NVMe interfaces. Must be supported by `machine_type`. + When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. + [See above](#local-ssd-storage) for more info. + EOT + type = number + default = null +} + +variable "local_ssd_count_nvme_block" { + description = <<-EOT + The number of local SSDs to attach to each node to back block storage. + Uses NVMe interfaces. Must be supported by `machine_type`. + When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. + [See above](#local-ssd-storage) for more info. + + EOT + type = number + default = null +} + +variable "autoscaling_total_min_nodes" { + description = "Total minimum number of nodes in the NodePool." + type = number + default = 0 +} + +variable "autoscaling_total_max_nodes" { + description = "Total maximum number of nodes in the NodePool." + type = number + default = 1000 +} + +variable "static_node_count" { + description = "The static number of nodes in the node pool. If set, autoscaling will be disabled." + type = number + default = null +} + +variable "is_reservation_active" { + description = "Whether the specified reservation is already created." + type = bool + default = true +} + +variable "auto_repair" { + description = "Whether the nodes will be automatically repaired." + type = bool + default = true +} + +variable "auto_upgrade" { + description = "Whether the nodes will be automatically upgraded." + type = bool + default = false +} + +variable "threads_per_core" { + description = <<-EOT + Sets the number of threads per physical core. By setting threads_per_core + to 2, Simultaneous Multithreading (SMT) is enabled extending the total number + of virtual cores. For example, a machine of type c2-standard-60 will have 60 + virtual cores with threads_per_core equal to 2. With threads_per_core equal + to 1 (SMT turned off), only the 30 physical cores will be available on the VM. + + The default value of \"0\" will turn off SMT for supported machine types, and + will fall back to GCE defaults for unsupported machine types (t2d, shared-core + instances, or instances with less than 2 vCPU). + + Disabling SMT can be more performant in many HPC workloads, therefore it is + disabled by default where compatible. + + null = SMT configuration will use the GCE defaults for the machine type + 0 = SMT will be disabled where compatible (default) + 1 = SMT will always be disabled (will fail on incompatible machine types) + 2 = SMT will always be enabled (will fail on incompatible machine types) + EOT + type = number + default = 0 + + validation { + condition = var.threads_per_core == null || try(var.threads_per_core >= 0, false) && try(var.threads_per_core <= 2, false) + error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." + } +} + +variable "spot" { + description = "Provision VMs using discounted Spot pricing, allowing for preemption" + type = bool + default = false +} + +# tflint-ignore: terraform_unused_declarations +variable "compact_placement" { + description = "DEPRECATED: Use `placement_policy`" + type = bool + default = null + validation { + condition = var.compact_placement == null + error_message = "`compact_placement` is deprecated. Use `placement_policy` instead" + } +} + +variable "placement_policy" { + description = <<-EOT + Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool. + It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + EOT + + type = object({ + type = string + name = optional(string) + tpu_topology = optional(string) + }) + default = { + type = null + name = null + tpu_topology = null + } + validation { + condition = var.placement_policy.type == null || try(contains(["COMPACT"], var.placement_policy.type), false) + error_message = "`COMPACT` is the only supported value for `placement_policy.type`." + } +} + +variable "service_account_email" { + description = "Service account e-mail address to use with the node pool" + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to to use with the node pool." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "taints" { + description = "Taints to be applied to the system node pool." + type = list(object({ + key = string + value = any + effect = string + })) + default = [] +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} + +variable "kubernetes_labels" { + description = <<-EOT + Kubernetes labels to be applied to each node in the node group. Key-value pairs. + (The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) + EOT + type = map(string) + default = null +} + +variable "timeout_create" { + description = "Timeout for creating a node pool" + type = string + default = null +} + +variable "timeout_update" { + description = "Timeout for updating a node pool" + type = string + default = null +} + +# Deprecated + +# tflint-ignore: terraform_unused_declarations +variable "total_min_nodes" { + description = "DEPRECATED: Use autoscaling_total_min_nodes." + type = number + default = null + validation { + condition = var.total_min_nodes == null + error_message = "total_min_nodes was renamed to autoscaling_total_min_nodes and is deprecated; use autoscaling_total_min_nodes" + } +} + +# tflint-ignore: terraform_unused_declarations +variable "total_max_nodes" { + description = "DEPRECATED: Use autoscaling_total_max_nodes." + type = number + default = null + validation { + condition = var.total_max_nodes == null + error_message = "total_max_nodes was renamed to autoscaling_total_max_nodes and is deprecated; use autoscaling_total_max_nodes" + } +} + +# tflint-ignore: terraform_unused_declarations +variable "service_account" { + description = "DEPRECATED: use service_account_email and scopes." + type = object({ + email = string, + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "service_account is deprecated and replaced with service_account_email and scopes." + } +} + +variable "additional_networks" { + description = "Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool" + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + })) + nullable = false +} + +variable "reservation_affinity" { + description = <<-EOT + Reservation resource to consume. When targeting SPECIFIC_RESERVATION, specific_reservations needs be specified. + Even though specific_reservations is a list, only one reservation is allowed by the NodePool API. + It is assumed that the specified reservation exists and has available capacity. + For a shared reservation, specify the project_id as well in which it was created. + To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared + EOT + type = object({ + consume_reservation_type = string + specific_reservations = optional(list(object({ + name = string + project = optional(string) + }))) + }) + default = { + consume_reservation_type = "NO_RESERVATION" + specific_reservations = [] + } + validation { + condition = contains(["NO_RESERVATION", "ANY_RESERVATION", "SPECIFIC_RESERVATION"], var.reservation_affinity.consume_reservation_type) + error_message = "Accepted values are: {NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION}" + } +} + +variable "host_maintenance_interval" { + description = "Specifies the frequency of planned maintenance events." + type = string + default = "" + nullable = false + validation { + condition = contains(["", "PERIODIC", "AS_NEEDED"], var.host_maintenance_interval) + error_message = "Invalid host_maintenance_interval value. Must be PERIODIC, AS_NEEDED or the empty string" + } +} + +variable "initial_node_count" { + description = "The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static_node_count and must be set to a value between autoscaling_total_min_nodes and autoscaling_total_max_nodes." + type = number + default = null +} + +variable "gke_version" { + description = "GKE version" + type = string +} + +variable "max_pods_per_node" { + description = "The maximum number of pods per node in this node pool. This will force replacement." + type = number + default = null +} + +variable "upgrade_settings" { + description = <<-EOT + Defines node pool upgrade settings. It is highly recommended that you define all max_surge and max_unavailable. + If max_surge is not specified, it would be set to a default value of 0. + If max_unavailable is not specified, it would be set to a default value of 1. + EOT + type = object({ + strategy = string + max_surge = optional(number) + max_unavailable = optional(number) + }) + default = { + strategy = "SURGE" + max_surge = 0 + max_unavailable = 1 + } +} + +variable "run_workload_script" { + description = "Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only." + type = bool + default = true +} + +variable "enable_queued_provisioning" { + description = "If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool." + type = bool + default = false +} + +variable "enable_flex_start" { + description = <<-EOT + If true, start the node pool with Flex Start provisioning model. + To learn more about flex-start mode, please refer to + https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and + https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest + EOT + type = bool + default = false +} + +variable "max_run_duration" { + description = "The duration (in whole seconds) of the instance. Instance will run and be terminated after then." + type = number + default = null +} + +variable "enable_private_nodes" { + description = "Whether nodes have internal IP addresses only." + type = bool + default = true +} + +variable "num_node_pools" { + description = "Number of node pools to create. This is same as num_slices." + type = number + default = 1 +} + +variable "num_slices" { + description = "Number of TPUs slices to create. This is same as num_node_pools." + type = number + default = 1 +} + +variable "enable_numa_aware_scheduling" { + description = "Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling." + type = bool + default = false +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf new file mode 100644 index 0000000000..f018d04fc5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf @@ -0,0 +1,38 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 7.2" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 7.2" + } + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" + } +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/README.md b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/README.md new file mode 100644 index 0000000000..3b769e8761 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/README.md @@ -0,0 +1,82 @@ +## Description + +This modules create a [resource policy for compute engines](https://cloud.google.com/compute/docs/instances/placement-policies-overview). This policy can be passed to a gke-node-pool module to apply the policy on the node-pool's nodes. + +Note: By default, you can't apply compact placement policies with a max distance value to A3 VMs. To request access to this feature, contact your [Technical Account Manager (TAM)](https://cloud.google.com/tam) or the [Sales team](https://cloud.google.com/contact). + +### Example + +The following example creates a group placement resource policy and applies it to a gke-node-pool. + +```yaml + - id: group_placement_1 + source: modules/compute/resource-policy + settings: + name: gp-np-1 + group_placement_max_distance: 2 + + - id: node_pool_1 + source: modules/compute/gke-node-pool + use: [group_placement_1] + settings: + machine_type: e2-standard-8 + outputs: [instructions] +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google-beta](#requirement\_google-beta) | >= 6.29.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google-beta](#provider\_google-beta) | >= 6.29.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_resource_policy.policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [group\_placement\_max\_distance](#input\_group\_placement\_max\_distance) | The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | `number` | `0` | no | +| [name](#input\_name) | The resource policy's name. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The project ID for the resource policy. | `string` | n/a | yes | +| [region](#input\_region) | The region for the the resource policy. | `string` | n/a | yes | +| [workload\_policy](#input\_workload\_policy) | Describes the workload policy | 
object({
    type                  = optional(string, null)
    max_topology_distance = optional(string, null)
    accelerator_topology  = optional(string, null)
  })
| 
{
  "accelerator_topology": null,
  "max_topology_distance": null,
  "type": null
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [placement\_policy](#output\_placement\_policy) | Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions.
The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. | + diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/main.tf b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/main.tf new file mode 100644 index 0000000000..906424ca7c --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + name = "${var.name}-${random_id.resource_name_suffix.hex}" +} + +resource "google_compute_resource_policy" "policy" { + name = local.name + region = var.region + project = var.project_id + provider = google-beta + + dynamic "workload_policy" { + for_each = var.workload_policy.type != null ? [1] : [] + + content { + type = var.workload_policy.type + max_topology_distance = var.workload_policy.max_topology_distance + accelerator_topology = var.workload_policy.accelerator_topology + } + } + + dynamic "group_placement_policy" { + for_each = var.group_placement_max_distance > 0 ? [1] : [] + + content { + collocation = "COLLOCATED" + max_distance = var.group_placement_max_distance + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/outputs.tf b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/outputs.tf new file mode 100644 index 0000000000..c1dc65bcbb --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "placement_policy" { + description = <<-EOT + Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. + It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. + EOT + + value = { + type = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? "COMPACT" : null + name = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? local.name : null + tpu_topology = (var.workload_policy.type != null) ? var.workload_policy.accelerator_topology : null + } +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/variables.tf b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/variables.tf new file mode 100644 index 0000000000..92434326ca --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/variables.tf @@ -0,0 +1,64 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID for the resource policy." + type = string +} + +variable "region" { + description = "The region for the the resource policy." + type = string +} + +variable "name" { + description = "The resource policy's name." + type = string + + validation { + # Check if the variable matches the GCP resource naming regex. + condition = can(regex("^[a-z]([-a-z0-9]{0,52}[a-z0-9])?$", var.name)) + error_message = <<-EOD + The resource policy name must be between 1 and 54 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. + Underscores are not allowed. A shorter length is enforced to accommodate a random suffix. + EOD + } +} + +variable "group_placement_max_distance" { + description = <<-EOT + The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + EOT + + type = number + default = 0 +} + +variable "workload_policy" { + description = "Describes the workload policy" + type = object({ + type = optional(string, null) + max_topology_distance = optional(string, null) + accelerator_topology = optional(string, null) + }) + default = { + type = null + max_topology_distance = null + accelerator_topology = null + } + nullable = false +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/versions.tf b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/versions.tf new file mode 100644 index 0000000000..f235fbade3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:resource-policy/v1.37.2" + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/README.md b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/README.md new file mode 100644 index 0000000000..0c4737e0d9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/README.md @@ -0,0 +1,257 @@ +## Description + +This module creates one or more +[compute VM instances](https://cloud.google.com/compute/docs/instances). + +### Example + +```yaml +- id: compute + source: modules/compute/vm-instance + use: [network1] + settings: + instance_count: 8 + name_prefix: compute + machine_type: c2-standard-60 +``` + +This creates a cluster of 8 compute VMs that are: + +* named `compute-[0-7]` +* on the network defined by the `network1` module +* of type c2-standard-60 + +> **_NOTE:_** Simultaneous Multithreading (SMT) is deactivated by default +> (threads_per_core=1), which means only the physical cores are visible on the +> VM. With SMT disabled, a machine of type c2-standard-60 will only have the 30 +> physical cores visible. To change this, set `threads_per_core=2` under +> settings. + +### VPC Networks + +There are two methods for adding network connectivity to the `vm-instance` +module. The first is shown in the example above, where a `vpc` module or +`pre-existing-vpc` module is used by the `vm-instance` module. When this +happens, the `network_self_link` and `subnetwork_self_link` outputs from the +network are provided as input to the `vm-instance` and a network interface is +defined based on that. This can also be done updating the `network_self_link` and +`subnetwork_self_link` settings directly. + +The alternative option can be used when more than one network needs to be added +to the `vm-instance` or further customization is needed beyond what is provided +via other variables. For this option, the `network_interfaces` variable can be +used to set up one or more network interfaces on the VM instance. The format is +consistent with the terraform `google_compute_instance` `network_interface` +block, and more information can be found in the +[terraform docs][network-interface-tf]. + +> **_NOTE:_** When supplying the `network_interfaces` variable, networks +> associated with the `vm-instance` via use will be ignored in favor of the +> networks added in `network_interfaces`. In addition, `bandwidth_tier` and +> `disable_public_ips` will not apply to networks defined in +> `network_interfaces`. + +[network-interface-tf]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + +### SSH key metadata + +This module will ignore all changes to the `ssh-keys` metadata field that are +typically set by [external Google Cloud tools that automate SSH access][gcpssh] +when not using OS Login. For example, clicking on the Google Cloud Console SSH +button next to VMs in the VM Instances list will temporarily modify VM metadata +to include a dynamically-generated SSH public key. + +[gcpssh]: https://cloud.google.com/compute/docs/connect/add-ssh-keys#metadata + +### Placement + +The `placement_policy` variable can be used to control where your VM instances +are physically located relative to each other within a zone. See the official +placement [guide][guide-link] and [api][api-link] documentation. + +[guide-link]: https://cloud.google.com/compute/docs/instances/define-instance-placement +[api-link]: https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement + +Use the following settings for compact placement: + +```yaml + ... + settings: + instance_count: 4 + machine_type: c2-standard-60 + placement_policy: + collocation: "COLLOCATED" +``` + +By default the above placement policy will always result in the most compact set +of VMs available. If you would like that provisioning failed if some level of +compactness is not obtainable, you can enforce this with the [`max_distance` +setting](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies): + +```yaml + ... + settings: + instance_count: 4 + machine_type: c2-standard-60 + placement_policy: + collocation: "COLLOCATED" + max_distance: 1 +``` + +Use the following settings for spread placement: + +```yaml + ... + settings: + instance_count: 4 + machine_type: n2-standard-4 + placement_policy: + availability_domain_count: 2 +``` + +When `vm_count` is not set, as shown in the examples above, then the VMs will be +added to the placement policy incrementally. This is the **recommended way** to +use placement policies. + +If `vm_count` is specified then VMs will stay in pending state until the +specified number of VMs are created. See the warning below if using this field. + +> [!WARNING] +> When creating a compact placement using `vm_count` with more than 10 VMs, you +> must add `-parallelism=` argument on apply. For example if you have 15 VMs +> in a placement group: `terraform apply -parallelism=15`. This is because +> terraform self limits to 10 parallel requests by default but the create +> instance requests will not succeed until all VMs in the placement group have +> been requested, forming a deadlock. + +### GPU Support + +More information on GPU support in `vm-instance` and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../docs/gpu-support.md) + +## Lifecycle + +The `vm-instance` module will be replaced when the `instance_image` variable is +changed and `terraform apply` is run on the deployment group folder or +`gcluster deploy` is run. However, it will not be automatically replaced if a new +image is created in a family. + +To selectively replace the vm-instance(s), consider running terraform +`apply -replace` such as: + +> See https://developer.hashicorp.com/terraform/cli/commands/plan#replace-address for precise syntax terraform apply -replace=ADDRESS + +```shell +terraform state list +# search for the module ID and resource +terraform apply -replace="address" +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 4.73.0 | +| [google-beta](#requirement\_google-beta) | >= 6.13.0 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.73.0 | +| [google-beta](#provider\_google-beta) | >= 6.13.0 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | +| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance) | resource | +| [google-beta_google_compute_resource_policy.placement_policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | +| [google_compute_address.compute_ip](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_disk.additional_disks](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.replace_vm_trigger_from_placement](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | +| [additional\_persistent\_disks](#input\_additional\_persistent\_disks) | Configurations of additional disks to be included on the partition nodes. | 
object({
    count = optional(number, 0)
    type  = optional(string, "pd-balanced")
    size  = optional(number, 200)
  })
| `{}` | no | +| [allocate\_ip](#input\_allocate\_ip) | If not null, allocate IPs with the given configuration. See details at
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address | 
object({
    address_type = optional(string, "INTERNAL")
    purpose      = optional(string),
    network_tier = optional(string),
    ip_version   = optional(string, "IPV4"),
  })
| `null` | no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | +| [automatic\_restart](#input\_automatic\_restart) | Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user). | `bool` | `null` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will optionally be used name resources according to `name_prefix` | `string` | n/a | yes | +| [disable\_public\_ips](#input\_disable\_public\_ips) | If set to true, instances will not have public IPs | `bool` | `false` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | +| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-standard"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | +| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_count](#input\_local\_ssd\_count) | The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd. | `number` | `0` | no | +| [local\_ssd\_interface](#input\_local\_ssd\_interface) | Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set. | `string` | `"NVME"` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface

**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply
to network interfaces defined in this variable.

Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "MRDMA", "IRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. Can use "default" for the default network. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `null` | no | +| [placement\_policy](#input\_placement\_policy) | Control where your VM instances are physically located relative to each other within a zone.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy#nested_group_placement_policy | `any` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [provisioning\_model](#input\_provisioning\_model) | Provisioning model for cloud instance. | `string` | `null` | no | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME
- RESERVATION\_NAME

Must be a "SPECIFIC\_RESERVATION"
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | +| [service\_account](#input\_service\_account) | DEPRECATED - Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [spot](#input\_spot) | DEPRECATED - Use `provisioning_model` instead. | `bool` | `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | +| [zone](#input\_zone) | Compute Platform zone | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [external\_ip](#output\_external\_ip) | External IP of the instances (if enabled) | +| [instructions](#output\_instructions) | Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions. | +| [internal\_ip](#output\_internal\_ip) | Internal IP of the instances | +| [name](#output\_name) | Names of instances created | +| [self\_link](#output\_self\_link) | The tuple URIs of the created instances | + diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/main.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/main.tf new file mode 100644 index 0000000000..0a8c7d354e --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/main.tf @@ -0,0 +1,334 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "vm-instance", ghpc_role = "compute" }) +} + +module "gpu" { + source = "../../internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + native_fstype = [] + startup_script = local.startup_from_network_storage != null ? ( + { startup-script = local.startup_from_network_storage }) : {} + network_storage = var.network_storage != null ? ( + { network_storage = jsonencode(var.network_storage) }) : {} + + prefix_optional_deployment_name = var.name_prefix != null ? var.name_prefix : var.deployment_name + prefix_always_deployment_name = var.name_prefix != null ? "${var.deployment_name}-${var.name_prefix}" : var.deployment_name + resource_prefix = var.add_deployment_name_before_prefix ? local.prefix_always_deployment_name : local.prefix_optional_deployment_name + + enable_gvnic = var.bandwidth_tier != "not_enabled" + enable_tier_1 = var.bandwidth_tier == "tier_1_enabled" + + provisioning_model = var.provisioning_model + + spot = var.provisioning_model == "SPOT" + + # compact_placement : true when placement policy is provided and collocation set; false if unset + compact_placement = try(var.placement_policy.collocation, null) != null + + gpu_attached = contains(["a2", "g2"], local.machine_family) || length(local.guest_accelerator) > 0 + + # both of these must be false if either compact placement or preemptible/spot instances are used + # automatic restart is tolerant of GPUs while on host maintenance is not + automatic_restart_default = local.compact_placement || local.spot ? false : null + on_host_maintenance_default = local.compact_placement || local.spot || local.gpu_attached ? "TERMINATE" : "MIGRATE" + + automatic_restart = ( + var.automatic_restart != null + ? var.automatic_restart + : local.automatic_restart_default + ) + + on_host_maintenance = ( + var.on_host_maintenance != null + ? var.on_host_maintenance + : local.on_host_maintenance_default + ) + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + # Network Interfaces + # Support for `use` input and base network parameters like `network_self_link` and `subnetwork_self_link` + empty_access_config = { + nat_ip = null, + public_ptr_domain_name = null, + network_tier = null + } + default_network_interface = { + network = var.network_self_link + subnetwork = var.subnetwork_self_link + subnetwork_project = null # will populate from subnetwork_self_link + network_ip = null + nic_type = local.enable_gvnic ? "GVNIC" : null + stack_type = null + queue_count = null + access_config = var.disable_public_ips ? [] : [local.empty_access_config] + ipv6_access_config = [] + alias_ip_range = [] + } + network_interfaces = coalescelist(var.network_interfaces, [local.default_network_interface]) + network_interfaces_with_ips = var.allocate_ip == null ? local.network_interfaces : [ + for i, interface in local.network_interfaces : + merge(interface, { + network_ip = google_compute_address.compute_ip[i].address + }) + ] +} + +resource "null_resource" "image" { + triggers = { + name = try(var.instance_image.name, null), + family = try(var.instance_image.family, null), + project = try(var.instance_image.project, null) + } +} + +resource "google_compute_disk" "additional_disks" { + project = var.project_id + + count = var.instance_count * var.additional_persistent_disks.count + + # NB: this resource array must be sliced accounting for var.instance_count + name = "${local.resource_prefix}-disk-${count.index}" + type = var.additional_persistent_disks.type + size = var.additional_persistent_disks.size + labels = local.labels + zone = var.zone +} + +resource "google_compute_resource_policy" "placement_policy" { + project = var.project_id + provider = google-beta + + count = var.placement_policy != null ? 1 : 0 + name = "${local.resource_prefix}-vm-instance-placement" + group_placement_policy { + vm_count = try(var.placement_policy.vm_count, null) + availability_domain_count = try(var.placement_policy.availability_domain_count, null) + collocation = try(var.placement_policy.collocation, null) + max_distance = try(var.placement_policy.max_distance, null) + } +} + +resource "null_resource" "replace_vm_trigger_from_placement" { + triggers = { + vm_count = try(tostring(var.placement_policy.vm_count), "") + availability_domain_count = try(tostring(var.placement_policy.availability_domain_count), "") + max_distance = try(tostring(var.placement_policy.max_distance), "") + collocation = try(var.placement_policy.collocation, "") + } +} + +resource "google_compute_address" "compute_ip" { + project = var.project_id + + count = var.allocate_ip != null ? length(local.network_interfaces) : 0 + + name = "${local.resource_prefix}-${count.index}" + + address = local.network_interfaces[count.index].network_ip + region = var.region + network = can(coalesce(local.network_interfaces[count.index].subnetwork)) ? null : local.network_interfaces[count.index].network + subnetwork = local.network_interfaces[count.index].subnetwork + address_type = var.allocate_ip.address_type + purpose = var.allocate_ip.purpose + network_tier = var.allocate_ip.network_tier + ip_version = var.allocate_ip.ip_version +} + +resource "google_compute_instance" "compute_vm" { + project = var.project_id + provider = google-beta + + count = var.instance_count + + depends_on = [var.network_self_link, var.network_storage] + + name = "${local.resource_prefix}-${count.index}" + min_cpu_platform = var.min_cpu_platform + machine_type = var.machine_type + zone = var.zone + + resource_policies = google_compute_resource_policy.placement_policy[*].self_link + + tags = var.tags + labels = local.labels + + boot_disk { + initialize_params { + image = data.google_compute_image.compute_image.self_link + size = var.disk_size_gb + type = var.disk_type + labels = local.labels + } + + device_name = "${local.resource_prefix}-boot-disk-${count.index}" + auto_delete = var.auto_delete_boot_disk + } + + dynamic "attached_disk" { + for_each = slice( + google_compute_disk.additional_disks, + var.additional_persistent_disks.count * count.index, + var.additional_persistent_disks.count * count.index + var.additional_persistent_disks.count, + ) + + content { + source = attached_disk.value.self_link + device_name = "additional-disk-${attached_disk.key}" + mode = "READ_WRITE" + } + } + + dynamic "scratch_disk" { + for_each = range(var.local_ssd_count) + content { + interface = var.local_ssd_interface + } + } + + dynamic "network_interface" { + for_each = local.network_interfaces_with_ips + + content { + network = network_interface.value.network + subnetwork = network_interface.value.subnetwork + subnetwork_project = network_interface.value.subnetwork_project + network_ip = network_interface.value.network_ip + nic_type = network_interface.value.nic_type + stack_type = network_interface.value.stack_type + queue_count = network_interface.value.queue_count + dynamic "access_config" { + for_each = network_interface.value.access_config + content { + nat_ip = access_config.value.nat_ip + public_ptr_domain_name = access_config.value.public_ptr_domain_name + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = network_interface.value.ipv6_access_config + content { + public_ptr_domain_name = ipv6_access_config.value.public_ptr_domain_name + network_tier = ipv6_access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = network_interface.value.alias_ip_range + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + } + } + + network_performance_config { + total_egress_bandwidth_tier = local.enable_tier_1 ? "TIER_1" : "DEFAULT" + } + + service_account { + email = var.service_account_email + scopes = var.service_account_scopes + } + + dynamic "guest_accelerator" { + for_each = local.guest_accelerator + content { + count = guest_accelerator.value.count + type = guest_accelerator.value.type + } + } + + scheduling { + on_host_maintenance = local.on_host_maintenance + automatic_restart = local.automatic_restart + preemptible = local.spot + provisioning_model = local.provisioning_model + } + + dynamic "advanced_machine_features" { + for_each = local.set_threads_per_core ? [1] : [] + content { + threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf + } + } + + dynamic "reservation_affinity" { + for_each = var.reservation_name == "" ? [] : [1] + content { + type = "SPECIFIC_RESERVATION" + specific_reservation { + key = "compute.googleapis.com/reservation-name" + values = [var.reservation_name] + } + } + } + + metadata = merge( + local.network_storage, + local.startup_script, + local.enable_oslogin, + local.disable_automatic_updates_metadata, + var.metadata + ) + + lifecycle { + ignore_changes = [ + metadata["ssh-keys"], + ] + + replace_triggered_by = [ + null_resource.replace_vm_trigger_from_placement + ] + + precondition { + condition = (length(var.network_interfaces) == 0) != (var.network_self_link == null && var.subnetwork_self_link == null) + error_message = "Exactly one of network_interfaces or network_self_link/subnetwork_self_link must be specified." + } + precondition { + condition = alltrue([for interface in var.network_interfaces : interface.network_ip == null]) || var.instance_count == 1 + error_message = <<-EOT + The network_ip cannot be statically set on vm-instance when the VM instance_count is greater than 1. + Either set the network_ip to null to allow it to be set dynamically for all instances, or create modules for each VM instance with its own network interface. + EOT + } + precondition { + condition = !contains([ + "c3-:pd-standard", + "h3-:pd-standard", + "h3-:pd-ssd", + ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") + error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/outputs.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/outputs.tf new file mode 100644 index 0000000000..eab8cb56bd --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "name" { + description = "Names of instances created" + value = google_compute_instance.compute_vm[*].name +} + +output "self_link" { + description = "The tuple URIs of the created instances" + value = google_compute_instance.compute_vm[*].self_link +} + +output "external_ip" { + description = "External IP of the instances (if enabled)" + value = try(google_compute_instance.compute_vm[*].network_interface[0].access_config[0].nat_ip, []) +} + +output "internal_ip" { + description = "Internal IP of the instances" + value = google_compute_instance.compute_vm[*].network_interface[0].network_ip +} + +locals { + first_instance_link = try(google_compute_instance.compute_vm[0].self_link, "no-instance") + ssh_instructions = <<-EOT + Use the following commands to SSH into the first VM created: + gcloud compute ssh ${local.first_instance_link} --project ${var.project_id} + If not accessible from the public internet, use an SSH tunnel through IAP: + gcloud compute ssh ${local.first_instance_link} --tunnel-through-iap --project ${var.project_id} + EOT +} + +output "instructions" { + description = "Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions." + value = var.instance_count > 0 ? local.ssh_instructions : "No instances were created." +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf new file mode 100644 index 0000000000..02bc58e4f7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "inputs": +# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. +# var.network_storage : to be passed into metadata somewhere else (not here) +# var.startup_script : to be changed into a more complete file system with all the fs runners + +# "outputs": +# local.startup_from_network_storage : A full startup script with all the runners that are not supported +# natively and were included in the network_storage structure + +locals { + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + # Pull out runners to include in startup script + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + startup_script_runner = [{ + content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" + destination = "passed_startup_script.sh" + type = "shell" + }] + + full_runner_list = concat( + local.storage_client_install_runners, + local.mount_runners, + local.startup_script_runner + ) + + startup_from_network_storage = module.netstorage_startup_script.startup_script +} + +module "netstorage_startup_script" { + source = "../../scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.full_runner_list +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf new file mode 100644 index 0000000000..e582db33da --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' + +# "inputs": +# var.machine_type: Machine type for the instance being evaluated. +# var.threads_per_core : Sets the number of threads per physical core, where 0 +# has behavior described in description. + +# "outputs": +# local.set_threads_per_core: bool that tells if threads per core should be set, +# to be used with a dynamic block. +# local.threads_per_core: actual threads_per_core to be used. + +locals { + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + machine_shared_core = length(local.machine_vals) <= 2 + machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) + + smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) + smt_capable_vcpu = local.machine_vcpus >= 2 + + smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core + set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) + threads_per_core = var.threads_per_core == 2 ? 2 : 1 +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/variables.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/variables.tf new file mode 100644 index 0000000000..5519b8cd40 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/variables.tf @@ -0,0 +1,452 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "instance_count" { + description = "Number of instances" + type = number + default = 1 +} + +variable "instance_image" { + description = "Instance Image" + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "disk_size_gb" { + description = "Size of disk for instances." + type = number + default = 200 +} + +variable "disk_type" { + description = "Disk type for instances." + type = string + default = "pd-standard" +} + +variable "auto_delete_boot_disk" { + description = "Controls if boot disk should be auto-deleted when instance is deleted." + type = bool + default = true +} + +variable "local_ssd_count" { + description = "The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd." + type = number + default = 0 +} + +variable "local_ssd_interface" { + description = "Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set." + type = string + default = "NVME" +} + +variable "additional_persistent_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = object({ + count = optional(number, 0) + type = optional(string, "pd-balanced") + size = optional(number, 200) + }) + default = {} +} + +variable "name_prefix" { + description = <<-EOT + An optional name for all VM and disk resources. + If not supplied, `deployment_name` will be used. + When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, + then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". + EOT + type = string + default = null +} + +variable "add_deployment_name_before_prefix" { + description = <<-EOT + If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. + See `name_prefix` for further details on resource naming behavior. + EOT + type = bool + default = false +} + +variable "disable_public_ips" { + description = "If set to true, instances will not have public IPs" + type = bool + default = false +} + +variable "machine_type" { + description = "Machine type to use for the instance creation" + type = string + default = "c2-standard-60" +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "deployment_name" { + description = "Name of the deployment, will optionally be used name resources according to `name_prefix`" + type = string +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} + +variable "service_account_email" { + description = "Service account e-mail address to use with the node pool" + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to to use with the node pool." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +# tflint-ignore: terraform_unused_declarations +variable "service_account" { + description = "DEPRECATED - Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string, + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "The 'service_account' setting is deprecated, please use 'var.service_account_email' and 'var.service_account_scopes' instead." + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the VM. Can use \"default\" for the default network." + type = string + default = null +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the VM." + type = string + default = null +} + +variable "network_interfaces" { + description = <<-EOT + A list of network interfaces. The options match that of the terraform + network_interface block of google_compute_instance. For descriptions of the + subfields or more information see the documentation: + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + + **_NOTE:_** If `network_interfaces` are set, `network_self_link` and + `subnetwork_self_link` will be ignored, even if they are provided through + the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply + to network interfaces defined in this variable. + + Subfields: + network (string, required if subnetwork is not supplied) + subnetwork (string, required if network is not supplied) + subnetwork_project (string, optional) + network_ip (string, optional) + nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "MRDMA", "IRDMA"]) + stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) + queue_count (number, optional) + access_config (object, optional) + ipv6_access_config (object, optional) + alias_ip_range (list(object), optional) + EOT + type = list(object({ + network = string, + subnetwork = string, + subnetwork_project = string, + network_ip = string, + nic_type = string, + stack_type = string, + queue_count = number, + access_config = list(object({ + nat_ip = string, + public_ptr_domain_name = string, + network_tier = string + })), + ipv6_access_config = list(object({ + public_ptr_domain_name = string, + network_tier = string + })), + alias_ip_range = list(object({ + ip_cidr_range = string, + subnetwork_range_name = string + })) + })) + default = [] + validation { + condition = alltrue([ + for ni in var.network_interfaces : (ni.network == null) != (ni.subnetwork == null) + ]) + error_message = "All additional network interfaces must define exactly one of \"network\" or \"subnetwork\"." + } + validation { + condition = alltrue([ + for ni in var.network_interfaces : ni.nic_type == "GVNIC" || ni.nic_type == "VIRTIO_NET" || ni.nic_type == "MRDMA" || ni.nic_type == "IRDMA" || ni.nic_type == null + ]) + error_message = "In the variable network_interfaces, field \"nic_type\" must be \"GVNIC\", \"VIRTIO_NET\", \"MRDMA\", \"IRDMA\", or null." + } + validation { + condition = alltrue([ + for ni in var.network_interfaces : ni.stack_type == "IPV4_ONLY" || ni.stack_type == "IPV4_IPV6" || ni.stack_type == null + ]) + error_message = "In the variable network_interfaces, field \"stack_type\" must be either \"IPV4_ONLY\", \"IPV4_IPV6\" or null." + } +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "zone" { + description = "Compute Platform zone" + type = string +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "startup_script" { + description = "Startup script used on the instance" + type = string + default = null +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false +} + +variable "automatic_restart" { + description = "Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user)." + type = bool + default = null +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" + type = string + default = null + validation { + condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) + error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." + } +} + +variable "bandwidth_tier" { + description = <= 0, false) && try(var.threads_per_core <= 2, false) + error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." + } + +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "allocate_ip" { + description = <<-EOT + If not null, allocate IPs with the given configuration. See details at + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address + EOT + type = object({ + address_type = optional(string, "INTERNAL") + purpose = optional(string), + network_tier = optional(string), + ip_version = optional(string, "IPV4"), + }) + default = null +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "reservation_name" { + description = <<-EOD + Name of the reservation to use for VM resources, should be in one of the following formats: + - projects/PROJECT_ID/reservations/RESERVATION_NAME + - RESERVATION_NAME + + Must be a "SPECIFIC_RESERVATION" + Set to empty string if using no reservation or automatically-consumed reservations + EOD + type = string + default = "" + nullable = false + + validation { + condition = length(regexall("^((projects/([a-z0-9-]+)/reservations/)?([a-z0-9-]+))?$", var.reservation_name)) > 0 + error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME', [...] is an optional part." + } +} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/versions.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/versions.tf new file mode 100644 index 0000000000..0429782c6d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/versions.tf @@ -0,0 +1,41 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.73.0" + } + + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.13.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/README.md b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/README.md new file mode 100644 index 0000000000..285a20bde2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/README.md @@ -0,0 +1,170 @@ +## Description + +This module creates a [Google Cloud Storage (GCS) bucket](https://cloud.google.com/storage). + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +### Example + +The following example will create a bucket named `simulation-results-xxxxxxxx`, +where `xxxxxxxx` is a randomly generated id. + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: + name_prefix: simulation-results + random_suffix: true +``` + +> **_NOTE:_** Use of `random_suffix` may cause the following error when used +> with other modules: +> `value depends on resource attributes that cannot be determined until apply`. +> To resolve this set `random_suffix` to `false` (default). + + + +> **_NOTE:_** Bucket namespace is shared by all users of Google Cloud so it is +> possible to have a bucket name clash with an existing bucket that is not in +> your project. To resolve this try to use a more unique name, or set the +> `random_suffix` variable to `true`. + +## Naming of Bucket + +There are potentially three parts to the bucket name. Each of these parts are +configurable in the blueprint. + +1. A **custom prefix**, provided by the user in the blueprint \ +Provide the custom prefix using the `name_prefix` setting. + +1. The **deployment name**, included by default \ +The deployment name can be excluded by setting `use_deployment_name_in_bucket_name: false`. + +1. A **random id** suffix, excluded by default \ +The random id can be included by setting `random_suffix: true`. + +If none of these are provided (no `name_prefix`, +`use_deployment_name_in_bucket_name: false`, & `random_suffix: false`), then the +bucket name will default to `no-bucket-name-provided`. + +Since bucket namespace is shared by all users of Google Cloud, it is more likely +to experience naming clashes than with other resources. In many cases, adding +the `random_suffix` will resolve the naming clash issue. + +> **Warning**: If a bucket is created with a `random_suffix` and then used as +> the bucket for a startup script in the same deployment group this will cause a +> `not known at apply time` error in terraform. The solution is to either create +> the bucket in a separate deployment group or to remove the random suffix. + +## Mounting + +To mount the Cloud Storage bucket you must first ensure that the GCS Fuse client +has been installed and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. + +If mounting is not automatically handled as described above, the +`cloud-storage-bucket` module outputs runners that can be used with the +`startup-script` module to install the client and mount the file system. See the +following example: + +```yaml + - id: bucket + source: modules/file-system/cloud-storage-bucket + settings: {local_mount: /data} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(bucket.client_install_runner) + - $(bucket.mount_runner) +``` + +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | +| [google-beta](#requirement\_google-beta) | >= 6.9.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [google-beta](#provider\_google-beta) | >= 6.9.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource | +| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [autoclass](#input\_autoclass) | Configure bucket autoclass setup

The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern.

The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time.
Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE')

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/autoclass | 
object({
    enabled                = optional(bool, false)
    terminal_storage_class = optional(string, null)
  })
| 
{
  "enabled": false
}
| no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the GCS bucket. | `string` | n/a | yes | +| [enable\_hierarchical\_namespace](#input\_enable\_hierarchical\_namespace) | If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket. | `bool` | `false` | no | +| [enable\_object\_retention](#input\_enable\_object\_retention) | If true, enables retention policy at per object level for the bucket.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/object-lock | `bool` | `false` | no | +| [enable\_versioning](#input\_enable\_versioning) | If true, enables versioning for the bucket. | `bool` | `false` | no | +| [force\_destroy](#input\_force\_destroy) | If true will destroy bucket with all objects stored within. | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to the GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | +| [lifecycle\_rules](#input\_lifecycle\_rules) | List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule | 
list(object({
    # Object with keys:
    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.
    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.
    action = object({
      type          = string
      storage_class = optional(string)
    })

    # Object with keys:
    # - age - (Optional) Minimum age of an object in days to satisfy this condition.
    # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field.
    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.
    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".
    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY.
    # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition.
    # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition.
    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.
    # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition.
    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.
    # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object.
    # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent.
    condition = object({
      age                        = optional(number)
      send_age_if_zero           = optional(bool)
      created_before             = optional(string)
      with_state                 = optional(string)
      matches_storage_class      = optional(string)
      matches_prefix             = optional(string)
      matches_suffix             = optional(string)
      num_newer_versions         = optional(number)
      custom_time_before         = optional(string)
      days_since_custom_time     = optional(number)
      days_since_noncurrent_time = optional(number)
      noncurrent_time_before     = optional(string)
    })
  }))
| `[]` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | +| [mount\_options](#input\_mount\_options) | Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories) | `string` | `"defaults,_netdev,implicit_dirs"` | no | +| [name\_prefix](#input\_name\_prefix) | Name Prefix. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | +| [public\_access\_prevention](#input\_public\_access\_prevention) | Bucket public access can be controlled by setting a value of either `inherited` or `enforced`.
When set to `enforced`, public access to the bucket is blocked.
If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/public-access-prevention | `string` | `null` | no | +| [random\_suffix](#input\_random\_suffix) | If true, a random id will be appended to the suffix of the bucket name. | `bool` | `false` | no | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [retention\_policy\_period](#input\_retention\_policy\_period) | If defined, this will configure retention\_policy with retention\_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/bucket-lock | `number` | `null` | no | +| [soft\_delete\_retention\_duration](#input\_soft\_delete\_retention\_duration) | If defined, this will configure soft\_delete\_policy with retention\_duration\_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days).
Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/soft-delete | `number` | `null` | no | +| [storage\_class](#input\_storage\_class) | The storage class of the GCS bucket. | `string` | `"REGIONAL"` | no | +| [uniform\_bucket\_level\_access](#input\_uniform\_bucket\_level\_access) | Allow uniform control access to the bucket. | `bool` | `true` | no | +| [use\_deployment\_name\_in\_bucket\_name](#input\_use\_deployment\_name\_in\_bucket\_name) | If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments. | `bool` | `true` | no | +| [viewers](#input\_viewers) | A list of additional accounts that can read packages from this bucket | `set(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use gcs fuse. | +| [gcs\_bucket\_name](#output\_gcs\_bucket\_name) | Bucket name. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | The gsutil bucket path with format of `gs://`. | +| [mount\_runner](#output\_mount\_runner) | Runner that mounts the cloud storage bucket with gcs fuse. | +| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | + diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf new file mode 100644 index 0000000000..81ba0ca6a9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf @@ -0,0 +1,126 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "cloud-storage-bucket", ghpc_role = "file-system" }) +} + +locals { + prefix = var.name_prefix != null ? var.name_prefix : "" + deployment = var.use_deployment_name_in_bucket_name ? var.deployment_name : "" + suffix = var.random_suffix ? random_id.resource_name_suffix.hex : "" + first_dash = (local.prefix != "" && (local.deployment != "" || local.suffix != "")) ? "-" : "" + second_dash = local.deployment != "" && local.suffix != "" ? "-" : "" + composite_name = "${local.prefix}${local.first_dash}${local.deployment}${local.second_dash}${local.suffix}" + name = local.composite_name == "" ? "no-bucket-name-provided" : local.composite_name +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket" "bucket" { + provider = google-beta + project = var.project_id + name = local.name + uniform_bucket_level_access = var.uniform_bucket_level_access + location = var.region + storage_class = var.storage_class + labels = local.labels + force_destroy = var.force_destroy + public_access_prevention = var.public_access_prevention + enable_object_retention = var.enable_object_retention + hierarchical_namespace { + enabled = var.enable_hierarchical_namespace + } + + dynamic "autoclass" { + for_each = var.autoclass.enabled ? [1] : [] + content { + enabled = var.autoclass.enabled + terminal_storage_class = var.autoclass.terminal_storage_class + } + } + + dynamic "soft_delete_policy" { + for_each = var.soft_delete_retention_duration == null ? [] : [1] + content { + retention_duration_seconds = var.soft_delete_retention_duration + } + } + + dynamic "retention_policy" { + for_each = var.retention_policy_period == null ? [] : [1] + content { + retention_period = var.retention_policy_period + } + } + + dynamic "versioning" { + for_each = var.enable_versioning ? [1] : [] + content { + enabled = var.enable_versioning + } + } + + dynamic "lifecycle_rule" { + for_each = var.lifecycle_rules + content { + action { + type = lifecycle_rule.value.action.type + storage_class = lookup(lifecycle_rule.value.action, "storage_class", null) + } + condition { + age = lookup(lifecycle_rule.value.condition, "age", null) + send_age_if_zero = lookup(lifecycle_rule.value.condition, "send_age_if_zero", null) + created_before = lookup(lifecycle_rule.value.condition, "created_before", null) + with_state = lookup(lifecycle_rule.value.condition, "with_state", contains(keys(lifecycle_rule.value.condition), "is_live") ? (lifecycle_rule.value.condition["is_live"] ? "LIVE" : null) : null) + matches_storage_class = lifecycle_rule.value.condition["matches_storage_class"] != null ? split(",", lifecycle_rule.value.condition["matches_storage_class"]) : null + matches_prefix = lifecycle_rule.value.condition["matches_prefix"] != null ? split(",", lifecycle_rule.value.condition["matches_prefix"]) : null + matches_suffix = lifecycle_rule.value.condition["matches_suffix"] != null ? split(",", lifecycle_rule.value.condition["matches_suffix"]) : null + num_newer_versions = lookup(lifecycle_rule.value.condition, "num_newer_versions", null) + custom_time_before = lookup(lifecycle_rule.value.condition, "custom_time_before", null) + days_since_custom_time = lookup(lifecycle_rule.value.condition, "days_since_custom_time", null) + days_since_noncurrent_time = lookup(lifecycle_rule.value.condition, "days_since_noncurrent_time", null) + noncurrent_time_before = lookup(lifecycle_rule.value.condition, "noncurrent_time_before", null) + } + } + } + + lifecycle { + precondition { + condition = !var.autoclass.enabled || !var.enable_hierarchical_namespace + error_message = "Hierarchical namespace is not compatible with Autoclass enabled." + } + + precondition { + condition = !var.enable_hierarchical_namespace || var.uniform_bucket_level_access + error_message = "Hierarchical namespace is not compatible with Uniform bucket level access disabled." + } + + precondition { + condition = !var.enable_versioning || !var.enable_hierarchical_namespace + error_message = "Hierarchical namespace is not compatible with Object versioning enabled." + } + } +} + +resource "google_storage_bucket_iam_binding" "viewers" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.objectViewer" + members = var.viewers +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf new file mode 100644 index 0000000000..29ddfef2d2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a remote network storage to be mounted by fs-tab." + value = { + remote_mount = local.name + local_mount = var.local_mount + fs_type = "gcsfuse" + mount_options = var.mount_options + server_ip = "" + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } +} + +locals { + client_install_runner = { + "type" = "shell" + "content" = file("${path.module}/scripts/install-gcs-fuse.sh") + "destination" = "install-gcsfuse${replace(var.local_mount, "/", "_")}.sh" + } + + mount_runner = { + "type" = "shell" + "destination" = "mount_gcs${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"not-used\" \"${local.name}\" \"${var.local_mount}\" \"gcsfuse\" \"${var.mount_options}\"" + "content" = file("${path.module}/scripts/mount.sh") + } +} + +output "client_install_runner" { + description = "Runner that performs client installation needed to use gcs fuse." + value = local.client_install_runner +} + +output "mount_runner" { + description = "Runner that mounts the cloud storage bucket with gcs fuse." + value = local.mount_runner +} + +output "gcs_bucket_path" { + description = "The gsutil bucket path with format of `gs://`." + # cannot use resource attribute, will cause lookup failure in startup-script + value = "gs://${local.name}" + + # needed to make sure bucket contents are deleted before bucket + depends_on = [ + google_storage_bucket.bucket + ] +} + +output "gcs_bucket_name" { + description = "Bucket name." + value = google_storage_bucket.bucket.name +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh new file mode 100644 index 0000000000..f8a990260b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ ! "$(which gcsfuse)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then + tee /etc/yum.repos.d/gcsfuse.repo >/dev/null </dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf new file mode 100644 index 0000000000..9804e4b268 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf @@ -0,0 +1,254 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which GCS bucket will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment; used as part of name of the GCS bucket." + type = string +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "labels" { + description = "Labels to add to the GCS bucket. Key-value pairs." + type = map(string) +} + +variable "local_mount" { + description = "The mount point where the contents of the device may be accessed after mounting." + type = string + default = "/mnt" +} + +variable "mount_options" { + description = "Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories)" + type = string + default = "defaults,_netdev,implicit_dirs" +} + +variable "name_prefix" { + description = "Name Prefix." + type = string + default = null +} + +variable "use_deployment_name_in_bucket_name" { + description = "If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments." + type = bool + default = true +} + +variable "random_suffix" { + description = "If true, a random id will be appended to the suffix of the bucket name." + type = bool + default = false +} + +variable "force_destroy" { + description = "If true will destroy bucket with all objects stored within." + type = bool + default = false +} + +variable "viewers" { + description = "A list of additional accounts that can read packages from this bucket" + type = set(string) + default = [] + + validation { + error_message = "All bucket viewers must be in IAM style: user:user@example.com, serviceAccount:sa@example.com, or group:group@example.com." + condition = alltrue([ + for viewer in var.viewers : length(regexall("^(user|serviceAccount|group):", viewer)) > 0 + ]) + } +} + +variable "enable_hierarchical_namespace" { + description = "If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket." + type = bool + default = false +} + +variable "uniform_bucket_level_access" { + description = "Allow uniform control access to the bucket." + type = bool + default = true +} + +variable "storage_class" { + description = "The storage class of the GCS bucket." + type = string + default = "REGIONAL" + validation { + condition = contains([ + "STANDARD", + "MULTI_REGIONAL", + "REGIONAL", + "NEARLINE", + "COLDLINE", + "ARCHIVE" + ], var.storage_class) + error_message = "Allowed values for GCS storage_class are 'STANDARD', 'MULTI_REGIONAL', 'REGIONAL', 'NEARLINE', 'COLDLINE', 'ARCHIVE'.\nhttps://cloud.google.com/storage/docs/storage-classes" + } +} + +variable "autoclass" { + description = <<-EOT + Configure bucket autoclass setup + + The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern. + + The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time. + Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE') + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/autoclass + EOT + type = object({ + enabled = optional(bool, false) + terminal_storage_class = optional(string, null) + }) + default = { + enabled = false + } + nullable = false + validation { + condition = !can(coalesce(var.autoclass.terminal_storage_class)) || var.autoclass.enabled + error_message = "Cannot set bucket var.autoclass.terminal_storage_class unless var.autoclass.enabled is true" + } +} + +variable "public_access_prevention" { + description = <<-EOT + Bucket public access can be controlled by setting a value of either `inherited` or `enforced`. + When set to `enforced`, public access to the bucket is blocked. + If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/public-access-prevention + EOT + type = string + default = null + validation { + condition = var.public_access_prevention == null ? true : contains([ + "inherited", + "enforced" + ], var.public_access_prevention) + error_message = "Allowed values for public_access_prevention are 'inherited', 'enforced'.\n" + } +} + +variable "soft_delete_retention_duration" { + description = <<-EOT + If defined, this will configure soft_delete_policy with retention_duration_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days). + Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/soft-delete + EOT + type = number + default = null + validation { + condition = var.soft_delete_retention_duration == null ? true : var.soft_delete_retention_duration == 0 || var.soft_delete_retention_duration >= 604800 && var.soft_delete_retention_duration <= 7776000 + error_message = "var.soft_delete_retention_duration value can be 0 or in between 604800(7 days) and 7776000(90 days)." + } +} + +variable "enable_versioning" { + description = "If true, enables versioning for the bucket." + type = bool + default = false +} + +variable "lifecycle_rules" { + description = "List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule" + type = list(object({ + # Object with keys: + # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass. + # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule. + action = object({ + type = string + storage_class = optional(string) + }) + + # Object with keys: + # - age - (Optional) Minimum age of an object in days to satisfy this condition. + # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field. + # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition. + # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY". + # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY. + # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition. + # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition. + # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition. + # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition. + # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true. + # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object. + # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent. + condition = object({ + age = optional(number) + send_age_if_zero = optional(bool) + created_before = optional(string) + with_state = optional(string) + matches_storage_class = optional(string) + matches_prefix = optional(string) + matches_suffix = optional(string) + num_newer_versions = optional(number) + custom_time_before = optional(string) + days_since_custom_time = optional(number) + days_since_noncurrent_time = optional(number) + noncurrent_time_before = optional(string) + }) + })) + default = [] +} + +variable "retention_policy_period" { + description = <<-EOT + If defined, this will configure retention_policy with retention_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/bucket-lock + EOT + type = number + default = null + validation { + condition = var.retention_policy_period == null ? true : var.retention_policy_period > 0 && var.retention_policy_period <= 3155760000 + error_message = "var.soft_delete_policy_retention_duration value must be in between 1 and 3155760000(100 years) seconds." + } +} + +variable "enable_object_retention" { + description = <<-EOT + If true, enables retention policy at per object level for the bucket. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/object-lock + EOT + type = bool + default = false +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf new file mode 100644 index 0000000000..217ee2f3a2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.9.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" + } + required_version = ">= 0.14.0" +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/README.md b/deletion-test/primary/modules/embedded/modules/file-system/filestore/README.md new file mode 100644 index 0000000000..3bf251828e --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/filestore/README.md @@ -0,0 +1,248 @@ +## Description + +This module creates a [filestore](https://cloud.google.com/filestore) +instance. Filestore is a high performance network file system that can be +mounted to one or more compute VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +### Deletion protection + +We recommend considering enabling [Filestore deletion protection][fdp]. Deletion +protection will prevent unintentional deletion of an entire Filestore instance. +It does not prevent deletion of files within the Filestore instance when mounted +by a VM. It is not available on some [tiers](#filestore-tiers), including the +default BASIC\_HDD tier or BASIC\_SSD tier. Follow the documentation link for +up to date details. + +Usage can be enabled in a blueprint with, for example: + +```yaml + - id: homefs + source: modules/file-system/filestore + use: [network] + settings: + deletion_protection: + enabled: true + reason: Avoid data loss + filestore_tier: ZONAL + local_mount: /home + size_gb: 1024 +``` + +[fdp]: https://cloud.google.com/filestore/docs/deletion-protection + +### Filestore tiers + +At the time of writing, Filestore supports 5 [tiers of service][tiers] that are +specified in the Toolkit using the following names: + +- Basic HDD: "BASIC\_HDD" ([preferred][tierapi]) or "STANDARD" (deprecated) +- Basic SSD: "BASIC\_SSD" ([preferred][tierapi]) or "PREMIUM" (deprecated) +- Zonal: "ZONAL" +- Enterprise: "ENTERPRISE" +- Regional: "REGIONAL" + +[tierapi]: https://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier + +**Please review the minimum storage requirements for each tier**. The Terraform +module can only enforce the minimum value of the `size_gb` parameter for the +lowest tier of service. If you supply a value that is too low, Filestore +creation will fail when you run `terraform apply`. + +[tiers]: https://cloud.google.com/filestore/docs/service-tiers + +### Filestore protocols and mount options +After Filestore instance is created, you can mount this to the compute node +using different mount options. Toolkit uses [default mount options](https://linux.die.net/man/8/mount) +for all tier services. Filestore has recommended mount options for different +service tiers which may overall improve performance. These can be found here: +[recommended mount options.](https://cloud.google.com/filestore/docs/mounting-fileshares) +While creating filestore module, you can overwrite these mount options as +mentioned below. + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /homefs + mount_options: defaults,hard,timeo=600,retrans=3,_netdev +``` + +Filestore supports NFS protocols `NFS_V3` (default) and `NFS_V4_1`. Protocol support depends on the selected tier: +- `NFS_V3`: Supported on all tiers (`BASIC_HDD`, `BASIC_SSD`, `HIGH_SCALE_SSD`, `ZONAL`, `ENTERPRISE`). +- `NFS_V4_1`: Supported only on `HIGH_SCALE_SSD`, `ZONAL`, `REGIONAL`, and `ENTERPRISE`. +This can be specified at creation time via the `protocol` variable. By default, `NFS_V3` is used for compatibility. +See the example below and [this page](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#protocol-1) for more information. + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /homefs + protocol: NFS_V4_1 + filestore_tier: ZONAL +``` + +### Filestore quota + +Your project must have unused quota for Cloud Filestore in the region you will +provision the storage. This can be found by browsing to the [Quota tab within IAM +& Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. +Please note that there are separate quota limits for HDD and SSD storage. + +All projects begin with 0 available quota for High Scale SSD tier. To use this +tier, [make a request and wait for it to be approved][hs-ssd-quota]. + +[hs-ssd-quota]: https://cloud.google.com/filestore/docs/high-scale + +### Example - Basic HDD + +The Filestore instance defined below will have the following attributes: + +- (default) `BASIC_HDD` tier +- (default) 1TiB capacity +- `homefs` module ID +- mount point at `/home` +- connected to the network defined in the `network1` module + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /home +``` + +### Example - High Scale SSD + +The Filestore instance defined below will have the following attributes: + +- `HIGH_SCALE_SSD` tier +- 10TiB capacity +- `highscale` module ID +- mount point at `/projects` +- connected to the VPC network defined in the `network1` module + +```yaml +- id: highscale + source: modules/file-system/filestore + use: [network1] + settings: + filestore_tier: HIGH_SCALE_SSD + size_gb: 10240 + local_mount: /projects +``` + +## Mounting + +To mount the Filestore instance you must first ensure that the NFS client has +been installed and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +See the [hpc-slurm](../../../examples/hpc-slurm.yaml) for +an example of using this module with Slurm. + +If mounting is not automatically handled as described above, the `filestore` +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + - id: filestore + source: modules/file-system/filestore + use: [network1] + settings: {local_mount: /scratch} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(filestore.install_nfs_client_runner) + - $(filestore.mount_runner) + +``` + +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 6.4 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.4 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_filestore_instance.filestore_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [connect\_mode](#input\_connect\_mode) | Used to select mode - supported values DIRECT\_PEERING and PRIVATE\_SERVICE\_ACCESS. | `string` | `"DIRECT_PEERING"` | no | +| [deletion\_protection](#input\_deletion\_protection) | Configure Filestore instance deletion protection | 
object({
    enabled = optional(bool, false)
    reason  = optional(string)
  })
| 
{
  "enabled": false
}
| no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the filestore instance if no name is specified. | `string` | n/a | yes | +| [description](#input\_description) | A description of the filestore instance. | `string` | `""` | no | +| [filestore\_share\_name](#input\_filestore\_share\_name) | Name of the file system share on the instance. | `string` | `"nfsshare"` | no | +| [filestore\_tier](#input\_filestore\_tier) | The service tier of the instance. | `string` | `"BASIC_HDD"` | no | +| [labels](#input\_labels) | Labels to add to the filestore instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mount](#input\_local\_mount) | Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md)). | `string` | `"/shared"` | no | +| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"defaults,_netdev"` | no | +| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [nfs\_export\_options](#input\_nfs\_export\_options) | Define NFS export options. | 
list(object({
    access_mode = optional(string)
    ip_ranges   = optional(list(string))
    squash_mode = optional(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | ID of project in which Filestore instance will be created. | `string` | n/a | yes | +| [protocol](#input\_protocol) | NFS protocol version. Default is NFS\_V3. NFS\_V4\_1 is only supported with HIGH\_SCALE\_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers. | `string` | `"NFS_V3"` | no | +| [region](#input\_region) | Location for Filestore instances at Enterprise tier. | `string` | n/a | yes | +| [reserved\_ip\_range](#input\_reserved\_ip\_range) | Reserved IP range for Filestore instance. Users are encouraged to set to null
for automatic selection. If supplied, it must be:

CIDR format when var.connect\_mode == "DIRECT\_PEERING"
Named IP Range when var.connect\_mode == "PRIVATE\_SERVICE\_ACCESS"

See Cloud documentation for more details:

https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range | `string` | `null` | no | +| [size\_gb](#input\_size\_gb) | Storage size of the filestore instance in GB. | `number` | `1024` | no | +| [zone](#input\_zone) | Location for Filestore instances below Enterprise tier. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | +| [filestore\_id](#output\_filestore\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | +| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | +| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | +| [network\_storage](#output\_network\_storage) | Describes a filestore instance. | + diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/filestore/main.tf new file mode 100644 index 0000000000..ce035dbb2b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/filestore/main.tf @@ -0,0 +1,116 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "filestore", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + is_high_capacity_tier = contains(["HIGH_SCALE_SSD", "ZONAL", "REGIONAL"], var.filestore_tier) && var.size_gb >= 10240 && var.size_gb <= 102400 + + timeouts = local.is_high_capacity_tier ? [1] : [] + server_ip = google_filestore_instance.filestore_instance.networks[0].ip_addresses[0] + remote_mount = format("/%s", google_filestore_instance.filestore_instance.file_shares[0].name) + fs_type = "nfs" + mount_options = var.mount_options + + install_nfs_client_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-nfs-client.sh" + "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" + } + mount_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" + } + + # id format: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network#id + split_network_id = split("/", var.network_id) + network_name = local.split_network_id[4] + network_project = local.split_network_id[1] + shared_vpc = local.network_project != var.project_id +} + +resource "google_filestore_instance" "filestore_instance" { + project = var.project_id + + name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + description = var.description + location = contains(["ENTERPRISE", "REGIONAL"], var.filestore_tier) ? var.region : var.zone + tier = var.filestore_tier + protocol = var.protocol + + deletion_protection_enabled = var.deletion_protection.enabled + deletion_protection_reason = var.deletion_protection.reason + + file_shares { + capacity_gb = var.size_gb + name = var.filestore_share_name + dynamic "nfs_export_options" { + for_each = var.nfs_export_options + content { + access_mode = nfs_export_options.value.access_mode + ip_ranges = nfs_export_options.value.ip_ranges + squash_mode = nfs_export_options.value.squash_mode + } + } + } + + labels = local.labels + + networks { + network = local.shared_vpc ? var.network_id : local.network_name + connect_mode = var.connect_mode + modes = ["MODE_IPV4"] + reserved_ip_range = var.reserved_ip_range + } + + dynamic "timeouts" { + for_each = local.timeouts + content { + create = "1h" + update = "1h" + delete = "1h" + } + } + + lifecycle { + precondition { + condition = ( + var.reserved_ip_range == null || + var.connect_mode == "PRIVATE_SERVICE_ACCESS" || + var.connect_mode == "DIRECT_PEERING" && can(cidrhost(var.reserved_ip_range, 0)) && contains(["24", "29"], try(split("/", var.reserved_ip_range)[1], "")) + ) + error_message = <<-EOT + If connect_mode is set to DIRECT_PEERING and reserved_ip_range is + specified then it must be a CIDR IP range with suffix range size 29 for + BASIC_HDD or BASIC_SSD tiers. Otherwise the range size must be 24. + EOT + } + + precondition { + condition = !startswith(var.filestore_tier, "BASIC") || var.protocol != "NFS_V4_1" + error_message = "NFS_V4_1 is not supported on BASIC Filestore tiers." + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/filestore/metadata.yaml new file mode 100644 index 0000000000..5298336f09 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/filestore/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - file.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/filestore/outputs.tf new file mode 100644 index 0000000000..9bdb3bdc7b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/filestore/outputs.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a filestore instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = local.mount_options + client_install_runner = local.install_nfs_client_runner + mount_runner = local.mount_runner + } +} + +output "install_nfs_client" { + description = "Script for installing NFS client" + value = file("${path.module}/scripts/install-nfs-client.sh") +} + +output "install_nfs_client_runner" { + description = "Runner to install NFS client using the startup-script module" + value = local.install_nfs_client_runner +} + +output "mount_runner" { + description = <<-EOT + Runner to mount the file-system using an ansible playbook. The startup-script + module will automatically handle installation of ansible. + - id: example-startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(your-fs-id.mount_runner) + ... + EOT + value = local.mount_runner +} + +output "filestore_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" + value = google_filestore_instance.filestore_instance.id +} + +output "capacity_gib" { + description = "File share capacity in GiB." + value = google_filestore_instance.filestore_instance.file_shares[0].capacity_gb +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh b/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/filestore/variables.tf new file mode 100644 index 0000000000..2d7e9258c0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/filestore/variables.tf @@ -0,0 +1,189 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which Filestore instance will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the filestore instance if no name is specified." + type = string +} + +variable "zone" { + description = "Location for Filestore instances below Enterprise tier." + type = string +} + +variable "region" { + description = "Location for Filestore instances at Enterprise tier." + type = string +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is connected given in the format: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "name" { + description = "The resource name of the instance." + type = string + default = null +} + +variable "filestore_share_name" { + description = "Name of the file system share on the instance." + type = string + default = "nfsshare" +} + +variable "local_mount" { + description = "Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md))." + type = string + default = "/shared" +} + +variable "size_gb" { + description = "Storage size of the filestore instance in GB." + type = number + default = 1024 + validation { + condition = var.size_gb >= 1024 + error_message = "No Filestore tier supports less than 1024GiB.\nSee https://cloud.google.com/filestore/docs/service-tiers." + } +} + +variable "filestore_tier" { + description = "The service tier of the instance." + type = string + default = "BASIC_HDD" + validation { + condition = var.filestore_tier != "STANDARD" + error_message = "The preferred name for STANDARD tier is now BASIC_HDD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." + } + validation { + condition = var.filestore_tier != "PREMIUM" + error_message = "The preferred name for PREMIUM tier is now BASIC_SSD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." + } + validation { + condition = contains([ + "BASIC_HDD", + "BASIC_SSD", + "HIGH_SCALE_SSD", + "ZONAL", + "REGIONAL", + "ENTERPRISE" + ], var.filestore_tier) + # Avoid adding the legacy tier name in error_message, for e.g. 'HIGH_SCALE_SSD', 'ENTERPRISE'. + # As we want to steer the customer to new one's, but also support the legacy ones for older customers. + error_message = "Allowed values for filestore_tier are 'BASIC_HDD','BASIC_SSD','ZONAL','REGIONAL'.\nhttps://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#tier\nhttps://cloud.google.com/filestore/docs/reference/rest/v1/Tier." + } +} + +variable "labels" { + description = "Labels to add to the filestore instance. Key-value pairs." + type = map(string) +} + +variable "connect_mode" { + description = "Used to select mode - supported values DIRECT_PEERING and PRIVATE_SERVICE_ACCESS." + type = string + default = "DIRECT_PEERING" + nullable = false + validation { + condition = contains(["DIRECT_PEERING", "PRIVATE_SERVICE_ACCESS"], var.connect_mode) + error_message = "Allowed values for connect_mode are \"DIRECT_PEERING\" or \"PRIVATE_SERVICE_ACCESS\"." + } +} + +variable "nfs_export_options" { + description = "Define NFS export options." + type = list(object({ + access_mode = optional(string) + ip_ranges = optional(list(string)) + squash_mode = optional(string) + })) + default = [] + nullable = false +} + +variable "reserved_ip_range" { + description = <<-EOT + Reserved IP range for Filestore instance. Users are encouraged to set to null + for automatic selection. If supplied, it must be: + + CIDR format when var.connect_mode == "DIRECT_PEERING" + Named IP Range when var.connect_mode == "PRIVATE_SERVICE_ACCESS" + + See Cloud documentation for more details: + + https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range + EOT + type = string + default = null + nullable = true +} + +variable "mount_options" { + description = "NFS mount options to mount file system." + type = string + default = "defaults,_netdev" +} + +variable "deletion_protection" { + description = "Configure Filestore instance deletion protection" + type = object({ + enabled = optional(bool, false) + reason = optional(string) + }) + default = { + enabled = false + } + nullable = false + + validation { + condition = !can(coalesce(var.deletion_protection.reason)) || var.deletion_protection.enabled + error_message = "Cannot set Filestore var.deletion_protection.reason unless var.deletion_protection.enabled is true" + } +} + +variable "protocol" { + description = "NFS protocol version. Default is NFS_V3. NFS_V4_1 is only supported with HIGH_SCALE_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers." + type = string + default = "NFS_V3" + validation { + condition = contains(["NFS_V3", "NFS_V4_1"], var.protocol) + error_message = "Allowed values for protocol are 'NFS_V3' or 'NFS_V4_1'." + } +} + +variable "description" { + description = "A description of the filestore instance." + type = string + default = "" + validation { + condition = length(var.description) <= 2048 + error_message = "Filestore description must be 2048 characters or fewer" + } +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/filestore/versions.tf new file mode 100644 index 0000000000..1ba0e7967e --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/filestore/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.4" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md new file mode 100644 index 0000000000..88ae4511e3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md @@ -0,0 +1,200 @@ +## Description + +This module creates Kubernetes Persistent Volumes (PV) and Persistent Volume +Claims (PVC) that can be used by a [gke-job-template]. + +`gke-persistent-volume` works with Filestore, Google Cloud Storage and Managed Lustre. Each +`gke-persistent-volume` can only be used with a single file system so if multiple +shared file systems are used then multiple `gke-persistent-volume` modules are +needed in the blueprint. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a Filestore and then uses the +`gke-persistent-volume` module to use the Filestore as shared storage in a +`gke-job-template`. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] + settings: + master_authorized_networks: + - display_name: deployment-machine + cidr_block: /32 + + - id: datafs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /data + + - id: datafs-pv + source: modules/file-system/gke-persistent-volume + use: [datafs, gke_cluster] + + - id: job-template + source: modules/compute/gke-job-template + use: [datafs-pv, compute_pool, gke_cluster] +``` + +The following example creates a GCS bucket and then uses the +`gke-persistent-volume` module to use the bucket as shared storage in a +`gke-job-template`. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] + settings: + master_authorized_networks: + - display_name: deployment-machine + cidr_block: /32 + + - id: data-bucket + source: modules/file-system/cloud-storage-bucket + settings: + local_mount: /data + + - id: datagcs-pv + source: modules/file-system/gke-persistent-volume + use: [data-bucket, gke_cluster] + + - id: job-template + source: modules/compute/gke-job-template + use: [datagcs-pv, compute_pool, gke_cluster] +``` + +The following example creates a Managed Lustre and then uses the +`gke-persistent-volume` module to use the Lustre as shared storage in a +`gke-job-template`. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] + settings: + master_authorized_networks: + - display_name: deployment-machine + cidr_block: /32 + + - id: data-managedlustre + source: modules/file-system/managed-lustre + settings: + local_mount: /data + + - id: datalustre-pv + source: modules/file-system/gke-persistent-volume + use: [data-managedlustre, gke_cluster] + + - id: job-template + source: modules/compute/gke-job-template + use: [datalustre-pv, compute_pool, gke_cluster] +``` + +See example +[storage-gke.yaml](../../../../examples/README.md#storage-gkeyaml--) blueprint +for a complete example. + +### Authorized Network + +Since the `gke-persistent-volume` module is making calls to the Kubernetes API +to create Kubernetes entities, the machine performing the deployment must be +authorized to connect to the Kubernetes API. You can add the +`master_authorized_networks` settings block, as shown in the example above, with +the IP address of the machine performing the deployment. This will ensure that +the deploying machine can connect to the cluster. + +### Connecting Via Use + +The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit +modules. For example the `gke-persistent-volume` module can `use` a +`gke-cluster` module and a `filestore` module, as shown in the example above. + +```mermaid + graph TD; + vpc--> |OneToMany| gke-cluster; + gke-cluster--> |OneToMany| gke-node-pool; + gke-node-pool--> |ManyToMany| gke-job-template; + gke-cluster--> |OneToMany| gke-persistent-volume; + gke-persistent-volume--> |ManyToMany| gke-job-template; + vpc--> |OneToMany| filestore; + vpc--> |OneToMany| gcs; + vpc--> |OneToMany| managed-lustre; + filestore--> |OneToOne| gke-persistent-volume; + gcs--> |OneToOne| gke-persistent-volume; + managed-lustre--> |OneToOne| gke-persistent-volume; + ``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [kubectl](#provider\_kubectl) | >= 1.7.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubectl_manifest.pv](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.pvc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.pvc_namespace](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [capacity\_gib](#input\_capacity\_gib) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | +| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | +| [filestore\_id](#input\_filestore\_id) | An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | +| [gcs\_bucket\_name](#input\_gcs\_bucket\_name) | The gcs bucket to be used with the persistent volume. | `string` | `null` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [lustre\_id](#input\_lustre\_id) | An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | +| [network\_storage](#input\_network\_storage) | Network attached storage mount to be configured. | 
object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  })
| n/a | yes | +| [pv\_name](#input\_pv\_name) | The name for PV. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | +| [pvc\_name](#input\_pvc\_name) | The name for PVC. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object describing the Kubernetes PersistentVolumeClaim created by this module. | +| [pvc\_name](#output\_pvc\_name) | The name of the Kubernetes PVC created by this module. | + diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf new file mode 100644 index 0000000000..818ebaf595 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf @@ -0,0 +1,155 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-persistent-volume", ghpc_role = "file-system" }) +} + +locals { + # Flags indicating which storage type is active based on input variables. + storage_type_active = { + gcs = var.gcs_bucket_name != null + lustre = var.lustre_id != null + filestore = var.filestore_id != null + } + + # Determine the active storage type name. + active_types = [for type, is_active in local.storage_type_active : type if is_active] + + # The precondition in kubectl_manifest.pv ensures exactly one type is active. + storage_type = length(local.active_types) > 0 ? local.active_types[0] : "unknown" + + # Map containing the base name derivation logic for each storage type. + base_name_map = { + gcs = var.gcs_bucket_name + lustre = var.lustre_id != null ? split("/", var.lustre_id)[5] : null + filestore = var.filestore_id != null ? split("/", var.filestore_id)[5] : null + } + # Retrieve the base name for the active storage type. + base_name = local.base_name_map[local.storage_type] + + # PV and PVC names + pv_name = var.pv_name != null ? var.pv_name : "${local.base_name}-pv" + pvc_name = var.pvc_name != null ? var.pvc_name : "${local.base_name}-pvc" + + # Template file paths + pv_templates = { + gcs = "${path.module}/templates/gcs-pv.yaml.tftpl" + lustre = "${path.module}/templates/managed-lustre-pv.yaml.tftpl" + filestore = "${path.module}/templates/filestore-pv.yaml.tftpl" + } + pvc_templates = { + gcs = "${path.module}/templates/gcs-pvc.yaml.tftpl" + lustre = "${path.module}/templates/managed-lustre-pvc.yaml.tftpl" + filestore = "${path.module}/templates/filestore-pvc.yaml.tftpl" + } + + # Common variables for all PVC templates + common_pvc_vars = { + pv_name = local.pv_name + pvc_name = local.pvc_name + labels = local.labels + capacity = "${var.capacity_gib}Gi" + namespace = var.namespace + } + + # Common variables for all PV templates + common_pv_vars = { + pv_name = local.pv_name + capacity = "${var.capacity_gib}Gi" + labels = local.labels + } + + # Variables for PV templates, merging common vars with type-specific ones. + pv_template_vars = { + gcs = merge(local.common_pv_vars, { + mount_options = var.gcs_bucket_name != null ? split(",", var.network_storage.mount_options) : [] + bucket_name = var.gcs_bucket_name + namespace = var.namespace + pvc_name = local.pvc_name + }) + lustre = merge(local.common_pv_vars, { + location = var.lustre_id != null ? split("/", var.lustre_id)[3] : null + project = split("/", var.cluster_id)[1] + instance_name = local.base_name + server_ip = var.lustre_id != null ? split("@", var.network_storage.server_ip)[0] : null + filesystem_name = var.network_storage.remote_mount + pvc_name = local.pvc_name + namespace = var.namespace + }) + filestore = merge(local.common_pv_vars, { + location = var.filestore_id != null ? split("/", var.filestore_id)[3] : null + filestore_name = local.base_name + share_name = trimprefix(var.network_storage.remote_mount, "/") + ip_address = var.network_storage.server_ip + pvc_name = local.pvc_name + namespace = var.namespace + }) + } + + # Rendered YAML contents + pv_content = templatefile( + local.pv_templates[local.storage_type], + local.pv_template_vars[local.storage_type] + ) + pvc_content = templatefile( + local.pvc_templates[local.storage_type], + local.common_pvc_vars + ) + + # GKE Cluster details + cluster_name = split("/", var.cluster_id)[5] + cluster_location = split("/", var.cluster_id)[3] +} + +data "google_container_cluster" "gke_cluster" { + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +provider "kubectl" { + host = "https://${data.google_container_cluster.gke_cluster.endpoint}" + cluster_ca_certificate = base64decode(data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) + token = data.google_client_config.default.access_token + load_config_file = false +} + +resource "kubectl_manifest" "pvc_namespace" { + count = var.namespace != "default" ? 1 : 0 + + yaml_body = templatefile("${path.module}/templates/namespace.yaml.tftpl", { + namespace = var.namespace + }) +} + +resource "kubectl_manifest" "pv" { + yaml_body = local.pv_content + + lifecycle { + precondition { + condition = length(local.active_types) == 1 + error_message = "Exactly one of gcs_bucket_name, filestore_id, or lustre_id must be set." + } + } +} + +resource "kubectl_manifest" "pvc" { + yaml_body = local.pvc_content + depends_on = [kubectl_manifest.pv, kubectl_manifest.pvc_namespace] +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf new file mode 100644 index 0000000000..60cf2dbe0f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "persistent_volume_claims" { + description = "An object describing the Kubernetes PersistentVolumeClaim created by this module." + value = { + name = local.pvc_name + namespace = var.namespace + mount_path = var.network_storage.local_mount + mount_options = var.network_storage.mount_options + storage_type = local.storage_type + } +} + +output "pvc_name" { + description = "The name of the Kubernetes PVC created by this module." + value = local.pvc_name +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl new file mode 100644 index 0000000000..06a1276c1e --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl @@ -0,0 +1,26 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${pv_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + storageClassName: "" + capacity: + storage: ${capacity} + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + volumeMode: Filesystem + csi: + driver: filestore.csi.storage.gke.io + volumeHandle: "modeInstance/${location}/${filestore_name}/${share_name}" + volumeAttributes: + ip: ${ip_address} + volume: ${share_name} + claimRef: + name: ${pvc_name} + namespace: ${namespace} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl new file mode 100644 index 0000000000..83cfb3bc8c --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl @@ -0,0 +1,18 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: ${pv_name} + resources: + requests: + storage: ${capacity} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl new file mode 100644 index 0000000000..aa0e570a8b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${pv_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + storageClassName: "" + capacity: + storage: ${capacity} + accessModes: + - ReadWriteMany + %{~ if mount_options != null ~} + mountOptions: + %{~ for key in mount_options ~} + - ${key} + %{~ endfor ~} + %{~ endif ~} + csi: + driver: gcsfuse.csi.storage.gke.io + volumeHandle: ${bucket_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl new file mode 100644 index 0000000000..4d02c85629 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: ${pv_name} + resources: + requests: + storage: ${capacity} + claimRef: + name: ${pvc_name} + namespace: ${namespace} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl new file mode 100644 index 0000000000..2b3b5e7738 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl @@ -0,0 +1,26 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${pv_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + storageClassName: "" + capacity: + storage: ${capacity} + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + volumeMode: Filesystem + claimRef: + namespace: ${namespace} + name: ${pvc_name} + csi: + driver: lustre.csi.storage.gke.io + volumeHandle: "${project}/${location}/${instance_name}/default-pool/default-container" + volumeAttributes: + ip: ${server_ip} + filesystem: ${filesystem_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl new file mode 100644 index 0000000000..83cfb3bc8c --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl @@ -0,0 +1,18 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: ${pv_name} + resources: + requests: + storage: ${capacity} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl new file mode 100644 index 0000000000..fa7647e33f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf new file mode 100644 index 0000000000..fd281756e7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf @@ -0,0 +1,93 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "cluster_id" { + description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" + type = string +} + +variable "network_storage" { + description = "Network attached storage mount to be configured." + type = object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + }) +} + +variable "filestore_id" { + description = "An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." + type = string + default = null + validation { + condition = ( + var.filestore_id == null || + try(length(split("/", var.filestore_id)), 0) == 6 + ) + error_message = "filestore_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." + } +} + +variable "lustre_id" { + description = "An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." + type = string + default = null + validation { + condition = ( + var.lustre_id == null || + try(length(split("/", var.lustre_id)), 0) == 6 + ) + error_message = "lustre_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." + } +} + +variable "gcs_bucket_name" { + description = "The gcs bucket to be used with the persistent volume." + type = string + default = null +} + +variable "capacity_gib" { + description = "The storage capacity with which to create the persistent volume." + type = number +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} + +variable "namespace" { + description = "Kubernetes namespace to deploy the storage PVC/PV" + type = string + default = "default" +} + +variable "pv_name" { + description = "The name for PV. IF not set, a name will be generated based on the storage name." + type = string + default = null +} + +variable "pvc_name" { + description = "The name for PVC. IF not set, a name will be generated based on the storage name." + type = string + default = null +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf new file mode 100644 index 0000000000..fa1c3e2b3f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf @@ -0,0 +1,30 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.7.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.74.0" + } +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/README.md b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/README.md new file mode 100644 index 0000000000..78ef5402aa --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/README.md @@ -0,0 +1,134 @@ +## Description + +This module creates Kubernetes Storage Class (SC) that can be used by a Persistent Volume Claim (PVC) +to dynamically provision GCP storage resources like Parallelstore. + +### Example + +The following example uses the `gke-storage` module to creates a Parallelstore Storage Class and Persistent Volume Claim, +then use them in a `gke-job-template` to dynamically provision the resource. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network] + settings: + enable_parallelstore_csi: true + + # Private Service Access (PSA) requires the compute.networkAdmin role which is + # included in the Owner role, but not Editor. + # PSA is required for all Parallelstore functionality. + # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: gke_storage + source: modules/file-system/gke-storage + use: [ gke_cluster, private_service_access ] + settings: + storage_type: Parallelstore + access_mode: ReadWriteMany + sc_volume_binding_mode: Immediate + sc_reclaim_policy: Delete + sc_topology_zones: [$(vars.zone)] + pvc_count: 2 + capacity_gb: 12000 + + - id: job_template + source: modules/compute/gke-job-template + use: [gke_storage, compute_pool] +``` + +See example +[gke-managed-parallelstore.yaml](../../../examples/README.md#gke-managed-parallelstoreyaml--) blueprint +for a complete example. + +### Authorized Network + +Since the `gke-storage` module is making calls to the Kubernetes API +to create Kubernetes entities, the machine performing the deployment must be +authorized to connect to the Kubernetes API. You can add the +`master_authorized_networks` settings block, as shown in the example above, with +the IP address of the machine performing the deployment. This will ensure that +the deploying machine can connect to the cluster. + +### Connecting Via Use + +The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit +modules. For example the `gke-storage` module can `use` a +`gke-cluster` module and a `private_service_access` module, as shown in the example above. + +```mermaid +graph TD; + vpc-->|OneToMany|gke-cluster; + gke-cluster-->|OneToMany|gke-node-pool; + gke-node-pool-->|ManyToMany|gke-job-template; + gke-cluster-->|OneToMany|gke-storage; + gke-storage-->|ManyToMany|gke-job-template; +``` + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_mode](#input\_access\_mode) | The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)
Valid access modes:
- ReadWriteOnce
- ReadOnlyMany
- ReadWriteMany
- ReadWriteOncePod | `string` | n/a | yes | +| [capacity\_gb](#input\_capacity\_gb) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | +| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [mount\_options](#input\_mount\_options) | Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class. | `string` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [pv\_mount\_path](#input\_pv\_mount\_path) | Path within the container at which the volume should be mounted. Must not contain ':'. | `string` | `"/data"` | no | +| [pvc\_count](#input\_pvc\_count) | How many PersistentVolumeClaims that will be created | `number` | `1` | no | +| [sc\_reclaim\_policy](#input\_sc\_reclaim\_policy) | Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted.
[More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming)
Supported value:
- Retain
- Delete | `string` | n/a | yes | +| [sc\_topology\_zones](#input\_sc\_topology\_zones) | Zone location that allow the volumes to be dynamically provisioned. | `list(string)` | `null` | no | +| [sc\_volume\_binding\_mode](#input\_sc\_volume\_binding\_mode) | Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound.
Supported value:
- Immediate
- WaitForFirstConsumer | `string` | `"WaitForFirstConsumer"` | no | +| [storage\_type](#input\_storage\_type) | The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview)
to used. This module currently support dynamic provisioning for the below storage options
- Parallelstore
- Hyperdisk-balanced
- Hyperdisk-throughput
- Hyperdisk-extreme | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object that describes a k8s PVC created by this module. | + diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/main.tf new file mode 100644 index 0000000000..9c9a641f79 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/main.tf @@ -0,0 +1,86 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-storage", ghpc_role = "file-system" }) +} + +locals { + storage_type = lower(var.storage_type) + storage_class_name = "${local.storage_type}-sc" + pvc_name_prefix = "${local.storage_type}-pvc" +} + +check "private_vpc_connection_peering" { + assert { + condition = lower(var.storage_type) != "parallelstore" ? true : var.private_vpc_connection_peering != null + error_message = <<-EOT + Parallelstore must be run within the same VPC as the GKE cluster and have private services access enabled. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access. + If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). + EOT + } +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + # count = var.pvc_count + apply_manifests = flatten( + [ + # create StorageClass in the cluster + { + content = templatefile( + "${path.module}/storage-class/${local.storage_class_name}.yaml.tftpl", + { + name = local.storage_class_name + labels = local.labels + volume_binding_mode = var.sc_volume_binding_mode + reclaim_policy = var.sc_reclaim_policy + topology_zones = var.sc_topology_zones + }) + }, + var.namespace != "default" ? [{ + content = templatefile( + "${path.module}/persistent-volume-claim/namespace.yaml.tftpl", + { + namespace = var.namespace + }) + }] : [], + # create PersistentVolumeClaim in the cluster + flatten([ + for idx in range(var.pvc_count) : [ + { + content = templatefile( + "${path.module}/persistent-volume-claim/${(local.pvc_name_prefix)}.yaml.tftpl", + { + pvc_name = "${local.pvc_name_prefix}-${idx}" + labels = local.labels + capacity = "${var.capacity_gb}Gi" + access_mode = var.access_mode + storage_class_name = local.storage_class_name + namespace = var.namespace + } + ) + } + ] + ]) + ]) +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml new file mode 100644 index 0000000000..8722823274 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf new file mode 100644 index 0000000000..ce80cdb266 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "persistent_volume_claims" { + description = "An object that describes a k8s PVC created by this module." + value = flatten([ + for idx in range(var.pvc_count) : [{ + name = "${local.pvc_name_prefix}-${idx}" + namespace = var.namespace + mount_path = "${var.pv_mount_path}/${local.pvc_name_prefix}-${idx}" + mount_options = var.mount_options + storage_type = local.storage_type + }] + ]) +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl new file mode 100644 index 0000000000..fa7647e33f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl new file mode 100644 index 0000000000..893b5e7103 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + namespace: ${namespace} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl new file mode 100644 index 0000000000..46e1f023d3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl @@ -0,0 +1,25 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + type: hyperdisk-balanced + provisioned-throughput-on-create: "250Mi" + provisioned-iops-on-create: "7000" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl new file mode 100644 index 0000000000..445020d001 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl @@ -0,0 +1,24 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + %{~ endfor ~} + type: hyperdisk-extreme + provisioned-iops-on-create: "50000" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl new file mode 100644 index 0000000000..ec404aec45 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl @@ -0,0 +1,24 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + type: hyperdisk-throughput + provisioned-throughput-on-create: "250Mi" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl new file mode 100644 index 0000000000..e6b8ea8d3e --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl @@ -0,0 +1,21 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: parallelstore.csi.storage.gke.io +parameters: +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/variables.tf new file mode 100644 index 0000000000..dba1c33b77 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/variables.tf @@ -0,0 +1,144 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" + type = string +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} + +variable "storage_type" { + description = <<-EOT + The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview) + to used. This module currently support dynamic provisioning for the below storage options + - Parallelstore + - Hyperdisk-balanced + - Hyperdisk-throughput + - Hyperdisk-extreme + EOT + type = string + nullable = false + validation { + condition = var.storage_type == null ? false : contains(["parallelstore", "hyperdisk-balanced", "hyperdisk-throughput", "hyperdisk-extreme"], lower(var.storage_type)) + error_message = "Allowed string values for var.storage_type are \"Parallelstore\", \"Hyperdisk-balanced\", \"Hyperdisk-throughput\", \"Hyperdisk-extreme\"." + } +} + +variable "access_mode" { + description = <<-EOT + The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) + Valid access modes: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + EOT + type = string + nullable = false + validation { + condition = var.access_mode == null ? false : contains(["readwriteonce", "readonlymany", "readwritemany", "readwriteoncepod"], lower(var.access_mode)) + error_message = "Allowed string values for var.access_mode are \"ReadWriteOnce\", \"ReadOnlyMany\", \"ReadWriteMany\", \"ReadWriteOncePod\"." + } +} + +variable "sc_volume_binding_mode" { + description = <<-EOT + Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound. + Supported value: + - Immediate + - WaitForFirstConsumer + EOT + type = string + default = "WaitForFirstConsumer" + validation { + condition = var.sc_volume_binding_mode == null ? true : contains(["immediate", "waitforfirstconsumer"], lower(var.sc_volume_binding_mode)) + error_message = "Allowed string values for var.sc_volume_binding_mode are \"Immediate\", \"WaitForFirstConsumer\"." + } +} + +variable "sc_reclaim_policy" { + description = <<-EOT + Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted. + [More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming) + Supported value: + - Retain + - Delete + EOT + type = string + nullable = false + validation { + condition = var.sc_reclaim_policy == null ? true : contains(["retain", "delete"], lower(var.sc_reclaim_policy)) + error_message = "Allowed string values for var.sc_reclaim_policy are \"Retain\", \"Delete\"." + } +} + +variable "sc_topology_zones" { + description = "Zone location that allow the volumes to be dynamically provisioned." + type = list(string) + default = null +} + +variable "pvc_count" { + description = "How many PersistentVolumeClaims that will be created" + type = number + default = 1 +} + +variable "pv_mount_path" { + description = "Path within the container at which the volume should be mounted. Must not contain ':'." + type = string + default = "/data" + validation { + condition = var.pv_mount_path == null ? true : !strcontains(var.pv_mount_path, ":") + error_message = "pv_mount_path must not contain ':', please correct it and retry" + } +} + +variable "mount_options" { + description = "Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class." + type = string + default = null +} + +variable "capacity_gb" { + description = "The storage capacity with which to create the persistent volume." + type = number +} + +variable "private_vpc_connection_peering" { + description = <<-EOT + The name of the VPC Network peering connection. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access and + If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). + EOT + type = string + default = null +} + +variable "namespace" { + description = "Kubernetes namespace to deploy the storage PVC/PV" + type = string + default = "default" +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/versions.tf new file mode 100644 index 0000000000..bcc803e41e --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/versions.tf @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.74.0" + } +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/README.md b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/README.md new file mode 100644 index 0000000000..28530a379f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/README.md @@ -0,0 +1,289 @@ +## Description + +This module creates a [Managed Lustre](https://cloud.google.com/managed-lustre) +instance. Managed Lustre is a high performance network file system that can be +mounted to one or more VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +### Supported Operating Systems + +A Managed Lustre instance can be used with Slurm cluster or compute +VM running Ubuntu 20.04, 22.04 or Rocky Linux 8 (including the HPC flavor). + +### Managed Lustre Access + +Managed Lustre must be enabled for your project by Google staff. Please contact +your sales representative for further steps. + +### Example - New VPC + +For Managed Lustre instance, the snippet below creates new VPC and configures +private-service-access for this newly created network. Both items are required +to be passed to the Lustre module to ensure that they're built in order and +that the correct subnetwork has private service access. + +```yaml + - id: network + source: modules/network/vpc + + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] +``` + +### Example - Slurm + +When using Slurm you must take into consideration whether or not you are using +an official image from the `schedmd-slurm-public` project or building your own. +The Lustre client modules are pre-installed in the official images. With the +official images, Lustre can be used as follows: + +```yaml +- id: managed_lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + +# Other modules: nodesets, partitions, login, etc. + +- id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: + - network + - lustre_partition + - managed_lustre + - slurm_login + settings: + machine_type: n2-standard-4 + enable_controller_public_ips: true +``` + +For custom images you must install the modules during the image build as the +Slurm cluster will not run the installation script like it does for the +standard VMs. + +Assuming you have a startup script for the Slurm image building, you can add +this Ansible playbook to correctly install the Lustre drivers into the image +(for Slurm-GCP versions greater than 6.10.0): + +```yaml +- type: data + destination: /var/tmp/slurm_vars.json + content: | + { + "reboot": false, + "install_cuda": false, + "install_gcsfuse": true, + "install_lustre": false, + "install_managed_lustre": true, + "install_nvidia_repo": true, + "install_ompi": true, + "allow_kernel_upgrades": false, + "monitoring_agent": "cloud-ops", + } +``` + +The `install_managed_lustre: true` line specifies that slurm-gcp should install +the correct modules within the slurm image. This runner should be placed +ahead of the script that calls the ansible build of the slurm-gcp image. + +### Example - Existing VPC + +If you want to use existing network with private-service-access configured, you need +to manually provide `private_vpc_connection_peering` to the Managed Lustre module. +You can get this details from the Google Cloud Console UI in `VPC network peering` +section. Below is the example of using existing network and creating Managed Lustre. +If existing network is not configured with private-service-access, you can follow +[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) +to set it up. + +```yaml + - id: network + source: modules/network/pre-existing-vpc + settings: + network_name: // Add network name + subnetwork_name: // Add subnetwork name + + - id: lustre + source: modules/file-system/managed-lustre + use: [network] + settings: + private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" +``` + +### Example - GKE compatibility + +By default the Managed Lustre instance that is deployed is not compatible with +GKE. To enable the compatibility use the `gke_support_enabled: true` option. +This creates a file `/etc/modprobe/lnet.conf` that changes the listening port +to 6988. + +```yaml + - id: managed-lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + gke_support_enabled: true +``` + +> [!WARNING] +> +> 1. VMs cannot connect to both GKE compatible and GKE incompatible lustre +> instances at the same time as they connect to different ports. Lustre can +> only listen to one port at a time. +> +> 2. Setting `gke_support_enabled: true` will not affect Slurm nodes, GKE +> compatibility must be built into the Slurm image. + +### Example - Importing data from GSC Bucket + +One option with the Managed Lustre instance is to import data from a GSC bucket +upon the lustre instance creation. To do this, use the `import_gcs_bucket_uri` +variable to dictate the bucket to pull data from. The data will be imported +under the directory specified by `local_mount` (`/shared` if unspecified). + +> [!NOTE] +> +> 1. This is a one way operation. Once the data has been copied to the lustre +> instance it will not be updated with any changes made to the GCS bucket. +> +> 2. Once the lustre instance has been created in Terraform, the copy process +> will proceed in the background. Data may not be appear in the mounted +> directory for a period of time after the deployment has completed (see below). + +```yaml +- id: managed_lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + import_gcs_bucket_uri: gs:// +``` + +> [!WARNING] +> Please follow [this guide](https://cloud.google.com/managed-lustre/docs/transfer-data#required_permissions) +> to set up the correct IAM permissions for importing data from GCS to lustre. +> Without this, the copy process may fail silently leaving an empty lustre +> instance. + +If an import is requested, gcluster will output a json response similar to: + +```json +{ + "name": "projects//locations//operations/", + "metadata": { + "@type": "type.googleapis.com/google.cloud.lustre.v1.ImportDataMetadata", + "createTime": "", + "target": "projects//locations//instances/", + "requestedCancellation": false, + "apiVersion": "v1" + }, + "done": false +} +``` + +You can retrieve more information about the transfer using the following +command, substituting with values from the json response above: + +```bash +gcloud lustre operations describe --location --project +``` + +This will provide information on if the transfer is complete or if any errors +have occurred. See more at +[Get operation](https://cloud.google.com/managed-lustre/docs/transfer-data#get_operation). + +## License + + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 6.27.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.27.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_lustre_instance.lustre_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/lustre_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | +| [google_storage_bucket.lustre_import_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the Lustre instance if no name is specified. | `string` | n/a | yes | +| [description](#input\_description) | Description of the created Lustre instance. | `string` | `"Lustre Instance"` | no | +| [gke\_support\_enabled](#input\_gke\_support\_enabled) | Set to true to create Managed Lustre instance with GKE compatibility.
Note: This does not work with Slurm, the Slurm image must be built with
the correct compatibility. | `bool` | `false` | no | +| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to managed lustre. Data will
be imported to the local\_mount directory. Changing this value will not
trigger a redeployment, to prevent data deletion. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the Managed Lustre instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mount](#input\_local\_mount) | Local mount point for the Managed Lustre instance. | `string` | `"/shared"` | no | +| [mount\_options](#input\_mount\_options) | Mounting options for the file system. | `string` | `"defaults,_netdev"` | no | +| [name](#input\_name) | Name of the Lustre instance | `string` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | Network self-link this instance will be on, required for checking private service access | `string` | n/a | yes | +| [per\_unit\_storage\_throughput](#input\_per\_unit\_storage\_throughput) | Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000. | `number` | `500` | no | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | ID of project in which Lustre instance will be created. | `string` | n/a | yes | +| [remote\_mount](#input\_remote\_mount) | Remote mount point of the Managed Lustre instance | `string` | n/a | yes | +| [size\_gib](#input\_size\_gib) | Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations | `number` | `36000` | no | +| [zone](#input\_zone) | Location for the Lustre instance. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | +| [install\_managed\_lustre\_client](#output\_install\_managed\_lustre\_client) | Script for installing Managed Lustre client | +| [lustre\_id](#output\_lustre\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | +| [network\_storage](#output\_network\_storage) | Describes a Managed Lustre instance. | + diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/main.tf new file mode 100644 index 0000000000..a969c53673 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/main.tf @@ -0,0 +1,104 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "managed-lustre", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +data "google_compute_network_peering" "private_peering" { + name = var.private_vpc_connection_peering + network = var.network_self_link +} + +locals { + server_ip = split(":", google_lustre_instance.lustre_instance.mount_point)[0] + remote_mount = split(":", google_lustre_instance.lustre_instance.mount_point)[1] + fs_type = "lustre" + mount_options = var.mount_options + instance_id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + destination_path = "/" + + install_managed_lustre_client_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-managed-lustre-client.sh" + "destination" = "install-managed-lustre-client${replace(var.local_mount, "/", "_")}.sh" + "args" = var.gke_support_enabled ? "1" : "0" + } + mount_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" + } + + bucket_count = try(length(data.google_storage_bucket.lustre_import_bucket), 0) +} + +data "google_storage_bucket" "lustre_import_bucket" { + count = try(length(var.import_gcs_bucket_uri) > 0, false) ? 1 : 0 + + name = split("//", var.import_gcs_bucket_uri)[1] +} + +resource "google_lustre_instance" "lustre_instance" { + project = var.project_id + + description = var.description + instance_id = local.instance_id + location = var.zone + + filesystem = var.remote_mount + capacity_gib = var.size_gib + per_unit_storage_throughput = var.per_unit_storage_throughput + + labels = local.labels + network = var.network_id + + gke_support_enabled = var.gke_support_enabled + + timeouts { + create = "1h" + update = "1h" + delete = "1h" + } + + depends_on = [var.private_vpc_connection_peering, data.google_storage_bucket.lustre_import_bucket] + + lifecycle { + precondition { + condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" + error_message = "The subnetwork that the lustre instance is hosted on must have private service access." + } + } + + provisioner "local-exec" { + command = < 0 ]]; then + curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + -d '{"gcsPath": {"uri":"${coalesce(var.import_gcs_bucket_uri, "gs://")}"}, "lustrePath": {"path":"${local.destination_path}"}}' \ + https://lustre.googleapis.com/v1/projects/${var.project_id}/locations/${var.zone}/instances/${local.instance_id}:importData + fi + EOF + interpreter = ["bash", "-c"] + } +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml new file mode 100644 index 0000000000..66da9827b6 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - lustre.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf new file mode 100644 index 0000000000..6de815524a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a Managed Lustre instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = local.mount_options + client_install_runner = local.install_managed_lustre_client_runner + mount_runner = local.mount_runner + } +} + +output "install_managed_lustre_client" { + description = "Script for installing Managed Lustre client" + value = file("${path.module}/scripts/install-managed-lustre-client.sh") +} + +output "lustre_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" + value = google_lustre_instance.lustre_instance.id +} + +output "capacity_gib" { + description = "File share capacity in GiB." + value = google_lustre_instance.lustre_instance.capacity_gib +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh new file mode 100644 index 0000000000..878130ab47 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install Managed Lustre client modules +# Based on these instructions: https://cloud.google.com/managed-lustre/docs/connect-from-compute-engine + +# The client modules currently only support Rocky 8, and Ubuntu 20.04/22.04 + +set -e + +GKE_ENABLED=$1 + +# Update lnet to enable GKE supported Lustre instance +if [[ $GKE_ENABLED == "1" ]]; then + if [[ -f "/etc/modprobe.d/lnet.conf" ]] && grep -Fq "options lnet accept_port" /etc/modprobe.d/lnet.conf; then + echo "Lnet accept port already set, continuing without updating /etc/modprobe.d/lnet.conf" + else + echo "options lnet accept_port=6988" >>/etc/modprobe.d/lnet.conf + fi +fi + +if grep -q lustre /proc/filesystems; then + echo "Skipping managed lustre client install as it is already supported" + exit 0 +fi + +# Get distro information +. /etc/os-release +DIST="NA" +if [[ $NAME == *"Ubuntu"* ]]; then + if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then + DIST="Ubuntu" + fi +elif [[ $NAME == *"Rocky"* ]]; then + if [[ $VERSION_ID == "8"* ]]; then + DIST="Rocky" + fi +fi + +if [[ ${DIST} == "Ubuntu" ]]; then + KEY_LOC=/etc/apt/keyrings + KEY_NAME=gcp-ar-repo.gpg + # Download new repo key + mkdir -p "${KEY_LOC}" + wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null + + # Set up apt repo + echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list + + # Install modules + apt update + apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) +elif [[ ${DIST} == "Rocky" ]]; then + # Set up yum repo + touch /etc/yum.repos.d/artifact-registry.repo + tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF + [lustre-client-rocky-8] + name=lustre-client-rocky-8 + baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + # Install modules + yum makecache + yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client +fi + +if [[ $DIST != "NA" ]]; then + # Load the new lustre client module + modprobe lustre +fi diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf new file mode 100644 index 0000000000..65607af66d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf @@ -0,0 +1,131 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which Lustre instance will be created." + type = string +} + +variable "description" { + description = "Description of the created Lustre instance." + type = string + default = "Lustre Instance" +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the Lustre instance if no name is specified." + type = string +} + +variable "zone" { + description = "Location for the Lustre instance." + type = string +} + +variable "name" { + description = "Name of the Lustre instance" + type = string +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is connected given in the format: + `projects//global/networks/`" + EOT + type = string + nullable = false + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "network_self_link" { + description = "Network self-link this instance will be on, required for checking private service access" + type = string + nullable = false +} + +variable "remote_mount" { + description = "Remote mount point of the Managed Lustre instance" + type = string + nullable = false +} + +variable "local_mount" { + description = "Local mount point for the Managed Lustre instance." + type = string + default = "/shared" +} + +variable "size_gib" { + description = "Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations" + type = number + default = 36000 +} + +variable "per_unit_storage_throughput" { + description = "Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000." + type = number + default = 500 +} + +variable "labels" { + description = "Labels to add to the Managed Lustre instance. Key-value pairs." + type = map(string) +} + +variable "mount_options" { + description = "Mounting options for the file system." + type = string + default = "defaults,_netdev" +} + +variable "private_vpc_connection_peering" { + description = <<-EOT + The name of the VPC Network peering connection. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access and + If using existing VPC with private-service-access enabled, set this manually." + EOT + type = string + nullable = false +} + +variable "gke_support_enabled" { + description = <<-EOT + Set to true to create Managed Lustre instance with GKE compatibility. + Note: This does not work with Slurm, the Slurm image must be built with + the correct compatibility. + EOT + type = bool + nullable = false + default = false +} + +variable "import_gcs_bucket_uri" { + description = <<-EOT + The name of the GCS bucket to import data from to managed lustre. Data will + be imported to the local_mount directory. Changing this value will not + trigger a redeployment, to prevent data deletion. + EOT + type = string + default = null + + validation { + condition = startswith(coalesce(var.import_gcs_bucket_uri, "gs://"), "gs://") + error_message = "The GCS bucket uri must start with 'gs://'" + } +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf new file mode 100644 index 0000000000..2322c9a8fd --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.27.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" + } + + required_version = ">= 1.3.0" +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/README.md b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/README.md new file mode 100644 index 0000000000..82332f3406 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/README.md @@ -0,0 +1,193 @@ +## Description + +This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) +storage pool. + +NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. +NetApp Volume provides: + +- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x +- a [rich feature set][service-levels] +- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) +- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data +- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost + +Support for NetApp Volumes is split into two modules. + +- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. +- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +### NetApp storage pool service levels + +The netapp-storage-pool module currently supports the following NetApp Volumes [service levels][service-levels]: + +- Standard: 16 KiBps throughput per provisioned KiB of volume capacity. +- Premium: 64 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. +- Extreme: 128 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. + +Check the [service level matrix][service-levels] for additional information on capability differences between service levels. Flex service levels are currently not supported, but you can connect to existing Flex volumes using the [pre-existing-network-storage module][pre-existing]. + +### On-boarding NetApp Volumes +NetApp Volumes uses [Private Service Access](https://cloud.google.com/vpc/docs/private-services-access) (PSA) to connect volumes to your network. Before you create a storage pool, make sure to [connect NetApp Volumes to your network](https://cloud.google.com/netapp/volumes/docs/get-started/configure-access/networking). + +Example of creating a storage pool using a new network: + +```yaml +deployment_groups: +- group: primary + modules: + - id: network + source: modules/network/vpc + settings: + region: $(vars.region) + + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + service_name: "netapp.servicenetworking.goog" + deletion_policy: "ABANDON" + + - id: netapp_pool + source: modules/file-system/netapp-storage-pool + use: [network, private_service_access] + settings: + pool_name: $(vars.deployment_name)-eda-pool + capacity_gib: 20000 + service_level: "EXTREME" + region: $(vars.region) +``` + +Example of creating a storage pool using an existing network which was already PSA-peered with NetApp Volume: + +```yaml +deployment_groups: + - group: primary + modules: + - id: network + source: modules/network/pre-existing-vpc + settings: + project_id: $(vars.project_id) + region: $(vars.region) + network_name: $(vars.network) + + - id: netapp_pool + source: modules/file-system/netapp-storage-pool + use: [network] + settings: + pool_name: "eda-pool" + capacity_gib: 20000 + service_level: "EXTREME" + region: $(vars.region) +``` + +### Storage pool example + +The following example shows all available parameters in use: + +```yaml + - id: netapp_pool + source: modules/file-system/netapp-storage-pool + use: [network, private_service_access] + settings: + pool_name: "mypool" + region: "us-west4" + capacity_gib: 2048 + service_level: "EXTREME" + active_directory_policy: "projects/myproject/locations/us-east4/activeDirectories/my-ad" + cmek_policy: "projects/myproject/locations/us-east4/kmsConfigs/my-cmek-policy" + ldap_enabled: false + allow_auto_tiering: false + description: "Demo storage pool" + labels: + owner: bob +``` + +### NetApp Volumes quota + +Your project must have unused quota for NetApp Volumes in the region you will +provision the storage pool. This can be found by browsing to the [Quota tab within IAM & Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. +Please note that there are separate quota limits for Standard and Premium/Extreme service levels. + +See also NetApp Volumes [default quotas](https://cloud.google.com/netapp/volumes/docs/quotas#netapp-volumes-default-quotas). + +[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels +[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering +[pre-existing]: ../pre-existing-network-storage/README.md +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [google](#requirement\_google) | >= 6.45.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.45.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_netapp_storage_pool.netapp_storage_pool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_storage_pool) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [active\_directory\_policy](#input\_active\_directory\_policy) | The ID of the Active Directory policy to apply to the storage pool in the format:
`projects//locations//activeDirectoryPolicies/` | `string` | `null` | no | +| [allow\_auto\_tiering](#input\_allow\_auto\_tiering) | Whether to allow automatic tiering for the storage pool. | `bool` | `false` | no | +| [capacity\_gib](#input\_capacity\_gib) | The capacity of the storage pool in GiB. | `number` | `2048` | no | +| [cmek\_policy](#input\_cmek\_policy) | The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format:
`projects//locations//kmsConfigs/` | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used as name of the NetApp storage pool if no name is specified. | `string` | n/a | yes | +| [description](#input\_description) | A description of the NetApp storage pool. | `string` | `""` | no | +| [labels](#input\_labels) | Labels to add to the NetApp storage pool. Key-value pairs. | `map(string)` | n/a | yes | +| [ldap\_enabled](#input\_ldap\_enabled) | Whether to enable LDAP for the storage pool. | `bool` | `false` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | Network self-link the pool will be on, required for checking private service access | `string` | n/a | yes | +| [pool\_name](#input\_pool\_name) | The name of the storage pool. Leave empty to generate name based on deployment name. | `string` | `null` | no | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the private VPC connection peering. | `string` | `"sn-netapp-prod"` | no | +| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | +| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | +| [service\_level](#input\_service\_level) | The service level of the storage pool. | `string` | `"PREMIUM"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gb](#output\_capacity\_gb) | Storage pool capacity in GiB. | +| [netapp\_storage\_pool\_id](#output\_netapp\_storage\_pool\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}` | + diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/main.tf new file mode 100644 index 0000000000..b9d63c11c3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/main.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "netapp-storage-pool", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +data "google_compute_network_peering" "private_peering" { + name = var.private_vpc_connection_peering + network = var.network_self_link +} + +resource "google_netapp_storage_pool" "netapp_storage_pool" { + project = var.project_id + + name = var.pool_name != null ? var.pool_name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + location = var.region + network = var.network_id + service_level = var.service_level + capacity_gib = var.capacity_gib + + active_directory = var.active_directory_policy + kms_config = var.cmek_policy + ldap_enabled = var.ldap_enabled + allow_auto_tiering = var.allow_auto_tiering + + description = var.description + labels = local.labels + + depends_on = [data.google_compute_network_peering.private_peering] + + lifecycle { + precondition { + condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" + error_message = "The network for the storage pool must have private service access." + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml new file mode 100644 index 0000000000..7a5291f9d5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - netapp.googleapis.com + - servicenetworking.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf new file mode 100644 index 0000000000..91379631c6 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "netapp_storage_pool_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}`" + value = google_netapp_storage_pool.netapp_storage_pool.id +} + +output "capacity_gb" { + description = "Storage pool capacity in GiB." + value = google_netapp_storage_pool.netapp_storage_pool.capacity_gib +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf new file mode 100644 index 0000000000..04f19fd3fb --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf @@ -0,0 +1,133 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which the NetApp storage pool will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the deployment, used as name of the NetApp storage pool if no name is specified." + type = string +} + +variable "region" { + description = "Location for NetApp storage pool." + type = string +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "network_self_link" { + description = "Network self-link the pool will be on, required for checking private service access" + type = string + nullable = false +} + +variable "private_vpc_connection_peering" { + description = "The name of the private VPC connection peering." + type = string + default = "sn-netapp-prod" +} + +variable "pool_name" { + description = "The name of the storage pool. Leave empty to generate name based on deployment name." + type = string + default = null +} + +variable "service_level" { + description = "The service level of the storage pool." + type = string + default = "PREMIUM" + validation { + condition = contains(["STANDARD", "PREMIUM", "EXTREME"], var.service_level) + error_message = "Allowed values for service_level are 'STANDARD', 'PREMIUM', or 'EXTREME'." + } +} + +variable "capacity_gib" { + description = "The capacity of the storage pool in GiB." + type = number + default = 2048 + validation { + condition = var.capacity_gib >= 2048 + error_message = "The minimum capacity for the storage pool is 2048 GiB." + } +} + +variable "active_directory_policy" { + description = <<-EOT + The ID of the Active Directory policy to apply to the storage pool in the format: + `projects//locations//activeDirectoryPolicies/` + EOT + type = string + default = null + validation { + condition = var.active_directory_policy == null ? true : length(split("/", var.active_directory_policy)) == 6 + error_message = "The active directory policy must be provided in the following format: projects//locations//activeDirectoryPolicies/." + } +} + +variable "cmek_policy" { + description = <<-EOT + The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format: + `projects//locations//kmsConfigs/` + EOT + type = string + default = null + validation { + condition = var.cmek_policy == null ? true : length(split("/", var.cmek_policy)) == 6 + error_message = "The CMEK policy must be provided in the following format: projects//locations//kmsConfigs/." + } +} + +variable "ldap_enabled" { + description = "Whether to enable LDAP for the storage pool." + type = bool + default = false +} + +variable "allow_auto_tiering" { + description = "Whether to allow automatic tiering for the storage pool." + type = bool + default = false +} + +variable "description" { + description = "A description of the NetApp storage pool." + type = string + default = "" + validation { + condition = length(var.description) <= 2048 + error_message = "NetApp storage pool description must be 2048 characters or fewer" + } +} + +variable "labels" { + description = "Labels to add to the NetApp storage pool. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf new file mode 100644 index 0000000000..f6501116cd --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.45.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" + } + + required_version = ">= 1.5.7" +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/README.md b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/README.md new file mode 100644 index 0000000000..6aaaf0cb05 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/README.md @@ -0,0 +1,201 @@ +## Description + +This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) +volume. + +NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. +NetApp Volume provides: + +- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x +- a [rich feature set][service-levels] +- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) +- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data +- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost + +Support for NetApp Volumes is split into two modules. + +- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. +- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +## Deletion protection +The netapp-volume module currently doesn't implement volume deletion protection. If you create a volume with Cluster Toolkit by using this module, Cluster Toolkit will also delete it when you run `gcluster destroy`. All the data in the volume will be gone. If you want to retain the volume instead, it is advised to [use existing volumes not created by Cluster Toolkit](#using-existing-volumes-not-created-by-cluster-toolkit). + +## Volumes overview +Volumes are filesystem containers which can be shared using NFS or SMB filesharing protocols. Volumes *live* inside of [storage pools](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview), which can be provisioned using the [netapp-storage-pool] module. Volumes inherit fundamental settings from the pool. They *consume* capacity provided by the pool. You can create one or multiple volumes *inside* a pool. + +[netapp-storage-pool]: ../netapp-storage-pool/README.md +[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels +[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering +[pre-existing]: ../pre-existing-network-storage/README.md +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## Volume examples +The following examples show the use of netapp-volume. They builds on top of an storage pool which can be provisioned using the [netapp-storage-pool][netapp-storage-pool] module. + +### Example with minimal parameters + +```yaml + - id: home_volume + source: modules/file-system/netapp-volume + use: [netapp_pool] # Create this pool using the netapp-storage-pool module + settings: + volume_name: "eda-home" + capacity_gib: 1024 # Size up to available capacity in the pool + local_mount: "/eda-home" # Mount point at client when client uses USE directive + protocols: ["NFSV3"] + region: $(vars.region) + # Default export policy exports to "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" and no_root_squash +``` + +### Example with all parameters + +```yaml + - id: shared_volume + source: modules/file-system/netapp-volume + use: [netapp_pool] # Create this pool using the netapp-storage-pool module + settings: + volume_name: "eda-shared" + capacity_gib: 25000 # Size up to available capacity in the pool + large_capacity: true + local_mount: "/shared" # Mount point at client when client uses USE directive + mount_options: "rw" # Allows customizing mount options for special workloads + protocols: ["NFSV3","NFSV4"] # List of protocols. ["NFSV3], ["NFSv4] or ["NFSV3, "NFSV4"] + region: $(vars.region) + unix_permissions: "0777" # Specify default permissions for roo inode owned by root:root + # If no export policy is specified, a permissive default policy will be applied, which is: + # allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" # RFC1918 + # has_root_access = true # no_root_squash enabled + # access_type = "READ_WRITE" + export_policy: + - allowed_clients: "10.10.20.8,10.10.20.9" + has_root_access: true # no_root_squash enabled + access_type: "READ_WRITE" + nfsv3: false # allow only NFSv4 for these hosts + nfsv4: true + - allowed_clients: "10.0.0.0/8" + has_root_access: false # no_root_squash disabled + access_type: "READ_WRITE" + nfsv3: true # allow only NFSv3 for these hosts + nfsv4: false + tiering_policy: # Enable auto-tiering. Requires auto-tiering enabled storage pool + tier_action: "ENABLED" + cooling_threshold_days: 31 # tier data blocks which have not been touched for 31 days + + description: "Shared volume for EDA job" + labels: + owner: bob +``` + +## Protocol support +Since Cluster Toolkit is currently built to provision Linux-based compute clusters, this module supports NFSv3 and NFSv4.1 only. SMB is blocked. + +## Large volumes +Volumes larger than 15 TiB can be created as [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). Such volumes can grow up to 3 PiB and can scale read performance up to 29 GiBps. They provide six IP addresses to the volume. They are exported via the `server_ips` output. When connecting a large volume to a client using the USE directive, cluster toolkit currently uses the first IP only. This will be improved in the future. + +This feature is allow-listed GA. To request allow-listing, see [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). + +## Auto-tiering support +For auto-tiering enabled storage pools you can enable auto-tiering on the volume. For more information, see [manage auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering). + +## Using existing volumes not created by Cluster Toolkit +NetApp Volumes volumes are regular NFS exports. You can use the [pre-existing-network-storage] module to integrate them into Cluster Toolkit. + +Example code: + +```yaml +- id: homefs + source: modules/file-system/pre-existing-network-storage + settings: + server_ip: ## Set server IP here ## + remote_mount: nfsshare + local_mount: /home + fs_type: nfs +``` + +This creates a resource in Cluster Toolkit which references the specified NFS export, which will be mounted at `/home` by clients which mount if via USE directive. + +Note that the `server_ip` must be known before deployment and this module does not allow +to specify a list of IPs for large volumes. + +[pre-existing-network-storage]: ../pre-existing-network-storage/README.md + +## FlexCache support +NetApp FlexCache technology accelerates data access, reduces WAN latency and lowers WAN bandwidth costs for read-intensive workloads, especially where clients need to access the same data repeatedly. When you create a FlexCache volume, you create a remote cache of an already existing (origin) volume that contains only the actively accessed data (hot data) of the origin volume. + +The FlexCache support in Google Cloud NetApp Volumes allows you to provision a cache volume in your Google network to improve performance for hybrid cloud environments. A FlexCache volume can help you transition workloads to the hybrid cloud by caching data from an on-premises data center to cloud. + +Deploying FlexCache volumes requires manual steps on the ONTAP origin side, which are not automated. Therefore this module has no support to deploy FlexCache volumes today. Deploy them manually and use the [pre-existing-network-storage](#using-existing-volumes-not-created-by-cluster-toolkit) instead. + +## License + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.7 | +| [google](#requirement\_google) | >= 6.45.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.45.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_netapp_volume.netapp_volume](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_volume) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [capacity\_gib](#input\_capacity\_gib) | The capacity of the volume in GiB. | `number` | `1024` | no | +| [description](#input\_description) | A description of the NetApp volume. | `string` | `""` | no | +| [export\_policy\_rules](#input\_export\_policy\_rules) | Define NFS export policy. | 
list(object({
    allowed_clients = optional(string)
    has_root_access = optional(bool, false)
    access_type     = optional(string, "READ_WRITE")
    nfsv3           = optional(bool)
    nfsv4           = optional(bool)
  }))
| 
[
  {
    "access_type": "READ_WRITE",
    "allowed_clients": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
    "has_root_access": true
  }
]
| no | +| [labels](#input\_labels) | Labels to add to the NetApp volume. Key-value pairs. | `map(string)` | n/a | yes | +| [large\_capacity](#input\_large\_capacity) | If true, the volume will be created with large capacity.
Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. | `bool` | `false` | no | +| [local\_mount](#input\_local\_mount) | Mountpoint for this volume. | `string` | `"/shared"` | no | +| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"rw,hard,rsize=65536,wsize=65536,tcp"` | no | +| [netapp\_storage\_pool\_id](#input\_netapp\_storage\_pool\_id) | The ID of the NetApp storage pool to use for the volume. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | +| [protocols](#input\_protocols) | The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported. | `list(string)` | 
[
  "NFSV3"
]
| no | +| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | +| [tiering\_policy](#input\_tiering\_policy) | Define the tiering policy for the NetApp volume. | 
object({
    tier_action            = optional(string)
    cooling_threshold_days = optional(number)
  })
| `null` | no | +| [unix\_permissions](#input\_unix\_permissions) | UNIX permissions for root inode in the volume. | `string` | `"0777"` | no | +| [volume\_name](#input\_volume\_name) | The name of the volume. Needs to be unique within the storage pool. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gb](#output\_capacity\_gb) | Volume capacity in GiB. | +| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | +| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | +| [netapp\_volume\_id](#output\_netapp\_volume\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}` | +| [network\_storage](#output\_network\_storage) | Describes a NetApp Volumes volume. | +| [server\_ips](#output\_server\_ips) | List of IP addresses of the volume. | + diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/main.tf new file mode 100644 index 0000000000..d8345bf347 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/main.tf @@ -0,0 +1,92 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "netapp-volume", ghpc_role = "file-system" }) +} + +# resource "random_id" "resource_name_suffix" { +# byte_length = 4 +# } + +locals { + full_path = split(":", google_netapp_volume.netapp_volume.mount_options[0].export_full) + server_ip = local.full_path[0] + remote_mount = local.full_path[1] + # Large volumes will have 6 IPs + server_ips = [for ip in google_netapp_volume.netapp_volume.mount_options[*].export_full : split(":", ip)[0]] + fs_type = "nfs" + mount_options = var.mount_options + + install_nfs_client_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-nfs-client.sh" + "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" + } + mount_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${join(",", local.server_ips)}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" + } + + split_pool_id = split("/", var.netapp_storage_pool_id) + pool_name = local.split_pool_id[5] +} + +resource "google_netapp_volume" "netapp_volume" { + project = var.project_id + + name = var.volume_name + share_name = var.volume_name + location = var.region + protocols = var.protocols + capacity_gib = var.capacity_gib + large_capacity = var.large_capacity + multiple_endpoints = var.large_capacity == true ? true : null + storage_pool = local.pool_name + unix_permissions = var.unix_permissions + + dynamic "tiering_policy" { + for_each = var.tiering_policy == null ? [] : [0] + content { + cooling_threshold_days = lookup(var.tiering_policy, "cooling_threshold_days", null) + tier_action = lookup(var.tiering_policy, "tier_action", null) + } + } + + description = var.description + labels = local.labels + + dynamic "export_policy" { + for_each = var.export_policy_rules == null ? [] : [0] + content { + dynamic "rules" { + for_each = var.export_policy_rules + content { + access_type = rules.value.access_type + allowed_clients = rules.value.allowed_clients + has_root_access = rules.value.has_root_access + nfsv3 = rules.value.nfsv3 == null ? contains([for p in var.protocols : lower(p)], "nfsv3") : rules.value.nfsv3 + nfsv4 = rules.value.nfsv4 == null ? contains([for p in var.protocols : lower(p)], "nfsv4") : rules.value.nfsv4 + } + } + } + } + + depends_on = [var.netapp_storage_pool_id] +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/metadata.yaml new file mode 100644 index 0000000000..e4a7aaaa14 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - netapp.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/outputs.tf new file mode 100644 index 0000000000..641eae007a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/outputs.tf @@ -0,0 +1,66 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +output "network_storage" { + description = "Describes a NetApp Volumes volume." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = local.mount_options + client_install_runner = local.install_nfs_client_runner + mount_runner = local.mount_runner + } +} + +output "install_nfs_client" { + description = "Script for installing NFS client" + value = file("${path.module}/scripts/install-nfs-client.sh") +} + +output "install_nfs_client_runner" { + description = "Runner to install NFS client using the startup-script module" + value = local.install_nfs_client_runner +} + +output "mount_runner" { + description = <<-EOT + Runner to mount the file-system using an ansible playbook. The startup-script + module will automatically handle installation of ansible. + - id: example-startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(your-fs-id.mount_runner) + ... + EOT + value = local.mount_runner +} + +output "netapp_volume_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}`" + value = google_netapp_volume.netapp_volume.id +} + +output "capacity_gb" { + description = "Volume capacity in GiB." + value = google_netapp_volume.netapp_volume.capacity_gib +} + +output "server_ips" { + description = "List of IP addresses of the volume." + value = local.server_ips +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..1b1595e5a4 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsupported distribution' + return 1 + fi +fi diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh new file mode 100644 index 0000000000..8253d40a24 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IPS=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +# accept a list of colon-separated IPs and randomly pick one to enable load balancing +# In recent changes cluster toolkit doesn't seem to use this file anymore, +# which makes all mounts use the first IP in the list. Needs to be investigated in future. +IFS="," read -r -a arrIPS <<<"${SERVER_IPS}" +rand1=$(od -vAn -t d -N1 /dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/variables.tf new file mode 100644 index 0000000000..272558ff77 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/variables.tf @@ -0,0 +1,133 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which the NetApp storage pool will be created." + type = string +} + +variable "netapp_storage_pool_id" { + description = "The ID of the NetApp storage pool to use for the volume." + type = string + validation { + condition = length(split("/", var.netapp_storage_pool_id)) == 6 + error_message = "The storage pool id must be provided in the following format: projects//locations//storagePools/." + } +} + +variable "region" { + description = "Location for NetApp storage pool." + type = string +} + +variable "volume_name" { + description = "The name of the volume. Needs to be unique within the storage pool." + type = string + default = null +} + +variable "capacity_gib" { + description = "The capacity of the volume in GiB." + type = number + default = 1024 + validation { + condition = var.capacity_gib >= 100 + error_message = "The minimum capacity for the volume is 100 GiB." + } +} + +variable "protocols" { + description = "The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported." + type = list(string) + default = ["NFSV3"] + validation { + condition = alltrue([for p in var.protocols : contains(["NFSV3", "NFSV4"], p)]) + error_message = "Allowed values for protocols are 'NFSV3' or 'NFSV4'." + } +} + +variable "description" { + description = "A description of the NetApp volume." + type = string + default = "" + validation { + condition = length(var.description) <= 2048 + error_message = "NetApp volume description must be 2048 characters or fewer" + } +} + +variable "labels" { + description = "Labels to add to the NetApp volume. Key-value pairs." + type = map(string) +} + +variable "local_mount" { + description = "Mountpoint for this volume." + type = string + default = "/shared" +} + +variable "mount_options" { + description = "NFS mount options to mount file system." + type = string + default = "rw,hard,rsize=65536,wsize=65536,tcp" +} + +variable "large_capacity" { + description = <<-EOT + If true, the volume will be created with large capacity. + Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. + EOT + type = bool + default = false +} + +variable "unix_permissions" { + description = "UNIX permissions for root inode in the volume." + type = string + default = "0777" + validation { + condition = length(var.unix_permissions) <= 4 + error_message = "UNIX permissions must be a 4-digit octal number." + } +} + +variable "tiering_policy" { + description = "Define the tiering policy for the NetApp volume." + type = object({ + tier_action = optional(string) + cooling_threshold_days = optional(number) + }) + default = null +} + +variable "export_policy_rules" { + description = "Define NFS export policy." + type = list(object({ + allowed_clients = optional(string) + has_root_access = optional(bool, false) + access_type = optional(string, "READ_WRITE") + nfsv3 = optional(bool) + nfsv4 = optional(bool) + })) + # Permissive default if user does not specify nfs_export_options. Allow all RFC1918 CIDRS with no_root_squash + default = [{ + allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", + has_root_access = true, + access_type = "READ_WRITE", + }] + nullable = true +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/versions.tf new file mode 100644 index 0000000000..c624d5100b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/versions.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.45.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" + } + + required_version = ">= 1.5.7" +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/README.md b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/README.md new file mode 100644 index 0000000000..0b942f067f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/README.md @@ -0,0 +1,196 @@ +## Description + +This module creates [parallelstore](https://cloud.google.com/parallelstore) +instance. Parallelstore is Google Cloud's first party parallel file system +service based on [Intel DAOS](https://docs.daos.io/v2.2/) + +### Supported Operating Systems + +A parallelstore instance can be used with Slurm cluster or compute +VM running Ubuntu 22.04, debian 12 or HPC Rocky Linux 8. + +### Parallelstore Quota + +To get access to a private preview of Parallelstore APIs, your project needs to +be allowlisted. To set this up, please work with your account representative. + +### Parallelstore mount options + +After parallelstore instance is created, you can specify mount options depending +upon your workload. DAOS is configured to deliver the best user experience for +interactive workloads with aggressive caching. If you are running parallel +workloads concurrently accessing the sane files from multiple client nodes, it +is recommended to disable the writeback cache to avoid cross-client consistency +issues. You can specify different mount options as follows, + +```yaml + - id: parallelstore + source: modules/file-system/parallelstore + use: [network, ps_connect] + settings: + mount_options: "disable-wb-cache,thread-count=20,eq-count=8" +``` + +### Example - New VPC + +For parallelstore instance, Below snippet creates new VPC and configures private-service-access +for this newly created network. + +```yaml + - id: network + source: modules/network/vpc + + # Private Service Access (PSA) requires the compute.networkAdmin role which is + # included in the Owner role, but not Editor. + # PSA is required for all Parallelstore functionality. + # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: parallelstore + source: modules/file-system/parallelstore + use: [network, private_service_access] +``` + +### Example - Existing VPC + +If you want to use existing network with private-service-access configured, you need +to manually provide `private_vpc_connection_peering` to the parallelstore module. +You can get this details from the Google Cloud Console UI in `VPC network peering` +section. Below is the example of using existing network and creating parallelstore. +If existing network is not configured with private-service-access, you can follow +[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) +to set it up. + +```yaml + - id: network + source: modules/network/pre-existing-vpc + settings: + network_name: // Add network name + subnetwork_name: // Add subnetwork name + + - id: parallelstore + source: modules/file-system/parallelstore + use: [network] + settings: + private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" +``` + +### Import data from GCS bucket + +You can import data from your GCS bucket to parallelstore instance. Important to +note that data may not be available to the instance immediately. This depends on +latency and size of data. Below is the example of importing data from bucket. + +```yaml + - id: parallelstore + source: modules/file-system/parallelstore + use: [network] + settings: + import_gcs_bucket_uri: gs://gcs-bucket/folder-path + import_destination_path: /gcs/import/ +``` + +Here you can replace `import_gcs_bucket_uri` with the uri of sub folder within GCS +bucket and `import_destination_path` with local directory within parallelstore +instance. + +### Additional configuration for DAOS agent and dfuse +Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + daos_agent_config: | + credential_config: + cache_expiration: 1m +``` + +Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/parallelstore + settings: + dfuse_environment: + D_LOG_FILE: /tmp/client.log + D_APPEND_PID_TO_LOG: 1 + D_LOG_MASK: debug +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13 | +| [google](#requirement\_google) | >= 6.13.0 | +| [null](#requirement\_null) | ~> 3.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.13.0 | +| [null](#provider\_null) | ~> 3.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_parallelstore_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/parallelstore_instance) | resource | +| [null_resource.hydration](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [daos\_agent\_config](#input\_daos\_agent\_config) | Additional configuration to be added to daos\_config.yml | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. | `string` | n/a | yes | +| [dfuse\_environment](#input\_dfuse\_environment) | Additional environment variables for DFuse process | `map(string)` | `{}` | no | +| [directory\_stripe](#input\_directory\_stripe) | The parallelstore stripe level for directories. | `string` | `null` | no | +| [file\_stripe](#input\_file\_stripe) | The parallelstore stripe level for files. | `string` | `null` | no | +| [import\_destination\_path](#input\_import\_destination\_path) | The name of local path to import data on parallelstore instance from GCS bucket. | `string` | `null` | no | +| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to parallelstore. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to parallel store instance. | `map(string)` | `{}` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/parallelstore"` | no | +| [mount\_options](#input\_mount\_options) | Options describing various aspects of the parallelstore instance. | `string` | `"disable-wb-cache,thread-count=16,eq-count=8"` | no | +| [name](#input\_name) | Name of parallelstore instance. | `string` | `null` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [size\_gb](#input\_size\_gb) | Storage size of the parallelstore instance in GB. | `number` | `12000` | no | +| [zone](#input\_zone) | Location for parallelstore instance. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions to monitor import-data operation from GCS bucket to parallelstore. | +| [network\_storage](#output\_network\_storage) | Describes a parallelstore instance. | + diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/main.tf new file mode 100644 index 0000000000..acc2a0551e --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/main.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "parallelstore", ghpc_role = "file-system" }) +} + +locals { + fs_type = "daos" + server_ip = "" + remote_mount = "" + id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + access_points = jsonencode(google_parallelstore_instance.instance.access_points) + destination_path = var.import_destination_path == null ? "/" : var.import_destination_path + + client_install_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-daos-client.sh" + "destination" = "install_daos_client.sh" + } + + mount_runner = { + "type" = "shell" + "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { + access_points = local.access_points + daos_agent_config = var.daos_agent_config + dfuse_environment = var.dfuse_environment + local_mount = var.local_mount + mount_options = join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) + }) + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + } +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_parallelstore_instance" "instance" { + project = var.project_id + instance_id = local.id + location = var.zone + capacity_gib = var.size_gb + network = var.network_id + file_stripe_level = var.file_stripe + directory_stripe_level = var.directory_stripe + + labels = local.labels + + depends_on = [var.private_vpc_connection_peering] +} + +resource "null_resource" "hydration" { + count = var.import_gcs_bucket_uri != null ? 1 : 0 + + depends_on = [resource.google_parallelstore_instance.instance] + provisioner "local-exec" { + command = "curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -d '{\"source_gcs_bucket\": {\"uri\":\"${var.import_gcs_bucket_uri}\"}, \"destination_parallelstore\": {\"path\":\"${local.destination_path}\"}}' https://parallelstore.googleapis.com/v1beta/projects/${var.project_id}/locations/${var.zone}/instances/${local.id}:importData" + } +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml new file mode 100644 index 0000000000..c0994d15bb --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - parallelstore.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf new file mode 100644 index 0000000000..f6e817ac8a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + operation_instructions = <<-EOT + Data is being imported from GCS bucket to parallelstore instance. It may + not be available immediately. + EOT +} + +output "network_storage" { + description = "Describes a parallelstore instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = var.mount_options + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } + + precondition { + condition = var.import_gcs_bucket_uri != null || var.import_destination_path == null + error_message = <<-EOD + Please specify import_gcs_bucket_uri to import data to parallelstore instance. + EOD + } +} + +output "instructions" { + description = "Instructions to monitor import-data operation from GCS bucket to parallelstore." + value = var.import_gcs_bucket_uri != null ? local.operation_instructions : "Data is not imported from GCS bucket." +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh new file mode 100644 index 0000000000..e96eadb56a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || + { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 +fi + +if [ -x /bin/daos ]; then + echo "DAOS already installed" + daos version +else + # Install the DAOS client library + # The following commands should be executed on each client vm. + ## For Rocky linux 8 / RedHat 8. + if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then + # 1) Add the Parallelstore package repository + cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF + [parallelstore-v2-6-el${OS_VERSION_MAJOR}] + name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 + baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + + ## TODO: Remove disable automatic update script after issue is fixed. + if [ -x /usr/bin/google_disable_automatic_updates ]; then + /usr/bin/google_disable_automatic_updates + fi + dnf clean all + dnf makecache + + # 2) Install daos-client + dnf install -y epel-release # needed for capstone + dnf install -y daos-client + + # 3) Upgrade libfabric + dnf upgrade -y libfabric + + # For Ubuntu 22.04 and debian 12, + elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then + # shellcheck disable=SC2034 + DEBIAN_FRONTEND=noninteractive + + # 1) Add the Parallelstore package repository + curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg + echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list + + apt-get update + + # 2) Install daos-client + apt-get install -y daos-client + + # 3) Create daos_agent.service (comes pre-installed with RedHat) + if ! getent passwd daos_agent >/dev/null 2>&1; then + useradd daos_agent + fi + cat >/etc/systemd/system/daos_agent.service <<-EOF + [Unit] + Description=DAOS Agent + StartLimitIntervalSec=60 + Wants=network-online.target + After=network-online.target + + [Service] + Type=notify + User=daos_agent + Group=daos_agent + RuntimeDirectory=daos_agent + RuntimeDirectoryMode=0755 + ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml + StandardOutput=journal + StandardError=journal + Restart=always + RestartSec=10 + LimitMEMLOCK=infinity + LimitCORE=infinity + StartLimitBurst=5 + + [Install] + WantedBy=multi-user.target + EOF + else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + fi +fi + +exit 0 diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl new file mode 100644 index 0000000000..c6f5d53660 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl @@ -0,0 +1,110 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; } && { [[ "$${OS_VERSION_MAJOR}" = "8" ]] || [[ "$${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "$${OS_ID}" = "ubuntu" ]] && [[ "$${OS_VERSION}" = "22.04" ]]; } || + { [[ "$${OS_ID}" = "debian" ]] && [[ "$${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system $${OS_ID} $${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + +fi + +# Edit agent config +daos_config=/etc/daos/daos_agent.yml + +# rewrite $daos_config from scratch +mv $${daos_config} $${daos_config}.orig + +exclude_fabric_ifaces="" +# Get names of network interfaces not in first PCI slot +# The first PCI slot is a standard network adapter while remaining interfaces +# are typically network cards dedicated to GPU or workload communication +if [[ "$${OS_ID}" == "debian" ]] || [[ "$${OS_ID}" = "ubuntu" ]]; then + extra_interfaces=$(find /sys/class/net/ -not -name 'enp0s*' -regextype posix-extended -regex '.*/enp[0-9]+s.*' -printf '"%f"\n' | paste -s -d ',') +elif [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; then + extra_interfaces=$(find /sys/class/net/ -not -name eth0 -regextype posix-extended -regex '.*/eth[0-9]+' -printf '"%f"\n' | paste -s -d ',') +fi + +cat > $daos_config </etc/systemd/system/"$${service_name}" </global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "import_gcs_bucket_uri" { + description = "The name of the GCS bucket to import data from to parallelstore." + type = string + default = null +} + +variable "import_destination_path" { + description = "The name of local path to import data on parallelstore instance from GCS bucket." + type = string + default = null +} + +variable "file_stripe" { + description = "The parallelstore stripe level for files." + type = string + default = null + validation { + condition = var.file_stripe == null ? true : contains([ + "FILE_STRIPE_LEVEL_UNSPECIFIED", + "FILE_STRIPE_LEVEL_MIN", + "FILE_STRIPE_LEVEL_BALANCED", + "FILE_STRIPE_LEVEL_MAX", + ], var.file_stripe) + error_message = "var.file_stripe must be set to \"FILE_STRIPE_LEVEL_UNSPECIFIED\", \"FILE_STRIPE_LEVEL_MIN\", \"FILE_STRIPE_LEVEL_BALANCED\", or \"FILE_STRIPE_LEVEL_MAX\"" + } +} + +variable "directory_stripe" { + description = "The parallelstore stripe level for directories." + type = string + default = null + validation { + condition = var.directory_stripe == null ? true : contains([ + "DIRECTORY_STRIPE_LEVEL_UNSPECIFIED", + "DIRECTORY_STRIPE_LEVEL_MIN", + "DIRECTORY_STRIPE_LEVEL_BALANCED", + "DIRECTORY_STRIPE_LEVEL_MAX", + ], var.directory_stripe) + error_message = "var.directory_stripe must be set to \"DIRECTORY_STRIPE_LEVEL_UNSPECIFIED\", \"DIRECTORY_STRIPE_LEVEL_MIN\", \"DIRECTORY_STRIPE_LEVEL_BALANCED\", or \"DIRECTORY_STRIPE_LEVEL_MAX\"" + } +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/versions.tf new file mode 100644 index 0000000000..174b5281e4 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.13.0" + } + + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md new file mode 100644 index 0000000000..47cf1518a1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md @@ -0,0 +1,192 @@ +## Description + +This module defines a file-system that already exists (i.e. it does not create +a new file system) in a way that can be shared with other modules. This allows +a compute VM to mount a filesystem that is not part of the current deployment +group. + +The pre-existing network storage can be referenced in the same way as any Cluster +Toolkit supported file-system such as [filestore](../filestore/README.md). + +For more information on network storage options in the Cluster Toolkit, see +the extended [Network Storage documentation](../../../docs/network_storage.md). + +### Example + +```yaml +- id: homefs + source: modules/file-system/pre-existing-network-storage + settings: + server_ip: ## Set server IP here ## + remote_mount: nfsshare + local_mount: /home + fs_type: nfs +``` + +This creates a pre-existing-network-storage module in terraform at the +provided IP in `server_ip` of type nfs that will be mounted at `/home`. Note +that the `server_ip` must be known before deployment. + +The following is an example of using `pre-existing-network-storage` with a GCS +bucket: + +```yaml +- id: data-bucket + source: modules/file-system/pre-existing-network-storage + settings: + remote_mount: my-bucket-name + local_mount: /data + fs_type: gcsfuse + mount_options: defaults,_netdev,implicit_dirs +``` + +The `implicit_dirs` mount option allows object paths to be treated as if they +were directories. This is important when working with files that were created by +another source, but there may have performance impacts. The `_netdev` mount option +denotes that the storage device requires network access. + +The following is an example of using `pre-existing-network-storage` with the `lustre` +filesystem: + +```yaml +- id: lustrefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: lustre + server_ip: 192.168.227.11@tcp + local_mount: /scratch + remote_mount: /exacloud +``` + +Note the use of the MGS NID (Network ID) in the `server_ip` field - in +particular, note the `@tcp` suffix. + +The following is an example of using `pre-existing-network-storage` with the +`managed_lustre` filesystem: + +```yaml +- id: lustrefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: managed_lustre + server_ip: 192.168.227.11@tcp + local_mount: /scratch + remote_mount: /mg_lustre +``` + +This is similar to the `lustre` filesystem, with the exception that it connects +with a managed Lustre instance hosted by GCP. Currently only Rocky 8 and +Ubuntu 20.04 and Ubuntu 22.04 are supported. + +The following is an example of using `pre-existing-network-storage` with the `daos` +filesystem. In order to use existing `parallelstore` instance, `fs_type` needs to be +explicitly mentioned in blueprint. The `remote_mount` option refers to `access_points` +for `parallelstore` instance. + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 +``` + +Parallelstore supports additional options for its mountpoints under `parallelstore_options` setting. +Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 + parallelstore_options: + daos_agent_config: | + credential_config: + cache_expiration: 1m +``` + +Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 + parallelstore_options: + dfuse_environment: + D_LOG_FILE: /tmp/client.log + D_APPEND_PID_TO_LOG: 1 + D_LOG_MASK: debug +``` + +### Mounting + +For the `fs_type` listed below, this module will provide `client_install_runner` +and `mount_runner` outputs. These can be used to create a startup script to +mount the network storage system. + +Supported `fs_type`: + +- nfs +- lustre +- managed_lustre +- gcsfuse +- daos + +[scripts/mount.sh](./scripts/mount.sh) is used as the contents of +`mount_runner`. This script will update `/etc/fstab` and mount the network +storage. This script will fail if the specified `local_mount` is already being +used by another entry in `/etc/fstab`. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. + +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## License + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [fs\_type](#input\_fs\_type) | Type of file system to be mounted (e.g., nfs, lustre) | `string` | `"nfs"` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | +| [managed\_lustre\_options](#input\_managed\_lustre\_options) | Managed Lustre specific options:
gke\_support\_enabled (bool, default = false)
Note: gke\_support\_enabled does not work with Slurm, the Slurm image must be built with
the correct compatibility. | 
object({
    gke_support_enabled = optional(bool, false)
  })
| `{}` | no | +| [mount\_options](#input\_mount\_options) | Options describing various aspects of the file system. Consider adding setting to 'defaults,\_netdev,implicit\_dirs' when using gcsfuse. | `string` | `"defaults,_netdev"` | no | +| [parallelstore\_options](#input\_parallelstore\_options) | Parallelstore specific options | 
object({
    daos_agent_config = optional(string, "")
    dfuse_environment = optional(map(string), {})
  })
| `{}` | no | +| [remote\_mount](#input\_remote\_mount) | Remote FS name or export. This is the exported directory for nfs, fs name for lustre, and bucket name (without gs://) for gcsfuse. | `string` | n/a | yes | +| [server\_ip](#input\_server\_ip) | The device name as supplied to fs-tab, excluding remote fs-name(for nfs, that is the server IP, for lustre [:]). This can be omitted for gcsfuse. | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use file system. | +| [mount\_runner](#output\_mount\_runner) | Runner that mounts the file system. | +| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | + diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf new file mode 100644 index 0000000000..203b6dfdac --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf @@ -0,0 +1,124 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a remote network storage to be mounted by fs-tab." + value = { + server_ip = var.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = var.mount_options + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } +} + +locals { + # Update remote mount to include a slash if the fs_type requires one to exist + remote_mount_with_slash = length(regexall("^/.*", var.remote_mount)) > 0 ? ( + var.remote_mount + ) : format("/%s", var.remote_mount) + remote_mount = contains(local.mount_vanilla_supported_fstype, local.fs_type) ? ( + local.remote_mount_with_slash + ) : var.remote_mount + + ml_gke_support_enabled = coalesce(try(var.managed_lustre_options.gke_support_enabled, false), false) + + # Collapse fs_type lustre and managed lustre for most uses, only needs to be + # different for client installation + fs_type = strcontains(var.fs_type, "lustre") ? "lustre" : var.fs_type + + # Client Install + ddn_lustre_client_install_script = templatefile( + "${path.module}/templates/ddn_exascaler_luster_client_install.tftpl", + { + server_ip = split("@", var.server_ip)[0] + remote_mount = local.remote_mount + local_mount = var.local_mount + } + ) + managed_lustre_client_install_script = file("${path.module}/scripts/install-managed-lustre-client.sh") + nfs_client_install_script = file("${path.module}/scripts/install-nfs-client.sh") + gcs_fuse_install_script = file("${path.module}/scripts/install-gcs-fuse.sh") + daos_client_install_script = file("${path.module}/scripts/install-daos-client.sh") + + install_scripts = { + "lustre" = local.ddn_lustre_client_install_script + "managed_lustre" = local.managed_lustre_client_install_script + "nfs" = local.nfs_client_install_script + "gcsfuse" = local.gcs_fuse_install_script + "daos" = local.daos_client_install_script + } + + client_install_runner = { + "type" = "shell" + "content" = lookup(local.install_scripts, var.fs_type, "echo 'skipping: client_install_runner not yet supported for ${var.fs_type}'") + "destination" = "install_filesystem_client${replace(var.local_mount, "/", "_")}.sh" + "args" = local.ml_gke_support_enabled ? "1" : "" + } + + mount_vanilla_supported_fstype = ["lustre", "nfs"] + mount_runner_vanilla = { + "type" = "shell" + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"${var.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" + "content" = ( + contains(local.mount_vanilla_supported_fstype, local.fs_type) ? + file("${path.module}/scripts/mount.sh") : + "echo 'skipping: mount_runner not yet supported for ${var.fs_type}'" + ) + } + gcsbucket = trimprefix(var.remote_mount, "gs://") + mount_runner_gcsfuse = { + "type" = "shell" + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"not-used\" \"${local.gcsbucket}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" + "content" = file("${path.module}/scripts/mount.sh") + } + + mount_runner_daos = { + "type" = "shell" + "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { + access_points = var.remote_mount + daos_agent_config = var.parallelstore_options.daos_agent_config + dfuse_environment = var.parallelstore_options.dfuse_environment + local_mount = var.local_mount + # avoid passing "--" as mount option to dfuse + mount_options = length(var.mount_options) == 0 ? "" : join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) + }) + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + } + + mount_scripts = { + "lustre" = local.mount_runner_vanilla + "nfs" = local.mount_runner_vanilla + "gcsfuse" = local.mount_runner_gcsfuse + "daos" = local.mount_runner_daos + } + + mount_runner = lookup(local.mount_scripts, local.fs_type, local.mount_runner_vanilla) +} + +output "client_install_runner" { + description = "Runner that performs client installation needed to use file system." + value = local.client_install_runner +} + +output "mount_runner" { + description = "Runner that mounts the file system." + value = local.mount_runner +} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh new file mode 100644 index 0000000000..e96eadb56a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || + { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 +fi + +if [ -x /bin/daos ]; then + echo "DAOS already installed" + daos version +else + # Install the DAOS client library + # The following commands should be executed on each client vm. + ## For Rocky linux 8 / RedHat 8. + if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then + # 1) Add the Parallelstore package repository + cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF + [parallelstore-v2-6-el${OS_VERSION_MAJOR}] + name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 + baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + + ## TODO: Remove disable automatic update script after issue is fixed. + if [ -x /usr/bin/google_disable_automatic_updates ]; then + /usr/bin/google_disable_automatic_updates + fi + dnf clean all + dnf makecache + + # 2) Install daos-client + dnf install -y epel-release # needed for capstone + dnf install -y daos-client + + # 3) Upgrade libfabric + dnf upgrade -y libfabric + + # For Ubuntu 22.04 and debian 12, + elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then + # shellcheck disable=SC2034 + DEBIAN_FRONTEND=noninteractive + + # 1) Add the Parallelstore package repository + curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg + echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list + + apt-get update + + # 2) Install daos-client + apt-get install -y daos-client + + # 3) Create daos_agent.service (comes pre-installed with RedHat) + if ! getent passwd daos_agent >/dev/null 2>&1; then + useradd daos_agent + fi + cat >/etc/systemd/system/daos_agent.service <<-EOF + [Unit] + Description=DAOS Agent + StartLimitIntervalSec=60 + Wants=network-online.target + After=network-online.target + + [Service] + Type=notify + User=daos_agent + Group=daos_agent + RuntimeDirectory=daos_agent + RuntimeDirectoryMode=0755 + ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml + StandardOutput=journal + StandardError=journal + Restart=always + RestartSec=10 + LimitMEMLOCK=infinity + LimitCORE=infinity + StartLimitBurst=5 + + [Install] + WantedBy=multi-user.target + EOF + else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + fi +fi + +exit 0 diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh new file mode 100644 index 0000000000..f8a990260b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ ! "$(which gcsfuse)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then + tee /etc/yum.repos.d/gcsfuse.repo >/dev/null <>/etc/modprobe.d/lnet.conf + fi +fi + +if grep -q lustre /proc/filesystems; then + echo "Skipping managed lustre client install as it is already supported" + exit 0 +fi + +# Get distro information +. /etc/os-release +DIST="NA" +if [[ $NAME == *"Ubuntu"* ]]; then + if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then + DIST="Ubuntu" + fi +elif [[ $NAME == *"Rocky"* ]]; then + if [[ $VERSION_ID == "8"* ]]; then + DIST="Rocky" + fi +fi + +if [[ ${DIST} == "Ubuntu" ]]; then + KEY_LOC=/etc/apt/keyrings + KEY_NAME=gcp-ar-repo.gpg + # Download new repo key + mkdir -p "${KEY_LOC}" + wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null + + # Set up apt repo + echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list + + # Install modules + apt update + apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) +elif [[ ${DIST} == "Rocky" ]]; then + # Set up yum repo + touch /etc/yum.repos.d/artifact-registry.repo + tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF + [lustre-client-rocky-8] + name=lustre-client-rocky-8 + baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + # Install modules + yum makecache + yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client +fi + +if [[ $DIST != "NA" ]]; then + # Load the new lustre client module + modprobe lustre +fi diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl new file mode 100644 index 0000000000..f5f0291e85 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl @@ -0,0 +1,50 @@ +#!/bin/sh + +# Copyright 2022 DataDirect Networks +# Modifications Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prior Art: https://github.com/DDNStorage/exascaler-cloud-terraform/blob/78deadbb2c1fa7e4603cf9605b0f7d1782117954/gcp/templates/client-script.tftpl + +# install new EXAScaler Cloud clients: +# all instances must be in the same zone +# and connected to the same network and subnet +# to set up EXAScaler Cloud filesystem on a new client instance, +# run the following commands on the client with root privileges: +set -e +if [[ ! -z $(cat /proc/filesystems | grep lustre) ]]; then + echo "Skipping lustre client install as it is already supported" + exit 0 +fi + +cat >/etc/esc-client.conf< $daos_config </etc/systemd/system/"$${service_name}" < +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string
    count = number
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [guest\_accelerator](#output\_guest\_accelerator) | Sanitized list of the type and count of accelerator cards attached to the instance. | +| [machine\_type\_guest\_accelerator](#output\_machine\_type\_guest\_accelerator) | List of the type and count of accelerator cards attached to the specified machine type. | + diff --git a/deletion-test/primary/modules/embedded/modules/internal/gpu-definition/main.tf b/deletion-test/primary/modules/embedded/modules/internal/gpu-definition/main.tf new file mode 100644 index 0000000000..f0861cddc9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/gpu-definition/main.tf @@ -0,0 +1,98 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "machine_type" { + description = "Machine type to use for the instance creation" + type = string +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string + count = number + gpu_driver_installation_config = optional(object({ + gpu_driver_version = string + }), { gpu_driver_version = "DEFAULT" }) + gpu_partition_size = optional(string) + gpu_sharing_config = optional(object({ + gpu_sharing_strategy = string + max_shared_clients_per_gpu = number + })) + })) + default = [] + nullable = false +} + +locals { + # example state; terraform will ignore diffs if last element of URL matches + # guest_accelerator = [ + # { + # count = 1 + # type = "https://www.googleapis.com/compute/beta/projects/PROJECT/zones/ZONE/acceleratorTypes/nvidia-tesla-a100" + # }, + # ] + accelerator_machines = { + "a2-highgpu-1g" = { type = "nvidia-tesla-a100", count = 1 }, + "a2-highgpu-2g" = { type = "nvidia-tesla-a100", count = 2 }, + "a2-highgpu-4g" = { type = "nvidia-tesla-a100", count = 4 }, + "a2-highgpu-8g" = { type = "nvidia-tesla-a100", count = 8 }, + "a2-megagpu-16g" = { type = "nvidia-tesla-a100", count = 16 }, + "a2-ultragpu-1g" = { type = "nvidia-a100-80gb", count = 1 }, + "a2-ultragpu-2g" = { type = "nvidia-a100-80gb", count = 2 }, + "a2-ultragpu-4g" = { type = "nvidia-a100-80gb", count = 4 }, + "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, + "a3-highgpu-1g" = { type = "nvidia-h100-80gb", count = 1 }, + "a3-highgpu-2g" = { type = "nvidia-h100-80gb", count = 2 }, + "a3-highgpu-4g" = { type = "nvidia-h100-80gb", count = 4 }, + "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, + "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, + "a4-highgpu-8g-lowmem" = { type = "nvidia-b200", count = 8 }, + "a4-highgpu-8g" = { type = "nvidia-b200", count = 8 }, + "a4x-highgpu-4g" = { type = "nvidia-gb200", count = 4 }, + "a4x-highgpu-4g-nolssd" = { type = "nvidia-gb200", count = 4 }, + "g2-standard-4" = { type = "nvidia-l4", count = 1 }, + "g2-standard-8" = { type = "nvidia-l4", count = 1 }, + "g2-standard-12" = { type = "nvidia-l4", count = 1 }, + "g2-standard-16" = { type = "nvidia-l4", count = 1 }, + "g2-standard-24" = { type = "nvidia-l4", count = 2 }, + "g2-standard-32" = { type = "nvidia-l4", count = 1 }, + "g2-standard-48" = { type = "nvidia-l4", count = 4 }, + "g2-standard-96" = { type = "nvidia-l4", count = 8 }, + } + generated_guest_accelerator = try([local.accelerator_machines[var.machine_type]], []) + + # Select in priority order: + # (1) var.guest_accelerator if not empty + # (2) local.generated_guest_accelerator if not empty + # (3) default to empty list if both are empty + guest_accelerator = try(coalescelist(var.guest_accelerator, local.generated_guest_accelerator), []) +} + +output "guest_accelerator" { + description = "Sanitized list of the type and count of accelerator cards attached to the instance." + value = local.guest_accelerator +} + +output "machine_type_guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the specified machine type." + value = local.generated_guest_accelerator +} + +terraform { + required_version = ">= 1.3" +} diff --git a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/README.md b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/README.md new file mode 100644 index 0000000000..21746fe0d8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/README.md @@ -0,0 +1,30 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [disk\_type](#input\_disk\_type) | The disk type to validate. | `string` | n/a | yes | +| [machine\_type](#input\_machine\_type) | The machine type to validate. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/main.tf b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/main.tf new file mode 100644 index 0000000000..d89d7edfec --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/main.tf @@ -0,0 +1,52 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +check "disk_type_c4_compatibility" { + assert { + condition = !(can(regex("^c4-", var.machine_type)) && var.disk_type == "pd-ssd") + error_message = "The C4 machine series does not support pd-ssd. Please use hyperdisk-balanced or another compatible disk type." + } +} + + +check "disk_type_c2_compatibility" { + assert { + condition = !(can(regex("^c2-", var.machine_type)) && can(regex("hyperdisk", var.disk_type))) + error_message = "The C2 machine series does not support Hyperdisk as a boot disk. Please use a compatible disk type like pd-ssd, pd-standard, or pd-balanced." + } +} + + +check "disk_type_pd_extreme_compatibility" { + assert { + condition = var.disk_type != "pd-extreme" || can(regex("^(m1-|m2-|m3-|n2-|n2d-)", var.machine_type)) + error_message = "pd-extreme disks are only supported for M1, M2, M3, N2, and N2D machine series." + } +} + + +check "disk_type_hyperdisk_extreme_compatibility" { + assert { + condition = var.disk_type != "hyperdisk-extreme" || can(regex("^(c3-|m1-|m3-|n2-)", var.machine_type)) + error_message = "hyperdisk-extreme disks are only supported for C3, M1, M3, and N2 machine series." + } +} + + +check "disk_type_hyperdisk_throughput_compatibility" { + assert { + condition = var.disk_type != "hyperdisk-throughput" || can(regex("^(c3-|c3d-|n4-|n2-|n2d-|n1-|t2d-|m1-)", var.machine_type)) + error_message = "hyperdisk-throughput disks are only supported for C3, C3D, N4, N2, N2D, N1, T2D, and M1 machine series." + } +} diff --git a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/variables.tf b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/variables.tf new file mode 100644 index 0000000000..23478051b3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/variables.tf @@ -0,0 +1,23 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "machine_type" { + type = string + description = "The machine type to validate." +} + +variable "disk_type" { + type = string + description = "The disk type to validate." +} diff --git a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/versions.tf b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/versions.tf new file mode 100644 index 0000000000..4702005614 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/versions.tf @@ -0,0 +1,17 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 0.15.0" +} diff --git a/deletion-test/primary/modules/embedded/modules/internal/network-attachment/README.md b/deletion-test/primary/modules/embedded/modules/internal/network-attachment/README.md new file mode 100644 index 0000000000..8aa9270a0a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/network-attachment/README.md @@ -0,0 +1,54 @@ + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | +| [google-beta](#requirement\_google-beta) | >= 6.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google-beta](#provider\_google-beta) | >= 6.0.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_network_attachment.self](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_network_attachment) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [connection\_preference](#input\_connection\_preference) | The connection preference of service attachment. | `string` | `"ACCEPT_AUTOMATIC"` | no | +| [name](#input\_name) | Name of the resource. Provided by the client when the resource is created | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The ID of the project in which the resource belongs. | `string` | n/a | yes | +| [region](#input\_region) | Region where the network attachment resides | `string` | n/a | yes | +| [subnetwork\_self\_links](#input\_subnetwork\_self\_links) | An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment. | `list(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [self\_link](#output\_self\_link) | Server-defined URL for the resource. | + diff --git a/deletion-test/primary/modules/embedded/modules/internal/network-attachment/main.tf b/deletion-test/primary/modules/embedded/modules/internal/network-attachment/main.tf new file mode 100644 index 0000000000..bbbece7085 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/network-attachment/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +variable "connection_preference" { + type = string + description = "The connection preference of service attachment." + default = "ACCEPT_AUTOMATIC" +} + +variable "subnetwork_self_links" { + type = list(string) + description = " An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment." +} + +variable "name" { + type = string + description = "Name of the resource. Provided by the client when the resource is created" +} + +variable "project_id" { + type = string + description = "The ID of the project in which the resource belongs." +} + +variable "region" { + type = string + description = "Region where the network attachment resides" +} + + +resource "google_compute_network_attachment" "self" { + provider = google-beta + + project = var.project_id + region = var.region + name = var.name + connection_preference = var.connection_preference + subnetworks = var.subnetwork_self_links +} + + +output "self_link" { + value = google_compute_network_attachment.self.self_link + description = "Server-defined URL for the resource." +} + +terraform { + required_version = ">= 0.15.0" + + required_providers { + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.0.0" + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml b/deletion-test/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml new file mode 100644 index 0000000000..e80fc96b9c --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/README.md b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/README.md new file mode 100644 index 0000000000..610d82c1b9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/README.md @@ -0,0 +1,85 @@ +## Description + +This is an internal helper module designed to encapsulate and centralize all hardware-specific logic for Google Cloud TPUs. It is intended to be called by parent modules like `gke-node-pool` to determine if a node pool is TPU-based and to retrieve its specific attributes. + +This module's primary responsibilities are: + +* Reliably detect if a node pool is for TPUs by checking its `placement_policy`. +* Determine the correct GKE `tpu-accelerator` label based on the machine type family. +* Determine the `number of chips per node` based on the specific machine type. +* Generate the standard **Kubernetes taint** that should be applied to TPU nodes. + +This follows the same design pattern as the `gpu-definition` internal module, promoting a clean separation of concerns within the gke-node-pool module. + +## Usage + +This module is not intended for direct use in a blueprint. It should be called from a parent module like `gke-node-pool`. + +```yaml +module "tpu" { + source = "../../internal/tpu-definition" + + # Pass the parent module's variables to this module + machine_type = var.machine_type + placement_policy = var.placement_policy +} + +# Example of consuming the module's outputs in the parent module +locals { + # The tpu_taint is then used in the node_config's dynamic "taint" block + tpu_taint = module.tpu.tpu_taint +} +``` + +## License + + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [machine\_type](#input\_machine\_type) | The machine type of the node pool. | `string` | n/a | yes | +| [placement\_policy](#input\_placement\_policy) | The placement policy for the node pool. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [is\_tpu](#output\_is\_tpu) | Boolean value indicating if the node pool is for TPUs. | +| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | +| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | +| [tpu\_taint](#output\_tpu\_taint) | A list containing the standard TPU taint object if the node pool is for TPUs. | +| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | + diff --git a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/main.tf b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/main.tf new file mode 100644 index 0000000000..c8ee417d71 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/main.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # Determine if this is a TPU node pool by checking if the machine_type exists in our authoritative map of TPU machine types. + is_tpu = contains(keys(local.tpu_chip_count_map), var.machine_type) + + tpu_taint = local.is_tpu ? [{ + key = "google.com/tpu" + value = "present" + effect = "NO_SCHEDULE" + }] : [] + + # Map of machine prefixes to GKE accelerator labels. + tpu_accelerator_map = { + "ct4p" = "tpu-v4-podslice" # TPU v4 + "ct5lp" = "tpu-v5-lite-podslice" # TPU v5e + "ct5p" = "tpu-v5p-slice" # TPU v5p + "ct6e" = "tpu-v6e-slice" # TPU v6e + "tpu7x" = "tpu7x" # TPU v7x + } + + # Map specific GCE machine types to the number of TPU chips per node (VM). + # The machine-type map must be updated to reflect new TPU releases with reference to public documentation: https://docs.cloud.google.com/tpu/docs/intro-to-tpu + tpu_chip_count_map = { + # v4 - ct4p + "ct4p-hightpu-4t" = 4 + + # v5e - ct5lp + "ct5lp-hightpu-1t" = 1 + "ct5lp-hightpu-4t" = 4 + "ct5lp-hightpu-8t" = 8 + + # v5p - ct5p + "ct5p-hightpu-1t" = 1 + "ct5p-hightpu-2t" = 2 + "ct5p-hightpu-4t" = 4 + + # v6e - ct6e + "ct6e-standard-1t" = 1 + "ct6e-standard-4t" = 4 + "ct6e-standard-8t" = 8 + + # v7x - tpu7x + "tpu7x-standard-4t" = 4 + } + + # Robustly extract the machine family prefix (e.g., "ct6e"). + tpu_machine_family = local.is_tpu ? element(split("-", var.machine_type), 0) : "" + tpu_accelerator_type = local.is_tpu ? lookup(local.tpu_accelerator_map, local.tpu_machine_family, null) : null + tpu_chips_per_node = local.is_tpu ? lookup(local.tpu_chip_count_map, var.machine_type, null) : null +} + +terraform { + required_version = ">= 1.3" +} diff --git a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/outputs.tf b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/outputs.tf new file mode 100644 index 0000000000..fa3c21fa34 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "is_tpu" { + description = "Boolean value indicating if the node pool is for TPUs." + value = local.is_tpu +} + +output "tpu_accelerator_type" { + description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." + value = local.tpu_accelerator_type +} + +output "tpu_topology" { + description = "The topology of the TPU slice (e.g., '4x4')." + value = local.is_tpu ? var.placement_policy.tpu_topology : null +} + +output "tpu_chips_per_node" { + description = "The number of TPU chips on each node in the pool." + value = local.tpu_chips_per_node +} + +output "tpu_taint" { + description = "A list containing the standard TPU taint object if the node pool is for TPUs." + value = local.tpu_taint +} diff --git a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/variables.tf b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/variables.tf new file mode 100644 index 0000000000..254488c02d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/variables.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "machine_type" { + description = "The machine type of the node pool." + type = string +} + +variable "placement_policy" { + description = "The placement policy for the node pool." + type = object({ + type = string + name = optional(string) + tpu_topology = optional(string) + }) +} diff --git a/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/README.md b/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/README.md new file mode 100644 index 0000000000..aefac9d187 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/README.md @@ -0,0 +1,56 @@ + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_network_peering.peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [export\_custom\_routes](#input\_export\_custom\_routes) | (Optional) Whether to export the custom routes to the peer network. Defaults to false. | `bool` | `null` | no | +| [import\_custom\_routes](#input\_import\_custom\_routes) | (Optional) Whether to import the custom routes from the peer network. Defaults to false. | `bool` | `null` | no | +| [import\_subnet\_routes\_with\_public\_ip](#input\_import\_subnet\_routes\_with\_public\_ip) | (Optional) Whether subnet routes with public IP range are imported. | `bool` | `null` | no | +| [name](#input\_name) | Name of the peering. | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | The primary network of the peering. | `string` | n/a | yes | +| [peer\_network\_self\_link](#input\_peer\_network\_self\_link) | The peer network in the peering. The peer network may belong to a different project. | `string` | n/a | yes | +| [stack\_type](#input\_stack\_type) | (Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [peering\_name](#output\_peering\_name) | Name of the peering. | + diff --git a/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/main.tf b/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/main.tf new file mode 100644 index 0000000000..386fa9377b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/main.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "name" { + type = string + description = "Name of the peering." +} + +variable "network_self_link" { + type = string + description = "The primary network of the peering." +} + +variable "peer_network_self_link" { + type = string + description = "The peer network in the peering. The peer network may belong to a different project." +} + +variable "export_custom_routes" { + type = bool + description = "(Optional) Whether to export the custom routes to the peer network. Defaults to false." + default = null +} + +variable "import_custom_routes" { + type = bool + description = "(Optional) Whether to import the custom routes from the peer network. Defaults to false." + default = null +} + +variable "import_subnet_routes_with_public_ip" { + type = bool + description = "(Optional) Whether subnet routes with public IP range are imported. " + default = null +} + +variable "stack_type" { + type = string + description = "(Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. " + default = null +} + +resource "google_compute_network_peering" "peering" { + name = var.name + network = var.network_self_link + peer_network = var.peer_network_self_link + export_custom_routes = var.export_custom_routes + import_custom_routes = var.import_custom_routes + import_subnet_routes_with_public_ip = var.import_subnet_routes_with_public_ip + stack_type = var.stack_type +} + +output "peering_name" { + value = google_compute_network_peering.peering.name + description = "Name of the peering." +} + +terraform { + required_version = ">= 0.15.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml b/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml new file mode 100644 index 0000000000..e80fc96b9c --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/README.md b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/README.md new file mode 100644 index 0000000000..d7054eb725 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/README.md @@ -0,0 +1,244 @@ +## Description + +This module simplifies the following functionality: + +* Applying Kubernetes manifests to GKE clusters: It provides flexible options for specifying manifests, allowing you to either directly embed them as strings content or reference them from URLs, files, templates, or entire .yaml and .tftpl files in directories. +* Deploying commonly used infrastructure like [Kueue](https://kueue.sigs.k8s.io/docs/) or [Jobset](https://jobset.sigs.k8s.io/docs/). + +> Note: Kueue can work with a variety of frameworks out of the box, find them [here](https://kueue.sigs.k8s.io/docs/tasks/run/) + +### Explanation + +* **Manifest:** + * **Raw String:** Specify manifests directly within the module configuration using the `content: manifest_body` format. + * **File/Template/Directory Reference:** Set `source` to the path to: + * A single URL to a manifest file. Ex.: `https://github.com/.../myrepo/manifest.yaml`. + + > **Note:** Applying from a URL has important limitations. Please review the [Considerations & Callouts for Applying from URLs](#applying-manifests-from-urls-considerations--callouts) section below. + * A single local YAML manifest file (`.yaml`). Ex.: `./manifest.yaml`. + * A template file (`.tftpl`) to generate a manifest. Ex.: `./template.yaml.tftpl`. You can pass the variables to format the template file in `template_vars`. + * A directory containing multiple YAML or template files. Ex: `./manifests/`. You can pass the variables to format the template files in `template_vars`. + +#### Manifest Example + +```yaml +- id: existing-gke-cluster + source: modules/scheduler/pre-existing-gke-cluster + settings: + project_id: $(vars.project_id) + cluster_name: my-gke-cluster + region: us-central1 + +- id: kubectl-apply + source: modules/management/kubectl-apply + use: [existing-gke-cluster] + settings: + - content: | + apiVersion: v1 + kind: Namespace + metadata: + name: my-namespace + - source: "https://github.com/kubernetes-sigs/jobset/releases/download/v0.6.0/manifests.yaml" + - source: $(ghpc_stage("manifests/configmap1.yaml")) + - source: $(ghpc_stage("manifests/configmap2.yaml.tftpl")) + template_vars: {name: "dev-config", public: "false"} + - source: $(ghpc_stage("manifests"))/ + template_vars: {name: "dev-config", public: "false"} +``` + +#### Pre-build infrastructure Example + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml")) + jobset: + install: true +``` + +The `config_path` field in `kueue` installation accepts a template file, too. You will need to provide variables for the template using `config_template_vars` field. + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) + config_template_vars: {name: "dev-config", public: "false"} + jobset: + install: true +``` + +You can specify a particular kueue version that you would like to use using the `version` flag. By default, we recommend customers to [use v0.10.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L68). You can find the list of supported kueue versions [here](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L18). + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + version: v0.10.0 + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) + config_template_vars: {name: "dev-config", public: "false"} + jobset: + install: true +``` + +> **_NOTE:_** +> +> The `project_id` and `region` settings would be inferred from the deployment variables of the same name, but they are included here for clarity. +> +> Terraform may apply resources in parallel, leading to potential dependency issues. If a resource's dependencies aren't ready, it will be applied again up to 15 times. + +## Callouts + +### Applying Manifests from URLs: Considerations & Callouts + +While this module supports applying manifests directly from remote `http://` or `https://` URLs, this method introduces complexities not present when using local files. For production environments, we recommend sourcing manifests from local paths or a version-controlled Git repository. Moreover, this method will be deprecated soon. Hence we recommend to use other methods to source manifests. + +If you choose to use the URL method, be aware of the following potential issues and their solutions. + +#### **1. Apply Order and Race Conditions** + +The module applies manifests from the `apply_manifests` list in parallel. This can create a **race condition** if one manifest depends on another. The most common example is applying a manifest with custom resources (like a `ClusterQueue`) at the same time as the manifest that defines it (the `CustomResourceDefinition` or CRD). + +There is **no guarantee** that the CRD will be applied before the resource that uses it. This can lead to non-deterministic deployment failures with errors like: + +```Error: resource [kueue.x-k8s.io/v1beta1/ClusterQueue] isn't valid for cluster``` + +##### **Recommended Workaround: Two-Stage Apply** + +To ensure a reliable deployment, you must manually enforce the correct order of operations. + +1. **Initial Deployment:** In your blueprint, include **only** the manifest(s) containing the `CustomResourceDefinition` (CRD) resources in the `apply_manifests` list. + + *Example `settings` for the first run:* + + ```yaml + settings: + apply_manifests: + # This manifest contains the CRDs for Kueue + - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" + server_side_apply: true + ``` + +2. **Run the deployment** (`gcluster deploy` or `terraform apply`). + +3. **Second Deployment:** Once the first apply is successful, **add** the manifests containing your custom resources (like `ClusterQueue`, `LocalQueue`) to the list. + + *Example `settings` for the second run:* + + ```yaml + settings: + apply_manifests: + # The CRD manifest is still present + - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" + server_side_apply: true + + # Now, add your configuration manifest + - source: "https://gist.githubusercontent.com/YourUser/..." # Your configuration URL + server_side_apply: true + ``` + +4. **Run the deployment command again.** Since the CRDs are now guaranteed to exist in the cluster, this second apply will succeed reliably. + +#### **2. Large Manifests (CRDs)** + +* **Issue:** Applying very large manifests can fail with a `metadata.annotations: Too long` error. +* **Solution:** Enable Server-Side Apply by setting `server_side_apply: true` for the manifest entry. + +#### **3. Conflicts on Re-application** + +* **Issue:** Re-running a deployment after a partial failure can cause server-side apply field manager `conflicts`. +* **Solution:** Forcibly take ownership of the resource fields by setting `force_conflicts: true`. + +#### **4. Terraform Template Files (`.tftpl`)** + +* **Limitation:** This module **cannot** render a template file (`.tftpl`) when sourced from a remote URL. +* **Workaround:** You must render the template into a pure YAML file locally, host that rendered file at a URL, and provide the URL of the rendered file in your blueprint. + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 7.2 | +| [helm](#requirement\_helm) | ~> 2.17 | +| [http](#requirement\_http) | ~> 3.0 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 7.2 | +| [http](#provider\_http) | ~> 3.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [configure\_kueue](#module\_configure\_kueue) | ./kubectl | n/a | +| [install\_gib](#module\_install\_gib) | ./kubectl | n/a | +| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | +| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | +| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | +| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | +| [kubectl\_apply\_manifests](#module\_kubectl\_apply\_manifests) | ./kubectl | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.gib_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.initial_gib_version](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.jobset_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.kueue_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | +| [http_http.manifest_from_url](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [apply\_manifests](#input\_apply\_manifests) | A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md).
NOTE: The `enable` input acts as a FF to apply a manifest or not. By default it is always set to `true`. | 
list(object({
    enable            = optional(bool, true)
    content           = optional(string, null)
    source            = optional(string, null)
    template_vars     = optional(map(any), null)
    server_side_apply = optional(bool, false)
    wait_for_rollout  = optional(bool, true)
  }))
| `[]` | no | +| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | +| [gib](#input\_gib) | Install the NCCL gIB plugin | 
object({
    install = bool
    path    = string
    template_vars = object({
      image   = optional(string, "us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib")
      version = string
      node_affinity = optional(any, {
        requiredDuringSchedulingIgnoredDuringExecution = {
          nodeSelectorTerms = [{
            matchExpressions = [{
              key      = "cloud.google.com/gke-gpu",
              operator = "In",
              values   = ["true"]
            }]
          }]
        }
      })
      accelerator_count = number
      max_unavailable   = optional(string, "50%")
    })
  })
| 
{
  "install": false,
  "path": "",
  "template_vars": {
    "accelerator_count": 0,
    "version": ""
  }
}
| no | +| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | +| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "0.10.1")
  })
| `{}` | no | +| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "0.13.3")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | +| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | +| [target\_architecture](#input\_target\_architecture) | The target architecture for the GKE nodes and gIB plugin (e.g., 'x86\_64' or 'arm64'). | `string` | `"x86_64"` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md new file mode 100644 index 0000000000..1957899617 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md @@ -0,0 +1,64 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [helm](#provider\_helm) | ~> 2.17 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | +| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | +| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | +| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | +| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | +| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | +| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | +| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | +| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | +| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | +| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | +| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | +| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | +| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | +| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | +| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | +| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | +| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | +| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | +| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | +| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | +| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | +| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | +| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | +| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | +| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | +| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | +| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | +| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | +| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf new file mode 100644 index 0000000000..8cc09bd3e2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf @@ -0,0 +1,79 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "helm_release" "apply_chart" { + # Required Identification + name = var.release_name + chart = var.chart_name + + # Chart Source & Version + repository = var.chart_repository + version = var.chart_version + devel = var.devel + + # Target Namespace + namespace = var.namespace + create_namespace = var.create_namespace + + # Values Configuration + values = var.values_yaml + + dynamic "set" { + for_each = var.set_values + content { + name = set.value.name + value = set.value.value + type = set.value.type + } + } + + # Installation/Upgrade Behavior + description = var.description + atomic = var.atomic + cleanup_on_fail = var.cleanup_on_fail + dependency_update = var.dependency_update + disable_crd_hooks = var.disable_crd_hooks + disable_openapi_validation = var.disable_openapi_validation + disable_webhooks = var.disable_webhooks + force_update = var.force_update + lint = var.lint + max_history = var.max_history + recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI + render_subchart_notes = var.render_subchart_notes + reset_values = var.reset_values + reuse_values = var.reuse_values + skip_crds = var.skip_crds + timeout = var.timeout + wait = var.wait + wait_for_jobs = var.wait_for_jobs + + # Verification & Credentials + keyring = var.keyring + pass_credentials = var.pass_credentials + verify = var.verify + + # Post Rendering + dynamic "postrender" { + # Only include the block if var.postrender is not null + for_each = var.postrender == null ? [] : [var.postrender] + content { + binary_path = postrender.value.binary_path + } + } + + # Lifecycle block (optional - generally avoid complex lifecycle in generic modules) + # lifecycle { + # ignore_changes = [] + # } +} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf new file mode 100644 index 0000000000..04e8e214fc --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf @@ -0,0 +1,212 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +# --- Required --- +variable "release_name" { + description = "Name of the Helm release." + type = string +} + +variable "chart_name" { + description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." + type = string +} + +# --- Chart Location & Version --- +variable "chart_repository" { + description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." + type = string + default = null +} + +variable "chart_version" { + description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." + type = string + default = null +} + +variable "devel" { + description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." + type = bool + default = false +} + +# --- Namespace --- +variable "namespace" { + description = "Kubernetes namespace to install the Helm release into." + type = string + default = "default" +} + +variable "create_namespace" { + description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." + type = bool + default = true # Common convenience setting +} + +# --- Values Customization --- +variable "values_yaml" { + description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." + type = list(string) + default = [] +} + +variable "set_values" { + description = "List of objects defining values to set ('helm install --set')." + type = list(object({ + name = string # Path to the value (e.g., 'service.type', 'replicaCount') + value = string # The value to set + type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') + })) + default = [] +} + +# --- Installation/Upgrade Behavior --- +variable "description" { + description = "Set an optional description for the Helm release." + type = string + default = null +} + +variable "atomic" { + description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." + type = bool + default = false +} + +variable "wait" { + description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." + type = bool + default = true # Often a good default for dependencies +} + +variable "wait_for_jobs" { + description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." + type = bool + default = false # Helm CLI default is false +} + +variable "timeout" { + description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." + type = number + default = 300 # 5 minutes (Helm CLI default) +} + +variable "cleanup_on_fail" { + description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." + type = bool + default = false +} + +variable "dependency_update" { + description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." + type = bool + default = false +} + +variable "disable_crd_hooks" { + description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." + type = bool + default = false +} + +variable "disable_openapi_validation" { + description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." + type = bool + default = false +} + +variable "disable_webhooks" { + description = "Prevent hooks from running ('helm install --no-hooks')." + type = bool + default = false +} + +variable "force_update" { + description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." + type = bool + default = false +} + +variable "lint" { + description = "Run the helm chart linter during the plan ('helm lint')." + type = bool + default = false +} + +variable "max_history" { + description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." + type = number + default = null # Terraform provider defaults to Helm's default (usually 10) +} + +variable "recreate_pods" { + description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." + type = bool + default = false +} + +variable "render_subchart_notes" { + description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." + type = bool + default = false +} + +variable "reset_values" { + description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." + type = bool + default = false +} + +variable "reuse_values" { + description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." + type = bool + default = false # Helm CLI default is false +} + +variable "skip_crds" { + description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." + type = bool + default = false +} + +# --- Verification & Credentials --- +variable "keyring" { + description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." + type = string + default = null # Defaults to Helm's default keyring location +} + +variable "pass_credentials" { + description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." + type = bool + default = false +} + +variable "verify" { + description = "Verify the package before installing it ('helm install --verify')." + type = bool + default = false +} + +# --- Advanced Rendering --- +variable "postrender" { + description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." + type = object({ + binary_path = string # Path to the post-renderer executable + }) + default = null # Disabled by default +} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf new file mode 100644 index 0000000000..09d912e2c9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml new file mode 100644 index 0000000000..92fc1bca22 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml @@ -0,0 +1,25 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For referencing the original jobset helm chart values, pull the latest jobset chart version +# `helm pull oci://registry.k8s.io/jobset/charts/jobset --version=0.10.1` (latest helm chart version) + +controller: + # It ensures the Jobset pod(s) can be scheduled on GKE clusters where the + # system node pool uses the default "gke-managed-components" taint. + tolerations: + - key: "components.gke.io/gke-managed-components" + operator: "Equal" + value: "true" + effect: "NoSchedule" diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md new file mode 100644 index 0000000000..691f4dc34a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md @@ -0,0 +1,55 @@ + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | + +## Providers + +| Name | Version | +|------|---------| +| [kubectl](#provider\_kubectl) | >= 1.7.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubectl_manifest.apply_doc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_path_documents.templates](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | +| [kubectl_path_documents.yamls](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | +| [force\_conflicts](#input\_force\_conflicts) | The force\_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller) | `bool` | `false` | no | +| [server\_side\_apply](#input\_server\_side\_apply) | Allow using kubectl server-side apply method. | `bool` | `false` | no | +| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `null` | no | +| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | +| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | + +## Outputs + +No outputs. + diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf new file mode 100644 index 0000000000..acf1d3c908 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf @@ -0,0 +1,92 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + yaml_separator = "\n---" + + # This locals block processes manifest inputs from one of four methods, + # evaluated in order of precedence using coalesce. + + # --- METHOD 1: Direct Content Input --- + # Used when manifest content is passed directly as a string. + content_yaml_body = var.content + + # Fallback for safe path checking in subsequent methods. + null_safe_source = coalesce(var.source_path, " ") + + # --- METHOD 2: Single Local YAML File --- + # Used when var.source_path points to a local .yaml file. + yaml_file = length(regexall("\\.yaml(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null + yaml_file_content = local.yaml_file != null ? file(local.yaml_file) : null + + # --- METHOD 3: Single Local Template File --- + # Used when var.source_path points to a local .tftpl file. + template_file = length(regexall("\\.tftpl(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null + template_file_content = local.template_file != null ? templatefile(local.template_file, var.template_vars) : null + + # --- CONSOLIDATE & PROCESS --- + # Coalesce finds the first non-null content from the methods above. + yaml_body = coalesce(local.content_yaml_body, local.yaml_file_content, local.template_file_content, " ") + # Ensure only valid YAML is processed + # It explicitly tests if the content can be decoded before including it. + yaml_body_docs = compact(flatten([ + for doc in split(local.yaml_separator, local.yaml_body) : [ + for content in [trimspace(doc)] : ( + # Use a temporary local variable and can() to test for successful YAML decoding. + # This handles malformed documents (like comment blocks) which cause yamldecode() to fail. + can(yamldecode(content)) && length(yamldecode(content)) > 0 ? content : null + ) + ] + ])) + + # --- METHOD 4: Directory of Files --- + # If no content was found via the methods above AND the source path looks like a directory, + # we assume this is the desired method. The data blocks below will handle it. + directory = length(local.yaml_body_docs) == 0 && endswith(local.null_safe_source, "/") ? abspath(var.source_path) : null + + # --- FINAL AGGREGATION --- + # Combine documents from single-source methods and directory-scan methods into one list. + docs_list = concat(try(local.yaml_body_docs, []), try(data.kubectl_path_documents.yamls[0].documents, []), try(data.kubectl_path_documents.templates[0].documents, [])) + docs_map = tomap({ + for index, doc in local.docs_list : index => doc + }) +} + +data "kubectl_path_documents" "yamls" { + count = local.directory != null ? 1 : 0 + pattern = "${local.directory}/*.yaml" +} + +data "kubectl_path_documents" "templates" { + count = local.directory != null ? 1 : 0 + pattern = "${local.directory}/*.tftpl" + vars = var.template_vars +} + +resource "kubectl_manifest" "apply_doc" { + for_each = local.docs_map + yaml_body = each.value + server_side_apply = var.server_side_apply + wait_for_rollout = var.wait_for_rollout + force_conflicts = var.force_conflicts + + lifecycle { + precondition { + condition = !var.force_conflicts || var.server_side_apply + error_message = "The 'force_conflicts' variable can only be set to true when 'server_side_apply' is also true." + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf new file mode 100644 index 0000000000..7bf34e089c --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "content" { + description = "The YAML body to apply to gke cluster." + type = string + default = null +} + +variable "source_path" { + description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." + type = string + default = null +} + +variable "template_vars" { + description = "The values to populate template file(s) with." + type = any + default = null +} + +variable "server_side_apply" { + description = "Allow using kubectl server-side apply method." + type = bool + default = false +} + +variable "wait_for_rollout" { + description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." + type = bool + default = true +} + +variable "force_conflicts" { + description = "The force_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller)" + type = bool + default = false +} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf new file mode 100644 index 0000000000..cce452239f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.7.0" + } + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml new file mode 100644 index 0000000000..7c0bef7013 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml @@ -0,0 +1,30 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For referencing the original Kueue helm chart values, pull the latest helm chart version +# `helm pull oci://registry.k8s.io/kueue/charts/kueue --version=0.13.3` (latest helm chart version) + +controllerManager: + # -- Enables the Topology-Aware Scheduling feature gate. + featureGates: + - name: TopologyAwareScheduling + enabled: true + + # It ensures the Kueue pod can schedule on GKE clusters where the + # system node pool uses the default "gke-managed-components" taint. + tolerations: + - key: "components.gke.io/gke-managed-components" + operator: "Equal" + value: "true" + effect: "NoSchedule" diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/main.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/main.tf new file mode 100644 index 0000000000..73a15ad1ab --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/main.tf @@ -0,0 +1,271 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] + project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] + + # 1. First, Identify manifests that are explicitly enabled. + enabled_manifests = { + for index, manifest in var.apply_manifests : index => manifest + if try(manifest.enable, true) + } + + # 2. Identify URL-based manifests + url_manifests = { + for index, manifest in local.enabled_manifests : index => manifest + if try(manifest.source, null) != null && (startswith(manifest.source, "http://") || startswith(manifest.source, "https://")) + } + + # 3. Rebuild the map by populating the 'content' field for URLs based manifest + processed_apply_manifests_map = tomap({ + for index, manifest in local.enabled_manifests : tostring(index) => { + # If this manifest was a URL, its content is the body from the HTTP call. + content = contains(keys(local.url_manifests), tostring(index)) ? data.http.manifest_from_url[tostring(index)].body : manifest.content + + # If this was a URL, its source path is now null. Otherwise, use original. + source = contains(keys(local.url_manifests), tostring(index)) ? null : manifest.source + + # Pass other vars + template_vars = manifest.template_vars + server_side_apply = manifest.server_side_apply + wait_for_rollout = manifest.wait_for_rollout + } + }) + + install_kueue = try(var.kueue.install, false) + install_jobset = try(var.jobset.install, false) + install_gpu_operator = try(var.gpu_operator.install, false) + install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) + install_gib = try(var.gib.install, false) +} + +data "http" "manifest_from_url" { + for_each = local.url_manifests + url = each.value.source +} + +data "google_container_cluster" "gke_cluster" { + project = local.project_id + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +module "kubectl_apply_manifests" { + for_each = local.processed_apply_manifests_map + source = "./kubectl" + depends_on = [var.gke_cluster_exists] + + content = each.value.content + source_path = each.value.source + template_vars = each.value.template_vars + server_side_apply = each.value.server_side_apply + wait_for_rollout = each.value.wait_for_rollout + + providers = { + kubectl = kubectl + } +} + +module "install_kueue" { + source = "./helm_install" + count = local.install_kueue ? 1 : 0 + wait = false + timeout = 1200 + release_name = "kueue" + chart_repository = "oci://registry.k8s.io/kueue/charts" + chart_name = "kueue" + chart_version = var.kueue.version + namespace = "kueue-system" + create_namespace = true + values_yaml = [ + file("${path.module}/kueue/kueue-helm-values.yaml") + ] + + depends_on = [var.gke_cluster_exists] +} + +module "configure_kueue" { + source = "./kubectl" + source_path = local.install_kueue ? try(var.kueue.config_path, "") : null + template_vars = local.install_kueue ? try(var.kueue.config_template_vars, null) : null + depends_on = [module.install_kueue] + + server_side_apply = true + wait_for_rollout = true + + providers = { + kubectl = kubectl + } +} + +module "install_jobset" { + source = "./helm_install" + count = local.install_jobset ? 1 : 0 + wait = false + timeout = 1200 + release_name = "jobset" + chart_repository = "oci://registry.k8s.io/jobset/charts" + chart_name = "jobset" + chart_version = var.jobset.version + namespace = "jobset-system" + create_namespace = true + values_yaml = [ + file("${path.module}/jobset/jobset-helm-values.yaml") + ] + depends_on = [var.gke_cluster_exists, module.configure_kueue] +} + +module "install_nvidia_dra_driver" { + count = local.install_nvidia_dra_driver ? 1 : 0 + depends_on = [module.kubectl_apply_manifests, var.gke_cluster_exists, module.configure_kueue] + source = "./helm_install" + + release_name = "nvidia-dra-driver-gpu" # The release name + chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts + chart_name = "nvidia-dra-driver-gpu" # The chart name + chart_version = var.nvidia_dra_driver.version # The chart version + namespace = "nvidia-dra-driver-gpu" # The target namespace + create_namespace = true # Equivalent to --create-namespace + + # Use the 'values' argument to pass the YAML content + # This corresponds to the -f <(cat < +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_monitoring_dashboard.dashboard](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_dashboard) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [base\_dashboard](#input\_base\_dashboard) | Baseline dashboard template, select from HPC or Empty | `string` | `"HPC"` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the monitoring dashboard instance. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [title](#input\_title) | Title of the created dashboard | `string` | `"Cluster Toolkit Dashboard"` | no | +| [widgets](#input\_widgets) | List of additional widgets to add to the base dashboard. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for accessing the monitoring dashboard | + diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl new file mode 100644 index 0000000000..f25cbbd2c6 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl @@ -0,0 +1,17 @@ +{ + "displayName": "${title}: ${deployment_name}", + "gridLayout": { + "columns": 2, + "widgets": [ + { + "text": { + "content": "Metrics from the ${deployment_name} deployment of the Cluster Toolkit.", + "format": "MARKDOWN" + }, + "title": "${title}" + }%{ for widget in widgets ~}, + ${widget} + %{endfor ~} + ] + } +} diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl new file mode 100644 index 0000000000..5b20435a9a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl @@ -0,0 +1,595 @@ +{ + "displayName": "${title}: ${deployment_name}", + "labels": ${jsonencode(labels)}, + "gridLayout": { + "columns": 2, + "widgets": [ + { + "text": { + "content": "HPC metrics from the ${deployment_name} deployment of the Cluster Toolkit.", + "format": "MARKDOWN" + }, + "title": "${title}" + }, + { + "title": "VM Instance - Memory utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/memory/percent_used\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - CPU Utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", + "pickTimeSeriesFilter": { + "direction": "TOP", + "numTimeSeries": 20, + "rankingMethod": "METHOD_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - CPU utilization (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + }, + "unitOverride": "%" + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk read operations", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/read_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk write operations", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/write_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk Read Bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/disk/read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk Write Bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/disk/write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Throttled read bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Throttled write bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Received packets", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/received_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - Sent packets", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - Received bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/received_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Sent bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_MEAN", + "groupByFields": [ + "metric.label.\"instance_name\"", + "metric.label.\"loadbalanced\"", + "resource.label.\"project_id\"", + "resource.label.\"instance_id\"", + "resource.label.\"zone\"" + ], + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Network Traffic Bytes (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/interface/traffic\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Network Packets (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/interface/packets\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "TCP connections", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/network/tcp_connections\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + }, + "unitOverride": "1" + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - CPU utilization for steal", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "STACKED_BAR", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MAX" + }, + "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\" metric.label.\"cpu_state\"=\"steal\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - CPU utilization [MEAN]", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }%{ for widget in widgets ~}, + ${widget} + %{endfor ~} + ] + } +} diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/main.tf b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/main.tf new file mode 100644 index 0000000000..df3c5c36b0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/main.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "dashboard", ghpc_role = "monitoring" }) +} + +locals { + dash_path = "${path.module}/dashboards/${var.base_dashboard}.json.tpl" +} + +resource "google_monitoring_dashboard" "dashboard" { + dashboard_json = templatefile(local.dash_path, { + widgets = var.widgets + deployment_name = var.deployment_name + title = var.title + labels = local.labels + } + ) + project = var.project_id +} diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml new file mode 100644 index 0000000000..de1a10f57d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - stackdriver.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf new file mode 100644 index 0000000000..b7ff35fb0e --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instructions" { + description = "Instructions for accessing the monitoring dashboard" + value = <<-EOT + A monitoring dashboard has been created. To view, navigate to the following URL: + https://console.cloud.google.com/monitoring/dashboards/builder${regex("/[0-9a-z-]*$", google_monitoring_dashboard.dashboard.id)} + EOT +} diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/variables.tf b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/variables.tf new file mode 100644 index 0000000000..8194f8b73a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/variables.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "base_dashboard" { + description = "Baseline dashboard template, select from HPC or Empty" + type = string + default = "HPC" + validation { + condition = contains(["HPC", "Empty"], var.base_dashboard) + error_message = "Must set var.base_dashboard to either \"HPC\" or \"Empty\"." + } +} + +variable "title" { + description = "Title of the created dashboard" + type = string + default = "Cluster Toolkit Dashboard" +} + +variable "widgets" { + description = "List of additional widgets to add to the base dashboard." + type = list(string) + default = [] +} + +variable "labels" { + description = "Labels to add to the monitoring dashboard instance. Key-value pairs." + type = map(string) +} diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/versions.tf b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/versions.tf new file mode 100644 index 0000000000..2717fe79f6 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/README.md b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/README.md new file mode 100644 index 0000000000..057f4b649d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/README.md @@ -0,0 +1,111 @@ +## Description + +This module facilitates the creation of custom firewall rules for existing +networks. + +## Example usage + +This module can be used by other Toolkit modules to create application-specific +firewall rules or in conjunction with the [pre-existing-vpc] module to enable +traffic in existing networks. The snippet below is drawn from the +[ml-slurm.yaml] example: + +```yaml +- group: primary + modules: + - id: network + source: modules/network/pre-existing-vpc + + # this example anticipates that the VPC default network has internal traffic + # allowed and IAP tunneling for SSH connections + - id: firewall_rule + source: modules/network/firewall-rules + use: + - network + settings: + ingress_rules: + - name: $(vars.deployment_name)-allow-internal-traffic + description: Allow internal traffic + destination_ranges: + - $(network.subnetwork_address) + source_ranges: + - $(network.subnetwork_address) + allow: + - protocol: tcp + ports: + - 0-65535 + - protocol: udp + ports: + - 0-65535 + - protocol: icmp + - name: $(vars.deployment_name)-allow-iap-ssh + description: Allow IAP-tunneled SSH connections + destination_ranges: + - $(network.subnetwork_address) + source_ranges: + - 35.235.240.0/20 + allow: + - protocol: tcp + ports: + - 22 +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [firewall\_rule](#module\_firewall\_rule) | terraform-google-modules/network/google//modules/firewall-rules | ~> 12.0 | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.pga_check](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [google_compute_subnetwork.subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [egress\_rules](#input\_egress\_rules) | List of egress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | +| [ingress\_rules](#input\_ingress\_rules) | List of ingress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | +| [network\_name](#input\_network\_name) | The name of the network to create firewall rules in | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The project ID to host the network in | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork whose global network firewall rules will be modified. | `string` | n/a | yes | + +## Outputs + +No outputs. + + +[pre-existing-vpc]: ../pre-existing-vpc/README.md +[ml-slurm.yaml]: ../../../examples/ml-slurm.yaml diff --git a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/main.tf b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/main.tf new file mode 100644 index 0000000000..05241278ad --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/main.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + use_subnetwork_data = (var.project_id == null || var.network_name == null) && var.subnetwork_self_link != null +} + +# the google_compute_network data source does not allow identification by +# self_link, which uniquely identifies subnet, project, and network +data "google_compute_subnetwork" "subnetwork" { + # Only instantiate this data source if needed + count = local.use_subnetwork_data ? 1 : 0 + self_link = var.subnetwork_self_link +} + +locals { + # Derived values from data source, null if data source is not used + derived_project_id = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].project : null + derived_network_name = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].network : null + + # Effective values: Use var if provided, otherwise use derived value + effective_project_id = coalesce(var.project_id, local.derived_project_id) + effective_network_name = coalesce(var.network_name, local.derived_network_name) +} + +# Module-level check for Private Google Access on the subnetwork. +# This check is only relevant if subnetwork_self_link was provided and used. +resource "terraform_data" "pga_check" { + count = local.use_subnetwork_data ? 1 : 0 + + lifecycle { + precondition { + condition = data.google_compute_subnetwork.subnetwork[0].private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.subnetwork[0].name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } + } +} + +module "firewall_rule" { + source = "terraform-google-modules/network/google//modules/firewall-rules" + version = "~> 12.0" + project_id = local.effective_project_id + network_name = local.effective_network_name + + ingress_rules = var.ingress_rules + egress_rules = var.egress_rules +} diff --git a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/variables.tf b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/variables.tf new file mode 100644 index 0000000000..05e9be4425 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/variables.tf @@ -0,0 +1,88 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork whose global network firewall rules will be modified." + type = string +} + +variable "project_id" { + description = "The project ID to host the network in" + type = string + default = null +} + +variable "network_name" { + description = "The name of the network to create firewall rules in" + type = string + default = null +} + +variable "ingress_rules" { + description = "List of ingress rules" + default = [] + type = list(object({ + name = string + description = optional(string, null) + disabled = optional(bool, null) + priority = optional(number, null) + destination_ranges = optional(list(string), []) + source_ranges = optional(list(string), []) + source_tags = optional(list(string)) + source_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + target_service_accounts = optional(list(string)) + + allow = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + deny = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + log_config = optional(object({ + metadata = string + })) + })) +} + +variable "egress_rules" { + description = "List of egress rules" + default = [] + type = list(object({ + name = string + description = optional(string, null) + disabled = optional(bool, null) + priority = optional(number, null) + destination_ranges = optional(list(string), []) + source_ranges = optional(list(string), []) + source_tags = optional(list(string)) + source_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + target_service_accounts = optional(list(string)) + + allow = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + deny = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + log_config = optional(object({ + metadata = string + })) + })) +} diff --git a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/versions.tf b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/versions.tf new file mode 100644 index 0000000000..9061dd3ae5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.74.0" + } + + required_version = ">= 1.5" +} diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md new file mode 100644 index 0000000000..abbfe3b97b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md @@ -0,0 +1,143 @@ +## Description + +This module accomplishes the following: + +* Creates one [VPC network][cft-network] + * Each VPC contains a variable number of subnetworks as specified in the + `subnetworks_template` variable + * Each subnetwork contains distinct IP address ranges +* Outputs the following unique parameters + * `subnetwork_interfaces` which is compatible with Slurm and vm-instance + modules + * `subnetwork_interfaces_gke` which is compatible with GKE modules + +This module is a simplified version of the VPC module and its main difference +is the variable `subnetwork_template` which is the template for all subnetworks +created within the network. This template contains the following values: + +1. `count`: The number of subnetworks to be created +1. `name_prefix`: The prefix for the subnetwork names +1. `ip_range`: [CIDR-formatted IP range][cidr] +1. `region`: The region where the subnetwork will be deployed + +> [!WARNING] +> The `ip_range` should be always be large enough to split into `count` +> subnetworks and the number of required connections within. + +[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v10.0.0 +[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation + +### Example + +This snippet uses the gpu-vpc module to create a new VPC network named +`test-rdma-net` with 8 subnetworks named `test-mrdma-sub-#` where # ranges from +0 to 7. The subnetworks will split the `ip_range` evenly, starting from bit 16 +(0 indexed). The networks are ingested by the Slurm nodeset within the +`additional_networks` setting. + +```yaml + - id: rdma-net + source: modules/network/gpu-rdma-vpc + settings: + network_name: test-rdma-net + network_profile: https://www.googleapis.com/compute/beta/projects/$(vars.project_id)/global/networkProfiles/$(vars.zone)-vpc-roce + network_routing_mode: REGIONAL + subnetworks_template: + name_prefix: test-mrdma-sub + count: 8 + ip_range: 192.168.0.0/16 + region: $(vars.region) + + - id: a3_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network0] + settings: + machine_type: a3-ultragpu-8g + additional_networks: + $(concat( + [{ + network=null, + subnetwork=network1.subnetwork_self_link, + subnetwork_project=vars.project_id, + nic_type="GVNIC", + queue_count=null, + network_ip="", + stack_type=null, + access_config=[], + ipv6_access_config=[], + alias_ip_range=[] + }], + rdma-net.subnetwork_interfaces + )) + ... +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | DEPRECATED: enable\_internal\_traffic can not be specified for gpu-rdma-vpc. | `bool` | `null` | no | +| [firewall\_log\_config](#input\_firewall\_log\_config) | DEPRECATED: firewall\_log\_config can not be specified for gpu-rdma-vpc. | `string` | `null` | no | +| [firewall\_rules](#input\_firewall\_rules) | DEPRECATED: firewall\_rules can not be specified for gpu-rdma-vpc. | `any` | `null` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}} | `string` | n/a | yes | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "REGIONAL") | `string` | `"REGIONAL"` | no | +| [nic\_type](#input\_nic\_type) | NIC type for use in modules that use the output | `string` | `"MRDMA"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | +| [subnetworks\_template](#input\_subnetworks\_template) | Specifications for the subnetworks that will be created within this VPC.

count (number, required, number of subnets to create, default is 8)
name\_prefix (string, required, subnet name prefix, default is deployment name)
ip\_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16)
region (string, optional, region to deploy subnets to, defaults to vars.region) | 
object({
    count       = number
    name_prefix = string
    ip_range    = string
    region      = optional(string)
  })
| 
{
  "count": 8,
  "ip_range": "192.168.0.0/16",
  "name_prefix": null,
  "region": null
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [network\_id](#output\_network\_id) | ID of the new VPC network | +| [network\_name](#output\_network\_name) | Name of the new VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | +| [subnetwork\_interfaces](#output\_subnetwork\_interfaces) | Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules) | +| [subnetwork\_interfaces\_gke](#output\_subnetwork\_interfaces\_gke) | Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool) | +| [subnetwork\_name\_prefix](#output\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | +| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | + diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf new file mode 100644 index 0000000000..e37db01976 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf @@ -0,0 +1,79 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + autoname = replace(var.deployment_name, "_", "-") + network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name + subnet_prefix = var.subnetworks_template.name_prefix == null ? "${local.autoname}-subnet" : var.subnetworks_template.name_prefix + + new_bits = ceil(log(var.subnetworks_template.count, 2)) + template_subnetworks = [for i in range(var.subnetworks_template.count) : + { + subnet_name = "${local.subnet_prefix}-${i}" + subnet_region = try(var.subnetworks_template.region, var.region) + subnet_ip = cidrsubnet(var.subnetworks_template.ip_range, local.new_bits, i) + } + ] + + firewall_rules = [] + + output_subnets = [ + for subnet in module.vpc.subnets : { + network = null + subnetwork = subnet.self_link + subnetwork_project = null # will populate from subnetwork_self_link + network_ip = null + nic_type = var.nic_type + stack_type = null + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } + ] + + output_subnets_gke = [ + for i in range(length(module.vpc.subnets)) : { + network = local.network_name + subnetwork = local.template_subnetworks[i].subnet_name + subnetwork_project = var.project_id + network_ip = null + nic_type = var.nic_type + stack_type = null + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } + ] +} + +module "vpc" { + source = "terraform-google-modules/network/google" + version = "~> 12.0" + + network_name = local.network_name + project_id = var.project_id + auto_create_subnetworks = false + subnets = local.template_subnetworks + routing_mode = var.network_routing_mode + mtu = var.mtu + description = var.network_description + shared_vpc_host = var.shared_vpc_host + delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes + firewall_rules = local.firewall_rules + network_profile = var.network_profile +} diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf new file mode 100644 index 0000000000..0a21f1d3f2 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the new VPC network" + value = module.vpc.network_name + depends_on = [module.vpc] +} + +output "network_id" { + description = "ID of the new VPC network" + value = module.vpc.network_id + depends_on = [module.vpc] +} + +output "network_self_link" { + description = "Self link of the new VPC network" + value = module.vpc.network_self_link + depends_on = [module.vpc] +} + +output "subnetworks" { + description = "Full list of subnetwork objects belonging to the new VPC network" + value = module.vpc.subnets + depends_on = [module.vpc] +} + +output "subnetwork_interfaces" { + description = "Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules)" + value = local.output_subnets + depends_on = [module.vpc] +} + +# The output subnetwork_interfaces is compatible with vm-instance module but not with gke-node-pool +# See https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/99493df21cecf6a092c45298bf7a45e0343cf622/modules/compute/vm-instance/variables.tf#L220 +# So, we need a separate output that makes the network and subnetwork names available +output "subnetwork_interfaces_gke" { + description = "Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool)" + value = local.output_subnets_gke + depends_on = [module.vpc] +} + +output "subnetwork_name_prefix" { + description = "Prefix of the RDMA subnetwork names" + value = var.subnetworks_template.name_prefix +} diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf new file mode 100644 index 0000000000..a30fb50e7d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf @@ -0,0 +1,164 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "network_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" + type = string + default = null +} + +variable "region" { + description = "The default region for Cloud resources" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "subnetworks_template" { + description = <<-EOT + Specifications for the subnetworks that will be created within this VPC. + + count (number, required, number of subnets to create, default is 8) + name_prefix (string, required, subnet name prefix, default is deployment name) + ip_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16) + region (string, optional, region to deploy subnets to, defaults to vars.region) + EOT + nullable = false + type = object({ + count = number + name_prefix = string + ip_range = string + region = optional(string) + }) + default = { + count = 8 + name_prefix = null + ip_range = "192.168.0.0/16" + region = null + } + + validation { + condition = var.subnetworks_template.count > 0 + error_message = "Number of subnetworks must be greater than 0" + } + + validation { + condition = can(cidrhost(var.subnetworks_template.ip_range, 0)) + error_message = "IP address range must be in CIDR format." + } +} + +variable "network_routing_mode" { + type = string + default = "REGIONAL" + description = "The network routing mode (default \"REGIONAL\")" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "shared_vpc_host" { + type = bool + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + default = false +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_internal_traffic" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." + type = bool + default = null + validation { + condition = var.enable_internal_traffic == null + error_message = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." + } +} + +variable "firewall_rules" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." + type = any + default = null + validation { + condition = var.firewall_rules == null + error_message = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." + } +} + +variable "firewall_log_config" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." + type = string + default = null + validation { + condition = var.firewall_log_config == null + error_message = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." + } +} + +variable "network_profile" { + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + EOT + type = string + nullable = false + + validation { + condition = can(coalesce(var.network_profile)) + error_message = "var.network_profile must be specified and not an empty string" + } +} + +variable "nic_type" { + description = "NIC type for use in modules that use the output" + type = string + nullable = true + default = "MRDMA" + + validation { + condition = contains(["MRDMA"], var.nic_type) + error_message = "The nic_type must be \"MRDMA\"." + } +} diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf new file mode 100644 index 0000000000..71b7106734 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.15.0" +} diff --git a/deletion-test/primary/modules/embedded/modules/network/multivpc/README.md b/deletion-test/primary/modules/embedded/modules/network/multivpc/README.md new file mode 100644 index 0000000000..973e6b32c9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/multivpc/README.md @@ -0,0 +1,136 @@ +## Description + +This module accomplishes the following: + +* Creates 2 to 8 [VPC networks][vpc] + * Each VPC contains exactly 1 subnetwork + * Each subnetwork contains distinct IP address ranges +* Outputs the `additional_networks` parameter, which is compatible with Slurm + modules + +There are 4 variables that differentiate this module from the standard VPC +module. + +1. `network_prefix`: The name prefix of the VPCs to be created. All + networks and subnetworks will start with this and end with a unique number. +1. `network_count`: The number of VPCs to be created. +1. `global_ip_address_range`: [CIDR-formatted IP range][cidr] +1. `network_cidr_suffix`: The CIDR suffix that defines the address + space that the individual VPCs will cover. + +> [!WARNING] +> The `network_cidr_suffix` should be always be larger than the CIDR suffix on +> `global_ip_address_range`. The difference between these two suffixes should +> be large enough to accommodate the number of VPCs that are being deployed +> (e.g. CIDR suffix bit difference <= `ceil(log2(network_count)))`). + +> [!NOTE] +> For deployments that need multiple VPCs that do not meet this use-case, users +> should deploy multiple individual VPC modules. + +[vpc]: ../vpc/README.md +[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation + +### Example + +This snippet uses the multivpc module to create 8 new VPC networks named +`multivpc-net-#` where # ranges from 0 to 7. Additionally, it creates 1 +subnetwork in each VPC. + +```yaml + - id: network + source: modules/network/vpc + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: a3_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network, multinetwork] + settings: + machine_type: a3-highgpu-8g + ... +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [vpcs](#module\_vpcs) | ../vpc | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.global_ip_cidr_suffix](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | +| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | +| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | +| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | +| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | +| [global\_ip\_address\_range](#input\_global\_ip\_address\_range) | IP address range (CIDR) that will span entire set of VPC networks | `string` | `"172.16.0.0/12"` | no | +| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT) | `number` | `2` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_count](#input\_network\_count) | The number of vpc nettworks to create | `number` | `4` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_interface\_defaults](#input\_network\_interface\_defaults) | The template of the network settings to be used on all vpcs. | 
object({
    network            = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string, "GVNIC")
    stack_type         = optional(string, "IPV4_ONLY")
    queue_count        = optional(string)
    access_config = optional(list(object({
      nat_ip                 = string
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  })
| 
{
  "access_config": [],
  "alias_ip_range": [],
  "ipv6_access_config": [],
  "network": null,
  "network_ip": "",
  "nic_type": "GVNIC",
  "queue_count": null,
  "stack_type": "IPV4_ONLY",
  "subnetwork": null,
  "subnetwork_project": null
}
| no | +| [network\_name\_prefix](#input\_network\_name\_prefix) | The base name of the vpcs and their subnets, will be appended with a sequence number | `string` | `""` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network dynamic routing mode | `string` | `"REGIONAL"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [subnetwork\_cidr\_suffix](#input\_subnetwork\_cidr\_suffix) | The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network. | `number` | `16` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [additional\_networks](#output\_additional\_networks) | Network interfaces for each subnetwork created by this module | +| [network\_ids](#output\_network\_ids) | IDs of the new VPC network | +| [network\_names](#output\_network\_names) | Names of the new VPC networks | +| [network\_self\_links](#output\_network\_self\_links) | Self link of the new VPC network | +| [subnetwork\_addresses](#output\_subnetwork\_addresses) | IP address range of the primary subnetwork | +| [subnetwork\_names](#output\_subnetwork\_names) | Names of the subnetwork created in each network | +| [subnetwork\_self\_links](#output\_subnetwork\_self\_links) | Self link of the primary subnetwork | + diff --git a/deletion-test/primary/modules/embedded/modules/network/multivpc/main.tf b/deletion-test/primary/modules/embedded/modules/network/multivpc/main.tf new file mode 100644 index 0000000000..ad06e793c1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/multivpc/main.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # this input variable is validated to be in CIDR format + network_name = coalesce(replace(var.network_name_prefix, "_", "-"), replace(var.deployment_name, "_", "-")) + global_ip_cidr_prefix = split("/", var.global_ip_address_range)[0] + global_ip_cidr_suffix = split("/", var.global_ip_address_range)[1] + global_ip_cidr_valid = "${local.global_ip_cidr_prefix}/${terraform_data.global_ip_cidr_suffix.output}" + subnetwork_new_bits = var.subnetwork_cidr_suffix - local.global_ip_cidr_suffix + maximum_subnetworks = pow(2, local.subnetwork_new_bits) + additional_networks = [ + for vpc in module.vpcs : + merge(var.network_interface_defaults, { + network = vpc.network_name + subnetwork = vpc.subnetwork_name + subnetwork_project = var.project_id + }) + ] +} + +resource "terraform_data" "global_ip_cidr_suffix" { + input = local.global_ip_cidr_suffix + lifecycle { + precondition { + condition = local.maximum_subnetworks >= var.network_count + error_message = < 1 + error_message = "The minimum VPCs able to be created by this module is 2. Use the standard Toolkit module at modules/network/vpc for count = 1" + } + validation { + condition = var.network_count <= 8 + error_message = "The maximum VPCs able to be created by this module is 8" + } +} + +variable "global_ip_address_range" { + description = "IP address range (CIDR) that will span entire set of VPC networks" + type = string + default = "172.16.0.0/12" + + validation { + condition = can(cidrhost(var.global_ip_address_range, 0)) + error_message = "var.global_ip_address_range must be an IPv4 CIDR range (e.g. \"172.16.0.0/12\")." + } +} + +variable "subnetwork_cidr_suffix" { + description = "The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network." + type = number + default = 16 +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "network_routing_mode" { + type = string + default = "REGIONAL" + description = "The network dynamic routing mode" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "ips_per_nat" { + type = number + description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT)" + default = 2 +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_iap_ssh_ingress" { + type = bool + description = "Enable a firewall rule to allow SSH access using IAP tunnels" + default = true +} + +variable "enable_iap_rdp_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" + default = false +} + +variable "enable_iap_winrm_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" + default = false +} + +variable "enable_internal_traffic" { + type = bool + description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" + default = true +} + +variable "extra_iap_ports" { + type = list(string) + description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" + default = [] +} + +variable "allowed_ssh_ip_ranges" { + type = list(string) + description = "A list of CIDR IP ranges from which to allow ssh access" + default = [] + + validation { + condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) + error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." + } +} + +variable "firewall_rules" { + type = any + description = "List of firewall rules" + default = [] +} + +variable "network_interface_defaults" { + type = object({ + network = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + network_ip = optional(string, "") + nic_type = optional(string, "GVNIC") + stack_type = optional(string, "IPV4_ONLY") + queue_count = optional(string) + access_config = optional(list(object({ + nat_ip = string + network_tier = string + public_ptr_domain_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + public_ptr_domain_name = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + }) + description = "The template of the network settings to be used on all vpcs." + default = { + network = null + subnetwork = null + subnetwork_project = null + network_ip = "" + nic_type = "GVNIC" + stack_type = "IPV4_ONLY" + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } +} + +variable "network_profile" { + type = string + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown + EOT + default = null +} diff --git a/deletion-test/primary/modules/embedded/modules/network/multivpc/versions.tf b/deletion-test/primary/modules/embedded/modules/network/multivpc/versions.tf new file mode 100644 index 0000000000..e75a67f7b6 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/multivpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.4.0" +} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md new file mode 100644 index 0000000000..4d63b17091 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md @@ -0,0 +1,94 @@ +## Description + +This module discovers a subnetwork that already exists in Google Cloud and +outputs subnetwork attributes that uniquely identify it for use by other modules. + +For example, the blueprint below discovers the referred to subnetwork. +With the `use` keyword, the [vm-instance] module accepts the `subnetwork_self_link` +input variables that uniquely identify the subnetwork in which the VM will be created. + +[vpc]: ../vpc/README.md +[vm-instance]: ../../compute/vm-instance/README.md + +> **_NOTE:_** Additional IAM work is needed for this to work correctly. + +### Example + +```yaml +- id: network + source: modules/network/pre-existing-subnetwork + settings: + subnetwork_self_link: https://www.googleapis.com/compute/v1/projects/name-of-host-project/regions/REGION/subnetworks/SUBNETNAME + +- id: example_vm + source: modules/compute/vm-instance + use: + - network + settings: + name_prefix: example + machine_type: c2-standard-4 +``` + +As described in documentation: +[https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork] + +If subnetwork_self_link is provided then name,region,project is ignored. + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [project](#input\_project) | Name of the project that owns the subnetwork | `string` | `null` | no | +| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | `null` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self-link of the subnet in the VPC | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | +| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | + diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf new file mode 100644 index 0000000000..9fb206f969 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +data "google_compute_subnetwork" "primary_subnetwork" { + name = var.subnetwork_name + region = var.region + project = var.project + self_link = var.subnetwork_self_link + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The subnetwork: ${coalesce(var.subnetwork_name, var.subnetwork_self_link)} could not be found." + } + } +} + +# Module-level check for Private Google Access on the subnetwork +check "private_google_access_enabled_subnetwork" { + assert { + condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } +} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml new file mode 100644 index 0000000000..6a6f1e5757 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com +ghpc: + has_to_be_used: true diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf new file mode 100644 index 0000000000..868708dc6b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "subnetwork" { + description = "Full subnetwork object in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork +} + +output "subnetwork_name" { + description = "Name of the subnetwork in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.name +} + +output "subnetwork_self_link" { + description = "Subnetwork self-link in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.self_link +} + +output "subnetwork_address" { + description = "Subnetwork IP range in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range +} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf new file mode 100644 index 0000000000..d5191843e8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "subnetwork_self_link" { + description = "Self-link of the subnet in the VPC" + type = string + default = null +} + +variable "project" { + description = "Name of the project that owns the subnetwork" + type = string + default = null +} + +variable "subnetwork_name" { + description = "Name of the pre-existing VPC subnetwork" + type = string + default = null +} + +variable "region" { + description = "Region in which to search for primary subnetwork" + type = string + default = null +} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf new file mode 100644 index 0000000000..917d948433 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.74.0" + } + + required_version = ">= 1.5" +} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/README.md b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/README.md new file mode 100644 index 0000000000..38a1840c2d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/README.md @@ -0,0 +1,110 @@ +## Description + +This module discovers a VPC network that already exists in Google Cloud and +outputs network attributes that uniquely identify it for use by other modules. +The module outputs are aligned with the [vpc module][vpc] so that it can be used +as a drop-in substitute when a VPC already exists. + +For example, the blueprint below discovers the "default" global network and the +"default" regional subnetwork in us-central1. With the `use` keyword, the +[vm-instance] module accepts the `network_self_link` and `subnetwork_self_link` +input variables that uniquely identify the network and subnetwork in which the +VM will be created. + +[vpc]: ../vpc/README.md +[vm-instance]: ../../compute/vm-instance/README.md + +### Example + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + settings: + project_id: $(vars.project_id) + region: us-central1 + +- id: example_vm + source: modules/compute/vm-instance + use: + - network1 + settings: + name_prefix: example + machine_type: c2-standard-4 +``` + +> **_NOTE:_** The `project_id` and `region` settings would be inferred from the +> deployment variables of the same name, but they are included here for clarity. + +### Use shared-vpc + +If a network is created in different project, this module can be used to +reference the network. To use a network from a different project first make sure +you have a [cloud nat][cloudnat] and [IAP][iap] forwarding. For more details, +refer [shared-vpc][shared-vpc-doc] + +[cloudnat]: https://cloud.google.com/nat/docs/overview +[iap]: https://cloud.google.com/iap/docs/using-tcp-forwarding +[shared-vpc-doc]: ../../../examples/README.md#hpc-slurm-sharedvpcyaml-community-badge-experimental-badge + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_network.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network) | data source | +| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [network\_name](#input\_network\_name) | Name of the existing VPC network | `string` | `"default"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | n/a | yes | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork; defaults to var.network\_name if set to null. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [network\_id](#output\_network\_id) | ID of the existing VPC network | +| [network\_name](#output\_network\_name) | Name of the existing VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the existing VPC network | +| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | +| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | + diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf new file mode 100644 index 0000000000..ed332bab72 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +data "google_compute_network" "vpc" { + name = var.network_name + project = var.project_id + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The network: ${var.network_name} could not be found in project: ${var.project_id}." + } + } +} + +locals { + subnetwork_name = var.subnetwork_name != null ? var.subnetwork_name : var.network_name +} + +data "google_compute_subnetwork" "primary_subnetwork" { + name = local.subnetwork_name + region = var.region + project = var.project_id + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The subnetwork: ${local.subnetwork_name} could not be found in project: ${var.project_id} and region: ${var.region}." + } + } +} + +# Module-level check for Private Google Access on the subnetwork +check "private_google_access_enabled_subnetwork" { + assert { + condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } +} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf new file mode 100644 index 0000000000..00861af5ca --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the existing VPC network" + value = data.google_compute_network.vpc.name +} + +output "network_id" { + description = "ID of the existing VPC network" + value = data.google_compute_network.vpc.id +} + +output "network_self_link" { + description = "Self link of the existing VPC network" + value = data.google_compute_network.vpc.self_link +} + +output "subnetwork" { + description = "Full subnetwork object in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork +} + +output "subnetwork_name" { + description = "Name of the subnetwork in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.name +} + +output "subnetwork_self_link" { + description = "Subnetwork self-link in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.self_link +} + +output "subnetwork_address" { + description = "Subnetwork IP range in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range +} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf new file mode 100644 index 0000000000..291a81604a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "network_name" { + description = "Name of the existing VPC network" + type = string + default = "default" +} + +variable "subnetwork_name" { + description = "Name of the pre-existing VPC subnetwork; defaults to var.network_name if set to null." + type = string + default = null +} + +variable "region" { + description = "Region in which to search for primary subnetwork" + type = string +} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf new file mode 100644 index 0000000000..81fe5aeff3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.74.0" + } + + required_version = ">= 1.5" +} diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/README.md b/deletion-test/primary/modules/embedded/modules/network/vpc/README.md new file mode 100644 index 0000000000..2c2b1aa1a3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/vpc/README.md @@ -0,0 +1,237 @@ +## Description + +This module creates a new [VPC network][vpc] with 1 or more subnetworks and +a [Cloud Router][router] for every region with a subnetwork. By default, it will +create: + +* A [Cloud NAT][nat] to enable outbound access to the public internet for VMs + without public IP addresses; VMs with public IP addresses bypass the NAT to + directly access the public internet +* A firewall rule that enables inbound SSH access from [Identity-Aware + Proxy][iap] +* A firewall rule that enables all traffic internal to the network + +This behavior is optional and can be configured as [described below](#inputs). +This module is based on networking support in the [Cloud Foundation +Toolkit][cft]. We recommend following the [documentation for the network +module][cft-network] and [submodules][cft-network-submodules] for more details. +In particular, the detailed structure of input variables can be found for: + +* [var.firewall\_rules](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/firewall-rules#inputs) +* [var.secondary\_ranges](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets#inputs) + +[vpc]: https://cloud.google.com/vpc +[router]: https://github.com/terraform-google-modules/terraform-google-cloud-router +[nat]: https://github.com/terraform-google-modules/terraform-google-cloud-nat +[iap]: https://cloud.google.com/iap +[cft]: https://cloud.google.com/foundation-toolkit +[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0 +[cft-network-submodules]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules + +Additionally, [Google Private Access][gpa] is enabled by default on all +subnetworks unless it is explicitly disabled. This setting ensures that all VMs +can use Google services such as [Cloud Storage][gcs] even if they do not have +public IP addresses or Cloud NAT is disabled. + +[gpa]: https://cloud.google.com/vpc/docs/private-google-access +[gcs]: https://cloud.google.com/storage + +### Example + +This creates a new VPC network named `cluster-net`. + +```yaml + - id: network1 + source: modules/network/vpc + settings: + network_name: cluster-net +``` + +### Deprecation warning + +The variables listed below have been deprecated and will be removed in a future +release. Until they are removed,You may continue to use them in Toolkit +blueprints with the same functionality as documented in the [Toolkit 1.0 +release][vpc1.0]. + +* Deprecated variables + * `var.primary_subnetwork` + * `var.additional_subnetworks` + * `var.subnetwork_size` + +[vpc1.0]: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/v1.0.0/modules/network/vpc/README.md + +The following variables have been added to support explicit IP ranges for +subnetworks while retaining existing functionality. We advise adopting them even +if not using explicit IP ranges . The Toolkit ***does not support*** mixing +deprecated variables with the new replacements. The new functionality is +described in [more detail below](#subnetworks). + +* New variables to adopt + * `var.subnetworks` + * A value for this can be generated by merging `var.primary_subnetwork` and + `var.additional_subnetworks` into a single list + * `var.default_primary_subnetwork_size` + * This variable has been renamed for clarity; its value can be directly + copied from an explicit setting for `var.subnetwork_size`; if your blueprint + does not have an explicit setting, the default values are the same + +### Subnetworks + +This module will always provision at least 1 "primary" subnetwork in which most +resources are expected to be provisioned. This primary subnetwork is determined +by + +1. The first element of [var.subnetworks](#input_subnetworks) if it is not the + empty list +2. A default subnetwork automatically calculated from + * [var.subnetwork_name](#input_subnetwork_name) + * [var.region](#input_region) + * [var.network_address_range](#input_network_address_range) + * [var.default_primary_subnetwork_size](#input_default_primary_subnetwork_size) + +If `var.subnetworks` is provided then the primary subnetwork name is taken +explicitly from it and `var.subnetwork_name` is ignored. + +`var.subnetworks` behaves identically to the [Cloud Foundation Toolkit subnets +module][cftsubnets] with the lone exception that one can provide ***one*** of +the following settings for each subnetwork: + +* `new_bits` +* `subnet_ip` + +If each subnetwork defines `subnet_ip` then these are taken to be their explicit +CIDR IP ranges. If each subnetwork defines `new_bits`, then these are taken to +be the size of the CIDR subnetwork (in bits). IP ranges for each subnetwork are +calculated using `var.network_address_range` as the base IP, producing the most +compact set of subnetworks possible. + +> **_NOTE:_** we do not presently support the modification of individual subnetworks +> when using this module to provision more than 1 subnetwork using automatically +> calculated IP ranges based upon `new_bits`. Doing so will cause IP ranges to be +> recalculated for each subnetwork. We advise appending new subnetworks to the end +> of `var.subnetworks`. + +[cftsubnets]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets + +### SSH Access + +By default a firewall rule is created to allow inbound SSH access from +[Identity-Aware Proxy][iap]. A user must have the `IAP-Secured Tunnel User` +(`roles/iap.tunnelResourceAccessor`) IAM role to be able to SSH over IAP. + +To allow regular SSH access from a known IP address you can add the following +`firewall_rules` setting to the `vpc` module: + +```yaml + - id: network1 + source: modules/network/vpc + settings: + firewall_rules: + - name: ssh-my-machine + direction: INGRESS + ranges: [/32] + allow: + - protocol: tcp + ports: [22] +``` + +> **Note**: You must populate the above example with the source IP address from +> which you plan to SSH from. You can use a service like +> [whatismyip.com](https://whatismyip.com) to determine your IP address. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +| Name | Version | +|------|---------| +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 7.3 | +| [nat\_ip\_addresses](#module\_nat\_ip\_addresses) | terraform-google-modules/address/google | ~> 4.1 | +| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.cloud_nat_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.network_profile_firewall_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.secondary_ranges_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_subnetworks](#input\_additional\_subnetworks) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `list(map(string))` | `null` | no | +| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | +| [default\_primary\_subnetwork\_size](#input\_default\_primary\_subnetwork\_size) | The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks | `number` | `15` | no | +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_cloud\_nat](#input\_enable\_cloud\_nat) | Enable the creation of Cloud NATs. | `bool` | `true` | no | +| [enable\_cloud\_router](#input\_enable\_cloud\_router) | Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview | `bool` | `true` | no | +| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | +| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | +| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | +| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | +| [firewall\_log\_config](#input\_firewall\_log\_config) | Firewall log configuration for Toolkit firewall rules (var.enable\_iap\_ssh\_ingress and others) | `string` | `"DISABLE_LOGGING"` | no | +| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | +| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples | `number` | `2` | no | +| [labels](#input\_labels) | Labels to add to network resources that support labels. Key-value pairs of strings. | `map(string)` | `{}` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_address\_range](#input\_network\_address\_range) | IP address range (CIDR) for global network | `string` | `"10.0.0.0/9"` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "GLOBAL") | `string` | `"GLOBAL"` | no | +| [primary\_subnetwork](#input\_primary\_subnetwork) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `map(string)` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [secondary\_ranges](#input\_secondary\_ranges) | "Secondary ranges associated with the subnets.
This will be deprecated in favour of secondary\_ranges\_list at a later date.
Please migrate to using the same." | `map(list(object({ range_name = string, ip_cidr_range = string })))` | `{}` | no | +| [secondary\_ranges\_list](#input\_secondary\_ranges\_list) | "List of secondary ranges associated with the subnetworks.
Each subnetwork must be specified at most once in this list." | 
list(object({
    subnetwork_name = string,
    ranges = list(object({
      range_name    = string,
      ip_cidr_range = string
    }))
  }))
| `[]` | no | +| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-primary-subnet") | `string` | `null` | no | +| [subnetwork\_size](#input\_subnetwork\_size) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `number` | `null` | no | +| [subnetworks](#input\_subnetworks) | List of subnetworks to create within the VPC. If left empty, it will be
replaced by a single, default subnetwork constructed from other parameters
(e.g. var.region). In all cases, the first subnetwork in the list is identified
by outputs as a "primary" subnetwork.

subnet\_name (string, required, name of subnet)
subnet\_region (string, required, region of subnet)
subnet\_ip (string, mutually exclusive with new\_bits, CIDR-formatted IP range for subnetwork)
new\_bits (number, mutually exclusive with subnet\_ip, CIDR bits used to calculate subnetwork range)
subnet\_private\_access (bool, optional, Enable Private Access on subnetwork)
subnet\_flow\_logs (map(string), optional, Configure Flow Logs see terraform-google-network module)
description (string, optional, Description of Network)
purpose (string, optional, related to Load Balancing)
role (string, optional, related to Load Balancing) | `list(map(string))` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nat\_ips](#output\_nat\_ips) | External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used) | +| [network\_id](#output\_network\_id) | ID of the new VPC network | +| [network\_name](#output\_network\_name) | Name of the new VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | +| [subnetwork](#output\_subnetwork) | Primary subnetwork object | +| [subnetwork\_address](#output\_subnetwork\_address) | IP address range of the primary subnetwork | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the primary subnetwork | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Self link of the primary subnetwork | +| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | + diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/main.tf b/deletion-test/primary/modules/embedded/modules/network/vpc/main.tf new file mode 100644 index 0000000000..24c8eb22bd --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/vpc/main.tf @@ -0,0 +1,256 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +resource "terraform_data" "secondary_ranges_validation" { + lifecycle { + precondition { + condition = !(length(var.secondary_ranges) > 0 && length(var.secondary_ranges_list) > 0) + error_message = "Only one of var.secondary_ranges or var.secondary_ranges_list should be specified" + } + } +} + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "vpc", ghpc_role = "network" }) +} + +locals { + autoname = replace(var.deployment_name, "_", "-") + network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name + subnetwork_name = var.subnetwork_name == null ? "${local.autoname}-primary-subnet" : var.subnetwork_name + + # define a default subnetwork for cases in which no explicit subnetworks are + # defined in var.subnetworks + default_primary_subnetwork_cidr_block = cidrsubnet(var.network_address_range, var.default_primary_subnetwork_size, 0) + default_primary_subnetwork = { + subnet_name = local.subnetwork_name + subnet_ip = local.default_primary_subnetwork_cidr_block + subnet_region = var.region + subnet_private_access = true + subnet_flow_logs = false + description = "primary subnetwork in ${local.network_name}" + purpose = null + role = null + } + + # Identify user-supplied primary subnetwork + # (1) explicit var.subnetworks[0] + # (2) implicit local default subnetwork + input_primary_subnetwork = coalesce(try(var.subnetworks[0], null), local.default_primary_subnetwork) + + # Identify user-supplied additional subnetworks + # (1) explicit var.subnetworks[1:end] + # (2) empty list + input_additional_subnetworks = try(slice(var.subnetworks, 1, length(var.subnetworks)), []) + + # at this point we have constructed a list of subnetworks but need to extract + # user-provided CIDR blocks or calculate them from user-provided new_bits + # after we complete deprecation, local.all_subnetworks can be replaced with + # var.subnetworks (or local.default_primary_subnetwork if that is null) + input_subnetworks = concat([local.input_primary_subnetwork], local.input_additional_subnetworks) + subnetworks_cidr_blocks = try( + local.input_subnetworks[*]["subnet_ip"], + cidrsubnets(var.network_address_range, local.input_subnetworks[*]["new_bits"]...) + ) + + # merge in the CIDR blocks (even when already there) and remove new_bits + subnetworks = [for i, subnet in local.input_subnetworks : + merge({ for k, v in subnet : k => v if k != "new_bits" }, { "subnet_ip" = local.subnetworks_cidr_blocks[i] }) + ] + + # gather the unique regions for purposes of creating Router/NAT + cloud_router_regions = var.enable_cloud_router ? distinct([for subnet in local.subnetworks : subnet.subnet_region]) : [] + cloud_nat_regions = var.enable_cloud_nat ? local.cloud_router_regions : [] + + # this comprehension should have 1 and only 1 match + output_primary_subnetwork = one([for k, v in module.vpc.subnets : v if k == "${local.subnetworks[0].subnet_region}/${local.subnetworks[0].subnet_name}"]) + output_primary_subnetwork_name = local.output_primary_subnetwork.name + output_primary_subnetwork_self_link = local.output_primary_subnetwork.self_link + output_primary_subnetwork_ip_cidr_range = local.output_primary_subnetwork.ip_cidr_range + + iap_ports = distinct(concat(compact([ + var.enable_iap_rdp_ingress ? "3389" : "", + var.enable_iap_ssh_ingress ? "22" : "", + var.enable_iap_winrm_ingress ? "5986" : "", + ]), var.extra_iap_ports)) + + firewall_log_api_values = { + "DISABLE_LOGGING" = null + "INCLUDE_ALL_METADATA" = { metadata = "INCLUDE_ALL_METADATA" }, + "EXCLUDE_ALL_METADATA" = { metadata = "EXCLUDE_ALL_METADATA" }, + } + firewall_log_config = lookup(local.firewall_log_api_values, var.firewall_log_config, null) + + allow_iap_ingress = { + name = "${local.network_name}-fw-allow-iap-ingress" + description = "allow TCP access via Identity-Aware Proxy" + direction = "INGRESS" + priority = null + ranges = ["35.235.240.0/20"] + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = local.iap_ports + }] + deny = [] + log_config = local.firewall_log_config + } + + allow_ssh_ingress = { + name = "${local.network_name}-fw-allow-ssh-ingress" + description = "allow SSH access" + direction = "INGRESS" + priority = null + ranges = var.allowed_ssh_ip_ranges + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = ["22"] + }] + deny = [] + log_config = local.firewall_log_config + } + + allow_internal_traffic = { + name = "${local.network_name}-fw-allow-internal-traffic" + priority = null + description = "allow traffic between nodes of this VPC" + direction = "INGRESS" + ranges = [var.network_address_range] + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = ["0-65535"] + }, { + protocol = "udp" + ports = ["0-65535"] + }, { + protocol = "icmp" + ports = null + }, + ] + deny = [] + log_config = local.firewall_log_config + } + + firewall_rules = concat( + var.firewall_rules, + length(var.allowed_ssh_ip_ranges) > 0 ? [local.allow_ssh_ingress] : [], + var.enable_internal_traffic ? [local.allow_internal_traffic] : [], + length(local.iap_ports) > 0 ? [local.allow_iap_ingress] : [] + ) + + secondary_ranges_map = { + for secondary_range in var.secondary_ranges_list : + secondary_range.subnetwork_name => secondary_range.ranges + } +} + +resource "terraform_data" "network_profile_firewall_validation" { + lifecycle { + precondition { + condition = !(try(strcontains(var.network_profile, "roce"), false) && length(local.firewall_rules) > 0) + error_message = "If var.network_profile contains 'roce', var.firewall_rules must be empty and var.enable_internal_traffic must be false, please see: https://cloud.google.com/vpc/docs/rdma-network-profiles#additional_features_that_dont_apply_to_traffic_from_rdma_nics" + } + } +} + +module "vpc" { + source = "terraform-google-modules/network/google" + version = "~> 12.0" + + depends_on = [terraform_data.network_profile_firewall_validation] + + network_name = local.network_name + project_id = var.project_id + auto_create_subnetworks = false + subnets = local.subnetworks + secondary_ranges = length(local.secondary_ranges_map) > 0 ? local.secondary_ranges_map : var.secondary_ranges + routing_mode = var.network_routing_mode + mtu = var.mtu + description = var.network_description + shared_vpc_host = var.shared_vpc_host + delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes + firewall_rules = local.firewall_rules + network_profile = var.network_profile +} + +resource "terraform_data" "cloud_nat_validation" { + lifecycle { + precondition { + condition = var.enable_cloud_router == true || var.enable_cloud_nat == false + error_message = <<-EOD + "Cannot have Cloud NAT without a Cloud Router. If you desire Cloud NAT functionality please set `enable_cloud_router` to true." + EOD + } + } +} + +# This use of the module may appear odd when var.ips_per_nat = 0. The module +# will be called for all regions with subnetworks but names will be set to the +# empty list. This is a perfectly valid value (the default!). In this scenario, +# no IP addresses are created and all module outputs are empty lists. +# +# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/variables.tf#L27 +# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/outputs.tf +module "nat_ip_addresses" { + source = "terraform-google-modules/address/google" + version = "~> 4.1" + + depends_on = [terraform_data.cloud_nat_validation] + + for_each = toset(local.cloud_nat_regions) + + project_id = var.project_id + region = each.value + # an external, regional (not global) IP address is suited for a regional NAT + address_type = "EXTERNAL" + global = false + labels = local.labels + names = [for idx in range(var.ips_per_nat) : "${local.network_name}-nat-ips-${each.value}-${idx}"] +} + +module "cloud_router" { + source = "terraform-google-modules/cloud-router/google" + version = "~> 7.3" + + depends_on = [terraform_data.cloud_nat_validation] + + for_each = toset(local.cloud_router_regions) + + project = var.project_id + name = "${local.network_name}-router" + region = each.value + network = module.vpc.network_name + # in scenario with no NAT IPs, no NAT is created even if router is created + # https://github.com/terraform-google-modules/terraform-google-cloud-router/blob/v2.0.0/nat.tf#L18-L20 + nats = length(module.nat_ip_addresses[each.value].self_links) == 0 ? [] : [ + { + name : "cloud-nat-${each.value}", + nat_ips : module.nat_ip_addresses[each.value].self_links + }, + ] +} diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/metadata.yaml b/deletion-test/primary/modules/embedded/modules/network/vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/outputs.tf b/deletion-test/primary/modules/embedded/modules/network/vpc/outputs.tf new file mode 100644 index 0000000000..c2ee6bdf6b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/vpc/outputs.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the new VPC network" + value = module.vpc.network_name + depends_on = [module.vpc, module.cloud_router] +} + +output "network_id" { + description = "ID of the new VPC network" + value = module.vpc.network_id + depends_on = [module.vpc, module.cloud_router] +} + +output "network_self_link" { + description = "Self link of the new VPC network" + value = module.vpc.network_self_link + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetworks" { + description = "Full list of subnetwork objects belonging to the new VPC network" + value = module.vpc.subnets + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork" { + description = "Primary subnetwork object" + value = local.output_primary_subnetwork + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_name" { + description = "Name of the primary subnetwork" + value = local.output_primary_subnetwork_name + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_self_link" { + description = "Self link of the primary subnetwork" + value = local.output_primary_subnetwork_self_link + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_address" { + description = "IP address range of the primary subnetwork" + value = local.output_primary_subnetwork_ip_cidr_range + depends_on = [module.vpc, module.cloud_router] +} + +output "nat_ips" { + description = "External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used)" + value = flatten([for ipmod in module.nat_ip_addresses : ipmod.addresses]) +} diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/variables.tf b/deletion-test/primary/modules/embedded/modules/network/vpc/variables.tf new file mode 100644 index 0000000000..e036189404 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/vpc/variables.tf @@ -0,0 +1,301 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "labels" { + description = "Labels to add to network resources that support labels. Key-value pairs of strings." + type = map(string) + default = {} + nullable = false +} + +variable "network_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" + type = string + default = null +} + +variable "subnetwork_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-primary-subnet\")" + type = string + default = null +} + +# tflint-ignore: terraform_unused_declarations +variable "subnetwork_size" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = number + default = null + validation { + condition = var.subnetwork_size == null + error_message = "subnetwork_size is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +variable "default_primary_subnetwork_size" { + description = "The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks" + type = number + default = 15 +} + +variable "region" { + description = "The default region for Cloud resources" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "network_address_range" { + description = "IP address range (CIDR) for global network" + type = string + default = "10.0.0.0/9" + + validation { + condition = can(cidrhost(var.network_address_range, 0)) + error_message = "IP address range must be in CIDR format." + } +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "subnetworks" { + description = <<-EOT + List of subnetworks to create within the VPC. If left empty, it will be + replaced by a single, default subnetwork constructed from other parameters + (e.g. var.region). In all cases, the first subnetwork in the list is identified + by outputs as a "primary" subnetwork. + + subnet_name (string, required, name of subnet) + subnet_region (string, required, region of subnet) + subnet_ip (string, mutually exclusive with new_bits, CIDR-formatted IP range for subnetwork) + new_bits (number, mutually exclusive with subnet_ip, CIDR bits used to calculate subnetwork range) + subnet_private_access (bool, optional, Enable Private Access on subnetwork) + subnet_flow_logs (map(string), optional, Configure Flow Logs see terraform-google-network module) + description (string, optional, Description of Network) + purpose (string, optional, related to Load Balancing) + role (string, optional, related to Load Balancing) + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_name"]) + ]) + error_message = "All subnetworks must define \"subnet_name\"." + } + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_region"]) + ]) + error_message = "All subnetworks must define \"subnet_region\"." + } + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_ip"]) != can(s["new_bits"]) + ]) + error_message = "All subnetworks must define exactly one of \"subnet_ip\" or \"new_bits\"." + } + validation { + condition = alltrue([for s in var.subnetworks : can(s["subnet_ip"])]) || alltrue([for s in var.subnetworks : can(s["new_bits"])]) + error_message = "All subnetworks must make same choice of \"subnet_ip\" or \"new_bits\"." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "primary_subnetwork" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = map(string) + default = null + validation { + condition = var.primary_subnetwork == null + error_message = "primary_subnetwork is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "additional_subnetworks" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = list(map(string)) + default = null + validation { + condition = var.additional_subnetworks == null + error_message = "additional_subnetworks is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +variable "secondary_ranges" { + type = map(list(object({ range_name = string, ip_cidr_range = string }))) + description = <<-EOT + "Secondary ranges associated with the subnets. + This will be deprecated in favour of secondary_ranges_list at a later date. + Please migrate to using the same." + EOT + default = {} +} + +variable "secondary_ranges_list" { + type = list(object({ + subnetwork_name = string, + ranges = list(object({ + range_name = string, + ip_cidr_range = string + })) + })) + description = <<-EOT + "List of secondary ranges associated with the subnetworks. + Each subnetwork must be specified at most once in this list." + EOT + default = [] + validation { + condition = (length(var.secondary_ranges_list[*].subnetwork_name) == + length(distinct(var.secondary_ranges_list[*].subnetwork_name))) + error_message = "Each subnetwork should be specified at most once in this list. Remove any duplicates." + } +} + +variable "network_routing_mode" { + type = string + default = "GLOBAL" + description = "The network routing mode (default \"GLOBAL\")" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "ips_per_nat" { + type = number + description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples" + default = 2 +} + +variable "shared_vpc_host" { + type = bool + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + default = false +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_iap_ssh_ingress" { + type = bool + description = "Enable a firewall rule to allow SSH access using IAP tunnels" + default = true +} + +variable "enable_iap_rdp_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" + default = false +} + +variable "enable_iap_winrm_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" + default = false +} + +variable "enable_internal_traffic" { + type = bool + description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" + default = true +} + +variable "enable_cloud_router" { + type = bool + description = "Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview" + default = true +} + +variable "enable_cloud_nat" { + type = bool + description = "Enable the creation of Cloud NATs." + default = true +} + +variable "extra_iap_ports" { + type = list(string) + description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" + default = [] +} + +variable "allowed_ssh_ip_ranges" { + type = list(string) + description = "A list of CIDR IP ranges from which to allow ssh access" + default = [] + + validation { + condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) + error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." + } +} + +variable "firewall_rules" { + type = any + description = "List of firewall rules" + default = [] +} + +variable "firewall_log_config" { + type = string + description = "Firewall log configuration for Toolkit firewall rules (var.enable_iap_ssh_ingress and others)" + default = "DISABLE_LOGGING" + nullable = false + + validation { + condition = contains([ + "INCLUDE_ALL_METADATA", + "EXCLUDE_ALL_METADATA", + "DISABLE_LOGGING", + ], var.firewall_log_config) + error_message = "var.firewall_log_config must be set to \"DISABLE_LOGGING\", or enable logging with \"INCLUDE_ALL_METADATA\" or \"EXCLUDE_ALL_METADATA\"" + } +} + +variable "network_profile" { + type = string + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown + EOT + default = null +} diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/versions.tf b/deletion-test/primary/modules/embedded/modules/network/vpc/versions.tf new file mode 100644 index 0000000000..71b7106734 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/network/vpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.15.0" +} diff --git a/deletion-test/primary/modules/embedded/modules/packer/custom-image/README.md b/deletion-test/primary/modules/embedded/modules/packer/custom-image/README.md new file mode 100644 index 0000000000..192d7575a5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/packer/custom-image/README.md @@ -0,0 +1,320 @@ +# Custom Images in the Cluster Toolkit (formerly HPC Toolkit) + +Please review the +[introduction to image building](../../../docs/image-building.md) for general +information on building custom images using the Toolkit. + +## Introduction + +This module uses [Packer](https://www.packer.io/) to create an image within an +Cluster Toolkit deployment. Packer operates by provisioning a short-lived VM in +Google Cloud on which it executes scripts to customize the boot disk for +repeated use. The VM's boot disk is specified from a source image that defaults +to the [HPC VM Image][hpcimage]. This Packer "template" supports customization +by the following approaches following a [recommended use](#recommended-use): + +- [startup-script metadata][startup-metadata] from [raw string][sss] or + [file][ssf] +- [Shell scripts][shell] uploaded from the Packer execution environment to the + VM +- [Ansible playbooks][ansible] uploaded from the Packer execution environment to + the VM + +They can be specified independently of one another, so that anywhere from 1 to 3 +solutions can be used simultaneously. In the case that 0 scripts are supplied, +the source boot disk is effectively copied to your project without +customization. This can be useful in scenarios where increased control over the +image maintenance lifecycle is desired or when policies restrict the use of +images to internal projects. + +## Minimum requirements + +### Outbound internet access + +Most customization scripts require access to resources on the public internet. +This can be achieved by one of the following 2 approaches: + +1. Using a public IP address on the VM + +- Set [var.omit_external_ip](#input_omit_external_ip) to `false` + +1. Configuring a VPC with a Cloud NAT in the region of the VM + +- Use the [vpc] module which automates NAT creation + +### Inbound internet access + +Read [order of execution](#order-of-execution) below for a discussion of VM +customization solutions and their requirements for inbound SSH access. +[Environments without SSH access](#environments-without-ssh-access) should use +the metadata-based startup-script solution. + +A simple way to enable inbound SSH access is to use the VPC module with +`allowed_ssh_ip_ranges` set to `0.0.0.0/0`. + +### User or service account executing Packer at command line + +The user or service account running Packer must have the permission to create +VMs in the selected VPC network and, if [use\_iap](#input_use_iap) is set, must +have the "IAP-Secured Tunnel User" role. Recommended roles are: + +- `roles/compute.instanceAdmin.v1` +- `roles/iap.tunnelResourceAccessor` + +### VM service account roles + +The service account attached to the temporary build VM created by Packer should +have the ability to write Cloud Logging entries so that you may inspect and +debug build logs. When using the metadata startup-script customization solution, +the service account attached to the temporary build VM created by Packer must +have the permission to modify its own metadata and to read from Cloud Storage +buckets. Recommended roles are: + +- `roles/compute.instanceAdmin.v1` +- `roles/iam.serviceAccountUser` +- `roles/logging.logWriter` +- `roles/monitoring.metricWriter` +- `roles/storage.objectViewer` + +It is recommended to create this service account as a separate step outside a +blueprint due to known delay in [IAM bindings propagation][iamprop]. + +## Example blueprints + +A recommended pattern for building images with this module is to use the +terraform based [startup-script] module along with this packer custom-image +module. Below you can find links to several examples of this pattern, including +usage instructions. + +### [Image Builder] + +The [Image Builder] blueprint demonstrates a solution that builds an image +using: + +- The [HPC VM Image][hpcimage] as a base upon which to customize +- A VPC network with firewall rules that allow IAP-based SSH tunnels +- A Toolkit runner that installs a custom script + +Please review the [examples README] for usage instructions. + +## Order of execution + +The startup script specified in metadata executes in parallel with the other +supported methods. However, the remaining methods execute in a well-defined +order relative to one another. + +1. All shell scripts will execute in the configured order +1. After shell scripts complete, all Ansible playbooks will execute in the + configured order + +> **_NOTE:_** if both [startup_script][sss] and [startup_script_file][ssf] are +> specified, then [startup_script_file][ssf] takes precedence. + +## Recommended use + +Because the [metadata startup script executes in parallel](#order-of-execution) +with the other solutions, conflicts can arise, especially when package managers +(`yum` or `apt`) lock their databases during package installation. Therefore, it +is recommended to choose one of the following approaches: + +1. Specify _either_ [startup_script][sss] _or_ [startup_script_file][ssf] and do + not specify [shell_scripts][shell] or [ansible_playbooks][ansible]. + - This can be especially useful in + [environments that restrict SSH access](#environments-without-ssh-access) +1. Specify any combination of [shell_scripts][shell] and + [ansible_playbooks][ansible] and do not specify [startup_script][sss] or + [startup_script_file][ssf]. + +If any of the startup script approaches fail by returning a code other than 0, +Packer will determine that the build has failed and refuse to save the image. + +## External access with SSH + +The [shell scripts][shell] and [Ansible playbooks][ansible] customization +solutions both require SSH access to the VM from the Packer execution +environment. SSH access can be enabled one of 2 ways: + +1. The VM is created without a public IP address and SSH tunnels are created + using [Identity-Aware Proxy (IAP)][iaptunnel]. + - Allow [use_iap](#input_use_iap) to take on its default value of `true` +1. The VM is created with an IP address on the public internet and firewall + rules allow SSH access from the Packer execution environment. + - Set `omit_external_ip = false` (or `omit_external_ip: false` in a + blueprint) + - Add firewall rules that open SSH to the VM + +The Packer template defaults to using to the 1st IAP-based solution because it +is more secure (no exposure to public internet) and because the [vpc] module +automatically sets up all necessary firewall rules for SSH tunneling and +outbound-only access to the internet through [Cloud NAT][cloudnat]. + +In either SSH solution, customization scripts should be supplied as files in the +[shell_scripts][shell] and [ansible_playbooks][ansible] settings. + +## Environments without SSH access + +Many network environments disallow SSH access to VMs. In these environments, the +[metadata-based startup scripts][startup-metadata] are appropriate because they +execute entirely independently of the Packer execution environment. + +In this scenario, a single scripts should be supplied in the form of a string to +the [startup_script][sss] input variable. This solution integrates well with +Toolkit runners. Runners operate by using a single startup script whose behavior +is extended by downloading and executing a customizable set of runners from +Cloud Storage at startup. + +> **_NOTE:_** Packer will attempt to use SSH if either [shell_scripts][shell] or +> [ansible_playbooks][ansible] are set to non-empty values. Leave them at their +> default, empty values to ensure access by SSH is disabled. + +## Supplying startup script as a string + +The [startup_script][sss] parameter accepts scripts formatted as strings. In +Packer and Terraform, multi-line strings can be specified using +[heredoc syntax](https://www.terraform.io/language/expressions/strings#heredoc-strings) +in an input [Packer variables file][pkrvars] (`*.pkrvars.hcl`) For example, the +following snippet defines a multi-line bash script followed by an integer +representing the size, in GiB, of the resulting image: + +```hcl +startup_script = <<-EOT + #!/bin/bash + yum install -y epel-release + yum install -y jq + EOT + +disk_size = 100 +``` + +In a blueprint, the equivalent syntax is: + +```yaml +... + settings: + startup_script: | + #!/bin/bash + yum install -y epel-release + yum install -y jq + disk_size: 100 +... +``` + +## Monitoring startup script execution + +When using startup script customization, Packer will print very limited output +to the console. For example: + +```text +==> example.googlecompute.toolkit_image: Waiting for any running startup script to finish... +==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... +==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... +==> example.googlecompute.toolkit_image: Startup script, if any, has finished running. +``` + +### Debugging startup-script failures + +> [!NOTE] +> There can be a delay in the propagation of the logs from the instance to +> Cloud Logging, so it may require waiting a few minutes to see the full logs. + +If the Packer image build fails, the module will output a `gcloud` command +that can be used directly to review startup-script execution. + +## License + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + +```text + http://www.apache.org/licenses/LICENSE-2.0 +``` + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_count](#input\_accelerator\_count) | Number of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `number` | `null` | no | +| [accelerator\_type](#input\_accelerator\_type) | Type of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `string` | `null` | no | +| [ansible\_playbooks](#input\_ansible\_playbooks) | A list of Ansible playbook configurations that will be uploaded to customize the VM image | 
list(object({
    playbook_file   = string
    galaxy_file     = string
    extra_arguments = list(string)
  }))
| `[]` | no | +| [communicator](#input\_communicator) | Communicator to use for provisioners that require access to VM ("ssh" or "winrm") | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name | `string` | n/a | yes | +| [disk\_size](#input\_disk\_size) | Size of disk image in GB | `number` | `null` | no | +| [disk\_type](#input\_disk\_type) | Type of persistent disk to provision | `string` | `"pd-balanced"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [image\_family](#input\_image\_family) | The family name of the image to be built. Defaults to `deployment_name` | `string` | `null` | no | +| [image\_name](#input\_image\_name) | The name of the image to be built. If not supplied, it will be set to image\_family-$ISO\_TIMESTAMP | `string` | `null` | no | +| [image\_storage\_locations](#input\_image\_storage\_locations) | Storage location, either regional or multi-regional, where snapshot content is to be stored and only accepts 1 value.
See https://developer.hashicorp.com/packer/plugins/builders/googlecompute#image_storage_locations | `list(string)` | `null` | no | +| [labels](#input\_labels) | Labels to apply to the short-lived VM | `map(string)` | `null` | no | +| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2-standard-4"` | no | +| [manifest\_file](#input\_manifest\_file) | File to which to write Packer build manifest | `string` | `"packer-manifest.json"` | no | +| [metadata](#input\_metadata) | Instance metadata for the builder VM (use var.startup\_script or var.startup\_script\_file to set startup-script metadata) | `map(string)` | `{}` | no | +| [network\_project\_id](#input\_network\_project\_id) | Project ID of Shared VPC network | `string` | `null` | no | +| [omit\_external\_ip](#input\_omit\_external\_ip) | Provision the image building VM without a public IP address | `bool` | `true` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which to create VM and image | `string` | n/a | yes | +| [scopes](#input\_scopes) | DEPRECATED: use var.service\_account\_scopes | `set(string)` | `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | The service account email to use. If null or 'default', then the default Compute Engine service account will be used. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Service account scopes to attach to the instance. See
https://cloud.google.com/compute/docs/access/service-accounts. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shell\_scripts](#input\_shell\_scripts) | A list of paths to local shell scripts which will be uploaded to customize the VM image | `list(string)` | `[]` | no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [source\_image](#input\_source\_image) | Source OS image to build from | `string` | `null` | no | +| [source\_image\_family](#input\_source\_image\_family) | Alternative to source\_image. Specify image family to build from latest image in family | `string` | `"hpc-rocky-linux-8"` | no | +| [source\_image\_project\_id](#input\_source\_image\_project\_id) | A list of project IDs to search for the source image. Packer will search the
first project ID in the list first, and fall back to the next in the list,
until it finds the source image. | `list(string)` | `null` | no | +| [ssh\_username](#input\_ssh\_username) | Username to use for SSH access to VM | `string` | `"hpc-toolkit-packer"` | no | +| [startup\_script](#input\_startup\_script) | Startup script (as raw string) used to build the custom Linux VM image (overridden by var.startup\_script\_file if both are set) | `string` | `null` | no | +| [startup\_script\_file](#input\_startup\_script\_file) | File path to local shell script that will be used to customize the Linux VM image (overrides var.startup\_script) | `string` | `null` | no | +| [state\_timeout](#input\_state\_timeout) | The time to wait for instance state changes, including image creation | `string` | `"10m"` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of subnetwork in which to provision image building VM | `string` | n/a | yes | +| [tags](#input\_tags) | Assign network tags to apply firewall rules to VM instance | `list(string)` | `null` | no | +| [use\_iap](#input\_use\_iap) | Use IAP proxy when connecting by SSH | `bool` | `true` | no | +| [use\_os\_login](#input\_use\_os\_login) | Use OS Login when connecting by SSH | `bool` | `false` | no | +| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | A list of strings containing PowerShell scripts which will customize a Windows VM image (requires WinRM communicator) | `list(string)` | `[]` | no | +| [wrap\_startup\_script](#input\_wrap\_startup\_script) | Wrap startup script with Packer-generated wrapper | `bool` | `true` | no | +| [zone](#input\_zone) | Cloud zone in which to provision image building VM | `string` | n/a | yes | + +## Outputs + +No outputs. + + +[ansible]: #input_ansible_playbooks +[cloudnat]: https://cloud.google.com/nat/docs/overview +[examples readme]: ../../../examples/README.md#image-builderyaml- +[hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[iamprop]: https://cloud.google.com/iam/docs/access-change-propagation +[iaptunnel]: https://cloud.google.com/iap/docs/using-tcp-forwarding +[image builder]: ../../../examples/image-builder.yaml +[logging-console]: https://console.cloud.google.com/logs/ +[logging-read-docs]: https://cloud.google.com/sdk/gcloud/reference/logging/read +[pkrvars]: https://www.packer.io/guides/hcl/variables#from-a-file +[shell]: #input_shell_scripts +[ssf]: #input_startup_script_file +[sss]: #input_startup_script +[startup-metadata]: https://cloud.google.com/compute/docs/instances/startup-scripts/linux +[startup-script]: ../../../modules/scripts/startup-script +[vpc]: ../../network/vpc/README.md diff --git a/deletion-test/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl b/deletion-test/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl new file mode 100644 index 0000000000..9282cf7433 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl @@ -0,0 +1,216 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "custom-image", ghpc_role = "packer" }) + + # construct a unique image name from the image family + image_family = var.image_family != null ? var.image_family : var.deployment_name + image_name_default = "${local.image_family}-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" + image_name = var.image_name != null ? var.image_name : local.image_name_default + + # construct vm image name for use when getting logs + instance_name = "packer-${substr(uuidv4(), 0, 6)}" + + # default to explicit var.communicator, otherwise in-order: ssh/winrm/none + shell_script_communicator = length(var.shell_scripts) > 0 ? "ssh" : "" + ansible_playbook_communicator = length(var.ansible_playbooks) > 0 ? "ssh" : "" + powershell_script_communicator = length(var.windows_startup_ps1) > 0 ? "winrm" : "" + communicator = coalesce( + var.communicator, + local.shell_script_communicator, + local.ansible_playbook_communicator, + local.powershell_script_communicator, + "none" + ) + + # must not enable IAP when no communicator is in use + use_iap = local.communicator == "none" ? false : var.use_iap + + # construct metadata from startup_script and metadata variables + startup_script_metadata = var.startup_script == null ? {} : { startup-script = var.startup_script } + + linux_user_metadata = { + block-project-ssh-keys = "TRUE" + shutdown-script = <<-EOT + #!/bin/bash + userdel -r ${var.ssh_username} + sed -i '/${var.ssh_username}/d' /var/lib/google/google_users + EOT + } + windows_packer_user = "packer_user" + windows_user_metadata = { + sysprep-specialize-script-cmd = "winrm quickconfig -quiet & net user /add ${local.windows_packer_user} & net localgroup administrators ${local.windows_packer_user} /add & winrm set winrm/config/service/auth @{Basic=\\\"true\\\"}" + windows-shutdown-script-cmd = <<-EOT + net user /delete ${local.windows_packer_user} + EOT + } + user_metadata = local.communicator == "winrm" ? local.windows_user_metadata : local.linux_user_metadata + + # merge metadata such that var.metadata always overrides user management + # metadata but always allow var.startup_script to override var.metadata + metadata = merge( + local.user_metadata, + var.metadata, + local.startup_script_metadata, + ) + + # determine best value for on_host_maintenance if not supplied by user + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + gpu_attached = contains(["a2", "g2"], local.machine_family) || var.accelerator_type != null + on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" + on_host_maintenance = ( + var.on_host_maintenance != null + ? var.on_host_maintenance + : local.on_host_maintenance_default + ) + + accelerator_type = var.accelerator_type == null ? null : "projects/${var.project_id}/zones/${var.zone}/acceleratorTypes/${var.accelerator_type}" + + winrm_username = local.communicator == "winrm" ? "packer_user" : null + winrm_insecure = local.communicator == "winrm" ? true : null + winrm_use_ssl = local.communicator == "winrm" ? true : null + + enable_integrity_monitoring = var.enable_shielded_vm && var.shielded_instance_config.enable_integrity_monitoring + enable_secure_boot = var.enable_shielded_vm && var.shielded_instance_config.enable_secure_boot + enable_vtpm = var.enable_shielded_vm && var.shielded_instance_config.enable_vtpm + + image_licenses = [ + "projects/click-to-deploy-images/global/licenses/hpc-toolkit-vm-image" + ] +} + +source "googlecompute" "toolkit_image" { + communicator = local.communicator + project_id = var.project_id + image_name = local.image_name + image_family = local.image_family + image_labels = local.labels + instance_name = local.instance_name + machine_type = var.machine_type + accelerator_type = local.accelerator_type + accelerator_count = var.accelerator_count + on_host_maintenance = local.on_host_maintenance + disk_size = var.disk_size + disk_type = var.disk_type + omit_external_ip = var.omit_external_ip + use_internal_ip = var.omit_external_ip + subnetwork = var.subnetwork_name + network_project_id = var.network_project_id + service_account_email = var.service_account_email + scopes = var.service_account_scopes + source_image = var.source_image + source_image_family = var.source_image_family + source_image_project_id = var.source_image_project_id + ssh_username = var.ssh_username + tags = var.tags + use_iap = local.use_iap + use_os_login = var.use_os_login + winrm_username = local.winrm_username + winrm_insecure = local.winrm_insecure + winrm_use_ssl = local.winrm_use_ssl + zone = var.zone + labels = local.labels + metadata = local.metadata + startup_script_file = var.startup_script_file + wrap_startup_script = var.wrap_startup_script + state_timeout = var.state_timeout + image_storage_locations = var.image_storage_locations + enable_secure_boot = local.enable_secure_boot + enable_vtpm = local.enable_vtpm + enable_integrity_monitoring = local.enable_integrity_monitoring + image_licenses = local.image_licenses +} + +build { + name = var.deployment_name + sources = ["sources.googlecompute.toolkit_image"] + + # using dynamic blocks to create provisioners ensures that there are no + # provisioner blocks when none are provided and we can use the none + # communicator when using startup-script + + # provisioner "shell" blocks + dynamic "provisioner" { + labels = ["shell"] + for_each = var.shell_scripts + content { + execute_command = "sudo -H sh -c '{{ .Vars }} {{ .Path }}'" + script = provisioner.value + } + } + + # provisioner "powershell" blocks + dynamic "provisioner" { + labels = ["powershell"] + for_each = var.windows_startup_ps1 + content { + inline = split("\n", provisioner.value) + } + } + + dynamic "provisioner" { + labels = ["powershell"] + for_each = length(var.windows_startup_ps1) > 0 ? [1] : [] + content { + inline = [ + "GCESysprep -no_shutdown" + ] + } + } + + # provisioner "ansible-local" blocks + # this installs custom roles/collections from ansible-galaxy in /home/packer + # which will be removed at the end; consider modifying /etc/ansible/ansible.cfg + dynamic "provisioner" { + labels = ["ansible-local"] + for_each = var.ansible_playbooks + content { + playbook_file = provisioner.value.playbook_file + galaxy_file = provisioner.value.galaxy_file + extra_arguments = provisioner.value.extra_arguments + } + } + + post-processor "manifest" { + output = var.manifest_file + strip_path = true + custom_data = { + built-by = "cloud-hpc-toolkit" + } + } + + # If there is an error during image creation, print out command for getting packer VM logs + error-cleanup-provisioner "shell-local" { + environment_vars = [ + "PRJ_ID=${var.project_id}", + "INST_NAME=${local.instance_name}", + "ZONE=${var.zone}", + ] + inline_shebang = "/bin/bash -e" + inline = [ + "type -P gcloud > /dev/null || exit 0", + "INST_ID=$(gcloud compute instances describe $INST_NAME --project $PRJ_ID --format=\"value(id)\" --zone=$ZONE)", + "echo 'Error building image try checking logs:'", + join(" ", ["echo \"gcloud logging --project $PRJ_ID read", + "'logName=(\\\"projects/$PRJ_ID/logs/GCEMetadataScripts\\\" OR \\\"projects/$PRJ_ID/logs/google_metadata_script_runner\\\") AND resource.labels.instance_id=$INST_ID'", + "--format=\\\"table(timestamp, resource.labels.instance_id, jsonPayload.message)\\\"", + "--order=asc\"" + ] + ) + ] + } +} diff --git a/deletion-test/primary/modules/embedded/modules/packer/custom-image/metadata.yaml b/deletion-test/primary/modules/embedded/modules/packer/custom-image/metadata.yaml new file mode 100644 index 0000000000..23108c4e17 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/packer/custom-image/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - logging.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl b/deletion-test/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl new file mode 100644 index 0000000000..3cede102ce --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl @@ -0,0 +1,276 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "deployment_name" { + description = "Cluster Toolkit deployment name" + type = string +} + +variable "project_id" { + description = "Project in which to create VM and image" + type = string +} + +variable "machine_type" { + description = "VM machine type on which to build new image" + type = string + default = "n2-standard-4" +} + +variable "disk_size" { + description = "Size of disk image in GB" + type = number + default = null +} + +variable "disk_type" { + description = "Type of persistent disk to provision" + type = string + default = "pd-balanced" +} + +variable "zone" { + description = "Cloud zone in which to provision image building VM" + type = string +} + +variable "network_project_id" { + description = "Project ID of Shared VPC network" + type = string + default = null +} + +variable "subnetwork_name" { + description = "Name of subnetwork in which to provision image building VM" + type = string +} + +variable "omit_external_ip" { + description = "Provision the image building VM without a public IP address" + type = bool + default = true +} + +variable "tags" { + description = "Assign network tags to apply firewall rules to VM instance" + type = list(string) + default = null +} + +variable "image_family" { + description = "The family name of the image to be built. Defaults to `deployment_name`" + type = string + default = null +} + +variable "image_name" { + description = "The name of the image to be built. If not supplied, it will be set to image_family-$ISO_TIMESTAMP" + type = string + default = null +} + +variable "source_image_project_id" { + description = < +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 4.0 | +| [local](#requirement\_local) | >= 2.0.0 | +| [null](#requirement\_null) | ~> 3.0 | +| [random](#requirement\_random) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.0 | +| [local](#provider\_local) | >= 2.0.0 | +| [null](#provider\_null) | ~> 3.0 | +| [random](#provider\_random) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_template](#module\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.submit_script](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [null_resource.submit_job](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.submit_job_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used for the job\_id | `string` | n/a | yes | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs | `bool` | `true` | no | +| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions. Valid inputs are `"alpha"`, `"beta"` and "" (empty string for default version) | `string` | `""` | no | +| [image](#input\_image) | DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided. | `any` | `null` | no | +| [instance\_image](#input\_instance\_image) | Google Cloud Batch compute node image. Ignored if `instance_template` is provided.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [instance\_template](#input\_instance\_template) | Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions. | `string` | `null` | no | +| [job\_filename](#input\_job\_filename) | The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified | `string` | `null` | no | +| [job\_id](#input\_job\_id) | An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs. | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided. | `map(string)` | n/a | yes | +| [log\_policy](#input\_log\_policy) | Create a block to define log policy.
When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging.
When set to `PATH`, path must be added to generated template.
When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. | `string` | `"CLOUD_LOGGING"` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided. | `string` | `"n2-standard-4"` | no | +| [mpi\_mode](#input\_mpi\_mode) | Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`. | `bool` | `false` | no | +| [native\_batch\_mounting](#input\_native\_batch\_mounting) | Batch can mount some fs\_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts. | `bool` | `true` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which to run the Google Cloud Batch job | `string` | n/a | yes | +| [runnable](#input\_runnable) | A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`. | `string` | `null` | no | +| [runnables](#input\_runnables) | A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template. | 
list(object({
    script = string
  }))
| `null` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/devstorage.read_only",
    "https://www.googleapis.com/auth/logging.write",
    "https://www.googleapis.com/auth/monitoring.write",
    "https://www.googleapis.com/auth/servicecontrol",
    "https://www.googleapis.com/auth/service.management.readonly",
    "https://www.googleapis.com/auth/trace.append"
  ]
}
| no | +| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided. | `string` | `null` | no | +| [submit](#input\_submit) | When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply. | `bool` | `false` | no | +| [subnetwork](#input\_subnetwork) | The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided. | `any` | `null` | no | +| [task\_count](#input\_task\_count) | Number of parallel tasks | `number` | `1` | no | +| [task\_count\_per\_node](#input\_task\_count\_per\_node) | Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value. | `number` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcloud\_version](#output\_gcloud\_version) | The version of gcloud to be used. | +| [instance\_template](#output\_instance\_template) | Instance template used by the Batch job. | +| [instructions](#output\_instructions) | Instructions for submitting the Batch job. | +| [job\_data](#output\_job\_data) | All data associated with the defined job, typically provided as input to clout-batch-login-node. | +| [network\_storage](#output\_network\_storage) | An array of network attached storage mounts used by the Batch job. | +| [startup\_script](#output\_startup\_script) | Startup script run before Google Cloud Batch job starts. | + diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf new file mode 100644 index 0000000000..0d681536c9 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf @@ -0,0 +1,149 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "batch-job-template", ghpc_role = "scheduler" }) +} + +locals { + instance_template = coalesce(var.instance_template, module.instance_template.self_link) + + tasks_per_node = var.task_count_per_node != null ? var.task_count_per_node : (var.mpi_mode ? 1 : null) + + one_line_runnable = coalesce(var.runnable, "## Add your workload here ##") + runnables = coalesce(var.runnables, [{ script = local.one_line_runnable }]) + + job_template_contents = templatefile( + "${path.module}/templates/batch-job-base.yaml.tftpl", + { + synchronized = var.mpi_mode + runnables = local.runnables + task_count = var.task_count + tasks_per_node = local.tasks_per_node + require_hosts_file = var.mpi_mode + permissive_ssh = var.mpi_mode + log_policy = var.log_policy + instance_template = local.instance_template + nfs_volumes = local.native_batch_network_storage + labels = local.labels + } + ) + + submit_job_id = "${var.job_id}-${random_id.submit_job_suffix.hex}" + job_filename = coalesce(var.job_filename, "${var.job_id}.yaml") + job_template_output_path = "${path.root}/${local.job_filename}" + + submit_script_contents = templatefile( + "${path.module}/templates/batch-submit.sh.tftpl", + { + project = var.project_id + location = var.region + config = local_file.job_template.filename + submit_job_id = local.submit_job_id + } + ) + submit_script_output_path = "${path.root}/submit-${var.job_id}.sh" + + subnetwork_name = var.subnetwork != null ? var.subnetwork.name : "default" + subnetwork_project = var.subnetwork != null ? var.subnetwork.project : var.project_id + + # Filter network_storage for native Batch support + native_fstype = var.native_batch_mounting ? ["nfs"] : [] + native_batch_network_storage = [ + for ns in var.network_storage : + ns if contains(local.native_fstype, ns.fs_type) + ] + # other processing happens in startup_from_network_storage.tf + + # this code is similar to code in Packer and vm-instance modules + # it differs in that this module does not (yet) expose var.guest_acclerator + # for attaching GPUs to N1 VMs. For now, identify only A2 types. + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + gpu_attached = contains(["a2", "g2"], local.machine_family) + on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" + + on_host_maintenance = coalesce(var.on_host_maintenance, local.on_host_maintenance_default) + + network_storage_metadata = var.network_storage != null ? ({ network_storage = jsonencode(var.network_storage) }) : {} + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.network_storage_metadata, + local.disable_automatic_updates_metadata + ) +} + +module "instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = var.instance_template == null ? "${var.job_id}-instance-template" : "unused-template" + project_id = var.project_id + subnetwork = local.subnetwork_name + subnetwork_project = local.subnetwork_project + service_account = var.service_account + access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + labels = local.labels + + machine_type = var.machine_type + startup_script = local.startup_from_network_storage + metadata = local.metadata + source_image_family = data.google_compute_image.compute_image.family + source_image = data.google_compute_image.compute_image.name + source_image_project = data.google_compute_image.compute_image.project + on_host_maintenance = local.on_host_maintenance +} + +resource "local_file" "job_template" { + content = local.job_template_contents + filename = local.job_template_output_path + + lifecycle { + precondition { + condition = var.runnable == null || var.runnables == null + error_message = "var.runnable and var.runnables (plural) cannot both be set." + } + } +} + +resource "random_id" "submit_job_suffix" { + byte_length = 4 + keepers = { + always_run = timestamp() + } +} + +resource "local_file" "submit_script" { + content = local.submit_script_contents + filename = local.submit_script_output_path +} + +resource "null_resource" "submit_job" { + depends_on = [local_file.job_template, local_file.submit_script] + count = var.submit ? 1 : 0 + + # A new deployment should always submit a new job. Old finished jobs aren't persistent parts of + # Cloud infrastructure. + triggers = { + always_run = timestamp() + } + + provisioner "local-exec" { + command = local.submit_script_output_path + } +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml new file mode 100644 index 0000000000..387e810962 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - batch.googleapis.com + - compute.googleapis.com +ghpc: + inject_module_id: job_id diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf new file mode 100644 index 0000000000..0b1295975a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + provided_instance_tpl_msg = "The Batch job template uses the existing VM instance template:" + generated_instance_tpl_msg = "The Batch job template uses a new VM instance template created matching the provided settings:" + submit_msg = <<-EOT + + The job has been submitted. See job status at: + https://console.cloud.google.com/batch/jobsDetail/regions/${var.region}/jobs/${local.submit_job_id}?project=${var.project_id} + EOT +} + +output "instructions" { + description = "Instructions for submitting the Batch job." + value = <<-EOT + + A Batch job template file has been created locally at: + ${abspath(local.job_template_output_path)} + + ${var.instance_template == null ? local.generated_instance_tpl_msg : local.provided_instance_tpl_msg} + ${local.instance_template} + ${var.submit ? local.submit_msg : ""} + + Use the following commands to: + Submit your job${var.submit ? " (Note: job has already been submitted)" : ""}: + gcloud ${var.gcloud_version} batch jobs submit ${local.submit_job_id} --config=${abspath(local.job_template_output_path)} --location=${var.region} --project=${var.project_id} + + Check status: + gcloud ${var.gcloud_version} batch jobs describe ${local.submit_job_id} --location=${var.region} --project=${var.project_id} | grep state: + + Delete job: + gcloud ${var.gcloud_version} batch jobs delete ${local.submit_job_id} --location=${var.region} --project=${var.project_id} + + List all jobs: + gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} + EOT +} + +output "job_data" { + description = "All data associated with the defined job, typically provided as input to clout-batch-login-node." + value = { + template_contents = local.job_template_contents, + filename = local.job_filename, + id = local.submit_job_id + } +} + +output "instance_template" { + description = "Instance template used by the Batch job." + value = local.instance_template +} + +output "network_storage" { + description = "An array of network attached storage mounts used by the Batch job." + value = var.network_storage +} + +output "startup_script" { + description = "Startup script run before Google Cloud Batch job starts." + value = var.startup_script +} + +output "gcloud_version" { + description = "The version of gcloud to be used." + value = var.gcloud_version +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf new file mode 100644 index 0000000000..02bc58e4f7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "inputs": +# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. +# var.network_storage : to be passed into metadata somewhere else (not here) +# var.startup_script : to be changed into a more complete file system with all the fs runners + +# "outputs": +# local.startup_from_network_storage : A full startup script with all the runners that are not supported +# natively and were included in the network_storage structure + +locals { + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + # Pull out runners to include in startup script + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + startup_script_runner = [{ + content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" + destination = "passed_startup_script.sh" + type = "shell" + }] + + full_runner_list = concat( + local.storage_client_install_runners, + local.mount_runners, + local.startup_script_runner + ) + + startup_from_network_storage = module.netstorage_startup_script.startup_script +} + +module "netstorage_startup_script" { + source = "../../scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.full_runner_list +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl new file mode 100644 index 0000000000..83fccde53b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl @@ -0,0 +1,53 @@ +taskGroups: + - taskSpec: + runnables: + %{~ if synchronized ~} + - barrier: + name: "wait-for-node-startup" + %{~ endif ~} + %{~ for runnable in runnables ~} + - script: + text: ${indent(12, chomp(yamlencode(runnable.script)))} + %{~ if synchronized ~} + - barrier: + name: "wait-for-script-to-complete" + %{~ endif ~} + %{~ endfor ~} + %{~ if length(nfs_volumes) > 0 ~} + volumes: + %{~ for index, vol in nfs_volumes ~} + - nfs: + server: "${vol.server_ip}" + remotePath: "${vol.remote_mount}" + %{~ if vol.mount_options != "" && vol.mount_options != null ~} + mountOptions: "${vol.mount_options}" + %{~ endif ~} + mountPath: "${vol.local_mount}" + %{~ endfor ~} + %{~ endif ~} + taskCount: ${task_count} + %{~ if tasks_per_node != null ~} + taskCountPerNode: ${tasks_per_node} + %{~ endif ~} + requireHostsFile: ${require_hosts_file} + permissiveSsh: ${permissive_ssh} +%{~ if instance_template != null } +allocationPolicy: + instances: + - instanceTemplate: "${instance_template}" +%{~ endif } +%{~ if log_policy == "CLOUD_LOGGING" } +logsPolicy: + destination: "CLOUD_LOGGING" +%{ endif } +%{~ if log_policy == "PATH" } +logsPolicy: + destination: "PATH" + logsPath: ## Add logging path here +%{ endif } +%{~ if length(labels) > 0 ~} +labels: +%{ for k, v in labels ~} + ${k}: "${v}" +%{ endfor } +%{~ endif ~} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl new file mode 100644 index 0000000000..25f89c3ceb --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl @@ -0,0 +1,10 @@ +#!/bin/bash +set -e -o pipefail +GCLOUD_MAJOR_VERSION=$(gcloud --version | head -n 1 | awk '{print $NF}' | cut -f1 --delimiter=.) +if [ $((GCLOUD_MAJOR_VERSION >= 461)) ]; then + gcloud batch jobs submit ${submit_job_id} --project=${project} --location=${location} --config=${config} + echo "batch job ${submit_job_id} successfully submitted" +else + echo "gcloud must be updated to version 461.0.0 or later." + exit 1 +fi diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf new file mode 100644 index 0000000000..f65fbd111e --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf @@ -0,0 +1,240 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "region" { + description = "The region in which to run the Google Cloud Batch job" + type = string +} + +variable "deployment_name" { + description = "Name of the deployment, used for the job_id" + type = string +} + +variable "labels" { + description = "Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided." + type = map(string) +} + +variable "job_id" { + description = "An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs." + type = string +} + +variable "job_filename" { + description = "The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified" + type = string + default = null +} + +variable "gcloud_version" { + description = "The version of the gcloud cli being used. Used for output instructions. Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default version)" + type = string + default = "" + + validation { + condition = contains(["alpha", "beta", ""], var.gcloud_version) + error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." + } +} + +variable "task_count" { + description = "Number of parallel tasks" + type = number + default = 1 +} + +variable "task_count_per_node" { + description = "Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value." + type = number + default = null +} + +variable "mpi_mode" { + description = "Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`." + type = bool + default = false +} + +variable "log_policy" { + description = <<-EOT + Create a block to define log policy. + When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging. + When set to `PATH`, path must be added to generated template. + When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. + EOT + type = string + default = "CLOUD_LOGGING" + + validation { + condition = contains(["CLOUD_LOGGING", "PATH", "DESTINATION_UNSPECIFIED"], var.log_policy) + error_message = "Allowed values for log_policy are 'CLOUD_LOGGING', 'PATH', or 'DESTINATION_UNSPECIFIED'." + } +} + +variable "runnables" { + description = "A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template." + type = list(object({ + script = string + })) + default = null +} + +variable "runnable" { + description = "A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`." + type = string + default = null +} + +variable "instance_template" { + description = "Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions." + type = string + default = null +} + +variable "subnetwork" { + description = "The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided." + type = any + default = null +} + +variable "enable_public_ips" { + description = "If set to true, instances will have public IPs" + type = bool + default = true +} + +variable "service_account" { + description = "Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided." + type = object({ + email = string, + scopes = set(string) + }) + default = { + email = null + scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/trace.append" + ] + } +} + +variable "machine_type" { + description = "Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided." + type = string + default = "n2-standard-4" +} + +variable "startup_script" { + description = "Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided." + type = string + default = null +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided." + type = list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "native_batch_mounting" { + description = "Batch can mount some fs_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts." + type = bool + default = true +} + +# Deprecated, replaced by instance_image +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided." + type = any + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + Google Cloud Batch compute node image. Ignored if `instance_template` is provided. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE`" + type = string + default = null + validation { + condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) + error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." + } +} + +variable "submit" { + description = "When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply." + type = bool + default = false +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf new file mode 100644 index 0000000000..a1161e1354 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + google = { + source = "hashicorp/google" + version = ">= 4.0" + } + } + required_version = ">= 1.1" +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/README.md b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/README.md new file mode 100644 index 0000000000..c20ca7dbeb --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/README.md @@ -0,0 +1,127 @@ +# Description + +This module creates a VM that acts as a login node to test and submit Google +Cloud Batch jobs. It is intended to be used along with the `batch-job-template` +module. + +This login node: + +- Uses the same VM settings as the first provided `batch-job-template`, such as + image, machine type, etc... +- Runs the same `startup-script` as the first provided `batch-job-template`. +- Has the same mounted file systems as the provided `batch-job-template`. +- Contains a folder with job templates generated by `batch-job-template` modules. + +Since the login node has the same mounted storage and is a homogeneous machine +to the Google Cloud Batch compute VMs, it can be used to inspect shared file +systems and test installed software before submitting a Google Cloud Batch job. + +## Example + +```yaml +- id: batch-job + source: modules/scheduler/batch-job-template + ... + +- id: batch-login + source: modules/scheduler/batch-login-node + use: [batch-job] + outputs: [instructions] +``` + +## Authentication + +To submit jobs from the login node, the service account attached to the VM needs +the `Batch Job Administrator` role. In most cases this service account will be +the Compute Engine default service account and will not be granted this role by +default. + +You can grant this role either by adding the `Batch Job Administrator` role to +the service account in the IAM page in the Google Cloud Console, or by running +the following command line: + +```bash +gcloud projects add-iam-policy-binding \ + --member=serviceAccount: \ + --role=roles/batch.jobsAdmin +``` + +## gcloud Batch Access + +Until the Google Cloud Batch API is generally available (GA), it may not be +available in all versions of the `gcloud` cli. You can test if the Google Cloud +Batch commands are available by running `gcloud [alpha|beta|] batch -h`. If the +Google Cloud Batch cli is not available it can generally be mitigated by either +updating `gcloud` by running `gcloud components update`, or using an image that +contains a more recent version of `gcloud`. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [login\_startup\_script](#module\_login\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_from_template.batch_login](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | +| [google_compute_instance_template.batch_instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [batch\_job\_directory](#input\_batch\_job\_directory) | The path of the directory on the login node in which to place the Google Cloud Batch job template | `string` | `"/home/batch-jobs"` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, also used for the job\_id | `string` | n/a | yes | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions.
Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default
version). Typically supplied by a batch-job-template module. If multiple
batch-job-template modules supply the gcloud\_version, only the first will be used. | `string` | `""` | no | +| [instance\_template](#input\_instance\_template) | Login VM instance template self-link. Typically supplied by a
batch-job-template module. If multiple batch-job-template modules supply the
instance\_template, the first will be used. | `string` | n/a | yes | +| [job\_data](#input\_job\_data) | List of jobs and supporting data for each, typically provided via "use" from the batch-job-template module. | 
list(object({
    template_contents = string,
    filename          = string,
    id                = string
  }))
| n/a | yes | +| [job\_filename](#input\_job\_filename) | Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [job\_id](#input\_job\_id) | Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions. | `string` | `null` | no | +| [job\_template\_contents](#input\_job\_template\_contents) | Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the login node. Key-value pairs | `map(string)` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which to create the login node | `string` | n/a | yes | +| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [zone](#input\_zone) | The zone in which to create the login node | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for accessing the login node and submitting Google Cloud Batch jobs | +| [login\_node\_name](#output\_login\_node\_name) | Name of the created VM | + diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf new file mode 100644 index 0000000000..6f539af122 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf @@ -0,0 +1,127 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "batch-login-node", ghpc_role = "scheduler" }) +} + +data "google_compute_instance_template" "batch_instance_template" { + name = var.instance_template +} + +locals { + job_template_runners = [for job in var.job_data : { + content = job.template_contents + destination = "${var.batch_job_directory}/${job.filename}" + type = "data" + }] + + instance_template_metadata = data.google_compute_instance_template.batch_instance_template.metadata + startup_metadata = { startup-script = module.login_startup_script.startup_script } + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + login_metadata = merge(local.instance_template_metadata, local.startup_metadata, local.oslogin_metadata) + + batch_command_instructions = join("\n", [for job in var.job_data : <<-EOT + ## For job: ${job.id} ## + + Submit your job from login node: + gcloud ${var.gcloud_version} batch jobs submit ${job.id} --config=${var.batch_job_directory}/${job.filename} --location=${var.region} --project=${var.project_id} + + Check status: + gcloud ${var.gcloud_version} batch jobs describe ${job.id} --location=${var.region} --project=${var.project_id} | grep state: + + Delete job: + gcloud ${var.gcloud_version} batch jobs delete ${job.id} --location=${var.region} --project=${var.project_id} + + EOT + ]) + + list_all_jobs = <<-EOT + List all jobs: + gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} + EOT + + readme_contents = <<-EOT + # Batch Job Templates + + This folder contains Batch job templates created by the Cluster Toolkit. + These templates can be edited before submitting to Batch to capture more + complex workloads. + + Use the following commands to: + ${local.list_all_jobs} + + ${local.batch_command_instructions} + EOT + + # Construct startup script for network storage + storage_client_install_runners = [ + for i, ns in var.network_storage : merge(ns.client_install_runner, { + destination = "${i}-${ns.client_install_runner.destination}" + }) if ns.client_install_runner != null + ] + mount_runners = [ + for i, ns in var.network_storage : merge(ns.mount_runner, { + destination = "${i}-${ns.mount_runner.destination}" + }) if ns.mount_runner != null + ] + + startup_script_runner = { + content = var.startup_script != null ? var.startup_script : "echo 'Batch job template had no startup script'" + destination = "passed_startup_script.sh" + type = "shell" + } +} + +module "login_startup_script" { + source = "../../scripts/startup-script" + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = concat( + local.storage_client_install_runners, + local.mount_runners, + [local.startup_script_runner], + local.job_template_runners, + [ + { + content = local.readme_contents + destination = "${var.batch_job_directory}/README.md" + type = "data" + } + ] + ) +} + +resource "google_compute_instance_from_template" "batch_login" { + name = "${var.deployment_name}-batch-login" + source_instance_template = var.instance_template + project = var.project_id + zone = var.zone + metadata = local.login_metadata + + service_account { + scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml new file mode 100644 index 0000000000..9af2319b4a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - batch.googleapis.com + - compute.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf new file mode 100644 index 0000000000..ea8eccf8d5 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "login_node_name" { + description = "Name of the created VM" + value = google_compute_instance_from_template.batch_login.name +} + +output "instructions" { + description = "Instructions for accessing the login node and submitting Google Cloud Batch jobs" + value = <<-EOT + + Batch job template files will be placed on the Batch login node in the following directory: + ${var.batch_job_directory} + + Use the following commands to: + SSH into the login node: + gcloud compute ssh --zone ${google_compute_instance_from_template.batch_login.zone} ${google_compute_instance_from_template.batch_login.name} --project ${google_compute_instance_from_template.batch_login.project} + + ${local.list_all_jobs} + + ${local.batch_command_instructions} + EOT +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf new file mode 100644 index 0000000000..3b9caa7001 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf @@ -0,0 +1,151 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "deployment_name" { + description = "Name of the deployment, also used for the job_id" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "region" { + description = "The region in which to create the login node" + type = string +} + +variable "zone" { + description = "The zone in which to create the login node" + type = string +} + +variable "labels" { + description = "Labels to add to the login node. Key-value pairs" + type = map(string) +} + +variable "instance_template" { + description = <<-EOT + Login VM instance template self-link. Typically supplied by a + batch-job-template module. If multiple batch-job-template modules supply the + instance_template, the first will be used. + EOT + type = string +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module." + type = list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "startup_script" { + description = "Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module." + type = string + default = null +} + +variable "job_data" { + description = "List of jobs and supporting data for each, typically provided via \"use\" from the batch-job-template module." + type = list(object({ + template_contents = string, + filename = string, + id = string + })) + validation { + condition = length(distinct([for job in var.job_data : job.filename])) == length(var.job_data) + error_message = "All filenames in var.job_data must be unique." + } + validation { + condition = length(distinct([for job in var.job_data : job.id])) == length(var.job_data) + error_message = "All job IDs in var.job_data must be unique." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_template_contents" { + description = "Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module." + type = string + default = null + validation { + condition = var.job_template_contents == null + error_message = "job_template_contents is deprecated. Please use `job_data` instead." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_filename" { + description = "Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module." + type = string + default = null + validation { + condition = var.job_filename == null + error_message = "job_filename is deprecated. Please use `job_data` instead." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_id" { + description = "Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions." + type = string + default = null + validation { + condition = var.job_id == null + error_message = "job_id is deprecated. Please use `job_data` instead." + } +} + +variable "gcloud_version" { + description = <<-EOT + The version of the gcloud cli being used. Used for output instructions. + Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default + version). Typically supplied by a batch-job-template module. If multiple + batch-job-template modules supply the gcloud_version, only the first will be used. + EOT + type = string + default = "" + + validation { + condition = contains(["alpha", "beta", ""], var.gcloud_version) + error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." + } +} + +variable "batch_job_directory" { + description = "The path of the directory on the login node in which to place the Google Cloud Batch job template" + type = string + default = "/home/batch-jobs" +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf new file mode 100644 index 0000000000..15337a1d7b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.74.0" + } + + required_version = ">= 0.14.0" +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/README.md b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/README.md new file mode 100644 index 0000000000..dd4f7fdaa7 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/README.md @@ -0,0 +1,220 @@ +## Description + +This module creates a Google Kubernetes Engine +([GKE](https://cloud.google.com/kubernetes-engine)) cluster. + +### Example + +The following example creates a GKE cluster and a VPC designed to work with GKE. +See [VPC Network](#vpc-network) section for more information about network +requirements. + +```yaml + - id: network1 + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### VPC Network + +This module is configured to create a +[VPC-native cluster](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips). +This means that alias IPs are used and that the subnetwork requires secondary +ranges for pods and services. In the example shown above these secondary ranges +are created in the VPC module. By default the `gke-cluster` module will look for +ranges with the names `pods` and `services`. These names can be configured using +the `pods_ip_range_name` and `services_ip_range_name` settings. + +### Multi-networking + +To [enable Multi-networking](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment), pass multivpc module to gke-cluster module as described in example below. Passing a multivpc module enables multi networking and [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en) on the cluster. + +```yaml + - id: network + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: gke-cluster + source: modules/scheduler/gke-cluster + use: [network, multinetwork] ## enables multi networking and Dataplane V2 on cluster + settings: + cluster_name: $(vars.deployment_name) +``` + +Find an example of multi networking in GKE [here](../../../examples/gke-a3-megagpu.yaml). + +### Cluster Limitations + +The current implementations has the following limitations: + +- Autopilot is disabled +- Auto-provisioning of new node pools is disabled +- Network policies are not supported +- General addon configuration is not supported +- Only regional cluster is supported + +### GKE Inference Gateway + +Setting `enable_inference_gateway` to `true` will enable the `HttpLoadBalancing` +addon and deploy the Inference Gateway CRDs. This feature requires a subnet with +`purpose` set to `REGIONAL_MANAGED_PROXY` in the VPC. For more information, see +the [GKE Inference Gateway documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/serve-with-gke-inference-gateway). + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 7.2 | +| [google-beta](#requirement\_google-beta) | >= 7.2 | +| [kubernetes](#requirement\_kubernetes) | >= 2.36 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 7.2 | +| [google-beta](#provider\_google-beta) | >= 7.2 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | +| [workload\_identity](#module\_workload\_identity) | terraform-google-modules/kubernetes-engine/google//modules/workload-identity | >= 40.0 | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_cluster) | resource | +| [google-beta_google_container_node_pool.system_node_pools](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | +| [google-beta_google_container_engine_versions.version_prefix_filter](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/data-sources/google_container_engine_versions) | data source | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks enables multi networking and creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [authenticator\_security\_group](#input\_authenticator\_security\_group) | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| [autoscaling\_profile](#input\_autoscaling\_profile) | (Beta) Optimize for utilization or availability when deciding to remove nodes. Can be BALANCED or OPTIMIZE\_UTILIZATION. | `string` | `"OPTIMIZE_UTILIZATION"` | no | +| [cloud\_dns\_config](#input\_cloud\_dns\_config) | Configuration for Using Cloud DNS for GKE.

additive\_vpc\_scope\_dns\_domain: This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = "CLOUD\_DNS" and cluster\_dns\_scope = "CLUSTER\_SCOPE" must both be set as well.
cluster\_dns: Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS.
cluster\_dns\_scope: The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE.
cluster\_dns\_domain: The suffix used for all cluster service records. | 
object({
    additive_vpc_scope_dns_domain = optional(string)
    cluster_dns                   = optional(string, "PROVIDER_UNSPECIFIED")
    cluster_dns_scope             = optional(string, "DNS_SCOPE_UNSPECIFIED")
    cluster_dns_domain            = optional(string)
  })
| 
{
  "additive_vpc_scope_dns_domain": null,
  "cluster_dns": "PROVIDER_UNSPECIFIED",
  "cluster_dns_domain": null,
  "cluster_dns_scope": "DNS_SCOPE_UNSPECIFIED"
}
| no | +| [cluster\_availability\_type](#input\_cluster\_availability\_type) | Type of cluster availability. Possible values are: {REGIONAL, ZONAL} | `string` | `"REGIONAL"` | no | +| [cluster\_reference\_type](#input\_cluster\_reference\_type) | How the google\_container\_node\_pool.system\_node\_pools refers to the cluster. Possible values are: {SELF\_LINK, NAME} | `string` | `"SELF_LINK"` | no | +| [configure\_workload\_identity\_sa](#input\_configure\_workload\_identity\_sa) | When true, a kubernetes service account will be created and bound using workload identity to the service account used to create the cluster. | `bool` | `false` | no | +| [default\_max\_pods\_per\_node](#input\_default\_max\_pods\_per\_node) | The default maximum number of pods per node in this cluster. | `number` | `null` | no | +| [deletion\_protection](#input\_deletion\_protection) | "Determines if the cluster can be deleted by gcluster commands or not".
To delete a cluster provisioned with deletion\_protection set to true, you must first set it to false and apply the changes.
Then proceed with deletion as usual. | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`. | `string` | n/a | yes | +| [enable\_dataplane\_v2](#input\_enable\_dataplane\_v2) | Enables [Dataplane v2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2). This setting is immutable on clusters. If null, will default to false unless using multi-networking, in which case it will default to true | `bool` | `null` | no | +| [enable\_dcgm\_monitoring](#input\_enable\_dcgm\_monitoring) | Enable GKE to collect DCGM metrics | `bool` | `false` | no | +| [enable\_external\_dns\_endpoint](#input\_enable\_external\_dns\_endpoint) | Allow [DNS-based approach](https://cloud.google.com/kubernetes-engine/docs/concepts/network-isolation#dns-based_endpoint) for accessing the GKE control plane.
Refer this [dedicated blog](https://cloud.google.com/blog/products/containers-kubernetes/new-dns-based-endpoint-for-the-gke-control-plane) for more details. | `bool` | `false` | no | +| [enable\_filestore\_csi](#input\_enable\_filestore\_csi) | The status of the Filestore Container Storage Interface (CSI) driver addon, which allows the usage of filestore instance as volumes. | `bool` | `false` | no | +| [enable\_gcsfuse\_csi](#input\_enable\_gcsfuse\_csi) | The status of the GCSFuse Container Storage Interface (CSI) driver addon, which allows the usage of a GCS bucket as volumes. | `bool` | `false` | no | +| [enable\_inference\_gateway](#input\_enable\_inference\_gateway) | If true, enables GKE features required for Inference Gateway, including the HttpLoadBalancing addon, and installs required CRDs. | `bool` | `false` | no | +| [enable\_k8s\_beta\_apis](#input\_enable\_k8s\_beta\_apis) | List of Enabled Kubernetes Beta APIs. | `list(string)` | `null` | no | +| [enable\_managed\_lustre\_csi](#input\_enable\_managed\_lustre\_csi) | The status of the Google Compute Engine Managed Lustre Container Storage Interface (CSI) driver addon, which allows the usage of a lustre as volumes. | `bool` | `false` | no | +| [enable\_master\_global\_access](#input\_enable\_master\_global\_access) | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `false` | no | +| [enable\_multi\_networking](#input\_enable\_multi\_networking) | Enables [multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) (Requires GKE Enterprise). This setting is immutable on clusters and enables [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en). If null, will determine state based on if additional\_networks are passed in. | `bool` | `null` | no | +| [enable\_node\_local\_dns\_cache](#input\_enable\_node\_local\_dns\_cache) | Enable GKE NodeLocal DNSCache addon to improve DNS lookup latency | `bool` | `false` | no | +| [enable\_parallelstore\_csi](#input\_enable\_parallelstore\_csi) | The status of the Google Compute Engine Parallelstore Container Storage Interface (CSI) driver addon, which allows the usage of a parallelstore as volumes. | `bool` | `false` | no | +| [enable\_persistent\_disk\_csi](#input\_enable\_persistent\_disk\_csi) | The status of the Google Compute Engine Persistent Disk Container Storage Interface (CSI) driver addon, which allows the usage of a PD as volumes. | `bool` | `true` | no | +| [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | (Beta) Whether the master's internal IP address is used as the cluster endpoint. | `bool` | `true` | no | +| [enable\_private\_ipv6\_google\_access](#input\_enable\_private\_ipv6\_google\_access) | The private IPv6 google access type for the VMs in this subnet. | `bool` | `true` | no | +| [enable\_private\_nodes](#input\_enable\_private\_nodes) | (Beta) Whether nodes have internal IP addresses only. | `bool` | `true` | no | +| [enable\_ray\_operator](#input\_enable\_ray\_operator) | The status of the Ray operator addon, This feature enables Kubernetes APIs for managing and scaling Ray clusters and jobs. You control and are responsible for managing ray.io custom resources in your cluster. This feature is not compatible with GKE clusters that already have another Ray operator installed. Supports clusters on Kubernetes version 1.29.8-gke.1054000 or later. | `bool` | `false` | no | +| [gcp\_public\_cidrs\_access\_enabled](#input\_gcp\_public\_cidrs\_access\_enabled) | Whether the cluster master is accessible via all the Google Compute Engine Public IPs. To view this list of IP addresses look here https://cloud.google.com/compute/docs/faq#find_ip_range | `bool` | `false` | no | +| [k8s\_network\_names](#input\_k8s\_network\_names) | Kubernetes network names details for GKE. If starting index is not specified for gvnic or rdma, it would be set to the default values. | 
object({
    gvnic_prefix      = optional(string, "")
    gvnic_start_index = optional(number, 1)
    gvnic_postfix     = optional(string, "")
    rdma_prefix       = optional(string, "")
    rdma_start_index  = optional(number, 0)
    rdma_postfix      = optional(string, "")
  })
| 
{
  "gvnic_postfix": "",
  "gvnic_prefix": "gvnic-",
  "gvnic_start_index": 1,
  "rdma_postfix": "",
  "rdma_prefix": "rdma-",
  "rdma_start_index": 0
}
| no | +| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account name to use with the gke cluster | `string` | `"workload-identity-k8s-sa"` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [maintenance\_exclusions](#input\_maintenance\_exclusions) | List of maintenance exclusions. A cluster can have up to three. | 
list(object({
    name            = string
    start_time      = string
    end_time        = string
    exclusion_scope = string
  }))
| `[]` | no | +| [maintenance\_start\_time](#input\_maintenance\_start\_time) | Start time for daily maintenance operations. Specified in GMT with `HH:MM` format. | `string` | `"09:00"` | no | +| [master\_authorized\_networks](#input\_master\_authorized\_networks) | External network that can access Kubernetes master through HTTPS. Must be specified in CIDR notation. | 
list(object({
    cidr_block   = string
    display_name = string
  }))
| `[]` | no | +| [master\_ipv4\_cidr\_block](#input\_master\_ipv4\_cidr\_block) | (Beta) The IP range in CIDR notation to use for the hosted master network. | `string` | `"172.16.0.32/28"` | no | +| [min\_master\_version](#input\_min\_master\_version) | The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release. | `string` | `null` | no | +| [name\_suffix](#input\_name\_suffix) | Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`. | `string` | `""` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`. | `string` | n/a | yes | +| [networking\_mode](#input\_networking\_mode) | Determines whether alias IPs or routes will be used for pod IPs in the cluster. Options are VPC\_NATIVE or ROUTES. VPC\_NATIVE enables IP aliasing. The default is VPC\_NATIVE. | `string` | `"VPC_NATIVE"` | no | +| [pods\_ip\_range\_name](#input\_pods\_ip\_range\_name) | The name of the secondary subnet ip range to use for pods. | `string` | `"pods"` | no | +| [prefix\_with\_deployment\_name](#input\_prefix\_with\_deployment\_name) | If true, cluster name will be prefixed by `deployment_name` (ex: -). | `bool` | `true` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [region](#input\_region) | The region to host the cluster in. | `string` | n/a | yes | +| [release\_channel](#input\_release\_channel) | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. | `string` | `"UNSPECIFIED"` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the system node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the system node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [services\_ip\_range\_name](#input\_services\_ip\_range\_name) | The name of the secondary subnet range to use for services. | `string` | `"services"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to host the cluster in. | `string` | n/a | yes | +| [system\_node\_pool\_disk\_size\_gb](#input\_system\_node\_pool\_disk\_size\_gb) | Size of disk for each node of the system node pool. | `number` | `100` | no | +| [system\_node\_pool\_disk\_type](#input\_system\_node\_pool\_disk\_type) | Disk type for each node of the system node pool. | `string` | `null` | no | +| [system\_node\_pool\_enable\_secure\_boot](#input\_system\_node\_pool\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | +| [system\_node\_pool\_enabled](#input\_system\_node\_pool\_enabled) | Create a system node pool. | `bool` | `true` | no | +| [system\_node\_pool\_image\_type](#input\_system\_node\_pool\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | +| [system\_node\_pool\_kubernetes\_labels](#input\_system\_node\_pool\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | +| [system\_node\_pool\_machine\_type](#input\_system\_node\_pool\_machine\_type) | Machine type for the system node pool. | `string` | `"e2-standard-4"` | no | +| [system\_node\_pool\_name](#input\_system\_node\_pool\_name) | Name of the system node pool. | `string` | `"system"` | no | +| [system\_node\_pool\_node\_count](#input\_system\_node\_pool\_node\_count) | The total min and max nodes to be maintained in the system node pool. | 
object({
    total_min_nodes = number
    total_max_nodes = number
  })
| 
{
  "total_max_nodes": 10,
  "total_min_nodes": 2
}
| no | +| [system\_node\_pool\_taints](#input\_system\_node\_pool\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "components.gke.io/gke-managed-components",
    "value": true
  }
]
| no | +| [system\_node\_pool\_zones](#input\_system\_node\_pool\_zones) | The zones to use for the system node pool. If not specified, the cluster default node zone(s) will be used. | `list(string)` | `null` | no | +| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | +| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | +| [upgrade\_settings](#input\_upgrade\_settings) | Defines gke cluster upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | +| [version\_prefix](#input\_version\_prefix) | If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally. | `string` | `"1.31."` | no | +| [zone](#input\_zone) | Zone for a zonal cluster. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | An identifier for the resource with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | +| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | +| [gke\_version](#output\_gke\_version) | GKE cluster's version. | +| [instructions](#output\_instructions) | Instructions on how to connect to the created cluster. | +| [k8s\_service\_account\_name](#output\_k8s\_service\_account\_name) | Name of k8s service account. | + diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf new file mode 100644 index 0000000000..6106f8d90f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf @@ -0,0 +1,470 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-cluster", ghpc_role = "scheduler" }) +} + +locals { + upgrade_settings = { + strategy = var.upgrade_settings.strategy + max_surge = coalesce(var.upgrade_settings.max_surge, 0) + max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) + } +} + +locals { + dash = var.prefix_with_deployment_name && var.name_suffix != "" ? "-" : "" + prefix = var.prefix_with_deployment_name ? var.deployment_name : "" + name_maybe_empty = "${local.prefix}${local.dash}${var.name_suffix}" + name = local.name_maybe_empty != "" ? local.name_maybe_empty : "NO-NAME-GIVEN" + + cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ + security_group = var.authenticator_security_group + }] + + default_sa_email = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" + sa_email = coalesce(var.service_account_email, local.default_sa_email) + + # additional VPCs enable multi networking + derived_enable_multi_networking = coalesce(var.enable_multi_networking, length(var.additional_networks) > 0) + + # multi networking needs enabled Dataplane v2 + derived_enable_dataplane_v2 = coalesce(var.enable_dataplane_v2, local.derived_enable_multi_networking) + + default_monitoring_component = [ + "SYSTEM_COMPONENTS", + "POD", + "DAEMONSET", + "DEPLOYMENT", + "STATEFULSET", + "STORAGE", + "HPA", + "CADVISOR", + "KUBELET" + ] + + default_logging_component = [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] +} + +data "google_project" "project" { + project_id = var.project_id +} + +data "google_container_engine_versions" "version_prefix_filter" { + provider = google-beta + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + version_prefix = var.version_prefix +} + +locals { + master_version = var.min_master_version != null ? var.min_master_version : data.google_container_engine_versions.version_prefix_filter.latest_master_version +} + +resource "google_container_cluster" "gke_cluster" { + provider = google-beta + + project = var.project_id + name = local.name + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + resource_labels = local.labels + networking_mode = var.networking_mode + # decouple node pool lifecycle from cluster life cycle + remove_default_node_pool = true + initial_node_count = 1 # must be set when remove_default_node_pool is set + node_locations = var.system_node_pool_zones + + deletion_protection = var.deletion_protection + + dynamic "enable_k8s_beta_apis" { + for_each = var.enable_k8s_beta_apis != null ? [1] : [] + content { + enabled_apis = var.enable_k8s_beta_apis + } + } + + network = var.network_id + subnetwork = var.subnetwork_self_link + + # Note: the existence of the "master_authorized_networks_config" block enables + # the master authorized networks even if it's empty. + master_authorized_networks_config { + dynamic "cidr_blocks" { + for_each = var.master_authorized_networks + content { + cidr_block = cidr_blocks.value.cidr_block + display_name = cidr_blocks.value.display_name + } + } + gcp_public_cidrs_access_enabled = var.gcp_public_cidrs_access_enabled + } + + private_ipv6_google_access = var.enable_private_ipv6_google_access ? "PRIVATE_IPV6_GOOGLE_ACCESS_TO_GOOGLE" : null + default_max_pods_per_node = var.default_max_pods_per_node + master_auth { + client_certificate_config { + issue_client_certificate = false + } + } + + enable_shielded_nodes = true + + cluster_autoscaling { + # Controls auto provisioning of node-pools + enabled = false + + # Controls autoscaling algorithm of node-pools + autoscaling_profile = var.autoscaling_profile + } + + datapath_provider = local.derived_enable_dataplane_v2 ? "ADVANCED_DATAPATH" : "LEGACY_DATAPATH" + + enable_multi_networking = local.derived_enable_multi_networking + + network_policy { + # Enabling NetworkPolicy for clusters with DatapathProvider=ADVANCED_DATAPATH + # is not allowed. Dataplane V2 will take care of network policy enforcement + # instead. + enabled = false + # GKE Dataplane V2 support. This must be set to PROVIDER_UNSPECIFIED in + # order to let the datapath_provider take effect. + # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/656#issuecomment-720398658 + provider = "PROVIDER_UNSPECIFIED" + } + + private_cluster_config { + enable_private_nodes = var.enable_private_nodes + enable_private_endpoint = var.enable_private_endpoint + master_ipv4_cidr_block = var.master_ipv4_cidr_block + master_global_access_config { + enabled = var.enable_master_global_access + } + } + + ip_allocation_policy { + cluster_secondary_range_name = var.pods_ip_range_name + services_secondary_range_name = var.services_ip_range_name + } + + workload_identity_config { + workload_pool = "${var.project_id}.svc.id.goog" + } + + dynamic "gateway_api_config" { + for_each = var.enable_inference_gateway ? [1] : [] + content { + channel = "CHANNEL_STANDARD" + } + } + + dynamic "authenticator_groups_config" { + for_each = local.cluster_authenticator_security_group + content { + security_group = authenticator_groups_config.value.security_group + } + } + + release_channel { + channel = var.release_channel + } + min_master_version = local.master_version + + maintenance_policy { + daily_maintenance_window { + start_time = var.maintenance_start_time + } + + dynamic "maintenance_exclusion" { + for_each = var.maintenance_exclusions + content { + exclusion_name = maintenance_exclusion.value.name + start_time = maintenance_exclusion.value.start_time + end_time = maintenance_exclusion.value.end_time + exclusion_options { + scope = maintenance_exclusion.value.exclusion_scope + } + } + } + } + + dynamic "dns_config" { + for_each = var.cloud_dns_config != null ? [1] : [] + content { + additive_vpc_scope_dns_domain = var.cloud_dns_config.additive_vpc_scope_dns_domain + cluster_dns = var.cloud_dns_config.cluster_dns + cluster_dns_scope = var.cloud_dns_config.cluster_dns_scope + cluster_dns_domain = var.cloud_dns_config.cluster_dns_domain + } + } + + addons_config { + gcp_filestore_csi_driver_config { + enabled = var.enable_filestore_csi + } + gcs_fuse_csi_driver_config { + enabled = var.enable_gcsfuse_csi + } + gce_persistent_disk_csi_driver_config { + enabled = var.enable_persistent_disk_csi + } + dns_cache_config { + enabled = var.enable_node_local_dns_cache + } + parallelstore_csi_driver_config { + enabled = var.enable_parallelstore_csi + } + ray_operator_config { + enabled = var.enable_ray_operator + } + lustre_csi_driver_config { + enabled = var.enable_managed_lustre_csi + } + dynamic "http_load_balancing" { + for_each = var.enable_inference_gateway ? [1] : [] + content { + disabled = false + } + } + } + + timeouts { + create = var.timeout_create + update = var.timeout_update + } + + node_config { + shielded_instance_config { + enable_secure_boot = var.system_node_pool_enable_secure_boot + enable_integrity_monitoring = true + } + } + + control_plane_endpoints_config { + dns_endpoint_config { + allow_external_traffic = var.enable_external_dns_endpoint + } + } + + lifecycle { + # Ignore all changes to the default node pool. It's being removed after creation. + ignore_changes = [ + node_config, + min_master_version, + ] + precondition { + condition = var.default_max_pods_per_node == null || var.networking_mode == "VPC_NATIVE" + error_message = "default_max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." + } + precondition { + condition = coalesce(var.enable_dataplane_v2, true) || !local.derived_enable_multi_networking + error_message = "'enable_dataplane_v2' cannot be false when enabling multi networking." + } + precondition { + condition = coalesce(var.enable_multi_networking, true) || length(var.additional_networks) == 0 + error_message = "'enable_multi_networking' cannot be false when using multivpc module, which passes additional_networks." + } + } + + monitoring_config { + enable_components = var.enable_dcgm_monitoring ? concat(local.default_monitoring_component, ["DCGM"]) : local.default_monitoring_component + managed_prometheus { + enabled = true + } + } + + logging_config { + enable_components = local.default_logging_component + } +} + +# We define explicit node pools, so that it can be modified without +# having to destroy the entire cluster. +resource "google_container_node_pool" "system_node_pools" { + provider = google-beta + count = var.system_node_pool_enabled ? 1 : 0 + + project = var.project_id + name = var.system_node_pool_name + cluster = var.cluster_reference_type == "NAME" ? google_container_cluster.gke_cluster.name : google_container_cluster.gke_cluster.self_link + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + node_locations = var.system_node_pool_zones + version = local.master_version + + autoscaling { + total_min_node_count = var.system_node_pool_node_count.total_min_nodes + total_max_node_count = var.system_node_pool_node_count.total_max_nodes + } + + upgrade_settings { + strategy = local.upgrade_settings.strategy + max_surge = local.upgrade_settings.max_surge + max_unavailable = local.upgrade_settings.max_unavailable + } + + management { + auto_repair = true + auto_upgrade = true + } + + node_config { + labels = var.system_node_pool_kubernetes_labels + resource_labels = local.labels + service_account = var.service_account_email + oauth_scopes = var.service_account_scopes + machine_type = var.system_node_pool_machine_type + disk_size_gb = var.system_node_pool_disk_size_gb + disk_type = var.system_node_pool_disk_type + + dynamic "taint" { + for_each = var.system_node_pool_taints + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } + + # Forcing the use of the Container-optimized image, as it is the only + # image with the proper logging daemon installed. + # + # cos images use Shielded VMs since v1.13.6-gke.0. + # https://cloud.google.com/kubernetes-engine/docs/how-to/node-images + # + # We use COS_CONTAINERD to be compatible with (optional) gVisor. + # https://cloud.google.com/kubernetes-engine/docs/how-to/sandbox-pods + image_type = var.system_node_pool_image_type + + shielded_instance_config { + enable_secure_boot = var.system_node_pool_enable_secure_boot + enable_integrity_monitoring = true + } + + gvnic { + enabled = var.system_node_pool_image_type == "COS_CONTAINERD" + } + + # Implied by Workload Identity + workload_metadata_config { + mode = "GKE_METADATA" + } + # Implied by workload identity. + metadata = { + "disable-legacy-endpoints" = "true" + } + } + + lifecycle { + ignore_changes = [ + node_config[0].labels, + node_config[0].taint, + version, + ] + precondition { + condition = contains(["SURGE"], local.upgrade_settings.strategy) + error_message = "Only SURGE strategy is supported" + } + precondition { + condition = local.upgrade_settings.max_unavailable >= 0 + error_message = "max_unavailable should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_surge >= 0 + error_message = "max_surge should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 + error_message = "At least one of max_unavailable or max_surge must greater than 0" + } + } +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${google_container_cluster.gke_cluster.endpoint}" + cluster_ca_certificate = base64decode(google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) + token = data.google_client_config.default.access_token +} + +module "workload_identity" { + count = var.configure_workload_identity_sa ? 1 : 0 + source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" + version = ">= 40.0" + + use_existing_gcp_sa = true + name = var.k8s_service_account_name + gcp_sa_name = local.sa_email + project_id = var.project_id + + # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1059 + depends_on = [ + data.google_project.project, + google_container_cluster.gke_cluster + ] +} + +locals { + k8s_service_account_name = one(module.workload_identity[*].k8s_service_account_name) +} + +locals { + # Separate gvnic and rdma networks and assign indexes + gvnic_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "GVNIC")] : + merge(net, { name = "${var.k8s_network_names.gvnic_prefix}${idx + var.k8s_network_names.gvnic_start_index}${var.k8s_network_names.gvnic_postfix}" }) + ] + + rdma_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "RDMA")] : + merge(net, { name = "${var.k8s_network_names.rdma_prefix}${idx + var.k8s_network_names.rdma_start_index}${var.k8s_network_names.rdma_postfix}" }) + ] + + all_networks = concat(local.gvnic_networks, local.rdma_networks) +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = google_container_cluster.gke_cluster.id + project_id = var.project_id + + apply_manifests = concat(flatten([ + for idx, network_info in local.all_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = network_info.name, + network_name = network_info.network + subnetwork_name = network_info.subnetwork, + device_mode = strcontains(upper(network_info.nic_type), "RDMA") ? "RDMA" : "NetDevice" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = network_info.name } + } + ] + ]), + var.enable_inference_gateway ? [ + { + source = "https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.0.0/manifests.yaml", + template_vars = {} + } + ] : [] + ) +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml new file mode 100644 index 0000000000..bd1517ce8f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf new file mode 100644 index 0000000000..3326a5468e --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf @@ -0,0 +1,104 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_id" { + description = "An identifier for the resource with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." + value = google_container_cluster.gke_cluster.id +} + +output "gke_cluster_exists" { + description = "A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations." + value = true + depends_on = [ + google_container_cluster.gke_cluster + ] +} + +locals { + private_endpoint_message = trimspace( + <<-EOT + This cluster was created with 'enable_private_endpoint: true'. + It cannot be accessed from a public IP addresses. + One way to access this cluster is from a VM created in the GKE cluster subnet. + EOT + ) + master_authorized_networks_message = length(var.master_authorized_networks) == 0 ? "" : trimspace( + <<-EOT + The following networks have been authorized to access this cluster: + ${join("\n", [for x in var.master_authorized_networks : " ${x.display_name}: ${x.cidr_block}"])}" + EOT + ) + public_endpoint_message = trimspace( + <<-EOT + To add authorized networks you can allowlist your IP with this command: + gcloud container clusters update ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} \ + --enable-master-authorized-networks \ + --master-authorized-networks /32 + EOT + ) + allowlist_your_ip_message = var.enable_private_endpoint ? local.private_endpoint_message : local.public_endpoint_message + kubernetes_service_account_message = local.k8s_service_account_name == null ? "" : trimspace( + <<-EOT + Use the following Kubernetes Service Account in the default namespace to run your workloads: + ${local.k8s_service_account_name} + The GCP Service Account mapped to this Kubernetes Service Account is: + ${local.sa_email} + EOT + ) + kubernetes_cluster_fetch_credential_message = var.enable_external_dns_endpoint ? trimspace( + <<-EOT + Use the following command to fetch credentials for the created cluster: + gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} \ + --dns-endpoint + EOT + ) : trimspace( + <<-EOT + Use the following command to fetch credentials for the created cluster: + gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} + EOT + ) +} + +output "instructions" { + description = "Instructions on how to connect to the created cluster." + value = trimspace( + <<-EOT + ${local.master_authorized_networks_message} + + ${local.allowlist_your_ip_message} + + ${local.kubernetes_cluster_fetch_credential_message} + + ${local.kubernetes_service_account_message} + EOT + ) +} + +output "k8s_service_account_name" { + description = "Name of k8s service account." + value = local.k8s_service_account_name +} + +output "gke_version" { + description = "GKE cluster's version." + value = google_container_cluster.gke_cluster.master_version +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl new file mode 100644 index 0000000000..d376a1a760 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl @@ -0,0 +1,9 @@ +--- +apiVersion: networking.gke.io/v1 +kind: GKENetworkParamSet +metadata: + name: ${name} +spec: + vpc: ${network_name} + vpcSubnet: ${subnetwork_name} + deviceMode: ${device_mode} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl new file mode 100644 index 0000000000..1571a92692 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl @@ -0,0 +1,11 @@ +--- +apiVersion: networking.gke.io/v1 +kind: Network +metadata: + name: ${name} +spec: + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: ${name} + type: Device diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf new file mode 100644 index 0000000000..8d863b1730 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf @@ -0,0 +1,533 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "name_suffix" { + description = "Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`." + type = string + default = "" +} + +variable "deployment_name" { + description = "Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`." + type = string +} + +variable "prefix_with_deployment_name" { + description = "If true, cluster name will be prefixed by `deployment_name` (ex: -)." + type = bool + default = true +} + +variable "region" { + description = "The region to host the cluster in." + type = string +} + +variable "zone" { + description = "Zone for a zonal cluster." + default = null + type = string +} + +variable "network_id" { + description = "The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`." + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to host the cluster in." + type = string +} + +variable "pods_ip_range_name" { + description = "The name of the secondary subnet ip range to use for pods." + type = string + default = "pods" +} + +variable "services_ip_range_name" { + description = "The name of the secondary subnet range to use for services." + type = string + default = "services" +} + +variable "enable_private_ipv6_google_access" { + description = "The private IPv6 google access type for the VMs in this subnet." + type = bool + default = true +} + +variable "release_channel" { + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`." + type = string + default = "UNSPECIFIED" +} + +variable "min_master_version" { + description = "The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release." + type = string + default = null +} + +variable "version_prefix" { + description = "If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally." + type = string + default = "1.31." +} + +variable "maintenance_start_time" { + description = "Start time for daily maintenance operations. Specified in GMT with `HH:MM` format." + type = string + default = "09:00" +} + +variable "maintenance_exclusions" { + description = "List of maintenance exclusions. A cluster can have up to three." + type = list(object({ + name = string + start_time = string + end_time = string + exclusion_scope = string + })) + default = [] + validation { + condition = alltrue([ + for x in var.maintenance_exclusions : + contains(["NO_UPGRADES", "NO_MINOR_UPGRADES", "NO_MINOR_OR_NODE_UPGRADES"], x.exclusion_scope) + ]) + error_message = "`exclusion_scope` must be set to `NO_UPGRADES` OR `NO_MINOR_UPGRADES` OR `NO_MINOR_OR_NODE_UPGRADES`." + } +} + +variable "cloud_dns_config" { + description = < **_NOTE:_** The `project_id` and `region` settings would be inferred from the +> deployment variables of the same name, but they are included here for clarity. + +### Multi-networking + +To create network objects in GKE cluster, you can pass a multivpc module to a pre-existing-gke-cluster module instead of [applying a manifest manually](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment). + +```yaml + - id: network + source: modules/network/vpc + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: existing-gke-cluster ## multinetworking must be enabled in advance when cluster creation + source: modules/scheduler/pre-existing-gke-cluster + use: [multinetwork] + settings: + cluster_name: $(vars.deployment_name) +``` + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_container_cluster.existing_gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the existing cluster | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project that hosts the existing cluster | `string` | n/a | yes | +| [rdma\_subnetwork\_name\_prefix](#input\_rdma\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | `string` | `null` | no | +| [region](#input\_region) | Region in which to search for the cluster | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | +| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster exists. | +| [gke\_version](#output\_gke\_version) | GKE cluster's version. | + diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf new file mode 100644 index 0000000000..926d2be100 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_container_cluster" "existing_gke_cluster" { + name = var.cluster_name + project = var.project_id + location = var.region +} + +locals { + rdma_networks = [for network_info in var.additional_networks : network_info if strcontains(upper(network_info.nic_type), "RDMA")] + non_rdma_networks = [for network_info in var.additional_networks : network_info if !strcontains(upper(network_info.nic_type), "RDMA")] + apply_manifests_rdma_networks = flatten([ + for idx, network_info in local.rdma_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = "${var.rdma_subnetwork_name_prefix}-${idx}", + network_name = network_info.network + subnetwork_name = "${var.rdma_subnetwork_name_prefix}-${idx}", + device_mode = "RDMA" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = "${var.rdma_subnetwork_name_prefix}-${idx}" } + } + ] + ]) + + apply_manifests_non_rdma_networks = flatten([ + for idx, network_info in local.non_rdma_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = network_info.subnetwork + network_name = network_info.network + subnetwork_name = network_info.subnetwork + device_mode = "NetDevice" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = network_info.subnetwork } + } + ] + ]) +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = data.google_container_cluster.existing_gke_cluster.id + project_id = var.project_id + + apply_manifests = concat(local.apply_manifests_non_rdma_networks, local.apply_manifests_rdma_networks) +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf new file mode 100644 index 0000000000..8884ee30b0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_id" { + description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." + value = data.google_container_cluster.existing_gke_cluster.id +} + +output "gke_cluster_exists" { + description = "A static flag that signals to downstream modules that a cluster exists." + value = true + depends_on = [ + data.google_container_cluster.existing_gke_cluster + ] +} + +output "gke_version" { + description = "GKE cluster's version." + value = data.google_container_cluster.existing_gke_cluster.master_version +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl new file mode 100644 index 0000000000..d376a1a760 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl @@ -0,0 +1,9 @@ +--- +apiVersion: networking.gke.io/v1 +kind: GKENetworkParamSet +metadata: + name: ${name} +spec: + vpc: ${network_name} + vpcSubnet: ${subnetwork_name} + deviceMode: ${device_mode} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl new file mode 100644 index 0000000000..1571a92692 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl @@ -0,0 +1,11 @@ +--- +apiVersion: networking.gke.io/v1 +kind: Network +metadata: + name: ${name} +spec: + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: ${name} + type: Device diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf new file mode 100644 index 0000000000..9e9ed98ed3 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project that hosts the existing cluster" + type = string +} + +variable "cluster_name" { + description = "Name of the existing cluster" + type = string +} + +variable "region" { + description = "Region in which to search for the cluster" + type = string +} + +variable "additional_networks" { + description = "Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster." + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + })) +} + +variable "rdma_subnetwork_name_prefix" { + description = "Prefix of the RDMA subnetwork names" + default = null + type = string +} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf new file mode 100644 index 0000000000..562d8647b1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "> 5.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.74.0" + } + + required_version = ">= 1.3" +} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/README.md b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/README.md new file mode 100644 index 0000000000..db9094909b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/README.md @@ -0,0 +1,355 @@ +## Description + +This module creates a startup script that will execute a list of runners in the +order they are specified. The runners are copied to a GCS bucket at deployment +time and then copied into the VM as they are executed after startup. + +Each runner receives the following attributes: + +- `destination`: (Required) The name of the file at the destination VM. If an + absolute path is provided, the file will be copied to that path, otherwise + the file will be created in a temporary folder and deleted once the startup + script runs. +- `type`: (Required) The type of the runner, one of the following: + - `shell`: The runner is a shell script and will be executed once copied to + the destination VM. + - `ansible-local`: The runner is an ansible playbook and will run on the VM + with the following command line flags: + + ```shell + ansible-playbook --connection=local --inventory=localhost, \ + --limit localhost <> + ``` + + - `data`: The data or file specified will be copied to `<>`. No + action will be performed after the data is staged. This data can be used by + subsequent runners or simply made available on the VM for later use. +- `content`: (Optional) Content to be uploaded and, if `type` is + either `shell` or `ansible-local`, executed. Must be defined if `source` is + not. +- `source`: (Optional) A path to the file or data you want to upload. Must be + defined if `content` is not. The source path is relative to the deployment + group directory. To ensure correctness of path use `ghpc_stage` function, that + would copy referenced file to the deployment group directory. For example: + + ```yaml + source: $(ghpc_stage("path/to/file")) + ``` + + For more examples with context, see the + [example blueprint snippet](#example). To reference any other source file, an + absolute path must be used. + +- `args`: (Optional) Arguments to be passed to `shell` or `ansible-local` + runners. For `shell` runners, these will be passed as arguments to the script + when it is executed. For `ansible-local` runners, they will be appended to + a list of default arguments that invoke `ansible-playbook` on the localhost. + Therefore`args` should not include any arguments that alter this behavior, + such as `--connection`, `--inventory`, or `--limit`. + +### Runner dependencies + +`ansible-local` runners require Ansible to be installed in the VM before +running. To support other playbook runners in the Cluster Toolkit, we install +version 2.11 of `ansible-core` as well as the larger package of collections +found in `ansible` version 4.10.0. + +If an `ansible-local` runner is found in the list supplied to this module, +a script to install Ansible will be prepended to the list of runners. This +behavior can be disabled by setting `var.prepend_ansible_installer` to `false`. +This script will do the following at VM startup: + +- Install system-wide python3 if not already installed using system package + managers (yum, apt-get, etc) +- Install `python3-distutils` system-wide in debian and ubuntu based + environments. This can be a missing dependency on system installations of + python3 for installing and upgrading pip. +- Install system-wide pip3 if not already installed and upgrade pip3 if the + version is not at least 18.0. +- Install and create a virtual environment located at `/usr/local/ghpc-venv`. +- Install ansible into this virtual environment if the current version of + ansible is not version 2.11 or higher. + +To use the virtual environment created by this script, you can activate it by +running the following command on the VM: + +```shell +source /usr/local/ghpc-venv/bin/activate +``` + +You may also need to provide the correct python interpreter as the python3 +binary in the virtual environment. This can be done by adding the following flag +when calling `ansible-playbook`: + +```shell +-e ansible_python_interpreter=/usr/local/ghpc-venv/bin/activate +``` + +> **_NOTE:_** ansible-playbook and other ansible command line tools will only be +> accessible from the command line (and in your PATH variable) after activating +> this environment. + +### Staging the runners + +Runners will be uploaded to a +[GCS bucket](https://cloud.google.com/storage/docs/creating-buckets). This +bucket will be created by this module and named as +`${var.deployment_name}-startup-scripts-${random_id}`. VMs using the startup +script created by this module will pull the runners content from a GCS bucket +and therefore must have access to GCS. + +> **_NOTE:_** To ensure access to GCS, set the following OAuth scope on the +> instance using the startup scripts: +> `https://www.googleapis.com/auth/devstorage.read_only`. +> +> This is set as a default scope in the [vm-instance], +> [schedMD-slurm-on-gcp-login-node] and [schedMD-slurm-on-gcp-controller] +> modules + +[vm-instance]: ../../compute/vm-instance/README.md +[schedMD-slurm-on-gcp-login-node]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md +[schedMD-slurm-on-gcp-controller]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md + +### Tracking startup script execution + +For more information on how to use startup scripts on Google Cloud Platform, +please refer to +[this document](https://cloud.google.com/compute/docs/instances/startup-scripts/linux). + +To debug startup scripts from a Linux VM created with startup script generated +by this module: + +```shell +sudo DEBUG=1 google_metadata_script_runner startup +``` + +To view outputs from a Linux startup script, run: + +```shell +sudo journalctl -u google-startup-scripts.service +``` + +### Monitoring Agent Installation + +This `startup-script` module has several options for installing a Google +monitoring agent. There are two relevant settings: `install_stackdriver_agent` +and `install_cloud_ops_agent`. + +The _Stackdriver Agent_ also called the _Legacy Cloud Monitoring Agent_ provides +better performance under some HPC workloads. While official documentation +recommends using the _Cloud Ops Agent_, it is recommended to use +`install_stackdriver_agent` when performance is important. + +#### Stackdriver Agent Installation + +If an image or machine already has Cloud Ops Agent installed and you would like +to instead use the Stackdriver Agent, the following script will remove the Cloud +Ops Agent and install the Stackdriver Agent. + +```bash +# Remove Cloud Ops Agent +sudo systemctl stop google-cloud-ops-agent.service +sudo systemctl disable google-cloud-ops-agent.service +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +sudo bash add-google-cloud-ops-agent-repo.sh --uninstall +sudo bash add-google-cloud-ops-agent-repo.sh --remove-repo + +# Install Stackdriver Agent +curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh +sudo bash add-monitoring-agent-repo.sh --also-install +curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh +sudo bash add-logging-agent-repo.sh --also-install +sudo service stackdriver-agent start +sudo service google-fluentd restart +``` + +#### Cloud Ops Agent Installation + +If an image or machine already has the Stackdriver Agent installed and you would +like to instead use the Cloud Ops Agent, the following script will remove the +Stackdriver Agent and install the Cloud Ops Agent. + +```bash +# UnInstall Stackdriver Agent + +sudo systemctl stop stackdriver-agent.service +sudo systemctl disable stackdriver-agent.service +curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh +sudo dpkg --configure -a +sudo bash add-monitoring-agent-repo.sh --uninstall +sudo bash add-monitoring-agent-repo.sh --remove-repo +sudo systemctl stop google-fluentd.service +sudo systemctl disable google-fluentd.service +sudo dpkg --configure -a +curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh +sudo bash add-logging-agent-repo.sh --uninstall +sudo bash add-logging-agent-repo.sh --remove-repo + +# Install ops-agent + +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +sudo bash add-google-cloud-ops-agent-repo.sh --also-install +sudo service google-cloud-ops-agent start +``` + +As a reminder, this should be in a startup script, which should run on all +Compute nodes via the `compute_startup_script` on the controller. + +#### Testing Installation + +You can test if one of the agents is running using the following commands: + +```bash +# For Cloud Ops Agent +$ sudo systemctl is-active google-cloud-ops-agent"*" +active +active +active +active + +# For Legacy Monitoring and Logging Agents +$ sudo service stackdriver-agent status +stackdriver-agent is running [ OK ] +$ sudo service google-fluentd status +google-fluentd is running [ OK ] +``` + +For official documentation see troubleshooting docs: + +- [Cloud Ops Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-install-startup) +- [Legacy Monitoring Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/monitoring/troubleshooting) +- [Legacy Logging Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/logging/troubleshooting) + +### Example + +```yaml +- id: startup + source: modules/scripts/startup-script + settings: + runners: + # Some modules such as filestore have runners as outputs for convenience: + - $(homefs.install_nfs_client_runner) + # These runners can still be created manually: + # - type: shell + # destination: "modules/filestore/scripts/install_nfs_client.sh" + # source: "modules/filestore/scripts/install_nfs_client.sh" + - type: ansible-local + destination: "modules/filestore/scripts/mount.yaml" + source: "modules/filestore/scripts/mount.yaml" + - type: data + source: /tmp/foo.tgz + destination: /tmp/bar.tgz + - type: shell + destination: "decompress.sh" + content: | + #!/bin/sh + echo $2 + tar zxvf /tmp/$1 -C / + args: "bar.tgz 'Expanding file'" + +- id: compute-cluster + source: modules/compute/vm-instance + use: [homefs, startup] +``` + +In the above example, a new GCS bucket is created to upload the startup-scripts. +But in the case where the user wants to reuse existing GCS bucket or folder, +they are able to do so by using the `gcs_bucket_path` as shown in the below example + +```yaml +- id: startup + source: modules/scripts/startup-script + settings: + gcs_bucket_path: gs://user-test-bucket/folder1/folder2 + install_stackdriver_agent: true + +- id: compute-cluster + source: modules/compute/vm-instance + use: [startup] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 6.41 | +| [local](#requirement\_local) | >= 2.0.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.41 | +| [local](#provider\_local) | >= 2.0.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.configs_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | +| [google_storage_bucket_object.scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [local_file.debug_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [ansible\_virtualenv\_path](#input\_ansible\_virtualenv\_path) | Virtual environment path in which to install Ansible | `string` | `"/usr/local/ghpc-venv"` | no | +| [bucket\_viewers](#input\_bucket\_viewers) | Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account) | `list(string)` | `[]` | no | +| [configure\_ssh\_host\_patterns](#input\_configure\_ssh\_host\_patterns) | If specified, it will automate ssh configuration by:
- Defining a Host block for every element of this variable and setting StrictHostKeyChecking to 'No'.
Ex: "hpc*", "hpc01*", "ml*"
- The first time users log-in, it will create ssh keys that are added to the authorized keys list
This requires a shared /home filesystem and relies on specifying the right prefix. | `list(string)` | `[]` | no | +| [debug\_file](#input\_debug\_file) | Path to an optional local to be written with 'startup\_script'. | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used to name GCS bucket for startup scripts. | `string` | n/a | yes | +| [docker](#input\_docker) | Install and configure Docker | 
object({
    enabled        = optional(bool, false)
    world_writable = optional(bool, false)
    daemon_config  = optional(string, "")
  })
| 
{
  "enabled": false
}
| no | +| [enable\_docker\_world\_writable](#input\_enable\_docker\_world\_writable) | DEPRECATED: use var.docker | `bool` | `null` | no | +| [enable\_gpu\_network\_wait\_online](#input\_enable\_gpu\_network\_wait\_online) | Enable a SystemD unit that blocks execution of startup-scripts until after all network interfaces are online. (Works on reboots or boots of an image built using this solution) | `bool` | `false` | no | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | `null` | no | +| [http\_no\_proxy](#input\_http\_no\_proxy) | Domains for which to disable http\_proxy behavior. Honored only if var.http\_proxy is set | `string` | `".google.com,.googleapis.com,metadata.google.internal,localhost,127.0.0.1"` | no | +| [http\_proxy](#input\_http\_proxy) | Web (http and https) proxy configuration for pip, apt, and yum/dnf and interactive shells | `string` | `""` | no | +| [install\_ansible](#input\_install\_ansible) | Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used. | `bool` | `null` | no | +| [install\_cloud\_ops\_agent](#input\_install\_cloud\_ops\_agent) | Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true. | `bool` | `false` | no | +| [install\_cloud\_rdma\_drivers](#input\_install\_cloud\_rdma\_drivers) | If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8. Should not be enabled if using the HPC VM Image. | `bool` | `false` | no | +| [install\_docker](#input\_install\_docker) | DEPRECATED: use var.docker. | `bool` | `null` | no | +| [install\_stackdriver\_agent](#input\_install\_stackdriver\_agent) | Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance. | `bool` | `false` | no | +| [labels](#input\_labels) | Labels for the created GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_filesystem](#input\_local\_ssd\_filesystem) | Create and mount a filesystem from local SSD disks (data will be lost if VMs are powered down without enabling migration); enable by setting mountpoint field to a valid directory path. | 
object({
    fs_type     = optional(string, "ext4")
    mountpoint  = optional(string, "")
    permissions = optional(string, "0755")
  })
| 
{
  "fs_type": "ext4",
  "mountpoint": "",
  "permissions": "0755"
}
| no | +| [managed\_lustre](#input\_managed\_lustre) | Configure Managed Lustre (assumes driver already installed) | 
object({
    enabled = optional(bool, false)
    port    = optional(number, 988)
  })
| 
{
  "enabled": false,
  "port": 988
}
| no | +| [prepend\_ansible\_installer](#input\_prepend\_ansible\_installer) | DEPRECATED. Use `install_ansible=false` to prevent ansible installation. | `bool` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [runners](#input\_runners) | List of runners to run on remote VM.
Runners can be of type ansible-local, shell or data.
A runner must specify one of 'source' or 'content'.
All runners must specify 'destination'. If 'destination' does not include a
path, it will be copied in a temporary folder and deleted after running.
Runners may also pass 'args', which will be passed as argument to shell runners only. | `list(map(string))` | `[]` | no | +| [set\_ofi\_cloud\_rdma\_tunables](#input\_set\_ofi\_cloud\_rdma\_tunables) | Controls whether to enable specific OFI environment variables for workloads using Cloud RDMA networking. Should be false for non-RDMA workloads. | `bool` | `false` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [compute\_startup\_script](#output\_compute\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | +| [controller\_startup\_script](#output\_controller\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | +| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | + diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml new file mode 100644 index 0000000000..02c449c7cb --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml @@ -0,0 +1,37 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Configure ssh between nodes + become: true + hosts: localhost + vars: + ssh_config_path: "/etc/ssh/ssh_config" + bashrc: "{{ '/etc/bashrc' if ansible_facts['os_family'] == 'RedHat' else '/etc/bash.bashrc' }}" + setup_ssh_script: "/bin/bash /usr/local/ghpc/setup-ssh-keys.sh" + tasks: + - name: "Set StrictHostKeyChecking to no" + ansible.builtin.blockinfile: + path: "{{ ssh_config_path }}" + block: | + Host "{{ item }}" + StrictHostKeyChecking no + marker: "# {mark} ANSIBLE MANAGED BLOCK {{item}}" + loop: "{{ host_name_prefix }}" + - name: "Create ssh keys in .bashrc if not already done" + ansible.builtin.lineinfile: + path: "{{ bashrc }}" + regexp: '^{{ setup_ssh_script }}' + line: "{{ setup_ssh_script }}" diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh new file mode 100644 index 0000000000..38c7ff9b5c --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +web_proxy="${1:-}" +if [ -z "$web_proxy" ]; then + echo "Error: must provide 1 argument identifying http/https proxy" + exit 1 +fi + +# configure pip to use proxy +PIP_CONF=/etc/pip.conf +if [ ! -f "$PIP_CONF" ]; then + cat <<-EOF >"$PIP_CONF" + [global] + proxy=$web_proxy + EOF +fi + +# configure yum or dnf to use proxy +if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + YUM_CONF="/etc/yum.conf" + if ! grep -q '^proxy=.*' "$YUM_CONF"; then + sed --follow-symlinks -i.bak "/^\[main]/a proxy=$web_proxy" "$YUM_CONF" + else + sed --follow-symlinks -i.bak "s,proxy=.*,proxy=$web_proxy," "$YUM_CONF" + fi +fi + +# configure apt to use proxy +if [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release 2>/dev/null || + grep -qi ubuntu /etc/os-release 2>/dev/null; then + APT_CONF_PROXY="/etc/apt/apt.conf.d/99proxy.conf" + if [ ! -f "$APT_CONF_PROXY" ]; then + cat <<-EOF >"$APT_CONF_PROXY" + Acquire::http::Proxy "$web_proxy"; + Acquire::https::Proxy "$web_proxy"; + EOF + fi +fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh new file mode 100644 index 0000000000..682e1352a1 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script applies fixes to VMs that must occur early in boot. For example, +# when yum or apt repositories are misconfigured, preventing most package +# operations from completing successfully. + +source /etc/os-release + +if [[ "$PRETTY_NAME" == "CentOS Linux 7 (Core)" ]]; then + echo "Applying hotfixes for CentOS 7" + if grep -q '^mirrorlist' /etc/yum.repos.d/CentOS-Base.repo; then + echo "Removing mirrorlist from default CentOS 7 repositories" + sed -i '/^mirrorlist/d' /etc/yum.repos.d/CentOS-Base.repo + fi + if grep -q '^#baseurl=http://mirror.centos.org' /etc/yum.repos.d/CentOS-Base.repo; then + echo "Reconfiguring default CentOS 7 repositories to use CentOS Vault" + sed -i 's,^#baseurl=http://mirror.centos.org/,baseurl=http://vault.centos.org/,' /etc/yum.repos.d/CentOS-Base.repo + fi +fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh new file mode 100644 index 0000000000..3a29ae808f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh @@ -0,0 +1,73 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Given a url and filename, download an object to the vardir. When the installed +# version of gcloud is >=402.0.0 (Sept. 2022), then gcloud storage is used to +# fetch from the bucket. Otherwise gsutil is used. Note, the service account for +# the instance must be properly configured with a role having authorization to +# get objects from the bucket. +# +# This function is intended for single file downloads and no attempt is made to +# verify the checksum other than the default behavior of gcloud or gsutil. +# +# This function has no other platform dependencies other than gcloud / gsutil. + +# This code originated from: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 +stdlib::get_from_bucket() { + local OPTIND opt url fname dir="${VARDIR:-/var/lib/startup}" + while getopts ":u:f:d:" opt; do + case "${opt}" in + u) url="${OPTARG}" ;; + f) fname="${OPTARG}" ;; + d) dir="${OPTARG}" ;; + :) + stdlib::mandatory_argument -n stdlib::get_from_bucket -f "$OPTARG" + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error 'Usage: stdlib::get_from_bucket -u -f -d ' + stdlib::info 'For example: stdlib::get_from_bucket -u gs://mybucket/foo.tgz -d /var/tmp' + return "${E_UNKNOWN_ARG}" + ;; + esac + done + # Trivially compute the filename from the URL if unspecified. + if [[ -z ${fname} ]]; then + fname=${url##*/} + stdlib::debug "Computed filename='${fname}' given URL." + fi + [[ -d ${dir} ]] || mkdir "${dir}" + local attempt=0 + local max_retries=7 + # store gcs command as array and then split when called by stdlib::cmd + if stdlib::cmd gcloud help storage cp &>/dev/null; then + gcs_command=(gcloud storage cp --no-user-output-enabled) + else + gcs_command=(gsutil -q cp) + fi + while [[ $attempt -le $max_retries ]]; do + if [[ $attempt -gt 0 ]]; then + local wait=$((2 ** attempt)) + stdlib::error "Retry attempt ${attempt} of ${max_retries} with exponential backoff: ${wait} seconds." + sleep $wait + fi + if stdlib::cmd "${gcs_command[@]}" "${url}" "${dir}/${fname}"; then + break + else + stdlib::error "${gcs_command[*]} reported non-zero exit code fetching ${url}." + ((attempt++)) + fi + done +} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh new file mode 100644 index 0000000000..eac2b2e32a --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh @@ -0,0 +1,247 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +REQ_ANSIBLE_VERSION=2.15 +REQ_ANSIBLE_PIP_VERSION=8.7.0 +REQ_PIP_WHEEL_VERSION=0.45.1 +REQ_PIP_SETUPTOOLS_VERSION=80.8.0 +REQ_PIP_MAJOR_VERSION=25 +REQ_PYTHON3_VERSION=9 + +apt_wait() { + while fuser /var/lib/apt/lists/lock >/dev/null 2>&1; do + echo "Sleeping for apt lists lock" + sleep 3 + done +} + +# Installs any dependencies needed for python based on the OS +install_python_deps() { + # this file is present on both Debian and Ubuntu OSes + if [ -f /etc/debian_version ]; then + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3-setuptools python3-venv + fi +} + +# Gets the name of the python executable for python starting with python3, then +# checking python. Sets the variable to an empty string if neither are found. +get_python_path() { + python_path="" + if command -v python3 1>/dev/null; then + python_path=$(command -v python3) + elif command -v python 1>/dev/null; then + python_path=$(command -v python) + fi +} + +# Returns the python major version. If provided, it will use the first argument +# as the python executable, otherwise it will default to simply "python". +get_python_major_version() { + python_path=${1:-python} + python_major_version=$(${python_path} -c "import sys; print(sys.version_info.major)") +} + +# Returns the python minor version. If provided, it will use the first argument +# as the python executable, otherwise it will default to simply "python". +get_python_minor_version() { + python_path=${1:-python} + python_minor_version=$(${python_path} -c "import sys; print(sys.version_info.minor)") +} + +# Install python3 with the yum package manager. Updates python_path to the +# newly installed packaged. +install_python3_dnf() { + major_version=$(rpm -E "%{rhel}") + set -- "--disablerepo=*" "--enablerepo=baseos,appstream" + if grep -qi 'ID="rhel"' /etc/os-release; then + # Do not set --disablerepo / --enablerepo on RedHat, due to + # complex repo names; clear array + set -- + fi + # On Rocky Linux 9, Python 3.9 is installed by default but this + # has already been dropped by ansible-core for control nodes. + # https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix + # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where + # it is available as "python3*" but must be named explicitly on + # older releases. It also ensures longer support for Ansible. + if [ "${major_version}" -lt "10" ]; then + dnf install "$@" -y python3.12 python3.12-pip + python_path=$(command -v python3.12) + else + dnf install "$@" -y python3 python3-pip + python_path=$(command -v python3) + fi +} + +# Install python3 with the apt package manager. Updates python_path to the +# newly installed packaged. +install_python3_apt() { + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3 python3-setuptools python3-pip python3-venv + python_path=$(command -v python3) +} + +install_python3() { + if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || + [ -f /etc/system-release ]; then + install_python3_dnf + elif [ -f /etc/debian_version ]; then + install_python3_apt + else + echo "Error: Unsupported Distribution" + return 1 + fi +} + +# Install pip3 with the dnf package manager. Updates python_path to the +# newly installed packaged. +install_pip3_dnf() { + major_version=$(rpm -E "%{rhel}") + set -- "--disablerepo=*" "--enablerepo=baseos,appstream" + if grep -qi 'ID="rhel"' /etc/os-release; then + # Do not set --disablerepo / --enablerepo on RedHat, due to complex repo names + # clear array + set -- + fi + # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where + # it is available as "python3*" but must be named explicitly on + # older releases. It also ensures longer support for Ansible. + if [ "${major_version}" -lt "10" ]; then + dnf install "$@" -y python3.12-pip + else + dnf install "$@" -y python3-pip + fi +} + +# Install pip3 with the apt package manager. Updates python_path to the +# newly installed packaged. +install_pip3_apt() { + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3-pip +} + +install_pip3() { + if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || + [ -f /etc/system-release ]; then + install_pip3_dnf + elif [ -f /etc/debian_version ]; then + install_pip3_apt + else + echo "Error: Unsupported Distribution" + return 1 + fi +} + +main() { + if [ $# -gt 1 ]; then + echo "Error: provide only 1 optional argument identifying virtual environment path for Ansible" + return 1 + fi + + venv_path="${1:-/usr/local/ghpc-venv}" + + # Get the python3 executable, or install it if not found + get_python_path + get_python_major_version "${python_path}" + get_python_minor_version "${python_path}" + if [ "${python_path}" = "" ] || [ "${python_major_version}" = "2" ] || [ "${python_minor_version}" -lt "${REQ_PYTHON3_VERSION}" ]; then + if ! install_python3; then + return 1 + fi + get_python_major_version "${python_path}" + get_python_minor_version "${python_path}" + else + install_python_deps + fi + + # Install OS-packaged pip + if ! ${python_path} -m pip --version 2>/dev/null; then + if ! install_pip3; then + return 1 + fi + fi + + # Create pip virtual environment for Cluster Toolkit + ${python_path} -m venv "${venv_path}" --copies + venv_python_path=${venv_path}/bin/python3 + + # Upgrade pip if necessary + pip_version=$(${venv_python_path} -m pip --version | sed -nr 's/^pip ([0-9]+\.[0-9]+).*$/\1/p') + pip_major_version=$(echo "${pip_version}" | cut -d '.' -f 1) + if [ "${pip_major_version}" -lt "${REQ_PIP_MAJOR_VERSION}" ]; then + ${venv_python_path} -m pip install --upgrade pip + fi + + # upgrade wheel if necessary + wheel_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^wheel" || true) + if [ "$wheel_pkg" != "wheel==${REQ_PIP_WHEEL_VERSION}" ]; then + ${venv_python_path} -m pip install -U wheel==${REQ_PIP_WHEEL_VERSION} + fi + + # upgrade setuptools if necessary + setuptools_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^setuptools" || true) + if [ "$setuptools_pkg" != "setuptools==${REQ_PIP_SETUPTOOLS_VERSION}" ]; then + ${venv_python_path} -m pip install -U setuptools==${REQ_PIP_SETUPTOOLS_VERSION} + fi + + # configure ansible to always use correct Python binary + if [ ! -f /etc/ansible/ansible.cfg ]; then + mkdir /etc/ansible + cat <<-EOF >/etc/ansible/ansible.cfg + [defaults] + interpreter_python=${venv_python_path} + stdout_callback=debug + stderr_callback=debug + EOF + fi + + # Install ansible + ansible_version="" + if command -v ansible-playbook 1>/dev/null; then + ansible_version=$(ansible-playbook --version 2>/dev/null | sed -nr 's/^ansible-playbook.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p') + ansible_major_vers=$(echo "${ansible_version}" | cut -d '.' -f 1) + ansible_minor_vers=$(echo "${ansible_version}" | cut -d '.' -f 2) + ansible_req_major_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 1) + ansible_req_minor_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 2) + fi + if [ -z "${ansible_version}" ] || [ "${ansible_major_vers}" -ne "${ansible_req_major_vers}" ] || + [ "${ansible_minor_vers}" -lt "${ansible_req_minor_vers}" ]; then + ${venv_python_path} -m pip install ansible=="${REQ_ANSIBLE_PIP_VERSION}" + fi + while read -r cmd; do + if ! [ -L "/usr/bin/${cmd}" ]; then + ln -s "${venv_path}/bin/${cmd}" "/usr/bin/${cmd}" + fi + done <<-EOF + ansible + ansible-config + ansible-connection + ansible-console + ansible-doc + ansible-galaxy + ansible-inventory + ansible-playbook + ansible-pull + ansible-test + ansible-vault + EOF +} + +main "$@" diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh new file mode 100644 index 0000000000..375792459b --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +OS_ID="$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g')" +OS_VERSION="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g')" +OS_VERSION_MAJOR="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//')" +REBOOT_FILE="/etc/.rdma_reboot" + +if { [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; } && { [ "${OS_VERSION_MAJOR}" = "8" ]; }; then + KMOD_VERSION="$(dnf list installed | awk '$1 ~ /^kmod-idpf-irdma(\.|$)/ {print $2}')" + + # For images that do not already have Cloud RDMA drivers installed + if [ -z "${KMOD_VERSION}" ] && [ -z "${REBOOT_FILE}" ]; then + sudo dnf update -y + sudo dnf install https://depot.ciq.com/public/files/gce-accelerator/irdma-kernel-modules-el8-x86_64/irdma-repos.rpm -y + sudo dnf install kmod-idpf-irdma rdma-core libibverbs-utils librdmacm-utils infiniband-diags perftest -y + sudo touch "${REBOOT_FILE}" + reboot + fi + echo "This image has IRDMA packages already installed, exiting." + exit 0 +else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. Cloud RDMA Drivers are only supported on Rocky Linux 8." + exit 1 +fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml new file mode 100644 index 0000000000..f9b0abeb14 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml @@ -0,0 +1,113 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install and configure Docker + hosts: all + become: true + vars: + docker_data_root: '' + docker_daemon_config: '' + enable_docker_world_writable: false + tasks: + - name: Check if docker is installed + ansible.builtin.stat: + path: /usr/bin/docker + register: docker_binary + - name: Download Docker Installer + ansible.builtin.get_url: + url: https://get.docker.com + dest: /tmp/get-docker.sh + owner: root + group: root + mode: '0644' + when: not docker_binary.stat.exists + - name: Install Docker + ansible.builtin.command: sh /tmp/get-docker.sh + register: docker_installed + changed_when: docker_installed.rc != 0 + when: not docker_binary.stat.exists + - name: Create Docker daemon configuration + ansible.builtin.copy: + dest: /etc/docker/daemon.json + mode: '0644' + content: '{{ docker_daemon_config }}' + validate: /usr/bin/dockerd --validate --config-file %s + when: docker_daemon_config + notify: + - Restart Docker + - name: Create Docker service override directory + ansible.builtin.file: + path: /etc/systemd/system/docker.service.d + state: directory + owner: root + group: root + mode: '0755' + - name: Create Docker service override configuration + ansible.builtin.copy: + dest: /etc/systemd/system/docker.service.d/data-root.conf + mode: '0644' + content: | + [Unit] + {% if docker_data_root %} + RequiresMountsFor={{ docker_data_root }} + {% endif %} + After=mount-localssd-raid.service + - name: Create Docker socket override directory + ansible.builtin.file: + path: /etc/systemd/system/docker.socket.d + state: directory + owner: root + group: root + mode: '0755' + when: enable_docker_world_writable + - name: Create Docker socket override configuration + ansible.builtin.copy: + dest: /etc/systemd/system/docker.socket.d/world-writable.conf + mode: '0644' + content: | + [Socket] + SocketMode=0666 + when: enable_docker_world_writable + notify: + - Reload SystemD + - Recreate Docker socket + - name: Delete Docker socket override configuration + ansible.builtin.file: + path: /etc/systemd/system/docker.socket.d/world-writable.conf + state: absent + when: not enable_docker_world_writable + notify: + - Reload SystemD + - Recreate Docker socket + + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + - name: Recreate Docker socket + ansible.builtin.service: + name: docker.socket + state: restarted + - name: Restart Docker + ansible.builtin.service: + name: docker.service + state: restarted + + post_tasks: + - name: Start Docker + ansible.builtin.service: + name: docker.service + state: started + enabled: true diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml new file mode 100644 index 0000000000..9d295dfc7d --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml @@ -0,0 +1,56 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install network wait service for A3/A4 variants + hosts: all + become: true + tasks: + + - name: Create universal SystemD service for GPU networking delay + when: ansible_os_family == "Debian" + ansible.builtin.copy: + dest: /etc/systemd/system/delay-gpu-network.service + owner: root + group: root + mode: "0644" + content: | + [Unit] + Description=Delay boot on multi-NIC VMs until networks are routable + After=network-online.target + Wants=network-online.target + Before=google-startup-scripts.service + + [Service] + # This condition checks if the machine type is one of the supported A3/A4 variants. + # The service will only run if the machine type matches. + ExecCondition=/bin/bash -c "/usr/bin/curl -s -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/machine-type | grep -qE '(/a3-highgpu-8g|/a3-megagpu-8g|/a3-ultragpu-8g|/a4-highgpu-8g|/a4x-highgpu-4g)$'" + ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -o routable --timeout=180 + ExecStartPost=/bin/sleep 30 + + [Install] + WantedBy=multi-user.target + notify: + - Reload SystemD + + - name: Enable universal GPU network delay service + when: ansible_os_family == "Debian" + ansible.builtin.systemd_service: + name: delay-gpu-network.service + enabled: true + + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml new file mode 100644 index 0000000000..94699471bb --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml @@ -0,0 +1,33 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure Managed Lustre (assumes driver already installed) + hosts: all + become: true + vars: + default_lustre_port: 988 + managed_lustre_port: "{{ default_lustre_port }}" + tasks: + # Ideally changes to this file would also trigger an execution of lnetctl + # command to update accept_port but it is unclear if lnetctl supports this. + - name: Update lnet to use non-default port + when: managed_lustre_port | int != {{ default_lustre_port }} + ansible.builtin.copy: + owner: root + group: root + mode: '0644' + dest: /etc/modprobe.d/lnet.conf + content: | + options lnet accept_port={{ managed_lustre_port | int }} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh new file mode 100644 index 0000000000..eb4bf899b8 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +LEGACY_MONITORING_PACKAGE='stackdriver-agent' +LEGACY_MONITORING_SCRIPT_URL='https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh' +LEGACY_LOGGING_PACKAGE='google-fluentd' +LEGACY_LOGGING_SCRIPT_URL='https://dl.google.com/cloudagents/add-logging-agent-repo.sh' + +OPSAGENT_PACKAGE='google-cloud-ops-agent' +OPSAGENT_SCRIPT_URL='https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh' + +ops_or_legacy="${1:-legacy}" + +fail() { + echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" + exit 1 +} + +handle_debian() { + is_legacy_monitoring_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_MONITORING_PACKAGE} | + grep "${LEGACY_MONITORING_PACKAGE} is installed" + } + + is_legacy_logging_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_LOGGING_PACKAGE} | + grep "${LEGACY_LOGGING_PACKAGE} is installed" + } + + is_legacy_installed() { + is_legacy_monitoring_installed || is_legacy_logging_installed + } + + is_opsagent_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${OPSAGENT_PACKAGE} | + grep "${OPSAGENT_PACKAGE} is installed" + } + + install_with_retry() { + MAX_RETRY=50 + RETRY=0 + until [ ${RETRY} -eq ${MAX_RETRY} ] || curl -s "${1}" | bash -s -- --also-install; do + RETRY=$((RETRY + 1)) + echo "WARNING: Installation of ${1} failed on try ${RETRY} of ${MAX_RETRY}" + sleep 5 + done + if [ $RETRY -eq $MAX_RETRY ]; then + echo "ERROR: Installation of ${1} was not successful after ${MAX_RETRY} attempts." + exit 1 + fi + } + + install_opsagent() { + install_with_retry "${OPSAGENT_SCRIPT_URL}" + } + + install_stackdriver_agent() { + install_with_retry "${LEGACY_MONITORING_SCRIPT_URL}" + install_with_retry "${LEGACY_LOGGING_SCRIPT_URL}" + service stackdriver-agent start + service google-fluentd start + } +} + +handle_redhat() { + is_legacy_monitoring_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_MONITORING_PACKAGE} | + grep "${LEGACY_MONITORING_PACKAGE} is installed" + } + + is_legacy_logging_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_LOGGING_PACKAGE} | + grep "${LEGACY_LOGGING_PACKAGE} is installed" + } + + is_legacy_installed() { + is_legacy_monitoring_installed || is_legacy_logging_installed + } + + is_opsagent_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${OPSAGENT_PACKAGE} | + grep "${OPSAGENT_PACKAGE} is installed" + } + + install_opsagent() { + curl -s "${OPSAGENT_SCRIPT_URL}" | bash -s -- --also-install + } + + install_stackdriver_agent() { + curl -sS "${LEGACY_MONITORING_SCRIPT_URL}" | bash -s -- --also-install + curl -sS "${LEGACY_LOGGING_SCRIPT_URL}" | bash -s -- --also-install + service stackdriver-agent start + service google-fluentd start + } +} + +main() { + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + handle_redhat + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + handle_debian + else + fail "Unsupported platform." + fi + + # Handle cases that agent is already installed + if [[ -z "$(is_legacy_monitoring_installed)" && -n $(is_legacy_logging_installed) ]] || + [[ -n "$(is_legacy_monitoring_installed)" && -z $(is_legacy_logging_installed) ]]; then + fail "Bad state: legacy agent is partially installed" + elif [[ "${ops_or_legacy}" == "legacy" ]] && is_legacy_installed; then + echo "Legacy agent is already installed" + exit 0 + elif [[ "${ops_or_legacy}" != "legacy" ]] && is_opsagent_installed; then + echo "Ops agent is already installed" + exit 0 + elif is_legacy_installed || is_opsagent_installed; then + fail "Agent is already installed but does not match requested agent of ${ops_or_legacy}" + fi + + # install agent + if [[ "${ops_or_legacy}" == "legacy" ]]; then + echo "Installing legacy monitoring agent (stackdriver)" + install_stackdriver_agent + else + echo "Installing cloud ops agent" + echo "WARNING: cloud ops agent may have a performance impact. Consider using legacy monitoring agent (stackdriver)." + install_opsagent + fi +} + +main diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh new file mode 100644 index 0000000000..738181aafb --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_COMPLETE_FILE="/run/startup_script_msg" + +# Ensure we're in an interactive terminal and not root +if [ -t 1 ] && [ "$(id -u)" -ne 0 ]; then + # Check if the file has contents otherwise skip + if [ -s "$SCRIPT_COMPLETE_FILE" ]; then + echo + cat "$SCRIPT_COMPLETE_FILE" + echo + fi +fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml new file mode 100644 index 0000000000..d94aac81fd --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml @@ -0,0 +1,100 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Configure local SSDs + become: true + hosts: localhost + vars: + raid_name: localssd + array_dev: /dev/md/{{ raid_name }} + fstype: ext4 + interface: nvme + mode: '0755' + mountpoint: /mnt/{{ raid_name }} + tasks: + - name: Get local SSD devices + ansible.builtin.find: + file_type: link + path: /dev/disk/by-id + patterns: google-local-{{ "nvme-" if interface == "nvme" else "" }}ssd-* + register: local_ssd_devices + + - name: Exit if zero local ssd found + ansible.builtin.meta: end_play + when: local_ssd_devices.files | length == 0 + + - name: Install mdadm + ansible.builtin.package: + name: mdadm + state: present + + # this service will act during the play and upon reboots to ensure that local + # SSD volumes are always assembled into a RAID and re-formatted if necessary; + # there are many scenarios where a VM can be stopped or migrated during + # maintenance and the contents of local SSD will be discarded + - name: Install service to create local SSD RAID and format it + ansible.builtin.copy: + dest: /etc/systemd/system/create-localssd-raid.service + mode: 0644 + content: | + [Unit] + After=local-fs.target + Before=slurmd.service docker.service + ConditionPathExists=!{{ array_dev }} + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/bash -c "/usr/sbin/mdadm --create {{ array_dev }} --name={{ raid_name }} --homehost=any --level=0 --raid-devices={{ local_ssd_devices.files | length }} /dev/disk/by-id/google-local-nvme-ssd-*{{ " --force" if local_ssd_devices.files | length == 1 else "" }}" + ExecStartPost=/usr/sbin/mkfs -t {{ fstype }}{{ " -m 0" if fstype == "ext4" else "" }} {{ array_dev }} + + [Install] + WantedBy=slurmd.service docker.service + + - name: Create RAID array and format + ansible.builtin.systemd: + name: create-localssd-raid.service + state: started + enabled: true + daemon_reload: true + + - name: Install service to mount local SSD array + ansible.builtin.copy: + dest: /etc/systemd/system/mount-localssd-raid.service + mode: 0644 + content: | + [Unit] + After=local-fs.target create-localssd-raid.service + Before=slurmd.service docker.service + Wants=create-localssd-raid.service + ConditionPathIsMountPoint=!{{ mountpoint }} + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/systemd-mount -t {{ fstype }} -o discard,defaults,nofail {{ array_dev }} {{ mountpoint }} + ExecStartPost=/usr/bin/chmod {{ mode }} {{ mountpoint }} + ExecStop=/usr/bin/systemd-umount {{ mountpoint }} + + [Install] + WantedBy=slurmd.service docker.service + + - name: Mount RAID array and set permissions + ansible.builtin.systemd: + name: mount-localssd-raid.service + state: started + enabled: true + daemon_reload: true diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh new file mode 100644 index 0000000000..1c8018fb01 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! -d ~/.ssh/ ]; then + source /usr/local/ghpc-venv/bin/activate + ansible-playbook /usr/local/ghpc/setup-ssh-keys.yml +fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml new file mode 100644 index 0000000000..692896bb9c --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml @@ -0,0 +1,40 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Setup SSH Keys for user + become: false + hosts: localhost + vars: + pub_key_path: "{{ ansible_env.HOME }}/.ssh" + pub_key_file: "{{ pub_key_path }}/id_rsa" + auth_key_file: "{{ pub_key_path }}/authorized_keys" + tasks: + - name: "Create .ssh folder" + ansible.builtin.file: + path: "{{ pub_key_path }}" + state: directory + mode: 0700 + owner: "{{ ansible_user_id }}" + - name: Create keys + community.crypto.openssh_keypair: + path: "{{ pub_key_file }}" + owner: "{{ ansible_user_id }}" + - name: Copy public key to authorized keys + ansible.builtin.copy: + src: "{{ pub_key_file }}.pub" + dest: "{{ auth_key_file }}" + owner: "{{ ansible_user_id }}" + mode: 0644 diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh new file mode 100644 index 0000000000..8ca40bc73f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh @@ -0,0 +1,39 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code contains minor changes from the original: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 + +stdlib::main() { + DELETE_AT_EXIT="$(mktemp -d)" + readonly DELETE_AT_EXIT + + # Initialize state required by other functions, e.g. debug() + stdlib::init + stdlib::debug "Loaded startup-script-stdlib as an executable." + + stdlib::load_config_values + + stdlib::load_runners +} + +# if script is being executed and not sourced. +if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then + stdlib::finish() { + [[ -d ${DELETE_AT_EXIT:-} ]] && rm -rf "${DELETE_AT_EXIT}" + } + trap stdlib::finish EXIT + + stdlib::main "$@" +fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh new file mode 100644 index 0000000000..589a3215ab --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh @@ -0,0 +1,266 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code contains minor changes from the original in: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 + +# Standard library of functions useful for startup scripts. + +# These are outside init_global_vars so logging functions work with the most +# basic case of `source startup-script-stdlib.sh` +readonly SYSLOG_DEBUG_PRIORITY="${SYSLOG_DEBUG_PRIORITY:-syslog.debug}" +readonly SYSLOG_INFO_PRIORITY="${SYSLOG_INFO_PRIORITY:-syslog.info}" +readonly SYSLOG_ERROR_PRIORITY="${SYSLOG_ERROR_PRIORITY:-syslog.error}" +# Global counter of how many times stdlib::init() has been called. +STARTUP_SCRIPT_STDLIB_INITIALIZED=0 + +# Error codes +readonly E_RUN_OR_DIE=5 +readonly E_MISSING_MANDATORY_ARG=9 +readonly E_UNKNOWN_ARG=10 + +SCRIPT_COMPLETE_FILE="/run/startup_script_msg" +SUCCESS_MESSAGE="* NOTICE **: The Cluster Toolkit startup scripts have finished running successfully." +readonly SUCCESS_MESSAGE +ERROR_MESSAGE="** ERROR **: The Cluster Toolkit startup scripts have finished running, but produced an error." +readonly ERROR_MESSAGE +WARNING_MESSAGE="** WARNING **: The Cluster Toolkit startup scripts are currently running." +readonly WARNING_MESSAGE + +stdlib::debug() { + [[ -z ${DEBUG:-} ]] && return 0 + local ds msg + msg="$*" + logger -p "${SYSLOG_DEBUG_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + [[ -n ${QUIET:-} ]] && return 0 + ds="$(date +"${DATE_FMT}") " + echo -e "${BLUE}${ds}Debug [$$]: ${msg}${NC}" >&2 +} + +stdlib::info() { + local ds msg + msg="$*" + logger -p "${SYSLOG_INFO_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + [[ -n ${QUIET:-} ]] && return 0 + ds="$(date +"${DATE_FMT}") " + echo -e "${GREEN}${ds}Info [$$]: ${msg}${NC}" >&2 +} + +stdlib::error() { + local ds msg + msg="$*" + ds="$(date +"${DATE_FMT}") " + logger -p "${SYSLOG_ERROR_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + echo -e "${RED}${ds}Error [$$]: ${msg}${NC}" >&2 +} + +stdlib::announce_runners_start() { + if [ -z "$recursive_proc" ]; then + wall -n "$WARNING_MESSAGE" + echo "$WARNING_MESSAGE" >"$SCRIPT_COMPLETE_FILE" + fi + export recursive_proc=$((${recursive_proc:=0} + 1)) +} + +stdlib::announce_runners_end() { + exit_code=$1 + export recursive_proc=$((${recursive_proc:=0} - 1)) + if [ "$recursive_proc" -le "0" ]; then + if [ "$exit_code" -ne "0" ]; then + wall -n "$ERROR_MESSAGE" + echo "$ERROR_MESSAGE" >"$SCRIPT_COMPLETE_FILE" + else + wall -n "$SUCCESS_MESSAGE" + echo -n "" >"$SCRIPT_COMPLETE_FILE" + fi + fi +} + +# The main initialization function of this library. This should be kept to the +# minimum amount of work required for all functions to operate cleanly. +stdlib::init() { + if [[ ${STARTUP_SCRIPT_STDLIB_INITIALIZED} -gt 0 ]]; then + stdlib::info 'stdlib::init()'" already initialized, no action taken." + return 0 + fi + ((STARTUP_SCRIPT_STDLIB_INITIALIZED++)) || true + stdlib::init_global_vars + stdlib::init_directories + stdlib::debug "stdlib::init(): startup-script-stdlib.sh initialized and ready" +} + +# Initialize global variables. +stdlib::init_global_vars() { + # The program name, used for logging. + readonly PROG="${PROG:-startup-script-stdlib}" + # Date format used for stderr logging. Passed to date + command. + readonly DATE_FMT="${DATE_FMT:-"%a %b %d %H:%M:%S %z %Y"}" + # var directory + readonly VARDIR="${VARDIR:-/var/lib/startup}" + # Override this with file://localhost/tmp/foo/bar in spec test context + readonly METADATA_BASE="${METADATA_BASE:-http://metadata.google.internal}" + + # Color variables + if [[ -n ${COLOR:-} ]]; then + readonly NC='\033[0m' # no color + readonly RED='\033[0;31m' # error + readonly GREEN='\033[0;32m' # info + readonly BLUE='\033[0;34m' # debug + else + readonly NC='' + readonly RED='' + readonly GREEN='' + readonly BLUE='' + fi + + return 0 +} + +stdlib::init_directories() { + if ! [[ -e ${VARDIR} ]]; then + install -d -m 0755 -o 0 -g 0 "${VARDIR}" + fi +} + +## +# Get a metadata key. When used without -o, this function is guaranteed to +# produce no output on STDOUT other than the retrieved value. This is intended +# to support the use case of +# FOO="$(stdlib::metadata_get -k instance/attributes/foo)" +# +# If the requested key does not exist, the error code will be 22 and zero bytes +# written to STDOUT. +stdlib::metadata_get() { + local OPTIND opt key outfile + local metadata="${METADATA_BASE%/}/computeMetadata/v1" + local exit_code + while getopts ":k:o:" opt; do + case "${opt}" in + k) key="${OPTARG}" ;; + o) outfile="${OPTARG}" ;; + :) + stdlib::error "Invalid option: -${OPTARG} requires an argument" + stdlib::metadata_get_usage + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error "Unknown option: -${opt}" + stdlib::metadata_get_usage + return "${E_UNKNOWN_ARG}" + ;; + esac + done + local url="${metadata}/${key#/}" + + stdlib::debug "Getting metadata resource url=${url}" + if [[ -z ${outfile:-} ]]; then + curl --location --silent --connect-timeout 1 --fail \ + -H 'Metadata-Flavor: Google' "$url" 2>/dev/null + exit_code=$? + else + stdlib::cmd curl --location \ + --silent \ + --connect-timeout 1 \ + --fail \ + --output "${outfile}" \ + -H 'Metadata-Flavor: Google' \ + "$url" + exit_code=$? + fi + case "${exit_code}" in + 22 | 37) + stdlib::debug "curl exit_code=${exit_code} for url=${url}" \ + "(Does not exist)" + ;; + esac + return "${exit_code}" +} + +stdlib::metadata_get_usage() { + stdlib::info 'Usage: stdlib::metadata_get -k ' + stdlib::info 'For example: stdlib::metadata_get -k instance/attributes/startup-config' +} + +# Load configuration values in the spirit of /etc/sysconfig defaults, but from +# metadata instead of the filesystem. +stdlib::load_config_values() { + local config_file + local key="instance/attributes/startup-script-config" + # shellcheck disable=SC2119 + config_file="$(stdlib::mktemp)" + stdlib::metadata_get -k "${key}" -o "${config_file}" + local status=$? + case "$status" in + 0) + stdlib::debug "SUCCESS: Configuration data sourced from $key" + ;; + 22 | 37) + stdlib::debug "no configuration data loaded from $key" + ;; + *) + stdlib::error "metadata_get -k $key returned unknown status=${status}" + ;; + esac + # shellcheck source=/dev/null + source "${config_file}" +} + +# Run a command logging the entry and exit. Intended for system level commands +# and operational debugging. Not intended for use with redirection. This is +# not named run() because bats uses a run() function. +stdlib::cmd() { + local exit_code argv=("$@") + stdlib::debug "BEGIN: stdlib::cmd() command=[${argv[*]}]" + "${argv[@]}" + exit_code=$? + stdlib::debug "END: stdlib::cmd() command=[${argv[*]}] exit_code=${exit_code}" + return $exit_code +} + +# Run a command successfully or exit the program with an error. +stdlib::run_or_die() { + if ! stdlib::cmd "$@"; then + stdlib::error "stdlib::run_or_die(): exiting with exit code ${E_RUN_OR_DIE}." + exit "${E_RUN_OR_DIE}" + fi +} + +# Intended to take advantage of automatic cleanup of startup script library +# temporary files without exporting a modified TMPDIR to child processes, which +# would cause the children to have their TMPDIR deleted out from under them. +# shellcheck disable=SC2120 +stdlib::mktemp() { + TMPDIR="${DELETE_AT_EXIT:-${TMPDIR}}" mktemp "$@" +} + +# Return a nice error message if a mandatory argument is missing. +stdlib::mandatory_argument() { + local OPTIND opt name flag + while getopts ":n:f:" opt; do + case "$opt" in + n) name="${OPTARG}" ;; + f) flag="${OPTARG}" ;; + :) + stdlib::error "Invalid argument: -${OPTARG} requires an argument to stdlib::mandatory_argument()" + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error "Unknown argument: -${OPTARG}" + stdlib::info "Usage: stdlib::mandatory_argument -n -f " + return "${E_UNKNOWN_ARG}" + ;; + esac + done + stdlib::error "Invalid argument: -${flag} requires an argument to ${name}()." +} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/main.tf b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/main.tf new file mode 100644 index 0000000000..02124eeddc --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/main.tf @@ -0,0 +1,306 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "startup-script", ghpc_role = "scripts" }) +} + +locals { + monitoring_agent_installer = ( + var.install_cloud_ops_agent || var.install_stackdriver_agent ? + [{ + type = "shell" + source = "${path.module}/files/install_monitoring_agent.sh" + destination = "install_monitoring_agent_automatic.sh" + args = var.install_cloud_ops_agent ? "ops" : "legacy" # install legacy (stackdriver) + }] : + [] + ) + + warnings = [ + { + type = "data" + content = file("${path.module}/files/running-script-warning.sh") + destination = "/etc/profile.d/99-running-script-warning.sh" + } + ] + + configure_ssh = length(var.configure_ssh_host_patterns) > 0 + host_args = { + host_name_prefix = var.configure_ssh_host_patterns + } + + prefix_file = "/tmp/prefix_file.json" + ansible_docker_settings_file = "/tmp/ansible_docker_settings.json" + + docker_config = try(jsondecode(var.docker.daemon_config), {}) + docker_data_root = try(local.docker_config.data-root, null) + + configure_ssh_runners = local.configure_ssh ? [ + { + type = "data" + source = "${path.module}/files/setup-ssh-keys.sh" + destination = "/usr/local/ghpc/setup-ssh-keys.sh" + }, + { + type = "data" + source = "${path.module}/files/setup-ssh-keys.yml" + destination = "/usr/local/ghpc/setup-ssh-keys.yml" + }, + { + type = "data" + content = jsonencode(local.host_args) + destination = local.prefix_file + }, + { + type = "ansible-local" + content = file("${path.module}/files/configure-ssh.yml") + destination = "configure-ssh.yml" + args = "-e @${local.prefix_file}" + } + ] : [] + + proxy_runner = var.http_proxy == "" ? [] : [ + { + type = "data" + destination = "/etc/profile.d/http_proxy.sh" + content = <<-EOT + #!/bin/bash + export http_proxy=${var.http_proxy} + export https_proxy=${var.http_proxy} + export NO_PROXY=${var.http_no_proxy} + EOT + }, + { + type = "shell" + source = "${path.module}/files/configure_proxy.sh" + destination = "configure_proxy.sh" + args = var.http_proxy + } + ] + + ofi_runner = !var.set_ofi_cloud_rdma_tunables ? [] : [ + { + type = "data" + destination = "/etc/profile.d/set_ofi_cloud_rdma_tunables.sh" + content = <<-EOT + #!/bin/bash + export FI_PROVIDER="verbs;ofi_rxm" + export FI_OFI_RXM_USE_RNDV_WRITE=0 + export FI_VERBS_INLINE_SIZE=39 + export I_MPI_FABRICS="shm:ofi" + export FI_UNIVERSE_SIZE=1024 + export I_MPI_ADJUST_ALLTOALL=1 + export I_MPI_ADJUST_IALLTOALL=1 + export I_MPI_ADJUST_BCAST=4 + export I_MPI_ADJUST_IBCAST=1 + EOT + }, + ] + + rdma_runner = !var.install_cloud_rdma_drivers ? [] : [ + { + type = "shell" + source = "${path.module}/files/install_cloud_rdma_drivers.sh" + destination = "install_cloud_rdma_drivers.sh" + } + ] + + docker_runner = !var.docker.enabled ? [] : [ + { + type = "data" + destination = local.ansible_docker_settings_file + content = jsonencode({ + enable_docker_world_writable = var.docker.world_writable + docker_daemon_config = var.docker.daemon_config + docker_data_root = local.docker_data_root + }) + }, + { + type = "ansible-local" + destination = "install_docker.yml" + content = file("${path.module}/files/install_docker.yml") + args = "-e \"@${local.ansible_docker_settings_file}\"" + }, + ] + + managed_lustre_runner = !var.managed_lustre.enabled ? [] : [ + { + type = "ansible-local" + destination = "install_managed_lustre.yml" + content = file("${path.module}/files/install_managed_lustre.yml") + args = "-e managed_lustre_port=${var.managed_lustre.port}" + }, + ] + + gpu_network_wait_online_runner = !var.enable_gpu_network_wait_online ? [] : [ + { + type = "ansible-local" + destination = "install_gpu_network_wait_online.yml" + content = file("${path.module}/files/install_gpu_network_wait_online.yml") + args = "" + }, + ] + + local_ssd_filesystem_enabled = can(coalesce(var.local_ssd_filesystem.mountpoint)) + raid_setup = !local.local_ssd_filesystem_enabled ? [] : [ + { + type = "ansible-local" + destination = "setup-raid.yml" + content = file("${path.module}/files/setup-raid.yml") + args = join(" ", [ + "-e mountpoint=${var.local_ssd_filesystem.mountpoint}", + "-e fs_type=${var.local_ssd_filesystem.fs_type}", + "-e mode=${var.local_ssd_filesystem.permissions}", + ]) + }, + ] + + supplied_ansible_runners = anytrue([for r in var.runners : r.type == "ansible-local"]) + has_ansible_runners = anytrue([ + local.supplied_ansible_runners, + local.configure_ssh, + var.docker.enabled, + var.managed_lustre.enabled, + var.enable_gpu_network_wait_online, + local.local_ssd_filesystem_enabled + ]) + + install_ansible = coalesce(var.install_ansible, local.has_ansible_runners) + ansible_installer = local.install_ansible ? [{ + type = "shell" + source = "${path.module}/files/install_ansible.sh" + destination = "install_ansible_automatic.sh" + args = var.ansible_virtualenv_path + }] : [] + + hotfix_runner = [{ + type = "shell" + source = "${path.module}/files/early_run_hotfixes.sh" + destination = "early_run_hotfixes.sh" + }] + + runners = concat( + local.warnings, + local.hotfix_runner, + local.proxy_runner, + local.ofi_runner, + local.rdma_runner, + local.monitoring_agent_installer, + local.ansible_installer, + local.raid_setup, # order RAID early to ensure filesystem is ready for subsequent runners + local.managed_lustre_runner, + local.configure_ssh_runners, + local.docker_runner, + local.gpu_network_wait_online_runner, + var.runners + ) + + bucket_regex = "^gs://([^/]*)/*(.*)" + gcs_bucket_path_trimmed = var.gcs_bucket_path == null ? null : trimsuffix(var.gcs_bucket_path, "/") + storage_folder_path = local.gcs_bucket_path_trimmed == null ? null : regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[1] + storage_folder_path_prefix = local.storage_folder_path == null || local.storage_folder_path == "" ? "" : "${local.storage_folder_path}/" + + user_provided_bucket_name = try(regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[0], null) + storage_bucket_name = coalesce(one(google_storage_bucket.configs_bucket[*].name), local.user_provided_bucket_name) + + load_runners = templatefile( + "${path.module}/templates/startup-script-custom.tftpl", + { + bucket = local.storage_bucket_name, + http_proxy = var.http_proxy, + no_proxy = var.http_no_proxy, + runners = [ + for runner in local.runners : { + object = google_storage_bucket_object.scripts[basename(runner["destination"])].output_name + type = runner["type"] + destination = runner["destination"] + args = contains(keys(runner), "args") ? runner["args"] : "" + } + ] + } + ) + + stdlib_head = file("${path.module}/files/startup-script-stdlib-head.sh") + get_from_bucket = file("${path.module}/files/get_from_bucket.sh") + stdlib_body = file("${path.module}/files/startup-script-stdlib-body.sh") + + # List representing complete content, to be concatenated together. + stdlib_list = [ + local.stdlib_head, + local.get_from_bucket, + local.load_runners, + local.stdlib_body, + ] + + # Final content output to the user + stdlib = join("", local.stdlib_list) + + runners_map = { for runner in local.runners : + basename(runner["destination"]) => { + content = lookup(runner, "content", null) + source = lookup(runner, "source", null) + } + } +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket" "configs_bucket" { + count = var.gcs_bucket_path == null ? 1 : 0 + project = var.project_id + name = "${var.deployment_name}-startup-scripts-${random_id.resource_name_suffix.hex}" + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +resource "google_storage_bucket_iam_binding" "viewers" { + bucket = local.storage_bucket_name + role = "roles/storage.objectViewer" + members = var.bucket_viewers +} + +resource "google_storage_bucket_object" "scripts" { + # this writes all scripts exactly once into GCS + for_each = local.runners_map + name = "${local.storage_folder_path_prefix}${each.key}-${substr(try(md5(each.value.content), filemd5(each.value.source)), 0, 4)}" + content = each.value.content + source = each.value.source + source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) + bucket = local.storage_bucket_name + timeouts { + create = "10m" + update = "10m" + } + + lifecycle { + precondition { + condition = !(var.install_cloud_ops_agent && var.install_stackdriver_agent) + error_message = "Only one of var.install_stackdriver_agent or var.install_cloud_ops_agent can be set. Stackdriver is recommended for best performance." + } + } +} + +resource "local_file" "debug_file" { + for_each = toset(var.debug_file != null ? [var.debug_file] : []) + filename = var.debug_file + content = local.stdlib +} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml new file mode 100644 index 0000000000..2ada34471f --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/outputs.tf b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/outputs.tf new file mode 100644 index 0000000000..6a15082814 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/outputs.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "script to load and run all runners, as a string value." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} + +output "compute_startup_script" { + description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} + +output "controller_startup_script" { + description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl new file mode 100644 index 0000000000..3c894b00b0 --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl @@ -0,0 +1,65 @@ + + +stdlib::run_playbook() { + if [ ! "$(which ansible-playbook)" ]; then + stdlib::error "ansible-playbook not found"\ + "Please install ansible before running ansible-local runners." + exit 1 + fi + ansible-playbook --connection=local --inventory=localhost, --limit localhost $1 $2 + ret_code=$? + return $${ret_code} +} + +stdlib::runner() { + + type=$1 + object=$2 + destination=$3 + tmpdir=$4 + args=$5 + + destpath="$(dirname $destination)" + filename="$(basename $destination)" + + if [ "$destpath" = "." ]; then + destpath=$tmpdir + fi + + stdlib::get_from_bucket -u "gs://${bucket}/$object" -d "$destpath" -f "$filename" + + stdlib::info "=== start executing runner: $object ===" + case "$1" in + ansible-local) stdlib::run_playbook "$destpath/$filename" "$args";; + shell) chmod u+x /$destpath/$filename && $destpath/$filename $args;; + esac + + exit_code=$? + stdlib::info "=== $object finished with exit_code=$exit_code ===" + if [ "$exit_code" -ne "0" ] ; then + stdlib::error "=== execution of $object failed, exiting ===" + stdlib::announce_runners_end "$exit_code" + exit $exit_code + fi +} + +stdlib::load_runners(){ + tmpdir="$(mktemp -d)" + + stdlib::debug "=== BEGIN Running runners ===" + stdlib::announce_runners_start + + %{if http_proxy != "" ~} + stdlib::info "=== Setting HTTP_PROXY,HTTPS_PROXY to ${http_proxy} ===" + export http_proxy=${http_proxy} + export https_proxy=${http_proxy} + export NO_PROXY=${no_proxy} + %{endif ~} + + %{for r in runners ~} + stdlib::runner "${r.type}" "${r.object}" "${r.destination}" $${tmpdir} "${r.args}" + %{endfor ~} + + stdlib::announce_runners_end "0" + stdlib::debug "=== END Running runners ===" +} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/variables.tf b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/variables.tf new file mode 100644 index 0000000000..7080085ece --- /dev/null +++ b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/variables.tf @@ -0,0 +1,298 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used to name GCS bucket for startup scripts." + type = string +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string + default = null +} + +variable "bucket_viewers" { + description = "Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account)" + type = list(string) + default = [] + + validation { + condition = alltrue([ + for u in var.bucket_viewers : length(regexall("^(allUsers$|allAuthenticatedUsers$|user:|group:|serviceAccount:|domain:)", u)) > 0 + ]) + error_message = "Bucket viewer members must begin with user/group/serviceAccount/domain following https://cloud.google.com/iam/docs/reference/rest/v1/Policy#Binding" + } +} + +variable "debug_file" { + description = "Path to an optional local to be written with 'startup_script'." + type = string + default = null +} + +variable "labels" { + description = "Labels for the created GCS bucket. Key-value pairs." + type = map(string) +} + +variable "runners" { + description = < 0 + error_message = "The POSIX permissions for the mountpoint must be represented as a 3 or 4-digit octal" + } + + default = { + fs_type = "ext4" + mountpoint = "" + permissions = "0755" + } + + nullable = false +} + +variable "install_cloud_ops_agent" { + description = "Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true." + type = bool + default = false +} + +variable "install_stackdriver_agent" { + description = "Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance." + type = bool + default = false +} + +variable "install_ansible" { + description = "Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used." + type = bool + default = null +} + +variable "configure_ssh_host_patterns" { + description = < **_NOTE:_** if both [startup_script][sss] and [startup_script_file][ssf] are +> specified, then [startup_script_file][ssf] takes precedence. + +## Recommended use + +Because the [metadata startup script executes in parallel](#order-of-execution) +with the other solutions, conflicts can arise, especially when package managers +(`yum` or `apt`) lock their databases during package installation. Therefore, it +is recommended to choose one of the following approaches: + +1. Specify _either_ [startup_script][sss] _or_ [startup_script_file][ssf] and do + not specify [shell_scripts][shell] or [ansible_playbooks][ansible]. + - This can be especially useful in + [environments that restrict SSH access](#environments-without-ssh-access) +1. Specify any combination of [shell_scripts][shell] and + [ansible_playbooks][ansible] and do not specify [startup_script][sss] or + [startup_script_file][ssf]. + +If any of the startup script approaches fail by returning a code other than 0, +Packer will determine that the build has failed and refuse to save the image. + +## External access with SSH + +The [shell scripts][shell] and [Ansible playbooks][ansible] customization +solutions both require SSH access to the VM from the Packer execution +environment. SSH access can be enabled one of 2 ways: + +1. The VM is created without a public IP address and SSH tunnels are created + using [Identity-Aware Proxy (IAP)][iaptunnel]. + - Allow [use_iap](#input_use_iap) to take on its default value of `true` +1. The VM is created with an IP address on the public internet and firewall + rules allow SSH access from the Packer execution environment. + - Set `omit_external_ip = false` (or `omit_external_ip: false` in a + blueprint) + - Add firewall rules that open SSH to the VM + +The Packer template defaults to using to the 1st IAP-based solution because it +is more secure (no exposure to public internet) and because the [vpc] module +automatically sets up all necessary firewall rules for SSH tunneling and +outbound-only access to the internet through [Cloud NAT][cloudnat]. + +In either SSH solution, customization scripts should be supplied as files in the +[shell_scripts][shell] and [ansible_playbooks][ansible] settings. + +## Environments without SSH access + +Many network environments disallow SSH access to VMs. In these environments, the +[metadata-based startup scripts][startup-metadata] are appropriate because they +execute entirely independently of the Packer execution environment. + +In this scenario, a single scripts should be supplied in the form of a string to +the [startup_script][sss] input variable. This solution integrates well with +Toolkit runners. Runners operate by using a single startup script whose behavior +is extended by downloading and executing a customizable set of runners from +Cloud Storage at startup. + +> **_NOTE:_** Packer will attempt to use SSH if either [shell_scripts][shell] or +> [ansible_playbooks][ansible] are set to non-empty values. Leave them at their +> default, empty values to ensure access by SSH is disabled. + +## Supplying startup script as a string + +The [startup_script][sss] parameter accepts scripts formatted as strings. In +Packer and Terraform, multi-line strings can be specified using +[heredoc syntax](https://www.terraform.io/language/expressions/strings#heredoc-strings) +in an input [Packer variables file][pkrvars] (`*.pkrvars.hcl`) For example, the +following snippet defines a multi-line bash script followed by an integer +representing the size, in GiB, of the resulting image: + +```hcl +startup_script = <<-EOT + #!/bin/bash + yum install -y epel-release + yum install -y jq + EOT + +disk_size = 100 +``` + +In a blueprint, the equivalent syntax is: + +```yaml +... + settings: + startup_script: | + #!/bin/bash + yum install -y epel-release + yum install -y jq + disk_size: 100 +... +``` + +## Monitoring startup script execution + +When using startup script customization, Packer will print very limited output +to the console. For example: + +```text +==> example.googlecompute.toolkit_image: Waiting for any running startup script to finish... +==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... +==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... +==> example.googlecompute.toolkit_image: Startup script, if any, has finished running. +``` + +### Debugging startup-script failures + +> [!NOTE] +> There can be a delay in the propagation of the logs from the instance to +> Cloud Logging, so it may require waiting a few minutes to see the full logs. + +If the Packer image build fails, the module will output a `gcloud` command +that can be used directly to review startup-script execution. + +## License + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + +```text + http://www.apache.org/licenses/LICENSE-2.0 +``` + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_count](#input\_accelerator\_count) | Number of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `number` | `null` | no | +| [accelerator\_type](#input\_accelerator\_type) | Type of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `string` | `null` | no | +| [ansible\_playbooks](#input\_ansible\_playbooks) | A list of Ansible playbook configurations that will be uploaded to customize the VM image | 
list(object({
    playbook_file   = string
    galaxy_file     = string
    extra_arguments = list(string)
  }))
| `[]` | no | +| [communicator](#input\_communicator) | Communicator to use for provisioners that require access to VM ("ssh" or "winrm") | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name | `string` | n/a | yes | +| [disk\_size](#input\_disk\_size) | Size of disk image in GB | `number` | `null` | no | +| [disk\_type](#input\_disk\_type) | Type of persistent disk to provision | `string` | `"pd-balanced"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [image\_family](#input\_image\_family) | The family name of the image to be built. Defaults to `deployment_name` | `string` | `null` | no | +| [image\_name](#input\_image\_name) | The name of the image to be built. If not supplied, it will be set to image\_family-$ISO\_TIMESTAMP | `string` | `null` | no | +| [image\_storage\_locations](#input\_image\_storage\_locations) | Storage location, either regional or multi-regional, where snapshot content is to be stored and only accepts 1 value.
See https://developer.hashicorp.com/packer/plugins/builders/googlecompute#image_storage_locations | `list(string)` | `null` | no | +| [labels](#input\_labels) | Labels to apply to the short-lived VM | `map(string)` | `null` | no | +| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2-standard-4"` | no | +| [manifest\_file](#input\_manifest\_file) | File to which to write Packer build manifest | `string` | `"packer-manifest.json"` | no | +| [metadata](#input\_metadata) | Instance metadata for the builder VM (use var.startup\_script or var.startup\_script\_file to set startup-script metadata) | `map(string)` | `{}` | no | +| [network\_project\_id](#input\_network\_project\_id) | Project ID of Shared VPC network | `string` | `null` | no | +| [omit\_external\_ip](#input\_omit\_external\_ip) | Provision the image building VM without a public IP address | `bool` | `true` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which to create VM and image | `string` | n/a | yes | +| [scopes](#input\_scopes) | DEPRECATED: use var.service\_account\_scopes | `set(string)` | `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | The service account email to use. If null or 'default', then the default Compute Engine service account will be used. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Service account scopes to attach to the instance. See
https://cloud.google.com/compute/docs/access/service-accounts. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shell\_scripts](#input\_shell\_scripts) | A list of paths to local shell scripts which will be uploaded to customize the VM image | `list(string)` | `[]` | no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [source\_image](#input\_source\_image) | Source OS image to build from | `string` | `null` | no | +| [source\_image\_family](#input\_source\_image\_family) | Alternative to source\_image. Specify image family to build from latest image in family | `string` | `"hpc-rocky-linux-8"` | no | +| [source\_image\_project\_id](#input\_source\_image\_project\_id) | A list of project IDs to search for the source image. Packer will search the
first project ID in the list first, and fall back to the next in the list,
until it finds the source image. | `list(string)` | `null` | no | +| [ssh\_username](#input\_ssh\_username) | Username to use for SSH access to VM | `string` | `"hpc-toolkit-packer"` | no | +| [startup\_script](#input\_startup\_script) | Startup script (as raw string) used to build the custom Linux VM image (overridden by var.startup\_script\_file if both are set) | `string` | `null` | no | +| [startup\_script\_file](#input\_startup\_script\_file) | File path to local shell script that will be used to customize the Linux VM image (overrides var.startup\_script) | `string` | `null` | no | +| [state\_timeout](#input\_state\_timeout) | The time to wait for instance state changes, including image creation | `string` | `"10m"` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of subnetwork in which to provision image building VM | `string` | n/a | yes | +| [tags](#input\_tags) | Assign network tags to apply firewall rules to VM instance | `list(string)` | `null` | no | +| [use\_iap](#input\_use\_iap) | Use IAP proxy when connecting by SSH | `bool` | `true` | no | +| [use\_os\_login](#input\_use\_os\_login) | Use OS Login when connecting by SSH | `bool` | `false` | no | +| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | A list of strings containing PowerShell scripts which will customize a Windows VM image (requires WinRM communicator) | `list(string)` | `[]` | no | +| [wrap\_startup\_script](#input\_wrap\_startup\_script) | Wrap startup script with Packer-generated wrapper | `bool` | `true` | no | +| [zone](#input\_zone) | Cloud zone in which to provision image building VM | `string` | n/a | yes | + +## Outputs + +No outputs. + + +[ansible]: #input_ansible_playbooks +[cloudnat]: https://cloud.google.com/nat/docs/overview +[examples readme]: ../../../examples/README.md#image-builderyaml- +[hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[iamprop]: https://cloud.google.com/iam/docs/access-change-propagation +[iaptunnel]: https://cloud.google.com/iap/docs/using-tcp-forwarding +[image builder]: ../../../examples/image-builder.yaml +[logging-console]: https://console.cloud.google.com/logs/ +[logging-read-docs]: https://cloud.google.com/sdk/gcloud/reference/logging/read +[pkrvars]: https://www.packer.io/guides/hcl/variables#from-a-file +[shell]: #input_shell_scripts +[ssf]: #input_startup_script_file +[sss]: #input_startup_script +[startup-metadata]: https://cloud.google.com/compute/docs/instances/startup-scripts/linux +[startup-script]: ../../../modules/scripts/startup-script +[vpc]: ../../network/vpc/README.md diff --git a/deletion-test/slurm-build/slurm-image/image.pkr.hcl b/deletion-test/slurm-build/slurm-image/image.pkr.hcl new file mode 100644 index 0000000000..9282cf7433 --- /dev/null +++ b/deletion-test/slurm-build/slurm-image/image.pkr.hcl @@ -0,0 +1,216 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "custom-image", ghpc_role = "packer" }) + + # construct a unique image name from the image family + image_family = var.image_family != null ? var.image_family : var.deployment_name + image_name_default = "${local.image_family}-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" + image_name = var.image_name != null ? var.image_name : local.image_name_default + + # construct vm image name for use when getting logs + instance_name = "packer-${substr(uuidv4(), 0, 6)}" + + # default to explicit var.communicator, otherwise in-order: ssh/winrm/none + shell_script_communicator = length(var.shell_scripts) > 0 ? "ssh" : "" + ansible_playbook_communicator = length(var.ansible_playbooks) > 0 ? "ssh" : "" + powershell_script_communicator = length(var.windows_startup_ps1) > 0 ? "winrm" : "" + communicator = coalesce( + var.communicator, + local.shell_script_communicator, + local.ansible_playbook_communicator, + local.powershell_script_communicator, + "none" + ) + + # must not enable IAP when no communicator is in use + use_iap = local.communicator == "none" ? false : var.use_iap + + # construct metadata from startup_script and metadata variables + startup_script_metadata = var.startup_script == null ? {} : { startup-script = var.startup_script } + + linux_user_metadata = { + block-project-ssh-keys = "TRUE" + shutdown-script = <<-EOT + #!/bin/bash + userdel -r ${var.ssh_username} + sed -i '/${var.ssh_username}/d' /var/lib/google/google_users + EOT + } + windows_packer_user = "packer_user" + windows_user_metadata = { + sysprep-specialize-script-cmd = "winrm quickconfig -quiet & net user /add ${local.windows_packer_user} & net localgroup administrators ${local.windows_packer_user} /add & winrm set winrm/config/service/auth @{Basic=\\\"true\\\"}" + windows-shutdown-script-cmd = <<-EOT + net user /delete ${local.windows_packer_user} + EOT + } + user_metadata = local.communicator == "winrm" ? local.windows_user_metadata : local.linux_user_metadata + + # merge metadata such that var.metadata always overrides user management + # metadata but always allow var.startup_script to override var.metadata + metadata = merge( + local.user_metadata, + var.metadata, + local.startup_script_metadata, + ) + + # determine best value for on_host_maintenance if not supplied by user + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + gpu_attached = contains(["a2", "g2"], local.machine_family) || var.accelerator_type != null + on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" + on_host_maintenance = ( + var.on_host_maintenance != null + ? var.on_host_maintenance + : local.on_host_maintenance_default + ) + + accelerator_type = var.accelerator_type == null ? null : "projects/${var.project_id}/zones/${var.zone}/acceleratorTypes/${var.accelerator_type}" + + winrm_username = local.communicator == "winrm" ? "packer_user" : null + winrm_insecure = local.communicator == "winrm" ? true : null + winrm_use_ssl = local.communicator == "winrm" ? true : null + + enable_integrity_monitoring = var.enable_shielded_vm && var.shielded_instance_config.enable_integrity_monitoring + enable_secure_boot = var.enable_shielded_vm && var.shielded_instance_config.enable_secure_boot + enable_vtpm = var.enable_shielded_vm && var.shielded_instance_config.enable_vtpm + + image_licenses = [ + "projects/click-to-deploy-images/global/licenses/hpc-toolkit-vm-image" + ] +} + +source "googlecompute" "toolkit_image" { + communicator = local.communicator + project_id = var.project_id + image_name = local.image_name + image_family = local.image_family + image_labels = local.labels + instance_name = local.instance_name + machine_type = var.machine_type + accelerator_type = local.accelerator_type + accelerator_count = var.accelerator_count + on_host_maintenance = local.on_host_maintenance + disk_size = var.disk_size + disk_type = var.disk_type + omit_external_ip = var.omit_external_ip + use_internal_ip = var.omit_external_ip + subnetwork = var.subnetwork_name + network_project_id = var.network_project_id + service_account_email = var.service_account_email + scopes = var.service_account_scopes + source_image = var.source_image + source_image_family = var.source_image_family + source_image_project_id = var.source_image_project_id + ssh_username = var.ssh_username + tags = var.tags + use_iap = local.use_iap + use_os_login = var.use_os_login + winrm_username = local.winrm_username + winrm_insecure = local.winrm_insecure + winrm_use_ssl = local.winrm_use_ssl + zone = var.zone + labels = local.labels + metadata = local.metadata + startup_script_file = var.startup_script_file + wrap_startup_script = var.wrap_startup_script + state_timeout = var.state_timeout + image_storage_locations = var.image_storage_locations + enable_secure_boot = local.enable_secure_boot + enable_vtpm = local.enable_vtpm + enable_integrity_monitoring = local.enable_integrity_monitoring + image_licenses = local.image_licenses +} + +build { + name = var.deployment_name + sources = ["sources.googlecompute.toolkit_image"] + + # using dynamic blocks to create provisioners ensures that there are no + # provisioner blocks when none are provided and we can use the none + # communicator when using startup-script + + # provisioner "shell" blocks + dynamic "provisioner" { + labels = ["shell"] + for_each = var.shell_scripts + content { + execute_command = "sudo -H sh -c '{{ .Vars }} {{ .Path }}'" + script = provisioner.value + } + } + + # provisioner "powershell" blocks + dynamic "provisioner" { + labels = ["powershell"] + for_each = var.windows_startup_ps1 + content { + inline = split("\n", provisioner.value) + } + } + + dynamic "provisioner" { + labels = ["powershell"] + for_each = length(var.windows_startup_ps1) > 0 ? [1] : [] + content { + inline = [ + "GCESysprep -no_shutdown" + ] + } + } + + # provisioner "ansible-local" blocks + # this installs custom roles/collections from ansible-galaxy in /home/packer + # which will be removed at the end; consider modifying /etc/ansible/ansible.cfg + dynamic "provisioner" { + labels = ["ansible-local"] + for_each = var.ansible_playbooks + content { + playbook_file = provisioner.value.playbook_file + galaxy_file = provisioner.value.galaxy_file + extra_arguments = provisioner.value.extra_arguments + } + } + + post-processor "manifest" { + output = var.manifest_file + strip_path = true + custom_data = { + built-by = "cloud-hpc-toolkit" + } + } + + # If there is an error during image creation, print out command for getting packer VM logs + error-cleanup-provisioner "shell-local" { + environment_vars = [ + "PRJ_ID=${var.project_id}", + "INST_NAME=${local.instance_name}", + "ZONE=${var.zone}", + ] + inline_shebang = "/bin/bash -e" + inline = [ + "type -P gcloud > /dev/null || exit 0", + "INST_ID=$(gcloud compute instances describe $INST_NAME --project $PRJ_ID --format=\"value(id)\" --zone=$ZONE)", + "echo 'Error building image try checking logs:'", + join(" ", ["echo \"gcloud logging --project $PRJ_ID read", + "'logName=(\\\"projects/$PRJ_ID/logs/GCEMetadataScripts\\\" OR \\\"projects/$PRJ_ID/logs/google_metadata_script_runner\\\") AND resource.labels.instance_id=$INST_ID'", + "--format=\\\"table(timestamp, resource.labels.instance_id, jsonPayload.message)\\\"", + "--order=asc\"" + ] + ) + ] + } +} diff --git a/deletion-test/slurm-build/slurm-image/metadata.yaml b/deletion-test/slurm-build/slurm-image/metadata.yaml new file mode 100644 index 0000000000..23108c4e17 --- /dev/null +++ b/deletion-test/slurm-build/slurm-image/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - logging.googleapis.com + - storage.googleapis.com diff --git a/deletion-test/slurm-build/slurm-image/variables.pkr.hcl b/deletion-test/slurm-build/slurm-image/variables.pkr.hcl new file mode 100644 index 0000000000..3cede102ce --- /dev/null +++ b/deletion-test/slurm-build/slurm-image/variables.pkr.hcl @@ -0,0 +1,276 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "deployment_name" { + description = "Cluster Toolkit deployment name" + type = string +} + +variable "project_id" { + description = "Project in which to create VM and image" + type = string +} + +variable "machine_type" { + description = "VM machine type on which to build new image" + type = string + default = "n2-standard-4" +} + +variable "disk_size" { + description = "Size of disk image in GB" + type = number + default = null +} + +variable "disk_type" { + description = "Type of persistent disk to provision" + type = string + default = "pd-balanced" +} + +variable "zone" { + description = "Cloud zone in which to provision image building VM" + type = string +} + +variable "network_project_id" { + description = "Project ID of Shared VPC network" + type = string + default = null +} + +variable "subnetwork_name" { + description = "Name of subnetwork in which to provision image building VM" + type = string +} + +variable "omit_external_ip" { + description = "Provision the image building VM without a public IP address" + type = bool + default = true +} + +variable "tags" { + description = "Assign network tags to apply firewall rules to VM instance" + type = list(string) + default = null +} + +variable "image_family" { + description = "The family name of the image to be built. Defaults to `deployment_name`" + type = string + default = null +} + +variable "image_name" { + description = "The name of the image to be built. If not supplied, it will be set to image_family-$ISO_TIMESTAMP" + type = string + default = null +} + +variable "source_image_project_id" { + description = < Date: Tue, 9 Dec 2025 19:05:46 +0000 Subject: [PATCH 07/19] to test labels --- tools/cloud-build/project-cleanup.yaml | 2 +- tools/exclusions.txt | 47 ++------------------------ 2 files changed, 3 insertions(+), 46 deletions(-) diff --git a/tools/cloud-build/project-cleanup.yaml b/tools/cloud-build/project-cleanup.yaml index 012da654fe..268b4b1353 100644 --- a/tools/cloud-build/project-cleanup.yaml +++ b/tools/cloud-build/project-cleanup.yaml @@ -32,7 +32,7 @@ steps: apt-get update -y && apt-get install -y jq # Set time variables - export CUTOFF_TIME=$(date -d '5 hours ago' -u +%Y-%m-%dT%H:%M:%S%z) + export CUTOFF_TIME=$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%S%z) export CUTOFF_TIME_IMAGES=$(date -d "60 days ago" -u +%Y-%m-%dT%H:%M:%S%z) attempt=1 diff --git a/tools/exclusions.txt b/tools/exclusions.txt index 263dd1d856..cbfd136504 100644 --- a/tools/exclusions.txt +++ b/tools/exclusions.txt @@ -1,45 +1,2 @@ -vertexui-do-not-kill -hpc-ctk1357 -hpc-toolkit-dev@appspot.gserviceaccount.com -build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com -cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com -cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com -cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com -508417052821-compute@developer.gserviceaccount.com -hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com -hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com -htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com -htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com -htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com -pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com -test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com -telemetry@hpc-toolkit-dev.iam.gserviceaccount.com -telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com -telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com -vertexui-do-not-kill-boot -vertexui-do-not-kill-data -default-router-us-west1 -default-router-us-west4 -default-net-router -default-router-australia-southeast1 -default-router-us-east4 -image-inspector-550 -image-inspector -gke-managed-lustre-basic-net-fw-allow-iap-ingress -gke-managed-lustre-basic-net-fw-allow-internal-traffic -a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com -hpc-vpc -allow-internal -allow-ssh -a4high-image-builder-20250214t220935z -chs-dcgmi-metric-u22-20250925t121709z -common-slurm-image-20250725t234825z -pbspro0 -a3u-image-u22-20250325t162635z -harsh-a4-image -rocka4hf-rocky9-20250910t040750z -rocka4h-rocky9-20250908t175724z -slurm-gcp-next-hpc-rocky-linux-8-1739990978 -slurm-gcp-next-hpc-rocky-linux-8-1740100297 -welp-insta-temp -testing2 \ No newline at end of file +a3mega-a3meganodeset-0 +a3mega-a3meganodeset-1 \ No newline at end of file From 1e549f926a7708988e8f617da63e72dac568b045 Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Wed, 10 Dec 2025 14:30:22 +0000 Subject: [PATCH 08/19] To review --- .../artifacts/DO_NOT_MODIFY_THIS_DIRECTORY | 1 - .../.ghpc/artifacts/expanded_blueprint.yaml | 651 ----- deletion-test/.gitignore | 48 - deletion-test/build_script/main.tf | 86 - .../embedded/community/modules/README.md | 7 - .../modules/compute/gke-nodeset/README.md | 55 - .../modules/compute/gke-nodeset/main.tf | 64 - .../modules/compute/gke-nodeset/metadata.yaml | 20 - .../modules/compute/gke-nodeset/output.tf | 18 - .../compute/gke-nodeset/persistent_volumes.tf | 50 - .../templates/nodeset-general.yaml.tftpl | 203 -- .../modules/compute/gke-nodeset/variables.tf | 118 - .../modules/compute/gke-nodeset/versions.tf | 27 - .../modules/compute/gke-partition/README.md | 39 - .../modules/compute/gke-partition/main.tf | 47 - .../compute/gke-partition/metadata.yaml | 19 - .../compute/gke-partition/variables.tf | 43 - .../modules/compute/gke-partition/versions.tf | 27 - .../compute/htcondor-execute-point/README.md | 271 -- .../htcondor-execute-point/compute_image.tf | 30 - .../files/htcondor_configure.yml | 74 - .../files/htcondor_configure_autoscaler.yml | 98 - .../compute/htcondor-execute-point/main.tf | 218 -- .../htcondor-execute-point/metadata.yaml | 20 - .../compute/htcondor-execute-point/outputs.tf | 25 - .../templates/condor_config.tftpl | 31 - .../download-condor-config.ps1.tftpl | 34 - .../htcondor-execute-point/variables.tf | 265 -- .../htcondor-execute-point/versions.tf | 34 - .../community/modules/compute/mig/README.md | 45 - .../community/modules/compute/mig/main.tf | 85 - .../modules/compute/mig/metadata.yaml | 21 - .../community/modules/compute/mig/outputs.tf | 18 - .../modules/compute/mig/variables.tf | 86 - .../community/modules/compute/mig/versions.tf | 27 - .../modules/compute/notebook/README.md | 112 - .../modules/compute/notebook/main.tf | 96 - .../modules/compute/notebook/metadata.yaml | 20 - .../modules/compute/notebook/variables.tf | 111 - .../modules/compute/notebook/versions.tf | 29 - .../README.md | 135 - .../main.tf | 128 - .../metadata.yaml | 20 - .../outputs.tf | 36 - .../source_image_logic.tf | 30 - .../variables.tf | 402 --- .../versions.tf | 22 - .../README.md | 85 - .../schedmd-slurm-gcp-v6-nodeset-tpu/main.tf | 59 - .../metadata.yaml | 21 - .../outputs.tf | 39 - .../variables.tf | 171 -- .../versions.tf | 23 - .../schedmd-slurm-gcp-v6-nodeset/README.md | 227 -- .../schedmd-slurm-gcp-v6-nodeset/main.tf | 232 -- .../metadata.yaml | 21 - .../schedmd-slurm-gcp-v6-nodeset/outputs.tf | 112 - .../source_image_logic.tf | 30 - .../schedmd-slurm-gcp-v6-nodeset/variables.tf | 641 ----- .../schedmd-slurm-gcp-v6-nodeset/versions.tf | 29 - .../schedmd-slurm-gcp-v6-partition/README.md | 105 - .../schedmd-slurm-gcp-v6-partition/main.tf | 41 - .../metadata.yaml | 20 - .../schedmd-slurm-gcp-v6-partition/outputs.tf | 54 - .../variables.tf | 311 --- .../versions.tf | 23 - .../container/artifact-registry/README.md | 157 -- .../container/artifact-registry/main.tf | 268 -- .../container/artifact-registry/metadata.yaml | 21 - .../container/artifact-registry/outputs.tf | 18 - .../container/artifact-registry/validation.tf | 49 - .../container/artifact-registry/variables.tf | 122 - .../container/artifact-registry/versions.tf | 27 - .../database/bigquery-dataset/README.md | 76 - .../modules/database/bigquery-dataset/main.tf | 32 - .../database/bigquery-dataset/metadata.yaml | 19 - .../database/bigquery-dataset/outputs.tf | 20 - .../database/bigquery-dataset/variables.tf | 36 - .../database/bigquery-dataset/versions.tf | 29 - .../modules/database/bigquery-table/README.md | 87 - .../modules/database/bigquery-table/main.tf | 37 - .../database/bigquery-table/metadata.yaml | 19 - .../database/bigquery-table/outputs.tf | 28 - .../database/bigquery-table/variables.tf | 46 - .../database/bigquery-table/versions.tf | 29 - .../slurm-cloudsql-federation/README.md | 107 - .../slurm-cloudsql-federation/main.tf | 165 -- .../slurm-cloudsql-federation/metadata.yaml | 21 - .../slurm-cloudsql-federation/outputs.tf | 27 - .../slurm-cloudsql-federation/variables.tf | 173 -- .../slurm-cloudsql-federation/versions.tf | 36 - .../file-system/DDN-EXAScaler/README.md | 158 -- .../modules/file-system/DDN-EXAScaler/main.tf | 72 - .../file-system/DDN-EXAScaler/metadata.yaml | 22 - .../file-system/DDN-EXAScaler/outputs.tf | 90 - .../file-system/DDN-EXAScaler/variables.tf | 502 ---- .../file-system/DDN-EXAScaler/versions.tf | 24 - .../modules/file-system/Intel-DAOS/README.md | 1 - .../modules/file-system/nfs-server/README.md | 152 -- .../modules/file-system/nfs-server/main.tf | 131 - .../file-system/nfs-server/metadata.yaml | 19 - .../modules/file-system/nfs-server/outputs.tf | 53 - .../nfs-server/scripts/install-nfs-client.sh | 37 - .../scripts/install-nfs-server.sh.tpl | 35 - .../file-system/nfs-server/scripts/mount.sh | 58 - .../file-system/nfs-server/scripts/mount.yaml | 39 - .../file-system/nfs-server/variables.tf | 194 -- .../file-system/nfs-server/versions.tf | 37 - .../file-system/sycomp-scale/README.md | 35 - .../modules/file-system/weka-client/README.md | 182 -- .../file-system/weka-client/metadata.yaml | 18 - .../file-system/weka-client/outputs.tf | 71 - .../templates/install-weka-client.yaml.tftpl | 133 - .../weka-client/templates/mount-weka.sh.tftpl | 101 - .../templates/mount-weka.yaml.tftpl | 54 - .../file-system/weka-client/variables.tf | 39 - .../file-system/weka-client/versions.tf | 19 - .../FSI_MonteCarlo.ipynb | 125 - .../files/fsi-montecarlo-on-batch/README.md | 97 - .../fsi-montecarlo-on-batch/iteration.sh | 23 - .../files/fsi-montecarlo-on-batch/main.tf | 102 - .../fsi-montecarlo-on-batch/mc_run.tpl.py | 157 -- .../fsi-montecarlo-on-batch/mc_run.tpl.yaml | 36 - .../fsi-montecarlo-on-batch/mc_run_reqs.txt | 9 - .../fsi-montecarlo-on-batch/metadata.yaml | 18 - .../fsi-montecarlo-on-batch/variables.tf | 51 - .../files/fsi-montecarlo-on-batch/versions.tf | 43 - .../internal/slurm-gcp/instance/README.md | 100 - .../internal/slurm-gcp/instance/main.tf | 126 - .../internal/slurm-gcp/instance/outputs.tf | 41 - .../internal/slurm-gcp/instance/variables.tf | 119 - .../internal/slurm-gcp/instance/versions.tf | 31 - .../slurm-gcp/instance_template/README.md | 87 - .../files/startup_sh_unlinted | 169 -- .../slurm-gcp/instance_template/main.tf | 171 -- .../slurm-gcp/instance_template/outputs.tf | 43 - .../slurm-gcp/instance_template/variables.tf | 431 ---- .../slurm-gcp/instance_template/versions.tf | 25 - .../internal_instance_template/README.md | 89 - .../internal_instance_template/main.tf | 234 -- .../internal_instance_template/outputs.tf | 33 - .../internal_instance_template/variables.tf | 398 --- .../internal_instance_template/versions.tf | 30 - .../internal/slurm-gcp/login/README.md | 52 - .../modules/internal/slurm-gcp/login/main.tf | 112 - .../internal/slurm-gcp/login/outputs.tf | 25 - .../internal/slurm-gcp/login/variables.tf | 188 -- .../internal/slurm-gcp/login/versions.tf | 29 - .../internal/slurm-gcp/nodeset_tpu/README.md | 95 - .../internal/slurm-gcp/nodeset_tpu/main.tf | 121 - .../internal/slurm-gcp/nodeset_tpu/outputs.tf | 30 - .../slurm-gcp/nodeset_tpu/variables.tf | 158 -- .../slurm-gcp/nodeset_tpu/versions.tf | 30 - .../dependencies-installer/README.md | 61 - .../helm_install/README.md | 64 - .../helm_install/main.tf | 75 - .../helm_install/metadata.yaml | 19 - .../helm_install/variables.tf | 212 -- .../helm_install/versions.tf | 24 - .../kubernetes_manifest/README.md | 40 - .../kubernetes_manifest/main.tf | 104 - .../kubernetes_manifest/metadata.yaml | 19 - .../kubernetes_manifest/variables.tf | 69 - .../kubernetes_manifest/versions.tf | 24 - .../management/dependencies-installer/main.tf | 183 -- .../dependencies-installer/metadata.yaml | 19 - .../dependencies-installer/providers.tf | 25 - .../dependencies-installer/variables.tf | 70 - .../dependencies-installer/versions.tf | 30 - .../network/private-service-access/README.md | 122 - .../network/private-service-access/main.tf | 61 - .../private-service-access/metadata.yaml | 20 - .../network/private-service-access/outputs.tf | 43 - .../private-service-access/variables.tf | 59 - .../private-service-access/versions.tf | 37 - .../modules/project/new-project/README.md | 128 - .../modules/project/service-account/README.md | 111 - .../modules/project/service-account/main.tf | 37 - .../project/service-account/metadata.yaml | 19 - .../project/service-account/outputs.tf | 36 - .../project/service-account/variables.tf | 113 - .../project/service-account/versions.tf | 22 - .../project/service-enablement/README.md | 70 - .../project/service-enablement/main.tf | 28 - .../project/service-enablement/metadata.yaml | 19 - .../project/service-enablement/variables.tf | 31 - .../project/service-enablement/versions.tf | 29 - .../modules/pubsub/bigquery-sub/README.md | 87 - .../modules/pubsub/bigquery-sub/main.tf | 57 - .../modules/pubsub/bigquery-sub/metadata.yaml | 19 - .../modules/pubsub/bigquery-sub/outputs.tf | 20 - .../modules/pubsub/bigquery-sub/variables.tf | 51 - .../modules/pubsub/bigquery-sub/versions.tf | 35 - .../community/modules/pubsub/topic/README.md | 82 - .../community/modules/pubsub/topic/main.tf | 48 - .../modules/pubsub/topic/metadata.yaml | 19 - .../community/modules/pubsub/topic/outputs.tf | 26 - .../modules/pubsub/topic/variables.tf | 74 - .../modules/pubsub/topic/versions.tf | 32 - .../chrome-remote-desktop/README.md | 113 - .../chrome-remote-desktop/main.tf | 111 - .../chrome-remote-desktop/metadata.yaml | 18 - .../chrome-remote-desktop/outputs.tf | 25 - .../scripts/configure-chrome-desktop.yml | 61 - .../scripts/configure-grid-drivers.yml | 163 -- .../scripts/disable-sleep.yml | 39 - .../chrome-remote-desktop/variables.tf | 277 -- .../chrome-remote-desktop/versions.tf | 19 - .../scheduler/htcondor-access-point/README.md | 187 -- .../files/htcondor_configure.yml | 120 - .../scheduler/htcondor-access-point/main.tf | 338 --- .../htcondor-access-point/metadata.yaml | 20 - .../htcondor-access-point/outputs.tf | 25 - .../templates/condor_config.tftpl | 70 - .../htcondor-access-point/variables.tf | 266 -- .../htcondor-access-point/versions.tf | 37 - .../htcondor-central-manager/README.md | 159 -- .../files/htcondor_configure.yml | 72 - .../htcondor-central-manager/main.tf | 226 -- .../htcondor-central-manager/metadata.yaml | 20 - .../htcondor-central-manager/outputs.tf | 30 - .../templates/condor_config.tftpl | 31 - .../htcondor-central-manager/variables.tf | 192 -- .../htcondor-central-manager/versions.tf | 33 - .../scheduler/htcondor-pool-secrets/README.md | 172 -- .../files/htcondor_secrets.yml | 102 - .../scheduler/htcondor-pool-secrets/main.tf | 168 -- .../htcondor-pool-secrets/metadata.yaml | 20 - .../htcondor-pool-secrets/outputs.tf | 50 - .../templates/fetch-idtoken.ps1.tftpl | 26 - .../htcondor-pool-secrets/variables.tf | 67 - .../htcondor-pool-secrets/versions.tf | 33 - .../htcondor-service-accounts/README.md | 128 - .../htcondor-service-accounts/main.tf | 51 - .../htcondor-service-accounts/metadata.yaml | 19 - .../htcondor-service-accounts/outputs.tf | 30 - .../htcondor-service-accounts/variables.tf | 56 - .../htcondor-service-accounts/versions.tf | 19 - .../scheduler/htcondor-setup/README.md | 118 - .../modules/scheduler/htcondor-setup/main.tf | 68 - .../scheduler/htcondor-setup/metadata.yaml | 21 - .../scheduler/htcondor-setup/outputs.tf | 27 - .../scheduler/htcondor-setup/variables.tf | 55 - .../scheduler/htcondor-setup/versions.tf | 19 - .../schedmd-slurm-gcp-v6-controller/README.md | 405 --- .../controller.tf | 213 -- .../etc/htc-slurm.conf.tpl | 65 - .../etc/htc-slurmdbd.conf.tpl | 34 - .../etc/long-prolog-slurm.conf.tpl | 71 - .../schedmd-slurm-gcp-v6-controller/login.tf | 50 - .../schedmd-slurm-gcp-v6-controller/main.tf | 35 - .../metadata.yaml | 21 - .../modules/cleanup_compute/README.md | 42 - .../modules/cleanup_compute/main.tf | 46 - .../scripts/cleanup_compute.sh | 100 - .../modules/cleanup_compute/variables.tf | 71 - .../modules/cleanup_compute/versions.tf | 27 - .../modules/cleanup_tpu/README.md | 79 - .../modules/cleanup_tpu/main.tf | 32 - .../cleanup_tpu/scripts/cleanup_tpu.sh | 63 - .../modules/cleanup_tpu/variables.tf | 60 - .../modules/cleanup_tpu/versions.tf | 27 - .../modules/slurm_files/README.md | 121 - .../modules/slurm_files/etc/cgroup.conf.tpl | 7 - .../modules/slurm_files/etc/slurm.conf.tpl | 67 - .../modules/slurm_files/etc/slurmdbd.conf.tpl | 31 - .../slurm_files/files/external_epilog.sh | 18 - .../slurm_files/files/external_prolog.sh | 18 - .../slurm_files/files/setup_external.sh | 117 - .../modules/slurm_files/main.tf | 406 --- .../modules/slurm_files/outputs.tf | 45 - .../modules/slurm_files/scripts/conf.py | 658 ----- .../modules/slurm_files/scripts/file_cache.py | 80 - .../slurm_files/scripts/get_tpu_vmcount.py | 76 - .../slurm_files/scripts/job_submit.lua.tpl | 103 - .../modules/slurm_files/scripts/load_bq.py | 352 --- .../slurm_files/scripts/local_pubsub.py | 196 -- .../modules/slurm_files/scripts/mig_flex.py | 254 -- .../slurm_files/scripts/requirements-dev.txt | 9 - .../slurm_files/scripts/requirements.txt | 18 - .../modules/slurm_files/scripts/resume.py | 703 ------ .../slurm_files/scripts/resume_wrapper.sh | 40 - .../modules/slurm_files/scripts/setup.py | 660 ----- .../scripts/setup_network_storage.py | 327 --- .../modules/slurm_files/scripts/slurmsync.py | 679 ----- .../modules/slurm_files/scripts/sort_nodes.py | 171 -- .../modules/slurm_files/scripts/suspend.py | 126 - .../slurm_files/scripts/suspend_wrapper.sh | 28 - .../slurm_files/scripts/tests/common.py | 116 - .../slurm_files/scripts/tests/test_conf.py | 226 -- .../slurm_files/scripts/tests/test_resume.py | 175 -- .../scripts/tests/test_topology.py | 215 -- .../slurm_files/scripts/tests/test_util.py | 668 ----- .../slurm_files/scripts/tools/gpu-test | 133 - .../slurm_files/scripts/tools/task-epilog | 67 - .../slurm_files/scripts/tools/task-prolog | 70 - .../modules/slurm_files/scripts/tpu.py | 331 --- .../modules/slurm_files/scripts/util.py | 2224 ----------------- .../slurm_files/scripts/watch_delete_vm_op.py | 124 - .../modules/slurm_files/variables.tf | 504 ---- .../modules/slurm_files/versions.tf | 37 - .../outputs.tf | 62 - .../partition.tf | 174 -- .../slurm_files.tf | 191 -- .../source_image_logic.tf | 30 - .../variables.tf | 814 ------ .../variables_controller_instance.tf | 382 --- .../versions.tf | 33 - .../schedmd-slurm-gcp-v6-login/README.md | 130 - .../schedmd-slurm-gcp-v6-login/main.tf | 115 - .../schedmd-slurm-gcp-v6-login/metadata.yaml | 21 - .../schedmd-slurm-gcp-v6-login/outputs.tf | 18 - .../source_image_logic.tf | 30 - .../schedmd-slurm-gcp-v6-login/variables.tf | 419 ---- .../schedmd-slurm-gcp-v6-login/versions.tf | 23 - .../modules/scheduler/slinky/README.md | 172 -- .../modules/scheduler/slinky/main.tf | 197 -- .../modules/scheduler/slinky/metadata.yaml | 19 - .../modules/scheduler/slinky/outputs.tf | 23 - .../modules/scheduler/slinky/providers.tf | 23 - .../modules/scheduler/slinky/variables.tf | 127 - .../modules/scheduler/slinky/versions.tf | 28 - .../scripts/htcondor-install/README.md | 149 -- .../htcondor-install/files/autoscaler.py | 417 ---- .../install-htcondor-autoscaler-deps.yml | 46 - .../files/install-htcondor.yaml | 94 - .../modules/scripts/htcondor-install/main.tf | 51 - .../scripts/htcondor-install/metadata.yaml | 18 - .../scripts/htcondor-install/outputs.tf | 30 - .../templates/install-htcondor.ps1.tftpl | 59 - .../scripts/htcondor-install/variables.tf | 51 - .../scripts/htcondor-install/versions.tf | 19 - .../modules/scripts/ramble-execute/README.md | 116 - .../modules/scripts/ramble-execute/main.tf | 71 - .../scripts/ramble-execute/metadata.yaml | 18 - .../modules/scripts/ramble-execute/outputs.tf | 53 - .../templates/ramble_execute.yml.tpl | 59 - .../scripts/ramble-execute/variables.tf | 114 - .../scripts/ramble-execute/versions.tf | 25 - .../modules/scripts/ramble-setup/README.md | 128 - .../modules/scripts/ramble-setup/main.tf | 113 - .../scripts/ramble-setup/metadata.yaml | 18 - .../modules/scripts/ramble-setup/outputs.tf | 61 - .../scripts/install_ramble_deps.yml | 50 - .../install_ramble_python_deps.yml.tftpl | 28 - .../templates/ramble_setup.yml.tftpl | 157 -- .../modules/scripts/ramble-setup/variables.tf | 97 - .../modules/scripts/ramble-setup/versions.tf | 30 - .../modules/scripts/spack-execute/README.md | 141 -- .../modules/scripts/spack-execute/main.tf | 70 - .../scripts/spack-execute/metadata.yaml | 18 - .../modules/scripts/spack-execute/outputs.tf | 45 - .../templates/execute_commands.yml.tpl | 59 - .../scripts/spack-execute/variables.tf | 103 - .../modules/scripts/spack-execute/versions.tf | 25 - .../modules/scripts/spack-setup/README.md | 382 --- .../modules/scripts/spack-setup/main.tf | 120 - .../modules/scripts/spack-setup/metadata.yaml | 19 - .../modules/scripts/spack-setup/outputs.tf | 56 - .../scripts/install_spack_deps.yml | 50 - .../templates/spack_setup.yml.tftpl | 157 -- .../modules/scripts/spack-setup/variables.tf | 106 - .../modules/scripts/spack-setup/versions.tf | 30 - .../scripts/wait-for-startup/README.md | 87 - .../modules/scripts/wait-for-startup/main.tf | 47 - .../scripts/wait-for-startup/metadata.yaml | 19 - .../scripts/wait-for-startup/outputs.tf | 15 - .../scripts/wait-for-startup-status.sh | 138 - .../scripts/wait-for-startup/variables.tf | 54 - .../scripts/wait-for-startup/versions.tf | 29 - .../scripts/windows-startup-script/README.md | 109 - .../scripts/windows-startup-script/main.tf | 34 - .../windows-startup-script/metadata.yaml | 18 - .../scripts/windows-startup-script/outputs.tf | 20 - .../templates/install_gpu_driver.ps1.tftpl | 38 - .../templates/setx_http_proxy.ps1 | 21 - .../windows-startup-script/variables.tf | 54 - .../windows-startup-script/versions.tf | 23 - .../modules/embedded/modules/README.md | 554 ---- .../compute/gke-job-template/README.md | 133 - .../modules/compute/gke-job-template/main.tf | 181 -- .../compute/gke-job-template/metadata.yaml | 18 - .../compute/gke-job-template/outputs.tf | 27 - .../templates/gke-job-base.yaml.tftpl | 128 - .../compute/gke-job-template/variables.tf | 206 -- .../compute/gke-job-template/versions.tf | 28 - .../modules/compute/gke-node-pool/README.md | 388 --- .../compute/gke-node-pool/disk_definitions.tf | 38 - .../sample-tcpx-workload-job.yaml | 50 - .../sample-tcpxo-workload-job.yaml | 70 - .../scripts/enable-tcpx-in-workload.py | 185 -- .../scripts/enable-tcpxo-in-workload.py | 186 -- .../compute/gke-node-pool/gpu_direct.tf | 87 - .../compute/gke-node-pool/guest_cpus.tf | 32 - .../modules/compute/gke-node-pool/main.tf | 482 ---- .../compute/gke-node-pool/metadata.yaml | 21 - .../modules/compute/gke-node-pool/outputs.tf | 152 -- .../gke-node-pool/reservation_definitions.tf | 107 - .../gke-node-pool/threads_per_core_calc.tf | 42 - .../compute/gke-node-pool/variables.tf | 487 ---- .../modules/compute/gke-node-pool/versions.tf | 38 - .../modules/compute/resource-policy/README.md | 82 - .../modules/compute/resource-policy/main.tf | 48 - .../compute/resource-policy/metadata.yaml | 19 - .../compute/resource-policy/outputs.tf | 30 - .../compute/resource-policy/variables.tf | 64 - .../compute/resource-policy/versions.tf | 34 - .../modules/compute/vm-instance/README.md | 257 -- .../compute/vm-instance/compute_image.tf | 30 - .../modules/compute/vm-instance/main.tf | 334 --- .../modules/compute/vm-instance/metadata.yaml | 19 - .../modules/compute/vm-instance/outputs.tf | 50 - .../startup_from_network_storage.tf | 65 - .../vm-instance/threads_per_core_calc.tf | 42 - .../modules/compute/vm-instance/variables.tf | 452 ---- .../modules/compute/vm-instance/versions.tf | 41 - .../cloud-storage-bucket/README.md | 170 -- .../file-system/cloud-storage-bucket/main.tf | 126 - .../cloud-storage-bucket/metadata.yaml | 18 - .../cloud-storage-bucket/outputs.tf | 69 - .../scripts/install-gcs-fuse.sh | 44 - .../cloud-storage-bucket/scripts/mount.sh | 58 - .../cloud-storage-bucket/variables.tf | 254 -- .../cloud-storage-bucket/versions.tf | 39 - .../modules/file-system/filestore/README.md | 248 -- .../modules/file-system/filestore/main.tf | 116 - .../file-system/filestore/metadata.yaml | 19 - .../modules/file-system/filestore/outputs.tf | 62 - .../filestore/scripts/install-nfs-client.sh | 37 - .../file-system/filestore/scripts/mount.sh | 58 - .../file-system/filestore/variables.tf | 189 -- .../modules/file-system/filestore/versions.tf | 36 - .../gke-persistent-volume/README.md | 200 -- .../file-system/gke-persistent-volume/main.tf | 155 -- .../gke-persistent-volume/metadata.yaml | 18 - .../gke-persistent-volume/outputs.tf | 31 - .../templates/filestore-pv.yaml.tftpl | 26 - .../templates/filestore-pvc.yaml.tftpl | 18 - .../templates/gcs-pv.yaml.tftpl | 24 - .../templates/gcs-pvc.yaml.tftpl | 21 - .../templates/managed-lustre-pv.yaml.tftpl | 26 - .../templates/managed-lustre-pvc.yaml.tftpl | 18 - .../templates/namespace.yaml.tftpl | 5 - .../gke-persistent-volume/variables.tf | 93 - .../gke-persistent-volume/versions.tf | 30 - .../modules/file-system/gke-storage/README.md | 134 - .../modules/file-system/gke-storage/main.tf | 86 - .../file-system/gke-storage/metadata.yaml | 18 - .../file-system/gke-storage/outputs.tf | 28 - .../hyperdisk-balanced-pvc.yaml.tftpl | 17 - .../hyperdisk-extreme-pvc.yaml.tftpl | 17 - .../hyperdisk-throughput-pvc.yaml.tftpl | 17 - .../namespace.yaml.tftpl | 5 - .../parallelstore-pvc.yaml.tftpl | 17 - .../hyperdisk-balanced-sc.yaml.tftpl | 25 - .../hyperdisk-extreme-sc.yaml.tftpl | 24 - .../hyperdisk-throughput-sc.yaml.tftpl | 24 - .../storage-class/parallelstore-sc.yaml.tftpl | 21 - .../file-system/gke-storage/variables.tf | 144 -- .../file-system/gke-storage/versions.tf | 21 - .../file-system/managed-lustre/README.md | 289 --- .../file-system/managed-lustre/main.tf | 104 - .../file-system/managed-lustre/metadata.yaml | 19 - .../file-system/managed-lustre/outputs.tf | 43 - .../scripts/install-managed-lustre-client.sh | 84 - .../managed-lustre/scripts/mount.sh | 58 - .../file-system/managed-lustre/variables.tf | 131 - .../file-system/managed-lustre/versions.tf | 36 - .../file-system/netapp-storage-pool/README.md | 193 -- .../file-system/netapp-storage-pool/main.tf | 56 - .../netapp-storage-pool/metadata.yaml | 20 - .../netapp-storage-pool/outputs.tf | 23 - .../netapp-storage-pool/variables.tf | 133 - .../netapp-storage-pool/versions.tf | 37 - .../file-system/netapp-volume/README.md | 201 -- .../modules/file-system/netapp-volume/main.tf | 92 - .../file-system/netapp-volume/metadata.yaml | 19 - .../file-system/netapp-volume/outputs.tf | 66 - .../scripts/install-nfs-client.sh | 37 - .../netapp-volume/scripts/mount.sh | 66 - .../file-system/netapp-volume/variables.tf | 133 - .../file-system/netapp-volume/versions.tf | 32 - .../file-system/parallelstore/README.md | 196 -- .../modules/file-system/parallelstore/main.tf | 74 - .../file-system/parallelstore/metadata.yaml | 19 - .../file-system/parallelstore/outputs.tf | 47 - .../scripts/install-daos-client.sh | 112 - .../templates/mount-daos.sh.tftpl | 110 - .../file-system/parallelstore/variables.tf | 137 - .../file-system/parallelstore/versions.tf | 36 - .../pre-existing-network-storage/README.md | 192 -- .../metadata.yaml | 18 - .../pre-existing-network-storage/outputs.tf | 124 - .../scripts/install-daos-client.sh | 112 - .../scripts/install-gcs-fuse.sh | 44 - .../scripts/install-managed-lustre-client.sh | 84 - .../scripts/install-nfs-client.sh | 37 - .../scripts/mount.sh | 58 - .../ddn_exascaler_luster_client_install.tftpl | 50 - .../templates/mount-daos.sh.tftpl | 110 - .../pre-existing-network-storage/variables.tf | 67 - .../pre-existing-network-storage/versions.tf | 19 - .../modules/internal/gpu-definition/README.md | 47 - .../modules/internal/gpu-definition/main.tf | 98 - .../internal/instance_validations/README.md | 30 - .../internal/instance_validations/main.tf | 52 - .../instance_validations/variables.tf | 23 - .../internal/instance_validations/versions.tf | 17 - .../internal/network-attachment/README.md | 54 - .../internal/network-attachment/main.tf | 70 - .../internal/network-attachment/metadata.yaml | 19 - .../modules/internal/tpu-definition/README.md | 85 - .../modules/internal/tpu-definition/main.tf | 69 - .../internal/tpu-definition/outputs.tf | 40 - .../internal/tpu-definition/variables.tf | 29 - .../modules/internal/vpc_peering/README.md | 56 - .../modules/internal/vpc_peering/main.tf | 80 - .../internal/vpc_peering/metadata.yaml | 19 - .../management/kubectl-apply/README.md | 244 -- .../kubectl-apply/helm_install/README.md | 64 - .../kubectl-apply/helm_install/main.tf | 79 - .../kubectl-apply/helm_install/metadata.yaml | 19 - .../kubectl-apply/helm_install/variables.tf | 212 -- .../kubectl-apply/helm_install/versions.tf | 24 - .../jobset/jobset-helm-values.yaml | 25 - .../kubectl-apply/kubectl/README.md | 55 - .../management/kubectl-apply/kubectl/main.tf | 92 - .../kubectl-apply/kubectl/metadata.yaml | 19 - .../kubectl-apply/kubectl/variables.tf | 51 - .../kubectl-apply/kubectl/versions.tf | 26 - .../kueue/kueue-helm-values.yaml | 30 - .../modules/management/kubectl-apply/main.tf | 271 -- .../management/kubectl-apply/metadata.yaml | 19 - .../management/kubectl-apply/providers.tf | 33 - .../management/kubectl-apply/variables.tf | 192 -- .../management/kubectl-apply/versions.tf | 42 - .../modules/monitoring/dashboard/README.md | 86 - .../dashboard/dashboards/Empty.json.tpl | 17 - .../dashboard/dashboards/HPC.json.tpl | 595 ----- .../modules/monitoring/dashboard/main.tf | 35 - .../monitoring/dashboard/metadata.yaml | 19 - .../modules/monitoring/dashboard/outputs.tf | 23 - .../modules/monitoring/dashboard/variables.tf | 52 - .../modules/monitoring/dashboard/versions.tf | 29 - .../modules/network/firewall-rules/README.md | 111 - .../modules/network/firewall-rules/main.tf | 60 - .../network/firewall-rules/metadata.yaml | 19 - .../network/firewall-rules/variables.tf | 88 - .../network/firewall-rules/versions.tf | 29 - .../modules/network/gpu-rdma-vpc/README.md | 143 -- .../modules/network/gpu-rdma-vpc/main.tf | 79 - .../network/gpu-rdma-vpc/metadata.yaml | 19 - .../modules/network/gpu-rdma-vpc/outputs.tf | 59 - .../modules/network/gpu-rdma-vpc/variables.tf | 164 -- .../modules/network/gpu-rdma-vpc/versions.tf | 19 - .../modules/network/multivpc/README.md | 136 - .../embedded/modules/network/multivpc/main.tf | 78 - .../modules/network/multivpc/metadata.yaml | 19 - .../modules/network/multivpc/outputs.tf | 50 - .../modules/network/multivpc/variables.tf | 201 -- .../modules/network/multivpc/versions.tf | 19 - .../network/pre-existing-subnetwork/README.md | 94 - .../network/pre-existing-subnetwork/main.tf | 38 - .../pre-existing-subnetwork/metadata.yaml | 21 - .../pre-existing-subnetwork/outputs.tf | 35 - .../pre-existing-subnetwork/variables.tf | 39 - .../pre-existing-subnetwork/versions.tf | 29 - .../network/pre-existing-vpc/README.md | 110 - .../modules/network/pre-existing-vpc/main.tf | 53 - .../network/pre-existing-vpc/metadata.yaml | 19 - .../network/pre-existing-vpc/outputs.tf | 50 - .../network/pre-existing-vpc/variables.tf | 37 - .../network/pre-existing-vpc/versions.tf | 29 - .../embedded/modules/network/vpc/README.md | 237 -- .../embedded/modules/network/vpc/main.tf | 256 -- .../modules/network/vpc/metadata.yaml | 19 - .../embedded/modules/network/vpc/outputs.tf | 68 - .../embedded/modules/network/vpc/variables.tf | 301 --- .../embedded/modules/network/vpc/versions.tf | 19 - .../modules/packer/custom-image/README.md | 320 --- .../modules/packer/custom-image/image.pkr.hcl | 216 -- .../modules/packer/custom-image/metadata.yaml | 21 - .../packer/custom-image/variables.pkr.hcl | 276 -- .../packer/custom-image/versions.pkr.hcl | 25 - .../scheduler/batch-job-template/README.md | 197 -- .../batch-job-template/compute_image.tf | 30 - .../scheduler/batch-job-template/main.tf | 149 -- .../batch-job-template/metadata.yaml | 22 - .../scheduler/batch-job-template/outputs.tf | 80 - .../startup_from_network_storage.tf | 65 - .../templates/batch-job-base.yaml.tftpl | 53 - .../templates/batch-submit.sh.tftpl | 10 - .../scheduler/batch-job-template/variables.tf | 240 -- .../scheduler/batch-job-template/versions.tf | 37 - .../scheduler/batch-login-node/README.md | 127 - .../scheduler/batch-login-node/main.tf | 127 - .../scheduler/batch-login-node/metadata.yaml | 21 - .../scheduler/batch-login-node/outputs.tf | 37 - .../scheduler/batch-login-node/variables.tf | 151 -- .../scheduler/batch-login-node/versions.tf | 29 - .../modules/scheduler/gke-cluster/README.md | 220 -- .../modules/scheduler/gke-cluster/main.tf | 470 ---- .../scheduler/gke-cluster/metadata.yaml | 19 - .../modules/scheduler/gke-cluster/outputs.tf | 104 - .../templates/gke-network-paramset.yaml.tftpl | 9 - .../templates/network-object.yaml.tftpl | 11 - .../scheduler/gke-cluster/variables.tf | 533 ---- .../modules/scheduler/gke-cluster/versions.tf | 39 - .../pre-existing-gke-cluster/README.md | 116 - .../pre-existing-gke-cluster/main.tf | 70 - .../pre-existing-gke-cluster/metadata.yaml | 19 - .../pre-existing-gke-cluster/outputs.tf | 33 - .../templates/gke-network-paramset.yaml.tftpl | 9 - .../templates/network-object.yaml.tftpl | 11 - .../pre-existing-gke-cluster/variables.tf | 61 - .../pre-existing-gke-cluster/versions.tf | 30 - .../modules/scripts/startup-script/README.md | 355 --- .../startup-script/files/configure-ssh.yml | 37 - .../startup-script/files/configure_proxy.sh | 54 - .../files/early_run_hotfixes.sh | 32 - .../startup-script/files/get_from_bucket.sh | 73 - .../startup-script/files/install_ansible.sh | 247 -- .../files/install_cloud_rdma_drivers.sh | 38 - .../startup-script/files/install_docker.yml | 113 - .../files/install_gpu_network_wait_online.yml | 56 - .../files/install_managed_lustre.yml | 33 - .../files/install_monitoring_agent.sh | 144 -- .../files/running-script-warning.sh | 26 - .../startup-script/files/setup-raid.yml | 100 - .../startup-script/files/setup-ssh-keys.sh | 19 - .../startup-script/files/setup-ssh-keys.yml | 40 - .../files/startup-script-stdlib-body.sh | 39 - .../files/startup-script-stdlib-head.sh | 266 -- .../modules/scripts/startup-script/main.tf | 306 --- .../scripts/startup-script/metadata.yaml | 19 - .../modules/scripts/startup-script/outputs.tf | 39 - .../templates/startup-script-custom.tftpl | 65 - .../scripts/startup-script/variables.tf | 298 --- .../scripts/startup-script/versions.tf | 37 - deletion-test/build_script/outputs.tf | 21 - deletion-test/build_script/providers.tf | 27 - deletion-test/build_script/variables.tf | 55 - deletion-test/build_script/versions.tf | 30 - deletion-test/cluster/main.tf | 227 -- .../embedded/community/modules/README.md | 7 - .../modules/compute/gke-nodeset/README.md | 55 - .../modules/compute/gke-nodeset/main.tf | 64 - .../modules/compute/gke-nodeset/metadata.yaml | 20 - .../modules/compute/gke-nodeset/output.tf | 18 - .../compute/gke-nodeset/persistent_volumes.tf | 50 - .../templates/nodeset-general.yaml.tftpl | 203 -- .../modules/compute/gke-nodeset/variables.tf | 118 - .../modules/compute/gke-nodeset/versions.tf | 27 - .../modules/compute/gke-partition/README.md | 39 - .../modules/compute/gke-partition/main.tf | 47 - .../compute/gke-partition/metadata.yaml | 19 - .../compute/gke-partition/variables.tf | 43 - .../modules/compute/gke-partition/versions.tf | 27 - .../compute/htcondor-execute-point/README.md | 271 -- .../htcondor-execute-point/compute_image.tf | 30 - .../files/htcondor_configure.yml | 74 - .../files/htcondor_configure_autoscaler.yml | 98 - .../compute/htcondor-execute-point/main.tf | 218 -- .../htcondor-execute-point/metadata.yaml | 20 - .../compute/htcondor-execute-point/outputs.tf | 25 - .../templates/condor_config.tftpl | 31 - .../download-condor-config.ps1.tftpl | 34 - .../htcondor-execute-point/variables.tf | 265 -- .../htcondor-execute-point/versions.tf | 34 - .../community/modules/compute/mig/README.md | 45 - .../community/modules/compute/mig/main.tf | 85 - .../modules/compute/mig/metadata.yaml | 21 - .../community/modules/compute/mig/outputs.tf | 18 - .../modules/compute/mig/variables.tf | 86 - .../community/modules/compute/mig/versions.tf | 27 - .../modules/compute/notebook/README.md | 112 - .../modules/compute/notebook/main.tf | 96 - .../modules/compute/notebook/metadata.yaml | 20 - .../modules/compute/notebook/variables.tf | 111 - .../modules/compute/notebook/versions.tf | 29 - .../README.md | 135 - .../main.tf | 128 - .../metadata.yaml | 20 - .../outputs.tf | 36 - .../source_image_logic.tf | 30 - .../variables.tf | 402 --- .../versions.tf | 22 - .../README.md | 85 - .../schedmd-slurm-gcp-v6-nodeset-tpu/main.tf | 59 - .../metadata.yaml | 21 - .../outputs.tf | 39 - .../variables.tf | 171 -- .../versions.tf | 23 - .../schedmd-slurm-gcp-v6-nodeset/README.md | 227 -- .../schedmd-slurm-gcp-v6-nodeset/main.tf | 232 -- .../metadata.yaml | 21 - .../schedmd-slurm-gcp-v6-nodeset/outputs.tf | 112 - .../source_image_logic.tf | 30 - .../schedmd-slurm-gcp-v6-nodeset/variables.tf | 641 ----- .../schedmd-slurm-gcp-v6-nodeset/versions.tf | 29 - .../schedmd-slurm-gcp-v6-partition/README.md | 105 - .../schedmd-slurm-gcp-v6-partition/main.tf | 41 - .../metadata.yaml | 20 - .../schedmd-slurm-gcp-v6-partition/outputs.tf | 54 - .../variables.tf | 311 --- .../versions.tf | 23 - .../container/artifact-registry/README.md | 157 -- .../container/artifact-registry/main.tf | 268 -- .../container/artifact-registry/metadata.yaml | 21 - .../container/artifact-registry/outputs.tf | 18 - .../container/artifact-registry/validation.tf | 49 - .../container/artifact-registry/variables.tf | 122 - .../container/artifact-registry/versions.tf | 27 - .../database/bigquery-dataset/README.md | 76 - .../modules/database/bigquery-dataset/main.tf | 32 - .../database/bigquery-dataset/metadata.yaml | 19 - .../database/bigquery-dataset/outputs.tf | 20 - .../database/bigquery-dataset/variables.tf | 36 - .../database/bigquery-dataset/versions.tf | 29 - .../modules/database/bigquery-table/README.md | 87 - .../modules/database/bigquery-table/main.tf | 37 - .../database/bigquery-table/metadata.yaml | 19 - .../database/bigquery-table/outputs.tf | 28 - .../database/bigquery-table/variables.tf | 46 - .../database/bigquery-table/versions.tf | 29 - .../slurm-cloudsql-federation/README.md | 107 - .../slurm-cloudsql-federation/main.tf | 165 -- .../slurm-cloudsql-federation/metadata.yaml | 21 - .../slurm-cloudsql-federation/outputs.tf | 27 - .../slurm-cloudsql-federation/variables.tf | 173 -- .../slurm-cloudsql-federation/versions.tf | 36 - .../file-system/DDN-EXAScaler/README.md | 158 -- .../modules/file-system/DDN-EXAScaler/main.tf | 72 - .../file-system/DDN-EXAScaler/metadata.yaml | 22 - .../file-system/DDN-EXAScaler/outputs.tf | 90 - .../file-system/DDN-EXAScaler/variables.tf | 502 ---- .../file-system/DDN-EXAScaler/versions.tf | 24 - .../modules/file-system/Intel-DAOS/README.md | 1 - .../modules/file-system/nfs-server/README.md | 152 -- .../modules/file-system/nfs-server/main.tf | 131 - .../file-system/nfs-server/metadata.yaml | 19 - .../modules/file-system/nfs-server/outputs.tf | 53 - .../nfs-server/scripts/install-nfs-client.sh | 37 - .../scripts/install-nfs-server.sh.tpl | 35 - .../file-system/nfs-server/scripts/mount.sh | 58 - .../file-system/nfs-server/scripts/mount.yaml | 39 - .../file-system/nfs-server/variables.tf | 194 -- .../file-system/nfs-server/versions.tf | 37 - .../file-system/sycomp-scale/README.md | 35 - .../modules/file-system/weka-client/README.md | 182 -- .../file-system/weka-client/metadata.yaml | 18 - .../file-system/weka-client/outputs.tf | 71 - .../templates/install-weka-client.yaml.tftpl | 133 - .../weka-client/templates/mount-weka.sh.tftpl | 101 - .../templates/mount-weka.yaml.tftpl | 54 - .../file-system/weka-client/variables.tf | 39 - .../file-system/weka-client/versions.tf | 19 - .../FSI_MonteCarlo.ipynb | 125 - .../files/fsi-montecarlo-on-batch/README.md | 97 - .../fsi-montecarlo-on-batch/iteration.sh | 23 - .../files/fsi-montecarlo-on-batch/main.tf | 102 - .../fsi-montecarlo-on-batch/mc_run.tpl.py | 157 -- .../fsi-montecarlo-on-batch/mc_run.tpl.yaml | 36 - .../fsi-montecarlo-on-batch/mc_run_reqs.txt | 9 - .../fsi-montecarlo-on-batch/metadata.yaml | 18 - .../fsi-montecarlo-on-batch/variables.tf | 51 - .../files/fsi-montecarlo-on-batch/versions.tf | 43 - .../internal/slurm-gcp/instance/README.md | 100 - .../internal/slurm-gcp/instance/main.tf | 126 - .../internal/slurm-gcp/instance/outputs.tf | 41 - .../internal/slurm-gcp/instance/variables.tf | 119 - .../internal/slurm-gcp/instance/versions.tf | 31 - .../slurm-gcp/instance_template/README.md | 87 - .../files/startup_sh_unlinted | 169 -- .../slurm-gcp/instance_template/main.tf | 171 -- .../slurm-gcp/instance_template/outputs.tf | 43 - .../slurm-gcp/instance_template/variables.tf | 431 ---- .../slurm-gcp/instance_template/versions.tf | 25 - .../internal_instance_template/README.md | 89 - .../internal_instance_template/main.tf | 234 -- .../internal_instance_template/outputs.tf | 33 - .../internal_instance_template/variables.tf | 398 --- .../internal_instance_template/versions.tf | 30 - .../internal/slurm-gcp/login/README.md | 52 - .../modules/internal/slurm-gcp/login/main.tf | 112 - .../internal/slurm-gcp/login/outputs.tf | 25 - .../internal/slurm-gcp/login/variables.tf | 188 -- .../internal/slurm-gcp/login/versions.tf | 29 - .../internal/slurm-gcp/nodeset_tpu/README.md | 95 - .../internal/slurm-gcp/nodeset_tpu/main.tf | 121 - .../internal/slurm-gcp/nodeset_tpu/outputs.tf | 30 - .../slurm-gcp/nodeset_tpu/variables.tf | 158 -- .../slurm-gcp/nodeset_tpu/versions.tf | 30 - .../dependencies-installer/README.md | 61 - .../helm_install/README.md | 64 - .../helm_install/main.tf | 75 - .../helm_install/metadata.yaml | 19 - .../helm_install/variables.tf | 212 -- .../helm_install/versions.tf | 24 - .../kubernetes_manifest/README.md | 40 - .../kubernetes_manifest/main.tf | 104 - .../kubernetes_manifest/metadata.yaml | 19 - .../kubernetes_manifest/variables.tf | 69 - .../kubernetes_manifest/versions.tf | 24 - .../management/dependencies-installer/main.tf | 183 -- .../dependencies-installer/metadata.yaml | 19 - .../dependencies-installer/providers.tf | 25 - .../dependencies-installer/variables.tf | 70 - .../dependencies-installer/versions.tf | 30 - .../network/private-service-access/README.md | 122 - .../network/private-service-access/main.tf | 61 - .../private-service-access/metadata.yaml | 20 - .../network/private-service-access/outputs.tf | 43 - .../private-service-access/variables.tf | 59 - .../private-service-access/versions.tf | 37 - .../modules/project/new-project/README.md | 128 - .../modules/project/service-account/README.md | 111 - .../modules/project/service-account/main.tf | 37 - .../project/service-account/metadata.yaml | 19 - .../project/service-account/outputs.tf | 36 - .../project/service-account/variables.tf | 113 - .../project/service-account/versions.tf | 22 - .../project/service-enablement/README.md | 70 - .../project/service-enablement/main.tf | 28 - .../project/service-enablement/metadata.yaml | 19 - .../project/service-enablement/variables.tf | 31 - .../project/service-enablement/versions.tf | 29 - .../modules/pubsub/bigquery-sub/README.md | 87 - .../modules/pubsub/bigquery-sub/main.tf | 57 - .../modules/pubsub/bigquery-sub/metadata.yaml | 19 - .../modules/pubsub/bigquery-sub/outputs.tf | 20 - .../modules/pubsub/bigquery-sub/variables.tf | 51 - .../modules/pubsub/bigquery-sub/versions.tf | 35 - .../community/modules/pubsub/topic/README.md | 82 - .../community/modules/pubsub/topic/main.tf | 48 - .../modules/pubsub/topic/metadata.yaml | 19 - .../community/modules/pubsub/topic/outputs.tf | 26 - .../modules/pubsub/topic/variables.tf | 74 - .../modules/pubsub/topic/versions.tf | 32 - .../chrome-remote-desktop/README.md | 113 - .../chrome-remote-desktop/main.tf | 111 - .../chrome-remote-desktop/metadata.yaml | 18 - .../chrome-remote-desktop/outputs.tf | 25 - .../scripts/configure-chrome-desktop.yml | 61 - .../scripts/configure-grid-drivers.yml | 163 -- .../scripts/disable-sleep.yml | 39 - .../chrome-remote-desktop/variables.tf | 277 -- .../chrome-remote-desktop/versions.tf | 19 - .../scheduler/htcondor-access-point/README.md | 187 -- .../files/htcondor_configure.yml | 120 - .../scheduler/htcondor-access-point/main.tf | 338 --- .../htcondor-access-point/metadata.yaml | 20 - .../htcondor-access-point/outputs.tf | 25 - .../templates/condor_config.tftpl | 70 - .../htcondor-access-point/variables.tf | 266 -- .../htcondor-access-point/versions.tf | 37 - .../htcondor-central-manager/README.md | 159 -- .../files/htcondor_configure.yml | 72 - .../htcondor-central-manager/main.tf | 226 -- .../htcondor-central-manager/metadata.yaml | 20 - .../htcondor-central-manager/outputs.tf | 30 - .../templates/condor_config.tftpl | 31 - .../htcondor-central-manager/variables.tf | 192 -- .../htcondor-central-manager/versions.tf | 33 - .../scheduler/htcondor-pool-secrets/README.md | 172 -- .../files/htcondor_secrets.yml | 102 - .../scheduler/htcondor-pool-secrets/main.tf | 168 -- .../htcondor-pool-secrets/metadata.yaml | 20 - .../htcondor-pool-secrets/outputs.tf | 50 - .../templates/fetch-idtoken.ps1.tftpl | 26 - .../htcondor-pool-secrets/variables.tf | 67 - .../htcondor-pool-secrets/versions.tf | 33 - .../htcondor-service-accounts/README.md | 128 - .../htcondor-service-accounts/main.tf | 51 - .../htcondor-service-accounts/metadata.yaml | 19 - .../htcondor-service-accounts/outputs.tf | 30 - .../htcondor-service-accounts/variables.tf | 56 - .../htcondor-service-accounts/versions.tf | 19 - .../scheduler/htcondor-setup/README.md | 118 - .../modules/scheduler/htcondor-setup/main.tf | 68 - .../scheduler/htcondor-setup/metadata.yaml | 21 - .../scheduler/htcondor-setup/outputs.tf | 27 - .../scheduler/htcondor-setup/variables.tf | 55 - .../scheduler/htcondor-setup/versions.tf | 19 - .../schedmd-slurm-gcp-v6-controller/README.md | 405 --- .../controller.tf | 213 -- .../etc/htc-slurm.conf.tpl | 65 - .../etc/htc-slurmdbd.conf.tpl | 34 - .../etc/long-prolog-slurm.conf.tpl | 71 - .../schedmd-slurm-gcp-v6-controller/login.tf | 50 - .../schedmd-slurm-gcp-v6-controller/main.tf | 35 - .../metadata.yaml | 21 - .../modules/cleanup_compute/README.md | 42 - .../modules/cleanup_compute/main.tf | 46 - .../scripts/cleanup_compute.sh | 100 - .../modules/cleanup_compute/variables.tf | 71 - .../modules/cleanup_compute/versions.tf | 27 - .../modules/cleanup_tpu/README.md | 79 - .../modules/cleanup_tpu/main.tf | 32 - .../cleanup_tpu/scripts/cleanup_tpu.sh | 63 - .../modules/cleanup_tpu/variables.tf | 60 - .../modules/cleanup_tpu/versions.tf | 27 - .../modules/slurm_files/README.md | 121 - .../build/slurm-gcp-devel-controller.zip | Bin 84485 -> 0 bytes .../slurm_files/build/slurm-gcp-devel.zip | Bin 63549 -> 0 bytes .../modules/slurm_files/etc/cgroup.conf.tpl | 7 - .../modules/slurm_files/etc/slurm.conf.tpl | 67 - .../modules/slurm_files/etc/slurmdbd.conf.tpl | 31 - .../slurm_files/files/external_epilog.sh | 18 - .../slurm_files/files/external_prolog.sh | 18 - .../slurm_files/files/setup_external.sh | 117 - .../modules/slurm_files/main.tf | 406 --- .../modules/slurm_files/outputs.tf | 45 - .../modules/slurm_files/scripts/conf.py | 658 ----- .../modules/slurm_files/scripts/file_cache.py | 80 - .../slurm_files/scripts/get_tpu_vmcount.py | 76 - .../slurm_files/scripts/job_submit.lua.tpl | 103 - .../modules/slurm_files/scripts/load_bq.py | 352 --- .../slurm_files/scripts/local_pubsub.py | 196 -- .../modules/slurm_files/scripts/mig_flex.py | 254 -- .../slurm_files/scripts/requirements-dev.txt | 9 - .../slurm_files/scripts/requirements.txt | 18 - .../modules/slurm_files/scripts/resume.py | 703 ------ .../slurm_files/scripts/resume_wrapper.sh | 40 - .../modules/slurm_files/scripts/setup.py | 660 ----- .../scripts/setup_network_storage.py | 327 --- .../modules/slurm_files/scripts/slurmsync.py | 679 ----- .../modules/slurm_files/scripts/sort_nodes.py | 171 -- .../modules/slurm_files/scripts/suspend.py | 126 - .../slurm_files/scripts/suspend_wrapper.sh | 28 - .../slurm_files/scripts/tests/common.py | 116 - .../slurm_files/scripts/tests/test_conf.py | 226 -- .../slurm_files/scripts/tests/test_resume.py | 175 -- .../scripts/tests/test_topology.py | 215 -- .../slurm_files/scripts/tests/test_util.py | 668 ----- .../slurm_files/scripts/tools/gpu-test | 133 - .../slurm_files/scripts/tools/task-epilog | 67 - .../slurm_files/scripts/tools/task-prolog | 70 - .../modules/slurm_files/scripts/tpu.py | 331 --- .../modules/slurm_files/scripts/util.py | 2224 ----------------- .../slurm_files/scripts/watch_delete_vm_op.py | 124 - .../modules/slurm_files/variables.tf | 504 ---- .../modules/slurm_files/versions.tf | 37 - .../outputs.tf | 62 - .../partition.tf | 174 -- .../slurm_files.tf | 191 -- .../source_image_logic.tf | 30 - .../variables.tf | 814 ------ .../variables_controller_instance.tf | 382 --- .../versions.tf | 33 - .../schedmd-slurm-gcp-v6-login/README.md | 130 - .../schedmd-slurm-gcp-v6-login/main.tf | 115 - .../schedmd-slurm-gcp-v6-login/metadata.yaml | 21 - .../schedmd-slurm-gcp-v6-login/outputs.tf | 18 - .../source_image_logic.tf | 30 - .../schedmd-slurm-gcp-v6-login/variables.tf | 419 ---- .../schedmd-slurm-gcp-v6-login/versions.tf | 23 - .../modules/scheduler/slinky/README.md | 172 -- .../modules/scheduler/slinky/main.tf | 197 -- .../modules/scheduler/slinky/metadata.yaml | 19 - .../modules/scheduler/slinky/outputs.tf | 23 - .../modules/scheduler/slinky/providers.tf | 23 - .../modules/scheduler/slinky/variables.tf | 127 - .../modules/scheduler/slinky/versions.tf | 28 - .../scripts/htcondor-install/README.md | 149 -- .../htcondor-install/files/autoscaler.py | 417 ---- .../install-htcondor-autoscaler-deps.yml | 46 - .../files/install-htcondor.yaml | 94 - .../modules/scripts/htcondor-install/main.tf | 51 - .../scripts/htcondor-install/metadata.yaml | 18 - .../scripts/htcondor-install/outputs.tf | 30 - .../templates/install-htcondor.ps1.tftpl | 59 - .../scripts/htcondor-install/variables.tf | 51 - .../scripts/htcondor-install/versions.tf | 19 - .../modules/scripts/ramble-execute/README.md | 116 - .../modules/scripts/ramble-execute/main.tf | 71 - .../scripts/ramble-execute/metadata.yaml | 18 - .../modules/scripts/ramble-execute/outputs.tf | 53 - .../templates/ramble_execute.yml.tpl | 59 - .../scripts/ramble-execute/variables.tf | 114 - .../scripts/ramble-execute/versions.tf | 25 - .../modules/scripts/ramble-setup/README.md | 128 - .../modules/scripts/ramble-setup/main.tf | 113 - .../scripts/ramble-setup/metadata.yaml | 18 - .../modules/scripts/ramble-setup/outputs.tf | 61 - .../scripts/install_ramble_deps.yml | 50 - .../install_ramble_python_deps.yml.tftpl | 28 - .../templates/ramble_setup.yml.tftpl | 157 -- .../modules/scripts/ramble-setup/variables.tf | 97 - .../modules/scripts/ramble-setup/versions.tf | 30 - .../modules/scripts/spack-execute/README.md | 141 -- .../modules/scripts/spack-execute/main.tf | 70 - .../scripts/spack-execute/metadata.yaml | 18 - .../modules/scripts/spack-execute/outputs.tf | 45 - .../templates/execute_commands.yml.tpl | 59 - .../scripts/spack-execute/variables.tf | 103 - .../modules/scripts/spack-execute/versions.tf | 25 - .../modules/scripts/spack-setup/README.md | 382 --- .../modules/scripts/spack-setup/main.tf | 120 - .../modules/scripts/spack-setup/metadata.yaml | 19 - .../modules/scripts/spack-setup/outputs.tf | 56 - .../scripts/install_spack_deps.yml | 50 - .../templates/spack_setup.yml.tftpl | 157 -- .../modules/scripts/spack-setup/variables.tf | 106 - .../modules/scripts/spack-setup/versions.tf | 30 - .../scripts/wait-for-startup/README.md | 87 - .../modules/scripts/wait-for-startup/main.tf | 47 - .../scripts/wait-for-startup/metadata.yaml | 19 - .../scripts/wait-for-startup/outputs.tf | 15 - .../scripts/wait-for-startup-status.sh | 138 - .../scripts/wait-for-startup/variables.tf | 54 - .../scripts/wait-for-startup/versions.tf | 29 - .../scripts/windows-startup-script/README.md | 109 - .../scripts/windows-startup-script/main.tf | 34 - .../windows-startup-script/metadata.yaml | 18 - .../scripts/windows-startup-script/outputs.tf | 20 - .../templates/install_gpu_driver.ps1.tftpl | 38 - .../templates/setx_http_proxy.ps1 | 21 - .../windows-startup-script/variables.tf | 54 - .../windows-startup-script/versions.tf | 23 - .../modules/embedded/modules/README.md | 554 ---- .../compute/gke-job-template/README.md | 133 - .../modules/compute/gke-job-template/main.tf | 181 -- .../compute/gke-job-template/metadata.yaml | 18 - .../compute/gke-job-template/outputs.tf | 27 - .../templates/gke-job-base.yaml.tftpl | 128 - .../compute/gke-job-template/variables.tf | 206 -- .../compute/gke-job-template/versions.tf | 28 - .../modules/compute/gke-node-pool/README.md | 388 --- .../compute/gke-node-pool/disk_definitions.tf | 38 - .../sample-tcpx-workload-job.yaml | 50 - .../sample-tcpxo-workload-job.yaml | 70 - .../scripts/enable-tcpx-in-workload.py | 185 -- .../scripts/enable-tcpxo-in-workload.py | 186 -- .../compute/gke-node-pool/gpu_direct.tf | 87 - .../compute/gke-node-pool/guest_cpus.tf | 32 - .../modules/compute/gke-node-pool/main.tf | 482 ---- .../compute/gke-node-pool/metadata.yaml | 21 - .../modules/compute/gke-node-pool/outputs.tf | 152 -- .../gke-node-pool/reservation_definitions.tf | 107 - .../gke-node-pool/threads_per_core_calc.tf | 42 - .../compute/gke-node-pool/variables.tf | 487 ---- .../modules/compute/gke-node-pool/versions.tf | 38 - .../modules/compute/resource-policy/README.md | 82 - .../modules/compute/resource-policy/main.tf | 48 - .../compute/resource-policy/metadata.yaml | 19 - .../compute/resource-policy/outputs.tf | 30 - .../compute/resource-policy/variables.tf | 64 - .../compute/resource-policy/versions.tf | 34 - .../modules/compute/vm-instance/README.md | 257 -- .../compute/vm-instance/compute_image.tf | 30 - .../modules/compute/vm-instance/main.tf | 334 --- .../modules/compute/vm-instance/metadata.yaml | 19 - .../modules/compute/vm-instance/outputs.tf | 50 - .../startup_from_network_storage.tf | 65 - .../vm-instance/threads_per_core_calc.tf | 42 - .../modules/compute/vm-instance/variables.tf | 452 ---- .../modules/compute/vm-instance/versions.tf | 41 - .../cloud-storage-bucket/README.md | 170 -- .../file-system/cloud-storage-bucket/main.tf | 126 - .../cloud-storage-bucket/metadata.yaml | 18 - .../cloud-storage-bucket/outputs.tf | 69 - .../scripts/install-gcs-fuse.sh | 44 - .../cloud-storage-bucket/scripts/mount.sh | 58 - .../cloud-storage-bucket/variables.tf | 254 -- .../cloud-storage-bucket/versions.tf | 39 - .../modules/file-system/filestore/README.md | 248 -- .../modules/file-system/filestore/main.tf | 116 - .../file-system/filestore/metadata.yaml | 19 - .../modules/file-system/filestore/outputs.tf | 62 - .../filestore/scripts/install-nfs-client.sh | 37 - .../file-system/filestore/scripts/mount.sh | 58 - .../file-system/filestore/variables.tf | 189 -- .../modules/file-system/filestore/versions.tf | 36 - .../gke-persistent-volume/README.md | 200 -- .../file-system/gke-persistent-volume/main.tf | 155 -- .../gke-persistent-volume/metadata.yaml | 18 - .../gke-persistent-volume/outputs.tf | 31 - .../templates/filestore-pv.yaml.tftpl | 26 - .../templates/filestore-pvc.yaml.tftpl | 18 - .../templates/gcs-pv.yaml.tftpl | 24 - .../templates/gcs-pvc.yaml.tftpl | 21 - .../templates/managed-lustre-pv.yaml.tftpl | 26 - .../templates/managed-lustre-pvc.yaml.tftpl | 18 - .../templates/namespace.yaml.tftpl | 5 - .../gke-persistent-volume/variables.tf | 93 - .../gke-persistent-volume/versions.tf | 30 - .../modules/file-system/gke-storage/README.md | 134 - .../modules/file-system/gke-storage/main.tf | 86 - .../file-system/gke-storage/metadata.yaml | 18 - .../file-system/gke-storage/outputs.tf | 28 - .../hyperdisk-balanced-pvc.yaml.tftpl | 17 - .../hyperdisk-extreme-pvc.yaml.tftpl | 17 - .../hyperdisk-throughput-pvc.yaml.tftpl | 17 - .../namespace.yaml.tftpl | 5 - .../parallelstore-pvc.yaml.tftpl | 17 - .../hyperdisk-balanced-sc.yaml.tftpl | 25 - .../hyperdisk-extreme-sc.yaml.tftpl | 24 - .../hyperdisk-throughput-sc.yaml.tftpl | 24 - .../storage-class/parallelstore-sc.yaml.tftpl | 21 - .../file-system/gke-storage/variables.tf | 144 -- .../file-system/gke-storage/versions.tf | 21 - .../file-system/managed-lustre/README.md | 289 --- .../file-system/managed-lustre/main.tf | 104 - .../file-system/managed-lustre/metadata.yaml | 19 - .../file-system/managed-lustre/outputs.tf | 43 - .../scripts/install-managed-lustre-client.sh | 84 - .../managed-lustre/scripts/mount.sh | 58 - .../file-system/managed-lustre/variables.tf | 131 - .../file-system/managed-lustre/versions.tf | 36 - .../file-system/netapp-storage-pool/README.md | 193 -- .../file-system/netapp-storage-pool/main.tf | 56 - .../netapp-storage-pool/metadata.yaml | 20 - .../netapp-storage-pool/outputs.tf | 23 - .../netapp-storage-pool/variables.tf | 133 - .../netapp-storage-pool/versions.tf | 37 - .../file-system/netapp-volume/README.md | 201 -- .../modules/file-system/netapp-volume/main.tf | 92 - .../file-system/netapp-volume/metadata.yaml | 19 - .../file-system/netapp-volume/outputs.tf | 66 - .../scripts/install-nfs-client.sh | 37 - .../netapp-volume/scripts/mount.sh | 66 - .../file-system/netapp-volume/variables.tf | 133 - .../file-system/netapp-volume/versions.tf | 32 - .../file-system/parallelstore/README.md | 196 -- .../modules/file-system/parallelstore/main.tf | 74 - .../file-system/parallelstore/metadata.yaml | 19 - .../file-system/parallelstore/outputs.tf | 47 - .../scripts/install-daos-client.sh | 112 - .../templates/mount-daos.sh.tftpl | 110 - .../file-system/parallelstore/variables.tf | 137 - .../file-system/parallelstore/versions.tf | 36 - .../pre-existing-network-storage/README.md | 192 -- .../metadata.yaml | 18 - .../pre-existing-network-storage/outputs.tf | 124 - .../scripts/install-daos-client.sh | 112 - .../scripts/install-gcs-fuse.sh | 44 - .../scripts/install-managed-lustre-client.sh | 84 - .../scripts/install-nfs-client.sh | 37 - .../scripts/mount.sh | 58 - .../ddn_exascaler_luster_client_install.tftpl | 50 - .../templates/mount-daos.sh.tftpl | 110 - .../pre-existing-network-storage/variables.tf | 67 - .../pre-existing-network-storage/versions.tf | 19 - .../modules/internal/gpu-definition/README.md | 47 - .../modules/internal/gpu-definition/main.tf | 98 - .../internal/instance_validations/README.md | 30 - .../internal/instance_validations/main.tf | 52 - .../instance_validations/variables.tf | 23 - .../internal/instance_validations/versions.tf | 17 - .../internal/network-attachment/README.md | 54 - .../internal/network-attachment/main.tf | 70 - .../internal/network-attachment/metadata.yaml | 19 - .../modules/internal/tpu-definition/README.md | 85 - .../modules/internal/tpu-definition/main.tf | 69 - .../internal/tpu-definition/outputs.tf | 40 - .../internal/tpu-definition/variables.tf | 29 - .../modules/internal/vpc_peering/README.md | 56 - .../modules/internal/vpc_peering/main.tf | 80 - .../internal/vpc_peering/metadata.yaml | 19 - .../management/kubectl-apply/README.md | 244 -- .../kubectl-apply/helm_install/README.md | 64 - .../kubectl-apply/helm_install/main.tf | 79 - .../kubectl-apply/helm_install/metadata.yaml | 19 - .../kubectl-apply/helm_install/variables.tf | 212 -- .../kubectl-apply/helm_install/versions.tf | 24 - .../jobset/jobset-helm-values.yaml | 25 - .../kubectl-apply/kubectl/README.md | 55 - .../management/kubectl-apply/kubectl/main.tf | 92 - .../kubectl-apply/kubectl/metadata.yaml | 19 - .../kubectl-apply/kubectl/variables.tf | 51 - .../kubectl-apply/kubectl/versions.tf | 26 - .../kueue/kueue-helm-values.yaml | 30 - .../modules/management/kubectl-apply/main.tf | 271 -- .../management/kubectl-apply/metadata.yaml | 19 - .../management/kubectl-apply/providers.tf | 33 - .../management/kubectl-apply/variables.tf | 192 -- .../management/kubectl-apply/versions.tf | 42 - .../modules/monitoring/dashboard/README.md | 86 - .../dashboard/dashboards/Empty.json.tpl | 17 - .../dashboard/dashboards/HPC.json.tpl | 595 ----- .../modules/monitoring/dashboard/main.tf | 35 - .../monitoring/dashboard/metadata.yaml | 19 - .../modules/monitoring/dashboard/outputs.tf | 23 - .../modules/monitoring/dashboard/variables.tf | 52 - .../modules/monitoring/dashboard/versions.tf | 29 - .../modules/network/firewall-rules/README.md | 111 - .../modules/network/firewall-rules/main.tf | 60 - .../network/firewall-rules/metadata.yaml | 19 - .../network/firewall-rules/variables.tf | 88 - .../network/firewall-rules/versions.tf | 29 - .../modules/network/gpu-rdma-vpc/README.md | 143 -- .../modules/network/gpu-rdma-vpc/main.tf | 79 - .../network/gpu-rdma-vpc/metadata.yaml | 19 - .../modules/network/gpu-rdma-vpc/outputs.tf | 59 - .../modules/network/gpu-rdma-vpc/variables.tf | 164 -- .../modules/network/gpu-rdma-vpc/versions.tf | 19 - .../modules/network/multivpc/README.md | 136 - .../embedded/modules/network/multivpc/main.tf | 78 - .../modules/network/multivpc/metadata.yaml | 19 - .../modules/network/multivpc/outputs.tf | 50 - .../modules/network/multivpc/variables.tf | 201 -- .../modules/network/multivpc/versions.tf | 19 - .../network/pre-existing-subnetwork/README.md | 94 - .../network/pre-existing-subnetwork/main.tf | 38 - .../pre-existing-subnetwork/metadata.yaml | 21 - .../pre-existing-subnetwork/outputs.tf | 35 - .../pre-existing-subnetwork/variables.tf | 39 - .../pre-existing-subnetwork/versions.tf | 29 - .../network/pre-existing-vpc/README.md | 110 - .../modules/network/pre-existing-vpc/main.tf | 53 - .../network/pre-existing-vpc/metadata.yaml | 19 - .../network/pre-existing-vpc/outputs.tf | 50 - .../network/pre-existing-vpc/variables.tf | 37 - .../network/pre-existing-vpc/versions.tf | 29 - .../embedded/modules/network/vpc/README.md | 237 -- .../embedded/modules/network/vpc/main.tf | 256 -- .../modules/network/vpc/metadata.yaml | 19 - .../embedded/modules/network/vpc/outputs.tf | 68 - .../embedded/modules/network/vpc/variables.tf | 301 --- .../embedded/modules/network/vpc/versions.tf | 19 - .../modules/packer/custom-image/README.md | 320 --- .../modules/packer/custom-image/image.pkr.hcl | 216 -- .../modules/packer/custom-image/metadata.yaml | 21 - .../packer/custom-image/variables.pkr.hcl | 276 -- .../packer/custom-image/versions.pkr.hcl | 25 - .../scheduler/batch-job-template/README.md | 197 -- .../batch-job-template/compute_image.tf | 30 - .../scheduler/batch-job-template/main.tf | 149 -- .../batch-job-template/metadata.yaml | 22 - .../scheduler/batch-job-template/outputs.tf | 80 - .../startup_from_network_storage.tf | 65 - .../templates/batch-job-base.yaml.tftpl | 53 - .../templates/batch-submit.sh.tftpl | 10 - .../scheduler/batch-job-template/variables.tf | 240 -- .../scheduler/batch-job-template/versions.tf | 37 - .../scheduler/batch-login-node/README.md | 127 - .../scheduler/batch-login-node/main.tf | 127 - .../scheduler/batch-login-node/metadata.yaml | 21 - .../scheduler/batch-login-node/outputs.tf | 37 - .../scheduler/batch-login-node/variables.tf | 151 -- .../scheduler/batch-login-node/versions.tf | 29 - .../modules/scheduler/gke-cluster/README.md | 220 -- .../modules/scheduler/gke-cluster/main.tf | 470 ---- .../scheduler/gke-cluster/metadata.yaml | 19 - .../modules/scheduler/gke-cluster/outputs.tf | 104 - .../templates/gke-network-paramset.yaml.tftpl | 9 - .../templates/network-object.yaml.tftpl | 11 - .../scheduler/gke-cluster/variables.tf | 533 ---- .../modules/scheduler/gke-cluster/versions.tf | 39 - .../pre-existing-gke-cluster/README.md | 116 - .../pre-existing-gke-cluster/main.tf | 70 - .../pre-existing-gke-cluster/metadata.yaml | 19 - .../pre-existing-gke-cluster/outputs.tf | 33 - .../templates/gke-network-paramset.yaml.tftpl | 9 - .../templates/network-object.yaml.tftpl | 11 - .../pre-existing-gke-cluster/variables.tf | 61 - .../pre-existing-gke-cluster/versions.tf | 30 - .../modules/scripts/startup-script/README.md | 355 --- .../startup-script/files/configure-ssh.yml | 37 - .../startup-script/files/configure_proxy.sh | 54 - .../files/early_run_hotfixes.sh | 32 - .../startup-script/files/get_from_bucket.sh | 73 - .../startup-script/files/install_ansible.sh | 247 -- .../files/install_cloud_rdma_drivers.sh | 38 - .../startup-script/files/install_docker.yml | 113 - .../files/install_gpu_network_wait_online.yml | 56 - .../files/install_managed_lustre.yml | 33 - .../files/install_monitoring_agent.sh | 144 -- .../files/running-script-warning.sh | 26 - .../startup-script/files/setup-raid.yml | 100 - .../startup-script/files/setup-ssh-keys.sh | 19 - .../startup-script/files/setup-ssh-keys.yml | 40 - .../files/startup-script-stdlib-body.sh | 39 - .../files/startup-script-stdlib-head.sh | 266 -- .../modules/scripts/startup-script/main.tf | 306 --- .../scripts/startup-script/metadata.yaml | 19 - .../modules/scripts/startup-script/outputs.tf | 39 - .../templates/startup-script-custom.tftpl | 65 - .../scripts/startup-script/variables.tf | 298 --- .../scripts/startup-script/versions.tf | 37 - deletion-test/cluster/outputs.tf | 20 - deletion-test/cluster/providers.tf | 27 - deletion-test/cluster/variables.tf | 120 - deletion-test/cluster/versions.tf | 30 - deletion-test/instructions.txt | 60 - deletion-test/primary/main.tf | 40 - .../embedded/community/modules/README.md | 7 - .../modules/compute/gke-nodeset/README.md | 55 - .../modules/compute/gke-nodeset/main.tf | 64 - .../modules/compute/gke-nodeset/metadata.yaml | 20 - .../modules/compute/gke-nodeset/output.tf | 18 - .../compute/gke-nodeset/persistent_volumes.tf | 50 - .../templates/nodeset-general.yaml.tftpl | 203 -- .../modules/compute/gke-nodeset/variables.tf | 118 - .../modules/compute/gke-nodeset/versions.tf | 27 - .../modules/compute/gke-partition/README.md | 39 - .../modules/compute/gke-partition/main.tf | 47 - .../compute/gke-partition/metadata.yaml | 19 - .../compute/gke-partition/variables.tf | 43 - .../modules/compute/gke-partition/versions.tf | 27 - .../compute/htcondor-execute-point/README.md | 271 -- .../htcondor-execute-point/compute_image.tf | 30 - .../files/htcondor_configure.yml | 74 - .../files/htcondor_configure_autoscaler.yml | 98 - .../compute/htcondor-execute-point/main.tf | 218 -- .../htcondor-execute-point/metadata.yaml | 20 - .../compute/htcondor-execute-point/outputs.tf | 25 - .../templates/condor_config.tftpl | 31 - .../download-condor-config.ps1.tftpl | 34 - .../htcondor-execute-point/variables.tf | 265 -- .../htcondor-execute-point/versions.tf | 34 - .../community/modules/compute/mig/README.md | 45 - .../community/modules/compute/mig/main.tf | 85 - .../modules/compute/mig/metadata.yaml | 21 - .../community/modules/compute/mig/outputs.tf | 18 - .../modules/compute/mig/variables.tf | 86 - .../community/modules/compute/mig/versions.tf | 27 - .../modules/compute/notebook/README.md | 112 - .../modules/compute/notebook/main.tf | 96 - .../modules/compute/notebook/metadata.yaml | 20 - .../modules/compute/notebook/variables.tf | 111 - .../modules/compute/notebook/versions.tf | 29 - .../README.md | 135 - .../main.tf | 128 - .../metadata.yaml | 20 - .../outputs.tf | 36 - .../source_image_logic.tf | 30 - .../variables.tf | 402 --- .../versions.tf | 22 - .../README.md | 85 - .../schedmd-slurm-gcp-v6-nodeset-tpu/main.tf | 59 - .../metadata.yaml | 21 - .../outputs.tf | 39 - .../variables.tf | 171 -- .../versions.tf | 23 - .../schedmd-slurm-gcp-v6-nodeset/README.md | 227 -- .../schedmd-slurm-gcp-v6-nodeset/main.tf | 232 -- .../metadata.yaml | 21 - .../schedmd-slurm-gcp-v6-nodeset/outputs.tf | 112 - .../source_image_logic.tf | 30 - .../schedmd-slurm-gcp-v6-nodeset/variables.tf | 641 ----- .../schedmd-slurm-gcp-v6-nodeset/versions.tf | 29 - .../schedmd-slurm-gcp-v6-partition/README.md | 105 - .../schedmd-slurm-gcp-v6-partition/main.tf | 41 - .../metadata.yaml | 20 - .../schedmd-slurm-gcp-v6-partition/outputs.tf | 54 - .../variables.tf | 311 --- .../versions.tf | 23 - .../container/artifact-registry/README.md | 157 -- .../container/artifact-registry/main.tf | 268 -- .../container/artifact-registry/metadata.yaml | 21 - .../container/artifact-registry/outputs.tf | 18 - .../container/artifact-registry/validation.tf | 49 - .../container/artifact-registry/variables.tf | 122 - .../container/artifact-registry/versions.tf | 27 - .../database/bigquery-dataset/README.md | 76 - .../modules/database/bigquery-dataset/main.tf | 32 - .../database/bigquery-dataset/metadata.yaml | 19 - .../database/bigquery-dataset/outputs.tf | 20 - .../database/bigquery-dataset/variables.tf | 36 - .../database/bigquery-dataset/versions.tf | 29 - .../modules/database/bigquery-table/README.md | 87 - .../modules/database/bigquery-table/main.tf | 37 - .../database/bigquery-table/metadata.yaml | 19 - .../database/bigquery-table/outputs.tf | 28 - .../database/bigquery-table/variables.tf | 46 - .../database/bigquery-table/versions.tf | 29 - .../slurm-cloudsql-federation/README.md | 107 - .../slurm-cloudsql-federation/main.tf | 165 -- .../slurm-cloudsql-federation/metadata.yaml | 21 - .../slurm-cloudsql-federation/outputs.tf | 27 - .../slurm-cloudsql-federation/variables.tf | 173 -- .../slurm-cloudsql-federation/versions.tf | 36 - .../file-system/DDN-EXAScaler/README.md | 158 -- .../modules/file-system/DDN-EXAScaler/main.tf | 72 - .../file-system/DDN-EXAScaler/metadata.yaml | 22 - .../file-system/DDN-EXAScaler/outputs.tf | 90 - .../file-system/DDN-EXAScaler/variables.tf | 502 ---- .../file-system/DDN-EXAScaler/versions.tf | 24 - .../modules/file-system/Intel-DAOS/README.md | 1 - .../modules/file-system/nfs-server/README.md | 152 -- .../modules/file-system/nfs-server/main.tf | 131 - .../file-system/nfs-server/metadata.yaml | 19 - .../modules/file-system/nfs-server/outputs.tf | 53 - .../nfs-server/scripts/install-nfs-client.sh | 37 - .../scripts/install-nfs-server.sh.tpl | 35 - .../file-system/nfs-server/scripts/mount.sh | 58 - .../file-system/nfs-server/scripts/mount.yaml | 39 - .../file-system/nfs-server/variables.tf | 194 -- .../file-system/nfs-server/versions.tf | 37 - .../file-system/sycomp-scale/README.md | 35 - .../modules/file-system/weka-client/README.md | 182 -- .../file-system/weka-client/metadata.yaml | 18 - .../file-system/weka-client/outputs.tf | 71 - .../templates/install-weka-client.yaml.tftpl | 133 - .../weka-client/templates/mount-weka.sh.tftpl | 101 - .../templates/mount-weka.yaml.tftpl | 54 - .../file-system/weka-client/variables.tf | 39 - .../file-system/weka-client/versions.tf | 19 - .../FSI_MonteCarlo.ipynb | 125 - .../files/fsi-montecarlo-on-batch/README.md | 97 - .../fsi-montecarlo-on-batch/iteration.sh | 23 - .../files/fsi-montecarlo-on-batch/main.tf | 102 - .../fsi-montecarlo-on-batch/mc_run.tpl.py | 157 -- .../fsi-montecarlo-on-batch/mc_run.tpl.yaml | 36 - .../fsi-montecarlo-on-batch/mc_run_reqs.txt | 9 - .../fsi-montecarlo-on-batch/metadata.yaml | 18 - .../fsi-montecarlo-on-batch/variables.tf | 51 - .../files/fsi-montecarlo-on-batch/versions.tf | 43 - .../internal/slurm-gcp/instance/README.md | 100 - .../internal/slurm-gcp/instance/main.tf | 126 - .../internal/slurm-gcp/instance/outputs.tf | 41 - .../internal/slurm-gcp/instance/variables.tf | 119 - .../internal/slurm-gcp/instance/versions.tf | 31 - .../slurm-gcp/instance_template/README.md | 87 - .../files/startup_sh_unlinted | 169 -- .../slurm-gcp/instance_template/main.tf | 171 -- .../slurm-gcp/instance_template/outputs.tf | 43 - .../slurm-gcp/instance_template/variables.tf | 431 ---- .../slurm-gcp/instance_template/versions.tf | 25 - .../internal_instance_template/README.md | 89 - .../internal_instance_template/main.tf | 234 -- .../internal_instance_template/outputs.tf | 33 - .../internal_instance_template/variables.tf | 398 --- .../internal_instance_template/versions.tf | 30 - .../internal/slurm-gcp/login/README.md | 52 - .../modules/internal/slurm-gcp/login/main.tf | 112 - .../internal/slurm-gcp/login/outputs.tf | 25 - .../internal/slurm-gcp/login/variables.tf | 188 -- .../internal/slurm-gcp/login/versions.tf | 29 - .../internal/slurm-gcp/nodeset_tpu/README.md | 95 - .../internal/slurm-gcp/nodeset_tpu/main.tf | 121 - .../internal/slurm-gcp/nodeset_tpu/outputs.tf | 30 - .../slurm-gcp/nodeset_tpu/variables.tf | 158 -- .../slurm-gcp/nodeset_tpu/versions.tf | 30 - .../dependencies-installer/README.md | 61 - .../helm_install/README.md | 64 - .../helm_install/main.tf | 75 - .../helm_install/metadata.yaml | 19 - .../helm_install/variables.tf | 212 -- .../helm_install/versions.tf | 24 - .../kubernetes_manifest/README.md | 40 - .../kubernetes_manifest/main.tf | 104 - .../kubernetes_manifest/metadata.yaml | 19 - .../kubernetes_manifest/variables.tf | 69 - .../kubernetes_manifest/versions.tf | 24 - .../management/dependencies-installer/main.tf | 183 -- .../dependencies-installer/metadata.yaml | 19 - .../dependencies-installer/providers.tf | 25 - .../dependencies-installer/variables.tf | 70 - .../dependencies-installer/versions.tf | 30 - .../network/private-service-access/README.md | 122 - .../network/private-service-access/main.tf | 61 - .../private-service-access/metadata.yaml | 20 - .../network/private-service-access/outputs.tf | 43 - .../private-service-access/variables.tf | 59 - .../private-service-access/versions.tf | 37 - .../modules/project/new-project/README.md | 128 - .../modules/project/service-account/README.md | 111 - .../modules/project/service-account/main.tf | 37 - .../project/service-account/metadata.yaml | 19 - .../project/service-account/outputs.tf | 36 - .../project/service-account/variables.tf | 113 - .../project/service-account/versions.tf | 22 - .../project/service-enablement/README.md | 70 - .../project/service-enablement/main.tf | 28 - .../project/service-enablement/metadata.yaml | 19 - .../project/service-enablement/variables.tf | 31 - .../project/service-enablement/versions.tf | 29 - .../modules/pubsub/bigquery-sub/README.md | 87 - .../modules/pubsub/bigquery-sub/main.tf | 57 - .../modules/pubsub/bigquery-sub/metadata.yaml | 19 - .../modules/pubsub/bigquery-sub/outputs.tf | 20 - .../modules/pubsub/bigquery-sub/variables.tf | 51 - .../modules/pubsub/bigquery-sub/versions.tf | 35 - .../community/modules/pubsub/topic/README.md | 82 - .../community/modules/pubsub/topic/main.tf | 48 - .../modules/pubsub/topic/metadata.yaml | 19 - .../community/modules/pubsub/topic/outputs.tf | 26 - .../modules/pubsub/topic/variables.tf | 74 - .../modules/pubsub/topic/versions.tf | 32 - .../chrome-remote-desktop/README.md | 113 - .../chrome-remote-desktop/main.tf | 111 - .../chrome-remote-desktop/metadata.yaml | 18 - .../chrome-remote-desktop/outputs.tf | 25 - .../scripts/configure-chrome-desktop.yml | 61 - .../scripts/configure-grid-drivers.yml | 163 -- .../scripts/disable-sleep.yml | 39 - .../chrome-remote-desktop/variables.tf | 277 -- .../chrome-remote-desktop/versions.tf | 19 - .../scheduler/htcondor-access-point/README.md | 187 -- .../files/htcondor_configure.yml | 120 - .../scheduler/htcondor-access-point/main.tf | 338 --- .../htcondor-access-point/metadata.yaml | 20 - .../htcondor-access-point/outputs.tf | 25 - .../templates/condor_config.tftpl | 70 - .../htcondor-access-point/variables.tf | 266 -- .../htcondor-access-point/versions.tf | 37 - .../htcondor-central-manager/README.md | 159 -- .../files/htcondor_configure.yml | 72 - .../htcondor-central-manager/main.tf | 226 -- .../htcondor-central-manager/metadata.yaml | 20 - .../htcondor-central-manager/outputs.tf | 30 - .../templates/condor_config.tftpl | 31 - .../htcondor-central-manager/variables.tf | 192 -- .../htcondor-central-manager/versions.tf | 33 - .../scheduler/htcondor-pool-secrets/README.md | 172 -- .../files/htcondor_secrets.yml | 102 - .../scheduler/htcondor-pool-secrets/main.tf | 168 -- .../htcondor-pool-secrets/metadata.yaml | 20 - .../htcondor-pool-secrets/outputs.tf | 50 - .../templates/fetch-idtoken.ps1.tftpl | 26 - .../htcondor-pool-secrets/variables.tf | 67 - .../htcondor-pool-secrets/versions.tf | 33 - .../htcondor-service-accounts/README.md | 128 - .../htcondor-service-accounts/main.tf | 51 - .../htcondor-service-accounts/metadata.yaml | 19 - .../htcondor-service-accounts/outputs.tf | 30 - .../htcondor-service-accounts/variables.tf | 56 - .../htcondor-service-accounts/versions.tf | 19 - .../scheduler/htcondor-setup/README.md | 118 - .../modules/scheduler/htcondor-setup/main.tf | 68 - .../scheduler/htcondor-setup/metadata.yaml | 21 - .../scheduler/htcondor-setup/outputs.tf | 27 - .../scheduler/htcondor-setup/variables.tf | 55 - .../scheduler/htcondor-setup/versions.tf | 19 - .../schedmd-slurm-gcp-v6-controller/README.md | 405 --- .../controller.tf | 213 -- .../etc/htc-slurm.conf.tpl | 65 - .../etc/htc-slurmdbd.conf.tpl | 34 - .../etc/long-prolog-slurm.conf.tpl | 71 - .../schedmd-slurm-gcp-v6-controller/login.tf | 50 - .../schedmd-slurm-gcp-v6-controller/main.tf | 35 - .../metadata.yaml | 21 - .../modules/cleanup_compute/README.md | 42 - .../modules/cleanup_compute/main.tf | 46 - .../scripts/cleanup_compute.sh | 100 - .../modules/cleanup_compute/variables.tf | 71 - .../modules/cleanup_compute/versions.tf | 27 - .../modules/cleanup_tpu/README.md | 79 - .../modules/cleanup_tpu/main.tf | 32 - .../cleanup_tpu/scripts/cleanup_tpu.sh | 63 - .../modules/cleanup_tpu/variables.tf | 60 - .../modules/cleanup_tpu/versions.tf | 27 - .../modules/slurm_files/README.md | 121 - .../modules/slurm_files/etc/cgroup.conf.tpl | 7 - .../modules/slurm_files/etc/slurm.conf.tpl | 67 - .../modules/slurm_files/etc/slurmdbd.conf.tpl | 31 - .../slurm_files/files/external_epilog.sh | 18 - .../slurm_files/files/external_prolog.sh | 18 - .../slurm_files/files/setup_external.sh | 117 - .../modules/slurm_files/main.tf | 406 --- .../modules/slurm_files/outputs.tf | 45 - .../modules/slurm_files/scripts/conf.py | 658 ----- .../modules/slurm_files/scripts/file_cache.py | 80 - .../slurm_files/scripts/get_tpu_vmcount.py | 76 - .../slurm_files/scripts/job_submit.lua.tpl | 103 - .../modules/slurm_files/scripts/load_bq.py | 352 --- .../slurm_files/scripts/local_pubsub.py | 196 -- .../modules/slurm_files/scripts/mig_flex.py | 254 -- .../slurm_files/scripts/requirements-dev.txt | 9 - .../slurm_files/scripts/requirements.txt | 18 - .../modules/slurm_files/scripts/resume.py | 703 ------ .../slurm_files/scripts/resume_wrapper.sh | 40 - .../modules/slurm_files/scripts/setup.py | 660 ----- .../scripts/setup_network_storage.py | 327 --- .../modules/slurm_files/scripts/slurmsync.py | 679 ----- .../modules/slurm_files/scripts/sort_nodes.py | 171 -- .../modules/slurm_files/scripts/suspend.py | 126 - .../slurm_files/scripts/suspend_wrapper.sh | 28 - .../slurm_files/scripts/tests/common.py | 116 - .../slurm_files/scripts/tests/test_conf.py | 226 -- .../slurm_files/scripts/tests/test_resume.py | 175 -- .../scripts/tests/test_topology.py | 215 -- .../slurm_files/scripts/tests/test_util.py | 668 ----- .../slurm_files/scripts/tools/gpu-test | 133 - .../slurm_files/scripts/tools/task-epilog | 67 - .../slurm_files/scripts/tools/task-prolog | 70 - .../modules/slurm_files/scripts/tpu.py | 331 --- .../modules/slurm_files/scripts/util.py | 2224 ----------------- .../slurm_files/scripts/watch_delete_vm_op.py | 124 - .../modules/slurm_files/variables.tf | 504 ---- .../modules/slurm_files/versions.tf | 37 - .../outputs.tf | 62 - .../partition.tf | 174 -- .../slurm_files.tf | 191 -- .../source_image_logic.tf | 30 - .../variables.tf | 814 ------ .../variables_controller_instance.tf | 382 --- .../versions.tf | 33 - .../schedmd-slurm-gcp-v6-login/README.md | 130 - .../schedmd-slurm-gcp-v6-login/main.tf | 115 - .../schedmd-slurm-gcp-v6-login/metadata.yaml | 21 - .../schedmd-slurm-gcp-v6-login/outputs.tf | 18 - .../source_image_logic.tf | 30 - .../schedmd-slurm-gcp-v6-login/variables.tf | 419 ---- .../schedmd-slurm-gcp-v6-login/versions.tf | 23 - .../modules/scheduler/slinky/README.md | 172 -- .../modules/scheduler/slinky/main.tf | 197 -- .../modules/scheduler/slinky/metadata.yaml | 19 - .../modules/scheduler/slinky/outputs.tf | 23 - .../modules/scheduler/slinky/providers.tf | 23 - .../modules/scheduler/slinky/variables.tf | 127 - .../modules/scheduler/slinky/versions.tf | 28 - .../scripts/htcondor-install/README.md | 149 -- .../htcondor-install/files/autoscaler.py | 417 ---- .../install-htcondor-autoscaler-deps.yml | 46 - .../files/install-htcondor.yaml | 94 - .../modules/scripts/htcondor-install/main.tf | 51 - .../scripts/htcondor-install/metadata.yaml | 18 - .../scripts/htcondor-install/outputs.tf | 30 - .../templates/install-htcondor.ps1.tftpl | 59 - .../scripts/htcondor-install/variables.tf | 51 - .../scripts/htcondor-install/versions.tf | 19 - .../modules/scripts/ramble-execute/README.md | 116 - .../modules/scripts/ramble-execute/main.tf | 71 - .../scripts/ramble-execute/metadata.yaml | 18 - .../modules/scripts/ramble-execute/outputs.tf | 53 - .../templates/ramble_execute.yml.tpl | 59 - .../scripts/ramble-execute/variables.tf | 114 - .../scripts/ramble-execute/versions.tf | 25 - .../modules/scripts/ramble-setup/README.md | 128 - .../modules/scripts/ramble-setup/main.tf | 113 - .../scripts/ramble-setup/metadata.yaml | 18 - .../modules/scripts/ramble-setup/outputs.tf | 61 - .../scripts/install_ramble_deps.yml | 50 - .../install_ramble_python_deps.yml.tftpl | 28 - .../templates/ramble_setup.yml.tftpl | 157 -- .../modules/scripts/ramble-setup/variables.tf | 97 - .../modules/scripts/ramble-setup/versions.tf | 30 - .../modules/scripts/spack-execute/README.md | 141 -- .../modules/scripts/spack-execute/main.tf | 70 - .../scripts/spack-execute/metadata.yaml | 18 - .../modules/scripts/spack-execute/outputs.tf | 45 - .../templates/execute_commands.yml.tpl | 59 - .../scripts/spack-execute/variables.tf | 103 - .../modules/scripts/spack-execute/versions.tf | 25 - .../modules/scripts/spack-setup/README.md | 382 --- .../modules/scripts/spack-setup/main.tf | 120 - .../modules/scripts/spack-setup/metadata.yaml | 19 - .../modules/scripts/spack-setup/outputs.tf | 56 - .../scripts/install_spack_deps.yml | 50 - .../templates/spack_setup.yml.tftpl | 157 -- .../modules/scripts/spack-setup/variables.tf | 106 - .../modules/scripts/spack-setup/versions.tf | 30 - .../scripts/wait-for-startup/README.md | 87 - .../modules/scripts/wait-for-startup/main.tf | 47 - .../scripts/wait-for-startup/metadata.yaml | 19 - .../scripts/wait-for-startup/outputs.tf | 15 - .../scripts/wait-for-startup-status.sh | 138 - .../scripts/wait-for-startup/variables.tf | 54 - .../scripts/wait-for-startup/versions.tf | 29 - .../scripts/windows-startup-script/README.md | 109 - .../scripts/windows-startup-script/main.tf | 34 - .../windows-startup-script/metadata.yaml | 18 - .../scripts/windows-startup-script/outputs.tf | 20 - .../templates/install_gpu_driver.ps1.tftpl | 38 - .../templates/setx_http_proxy.ps1 | 21 - .../windows-startup-script/variables.tf | 54 - .../windows-startup-script/versions.tf | 23 - .../modules/embedded/modules/README.md | 554 ---- .../compute/gke-job-template/README.md | 133 - .../modules/compute/gke-job-template/main.tf | 181 -- .../compute/gke-job-template/metadata.yaml | 18 - .../compute/gke-job-template/outputs.tf | 27 - .../templates/gke-job-base.yaml.tftpl | 128 - .../compute/gke-job-template/variables.tf | 206 -- .../compute/gke-job-template/versions.tf | 28 - .../modules/compute/gke-node-pool/README.md | 388 --- .../compute/gke-node-pool/disk_definitions.tf | 38 - .../sample-tcpx-workload-job.yaml | 50 - .../sample-tcpxo-workload-job.yaml | 70 - .../scripts/enable-tcpx-in-workload.py | 185 -- .../scripts/enable-tcpxo-in-workload.py | 186 -- .../compute/gke-node-pool/gpu_direct.tf | 87 - .../compute/gke-node-pool/guest_cpus.tf | 32 - .../modules/compute/gke-node-pool/main.tf | 482 ---- .../compute/gke-node-pool/metadata.yaml | 21 - .../modules/compute/gke-node-pool/outputs.tf | 152 -- .../gke-node-pool/reservation_definitions.tf | 107 - .../gke-node-pool/threads_per_core_calc.tf | 42 - .../compute/gke-node-pool/variables.tf | 487 ---- .../modules/compute/gke-node-pool/versions.tf | 38 - .../modules/compute/resource-policy/README.md | 82 - .../modules/compute/resource-policy/main.tf | 48 - .../compute/resource-policy/metadata.yaml | 19 - .../compute/resource-policy/outputs.tf | 30 - .../compute/resource-policy/variables.tf | 64 - .../compute/resource-policy/versions.tf | 34 - .../modules/compute/vm-instance/README.md | 257 -- .../compute/vm-instance/compute_image.tf | 30 - .../modules/compute/vm-instance/main.tf | 334 --- .../modules/compute/vm-instance/metadata.yaml | 19 - .../modules/compute/vm-instance/outputs.tf | 50 - .../startup_from_network_storage.tf | 65 - .../vm-instance/threads_per_core_calc.tf | 42 - .../modules/compute/vm-instance/variables.tf | 452 ---- .../modules/compute/vm-instance/versions.tf | 41 - .../cloud-storage-bucket/README.md | 170 -- .../file-system/cloud-storage-bucket/main.tf | 126 - .../cloud-storage-bucket/metadata.yaml | 18 - .../cloud-storage-bucket/outputs.tf | 69 - .../scripts/install-gcs-fuse.sh | 44 - .../cloud-storage-bucket/scripts/mount.sh | 58 - .../cloud-storage-bucket/variables.tf | 254 -- .../cloud-storage-bucket/versions.tf | 39 - .../modules/file-system/filestore/README.md | 248 -- .../modules/file-system/filestore/main.tf | 116 - .../file-system/filestore/metadata.yaml | 19 - .../modules/file-system/filestore/outputs.tf | 62 - .../filestore/scripts/install-nfs-client.sh | 37 - .../file-system/filestore/scripts/mount.sh | 58 - .../file-system/filestore/variables.tf | 189 -- .../modules/file-system/filestore/versions.tf | 36 - .../gke-persistent-volume/README.md | 200 -- .../file-system/gke-persistent-volume/main.tf | 155 -- .../gke-persistent-volume/metadata.yaml | 18 - .../gke-persistent-volume/outputs.tf | 31 - .../templates/filestore-pv.yaml.tftpl | 26 - .../templates/filestore-pvc.yaml.tftpl | 18 - .../templates/gcs-pv.yaml.tftpl | 24 - .../templates/gcs-pvc.yaml.tftpl | 21 - .../templates/managed-lustre-pv.yaml.tftpl | 26 - .../templates/managed-lustre-pvc.yaml.tftpl | 18 - .../templates/namespace.yaml.tftpl | 5 - .../gke-persistent-volume/variables.tf | 93 - .../gke-persistent-volume/versions.tf | 30 - .../modules/file-system/gke-storage/README.md | 134 - .../modules/file-system/gke-storage/main.tf | 86 - .../file-system/gke-storage/metadata.yaml | 18 - .../file-system/gke-storage/outputs.tf | 28 - .../hyperdisk-balanced-pvc.yaml.tftpl | 17 - .../hyperdisk-extreme-pvc.yaml.tftpl | 17 - .../hyperdisk-throughput-pvc.yaml.tftpl | 17 - .../namespace.yaml.tftpl | 5 - .../parallelstore-pvc.yaml.tftpl | 17 - .../hyperdisk-balanced-sc.yaml.tftpl | 25 - .../hyperdisk-extreme-sc.yaml.tftpl | 24 - .../hyperdisk-throughput-sc.yaml.tftpl | 24 - .../storage-class/parallelstore-sc.yaml.tftpl | 21 - .../file-system/gke-storage/variables.tf | 144 -- .../file-system/gke-storage/versions.tf | 21 - .../file-system/managed-lustre/README.md | 289 --- .../file-system/managed-lustre/main.tf | 104 - .../file-system/managed-lustre/metadata.yaml | 19 - .../file-system/managed-lustre/outputs.tf | 43 - .../scripts/install-managed-lustre-client.sh | 84 - .../managed-lustre/scripts/mount.sh | 58 - .../file-system/managed-lustre/variables.tf | 131 - .../file-system/managed-lustre/versions.tf | 36 - .../file-system/netapp-storage-pool/README.md | 193 -- .../file-system/netapp-storage-pool/main.tf | 56 - .../netapp-storage-pool/metadata.yaml | 20 - .../netapp-storage-pool/outputs.tf | 23 - .../netapp-storage-pool/variables.tf | 133 - .../netapp-storage-pool/versions.tf | 37 - .../file-system/netapp-volume/README.md | 201 -- .../modules/file-system/netapp-volume/main.tf | 92 - .../file-system/netapp-volume/metadata.yaml | 19 - .../file-system/netapp-volume/outputs.tf | 66 - .../scripts/install-nfs-client.sh | 37 - .../netapp-volume/scripts/mount.sh | 66 - .../file-system/netapp-volume/variables.tf | 133 - .../file-system/netapp-volume/versions.tf | 32 - .../file-system/parallelstore/README.md | 196 -- .../modules/file-system/parallelstore/main.tf | 74 - .../file-system/parallelstore/metadata.yaml | 19 - .../file-system/parallelstore/outputs.tf | 47 - .../scripts/install-daos-client.sh | 112 - .../templates/mount-daos.sh.tftpl | 110 - .../file-system/parallelstore/variables.tf | 137 - .../file-system/parallelstore/versions.tf | 36 - .../pre-existing-network-storage/README.md | 192 -- .../metadata.yaml | 18 - .../pre-existing-network-storage/outputs.tf | 124 - .../scripts/install-daos-client.sh | 112 - .../scripts/install-gcs-fuse.sh | 44 - .../scripts/install-managed-lustre-client.sh | 84 - .../scripts/install-nfs-client.sh | 37 - .../scripts/mount.sh | 58 - .../ddn_exascaler_luster_client_install.tftpl | 50 - .../templates/mount-daos.sh.tftpl | 110 - .../pre-existing-network-storage/variables.tf | 67 - .../pre-existing-network-storage/versions.tf | 19 - .../modules/internal/gpu-definition/README.md | 47 - .../modules/internal/gpu-definition/main.tf | 98 - .../internal/instance_validations/README.md | 30 - .../internal/instance_validations/main.tf | 52 - .../instance_validations/variables.tf | 23 - .../internal/instance_validations/versions.tf | 17 - .../internal/network-attachment/README.md | 54 - .../internal/network-attachment/main.tf | 70 - .../internal/network-attachment/metadata.yaml | 19 - .../modules/internal/tpu-definition/README.md | 85 - .../modules/internal/tpu-definition/main.tf | 69 - .../internal/tpu-definition/outputs.tf | 40 - .../internal/tpu-definition/variables.tf | 29 - .../modules/internal/vpc_peering/README.md | 56 - .../modules/internal/vpc_peering/main.tf | 80 - .../internal/vpc_peering/metadata.yaml | 19 - .../management/kubectl-apply/README.md | 244 -- .../kubectl-apply/helm_install/README.md | 64 - .../kubectl-apply/helm_install/main.tf | 79 - .../kubectl-apply/helm_install/metadata.yaml | 19 - .../kubectl-apply/helm_install/variables.tf | 212 -- .../kubectl-apply/helm_install/versions.tf | 24 - .../jobset/jobset-helm-values.yaml | 25 - .../kubectl-apply/kubectl/README.md | 55 - .../management/kubectl-apply/kubectl/main.tf | 92 - .../kubectl-apply/kubectl/metadata.yaml | 19 - .../kubectl-apply/kubectl/variables.tf | 51 - .../kubectl-apply/kubectl/versions.tf | 26 - .../kueue/kueue-helm-values.yaml | 30 - .../modules/management/kubectl-apply/main.tf | 271 -- .../management/kubectl-apply/metadata.yaml | 19 - .../management/kubectl-apply/providers.tf | 33 - .../management/kubectl-apply/variables.tf | 192 -- .../management/kubectl-apply/versions.tf | 42 - .../modules/monitoring/dashboard/README.md | 86 - .../dashboard/dashboards/Empty.json.tpl | 17 - .../dashboard/dashboards/HPC.json.tpl | 595 ----- .../modules/monitoring/dashboard/main.tf | 35 - .../monitoring/dashboard/metadata.yaml | 19 - .../modules/monitoring/dashboard/outputs.tf | 23 - .../modules/monitoring/dashboard/variables.tf | 52 - .../modules/monitoring/dashboard/versions.tf | 29 - .../modules/network/firewall-rules/README.md | 111 - .../modules/network/firewall-rules/main.tf | 60 - .../network/firewall-rules/metadata.yaml | 19 - .../network/firewall-rules/variables.tf | 88 - .../network/firewall-rules/versions.tf | 29 - .../modules/network/gpu-rdma-vpc/README.md | 143 -- .../modules/network/gpu-rdma-vpc/main.tf | 79 - .../network/gpu-rdma-vpc/metadata.yaml | 19 - .../modules/network/gpu-rdma-vpc/outputs.tf | 59 - .../modules/network/gpu-rdma-vpc/variables.tf | 164 -- .../modules/network/gpu-rdma-vpc/versions.tf | 19 - .../modules/network/multivpc/README.md | 136 - .../embedded/modules/network/multivpc/main.tf | 78 - .../modules/network/multivpc/metadata.yaml | 19 - .../modules/network/multivpc/outputs.tf | 50 - .../modules/network/multivpc/variables.tf | 201 -- .../modules/network/multivpc/versions.tf | 19 - .../network/pre-existing-subnetwork/README.md | 94 - .../network/pre-existing-subnetwork/main.tf | 38 - .../pre-existing-subnetwork/metadata.yaml | 21 - .../pre-existing-subnetwork/outputs.tf | 35 - .../pre-existing-subnetwork/variables.tf | 39 - .../pre-existing-subnetwork/versions.tf | 29 - .../network/pre-existing-vpc/README.md | 110 - .../modules/network/pre-existing-vpc/main.tf | 53 - .../network/pre-existing-vpc/metadata.yaml | 19 - .../network/pre-existing-vpc/outputs.tf | 50 - .../network/pre-existing-vpc/variables.tf | 37 - .../network/pre-existing-vpc/versions.tf | 29 - .../embedded/modules/network/vpc/README.md | 237 -- .../embedded/modules/network/vpc/main.tf | 256 -- .../modules/network/vpc/metadata.yaml | 19 - .../embedded/modules/network/vpc/outputs.tf | 68 - .../embedded/modules/network/vpc/variables.tf | 301 --- .../embedded/modules/network/vpc/versions.tf | 19 - .../modules/packer/custom-image/README.md | 320 --- .../modules/packer/custom-image/image.pkr.hcl | 216 -- .../modules/packer/custom-image/metadata.yaml | 21 - .../packer/custom-image/variables.pkr.hcl | 276 -- .../packer/custom-image/versions.pkr.hcl | 25 - .../scheduler/batch-job-template/README.md | 197 -- .../batch-job-template/compute_image.tf | 30 - .../scheduler/batch-job-template/main.tf | 149 -- .../batch-job-template/metadata.yaml | 22 - .../scheduler/batch-job-template/outputs.tf | 80 - .../startup_from_network_storage.tf | 65 - .../templates/batch-job-base.yaml.tftpl | 53 - .../templates/batch-submit.sh.tftpl | 10 - .../scheduler/batch-job-template/variables.tf | 240 -- .../scheduler/batch-job-template/versions.tf | 37 - .../scheduler/batch-login-node/README.md | 127 - .../scheduler/batch-login-node/main.tf | 127 - .../scheduler/batch-login-node/metadata.yaml | 21 - .../scheduler/batch-login-node/outputs.tf | 37 - .../scheduler/batch-login-node/variables.tf | 151 -- .../scheduler/batch-login-node/versions.tf | 29 - .../modules/scheduler/gke-cluster/README.md | 220 -- .../modules/scheduler/gke-cluster/main.tf | 470 ---- .../scheduler/gke-cluster/metadata.yaml | 19 - .../modules/scheduler/gke-cluster/outputs.tf | 104 - .../templates/gke-network-paramset.yaml.tftpl | 9 - .../templates/network-object.yaml.tftpl | 11 - .../scheduler/gke-cluster/variables.tf | 533 ---- .../modules/scheduler/gke-cluster/versions.tf | 39 - .../pre-existing-gke-cluster/README.md | 116 - .../pre-existing-gke-cluster/main.tf | 70 - .../pre-existing-gke-cluster/metadata.yaml | 19 - .../pre-existing-gke-cluster/outputs.tf | 33 - .../templates/gke-network-paramset.yaml.tftpl | 9 - .../templates/network-object.yaml.tftpl | 11 - .../pre-existing-gke-cluster/variables.tf | 61 - .../pre-existing-gke-cluster/versions.tf | 30 - .../modules/scripts/startup-script/README.md | 355 --- .../startup-script/files/configure-ssh.yml | 37 - .../startup-script/files/configure_proxy.sh | 54 - .../files/early_run_hotfixes.sh | 32 - .../startup-script/files/get_from_bucket.sh | 73 - .../startup-script/files/install_ansible.sh | 247 -- .../files/install_cloud_rdma_drivers.sh | 38 - .../startup-script/files/install_docker.yml | 113 - .../files/install_gpu_network_wait_online.yml | 56 - .../files/install_managed_lustre.yml | 33 - .../files/install_monitoring_agent.sh | 144 -- .../files/running-script-warning.sh | 26 - .../startup-script/files/setup-raid.yml | 100 - .../startup-script/files/setup-ssh-keys.sh | 19 - .../startup-script/files/setup-ssh-keys.yml | 40 - .../files/startup-script-stdlib-body.sh | 39 - .../files/startup-script-stdlib-head.sh | 266 -- .../modules/scripts/startup-script/main.tf | 306 --- .../scripts/startup-script/metadata.yaml | 19 - .../modules/scripts/startup-script/outputs.tf | 39 - .../templates/startup-script-custom.tftpl | 65 - .../scripts/startup-script/variables.tf | 298 --- .../scripts/startup-script/versions.tf | 37 - deletion-test/primary/outputs.tf | 37 - deletion-test/primary/providers.tf | 27 - deletion-test/primary/variables.tf | 55 - deletion-test/primary/versions.tf | 30 - .../slurm-build/slurm-image/README.md | 320 --- .../slurm-build/slurm-image/image.pkr.hcl | 216 -- .../slurm-build/slurm-image/metadata.yaml | 21 - .../slurm-build/slurm-image/variables.pkr.hcl | 276 -- .../slurm-build/slurm-image/versions.pkr.hcl | 25 - .../a3mega-slurm-deployment.yaml | 19 +- tools/cleanup.sh | 646 +++-- tools/cloud-build/project-cleanup.yaml | 13 +- tools/exclusions.txt | 49 +- 1935 files changed, 492 insertions(+), 189297 deletions(-) delete mode 100644 deletion-test/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY delete mode 100644 deletion-test/.ghpc/artifacts/expanded_blueprint.yaml delete mode 100644 deletion-test/.gitignore delete mode 100644 deletion-test/build_script/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/output.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/mig/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/notebook/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/notebook/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/notebook/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/notebook/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/notebook/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/validation.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/Intel-DAOS/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/sycomp-scale/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/providers.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/new-project/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-account/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables_controller_instance.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/providers.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/README.md delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpx-in-workload.py delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/resource-policy/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/compute_image.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/compute/vm-instance/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/mount.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/mount.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/filestore/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-managed-lustre-client.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/templates/mount-daos.sh.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/gpu-definition/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/gpu-definition/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/instance_validations/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/instance_validations/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/instance_validations/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/instance_validations/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/network-attachment/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/network-attachment/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/network-attachment/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/providers.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/firewall-rules/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/firewall-rules/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/firewall-rules/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/firewall-rules/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/firewall-rules/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/multivpc/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/network/vpc/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/packer/custom-image/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/packer/custom-image/image.pkr.hcl delete mode 100644 deletion-test/build_script/modules/embedded/modules/packer/custom-image/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/packer/custom-image/variables.pkr.hcl delete mode 100644 deletion-test/build_script/modules/embedded/modules/packer/custom-image/versions.pkr.hcl delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/README.md delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_docker.yml delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/main.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/metadata.yaml delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/outputs.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/variables.tf delete mode 100644 deletion-test/build_script/modules/embedded/modules/scripts/startup-script/versions.tf delete mode 100644 deletion-test/build_script/outputs.tf delete mode 100644 deletion-test/build_script/providers.tf delete mode 100644 deletion-test/build_script/variables.tf delete mode 100644 deletion-test/build_script/versions.tf delete mode 100644 deletion-test/cluster/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/output.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/mig/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/notebook/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/notebook/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/notebook/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/notebook/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/notebook/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/validation.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/Intel-DAOS/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/sycomp-scale/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/providers.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/new-project/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-account/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel-controller.zip delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel.zip delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables_controller_instance.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/providers.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/README.md delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpx-in-workload.py delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/resource-policy/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/compute_image.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/compute/vm-instance/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/mount.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/mount.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/filestore/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-managed-lustre-client.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/templates/mount-daos.sh.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/gpu-definition/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/gpu-definition/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/instance_validations/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/instance_validations/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/instance_validations/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/instance_validations/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/network-attachment/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/network-attachment/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/network-attachment/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/providers.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/firewall-rules/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/firewall-rules/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/firewall-rules/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/firewall-rules/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/firewall-rules/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/multivpc/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/network/vpc/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/packer/custom-image/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/packer/custom-image/image.pkr.hcl delete mode 100644 deletion-test/cluster/modules/embedded/modules/packer/custom-image/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/packer/custom-image/variables.pkr.hcl delete mode 100644 deletion-test/cluster/modules/embedded/modules/packer/custom-image/versions.pkr.hcl delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/README.md delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_docker.yml delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/main.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/metadata.yaml delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/outputs.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/variables.tf delete mode 100644 deletion-test/cluster/modules/embedded/modules/scripts/startup-script/versions.tf delete mode 100644 deletion-test/cluster/outputs.tf delete mode 100644 deletion-test/cluster/providers.tf delete mode 100644 deletion-test/cluster/variables.tf delete mode 100644 deletion-test/cluster/versions.tf delete mode 100644 deletion-test/instructions.txt delete mode 100644 deletion-test/primary/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/output.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/mig/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/notebook/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/notebook/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/notebook/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/notebook/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/sycomp-scale/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb delete mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh delete mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt delete mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/providers.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/network/private-service-access/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/new-project/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-account/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-enablement/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-enablement/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-enablement/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/project/service-enablement/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/pubsub/topic/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables_controller_instance.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/providers.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-job-template/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpx-in-workload.py delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/resource-policy/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/compute/vm-instance/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/mount.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/filestore/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/gke-storage/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/parallelstore/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-managed-lustre-client.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/mount-daos.sh.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/gpu-definition/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/gpu-definition/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/instance_validations/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/instance_validations/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/instance_validations/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/instance_validations/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/network-attachment/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/network-attachment/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/tpu-definition/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/tpu-definition/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/tpu-definition/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/tpu-definition/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/vpc_peering/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/vpc_peering/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/providers.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/management/kubectl-apply/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl delete mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl delete mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/monitoring/dashboard/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/firewall-rules/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/network/firewall-rules/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/network/firewall-rules/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/firewall-rules/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/multivpc/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/network/vpc/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/packer/custom-image/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl delete mode 100644 deletion-test/primary/modules/embedded/modules/packer/custom-image/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl delete mode 100644 deletion-test/primary/modules/embedded/modules/packer/custom-image/versions.pkr.hcl delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/README.md delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/main.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/outputs.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/variables.tf delete mode 100644 deletion-test/primary/modules/embedded/modules/scripts/startup-script/versions.tf delete mode 100644 deletion-test/primary/outputs.tf delete mode 100644 deletion-test/primary/providers.tf delete mode 100644 deletion-test/primary/variables.tf delete mode 100644 deletion-test/primary/versions.tf delete mode 100644 deletion-test/slurm-build/slurm-image/README.md delete mode 100644 deletion-test/slurm-build/slurm-image/image.pkr.hcl delete mode 100644 deletion-test/slurm-build/slurm-image/metadata.yaml delete mode 100644 deletion-test/slurm-build/slurm-image/variables.pkr.hcl delete mode 100644 deletion-test/slurm-build/slurm-image/versions.pkr.hcl diff --git a/deletion-test/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY b/deletion-test/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY deleted file mode 100644 index 56f49c329a..0000000000 --- a/deletion-test/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY +++ /dev/null @@ -1 +0,0 @@ -Files in this directory are managed by gcluster. Do not modify them manually! diff --git a/deletion-test/.ghpc/artifacts/expanded_blueprint.yaml b/deletion-test/.ghpc/artifacts/expanded_blueprint.yaml deleted file mode 100644 index 4c9f667e28..0000000000 --- a/deletion-test/.ghpc/artifacts/expanded_blueprint.yaml +++ /dev/null @@ -1,651 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -blueprint_name: a3mega-slurm -ghpc_version: v1.74.0-31-gedd55b924-dirty -validators: - - validator: test_deployment_variable_not_used - skip: true -vars: - a3mega_cluster_size: 2 - a3mega_dws_flex_enabled: false - a3mega_enable_spot_vm: true - a3mega_partition_name: a3mega - a3mega_reservation_name: "" - deployment_name: deletion-test - disk_size_gb: 200 - enable_controller_public_ips: true - enable_login_public_ips: true - enable_nvidia_dcgm: true - enable_nvidia_persistenced: true - enable_ops_agent: true - final_image_family: slurm-a3mega - instance_image: - family: ((var.final_image_family)) - project: ((var.project_id)) - labels: - ghpc_blueprint: a3mega-slurm - ghpc_deployment: ((var.deployment_name)) - local_mount_homefs: /home - localssd_mountpoint: /mnt/localssd - network_name_system: deletion-test-sys-net - project_id: hpc-toolkit-dev - region: europe-west1 - slurm_cluster_name: a3mega - source_image_family: ubuntu-accelerator-2204-amd64-with-nvidia-570 - source_image_project_id: - - ubuntu-os-accelerator-images - subnetwork_name_system: deletion-test-sys-subnet - sys_net_range: 172.16.0.0/16 - zone: europe-west1-c -deployment_groups: - - group: primary - terraform_backend: - type: gcs - configuration: - bucket: simranka - prefix: (("a3mega-slurm/${var.deployment_name}/primary")) - terraform_providers: - google: - source: hashicorp/google - version: '>= 6.9.0, <= 7.12.0' - configuration: - project: ((var.project_id)) - region: ((var.region)) - zone: ((var.zone)) - google-beta: - source: hashicorp/google-beta - version: '>= 6.9.0, <= 7.12.0' - configuration: - project: ((var.project_id)) - region: ((var.region)) - zone: ((var.zone)) - modules: - - source: modules/network/vpc - kind: terraform - id: sysnet - outputs: - - name: network_name - - name: subnetwork_name - - name: network_id - description: Automatically-generated output exported for use by later deployment groups - sensitive: true - - name: subnetwork_self_link - description: Automatically-generated output exported for use by later deployment groups - sensitive: true - settings: - deployment_name: ((var.deployment_name)) - labels: ((var.labels)) - mtu: 8244 - network_address_range: ((var.sys_net_range)) - network_name: ((var.network_name_system)) - project_id: ((var.project_id)) - region: ((var.region)) - subnetworks: - - description: primary subnetwork in gsc-sys-net - new_bits: 4 - subnet_name: ((var.subnetwork_name_system)) - subnet_private_access: true - subnet_region: ((var.region)) - - group: build_script - terraform_backend: - type: gcs - configuration: - bucket: simranka - prefix: (("a3mega-slurm/${var.deployment_name}/build_script")) - terraform_providers: - google: - source: hashicorp/google - version: '>= 6.9.0, <= 7.12.0' - configuration: - project: ((var.project_id)) - region: ((var.region)) - zone: ((var.zone)) - google-beta: - source: hashicorp/google-beta - version: '>= 6.9.0, <= 7.12.0' - configuration: - project: ((var.project_id)) - region: ((var.region)) - zone: ((var.zone)) - modules: - - source: modules/scripts/startup-script - kind: terraform - id: image_build_script - outputs: - - name: startup_script - description: Automatically-generated output exported for use by later deployment groups - sensitive: true - settings: - configure_ssh_host_patterns: - - 10.0.0.* - - 10.1.0.* - - 10.2.0.* - - 10.3.0.* - - 10.4.0.* - - 10.5.0.* - - 10.6.0.* - - 10.7.0.* - - (("${var.slurm_cluster_name}*")) - deployment_name: ((var.deployment_name)) - docker: - enabled: true - world_writable: true - enable_gpu_network_wait_online: true - install_ansible: true - labels: ((var.labels)) - project_id: ((var.project_id)) - region: ((var.region)) - runners: - - content: | - --- - - name: Hold nvidia packages - hosts: all - become: true - vars: - nvidia_packages_to_hold: - - libnvidia-cfg1-*-server - - libnvidia-compute-*-server - - libnvidia-nscq-* - - nvidia-compute-utils-*-server - - nvidia-fabricmanager-* - - nvidia-utils-*-server - - nvidia-imex-* - tasks: - - name: Hold nvidia packages - ansible.builtin.command: - argv: - - apt-mark - - hold - - "{{ item }}" - loop: "{{ nvidia_packages_to_hold }}" - destination: hold-nvidia-packages.yml - type: ansible-local - - content: | - #!/bin/bash - set -e -o pipefail - apt-mark hold google-compute-engine - apt-mark hold google-compute-engine-oslogin - apt-mark hold google-guest-agent - apt-mark hold google-osconfig-agent - destination: prevent_google_compute_upgrades.sh - type: shell - - content: | - { - "reboot": false, - "install_cuda": false, - "install_ompi": true, - "install_lustre": false, - "install_managed_lustre": false, - "install_gcsfuse": true, - "monitoring_agent": "cloud-ops", - "use_open_drivers": true - } - destination: /var/tmp/slurm_vars.json - type: data - - content: | - #!/bin/bash - set -e -o pipefail - apt-get update - apt-get install -y git - ansible-galaxy role install googlecloudplatform.google_cloud_ops_agents - ansible-pull \ - -U https://github.com/GoogleCloudPlatform/slurm-gcp -C 6.10.6 \ - -i localhost, --limit localhost --connection=local \ - -e @/var/tmp/slurm_vars.json \ - ansible/playbook.yml - destination: install_slurm.sh - type: shell - - content: | - --- - - name: Install updated gVNIC driver from GitHub - hosts: all - become: true - vars: - package_url: https://github.com/GoogleCloudPlatform/compute-virtual-ethernet-linux/releases/download/v1.4.3/gve-dkms_1.4.3_all.deb - package_filename: /tmp/{{ package_url | basename }} - tasks: - - name: Install driver dependencies - ansible.builtin.apt: - name: - - dkms - - name: Download gVNIC package - ansible.builtin.get_url: - url: "{{ package_url }}" - dest: "{{ package_filename }}" - - name: Install updated gVNIC - ansible.builtin.apt: - deb: "{{ package_filename }}" - state: present - destination: update-gvnic.yml - type: ansible-local - - content: | - #!/bin/bash - set -ex -o pipefail - add-nvidia-repositories -y - apt update -y - apt install -y cuda-toolkit-12-8 - apt install -y nvidia-container-toolkit - apt install -y datacenter-gpu-manager-4-cuda12 - apt install -y datacenter-gpu-manager-4-dev - destination: install-cuda-toolkit.sh - type: shell - - content: | - * - memlock unlimited - * - nproc unlimited - * - stack unlimited - * - nofile 1048576 - * - cpu unlimited - * - rtprio unlimited - destination: /etc/security/limits.d/99-unlimited.conf - type: data - - content: | - ENROOT_CONFIG_PATH ${HOME}/.enroot - ENROOT_RUNTIME_PATH /mnt/localssd/${UID}/enroot/runtime - ENROOT_CACHE_PATH /mnt/localssd/${UID}/enroot/cache - ENROOT_DATA_PATH /mnt/localssd/${UID}/enroot/data - ENROOT_TEMP_PATH /mnt/localssd/${UID}/enroot - destination: /etc/enroot/enroot.conf - type: data - - content: '(("---\n- name: Install CUDA & DCGM & Configure Ops Agent\n hosts: all\n become: true\n vars:\n enable_ops_agent: ${var.enable_ops_agent}\n enable_nvidia_dcgm: ${var.enable_nvidia_dcgm}\n tasks:\n - name: Create nvidia-persistenced override directory\n ansible.builtin.file:\n path: /etc/systemd/system/nvidia-persistenced.service.d\n state: directory\n owner: root\n group: root\n mode: 0o755\n - name: Configure nvidia-persistenced override\n ansible.builtin.copy:\n dest: /etc/systemd/system/nvidia-persistenced.service.d/persistence_mode.conf\n owner: root\n group: root\n mode: 0o644\n content: |\n [Service]\n ExecStart=\n ExecStart=/usr/bin/nvidia-persistenced --user nvidia-persistenced --verbose\n notify: Reload SystemD\n handlers:\n - name: Reload SystemD\n ansible.builtin.systemd:\n daemon_reload: true\n post_tasks:\n - name: Enable Google Cloud Ops Agent\n ansible.builtin.service:\n name: google-cloud-ops-agent.service\n state: \"{{ ''started'' if enable_ops_agent else ''stopped'' }}\"\n enabled: \"{{ enable_ops_agent }}\"\n - name: Disable NVIDIA DCGM by default (enable during boot on GPU nodes)\n ansible.builtin.service:\n name: nvidia-dcgm.service\n state: stopped\n enabled: \"{{ enable_nvidia_dcgm }}\"\n - name: Disable nvidia-persistenced SystemD unit (enable during boot on GPU nodes)\n ansible.builtin.service:\n name: nvidia-persistenced.service\n state: stopped\n enabled: false\n"))' - destination: configure_gpu_monitoring.yml - type: ansible-local - - content: | - --- - - name: Install DMBABUF import helper - hosts: all - become: true - tasks: - - name: Setup apt-transport-artifact-registry repository - ansible.builtin.apt_repository: - repo: deb http://packages.cloud.google.com/apt apt-transport-artifact-registry-stable main - state: present - - name: Install driver dependencies - ansible.builtin.apt: - name: - - dkms - - apt-transport-artifact-registry - - name: Setup gpudirect-tcpxo apt repository - ansible.builtin.apt_repository: - repo: deb [arch=all trusted=yes ] ar+https://us-apt.pkg.dev/projects/gce-ai-infra gpudirect-tcpxo-apt main - state: present - - name: Install DMABUF import helper DKMS package - ansible.builtin.apt: - name: dmabuf-import-helper - state: present - destination: install_dmabuf.yml - type: ansible-local - - content: | - --- - - name: Setup GPUDirect-TCPXO aperture devices - hosts: all - become: true - tasks: - - name: Mount aperture devices to /dev and make writable - ansible.builtin.copy: - dest: /etc/udev/rules.d/00-a3-megagpu.rules - owner: root - group: root - mode: 0o644 - content: | - ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x1ae0", ATTR{device}=="0x0084", TAG+="systemd", \ - RUN+="/usr/bin/mkdir --mode=0755 -p /dev/aperture_devices", \ - RUN+="/usr/bin/systemd-mount --type=none --options=bind --collect %S/%p /dev/aperture_devices/%k", \ - RUN+="/usr/bin/bash -c '/usr/bin/chmod 0666 /dev/aperture_devices/%k/resource*'" - notify: Update initramfs - handlers: - - name: Update initramfs - ansible.builtin.command: /usr/sbin/update-initramfs -u -k all - destination: aperture_devices.yml - type: ansible-local - - content: | - #!/bin/bash - # IMPORTANT: This script should be run *last* in any sequence of setup steps - # that use 'gsutil' or other gcloud commands. - # This is because removing the Snap version of the GCloud SDK can temporarily - # break existing 'gsutil' paths, which might disrupt other scripts still running - # that rely on the Snap-installed version. - - set -e -o pipefail - - # Remove the previously installed Google Cloud SDK (google-cloud-cli) and - # the LXD container manager, both of which might have been installed via Snap. - # This step is crucial to prevent conflicts with the upcoming APT installation - # and address potential issues with Snapd and NFS mounts in specific environments - snap remove google-cloud-cli lxd - # Install key and google-cloud-cli from apt repo - GCLOUD_APT_SOURCE="/etc/apt/sources.list.d/google-cloud-sdk.list" - if [ ! -f "${GCLOUD_APT_SOURCE}" ]; then - # indentation matters in EOT below; do not blindly edit! - cat < "${GCLOUD_APT_SOURCE}" - deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt cloud-sdk main - EOT - fi - curl -o /usr/share/keyrings/cloud.google.asc https://packages.cloud.google.com/apt/doc/apt-key.gpg - apt-get update - apt-get install --assume-yes google-cloud-cli - # Clean up the bash executable hash for subsequent steps using gsutil - hash -r - destination: remove_snap_gcloud.sh - type: shell - - group: slurm-build - terraform_backend: - type: gcs - configuration: - bucket: simranka - prefix: (("a3mega-slurm/${var.deployment_name}/slurm-build")) - modules: - - source: modules/packer/custom-image - kind: packer - id: slurm-image - use: - - image_build_script - - sysnet - settings: - deployment_name: ((var.deployment_name)) - disk_size: ((var.disk_size_gb)) - image_family: ((var.final_image_family)) - labels: ((var.labels)) - machine_type: c2-standard-8 - metadata: - user-data: | - #cloud-config - write_files: - - path: /etc/apt/apt.conf.d/20auto-upgrades - permissions: '0644' - owner: root - content: | - APT::Periodic::Update-Package-Lists "0"; - APT::Periodic::Unattended-Upgrade "0"; - omit_external_ip: false - project_id: ((var.project_id)) - source_image_family: ((var.source_image_family)) - source_image_project_id: ((var.source_image_project_id)) - startup_script: ((module.image_build_script.startup_script)) - subnetwork_name: ((module.sysnet.subnetwork_name)) - zone: ((var.zone)) - - group: cluster - terraform_backend: - type: gcs - configuration: - bucket: simranka - prefix: (("a3mega-slurm/${var.deployment_name}/cluster")) - terraform_providers: - google: - source: hashicorp/google - version: '>= 6.9.0, <= 7.12.0' - configuration: - project: ((var.project_id)) - region: ((var.region)) - zone: ((var.zone)) - google-beta: - source: hashicorp/google-beta - version: '>= 6.9.0, <= 7.12.0' - configuration: - project: ((var.project_id)) - region: ((var.region)) - zone: ((var.zone)) - modules: - - source: modules/file-system/cloud-storage-bucket - kind: terraform - id: data-bucket - settings: - deployment_name: ((var.deployment_name)) - labels: ((var.labels)) - local_mount: /gcs - mount_options: defaults,rw,_netdev,implicit_dirs,allow_other,implicit_dirs,file_mode=777,dir_mode=777 - project_id: ((var.project_id)) - random_suffix: true - region: ((var.region)) - - source: modules/network/multivpc - kind: terraform - id: gpunets - settings: - deployment_name: ((var.deployment_name)) - global_ip_address_range: 10.0.0.0/9 - network_count: 8 - network_name_prefix: (("${var.deployment_name}-gpunet")) - project_id: ((var.project_id)) - region: ((var.region)) - subnetwork_cidr_suffix: 20 - - source: community/modules/network/private-service-access - kind: terraform - id: private_service_access - use: - - sysnet - settings: - labels: ((var.labels)) - network_id: ((module.sysnet.network_id)) - project_id: ((var.project_id)) - - source: modules/file-system/filestore - kind: terraform - id: homefs - use: - - sysnet - - private_service_access - outputs: - - name: network_storage - settings: - connect_mode: ((module.private_service_access.connect_mode)) - deletion_protection: - enabled: true - reason: Avoid data loss - deployment_name: ((var.deployment_name)) - filestore_tier: HIGH_SCALE_SSD - labels: ((var.labels)) - local_mount: /home - mount_options: defaults,hard - network_id: ((module.sysnet.network_id)) - project_id: ((var.project_id)) - region: ((var.region)) - reserved_ip_range: ((module.private_service_access.reserved_ip_range)) - size_gb: 10240 - zone: ((var.zone)) - - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - kind: terraform - id: debug_nodeset - use: - - sysnet - settings: - disk_size_gb: ((var.disk_size_gb)) - instance_image: ((var.instance_image)) - labels: ((var.labels)) - machine_type: n2-standard-2 - name: debug_nodeset - node_count_dynamic_max: 4 - node_count_static: 0 - project_id: ((var.project_id)) - region: ((var.region)) - subnetwork_self_link: ((module.sysnet.subnetwork_self_link)) - zone: ((var.zone)) - - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - kind: terraform - id: debug_partition - use: - - debug_nodeset - settings: - exclusive: false - nodeset: ((flatten([module.debug_nodeset.nodeset]))) - partition_name: debug - - source: modules/scripts/startup-script - kind: terraform - id: a3mega_startup - settings: - deployment_name: ((var.deployment_name)) - docker: - daemon_config: '(("{\n \"data-root\": \"${var.localssd_mountpoint}/docker\"\n}\n"))' - enabled: true - world_writable: true - labels: ((var.labels)) - local_ssd_filesystem: - mountpoint: ((var.localssd_mountpoint)) - permissions: "1777" - project_id: ((var.project_id)) - region: ((var.region)) - runners: - - content: | - --- - - name: Configure Slurm to depend upon aperture devices - hosts: all - become: true - vars: {} - tasks: - - name: Ensure slurmd starts after aperture devices are ready - ansible.builtin.copy: - dest: /etc/systemd/system/slurmd.service.d/aperture.conf - owner: root - group: root - mode: 0o644 - content: | - [Service] - ExecCondition=/usr/bin/test -d /dev/aperture_devices/ - notify: Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - destination: slurm_aperture.yml - type: ansible-local - - content: '(("---\n- name: Enable NVIDIA DCGM on GPU nodes\n hosts: all\n become: true\n vars:\n enable_ops_agent: ${var.enable_ops_agent}\n enable_nvidia_dcgm: ${var.enable_nvidia_dcgm}\n enable_nvidia_persistenced: ${var.enable_nvidia_persistenced}\n tasks:\n - name: Update Ops Agent configuration\n ansible.builtin.blockinfile:\n path: /etc/google-cloud-ops-agent/config.yaml\n insertafter: EOF\n block: |\n metrics:\n receivers:\n dcgm:\n type: dcgm\n service:\n pipelines:\n dcgm:\n receivers:\n - dcgm\n notify:\n - Restart Google Cloud Ops Agent\n handlers:\n - name: Restart Google Cloud Ops Agent\n ansible.builtin.service:\n name: google-cloud-ops-agent.service\n state: \"{{ ''restarted'' if enable_ops_agent else ''stopped'' }}\"\n enabled: \"{{ enable_ops_agent }}\"\n post_tasks:\n - name: Enable Google Cloud Ops Agent\n ansible.builtin.service:\n name: google-cloud-ops-agent.service\n state: \"{{ ''started'' if enable_ops_agent else ''stopped'' }}\"\n enabled: \"{{ enable_ops_agent }}\"\n - name: Enable NVIDIA DCGM\n ansible.builtin.service:\n name: nvidia-dcgm.service\n state: \"{{ ''started'' if enable_nvidia_dcgm else ''stopped'' }}\"\n enabled: \"{{ enable_nvidia_dcgm }}\"\n - name: Enable NVIDIA Persistence Daemon\n ansible.builtin.service:\n name: nvidia-persistenced.service\n state: \"{{ ''started'' if enable_nvidia_persistenced else ''stopped'' }}\"\n enabled: \"{{ enable_nvidia_persistenced }}\"\n"))' - destination: enable_dcgm.yml - type: ansible-local - - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - kind: terraform - id: a3mega_nodeset - use: - - sysnet - - gpunets - settings: - additional_networks: ((flatten([module.gpunets.additional_networks]))) - advanced_machine_features: - threads_per_core: null - bandwidth_tier: gvnic_enabled - disk_size_gb: ((var.disk_size_gb)) - disk_type: pd-ssd - dws_flex: - enabled: ((var.a3mega_dws_flex_enabled)) - enable_public_ips: false - enable_spot_vm: ((var.a3mega_enable_spot_vm)) - instance_image: ((var.instance_image)) - labels: ((var.labels)) - machine_type: a3-megagpu-8g - name: a3mega_nodeset - node_conf: - CoresPerSocket: 52 - ThreadsPerCore: 2 - node_count_dynamic_max: 0 - node_count_static: ((var.a3mega_cluster_size)) - on_host_maintenance: TERMINATE - project_id: ((var.project_id)) - region: ((var.region)) - reservation_name: ((var.a3mega_reservation_name)) - startup_script: ((module.a3mega_startup.startup_script)) - subnetwork_self_link: ((module.sysnet.subnetwork_self_link)) - zone: ((var.zone)) - - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - kind: terraform - id: a3mega_partition - use: - - a3mega_nodeset - settings: - exclusive: false - is_default: true - nodeset: ((flatten([module.a3mega_nodeset.nodeset]))) - partition_conf: - OverSubscribe: EXCLUSIVE - ResumeTimeout: 900 - SuspendTimeout: 600 - partition_name: ((var.a3mega_partition_name)) - - source: modules/scripts/startup-script - kind: terraform - id: controller_startup - settings: - deployment_name: ((var.deployment_name)) - labels: ((var.labels)) - project_id: ((var.project_id)) - region: ((var.region)) - runners: - - content: (("#!/bin/bash\nSLURM_ROOT=/opt/apps/adm/slurm\nmkdir -m 0755 -p \"$${SLURM_ROOT}/scripts\"\nmkdir -p \"$${SLURM_ROOT}/partition-${var.a3mega_partition_name}-prolog_slurmd.d\"\nmkdir -p \"$${SLURM_ROOT}/partition-${var.a3mega_partition_name}-epilog_slurmd.d\"\nmkdir -p \"$${SLURM_ROOT}/prolog_slurmd.d\"\nmkdir -p \"$${SLURM_ROOT}/epilog_slurmd.d\"\n# enable the use of password-free sudo within Slurm jobs on all compute nodes\n# feature is restricted to users with OS Admin Login IAM role\n# https://cloud.google.com/iam/docs/understanding-roles#compute.osAdminLogin\ncurl -s -o \"$${SLURM_ROOT}/scripts/sudo-oslogin\" \\\n https://raw.githubusercontent.com/GoogleCloudPlatform/slurm-gcp/master/tools/prologs-epilogs/sudo-oslogin\nchmod 0755 \"$${SLURM_ROOT}/scripts/sudo-oslogin\"\nln -s \"$${SLURM_ROOT}/scripts/sudo-oslogin\" \"$${SLURM_ROOT}/prolog_slurmd.d/sudo-oslogin.prolog_slurmd\"\nln -s \"$${SLURM_ROOT}/scripts/sudo-oslogin\" \"$${SLURM_ROOT}/epilog_slurmd.d/sudo-oslogin.epilog_slurmd\"\ncurl -s -o \"$${SLURM_ROOT}/scripts/rxdm\" \\\n https://raw.githubusercontent.com/GoogleCloudPlatform/slurm-gcp/master/tools/prologs-epilogs/receive-data-path-manager-mega\nchmod 0755 \"$${SLURM_ROOT}/scripts/rxdm\"\nln -s \"$${SLURM_ROOT}/scripts/rxdm\" \"$${SLURM_ROOT}/partition-${var.a3mega_partition_name}-prolog_slurmd.d/rxdm.prolog_slurmd\"\nln -s \"$${SLURM_ROOT}/scripts/rxdm\" \"$${SLURM_ROOT}/partition-${var.a3mega_partition_name}-epilog_slurmd.d/rxdm.epilog_slurmd\"\n# enable a GPU health check that runs at the completion of all jobs on A3mega nodes\nln -s \"/slurm/scripts/tools/gpu-test\" \"$${SLURM_ROOT}/partition-${var.a3mega_partition_name}-epilog_slurmd.d/gpu-test.epilog_slurmd\"\n")) - destination: stage_scripts.sh - type: shell - - content: | - #!/bin/bash - # reset enroot to defaults of files under /home and running under /run - # allows basic enroot testing with reduced I/O performance - rm -f /etc/enroot/enroot.conf - destination: reset_enroot.sh - type: shell - - source: community/modules/scheduler/schedmd-slurm-gcp-v6-login - kind: terraform - id: slurm_login - use: - - sysnet - settings: - disk_size_gb: ((var.disk_size_gb)) - disk_type: pd-balanced - enable_login_public_ips: ((var.enable_login_public_ips)) - instance_image: ((var.instance_image)) - labels: ((var.labels)) - machine_type: c2-standard-4 - name_prefix: login - project_id: ((var.project_id)) - region: ((var.region)) - subnetwork_self_link: ((module.sysnet.subnetwork_self_link)) - zone: ((var.zone)) - - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - kind: terraform - id: slurm_controller - use: - - sysnet - - a3mega_partition - - debug_partition - - slurm_login - - homefs - - data-bucket - settings: - controller_startup_script: ((module.controller_startup.startup_script)) - deployment_name: ((var.deployment_name)) - disk_size_gb: ((var.disk_size_gb)) - enable_cleanup_compute: true - enable_controller_public_ips: ((var.enable_controller_public_ips)) - enable_external_prolog_epilog: true - instance_image: ((var.instance_image)) - labels: ((var.labels)) - login_nodes: ((flatten([module.slurm_login.login_nodes]))) - login_startup_script: | - #!/bin/bash - # reset enroot to defaults of files under /home and running under /run - # allows basic enroot testing with reduced I/O performance - rm -f /etc/enroot/enroot.conf - machine_type: c2-standard-8 - network_storage: ((flatten([module.data-bucket.network_storage, flatten([module.homefs.network_storage])]))) - nodeset: ((flatten([module.debug_partition.nodeset, flatten([module.a3mega_partition.nodeset])]))) - nodeset_dyn: ((flatten([module.debug_partition.nodeset_dyn, flatten([module.a3mega_partition.nodeset_dyn])]))) - nodeset_tpu: ((flatten([module.debug_partition.nodeset_tpu, flatten([module.a3mega_partition.nodeset_tpu])]))) - partitions: ((flatten([module.debug_partition.partitions, flatten([module.a3mega_partition.partitions])]))) - project_id: ((var.project_id)) - prolog_scripts: - - content: | - #!/bin/bash - hostname | tee /etc/hostname - filename: set_hostname_for_enroot.sh - region: ((var.region)) - slurm_cluster_name: ((var.slurm_cluster_name)) - slurm_conf_tpl: modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl - subnetwork_self_link: ((module.sysnet.subnetwork_self_link)) - zone: ((var.zone)) -terraform_backend_defaults: - type: gcs - configuration: - bucket: simranka diff --git a/deletion-test/.gitignore b/deletion-test/.gitignore deleted file mode 100644 index 1e44b25074..0000000000 --- a/deletion-test/.gitignore +++ /dev/null @@ -1,48 +0,0 @@ -# Local .terraform directories -**/.terraform/* - -# .tfstate files -*.tfstate -*.tfstate.* - -# Crash log files -crash.log -crash.*.log - -# Exclude all .tfvars files, which are likely to contain sensitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject -# to change depending on the environment. -*.tfvars -*.tfvars.json - -# Ignore override files as they are usually used to override resources locally and so -# are not checked in -override.tf -override.tf.json -*_override.tf -*_override.tf.json - -# Include override files you do wish to add to version control using negated pattern -# !example_override.tf - -# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan -# example: *tfplan* - -# Ignore CLI configuration files -.terraformrc -terraform.rc - -# Cache objects -packer_cache/ - -# https://www.packer.io/guides/hcl/variables -# Exclude all .pkrvars.hcl files, which are likely to contain sensitive data, -# such as password, private keys, and other secrets. These should not be part of -# version control as they are data points which are potentially sensitive and -# subject to change depending on the environment. -# -*.pkrvars.hcl - -# For built boxes -*.box diff --git a/deletion-test/build_script/main.tf b/deletion-test/build_script/main.tf deleted file mode 100644 index 18d593011a..0000000000 --- a/deletion-test/build_script/main.tf +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - backend "gcs" { - bucket = "simranka" - prefix = "a3mega-slurm/deletion-test/build_script" - } -} - -module "image_build_script" { - source = "./modules/embedded/modules/scripts/startup-script" - configure_ssh_host_patterns = ["10.0.0.*", "10.1.0.*", "10.2.0.*", "10.3.0.*", "10.4.0.*", "10.5.0.*", "10.6.0.*", "10.7.0.*", "${var.slurm_cluster_name}*"] - deployment_name = var.deployment_name - docker = { - enabled = true - world_writable = true - } - enable_gpu_network_wait_online = true - install_ansible = true - labels = var.labels - project_id = var.project_id - region = var.region - runners = [{ - content = "---\n- name: Hold nvidia packages\n hosts: all\n become: true\n vars:\n nvidia_packages_to_hold:\n - libnvidia-cfg1-*-server\n - libnvidia-compute-*-server\n - libnvidia-nscq-*\n - nvidia-compute-utils-*-server\n - nvidia-fabricmanager-*\n - nvidia-utils-*-server\n - nvidia-imex-*\n tasks:\n - name: Hold nvidia packages\n ansible.builtin.command:\n argv:\n - apt-mark\n - hold\n - \"{{ item }}\"\n loop: \"{{ nvidia_packages_to_hold }}\"\n" - destination = "hold-nvidia-packages.yml" - type = "ansible-local" - }, { - content = "#!/bin/bash\nset -e -o pipefail\napt-mark hold google-compute-engine\napt-mark hold google-compute-engine-oslogin\napt-mark hold google-guest-agent\napt-mark hold google-osconfig-agent\n" - destination = "prevent_google_compute_upgrades.sh" - type = "shell" - }, { - content = "{\n \"reboot\": false,\n \"install_cuda\": false,\n \"install_ompi\": true,\n \"install_lustre\": false,\n \"install_managed_lustre\": false,\n \"install_gcsfuse\": true,\n \"monitoring_agent\": \"cloud-ops\",\n \"use_open_drivers\": true\n}\n" - destination = "/var/tmp/slurm_vars.json" - type = "data" - }, { - content = "#!/bin/bash\nset -e -o pipefail\napt-get update\napt-get install -y git\nansible-galaxy role install googlecloudplatform.google_cloud_ops_agents\nansible-pull \\\n -U https://github.com/GoogleCloudPlatform/slurm-gcp -C 6.10.6 \\\n -i localhost, --limit localhost --connection=local \\\n -e @/var/tmp/slurm_vars.json \\\n ansible/playbook.yml\n" - destination = "install_slurm.sh" - type = "shell" - }, { - content = "---\n- name: Install updated gVNIC driver from GitHub\n hosts: all\n become: true\n vars:\n package_url: https://github.com/GoogleCloudPlatform/compute-virtual-ethernet-linux/releases/download/v1.4.3/gve-dkms_1.4.3_all.deb\n package_filename: /tmp/{{ package_url | basename }}\n tasks:\n - name: Install driver dependencies\n ansible.builtin.apt:\n name:\n - dkms\n - name: Download gVNIC package\n ansible.builtin.get_url:\n url: \"{{ package_url }}\"\n dest: \"{{ package_filename }}\"\n - name: Install updated gVNIC\n ansible.builtin.apt:\n deb: \"{{ package_filename }}\"\n state: present\n" - destination = "update-gvnic.yml" - type = "ansible-local" - }, { - content = "#!/bin/bash\nset -ex -o pipefail\nadd-nvidia-repositories -y\napt update -y\napt install -y cuda-toolkit-12-8\napt install -y nvidia-container-toolkit\napt install -y datacenter-gpu-manager-4-cuda12\napt install -y datacenter-gpu-manager-4-dev\n" - destination = "install-cuda-toolkit.sh" - type = "shell" - }, { - content = "* - memlock unlimited\n* - nproc unlimited\n* - stack unlimited\n* - nofile 1048576\n* - cpu unlimited\n* - rtprio unlimited\n" - destination = "/etc/security/limits.d/99-unlimited.conf" - type = "data" - }, { - content = "ENROOT_CONFIG_PATH $${HOME}/.enroot\nENROOT_RUNTIME_PATH /mnt/localssd/$${UID}/enroot/runtime\nENROOT_CACHE_PATH /mnt/localssd/$${UID}/enroot/cache\nENROOT_DATA_PATH /mnt/localssd/$${UID}/enroot/data\nENROOT_TEMP_PATH /mnt/localssd/$${UID}/enroot\n" - destination = "/etc/enroot/enroot.conf" - type = "data" - }, { - content = "---\n- name: Install CUDA & DCGM & Configure Ops Agent\n hosts: all\n become: true\n vars:\n enable_ops_agent: ${var.enable_ops_agent}\n enable_nvidia_dcgm: ${var.enable_nvidia_dcgm}\n tasks:\n - name: Create nvidia-persistenced override directory\n ansible.builtin.file:\n path: /etc/systemd/system/nvidia-persistenced.service.d\n state: directory\n owner: root\n group: root\n mode: 0o755\n - name: Configure nvidia-persistenced override\n ansible.builtin.copy:\n dest: /etc/systemd/system/nvidia-persistenced.service.d/persistence_mode.conf\n owner: root\n group: root\n mode: 0o644\n content: |\n [Service]\n ExecStart=\n ExecStart=/usr/bin/nvidia-persistenced --user nvidia-persistenced --verbose\n notify: Reload SystemD\n handlers:\n - name: Reload SystemD\n ansible.builtin.systemd:\n daemon_reload: true\n post_tasks:\n - name: Enable Google Cloud Ops Agent\n ansible.builtin.service:\n name: google-cloud-ops-agent.service\n state: \"{{ 'started' if enable_ops_agent else 'stopped' }}\"\n enabled: \"{{ enable_ops_agent }}\"\n - name: Disable NVIDIA DCGM by default (enable during boot on GPU nodes)\n ansible.builtin.service:\n name: nvidia-dcgm.service\n state: stopped\n enabled: \"{{ enable_nvidia_dcgm }}\"\n - name: Disable nvidia-persistenced SystemD unit (enable during boot on GPU nodes)\n ansible.builtin.service:\n name: nvidia-persistenced.service\n state: stopped\n enabled: false\n" - destination = "configure_gpu_monitoring.yml" - type = "ansible-local" - }, { - content = "---\n- name: Install DMBABUF import helper\n hosts: all\n become: true\n tasks:\n - name: Setup apt-transport-artifact-registry repository\n ansible.builtin.apt_repository:\n repo: deb http://packages.cloud.google.com/apt apt-transport-artifact-registry-stable main\n state: present\n - name: Install driver dependencies\n ansible.builtin.apt:\n name:\n - dkms\n - apt-transport-artifact-registry\n - name: Setup gpudirect-tcpxo apt repository\n ansible.builtin.apt_repository:\n repo: deb [arch=all trusted=yes ] ar+https://us-apt.pkg.dev/projects/gce-ai-infra gpudirect-tcpxo-apt main\n state: present\n - name: Install DMABUF import helper DKMS package\n ansible.builtin.apt:\n name: dmabuf-import-helper\n state: present\n" - destination = "install_dmabuf.yml" - type = "ansible-local" - }, { - content = "---\n- name: Setup GPUDirect-TCPXO aperture devices\n hosts: all\n become: true\n tasks:\n - name: Mount aperture devices to /dev and make writable\n ansible.builtin.copy:\n dest: /etc/udev/rules.d/00-a3-megagpu.rules\n owner: root\n group: root\n mode: 0o644\n content: |\n ACTION==\"add\", SUBSYSTEM==\"pci\", ATTR{vendor}==\"0x1ae0\", ATTR{device}==\"0x0084\", TAG+=\"systemd\", \\\n RUN+=\"/usr/bin/mkdir --mode=0755 -p /dev/aperture_devices\", \\\n RUN+=\"/usr/bin/systemd-mount --type=none --options=bind --collect %S/%p /dev/aperture_devices/%k\", \\\n RUN+=\"/usr/bin/bash -c '/usr/bin/chmod 0666 /dev/aperture_devices/%k/resource*'\"\n notify: Update initramfs\n handlers:\n - name: Update initramfs\n ansible.builtin.command: /usr/sbin/update-initramfs -u -k all\n" - destination = "aperture_devices.yml" - type = "ansible-local" - }, { - content = "#!/bin/bash\n# IMPORTANT: This script should be run *last* in any sequence of setup steps\n# that use 'gsutil' or other gcloud commands.\n# This is because removing the Snap version of the GCloud SDK can temporarily\n# break existing 'gsutil' paths, which might disrupt other scripts still running\n# that rely on the Snap-installed version.\n\nset -e -o pipefail\n\n# Remove the previously installed Google Cloud SDK (google-cloud-cli) and\n# the LXD container manager, both of which might have been installed via Snap.\n# This step is crucial to prevent conflicts with the upcoming APT installation\n# and address potential issues with Snapd and NFS mounts in specific environments\nsnap remove google-cloud-cli lxd\n# Install key and google-cloud-cli from apt repo\nGCLOUD_APT_SOURCE=\"/etc/apt/sources.list.d/google-cloud-sdk.list\"\nif [ ! -f \"$${GCLOUD_APT_SOURCE}\" ]; then\n # indentation matters in EOT below; do not blindly edit!\n cat < \"$${GCLOUD_APT_SOURCE}\"\ndeb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt cloud-sdk main\nEOT\nfi\ncurl -o /usr/share/keyrings/cloud.google.asc https://packages.cloud.google.com/apt/doc/apt-key.gpg\napt-get update\napt-get install --assume-yes google-cloud-cli\n# Clean up the bash executable hash for subsequent steps using gsutil\nhash -r\n" - destination = "remove_snap_gcloud.sh" - type = "shell" - }] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/README.md b/deletion-test/build_script/modules/embedded/community/modules/README.md deleted file mode 100644 index 0c83b9d30c..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Community Modules - -This directory contains modules that rely on partner resources, have been -contributed by outside developers or are in early development by the Cluster Toolkit -team. The modules in this directory are listed alongside core modules in the -[core modules README](../../modules/README.md). There you can also learn more -about general use and how to write custom Cluster Toolkit modules. diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/README.md deleted file mode 100644 index 0ee685d93e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/README.md +++ /dev/null @@ -1,55 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 4.84 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [home\_pv](#module\_home\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | -| [kubectl\_apply](#module\_kubectl\_apply) | ../../../../modules/management/kubectl-apply | n/a | -| [slurm\_key\_pv](#module\_slurm\_key\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.gke_nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | -| [filestore\_id](#input\_filestore\_id) | An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `list(string)` | n/a | yes | -| [image](#input\_image) | The image for slurm daemon | `string` | n/a | yes | -| [instance\_templates](#input\_instance\_templates) | The URLs of Instance Templates | `list(string)` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| n/a | yes | -| [node\_count\_static](#input\_node\_count\_static) | The number of static nodes in node-pool | `number` | n/a | yes | -| [node\_pool\_names](#input\_node\_pool\_names) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `list(string)` | n/a | yes | -| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | -| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used in slurm controller | `string` | n/a | yes | -| [slurm\_controller\_instance](#input\_slurm\_controller\_instance) | Slurm cluster controller instance | `any` | n/a | yes | -| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | -| [subnetwork](#input\_subnetwork) | Primary subnetwork object | `any` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset\_name](#output\_nodeset\_name) | Name of the new Slinky nodset | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/main.tf deleted file mode 100644 index 8b2f1deeac..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/main.tf +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -### GKE NodeSet -locals { - manifest_path = "${path.module}/templates/nodeset-general.yaml.tftpl" -} - -module "kubectl_apply" { - source = "../../../../modules/management/kubectl-apply" - - cluster_id = var.cluster_id - project_id = var.project_id - - apply_manifests = [{ - source = local.manifest_path, - template_vars = { - slurm_namespace = var.slurm_namespace, - nodeset_name = "${var.slurm_cluster_name}-${var.nodeset_name}", - nodeset_cr_name = "${var.slurm_cluster_name}-${var.nodeset_name}", - controller_name = "${var.slurm_cluster_name}-controller", - node_pool_name = var.node_pool_names[0], - node_count = var.node_count_static, - image = var.image, - home_pvc = module.home_pv.pvc_name - slurm_key_pvc = module.slurm_key_pv.pvc_name - } - }] -} - -data "google_storage_bucket" "this" { - name = var.slurm_bucket[0].name - - depends_on = [var.slurm_bucket] -} - -### Slurm NodeSet -locals { - nodeset = { - gke_nodepool = var.node_pool_names[0] - nodeset_name = var.nodeset_name - node_count_static = var.node_count_static - subnetwork = "https://www.googleapis.com/compute/v1/projects/${var.project_id}/regions/${var.subnetwork.region}/subnetworks/${var.subnetwork.name}" - instance_template = var.instance_templates[0] - } -} - -resource "google_storage_bucket_object" "gke_nodeset_config" { - bucket = data.google_storage_bucket.this.name - name = "${var.slurm_bucket_dir}/nodeset_configs/${var.nodeset_name}.yaml" - content = yamlencode(local.nodeset) -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml deleted file mode 100644 index ea2cfc221e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/output.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/output.tf deleted file mode 100644 index 15970ff0b7..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/output.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset_name" { - description = "Name of the new Slinky nodset" - value = local.nodeset.nodeset_name -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf deleted file mode 100644 index 8a190c4019..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - slurm_key_storage = { - server_ip = var.slurm_controller_instance.network_interface[0].network_ip - remote_mount = "/slurm/key_distribution" # defined in /community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py - client_install_runner = {} - mount_runner = {} - fs_type = "" - local_mount = "" - mount_options = "" - } -} - -module "slurm_key_pv" { - source = "../../../../modules/file-system/gke-persistent-volume" - labels = {} - capacity_gib = 1 - cluster_id = var.cluster_id - filestore_id = "projects/empty/locations/empty/instances/empty" # this does not apply since this NFS is not a filestore - namespace = var.slurm_namespace - network_storage = local.slurm_key_storage - pv_name = "slurm-key-pv" - pvc_name = "slurm-key-pvc" -} - -# Assume the var.network_storage[0] will be home and only one home pv is accepted for now. -module "home_pv" { - source = "../../../../modules/file-system/gke-persistent-volume" - labels = {} - capacity_gib = 1024 - cluster_id = var.cluster_id - filestore_id = var.filestore_id[0] - network_storage = var.network_storage[0] - namespace = var.slurm_namespace - pv_name = "home-pv" - pvc_name = "home-pvc" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl deleted file mode 100644 index a5a4a5e7ac..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl +++ /dev/null @@ -1,203 +0,0 @@ -apiVersion: slinky.slurm.net/v1alpha1 -kind: NodeSet -metadata: - annotations: - meta.helm.sh/release-name: slurm - meta.helm.sh/release-namespace: ${slurm_namespace} - labels: - app.kubernetes.io/component: compute - app.kubernetes.io/instance: slurm - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: slurmd - app.kubernetes.io/part-of: slurm - app.kubernetes.io/version: "24.11" - helm.sh/chart: slurm-0.3.0 - nodeset.slinky.slurm.net/name: ${nodeset_name} - name: ${nodeset_name} - namespace: ${slurm_namespace} -spec: - clusterName: slurm - persistentVolumeClaimRetentionPolicy: - whenDeleted: Retain - whenScaled: Retain - replicas: ${node_count} - revisionHistoryLimit: 0 - selector: - matchLabels: - app.kubernetes.io/instance: slurm - app.kubernetes.io/name: slurmd - nodeset.slinky.slurm.net/name: ${nodeset_name} - serviceName: slurm-compute - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: slurmd - labels: - app.kubernetes.io/component: compute - app.kubernetes.io/instance: slurm - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: slurmd - app.kubernetes.io/part-of: slurm - app.kubernetes.io/version: "24.11" - helm.sh/chart: slurm-0.3.0 - nodeset.slinky.slurm.net/name: ${nodeset_name} - spec: - automountServiceAccountToken: false - containers: - - args: - - -g - - -- - - bash - - -c - - | - mkdir -p /usr/local/lib/slurm - ln -s /usr/lib/x86_64-linux-gnu/slurm/spank_pyxis.so /usr/local/lib/slurm/spank_pyxis.so - /usr/local/bin/entrypoint.sh -Z --conf-server ${controller_name}:6825 -N $NODE_NAME - command: - - tini - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_CPUS - value: "0" - - name: POD_MEMORY - value: "0" - image: ${image} - imagePullPolicy: IfNotPresent - name: slurmd - ports: - - containerPort: 6818 - name: slurmd - protocol: TCP - readinessProbe: - exec: - command: - - scontrol - - show - - slurmd - resources: {} - securityContext: - capabilities: - add: - - BPF - - NET_ADMIN - - SYS_ADMIN - - SYS_NICE - privileged: true - volumeMounts: - - mountPath: /etc/slurm - name: etc-slurm - - mountPath: /run - name: run - - mountPath: /var/spool/slurmd - name: slurm-spool - - mountPath: /var/log/slurm - name: slurm-log - - mountPath: /home - name: home-pvc - dnsConfig: - searches: - - ${controller_name} - hostNetwork: true - initContainers: - - command: - - tini - - -g - - -- - - bash - - -c - - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# - SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env - contains:\n# SLURM_USER - username or UID\n\nfunction init::common() {\n\tlocal - dir\n\n\tdir=/var/spool/slurmd\n\tmkdir -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" - \"$dir\"\n\tchmod -v 700 \"$dir\"\n\n\tdir=/var/spool/slurmctld\n\tmkdir - -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" \"$dir\"\n\tchmod - -v 700 \"$dir\"\n}\n\nfunction init::slurm() {\n\tSLURM_MOUNT=/mnt/slurm\n\tSLURM_DIR=/mnt/etc/slurm\n\n\t# - Workaround to ephemeral volumes not supporting securityContext\n\t# https://github.com/kubernetes/kubernetes/issues/81089\n\n\t# - Copy Slurm config files, secrets, and scripts\n\tmkdir -p \"$SLURM_DIR\"\n\tfind - \"$${SLURM_MOUNT}\" -type f -name \"*.conf\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind - \"$${SLURM_MOUNT}\" -type f -name \"*.key\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind - \"$${SLURM_MOUNT}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" - -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind \"$${SLURM_MOUNT}\" -type - f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 | xargs - -0r cp -vt \"$${SLURM_DIR}\"\n\n\t# Set general permissions and ownership\n\tfind - \"$${SLURM_DIR}\" -type f -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind - \"$${SLURM_DIR}\" -type f -name \"*.conf\" -print0 | xargs -0r chmod -v 644\n\tfind - \"$${SLURM_DIR}\" -type f -name \"*.key\" -print0 | xargs -0r chmod -v 600\n\tfind - \"$${SLURM_DIR}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" - -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind \"$${SLURM_DIR}\" - -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 - | xargs -0r chmod -v 755\n\n\t# Inject secrets into certain config files\n\tlocal - dbd_conf=\"slurmdbd.conf\"\n\tif [[ -f \"$${SLURM_MOUNT}/$${dbd_conf}\" ]]; - then\n\t\techo \"Injecting secrets from environment into: $${dbd_conf}\"\n\t\trm - -f \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tenvsubst <\"$${SLURM_MOUNT}/$${dbd_conf}\" - >\"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" - \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchmod -v 600 \"$${SLURM_DIR}/$${dbd_conf}\"\n\tfi\n\n\t# - Display Slurm directory files\n\tls -lAF \"$${SLURM_DIR}\"\n}\n\nfunction - main() {\n\tinit::common\n\tinit::slurm\n}\nmain\n" - env: - - name: SLURM_USER - value: slurm - image: ${image} - imagePullPolicy: IfNotPresent - name: init - resources: {} - volumeMounts: - - mountPath: /mnt/slurm - name: slurm-config - - mountPath: /mnt/etc/slurm - name: etc-slurm - - command: - - tini - - -g - - -- - - bash - - -c - - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# - SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env - contains:\n# SOCKET - Named socket to read from\n\nmkdir -v -p \"$(dirname - \"$SOCKET\")\"\nrm -f \"$SOCKET\"\nif ! [ -f \"$SOCKET\" ]; then\n\tmkfifo - -m 777 \"$SOCKET\"\nfi\nwhile IFS=\"\" read data; do\n\techo $data\ndone - <\"$SOCKET\"\n" - env: - - name: SOCKET - value: /var/log/slurm/slurmd.log - image: ghcr.io/slinkyproject/sackd:24.11-ubuntu24.04 - imagePullPolicy: IfNotPresent - name: logfile - resources: {} - restartPolicy: Always - volumeMounts: - - mountPath: /var/log/slurm - name: slurm-log - nodeSelector: - cloud.google.com/gke-nodepool: ${node_pool_name} - tolerations: - - effect: NoSchedule - key: nvidia.com/gpu - operator: Equal - value: present - volumes: - - emptyDir: - medium: Memory - name: etc-slurm - - emptyDir: {} - name: run - - name: slurm-config - persistentVolumeClaim: - claimName: ${slurm_key_pvc} - - emptyDir: - medium: Memory - name: slurm-spool - - emptyDir: - medium: Memory - name: slurm-log - - name: home-pvc - persistentVolumeClaim: - claimName: ${home_pvc} - updateStrategy: - rollingUpdate: - maxUnavailable: 20% - type: RollingUpdate diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/variables.tf deleted file mode 100644 index c091a0da86..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/variables.tf +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "cluster_id" { - description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" - type = string -} - -variable "slurm_cluster_name" { - type = string - description = "Cluster name, used in slurm controller" - - validation { - condition = var.slurm_cluster_name != null && can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) - error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." - } -} - -variable "slurm_controller_instance" { - type = any - description = "Slurm cluster controller instance" -} - -variable "image" { - description = "The image for slurm daemon" - type = string - nullable = false -} - -variable "node_pool_names" { - description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." - type = list(string) - nullable = false -} - -variable "node_count_static" { - description = "The number of static nodes in node-pool" - type = number -} - -variable "subnetwork" { - description = "Primary subnetwork object" - type = any -} - -variable "slurm_namespace" { - description = "slurm namespace for charts" - type = string - default = "slurm" -} - -variable "nodeset_name" { - description = "The nodeset name" - type = string - default = "gkenodeset" -} - -variable "slurm_bucket_dir" { - description = "Path directory within `bucket_name` for Slurm cluster file storage." - type = string - nullable = false -} - -variable "slurm_bucket" { - description = "GCS Bucket of Slurm cluster file storage." - type = any - nullable = true -} - -variable "instance_templates" { - description = "The URLs of Instance Templates" - type = list(string) - nullable = false -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured on nodes." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - - validation { - condition = length(var.network_storage) == 1 && var.network_storage[0].local_mount == "/home" - error_message = "The 'network_storage' variable must contain exactly one element, and that element's 'local_mount' attribute must be \"/home\"." - } -} - -variable "filestore_id" { - description = "An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." - type = list(string) - - validation { - condition = length(var.filestore_id) == 1 - error_message = "The 'filestore_id' variable must contain exactly one element." - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/versions.tf deleted file mode 100644 index 3d7237cb92..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-nodeset/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.84" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-nodeset/v1.51.0" - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/README.md deleted file mode 100644 index 2a7c363a87..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/README.md +++ /dev/null @@ -1,39 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 4.84 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [has\_tpu](#input\_has\_tpu) | If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu. | `bool` | `false` | no | -| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | -| [partition\_name](#input\_partition\_name) | The partition name | `string` | `"gke"` | no | -| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | -| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/main.tf deleted file mode 100644 index 2949fd6594..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/main.tf +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -data "google_storage_bucket" "this" { - name = var.slurm_bucket[0].name - - depends_on = [var.slurm_bucket] -} - -### Slurm Partition -locals { - partition_conf = { - "PowerDownOnIdle" = "NO" - "SuspendTime" = "INFINITE" - "SuspendTimeout" = var.has_tpu ? 240 : 120 - "ResumeTimeout" = var.has_tpu ? 600 : 300 - } - - partition = { - partition_name = var.partition_name - partition_conf = local.partition_conf - - partition_nodeset = [var.nodeset_name] - partition_nodeset_tpu = [] - partition_nodeset_dyn = [] - # Options - enable_job_exclusive = true - power_down_on_idle = false - } -} - -resource "google_storage_bucket_object" "parition_config" { - bucket = data.google_storage_bucket.this.name - name = "${var.slurm_bucket_dir}/partition_configs/${var.partition_name}.yaml" - content = yamlencode(local.partition) -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/metadata.yaml deleted file mode 100644 index 557e1fc2ae..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/variables.tf deleted file mode 100644 index 3aeed2e59a..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/variables.tf +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "has_tpu" { - description = "If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu." - type = bool - default = false -} - -variable "nodeset_name" { - description = "The nodeset name" - type = string - default = "gkenodeset" -} - -variable "partition_name" { - description = "The partition name" - type = string - default = "gke" -} - -variable "slurm_bucket_dir" { - description = "Path directory within `bucket_name` for Slurm cluster file storage." - type = string - nullable = false -} - -variable "slurm_bucket" { - description = "GCS Bucket of Slurm cluster file storage." - type = any - nullable = true -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/versions.tf deleted file mode 100644 index aede55263c..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/gke-partition/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.84" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-partition/v1.51.0" - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/README.md deleted file mode 100644 index 4f65411ddf..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/README.md +++ /dev/null @@ -1,271 +0,0 @@ -## Description - -This module performs the following tasks: - -- create an instance template from which execute points will be created -- create a managed instance group ([MIG][mig]) for execute points -- create a Toolkit runner to configure the autoscaler to scale the MIG - -It is expected to be used with the [htcondor-install] and [htcondor-setup] -modules. - -[htcondor-install]: ../../scripts/htcondor-install/README.md -[htcondor-setup]: ../../scheduler/htcondor-setup/README.md -[mig]: https://cloud.google.com/compute/docs/instance-groups/ - -### Known limitations - -This module may be used multiple times in a blueprint to create sets of -execute points in an HTCondor pool. If used more than 1 time, the setting -[name_prefix](#input_name_prefix) must be set to a value that is unique across -all uses of the htcondor-execute-point module. If you do not follow this -constraint, you will likely receive an error while running `terraform apply` -similar to that shown below. - -```text -Error: Invalid value for variable - - on modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf line 136, in module "startup_script": - 136: runners = local.all_runners - ├──────────────── - │ var.runners is list of map of string with 5 elements - -All startup-script runners must have a unique destination. -``` - -### How to configure jobs to select execute points - -HTCondor access points provisioned by the Toolkit are specially configured to -honor an attribute named `RequireId` in each [Job ClassAd][jobad]. This value -must be set to the ID of a MIG created by an instance of this module. The -[htcondor-access-point] module includes a setting `var.default_mig_id` that will -set this value automatically to the MIG ID corresponding to the module's -execute points. If this setting is left unset each job must specify `+RequireId` -explicitly. In all cases, the default value can be overridden explicitly as shown -below: - -```text -universe = vanilla -executable = /bin/echo -arguments = "Hello, World!" -output = out.$(ClusterId).$(ProcId) -error = err.$(ClusterId).$(ProcId) -log = log.$(ClusterId).$(ProcId) -request_cpus = 1 -request_memory = 100MB -+RequireId = "htcondor-pool-ep-mig" -queue -``` - -[htcondor-access-point]: ../../scheduler/htcondor-access-point/README.md -[jobad]: https://htcondor.readthedocs.io/en/latest/users-manual/matchmaking-with-classads.html - -### Example - -A full example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -The following code snippet creates a pool with 2 sets of HTCondor execute -points, one using On-demand pricing and the other using Spot pricing. They use -a startup script and network created in previous steps. - -```yaml -- id: htcondor_execute_point - source: community/modules/compute/htcondor-execute-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - min_idle: 2 - -- id: htcondor_execute_point_spot - source: community/modules/compute/htcondor-execute-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - spot: true - -- id: htcondor_access - source: community/modules/scheduler/htcondor-access-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - - htcondor_execute_point - - htcondor_execute_point_spot - settings: - default_mig_id: $(htcondor_execute_point.mig_id) - enable_public_ips: true - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - access_point_ips - - access_point_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## Behavior of Managed Instance Group (MIG) - -Regional [MIGs][mig] are used to provision Execute Points. By default, VMs -will be provisioned in any of the zones available in that region, however, it -can be constrained to run in fewer zones (or a single zone) using -[var.zones](#input_zones). - -When the configuration of an Execute Point is changed, the MIG can be configured -to [replace the VM][replacement] using a "proactive" or "opportunistic" policy. -By default, the policy is set to opportunistic. In practice, this means that -Execute Points will _NOT_ be automatically replaced by Terraform when changes to -the instance template / HTCondor configuration are made. We recommend leaving -this at the default value as it will allow the HTCondor autoscaler to replace -VMs when they become idle without disrupting running jobs. - -However, if it is desired [var.update_policy](#input_update_policy) can be set -to "PROACTIVE" to enable automatic replacement. This will disrupt running jobs -and send them back to the queue. Alternatively, one can leave the setting at -the default value of "OPPORTUNISTIC" and update: - -- intentionally by issuing an update via Cloud Console or using gcloud (below) -- VMs becomes unhealthy or are otherwise automatically replaced (e.g. regular - Google Cloud maintenance) - -For example, to manually update all instances in a MIG: - -```text -gcloud compute instance-groups managed update-instances \ - <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - -## Known Issues - -When using OS Login with "external users" (outside of the Google Cloud -organization), then Docker universe jobs will fail and cause the Docker daemon -to crash. This stems from the use of POSIX user ids (uid) outside the range -supported by Docker. Please consider disabling OS Login if this atypical -situation applies. - -```yaml -vars: - # add setting below to existing deployment variables - enable_oslogin: DISABLE -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 4.0 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.0 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [execute\_point\_instance\_template](#module\_execute\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [mig](#module\_mig) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.execute_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.execute_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Disk type for template | `string` | `"pd-balanced"` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape across zones for instance group managing execute points | `string` | `"ANY"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [execute\_point\_runner](#input\_execute\_point\_runner) | A list of Toolkit runners for configuring an HTCondor execute point | `list(map(string))` | `[]` | no | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account for HTCondor execute point (e-mail format) | `string` | n/a | yes | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | HTCondor execute point VM image

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to HTConodr execute points | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor execute points | `string` | `"n2-standard-4"` | no | -| [max\_size](#input\_max\_size) | Maximum size of the HTCondor execute point pool. | `number` | `5` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor execute points | `map(string)` | `{}` | no | -| [min\_idle](#input\_min\_idle) | Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max\_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly. | `number` | `0` | no | -| [name\_prefix](#input\_name\_prefix) | Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network HTCondor execute points will join | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HTCondor execute points will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which HTCondor execute points will be created | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork HTCondor execute points will join | `string` | `null` | no | -| [target\_size](#input\_target\_size) | Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size. | `number` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | -| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | Startup script to run at boot-time for Windows-based HTCondor execute points | `list(string)` | `[]` | no | -| [zones](#input\_zones) | Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [autoscaler\_runner](#output\_autoscaler\_runner) | Toolkit runner to configure the HTCondor autoscaler | -| [mig\_id](#output\_mig\_id) | ID of the managed instance group containing the execute points | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml deleted file mode 100644 index 375ae036cd..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Role - hosts: localhost - become: true - vars: - spool_dir: /var/lib/condor/spool - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - handlers: - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml deleted file mode 100644 index a85158fdfc..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This playbook makes the assumption that a virtual environment has been created -# with the autoscaler and its dependencies previously installed. A runner that -# does this is provided as an output of the htcondor-install module within the -# Cluster Toolkit at community/modules/scripts/htcondor-install. - ---- -- name: Configure HTCondor Autoscaler - hosts: all - vars: - python: /usr/local/htcondor/bin/python3 - autoscaler: /usr/local/htcondor/bin/autoscaler.py - systemd_override_path: /etc/systemd/system - become: true - tasks: - - name: User must supply HTCondor role - ansible.builtin.assert: - that: - - project_id is defined - - region is defined - - zone is defined - - mig_id is defined - - max_size is defined - - name: Create SystemD service for HTCondor autoscaler - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@.service" - mode: 0644 - content: | - [Unit] - Description=HTCondor Autoscaler MIG: %i - - [Service] - User=condor - Type=oneshot - ExecStart={{ python }} {{ autoscaler }} --p $PROJECT_ID --r $REGION --z $ZONE --mz --g %i --c $MAX_SIZE --i $MIN_IDLE - notify: - - Reload SystemD - - name: Create SystemD override directory for autoscaler configuration - ansible.builtin.file: - path: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d" - state: directory - owner: root - group: root - mode: 0755 - - name: Create autoscaler configuration - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d/miglimit.conf" - mode: 0644 - content: | - [Service] - Environment=PROJECT_ID={{ project_id }} - Environment=REGION={{ region }} - Environment=ZONE={{ zone }} - Environment=MAX_SIZE={{ max_size }} - Environment=MIN_IDLE={{ min_idle }} - notify: - - Reload SystemD - - name: Create SystemD timer for HTCondor autoscaler - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@.timer" - mode: 0644 - content: | - [Unit] - Description=Run HTCondor Autoscaler Periodically - - [Timer] - OnCalendar=minutely - AccuracySec=1us - RandomizedDelaySec=30 - # the directive below is ignored harmlessly on CentOS 7; this has impact - # that timing averages to 1 minute but is not precisely 1 minute; still - # useful to ensure that timers for different MIGs do not overlap - FixedRandomDelay=true - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - post_tasks: - - name: Activate HTCondor Autoscaler timer - ansible.builtin.systemd: - name: htcondor-autoscaler@{{ mig_id }}.timer - enabled: true - state: started diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf deleted file mode 100644 index 7b0df94987..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf +++ /dev/null @@ -1,218 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-execute-point", ghpc_role = "compute" }) -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - zones = coalescelist(var.zones, data.google_compute_zones.available.names) - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - windows_startup_ps1 = join("\n\n", flatten([var.windows_startup_ps1, local.execute_config_windows_startup_ps1])) - - is_windows_image = anytrue([for l in data.google_compute_image.compute_image.licenses : length(regexall("windows-cloud", l)) > 0]) - windows_startup_metadata = local.is_windows_image && local.windows_startup_ps1 != "" ? { - windows-startup-script-ps1 = local.windows_startup_ps1 - } : {} - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.windows_startup_metadata, - local.network_storage_metadata, - local.enable_oslogin, - local.disable_automatic_updates_metadata, - var.metadata - ) - - autoscaler_runner = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_configure_autoscaler.yml") - "destination" = "htcondor_configure_autoscaler_${module.mig.instance_group_manager.name}.yml" - "args" = join(" ", [ - "-e project_id=${var.project_id}", - "-e region=${var.region}", - "-e zone=${local.zones[0]}", # this value is required, but ignored by regional MIG autoscaler - "-e mig_id=${module.mig.instance_group_manager.name}", - "-e max_size=${var.max_size}", - "-e min_idle=${var.min_idle}", - ]) - } - - execute_config = templatefile("${path.module}/templates/condor_config.tftpl", { - htcondor_role = "get_htcondor_execute", - central_manager_ips = var.central_manager_ips, - guest_accelerator = local.guest_accelerator, - }) - - execute_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.execute_config.output_name}" - execute_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e htcondor_role=get_htcondor_execute", - "-e config_object=${local.execute_object}", - ]) - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.execute_point_runner, - [local.execute_runner], - ) - - execute_config_windows_startup_ps1 = templatefile( - "${path.module}/templates/download-condor-config.ps1.tftpl", - { - config_object = local.execute_object, - } - ) - - name_prefix = "${var.deployment_name}-${var.name_prefix}-ep" -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -resource "null_resource" "execute_config" { - triggers = { - config = local.execute_config - } -} - -resource "google_storage_bucket_object" "execute_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.execute_config.id), 0, 4)}" - content = local.execute_config - bucket = var.htcondor_bucket_name -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -module "execute_point_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.execute_point_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - gpu = one(local.guest_accelerator) - preemptible = var.spot - startup_script = local.is_windows_image ? null : module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.compute_image.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config -} - -module "mig" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = var.target_size - hostname = local.name_prefix - mig_name = local.name_prefix - instance_template = module.execute_point_instance_template.self_link - - health_check_name = "health-htcondor-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "SUBSTITUTE" - max_surge_fixed = length(local.zones) - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf deleted file mode 100644 index b31f40130f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "autoscaler_runner" { - value = local.autoscaler_runner - description = "Toolkit runner to configure the HTCondor autoscaler" -} - -output "mig_id" { - value = module.mig.instance_group_manager.name - description = "ID of the managed instance group containing the execute points" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl deleted file mode 100644 index c8f5ce31a8..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:${htcondor_role} -CONDOR_HOST = ${join(",", central_manager_ips)} - -# StartD configuration settings -%{ if length(guest_accelerator) > 0 ~} -use feature:GPUs -%{ endif ~} -use feature:PartitionableSlot -use feature:CommonCloudAttributesGoogle("-c created-by") -UPDATE_INTERVAL = 30 -TRUST_UID_DOMAIN = True -STARTER_ALLOW_RUNAS_OWNER = True -RUNBENCHMARKS = False diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl deleted file mode 100644 index 19789f122e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl +++ /dev/null @@ -1,34 +0,0 @@ -# create directory for local condor_config customizations -$config_dir = 'C:\Condor\config' -if(!(test-path -PathType container -Path $config_dir)) -{ - New-Item -ItemType Directory -Path $config_dir -} - -# update local condor_config if blueprint has changed -$config_file = "$config_dir\50-ghpc-managed" -if (Test-Path -Path $config_file -PathType Leaf) -{ - $local_hash = gcloud --format="value(md5_hash)" storage hash $config_file -} -else -{ - $local_hash = "INVALID-HASH" -} - -$remote_hash = gcloud --format="value(md5_hash)" storage hash ${config_object} -if ($local_hash -cne $remote_hash) -{ - Write-Output "Updating condor configuration" - gcloud storage cp ${config_object} $config_file - if ($LASTEXITCODE -ne 0) - { - throw "Could not download HTCondor configuration; exiting startup script" - } - Restart-Service condor -} - -# ignored if service is already running; must be here to handle case where -# machine is rebooted, but configuration has previously been downloaded -# and service is disabled from automatic start -Start-Service condor diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf deleted file mode 100644 index aab8a54c2d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf +++ /dev/null @@ -1,265 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HTCondor execute points will be created" - type = string -} - -variable "region" { - description = "The region in which HTCondor execute points will be created" - type = string -} - -variable "zones" { - description = "Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region." - type = list(string) - default = [] - nullable = false -} - -variable "distribution_policy_target_shape" { - description = "Target shape across zones for instance group managing execute points" - type = string - default = "ANY" -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to HTConodr execute points" - type = map(string) -} - -variable "machine_type" { - description = "Machine type to use for HTCondor execute points" - type = string - default = "n2-standard-4" -} - -variable "execute_point_runner" { - description = "A list of Toolkit runners for configuring an HTCondor execute point" - type = list(map(string)) - default = [] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "instance_image" { - description = <<-EOD - HTCondor execute point VM image - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "execute_point_service_account_email" { - description = "Service account for HTCondor execute point (e-mail format)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_self_link" { - description = "The self link of the network HTCondor execute points will join" - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork HTCondor execute points will join" - type = string - default = null -} - -variable "target_size" { - description = "Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size." - type = number - default = null -} - -variable "max_size" { - description = "Maximum size of the HTCondor execute point pool." - type = number - default = 5 -} - -variable "min_idle" { - description = "Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly." - type = number - default = 0 -} - -variable "metadata" { - description = "Metadata to add to HTCondor execute points" - type = map(string) - default = {} -} - -# this default is deliberately the opposite of vm-instance because of observed -# issues running HTCondor docker universe jobs with OS Login enabled and running -# jobs as a user with uid>2^31; these uids occur when users outside the GCP -# organization login to a VM and OS Login is enabled. -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "spot" { - description = "Provision VMs using discounted Spot pricing, allowing for preemption" - type = bool - default = false -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 100 -} - -variable "disk_type" { - description = "Disk type for template" - type = string - default = "pd-balanced" -} - -variable "windows_startup_ps1" { - description = "Startup script to run at boot-time for Windows-based HTCondor execute points" - type = list(string) - default = [] - nullable = false -} - -variable "central_manager_ips" { - description = "List of IP addresses of HTCondor Central Managers" - type = list(string) -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The HTCondor module supports 0 or 1 models of accelerator card on each execute point" - } -} - -variable "name_prefix" { - description = "Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module" - type = string - nullable = false - validation { - condition = length(var.name_prefix) > 0 - error_message = "var.name_prefix must be a set to a non-empty string and must also be unique across all instances of htcondor-execute-point" - } -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" - type = string - default = "OPPORTUNISTIC" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf deleted file mode 100644 index 729dc3cda5..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 1.1" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.74.0" - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/README.md deleted file mode 100644 index 278207b04a..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/README.md +++ /dev/null @@ -1,45 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_group_manager.mig](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_group_manager) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [base\_instance\_name](#input\_base\_instance\_name) | Base name for the instances in the MIG | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will be used to name MIG if `var.name` is not provided | `string` | n/a | yes | -| [ghpc\_module\_id](#input\_ghpc\_module\_id) | Internal GHPC field, do not set this value | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the MIG | `map(string)` | n/a | yes | -| [name](#input\_name) | Name of the MIG. If not provided, will be generated from `var.deployment_name` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the MIG will be created | `string` | n/a | yes | -| [target\_size](#input\_target\_size) | Target number of instances in the MIG | `number` | `0` | no | -| [versions](#input\_versions) | Application versions managed by this instance group. Each version deals with a specific instance template | 
list(object({
    name              = string
    instance_template = string
    target_size = optional(object({
      fixed   = optional(number)
      percent = optional(number)
    }))
  }))
| n/a | yes | -| [wait\_for\_instances](#input\_wait\_for\_instances) | Whether to wait for all instances to be created/updated before returning | `bool` | `false` | no | -| [zone](#input\_zone) | Compute Platform zone. Required, currently only zonal MIGs are supported | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [self\_link](#output\_self\_link) | The URL of the created MIG | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/main.tf deleted file mode 100644 index 0e7cf186c2..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/main.tf +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "mig", ghpc_role = "compute" }) -} - -locals { - sanitized_deploy_name = try(replace(lower(var.deployment_name), "/[^a-z0-9]/", ""), null) - sanitized_module_id = try(replace(lower(var.ghpc_module_id), "/[^a-z0-9]/", ""), null) - synth_mig_name = try("${local.sanitized_deploy_name}-${local.sanitized_module_id}", null) - - mig_name = var.name == null ? local.synth_mig_name : var.name - base_instance_name = var.base_instance_name == null ? local.mig_name : var.base_instance_name -} - -resource "google_compute_instance_group_manager" "mig" { - # REQUIRED - name = local.mig_name - base_instance_name = local.base_instance_name - zone = var.zone - - dynamic "version" { - for_each = var.versions - content { - name = version.value.name - instance_template = version.value.instance_template - dynamic "target_size" { - for_each = version.value.target_size != null ? [version.value.target_size] : [] - content { - fixed = target_size.value.fixed - percent = target_size.value.percent - } - } - } - } - - # OPTIONAL - project = var.project_id - target_size = var.target_size - wait_for_instances = var.wait_for_instances - - all_instances_config { - # TODO: validate that template metadata not getting wiped out - # TODO: validate that template labels not getting wiped out - labels = local.labels - } - - # OMITTED: - # * description - # * named_port - # * list_managed_instances_results - # * target_pools - specific for Load Balancers usage - # * wait_for_instances_status - # * auto_healing_policies - # * stateful_disk - # * stateful_internal_ip - # * update_policy - # * params - - - lifecycle { - precondition { - condition = local.mig_name != null - error_message = "Could not come up with a name for the MIG, specify `var.name`" - } - - precondition { - condition = local.base_instance_name != null - error_message = "Could not come up with a base_instance_name, specify `var.base_instance_name`" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/metadata.yaml deleted file mode 100644 index 97a4fa9a89..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com -ghpc: - inject_module_id: ghpc_module_id diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/outputs.tf deleted file mode 100644 index 23c66a3535..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "self_link" { - description = "The URL of the created MIG" - value = google_compute_instance_group_manager.mig.self_link -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/variables.tf deleted file mode 100644 index b6c3c0e78a..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/variables.tf +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "Project in which the MIG will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the deployment, will be used to name MIG if `var.name` is not provided" - type = string -} - -variable "labels" { - description = "Labels to add to the MIG" - type = map(string) -} - -variable "zone" { - description = "Compute Platform zone. Required, currently only zonal MIGs are supported" - type = string -} - - -variable "versions" { - description = <<-EOD - Application versions managed by this instance group. Each version deals with a specific instance template - EOD - type = list(object({ - name = string - instance_template = string - target_size = optional(object({ - fixed = optional(number) - percent = optional(number) - })) - })) - - validation { - condition = length(var.versions) > 0 - error_message = "At least one version must be provided" - } - -} - - -variable "ghpc_module_id" { - description = "Internal GHPC field, do not set this value" - type = string - default = null -} - -variable "name" { - description = "Name of the MIG. If not provided, will be generated from `var.deployment_name`" - type = string - default = null -} - -variable "base_instance_name" { - description = "Base name for the instances in the MIG" - type = string - default = null -} - - -variable "target_size" { - description = "Target number of instances in the MIG" - type = number - default = 0 -} - -variable "wait_for_instances" { - description = "Whether to wait for all instances to be created/updated before returning" - type = bool - default = false -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/mig/versions.tf deleted file mode 100644 index 4147447b44..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/mig/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - google = { - source = "hashicorp/google" - version = "> 5.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:mig/v1.74.0" - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/README.md deleted file mode 100644 index 1dcacc57e9..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# Description - -This module creates the Vertex AI Notebook, to be used in tutorials. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage - -This is a simple usage, using the default network: - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: - name_prefix: my-bucket - local_mount: /home/jupyter/my-bucket - - - id: notebook - source: community/modules/compute/notebook - use: [bucket] - settings: - name_prefix: notebook - machine_type: n1-standard-4 - -``` - -If the user wants do specify a custom subnetwork, or specific external IP restrictions, they can use the `network_interfaces` variable, here is an example on how to use a Shared VPC Subnet with an ephemeral external IP: - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: - name_prefix: my-bucket - local_mount: /home/jupyter/my-bucket - - - id: notebook - source: community/modules/compute/notebook - use: [bucket] - settings: - name_prefix: notebook - machine_type: n1-standard-4 - network_interfaces: - - network: "projects/HOST_PROJECT_ID/global/networks/SHARED_VPC_NAME" - subnet: "projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME" - nic_type: "VIRTIO_NET" -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 5.34 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 5.34 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.mount_script](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_workbench_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/workbench_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the notebook. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name, can be provided from the google-cloud-storage module | `string` | `null` | no | -| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "tf-latest-cpu",
  "name": null,
  "project": "deeplearning-platform-release"
}
| no | -| [labels](#input\_labels) | Labels to add to the resource Key-value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | The machine type to employ | `string` | n/a | yes | -| [mount\_runner](#input\_mount\_runner) | mount content from the google-cloud-storage module | `map(string)` | n/a | yes | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces for the VM instance. Each network interface is represented by an object with the following fields:

- network: (Optional) The name of the Virtual Private Cloud (VPC) network that this VM instance is connected to.

- subnet: (Optional) The name of the subnetwork within the specified VPC that this VM instance is connected to.

- nic\_type: (Optional) The type of vNIC to be used on this interface. Possible values are: `VIRTIO_NET`, `GVNIC`.

- access\_configs: (Optional) An array of access configurations for this network interface. The access\_config object contains:
* external\_ip: (Required) An external IP address associated with this instance. Specify an unused static external IP address available to the project or leave this field undefined to use an IP from a shared ephemeral IP address pool. If you specify a static external IP address, it must live in the same region as the zone of the instance. | 
list(object({
    network  = optional(string)
    subnet   = optional(string)
    nic_type = optional(string)
    access_configs = optional(list(object({
      external_ip = optional(string)
    })))
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | ID of project in which the notebook will be created. | `string` | n/a | yes | -| [service\_account\_email](#input\_service\_account\_email) | If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account | `string` | `null` | no | -| [zone](#input\_zone) | The zone to deploy to | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/main.tf deleted file mode 100644 index cd3ce3b4ea..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/main.tf +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "notebook", ghpc_role = "compute" }) -} - -locals { - suffix = random_id.resource_name_suffix.hex - #name = "thenotebook" - name = "notebook-${var.deployment_name}-${local.suffix}" - bucket = replace(var.gcs_bucket_path, "gs://", "") - post_script_filename = "mount-${local.suffix}.sh" - - # mount_runner_args is defined in the file: cluster-toolkit/modules/file-system/cloud-storage-bucket/outputs.tf - mount_args = split(" ", var.mount_runner.args) - - unused = local.mount_args[0] - remote_mount = local.mount_args[1] - local_mount = local.mount_args[2] - fs_type = local.mount_args[3] - # These options provide a "rw" mount of the GCS bucket - mount_options = "defaults,_netdev,allow_other,implicit_dirs,gid=1000,uid=1000" - - content0 = var.mount_runner.content - content1 = replace(local.content0, "$1", local.unused) - content2 = replace(local.content1, "$2", local.remote_mount) - content3 = replace(local.content2, "$3", local.local_mount) - content4 = replace(local.content3, "$4", local.fs_type) - content5 = replace(local.content4, "$5", local.mount_options) - -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket_object" "mount_script" { - name = local.post_script_filename - content = local.content5 - bucket = local.bucket -} - -resource "google_workbench_instance" "instance" { - name = local.name - location = var.zone - project = var.project_id - labels = local.labels - gce_setup { - machine_type = var.machine_type - metadata = { - post-startup-script = "${var.gcs_bucket_path}/${google_storage_bucket_object.mount_script.name}" - } - vm_image { - project = var.instance_image.project - family = var.instance_image.family - } - - dynamic "service_accounts" { - for_each = var.service_account_email == null ? [] : [1] - content { - email = var.service_account_email - } - } - - dynamic "network_interfaces" { - for_each = var.network_interfaces - content { - network = network_interfaces.value.network - subnet = network_interfaces.value.subnet - nic_type = network_interfaces.value.nic_type - - dynamic "access_configs" { - for_each = network_interfaces.value.access_configs != null ? network_interfaces.value.access_configs : [] - content { - external_ip = access_configs.value.external_ip - } - } - } - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/metadata.yaml deleted file mode 100644 index 4a7d5397ca..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - notebooks.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/variables.tf deleted file mode 100644 index 4359de8c10..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/notebook/variables.tf +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which the notebook will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment; used as part of name of the notebook." - type = string - # notebook name can have: lowercase letters, numbers, or hyphens (-) and cannot end with a hyphen - validation { - error_message = "The notebook name uses 'deployment_name' -- can only have: lowercase letters, numbers, or hyphens" - condition = can(regex("^[a-z0-9]+(?:-[a-z0-9]+)*$", var.deployment_name)) - } -} - -variable "zone" { - description = "The zone to deploy to" - type = string -} - -variable "machine_type" { - description = "The machine type to employ" - type = string -} - -variable "labels" { - description = "Labels to add to the resource Key-value pairs." - type = map(string) -} - -variable "instance_image" { - description = "Instance Image" - type = map(string) - default = { - project = "deeplearning-platform-release" - family = "tf-latest-cpu" - name = null - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "gcs_bucket_path" { - description = "Bucket name, can be provided from the google-cloud-storage module" - type = string - default = null -} - -variable "mount_runner" { - description = "mount content from the google-cloud-storage module" - type = map(string) - - validation { - condition = (length(split(" ", var.mount_runner.args)) == 5) - error_message = "There must be 5 elements in the Mount Runner Arguments: ${var.mount_runner.args} \n " - } -} - -variable "service_account_email" { - description = "If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account" - type = string - default = null -} - -variable "network_interfaces" { - type = list(object({ - network = optional(string) - subnet = optional(string) - nic_type = optional(string) - access_configs = optional(list(object({ - external_ip = optional(string) - }))) - })) - default = [] - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | -| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name    = string
    device_name  = string
    disk_size_gb = number
    disk_type    = string
    disk_labels  = map(string)
    auto_delete  = bool
    boot         = bool
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | -| [feature](#input\_feature) | The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used. | `string` | `null` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | Path to the Slurm bucket. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster. | `string` | n/a | yes | -| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_template\_self\_link](#output\_instance\_template\_self\_link) | The URI of the template. | -| [node\_name\_prefix](#output\_node\_name\_prefix) | The prefix to be used for the node names.

Make sure that nodes are named `-`
This temporary required for proper functioning of the nodes.
While Slurm scheduler uses "features" to bind node and nodeset,
the SlurmGCP relies on node names for this (to be switched to features as well). | -| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf deleted file mode 100644 index 31d9f14ae7..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset-dynamic", ghpc_role = "compute" }) -} - -module "instance_validation" { - source = "../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - nodeset_name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - feature = coalesce(var.feature, local.nodeset_name) - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - universe_domain = { "universe_domain" = var.universe_domain } - - metadata = merge( - local.disable_automatic_updates_metadata, - local.universe_domain, - { slurmd_feature = local.feature }, - var.metadata - ) - - nodeset = { - nodeset_name = local.nodeset_name - nodeset_feature : local.feature - startup_script = local.ghpc_startup_script - network_storage = var.network_storage - } - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - } - ] - - public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - ghpc_startup_script = [{ - filename = "ghpc_nodeset_startup.sh" - content = var.startup_script - }] - -} - -module "slurm_nodeset_template" { - source = "../../internal/slurm-gcp/instance_template" - - project_id = var.project_id - region = var.region - name_prefix = local.nodeset_name - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = "compute" - slurm_bucket_path = var.slurm_bucket_path - metadata = local.metadata - - additional_disks = local.additional_disks - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(local.labels, var.disk_labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - - bandwidth_tier = var.bandwidth_tier - can_ip_forward = var.can_ip_forward - - advanced_machine_features = var.advanced_machine_features - enable_confidential_vm = var.enable_confidential_vm - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - labels = local.labels - machine_type = var.machine_type - - min_cpu_platform = var.min_cpu_platform - on_host_maintenance = var.on_host_maintenance - termination_action = try(var.spot_instance_config.termination_action, null) - preemptible = var.preemptible - spot = var.enable_spot_vm - service_account = local.service_account - gpu = one(local.guest_accelerator) # requires gpu_definition.tf - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - subnetwork = var.subnetwork_self_link - additional_networks = var.additional_networks - access_config = local.access_config - tags = concat([var.slurm_cluster_name], var.tags) -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml deleted file mode 100644 index a99e59d09f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [compute.googleapis.com] -ghpc: - inject_module_id: name diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf deleted file mode 100644 index 2d2d1415cf..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset_dyn" { - description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset -} - -output "instance_template_self_link" { - description = "The URI of the template." - value = module.slurm_nodeset_template.self_link -} - -output "node_name_prefix" { - description = <<-EOD - The prefix to be used for the node names. - - Make sure that nodes are named `-` - This temporary required for proper functioning of the nodes. - While Slurm scheduler uses "features" to bind node and nodeset, - the SlurmGCP relies on node names for this (to be switched to features as well). - EOD - value = "${var.slurm_cluster_name}-${local.nodeset_name}" - -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf deleted file mode 100644 index ec6206e317..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf +++ /dev/null @@ -1,402 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "feature" { - type = string - description = "The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used." - default = null -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "slurm_cluster_name" { - description = "Name of the Slurm cluster." - type = string -} - -variable "slurm_bucket_path" { - description = "Path to the Slurm bucket." - type = string -} - - -variable "machine_type" { - description = "Compute Platform machine type to use for this partition compute nodes." - type = string - default = "c2-standard-60" -} - -variable "metadata" { - type = map(string) - description = "Metadata, provided as a map." - default = {} -} - -variable "instance_image" { - description = <<-EOD - Defines the image that will be used in the Slurm node group VM instances. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - - For more information on creating custom images that comply with Slurm on GCP - see the "Slurm on GCP Custom Images" section in docs/vm-images.md. - EOD - type = map(string) - default = { - family = "slurm-gcp-6-11-hpc-rocky-linux-8" - project = "schedmd-slurm-public" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations - description = <<-EOD - A flag that designates that the user is aware that they are requesting - to use a custom and potentially incompatible image for this Slurm on - GCP module. - - If the field is set to false, only the compatible families and project - names will be accepted. The deployment will fail with any other image - family or name. If set to true, no checks will be done. - - See: https://goo.gle/hpc-slurm-images - EOD - type = bool - default = false -} - - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "tags" { - type = list(string) - description = "Network tag list." - default = [] -} - -variable "disk_type" { - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - type = string - default = "pd-standard" -} - -variable "disk_size_gb" { - description = "Size of boot disk to create for the partition compute nodes." - type = number - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "additional_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = list(object({ - disk_name = string - device_name = string - disk_size_gb = number - disk_type = string - disk_labels = map(string) - auto_delete = bool - boot = bool - })) - default = [] -} - -variable "enable_confidential_vm" { - type = bool - description = "Enable the Confidential VM configuration. Note: the instance image must support option." - default = false -} - -variable "enable_shielded_vm" { - type = bool - description = "Enable the Shielded VM configuration. Note: the instance image must support option." - default = false -} - -variable "shielded_instance_config" { - type = object({ - enable_integrity_monitoring = bool - enable_secure_boot = bool - enable_vtpm = bool - }) - description = <<-EOD - Shielded VM configuration for the instance. Note: not used unless - enable_shielded_vm is 'true'. - - enable_integrity_monitoring : Compare the most recent boot measurements to the - integrity policy baseline and return a pair of pass/fail results depending on - whether they match or not. - - enable_secure_boot : Verify the digital signature of all boot components, and - halt the boot process if signature verification fails. - - enable_vtpm : Use a virtualized trusted platform module, which is a - specialized computer chip you can use to encrypt objects like keys and - certificates. - EOD - default = { - enable_integrity_monitoring = true - enable_secure_boot = true - enable_vtpm = true - } -} - - -variable "enable_oslogin" { - type = bool - description = <<-EOD - Enables Google Cloud os-login for user login and authentication for VMs. - See https://cloud.google.com/compute/docs/oslogin - EOD - default = true -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example." - type = bool - default = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "labels" { - description = "Labels to add to partition compute instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "min_cpu_platform" { - description = "The name of the minimum CPU platform that you want the instance to use." - type = string - default = null -} - -variable "on_host_maintenance" { - type = string - description = <<-EOD - Instance availability Policy. - - Note: Placement groups are not supported when on_host_maintenance is set to - "MIGRATE" and will be deactivated regardless of the value of - enable_placement. To support enable_placement, ensure on_host_maintenance is - set to "TERMINATE". - EOD - default = "TERMINATE" -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." - } -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - - -variable "service_account_email" { - description = "Service account e-mail address to attach to the compute instances." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the compute instances." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "enable_spot_vm" { - description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." - type = bool - default = false -} - -variable "spot_instance_config" { - description = "Configuration for spot VMs." - type = object({ - termination_action = string - }) - default = null -} - -variable "bandwidth_tier" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | -| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | -| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | -| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `null` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling worker nodes allowed in this partition.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of worker nodes to be statically created.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | -| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. | `string` | `""` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the TPU-vm. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the TPU-vm. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | -| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | `"2.14.0"` | no | -| [zone](#input\_zone) | Zone in which to create compute VMs. TPU partitions can only specify a single zone. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf deleted file mode 100644 index ac9b119702..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# locals { -# # This label allows for billing report tracking based on module. -# labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) -# } - -locals { - name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - nodeset_tpu = { - node_count_static = var.node_count_static - node_count_dynamic_max = var.node_count_dynamic_max - nodeset_name = local.name - node_type = var.node_type - - accelerator_config = var.accelerator_config - tf_version = var.tf_version - preemptible = var.preemptible - preserve_tpu = var.preserve_tpu - - data_disks = var.data_disks - docker_image = var.docker_image - - enable_public_ip = var.enable_public_ips - # TODO: rename to subnetwork_self_link, requires changes to the scripts - subnetwork = var.subnetwork_self_link - service_account = local.service_account - zone = var.zone - - project_id = var.project_id - reserved = var.reserved - network_storage = var.network_storage - } - - node_type_core_count = var.node_type == "" ? 0 : tonumber(regex("-(.*)", var.node_type)[0]) - - accelerator_core_list = var.accelerator_config.topology == "" ? [0, 0] : regexall("\\d+", var.accelerator_config.topology) - accelerator_core_count = length(local.accelerator_core_list) > 2 ? (local.accelerator_core_list[0] * local.accelerator_core_list[1] * local.accelerator_core_list[2]) * 2 : (local.accelerator_core_list[0] * local.accelerator_core_list[1]) * 2 - - tpu_core_count = local.accelerator_core_count == 0 ? local.node_type_core_count : local.accelerator_core_count -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml deleted file mode 100644 index 95b6d1c730..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name - has_to_be_used: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf deleted file mode 100644 index 8cb7b8663e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset_tpu" { - description = "Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset_tpu - - precondition { - condition = (var.node_type == "") != (var.accelerator_config == { topology : "", version : "" }) - error_message = "Either a node_type or an accelerator_config must be provided." - } - - precondition { - condition = ((local.tpu_core_count / 8) <= var.node_count_dynamic_max) || ((local.tpu_core_count / 8) <= var.node_count_static) - error_message = <<-EOD - When using TPUs there should be at least one node per every 8 cores. - Currently there are ${local.tpu_core_count} cores but only ${var.node_count_static} static nodes and ${var.node_count_dynamic_max} dynamic nodes. - EOD - } - - precondition { - condition = (var.node_count_dynamic_max % (local.tpu_core_count / 8) == 0) && (var.node_count_static % (local.tpu_core_count / 8) == 0) - error_message = <<-EOD - The number of worker nodes should be a multiple of ${local.tpu_core_count / 8}. - This is to ensure each node has a TPU machine for job scheduling. - EOD - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf deleted file mode 100644 index 367b0bee09..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "node_count_static" { - description = <<-EOD - Number of worker nodes to be statically created. - For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). - See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. - EOD - type = number - default = 0 -} - -variable "node_count_dynamic_max" { - description = <<-EOD - Maximum number of auto-scaling worker nodes allowed in this partition. - For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). - See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. - EOD - type = number - default = 0 -} - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "enable_public_ips" { - description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." - type = bool - default = false -} - -variable "disable_public_ips" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `enable_public_ips` instead." - type = bool - default = null - validation { - condition = var.disable_public_ips == null - error_message = "DEPRECATED: Use `enable_public_ips` instead." - } -} - -variable "node_type" { - description = "Specify a node type to base the vm configuration upon it." - type = string - default = "" -} - -variable "accelerator_config" { - description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." - type = object({ - topology = string - version = string - }) - default = { - topology = "" - version = "" - } - validation { - condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], var.accelerator_config.version) - error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" - } - validation { - condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) - error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." - } -} - -variable "tf_version" { - description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." - type = string - default = "2.14.0" -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - -variable "preserve_tpu" { - description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" - type = bool - default = false -} - -variable "zone" { - description = "Zone in which to create compute VMs. TPU partitions can only specify a single zone." - type = string -} - -variable "data_disks" { - description = "The data disks to include in the TPU node" - type = list(string) - default = [] -} - -variable "docker_image" { - description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" - type = string - default = null -} - -variable "subnetwork_self_link" { - type = string - description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." -} - -variable "service_account_email" { - description = "Service account e-mail address to attach to the TPU-vm." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the TPU-vm." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "service_account" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - } -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "reserved" { - description = "Specify whether TPU-vms in this nodeset are created under a reservation." - type = bool - default = false -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured on nodes." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - })) - default = [] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf deleted file mode 100644 index 398eeffdda..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.3" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.74.0" - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md deleted file mode 100644 index 7c9e32debf..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md +++ /dev/null @@ -1,227 +0,0 @@ -## Description - -This module creates a nodeset data structure intended to be input to the -[schedmd-slurm-gcp-v6-partition](../schedmd-slurm-gcp-v6-partition/) module. - -Nodesets allow adding heterogeneous node types to a partition, and hence -running jobs that mix multiple node characteristics. See the [heterogeneous jobs -section][hetjobs] of the SchedMD documentation for more information. - -To specify nodes from a specific nodesets in a partition, the [`--nodelist`] -(or `-w`) flag can be used, for example: - -```bash -srun -N 3 -p compute --nodelist cluster-compute-group-[0-2] hostname -``` - -Where the 3 nodes will be selected from the nodes `cluster-compute-group-[0-2]` -in the compute partition. - -Additionally, depending on how the nodes differ, a constraint can be added via -the [`--constraint`] (or `-C`) flag or other flags such as `--mincpus` can be -used to specify nodes with the desired characteristics. - -[`--nodelist`]: https://slurm.schedmd.com/srun.html#OPT_nodelist -[`--constraint`]: https://slurm.schedmd.com/srun.html#OPT_constraint -[hetjobs]: https://slurm.schedmd.com/heterogeneous_jobs.html - -### Example - -The following code snippet creates a partition module using the `nodeset` -module as input with: - -* a max node count of 200 -* VM machine type of `c2-standard-30` -* partition name of "compute" -* default nodeset name of "ghpc" -* connected to the `network` module via `use` -* nodes mounted to homefs via `use` - -```yaml -- id: nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - node_count_dynamic_max: 200 - machine_type: c2-standard-30 - -- id: compute_partition - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - use: - - homefs - - nodeset - settings: - partition_name: compute -``` - -## Custom Images - -For more information on creating valid custom images for the node group VM -instances or for custom instance templates, see our [vm-images.md] documentation -page. - -[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images - -## GPU Support - -More information on GPU support in Slurm on GCP and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) - -### Compute VM Zone Policies - -The Slurm on GCP nodeset module allows you to specify additional zones in -which to create VMs through [bulk creation][bulk]. This is valuable when -configuring partitions with popular VM families and you desire access to -more compute resources across zones. - -[bulk]: https://cloud.google.com/compute/docs/instances/multiple/about-bulk-creation -[networkpricing]: https://cloud.google.com/vpc/network-pricing - -> **_WARNING:_** Lenient zone policies can lead to additional egress costs when -> moving large amounts of data between zones in the same region. For example, -> traffic between VMs and traffic from VMs to shared filesystems such as -> Filestore. For more information on egress fees, see the -> [Network Pricing][networkpricing] Google Cloud documentation. -> -> To avoid egress charges, ensure your compute nodes are created in a single -> zone by setting var.zone and leaving var.zones to its default value of the -> empty list. -> -> **_NOTE:_** If a new zone is added to the region while the cluster is active, -> nodes in the partition may be created in that zone. In this case, the -> partition may need to be redeployed to ensure the newly added zone is denied. - -In the zonal example below, the nodeset's zone implicitly defaults to the -deployment variable `vars.zone`: - -```yaml -vars: - zone: us-central1-f - -- id: zonal-nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset -``` - -In the example below, we enable creation in additional zones: - -```yaml -vars: - zone: us-central1-f - -- id: multi-zonal-nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - settings: - zones: - - us-central1-a - - us-central1-b -``` - -## Support -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4 | -| [google](#requirement\_google) | >= 5.11 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 5.11 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.machine_type_zone_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [google_compute_machine_types.machine_types_by_zone](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | -| [google_compute_reservation.reservation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_topology](#input\_accelerator\_topology) | Specifies the shape of the Accelerator (GPU/TPU) slice. | `string` | `null` | no | -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = optional(string)
    subnetwork         = string
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    stack_type         = optional(string)
    queue_count        = optional(number)
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | -| [dws\_flex](#input\_dws\_flex) | If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes.
See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler
Options:
- enable: Enable DWS Flex Start
- max\_run\_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week).
- use\_job\_duration: Use the job duration to determine the max\_run\_duration, if job duration is not set, max\_run\_duration will be used.
- use\_bulk\_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances

Limitations:
- CAN NOT be used with reservations;
- CAN NOT be used with placement groups; | 
object({
    enabled          = optional(bool, true)
    max_run_duration = optional(number, 604800) # one week
    use_job_duration = optional(bool, false)
    use_bulk_insert  = optional(bool, false)
  })
| 
{
  "enabled": false
}
| no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_maintenance\_reservation](#input\_enable\_maintenance\_reservation) | Enables slurm reservation for scheduled maintenance. | `bool` | `false` | no | -| [enable\_opportunistic\_maintenance](#input\_enable\_opportunistic\_maintenance) | On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_placement](#input\_enable\_placement) | Use placement policy for VMs in this nodeset.
See: https://cloud.google.com/compute/docs/instances/placement-policies-overview
To set max\_distance of used policy, use `placement_max_distance` variable.

Enabled by default, reasons for users to disable it:
- If non-dense reservation is used, user can avoid extra-cost of creating placement policies;
- If user wants to avoid "all or nothing" VM provisioning behaviour;
- If user wants to intentionally have "spread" VMs (e.g. for reliability reasons) | `bool` | `true` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | -| [future\_reservation](#input\_future\_reservation) | If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT\_ID/zones/ZONE/futureReservations/FUTURE\_RESERVATION\_NAME'.
See https://cloud.google.com/compute/docs/instances/future-reservations-overview | `string` | `""` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_properties](#input\_instance\_properties) | Override the instance properties. Used to test features not supported by Slurm GCP,
recommended for advanced usage only.
See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert
If any sub-field (e.g. scheduling) is set, it will override the values computed by
SlurmGCP and ignoring values of provided vars. | `any` | `null` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for compute nodes. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | -| [maintenance\_interval](#input\_maintenance\_interval) | Sets the maintenance interval for instances in this nodeset.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. | `string` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_conf](#input\_node\_conf) | Map of Slurm node line configuration. | `map(any)` | `{}` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling nodes allowed in this partition. | `number` | `10` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | -| [placement\_max\_distance](#input\_placement\_max\_distance) | Maximum distance between nodes in the placement group. Requires enable\_placement to be true. Values must be supported by the chosen machine type. | `number` | `null` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | -| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]
- RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]

Must be a "SPECIFIC" reservation
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [zone](#input\_zone) | Zone in which to create compute VMs. Additional zones in the same region can be specified in var.zones. | `string` | n/a | yes | -| [zone\_target\_shape](#input\_zone\_target\_shape) | Strategy for distributing VMs across zones in a region.
ANY
GCE picks zones for creating VM instances to fulfill the requested number of VMs
within present resource constraints and to maximize utilization of unused zonal
reservations.
ANY\_SINGLE\_ZONE (default)
GCE always selects a single zone for all the VMs, optimizing for resource quotas,
available reservations and general capacity.
BALANCED
GCE prioritizes acquisition of resources, scheduling VMs in zones where resources
are available while distributing VMs as evenly as possible across allowed zones
to minimize the impact of zonal failure. | `string` | `"ANY_SINGLE_ZONE"` | no | -| [zones](#input\_zones) | Additional zones in which to allow creation of partition nodes. Google Cloud
will find zone based on availability, quota and reservations.
Should not be set if SPECIFIC reservation is used. | `set(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf deleted file mode 100644 index da6aae33ee..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) -} - -module "instance_validation" { - source = "../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata - ) - - name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.disk_resource_manager_tags - } - ] - - public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - ghpc_startup_script = [{ - filename = "ghpc_nodeset_startup.sh" - content = var.startup_script - }] - - termination_action = (var.dws_flex.enabled && !var.dws_flex.use_bulk_insert) ? "DELETE" : try(var.spot_instance_config.termination_action, null) - - nodeset = { - node_count_static = var.node_count_static - node_count_dynamic_max = var.node_count_dynamic_max - node_conf = var.node_conf - nodeset_name = local.name - dws_flex = var.dws_flex - - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(local.labels, var.disk_labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - - bandwidth_tier = var.bandwidth_tier - can_ip_forward = var.can_ip_forward - - enable_confidential_vm = var.enable_confidential_vm - enable_placement = var.enable_placement - placement_max_distance = var.placement_max_distance - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - gpu = one(local.guest_accelerator) - accelerator_topology = var.accelerator_topology - - labels = local.labels - machine_type = terraform_data.machine_type_zone_validation.output - advanced_machine_features = var.advanced_machine_features - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - region = var.region - resource_manager_tags = var.resource_manager_tags - service_account = local.service_account - shielded_instance_config = var.shielded_instance_config - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - subnetwork_self_link = var.subnetwork_self_link - additional_networks = var.additional_networks - access_config = local.access_config - tags = var.tags - spot = var.enable_spot_vm - termination_action = local.termination_action - reservation_name = local.reservation_name - future_reservation = local.future_reservation - maintenance_interval = var.maintenance_interval - instance_properties_json = jsonencode(var.instance_properties) - - zone_target_shape = var.zone_target_shape - zone_policy_allow = local.zones - zone_policy_deny = local.zones_deny - - startup_script = local.ghpc_startup_script - network_storage = var.network_storage - - enable_maintenance_reservation = var.enable_maintenance_reservation - enable_opportunistic_maintenance = var.enable_opportunistic_maintenance - } -} - -locals { - zones = setunion(var.zones, [var.zone]) - zones_deny = setsubtract(data.google_compute_zones.available.names, local.zones) -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region - - lifecycle { - postcondition { - condition = length(setsubtract(local.zones, self.names)) == 0 - error_message = <<-EOD - Invalid zones=${jsonencode(setsubtract(local.zones, self.names))} - Available zones=${jsonencode(self.names)} - EOD - } - } -} - -locals { - res_match = regex("^(?P(?Pprojects/(?P[a-z0-9-]+)/reservations/)?(?P[a-z0-9-]+)(?P/reservationBlocks/[a-z0-9-]+)?)?$", var.reservation_name) - - res_short_name = local.res_match.name - res_project = coalesce(local.res_match.project, var.project_id) - res_prefix = coalesce(local.res_match.prefix, "projects/${local.res_project}/reservations/") - res_suffix = local.res_match.suffix == null ? "" : local.res_match.suffix - - reservation_name = local.res_match.whole == null ? "" : "${local.res_prefix}${local.res_short_name}${local.res_suffix}" -} - -locals { - fr_match = regex("^(?Pprojects/(?P[a-z0-9-]+)/zones/(?P[a-z0-9-]+)/futureReservations/)?(?P[a-z0-9-]+)?$", var.future_reservation) - - fr_name = local.fr_match.name - fr_project = coalesce(local.fr_match.project, var.project_id) - fr_zone = coalesce(local.fr_match.zone, var.zone) - - future_reservation = var.future_reservation == "" ? "" : "projects/${local.fr_project}/zones/${local.fr_zone}/futureReservations/${local.fr_name}" -} - - -# tflint-ignore: terraform_unused_declarations -data "google_compute_reservation" "reservation" { - count = length(local.reservation_name) > 0 ? 1 : 0 - - name = local.res_short_name - project = local.res_project - zone = var.zone - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "Couldn't find the reservation ${var.reservation_name}" - } - - postcondition { - condition = coalesce(self.specific_reservation_required, true) - error_message = < 0] -} - -resource "terraform_data" "machine_type_zone_validation" { - input = var.machine_type - lifecycle { - precondition { - condition = length(local.zones_with_machine_type) > 0 - error_message = <<-EOT - machine type ${var.machine_type} is not available in any of the zones ${jsonencode(local.zones)}". To list zones in which it is available, run: - - gcloud compute machine-types list --filter="name=${var.machine_type}" - EOT - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml deleted file mode 100644 index 95b6d1c730..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name - has_to_be_used: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf deleted file mode 100644 index 18ed74e2d5..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset" { - description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset - - precondition { - condition = !contains([ - "c3-:pd-standard", - "h3-:pd-standard", - "h3-:pd-ssd", - ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") - error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." - } - - precondition { - condition = var.reservation_name == "" || length(var.zones) == 0 - error_message = <<-EOD - If a reservation is specified, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.accelerator_topology == null || var.enable_placement - error_message = "accelerator_topology requires enable_placement to be set to true." - } - - precondition { - condition = (var.accelerator_topology == null) || try(tonumber(split("x", var.accelerator_topology)[1]) % local.guest_accelerator[0].count == 0, false) - error_message = "accelerator_topology must be divisible by number of gpus in machine." - } - - precondition { - condition = var.placement_max_distance == null || var.enable_placement - error_message = "placement_max_distance requires enable_placement to be set to true." - } - - precondition { - condition = !(startswith(var.machine_type, "a3-") && var.placement_max_distance == 1) - error_message = "A3 machines do not support a placement_max_distance of 1." - } - - precondition { - condition = var.reservation_name == "" || !var.dws_flex.enabled - error_message = "Cannot use reservations with DWS Flex." - } - - precondition { - condition = !var.enable_placement || !var.dws_flex.enabled - error_message = "Cannot use DWS Flex with `enable_placement`." - } - - precondition { - condition = length(var.zones) == 0 || !var.dws_flex.enabled - error_message = <<-EOD - If a DWS Flex is enabled, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.on_host_maintenance == "TERMINATE" || !var.dws_flex.enabled - error_message = "If DWS Flex is used, `on_host_maintenance` should be set to 'TERMINATE'" - } - - precondition { - condition = !var.enable_spot_vm || !var.dws_flex.enabled - error_message = "Cannot use both Flex-Start and Spot VMs for provisioning." - } - - precondition { - condition = var.reservation_name == "" || var.future_reservation == "" - error_message = "Cannot use reservations and future reservations in the same nodeset" - } - - precondition { - condition = !var.enable_placement || var.future_reservation == "" - error_message = "Cannot use `enable_placement` with future reservations." - } - - precondition { - condition = var.future_reservation == "" || length(var.zones) == 0 - error_message = <<-EOD - If a future reservation is specified, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.future_reservation == "" || local.fr_zone == var.zone - error_message = <<-EOD - The zone of the deployment must match that of the future reservation - EOD - } - - precondition { - condition = var.node_count_dynamic_max > 0 || var.node_count_static > 0 - error_message = <<-EOD - This nodeset contains zero nodes, there should be at least one static or dynamic node - EOD - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf deleted file mode 100644 index 06ef5aac6f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf +++ /dev/null @@ -1,641 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "node_conf" { - description = "Map of Slurm node line configuration." - type = map(any) - default = {} - validation { - condition = lookup(var.node_conf, "Sockets", null) == null - error_message = <<-EOD - `Sockets` field is in conflict with `SocketsPerBoard` which is automatically generated by SlurmGCP. - Instead, you can override the following fields: `Boards`, `SocketsPerBoard`, `CoresPerSocket`, and `ThreadsPerCore`. - See: https://slurm.schedmd.com/slurm.conf.html#OPT_Boards and https://slurm.schedmd.com/slurm.conf.html#OPT_Sockets_1 - EOD - } -} - -variable "node_count_static" { - description = "Number of nodes to be statically created." - type = number - default = 0 -} - -variable "node_count_dynamic_max" { - description = "Maximum number of auto-scaling nodes allowed in this partition." - type = number - default = 10 -} - -## VM Definition -variable "instance_template" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Instance template can not be specified for compute nodes." - type = string - default = null - validation { - condition = var.instance_template == null - error_message = "DEPRECATED: Instance template can not be specified for compute nodes." - } -} - -variable "machine_type" { - description = "Compute Platform machine type to use for this partition compute nodes." - type = string - default = "c2-standard-60" -} - -variable "metadata" { - type = map(string) - description = "Metadata, provided as a map." - default = {} -} - -variable "instance_image" { - description = <<-EOD - Defines the image that will be used in the Slurm node group VM instances. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - - For more information on creating custom images that comply with Slurm on GCP - see the "Slurm on GCP Custom Images" section in docs/vm-images.md. - EOD - type = map(string) - default = { - family = "slurm-gcp-6-11-hpc-rocky-linux-8" - project = "schedmd-slurm-public" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations - description = <<-EOD - A flag that designates that the user is aware that they are requesting - to use a custom and potentially incompatible image for this Slurm on - GCP module. - - If the field is set to false, only the compatible families and project - names will be accepted. The deployment will fail with any other image - family or name. If set to true, no checks will be done. - - See: https://goo.gle/hpc-slurm-images - EOD - type = bool - default = false -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "tags" { - type = list(string) - description = "Network tag list." - default = [] -} - -variable "disk_type" { - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - type = string - default = "pd-standard" -} - -variable "disk_size_gb" { - description = "Size of boot disk to create for the partition compute nodes." - type = number - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string)) - auto_delete = optional(bool) - boot = optional(bool) - disk_resource_manager_tags = optional(map(string)) - })) - default = [] -} - -variable "enable_confidential_vm" { - type = bool - description = "Enable the Confidential VM configuration. Note: the instance image must support option." - default = false -} - -variable "enable_shielded_vm" { - type = bool - description = "Enable the Shielded VM configuration. Note: the instance image must support option." - default = false -} - -variable "shielded_instance_config" { - type = object({ - enable_integrity_monitoring = bool - enable_secure_boot = bool - enable_vtpm = bool - }) - description = <<-EOD - Shielded VM configuration for the instance. Note: not used unless - enable_shielded_vm is 'true'. - - enable_integrity_monitoring : Compare the most recent boot measurements to the - integrity policy baseline and return a pair of pass/fail results depending on - whether they match or not. - - enable_secure_boot : Verify the digital signature of all boot components, and - halt the boot process if signature verification fails. - - enable_vtpm : Use a virtualized trusted platform module, which is a - specialized computer chip you can use to encrypt objects like keys and - certificates. - EOD - default = { - enable_integrity_monitoring = true - enable_secure_boot = true - enable_vtpm = true - } -} - - -variable "enable_oslogin" { - type = bool - description = <<-EOD - Enables Google Cloud os-login for user login and authentication for VMs. - See https://cloud.google.com/compute/docs/oslogin - EOD - default = true -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example." - type = bool - default = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "labels" { - description = "Labels to add to partition compute instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "min_cpu_platform" { - description = "The name of the minimum CPU platform that you want the instance to use." - type = string - default = null -} - -variable "on_host_maintenance" { - type = string - description = <<-EOD - Instance availability Policy. - - Note: Placement groups are not supported when on_host_maintenance is set to - "MIGRATE" and will be deactivated regardless of the value of - enable_placement. To support enable_placement, ensure on_host_maintenance is - set to "TERMINATE". - EOD - default = "TERMINATE" -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." - } -} - -variable "accelerator_topology" { - type = string - description = "Specifies the shape of the Accelerator (GPU/TPU) slice." - nullable = true - default = null -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - - -variable "service_account_email" { - description = "Service account e-mail address to attach to the compute instances." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the compute instances." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "service_account" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - } -} - -variable "enable_spot_vm" { - description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." - type = bool - default = false -} - -variable "spot_instance_config" { - description = "Configuration for spot VMs." - type = object({ - termination_action = string - }) - default = null -} - -variable "bandwidth_tier" { - description = < 0 - error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME[/reservationBlocks/BLOCK_ID]', [...] are optional parts." - } -} - -variable "future_reservation" { - description = <<-EOD - If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zones/ZONE/futureReservations/FUTURE_RESERVATION_NAME'. - See https://cloud.google.com/compute/docs/instances/future-reservations-overview - EOD - type = string - default = "" - nullable = false - - validation { - condition = length(regexall("^(projects/([a-z0-9-]+)/zones/([a-z0-9-]+)/futureReservations/([a-z0-9-]+))?$", var.future_reservation)) > 0 || length(regexall("^([a-z0-9-]+)$", var.future_reservation)) > 0 - error_message = "Future reservation must be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zone/ZONE/futureReservations/FUTURE_RESERVATION_NAME'." - } -} - -variable "maintenance_interval" { - description = <<-EOD - Sets the maintenance interval for instances in this nodeset. - See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. - EOD - type = string - default = null -} - -variable "startup_script" { - description = "Startup script used by VMs in this nodeset" - type = string - default = "# no-op" -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured on nodes." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - })) - default = [] -} - - -variable "instance_properties" { - description = <<-EOD - Override the instance properties. Used to test features not supported by Slurm GCP, - recommended for advanced usage only. - See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert - If any sub-field (e.g. scheduling) is set, it will override the values computed by - SlurmGCP and ignoring values of provided vars. - EOD - type = any - default = null -} - - -variable "enable_maintenance_reservation" { - type = bool - description = "Enables slurm reservation for scheduled maintenance." - default = false -} - - -variable "enable_opportunistic_maintenance" { - type = bool - description = "On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle." - default = false -} - - -variable "dws_flex" { - description = <<-EOD - If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes. - See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler - Options: - - enable: Enable DWS Flex Start - - max_run_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week). - - use_job_duration: Use the job duration to determine the max_run_duration, if job duration is not set, max_run_duration will be used. - - use_bulk_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances - - Limitations: - - CAN NOT be used with reservations; - - CAN NOT be used with placement groups; - - EOD - - type = object({ - enabled = optional(bool, true) - max_run_duration = optional(number, 604800) # one week - use_job_duration = optional(bool, false) - use_bulk_insert = optional(bool, false) - }) - default = { - enabled = false - } - validation { - condition = var.dws_flex.max_run_duration >= 600 && var.dws_flex.max_run_duration <= 604800 - error_message = "Max duration must be at least than 10 minutes, and cannot be more than one week." - } -} - -variable "placement_max_distance" { - type = number - description = "Maximum distance between nodes in the placement group. Requires enable_placement to be true. Values must be supported by the chosen machine type." - nullable = true - default = null - - validation { - condition = coalesce(var.placement_max_distance, 1) >= 1 && coalesce(var.placement_max_distance, 3) <= 3 - error_message = "Invalid value for placement_max_distance. Valid values are null, 1, 2, or 3." - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf deleted file mode 100644 index e014c318e4..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.4" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 5.11" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.74.0" - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md deleted file mode 100644 index d3dbcd959e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md +++ /dev/null @@ -1,105 +0,0 @@ -## Description - -This module creates a compute partition that can be used as input to the -[schedmd-slurm-gcp-v6-controller](../../scheduler/schedmd-slurm-gcp-v6-controller/README.md). - -The partition module is designed to work alongside the -[schedmd-slurm-gcp-v6-nodeset](../schedmd-slurm-gcp-v6-nodeset/README.md) -module. A partition can be made up of one or -more nodesets, provided either through `use` (preferred) or defined manually -in the `nodeset` variable. - -### Example - -The following code snippet creates a partition module with: - -* 2 nodesets added via `use`. - * The first nodeset is made up of machines of type `c2-standard-30`. - * The second nodeset is made up of machines of type `c2-standard-60`. - * Both nodesets have a maximum count of 200 dynamically created nodes. -* partition name of "compute". -* connected to the `network` module via `use`. -* nodes mounted to homefs via `use`. - -```yaml -- id: nodeset_1 - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - name: c30 - node_count_dynamic_max: 200 - machine_type: c2-standard-30 - -- id: nodeset_2 - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - name: c60 - node_count_dynamic_max: 200 - machine_type: c2-standard-60 - -- id: compute_partition - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - use: - - homefs - - nodeset_1 - - nodeset_2 - settings: - partition_name: compute -``` - -## Support - -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [exclusive](#input\_exclusive) | Exclusive job access to nodes. When set to true nodes execute single job and are deleted
after job exits. If set to false, multiple jobs can be scheduled on one node. | `bool` | `true` | no | -| [is\_default](#input\_is\_default) | Sets this partition as the default partition by updating the partition\_conf.
If "Default" is already set in partition\_conf, this variable will have no effect. | `bool` | `false` | no | -| [network\_storage](#input\_network\_storage) | DEPRECATED | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | A list of nodesets.
For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip     = string
      remote_mount  = string
      local_mount   = string
      fs_type       = string
      mount_options = string
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | -| [partition\_conf](#input\_partition\_conf) | Slurm partition configuration as a map.
See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION | `map(string)` | `{}` | no | -| [partition\_name](#input\_partition\_name) | The name of the slurm partition. | `string` | n/a | yes | -| [resume\_timeout](#input\_resume\_timeout) | Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'ResumeTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. | `number` | `null` | no | -| [suspend\_time](#input\_suspend\_time) | Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram.
This sets 'SuspendTime' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details.
NOTE: use value -1 to exclude partition from suspend.
NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. | `number` | `300` | no | -| [suspend\_timeout](#input\_suspend\_timeout) | Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'SuspendTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. | `number` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Details of a nodesets in this partition | -| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of a dynamic nodesets in this partition | -| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of a TPU nodesets in this partition | -| [partitions](#output\_partitions) | Details of a slurm partition | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf deleted file mode 100644 index 1618c64280..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - use_static = [for ns in concat(var.nodeset, var.nodeset_tpu) : ns.nodeset_name if ns.node_count_static > 0] - - has_node = length(var.nodeset) > 0 - has_dyn = length(var.nodeset_dyn) > 0 - has_tpu = length(var.nodeset_tpu) > 0 - has_flex = length([for ns in var.nodeset : ns.dws_flex.enabled if ns.dws_flex.enabled]) > 0 -} - -locals { - partition_conf = merge({ - "Default" = var.is_default ? "YES" : null - "SuspendTime" = var.suspend_time < 0 ? "INFINITE" : var.suspend_time - "SuspendTimeout" = var.suspend_timeout != null ? var.suspend_timeout : (local.has_tpu ? 240 : 120) - }, var.partition_conf, { "ResumeTimeout" = local.has_flex ? 65535 : try(var.partition_conf["ResumeTimeout"], coalesce(var.resume_timeout, (local.has_tpu ? 600 : 300))) }) - - partition = { - partition_name = var.partition_name - partition_conf = local.partition_conf - - partition_nodeset = [for ns in var.nodeset : ns.nodeset_name] - partition_nodeset_tpu = [for ns in var.nodeset_tpu : ns.nodeset_name] - partition_nodeset_dyn = [for ns in var.nodeset_dyn : ns.nodeset_name] - # Options - enable_job_exclusive = var.exclusive - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml deleted file mode 100644 index 13ea127b3c..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - has_to_be_used: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf deleted file mode 100644 index 35dece64fb..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "partitions" { - description = "Details of a slurm partition" - - value = [local.partition] - - precondition { - condition = (length(local.use_static) == 0) || !var.exclusive - error_message = <<-EOD - Can't use static nodes within partition with `var.exclusive` set to `true`. - NOTE: Partition's `var.exclusive` is set to `true` by default. Set it to `false` explicitly to use static nodes. - EOD - } - - precondition { - # Can not mix TPU with other non-TPU nodesets due to SlurmGCP specific limitations; - # Can not mix dynamic with non-dynamic nodesets due to Slurms inability to - # turn off "power management" at nodeset level (can only do it at partition or node level). - condition = sum([for b in [local.has_node, local.has_dyn, local.has_tpu] : b ? 1 : 0]) == 1 - error_message = "Partition must contain exactly one type of nodeset." - } -} - -output "nodeset" { - description = "Details of a nodesets in this partition" - - value = var.nodeset -} - -output "nodeset_tpu" { - description = "Details of a TPU nodesets in this partition" - - value = var.nodeset_tpu -} - - -output "nodeset_dyn" { - description = "Details of a dynamic nodesets in this partition" - - value = var.nodeset_dyn -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf deleted file mode 100644 index a1c85adb90..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "partition_name" { - description = "The name of the slurm partition." - type = string - - validation { - condition = can(regex("^[a-z](?:[a-z0-9]*)$", var.partition_name)) - error_message = "Variable 'partition_name' must be a match of regex '^[a-z](?:[a-z0-9]*)$'." - } -} - -variable "partition_conf" { - description = <<-EOD - Slurm partition configuration as a map. - See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION - EOD - type = map(string) - default = {} -} - -variable "is_default" { - description = <<-EOD - Sets this partition as the default partition by updating the partition_conf. - If "Default" is already set in partition_conf, this variable will have no effect. - EOD - type = bool - default = false -} - -variable "exclusive" { - description = <<-EOD - Exclusive job access to nodes. When set to true nodes execute single job and are deleted - after job exits. If set to false, multiple jobs can be scheduled on one node. - EOD - type = bool - default = true -} - -variable "nodeset" { - description = <<-EOD - A list of nodesets. - For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset - EOD - type = list(object({ - node_count_static = optional(number, 0) - node_count_dynamic_max = optional(number, 1) - node_conf = optional(map(string), {}) - nodeset_name = string - additional_disks = optional(list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string), {}) - auto_delete = optional(bool, true) - boot = optional(bool, false) - disk_resource_manager_tags = optional(map(string), {}) - })), []) - bandwidth_tier = optional(string, "platform_default") - can_ip_forward = optional(bool, false) - disk_auto_delete = optional(bool, true) - disk_labels = optional(map(string), {}) - disk_resource_manager_tags = optional(map(string), {}) - disk_size_gb = optional(number) - disk_type = optional(string) - enable_confidential_vm = optional(bool, false) - enable_placement = optional(bool, false) - placement_max_distance = optional(number, null) - enable_oslogin = optional(bool, true) - enable_shielded_vm = optional(bool, false) - enable_maintenance_reservation = optional(bool, false) - enable_opportunistic_maintenance = optional(bool, false) - gpu = optional(object({ - count = number - type = string - })) - accelerator_topology = optional(string, null) - dws_flex = object({ - enabled = bool - max_run_duration = number - use_job_duration = bool - use_bulk_insert = bool - }) - labels = optional(map(string), {}) - machine_type = optional(string) - advanced_machine_features = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - maintenance_interval = optional(string) - instance_properties_json = string - metadata = optional(map(string), {}) - min_cpu_platform = optional(string) - network_tier = optional(string, "STANDARD") - network_storage = optional(list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = optional(map(string)) - mount_runner = optional(map(string)) - })), []) - on_host_maintenance = optional(string) - preemptible = optional(bool, false) - region = optional(string) - resource_manager_tags = optional(map(string), {}) - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - shielded_instance_config = optional(object({ - enable_integrity_monitoring = optional(bool, true) - enable_secure_boot = optional(bool, true) - enable_vtpm = optional(bool, true) - })) - source_image_family = optional(string) - source_image_project = optional(string) - source_image = optional(string) - subnetwork_self_link = string - additional_networks = optional(list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - }))) - access_config = optional(list(object({ - nat_ip = string - network_tier = string - }))) - spot = optional(bool, false) - tags = optional(list(string), []) - termination_action = optional(string) - reservation_name = optional(string) - future_reservation = string - startup_script = optional(list(object({ - filename = string - content = string })), []) - - zone_target_shape = string - zone_policy_allow = set(string) - zone_policy_deny = set(string) - })) - default = [] - - validation { - condition = length(distinct(var.nodeset[*].nodeset_name)) == length(var.nodeset) - error_message = "All nodesets must have a unique name." - } -} - -variable "nodeset_tpu" { - description = "Define TPU nodesets, as a list." - type = list(object({ - node_count_static = optional(number, 0) - node_count_dynamic_max = optional(number, 5) - nodeset_name = string - enable_public_ip = optional(bool, false) - node_type = string - accelerator_config = optional(object({ - topology = string - version = string - }), { - topology = "" - version = "" - }) - tf_version = string - preemptible = optional(bool, false) - preserve_tpu = optional(bool, false) - zone = string - data_disks = optional(list(string), []) - docker_image = optional(string, "") - network_storage = optional(list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - })), []) - subnetwork = string - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - project_id = string - reserved = optional(string, false) - })) - default = [] - - validation { - condition = length(distinct([for x in var.nodeset_tpu : x.nodeset_name])) == length(var.nodeset_tpu) - error_message = "All TPU nodesets must have a unique name." - } -} - -variable "nodeset_dyn" { - description = "Defines dynamic nodesets, as a list." - type = list(object({ - nodeset_name = string - nodeset_feature = string - })) - default = [] - - validation { - condition = length(distinct([for x in var.nodeset_dyn : x.nodeset_name])) == length(var.nodeset_dyn) - error_message = "All dynamic nodesets must have a unique name." - } -} - -variable "resume_timeout" { - description = <<-EOD - Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use. - If null is given, then a smart default will be chosen depending on nodesets in partition. - This sets 'ResumeTimeout' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. - EOD - type = number - default = null - - validation { - condition = var.resume_timeout == null ? true : var.resume_timeout > 0 && var.resume_timeout < 65536 - error_message = "Value must be > 0 and < 65536" - } -} - -variable "suspend_time" { - description = <<-EOD - Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram. - This sets 'SuspendTime' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details. - NOTE: use value -1 to exclude partition from suspend. - NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. - EOD - type = number - default = 300 - - validation { - condition = var.suspend_time >= -1 - error_message = "Value must be >= -1." - } -} - -variable "suspend_timeout" { - description = <<-EOD - Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown. - If null is given, then a smart default will be chosen depending on nodesets in partition. - This sets 'SuspendTimeout' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. - EOD - type = number - default = null - - validation { - condition = var.suspend_timeout == null ? true : var.suspend_timeout > 0 - error_message = "Value must be > 0." - } -} - - -# tflint-ignore: terraform_unused_declarations -variable "network_storage" { - description = "DEPRECATED" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] - validation { - condition = length(var.network_storage) == 0 - error_message = <<-EOD - network_storage in partition module is deprecated and should not be set. - To add network storage to compute nodes, use network_storage of nodeset module instead. - EOD - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf deleted file mode 100644 index d388f4bfdd..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.3" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.74.0" - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/README.md b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/README.md deleted file mode 100644 index 994f1500ba..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/README.md +++ /dev/null @@ -1,157 +0,0 @@ -## Description - -This module provides ways to create and manage Google Cloud Artifact Registry repositories. - -Currently this module is built to support repositories in Docker format although there are placeholder variables for other types which may work too. Remote repositories with pull-through cache functionality integrated with Google Secret Manager is currently supported. The aim of this module is to eventually offer feature parity with this [Terraform module](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository#nested_remote_repository_config), allowing creation of repositories in various formats, including Docker, Maven, NPM, Python, APT, YUM, and COMMON. - -This module is best suited for managing artifact repositories in HPC/AI containerized environments where artifacts need to be shared across distributed systems. It includes IAM role configurations and secret access handling for seamless integration with CI/CD pipelines and other services too. - -It is designed to help facilitate containerized workloads running in the Cluster Toolkit with SLURM leveraging [Enroot](https://github.com/NVIDIA/enroot) and [Pyxis](https://github.com/NVIDIA/pyxis). Docker repositories can store container images that are used in job submissions, enabling efficient and scalable execution of containerized HPC or AI based workloads. - -## Usage - -### Service Account / APIs - -You will need to enable the relevant APIs and create a Service Account for your cluster with the following Artifact Registry permissions. - -```yaml - - id: services-api - source: community/modules/project/service-enablement - settings: - gcp_service_list: - - secretmanager.googleapis.com - - cloudbuild.googleapis.com - - artifactregistry.googleapis.com - - - source: community/modules/project/service-account - kind: terraform - id: hpc_service_account - settings: - project_id: project_name - name: service_account_name - project_roles: - - artifactregistry.reader - - artifactregistry.writer - - secretmanager.secretAccessor -``` - -### Deployment - -Create a standard Docker repository. - -```yaml -- id: registry - source: community/modules/container/artifact-registry - settings: - repo_mode: STANDARD_REPOSITORY - format: DOCKER -``` - -Mirror of public Docker Hub repository. - -```yaml -- id: dockerhub_registry - source: community/modules/container/artifact-registry - settings: - repo_mode: REMOTE_REPOSITORY - format: DOCKER - repo_public_repository: DOCKER_HUB -``` - -Mirror of NVIDIA's [NGC Catalog](https://catalog.ngc.nvidia.com/containers). [API key](https://org.ngc.nvidia.com/setup/api-key) used in blueprint is stored in Secret Manager. - -```yaml -- id: ngc_registry - source: community/modules/container/artifact-registry - settings: - repo_mode: REMOTE_REPOSITORY - format: DOCKER - repo_mirror_url: "https://nvcr.io" - repo_username: $oauthtoken - repo_password: api_key_here - use_upstream_credentials: True -``` - -### Container Operations - -Retrieve `$REPOSITORY_NAME` from [Artifact Registry](https://console.cloud.google.com/artifacts) or by using `gcloud`. - -```yaml -gcloud artifacts repositories list --project="${PROJECT_ID}" -``` - -Pulling containers from your mirrored internal Artifact Repositories. - -Pull [Ubuntu](https://hub.docker.com/_/ubuntu) from Docker Hub mirror. - -```yaml -docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/library/ubuntu:latest -``` - -Pull [Pytorch](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch) from NGC Catalog mirror. - -```yaml -docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/nvidia/pytorch:24.11-py3 -``` - -Alternatively, proceed with running SLURM's [NVIDIA/pyxis](https://github.com/NVIDIA/pyxis) plugin, which will now be able to pull and use these containers directly from the mirrored repositories. - -Note: only Docker registries have been tested so far. Placeholders do exist for other registry types which may or may not work. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_artifact_registry_repository.artifact_registry](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository) | resource | -| [google_secret_manager_secret.repo_password_secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_version.repo_password_secret_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [random_password.repo_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | -| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment. | `string` | n/a | yes | -| [format](#input\_format) | Artifact Registry format (e.g., DOCKER). | `string` | `"DOCKER"` | no | -| [labels](#input\_labels) | Labels to add to the artifact registry. Key-value pairs. | `map(string)` | `{}` | no | -| [project\_id](#input\_project\_id) | Project ID where the artifact registry and secret are created. | `string` | n/a | yes | -| [region](#input\_region) | Region for the artifact registry. | `string` | n/a | yes | -| [repo\_mirror\_url](#input\_repo\_mirror\_url) | For REMOTE\_REPOSITORY, URL for a custom or common mirror. | `string` | `null` | no | -| [repo\_mode](#input\_repo\_mode) | Artifact Registry mode (STANDARD\_REPOSITORY, REMOTE\_REPOSITORY, etc.). | `string` | `"STANDARD_REPOSITORY"` | no | -| [repo\_password](#input\_repo\_password) | Optional password/API key. If null, one will be randomly generated. | `string` | `null` | no | -| [repo\_public\_repository](#input\_repo\_public\_repository) | For REMOTE\_REPOSITORY, name of a known public repo as per the Terraform module
(e.g., DOCKER\_HUB) or null for custom repo. | `string` | `null` | no | -| [repo\_username](#input\_repo\_username) | Username for external repository. | `string` | `null` | no | -| [repository\_base](#input\_repository\_base) | For APT/YUM public repos, repository\_base (e.g., 'DEBIAN', 'UBUNTU'). | `string` | `null` | no | -| [repository\_path](#input\_repository\_path) | For APT/YUM public repos, repository\_path (e.g., 'debian/dists/buster'). | `string` | `null` | no | -| [use\_upstream\_credentials](#input\_use\_upstream\_credentials) | Configure Service Account to use upstream credentials for REMOTE\_REPOSITORY:
If true, a username/password is used for the REMOTE\_REPOSITORY mirror.
If false (or if repo\_password == null), no password is created at all.
Note: Blueprint credentials will be stored in Secrets Manager. | `bool` | `false` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | (Optional) A list of objects to enable user-managed replication.
Each object can have:
location = string
kms\_key\_name = optional(string)
If empty, auto replication is used. | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [registry\_url](#output\_registry\_url) | The URL of the created artifact registry. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/main.tf b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/main.tf deleted file mode 100644 index c3406af607..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/main.tf +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "artifact-registry", ghpc_role = "container" }) -} - -locals { - # Auto (i.e., empty) vs user-managed replication - auto = length(var.user_managed_replication) == 0 ? true : false - - # For remote custom repositories, parse out host to create a base_component name - mirror_url_no_proto = var.repo_mirror_url != null ? replace(replace(var.repo_mirror_url, "https://", ""), "http://", "") : "" - mirror_host = local.mirror_url_no_proto != "" ? split("/", local.mirror_url_no_proto)[0] : "" - - base_component = replace( - replace( - replace( - lower( - local.mirror_host != "" - ? "${var.format}-${var.repo_mode}-${local.mirror_host}" - : "${var.format}-${var.repo_mode}-nohost" - ), - "\\.", "-" - ), - "/", "-" - ), - "_", "-" - ) - - repository_suffix = random_id.resource_name_suffix.hex - - # The final name for the artifact registry repository - repository_name = replace( - replace( - lower( - format("%s-%s", local.base_component, local.repository_suffix) - ), - ".", "-" - ), - "/", "-" - ) - - # The secret name is derived from the repository name - # with a suffix like "-secret". - derived_secret_name = format("%s-secret", local.repository_name) -} - -############################## -# PASSWORD / SECRET -############################## - -# Only create a random password if user didn't supply one -resource "random_password" "repo_password" { - count = var.use_upstream_credentials && var.repo_password == null ? 1 : 0 - length = 24 - special = true - override_special = "_-#=." -} - -resource "google_secret_manager_secret" "repo_password_secret" { - count = var.use_upstream_credentials ? 1 : 0 - project = var.project_id - - # Derive the secret ID from the repository name - secret_id = local.derived_secret_name - - labels = local.labels - - replication { - dynamic "auto" { - for_each = local.auto ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = local.auto ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = replicas.value.kms_key_name != null ? [1] : [] - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_version" "repo_password_secret_version" { - count = var.use_upstream_credentials ? 1 : 0 - secret = google_secret_manager_secret.repo_password_secret[0].id - - # If user provided a password, use it. Otherwise use the random password. - secret_data = var.repo_password != null ? var.repo_password : random_password.repo_password[0].result -} - -############################## -# IAM BINDINGS -############################## - -############################## -# ARTIFACT REGISTRY -############################## - -resource "random_id" "resource_name_suffix" { - byte_length = 2 -} - -resource "google_artifact_registry_repository" "artifact_registry" { - project = var.project_id - location = var.region - format = var.format - mode = var.repo_mode - description = var.deployment_name - labels = local.labels - repository_id = local.repository_name - - # Only create remote_repository_config if REMOTE_REPOSITORY - dynamic "remote_repository_config" { - for_each = var.repo_mode == "REMOTE_REPOSITORY" ? [1] : [] - content { - description = "Pull-through cache" - - dynamic "docker_repository" { - for_each = var.format == "DOCKER" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "docker_repository" { - for_each = var.format == "DOCKER" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "maven_repository" { - for_each = var.format == "MAVEN" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "maven_repository" { - for_each = var.format == "MAVEN" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "npm_repository" { - for_each = var.format == "NPM" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "npm_repository" { - for_each = var.format == "NPM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "python_repository" { - for_each = var.format == "PYTHON" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "python_repository" { - for_each = var.format == "PYTHON" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "apt_repository" { - for_each = var.format == "APT" && var.repo_public_repository != null ? [1] : [] - content { - public_repository { - repository_base = var.repository_base - repository_path = var.repository_path - } - } - } - - dynamic "apt_repository" { - for_each = var.format == "APT" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "yum_repository" { - for_each = var.format == "YUM" && var.repo_public_repository != null ? [1] : [] - content { - public_repository { - repository_base = var.repository_base - repository_path = var.repository_path - } - } - } - - dynamic "yum_repository" { - for_each = var.format == "YUM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "common_repository" { - for_each = var.format == "COMMON" ? [1] : [] - content { - uri = var.repo_mirror_url - } - } - - # Only enable upstream credentials if user wants it - dynamic "upstream_credentials" { - for_each = var.use_upstream_credentials ? [1] : [] - content { - username_password_credentials { - username = var.repo_username - password_secret_version = google_secret_manager_secret_version.repo_password_secret_version[0].name - } - } - } - } - } - - depends_on = [ - google_secret_manager_secret.repo_password_secret, - google_secret_manager_secret_version.repo_password_secret_version, - ] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/metadata.yaml deleted file mode 100644 index 6b68c98a54..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - secretmanager.googleapis.com - - artifactregistry.googleapis.com - - cloudbuild.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/outputs.tf deleted file mode 100644 index 92b6dbb165..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "registry_url" { - description = "The URL of the created artifact registry." - value = "${var.region}-docker.pkg.dev/${var.project_id}/${var.deployment_name}" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/validation.tf b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/validation.tf deleted file mode 100644 index a795060fb7..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/validation.tf +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "terraform_data" "input_validation" { - lifecycle { - precondition { - condition = ( - var.repo_password == null || - (var.use_upstream_credentials && var.repo_mode == "REMOTE_REPOSITORY") - ) - error_message = "repo_password may be set only when repo_mode=REMOTE_REPOSITORY and use_upstream_credentials=true." - } - - precondition { - condition = ( - !var.use_upstream_credentials || - var.repo_mode == "REMOTE_REPOSITORY" - ) - error_message = "use_upstream_credentials is allowed only when repo_mode is REMOTE_REPOSITORY." - } - - precondition { - condition = ( - var.repo_mode != "REMOTE_REPOSITORY" || - (var.repo_public_repository != null || var.repo_mirror_url != null) - ) - error_message = "For a REMOTE_REPOSITORY you must set repo_public_repository or repo_mirror_url." - } - - precondition { - condition = ( - !contains(["APT", "YUM"], var.format) || - (var.repository_base != null && var.repository_path != null) - ) - error_message = "APT/YUM formats require repository_base and repository_path." - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/variables.tf deleted file mode 100644 index 9a4eecb921..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/variables.tf +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "Project ID where the artifact registry and secret are created." - type = string -} - -variable "region" { - description = "Region for the artifact registry." - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment." - type = string -} - -variable "labels" { - description = "Labels to add to the artifact registry. Key-value pairs." - type = map(string) - default = {} -} - -variable "repo_password" { - description = "Optional password/API key. If null, one will be randomly generated." - type = string - default = null -} - -variable "user_managed_replication" { - description = <<-DOC - (Optional) A list of objects to enable user-managed replication. - Each object can have: - location = string - kms_key_name = optional(string) - If empty, auto replication is used. - DOC - type = list(object({ - location = string - kms_key_name = optional(string) - })) - default = [] -} - -variable "format" { - description = "Artifact Registry format (e.g., DOCKER)." - type = string - default = "DOCKER" -} - -variable "repo_mode" { - description = "Artifact Registry mode (STANDARD_REPOSITORY, REMOTE_REPOSITORY, etc.)." - type = string - default = "STANDARD_REPOSITORY" - - validation { - condition = can(regex("^(STANDARD_REPOSITORY|REMOTE_REPOSITORY|VIRTUAL_REPOSITORY)$", var.repo_mode)) - error_message = "repo_mode must be one of STANDARD_REPOSITORY, REMOTE_REPOSITORY, or VIRTUAL_REPOSITORY." - } -} - -variable "repo_public_repository" { - description = <<-DOC - For REMOTE_REPOSITORY, name of a known public repo as per the Terraform module - (e.g., DOCKER_HUB) or null for custom repo. - DOC - type = string - default = null - - # To Do: implement validation - # validation { - # condition = ((var.repo_mode != "REMOTE_REPOSITORY" && var.repo_public_repository == null) || (var.repo_mode == "REMOTE_REPOSITORY" && (var.repo_public_repository != null || var.repo_mirror_url != null))) - # error_message = "If repo_mode is REMOTE_REPOSITORY, you must set either repo_public_repository or repo_mirror_url. Otherwise, leave them null." - # } -} - -variable "repo_mirror_url" { - description = "For REMOTE_REPOSITORY, URL for a custom or common mirror." - type = string - default = null -} - -variable "use_upstream_credentials" { - description = <<-DOC - Configure Service Account to use upstream credentials for REMOTE_REPOSITORY: - If true, a username/password is used for the REMOTE_REPOSITORY mirror. - If false (or if repo_password == null), no password is created at all. - Note: Blueprint credentials will be stored in Secrets Manager. - DOC - type = bool - default = false -} - -variable "repo_username" { - description = "Username for external repository." - type = string - default = null -} - -variable "repository_base" { - description = "For APT/YUM public repos, repository_base (e.g., 'DEBIAN', 'UBUNTU')." - type = string - default = null -} - -variable "repository_path" { - description = "For APT/YUM public repos, repository_path (e.g., 'debian/dists/buster')." - type = string - default = null -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/versions.tf deleted file mode 100644 index 392a7131d2..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/container/artifact-registry/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/README.md b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/README.md deleted file mode 100644 index 23bf87398a..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/README.md +++ /dev/null @@ -1,76 +0,0 @@ -## Description - -Creates a BigQuery dataset. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage -This is a simple usage. - -```yaml - - id: bq-dataset - source: community/modules/database/bigquery-dataset - settings: - dataset_id: my_dataset -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_dataset.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | The name of the dataset to be created | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the dataset. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [dataset\_id](#output\_dataset\_id) | Name of the dataset that was created. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/main.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/main.tf deleted file mode 100644 index 1a9c4bba60..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/main.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-dataset", ghpc_role = "database" }) -} -locals { - dataset_id = var.dataset_id != null ? var.dataset_id : replace("${var.deployment_name}_dataset_${random_id.resource_name_suffix.hex}", "-", "_") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_bigquery_dataset" "pbsb" { - dataset_id = local.dataset_id - project = var.project_id - labels = local.labels -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml deleted file mode 100644 index 87ff9357e4..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigquery.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf deleted file mode 100644 index 9cd8e5df31..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "dataset_id" { - description = "Name of the dataset that was created." - value = google_bigquery_dataset.pbsb.dataset_id -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/variables.tf deleted file mode 100644 index 90c229af6b..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/variables.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "dataset_id" { - description = "The name of the dataset to be created" - type = string - default = null -} - -variable "labels" { - description = "Labels to add to the dataset. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/versions.tf deleted file mode 100644 index 12ddbe842d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-dataset/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/README.md b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/README.md deleted file mode 100644 index ef67cfef01..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/README.md +++ /dev/null @@ -1,87 +0,0 @@ -## Description - -Creates a BigQuery table with a specified schema. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage - -```yaml -id: bq-table - source: community/modules/database/bigquery-table - use: [bq-dataset] - settings: - table_schema: - ' - [ - { - "name": "id", "type": "STRING" - } - ] - ' -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_table.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Dataset name to be used to create the new BQ Table | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the tables. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [table\_id](#input\_table\_id) | Table name to be used to create the new BQ Table | `string` | `null` | no | -| [table\_schema](#input\_table\_schema) | Schema used to create the new BQ Table | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [dataset\_id](#output\_dataset\_id) | ID of BQ dataset | -| [table\_id](#output\_table\_id) | ID of created BQ table | -| [table\_name](#output\_table\_name) | Name of created BQ table | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/main.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/main.tf deleted file mode 100644 index 73f3923e00..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-table", ghpc_role = "database" }) -} - -locals { - table_id = var.table_id != null ? var.table_id : replace("${var.deployment_name}_table_${random_id.resource_name_suffix.hex}", "-", "_") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_bigquery_table" "pbsb" { - deletion_protection = false - project = var.project_id - table_id = local.table_id - dataset_id = var.dataset_id - schema = var.table_schema - labels = local.labels -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/metadata.yaml deleted file mode 100644 index 87ff9357e4..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigquery.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/outputs.tf deleted file mode 100644 index 4220ec1390..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/outputs.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "table_name" { - description = "Name of created BQ table" - value = google_bigquery_table.pbsb.friendly_name -} -output "table_id" { - description = "ID of created BQ table" - value = google_bigquery_table.pbsb.table_id -} -output "dataset_id" { - description = "ID of BQ dataset" - value = google_bigquery_table.pbsb.dataset_id -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/variables.tf deleted file mode 100644 index ec474b4e64..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/variables.tf +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "labels" { - description = "Labels to add to the tables. Key-value pairs." - type = map(string) -} - -variable "table_id" { - description = "Table name to be used to create the new BQ Table" - type = string - default = null -} - -variable "dataset_id" { - description = "Dataset name to be used to create the new BQ Table" - type = string -} - -variable "table_schema" { - description = "Schema used to create the new BQ Table" - type = string -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/versions.tf deleted file mode 100644 index 12ddbe842d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/bigquery-table/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md deleted file mode 100644 index 08364c175b..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md +++ /dev/null @@ -1,107 +0,0 @@ -## Description - -terraform-google-sql makes it easy to create a Google CloudSQL instance and -implement high availability settings. This module is meant for use with -Terraform 0.13+ and tested using Terraform 1.0+. - -The cloudsql created here is used to integrate with the slurm cluster to enable -accounting data storage. - -### Example - -```yaml -- id: cloudsql - source: community/modules/database/slurm-cloudsql-federation - use: [network] - settings: - sql_instance_name: slurm-sql6-demo - tier: "db-f1-micro" -``` - -This creates a cloud sql instance, including a database, user that would allow -the slurm cluster to use as an external DB. In addition, it will allow BigQuery -to run federated query through it. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | -| [google](#requirement\_google) | >= 3.83 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_connection.connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_connection) | resource | -| [google_compute_address.psc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_forwarding_rule.psc_consumer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_forwarding_rule) | resource | -| [google_sql_database.database](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database) | resource | -| [google_sql_database_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance) | resource | -| [google_sql_user.users](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [random_password.password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [authorized\_networks](#input\_authorized\_networks) | IP address ranges as authorized networks of the Cloud SQL for MySQL instances | `list(string)` | `[]` | no | -| [data\_cache\_enabled](#input\_data\_cache\_enabled) | Whether data cache is enabled for the instance. Can be used with ENTERPRISE\_PLUS edition. | `bool` | `false` | no | -| [database\_flags](#input\_database\_flags) | Database flags to set on instance. | `map(string)` | `{}` | no | -| [database\_version](#input\_database\_version) | The version of the database to be created. | `string` | `"MYSQL_8_0"` | no | -| [deletion\_protection](#input\_deletion\_protection) | Whether or not to allow Terraform to destroy the instance. | `string` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [disk\_autoresize](#input\_disk\_autoresize) | Set to false to disable automatic disk grow. | `bool` | `true` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of the database disk in GiB. | `number` | `null` | no | -| [edition](#input\_edition) | value | `string` | `"ENTERPRISE"` | no | -| [enable\_backups](#input\_enable\_backups) | Set true to enable backups | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is going to be created in.:
`projects//global/networks/`" | `string` | n/a | yes | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [query\_insights](#input\_query\_insights) | Query insights configuration. | 
object({
    enabled                 = optional(bool, false)
    query_plans_per_minute  = optional(number)
    query_string_length     = optional(number)
    record_application_tags = optional(bool)
    record_client_address   = optional(bool)
  })
| `{}` | no | -| [region](#input\_region) | The region where SQL instance will be configured | `string` | n/a | yes | -| [sql\_instance\_name](#input\_sql\_instance\_name) | name given to the sql instance for ease of identificaion | `string` | n/a | yes | -| [sql\_password](#input\_sql\_password) | Password for the SQL database. | `any` | `null` | no | -| [sql\_username](#input\_sql\_username) | Username for the SQL database | `string` | `"slurm"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self link of the network where Cloud SQL instance PSC endpoint will be created | `string` | `null` | no | -| [tier](#input\_tier) | The machine type to use for the SQL instance | `string` | n/a | yes | -| [use\_psc\_connection](#input\_use\_psc\_connection) | Create Private Service Connection instead of using Private Service Access peering | `bool` | `false` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cloudsql](#output\_cloudsql) | Describes the cloudsql instance. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf deleted file mode 100644 index 9b518a1b5f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "slurm-cloudsql-federation", ghpc_role = "database" }) -} - -locals { - user_managed_replication = var.user_managed_replication -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "random_password" "password" { - length = 12 - special = false -} - -locals { - sql_instance_name = var.sql_instance_name == null ? "${var.deployment_name}-sql-${random_id.resource_name_suffix.hex}" : var.sql_instance_name - sql_password = var.sql_password == null ? random_password.password.result : var.sql_password -} - - -resource "google_sql_database_instance" "instance" { - project = var.project_id - depends_on = [var.private_vpc_connection_peering] - name = local.sql_instance_name - region = var.region - deletion_protection = var.deletion_protection - database_version = var.database_version - - settings { - disk_size = var.disk_size_gb - disk_autoresize = var.disk_autoresize - edition = var.edition - tier = var.tier - user_labels = local.labels - - dynamic "data_cache_config" { - for_each = var.edition == "ENTERPRISE_PLUS" ? [""] : [] - content { - data_cache_enabled = var.data_cache_enabled - } - } - - dynamic "database_flags" { - for_each = var.database_flags - content { - name = database_flags.key - value = database_flags.value - } - } - - insights_config { - query_insights_enabled = var.query_insights.enabled - query_plans_per_minute = var.query_insights.query_plans_per_minute - query_string_length = var.query_insights.query_string_length - record_application_tags = var.query_insights.record_application_tags - record_client_address = var.query_insights.record_client_address - } - - ip_configuration { - ipv4_enabled = false - private_network = var.use_psc_connection ? null : var.network_id - enable_private_path_for_google_cloud_services = true - - dynamic "authorized_networks" { - for_each = var.use_psc_connection ? [] : var.authorized_networks - iterator = ip_range - - content { - value = ip_range.value - } - } - dynamic "psc_config" { - for_each = var.use_psc_connection ? [""] : [] - content { - psc_enabled = true - allowed_consumer_projects = [var.project_id] - } - } - } - - backup_configuration { - enabled = var.enable_backups - # to allow easy switching between ENTERPRISE and ENTERPRISE_PLUS - transaction_log_retention_days = 7 - } - } - lifecycle { - precondition { - condition = var.disk_autoresize && var.disk_size_gb == null || !var.disk_autoresize - error_message = "If setting disk_size_gb set disk_autorize to false to prevent re-provisioning of the instance after disk auto-expansion." - } - } -} - - - -resource "google_compute_address" "psc" { - count = var.use_psc_connection ? 1 : 0 - project = var.project_id - name = local.sql_instance_name - address_type = "INTERNAL" - region = var.region - subnetwork = var.subnetwork_self_link - labels = local.labels -} - -resource "google_compute_forwarding_rule" "psc_consumer" { - count = var.use_psc_connection ? 1 : 0 - name = local.sql_instance_name - project = var.project_id - region = var.region - subnetwork = var.subnetwork_self_link - ip_address = google_compute_address.psc[0].self_link - load_balancing_scheme = "" - recreate_closed_psc = true - target = google_sql_database_instance.instance.psc_service_attachment_link -} - -resource "google_sql_database" "database" { - project = var.project_id - name = "slurm_accounting" - instance = google_sql_database_instance.instance.name -} - -resource "google_sql_user" "users" { - project = var.project_id - name = var.sql_username - instance = google_sql_database_instance.instance.name - password = local.sql_password -} - -resource "google_bigquery_connection" "connection" { - provider = google - project = var.project_id - location = var.region - cloud_sql { - instance_id = google_sql_database_instance.instance.connection_name - database = google_sql_database.database.name - type = "MYSQL" - credential { - username = google_sql_user.users.name - password = google_sql_user.users.password - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml deleted file mode 100644 index fc0cae0859..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigqueryconnection.googleapis.com - - sqladmin.googleapis.com - - servicenetworking.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf deleted file mode 100644 index 0d05221cd8..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "cloudsql" { - description = "Describes the cloudsql instance." - sensitive = true - value = { - server_ip = var.use_psc_connection ? google_compute_address.psc[0].address : google_sql_database_instance.instance.ip_address[0].ip_address - user = google_sql_user.users.name - password = google_sql_user.users.password - db_name = google_sql_database.database.name - user_managed_replication = local.user_managed_replication - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf deleted file mode 100644 index a2f150419e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "authorized_networks" { - description = "IP address ranges as authorized networks of the Cloud SQL for MySQL instances" - type = list(string) - default = [] - nullable = false -} - -variable "database_version" { - description = "The version of the database to be created." - type = string - default = "MYSQL_8_0" - validation { - condition = contains(["MYSQL_5_7", "MYSQL_8_0", "MYSQL_8_4"], var.database_version) - error_message = "The database version must be either MYSQL_5_7, MYSQL_8_0 or MYSQL_8_4." - } -} - -variable "data_cache_enabled" { - description = "Whether data cache is enabled for the instance. Can be used with ENTERPRISE_PLUS edition." - type = bool - default = false -} - -variable "database_flags" { - description = "Database flags to set on instance." - type = map(string) - default = {} - nullable = false -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "disk_autoresize" { - description = "Set to false to disable automatic disk grow." - type = bool - default = true -} - -variable "disk_size_gb" { - description = "Size of the database disk in GiB." - type = number - default = null -} - -variable "edition" { - description = "value" - type = string - validation { - condition = contains(["ENTERPRISE", "ENTERPRISE_PLUS"], var.edition) - error_message = "The database edition must be either ENTERPRISE or ENTERPRISE_PLUS" - } - default = "ENTERPRISE" -} - -variable "enable_backups" { - description = "Set true to enable backups" - type = bool - default = false -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "query_insights" { - description = "Query insights configuration." - nullable = false - default = {} - type = object({ - enabled = optional(bool, false) - query_plans_per_minute = optional(number) - query_string_length = optional(number) - record_application_tags = optional(bool) - record_client_address = optional(bool) - }) -} - -variable "region" { - description = "The region where SQL instance will be configured" - type = string -} - -variable "tier" { - description = "The machine type to use for the SQL instance" - type = string -} - -variable "sql_instance_name" { - description = "name given to the sql instance for ease of identificaion" - type = string -} - -variable "deletion_protection" { - description = "Whether or not to allow Terraform to destroy the instance." - type = string - default = false -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} - -variable "sql_username" { - description = "Username for the SQL database" - type = string - default = "slurm" -} - -variable "sql_password" { - description = "Password for the SQL database." - type = any - default = null -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is going to be created in.: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "private_vpc_connection_peering" { - description = "The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation." - type = string - default = null -} - -variable "subnetwork_self_link" { - description = "Self link of the network where Cloud SQL instance PSC endpoint will be created" - type = string - default = null -} - -variable "user_managed_replication" { - type = list(object({ - location = string - kms_key_name = optional(string) - })) - description = "Replication parameters that will be used for defined secrets" - default = [] -} - -variable "use_psc_connection" { - description = "Create Private Service Connection instead of using Private Service Access peering" - type = bool - default = false -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf deleted file mode 100644 index 7e672858b6..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" - } - - required_version = ">= 0.13.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md deleted file mode 100644 index d39a58afe1..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md +++ /dev/null @@ -1,158 +0,0 @@ -> [!WARNING] -> This module is deprecated and will be removed on July 1, 2025. The -> recommended replacement is the -> [GCP Managed Lustre module](../../../../modules/file-system/managed-lustre/README.md) - -## Description -This module creates a DDN EXAScaler Cloud Lustre file system using code based on DDN's -[exascaler-cloud-terraform](https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp) (`scripts/2.2.2` is last release with GCP-specific module). - -More information about the architecture can be found at -[Overview of Lustre and EXAScaler Cloud][architecture]. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -> **Warning**: This file system has a license cost as described in the pricing -> section of the [DDN EXAScaler Cloud Marketplace Solution][marketplace]. -> -> **Note**: By default security.public_key is set to `null`, therefore the -> admin user is not created. To ensure the admin user is created, provide a -> public key via the security setting. -> -> **Note**: This module's instances require access to Google APIs and -> therefore, instances must have public IP address or it must be used in a -> subnetwork where [Private Google Access][private-google-access] is enabled. - -[private-google-access]: https://cloud.google.com/vpc/docs/configure-private-google-access -[marketplace]: https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud -[architecture]: https://cloud.google.com/architecture/parallel-file-systems-for-hpc#overview_of_lustre_and_exascaler_cloud - -## Mounting - -To mount the DDN EXAScaler Lustre file system you must first install the DDN -Lustre client and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -the [hpc-enterprise-slurm.yaml](../../../../examples/hpc-enterprise-slurm.yaml) for an -example of using this module with Slurm. - -If mounting is not automatically handled as described above, the DDN-EXAScaler -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - # This file system has an associated license cost. - # https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud - - id: lustrefs - source: community/modules/file-system/DDN-EXAScaler - use: [network1] - settings: {local_mount: /scratch} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(lustrefs.install_ddn_lustre_client_runner) - - $(lustrefs.mount_runner) - -``` - -See [additional documentation][ddn-install-docs] from DDN EXAScaler. - -[ddn-install-docs]: https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp#install-new-exascaler-cloud-clients -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## Support - -EXAScaler Cloud includes self-help support with access to publicly available -documents and videos. Premium support includes 24x7x365 access to DDN's experts, -along with support community access, automated notifications of updates and -other premium support features. For more information, visit -[EXAscaler Cloud on GCP][exa-gcp]. - -[exa-gcp]: https://console.cloud.google.com/marketplace/product/ddnstorage/exascaler-cloud - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [ddn\_exascaler](#module\_ddn\_exascaler) | github.com/DDNStorage/exascaler-cloud-terraform//gcp | a3355d50deebe45c0556b45bd599059b7c06988d | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [boot](#input\_boot) | Boot disk properties | 
object({
    disk_type   = string
    auto_delete = bool
    script_url  = string
  })
| 
{
  "auto_delete": true,
  "disk_type": "pd-standard",
  "script_url": null
}
| no | -| [cls](#input\_cls) | Compute client properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 0,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-2",
  "public_ip": true
}
| no | -| [clt](#input\_clt) | Compute client target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 0,
  "disk_size": 256,
  "disk_type": "pd-standard"
}
| no | -| [fsname](#input\_fsname) | EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long | `string` | `"exacloud"` | no | -| [image](#input\_image) | DEPRECATED: Source image properties | `any` | `null` | no | -| [instance\_image](#input\_instance\_image) | Source image properties

Expected Fields:
name: Unavailable with this module.
family: The image family to use.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp",
  "project": "ddn-public"
}
| no | -| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. Key-value pairs. | `map(string)` | `{}` | no | -| [local\_mount](#input\_local\_mount) | Mountpoint (at the client instances) for this EXAScaler system | `string` | `"/shared"` | no | -| [mds](#input\_mds) | Metadata server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | -| [mdt](#input\_mdt) | Metadata target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | -| [mgs](#input\_mgs) | Management server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | -| [mgt](#input\_mgt) | Management target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | -| [mnt](#input\_mnt) | Monitoring target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | -| [network\_properties](#input\_network\_properties) | Network options. 'network\_self\_link' or 'network\_properties' must be provided. | 
object({
    routing = string
    tier    = string
    id      = string
    auto    = bool
    mtu     = number
    new     = bool
    nat     = bool
  })
| `null` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self-link of the VPC network to where the system is connected. Ignored if 'network\_properties' is provided. 'network\_self\_link' or 'network\_properties' must be provided. | `string` | `null` | no | -| [oss](#input\_oss) | Object Storage server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 3,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-16",
  "public_ip": true
}
| no | -| [ost](#input\_ost) | Object Storage target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | -| [prefix](#input\_prefix) | EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud') | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Compute Platform project that will host the EXAScaler filesystem | `string` | n/a | yes | -| [security](#input\_security) | Security options | 
object({
    admin              = string
    public_key         = string
    block_project_keys = bool
    enable_os_login    = bool
    enable_local       = bool
    enable_ssh         = bool
    enable_http        = bool
    ssh_source_ranges  = list(string)
    http_source_ranges = list(string)
  })
| 
{
  "admin": "stack",
  "block_project_keys": false,
  "enable_http": false,
  "enable_local": false,
  "enable_os_login": true,
  "enable_ssh": false,
  "http_source_ranges": [
    "0.0.0.0/0"
  ],
  "public_key": null,
  "ssh_source_ranges": [
    "0.0.0.0/0"
  ]
}
| no | -| [service\_account](#input\_service\_account) | Service account name used by deploy application | 
object({
    new   = bool
    email = string
  })
| 
{
  "email": null,
  "new": false
}
| no | -| [subnetwork\_address](#input\_subnetwork\_address) | The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork\_properties' is provided. | `string` | `null` | no | -| [subnetwork\_properties](#input\_subnetwork\_properties) | Subnetwork properties. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | 
object({
    address = string
    private = bool
    id      = string
    new     = bool
  })
| `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork\_properties' is provided. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | `string` | `null` | no | -| [waiter](#input\_waiter) | Waiter to check progress and result for deployment. | `string` | `null` | no | -| [zone](#input\_zone) | Compute Platform zone where the servers will be located | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_config\_script](#output\_client\_config\_script) | Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler. | -| [http\_console](#output\_http\_console) | HTTP address to access the system web console. | -| [install\_ddn\_lustre\_client\_runner](#output\_install\_ddn\_lustre\_client\_runner) | Runner that encapsulates the `client_config_script` output on this module. | -| [mount\_command](#output\_mount\_command) | Command to mount the file system. `client_config_script` must be run first. | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the DDN EXAScaler Lustre file system | -| [network\_storage](#output\_network\_storage) | Describes a EXAScaler system to be mounted by other systems. | -| [private\_addresses](#output\_private\_addresses) | Private IP addresses for all instances. | -| [ssh\_console](#output\_ssh\_console) | Instructions to ssh into the instances. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf deleted file mode 100644 index 6a2fc4b702..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ddn-exascaler", ghpc_role = "file-system" }) -} - -locals { - - network_id = var.network_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.network_self_link)[0] : null - named_net = { - routing = "REGIONAL" - tier = "STANDARD" - id = local.network_id - auto = false - mtu = 1500 - new = false - nat = false - } - - subnetwork_id = var.subnetwork_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.subnetwork_self_link)[0] : null - named_subnet = { - address = var.subnetwork_address - private = true - id = local.subnetwork_id - new = false - } -} - -module "ddn_exascaler" { - source = "github.com/DDNStorage/exascaler-cloud-terraform//gcp?ref=a3355d50deebe45c0556b45bd599059b7c06988d" - fsname = var.fsname - zone = var.zone - project = var.project_id - prefix = var.prefix - labels = local.labels - security = var.security - service_account = var.service_account - waiter = var.waiter - network = var.network_properties == null ? local.named_net : var.network_properties - subnetwork = var.subnetwork_properties == null ? local.named_subnet : var.subnetwork_properties - boot = var.boot - image = var.instance_image - mgs = var.mgs - mgt = var.mgt - mnt = var.mnt - mds = var.mds - mdt = var.mdt - oss = var.oss - ost = var.ost - cls = var.cls - clt = var.clt -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml deleted file mode 100644 index b995bd4358..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - deploymentmanager.googleapis.com - - iam.googleapis.com - - runtimeconfig.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf deleted file mode 100644 index 2e9ae732ae..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -output "private_addresses" { - description = "Private IP addresses for all instances." - value = module.ddn_exascaler.private_addresses -} - -output "ssh_console" { - description = "Instructions to ssh into the instances." - value = module.ddn_exascaler.ssh_console -} - -output "client_config_script" { - description = "Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler." - value = module.ddn_exascaler.client_config -} - -output "install_ddn_lustre_client_runner" { - description = "Runner that encapsulates the `client_config_script` output on this module." - value = local.client_install_runner -} - -locals { - client_install_runner = { - "type" = "shell" - "content" = module.ddn_exascaler.client_config - "destination" = "install_ddn_lustre_client.sh" - } - - # Mount command provided by DDN does not support custom local mount - split_mount_cmd = split(" ", module.ddn_exascaler.mount_command) - split_mount_cmd_wo_mountpoint = slice(local.split_mount_cmd, 0, length(local.split_mount_cmd) - 1) - mount_cmd = "${join(" ", local.split_mount_cmd_wo_mountpoint)} ${var.local_mount}" - mount_cmd_w_mkdir = "mkdir -p ${var.local_mount} && ${local.mount_cmd}" - mount_runner = { - "type" = "shell" - "content" = local.mount_cmd_w_mkdir - "destination" = "mount-ddn-lustre.sh" - } -} - -output "mount_command" { - description = "Command to mount the file system. `client_config_script` must be run first." - value = local.mount_cmd_w_mkdir -} - -output "mount_runner" { - description = "Runner to mount the DDN EXAScaler Lustre file system" - value = local.mount_runner -} - -output "http_console" { - description = "HTTP address to access the system web console." - value = module.ddn_exascaler.http_console -} - -output "network_storage" { - description = "Describes a EXAScaler system to be mounted by other systems." - value = { - server_ip = split(":", split(" ", module.ddn_exascaler.mount_command)[3])[0] - remote_mount = length(regexall("^/.*", var.fsname)) > 0 ? var.fsname : format("/%s", var.fsname) - local_mount = var.local_mount != null ? var.local_mount : format("/mnt/%s", var.fsname) - fs_type = "lustre" - mount_options = "" - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } - depends_on = [ - module.ddn_exascaler - ] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf deleted file mode 100644 index 68bcc8a8ba..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf +++ /dev/null @@ -1,502 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -# EXAScaler filesystem name -# only alphanumeric characters are allowed, -# and the value must be 1-8 characters long -variable "fsname" { - description = "EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long" - type = string - default = "exacloud" -} - -# Project ID to manage resources -# https://cloud.google.com/resource-manager/docs/creating-managing-projects -variable "project_id" { - description = "Compute Platform project that will host the EXAScaler filesystem" - type = string -} - -# Zone name to manage resources -# https://cloud.google.com/compute/docs/regions-zones -variable "zone" { - description = "Compute Platform zone where the servers will be located" - type = string -} - -# Service account name used by deploy application -# https://cloud.google.com/iam/docs/service-accounts -# new: create a new custom service account or use an existing one: true or false -# email: existing service account email address, will be using if new is false -# set email = null to use the default compute service account -variable "service_account" { - description = "Service account name used by deploy application" - type = object({ - new = bool - email = string - }) - default = { - new = false - email = null - } -} - -# Waiter to check progress and result for deployment. -# To use Google Deployment Manager: -# waiter = "deploymentmanager" -# To use generic Google Cloud SDK command line: -# waiter = "sdk" -# If you don’t want to wait until the deployment is complete: -# waiter = null -# https://cloud.google.com/deployment-manager/runtime-configurator/creating-a-waiter -variable "waiter" { - description = "Waiter to check progress and result for deployment." - type = string - default = null -} - -# Security options -# admin: optional user name for remote SSH access -# Set admin = null to disable creation admin user -# public_key: path to the SSH public key on the local host -# Set public_key = null to disable creation admin user -# block_project_keys: true or false -# Block project-wide public SSH keys if you want to restrict -# deployment to only user with deployment-level public SSH key. -# https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys -# enable_os_login: true or false -# Enable or disable OS Login feature. -# Please note, enabling this option disables other security options: -# admin, public_key and block_project_keys. -# https://cloud.google.com/compute/docs/instances/managing-instance-access#enable_oslogin -# enable_local: true or false, enable or disable firewall rules for local access -# enable_ssh: true or false, enable or disable remote SSH access -# ssh_source_ranges: source IP ranges for remote SSH access in CIDR notation -# enable_http: true or false, enable or disable remote HTTP access -# http_source_ranges: source IP ranges for remote HTTP access in CIDR notation -variable "security" { - description = "Security options" - type = object({ - admin = string - public_key = string - block_project_keys = bool - enable_os_login = bool - enable_local = bool - enable_ssh = bool - enable_http = bool - ssh_source_ranges = list(string) - http_source_ranges = list(string) - }) - - default = { - admin = "stack" - public_key = null - block_project_keys = false - enable_os_login = true - enable_local = false - enable_ssh = false - enable_http = false - ssh_source_ranges = [ - "0.0.0.0/0" - ] - http_source_ranges = [ - "0.0.0.0/0" - ] - } -} - -variable "network_self_link" { - description = "The self-link of the VPC network to where the system is connected. Ignored if 'network_properties' is provided. 'network_self_link' or 'network_properties' must be provided." - type = string - default = null -} - -# Network properties -# https://cloud.google.com/vpc/docs/vpc -# routing: network-wide routing mode: REGIONAL or GLOBAL -# tier: networking tier for VM interfaces: STANDARD or PREMIUM -# id: existing network id, will be using if new is false -# auto: create subnets in each region automatically: false or true -# mtu: maximum transmission unit in bytes: 1460 - 1500 -# new: create a new network or use an existing one: true or false -# nat: allow instances without external IP to communicate with the outside world: true or false -variable "network_properties" { - description = "Network options. 'network_self_link' or 'network_properties' must be provided." - type = object({ - routing = string - tier = string - id = string - auto = bool - mtu = number - new = bool - nat = bool - }) - - default = null -} - -variable "subnetwork_self_link" { - description = "The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork_properties' is provided. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." - type = string - default = null -} - -variable "subnetwork_address" { - description = "The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork_properties' is provided." - type = string - default = null -} - -# Subnetwork properties -# https://cloud.google.com/vpc/docs/vpc -# address: IP range of internal addresses for a new subnetwork -# private: when enabled VMs in this subnetwork without external -# IP addresses can access Google APIs and services by using -# Private Google Access: true or false -# https://cloud.google.com/vpc/docs/private-access-options -# id: existing subnetwork id, will be using if new is false -# new: create a new subnetwork or use an existing one: true or false -variable "subnetwork_properties" { - description = "Subnetwork properties. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." - type = object({ - address = string - private = bool - id = string - new = bool - }) - default = null -} -# Boot disk properties -# disk_type: pd-standard, pd-ssd or pd-balanced -# auto_delete: true or false -# whether the disk will be auto-deleted when the instance is deleted -variable "boot" { - description = "Boot disk properties" - type = object({ - disk_type = string - auto_delete = bool - script_url = string - }) - default = { - disk_type = "pd-standard" - auto_delete = true - script_url = null - } -} - -# Source image properties -# project: project name -# family: image family name -# name: !!DEPRECATED!! - image name -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: Source image properties" - type = any - # Omitting type checking so validation can provide more useful error message - # type = object({ - # project = string - # family = string - # }) - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - Source image properties - - Expected Fields: - name: Unavailable with this module. - family: The image family to use. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "ddn-public" - family = "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp" - } - - validation { - condition = !can(coalesce(var.instance_image.name)) - error_message = "In var.instance_image, the \"name\" field is not used, please use the \"family\" setting." - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, the \"family\" field must be a string set to the image family." - } -} - -# Management server properties -# node_type: type of management server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of management servers -variable "mgs" { - description = "Management server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-32" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 1 - } -} - -# Management target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of management target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of management target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of management target in GB (scratch disk size must be exactly 375) -# disk_count: number of management targets -# disk_raid: create striped management target, true or false -variable "mgt" { - description = "Management target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 128 - disk_count = 1 - disk_raid = false - } -} - - -# Monitoring target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of monitoring target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of monitoring target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of monitoring target in GB (scratch disk size must be exactly 375) -# disk_count: number of monitoring targets -# disk_raid: create striped monitoring target, true or false -variable "mnt" { - description = "Monitoring target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 128 - disk_count = 1 - disk_raid = false - } -} - -# Metadata server properties -# node_type: type of metadata server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of metadata servers -variable "mds" { - description = "Metadata server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-32" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 1 - } -} - -# Metadata target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of metadata target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of metadata target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of metadata target in GB (scratch disk size must be exactly 375) -# disk_count: number of metadata targets -# disk_raid: create striped metadata target, true or false -variable "mdt" { - description = "Metadata target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-ssd" - disk_size = 3500 - disk_count = 1 - disk_raid = false - } -} - -# Object Storage server properties -# node_type: type of storage server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of storage servers -variable "oss" { - description = "Object Storage server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-16" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 3 - } -} - -# Object Storage target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of storage target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of storage target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of storage target in GB (scratch disk size must be exactly 375) -# disk_count: number of storage targets -# disk_raid: create striped storage target, true or false -variable "ost" { - description = "Object Storage target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-ssd" - disk_size = 3500 - disk_count = 1 - disk_raid = false - } -} - -# Compute client properties -# node_type: type of compute client -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of compute clients -variable "cls" { - description = "Compute client properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-2" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 0 - } -} -# Compute client target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of compute target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of compute target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of compute target in GB (scratch disk size must be exactly 375) -# disk_count: number of compute targets -variable "clt" { - description = "Compute client target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 256 - disk_count = 0 - } -} -variable "local_mount" { - description = "Mountpoint (at the client instances) for this EXAScaler system" - type = string - default = "/shared" -} - -variable "prefix" { - description = "EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud')" - type = string - default = null -} - -variable "labels" { - description = "Labels to add to EXAScaler Cloud deployment. Key-value pairs." - type = map(string) - default = {} -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf deleted file mode 100644 index 2981b4dd75..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/Intel-DAOS/README.md b/deletion-test/build_script/modules/embedded/community/modules/file-system/Intel-DAOS/README.md deleted file mode 100644 index 04db0acb8c..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/Intel-DAOS/README.md +++ /dev/null @@ -1 +0,0 @@ -> **_NOTE:_** Cluster Toolkit is dropping support for the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository. The DAOS example blueprints (`hpc-slurm-daos.yaml` and `pfs-daos.yaml`) have been removed from the Cluster Toolkit. We recommend migrating to the first-party [Parallelstore](../../../../modules/file-system/parallelstore/) module for similar functionality. To help with this transition, see the Parallelstore example blueprints ([pfs-parallelstore.yaml](../../../../examples/pfs-parallelstore.yaml) and [ps-slurm.yaml](../../../../examples/ps-slurm.yaml)). If the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository is necessary, we recommend using the last Cluster Toolkit [v1.41.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/releases/tag/v1.41.0). diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/README.md b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/README.md deleted file mode 100644 index 66aaaa46af..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/README.md +++ /dev/null @@ -1,152 +0,0 @@ -## Description - -This module creates a Network File Sharing (NFS) file system based on a VM -instance and [compute disk][disk]. This file system can share directories and -files with other clients over a network. `nfs-server` can be used by -[vm-instance](../../../../modules/compute/vm-instance/README.md) and SchedMD -community modules that create compute VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -If you are using Hyperdisk storage, check the possible disk size, IOPS, and throughput values for each disk type in the [Hyperdisk limits documentation](https://cloud.google.com/compute/docs/disks/hyperdisks#limits-disk). - -> **_WARNING:_** This module has only been tested against the HPC centos7 OS -> disk image (the default). Using other images may work, but have not been -> verified. - -[disk]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk - -### Example - -```yaml -- id: homefs - source: community/modules/file-system/nfs-server - use: [network1] -``` - -This creates a NFS on a virtual machine which allow other VMs to mount the -volume as an external file system. - -> **_NOTE:_** All disks are destroyed along with the instance, during a `gcluster destroy`/`terraform destroy` event. However, you can setup data retention with `create_boot_snapshot_before_destroy` (boot disk) and `create_snapshot_before_destroy` (data disk). - -## Mounting - -To mount the NFS Server you must first ensure that the NFS client has been -installed the and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -See the [hpc-centos-ss.yaml] test config for an example of using this module -with a `vm-instance` module. - -If mounting is not automatically handled as described above, the `nfs-server` -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - - id: nfs - source: community/modules/file-system/nfs-server - use: [network1] - settings: {local_mounts: [/mnt1]} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(nfs.install_nfs_client_runner) - - $(nfs.mount_runner) - -``` - -[hpc-centos-ss.yaml]: ../../../../tools/validate_configs/test_configs/hpc-centos-ss.yaml -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 6.14 | -| [null](#requirement\_null) | >= 3.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.14 | -| [null](#provider\_null) | >= 3.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_disk.attached_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_instance.compute_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource | -| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_default_service_account.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_default_service_account) | data source | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [auto\_delete\_disk](#input\_auto\_delete\_disk) | DEPRECATED: Whether or not the NFS disk should be auto-deleted | `string` | `null` | no | -| [boot\_disk\_size](#input\_boot\_disk\_size) | Storage size in GB for the boot disk | `number` | `null` | no | -| [boot\_disk\_type](#input\_boot\_disk\_type) | Storage type for the boot disk | `string` | `null` | no | -| [create\_boot\_snapshot\_before\_destroy](#input\_create\_boot\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the boot disk | `bool` | `false` | no | -| [create\_snapshot\_before\_destroy](#input\_create\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the NFS data disk | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the NFS instance if no name is specified. | `string` | n/a | yes | -| [disk\_size](#input\_disk\_size) | Storage size in GB for the NFS data disk | `number` | `"100"` | no | -| [image](#input\_image) | DEPRECATED: The VM image used by the NFS server | `string` | `null` | no | -| [instance\_image](#input\_instance\_image) | The VM image used by the NFS server.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to the NFS instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mounts](#input\_local\_mounts) | Mountpoint for this NFS compute instance | `list(string)` | 
[
  "/data"
]
| no | -| [machine\_type](#input\_machine\_type) | Type of the VM instance to use | `string` | `"n2d-standard-2"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the NFS VM. | `string` | `"default"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [provisioned\_iops](#input\_provisioned\_iops) | Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput | `number` | `null` | no | -| [provisioned\_throughput](#input\_provisioned\_throughput) | Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme | `number` | `null` | no | -| [scopes](#input\_scopes) | Scopes to apply to the controller | `list(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [service\_account](#input\_service\_account) | Service Account for the NFS server | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the NFS VM. | `string` | `null` | no | -| [type](#input\_type) | Storage type for the NFS data disk | `string` | `"pd-ssd"` | no | -| [zone](#input\_zone) | The zone name where the NFS instance located in. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | -| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | -| [network\_storage](#output\_network\_storage) | export of all desired folder directories | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/main.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/main.tf deleted file mode 100644 index a00d2681ba..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/main.tf +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "nfs-server", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - server_ip = google_compute_instance.compute_instance.network_interface[0].network_ip - fs_type = "nfs" - mount_options = "defaults,hard,intr" - install_nfs_client_runners = [for mount in var.local_mounts : - { - "type" = "shell" - "source" = "${path.module}/scripts/install-nfs-client.sh" - "destination" = "install-nfs${replace(mount, "/", "_")}.sh" - } - ] - mount_runners = [for mount in var.local_mounts : - { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"/exports${mount}\" \"${mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(mount, "/", "_")}.sh" - } - ] - ansible_mount_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/mount.yaml" - "destination" = "mount.yaml" - } -} - -data "google_compute_default_service_account" "default" {} - -resource "google_compute_disk" "attached_disk" { - project = var.project_id - name = "${local.name}-nfs-instance-disk" - size = var.disk_size - type = var.type - zone = var.zone - labels = local.labels - provisioned_iops = var.provisioned_iops - provisioned_throughput = var.provisioned_throughput - create_snapshot_before_destroy = var.create_snapshot_before_destroy -} - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project -} - -resource "null_resource" "image" { - triggers = { - name = try(var.instance_image.name, null), - family = try(var.instance_image.family, null), - project = var.instance_image.project - } -} - -resource "google_compute_disk" "boot_disk" { - project = var.project_id - - name = "${local.name}-boot-disk" - size = var.boot_disk_size - type = var.boot_disk_type - image = data.google_compute_image.compute_image.self_link - labels = local.labels - zone = var.zone - create_snapshot_before_destroy = var.create_boot_snapshot_before_destroy - - lifecycle { - replace_triggered_by = [null_resource.image] - ignore_changes = [ - image - ] - } -} - -resource "google_compute_instance" "compute_instance" { - project = var.project_id - name = "${local.name}-nfs-instance" - zone = var.zone - machine_type = var.machine_type - - boot_disk { - auto_delete = false - source = google_compute_disk.boot_disk.self_link - device_name = google_compute_disk.boot_disk.name - } - - attached_disk { - source = google_compute_disk.attached_disk.id - device_name = "attached_disk" - } - - network_interface { - network = var.network_self_link - subnetwork = var.subnetwork_self_link - } - - service_account { - email = var.service_account == null ? data.google_compute_default_service_account.default.email : var.service_account - scopes = var.scopes - } - - metadata = var.metadata - metadata_startup_script = templatefile("${path.module}/scripts/install-nfs-server.sh.tpl", { local_mounts = var.local_mounts }) - - labels = local.labels -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/outputs.tf deleted file mode 100644 index e23b94e2b2..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/outputs.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -# render the content for each folder -output "network_storage" { - description = "export of all desired folder directories" - value = [for i, mount in var.local_mounts : { - remote_mount = "/exports${mount}" - local_mount = mount - fs_type = local.fs_type - mount_options = local.mount_options - server_ip = local.server_ip - client_install_runner = local.install_nfs_client_runners[i] - mount_runner = local.mount_runners[i] - } - ] -} - -output "install_nfs_client" { - description = "Script for installing NFS client" - value = file("${path.module}/scripts/install-nfs-client.sh") -} - -output "install_nfs_client_runner" { - description = "Runner to install NFS client using the startup-script module" - value = local.install_nfs_client_runners[0] -} - -output "mount_runner" { - description = <<-EOT - Runner to mount the file-system using an ansible playbook. The startup-script - module will automatically handle installation of ansible. - - id: example-startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(your-fs-id.mount_runner) - ... - EOT - value = local.ansible_mount_runner -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl deleted file mode 100644 index 1b06a5f032..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -ex - -if [ ! -d "/exports" ]; then # first load, format and mount the disk - # See https://cloud.google.com/compute/docs/disks/add-persistent-disk - uuid=$(uuidgen) - mkfs.ext4 -F -m 0 -U "$uuid" -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/disk/by-id/google-attached_disk - - mkdir /exports - echo "UUID=$uuid /exports ext4 discard,defaults 0 0" >> /etc/fstab - mount --target /exports/ - - %{ for mount in local_mounts ~} - mkdir -p /exports${mount} - chmod 755 /exports${mount} - echo '/exports${mount} *(rw,sync,no_root_squash)' >> "/etc/exports" - %{ endfor ~} -fi - -systemctl start nfs-server rpcbind -systemctl enable nfs-server -exportfs -r diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml deleted file mode 100644 index f7fbe58d5e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Mounts the file systems specified in the metadata network_storage key - hosts: localhost - become: true - vars: - meta_key: "network_storage" - url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" - tasks: - - name: Read metadata network_storage information - ansible.builtin.uri: - url: "{{ url }}/{{ meta_key }}" - method: GET - headers: - Metadata-Flavor: "Google" - register: storage - - name: Mount file systems - ansible.posix.mount: - src: "{{ item.server_ip }}:/{{ item.remote_mount }}" - path: "{{ item.local_mount }}" - opts: "{{ item.mount_options }}" - boot: true - fstype: "{{ item.fs_type }}" - state: "mounted" - loop: "{{ storage.json }}" diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/variables.tf deleted file mode 100644 index 9a58da641e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/variables.tf +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the NFS instance if no name is specified." - type = string -} - -variable "name" { - description = "The resource name of the instance." - type = string - default = null -} - -variable "zone" { - description = "The zone name where the NFS instance located in." - type = string -} - -variable "boot_disk_size" { - description = "Storage size in GB for the boot disk" - type = number - default = null -} - -variable "boot_disk_type" { - description = "Storage type for the boot disk" - type = string - default = null -} - -variable "create_boot_snapshot_before_destroy" { - description = "Whether to create a snapshot before destroying the boot disk" - type = bool - default = false -} - -variable "disk_size" { - description = "Storage size in GB for the NFS data disk" - type = number - default = "100" -} - -variable "type" { - description = "Storage type for the NFS data disk" - type = string - default = "pd-ssd" -} - -variable "create_snapshot_before_destroy" { - description = "Whether to create a snapshot before destroying the NFS data disk" - type = bool - default = false -} - -variable "provisioned_iops" { - description = "Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput" - type = number - default = null -} - -variable "provisioned_throughput" { - description = "Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme" - type = number - default = null -} - -# Deprecated, replaced by instance_image -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: The VM image used by the NFS server" - type = string - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - The VM image used by the NFS server. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -# Deprecated, replaced by create_snapshot_before_destroy and create_boot_snapshot_before_destroy -# tflint-ignore: terraform_unused_declarations -variable "auto_delete_disk" { - description = "DEPRECATED: Whether or not the NFS disk should be auto-deleted" - type = string - default = null - - validation { - condition = var.auto_delete_disk == null - error_message = "The 'var.auto_delete_disk' setting is broken in Cluster Toolkit versions >1.25.0 and deprecated in versions >1.48.0, please use 'var.create_snapshot_before_destroy' and 'var.create_boot_snapshot_before_destroy' instead." - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the NFS VM." - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the NFS VM." - type = string - default = null -} - -variable "machine_type" { - description = "Type of the VM instance to use" - type = string - default = "n2d-standard-2" -} - -variable "labels" { - description = "Labels to add to the NFS instance. Key-value pairs." - type = map(string) -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "service_account" { - description = "Service Account for the NFS server" - type = string - default = null -} - -variable "scopes" { - description = "Scopes to apply to the controller" - type = list(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "local_mounts" { - description = "Mountpoint for this NFS compute instance" - type = list(string) - default = ["/data"] - - validation { - condition = alltrue([ - for m in var.local_mounts : substr(m, 0, 1) == "/" - ]) - error_message = "Local mountpoints have to start with '/'." - } - validation { - condition = length(var.local_mounts) > 0 - error_message = "At least one local mount must be specified in var.local_mounts." - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/versions.tf deleted file mode 100644 index 63443806b8..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/nfs-server/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.14" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/sycomp-scale/README.md b/deletion-test/build_script/modules/embedded/community/modules/file-system/sycomp-scale/README.md deleted file mode 100644 index 79ff12bc18..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/sycomp-scale/README.md +++ /dev/null @@ -1,35 +0,0 @@ -## Description - -This document provides information on how to deploy an instance of [Sycomp Intelligent Data Storage Platform](https://sycomp.com/solution/hpc/storage/) on Google Cloud Platform ([GCP](https://cloud.google.com/)) using the Google Cluster Toolkit. - -> **_NOTE:_** -> Sycomp Storage on GCP does not require an HPC Toolkit wrapper. -> Terraform modules are sourced directly from GitLab. - -Terraform modules for Sycomp Intelligent Data Storage Platform are downloaded on deployment using the Google Cloud Toolkit. - -The Terraform module parameters are documented in the `README.md` files in the respective module directories of the source GitLab repository. The main modules are: - -- `sycomp-scale` -- `sycomp-scale-expansion` - -## Examples - -The community examples folder (community/examples/sycomp/) contains four example blueprints that you can use to deploy or expand a Sycomp Storage cluster. - -- [community/examples/sycomp/sycomp-storage.yaml][sycomp-storage-yaml] - - Blueprint for deploying a Sycomp Storage cluster consisting of 3 storage servers. - -- [community/examples/sycomp/sycomp-storage-expansion.yaml][sycomp-storage-expansion-yaml] - - Blueprint for expanding the above created cluster from 3 to 4 storage servers. - -- [community/examples/sycomp/sycomp-storage-ece.yaml][sycomp-storage-ece-yaml] - - Blueprint for deploying a Sycomp Storage cluster consisting of 7 storage servers with ECE (Erasure Code Edition) software RAID. - -- [community/examples/sycomp/sycomp-storage-slurm.yaml][sycomp-storage-slurm-yaml] - - Blueprint for deploying a Slurm cluster and Sycomp Storage cluster with 3 servers. The Slurm compute nodes are configured as NFS clients and have the ability to use the Sycomp Storage filesystem. - -[sycomp-storage-yaml]: ../../../examples/sycomp/sycomp-storage.yaml -[sycomp-storage-expansion-yaml]: ../../../examples/sycomp/sycomp-storage-expansion.yaml -[sycomp-storage-ece-yaml]: ../../../examples/sycomp/sycomp-storage-ece.yaml -[sycomp-storage-slurm-yaml]: ../../../examples/sycomp/sycomp-storage-slurm.yaml diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/README.md b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/README.md deleted file mode 100644 index 0e2a936167..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/README.md +++ /dev/null @@ -1,182 +0,0 @@ -## Description - -This module provides scripts for client installation and mounting [WEKA] -filesystems. Client supports both UDP and DPDK modes and allows customization of -mount parameters using Compute VM instance metadata. - -For deploying Weka cluster please consult [WEKA installation on GCP]. - -[WEKA]: https://www.weka.io/ -[WEKA installation on GCP]: https://docs.weka.io/planning-and-installation/weka-installation-on-gcp - -## Prerequisites - -* up and running Weka cluster -* running on a [supported OS](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#operating-system) -* [open firewall](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#required-ports) - between WEKA backend servers and clients -* VPC peering configuration: - * if clients share VPCs created for WEKA cluster, no additional configuration - is necessary - * if dedicated VPCs are in use for clients, then WEKA VPCs needs to be peered - with VPCs that are used as: - * primary interface on client - * interfaces dedicated for DPDK client - * if dedicated VPCs are in use for clients, then those VPCs needs to be peered - with each other - -## Mounting -This example creates mount scripts that will mount `default` filesystem from -`10.0.0.3` WEKA backend: - -```yaml - - id: wekafs - source: community/modules/file-system/weka-client - settings: - local_mount: /scratch - server_ip: 10.0.0.3 - remote_mount: default - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: $(wekafs.runners) -``` - -If you need to add mount script along other runners, remember to add all 4 -runners provided by this script as shown in this example: - -```yaml - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(wekafs.client_install_runner) - - $(wekafs.mount_runner) - - type: shell - content: | - #!/bin/bash - - echo Sample - destination: sample-script.sh -``` - -To use the client within Slurm partition, with DPDK, remember to set additional -networks, and configure metadata. In this example, all four additional interfaces -are dedicated to WEKA DPDK - -```yaml - - id: c2_60_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - - mount-at-startup # as defined in previous examples - settings: - bandwidth_tier: virtio_enabled # Weka requires VirtIO, from WEKA 4.4.1, DPDK is also supported on gVNIC - additional_networks: - - subnetwork: weka-client-1 - nic_type: VIRTIO_NET - - subnetwork: weka-client-2 - nic_type: VIRTIO_NET - - subnetwork: weka-client-3 - nic_type: VIRTIO_NET - - subnetwork: weka-client-4 - nic_type: VIRTIO_NET - machine_type: c2-standard-60 - metadata: - weka-data_interfaces: 1,2,3,4 # allocate interfaces 1, 2, 3 and 4 to DPDK - weka-mode: dpdk - weka-options: num_cores=4,dpdk_base_memory_mb=16 - node_conf: - # From https://docs.weka.io/planning-and-installation/bare-metal/planning-a-weka-system-installation - # do not set RealMem as this is set automatically by Cluster Toolkit - CoreSpecCount: 4 - MemSpecLimit: 5120 -``` - -Due to the fact, that client installation takes ~6-7 minutes, if you use WEKA together with Slurm and do not bundle -client in the instance image, you may need to increase the timeout for startups scripts. - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - settings: - compute_startup_scripts_timeout: 600 - login_startup_scripts_timeout: 600 - ... - - id: compute_partition - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - settings: - resume_timeout: 600 - ... -``` - -## Supported VM metadata options -Client scripts do support following metadata keys: -* `weka-mode` - one of `udp` or `dpdk`. Defaults to `udp`. Sets client mode. -* `weka-data_interfaces` - comma separated list of interface identifiers, - specifying which interfaces are dedicated for data plane. Set to `1` to - dedicate second interface of instance for WEKA DPDK. Set to `2,5` to dedicate - third and sixth interface of instance for WEKA DPDK. -* `weka-mgmt_interface` - identifier of management interface, defaults to `0`, - which means to use primary interface as management interface. -* `weka-options` - additional [mount command options](https://docs.weka.io/weka-filesystems-and-object-stores/mounting-filesystems#mount-command-options) - to pass to `mount` command - -## Adding client to the OS image -To save time during the mount command install and precompile DPDK drivers in the -OS image. Following scripts compiles DPDK driver for currently running kernel. - -```shell -#!/bin/bash - -set -e -o pipefail - -echo Downloading and installing Weka client -curl --max-time 10 "{{ weka backend endpoint }}/dist/v1/install" | sh -WEKA_VERSION=$(weka -v | sed -e 's/^[^0-9]*//') -echo Installing Weka version: ${WEKA_VERSION} -weka version get "${WEKA_VERSION}" -weka version set "${WEKA_VERSION}" -# run setup for the second time, if it fails for the first time -weka local setup weka || weka local setup weka -weka version prepare "${WEKA_VERSION}" -weka local stop -weka local rm -f --all -``` - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | -| [mount\_options](#input\_mount\_options) | Mount options for filesystem shared by all clients. | `string` | `""` | no | -| [remote\_mount](#input\_remote\_mount) | Weka filesystem name. | `string` | n/a | yes | -| [server\_ip](#input\_server\_ip) | Weka backend IP address used for bootstrapping. | `string` | `""` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_install\_runner](#output\_client\_install\_runner) | Ansible runner that performs client installation needed to use file system. | -| [mount\_runner](#output\_mount\_runner) | Ansible runner that mounts the file system. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/metadata.yaml deleted file mode 100644 index 419bc3fe46..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/outputs.tf deleted file mode 100644 index 0bd9098d80..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/outputs.tf +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - template_args = { - local_mount = var.local_mount - mount_options = var.mount_options == "" ? "" : "-o ${var.mount_options}" - remote_mount = var.remote_mount - server_ip = var.server_ip - service_name = "weka-mount${replace(var.local_mount, "/", "-")}" - } - mount_script = templatefile("${path.module}/templates/mount-weka.sh.tftpl", local.template_args) - - mount_runner_ansible = { - type = "ansible-local" - content = templatefile( - "${path.module}/templates/mount-weka.yaml.tftpl", - merge( - local.template_args, - { mount_weka_script = local.mount_script } - ) - ) - destination = "mount_filesystem${replace(var.local_mount, "/", "_")}.yaml" - } - - client_install_runner = { - type = "ansible-local" - content = templatefile("${path.module}/templates/install-weka-client.yaml.tftpl", local.template_args) - destination = "install_filesystem${replace(var.local_mount, "/", "_")}.yaml" - } -} - -# currently WEKA mounts are not compatible with network_storage logic, as WEKA volumes needs to be mounted by -# systemd script and not /etc/fstab entry, as the mount command needs to have network configuration which may change -# between restarts -# -#output "network_storage" { -# description = "Describes a remote network storage to be mounted by fs-tab." -# value = { -# server_ip = var.server_ip -# remote_mount = var.remote_mount -# local_mount = var.local_mount -# fs_type = var.fs_type -# mount_options = var.mount_options -# client_install_runner = local.client_install_runner -# mount_runner = local.mount_runner -# } -#} -# -output "client_install_runner" { - description = "Ansible runner that performs client installation needed to use file system." - value = local.client_install_runner -} - -output "mount_runner" { - description = "Ansible runner that mounts the file system." - value = local.mount_runner_ansible -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl deleted file mode 100644 index ddc3acdb5d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Mounts the file systems specified in the metadata network_storage key - hosts: localhost - become: true - vars: - meta_key: "network_storage" - url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" - tasks: - - name: Check if weka is installed - ansible.builtin.stat: - path: /usr/bin/weka - register: weka_binary - - - name: Create temporary location for installation script - ansible.builtin.tempfile: - state: file - register: - install_script - when: not weka_binary.stat.exists - - - name: Download WEKA client - ansible.builtin.get_url: - url: http://${server_ip}:14000/dist/v1/install - dest: "{{ install_script.path }}" - mode: "700" - when: not weka_binary.stat.exists - - - name: Run WEKA installation script - ansible.builtin.shell: - cmd: "{{ install_script.path }}" - when: not weka_binary.stat.exists - register: weka_install_result - changed_when: weka_install_result.rc == 0 - - - name: Read metadata network_storage information - ansible.builtin.uri: - url: "{{ url }}/weka-version" - method: GET - headers: - Metadata-Flavor: "Google" - status_code: - - 200 - - 404 - register: get_weka_version - - - name: Set WEKA version from metadata server - ansible.builtin.set_fact: - weka_version: "{{ get_weka_version.body }}" - when: get_weka_version.status == 200 - - - name: Get version of WEKA installation client - ansible.builtin.shell: - cmd: weka -v | sed -e 's/^[^0-9.]*\([0-9.]*\)[^0-9.]*$/\1/' - register: get_weka_client_version - changed_when: get_weka_client_version.rc == 0 - - - name: Set WEKA version from WEKA installation client - ansible.builtin.set_fact: - weka_version: "{{ get_weka_client_version.stdout }}" - when: get_weka_version.status == 404 - - - name: Download user-defined WEKA version - ansible.builtin.shell: - cmd: weka version get {{ weka_version }} - register: result - changed_when: result.rc == 0 - - - name: Set user-defined WEKA version - ansible.builtin.shell: - cmd: weka version set {{ weka_version }} - register: result - changed_when: result.rc == 0 - - - name: Setup WEKA client - ansible.builtin.shell: - cmd: weka local setup weka - register: setup_1_result - changed_when: setup_1_result.rc == 0 - failed_when: false # ignore errors - - - name: Setup WEKA client (2nd try) - ansible.builtin.shell: - cmd: weka local setup weka - register: result - changed_when: result.rc == 0 - when: setup_1_result.rc != 0 - - - name: Prepare WEKA version - ansible.builtin.shell: - cmd: weka version prepare {{ weka_version }} - register: result - changed_when: result.rc == 0 - - - name: Stop WEKA client - ansible.builtin.shell: - cmd: weka local stop - async: 30 - poll: 10 - register: weka_stop - changed_when: weka_stop.get("rc") == 0 # when killed by async, rc is not defined - failed_when: false # ignore errors - - - name: Stop WEKA client (2nd try) - ansible.builtin.shell: - cmd: weka local stop - async: 30 - poll: 10 - register: result - changed_when: result.rc == 0 - failed_when: false # ignore errors - when: weka_stop.get("rc") != 0 - - - name: Remove WEKA containers - ansible.builtin.shell: - cmd: weka local rm -f --all - register: result - changed_when: result.rc == 0 - failed_when: false # ignore errors diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl deleted file mode 100644 index 19c6dc1fdc..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -# shellcheck disable=SC2034 -METADATA_BASE_URL="http://metadata.google.internal/computeMetadata/v1/instance" -# shellcheck disable=SC2034 -ATTR_URL="$${METADATA_BASE_URL}/attributes/weka-" -NET_URL="$${METADATA_BASE_URL}/network-interfaces" - -# shellcheck disable=SC1083 -WEKA_MODE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mode || echo -n udp) -# shellcheck disable=SC1083 -WEKA_DATA_INTERFACES=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}data_interfaces || exit 0) -# shellcheck disable=SC1083 -WEKA_MGMT_INTERFACE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mgmt_interface || echo -n 0) -# shellcheck disable=SC1083 -WEKA_OPTIONS=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}options || exit 0) - -WEKA_OPTIONS="$${WEKA_OPTIONS:+-o $WEKA_OPTIONS}" - -netmask_to_cidr () { - c=0 - # shellcheck disable=SC2086,SC1083 - x=0$( printf '%o' $${1//./ } ) - while [ "$x" -gt 0 ]; do - c=$(( c + x%2 )) - x=$(( x >> 1)) - done - echo $c ; -} - -# detect network interface naming scheme -if [[ -e /sys/class/net/eth0 ]] ; then - DEVICE_NAME="eth" - DEVICE_INDEX_BASE=0 -elif [[ -e /sys/class/net/ens4 ]] ; then - DEVICE_NAME="ens" - DEVICE_INDEX_BASE=4 -else - echo "Can't detect device names. Both /sys/class/net/eth0 and /sys/class/net/ens4 do not exists" - exit 1 -fi - -# ensure that /etc/hosts contains entry for hostname pointing to primary interface -NEW_IP=$(ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) | head -n 1 | sed -e 's/^.*inet \([0-9\.]\+\)\/.*$/\1/') -if [ -n "$NEW_IP" ] ; then - HOSTNAME=$(hostname) - sed -i -e "/$HOSTNAME/s/^[0-9\.]\+ $HOSTNAME/$NEW_IP $HOSTNAME/" /etc/hosts -else - echo "Failed to find primary interface address" - ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) - exit 1 -fi - -# shellcheck disable=SC2154 -echo "Mounting Weka ${server_ip}/${remote_mount} to ${local_mount}" -mkdir -p "${local_mount}" -service weka-agent start -if [[ $WEKA_MODE == "udp" ]] ; then - # shellcheck disable=SC2086,SC2154,SC2086 - mount -t wekafs ${mount_options} -o net=udp $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" - -elif [[ $WEKA_MODE == "dpdk" ]] ; then - declare -a DATA_INTERFACES - # split WEKA_DATA_INTERFACES by comma into array - # shellcheck disable=SC2034 - IFS=',' read -r -a DATA_INTERFACES <<< "$WEKA_DATA_INTERFACES" - - DATA_OPTIONS="" - # shellcheck disable=SC2066 - for interface in "$${DATA_INTERFACES[@]}" ; do - INTERFACE_IP=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/ip") - INTERFACE_MASK=$(netmask_to_cidr "$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/subnetmask")") - INTERFACE_GATEWAY=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/gateway") - - DATA_OPTIONS+="-o net=$DEVICE_NAME$((DEVICE_INDEX_BASE + interface))/$INTERFACE_IP/$INTERFACE_MASK/$INTERFACE_GATEWAY " - done - - # shellcheck disable=SC2086 - mount -t wekafs \ - ${mount_options} \ - -o mgmt_ip="$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$WEKA_MGMT_INTERFACE/ip")" \ - $DATA_OPTIONS $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" -else - echo "Unknown weka:mode metadata value: $${WEKA_MODE}. Allowed values: udp and dpdk" - exit 1 -fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl deleted file mode 100644 index 84587103a9..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Mount the WEKA file systems - hosts: localhost - become: true - vars: - local_mount: "${local_mount}" - remote_mount: "${remote_mount}" - server_ip: "${server_ip}" - service_name: "weka-mount-${replace(local_mount, "/", "_")}" - tasks: - - name: Create mount script - ansible.builtin.copy: - dest: "/etc/{{ service_name }}.sh" - mode: "0755" - content: | - ${indent(8, mount_weka_script)} - - - name: Create systemd service for weka mount - ansible.builtin.copy: - dest: "/etc/systemd/system/{{ service_name }}.service" - mode: "0644" - content: | - [Install] - WantedBy=multi-user.target - [Unit] - Description=Mount Weka {{ server_ip }}/{{ remote_mount }} at {{ local_mount }} - After=network-online.target - Wants=network-online.target - [Service] - RemainAfterExit=true - Type=oneshot - ExecStart=/bin/bash -c "/etc/{{ service_name }}.sh" - - - name: Enable and start weka mount service - ansible.builtin.systemd: - name: "{{ service_name }}" - daemon_reload: true - enabled: true - state: started diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/variables.tf deleted file mode 100644 index f07961d64c..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "local_mount" { - description = "The mount point where the contents of the device may be accessed after mounting." - type = string - default = "/mnt" -} - -variable "mount_options" { - description = "Mount options for filesystem shared by all clients." - type = string - default = "" - nullable = false -} - -variable "remote_mount" { - description = "Weka filesystem name." - type = string -} - -variable "server_ip" { - description = "Weka backend IP address used for bootstrapping." - type = string - default = "" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/versions.tf deleted file mode 100644 index 9e6af1fa7f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/file-system/weka-client/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 0.14.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb deleted file mode 100644 index f13726f691..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb +++ /dev/null @@ -1,125 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "project_id = \"${project_id}\"\n", - "dataset_id = \"${dataset_id}\"\n", - "table_id = \"${table_id}\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ONI1Xo0-KtAD", - "outputId": "fb9ca475-e4ec-4cd0-e0e6-14f409eefd7a" - }, - "outputs": [], - "source": [ - "from google.cloud import bigquery\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "\n", - "client = bigquery.Client(project=project_id)\n", - "\n", - "df = client.query(f'''\n", - "SELECT ticker, cast(price AS FLOAT64) AS price, CAST(OFFSET as INTEGER) AS offset, start_date, end_date, iteration\n", - "FROM `{project_id}.{dataset_id}.{table_id}`,\n", - "UNNEST(simulation_results) as NUMERIC with OFFSET\n", - "WHERE epoch_time IN\n", - " # Get the latest simulation runs for each Ticker Symbol\n", - "(SELECT MAX(epoch_time) FROM `{project_id}.{dataset_id}.{table_id}` GROUP BY ticker)\n", - "'''\n", - ").to_dataframe()\n", - "# Display the data\n", - "df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Define a function to plot the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "def plot_ticker(t,df):\n", - "\n", - " dtf = df[(df.ticker==t) &(df.offset == 250)].price.describe(include=[np.float64], percentiles=[.05, .01, .001])\n", - " cellText = []\n", - " for v in dtf.values:\n", - " cellText.append([v])\n", - " \n", - " pltf = df[df.ticker==t].pivot(index='offset', columns='iteration', values='price')\n", - " \n", - " fig = plt.figure(figsize=(10,5))\n", - " ax1 = fig.add_subplot(122)\n", - " pltf.plot(legend=False, ax=ax1, xlabel='Time(days)', ylabel='US$', title=f\"{ df[(df.ticker == t) & (df.offset == 0) & (df.iteration == 4)]}\")\n", - " ax2 = fig.add_subplot(121)\n", - " font_size=10\n", - " bbox=[0, 0, .5, 1]\n", - " ax2.axis('off')\n", - " mpl_table = ax2.table(cellText = cellText, rowLabels=dtf.index.values, bbox=bbox)\n", - " mpl_table.auto_set_font_size(False)\n", - " mpl_table.set_fontsize(font_size)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 808 - }, - "id": "jvBmb_KceX7z", - "outputId": "42a3ba9f-b68f-4c7b-d928-0fedeed9216c" - }, - "outputs": [], - "source": [ - "ticker_list = df.ticker.unique()\n", - "for t in ticker_list:\n", - " plot_ticker(t,df)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md deleted file mode 100644 index e54893a1bb..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md +++ /dev/null @@ -1,97 +0,0 @@ -## Description - -Copy files to a target GCS bucket. - -Primarily used for FSI - MonteCarlo Tutorial **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: -../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage -This copies the module files to the specified GCS bucket. It is expected that -the bucket will be mounted on the target VM. - -Some of the files are templates, and `main.tf` translates the files with the -passed variable values. This way the user does not have to change things like -pointing to the correct bigquery table or adding in the project_id. - -```yaml - - id: fsi_tutorial_files - source: community/modules/files/fsi-montecarlo-on-batch - use: [bq-dataset, bq-table, fsi_bucket, pubsub_topic] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 3.83 | -| [http](#requirement\_http) | ~> 3.0 | -| [random](#requirement\_random) | ~> 3.0 | -| [template](#requirement\_template) | ~> 2.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [http](#provider\_http) | ~> 3.0 | -| [random](#provider\_random) | ~> 3.0 | -| [template](#provider\_template) | ~> 2.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.get_iteration_sh](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.get_mc_reqs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.get_requirements](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.ipynb_obj_fsi](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.mc_obj_yaml](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.mc_run](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.run_batch_py](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [http_http.batch_py](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | -| [http_http.batch_requirements](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | -| [template_file.ipynb_fsi](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | -| [template_file.mc_run_py](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | -| [template_file.mc_run_yaml](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Bigquery dataset id | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to run project | `string` | n/a | yes | -| [table\_id](#input\_table\_id) | Bigquery table id | `string` | n/a | yes | -| [topic\_id](#input\_topic\_id) | Pubsub Topic Name | `string` | n/a | yes | -| [topic\_schema](#input\_topic\_schema) | Pubsub Topic schema | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh deleted file mode 100644 index 50aa865a31..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ticker=("GOOG" "AMZN" "MSFT" "NVDA" "META" "TSLA" "PEP" "COST") -echo "BI: $BATCH_TASK_INDEX" -echo "TI: ${ticker[$BATCH_TASK_INDEX]}" -python3 -m pip install -r /mnt/disks/fsi/mc_run_reqs.txt -python3 /mnt/disks/fsi/mc_run.py \ - --ticker "${ticker[$BATCH_TASK_INDEX]}" \ - --iterations 500 \ - --start_date 2022-01-01 diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf deleted file mode 100644 index 83dc7fe9cf..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - bucket = replace(var.gcs_bucket_path, "gs://", "") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -data "template_file" "mc_run_py" { - template = file("${path.module}/mc_run.tpl.py") - vars = { - project_id = var.project_id - topic_id = var.topic_id - topic_schema = var.topic_schema - dataset_id = var.dataset_id - table_id = var.table_id - } -} - -resource "google_storage_bucket_object" "mc_run" { - name = "mc_run.py" - content = data.template_file.mc_run_py.rendered - bucket = local.bucket -} - -data "template_file" "mc_run_yaml" { - template = file("${path.module}/mc_run.tpl.yaml") - vars = { - project_id = var.project_id - bucket_name = local.bucket - region = var.region - } -} - -resource "google_storage_bucket_object" "mc_obj_yaml" { - name = "mc_run.yaml" - content = data.template_file.mc_run_yaml.rendered - bucket = local.bucket -} - -data "template_file" "ipynb_fsi" { - template = file("${path.module}/FSI_MonteCarlo.ipynb") - vars = { - project_id = var.project_id - dataset_id = var.dataset_id - table_id = var.table_id - } -} -resource "google_storage_bucket_object" "ipynb_obj_fsi" { - name = "FSI_MonteCarlo.ipynb" - content = data.template_file.ipynb_fsi.rendered - bucket = local.bucket -} - -data "http" "batch_py" { - url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/batch.py" -} - -resource "google_storage_bucket_object" "run_batch_py" { - name = "batch.py" - content = data.http.batch_py.response_body - bucket = local.bucket -} - -data "http" "batch_requirements" { - url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/requirements.txt" -} - -resource "google_storage_bucket_object" "get_requirements" { - name = "requirements.txt" - content = data.http.batch_requirements.response_body - bucket = local.bucket -} - -resource "google_storage_bucket_object" "get_iteration_sh" { - name = "iteration.sh" - content = file("${path.module}/iteration.sh") - bucket = local.bucket -} - -resource "google_storage_bucket_object" "get_mc_reqs" { - name = "mc_run_reqs.txt" - content = file("${path.module}/mc_run_reqs.txt") - bucket = local.bucket -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py deleted file mode 100644 index 4e0a64e363..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Run MC simulation for VaR portfolio risk -""" - -import avro.schema -import io -import google.auth -import numpy -import time -import yfinance as yf - -from absl import app -from absl import flags -from avro.io import DatumWriter, BinaryEncoder, BinaryDecoder, DatumReader -from datetime import datetime -from datetime import timedelta -from google.cloud import pubsub_v1, bigquery -from google.cloud.pubsub import SchemaServiceClient - -PROJECT_ID = '${project_id}' -INCOMING_TOPIC_ID = '${topic_id}' -INCOMING_TOPIC_SCHEMA = '${topic_schema}' -DATASET_ID = '${dataset_id}' -TABLE_ID = '${table_id}' - - -FLAGS = flags.FLAGS - -flags.DEFINE_string("ticker", 'GOOG', "Nasdaq Stock Ticker to run, default GOOG") -flags.DEFINE_string("start_date", '2022-01-01' , "Start data for data query, default 2022-01-01") -flags.DEFINE_integer("calendar_days", 365 , "How many calendar days to include in the calculation") -flags.DEFINE_integer("epoch_time", f'{int(time.time())}' , "Epoch time, number of seconds since January 1st, 1970 at 00:00:00 UTC.") -flags.DEFINE_integer("iterations", 100 , "Number of iterations to run.") -flags.DEFINE_boolean("print_raw", False, "Dump raw data.") - -class VaRSimulator: - - def __init__(self): - pass - - def get_data(self): - self.get_historical_data_yahoo() - - def get_historical_data_yahoo(self): - - # get historical market data: https://pypi.org/project/yfinance/ - - self.raw_data = yf.Ticker(self.ticker).history(start=self.start_date, end=self.end_date ) - self.data = self.raw_data.Close - - def print_raw(self): - print(self.get_stats()) - print(type(self.raw_data)) - print(self.raw_data) - - def get_stats(self): - close = self.data - self.first = close[0] - self.last = close[-1] - self.trading_days = len(close) - self.cagr = (self.last / self.first) ** (365.0/self.calendar_days) -1.0 - self.volatility = self.data.pct_change().std() - return(self.first, self.last, self.trading_days, self.cagr, self.volatility) - - def run_simulation(self): - - returns = numpy.random.normal(self.cagr/self.trading_days, self.volatility, self.trading_days) + 1 - returns = numpy.insert(returns,0,1.0) - self.simulation_results = self.last * returns.cumprod() - return(self.simulation_results) - - def create_object(self): - self.object = { - "ticker": self.ticker, - "epoch_time": self.epoch_time, - "iteration": self.iteration, - "start_date": self.start_date, - "end_date": self.end_date, - "simulation_results": list(map(lambda x: {"price":x}, self.simulation_results)) - } - return(self.object) - - -class PubsubToBiquery: - - def __init__(self): - - the_time = int(time.time()) - - self.project_id = PROJECT_ID - - self.publisher_client = pubsub_v1.PublisherClient() - self.topic_path = self.publisher_client.topic_path(self.project_id, INCOMING_TOPIC_ID) - - self.schema_client = SchemaServiceClient() - self.schema_path = self.schema_client.schema_path(self.project_id, INCOMING_TOPIC_SCHEMA) - - pubsub_schema = self.schema_client.get_schema(request={"name": self.schema_path}) - avro_schema = avro.schema.parse(pubsub_schema.definition) - - self.writer = DatumWriter(avro_schema) - - - def publish_record(self,record): - - byte_stream = io.BytesIO() - encoder = BinaryEncoder(byte_stream) - self.writer.write(record, encoder) - data = byte_stream.getvalue() - byte_stream.flush() - future = self.publisher_client.publish(self.topic_path, data) - if(FLAGS.print_raw): - print(f"Published message ID: {future.result()}") - - -def main(argv): - - vr = VaRSimulator() - pbbq = PubsubToBiquery() - - vr.ticker =FLAGS.ticker - vr.start_date =FLAGS.start_date - vr.end_date =f'{(datetime.strptime(FLAGS.start_date,"%Y-%m-%d") + timedelta(days = FLAGS.calendar_days)).date()}' - vr.calendar_days = FLAGS.calendar_days - vr.epoch_time = FLAGS.epoch_time - vr.iteration = 1 - - vr.get_data() - vr.get_stats() - - for i in range(FLAGS.iterations): - vr.iteration = i - vr.run_simulation() - pbbq.publish_record(vr.create_object()) - - if(FLAGS.print_raw): - vr.print_raw() - - -if __name__ == "__main__": - """ This is executed when run from the command line """ - app.run(main) diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml deleted file mode 100644 index 7f7de4840b..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -project_id: "${project_id}" -region: "${region}" - -job_prefix: 'fsi-' -machine_type: "n2-standard-2" -volumes: -- {bucket_name: "${bucket_name}", gcs_path: "/mnt/disks/fsi"} - -container: - image_uri: "python" - entry_point: "/bin/bash" - commands: ["/mnt/disks/fsi/iteration.sh", "$BATCH_TASK_INDEX"] - -task_count: 8 #optional -parallelism: 4 #optional -task_count_per_node: 2 #optional -cpu_milli: 1000 #optional -memory_mib: 102400 #optional - - -labels: - env: "monte" - type: "carlo" diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt deleted file mode 100644 index 105ed70ad2..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt +++ /dev/null @@ -1,9 +0,0 @@ -absl-py -avro -google-auth -google-cloud -google-cloud-batch -google-cloud-pubsub -google-cloud-bigquery -yfinance -PyYAML diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml deleted file mode 100644 index 268c8faa9a..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [storage.googleapis.com] diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf deleted file mode 100644 index eddf3c9478..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which GCS bucket will be created." - type = string -} - -variable "gcs_bucket_path" { - description = "Bucket name" - type = string - default = null -} - -variable "topic_id" { - description = "Pubsub Topic Name" - type = string -} - -variable "topic_schema" { - description = "Pubsub Topic schema" - type = string -} - -variable "dataset_id" { - description = "Bigquery dataset id" - type = string -} - -variable "table_id" { - description = "Bigquery table id" - type = string -} - -variable "region" { - description = "Region to run project" - type = string -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf deleted file mode 100644 index 86dcb4dc52..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - http = { - source = "hashicorp/http" - version = "~> 3.0" - } - template = { - source = "hashicorp/template" - version = "~> 2.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md deleted file mode 100644 index ae8462d763..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md +++ /dev/null @@ -1,100 +0,0 @@ -# Module: Slurm Instance - - - -- [Module: Slurm Instance](#module-slurm-instance) - - [Overview](#overview) - - [Module API](#module-api) - - - -## Overview - -This module creates a [compute instance](../../../../docs/glossary.md#vm) from -[instance template](../../../../docs/glossary.md#instance-template) for a -[Slurm cluster](../slurm_cluster/README.md). - -> **NOTE:** This module is only intended to be used by Slurm modules. For -> general usage, please consider using: -> -> - [terraform-google-modules/vm/google//modules/compute_instance](https://registry.terraform.io/modules/terraform-google-modules/vm/google/latest/submodules/compute_instance). -> **WARNING:** The source image is not modified. Make sure to use a compatible -> source image. - -## Module API - -For the terraform module API reference, please see -[README_TF.md](./README_TF.md). - - -Copyright (C) SchedMD LLC. -Copyright 2018 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.0 | -| [google](#requirement\_google) | >= 3.43 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.43 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_from_template.slurm_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | -| [null_resource.replace_trigger](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_instance_template.base](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | -| [hostname](#input\_hostname) | Hostname of instances | `string` | n/a | yes | -| [instance\_template](#input\_instance\_template) | Instance template self\_link used to create compute instances | `string` | n/a | yes | -| [network](#input\_network) | Network to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | -| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | -| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | -| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | -| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances | `list(string)` | `[]` | no | -| [subnetwork](#input\_subnetwork) | Subnet to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The project that subnetwork belongs to | `string` | `null` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [available\_zones](#output\_available\_zones) | List of available zones in region | -| [instances\_details](#output\_instances\_details) | List of all details for compute instances | -| [instances\_self\_links](#output\_instances\_self\_links) | List of self-links for compute instances | -| [names](#output\_names) | List of available zones in region | -| [slurm\_instances](#output\_slurm\_instances) | List of all resource objects for compute instances | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf deleted file mode 100644 index 2af9008a0e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########## -# LOCALS # -########## - -locals { - num_instances = length(var.static_ips) == 0 ? var.num_instances : length(var.static_ips) - - # local.static_ips is the same as var.static_ips with a dummy element appended - # at the end of the list to work around "list does not have any elements so cannot - # determine type" error when var.static_ips is empty - static_ips = concat(var.static_ips, ["NOT_AN_IP"]) - - network_interfaces = [for index in range(local.num_instances) : - concat([ - { - access_config = var.access_config - alias_ip_range = [] - ipv6_access_config = [] - network = var.network - network_ip = length(var.static_ips) == 0 ? "" : element(local.static_ips, index) - nic_type = null - queue_count = null - stack_type = null - subnetwork = var.subnetwork - subnetwork_project = var.subnetwork_project - } - ], - var.additional_networks - ) - ] -} - -################ -# DATA SOURCES # -################ - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -data "google_compute_instance_template" "base" { - project = var.project_id - name = var.instance_template -} - -############# -# INSTANCES # -############# -resource "null_resource" "replace_trigger" { - triggers = { - trigger = var.replace_trigger - } -} - -# TODO: `internal/slurm-gcp/login` is ONLY user of `internal/slurm-gcp/instance` -# Remove this module, add functionality (+ prune generality) to the login module directly. -resource "google_compute_instance_from_template" "slurm_instance" { - count = local.num_instances - name = format("%s-%s", var.hostname, format("%03d", count.index + 1)) - project = var.project_id - zone = var.zone == null ? data.google_compute_zones.available.names[count.index % length(data.google_compute_zones.available.names)] : var.zone - - allow_stopping_for_update = true - - dynamic "network_interface" { - for_each = local.network_interfaces[count.index] - iterator = nic - content { - dynamic "access_config" { - for_each = nic.value.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = nic.value.alias_ip_range - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - dynamic "ipv6_access_config" { - for_each = nic.value.ipv6_access_config - iterator = access_config - content { - network_tier = access_config.value.network_tier - } - } - network = nic.value.network - network_ip = nic.value.network_ip - nic_type = nic.value.nic_type - queue_count = nic.value.queue_count - subnetwork = nic.value.subnetwork - subnetwork_project = nic.value.subnetwork_project - } - } - - source_instance_template = data.google_compute_instance_template.base.self_link - # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 - # we have to explicitly override instance labels instead of inheriting them from template. - labels = data.google_compute_instance_template.base.labels - - - lifecycle { - replace_triggered_by = [null_resource.replace_trigger.id] - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf deleted file mode 100644 index 4eba78a7e8..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "slurm_instances" { - description = "List of all resource objects for compute instances" - value = google_compute_instance_from_template.slurm_instance -} - -output "instances_self_links" { - description = "List of self-links for compute instances" - value = google_compute_instance_from_template.slurm_instance[*].self_link -} - -output "instances_details" { - description = "List of all details for compute instances" - value = google_compute_instance_from_template.slurm_instance[*] -} - -output "available_zones" { - description = "List of available zones in region" - value = data.google_compute_zones.available.names -} - -output "names" { - description = "List of available zones in region" - value = google_compute_instance_from_template.slurm_instance[*].name -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf deleted file mode 100644 index 11111a2c05..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "The GCP project ID" - default = null -} - -variable "network" { - description = "Network to deploy to. Only one of network or subnetwork should be specified." - type = string - default = "" -} - -variable "subnetwork" { - description = "Subnet to deploy to. Only one of network or subnetwork should be specified." - type = string - default = "" -} - -variable "subnetwork_project" { - description = "The project that subnetwork belongs to" - type = string - default = null -} - -variable "hostname" { - description = "Hostname of instances" - type = string -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })) - nullable = false -} - -variable "static_ips" { - description = "List of static IPs for VM instances" - type = list(string) - default = [] -} - -variable "access_config" { - description = "Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet." - type = list(object({ - nat_ip = string - network_tier = string - })) - default = [] -} - -variable "num_instances" { - description = "Number of instances to create. This value is ignored if static_ips is provided." - type = number - default = 1 -} - -variable "instance_template" { - description = "Instance template self_link used to create compute instances" - type = string -} - -variable "region" { - description = "Region where the instances should be created." - type = string - default = null -} - -variable "zone" { - description = "Zone where the instances should be created. If not specified, instances will be spread across available zones in the region." - type = string - default = null -} - -######### -# SLURM # -######### - -variable "replace_trigger" { - description = "Trigger value to replace the instances." - type = string - default = "" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf deleted file mode 100644 index a3e84c09bf..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = "~> 1.0" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.43" - } - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md deleted file mode 100644 index 87394bef6a..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md +++ /dev/null @@ -1,87 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.0 | -| [local](#requirement\_local) | ~> 2.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | ~> 2.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance\_template](#module\_instance\_template) | ../internal_instance_template | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.startup](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    disk_type                  = optional(string)
    disk_size_gb               = optional(number)
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `virtio_enabled` setting will only enable VirtioNet and will not enable TIER\_1.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard. | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See
https://cloud.google.com/compute/docs/gpus more details.
- type : the GPU type
- count : number of GPUs | 
object({
    type  = string
    count = number
  })
| `null` | no | -| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"n1-standard-1"` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Prefix for template resource. | `string` | `"default"` | no | -| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network
attribute for Legacy or Auto subnetted networks and subnetwork for custom
subnetted networks. | `string` | `null` | no | -| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | -| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the instances. See
'main.tf:local.service\_account' for the default. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming. | `string` | n/a | yes | -| [slurm\_instance\_role](#input\_slurm\_instance\_role) | Slurm instance type. Must be one of: controller; login; compute; or null. | `string` | n/a | yes | -| [source\_image](#input\_source\_image) | Source disk image. | `string` | `""` | no | -| [source\_image\_family](#input\_source\_image\_family) | Source image family. | `string` | `""` | no | -| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. If it is not provided, the provider project is used. | `string` | `""` | no | -| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must
exist in the same region this instance will be created in. Either network or
subnetwork must be provided. | `string` | `null` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [termination\_action](#input\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_template](#output\_instance\_template) | Instance template details | -| [labels](#output\_labels) | Labels attached to the instance template | -| [name](#output\_name) | Name of instance template | -| [self\_link](#output\_self\_link) | Self\_link of instance template | -| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | -| [tags](#output\_tags) | Tags that will be associated with instance(s) | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted deleted file mode 100644 index 2edaa942d2..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -SLURM_DIR=/slurm -FLAGFILE=$SLURM_DIR/slurm_configured_do_not_remove -SCRIPTS_DIR=$SLURM_DIR/scripts -if [[ -z "$HOME" ]]; then - # google-startup-scripts.service lacks environment variables - HOME="$(getent passwd "$(whoami)" | cut -d: -f6)" -fi - -# Temporary workaround for transition period when some of older images -# don't have "baked in" python yet. -# TODO: Remove -SLURM_PY="/slurm/python/venv/bin/python3.13" -SYSTEM_PY="/usr/bin/python3" -if [[ ! -e "$SLURM_PY" ]]; then - echo "Symlink $SLURM_PY does not exist. Creating symlink to $SYSTEM_PY" - mkdir -p /slurm/python/venv/bin - ln -s "$SYSTEM_PY" "$SLURM_PY" -fi - -METADATA_SERVER="metadata.google.internal" -URL="http://$METADATA_SERVER/computeMetadata/v1" -CURL="curl -sS --fail --header Metadata-Flavor:Google" - -PING_METADATA="ping -q -w1 -c1 $METADATA_SERVER" -echo "INFO: $PING_METADATA" -for i in $(seq 10); do - [ $i -gt 1 ] && sleep 5; - $PING_METADATA > /dev/null && s=0 && break || s=$?; - echo "ERROR: Failed to contact metadata server, will retry" -done -if [ $s -ne 0 ]; then - echo "ERROR: Unable to contact metadata server, aborting" - wall -n '*** Slurm setup failed in the startup script! see `journalctl -u google-startup-scripts` ***' - exit 1 -else - echo "INFO: Successfully contacted metadata server" -fi - -PING_GOOGLE="ping -q -w1 -c1 8.8.8.8" -echo "INFO: $PING_GOOGLE" -for i in $(seq 5); do - [ $i -gt 1 ] && sleep 2; - $PING_GOOGLE > /dev/null && s=0 && break || s=$?; - echo "failed to ping Google DNS, will retry" -done -if [ $s -ne 0 ]; then - echo "WARNING: No internet access detected" -else - echo "INFO: Internet access detected" -fi - -mkdir -p $SCRIPTS_DIR -UNIVERSE_DOMAIN="$($CURL $URL/instance/attributes/universe_domain)" -BUCKET="$($CURL $URL/instance/attributes/slurm_bucket_path)" -if [[ -z $BUCKET ]]; then - echo "ERROR: No bucket path detected." - exit 1 -fi - -SCRIPTS_ZIP="$HOME/slurm-gcp-scripts.zip" -export CLOUDSDK_CORE_UNIVERSE_DOMAIN="$UNIVERSE_DOMAIN" - -INSTANCE_ROLE="$($CURL $URL/instance/attributes/slurm_instance_role)" - -if [ "$INSTANCE_ROLE" == "controller" ]; then - DEVEL_ZIP="slurm-gcp-devel-controller.zip" -else - DEVEL_ZIP="slurm-gcp-devel.zip" -fi -until gcloud storage cp "$BUCKET/$DEVEL_ZIP" "$SCRIPTS_ZIP"; do - echo "WARN: Could not download SlurmGCP scripts, retrying in 5 seconds." - # Remove marker used to determine if gcloud is being used in a GCE VM. - # This can get mistakenly set to False in some cases. - rm -f /root/.config/gcloud/gce - sleep 5 -done -unzip -o "$SCRIPTS_ZIP" -d "$SCRIPTS_DIR" -rm -rf "$SCRIPTS_ZIP" - -#temporary hack to not make the script fail on TPU vm -chown slurm:slurm -R "$SCRIPTS_DIR" || true -chmod 700 -R "$SCRIPTS_DIR" - - -if [ -f $FLAGFILE ]; then - echo "WARNING: Slurm was previously configured, quitting" - exit 0 -fi -touch $FLAGFILE - -function tpu_setup { - #allow the following command to fail, as this attribute does not exist for regular nodes - docker_image=$($CURL $URL/instance/attributes/slurm_docker_image 2> /dev/null || true) - if [ -z $docker_image ]; then #Not a tpu node, do not do anything - return - fi - if [ "$OS_ENV" == "slurm_container" ]; then #Already inside the slurm container, we should continue starting - return - fi - - #given a input_string like "WORKER_0:Joseph;WORKER_1:richard;WORKER_2:edward;WORKER_3:john" and a number 1, this function will print richard - parse_metadata() { - local number=$1 - local input_string=$2 - local word=$(echo "$input_string" | awk -v n="$number" -F ':|;' '{ for (i = 1; i <= NF; i+=2) if ($(i) == "WORKER_"n) print $(i+1) }') - echo "$word" - } - - input_string=$($CURL $URL/instance/attributes/slurm_names) - worker_id=$($CURL $URL/instance/attributes/tpu-env | awk '/WORKER_ID/ {print $2}' | tr -d \') - real_name=$(parse_metadata $worker_id $input_string) - - #Prepare to docker pull with gcloud - mkdir -p /root/.docker - cat << EOF > /root/.docker/config.json -{ - "credHelpers": { - "gcr.io": "gcloud", - "us-docker.pkg.dev": "gcloud" - } -} -EOF - #cgroup detection - CGV=1 - CGROUP_FLAGS="-v /sys/fs/cgroup:/sys/fs/cgroup:rw" - if [ -f /sys/fs/cgroup/cgroup.controllers ]; then #CGV2 - CGV=2 - fi - if [ $CGV == 2 ]; then - CGROUP_FLAGS="--cgroup-parent=docker.slice --cgroupns=private --tmpfs /run --tmpfs /run/lock --tmpfs /tmp" - if [ ! -f /etc/systemd/system/docker.slice ]; then #In case that there is no slice prepared for hosting the containers create it - printf "[Unit]\nDescription=docker slice\nBefore=slices.target\n[Slice]\nCPUAccounting=true\nMemoryAccounting=true" > /etc/systemd/system/docker.slice - systemctl start docker.slice - fi - fi - #for the moment always use --privileged, as systemd might not work properly otherwise - TPU_FLAGS="--privileged" - # TPU_FLAGS="--cap-add SYS_RESOURCE --device /dev/accel0 --device /dev/accel1 --device /dev/accel2 --device /dev/accel3" - # if [ $CGV == 2 ]; then #In case that we are in CGV2 for systemd to work correctly for the moment we go with privileged - # TPU_FLAGS="--privileged" - # fi - - docker run -d $CGROUP_FLAGS $TPU_FLAGS --net=host --name=slurmd --hostname=$real_name --entrypoint=/usr/bin/systemd --restart unless-stopped $docker_image - exit 0 -} - -tpu_setup #will do nothing for normal nodes or the container spawned inside TPU - -echo "INFO: Running python cluster setup script" -SETUP_SCRIPT_FILE=$SCRIPTS_DIR/setup.py -chmod +x $SETUP_SCRIPT_FILE -exec $SETUP_SCRIPT_FILE diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf deleted file mode 100644 index c91bbc4fd1..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "instance_validation" { - source = "../../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -########## -# LOCALS # -########## - -locals { - additional_disks = [ - for disk in var.additional_disks : { - disk_name = disk.disk_name - device_name = disk.device_name - auto_delete = disk.auto_delete - source = disk.source - boot = disk.boot - disk_size_gb = disk.disk_size_gb - disk_type = disk.disk_type - disk_labels = merge( - disk.disk_labels, - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) - disk_resource_manager_tags = disk.disk_resource_manager_tags - } - ] - - service_account = { - email = try(var.service_account.email, null) - scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) - } - - source_image_family = ( - var.source_image_family != "" && var.source_image_family != null - ? var.source_image_family - : "slurm-gcp-6-11-hpc-rocky-linux-8" - ) - source_image_project = ( - var.source_image_project != "" && var.source_image_project != null - ? var.source_image_project - : "projects/schedmd-slurm-public/global/images/family" - ) - - source_image = ( - var.source_image != null - ? var.source_image - : "" - ) - - - name_prefix = "${var.slurm_cluster_name}-${var.slurm_instance_role}-${var.name_prefix}" - - total_egress_bandwidth_tier = var.bandwidth_tier == "tier_1_enabled" ? "TIER_1" : "DEFAULT" - - nic_type_map = { - platform_default = null - virtio_enabled = "VIRTIO_NET" - gvnic_enabled = "GVNIC" - tier_1_enabled = "GVNIC" - } - nic_type = lookup(local.nic_type_map, var.bandwidth_tier, null) - - labels = merge(var.labels, - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) -} - -######## -# DATA # -######## - -data "local_file" "startup" { - filename = "${path.module}/files/startup_sh_unlinted" -} - -############ -# TEMPLATE # -############ - -module "instance_template" { - source = "../internal_instance_template" - - project_id = var.project_id - - # Network - can_ip_forward = var.can_ip_forward - network_ip = var.network_ip - network = var.network - nic_type = local.nic_type - region = var.region - subnetwork_project = var.subnetwork_project - subnetwork = var.subnetwork - tags = var.tags - total_egress_bandwidth_tier = local.total_egress_bandwidth_tier - additional_networks = var.additional_networks - access_config = var.access_config - - # Instance - machine_type = var.machine_type - min_cpu_platform = var.min_cpu_platform - name_prefix = local.name_prefix - gpu = var.gpu - service_account = local.service_account - shielded_instance_config = var.shielded_instance_config - advanced_machine_features = var.advanced_machine_features - enable_confidential_vm = var.enable_confidential_vm - enable_shielded_vm = var.enable_shielded_vm - preemptible = var.preemptible - spot = var.spot - on_host_maintenance = var.on_host_maintenance - labels = local.labels - instance_termination_action = var.termination_action - resource_manager_tags = var.resource_manager_tags - - # Metadata - startup_script = coalesce(var.internal_startup_script, data.local_file.startup.content) - metadata = merge( - var.metadata, - { - enable-oslogin = upper(var.enable_oslogin) - slurm_bucket_path = var.slurm_bucket_path - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) - - # Image - source_image_project = local.source_image_project - source_image_family = local.source_image_family - source_image = local.source_image - - # Disk - disk_type = var.disk_type - disk_size_gb = var.disk_size_gb - auto_delete = var.disk_auto_delete - disk_labels = merge( - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - var.disk_labels, - ) - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - - max_run_duration = var.max_run_duration - provisioning_model = var.provisioning_model - reservation_affinity = var.reservation_affinity -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf deleted file mode 100644 index 65da41052e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "instance_template" { - description = "Instance template details" - value = module.instance_template -} - -output "self_link" { - description = "Self_link of instance template" - value = module.instance_template.self_link -} - -output "name" { - description = "Name of instance template" - value = module.instance_template.name -} - -output "tags" { - description = "Tags that will be associated with instance(s)" - value = module.instance_template.tags -} - -output "service_account" { - description = "Service account object, includes email and scopes." - value = module.instance_template.service_account -} - -output "labels" { - description = "Labels attached to the instance template" - value = local.labels -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf deleted file mode 100644 index 35dd9c376f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf +++ /dev/null @@ -1,431 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -########### -# GENERAL # -########### - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "on_host_maintenance" { - type = string - description = "Instance availability Policy" - default = "MIGRATE" -} - -variable "labels" { - type = map(string) - description = "Labels, provided as a map" - default = {} -} - -variable "enable_oslogin" { - type = bool - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >=0.13.0 | -| [google](#requirement\_google) | >= 3.88 | -| [google-beta](#requirement\_google-beta) | >= 6.13.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.88 | -| [google-beta](#provider\_google-beta) | >= 6.13.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance_template.tpl](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_template) | resource | -| [google_project.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    auto_delete                = bool
    boot                       = bool
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = map(string)
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | -| [alias\_ip\_range](#input\_alias\_ip\_range) | An array of alias IP ranges for this network interface. Can only be specified for network interfaces on subnet-mode networks.
ip\_cidr\_range: The IP CIDR range represented by this alias IP range. This IP CIDR range must belong to the specified subnetwork and cannot contain IP addresses reserved by system or used by other network interfaces. At the time of writing only a netmask (e.g. /24) may be supplied, with a CIDR format resulting in an API error.
subnetwork\_range\_name: The subnetwork secondary range name specifying the secondary range from which to allocate the IP CIDR range for this alias IP range. If left unspecified, the primary range of the subnetwork will be used. | 
object({
    ip_cidr_range         = string
    subnetwork_range_name = string
  })
| `null` | no | -| [auto\_delete](#input\_auto\_delete) | Whether or not the boot disk should be auto-deleted | `string` | `"true"` | no | -| [automatic\_restart](#input\_automatic\_restart) | (Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user). | `bool` | `true` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example | `string` | `"false"` | no | -| [disk\_encryption\_key](#input\_disk\_encryption\_key) | The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance | `string` | `null` | no | -| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `string` | `"100"` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Whether to enable the Confidential VM configuration on the instance. Note that the instance image must support Confidential VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Whether to enable the Shielded VM configuration on the instance. Note that the instance image must support Shielded VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | -| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See https://cloud.google.com/compute/docs/gpus more details | 
object({
    type  = string
    count = number
  })
| `null` | no | -| [instance\_termination\_action](#input\_instance\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | -| [ipv6\_access\_config](#input\_ipv6\_access\_config) | IPv6 access configurations. Currently a max of 1 IPv6 access configuration is supported. If not specified, the instance will have no external IPv6 Internet access. | 
list(object({
    network_tier = string
  }))
| `[]` | no | -| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create, e.g. n1-standard-1 | `string` | `"n1-standard-1"` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Name prefix for the instance template | `string` | n/a | yes | -| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks. | `string` | `""` | no | -| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | -| [nic\_type](#input\_nic\_type) | The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO\_NET. | `string` | `null` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | -| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | -| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#service_account. | 
object({
    email  = optional(string)
    scopes = set(string)
  })
| n/a | yes | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Not used unless enable\_shielded\_vm is true. Shielded VM configuration for the instance. | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [source\_image](#input\_source\_image) | Source disk image. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `""` | no | -| [source\_image\_family](#input\_source\_image\_family) | Source image family. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `"centos-7"` | no | -| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. The default project contains CentOS images. | `string` | `"centos-cloud"` | no | -| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | -| [stack\_type](#input\_stack\_type) | The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4\_ONLY. | `string` | `null` | no | -| [startup\_script](#input\_startup\_script) | User startup script to run when instances spin up | `string` | `""` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided. | `string` | `""` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [total\_egress\_bandwidth\_tier](#input\_total\_egress\_bandwidth\_tier) | Network bandwidth tier. Note: machine\_type must be a supported type. Values are 'TIER\_1' or 'DEFAULT'.
See https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration for details. | `string` | `"DEFAULT"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [name](#output\_name) | Name of instance template | -| [self\_link](#output\_self\_link) | Self-link of instance template | -| [service\_account](#output\_service\_account) | value | -| [tags](#output\_tags) | Tags that will be associated with instance(s) | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf deleted file mode 100644 index f8d2813ece..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "instance_validation" { - source = "../../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -######### -# Locals -######### - -locals { - source_image = var.source_image != "" ? var.source_image : "centos-7-v20201112" - source_image_family = var.source_image_family != "" ? var.source_image_family : "centos-7" - source_image_project = var.source_image_project != "" ? var.source_image_project : "centos-cloud" - - boot_disk = [ - { - source_image = var.source_image != "" ? format("${local.source_image_project}/${local.source_image}") : format("${local.source_image_project}/${local.source_image_family}") - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_labels = var.disk_labels - auto_delete = var.auto_delete - disk_resource_manager_tags = var.disk_resource_manager_tags - boot = "true" - }, - ] - - all_disks = concat(local.boot_disk, var.additional_disks) - - # NOTE: Even if all the shielded_instance_config or confidential_instance_config - # values are false, if the config block exists and an unsupported image is chosen, - # the apply will fail so we use a single-value array with the default value to - # initialize the block only if it is enabled. - shielded_vm_configs = var.enable_shielded_vm ? [true] : [] - - gpu_enabled = var.gpu != null - alias_ip_range_enabled = var.alias_ip_range != null - preemptible = var.preemptible || var.spot - on_host_maintenance = ( - local.preemptible || var.enable_confidential_vm || local.gpu_enabled - ? "TERMINATE" - : var.on_host_maintenance - ) - automatic_restart = ( - # must be false when preemptible is true - local.preemptible ? false : var.automatic_restart - ) - - nic_type = var.total_egress_bandwidth_tier == "TIER_1" ? "GVNIC" : var.nic_type - - - provisioning_model = coalesce(var.provisioning_model, local.preemptible ? "SPOT" : "STANDARD") -} - -data "google_project" "this" { - project_id = var.project_id -} - -#################### -# Instance Template -#################### -resource "google_compute_instance_template" "tpl" { - provider = google-beta - name_prefix = "${var.name_prefix}-" - project = var.project_id - machine_type = var.machine_type - labels = var.labels - metadata = var.metadata - tags = var.tags - can_ip_forward = var.can_ip_forward - metadata_startup_script = var.startup_script - region = var.region - min_cpu_platform = var.min_cpu_platform - resource_manager_tags = var.resource_manager_tags - - service_account { - email = coalesce(var.service_account.email, "${data.google_project.this.number}-compute@developer.gserviceaccount.com") - scopes = lookup(var.service_account, "scopes", null) - } - - dynamic "disk" { - for_each = local.all_disks - content { - auto_delete = lookup(disk.value, "auto_delete", null) - boot = lookup(disk.value, "boot", null) - device_name = lookup(disk.value, "device_name", null) - disk_name = lookup(disk.value, "disk_name", null) - disk_size_gb = lookup(disk.value, "disk_size_gb", lookup(disk.value, "disk_type", null) == "local-ssd" ? "375" : null) - disk_type = lookup(disk.value, "disk_type", null) - interface = lookup(disk.value, "interface", lookup(disk.value, "disk_type", null) == "local-ssd" ? "NVME" : null) - mode = lookup(disk.value, "mode", null) - source = lookup(disk.value, "source", null) - source_image = lookup(disk.value, "source_image", null) - type = lookup(disk.value, "disk_type", null) == "local-ssd" ? "SCRATCH" : "PERSISTENT" - labels = (lookup(disk.value, "source", null) != null || lookup(disk.value, "disk_type", null) == "local-ssd") ? null : lookup(disk.value, "disk_labels", null) - resource_manager_tags = lookup(disk.value, "disk_resource_manager_tags", {}) - - dynamic "disk_encryption_key" { - for_each = compact([var.disk_encryption_key == null ? null : 1]) - content { - kms_key_self_link = var.disk_encryption_key - } - } - } - } - - network_interface { - network = var.network - subnetwork = var.subnetwork - subnetwork_project = var.subnetwork_project - network_ip = try(coalesce(var.network_ip), null) - nic_type = local.nic_type - stack_type = var.stack_type - dynamic "access_config" { - for_each = var.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = var.ipv6_access_config - content { - network_tier = ipv6_access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = local.alias_ip_range_enabled ? [var.alias_ip_range] : [] - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - } - - dynamic "network_interface" { - for_each = var.additional_networks - content { - network = network_interface.value.network - subnetwork = network_interface.value.subnetwork - subnetwork_project = network_interface.value.subnetwork_project - network_ip = try(coalesce(network_interface.value.network_ip), null) - nic_type = try(coalesce(network_interface.value.nic_type), null) - dynamic "access_config" { - for_each = network_interface.value.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = network_interface.value.ipv6_access_config - content { - network_tier = ipv6_access_config.value.network_tier - } - } - } - } - - network_performance_config { - total_egress_bandwidth_tier = coalesce(var.total_egress_bandwidth_tier, "DEFAULT") - } - - lifecycle { - create_before_destroy = "true" - } - - scheduling { - preemptible = local.preemptible - provisioning_model = local.provisioning_model - automatic_restart = local.automatic_restart - on_host_maintenance = local.on_host_maintenance - instance_termination_action = var.instance_termination_action - - dynamic "max_run_duration" { - for_each = var.max_run_duration != null ? [var.max_run_duration] : [] - content { - seconds = max_run_duration.value - } - } - } - - dynamic "reservation_affinity" { - for_each = var.reservation_affinity != null ? [var.reservation_affinity] : [] - content { - type = reservation_affinity.value.type - } - } - - advanced_machine_features { - enable_nested_virtualization = var.advanced_machine_features.enable_nested_virtualization - threads_per_core = var.advanced_machine_features.threads_per_core - turbo_mode = var.advanced_machine_features.turbo_mode - visible_core_count = var.advanced_machine_features.visible_core_count - performance_monitoring_unit = var.advanced_machine_features.performance_monitoring_unit - enable_uefi_networking = var.advanced_machine_features.enable_uefi_networking - } - - dynamic "shielded_instance_config" { - for_each = local.shielded_vm_configs - content { - enable_secure_boot = lookup(var.shielded_instance_config, "enable_secure_boot", shielded_instance_config.value) - enable_vtpm = lookup(var.shielded_instance_config, "enable_vtpm", shielded_instance_config.value) - enable_integrity_monitoring = lookup(var.shielded_instance_config, "enable_integrity_monitoring", shielded_instance_config.value) - } - } - - confidential_instance_config { - enable_confidential_compute = var.enable_confidential_vm - } - - dynamic "guest_accelerator" { - for_each = local.gpu_enabled ? [var.gpu] : [] - content { - type = guest_accelerator.value.type - count = guest_accelerator.value.count - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf deleted file mode 100644 index 69f8d3b98c..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "self_link" { - description = "Self-link of instance template" - value = google_compute_instance_template.tpl.self_link -} - -output "name" { - description = "Name of instance template" - value = google_compute_instance_template.tpl.name -} - -output "tags" { - description = "Tags that will be associated with instance(s)" - value = google_compute_instance_template.tpl.tags -} - -output "service_account" { - description = "value" - value = google_compute_instance_template.tpl.service_account[0] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf deleted file mode 100644 index c285c3fea5..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf +++ /dev/null @@ -1,398 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - type = string - description = "The GCP project ID" - default = null -} - -variable "name_prefix" { - description = "Name prefix for the instance template" - type = string -} - -variable "machine_type" { - description = "Machine type to create, e.g. n1-standard-1" - type = string - default = "n1-standard-1" -} - -variable "min_cpu_platform" { - description = "Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform" - type = string - default = null -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example" - type = string - default = "false" -} - -variable "tags" { - type = list(string) - description = "Network tags, provided as a list" - default = [] -} - -variable "labels" { - type = map(string) - description = "Labels, provided as a map" - default = {} -} - -variable "preemptible" { - type = bool - description = "Allow the instance to be preempted" - default = false -} - -variable "spot" { - description = <<-EOD - Provision as a SPOT preemptible instance. - See https://cloud.google.com/compute/docs/instances/spot for more details. - EOD - type = bool - default = false -} - -variable "instance_termination_action" { - description = <<-EOD - Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'. - See https://cloud.google.com/compute/docs/instances/spot for more details. - EOD - type = string - default = null -} - -variable "automatic_restart" { - type = bool - description = "(Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user)." - default = true -} - -variable "on_host_maintenance" { - type = string - description = "Instance availability Policy" - default = "MIGRATE" -} - -variable "region" { - type = string - description = "Region where the instance template should be created." - nullable = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -####### -# disk -####### -variable "source_image" { - description = "Source disk image. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." - type = string - default = "" -} - -variable "source_image_family" { - description = "Source image family. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." - type = string - default = "centos-7" -} - -variable "source_image_project" { - description = "Project where the source image comes from. The default project contains CentOS images." - type = string - default = "centos-cloud" -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = string - default = "100" -} - -variable "disk_type" { - description = "Boot disk type, can be either pd-ssd, local-ssd, or pd-standard" - type = string - default = "pd-standard" -} - -variable "disk_labels" { - description = "Labels to be assigned to boot disk, provided as a map" - type = map(string) - default = {} -} - -variable "disk_encryption_key" { - description = "The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance" - type = string - default = null -} - -variable "auto_delete" { - description = "Whether or not the boot disk should be auto-deleted" - type = string - default = "true" -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - description = "List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name" - type = list(object({ - source = optional(string) - disk_name = optional(string) - device_name = string - auto_delete = bool - boot = bool - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = map(string) - disk_resource_manager_tags = map(string) - })) - default = [] -} - -#################### -# network_interface -#################### -variable "network" { - description = "The name or self_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks." - type = string - default = "" -} - -variable "nic_type" { - description = "The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO_NET." - type = string - default = null -} - -variable "subnetwork" { - description = "The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided." - type = string - default = "" -} - -variable "subnetwork_project" { - description = "The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used." - type = string - default = null -} - -variable "network_ip" { - description = "Private IP address to assign to the instance if desired." - type = string - default = "" -} - -variable "stack_type" { - description = "The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4_ONLY." - type = string - default = null -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - })) -} - -variable "total_egress_bandwidth_tier" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 6.41 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.41 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance](#module\_instance) | ../instance | n/a | -| [template](#module\_template) | ../instance_template | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | -| [login\_nodes](#input\_login\_nodes) | Slurm login instance definitions. | 
object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  })
| n/a | yes | -| [network\_storage](#input\_network\_storage) | Storage to mounted on login instances
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | -| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path to directory in the bucket for configs | `string` | n/a | yes | -| [slurm\_bucket\_name](#input\_slurm\_bucket\_name) | Name of the bucket for configs | `string` | n/a | yes | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name | `string` | n/a | yes | -| [startup\_scripts](#input\_startup\_scripts) | List of scripts to be ran on login VMs startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | -| [startup\_scripts\_timeout](#input\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script. If any script exceeds this timeout,
then the instance setup process is considered failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instances](#output\_instances) | VM instances of login nodes | -| [service\_account](#output\_service\_account) | Service Account used by login VMs | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf deleted file mode 100644 index 605461f7e6..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "template" { - source = "../instance_template" - - project_id = var.project_id - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = "login" - slurm_bucket_path = var.slurm_bucket_path - name_prefix = local.name - - additional_disks = var.login_nodes.additional_disks - bandwidth_tier = var.login_nodes.bandwidth_tier - can_ip_forward = var.login_nodes.can_ip_forward - advanced_machine_features = var.login_nodes.advanced_machine_features - disk_auto_delete = var.login_nodes.disk_auto_delete - disk_labels = var.login_nodes.disk_labels - disk_resource_manager_tags = var.login_nodes.disk_resource_manager_tags - disk_size_gb = var.login_nodes.disk_size_gb - disk_type = var.login_nodes.disk_type - enable_confidential_vm = var.login_nodes.enable_confidential_vm - enable_oslogin = var.login_nodes.enable_oslogin - enable_shielded_vm = var.login_nodes.enable_shielded_vm - gpu = var.login_nodes.gpu - labels = var.login_nodes.labels - machine_type = var.login_nodes.machine_type - metadata = merge(var.login_nodes.metadata, { - "universe_domain" = var.universe_domain, - "slurm_login_group" = local.name - }) - min_cpu_platform = var.login_nodes.min_cpu_platform - on_host_maintenance = var.login_nodes.on_host_maintenance - preemptible = var.login_nodes.preemptible - region = var.login_nodes.region - resource_manager_tags = var.login_nodes.resource_manager_tags - service_account = var.login_nodes.service_account - shielded_instance_config = var.login_nodes.shielded_instance_config - source_image_family = var.login_nodes.source_image_family - source_image_project = var.login_nodes.source_image_project - source_image = var.login_nodes.source_image - spot = var.login_nodes.spot - subnetwork = var.login_nodes.subnetwork - tags = concat([var.slurm_cluster_name], var.login_nodes.tags) - termination_action = var.login_nodes.termination_action - - internal_startup_script = var.internal_startup_script -} - -module "instance" { - source = "../instance" - - access_config = var.login_nodes.access_config - hostname = "${var.slurm_cluster_name}-${local.name}" - - project_id = var.project_id - - instance_template = module.template.self_link - num_instances = var.login_nodes.num_instances - - additional_networks = var.login_nodes.additional_networks - region = var.login_nodes.region - static_ips = var.login_nodes.static_ips - subnetwork = var.login_nodes.subnetwork - zone = var.login_nodes.zone - - replace_trigger = var.replace_trigger -} - -resource "google_storage_bucket_object" "startup_scripts" { - for_each = { - for s in var.startup_scripts : format( - "slurm-login-%s-script-%s", local.name, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_") - ) => s.content - } - - bucket = var.slurm_bucket_name - name = "${var.slurm_bucket_dir}/${each.key}" - content = each.value - source_md5hash = md5(each.value) -} - -locals { - name = var.login_nodes.group_name # short hand - - config = { - group_name = local.name - startup_scripts_timeout = var.startup_scripts_timeout - network_storage = var.network_storage - } -} - -resource "google_storage_bucket_object" "config" { - bucket = var.slurm_bucket_name - name = "${var.slurm_bucket_dir}/login_group_configs/${local.name}.yaml" - content = yamlencode(local.config) - source_md5hash = md5(yamlencode(local.config)) - - # To ensure that login group "is not ready" until all startup scripts are written down - depends_on = [google_storage_bucket_object.startup_scripts] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf deleted file mode 100644 index 04de18a188..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "service_account" { - value = module.template.service_account - description = "Service Account used by login VMs" -} - -output "instances" { - value = module.instance.slurm_instances - description = "VM instances of login nodes" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf deleted file mode 100644 index 3efd862942..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "slurm_cluster_name" { - type = string - description = "Cluster name" -} - -variable "slurm_bucket_path" { - type = string - description = "GCS Bucket URI of Slurm cluster file storage." -} - - -variable "slurm_bucket_name" { - type = string - description = "Name of the bucket for configs" -} - -variable "slurm_bucket_dir" { - type = string - description = "Path to directory in the bucket for configs" -} - - -variable "universe_domain" { - description = "Domain address for alternate API universe" - type = string - default = "googleapis.com" -} - -variable "login_nodes" { - description = "Slurm login instance definitions." - type = object({ - group_name = string - access_config = optional(list(object({ - nat_ip = string - network_tier = string - }))) - additional_disks = optional(list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string), {}) - auto_delete = optional(bool, true) - boot = optional(bool, false) - disk_resource_manager_tags = optional(map(string), {}) - })), []) - additional_networks = optional(list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })), []) - bandwidth_tier = optional(string, "platform_default") - can_ip_forward = optional(bool, false) - disk_auto_delete = optional(bool, true) - disk_labels = optional(map(string), {}) - disk_resource_manager_tags = optional(map(string), {}) - disk_size_gb = optional(number) - disk_type = optional(string, "n1-standard-1") - enable_confidential_vm = optional(bool, false) - enable_oslogin = optional(bool, true) - enable_shielded_vm = optional(bool, false) - gpu = optional(object({ - count = number - type = string - })) - labels = optional(map(string), {}) - machine_type = optional(string) - advanced_machine_features = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - metadata = optional(map(string), {}) - min_cpu_platform = optional(string) - num_instances = optional(number, 1) - on_host_maintenance = optional(string) - preemptible = optional(bool, false) - region = optional(string) - resource_manager_tags = optional(map(string), {}) - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - shielded_instance_config = optional(object({ - enable_integrity_monitoring = optional(bool, true) - enable_secure_boot = optional(bool, true) - enable_vtpm = optional(bool, true) - })) - source_image_family = optional(string) - source_image_project = optional(string) - source_image = optional(string) - static_ips = optional(list(string), []) - subnetwork = string - spot = optional(bool, false) - tags = optional(list(string), []) - zone = optional(string) - termination_action = optional(string) - }) -} - - -variable "startup_scripts" { - description = "List of scripts to be ran on login VMs startup." - type = list(object({ - filename = string - content = string - })) - default = [] -} - -variable "startup_scripts_timeout" { - description = < - -- [Module: Slurm Nodeset (TPU)](#module-slurm-nodeset-tpu) - - [Overview](#overview) - - [Module API](#module-api) - - - -## Overview - -This is a submodule of [slurm_cluster](../../../slurm_cluster/README.md). It -creates a Slurm TPU nodeset for [slurm_partition](../slurm_partition/README.md). - -## Module API - -For the terraform module API reference, please see -[README_TF.md](./README_TF.md). - - -Copyright (C) SchedMD LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.2 | -| [google](#requirement\_google) | >= 3.53 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.53 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.nodeset_tpu](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_subnetwork.nodeset_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | -| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | -| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `""` | no | -| [enable\_public\_ip](#input\_enable\_public\_ip) | Enables IP address to access the Internet. | `bool` | `false` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of nodes allowed in this partition to be created dynamically. | `number` | `0` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | -| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. Not needed if you use accelerator\_config | `string` | `null` | no | -| [nodeset\_name](#input\_nodeset\_name) | Name of Slurm nodeset. | `string` | n/a | yes | -| [preemptible](#input\_preemptible) | Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details. | `bool` | `false` | no | -| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `true` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the TPU-vm.
If none is given, the default service account and scopes will be used. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | -| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | n/a | yes | -| [zone](#input\_zone) | Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Nodeset details. | -| [nodeset\_name](#output\_nodeset\_name) | Nodeset name. | -| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf deleted file mode 100644 index 1a6a9cfba1..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########### -# NODESET # -########### - -locals { - node_conf_hw = { - Mem334CPU96 = { - CPUs = 96 - Boards = 1 - Sockets = 2 - CoresPerSocket = 24 - ThreadsPerCore = 2 - RealMemory = 307200 - } - Mem400CPU240 = { - CPUs = 240 - Boards = 1 - Sockets = 2 - CoresPerSocket = 60 - ThreadsPerCore = 2 - RealMemory = 400000 - } - } - node_conf_mappings = { - "v2" = local.node_conf_hw.Mem334CPU96 - "v3" = local.node_conf_hw.Mem334CPU96 - "v4" = local.node_conf_hw.Mem400CPU240 - } - simple_nodes = ["v2-8", "v3-8", "v4-8"] -} - -locals { - snetwork = data.google_compute_subnetwork.nodeset_subnetwork.name - region = join("-", slice(split("-", var.zone), 0, 2)) - tpu_fam = var.accelerator_config.version != "" ? lower(var.accelerator_config.version) : split("-", var.node_type)[0] - #If subnetwork is specified and it does not have private_ip_google_access, we need to have public IPs on the TPU - #if no subnetwork is specified, the default one will be used, this does not have private_ip_google_access so we need public IPs too - pub_need = !data.google_compute_subnetwork.nodeset_subnetwork.private_ip_google_access - can_preempt = var.node_type != null ? contains(local.simple_nodes, var.node_type) : false - nodeset_tpu = { - nodeset_name = var.nodeset_name - node_conf = local.node_conf_mappings[local.tpu_fam] - node_type = var.node_type - accelerator_config = var.accelerator_config - tf_version = var.tf_version - preemptible = local.can_preempt ? var.preemptible : false - reserved = var.reserved - node_count_dynamic_max = var.node_count_dynamic_max - node_count_static = var.node_count_static - enable_public_ip = var.enable_public_ip - zone = var.zone - service_account = var.service_account != null ? var.service_account : local.service_account - preserve_tpu = local.can_preempt ? var.preserve_tpu : false - data_disks = var.data_disks - docker_image = var.docker_image != "" ? var.docker_image : "us-docker.pkg.dev/schedmd-slurm-public/tpu/slurm-gcp-6-9:tf-${var.tf_version}" - subnetwork = local.snetwork - network_storage = var.network_storage - } - - service_account = { - email = try(var.service_account.email, null) - scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) - } -} - -data "google_compute_subnetwork" "nodeset_subnetwork" { - name = var.subnetwork - region = local.region - project = var.project_id - - self_link = ( - length(regexall("/projects/([^/]*)", var.subnetwork)) > 0 - && length(regexall("/regions/([^/]*)", var.subnetwork)) > 0 - ? var.subnetwork - : null - ) -} - -resource "null_resource" "nodeset_tpu" { - triggers = { - nodeset = sha256(jsonencode(local.nodeset_tpu)) - } - lifecycle { - precondition { - condition = sum([var.node_count_dynamic_max, var.node_count_static]) > 0 - error_message = "Sum of node_count_dynamic_max and node_count_static must be > 0." - } - precondition { - condition = !(var.preemptible && var.reserved) - error_message = "Nodeset cannot be preemptible and reserved at the same time." - } - precondition { - condition = !(var.subnetwork == null && !var.enable_public_ip) - error_message = "Using the default subnetwork for the TPU nodeset requires enable_public_ip set to true." - } - precondition { - condition = !(var.subnetwork != null && (local.pub_need && !var.enable_public_ip)) - error_message = "The subnetwork specified does not have Private Google Access enabled. This is required when enable_public_ip is set to false." - } - precondition { - condition = !(var.node_type == null && (var.accelerator_config.topology == "" && var.accelerator_config.version == "")) - error_message = "Either a node type or an accelerator_config must be provided." - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf deleted file mode 100644 index fce700d567..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "nodeset_name" { - description = "Nodeset name." - value = local.nodeset_tpu.nodeset_name -} - -output "nodeset" { - description = "Nodeset details." - value = local.nodeset_tpu -} - -output "service_account" { - description = "Service account object, includes email and scopes." - value = local.service_account -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf deleted file mode 100644 index a8c470dec9..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "nodeset_name" { - description = "Name of Slurm nodeset." - type = string - - validation { - condition = can(regex("^[a-z](?:[a-z0-9]{0,14})$", var.nodeset_name)) - error_message = "Variable 'nodeset_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,14})$'." - } -} - -variable "node_type" { - description = "Specify a node type to base the vm configuration upon it. Not needed if you use accelerator_config" - type = string - default = null -} - -variable "accelerator_config" { - description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." - type = object({ - topology = string - version = string - }) - default = { - topology = "" - version = "" - } - validation { - condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], upper(var.accelerator_config.version)) - error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" - } - validation { - condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) - error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." - } -} - -variable "docker_image" { - description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" - type = string - default = "" -} - -variable "tf_version" { - description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." - type = string -} - -variable "zone" { - description = "Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it." - type = string - - validation { - condition = can(coalesce(var.zone)) - error_message = "Zone cannot be null or empty." - } -} - -variable "preemptible" { - description = "Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details." - type = bool - default = false -} - -variable "reserved" { - description = "Specify whether TPU-vms in this nodeset are created under a reservation." - type = bool - default = false -} - -variable "preserve_tpu" { - description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" - type = bool - default = true -} - -variable "node_count_static" { - description = "Number of nodes to be statically created." - type = number - default = 0 - - validation { - condition = var.node_count_static >= 0 - error_message = "Value must be >= 0." - } -} - -variable "node_count_dynamic_max" { - description = "Maximum number of nodes allowed in this partition to be created dynamically." - type = number - default = 0 - - validation { - condition = var.node_count_dynamic_max >= 0 - error_message = "Value must be >= 0." - } -} - -variable "enable_public_ip" { - description = "Enables IP address to access the Internet." - type = bool - default = false -} - -variable "data_disks" { - type = list(string) - description = "The data disks to include in the TPU node" - default = [] -} - -variable "subnetwork" { - description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." - type = string -} - -variable "service_account" { - type = object({ - email = string - scopes = set(string) - }) - description = < -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | -| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | -| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | -| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | -| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | -| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "v0.7.2")
  })
| `{}` | no | -| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "v0.11.4")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | -| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md deleted file mode 100644 index 1957899617..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [helm](#provider\_helm) | ~> 2.17 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | -| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | -| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | -| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | -| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | -| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | -| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | -| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | -| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | -| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | -| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | -| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | -| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | -| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | -| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | -| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | -| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | -| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | -| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | -| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | -| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | -| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | -| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | -| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | -| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | -| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | -| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | -| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | -| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | -| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf deleted file mode 100644 index bd2383b772..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "helm_release" "apply_chart" { - # Required Identification - name = var.release_name - chart = var.chart_name - - # Chart Source & Version - repository = var.chart_repository - version = var.chart_version - devel = var.devel - - # Target Namespace - namespace = var.namespace - create_namespace = var.create_namespace - - # Values Configuration - values = var.values_yaml - - dynamic "set" { - for_each = var.set_values - content { - name = set.value.name - value = set.value.value - type = set.value.type - } - } - - # Installation/Upgrade Behavior - description = var.description - atomic = var.atomic - cleanup_on_fail = var.cleanup_on_fail - dependency_update = var.dependency_update - disable_crd_hooks = var.disable_crd_hooks - disable_openapi_validation = var.disable_openapi_validation - disable_webhooks = var.disable_webhooks - force_update = var.force_update - lint = var.lint - max_history = var.max_history - recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI - render_subchart_notes = var.render_subchart_notes - reset_values = var.reset_values - reuse_values = var.reuse_values - skip_crds = var.skip_crds - timeout = var.timeout - wait = var.wait - wait_for_jobs = var.wait_for_jobs - - # Verification & Credentials - keyring = var.keyring - pass_credentials = var.pass_credentials - verify = var.verify - - # Post Rendering - dynamic "postrender" { - # Only include the block if var.postrender is not null - for_each = var.postrender == null ? [] : [var.postrender] - content { - binary_path = postrender.value.binary_path - } - } - -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml deleted file mode 100644 index e18197e2b7..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf deleted file mode 100644 index 04e8e214fc..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -# --- Required --- -variable "release_name" { - description = "Name of the Helm release." - type = string -} - -variable "chart_name" { - description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." - type = string -} - -# --- Chart Location & Version --- -variable "chart_repository" { - description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." - type = string - default = null -} - -variable "chart_version" { - description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." - type = string - default = null -} - -variable "devel" { - description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." - type = bool - default = false -} - -# --- Namespace --- -variable "namespace" { - description = "Kubernetes namespace to install the Helm release into." - type = string - default = "default" -} - -variable "create_namespace" { - description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." - type = bool - default = true # Common convenience setting -} - -# --- Values Customization --- -variable "values_yaml" { - description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." - type = list(string) - default = [] -} - -variable "set_values" { - description = "List of objects defining values to set ('helm install --set')." - type = list(object({ - name = string # Path to the value (e.g., 'service.type', 'replicaCount') - value = string # The value to set - type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') - })) - default = [] -} - -# --- Installation/Upgrade Behavior --- -variable "description" { - description = "Set an optional description for the Helm release." - type = string - default = null -} - -variable "atomic" { - description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." - type = bool - default = false -} - -variable "wait" { - description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." - type = bool - default = true # Often a good default for dependencies -} - -variable "wait_for_jobs" { - description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." - type = bool - default = false # Helm CLI default is false -} - -variable "timeout" { - description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." - type = number - default = 300 # 5 minutes (Helm CLI default) -} - -variable "cleanup_on_fail" { - description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." - type = bool - default = false -} - -variable "dependency_update" { - description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." - type = bool - default = false -} - -variable "disable_crd_hooks" { - description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." - type = bool - default = false -} - -variable "disable_openapi_validation" { - description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." - type = bool - default = false -} - -variable "disable_webhooks" { - description = "Prevent hooks from running ('helm install --no-hooks')." - type = bool - default = false -} - -variable "force_update" { - description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." - type = bool - default = false -} - -variable "lint" { - description = "Run the helm chart linter during the plan ('helm lint')." - type = bool - default = false -} - -variable "max_history" { - description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." - type = number - default = null # Terraform provider defaults to Helm's default (usually 10) -} - -variable "recreate_pods" { - description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." - type = bool - default = false -} - -variable "render_subchart_notes" { - description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." - type = bool - default = false -} - -variable "reset_values" { - description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." - type = bool - default = false -} - -variable "reuse_values" { - description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." - type = bool - default = false # Helm CLI default is false -} - -variable "skip_crds" { - description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." - type = bool - default = false -} - -# --- Verification & Credentials --- -variable "keyring" { - description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." - type = string - default = null # Defaults to Helm's default keyring location -} - -variable "pass_credentials" { - description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." - type = bool - default = false -} - -variable "verify" { - description = "Verify the package before installing it ('helm install --verify')." - type = bool - default = false -} - -# --- Advanced Rendering --- -variable "postrender" { - description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." - type = object({ - binary_path = string # Path to the post-renderer executable - }) - default = null # Disabled by default -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf deleted file mode 100644 index 09d912e2c9..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md deleted file mode 100644 index 46bfe51a32..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md +++ /dev/null @@ -1,40 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.23 | - -## Providers - -| Name | Version | -|------|---------| -| [kubernetes](#provider\_kubernetes) | ~> 2.23 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubernetes_manifest.apply_manifests](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | -| [field\_manager](#input\_field\_manager) | (Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts. | 
object({
    name            = optional(string, null)
    force_conflicts = optional(bool, false)
  })
| `null` | no | -| [resource\_timeouts](#input\_resource\_timeouts) | (Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met. | 
object({
    create = optional(string, null)
    update = optional(string, null)
    delete = optional(string, null)
  })
| 
{
  "create": "15m",
  "delete": "5m",
  "update": "10m"
}
| no | -| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `""` | no | -| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | -| [wait\_for\_fields](#input\_wait\_for\_fields) | (Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern. | `map(string)` | `{}` | no | -| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf deleted file mode 100644 index f97f26038d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - yaml_separator = "\n---" - - # --- 1. Determine the primary source of YAML content --- - # Prioritize 'content' variable if provided - primary_content_body = var.content != "" ? var.content : null - - # --- 2. Handle 'source_path' based on its type (File vs. Directory) --- - - # Check if source_path is a directory (indicated by trailing slash) - is_directory = endswith(var.source_path, "/") - directory_absolute_path = local.is_directory ? abspath(var.source_path) : null - - # Check if source_path is a single yaml or tftpl file (only if not a directory) - is_single_file = !local.is_directory && ( - length(regexall("\\.yaml$", lower(var.source_path))) > 0 || - length(regexall("\\.tftpl$", lower(var.source_path))) > 0 - ) - single_file_raw_content = local.is_single_file ? ( - length(regexall("\\.tftpl$", lower(var.source_path))) > 0 ? - templatefile(abspath(var.source_path), var.template_vars) : - file(abspath(var.source_path)) - ) : null - - # Docs from primary_content_body - docs_from_primary_source = [ - for doc in split(local.yaml_separator, coalesce(local.primary_content_body, local.single_file_raw_content, "")) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - - # Docs from .yaml files in a directory - directory_yaml_files = local.is_directory ? fileset(local.directory_absolute_path, "*.yaml") : [] - docs_from_directory_yamls = flatten([ - for file_name in local.directory_yaml_files : - [ - for doc in split(local.yaml_separator, file(format("%s/%s", local.directory_absolute_path, file_name))) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - ]) - - # Docs from .tftpl files in a directory - directory_template_files = local.is_directory ? fileset(local.directory_absolute_path, "*.tftpl") : [] - docs_from_directory_templates = flatten([ - for file_name in local.directory_template_files : - [ - for doc in split(local.yaml_separator, templatefile(format("%s/%s", local.directory_absolute_path, file_name), var.template_vars)) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - ]) - - all_parsed_docs = concat( - local.docs_from_primary_source, - local.docs_from_directory_yamls, - local.docs_from_directory_templates - ) - - # --- 5. Create the final map for `for_each` (keys must be unique strings) --- - docs_map = tomap({ - for index, doc in local.all_parsed_docs : index => doc - if length(trimspace(doc)) > 0 - }) -} - -# Apply all manifest files dynamically -resource "kubernetes_manifest" "apply_manifests" { - for_each = local.docs_map - manifest = yamldecode(each.value) - timeouts { - create = var.resource_timeouts.create - update = var.resource_timeouts.update - delete = var.resource_timeouts.delete - } - - dynamic "wait" { - for_each = var.wait_for_rollout ? [1] : [] - content { - rollout = var.wait_for_rollout - fields = var.wait_for_fields - } - } - - # Configure the 'field_manager' block dynamically - dynamic "field_manager" { - for_each = var.field_manager != null ? [var.field_manager] : [] - content { - name = field_manager.value.name - force_conflicts = field_manager.value.force_conflicts - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml deleted file mode 100644 index e18197e2b7..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf deleted file mode 100644 index 0b846189ea..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -variable "content" { - description = "The YAML body to apply to gke cluster." - type = string - default = null -} - -variable "source_path" { - description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." - type = string - default = "" -} - -variable "template_vars" { - description = "The values to populate template file(s) with." - type = any - default = null -} - -variable "wait_for_rollout" { - description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." - type = bool - default = true -} - - -variable "wait_for_fields" { - description = "(Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern." - type = map(string) - default = {} -} - -variable "resource_timeouts" { - description = "(Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met." - type = object({ - create = optional(string, null) - update = optional(string, null) - delete = optional(string, null) - }) - default = { - create = "15m" # Default create timeout, also covers waiting for initial conditions - update = "10m" # Default update timeout, also covers waiting for update conditions - delete = "5m" # Default delete timeout - } -} - -variable "field_manager" { - description = "(Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts." - type = object({ - name = optional(string, null) - force_conflicts = optional(bool, false) - }) - default = null -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf deleted file mode 100644 index 61786b06de..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - # Defines the providers that this module depends on and their versions. - required_providers { - kubernetes = { - source = "hashicorp/kubernetes" - version = "~> 2.23" - } - } - required_version = ">= 1.3" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/main.tf b/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/main.tf deleted file mode 100644 index 8db4870452..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/management/dependencies-installer/main.tf +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] - project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] - - install_gpu_operator = try(var.gpu_operator.install, false) - install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) -} - -data "google_container_cluster" "gke_cluster" { - project = local.project_id - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -module "install_kueue" { - source = "./helm_install" - depends_on = [var.gke_cluster_exists] - - release_name = "kueue" - - chart_name = "oci://registry.k8s.io/kueue/charts/kueue" - chart_version = var.kueue.version # Specify your desired Kueue version - - create_namespace = true # Helm can also create the namespace - wait = true - timeout = 600 # seconds -} - -module "install_jobset" { - source = "./helm_install" - depends_on = [var.gke_cluster_exists, module.install_kueue] - release_name = "jobset-controller" # The release name for your JobSet installation - chart_name = "oci://registry.k8s.io/jobset/charts/jobset" # The Helm repository URL for nvidia charts - chart_version = var.jobset.version - create_namespace = true - namespace = "jobset-system" -} - -module "install_nvidia_dra_driver" { - count = local.install_nvidia_dra_driver ? 1 : 0 - depends_on = [var.gke_cluster_exists] - source = "./helm_install" - - release_name = "nvidia-dra-driver-gpu" # The release name - chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts - chart_name = "nvidia-dra-driver-gpu" # The chart name - chart_version = var.nvidia_dra_driver.version # The chart version - namespace = "nvidia-dra-driver-gpu" # The target namespace - create_namespace = true # Equivalent to --create-namespace - - # Use the 'values' argument to pass the YAML content - # This corresponds to the -f <(cat < -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.2 | -| [google](#requirement\_google) | >= 6.40 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.40 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_global_address.private_ip_alloc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_global_address) | resource | -| [google_compute_network_peering_routes_config.private_vpc_peering_routes_gcnv](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering_routes_config) | resource | -| [google_service_networking_connection.private_vpc_connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_networking_connection) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [address](#input\_address) | The IP address or beginning of the address range allocated for the Private Service Access. | `string` | `null` | no | -| [deletion\_policy](#input\_deletion\_policy) | The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to supporting resources. Key-value pairs. | `map(string)` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to configure Private Service Access:
`projects//global/networks/`" | `string` | n/a | yes | -| [prefix\_length](#input\_prefix\_length) | The prefix length of the IP range allocated for the Private Service Access. | `number` | `16` | no | -| [project\_id](#input\_project\_id) | ID of project in which Private Service Access will be created. | `string` | n/a | yes | -| [service\_name](#input\_service\_name) | The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'. | `string` | `"servicenetworking.googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cidr\_range](#output\_cidr\_range) | CIDR range of the created google\_compute\_global\_address | -| [connect\_mode](#output\_connect\_mode) | Services that use Private Service Access typically specify connect\_mode
"PRIVATE\_SERVICE\_ACCESS". This output value sets connect\_mode and additionally
blocks terraform actions until the VPC connection has been created. | -| [private\_vpc\_connection\_peering](#output\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection that was created by the service provider. | -| [reserved\_ip\_range](#output\_reserved\_ip\_range) | Named IP range to be used by services connected with Private Service Access. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/main.tf b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/main.tf deleted file mode 100644 index 429e4d93f0..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/main.tf +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "private-service-access", ghpc_role = "network" }) -} - -locals { - split_network_id = split("/", var.network_id) - network_name = local.split_network_id[4] - network_project = local.split_network_id[1] -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_compute_global_address" "private_ip_alloc" { - provider = google - name = "global-psconnect-ip-${random_id.resource_name_suffix.hex}" - project = var.project_id - purpose = "VPC_PEERING" - address_type = "INTERNAL" - network = var.network_id - prefix_length = var.prefix_length - labels = local.labels - address = var.address -} - -resource "google_service_networking_connection" "private_vpc_connection" { - network = var.network_id - service = var.service_name - reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name] - deletion_policy = var.deletion_policy - update_on_creation_fail = var.deletion_policy == "ABANDON" ? true : null -} - -# Google Cloud NetApp Volumes need enablement of custom_route import and export -resource "google_compute_network_peering_routes_config" "private_vpc_peering_routes_gcnv" { - count = var.service_name == "netapp.servicenetworking.goog" ? 1 : 0 - project = local.network_project - network = local.network_name - peering = google_service_networking_connection.private_vpc_connection.peering - - export_custom_routes = true - import_custom_routes = true -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/metadata.yaml deleted file mode 100644 index 93e8b3970e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - servicenetworking.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/outputs.tf deleted file mode 100644 index 296f2e9140..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "private_vpc_connection_peering" { - description = "The name of the VPC Network peering connection that was created by the service provider." - sensitive = true - value = google_service_networking_connection.private_vpc_connection.peering -} - -output "connect_mode" { - description = <<-EOT - Services that use Private Service Access typically specify connect_mode - "PRIVATE_SERVICE_ACCESS". This output value sets connect_mode and additionally - blocks terraform actions until the VPC connection has been created. - EOT - value = "PRIVATE_SERVICE_ACCESS" - depends_on = [ - google_service_networking_connection.private_vpc_connection, - ] -} - -output "reserved_ip_range" { - description = "Named IP range to be used by services connected with Private Service Access." - value = google_compute_global_address.private_ip_alloc.name -} - -output "cidr_range" { - description = "CIDR range of the created google_compute_global_address" - value = "${google_compute_global_address.private_ip_alloc.address}/${google_compute_global_address.private_ip_alloc.prefix_length}" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/variables.tf deleted file mode 100644 index 4b0a3e796f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/variables.tf +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "address" { - description = "The IP address or beginning of the address range allocated for the Private Service Access." - type = string - default = null -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to configure Private Service Access: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "labels" { - description = "Labels to add to supporting resources. Key-value pairs." - type = map(string) -} - -variable "prefix_length" { - description = "The prefix length of the IP range allocated for the Private Service Access." - type = number - default = 16 -} - -variable "project_id" { - description = "ID of project in which Private Service Access will be created." - type = string -} - -variable "service_name" { - description = "The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'." - type = string - default = "servicenetworking.googleapis.com" -} - -variable "deletion_policy" { - description = "The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON." - type = string - default = null -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/versions.tf deleted file mode 100644 index df2914cdb9..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/network/private-service-access/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.40" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" - } - - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" - } - - required_version = ">= 1.2" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/new-project/README.md b/deletion-test/build_script/modules/embedded/community/modules/project/new-project/README.md deleted file mode 100644 index 5e5cabe9d5..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/new-project/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module allows you to create opinionated Google Cloud Platform projects. It -creates projects and configures aspects like Shared VPC connectivity, IAM -access, Service Accounts, and API enablement to follow best practices. - -This module is meant for use with Terraform 0.13. - -**Note:** This module has been removed from the Cluster Toolkit. The upstream module (`terraform-google-project-factory`) is now the recommended way to create and manage GCP projects. - -### Example - -```yaml -- id: project - source: github.com/terraform-google-modules/terraform-google-project-factory?rev=v17.0.0&depth=1 -``` - -This creates a new project with pre-defined project ID, a designated folder and -organization and associated billing account which will be used to pay for -services consumed. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [project\_factory](#module\_project\_factory) | terraform-google-modules/project-factory/google | ~> 11.3 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [activate\_api\_identities](#input\_activate\_api\_identities) | The list of service identities (Google Managed service account for the API) to force-create for the project (e.g. in order to grant additional roles).
APIs in this list will automatically be appended to `activate_apis`.
Not including the API in this list will follow the default behaviour for identity creation (which is usually when the first resource using the API is created).
Any roles (e.g. service agent role) must be explicitly listed. See https://cloud.google.com/iam/docs/understanding-roles#service-agent-roles-roles for a list of related roles. | 
list(object({
    api   = string
    roles = list(string)
  }))
| `[]` | no | -| [activate\_apis](#input\_activate\_apis) | The list of apis to activate within the project | `list(string)` | 
[
  "compute.googleapis.com",
  "serviceusage.googleapis.com",
  "storage.googleapis.com"
]
| no | -| [auto\_create\_network](#input\_auto\_create\_network) | Create the default network | `bool` | `false` | no | -| [billing\_account](#input\_billing\_account) | The ID of the billing account to associate this project with | `string` | n/a | yes | -| [bucket\_force\_destroy](#input\_bucket\_force\_destroy) | Force the deletion of all objects within the GCS bucket when deleting the bucket (optional) | `bool` | `false` | no | -| [bucket\_labels](#input\_bucket\_labels) | A map of key/value label pairs to assign to the bucket (optional) | `map(string)` | `{}` | no | -| [bucket\_location](#input\_bucket\_location) | The location for a GCS bucket to create (optional) | `string` | `"US"` | no | -| [bucket\_name](#input\_bucket\_name) | A name for a GCS bucket to create (in the bucket\_project project), useful for Terraform state (optional) | `string` | `""` | no | -| [bucket\_project](#input\_bucket\_project) | A project to create a GCS bucket (bucket\_name) in, useful for Terraform state (optional) | `string` | `""` | no | -| [bucket\_ula](#input\_bucket\_ula) | Enable Uniform Bucket Level Access | `bool` | `true` | no | -| [bucket\_versioning](#input\_bucket\_versioning) | Enable versioning for a GCS bucket to create (optional) | `bool` | `false` | no | -| [budget\_alert\_pubsub\_topic](#input\_budget\_alert\_pubsub\_topic) | The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}` | `string` | `null` | no | -| [budget\_alert\_spent\_percents](#input\_budget\_alert\_spent\_percents) | A list of percentages of the budget to alert on when threshold is exceeded | `list(number)` | 
[
  0.5,
  0.7,
  1
]
| no | -| [budget\_amount](#input\_budget\_amount) | The amount to use for a budget alert | `number` | `null` | no | -| [budget\_display\_name](#input\_budget\_display\_name) | The display name of the budget. If not set defaults to `Budget For ` | `string` | `null` | no | -| [budget\_monitoring\_notification\_channels](#input\_budget\_monitoring\_notification\_channels) | A list of monitoring notification channels in the form `[projects/{project_id}/notificationChannels/{channel_id}]`. A maximum of 5 channels are allowed. | `list(string)` | `[]` | no | -| [consumer\_quotas](#input\_consumer\_quotas) | The quotas configuration you want to override for the project. | 
list(object({
    service = string,
    metric  = string,
    limit   = string,
    value   = string,
  }))
| `[]` | no | -| [create\_project\_sa](#input\_create\_project\_sa) | Whether the default service account for the project shall be created | `bool` | `true` | no | -| [default\_network\_tier](#input\_default\_network\_tier) | Default Network Service Tier for resources created in this project. If unset, the value will not be modified. See https://cloud.google.com/network-tiers/docs/using-network-service-tiers and https://cloud.google.com/network-tiers. | `string` | `""` | no | -| [default\_service\_account](#input\_default\_service\_account) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | `string` | `"keep"` | no | -| [disable\_dependent\_services](#input\_disable\_dependent\_services) | Whether services that are enabled and which depend on this service should also be disabled when this service is destroyed. | `bool` | `true` | no | -| [disable\_services\_on\_destroy](#input\_disable\_services\_on\_destroy) | Whether project services will be disabled when the resources are destroyed | `bool` | `true` | no | -| [domain](#input\_domain) | The domain name (optional). | `string` | `""` | no | -| [enable\_shared\_vpc\_host\_project](#input\_enable\_shared\_vpc\_host\_project) | If this project is a shared VPC host project. If true, you must *not* set svpc\_host\_project\_id variable. Default is false. | `bool` | `false` | no | -| [folder\_id](#input\_folder\_id) | The ID of a folder to host this project | `string` | `""` | no | -| [grant\_services\_network\_role](#input\_grant\_services\_network\_role) | Whether or not to grant service agents the network roles on the host project | `bool` | `true` | no | -| [grant\_services\_security\_admin\_role](#input\_grant\_services\_security\_admin\_role) | Whether or not to grant Kubernetes Engine Service Agent the Security Admin role on the host project so it can manage firewall rules | `bool` | `false` | no | -| [group\_name](#input\_group\_name) | A group to control the project by being assigned group\_role (defaults to project editor) | `string` | `""` | no | -| [group\_role](#input\_group\_role) | The role to give the controlling group (group\_name) over the project (defaults to project editor) | `string` | `"roles/editor"` | no | -| [labels](#input\_labels) | Map of labels for project | `map(string)` | `{}` | no | -| [lien](#input\_lien) | Add a lien on the project to prevent accidental deletion | `bool` | `false` | no | -| [name](#input\_name) | The name for the project | `string` | `null` | no | -| [org\_id](#input\_org\_id) | The organization ID. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The ID to give the project. If not provided, the `name` will be used. | `string` | `""` | no | -| [project\_sa\_name](#input\_project\_sa\_name) | Default service account name for the project. | `string` | `"project-service-account"` | no | -| [random\_project\_id](#input\_random\_project\_id) | Adds a suffix of 4 random characters to the `project_id` | `bool` | `false` | no | -| [sa\_role](#input\_sa\_role) | A role to give the default Service Account for the project (defaults to none) | `string` | `""` | no | -| [shared\_vpc\_subnets](#input\_shared\_vpc\_subnets) | List of subnets fully qualified subnet IDs (ie. projects/$project\_id/regions/$region/subnetworks/$subnet\_id) | `list(string)` | `[]` | no | -| [svpc\_host\_project\_id](#input\_svpc\_host\_project\_id) | The ID of the host project which hosts the shared VPC | `string` | `""` | no | -| [usage\_bucket\_name](#input\_usage\_bucket\_name) | Name of a GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | -| [usage\_bucket\_prefix](#input\_usage\_bucket\_prefix) | Prefix in the GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | -| [vpc\_service\_control\_attach\_enabled](#input\_vpc\_service\_control\_attach\_enabled) | Whether the project will be attached to a VPC Service Control Perimeter | `bool` | `false` | no | -| [vpc\_service\_control\_perimeter\_name](#input\_vpc\_service\_control\_perimeter\_name) | The name of a VPC Service Control Perimeter to add the created project to | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [api\_s\_account](#output\_api\_s\_account) | API service account email | -| [api\_s\_account\_fmt](#output\_api\_s\_account\_fmt) | API service account email formatted for terraform use | -| [budget\_name](#output\_budget\_name) | The name of the budget if created | -| [domain](#output\_domain) | The organization's domain | -| [enabled\_api\_identities](#output\_enabled\_api\_identities) | Enabled API identities in the project | -| [enabled\_apis](#output\_enabled\_apis) | Enabled APIs in the project | -| [group\_email](#output\_group\_email) | The email of the G Suite group with group\_name | -| [project\_bucket\_self\_link](#output\_project\_bucket\_self\_link) | Project's bucket selfLink | -| [project\_bucket\_url](#output\_project\_bucket\_url) | Project's bucket url | -| [project\_id](#output\_project\_id) | ID of the project that was created | -| [project\_name](#output\_project\_name) | Name of the project that was created | -| [project\_number](#output\_project\_number) | Number of the project that was created | -| [service\_account\_display\_name](#output\_service\_account\_display\_name) | The display name of the default service account | -| [service\_account\_email](#output\_service\_account\_email) | The email of the default service account | -| [service\_account\_id](#output\_service\_account\_id) | The id of the default service account | -| [service\_account\_name](#output\_service\_account\_name) | The fully-qualified name of the default service account | -| [service\_account\_unique\_id](#output\_service\_account\_unique\_id) | The unique id of the default service account | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/README.md b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/README.md deleted file mode 100644 index 0f5c10c7e4..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/README.md +++ /dev/null @@ -1,111 +0,0 @@ -## Description - -Allows creation of service accounts for a Google Cloud Platform project. - -### Example - -```yaml -- id: service_acct - source: community/modules/project/service-account - settings: - project_id: $(vars.project_id) - name: instance_acct - project_roles: - - logging.logWriter - - monitoring.metricWriter - - storage.objectViewer -``` - -This creates a service account in GCP project "project_id" with the name -"instance_acct". It will have the 3 roles listed for all resources within the -project. - -### Usage with startup-script module - -When this module is used in conjunction with the [startup-script] module, the -service account must be granted (at least) read access to the bucket. This can -be achieved by granting project-wide access as shown above or by specifying the -service account as a bucket viewer in the startup-script module: - -```yaml -- id: service_acct - source: community/modules/project/service-account - settings: - project_id: $(vars.project_id) - name: instance_acct - project_roles: - - logging.logWriter - - monitoring.metricWriter -- id: script - source: modules/scripts/startup-script - settings: - bucket_viewers: - - $(service_acct.service_account_iam_email) -``` - -[startup-script]: ../../../../modules/scripts/startup-script/README.md - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [service\_account](#module\_service\_account) | terraform-google-modules/service-accounts/google | ~> 4.2 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [billing\_account\_id](#input\_billing\_account\_id) | If assigning billing role, specify a billing account (default is to assign at the organizational level). | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment (will be prepended to service account name) | `string` | n/a | yes | -| [description](#input\_description) | Description of the created service account. | `string` | `"Service Account"` | no | -| [descriptions](#input\_descriptions) | Deprecated; create single service accounts using var.description. | `list(string)` | `null` | no | -| [display\_name](#input\_display\_name) | Display name of the created service account. | `string` | `"Service Account"` | no | -| [generate\_keys](#input\_generate\_keys) | Generate keys for service account. | `bool` | `false` | no | -| [grant\_billing\_role](#input\_grant\_billing\_role) | Grant billing user role. | `bool` | `false` | no | -| [grant\_xpn\_roles](#input\_grant\_xpn\_roles) | Grant roles for shared VPC management. | `bool` | `true` | no | -| [name](#input\_name) | Name of the service account to create. | `string` | n/a | yes | -| [names](#input\_names) | Deprecated; create single service accounts using var.name. | `list(string)` | `null` | no | -| [org\_id](#input\_org\_id) | Id of the organization for org-level roles. | `string` | `""` | no | -| [prefix](#input\_prefix) | Deprecated; prefix now set using var.deployment\_name | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | -| [project\_roles](#input\_project\_roles) | List of roles to grant to service account (e.g. "storage.objectViewer" or "compute.instanceAdmin.v1" | `list(string)` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [key](#output\_key) | Service account key (if creation was requested) | -| [service\_account\_email](#output\_service\_account\_email) | Service account e-mail address | -| [service\_account\_iam\_email](#output\_service\_account\_iam\_email) | Service account IAM binding format (serviceAccount:name@example.com) | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/main.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/main.tf deleted file mode 100644 index e8a69be642..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - display_name = "${var.display_name} (${var.deployment_name})" - description = "${var.description} (${var.deployment_name})" -} - -module "service_account" { - source = "terraform-google-modules/service-accounts/google" - version = "~> 4.2" - - billing_account_id = var.billing_account_id - description = local.description - display_name = local.display_name - generate_keys = var.generate_keys - grant_billing_role = var.grant_billing_role - grant_xpn_roles = var.grant_xpn_roles - names = [var.name] - org_id = var.org_id - prefix = var.deployment_name - project_id = var.project_id - project_roles = [for role in var.project_roles : "${var.project_id}=>roles/${role}"] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/metadata.yaml deleted file mode 100644 index c4dcdffdf4..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/outputs.tf deleted file mode 100644 index f9c9be05c8..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/outputs.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "key" { - description = "Service account key (if creation was requested)" - value = module.service_account.key -} - -output "service_account_email" { - description = "Service account e-mail address" - value = module.service_account.email - depends_on = [ - module.service_account, - ] -} - -output "service_account_iam_email" { - description = "Service account IAM binding format (serviceAccount:name@example.com)" - value = module.service_account.iam_email - depends_on = [ - module.service_account, - ] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/variables.tf deleted file mode 100644 index 53267f47e7..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/variables.tf +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "billing_account_id" { - description = "If assigning billing role, specify a billing account (default is to assign at the organizational level)." - type = string - default = "" -} - -variable "deployment_name" { - description = "Name of the deployment (will be prepended to service account name)" - type = string -} - -variable "description" { - description = "Description of the created service account." - type = string - default = "Service Account" -} - -# tflint-ignore: terraform_unused_declarations -variable "descriptions" { - description = "Deprecated; create single service accounts using var.description." - type = list(string) - default = null - - validation { - condition = var.descriptions == null - error_message = "var.descriptions has been deprecated in favor of creating single accounts with var.description" - } -} - -variable "display_name" { - description = "Display name of the created service account." - type = string - default = "Service Account" -} - -variable "generate_keys" { - description = "Generate keys for service account." - type = bool - default = false -} - -variable "grant_billing_role" { - description = "Grant billing user role." - type = bool - default = false -} - -variable "grant_xpn_roles" { - description = "Grant roles for shared VPC management." - type = bool - default = true -} - -variable "name" { - description = "Name of the service account to create." - type = string -} - -# tflint-ignore: terraform_unused_declarations -variable "names" { - description = "Deprecated; create single service accounts using var.name." - type = list(string) - default = null - - validation { - condition = var.names == null - error_message = "var.names has been deprecated in favor of creating single accounts with var.name" - } -} - -variable "org_id" { - description = "Id of the organization for org-level roles." - type = string - default = "" -} - -# tflint-ignore: terraform_unused_declarations -variable "prefix" { - description = "Deprecated; prefix now set using var.deployment_name" - type = string - default = null - - validation { - condition = var.prefix == null - error_message = "var.prefix has been deprecated in favor of setting prefix with var.deployment_name" - } -} - -variable "project_id" { - description = "ID of the project" - type = string -} - -variable "project_roles" { - description = "List of roles to grant to service account (e.g. \"storage.objectViewer\" or \"compute.instanceAdmin.v1\"" - type = list(string) -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-account/versions.tf deleted file mode 100644 index 38e6e71945..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/service-account/versions.tf +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/README.md b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/README.md deleted file mode 100644 index 266eac26ec..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/README.md +++ /dev/null @@ -1,70 +0,0 @@ -## Description - -Allows management of multiple API services for a Google Cloud Platform project. - -### Example - -```yaml -- id: services-api - source: community/modules/project/service-enablement - settings: - gcp_service_list: [ - "file.googleapis.com", - "compute.googleapis.com" - ] -``` - -This allows the project to enable both the filestore API as well as the compute API. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_project_service.gcp_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [disable\_on\_destroy](#input\_disable\_on\_destroy) | Disable services on destroy if they were enabled (or already enabled) during apply (default: false) | `bool` | `false` | no | -| [gcp\_service\_list](#input\_gcp\_service\_list) | list of APIs to be enabled for the project | `list(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/main.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/main.tf deleted file mode 100644 index 965e93c549..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/main.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -resource "google_project_service" "gcp_services" { - count = length(var.gcp_service_list) - project = var.project_id - service = var.gcp_service_list[count.index] - timeouts { - create = "30m" - update = "40m" - } - - disable_dependent_services = true - disable_on_destroy = var.disable_on_destroy -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/metadata.yaml deleted file mode 100644 index c594c8f819..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - serviceusage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/variables.tf deleted file mode 100644 index 08f13999fe..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/variables.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "ID of the project" - type = string -} - -variable "gcp_service_list" { - description = "list of APIs to be enabled for the project" - type = list(string) -} - -variable "disable_on_destroy" { - description = "Disable services on destroy if they were enabled (or already enabled) during apply (default: false)" - type = bool - default = false -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/versions.tf deleted file mode 100644 index 07f25fb045..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/project/service-enablement/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/README.md b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/README.md deleted file mode 100644 index 052e6aee23..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Description - -This module creates a Bigquery Pub/Sub Subscription. - -Primarily used for FSI - MonteCarlo Tutorial: -**[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Example - -The following example creates a Bigquery subscription using a Bigquery table and -Pub/Sub topic. - -```yaml - - id: bq_subscription - source: community/modules/pubsub/bigquery-sub - use: [bq-table, pubsub_topic] -``` - -Also see usages in this -[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_project_iam_member.editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_iam_member.viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_pubsub_subscription.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Name of the dataset that was created. Can be provided by the bigquery-table module | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [subscription\_id](#input\_subscription\_id) | The name of the pubsub subscription to be created | `string` | `null` | no | -| [table\_id](#input\_table\_id) | ID of created BQ table. Can be provided by the bigquery-table module | `string` | n/a | yes | -| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [subscription\_id](#output\_subscription\_id) | Name of the subscription that was created. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf deleted file mode 100644 index 8edbc6b24e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-sub", ghpc_role = "pubsub" }) -} - -locals { - subscription_id = var.subscription_id != null ? var.subscription_id : "${var.deployment_name}_subscription_${random_id.resource_name_suffix.hex}" -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} -data "google_project" "project" { - project_id = var.project_id -} - -resource "google_project_iam_member" "viewer" { - project = data.google_project.project.project_id - role = "roles/bigquery.metadataViewer" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" -} - -resource "google_project_iam_member" "editor" { - project = data.google_project.project.project_id - role = "roles/bigquery.dataEditor" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" -} - -resource "google_pubsub_subscription" "example" { - depends_on = [google_project_iam_member.editor, google_project_iam_member.viewer] - name = local.subscription_id - topic = var.topic_id - project = var.project_id - labels = local.labels - bigquery_config { - table = "${var.project_id}.${var.dataset_id}.${var.table_id}" - use_topic_schema = true - write_metadata = true - } - -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml deleted file mode 100644 index 9aedef48dc..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - pubsub.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf deleted file mode 100644 index fc81859503..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "subscription_id" { - description = "Name of the subscription that was created." - value = google_pubsub_subscription.example.name -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf deleted file mode 100644 index ee4dbbed8e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "topic_id" { - description = "The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module" - type = string -} - -variable "subscription_id" { - description = "The name of the pubsub subscription to be created" - type = string - default = null -} - -variable "dataset_id" { - description = "Name of the dataset that was created. Can be provided by the bigquery-table module" - type = string -} - -variable "table_id" { - description = "ID of created BQ table. Can be provided by the bigquery-table module" - type = string -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf deleted file mode 100644 index 46ad6e17c8..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" - } - required_version = ">= 1.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/README.md b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/README.md deleted file mode 100644 index 177f799dc6..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/README.md +++ /dev/null @@ -1,82 +0,0 @@ -## Description - -Creates a Pub/Sub topic - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -### Example - -The following example creates a Pub/Sub topic. - -```yaml - - id: pubsub_topic - source: community/modules/pubsub/topic -``` - -Also see usages in this -[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_pubsub_schema.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_schema) | resource | -| [google_pubsub_topic.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [schema\_id](#input\_schema\_id) | The name of the pubsub schema to be created | `string` | `null` | no | -| [schema\_json](#input\_schema\_json) | The JSON definition of the pubsub topic schema | `string` | `"{ \n \"name\" : \"Avro\", \n \"type\" : \"record\", \n \"fields\" : \n [\n {\"name\" : \"ticker\", \"type\" : \"string\"},\n {\"name\" : \"epoch_time\", \"type\" : \"int\"},\n {\"name\" : \"iteration\", \"type\" : \"int\"},\n {\"name\" : \"start_date\", \"type\" : \"string\"},\n {\"name\" : \"end_date\", \"type\" : \"string\"},\n {\n \"name\":\"simulation_results\",\n \"type\":{\n \"type\": \"array\", \n \"items\":{\n \"name\":\"Child\",\n \"type\":\"record\",\n \"fields\":[\n {\"name\":\"price\", \"type\":\"double\"}\n ]\n }\n }\n }\n ]\n }\n"` | no | -| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to be created | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [topic\_id](#output\_topic\_id) | Name of the topic that was created. | -| [topic\_schema](#output\_topic\_schema) | Name of the topic schema that was created. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/main.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/main.tf deleted file mode 100644 index 4ba68fb5d0..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/main.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "topic", ghpc_role = "pubsub" }) -} - -locals { - topic_id = var.topic_id != null ? var.topic_id : "${var.deployment_name}_topic_${random_id.resource_name_suffix.hex}" - schema_id = var.schema_id != null ? var.schema_id : "${var.deployment_name}_schema_${random_id.resource_name_suffix.hex}" -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_pubsub_topic" "example" { - name = local.topic_id - depends_on = [google_pubsub_schema.example] - project = var.project_id - labels = local.labels - schema_settings { - schema = "projects/${var.project_id}/schemas/${local.schema_id}" - encoding = "BINARY" - } -} - -resource "google_pubsub_schema" "example" { - name = local.schema_id - project = var.project_id - type = "AVRO" - - definition = var.schema_json -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/metadata.yaml deleted file mode 100644 index 9aedef48dc..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - pubsub.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/outputs.tf deleted file mode 100644 index 3ea9d951b2..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/outputs.tf +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "topic_id" { - description = "Name of the topic that was created." - value = google_pubsub_topic.example.name -} - - -output "topic_schema" { - description = "Name of the topic schema that was created." - value = local.schema_id -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/variables.tf deleted file mode 100644 index dca575d21d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/pubsub/topic/variables.tf +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "topic_id" { - description = "The name of the pubsub topic to be created" - type = string - default = null -} - -variable "schema_id" { - description = "The name of the pubsub schema to be created" - type = string - default = null -} - -variable "schema_json" { - description = "The JSON definition of the pubsub topic schema" - type = string - default = < **Note**: This is an experimental module. This module has only been tested in -> limited capacity with the Cluster Toolkit. The module interface may have undergo -> breaking changes in the future. - -### Example - -The following example will create a single GPU accelerated remote desktop. - -```yaml - - id: remote-desktop - source: community/modules/remote-desktop/chrome-remote-desktop - use: [network1] - settings: - install_nvidia_driver: true -``` - -### Setting up the Remote Desktop - -1. Once the remote desktop has been deployed, navigate to https://remotedesktop.google.com/headless. -1. Click through `Begin`, `Next`, & `Authorize`. -1. Copy the code snippet for `Debian Linux`. -1. SSH into the remote desktop machine. It will be listed under - [VM Instances](https://console.cloud.google.com/compute/instances) in the - Google Cloud web console. -1. Run the copied command and follow instructions to set up a PIN. -1. You should now see your machine listed on the - [Chrome Remote Desktop page](https://remotedesktop.google.com/access) under `Remote devices`. -1. Click on your machine and enter PIN if prompted. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.12.31 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [client\_startup\_script](#module\_client\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | -| [instances](#module\_instances) | ../../../../modules/compute/vm-instance | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | -| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image th
at supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-v
m-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. Cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | -| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-balanced"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs on the internet. | `bool` | `true` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required | 
list(object({
    type  = string,
    count = number
  }))
| 
[
  {
    "count": 1,
    "type": "nvidia-tesla-t4-vws"
  }
]
| no | -| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Installs the nvidia driver (true/false). For details, see https://cloud.google.com/compute/docs/gpus/install-drivers-gpu | `bool` | n/a | yes | -| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | -| [instance\_image](#input\_instance\_image) | Image used to build chrome remote desktop node. The default image is
name="debian-12-bookworm-v20250610" and project="debian-cloud".
NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues.

An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud".

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "name": "debian-12-bookworm-v20250610",
  "project": "debian-cloud"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation. Must be N1 family if GPU is used. | `string` | `"n1-standard-8"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface
**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.
Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "RDMA", "IRDMA", "MRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `"TERMINATE"` | no | -| [project\_id](#input\_project\_id) | Project in which Google Cloud resources will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/cloud-platform"
  ]
}
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core | `number` | `2` | no | -| [zone](#input\_zone) | Default zone for creating resources | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_name](#output\_instance\_name) | Name of the first instance created, if any. | -| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf deleted file mode 100644 index a5cf7c5d37..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "chrome-remote-desktop", ghpc_role = "remote-desktop" }) -} - -locals { - - user_startup_script_runners = var.startup_script == null ? [] : [ - { - type = "shell" - content = var.startup_script - destination = "user_startup_script.sh" - } - ] - - configure_nvidia_driver_runners = var.install_nvidia_driver == false ? [] : [ - { - type = "ansible-local" - content = file("${path.module}/scripts/configure-grid-drivers.yml") - destination = "/usr/local/ghpc/configure-grid-drivers.yml" - } - ] - - configure_chrome_remote_desktop_runners = [ - { - type = "ansible-local" - content = file("${path.module}/scripts/configure-chrome-desktop.yml") - destination = "/usr/local/ghpc/configure-chrome-desktop.yml" - } - ] - - disable_sleep = [ - { - type = "ansible-local" - content = file("${path.module}/scripts/disable-sleep.yml") - destination = "/usr/local/ghpc/disable-sleep.yml" - } - ] -} - -module "client_startup_script" { - source = "../../../../modules/scripts/startup-script" - - deployment_name = var.deployment_name - project_id = var.project_id - region = var.region - labels = local.labels - - runners = flatten([ - local.user_startup_script_runners, - local.configure_nvidia_driver_runners, - local.configure_chrome_remote_desktop_runners, - local.disable_sleep - ]) -} - -module "instances" { - source = "../../../../modules/compute/vm-instance" - - instance_count = var.instance_count - name_prefix = var.name_prefix - add_deployment_name_before_prefix = var.add_deployment_name_before_prefix - provisioning_model = var.spot ? "SPOT" : null - - deployment_name = var.deployment_name - project_id = var.project_id - region = var.region - zone = var.zone - labels = local.labels - - machine_type = var.machine_type - service_account_email = var.service_account.email - metadata = var.metadata - startup_script = module.client_startup_script.startup_script - enable_oslogin = var.enable_oslogin - - instance_image = var.instance_image - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - auto_delete_boot_disk = var.auto_delete_boot_disk - - disable_public_ips = !var.enable_public_ips - network_self_link = var.network_self_link - subnetwork_self_link = var.subnetwork_self_link - network_interfaces = var.network_interfaces - bandwidth_tier = var.bandwidth_tier - tags = var.tags - - threads_per_core = var.threads_per_core - guest_accelerator = var.guest_accelerator - on_host_maintenance = var.on_host_maintenance - - network_storage = var.network_storage - -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf deleted file mode 100644 index bcf8ece52d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "script to load and run all runners, as a string value." - value = module.client_startup_script.startup_script -} - -output "instance_name" { - description = "Name of the first instance created, if any." - value = var.instance_count > 0 ? module.instances.name[0] : null -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml deleted file mode 100644 index 391aa86433..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Ensure Desktop OS and Chrome Remote Desktop is installed - hosts: localhost - become: true - module_defaults: - ansible.builtin.apt: - update_cache: true - cache_valid_time: 3600 - tasks: - - name: Install desktop packages - ansible.builtin.apt: - name: - - xfce4 - - xfce4-goodies - state: present - register: apt_result - retries: 10 - delay: 30 - until: apt_result is success - - - name: Download and configure CRD - ansible.builtin.get_url: - url: https://dl.google.com/linux/direct/chrome-remote-desktop_current_amd64.deb - dest: /tmp/chrome-remote-desktop_current_amd64.deb - mode: "0755" - timeout: 30 - - - name: Install CRD - ansible.builtin.apt: - deb: /tmp/chrome-remote-desktop_current_amd64.deb - environment: - DEBIAN_FRONTEND: noninteractive - register: apt_result - retries: 10 - delay: 30 - until: apt_result is success - - - name: Configure CRD to use Xfce by default - ansible.builtin.copy: - dest: /etc/chrome-remote-desktop-session - content: "exec /etc/X11/Xsession /usr/bin/xfce4-session" - mode: 0644 - - - name: Start Chrome remote desktop - ansible.builtin.command: /etc/init.d/chrome-remote-desktop start - register: result - changed_when: result.rc == 0 diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml deleted file mode 100644 index daae08176d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Ensure nvidia grid drivers and other binaries are installed - hosts: localhost - become: true - vars: - dist_settings: - bullseye: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-510.85.02-grid.run - grid_ver: vGPU14.2 - bookworm: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-550.54.15-grid.run - grid_ver: vGPU17.1 - jammy: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - gcc-12 # must match compiler used to build kernel on latest Ubuntu 22 - - pkg-config # observed to be necessary for GRID driver installation on latest Ubuntu 22 - - libglvnd-dev # observed to be necessary for GRID driver installation on latest Ubuntu 22 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-525.125.06-grid.run - grid_ver: vGPU15.3 - tasks: - - name: Fail if using wrong OS - ansible.builtin.assert: - that: - - ansible_os_family in ["Debian", "Ubuntu"] - - ansible_distribution_release in dist_settings.keys() | list - fail_msg: "ansible_os_family: {{ ansible_os_family }} or ansible_distribution_release: {{ansible_distribution_release}} was not acceptable." - - - name: Check if GRID driver installed - ansible.builtin.command: which nvidia-smi - register: nvidiasmi_result - ignore_errors: true - changed_when: false - - - name: Install binaries for GRID drivers - ansible.builtin.apt: - name: '{{ dist_settings[ansible_distribution_release]["packages"] }}' - state: present - update_cache: true - register: apt_result - retries: 6 - delay: 10 - until: apt_result is success - - - name: Install GRID driver if not existing - when: nvidiasmi_result is failed - block: - - name: Download GPU driver - ansible.builtin.get_url: - url: https://storage.googleapis.com/nvidia-drivers-us-public/GRID/{{ dist_settings[ansible_distribution_release]["grid_ver"] }}/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} - dest: /tmp/ - mode: "0755" - timeout: 30 - - - name: Stop gdm service - ansible.builtin.systemd: - name: gdm - state: stopped - - - name: Install GPU driver - ansible.builtin.shell: | - #jinja2: trim_blocks: "True" - {% if ansible_distribution_release == "jammy" %} - CC=gcc-12 /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent - {% else %} - /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent - {% endif %} - register: result - changed_when: result.rc == 0 - - - name: Download VirtualGL driver - ansible.builtin.get_url: - url: https://sourceforge.net/projects/virtualgl/files/3.0.2/virtualgl_3.0.2_amd64.deb/download - dest: /tmp/virtualgl_3.0.2_amd64.deb - mode: "0755" - timeout: 30 - - - name: Install VirtualGL - ansible.builtin.command: gdebi /tmp/virtualgl_3.0.2_amd64.deb --non-interactive - register: result - changed_when: result.rc == 0 - - - name: Fix headless Nvidia issue - block: - - name: Lookup gpu info - ansible.builtin.command: nvidia-xconfig --query-gpu-info - register: gpu_info - failed_when: gpu_info.rc != 0 - changed_when: false - - - name: Extract PCI ID - ansible.builtin.shell: | - set -o pipefail - echo "{{ gpu_info.stdout }}" | grep "PCI BusID " | head -n 1 | cut -d':' -f2-99 | xargs - args: - executable: /bin/bash - register: pci_id - changed_when: false - - - name: Configure nvidia-xconfig - ansible.builtin.command: nvidia-xconfig -a --allow-empty-initial-configuration --enable-all-gpus --virtual=1920x1200 --busid={{ pci_id.stdout }} - register: result - changed_when: result.rc == 0 - - - name: Set HardDPMS to false - ansible.builtin.replace: - path: /etc/X11/xorg.conf - regexp: "Section \"Device\"" - replace: "Section \"Device\"\n Option \"HardDPMS\" \"false\"" - - - name: Configure VirtualGL for X - ansible.builtin.command: vglserver_config +glx +s +f -t - register: result - changed_when: result.rc == 0 - - - name: Configure gdm for X - block: - - name: Configure default display manager - ansible.builtin.copy: - dest: /etc/X11/default-display-manager - content: "/usr/sbin/gdm3" - mode: 0644 - - - name: Switch boot target to gui - ansible.builtin.command: systemctl set-default graphical.target - register: result - changed_when: result.rc == 0 - - - name: Start gdm service - ansible.builtin.systemd: - name: gdm - daemon_reload: true - state: started diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml deleted file mode 100644 index 6767b05fb2..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Mask sleep, suspend, hibernate, and hybrid-sleep targets - hosts: localhost - become: true - tasks: - - - name: Mask sleep target - ansible.builtin.systemd: - name: sleep.target - masked: true - - - name: Mask suspend target - ansible.builtin.systemd: - name: suspend.target - masked: true - - - name: Mask hibernate target - ansible.builtin.systemd: - name: hibernate.target - masked: true - - - name: Mask hybrid-sleep target - ansible.builtin.systemd: - name: hybrid-sleep.target - masked: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf deleted file mode 100644 index ac4c3b1869..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf +++ /dev/null @@ -1,277 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which Google Cloud resources will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. Cloud resource names will include this value." - type = string - #default = "chrome-remote-desktop" -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zone" { - description = "Default zone for creating resources" - type = string -} - -variable "instance_count" { - description = "Number of instances" - type = number - default = 1 -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "instance_image" { - description = <<-EOD - Image used to build chrome remote desktop node. The default image is - name="debian-12-bookworm-v20250610" and project="debian-cloud". - NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues. - - An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud". - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "debian-cloud" - name = "debian-12-bookworm-v20250610" - } -} - -variable "disk_size_gb" { - description = "Size of disk for instances." - type = number - default = 200 -} - -variable "disk_type" { - description = "Disk type for instances." - type = string - default = "pd-balanced" -} - -variable "auto_delete_boot_disk" { - description = "Controls if boot disk should be auto-deleted when instance is deleted." - type = bool - default = true -} - -variable "name_prefix" { - description = <<-EOT - An optional name for all VM and disk resources. - If not supplied, `deployment_name` will be used. - When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, - then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". - EOT - type = string - default = null -} - -variable "add_deployment_name_before_prefix" { - description = <<-EOT - If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. - See `name_prefix` for further details on resource naming behavior. - EOT - type = bool - default = false -} - -variable "enable_public_ips" { - description = "If set to true, instances will have public IPs on the internet." - type = bool - default = true -} - -variable "machine_type" { - description = "Machine type to use for the instance creation. Must be N1 family if GPU is used." - type = string - default = "n1-standard-8" -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "service_account" { - description = "Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account." - type = object({ - email = string, - scopes = set(string) - }) - default = { - email = null - scopes = [ - "https://www.googleapis.com/auth/cloud-platform", - ] - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the VM." - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the VM." - type = string - default = null -} - -variable "network_interfaces" { - description = <<-EOT - A list of network interfaces. The options match that of the terraform - network_interface block of google_compute_instance. For descriptions of the - subfields or more information see the documentation: - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - **_NOTE:_** If `network_interfaces` are set, `network_self_link` and - `subnetwork_self_link` will be ignored, even if they are provided through - the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply - to network interfaces defined in this variable. - Subfields: - network (string, required if subnetwork is not supplied) - subnetwork (string, required if network is not supplied) - subnetwork_project (string, optional) - network_ip (string, optional) - nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "RDMA", "IRDMA", "MRDMA"]) - stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) - queue_count (number, optional) - access_config (object, optional) - ipv6_access_config (object, optional) - alias_ip_range (list(object), optional) - EOT - type = list(object({ - network = string, - subnetwork = string, - subnetwork_project = string, - network_ip = string, - nic_type = string, - stack_type = string, - queue_count = number, - access_config = list(object({ - nat_ip = string, - public_ptr_domain_name = string, - network_tier = string - })), - ipv6_access_config = list(object({ - public_ptr_domain_name = string, - network_tier = string - })), - alias_ip_range = list(object({ - ip_cidr_range = string, - subnetwork_range_name = string - })) - })) - default = [] -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "startup_script" { - description = "Startup script used on the instance" - type = string - default = null -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required" - type = list(object({ - type = string, - count = number - })) - default = [{ - type = "nvidia-tesla-t4-vws" - count = 1 - }] -} - -variable "threads_per_core" { - description = "Sets the number of threads per physical core" - type = number - default = 2 -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" - type = string - default = "TERMINATE" -} - -variable "bandwidth_tier" { - description = <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -This mode can be switched to proactive (automatic) replacement by setting -[var.update_policy](#input_update_policy) to "PROACTIVE". In this case we -recommend the use of Filestore to store the job queue state ("spool") and -setting [var.spool_parent_dir][#input_spool_parent_dir] to its mount point: - -```yaml - - id: spoolfs - source: modules/file-system/filestore - use: - - network1 - settings: - filestore_tier: ENTERPRISE - local_mount: /shared - -... - - - id: htcondor_access - source: community/modules/scheduler/htcondor-access-point - use: - - network1 - - spoolfs - - htcondor_secrets - - htcondor_setup - - htcondor_cm - - htcondor_execute_point_group - settings: - spool_parent_dir: /shared -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 3.83 | -| [null](#requirement\_null) | >= 3.0 | -| [random](#requirement\_random) | ~> 3.6 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [null](#provider\_null) | >= 3.0 | -| [random](#provider\_random) | ~> 3.6 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [access\_point\_instance\_template](#module\_access\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [htcondor\_ap](#module\_htcondor\_ap) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_address.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_region_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_region_disk) | resource | -| [google_storage_bucket_object.ap_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.ap_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_shuffle.zones](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/shuffle) | resource | -| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_instance.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | -| [google_compute_region_instance_group.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_runner](#input\_access\_point\_runner) | A list of Toolkit runners for configuring an HTCondor access point | `list(map(string))` | `[]` | no | -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account for access point (e-mail format) | `string` | n/a | yes | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [autoscaler\_runner](#input\_autoscaler\_runner) | A list of Toolkit runners for configuring autoscaling daemons | `list(map(string))` | `[]` | no | -| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | -| [default\_mig\_id](#input\_default\_mig\_id) | Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | -| [disk\_type](#input\_disk\_type) | Boot disk size in GB | `string` | `"pd-balanced"` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape acoss zones for instance group managing high availability of access point | `string` | `"ANY_SINGLE_ZONE"` | no | -| [enable\_high\_availability](#input\_enable\_high\_availability) | Provision HTCondor access point in high availability mode | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | Enable Public IPs on the access points | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor and Toolkit support installed."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | -| [mig\_id](#input\_mig\_id) | List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module) | `list(string)` | `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spool\_disk\_size\_gb](#input\_spool\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | -| [spool\_disk\_type](#input\_spool\_disk\_type) | Boot disk size in GB | `string` | `"pd-ssd"` | no | -| [spool\_parent\_dir](#input\_spool\_parent\_dir) | HTCondor access point configuration SPOOL will be set to subdirectory named "spool" | `string` | `"/var/lib/condor"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | -| [zones](#input\_zones) | Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_ips](#output\_access\_point\_ips) | IP addresses of the access points provisioned by this module | -| [access\_point\_name](#output\_access\_point\_name) | Name of the access point provisioned by this module | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml deleted file mode 100644 index 6a2f50c831..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Access Point - hosts: localhost - become: true - vars: - spool_dir: /var/lib/condor/spool - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - htcondor_spool_disk_device: /dev/disk/by-id/google-htcondor-spool-disk - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - - name: Configure HTCondor SchedD - when: htcondor_role == 'get_htcondor_submit' - block: - - name: Format spool disk - community.general.filesystem: - fstype: ext4 - state: present - dev: "{{ htcondor_spool_disk_device }}" - # RUN TUNE2FS - - name: Mount spool (creates mount point) - ansible.posix.mount: - path: "{{ spool_dir }}" - src: "{{ htcondor_spool_disk_device }}" - fstype: ext4 - opts: defaults - state: mounted - - name: Ensure spool free space - ansible.builtin.command: tune2fs -r 0 {{ htcondor_spool_disk_device }} - - name: Setup spool directory - ansible.builtin.file: - path: "{{ spool_dir }}" - state: directory - owner: condor - group: condor - mode: 0755 - recurse: true - - name: Create SystemD override directory for HTCondor - ansible.builtin.file: - path: /etc/systemd/system/condor.service.d - state: directory - owner: root - group: root - mode: 0755 - - name: Ensure HTCondor starts after shared filesystem is mounted - ansible.builtin.copy: - dest: /etc/systemd/system/condor.service.d/mount-spool.conf - mode: 0644 - content: | - [Unit] - RequiresMountsFor={{ spool_dir }} - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor configuration complete; startup-script may still be executing ********" diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf deleted file mode 100644 index fdbcf5c32f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf +++ /dev/null @@ -1,338 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-access-point", ghpc_role = "scheduler" }) -} - -locals { - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - metadata = merge( - local.network_storage_metadata, - local.enable_oslogin_metadata, - local.disable_automatic_updates_metadata, - var.metadata - ) - - host_count = 1 - name_prefix = "${var.deployment_name}-ap" - - example_runner = { - type = "data" - destination = "/var/tmp/helloworld.sub" - content = <<-EOT - universe = vanilla - executable = /bin/sleep - arguments = 1000 - output = out.$(ClusterId).$(ProcId) - error = err.$(ClusterId).$(ProcId) - log = log.$(ClusterId).$(ProcId) - request_cpus = 1 - request_memory = 100MB - queue - EOT - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.access_point_runner, - [local.schedd_runner], - var.autoscaler_runner, - [local.example_runner] - ) - - ap_config = templatefile("${path.module}/templates/condor_config.tftpl", { - htcondor_role = "get_htcondor_submit", - central_manager_ips = var.central_manager_ips - spool_dir = "${var.spool_parent_dir}/spool", - mig_ids = var.mig_id, - default_mig_id = var.default_mig_id - }) - - ap_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.ap_config.output_name}" - schedd_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e htcondor_role=get_htcondor_submit", - "-e config_object=${local.ap_object}", - "-e spool_dir=${var.spool_parent_dir}/spool", - "-e htcondor_spool_disk_device=/dev/disk/by-id/google-${local.spool_disk_device_name}", - ]) - } - - access_point_ips = google_compute_address.ap.address - access_point_name = data.google_compute_instance.ap.name - - spool_disk_resource_name = "${var.deployment_name}-spool-disk" - spool_disk_device_name = "htcondor-spool-disk" - spool_disk_source = try(google_compute_disk.spool[0].name, google_compute_region_disk.spool[0].self_link) - - zones = coalescelist(var.zones, random_shuffle.zones.result) - - vm_family = split("-", var.machine_type)[0] - regional_pd_families = ["e2", "n1", "n2", "n2d"] -} - -data "google_compute_image" "htcondor" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - postcondition { - condition = self.disk_size_gb <= var.disk_size_gb - error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region - - lifecycle { - postcondition { - condition = alltrue([ - for z in var.zones : contains(self.names, z) - ]) - error_message = "Each entry in var.zones must be a zone in var.region: ${var.region}" - } - } -} - -resource "random_shuffle" "zones" { - input = data.google_compute_zones.available.names - result_count = var.enable_high_availability ? 2 : 1 -} - -data "google_compute_region_instance_group" "ap" { - self_link = module.htcondor_ap.self_link - lifecycle { - postcondition { - condition = length(self.instances) == local.host_count - error_message = "There should be ${local.host_count} access points found" - } - } -} - -data "google_compute_instance" "ap" { - self_link = data.google_compute_region_instance_group.ap.instances[0].instance -} - -resource "null_resource" "ap_config" { - triggers = { - config = local.ap_config - } -} - -resource "google_storage_bucket_object" "ap_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.ap_config.id), 0, 4)}" - content = local.ap_config - bucket = var.htcondor_bucket_name - - lifecycle { - precondition { - condition = var.default_mig_id == "" || contains(var.mig_id, var.default_mig_id) - error_message = "If set, var.default_mig_id must be an element in var.mig_id" - } - - # by construction, this precondition only fails when the user has set - # var.zones to a non-empty list of length not equal to 2 - precondition { - condition = !var.enable_high_availability || length(local.zones) == 2 - error_message = "When using HTCondor access point high availability, var.zones must be of length 2." - } - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -resource "google_compute_region_disk" "spool" { - count = var.enable_high_availability ? 1 : 0 - name = local.spool_disk_resource_name - labels = local.labels - type = var.spool_disk_type - region = var.region - size = var.spool_disk_size_gb - - replica_zones = local.zones - - lifecycle { - precondition { - condition = var.spool_disk_size_gb >= 200 - error_message = "When using HTCondor access point high availability, var.spool_disk_size_gb must be set to 200 or greater." - } - - precondition { - condition = contains(local.regional_pd_families, local.vm_family) - error_message = "When using HTCondor access point high availability, var.machine_type must be one of ${jsonencode(local.regional_pd_families)}." - } - } -} - -resource "google_compute_disk" "spool" { - count = var.enable_high_availability ? 0 : 1 - name = local.spool_disk_resource_name - labels = local.labels - type = var.spool_disk_type - zone = local.zones[0] - size = var.spool_disk_size_gb -} - -resource "google_compute_address" "ap" { - project = var.project_id - name = local.name_prefix - region = var.region - subnetwork = var.subnetwork_self_link - address_type = "INTERNAL" - purpose = "GCE_ENDPOINT" -} - -module "access_point_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.access_point_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - preemptible = false - startup_script = module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.htcondor.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - network_ip = google_compute_address.ap.id - - # spool disk - additional_disks = [ - { - source = local.spool_disk_source - device_name = local.spool_disk_device_name - } - ] -} - -module "htcondor_ap" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = local.host_count - hostname = local.name_prefix - instance_template = module.access_point_instance_template.self_link - - health_check_name = "health-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) - max_surge_fixed = 0 # must be 0 to preserve hostnames - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - - stateful_disks = [{ - device_name = local.spool_disk_device_name - delete_rule = "ON_PERMANENT_INSTANCE_DELETION" - }] - stateful_ips = var.enable_public_ips ? [{ - interface_name = "nic0" - delete_rule = "ON_PERMANENT_INSTANCE_DELETION" - is_external = true - }] : [] - - # the timeouts below are default for resource - wait_for_instances = true - mig_timeouts = { - create = "15m" - delete = "15m" - update = "15m" - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf deleted file mode 100644 index f7424c6d5d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "access_point_ips" { - description = "IP addresses of the access points provisioned by this module" - value = local.access_point_ips -} - -output "access_point_name" { - description = "Name of the access point provisioned by this module" - value = local.access_point_name -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl deleted file mode 100644 index 214fbc726f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:${htcondor_role} -CONDOR_HOST = ${join(",", central_manager_ips)} - -SPOOL = ${spool_dir} -SCHEDD_INTERVAL = 30 -TRUST_UID_DOMAIN = True -SUBMIT_ATTRS = RunAsOwner -RunAsOwner = True - -# When a job matches to a machine, add machine attributes to the job for -# condor_history (e.g. VM Instance ID) -use feature:JobsHaveInstanceIDs -SYSTEM_JOB_MACHINE_ATTRS = $(SYSTEM_JOB_MACHINE_ATTRS) \ - CloudVMType CloudZone CloudInterruptible -SYSTEM_JOB_MACHINE_ATTRS_HISTORY_LENGTH = 10 - -# Add Cloud attributes to SchedD ClassAd -use feature:ScheddCronOneShot(cloud, $(LIBEXEC)/common-cloud-attributes-google.py) -SCHEDD_CRON_cloud_PREFIX = Cloud - -# aid the user by automatically using RequireSpot in their Requirements, unless -# the user has explicitly used CloudInterruptible -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SPOT -JOB_TRANSFORM_SPOT @=end - REQUIREMENTS ! isUndefined(RequireSpot) && ! unresolved(Requirements, "^CloudInterruptible$") - SET Requirements ($(MY.Requirements)) && (CloudInterruptible is My.RequireSpot) -@end - -# help the user by enforcing that RequireSpot is undefined or a boolean -SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) SPOT -SUBMIT_REQUIREMENT_SPOT = isUndefined(RequireSpot) || isBoolean(RequireSpot) -SUBMIT_REQUIREMENT_SPOT_REASON = "If +RequireSpot is defined, it must be either True or False" - -%{ if length(mig_ids) > 0 ~} -MIG_IDS = "${join(" ", mig_ids)}" -MIG_ID_LIST = split($(MIG_IDS)) -%{ if default_mig_id != "" ~} -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) ID_DEFAULT -JOB_TRANSFORM_ID_DEFAULT @=end - DEFAULT RequireId "${default_mig_id}" -@end -%{ endif ~} -SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) MIGID -SUBMIT_REQUIREMENT_MIGID = !isUndefined(RequireId) && member(RequireId, $(MIG_ID_LIST)) -SUBMIT_REQUIREMENT_MIGID_REASON = strcat("Jobs must set +RequireId to one of following values surrounded by quotation marks:\n", $(MIG_IDS)) - -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) MIGID -JOB_TRANSFORM_MIGID @=end - REQUIREMENTS ! isUndefined(RequireId) && ! unresolved(Requirements, "^CloudCreatedBy$") - SET Requirements ($(MY.Requirements)) && regexp(strcat("/", My.RequireId, "$"), CloudCreatedBy) -@end -%{ endif ~} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf deleted file mode 100644 index f54a88ac2e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zones" { - description = "Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region." - type = list(string) - default = [] - nullable = false - - validation { - condition = length(var.zones) <= 2 - error_message = "Set var.zones to the empty list or up to 2 zones in var.region" - } -} - -variable "distribution_policy_target_shape" { - description = "Target shape acoss zones for instance group managing high availability of access point" - type = string - default = "ANY_SINGLE_ZONE" -} - -variable "network_self_link" { - description = "The self link of the network in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "access_point_service_account_email" { - description = "Service account for access point (e-mail format)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 32 - nullable = false -} - -variable "disk_type" { - description = "Boot disk size in GB" - type = string - default = "pd-balanced" - nullable = false -} - -variable "spool_disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 32 - nullable = false -} - -variable "spool_disk_type" { - description = "Boot disk size in GB" - type = string - default = "pd-ssd" - nullable = false -} - -variable "metadata" { - description = "Metadata to add to HTCondor central managers" - type = map(string) - default = {} -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - nullable = false - validation { - condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "enable_high_availability" { - description = "Provision HTCondor access point in high availability mode" - type = bool - default = false -} - -variable "instance_image" { - description = <<-EOD - Custom VM image with HTCondor and Toolkit support installed." - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "machine_type" { - description = "Machine type to use for HTCondor central managers" - type = string - default = "n2-standard-4" -} - -variable "access_point_runner" { - description = "A list of Toolkit runners for configuring an HTCondor access point" - type = list(map(string)) - default = [] -} - -variable "autoscaler_runner" { - description = "A list of Toolkit runners for configuring autoscaling daemons" - type = list(map(string)) - default = [] -} - -variable "spool_parent_dir" { - description = "HTCondor access point configuration SPOOL will be set to subdirectory named \"spool\"" - type = string - default = "/var/lib/condor" -} - -variable "central_manager_ips" { - description = "List of IP addresses of HTCondor Central Managers" - type = list(string) -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "enable_public_ips" { - description = "Enable Public IPs on the access points" - type = bool - default = false -} - -variable "mig_id" { - description = "List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module)" - type = list(string) - default = [] - nullable = false - - validation { - condition = length(var.mig_id) > 0 - error_message = "At least 1 MIG containing execute points must be provided to this module" - } -} - -variable "default_mig_id" { - description = "Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id" - type = string - default = "" - nullable = false -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" - type = string - default = "OPPORTUNISTIC" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf deleted file mode 100644 index 0d07e7abf1..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.6" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.74.0" - } - - required_version = ">= 1.1" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md deleted file mode 100644 index dfab563a55..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md +++ /dev/null @@ -1,159 +0,0 @@ -## Description - -This module provisions a highly available HTCondor central manager using a [Managed -Instance Group (MIG)][mig] with auto-healing. - -[mig]: https://cloud.google.com/compute/docs/instance-groups - -## Usage - -This module provisions an HTCondor central manager with a standard -configuration. For the node to function correctly, you must supply the input -variable described below: - -- [var.central_manager_runner](#input_central_manager_runner) - - Runner must download a POOL password / signing key and create an [IDTOKEN] - with no scopes (full authorization). - -A reference implementation is included in the Toolkit module -[htcondor-pool-secrets]. You may substitute implementations so long as they -duplicate the functionality in the references. Usage is demonstrated in the -[HTCondor example][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- -[htcondor-pool-secrets]: ../htcondor-pool-secrets/README.md -[IDTOKEN]: https://htcondor.readthedocs.io/en/latest/admin-manual/security.html#introducing-idtokens - -## Behavior of Managed Instance Group (MIG) - -A regional [MIG][mig] is used to provision the central manager, although only -1 node will ever be active at a time. By default, the node will be provisioned -in any of the zones available in that region, however, it can be constrained to -run in fewer zones (or a single zone) using [var.zones](#input_zones). - -When the configuration of the Central Manager is changed, the MIG can be -configured to [replace the VM][replacement] using a "proactive" or -"opportunistic" policy. By default, the Central Manager replacement policy is -set to proactive. In practice, this means that the Central Manager will be -replaced by Terraform when changes to the instance template / HTCondor -configuration are made. The Central Manager is safe to replace automatically as -it gathers its state information from periodic messages exchanged with the rest -of the HTCondor pool. - -This mode can be configured by setting [var.update_policy](#input_update_policy) -to either "PROACTIVE" (default) or "OPPORTUNISTIC". If set to opportunistic -replacement, the Central Manager will be replaced only when: - -- intentionally by issuing an update via Cloud Console or using gcloud (below) -- the VM becomes unhealthy or is otherwise automatically replaced (e.g. regular - Google Cloud maintenance) - -For example, to manually update all instances in a MIG: - -```text -gcloud compute instance-groups managed update-instances \ - <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - -## Limiting inter-zone egress - -Because all the elements of the HTCondor pool use regional MIGs, they may be -subject to [interzone egress fees][network-pricing]. The primary traffic between -nodes of an HTCondor pool running embarrassingly parallel jobs is expected to -be limited to API traffic for job scheduling and monitoring. Please review the -[network pricing][network-pricing] documentation and determine if this cost is -a concern. If it is, use [var.zones](#input_zones) to constrain each node within -your HTCondor pool to operate within a single zone. - -[network-pricing]: https://cloud.google.com/vpc/network-pricing - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1.0 | -| [google](#requirement\_google) | >= 3.83 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [central\_manager\_instance\_template](#module\_central\_manager\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [htcondor\_cm](#module\_htcondor\_cm) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_address.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_storage_bucket_object.cm_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.cm_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_instance.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | -| [google_compute_region_instance_group.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [central\_manager\_runner](#input\_central\_manager\_runner) | A list of Toolkit runners for configuring an HTCondor central manager | `list(map(string))` | `[]` | no | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for central manager (can be supplied by htcondor-setup module) | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `20` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape for instance group managing high availability of central manager | `string` | `"ANY_SINGLE_ZONE"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor installed using the htcondor-install module."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor central manager will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Central Manager ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle). | `string` | `"PROACTIVE"` | no | -| [zones](#input\_zones) | Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [central\_manager\_ips](#output\_central\_manager\_ips) | IP addresses of the central managers provisioned by this module | -| [central\_manager\_name](#output\_central\_manager\_name) | Name of the central managers provisioned by this module | -| [list\_instances\_command](#output\_list\_instances\_command) | Command to list central managers provisioned by this module | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml deleted file mode 100644 index 7408af6370..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor central manager - hosts: localhost - become: true - vars: - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - handlers: - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf deleted file mode 100644 index d288a91144..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-central-manager", ghpc_role = "scheduler" }) -} - -locals { - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - metadata = merge( - local.network_storage_metadata, - local.enable_oslogin_metadata, - local.disable_automatic_updates_metadata, - var.metadata - ) - - name_prefix = "${var.deployment_name}-cm" - - cm_config = templatefile("${path.module}/templates/condor_config.tftpl", {}) - - cm_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.cm_config.output_name}" - schedd_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e config_object=${local.cm_object}", - ]) - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.central_manager_runner, - [local.schedd_runner] - ) - - central_manager_ips = google_compute_address.cm.address - central_manager_name = data.google_compute_instance.cm.name - - list_instances_command = "gcloud compute instance-groups list-instances ${data.google_compute_region_instance_group.cm.name} --region ${var.region} --project ${var.project_id}" - - zones = coalescelist(var.zones, data.google_compute_zones.available.names) -} - -data "google_compute_image" "htcondor" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - postcondition { - condition = self.disk_size_gb <= var.disk_size_gb - error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -data "google_compute_region_instance_group" "cm" { - self_link = module.htcondor_cm.self_link - lifecycle { - postcondition { - condition = length(self.instances) == 1 - error_message = "There should only be 1 central manager found" - } - } -} - -data "google_compute_instance" "cm" { - self_link = data.google_compute_region_instance_group.cm.instances[0].instance -} - -resource "null_resource" "cm_config" { - triggers = { - config = local.cm_config - } -} - -resource "google_storage_bucket_object" "cm_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.cm_config.id), 0, 4)}" - content = local.cm_config - bucket = var.htcondor_bucket_name -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -resource "google_compute_address" "cm" { - project = var.project_id - name = local.name_prefix - region = var.region - subnetwork = var.subnetwork_self_link - address_type = "INTERNAL" - purpose = "GCE_ENDPOINT" -} - -module "central_manager_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.central_manager_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - preemptible = false - startup_script = module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.htcondor.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - network_ip = google_compute_address.cm.id -} - -module "htcondor_cm" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = 1 - hostname = local.name_prefix - instance_template = module.central_manager_instance_template.self_link - - health_check_name = "health-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) - max_surge_fixed = 0 # must be 0 to preserve hostnames - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - - # the timeouts below are default for resource - wait_for_instances = true - mig_timeouts = { - create = "15m" - delete = "15m" - update = "15m" - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf deleted file mode 100644 index a6272e7ca2..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "list_instances_command" { - description = "Command to list central managers provisioned by this module" - value = local.list_instances_command -} - -output "central_manager_ips" { - description = "IP addresses of the central managers provisioned by this module" - value = local.central_manager_ips -} - -output "central_manager_name" { - description = "Name of the central managers provisioned by this module" - value = local.central_manager_name -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl deleted file mode 100644 index 5b9676457e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:get_htcondor_central_manager -CONDOR_HOST = $(IPV4_ADDRESS) - -# Central Manager configuration settings -# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-collector-configuration-file-entries -# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-negotiator-configuration-file-entries -# set classad lifetime (expiration) to ~5x the update interval for all daemons -# defaults to 900s -CLASSAD_LIFETIME = 180 -COLLECTOR_UPDATE_INTERVAL = 30 -NEGOTIATOR_UPDATE_INTERVAL = 30 -NEGOTIATOR_DEPTH_FIRST = True -NEGOTIATOR_UPDATE_AFTER_CYCLE = True diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf deleted file mode 100644 index 7f85861c3f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf +++ /dev/null @@ -1,192 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor central manager will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zones" { - description = "Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region." - type = list(string) - default = [] - nullable = false -} - -variable "distribution_policy_target_shape" { - description = "Target shape for instance group managing high availability of central manager" - type = string - default = "ANY_SINGLE_ZONE" -} - -variable "network_self_link" { - description = "The self link of the network in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "central_manager_service_account_email" { - description = "Service account e-mail for central manager (can be supplied by htcondor-setup module)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 20 - nullable = false -} - -variable "metadata" { - description = "Metadata to add to HTCondor central managers" - type = map(string) - default = {} -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - nullable = false - validation { - condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "instance_image" { - description = <<-EOD - Custom VM image with HTCondor installed using the htcondor-install module." - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "machine_type" { - description = "Machine type to use for HTCondor central managers" - type = string - default = "n2-standard-4" -} - -variable "central_manager_runner" { - description = "A list of Toolkit runners for configuring an HTCondor central manager" - type = list(map(string)) - default = [] -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Central Manager (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)." - type = string - default = "PROACTIVE" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf deleted file mode 100644 index 4dee3adac7..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.74.0" - } - - required_version = ">= 1.1.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md deleted file mode 100644 index 7158e7bac6..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md +++ /dev/null @@ -1,172 +0,0 @@ -## Description - -This module is responsible for the following actions: - -- store an HTCondor Pool password in Google Cloud Secret Manager - - will generate a new password if one is not supplied -- create a secret in Google Cloud Secret Manager in which the HTCondor central - manager can place IDTOKENs (JWT Authorizations) for execute points to download -- create a Toolkit runner for the central manager - - download the POOL password / signing key - - create a local IDTOKEN for itself - - upload the execute point IDTOKEN secret -- create a Toolkit runner for access points - - download the POOL password / signing key - - create a local IDTOKEN for itself -- create a Toolkit runner for execute points - - Fetch the IDTOKEN secret generated by the central manager - -It is expected to be used with the [htcondor-install] and -[htcondor-execute-point] modules. - -[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[htcondor-install]: ../../scripts/htcondor-setup/README.md -[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md - -[htcrole]: https://htcondor.readthedocs.io/en/latest/getting-htcondor/admin-quick-start.html#what-get-htcondor-does-to-configure-a-role - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_setup - - - id: htcondor_startup_central_manager - source: modules/scripts/startup-script - settings: - runners: - - $(htcondor_install.install_htcondor_runner) - - $(htcondor_secrets.central_manager_runner) - - $(htcondor_setup.central_manager_runner) - -- id: htcondor_cm - source: modules/compute/vm-instance - use: - - network1 - - htcondor_startup_central_manager - settings: - name_prefix: cm0 - machine_type: c2-standard-4 - disable_public_ips: true - service_account: - email: $(htcondor_setup.central_manager_service_account) - scopes: - - cloud-platform - network_interfaces: - - network: null - subnetwork: $(network1.subnetwork_self_link) - subnetwork_project: $(vars.project_id) - network_ip: $(htcondor_setup.central_manager_internal_ip) - stack_type: null - access_config: [] - ipv6_access_config: [] - alias_ip_range: [] - nic_type: VIRTIO_NET - queue_count: null - outputs: - - internal_ip -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 4.84 | -| [random](#requirement\_random) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | -| [random](#provider\_random) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_secret_manager_secret.execute_point_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_iam_member.access_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.central_manager_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.central_manager_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.execute_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_version.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [random_password.pool](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [pool\_password](#input\_pool\_password) | HTCondor Pool Password | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [trust\_domain](#input\_trust\_domain) | Trust domain for HTCondor pool (if not supplied, will be set based on project\_id) | `string` | `""` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_runner](#output\_access\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor access point | -| [central\_manager\_runner](#output\_central\_manager\_runner) | Toolkit Runner to download pool secrets to an HTCondor central manager | -| [execute\_point\_runner](#output\_execute\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor execute point | -| [pool\_password\_secret\_id](#output\_pool\_password\_secret\_id) | Google Cloud Secret Manager ID containing HTCondor Pool Password | -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | PowerShell script to download pool secrets to an HTCondor execute point | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml deleted file mode 100644 index 538c809c2a..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Secrets - hosts: localhost - become: true - vars: - condor_config_root: /etc/condor - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - password_id is defined - - trust_domain is defined - - name: Set Pool Trust Domain - ansible.builtin.copy: - dest: "{{ condor_config_root }}/config.d/51-ghpc-trust-domain" - mode: 0644 - content: | - # these lines must appear AFTER any "use role:" settings - UID_DOMAIN = {{ trust_domain }} - TRUST_DOMAIN = {{ trust_domain }} - - name: Get HTCondor Pool password (token signing key) - when: htcondor_role != 'get_htcondor_execute' - ansible.builtin.shell: | - set -e -o pipefail +o history - POOL_PASSWORD=$(gcloud secrets versions access latest --secret={{ password_id }}) - echo -n "$POOL_PASSWORD" | sh -c "condor_store_cred add -c -i -" - args: - creates: "{{ condor_config_root }}/passwords.d/POOL" - executable: /bin/bash - - name: Configure HTCondor Central Manager - when: htcondor_role == 'get_htcondor_central_manager' - block: - - name: Create IDTOKEN for Central Manager - ansible.builtin.shell: | - umask 0077 - condor_token_create -identity condor@{{ trust_domain }} \ - -token condor@{{ trust_domain }} - args: - creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" - - name: Create IDTOKEN secret for Execute Points - when: xp_idtoken_secret_id | length > 0 - changed_when: true - ansible.builtin.shell: | - umask 0077 - TMPFILE=$(mktemp) - condor_token_create -authz READ -authz ADVERTISE_MASTER \ - -authz ADVERTISE_STARTD -identity condor@{{ trust_domain }} > "$TMPFILE" - gcloud secrets versions add --data-file "$TMPFILE" {{ xp_idtoken_secret_id }} - rm -f "$TMPFILE" - - name: Configure HTCondor SchedD - when: htcondor_role == 'get_htcondor_submit' - block: - - name: Create IDTOKEN to advertise access point - ansible.builtin.shell: | - umask 0077 - # DAEMON authorization can likely be removed in future when scopes - # needed to trigger a negotiation cycle are changed. Suggest review - # https://opensciencegrid.atlassian.net/jira/software/c/projects/HTCONDOR/issues/?filter=allissues - condor_token_create -authz READ -authz ADVERTISE_MASTER \ - -authz ADVERTISE_SCHEDD -authz DAEMON -identity condor@{{ trust_domain }} \ - -token condor@{{ trust_domain }} - args: - creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" - - name: Configure HTCondor StartD - when: htcondor_role == 'get_htcondor_execute' - block: - - name: Create SystemD override directory for HTCondor Execute Point - ansible.builtin.file: - path: /etc/systemd/system/condor.service.d - state: directory - owner: root - group: root - mode: 0755 - - name: Fetch IDTOKEN to advertise execute point - ansible.builtin.copy: - dest: "/etc/systemd/system/condor.service.d/htcondor-token-fetcher.conf" - mode: 0644 - content: | - [Service] - ExecStartPre=gcloud secrets versions access latest --secret {{ xp_idtoken_secret_id }} \ - --out-file {{ condor_config_root }}/tokens.d/condor@{{ trust_domain }} - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf deleted file mode 100644 index 1a7c761760..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-pool-secrets", ghpc_role = "scheduler" }) -} - -locals { - pool_password = coalesce(var.pool_password, random_password.pool.result) - auto = length(var.user_managed_replication) == 0 ? "" : "-user" - access_point_service_account_iam_email = "serviceAccount:${var.access_point_service_account_email}" - central_manager_service_account_iam_email = "serviceAccount:${var.central_manager_service_account_email}" - execute_point_service_account_iam_email = "serviceAccount:${var.execute_point_service_account_email}" - - trust_domain = coalesce(var.trust_domain, "c.${var.project_id}.internal") - - runner_cm = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_central_manager", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - - runner_access = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_submit", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - - runner_execute = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_execute", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - windows_startup_ps1 = templatefile( - "${path.module}/templates/fetch-idtoken.ps1.tftpl", - { - trust_domain = local.trust_domain, - xp_idtoken_secret_id = google_secret_manager_secret.execute_point_idtoken.secret_id, - } - ) -} - -resource "random_password" "pool" { - length = 24 - special = true - override_special = "_-#=." -} - -resource "google_secret_manager_secret" "pool_password" { - secret_id = "${var.deployment_name}-pool-password${local.auto}" - - labels = local.labels - - replication { - dynamic "auto" { - for_each = length(var.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_version" "pool_password" { - secret = google_secret_manager_secret.pool_password.id - secret_data = local.pool_password -} - -# this secret will be populated by the Central Manager -resource "google_secret_manager_secret" "execute_point_idtoken" { - secret_id = "${var.deployment_name}-execute-point-idtoken${local.auto}" - - labels = local.labels - - replication { - dynamic "auto" { - for_each = length(var.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_iam_member" "central_manager_password" { - secret_id = google_secret_manager_secret.pool_password.id - role = "roles/secretmanager.secretAccessor" - member = local.central_manager_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "central_manager_idtoken" { - secret_id = google_secret_manager_secret.execute_point_idtoken.id - role = "roles/secretmanager.secretVersionManager" - member = local.central_manager_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "access_point" { - secret_id = google_secret_manager_secret.pool_password.id - role = "roles/secretmanager.secretAccessor" - member = local.access_point_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "execute_point" { - secret_id = google_secret_manager_secret.execute_point_idtoken.id - role = "roles/secretmanager.secretAccessor" - member = local.execute_point_service_account_iam_email -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml deleted file mode 100644 index 4b0bdbd616..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com - - secretmanager.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf deleted file mode 100644 index 81c4986b16..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "pool_password_secret_id" { - description = "Google Cloud Secret Manager ID containing HTCondor Pool Password" - value = google_secret_manager_secret.pool_password.secret_id - sensitive = true -} - -output "central_manager_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor central manager" - value = local.runner_cm - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "access_point_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor access point" - value = local.runner_access - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "execute_point_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor execute point" - value = local.runner_execute - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "windows_startup_ps1" { - description = "PowerShell script to download pool secrets to an HTCondor execute point" - value = local.windows_startup_ps1 -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl deleted file mode 100644 index 04c96291ee..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl +++ /dev/null @@ -1,26 +0,0 @@ -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -$config_dir = 'C:\Condor\config' -if(!(test-path -PathType container -Path $config_dir)) -{ - New-Item -ItemType Directory -Path $config_dir -} -$config_file = "$config_dir\51-ghpc-trust-domain" - -$config_string = @' -# these lines must appear AFTER any "use role:" settings -UID_DOMAIN = ${trust_domain} -TRUST_DOMAIN = ${trust_domain} -'@ - -Set-Content -Path "$config_file" -Value "$config_string" - -# obtain IDTOKEN for authentication by StartD to Central Manager -gcloud secrets versions access latest --secret ${xp_idtoken_secret_id} ` - --out-file C:\condor\tokens.d\condor@${trust_domain} - -if ($LASTEXITCODE -ne 0) -{ - throw "Could not download HTCondor IDTOKEN; exiting startup script" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf deleted file mode 100644 index 22ef3644e8..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "access_point_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "central_manager_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "execute_point_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "pool_password" { - description = "HTCondor Pool Password" - type = string - sensitive = true - default = null -} - -variable "trust_domain" { - description = "Trust domain for HTCondor pool (if not supplied, will be set based on project_id)" - type = string - default = "" -} - -variable "user_managed_replication" { - type = list(object({ - location = string - kms_key_name = optional(string) - })) - description = "Replication parameters that will be used for defined secrets" - default = [] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf deleted file mode 100644 index d8a1d96f5f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.84" - } - random = { - source = "hashicorp/random" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md deleted file mode 100644 index 5a403c0a38..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module creates the service accounts for use by the primary elements of an -[HTCondor pool][pool]: - -- Central Managers -- Access Points -- Execute Points - -Each service account is assigned common roles necessary for the VM to function -properly. In particular, nearly every VM requires the ability to read from Cloud -Storage buckets and write Cloud Logging entries. These roles are configurable -as described below. - -[pool]: https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-admin-manual.html#the-different-roles-a-machine-can-play - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_service_accounts - source: community/modules/scheduler/htcondor-service-accounts - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - - htcondor_service_accounts - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_service_accounts - -- id: htcondor_cm - source: community/modules/scheduler/htcondor-central-manager - use: - - network1 - - htcondor_secrets - - htcondor_service_accounts - - htcondor_setup - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - central_manager_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [access\_point\_service\_account](#module\_access\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | -| [central\_manager\_service\_account](#module\_central\_manager\_service\_account) | ../../../../community/modules/project/service-account | n/a | -| [execute\_point\_service\_account](#module\_execute\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_roles](#input\_access\_point\_roles) | Project-wide roles for HTCondor Access Point service account | `list(string)` | 
[
  "compute.instanceAdmin.v1",
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [central\_manager\_roles](#input\_central\_manager\_roles) | Project-wide roles for HTCondor Central Manager service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_roles](#input\_execute\_point\_roles) | Project-wide roles for HTCondor Execute Point service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_service\_account\_email](#output\_access\_point\_service\_account\_email) | HTCondor Access Point Service Account (e-mail format) | -| [central\_manager\_service\_account\_email](#output\_central\_manager\_service\_account\_email) | HTCondor Central Manager Service Account (e-mail format) | -| [execute\_point\_service\_account\_email](#output\_execute\_point\_service\_account\_email) | HTCondor Execute Point Service Account (e-mail format) | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf deleted file mode 100644 index 9d97b18642..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# NB: the community/modules/project/service-account module will not output the -# service account e-mail address until all IAM bindings have been created; if -# underlying implementation changes, this module should declare explicit -# depends_on the IAM bindings to prevent race conditions for services that -# require them - -module "access_point_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Access Point" - deployment_name = var.deployment_name - name = "access" - project_roles = var.access_point_roles -} - -module "execute_point_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Execute Point" - deployment_name = var.deployment_name - name = "execute" - project_roles = var.execute_point_roles -} - -module "central_manager_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Central Manager" - deployment_name = var.deployment_name - name = "cm" - project_roles = var.central_manager_roles -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml deleted file mode 100644 index c4dcdffdf4..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf deleted file mode 100644 index 28f3a79457..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "access_point_service_account_email" { - description = "HTCondor Access Point Service Account (e-mail format)" - value = module.access_point_service_account.service_account_email -} - -output "central_manager_service_account_email" { - description = "HTCondor Central Manager Service Account (e-mail format)" - value = module.central_manager_service_account.service_account_email -} - -output "execute_point_service_account_email" { - description = "HTCondor Execute Point Service Account (e-mail format)" - value = module.execute_point_service_account.service_account_email -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf deleted file mode 100644 index ee186e0971..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "access_point_roles" { - description = "Project-wide roles for HTCondor Access Point service account" - type = list(string) - default = [ - "compute.instanceAdmin.v1", - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} - -variable "central_manager_roles" { - description = "Project-wide roles for HTCondor Central Manager service account" - type = list(string) - default = [ - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} - -variable "execute_point_roles" { - description = "Project-wide roles for HTCondor Execute Point service account" - type = list(string) - default = [ - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/README.md deleted file mode 100644 index 1722702ceb..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/README.md +++ /dev/null @@ -1,118 +0,0 @@ -## Description - -This module creates a bucket in which to store HTCondor configurations and -a firewall rule that allows Managed Instance Group health checks to probe the -health of HTCondor VMs. - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_service_accounts - source: community/modules/scheduler/htcondor-service-accounts - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - - htcondor_service_accounts - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_service_accounts - -- id: htcondor_cm - source: community/modules/scheduler/htcondor-central-manager - use: - - network1 - - htcondor_secrets - - htcondor_service_accounts - - htcondor_setup - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - central_manager_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [health\_check\_firewall\_rule](#module\_health\_check\_firewall\_rule) | ../../../../modules/network/firewall-rules | n/a | -| [htcondor\_bucket](#module\_htcondor\_bucket) | ../../../../modules/file-system/cloud-storage-bucket | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account e-mail for HTCondor Access Point | `string` | n/a | yes | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for HTCondor Central Manager | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account e-mail for HTCondor Execute Points | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which Central Managers will be placed. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [htcondor\_bucket\_name](#output\_htcondor\_bucket\_name) | Name of the HTCondor configuration bucket | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf deleted file mode 100644 index e048362663..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-setup", ghpc_role = "scheduler" }) -} - -locals { - service_account_iam_email = [ - "serviceAccount:${var.access_point_service_account_email}", - "serviceAccount:${var.central_manager_service_account_email}", - "serviceAccount:${var.execute_point_service_account_email}", - ] - service_account_email = [ - var.access_point_service_account_email, - var.central_manager_service_account_email, - var.execute_point_service_account_email, - ] -} - -module "health_check_firewall_rule" { - source = "../../../../modules/network/firewall-rules" - - subnetwork_self_link = var.subnetwork_self_link - - ingress_rules = [{ - name = "allow-health-check-${var.deployment_name}" - description = "Allow Managed Instance Group Health Checks for HTCondor VMs" - direction = "INGRESS" - source_ranges = [ - "130.211.0.0/22", - "35.191.0.0/16", - ] - target_service_accounts = local.service_account_email - allow = [{ - protocol = "tcp" - ports = ["9618"] - }] - }] -} - -module "htcondor_bucket" { - source = "../../../../modules/file-system/cloud-storage-bucket" - - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - name_prefix = "${var.deployment_name}-htcondor-config" - random_suffix = true - labels = local.labels - viewers = local.service_account_iam_email - - use_deployment_name_in_bucket_name = false -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml deleted file mode 100644 index 7b4918b962..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - iam.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf deleted file mode 100644 index a44223faee..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "htcondor_bucket_name" { - description = "Name of the HTCondor configuration bucket" - value = module.htcondor_bucket.gcs_bucket_name - - # ensure that all IAM bindings to the bucket and firewall rules are active - # before this modules output is allowed to propagate - depends_on = [ - module.htcondor_bucket, - module.health_check_firewall_rule - ] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf deleted file mode 100644 index 147a2ca88d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which Central Managers will be placed." - type = string -} - -variable "access_point_service_account_email" { - description = "Service account e-mail for HTCondor Access Point" - type = string -} - -variable "central_manager_service_account_email" { - description = "Service account e-mail for HTCondor Central Manager" - type = string -} - -variable "execute_point_service_account_email" { - description = "Service account e-mail for HTCondor Execute Points" - type = string -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md deleted file mode 100644 index 43254cbfa8..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md +++ /dev/null @@ -1,405 +0,0 @@ -## Description - -This module creates a slurm controller node via the internal -[slurm\_instance\_template] module. - -More information about Slurm On GCP can be found at the -[project's GitHub page][slurm-gcp] and in the -[Slurm on Google Cloud User Guide][slurm-ug]. - -The [user guide][slurm-ug] provides detailed instructions on customizing and -enhancing the Slurm on GCP cluster as well as recommendations on configuring the -controller for optimal performance at different scales. - -[slurm\_instance\_template]: /community/modules/internal/slurm-gcp/instance_template/README.md -[slurm-ug]: https://goo.gle/slurm-gcp-user-guide. -[enable\_cleanup\_compute]: #input\_enable\_cleanup\_compute -[enable\_cleanup\_subscriptions]: #input\_enable\_cleanup\_subscriptions -[enable\_reconfigure]: #input\_enable\_reconfigure - -### Example - -```yaml -- id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: - - network - - homefs - - compute_partition - settings: - machine_type: c2-standard-8 -``` - -This creates a controller node with the following attributes: - -* connected to the primary subnetwork of `network` -* the filesystem with the ID `homefs` (defined elsewhere in the blueprint) - mounted -* One partition with the ID `compute_partition` (defined elsewhere in the - blueprint) -* machine type upgraded from the default `c2-standard-4` to `c2-standard-8` - -### Live Cluster Reconfiguration - -The `schedmd-slurm-gcp-v6-controller` module supports the reconfiguration of -partitions and slurm configuration in a running, active cluster. - -To reconfigure a running cluster: - -1. Edit the blueprint with the desired configuration changes -2. Call `gcluster create -w` to overwrite the deployment directory -3. Follow instructions in terminal to deploy - -The following are examples of updates that can be made to a running cluster: - -* Add or remove a partition to the cluster -* Resize an existing partition -* Attach new network storage to an existing partition - -> **NOTE**: Changing the VM `machine_type` of a partition may not work. -> It is better to create a new partition and delete the old one. - -## Custom Images - -For more information on creating valid custom images for the controller VM -instance or for custom instance templates, see our [vm-images.md] documentation -page. - -[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images - -## GPU Support - -More information on GPU support in Slurm on GCP and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) - -## Reservation for Scheduled Maintenance - -A [maintenance event](https://cloud.google.com/compute/docs/instances/host-maintenance-overview#maintenanceevents) is when a compute engine stops a VM to perform a hardware or -software update which is determined by the host maintenance policy. This can -also affect the running jobs if the maintenance kicks in. Now, Customers can -protect jobs from getting terminated due to maintenance using the cluster -toolkit. You can enable creation of reservation for scheduled maintenance for -your compute nodeset and Slurm will reserve your node for maintenance during the -maintenance window. If you try to schedule any jobs which overlap with the -maintenance reservation, Slurm would not schedule any job. - -You can specify in your blueprint like - -```yaml - - id: compute_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network] - settings: - enable_maintenance_reservation: true -``` - -To enable creation of reservation for maintenance. - -While running job on slurm cluster, you can specify total run time of the job -using [-t flag](https://slurm.schedmd.com/srun.html#OPT_time).This would only -run the job outside of the maintenance window. - -```shell -srun -n1 -pcompute -t 10:00 -``` - -Currently upcoming maintenance notification is supported in ALPHA version of -compute API. You can update the API version from your blueprint, - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - settings: - endpoint_versions: - compute: "alpha" -``` - -## Opportunistic GCP maintenance in Slurm - -Customers can also enable running GCP maintenance as Slurm job opportunistically -to perform early maintenance. If a node is detected for maintenance, Slurm will -create a job to perform maintenance and put it in the job queue. - -If [backfill](https://slurm.schedmd.com/sched_config.html#backfill) scheduler is -used, Slurm will backfill maintenance job if it can find any empty time window. - -Customer can also choose builtin scheduler type. In this case, Slurm would run -maintenance job in strictly priority order. If the maintenance job doesn't kick -in, then forced maintenance will take place at scheduled window. - -Customer can enable this feature at nodeset level by, - -```yaml - - id: debug_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network] - settings: - enable_opportunistic_maintenance: true -``` - -## Placement Max Distance - -When using -[enable_placement](../../../../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md#input_enable_placement) -with Slurm, Google Compute Engine will attempt to place VMs as physically close -together as possible. Capacity constraints at the time of VM creation may still -force VMs to be spread across multiple racks. Google provides the `max-distance` -flag which can used to control the maximum spreading allowed. Read more about -`max-distance` in the -[official docs](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies -). - -You can use the `placement_max_distance` setting on the nodeset module to control the `max-distance` behavior. See the following example: - -```yaml - - id: nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [ network ] - settings: - machine_type: c2-standard-4 - node_count_dynamic_max: 30 - enable_placement: true - placement_max_distance: 1 - -> [!NOTE] -> `schedmd-slurm-gcp-v6-nodeset.settings.enable_placement: true` must also be -> set for placement_max_distance to take effect. - -In the above case using a value of 1 will restrict VM to be placed on the same -rack. You can confirm that the `max-distance` was applied by calling the -following command while jobs are running: - -```shell -gcloud beta compute resource-policies list \ - --format='yaml(name,groupPlacementPolicy.maxDistance)' -``` - -> [!WARNING] -> If a zone lacks capacity, using a lower `max-distance` value (such as 1) is -> more likely to cause VMs creation to fail. - -## TreeWidth and Node Communication - -Slurm uses a fan out mechanism to communicate large groups of nodes. The shape -of this fan out tree is determined by the -[TreeWidth](https://slurm.schedmd.com/slurm.conf.html#OPT_TreeWidth) -configuration variable. - -In the cloud, this fan out mechanism can become unstable when nodes restart with -new IP addresses. You can enforce that all nodes communicate directly with the -controller by setting TreeWidth to a value >= largest partition. - -If the largest partition was 200 nodes, configure the blueprint as follows: - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - ... - settings: - cloud_parameters: - tree_width: 200 -``` - -The default has been set to 128. Values above this have not been fully tested -and may cause congestion on the controller. A more scalable solution is under -way. - -## ResumeRate and Node Resumption - -The `ResumeRate` parameter in `slurm.conf` controls the maximum number of nodes -that Slurm attempts to resume (power up) per minute. This is particularly -important in cloud environments where auto-scaling can lead to a large number of -nodes starting concurrently. - -When many nodes start simultaneously, they can place a heavy load on shared -resources, especially shared filesystems, as they all try to mount filesystems -and access configuration files at the same time. By limiting the `ResumeRate`, -you can stagger the node startup process, reducing the peak load on these shared -resources and improving overall cluster stability during scaling events. - -For example, to limit the node resumption rate to 100 nodes per minute, -configure the blueprint as follows: - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - ... - settings: - cloud_parameters: - resume_rate: 100 -``` - -Adjust this value based on the capabilities of your shared filesystem and the -expected scaling behavior of your cluster. - -## Support -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 6.41 | -| [google-beta](#requirement\_google-beta) | >= 6.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.41 | -| [google-beta](#provider\_google-beta) | >= 6.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [bucket](#module\_bucket) | terraform-google-modules/cloud-storage/google | >= 6.1 | -| [daos\_network\_storage\_scripts](#module\_daos\_network\_storage\_scripts) | ../../../../modules/scripts/startup-script | n/a | -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [login](#module\_login) | ../../internal/slurm-gcp/login | n/a | -| [nodeset\_cleanup](#module\_nodeset\_cleanup) | ./modules/cleanup_compute | n/a | -| [nodeset\_cleanup\_tpu](#module\_nodeset\_cleanup\_tpu) | ./modules/cleanup_tpu | n/a | -| [slurm\_controller\_template](#module\_slurm\_controller\_template) | ../../internal/slurm-gcp/instance_template | n/a | -| [slurm\_files](#module\_slurm\_files) | ./modules/slurm_files | n/a | -| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | -| [slurm\_nodeset\_tpu](#module\_slurm\_nodeset\_tpu) | ../../internal/slurm-gcp/nodeset_tpu | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance_from_template.controller](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_from_template) | resource | -| [google_compute_disk.controller_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_secret_manager_secret.cloudsql](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_iam_member.cloudsql_secret_accessor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_version.cloudsql_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [google_storage_bucket_iam_member.legacy_readers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | -| [google_storage_bucket_iam_member.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | -| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_project.controller_project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = string
    device_name                = string
    disk_type                  = string
    disk_size_gb               = number
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. If not specified, then one will be chosen based on slurm\_cluster\_name. | `string` | `null` | no | -| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket.
Ignored when 'create\_bucket' is true. | `string` | `null` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | -| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Defaults inherited from [Slurm GCP repo](https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/terraform/slurm_cluster/modules/slurm_files/README_TF.md#input_cloud_parameters) | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | -| [cloudsql](#input\_cloudsql) | Use this database instead of the one on the controller.
server\_ip : Address of the database server.
user : The user to access the database as.
password : The password, given the user, to access the given database. (sensitive)
db\_name : The database to access.
user\_managed\_replication : The list of location and (optional) kms\_key\_name for secret | 
object({
    server_ip = string
    user      = string
    password  = string # sensitive
    db_name   = string
    user_managed_replication = optional(list(object({
      location     = string
      kms_key_name = optional(string)
    })), [])
  })
| `null` | no | -| [compute\_startup\_script](#input\_compute\_startup\_script) | DEPRECATED: `compute_startup_script` has been deprecated.
Use `startup_script` of nodeset module instead. | `any` | `null` | no | -| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script in compute nodes. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | -| [controller\_project\_id](#input\_controller\_project\_id) | Optionally. Provision controller and config bucket in the different project | `string` | `null` | no | -| [controller\_startup\_script](#input\_controller\_startup\_script) | Startup script used by the controller VM. | `string` | `"# no-op"` | no | -| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    type = string
    size = number
  })
| 
{
  "size": 50,
  "type": "pd-ssd"
}
| no | -| [create\_bucket](#input\_create\_bucket) | Create GCS bucket instead of using an existing one. | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment. | `string` | n/a | yes | -| [disable\_controller\_public\_ips](#input\_disable\_controller\_public\_ips) | DEPRECATED: Use `enable_controller_public_ips` instead. | `bool` | `null` | no | -| [disable\_default\_mounts](#input\_disable\_default\_mounts) | DEPRECATED: Use `enable_default_mounts` instead. | `bool` | `null` | no | -| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | -| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | `true` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_controller\_public\_ips](#input\_enable\_controller\_public\_ips) | If set to true. The controller will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. | `bool` | `false` | no | -| [enable\_default\_mounts](#input\_enable\_default\_mounts) | Enable default global network storage from the controller
- /home
- /opt/apps | `bool` | `true` | no | -| [enable\_devel](#input\_enable\_devel) | DEPRECATED: `enable_devel` is always on. | `bool` | `null` | no | -| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/tools/prologs-epilogs/README.md | `bool` | `null` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | -| [enable\_slurm\_gcp\_plugins](#input\_enable\_slurm\_gcp\_plugins) | DEPRECATED: Slurm GCP plugins have been deprecated.
Instead of 'max\_hops' plugin please use the 'placement\_max\_distance' nodeset property.
Instead of 'enable\_vpmu' plugin please use 'advanced\_machine\_features.performance\_monitoring\_unit' nodeset property. | `any` | `null` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": "beta"
}
| no | -| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for controller. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | -| [login\_network\_storage](#input\_login\_network\_storage) | An array of network attached storage mounts to be configured on all login nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [login\_nodes](#input\_login\_nodes) | List of slurm login instance definitions. | 
list(object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  }))
| `[]` | no | -| [login\_startup\_script](#input\_login\_startup\_script) | Startup script used by the login VMs. | `string` | `"# no-op"` | no | -| [login\_startup\_scripts\_timeout](#input\_login\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in login\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = optional(map(string))
    mount_runner          = optional(map(string))
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | Define nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | -| [partitions](#input\_partitions) | Cluster partitions as a list. See module slurm\_partition. | 
list(object({
    partition_name        = string
    partition_conf        = optional(map(string), {})
    partition_nodeset     = optional(list(string), [])
    partition_nodeset_dyn = optional(list(string), [])
    partition_nodeset_tpu = optional(list(string), [])
    enable_job_exclusive  = optional(bool, false)
  }))
| `[]` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [region](#input\_region) | The default region to place resources in. | `string` | n/a | yes | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the controller instance. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the controller instance. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming and slurm accounting.
If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). | `string` | `null` | no | -| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | -| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | -| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Post deployment instructions. | -| [slurm\_bucket](#output\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | -| [slurm\_bucket\_dir](#output\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | -| [slurm\_bucket\_name](#output\_slurm\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | -| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | Bucket path used by cluster. | -| [slurm\_cluster\_name](#output\_slurm\_cluster\_name) | Slurm cluster name. | -| [slurm\_controller\_instance](#output\_slurm\_controller\_instance) | Compute instance of controller node | -| [slurm\_login\_instances](#output\_slurm\_login\_instances) | Compute instances of login nodes | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf deleted file mode 100644 index 4a887b99cf..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.disk_resource_manager_tags - } - ] - - state_disk = var.controller_state_disk != null ? [{ - source = google_compute_disk.controller_disk[0].name - device_name = google_compute_disk.controller_disk[0].name - disk_labels = null - auto_delete = false - boot = false - }] : [] - - synth_def_sa_email = "${data.google_project.controller_project.number}-compute@developer.gserviceaccount.com" - - service_account = { - email = coalesce(var.service_account_email, local.synth_def_sa_email) - scopes = var.service_account_scopes - } - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata, - local.universe_domain - ) - - controller_project_id = coalesce(var.controller_project_id, var.project_id) -} - -data "google_project" "controller_project" { - project_id = local.controller_project_id -} - -resource "google_compute_disk" "controller_disk" { - count = var.controller_state_disk != null ? 1 : 0 - - project = local.controller_project_id - name = "${local.slurm_cluster_name}-controller-save" - type = var.controller_state_disk.type - size = var.controller_state_disk.size - zone = var.zone -} - -# INSTANCE TEMPLATE -module "slurm_controller_template" { - source = "../../internal/slurm-gcp/instance_template" - - project_id = local.controller_project_id - region = var.region - slurm_instance_role = "controller" - slurm_cluster_name = local.slurm_cluster_name - labels = local.labels - - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(var.disk_labels, local.labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = concat(local.additional_disks, local.state_disk) - - bandwidth_tier = var.bandwidth_tier - slurm_bucket_path = module.slurm_files.slurm_bucket_path - can_ip_forward = var.can_ip_forward - advanced_machine_features = var.advanced_machine_features - resource_manager_tags = var.resource_manager_tags - - enable_confidential_vm = var.enable_confidential_vm - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - gpu = one(module.gpu.guest_accelerator) - - machine_type = var.machine_type - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - service_account = local.service_account - - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - subnetwork = var.subnetwork_self_link - - tags = concat([local.slurm_cluster_name], var.tags) - # termination_action = TODO: add support for termination_action (?) -} - -# INSTANCE -resource "google_compute_instance_from_template" "controller" { - provider = google-beta - - name = "${local.slurm_cluster_name}-controller" - project = local.controller_project_id - zone = var.zone - source_instance_template = module.slurm_controller_template.self_link - # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 - # we have to explicitly override instance labels instead of inheriting them from template. - labels = module.slurm_controller_template.labels - - allow_stopping_for_update = true - - # Can't rely on template to specify nics due to usage of static_ip - network_interface { - dynamic "access_config" { - for_each = var.enable_controller_public_ips ? ["unit"] : [] - content { - nat_ip = null - network_tier = null - } - } - network_ip = length(var.static_ips) == 0 ? "" : var.static_ips[0] - subnetwork = var.subnetwork_self_link - } - - dynamic "network_interface" { - for_each = var.controller_network_attachment != null ? [1] : [] - content { - network_attachment = var.controller_network_attachment - } - } -} - -moved { - from = module.slurm_controller_instance.google_compute_instance_from_template.slurm_instance[0] - to = google_compute_instance_from_template.controller -} - -# SECRETS: CLOUDSQL -resource "google_secret_manager_secret" "cloudsql" { - count = var.cloudsql != null ? 1 : 0 - - secret_id = "${local.slurm_cluster_name}-slurm-secret-cloudsql" - project = var.project_id - - replication { - dynamic "auto" { - for_each = length(var.cloudsql.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.cloudsql.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = nonsensitive(var.cloudsql.user_managed_replication) - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } - - labels = { - slurm_cluster_name = local.slurm_cluster_name - } -} - -resource "google_secret_manager_secret_version" "cloudsql_version" { - count = var.cloudsql != null ? 1 : 0 - - secret = google_secret_manager_secret.cloudsql[0].id - secret_data = jsonencode(var.cloudsql) -} - -resource "google_secret_manager_secret_iam_member" "cloudsql_secret_accessor" { - count = var.cloudsql != null ? 1 : 0 - - secret_id = google_secret_manager_secret.cloudsql[0].id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${local.service_account.email}" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl deleted file mode 100644 index 219bdc5227..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl +++ /dev/null @@ -1,65 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/high_throughput.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxArraySize=10001 -MaxJobCount=500000 -MaxNodeCount=65536 -MinJobAge=60 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -SlurmctldDebug=error -SlurmdDebug=error - -# -# -# TIMERS -MessageTimeout=60 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=120 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl deleted file mode 100644 index 93ac47e341..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl +++ /dev/null @@ -1,34 +0,0 @@ -# slurmdbd.conf -# https://slurm.schedmd.com/slurmdbd.conf.html - -DebugLevel=info -PidFile=/var/run/slurm/slurmdbd.pid - -# https://slurm.schedmd.com/slurmdbd.conf.html#OPT_CommitDelay -CommitDelay=1 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -AuthType=auth/{auth_key} -AuthAltTypes=auth/jwt -AuthAltParameters=jwt_key={state_save}/jwt_hs256.key - -DbdHost={control_host} - -LogFile={slurmlog}/slurmdbd.log - -SlurmUser=slurm - -StorageLoc={db_name} - -StorageType=accounting_storage/mysql -StorageHost={db_host} -StoragePort={db_port} -StorageUser={db_user} -StoragePass={db_pass} - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl deleted file mode 100644 index d3f2615a68..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl +++ /dev/null @@ -1,71 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/slurm.conf.html -# https://slurm.schedmd.com/configurator.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxNodeCount=64000 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -AccountingStoreFlags=job_comment -JobAcctGatherFrequency=30 -JobAcctGatherType=jobacct_gather/cgroup -SlurmctldDebug=info -SlurmdDebug=info -DebugFlags=Power - -# -# -# TIMERS -MessageTimeout=600 -BatchStartTimeout=600 -PrologEpilogTimeout=600 -PrologFlags=Contain - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=600 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf deleted file mode 100644 index 21e915a125..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -locals { - # TODO: deprecate `var.login_[ startup_script, startup_scripts_timeout, network_storage]` - # in favour of vars defined in user-facing login module - ghpc_startup_login = [{ - filename = "ghpc_startup.sh" - content = var.login_startup_script - }] - - login_startup_scripts = concat(local.common_scripts, local.ghpc_startup_login) -} - -module "login" { - source = "../../internal/slurm-gcp/login" - for_each = { for x in var.login_nodes : x.group_name => x } - - project_id = var.project_id - - slurm_cluster_name = local.slurm_cluster_name - slurm_bucket_path = module.slurm_files.slurm_bucket_path - slurm_bucket_name = module.slurm_files.bucket_name - slurm_bucket_dir = module.slurm_files.bucket_dir - - login_nodes = each.value - - startup_scripts = local.login_startup_scripts - startup_scripts_timeout = var.login_startup_scripts_timeout - - network_storage = var.login_network_storage - - universe_domain = var.universe_domain - - # trigger replacement of login nodes when the controller instance is replaced - # Needed for re-mounting volumes hosted on controller - replace_trigger = google_compute_instance_from_template.controller.self_link -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf deleted file mode 100644 index 7622bdffef..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-controller", ghpc_role = "scheduler" }) -} - -locals { - # Since deployment name may be used to create a cluster name, we remove any invalid character from the beginning - # Also, slurm imposed a lot of restrictions to this name, so we format it to an acceptable string - tmp_cluster_name = substr(replace(lower(var.deployment_name), "/^[^a-z]*|[^a-z0-9]/", ""), 0, 10) - slurm_cluster_name = coalesce(var.slurm_cluster_name, local.tmp_cluster_name) - - universe_domain = { "universe_domain" = var.universe_domain } -} - -# See -# * slurm_files.tf -# * controller.tf -# * partition.tf -# * login.tf diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml deleted file mode 100644 index 7b4918b962..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - iam.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md deleted file mode 100644 index 002bf14145..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md +++ /dev/null @@ -1,42 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name         = string
    subnetwork_self_link = string
    additional_networks = list(object({
      subnetwork = string
    }))
  })
| n/a | yes | -| [nodeset\_template](#input\_nodeset\_template) | Self link of the nodeset template | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf deleted file mode 100644 index bd8773cf84..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - cleanup_dependencies_agg = flatten([ - var.nodeset.subnetwork_self_link, - var.nodeset.additional_networks[*].subnetwork, - var.nodeset_template]) -} - -# Can not use variadic list in `depends_on`, wrap it into a collection of `null_resource` -resource "null_resource" "dependencies" { - count = length(local.cleanup_dependencies_agg) -} - -resource "null_resource" "script" { - count = var.enable_cleanup_compute ? 1 : 0 - - triggers = { - project_id = var.project_id - cluster_name = var.slurm_cluster_name - nodeset_name = var.nodeset.nodeset_name - universe_domain = var.universe_domain - compute_endpoint_version = var.endpoint_versions.compute - gcloud_path_override = var.gcloud_path_override - } - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/cleanup_compute.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" - when = destroy - } - - # Ensure that clean up is done before attempt to delete the networks - depends_on = [null_resource.dependencies] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh deleted file mode 100644 index a98243d464..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -project="$1" -cluster_name="$2" -nodeset_name="$3" -universe_domain="$4" -compute_endpoint_version="$5" -gcloud_dir="$6" -MAX_ATTEMPTS=3 - -if [[ $# -ne 5 ]] && [[ $# -ne 6 ]]; then - echo "Usage: $0 []" - exit 1 -fi - -if [[ -n "${gcloud_dir}" ]]; then - export PATH="$gcloud_dir:$PATH" -fi - -export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" -export CLOUDSDK_CORE_PROJECT="${project}" - -if ! type -P gcloud 1>/dev/null; then - echo "gcloud is not available and your compute resources are not being cleaned up" - echo "https://console.cloud.google.com/compute/instances?project=${project}" - exit 1 -fi - -tmpfile=$(mktemp) # have to use a temp file, since `< <(gcloud ...)` doesn't work nicely with `head` -trap 'rm -f "$tmpfile"' EXIT - -echo "Deleting managed instance groups" -mig_filter="name:${cluster_name}-${nodeset_name}-*" -gcloud compute instance-groups managed list --format="value(self_link)" --filter="${mig_filter}" >"$tmpfile" -while batch="$(head -n 5)" && [[ ${#batch} -gt 0 ]]; do - groups=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line - # The lack of quotes around ${groups} is intentional and causes each new space-separated "word" to - # be treated as independent arguments. See PR#2523 - # shellcheck disable=SC2086 - for _ in $( #occasionally MIGs will fail to delete due to some active transformation happening, so let's retry - seq 1 $MAX_ATTEMPTS - ); do - if gcloud compute instance-groups managed delete --quiet ${groups}; then - break - fi - echo "MIG deletion failed, retrying" - done -done <"$tmpfile" -true >"$tmpfile" # Wipe contents of tmp file - -echo "Deleting compute nodes" -node_filter="name:${cluster_name}-${nodeset_name}-* labels.slurm_cluster_name=${cluster_name} AND labels.slurm_instance_role=compute" - -running_nodes_filter="${node_filter} AND status!=STOPPING" -# List all currently running instances and attempt to delete them -gcloud compute instances list --format="value(selfLink)" --filter="${running_nodes_filter}" >"$tmpfile" -# Do 500 instances at a time -while batch="$(head -n 500)" && [[ ${#batch} -gt 0 ]]; do - nodes=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line - # The lack of quotes around ${nodes} is intentional and causes each new space-separated "word" to - # be treated as independent arguments. See PR#2523 - # shellcheck disable=SC2086 - gcloud compute instances delete --quiet ${nodes} || echo "Failed to delete some instances" -done <"$tmpfile" - -# In case if controller tries to delete the nodes as well, -# wait until nodes in STOPPING state are deleted, before deleting the resource policies -stopping_nodes_filter="${node_filter} AND status=STOPPING" -while true; do - node=$(gcloud compute instances list --format="value(name)" --filter="${stopping_nodes_filter}" --limit=1) - if [[ -z "${node}" ]]; then - break - fi - echo "Waiting for instances to be deleted: ${node}" - sleep 5 -done - -echo "Deleting resource policies" -policies_filter="name:${cluster_name}-slurmgcp-managed-${nodeset_name}-*" -gcloud compute resource-policies list --format="value(selfLink)" --filter="${policies_filter}" | while read -r line; do - echo "Deleting resource policy: $line" - gcloud compute resource-policies delete --quiet "${line}" || { - echo "Failed to delete resource policy: $line" - } -done diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf deleted file mode 100644 index b6da69931c..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "Project ID" -} - - -variable "slurm_cluster_name" { - type = string - description = "Name of the Slurm cluster" -} - -variable "enable_cleanup_compute" { - description = < [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | 3.2.3 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf deleted file mode 100644 index ec86a03a24..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "null_resource" "script" { - count = var.enable_cleanup_compute ? 1 : 0 - - triggers = { - project_id = var.project_id - cluster_name = var.slurm_cluster_name - nodeset_name = var.nodeset.nodeset_name - zone = var.nodeset.zone - universe_domain = var.universe_domain - compute_endpoint_version = var.endpoint_versions.compute - gcloud_path_override = var.gcloud_path_override - } - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/cleanup_tpu.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.zone} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" - when = destroy - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh deleted file mode 100644 index c724e342c3..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -project="$1" -cluster_name="$2" -nodeset_name="$3" -zone="$4" -universe_domain="$5" -compute_endpoint_version="$6" -gcloud_dir="$7" - -if [[ $# -ne 6 ]] && [[ $# -ne 7 ]]; then - echo "Usage: $0 []" - exit 1 -fi - -if [[ -n "${gcloud_dir}" ]]; then - export PATH="$gcloud_dir:$PATH" -fi - -export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" -export CLOUDSDK_CORE_PROJECT="${project}" - -if ! type -P gcloud 1>/dev/null; then - echo "gcloud is not available and your compute resources are not being cleaned up" - echo "https://console.cloud.google.com/compute/instances?project=${project}" - exit 1 -fi - -echo "Deleting TPU nodes" -node_filter="name~${cluster_name}-${nodeset_name}" -running_nodes_filter="${node_filter} AND state!=DELETING" - -# List all currently running nodes and attempt to delete them -gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${running_nodes_filter}" | while read -r name; do - echo "Deleting TPU node: $name" - gcloud compute tpus tpu-vm delete --async --zone="${zone}" --quiet "${name}" || echo "Failed to delete $name" -done - -# Wait until nodes in DELETING state are deleted, before deleting the resource policies -deleting_nodes_filter="${node_filter} AND state=DELETING" -while true; do - node=$(gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${deleting_nodes_filter}" --limit=1) - if [[ -z "${node}" ]]; then - break - fi - echo "Waiting for nodes to be deleted: ${node}" - sleep 5 -done diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf deleted file mode 100644 index 1ac6f64b75..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright (C) Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "Project ID" -} - -variable "slurm_cluster_name" { - type = string - description = "Name of the Slurm cluster" -} - -variable "enable_cleanup_compute" { - description = < -Copyright (C) SchedMD LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.3 | -| [archive](#requirement\_archive) | ~> 2.0 | -| [google](#requirement\_google) | >= 6.41 | -| [local](#requirement\_local) | ~> 2.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [archive](#provider\_archive) | ~> 2.0 | -| [google](#provider\_google) | >= 6.41 | -| [local](#provider\_local) | ~> 2.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.controller_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.devel](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.devel_compute](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_dyn_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_tpu_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.task_epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.task_prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [random_uuid.cluster_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/uuid) | resource | -| [archive_file.slurm_gcp_devel_compute_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | -| [archive_file.slurm_gcp_devel_controller_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | -| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | -| [local_file.chs_gpu_health_check](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.external_epilog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.external_prolog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.setup_external](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. | `string` | `null` | no | -| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket to use. | `string` | n/a | yes | -| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | -| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Default behavior defined in scripts/conf.py | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | -| [cloudsql\_secret](#input\_cloudsql\_secret) | Secret URI to cloudsql secret. | `string` | `null` | no | -| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in compute\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | -| [controller\_startup\_scripts](#input\_controller\_startup\_scripts) | List of scripts to be ran on controller VM startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | -| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    device_name = string
  })
| 
{
  "device_name": null
}
| no | -| [disable\_default\_mounts](#input\_disable\_default\_mounts) | Disable default global network storage from the controller
- /home
- /apps | `bool` | `false` | no | -| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. Not for production use. | `bool` | `false` | no | -| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/v5/tools/prologs-epilogs/README.md | `bool` | `false` | no | -| [enable\_hybrid](#input\_enable\_hybrid) | Enables use of hybrid controller mode. When true, controller\_hybrid\_config will
be used instead of controller\_instance\_config and will disable login instances. | `bool` | `false` | no | -| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": null
}
| no | -| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | -| [google\_app\_cred\_path](#input\_google\_app\_cred\_path) | Path to Google Application Credentials. | `string` | `null` | no | -| [install\_dir](#input\_install\_dir) | Directory where the hybrid configuration directory will be installed on the
on-premise controller (e.g. /etc/slurm/hybrid). This updates the prefix path
for the resume and suspend scripts in the generated `cloud.conf` file.

This variable should be used when the TerraformHost and the SlurmctldHost
are different.

This will default to var.output\_dir if null. | `string` | `null` | no | -| [munge\_mount](#input\_munge\_mount) | Remote munge mount for compute and login nodes to acquire the munge.key.
By default, the munge mount server will be assumed to be the
`var.slurm_control_host` (or `var.slurm_control_addr` if non-null) when
`server_ip=null`. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| 
{
  "fs_type": "nfs",
  "mount_options": "",
  "remote_mount": "/etc/munge/",
  "server_ip": null
}
| no | -| [network\_storage](#input\_network\_storage) | Storage to mounted on all instances.
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | Cluster nodenets, as a list. | `list(any)` | `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Cluster nodenets (dynamic), as a list. | `list(any)` | `[]` | no | -| [nodeset\_startup\_scripts](#input\_nodeset\_startup\_scripts) | List of scripts to be ran on compute VM startup in the specific nodeset. | 
map(list(object({
    filename = string
    content  = string
  })))
| `{}` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Cluster nodenets (TPU), as a list. | `list(any)` | `[]` | no | -| [output\_dir](#input\_output\_dir) | Directory where this module will write its files to. These files include:
cloud.conf; cloud\_gres.conf; config.yaml; resume.py; suspend.py; and util.py. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The GCP project ID. | `string` | n/a | yes | -| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [slurm\_bin\_dir](#input\_slurm\_bin\_dir) | Path to directory of Slurm binary commands (e.g. scontrol, sinfo). If 'null',
then it will be assumed that binaries are in $PATH. | `string` | `null` | no | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | The cluster name, used for resource naming and slurm accounting. | `string` | n/a | yes | -| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | -| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | -| [slurm\_control\_addr](#input\_slurm\_control\_addr) | The IP address or a name by which the address can be identified.

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | -| [slurm\_control\_host](#input\_slurm\_control\_host) | The short, or long, hostname of the machine where Slurm control daemon is
executed (i.e. the name returned by the command "hostname -s").

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | -| [slurm\_control\_host\_port](#input\_slurm\_control\_host\_port) | The port number that the Slurm controller, slurmctld, listens to for work.

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldPort | `string` | `"6818"` | no | -| [slurm\_key\_mount](#input\_slurm\_key\_mount) | Remote mount for compute and login nodes to acquire the slurm.key. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| `null` | no | -| [slurm\_log\_dir](#input\_slurm\_log\_dir) | Directory where Slurm logs to. | `string` | `"/var/log/slurm"` | no | -| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | -| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [bucket\_dir](#output\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | -| [bucket\_name](#output\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | -| [config](#output\_config) | Cluster configuration. | -| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl deleted file mode 100644 index ffeb167cfc..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl +++ /dev/null @@ -1,7 +0,0 @@ -# cgroup.conf -# https://slurm.schedmd.com/cgroup.conf.html - -ConstrainCores=yes -ConstrainRamSpace=yes -ConstrainSwapSpace=no -ConstrainDevices=yes diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl deleted file mode 100644 index 4951289842..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl +++ /dev/null @@ -1,67 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/slurm.conf.html -# https://slurm.schedmd.com/configurator.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxNodeCount=64000 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -AccountingStoreFlags=job_comment -JobAcctGatherFrequency=30 -JobAcctGatherType=jobacct_gather/cgroup -SlurmctldDebug=info -SlurmdDebug=info -DebugFlags=Power - -# -# -# TIMERS -MessageTimeout=60 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=120 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl deleted file mode 100644 index 8c90a9dfbe..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl +++ /dev/null @@ -1,31 +0,0 @@ -# slurmdbd.conf -# https://slurm.schedmd.com/slurmdbd.conf.html - -DebugLevel=info -PidFile=/var/run/slurm/slurmdbd.pid - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -AuthType=auth/{auth_key} -AuthAltTypes=auth/jwt -AuthAltParameters=jwt_key={state_save}/jwt_hs256.key - -DbdHost={control_host} - -LogFile={slurmlog}/slurmdbd.log - -SlurmUser=slurm - -StorageLoc={db_name} - -StorageType=accounting_storage/mysql -StorageHost={db_host} -StoragePort={db_port} -StorageUser={db_user} -StoragePass={db_pass} - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh deleted file mode 100644 index db514fc9e5..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -x /opt/apps/adm/slurm/slurm_epilog ]]; then - exec /opt/apps/adm/slurm/slurm_epilog -fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh deleted file mode 100644 index 37a91bb1ea..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -x /opt/apps/adm/slurm/slurm_prolog ]]; then - exec /opt/apps/adm/slurm/slurm_prolog -fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh deleted file mode 100644 index 0877ff3b19..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SLURM_EXTERNAL_ROOT="/opt/apps/adm/slurm" -SLURM_MUX_FILE="slurm_mux" - -mkdir -p "${SLURM_EXTERNAL_ROOT}" -mkdir -p "${SLURM_EXTERNAL_ROOT}/logs" -mkdir -p "${SLURM_EXTERNAL_ROOT}/etc" - -# create common prolog / epilog "multiplex" script -if [ ! -f "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" ]; then - # indentation matters in EOT below; do not blindly edit! - cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -CMD="${0##*/}" -# Locate script -BASE=$(readlink -f $0) -BASE=${BASE%/*} - -export CLUSTER_ADM_BASE=${BASE} - -# Source config file if it exists for extra DEBUG settings -# used below -SLURM_MUX_CONF=${CLUSTER_ADM_BASE}/etc/slurm_mux.conf -if [[ -r ${SLURM_MUX_CONF} ]]; then - source ${SLURM_MUX_CONF} -fi - -# Setup logging if configured and directory exists -LOGFILE="/dev/null" -if [[ -d ${DEBUG_SLURM_MUX_LOG_DIR} && ${DEBUG_SLURM_MUX_ENABLE_LOG} == "yes" ]]; then - LOGFILE="${DEBUG_SLURM_MUX_LOG_DIR}/${CMD}-${SLURM_SCRIPT_CONTEXT}-job-${SLURMD_NODENAME}.log" - exec >>${LOGFILE} 2>&1 -fi - -# Global scriptlets -for SCRIPTLET in ${BASE}/${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do - if [[ -x ${SCRIPTLET} ]]; then - echo "Running ${SCRIPTLET}" - ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 - echo "Running ${SCRIPTLET} returned $?" - fi -done - -# Per partition scriptlets -for SCRIPTLET in ${BASE}/partition-${SLURM_JOB_PARTITION}-${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do - if [[ -x ${SCRIPTLET} ]]; then - echo "Running ${SCRIPTLET}" - ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 - echo "Running ${SCRIPTLET} returned $?" - fi -done -EOT -fi - -# ensure proper permissions on slurm_mux script -chmod 0755 "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" - -# create default slurm_mux configuration file -if [ ! -f "${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" ]; then - cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" -# these settings are intended for temporary debugging purposes only; leaving -# them enabled will write files for each job to a shared NFS directory without -# any automated cleanup -DEBUG_SLURM_MUX_LOG_DIR=/opt/apps/adm/slurm/logs -DEBUG_SLURM_MUX_ENABLE_LOG=no -EOT -fi - -# create epilog symbolic link -if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_epilog" ]; then - cd ${SLURM_EXTERNAL_ROOT} - # delete existing file if necessary - rm -f slurm_epilog - ln -s ${SLURM_MUX_FILE} slurm_epilog - cd - >/dev/null -fi - -# create prolog symbolic link -if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_prolog" ]; then - cd ${SLURM_EXTERNAL_ROOT} - # delete existing file if necessary - rm -f slurm_prolog - ln -s ${SLURM_MUX_FILE} slurm_prolog - cd - >/dev/null -fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf deleted file mode 100644 index e63b2d1100..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf +++ /dev/null @@ -1,406 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - scripts_dir = abspath("${path.module}/scripts") - - bucket_dir = coalesce(var.bucket_dir, format("%s-files", var.slurm_cluster_name)) -} - -######## -# DATA # -######## - -data "google_storage_bucket" "this" { - name = var.bucket_name -} - -########## -# RANDOM # -########## - -resource "random_uuid" "cluster_id" { -} - -################## -# CLUSTER CONFIG # -################## - -locals { - config = { - enable_bigquery_load = var.enable_bigquery_load - cloudsql_secret = var.cloudsql_secret - cluster_id = random_uuid.cluster_id.result - project = var.project_id - slurm_cluster_name = var.slurm_cluster_name - enable_slurm_auth = var.enable_slurm_auth - bucket_path = local.bucket_path - enable_debug_logging = var.enable_debug_logging - extra_logging_flags = var.extra_logging_flags - controller_state_disk = var.controller_state_disk - - # storage - disable_default_mounts = var.disable_default_mounts - network_storage = var.network_storage - - # timeouts - controller_startup_scripts_timeout = var.controller_startup_scripts_timeout - compute_startup_scripts_timeout = var.compute_startup_scripts_timeout - - munge_mount = local.munge_mount - slurm_key_mount = var.slurm_key_mount - - # slurm conf - prolog_scripts = [for k, v in google_storage_bucket_object.prolog_scripts : k] - epilog_scripts = [for k, v in google_storage_bucket_object.epilog_scripts : k] - task_prolog_scripts = [for k, v in google_storage_bucket_object.task_prolog_scripts : k] - task_epilog_scripts = [for k, v in google_storage_bucket_object.task_epilog_scripts : k] - cloud_parameters = var.cloud_parameters - - # hybrid - hybrid = var.enable_hybrid - google_app_cred_path = var.enable_hybrid ? local.google_app_cred_path : null - output_dir = var.enable_hybrid ? local.output_dir : null - install_dir = var.enable_hybrid ? local.install_dir : null - slurm_control_host = var.enable_hybrid ? var.slurm_control_host : null - slurm_control_host_port = var.enable_hybrid ? local.slurm_control_host_port : null - slurm_control_addr = var.enable_hybrid ? var.slurm_control_addr : null - slurm_bin_dir = var.enable_hybrid ? local.slurm_bin_dir : null - slurm_log_dir = var.enable_hybrid ? local.slurm_log_dir : null - controller_network_attachment = var.controller_network_attachment - - - # config files templates - slurmdbd_conf_tpl = file(coalesce(var.slurmdbd_conf_tpl, "${local.etc_dir}/slurmdbd.conf.tpl")) - slurm_conf_tpl = var.slurm_conf_template != null ? var.slurm_conf_template : file(coalesce(var.slurm_conf_tpl, "${local.etc_dir}/slurm.conf.tpl")) - cgroup_conf_tpl = file(coalesce(var.cgroup_conf_tpl, "${local.etc_dir}/cgroup.conf.tpl")) - - # Providers - endpoint_versions = var.endpoint_versions - } - - x_nodeset = toset(var.nodeset[*].nodeset_name) - x_nodeset_dyn = toset(var.nodeset_dyn[*].nodeset_name) - x_nodeset_tpu = toset(var.nodeset_tpu[*].nodeset.nodeset_name) - x_nodeset_overlap = setintersection([], local.x_nodeset, local.x_nodeset_dyn, local.x_nodeset_tpu) - - etc_dir = abspath("${path.module}/etc") - - bucket_path = format("%s/%s", data.google_storage_bucket.this.url, local.bucket_dir) - - slurm_control_host_port = coalesce(var.slurm_control_host_port, "6818") - - google_app_cred_path = var.google_app_cred_path != null ? abspath(var.google_app_cred_path) : null - slurm_bin_dir = var.slurm_bin_dir != null ? abspath(var.slurm_bin_dir) : null - slurm_log_dir = var.slurm_log_dir != null ? abspath(var.slurm_log_dir) : null - - munge_mount = var.enable_hybrid ? { - server_ip = lookup(var.munge_mount, "server_ip", coalesce(var.slurm_control_addr, var.slurm_control_host)) - remote_mount = lookup(var.munge_mount, "remote_mount", "/etc/munge/") - fs_type = lookup(var.munge_mount, "fs_type", "nfs") - mount_options = lookup(var.munge_mount, "mount_options", "") - } : null - - output_dir = can(coalesce(var.output_dir)) ? abspath(var.output_dir) : abspath(".") - install_dir = can(coalesce(var.install_dir)) ? abspath(var.install_dir) : local.output_dir -} - -resource "google_storage_bucket_object" "config" { - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/config.yaml" - content = yamlencode(local.config) - source_md5hash = md5(yamlencode(local.config)) - - # Take dependency on all other "config artifacts" so creation of `config.yaml` - # can be used as a signal for setup.py that "everything is ready". - # Some of following files, particularly mount scripts for new NFSes, can take a while to be created. - depends_on = [ - google_storage_bucket_object.controller_startup_scripts, - google_storage_bucket_object.nodeset_startup_scripts, - google_storage_bucket_object.prolog_scripts, - google_storage_bucket_object.epilog_scripts, - google_storage_bucket_object.task_prolog_scripts, - google_storage_bucket_object.task_epilog_scripts - ] -} - -resource "google_storage_bucket_object" "nodeset_config" { - for_each = { for ns in var.nodeset : ns.nodeset_name => merge(ns, { - instance_properties = jsondecode(ns.instance_properties_json) - }) } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -resource "google_storage_bucket_object" "nodeset_dyn_config" { - for_each = { for ns in var.nodeset_dyn : ns.nodeset_name => ns } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_dyn_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -resource "google_storage_bucket_object" "nodeset_tpu_config" { - for_each = { for n in var.nodeset_tpu[*].nodeset : n.nodeset_name => n } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_tpu_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -######### -# DEVEL # -######### - -locals { - build_dir = abspath("${path.module}/build") - - slurm_gcp_devel_controller_zip = "slurm-gcp-devel-controller.zip" - slurm_gcp_devel_compute_zip = "slurm-gcp-devel.zip" - slurm_gcp_devel_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_controller_zip) - slurm_gcp_devel_compute_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_compute_zip) - - controller_files = [ - "tools/gpu-test", - "tools/task-epilog", - "tools/task-prolog", - "conf.py", - "file_cache.py", - "get_tpu_vmcount.py", - "job_submit.lua.tpl", - "load_bq.py", - "local_pubsub.py", - "mig_flex.py", - "resume_wrapper.sh", - "resume.py", - "setup_network_storage.py", - "setup.py", - "slurmsync.py", - "sort_nodes.py", - "suspend_wrapper.sh", - "suspend.py", - "tpu.py", - "util.py", - "watch_delete_vm_op.py", - ] - - compute_files = [ - "tools/gpu-test", - "tools/task-epilog", - "tools/task-prolog", - "file_cache.py", - "get_tpu_vmcount.py", - "job_submit.lua.tpl", - "local_pubsub.py", - "mig_flex.py", - "setup_network_storage.py", - "setup.py", - "slurmsync.py", - "sort_nodes.py", - "suspend.py", - "tpu.py", - "util.py", - "watch_delete_vm_op.py", - ] -} - -data "archive_file" "slurm_gcp_devel_controller_zip" { - output_path = "${local.build_dir}/${local.slurm_gcp_devel_controller_zip}" - type = "zip" - - dynamic "source" { - for_each = local.controller_files - content { - content = file("${local.scripts_dir}/${source.value}") - filename = source.value - } - } -} - -data "archive_file" "slurm_gcp_devel_compute_zip" { - output_path = "${local.build_dir}/${local.slurm_gcp_devel_compute_zip}" - type = "zip" - - dynamic "source" { - for_each = local.compute_files - content { - content = file("${local.scripts_dir}/${source.value}") - filename = source.value - } - } -} - -resource "google_storage_bucket_object" "devel" { - bucket = var.bucket_name - name = local.slurm_gcp_devel_zip_bucket - source = data.archive_file.slurm_gcp_devel_controller_zip.output_path - source_md5hash = data.archive_file.slurm_gcp_devel_controller_zip.output_md5 -} - -resource "google_storage_bucket_object" "devel_compute" { - bucket = var.bucket_name - name = local.slurm_gcp_devel_compute_zip_bucket - source = data.archive_file.slurm_gcp_devel_compute_zip.output_path - source_md5hash = data.archive_file.slurm_gcp_devel_compute_zip.output_md5 -} - -########### -# SCRIPTS # -########### - -resource "google_storage_bucket_object" "controller_startup_scripts" { - for_each = { - for x in local.controller_startup_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-controller-script-%s", local.bucket_dir, each.key) - content = each.value.content - source_md5hash = md5(each.value.content) -} - -resource "google_storage_bucket_object" "nodeset_startup_scripts" { - for_each = { for x in flatten([ - for nodeset, scripts in var.nodeset_startup_scripts - : [for s in scripts - : { - content = s.content, - name = format("slurm-nodeset-%s-script-%s", nodeset, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_")) } - ]]) : x.name => x.content } - - bucket = var.bucket_name - name = format("%s/%s", local.bucket_dir, each.key) - content = each.value - source_md5hash = md5(each.value) -} - -resource "google_storage_bucket_object" "prolog_scripts" { - for_each = { - for x in local.prolog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-prolog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -resource "google_storage_bucket_object" "epilog_scripts" { - for_each = { - for x in local.epilog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-epilog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -resource "google_storage_bucket_object" "task_prolog_scripts" { - for_each = { - for x in local.task_prolog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-task_prolog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -resource "google_storage_bucket_object" "task_epilog_scripts" { - for_each = { - for x in local.task_epilog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-task_epilog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -############################ -# DATA: CHS GPU HEALTH CHECK -############################ - -data "local_file" "chs_gpu_health_check" { - filename = "${path.module}/scripts/tools/gpu-test" -} - -################################ -# DATA: EXTERNAL PROLOG/EPILOG # -################################ - -data "local_file" "external_epilog" { - filename = "${path.module}/files/external_epilog.sh" -} - -data "local_file" "external_prolog" { - filename = "${path.module}/files/external_prolog.sh" -} - -data "local_file" "setup_external" { - filename = "${path.module}/files/setup_external.sh" -} - -locals { - external_epilog = [{ - filename = "z_external_epilog.sh" - content = data.local_file.external_epilog.content - source = null - }] - external_prolog = [{ - filename = "z_external_prolog.sh" - content = data.local_file.external_prolog.content - source = null - }] - setup_external = [{ - filename = "z_setup_external.sh" - content = data.local_file.setup_external.content - }] - chs_gpu_health_check = [{ - filename = "a_chs_gpu_health_check.sh" - content = data.local_file.chs_gpu_health_check.content - source = null - }] - - chs_prolog = var.enable_chs_gpu_health_check_prolog ? local.chs_gpu_health_check : [] - ext_prolog = var.enable_external_prolog_epilog ? local.external_prolog : [] - prolog_scripts = concat(local.chs_prolog, local.ext_prolog, var.prolog_scripts) - task_prolog_scripts = var.task_prolog_scripts - - chs_epilog = var.enable_chs_gpu_health_check_epilog ? local.chs_gpu_health_check : [] - ext_epilog = var.enable_external_prolog_epilog ? local.external_epilog : [] - epilog_scripts = concat(local.chs_epilog, local.ext_epilog, var.epilog_scripts) - task_epilog_scripts = var.task_epilog_scripts - - controller_startup_scripts = var.enable_external_prolog_epilog ? concat(local.setup_external, var.controller_startup_scripts) : var.controller_startup_scripts - - -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf deleted file mode 100644 index 111c997d62..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "slurm_bucket_path" { - description = "GCS Bucket URI of Slurm cluster file storage." - value = local.bucket_path -} - -output "bucket_name" { - description = "GCS Bucket name of Slurm cluster file storage." - value = data.google_storage_bucket.this.name -} - -output "bucket_dir" { - description = "Path directory within `bucket_name` for Slurm cluster file storage." - value = local.bucket_dir -} - -output "config" { - description = "Cluster configuration." - value = local.config - - precondition { - condition = var.enable_hybrid ? can(coalesce(var.slurm_control_host)) : true - error_message = "Input slurm_control_host is required." - } - - precondition { - condition = length(local.x_nodeset_overlap) == 0 - error_message = "All nodeset names must be unique among all nodeset types." - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py deleted file mode 100644 index 89ceefa3df..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py +++ /dev/null @@ -1,658 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional, Iterable, Dict, Set, Tuple -from itertools import chain -from collections import defaultdict -import json -from pathlib import Path -import util -from util import dirs, slurmdirs -import tpu -from addict import Dict as NSDict # type: ignore - -FILE_PREAMBLE = """ -# Warning: -# This file is managed by a script. Manual modifications will be overwritten. -""" - - - -def dict_to_conf(conf, delim=" ") -> str: - """convert dict to delimited slurm-style key-value pairs""" - - def filter_conf(pair): - k, v = pair - if isinstance(v, list): - v = ",".join(str(el) for el in v if el is not None) - return k, (v if bool(v) or v == 0 else None) - - return delim.join( - f"{k}={v}" for k, v in map(filter_conf, conf.items()) if v is not None - ) - - -TOPOLOGY_PLUGIN_TREE = "topology/tree" - -def topology_plugin(lkp: util.Lookup) -> str: - """ - Returns configured topology plugin, defaults to `topology/tree`. - """ - cp, key = lkp.cfg.cloud_parameters, "topology_plugin" - if key not in cp or cp[key] is None: - return TOPOLOGY_PLUGIN_TREE - return cp[key] - -def conflines(lkp: util.Lookup) -> str: - params = lkp.cfg.cloud_parameters - def get(key, default): - """ - Returns the value of the key in params if it exists and is not None, - otherwise returns supplied default. - We can't rely on the `dict.get` method because the value could be `None` as - well as empty NSDict, depending on type of the `cfg.cloud_parameters`. - TODO: Simplify once NSDict is removed from the codebase. - """ - if key not in params or params[key] is None: - return default - return params[key] - - no_comma_params = get("no_comma_params", False) - - any_gpus = any( - lkp.template_info(nodeset.instance_template).gpu - for nodeset in lkp.cfg.nodeset.values() - ) - - any_tpu = any( - tpu_nodeset is not None - for part in lkp.cfg.partitions.values() - for tpu_nodeset in part.partition_nodeset_tpu - ) - - any_gke = any( - lkp.nodeset_is_gke(nodeset) - for nodeset in lkp.cfg.nodeset.values() - ) - - any_dynamic = any(bool(p.partition_feature) for p in lkp.cfg.partitions.values()) - comma_params = { - "LaunchParameters": [ - "enable_nss_slurm", - "use_interactive_step", - ], - "SlurmctldParameters": [ - "cloud_reg_addrs" if any_dynamic or any_tpu or any_gke else "cloud_dns", - "enable_configless", - "idle_on_node_suspend", - ], - "GresTypes": [ - "gpu" if any_gpus else None, - ], - } - - scripts_dir = lkp.cfg.install_dir or dirs.scripts - prolog_path = Path(dirs.custom_scripts / "prolog.d") - epilog_path = Path(dirs.custom_scripts / "epilog.d") - task_prolog_path = Path(dirs.custom_scripts / "task_prolog.d") - task_epilog_path = Path(dirs.custom_scripts / "task_epilog.d") - default_tree_width = 65533 if any_dynamic else 128 - - conf_options = { - **(comma_params if not no_comma_params else {}), - "Prolog": f"{prolog_path}/*" if lkp.cfg.prolog_scripts else None, - "Epilog": f"{epilog_path}/*" if lkp.cfg.epilog_scripts else None, - "TaskProlog": f"{task_prolog_path}/task-prolog" if lkp.cfg.task_prolog_scripts else None, - "TaskEpilog": f"{task_epilog_path}/task-epilog" if lkp.cfg.task_epilog_scripts else None, - "PrologFlags": get("prolog_flags", None), - "SwitchType": get("switch_type", None), - "PrivateData": get("private_data", []), - "SchedulerParameters": get("scheduler_parameters", [ - "bf_continue", - "salloc_wait_nodes", - "ignore_prefer_validation", - ]), - "ResumeProgram": f"{scripts_dir}/resume_wrapper.sh", - "ResumeFailProgram": f"{scripts_dir}/suspend_wrapper.sh", - "ResumeRate": get("resume_rate", 0), - "ResumeTimeout": get("resume_timeout", 300), - "SuspendProgram": f"{scripts_dir}/suspend_wrapper.sh", - "SuspendRate": get("suspend_rate", 0), - "SuspendTimeout": get("suspend_timeout", 300), - "SlurmdTimeout": get("slurmd_timeout", 300), - "UnkillableStepTimeout": get("unkillable_step_timeout", 300), - "TreeWidth": get("tree_width", default_tree_width), - "JobSubmitPlugins": "lua" if any_tpu else None, - "TopologyPlugin": topology_plugin(lkp), - "TopologyParam": get("topology_param", "SwitchAsNodeRank"), - } - return dict_to_conf(conf_options, delim="\n") - - - - -def nodeset_lines(nodeset, lkp: util.Lookup) -> str: - template_info = lkp.template_info(nodeset.instance_template) - machine_conf = lkp.template_machine_conf(nodeset.instance_template) - - # follow https://slurm.schedmd.com/slurm.conf.html#OPT_Boards - # by setting Boards, SocketsPerBoard, CoresPerSocket, and ThreadsPerCore - gres = f"gpu:{template_info.gpu.count}" if template_info.gpu else None - node_conf = { - "RealMemory": machine_conf.memory, - "Boards": machine_conf.boards, - "SocketsPerBoard": machine_conf.sockets_per_board, - "CoresPerSocket": machine_conf.cores_per_socket, - "ThreadsPerCore": machine_conf.threads_per_core, - "CPUs": machine_conf.cpus, - "Gres": gres, - **nodeset.node_conf, - } - nodelist = lkp.nodelist(nodeset) - - return "\n".join( - map( - dict_to_conf, - [ - {"NodeName": nodelist, "State": "CLOUD", **node_conf}, - {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, - ], - ) - ) - - -def nodeset_tpu_lines(nodeset, lkp: util.Lookup) -> str: - nodelist = lkp.nodelist(nodeset) - return "\n".join( - map( - dict_to_conf, - [ - {"NodeName": nodelist, "State": "CLOUD", **nodeset.node_conf}, - {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, - ], - ) - ) - - -def nodeset_dyn_lines(nodeset): - """generate slurm NodeSet definition for dynamic nodeset""" - return dict_to_conf( - {"NodeSet": nodeset.nodeset_name, "Feature": nodeset.nodeset_feature} - ) - - -def partitionlines(partition, lkp: util.Lookup) -> str: - """Make a partition line for the slurm.conf""" - MIN_MEM_PER_CPU = 100 - - def defmempercpu(nodeset_name: str) -> int: - nodeset = lkp.cfg.nodeset.get(nodeset_name) - template = nodeset.instance_template - machine = lkp.template_machine_conf(template) - mem_spec_limit = int(nodeset.node_conf.get("MemSpecLimit", 0)) - return max(MIN_MEM_PER_CPU, (machine.memory - mem_spec_limit) // machine.cpus) - - defmem = min( - map(defmempercpu, partition.partition_nodeset), default=MIN_MEM_PER_CPU - ) - - nodesets = list( - chain( - partition.partition_nodeset, - partition.partition_nodeset_dyn, - partition.partition_nodeset_tpu, - ) - ) - - is_tpu = len(partition.partition_nodeset_tpu) > 0 - is_dyn = len(partition.partition_nodeset_dyn) > 0 - - oversub_exlusive = partition.enable_job_exclusive or is_tpu - power_down_on_idle = partition.enable_job_exclusive and not is_dyn - - line_elements = { - "PartitionName": partition.partition_name, - "Nodes": ",".join(nodesets), - "State": "UP", - "DefMemPerCPU": defmem, - "SuspendTime": 300, - "Oversubscribe": "Exclusive" if oversub_exlusive else None, - "PowerDownOnIdle": "YES" if power_down_on_idle else None, - **partition.partition_conf, - } - - return dict_to_conf(line_elements) - - -def suspend_exc_lines(lkp: util.Lookup) -> Iterable[str]: - static_nodelists = [] - for ns in lkp.power_managed_nodesets(): - if ns.node_count_static: - nodelist = lkp.nodelist_range(ns.nodeset_name, 0, ns.node_count_static) - static_nodelists.append(nodelist) - suspend_exc_nodes = {"SuspendExcNodes": static_nodelists} - - dyn_parts = [ - p.partition_name - for p in lkp.cfg.partitions.values() - if len(p.partition_nodeset_dyn) > 0 - ] - suspend_exc_parts = {"SuspendExcParts": [*dyn_parts]} - - return filter( - None, - [ - dict_to_conf(suspend_exc_nodes) if static_nodelists else None, - dict_to_conf(suspend_exc_parts), - ], - ) - - -def make_cloud_conf(lkp: util.Lookup) -> str: - """generate cloud.conf snippet""" - lines = [ - FILE_PREAMBLE, - conflines(lkp), - *(nodeset_lines(n, lkp) for n in lkp.cfg.nodeset.values()), - *(nodeset_dyn_lines(n) for n in lkp.cfg.nodeset_dyn.values()), - *(nodeset_tpu_lines(n, lkp) for n in lkp.cfg.nodeset_tpu.values()), - *(partitionlines(p, lkp) for p in lkp.cfg.partitions.values()), - *(suspend_exc_lines(lkp)), - ] - return "\n\n".join(filter(None, lines)) - - -def gen_cloud_conf(lkp: util.Lookup) -> None: - content = make_cloud_conf(lkp) - - conf_file = lkp.etc_dir / "cloud.conf" - conf_file.write_text(content) - util.chown_slurm(conf_file, mode=0o644) - - -def install_slurm_conf(lkp: util.Lookup) -> None: - """install slurm.conf""" - if lkp.cfg.ompi_version: - mpi_default = "pmi2" - else: - mpi_default = "none" - - conf_options = { - "name": lkp.cfg.slurm_cluster_name, - "control_addr": lkp.control_addr if lkp.control_addr else lkp.hostname_fqdn, - "control_host": lkp.control_host, - "accounting_storage_host": lkp.control_addr if lkp.cfg.controller_network_attachment else lkp.control_host, - "control_host_port": lkp.control_host_port, - "scripts": dirs.scripts, - "slurmlog": dirs.log, - "state_save": slurmdirs.state, - "mpi_default": mpi_default, - "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", - } - - conf = lkp.cfg.slurm_conf_tpl.format(**conf_options) - - conf_file = lkp.etc_dir / "slurm.conf" - conf_file.write_text(conf) - util.chown_slurm(conf_file, mode=0o644) - - -def install_slurmdbd_conf(lkp: util.Lookup) -> None: - """install slurmdbd.conf""" - conf_options = { - "control_host": lkp.control_host, - "slurmlog": dirs.log, - "state_save": slurmdirs.state, - "db_name": "slurm_acct_db", - "db_user": "slurm", - "db_pass": '""', - "db_host": "localhost", - "db_port": "3306", - "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", - } - - if lkp.cfg.cloudsql_secret: - secret_name = f"{lkp.cfg.slurm_cluster_name}-slurm-secret-cloudsql" - payload = json.loads(util.access_secret_version(lkp.project, secret_name)) - - if payload["db_name"] and payload["db_name"] != "": - conf_options["db_name"] = payload["db_name"] - if payload["user"] and payload["user"] != "": - conf_options["db_user"] = payload["user"] - if payload["password"] and payload["password"] != "": - conf_options["db_pass"] = payload["password"] - - db_host_str = payload["server_ip"].split(":") - if db_host_str[0]: - conf_options["db_host"] = db_host_str[0] - conf_options["db_port"] = ( - db_host_str[1] if len(db_host_str) >= 2 else "3306" - ) - - conf = lkp.cfg.slurmdbd_conf_tpl.format(**conf_options) - - conf_file = lkp.etc_dir / "slurmdbd.conf" - conf_file.write_text(conf) - util.chown_slurm(conf_file, 0o600) - - -def install_cgroup_conf(lkp: util.Lookup) -> None: - """install cgroup.conf""" - conf_file = lkp.etc_dir / "cgroup.conf" - conf_file.write_text(lkp.cfg.cgroup_conf_tpl) - util.chown_slurm(conf_file, mode=0o600) - - -def install_jobsubmit_lua(lkp: util.Lookup) -> None: - """install job_submit.lua if there are tpu nodes in the cluster""" - if not any( - tpu_nodeset is not None - for part in lkp.cfg.partitions.values() - for tpu_nodeset in part.partition_nodeset_tpu - ): - return # No TPU partitions, no need for job_submit.lua - - scripts_dir = lkp.cfg.slurm_scripts_dir or dirs.scripts - tpl = (scripts_dir / "job_submit.lua.tpl").read_text() - conf = tpl.format(scripts_dir=scripts_dir) - - conf_file = lkp.etc_dir / "job_submit.lua" - conf_file.write_text(conf) - util.chown_slurm(conf_file, 0o600) - - -def gen_cloud_gres_conf_lines(lkp: util.Lookup) -> str: - """generate cloud_gres.conf's content""" - - gpu_nodes = defaultdict(list) - for nodeset in lkp.cfg.nodeset.values(): - ti = lkp.template_info(nodeset.instance_template) - gpu_count = ti.gpu.count if ti.gpu else 0 - gpu_type = ti.gpu.type if ti.gpu else None - if gpu_count: - gpu_nodes[(gpu_count, gpu_type)].append(lkp.nodelist(nodeset)) - - lines = [ - dict_to_conf( - { - "NodeName": names, - "Name": "gpu", - "Type": gpu_type, - "File": "/dev/nvidia{}".format(f"[0-{gpu_count-1}]" if gpu_count > 1 else "0"), - } - ) - for (gpu_count, gpu_type), names in gpu_nodes.items() - ] - lines.append("\n") - return "\n".join(lines) - - -def gen_cloud_gres_conf(lkp: util.Lookup) -> None: - """create cloud_gres.conf file""" - - content = FILE_PREAMBLE + gen_cloud_gres_conf_lines(lkp) - - conf_file = lkp.etc_dir / "cloud_gres.conf" - conf_file.write_text(content) - util.chown_slurm(conf_file, mode=0o600) - - -def install_gres_conf(lkp: util.Lookup) -> None: - conf_file = lkp.etc_dir / "cloud_gres.conf" - gres_conf = lkp.etc_dir / "gres.conf" - if not gres_conf.exists(): - gres_conf.symlink_to(conf_file) - util.chown_slurm(gres_conf, mode=0o600) - - -class Switch: - """ - Represents a switch in the topology.conf file. - NOTE: It's class user job to make sure that there is no leaf-less Switches in the tree - """ - - def __init__( - self, - name: str, - nodes: Optional[Iterable[str]] = None, - switches: Optional[Dict[str, "Switch"]] = None, - ): - self.name = name - self.nodes = nodes or [] - self.switches = switches or {} - - def conf_line(self) -> str: - d = {"SwitchName": self.name} - if self.nodes: - d["Nodes"] = util.to_hostlist(self.nodes) - if self.switches: - d["Switches"] = util.to_hostlist(self.switches.keys()) - return dict_to_conf(d) - - def render_conf_lines(self) -> Iterable[str]: - yield self.conf_line() - for s in sorted(self.switches.values(), key=lambda s: s.name): - yield from s.render_conf_lines() - -class TopologySummary: - """ - Represents a summary of the topology, to make judgements about changes. - To be stored in JSON file along side of topology.conf to simplify parsing. - """ - def __init__( - self, - physical_host: Optional[Dict[str, str]] = None, - down_nodes: Optional[Iterable[str]] = None, - tpu_nodes: Optional[Iterable[str]] = None, - ) -> None: - self.physical_host = physical_host or {} - self.down_nodes = set(down_nodes or []) - self.tpu_nodes = set(tpu_nodes or []) - - - @classmethod - def path(cls, lkp: util.Lookup) -> Path: - return lkp.etc_dir / "cloud_topology.summary.json" - - @classmethod - def loads(cls, s: str) -> "TopologySummary": - d = json.loads(s) - return cls( - physical_host=d.get("physical_host"), - down_nodes=d.get("down_nodes"), - tpu_nodes=d.get("tpu_nodes"), - ) - - @classmethod - def load(cls, lkp: util.Lookup) -> "TopologySummary": - p = cls.path(lkp) - if not p.exists(): - return cls() # Return empty instance - return cls.loads(p.read_text()) - - def dumps(self) -> str: - return json.dumps( - { - "physical_host": self.physical_host, - "down_nodes": list(self.down_nodes), - "tpu_nodes": list(self.tpu_nodes), - }, - indent=2) - - def dump(self, lkp: util.Lookup) -> None: - TopologySummary.path(lkp).write_text(self.dumps()) - - def _nodenames(self) -> Set[str]: - return set(self.physical_host) | self.down_nodes | self.tpu_nodes - - def requires_reconfigure(self, prev: "TopologySummary") -> bool: - """ - Reconfigure IFF one of the following occurs: - * A node is added - * A node get a non-empty physicalHost - """ - if len(self._nodenames() - prev._nodenames()) > 0: - return True - for n, ph in self.physical_host.items(): - if ph and ph != prev.physical_host.get(n): - return True - return False - -class TopologyBuilder: - def __init__(self) -> None: - self._r = Switch("") # fake root, not part of the tree - self.summary = TopologySummary() - - def add(self, path: List[str], nodes: Iterable[str]) -> None: - n = self._r - assert path - for p in path: - n = n.switches.setdefault(p, Switch(p)) - n.nodes = [*n.nodes, *nodes] - - def render_conf_lines(self) -> Iterable[str]: - if not self._r.switches: - return [] # type: ignore - for s in sorted(self._r.switches.values(), key=lambda s: s.name): - yield from s.render_conf_lines() - - def compress(self) -> "TopologyBuilder": - compressed = TopologyBuilder() - compressed.summary = self.summary - def _walk( - u: Switch, c: Switch - ): # u: uncompressed node, c: its counterpart in compressed tree - pref = f"{c.name}_" if c != compressed._r else "s" - for i, us in enumerate(sorted(u.switches.values(), key=lambda s: s.name)): - cs = Switch(f"{pref}{i}", nodes=us.nodes) - c.switches[cs.name] = cs - _walk(us, cs) - - _walk(self._r, compressed._r) - return compressed - - -def add_tpu_nodeset_topology(nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup): - tpuobj = tpu.TPU.make(nodeset.nodeset_name, lkp) - static, dynamic = lkp.nodenames(nodeset) - - pref = ["tpu-root", f"ns_{nodeset.nodeset_name}"] - if tpuobj.vmcount == 1: # Put all nodes in one switch - all_nodes = list(chain(static, dynamic)) - bldr.add(pref, all_nodes) - bldr.summary.tpu_nodes.update(all_nodes) - return - - # Chunk nodes into sub-switches of size `vmcount` - chunk_num = 0 - for nodenames in (static, dynamic): - for nodeschunk in util.chunked(nodenames, n=tpuobj.vmcount): - chunk_name = f"{nodeset.nodeset_name}-{chunk_num}" - chunk_num += 1 - bldr.add([*pref, chunk_name], nodeschunk) - bldr.summary.tpu_nodes.update(nodeschunk) - -_SLURM_TOPO_ROOT = "slurm-root" - -def _make_physical_path(physical_host: str) -> List[str]: - assert physical_host.startswith("/"), f"Unexpected physicalHost: {physical_host}" - parts = physical_host[1:].split("/") - # Due to issues with Slurm's topology plugin, we can not use all components of `physicalHost`, - # trim it down to `cluster/rack`. - short_path = parts[:2] - return [_SLURM_TOPO_ROOT, *short_path] - -def add_nodeset_topology( - nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup -) -> None: - up_nodes = set() - default_path = [_SLURM_TOPO_ROOT, f"ns_{nodeset.nodeset_name}"] - - for inst in lkp.instances().values(): - try: - if lkp.node_nodeset_name(inst.name) != nodeset.nodeset_name: - continue - except Exception: - continue - - phys_host = inst.resource_status.physical_host or "" - bldr.summary.physical_host[inst.name] = phys_host - up_nodes.add(inst.name) - - if phys_host: - bldr.add(_make_physical_path(phys_host), [inst.name]) - else: - bldr.add(default_path, [inst.name]) - - down_nodes = [] - for node in chain(*lkp.nodenames(nodeset)): - if node not in up_nodes: - down_nodes.append(node) - if down_nodes: - bldr.add(default_path, down_nodes) - bldr.summary.down_nodes.update(down_nodes) - -def gen_topology(lkp: util.Lookup) -> TopologyBuilder: - bldr = TopologyBuilder() - for ns in lkp.cfg.nodeset_tpu.values(): - add_tpu_nodeset_topology(ns, bldr, lkp) - for ns in lkp.cfg.nodeset.values(): - add_nodeset_topology(ns, bldr, lkp) - return bldr - -def gen_topology_conf(lkp: util.Lookup) -> Tuple[bool, TopologySummary]: - """ - Generates slurm topology.conf. - Returns whether the topology.conf got updated. - """ - topo = gen_topology(lkp).compress() - conf_file = lkp.etc_dir / "cloud_topology.conf" - - with open(conf_file, "w") as f: - f.writelines(FILE_PREAMBLE + "\n") - for line in topo.render_conf_lines(): - f.write(line) - f.write("\n") - f.write("\n") - - prev_summary = TopologySummary.load(lkp) - return topo.summary.requires_reconfigure(prev_summary), topo.summary - -def install_topology_conf(lkp: util.Lookup) -> None: - conf_file = lkp.etc_dir / "cloud_topology.conf" - summary_file = lkp.etc_dir / "cloud_topology.summary.json" - topo_conf = lkp.etc_dir / "topology.conf" - - if not topo_conf.exists(): - topo_conf.symlink_to(conf_file) - - util.chown_slurm(conf_file, mode=0o600) - util.chown_slurm(summary_file, mode=0o600) - - -def gen_controller_configs(lkp: util.Lookup) -> None: - install_slurm_conf(lkp) - install_slurmdbd_conf(lkp) - gen_cloud_conf(lkp) - gen_cloud_gres_conf(lkp) - install_gres_conf(lkp) - install_cgroup_conf(lkp) - install_jobsubmit_lua(lkp) - - if topology_plugin(lkp) == TOPOLOGY_PLUGIN_TREE: - _, summary = gen_topology_conf(lkp) - summary.dump(lkp) - install_topology_conf(lkp) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py deleted file mode 100644 index cd2e41e5af..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any -from pathlib import Path -import shutil -import pickle - -import logging -log = logging.getLogger() - -# Can't reuse tool from util.py to avoid circular dependencies -# TODO: break down util.py for better modularity. -def _chown_slurm(path: Path) -> None: - shutil.chown(path, user="slurm", group="slurm") - -class FileCache: - def __init__(self, path: Path): - self.path = path - - def get(self, key: str) -> Any | None: - p = self.path / key - if not p.exists(): - return None - - try: - with p.open("rb") as f: - return pickle.load(f) - - except Exception as e: - log.warning(f"Failed to read cached value at {p}: {e}") - return None - - def set(self, key: str, data: Any) -> None: - p = self.path / key - - try: - # Create & chown before writing to minimize chances - # of ending up with root-owned corrupted file that can't be cleaned up - # TODO: restrict usage of cache by root to avoid all this complexity - # or have a cache per user. - p.touch(exist_ok=True) - _chown_slurm(p) - with p.open("wb") as f: - pickle.dump(data, f) - - except Exception as e: - log.warning(f"Failed to write cached value at {p}: {e}") - - -class NoCache: - def get(self, key: str) -> Any: - log.warning("No cache used") - return None - - def set(self, key: str, data: Any) -> None: - log.warning("No cache used") - - -def cache(name: str) -> FileCache | NoCache: - try: - path = Path("/tmp/slurm_gcp_cache/") / name - if not path.exists(): - path.mkdir(exist_ok=True, parents=True) - _chown_slurm(path) - return FileCache(path) - except: - log.exception(f"Failed to create cache, fallback to NoCache") - return NoCache() diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py deleted file mode 100644 index df0fd8ebe0..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright 2024 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import util -import tpu - - -def get_vmcount_of_tpu_part(part): - res = 0 - lkp = util.lookup() - for ns in lkp.cfg.partitions[part].partition_nodeset_tpu: - tpu_obj = tpu.TPU.make(ns, lkp) - if res == 0: - res = tpu_obj.vmcount - else: - if res != tpu_obj.vmcount: - # this should not happen, that in the same partition there are different vmcount nodesets - return -1 - return res - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--partitions", - "-p", - help="The partition(s) to retrieve the TPU vmcount value for.", - ) - args = parser.parse_args() - if not args.partitions: - exit(0) - - # useful exit code - # partition does not exists in config.yaml, thus do not exist in slurm - PART_INVALID = -1 - # in the same partition there are nodesets with different vmcounts - DIFF_VMCOUNTS_SAME_PART = -2 - # partition is a list of partitions in which at least two of them have different vmcount - DIFF_PART_DIFFERENT_VMCOUNTS = -3 - vmcounts = [] - # valid equals to 0 means that we are ok, otherwise it will be set to one of the previously defined exit codes - valid = 0 - for part in args.partitions.split(","): - if part not in util.lookup().cfg.partitions: - valid = PART_INVALID - break - else: - if util.lookup().partition_is_tpu(part): - vmcount = get_vmcount_of_tpu_part(part) - if vmcount == -1: - valid = DIFF_VMCOUNTS_SAME_PART - break - vmcounts.append(vmcount) - else: - vmcounts.append(0) - # this means that there are different vmcounts for these partitions - if valid == 0 and len(set(vmcounts)) != 1: - valid = DIFF_PART_DIFFERENT_VMCOUNTS - if valid != 0: - print(f"VMCOUNT:{valid}") - else: - print(f"VMCOUNT:{vmcounts[0]}") diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl deleted file mode 100644 index 810a0742b0..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl +++ /dev/null @@ -1,103 +0,0 @@ -SCRIPTS_DIR = "{scripts_dir}" -NO_VAL = 4294967294 --- get_tpu_vmcount.py exit code -PART_INVALID = -1 -- partition does not exists in config.yaml, thus do not exist in slurm -DIFF_VMCOUNTS_SAME_PART = -2 -- in the same partition there are nodesets with different vmcounts -DIFF_PART_DIFFERENT_VMCOUNTS = -3 -- partition is a list of partitions in which at least two of them have different vmcount -UNKWOWN_ERROR = -4 -- get_tpu_vmcount.py did not return a valid response - -function get_part(job_desc, part_list) - if job_desc.partition then - return job_desc.partition - end - for name, val in pairs(part_list) do - if val.flag_default == 1 then - return name - end - end - return nil -end - -function os.capture(cmd, raw) - local handle = assert(io.popen(cmd, 'r')) - local output = assert(handle:read('*a')) - handle:close() - return output -end - -function get_vmcount(part) - local cmd = SCRIPTS_DIR .. "/get_tpu_vmcount.py -p " .. part - local out = os.capture(cmd, true) - for line in out:gmatch("(.-)\r?\n") do - local tag, val = line:match("([^:]+):([^:]+)") - if tag == "VMCOUNT" then - return tonumber(val) - end - end - return UNKWOWN_ERROR -end - -function slurm_job_submit(job_desc, part_list, submit_uid) - local part = get_part(job_desc, part_list) - local vmcount = get_vmcount(part) - -- Only do something if the job is in a TPU partition, if vmcount is 0, it implies that the partition(s) specified are not TPU ones - if vmcount == 0 then - return slurm.SUCCESS - end - -- This is a TPU job, but as the vmcount is 1 it can he handled the same way - if vmcount == 1 then - return slurm.SUCCESS - end - -- Check for errors - if vmcount == PART_INVALID then - slurm.log_user("Invalid partition specified " .. part) - return slurm.FAILURE - end - if vmcount == DIFF_VMCOUNTS_SAME_PART then - slurm.log_user("In partition(s) " .. part .. - " there are more than one tpu nodeset vmcount, this should not happen.") - return slurm.ERROR - end - if vmcount == DIFF_PART_DIFFERENT_VMCOUNTS then - slurm.log_user("In partition list " .. part .. - " there are more than one TPU types, cannot determine which is the correct vmcount to use, please retry with only one partition.") - return slurm.FAILURE - end - if vmcount == UNKWOWN_ERROR then - slurm.log_user("Something went wrong while executing get_tpu_vmcount.py.") - return slurm.ERROR - end - -- This is surely a TPU node - if vmcount > 1 then - local min_nodes = job_desc.min_nodes - local max_nodes = job_desc.max_nodes - -- if not specified assume it is one, this should be improved taking into account the cpus, mem, and other factors - if min_nodes == NO_VAL then - min_nodes = 1 - max_nodes = 1 - end - -- as max_nodes can be higher than the nodes in the partition, we are not able to calculate with certainty the nodes that this job will have if this value is set to something - -- different than min_nodes - if min_nodes ~= max_nodes then - slurm.log_user("Max nodes cannot be set different than min nodes for the TPU partitions.") - return slurm.ERROR - end - -- Set the number of switches to the number of nodes originally requested by the job, as the job requests "TPU groups" - job_desc.req_switch = min_nodes - - -- Apply the node increase into the job description. - job_desc.min_nodes = min_nodes * vmcount - job_desc.max_nodes = max_nodes * vmcount - -- if job_desc.features then - -- slurm.log_user("Features: %s",job_desc.features) - -- end - end - - return slurm.SUCCESS -end - -function slurm_job_modify(job_desc, job_rec, part_list, modify_uid) - return slurm.SUCCESS -end - -return slurm.SUCCESS diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py deleted file mode 100644 index cabd6e3e9f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py +++ /dev/null @@ -1,352 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, Callable, Any -import argparse -import os -import shelve -import uuid -from collections import namedtuple -from datetime import datetime, timedelta, timezone -from pathlib import Path -from pprint import pprint - -import util -from google.api_core import exceptions, retry -from google.cloud import bigquery as bq -from google.cloud.bigquery import SchemaField # type: ignore -from util import lookup, run - -SACCT = "sacct" -script = Path(__file__).resolve() - -DEFAULT_TIMESTAMP_FILE = script.parent / "bq_timestamp" -timestamp_file = Path(os.environ.get("TIMESTAMP_FILE", DEFAULT_TIMESTAMP_FILE)) -# The maximum request to insert_rows is 10MB, each sacct row is about 1200 bytes or ~ 8000 rows. -# Set to 5000 for a little wiggle room. -BQ_ROW_BATCH_SIZE = 5000 - -# cluster_id_file = script.parent / 'cluster_uuid' -# try: -# cluster_id = cluster_id_file.read_text().rstrip() -# except FileNotFoundError: -# cluster_id = uuid.uuid4().hex -# cluster_id_file.write_text(cluster_id) - -job_idx_cache_path = script.parent / "bq_job_idx_cache" - -SLURM_TIME_FORMAT = r"%Y-%m-%dT%H:%M:%S" - - -def make_datetime(time_string): - if time_string == "None": - return None - return datetime.strptime(time_string, SLURM_TIME_FORMAT).replace( - tzinfo=timezone.utc - ) - - -def make_time_interval(seconds): - sign = 1 - if seconds < 0: - sign = -1 - seconds = abs(seconds) - d, r = divmod(seconds, 60 * 60 * 24) - h, r = divmod(r, 60 * 60) - m, s = divmod(r, 60) - d *= sign - h *= sign - return f"{d}D {h:02}:{m:02}:{s}" - - -converters: Dict[str, Callable[[Any], Any]] = { - "DATETIME": make_datetime, - "INTERVAL": make_time_interval, - "STRING": str, - "INT64": lambda n: int(n or 0), -} - - -def schema_field(field_name, data_type, description, required=False): - return SchemaField( - field_name, - data_type, - description=description, - mode="REQUIRED" if required else "NULLABLE", - ) - - -schema_fields = [ - schema_field("cluster_name", "STRING", "cluster name", required=True), - schema_field("cluster_id", "STRING", "UUID for the cluster", required=True), - schema_field("entry_uuid", "STRING", "entry UUID for the job row", required=True), - schema_field( - "job_db_uuid", "STRING", "job db index from the slurm database", required=True - ), - schema_field("job_id_raw", "INT64", "raw job id", required=True), - schema_field("job_id", "STRING", "job id", required=True), - schema_field("state", "STRING", "final job state", required=True), - schema_field("job_name", "STRING", "job name"), - schema_field("partition", "STRING", "job partition"), - schema_field("submit_time", "DATETIME", "job submit time"), - schema_field("start_time", "DATETIME", "job start time"), - schema_field("end_time", "DATETIME", "job end time"), - schema_field("elapsed_raw", "INT64", "STRING", "job run time in seconds"), - # schema_field("elapsed_time", "INTERVAL", "STRING", "job run time interval"), - schema_field("timelimit_raw", "STRING", "job timelimit in minutes"), - schema_field("timelimit", "STRING", "job timelimit"), - # schema_field("num_tasks", "INT64", "number of allocated tasks in job"), - schema_field("nodelist", "STRING", "names of nodes allocated to job"), - schema_field("user", "STRING", "user responsible for job"), - schema_field("uid", "INT64", "uid of job user"), - schema_field("group", "STRING", "group of job user"), - schema_field("gid", "INT64", "gid of job user"), - schema_field("wckey", "STRING", "job wckey"), - schema_field("qos", "STRING", "job qos"), - schema_field("comment", "STRING", "job comment"), - schema_field("admin_comment", "STRING", "job admin comment"), - # extra will be added in 23.02 - # schema_field("extra", "STRING", "job extra field"), - schema_field("exitcode", "STRING", "job exit code"), - schema_field("alloc_cpus", "INT64", "count of allocated CPUs"), - schema_field("alloc_nodes", "INT64", "number of nodes allocated to job"), - schema_field("alloc_tres", "STRING", "allocated trackable resources (TRES)"), - # schema_field("system_cpu", "INTERVAL", "cpu time used by parent processes"), - # schema_field("cpu_time", "INTERVAL", "CPU time used (elapsed * cpu count)"), - schema_field("cpu_time_raw", "INT64", "CPU time used (elapsed * cpu count)"), - # schema_field("ave_cpu", "INT64", "Average CPU time of all tasks in job"), - # schema_field( - # "tres_usage_tot", - # "STRING", - # "Tres total usage by all tasks in job", - # ), -] - - -slurm_field_map = { - "job_db_uuid": "DBIndex", - "job_id_raw": "JobIDRaw", - "job_id": "JobID", - "state": "State", - "job_name": "JobName", - "partition": "Partition", - "submit_time": "Submit", - "start_time": "Start", - "end_time": "End", - "elapsed_raw": "ElapsedRaw", - "elapsed_time": "Elapsed", - "timelimit_raw": "TimelimitRaw", - "timelimit": "Timelimit", - "num_tasks": "NTasks", - "nodelist": "Nodelist", - "user": "User", - "uid": "Uid", - "group": "Group", - "gid": "Gid", - "wckey": "Wckey", - "qos": "Qos", - "comment": "Comment", - "admin_comment": "AdminComment", - # "extra": "Extra", - "exit_code": "ExitCode", - "alloc_cpus": "AllocCPUs", - "alloc_nodes": "AllocNodes", - "alloc_tres": "AllocTres", - "system_cpu": "SystemCPU", - "cpu_time": "CPUTime", - "cpu_time_raw": "CPUTimeRaw", - "ave_cpu": "AveCPU", - "tres_usage_tot": "TresUsageInTot", -} - -# new field name is the key for job_schema. Used to lookup the datatype when -# creating the job rows -job_schema = {field.name: field for field in schema_fields} -# Order is important here, as that is how they are parsed from sacct output -Job = namedtuple("Job", job_schema.keys()) # type: ignore -# ... see https://github.com/python/mypy/issues/848 - -client = bq.Client( - project=lookup().cfg.project, - credentials=util.default_credentials(), - client_options=util.create_client_options(util.ApiEndpoint.BQ), -) -dataset_id = f"{lookup().cfg.slurm_cluster_name}_job_data" -dataset = bq.DatasetReference(project=lookup().project, dataset_id=dataset_id) -table = bq.Table( - bq.TableReference(dataset, f"jobs_{lookup().cfg.slurm_cluster_name}"), schema_fields -) - - -class JobInsertionFailed(Exception): - pass - - -def make_job_row(job): - job_row = { - field_name: converters[field.field_type](job[field_name]) - for field_name, field in job_schema.items() - if field_name in job - } - job_row["entry_uuid"] = uuid.uuid4().hex - job_row["cluster_id"] = lookup().cfg.cluster_id - job_row["cluster_name"] = lookup().cfg.slurm_cluster_name - return job_row - - -def load_slurm_jobs(start, end): - states = ",".join( - ( - "BOOT_FAIL", - "CANCELLED", - "COMPLETED", - "DEADLINE", - "FAILED", - "NODE_FAIL", - "OUT_OF_MEMORY", - "PREEMPTED", - "REQUEUED", - "REVOKED", - "TIMEOUT", - ) - ) - start_iso = start.isoformat(timespec="seconds") - end_iso = end.isoformat(timespec="seconds") - # slurm_fields and bq_fields will be in matching order - slurm_fields = ",".join(slurm_field_map.values()) - bq_fields = slurm_field_map.keys() - cmd = ( - f"{SACCT} --start {start_iso} --end {end_iso} -X -D --format={slurm_fields} " - f"--state={states} --parsable2 --noheader --allusers --duplicates" - ) - text = run(cmd).stdout.splitlines() - # zip pairs bq_fields with the value from sacct - jobs = [dict(zip(bq_fields, line.split("|"))) for line in text] - - # The job index cache allows us to avoid sending duplicate jobs. This avoids a race condition with updating the database. - with shelve.open(str(job_idx_cache_path), flag="r") as job_idx_cache: - job_rows = [ - make_job_row(job) - for job in jobs - if str(job["job_db_uuid"]) not in job_idx_cache - ] - return job_rows - - -def init_table(): - global dataset - global table - dataset = client.create_dataset(dataset, exists_ok=True) # type: ignore - table = client.create_table(table, exists_ok=True) - until_found = retry.Retry(predicate=retry.if_exception_type(exceptions.NotFound)) - table = client.get_table(table, retry=until_found) - # cannot add required fields to an existing schema - table.schema = schema_fields - table = client.update_table(table, ["schema"]) - - -def purge_job_idx_cache(): - purge_time = datetime.now() - timedelta(minutes=30) - with shelve.open(str(job_idx_cache_path), writeback=True) as cache: - to_delete = [] - for idx, stamp in cache.items(): - if stamp < purge_time: - to_delete.append(idx) - for idx in to_delete: - del cache[idx] - - -def bq_submit(jobs): - try: - result = client.insert_rows(table, jobs) - except exceptions.NotFound as e: - print(f"failed to upload job data, table not yet found: {e}") - raise e - except Exception as e: - print(f"failed to upload job data: {e}") - raise e - if result: - pprint(jobs) - pprint(result) - raise JobInsertionFailed("failed to upload job data to big query") - print(f"successfully loaded {len(jobs)} jobs") - - -def get_time_window(): - if not timestamp_file.is_file(): - timestamp_file.touch() - try: - timestamp = datetime.strptime( - timestamp_file.read_text().rstrip(), SLURM_TIME_FORMAT - ) - # time window will overlap the previous by 10 minutes. Duplicates will be filtered out by the job_idx_cache - start = timestamp - timedelta(minutes=10) - except ValueError: - # timestamp 1 is 1 second after the epoch; timestamp 0 is special for sacct - start = datetime.fromtimestamp(1) - # end is now() truncated to the last second - end = datetime.now().replace(microsecond=0) - return start, end - - -def write_timestamp(time): - timestamp_file.write_text(time.isoformat(timespec="seconds")) - - -def update_job_idx_cache(jobs, timestamp): - with shelve.open(str(job_idx_cache_path), writeback=True) as job_idx_cache: - for job in jobs: - job_idx = str(job["job_db_uuid"]) - job_idx_cache[job_idx] = timestamp - - -def main(): - if not lookup().cfg.enable_bigquery_load: - print("bigquery load is not currently enabled") - exit(0) - init_table() - - start, end = get_time_window() - jobs = load_slurm_jobs(start, end) - # on failure, an exception will cause the timestamp not to be rewritten. So - # it will try again next time. If some writes succeed, we don't currently - # have a way to not submit duplicates next time. - if jobs: - num_batches = (len(jobs) - 1) // BQ_ROW_BATCH_SIZE + 1 - print( - f"loading {num_batches} batches of BigQuery data in batches of size : {BQ_ROW_BATCH_SIZE}" - ) - for batch_indx, job_indx in enumerate(range(0, len(jobs), BQ_ROW_BATCH_SIZE)): - print(f"loading BigQuery data batch {batch_indx} of {num_batches}") - bq_submit(jobs[job_indx : job_indx + BQ_ROW_BATCH_SIZE]) - write_timestamp(end) - update_job_idx_cache(jobs, end) - - -parser = argparse.ArgumentParser(description="submit slurm job data to big query") -parser.add_argument( - "timestamp_file", - nargs="?", - action="store", - type=Path, - help="specify timestamp file for reading and writing the time window start. Precedence over TIMESTAMP_FILE env var.", -) - -purge_job_idx_cache() -if __name__ == "__main__": - args = parser.parse_args() - if args.timestamp_file: - timestamp_file = args.timestamp_file.resolve() - main() diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py deleted file mode 100644 index d4a4477f83..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -Implementation of message queue that mimics interface of GCP (PubSub)[https://cloud.google.com/pubsub] - -Messages are stored on controller state disk (to survive controller re-creation) with following layout: - -// -├- -| └- -└- .staging - └- - └- - -One message is one immutable file, that will be deleted after acknowledgement. -NOTE: Implementation assumes that both `` and `.staging/` are on the same disk device, -so it can rely on atomic "move / rename" operation. -""" -from typing import Any -import util -import json -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path -import os -import uuid - -import logging -log = logging.getLogger() - - -@dataclass(frozen=True) -class Message: - id: str - created: datetime - data: Any - - def to_json(self) -> dict[str, str]: - return dict( - id=self.id, - created=self.created.isoformat(), - data=self.data) - - @classmethod - def from_json(cls, data: dict[str, str]) -> 'Message': - return cls( - id=data['id'], - created=datetime.fromisoformat(data['created']), - data=data['data']) - -class Topic: - """ - Acts as PubSub topic (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics). - We can have multiple instances of - """ - def __init__(self, path: Path, staging: Path) -> None: - self._path = path - self._staging = staging - - def _gen_id(self, created: datetime) -> str: - ts = created.strftime("%Y_%m_%d-%H_%M_%S") - suf = str(uuid.uuid4())[:8] - return f"{ts}-{suf}" - - def publish(self, data: Any) -> None: - created = util.now() - id = self._gen_id(created) - msg = Message(id=id, created=created, data=data) - - staged = self._staging / msg.id - dst = self._path / msg.id - - # Write to stagin area first then perform atomic move - # to prevent "reads of partial writes" - staged.write_text(json.dumps(msg.to_json())) - util.chown_slurm(staged) - staged.rename(dst) - - -class Subscription: - """ - Acts as PubSub subscription (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) - with following settings: - - ``` - ackDeadlineSeconds = +Inf # don't resend message that was already being delivered but not acked yet - retainAckedMessages = False # don't persist messages that were already acked - enableMessageOrdering = True # delivers messages in chronoligical order - messageRetentionDuration = +Inf # don't expire messages - deadLetterPolicy = None # "deadlettering" is disabled, subscriber should take care of any poisonous messages - retryPolicy = { # NACKed message will be re-delievered after some time - minimumBackoff = 30s # NOTE: Practically there is no timer, but Subscription instance will not try to re-deliver NACKed messages. - maximumBackoff = 30s # Assumes that slurmsync runs every 30+ sec. - } - ``` - - IMPORTANT: Should only be run as part of slurmsync, - this is our way to ensure that at most one instance exists at a time. - There is no concurancy safeguards in place, avoid multithreaded `pull`, - while multithreaded `ack` & `modify_ack_deadline` are OK. - """ - - def __init__(self, path: Path) -> None: - self._path: Path = path - # contains ALL messages pulled by this subscription instance - # both acked, nacked, and still being processed - # used to prevent double delivery within lifetime of subscription (slurmsync) - self._pulled: set[str] = set() - - def _delete(self, id: str) -> None: - log.debug(f"removing {id}") - try: - os.unlink(self._path / id) - except: - log.exception(f"Failed to remove message {id}") - - def _read_msg(self, id: str) -> Message | None: - try: - with open(self._path / id, 'r') as f: - content = json.loads(f.read()) - return Message.from_json(content) - except Exception: - log.exception(f"Failed to read message {id}") - self._delete(id) # delete message to reduce "deadlettering" - return None - - def pull(self, max_messages: int) -> list[Message]: - if not self._path.exists(): - log.warning(f"Topic {self._path} does not exist") - return [] - res = [] - ls = sorted(os.listdir(self._path)) - for name in ls: - msg = self._read_msg(name) - if msg is not None and msg.id not in self._pulled: - self._pulled.add(msg.id) - res.append(msg) - - if len(res) >= max_messages: - break - return res - - - def ack(self, ids: list[str]) -> None: - for id in ids: - self._delete(id) - - - def modify_ack_deadline(self, ids: list[str], deadline: int) -> None: - """ - Modifies the ack deadline for a specific message. - IMPORTANT: Only accepts deadline=0, which is a way to NACK - Any other values are also meaningless due to ackDeadlineSeconds==+Inf - """ - assert deadline == 0 # no op, next subscriber (slurmsync) will pick this up - - -# Topics and Subscriptions are singletons -# TODO: consider making thread-safe -_topics = {} -_subscriptions = {} - -def _make_path(name: str) -> Path: - p = util.slurmdirs.state / "pubsub" / name - p.mkdir(parents=True, exist_ok=True) - util.chown_slurm(p) - return p - -def _make_staging_path(name: str) -> Path: - p = util.slurmdirs.state / "pubsub" / ".staging" / name - p.mkdir(parents=True, exist_ok=True) - util.chown_slurm(p) - return p - -def topic(name: str) -> Topic: - if name not in _topics: - _topics[name] = Topic(_make_path(name), _make_staging_path(name)) - return _topics[name] - -def subscription(name: str) -> Subscription: - if name not in _subscriptions: - _subscriptions[name] = Subscription(_make_path(name)) - return _subscriptions[name] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py deleted file mode 100644 index 8ea3d0657e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional - -import util -import uuid -from addict import Dict as NSDict # type: ignore -from datetime import datetime, timedelta -from collections import defaultdict -import logging -from time import sleep - -log = logging.getLogger() - -DWS_EOL_RESERVATION_DURATION = 10 # minutes - -def _duration(flex_options: NSDict, job_id: Optional[int], lkp: util.Lookup) -> int: - dur = flex_options.max_run_duration - if not job_id or not flex_options.use_job_duration: - return dur - - job = lkp.job(job_id) - if not job or not job.duration: - return dur - - if timedelta(minutes=10) <= job.duration <= timedelta(weeks=1): - return int(job.duration.total_seconds()) - - log.info("Job TimeLimit cannot be less than 10 minutes or exceed one week") - return dur - -def _create_slurm_reservation(node_name: str, boot_time: datetime, run_duration: int, lkp: util.Lookup): - """ - Create a Slurm reservation starting at EOL - buffer time. - """ - eol = boot_time + timedelta(seconds=run_duration) - start_str = eol.strftime("%Y-%m-%dT%H:%M:%S") - reservation_name = f"dws-eol-{node_name}" - log.debug(f"creating slurm reservation for {node_name}") - try: - util.run(f"{lkp.scontrol} create reservation user=slurm starttime={start_str} duration={DWS_EOL_RESERVATION_DURATION} nodes={node_name} reservationname={reservation_name} flags=maint,ignore_jobs") - except Exception as e: - log.error(f"Failed to create reservation for {node_name}: {e}") - -def _delete_slurm_reservation(node_name: str, lkp: util.Lookup): - """ - Delete the Slurm reservation for the given node. - """ - reservation_name = f"dws-eol-{node_name}" - try: - util.run(f"{lkp.scontrol} delete reservation {reservation_name}") - log.debug(f"Deleted Slurm reservation {reservation_name} for {node_name}") - except Exception as e: - log.error(f"Failed to delete reservation for {node_name}: {e}") - -def resume_flex_chunk(nodes: List[str], job_id: Optional[int], lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - assert len(nodeset.zone_policy_allow) > 0 - region = lkp.node_region(model) - - assert nodeset.dws_flex.enabled - - uid = str(uuid.uuid4())[:8] - if job_id: - mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-job-{job_id}-{uid}" - else: - mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-{uid}" - - # Create MIG - req = lkp.compute.regionInstanceGroupManagers().insert( - project=lkp.project, - region=region, - body=dict( - name=mig_name, - versions=[dict(instanceTemplate=nodeset.instance_template)], - targetSize=0, - distributionPolicy=dict( - zones=[ - dict(zone=f"zones/{z}") for z in nodeset.zone_policy_allow - ], - targetShape="ANY_SINGLE_ZONE" ), - updatePolicy = dict(instanceRedistributionType = "NONE" ), - instanceLifecyclePolicy=dict(defaultActionOnFailure= "DO_NOTHING" ), # TODO(FLEX): Not supported yet, migrate once supported - ) - ) - util.log_api_request(req) - op = req.execute() - res = util.wait_for_operation(op) - assert "error" not in res, f"{res}" - - # Create resize request - duration_seconds = _duration(nodeset.dws_flex, job_id, lkp) - req = lkp.compute.regionInstanceGroupManagerResizeRequests().insert( - project=lkp.project, - region=region, - instanceGroupManager=mig_name, - body=dict( - name="initial-resize", - instances=[dict(name=n) for n in nodes], - requested_run_duration=dict( - seconds=duration_seconds - ) - ) - ) - util.log_api_request(req) - op = req.execute() - res = util.wait_for_operation(op) - - # Create Slurm reservations if use_job_duration is set - if nodeset.dws_flex.use_job_duration: - # Get run duration (seconds) - run_duration = duration_seconds - for node_name in nodes: - # Fetch instance creation time from GCP instance (via util.py) - instance = lkp.instance(node_name) - if(instance and instance.creation_timestamp): - log.debug("creating with creation_timestamp") - boot_time = instance.creation_timestamp # Already a datetime object - else: - boot_time = datetime.utcnow() - log.debug("creating with utcnow time: {boot_time}") - _create_slurm_reservation(node_name, boot_time, run_duration, lkp) - - assert "error" not in res, f"{res}" - -def _suspend_flex_mig(mig_self_link: str, nodes: List[str], lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - assert len(nodeset.zone_policy_allow) > 0 - region = lkp.node_region(model) - project=lkp.project - instanceGroupManager=util.trim_self_link(mig_self_link) - - links = [ - f"zones/{inst.zone}/instances/{inst.name}" - for inst in [ - lkp.instance(node) for node in nodes - ] if inst - ] - - target_mig=lkp.get_mig(lkp.project, region, instanceGroupManager) - assert target_mig - - # TODO(FLEX): This will not work if MIG didn't obtain capacity yet. - # The request will fail and MIG will continue provisioning. - # Instead whole MIG should be deleted. - # + All other instances in MIG are not provisioned also, safe to delete - # - Need to come up will clear test to differentiate non-provisioned MIG and single VM being down; - # Particularly CRITICAL due to ActionOnFailure=DO_NOTHING - # - Need to `down_nodes_notify_jobs` for all nodes in MIG, make sure that it doesn't interfere with Slurm suspend-flow. - - if target_mig["targetSize"] == len(nodes): #We can just delete the whole MIG in this case - req = lkp.compute.regionInstanceGroupManagers().delete( - project=project, - region=region, - instanceGroupManager=instanceGroupManager, - ) - else: - req = lkp.compute.regionInstanceGroupManagers().deleteInstances( - project=project, - region=region, - instanceGroupManager=instanceGroupManager, - body=dict( - instances=links, - skipInstancesOnValidationError=True, - ) - ) - - util.log_api_request(req) - op = req.execute() - - res = util.wait_for_operation(op) - - # Delete Slurm reservations for nodes being deprovisioned - for node_name in nodes: - log.info("delete dws reservation") - _delete_slurm_reservation(node_name, lkp) - - assert "error" not in res, f"{res}" - -def _suspend_provisioning_inst(nodes:List[str], node_template:str, lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - assert len(nodeset.zone_policy_allow) > 0 - region = lkp.node_region(model) - - mig_list=lkp.get_mig_list(lkp.project, region) - - # FLEX (#TODO): If we enter this conditional it's likely this was called so early that MIG creation hasn't started - # Consider potentially retrying? No natural mechanism for retry currently but we could - # perhaps use slurmsync and then try it again to ensure it wasn't a case of being too early. - # This is important since we're now enabling long ResumeTimeout (Slurm won't call suspend on node within reasonable timeframe) - # so until we do this is slurmsync this is a temporary workaround. - - if not mig_list or not mig_list.get("items"): - log.info("No matching MIG found to delete! Retrying...") - sleep(5) - mig_list=lkp.get_mig_list(lkp.project, region) - if not mig_list or not mig_list.get("items"): - return - - for mig in mig_list["items"]: - if mig["instanceTemplate"] == node_template: - if mig["currentActions"]["creating"] > 0 and mig["targetSize"] == mig["currentActions"]["creating"]: - req = lkp.compute.regionInstanceGroupManagers().delete( - project=lkp.project, - region=region, - instanceGroupManager=util.trim_self_link(mig["selfLink"]), - ) - - util.log_api_request(req) - op = req.execute() - - res = util.wait_for_operation(op) - assert "error" not in res, f"{res}" - return - - log.info("No matching MIG found to delete!") - -def suspend_flex_nodes(nodes: List[str], lkp: util.Lookup) -> None: - by_mig = defaultdict(list) - not_provisioned = defaultdict(list) - for node in nodes: - inst = lkp.instance(node) - if not inst: - not_provisioned[lkp.node_template(node)].append(node) - else: - mig = inst.metadata.get("created-by") - if not mig: - log.error(f"Can not suspend {node}, can not find associated MIG") - continue - by_mig[mig].append(node) - - for mig, nodes in by_mig.items(): - _suspend_flex_mig(mig, nodes, lkp) - - for node_template, nodes in not_provisioned.items(): - _suspend_provisioning_inst(nodes, node_template, lkp) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt deleted file mode 100644 index 2ab3162ccf..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ -pytest -pytest-mock -pytest_unordered -mock - -types-mock -types-httplib2 -types-requests -types-PyYAML diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt deleted file mode 100644 index e923e53dbf..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -addict==2.4.0 -google-api-core==2.19.0 -google-api-python-client==2.93.0 -google-auth==2.40.3 -google-auth-httplib2==0.1.0 -google-cloud-bigquery==3.11.3 -google-cloud-core==2.3.3 -google-cloud-secret-manager~=2.22 -google-cloud-storage==2.10.0 -google-cloud-tpu==1.10.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.1 -grpcio==1.60.0 -grpcio-status==1.60.0 -httplib2==0.22.0 -more-executors==2.11.4 -pyyaml==6.0.2 -requests==2.32.4 diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py deleted file mode 100644 index ea0012a0b1..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py +++ /dev/null @@ -1,703 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional, Dict, Any -import argparse -from datetime import timedelta -import shlex -import json -import logging -import os -import yaml -import collections -from pathlib import Path -from dataclasses import dataclass -from addict import Dict as NSDict # type: ignore - -import util -from util import ( - chunked, - ensure_execute, - execute_with_futures, - log_api_request, - map_with_futures, - run, - separate, - to_hostlist, - trim_self_link, - wait_for_operation, -) -from util import lookup, ReservationDetails -import tpu -import mig_flex - -log = logging.getLogger() - -PLACEMENT_MAX_CNT = 1500 -# Placement group needs to be the same for an entire bulk_insert hence -# if placement is used the actual BULK_INSERT_LIMIT will be -# max([1000, PLACEMENT_MAX_CNT]) -BULK_INSERT_LIMIT = 5000 - -# https://cloud.google.com/compute/docs/instance-groups#types_of_managed_instance_groups -ZONAL_MIG_SIZE_LIMIT = 1000 - - -@dataclass(frozen=True) -class ResumeJobData: - job_id: int - partition: str - nodes_alloc: List[str] - -@dataclass(frozen=True) -class ResumeData: - jobs: List[ResumeJobData] - - -def get_resume_file_data() -> Optional[ResumeData]: - if not (path := os.getenv("SLURM_RESUME_FILE")): - log.error("SLURM_RESUME_FILE was not in environment. Cannot get detailed job, node, partition allocation data.") - return None - blob = Path(path).read_text() - log.debug(f"Resume data: {blob}") - data = json.loads(blob) - - jobs = [] - for jo in data.get("jobs", []): - job = ResumeJobData( - job_id = jo.get("job_id"), - partition = jo.get("partition"), - nodes_alloc = util.to_hostnames(jo.get("nodes_alloc")), - ) - jobs.append(job) - return ResumeData(jobs=jobs) - -def instance_properties(nodeset: NSDict, model:str, placement_group:Optional[str], labels:Optional[dict], job_id:Optional[int]): - props = NSDict() - - if labels: # merge in extra labels on instance and disks - template_link = lookup().node_template(model) - template_info = lookup().template_info(template_link) - - props.labels = {**template_info.labels, **labels} - - for disk in template_info.disks: - if disk.initializeParams.get("diskType", "local-ssd") == "local-ssd": - continue # do not label local ssd - disk.initializeParams.labels.update(labels) - props.disks = template_info.disks - - if placement_group: - props.resourcePolicies = [placement_group] - - if reservation := lookup().nodeset_reservation(nodeset): - update_reservation_props(reservation, props, placement_group, reservation.calendar) - - if (fr := lookup().future_reservation(nodeset)) and fr.specific: - assert fr.active_reservation - update_reservation_props(fr.active_reservation, props, placement_group, fr.calendar) - - if props.resourcePolicies: - props.scheduling.onHostMaintenance = "TERMINATE" - - if nodeset.maintenance_interval: - props.scheduling.maintenanceInterval = nodeset.maintenance_interval - - if nodeset.dws_flex.enabled and nodeset.dws_flex.use_bulk_insert: - update_props_dws(props, nodeset.dws_flex, job_id) - - # Override with properties explicit specified in the nodeset - props.update(nodeset.get("instance_properties") or {}) - return props - -def update_reservation_props(reservation:ReservationDetails, props:NSDict, placement_group:Optional[str], calendar_mode:bool) -> None: - props.reservationAffinity = { - "consumeReservationType": "SPECIFIC_RESERVATION", - "key": f"compute.{util.universe_domain()}/reservation-name", - "values": [reservation.bulk_insert_name], - } - - if reservation.dense or calendar_mode: - props.scheduling.provisioningModel = "RESERVATION_BOUND" - - # Figure out `resourcePolicies` - if reservation.policies: # use ones already attached to reservations - props.resourcePolicies = reservation.policies - elif reservation.dense and placement_group: # use once created by Slurm - props.resourcePolicies = [placement_group] - else: # vanilla reservations don't support external policies - props.resourcePolicies = [] - log.info( - f"reservation {reservation.bulk_insert_name} is being used with resourcePolicies: {props.resourcePolicies}") - -def update_props_dws(props: NSDict, dws_flex: NSDict, job_id: Optional[int]) -> None: - props.scheduling.onHostMaintenance = "TERMINATE" - props.scheduling.instanceTerminationAction = "DELETE" - props.reservationAffinity['consumeReservationType'] = "NO_RESERVATION" - props.scheduling.maxRunDuration['seconds'] = dws_flex_duration(dws_flex, job_id) - -def dws_flex_duration(dws_flex: NSDict, job_id: Optional[int]) -> int: - max_duration = dws_flex.max_run_duration - if dws_flex.use_job_duration and job_id is not None and (job := lookup().job(job_id)) and job.duration: - if timedelta(seconds=30) <= job.duration <= timedelta(weeks=1): - max_duration = int(job.duration.total_seconds()) - else: - log.info("Job TimeLimit cannot be less than 30 seconds or exceed one week") - return max_duration - -def create_instances_request(nodes: List[str], placement_group: Optional[str], excl_job_id: Optional[int]): - """Call regionInstances.bulkInsert to create instances""" - assert 0 < len(nodes) <= BULK_INSERT_LIMIT - - # model here indicates any node that can be used to describe the rest - model = next(iter(nodes)) - log.debug(f"create_instances_request: {model} placement: {placement_group}") - - nodeset = lookup().node_nodeset(model) - template = lookup().node_template(model) - labels = {"slurm_job_id": excl_job_id} if excl_job_id else None - - body = dict( - count = len(nodes), - sourceInstanceTemplate = template, - # key is instance name, value overwrites properties (no overwrites) - perInstanceProperties = {k: {} for k in nodes}, - instanceProperties = instance_properties( - nodeset, model, placement_group, labels, excl_job_id - ), - ) - - if placement_group and excl_job_id is not None: - pass # do not set minCount to force "all or nothing" behavior - else: - body["minCount"] = 1 - - zone_allow = nodeset.zone_policy_allow or [] - zone_deny = nodeset.zone_policy_deny or [] - - if len(zone_allow) == 1: # if only one zone is used, use zonal BulkInsert API, as less prone to errors - api_method = lookup().compute.instances().bulkInsert - method_args = {"zone": zone_allow[0]} - else: - api_method = lookup().compute.regionInstances().bulkInsert - method_args = {"region": lookup().node_region(model)} - - body["locationPolicy"] = dict( - locations = { - **{ f"zones/{z}": {"preference": "ALLOW"} for z in zone_allow }, - **{ f"zones/{z}": {"preference": "DENY"} for z in zone_deny }}, - targetShape = nodeset.zone_target_shape, - ) - - req = api_method( - project=lookup().project, - body=body, - **method_args) - log.debug(f"new request: endpoint={req.methodId} nodes={to_hostlist(nodes)}") - log_api_request(req) - return req - -@dataclass() -class PlacementAndNodes: - placement: Optional[str] - nodes: List[str] - -@dataclass(frozen=True) -class BulkChunk: - nodes: List[str] - prefix: str # - - chunk_idx: int - excl_job_id: Optional[int] - placement_group: Optional[str] = None - - @property - def name(self): - if self.placement_group is not None: - return f"{self.prefix}:job{self.excl_job_id}:{self.placement_group}:{self.chunk_idx}" - if self.excl_job_id is not None: - return f"{self.prefix}:job{self.excl_job_id}:{self.chunk_idx}" - return f"{self.prefix}:{self.chunk_idx}" - - -def group_nodes_bulk(nodes: List[str], resume_data: Optional[ResumeData], lkp: util.Lookup): - """group nodes by nodeset, placement_group, exclusive_job_id if any""" - if resume_data is None: # all nodes will be considered jobless - resume_data = ResumeData(jobs=[]) - - nodes_set = set(nodes) # turn into set to simplify intersection - non_excl = nodes_set.copy() - groups : Dict[Optional[int], List[PlacementAndNodes]] = {} # excl_job_id|none -> PlacementAndNodes - - # expand all exclusive job nodelists - for job in resume_data.jobs: - if not lkp.cfg.partitions[job.partition].enable_job_exclusive: - continue - - groups[job.job_id] = [] - # placement group assignment is based on all allocated nodes, ... - for pn in create_placements(job.nodes_alloc, job.job_id, lkp): - groups[job.job_id].append( - PlacementAndNodes( - placement=pn.placement, - #... but we only want to handle nodes in nodes_resume in this run. - nodes = sorted(set(pn.nodes) & nodes_set) - )) - non_excl.difference_update(job.nodes_alloc) - - groups[None] = create_placements(sorted(non_excl), excl_job_id=None, lkp=lkp) - - def chunk_nodes(nodes: List[str]): - if not nodes: - return [] - - model = nodes[0] - - if lkp.is_flex_node(model): - chunk_size = ZONAL_MIG_SIZE_LIMIT - elif lkp.node_is_tpu(model): - ns_name = lkp.node_nodeset_name(model) - chunk_size = tpu.TPU.make(ns_name, lkp).vmcount - else: - chunk_size = BULK_INSERT_LIMIT - - return chunked(nodes, n=chunk_size) - - chunks = [ - BulkChunk( - nodes=nodes_chunk, - prefix=lkp.node_prefix(nodes_chunk[0]), # - - excl_job_id = job_id, - placement_group=pn.placement, - chunk_idx=i) - - for job_id, placements in groups.items() - for pn in placements if pn.nodes - for i, nodes_chunk in enumerate(chunk_nodes(pn.nodes)) - ] - return {chunk.name: chunk for chunk in chunks} - - -def resume_nodes(nodes: List[str], resume_data: Optional[ResumeData]): - """resume nodes in nodelist""" - lkp = lookup() - # Prevent dormant nodes associated with a reservation from being resumed - nodes, dormant_res_nodes = util.separate(lkp.is_dormant_res_node, nodes) - - if dormant_res_nodes: - log.warning(f"Resume was unable to resume reservation nodes={dormant_res_nodes}") - down_nodes_notify_jobs(dormant_res_nodes, "Reservation is not active, nodes cannot be resumed", resume_data) - - nodes, flex_managed = util.separate(lkp.is_provisioning_flex_node, nodes) - if flex_managed: - log.warning(f"Resume was unable to resume nodes={flex_managed} already managed by MIGs") - down_nodes_notify_jobs(flex_managed, "VM is managed MIG, can not be resumed", resume_data) - - if not nodes: - log.info("No nodes to resume") - return - - nodes = sorted(nodes, key=lkp.node_prefix) - grouped_nodes = group_nodes_bulk(nodes, resume_data, lkp) - - if log.isEnabledFor(logging.DEBUG): - grouped_nodelists = { - group: to_hostlist(chunk.nodes) for group, chunk in grouped_nodes.items() - } - log.debug( - "node bulk groups: \n{}".format(yaml.safe_dump(grouped_nodelists).rstrip()) - ) - - tpu_chunks, flex_chunks = [], [] - bi_inserts = {} - - for group, chunk in grouped_nodes.items(): - model = chunk.nodes[0] - - if lkp.node_is_tpu(model): - tpu_chunks.append(chunk.nodes) - elif lkp.is_flex_node(model): - flex_chunks.append(chunk) - else: - bi_inserts[group] = create_instances_request( - chunk.nodes, chunk.placement_group, chunk.excl_job_id - ) - - for chunk in flex_chunks: - mig_flex.resume_flex_chunk(chunk.nodes, chunk.excl_job_id, lkp) - - # execute all bulkInsert requests with batch - bulk_ops = dict( - zip(bi_inserts.keys(), map_with_futures(ensure_execute, bi_inserts.values())) - ) - log.debug(f"bulk_ops={yaml.safe_dump(bulk_ops)}") - started = { - group: op for group, op in bulk_ops.items() if not isinstance(op, Exception) - } - failed = { - group: err for group, err in bulk_ops.items() if isinstance(err, Exception) - } - if failed: - failed_reqs = [str(e) for e in failed.items()] - log.error("bulkInsert API failures: {}".format("; ".join(failed_reqs))) - for ident, exc in failed.items(): - down_nodes_notify_jobs(grouped_nodes[ident].nodes, f"GCP Error: {exc._get_reason()}", resume_data) # type: ignore - - if log.isEnabledFor(logging.DEBUG): - for group, op in started.items(): - group_nodes = grouped_nodelists[group] - name = op["name"] - gid = op["operationGroupId"] - log.debug( - f"new bulkInsert operation started: group={group} nodes={group_nodes} name={name} operationGroupId={gid}" - ) - # wait for all bulkInserts to complete and log any errors - bulk_operations = {group: wait_for_operation(op) for group, op in started.items()} - - # Start TPU after regular nodes so that regular nodes are not affected by the slower TPU nodes - execute_with_futures(tpu.start_tpu, tpu_chunks) - - for group, op in bulk_operations.items(): - _handle_bulk_insert_op(op, grouped_nodes[group].nodes, resume_data) - - -def _get_failed_zonal_instance_inserts(bulk_op: Any, zone: str, lkp: util.Lookup) -> list[Any]: - group_id = bulk_op["operationGroupId"] - user = bulk_op["user"] - started = bulk_op["startTime"] - ended = bulk_op["endTime"] - - fltr = f'(user eq "{user}") AND (operationType eq "insert") AND (creationTimestamp > "{started}") AND (creationTimestamp < "{ended}")' - act = lkp.compute.zoneOperations() - req = act.list(project=lkp.project, zone=zone, filter=fltr) - ops = [] - while req is not None: - result = util.ensure_execute(req) - for op in result.get("items", []): - if op.get("operationGroupId") == group_id and "error" in op: - ops.append(op) - req = act.list_next(req, result) - return ops - - -def _get_failed_instance_inserts(bulk_op: Any, lkp: util.Lookup) -> list[Any]: - zones = set() # gather zones that had failed inserts - for loc, stat in bulk_op.get("instancesBulkInsertOperationMetadata", {}).get("perLocationStatus", {}).items(): - pref, zone = loc.split("/", 1) - if not pref == "zones": - log.error(f"Unexpected location: {loc} in operation {bulk_op['name']}") - continue - if stat.get("targetVmCount", 0) != stat.get("createdVmCount", 0): - zones.add(zone) - - res = [] - for zone in zones: - res.extend(_get_failed_zonal_instance_inserts(bulk_op, zone, lkp)) - return res - -def _handle_bulk_insert_op(op: Dict, nodes: List[str], resume_data: Optional[ResumeData]) -> None: - """ - Handles **DONE** BulkInsert operations - """ - assert op["operationType"] == "bulkInsert" and op["status"] == "DONE", f"unexpected op: {op}" - - group_id = op["operationGroupId"] - if "error" in op: - error = op["error"]["errors"][0] - log.error( - f"bulkInsert operation error: {error['code']} name={op['name']} operationGroupId={group_id} nodes={to_hostlist(nodes)}" - ) - - created = 0 - for status in op["instancesBulkInsertOperationMetadata"]["perLocationStatus"].values(): - created += status.get("createdVmCount", 0) - if created == len(nodes): - log.info(f"created {len(nodes)} instances: nodes={to_hostlist(nodes)}") - return # no need to gather status of insert-operations. - - # TODO: don't gather insert-operations per bulkInsert request, instead aggregate it - # across all bulkInserts (goes one level above this function) - failed = _get_failed_instance_inserts(op, util.lookup()) - - # Multiple errors are possible, group by all of them (joined string codes) - by_error_inserts = util.groupby_unsorted( - failed, - lambda op: "+".join(err["code"] for err in op["error"]["errors"]), - ) - for code, failed_ops in by_error_inserts: - failed_ops = list(failed_ops) - failed_nodes = [trim_self_link(op["targetLink"]) for op in failed_ops] - hostlist = util.to_hostlist(failed_nodes) - log.error( - f"{len(failed_nodes)} instances failed to start: {code} ({hostlist}) operationGroupId={group_id}" - ) - - msg = "; ".join( - f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" - for err in failed_ops[0]["error"]["errors"] - ) - if code != "RESOURCE_ALREADY_EXISTS": - down_nodes_notify_jobs(failed_nodes, f"GCP Error: {msg}", resume_data) - log.error( - f"errors from insert for node '{failed_nodes[0]}' ({failed_ops[0]['name']}): {msg}" - ) - - -def down_nodes_notify_jobs(nodes: List[str], reason: str, resume_data: Optional[ResumeData]) -> None: - """set nodes down with reason""" - nodes_set = set(nodes) # turn into set to speed up intersection - jobs = resume_data.jobs if resume_data else [] - reason_quoted = shlex.quote(reason) - - for job in jobs: - if not (set(job.nodes_alloc) & nodes_set): - continue - run(f"{lookup().scontrol} update jobid={job.job_id} admincomment={reason_quoted}", check=False) - run(f"{lookup().scontrol} notify {job.job_id} {reason_quoted}", check=False) - - nodelist = util.to_hostlist(nodes) - log.error(f"Marking nodes {nodelist} as DOWN, reason: {reason}") - run(f"{lookup().scontrol} update nodename={nodelist} state=down reason={reason_quoted}", check=False) - - - - -def create_placement_request(pg_name: str, region: str, max_distance: Optional[int], accelerator_topology: Optional[str]): - config = { - "name": pg_name, - "region": region, - "groupPlacementPolicy": { - "collocation": "COLLOCATED", - "maxDistance": max_distance, - "gpuTopology": accelerator_topology, - }, - } - - request = lookup().compute.resourcePolicies().insert( - project=lookup().project, region=region, body=config - ) - log_api_request(request) - return request - - -def create_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - nodeset_map = collections.defaultdict(list) - for node in nodes: # split nodes on nodesets - nodeset_map[lkp.node_nodeset_name(node)].append(node) - - placements = [] - for _, ns_nodes in nodeset_map.items(): - placements.extend(create_nodeset_placements(ns_nodes, excl_job_id, lkp)) - return placements - - -def _allocate_nodes_to_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - # canned result for no placement policies created - no_pp = [PlacementAndNodes(placement=None, nodes=nodes)] - - model = nodes[0] - nodeset = lkp.node_nodeset(model) - - is_slice = bool(getattr(nodeset, 'accelerator_topology', None)) - - excl_job_placement = (excl_job_id is not None) and (not is_slice) - - if excl_job_placement and len(nodes) < 2: - return no_pp # don't create placement_policy for just one node - - if lkp.is_flex_node(model): - return no_pp # TODO(FLEX): Add support for workload policies - if lkp.node_is_tpu(model): - return no_pp - if not (nodeset.enable_placement and valid_placement_node(model)): - return no_pp - - max_count = calculate_chunk_size(nodeset, lkp) - - name_prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-{nodeset.nodeset_name}" - - if excl_job_placement: # simply chunk given nodes by max size of placement - return [ - PlacementAndNodes(placement=f"{name_prefix}-{excl_job_id}-{i}", nodes=chunk) - for i, chunk in enumerate(chunked(nodes, n=max_count)) - ] - - # split whole nodeset (not only nodes to resume) into chunks of max size of placement - # create placements (most likely already exists) placements for requested nodes - chunks = collections.defaultdict(list) # chunk_id -> nodes - invalid = [] - - for node in nodes: - try: - chunk = lkp.node_index(node) // max_count - chunks[chunk].append(node) - except: - invalid.append(node) - - placements = [ - # NOTE: use 0 instead of job_id for consistency with previous SlurmGCP behavior - PlacementAndNodes(placement=f"{name_prefix}-0-{c_id}", nodes=c_nodes) - for c_id, c_nodes in chunks.items() - ] - - if invalid: - placements.append(PlacementAndNodes(placement=None, nodes=invalid)) - log.error(f"Could not find placement for nodes with unexpected names: {to_hostlist(invalid)}") - - return placements - -def calculate_hosts_per_topo(accelerator_topology: str, machine_type: NSDict) -> int: - # Calculate total number of hosts per topology (Assumes format: '1x72') - try: - top_split = [int(x) for x in accelerator_topology.split("x")] - except Exception as e: - log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") - raise e - - if len(machine_type.accelerators) == 0: - gpus_per_machine = 0 - else: - gpus_per_machine = machine_type.accelerators[0].count - - if len(top_split) != 2: - log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") - elif top_split[0] <= 0 or top_split[1] <= 0: - log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") - elif gpus_per_machine <= 0: - log.error(f"The machine type has no accelerators. Cannot use accelerator topology {accelerator_topology}.") - elif top_split[1] % gpus_per_machine: - log.error(f"The GPU count {gpus_per_machine} per node is not a factor of the accelerator topology {accelerator_topology}") - - return (top_split[0] * top_split[1]) // gpus_per_machine - -def calculate_chunk_size(nodeset: NSDict, lkp: util.Lookup) -> int: - # Calculates the chunk size based on max distance value received or accelerator topology - # Assuming nodeset is not tpu - machine_type = lkp.template_info(nodeset.instance_template).machine_type - max_distance = nodeset.placement_max_distance - accelerator_topology = nodeset.accelerator_topology - - # Look for accelerator topology first - if accelerator_topology: - hosts_per_topo = calculate_hosts_per_topo(accelerator_topology, machine_type) - return hosts_per_topo - - if max_distance == 1: - return 22 - elif max_distance == 2: - if machine_type.family.startswith("a3"): - return 256 - else: - return 150 - elif max_distance == 3: - return 1500 - else: - return PLACEMENT_MAX_CNT - -def create_nodeset_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - placements = _allocate_nodes_to_placements(nodes, excl_job_id, lkp) - region = lkp.node_region(nodes[0]) - max_distance = lkp.node_nodeset(nodes[0]).get('placement_max_distance') - accelerator_topology = lkp.nodeset_accelerator_topology(lkp.node_nodeset_name(nodes[0])) - - if log.isEnabledFor(logging.DEBUG): - debug_p = {p.placement: to_hostlist(p.nodes) for p in placements} - log.debug( - f"creating {len(placements)} placement groups: \n{yaml.safe_dump(debug_p).rstrip()}" - ) - - requests = { - p.placement: create_placement_request(p.placement, region, max_distance, accelerator_topology) for p in placements if p.placement - } - if not requests: - return placements - # TODO: aggregate all requests for whole resume and execute them at once (don't limit to nodeset/job) - ops = dict( - zip(requests.keys(), map_with_futures(ensure_execute, requests.values())) - ) - - def classify_result(item): - op = item[1] - if not isinstance(op, Exception): - return "submitted" - if all(e.get("reason") == "alreadyExists" for e in op.error_details): # type: ignore - return "redundant" - return "failed" - - grouped_ops = dict(util.groupby_unsorted(list(ops.items()), classify_result)) - submitted, redundant, failed = ( - dict(grouped_ops.get(key, {})) for key in ("submitted", "redundant", "failed") - ) - if redundant: - log.warning( - "placement policies already exist: {}".format(",".join(redundant.keys())) - ) - if failed: - reqs = [f"{e}" for _, e in failed.values()] - log.fatal("failed to create placement policies: {}".format("; ".join(reqs))) - operations = {group: wait_for_operation(op) for group, op in submitted.items()} - for group, op in operations.items(): - if "error" in op: - msg = "; ".join( - f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" - for err in op["error"]["errors"] - ) - log.error( - f"placement group failed to create: '{group}' ({op['name']}): {msg}" - ) - - log.info( - f"created {len(operations)} placement groups ({to_hostlist(operations.keys())})" - ) - return placements - - -def valid_placement_node(node: str) -> bool: - invalid_types = frozenset(["e2", "t2d", "n1", "t2a", "m1", "m2", "m3"]) - mt = lookup().node_template_info(node).machineType - if mt.split("-")[0] in invalid_types: - log.warn(f"Unsupported machine type for placement policy: {mt}.") - log.warn( - f"Please do not use any the following machine types with placement policy: ({','.join(invalid_types)})" - ) - return False - return True - - -def main(nodelist: str) -> None: - """main called when run as script""" - log.debug(f"ResumeProgram {nodelist}") - # Filter out nodes not in config.yaml - other_nodes, nodes = separate( - lookup().is_power_managed_node, util.to_hostnames(nodelist) - ) - if other_nodes: - log.error( - f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" - ) - - if not nodes: - log.info("No nodes to resume") - return - resume_data = get_resume_file_data() - log.info(f"resume {util.to_hostlist(nodes)}") - resume_nodes(nodes, resume_data) - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("nodelist", help="list of nodes to resume") - args = util.init_log_and_parse(parser) - main(args.nodelist) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh deleted file mode 100644 index 023d246f01..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PYTHON_SCRIPT="${SCRIPT_DIR}/resume.py" - -# Capture all arguments passed by Slurm (the nodelist). -ALL_ARGS=("$@") - -# This array will hold extra argument for resume.py, like the resume data file. -UNIQUE_RESUME_FILE="" - -# Handle SLURM_RESUME_FILE if provided -if [ -n "${SLURM_RESUME_FILE-}" ] && [ -f "$SLURM_RESUME_FILE" ]; then - SAFE_DIR="/tmp/slurm_resume_data" - mkdir -p "$SAFE_DIR" - - UNIQUE_RESUME_FILE="${SAFE_DIR}/resumedata.$$.json" - cp "$SLURM_RESUME_FILE" "$UNIQUE_RESUME_FILE" -fi - -SLURM_RESUME_FILE="${UNIQUE_RESUME_FILE}" -setsid "${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & - -exit 0 diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py deleted file mode 100644 index 846524adf2..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py +++ /dev/null @@ -1,660 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import logging -import os -import shutil -import subprocess -import stat -import time -import yaml -from pathlib import Path -import functools - -import util -from util import ( - lookup, - dirs, - slurmdirs, - run, - install_custom_scripts, -) -import conf -import slurmsync - -from setup_network_storage import ( - setup_network_storage, - setup_nfs_exports, -) - - -log = logging.getLogger() - - -MOTD_HEADER = """ - SSSSSSS - SSSSSSSSS - SSSSSSSSS - SSSSSSSSS - SSSS SSSSSSS SSSS - SSSSSS SSSSSS - SSSSSS SSSSSSS SSSSSS - SSSS SSSSSSSSS SSSS - SSS SSSSSSSSS SSS - SSSSS SSSS SSSSSSSSS SSSS SSSSS - SSS SSSSSS SSSSSSSSS SSSSSS SSS - SSSSSS SSSSSSS SSSSSS - SSS SSSSSS SSSSSS SSS - SSSSS SSSS SSSSSSS SSSS SSSSS - S SSS SSSSSSSSS SSS S - SSS SSSS SSSSSSSSS SSSS SSS - S SSS SSSSSS SSSSSSSSS SSSSSS SSS S - SSSSS SSSSSS SSSSSSSSS SSSSSS SSSSS - S SSSSS SSSS SSSSSSS SSSS SSSSS S - S SSS SSS SSS SSS S - S S S S - SSS - SSS - SSS - SSS - SSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSS SSSSSSSSSSSSSSSSSSSS -SSSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSSS SSSSSSSSSSSSSSSSSSSSSS -SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSSS SSS SSSSSSSSSSSSSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSS SSS SSSSSSSSSSSSS SSSS SSSS SSSS SSSS - -""" -_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" - -def start_motd(): - """advise in motd that slurm is currently configuring""" - wall_msg = "*** Slurm is currently being configured in the background. ***" - motd_msg = MOTD_HEADER + wall_msg + "\n\n" - Path("/etc/motd").write_text(motd_msg) - util.run(f"wall -n '{wall_msg}'", timeout=30) - - -def end_motd(broadcast=True): - """modify motd to signal that setup is complete""" - Path("/etc/motd").write_text(MOTD_HEADER) - - if not broadcast: - return - - run( - "wall -n '*** Slurm {} setup complete ***'".format(lookup().instance_role), - timeout=30, - ) - if not lookup().is_controller: - run( - """wall -n ' -/home on the controller was mounted over the existing /home. -Log back in to ensure your home directory is correct. -'""", - timeout=30, - ) - - -def failed_motd(): - """modify motd to signal that setup is failed""" - wall_msg = f"*** Slurm setup failed! Please view log: {util.get_log_path()} ***" - motd_msg = MOTD_HEADER + wall_msg + "\n\n" - Path("/etc/motd").write_text(motd_msg) - util.run(f"wall -n '{wall_msg}'", timeout=30) - - -def _startup_script_timeout(lkp: util.Lookup) -> int: - if lkp.is_controller: - return lkp.cfg.get("controller_startup_scripts_timeout", 300) - elif lkp.instance_role == "compute": - return lkp.cfg.get("compute_startup_scripts_timeout", 300) - elif lkp.is_login_node: - return lkp.cfg.login_groups[util.instance_login_group()].get("startup_scripts_timeout", 300) - return 300 - - -def run_custom_scripts(): - """run custom scripts based on instance_role""" - custom_dir = dirs.custom_scripts - if lookup().is_controller: - # controller has all scripts, but only runs controller.d - custom_dirs = [custom_dir / "controller.d"] - elif lookup().instance_role == "compute": - # compute setup with nodeset.d - custom_dirs = [custom_dir / "nodeset.d"] - elif lookup().is_login_node: - # login setup with only login.d - custom_dirs = [custom_dir / "login.d"] - else: - # Unknown role: run nothing - custom_dirs = [] - - timeout = _startup_script_timeout(lookup()) - - custom_scripts = [ - p - for d in custom_dirs - for p in d.rglob("*") - if p.is_file() and not p.name.endswith(".disabled") - ] - print_scripts = ",".join(str(s.relative_to(custom_dir)) for s in custom_scripts) - log.debug(f"custom scripts to run: {custom_dir}/({print_scripts})") - - try: - for script in custom_scripts: - log.info(f"running script {script.name} with timeout={timeout}") - result = run(str(script), timeout=timeout, check=False, shell=True) - runlog = ( - f"{script.name} returncode={result.returncode}\n" - f"stdout={result.stdout}stderr={result.stderr}" - ) - log.info(runlog) - result.check_returncode() - except OSError as e: - log.error(f"script {script} is not executable") - raise e - except subprocess.TimeoutExpired as e: - log.error(f"script {script} did not complete within timeout={timeout}") - raise e - except Exception as e: - log.exception(f"script {script} encountered an exception") - raise e - -def mount_save_state_disk(): - disk_name = f"/dev/disk/by-id/google-{lookup().cfg.controller_state_disk.device_name}" - mount_point = util.slurmdirs.state - fs_type = "ext4" - - rdevice = util.run(f"realpath {disk_name}").stdout.strip() - file_output = util.run(f"file -s {rdevice}").stdout.strip() - if "filesystem" not in file_output: - util.run(f"mkfs -t {fs_type} -q {rdevice}") - - fstab_entry = f"{disk_name} {mount_point} {fs_type}" - with open("/etc/fstab", "r") as f: - fstab = f.readlines() - if fstab_entry not in fstab: - with open("/etc/fstab", "a") as f: - f.write(f"{fstab_entry} defaults 0 0\n") - - util.run(f"systemctl daemon-reload") - - os.makedirs(mount_point, exist_ok=True) - util.run(f"mount {mount_point}") - - util.chown_slurm(mount_point) - - -def setup_jwt_key(): - jwt_key = Path(slurmdirs.state / "jwt_hs256.key") - - if jwt_key.exists(): - log.info("JWT key already exists. Skipping key generation.") - else: - run("dd if=/dev/urandom bs=32 count=1 > " + str(jwt_key), shell=True) - - util.chown_slurm(jwt_key, mode=0o400) - - -def _generate_key(p: Path) -> None: - run(f"dd if=/dev/random of={p} bs=1024 count=1") - - -def setup_key(lkp: util.Lookup) -> None: - file_name = "munge.key" - dir = dirs.munge - - if lkp.cfg.enable_slurm_auth: - file_name = "slurm.key" - dir = slurmdirs.etc - - dst = Path(dir / file_name) - - if lkp.cfg.controller_state_disk.device_name: - # Copy key from persistent state disk - persist = slurmdirs.state / file_name - if not persist.exists(): - _generate_key(persist) - - shutil.copyfile(persist, dst) - if lkp.cfg.enable_slurm_auth: - util.chown_slurm(dst, mode=0o400) - util.chown_slurm(persist, mode=0o400) - else: - shutil.chown(dst, user="munge", group="munge") - os.chmod(dst, stat.S_IRUSR) - else: - if dst.exists(): - log.info("key already exists. Skipping key generation.") - else: - _generate_key(dst) - if lkp.cfg.enable_slurm_auth: - util.chown_slurm(dst, mode=0o400) - else: - shutil.chown(dst, user="munge", group="munge") - os.chmod(dst, stat.S_IRUSR) - - if lkp.cfg.enable_slurm_auth: - # Put key into shared volume for distribution - distributed = util.slurmdirs.key_distribution / file_name - shutil.copyfile(dst, distributed) - util.chown_slurm(distributed, mode=0o400) - # Munge is distributed from /etc/munge. - else: - run("systemctl restart munge", timeout=30) - - -def setup_nss_slurm(): - """install and configure nss_slurm""" - # setup nss_slurm - util.mkdirp(Path("/var/spool/slurmd")) - run( - "ln -s {}/lib/libnss_slurm.so.2 /usr/lib64/libnss_slurm.so.2".format( - slurmdirs.prefix - ), - check=False, - ) - run(r"sed -i 's/\(^\(passwd\|group\):\s\+\)/\1slurm /g' /etc/nsswitch.conf") - - -def setup_sudoers(): - content = """ -# Allow SlurmUser to manage the slurm daemons -slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmd.service -slurm ALL= NOPASSWD: /usr/bin/systemctl restart sackd.service -slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmctld.service -""" - sudoers_file = Path("/etc/sudoers.d/slurm") - sudoers_file.write_text(content) - sudoers_file.chmod(0o0440) - - -def setup_maintenance_script(): - perform_maintenance = """#!/bin/bash - -#SBATCH --priority=low -#SBATCH --time=180 - -VM_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") -ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" | cut -d '/' -f 4) - -gcloud compute instances perform-maintenance $VM_NAME \ - --zone=$ZONE -""" - - - with open(_MAINTENANCE_SBATCH_SCRIPT_PATH, "w") as f: - f.write(perform_maintenance) - - util.chown_slurm(_MAINTENANCE_SBATCH_SCRIPT_PATH, mode=0o755) - - -def update_system_config(file, content): - """Add system defaults options for service files""" - sysconfig = Path("/etc/sysconfig") - default = Path("/etc/default") - - if sysconfig.exists(): - conf_dir = sysconfig - elif default.exists(): - conf_dir = default - else: - raise Exception("Cannot determine system configuration directory.") - - slurmd_file = Path(conf_dir, file) - slurmd_file.write_text(content) - -def _symlink_mysql_datadir(lkp: util.Lookup) -> None: - """ Symlink /var/lib/mysql to controller state disk if needed. """ - if not lkp.cfg.controller_state_disk.device_name: - return - - datadir = Path("/var/lib/mysql") - dst = slurmdirs.state / "mysql" - - if dst.exists(): - run(f"rm -rf {datadir}") - else: - shutil.move(datadir, dst) - - datadir.symlink_to(dst, target_is_directory=True) - shutil.chown(datadir, user="mysql", group="mysql") - run(f"chown -R mysql:mysql {dst}") - -def configure_mysql(lkp: util.Lookup) -> None: - cnfdir = Path("/etc/my.cnf.d") - if not cnfdir.exists(): - cnfdir = Path("/etc/mysql/conf.d") - if not (cnfdir / "mysql_slurm.cnf").exists(): - (cnfdir / "mysql_slurm.cnf").write_text( - """ -[mysqld] -bind-address=127.0.0.1 -innodb_buffer_pool_size=1024M -innodb_log_file_size=64M -innodb_lock_wait_timeout=900 -""" - ) - - run("systemctl stop mariadb", timeout=30) - _symlink_mysql_datadir(lkp) - - run("systemctl enable mariadb", timeout=30) - run("systemctl restart mariadb", timeout=30) - - db_name = "slurm_acct_db" - - - cmd = "mysql -u root -e" - for host in ("localhost", lkp.control_host): - run(f"""{cmd} "drop user if exists 'slurm'@'{host}'";""", timeout=30) - run(f"""{cmd} "create user 'slurm'@'{host}'";""", timeout=30) - run(f"""{cmd} "grant all on {db_name}.* TO 'slurm'@'{host}'";""", timeout=30) - - -def configure_dirs(): - for p in dirs.values(): - util.mkdirp(p) - - for p in (dirs.slurm, dirs.scripts, dirs.custom_scripts): - util.chown_slurm(p) - - for p in slurmdirs.values(): - util.mkdirp(p) - util.chown_slurm(p) - - for sl, tgt in ( # create symlinks - (Path("/etc/slurm"), slurmdirs.etc), - (dirs.scripts / "etc", slurmdirs.etc), - (dirs.scripts / "log", dirs.log), - ): - if sl.exists() and sl.is_symlink(): - sl.unlink() - sl.symlink_to(tgt) - - # copy auxiliary scripts - for dst_folder, src_file in ((lookup().cfg.slurm_bin_dir, - Path("sort_nodes.py")), - (dirs.custom_scripts / "task_prolog.d", - Path("tools/task-prolog")), - (dirs.custom_scripts / "task_epilog.d", - Path("tools/task-epilog"))): - dst = Path(dst_folder) / src_file.name - util.mkdirp(dst.parent) - shutil.copyfile(util.scripts_dir / src_file, dst) - os.chmod(dst, 0o755) - - -def self_report_controller_address(lkp: util.Lookup) -> None: - if not lkp.cfg.controller_network_attachment: - return # only self report address if network attachment is used - data = { "slurm_control_addr": lkp.cfg.slurm_control_addr } - bucket, prefix = util._get_bucket_and_common_prefix() - blob = util.storage_client().bucket(bucket).blob(f"{prefix}/controller_addr.yaml") - with blob.open('w') as f: - f.write(yaml.dump(data)) - -def setup_controller(): - """Run controller setup""" - log.info("Setting up controller") - lkp = util.lookup() - util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) - install_custom_scripts() - conf.gen_controller_configs(lkp) - - if lkp.cfg.controller_state_disk.device_name != None: - mount_save_state_disk() - - setup_jwt_key() - setup_key(lkp) - - setup_sudoers() - setup_network_storage() - - run_custom_scripts() - - if not lkp.cfg.cloudsql_secret: - configure_mysql(lkp) - - run("systemctl enable slurmdbd", timeout=30) - run("systemctl restart slurmdbd", timeout=30) - - # Wait for slurmdbd to come up - time.sleep(5) - - sacctmgr = f"{slurmdirs.prefix}/bin/sacctmgr -i" - result = run( - f"{sacctmgr} add cluster {lkp.cfg.slurm_cluster_name}", timeout=30, check=False - ) - if "already exists" in result.stdout: - log.info(result.stdout) - elif result.returncode > 1: - result.check_returncode() # will raise error - - run("systemctl enable slurmctld", timeout=30) - run("systemctl restart slurmctld", timeout=30) - - run("systemctl enable slurmrestd", timeout=30) - run("systemctl restart slurmrestd", timeout=30) - - # Export at the end to signal that everything is up - run("systemctl enable nfs-server", timeout=30) - run("systemctl start nfs-server", timeout=30) - - setup_nfs_exports() - run("systemctl enable --now slurmcmd.timer", timeout=30) - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status slurmdbd", timeout=30) - run("systemctl status slurmctld", timeout=30) - run("systemctl status slurmrestd", timeout=30) - - try: - slurmsync.sync_instances() - except Exception: - log.exception("Failed to sync instances, will try next time.") - - run("systemctl enable slurm_load_bq.timer", timeout=30) - run("systemctl start slurm_load_bq.timer", timeout=30) - run("systemctl status slurm_load_bq.timer", timeout=30) - - # Add script to perform maintenance - setup_maintenance_script() - - self_report_controller_address(lkp) - - log.info("Done setting up controller") - pass - - -def setup_login(): - """run login node setup""" - log.info("Setting up login") - - lkp = lookup() - slurmctld_host = f"{lkp.control_host}" - if lkp.control_addr: - slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" - sackd_options = [ - f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', - ] - sysconf = f"""SACKD_OPTIONS='{" ".join(sackd_options)}'""" - update_system_config("sackd", sysconf) - install_custom_scripts() - - setup_network_storage() - setup_sudoers() - if not lkp.cfg.enable_slurm_auth: - run("systemctl restart munge", timeout=30) - run("systemctl enable sackd", timeout=30) - run("systemctl restart sackd", timeout=30) - run("systemctl enable --now slurmcmd.timer", timeout=30) - - run_custom_scripts() - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status sackd", timeout=30) - - log.info("Done setting up login") - - -def setup_compute(): - """run compute node setup""" - log.info("Setting up compute") - - lkp = lookup() - util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) - slurmctld_host = f"{lkp.control_host}" - if lkp.control_addr: - slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" - slurmd_options = [ - f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', - ] - - try: - slurmd_feature = util.instance_metadata("attributes/slurmd_feature", silent=True) - except util.MetadataNotFoundError: - slurmd_feature = None - - if slurmd_feature is not None: - slurmd_options.append(f'--conf="Feature={slurmd_feature}"') - slurmd_options.append("-Z") - - sysconf = f"""SLURMD_OPTIONS='{" ".join(slurmd_options)}'""" - update_system_config("slurmd", sysconf) - install_custom_scripts() - - setup_nss_slurm() - setup_network_storage() - - has_gpu = run("lspci | grep --ignore-case 'NVIDIA' | wc -l", shell=True).returncode - if has_gpu: - run("nvidia-smi") - - run_custom_scripts() - - setup_sudoers() - if not lkp.cfg.enable_slurm_auth: - run("systemctl restart munge", timeout=30) - run("systemctl enable slurmd", timeout=30) - run("systemctl restart slurmd", timeout=30) - run("systemctl enable --now slurmcmd.timer", timeout=30) - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status slurmd", timeout=30) - - log.info("Done setting up compute") - -def setup_cloud_ops() -> None: - """Add health checks, deployment info, and updated setup path to cloud ops config.""" - cloudOpsStatus = run( - "systemctl is-active --quiet google-cloud-ops-agent.service", check=False - ).returncode - - if cloudOpsStatus != 0: - return - - with open("/etc/google-cloud-ops-agent/config.yaml", "r") as f: - file = yaml.safe_load(f) - - # Update setup receiver path - file["logging"]["receivers"]["setup"]["include_paths"] = ["/var/log/slurm/setup.log"] - - cluster_info = { - 'type':'modify_fields', - 'fields': { - 'labels."cluster_name"':{ - 'static_value':f"{lookup().cfg.slurm_cluster_name}" - }, - 'labels."hostname"':{ - 'static_value': f"{lookup().hostname}" - } - } - } - - file["logging"]["processors"]["add_cluster_info"] = cluster_info - file["logging"]["service"]["pipelines"]["slurmlog_pipeline"]["processors"].append("add_cluster_info") - file["logging"]["service"]["pipelines"]["slurmlog2_pipeline"]["processors"].append("add_cluster_info") - - with open("/etc/google-cloud-ops-agent/config.yaml", "w") as f: - yaml.safe_dump(file, f, sort_keys=False) - - retries = 2 - for _ in range(retries): - try: - run("systemctl restart google-cloud-ops-agent.service", timeout=120) - break - except subprocess.TimeoutExpired: - log.error("google-cloud-ops-agent.service did not restart within 120s.") - result=run("cat /var/log/google-cloud-ops-agent/subagents/logging-module.log", timeout=120, shell=True) - if result.stdout: - log.error(f"Logs for google-cloud-ops-agent (logging-module.log file):\n{result.stdout}") - raise - - -def main(): - start_motd() - - log.info("Starting setup, fetching config") - sleep_seconds = 5 - while True: - try: - _, cfg = util.fetch_config() - util.update_config(cfg) - break - except util.DeffetiveStoredConfigError as e: - log.warning(f"config is not ready yet: {e}, sleeping for {sleep_seconds}s") - except Exception as e: - log.exception(f"unexpected error while fetching config, sleeping for {sleep_seconds}s") - time.sleep(sleep_seconds) - log.info("Config fetched") - setup_cloud_ops() - configure_dirs() - # call the setup function for the instance type - { - "controller": setup_controller, - "compute": setup_compute, - "login": setup_login, - }.get( - lookup().instance_role, - lambda: log.fatal(f"Unknown node role: {lookup().instance_role}"))() - - end_motd() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--slurmd-feature", dest="slurmd_feature", help="Unused, to be removed.") - _ = util.init_log_and_parse(parser) - - try: - main() - except Exception: - log.exception("Aborting setup...") - failed_motd() diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py deleted file mode 100644 index 095f42e758..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py +++ /dev/null @@ -1,327 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List - -import os -import sys -import stat -import time -import logging -import uuid - -import shutil -from pathlib import Path -from concurrent.futures import as_completed -from addict import Dict as NSDict # type: ignore - -import util -from util import NSMount, lookup, run, dirs, separate -from more_executors import Executors, ExceptionRetryPolicy - - -log = logging.getLogger() - -def mounts_by_local(mounts: list[NSMount]) -> dict[str, NSMount]: - """convert list of mounts to dict of mounts, local_mount as key""" - return {str(m.local_mount.resolve()): m for m in mounts} - - -def _get_default_mounts(lkp: util.Lookup) -> list[NSMount]: - if lkp.cfg.disable_default_mounts: - return [] - return [ - NSMount( - server_ip=lkp.controller_mount_server_ip(), - remote_mount=path, - local_mount=path, - fs_type="nfs", - mount_options="defaults,hard,intr", - ) - for path in ( - dirs.home, - dirs.apps, - ) - ] - -def get_slurm_bucket_mount() -> NSMount: - bucket, path = util._get_bucket_and_common_prefix() - return NSMount( - fs_type="gcsfuse", - server_ip="", - remote_mount=Path(bucket), - local_mount=dirs.slurm_bucket_mount, - mount_options=f"defaults,_netdev,implicit_dirs,only_dir={path}", - ) - -def resolve_network_storage() -> List[NSMount]: - """Combine appropriate network_storage fields to a single list""" - lkp = lookup() - - # create dict of mounts, local_mount: mount_info - mounts = mounts_by_local(_get_default_mounts(lkp)) - - if lkp.is_controller and util.should_mount_slurm_bucket(): - mounts.update(mounts_by_local([get_slurm_bucket_mount()])) - - # On non-controller instances, entries in network_storage could overwrite - # default exports from the controller. Be careful, of course - common = [lkp.normalize_ns_mount(m) for m in lkp.cfg.network_storage] - mounts.update(mounts_by_local(common)) - - if lkp.is_login_node: - login_group = lkp.cfg.login_groups[util.instance_login_group()] - login_ns = [lkp.normalize_ns_mount(m) for m in login_group.network_storage] - mounts.update(mounts_by_local(login_ns)) - - if lkp.instance_role == "compute": - try: - nodeset = lkp.node_nodeset() - except Exception: - pass # external nodename, skip lookup - else: - nodeset_ns = [lkp.normalize_ns_mount(m) for m in nodeset.network_storage] - mounts.update(mounts_by_local(nodeset_ns)) - - return list(mounts.values()) - - -def is_controller_mount(mount) -> bool: - # NOTE: Valid Lustre server_ip can take the form of '@tcp' - server_ip = mount.server_ip.split("@")[0] - mount_addr = util.host_lookup(server_ip) - return mount_addr == lookup().control_host_addr - -def setup_network_storage(): - """prepare network fs mounts and add them to fstab""" - log.info("Set up network storage") - - all_mounts = resolve_network_storage() - if lookup().is_controller: - mounts, _ = separate(is_controller_mount, all_mounts) - else: - mounts = all_mounts - - # Determine fstab entries and write them out - fstab_entries = [] - for mount in mounts: - local_mount = mount.local_mount - fs_type = mount.fs_type - server_ip = mount.server_ip or "" - src = mount.remote_mount if fs_type == "gcsfuse" else f"{server_ip}:{mount.remote_mount}" - - log.info(f"Setting up mount ({fs_type}) {src} to {local_mount}") - util.mkdirp(local_mount) - - mount_options = mount.mount_options.split(",") if mount.mount_options else [] - if "_netdev" not in mount_options: - mount_options += ["_netdev"] - options_line = ",".join(mount_options) - - - fstab_entries.append(f"{src} {local_mount} {fs_type} {options_line} 0 0") - - fstab = Path("/etc/fstab") - if not Path(fstab.with_suffix(".bak")).is_file(): - shutil.copy2(fstab, fstab.with_suffix(".bak")) - shutil.copy2(fstab.with_suffix(".bak"), fstab) - with open(fstab, "a") as f: - f.write("\n") - for entry in fstab_entries: - f.write(entry) - f.write("\n") - - mount_fstab(mounts, log) - if lookup().cfg.enable_slurm_auth: - slurm_key_mount_handler() - else: - munge_mount_handler() - - -def mount_fstab(mounts: list[NSMount], log): - """Wait on each mount, then make sure all fstab is mounted""" - def mount_path(path: Path): - log.info(f"Waiting for '{path}' to be mounted...") - try: - run(f"mount {path}", timeout=120) - except Exception as e: - exc_type, _, _ = sys.exc_info() - log.error(f"mount of path '{path}' failed: {exc_type}: {e}") - raise e - log.info(f"Mount point '{path}' was mounted.") - - MAX_MOUNT_TIMEOUT = 60 * 5 - future_list = [] - retry_policy = ExceptionRetryPolicy( - max_attempts=120, exponent=1.6, sleep=1.0, max_sleep=16.0 - ) - with Executors.thread_pool().with_timeout(MAX_MOUNT_TIMEOUT).with_retry( - retry_policy=retry_policy - ) as exe: - for m in mounts: - future = exe.submit(mount_path, m.local_mount) - future_list.append(future) - - # Iterate over futures, checking for exceptions - for future in as_completed(future_list): - try: - future.result() - except Exception as e: - raise e - - -def munge_mount_handler(): - if lookup().is_controller: - return - mnt = lookup().munge_mount - - log.info(f"Mounting munge share to: {mnt.local_mount}") - mnt.local_mount.mkdir() - if mnt.fs_type == "gcsfuse": - cmd = [ - "gcsfuse", - f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, - mnt.server_ip, - str(mnt.local_mount), - ] - else: - cmd = [ - "mount", - f"--types={mnt.fs_type}", - f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, - f"{mnt.server_ip}:{mnt.remote_mount}", - str(mnt.local_mount), - ] - # wait max 240s for munge mount - timeout = 240 - for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): - try: - run(cmd, timeout=timeout) - break - except Exception as e: - log.error( - f"munge mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" - ) - time.sleep(wait) - err = e - continue - else: - raise err - - munge_key = Path(dirs.munge / "munge.key") - log.info(f"Copy munge.key from: {mnt.local_mount}") - shutil.copy2(Path(mnt.local_mount / "munge.key"), munge_key) - - log.info("Restrict permissions of munge.key") - shutil.chown(munge_key, user="munge", group="munge") - os.chmod(munge_key, stat.S_IRUSR) - - log.info(f"Unmount {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - run(f"fusermount -u {mnt.local_mount}", timeout=120) - else: - run(f"umount {mnt.local_mount}", timeout=120) - shutil.rmtree(mnt.local_mount) - -def slurm_key_mount_handler(): - if lookup().is_controller: - return - mnt = lookup().slurm_key_mount - - log.info(f"Mounting slurm_key share to: {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - cmd = [ - "gcsfuse", - f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, - mnt.server_ip, - str(mnt.local_mount), - ] - else: - cmd = [ - "mount", - f"--types={mnt.fs_type}", - f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, - f"{mnt.server_ip}:{mnt.remote_mount}", - str(mnt.local_mount), - ] - timeout = 120 # wait max 120s to mount - for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): - try: - run(cmd, timeout=timeout) - break - except Exception as e: - log.error( - f"slurm key mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" - ) - time.sleep(wait) - err = e - continue - else: - raise err - - file_name = "slurm.key" - dst = Path(util.slurmdirs.etc / file_name) - log.info(f"Copy slurm.key from: {mnt.local_mount}") - shutil.copy2(mnt.local_mount / file_name, dst) - - log.info("Restrict permissions of slurm.key") - util.chown_slurm(dst, mode=0o400) - - log.info(f"Unmount {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - run(f"fusermount -u {mnt.local_mount}", timeout=120) - else: - run(f"umount {mnt.local_mount}", timeout=120) - shutil.rmtree(mnt.local_mount) - - -def setup_nfs_exports(): - """nfs export all needed directories""" - lkp = util.lookup() - assert lkp.is_controller - - # The controller only needs to set up exports for cluster-internal mounts - exported_mounts = [m for m in resolve_network_storage() if is_controller_mount(m)] - - # key by remote mount path since that is what needs exporting - to_export = {m.remote_mount: "*(rw,no_subtree_check,no_root_squash)" for m in exported_mounts} - - key_mount = lkp.slurm_key_mount if lkp.cfg.enable_slurm_auth else lkp.munge_mount - if is_controller_mount(key_mount): - # Export key mount as read-only - to_export[key_mount.remote_mount] = "*(ro,no_subtree_check,no_root_squash)" - - if util.should_mount_slurm_bucket(): - mnt = get_slurm_bucket_mount() - # FSID is required for virtual filesystem that is not based on a device - # Also export it as read-only - fsid=str(uuid.uuid4()) - to_export[mnt.local_mount] = f"*(ro,no_subtree_check,no_root_squash,fsid={fsid})" - - # export path if corresponding selector boolean is True - lines = [] - for path,options in to_export.items(): - util.mkdirp(Path(path)) - run(rf"sed -i '\#{path}#d' /etc/exports", timeout=30) - lines.append(f"{path} {options}") - - exportsd = Path("/etc/exports.d") - util.mkdirp(exportsd) - with (exportsd / "slurm.exports").open("w") as f: - f.write("\n") - f.write("\n".join(lines)) - run("exportfs -a", timeout=30) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py deleted file mode 100644 index 1bfdd5acce..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py +++ /dev/null @@ -1,679 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import fcntl -import json -import logging -import re -import sys -import shlex -from datetime import datetime, timedelta -from itertools import chain -from pathlib import Path -from dataclasses import dataclass -from typing import Dict, Tuple, List, Optional, Protocol, Any -from functools import lru_cache - -import util -from util import ( - batch_execute, - ensure_execute, - execute_with_futures, - FutureReservation, - install_custom_scripts, - run, - separate, - to_hostlist, - NodeState, - chunked, - dirs, -) -from util import lookup -from suspend import delete_instances -import tpu -import conf -import watch_delete_vm_op - -log = logging.getLogger() - -TOT_REQ_CNT = 1000 -_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" - -class NodeAction(Protocol): - def apply(self, nodes:List[str]) -> None: - ... - - def __hash__(self): - ... - -@dataclass(frozen=True) -class NodeActionPowerUp(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to resume ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=power_up") - -@dataclass(frozen=True) -class NodeActionIdle(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} nodes to idle ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=resume") - -@dataclass(frozen=True) -class NodeActionPowerDown(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to power down ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=power_down") - - -@dataclass(frozen=True) -class NodeActionPowerDownForce(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to power down ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=power_down_force") - - -@dataclass(frozen=True) -class NodeActionDelete(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to delete ({hostlist})") - delete_instances(nodes) - -@dataclass(frozen=True) -class NodeActionPrempt(): - def apply(self, nodes:List[str]) -> None: - NodeActionDown(reason="Preempted instance").apply(nodes) - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances restarted ({hostlist})") - start_instances(nodes) - -@dataclass(frozen=True) -class NodeActionUnchanged(): - def apply(self, nodes:List[str]) -> None: - pass - -@dataclass(frozen=True) -class NodeActionDown(): - reason: str - - def apply(self, nodes: List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} nodes set down ({hostlist}) with reason={self.reason}") - run(f"{lookup().scontrol} update nodename={hostlist} state=down reason={shlex.quote(self.reason)}") - -@dataclass(frozen=True) -class NodeActionUnknown(): - slurm_state: Optional[NodeState] - instance_state: Optional[str] - - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.error(f"{len(nodes)} nodes have unexpected {self.slurm_state} and instance state:{self.instance_state}, ({hostlist})") - -def start_instance_op(node: str) -> Any: - inst = lookup().instance(node) - assert inst - - return lookup().compute.instances().start( - project=lookup().project, - zone=inst.zone, - instance=inst.name, - ) - - -def start_instances(node_list): - log.info("{} instances to start ({})".format(len(node_list), ",".join(node_list))) - lkp = lookup() - # TODO: use code from resume.py to assign proper placement - normal, tpu_nodes = separate(lkp.node_is_tpu, node_list) - ops = {node: start_instance_op(node) for node in normal} - - done, failed = batch_execute(ops) - - tpu_start_data = [] - for ns, nodes in util.groupby_unsorted(tpu_nodes, lkp.node_nodeset_name): - tpuobj = tpu.TPU.make(ns, lkp) - for snodes in chunked(nodes, n=tpuobj.vmcount): - tpu_start_data.append({"tpu": tpuobj, "node": snodes}) - execute_with_futures(tpu.start_tpu, tpu_start_data) - - -def _find_dynamic_node_status() -> NodeAction: - # TODO: cover more cases: - # * delete dead dynamic nodes - # * delete orhpaned instances - return NodeActionUnchanged() # don't touch dynamic nodes - -def get_fr_action(fr: FutureReservation, state:Optional[NodeState]) -> Optional[NodeAction]: - now = util.now() - if state is None: - return None # handle like any other node - if fr.start_time < now < fr.end_time: - return None # handle like any other node - - if state.base == "DOWN": - return NodeActionUnchanged() - if fr.start_time >= now: - msg = f"Waiting for reservation:{fr.name} to start at {fr.start_time}" - else: - msg = f"Reservation:{fr.name} is after its end-time" - return NodeActionDown(reason=msg) - -def _find_tpu_node_action(nodename, state) -> NodeAction: - lkp = lookup() - tpuobj = tpu.TPU.make(lkp.node_nodeset_name(nodename), lkp) - inst = tpuobj.get_node(nodename) - # If we do not find the node but it is from a Tpu that has multiple vms look for the master node - if inst is None and tpuobj.vmcount > 1: - # Get the tpu slurm nodelist of the nodes in the same tpu group as nodename - nodelist = run( - f"{lkp.scontrol} show topo {nodename}" - + " | awk -F'=' '/Level=0/ { print $NF }'", - shell=True, - ).stdout - l_nodelist = util.to_hostnames(nodelist) - group_names = set(l_nodelist) - # get the list of all the existing tpus in the nodeset - tpus_list = set(tpuobj.list_node_names()) - # In the intersection there must be only one node that is the master - tpus_int = list(group_names.intersection(tpus_list)) - if len(tpus_int) > 1: - log.error( - f"More than one cloud tpu node for tpu group {nodelist}, there should be only one that should be {l_nodelist[0]}, but we have found {tpus_int}" - ) - return NodeActionUnknown(slurm_state=state, instance_state=None) - if len(tpus_int) == 1: - inst = tpuobj.get_node(tpus_int[0]) - # if len(tpus_int ==0) this case is not relevant as this would be the case always that a TPU group is not running - if inst is None: - if state.base == "DOWN" and "POWERED_DOWN" in state.flags: - return NodeActionIdle() - if "POWERING_DOWN" in state.flags: - return NodeActionIdle() - if "COMPLETING" in state.flags: - return NodeActionDown(reason="Unbacked instance") - if state.base != "DOWN" and not ( - set(("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN")) - & state.flags - ): - return NodeActionDown(reason="Unbacked instance") - if lkp.is_static_node(nodename): - return NodeActionPowerUp() - elif ( - state is not None - and "POWERED_DOWN" not in state.flags - and "POWERING_DOWN" not in state.flags - and inst.state == tpu.TPU.State.STOPPED - ): - if tpuobj.preemptible: - return NodeActionPrempt() - if state.base != "DOWN": - return NodeActionDown(reason="Instance terminated") - elif ( - state is None or "POWERED_DOWN" in state.flags - ) and inst.state == tpu.TPU.State.READY: - return NodeActionDelete() - elif state is None: - # if state is None here, the instance exists but it's not in Slurm - return NodeActionUnknown(slurm_state=state, instance_state=inst.status) - - return NodeActionUnchanged() - -def get_node_action(nodename: str) -> NodeAction: - """Determine node/instance status that requires action""" - lkp = lookup() - state = lkp.node_state(nodename) - - if lkp.node_is_gke(nodename): - return NodeActionUnchanged() - - if lkp.node_is_fr(nodename): - fr = lkp.future_reservation(lkp.node_nodeset(nodename)) - assert fr - if action := get_fr_action(fr, state): - return action - - if lkp.node_is_dyn(nodename): - return _find_dynamic_node_status() - - if lkp.node_is_tpu(nodename): - return _find_tpu_node_action(nodename, state) - - # split below is workaround for VMs whose hostname is FQDN - inst = lkp.instance(nodename.split(".")[0]) - power_flags = frozenset( - ("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN") - ) & (state.flags if state is not None else set()) - - if (state is None) and (inst is None): - # Should never happen - return NodeActionUnknown(None, None) - if inst is None: - assert state is not None # to keep type-checker happy - if "POWERING_UP" in state.flags: - return NodeActionUnchanged() - if state.base == "DOWN" and "POWERED_DOWN" in state.flags: - return NodeActionIdle() - if "POWERING_DOWN" in state.flags: - return NodeActionIdle() - if "COMPLETING" in state.flags: - return NodeActionDown(reason="Unbacked instance") - if state.base != "DOWN" and not power_flags: - return NodeActionDown(reason="Unbacked instance") - if state.base == "DOWN" and not power_flags: - return NodeActionPowerDown() - if "NOT_RESPONDING" in state.flags: - return NodeActionPowerDown() - if "POWERED_DOWN" in state.flags and lkp.is_static_node(nodename): - return NodeActionPowerUp() - elif ( - state is not None - and "POWERED_DOWN" not in state.flags - and "POWERING_DOWN" not in state.flags - and inst.status == "TERMINATED" - ): - if inst.scheduling.preemptible: - return NodeActionPrempt() - if state.base != "DOWN": - return NodeActionDown(reason="Instance terminated") - elif (state is None or "POWERED_DOWN" in state.flags) and inst.status == "RUNNING": - log.info("%s is potential orphan node", nodename) - threshold = timedelta(seconds=90) - age = util.now() - inst.creation_timestamp - log.info(f"{nodename} state: {state}, age: {age}") - if age < threshold: - log.info(f"{nodename} not marked as orphan, it started less than {threshold.seconds}s ago ({age.seconds}s)") - return NodeActionUnchanged() - return NodeActionDelete() - elif state is None: - # if state is None here, the instance exists but it's not in Slurm - return NodeActionUnknown(slurm_state=state, instance_state=inst.status) - elif lkp.is_flex_node(nodename) and "POWERING_UP" in state.flags: - threshold = timedelta(seconds=int(lkp.cfg.compute_startup_scripts_timeout) * 2) #extra buffer for unexpectedly long startup scripts - if util.now() - inst.creation_timestamp > threshold: - log.info(f"{nodename} was unable to join the cluster after {threshold.seconds}s, potential failure on VM startup. Powering down...") - return NodeActionPowerDownForce() - return NodeActionUnchanged() - - -def delete_resource_policies(links: list[str], lkp: util.Lookup) -> None: - requests = {} - for link in links: - name = util.trim_self_link(link) - region = util.parse_self_link(link).region - requests[name] = lkp.compute.resourcePolicies().delete(project=lkp.project, region=region, resourcePolicy=name) - - def swallow_err(_: str) -> None: - pass - - done, failed = batch_execute(requests, log_err=swallow_err) - if failed: - # Filter out resourceInUseByAnotherResource errors , they are expected to happen - def ignore_err(e) -> bool: - return "resourceInUseByAnotherResource" in str(e) - - failures = [f"{n}: {e}" for n, (_, e) in failed.items() if not ignore_err(e)] - if failures: - log.error(f"some placement groups failed to delete: {failures}") - log.info( - f"deleted {len(done)} of {len(links)} placement groups ({to_hostlist(done.keys())})" - ) - - - -@lru_cache -def _get_resource_policies_in_region(lkp: util.Lookup, region: str) -> list[Any]: - res = [] - act = lkp.compute.resourcePolicies() - op = act.list(project=lkp.project, region=region) - prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-" - while op is not None: - result = ensure_execute(op) - res.extend([p for p in result.get("items", []) if p.get("name", "").startswith(prefix)]) - op = act.list_next(op, result) - return res - - -@lru_cache -def _get_resource_policies(lkp: util.Lookup) -> list[Any]: - res = [] - for region in lkp.cluster_regions(): - res.extend(_get_resource_policies_in_region(lkp, region)) - return res - -def sync_placement_groups(): - """Delete placement policies that are for jobs that have completed/terminated""" - keep_states = frozenset( - [ - "RUNNING", - "CONFIGURING", - "STOPPED", - "SUSPENDED", - "COMPLETING", - "PENDING", - ] - ) - - lkp = lookup() - keep_jobs = { - str(job.id) - for job in lkp.get_jobs() - if job.job_state in keep_states - } - keep_jobs.add("0") # Job 0 is a placeholder for static node placement - - to_delete = [] - pg_regex = re.compile( - rf"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-(?P[^\s\-]+)-(?P\d+)-(?P\d+)" - ) - - for pg in _get_resource_policies(lkp): - name = pg["name"] - - if (mtch := pg_regex.match(name)) is None: - log.warning(f"Unexpected resource policy {name=}") - continue - if mtch.group("job_id") not in keep_jobs: - to_delete.append(pg["selfLink"]) - - if to_delete: - delete_resource_policies(to_delete, lkp) - - -def sync_instances(): - compute_instances = { - name for name, inst in lookup().instances().items() if inst.role == "compute" - } - slurm_nodes = set(lookup().slurm_nodes().keys()) - log.debug(f"reconciling {len(compute_instances)} GCP instances and {len(slurm_nodes)} Slurm nodes.") - - for action, nodes in util.groupby_unsorted(list(compute_instances | slurm_nodes), get_node_action): - action.apply(list(nodes)) - - -def reconfigure_slurm(): - update_msg = "*** slurm configuration was updated ***" - if lookup().cfg.hybrid: - # terraform handles generating the config.yaml, don't do it here - return - - upd, cfg_new = util.fetch_config() - if not upd: - log.debug("No changes in config detected.") - return - log.debug("Changes in config detected. Reconfiguring Slurm now.") - util.update_config(cfg_new) - - if lookup().is_controller: - conf.gen_controller_configs(lookup()) - log.info("Restarting slurmctld to make changes take effect.") - try: - # TODO: consider removing "restart" since "reconfigure" should restart slurmctld as well - run("sudo systemctl restart slurmctld.service", check=False) - util.scontrol_reconfigure(lookup()) - except Exception: - log.exception("failed to reconfigure slurmctld") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - elif lookup().instance_role_safe == "compute": - log.info("Restarting slurmd to make changes take effect.") - run("systemctl restart slurmd") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - elif lookup().is_login_node: - log.info("Restarting sackd to make changes take effect.") - run("systemctl restart sackd") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - - -def update_topology(lkp: util.Lookup) -> None: - if conf.topology_plugin(lkp) != conf.TOPOLOGY_PLUGIN_TREE: - return - updated, summary = conf.gen_topology_conf(lkp) - if updated: - log.info("Topology configuration updated. Reconfiguring Slurm.") - util.scontrol_reconfigure(lkp) - # Safe summary only after Slurm got reconfigured, so summary reflects Slurm POV - summary.dump(lkp) - - -def delete_reservation(lkp: util.Lookup, reservation_name: str) -> None: - util.run(f"{lkp.scontrol} delete reservation {reservation_name}") - - -def create_reservation(lkp: util.Lookup, reservation_name: str, node: str, start_time: datetime) -> None: - # Format time to be compatible with slurm reservation. - formatted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S') - - util.run(f"{lkp.scontrol} create reservation user=slurm starttime={formatted_start_time} duration=180 nodes={node} reservationname={reservation_name} flags=maint,ignore_jobs") - - -def get_slurm_reservation_maintenance(lkp: util.Lookup) -> Dict[str, datetime]: - res = util.run(f"{lkp.scontrol} show reservation --json") - all_reservations = json.loads(res.stdout) - reservation_map = {} - - for reservation in all_reservations['reservations']: - name = reservation.get('name') - nodes = reservation.get('node_list') - time_epoch = reservation.get('start_time', {}).get('number') - - if name is None or nodes is None or time_epoch is None: - continue - - if reservation.get('node_count') != 1: - continue - - if name != f"{nodes}_maintenance": - continue - - reservation_map[name] = datetime.fromtimestamp(time_epoch) - - return reservation_map - -@lru_cache -def get_upcoming_maintenance(lkp: util.Lookup) -> Dict[str, Tuple[str, datetime]]: - upc_maint_map = {} - - for node, inst in lkp.instances().items(): - if inst.resource_status.upcoming_maintenance: - upc_maint_map[node + "_maintenance"] = (node, inst.resource_status.upcoming_maintenance.window_start_time) - - return upc_maint_map - - -def sync_maintenance_reservation(lkp: util.Lookup) -> None: - upc_maint_map = get_upcoming_maintenance(lkp) # map reservation_name -> (node_name, time) - log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") - - curr_reservation_map = get_slurm_reservation_maintenance(lkp) # map reservation_name -> time - log.debug(f"curr-reservation-map: {curr_reservation_map}") - - del_reservation = set(curr_reservation_map.keys() - upc_maint_map.keys()) - create_reservation_map = {} - - for res_name, (node, start_time) in upc_maint_map.items(): - try: - enabled = lkp.node_nodeset(node).enable_maintenance_reservation - except Exception: - enabled = False - - if not enabled: - if res_name in curr_reservation_map: - del_reservation.add(res_name) - continue - - if res_name in curr_reservation_map: - diff = curr_reservation_map[res_name] - start_time - if abs(diff) <= timedelta(seconds=1): - continue - else: - del_reservation.add(res_name) - create_reservation_map[res_name] = (node, start_time) - else: - create_reservation_map[res_name] = (node, start_time) - - log.debug(f"del-reservation: {del_reservation}") - for res_name in del_reservation: - delete_reservation(lkp, res_name) - - log.debug(f"create-reservation-map: {create_reservation_map}") - for res_name, (node, start_time) in create_reservation_map.items(): - create_reservation(lkp, res_name, node, start_time) - - -def delete_maintenance_job(job_name: str) -> None: - util.run(f"scancel --name={job_name}") - - -def create_maintenance_job(job_name: str, node: str) -> None: - util.run(f"sbatch --job-name={job_name} --nodelist={node} {_MAINTENANCE_SBATCH_SCRIPT_PATH}") - - -def get_slurm_maintenance_job(lkp: util.Lookup) -> Dict[str, str]: - jobs = {} - - for job in lkp.get_jobs(): - if job.name is None or job.required_nodes is None or job.job_state is None: - continue - - if job.name != f"{job.required_nodes}_maintenance": - continue - - if job.job_state != "PENDING": - continue - - jobs[job.name] = job.required_nodes - - return jobs - - -def sync_opportunistic_maintenance(lkp: util.Lookup) -> None: - upc_maint_map = get_upcoming_maintenance(lkp) # map job_name -> (node_name, time) - log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") - - curr_jobs = get_slurm_maintenance_job(lkp) # map job_name -> node. - log.debug(f"curr-maintenance-job-map: {curr_jobs}") - - del_jobs = set(curr_jobs.keys() - upc_maint_map.keys()) - create_jobs = {} - - for job_name, (node, _) in upc_maint_map.items(): - try: - enabled = lkp.node_nodeset(node).enable_opportunistic_maintenance - except Exception: - enabled = False - - if not enabled: - if job_name in curr_jobs: - del_jobs.add(job_name) - continue - - if job_name not in curr_jobs: - create_jobs[job_name] = node - - log.debug(f"del-maintenance-job: {del_jobs}") - for job_name in del_jobs: - delete_maintenance_job(job_name) - - log.debug(f"create-maintenance-job: {create_jobs}") - for job_name, node in create_jobs.items(): - create_maintenance_job(job_name, node) - - - -def sync_flex_migs(lkp: util.Lookup) -> None: - pass - - -def process_messages(lkp: util.Lookup) -> None: - try: - watch_delete_vm_op.watch_vm_delete_ops(lkp) - except: - log.exception("failed during watching delete VM operations") - - -def main(): - lkp = lookup() - if util.should_mount_slurm_bucket() and not lkp.is_controller: - return - try: - reconfigure_slurm() - except Exception: - log.exception("failed to reconfigure slurm") - if lkp.is_controller: - try: - process_messages(lkp) - except: - log.exception("failed to process messages") - - try: - sync_instances() - except Exception: - log.exception("failed to sync instances") - - try: - sync_flex_migs(lkp) - except Exception: - log.exception("failed to sync DWS Flex MIGs") - - try: - sync_placement_groups() - except Exception: - log.exception("failed to sync placement groups") - - try: - update_topology(lkp) - except Exception: - log.exception("failed to update topology") - - try: - sync_maintenance_reservation(lkp) - except Exception: - log.exception("failed to sync slurm reservation for scheduled maintenance") - - try: - sync_opportunistic_maintenance(lkp) - except Exception: - log.exception("failed to sync opportunistic reservation for scheduled maintenance") - - - try: - # TODO: it performs 1 to 4 GCS list requests, - # use cached version, combine with `_list_config_blobs` - install_custom_scripts(check_hash=True) - except Exception: - log.exception("failed to sync custom scripts") - - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - _ = util.init_log_and_parse(parser) - - pid_file = (Path("/tmp") / Path(__file__).name).with_suffix(".pid") - with pid_file.open("w") as fp: - try: - fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) - main() - except BlockingIOError: - sys.exit(0) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py deleted file mode 100644 index ae36c54222..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This script sorts nodes based on their `physicalHost`. - -See https://cloud.google.com/compute/docs/instances/use-compact-placement-policies - -You can reduce latency in tightly coupled HPC workloads (including distributed ML training) -by deploying them to machines that are located close together. -For example, if you deploy your workload on a single physical rack, you can expect lower latency -than if your workload is spread across multiple racks. -Sending data across multiple rack requires sending data through additional network switches. - -Example usage: -``` my_sbatch.sh -#SBATCH --ntasks-per-node=8 -#SBATCH --nodes=64 - -export SLURM_HOSTFILE=$(sort_nodes.py) - -srun -l hostname | sort -``` -""" -import os -import subprocess -import uuid -from typing import List, Optional, Dict -from collections import OrderedDict - -def order(paths: List[List[str]]) -> List[str]: - """ - Orders the leaves of the tree in a way that minimizes the sum of distance in between - each pair of neighboring nodes in the resulting order. - The resulting order will always start from the first node in the input list. - The ordering is "stable" with respect to the input order of the leaves i.e. - given a choice between two nodes (identical in other ways) it will select "nodelist-smallest" one. - - Returns a list of nodenames, ordered as described above. - """ - if not paths: return [] - class Vert: - "Represents a vertex in a *network* tree." - def __init__(self, name: str, parent: Optional["Vert"]): - self.name = name - self.parent = parent - # Use `OrderedDict` to preserve insertion order - # TODO: once we move to Python 3.7+ use regular `dict` since it has the same guarantee - self.children: OrderedDict = OrderedDict() - - # build a tree, children are ordered by insertion order - root = Vert("", None) - for path in paths: - n = root - for v in path: - if v not in n.children: - n.children[v] = Vert(v, n) - n = n.children[v] - - # walk the tree in insertion order, gather leaves - result = [] - def gather_nodes(v: Vert) -> None: - if not v.children: # this is a Slurm node - result.append(v.name) - for u in v.children.values(): - gather_nodes(u) - gather_nodes(root) - return result - - -class Instance: - def __init__(self, name: str, zone: str, physical_host: Optional[str]): - self.name = name - self.zone = zone - self.physical_host = physical_host - - -def make_path(node_name: str, inst: Optional[Instance]) -> List[str]: - if not inst: # node with unknown instance (e.g. hybrid cluster) - return ["unknown", node_name] - zone = f"zone_{inst.zone}" - if not inst.physical_host: # node without physical host info (e.g. no placement policy) - return [zone, "unknown", node_name] - - assert inst.physical_host.startswith("/"), f"Unexpected physicalHost: {inst.physical_host}" - parts = inst.physical_host[1:].split("/") - if len(parts) >= 4: - return [*parts, node_name] - return [zone, *parts, node_name] - - -def to_hostnames(nodelist: str) -> List[str]: - cmd = ["scontrol", "show", "hostnames", nodelist] - out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout - return [n.decode("utf-8") for n in out.splitlines()] - - -def get_instances(node_names: List[str]) -> Dict[str, Optional[Instance]]: - fmt = ( - "--format=csv[no-heading,separator=','](zone,resourceStatus.physicalHost,name)" - ) - cmd = ["gcloud", "compute", "instances", "list", fmt] - - scp = os.path.commonprefix(node_names) - if scp: - cmd.append(f"--filter=name~'{scp}.*'") - out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout - d = {} - for line in out.splitlines(): - zone, physical_host, name = line.decode("utf-8").split(",") - d[name] = Instance(name, zone, physical_host) - return {n: d.get(n) for n in node_names} - - -def main(args) -> None: - nodelist = args.nodelist or os.getenv("SLURM_NODELIST") - if not nodelist: - raise ValueError("nodelist is not provided and SLURM_NODELIST is not set") - - if args.ntasks_per_node is None: - args.ntasks_per_node = int(os.getenv("SLURM_NTASKS_PER_NODE", "") or 1) - assert args.ntasks_per_node > 0 - - output = args.output or f"hosts.{uuid.uuid4()}" - - node_names = to_hostnames(nodelist) - instannces = get_instances(node_names) - paths = [make_path(n, instannces[n]) for n in node_names] - ordered = order(paths) - - with open(output, "w") as f: - for node in ordered: - for _ in range(args.ntasks_per_node): - f.write(node) - f.write("\n") - print(output) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument( - "--nodelist", - type=str, - help="Slurm 'hostlist expression' of nodes to sort, if not set the value of SLURM_NODELIST environment variable will be used", - ) - parser.add_argument( - "--ntasks-per-node", - type=int, - help="""Number of times to repeat each node in resulting sorted list. -If not set, the value of SLURM_NTASKS_PER_NODE environment variable will be used, -if neither is set, defaults to 1""", - ) - parser.add_argument( - "--output", type=str, help="Output file to write, defaults to 'hosts.'" - ) - args = parser.parse_args() - main(args) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py deleted file mode 100644 index ecef70f1cc..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Any -import argparse -import logging - -import util -from util import ( - log_api_request, - batch_execute, - to_hostlist, - separate, -) -from util import lookup -import tpu -import mig_flex -import watch_delete_vm_op - -log = logging.getLogger() - -TOT_REQ_CNT = 1000 - - -def truncate_iter(iterable, max_count): - end = "..." - _iter = iter(iterable) - for i, el in enumerate(_iter, start=1): - if i >= max_count: - yield end - break - yield el - - -def delete_instance_request(name: str) -> Any: - inst = lookup().instance(name) - assert inst - - request = lookup().compute.instances().delete( - project=lookup().project, - zone=inst.zone, - instance=name, - ) - log_api_request(request) - return request - - -def delete_instances(instances): - """delete instances individually""" - invalid, valid = separate(lambda inst: bool(lookup().instance(inst)), instances) - if len(invalid) > 0: - log.debug("instances do not exist: {}".format(",".join(invalid))) - if len(valid) == 0: - log.debug("No instances to delete") - return - - requests = {inst: delete_instance_request(inst) for inst in valid} - - log.info(f"to delete {len(valid)} instances ({to_hostlist(valid)})") - ops, failed = batch_execute(requests) - for node, (_, err) in failed.items(): - log.error(f"instance {node} failed to delete: {err}") - - log.info(f"deleting {len(ops)} instances {to_hostlist(ops.keys())}") - - topic = watch_delete_vm_op.watch_delete_vm_op_topic() - for node, op in ops.items(): - topic.publish(op, node) - - - - -def suspend_nodes(nodes: List[str]) -> None: - lkp = lookup() - other_nodes, tpu_nodes = util.separate(lkp.node_is_tpu, nodes) - bulk_nodes, flex_nodes = util.separate(lkp.is_flex_node, other_nodes) - - mig_flex.suspend_flex_nodes(flex_nodes, lkp) - delete_instances(bulk_nodes) - tpu.delete_tpu_instances(tpu_nodes) - - -def main(nodelist): - """main called when run as script""" - log.debug(f"SuspendProgram {nodelist}") - - # Filter out nodes not in config.yaml - other_nodes, pm_nodes = separate( - lookup().is_power_managed_node, util.to_hostnames(nodelist) - ) - if other_nodes: - log.debug( - f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" - ) - if pm_nodes: - log.debug(f"Suspending nodes '{to_hostlist(pm_nodes)}' from '{nodelist}'") - else: - log.debug("No cloud nodes to suspend") - return - - log.info(f"suspend {nodelist}") - suspend_nodes(pm_nodes) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument("nodelist", help="list of nodes to suspend") - args = util.init_log_and_parse(parser) - - main(args.nodelist) diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh deleted file mode 100644 index 9079e4e4b0..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PYTHON_SCRIPT="${SCRIPT_DIR}/suspend.py" - -# Capture all arguments passed by Slurm (the nodelist). -ALL_ARGS=("$@") - -"${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & -disown - -exit 0 diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py deleted file mode 100644 index 0ce7fb5ec4..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Any -import sys -from dataclasses import dataclass, field -from datetime import datetime - -SCRIPTS_DIR = "community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts" -if SCRIPTS_DIR not in sys.path: - sys.path.append(SCRIPTS_DIR) # TODO: make this more robust - -import util - - -SOME_TS = datetime.fromisoformat("2018-09-03T20:56:35.450686+00:00") -# TODO: use "real" classes once they are defined (instead of NSDict) - -@dataclass -class Placeholder: - pass - -@dataclass -class TstNodeset: - nodeset_name: str = "cantor" - node_count_static: int = 0 - node_count_dynamic_max: int = 0 - node_conf: dict[str, Any] = field(default_factory=dict) - instance_template: Optional[str] = None - reservation_name: Optional[str] = "" - zone_policy_allow: Optional[list[str]] = field(default_factory=list) - enable_placement: bool = True - placement_max_distance: Optional[int] = None - accelerator_topology: Optional[str] = "" - future_reservation: Optional[str] = "" - -@dataclass -class TstPartition: - partition_name: str = "euler" - partition_nodeset: list[str] = field(default_factory=list) - partition_nodeset_tpu: list[str] = field(default_factory=list) - enable_job_exclusive: bool = False - -@dataclass -class TstCfg: - slurm_cluster_name: str = "m22" - cloud_parameters: dict[str, Any] = field(default_factory=dict) - - partitions: dict[str, TstPartition] = field(default_factory=dict) - nodeset: dict[str, TstNodeset] = field(default_factory=dict) - nodeset_tpu: dict[str, TstNodeset] = field(default_factory=dict) - nodeset_dyn: dict[str, TstNodeset] = field(default_factory=dict) - - install_dir: Optional[str] = None - output_dir: Optional[str] = None - - prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - task_prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - task_epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - - -@dataclass -class TstTPU: # to prevent client initialization durint "TPU.__init__" - vmcount: int - -@dataclass -class TstMachineConf: - cpus: int - memory: int - sockets: int - sockets_per_board: int - cores_per_socket: int - boards: int - threads_per_core: int - - -@dataclass -class TstTemplateInfo: - gpu: Optional[util.AcceleratorInfo] - -def tstInstance(name: str, physical_host: Optional[str] = None): - return util.Instance( - name=name, - zone="anorien", - status="RUNNING", - creation_timestamp=SOME_TS, - resource_status=util.InstanceResourceStatus( - physical_host=physical_host, - upcoming_maintenance=None, - ), - scheduling=util.NSDict(), - role="compute", - metadata={}, - ) - -def make_to_hostnames_mock(tbl: Optional[dict[str, list[str]]]): - tbl = tbl or {} - - def se(k: str) -> list[str]: - if k not in tbl: - raise AssertionError(f"to_hostnames mock: unexpected nodelist: '{k}'") - return tbl[k] - - return se diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py deleted file mode 100644 index 6bd6762748..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from mock import Mock -from common import TstNodeset, TstCfg, TstMachineConf, TstTemplateInfo, Placeholder - -import addict # type: ignore -import conf -import util - - -def test_nodeset_tpu_lines(): - nodeset = TstNodeset( - "turbo", - node_count_static=2, - node_count_dynamic_max=3, - node_conf={"red": "velvet"}, - ) - assert conf.nodeset_tpu_lines(nodeset, util.Lookup(TstCfg())) == "\n".join( - [ - "NodeName=m22-turbo-[0-4] State=CLOUD red=velvet", - "NodeSet=turbo Nodes=m22-turbo-[0-4]", - ] - ) - - -def test_nodeset_lines(): - nodeset = TstNodeset( - "turbo", - node_count_static=2, - node_count_dynamic_max=3, - node_conf={"red": "velvet", "CPUs": 55}, - ) - lkp = util.Lookup(TstCfg()) - lkp.template_info = Mock(return_value=TstTemplateInfo( - gpu=util.AcceleratorInfo(type="Popov", count=33) - )) - mc = TstMachineConf( - cpus=5, - memory=6, - sockets=7, - sockets_per_board=8, - boards=9, - threads_per_core=10, - cores_per_socket=11, - ) - lkp.template_machine_conf = Mock(return_value=mc) # type: ignore[method-assign] - assert conf.nodeset_lines(nodeset, lkp) == "\n".join( - [ - "NodeName=m22-turbo-[0-4] State=CLOUD RealMemory=6 Boards=9 SocketsPerBoard=8 CoresPerSocket=11 ThreadsPerCore=10 CPUs=55 Gres=gpu:33 red=velvet", - "NodeSet=turbo Nodes=m22-turbo-[0-4]", - ] - ) - - -@pytest.mark.parametrize( - "value,want", - [ - ({"a": 1}, "a=1"), - ({"a": "two"}, "a=two"), - ({"a": [3, 4]}, "a=3,4"), - ({"a": ["five", "six"]}, "a=five,six"), - ({"a": None}, ""), - ({"a": ["seven", None, 8]}, "a=seven,8"), - ({"a": 1, "b": "two"}, "a=1 b=two"), - ({"a": 1, "b": None, "c": "three"}, "a=1 c=three"), - ({"a": 0, "b": None, "c": 0.0, "e": ""}, "a=0 c=0.0"), - ({"a": [0, 0.0, None, "X", "", "Y"]}, "a=0,0.0,X,,Y"), - ]) -def test_dict_to_conf(value: dict, want: str): - assert conf.dict_to_conf(value) == want - - - -@pytest.mark.parametrize( - "cfg,want", - [ - (TstCfg( - install_dir="ukulele", - ), - """LaunchParameters=enable_nss_slurm,use_interactive_step -SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend -SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -SlurmdTimeout=300 -UnkillableStepTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - (TstCfg( - install_dir="ukulele", - cloud_parameters={ - "no_comma_params": True, - "private_data": None, - "scheduler_parameters": None, - "resume_rate": None, - "resume_timeout": None, - "suspend_rate": None, - "suspend_timeout": None, - "unkillable_step_timeout": None, - "slurmd_timeout": None, - "topology_plugin": None, - "topology_param": None, - "tree_width": None, - }, - ), - """SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -SlurmdTimeout=300 -UnkillableStepTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - (TstCfg( - install_dir="ukulele", - cloud_parameters={ - "no_comma_params": True, - "private_data": [ - "events", - "jobs", - ], - "scheduler_parameters": [ - "bf_busy_nodes", - "bf_continue", - "ignore_prefer_validation", - "nohold_on_prolog_fail", - ], - "resume_rate": 1, - "resume_timeout": 2, - "suspend_rate": 3, - "suspend_timeout": 4, - "slurmd_timeout": 5, - "unkillable_step_timeout": 6, - "tree_width": 7, - "topology_plugin": "guess", - "topology_param": "yellow", - }, - ), - """PrivateData=events,jobs -SchedulerParameters=bf_busy_nodes,bf_continue,ignore_prefer_validation,nohold_on_prolog_fail -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=1 -ResumeTimeout=2 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=3 -SuspendTimeout=4 -SlurmdTimeout=5 -UnkillableStepTimeout=6 -TreeWidth=7 -TopologyPlugin=guess -TopologyParam=yellow"""), - (TstCfg( - install_dir="ukulele", - task_prolog_scripts=[Placeholder()], - task_epilog_scripts=[Placeholder()], - ), - """LaunchParameters=enable_nss_slurm,use_interactive_step -SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend -TaskProlog=/slurm/custom_scripts/task_prolog.d/task-prolog -TaskEpilog=/slurm/custom_scripts/task_epilog.d/task-epilog -SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -SlurmdTimeout=300 -UnkillableStepTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - ]) -def test_conflines(cfg, want): - assert conf.conflines(util.Lookup(cfg)) == want - - cfg.cloud_parameters = addict.Dict(cfg.cloud_parameters) - assert conf.conflines(util.Lookup(cfg)) == want - - -@pytest.mark.parametrize( - "cfg,gputype,gpucount,want", - [ - (TstCfg(), - "", - 0, - "\n"), - (TstCfg( - nodeset={"turbo": TstNodeset("turbo")} - ), - "Popov", - 8, - "Name=gpu Type=Popov File=/dev/nvidia[0-7]\n\n"), - ]) -def test_gen_cloud_gres_conf_lines(cfg, gputype, gpucount, want): - lkp = util.Lookup(cfg) - lkp.template_info = Mock(return_value=TstTemplateInfo( - gpu=util.AcceleratorInfo(type=gputype, count=gpucount) - )) - assert conf.gen_cloud_gres_conf_lines(lkp) == want diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py deleted file mode 100644 index 77f1229605..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional - -import os -import pytest -import unittest.mock -import unittest -import tempfile - -from common import TstCfg, TstNodeset, TstPartition, TstTPU # needed to import util -import util -import resume -from resume import ResumeData, ResumeJobData, BulkChunk, PlacementAndNodes - -def test_get_resume_file_data_no_env(): - with unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": ""}): - assert resume.get_resume_file_data() is None - - -def test_get_resume_file_data(): - with tempfile.NamedTemporaryFile() as f: - f.write(b"""{ - "jobs": [ - { - "extra": null, - "job_id": 1, - "features": null, - "nodes_alloc": "green-[0-2]", - "nodes_resume": "green-[0-1]", - "oversubscribe": "OK", - "partition": "red", - "reservation": null - } - ], - "all_nodes_resume": "green-[0-1]" -}""") - f.flush() - with ( - unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": f.name}), - unittest.mock.patch("util.to_hostnames") as mock_to_hostnames, - ): - mock_to_hostnames.return_value = ["green-0", "green-1", "green-2"] - assert resume.get_resume_file_data() == ResumeData(jobs=[ - ResumeJobData( - job_id = 1, - partition="red", - nodes_alloc=["green-0", "green-1", "green-2"], - ) - ]) - mock_to_hostnames.assert_called_once_with("green-[0-2]") - - -@unittest.mock.patch("tpu.TPU.make") -@unittest.mock.patch("resume.create_placements") -def test_group_nodes_bulk(mock_create_placements, mock_tpu): - cfg = TstCfg( - nodeset={ - "n": TstNodeset(nodeset_name="n"), - }, - nodeset_tpu={ - "t": TstNodeset(nodeset_name="t"), - }, - partitions={ - "p1": TstPartition( - partition_name="p1", - enable_job_exclusive=True, - ), - "p2": TstPartition( - partition_name="p2", - partition_nodeset_tpu=["t"], - enable_job_exclusive=True, - ) - } - ) - lkp = util.Lookup(cfg) - - def mock_create_placements_se(nodes, excl_job_id, lkp): - args = (set(nodes), excl_job_id) - if ({'c-n-1', 'c-n-2', 'c-t-8', 'c-t-9'}, None) == args: - return [ - PlacementAndNodes("g0", ["c-n-1", "c-n-2"]), - PlacementAndNodes(None, ['c-t-8', 'c-t-9']), - ] - if ({"c-n-0", "c-n-8"}, 1) == args: - return [ - PlacementAndNodes("g10", ["c-n-0"]), - PlacementAndNodes("g11", ["c-n-8"]), - ] - if ({'c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5'}, 2) == args: - return [ - PlacementAndNodes(None, ['c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5']) - ] - raise AssertionError(f"unexpected invocation: '{args}'") - mock_create_placements.side_effect = mock_create_placements_se - - def mock_tpu_se(ns: str, lkp) -> TstTPU: - if ns == "t": - return TstTPU(vmcount=2) - raise AssertionError(f"unexpected invocation: '{ns}'") - mock_tpu.side_effect = mock_tpu_se - - got = resume.group_nodes_bulk( - ["c-n-0", "c-n-1", "c-n-2", "c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-8", "c-t-9"], - ResumeData(jobs=[ - ResumeJobData(job_id=1, partition="p1", nodes_alloc=["c-n-0", "c-n-8"]), - ResumeJobData(job_id=2, partition="p2", nodes_alloc=["c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-4", "c-t-5"]), - ]), lkp) - mock_create_placements.assert_called() - assert got == { - "c-n:jobNone:g0:0": BulkChunk( - nodes=["c-n-1", "c-n-2"], prefix="c-n", chunk_idx=0, excl_job_id=None, placement_group="g0"), - "c-n:job1:g10:0": BulkChunk( - nodes=["c-n-0"], prefix="c-n", chunk_idx=0, excl_job_id=1, placement_group="g10"), - "c-t:0": BulkChunk( - nodes=["c-t-8", "c-t-9"], prefix="c-t", chunk_idx=0, excl_job_id=None, placement_group=None), - "c-t:job2:0": BulkChunk( - nodes=["c-t-0", "c-t-1"], prefix="c-t", chunk_idx=0, excl_job_id=2, placement_group=None), - "c-t:job2:1": BulkChunk( - nodes=["c-t-2", "c-t-3"], prefix="c-t", chunk_idx=1, excl_job_id=2, placement_group=None), - } - - -@pytest.mark.parametrize( - "nodes,excl_job_id,expected", - [ - ( # TPU - no placements - ["c-t-0", "c-t-2"], 4, [PlacementAndNodes(None, ["c-t-0", "c-t-2"])] - ), - ( # disabled placements - no placemens - ["c-x-0", "c-x-2"], 4, [PlacementAndNodes(None, ["c-x-0", "c-x-2"])] - ), - ( # excl_job - ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], 4, [ - PlacementAndNodes("c-slurmgcp-managed-n-4-0", ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"]) - ] - ), - ( # no excl_job - ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], None, [ - PlacementAndNodes("c-slurmgcp-managed-n-0-0", ["c-n-0", "c-n-2"]), - PlacementAndNodes('c-slurmgcp-managed-n-0-1', ['c-n-2011']), - PlacementAndNodes(None, ["c-n-uno"]), - ] - ), - ], -) -def test_allocate_nodes_to_placements(nodes: list[str], excl_job_id: Optional[int], expected: list[PlacementAndNodes]): - cfg = TstCfg( - slurm_cluster_name="c", - nodeset={ - "n": TstNodeset(nodeset_name="n", enable_placement=True), - "x": TstNodeset(nodeset_name="x", enable_placement=False) - }, - nodeset_tpu={ - "t": TstNodeset(nodeset_name="t") - }) - lkp = util.Lookup(cfg) - - with unittest.mock.patch("resume.valid_placement_node") as mock_valid_placement_node: - mock_valid_placement_node.return_value = True - lkp.template_info = unittest.mock.Mock(return_value=unittest.mock.Mock(machine_type=unittest.mock.Mock(family="n1"))) - - assert resume._allocate_nodes_to_placements(nodes, excl_job_id, lkp) == expected diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py deleted file mode 100644 index df9f3a0137..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -import json -import mock -from pytest_unordered import unordered -from common import TstCfg, TstNodeset, TstTPU, tstInstance -import sort_nodes - -import util -import conf -import tempfile - -PRELUDE = """ -# Warning: -# This file is managed by a script. Manual modifications will be overwritten. - -""" - -def test_gen_topology_conf_empty(): - out_dir = tempfile.mkdtemp() - cfg = TstCfg(output_dir=out_dir) - conf.gen_topology_conf(util.Lookup(cfg)) - assert open(out_dir + "/cloud_topology.conf").read() == PRELUDE + "\n" - - -@mock.patch("tpu.TPU.make") -def test_gen_topology_conf(tpu_mock): - output_dir = tempfile.mkdtemp() - cfg = TstCfg( - nodeset_tpu={ - "a": TstNodeset("bold", node_count_static=4, node_count_dynamic_max=5), - "b": TstNodeset("slim", node_count_dynamic_max=3), - }, - nodeset={ - "c": TstNodeset("green", node_count_static=2, node_count_dynamic_max=3), - "d": TstNodeset("blue", node_count_static=7), - "e": TstNodeset("pink", node_count_dynamic_max=4), - }, - output_dir=output_dir, - ) - - def tpu_se(ns: str, lkp) -> TstTPU: - if ns == "bold": - return TstTPU(vmcount=3) - if ns == "slim": - return TstTPU(vmcount=1) - raise AssertionError(f"unexpected TPU name: '{ns}'") - - tpu_mock.side_effect = tpu_se - - lkp = util.Lookup(cfg) - lkp.instances = lambda: { n.name: n for n in [ # type: ignore[assignment] - # nodeset blue - tstInstance("m22-blue-0"), # no physicalHost - tstInstance("m22-blue-0", physical_host="/a/a/a"), - tstInstance("m22-blue-1", physical_host="/a/a/b"), - tstInstance("m22-blue-2", physical_host="/a/b/a"), - tstInstance("m22-blue-3", physical_host="/b/a/a"), - # nodeset green - tstInstance("m22-green-3", physical_host="/a/a/c"), - ]} - - uncompressed = conf.gen_topology(lkp) - want_uncompressed = [ - #NOTE: the switch names are not unique, it's not valid content for topology.conf - # The uniquefication and compression of names are done in the compress() method - "SwitchName=slurm-root Switches=a,b,ns_blue,ns_green,ns_pink", - # "physical" topology - 'SwitchName=a Switches=a,b', - 'SwitchName=a Nodes=m22-blue-[0-1],m22-green-3', - 'SwitchName=b Nodes=m22-blue-2', - 'SwitchName=b Switches=a', - 'SwitchName=a Nodes=m22-blue-3', - # topology "by nodeset" - "SwitchName=ns_blue Nodes=m22-blue-[4-6]", - "SwitchName=ns_green Nodes=m22-green-[0-2,4]", - "SwitchName=ns_pink Nodes=m22-pink-[0-3]", - # TPU topology - "SwitchName=tpu-root Switches=ns_bold,ns_slim", - "SwitchName=ns_bold Switches=bold-[0-3]", - "SwitchName=bold-0 Nodes=m22-bold-[0-2]", - "SwitchName=bold-1 Nodes=m22-bold-3", - "SwitchName=bold-2 Nodes=m22-bold-[4-6]", - "SwitchName=bold-3 Nodes=m22-bold-[7-8]", - "SwitchName=ns_slim Nodes=m22-slim-[0-2]"] - assert list(uncompressed.render_conf_lines()) == want_uncompressed - - compressed = uncompressed.compress() - want_compressed = [ - "SwitchName=s0 Switches=s0_[0-4]", # root - # "physical" topology - 'SwitchName=s0_0 Switches=s0_0_[0-1]', # /a - 'SwitchName=s0_0_0 Nodes=m22-blue-[0-1],m22-green-3', # /a/a - 'SwitchName=s0_0_1 Nodes=m22-blue-2', # /a/b - 'SwitchName=s0_1 Switches=s0_1_0', # /b - 'SwitchName=s0_1_0 Nodes=m22-blue-3', # /b/a - # topology "by nodeset" - "SwitchName=s0_2 Nodes=m22-blue-[4-6]", - "SwitchName=s0_3 Nodes=m22-green-[0-2,4]", - "SwitchName=s0_4 Nodes=m22-pink-[0-3]", - # TPU topology - "SwitchName=s1 Switches=s1_[0-1]", - "SwitchName=s1_0 Switches=s1_0_[0-3]", - "SwitchName=s1_0_0 Nodes=m22-bold-[0-2]", - "SwitchName=s1_0_1 Nodes=m22-bold-3", - "SwitchName=s1_0_2 Nodes=m22-bold-[4-6]", - "SwitchName=s1_0_3 Nodes=m22-bold-[7-8]", - "SwitchName=s1_1 Nodes=m22-slim-[0-2]"] - assert list(compressed.render_conf_lines()) == want_compressed - - upd, summary = conf.gen_topology_conf(lkp) - assert upd == True - want_written = PRELUDE + "\n".join(want_compressed) + "\n\n" - assert open(output_dir + "/cloud_topology.conf").read() == want_written - - summary.dump(lkp) - summary_got = json.loads(open(output_dir + "/cloud_topology.summary.json").read()) - - assert summary_got == { - "down_nodes": unordered( - [f"m22-blue-{i}" for i in (4,5,6)] + - [f"m22-green-{i}" for i in (0,1,2,4)] + - [f"m22-pink-{i}" for i in range(4)]), - "tpu_nodes": unordered( - [f"m22-bold-{i}" for i in range(9)] + - [f"m22-slim-{i}" for i in range(3)]), - 'physical_host': { - 'm22-blue-0': '/a/a/a', - 'm22-blue-1': '/a/a/b', - 'm22-blue-2': '/a/b/a', - 'm22-blue-3': '/b/a/a', - 'm22-green-3': '/a/a/c'}, - } - - - -def test_gen_topology_conf_update(): - cfg = TstCfg( - nodeset={ - "c": TstNodeset("green", node_count_static=2), - }, - output_dir=tempfile.mkdtemp(), - ) - lkp = util.Lookup(cfg) - lkp.instances = lambda: { # type: ignore[assignment] - # no instances - } - - # initial generation - reconfigure - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # add node: node_count_static 2 -> 3 - reconfigure - lkp.cfg.nodeset["c"].node_count_static = 3 - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # remove node: node_count_static 3 -> 2 - no reconfigure - lkp.cfg.nodeset["c"].node_count_static = 2 - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - # set empty physicalHost - no reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - # set physicalHost - reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/c")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # change physicalHost - reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/z")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # shut down node - no reconfigure - lkp.instances = lambda: {} # type: ignore[assignment] - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - -@pytest.mark.parametrize( - "paths,expected", - [ - (["z/n-0", "z/n-1", "z/n-2", "z/n-3", "z/n-4", "z/n-10"], ['n-0', 'n-1', 'n-2', 'n-3', 'n-4', 'n-10']), - (["y/n-0", "z/n-1", "x/n-2", "x/n-3", "y/n-4", "g/n-10"], ['n-0', 'n-4', 'n-1', 'n-2', 'n-3', 'n-10']), - ]) -def test_sort_nodes_order(paths: list[str], expected: list[str]) -> None: - paths_expanded = [l.split("/") for l in paths] - assert sort_nodes.order(paths_expanded) == expected diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py deleted file mode 100644 index 69617d0301..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py +++ /dev/null @@ -1,668 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Type - -import pytest -from mock import Mock -from datetime import datetime, timezone, timedelta -import unittest - -from common import TstNodeset, TstCfg # needed to import util -import util -from util import NodeState, MachineType, AcceleratorInfo, UpcomingMaintenance, InstanceResourceStatus, FutureReservation, ReservationDetails -from google.api_core.client_options import ClientOptions # noqa: E402 -from addict import Dict as NSDict # type: ignore - -# Note: need to install pytest-mock - -@pytest.mark.parametrize( - "name,expected", - [ - ( - "az-buka-23", - { - "cluster": "az", - "nodeset": "buka", - "node": "23", - "prefix": "az-buka", - "range": None, - "suffix": "23", - }, - ), - ( - "az-buka-xyzf", - { - "cluster": "az", - "nodeset": "buka", - "node": "xyzf", - "prefix": "az-buka", - "range": None, - "suffix": "xyzf", - }, - ), - ( - "az-buka-[2-3]", - { - "cluster": "az", - "nodeset": "buka", - "node": "[2-3]", - "prefix": "az-buka", - "range": "[2-3]", - "suffix": None, - }, - ), - ], -) -def test_node_desc(name, expected): - assert util.lookup()._node_desc(name) == expected - - -@pytest.mark.parametrize( - "name,expected", - [ - ("az-buka-23", 23), - ("az-buka-0", 0), - ("az-buka", Exception), - ("az-buka-xyzf", ValueError), - ("az-buka-[2-3]", ValueError), - ], -) -def test_node_index(name, expected): - if type(expected) is type and issubclass(expected, Exception): - with pytest.raises(expected): - util.lookup().node_index(name) - else: - assert util.lookup().node_index(name) == expected - - -@pytest.mark.parametrize( - "name", - [ - "az-buka", - ], -) -def test_node_desc_fail(name): - with pytest.raises(Exception): - util.lookup()._node_desc(name) - - -@pytest.mark.parametrize( - "names,expected", - [ - ("pedro,pedro-1,pedro-2,pedro-01,pedro-02", "pedro,pedro-[1-2,01-02]"), - ("pedro,,pedro-1,,pedro-2", "pedro,pedro-[1-2]"), - ("pedro-8,pedro-9,pedro-10,pedro-11", "pedro-[8-9,10-11]"), - ("pedro-08,pedro-09,pedro-10,pedro-11", "pedro-[08-11]"), - ("pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-09]"), - ("pedro-10,pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-10]"), - ("pedro-8,pedro-9,juan-10,juan-11", "juan-[10-11],pedro-[8-9]"), - ("az,buki,vedi", "az,buki,vedi"), - ("a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12", "a[0-9,10-12]"), - ("a0,a2,a4,a6,a7,a8,a11,a12", "a[0,2,4,6-8,11-12]"), - ("seas7-0,seas7-1", "seas7-[0-1]"), - ], -) -def test_to_hostlist(names, expected): - assert util.to_hostlist(names.split(",")) == expected - - -@pytest.mark.parametrize( - "api,ep_ver,expected", - [ - ( - util.ApiEndpoint.BQ, - "v1", - ClientOptions(api_endpoint="https://bq.googleapis.com/v1/"), - ), - ( - util.ApiEndpoint.COMPUTE, - "staging_v1", - ClientOptions(api_endpoint="https://compute.googleapis.com/staging_v1/"), - ), - ( - util.ApiEndpoint.SECRET, - "v1", - ClientOptions(api_endpoint="https://secret_manager.googleapis.com/v1/"), - ), - ( - util.ApiEndpoint.STORAGE, - "beta", - ClientOptions(api_endpoint="https://storage.googleapis.com/beta/"), - ), - ( - util.ApiEndpoint.TPU, - "alpha", - ClientOptions(api_endpoint="https://tpu.googleapis.com/alpha/"), - ), - ], -) -def test_create_client_options( - api: util.ApiEndpoint, ep_ver: str, expected: ClientOptions, mocker -): - ud_mock = mocker.patch("util.universe_domain") - ep_mock = mocker.patch("util.endpoint_version") - ud_mock.return_value = "googleapis.com" - ep_mock.return_value = ep_ver - assert util.create_client_options(api).__repr__() == expected.__repr__() - - - -@pytest.mark.parametrize( - "nodeset,err", - [ - (TstNodeset(reservation_name="projects/x/reservations/y"), AssertionError), # no zones - (TstNodeset( - reservation_name="projects/x/reservations/y", - zone_policy_allow=["eine", "zwei"]), AssertionError), # multiples zones - (TstNodeset( - reservation_name="robin", - zone_policy_allow=["eine"]), ValueError), # invalid name - (TstNodeset( - reservation_name="projects/reservations/y", - zone_policy_allow=["eine"]), ValueError), # invalid name - (TstNodeset( - reservation_name="projects/x/zones/z/reservations/y", - zone_policy_allow=["eine"]), ValueError), # invalid name - ] -) -def test_nodeset_reservation_err(nodeset, err): - lkp = util.Lookup(TstCfg()) - lkp._get_reservation = Mock() - with pytest.raises(err): - lkp.nodeset_reservation(nodeset) - lkp._get_reservation.assert_not_called() # type: ignore - -@pytest.mark.parametrize( - "nodeset,policies,expected", - [ - (TstNodeset(), [], None), # no reservation - (TstNodeset( - reservation_name="projects/bobin/reservations/robin", - zone_policy_allow=["eine"]), - [], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=[], - deployment_type=None, - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/bobin/reservations/robin")), - (TstNodeset( - reservation_name="projects/bobin/reservations/robin", - zone_policy_allow=["eine"]), - ["seven/wanders", "five/red/apples", "yum"], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=["wanders", "apples", "yum"], - deployment_type=None, - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/bobin/reservations/robin")), - (TstNodeset( - reservation_name="projects/bobin/reservations/robin/snek/cheese-brie-6", - zone_policy_allow=["eine"]), - [], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=[], - deployment_type=None, - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/bobin/reservations/robin/snek/cheese-brie-6")), - - ]) - -def test_nodeset_reservation_ok(nodeset, policies, expected): - lkp = util.Lookup(TstCfg()) - lkp._get_reservation = Mock() - - if not expected: - assert lkp.nodeset_reservation(nodeset) is None - lkp._get_reservation.assert_not_called() # type: ignore - return - - lkp._get_reservation.return_value = { # type: ignore - "resourcePolicies": {i: p for i, p in enumerate(policies)}, - } - assert lkp.nodeset_reservation(nodeset) == expected - lkp._get_reservation.assert_called_once_with(expected.project, expected.zone, expected.name) # type: ignore - -@pytest.mark.parametrize( - "job_info,expected_job", - [ - ( - """JobId=123 - TimeLimit=02:00:00 - JobName=myjob - JobState=PENDING - ReqNodeList=node-[1-10]""", - util.Job( - id=123, - duration=timedelta(days=0, hours=2, minutes=0, seconds=0), - name="myjob", - job_state="PENDING", - required_nodes="node-[1-10]" - ), - ), - ( - """JobId=456 - JobName=anotherjob - JobState=PENDING - ReqNodeList=node-group1""", - util.Job( - id=456, - duration=None, - name="anotherjob", - job_state="PENDING", - required_nodes="node-group1" - ), - ), - ( - """JobId=789 - TimeLimit=00:30:00 - JobState=COMPLETED""", - util.Job( - id=789, - duration=timedelta(minutes=30), - name=None, - job_state="COMPLETED", - required_nodes=None - ), - ), - ( - """JobId=101112 - TimeLimit=1-00:30:00 - JobState=COMPLETED, - ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", - util.Job( - id=101112, - duration=timedelta(days=1, hours=0, minutes=30, seconds=0), - name=None, - job_state="COMPLETED", - required_nodes="node-[1-10],grob-pop-[2,1,44-77]" - ), - ), - ( - """JobId=131415 - TimeLimit=1-00:30:00 - JobName=mynode-1_maintenance - JobState=COMPLETED, - ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", - util.Job( - id=131415, - duration=timedelta(days=1, hours=0, minutes=30, seconds=0), - name="mynode-1_maintenance", - job_state="COMPLETED", - required_nodes="node-[1-10],grob-pop-[2,1,44-77]" - ), - ), - ], -) -def test_parse_job_info(job_info, expected_job): - lkp = util.Lookup(TstCfg()) - assert lkp._parse_job_info(job_info) == expected_job - - - -@pytest.mark.parametrize( - "node,state,want", - [ - ("c-n-2", NodeState("DOWN", frozenset([])), NodeState("DOWN", frozenset([]))), # happy scenario - ("c-d-vodoo", None, None), # dynamic nodeset - ("c-x-44", None, None), # unknown(removed) nodeset - ("c-n-7", None, None), # Out of bounds: c-n-[0-4] - downsized nodeset - ("c-t-7", None, None), # Out of bounds: c-t-[0-4] - downsized nodeset TPU - ("c-n-2", None, RuntimeError), # something is wrong - ("c-t-2", None, RuntimeError), # something is wrong, but TPU - - # Check boundaries match [0-5) - ("c-n-5", None, None), # out of boundaries - ("c-n-4", None, RuntimeError), # within boundaries - ]) -def test_node_state(node: str, state: Optional[NodeState], want: NodeState | None | Type[Exception]): - cfg = TstCfg( - slurm_cluster_name="c", - nodeset={ - "n": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, - nodeset_tpu={ - "t": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, - nodeset_dyn={ - "d": TstNodeset()}, - ) - lkp = util.Lookup(cfg) - lkp.slurm_nodes = lambda: {node: state} if state else {} # type: ignore[assignment] - # ... see https://github.com/python/typeshed/issues/6347 - - if type(want) is type and issubclass(want, Exception): - with pytest.raises(want): - lkp.node_state(node) - else: - assert lkp.node_state(node) == want - - - -@pytest.mark.parametrize( - "jo,want", - [ - ({ - "accelerators": [ { "guestAcceleratorCount": 1, "guestAcceleratorType": "nvidia-tesla-a100" } ], - "creationTimestamp": "1969-12-31T16:00:00.000-08:00", - "description": "Accelerator Optimized: 1 NVIDIA Tesla A100 GPU, 12 vCPUs, 85GB RAM", - "guestCpus": 12, - "id": "1000012", - "imageSpaceGb": 0, - "isSharedCpu": False, - "kind": "compute#machineType", - "maximumPersistentDisks": 128, - "maximumPersistentDisksSizeGb": "263168", - "memoryMb": 87040, - "name": "a2-highgpu-1g", - "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/us-central1-a/machineTypes/a2-highgpu-1g", - "zone": "us-central1-a" - }, MachineType( - name="a2-highgpu-1g", - guest_cpus=12, - memory_mb=87040, - accelerators=[ - AcceleratorInfo(type="nvidia-tesla-a100", count=1) - ] - )), - ({ - "architecture": "X86_64", - "creationTimestamp": "1969-12-31T16:00:00.000-08:00", - "description": "8 vCPUs, 32 GB RAM", - "guestCpus": 8, - "id": "1210008", - "imageSpaceGb": 0, - "isSharedCpu": False, - "kind": "compute#machineType", - "maximumPersistentDisks": 128, - "maximumPersistentDisksSizeGb": "263168", - "memoryMb": 32768, - "name": "t2d-standard-8", - "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/europe-north2-b/machineTypes/t2d-standard-8", - "zone": "europe-north2-b" - }, MachineType( - name="t2d-standard-8", - guest_cpus=8, - memory_mb=32768, - accelerators=[] - )), - ]) -def test_MachineType_from_json(jo: dict, want: MachineType): - assert MachineType.from_json(jo) == want - - -@pytest.mark.parametrize( - "template,expected", - [ - ( - NSDict({ - "machine_type": MachineType( - name="e2", - guest_cpus=12, - memory_mb=87040, - accelerators=[]), - }), - None - ), - ( - NSDict({ - "machine_type": MachineType( - name="tpu-machine", - guest_cpus=12, - memory_mb=87040, - accelerators=[ - AcceleratorInfo(type="tpu-v6", count=1) - ]), - }), - None - ), - ( - NSDict({ - "machine_type": MachineType( - name="a2-highgpu-1g", - guest_cpus=12, - memory_mb=87040, - accelerators=[AcceleratorInfo(type="nvidia-tesla-a100", count=1)] - ), - }), - AcceleratorInfo(type="nvidia-tesla-a100", count=1) - ), - ( - NSDict({ - "machine_type": MachineType( - name="a2-highgpu-1g", - guest_cpus=12, - memory_mb=87040, - accelerators=[]), - "guestAccelerators":[ { "acceleratorCount": 1, "acceleratorType": "nvidia-tesla-a100" } ], - }), - AcceleratorInfo(type="nvidia-tesla-a100", count=1) - ), - ], -) -def test_get_template_gpu(template, expected): - assert util.get_template_gpu(template) == expected - - -UTC, PST = timezone.utc, timezone(timedelta(hours=-8)) - -@pytest.mark.parametrize( - "got,want", - [ - # from instance.creationTimestamp: - ("2024-11-30T12:47:51.676-08:00", datetime(2024, 11, 30, 12, 47, 51, 676000, tzinfo=PST)), - # from futureReservation.creationTimestamp - ("2024-11-05T15:23:33.702-08:00", datetime(2024, 11, 5, 15, 23, 33, 702000, tzinfo=PST)), - # from futureReservation.timeWindow.endTime - ("2025-01-15T00:00:00Z", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), - # fallback to UTC if no tz is specified - ("2025-01-15T00:00:00", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), - ]) -def test_parse_gcp_timestamp(got: str, want: datetime): - assert util.parse_gcp_timestamp(got) == want - - -@pytest.mark.parametrize( - "got,want", - [ - (None, None), - (dict( - windowStartTime="2025-01-15T00:00:00Z", - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - (dict( - startTimeWindow=dict( - earliest="2025-01-15T00:00:00Z"), - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - (dict( - windowStartTime="2025-01-15T00:00:00Z", - startTimeWindow=dict( - earliest="2025-01-25T00:00:00Z"), # ignored - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - ]) -def tests_parse_UpcomingMaintenance_OK(got: dict, want: Optional[UpcomingMaintenance]): - assert UpcomingMaintenance.from_json(got) == want - - -@pytest.mark.parametrize( - "got", - [ - {}, - dict( - windowStartTime=dict( - earliest="2025-01-15T00:00:00Z")), - ]) -def tests_parse_UpcomingMaintenance_FAIL(got: dict): - with pytest.raises(ValueError): - UpcomingMaintenance.from_json(got) - - -@pytest.mark.parametrize( - "got,want", - [ - (None, InstanceResourceStatus( - physical_host=None, - upcoming_maintenance=None)), - ({}, InstanceResourceStatus( - physical_host=None, - upcoming_maintenance=None)), - (dict( - physicalHost="/aaa/bbb/ccc"), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=None)), - (dict( # invalid upcomingMaintenance field to be ignored - physicalHost="/aaa/bbb/ccc", - upcomingMaintenance="maintenance is upon us"), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=None)), - (dict( - physicalHost="/aaa/bbb/ccc", - upcomingMaintenance=dict(windowStartTime="2025-01-15T00:00:00Z")), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=UpcomingMaintenance( - window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC)))), - ]) -def test_parse_InstanceResourceStatus(got: dict, want: Optional[InstanceResourceStatus]): - assert InstanceResourceStatus.from_json(got) == want - - -@pytest.mark.parametrize( - "link,component_name,expected", - [ - ( - "mylink/regions/us-cental1/other", - "regions", - "us-cental1" - ), - ( - "mylink/global/other", - "regions", - None - ), - ], -) -def test_get_self_link_component(link, component_name, expected): - assert util.get_self_link_component(link, component_name) == expected - - -def test_future_reservation_none(): - lkp = util.Lookup(TstCfg()) - assert lkp.future_reservation(TstNodeset()) == None - - -def test_future_reservation_declined(): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-03T23:30:00Z" }, - status = {"procurementStatus": "DECLINED"}, - reservationMode = "CALENDAR", - specificReservationRequired = True, - )) - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 3, 23, 30, tzinfo=timezone.utc), - reservation_mode="CALENDAR", - active_reservation=None) - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - -@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 13, 0, 0, tzinfo=timezone.utc)) -def test_future_reservation_active(_): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, - status = { - "procurementStatus": "FULFILLED", - "autoCreatedReservations": [ - "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" - ], - }, - specificReservationRequired = True, - )) - lkp._get_reservation = Mock(return_value=dict()) - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), - reservation_mode=None, - active_reservation=ReservationDetails( - project='manhattan', - zone='danger', - name='melon', - policies=[], - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/manhattan/reservations/melon", - deployment_type=None)) - - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - lkp._get_reservation.assert_called_once_with("manhattan", "danger", "melon") - -@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 28, 0, 0, tzinfo=timezone.utc)) -def test_future_reservation_inactive(_): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, - status = { - "procurementStatus": "FULFILLED", - "autoCreatedReservations": [ - "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" - ], - }, - reservationMode = "DEFAULT", - specificReservationRequired = True, - )) - lkp._get_reservation = Mock() - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), - reservation_mode="DEFAULT", - active_reservation=None) - - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - lkp._get_reservation.assert_not_called() diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test deleted file mode 100644 index a583642015..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -unset CUDA_VISIBLE_DEVICES - -LOG_FILE="/var/log/slurm/chs_health_check.log" -TMP_DCGM_OUT="/tmp/dcgm.out" -TMP_ECC_ERRORS_OUT="/tmp/ecc_errors.out" - -log_step() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" -} - -# Fail gracefully if nvidia-smi or dcgmi doesn't exist -if ! type -P nvidia-smi 1>/dev/null; then - log_step "nvidia-smi not found - this script requires nvidia-smi to function" - exit 0 -fi - -if ! type -P dcgmi 1>/dev/null; then - log_step "dcgmi not found - this script requires dcgmi to function" - exit 0 -fi - -if ! type -P nv-hostengine 1>/dev/null; then - log_step "nv-hostengine not found - this script requires nv-hostengine to function" - exit 0 -fi - -################################################### -# Disable running health checks -################################################### -# Check if the environment variable '$SLURM_JOB_EXTRA' is set and contains the -# substring 'healthchecks_prolog=off' -if [[ -n "$SLURM_JOB_EXTRA" ]]; then - log_step "Environment variable SLURM_JOB_EXTRA is set. Checking if it contains healthchecks_prolog=off." - # Check if the value of the variable matches the string "healthchecks_prolog=off" - if [[ "$SLURM_JOB_EXTRA" == *"healthchecks_prolog=off"* ]]; then - log_step "Environment variable SLURM_JOB_EXTRA matches substring healthchecks_prolog=off. Skipping health checks." - exit 0 - else - log_step "Environment variable SLURM_JOB_EXTRA does NOT match substring healthchecks_prolog=off. Attempting to run health checks." - fi -else - log_step "Environment variable SLURM_JOB_EXTRA is NOT set. Attempting to run health checks." -fi - -# Exit if GPU isn't H/B 100/200 -GPU_MODEL=$(nvidia-smi --query-gpu=name --format=csv,noheader) -if ! [[ "$GPU_MODEL" =~ [BH][1-2]00 ]]; then - log_step "No Supported GPU detected" - exit 0 -fi - -NUMGPUS=$(nvidia-smi -L | wc -l) - -# Check that all GPUs are healthy via DCGM and check for ECC errors -if [ $NUMGPUS -gt 0 ]; then - log_step "Execute DCGM health check, ECC error check, and NVLink error check for GPUs" - GPULIST=$(nvidia-smi --query-gpu=index --format=csv,noheader | tr '\n' ',' | sed 's/,$//') - rm -f $TMP_DCGM_OUT - rm -f $TMP_ECC_ERRORS_OUT - - # Run DCGM checks - START_HOSTENGINE=false - if ! pidof nv-hostengine > /dev/null; then - log_step "Starting nv-hostengine..." - nv-hostengine >> "$LOG_FILE" 2>&1 - sleep 1 # Give it a moment to start up - START_HOSTENGINE=true - fi - GROUPID=$(dcgmi group -c gpuinfo | awk '{print $NF}' | tr -d ' ') - dcgmi group -g $GROUPID -a $GPULIST >> "$LOG_FILE" 2>&1 - dcgmi diag -g $GROUPID -r 1 > "$TMP_DCGM_OUT" 2>&1 - cat "$TMP_DCGM_OUT" >> "$LOG_FILE" - dcgmi group -d $GROUPID >> "$LOG_FILE" 2>&1 - - # Terminate the host engine if it was manually started - if [ "$START_HOSTENGINE" = true ]; then - log_step "Terminating nv-hostengine..." - nv-hostengine -t >> "$LOG_FILE" 2>&1 - fi - - # Check for DCGM failures - DCGM_FAILED=0 - if grep -i fail "$TMP_DCGM_OUT" > /dev/null; then - DCGM_FAILED=1 - fi - - # Check for ECC errors - nvidia-smi --query-gpu=ecc.errors.uncorrected.volatile.total --format=csv,noheader > "$TMP_ECC_ERRORS_OUT" - cat "$TMP_ECC_ERRORS_OUT" >> "$LOG_FILE" - ECC_ERRORS=$(awk -F', ' '{sum += $2} END {print sum}' "$TMP_ECC_ERRORS_OUT") - log_step "ECC Errors: $ECC_ERRORS" - - # Check for NVLink errors - NVLINK_ERRORS=$(nvidia-smi nvlink -sc 0bz -i 0 2>/dev/null | grep -i "Error Count" | awk '{sum += $3} END {print sum}') - # Set to 0 if empty/null - NVLINK_ERRORS=${NVLINK_ERRORS:-0} - log_step "NVLink Errors: $NVLINK_ERRORS" - - if [ $DCGM_FAILED -eq 1 ] || \ - [ $ECC_ERRORS -gt 0 ] || \ - [ $NVLINK_ERRORS -gt 0 ]; then - REASON="GPU issues detected: " - if [ $DCGM_FAILED -eq 1 ]; then - REASON+="DCGM test failed, " - fi - if [ $ECC_ERRORS -gt 0 ]; then - REASON+="ECC errors found ($ECC_ERRORS double-bit errors), " - fi - if [ $NVLINK_ERRORS -gt 0 ]; then - REASON+="NVLink errors detected ($NVLINK_ERRORS errors), " - fi - REASON+="see $LOG_FILE" - log_step "$REASON" - exit 1 - fi -fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog deleted file mode 100644 index a22ddea9e5..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Main TaskEpilog Script -# This script executes all *.sh scripts found in /slurm/custom_scripts/task_epilog.d/ -# -# slurm.conf configuration: -# TaskEpilog=/slurm/scripts/tools/task-epilog - -# Directory containing the individual task epilog scripts -EPILOG_D_DIR="/slurm/custom_scripts/task_epilog.d" - -# --- Output Handling for TaskEpilog --- -# The stdout and stderr of this script (and the sub-scripts it calls) -# are typically captured by Slurm and written to the job's output/error file -# or a separate Slurm log, depending on configuration. -# Unlike TaskProlog, stdout is not typically parsed for special commands -# like 'export' or 'print' to affect the (now finished) task's environment. -# -# --- Error Handling --- -# If any script in EPILOG_D_DIR exits with a non-zero status, -# this main script will also exit with a non-zero status. -# Slurm will log this. Depending on Slurm's configuration, -# frequent epilog failures might lead to node issues or alerts. -set -e # Exit immediately if a command exits with a non-zero status. - -# Check if the directory exists -if [[ ! -d "$EPILOG_D_DIR" ]]; then - # Log in task stdout and exit if the directory is missing. This likely indicates a configuration error. - echo "print TaskEpilog Error: Directory '$EPILOG_D_DIR' not found. Check Slurm configuration." - exit 1 -fi - -# Find and execute all *.sh scripts in the directory -# Scripts will be executed in reverse alphabetical order of their filenames. -find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do - if [[ -x "$script" ]]; then - # Execute the script. Its stdout will be captured by this wrapper. - # Its stderr will also be passed through. - # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. - "$script" - else - # Log in task stdout a warning if a *.sh file is found but is not executable - echo "print TaskEpilog Warning: Script '$script' is not executable and will be skipped." - fi -done - -# Check if any scripts were found and executed -if [[ $(find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then - # Log in task stdout if no scripts were found to execute - echo "print TaskEpilog Info: No executable *.sh scripts found in $EPILOG_D_DIR." -fi - -# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) -exit 0 diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog deleted file mode 100644 index feddb23209..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Main TaskProlog Script -# This script executes all *.sh scripts found in /slurm/custom_scripts/task_prolog.d/ -# -# slurm.conf configuration: -# TaskProlog=/slurm/scripts/tools/task-prolog - -# Directory containing the individual task prolog scripts -PROLOG_D_DIR="/slurm/custom_scripts/task_prolog.d" - -# --- Output Handling for TaskProlog --- -# Slurm's TaskProlog can interpret specific stdout lines: -# - "export NAME=value" : Sets an environment variable for the task. -# - "unset NAME" : Unsets an environment variable for the task. -# - "print message" : Prints a message to the task's standard output. -# -# This wrapper script will concatenate the stdout of all sub-scripts. -# If sub-scripts need to set/unset environment variables or print messages -# for the task, they should output the appropriate "export", "unset", or "print" -# commands to their own stdout. - -# --- Error Handling --- -# If any script in PROLOG_D_DIR exits with a non-zero status, -# this main script will also exit with a non-zero status. -# This will typically cause the task to fail. -set -e # Exit immediately if a command exits with a non-zero status. - -# Check if the directory exists -if [[ ! -d "$PROLOG_D_DIR" ]]; then - # Log in task stdout and exit if the directory is missing. All jobs will be failed. - echo "print TaskProlog Error: Directory '$PROLOG_D_DIR' not found. Check Slurm configuration." - exit 1 -fi - -# Find and execute all *.sh scripts in the directory -# Scripts will be executed in reverse alphabetical order of their filenames. -find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do - if [[ -x "$script" ]]; then - # Execute the script. Its stdout will be captured by this wrapper. - # Its stderr will also be passed through. - # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. - "$script" - else - # Log a warning in task stdout if a *.sh file is found but is not executable - echo "print TaskProlog Warning: Script '$script' is not executable and will be skipped." - fi -done - -# Check if any scripts were found and executed -if [[ $(find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then - # Log in task stdout if no scripts were found to execute - echo "print TaskProlog Info: No executable *.sh scripts found in $PROLOG_D_DIR." -fi - -# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) -exit 0 diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py deleted file mode 100644 index 531f0348dc..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py +++ /dev/null @@ -1,331 +0,0 @@ -# mypy: ignore-errors -# This implementation of TPU integration is to be deprecated - -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List - -import socket -import logging -from pathlib import Path -import yaml - -import util -from util import create_client_options, ApiEndpoint - -from google.cloud import tpu_v2 as tpu # noqa: E402 -import google.api_core.exceptions as gExceptions # noqa: E402 - -log = logging.getLogger() - -_tpu_cache = {} - -class TPU: - """Class for handling the TPU-vm nodes""" - - State = tpu.types.cloud_tpu.Node.State - TPUS_PER_VM = 4 - __expected_states = { - "create": State.READY, - "start": State.READY, - "stop": State.STOPPED, - } - - __tpu_version_mapping = { - "V2": tpu.AcceleratorConfig().Type.V2, - "V3": tpu.AcceleratorConfig().Type.V3, - "V4": tpu.AcceleratorConfig().Type.V4, - } - - @classmethod - def make(cls, nodeset_name: str, lkp: util.Lookup) -> "TPU": - key = (id(lkp), nodeset_name) - if key not in _tpu_cache: - nodeset = lkp.cfg.nodeset_tpu[nodeset_name] - _tpu_cache[key] = cls(nodeset, lkp) - return _tpu_cache[key] - - - def __init__(self, nodeset: object, lkp: util.Lookup): - self._nodeset = nodeset - self.lkp = lkp - self._parent = f"projects/{lkp.project}/locations/{nodeset.zone}" - co = create_client_options(ApiEndpoint.TPU) - self._client = tpu.TpuClient(client_options=co) - self.data_disks = [] - for data_disk in nodeset.data_disks: - ad = tpu.AttachedDisk() - ad.source_disk = data_disk - ad.mode = tpu.AttachedDisk.DiskMode.DISK_MODE_UNSPECIFIED - self.data_disks.append(ad) - ns_ac = nodeset.accelerator_config - if ns_ac.topology != "" and ns_ac.version != "": - ac = tpu.AcceleratorConfig() - ac.topology = ns_ac.topology - ac.type_ = self.__tpu_version_mapping[ns_ac.version] - self.ac = ac - else: - req = tpu.GetAcceleratorTypeRequest( - name=f"{self._parent}/acceleratorTypes/{nodeset.node_type}" - ) - self.ac = self._client.get_accelerator_type(req).accelerator_configs[0] - self.vmcount = self.__calc_vm_from_topology(self.ac.topology) - - @property - def nodeset(self): - return self._nodeset - - @property - def preserve_tpu(self): - return self._nodeset.preserve_tpu - - @property - def node_type(self): - return self._nodeset.node_type - - @property - def tf_version(self): - return self._nodeset.tf_version - - @property - def enable_public_ip(self): - return self._nodeset.enable_public_ip - - @property - def preemptible(self): - return self._nodeset.preemptible - - @property - def reserved(self): - return self._nodeset.reserved - - @property - def service_account(self): - return self._nodeset.service_account - - @property - def zone(self): - return self._nodeset.zone - - def check_node_type(self): - if self.node_type is None: - return False - try: - request = tpu.GetAcceleratorTypeRequest( - name=f"{self._parent}/acceleratorTypes/{self.node_type}" - ) - return self._client.get_accelerator_type(request=request) is not None - except Exception: - return False - - def check_tf_version(self): - try: - request = tpu.GetRuntimeVersionRequest( - name=f"{self._parent}/runtimeVersions/{self.tf_version}" - ) - return self._client.get_runtime_version(request=request) is not None - except Exception: - return False - - def __calc_vm_from_topology(self, topology): - topo = topology.split("x") - tot = 1 - for num in topo: - tot = tot * int(num) - return tot // self.TPUS_PER_VM - - def __check_resp(self, response, op_name): - des_state = self.__expected_states.get(op_name) - # If the state is not in the table just print the response - if des_state is None: - return False - if response.__class__.__name__ != "Node": # If the response is not a node fail - return False - if response.state == des_state: - return True - return False - - def list_nodes(self): - try: - request = tpu.ListNodesRequest(parent=self._parent) - res = self._client.list_nodes(request=request) - except gExceptions.NotFound: - res = None - return res - - def list_node_names(self): - return [node.name.split("/")[-1] for node in self.list_nodes()] - - def start_node(self, nodename): - request = tpu.StartNodeRequest(name=f"{self._parent}/nodes/{nodename}") - resp = self._client.start_node(request=request).result() - return self.__check_resp(resp, "start") - - def stop_node(self, nodename): - request = tpu.StopNodeRequest(name=f"{self._parent}/nodes/{nodename}") - resp = self._client.stop_node(request=request).result() - return self.__check_resp(resp, "stop") - - def get_node(self, nodename): - try: - request = tpu.GetNodeRequest(name=f"{self._parent}/nodes/{nodename}") - res = self._client.get_node(request=request) - except gExceptions.NotFound: - res = None - return res - - def _register_node(self, nodename, ip_addr): - dns_name = socket.getnameinfo((ip_addr, 0), 0)[0] - util.run( - f"{self.lkp.scontrol} update nodename={nodename} nodeaddr={ip_addr} nodehostname={dns_name}" - ) - - def create_node(self, nodename): - if self.vmcount > 1 and not isinstance(nodename, list): - log.error( - f"Tried to create a {self.vmcount} node TPU on nodeset {self._nodeset.nodeset_name} but only received one nodename {nodename}" - ) - return False - if self.vmcount > 1 and ( - isinstance(nodename, list) and len(nodename) != self.vmcount - ): - log.error( - f"Expected to receive a list of {self.vmcount} nodenames for TPU node creation in nodeset {self._nodeset.nodeset_name}, but received this list {nodename}" - ) - return False - - node = tpu.Node() - node.accelerator_config = self.ac - node.runtime_version = f"tpu-vm-tf-{self.tf_version}" - startup_script = """ - #!/bin/bash - echo "startup script not found > /var/log/startup_error.log" - """ - with open( - Path(self.lkp.cfg.slurm_scripts_dir or util.dirs.scripts) / "startup.sh", "r" - ) as script: - startup_script = script.read() - if isinstance(nodename, list): - node_id = nodename[0] - slurm_names = [] - wid = 0 - for node_wid in nodename: - slurm_names.append(f"WORKER_{wid}:{node_wid}") - wid += 1 - else: - node_id = nodename - slurm_names = [f"WORKER_0:{nodename}"] - node.metadata = { - "slurm_docker_image": self.nodeset.docker_image, - "startup-script": startup_script, - "slurm_instance_role": "compute", - "slurm_cluster_name": self.lkp.cfg.slurm_cluster_name, - "slurm_bucket_path": self.lkp.cfg.bucket_path, - "slurm_names": ";".join(slurm_names), - "universe_domain": util.universe_domain(), - } - node.tags = [self.lkp.cfg.slurm_cluster_name] - if self.nodeset.service_account: - node.service_account.email = self.nodeset.service_account.email - node.service_account.scope = self.nodeset.service_account.scopes - node.scheduling_config.preemptible = self.preemptible - node.scheduling_config.reserved = self.reserved - node.network_config.subnetwork = self.nodeset.subnetwork - node.network_config.enable_external_ips = self.enable_public_ip - if self.data_disks: - node.data_disks = self.data_disks - - request = tpu.CreateNodeRequest(parent=self._parent, node=node, node_id=node_id) - resp = self._client.create_node(request=request).result() - if not self.__check_resp(resp, "create"): - return False - if isinstance(nodename, list): - for node_id, net_endpoint in zip(nodename, resp.network_endpoints): - self._register_node(node_id, net_endpoint.ip_address) - else: - ip_add = resp.network_endpoints[0].ip_address - self._register_node(nodename, ip_add) - return True - - def delete_node(self, nodename): - request = tpu.DeleteNodeRequest(name=f"{self._parent}/nodes/{nodename}") - try: - resp = self._client.delete_node(request=request).result() - if resp: - return self.get_node(nodename=nodename) is None - return False - except gExceptions.NotFound: - # log only error if vmcount is 1 as for other tpu vm count, this could be "phantom" nodes - if self.vmcount == 1: - log.error(f"Tpu single node {nodename} not found") - else: - # for the TPU nodes that consist in more than one vm, only the first node of the TPU a.k.a. the master node will - # exist as real TPU nodes, so the other ones are expected to not be found, check the hostname of the node that has - # not been found, and if it ends in 0, it means that is the master node and it should have been found, and in consequence - # log an error - nodehostname = yaml.safe_load( - util.run(f"{self.lkp.scontrol} --yaml show node {nodename}").stdout.rstrip() - )["nodes"][0]["hostname"] - if nodehostname.split("-")[-1] == "0": - log.error(f"TPU master node {nodename} not found") - else: - log.info(f"Deleted TPU 'phantom' node {nodename}") - # If the node is not found it is tecnichally deleted, so return success. - return True - -def _stop_tpu(node: str) -> None: - lkp = util.lookup() - tpuobj = TPU.make(lkp.node_nodeset_name(node), lkp) - if tpuobj.nodeset.preserve_tpu and tpuobj.vmcount == 1: - log.info(f"stopping node {node}") - if tpuobj.stop_node(node): - return - log.error("Error stopping node {node} will delete instead") - log.info(f"deleting node {node}") - if not tpuobj.delete_node(node): - log.error("Error deleting node {node}") - - -def delete_tpu_instances(instances: List[str]) -> None: - util.execute_with_futures(_stop_tpu, instances) - - -def start_tpu(node: List[str]): - lkp = util.lookup() - tpuobj = TPU.make(lkp.node_nodeset_name(node[0]), lkp) - - if len(node) == 1: - node = node[0] - log.debug( - f"Will create a TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node}" - ) - tpunode = tpuobj.get_node(node) - if tpunode is None: - if not tpuobj.create_node(nodename=node): - log.error("Error creating tpu node {node}") - else: - if tpuobj.preserve_tpu: - if not tpuobj.start_node(nodename=node): - log.error("Error starting tpu node {node}") - else: - log.info( - f"Tpu node {node} is already created, but will not start it because nodeset does not have preserve_tpu option active." - ) - else: - log.debug( - f"Will create a multi-vm TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node[0]}" - ) - if not tpuobj.create_node(nodename=node): - log.error("Error creating tpu node {node}") diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py deleted file mode 100644 index 217fd0bca2..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py +++ /dev/null @@ -1,2224 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Iterable, List, Tuple, Optional, Any, Dict, Sequence, Type, Callable, Union -import argparse -import base64 -from dataclasses import dataclass, field -from datetime import timedelta, datetime, timezone -import hashlib -import inspect -import json -import logging -import logging.config -import logging.handlers -import math -import os -import re -import shlex -import shutil -import socket -import subprocess -import sys -from enum import Enum -from collections import defaultdict -from concurrent.futures import ThreadPoolExecutor, as_completed -from contextlib import contextmanager -from functools import lru_cache, reduce, wraps -from itertools import chain, islice -from pathlib import Path -from time import sleep, time - -# TODO: remove "type: ignore" once moved to newer version of libraries -from google.cloud import secretmanager -from google.cloud import storage # type: ignore - -import google.auth # type: ignore -from google.oauth2 import service_account # type: ignore -import googleapiclient.discovery # type: ignore -import google_auth_httplib2 # type: ignore -from googleapiclient.http import set_user_agent # type: ignore -from google.api_core.client_options import ClientOptions -import httplib2 - -import google.api_core.exceptions as gExceptions - -import requests as requests_lib - -import yaml -from addict import Dict as NSDict # type: ignore -import file_cache - -USER_AGENT = "Slurm_GCP_Scripts/1.5 (GPN:SchedMD)" -ENV_CONFIG_YAML = os.getenv("SLURM_CONFIG_YAML") -if ENV_CONFIG_YAML: - CONFIG_FILE = Path(ENV_CONFIG_YAML) -else: - CONFIG_FILE = Path(__file__).with_name("config.yaml") -API_REQ_LIMIT = 2000 - - -def mkdirp(path: Path) -> None: - path.mkdir(parents=True, exist_ok=True) - - -scripts_dir = next( - p for p in (Path(__file__).parent, Path("/slurm/scripts")) if p.is_dir() -) - - -# load all directories as Paths into a dict-like namespace -dirs = NSDict( - home = Path("/home"), - apps = Path("/opt/apps"), - slurm = Path("/slurm"), - scripts = scripts_dir, - custom_scripts = Path("/slurm/custom_scripts"), - munge = Path("/etc/munge"), - secdisk = Path("/mnt/disks/sec"), - log = Path("/var/log/slurm"), - slurm_bucket_mount = Path("/slurm/bucket"), -) - -slurmdirs = NSDict( - prefix = Path("/usr/local"), - etc = Path("/usr/local/etc/slurm"), - state = Path("/var/spool/slurm"), - key_distribution = Path("/slurm/key_distribution"), -) - - -# TODO: Remove this hack (relies on undocumented behavior of PyYAML) -# No need to represent NSDict and Path once we move to properly typed & serializable config. -yaml.SafeDumper.yaml_representers[ - None # type: ignore -] = lambda self, data: yaml.representer.SafeRepresenter.represent_str(self, str(data)) # type: ignore - - -class ApiEndpoint(Enum): - COMPUTE = "compute" - BQ = "bq" - STORAGE = "storage" - TPU = "tpu" - SECRET = "secret_manager" - - -@dataclass(frozen=True) -class AcceleratorInfo: - type: str - count: int - - @classmethod - def from_json(cls, jo: dict) -> "AcceleratorInfo": - return cls( - type=jo["guestAcceleratorType"], - count=jo["guestAcceleratorCount"]) - -@dataclass(frozen=True) -class MachineType: - name: str - guest_cpus: int - memory_mb: int - accelerators: List[AcceleratorInfo] - - @classmethod - def from_json(cls, jo: dict) -> "MachineType": - return cls( - name=jo["name"], - guest_cpus=jo["guestCpus"], - memory_mb=jo["memoryMb"], - accelerators=[ - AcceleratorInfo.from_json(a) for a in jo.get("accelerators", [])], - ) - - @property - def family(self) -> str: - # TODO: doesn't work with N1 custom machine types - # See https://cloud.google.com/compute/docs/instances/creating-instance-with-custom-machine-type#create - return self.name.split("-")[0] - - @property - def supports_smt(self) -> bool: - # https://cloud.google.com/compute/docs/cpu-platforms - if self.family in ("t2a", "t2d", "h3", "c4a", "h4d",): - return False - if self.guest_cpus == 1: - return False - return True - - @property - def sockets(self) -> int: - return { - "h3": 2, - "h4d": 2, - "c2d": 2 if self.guest_cpus > 56 else 1, - "a3": 2, - "c2": 2 if self.guest_cpus > 30 else 1, - "c3": 2 if self.guest_cpus > 88 else 1, - "c3d": 2 if self.guest_cpus > 180 else 1, - "c4": 2 if self.guest_cpus > 96 else 1, - "c4d": 2 if self.guest_cpus > 192 else 1, - }.get( - self.family, - 1, # assume 1 socket for all other families - ) - - -@dataclass(frozen=True) -class UpcomingMaintenance: - window_start_time: datetime - - @classmethod - def from_json(cls, jo: Optional[dict]) -> Optional["UpcomingMaintenance"]: - if jo is None: - return None - try: - if "windowStartTime" in jo: - ts = parse_gcp_timestamp(jo["windowStartTime"]) - elif "startTimeWindow" in jo: - ts = parse_gcp_timestamp(jo["startTimeWindow"]["earliest"]) - else: - raise Exception("Neither windowStartTime nor startTimeWindow are found") - except BaseException as e: - raise ValueError(f"Unexpected format for upcomingMaintenance: {jo}") from e - return cls(window_start_time=ts) - -@dataclass(frozen=True) -class InstanceResourceStatus: - physical_host: Optional[str] - upcoming_maintenance: Optional[UpcomingMaintenance] - - @classmethod - def from_json(cls, jo: Optional[dict]) -> "InstanceResourceStatus": - if not jo: - return cls( - physical_host=None, - upcoming_maintenance=None, - ) - - try: - maint = UpcomingMaintenance.from_json(jo.get("upcomingMaintenance")) - except ValueError as e: - log.exception("Failed to parse upcomingMaintenance, ignoring") - maint = None # intentionally swallow exception - - return cls( - physical_host=jo.get("physicalHost"), - upcoming_maintenance=maint, - ) - - -@dataclass(frozen=True) -class Instance: - name: str - zone: str - status: str - creation_timestamp: datetime - role: Optional[str] - resource_status: InstanceResourceStatus - metadata: Dict[str, str] - # TODO: use proper InstanceScheduling class - scheduling: NSDict - - @classmethod - def from_json(cls, jo: dict) -> "Instance": - return cls( - name=jo["name"], - zone=trim_self_link(jo["zone"]), - status=jo["status"], - creation_timestamp=parse_gcp_timestamp(jo["creationTimestamp"]), - resource_status=InstanceResourceStatus.from_json(jo.get("resourceStatus")), - scheduling=NSDict(jo.get("scheduling")), - role = jo.get("labels", {}).get("slurm_instance_role"), - metadata = {k["key"]: k["value"] for k in jo.get("metadata", {}).get("items", [])} - ) - - -@dataclass(frozen=True) -class NSMount: - server_ip: str - local_mount: Path - remote_mount: Path - fs_type: str - mount_options: str - -@lru_cache(maxsize=1) -def default_credentials(): - return google.auth.default()[0] - - -@lru_cache(maxsize=1) -def authentication_project(): - return google.auth.default()[1] - - -DEFAULT_UNIVERSE_DOMAIN = "googleapis.com" - - -def now() -> datetime: - """ - Return current time as timezone-aware datetime. - - IMPORTANT: DO NOT use `datetime.now()`, unless you explicitly need to have tz-naive datetime. - Otherwise there is a risk of getting: "cannot compare naive and aware datetimes" error, - since all timetstamps we receive from GCP API are tz-aware. - - Another motivation for this function is to allow to mock time in tests. - """ - return datetime.now(timezone.utc) - -def parse_gcp_timestamp(s: str) -> datetime: - """ - Parse timestamp strings returned by GCP API into datetime. - Works with both Zulu and non-Zulu timestamps. - NOTE: It always return tz-aware datetime (fallbacks to UTC and logs error). - """ - # Requires Python >= 3.7 - # TODO: Remove this "hack" of trimming the Z from timestamps once we move to Python 3.11 - # (context: https://discuss.python.org/t/parse-z-timezone-suffix-in-datetime/2220/30) - ts = datetime.fromisoformat(s.replace('Z', '+00:00')) - if ts.tzinfo is None: # fallback to UTC - log.error(f"Received timestamp without timezone info: {s}") - ts = ts.replace(tzinfo=timezone.utc) - return ts - - -def universe_domain() -> str: - try: - return instance_metadata("attributes/universe_domain") - except MetadataNotFoundError: - return DEFAULT_UNIVERSE_DOMAIN - - -def endpoint_version(api: ApiEndpoint) -> Optional[str]: - return lookup().endpoint_versions.get(api.value, None) - - -@lru_cache(maxsize=1) -def get_credentials() -> Optional[service_account.Credentials]: - """Get credentials for service account""" - key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") - if key_path is not None: - credentials = service_account.Credentials.from_service_account_file( - key_path, scopes=[f"https://www.{universe_domain()}/auth/cloud-platform"] - ) - else: - credentials = default_credentials() - - return credentials - - -@lru_cache(maxsize=1) -def get_dev_key() -> Optional[str]: - """Get dev key for project (uses json or yaml format)""" - try: - with open("/etc/slurm/slurm_vars.yaml", 'r') as file: - data = yaml.safe_load(file) - return data['google_developer_key'] - except: - return None - - -def create_client_options(api: ApiEndpoint) -> ClientOptions: - """Create client options for cloud endpoints""" - ver = endpoint_version(api) - ud = universe_domain() - options = {} - if ud and ud != DEFAULT_UNIVERSE_DOMAIN: - options["universe_domain"] = ud - if ver: - options["api_endpoint"] = f"https://{api.value}.{ud}/{ver}/" - co = ClientOptions(**options) - log.debug(f"Using ClientOptions = {co} for API: {api.value}") - return co - -log = logging.getLogger() - - -def access_secret_version(project_id, secret_id, version_id="latest"): - """ - Access the payload for the given secret version if one exists. The version - can be a version number as a string (e.g. "5") or an alias (e.g. "latest"). - """ - co = create_client_options(ApiEndpoint.SECRET) - client = secretmanager.SecretManagerServiceClient(client_options=co) - name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" - try: - response = client.access_secret_version(request={"name": name}) - log.debug(f"Secret '{name}' was found.") - payload = response.payload.data.decode("UTF-8") - except gExceptions.NotFound: - log.debug(f"Secret '{name}' was not found!") - payload = None - - return payload - - -def parse_self_link(self_link: str): - """Parse a selfLink url, extracting all useful values - https://.../v1/projects//regions//... - {'project': , 'region': , ...} - can also extract zone, instance (name), image, etc - """ - link_patt = re.compile(r"(?P[^\/\s]+)s\/(?P[^\s\/]+)") - return NSDict(link_patt.findall(self_link)) - - -def parse_bucket_uri(uri: str): - """ - Parse a bucket url - E.g. gs:/// - """ - pattern = re.compile(r"gs://(?P[^/\s]+)/(?P([^/\s]+)(/[^/\s]+)*)") - matches = pattern.match(uri) - assert matches, f"Unexpected bucker URI: '{uri}'" - return matches.group("bucket"), matches.group("path") - - -def get_template_gpu(template): - """get gpu info from machine type or guest accelerators""" - gpu_keyword = "nvidia" - gpu = None - if template.machine_type.accelerators: - tma = template.machine_type.accelerators[0] - if gpu_keyword in tma.type.lower(): - gpu = tma - elif template.guestAccelerators: - tga = template.guestAccelerators[0] - if gpu_keyword in tga.acceleratorType.lower(): - gpu = AcceleratorInfo( - type=tga.acceleratorType, - count=tga.acceleratorCount) - return gpu - - -def trim_self_link(link: str): - """get resource name from self link url, eg. - https://.../v1/projects//regions/ - -> - """ - try: - return link[link.rindex("/") + 1 :] - except ValueError: - raise Exception(f"'/' not found, not a self link: '{link}' ") - - -def get_self_link_component(link: str, component_name: str): - """ - Extracts a component (e.g., 'region', 'project') from a self-link URL. - Args: - link: The self-link URL string. - component_name: The name of the component to extract (e.g., 'regions', 'projects'). - Returns: - The extracted component value (e.g., '', ''), - or None if the component is not found in the link. - """ - search_string = f"/{component_name}/" - start_index = link.rfind(search_string) - - if start_index == -1: - return None - - start_index += len(search_string) - end_index = link.find("/", start_index) - - if end_index == -1: - # If no further slash, the rest of the string is the component - return link[start_index:] - else: - return link[start_index:end_index] - - -def execute_with_futures(func, seq): - with ThreadPoolExecutor() as exe: - futures = [] - for i in seq: - future = exe.submit(func, i) - futures.append(future) - for future in as_completed(futures): - result = future.exception() - if result is not None: - raise result - - -def map_with_futures(func, seq): - with ThreadPoolExecutor() as exe: - futures = [] - for i in seq: - future = exe.submit(func, i) - futures.append(future) - for future in futures: - # Will be result or raise Exception - res = None - try: - res = future.result() - except Exception as e: - res = e - yield res - -def should_mount_slurm_bucket() -> bool: - try: - return instance_metadata("attributes/slurm_bucket_mount", silent=True).lower() == "true" - except MetadataNotFoundError: - return False - - -def _get_bucket_and_common_prefix() -> Tuple[str, str]: - uri = instance_metadata("attributes/slurm_bucket_path") - return parse_bucket_uri(uri) - -def blob_get(file): - bucket_name, path = _get_bucket_and_common_prefix() - blob_name = f"{path}/{file}" - return storage_client().get_bucket(bucket_name).blob(blob_name) - - -def blob_list(prefix="", delimiter=None): - bucket_name, path = _get_bucket_and_common_prefix() - blob_prefix = f"{path}/{prefix}" - # Note: The call returns a response only when the iterator is consumed. - blobs = storage_client().list_blobs( - bucket_name, prefix=blob_prefix, delimiter=delimiter - ) - return [blob for blob in blobs] - -def file_list(prefix="", subpath="") -> List[os.DirEntry]: - path = dirs.slurm_bucket_mount - file_prefix = f"{path}/{subpath}" - try: - files = os.scandir(file_prefix) - return [file for file in files if file.name.startswith(prefix)] - except: - return [] - # Not considering lack of file's existence as fatal (we may check for files we know don't exist). - # Responsibility of callee to determine if it is fatal or not, blob_list returns empty iterator in similar cases. - -def hash_file(fullpath: Path) -> str: - with open(fullpath, "rb") as f: - file_hash = hashlib.md5() - chunk = f.read(8192) - while chunk: - file_hash.update(chunk) - chunk = f.read(8192) - return base64.b64encode(file_hash.digest()).decode("utf-8") - - -def install_custom_scripts(check_hash:bool=False): - """download custom scripts from gcs bucket""" - role, tokens = lookup().instance_role, [] - - mounted_scripts=False - if should_mount_slurm_bucket() and role != "controller": - mounted_scripts=True - - all_prolog_tokens = ["prolog", "epilog", "task_prolog", "task_epilog"] - if role == "controller": - tokens = ["controller"] + all_prolog_tokens - elif role == "compute": - tokens = [f"nodeset-{lookup().node_nodeset_name()}"] + all_prolog_tokens - elif role == "login": - tokens = [f"login-{instance_login_group()}"] - - prefixes = [f"slurm-{tok}-script" for tok in tokens] - - # TODO: use single `blob_list`, to reduce ~4x number of GCS requests - if mounted_scripts: - source_collection = list(chain.from_iterable(file_list(prefix=p) for p in prefixes)) - else: - source_collection = list(chain.from_iterable(blob_list(prefix=p) for p in prefixes)) - - script_pattern = re.compile(r"^slurm-(?P\S+)-script-(?P\S+)") - for source in source_collection: - if mounted_scripts: - m = script_pattern.match(source.name) - else: - m = script_pattern.match(Path(source.name).name) - - if not m: - log.warning(f"found blob that doesn't match expected pattern: {source.name}") - continue - path_parts = m["path"].split("-") - path_parts[0] += ".d" - stem, _, ext = m["name"].rpartition("_") - filename = ".".join((stem, ext)) - - path = Path(*path_parts, filename) - fullpath = (dirs.custom_scripts / path).resolve() - mkdirp(fullpath.parent) - - for par in path.parents: - chown_slurm(dirs.custom_scripts / par) - need_update = True - - if check_hash and fullpath.exists() and isinstance(source,storage.Blob): - # TODO: MD5 reported by gcloud may differ from the one calculated here (e.g. if blob got gzipped), - # consider using gCRC32C - need_update = hash_file(fullpath) != source.md5_hash - - log.info(f"installing custom script: {path} from {source.name}") - - if isinstance(source,os.DirEntry): - shutil.copy(source.path, fullpath) #Needs to be copied since mounted nfs is read-only - chown_slurm(fullpath, mode=0o755) - - elif need_update: - with fullpath.open("wb") as f: - source.download_to_file(f) - chown_slurm(fullpath, mode=0o755) - -def compute_service(version="beta"): - """Make thread-safe compute service handle - creates a new Http for each request - """ - credentials = get_credentials() - dev_key = get_dev_key() - - def build_request(http, *args, **kwargs): - new_http = set_user_agent(httplib2.Http(), USER_AGENT) - if credentials is not None: - new_http = google_auth_httplib2.AuthorizedHttp(credentials, http=new_http) - return googleapiclient.http.HttpRequest(new_http, *args, **kwargs) - - ver = endpoint_version(ApiEndpoint.COMPUTE) - disc_url = googleapiclient.discovery.DISCOVERY_URI - if ver: - version = ver - disc_url = disc_url.replace(DEFAULT_UNIVERSE_DOMAIN, universe_domain()) - - log.debug(f"Using version={version} of Google Compute Engine API") - return googleapiclient.discovery.build( - "compute", - version, - requestBuilder=build_request, - credentials=credentials, - developerKey=dev_key, - discoveryServiceUrl=disc_url, - cache_discovery=False, # See https://github.com/googleapis/google-api-python-client/issues/299 - ) - -def storage_client() -> storage.Client: - """ - Config-independent storage client - """ - ud = universe_domain() - co = {} - if ud and ud != DEFAULT_UNIVERSE_DOMAIN: - co["universe_domain"] = ud - return storage.Client(client_options=ClientOptions(**co)) - - -class DeffetiveStoredConfigError(Exception): - """ - Raised when config can not be loaded and assembled from bucket - """ - pass - - -def _fill_cfg_defaults(cfg: NSDict) -> NSDict: - if not cfg.slurm_log_dir: - cfg.slurm_log_dir = dirs.log - if not cfg.slurm_bin_dir: - cfg.slurm_bin_dir = slurmdirs.prefix / "bin" - if not cfg.slurm_control_host: - try: - control_dns_name = instance_metadata("attributes/slurm_control_dns", silent=True) - cfg.slurm_control_host = control_dns_name - except MetadataNotFoundError: - cfg.slurm_control_host = f"{cfg.slurm_cluster_name}-controller" - if not cfg.slurm_control_host_port: - cfg.slurm_control_host_port = "6820-6830" - return cfg - -@dataclass -class _ConfigBlobs: - """ - "Private" class that represent a collection of GCS blobs for configuration - """ - core: storage.Blob - controller_addr: Optional[storage.Blob] - partition: List[storage.Blob] = field(default_factory=list) - nodeset: List[storage.Blob] = field(default_factory=list) - nodeset_dyn: List[storage.Blob] = field(default_factory=list) - nodeset_tpu: List[storage.Blob] = field(default_factory=list) - login_group: List[storage.Blob] = field(default_factory=list) - - @property - def hash(self) -> str: - h = hashlib.md5() - all = [self.core] + self.partition + self.nodeset + self.nodeset_dyn + self.nodeset_tpu - if self.controller_addr: - all.append(self.controller_addr) - - # sort blobs so hash is consistent - for blob in sorted(all, key=lambda b: b.name): - h.update(blob.md5_hash.encode("utf-8")) - return h.hexdigest() - -@dataclass -class _ConfigFiles: - """ - "Private" class that represent a collection of files for configuration - """ - core: Path - controller_addr: Optional[Path] - partition: List[Path] = field(default_factory=list) - nodeset: List[Path] = field(default_factory=list) - nodeset_dyn: List[Path] = field(default_factory=list) - nodeset_tpu: List[Path] = field(default_factory=list) - login_group: List[Path] = field(default_factory=list) - -def _list_config_blobs() -> _ConfigBlobs: - _, common_prefix = _get_bucket_and_common_prefix() - - core: Optional[storage.Blob] = None - controller_addr: Optional[storage.Blob] = None - rest: Dict[str, List[storage.Blob]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} - - is_controller = instance_role() == "controller" - - for blob in blob_list(prefix=""): - if blob.name == f"{common_prefix}/config.yaml": - core = blob - if blob.name == f"{common_prefix}/controller_addr.yaml" and not is_controller: - # Don't add this config blobs for controller to avoid "double reconfiguration": - # Initially this file doesn't exist and produce later by `setup_controller`; - # Appearance of this blob would trigger change in combined hash of config files; - # Ignore existence of this file for controller, assume that - # no other instance nodes will proceed with configuration until this file is created. - controller_addr = blob - for key in rest.keys(): - if blob.name.startswith(f"{common_prefix}/{key}_configs/"): - rest[key].append(blob) - - if core is None: - raise DeffetiveStoredConfigError(f"{common_prefix}/config.yaml not found in bucket") - - return _ConfigBlobs(core=core, controller_addr=controller_addr, **rest) - -def _list_config_files() -> _ConfigFiles: - file_dir = dirs.slurm_bucket_mount - core: Optional[Path] = None - controller_addr: Optional[Path] = None - rest: Dict[str, List[Path]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} - - if Path(f"{file_dir}/config.yaml").exists(): - core = Path(f"{file_dir}/config.yaml") - - for key in rest.keys(): - for f in file_list(subpath=f"{key}_configs"): - rest[key].append(f.path) - - if core is None: - raise Exception(f"config.yaml was not found in mounted folder: {dirs.slurm_bucket_mount}") #Intentionally not using DeffetiveStoredConfigError as this is considered a fatal error - - return _ConfigFiles(core=core, controller_addr=None, **rest) - -def _fetch_config(old_hash: Optional[str]) -> Optional[Tuple[NSDict, str]]: - """Fetch config from bucket, returns None if no changes are detected.""" - blobs = _list_config_blobs() - if old_hash == blobs.hash: - return None - - def _download(bs) -> List[Any]: - return [yaml.safe_load(b.download_as_text()) for b in bs] - - return _assemble_config( - core=_download([blobs.core])[0], - controller_addr=_download([blobs.controller_addr])[0] if blobs.controller_addr else None, - partitions=_download(blobs.partition), - nodesets=_download(blobs.nodeset), - nodesets_dyn=_download(blobs.nodeset_dyn), - nodesets_tpu=_download(blobs.nodeset_tpu), - login_groups=_download(blobs.login_group), - ), blobs.hash - -def _fetch_mounted_config() -> Optional[Tuple[NSDict, str]]: - if not dirs.slurm_bucket_mount.is_mount(): - raise Exception(f"{dirs.slurm_bucket_mount} is not mounted") - - files = _list_config_files() - - def _load(files) -> List[Any]: - file_yaml=[] - for file in files: - with open(file, "r") as f: - file_yaml.append(yaml.safe_load(f)) - return file_yaml - - return _assemble_config( - core=_load([files.core])[0], - controller_addr=None, - partitions=_load(files.partition), - nodesets=_load(files.nodeset), - nodesets_dyn=_load(files.nodeset_dyn), - nodesets_tpu=_load(files.nodeset_tpu), - login_groups=_load(files.login_group), - ) - -def controller_lookup_self_ip() -> str: - assert instance_role() == "controller" - # Get IP of LAST network-interface - # TODO: Consider change order of NICs definition, so right NIC is always @0. - idx = instance_metadata("network-interfaces").split()[-1] # either `0/` or `1/` - return instance_metadata(f"network-interfaces/{idx}ip") - -def _assemble_config( - core: Any, - controller_addr: Optional[Any], - partitions: List[Any], - nodesets: List[Any], - nodesets_dyn: List[Any], - nodesets_tpu: List[Any], - login_groups: List[Any], - ) -> NSDict: - cfg = NSDict(core) - - if cfg.controller_network_attachment: - # lookup controller address - if instance_role() == "controller": - # ignore stored value of `controller_addr`, it will be overwritten during `setup_controller` - cfg.slurm_control_addr = controller_lookup_self_ip() - else: - if not controller_addr: - raise DeffetiveStoredConfigError("controller_addr.yaml not found in bucket") - cfg.slurm_control_addr = controller_addr["slurm_control_addr"] - - # add partition configs - for p_yaml in partitions: - p_cfg = NSDict(p_yaml) - assert p_cfg.get("partition_name"), "partition_name is required" - p_name = p_cfg.partition_name - assert p_name not in cfg.partitions, f"partition {p_name} already defined" - cfg.partitions[p_name] = p_cfg - - # add nodeset configs - ns_names = set() - def _add_nodesets(yamls: List[Any], target: dict): - for ns_yaml in yamls: - ns_cfg = NSDict(ns_yaml) - assert ns_cfg.get("nodeset_name"), "nodeset_name is required" - ns_name = ns_cfg.nodeset_name - assert ns_name not in ns_names, f"nodeset {ns_name} already defined" - target[ns_name] = ns_cfg - ns_names.add(ns_name) - - _add_nodesets(nodesets, cfg.nodeset) - _add_nodesets(nodesets_dyn, cfg.nodeset_dyn) - _add_nodesets(nodesets_tpu, cfg.nodeset_tpu) - - # validate that configs for all referenced nodesets are present - for p in cfg.partitions.values(): - for ns_name in chain(p.partition_nodeset, p.partition_nodeset_dyn, p.partition_nodeset_tpu): - if ns_name not in ns_names: - raise DeffetiveStoredConfigError(f"nodeset {ns_name} not defined in config") - - for lg_yaml in login_groups: - lg_cfg = NSDict(lg_yaml) - assert lg_cfg.get("group_name"), "group_name is required" - lg_name = lg_cfg.group_name - assert lg_name not in cfg.login_groups - cfg.login_groups[lg_name] = lg_cfg - - if instance_role() == "login": - group = instance_login_group() - if group not in cfg.login_groups: - raise DeffetiveStoredConfigError(f"login group '{group}' does not exist in config") - - return _fill_cfg_defaults(cfg) - -def fetch_config() -> Tuple[bool, NSDict]: - """ - Fetches config from bucket and saves it locally - Returns True if new (updated) config was fetched - """ - hash_file = Path("/slurm/scripts/.config.hash") - old_hash = hash_file.read_text() if hash_file.exists() else None - - if should_mount_slurm_bucket() and instance_role() != "controller": - cfg = _fetch_mounted_config() - CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) - chown_slurm(CONFIG_FILE) - return False, cfg - - cfg_and_hash = _fetch_config(old_hash=old_hash) - - if not cfg_and_hash: - return False, _load_config() - - cfg, hash = cfg_and_hash - hash_file.write_text(hash) - chown_slurm(hash_file) - CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) - chown_slurm(CONFIG_FILE) - return True, cfg - -def owned_file_handler(filename): - """create file handler""" - chown_slurm(filename) - return logging.handlers.WatchedFileHandler(filename, delay=True) - -def get_log_path() -> Path: - """ - Returns path to log file for the current script. - e.g. resume.py -> /var/log/slurm/resume.log - """ - cfg_log_dir = lookup().cfg.slurm_log_dir - log_dir = Path(cfg_log_dir) if cfg_log_dir else dirs.log - return (log_dir / Path(sys.argv[0]).name).with_suffix(".log") - -def init_log_and_parse(parser: argparse.ArgumentParser) -> argparse.Namespace: - parser.add_argument( - "--debug", - "-d", - dest="loglevel", - action="store_const", - const=logging.DEBUG, - default=logging.INFO, - help="Enable debugging output", - ) - parser.add_argument( - "--trace-api", - "-t", - action="store_true", - help="Enable detailed api request output", - ) - args = parser.parse_args() - loglevel = args.loglevel - if lookup().cfg.enable_debug_logging: - loglevel = logging.DEBUG - if args.trace_api: - lookup().cfg.extra_logging_flags["trace_api"] = True - # Configure root logger - logging.config.dictConfig({ - "version": 1, - "disable_existing_loggers": True, - "formatters": { - "standard": { - "format": "%(levelname)s: %(message)s", - }, - "stamp": { - "format": "%(asctime)s %(levelname)s: %(message)s", - }, - }, - "handlers": { - "stdout_handler": { - "level": logging.DEBUG, - "formatter": "standard", - "class": "logging.StreamHandler", - "stream": sys.stdout, - }, - "file_handler": { - "()": owned_file_handler, - "level": logging.DEBUG, - "formatter": "stamp", - "filename": get_log_path(), - }, - }, - "root": { - "handlers": ["stdout_handler", "file_handler"], - "level": loglevel, - }, - }) - - sys.excepthook = _handle_exception - - return args - - -def log_api_request(request): - """log.trace info about a compute API request""" - if not lookup().cfg.extra_logging_flags.get("trace_api"): - return - # output the whole request object as pretty yaml - # the body is nested json, so load it as well - rep = json.loads(request.to_json()) - if rep.get("body", None) is not None: - rep["body"] = json.loads(rep["body"]) - pretty_req = yaml.safe_dump(rep).rstrip() - # label log message with the calling function - log.debug(f"{inspect.stack()[1].function}:\n{pretty_req}") - - -def _handle_exception(exc_type, exc_value, exc_trace): - """log exceptions other than KeyboardInterrupt""" - if not issubclass(exc_type, KeyboardInterrupt): - log.exception("Fatal exception", exc_info=(exc_type, exc_value, exc_trace)) - sys.__excepthook__(exc_type, exc_value, exc_trace) - - -def run( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False, - timeout=None, - check=True, - universal_newlines=True, - **kwargs, -): - """Wrapper for subprocess.run() with convenient defaults""" - if isinstance(args, list): - args = list(filter(lambda x: x is not None, args)) - args = " ".join(args) - if not shell and isinstance(args, str): - args = shlex.split(args) - log.debug(f"run: {args}") - try: - result = subprocess.run( - args, - stdout=stdout, - stderr=stderr, - shell=shell, - timeout=timeout, - check=check, - universal_newlines=universal_newlines, - **kwargs, - ) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - log_subprocess(e) - raise - log_subprocess(result) - return result - -def log_subprocess(subj: subprocess.CalledProcessError | subprocess.TimeoutExpired | subprocess.CompletedProcess) -> None: - match subj: - case subprocess.CompletedProcess(returncode=0): - # Do not log successful runs, to not overwhelm logs (e.g. scontrol show jobs --json) - # TODO: consider still doing it in DEBUG or trim output to few KBs. - return - case subprocess.CompletedProcess(): # non-zero returncode - log.error(f"Command '{subj.args}' returned exit status {subj.returncode}.") - case subprocess.CalledProcessError() | subprocess.TimeoutExpired(): - log.error(str(subj)) - - - def normalize(out: None | str | bytes) -> None | str: - """ - Turns stderr and stdout into string: - > A bytes sequence, or a string if run() was called with an encoding, errors, or text=True. None if was not captured. - """ - match out: - case None: - return None - case str(): - return out.strip() - case bytes(): - return out.decode().strip() - case _: - return repr(out) - - if stdout := normalize(subj.stdout): - log.error(f"stdout: {stdout}") - if stderr := normalize(subj.stderr): - log.error(f"stderr: {stderr}") - - -def chown_slurm(path: Path, mode=None) -> None: - if path.exists(): - if mode: - path.chmod(mode) - else: - mkdirp(path.parent) - if mode: - path.touch(mode=mode) - else: - path.touch() - try: - shutil.chown(path, user="slurm", group="slurm") - except LookupError: - log.warning(f"User 'slurm' does not exist. Cannot 'chown slurm:slurm {path}'.") - except PermissionError: - log.warning(f"Not authorized to 'chown slurm:slurm {path}'.") - except Exception as err: - log.error(err) - - -@contextmanager -def cd(path): - """Change working directory for context""" - prev = Path.cwd() - os.chdir(path) - try: - yield - finally: - os.chdir(prev) - - -def cached_property(f): - return property(lru_cache()(f)) - - -def retry(max_retries: int, init_wait_time: float, warn_msg: str, exc_type: Type[Exception]): - """Retries functions that raises the exception exc_type. - Retry time is increased by a factor of two for every iteration. - - Args: - max_retries (int): Maximum number of retries - init_wait_time (float): Initial wait time in secs - warn_msg (str): Message to print during retries - exc_type (Exception): Exception type to check for - """ - - if max_retries <= 0: - raise ValueError("Incorrect value for max_retries, must be >= 1") - if init_wait_time <= 0.0: - raise ValueError("Invalid value for init_wait_time, must be > 0.0") - - def decorator(f): - @wraps(f) - def wrapper(*args, **kwargs): - retry = 0 - secs = init_wait_time - captured_exc: Optional[BaseException] = None - while retry < max_retries: - try: - return f(*args, **kwargs) - except exc_type as e: - captured_exc = e - log.warn(f"{warn_msg}, retrying in {secs}") - sleep(secs) - retry += 1 - secs *= 2 - assert captured_exc - raise captured_exc - - return wrapper - - return decorator - - -def separate(pred: Callable[[Any], bool], coll: Iterable[Any]) -> Tuple[List[Any], List[Any]]: - """filter into 2 lists based on pred returning True or False - returns ([False], [True]) - """ - res: Tuple[List[Any], List[Any]] = ([],[]) - for el in coll: - res[pred(el)].append(el) - return res - - -def chunked(iterable, n=API_REQ_LIMIT): - """group iterator into chunks of max size n""" - it = iter(iterable) - while True: - chunk = list(islice(it, n)) - if not chunk: - return - yield chunk - -def groupby_unsorted(seq: Sequence[Any], key): - indices = defaultdict(list) - for i, el in enumerate(seq): - indices[key(el)].append(i) - for k, idxs in indices.items(): - yield k, (seq[i] for i in idxs) - - -@lru_cache(maxsize=32) -def find_ratio(a, n, s, r0=None): - """given the start (a), count (n), and sum (s), find the ratio required""" - if n == 2: - return s / a - 1 - an = a * n - if n == 1 or s == an: - return 1 - if r0 is None: - # we only need to know which side of 1 to guess, and the iteration will work - r0 = 1.1 if an < s else 0.9 - - # geometric sum formula - def f(r): - return a * (1 - r**n) / (1 - r) - s - - # derivative of f - def df(r): - rm1 = r - 1 - rn = r**n - return (a * (rn * (n * rm1 - r) + r)) / (r * rm1**2) - - MIN_DR = 0.0001 # negligible change - r = r0 - # print(f"r(0)={r0}") - MAX_TRIES = 64 - for i in range(1, MAX_TRIES + 1): - try: - dr = f(r) / df(r) - except ZeroDivisionError: - log.error(f"Failed to find ratio due to zero division! Returning r={r0}") - return r0 - r = r - dr - # print(f"r({i})={r}") - # if the change in r is small, we are close enough - if abs(dr) < MIN_DR: - break - else: - log.error(f"Could not find ratio after {MAX_TRIES}! Returning r={r0}") - return r0 - return r - - -def backoff_delay(start, timeout=None, ratio=None, count: int = 0): - """generates `count` waits starting at `start` - sum of waits is `timeout` or each one is `ratio` bigger than the last - the last wait is always 0""" - # timeout or ratio must be set but not both - assert (timeout is None) ^ (ratio is None) - assert ratio is None or ratio > 0 - assert timeout is None or timeout >= start - assert (count > 1 or timeout is not None) and isinstance(count, int) - assert start > 0 - - if count == 0: - # Equation for auto-count is tuned to have a max of - # ~int(timeout) counts with a start wait of <0.01. - # Increasing start wait decreases count eg. - # backoff_delay(10, timeout=60) -> count = 5 - count = int( - (timeout / ((start + 0.05) ** (1 / 2)) + 2) // math.log(timeout + 2) - ) - - yield start - # if ratio is set: - # timeout = start * (1 - ratio**(count - 1)) / (1 - ratio) - if ratio is None: - ratio = find_ratio(start, count - 1, timeout) - - wait = start - # we have start and 0, so we only need to generate count - 2 - for _ in range(count - 2): - wait *= ratio - yield wait - yield 0 - return - - -ROOT_URL = "http://metadata.google.internal/computeMetadata/v1" - -class MetadataNotFoundError(Exception): - pass - -def get_metadata(path:str, silent=False) -> str: - """Get metadata relative to metadata/computeMetadata/v1""" - HEADERS = {"Metadata-Flavor": "Google"} - url = f"{ROOT_URL}/{path}" - try: - resp = requests_lib.get(url, headers=HEADERS) - resp.raise_for_status() - return resp.text - except requests_lib.exceptions.HTTPError: - if not silent: - log.warning(f"metadata not found ({url})") - raise MetadataNotFoundError(f"failed to get_metadata from {url}") - - -@lru_cache(maxsize=None) -def instance_metadata(path: str, silent:bool=False) -> str: - return get_metadata(f"instance/{path}", silent=silent) - -def instance_role(): - return instance_metadata("attributes/slurm_instance_role") - - -def instance_login_group(): - return instance_metadata("attributes/slurm_login_group") - - -def natural_sort(text): - def atoi(text): - return int(text) if text.isdigit() else text - - return [atoi(w) for w in re.split(r"(\d+)", text)] - - -def to_hostlist(names: Iterable[str]) -> str: - """ - Fast implementation of `hostlist` that doesn't invoke `scontrol` - IMPORTANT: - * Acts as `scontrol show hostlistsorted`, i.e. original order is not preserved - * Achieves worse compression than `scontrol show hostlist` for some cases - """ - pref = defaultdict(list) - tokenizer = re.compile(r"^(.*?)(\d*)$") - for name in filter(None, names): - matches = tokenizer.match(name) - assert matches, name - p, s = matches.groups() - pref[p].append(s) - - def _compress_suffixes(ss: List[str]) -> List[str]: - cur, res = None, [] - - def cur_repr(): - assert cur - nums, strs = cur - if nums[0] == nums[1]: - return strs[0] - return f"{strs[0]}-{strs[1]}" - - for s in sorted(ss, key=int): - n = int(s) - if cur is None: - cur = ((n, n), (s, s)) - continue - - nums, strs = cur - if n == nums[1] + 1: - cur = ((nums[0], n), (strs[0], s)) - else: - res.append(cur_repr()) - cur = ((n, n), (s, s)) - if cur: - res.append(cur_repr()) - return res - - res = [] - for p in sorted(pref.keys()): - sl = defaultdict(list) - for s in pref[p]: - sl[len(s)].append(s) - cs = [] - for ln in sorted(sl.keys()): - if ln == 0: - res.append(p) - else: - cs.extend(_compress_suffixes(sl[ln])) - if not cs: - continue - if len(cs) == 1 and "-" not in cs[0]: - res.append(f"{p}{cs[0]}") - else: - res.append(f"{p}[{','.join(cs)}]") - return ",".join(res) - -@lru_cache(maxsize=None) -def to_hostnames(nodelist: str) -> List[str]: - """make list of hostnames from hostlist expression""" - if not nodelist: - return [] # avoid degenerate invocation of scontrol - if isinstance(nodelist, str): - hostlist = nodelist - else: - hostlist = ",".join(nodelist) - hostnames = run(f"{lookup().scontrol} show hostnames {hostlist}").stdout.splitlines() - return hostnames - - -def retry_exception(exc) -> bool: - """return true for exceptions that should always be retried""" - msg = str(exc) - retry_errors = ( - "Rate Limit Exceeded", - "Quota Exceeded", - "Quota exceeded", - ) - return any(err in msg for err in retry_errors) - - -def ensure_execute(request): - """Handle rate limits and socket time outs""" - - for retry, wait in enumerate(backoff_delay(0.5, timeout=10 * 60, count=20)): - try: - return request.execute() - except googleapiclient.errors.HttpError as e: - if retry_exception(e): - log.error(f"retry:{retry} '{e}'") - sleep(wait) - continue - raise - - except socket.timeout as e: - # socket timed out, try again - log.debug(e) - - except Exception as e: - log.error(e, exc_info=True) - raise - - break - - -def batch_execute(requests, retry_cb=None, log_err=log.error): - """execute list or dict as batch requests - retry if retry_cb returns true - """ - BATCH_LIMIT = 1000 - if not isinstance(requests, dict): - requests = {str(k): v for k, v in enumerate(requests)} # rid generated here - done = {} - failed = {} - timestamps: List[float] = [] - rate_limited = False - - def batch_callback(rid, resp, exc): - nonlocal rate_limited - if exc is not None: - log_err(f"compute request exception {rid}: {exc}") - if retry_exception(exc): - rate_limited = True - else: - req = requests.pop(rid) - failed[rid] = (req, exc) - else: - # if retry_cb is set, don't move to done until it returns false - if retry_cb is None or not retry_cb(resp): - requests.pop(rid) - done[rid] = resp - - def batch_request(reqs): - batch = lookup().compute.new_batch_http_request(callback=batch_callback) - for rid, req in reqs: - batch.add(req, request_id=rid) - return batch - - while requests: - if timestamps: - timestamps = [stamp for stamp in timestamps if stamp > time()] - if rate_limited and timestamps: - stamp = next(iter(timestamps)) - sleep(max(stamp - time(), 0)) - rate_limited = False - # up to API_REQ_LIMIT (2000) requests - # in chunks of up to BATCH_LIMIT (1000) - batches = [ - batch_request(chunk) - for chunk in chunked(islice(requests.items(), API_REQ_LIMIT), BATCH_LIMIT) - ] - timestamps.append(time() + 100) - with ThreadPoolExecutor() as exe: - futures = [] - for batch in batches: - future = exe.submit(ensure_execute, batch) - futures.append(future) - for future in futures: - result = future.exception() - if result is not None: - raise result - - return done, failed - - -def get_operation_req(lkp: "Lookup", name: str, region: Optional[str]=None, zone: Optional[str]=None) -> Any: - if zone: - return lkp.compute.zoneOperations().get(project=lkp.project, zone=zone, operation=name) - elif region: - return lkp.compute.regionOperations().get(project=lkp.project, region=region, operation=name) - return lkp.compute.globalOperations().get(project=lkp.project, operation=name) - -def wait_request(operation, project: str): - """makes the appropriate wait request for a given operation""" - if "zone" in operation: - req = lookup().compute.zoneOperations().wait( - project=project, - zone=trim_self_link(operation["zone"]), - operation=operation["name"], - ) - elif "region" in operation: - req = lookup().compute.regionOperations().wait( - project=project, - region=trim_self_link(operation["region"]), - operation=operation["name"], - ) - else: - req = lookup().compute.globalOperations().wait( - project=project, operation=operation["name"] - ) - return req - - -def wait_for_operation(operation) -> Dict[str, Any]: - """wait for given operation""" - project = parse_self_link(operation["selfLink"]).project - wait_req = wait_request(operation, project=project) - - while True: - result = ensure_execute(wait_req) - if result["status"] == "DONE": - log_errors = " with errors" if "error" in result else "" - log.debug( - f"operation complete{log_errors}: type={result['operationType']}, name={result['name']}" - ) - return result - - - -def getThreadsPerCore(template) -> int: - if not template.machine_type.supports_smt: - return 1 - return template.advancedMachineFeatures.threadsPerCore or 2 - - -@retry( - max_retries=9, - init_wait_time=1, - warn_msg="Temporary failure in name resolution", - exc_type=socket.gaierror, -) -def host_lookup(host_name: str) -> str: - return socket.gethostbyname(host_name) - - -class Dumper(yaml.SafeDumper): - """Add representers for pathlib.Path and NSDict for yaml serialization""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.add_representer(NSDict, self.represent_nsdict) - self.add_multi_representer(Path, self.represent_path) - - @staticmethod - def represent_nsdict(dumper, data): - return dumper.represent_mapping("tag:yaml.org,2002:map", data.items()) - - @staticmethod - def represent_path(dumper, path): - return dumper.represent_scalar("tag:yaml.org,2002:str", str(path)) - - -@dataclass(frozen=True) -class ReservationDetails: - project: str - zone: str - name: str - policies: List[str] # names (not URLs) of resource policies - bulk_insert_name: str # name in format suitable for bulk insert (currently identical to user supplied name in long format) - deployment_type: Optional[str] - reservation_mode: Optional[str] - assured_count: int - delete_at_time: Optional[datetime] - - @property - def dense(self) -> bool: - return self.deployment_type == "DENSE" - - @property - def calendar(self) -> bool: - return self.reservation_mode == "CALENDAR" - -@dataclass(frozen=True) -class FutureReservation: - project: str - zone: str - name: str - specific: bool - start_time: datetime - end_time: datetime - reservation_mode: Optional[str] - active_reservation: Optional[ReservationDetails] - - @property - def calendar(self) -> bool: - return self.reservation_mode == "CALENDAR" - -@dataclass -class Job: - id: int - name: Optional[str] = None - required_nodes: Optional[str] = None - job_state: Optional[str] = None - duration: Optional[timedelta] = None - -@dataclass(frozen=True) -class NodeState: - base: str - flags: frozenset - -class Lookup: - """Wrapper class for cached data access""" - - def __init__(self, cfg): - self._cfg = cfg - - @property - def cfg(self): - return self._cfg - - @property - def project(self): - return self.cfg.project or authentication_project() - - @cached_property - def control_addr(self) -> Optional[str]: - return self.cfg.get("slurm_control_addr", None) - - @property - def control_host(self): - return self.cfg.slurm_control_host - - @cached_property - def control_host_addr(self): - return self.control_addr or host_lookup(self.cfg.slurm_control_host) - - @property - def control_host_port(self): - return self.cfg.slurm_control_host_port - - @property - def endpoint_versions(self): - return self.cfg.endpoint_versions - - @property - def scontrol(self): - return Path(self.cfg.slurm_bin_dir or "") / "scontrol" - - @cached_property - def instance_role(self): - return instance_role() - - @cached_property - def instance_role_safe(self): - try: - role = self.instance_role - except Exception as e: - log.error(e) - role = None - return role - - @property - def is_controller(self): - return self.instance_role_safe == "controller" - - @property - def is_login_node(self): - return self.instance_role_safe == "login" - - @cached_property - def compute(self): - # TODO evaluate when we need to use google_app_cred_path - if self.cfg.google_app_cred_path: - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.cfg.google_app_cred_path - return compute_service() - - @cached_property - def hostname(self): - return socket.gethostname() - - @cached_property - def hostname_fqdn(self): - return socket.getfqdn() - - @cached_property - def zone(self): - return instance_metadata("zone") - - node_desc_regex = re.compile( - r"^(?P(?P[^\s\-]+)-(?P\S+))-(?P(?P\w+)|(?P\[[\d,-]+\]))$" - ) - - @lru_cache(maxsize=None) - def _node_desc(self, node_name): - """Get parts from node name""" - if not node_name: - node_name = self.hostname - # workaround below is for VMs whose hostname is FQDN - node_name_short = node_name.split(".")[0] - m = self.node_desc_regex.match(node_name_short) - if not m: - raise Exception(f"node name {node_name} is not valid") - return m.groupdict() - - def node_prefix(self, node_name=None): - return self._node_desc(node_name)["prefix"] - - def node_index(self, node: str) -> int: - """ node_index("cluster-nodeset-45") == 45 """ - suff = self._node_desc(node)["suffix"] - - if suff is None: - raise ValueError(f"Node {node} name does not end with numeric index") - return int(suff) - - def node_nodeset_name(self, node_name=None): - return self._node_desc(node_name)["nodeset"] - - def node_nodeset(self, node_name=None): - nodeset_name = self.node_nodeset_name(node_name) - if nodeset_name in self.cfg.nodeset_tpu: - return self.cfg.nodeset_tpu[nodeset_name] - - return self.cfg.nodeset[nodeset_name] - - def partition_is_tpu(self, part: str) -> bool: - """check if partition with name part contains a nodeset of type tpu""" - return len(self.cfg.partitions[part].partition_nodeset_tpu) > 0 - - - def node_is_tpu(self, node_name=None): - nodeset_name = self.node_nodeset_name(node_name) - return self.cfg.nodeset_tpu.get(nodeset_name) is not None - - def nodeset_is_tpu(self, nodeset_name=None) -> bool: - return self.cfg.nodeset_tpu.get(nodeset_name) is not None - - def node_is_fr(self, node_name:str) -> bool: - return bool(self.node_nodeset(node_name).future_reservation) - - def is_dormant_res_node(self, node_name:str) -> bool: - fr = self.future_reservation(self.node_nodeset(node_name)) - res = self.nodeset_reservation(self.node_nodeset(node_name)) - - if fr is None and res is None: - return False - - if fr: - return fr.active_reservation is None - - if res: - if res.calendar: - # If reservation is calendar based, check if it is past the delete_at_time - if res.delete_at_time is not None and now() >= res.delete_at_time: - log.debug(f"DWS calendar reservation {res.bulk_insert_name} is past deletion time {res.delete_at_time}, skipping resume.") - return True - - # If assured_count is 0 do not resume nodes as they are not active yet - if res.delete_at_time is not None and res.assured_count <= 0: - log.debug(f"DWS calendar reservation {res.bulk_insert_name} is not active yet, skipping resume.") - return True - - return False - - def node_is_dyn(self, node_name=None) -> bool: - nodeset = self.node_nodeset_name(node_name) - return self.cfg.nodeset_dyn.get(nodeset) is not None - - def node_is_gke(self, node_name=None) -> bool: - return self.nodeset_is_gke(self.node_nodeset(node_name)) - - def nodeset_is_gke(self, nodeset=None) -> bool: - return "gke_nodepool" in nodeset - - def node_template(self, node_name=None) -> str: - """ Self link of nodeset template """ - return self.node_nodeset(node_name).instance_template - - def node_template_info(self, node_name=None): - return self.template_info(self.node_template(node_name)) - - def node_region(self, node_name=None): - nodeset = self.node_nodeset(node_name) - return parse_self_link(nodeset.subnetwork).region - - def nodeset_accelerator_topology(self, nodeset_name: str) -> Optional[str]: - if not self.nodeset_is_tpu(nodeset_name): - return getattr(self.cfg.nodeset[nodeset_name], 'accelerator_topology', None) - return None - - def nodeset_prefix(self, nodeset_name): - return f"{self.cfg.slurm_cluster_name}-{nodeset_name}" - - def nodelist_range(self, nodeset_name: str, start: int, count: int) -> str: - assert 0 <= start and 0 < count - pref = self.nodeset_prefix(nodeset_name) - if count == 1: - return f"{pref}-{start}" - return f"{pref}-[{start}-{start + count - 1}]" - - def static_dynamic_sizes(self, nodeset: NSDict) -> Tuple[int, int]: - return (nodeset.node_count_static or 0, nodeset.node_count_dynamic_max or 0) - - def nodelist(self, nodeset) -> str: - cnt = sum(self.static_dynamic_sizes(nodeset)) - if cnt == 0: - return "" - return self.nodelist_range(nodeset.nodeset_name, 0, cnt) - - def nodenames(self, nodeset) -> Tuple[Iterable[str], Iterable[str]]: - pref = self.nodeset_prefix(nodeset.nodeset_name) - s_count, d_count = self.static_dynamic_sizes(nodeset) - return ( - (f"{pref}-{i}" for i in range(s_count)), - (f"{pref}-{i}" for i in range(s_count, s_count + d_count)), - ) - - def power_managed_nodesets(self) -> Iterable[NSDict]: - return chain(self.cfg.nodeset.values(), self.cfg.nodeset_tpu.values()) - - def is_power_managed_node(self, node_name: str) -> bool: - try: - ns = self.node_nodeset(node_name) - if ns is None: - return False - idx = int(self._node_desc(node_name)["suffix"]) - return idx < sum(self.static_dynamic_sizes(ns)) - except Exception: - return False - - def is_static_node(self, node_name: str) -> bool: - if not self.is_power_managed_node(node_name): - return False - idx = int(self._node_desc(node_name)["suffix"]) - return idx < self.node_nodeset(node_name).node_count_static - - @lru_cache(maxsize=None) - def slurm_nodes(self) -> Dict[str, NodeState]: - def parse_line(node_line) -> Tuple[str, NodeState]: - """turn node,state line to (node, NodeState)""" - # state flags include: CLOUD, COMPLETING, DRAIN, FAIL, POWERED_DOWN, - # POWERING_DOWN - node, fullstate = node_line.split(",") - state = fullstate.split("+") - state_tuple = NodeState(base=state[0], flags=frozenset(state[1:])) - return (node, state_tuple) - - cmd = ( - f"{self.scontrol} show nodes | " - r"grep -oP '^NodeName=\K(\S+)|\s+State=\K(\S+)' | " - r"paste -sd',\n'" - ) - node_lines = run(cmd, shell=True).stdout.rstrip().splitlines() - nodes = { - node: state - for node, state in map(parse_line, node_lines) - if "CLOUD" in state.flags or "DYNAMIC_NORM" in state.flags - } - return nodes - - def node_state(self, nodename: str) -> Optional[NodeState]: - state = self.slurm_nodes().get(nodename) - if state is not None: - return state - - # state is None => Slurm doesn't know this node, - # there are two reasons: - # * happy: - # * node belongs to removed nodeset - # * node belongs to downsized portion of nodeset - # * dynamic node that didn't register itself - # * unhappy: - # * there is a drift in Slurm and SlurmGCP configurations - # * `slurm_nodes` function failed to handle `scontrol show nodes`, - # TODO: make `slurm_nodes` robust by using `scontrol show nodes --json` - # In either of "unhappy" cases it's too dangerous to proceed - abort slurmsync. - try: - ns = self.node_nodeset(nodename) - except: - log.info(f"Unknown node {nodename}, belongs to unknown nodeset") - return None # Can't find nodeset, may be belongs to removed nodeset - - if self.node_is_dyn(nodename): - log.info(f"Unknown node {nodename}, belongs to dynamic nodeset") - return None # we can't make any judjment for dynamic nodes - - cnt = sum(self.static_dynamic_sizes(ns)) - if self.node_index(nodename) >= cnt: - log.info(f"Unknown node {nodename}, out of nodeset size boundaries ({cnt})") - return None # node belongs to downsized nodeset - - raise RuntimeError(f"Slurm does not recognize node {nodename}, potential misconfiguration.") - - - @lru_cache(maxsize=1) - def instances(self) -> Dict[str, Instance]: - instance_information_fields = [ - "creationTimestamp", - "name", - "resourceStatus", - "scheduling", - "status", - "labels.slurm_instance_role", - "zone", - "metadata", - ] - - instance_fields = ",".join(sorted(instance_information_fields)) - fields = f"items.zones.instances({instance_fields}),nextPageToken" - flt = f"labels.slurm_cluster_name={self.cfg.slurm_cluster_name} AND name:{self.cfg.slurm_cluster_name}-*" - act = self.compute.instances() - op = act.aggregatedList(project=self.project, fields=fields, filter=flt) - - instances = {} - while op is not None: - result = ensure_execute(op) - for zone in result.get("items", {}).values(): - for jo in zone.get("instances", []): - inst = Instance.from_json(jo) - if inst.name in instances: - log.error(f"Duplicate VM name {inst.name} across multiple zones") - instances[inst.name] = inst - op = act.aggregatedList_next(op, result) - return instances - - def instance(self, instance_name: str) -> Optional[Instance]: - return self.instances().get(instance_name) - - @lru_cache() - def _get_reservation(self, project: str, zone: str, name: str) -> Any: - """See https://cloud.google.com/compute/docs/reference/rest/v1/reservations""" - return self.compute.reservations().get( - project=project, zone=zone, reservation=name).execute() - - @lru_cache() - def get_mig(self, project: str, region: str, self_link:str) -> Any: - """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" - return self.compute.regionInstanceGroupManagers().get(project=project, region=region, instanceGroupManager=self_link).execute() - - @lru_cache - def get_mig_instances(self, project: str, region: str, self_link:str) -> Any: - return self.compute.regionInstanceGroupManagers().listManagedInstances(project=project, region=region, instanceGroupManager=self_link).execute() - - @lru_cache() - def get_mig_list(self, project: str, region: str) -> Any: - """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" - return self.compute.regionInstanceGroupManagers().list(project=project, region=region).execute() - - @lru_cache() - def _get_future_reservation(self, project:str, zone:str, name: str) -> Any: - """See https://cloud.google.com/compute/docs/reference/rest/v1/futureReservations""" - return self.compute.futureReservations().get(project=project, zone=zone, futureReservation=name).execute() - - def get_reservation_details(self, project:str, zone:str, name:str, bulk_insert_name:str) -> ReservationDetails: - reservation = self._get_reservation(project, zone, name) - - # Converts policy URLs to names, e.g.: - # projects/111111/regions/us-central1/resourcePolicies/zebra -> zebra - policies = [u.split("/")[-1] for u in reservation.get("resourcePolicies", {}).values()] - - return ReservationDetails( - project=project, - zone=zone, - name=name, - policies=policies, - deployment_type=reservation.get("deploymentType"), - reservation_mode=reservation.get("reservationMode"), - assured_count=int(reservation.get("specificReservation", {}).get("assuredCount", 0)), - delete_at_time=parse_gcp_timestamp(reservation.get("deleteAtTime")) if reservation.get("deleteAtTime") else None, - bulk_insert_name=bulk_insert_name) - - def nodeset_reservation(self, nodeset: NSDict) -> Optional[ReservationDetails]: - if not nodeset.reservation_name: - return None - - zones = list(nodeset.zone_policy_allow or []) - assert len(zones) == 1, "Only single zone is supported if using a reservation" - zone = zones[0] - - regex = re.compile(r'^projects/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$') - if not (match := regex.match(nodeset.reservation_name)): - raise ValueError( - f"Invalid reservation name: '{nodeset.reservation_name}', expected format is 'projects/PROJECT/reservations/NAME'" - ) - - project, name = match.group("project", "reservation") - return self.get_reservation_details(project, zone, name, nodeset.reservation_name) - - def future_reservation(self, nodeset: NSDict) -> Optional[FutureReservation]: - if not nodeset.future_reservation: - return None - - active_reservation = None - match = re.search(r'^projects/(?P[^/]+)/zones/(?P[^/]+)/futureReservations/(?P[^/]+)(/.*)?$', nodeset.future_reservation) - assert match, f"Invalid future reservation name '{nodeset.future_reservation}'" - project, zone, name = match.group("project","zone","name") - fr = self._get_future_reservation(project,zone,name) - - start_time = parse_gcp_timestamp(fr["timeWindow"]["startTime"]) - end_time = parse_gcp_timestamp(fr["timeWindow"]["endTime"]) - - if "autoCreatedReservations" in fr["status"] and (res:=fr["status"]["autoCreatedReservations"][0]): - if start_time <= now() <=end_time: - match = re.search(r'projects/(?P[^/]+)/zones/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$',res) - assert match, f"Unexpected reservation name '{res}'" - res_name = match.group("name") - bulk_insert_name = f"projects/{project}/reservations/{res_name}" - active_reservation = self.get_reservation_details(project, zone, res_name, bulk_insert_name) - - return FutureReservation( - project=project, - zone=zone, - name=name, - specific=fr["specificReservationRequired"], - start_time=start_time, - end_time=end_time, - reservation_mode=fr.get("reservationMode"), - active_reservation=active_reservation - ) - - @lru_cache(maxsize=1) - def machine_types(self): - field_names = "name,zone,guestCpus,memoryMb,accelerators" - fields = f"items.zones.machineTypes({field_names}),nextPageToken" - - machines: Dict[str, Dict[str, Any]] = defaultdict(dict) - act = self.compute.machineTypes() - op = act.aggregatedList(project=self.project, fields=fields) - while op is not None: - result = ensure_execute(op) - machine_iter = chain.from_iterable( - scope.get("machineTypes", []) for scope in result["items"].values() - ) - for machine in machine_iter: - name = machine["name"] - zone = machine["zone"] - machines[name][zone] = machine - - op = act.aggregatedList_next(op, result) - return machines - - def machine_type(self, name: str) -> MachineType: - custom_patt = re.compile( - r"((?P\w+)-)?custom-(?P\d+)-(?P\d+)" - ) - if match := custom_patt.match(name): - return MachineType( - name=name, - guest_cpus=int(match.group("cpus")), - memory_mb=int(match.group("mem")), - accelerators=[], - ) - - machines = self.machine_types() - if name not in machines: - raise Exception(f"machine type {name} not found") - per_zone = machines[name] - assert per_zone - return MachineType.from_json( - next(iter(per_zone.values())) # pick the first/any zone - ) - - def template_machine_conf(self, template_link): - template = self.template_info(template_link) - machine = template.machine_type - - machine_conf = NSDict() - machine_conf.boards = 1 # No information, assume 1 - machine_conf.sockets = machine.sockets - # the value below for SocketsPerBoard must be type int - machine_conf.sockets_per_board = machine_conf.sockets // machine_conf.boards - machine_conf.threads_per_core = 1 - _div = 2 if getThreadsPerCore(template) == 1 else 1 - machine_conf.cpus = ( - int(machine.guest_cpus / _div) if machine.supports_smt else machine.guest_cpus - ) - machine_conf.cores_per_socket = int(machine_conf.cpus / machine_conf.sockets) - # Because the actual memory on the host will be different than - # what is configured (e.g. kernel will take it). From - # experiments, about 16 MB per GB are used (plus about 400 MB - # buffer for the first couple of GB's. Using 30 MB to be safe. - gb = machine.memory_mb // 1024 - machine_conf.memory = machine.memory_mb - (400 + (30 * gb)) - return machine_conf - - @lru_cache(maxsize=None) - def template_info(self, template_link): - template_name = trim_self_link(template_link) - cache = file_cache.cache("template_cache") - - if cached := cache.get(template_name): - return NSDict(cached) - - region = get_self_link_component(template_link, "regions") - - template = ensure_execute( - self.compute.instanceTemplates().get( - project=self.project, instanceTemplate=template_name - ) if region is None else - self.compute.regionInstanceTemplates().get( - project=self.project, region=region, instanceTemplate=template_name - ) - ).get("properties") - template = NSDict(template) - # name and link are not in properties, so stick them in - template.name = template_name - template.link = template_link - template.machine_type = self.machine_type(template.machineType) - # TODO delete metadata to reduce memory footprint? - # del template.metadata - - template.gpu = get_template_gpu(template) - - cache.set(template_name, template.to_dict()) - return template - - def _parse_job_info(self, job_info: str) -> Job: - """Extract job details""" - if match:= re.search(r"JobId=(\d+)", job_info): - job_id = int(match.group(1)) - else: - raise ValueError(f"Job ID not found in the job info: {job_info}") - - if match:= re.search(r"TimeLimit=(?:(\d+)-)?(\d{2}):(\d{2}):(\d{2})", job_info): - days, hours, minutes, seconds = match.groups() - duration = timedelta( - days=int(days) if days else 0, - hours=int(hours), - minutes=int(minutes), - seconds=int(seconds) - ) - else: - duration = None - - if match := re.search(r"JobName=([^\n]+)", job_info): - name = match.group(1) - else: - name = None - - if match := re.search(r"JobState=(\w+)", job_info): - job_state = match.group(1) - else: - job_state = None - - if match := re.search(r"ReqNodeList=([^ ]+)", job_info): - required_nodes = match.group(1) - else: - required_nodes = None - - return Job(id=job_id, duration=duration, name=name, job_state=job_state, required_nodes=required_nodes) - - @lru_cache - def get_jobs(self) -> List[Job]: - res = run(f"{self.scontrol} show jobs", timeout=30) - - return [self._parse_job_info(job) for job in res.stdout.split("\n\n")[:-1]] - - @lru_cache - def job(self, job_id: int) -> Optional[Job]: - job_info = run(f"{self.scontrol} show jobid {job_id}", check=False).stdout.rstrip() - if not job_info: - return None - - return self._parse_job_info(job_info=job_info) - - @property - def etc_dir(self) -> Path: - return Path(self.cfg.output_dir or slurmdirs.etc) - - def controller_mount_server_ip(self) -> str: - return self.control_addr or self.control_host - - def normalize_ns_mount(self, ns: Union[dict, NSMount]) -> NSMount: - if isinstance(ns, NSMount): - return ns - - server_ip = ns.get("server_ip") or "$controller" - if server_ip == "$controller": - server_ip = self.controller_mount_server_ip() - - return NSMount( - server_ip=server_ip, - local_mount=Path(ns["local_mount"]), - remote_mount=Path(ns["remote_mount"]), - fs_type=ns["fs_type"], - mount_options=ns["mount_options"], - ) - - @property - def munge_mount(self) -> NSMount: - if self.cfg.munge_mount: - mnt = self.cfg.munge_mount - mnt.local_mount = mnt.local_mount or "/mnt/munge" - return self.normalize_ns_mount(mnt) - else: - return NSMount( - server_ip=self.controller_mount_server_ip(), - local_mount=Path("/mnt/munge"), - remote_mount=dirs.munge, - fs_type="nfs", - mount_options="defaults,hard,intr,_netdev", - ) - - @property - def slurm_key_mount(self) -> NSMount: - if self.cfg.slurm_key_mount: - mnt = self.cfg.slurm_key_mount - mnt.local_mount = mnt.local_mount or slurmdirs.key_distribution - return self.normalize_ns_mount(mnt) - else: - return NSMount( - server_ip=self.controller_mount_server_ip(), - local_mount=slurmdirs.key_distribution, - remote_mount=slurmdirs.key_distribution, - fs_type="nfs", - mount_options="defaults,hard,intr,_netdev", - ) - - def is_flex_node(self, node: str) -> bool: - try: - nodeset = self.node_nodeset(node) - if nodeset.dws_flex.use_bulk_insert: - return False #For legacy flex support - return bool(nodeset.dws_flex.enabled) - except: - return False - - def is_provisioning_flex_node(self, node:str) -> bool: - if not self.is_flex_node(node): - return False - if self.instance(node) is not None: - return True - - nodeset = self.node_nodeset(node) - zones = nodeset.zone_policy_allow - assert len(zones) > 0 - region = self.node_region(node) - - potential_migs=[] - mig_list=self.get_mig_list(self.project, region) - - if not mig_list or not mig_list.get("items"): - return False - - for mig in mig_list["items"]: - if not mig.get("instanceTemplate"): #possibly an old MIG - return False - if mig["instanceTemplate"] == self.node_template(node) and mig["currentActions"]["creating"] > 0: - potential_migs.append(self.get_mig_instances(self.project, region, trim_self_link(mig["selfLink"]))) - - if not potential_migs: - return False - - for instance_collection in potential_migs[0]["managedInstances"]: - if node in instance_collection["name"] and instance_collection["currentAction"]=="CREATING": - return True - return False - - def cluster_regions(self) -> list[str]: - """ - Returns all regions used in cluster - NOTE: only concerned with normal nodesets, - neither TPU, nor dynamic, nor login node, nor controller node are considered - """ - res = set() - for nodeset in self.cfg.nodeset.values(): - res.add(parse_self_link(nodeset.subnetwork).region) - return list(res) - - - -_lkp: Optional[Lookup] = None - -def _load_config() -> NSDict: - return NSDict(yaml.safe_load(CONFIG_FILE.read_text())) - -def lookup() -> Lookup: - global _lkp - if _lkp is None: - try: - cfg = _load_config() - except FileNotFoundError: - log.error(f"config file not found: {CONFIG_FILE}") - cfg = NSDict() # TODO: fail here, once all code paths are covered (mainly init_logging) - _lkp = Lookup(cfg) - return _lkp - -def update_config(cfg: NSDict) -> None: - global _lkp - _lkp = Lookup(cfg) - -def scontrol_reconfigure(lkp: Lookup) -> None: - log.info("Running systemctl restart slurmctld.service") - run("sudo systemctl restart slurmctld.service", timeout=30) - log.info("Running scontrol reconfigure") - run(f"{lkp.scontrol} reconfigure") diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py deleted file mode 100644 index d1d77a1833..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any - - -from dataclasses import dataclass, asdict -import util -import local_pubsub - -import logging -log = logging.getLogger() - -# Name of the topic -TOPIC = "watch_delete_vm_op" - -@dataclass(frozen=True) -class WatchDeleteVmOp_Message: - op_name: str - zone: str - node: str - -class WatchDeleteVmOp_Topic: - def __init__(self, topic: local_pubsub.Topic) -> None: - self._t = topic - - def publish(self, op: dict[str, Any], node: str) -> None: - assert op.get("operationType") == "delete" - assert op.get("zone") - assert node - - msg = WatchDeleteVmOp_Message(op_name=op["name"], zone=op["zone"], node=node) - self._t.publish(data=asdict(msg)) - - -def watch_delete_vm_op_topic() -> WatchDeleteVmOp_Topic: - return WatchDeleteVmOp_Topic(local_pubsub.topic(TOPIC)) - - -def _watch_op(lkp: util.Lookup, m: WatchDeleteVmOp_Message) -> bool: - """ - Processes VM delete-operation. - If operation is still running - do nothing - If operation failed - log error & remove op from watch list - If operation is done - remove op from watch list do nothing - - To avoid querying status for each op individually, use list of VM instances as - a source of data. Don't query op for instance X if instance X is not present - (presumably deleted). - NOTE: This optimization can lead to false-positives - - absence of error-logs in case op failed, but VM got deleted by other means. - - Returns True if message should be marked as processed (ack). - """ - - inst = lkp.instance(m.node) - - if not inst: - log.debug(f"Stop watching op {m.op_name}, VM {m.node} appears to be deleted") - return True # ack, potentially false-positive - - if inst.status == "TERMINATED": - log.debug(f"Stop watching op {m.op_name}, VM {m.node} is TERMINATED") - return True # ack, potentially false-positive - - if inst.status == "STOPPING": - log.debug(f"Skipping op {m.op_name}, VM {m.node} is STOPPING") - return False # try later - - try: - op = util.get_operation_req(lkp, m.op_name, zone=m.zone).execute() - except: - # TODO: consider less conservative handling, but be careful not to cause deadlettering. - log.exception(f"Failed to get operation {m.op_name}, will not retry") - return True # ack (remove) - - if op["status"] != "DONE": - log.debug(f"Watching op {m.op_name} is still not done ({op['status']})") - return False # try later - - if "error" in op: - log.error(f"Operation {m.op_name} to delete {m.node} finished with error: {op['error']}") - else: - log.debug(f"Operation {m.op_name} to delete {m.node} successfully finished") - return True # ack - - -def watch_vm_delete_ops(lkp: util.Lookup) -> None: - sub = local_pubsub.subscription(TOPIC) - - # Pull once instead of "pulling until empty", motivation: - # Bulk of cases processed by `_watch_op` relies on freshness of `lkp.instances`, - # `lkp.instances` are fetched once during run of `slurmsync`. - # Therefore we shouldn't try to re-process messages that has been already NACKed in this run, - # since they will be handled with the same `lkp.instance` as a previous attempt. - msgs = sub.pull(max_messages=1000) # 1000 is arbitrary number to be adjusted if needed. - log.debug(f"Processing {len(msgs)} delete VM operations") - # TODO: handle messages in butches to improve latency - for m in msgs: - try: - dm = WatchDeleteVmOp_Message(**m.data) - ack = _watch_op(lkp, dm) - except Exception: - log.exception(f"Failed to process the message {m.id}, removing") - ack = True - if ack: - sub.ack([m.id]) - else: - sub.modify_ack_deadline([m.id], deadline=0) # NACK - - - - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf deleted file mode 100644 index 71905a0342..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf +++ /dev/null @@ -1,504 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "bucket_name" { - description = <<-EOD - Name of GCS bucket to use. - EOD - type = string -} - -variable "bucket_dir" { - description = "Bucket directory for cluster files to be put into." - type = string - default = null -} - -variable "enable_debug_logging" { - type = bool - description = "Enables debug logging mode. Not for production use." - default = false -} - -variable "extra_logging_flags" { - type = map(bool) - description = "The only available flag is `trace_api`" - default = {} -} - -variable "project_id" { - description = "The GCP project ID." - type = string -} - -variable "enable_slurm_auth" { - description = < x... } - nodeset_map = { for k, vs in local.nodeset_map_ell : k => vs[0] } - - nodeset_tpu_map_ell = { for x in var.nodeset_tpu : x.nodeset_name => x... } - nodeset_tpu_map = { for k, vs in local.nodeset_tpu_map_ell : k => vs[0] } - - nodeset_dyn_map_ell = { for x in var.nodeset_dyn : x.nodeset_name => x... } - nodeset_dyn_map = { for k, vs in local.nodeset_dyn_map_ell : k => vs[0] } - - - no_reservation_affinity = { type : "NO_RESERVATION" } -} - -# NODESET -module "slurm_nodeset_template" { - source = "../../internal/slurm-gcp/instance_template" - for_each = local.nodeset_map - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - slurm_instance_role = "compute" - slurm_bucket_path = module.slurm_files.slurm_bucket_path - - additional_disks = each.value.additional_disks - bandwidth_tier = each.value.bandwidth_tier - can_ip_forward = each.value.can_ip_forward - advanced_machine_features = each.value.advanced_machine_features - disk_auto_delete = each.value.disk_auto_delete - disk_labels = each.value.disk_labels - disk_resource_manager_tags = each.value.disk_resource_manager_tags - disk_size_gb = each.value.disk_size_gb - disk_type = each.value.disk_type - enable_confidential_vm = each.value.enable_confidential_vm - enable_oslogin = each.value.enable_oslogin - enable_shielded_vm = each.value.enable_shielded_vm - gpu = each.value.gpu - labels = merge(each.value.labels, { slurm_nodeset = each.value.nodeset_name }) - machine_type = each.value.machine_type - metadata = merge(each.value.metadata, local.universe_domain) - min_cpu_platform = each.value.min_cpu_platform - name_prefix = each.value.nodeset_name - on_host_maintenance = each.value.on_host_maintenance - preemptible = each.value.preemptible - region = each.value.region - resource_manager_tags = each.value.resource_manager_tags - spot = each.value.spot - termination_action = each.value.termination_action - service_account = each.value.service_account - shielded_instance_config = each.value.shielded_instance_config - source_image_family = each.value.source_image_family - source_image_project = each.value.source_image_project - source_image = each.value.source_image - subnetwork = each.value.subnetwork_self_link - additional_networks = each.value.additional_networks - access_config = each.value.access_config - tags = concat([local.slurm_cluster_name], each.value.tags) - - max_run_duration = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? each.value.dws_flex.max_run_duration : null - provisioning_model = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? "FLEX_START" : null - reservation_affinity = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? local.no_reservation_affinity : null -} - -module "nodeset_cleanup" { - source = "./modules/cleanup_compute" - for_each = local.nodeset_map - - nodeset = each.value - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - enable_cleanup_compute = var.enable_cleanup_compute - universe_domain = var.universe_domain - endpoint_versions = var.endpoint_versions - gcloud_path_override = var.gcloud_path_override - nodeset_template = module.slurm_nodeset_template[each.value.nodeset_name].self_link -} - -locals { - nodesets = [for name, ns in local.nodeset_map : { - nodeset_name = ns.nodeset_name - node_conf = ns.node_conf - dws_flex = ns.dws_flex - instance_template = module.slurm_nodeset_template[ns.nodeset_name].self_link - node_count_dynamic_max = ns.node_count_dynamic_max - node_count_static = ns.node_count_static - subnetwork = ns.subnetwork_self_link - reservation_name = ns.reservation_name - future_reservation = ns.future_reservation - maintenance_interval = ns.maintenance_interval - instance_properties_json = ns.instance_properties_json - enable_placement = ns.enable_placement - placement_max_distance = ns.placement_max_distance - network_storage = ns.network_storage - zone_target_shape = ns.zone_target_shape - zone_policy_allow = ns.zone_policy_allow - zone_policy_deny = ns.zone_policy_deny - enable_maintenance_reservation = ns.enable_maintenance_reservation - enable_opportunistic_maintenance = ns.enable_opportunistic_maintenance - accelerator_topology = ns.accelerator_topology - }] -} - -# NODESET TPU -module "slurm_nodeset_tpu" { - source = "../../internal/slurm-gcp/nodeset_tpu" - for_each = local.nodeset_tpu_map - - project_id = var.project_id - node_count_dynamic_max = each.value.node_count_dynamic_max - node_count_static = each.value.node_count_static - nodeset_name = each.value.nodeset_name - zone = each.value.zone - node_type = each.value.node_type - accelerator_config = each.value.accelerator_config - tf_version = each.value.tf_version - preemptible = each.value.preemptible - preserve_tpu = each.value.preserve_tpu - enable_public_ip = each.value.enable_public_ip - service_account = each.value.service_account - data_disks = each.value.data_disks - docker_image = each.value.docker_image - subnetwork = each.value.subnetwork -} - -module "nodeset_cleanup_tpu" { - source = "./modules/cleanup_tpu" - for_each = local.nodeset_tpu_map - - nodeset = { - nodeset_name = each.value.nodeset_name - zone = each.value.zone - } - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - enable_cleanup_compute = var.enable_cleanup_compute - universe_domain = var.universe_domain - endpoint_versions = var.endpoint_versions - gcloud_path_override = var.gcloud_path_override - - depends_on = [ - # Depend on controller network, as a best effort to avoid - # subnetwork resourceInUseByAnotherResource error - var.subnetwork_self_link - ] -} - -resource "google_storage_bucket_object" "parition_config" { - for_each = { for p in var.partitions : p.partition_name => p } - - bucket = module.slurm_files.bucket_name - name = "${module.slurm_files.bucket_dir}/partition_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -moved { - from = module.slurm_files.google_storage_bucket_object.parition_config - to = google_storage_bucket_object.parition_config -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf deleted file mode 100644 index 218c36e392..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# BUCKET - -locals { - synt_suffix = substr(md5("${local.controller_project_id}${var.deployment_name}"), 0, 5) - synth_bucket_name = "${local.slurm_cluster_name}${local.synt_suffix}" - - bucket_name = var.create_bucket ? module.bucket[0].name : var.bucket_name -} - -module "bucket" { - source = "terraform-google-modules/cloud-storage/google" - version = ">= 6.1" - - count = var.create_bucket ? 1 : 0 - - location = var.region - names = [local.synth_bucket_name] - prefix = "slurm" - project_id = local.controller_project_id - - force_destroy = { - (local.synth_bucket_name) = true - } - - labels = merge(local.labels, { - slurm_cluster_name = local.slurm_cluster_name - }) -} - -# BUCKET IAMs -locals { - compute_sa = toset(flatten([for x in module.slurm_nodeset_template : x.service_account])) - compute_tpu_sa = toset(flatten([for x in module.slurm_nodeset_tpu : x.service_account])) - login_sa = toset(flatten([for x in module.login : x.service_account])) - - viewers = toset(flatten([ - "serviceAccount:${module.slurm_controller_template.service_account.email}", - formatlist("serviceAccount:%s", [for x in local.compute_sa : x.email]), - formatlist("serviceAccount:%s", [for x in local.compute_tpu_sa : x.email if x.email != null]), - formatlist("serviceAccount:%s", [for x in local.login_sa : x.email]), - ])) -} - - -resource "google_storage_bucket_iam_member" "viewers" { - for_each = local.viewers - bucket = local.bucket_name - role = "roles/storage.objectViewer" - member = each.value -} - -resource "google_storage_bucket_iam_member" "legacy_readers" { - for_each = local.viewers - bucket = local.bucket_name - role = "roles/storage.legacyBucketReader" - member = each.value -} - -locals { - daos_ns = [ - for ns in var.network_storage : - ns if ns.fs_type == "daos" - ] - - daos_client_install_runners = [ - for ns in local.daos_ns : - ns.client_install_runner if ns.client_install_runner != null - ] - - daos_mount_runners = [ - for ns in local.daos_ns : - ns.mount_runner if ns.mount_runner != null - ] - - daos_network_storage_runners = concat( - local.daos_client_install_runners, - local.daos_mount_runners, - ) - - daos_install_mount_script = { - filename = "ghpc_daos_mount.sh" - content = length(local.daos_ns) > 0 ? module.daos_network_storage_scripts[0].startup_script : "" - } - - common_scripts = length(local.daos_ns) > 0 ? [local.daos_install_mount_script] : [] -} - -# SLURM FILES -locals { - ghpc_startup_script_controller = concat( - local.common_scripts, - [{ - filename = "ghpc_startup.sh" - content = var.controller_startup_script - }]) - - controller_state_disk = { - device_name : try(google_compute_disk.controller_disk[0].name, null) - } - - - nodeset_startup_scripts = { for k, v in local.nodeset_map : k => concat(local.common_scripts, v.startup_script) } -} - -module "daos_network_storage_scripts" { - count = length(local.daos_ns) > 0 ? 1 : 0 - - source = "../../../../modules/scripts/startup-script" - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.daos_network_storage_runners -} - -module "slurm_files" { - source = "./modules/slurm_files" - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - bucket_dir = var.bucket_dir - bucket_name = local.bucket_name - controller_network_attachment = var.controller_network_attachment - - slurmdbd_conf_tpl = var.slurmdbd_conf_tpl - slurm_conf_tpl = var.slurm_conf_tpl - slurm_conf_template = var.slurm_conf_template - cgroup_conf_tpl = var.cgroup_conf_tpl - cloud_parameters = var.cloud_parameters - cloudsql_secret = try( - one(google_secret_manager_secret_version.cloudsql_version[*].id), - null) - - controller_startup_scripts = local.ghpc_startup_script_controller - controller_startup_scripts_timeout = var.controller_startup_scripts_timeout - nodeset_startup_scripts = local.nodeset_startup_scripts - compute_startup_scripts_timeout = var.compute_startup_scripts_timeout - controller_state_disk = local.controller_state_disk - - enable_debug_logging = var.enable_debug_logging - extra_logging_flags = var.extra_logging_flags - - enable_slurm_auth = var.enable_slurm_auth - - enable_bigquery_load = var.enable_bigquery_load - enable_external_prolog_epilog = var.enable_external_prolog_epilog - enable_chs_gpu_health_check_prolog = var.enable_chs_gpu_health_check_prolog - enable_chs_gpu_health_check_epilog = var.enable_chs_gpu_health_check_epilog - epilog_scripts = var.epilog_scripts - prolog_scripts = var.prolog_scripts - task_epilog_scripts = var.task_epilog_scripts - task_prolog_scripts = var.task_prolog_scripts - - disable_default_mounts = !var.enable_default_mounts - network_storage = [ - for storage in var.network_storage : { - server_ip = storage.server_ip, - remote_mount = storage.remote_mount, - local_mount = storage.local_mount, - fs_type = storage.fs_type, - mount_options = storage.mount_options - } - if storage.fs_type != "daos" - ] - - nodeset = local.nodesets - nodeset_dyn = values(local.nodeset_dyn_map) - # Use legacy format for now - nodeset_tpu = values(module.slurm_nodeset_tpu)[*] - - - depends_on = [module.bucket] - - # Providers - endpoint_versions = var.endpoint_versions -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf deleted file mode 100644 index 85ad10fa21..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf +++ /dev/null @@ -1,814 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########### -# GENERAL # -########### - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "deployment_name" { - description = "Name of the deployment." - type = string -} - -variable "slurm_cluster_name" { - type = string - description = <<-EOD - Cluster name, used for resource naming and slurm accounting. - If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). - EOD - default = null - - validation { - condition = var.slurm_cluster_name == null || can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) - error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." - } -} - -variable "region" { - type = string - description = "The default region to place resources in." -} - -variable "zone" { - type = string - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disable\_login\_public\_ips](#input\_disable\_login\_public\_ips) | DEPRECATED: Use `enable_login_public_ips` instead. | `bool` | `null` | no | -| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_login\_public\_ips](#input\_enable\_login\_public\_ips) | If set to true. The login node will have a random public IP assigned to it. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for login nodes. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Unique name prefix for login nodes. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all login groups. | `string` | n/a | yes | -| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the login instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the login instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [login\_nodes](#output\_login\_nodes) | Slurm login instance definition. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf deleted file mode 100644 index 6ebe5902dc..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-login", ghpc_role = "scheduler" }) -} - -module "instance_validation" { - source = "../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata - ) - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.disk_resource_manager_tags - } - ] - - public_access_config = [{ nat_ip = null, network_tier = null }] - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - # lower, replace `_` with `-`, and remove any non-alphanumeric characters - group_name = replace( - replace( - lower(var.name_prefix), - "_", "-"), - "/[^-a-z0-9]/", "") - - - login_node = { - group_name = local.group_name - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(var.disk_labels, local.labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - additional_networks = var.additional_networks - - can_ip_forward = var.can_ip_forward - advanced_machine_features = var.advanced_machine_features - - enable_confidential_vm = var.enable_confidential_vm - access_config = var.enable_login_public_ips ? local.public_access_config : [] - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - gpu = one(local.guest_accelerator) - labels = local.labels - machine_type = var.machine_type - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - num_instances = var.num_instances - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - region = var.region - resource_manager_tags = var.resource_manager_tags - zone = var.zone - - service_account = local.service_account - - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - static_ips = var.static_ips - bandwidth_tier = var.bandwidth_tier - - subnetwork = var.subnetwork_self_link - tags = var.tags - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml deleted file mode 100644 index 47f003258e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name_prefix - has_to_be_used: true diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf deleted file mode 100644 index e700542794..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "login_nodes" { - description = "Slurm login instance definition." - value = [local.login_node] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf deleted file mode 100644 index 7c1a2e06b5..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf +++ /dev/null @@ -1,419 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -variable "project_id" { # tflint-ignore: terraform_unused_declarations - type = string - description = "Project ID to create resources in." -} - -variable "region" { - type = string - description = "Region where the instances should be created." - default = null -} - -variable "zone" { - type = string - description = <<-EOD - Zone where the instances should be created. If not specified, instances will be - spread across available zones in the region. - EOD - default = null -} - -variable "name_prefix" { - type = string - description = <<-EOD - Unique name prefix for login nodes. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all login groups. - EOD -} - -variable "num_instances" { - type = number - description = "Number of instances to create. This value is ignored if static_ips is provided." - default = 1 -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} -} - -variable "disk_type" { - type = string - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - default = "pd-ssd" -} - -variable "disk_size_gb" { - type = number - description = "Boot disk size in GB." - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - type = list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string)) - auto_delete = optional(bool) - boot = optional(bool) - disk_resource_manager_tags = optional(map(string)) - })) - description = "List of maps of disks." - default = [] -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })) - nullable = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "disable_smt" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - type = bool - default = null - validation { - condition = var.disable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "static_ips" { - type = list(string) - description = "List of static IPs for VM instances." - default = [] -} - -variable "bandwidth_tier" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 6.16 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.16 | -| [helm](#provider\_helm) | ~> 2.17 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.cert_manager](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.prometheus](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.slurm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.slurm_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cert\_manager\_chart\_version](#input\_cert\_manager\_chart\_version) | Version of the Cert Manager chart to install. | `string` | `"v1.18.2"` | no | -| [cert\_manager\_values](#input\_cert\_manager\_values) | Value overrides for the Cert Manager release | `any` | 
{
  "crds": {
    "enabled": true
  }
}
| no | -| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | -| [install\_kube\_prometheus\_stack](#input\_install\_kube\_prometheus\_stack) | Install the Kube Prometheus Stack. | `bool` | `false` | no | -| [install\_slurm\_chart](#input\_install\_slurm\_chart) | Install slurm-operator chart. | `bool` | `true` | no | -| [install\_slurm\_operator\_chart](#input\_install\_slurm\_operator\_chart) | Install slurm-operator chart. | `bool` | `true` | no | -| [node\_pool\_names](#input\_node\_pool\_names) | Names of node pools, for use in node affinities (Slinky system components). | `list(string)` | `null` | no | -| [project\_id](#input\_project\_id) | The project ID that hosts the GKE cluster. | `string` | n/a | yes | -| [prometheus\_chart\_version](#input\_prometheus\_chart\_version) | Version of the Kube Prometheus Stack chart to install. | `string` | `"77.0.1"` | no | -| [prometheus\_values](#input\_prometheus\_values) | Value overrides for the Prometheus release | `any` | 
{
  "installCRDs": true
}
| no | -| [slurm\_chart\_version](#input\_slurm\_chart\_version) | Version of the Slurm chart to install. | `string` | `"0.3.1"` | no | -| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | -| [slurm\_operator\_chart\_version](#input\_slurm\_operator\_chart\_version) | Version of the Slurm Operator chart to install. | `string` | `"0.3.1"` | no | -| [slurm\_operator\_namespace](#input\_slurm\_operator\_namespace) | slurm namespace for charts | `string` | `"slinky"` | no | -| [slurm\_operator\_repository](#input\_slurm\_operator\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | -| [slurm\_operator\_values](#input\_slurm\_operator\_values) | Value overrides for the Slinky release | `any` | `{}` | no | -| [slurm\_repository](#input\_slurm\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | -| [slurm\_values](#input\_slurm\_values) | Value overrides for the Slurm release | `any` | `{}` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [slurm\_namespace](#output\_slurm\_namespace) | namespace for the slurm chart | -| [slurm\_operator\_namespace](#output\_slurm\_operator\_namespace) | namespace for the slinky operator chart | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/main.tf deleted file mode 100644 index aff33b73a0..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/main.tf +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] - project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] - - # Define affinity settings when node pools are specified - node_pool_affinity = var.node_pool_names != null ? { - nodeAffinity = { - requiredDuringSchedulingIgnoredDuringExecution = { - nodeSelectorTerms = [{ - matchExpressions = [{ - key = "cloud.google.com/gke-nodepool" - operator = "In" - values = var.node_pool_names - }] - }] - } - } - } : {} -} - -data "google_client_config" "default" {} - -data "google_container_cluster" "gke_cluster" { - project = local.project_id - name = local.cluster_name - location = local.cluster_location -} - -resource "helm_release" "cert_manager" { - name = "cert-manager" - chart = "cert-manager" - repository = "https://charts.jetstack.io" - version = var.cert_manager_chart_version - namespace = "cert-manager" - create_namespace = true - - values = concat( - [yamlencode({ - affinity = local.node_pool_affinity - webhook = { - affinity = local.node_pool_affinity - } - cainjector = { - affinity = local.node_pool_affinity - } - startupapicheck = { - affinity = local.node_pool_affinity - } - })], - [yamlencode(var.cert_manager_values)] - ) -} - -resource "helm_release" "slurm_operator" { - count = var.install_slurm_operator_chart ? 1 : 0 - name = "slurm-operator" - chart = "slurm-operator" - repository = var.slurm_operator_repository - version = var.slurm_operator_chart_version - namespace = var.slurm_operator_namespace - create_namespace = true - - # The Cert Manager webhook deployment must be running to provision the Operator - depends_on = [ - helm_release.cert_manager - ] - - values = concat( - [yamlencode({ - operator = { - affinity = local.node_pool_affinity - } - webhook = { - affinity = local.node_pool_affinity - } - })], - [yamlencode(var.slurm_operator_values)] - ) -} - -resource "helm_release" "slurm" { - count = var.install_slurm_chart ? 1 : 0 - name = "slurm" - chart = "slurm" - repository = var.slurm_repository - version = var.slurm_chart_version - namespace = var.slurm_namespace - create_namespace = true - - # The Slurm Operator must be running to provision Slurm clusters/nodesets - depends_on = [ - helm_release.slurm_operator - ] - - values = concat( - [yamlencode({ - controller = { - affinity = local.node_pool_affinity - } - accounting = { - affinity = local.node_pool_affinity - } - mariadb = { - primary = { - affinity = local.node_pool_affinity - } - secondary = { - affinity = local.node_pool_affinity - } - } - restapi = { - affinity = local.node_pool_affinity - } - slurm-exporter = { - exporter = { - affinity = local.node_pool_affinity - } - } - })], - [yamlencode(var.slurm_values)] - ) -} - -resource "helm_release" "prometheus" { - count = var.install_kube_prometheus_stack ? 1 : 0 - name = "prometheus" - chart = "kube-prometheus-stack" - repository = "https://prometheus-community.github.io/helm-charts" - version = var.prometheus_chart_version - namespace = "prometheus" - create_namespace = true - - values = concat( - [yamlencode({ - crds = { - upgradeJob = { - affinity = local.node_pool_affinity - } - } - alertmanager = { - alertmanagerSpec = { - affinity = local.node_pool_affinity - } - } - prometheusOperator = { - admissionWebhooks = { - deployment = { - affinity = local.node_pool_affinity - } - patch = { - affinity = local.node_pool_affinity - } - } - affinity = local.node_pool_affinity - } - prometheus = { - prometheusSpec = { - affinity = local.node_pool_affinity - } - } - thanosRuler = { - thanosRulerSpec = { - affinity = local.node_pool_affinity - } - } - kube-state-metrics = { - affinity = local.node_pool_affinity - } - grafana = { - affinity = local.node_pool_affinity - imageRenderer = { - affinity = local.node_pool_affinity - } - } - prometheus-windows-exporter = { - affinity = local.node_pool_affinity - } - })], - [yamlencode(var.prometheus_values)] - ) -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/metadata.yaml deleted file mode 100644 index e18197e2b7..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/outputs.tf deleted file mode 100644 index 8ea6385905..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "slurm_namespace" { - description = "namespace for the slurm chart" - value = var.slurm_namespace -} - -output "slurm_operator_namespace" { - description = "namespace for the slinky operator chart" - value = var.slurm_operator_namespace -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/providers.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/providers.tf deleted file mode 100644 index 313d6dc58e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/providers.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -provider "helm" { - kubernetes { - host = "https://${data.google_container_cluster.gke_cluster.endpoint}" - token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode( - data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate, - ) - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/variables.tf deleted file mode 100644 index 8acaf78562..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/variables.tf +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "The project ID that hosts the GKE cluster." - type = string -} - -variable "cluster_id" { - description = "An identifier for the GKE cluster resource with format projects//locations//clusters/." - type = string - nullable = false -} - -variable "node_pool_names" { - description = "Names of node pools, for use in node affinities (Slinky system components)." - type = list(string) - default = null -} - -variable "cert_manager_chart_version" { - description = "Version of the Cert Manager chart to install." - type = string - default = "v1.18.2" -} - -variable "cert_manager_values" { - description = "Value overrides for the Cert Manager release" - type = any - default = { - crds = { - enabled = true - } - } -} - -variable "slurm_operator_chart_version" { - description = "Version of the Slurm Operator chart to install." - type = string - default = "0.3.1" -} - -variable "slurm_operator_values" { - description = "Value overrides for the Slinky release" - type = any - default = {} -} - -variable "slurm_chart_version" { - description = "Version of the Slurm chart to install." - type = string - default = "0.3.1" -} - -variable "slurm_values" { - description = "Value overrides for the Slurm release" - type = any - default = {} -} - -variable "install_kube_prometheus_stack" { - # Components detailed at https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack - description = "Install the Kube Prometheus Stack." - type = bool - default = false -} - -variable "prometheus_chart_version" { - description = "Version of the Kube Prometheus Stack chart to install." - type = string - default = "77.0.1" -} - -variable "prometheus_values" { - description = "Value overrides for the Prometheus release" - type = any - default = { - installCRDs = true - } -} - -variable "slurm_namespace" { - description = "slurm namespace for charts" - type = string - default = "slurm" -} - -variable "slurm_operator_namespace" { - description = "slurm namespace for charts" - type = string - default = "slinky" -} - -variable "install_slurm_chart" { - description = "Install slurm-operator chart." - type = bool - default = true -} - -variable "install_slurm_operator_chart" { - description = "Install slurm-operator chart." - type = bool - default = true -} - -variable "slurm_repository" { - description = "Value overrides for the Slinky release" - type = string - default = "oci://ghcr.io/slinkyproject/charts" -} - -variable "slurm_operator_repository" { - description = "Value overrides for the Slinky release" - type = string - default = "oci://ghcr.io/slinkyproject/charts" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/versions.tf deleted file mode 100644 index ae4327aeef..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scheduler/slinky/versions.tf +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - google = { - source = "hashicorp/google" - version = ">= 6.16" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/README.md deleted file mode 100644 index 71a862fd6c..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/README.md +++ /dev/null @@ -1,149 +0,0 @@ -## Description - -This module creates a Toolkit runner that will install HTCondor on RedHat 7 or -8 and its derivative operating systems. These include the CentOS 7 and Rocky -Linux 8 releases of the [HPC VM Image][hpcvmimage]. It may also function on -RedHat 9 and derivatives, however it is not yet supported. Please report any -[issues] on these 3 distributions or open a [discussion] to request support on -Debian or Ubuntu distributions. - -[issues]: https://github.com/GoogleCloudPlatform/hpc-toolkit/issues -[discussion]: https://github.com/GoogleCloudPlatform/hpc-toolkit/discussions - -It also exports a list of Google Cloud APIs which must be enabled prior to -provisioning an HTCondor Pool. - -It is expected to be used with the [htcondor-setup] and -[htcondor-execute-point] modules. - -[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[htcondor-setup]: ../../scheduler/htcondor-setup/README.md -[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md - -### Example - -The following code snippet uses this module to create startup scripts that -install the HTCondor software into a custom VM image. - -```yaml -deployment_groups: -- group: primary - modules: - - id: network1 - source: modules/network/vpc - outputs: - - network_name - - - id: htcondor_install - source: community/modules/scripts/htcondor-install - - - id: htcondor_install_script - source: modules/scripts/startup-script - use: - - htcondor_install - -- group: packer - modules: - - id: custom-image - source: modules/packer/custom-image - kind: packer - use: - - network1 - - htcondor_install_script - settings: - disk_size: 50 - source_image_family: hpc-rocky-linux-8 - image_family: "htcondor-10x" -``` - -A full example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -## Important note - -All POSIX users and HTCondor jobs can act as the service account attached to -VMs within the pool. This enables the use of IAM restrictions via service -accounts but also allows users to access services to which system daemons need -access (e.g. to create Cloud Logging entries). If this is undesirable, one can -restrict access to the instance metadata server to the `root` and `condor` -users. This will allow system services to use the service account, but not -other POSIX users or HTCondor jobs. The firewall example below is appropriate -for CentOS 7. - -```shell -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 1 \ - -m owner --uid-owner root -p tcp -d metadata.google.internal --dport 80 -j ACCEPT -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 2 \ - -m owner --uid-owner condor -p tcp -d metadata.google.internal --dport 80 -j ACCEPT -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 3 \ - -p tcp -d metadata.google.internal --dport 80 -j DROP -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 4 \ - -p tcp -d metadata.google.internal --dport 8080 -j DROP -firewall-cmd --permanent --zone=public --add-port=9618/tcp -firewall-cmd --reload -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [condor\_version](#input\_condor\_version) | Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: "23.0.0","23.*")) | `string` | `"23.*"` | no | -| [enable\_docker](#input\_enable\_docker) | Install and enable docker daemon alongside HTCondor | `bool` | `true` | no | -| [http\_proxy](#input\_http\_proxy) | Set system default web (http and https) proxy for Windows HTCondor installation | `string` | `""` | no | -| [python\_windows\_installer\_url](#input\_python\_windows\_installer\_url) | URL of Python installer for Windows | `string` | `"https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [gcp\_service\_list](#output\_gcp\_service\_list) | Google Cloud APIs required by HTCondor | -| [runners](#output\_runners) | Runner to install HTCondor using startup-scripts | -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | Windows PowerShell script to install HTCondor | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py deleted file mode 100644 index 77bafa0310..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py +++ /dev/null @@ -1,417 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -# Copyright 2018 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script for resizing managed instance group (MIG) cluster size based -# on the number of jobs in the Condor Queue. - -from absl import app -from absl import flags -from collections import OrderedDict -from datetime import datetime -from pprint import pprint -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials - -import argparse -import os -import math -import time -import htcondor -import classad - -parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) -parser.add_argument("--p", required=True, help="Project id", type=str) -parser.add_argument( - "--z", - required=True, - help="Name of GCP zone where the managed instance group is located", - type=str, -) -parser.add_argument( - "--r", - required=True, - help="Name of GCP region where the managed instance group is located", - type=str, -) -parser.add_argument( - "--mz", - required=False, - help="Enabled multizone (regional) managed instance group", - action="store_true", -) -parser.add_argument( - "--g", required=True, help="Name of the managed instance group", type=str -) -parser.add_argument( - "--i", - default=0, - help="Minimum number of idle compute instances", - type=int -) -parser.add_argument( - "--c", required=True, help="Maximum number of compute instances", type=int -) -parser.add_argument( - "--v", - default=0, - help="Increase output verbosity. 1-show basic debug info. 2-show detail debug info", - type=int, - choices=[0, 1, 2], -) -parser.add_argument( - "--d", - default=0, - help="Dry Run, default=0, if 1, then no scaling actions", - type=int, - choices=[0, 1], -) - -args = parser.parse_args() - -class AutoScaler: - def __init__(self, multizone=False): - - self.multizone = multizone - # Obtain credentials - self.credentials = GoogleCredentials.get_application_default() - self.service = discovery.build("compute", "v1", credentials=self.credentials) - - if self.multizone: - self.instanceGroupManagers = self.service.regionInstanceGroupManagers() - else: - self.instanceGroupManagers = self.service.instanceGroupManagers() - - # Remove specified instances from MIG and decrease MIG size - def deleteFromMig(self, node_self_links): - requestDelInstance = self.instanceGroupManagers.deleteInstances( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - body={ "instances": node_self_links }, - ) - - # execute if not a dry-run - if not self.dryrun: - response = requestDelInstance.execute() - if self.debug > 0: - pprint(response) - return response - return "Dry Run" - - def getInstanceTemplateInfo(self): - requestTemplateName = self.instanceGroupManagers.get( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - fields="instanceTemplate", - ) - responseTemplateName = requestTemplateName.execute() - template_name = "" - - if self.debug > 1: - print("Request for the template name") - pprint(responseTemplateName) - - if len(responseTemplateName) > 0: - template_url = responseTemplateName.get("instanceTemplate") - template_url_partitioned = template_url.split("/") - template_name = template_url_partitioned[len(template_url_partitioned) - 1] - - requestInstanceTemplate = self.service.instanceTemplates().get( - project=self.project, instanceTemplate=template_name, fields="properties" - ) - responseInstanceTemplateInfo = requestInstanceTemplate.execute() - - if self.debug > 1: - print("Template information") - pprint(responseInstanceTemplateInfo["properties"]) - - machine_type = responseInstanceTemplateInfo["properties"]["machineType"] - is_spot = responseInstanceTemplateInfo["properties"]["scheduling"][ - "preemptible" - ] - if self.debug > 0: - print("Machine Type: " + machine_type) - print("Is spot: " + str(is_spot)) - request = self.service.machineTypes().get( - project=self.project, zone=self.zone, machineType=machine_type - ) - response = request.execute() - guest_cpus = response["guestCpus"] - if self.debug > 1: - print("Machine information") - pprint(responseInstanceTemplateInfo["properties"]) - if self.debug > 0: - print("Guest CPUs: " + str(guest_cpus)) - - instanceTemplateInfo = { - "machine_type": machine_type, - "is_spot": is_spot, - "guest_cpus": guest_cpus, - } - return instanceTemplateInfo - - def scale(self): - # diagnosis - if self.debug > 1: - print("Launching autoscaler.py with the following arguments:") - print("project_id: " + self.project) - print("zone: " + self.zone) - print("region: " + self.region) - print(f"multizone: {self.multizone}") - print("group_manager: " + self.instance_group_manager) - print("computeinstancelimit: " + str(self.compute_instance_limit)) - print("debuglevel: " + str(self.debug)) - - if self.multizone: - self.zoneargs = {"region": self.region} - else: - self.zoneargs = {"zone": self.zone} - - # Each HTCondor scheduler (SchedD), maintains a list of jobs under its - # stewardship. A full list of Job ClassAd attributes can be found at - # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html - schedd = htcondor.Schedd() - # encourage the job queue to start a new negotiation cycle; there are - # internal unconfigurable rate limits so not guaranteed; this is not - # strictly required for success, but may reduce latency of autoscaling - schedd.reschedule() - REQUEST_CPUS_ATTRIBUTE = "RequestCpus" - REQUEST_GPUS_ATTRIBUTE = "RequestGpus" - REQUEST_MEMORY_ATTRIBUTE = "RequestMemory" - job_attributes = [ - REQUEST_CPUS_ATTRIBUTE, - REQUEST_GPUS_ATTRIBUTE, - REQUEST_MEMORY_ATTRIBUTE, - ] - - instanceTemplateInfo = self.getInstanceTemplateInfo() - self.is_spot = instanceTemplateInfo["is_spot"] - self.cores_per_node = instanceTemplateInfo["guest_cpus"] - print(f"MIG is configured for Spot pricing: {self.is_spot}") - print("Number of CPU per compute node: " + str(self.cores_per_node)) - - # this query will constrain the search for jobs to those that either - # require spot VMs or do not require Spot VMs based on whether the - # VM instance template is configured for Spot pricing - spot_query = classad.ExprTree(f"RequireId == \"{self.instance_group_manager}\"") - - # For purpose of scaling a Managed Instance Group, count only jobs that - # are idle and likely participated in a negotiation cycle (there does - # not appear to be a single classad attribute for this). - # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html#JobStatus - LAST_CYCLE_ATTRIBUTE = "LastNegotiationCycleTime0" - coll = htcondor.Collector() - negotiator_ad = coll.query(htcondor.AdTypes.Negotiator, projection=[LAST_CYCLE_ATTRIBUTE]) - if len(negotiator_ad) != 1: - print(f"There should be exactly 1 negotiator in the pool. There is {len(negotiator_ad)}") - exit() - last_negotiation_cycle_time = negotiator_ad[0].get(LAST_CYCLE_ATTRIBUTE) - if not last_negotiation_cycle_time: - print(f"The negotiator has not yet started a match cycle. Exiting auto-scaling.") - exit() - - print(f"Last negotiation cycle occurred at: {datetime.fromtimestamp(last_negotiation_cycle_time)}") - idle_job_query = classad.ExprTree(f"JobStatus == 1 && QDate < {last_negotiation_cycle_time}") - idle_job_ads = schedd.query(constraint=idle_job_query.and_(spot_query), - projection=job_attributes) - - total_idle_request_cpus = sum(j[REQUEST_CPUS_ATTRIBUTE] for j in idle_job_ads) - print(f"Total CPUs requested by idle jobs: {total_idle_request_cpus}") - - if self.debug > 1: - print("Information about the compute instance template") - pprint(instanceTemplateInfo) - - # Calculate the minimum number of instances that, for fully packed - # execute points, could satisfy current job queue - min_hosts_for_idle_jobs = math.ceil(total_idle_request_cpus / self.cores_per_node) - if self.debug > 0: - print(f"Minimum hosts needed: {total_idle_request_cpus} / {self.cores_per_node} = {min_hosts_for_idle_jobs}") - - # Get current number of instances in the MIG - requestGroupInfo = self.instanceGroupManagers.get( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - ) - responseGroupInfo = requestGroupInfo.execute() - current_target = responseGroupInfo["targetSize"] - print(f"Current MIG target size: {current_target}") - - # Find instances that are being modified by the MIG (currentAction is - # any value other than "NONE"). A common reason an instance is modified - # is it because it has failed a health check. - reqModifyingInstances = self.instanceGroupManagers.listManagedInstances( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - filter="currentAction != \"NONE\"", - orderBy="creationTimestamp desc" - ) - respModifyingInstances = reqModifyingInstances.execute() - - # Find VMs that are idle (no dynamic slots created from partitionable - # slots) in the MIG handled by this autoscaler - filter_idle_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots==0") - filter_claimed_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots>0") - filter_mig = classad.ExprTree(f"regexp(\".*/{self.instance_group_manager}$\", CloudCreatedBy)") - # A full list of Machine (StartD) ClassAd attributes can be found at - # https://htcondor.readthedocs.io/en/latest/classad-attributes/machine-classad-attributes.html - idle_node_ads = coll.query(htcondor.AdTypes.Startd, - constraint=filter_idle_vms.and_(filter_mig), - projection=["Machine", "CloudZone"]) - - NODENAME_ATTRIBUTE = "Machine" - claimed_node_ads = coll.query(htcondor.AdTypes.Startd, - constraint=filter_claimed_vms.and_(filter_mig), - projection=[NODENAME_ATTRIBUTE]) - claimed_nodes = [ ad[NODENAME_ATTRIBUTE].split(".")[0] for ad in claimed_node_ads] - - # treat OrderedDict as a set by ignoring key values; this set will - # contain VMs we would consider deleting, in inverse order of - # their readiness to join pool (creating, unhealthy, healthy+idle) - idle_nodes = OrderedDict() - try: - modifyingInstances = respModifyingInstances["managedInstances"] - except KeyError: - modifyingInstances = [] - - print(f"There are {len(modifyingInstances)} VMs being modified by the managed instance group") - - # there is potential for nodes in MIG health check "VERIFYING" state - # to have already joined the pool and be running jobs - for instance in modifyingInstances: - self_link = instance["instance"] - node_name = self_link.rsplit("/", 1)[-1] - if node_name not in claimed_nodes: - idle_nodes[self_link] = "modifying" - - for ad in idle_node_ads: - node = ad["Machine"].split(".")[0] - zone = ad["CloudZone"] - self_link = "https://www.googleapis.com/compute/v1/projects/" + \ - self.project + "/zones/" + zone + "/instances/" + node - # there is potential for nodes in MIG health check "VERIFYING" state - # to have already joined the pool and be idle; delete them last - if self_link in idle_nodes: - idle_nodes.move_to_end(self_link) - idle_nodes[self_link] = "idle" - n_idle = len(idle_nodes) - - print(f"There are {n_idle} VMs being modified or idle in the pool") - if self.debug > 1: - print("Listing idle nodes:") - pprint(idle_nodes) - - # always keep size tending toward the minimum idle VMs requested - new_target = current_target + self.compute_instance_min_idle - n_idle + min_hosts_for_idle_jobs - if new_target > self.compute_instance_limit: - self.size = self.compute_instance_limit - print(f"MIG target size will be limited by {self.compute_instance_limit}") - else: - self.size = new_target - - print(f"New MIG target size: {self.size}") - - if self.debug > 1: - print("MIG Information:") - print(responseGroupInfo) - - if self.size == current_target: - if current_target == 0: - print("Queue is empty") - print("Running correct number of VMs to handle queue") - exit() - - if self.size < current_target: - print("Scaling down. Looking for nodes that can be shut down") - - if self.debug > 1: - print("Compute node busy status:") - for node in idle_nodes: - print(node) - - # Shut down idle nodes up to our calculated limit - nodes_to_delete = list(idle_nodes.keys())[0:current_target-self.size] - for node in nodes_to_delete: - print(f"Attempting to delete: {node.rsplit('/',1)[-1]}") - respDel = self.deleteFromMig(nodes_to_delete) - - if self.debug > 1: - print("Scaling down complete") - - if self.size > current_target: - print( - "Scaling up. Need to increase number of instances to " + str(self.size) - ) - # Request to resize - request = self.instanceGroupManagers.resize( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - size=self.size, - ) - response = request.execute() - if self.debug > 1: - print("Requesting to increase MIG size") - pprint(response) - print("Scaling up complete") - - -def main(): - - scaler = AutoScaler(args.mz) - - # Project ID - scaler.project = args.p # Ex:'slurm-var-demo' - - # Name of the zone where the managed instance group is located - scaler.zone = args.z # Ex: 'us-central1-f' - - # Name of the region where the managed instance group is located - scaler.region = args.r # Ex: 'us-central1' - - # The name of the managed instance group. - scaler.instance_group_manager = args.g # Ex: 'condor-compute-igm' - - # Default number of cores per instance, will be replaced with actual value - scaler.cores_per_node = 4 - - # Default number of running instances that the managed instance group should maintain at any given time. This number will go up and down based on the load (number of jobs in the queue) - scaler.size = 0 - - scaler.compute_instance_min_idle = args.i - - # Dry run: : 0, run scaling; 1, only provide info. - scaler.dryrun = args.d > 0 - - # Debug level: 1-print debug information, 2 - print detail debug information - scaler.debug = 0 - if args.v: - scaler.debug = args.v - - # Limit for the maximum number of compute instance. If zero (default setting), no limit will be enforced by the script - scaler.compute_instance_limit = 0 - if args.c: - scaler.compute_instance_limit = abs(args.c) - - scaler.scale() - - -if __name__ == "__main__": - main() diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml deleted file mode 100644 index db989f9d40..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Install but do not activate HTCondor autoscaler - become: true - hosts: localhost - tasks: - - name: Install Python 3 pip - ansible.builtin.package: - name: python3-pip - state: present - - name: Create virtual environment for HTCondor autoscaler - ansible.builtin.pip: - name: pip - version: 21.3.1 # last Python 3.6-compatible release - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv - - name: Install latest setuptools - ansible.builtin.pip: - name: setuptools - version: 59.6.0 # last Python 3.6-compatible release - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv - - name: Install HTCondor autoscaler dependencies - with_items: - - oauth2client - - google-api-python-client - - absl-py - - htcondor - ansible.builtin.pip: - name: "{{ item }}" - state: present # rely on pip resolver to pick latest compatible releases - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml deleted file mode 100644 index 4d3abbbfd6..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The instructions for installing HTCondor may change with time, although we -# anticipate that they will stay fixed for the 23.0 releases. Find up-to-date -# recommendations at: -## https://htcondor.readthedocs.io/en/latest/getting-htcondor/from-our-repositories.html - ---- -- name: Ensure HTCondor is installed - hosts: all - vars: - enable_docker: true - htcondor_key: https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-23.0-Key - docker_key: https://download.docker.com/linux/centos/gpg - become: true - module_defaults: - ansible.builtin.yum: - lock_timeout: 300 - tasks: - - name: Enable EPEL repository - ansible.builtin.yum: - name: - - epel-release - - name: Directly install RPM verification keys - ansible.builtin.rpm_key: - state: present - key: "{{ item }}" - loop: - - "{{ htcondor_key }}" - - "{{ docker_key }}" - register: key_install - retries: 10 - delay: 60 - until: key_install is success - - name: Enable HTCondor LTS Release repository - ansible.builtin.yum_repository: - name: htcondor-feature - description: HTCondor LTS Release (23.0) - file: htcondor - baseurl: https://research.cs.wisc.edu/htcondor/repo/23.0/el$releasever/$basearch/release - gpgkey: "{{ htcondor_key }}" - gpgcheck: true - repo_gpgcheck: true - priority: "90" - - name: Install HTCondor - ansible.builtin.yum: - name: condor-{{ condor_version | default("23.*") | string }} - state: present - - name: Ensure token directory - ansible.builtin.file: - path: /etc/condor/tokens.d - mode: 0700 - owner: root - group: root - - name: Install Docker and configure HTCondor to use it - when: enable_docker | bool # allows string to be passed at CLI - block: - - name: Setup Docker repo - ansible.builtin.yum_repository: - name: docker-ce-stable - description: Docker CE Stable - $basearch - baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable - enabled: yes - gpgcheck: yes - gpgkey: "{{ docker_key }}" - - name: Install Docker - ansible.builtin.yum: - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-compose-plugin - - name: Enable Docker - ansible.builtin.service: - name: docker - state: started - enabled: true - - name: Add condor to docker group - ansible.builtin.user: - name: condor - groups: docker - append: yes diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/main.tf deleted file mode 100644 index 0853e035f4..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/main.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - runners = [ - { - "type" = "ansible-local" - "source" = "${path.module}/files/install-htcondor.yaml" - "destination" = "install-htcondor.yaml" - "args" = join(" ", [ - "-e enable_docker=${var.enable_docker}", - "-e condor_version=${var.condor_version}", - ]) - }, - { - "type" = "ansible-local" - "content" = file("${path.module}/files/install-htcondor-autoscaler-deps.yml") - "destination" = "install-htcondor-autoscaler-deps.yml" - }, - { - "type" = "data" - "content" = file("${path.module}/files/autoscaler.py") - "destination" = "/usr/local/htcondor/bin/autoscaler.py" - }, - ] - - install_htcondor_ps1 = templatefile( - "${path.module}/templates/install-htcondor.ps1.tftpl", { - condor_version = var.condor_version, - http_proxy = var.http_proxy, - python_windows_installer_url = var.python_windows_installer_url, - }) - - required_apis = [ - "compute.googleapis.com", - "secretmanager.googleapis.com", - ] -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf deleted file mode 100644 index c7951737ff..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "runners" { - description = "Runner to install HTCondor using startup-scripts" - value = local.runners -} - -output "windows_startup_ps1" { - description = "Windows PowerShell script to install HTCondor" - value = local.install_htcondor_ps1 -} - -output "gcp_service_list" { - description = "Google Cloud APIs required by HTCondor" - value = local.required_apis -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl deleted file mode 100644 index 7492da3c12..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl +++ /dev/null @@ -1,59 +0,0 @@ -#Requires -RunAsAdministrator - -# Windows 2016 needs forced upgrade to TLS 1.2 -[Net.ServicePointManager]::SecurityProtocol = 'Tls12' - -# important for catching exception in Invoke-WebRequest -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -%{ if http_proxy != "" ~} -[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") -%{ endif ~} - -# do not show progress bar when running Invoke-WebRequest -$ProgressPreference = 'SilentlyContinue' - -# download C Runtime DLL necessary for HTCondor installer -$runtime_installer = 'C:\vc_redist.x64.exe' -Invoke-WebRequest https://aka.ms/vs/17/release/vc_redist.x64.exe -OutFile "$runtime_installer" -Start-Process -FilePath "$runtime_installer" -Wait -ArgumentList "/norestart /quiet /log c:\vc_redist_log.txt" -Remove-Item "$runtime_installer" - -# download HTCondor installer -$htcondor_installer = 'C:\htcondor.msi' -%{ if condor_version == "23.*" } -Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/current/condor-Windows-x64.msi -OutFile "$htcondor_installer" -%{ else ~} -Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/${condor_version}/release/condor-${condor_version}-Windows-x64.msi -OutFile "$htcondor_installer" -%{ endif ~} -$args='/qn /l* condor-install-log.txt /i' -$args=$args + " $htcondor_installer" -$args=$args + ' NEWPOOL="N"' -$args=$args + ' RUNJOBS="N"' -$args=$args + ' SUBMITJOBS="N"' -$args=$args + ' INSTALLDIR="C:\Condor"' -Start-Process "msiexec.exe" -Wait -ArgumentList "$args" -Remove-Item "$htcondor_installer" - -# do not start HTCondor on boot by default. Allow startup script to download -# configuration first and then start HTCondor -Set-Service -StartupType Manual condor - -# remove settings from condor_config that we want to override in configuration step -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^CONDOR_HOST' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^INSTALL_USER' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^DAEMON_LIST' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^use SECURITY' -NotMatch) - -# install Python so that custom ClassAd hooks can execute -$python_installer = 'C:\python-installer.exe' -Invoke-WebRequest -Uri "${python_windows_installer_url}" -OutFile "$python_installer" -Start-Process -FilePath "$python_installer" -Wait -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1 Include_test=0' -%{ if http_proxy == "" ~} -Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --no-warn-script-location requests" -%{ else ~} -Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --proxy ${http_proxy} --no-warn-script-location requests" -%{ endif ~} -Invoke-WebRequest -Uri "https://raw.githubusercontent.com/htcondor/htcondor/main/src/condor_scripts/common-cloud-attributes-google.py" -OutFile "C:\Condor\bin\common-cloud-attributes-google.py" -Remove-Item "$python_installer" diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/variables.tf deleted file mode 100644 index 1afdf4e0eb..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "enable_docker" { - description = "Install and enable docker daemon alongside HTCondor" - type = bool - default = true -} - -variable "condor_version" { - description = "Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: \"23.0.0\",\"23.*\"))" - type = string - default = "23.*" - - validation { - error_message = "var.condor_version must be set to \"23.*\" for latest 23.0 release or to a specific \"23.0.y\" release." - condition = var.condor_version == "23.*" || ( - length(split(".", var.condor_version)) == 3 && alltrue([ - for v in split(".", var.condor_version) : can(tonumber(v)) - ]) && split(".", var.condor_version)[0] == "23" - && split(".", var.condor_version)[1] == "0" - ) - } -} - -variable "http_proxy" { - description = "Set system default web (http and https) proxy for Windows HTCondor installation" - type = string - default = "" - nullable = false -} - -variable "python_windows_installer_url" { - description = "URL of Python installer for Windows" - type = string - default = "https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe" - nullable = false -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/htcondor-install/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/README.md deleted file mode 100644 index 55c2fc7e4e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/README.md +++ /dev/null @@ -1,116 +0,0 @@ -## Description - -This module will create a startup-script runner that will execute Ramble commands. - -Ramble is a multi-platform experimentation framework capable of driving -software installation, acquiring input files, configuring experiments, and -extracting results. For more information about Ramble, see: -https://github.com/GoogleCloudPlatform/ramble - -This module outputs a startup script runner, which can be combined with other -startup script runners to execute a set of Ramble commands. - -Ramble makes extensive use of Spack. It must be installed with a Toolkit runner -generated by the [spack-setup module](../spack-setup/README.md) following the -[basic example](#basic-example) below. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -# Examples - -## Basic Example - -Below is a basic example of using this module. - -```yaml - - id: spack - source: community/modules/scripts/spack-setup - - - id: ramble-setup - source: community/modules/scripts/ramble-setup - - - id: ramble-execute - source: community/modules/scripts/ramble-execute - use: [spack, ramble-setup] - settings: - commands: - - ramble list -``` - -This example shows installing Spack and Ramble with their own modules -(spack-setup and ramble-setup respectively). Then the ramble-execute module -is added to simply list all applications Ramble knows about. - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | -| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [log\_file](#input\_log\_file) | Log file to write output from Ramble execute steps into | `string` | `"/var/log/ramble-execute.log"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by an instance of ramble-setup.
Can be defined explicitly, or by chaining an instance of a ramble-setup module
through a `use` setting. | `string` | n/a | yes | -| [ramble\_runner](#input\_ramble\_runner) | Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | -| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting.
Defaults to /etc/profile.d/spack.sh if not set. | `string` | `"/etc/profile.d/spack.sh"` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the ramble-setup module. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble startup script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for ramble, to be reused by ramble-execute module. | -| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | -| [ramble\_runner](#output\_ramble\_runner) | Runner to execute Ramble commands using an ansible playbook. The startup-script module
will automatically handle installation of ansible. | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to Spack profile script. | -| [startup\_script](#output\_startup\_script) | Ramble startup script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/main.tf deleted file mode 100644 index 7ef0b029e3..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/main.tf +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ramble-execute", ghpc_role = "scripts" }) -} - -locals { - commands_content = var.commands == null ? "echo 'no ramble commands provided'" : indent(4, yamlencode(var.commands)) - - execute_contents = templatefile( - "${path.module}/templates/ramble_execute.yml.tpl", - { - pre_script = "if [ -f ${var.spack_profile_script_path} ]; then . ${var.spack_profile_script_path}; fi; . ${var.ramble_profile_script_path}" - log_file = var.log_file - commands = local.commands_content - system_user_name = var.system_user_name - } - ) - - data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] - - execute_md5 = substr(md5(local.execute_contents), 0, 4) - execute_runner = { - type = "ansible-local" - content = local.execute_contents - destination = "ramble_execute_${local.execute_md5}.yml" - } - - previous_runners = var.ramble_runner != null ? [var.ramble_runner] : [] - runners = concat(local.previous_runners, local.data_runners, [local.execute_runner]) - - # Destinations should be unique while also being known at time of apply - combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) - combined_md5 = substr(md5(local.combined_unique_string), 0, 4) - combined_runner = { - type = "shell" - content = module.startup_script.startup_script - destination = "combined_install_ramble_${local.combined_md5}.sh" - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = var.gcs_bucket_path -} - -resource "local_file" "debug_file_ansible_execute" { - content = local.execute_contents - filename = "${path.module}/debug_execute_${local.execute_md5}.yml" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf deleted file mode 100644 index 4e6c3a44d8..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Ramble startup script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Ramble startup script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "ramble_runner" { - description = <<-EOT - Runner to execute Ramble commands using an ansible playbook. The startup-script module - will automatically handle installation of ansible. - EOT - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for ramble, to be reused by ramble-execute module." - value = var.gcs_bucket_path -} - -output "spack_profile_script_path" { - description = "Path to Spack profile script." - value = var.spack_profile_script_path -} - -output "ramble_profile_script_path" { - description = "Path to Ramble profile script." - value = var.ramble_profile_script_path -} - -output "system_user_name" { - description = "The system user used to execute commands." - value = var.system_user_name -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl deleted file mode 100644 index 0e98f3aa2c..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Execute Commands - hosts: localhost - vars: - pre_script: ${pre_script} - log_file: ${log_file} - commands: ${commands} - system_user_name: ${system_user_name} - tasks: - - name: Execute command block - block: - - name: Print commands to be executed - ansible.builtin.debug: - msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" - - - name: Streaming log info - ansible.builtin.debug: - msg: | - Logs from commands will not be printed here until success (or failure) - Streaming logs can be found at {{ log_file }} - - - name: Ensure user can write to log file - ansible.builtin.file: - path: "{{ log_file }}" - state: touch - owner: "{{ system_user_name }}" - - - name: Execute commands - ansible.builtin.shell: | - set -eo pipefail - { - {{ pre_script }} - echo " === Starting commands ===" - {{ commands }} - echo " === Finished commands ===" - } 2>&1 | tee -a {{ log_file }} - args: - executable: /bin/bash - register: output - become: true - become_user: "{{ system_user_name }}" - - always: - - name: Print commands output - ansible.builtin.debug: - var: output.stdout_lines diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/variables.tf deleted file mode 100644 index ec67228df5..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/variables.tf +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing spack scripts." - type = string -} - -variable "region" { - description = "Region to place bucket containing spack scripts." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "log_file" { - description = "Log file to write output from Ramble execute steps into" - default = "/var/log/ramble-execute.log" - type = string -} - -variable "data_files" { - description = <<-EOT - A list of files to be transferred prior to running commands. - It must specify one of 'source' (absolute local file path) or 'content' (string). - It must specify a 'destination' with absolute path where file should be placed. - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) - error_message = "All destinations must be absolute paths and start with '/'." - } - validation { - condition = alltrue([ - for r in var.data_files : - can(r["content"]) != can(r["source"]) - ]) - error_message = "A data_file must specify either 'content' or 'source', but never both." - } - validation { - condition = alltrue([ - for r in var.data_files : - lookup(r, "content", lookup(r, "source", null)) != null - ]) - error_message = "A data_file must specify a non-null 'content' or 'source'." - } -} - -variable "commands" { - description = "String of commands to run within this module" - default = null - type = string -} - -variable "ramble_runner" { - description = "Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module." - type = object({ - type = string - content = string - destination = string - }) -} - -variable "system_user_name" { - description = "Name of the system user used to execute commands. Generally passed from the ramble-setup module." - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string -} - -variable "spack_profile_script_path" { - description = <<-EOT - Path to the Spack profile.d script. - Can be defined explicitly, or by chaining an instance of a spack-setup module - through a `use` setting. - Defaults to /etc/profile.d/spack.sh if not set. - EOT - type = string - default = "/etc/profile.d/spack.sh" -} - -variable "ramble_profile_script_path" { - description = <<-EOT - Path to the Ramble profile.d script. Created by an instance of ramble-setup. - Can be defined explicitly, or by chaining an instance of a ramble-setup module - through a `use` setting. - EOT - type = string -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/versions.tf deleted file mode 100644 index 9b23317323..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-execute/versions.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/README.md deleted file mode 100644 index 9891088105..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module will create a set of startup-script runners that will setup Ramble, -and install Ramble’s dependencies. - -Ramble is a multi-platform experimentation framework capable of driving -software installation, acquiring input files, configuring experiments, and -extracting results. For more information about ramble, see: -https://github.com/GoogleCloudPlatform/ramble - -This module outputs two startup script runners, which can be added to startup -scripts to setup, ramble and its dependencies. - -For this module to be completely functional, it depends on a spack -installation. For more information, see Cluster-Toolkit’s Spack module. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -# Examples - -## Basic Example - -```yaml -- id: ramble-setup - source: community/modules/scripts/ramble-setup -``` - -This example simply installs ramble on a VM. - -## Full Example - -```yaml -- id: ramble-setup - source: community/modules/scripts/ramble-setup - settings: - install_dir: /ramble - ramble_url: https://github.com/GoogleCloudPlatform/ramble - ramble_ref: v0.2.1 - log_file: /var/log/ramble.log - chown_owner: “owner” - chgrp_group: “user_group” - chmod_mode: “a+r” -``` - -This example simply installs ramble into a VM at the location `/ramble`, checks -out the v0.2.1 tag, changes the owner and group to “owner” and “user_group”, -and chmod’s the clone to make it world readable. - -Also see a more complete [Ramble example blueprint](../../../examples/ramble.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 4.42 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [chmod\_mode](#input\_chmod\_mode) | Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify).
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | -| [install\_dir](#input\_install\_dir) | Destination directory of installation of Ramble. | `string` | `"/apps/ramble"` | no | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by this module | `string` | `"/etc/profile.d/ramble.sh"` | no | -| [ramble\_ref](#input\_ramble\_ref) | Git ref to checkout for Ramble. | `string` | `"develop"` | no | -| [ramble\_url](#input\_ramble\_url) | URL for Ramble repository to clone. | `string` | `"https://github.com/GoogleCloudPlatform/ramble"` | no | -| [ramble\_virtualenv\_path](#input\_ramble\_virtualenv\_path) | Virtual environment path in which to install Ramble Python interpreter and other dependencies | `string` | `"/usr/local/ramble-python"` | no | -| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | -| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Ramble. It will be created if it does not exist. | `string` | `"ramble"` | no | -| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble installation script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module. | -| [ramble\_path](#output\_ramble\_path) | Location ramble is installed into. | -| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | -| [ramble\_ref](#output\_ramble\_ref) | Git ref the ramble install is checked out to use | -| [ramble\_runner](#output\_ramble\_runner) | Runner to be used with startup-script module or passed to ramble-execute module.
- installs Ramble dependencies
- installs Ramble
- generates profile.d script to enable access to Ramble
This is safe to run in parallel by multiple machines. | -| [startup\_script](#output\_startup\_script) | Ramble installation script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/main.tf deleted file mode 100644 index 4389af7d33..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/main.tf +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ramble-setup", ghpc_role = "scripts" }) -} - -locals { - profile_script = <<-EOF - if [ -f ${var.install_dir}/share/ramble/setup-env.sh ]; then - test -t 1 && echo "** Ramble's python virtualenv (/usr/local/ramble-python) is activated. Call 'deactivate' to deactivate." - VIRTUAL_ENV_DISABLE_PROMPT=1 . ${var.ramble_virtualenv_path}/bin/activate - . ${var.install_dir}/share/ramble/setup-env.sh - fi - EOF - - script_content = templatefile( - "${path.module}/templates/ramble_setup.yml.tftpl", - { - sw_name = "ramble" - profile_script = indent(4, yamlencode(local.profile_script)) - install_dir = var.install_dir - git_url = var.ramble_url - git_ref = var.ramble_ref - chmod_mode = var.chmod_mode - system_user_name = var.system_user_name - system_user_uid = var.system_user_uid - system_user_gid = var.system_user_gid - finalize_setup_script = "echo 'no finalize setup script'" - profile_script_path = var.ramble_profile_script_path - } - ) - - install_ramble_deps_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/install_ramble_deps.yml" - "destination" = "install_ramble_deps.yml" - "args" = "-e virtualenv_path=${var.ramble_virtualenv_path}" - } - - python_reqs_content = templatefile( - "${path.module}/templates/install_ramble_python_deps.yml.tftpl", - { - install_dir = var.install_dir - virtualenv_path = var.ramble_virtualenv_path - } - ) - - python_reqs_runner = { - "type" = "ansible-local" - "content" = local.python_reqs_content - "destination" = "install_ramble_reqs.yml" - } - - install_ramble_runner = { - "type" = "ansible-local" - "content" = local.script_content - "destination" = "install_ramble.yml" - } - - bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}"), 0, 8) - # Max bucket name length is 63, so truncate deployment_name if necessary. - # The string "-ramble-scripts-" is 16 characters and bucket_md5 is 8 characters, - # leaving 63-16-8=39 chars for deployment_name. - bucket_name = "${substr(var.deployment_name, 0, 39)}-ramble-scripts-${local.bucket_md5}" - runners = [local.install_ramble_deps_runner, local.install_ramble_runner, local.python_reqs_runner] - - combined_runner = { - "type" = "shell" - "content" = module.startup_script.startup_script - "destination" = "ramble-install-and-setup.sh" - } - -} - -resource "google_storage_bucket" "bucket" { - project = var.project_id - name = local.bucket_name - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" -} - -resource "local_file" "debug_file_shell_install" { - content = local.script_content - filename = "${path.module}/debug_install.yml" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf deleted file mode 100644 index e587470eac..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Ramble installation script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Ramble installation script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "ramble_runner" { - description = <<-EOT - Runner to be used with startup-script module or passed to ramble-execute module. - - installs Ramble dependencies - - installs Ramble - - generates profile.d script to enable access to Ramble - This is safe to run in parallel by multiple machines. - EOT - value = local.combined_runner -} - -output "ramble_path" { - description = "Location ramble is installed into." - value = var.install_dir -} - -output "ramble_ref" { - description = "Git ref the ramble install is checked out to use" - value = var.ramble_ref -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module." - value = "gs://${google_storage_bucket.bucket.name}" -} - -output "ramble_profile_script_path" { - description = "Path to Ramble profile script." - value = var.ramble_profile_script_path -} - -output "system_user_name" { - description = "The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands." - value = var.system_user_name -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml deleted file mode 100644 index b7905bbe9e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Create python virtual env for a tool - become: yes - hosts: localhost - vars: - virtualenv_path: ${virtualenv_path} - tasks: - - name: Install dependencies through system package manager - ansible.builtin.package: - name: - - python3 - - python3-pip - - git - register: package - changed_when: package.changed - retries: 5 - delay: 10 - until: package is success - - - name: Create virtualenv for tool - # Python 3.6 is minimum we wish to support due to ease of installation on - # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip - # supported by 3.6. Additionally, recent versions of pip are necessary for - # proper dependency resolution of real-world problems with google-cloud-* - # (and third-party) Python packages (20.3+ probably effective minimum). - ansible.builtin.pip: - name: pip>=21.3.1 - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv - - - name: Add google-cloud-storage to virtualenv - ansible.builtin.pip: - name: google-cloud-storage - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl deleted file mode 100644 index ea14780a58..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Install Python Requirements - hosts: localhost - vars: - install_dir: ${install_dir} - virtualenv_path: ${virtualenv_path} - tasks: - - - name: Install dependencies - ansible.builtin.pip: - requirements: "{{ install_dir }}/requirements.txt" - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl deleted file mode 100644 index ca48a5afa0..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Install Software - hosts: localhost - vars: - sw_name: ${sw_name} - profile_script: ${profile_script} - install_dir: ${install_dir} - git_url: ${git_url} - git_ref: ${git_ref} - chmod_mode: ${chmod_mode} - system_user_name: ${system_user_name} - system_user_uid: ${system_user_uid} - system_user_gid: ${system_user_gid} - finalize_setup_script: ${finalize_setup_script} - profile_script_path: ${profile_script_path} - tasks: - - name: Print software name - ansible.builtin.debug: - msg: "Running installation for software: {{ sw_name }}" - - - name: Add profile script for software - ansible.builtin.copy: - dest: "{{ profile_script_path }}" - mode: '0644' - content: "{{ profile_script }}" - when: profile_script - - - name: Look up user to use for install - block: - - - name: Check if user already exists - ansible.builtin.getent: - database: passwd - key: "{{ system_user_name }}" - - - name: Look up existing user details - ansible.builtin.user: - name: "{{ system_user_name }}" - register: system_user - - rescue: - - name: User did not exist, create group for system user - ansible.builtin.group: - name: "{{ system_user_name }}" - gid: "{{ system_user_gid }}" - system: true - register: system_group - - - name: Create system user - ansible.builtin.user: - name: "{{ system_user_name }}" - comment: "{{ sw_name }} installation" - uid: "{{ system_user_uid }}" - group: "{{ system_group.name }}" - system: true - register: system_user - - - name: Create parent of install directory - ansible.builtin.file: - path: "{{ install_dir | dirname }}" - state: directory - - - name: Set lock dir - ansible.builtin.set_fact: - lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" - - - name: Acquire lock - ansible.builtin.command: - mkdir "{{ lock_dir }}" - register: lock_out - changed_when: lock_out.rc == 0 - failed_when: false - - - name: Add hostname to lock_dir - ansible.builtin.file: - path: "{{ lock_dir }}/{{ ansible_hostname }}" - state: touch - when: lock_out.rc == 0 - - - name: Clone branch or tag into installation directory - ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} - failed_when: false - register: clone_res - when: lock_out.rc == 0 - - - name: Clone commit hash into installation directory - ansible.builtin.command: "{{ item }}" - with_items: - - git clone {{ git_url }} {{ install_dir }} - - git -C {{ install_dir }} checkout {{ git_ref }} - when: lock_out.rc == 0 and clone_res.rc != 0 - - - name: Transfer ownership to system user - ansible.builtin.file: - path: "{{ install_dir }}" - owner: "{{ system_user.name }}" - group: "{{ system_user.group }}" - recurse: true - follow: false - when: lock_out.rc == 0 - - - name: Finalize setup - ansible.builtin.shell: "{{ finalize_setup_script }}" - when: lock_out.rc == 0 and finalize_setup_script - become: true - become_user: "{{ system_user.name }}" - - - name: Apply chmod - ansible.builtin.file: - path: "{{ install_dir }}" - mode: "{{ chmod_mode | default(omit, true) }}" - recurse: true - follow: false - when: (lock_out.rc == 0) and (chmod_mode != None) - - - name: Release lock - ansible.builtin.file: - path: "{{ lock_dir }}/done" - state: touch - when: lock_out.rc == 0 - - - name: Wait for lock - block: - - name: Wait for lock - ansible.builtin.wait_for: - path: "{{ lock_dir }}/done" - state: present - timeout: 600 - sleep: 10 - when: lock_out.rc != 0 - - rescue: - - name: Timed out on waiting for lock, get lock directory contents - ansible.builtin.find: - paths: "{{ lock_dir }}" - register: lock_dir_contents - - - name: Print lock directory contents, it should contain name of host that is holding lock - ansible.builtin.debug: - msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" - - - name: Failed to get lock - ansible.builtin.fail: - msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/variables.tf deleted file mode 100644 index 0d3a8eed05..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/variables.tf +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "install_dir" { - description = "Destination directory of installation of Ramble." - default = "/apps/ramble" - type = string -} - -variable "ramble_url" { - description = "URL for Ramble repository to clone." - default = "https://github.com/GoogleCloudPlatform/ramble" - type = string -} - -variable "ramble_ref" { - description = "Git ref to checkout for Ramble." - default = "develop" - type = string -} - -variable "chmod_mode" { - description = <<-EOT - Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify). - For usage information see: - https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode - EOT - default = "" - type = string - nullable = false -} - -variable "system_user_name" { - description = "Name of system user that will perform installation of Ramble. It will be created if it does not exist." - default = "ramble" - type = string - nullable = false -} - -variable "system_user_uid" { - description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." - default = 1104762904 - type = number - nullable = false -} - -variable "system_user_gid" { - description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." - default = 1104762904 - type = number - nullable = false -} - -variable "ramble_virtualenv_path" { - description = "Virtual environment path in which to install Ramble Python interpreter and other dependencies" - default = "/usr/local/ramble-python" - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing startup script." - type = string -} - -variable "region" { - description = "Region to place bucket containing startup script." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "ramble_profile_script_path" { - description = "Path to the Ramble profile.d script. Created by this module" - type = string - default = "/etc/profile.d/ramble.sh" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/versions.tf deleted file mode 100644 index 936b4a5b80..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/ramble-setup/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/README.md deleted file mode 100644 index 8cbb75fb42..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/README.md +++ /dev/null @@ -1,141 +0,0 @@ -## Description - -This module creates a script that defines a software build using Spack and -performs any additional customization to a Spack installation. - -There are two main variable inputs that can be used to define a Spack build: -`data_files` and `commands`. - -- `data_files`: Any files specified will be transferred to the machine running - outputted script. Data file `content` can be defined inline in the blueprint - or can point to a `source`, an absolute local path of a file. This can be used - to transfer environment definition files, config definition files, GPG keys, - or software licenses. `data_files` are transferred before `commands` are run. -- `commands`: A script that is run. This can be used to perform actions such as - installation of compilers & packages, environment creation, adding a build - cache, and modifying the spack configuration. - -## Example - -The `spack-execute` module should `use` a `spack-setup` module. This will -prepend the installation of Spack and its dependencies to the build. Then -`spack-execute` can be used by a module that takes `startup-script` as an input. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: builder-vm - source: modules/compute/vm-instance - use: [network1, spack-build] -``` - -To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. - -[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml - -### Using with `startup-script` module - -The `spack-runner` output can be used by the `startup-script` module. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(spack-build.spack-runner) - - type: shell - destination: "my-script.sh" - content: echo 'hello world' - - - id: workstation - source: modules/compute/vm-instance - use: [network1, startup-script] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | -| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [log\_file](#input\_log\_file) | Defines the logfile that script output will be written to | `string` | `"/var/log/spack.log"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by an instance of spack-setup.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting. | `string` | n/a | yes | -| [spack\_runner](#input\_spack\_runner) | Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | -| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the spack-setup module. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack startup script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | -| [spack\_runner](#output\_spack\_runner) | Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules. | -| [startup\_script](#output\_startup\_script) | Spack startup script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/main.tf deleted file mode 100644 index 04ebcf7d49..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "spack-execute", ghpc_role = "scripts" }) -} - -locals { - commands_content = var.commands == null ? "echo 'no spack commands provided'" : indent(4, yamlencode(var.commands)) - - execute_contents = templatefile( - "${path.module}/templates/execute_commands.yml.tpl", - { - pre_script = ". ${var.spack_profile_script_path}" - log_file = var.log_file - commands = local.commands_content - system_user_name = var.system_user_name - } - ) - - data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] - - execute_md5 = substr(md5(local.execute_contents), 0, 4) - execute_runner = { - type = "ansible-local" - content = local.execute_contents - destination = "spack_execute_${local.execute_md5}.yml" - } - - runners = concat([var.spack_runner], local.data_runners, [local.execute_runner]) - - # Destinations should be unique while also being known at time of apply - combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) - combined_md5 = substr(md5(local.combined_unique_string), 0, 4) - combined_runner = { - type = "shell" - content = module.startup_script.startup_script - destination = "combined_install_spack_${local.combined_md5}.sh" - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = var.gcs_bucket_path -} - -resource "local_file" "debug_file_ansible_execute" { - content = local.execute_contents - filename = "${path.module}/debug_execute_${local.execute_md5}.yml" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/outputs.tf deleted file mode 100644 index 4a52532d51..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/outputs.tf +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Spack startup script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Spack startup script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "spack_runner" { - description = "Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules." - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." - value = var.gcs_bucket_path -} - -output "spack_profile_script_path" { - description = "Path to the Spack profile.d script." - value = var.spack_profile_script_path -} - -output "system_user_name" { - description = "The system user used to execute commands." - value = var.system_user_name -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl deleted file mode 100644 index 0e98f3aa2c..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Execute Commands - hosts: localhost - vars: - pre_script: ${pre_script} - log_file: ${log_file} - commands: ${commands} - system_user_name: ${system_user_name} - tasks: - - name: Execute command block - block: - - name: Print commands to be executed - ansible.builtin.debug: - msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" - - - name: Streaming log info - ansible.builtin.debug: - msg: | - Logs from commands will not be printed here until success (or failure) - Streaming logs can be found at {{ log_file }} - - - name: Ensure user can write to log file - ansible.builtin.file: - path: "{{ log_file }}" - state: touch - owner: "{{ system_user_name }}" - - - name: Execute commands - ansible.builtin.shell: | - set -eo pipefail - { - {{ pre_script }} - echo " === Starting commands ===" - {{ commands }} - echo " === Finished commands ===" - } 2>&1 | tee -a {{ log_file }} - args: - executable: /bin/bash - register: output - become: true - become_user: "{{ system_user_name }}" - - always: - - name: Print commands output - ansible.builtin.debug: - var: output.stdout_lines diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/variables.tf deleted file mode 100644 index 851cd1aed8..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/variables.tf +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing spack scripts." - type = string -} - -variable "region" { - description = "Region to place bucket containing spack scripts." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "log_file" { - description = "Defines the logfile that script output will be written to" - default = "/var/log/spack.log" - type = string -} - -variable "data_files" { - description = <<-EOT - A list of files to be transferred prior to running commands. - It must specify one of 'source' (absolute local file path) or 'content' (string). - It must specify a 'destination' with absolute path where file should be placed. - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) - error_message = "All destinations must be absolute paths and start with '/'." - } - validation { - condition = alltrue([ - for r in var.data_files : - can(r["content"]) != can(r["source"]) - ]) - error_message = "A data_file must specify either 'content' or 'source', but never both." - } - validation { - condition = alltrue([ - for r in var.data_files : - lookup(r, "content", lookup(r, "source", null)) != null - ]) - error_message = "A data_file must specify a non-null 'content' or 'source'." - } -} - -variable "commands" { - description = "String of commands to run within this module" - type = string - default = null -} - -variable "spack_runner" { - description = "Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module." - type = object({ - type = string - content = string - destination = string - }) -} - -variable "system_user_name" { - description = "Name of the system user used to execute commands. Generally passed from the spack-setup module." - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string -} - -variable "spack_profile_script_path" { - description = <<-EOT - Path to the Spack profile.d script. Created by an instance of spack-setup. - Can be defined explicitly, or by chaining an instance of a spack-setup module - through a `use` setting. - EOT - type = string -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/versions.tf deleted file mode 100644 index 09583c3d43..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-execute/versions.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 1.0.0" - required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/README.md deleted file mode 100644 index 01d3e6d389..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/README.md +++ /dev/null @@ -1,382 +0,0 @@ -## Description - -This module can be used to setup and install Spack on a VM. To actually run -Spack commands to install other software use the -[spack-execute](../spack-execute/) module. - -This module generates a script that performs the following: - -1. Install system dependencies needed for Spack -1. Clone Spack into a predefined directory -1. Check out a specific version of Spack - -There are several options on how to consume the outputs of this module: - -> [!IMPORTANT] -> Breaking changes between after v1.21.0. `spack-install` module replaced by -> `spack-setup` and `spack-execute` modules. -> [Details Below](#deprecations-and-breaking-changes) - -## Examples - -### `use` `spack-setup` with `spack-execute` - -This will prepend the `spack-setup` script to the `spack-execute` commands. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: builder - source: modules/compute/vm-instance - use: [network1, spack-build] -``` - -### `use` `spack-setup` with `vm-instance` or Slurm module - -This will run `spack-setup` scripts on the downstream compute resource. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-installer - source: modules/compute/vm-instance - use: [network1, spack-setup] -``` - -OR - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: [network1, partition1, spack-setup] -``` - -### Build `starup-script` with `spack-runner` output - -This will use the generated `spack-setup` script as one step in `startup-script`. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(spack-setup.spack-runner) - - type: shell - destination: "my-script.sh" - content: echo 'hello world' - - - id: workstation - source: modules/compute/vm-instance - use: [network1, startup-script] -``` - -To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. - -[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml - -## Environment Setup - -### Activating Spack - -[Spack installation] produces a setup script that adds `spack` to your `PATH` as -well as some other command-line integration tools. This script can be found at -`/share/spack/setup-env.sh`. This script will be automatically -added to bash startup by any machine that runs the `spack_runner`. - -If you have multiple machines that all want to use the same shared Spack -installation you can just have both machines run the `spack_runner`. - -[Spack installation]: https://spack-tutorial.readthedocs.io/en/latest/tutorial_basics.html#installing-spack - -### Managing Spack Python dependencies - -Spack is configured with [SPACK_PYTHON] to ensure that Spack itself uses a -Python virtual environment with a supported copy of Python with the package -`google-cloud-storage` pre-installed. This enables Spack to use mirrors and -[build caches][builds] on Google Cloud Storage. It does not configure Python -packages *inside* Spack virtual environments. If you need to add more Python -dependencies for Spack itself, use the `spack python` command: - -```shell -sudo -i spack python -m pip install package-name -``` - -[SPACK_PYTHON]: https://spack.readthedocs.io/en/latest/getting_started.html#shell-support -[builds]: https://spack.readthedocs.io/en/latest/binary_caches.html - -## Spack Permissions - -### System `spack` user is created - Default - -By default this module will create a `spack` linux user and group with -consistent UID and GID. This user and group will own the Spack installation. To -allow a user to manually add Spack packages to the system Spack installation, -you can add the user to the spack group: - -```sh -sudo usermod -a -G spack -``` - -Log out and back in so the group change will take effect, then `` will -be able to call `spack install `. - -> [!NOTE] -> A background persistent SSH connections may prevent the group change from -> taking effect. - -You can use the `system_user_name`, `system_user_uid`, and `system_user_gid` to -customize the name and ids of the system user. While unlikely, it is possible -that the default `system_user_uid` or `system_user_gid` could conflict with -existing UIDs. - -### Use and existing user - -Alternatively, if `system_user_name` is a user already on the system, then this -existing user will be used for Spack installation. - -#### OS Login User - -If OS Login is enabled (default for most Cluster Toolkit modules) then you can -provide an OS Login user name: - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - system_user_name: username_company_com -``` - -This will work even if the user has not yet logged onto the machine. When the -specified user does log on to the machine they will be able to call -`spack install` without any further configuration. - -#### Pre-configured user - -You can also use a startup script to configure a user: - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - system_user_name: special-user - - - id: startup - source: modules/scripts/startup-script - settings: - runners: - - type: shell - destination: "create_user.sh" - content: | - #!/bin/bash - sudo useradd -u 799 special-user - sudo groupadd -g 922 org-group - sudo usermod -g org-group special-user - - $(spack-setup.spack_runner) - - - id: spack-vms - source: modules/compute/vm-instance - use: [network1, startup] - settings: - name_prefix: spack-vm - machine_type: n2d-standard-2 - instance_count: 5 -``` - -### Chaining spack installations - -If there is a need to have a non-root user to install spack packages it is -recommended to create a separate installation for that user and chain Spack installations -([Spack docs](https://spack.readthedocs.io/en/latest/chain.html#chaining-spack-installations)). - -Steps to chain Spack installations: - -1. Get the version of the system Spack: - - ```sh - $ spack --version - - 0.20.0 (e493ab31c6f81a9e415a4b0e0e2263374c61e758) - # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - # Note commit hash and use in next step - ``` - -1. Clone a new spack installation: - - ```sh - git clone -c feature.manyFiles=true https://github.com/spack/spack.git /spack - git -C /spack checkout - ``` - -1. Point the new Spack installation to the system Spack installation. Create a - file at `/spack/etc/spack/upstreams.yaml` with the following - contents: - - ```yaml - upstreams: - spack-instance-1: - install_tree: /sw/spack/opt/spack/ - ``` - -1. Add the following line to your `.bashrc` to make sure the new `spack` is in - your `PATH`. - - ```sh - . /spack/share/spack/setup-env.sh - ``` - -## Deprecations and Breaking Changes - -The old `spack-install` module has been replaced by the `spack-setup` and -`spack-execute` modules. Generally this change strives to allow for a more -flexible definition of a Spack build by using native Spack commands. - -For every deprecated variable from `spack-install` there is documentation on how -to perform the equivalent action using `commands` and `data_files`. The -documentation can be found on the [inputs table](#inputs) below. - -Below is a simple example of the same functionality shown before and after the -breaking changes. - -```yaml - # Before - - id: spack-install - source: community/modules/scripts/spack-install - settings: - install_dir: /sw/spack - compilers: - - gcc@10.3.0 target=x86_64 - packages: - - intel-mpi@2018.4.274%gcc@10.3.0 - -- id: spack-startup - source: modules/scripts/startup-script - settings: - runners: - - $(spack.install_spack_deps_runner) - - $(spack.install_spack_runner) -``` - -```yaml - # After - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - install_dir: /sw/spack - - - id: spack-execute - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - spack load gcc@10.3.0 target=x86_64 - spack compiler find --scope site - spack install intel-mpi@2018.4.274%gcc@10.3.0 - -- id: spack-startup - source: modules/scripts/startup-script - settings: - runners: - - $(spack-execute.spack-runner) -``` - -Although the old `spack-install` module will no longer be maintained, it is -still possible to use the old module in a blueprint by referencing an old -version from GitHub. Note the source line in the following example. - -```yaml - - id: spack-install - source: github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/spack-install?ref=v1.22.1&depth=1 -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 4.42 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [chmod\_mode](#input\_chmod\_mode) | `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification.
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `"g+w"` | no | -| [configure\_for\_google](#input\_configure\_for\_google) | When true, the spack installation will be configured to pull from Google's Spack binary cache. | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | -| [install\_dir](#input\_install\_dir) | Directory to install spack into. | `string` | `"/sw/spack"` | no | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by this module | `string` | `"/etc/profile.d/spack.sh"` | no | -| [spack\_ref](#input\_spack\_ref) | Git ref to checkout for spack. | `string` | `"v0.20.0"` | no | -| [spack\_url](#input\_spack\_url) | URL to clone the spack repo from. | `string` | `"https://github.com/spack/spack"` | no | -| [spack\_virtualenv\_path](#input\_spack\_virtualenv\_path) | Virtual environment path in which to install Spack Python interpreter and other dependencies | `string` | `"/usr/local/spack-python"` | no | -| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Spack. It will be created if it does not exist. | `string` | `"spack"` | no | -| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack installation script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | -| [spack\_path](#output\_spack\_path) | Path to the root of the spack installation | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | -| [spack\_runner](#output\_spack\_runner) | Runner to be used with startup-script module or passed to spack-execute module.
- installs Spack dependencies
- installs Spack
- generates profile.d script to enable access to Spack
This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. | -| [startup\_script](#output\_startup\_script) | Spack installation script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Spack. It can be reused by spack-execute module to install spack packages. | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/main.tf deleted file mode 100644 index d45f5d1be3..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/main.tf +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "spack-setup", ghpc_role = "scripts" }) -} - -locals { - profile_script = <<-EOF - SPACK_PYTHON=${var.spack_virtualenv_path}/bin/python3 - if [ -f ${var.install_dir}/share/spack/setup-env.sh ]; then - test -t 1 && echo "Running Spack setup, this may take a moment on first login." - . ${var.install_dir}/share/spack/setup-env.sh - fi - EOF - - supported_cache_versions = ["v0.19.0", "v0.20.0"] - cache_version = contains(local.supported_cache_versions, var.spack_ref) ? var.spack_ref : "latest" - add_google_mirror_script = !var.configure_for_google ? "" : <<-EOF - if ! spack mirror list | grep -q google_binary_cache; then - spack mirror add --scope site google_binary_cache gs://spack/${local.cache_version} - spack buildcache keys --install --trust - fi - EOF - - finalize_setup_script = <<-EOF - set -e - . ${var.spack_profile_script_path} - spack config --scope site add 'packages:all:permissions:read:world' - spack config --scope site add 'packages:all:permissions:write:group' - spack gpg init - spack compiler find --scope site - ${local.add_google_mirror_script} - # perform fast install to make sure Spack is fully initialized - spack install xz - spack uninstall --yes-to-all xz - EOF - - script_content = templatefile( - "${path.module}/templates/spack_setup.yml.tftpl", - { - sw_name = "spack" - profile_script = indent(4, yamlencode(local.profile_script)) - install_dir = var.install_dir - git_url = var.spack_url - git_ref = var.spack_ref - chmod_mode = var.chmod_mode - system_user_name = var.system_user_name - system_user_uid = var.system_user_uid - system_user_gid = var.system_user_gid - finalize_setup_script = indent(4, yamlencode(local.finalize_setup_script)) - profile_script_path = var.spack_profile_script_path - } - ) - - install_spack_deps_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/install_spack_deps.yml" - "destination" = "install_spack_deps.yml" - "args" = "-e virtualenv_path=${var.spack_virtualenv_path}" - } - install_spack_runner = { - "type" = "ansible-local" - "content" = local.script_content - "destination" = "install_spack.yml" - } - - bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}.${local.script_content}"), 0, 8) - # Max bucket name length is 63, so truncate deployment_name if necessary. - # The string "-spack-scripts-" is 15 characters and bucket_md5 is 8 characters, - # leaving 63-15-8=40 chars for deployment_name. Using 39 so it has the same prefix as the - # ramble-setup module's GCS bucket. - bucket_name = "${substr(var.deployment_name, 0, 39)}-spack-scripts-${local.bucket_md5}" - runners = [local.install_spack_deps_runner, local.install_spack_runner] - - combined_runner = { - "type" = "shell" - "content" = module.startup_script.startup_script - "destination" = "spack-install-and-setup.sh" - } -} - -resource "google_storage_bucket" "bucket" { - project = var.project_id - name = local.bucket_name - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" -} - -resource "local_file" "debug_file_shell_install" { - content = local.script_content - filename = "${path.module}/debug_install.yml" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml deleted file mode 100644 index 2ada34471f..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/outputs.tf deleted file mode 100644 index d94b9757db..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/outputs.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Spack installation script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Spack installation script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "spack_path" { - description = "Path to the root of the spack installation" - value = var.install_dir -} - -output "spack_runner" { - description = <<-EOT - Runner to be used with startup-script module or passed to spack-execute module. - - installs Spack dependencies - - installs Spack - - generates profile.d script to enable access to Spack - This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. - EOT - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." - value = "gs://${google_storage_bucket.bucket.name}" -} - -output "spack_profile_script_path" { - description = "Path to the Spack profile.d script." - value = var.spack_profile_script_path -} - -output "system_user_name" { - description = "The system user used to install Spack. It can be reused by spack-execute module to install spack packages." - value = var.system_user_name -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml deleted file mode 100644 index b7905bbe9e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Create python virtual env for a tool - become: yes - hosts: localhost - vars: - virtualenv_path: ${virtualenv_path} - tasks: - - name: Install dependencies through system package manager - ansible.builtin.package: - name: - - python3 - - python3-pip - - git - register: package - changed_when: package.changed - retries: 5 - delay: 10 - until: package is success - - - name: Create virtualenv for tool - # Python 3.6 is minimum we wish to support due to ease of installation on - # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip - # supported by 3.6. Additionally, recent versions of pip are necessary for - # proper dependency resolution of real-world problems with google-cloud-* - # (and third-party) Python packages (20.3+ probably effective minimum). - ansible.builtin.pip: - name: pip>=21.3.1 - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv - - - name: Add google-cloud-storage to virtualenv - ansible.builtin.pip: - name: google-cloud-storage - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl deleted file mode 100644 index ca48a5afa0..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Install Software - hosts: localhost - vars: - sw_name: ${sw_name} - profile_script: ${profile_script} - install_dir: ${install_dir} - git_url: ${git_url} - git_ref: ${git_ref} - chmod_mode: ${chmod_mode} - system_user_name: ${system_user_name} - system_user_uid: ${system_user_uid} - system_user_gid: ${system_user_gid} - finalize_setup_script: ${finalize_setup_script} - profile_script_path: ${profile_script_path} - tasks: - - name: Print software name - ansible.builtin.debug: - msg: "Running installation for software: {{ sw_name }}" - - - name: Add profile script for software - ansible.builtin.copy: - dest: "{{ profile_script_path }}" - mode: '0644' - content: "{{ profile_script }}" - when: profile_script - - - name: Look up user to use for install - block: - - - name: Check if user already exists - ansible.builtin.getent: - database: passwd - key: "{{ system_user_name }}" - - - name: Look up existing user details - ansible.builtin.user: - name: "{{ system_user_name }}" - register: system_user - - rescue: - - name: User did not exist, create group for system user - ansible.builtin.group: - name: "{{ system_user_name }}" - gid: "{{ system_user_gid }}" - system: true - register: system_group - - - name: Create system user - ansible.builtin.user: - name: "{{ system_user_name }}" - comment: "{{ sw_name }} installation" - uid: "{{ system_user_uid }}" - group: "{{ system_group.name }}" - system: true - register: system_user - - - name: Create parent of install directory - ansible.builtin.file: - path: "{{ install_dir | dirname }}" - state: directory - - - name: Set lock dir - ansible.builtin.set_fact: - lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" - - - name: Acquire lock - ansible.builtin.command: - mkdir "{{ lock_dir }}" - register: lock_out - changed_when: lock_out.rc == 0 - failed_when: false - - - name: Add hostname to lock_dir - ansible.builtin.file: - path: "{{ lock_dir }}/{{ ansible_hostname }}" - state: touch - when: lock_out.rc == 0 - - - name: Clone branch or tag into installation directory - ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} - failed_when: false - register: clone_res - when: lock_out.rc == 0 - - - name: Clone commit hash into installation directory - ansible.builtin.command: "{{ item }}" - with_items: - - git clone {{ git_url }} {{ install_dir }} - - git -C {{ install_dir }} checkout {{ git_ref }} - when: lock_out.rc == 0 and clone_res.rc != 0 - - - name: Transfer ownership to system user - ansible.builtin.file: - path: "{{ install_dir }}" - owner: "{{ system_user.name }}" - group: "{{ system_user.group }}" - recurse: true - follow: false - when: lock_out.rc == 0 - - - name: Finalize setup - ansible.builtin.shell: "{{ finalize_setup_script }}" - when: lock_out.rc == 0 and finalize_setup_script - become: true - become_user: "{{ system_user.name }}" - - - name: Apply chmod - ansible.builtin.file: - path: "{{ install_dir }}" - mode: "{{ chmod_mode | default(omit, true) }}" - recurse: true - follow: false - when: (lock_out.rc == 0) and (chmod_mode != None) - - - name: Release lock - ansible.builtin.file: - path: "{{ lock_dir }}/done" - state: touch - when: lock_out.rc == 0 - - - name: Wait for lock - block: - - name: Wait for lock - ansible.builtin.wait_for: - path: "{{ lock_dir }}/done" - state: present - timeout: 600 - sleep: 10 - when: lock_out.rc != 0 - - rescue: - - name: Timed out on waiting for lock, get lock directory contents - ansible.builtin.find: - paths: "{{ lock_dir }}" - register: lock_dir_contents - - - name: Print lock directory contents, it should contain name of host that is holding lock - ansible.builtin.debug: - msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" - - - name: Failed to get lock - ansible.builtin.fail: - msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/variables.tf deleted file mode 100644 index 85baeec401..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/variables.tf +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -# spack-setup variables - -variable "install_dir" { - description = "Directory to install spack into." - type = string - default = "/sw/spack" -} - -variable "spack_url" { - description = "URL to clone the spack repo from." - type = string - default = "https://github.com/spack/spack" -} - -variable "spack_ref" { - description = "Git ref to checkout for spack." - type = string - default = "v0.20.0" -} - -variable "configure_for_google" { - description = "When true, the spack installation will be configured to pull from Google's Spack binary cache." - type = bool - default = true -} - - -variable "chmod_mode" { - description = <<-EOT - `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification. - For usage information see: - https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode - EOT - default = "g+w" - type = string - nullable = false -} - -variable "system_user_name" { - description = "Name of system user that will perform installation of Spack. It will be created if it does not exist." - default = "spack" - type = string - nullable = false -} - -variable "system_user_uid" { - description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." - default = 1104762903 - type = number - nullable = false -} - -variable "system_user_gid" { - description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." - default = 1104762903 - type = number - nullable = false -} - -variable "spack_virtualenv_path" { - description = "Virtual environment path in which to install Spack Python interpreter and other dependencies" - default = "/usr/local/spack-python" - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing startup script." - type = string -} - -variable "region" { - description = "Region to place bucket containing startup script." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "spack_profile_script_path" { - description = "Path to the Spack profile.d script. Created by this module" - type = string - default = "/etc/profile.d/spack.sh" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/versions.tf deleted file mode 100644 index ff1180fc1b..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/spack-setup/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/README.md deleted file mode 100644 index ee9c057c39..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/README.md +++ /dev/null @@ -1,87 +0,0 @@ -## Description - -This module will insert a dependency on the completion of the startup script -for one or more specified compute VMs and report back if it fails. This can be useful when running -post-boot installation scripts that require the startup script to finish setting up a node. - -> **_WARNING:_**: this module is experimental and not fully supported. - -### Additional Dependencies - -* [**gcloud**](https://cloud.google.com/sdk/gcloud) must be present in the path - of the machine where `terraform apply` is run. - -### Example - -```yaml -- id: workstation - source: modules/compute/vm-instance - use: - - network1 - - my-startup-script - settings: - instance_count: 4 - -# Wait for all instances of the above VM to finish running startup scripts. -- id: wait - source: community/modules/scripts/wait-for-startup - settings: - instance_names: $(workstation.name) -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.validate_instance_names](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.wait_for_startup](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | -| [instance\_name](#input\_instance\_name) | Name of the instance we are waiting for (can be null if 'instance\_names' is not empty) | `string` | `null` | no | -| [instance\_names](#input\_instance\_names) | A list of instance names we are waiting for, in addition to the one mentioned in 'instance\_name' (if any) | `list(string)` | `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [timeout](#input\_timeout) | Timeout in seconds | `number` | `1200` | no | -| [zone](#input\_zone) | The GCP zone where the instance is running | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/main.tf deleted file mode 100644 index 3f6b416251..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/main.tf +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - combined_instance_names = concat(var.instance_names, [var.instance_name]) -} - -resource "null_resource" "validate_instance_names" { - lifecycle { - precondition { - condition = var.instance_name != null || length(var.instance_names) > 0 - error_message = "At least one instance name must be provided" - } - } -} - -resource "null_resource" "wait_for_startup" { - count = length(local.combined_instance_names) - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/wait-for-startup-status.sh" - environment = { - INSTANCE_NAME = self.triggers.instance_name - ZONE = var.zone - PROJECT_ID = var.project_id - TIMEOUT = var.timeout - GCLOUD_PATH = var.gcloud_path_override - } - } - - triggers = { - instance_name = local.combined_instance_names[count.index] - } -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf deleted file mode 100644 index 11a2ddf118..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh deleted file mode 100644 index fae5833121..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -z "${INSTANCE_NAME}" ]]; then - echo "INSTANCE_NAME is unset... exiting" - exit 0 -fi -if [[ -z "${ZONE}" ]]; then - echo "ZONE is unset" - exit 1 -fi -if [[ -z "${PROJECT_ID}" ]]; then - echo "PROJECT_ID is unset" - exit 1 -fi -if [[ -z "${TIMEOUT}" ]]; then - echo "TIMEOUT is unset" - exit 1 -fi - -if [[ -n "${GCLOUD_PATH}" ]]; then - export PATH="$GCLOUD_PATH:$PATH" -fi - -echo "Waiting for startup: instance_name='${INSTANCE_NAME}', zone='${ZONE}', project_id='${PROJECT_ID}', timeout_seconds='${TIMEOUT}'" - -# Wrapper around grep that swallows the error status code 1 -c1grep() { grep "$@" || test $? = 1; } - -now=$(date +%s) - -# If VM was created more than 30 days ago, serial port logs may no longer exist. -# Exit without errors if the instance is older than 30 days. -logsExpiryDays=30 -createdTimestampIso=$(gcloud compute instances describe "${INSTANCE_NAME}" --project "${PROJECT_ID}" --zone "${ZONE}" --format "value(creationTimestamp)") -earliestAllowedCreatedTimestamp=$(date -d "${createdTimestampIso} +${logsExpiryDays} day" +%s) -if [[ "$earliestAllowedCreatedTimestamp" -lt "$now" ]]; then - echo "Instance was created more than 30 days ago - serial port 1 logs are likely expired... exiting" - exit 0 -fi - -deadline=$((now + TIMEOUT)) -error_file=$(mktemp) -fetch_cmd="gcloud compute instances get-serial-port-output ${INSTANCE_NAME} --port 1 --zone ${ZONE} --project ${PROJECT_ID}" -# Match string for all finish types of the old guest agent and successful -# finishes on the new guest agent -FINISH_LINE="startup-script exit status" -# Match string for failures on the new guest agent -FINISH_LINE_ERR="Script \"startup-script\" failed with error:" - -# NEW: Accept also these finish lines as success. -STARTUP_SCRIPT_SUCCEEDED_LINE="google-startup-scripts.service: Succeeded." -STARTUP_SCRIPT_FINISHED_LINE="Finished Google Compute Engine Startup Scripts." -STARTUP_SCRIPT_SERVICE_FINISHED_LINE="Finished google-startup-scripts.service - Google Compute Engine Startup Scripts." - -NON_FATAL_ERRORS=( - "Internal error" -) - -until [[ now -gt deadline ]]; do - ser_log=$( - set -o pipefail - ${fetch_cmd} 2>"${error_file}" | - c1grep "${FINISH_LINE}\|${FINISH_LINE_ERR}\|${STARTUP_SCRIPT_SUCCEEDED_LINE}\|${STARTUP_SCRIPT_FINISHED_LINE}\|${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}" - ) || { - err=$(cat "${error_file}") - echo "$err" - fatal_error="true" - for e in "${NON_FATAL_ERRORS[@]}"; do - if [[ $err = *"$e"* ]]; then - fatal_error="false" - break - fi - done - - if [[ $fatal_error = "true" ]]; then - exit 1 - fi - } - if [[ -n "${ser_log}" ]]; then break; fi - sleep 5 - now=$(date +%s) -done - -# This line checks for an exit code - the assumption is that there is a number -# at the end of the line and it is an exit code. -# Modified to correctly extract the last numeric exit status from the relevant log line. -LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=Script \"startup-script\" failed with error: exit status )[0-9]+" | tail -n 1) -if [[ -z "${LAST_EXIT_STATUS}" ]]; then - LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=startup-script exit status )[0-9]+" | tail -n 1) -fi - -# This specific text is monitored for in tests, do not change. -INSPECT_OUTPUT_TEXT="To inspect the startup script output, please run:" - -# --- Prioritize explicit failure from the script itself --- -if [[ "${LAST_EXIT_STATUS}" == 1 ]]; then - echo "startup-script finished with errors, ${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" - exit 1 -# --- Then explicit success from the script itself --- -elif [[ "${LAST_EXIT_STATUS}" == 0 ]]; then - echo "startup-script finished successfully" - exit 0 -elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SUCCEEDED_LINE}"; then - echo "startup-script finished successfully (startup script succeeded line detected)" - exit 0 -elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_FINISHED_LINE}"; then - echo "startup-script finished successfully (startup script finished line detected)" - exit 0 -elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}"; then - echo "startup-script finished successfully (startup script service finished line detected)" - exit 0 -# --- If we reached deadline, it's a timeout --- -elif [[ now -ge deadline ]]; then - echo "startup-script timed out after ${TIMEOUT} seconds" - echo "${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" - exit 1 -# --- All other cases are considered failure or invalid state --- -else - echo "Invalid or undetermined startup script status. Last detected exit status: '${LAST_EXIT_STATUS}'" - echo "${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" - exit "${LAST_EXIT_STATUS}" -fi diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf deleted file mode 100644 index fe6410a920..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "instance_name" { - description = "Name of the instance we are waiting for (can be null if 'instance_names' is not empty)" - type = string - default = null -} - -variable "instance_names" { - description = "A list of instance names we are waiting for, in addition to the one mentioned in 'instance_name' (if any)" - type = list(string) - default = [] -} - -variable "zone" { - description = "The GCP zone where the instance is running" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "timeout" { - description = "Timeout in seconds" - type = number - default = 1200 - validation { - condition = var.timeout >= 0 - error_message = "The timeout should be non-negative" - } -} - -variable "gcloud_path_override" { - description = "Directory of the gcloud executable to be used during cleanup" - type = string - default = "" - nullable = false -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf deleted file mode 100644 index 8cd43b944e..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/README.md b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/README.md deleted file mode 100644 index fc25bc0a55..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/README.md +++ /dev/null @@ -1,109 +0,0 @@ -## Description - -This module contains a set of scripts to be used in customizing Windows VMs at -boot or during image building. Please note that the installation of NVIDIA GPU -drivers takes, at minimum, 30-60 minutes. It is therefore recommended to build -a custom image and reuse it as shown below, rather than install GPU drivers at -boot time. - -> NOTE: the output `windows_startup_ps1` must be passed explicitly as shown -> below when used with Packer modules. This is due to a limitation in the `use` -> keyword and inputs of type `list` in Packer modules; this does not impact -> Terraform modules - -### NVIDIA Drivers and CUDA Toolkit - -Many Google Cloud VM families include or can have NVIDIA GPUs attached to them. -This module supports GPU applications by enabling you to easily install -a compatible release of NVIDIA drivers and of the CUDA Toolkit. The script is -the [solution recommended by our documentation][docs] and is [directly sourced -from GitHub][script-src]. - -[docs]: https://cloud.google.com/compute/docs/gpus/install-drivers-gpu#windows -[script-src]: https://github.com/GoogleCloudPlatform/compute-gpu-installation/blob/24dac3004360e0696c49560f2da2cd60fcb80107/windows/install_gpu_driver.ps1 - -```yaml -- group: primary - modules: - - id: network1 - source: modules/network/vpc - settings: - enable_iap_rdp_ingress: true - enable_iap_winrm_ingress: true - - - id: windows_startup - source: community/modules/scripts/windows-startup-script - settings: - install_nvidia_driver: true - -- group: packer - modules: - - id: image - source: modules/packer/custom-image - kind: packer - use: - - network1 - - windows_startup - settings: - source_image_family: windows-2016 - machine_type: n1-standard-8 - accelerator_count: 1 - accelerator_type: nvidia-tesla-t4 - disk_size: 75 - disk_type: pd-ssd - omit_external_ip: false - state_timeout: 15m -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [http\_proxy](#input\_http\_proxy) | Set http and https proxy for use by Invoke-WebRequest commands | `string` | `""` | no | -| [http\_proxy\_set\_environment](#input\_http\_proxy\_set\_environment) | Set system default environment variables http\_proxy and https\_proxy for all commands | `bool` | `false` | no | -| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install\_nvidia\_driver\_script | `bool` | `false` | no | -| [install\_nvidia\_driver\_args](#input\_install\_nvidia\_driver\_args) | Arguments to supply to NVIDIA driver install script | `string` | `"/s /n"` | no | -| [install\_nvidia\_driver\_script](#input\_install\_nvidia\_driver\_script) | Install script for NVIDIA drivers specified by http/https URL | `string` | `"https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe"` | no | -| [no\_proxy](#input\_no\_proxy) | Environment variables no\_proxy (only used if var.http\_proxy\_set\_environment is enabled) | `string` | `"169.254.169.254,metadata,metadata.google.internal,.googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | A string list of scripts selected by this module | - diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/main.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/main.tf deleted file mode 100644 index 5e6bc8b94d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/main.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - setx_http_proxy_ps1 = !var.http_proxy_set_environment ? [] : [ - templatefile("${path.module}/templates/setx_http_proxy.ps1", { - "http_proxy" : var.http_proxy, - "no_proxy" : var.no_proxy, - }) - ] - - nvidia_ps1 = !var.install_nvidia_driver ? [] : [ - templatefile("${path.module}/templates/install_gpu_driver.ps1.tftpl", { - "url" : var.install_nvidia_driver_script - "args" : var.install_nvidia_driver_args - "http_proxy" : var.http_proxy, - }) - ] - - startup_ps1 = concat(local.setx_http_proxy_ps1, local.nvidia_ps1) -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf deleted file mode 100644 index 006ea312ad..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "windows_startup_ps1" { - description = "A string list of scripts selected by this module" - value = local.startup_ps1 -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl deleted file mode 100644 index 55c4a2a3cd..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl +++ /dev/null @@ -1,38 +0,0 @@ -#Requires -RunAsAdministrator - -# Windows 2016 needs forced upgrade to TLS 1.2 -[Net.ServicePointManager]::SecurityProtocol = 'Tls12' - -# important for catching exception in Invoke-WebRequest -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -%{ if http_proxy != "" } -[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") -%{ endif } - -# Create the folder for the driver download -$file_dir = 'C:\NVIDIA-Driver\nvidia_installer_windows.exe' -if (!(Test-Path -Path 'C:\NVIDIA-Driver')) { - New-Item -Path 'C:\' -Name 'NVIDIA-Driver' -ItemType 'directory' | Out-Null -} - -# Download the file to a specified directory -Write-Output "Downloading ${url} to $file_dir" -# Disabling progress bar has surprising large (10-100x) impact on speed -$ProgressPreference = 'SilentlyContinue' -try { - Invoke-WebRequest -Uri "${url}" -OutFile "$file_dir" -} catch { - Write-Output "$_" - throw "Failed to download ${url}; exiting startup script" -} - -# Install the file with the specified path from earlier as well as the RunAs admin option -Write-Output "Executing $file_dir with arguments '${args}'" -try { - Start-Process -FilePath "$file_dir" -ArgumentList '${args}' -Wait -} catch { - Write-Output "$_" - throw "Could not install NVIDIA driver; exiting startup script" -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 deleted file mode 100644 index ca4d13f98b..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 +++ /dev/null @@ -1,21 +0,0 @@ -<# - Copyright 2025 "Google LLC" - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -#> - -#Requires -RunAsAdministrator - -setx http_proxy ${http_proxy} /m -setx https_proxy ${http_proxy} /m -setx no_proxy ${no_proxy} /m diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf deleted file mode 100644 index 9e4fb9e67d..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "install_nvidia_driver" { - description = "Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install_nvidia_driver_script" - type = bool - default = false -} - -variable "install_nvidia_driver_script" { - description = "Install script for NVIDIA drivers specified by http/https URL" - type = string - default = "https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe" -} - -variable "install_nvidia_driver_args" { - description = "Arguments to supply to NVIDIA driver install script" - type = string - default = "/s /n" -} - -variable "http_proxy" { - description = "Set http and https proxy for use by Invoke-WebRequest commands" - type = string - default = "" - nullable = false -} - -variable "http_proxy_set_environment" { - description = "Set system default environment variables http_proxy and https_proxy for all commands" - type = bool - default = false - nullable = false -} - -variable "no_proxy" { - description = "Environment variables no_proxy (only used if var.http_proxy_set_environment is enabled)" - type = string - default = "169.254.169.254,metadata,metadata.google.internal,.googleapis.com" - nullable = false -} diff --git a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf b/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf deleted file mode 100644 index dfeeac34f8..0000000000 --- a/deletion-test/build_script/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/build_script/modules/embedded/modules/README.md b/deletion-test/build_script/modules/embedded/modules/README.md deleted file mode 100644 index 6886b3f330..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/README.md +++ /dev/null @@ -1,554 +0,0 @@ -# Modules - -This directory contains a set of core modules built for the Cluster Toolkit. Modules -describe the building blocks of an AI/ML and HPC deployment. The expected fields in a -module are listed in more detail [below](#module-fields). Blueprints can be -extended in functionality by incorporating [modules from GitHub -repositories][ghmods]. - -[ghmods]: #github-modules - -## All Modules - -Modules from various sources are all listed here for visibility. Badges are used -to indicate the source and status of many of these resources. - -Modules listed below with the ![core-badge] badge are located in this -folder and are tested and maintained by the Cluster Toolkit team. - -Modules labeled with the ![community-badge] badge are contributed by -the community (including the Cluster Toolkit team, partners, etc.). Community modules -are located in the [community folder](../community/modules/README.md). - -Modules labeled with the ![deprecated-badge] badge are now deprecated and may be -removed in the future. Customers are advised to transition to alternatives. - -Modules that are still in development and less stable are labeled with the -![experimental-badge] badge. - -[core-badge]: https://img.shields.io/badge/-core-blue?style=plastic -[community-badge]: https://img.shields.io/badge/-community-%23b8def4?style=plastic -[stable-badge]: https://img.shields.io/badge/-stable-lightgrey?style=plastic -[experimental-badge]: https://img.shields.io/badge/-experimental-%23febfa2?style=plastic -[deprecated-badge]: https://img.shields.io/badge/-deprecated-%23fea2a2?style=plastic - -### Compute - -* **[vm-instance]** ![core-badge] : Creates one or more VM instances. -* **[schedmd-slurm-gcp-v6-partition]** ![core-badge] : - Creates a partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. -* **[schedmd-slurm-gcp-v6-nodeset]** ![core-badge] : - Creates a nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. -* **[schedmd-slurm-gcp-v6-nodeset-tpu]** ![core-badge] : - Creates a TPU nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. -* **[schedmd-slurm-gcp-v6-nodeset-dynamic]** ![core-badge] ![experimental-badge]: - Creates a dynamic nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module and instance template. -* **[gke-node-pool]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes node pool using GKE. -* **[resource-policy]** ![core-badge] ![experimental-badge] : Create a resource policy for compute engines that can be applied to gke-node-pool's nodes. -* **[gke-job-template]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes job file to be used with a [gke-node-pool]. -* **[htcondor-execute-point]** ![community-badge] ![experimental-badge] : - Manages a group of execute points for use in an [HTCondor - pool][htcondor-setup]. -* **[mig]** ![community-badge] ![experimental-badge] : Creates a Managed Instance Group. -* **[notebook]** ![community-badge] ![experimental-badge] : Creates a Vertex AI - Notebook. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. -* **[gke-nodeset]** ![community-badge] ![experimental-badge] : Create a slinky nodeset to be used by the [gke-partition] module. -* **[gke-partition]** ![community-badge] ![experimental-badge] : Creates a slinky partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. - -[vm-instance]: compute/vm-instance/README.md -[gke-node-pool]: ../modules/compute/gke-node-pool/README.md -[resource-policy]: ../modules/compute/resource-policy/README.md -[gke-job-template]: ../modules/compute/gke-job-template/README.md -[schedmd-slurm-gcp-v6-partition]: ../community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md -[schedmd-slurm-gcp-v6-nodeset]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md -[schedmd-slurm-gcp-v6-nodeset-tpu]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md -[schedmd-slurm-gcp-v6-nodeset-dynamic]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md -[htcondor-execute-point]: ../community/modules/compute/htcondor-execute-point/README.md -[mig]: ../community/modules/compute/mig/README.md -[notebook]: ../community/modules/compute/notebook/README.md -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -### Database - -* **[slurm-cloudsql-federation]** ![community-badge] ![experimental-badge] : - Creates a [Google SQL Instance](https://cloud.google.com/sql/) meant to be - integrated with a [slurm-controller][schedmd-slurm-gcp-v6-controller]. -* **[bigquery-dataset]** ![community-badge] ![experimental-badge] : Creates a BQ - dataset. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. -* **[bigquery-table]** ![community-badge] ![experimental-badge] : Creates a BQ - table. Primarily used for - [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. - -[slurm-cloudsql-federation]: ../community/modules/database/slurm-cloudsql-federation/README.md -[bigquery-dataset]: ../community/modules/database/bigquery-dataset/README.md -[bigquery-table]: ../community/modules/database/bigquery-table/README.md -[fsi-montecarlo-on-batch]: ../community/modules/files/fsi-montecarlo-on-batch/README.md - -### File System - -* **[filestore]** ![core-badge] : Creates a - [filestore](https://cloud.google.com/filestore) file system. -* **[parallelstore]** ![core-badge] ![experimental-badge]: Creates a - [parallelstore](https://cloud.google.com/parallelstore) file system. -* **[pre-existing-network-storage]** ![core-badge] : Specifies a - pre-existing file system that can be mounted on a VM. -* **[managed-lustre]** ![core-badge] ![experimental-badge]: Creates a - [managed-lustred](https://cloud.google.com/managed-lustre) file system. -* **[DDN-EXAScaler]** ![community-badge] ![deprecated-badge] : Creates - a [DDN EXAscaler lustre](https://www.ddn.com/partners/google-cloud-platform/) - file system. This module is deprecated and will be removed by July 1, 2025. Consider migrating to managed-lustre. -* **[cloud-storage-bucket]** ![core-badge] : Creates a Google Cloud Storage (GCS) bucket. -* **[gke-persistent-volume]** ![core-badge] ![experimental-badge] : Creates - persistent volumes and persistent volume claims for shared storage. -* **[nfs-server]** ![community-badge] ![experimental-badge] : Creates a VM and - configures an NFS server that can be mounted by other VM. -* **[weka-client]** ![community-badge] ![experimental-badge] : Installs client - and mounts [WEKA](https://www.weka.io/) filesystems. - -[filestore]: file-system/filestore/README.md -[parallelstore]: file-system/parallelstore/README.md -[pre-existing-network-storage]: file-system/pre-existing-network-storage/README.md -[managed-lustre]: file-system/managed-lustre/README.md -[ddn-exascaler]: ../community/modules/file-system/DDN-EXAScaler/README.md -[nfs-server]: ../community/modules/file-system/nfs-server/README.md -[cloud-storage-bucket]: file-system/cloud-storage-bucket/README.md -[gke-persistent-volume]: file-system/gke-persistent-volume/README.md -[weka-client]: ../community/modules/file-system/weka-client/README.md - -### Monitoring - -* **[dashboard]** ![core-badge] : Creates a - [monitoring dashboard](https://cloud.google.com/monitoring/dashboards) for - visually tracking a Cluster Toolkit deployment. - -[dashboard]: monitoring/dashboard/README.md - -### Network - -* **[vpc]** ![core-badge] : Creates a - [Virtual Private Cloud (VPC)](https://cloud.google.com/vpc) network with - regional subnetworks and firewall rules. -* **[multivpc]** ![core-badge] ![experimental-badge]: Creates a variable - number of VPC networks using the [vpc] module. -* **[pre-existing-vpc]** ![core-badge] : Used to connect newly - built components to a pre-existing VPC network. -* **[firewall-rules]** ![core-badge] ![experimental-badge] : Add custom firewall - rules to existing networks (commonly used with [pre-existing-vpc]). -* **[private-service-access]** ![community-badge] ![experimental-badge] : - Configures Private Services Access for a VPC network (commonly used with [filestore] and [slurm-cloudsql-federation]). - -[vpc]: network/vpc/README.md -[multivpc]: network/multivpc/README.md -[pre-existing-vpc]: network/pre-existing-vpc/README.md -[firewall-rules]: network/firewall-rules/README.md -[private-service-access]: ../community/modules/network/private-service-access/README.md - -### Packer - -* **[custom-image]** ![core-badge] : Creates a custom VM Image - based on the GCP HPC VM image. - -[custom-image]: packer/custom-image/README.md - -### Project - -* **[service-account]** ![community-badge] ![experimental-badge] : Creates [service - accounts](https://cloud.google.com/iam/docs/service-accounts) for a GCP - project. -* **[service-enablement]** ![community-badge] ![experimental-badge] : Allows enabling - various APIs for a Google Cloud Project. - -[service-account]: ../community/modules/project/service-account/README.md -[service-enablement]: ../community/modules/project/service-enablement/README.md - -### Pub/Sub - -* **[topic]** ![community-badge] ![experimental-badge] : Creates a -Pub/Sub topic. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. -* **[bigquery-sub]** ![community-badge] ![experimental-badge] : Creates a -Pub/Sub subscription. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. - -[topic]: ../community/modules/pubsub/topic/README.md -[bigquery-sub]: ../community/modules/pubsub/bigquery-sub/README.md - -### Remote Desktop - -* **[chrome-remote-desktop]** ![community-badge] ![experimental-badge] : Creates - a GPU accelerated Chrome Remote Desktop. - -[chrome-remote-desktop]: ../community/modules/remote-desktop/chrome-remote-desktop/README.md - -### Scheduler - -* **[batch-job-template]** ![core-badge] : Creates a Google Cloud Batch job - template that works with other Toolkit modules. -* **[batch-login-node]** ![core-badge] : Creates a VM that can be used for - submission of Google Cloud Batch jobs. -* **[gke-cluster]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes cluster using GKE. -* **[pre-existing-gke-cluster]** ![core-badge] ![experimental-badge] : Retrieves an existing GKE cluster. Substitute for ([gke-cluster]) module. -* **[schedmd-slurm-gcp-v6-controller]** ![core-badge] : - Creates a Slurm controller node. -* **[schedmd-slurm-gcp-v6-login]** ![core-badge] : - Creates a Slurm login node. -* **[htcondor-setup]** ![community-badge] ![experimental-badge] : Creates the - base infrastructure for an HTCondor pool (service accounts and Cloud Storage bucket). -* **[htcondor-pool-secrets]** ![community-badge] ![experimental-badge] : Creates - and manages access to the secrets necessary for secure operation of an - HTCondor pool. -* **[htcondor-access-point]** ![community-badge] ![experimental-badge] : Creates - a regional instance group managing a highly available HTCondor access point - (login node). - -[batch-job-template]: ../modules/scheduler/batch-job-template/README.md -[batch-login-node]: ../modules/scheduler/batch-login-node/README.md -[gke-cluster]: ../modules/scheduler/gke-cluster/README.md -[pre-existing-gke-cluster]: ../modules/scheduler/pre-existing-gke-cluster/README.md -[htcondor-setup]: ../community/modules/scheduler/htcondor-setup/README.md -[htcondor-pool-secrets]: ../community/modules/scheduler/htcondor-pool-secrets/README.md -[htcondor-access-point]: ../community/modules/scheduler/htcondor-access-point/README.md -[schedmd-slurm-gcp-v6-controller]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md -[schedmd-slurm-gcp-v6-login]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md - -### Scripts - -* **[startup-script]** ![core-badge] : Creates a customizable startup script - that can be fed into compute VMs. -* **[windows-startup-script]** ![community-badge] ![experimental-badge]: Creates - Windows PowerShell (PS1) scripts that can be used to customize Windows VMs - and VM images. -* **[htcondor-install]** ![community-badge] ![experimental-badge] : Creates - a startup script to install HTCondor and exports a list of required APIs -* **[ramble-execute]** ![community-badge] ![experimental-badge] : Creates a - startup script to execute - [Ramble](https://github.com/GoogleCloudPlatform/ramble) commands on a target - VM -* **[ramble-setup]** ![community-badge] ![experimental-badge] : Creates a - startup script to install - [Ramble](https://github.com/GoogleCloudPlatform/ramble) on an instance or a - slurm login or controller. -* **[spack-setup]** ![community-badge] ![experimental-badge] : Creates a startup - script to install [Spack](https://github.com/spack/spack) on an instance or a - slurm login or controller. -* **[spack-execute]** ![community-badge] ![experimental-badge] : Defines a - software build using [Spack](https://github.com/spack/spack). -* **[wait-for-startup]** ![community-badge] ![experimental-badge] : Waits for - successful completion of a startup script on a compute VM. - -[startup-script]: scripts/startup-script/README.md -[windows-startup-script]: ../community/modules/scripts/windows-startup-script/README.md -[htcondor-install]: ../community/modules/scripts/htcondor-install/README.md -[kubernetes-operations]: ../community/modules/scripts/kubernetes-operations/README.md -[ramble-execute]: ../community/modules/scripts/ramble-execute/README.md -[ramble-setup]: ../community/modules/scripts/ramble-setup/README.md -[spack-setup]: ../community/modules/scripts/spack-setup/README.md -[spack-execute]: ../community/modules/scripts/spack-execute/README.md -[wait-for-startup]: ../community/modules/scripts/wait-for-startup/README.md - -## Module Fields - -### ID (Required) - -The `id` field is used to uniquely identify and reference a defined module. -ID's are used in [variables](../examples/README.md#variables) and become the -name of each module when writing the terraform `main.tf` file. They are also -used in the [use](#use-optional) and [outputs](#outputs-optional) lists -described below. - -For terraform modules, the ID will be rendered into the terraform module label -at the top level main.tf file. - -### Source (Required) - -The source is a path or URL that points to the source files for Packer or -Terraform modules. A source can either be a filesystem path or a URL to a git -repository: - -* Filesystem paths - * modules embedded in the `gcluster` executable - * modules in the local filesystem -* Remote modules using [Terraform URL syntax](https://developer.hashicorp.com/terraform/language/modules/sources) - * Hosted on [GitHub](https://developer.hashicorp.com/terraform/language/modules/sources#github) - * Google Cloud Storage [Buckets](https://developer.hashicorp.com/terraform/language/modules/sources#gcs-bucket) - * Generic [git repositories](https://developer.hashicorp.com/terraform/language/modules/sources#generic-git-repository) - - when modules are in a subdirectory of the git repository, a special - double-slash `//` notation can be required as described below - -An important distinction is that those URLs are natively supported by Terraform so -they are not copied to your deployment directory. Packer does not have native -support for git-hosted modules so the Toolkit will copy these modules into the -deployment folder on your behalf. - -#### Embedded Modules - -Embedded modules are added to the gcluster binary during compilation and cannot -be edited. To refer to embedded modules, set the source path to -`modules/<>` or `community/modules/<>`. - -The paths match the modules in the repository structure for [core modules](./) -and [community modules](../community/modules/). Because the modules are embedded -during compilation, your local copies may differ unless you recompile gcluster. - -For example, this example snippet uses the embedded pre-existing-vpc module: - -```yaml - - id: network1 - source: modules/network/pre-existing-vpc -``` - -#### Local Modules - -Local modules point to a module in the file system and can easily be edited. -They are very useful during module development. To use a local module, set -the source to a path starting with `/`, `./`, or `../`. For instance, the -following module definition refers the local pre-existing-vpc modules. - -```yaml - - id: network1 - source: modules/network/pre-existing-vpc -``` - -> **_NOTE:_** Relative paths (beginning with `.` or `..` must be relative to the -> working directory from which `gcluster` is executed. This example would have to be -> run from a local copy of the Cluster Toolkit repository. An alternative is to use -> absolute paths to modules. - -#### GitHub-hosted Modules and Packages - -To use a Terraform module available on GitHub, set the source to a path starting -with `github.com` (HTTPS) or `git@github.com` (SSH). For instance, the following -module definition sources the Toolkit vpc module: - -```yaml - - id: network1 - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc -``` - -This example uses the [double-slash notation][tfsubdir] (`//`) to indicate that -the Toolkit is a "package" of multiple modules whose root directory is the root -of the git repository. The remainder of the path indicates the sub-directory of -the vpc module. - -The example above uses the default `main` branch of the Toolkit. Specific -[revisions][tfrev] can be selected with any valid [git reference][gitref]. -(git branch, commit hash or tag). If the git reference is a tag or branch, we -recommend setting `&depth=1` to reduce the data transferred over the network. -This option cannot be set when the reference is a commit hash. The following -examples select the vpc module on the active `develop` branch and also an older -release of the filestore module: - -```yaml - - id: network1 - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc?ref=develop - ... - - id: homefs - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/file-system/filestore?ref=v1.22.1&depth=1 -``` - -Because Terraform modules natively support this syntax, gcluster will not copy -GitHub-hosted modules into your deployment folder. Terraform will download them -into a hidden folder when you run `terraform init`. - -[tfrev]: https://www.terraform.io/language/modules/sources#selecting-a-revision -[gitref]: https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection#_single_revisions -[tfsubdir]: https://www.terraform.io/language/modules/sources#modules-in-package-sub-directories - -##### GitHub-hosted Packer modules - -Packer does not natively support GitHub-hosted modules so `gcluster create` will -copy modules into your deployment folder. - -If the module uses `//` package notation, `gcluster create` will copy the entire -repository to the module path: `deployment_name/group_name/module_id`. However, -when `gcluster deploy` is invoked, it will run Packer from the subdirectory -`deployment_name/group_name/module_id/subdirectory/after/double_slash`. - -If the module does not use `//` package notation, `gcluster create` will copy -only the final directory in the path to `deployment_name/group_name/module_id`. - -In all cases, `gcluster create` will remove the `.git` directory from the packer -module to ensure that you can manage the entire deployment directory with its -own git versioning. - -##### GitHub over SSH - -Get module from GitHub over SSH: - -```yaml - - id: network1 - source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc -``` - -Specific versions can be selected as for HTTPS: - -```yaml - - id: network1 - source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc?ref=v1.22.1&depth=1 -``` - -##### Generic Git Modules - -To use a Terraform module available in a non-GitHub git repository such as -gitlab, set the source to a path starting `git::`. Two Standard git protocols -are supported, `git::https://` for HTTPS or `git::git@github.com` for SSH. - -Additional formatting and features after `git::` are identical to that of the -[GitHub Modules](#github-modules) described above. - -#### Google Cloud Storage Modules - -To use a Terraform module available in a Google Cloud Storage bucket, set the source -to a URL with the special `gcs::` prefix, followed by a [GCS bucket object URL](https://cloud.google.com/storage/docs/request-endpoints#typical). - -For example: `gcs::https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH_TO_MODULE` - -### Kind (May be Required) - -`kind` refers to the way in which a module is deployed. Currently, `kind` can be -either `terraform` or `packer`. It must be specified for modules of type -`packer`. If omitted, it will default to `terraform`. - -### Settings (May Be Required) - -The settings field is a map that supplies any user-defined variables for each -module. Settings values can be simple strings, numbers or booleans, but can -also support complex data types like maps and lists of variable depth. These -settings will become the values for the variables defined in either the -`variables.tf` file for Terraform or `variable.pkr.hcl` file for Packer. - -For some modules, there are mandatory variables that must be set, -therefore `settings` is a required field in that case. In many situations, a -combination of sensible defaults, deployment variables and used modules can -populated all required settings and therefore the settings field can be omitted. - -### Use (Optional) - -The `use` field is a powerful way of linking a module to one or more other -modules. When a module "uses" another module, the outputs of the used -module are compared to the settings of the current module. If they have -matching names and the setting has no explicit value, then it will be set to -the used module's output. For example, see the following blueprint snippet: - -```yaml -modules: -- id: network1 - source: modules/network/vpc - -- id: workstation - source: modules/compute/vm-instance - use: [network1] - settings: - ... -``` - -In this snippet, the VM instance `workstation` uses the outputs of vpc -`network1`. - -In this case both `network_self_link` and `subnetwork_self_link` in the -[workstation settings](compute/vm-instance/README.md#Inputs) will be set -to `$(network1.network_self_link)` and `$(network1.subnetwork_self_link)` which -refer to the [network1 outputs](network/vpc/README#Outputs) -of the same names. - -The order of precedence that `gcluster` uses in determining when to infer a setting -value is in the following priority order: - -1. Explicitly set in the blueprint using the `settings` field -1. Output from a used module, taken in the order provided in the `use` list -1. Deployment variable (`vars`) of the same name -1. Default value for the setting - -> **_NOTE:_** See the -> [network storage documentation](./../docs/network_storage.md) for more -> information about mounting network storage file systems via the `use` field. - -### Outputs (Optional) - -The `outputs` field adds the output of individual Terraform modules to the -output of its deployment group. This enables the value to be available via -`terraform output`. This can useful for displaying the IP of a login node or -printing instructions on how to use a module, as we have in the -[monitoring dashboard module](monitoring/dashboard/README.md#Outputs). - -The outputs field is a lists that it can be in either of two formats: a string -equal to the name of the module output, or a map specifying the `name`, -`description`, and whether the value is `sensitive` and should be suppressed -from the standard output of Terraform commands. An example is shown below -that displays the internal and public IP addresses of a VM created by the -vm-instance module: - -```yaml - - id: vm - source: modules/compute/vm-instance - use: - - network1 - settings: - machine_type: e2-medium - outputs: - - internal_ip - - name: external_ip - description: "External IP of VM" - sensitive: true -``` - -The outputs shown after running Terraform apply will resemble: - -```text -Apply complete! Resources: 7 added, 0 changed, 0 destroyed. - -Outputs: - -external_ip_simplevm = -internal_ip_simplevm = [ - "10.128.0.19", -] -``` - -### Required Services (APIs) (optional) - -Each Toolkit module depends upon Google Cloud services ("APIs") being enabled -in the project used by the AI/ML and HPC environment. For example, the [creation of -VMs](compute/vm-instance/) requires the Compute Engine API -(compute.googleapis.com). The [startup-script](scripts/startup-script/) module -requires the Cloud Storage API (storage.googleapis.com) for storage of the -scripts themselves. Each module included in the Toolkit source code describes -its required APIs internally. The Toolkit will merge the requirements from all -modules and [automatically validate](../README.md#blueprint-validation) that all -APIs are enabled in the project specified by `$(vars.project_id)`. - -## Common Settings - -The following common naming conventions should be used to decrease the verbosity -needed to define a blueprint. This is intentional to allow multiple -modules to share inferred settings from deployment variables or from other -modules listed under the `use` field. - -For example, if all modules are to be created in a single region, that region -can be defined as a deployment variable named `region`, which is shared between -all modules without an explicit setting. Similarly, if many modules need to be -connected to the same VPC network, they all can add the vpc module ID to their -`use` list so that `network_self_link` would be inferred from that vpc module rather -than having to set it manually. - -* **project_id**: The GCP project ID in which to create the GCP resources. -* **deployment_name**: The name of the current deployment of a blueprint. This - can help to avoid naming conflicts of modules when multiple deployments are - created from the same blueprint. -* **region**: The GCP - [region](https://cloud.google.com/compute/docs/regions-zones) the module - will be created in. -* **zone**: The GCP [zone](https://cloud.google.com/compute/docs/regions-zones) - the module will be created in. -* **labels**: - [Labels](https://cloud.google.com/resource-manager/docs/creating-managing-labels) - added to the module. In order to include any module in advanced - monitoring, labels must be exposed. We strongly recommend that all modules - expose this variable. - -## Writing Custom Cluster Toolkit Modules - -Modules are flexible by design, however we define some [best practices](../docs/module-guidelines.md) when -creating a new module meant to be used with the Cluster Toolkit. diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/README.md b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/README.md deleted file mode 100644 index f807cd727e..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/README.md +++ /dev/null @@ -1,133 +0,0 @@ -## Description - -This module is used to create a Kubernetes job template file. - -The job template file can be submitted as is or used as a template for further -customization. Add the `instructions` output to a blueprint (as shown below) to -get instructions on how to use `kubectl` to submit the job. - -This module is designed to `use` one or more `gke-node-pool` modules. The job -will be configured to run on any of the specified node pools. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a GKE job template file. - -```yaml - - id: job-template - source: modules/compute/gke-job-template - use: [compute_pool] - settings: - node_count: 3 - outputs: [instructions] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### Storage Options - -This module natively supports: - -* Filestore as a shared file system between pods/nodes. -* Pod level ephemeral storage options: - * memory backed emptyDir - * local SSD backed emptyDir - * SSD persistent disk backed ephemeral volume - * balanced persistent disk backed ephemeral volume - -See the [storage-gke.yaml blueprint](../../../examples/storage-gke.yaml) and the -associated [documentation](../../../../examples/README.md#storage-gkeyaml--) for -examples of how to use Filestore and ephemeral storage with this module. - -### Requested Resources - -When one or more `gke-node-pool` modules are referenced with the `use` field. -The requested resources will be populated to achieve a 1 pod per node packing -while still leaving some headroom for required system pods. - -This functionality can be overridden by specifying the desired cpu requirement -using the `requested_cpu_per_pod` setting. - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.2 | -| [local](#requirement\_local) | >= 2.0.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allocatable\_cpu\_per\_node](#input\_allocatable\_cpu\_per\_node) | The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | -| [allocatable\_gpu\_per\_node](#input\_allocatable\_gpu\_per\_node) | The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | -| [backoff\_limit](#input\_backoff\_limit) | Controls the number of retries before considering a Job as failed. Set to zero for shared fate. | `number` | `0` | no | -| [command](#input\_command) | The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes. | `list(string)` | 
[
  "hostname"
]
| no | -| [completion\_mode](#input\_completion\_mode) | Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information | `string` | `"Indexed"` | no | -| [ephemeral\_volumes](#input\_ephemeral\_volumes) | Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB. | 
list(object({
    type       = string
    mount_path = string
    size_gb    = number
  }))
| `[]` | no | -| [has\_gpu](#input\_has\_gpu) | Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module. | `list(bool)` | 
[
  false
]
| no | -| [image](#input\_image) | The container image the job should use. | `string` | `"debian"` | no | -| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account to run the job as. If null then no service account is specified. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the GKE job template. Key-value pairs. | `map(string)` | n/a | yes | -| [machine\_family](#input\_machine\_family) | The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria. | `string` | `null` | no | -| [name](#input\_name) | The name of the job. | `string` | `"my-job"` | no | -| [node\_count](#input\_node\_count) | How many nodes the job should run in parallel. | `number` | `1` | no | -| [node\_pool\_names](#input\_node\_pool\_names) | A list of node pool names on which to run the job. Can be populated via `use` field. | `list(string)` | `[]` | no | -| [node\_selectors](#input\_node\_selectors) | A list of node selectors to use to place the job. | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | -| [persistent\_volume\_claims](#input\_persistent\_volume\_claims) | A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module. | 
list(object({
    name          = string
    namespace     = string
    mount_path    = string
    mount_options = string
    storage_type  = string
  }))
| `[]` | no | -| [random\_name\_sufix](#input\_random\_name\_sufix) | Appends a random suffix to the job name to avoid clashes. | `bool` | `true` | no | -| [requested\_cpu\_per\_pod](#input\_requested\_cpu\_per\_pod) | The requested cpu per pod. If null, allocatable\_cpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_cpu\_per\_node. | `number` | `-1` | no | -| [requested\_gpu\_per\_pod](#input\_requested\_gpu\_per\_pod) | The requested gpu per pod. If null, allocatable\_gpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_gpu\_per\_node. | `number` | `-1` | no | -| [restart\_policy](#input\_restart\_policy) | Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed. | `string` | `"Never"` | no | -| [security\_context](#input\_security\_context) | The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | -| [tolerations](#input\_tolerations) | Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field. | 
list(object({
    key      = string
    operator = string
    value    = string
    effect   = string
  }))
| 
[
  {
    "effect": "NoSchedule",
    "key": "user-workload",
    "operator": "Equal",
    "value": "true"
  }
]
| no | -| [tpu\_accelerator\_type](#input\_tpu\_accelerator\_type) | The TPU accelerator type label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | -| [tpu\_chips\_per\_node](#input\_tpu\_chips\_per\_node) | The number of TPU chips per node. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | -| [tpu\_topology](#input\_tpu\_topology) | The TPU topology label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for submitting the GKE job. | - diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/main.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/main.tf deleted file mode 100644 index e84138bb3f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/main.tf +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-job-template", ghpc_role = "compute" }) -} - -locals { - tpu_accelerator_node_selector = var.tpu_accelerator_type[0] != null ? [{ - key = "cloud.google.com/gke-tpu-accelerator" - value = var.tpu_accelerator_type[0] - }] : [] - - tpu_topology_node_selector = var.tpu_topology[0] != null ? [{ - key = "cloud.google.com/gke-tpu-topology" - value = var.tpu_topology[0] - }] : [] -} - -locals { - # Start with the minimum cpu available of used node pools - min_allocatable_cpu = min(var.allocatable_cpu_per_node...) - full_node_cpu_request = ( - local.min_allocatable_cpu > 2 ? # if large enough - local.min_allocatable_cpu - 1 : # leave headroom for 1 cpu - local.min_allocatable_cpu / 2 + 0.1 # else take just over half - ) - (local.any_gcs ? 0.25 : 0) # save room for gcs side car - - cpu_request = ( - var.requested_cpu_per_pod >= 0 ? # if user supplied requested cpu - var.requested_cpu_per_pod : # then honor it - ( # else - local.min_allocatable_cpu >= 0 ? # if allocatable cpu was supplied - local.full_node_cpu_request : # then claim the full node - -1 # else do not set a limit - ) - ) - millicpu = floor(local.cpu_request * 1000) - cpu_request_string = local.millicpu >= 0 ? "${local.millicpu}m" : null - full_node_request = local.min_allocatable_cpu >= 0 && var.requested_cpu_per_pod < 0 - - memory_request_value = try(sum([for ed in var.ephemeral_volumes : - ed.size_gb - if ed.type == "memory" - ]), 0) - memory_request_string = local.memory_request_value > 0 ? "${local.memory_request_value}Gi" : null - - ephemeral_request_value = try(sum([for ed in var.ephemeral_volumes : - ed.size_gb - if ed.type == "local-ssd" - ]), 0) - ephemeral_request_string = local.ephemeral_request_value > 0 ? "${local.ephemeral_request_value}Gi" : null - - uses_local_ssd = anytrue([for ed in var.ephemeral_volumes : - ed.type == "local-ssd" - ]) - local_ssd_node_selector = local.uses_local_ssd ? [{ - key = "cloud.google.com/gke-ephemeral-storage-local-ssd" - value = "true" - }] : [] - - # Setup limit for GPUs per pod - min_allocatable_gpu = min(var.allocatable_gpu_per_node...) - min_allocatable_gpu_per_pod = local.min_allocatable_gpu > 0 ? local.min_allocatable_gpu : null - gpu_limit_per_pod = var.requested_gpu_per_pod > 0 ? var.requested_gpu_per_pod : local.min_allocatable_gpu_per_pod - gpu_limit_string = alltrue(var.has_gpu) ? tostring(local.gpu_limit_per_pod) : null - - empty_dir_volumes = [for ed in var.ephemeral_volumes : - { - name = replace(trim(ed.mount_path, "/"), "/", "-") - mount_path = ed.mount_path - size_limit = "${ed.size_gb}Gi" - in_memory = ed.type == "memory" - } - if contains(["memory", "local-ssd"], ed.type) - ] - - ephemeral_pd_volumes = [for pd in var.ephemeral_volumes : - { - name = replace(trim(pd.mount_path, "/"), "/", "-") - mount_path = pd.mount_path - storage_class_name = pd.type == "pd-ssd" ? "premium-rwo" : "standard-rwo" - storage = "${pd.size_gb}Gi" - } - if contains(["pd-balanced", "pd-ssd"], pd.type) - ] - - pvc_volumes = [for pvc in var.persistent_volume_claims : - { - name = replace(trim(pvc.mount_path, "/"), "/", "-") - mount_path = pvc.mount_path - claim_name = pvc.name - } - ] - - volume_mounts = [for v in concat(local.empty_dir_volumes, local.ephemeral_pd_volumes, local.pvc_volumes) : - { - name = v.name - mount_path = v.mount_path - } - ] - - suffix = var.random_name_sufix ? "-${random_id.resource_name_suffix.hex}" : "" - machine_family_node_selector = var.machine_family != null ? [{ - key = "cloud.google.com/machine-family" - value = var.machine_family - }] : [] - node_selectors = concat(local.machine_family_node_selector, local.local_ssd_node_selector, local.tpu_accelerator_node_selector, local.tpu_topology_node_selector, var.node_selectors) - - any_gcs = anytrue([for pvc in var.persistent_volume_claims : - pvc.storage_type == "gcs" - ]) - - job_template_contents = templatefile( - "${path.module}/templates/gke-job-base.yaml.tftpl", - { - name = var.name - suffix = local.suffix - image = var.image - command = var.command - node_count = var.node_count - completion_mode = var.completion_mode - k8s_service_account_name = var.k8s_service_account_name - node_pool_names = var.node_pool_names - node_selectors = local.node_selectors - tpu_limit = var.tpu_chips_per_node[0] - full_node_request = local.full_node_request - cpu_request = local.cpu_request_string - gpu_limit = local.gpu_limit_string - restart_policy = var.restart_policy - backoff_limit = var.backoff_limit - tolerations = distinct(var.tolerations) - security_context = var.security_context - labels = local.labels - - empty_dir_volumes = local.empty_dir_volumes - ephemeral_pd_volumes = local.ephemeral_pd_volumes - pvc_volumes = local.pvc_volumes - volume_mounts = local.volume_mounts - memory_request = local.memory_request_string - ephemeral_request = local.ephemeral_request_string - gcs_annotation = local.any_gcs - } - ) - - job_template_output_path = "${path.root}/${var.name}${local.suffix}.yaml" - -} - -resource "random_id" "resource_name_suffix" { - byte_length = 2 - keepers = { - timestamp = timestamp() - } -} - -resource "local_file" "job_template" { - content = local.job_template_contents - filename = local.job_template_output_path - - lifecycle { - precondition { - condition = local.any_gcs ? var.k8s_service_account_name != null : true - error_message = "When using GCS, a kubernetes service account with workload identity is required. gke-cluster module will perform this setup when var.configure_workload_identity_sa is set to true." - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/outputs.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/outputs.tf deleted file mode 100644 index adf78e936d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "instructions" { - description = "Instructions for submitting the GKE job." - value = <<-EOT - A GKE job file has been created locally at: - ${abspath(local.job_template_output_path)} - - Use the following commands to: - Submit your job: - kubectl create -f ${abspath(local.job_template_output_path)} - EOT -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl deleted file mode 100644 index 11df39ce2c..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl +++ /dev/null @@ -1,128 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: ${name}${suffix} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - parallelism: ${node_count} - completions: ${node_count} - completionMode: ${completion_mode} - template: - %{~ if gcs_annotation ~} - metadata: - annotations: - gke-gcsfuse/volumes: "true" - %{~ endif ~} - spec: - %{~ if length(security_context) > 0 ~} - securityContext: - %{~ for context in security_context ~} - ${context.key}: ${context.value} - %{~ endfor ~} - %{~ endif ~} - %{~ if k8s_service_account_name != null ~} - serviceAccountName: ${k8s_service_account_name} - %{~ endif ~} - %{~ if length(node_pool_names) > 0 ~} - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.google.com/gke-nodepool - operator: In - values: - %{~ for node_pool in node_pool_names ~} - - ${node_pool} - %{~ endfor ~} - %{~ endif ~} - %{~ if length(node_selectors) > 0 ~} - nodeSelector: - %{~ for selector in node_selectors ~} - ${selector.key}: "${selector.value}" - %{~ endfor ~} - %{~ endif ~} - tolerations: - %{~ for toleration in tolerations ~} - - key: ${toleration.key} - operator: ${toleration.operator} - value: "${toleration.value}" - effect: ${toleration.effect} - %{~ endfor ~} - containers: - - name: ${name}-container - image: ${image} - command: - %{for s in command}- ${indent(8, yamlencode(s))}%{~ endfor } - %{~ if gpu_limit != null || cpu_request != null || tpu_limit != null ~} - resources: - %{~ if gpu_limit != null || tpu_limit != null ~} - limits: - %{~ if gpu_limit != null ~} - # GPUs should only be specified as limits - # https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/ - nvidia.com/gpu: ${gpu_limit} - %{~ endif ~} - %{~ if tpu_limit != null ~} - google.com/tpu: ${tpu_limit} - %{~ endif ~} - %{~ endif ~} - %{~ if cpu_request != null || memory_request != null || ephemeral_request != null || tpu_limit != null ~} - requests: - %{~ if full_node_request ~} - # cpu request attempts full node per pod - %{~ endif ~} - %{~ if cpu_request != null ~} - cpu: ${cpu_request} - %{~ endif ~} - %{~ if tpu_limit != null ~} - google.com/tpu: ${tpu_limit} - %{~ endif ~} - %{~ if memory_request != null ~} - memory: ${memory_request} - %{~ endif ~} - %{~ if ephemeral_request != null ~} - ephemeral-storage: ${ephemeral_request} - %{~ endif ~} - %{~ endif ~} - %{~ endif ~} - %{~ if length(volume_mounts) > 0 ~} - volumeMounts: - %{~ for v in volume_mounts ~} - - name: ${v.name} - mountPath: ${v.mount_path} - %{~ endfor ~} - %{~ endif ~} - %{~ if length(volume_mounts) > 0 ~} - volumes: - %{~ for ed in empty_dir_volumes ~} - - name: ${ed.name} - emptyDir: - sizeLimit: ${ed.size_limit} - %{~ if ed.in_memory ~} - medium: "Memory" - %{~ endif ~} - %{~ endfor ~} - %{~ for pd in ephemeral_pd_volumes ~} - - name: ${pd.name} - ephemeral: - volumeClaimTemplate: - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: ${pd.storage_class_name} - resources: - requests: - storage: ${pd.storage} - %{~ endfor ~} - %{~ for pvc in pvc_volumes ~} - - name: ${pvc.name} - persistentVolumeClaim: - claimName: ${pvc.claim_name} - %{~ endfor ~} - %{~ endif ~} - restartPolicy: ${restart_policy} - backoffLimit: ${backoff_limit} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/variables.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/variables.tf deleted file mode 100644 index fd83f2b692..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/variables.tf +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "name" { - description = "The name of the job." - type = string - default = "my-job" -} - -variable "node_count" { - description = "How many nodes the job should run in parallel." - type = number - default = 1 -} - -variable "completion_mode" { - description = "Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information" - type = string - default = "Indexed" -} - -variable "command" { - description = "The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes." - type = list(string) - default = ["hostname"] -} - -variable "image" { - description = "The container image the job should use." - type = string - default = "debian" -} - -variable "k8s_service_account_name" { - description = "Kubernetes service account to run the job as. If null then no service account is specified." - type = string - default = null -} - -variable "node_pool_names" { - description = "A list of node pool names on which to run the job. Can be populated via `use` field." - type = list(string) - default = [] -} - -variable "allocatable_cpu_per_node" { - description = "The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." - type = list(number) - default = [-1] -} - -variable "has_gpu" { - description = "Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module." - type = list(bool) - default = [false] -} - -variable "requested_cpu_per_pod" { - description = "The requested cpu per pod. If null, allocatable_cpu_per_node will be used to claim whole nodes. If provided will override allocatable_cpu_per_node." - type = number - default = -1 -} - -variable "allocatable_gpu_per_node" { - description = "The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." - type = list(number) - default = [-1] -} - -variable "requested_gpu_per_pod" { - description = "The requested gpu per pod. If null, allocatable_gpu_per_node will be used to claim whole nodes. If provided will override allocatable_gpu_per_node." - type = number - default = -1 -} - -variable "tolerations" { - description = "Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field." - type = list(object({ - key = string - operator = string - value = string - effect = string - })) - default = [ - { - key = "user-workload" - operator = "Equal" - value = "true" - effect = "NoSchedule" - } - ] -} - -variable "security_context" { - description = "The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/" - type = list(object({ - key = string - value = string - })) - default = [] -} - -variable "machine_family" { - description = "The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria." - type = string - default = null -} - -variable "node_selectors" { - description = "A list of node selectors to use to place the job." - type = list(object({ - key = string - value = string - })) - default = [] -} - -variable "restart_policy" { - description = "Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed." - type = string - default = "Never" -} - -variable "backoff_limit" { - description = "Controls the number of retries before considering a Job as failed. Set to zero for shared fate." - type = number - default = 0 -} - -variable "random_name_sufix" { - description = "Appends a random suffix to the job name to avoid clashes." - type = bool - default = true -} - -variable "persistent_volume_claims" { - description = "A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module." - type = list(object({ - name = string - namespace = string - mount_path = string - mount_options = string - storage_type = string - })) - default = [] -} - -variable "ephemeral_volumes" { - description = "Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB." - type = list(object({ - type = string - mount_path = string - size_gb = number - })) - default = [] - validation { - condition = alltrue([ - for v in var.ephemeral_volumes : - contains(["pd-balanced", "pd-ssd", "memory", "local-ssd"], v.type) - ]) - error_message = "Type must be one of 'pd-balanced', 'pd-ssd', 'memory', 'local-ssd'." - } - validation { - condition = alltrue([ - for v in var.ephemeral_volumes : - substr(v.mount_path, 0, 1) == "/" - ]) - error_message = "Mount path must start with the '/' character." - } -} - -variable "labels" { - description = "Labels to add to the GKE job template. Key-value pairs." - type = map(string) -} - -variable "tpu_accelerator_type" { - description = "The TPU accelerator type label. Populated from gke-node-pool via `use` field." - type = list(string) - default = [null] -} - -variable "tpu_topology" { - description = "The TPU topology label. Populated from gke-node-pool via `use` field." - type = list(string) - default = [null] -} - -variable "tpu_chips_per_node" { - description = "The number of TPU chips per node. Populated from gke-node-pool via `use` field." - type = list(string) - default = [null] -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/versions.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/versions.tf deleted file mode 100644 index 0f902ac8c5..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-job-template/versions.tf +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.2" - - required_providers { - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/README.md b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/README.md deleted file mode 100644 index b25d905252..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/README.md +++ /dev/null @@ -1,388 +0,0 @@ -## Description - -This module creates a Google Kubernetes Engine -([GKE](https://cloud.google.com/kubernetes-engine)) node pool. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a GKE node group. - -```yaml - - id: compute_pool - source: modules/compute/gke-node-pool - use: [gke_cluster] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### Taints and Tolerations - -By default node pools created with this module will be tainted with -`user-workload=true:NoSchedule` to prevent system pods from being scheduled. -User jobs targeting the node pool should include this toleration. This behavior -can be overridden using the `taints` setting. See -[docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-taints) for -more info. - -### Local SSD Storage -GKE offers two options for managing locally attached SSDs. - -The first, and recommended, option is for GKE to manage the ephemeral storage -space on the node, which will then be automatically attached to pods which -request an `emptyDir` volume. This can be accomplished using the -[`local_ssd_count_ephemeral_storage`] variable. - -The second, more complex, option is for GCP to attach these nodes as raw block -storage. In this case, the cluster administrator is responsible for software -RAID settings, partitioning, formatting and mounting these disks on the host -OS. Still, this may be desired behavior in use cases which aren't supported -by an `emptyDir` volume (for example, a `ReadOnlyMany` or `ReadWriteMany` PV). -This can be accomplished using the [`local_ssd_count_nvme_block`] variable. - -The [`local_ssd_count_ephemeral_storage`] and [`local_ssd_count_nvme_block`] -variables are mutually exclusive and cannot be mixed together. - -Also, the number of SSDs which can be attached to a node depends on the -[machine type](https://cloud.google.com/compute/docs/disks#local_ssd_machine_type_restrictions). - -See [docs](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd) -for more info. - -[`local_ssd_count_ephemeral_storage`]: #input\_local\_ssd\_count\_ephemeral\_storage -[`local_ssd_count_nvme_block`]: #input\_local\_ssd\_count\_nvme\_block - -### Considerations with GPUs - -When a GPU is attached to a node an additional taint is automatically added: -`nvidia.com/gpu=present:NoSchedule`. For jobs to get placed on these nodes, the -equivalent toleration is required. The `gke-job-template` module will -automatically apply this toleration when using a node pool with GPUs. - -Nvidia GPU drivers must be installed. The recommended approach for GKE to install -GPU dirvers is by applying a DaemonSet to the cluster. See -[these instructions](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus#cos). - -However, in some cases it may be desired to compile a different driver (such as -a desire to install a newer version, compatibility with the -[Nvidia GPU-operator](https://github.com/NVIDIA/gpu-operator) or other -use-cases). In this case, ensure that you turn off the -[enable_secure_boot](#input\_enable\_secure\_boot) option to allow unsigned -kernel modules to be loaded. - -#### Maximize GPU network bandwidth with GPUDirect and multi-networking -For A3 Series machines to achieve optimal performance , GKE provide two networking stacks for remote direct memory access (RDMA): - -- A3 High machine types (a3-highgpu-8g): utilize GPUDirect-TCPX to reduce the overhead required to transfer packet payloads to and from GPUs, which significantly improves throughput at scale compared to GPUs that don't use GPUDirect. -- A3 Mega machine types (a3-megagpu-8g): utilize GPUDirect-TCPXO to improve GPU to GPU communication, and further improves GPU to VM communication. - -To achieve this, when creating nodepools with A3 Series machine type, pass in a multivpc module to the gke-node-pool module, and the gke-node-pool module would detect the eligible machine type and enable GPUDirect for it. More specifically, the below components will be installed in the nodepool for enabling GPUDirect. - -- Install NCCL plugin for GPUDirect [TCPX](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpx) or [TCPXO](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpxo) -- Install [NRI](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nri_device_injector) device injector plugin -- Provide support for injecting GPUDirect required components(annotations, volumes, rxdm sidecar etc.) into the user workload in the form of Kubernetes Job. - - Provide sample workload to showcase how it will be updated with the required components injected, and how it can be deployed. - - Allow user to use the provided script to update their own workload and deploy. - -The GPUDirect supports included in the Cluster Toolkit aim to automate the [GPUDirect User Guid](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#install-gpudirect-tcpx-nccl) and provide better usability. - -> **_NOTE:_** You must [enable multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) feature when creating the GKE cluster. When gke-cluster depends on multivpc (with the use keyword), multi networking will be automatically enabled on the cluster creation. -> When gke-cluster or pre-existing-gke-cluster depends on multivpc (with the use keyword), the [network objects](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment) required for multi networking will be created on the cluster. - -### GPUs Examples - -There are several ways to add GPUs to a GKE node pool. See -[docs](https://cloud.google.com/compute/docs/gpus) for more info on GPUs. - -The following is a node pool that uses `a2`, `a3` or `g2` machine types which has a -fixed number of attached GPUs, let's call these machine types as "pre-defined gpu machine families": - -```yaml - - id: simple-a2-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g -``` - -> **Note**: It is not necessary to define the [`guest_accelerator`] setting when -> using pre-defined gpu machine families as information about GPUs, such as type, count and -> `gpu_driver_installation_config`, is automatically inferred from the machine type. -> Optional fields such as `gpu_partition_size` need to be specified only if they have -> non-default values. - -The following scenarios require the [`guest_accelerator`] block is specified: - -- To partition an A100 GPU into multiple GPUs on an A2 family machine. -- To specify a time sharing configuration on a GPUs. -- To attach a GPU to an N1 family machine. - -The following is an example of -[partitioning](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus-multi) -an A100 GPU: - -> **Note**: In the following example, `type`, `count` and `gpu_driver_installation_config` are picked up automatically. - -```yaml - - id: multi-instance-gpu-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g - guest_accelerator: - - gpu_partition_size: 1g.5gb -``` - -[`guest_accelerator`]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_guest_accelerator - -The following is an example of -[GPU time sharing](https://cloud.google.com/kubernetes-engine/docs/concepts/timesharing-gpus) -(with partitioned GPUs): - -```yaml - - id: time-sharing-gpu-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g - guest_accelerator: - - gpu_partition_size: 1g.5gb - gpu_sharing_config: - gpu_sharing_strategy: TIME_SHARING - max_shared_clients_per_gpu: 3 -``` - -Following is an example of using a GPU attached to an `n1` machine: - -```yaml - - id: t4-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: n1-standard-16 - guest_accelerator: - - type: nvidia-tesla-t4 - count: 2 -``` - -The following is an example of using a GPU (with sharing config) attached to an `n1` machine: - -```yaml - - id: n1-t4-pool - source: community/modules/compute/gke-node-pool - use: [gke_cluster] - settings: - name: n1-t4-pool - machine_type: n1-standard-1 - guest_accelerator: - - type: nvidia-tesla-t4 - count: 2 - gpu_driver_installation_config: - gpu_driver_version: "LATEST" - gpu_sharing_config: - max_shared_clients_per_gpu: 2 - gpu_sharing_strategy: "TIME_SHARING" -``` - -Finally, the following is adding multivpc to a node pool: - -```yaml - - id: network - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: gke-cluster - source: modules/scheduler/gke-cluster - use: [network, multinetwork] - settings: - cluster_name: $(vars.deployment_name) - - - id: a3-megagpu_pool - source: modules/compute/gke-node-pool - use: [gke-cluster, multinetwork] - settings: - machine_type: a3-megagpu-8g - ... -``` - -## Using GCE Reservations -You can reserve Google Compute Engine instances in a specific zone to ensure resources are available for their workloads when needed. For more details on how to manage reservations, see [Reserving Compute Engine zonal resources](https://cloud.google.com/compute/docs/instances/reserving-zonal-resources). - -After creating a reservation, you can consume the reserved GCE VM instances in GKE. GKE clusters deployed using Cluster Toolkit support the same consumption modes as Compute Engine: NO_RESERVATION(default), ANY_RESERVATION, SPECIFIC_RESERVATION. - -This can be accomplished using [`reservation_affinity`](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/compute/gke-node-pool/README.md#input_reservation_affinity). - -```yaml -# Target any reservation -reservation_affinity: - consume_reservation_type: ANY_RESERVATION - -# Target a specific reservation -reservation_affinity: - consume_reservation_type: SPECIFIC_RESERVATION - specific_reservations: - - name: specific-reservation-1 -``` - -The following requirements need to be satisfied for the node pool nodes to be able to use a specific reservation: -1. A reservation with the name must exist in the specified project(`var.project_id`) and one of the specified zones(`var.zones`). -2. Its consumption type must be `specific`. -3. Its GCE VM Properties must match with those of the Node Pool; Machine type, Accelerators (GPU Type and count), Local SSD disk type and count. - -If you want to utilise a shared reservation, the owner project of the shared reservation needs to be explicitly specified like the following. Note that a shared reservation can be used by the project that hosts the reservation (owner project) and by the projects the reservation is shared with (consumer projects). See how to [create and use a shared reservation](https://cloud.google.com/compute/docs/instances/reservations-shared). - -```yaml -reservation_affinity: - consume_reservation_type: SPECIFIC_RESERVATION - specific_reservations: - - name: specific-reservation-shared - project: shared_reservation_owner_project_id -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 7.2 | -| [google-beta](#requirement\_google-beta) | >= 7.2 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 7.2 | -| [google-beta](#provider\_google-beta) | >= 7.2 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | -| [tpu](#module\_tpu) | ../../internal/tpu-definition | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_container_node_pool.node_pool](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | -| [null_resource.enable_tcpx_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.enable_tcpxo_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.install_dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_machine_types.machine_info](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | -| [google_compute_region_instance_template.instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_template) | data source | -| [google_compute_reservation.specific_reservations](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [auto\_repair](#input\_auto\_repair) | Whether the nodes will be automatically repaired. | `bool` | `true` | no | -| [auto\_upgrade](#input\_auto\_upgrade) | Whether the nodes will be automatically upgraded. | `bool` | `false` | no | -| [autoscaling\_total\_max\_nodes](#input\_autoscaling\_total\_max\_nodes) | Total maximum number of nodes in the NodePool. | `number` | `1000` | no | -| [autoscaling\_total\_min\_nodes](#input\_autoscaling\_total\_min\_nodes) | Total minimum number of nodes in the NodePool. | `number` | `0` | no | -| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | -| [compact\_placement](#input\_compact\_placement) | DEPRECATED: Use `placement_policy` | `bool` | `null` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for each node. | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Disk type for each node. | `string` | `null` | no | -| [enable\_flex\_start](#input\_enable\_flex\_start) | If true, start the node pool with Flex Start provisioning model.
To learn more about flex-start mode, please refer to
https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and
https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest | `bool` | `false` | no | -| [enable\_gcfs](#input\_enable\_gcfs) | Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config). | `bool` | `false` | no | -| [enable\_numa\_aware\_scheduling](#input\_enable\_numa\_aware\_scheduling) | Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling. | `bool` | `false` | no | -| [enable\_private\_nodes](#input\_enable\_private\_nodes) | Whether nodes have internal IP addresses only. | `bool` | `true` | no | -| [enable\_queued\_provisioning](#input\_enable\_queued\_provisioning) | If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool. | `bool` | `false` | no | -| [enable\_secure\_boot](#input\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | -| [gke\_version](#input\_gke\_version) | GKE version | `string` | n/a | yes | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = optional(string)
    count = optional(number, 0)
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | -| [host\_maintenance\_interval](#input\_host\_maintenance\_interval) | Specifies the frequency of planned maintenance events. | `string` | `""` | no | -| [image\_type](#input\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | -| [initial\_node\_count](#input\_initial\_node\_count) | The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static\_node\_count and must be set to a value between autoscaling\_total\_min\_nodes and autoscaling\_total\_max\_nodes. | `number` | `null` | no | -| [internal\_ghpc\_module\_id](#input\_internal\_ghpc\_module\_id) | DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide). | `string` | n/a | yes | -| [is\_reservation\_active](#input\_is\_reservation\_active) | Whether the specified reservation is already created. | `bool` | `true` | no | -| [kubernetes\_labels](#input\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_count\_ephemeral\_storage](#input\_local\_ssd\_count\_ephemeral\_storage) | The number of local SSDs to attach to each node to back ephemeral storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | -| [local\_ssd\_count\_nvme\_block](#input\_local\_ssd\_count\_nvme\_block) | The number of local SSDs to attach to each node to back block storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | -| [machine\_type](#input\_machine\_type) | The name of a Google Compute Engine machine type. | `string` | `"c2-standard-60"` | no | -| [max\_pods\_per\_node](#input\_max\_pods\_per\_node) | The maximum number of pods per node in this node pool. This will force replacement. | `number` | `null` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [name](#input\_name) | The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix.
If setting manually, ensure a unique value across all gke-node-pools. | `string` | `null` | no | -| [num\_node\_pools](#input\_num\_node\_pools) | Number of node pools to create. This is same as num\_slices. | `number` | `1` | no | -| [num\_slices](#input\_num\_slices) | Number of TPUs slices to create. This is same as num\_node\_pools. | `number` | `1` | no | -| [placement\_policy](#input\_placement\_policy) | Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| 
{
  "name": null,
  "tpu_topology": null,
  "type": null
}
| no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Reservation resource to consume. When targeting SPECIFIC\_RESERVATION, specific\_reservations needs be specified.
Even though specific\_reservations is a list, only one reservation is allowed by the NodePool API.
It is assumed that the specified reservation exists and has available capacity.
For a shared reservation, specify the project\_id as well in which it was created.
To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared | 
object({
    consume_reservation_type = string
    specific_reservations = optional(list(object({
      name    = string
      project = optional(string)
    })))
  })
| 
{
  "consume_reservation_type": "NO_RESERVATION",
  "specific_reservations": []
}
| no | -| [run\_workload\_script](#input\_run\_workload\_script) | Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only. | `bool` | `true` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [static\_node\_count](#input\_static\_node\_count) | The static number of nodes in the node pool. If set, autoscaling will be disabled. | `number` | `null` | no | -| [taints](#input\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | -| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | -| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | -| [total\_max\_nodes](#input\_total\_max\_nodes) | DEPRECATED: Use autoscaling\_total\_max\_nodes. | `number` | `null` | no | -| [total\_min\_nodes](#input\_total\_min\_nodes) | DEPRECATED: Use autoscaling\_total\_min\_nodes. | `number` | `null` | no | -| [upgrade\_settings](#input\_upgrade\_settings) | Defines node pool upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | -| [zones](#input\_zones) | A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable. | `list(string)` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [allocatable\_cpu\_per\_node](#output\_allocatable\_cpu\_per\_node) | Number of CPUs available for scheduling pods on each node. | -| [allocatable\_gpu\_per\_node](#output\_allocatable\_gpu\_per\_node) | Number of GPUs available for scheduling pods on each node. | -| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | -| [guest\_accelerator](#output\_guest\_accelerator) | The accelerator type of the nodes. | -| [has\_gpu](#output\_has\_gpu) | Boolean value indicating whether nodes in the pool are configured with GPUs. | -| [instance\_templates](#output\_instance\_templates) | The URLs of Instance Templates | -| [instructions](#output\_instructions) | Instructions for submitting the sample GPUDirect enabled job. | -| [machine\_type](#output\_machine\_type) | Machine Type | -| [node\_count\_static](#output\_node\_count\_static) | The number of static nodes in node-pool. | -| [node\_pool\_names](#output\_node\_pool\_names) | Names of the node pools. | -| [static\_gpu\_count](#output\_static\_gpu\_count) | Total number of GPUs in the node pool. Available only for static node pools. | -| [tolerations](#output\_tolerations) | Tolerations needed for a pod to be scheduled on this node pool. | -| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | -| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | -| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | - diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf deleted file mode 100644 index 0c1c255255..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -## Required variables: -# local_ssd_count_ephemeral_storage -# local_ssd_count_nvme_block -# machine_type - -locals { - - local_ssd_machines = { - "a3-highgpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, - "a3-megagpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, - "a3-ultragpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, - "a4-highgpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, - } - - generated_local_ssd_config = lookup(local.local_ssd_machines, var.machine_type, { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = null }) - - # Select in priority order: - # (1) var.local_ssd_count_ephemeral_storage and var.local_ssd_count_nvme_block if any is not null - # (2) local.local_ssd_machines if not empty - # (3) default to null value for both local_ssd_count_ephemeral_storage and local_ssd_count_nvme_block - local_ssd_config = (var.local_ssd_count_ephemeral_storage == null && var.local_ssd_count_nvme_block == null) ? local.generated_local_ssd_config : { local_ssd_count_ephemeral_storage = var.local_ssd_count_ephemeral_storage, local_ssd_count_nvme_block = var.local_ssd_count_nvme_block } -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml deleted file mode 100644 index 1106f63479..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: batch/v1 -kind: Job -metadata: - name: my-sample-job -spec: - parallelism: 2 - completions: 2 - completionMode: Indexed - template: - spec: - containers: - - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx-dev:v3.1.9 - imagePullPolicy: Always - command: - - /bin/sh - - -c - - | - service ssh restart; - sleep infinity; - env: - - name: LD_LIBRARY_PATH - value: /usr/local/nvidia/lib64 - volumeMounts: - - name: config-volume - mountPath: /configs - resources: - limits: - nvidia.com/gpu: 8 - volumes: - - name: config-volume - configMap: - name: nccl-configmap - defaultMode: 0777 - restartPolicy: Never - backoffLimit: 0 diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml deleted file mode 100644 index bce6720681..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: batch/v1 -kind: Job -metadata: - name: my-sample-job -spec: - parallelism: 2 - completions: 2 - completionMode: Indexed - template: - spec: - hostname: host1 - subdomain: nccl-host-1 - containers: - - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.14 - imagePullPolicy: Always - command: - - /bin/sh - - -c - - | - set -ex - chmod 755 /scripts/demo-run-nccl-test-tcpxo-via-mpi.sh - cat >/scripts/allgather.sh < 0: - container["env"].extend(env_vars) - container["volumeMounts"].extend(volume_mounts) - -if __name__ == "__main__": - main() diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py deleted file mode 100644 index db9fb3e7ff..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import yaml -import argparse -import os - -def main(): - parser = argparse.ArgumentParser(description="TCPXO Job Manifest Generator") - parser.add_argument("-f", "--file", required=True, help="Path to your job template YAML file") - parser.add_argument("-r", "--rxdm", required=True, help="RxDM version") - - args = parser.parse_args() - - # Get the YAML file from the user - if not args.file: - args.file = input("Please provide the path to your job template YAML file: ") - - # Get component versions from user - if not args.rxdm: - args.rxdm = input("Enter the RxDM version: ") - - # Load and modify the YAML - with open(args.file, "r") as file: - job_manifest = yaml.load(file, Loader=yaml.BaseLoader) - - # Update annotations - add_annotations(job_manifest) - - # Update volumes - add_volumes(job_manifest) - - # Update tolerations - add_tolerations(job_manifest) - - # Add tcpxo-daemon container - add_tcpxo_daemon_container(job_manifest, args.rxdm) - - # Update environment variables and volumeMounts for GPU containers - update_gpu_containers(job_manifest) - - # Generate the new YAML file - updated_job = str(yaml.dump(job_manifest, default_flow_style=False, width=1000, default_style="|", sort_keys=False)).replace("|-", "") - - new_file_name = args.file.replace(".yaml", "-tcpxo.yaml") - with open(new_file_name, "w", encoding="utf-8") as file: - file.write(updated_job) - - # Step 7: Provide instructions to the user - print("\nA new manifest has been generated and updated to have TCPXO enabled based on the provided workload") - print("It can be found in {path}".format(path=os.path.abspath(new_file_name))) - print("You can use the following commands to submit the sample job:") - print(" kubectl create -f {path}".format(path=os.path.abspath(new_file_name))) - -def add_annotations(job_manifest): - annotations = { - 'devices.gke.io/container.tcpxo-daemon':"""|+ -- path: /dev/nvidia0 -- path: /dev/nvidia1 -- path: /dev/nvidia2 -- path: /dev/nvidia3 -- path: /dev/nvidia4 -- path: /dev/nvidia5 -- path: /dev/nvidia6 -- path: /dev/nvidia7 -- path: /dev/nvidiactl -- path: /dev/nvidia-uvm -- path: /dev/dmabuf_import_helper""", - "networking.gke.io/default-interface": "eth0", - "networking.gke.io/interfaces": """| -[ - {"interfaceName":"eth0","network":"default"}, - {"interfaceName":"eth1","network":"vpc1"}, - {"interfaceName":"eth2","network":"vpc2"}, - {"interfaceName":"eth3","network":"vpc3"}, - {"interfaceName":"eth4","network":"vpc4"}, - {"interfaceName":"eth5","network":"vpc5"}, - {"interfaceName":"eth6","network":"vpc6"}, - {"interfaceName":"eth7","network":"vpc7"}, - {"interfaceName":"eth8","network":"vpc8"} -]""", - } - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("metadata", {}) - - # Add/update annotations - pod_template_spec = job_manifest["spec"]["template"]["metadata"] - if "annotations" in pod_template_spec: - pod_template_spec["annotations"].update(annotations) - else: - pod_template_spec["annotations"] = annotations - -def add_tolerations(job_manifest): - tolerations = [ - {"key": "user-workload", "operator": "Equal", "value": """\"true\"""", "effect": "NoSchedule"}, - ] - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add tolerations - pod_spec = job_manifest["spec"]["template"]["spec"] - if "tolerations" in pod_spec: - pod_spec["tolerations"].extend(tolerations) - else: - pod_spec["tolerations"] = tolerations - -def add_volumes(job_manifest): - volumes = [ - {"name": "nvidia-install-dir-host", "hostPath": {"path": "/home/kubernetes/bin/nvidia"}}, - {"name": "sys", "hostPath": {"path": "/sys"}}, - {"name": "proc-sys", "hostPath": {"path": "/proc/sys"}}, - {"name": "aperture-devices", "hostPath": {"path": "/dev/aperture_devices"}}, - ] - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add volumes - pod_spec = job_manifest["spec"]["template"]["spec"] - if "volumes" in pod_spec: - pod_spec["volumes"].extend(volumes) - else: - pod_spec["volumes"] = volumes - - -def add_tcpxo_daemon_container(job_template, rxdm_version): - tcpxo_daemon_container = { - "name": "tcpxo-daemon", - "image": f"us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:{rxdm_version}", # Use provided RxDM version - "imagePullPolicy": "Always", - "command": ["/bin/sh", "-c"], - "args": [ - """| - set -ex - chmod 755 /fts/entrypoint_rxdm_container.sh - /fts/entrypoint_rxdm_container.sh --num_hops=2 --num_nics=8 --uid= --alsologtostderr""" - ], - "securityContext": { - "capabilities": {"add": ["NET_ADMIN", "NET_BIND_SERVICE"]} - }, - "volumeMounts": [ - {"name": "nvidia-install-dir-host", "mountPath": "/usr/local/nvidia"}, - {"name": "sys", "mountPath": "/hostsysfs"}, - {"name": "proc-sys", "mountPath": "/hostprocsysfs"}, - ], - "env": [{"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}], - } - - # Create path if it doesn't exist - job_template.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add container - pod_spec = job_template["spec"]["template"]["spec"] - pod_spec.setdefault("containers", []).insert(0, tcpxo_daemon_container) - -def update_gpu_containers(job_manifest): - env_vars = [ - {"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}, - {"name": "NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY", "value": "/dev/aperture_devices"}, - ] - volume_mounts = [{"name": "aperture-devices", "mountPath": "/dev/aperture_devices"}] - - pod_spec = job_manifest.get("spec", {}).get("template", {}).get("spec", {}) - for container in pod_spec.get("containers", []): - # Create path if it doesn't exist - container.setdefault("env", []) - container.setdefault("volumeMounts", []) - if int(container.get("resources", {}).get("limits", {}).get("nvidia.com/gpu", 0)) > 0: - container["env"].extend(env_vars) - container["volumeMounts"].extend(volume_mounts) - -if __name__ == "__main__": - main() diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf deleted file mode 100644 index d23d050986..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# Enable GPUDirect for A3 and A3Mega VMs, this involve multiple kubectl steps to integrate with the created cluster -# 1. Install NCCL plugin daemonset -# 2. Install NRI plugin daemonset -# 3. Update provided workload to inject rxdm sidecar and other required annotation, volume etc. -locals { - workload_path_tcpx = "${path.module}/gpu-direct-workload/sample-tcpx-workload-job.yaml" - workload_path_tcpxo = "${path.module}/gpu-direct-workload/sample-tcpxo-workload-job.yaml" - - gpu_direct_settings = { - "a3-highgpu-8g" = { - # Manifest to be installed for enabling TCPX on a3-highgpu-8g machines - gpu_direct_manifests = [ - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-tcpx-installer.yaml", # nccl_plugin v3.1.9 for tcpx - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-config.yaml", # nccl_configmap - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/nri_device_injector/nri-device-injector.yaml", # nri_plugin - ] - updated_workload_path = replace(local.workload_path_tcpx, ".yaml", "-tcpx.yaml") - rxdm_version = "v2.0.12" # matching nccl-tcpx-installer version v3.1.9 - min_additional_networks = 4 - major_minor_version_acceptable_map = { - "1.27" = "1.27.7-gke.1121000" - "1.28" = "1.28.8-gke.1095000" - "1.29" = "1.29.3-gke.1093000" - "1.30" = "1.30.2-gke.1023000" - } - } - "a3-megagpu-8g" = { - # Manifest to be installed for enabling TCPXO on a3-megagpu-8g machines - gpu_direct_manifests = [ - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/gpudirect-tcpxo/nccl-tcpxo-installer.yaml", # nccl_plugin v1.0.14 for tcpxo - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/nri_device_injector/nri-device-injector.yaml", # nri_plugin - ] - updated_workload_path = replace(local.workload_path_tcpxo, ".yaml", "-tcpxo.yaml") - rxdm_version = "v1.0.20" # matching nccl-tcpxo-installer version v1.0.14 - min_additional_networks = 8 - major_minor_version_acceptable_map = { - "1.28" = "1.28.9-gke.1250000" - "1.29" = "1.29.4-gke.1542000" - "1.30" = "1.30.4-gke.1129000" - "1.31" = "1.31.1-gke.2008000" - "1.32" = "1.32.2-gke.1489001" - } - } - } - - min_additional_networks = try(local.gpu_direct_settings[var.machine_type].min_additional_networks, 0) - - gke_version_regex = "(\\d+\\.\\d+)\\.(\\d+)-gke\\.(\\d+)" # GKE version format: 1.X.Y-gke.Z , regex output: ["1.X" , "Y", "Z"] - - gke_version_parts = regex(local.gke_version_regex, var.gke_version) - gke_version_major = local.gke_version_parts[0] - - major_minor_version_acceptable_map = try(local.gpu_direct_setting[var.machine_type].major_minor_version_acceptable_map, null) - minor_version_acceptable = try(contains(keys(local.major_minor_version_acceptable_map), local.gke_version_major), false) ? local.major_minor_version_acceptable_map[local.gke_version_major] : "1.0.0-gke.0" - minor_version_acceptable_parts = regex(local.gke_version_regex, local.minor_version_acceptable) - gke_gpudirect_compatible = local.gke_version_parts[1] > local.minor_version_acceptable_parts[1] || (local.gke_version_parts[1] == local.minor_version_acceptable_parts[1] && local.gke_version_parts[2] >= local.minor_version_acceptable_parts[2]) -} - -check "gpu_direct_check_multi_vpc" { - assert { - condition = length(var.additional_networks) >= local.min_additional_networks - error_message = "To achieve optimal performance for ${var.machine_type} machine, at least ${local.min_additional_networks} additional vpc is recommended. You could configure it in the blueprint through modules/network/multivpc with network_count set as ${local.min_additional_networks}" - } -} - -check "gke_version_requirements" { - assert { - condition = local.gke_gpudirect_compatible - error_message = "GPUDirect is not supported on GKE version ${var.gke_version} for ${var.machine_type} machine. For supported version details visit https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#requirements" - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf deleted file mode 100644 index 1ddc7ba8c3..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -data "google_compute_machine_types" "machine_info" { - for_each = var.zones == null ? toset([]) : toset(var.zones) - - project = var.project_id - zone = each.key - filter = "name = \"${var.machine_type}\"" -} - -locals { - valid_machine_info = { - for zone, data in data.google_compute_machine_types.machine_info : - zone => data.machine_types if length(data.machine_types) > 0 - } - - guest_cpus = try(local.valid_machine_info[0].guest_cpus, 0) -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/main.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/main.tf deleted file mode 100644 index 05314497fc..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/main.tf +++ /dev/null @@ -1,482 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-node-pool", ghpc_role = "compute" }) -} - -locals { - upgrade_settings = { - strategy = var.upgrade_settings.strategy - max_surge = coalesce(var.upgrade_settings.max_surge, 0) - max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) - } -} - -module "gpu" { - source = "../../internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - has_gpu = length(local.guest_accelerator) > 0 - allocatable_gpu_per_node = local.has_gpu ? max(local.guest_accelerator[*].count...) : -1 - is_static_node_pool_with_gpus = var.static_node_count != null && local.allocatable_gpu_per_node != -1 - static_gpu_count = local.is_static_node_pool_with_gpus ? var.static_node_count * local.allocatable_gpu_per_node : 0 - gpu_taint = local.has_gpu ? [{ - key = "nvidia.com/gpu" - value = "present" - effect = "NO_SCHEDULE" - }] : [] - - autoscale_set = var.autoscaling_total_min_nodes != 0 || var.autoscaling_total_max_nodes != 1000 - static_node_set = var.static_node_count != null - initial_node_set = try(var.initial_node_count > 0, false) - - module_unique_id = replace(lower(var.internal_ghpc_module_id), "/[^a-z0-9\\-]/", "") -} - - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] -} - -module "tpu" { - source = "../../internal/tpu-definition" - - machine_type = var.machine_type - placement_policy = var.placement_policy -} - - -data "google_container_cluster" "gke_cluster" { - name = local.cluster_name - location = local.cluster_location -} - -resource "google_container_node_pool" "node_pool" { - provider = google-beta - - count = max(var.num_node_pools, var.num_slices) - - name = (max(var.num_node_pools, var.num_slices) == 1) ? coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])) : join("-", [coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])), count.index]) - cluster = var.cluster_id - node_locations = var.zones - - node_count = var.static_node_count - dynamic "autoscaling" { - for_each = local.static_node_set ? [] : [1] - content { - total_min_node_count = var.autoscaling_total_min_nodes - total_max_node_count = var.autoscaling_total_max_nodes - location_policy = "ANY" - } - } - - initial_node_count = var.initial_node_count - - max_pods_per_node = var.max_pods_per_node - - management { - auto_repair = var.auto_repair - auto_upgrade = var.auto_upgrade - } - - upgrade_settings { - strategy = local.upgrade_settings.strategy - max_surge = local.upgrade_settings.max_surge - max_unavailable = local.upgrade_settings.max_unavailable - } - - dynamic "placement_policy" { - for_each = var.placement_policy.type != null ? [1] : [] - content { - type = var.placement_policy.type - policy_name = var.placement_policy.name - tpu_topology = module.tpu.is_tpu ? var.placement_policy.tpu_topology : null - } - } - - dynamic "queued_provisioning" { - for_each = var.enable_queued_provisioning ? [1] : [] - content { - enabled = true - } - } - - node_config { - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - resource_labels = local.labels - labels = var.kubernetes_labels - service_account = var.service_account_email - oauth_scopes = var.service_account_scopes - machine_type = var.machine_type - spot = var.spot - image_type = var.image_type - flex_start = var.enable_flex_start - max_run_duration = var.max_run_duration != null ? "${var.max_run_duration}s" : null - - dynamic "guest_accelerator" { - for_each = local.guest_accelerator - iterator = ga - content { - type = coalesce(ga.value.type, try(local.generated_guest_accelerator[0].type, "")) - count = coalesce(try(ga.value.count, 0) > 0 ? ga.value.count : try(local.generated_guest_accelerator[0].count, "0")) - - gpu_partition_size = try(ga.value.gpu_partition_size, null) - - dynamic "gpu_driver_installation_config" { - # in case user did not specify guest_accelerator settings, we need a try to default to [] - for_each = try([ga.value.gpu_driver_installation_config], [{ gpu_driver_version = "DEFAULT" }]) - iterator = gdic - content { - gpu_driver_version = gdic.value.gpu_driver_version - } - } - - dynamic "gpu_sharing_config" { - for_each = try(ga.value.gpu_sharing_config == null, true) ? [] : [ga.value.gpu_sharing_config] - iterator = gsc - content { - gpu_sharing_strategy = gsc.value.gpu_sharing_strategy - max_shared_clients_per_gpu = gsc.value.max_shared_clients_per_gpu - } - } - } - } - - dynamic "taint" { - for_each = concat(var.taints, local.gpu_taint, module.tpu.tpu_taint) - content { - key = taint.value.key - value = taint.value.value - effect = taint.value.effect - } - } - - dynamic "ephemeral_storage_local_ssd_config" { - for_each = local.local_ssd_config.local_ssd_count_ephemeral_storage != null ? [1] : [] - content { - local_ssd_count = local.local_ssd_config.local_ssd_count_ephemeral_storage - } - } - - dynamic "local_nvme_ssd_block_config" { - for_each = local.local_ssd_config.local_ssd_count_nvme_block != null ? [1] : [] - content { - local_ssd_count = local.local_ssd_config.local_ssd_count_nvme_block - } - } - - shielded_instance_config { - enable_secure_boot = var.enable_secure_boot - enable_integrity_monitoring = true - } - - dynamic "gcfs_config" { - for_each = var.enable_gcfs ? [1] : [] - content { - enabled = true - } - } - - gvnic { - enabled = var.image_type == "COS_CONTAINERD" - } - - dynamic "advanced_machine_features" { - for_each = local.set_threads_per_core ? [1] : [] - content { - threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf - } - } - - # Implied by Workload Identity - workload_metadata_config { - mode = "GKE_METADATA" - } - # Implied by workload identity. - metadata = { - "disable-legacy-endpoints" = "true" - } - - linux_node_config { - sysctls = { - "net.ipv4.tcp_rmem" = "4096 87380 16777216" - "net.ipv4.tcp_wmem" = "4096 16384 16777216" - } - } - - reservation_affinity { - consume_reservation_type = var.reservation_affinity.consume_reservation_type - key = local.is_valid_reservation ? local.reservation_resource_api_label : null - values = local.is_valid_reservation ? (var.is_reservation_active ? local.active_reservation_values : local.default_reservation_values) : null - } - - dynamic "host_maintenance_policy" { - for_each = var.host_maintenance_interval != "" ? [1] : [] - content { - maintenance_interval = var.host_maintenance_interval - } - } - - kubelet_config { - cpu_manager_policy = var.enable_numa_aware_scheduling ? "static" : null - dynamic "topology_manager" { - for_each = var.enable_numa_aware_scheduling ? [1] : [] - content { - policy = "restricted" - } - } - dynamic "memory_manager" { - for_each = var.enable_numa_aware_scheduling ? [1] : [] - content { - policy = "Static" - } - } - } - } - - network_config { - dynamic "additional_node_network_configs" { - for_each = var.additional_networks - - content { - network = additional_node_network_configs.value.network - subnetwork = additional_node_network_configs.value.subnetwork - } - } - - enable_private_nodes = var.enable_private_nodes - } - - timeouts { - create = var.timeout_create - update = var.timeout_update - } - - lifecycle { - ignore_changes = [ - node_config[0].labels, - initial_node_count, - # Ignore local/ephemeral ssd configs as they are tied to machine types. - node_config[0].ephemeral_storage_local_ssd_config, - node_config[0].local_nvme_ssd_block_config, - ] - precondition { - condition = (var.max_pods_per_node == null) || (data.google_container_cluster.gke_cluster.networking_mode == "VPC_NATIVE") - error_message = "max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." - } - precondition { - condition = !local.static_node_set || !local.autoscale_set - error_message = "static_node_count cannot be set with either autoscaling_total_min_nodes or autoscaling_total_max_nodes." - } - precondition { - condition = !local.static_node_set || !local.initial_node_set - error_message = "initial_node_count cannot be set with static_node_count." - } - precondition { - condition = !local.initial_node_set || (coalesce(var.initial_node_count, 0) >= var.autoscaling_total_min_nodes && coalesce(var.initial_node_count, 0) <= var.autoscaling_total_max_nodes) - error_message = "initial_node_count must be between autoscaling_total_min_nodes and autoscaling_total_max_nodes included." - } - precondition { - condition = !(coalesce(local.local_ssd_config.local_ssd_count_ephemeral_storage, 0) > 0 && coalesce(local.local_ssd_config.local_ssd_count_nvme_block, 0) > 0) - error_message = "Only one of local_ssd_count_ephemeral_storage or local_ssd_count_nvme_block can be set to a non-zero value." - } - precondition { - condition = ( - (var.reservation_affinity.consume_reservation_type != "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 0) || - (var.reservation_affinity.consume_reservation_type == "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 1) - ) - error_message = <<-EOT - When using NO_RESERVATION or ANY_RESERVATION as the `consume_reservation_type`, `specific_reservations` cannot be set. - On the other hand, with SPECIFIC_RESERVATION you must set `specific_reservations`. - EOT - } - precondition { - condition = ( - (local.input_specific_reservations_count == 0) || - ((length(local.verified_specific_reservations) == 1 || !var.is_reservation_active) && - length(local.specific_reservation_requirement_violations) == 0) - ) - error_message = <<-EOT - Check if your reservation is configured correctly: - - A reservation with the name must exist in the specified project and one of the specified zones - - - Its consumption type must be "specific" - %{for property in local.specific_reservation_requirement_violations} - - ${local.specific_reservation_requirement_violation_messages[property]} - %{endfor} - EOT - } - precondition { - condition = ( - (local.input_specific_reservations_count == 0) || - (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) == 0) || - (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) > 0 && try(local.input_reservation_projects[0], var.project_id) == var.project_id) - ) - error_message = "Shared extended reservations are not supported by GKE." - } - precondition { - condition = contains(["SURGE"], local.upgrade_settings.strategy) - error_message = "Only SURGE strategy is supported" - } - precondition { - condition = local.upgrade_settings.max_unavailable >= 0 - error_message = "max_unavailable should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_surge >= 0 - error_message = "max_surge should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 - error_message = "At least one of max_unavailable or max_surge must greater than 0" - } - precondition { - condition = var.placement_policy.type != "COMPACT" || (var.zones != null ? (length(var.zones) == 1) : false) - error_message = "Compact placement is only available for node pools operating in a single zone." - } - precondition { - condition = var.placement_policy.type != "COMPACT" || local.upgrade_settings.strategy != "BLUE_GREEN" - error_message = "Compact placement is not supported with blue-green upgrades." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.placement_policy.type == "COMPACT") - error_message = "placement_policy cannot be COMPACT when enable_queued_provisioning is true." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.reservation_affinity.consume_reservation_type != "NO_RESERVATION") - error_message = "reservation_affinity should be NO_RESERVATION when enable_queued_provisioning is true." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.autoscaling_total_min_nodes != 0) - error_message = "autoscaling_total_min_nodes should be 0 when enable_queued_provisioning is true." - } - precondition { - condition = !(var.num_node_pools > 1 && var.num_slices > 1) - error_message = "num_node_pools is for CPUs and GPUS, and num_slices is for TPUs. Both cannot be set at the same time to create a group of identical nodepools / slices." - } - precondition { - condition = !(var.num_node_pools == 0 && var.num_slices == 0) - error_message = "Either num_node_pools (for CPUs and GPUS) or num_slices (for TPUs) should be set to a positive integer value." - } - precondition { - condition = !(var.num_node_pools < 0 || var.num_slices < 0) - error_message = "Negative integer value of num_node_pools or num_slices is not valid. Please use a positive integer value to set num_node_pools for CPUs and GPUS, and num_slices for TPUs." - } - precondition { - condition = var.enable_flex_start == true ? (var.auto_repair == false) : true - error_message = "enable_flex_start needs node auto_repair set to false." - } - precondition { - condition = var.enable_flex_start == true ? (var.static_node_count == null) : true - error_message = "enable_flex_start does not work with static_node_count. static_node_count should be set to null." - } - precondition { - condition = var.enable_flex_start == true ? (var.reservation_affinity.consume_reservation_type == "NO_RESERVATION") : true - error_message = "enable_flex_start only works with reservation_affinity consume_reservation_type NO_RESERVATION." - } - precondition { - condition = var.enable_flex_start == true ? (var.spot == false) : true - error_message = "Both enable_flex_start and spot consumption option cannot be set to true at the same time." - } - } -} - -locals { - supported_machine_types_for_install_dependencies = ["a3-highgpu-8g", "a3-megagpu-8g"] -} - -# Replicates GKE's naming logic for its instance templates. The full -# pattern is "gke-{cluster_name}-{nodepool_name}-{hash}". -# -# This code builds the "{cluster_name}-{nodepool_name}" prefix, which is -# capped at 32 characters plus a dash '-' in between, by truncating names if needed: -# - If both names > 16 chars, both are cut to 16. -# - If one name > 16, it's shortened so the combined name length is 32. -data "google_compute_region_instance_template" "instance_template" { - for_each = { for idx, np in google_container_node_pool.node_pool : idx => np } - project = var.project_id - filter = "name: gke-${ - (length(local.cluster_name) <= 16 && length(each.value.name) <= 16) ? "${local.cluster_name}-${each.value.name}" : - (length(local.cluster_name) > 16 && length(each.value.name) > 16) ? "${substr(local.cluster_name, 0, 16)}-${substr(each.value.name, 0, 16)}" : - (length(local.cluster_name) > 16) ? "${substr(local.cluster_name, 0, 32 - length(each.value.name))}-${each.value.name}" : - "${local.cluster_name}-${substr(each.value.name, 0, 32 - length(local.cluster_name))}" - }*" - most_recent = true -} - -resource "null_resource" "install_dependencies" { - count = var.run_workload_script && contains(local.supported_machine_types_for_install_dependencies, var.machine_type) ? 1 : 0 - provisioner "local-exec" { - command = "pip3 install pyyaml" - } -} - -locals { - gpu_direct_setting = lookup(local.gpu_direct_settings, var.machine_type, { gpu_direct_manifests = [], updated_workload_path = "", rxdm_version = "" }) -} - -# execute script to inject rxdm sidecar into workload to enable tcpx for a3-highgpu-8g VM workload -resource "null_resource" "enable_tcpx_in_workload" { - count = var.run_workload_script && var.machine_type == "a3-highgpu-8g" ? 1 : 0 - triggers = { - always_run = timestamp() - } - provisioner "local-exec" { - command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpx-in-workload.py --file ${local.workload_path_tcpx} --rxdm ${local.gpu_direct_setting.rxdm_version}" - } - - depends_on = [null_resource.install_dependencies] -} - -# execute script to inject rxdm sidecar into workload to enable tcpxo for a3-megagpu-8g VM workload -resource "null_resource" "enable_tcpxo_in_workload" { - count = var.run_workload_script && var.machine_type == "a3-megagpu-8g" ? 1 : 0 - triggers = { - always_run = timestamp() - } - provisioner "local-exec" { - command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py --file ${local.workload_path_tcpxo} --rxdm ${local.gpu_direct_setting.rxdm_version}" - } - - depends_on = [null_resource.install_dependencies] -} - -# apply manifest to enable tcpx -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = var.cluster_id - project_id = var.project_id - - apply_manifests = flatten([ - for manifest in local.gpu_direct_setting.gpu_direct_manifests : [ - { - source = manifest - } - ] - ]) -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/metadata.yaml deleted file mode 100644 index e980d595a2..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com -ghpc: - inject_module_id: internal_ghpc_module_id diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/outputs.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/outputs.tf deleted file mode 100644 index 44e1c3d971..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/outputs.tf +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "node_pool_names" { - description = "Names of the node pools." - value = google_container_node_pool.node_pool[*].name -} - -locals { - # Shared core machines only have 1 cpu allocatable, even if they have 2 cpu capacity - vcpu = local.machine_shared_core ? 1 : local.guest_cpus - useable_cpu = local.set_threads_per_core ? local.threads_per_core * local.vcpu / 2 : local.vcpu - - # allocatable resource definition: https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes#cpu_reservations - second_core = local.useable_cpu > 1 ? 1 : 0 - third_fourth_core = local.useable_cpu == 3 ? 1 : local.useable_cpu > 3 ? 2 : 0 - cores_above_four = local.useable_cpu > 4 ? local.useable_cpu - 4 : 0 - - allocatable_cpu = 0.94 + (0.99 * local.second_core) + (0.995 * local.third_fourth_core) + (0.9975 * local.cores_above_four) -} - -output "allocatable_cpu_per_node" { - description = "Number of CPUs available for scheduling pods on each node." - value = local.allocatable_cpu -} - -output "has_gpu" { - description = "Boolean value indicating whether nodes in the pool are configured with GPUs." - value = local.has_gpu -} - -output "allocatable_gpu_per_node" { - description = "Number of GPUs available for scheduling pods on each node." - value = local.allocatable_gpu_per_node -} - -output "static_gpu_count" { - description = "Total number of GPUs in the node pool. Available only for static node pools." - value = local.static_gpu_count -} - -locals { - translate_toleration = { - PREFER_NO_SCHEDULE = "PreferNoSchedule" - NO_SCHEDULE = "NoSchedule" - NO_EXECUTE = "NoExecute" - } - taints = google_container_node_pool.node_pool[0].node_config[0].taint - tolerations = [for taint in local.taints : { - key = taint.key - operator = "Equal" - value = taint.value - effect = lookup(local.translate_toleration, taint.effect, null) - }] -} - -output "tolerations" { - description = "Tolerations needed for a pod to be scheduled on this node pool." - value = local.tolerations -} - -locals { - gpu_direct_enabled = var.machine_type == "a3-highgpu-8g" || var.machine_type == "a3-megagpu-8g" - script_path = { - a3-highgpu-8g = "enable-tcpx-in-workload.py", - a3-megagpu-8g = "enable-tcpxo-in-workload.py" - } - nccl_path = var.machine_type == "a3-highgpu-8g" ? "configs" : "scripts" - gpu_direct_instruction = <<-EOT - Since you are using ${var.machine_type} machine type that has GPUDirect support, your nodepool had been configured with the required plugins. - To fully utilize GPUDirect you will need to add some components into your workload manifest. Details below: - - A sample GKE job that has GPUDirect enabled and NCCL test included has been generated locally at: - ${abspath(local.gpu_direct_setting.updated_workload_path)} - - You can use the following commands to submit the sample job: - kubectl create -f ${abspath(local.gpu_direct_setting.updated_workload_path)} - After submitting the sample job, you can validate the GPU performance by initiating NCCL test included in the sample workload: - NCCL test can be initiated from any one of the sample job Pods and coordinate with the peer Pods: - export POD_NAME=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | head -n 1) - export PEER_POD_IPS=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.status.podIP}}{{" "}}{{end}}') - kubectl exec --stdin --tty --container=nccl-test $POD_NAME -- /${local.nccl_path}/allgather.sh $PEER_POD_IPS - - If you would like to enable GPUDirect for your own workload, please follow the below steps: - export WORKLOAD_PATH=<> - python3 ${abspath("${path.module}/gpu-direct-workload/scripts/${lookup(local.script_path, var.machine_type, "")}")} --file $WORKLOAD_PATH --rxdm ${local.gpu_direct_setting.rxdm_version} - **WARNING** - The "--rxdm" version is tied to the nccl-tcpx/o-installer that had been deployed to your cluster, changing it to other value might have impact on performance - **WARNING** - - Or you can also follow our GPUDirect user guide to update your workload - https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#add-gpudirect-manifests - EOT -} - -output "instructions" { - description = "Instructions for submitting the sample GPUDirect enabled job." - value = local.gpu_direct_enabled ? local.gpu_direct_instruction : null -} - -output "node_count_static" { - description = "The number of static nodes in node-pool." - value = coalesce(var.static_node_count, var.initial_node_count, 0) -} - -output "guest_accelerator" { - description = "The accelerator type of the nodes." - value = local.guest_accelerator -} - -output "cluster_id" { - description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." - value = var.cluster_id -} - -output "machine_type" { - description = "Machine Type" - value = var.machine_type -} - -output "instance_templates" { - description = "The URLs of Instance Templates" - value = [for key, template in data.google_compute_region_instance_template.instance_template : template.self_link] -} - -output "tpu_accelerator_type" { - description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." - value = module.tpu.is_tpu ? module.tpu.tpu_accelerator_type : null -} - -output "tpu_topology" { - description = "The topology of the TPU slice (e.g., '4x4')." - value = module.tpu.is_tpu ? module.tpu.tpu_topology : null -} - -output "tpu_chips_per_node" { - description = "The number of TPU chips on each node in the pool." - value = module.tpu.is_tpu ? module.tpu.tpu_chips_per_node : null -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf deleted file mode 100644 index 7c29e3902a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# Split the input into three different lists where the details of a given reservation are at the same index across these lists. -locals { - # Specific block of an extended reservation can be targeted with exr-one/reservationBlocks/exr-one-block-1 - # Data source needs to be queried with the reservation name only. So, we extract the reservation name - input_reservation_names = [for r in try(var.reservation_affinity.specific_reservations, []) : split("/", r.name)[0]] - input_reservation_projects = [for r in try(var.reservation_affinity.specific_reservations, []) : coalesce(r.project, var.project_id)] - # We, also, remember the suffix "/reservationBlocks/exr-one-block-1" for use elsewhere afterwards - input_reservation_suffixes = [for r in try(var.reservation_affinity.specific_reservations, []) : substr(r.name, length(split("/", r.name)[0]), -1)] - # Adding this variable to by-pass the machine-type validation for TPUs - is_tpu = var.placement_policy.tpu_topology != null -} - -data "google_compute_reservation" "specific_reservations" { - for_each = ( - local.input_specific_reservations_count == 0 ? - {} : - { - for pair in flatten([ - for zone in try(var.zones, []) : [ - for i, reservation_name in try(local.input_reservation_names, []) : { - key : "${local.input_reservation_projects[i]}/${zone}/${reservation_name}" - zone : zone - reservation_name : reservation_name - project : local.input_reservation_projects[i] - } - ] - ]) : - pair.key => pair - } - ) - name = each.value.reservation_name - zone = each.value.zone - project = each.value.project -} - -locals { - generated_guest_accelerator = module.gpu.machine_type_guest_accelerator - reservation_resource_api_label = "compute.googleapis.com/reservation-name" - input_specific_reservations_count = try(length(var.reservation_affinity.specific_reservations), 0) - - # Filter specific reservations - verified_specific_reservations = [for k, v in data.google_compute_reservation.specific_reservations : v if(v.specific_reservation != null && v.specific_reservation_required == true)] - - # Build two maps to be used to compare the VM properties between reservations and the node pool - # Validation of only machine-type for CPUs and and both machine-type and guest-accelerators for GPUs - # Skip this for TPUs ( returns an empty list to skip the machine-type validation for aggregate TPU reservations) - reservation_vm_properties = local.is_tpu ? [] : [for reservation in local.verified_specific_reservations : { - "machine_type" : try(reservation.specific_reservation[0].instance_properties[0].machine_type, "") - "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs - { for acc in try(reservation.specific_reservation[0].instance_properties[0].guest_accelerators, []) : acc.accelerator_type => acc.accelerator_count } - ) : {} # If no GPUs, it's an empty map {} - }] - - nodepool_vm_properties = { - "machine_type" : var.machine_type - "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs - { for acc in try(local.guest_accelerator, []) : coalesce(acc.type, try(local.generated_guest_accelerator[0].type, "")) => coalesce(acc.count, try(local.generated_guest_accelerator[0].count, 0)) } - ) : {} # If no GPUs, it's an empty map {} - } - - # Compare two maps by counting the keys that mismatch. - # Know that in map comparison the order of keys does not matter. That is {NVME: x, SCSI: y} and {SCSI: y, NVME: x} are equal - # As of this writing, there is only one reservation supported by the Node Pool API. So, directly accessing it from the list - specific_reservation_requirement_violations = length(local.reservation_vm_properties) == 0 ? [] : [for k, v in local.nodepool_vm_properties : k if v != local.reservation_vm_properties[0][k]] - - specific_reservation_requirement_violation_messages = { - "machine_type" : <<-EOT - The reservation has "${try(local.reservation_vm_properties[0].machine_type, "")}" machine type and the node pool has "${local.nodepool_vm_properties.machine_type}". Check the relevant node pool setting: "machine_type" - EOT - "guest_accelerators" : <<-EOT - The reservation has ${jsonencode(try(local.reservation_vm_properties[0].guest_accelerators, {}))} accelerators and the node pool has ${jsonencode(try(local.nodepool_vm_properties.guest_accelerators, {}))}. Check the relevant node pool setting: "guest_accelerator". When unspecified, for the machine_type=${var.machine_type}, the default is guest_accelerator=${jsonencode(try(local.generated_guest_accelerator, [{}]))}. - EOT - } -} - -locals { - # Check if reservation is valid, that is, if it exists, there should be only 1 verified specific reservation or the reservation doesn't exist - is_valid_reservation = length(local.verified_specific_reservations) == 1 || !var.is_reservation_active - - # Build the list of reservation names when var.is_reservation_active is true - active_reservation_values = [ - for i, r in local.verified_specific_reservations : - length(local.input_reservation_suffixes[i]) > 0 ? - format("%s%s", r.name, local.input_reservation_suffixes[i]) : - "projects/${r.project}/reservations/${r.name}" - ] - - # Define a default reservation value if no specific reservations are present - specific_reservation_name = length(local.input_reservation_names) > 0 ? local.input_reservation_names[0] : "" - default_reservation_values = ["projects/${var.project_id}/reservations/${local.specific_reservation_name}"] -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf deleted file mode 100644 index e582db33da..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' - -# "inputs": -# var.machine_type: Machine type for the instance being evaluated. -# var.threads_per_core : Sets the number of threads per physical core, where 0 -# has behavior described in description. - -# "outputs": -# local.set_threads_per_core: bool that tells if threads per core should be set, -# to be used with a dynamic block. -# local.threads_per_core: actual threads_per_core to be used. - -locals { - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - machine_shared_core = length(local.machine_vals) <= 2 - machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) - - smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) - smt_capable_vcpu = local.machine_vcpus >= 2 - - smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core - set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) - threads_per_core = var.threads_per_core == 2 ? 2 : 1 -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/variables.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/variables.tf deleted file mode 100644 index b44ea28d57..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/variables.tf +++ /dev/null @@ -1,487 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "cluster_id" { - description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" - type = string -} - -variable "zones" { - description = "A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable." - type = list(string) - default = null -} - -variable "name" { - description = <<-EOD - The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix. - If setting manually, ensure a unique value across all gke-node-pools. - EOD - type = string - default = null - - validation { - # Check if the variable is null OR if it matches the GCP resource naming regex. - condition = var.name == null || can(regex("^[a-z]([-a-z0-9]{0,34}[a-z0-9])?$", var.name)) - error_message = <<-EOD - If provided, the node pool name must be between 1 and 36 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. - Underscores are not allowed. A shorter length is enforced to accommodate a suffix when creating multiple node pools. - EOD - } -} - -variable "internal_ghpc_module_id" { - description = "DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide)." - type = string -} - -variable "machine_type" { - description = "The name of a Google Compute Engine machine type." - type = string - default = "c2-standard-60" -} - -variable "disk_size_gb" { - description = "Size of disk for each node." - type = number - default = 100 -} - -variable "disk_type" { - description = "Disk type for each node." - type = string - default = null -} - -variable "enable_gcfs" { - description = "Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config)." - type = bool - default = false -} - -variable "enable_secure_boot" { - description = "Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info." - type = bool - default = true -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = optional(string) - count = optional(number, 0) - gpu_driver_installation_config = optional(object({ - gpu_driver_version = string - }), { gpu_driver_version = "DEFAULT" }) - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = string - max_shared_clients_per_gpu = number - })) - })) - default = [] - nullable = false - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.count != null]) - error_message = "var.guest_accelerator[*].count cannot be null" - } - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.count >= 0]) - error_message = "var.guest_accelerator[*].count must never be negative" - } - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.gpu_driver_installation_config != null]) - error_message = "var.guest_accelerator[*].gpu_driver_installation_config must not be null; leave unset to enable GKE to select default GPU driver installation" - } -} - -variable "image_type" { - description = "The default image type used by NAP once a new node pool is being created. Use either COS_CONTAINERD or UBUNTU_CONTAINERD." - type = string - default = "COS_CONTAINERD" -} - -variable "local_ssd_count_ephemeral_storage" { - description = <<-EOT - The number of local SSDs to attach to each node to back ephemeral storage. - Uses NVMe interfaces. Must be supported by `machine_type`. - When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. - [See above](#local-ssd-storage) for more info. - EOT - type = number - default = null -} - -variable "local_ssd_count_nvme_block" { - description = <<-EOT - The number of local SSDs to attach to each node to back block storage. - Uses NVMe interfaces. Must be supported by `machine_type`. - When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. - [See above](#local-ssd-storage) for more info. - - EOT - type = number - default = null -} - -variable "autoscaling_total_min_nodes" { - description = "Total minimum number of nodes in the NodePool." - type = number - default = 0 -} - -variable "autoscaling_total_max_nodes" { - description = "Total maximum number of nodes in the NodePool." - type = number - default = 1000 -} - -variable "static_node_count" { - description = "The static number of nodes in the node pool. If set, autoscaling will be disabled." - type = number - default = null -} - -variable "is_reservation_active" { - description = "Whether the specified reservation is already created." - type = bool - default = true -} - -variable "auto_repair" { - description = "Whether the nodes will be automatically repaired." - type = bool - default = true -} - -variable "auto_upgrade" { - description = "Whether the nodes will be automatically upgraded." - type = bool - default = false -} - -variable "threads_per_core" { - description = <<-EOT - Sets the number of threads per physical core. By setting threads_per_core - to 2, Simultaneous Multithreading (SMT) is enabled extending the total number - of virtual cores. For example, a machine of type c2-standard-60 will have 60 - virtual cores with threads_per_core equal to 2. With threads_per_core equal - to 1 (SMT turned off), only the 30 physical cores will be available on the VM. - - The default value of \"0\" will turn off SMT for supported machine types, and - will fall back to GCE defaults for unsupported machine types (t2d, shared-core - instances, or instances with less than 2 vCPU). - - Disabling SMT can be more performant in many HPC workloads, therefore it is - disabled by default where compatible. - - null = SMT configuration will use the GCE defaults for the machine type - 0 = SMT will be disabled where compatible (default) - 1 = SMT will always be disabled (will fail on incompatible machine types) - 2 = SMT will always be enabled (will fail on incompatible machine types) - EOT - type = number - default = 0 - - validation { - condition = var.threads_per_core == null || try(var.threads_per_core >= 0, false) && try(var.threads_per_core <= 2, false) - error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." - } -} - -variable "spot" { - description = "Provision VMs using discounted Spot pricing, allowing for preemption" - type = bool - default = false -} - -# tflint-ignore: terraform_unused_declarations -variable "compact_placement" { - description = "DEPRECATED: Use `placement_policy`" - type = bool - default = null - validation { - condition = var.compact_placement == null - error_message = "`compact_placement` is deprecated. Use `placement_policy` instead" - } -} - -variable "placement_policy" { - description = <<-EOT - Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool. - It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - EOT - - type = object({ - type = string - name = optional(string) - tpu_topology = optional(string) - }) - default = { - type = null - name = null - tpu_topology = null - } - validation { - condition = var.placement_policy.type == null || try(contains(["COMPACT"], var.placement_policy.type), false) - error_message = "`COMPACT` is the only supported value for `placement_policy.type`." - } -} - -variable "service_account_email" { - description = "Service account e-mail address to use with the node pool" - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to to use with the node pool." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "taints" { - description = "Taints to be applied to the system node pool." - type = list(object({ - key = string - value = any - effect = string - })) - default = [] -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} - -variable "kubernetes_labels" { - description = <<-EOT - Kubernetes labels to be applied to each node in the node group. Key-value pairs. - (The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) - EOT - type = map(string) - default = null -} - -variable "timeout_create" { - description = "Timeout for creating a node pool" - type = string - default = null -} - -variable "timeout_update" { - description = "Timeout for updating a node pool" - type = string - default = null -} - -# Deprecated - -# tflint-ignore: terraform_unused_declarations -variable "total_min_nodes" { - description = "DEPRECATED: Use autoscaling_total_min_nodes." - type = number - default = null - validation { - condition = var.total_min_nodes == null - error_message = "total_min_nodes was renamed to autoscaling_total_min_nodes and is deprecated; use autoscaling_total_min_nodes" - } -} - -# tflint-ignore: terraform_unused_declarations -variable "total_max_nodes" { - description = "DEPRECATED: Use autoscaling_total_max_nodes." - type = number - default = null - validation { - condition = var.total_max_nodes == null - error_message = "total_max_nodes was renamed to autoscaling_total_max_nodes and is deprecated; use autoscaling_total_max_nodes" - } -} - -# tflint-ignore: terraform_unused_declarations -variable "service_account" { - description = "DEPRECATED: use service_account_email and scopes." - type = object({ - email = string, - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "service_account is deprecated and replaced with service_account_email and scopes." - } -} - -variable "additional_networks" { - description = "Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool" - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - })) - nullable = false -} - -variable "reservation_affinity" { - description = <<-EOT - Reservation resource to consume. When targeting SPECIFIC_RESERVATION, specific_reservations needs be specified. - Even though specific_reservations is a list, only one reservation is allowed by the NodePool API. - It is assumed that the specified reservation exists and has available capacity. - For a shared reservation, specify the project_id as well in which it was created. - To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared - EOT - type = object({ - consume_reservation_type = string - specific_reservations = optional(list(object({ - name = string - project = optional(string) - }))) - }) - default = { - consume_reservation_type = "NO_RESERVATION" - specific_reservations = [] - } - validation { - condition = contains(["NO_RESERVATION", "ANY_RESERVATION", "SPECIFIC_RESERVATION"], var.reservation_affinity.consume_reservation_type) - error_message = "Accepted values are: {NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION}" - } -} - -variable "host_maintenance_interval" { - description = "Specifies the frequency of planned maintenance events." - type = string - default = "" - nullable = false - validation { - condition = contains(["", "PERIODIC", "AS_NEEDED"], var.host_maintenance_interval) - error_message = "Invalid host_maintenance_interval value. Must be PERIODIC, AS_NEEDED or the empty string" - } -} - -variable "initial_node_count" { - description = "The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static_node_count and must be set to a value between autoscaling_total_min_nodes and autoscaling_total_max_nodes." - type = number - default = null -} - -variable "gke_version" { - description = "GKE version" - type = string -} - -variable "max_pods_per_node" { - description = "The maximum number of pods per node in this node pool. This will force replacement." - type = number - default = null -} - -variable "upgrade_settings" { - description = <<-EOT - Defines node pool upgrade settings. It is highly recommended that you define all max_surge and max_unavailable. - If max_surge is not specified, it would be set to a default value of 0. - If max_unavailable is not specified, it would be set to a default value of 1. - EOT - type = object({ - strategy = string - max_surge = optional(number) - max_unavailable = optional(number) - }) - default = { - strategy = "SURGE" - max_surge = 0 - max_unavailable = 1 - } -} - -variable "run_workload_script" { - description = "Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only." - type = bool - default = true -} - -variable "enable_queued_provisioning" { - description = "If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool." - type = bool - default = false -} - -variable "enable_flex_start" { - description = <<-EOT - If true, start the node pool with Flex Start provisioning model. - To learn more about flex-start mode, please refer to - https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and - https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest - EOT - type = bool - default = false -} - -variable "max_run_duration" { - description = "The duration (in whole seconds) of the instance. Instance will run and be terminated after then." - type = number - default = null -} - -variable "enable_private_nodes" { - description = "Whether nodes have internal IP addresses only." - type = bool - default = true -} - -variable "num_node_pools" { - description = "Number of node pools to create. This is same as num_slices." - type = number - default = 1 -} - -variable "num_slices" { - description = "Number of TPUs slices to create. This is same as num_node_pools." - type = number - default = 1 -} - -variable "enable_numa_aware_scheduling" { - description = "Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling." - type = bool - default = false -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/versions.tf b/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/versions.tf deleted file mode 100644 index f018d04fc5..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/gke-node-pool/versions.tf +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.5" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 7.2" - } - google-beta = { - source = "hashicorp/google-beta" - version = ">= 7.2" - } - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/README.md b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/README.md deleted file mode 100644 index 3b769e8761..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/README.md +++ /dev/null @@ -1,82 +0,0 @@ -## Description - -This modules create a [resource policy for compute engines](https://cloud.google.com/compute/docs/instances/placement-policies-overview). This policy can be passed to a gke-node-pool module to apply the policy on the node-pool's nodes. - -Note: By default, you can't apply compact placement policies with a max distance value to A3 VMs. To request access to this feature, contact your [Technical Account Manager (TAM)](https://cloud.google.com/tam) or the [Sales team](https://cloud.google.com/contact). - -### Example - -The following example creates a group placement resource policy and applies it to a gke-node-pool. - -```yaml - - id: group_placement_1 - source: modules/compute/resource-policy - settings: - name: gp-np-1 - group_placement_max_distance: 2 - - - id: node_pool_1 - source: modules/compute/gke-node-pool - use: [group_placement_1] - settings: - machine_type: e2-standard-8 - outputs: [instructions] -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google-beta](#requirement\_google-beta) | >= 6.29.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google-beta](#provider\_google-beta) | >= 6.29.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_resource_policy.policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [group\_placement\_max\_distance](#input\_group\_placement\_max\_distance) | The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | `number` | `0` | no | -| [name](#input\_name) | The resource policy's name. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The project ID for the resource policy. | `string` | n/a | yes | -| [region](#input\_region) | The region for the the resource policy. | `string` | n/a | yes | -| [workload\_policy](#input\_workload\_policy) | Describes the workload policy | 
object({
    type                  = optional(string, null)
    max_topology_distance = optional(string, null)
    accelerator_topology  = optional(string, null)
  })
| 
{
  "accelerator_topology": null,
  "max_topology_distance": null,
  "type": null
}
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [placement\_policy](#output\_placement\_policy) | Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions.
The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. | - diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/main.tf b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/main.tf deleted file mode 100644 index 906424ca7c..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/main.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - name = "${var.name}-${random_id.resource_name_suffix.hex}" -} - -resource "google_compute_resource_policy" "policy" { - name = local.name - region = var.region - project = var.project_id - provider = google-beta - - dynamic "workload_policy" { - for_each = var.workload_policy.type != null ? [1] : [] - - content { - type = var.workload_policy.type - max_topology_distance = var.workload_policy.max_topology_distance - accelerator_topology = var.workload_policy.accelerator_topology - } - } - - dynamic "group_placement_policy" { - for_each = var.group_placement_max_distance > 0 ? [1] : [] - - content { - collocation = "COLLOCATED" - max_distance = var.group_placement_max_distance - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/outputs.tf b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/outputs.tf deleted file mode 100644 index c1dc65bcbb..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "placement_policy" { - description = <<-EOT - Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. - It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. - EOT - - value = { - type = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? "COMPACT" : null - name = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? local.name : null - tpu_topology = (var.workload_policy.type != null) ? var.workload_policy.accelerator_topology : null - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/variables.tf b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/variables.tf deleted file mode 100644 index 92434326ca..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/variables.tf +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "The project ID for the resource policy." - type = string -} - -variable "region" { - description = "The region for the the resource policy." - type = string -} - -variable "name" { - description = "The resource policy's name." - type = string - - validation { - # Check if the variable matches the GCP resource naming regex. - condition = can(regex("^[a-z]([-a-z0-9]{0,52}[a-z0-9])?$", var.name)) - error_message = <<-EOD - The resource policy name must be between 1 and 54 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. - Underscores are not allowed. A shorter length is enforced to accommodate a random suffix. - EOD - } -} - -variable "group_placement_max_distance" { - description = <<-EOT - The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - EOT - - type = number - default = 0 -} - -variable "workload_policy" { - description = "Describes the workload policy" - type = object({ - type = optional(string, null) - max_topology_distance = optional(string, null) - accelerator_topology = optional(string, null) - }) - default = { - type = null - max_topology_distance = null - accelerator_topology = null - } - nullable = false -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/versions.tf b/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/versions.tf deleted file mode 100644 index f235fbade3..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/resource-policy/versions.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.29.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:resource-policy/v1.37.2" - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/README.md b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/README.md deleted file mode 100644 index 0c4737e0d9..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/README.md +++ /dev/null @@ -1,257 +0,0 @@ -## Description - -This module creates one or more -[compute VM instances](https://cloud.google.com/compute/docs/instances). - -### Example - -```yaml -- id: compute - source: modules/compute/vm-instance - use: [network1] - settings: - instance_count: 8 - name_prefix: compute - machine_type: c2-standard-60 -``` - -This creates a cluster of 8 compute VMs that are: - -* named `compute-[0-7]` -* on the network defined by the `network1` module -* of type c2-standard-60 - -> **_NOTE:_** Simultaneous Multithreading (SMT) is deactivated by default -> (threads_per_core=1), which means only the physical cores are visible on the -> VM. With SMT disabled, a machine of type c2-standard-60 will only have the 30 -> physical cores visible. To change this, set `threads_per_core=2` under -> settings. - -### VPC Networks - -There are two methods for adding network connectivity to the `vm-instance` -module. The first is shown in the example above, where a `vpc` module or -`pre-existing-vpc` module is used by the `vm-instance` module. When this -happens, the `network_self_link` and `subnetwork_self_link` outputs from the -network are provided as input to the `vm-instance` and a network interface is -defined based on that. This can also be done updating the `network_self_link` and -`subnetwork_self_link` settings directly. - -The alternative option can be used when more than one network needs to be added -to the `vm-instance` or further customization is needed beyond what is provided -via other variables. For this option, the `network_interfaces` variable can be -used to set up one or more network interfaces on the VM instance. The format is -consistent with the terraform `google_compute_instance` `network_interface` -block, and more information can be found in the -[terraform docs][network-interface-tf]. - -> **_NOTE:_** When supplying the `network_interfaces` variable, networks -> associated with the `vm-instance` via use will be ignored in favor of the -> networks added in `network_interfaces`. In addition, `bandwidth_tier` and -> `disable_public_ips` will not apply to networks defined in -> `network_interfaces`. - -[network-interface-tf]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - -### SSH key metadata - -This module will ignore all changes to the `ssh-keys` metadata field that are -typically set by [external Google Cloud tools that automate SSH access][gcpssh] -when not using OS Login. For example, clicking on the Google Cloud Console SSH -button next to VMs in the VM Instances list will temporarily modify VM metadata -to include a dynamically-generated SSH public key. - -[gcpssh]: https://cloud.google.com/compute/docs/connect/add-ssh-keys#metadata - -### Placement - -The `placement_policy` variable can be used to control where your VM instances -are physically located relative to each other within a zone. See the official -placement [guide][guide-link] and [api][api-link] documentation. - -[guide-link]: https://cloud.google.com/compute/docs/instances/define-instance-placement -[api-link]: https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement - -Use the following settings for compact placement: - -```yaml - ... - settings: - instance_count: 4 - machine_type: c2-standard-60 - placement_policy: - collocation: "COLLOCATED" -``` - -By default the above placement policy will always result in the most compact set -of VMs available. If you would like that provisioning failed if some level of -compactness is not obtainable, you can enforce this with the [`max_distance` -setting](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies): - -```yaml - ... - settings: - instance_count: 4 - machine_type: c2-standard-60 - placement_policy: - collocation: "COLLOCATED" - max_distance: 1 -``` - -Use the following settings for spread placement: - -```yaml - ... - settings: - instance_count: 4 - machine_type: n2-standard-4 - placement_policy: - availability_domain_count: 2 -``` - -When `vm_count` is not set, as shown in the examples above, then the VMs will be -added to the placement policy incrementally. This is the **recommended way** to -use placement policies. - -If `vm_count` is specified then VMs will stay in pending state until the -specified number of VMs are created. See the warning below if using this field. - -> [!WARNING] -> When creating a compact placement using `vm_count` with more than 10 VMs, you -> must add `-parallelism=` argument on apply. For example if you have 15 VMs -> in a placement group: `terraform apply -parallelism=15`. This is because -> terraform self limits to 10 parallel requests by default but the create -> instance requests will not succeed until all VMs in the placement group have -> been requested, forming a deadlock. - -### GPU Support - -More information on GPU support in `vm-instance` and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../docs/gpu-support.md) - -## Lifecycle - -The `vm-instance` module will be replaced when the `instance_image` variable is -changed and `terraform apply` is run on the deployment group folder or -`gcluster deploy` is run. However, it will not be automatically replaced if a new -image is created in a family. - -To selectively replace the vm-instance(s), consider running terraform -`apply -replace` such as: - -> See https://developer.hashicorp.com/terraform/cli/commands/plan#replace-address for precise syntax terraform apply -replace=ADDRESS - -```shell -terraform state list -# search for the module ID and resource -terraform apply -replace="address" -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 4.73.0 | -| [google-beta](#requirement\_google-beta) | >= 6.13.0 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.73.0 | -| [google-beta](#provider\_google-beta) | >= 6.13.0 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | -| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance) | resource | -| [google-beta_google_compute_resource_policy.placement_policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | -| [google_compute_address.compute_ip](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_disk.additional_disks](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.replace_vm_trigger_from_placement](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | -| [additional\_persistent\_disks](#input\_additional\_persistent\_disks) | Configurations of additional disks to be included on the partition nodes. | 
object({
    count = optional(number, 0)
    type  = optional(string, "pd-balanced")
    size  = optional(number, 200)
  })
| `{}` | no | -| [allocate\_ip](#input\_allocate\_ip) | If not null, allocate IPs with the given configuration. See details at
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address | 
object({
    address_type = optional(string, "INTERNAL")
    purpose      = optional(string),
    network_tier = optional(string),
    ip_version   = optional(string, "IPV4"),
  })
| `null` | no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | -| [automatic\_restart](#input\_automatic\_restart) | Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user). | `bool` | `null` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will optionally be used name resources according to `name_prefix` | `string` | n/a | yes | -| [disable\_public\_ips](#input\_disable\_public\_ips) | If set to true, instances will not have public IPs | `bool` | `false` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | -| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-standard"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | -| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_count](#input\_local\_ssd\_count) | The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd. | `number` | `0` | no | -| [local\_ssd\_interface](#input\_local\_ssd\_interface) | Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set. | `string` | `"NVME"` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface

**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply
to network interfaces defined in this variable.

Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "MRDMA", "IRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. Can use "default" for the default network. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `null` | no | -| [placement\_policy](#input\_placement\_policy) | Control where your VM instances are physically located relative to each other within a zone.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy#nested_group_placement_policy | `any` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [provisioning\_model](#input\_provisioning\_model) | Provisioning model for cloud instance. | `string` | `null` | no | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME
- RESERVATION\_NAME

Must be a "SPECIFIC\_RESERVATION"
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | -| [service\_account](#input\_service\_account) | DEPRECATED - Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [spot](#input\_spot) | DEPRECATED - Use `provisioning_model` instead. | `bool` | `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | -| [zone](#input\_zone) | Compute Platform zone | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [external\_ip](#output\_external\_ip) | External IP of the instances (if enabled) | -| [instructions](#output\_instructions) | Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions. | -| [internal\_ip](#output\_internal\_ip) | Internal IP of the instances | -| [name](#output\_name) | Names of instances created | -| [self\_link](#output\_self\_link) | The tuple URIs of the created instances | - diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/compute_image.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/main.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/main.tf deleted file mode 100644 index 0a8c7d354e..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/main.tf +++ /dev/null @@ -1,334 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "vm-instance", ghpc_role = "compute" }) -} - -module "gpu" { - source = "../../internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - native_fstype = [] - startup_script = local.startup_from_network_storage != null ? ( - { startup-script = local.startup_from_network_storage }) : {} - network_storage = var.network_storage != null ? ( - { network_storage = jsonencode(var.network_storage) }) : {} - - prefix_optional_deployment_name = var.name_prefix != null ? var.name_prefix : var.deployment_name - prefix_always_deployment_name = var.name_prefix != null ? "${var.deployment_name}-${var.name_prefix}" : var.deployment_name - resource_prefix = var.add_deployment_name_before_prefix ? local.prefix_always_deployment_name : local.prefix_optional_deployment_name - - enable_gvnic = var.bandwidth_tier != "not_enabled" - enable_tier_1 = var.bandwidth_tier == "tier_1_enabled" - - provisioning_model = var.provisioning_model - - spot = var.provisioning_model == "SPOT" - - # compact_placement : true when placement policy is provided and collocation set; false if unset - compact_placement = try(var.placement_policy.collocation, null) != null - - gpu_attached = contains(["a2", "g2"], local.machine_family) || length(local.guest_accelerator) > 0 - - # both of these must be false if either compact placement or preemptible/spot instances are used - # automatic restart is tolerant of GPUs while on host maintenance is not - automatic_restart_default = local.compact_placement || local.spot ? false : null - on_host_maintenance_default = local.compact_placement || local.spot || local.gpu_attached ? "TERMINATE" : "MIGRATE" - - automatic_restart = ( - var.automatic_restart != null - ? var.automatic_restart - : local.automatic_restart_default - ) - - on_host_maintenance = ( - var.on_host_maintenance != null - ? var.on_host_maintenance - : local.on_host_maintenance_default - ) - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - # Network Interfaces - # Support for `use` input and base network parameters like `network_self_link` and `subnetwork_self_link` - empty_access_config = { - nat_ip = null, - public_ptr_domain_name = null, - network_tier = null - } - default_network_interface = { - network = var.network_self_link - subnetwork = var.subnetwork_self_link - subnetwork_project = null # will populate from subnetwork_self_link - network_ip = null - nic_type = local.enable_gvnic ? "GVNIC" : null - stack_type = null - queue_count = null - access_config = var.disable_public_ips ? [] : [local.empty_access_config] - ipv6_access_config = [] - alias_ip_range = [] - } - network_interfaces = coalescelist(var.network_interfaces, [local.default_network_interface]) - network_interfaces_with_ips = var.allocate_ip == null ? local.network_interfaces : [ - for i, interface in local.network_interfaces : - merge(interface, { - network_ip = google_compute_address.compute_ip[i].address - }) - ] -} - -resource "null_resource" "image" { - triggers = { - name = try(var.instance_image.name, null), - family = try(var.instance_image.family, null), - project = try(var.instance_image.project, null) - } -} - -resource "google_compute_disk" "additional_disks" { - project = var.project_id - - count = var.instance_count * var.additional_persistent_disks.count - - # NB: this resource array must be sliced accounting for var.instance_count - name = "${local.resource_prefix}-disk-${count.index}" - type = var.additional_persistent_disks.type - size = var.additional_persistent_disks.size - labels = local.labels - zone = var.zone -} - -resource "google_compute_resource_policy" "placement_policy" { - project = var.project_id - provider = google-beta - - count = var.placement_policy != null ? 1 : 0 - name = "${local.resource_prefix}-vm-instance-placement" - group_placement_policy { - vm_count = try(var.placement_policy.vm_count, null) - availability_domain_count = try(var.placement_policy.availability_domain_count, null) - collocation = try(var.placement_policy.collocation, null) - max_distance = try(var.placement_policy.max_distance, null) - } -} - -resource "null_resource" "replace_vm_trigger_from_placement" { - triggers = { - vm_count = try(tostring(var.placement_policy.vm_count), "") - availability_domain_count = try(tostring(var.placement_policy.availability_domain_count), "") - max_distance = try(tostring(var.placement_policy.max_distance), "") - collocation = try(var.placement_policy.collocation, "") - } -} - -resource "google_compute_address" "compute_ip" { - project = var.project_id - - count = var.allocate_ip != null ? length(local.network_interfaces) : 0 - - name = "${local.resource_prefix}-${count.index}" - - address = local.network_interfaces[count.index].network_ip - region = var.region - network = can(coalesce(local.network_interfaces[count.index].subnetwork)) ? null : local.network_interfaces[count.index].network - subnetwork = local.network_interfaces[count.index].subnetwork - address_type = var.allocate_ip.address_type - purpose = var.allocate_ip.purpose - network_tier = var.allocate_ip.network_tier - ip_version = var.allocate_ip.ip_version -} - -resource "google_compute_instance" "compute_vm" { - project = var.project_id - provider = google-beta - - count = var.instance_count - - depends_on = [var.network_self_link, var.network_storage] - - name = "${local.resource_prefix}-${count.index}" - min_cpu_platform = var.min_cpu_platform - machine_type = var.machine_type - zone = var.zone - - resource_policies = google_compute_resource_policy.placement_policy[*].self_link - - tags = var.tags - labels = local.labels - - boot_disk { - initialize_params { - image = data.google_compute_image.compute_image.self_link - size = var.disk_size_gb - type = var.disk_type - labels = local.labels - } - - device_name = "${local.resource_prefix}-boot-disk-${count.index}" - auto_delete = var.auto_delete_boot_disk - } - - dynamic "attached_disk" { - for_each = slice( - google_compute_disk.additional_disks, - var.additional_persistent_disks.count * count.index, - var.additional_persistent_disks.count * count.index + var.additional_persistent_disks.count, - ) - - content { - source = attached_disk.value.self_link - device_name = "additional-disk-${attached_disk.key}" - mode = "READ_WRITE" - } - } - - dynamic "scratch_disk" { - for_each = range(var.local_ssd_count) - content { - interface = var.local_ssd_interface - } - } - - dynamic "network_interface" { - for_each = local.network_interfaces_with_ips - - content { - network = network_interface.value.network - subnetwork = network_interface.value.subnetwork - subnetwork_project = network_interface.value.subnetwork_project - network_ip = network_interface.value.network_ip - nic_type = network_interface.value.nic_type - stack_type = network_interface.value.stack_type - queue_count = network_interface.value.queue_count - dynamic "access_config" { - for_each = network_interface.value.access_config - content { - nat_ip = access_config.value.nat_ip - public_ptr_domain_name = access_config.value.public_ptr_domain_name - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = network_interface.value.ipv6_access_config - content { - public_ptr_domain_name = ipv6_access_config.value.public_ptr_domain_name - network_tier = ipv6_access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = network_interface.value.alias_ip_range - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - } - } - - network_performance_config { - total_egress_bandwidth_tier = local.enable_tier_1 ? "TIER_1" : "DEFAULT" - } - - service_account { - email = var.service_account_email - scopes = var.service_account_scopes - } - - dynamic "guest_accelerator" { - for_each = local.guest_accelerator - content { - count = guest_accelerator.value.count - type = guest_accelerator.value.type - } - } - - scheduling { - on_host_maintenance = local.on_host_maintenance - automatic_restart = local.automatic_restart - preemptible = local.spot - provisioning_model = local.provisioning_model - } - - dynamic "advanced_machine_features" { - for_each = local.set_threads_per_core ? [1] : [] - content { - threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf - } - } - - dynamic "reservation_affinity" { - for_each = var.reservation_name == "" ? [] : [1] - content { - type = "SPECIFIC_RESERVATION" - specific_reservation { - key = "compute.googleapis.com/reservation-name" - values = [var.reservation_name] - } - } - } - - metadata = merge( - local.network_storage, - local.startup_script, - local.enable_oslogin, - local.disable_automatic_updates_metadata, - var.metadata - ) - - lifecycle { - ignore_changes = [ - metadata["ssh-keys"], - ] - - replace_triggered_by = [ - null_resource.replace_vm_trigger_from_placement - ] - - precondition { - condition = (length(var.network_interfaces) == 0) != (var.network_self_link == null && var.subnetwork_self_link == null) - error_message = "Exactly one of network_interfaces or network_self_link/subnetwork_self_link must be specified." - } - precondition { - condition = alltrue([for interface in var.network_interfaces : interface.network_ip == null]) || var.instance_count == 1 - error_message = <<-EOT - The network_ip cannot be statically set on vm-instance when the VM instance_count is greater than 1. - Either set the network_ip to null to allow it to be set dynamically for all instances, or create modules for each VM instance with its own network interface. - EOT - } - precondition { - condition = !contains([ - "c3-:pd-standard", - "h3-:pd-standard", - "h3-:pd-ssd", - ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") - error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/outputs.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/outputs.tf deleted file mode 100644 index eab8cb56bd..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "name" { - description = "Names of instances created" - value = google_compute_instance.compute_vm[*].name -} - -output "self_link" { - description = "The tuple URIs of the created instances" - value = google_compute_instance.compute_vm[*].self_link -} - -output "external_ip" { - description = "External IP of the instances (if enabled)" - value = try(google_compute_instance.compute_vm[*].network_interface[0].access_config[0].nat_ip, []) -} - -output "internal_ip" { - description = "Internal IP of the instances" - value = google_compute_instance.compute_vm[*].network_interface[0].network_ip -} - -locals { - first_instance_link = try(google_compute_instance.compute_vm[0].self_link, "no-instance") - ssh_instructions = <<-EOT - Use the following commands to SSH into the first VM created: - gcloud compute ssh ${local.first_instance_link} --project ${var.project_id} - If not accessible from the public internet, use an SSH tunnel through IAP: - gcloud compute ssh ${local.first_instance_link} --tunnel-through-iap --project ${var.project_id} - EOT -} - -output "instructions" { - description = "Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions." - value = var.instance_count > 0 ? local.ssh_instructions : "No instances were created." -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf deleted file mode 100644 index 02bc58e4f7..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "inputs": -# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. -# var.network_storage : to be passed into metadata somewhere else (not here) -# var.startup_script : to be changed into a more complete file system with all the fs runners - -# "outputs": -# local.startup_from_network_storage : A full startup script with all the runners that are not supported -# natively and were included in the network_storage structure - -locals { - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - # Pull out runners to include in startup script - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - startup_script_runner = [{ - content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" - destination = "passed_startup_script.sh" - type = "shell" - }] - - full_runner_list = concat( - local.storage_client_install_runners, - local.mount_runners, - local.startup_script_runner - ) - - startup_from_network_storage = module.netstorage_startup_script.startup_script -} - -module "netstorage_startup_script" { - source = "../../scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.full_runner_list -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf deleted file mode 100644 index e582db33da..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' - -# "inputs": -# var.machine_type: Machine type for the instance being evaluated. -# var.threads_per_core : Sets the number of threads per physical core, where 0 -# has behavior described in description. - -# "outputs": -# local.set_threads_per_core: bool that tells if threads per core should be set, -# to be used with a dynamic block. -# local.threads_per_core: actual threads_per_core to be used. - -locals { - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - machine_shared_core = length(local.machine_vals) <= 2 - machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) - - smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) - smt_capable_vcpu = local.machine_vcpus >= 2 - - smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core - set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) - threads_per_core = var.threads_per_core == 2 ? 2 : 1 -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/variables.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/variables.tf deleted file mode 100644 index 5519b8cd40..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/variables.tf +++ /dev/null @@ -1,452 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "instance_count" { - description = "Number of instances" - type = number - default = 1 -} - -variable "instance_image" { - description = "Instance Image" - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "disk_size_gb" { - description = "Size of disk for instances." - type = number - default = 200 -} - -variable "disk_type" { - description = "Disk type for instances." - type = string - default = "pd-standard" -} - -variable "auto_delete_boot_disk" { - description = "Controls if boot disk should be auto-deleted when instance is deleted." - type = bool - default = true -} - -variable "local_ssd_count" { - description = "The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd." - type = number - default = 0 -} - -variable "local_ssd_interface" { - description = "Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set." - type = string - default = "NVME" -} - -variable "additional_persistent_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = object({ - count = optional(number, 0) - type = optional(string, "pd-balanced") - size = optional(number, 200) - }) - default = {} -} - -variable "name_prefix" { - description = <<-EOT - An optional name for all VM and disk resources. - If not supplied, `deployment_name` will be used. - When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, - then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". - EOT - type = string - default = null -} - -variable "add_deployment_name_before_prefix" { - description = <<-EOT - If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. - See `name_prefix` for further details on resource naming behavior. - EOT - type = bool - default = false -} - -variable "disable_public_ips" { - description = "If set to true, instances will not have public IPs" - type = bool - default = false -} - -variable "machine_type" { - description = "Machine type to use for the instance creation" - type = string - default = "c2-standard-60" -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "deployment_name" { - description = "Name of the deployment, will optionally be used name resources according to `name_prefix`" - type = string -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} - -variable "service_account_email" { - description = "Service account e-mail address to use with the node pool" - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to to use with the node pool." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -# tflint-ignore: terraform_unused_declarations -variable "service_account" { - description = "DEPRECATED - Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string, - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "The 'service_account' setting is deprecated, please use 'var.service_account_email' and 'var.service_account_scopes' instead." - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the VM. Can use \"default\" for the default network." - type = string - default = null -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the VM." - type = string - default = null -} - -variable "network_interfaces" { - description = <<-EOT - A list of network interfaces. The options match that of the terraform - network_interface block of google_compute_instance. For descriptions of the - subfields or more information see the documentation: - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - - **_NOTE:_** If `network_interfaces` are set, `network_self_link` and - `subnetwork_self_link` will be ignored, even if they are provided through - the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply - to network interfaces defined in this variable. - - Subfields: - network (string, required if subnetwork is not supplied) - subnetwork (string, required if network is not supplied) - subnetwork_project (string, optional) - network_ip (string, optional) - nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "MRDMA", "IRDMA"]) - stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) - queue_count (number, optional) - access_config (object, optional) - ipv6_access_config (object, optional) - alias_ip_range (list(object), optional) - EOT - type = list(object({ - network = string, - subnetwork = string, - subnetwork_project = string, - network_ip = string, - nic_type = string, - stack_type = string, - queue_count = number, - access_config = list(object({ - nat_ip = string, - public_ptr_domain_name = string, - network_tier = string - })), - ipv6_access_config = list(object({ - public_ptr_domain_name = string, - network_tier = string - })), - alias_ip_range = list(object({ - ip_cidr_range = string, - subnetwork_range_name = string - })) - })) - default = [] - validation { - condition = alltrue([ - for ni in var.network_interfaces : (ni.network == null) != (ni.subnetwork == null) - ]) - error_message = "All additional network interfaces must define exactly one of \"network\" or \"subnetwork\"." - } - validation { - condition = alltrue([ - for ni in var.network_interfaces : ni.nic_type == "GVNIC" || ni.nic_type == "VIRTIO_NET" || ni.nic_type == "MRDMA" || ni.nic_type == "IRDMA" || ni.nic_type == null - ]) - error_message = "In the variable network_interfaces, field \"nic_type\" must be \"GVNIC\", \"VIRTIO_NET\", \"MRDMA\", \"IRDMA\", or null." - } - validation { - condition = alltrue([ - for ni in var.network_interfaces : ni.stack_type == "IPV4_ONLY" || ni.stack_type == "IPV4_IPV6" || ni.stack_type == null - ]) - error_message = "In the variable network_interfaces, field \"stack_type\" must be either \"IPV4_ONLY\", \"IPV4_IPV6\" or null." - } -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "zone" { - description = "Compute Platform zone" - type = string -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "startup_script" { - description = "Startup script used on the instance" - type = string - default = null -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false -} - -variable "automatic_restart" { - description = "Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user)." - type = bool - default = null -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" - type = string - default = null - validation { - condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) - error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." - } -} - -variable "bandwidth_tier" { - description = <= 0, false) && try(var.threads_per_core <= 2, false) - error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." - } - -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "allocate_ip" { - description = <<-EOT - If not null, allocate IPs with the given configuration. See details at - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address - EOT - type = object({ - address_type = optional(string, "INTERNAL") - purpose = optional(string), - network_tier = optional(string), - ip_version = optional(string, "IPV4"), - }) - default = null -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "reservation_name" { - description = <<-EOD - Name of the reservation to use for VM resources, should be in one of the following formats: - - projects/PROJECT_ID/reservations/RESERVATION_NAME - - RESERVATION_NAME - - Must be a "SPECIFIC_RESERVATION" - Set to empty string if using no reservation or automatically-consumed reservations - EOD - type = string - default = "" - nullable = false - - validation { - condition = length(regexall("^((projects/([a-z0-9-]+)/reservations/)?([a-z0-9-]+))?$", var.reservation_name)) > 0 - error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME', [...] is an optional part." - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/versions.tf b/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/versions.tf deleted file mode 100644 index 0429782c6d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/compute/vm-instance/versions.tf +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.73.0" - } - - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.13.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/README.md deleted file mode 100644 index 285a20bde2..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/README.md +++ /dev/null @@ -1,170 +0,0 @@ -## Description - -This module creates a [Google Cloud Storage (GCS) bucket](https://cloud.google.com/storage). - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -### Example - -The following example will create a bucket named `simulation-results-xxxxxxxx`, -where `xxxxxxxx` is a randomly generated id. - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: - name_prefix: simulation-results - random_suffix: true -``` - -> **_NOTE:_** Use of `random_suffix` may cause the following error when used -> with other modules: -> `value depends on resource attributes that cannot be determined until apply`. -> To resolve this set `random_suffix` to `false` (default). - - - -> **_NOTE:_** Bucket namespace is shared by all users of Google Cloud so it is -> possible to have a bucket name clash with an existing bucket that is not in -> your project. To resolve this try to use a more unique name, or set the -> `random_suffix` variable to `true`. - -## Naming of Bucket - -There are potentially three parts to the bucket name. Each of these parts are -configurable in the blueprint. - -1. A **custom prefix**, provided by the user in the blueprint \ -Provide the custom prefix using the `name_prefix` setting. - -1. The **deployment name**, included by default \ -The deployment name can be excluded by setting `use_deployment_name_in_bucket_name: false`. - -1. A **random id** suffix, excluded by default \ -The random id can be included by setting `random_suffix: true`. - -If none of these are provided (no `name_prefix`, -`use_deployment_name_in_bucket_name: false`, & `random_suffix: false`), then the -bucket name will default to `no-bucket-name-provided`. - -Since bucket namespace is shared by all users of Google Cloud, it is more likely -to experience naming clashes than with other resources. In many cases, adding -the `random_suffix` will resolve the naming clash issue. - -> **Warning**: If a bucket is created with a `random_suffix` and then used as -> the bucket for a startup script in the same deployment group this will cause a -> `not known at apply time` error in terraform. The solution is to either create -> the bucket in a separate deployment group or to remove the random suffix. - -## Mounting - -To mount the Cloud Storage bucket you must first ensure that the GCS Fuse client -has been installed and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. - -If mounting is not automatically handled as described above, the -`cloud-storage-bucket` module outputs runners that can be used with the -`startup-script` module to install the client and mount the file system. See the -following example: - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: {local_mount: /data} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(bucket.client_install_runner) - - $(bucket.mount_runner) -``` - -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | -| [google-beta](#requirement\_google-beta) | >= 6.9.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [google-beta](#provider\_google-beta) | >= 6.9.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource | -| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [autoclass](#input\_autoclass) | Configure bucket autoclass setup

The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern.

The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time.
Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE')

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/autoclass | 
object({
    enabled                = optional(bool, false)
    terminal_storage_class = optional(string, null)
  })
| 
{
  "enabled": false
}
| no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the GCS bucket. | `string` | n/a | yes | -| [enable\_hierarchical\_namespace](#input\_enable\_hierarchical\_namespace) | If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket. | `bool` | `false` | no | -| [enable\_object\_retention](#input\_enable\_object\_retention) | If true, enables retention policy at per object level for the bucket.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/object-lock | `bool` | `false` | no | -| [enable\_versioning](#input\_enable\_versioning) | If true, enables versioning for the bucket. | `bool` | `false` | no | -| [force\_destroy](#input\_force\_destroy) | If true will destroy bucket with all objects stored within. | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to the GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | -| [lifecycle\_rules](#input\_lifecycle\_rules) | List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule | 
list(object({
    # Object with keys:
    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.
    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.
    action = object({
      type          = string
      storage_class = optional(string)
    })

    # Object with keys:
    # - age - (Optional) Minimum age of an object in days to satisfy this condition.
    # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field.
    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.
    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".
    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY.
    # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition.
    # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition.
    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.
    # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition.
    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.
    # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object.
    # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent.
    condition = object({
      age                        = optional(number)
      send_age_if_zero           = optional(bool)
      created_before             = optional(string)
      with_state                 = optional(string)
      matches_storage_class      = optional(string)
      matches_prefix             = optional(string)
      matches_suffix             = optional(string)
      num_newer_versions         = optional(number)
      custom_time_before         = optional(string)
      days_since_custom_time     = optional(number)
      days_since_noncurrent_time = optional(number)
      noncurrent_time_before     = optional(string)
    })
  }))
| `[]` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | -| [mount\_options](#input\_mount\_options) | Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories) | `string` | `"defaults,_netdev,implicit_dirs"` | no | -| [name\_prefix](#input\_name\_prefix) | Name Prefix. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | -| [public\_access\_prevention](#input\_public\_access\_prevention) | Bucket public access can be controlled by setting a value of either `inherited` or `enforced`.
When set to `enforced`, public access to the bucket is blocked.
If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/public-access-prevention | `string` | `null` | no | -| [random\_suffix](#input\_random\_suffix) | If true, a random id will be appended to the suffix of the bucket name. | `bool` | `false` | no | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [retention\_policy\_period](#input\_retention\_policy\_period) | If defined, this will configure retention\_policy with retention\_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/bucket-lock | `number` | `null` | no | -| [soft\_delete\_retention\_duration](#input\_soft\_delete\_retention\_duration) | If defined, this will configure soft\_delete\_policy with retention\_duration\_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days).
Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/soft-delete | `number` | `null` | no | -| [storage\_class](#input\_storage\_class) | The storage class of the GCS bucket. | `string` | `"REGIONAL"` | no | -| [uniform\_bucket\_level\_access](#input\_uniform\_bucket\_level\_access) | Allow uniform control access to the bucket. | `bool` | `true` | no | -| [use\_deployment\_name\_in\_bucket\_name](#input\_use\_deployment\_name\_in\_bucket\_name) | If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments. | `bool` | `true` | no | -| [viewers](#input\_viewers) | A list of additional accounts that can read packages from this bucket | `set(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use gcs fuse. | -| [gcs\_bucket\_name](#output\_gcs\_bucket\_name) | Bucket name. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | The gsutil bucket path with format of `gs://`. | -| [mount\_runner](#output\_mount\_runner) | Runner that mounts the cloud storage bucket with gcs fuse. | -| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | - diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf deleted file mode 100644 index 81ba0ca6a9..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "cloud-storage-bucket", ghpc_role = "file-system" }) -} - -locals { - prefix = var.name_prefix != null ? var.name_prefix : "" - deployment = var.use_deployment_name_in_bucket_name ? var.deployment_name : "" - suffix = var.random_suffix ? random_id.resource_name_suffix.hex : "" - first_dash = (local.prefix != "" && (local.deployment != "" || local.suffix != "")) ? "-" : "" - second_dash = local.deployment != "" && local.suffix != "" ? "-" : "" - composite_name = "${local.prefix}${local.first_dash}${local.deployment}${local.second_dash}${local.suffix}" - name = local.composite_name == "" ? "no-bucket-name-provided" : local.composite_name -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket" "bucket" { - provider = google-beta - project = var.project_id - name = local.name - uniform_bucket_level_access = var.uniform_bucket_level_access - location = var.region - storage_class = var.storage_class - labels = local.labels - force_destroy = var.force_destroy - public_access_prevention = var.public_access_prevention - enable_object_retention = var.enable_object_retention - hierarchical_namespace { - enabled = var.enable_hierarchical_namespace - } - - dynamic "autoclass" { - for_each = var.autoclass.enabled ? [1] : [] - content { - enabled = var.autoclass.enabled - terminal_storage_class = var.autoclass.terminal_storage_class - } - } - - dynamic "soft_delete_policy" { - for_each = var.soft_delete_retention_duration == null ? [] : [1] - content { - retention_duration_seconds = var.soft_delete_retention_duration - } - } - - dynamic "retention_policy" { - for_each = var.retention_policy_period == null ? [] : [1] - content { - retention_period = var.retention_policy_period - } - } - - dynamic "versioning" { - for_each = var.enable_versioning ? [1] : [] - content { - enabled = var.enable_versioning - } - } - - dynamic "lifecycle_rule" { - for_each = var.lifecycle_rules - content { - action { - type = lifecycle_rule.value.action.type - storage_class = lookup(lifecycle_rule.value.action, "storage_class", null) - } - condition { - age = lookup(lifecycle_rule.value.condition, "age", null) - send_age_if_zero = lookup(lifecycle_rule.value.condition, "send_age_if_zero", null) - created_before = lookup(lifecycle_rule.value.condition, "created_before", null) - with_state = lookup(lifecycle_rule.value.condition, "with_state", contains(keys(lifecycle_rule.value.condition), "is_live") ? (lifecycle_rule.value.condition["is_live"] ? "LIVE" : null) : null) - matches_storage_class = lifecycle_rule.value.condition["matches_storage_class"] != null ? split(",", lifecycle_rule.value.condition["matches_storage_class"]) : null - matches_prefix = lifecycle_rule.value.condition["matches_prefix"] != null ? split(",", lifecycle_rule.value.condition["matches_prefix"]) : null - matches_suffix = lifecycle_rule.value.condition["matches_suffix"] != null ? split(",", lifecycle_rule.value.condition["matches_suffix"]) : null - num_newer_versions = lookup(lifecycle_rule.value.condition, "num_newer_versions", null) - custom_time_before = lookup(lifecycle_rule.value.condition, "custom_time_before", null) - days_since_custom_time = lookup(lifecycle_rule.value.condition, "days_since_custom_time", null) - days_since_noncurrent_time = lookup(lifecycle_rule.value.condition, "days_since_noncurrent_time", null) - noncurrent_time_before = lookup(lifecycle_rule.value.condition, "noncurrent_time_before", null) - } - } - } - - lifecycle { - precondition { - condition = !var.autoclass.enabled || !var.enable_hierarchical_namespace - error_message = "Hierarchical namespace is not compatible with Autoclass enabled." - } - - precondition { - condition = !var.enable_hierarchical_namespace || var.uniform_bucket_level_access - error_message = "Hierarchical namespace is not compatible with Uniform bucket level access disabled." - } - - precondition { - condition = !var.enable_versioning || !var.enable_hierarchical_namespace - error_message = "Hierarchical namespace is not compatible with Object versioning enabled." - } - } -} - -resource "google_storage_bucket_iam_binding" "viewers" { - bucket = google_storage_bucket.bucket.name - role = "roles/storage.objectViewer" - members = var.viewers -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf deleted file mode 100644 index 29ddfef2d2..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a remote network storage to be mounted by fs-tab." - value = { - remote_mount = local.name - local_mount = var.local_mount - fs_type = "gcsfuse" - mount_options = var.mount_options - server_ip = "" - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } -} - -locals { - client_install_runner = { - "type" = "shell" - "content" = file("${path.module}/scripts/install-gcs-fuse.sh") - "destination" = "install-gcsfuse${replace(var.local_mount, "/", "_")}.sh" - } - - mount_runner = { - "type" = "shell" - "destination" = "mount_gcs${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"not-used\" \"${local.name}\" \"${var.local_mount}\" \"gcsfuse\" \"${var.mount_options}\"" - "content" = file("${path.module}/scripts/mount.sh") - } -} - -output "client_install_runner" { - description = "Runner that performs client installation needed to use gcs fuse." - value = local.client_install_runner -} - -output "mount_runner" { - description = "Runner that mounts the cloud storage bucket with gcs fuse." - value = local.mount_runner -} - -output "gcs_bucket_path" { - description = "The gsutil bucket path with format of `gs://`." - # cannot use resource attribute, will cause lookup failure in startup-script - value = "gs://${local.name}" - - # needed to make sure bucket contents are deleted before bucket - depends_on = [ - google_storage_bucket.bucket - ] -} - -output "gcs_bucket_name" { - description = "Bucket name." - value = google_storage_bucket.bucket.name -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh deleted file mode 100644 index f8a990260b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -if [ ! "$(which gcsfuse)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then - tee /etc/yum.repos.d/gcsfuse.repo >/dev/null </dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf deleted file mode 100644 index 9804e4b268..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf +++ /dev/null @@ -1,254 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which GCS bucket will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment; used as part of name of the GCS bucket." - type = string -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "labels" { - description = "Labels to add to the GCS bucket. Key-value pairs." - type = map(string) -} - -variable "local_mount" { - description = "The mount point where the contents of the device may be accessed after mounting." - type = string - default = "/mnt" -} - -variable "mount_options" { - description = "Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories)" - type = string - default = "defaults,_netdev,implicit_dirs" -} - -variable "name_prefix" { - description = "Name Prefix." - type = string - default = null -} - -variable "use_deployment_name_in_bucket_name" { - description = "If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments." - type = bool - default = true -} - -variable "random_suffix" { - description = "If true, a random id will be appended to the suffix of the bucket name." - type = bool - default = false -} - -variable "force_destroy" { - description = "If true will destroy bucket with all objects stored within." - type = bool - default = false -} - -variable "viewers" { - description = "A list of additional accounts that can read packages from this bucket" - type = set(string) - default = [] - - validation { - error_message = "All bucket viewers must be in IAM style: user:user@example.com, serviceAccount:sa@example.com, or group:group@example.com." - condition = alltrue([ - for viewer in var.viewers : length(regexall("^(user|serviceAccount|group):", viewer)) > 0 - ]) - } -} - -variable "enable_hierarchical_namespace" { - description = "If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket." - type = bool - default = false -} - -variable "uniform_bucket_level_access" { - description = "Allow uniform control access to the bucket." - type = bool - default = true -} - -variable "storage_class" { - description = "The storage class of the GCS bucket." - type = string - default = "REGIONAL" - validation { - condition = contains([ - "STANDARD", - "MULTI_REGIONAL", - "REGIONAL", - "NEARLINE", - "COLDLINE", - "ARCHIVE" - ], var.storage_class) - error_message = "Allowed values for GCS storage_class are 'STANDARD', 'MULTI_REGIONAL', 'REGIONAL', 'NEARLINE', 'COLDLINE', 'ARCHIVE'.\nhttps://cloud.google.com/storage/docs/storage-classes" - } -} - -variable "autoclass" { - description = <<-EOT - Configure bucket autoclass setup - - The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern. - - The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time. - Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE') - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/autoclass - EOT - type = object({ - enabled = optional(bool, false) - terminal_storage_class = optional(string, null) - }) - default = { - enabled = false - } - nullable = false - validation { - condition = !can(coalesce(var.autoclass.terminal_storage_class)) || var.autoclass.enabled - error_message = "Cannot set bucket var.autoclass.terminal_storage_class unless var.autoclass.enabled is true" - } -} - -variable "public_access_prevention" { - description = <<-EOT - Bucket public access can be controlled by setting a value of either `inherited` or `enforced`. - When set to `enforced`, public access to the bucket is blocked. - If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/public-access-prevention - EOT - type = string - default = null - validation { - condition = var.public_access_prevention == null ? true : contains([ - "inherited", - "enforced" - ], var.public_access_prevention) - error_message = "Allowed values for public_access_prevention are 'inherited', 'enforced'.\n" - } -} - -variable "soft_delete_retention_duration" { - description = <<-EOT - If defined, this will configure soft_delete_policy with retention_duration_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days). - Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/soft-delete - EOT - type = number - default = null - validation { - condition = var.soft_delete_retention_duration == null ? true : var.soft_delete_retention_duration == 0 || var.soft_delete_retention_duration >= 604800 && var.soft_delete_retention_duration <= 7776000 - error_message = "var.soft_delete_retention_duration value can be 0 or in between 604800(7 days) and 7776000(90 days)." - } -} - -variable "enable_versioning" { - description = "If true, enables versioning for the bucket." - type = bool - default = false -} - -variable "lifecycle_rules" { - description = "List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule" - type = list(object({ - # Object with keys: - # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass. - # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule. - action = object({ - type = string - storage_class = optional(string) - }) - - # Object with keys: - # - age - (Optional) Minimum age of an object in days to satisfy this condition. - # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field. - # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition. - # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY". - # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY. - # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition. - # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition. - # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition. - # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition. - # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true. - # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object. - # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent. - condition = object({ - age = optional(number) - send_age_if_zero = optional(bool) - created_before = optional(string) - with_state = optional(string) - matches_storage_class = optional(string) - matches_prefix = optional(string) - matches_suffix = optional(string) - num_newer_versions = optional(number) - custom_time_before = optional(string) - days_since_custom_time = optional(number) - days_since_noncurrent_time = optional(number) - noncurrent_time_before = optional(string) - }) - })) - default = [] -} - -variable "retention_policy_period" { - description = <<-EOT - If defined, this will configure retention_policy with retention_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/bucket-lock - EOT - type = number - default = null - validation { - condition = var.retention_policy_period == null ? true : var.retention_policy_period > 0 && var.retention_policy_period <= 3155760000 - error_message = "var.soft_delete_policy_retention_duration value must be in between 1 and 3155760000(100 years) seconds." - } -} - -variable "enable_object_retention" { - description = <<-EOT - If true, enables retention policy at per object level for the bucket. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/object-lock - EOT - type = bool - default = false -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf deleted file mode 100644 index 217ee2f3a2..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.9.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" - } - required_version = ">= 0.14.0" -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/README.md deleted file mode 100644 index 3bf251828e..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/README.md +++ /dev/null @@ -1,248 +0,0 @@ -## Description - -This module creates a [filestore](https://cloud.google.com/filestore) -instance. Filestore is a high performance network file system that can be -mounted to one or more compute VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -### Deletion protection - -We recommend considering enabling [Filestore deletion protection][fdp]. Deletion -protection will prevent unintentional deletion of an entire Filestore instance. -It does not prevent deletion of files within the Filestore instance when mounted -by a VM. It is not available on some [tiers](#filestore-tiers), including the -default BASIC\_HDD tier or BASIC\_SSD tier. Follow the documentation link for -up to date details. - -Usage can be enabled in a blueprint with, for example: - -```yaml - - id: homefs - source: modules/file-system/filestore - use: [network] - settings: - deletion_protection: - enabled: true - reason: Avoid data loss - filestore_tier: ZONAL - local_mount: /home - size_gb: 1024 -``` - -[fdp]: https://cloud.google.com/filestore/docs/deletion-protection - -### Filestore tiers - -At the time of writing, Filestore supports 5 [tiers of service][tiers] that are -specified in the Toolkit using the following names: - -- Basic HDD: "BASIC\_HDD" ([preferred][tierapi]) or "STANDARD" (deprecated) -- Basic SSD: "BASIC\_SSD" ([preferred][tierapi]) or "PREMIUM" (deprecated) -- Zonal: "ZONAL" -- Enterprise: "ENTERPRISE" -- Regional: "REGIONAL" - -[tierapi]: https://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier - -**Please review the minimum storage requirements for each tier**. The Terraform -module can only enforce the minimum value of the `size_gb` parameter for the -lowest tier of service. If you supply a value that is too low, Filestore -creation will fail when you run `terraform apply`. - -[tiers]: https://cloud.google.com/filestore/docs/service-tiers - -### Filestore protocols and mount options -After Filestore instance is created, you can mount this to the compute node -using different mount options. Toolkit uses [default mount options](https://linux.die.net/man/8/mount) -for all tier services. Filestore has recommended mount options for different -service tiers which may overall improve performance. These can be found here: -[recommended mount options.](https://cloud.google.com/filestore/docs/mounting-fileshares) -While creating filestore module, you can overwrite these mount options as -mentioned below. - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /homefs - mount_options: defaults,hard,timeo=600,retrans=3,_netdev -``` - -Filestore supports NFS protocols `NFS_V3` (default) and `NFS_V4_1`. Protocol support depends on the selected tier: -- `NFS_V3`: Supported on all tiers (`BASIC_HDD`, `BASIC_SSD`, `HIGH_SCALE_SSD`, `ZONAL`, `ENTERPRISE`). -- `NFS_V4_1`: Supported only on `HIGH_SCALE_SSD`, `ZONAL`, `REGIONAL`, and `ENTERPRISE`. -This can be specified at creation time via the `protocol` variable. By default, `NFS_V3` is used for compatibility. -See the example below and [this page](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#protocol-1) for more information. - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /homefs - protocol: NFS_V4_1 - filestore_tier: ZONAL -``` - -### Filestore quota - -Your project must have unused quota for Cloud Filestore in the region you will -provision the storage. This can be found by browsing to the [Quota tab within IAM -& Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. -Please note that there are separate quota limits for HDD and SSD storage. - -All projects begin with 0 available quota for High Scale SSD tier. To use this -tier, [make a request and wait for it to be approved][hs-ssd-quota]. - -[hs-ssd-quota]: https://cloud.google.com/filestore/docs/high-scale - -### Example - Basic HDD - -The Filestore instance defined below will have the following attributes: - -- (default) `BASIC_HDD` tier -- (default) 1TiB capacity -- `homefs` module ID -- mount point at `/home` -- connected to the network defined in the `network1` module - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /home -``` - -### Example - High Scale SSD - -The Filestore instance defined below will have the following attributes: - -- `HIGH_SCALE_SSD` tier -- 10TiB capacity -- `highscale` module ID -- mount point at `/projects` -- connected to the VPC network defined in the `network1` module - -```yaml -- id: highscale - source: modules/file-system/filestore - use: [network1] - settings: - filestore_tier: HIGH_SCALE_SSD - size_gb: 10240 - local_mount: /projects -``` - -## Mounting - -To mount the Filestore instance you must first ensure that the NFS client has -been installed and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -See the [hpc-slurm](../../../examples/hpc-slurm.yaml) for -an example of using this module with Slurm. - -If mounting is not automatically handled as described above, the `filestore` -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - - id: filestore - source: modules/file-system/filestore - use: [network1] - settings: {local_mount: /scratch} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(filestore.install_nfs_client_runner) - - $(filestore.mount_runner) - -``` - -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 6.4 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.4 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_filestore_instance.filestore_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [connect\_mode](#input\_connect\_mode) | Used to select mode - supported values DIRECT\_PEERING and PRIVATE\_SERVICE\_ACCESS. | `string` | `"DIRECT_PEERING"` | no | -| [deletion\_protection](#input\_deletion\_protection) | Configure Filestore instance deletion protection | 
object({
    enabled = optional(bool, false)
    reason  = optional(string)
  })
| 
{
  "enabled": false
}
| no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the filestore instance if no name is specified. | `string` | n/a | yes | -| [description](#input\_description) | A description of the filestore instance. | `string` | `""` | no | -| [filestore\_share\_name](#input\_filestore\_share\_name) | Name of the file system share on the instance. | `string` | `"nfsshare"` | no | -| [filestore\_tier](#input\_filestore\_tier) | The service tier of the instance. | `string` | `"BASIC_HDD"` | no | -| [labels](#input\_labels) | Labels to add to the filestore instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mount](#input\_local\_mount) | Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md)). | `string` | `"/shared"` | no | -| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"defaults,_netdev"` | no | -| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [nfs\_export\_options](#input\_nfs\_export\_options) | Define NFS export options. | 
list(object({
    access_mode = optional(string)
    ip_ranges   = optional(list(string))
    squash_mode = optional(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | ID of project in which Filestore instance will be created. | `string` | n/a | yes | -| [protocol](#input\_protocol) | NFS protocol version. Default is NFS\_V3. NFS\_V4\_1 is only supported with HIGH\_SCALE\_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers. | `string` | `"NFS_V3"` | no | -| [region](#input\_region) | Location for Filestore instances at Enterprise tier. | `string` | n/a | yes | -| [reserved\_ip\_range](#input\_reserved\_ip\_range) | Reserved IP range for Filestore instance. Users are encouraged to set to null
for automatic selection. If supplied, it must be:

CIDR format when var.connect\_mode == "DIRECT\_PEERING"
Named IP Range when var.connect\_mode == "PRIVATE\_SERVICE\_ACCESS"

See Cloud documentation for more details:

https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range | `string` | `null` | no | -| [size\_gb](#input\_size\_gb) | Storage size of the filestore instance in GB. | `number` | `1024` | no | -| [zone](#input\_zone) | Location for Filestore instances below Enterprise tier. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | -| [filestore\_id](#output\_filestore\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | -| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | -| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | -| [network\_storage](#output\_network\_storage) | Describes a filestore instance. | - diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/main.tf deleted file mode 100644 index ce035dbb2b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/main.tf +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "filestore", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - is_high_capacity_tier = contains(["HIGH_SCALE_SSD", "ZONAL", "REGIONAL"], var.filestore_tier) && var.size_gb >= 10240 && var.size_gb <= 102400 - - timeouts = local.is_high_capacity_tier ? [1] : [] - server_ip = google_filestore_instance.filestore_instance.networks[0].ip_addresses[0] - remote_mount = format("/%s", google_filestore_instance.filestore_instance.file_shares[0].name) - fs_type = "nfs" - mount_options = var.mount_options - - install_nfs_client_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-nfs-client.sh" - "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" - } - mount_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" - } - - # id format: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network#id - split_network_id = split("/", var.network_id) - network_name = local.split_network_id[4] - network_project = local.split_network_id[1] - shared_vpc = local.network_project != var.project_id -} - -resource "google_filestore_instance" "filestore_instance" { - project = var.project_id - - name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - description = var.description - location = contains(["ENTERPRISE", "REGIONAL"], var.filestore_tier) ? var.region : var.zone - tier = var.filestore_tier - protocol = var.protocol - - deletion_protection_enabled = var.deletion_protection.enabled - deletion_protection_reason = var.deletion_protection.reason - - file_shares { - capacity_gb = var.size_gb - name = var.filestore_share_name - dynamic "nfs_export_options" { - for_each = var.nfs_export_options - content { - access_mode = nfs_export_options.value.access_mode - ip_ranges = nfs_export_options.value.ip_ranges - squash_mode = nfs_export_options.value.squash_mode - } - } - } - - labels = local.labels - - networks { - network = local.shared_vpc ? var.network_id : local.network_name - connect_mode = var.connect_mode - modes = ["MODE_IPV4"] - reserved_ip_range = var.reserved_ip_range - } - - dynamic "timeouts" { - for_each = local.timeouts - content { - create = "1h" - update = "1h" - delete = "1h" - } - } - - lifecycle { - precondition { - condition = ( - var.reserved_ip_range == null || - var.connect_mode == "PRIVATE_SERVICE_ACCESS" || - var.connect_mode == "DIRECT_PEERING" && can(cidrhost(var.reserved_ip_range, 0)) && contains(["24", "29"], try(split("/", var.reserved_ip_range)[1], "")) - ) - error_message = <<-EOT - If connect_mode is set to DIRECT_PEERING and reserved_ip_range is - specified then it must be a CIDR IP range with suffix range size 29 for - BASIC_HDD or BASIC_SSD tiers. Otherwise the range size must be 24. - EOT - } - - precondition { - condition = !startswith(var.filestore_tier, "BASIC") || var.protocol != "NFS_V4_1" - error_message = "NFS_V4_1 is not supported on BASIC Filestore tiers." - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/metadata.yaml deleted file mode 100644 index 5298336f09..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - file.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/outputs.tf deleted file mode 100644 index 9bdb3bdc7b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/outputs.tf +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a filestore instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = local.mount_options - client_install_runner = local.install_nfs_client_runner - mount_runner = local.mount_runner - } -} - -output "install_nfs_client" { - description = "Script for installing NFS client" - value = file("${path.module}/scripts/install-nfs-client.sh") -} - -output "install_nfs_client_runner" { - description = "Runner to install NFS client using the startup-script module" - value = local.install_nfs_client_runner -} - -output "mount_runner" { - description = <<-EOT - Runner to mount the file-system using an ansible playbook. The startup-script - module will automatically handle installation of ansible. - - id: example-startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(your-fs-id.mount_runner) - ... - EOT - value = local.mount_runner -} - -output "filestore_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" - value = google_filestore_instance.filestore_instance.id -} - -output "capacity_gib" { - description = "File share capacity in GiB." - value = google_filestore_instance.filestore_instance.file_shares[0].capacity_gb -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/mount.sh b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/variables.tf deleted file mode 100644 index 2d7e9258c0..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/variables.tf +++ /dev/null @@ -1,189 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which Filestore instance will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the filestore instance if no name is specified." - type = string -} - -variable "zone" { - description = "Location for Filestore instances below Enterprise tier." - type = string -} - -variable "region" { - description = "Location for Filestore instances at Enterprise tier." - type = string -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is connected given in the format: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "name" { - description = "The resource name of the instance." - type = string - default = null -} - -variable "filestore_share_name" { - description = "Name of the file system share on the instance." - type = string - default = "nfsshare" -} - -variable "local_mount" { - description = "Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md))." - type = string - default = "/shared" -} - -variable "size_gb" { - description = "Storage size of the filestore instance in GB." - type = number - default = 1024 - validation { - condition = var.size_gb >= 1024 - error_message = "No Filestore tier supports less than 1024GiB.\nSee https://cloud.google.com/filestore/docs/service-tiers." - } -} - -variable "filestore_tier" { - description = "The service tier of the instance." - type = string - default = "BASIC_HDD" - validation { - condition = var.filestore_tier != "STANDARD" - error_message = "The preferred name for STANDARD tier is now BASIC_HDD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." - } - validation { - condition = var.filestore_tier != "PREMIUM" - error_message = "The preferred name for PREMIUM tier is now BASIC_SSD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." - } - validation { - condition = contains([ - "BASIC_HDD", - "BASIC_SSD", - "HIGH_SCALE_SSD", - "ZONAL", - "REGIONAL", - "ENTERPRISE" - ], var.filestore_tier) - # Avoid adding the legacy tier name in error_message, for e.g. 'HIGH_SCALE_SSD', 'ENTERPRISE'. - # As we want to steer the customer to new one's, but also support the legacy ones for older customers. - error_message = "Allowed values for filestore_tier are 'BASIC_HDD','BASIC_SSD','ZONAL','REGIONAL'.\nhttps://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#tier\nhttps://cloud.google.com/filestore/docs/reference/rest/v1/Tier." - } -} - -variable "labels" { - description = "Labels to add to the filestore instance. Key-value pairs." - type = map(string) -} - -variable "connect_mode" { - description = "Used to select mode - supported values DIRECT_PEERING and PRIVATE_SERVICE_ACCESS." - type = string - default = "DIRECT_PEERING" - nullable = false - validation { - condition = contains(["DIRECT_PEERING", "PRIVATE_SERVICE_ACCESS"], var.connect_mode) - error_message = "Allowed values for connect_mode are \"DIRECT_PEERING\" or \"PRIVATE_SERVICE_ACCESS\"." - } -} - -variable "nfs_export_options" { - description = "Define NFS export options." - type = list(object({ - access_mode = optional(string) - ip_ranges = optional(list(string)) - squash_mode = optional(string) - })) - default = [] - nullable = false -} - -variable "reserved_ip_range" { - description = <<-EOT - Reserved IP range for Filestore instance. Users are encouraged to set to null - for automatic selection. If supplied, it must be: - - CIDR format when var.connect_mode == "DIRECT_PEERING" - Named IP Range when var.connect_mode == "PRIVATE_SERVICE_ACCESS" - - See Cloud documentation for more details: - - https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range - EOT - type = string - default = null - nullable = true -} - -variable "mount_options" { - description = "NFS mount options to mount file system." - type = string - default = "defaults,_netdev" -} - -variable "deletion_protection" { - description = "Configure Filestore instance deletion protection" - type = object({ - enabled = optional(bool, false) - reason = optional(string) - }) - default = { - enabled = false - } - nullable = false - - validation { - condition = !can(coalesce(var.deletion_protection.reason)) || var.deletion_protection.enabled - error_message = "Cannot set Filestore var.deletion_protection.reason unless var.deletion_protection.enabled is true" - } -} - -variable "protocol" { - description = "NFS protocol version. Default is NFS_V3. NFS_V4_1 is only supported with HIGH_SCALE_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers." - type = string - default = "NFS_V3" - validation { - condition = contains(["NFS_V3", "NFS_V4_1"], var.protocol) - error_message = "Allowed values for protocol are 'NFS_V3' or 'NFS_V4_1'." - } -} - -variable "description" { - description = "A description of the filestore instance." - type = string - default = "" - validation { - condition = length(var.description) <= 2048 - error_message = "Filestore description must be 2048 characters or fewer" - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/filestore/versions.tf deleted file mode 100644 index 1ba0e7967e..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/filestore/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.4" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/README.md deleted file mode 100644 index 88ae4511e3..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/README.md +++ /dev/null @@ -1,200 +0,0 @@ -## Description - -This module creates Kubernetes Persistent Volumes (PV) and Persistent Volume -Claims (PVC) that can be used by a [gke-job-template]. - -`gke-persistent-volume` works with Filestore, Google Cloud Storage and Managed Lustre. Each -`gke-persistent-volume` can only be used with a single file system so if multiple -shared file systems are used then multiple `gke-persistent-volume` modules are -needed in the blueprint. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a Filestore and then uses the -`gke-persistent-volume` module to use the Filestore as shared storage in a -`gke-job-template`. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] - settings: - master_authorized_networks: - - display_name: deployment-machine - cidr_block: /32 - - - id: datafs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /data - - - id: datafs-pv - source: modules/file-system/gke-persistent-volume - use: [datafs, gke_cluster] - - - id: job-template - source: modules/compute/gke-job-template - use: [datafs-pv, compute_pool, gke_cluster] -``` - -The following example creates a GCS bucket and then uses the -`gke-persistent-volume` module to use the bucket as shared storage in a -`gke-job-template`. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] - settings: - master_authorized_networks: - - display_name: deployment-machine - cidr_block: /32 - - - id: data-bucket - source: modules/file-system/cloud-storage-bucket - settings: - local_mount: /data - - - id: datagcs-pv - source: modules/file-system/gke-persistent-volume - use: [data-bucket, gke_cluster] - - - id: job-template - source: modules/compute/gke-job-template - use: [datagcs-pv, compute_pool, gke_cluster] -``` - -The following example creates a Managed Lustre and then uses the -`gke-persistent-volume` module to use the Lustre as shared storage in a -`gke-job-template`. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] - settings: - master_authorized_networks: - - display_name: deployment-machine - cidr_block: /32 - - - id: data-managedlustre - source: modules/file-system/managed-lustre - settings: - local_mount: /data - - - id: datalustre-pv - source: modules/file-system/gke-persistent-volume - use: [data-managedlustre, gke_cluster] - - - id: job-template - source: modules/compute/gke-job-template - use: [datalustre-pv, compute_pool, gke_cluster] -``` - -See example -[storage-gke.yaml](../../../../examples/README.md#storage-gkeyaml--) blueprint -for a complete example. - -### Authorized Network - -Since the `gke-persistent-volume` module is making calls to the Kubernetes API -to create Kubernetes entities, the machine performing the deployment must be -authorized to connect to the Kubernetes API. You can add the -`master_authorized_networks` settings block, as shown in the example above, with -the IP address of the machine performing the deployment. This will ensure that -the deploying machine can connect to the cluster. - -### Connecting Via Use - -The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit -modules. For example the `gke-persistent-volume` module can `use` a -`gke-cluster` module and a `filestore` module, as shown in the example above. - -```mermaid - graph TD; - vpc--> |OneToMany| gke-cluster; - gke-cluster--> |OneToMany| gke-node-pool; - gke-node-pool--> |ManyToMany| gke-job-template; - gke-cluster--> |OneToMany| gke-persistent-volume; - gke-persistent-volume--> |ManyToMany| gke-job-template; - vpc--> |OneToMany| filestore; - vpc--> |OneToMany| gcs; - vpc--> |OneToMany| managed-lustre; - filestore--> |OneToOne| gke-persistent-volume; - gcs--> |OneToOne| gke-persistent-volume; - managed-lustre--> |OneToOne| gke-persistent-volume; - ``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [kubectl](#provider\_kubectl) | >= 1.7.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubectl_manifest.pv](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.pvc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.pvc_namespace](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [capacity\_gib](#input\_capacity\_gib) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | -| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | -| [filestore\_id](#input\_filestore\_id) | An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | -| [gcs\_bucket\_name](#input\_gcs\_bucket\_name) | The gcs bucket to be used with the persistent volume. | `string` | `null` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [lustre\_id](#input\_lustre\_id) | An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | Network attached storage mount to be configured. | 
object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  })
| n/a | yes | -| [pv\_name](#input\_pv\_name) | The name for PV. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | -| [pvc\_name](#input\_pvc\_name) | The name for PVC. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object describing the Kubernetes PersistentVolumeClaim created by this module. | -| [pvc\_name](#output\_pvc\_name) | The name of the Kubernetes PVC created by this module. | - diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/main.tf deleted file mode 100644 index 818ebaf595..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/main.tf +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-persistent-volume", ghpc_role = "file-system" }) -} - -locals { - # Flags indicating which storage type is active based on input variables. - storage_type_active = { - gcs = var.gcs_bucket_name != null - lustre = var.lustre_id != null - filestore = var.filestore_id != null - } - - # Determine the active storage type name. - active_types = [for type, is_active in local.storage_type_active : type if is_active] - - # The precondition in kubectl_manifest.pv ensures exactly one type is active. - storage_type = length(local.active_types) > 0 ? local.active_types[0] : "unknown" - - # Map containing the base name derivation logic for each storage type. - base_name_map = { - gcs = var.gcs_bucket_name - lustre = var.lustre_id != null ? split("/", var.lustre_id)[5] : null - filestore = var.filestore_id != null ? split("/", var.filestore_id)[5] : null - } - # Retrieve the base name for the active storage type. - base_name = local.base_name_map[local.storage_type] - - # PV and PVC names - pv_name = var.pv_name != null ? var.pv_name : "${local.base_name}-pv" - pvc_name = var.pvc_name != null ? var.pvc_name : "${local.base_name}-pvc" - - # Template file paths - pv_templates = { - gcs = "${path.module}/templates/gcs-pv.yaml.tftpl" - lustre = "${path.module}/templates/managed-lustre-pv.yaml.tftpl" - filestore = "${path.module}/templates/filestore-pv.yaml.tftpl" - } - pvc_templates = { - gcs = "${path.module}/templates/gcs-pvc.yaml.tftpl" - lustre = "${path.module}/templates/managed-lustre-pvc.yaml.tftpl" - filestore = "${path.module}/templates/filestore-pvc.yaml.tftpl" - } - - # Common variables for all PVC templates - common_pvc_vars = { - pv_name = local.pv_name - pvc_name = local.pvc_name - labels = local.labels - capacity = "${var.capacity_gib}Gi" - namespace = var.namespace - } - - # Common variables for all PV templates - common_pv_vars = { - pv_name = local.pv_name - capacity = "${var.capacity_gib}Gi" - labels = local.labels - } - - # Variables for PV templates, merging common vars with type-specific ones. - pv_template_vars = { - gcs = merge(local.common_pv_vars, { - mount_options = var.gcs_bucket_name != null ? split(",", var.network_storage.mount_options) : [] - bucket_name = var.gcs_bucket_name - namespace = var.namespace - pvc_name = local.pvc_name - }) - lustre = merge(local.common_pv_vars, { - location = var.lustre_id != null ? split("/", var.lustre_id)[3] : null - project = split("/", var.cluster_id)[1] - instance_name = local.base_name - server_ip = var.lustre_id != null ? split("@", var.network_storage.server_ip)[0] : null - filesystem_name = var.network_storage.remote_mount - pvc_name = local.pvc_name - namespace = var.namespace - }) - filestore = merge(local.common_pv_vars, { - location = var.filestore_id != null ? split("/", var.filestore_id)[3] : null - filestore_name = local.base_name - share_name = trimprefix(var.network_storage.remote_mount, "/") - ip_address = var.network_storage.server_ip - pvc_name = local.pvc_name - namespace = var.namespace - }) - } - - # Rendered YAML contents - pv_content = templatefile( - local.pv_templates[local.storage_type], - local.pv_template_vars[local.storage_type] - ) - pvc_content = templatefile( - local.pvc_templates[local.storage_type], - local.common_pvc_vars - ) - - # GKE Cluster details - cluster_name = split("/", var.cluster_id)[5] - cluster_location = split("/", var.cluster_id)[3] -} - -data "google_container_cluster" "gke_cluster" { - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -provider "kubectl" { - host = "https://${data.google_container_cluster.gke_cluster.endpoint}" - cluster_ca_certificate = base64decode(data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) - token = data.google_client_config.default.access_token - load_config_file = false -} - -resource "kubectl_manifest" "pvc_namespace" { - count = var.namespace != "default" ? 1 : 0 - - yaml_body = templatefile("${path.module}/templates/namespace.yaml.tftpl", { - namespace = var.namespace - }) -} - -resource "kubectl_manifest" "pv" { - yaml_body = local.pv_content - - lifecycle { - precondition { - condition = length(local.active_types) == 1 - error_message = "Exactly one of gcs_bucket_name, filestore_id, or lustre_id must be set." - } - } -} - -resource "kubectl_manifest" "pvc" { - yaml_body = local.pvc_content - depends_on = [kubectl_manifest.pv, kubectl_manifest.pvc_namespace] -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf deleted file mode 100644 index 60cf2dbe0f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "persistent_volume_claims" { - description = "An object describing the Kubernetes PersistentVolumeClaim created by this module." - value = { - name = local.pvc_name - namespace = var.namespace - mount_path = var.network_storage.local_mount - mount_options = var.network_storage.mount_options - storage_type = local.storage_type - } -} - -output "pvc_name" { - description = "The name of the Kubernetes PVC created by this module." - value = local.pvc_name -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl deleted file mode 100644 index 06a1276c1e..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ${pv_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - storageClassName: "" - capacity: - storage: ${capacity} - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - volumeMode: Filesystem - csi: - driver: filestore.csi.storage.gke.io - volumeHandle: "modeInstance/${location}/${filestore_name}/${share_name}" - volumeAttributes: - ip: ${ip_address} - volume: ${share_name} - claimRef: - name: ${pvc_name} - namespace: ${namespace} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl deleted file mode 100644 index 83cfb3bc8c..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl +++ /dev/null @@ -1,18 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - volumeName: ${pv_name} - resources: - requests: - storage: ${capacity} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl deleted file mode 100644 index aa0e570a8b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ${pv_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - storageClassName: "" - capacity: - storage: ${capacity} - accessModes: - - ReadWriteMany - %{~ if mount_options != null ~} - mountOptions: - %{~ for key in mount_options ~} - - ${key} - %{~ endfor ~} - %{~ endif ~} - csi: - driver: gcsfuse.csi.storage.gke.io - volumeHandle: ${bucket_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl deleted file mode 100644 index 4d02c85629..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl +++ /dev/null @@ -1,21 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - volumeName: ${pv_name} - resources: - requests: - storage: ${capacity} - claimRef: - name: ${pvc_name} - namespace: ${namespace} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl deleted file mode 100644 index 2b3b5e7738..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ${pv_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - storageClassName: "" - capacity: - storage: ${capacity} - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - volumeMode: Filesystem - claimRef: - namespace: ${namespace} - name: ${pvc_name} - csi: - driver: lustre.csi.storage.gke.io - volumeHandle: "${project}/${location}/${instance_name}/default-pool/default-container" - volumeAttributes: - ip: ${server_ip} - filesystem: ${filesystem_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl deleted file mode 100644 index 83cfb3bc8c..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl +++ /dev/null @@ -1,18 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - volumeName: ${pv_name} - resources: - requests: - storage: ${capacity} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl deleted file mode 100644 index fa7647e33f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf deleted file mode 100644 index fd281756e7..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "cluster_id" { - description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" - type = string -} - -variable "network_storage" { - description = "Network attached storage mount to be configured." - type = object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - }) -} - -variable "filestore_id" { - description = "An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." - type = string - default = null - validation { - condition = ( - var.filestore_id == null || - try(length(split("/", var.filestore_id)), 0) == 6 - ) - error_message = "filestore_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." - } -} - -variable "lustre_id" { - description = "An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." - type = string - default = null - validation { - condition = ( - var.lustre_id == null || - try(length(split("/", var.lustre_id)), 0) == 6 - ) - error_message = "lustre_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." - } -} - -variable "gcs_bucket_name" { - description = "The gcs bucket to be used with the persistent volume." - type = string - default = null -} - -variable "capacity_gib" { - description = "The storage capacity with which to create the persistent volume." - type = number -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} - -variable "namespace" { - description = "Kubernetes namespace to deploy the storage PVC/PV" - type = string - default = "default" -} - -variable "pv_name" { - description = "The name for PV. IF not set, a name will be generated based on the storage name." - type = string - default = null -} - -variable "pvc_name" { - description = "The name for PVC. IF not set, a name will be generated based on the storage name." - type = string - default = null -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf deleted file mode 100644 index fa1c3e2b3f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.7.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.74.0" - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/README.md deleted file mode 100644 index 78ef5402aa..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/README.md +++ /dev/null @@ -1,134 +0,0 @@ -## Description - -This module creates Kubernetes Storage Class (SC) that can be used by a Persistent Volume Claim (PVC) -to dynamically provision GCP storage resources like Parallelstore. - -### Example - -The following example uses the `gke-storage` module to creates a Parallelstore Storage Class and Persistent Volume Claim, -then use them in a `gke-job-template` to dynamically provision the resource. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network] - settings: - enable_parallelstore_csi: true - - # Private Service Access (PSA) requires the compute.networkAdmin role which is - # included in the Owner role, but not Editor. - # PSA is required for all Parallelstore functionality. - # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: gke_storage - source: modules/file-system/gke-storage - use: [ gke_cluster, private_service_access ] - settings: - storage_type: Parallelstore - access_mode: ReadWriteMany - sc_volume_binding_mode: Immediate - sc_reclaim_policy: Delete - sc_topology_zones: [$(vars.zone)] - pvc_count: 2 - capacity_gb: 12000 - - - id: job_template - source: modules/compute/gke-job-template - use: [gke_storage, compute_pool] -``` - -See example -[gke-managed-parallelstore.yaml](../../../examples/README.md#gke-managed-parallelstoreyaml--) blueprint -for a complete example. - -### Authorized Network - -Since the `gke-storage` module is making calls to the Kubernetes API -to create Kubernetes entities, the machine performing the deployment must be -authorized to connect to the Kubernetes API. You can add the -`master_authorized_networks` settings block, as shown in the example above, with -the IP address of the machine performing the deployment. This will ensure that -the deploying machine can connect to the cluster. - -### Connecting Via Use - -The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit -modules. For example the `gke-storage` module can `use` a -`gke-cluster` module and a `private_service_access` module, as shown in the example above. - -```mermaid -graph TD; - vpc-->|OneToMany|gke-cluster; - gke-cluster-->|OneToMany|gke-node-pool; - gke-node-pool-->|ManyToMany|gke-job-template; - gke-cluster-->|OneToMany|gke-storage; - gke-storage-->|ManyToMany|gke-job-template; -``` - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_mode](#input\_access\_mode) | The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)
Valid access modes:
- ReadWriteOnce
- ReadOnlyMany
- ReadWriteMany
- ReadWriteOncePod | `string` | n/a | yes | -| [capacity\_gb](#input\_capacity\_gb) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | -| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [mount\_options](#input\_mount\_options) | Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class. | `string` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [pv\_mount\_path](#input\_pv\_mount\_path) | Path within the container at which the volume should be mounted. Must not contain ':'. | `string` | `"/data"` | no | -| [pvc\_count](#input\_pvc\_count) | How many PersistentVolumeClaims that will be created | `number` | `1` | no | -| [sc\_reclaim\_policy](#input\_sc\_reclaim\_policy) | Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted.
[More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming)
Supported value:
- Retain
- Delete | `string` | n/a | yes | -| [sc\_topology\_zones](#input\_sc\_topology\_zones) | Zone location that allow the volumes to be dynamically provisioned. | `list(string)` | `null` | no | -| [sc\_volume\_binding\_mode](#input\_sc\_volume\_binding\_mode) | Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound.
Supported value:
- Immediate
- WaitForFirstConsumer | `string` | `"WaitForFirstConsumer"` | no | -| [storage\_type](#input\_storage\_type) | The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview)
to used. This module currently support dynamic provisioning for the below storage options
- Parallelstore
- Hyperdisk-balanced
- Hyperdisk-throughput
- Hyperdisk-extreme | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object that describes a k8s PVC created by this module. | - diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/main.tf deleted file mode 100644 index 9c9a641f79..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/main.tf +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-storage", ghpc_role = "file-system" }) -} - -locals { - storage_type = lower(var.storage_type) - storage_class_name = "${local.storage_type}-sc" - pvc_name_prefix = "${local.storage_type}-pvc" -} - -check "private_vpc_connection_peering" { - assert { - condition = lower(var.storage_type) != "parallelstore" ? true : var.private_vpc_connection_peering != null - error_message = <<-EOT - Parallelstore must be run within the same VPC as the GKE cluster and have private services access enabled. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access. - If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). - EOT - } -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = var.cluster_id - project_id = var.project_id - - # count = var.pvc_count - apply_manifests = flatten( - [ - # create StorageClass in the cluster - { - content = templatefile( - "${path.module}/storage-class/${local.storage_class_name}.yaml.tftpl", - { - name = local.storage_class_name - labels = local.labels - volume_binding_mode = var.sc_volume_binding_mode - reclaim_policy = var.sc_reclaim_policy - topology_zones = var.sc_topology_zones - }) - }, - var.namespace != "default" ? [{ - content = templatefile( - "${path.module}/persistent-volume-claim/namespace.yaml.tftpl", - { - namespace = var.namespace - }) - }] : [], - # create PersistentVolumeClaim in the cluster - flatten([ - for idx in range(var.pvc_count) : [ - { - content = templatefile( - "${path.module}/persistent-volume-claim/${(local.pvc_name_prefix)}.yaml.tftpl", - { - pvc_name = "${local.pvc_name_prefix}-${idx}" - labels = local.labels - capacity = "${var.capacity_gb}Gi" - access_mode = var.access_mode - storage_class_name = local.storage_class_name - namespace = var.namespace - } - ) - } - ] - ]) - ]) -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/metadata.yaml deleted file mode 100644 index 8722823274..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/outputs.tf deleted file mode 100644 index ce80cdb266..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/outputs.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "persistent_volume_claims" { - description = "An object that describes a k8s PVC created by this module." - value = flatten([ - for idx in range(var.pvc_count) : [{ - name = "${local.pvc_name_prefix}-${idx}" - namespace = var.namespace - mount_path = "${var.pv_mount_path}/${local.pvc_name_prefix}-${idx}" - mount_options = var.mount_options - storage_type = local.storage_type - }] - ]) -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl deleted file mode 100644 index fa7647e33f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl deleted file mode 100644 index 46e1f023d3..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - type: hyperdisk-balanced - provisioned-throughput-on-create: "250Mi" - provisioned-iops-on-create: "7000" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl deleted file mode 100644 index 445020d001..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - %{~ endfor ~} - type: hyperdisk-extreme - provisioned-iops-on-create: "50000" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl deleted file mode 100644 index ec404aec45..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - type: hyperdisk-throughput - provisioned-throughput-on-create: "250Mi" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl deleted file mode 100644 index e6b8ea8d3e..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: parallelstore.csi.storage.gke.io -parameters: -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/variables.tf deleted file mode 100644 index dba1c33b77..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/variables.tf +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "cluster_id" { - description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" - type = string -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} - -variable "storage_type" { - description = <<-EOT - The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview) - to used. This module currently support dynamic provisioning for the below storage options - - Parallelstore - - Hyperdisk-balanced - - Hyperdisk-throughput - - Hyperdisk-extreme - EOT - type = string - nullable = false - validation { - condition = var.storage_type == null ? false : contains(["parallelstore", "hyperdisk-balanced", "hyperdisk-throughput", "hyperdisk-extreme"], lower(var.storage_type)) - error_message = "Allowed string values for var.storage_type are \"Parallelstore\", \"Hyperdisk-balanced\", \"Hyperdisk-throughput\", \"Hyperdisk-extreme\"." - } -} - -variable "access_mode" { - description = <<-EOT - The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) - Valid access modes: - - ReadWriteOnce - - ReadOnlyMany - - ReadWriteMany - - ReadWriteOncePod - EOT - type = string - nullable = false - validation { - condition = var.access_mode == null ? false : contains(["readwriteonce", "readonlymany", "readwritemany", "readwriteoncepod"], lower(var.access_mode)) - error_message = "Allowed string values for var.access_mode are \"ReadWriteOnce\", \"ReadOnlyMany\", \"ReadWriteMany\", \"ReadWriteOncePod\"." - } -} - -variable "sc_volume_binding_mode" { - description = <<-EOT - Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound. - Supported value: - - Immediate - - WaitForFirstConsumer - EOT - type = string - default = "WaitForFirstConsumer" - validation { - condition = var.sc_volume_binding_mode == null ? true : contains(["immediate", "waitforfirstconsumer"], lower(var.sc_volume_binding_mode)) - error_message = "Allowed string values for var.sc_volume_binding_mode are \"Immediate\", \"WaitForFirstConsumer\"." - } -} - -variable "sc_reclaim_policy" { - description = <<-EOT - Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted. - [More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming) - Supported value: - - Retain - - Delete - EOT - type = string - nullable = false - validation { - condition = var.sc_reclaim_policy == null ? true : contains(["retain", "delete"], lower(var.sc_reclaim_policy)) - error_message = "Allowed string values for var.sc_reclaim_policy are \"Retain\", \"Delete\"." - } -} - -variable "sc_topology_zones" { - description = "Zone location that allow the volumes to be dynamically provisioned." - type = list(string) - default = null -} - -variable "pvc_count" { - description = "How many PersistentVolumeClaims that will be created" - type = number - default = 1 -} - -variable "pv_mount_path" { - description = "Path within the container at which the volume should be mounted. Must not contain ':'." - type = string - default = "/data" - validation { - condition = var.pv_mount_path == null ? true : !strcontains(var.pv_mount_path, ":") - error_message = "pv_mount_path must not contain ':', please correct it and retry" - } -} - -variable "mount_options" { - description = "Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class." - type = string - default = null -} - -variable "capacity_gb" { - description = "The storage capacity with which to create the persistent volume." - type = number -} - -variable "private_vpc_connection_peering" { - description = <<-EOT - The name of the VPC Network peering connection. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access and - If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). - EOT - type = string - default = null -} - -variable "namespace" { - description = "Kubernetes namespace to deploy the storage PVC/PV" - type = string - default = "default" -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/versions.tf deleted file mode 100644 index bcc803e41e..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/gke-storage/versions.tf +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.5" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.74.0" - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/README.md deleted file mode 100644 index 28530a379f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/README.md +++ /dev/null @@ -1,289 +0,0 @@ -## Description - -This module creates a [Managed Lustre](https://cloud.google.com/managed-lustre) -instance. Managed Lustre is a high performance network file system that can be -mounted to one or more VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -### Supported Operating Systems - -A Managed Lustre instance can be used with Slurm cluster or compute -VM running Ubuntu 20.04, 22.04 or Rocky Linux 8 (including the HPC flavor). - -### Managed Lustre Access - -Managed Lustre must be enabled for your project by Google staff. Please contact -your sales representative for further steps. - -### Example - New VPC - -For Managed Lustre instance, the snippet below creates new VPC and configures -private-service-access for this newly created network. Both items are required -to be passed to the Lustre module to ensure that they're built in order and -that the correct subnetwork has private service access. - -```yaml - - id: network - source: modules/network/vpc - - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] -``` - -### Example - Slurm - -When using Slurm you must take into consideration whether or not you are using -an official image from the `schedmd-slurm-public` project or building your own. -The Lustre client modules are pre-installed in the official images. With the -official images, Lustre can be used as follows: - -```yaml -- id: managed_lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - -# Other modules: nodesets, partitions, login, etc. - -- id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: - - network - - lustre_partition - - managed_lustre - - slurm_login - settings: - machine_type: n2-standard-4 - enable_controller_public_ips: true -``` - -For custom images you must install the modules during the image build as the -Slurm cluster will not run the installation script like it does for the -standard VMs. - -Assuming you have a startup script for the Slurm image building, you can add -this Ansible playbook to correctly install the Lustre drivers into the image -(for Slurm-GCP versions greater than 6.10.0): - -```yaml -- type: data - destination: /var/tmp/slurm_vars.json - content: | - { - "reboot": false, - "install_cuda": false, - "install_gcsfuse": true, - "install_lustre": false, - "install_managed_lustre": true, - "install_nvidia_repo": true, - "install_ompi": true, - "allow_kernel_upgrades": false, - "monitoring_agent": "cloud-ops", - } -``` - -The `install_managed_lustre: true` line specifies that slurm-gcp should install -the correct modules within the slurm image. This runner should be placed -ahead of the script that calls the ansible build of the slurm-gcp image. - -### Example - Existing VPC - -If you want to use existing network with private-service-access configured, you need -to manually provide `private_vpc_connection_peering` to the Managed Lustre module. -You can get this details from the Google Cloud Console UI in `VPC network peering` -section. Below is the example of using existing network and creating Managed Lustre. -If existing network is not configured with private-service-access, you can follow -[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) -to set it up. - -```yaml - - id: network - source: modules/network/pre-existing-vpc - settings: - network_name: // Add network name - subnetwork_name: // Add subnetwork name - - - id: lustre - source: modules/file-system/managed-lustre - use: [network] - settings: - private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" -``` - -### Example - GKE compatibility - -By default the Managed Lustre instance that is deployed is not compatible with -GKE. To enable the compatibility use the `gke_support_enabled: true` option. -This creates a file `/etc/modprobe/lnet.conf` that changes the listening port -to 6988. - -```yaml - - id: managed-lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - gke_support_enabled: true -``` - -> [!WARNING] -> -> 1. VMs cannot connect to both GKE compatible and GKE incompatible lustre -> instances at the same time as they connect to different ports. Lustre can -> only listen to one port at a time. -> -> 2. Setting `gke_support_enabled: true` will not affect Slurm nodes, GKE -> compatibility must be built into the Slurm image. - -### Example - Importing data from GSC Bucket - -One option with the Managed Lustre instance is to import data from a GSC bucket -upon the lustre instance creation. To do this, use the `import_gcs_bucket_uri` -variable to dictate the bucket to pull data from. The data will be imported -under the directory specified by `local_mount` (`/shared` if unspecified). - -> [!NOTE] -> -> 1. This is a one way operation. Once the data has been copied to the lustre -> instance it will not be updated with any changes made to the GCS bucket. -> -> 2. Once the lustre instance has been created in Terraform, the copy process -> will proceed in the background. Data may not be appear in the mounted -> directory for a period of time after the deployment has completed (see below). - -```yaml -- id: managed_lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - import_gcs_bucket_uri: gs:// -``` - -> [!WARNING] -> Please follow [this guide](https://cloud.google.com/managed-lustre/docs/transfer-data#required_permissions) -> to set up the correct IAM permissions for importing data from GCS to lustre. -> Without this, the copy process may fail silently leaving an empty lustre -> instance. - -If an import is requested, gcluster will output a json response similar to: - -```json -{ - "name": "projects//locations//operations/", - "metadata": { - "@type": "type.googleapis.com/google.cloud.lustre.v1.ImportDataMetadata", - "createTime": "", - "target": "projects//locations//instances/", - "requestedCancellation": false, - "apiVersion": "v1" - }, - "done": false -} -``` - -You can retrieve more information about the transfer using the following -command, substituting with values from the json response above: - -```bash -gcloud lustre operations describe --location --project -``` - -This will provide information on if the transfer is complete or if any errors -have occurred. See more at -[Get operation](https://cloud.google.com/managed-lustre/docs/transfer-data#get_operation). - -## License - - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 6.27.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.27.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_lustre_instance.lustre_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/lustre_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | -| [google_storage_bucket.lustre_import_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the Lustre instance if no name is specified. | `string` | n/a | yes | -| [description](#input\_description) | Description of the created Lustre instance. | `string` | `"Lustre Instance"` | no | -| [gke\_support\_enabled](#input\_gke\_support\_enabled) | Set to true to create Managed Lustre instance with GKE compatibility.
Note: This does not work with Slurm, the Slurm image must be built with
the correct compatibility. | `bool` | `false` | no | -| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to managed lustre. Data will
be imported to the local\_mount directory. Changing this value will not
trigger a redeployment, to prevent data deletion. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the Managed Lustre instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mount](#input\_local\_mount) | Local mount point for the Managed Lustre instance. | `string` | `"/shared"` | no | -| [mount\_options](#input\_mount\_options) | Mounting options for the file system. | `string` | `"defaults,_netdev"` | no | -| [name](#input\_name) | Name of the Lustre instance | `string` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | Network self-link this instance will be on, required for checking private service access | `string` | n/a | yes | -| [per\_unit\_storage\_throughput](#input\_per\_unit\_storage\_throughput) | Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000. | `number` | `500` | no | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | ID of project in which Lustre instance will be created. | `string` | n/a | yes | -| [remote\_mount](#input\_remote\_mount) | Remote mount point of the Managed Lustre instance | `string` | n/a | yes | -| [size\_gib](#input\_size\_gib) | Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations | `number` | `36000` | no | -| [zone](#input\_zone) | Location for the Lustre instance. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | -| [install\_managed\_lustre\_client](#output\_install\_managed\_lustre\_client) | Script for installing Managed Lustre client | -| [lustre\_id](#output\_lustre\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | -| [network\_storage](#output\_network\_storage) | Describes a Managed Lustre instance. | - diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/main.tf deleted file mode 100644 index a969c53673..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/main.tf +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "managed-lustre", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -data "google_compute_network_peering" "private_peering" { - name = var.private_vpc_connection_peering - network = var.network_self_link -} - -locals { - server_ip = split(":", google_lustre_instance.lustre_instance.mount_point)[0] - remote_mount = split(":", google_lustre_instance.lustre_instance.mount_point)[1] - fs_type = "lustre" - mount_options = var.mount_options - instance_id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - destination_path = "/" - - install_managed_lustre_client_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-managed-lustre-client.sh" - "destination" = "install-managed-lustre-client${replace(var.local_mount, "/", "_")}.sh" - "args" = var.gke_support_enabled ? "1" : "0" - } - mount_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" - } - - bucket_count = try(length(data.google_storage_bucket.lustre_import_bucket), 0) -} - -data "google_storage_bucket" "lustre_import_bucket" { - count = try(length(var.import_gcs_bucket_uri) > 0, false) ? 1 : 0 - - name = split("//", var.import_gcs_bucket_uri)[1] -} - -resource "google_lustre_instance" "lustre_instance" { - project = var.project_id - - description = var.description - instance_id = local.instance_id - location = var.zone - - filesystem = var.remote_mount - capacity_gib = var.size_gib - per_unit_storage_throughput = var.per_unit_storage_throughput - - labels = local.labels - network = var.network_id - - gke_support_enabled = var.gke_support_enabled - - timeouts { - create = "1h" - update = "1h" - delete = "1h" - } - - depends_on = [var.private_vpc_connection_peering, data.google_storage_bucket.lustre_import_bucket] - - lifecycle { - precondition { - condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" - error_message = "The subnetwork that the lustre instance is hosted on must have private service access." - } - } - - provisioner "local-exec" { - command = < 0 ]]; then - curl -X POST \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $(gcloud auth print-access-token)" \ - -d '{"gcsPath": {"uri":"${coalesce(var.import_gcs_bucket_uri, "gs://")}"}, "lustrePath": {"path":"${local.destination_path}"}}' \ - https://lustre.googleapis.com/v1/projects/${var.project_id}/locations/${var.zone}/instances/${local.instance_id}:importData - fi - EOF - interpreter = ["bash", "-c"] - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/metadata.yaml deleted file mode 100644 index 66da9827b6..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - lustre.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/outputs.tf deleted file mode 100644 index 6de815524a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a Managed Lustre instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = local.mount_options - client_install_runner = local.install_managed_lustre_client_runner - mount_runner = local.mount_runner - } -} - -output "install_managed_lustre_client" { - description = "Script for installing Managed Lustre client" - value = file("${path.module}/scripts/install-managed-lustre-client.sh") -} - -output "lustre_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" - value = google_lustre_instance.lustre_instance.id -} - -output "capacity_gib" { - description = "File share capacity in GiB." - value = google_lustre_instance.lustre_instance.capacity_gib -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh deleted file mode 100644 index 878130ab47..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Install Managed Lustre client modules -# Based on these instructions: https://cloud.google.com/managed-lustre/docs/connect-from-compute-engine - -# The client modules currently only support Rocky 8, and Ubuntu 20.04/22.04 - -set -e - -GKE_ENABLED=$1 - -# Update lnet to enable GKE supported Lustre instance -if [[ $GKE_ENABLED == "1" ]]; then - if [[ -f "/etc/modprobe.d/lnet.conf" ]] && grep -Fq "options lnet accept_port" /etc/modprobe.d/lnet.conf; then - echo "Lnet accept port already set, continuing without updating /etc/modprobe.d/lnet.conf" - else - echo "options lnet accept_port=6988" >>/etc/modprobe.d/lnet.conf - fi -fi - -if grep -q lustre /proc/filesystems; then - echo "Skipping managed lustre client install as it is already supported" - exit 0 -fi - -# Get distro information -. /etc/os-release -DIST="NA" -if [[ $NAME == *"Ubuntu"* ]]; then - if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then - DIST="Ubuntu" - fi -elif [[ $NAME == *"Rocky"* ]]; then - if [[ $VERSION_ID == "8"* ]]; then - DIST="Rocky" - fi -fi - -if [[ ${DIST} == "Ubuntu" ]]; then - KEY_LOC=/etc/apt/keyrings - KEY_NAME=gcp-ar-repo.gpg - # Download new repo key - mkdir -p "${KEY_LOC}" - wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null - - # Set up apt repo - echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list - - # Install modules - apt update - apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) -elif [[ ${DIST} == "Rocky" ]]; then - # Set up yum repo - touch /etc/yum.repos.d/artifact-registry.repo - tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF - [lustre-client-rocky-8] - name=lustre-client-rocky-8 - baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - # Install modules - yum makecache - yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client -fi - -if [[ $DIST != "NA" ]]; then - # Load the new lustre client module - modprobe lustre -fi diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/variables.tf deleted file mode 100644 index 65607af66d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/variables.tf +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which Lustre instance will be created." - type = string -} - -variable "description" { - description = "Description of the created Lustre instance." - type = string - default = "Lustre Instance" -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the Lustre instance if no name is specified." - type = string -} - -variable "zone" { - description = "Location for the Lustre instance." - type = string -} - -variable "name" { - description = "Name of the Lustre instance" - type = string -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is connected given in the format: - `projects//global/networks/`" - EOT - type = string - nullable = false - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "network_self_link" { - description = "Network self-link this instance will be on, required for checking private service access" - type = string - nullable = false -} - -variable "remote_mount" { - description = "Remote mount point of the Managed Lustre instance" - type = string - nullable = false -} - -variable "local_mount" { - description = "Local mount point for the Managed Lustre instance." - type = string - default = "/shared" -} - -variable "size_gib" { - description = "Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations" - type = number - default = 36000 -} - -variable "per_unit_storage_throughput" { - description = "Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000." - type = number - default = 500 -} - -variable "labels" { - description = "Labels to add to the Managed Lustre instance. Key-value pairs." - type = map(string) -} - -variable "mount_options" { - description = "Mounting options for the file system." - type = string - default = "defaults,_netdev" -} - -variable "private_vpc_connection_peering" { - description = <<-EOT - The name of the VPC Network peering connection. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access and - If using existing VPC with private-service-access enabled, set this manually." - EOT - type = string - nullable = false -} - -variable "gke_support_enabled" { - description = <<-EOT - Set to true to create Managed Lustre instance with GKE compatibility. - Note: This does not work with Slurm, the Slurm image must be built with - the correct compatibility. - EOT - type = bool - nullable = false - default = false -} - -variable "import_gcs_bucket_uri" { - description = <<-EOT - The name of the GCS bucket to import data from to managed lustre. Data will - be imported to the local_mount directory. Changing this value will not - trigger a redeployment, to prevent data deletion. - EOT - type = string - default = null - - validation { - condition = startswith(coalesce(var.import_gcs_bucket_uri, "gs://"), "gs://") - error_message = "The GCS bucket uri must start with 'gs://'" - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/versions.tf deleted file mode 100644 index 2322c9a8fd..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/managed-lustre/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.27.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/README.md deleted file mode 100644 index 82332f3406..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/README.md +++ /dev/null @@ -1,193 +0,0 @@ -## Description - -This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) -storage pool. - -NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. -NetApp Volume provides: - -- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x -- a [rich feature set][service-levels] -- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) -- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data -- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost - -Support for NetApp Volumes is split into two modules. - -- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. -- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -### NetApp storage pool service levels - -The netapp-storage-pool module currently supports the following NetApp Volumes [service levels][service-levels]: - -- Standard: 16 KiBps throughput per provisioned KiB of volume capacity. -- Premium: 64 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. -- Extreme: 128 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. - -Check the [service level matrix][service-levels] for additional information on capability differences between service levels. Flex service levels are currently not supported, but you can connect to existing Flex volumes using the [pre-existing-network-storage module][pre-existing]. - -### On-boarding NetApp Volumes -NetApp Volumes uses [Private Service Access](https://cloud.google.com/vpc/docs/private-services-access) (PSA) to connect volumes to your network. Before you create a storage pool, make sure to [connect NetApp Volumes to your network](https://cloud.google.com/netapp/volumes/docs/get-started/configure-access/networking). - -Example of creating a storage pool using a new network: - -```yaml -deployment_groups: -- group: primary - modules: - - id: network - source: modules/network/vpc - settings: - region: $(vars.region) - - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - service_name: "netapp.servicenetworking.goog" - deletion_policy: "ABANDON" - - - id: netapp_pool - source: modules/file-system/netapp-storage-pool - use: [network, private_service_access] - settings: - pool_name: $(vars.deployment_name)-eda-pool - capacity_gib: 20000 - service_level: "EXTREME" - region: $(vars.region) -``` - -Example of creating a storage pool using an existing network which was already PSA-peered with NetApp Volume: - -```yaml -deployment_groups: - - group: primary - modules: - - id: network - source: modules/network/pre-existing-vpc - settings: - project_id: $(vars.project_id) - region: $(vars.region) - network_name: $(vars.network) - - - id: netapp_pool - source: modules/file-system/netapp-storage-pool - use: [network] - settings: - pool_name: "eda-pool" - capacity_gib: 20000 - service_level: "EXTREME" - region: $(vars.region) -``` - -### Storage pool example - -The following example shows all available parameters in use: - -```yaml - - id: netapp_pool - source: modules/file-system/netapp-storage-pool - use: [network, private_service_access] - settings: - pool_name: "mypool" - region: "us-west4" - capacity_gib: 2048 - service_level: "EXTREME" - active_directory_policy: "projects/myproject/locations/us-east4/activeDirectories/my-ad" - cmek_policy: "projects/myproject/locations/us-east4/kmsConfigs/my-cmek-policy" - ldap_enabled: false - allow_auto_tiering: false - description: "Demo storage pool" - labels: - owner: bob -``` - -### NetApp Volumes quota - -Your project must have unused quota for NetApp Volumes in the region you will -provision the storage pool. This can be found by browsing to the [Quota tab within IAM & Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. -Please note that there are separate quota limits for Standard and Premium/Extreme service levels. - -See also NetApp Volumes [default quotas](https://cloud.google.com/netapp/volumes/docs/quotas#netapp-volumes-default-quotas). - -[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels -[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering -[pre-existing]: ../pre-existing-network-storage/README.md -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5.7 | -| [google](#requirement\_google) | >= 6.45.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.45.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_netapp_storage_pool.netapp_storage_pool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_storage_pool) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [active\_directory\_policy](#input\_active\_directory\_policy) | The ID of the Active Directory policy to apply to the storage pool in the format:
`projects//locations//activeDirectoryPolicies/` | `string` | `null` | no | -| [allow\_auto\_tiering](#input\_allow\_auto\_tiering) | Whether to allow automatic tiering for the storage pool. | `bool` | `false` | no | -| [capacity\_gib](#input\_capacity\_gib) | The capacity of the storage pool in GiB. | `number` | `2048` | no | -| [cmek\_policy](#input\_cmek\_policy) | The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format:
`projects//locations//kmsConfigs/` | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used as name of the NetApp storage pool if no name is specified. | `string` | n/a | yes | -| [description](#input\_description) | A description of the NetApp storage pool. | `string` | `""` | no | -| [labels](#input\_labels) | Labels to add to the NetApp storage pool. Key-value pairs. | `map(string)` | n/a | yes | -| [ldap\_enabled](#input\_ldap\_enabled) | Whether to enable LDAP for the storage pool. | `bool` | `false` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | Network self-link the pool will be on, required for checking private service access | `string` | n/a | yes | -| [pool\_name](#input\_pool\_name) | The name of the storage pool. Leave empty to generate name based on deployment name. | `string` | `null` | no | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the private VPC connection peering. | `string` | `"sn-netapp-prod"` | no | -| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | -| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | -| [service\_level](#input\_service\_level) | The service level of the storage pool. | `string` | `"PREMIUM"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gb](#output\_capacity\_gb) | Storage pool capacity in GiB. | -| [netapp\_storage\_pool\_id](#output\_netapp\_storage\_pool\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}` | - diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/main.tf deleted file mode 100644 index b9d63c11c3..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/main.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "netapp-storage-pool", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -data "google_compute_network_peering" "private_peering" { - name = var.private_vpc_connection_peering - network = var.network_self_link -} - -resource "google_netapp_storage_pool" "netapp_storage_pool" { - project = var.project_id - - name = var.pool_name != null ? var.pool_name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - location = var.region - network = var.network_id - service_level = var.service_level - capacity_gib = var.capacity_gib - - active_directory = var.active_directory_policy - kms_config = var.cmek_policy - ldap_enabled = var.ldap_enabled - allow_auto_tiering = var.allow_auto_tiering - - description = var.description - labels = local.labels - - depends_on = [data.google_compute_network_peering.private_peering] - - lifecycle { - precondition { - condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" - error_message = "The network for the storage pool must have private service access." - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml deleted file mode 100644 index 7a5291f9d5..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - netapp.googleapis.com - - servicenetworking.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf deleted file mode 100644 index 91379631c6..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "netapp_storage_pool_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}`" - value = google_netapp_storage_pool.netapp_storage_pool.id -} - -output "capacity_gb" { - description = "Storage pool capacity in GiB." - value = google_netapp_storage_pool.netapp_storage_pool.capacity_gib -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf deleted file mode 100644 index 04f19fd3fb..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which the NetApp storage pool will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the deployment, used as name of the NetApp storage pool if no name is specified." - type = string -} - -variable "region" { - description = "Location for NetApp storage pool." - type = string -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "network_self_link" { - description = "Network self-link the pool will be on, required for checking private service access" - type = string - nullable = false -} - -variable "private_vpc_connection_peering" { - description = "The name of the private VPC connection peering." - type = string - default = "sn-netapp-prod" -} - -variable "pool_name" { - description = "The name of the storage pool. Leave empty to generate name based on deployment name." - type = string - default = null -} - -variable "service_level" { - description = "The service level of the storage pool." - type = string - default = "PREMIUM" - validation { - condition = contains(["STANDARD", "PREMIUM", "EXTREME"], var.service_level) - error_message = "Allowed values for service_level are 'STANDARD', 'PREMIUM', or 'EXTREME'." - } -} - -variable "capacity_gib" { - description = "The capacity of the storage pool in GiB." - type = number - default = 2048 - validation { - condition = var.capacity_gib >= 2048 - error_message = "The minimum capacity for the storage pool is 2048 GiB." - } -} - -variable "active_directory_policy" { - description = <<-EOT - The ID of the Active Directory policy to apply to the storage pool in the format: - `projects//locations//activeDirectoryPolicies/` - EOT - type = string - default = null - validation { - condition = var.active_directory_policy == null ? true : length(split("/", var.active_directory_policy)) == 6 - error_message = "The active directory policy must be provided in the following format: projects//locations//activeDirectoryPolicies/." - } -} - -variable "cmek_policy" { - description = <<-EOT - The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format: - `projects//locations//kmsConfigs/` - EOT - type = string - default = null - validation { - condition = var.cmek_policy == null ? true : length(split("/", var.cmek_policy)) == 6 - error_message = "The CMEK policy must be provided in the following format: projects//locations//kmsConfigs/." - } -} - -variable "ldap_enabled" { - description = "Whether to enable LDAP for the storage pool." - type = bool - default = false -} - -variable "allow_auto_tiering" { - description = "Whether to allow automatic tiering for the storage pool." - type = bool - default = false -} - -variable "description" { - description = "A description of the NetApp storage pool." - type = string - default = "" - validation { - condition = length(var.description) <= 2048 - error_message = "NetApp storage pool description must be 2048 characters or fewer" - } -} - -variable "labels" { - description = "Labels to add to the NetApp storage pool. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf deleted file mode 100644 index f6501116cd..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.45.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" - } - - required_version = ">= 1.5.7" -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/README.md deleted file mode 100644 index 6aaaf0cb05..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/README.md +++ /dev/null @@ -1,201 +0,0 @@ -## Description - -This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) -volume. - -NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. -NetApp Volume provides: - -- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x -- a [rich feature set][service-levels] -- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) -- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data -- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost - -Support for NetApp Volumes is split into two modules. - -- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. -- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -## Deletion protection -The netapp-volume module currently doesn't implement volume deletion protection. If you create a volume with Cluster Toolkit by using this module, Cluster Toolkit will also delete it when you run `gcluster destroy`. All the data in the volume will be gone. If you want to retain the volume instead, it is advised to [use existing volumes not created by Cluster Toolkit](#using-existing-volumes-not-created-by-cluster-toolkit). - -## Volumes overview -Volumes are filesystem containers which can be shared using NFS or SMB filesharing protocols. Volumes *live* inside of [storage pools](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview), which can be provisioned using the [netapp-storage-pool] module. Volumes inherit fundamental settings from the pool. They *consume* capacity provided by the pool. You can create one or multiple volumes *inside* a pool. - -[netapp-storage-pool]: ../netapp-storage-pool/README.md -[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels -[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering -[pre-existing]: ../pre-existing-network-storage/README.md -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## Volume examples -The following examples show the use of netapp-volume. They builds on top of an storage pool which can be provisioned using the [netapp-storage-pool][netapp-storage-pool] module. - -### Example with minimal parameters - -```yaml - - id: home_volume - source: modules/file-system/netapp-volume - use: [netapp_pool] # Create this pool using the netapp-storage-pool module - settings: - volume_name: "eda-home" - capacity_gib: 1024 # Size up to available capacity in the pool - local_mount: "/eda-home" # Mount point at client when client uses USE directive - protocols: ["NFSV3"] - region: $(vars.region) - # Default export policy exports to "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" and no_root_squash -``` - -### Example with all parameters - -```yaml - - id: shared_volume - source: modules/file-system/netapp-volume - use: [netapp_pool] # Create this pool using the netapp-storage-pool module - settings: - volume_name: "eda-shared" - capacity_gib: 25000 # Size up to available capacity in the pool - large_capacity: true - local_mount: "/shared" # Mount point at client when client uses USE directive - mount_options: "rw" # Allows customizing mount options for special workloads - protocols: ["NFSV3","NFSV4"] # List of protocols. ["NFSV3], ["NFSv4] or ["NFSV3, "NFSV4"] - region: $(vars.region) - unix_permissions: "0777" # Specify default permissions for roo inode owned by root:root - # If no export policy is specified, a permissive default policy will be applied, which is: - # allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" # RFC1918 - # has_root_access = true # no_root_squash enabled - # access_type = "READ_WRITE" - export_policy: - - allowed_clients: "10.10.20.8,10.10.20.9" - has_root_access: true # no_root_squash enabled - access_type: "READ_WRITE" - nfsv3: false # allow only NFSv4 for these hosts - nfsv4: true - - allowed_clients: "10.0.0.0/8" - has_root_access: false # no_root_squash disabled - access_type: "READ_WRITE" - nfsv3: true # allow only NFSv3 for these hosts - nfsv4: false - tiering_policy: # Enable auto-tiering. Requires auto-tiering enabled storage pool - tier_action: "ENABLED" - cooling_threshold_days: 31 # tier data blocks which have not been touched for 31 days - - description: "Shared volume for EDA job" - labels: - owner: bob -``` - -## Protocol support -Since Cluster Toolkit is currently built to provision Linux-based compute clusters, this module supports NFSv3 and NFSv4.1 only. SMB is blocked. - -## Large volumes -Volumes larger than 15 TiB can be created as [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). Such volumes can grow up to 3 PiB and can scale read performance up to 29 GiBps. They provide six IP addresses to the volume. They are exported via the `server_ips` output. When connecting a large volume to a client using the USE directive, cluster toolkit currently uses the first IP only. This will be improved in the future. - -This feature is allow-listed GA. To request allow-listing, see [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). - -## Auto-tiering support -For auto-tiering enabled storage pools you can enable auto-tiering on the volume. For more information, see [manage auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering). - -## Using existing volumes not created by Cluster Toolkit -NetApp Volumes volumes are regular NFS exports. You can use the [pre-existing-network-storage] module to integrate them into Cluster Toolkit. - -Example code: - -```yaml -- id: homefs - source: modules/file-system/pre-existing-network-storage - settings: - server_ip: ## Set server IP here ## - remote_mount: nfsshare - local_mount: /home - fs_type: nfs -``` - -This creates a resource in Cluster Toolkit which references the specified NFS export, which will be mounted at `/home` by clients which mount if via USE directive. - -Note that the `server_ip` must be known before deployment and this module does not allow -to specify a list of IPs for large volumes. - -[pre-existing-network-storage]: ../pre-existing-network-storage/README.md - -## FlexCache support -NetApp FlexCache technology accelerates data access, reduces WAN latency and lowers WAN bandwidth costs for read-intensive workloads, especially where clients need to access the same data repeatedly. When you create a FlexCache volume, you create a remote cache of an already existing (origin) volume that contains only the actively accessed data (hot data) of the origin volume. - -The FlexCache support in Google Cloud NetApp Volumes allows you to provision a cache volume in your Google network to improve performance for hybrid cloud environments. A FlexCache volume can help you transition workloads to the hybrid cloud by caching data from an on-premises data center to cloud. - -Deploying FlexCache volumes requires manual steps on the ONTAP origin side, which are not automated. Therefore this module has no support to deploy FlexCache volumes today. Deploy them manually and use the [pre-existing-network-storage](#using-existing-volumes-not-created-by-cluster-toolkit) instead. - -## License - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5.7 | -| [google](#requirement\_google) | >= 6.45.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.45.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_netapp_volume.netapp_volume](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_volume) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [capacity\_gib](#input\_capacity\_gib) | The capacity of the volume in GiB. | `number` | `1024` | no | -| [description](#input\_description) | A description of the NetApp volume. | `string` | `""` | no | -| [export\_policy\_rules](#input\_export\_policy\_rules) | Define NFS export policy. | 
list(object({
    allowed_clients = optional(string)
    has_root_access = optional(bool, false)
    access_type     = optional(string, "READ_WRITE")
    nfsv3           = optional(bool)
    nfsv4           = optional(bool)
  }))
| 
[
  {
    "access_type": "READ_WRITE",
    "allowed_clients": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
    "has_root_access": true
  }
]
| no | -| [labels](#input\_labels) | Labels to add to the NetApp volume. Key-value pairs. | `map(string)` | n/a | yes | -| [large\_capacity](#input\_large\_capacity) | If true, the volume will be created with large capacity.
Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. | `bool` | `false` | no | -| [local\_mount](#input\_local\_mount) | Mountpoint for this volume. | `string` | `"/shared"` | no | -| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"rw,hard,rsize=65536,wsize=65536,tcp"` | no | -| [netapp\_storage\_pool\_id](#input\_netapp\_storage\_pool\_id) | The ID of the NetApp storage pool to use for the volume. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | -| [protocols](#input\_protocols) | The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported. | `list(string)` | 
[
  "NFSV3"
]
| no | -| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | -| [tiering\_policy](#input\_tiering\_policy) | Define the tiering policy for the NetApp volume. | 
object({
    tier_action            = optional(string)
    cooling_threshold_days = optional(number)
  })
| `null` | no | -| [unix\_permissions](#input\_unix\_permissions) | UNIX permissions for root inode in the volume. | `string` | `"0777"` | no | -| [volume\_name](#input\_volume\_name) | The name of the volume. Needs to be unique within the storage pool. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gb](#output\_capacity\_gb) | Volume capacity in GiB. | -| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | -| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | -| [netapp\_volume\_id](#output\_netapp\_volume\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}` | -| [network\_storage](#output\_network\_storage) | Describes a NetApp Volumes volume. | -| [server\_ips](#output\_server\_ips) | List of IP addresses of the volume. | - diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/main.tf deleted file mode 100644 index d8345bf347..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/main.tf +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "netapp-volume", ghpc_role = "file-system" }) -} - -# resource "random_id" "resource_name_suffix" { -# byte_length = 4 -# } - -locals { - full_path = split(":", google_netapp_volume.netapp_volume.mount_options[0].export_full) - server_ip = local.full_path[0] - remote_mount = local.full_path[1] - # Large volumes will have 6 IPs - server_ips = [for ip in google_netapp_volume.netapp_volume.mount_options[*].export_full : split(":", ip)[0]] - fs_type = "nfs" - mount_options = var.mount_options - - install_nfs_client_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-nfs-client.sh" - "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" - } - mount_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${join(",", local.server_ips)}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" - } - - split_pool_id = split("/", var.netapp_storage_pool_id) - pool_name = local.split_pool_id[5] -} - -resource "google_netapp_volume" "netapp_volume" { - project = var.project_id - - name = var.volume_name - share_name = var.volume_name - location = var.region - protocols = var.protocols - capacity_gib = var.capacity_gib - large_capacity = var.large_capacity - multiple_endpoints = var.large_capacity == true ? true : null - storage_pool = local.pool_name - unix_permissions = var.unix_permissions - - dynamic "tiering_policy" { - for_each = var.tiering_policy == null ? [] : [0] - content { - cooling_threshold_days = lookup(var.tiering_policy, "cooling_threshold_days", null) - tier_action = lookup(var.tiering_policy, "tier_action", null) - } - } - - description = var.description - labels = local.labels - - dynamic "export_policy" { - for_each = var.export_policy_rules == null ? [] : [0] - content { - dynamic "rules" { - for_each = var.export_policy_rules - content { - access_type = rules.value.access_type - allowed_clients = rules.value.allowed_clients - has_root_access = rules.value.has_root_access - nfsv3 = rules.value.nfsv3 == null ? contains([for p in var.protocols : lower(p)], "nfsv3") : rules.value.nfsv3 - nfsv4 = rules.value.nfsv4 == null ? contains([for p in var.protocols : lower(p)], "nfsv4") : rules.value.nfsv4 - } - } - } - } - - depends_on = [var.netapp_storage_pool_id] -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/metadata.yaml deleted file mode 100644 index e4a7aaaa14..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - netapp.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/outputs.tf deleted file mode 100644 index 641eae007a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/outputs.tf +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -output "network_storage" { - description = "Describes a NetApp Volumes volume." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = local.mount_options - client_install_runner = local.install_nfs_client_runner - mount_runner = local.mount_runner - } -} - -output "install_nfs_client" { - description = "Script for installing NFS client" - value = file("${path.module}/scripts/install-nfs-client.sh") -} - -output "install_nfs_client_runner" { - description = "Runner to install NFS client using the startup-script module" - value = local.install_nfs_client_runner -} - -output "mount_runner" { - description = <<-EOT - Runner to mount the file-system using an ansible playbook. The startup-script - module will automatically handle installation of ansible. - - id: example-startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(your-fs-id.mount_runner) - ... - EOT - value = local.mount_runner -} - -output "netapp_volume_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}`" - value = google_netapp_volume.netapp_volume.id -} - -output "capacity_gb" { - description = "Volume capacity in GiB." - value = google_netapp_volume.netapp_volume.capacity_gib -} - -output "server_ips" { - description = "List of IP addresses of the volume." - value = local.server_ips -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh deleted file mode 100644 index 1b1595e5a4..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsupported distribution' - return 1 - fi -fi diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh deleted file mode 100644 index 8253d40a24..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IPS=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -# accept a list of colon-separated IPs and randomly pick one to enable load balancing -# In recent changes cluster toolkit doesn't seem to use this file anymore, -# which makes all mounts use the first IP in the list. Needs to be investigated in future. -IFS="," read -r -a arrIPS <<<"${SERVER_IPS}" -rand1=$(od -vAn -t d -N1 /dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/variables.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/variables.tf deleted file mode 100644 index 272558ff77..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/variables.tf +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which the NetApp storage pool will be created." - type = string -} - -variable "netapp_storage_pool_id" { - description = "The ID of the NetApp storage pool to use for the volume." - type = string - validation { - condition = length(split("/", var.netapp_storage_pool_id)) == 6 - error_message = "The storage pool id must be provided in the following format: projects//locations//storagePools/." - } -} - -variable "region" { - description = "Location for NetApp storage pool." - type = string -} - -variable "volume_name" { - description = "The name of the volume. Needs to be unique within the storage pool." - type = string - default = null -} - -variable "capacity_gib" { - description = "The capacity of the volume in GiB." - type = number - default = 1024 - validation { - condition = var.capacity_gib >= 100 - error_message = "The minimum capacity for the volume is 100 GiB." - } -} - -variable "protocols" { - description = "The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported." - type = list(string) - default = ["NFSV3"] - validation { - condition = alltrue([for p in var.protocols : contains(["NFSV3", "NFSV4"], p)]) - error_message = "Allowed values for protocols are 'NFSV3' or 'NFSV4'." - } -} - -variable "description" { - description = "A description of the NetApp volume." - type = string - default = "" - validation { - condition = length(var.description) <= 2048 - error_message = "NetApp volume description must be 2048 characters or fewer" - } -} - -variable "labels" { - description = "Labels to add to the NetApp volume. Key-value pairs." - type = map(string) -} - -variable "local_mount" { - description = "Mountpoint for this volume." - type = string - default = "/shared" -} - -variable "mount_options" { - description = "NFS mount options to mount file system." - type = string - default = "rw,hard,rsize=65536,wsize=65536,tcp" -} - -variable "large_capacity" { - description = <<-EOT - If true, the volume will be created with large capacity. - Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. - EOT - type = bool - default = false -} - -variable "unix_permissions" { - description = "UNIX permissions for root inode in the volume." - type = string - default = "0777" - validation { - condition = length(var.unix_permissions) <= 4 - error_message = "UNIX permissions must be a 4-digit octal number." - } -} - -variable "tiering_policy" { - description = "Define the tiering policy for the NetApp volume." - type = object({ - tier_action = optional(string) - cooling_threshold_days = optional(number) - }) - default = null -} - -variable "export_policy_rules" { - description = "Define NFS export policy." - type = list(object({ - allowed_clients = optional(string) - has_root_access = optional(bool, false) - access_type = optional(string, "READ_WRITE") - nfsv3 = optional(bool) - nfsv4 = optional(bool) - })) - # Permissive default if user does not specify nfs_export_options. Allow all RFC1918 CIDRS with no_root_squash - default = [{ - allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", - has_root_access = true, - access_type = "READ_WRITE", - }] - nullable = true -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/versions.tf deleted file mode 100644 index c624d5100b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/netapp-volume/versions.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.45.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" - } - - required_version = ">= 1.5.7" -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/README.md deleted file mode 100644 index 0b942f067f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/README.md +++ /dev/null @@ -1,196 +0,0 @@ -## Description - -This module creates [parallelstore](https://cloud.google.com/parallelstore) -instance. Parallelstore is Google Cloud's first party parallel file system -service based on [Intel DAOS](https://docs.daos.io/v2.2/) - -### Supported Operating Systems - -A parallelstore instance can be used with Slurm cluster or compute -VM running Ubuntu 22.04, debian 12 or HPC Rocky Linux 8. - -### Parallelstore Quota - -To get access to a private preview of Parallelstore APIs, your project needs to -be allowlisted. To set this up, please work with your account representative. - -### Parallelstore mount options - -After parallelstore instance is created, you can specify mount options depending -upon your workload. DAOS is configured to deliver the best user experience for -interactive workloads with aggressive caching. If you are running parallel -workloads concurrently accessing the sane files from multiple client nodes, it -is recommended to disable the writeback cache to avoid cross-client consistency -issues. You can specify different mount options as follows, - -```yaml - - id: parallelstore - source: modules/file-system/parallelstore - use: [network, ps_connect] - settings: - mount_options: "disable-wb-cache,thread-count=20,eq-count=8" -``` - -### Example - New VPC - -For parallelstore instance, Below snippet creates new VPC and configures private-service-access -for this newly created network. - -```yaml - - id: network - source: modules/network/vpc - - # Private Service Access (PSA) requires the compute.networkAdmin role which is - # included in the Owner role, but not Editor. - # PSA is required for all Parallelstore functionality. - # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: parallelstore - source: modules/file-system/parallelstore - use: [network, private_service_access] -``` - -### Example - Existing VPC - -If you want to use existing network with private-service-access configured, you need -to manually provide `private_vpc_connection_peering` to the parallelstore module. -You can get this details from the Google Cloud Console UI in `VPC network peering` -section. Below is the example of using existing network and creating parallelstore. -If existing network is not configured with private-service-access, you can follow -[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) -to set it up. - -```yaml - - id: network - source: modules/network/pre-existing-vpc - settings: - network_name: // Add network name - subnetwork_name: // Add subnetwork name - - - id: parallelstore - source: modules/file-system/parallelstore - use: [network] - settings: - private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" -``` - -### Import data from GCS bucket - -You can import data from your GCS bucket to parallelstore instance. Important to -note that data may not be available to the instance immediately. This depends on -latency and size of data. Below is the example of importing data from bucket. - -```yaml - - id: parallelstore - source: modules/file-system/parallelstore - use: [network] - settings: - import_gcs_bucket_uri: gs://gcs-bucket/folder-path - import_destination_path: /gcs/import/ -``` - -Here you can replace `import_gcs_bucket_uri` with the uri of sub folder within GCS -bucket and `import_destination_path` with local directory within parallelstore -instance. - -### Additional configuration for DAOS agent and dfuse -Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - daos_agent_config: | - credential_config: - cache_expiration: 1m -``` - -Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/parallelstore - settings: - dfuse_environment: - D_LOG_FILE: /tmp/client.log - D_APPEND_PID_TO_LOG: 1 - D_LOG_MASK: debug -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13 | -| [google](#requirement\_google) | >= 6.13.0 | -| [null](#requirement\_null) | ~> 3.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.13.0 | -| [null](#provider\_null) | ~> 3.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_parallelstore_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/parallelstore_instance) | resource | -| [null_resource.hydration](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [daos\_agent\_config](#input\_daos\_agent\_config) | Additional configuration to be added to daos\_config.yml | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. | `string` | n/a | yes | -| [dfuse\_environment](#input\_dfuse\_environment) | Additional environment variables for DFuse process | `map(string)` | `{}` | no | -| [directory\_stripe](#input\_directory\_stripe) | The parallelstore stripe level for directories. | `string` | `null` | no | -| [file\_stripe](#input\_file\_stripe) | The parallelstore stripe level for files. | `string` | `null` | no | -| [import\_destination\_path](#input\_import\_destination\_path) | The name of local path to import data on parallelstore instance from GCS bucket. | `string` | `null` | no | -| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to parallelstore. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to parallel store instance. | `map(string)` | `{}` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/parallelstore"` | no | -| [mount\_options](#input\_mount\_options) | Options describing various aspects of the parallelstore instance. | `string` | `"disable-wb-cache,thread-count=16,eq-count=8"` | no | -| [name](#input\_name) | Name of parallelstore instance. | `string` | `null` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [size\_gb](#input\_size\_gb) | Storage size of the parallelstore instance in GB. | `number` | `12000` | no | -| [zone](#input\_zone) | Location for parallelstore instance. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions to monitor import-data operation from GCS bucket to parallelstore. | -| [network\_storage](#output\_network\_storage) | Describes a parallelstore instance. | - diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/main.tf b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/main.tf deleted file mode 100644 index acc2a0551e..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/main.tf +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "parallelstore", ghpc_role = "file-system" }) -} - -locals { - fs_type = "daos" - server_ip = "" - remote_mount = "" - id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - access_points = jsonencode(google_parallelstore_instance.instance.access_points) - destination_path = var.import_destination_path == null ? "/" : var.import_destination_path - - client_install_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-daos-client.sh" - "destination" = "install_daos_client.sh" - } - - mount_runner = { - "type" = "shell" - "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { - access_points = local.access_points - daos_agent_config = var.daos_agent_config - dfuse_environment = var.dfuse_environment - local_mount = var.local_mount - mount_options = join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) - }) - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - } -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_parallelstore_instance" "instance" { - project = var.project_id - instance_id = local.id - location = var.zone - capacity_gib = var.size_gb - network = var.network_id - file_stripe_level = var.file_stripe - directory_stripe_level = var.directory_stripe - - labels = local.labels - - depends_on = [var.private_vpc_connection_peering] -} - -resource "null_resource" "hydration" { - count = var.import_gcs_bucket_uri != null ? 1 : 0 - - depends_on = [resource.google_parallelstore_instance.instance] - provisioner "local-exec" { - command = "curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -d '{\"source_gcs_bucket\": {\"uri\":\"${var.import_gcs_bucket_uri}\"}, \"destination_parallelstore\": {\"path\":\"${local.destination_path}\"}}' https://parallelstore.googleapis.com/v1beta/projects/${var.project_id}/locations/${var.zone}/instances/${local.id}:importData" - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/metadata.yaml deleted file mode 100644 index c0994d15bb..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - parallelstore.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/outputs.tf deleted file mode 100644 index f6e817ac8a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/outputs.tf +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - operation_instructions = <<-EOT - Data is being imported from GCS bucket to parallelstore instance. It may - not be available immediately. - EOT -} - -output "network_storage" { - description = "Describes a parallelstore instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = var.mount_options - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } - - precondition { - condition = var.import_gcs_bucket_uri != null || var.import_destination_path == null - error_message = <<-EOD - Please specify import_gcs_bucket_uri to import data to parallelstore instance. - EOD - } -} - -output "instructions" { - description = "Instructions to monitor import-data operation from GCS bucket to parallelstore." - value = var.import_gcs_bucket_uri != null ? local.operation_instructions : "Data is not imported from GCS bucket." -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh deleted file mode 100644 index e96eadb56a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || - { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 -fi - -if [ -x /bin/daos ]; then - echo "DAOS already installed" - daos version -else - # Install the DAOS client library - # The following commands should be executed on each client vm. - ## For Rocky linux 8 / RedHat 8. - if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then - # 1) Add the Parallelstore package repository - cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF - [parallelstore-v2-6-el${OS_VERSION_MAJOR}] - name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 - baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - - ## TODO: Remove disable automatic update script after issue is fixed. - if [ -x /usr/bin/google_disable_automatic_updates ]; then - /usr/bin/google_disable_automatic_updates - fi - dnf clean all - dnf makecache - - # 2) Install daos-client - dnf install -y epel-release # needed for capstone - dnf install -y daos-client - - # 3) Upgrade libfabric - dnf upgrade -y libfabric - - # For Ubuntu 22.04 and debian 12, - elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then - # shellcheck disable=SC2034 - DEBIAN_FRONTEND=noninteractive - - # 1) Add the Parallelstore package repository - curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg - echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list - - apt-get update - - # 2) Install daos-client - apt-get install -y daos-client - - # 3) Create daos_agent.service (comes pre-installed with RedHat) - if ! getent passwd daos_agent >/dev/null 2>&1; then - useradd daos_agent - fi - cat >/etc/systemd/system/daos_agent.service <<-EOF - [Unit] - Description=DAOS Agent - StartLimitIntervalSec=60 - Wants=network-online.target - After=network-online.target - - [Service] - Type=notify - User=daos_agent - Group=daos_agent - RuntimeDirectory=daos_agent - RuntimeDirectoryMode=0755 - ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml - StandardOutput=journal - StandardError=journal - Restart=always - RestartSec=10 - LimitMEMLOCK=infinity - LimitCORE=infinity - StartLimitBurst=5 - - [Install] - WantedBy=multi-user.target - EOF - else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - fi -fi - -exit 0 diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl deleted file mode 100644 index c6f5d53660..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; } && { [[ "$${OS_VERSION_MAJOR}" = "8" ]] || [[ "$${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "$${OS_ID}" = "ubuntu" ]] && [[ "$${OS_VERSION}" = "22.04" ]]; } || - { [[ "$${OS_ID}" = "debian" ]] && [[ "$${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system $${OS_ID} $${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - -fi - -# Edit agent config -daos_config=/etc/daos/daos_agent.yml - -# rewrite $daos_config from scratch -mv $${daos_config} $${daos_config}.orig - -exclude_fabric_ifaces="" -# Get names of network interfaces not in first PCI slot -# The first PCI slot is a standard network adapter while remaining interfaces -# are typically network cards dedicated to GPU or workload communication -if [[ "$${OS_ID}" == "debian" ]] || [[ "$${OS_ID}" = "ubuntu" ]]; then - extra_interfaces=$(find /sys/class/net/ -not -name 'enp0s*' -regextype posix-extended -regex '.*/enp[0-9]+s.*' -printf '"%f"\n' | paste -s -d ',') -elif [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; then - extra_interfaces=$(find /sys/class/net/ -not -name eth0 -regextype posix-extended -regex '.*/eth[0-9]+' -printf '"%f"\n' | paste -s -d ',') -fi - -cat > $daos_config </etc/systemd/system/"$${service_name}" </global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "import_gcs_bucket_uri" { - description = "The name of the GCS bucket to import data from to parallelstore." - type = string - default = null -} - -variable "import_destination_path" { - description = "The name of local path to import data on parallelstore instance from GCS bucket." - type = string - default = null -} - -variable "file_stripe" { - description = "The parallelstore stripe level for files." - type = string - default = null - validation { - condition = var.file_stripe == null ? true : contains([ - "FILE_STRIPE_LEVEL_UNSPECIFIED", - "FILE_STRIPE_LEVEL_MIN", - "FILE_STRIPE_LEVEL_BALANCED", - "FILE_STRIPE_LEVEL_MAX", - ], var.file_stripe) - error_message = "var.file_stripe must be set to \"FILE_STRIPE_LEVEL_UNSPECIFIED\", \"FILE_STRIPE_LEVEL_MIN\", \"FILE_STRIPE_LEVEL_BALANCED\", or \"FILE_STRIPE_LEVEL_MAX\"" - } -} - -variable "directory_stripe" { - description = "The parallelstore stripe level for directories." - type = string - default = null - validation { - condition = var.directory_stripe == null ? true : contains([ - "DIRECTORY_STRIPE_LEVEL_UNSPECIFIED", - "DIRECTORY_STRIPE_LEVEL_MIN", - "DIRECTORY_STRIPE_LEVEL_BALANCED", - "DIRECTORY_STRIPE_LEVEL_MAX", - ], var.directory_stripe) - error_message = "var.directory_stripe must be set to \"DIRECTORY_STRIPE_LEVEL_UNSPECIFIED\", \"DIRECTORY_STRIPE_LEVEL_MIN\", \"DIRECTORY_STRIPE_LEVEL_BALANCED\", or \"DIRECTORY_STRIPE_LEVEL_MAX\"" - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/versions.tf b/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/versions.tf deleted file mode 100644 index 174b5281e4..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/parallelstore/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.13.0" - } - - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/README.md b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/README.md deleted file mode 100644 index 47cf1518a1..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/README.md +++ /dev/null @@ -1,192 +0,0 @@ -## Description - -This module defines a file-system that already exists (i.e. it does not create -a new file system) in a way that can be shared with other modules. This allows -a compute VM to mount a filesystem that is not part of the current deployment -group. - -The pre-existing network storage can be referenced in the same way as any Cluster -Toolkit supported file-system such as [filestore](../filestore/README.md). - -For more information on network storage options in the Cluster Toolkit, see -the extended [Network Storage documentation](../../../docs/network_storage.md). - -### Example - -```yaml -- id: homefs - source: modules/file-system/pre-existing-network-storage - settings: - server_ip: ## Set server IP here ## - remote_mount: nfsshare - local_mount: /home - fs_type: nfs -``` - -This creates a pre-existing-network-storage module in terraform at the -provided IP in `server_ip` of type nfs that will be mounted at `/home`. Note -that the `server_ip` must be known before deployment. - -The following is an example of using `pre-existing-network-storage` with a GCS -bucket: - -```yaml -- id: data-bucket - source: modules/file-system/pre-existing-network-storage - settings: - remote_mount: my-bucket-name - local_mount: /data - fs_type: gcsfuse - mount_options: defaults,_netdev,implicit_dirs -``` - -The `implicit_dirs` mount option allows object paths to be treated as if they -were directories. This is important when working with files that were created by -another source, but there may have performance impacts. The `_netdev` mount option -denotes that the storage device requires network access. - -The following is an example of using `pre-existing-network-storage` with the `lustre` -filesystem: - -```yaml -- id: lustrefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: lustre - server_ip: 192.168.227.11@tcp - local_mount: /scratch - remote_mount: /exacloud -``` - -Note the use of the MGS NID (Network ID) in the `server_ip` field - in -particular, note the `@tcp` suffix. - -The following is an example of using `pre-existing-network-storage` with the -`managed_lustre` filesystem: - -```yaml -- id: lustrefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: managed_lustre - server_ip: 192.168.227.11@tcp - local_mount: /scratch - remote_mount: /mg_lustre -``` - -This is similar to the `lustre` filesystem, with the exception that it connects -with a managed Lustre instance hosted by GCP. Currently only Rocky 8 and -Ubuntu 20.04 and Ubuntu 22.04 are supported. - -The following is an example of using `pre-existing-network-storage` with the `daos` -filesystem. In order to use existing `parallelstore` instance, `fs_type` needs to be -explicitly mentioned in blueprint. The `remote_mount` option refers to `access_points` -for `parallelstore` instance. - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 -``` - -Parallelstore supports additional options for its mountpoints under `parallelstore_options` setting. -Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 - parallelstore_options: - daos_agent_config: | - credential_config: - cache_expiration: 1m -``` - -Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 - parallelstore_options: - dfuse_environment: - D_LOG_FILE: /tmp/client.log - D_APPEND_PID_TO_LOG: 1 - D_LOG_MASK: debug -``` - -### Mounting - -For the `fs_type` listed below, this module will provide `client_install_runner` -and `mount_runner` outputs. These can be used to create a startup script to -mount the network storage system. - -Supported `fs_type`: - -- nfs -- lustre -- managed_lustre -- gcsfuse -- daos - -[scripts/mount.sh](./scripts/mount.sh) is used as the contents of -`mount_runner`. This script will update `/etc/fstab` and mount the network -storage. This script will fail if the specified `local_mount` is already being -used by another entry in `/etc/fstab`. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. - -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## License - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [fs\_type](#input\_fs\_type) | Type of file system to be mounted (e.g., nfs, lustre) | `string` | `"nfs"` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | -| [managed\_lustre\_options](#input\_managed\_lustre\_options) | Managed Lustre specific options:
gke\_support\_enabled (bool, default = false)
Note: gke\_support\_enabled does not work with Slurm, the Slurm image must be built with
the correct compatibility. | 
object({
    gke_support_enabled = optional(bool, false)
  })
| `{}` | no | -| [mount\_options](#input\_mount\_options) | Options describing various aspects of the file system. Consider adding setting to 'defaults,\_netdev,implicit\_dirs' when using gcsfuse. | `string` | `"defaults,_netdev"` | no | -| [parallelstore\_options](#input\_parallelstore\_options) | Parallelstore specific options | 
object({
    daos_agent_config = optional(string, "")
    dfuse_environment = optional(map(string), {})
  })
| `{}` | no | -| [remote\_mount](#input\_remote\_mount) | Remote FS name or export. This is the exported directory for nfs, fs name for lustre, and bucket name (without gs://) for gcsfuse. | `string` | n/a | yes | -| [server\_ip](#input\_server\_ip) | The device name as supplied to fs-tab, excluding remote fs-name(for nfs, that is the server IP, for lustre [:]). This can be omitted for gcsfuse. | `string` | `""` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use file system. | -| [mount\_runner](#output\_mount\_runner) | Runner that mounts the file system. | -| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | - diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf deleted file mode 100644 index 203b6dfdac..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a remote network storage to be mounted by fs-tab." - value = { - server_ip = var.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = var.mount_options - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } -} - -locals { - # Update remote mount to include a slash if the fs_type requires one to exist - remote_mount_with_slash = length(regexall("^/.*", var.remote_mount)) > 0 ? ( - var.remote_mount - ) : format("/%s", var.remote_mount) - remote_mount = contains(local.mount_vanilla_supported_fstype, local.fs_type) ? ( - local.remote_mount_with_slash - ) : var.remote_mount - - ml_gke_support_enabled = coalesce(try(var.managed_lustre_options.gke_support_enabled, false), false) - - # Collapse fs_type lustre and managed lustre for most uses, only needs to be - # different for client installation - fs_type = strcontains(var.fs_type, "lustre") ? "lustre" : var.fs_type - - # Client Install - ddn_lustre_client_install_script = templatefile( - "${path.module}/templates/ddn_exascaler_luster_client_install.tftpl", - { - server_ip = split("@", var.server_ip)[0] - remote_mount = local.remote_mount - local_mount = var.local_mount - } - ) - managed_lustre_client_install_script = file("${path.module}/scripts/install-managed-lustre-client.sh") - nfs_client_install_script = file("${path.module}/scripts/install-nfs-client.sh") - gcs_fuse_install_script = file("${path.module}/scripts/install-gcs-fuse.sh") - daos_client_install_script = file("${path.module}/scripts/install-daos-client.sh") - - install_scripts = { - "lustre" = local.ddn_lustre_client_install_script - "managed_lustre" = local.managed_lustre_client_install_script - "nfs" = local.nfs_client_install_script - "gcsfuse" = local.gcs_fuse_install_script - "daos" = local.daos_client_install_script - } - - client_install_runner = { - "type" = "shell" - "content" = lookup(local.install_scripts, var.fs_type, "echo 'skipping: client_install_runner not yet supported for ${var.fs_type}'") - "destination" = "install_filesystem_client${replace(var.local_mount, "/", "_")}.sh" - "args" = local.ml_gke_support_enabled ? "1" : "" - } - - mount_vanilla_supported_fstype = ["lustre", "nfs"] - mount_runner_vanilla = { - "type" = "shell" - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"${var.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" - "content" = ( - contains(local.mount_vanilla_supported_fstype, local.fs_type) ? - file("${path.module}/scripts/mount.sh") : - "echo 'skipping: mount_runner not yet supported for ${var.fs_type}'" - ) - } - gcsbucket = trimprefix(var.remote_mount, "gs://") - mount_runner_gcsfuse = { - "type" = "shell" - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"not-used\" \"${local.gcsbucket}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" - "content" = file("${path.module}/scripts/mount.sh") - } - - mount_runner_daos = { - "type" = "shell" - "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { - access_points = var.remote_mount - daos_agent_config = var.parallelstore_options.daos_agent_config - dfuse_environment = var.parallelstore_options.dfuse_environment - local_mount = var.local_mount - # avoid passing "--" as mount option to dfuse - mount_options = length(var.mount_options) == 0 ? "" : join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) - }) - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - } - - mount_scripts = { - "lustre" = local.mount_runner_vanilla - "nfs" = local.mount_runner_vanilla - "gcsfuse" = local.mount_runner_gcsfuse - "daos" = local.mount_runner_daos - } - - mount_runner = lookup(local.mount_scripts, local.fs_type, local.mount_runner_vanilla) -} - -output "client_install_runner" { - description = "Runner that performs client installation needed to use file system." - value = local.client_install_runner -} - -output "mount_runner" { - description = "Runner that mounts the file system." - value = local.mount_runner -} diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh deleted file mode 100644 index e96eadb56a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || - { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 -fi - -if [ -x /bin/daos ]; then - echo "DAOS already installed" - daos version -else - # Install the DAOS client library - # The following commands should be executed on each client vm. - ## For Rocky linux 8 / RedHat 8. - if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then - # 1) Add the Parallelstore package repository - cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF - [parallelstore-v2-6-el${OS_VERSION_MAJOR}] - name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 - baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - - ## TODO: Remove disable automatic update script after issue is fixed. - if [ -x /usr/bin/google_disable_automatic_updates ]; then - /usr/bin/google_disable_automatic_updates - fi - dnf clean all - dnf makecache - - # 2) Install daos-client - dnf install -y epel-release # needed for capstone - dnf install -y daos-client - - # 3) Upgrade libfabric - dnf upgrade -y libfabric - - # For Ubuntu 22.04 and debian 12, - elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then - # shellcheck disable=SC2034 - DEBIAN_FRONTEND=noninteractive - - # 1) Add the Parallelstore package repository - curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg - echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list - - apt-get update - - # 2) Install daos-client - apt-get install -y daos-client - - # 3) Create daos_agent.service (comes pre-installed with RedHat) - if ! getent passwd daos_agent >/dev/null 2>&1; then - useradd daos_agent - fi - cat >/etc/systemd/system/daos_agent.service <<-EOF - [Unit] - Description=DAOS Agent - StartLimitIntervalSec=60 - Wants=network-online.target - After=network-online.target - - [Service] - Type=notify - User=daos_agent - Group=daos_agent - RuntimeDirectory=daos_agent - RuntimeDirectoryMode=0755 - ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml - StandardOutput=journal - StandardError=journal - Restart=always - RestartSec=10 - LimitMEMLOCK=infinity - LimitCORE=infinity - StartLimitBurst=5 - - [Install] - WantedBy=multi-user.target - EOF - else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - fi -fi - -exit 0 diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh deleted file mode 100644 index f8a990260b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -if [ ! "$(which gcsfuse)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then - tee /etc/yum.repos.d/gcsfuse.repo >/dev/null <>/etc/modprobe.d/lnet.conf - fi -fi - -if grep -q lustre /proc/filesystems; then - echo "Skipping managed lustre client install as it is already supported" - exit 0 -fi - -# Get distro information -. /etc/os-release -DIST="NA" -if [[ $NAME == *"Ubuntu"* ]]; then - if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then - DIST="Ubuntu" - fi -elif [[ $NAME == *"Rocky"* ]]; then - if [[ $VERSION_ID == "8"* ]]; then - DIST="Rocky" - fi -fi - -if [[ ${DIST} == "Ubuntu" ]]; then - KEY_LOC=/etc/apt/keyrings - KEY_NAME=gcp-ar-repo.gpg - # Download new repo key - mkdir -p "${KEY_LOC}" - wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null - - # Set up apt repo - echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list - - # Install modules - apt update - apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) -elif [[ ${DIST} == "Rocky" ]]; then - # Set up yum repo - touch /etc/yum.repos.d/artifact-registry.repo - tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF - [lustre-client-rocky-8] - name=lustre-client-rocky-8 - baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - # Install modules - yum makecache - yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client -fi - -if [[ $DIST != "NA" ]]; then - # Load the new lustre client module - modprobe lustre -fi diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl b/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl deleted file mode 100644 index f5f0291e85..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh - -# Copyright 2022 DataDirect Networks -# Modifications Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Prior Art: https://github.com/DDNStorage/exascaler-cloud-terraform/blob/78deadbb2c1fa7e4603cf9605b0f7d1782117954/gcp/templates/client-script.tftpl - -# install new EXAScaler Cloud clients: -# all instances must be in the same zone -# and connected to the same network and subnet -# to set up EXAScaler Cloud filesystem on a new client instance, -# run the following commands on the client with root privileges: -set -e -if [[ ! -z $(cat /proc/filesystems | grep lustre) ]]; then - echo "Skipping lustre client install as it is already supported" - exit 0 -fi - -cat >/etc/esc-client.conf< $daos_config </etc/systemd/system/"$${service_name}" < -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string
    count = number
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [guest\_accelerator](#output\_guest\_accelerator) | Sanitized list of the type and count of accelerator cards attached to the instance. | -| [machine\_type\_guest\_accelerator](#output\_machine\_type\_guest\_accelerator) | List of the type and count of accelerator cards attached to the specified machine type. | - diff --git a/deletion-test/build_script/modules/embedded/modules/internal/gpu-definition/main.tf b/deletion-test/build_script/modules/embedded/modules/internal/gpu-definition/main.tf deleted file mode 100644 index f0861cddc9..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/gpu-definition/main.tf +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "machine_type" { - description = "Machine type to use for the instance creation" - type = string -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string - count = number - gpu_driver_installation_config = optional(object({ - gpu_driver_version = string - }), { gpu_driver_version = "DEFAULT" }) - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = string - max_shared_clients_per_gpu = number - })) - })) - default = [] - nullable = false -} - -locals { - # example state; terraform will ignore diffs if last element of URL matches - # guest_accelerator = [ - # { - # count = 1 - # type = "https://www.googleapis.com/compute/beta/projects/PROJECT/zones/ZONE/acceleratorTypes/nvidia-tesla-a100" - # }, - # ] - accelerator_machines = { - "a2-highgpu-1g" = { type = "nvidia-tesla-a100", count = 1 }, - "a2-highgpu-2g" = { type = "nvidia-tesla-a100", count = 2 }, - "a2-highgpu-4g" = { type = "nvidia-tesla-a100", count = 4 }, - "a2-highgpu-8g" = { type = "nvidia-tesla-a100", count = 8 }, - "a2-megagpu-16g" = { type = "nvidia-tesla-a100", count = 16 }, - "a2-ultragpu-1g" = { type = "nvidia-a100-80gb", count = 1 }, - "a2-ultragpu-2g" = { type = "nvidia-a100-80gb", count = 2 }, - "a2-ultragpu-4g" = { type = "nvidia-a100-80gb", count = 4 }, - "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, - "a3-highgpu-1g" = { type = "nvidia-h100-80gb", count = 1 }, - "a3-highgpu-2g" = { type = "nvidia-h100-80gb", count = 2 }, - "a3-highgpu-4g" = { type = "nvidia-h100-80gb", count = 4 }, - "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, - "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, - "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, - "a4-highgpu-8g-lowmem" = { type = "nvidia-b200", count = 8 }, - "a4-highgpu-8g" = { type = "nvidia-b200", count = 8 }, - "a4x-highgpu-4g" = { type = "nvidia-gb200", count = 4 }, - "a4x-highgpu-4g-nolssd" = { type = "nvidia-gb200", count = 4 }, - "g2-standard-4" = { type = "nvidia-l4", count = 1 }, - "g2-standard-8" = { type = "nvidia-l4", count = 1 }, - "g2-standard-12" = { type = "nvidia-l4", count = 1 }, - "g2-standard-16" = { type = "nvidia-l4", count = 1 }, - "g2-standard-24" = { type = "nvidia-l4", count = 2 }, - "g2-standard-32" = { type = "nvidia-l4", count = 1 }, - "g2-standard-48" = { type = "nvidia-l4", count = 4 }, - "g2-standard-96" = { type = "nvidia-l4", count = 8 }, - } - generated_guest_accelerator = try([local.accelerator_machines[var.machine_type]], []) - - # Select in priority order: - # (1) var.guest_accelerator if not empty - # (2) local.generated_guest_accelerator if not empty - # (3) default to empty list if both are empty - guest_accelerator = try(coalescelist(var.guest_accelerator, local.generated_guest_accelerator), []) -} - -output "guest_accelerator" { - description = "Sanitized list of the type and count of accelerator cards attached to the instance." - value = local.guest_accelerator -} - -output "machine_type_guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the specified machine type." - value = local.generated_guest_accelerator -} - -terraform { - required_version = ">= 1.3" -} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/README.md b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/README.md deleted file mode 100644 index 21746fe0d8..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/README.md +++ /dev/null @@ -1,30 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [disk\_type](#input\_disk\_type) | The disk type to validate. | `string` | n/a | yes | -| [machine\_type](#input\_machine\_type) | The machine type to validate. | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/main.tf b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/main.tf deleted file mode 100644 index d89d7edfec..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/main.tf +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -check "disk_type_c4_compatibility" { - assert { - condition = !(can(regex("^c4-", var.machine_type)) && var.disk_type == "pd-ssd") - error_message = "The C4 machine series does not support pd-ssd. Please use hyperdisk-balanced or another compatible disk type." - } -} - - -check "disk_type_c2_compatibility" { - assert { - condition = !(can(regex("^c2-", var.machine_type)) && can(regex("hyperdisk", var.disk_type))) - error_message = "The C2 machine series does not support Hyperdisk as a boot disk. Please use a compatible disk type like pd-ssd, pd-standard, or pd-balanced." - } -} - - -check "disk_type_pd_extreme_compatibility" { - assert { - condition = var.disk_type != "pd-extreme" || can(regex("^(m1-|m2-|m3-|n2-|n2d-)", var.machine_type)) - error_message = "pd-extreme disks are only supported for M1, M2, M3, N2, and N2D machine series." - } -} - - -check "disk_type_hyperdisk_extreme_compatibility" { - assert { - condition = var.disk_type != "hyperdisk-extreme" || can(regex("^(c3-|m1-|m3-|n2-)", var.machine_type)) - error_message = "hyperdisk-extreme disks are only supported for C3, M1, M3, and N2 machine series." - } -} - - -check "disk_type_hyperdisk_throughput_compatibility" { - assert { - condition = var.disk_type != "hyperdisk-throughput" || can(regex("^(c3-|c3d-|n4-|n2-|n2d-|n1-|t2d-|m1-)", var.machine_type)) - error_message = "hyperdisk-throughput disks are only supported for C3, C3D, N4, N2, N2D, N1, T2D, and M1 machine series." - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/variables.tf b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/variables.tf deleted file mode 100644 index 23478051b3..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/variables.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "machine_type" { - type = string - description = "The machine type to validate." -} - -variable "disk_type" { - type = string - description = "The disk type to validate." -} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/versions.tf b/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/versions.tf deleted file mode 100644 index 4702005614..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/instance_validations/versions.tf +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 0.15.0" -} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/README.md b/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/README.md deleted file mode 100644 index 8aa9270a0a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/README.md +++ /dev/null @@ -1,54 +0,0 @@ - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | -| [google-beta](#requirement\_google-beta) | >= 6.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google-beta](#provider\_google-beta) | >= 6.0.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_network_attachment.self](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_network_attachment) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [connection\_preference](#input\_connection\_preference) | The connection preference of service attachment. | `string` | `"ACCEPT_AUTOMATIC"` | no | -| [name](#input\_name) | Name of the resource. Provided by the client when the resource is created | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The ID of the project in which the resource belongs. | `string` | n/a | yes | -| [region](#input\_region) | Region where the network attachment resides | `string` | n/a | yes | -| [subnetwork\_self\_links](#input\_subnetwork\_self\_links) | An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment. | `list(string)` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [self\_link](#output\_self\_link) | Server-defined URL for the resource. | - diff --git a/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/main.tf b/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/main.tf deleted file mode 100644 index bbbece7085..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -variable "connection_preference" { - type = string - description = "The connection preference of service attachment." - default = "ACCEPT_AUTOMATIC" -} - -variable "subnetwork_self_links" { - type = list(string) - description = " An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment." -} - -variable "name" { - type = string - description = "Name of the resource. Provided by the client when the resource is created" -} - -variable "project_id" { - type = string - description = "The ID of the project in which the resource belongs." -} - -variable "region" { - type = string - description = "Region where the network attachment resides" -} - - -resource "google_compute_network_attachment" "self" { - provider = google-beta - - project = var.project_id - region = var.region - name = var.name - connection_preference = var.connection_preference - subnetworks = var.subnetwork_self_links -} - - -output "self_link" { - value = google_compute_network_attachment.self.self_link - description = "Server-defined URL for the resource." -} - -terraform { - required_version = ">= 0.15.0" - - required_providers { - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.0.0" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/metadata.yaml deleted file mode 100644 index e80fc96b9c..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/network-attachment/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/README.md b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/README.md deleted file mode 100644 index 610d82c1b9..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/README.md +++ /dev/null @@ -1,85 +0,0 @@ -## Description - -This is an internal helper module designed to encapsulate and centralize all hardware-specific logic for Google Cloud TPUs. It is intended to be called by parent modules like `gke-node-pool` to determine if a node pool is TPU-based and to retrieve its specific attributes. - -This module's primary responsibilities are: - -* Reliably detect if a node pool is for TPUs by checking its `placement_policy`. -* Determine the correct GKE `tpu-accelerator` label based on the machine type family. -* Determine the `number of chips per node` based on the specific machine type. -* Generate the standard **Kubernetes taint** that should be applied to TPU nodes. - -This follows the same design pattern as the `gpu-definition` internal module, promoting a clean separation of concerns within the gke-node-pool module. - -## Usage - -This module is not intended for direct use in a blueprint. It should be called from a parent module like `gke-node-pool`. - -```yaml -module "tpu" { - source = "../../internal/tpu-definition" - - # Pass the parent module's variables to this module - machine_type = var.machine_type - placement_policy = var.placement_policy -} - -# Example of consuming the module's outputs in the parent module -locals { - # The tpu_taint is then used in the node_config's dynamic "taint" block - tpu_taint = module.tpu.tpu_taint -} -``` - -## License - - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [machine\_type](#input\_machine\_type) | The machine type of the node pool. | `string` | n/a | yes | -| [placement\_policy](#input\_placement\_policy) | The placement policy for the node pool. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [is\_tpu](#output\_is\_tpu) | Boolean value indicating if the node pool is for TPUs. | -| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | -| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | -| [tpu\_taint](#output\_tpu\_taint) | A list containing the standard TPU taint object if the node pool is for TPUs. | -| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | - diff --git a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/main.tf b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/main.tf deleted file mode 100644 index c8ee417d71..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/main.tf +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # Determine if this is a TPU node pool by checking if the machine_type exists in our authoritative map of TPU machine types. - is_tpu = contains(keys(local.tpu_chip_count_map), var.machine_type) - - tpu_taint = local.is_tpu ? [{ - key = "google.com/tpu" - value = "present" - effect = "NO_SCHEDULE" - }] : [] - - # Map of machine prefixes to GKE accelerator labels. - tpu_accelerator_map = { - "ct4p" = "tpu-v4-podslice" # TPU v4 - "ct5lp" = "tpu-v5-lite-podslice" # TPU v5e - "ct5p" = "tpu-v5p-slice" # TPU v5p - "ct6e" = "tpu-v6e-slice" # TPU v6e - "tpu7x" = "tpu7x" # TPU v7x - } - - # Map specific GCE machine types to the number of TPU chips per node (VM). - # The machine-type map must be updated to reflect new TPU releases with reference to public documentation: https://docs.cloud.google.com/tpu/docs/intro-to-tpu - tpu_chip_count_map = { - # v4 - ct4p - "ct4p-hightpu-4t" = 4 - - # v5e - ct5lp - "ct5lp-hightpu-1t" = 1 - "ct5lp-hightpu-4t" = 4 - "ct5lp-hightpu-8t" = 8 - - # v5p - ct5p - "ct5p-hightpu-1t" = 1 - "ct5p-hightpu-2t" = 2 - "ct5p-hightpu-4t" = 4 - - # v6e - ct6e - "ct6e-standard-1t" = 1 - "ct6e-standard-4t" = 4 - "ct6e-standard-8t" = 8 - - # v7x - tpu7x - "tpu7x-standard-4t" = 4 - } - - # Robustly extract the machine family prefix (e.g., "ct6e"). - tpu_machine_family = local.is_tpu ? element(split("-", var.machine_type), 0) : "" - tpu_accelerator_type = local.is_tpu ? lookup(local.tpu_accelerator_map, local.tpu_machine_family, null) : null - tpu_chips_per_node = local.is_tpu ? lookup(local.tpu_chip_count_map, var.machine_type, null) : null -} - -terraform { - required_version = ">= 1.3" -} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/outputs.tf b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/outputs.tf deleted file mode 100644 index fa3c21fa34..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/outputs.tf +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "is_tpu" { - description = "Boolean value indicating if the node pool is for TPUs." - value = local.is_tpu -} - -output "tpu_accelerator_type" { - description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." - value = local.tpu_accelerator_type -} - -output "tpu_topology" { - description = "The topology of the TPU slice (e.g., '4x4')." - value = local.is_tpu ? var.placement_policy.tpu_topology : null -} - -output "tpu_chips_per_node" { - description = "The number of TPU chips on each node in the pool." - value = local.tpu_chips_per_node -} - -output "tpu_taint" { - description = "A list containing the standard TPU taint object if the node pool is for TPUs." - value = local.tpu_taint -} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/variables.tf b/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/variables.tf deleted file mode 100644 index 254488c02d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/tpu-definition/variables.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "machine_type" { - description = "The machine type of the node pool." - type = string -} - -variable "placement_policy" { - description = "The placement policy for the node pool." - type = object({ - type = string - name = optional(string) - tpu_topology = optional(string) - }) -} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/README.md b/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/README.md deleted file mode 100644 index aefac9d187..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/README.md +++ /dev/null @@ -1,56 +0,0 @@ - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_network_peering.peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [export\_custom\_routes](#input\_export\_custom\_routes) | (Optional) Whether to export the custom routes to the peer network. Defaults to false. | `bool` | `null` | no | -| [import\_custom\_routes](#input\_import\_custom\_routes) | (Optional) Whether to import the custom routes from the peer network. Defaults to false. | `bool` | `null` | no | -| [import\_subnet\_routes\_with\_public\_ip](#input\_import\_subnet\_routes\_with\_public\_ip) | (Optional) Whether subnet routes with public IP range are imported. | `bool` | `null` | no | -| [name](#input\_name) | Name of the peering. | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | The primary network of the peering. | `string` | n/a | yes | -| [peer\_network\_self\_link](#input\_peer\_network\_self\_link) | The peer network in the peering. The peer network may belong to a different project. | `string` | n/a | yes | -| [stack\_type](#input\_stack\_type) | (Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [peering\_name](#output\_peering\_name) | Name of the peering. | - diff --git a/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/main.tf b/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/main.tf deleted file mode 100644 index 386fa9377b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/main.tf +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "name" { - type = string - description = "Name of the peering." -} - -variable "network_self_link" { - type = string - description = "The primary network of the peering." -} - -variable "peer_network_self_link" { - type = string - description = "The peer network in the peering. The peer network may belong to a different project." -} - -variable "export_custom_routes" { - type = bool - description = "(Optional) Whether to export the custom routes to the peer network. Defaults to false." - default = null -} - -variable "import_custom_routes" { - type = bool - description = "(Optional) Whether to import the custom routes from the peer network. Defaults to false." - default = null -} - -variable "import_subnet_routes_with_public_ip" { - type = bool - description = "(Optional) Whether subnet routes with public IP range are imported. " - default = null -} - -variable "stack_type" { - type = string - description = "(Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. " - default = null -} - -resource "google_compute_network_peering" "peering" { - name = var.name - network = var.network_self_link - peer_network = var.peer_network_self_link - export_custom_routes = var.export_custom_routes - import_custom_routes = var.import_custom_routes - import_subnet_routes_with_public_ip = var.import_subnet_routes_with_public_ip - stack_type = var.stack_type -} - -output "peering_name" { - value = google_compute_network_peering.peering.name - description = "Name of the peering." -} - -terraform { - required_version = ">= 0.15.0" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/metadata.yaml deleted file mode 100644 index e80fc96b9c..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/internal/vpc_peering/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/README.md b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/README.md deleted file mode 100644 index d7054eb725..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/README.md +++ /dev/null @@ -1,244 +0,0 @@ -## Description - -This module simplifies the following functionality: - -* Applying Kubernetes manifests to GKE clusters: It provides flexible options for specifying manifests, allowing you to either directly embed them as strings content or reference them from URLs, files, templates, or entire .yaml and .tftpl files in directories. -* Deploying commonly used infrastructure like [Kueue](https://kueue.sigs.k8s.io/docs/) or [Jobset](https://jobset.sigs.k8s.io/docs/). - -> Note: Kueue can work with a variety of frameworks out of the box, find them [here](https://kueue.sigs.k8s.io/docs/tasks/run/) - -### Explanation - -* **Manifest:** - * **Raw String:** Specify manifests directly within the module configuration using the `content: manifest_body` format. - * **File/Template/Directory Reference:** Set `source` to the path to: - * A single URL to a manifest file. Ex.: `https://github.com/.../myrepo/manifest.yaml`. - - > **Note:** Applying from a URL has important limitations. Please review the [Considerations & Callouts for Applying from URLs](#applying-manifests-from-urls-considerations--callouts) section below. - * A single local YAML manifest file (`.yaml`). Ex.: `./manifest.yaml`. - * A template file (`.tftpl`) to generate a manifest. Ex.: `./template.yaml.tftpl`. You can pass the variables to format the template file in `template_vars`. - * A directory containing multiple YAML or template files. Ex: `./manifests/`. You can pass the variables to format the template files in `template_vars`. - -#### Manifest Example - -```yaml -- id: existing-gke-cluster - source: modules/scheduler/pre-existing-gke-cluster - settings: - project_id: $(vars.project_id) - cluster_name: my-gke-cluster - region: us-central1 - -- id: kubectl-apply - source: modules/management/kubectl-apply - use: [existing-gke-cluster] - settings: - - content: | - apiVersion: v1 - kind: Namespace - metadata: - name: my-namespace - - source: "https://github.com/kubernetes-sigs/jobset/releases/download/v0.6.0/manifests.yaml" - - source: $(ghpc_stage("manifests/configmap1.yaml")) - - source: $(ghpc_stage("manifests/configmap2.yaml.tftpl")) - template_vars: {name: "dev-config", public: "false"} - - source: $(ghpc_stage("manifests"))/ - template_vars: {name: "dev-config", public: "false"} -``` - -#### Pre-build infrastructure Example - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml")) - jobset: - install: true -``` - -The `config_path` field in `kueue` installation accepts a template file, too. You will need to provide variables for the template using `config_template_vars` field. - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) - config_template_vars: {name: "dev-config", public: "false"} - jobset: - install: true -``` - -You can specify a particular kueue version that you would like to use using the `version` flag. By default, we recommend customers to [use v0.10.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L68). You can find the list of supported kueue versions [here](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L18). - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - version: v0.10.0 - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) - config_template_vars: {name: "dev-config", public: "false"} - jobset: - install: true -``` - -> **_NOTE:_** -> -> The `project_id` and `region` settings would be inferred from the deployment variables of the same name, but they are included here for clarity. -> -> Terraform may apply resources in parallel, leading to potential dependency issues. If a resource's dependencies aren't ready, it will be applied again up to 15 times. - -## Callouts - -### Applying Manifests from URLs: Considerations & Callouts - -While this module supports applying manifests directly from remote `http://` or `https://` URLs, this method introduces complexities not present when using local files. For production environments, we recommend sourcing manifests from local paths or a version-controlled Git repository. Moreover, this method will be deprecated soon. Hence we recommend to use other methods to source manifests. - -If you choose to use the URL method, be aware of the following potential issues and their solutions. - -#### **1. Apply Order and Race Conditions** - -The module applies manifests from the `apply_manifests` list in parallel. This can create a **race condition** if one manifest depends on another. The most common example is applying a manifest with custom resources (like a `ClusterQueue`) at the same time as the manifest that defines it (the `CustomResourceDefinition` or CRD). - -There is **no guarantee** that the CRD will be applied before the resource that uses it. This can lead to non-deterministic deployment failures with errors like: - -```Error: resource [kueue.x-k8s.io/v1beta1/ClusterQueue] isn't valid for cluster``` - -##### **Recommended Workaround: Two-Stage Apply** - -To ensure a reliable deployment, you must manually enforce the correct order of operations. - -1. **Initial Deployment:** In your blueprint, include **only** the manifest(s) containing the `CustomResourceDefinition` (CRD) resources in the `apply_manifests` list. - - *Example `settings` for the first run:* - - ```yaml - settings: - apply_manifests: - # This manifest contains the CRDs for Kueue - - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" - server_side_apply: true - ``` - -2. **Run the deployment** (`gcluster deploy` or `terraform apply`). - -3. **Second Deployment:** Once the first apply is successful, **add** the manifests containing your custom resources (like `ClusterQueue`, `LocalQueue`) to the list. - - *Example `settings` for the second run:* - - ```yaml - settings: - apply_manifests: - # The CRD manifest is still present - - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" - server_side_apply: true - - # Now, add your configuration manifest - - source: "https://gist.githubusercontent.com/YourUser/..." # Your configuration URL - server_side_apply: true - ``` - -4. **Run the deployment command again.** Since the CRDs are now guaranteed to exist in the cluster, this second apply will succeed reliably. - -#### **2. Large Manifests (CRDs)** - -* **Issue:** Applying very large manifests can fail with a `metadata.annotations: Too long` error. -* **Solution:** Enable Server-Side Apply by setting `server_side_apply: true` for the manifest entry. - -#### **3. Conflicts on Re-application** - -* **Issue:** Re-running a deployment after a partial failure can cause server-side apply field manager `conflicts`. -* **Solution:** Forcibly take ownership of the resource fields by setting `force_conflicts: true`. - -#### **4. Terraform Template Files (`.tftpl`)** - -* **Limitation:** This module **cannot** render a template file (`.tftpl`) when sourced from a remote URL. -* **Workaround:** You must render the template into a pure YAML file locally, host that rendered file at a URL, and provide the URL of the rendered file in your blueprint. - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 7.2 | -| [helm](#requirement\_helm) | ~> 2.17 | -| [http](#requirement\_http) | ~> 3.0 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 7.2 | -| [http](#provider\_http) | ~> 3.0 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [configure\_kueue](#module\_configure\_kueue) | ./kubectl | n/a | -| [install\_gib](#module\_install\_gib) | ./kubectl | n/a | -| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | -| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | -| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | -| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | -| [kubectl\_apply\_manifests](#module\_kubectl\_apply\_manifests) | ./kubectl | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.gib_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.initial_gib_version](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.jobset_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.kueue_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | -| [http_http.manifest_from_url](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [apply\_manifests](#input\_apply\_manifests) | A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md).
NOTE: The `enable` input acts as a FF to apply a manifest or not. By default it is always set to `true`. | 
list(object({
    enable            = optional(bool, true)
    content           = optional(string, null)
    source            = optional(string, null)
    template_vars     = optional(map(any), null)
    server_side_apply = optional(bool, false)
    wait_for_rollout  = optional(bool, true)
  }))
| `[]` | no | -| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | -| [gib](#input\_gib) | Install the NCCL gIB plugin | 
object({
    install = bool
    path    = string
    template_vars = object({
      image   = optional(string, "us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib")
      version = string
      node_affinity = optional(any, {
        requiredDuringSchedulingIgnoredDuringExecution = {
          nodeSelectorTerms = [{
            matchExpressions = [{
              key      = "cloud.google.com/gke-gpu",
              operator = "In",
              values   = ["true"]
            }]
          }]
        }
      })
      accelerator_count = number
      max_unavailable   = optional(string, "50%")
    })
  })
| 
{
  "install": false,
  "path": "",
  "template_vars": {
    "accelerator_count": 0,
    "version": ""
  }
}
| no | -| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | -| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "0.10.1")
  })
| `{}` | no | -| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "0.13.3")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | -| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | -| [target\_architecture](#input\_target\_architecture) | The target architecture for the GKE nodes and gIB plugin (e.g., 'x86\_64' or 'arm64'). | `string` | `"x86_64"` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/README.md b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/README.md deleted file mode 100644 index 1957899617..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [helm](#provider\_helm) | ~> 2.17 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | -| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | -| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | -| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | -| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | -| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | -| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | -| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | -| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | -| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | -| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | -| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | -| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | -| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | -| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | -| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | -| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | -| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | -| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | -| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | -| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | -| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | -| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | -| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | -| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | -| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | -| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | -| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | -| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | -| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf deleted file mode 100644 index 8cc09bd3e2..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "helm_release" "apply_chart" { - # Required Identification - name = var.release_name - chart = var.chart_name - - # Chart Source & Version - repository = var.chart_repository - version = var.chart_version - devel = var.devel - - # Target Namespace - namespace = var.namespace - create_namespace = var.create_namespace - - # Values Configuration - values = var.values_yaml - - dynamic "set" { - for_each = var.set_values - content { - name = set.value.name - value = set.value.value - type = set.value.type - } - } - - # Installation/Upgrade Behavior - description = var.description - atomic = var.atomic - cleanup_on_fail = var.cleanup_on_fail - dependency_update = var.dependency_update - disable_crd_hooks = var.disable_crd_hooks - disable_openapi_validation = var.disable_openapi_validation - disable_webhooks = var.disable_webhooks - force_update = var.force_update - lint = var.lint - max_history = var.max_history - recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI - render_subchart_notes = var.render_subchart_notes - reset_values = var.reset_values - reuse_values = var.reuse_values - skip_crds = var.skip_crds - timeout = var.timeout - wait = var.wait - wait_for_jobs = var.wait_for_jobs - - # Verification & Credentials - keyring = var.keyring - pass_credentials = var.pass_credentials - verify = var.verify - - # Post Rendering - dynamic "postrender" { - # Only include the block if var.postrender is not null - for_each = var.postrender == null ? [] : [var.postrender] - content { - binary_path = postrender.value.binary_path - } - } - - # Lifecycle block (optional - generally avoid complex lifecycle in generic modules) - # lifecycle { - # ignore_changes = [] - # } -} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf deleted file mode 100644 index 04e8e214fc..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -# --- Required --- -variable "release_name" { - description = "Name of the Helm release." - type = string -} - -variable "chart_name" { - description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." - type = string -} - -# --- Chart Location & Version --- -variable "chart_repository" { - description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." - type = string - default = null -} - -variable "chart_version" { - description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." - type = string - default = null -} - -variable "devel" { - description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." - type = bool - default = false -} - -# --- Namespace --- -variable "namespace" { - description = "Kubernetes namespace to install the Helm release into." - type = string - default = "default" -} - -variable "create_namespace" { - description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." - type = bool - default = true # Common convenience setting -} - -# --- Values Customization --- -variable "values_yaml" { - description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." - type = list(string) - default = [] -} - -variable "set_values" { - description = "List of objects defining values to set ('helm install --set')." - type = list(object({ - name = string # Path to the value (e.g., 'service.type', 'replicaCount') - value = string # The value to set - type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') - })) - default = [] -} - -# --- Installation/Upgrade Behavior --- -variable "description" { - description = "Set an optional description for the Helm release." - type = string - default = null -} - -variable "atomic" { - description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." - type = bool - default = false -} - -variable "wait" { - description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." - type = bool - default = true # Often a good default for dependencies -} - -variable "wait_for_jobs" { - description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." - type = bool - default = false # Helm CLI default is false -} - -variable "timeout" { - description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." - type = number - default = 300 # 5 minutes (Helm CLI default) -} - -variable "cleanup_on_fail" { - description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." - type = bool - default = false -} - -variable "dependency_update" { - description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." - type = bool - default = false -} - -variable "disable_crd_hooks" { - description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." - type = bool - default = false -} - -variable "disable_openapi_validation" { - description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." - type = bool - default = false -} - -variable "disable_webhooks" { - description = "Prevent hooks from running ('helm install --no-hooks')." - type = bool - default = false -} - -variable "force_update" { - description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." - type = bool - default = false -} - -variable "lint" { - description = "Run the helm chart linter during the plan ('helm lint')." - type = bool - default = false -} - -variable "max_history" { - description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." - type = number - default = null # Terraform provider defaults to Helm's default (usually 10) -} - -variable "recreate_pods" { - description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." - type = bool - default = false -} - -variable "render_subchart_notes" { - description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." - type = bool - default = false -} - -variable "reset_values" { - description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." - type = bool - default = false -} - -variable "reuse_values" { - description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." - type = bool - default = false # Helm CLI default is false -} - -variable "skip_crds" { - description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." - type = bool - default = false -} - -# --- Verification & Credentials --- -variable "keyring" { - description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." - type = string - default = null # Defaults to Helm's default keyring location -} - -variable "pass_credentials" { - description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." - type = bool - default = false -} - -variable "verify" { - description = "Verify the package before installing it ('helm install --verify')." - type = bool - default = false -} - -# --- Advanced Rendering --- -variable "postrender" { - description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." - type = object({ - binary_path = string # Path to the post-renderer executable - }) - default = null # Disabled by default -} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf deleted file mode 100644 index 09d912e2c9..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml deleted file mode 100644 index 92fc1bca22..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# For referencing the original jobset helm chart values, pull the latest jobset chart version -# `helm pull oci://registry.k8s.io/jobset/charts/jobset --version=0.10.1` (latest helm chart version) - -controller: - # It ensures the Jobset pod(s) can be scheduled on GKE clusters where the - # system node pool uses the default "gke-managed-components" taint. - tolerations: - - key: "components.gke.io/gke-managed-components" - operator: "Equal" - value: "true" - effect: "NoSchedule" diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/README.md b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/README.md deleted file mode 100644 index 691f4dc34a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/README.md +++ /dev/null @@ -1,55 +0,0 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | - -## Providers - -| Name | Version | -|------|---------| -| [kubectl](#provider\_kubectl) | >= 1.7.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubectl_manifest.apply_doc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_path_documents.templates](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | -| [kubectl_path_documents.yamls](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | -| [force\_conflicts](#input\_force\_conflicts) | The force\_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller) | `bool` | `false` | no | -| [server\_side\_apply](#input\_server\_side\_apply) | Allow using kubectl server-side apply method. | `bool` | `false` | no | -| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `null` | no | -| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | -| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf deleted file mode 100644 index acf1d3c908..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - yaml_separator = "\n---" - - # This locals block processes manifest inputs from one of four methods, - # evaluated in order of precedence using coalesce. - - # --- METHOD 1: Direct Content Input --- - # Used when manifest content is passed directly as a string. - content_yaml_body = var.content - - # Fallback for safe path checking in subsequent methods. - null_safe_source = coalesce(var.source_path, " ") - - # --- METHOD 2: Single Local YAML File --- - # Used when var.source_path points to a local .yaml file. - yaml_file = length(regexall("\\.yaml(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null - yaml_file_content = local.yaml_file != null ? file(local.yaml_file) : null - - # --- METHOD 3: Single Local Template File --- - # Used when var.source_path points to a local .tftpl file. - template_file = length(regexall("\\.tftpl(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null - template_file_content = local.template_file != null ? templatefile(local.template_file, var.template_vars) : null - - # --- CONSOLIDATE & PROCESS --- - # Coalesce finds the first non-null content from the methods above. - yaml_body = coalesce(local.content_yaml_body, local.yaml_file_content, local.template_file_content, " ") - # Ensure only valid YAML is processed - # It explicitly tests if the content can be decoded before including it. - yaml_body_docs = compact(flatten([ - for doc in split(local.yaml_separator, local.yaml_body) : [ - for content in [trimspace(doc)] : ( - # Use a temporary local variable and can() to test for successful YAML decoding. - # This handles malformed documents (like comment blocks) which cause yamldecode() to fail. - can(yamldecode(content)) && length(yamldecode(content)) > 0 ? content : null - ) - ] - ])) - - # --- METHOD 4: Directory of Files --- - # If no content was found via the methods above AND the source path looks like a directory, - # we assume this is the desired method. The data blocks below will handle it. - directory = length(local.yaml_body_docs) == 0 && endswith(local.null_safe_source, "/") ? abspath(var.source_path) : null - - # --- FINAL AGGREGATION --- - # Combine documents from single-source methods and directory-scan methods into one list. - docs_list = concat(try(local.yaml_body_docs, []), try(data.kubectl_path_documents.yamls[0].documents, []), try(data.kubectl_path_documents.templates[0].documents, [])) - docs_map = tomap({ - for index, doc in local.docs_list : index => doc - }) -} - -data "kubectl_path_documents" "yamls" { - count = local.directory != null ? 1 : 0 - pattern = "${local.directory}/*.yaml" -} - -data "kubectl_path_documents" "templates" { - count = local.directory != null ? 1 : 0 - pattern = "${local.directory}/*.tftpl" - vars = var.template_vars -} - -resource "kubectl_manifest" "apply_doc" { - for_each = local.docs_map - yaml_body = each.value - server_side_apply = var.server_side_apply - wait_for_rollout = var.wait_for_rollout - force_conflicts = var.force_conflicts - - lifecycle { - precondition { - condition = !var.force_conflicts || var.server_side_apply - error_message = "The 'force_conflicts' variable can only be set to true when 'server_side_apply' is also true." - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf deleted file mode 100644 index 7bf34e089c..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "content" { - description = "The YAML body to apply to gke cluster." - type = string - default = null -} - -variable "source_path" { - description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." - type = string - default = null -} - -variable "template_vars" { - description = "The values to populate template file(s) with." - type = any - default = null -} - -variable "server_side_apply" { - description = "Allow using kubectl server-side apply method." - type = bool - default = false -} - -variable "wait_for_rollout" { - description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." - type = bool - default = true -} - -variable "force_conflicts" { - description = "The force_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller)" - type = bool - default = false -} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf deleted file mode 100644 index cce452239f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.7.0" - } - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml deleted file mode 100644 index 7c0bef7013..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# For referencing the original Kueue helm chart values, pull the latest helm chart version -# `helm pull oci://registry.k8s.io/kueue/charts/kueue --version=0.13.3` (latest helm chart version) - -controllerManager: - # -- Enables the Topology-Aware Scheduling feature gate. - featureGates: - - name: TopologyAwareScheduling - enabled: true - - # It ensures the Kueue pod can schedule on GKE clusters where the - # system node pool uses the default "gke-managed-components" taint. - tolerations: - - key: "components.gke.io/gke-managed-components" - operator: "Equal" - value: "true" - effect: "NoSchedule" diff --git a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/main.tf b/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/main.tf deleted file mode 100644 index 73a15ad1ab..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/management/kubectl-apply/main.tf +++ /dev/null @@ -1,271 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] - project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] - - # 1. First, Identify manifests that are explicitly enabled. - enabled_manifests = { - for index, manifest in var.apply_manifests : index => manifest - if try(manifest.enable, true) - } - - # 2. Identify URL-based manifests - url_manifests = { - for index, manifest in local.enabled_manifests : index => manifest - if try(manifest.source, null) != null && (startswith(manifest.source, "http://") || startswith(manifest.source, "https://")) - } - - # 3. Rebuild the map by populating the 'content' field for URLs based manifest - processed_apply_manifests_map = tomap({ - for index, manifest in local.enabled_manifests : tostring(index) => { - # If this manifest was a URL, its content is the body from the HTTP call. - content = contains(keys(local.url_manifests), tostring(index)) ? data.http.manifest_from_url[tostring(index)].body : manifest.content - - # If this was a URL, its source path is now null. Otherwise, use original. - source = contains(keys(local.url_manifests), tostring(index)) ? null : manifest.source - - # Pass other vars - template_vars = manifest.template_vars - server_side_apply = manifest.server_side_apply - wait_for_rollout = manifest.wait_for_rollout - } - }) - - install_kueue = try(var.kueue.install, false) - install_jobset = try(var.jobset.install, false) - install_gpu_operator = try(var.gpu_operator.install, false) - install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) - install_gib = try(var.gib.install, false) -} - -data "http" "manifest_from_url" { - for_each = local.url_manifests - url = each.value.source -} - -data "google_container_cluster" "gke_cluster" { - project = local.project_id - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -module "kubectl_apply_manifests" { - for_each = local.processed_apply_manifests_map - source = "./kubectl" - depends_on = [var.gke_cluster_exists] - - content = each.value.content - source_path = each.value.source - template_vars = each.value.template_vars - server_side_apply = each.value.server_side_apply - wait_for_rollout = each.value.wait_for_rollout - - providers = { - kubectl = kubectl - } -} - -module "install_kueue" { - source = "./helm_install" - count = local.install_kueue ? 1 : 0 - wait = false - timeout = 1200 - release_name = "kueue" - chart_repository = "oci://registry.k8s.io/kueue/charts" - chart_name = "kueue" - chart_version = var.kueue.version - namespace = "kueue-system" - create_namespace = true - values_yaml = [ - file("${path.module}/kueue/kueue-helm-values.yaml") - ] - - depends_on = [var.gke_cluster_exists] -} - -module "configure_kueue" { - source = "./kubectl" - source_path = local.install_kueue ? try(var.kueue.config_path, "") : null - template_vars = local.install_kueue ? try(var.kueue.config_template_vars, null) : null - depends_on = [module.install_kueue] - - server_side_apply = true - wait_for_rollout = true - - providers = { - kubectl = kubectl - } -} - -module "install_jobset" { - source = "./helm_install" - count = local.install_jobset ? 1 : 0 - wait = false - timeout = 1200 - release_name = "jobset" - chart_repository = "oci://registry.k8s.io/jobset/charts" - chart_name = "jobset" - chart_version = var.jobset.version - namespace = "jobset-system" - create_namespace = true - values_yaml = [ - file("${path.module}/jobset/jobset-helm-values.yaml") - ] - depends_on = [var.gke_cluster_exists, module.configure_kueue] -} - -module "install_nvidia_dra_driver" { - count = local.install_nvidia_dra_driver ? 1 : 0 - depends_on = [module.kubectl_apply_manifests, var.gke_cluster_exists, module.configure_kueue] - source = "./helm_install" - - release_name = "nvidia-dra-driver-gpu" # The release name - chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts - chart_name = "nvidia-dra-driver-gpu" # The chart name - chart_version = var.nvidia_dra_driver.version # The chart version - namespace = "nvidia-dra-driver-gpu" # The target namespace - create_namespace = true # Equivalent to --create-namespace - - # Use the 'values' argument to pass the YAML content - # This corresponds to the -f <(cat < -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_monitoring_dashboard.dashboard](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_dashboard) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [base\_dashboard](#input\_base\_dashboard) | Baseline dashboard template, select from HPC or Empty | `string` | `"HPC"` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the monitoring dashboard instance. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [title](#input\_title) | Title of the created dashboard | `string` | `"Cluster Toolkit Dashboard"` | no | -| [widgets](#input\_widgets) | List of additional widgets to add to the base dashboard. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for accessing the monitoring dashboard | - diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl deleted file mode 100644 index f25cbbd2c6..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl +++ /dev/null @@ -1,17 +0,0 @@ -{ - "displayName": "${title}: ${deployment_name}", - "gridLayout": { - "columns": 2, - "widgets": [ - { - "text": { - "content": "Metrics from the ${deployment_name} deployment of the Cluster Toolkit.", - "format": "MARKDOWN" - }, - "title": "${title}" - }%{ for widget in widgets ~}, - ${widget} - %{endfor ~} - ] - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl deleted file mode 100644 index 5b20435a9a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl +++ /dev/null @@ -1,595 +0,0 @@ -{ - "displayName": "${title}: ${deployment_name}", - "labels": ${jsonencode(labels)}, - "gridLayout": { - "columns": 2, - "widgets": [ - { - "text": { - "content": "HPC metrics from the ${deployment_name} deployment of the Cluster Toolkit.", - "format": "MARKDOWN" - }, - "title": "${title}" - }, - { - "title": "VM Instance - Memory utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/memory/percent_used\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - CPU Utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", - "pickTimeSeriesFilter": { - "direction": "TOP", - "numTimeSeries": 20, - "rankingMethod": "METHOD_MEAN" - } - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - CPU utilization (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - }, - "unitOverride": "%" - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk read operations", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/read_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk write operations", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/write_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk Read Bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/disk/read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk Write Bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/disk/write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Throttled read bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Throttled write bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Received packets", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/received_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - Sent packets", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - Received bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/received_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Sent bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_MEAN", - "groupByFields": [ - "metric.label.\"instance_name\"", - "metric.label.\"loadbalanced\"", - "resource.label.\"project_id\"", - "resource.label.\"instance_id\"", - "resource.label.\"zone\"" - ], - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - } - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Network Traffic Bytes (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/interface/traffic\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Network Packets (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/interface/packets\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "TCP connections", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/network/tcp_connections\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - }, - "unitOverride": "1" - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - CPU utilization for steal", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "STACKED_BAR", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MAX" - }, - "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\" metric.label.\"cpu_state\"=\"steal\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - CPU utilization [MEAN]", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }%{ for widget in widgets ~}, - ${widget} - %{endfor ~} - ] - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/main.tf b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/main.tf deleted file mode 100644 index df3c5c36b0..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/main.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "dashboard", ghpc_role = "monitoring" }) -} - -locals { - dash_path = "${path.module}/dashboards/${var.base_dashboard}.json.tpl" -} - -resource "google_monitoring_dashboard" "dashboard" { - dashboard_json = templatefile(local.dash_path, { - widgets = var.widgets - deployment_name = var.deployment_name - title = var.title - labels = local.labels - } - ) - project = var.project_id -} diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/metadata.yaml deleted file mode 100644 index de1a10f57d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - stackdriver.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/outputs.tf b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/outputs.tf deleted file mode 100644 index b7ff35fb0e..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "instructions" { - description = "Instructions for accessing the monitoring dashboard" - value = <<-EOT - A monitoring dashboard has been created. To view, navigate to the following URL: - https://console.cloud.google.com/monitoring/dashboards/builder${regex("/[0-9a-z-]*$", google_monitoring_dashboard.dashboard.id)} - EOT -} diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/variables.tf b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/variables.tf deleted file mode 100644 index 8194f8b73a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/variables.tf +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "base_dashboard" { - description = "Baseline dashboard template, select from HPC or Empty" - type = string - default = "HPC" - validation { - condition = contains(["HPC", "Empty"], var.base_dashboard) - error_message = "Must set var.base_dashboard to either \"HPC\" or \"Empty\"." - } -} - -variable "title" { - description = "Title of the created dashboard" - type = string - default = "Cluster Toolkit Dashboard" -} - -variable "widgets" { - description = "List of additional widgets to add to the base dashboard." - type = list(string) - default = [] -} - -variable "labels" { - description = "Labels to add to the monitoring dashboard instance. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/versions.tf b/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/versions.tf deleted file mode 100644 index 2717fe79f6..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/monitoring/dashboard/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/README.md b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/README.md deleted file mode 100644 index 057f4b649d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/README.md +++ /dev/null @@ -1,111 +0,0 @@ -## Description - -This module facilitates the creation of custom firewall rules for existing -networks. - -## Example usage - -This module can be used by other Toolkit modules to create application-specific -firewall rules or in conjunction with the [pre-existing-vpc] module to enable -traffic in existing networks. The snippet below is drawn from the -[ml-slurm.yaml] example: - -```yaml -- group: primary - modules: - - id: network - source: modules/network/pre-existing-vpc - - # this example anticipates that the VPC default network has internal traffic - # allowed and IAP tunneling for SSH connections - - id: firewall_rule - source: modules/network/firewall-rules - use: - - network - settings: - ingress_rules: - - name: $(vars.deployment_name)-allow-internal-traffic - description: Allow internal traffic - destination_ranges: - - $(network.subnetwork_address) - source_ranges: - - $(network.subnetwork_address) - allow: - - protocol: tcp - ports: - - 0-65535 - - protocol: udp - ports: - - 0-65535 - - protocol: icmp - - name: $(vars.deployment_name)-allow-iap-ssh - description: Allow IAP-tunneled SSH connections - destination_ranges: - - $(network.subnetwork_address) - source_ranges: - - 35.235.240.0/20 - allow: - - protocol: tcp - ports: - - 22 -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [firewall\_rule](#module\_firewall\_rule) | terraform-google-modules/network/google//modules/firewall-rules | ~> 12.0 | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.pga_check](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [google_compute_subnetwork.subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [egress\_rules](#input\_egress\_rules) | List of egress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | -| [ingress\_rules](#input\_ingress\_rules) | List of ingress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | -| [network\_name](#input\_network\_name) | The name of the network to create firewall rules in | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The project ID to host the network in | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork whose global network firewall rules will be modified. | `string` | n/a | yes | - -## Outputs - -No outputs. - - -[pre-existing-vpc]: ../pre-existing-vpc/README.md -[ml-slurm.yaml]: ../../../examples/ml-slurm.yaml diff --git a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/main.tf b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/main.tf deleted file mode 100644 index 05241278ad..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/main.tf +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - use_subnetwork_data = (var.project_id == null || var.network_name == null) && var.subnetwork_self_link != null -} - -# the google_compute_network data source does not allow identification by -# self_link, which uniquely identifies subnet, project, and network -data "google_compute_subnetwork" "subnetwork" { - # Only instantiate this data source if needed - count = local.use_subnetwork_data ? 1 : 0 - self_link = var.subnetwork_self_link -} - -locals { - # Derived values from data source, null if data source is not used - derived_project_id = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].project : null - derived_network_name = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].network : null - - # Effective values: Use var if provided, otherwise use derived value - effective_project_id = coalesce(var.project_id, local.derived_project_id) - effective_network_name = coalesce(var.network_name, local.derived_network_name) -} - -# Module-level check for Private Google Access on the subnetwork. -# This check is only relevant if subnetwork_self_link was provided and used. -resource "terraform_data" "pga_check" { - count = local.use_subnetwork_data ? 1 : 0 - - lifecycle { - precondition { - condition = data.google_compute_subnetwork.subnetwork[0].private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.subnetwork[0].name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } - } -} - -module "firewall_rule" { - source = "terraform-google-modules/network/google//modules/firewall-rules" - version = "~> 12.0" - project_id = local.effective_project_id - network_name = local.effective_network_name - - ingress_rules = var.ingress_rules - egress_rules = var.egress_rules -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/variables.tf b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/variables.tf deleted file mode 100644 index 05e9be4425..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/variables.tf +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork whose global network firewall rules will be modified." - type = string -} - -variable "project_id" { - description = "The project ID to host the network in" - type = string - default = null -} - -variable "network_name" { - description = "The name of the network to create firewall rules in" - type = string - default = null -} - -variable "ingress_rules" { - description = "List of ingress rules" - default = [] - type = list(object({ - name = string - description = optional(string, null) - disabled = optional(bool, null) - priority = optional(number, null) - destination_ranges = optional(list(string), []) - source_ranges = optional(list(string), []) - source_tags = optional(list(string)) - source_service_accounts = optional(list(string)) - target_tags = optional(list(string)) - target_service_accounts = optional(list(string)) - - allow = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - deny = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - log_config = optional(object({ - metadata = string - })) - })) -} - -variable "egress_rules" { - description = "List of egress rules" - default = [] - type = list(object({ - name = string - description = optional(string, null) - disabled = optional(bool, null) - priority = optional(number, null) - destination_ranges = optional(list(string), []) - source_ranges = optional(list(string), []) - source_tags = optional(list(string)) - source_service_accounts = optional(list(string)) - target_tags = optional(list(string)) - target_service_accounts = optional(list(string)) - - allow = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - deny = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - log_config = optional(object({ - metadata = string - })) - })) -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/versions.tf deleted file mode 100644 index 9061dd3ae5..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/firewall-rules/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.74.0" - } - - required_version = ">= 1.5" -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/README.md b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/README.md deleted file mode 100644 index abbfe3b97b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/README.md +++ /dev/null @@ -1,143 +0,0 @@ -## Description - -This module accomplishes the following: - -* Creates one [VPC network][cft-network] - * Each VPC contains a variable number of subnetworks as specified in the - `subnetworks_template` variable - * Each subnetwork contains distinct IP address ranges -* Outputs the following unique parameters - * `subnetwork_interfaces` which is compatible with Slurm and vm-instance - modules - * `subnetwork_interfaces_gke` which is compatible with GKE modules - -This module is a simplified version of the VPC module and its main difference -is the variable `subnetwork_template` which is the template for all subnetworks -created within the network. This template contains the following values: - -1. `count`: The number of subnetworks to be created -1. `name_prefix`: The prefix for the subnetwork names -1. `ip_range`: [CIDR-formatted IP range][cidr] -1. `region`: The region where the subnetwork will be deployed - -> [!WARNING] -> The `ip_range` should be always be large enough to split into `count` -> subnetworks and the number of required connections within. - -[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v10.0.0 -[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation - -### Example - -This snippet uses the gpu-vpc module to create a new VPC network named -`test-rdma-net` with 8 subnetworks named `test-mrdma-sub-#` where # ranges from -0 to 7. The subnetworks will split the `ip_range` evenly, starting from bit 16 -(0 indexed). The networks are ingested by the Slurm nodeset within the -`additional_networks` setting. - -```yaml - - id: rdma-net - source: modules/network/gpu-rdma-vpc - settings: - network_name: test-rdma-net - network_profile: https://www.googleapis.com/compute/beta/projects/$(vars.project_id)/global/networkProfiles/$(vars.zone)-vpc-roce - network_routing_mode: REGIONAL - subnetworks_template: - name_prefix: test-mrdma-sub - count: 8 - ip_range: 192.168.0.0/16 - region: $(vars.region) - - - id: a3_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network0] - settings: - machine_type: a3-ultragpu-8g - additional_networks: - $(concat( - [{ - network=null, - subnetwork=network1.subnetwork_self_link, - subnetwork_project=vars.project_id, - nic_type="GVNIC", - queue_count=null, - network_ip="", - stack_type=null, - access_config=[], - ipv6_access_config=[], - alias_ip_range=[] - }], - rdma-net.subnetwork_interfaces - )) - ... -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | DEPRECATED: enable\_internal\_traffic can not be specified for gpu-rdma-vpc. | `bool` | `null` | no | -| [firewall\_log\_config](#input\_firewall\_log\_config) | DEPRECATED: firewall\_log\_config can not be specified for gpu-rdma-vpc. | `string` | `null` | no | -| [firewall\_rules](#input\_firewall\_rules) | DEPRECATED: firewall\_rules can not be specified for gpu-rdma-vpc. | `any` | `null` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}} | `string` | n/a | yes | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "REGIONAL") | `string` | `"REGIONAL"` | no | -| [nic\_type](#input\_nic\_type) | NIC type for use in modules that use the output | `string` | `"MRDMA"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | -| [subnetworks\_template](#input\_subnetworks\_template) | Specifications for the subnetworks that will be created within this VPC.

count (number, required, number of subnets to create, default is 8)
name\_prefix (string, required, subnet name prefix, default is deployment name)
ip\_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16)
region (string, optional, region to deploy subnets to, defaults to vars.region) | 
object({
    count       = number
    name_prefix = string
    ip_range    = string
    region      = optional(string)
  })
| 
{
  "count": 8,
  "ip_range": "192.168.0.0/16",
  "name_prefix": null,
  "region": null
}
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [network\_id](#output\_network\_id) | ID of the new VPC network | -| [network\_name](#output\_network\_name) | Name of the new VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | -| [subnetwork\_interfaces](#output\_subnetwork\_interfaces) | Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules) | -| [subnetwork\_interfaces\_gke](#output\_subnetwork\_interfaces\_gke) | Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool) | -| [subnetwork\_name\_prefix](#output\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | -| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | - diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/main.tf b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/main.tf deleted file mode 100644 index e37db01976..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/main.tf +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - autoname = replace(var.deployment_name, "_", "-") - network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name - subnet_prefix = var.subnetworks_template.name_prefix == null ? "${local.autoname}-subnet" : var.subnetworks_template.name_prefix - - new_bits = ceil(log(var.subnetworks_template.count, 2)) - template_subnetworks = [for i in range(var.subnetworks_template.count) : - { - subnet_name = "${local.subnet_prefix}-${i}" - subnet_region = try(var.subnetworks_template.region, var.region) - subnet_ip = cidrsubnet(var.subnetworks_template.ip_range, local.new_bits, i) - } - ] - - firewall_rules = [] - - output_subnets = [ - for subnet in module.vpc.subnets : { - network = null - subnetwork = subnet.self_link - subnetwork_project = null # will populate from subnetwork_self_link - network_ip = null - nic_type = var.nic_type - stack_type = null - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } - ] - - output_subnets_gke = [ - for i in range(length(module.vpc.subnets)) : { - network = local.network_name - subnetwork = local.template_subnetworks[i].subnet_name - subnetwork_project = var.project_id - network_ip = null - nic_type = var.nic_type - stack_type = null - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } - ] -} - -module "vpc" { - source = "terraform-google-modules/network/google" - version = "~> 12.0" - - network_name = local.network_name - project_id = var.project_id - auto_create_subnetworks = false - subnets = local.template_subnetworks - routing_mode = var.network_routing_mode - mtu = var.mtu - description = var.network_description - shared_vpc_host = var.shared_vpc_host - delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes - firewall_rules = local.firewall_rules - network_profile = var.network_profile -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf deleted file mode 100644 index 0a21f1d3f2..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the new VPC network" - value = module.vpc.network_name - depends_on = [module.vpc] -} - -output "network_id" { - description = "ID of the new VPC network" - value = module.vpc.network_id - depends_on = [module.vpc] -} - -output "network_self_link" { - description = "Self link of the new VPC network" - value = module.vpc.network_self_link - depends_on = [module.vpc] -} - -output "subnetworks" { - description = "Full list of subnetwork objects belonging to the new VPC network" - value = module.vpc.subnets - depends_on = [module.vpc] -} - -output "subnetwork_interfaces" { - description = "Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules)" - value = local.output_subnets - depends_on = [module.vpc] -} - -# The output subnetwork_interfaces is compatible with vm-instance module but not with gke-node-pool -# See https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/99493df21cecf6a092c45298bf7a45e0343cf622/modules/compute/vm-instance/variables.tf#L220 -# So, we need a separate output that makes the network and subnetwork names available -output "subnetwork_interfaces_gke" { - description = "Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool)" - value = local.output_subnets_gke - depends_on = [module.vpc] -} - -output "subnetwork_name_prefix" { - description = "Prefix of the RDMA subnetwork names" - value = var.subnetworks_template.name_prefix -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf deleted file mode 100644 index a30fb50e7d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "network_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" - type = string - default = null -} - -variable "region" { - description = "The default region for Cloud resources" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "subnetworks_template" { - description = <<-EOT - Specifications for the subnetworks that will be created within this VPC. - - count (number, required, number of subnets to create, default is 8) - name_prefix (string, required, subnet name prefix, default is deployment name) - ip_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16) - region (string, optional, region to deploy subnets to, defaults to vars.region) - EOT - nullable = false - type = object({ - count = number - name_prefix = string - ip_range = string - region = optional(string) - }) - default = { - count = 8 - name_prefix = null - ip_range = "192.168.0.0/16" - region = null - } - - validation { - condition = var.subnetworks_template.count > 0 - error_message = "Number of subnetworks must be greater than 0" - } - - validation { - condition = can(cidrhost(var.subnetworks_template.ip_range, 0)) - error_message = "IP address range must be in CIDR format." - } -} - -variable "network_routing_mode" { - type = string - default = "REGIONAL" - description = "The network routing mode (default \"REGIONAL\")" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "shared_vpc_host" { - type = bool - description = "Makes this project a Shared VPC host if 'true' (default 'false')" - default = false -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_internal_traffic" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." - type = bool - default = null - validation { - condition = var.enable_internal_traffic == null - error_message = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." - } -} - -variable "firewall_rules" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." - type = any - default = null - validation { - condition = var.firewall_rules == null - error_message = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." - } -} - -variable "firewall_log_config" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." - type = string - default = null - validation { - condition = var.firewall_log_config == null - error_message = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." - } -} - -variable "network_profile" { - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - EOT - type = string - nullable = false - - validation { - condition = can(coalesce(var.network_profile)) - error_message = "var.network_profile must be specified and not an empty string" - } -} - -variable "nic_type" { - description = "NIC type for use in modules that use the output" - type = string - nullable = true - default = "MRDMA" - - validation { - condition = contains(["MRDMA"], var.nic_type) - error_message = "The nic_type must be \"MRDMA\"." - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf deleted file mode 100644 index 71b7106734..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 0.15.0" -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/multivpc/README.md b/deletion-test/build_script/modules/embedded/modules/network/multivpc/README.md deleted file mode 100644 index 973e6b32c9..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/multivpc/README.md +++ /dev/null @@ -1,136 +0,0 @@ -## Description - -This module accomplishes the following: - -* Creates 2 to 8 [VPC networks][vpc] - * Each VPC contains exactly 1 subnetwork - * Each subnetwork contains distinct IP address ranges -* Outputs the `additional_networks` parameter, which is compatible with Slurm - modules - -There are 4 variables that differentiate this module from the standard VPC -module. - -1. `network_prefix`: The name prefix of the VPCs to be created. All - networks and subnetworks will start with this and end with a unique number. -1. `network_count`: The number of VPCs to be created. -1. `global_ip_address_range`: [CIDR-formatted IP range][cidr] -1. `network_cidr_suffix`: The CIDR suffix that defines the address - space that the individual VPCs will cover. - -> [!WARNING] -> The `network_cidr_suffix` should be always be larger than the CIDR suffix on -> `global_ip_address_range`. The difference between these two suffixes should -> be large enough to accommodate the number of VPCs that are being deployed -> (e.g. CIDR suffix bit difference <= `ceil(log2(network_count)))`). - -> [!NOTE] -> For deployments that need multiple VPCs that do not meet this use-case, users -> should deploy multiple individual VPC modules. - -[vpc]: ../vpc/README.md -[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation - -### Example - -This snippet uses the multivpc module to create 8 new VPC networks named -`multivpc-net-#` where # ranges from 0 to 7. Additionally, it creates 1 -subnetwork in each VPC. - -```yaml - - id: network - source: modules/network/vpc - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: a3_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network, multinetwork] - settings: - machine_type: a3-highgpu-8g - ... -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | - -## Providers - -| Name | Version | -|------|---------| -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [vpcs](#module\_vpcs) | ../vpc | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.global_ip_cidr_suffix](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | -| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | -| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | -| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | -| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | -| [global\_ip\_address\_range](#input\_global\_ip\_address\_range) | IP address range (CIDR) that will span entire set of VPC networks | `string` | `"172.16.0.0/12"` | no | -| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT) | `number` | `2` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_count](#input\_network\_count) | The number of vpc nettworks to create | `number` | `4` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_interface\_defaults](#input\_network\_interface\_defaults) | The template of the network settings to be used on all vpcs. | 
object({
    network            = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string, "GVNIC")
    stack_type         = optional(string, "IPV4_ONLY")
    queue_count        = optional(string)
    access_config = optional(list(object({
      nat_ip                 = string
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  })
| 
{
  "access_config": [],
  "alias_ip_range": [],
  "ipv6_access_config": [],
  "network": null,
  "network_ip": "",
  "nic_type": "GVNIC",
  "queue_count": null,
  "stack_type": "IPV4_ONLY",
  "subnetwork": null,
  "subnetwork_project": null
}
| no | -| [network\_name\_prefix](#input\_network\_name\_prefix) | The base name of the vpcs and their subnets, will be appended with a sequence number | `string` | `""` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network dynamic routing mode | `string` | `"REGIONAL"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [subnetwork\_cidr\_suffix](#input\_subnetwork\_cidr\_suffix) | The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network. | `number` | `16` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [additional\_networks](#output\_additional\_networks) | Network interfaces for each subnetwork created by this module | -| [network\_ids](#output\_network\_ids) | IDs of the new VPC network | -| [network\_names](#output\_network\_names) | Names of the new VPC networks | -| [network\_self\_links](#output\_network\_self\_links) | Self link of the new VPC network | -| [subnetwork\_addresses](#output\_subnetwork\_addresses) | IP address range of the primary subnetwork | -| [subnetwork\_names](#output\_subnetwork\_names) | Names of the subnetwork created in each network | -| [subnetwork\_self\_links](#output\_subnetwork\_self\_links) | Self link of the primary subnetwork | - diff --git a/deletion-test/build_script/modules/embedded/modules/network/multivpc/main.tf b/deletion-test/build_script/modules/embedded/modules/network/multivpc/main.tf deleted file mode 100644 index ad06e793c1..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/multivpc/main.tf +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # this input variable is validated to be in CIDR format - network_name = coalesce(replace(var.network_name_prefix, "_", "-"), replace(var.deployment_name, "_", "-")) - global_ip_cidr_prefix = split("/", var.global_ip_address_range)[0] - global_ip_cidr_suffix = split("/", var.global_ip_address_range)[1] - global_ip_cidr_valid = "${local.global_ip_cidr_prefix}/${terraform_data.global_ip_cidr_suffix.output}" - subnetwork_new_bits = var.subnetwork_cidr_suffix - local.global_ip_cidr_suffix - maximum_subnetworks = pow(2, local.subnetwork_new_bits) - additional_networks = [ - for vpc in module.vpcs : - merge(var.network_interface_defaults, { - network = vpc.network_name - subnetwork = vpc.subnetwork_name - subnetwork_project = var.project_id - }) - ] -} - -resource "terraform_data" "global_ip_cidr_suffix" { - input = local.global_ip_cidr_suffix - lifecycle { - precondition { - condition = local.maximum_subnetworks >= var.network_count - error_message = < 1 - error_message = "The minimum VPCs able to be created by this module is 2. Use the standard Toolkit module at modules/network/vpc for count = 1" - } - validation { - condition = var.network_count <= 8 - error_message = "The maximum VPCs able to be created by this module is 8" - } -} - -variable "global_ip_address_range" { - description = "IP address range (CIDR) that will span entire set of VPC networks" - type = string - default = "172.16.0.0/12" - - validation { - condition = can(cidrhost(var.global_ip_address_range, 0)) - error_message = "var.global_ip_address_range must be an IPv4 CIDR range (e.g. \"172.16.0.0/12\")." - } -} - -variable "subnetwork_cidr_suffix" { - description = "The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network." - type = number - default = 16 -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "network_routing_mode" { - type = string - default = "REGIONAL" - description = "The network dynamic routing mode" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "ips_per_nat" { - type = number - description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT)" - default = 2 -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_iap_ssh_ingress" { - type = bool - description = "Enable a firewall rule to allow SSH access using IAP tunnels" - default = true -} - -variable "enable_iap_rdp_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" - default = false -} - -variable "enable_iap_winrm_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" - default = false -} - -variable "enable_internal_traffic" { - type = bool - description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" - default = true -} - -variable "extra_iap_ports" { - type = list(string) - description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" - default = [] -} - -variable "allowed_ssh_ip_ranges" { - type = list(string) - description = "A list of CIDR IP ranges from which to allow ssh access" - default = [] - - validation { - condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) - error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." - } -} - -variable "firewall_rules" { - type = any - description = "List of firewall rules" - default = [] -} - -variable "network_interface_defaults" { - type = object({ - network = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - network_ip = optional(string, "") - nic_type = optional(string, "GVNIC") - stack_type = optional(string, "IPV4_ONLY") - queue_count = optional(string) - access_config = optional(list(object({ - nat_ip = string - network_tier = string - public_ptr_domain_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - public_ptr_domain_name = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - }) - description = "The template of the network settings to be used on all vpcs." - default = { - network = null - subnetwork = null - subnetwork_project = null - network_ip = "" - nic_type = "GVNIC" - stack_type = "IPV4_ONLY" - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } -} - -variable "network_profile" { - type = string - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown - EOT - default = null -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/multivpc/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/multivpc/versions.tf deleted file mode 100644 index e75a67f7b6..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/multivpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.4.0" -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/README.md b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/README.md deleted file mode 100644 index 4d63b17091..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/README.md +++ /dev/null @@ -1,94 +0,0 @@ -## Description - -This module discovers a subnetwork that already exists in Google Cloud and -outputs subnetwork attributes that uniquely identify it for use by other modules. - -For example, the blueprint below discovers the referred to subnetwork. -With the `use` keyword, the [vm-instance] module accepts the `subnetwork_self_link` -input variables that uniquely identify the subnetwork in which the VM will be created. - -[vpc]: ../vpc/README.md -[vm-instance]: ../../compute/vm-instance/README.md - -> **_NOTE:_** Additional IAM work is needed for this to work correctly. - -### Example - -```yaml -- id: network - source: modules/network/pre-existing-subnetwork - settings: - subnetwork_self_link: https://www.googleapis.com/compute/v1/projects/name-of-host-project/regions/REGION/subnetworks/SUBNETNAME - -- id: example_vm - source: modules/compute/vm-instance - use: - - network - settings: - name_prefix: example - machine_type: c2-standard-4 -``` - -As described in documentation: -[https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork] - -If subnetwork_self_link is provided then name,region,project is ignored. - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [project](#input\_project) | Name of the project that owns the subnetwork | `string` | `null` | no | -| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | `null` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self-link of the subnet in the VPC | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | -| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | - diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/main.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/main.tf deleted file mode 100644 index 9fb206f969..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/main.tf +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -data "google_compute_subnetwork" "primary_subnetwork" { - name = var.subnetwork_name - region = var.region - project = var.project - self_link = var.subnetwork_self_link - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The subnetwork: ${coalesce(var.subnetwork_name, var.subnetwork_self_link)} could not be found." - } - } -} - -# Module-level check for Private Google Access on the subnetwork -check "private_google_access_enabled_subnetwork" { - assert { - condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml deleted file mode 100644 index 6a6f1e5757..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com -ghpc: - has_to_be_used: true diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf deleted file mode 100644 index 868708dc6b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "subnetwork" { - description = "Full subnetwork object in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork -} - -output "subnetwork_name" { - description = "Name of the subnetwork in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.name -} - -output "subnetwork_self_link" { - description = "Subnetwork self-link in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.self_link -} - -output "subnetwork_address" { - description = "Subnetwork IP range in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf deleted file mode 100644 index d5191843e8..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "subnetwork_self_link" { - description = "Self-link of the subnet in the VPC" - type = string - default = null -} - -variable "project" { - description = "Name of the project that owns the subnetwork" - type = string - default = null -} - -variable "subnetwork_name" { - description = "Name of the pre-existing VPC subnetwork" - type = string - default = null -} - -variable "region" { - description = "Region in which to search for primary subnetwork" - type = string - default = null -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf deleted file mode 100644 index 917d948433..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.74.0" - } - - required_version = ">= 1.5" -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/README.md b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/README.md deleted file mode 100644 index 38a1840c2d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/README.md +++ /dev/null @@ -1,110 +0,0 @@ -## Description - -This module discovers a VPC network that already exists in Google Cloud and -outputs network attributes that uniquely identify it for use by other modules. -The module outputs are aligned with the [vpc module][vpc] so that it can be used -as a drop-in substitute when a VPC already exists. - -For example, the blueprint below discovers the "default" global network and the -"default" regional subnetwork in us-central1. With the `use` keyword, the -[vm-instance] module accepts the `network_self_link` and `subnetwork_self_link` -input variables that uniquely identify the network and subnetwork in which the -VM will be created. - -[vpc]: ../vpc/README.md -[vm-instance]: ../../compute/vm-instance/README.md - -### Example - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - settings: - project_id: $(vars.project_id) - region: us-central1 - -- id: example_vm - source: modules/compute/vm-instance - use: - - network1 - settings: - name_prefix: example - machine_type: c2-standard-4 -``` - -> **_NOTE:_** The `project_id` and `region` settings would be inferred from the -> deployment variables of the same name, but they are included here for clarity. - -### Use shared-vpc - -If a network is created in different project, this module can be used to -reference the network. To use a network from a different project first make sure -you have a [cloud nat][cloudnat] and [IAP][iap] forwarding. For more details, -refer [shared-vpc][shared-vpc-doc] - -[cloudnat]: https://cloud.google.com/nat/docs/overview -[iap]: https://cloud.google.com/iap/docs/using-tcp-forwarding -[shared-vpc-doc]: ../../../examples/README.md#hpc-slurm-sharedvpcyaml-community-badge-experimental-badge - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_network.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network) | data source | -| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [network\_name](#input\_network\_name) | Name of the existing VPC network | `string` | `"default"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | n/a | yes | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork; defaults to var.network\_name if set to null. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [network\_id](#output\_network\_id) | ID of the existing VPC network | -| [network\_name](#output\_network\_name) | Name of the existing VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the existing VPC network | -| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | -| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | - diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/main.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/main.tf deleted file mode 100644 index ed332bab72..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/main.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -data "google_compute_network" "vpc" { - name = var.network_name - project = var.project_id - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The network: ${var.network_name} could not be found in project: ${var.project_id}." - } - } -} - -locals { - subnetwork_name = var.subnetwork_name != null ? var.subnetwork_name : var.network_name -} - -data "google_compute_subnetwork" "primary_subnetwork" { - name = local.subnetwork_name - region = var.region - project = var.project_id - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The subnetwork: ${local.subnetwork_name} could not be found in project: ${var.project_id} and region: ${var.region}." - } - } -} - -# Module-level check for Private Google Access on the subnetwork -check "private_google_access_enabled_subnetwork" { - assert { - condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/outputs.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/outputs.tf deleted file mode 100644 index 00861af5ca..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the existing VPC network" - value = data.google_compute_network.vpc.name -} - -output "network_id" { - description = "ID of the existing VPC network" - value = data.google_compute_network.vpc.id -} - -output "network_self_link" { - description = "Self link of the existing VPC network" - value = data.google_compute_network.vpc.self_link -} - -output "subnetwork" { - description = "Full subnetwork object in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork -} - -output "subnetwork_name" { - description = "Name of the subnetwork in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.name -} - -output "subnetwork_self_link" { - description = "Subnetwork self-link in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.self_link -} - -output "subnetwork_address" { - description = "Subnetwork IP range in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/variables.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/variables.tf deleted file mode 100644 index 291a81604a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/variables.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "network_name" { - description = "Name of the existing VPC network" - type = string - default = "default" -} - -variable "subnetwork_name" { - description = "Name of the pre-existing VPC subnetwork; defaults to var.network_name if set to null." - type = string - default = null -} - -variable "region" { - description = "Region in which to search for primary subnetwork" - type = string -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/versions.tf deleted file mode 100644 index 81fe5aeff3..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/pre-existing-vpc/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.74.0" - } - - required_version = ">= 1.5" -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/README.md b/deletion-test/build_script/modules/embedded/modules/network/vpc/README.md deleted file mode 100644 index 2c2b1aa1a3..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/vpc/README.md +++ /dev/null @@ -1,237 +0,0 @@ -## Description - -This module creates a new [VPC network][vpc] with 1 or more subnetworks and -a [Cloud Router][router] for every region with a subnetwork. By default, it will -create: - -* A [Cloud NAT][nat] to enable outbound access to the public internet for VMs - without public IP addresses; VMs with public IP addresses bypass the NAT to - directly access the public internet -* A firewall rule that enables inbound SSH access from [Identity-Aware - Proxy][iap] -* A firewall rule that enables all traffic internal to the network - -This behavior is optional and can be configured as [described below](#inputs). -This module is based on networking support in the [Cloud Foundation -Toolkit][cft]. We recommend following the [documentation for the network -module][cft-network] and [submodules][cft-network-submodules] for more details. -In particular, the detailed structure of input variables can be found for: - -* [var.firewall\_rules](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/firewall-rules#inputs) -* [var.secondary\_ranges](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets#inputs) - -[vpc]: https://cloud.google.com/vpc -[router]: https://github.com/terraform-google-modules/terraform-google-cloud-router -[nat]: https://github.com/terraform-google-modules/terraform-google-cloud-nat -[iap]: https://cloud.google.com/iap -[cft]: https://cloud.google.com/foundation-toolkit -[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0 -[cft-network-submodules]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules - -Additionally, [Google Private Access][gpa] is enabled by default on all -subnetworks unless it is explicitly disabled. This setting ensures that all VMs -can use Google services such as [Cloud Storage][gcs] even if they do not have -public IP addresses or Cloud NAT is disabled. - -[gpa]: https://cloud.google.com/vpc/docs/private-google-access -[gcs]: https://cloud.google.com/storage - -### Example - -This creates a new VPC network named `cluster-net`. - -```yaml - - id: network1 - source: modules/network/vpc - settings: - network_name: cluster-net -``` - -### Deprecation warning - -The variables listed below have been deprecated and will be removed in a future -release. Until they are removed,You may continue to use them in Toolkit -blueprints with the same functionality as documented in the [Toolkit 1.0 -release][vpc1.0]. - -* Deprecated variables - * `var.primary_subnetwork` - * `var.additional_subnetworks` - * `var.subnetwork_size` - -[vpc1.0]: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/v1.0.0/modules/network/vpc/README.md - -The following variables have been added to support explicit IP ranges for -subnetworks while retaining existing functionality. We advise adopting them even -if not using explicit IP ranges . The Toolkit ***does not support*** mixing -deprecated variables with the new replacements. The new functionality is -described in [more detail below](#subnetworks). - -* New variables to adopt - * `var.subnetworks` - * A value for this can be generated by merging `var.primary_subnetwork` and - `var.additional_subnetworks` into a single list - * `var.default_primary_subnetwork_size` - * This variable has been renamed for clarity; its value can be directly - copied from an explicit setting for `var.subnetwork_size`; if your blueprint - does not have an explicit setting, the default values are the same - -### Subnetworks - -This module will always provision at least 1 "primary" subnetwork in which most -resources are expected to be provisioned. This primary subnetwork is determined -by - -1. The first element of [var.subnetworks](#input_subnetworks) if it is not the - empty list -2. A default subnetwork automatically calculated from - * [var.subnetwork_name](#input_subnetwork_name) - * [var.region](#input_region) - * [var.network_address_range](#input_network_address_range) - * [var.default_primary_subnetwork_size](#input_default_primary_subnetwork_size) - -If `var.subnetworks` is provided then the primary subnetwork name is taken -explicitly from it and `var.subnetwork_name` is ignored. - -`var.subnetworks` behaves identically to the [Cloud Foundation Toolkit subnets -module][cftsubnets] with the lone exception that one can provide ***one*** of -the following settings for each subnetwork: - -* `new_bits` -* `subnet_ip` - -If each subnetwork defines `subnet_ip` then these are taken to be their explicit -CIDR IP ranges. If each subnetwork defines `new_bits`, then these are taken to -be the size of the CIDR subnetwork (in bits). IP ranges for each subnetwork are -calculated using `var.network_address_range` as the base IP, producing the most -compact set of subnetworks possible. - -> **_NOTE:_** we do not presently support the modification of individual subnetworks -> when using this module to provision more than 1 subnetwork using automatically -> calculated IP ranges based upon `new_bits`. Doing so will cause IP ranges to be -> recalculated for each subnetwork. We advise appending new subnetworks to the end -> of `var.subnetworks`. - -[cftsubnets]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets - -### SSH Access - -By default a firewall rule is created to allow inbound SSH access from -[Identity-Aware Proxy][iap]. A user must have the `IAP-Secured Tunnel User` -(`roles/iap.tunnelResourceAccessor`) IAM role to be able to SSH over IAP. - -To allow regular SSH access from a known IP address you can add the following -`firewall_rules` setting to the `vpc` module: - -```yaml - - id: network1 - source: modules/network/vpc - settings: - firewall_rules: - - name: ssh-my-machine - direction: INGRESS - ranges: [/32] - allow: - - protocol: tcp - ports: [22] -``` - -> **Note**: You must populate the above example with the source IP address from -> which you plan to SSH from. You can use a service like -> [whatismyip.com](https://whatismyip.com) to determine your IP address. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | - -## Providers - -| Name | Version | -|------|---------| -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 7.3 | -| [nat\_ip\_addresses](#module\_nat\_ip\_addresses) | terraform-google-modules/address/google | ~> 4.1 | -| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.cloud_nat_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.network_profile_firewall_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.secondary_ranges_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_subnetworks](#input\_additional\_subnetworks) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `list(map(string))` | `null` | no | -| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | -| [default\_primary\_subnetwork\_size](#input\_default\_primary\_subnetwork\_size) | The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks | `number` | `15` | no | -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_cloud\_nat](#input\_enable\_cloud\_nat) | Enable the creation of Cloud NATs. | `bool` | `true` | no | -| [enable\_cloud\_router](#input\_enable\_cloud\_router) | Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview | `bool` | `true` | no | -| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | -| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | -| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | -| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | -| [firewall\_log\_config](#input\_firewall\_log\_config) | Firewall log configuration for Toolkit firewall rules (var.enable\_iap\_ssh\_ingress and others) | `string` | `"DISABLE_LOGGING"` | no | -| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | -| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples | `number` | `2` | no | -| [labels](#input\_labels) | Labels to add to network resources that support labels. Key-value pairs of strings. | `map(string)` | `{}` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_address\_range](#input\_network\_address\_range) | IP address range (CIDR) for global network | `string` | `"10.0.0.0/9"` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "GLOBAL") | `string` | `"GLOBAL"` | no | -| [primary\_subnetwork](#input\_primary\_subnetwork) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `map(string)` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [secondary\_ranges](#input\_secondary\_ranges) | "Secondary ranges associated with the subnets.
This will be deprecated in favour of secondary\_ranges\_list at a later date.
Please migrate to using the same." | `map(list(object({ range_name = string, ip_cidr_range = string })))` | `{}` | no | -| [secondary\_ranges\_list](#input\_secondary\_ranges\_list) | "List of secondary ranges associated with the subnetworks.
Each subnetwork must be specified at most once in this list." | 
list(object({
    subnetwork_name = string,
    ranges = list(object({
      range_name    = string,
      ip_cidr_range = string
    }))
  }))
| `[]` | no | -| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-primary-subnet") | `string` | `null` | no | -| [subnetwork\_size](#input\_subnetwork\_size) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `number` | `null` | no | -| [subnetworks](#input\_subnetworks) | List of subnetworks to create within the VPC. If left empty, it will be
replaced by a single, default subnetwork constructed from other parameters
(e.g. var.region). In all cases, the first subnetwork in the list is identified
by outputs as a "primary" subnetwork.

subnet\_name (string, required, name of subnet)
subnet\_region (string, required, region of subnet)
subnet\_ip (string, mutually exclusive with new\_bits, CIDR-formatted IP range for subnetwork)
new\_bits (number, mutually exclusive with subnet\_ip, CIDR bits used to calculate subnetwork range)
subnet\_private\_access (bool, optional, Enable Private Access on subnetwork)
subnet\_flow\_logs (map(string), optional, Configure Flow Logs see terraform-google-network module)
description (string, optional, Description of Network)
purpose (string, optional, related to Load Balancing)
role (string, optional, related to Load Balancing) | `list(map(string))` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nat\_ips](#output\_nat\_ips) | External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used) | -| [network\_id](#output\_network\_id) | ID of the new VPC network | -| [network\_name](#output\_network\_name) | Name of the new VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | -| [subnetwork](#output\_subnetwork) | Primary subnetwork object | -| [subnetwork\_address](#output\_subnetwork\_address) | IP address range of the primary subnetwork | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the primary subnetwork | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Self link of the primary subnetwork | -| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | - diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/main.tf b/deletion-test/build_script/modules/embedded/modules/network/vpc/main.tf deleted file mode 100644 index 24c8eb22bd..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/vpc/main.tf +++ /dev/null @@ -1,256 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -resource "terraform_data" "secondary_ranges_validation" { - lifecycle { - precondition { - condition = !(length(var.secondary_ranges) > 0 && length(var.secondary_ranges_list) > 0) - error_message = "Only one of var.secondary_ranges or var.secondary_ranges_list should be specified" - } - } -} - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "vpc", ghpc_role = "network" }) -} - -locals { - autoname = replace(var.deployment_name, "_", "-") - network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name - subnetwork_name = var.subnetwork_name == null ? "${local.autoname}-primary-subnet" : var.subnetwork_name - - # define a default subnetwork for cases in which no explicit subnetworks are - # defined in var.subnetworks - default_primary_subnetwork_cidr_block = cidrsubnet(var.network_address_range, var.default_primary_subnetwork_size, 0) - default_primary_subnetwork = { - subnet_name = local.subnetwork_name - subnet_ip = local.default_primary_subnetwork_cidr_block - subnet_region = var.region - subnet_private_access = true - subnet_flow_logs = false - description = "primary subnetwork in ${local.network_name}" - purpose = null - role = null - } - - # Identify user-supplied primary subnetwork - # (1) explicit var.subnetworks[0] - # (2) implicit local default subnetwork - input_primary_subnetwork = coalesce(try(var.subnetworks[0], null), local.default_primary_subnetwork) - - # Identify user-supplied additional subnetworks - # (1) explicit var.subnetworks[1:end] - # (2) empty list - input_additional_subnetworks = try(slice(var.subnetworks, 1, length(var.subnetworks)), []) - - # at this point we have constructed a list of subnetworks but need to extract - # user-provided CIDR blocks or calculate them from user-provided new_bits - # after we complete deprecation, local.all_subnetworks can be replaced with - # var.subnetworks (or local.default_primary_subnetwork if that is null) - input_subnetworks = concat([local.input_primary_subnetwork], local.input_additional_subnetworks) - subnetworks_cidr_blocks = try( - local.input_subnetworks[*]["subnet_ip"], - cidrsubnets(var.network_address_range, local.input_subnetworks[*]["new_bits"]...) - ) - - # merge in the CIDR blocks (even when already there) and remove new_bits - subnetworks = [for i, subnet in local.input_subnetworks : - merge({ for k, v in subnet : k => v if k != "new_bits" }, { "subnet_ip" = local.subnetworks_cidr_blocks[i] }) - ] - - # gather the unique regions for purposes of creating Router/NAT - cloud_router_regions = var.enable_cloud_router ? distinct([for subnet in local.subnetworks : subnet.subnet_region]) : [] - cloud_nat_regions = var.enable_cloud_nat ? local.cloud_router_regions : [] - - # this comprehension should have 1 and only 1 match - output_primary_subnetwork = one([for k, v in module.vpc.subnets : v if k == "${local.subnetworks[0].subnet_region}/${local.subnetworks[0].subnet_name}"]) - output_primary_subnetwork_name = local.output_primary_subnetwork.name - output_primary_subnetwork_self_link = local.output_primary_subnetwork.self_link - output_primary_subnetwork_ip_cidr_range = local.output_primary_subnetwork.ip_cidr_range - - iap_ports = distinct(concat(compact([ - var.enable_iap_rdp_ingress ? "3389" : "", - var.enable_iap_ssh_ingress ? "22" : "", - var.enable_iap_winrm_ingress ? "5986" : "", - ]), var.extra_iap_ports)) - - firewall_log_api_values = { - "DISABLE_LOGGING" = null - "INCLUDE_ALL_METADATA" = { metadata = "INCLUDE_ALL_METADATA" }, - "EXCLUDE_ALL_METADATA" = { metadata = "EXCLUDE_ALL_METADATA" }, - } - firewall_log_config = lookup(local.firewall_log_api_values, var.firewall_log_config, null) - - allow_iap_ingress = { - name = "${local.network_name}-fw-allow-iap-ingress" - description = "allow TCP access via Identity-Aware Proxy" - direction = "INGRESS" - priority = null - ranges = ["35.235.240.0/20"] - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = local.iap_ports - }] - deny = [] - log_config = local.firewall_log_config - } - - allow_ssh_ingress = { - name = "${local.network_name}-fw-allow-ssh-ingress" - description = "allow SSH access" - direction = "INGRESS" - priority = null - ranges = var.allowed_ssh_ip_ranges - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = ["22"] - }] - deny = [] - log_config = local.firewall_log_config - } - - allow_internal_traffic = { - name = "${local.network_name}-fw-allow-internal-traffic" - priority = null - description = "allow traffic between nodes of this VPC" - direction = "INGRESS" - ranges = [var.network_address_range] - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = ["0-65535"] - }, { - protocol = "udp" - ports = ["0-65535"] - }, { - protocol = "icmp" - ports = null - }, - ] - deny = [] - log_config = local.firewall_log_config - } - - firewall_rules = concat( - var.firewall_rules, - length(var.allowed_ssh_ip_ranges) > 0 ? [local.allow_ssh_ingress] : [], - var.enable_internal_traffic ? [local.allow_internal_traffic] : [], - length(local.iap_ports) > 0 ? [local.allow_iap_ingress] : [] - ) - - secondary_ranges_map = { - for secondary_range in var.secondary_ranges_list : - secondary_range.subnetwork_name => secondary_range.ranges - } -} - -resource "terraform_data" "network_profile_firewall_validation" { - lifecycle { - precondition { - condition = !(try(strcontains(var.network_profile, "roce"), false) && length(local.firewall_rules) > 0) - error_message = "If var.network_profile contains 'roce', var.firewall_rules must be empty and var.enable_internal_traffic must be false, please see: https://cloud.google.com/vpc/docs/rdma-network-profiles#additional_features_that_dont_apply_to_traffic_from_rdma_nics" - } - } -} - -module "vpc" { - source = "terraform-google-modules/network/google" - version = "~> 12.0" - - depends_on = [terraform_data.network_profile_firewall_validation] - - network_name = local.network_name - project_id = var.project_id - auto_create_subnetworks = false - subnets = local.subnetworks - secondary_ranges = length(local.secondary_ranges_map) > 0 ? local.secondary_ranges_map : var.secondary_ranges - routing_mode = var.network_routing_mode - mtu = var.mtu - description = var.network_description - shared_vpc_host = var.shared_vpc_host - delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes - firewall_rules = local.firewall_rules - network_profile = var.network_profile -} - -resource "terraform_data" "cloud_nat_validation" { - lifecycle { - precondition { - condition = var.enable_cloud_router == true || var.enable_cloud_nat == false - error_message = <<-EOD - "Cannot have Cloud NAT without a Cloud Router. If you desire Cloud NAT functionality please set `enable_cloud_router` to true." - EOD - } - } -} - -# This use of the module may appear odd when var.ips_per_nat = 0. The module -# will be called for all regions with subnetworks but names will be set to the -# empty list. This is a perfectly valid value (the default!). In this scenario, -# no IP addresses are created and all module outputs are empty lists. -# -# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/variables.tf#L27 -# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/outputs.tf -module "nat_ip_addresses" { - source = "terraform-google-modules/address/google" - version = "~> 4.1" - - depends_on = [terraform_data.cloud_nat_validation] - - for_each = toset(local.cloud_nat_regions) - - project_id = var.project_id - region = each.value - # an external, regional (not global) IP address is suited for a regional NAT - address_type = "EXTERNAL" - global = false - labels = local.labels - names = [for idx in range(var.ips_per_nat) : "${local.network_name}-nat-ips-${each.value}-${idx}"] -} - -module "cloud_router" { - source = "terraform-google-modules/cloud-router/google" - version = "~> 7.3" - - depends_on = [terraform_data.cloud_nat_validation] - - for_each = toset(local.cloud_router_regions) - - project = var.project_id - name = "${local.network_name}-router" - region = each.value - network = module.vpc.network_name - # in scenario with no NAT IPs, no NAT is created even if router is created - # https://github.com/terraform-google-modules/terraform-google-cloud-router/blob/v2.0.0/nat.tf#L18-L20 - nats = length(module.nat_ip_addresses[each.value].self_links) == 0 ? [] : [ - { - name : "cloud-nat-${each.value}", - nat_ips : module.nat_ip_addresses[each.value].self_links - }, - ] -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/network/vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/outputs.tf b/deletion-test/build_script/modules/embedded/modules/network/vpc/outputs.tf deleted file mode 100644 index c2ee6bdf6b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/vpc/outputs.tf +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the new VPC network" - value = module.vpc.network_name - depends_on = [module.vpc, module.cloud_router] -} - -output "network_id" { - description = "ID of the new VPC network" - value = module.vpc.network_id - depends_on = [module.vpc, module.cloud_router] -} - -output "network_self_link" { - description = "Self link of the new VPC network" - value = module.vpc.network_self_link - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetworks" { - description = "Full list of subnetwork objects belonging to the new VPC network" - value = module.vpc.subnets - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork" { - description = "Primary subnetwork object" - value = local.output_primary_subnetwork - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_name" { - description = "Name of the primary subnetwork" - value = local.output_primary_subnetwork_name - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_self_link" { - description = "Self link of the primary subnetwork" - value = local.output_primary_subnetwork_self_link - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_address" { - description = "IP address range of the primary subnetwork" - value = local.output_primary_subnetwork_ip_cidr_range - depends_on = [module.vpc, module.cloud_router] -} - -output "nat_ips" { - description = "External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used)" - value = flatten([for ipmod in module.nat_ip_addresses : ipmod.addresses]) -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/variables.tf b/deletion-test/build_script/modules/embedded/modules/network/vpc/variables.tf deleted file mode 100644 index e036189404..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/vpc/variables.tf +++ /dev/null @@ -1,301 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "labels" { - description = "Labels to add to network resources that support labels. Key-value pairs of strings." - type = map(string) - default = {} - nullable = false -} - -variable "network_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" - type = string - default = null -} - -variable "subnetwork_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-primary-subnet\")" - type = string - default = null -} - -# tflint-ignore: terraform_unused_declarations -variable "subnetwork_size" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = number - default = null - validation { - condition = var.subnetwork_size == null - error_message = "subnetwork_size is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -variable "default_primary_subnetwork_size" { - description = "The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks" - type = number - default = 15 -} - -variable "region" { - description = "The default region for Cloud resources" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "network_address_range" { - description = "IP address range (CIDR) for global network" - type = string - default = "10.0.0.0/9" - - validation { - condition = can(cidrhost(var.network_address_range, 0)) - error_message = "IP address range must be in CIDR format." - } -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "subnetworks" { - description = <<-EOT - List of subnetworks to create within the VPC. If left empty, it will be - replaced by a single, default subnetwork constructed from other parameters - (e.g. var.region). In all cases, the first subnetwork in the list is identified - by outputs as a "primary" subnetwork. - - subnet_name (string, required, name of subnet) - subnet_region (string, required, region of subnet) - subnet_ip (string, mutually exclusive with new_bits, CIDR-formatted IP range for subnetwork) - new_bits (number, mutually exclusive with subnet_ip, CIDR bits used to calculate subnetwork range) - subnet_private_access (bool, optional, Enable Private Access on subnetwork) - subnet_flow_logs (map(string), optional, Configure Flow Logs see terraform-google-network module) - description (string, optional, Description of Network) - purpose (string, optional, related to Load Balancing) - role (string, optional, related to Load Balancing) - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_name"]) - ]) - error_message = "All subnetworks must define \"subnet_name\"." - } - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_region"]) - ]) - error_message = "All subnetworks must define \"subnet_region\"." - } - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_ip"]) != can(s["new_bits"]) - ]) - error_message = "All subnetworks must define exactly one of \"subnet_ip\" or \"new_bits\"." - } - validation { - condition = alltrue([for s in var.subnetworks : can(s["subnet_ip"])]) || alltrue([for s in var.subnetworks : can(s["new_bits"])]) - error_message = "All subnetworks must make same choice of \"subnet_ip\" or \"new_bits\"." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "primary_subnetwork" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = map(string) - default = null - validation { - condition = var.primary_subnetwork == null - error_message = "primary_subnetwork is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "additional_subnetworks" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = list(map(string)) - default = null - validation { - condition = var.additional_subnetworks == null - error_message = "additional_subnetworks is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -variable "secondary_ranges" { - type = map(list(object({ range_name = string, ip_cidr_range = string }))) - description = <<-EOT - "Secondary ranges associated with the subnets. - This will be deprecated in favour of secondary_ranges_list at a later date. - Please migrate to using the same." - EOT - default = {} -} - -variable "secondary_ranges_list" { - type = list(object({ - subnetwork_name = string, - ranges = list(object({ - range_name = string, - ip_cidr_range = string - })) - })) - description = <<-EOT - "List of secondary ranges associated with the subnetworks. - Each subnetwork must be specified at most once in this list." - EOT - default = [] - validation { - condition = (length(var.secondary_ranges_list[*].subnetwork_name) == - length(distinct(var.secondary_ranges_list[*].subnetwork_name))) - error_message = "Each subnetwork should be specified at most once in this list. Remove any duplicates." - } -} - -variable "network_routing_mode" { - type = string - default = "GLOBAL" - description = "The network routing mode (default \"GLOBAL\")" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "ips_per_nat" { - type = number - description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples" - default = 2 -} - -variable "shared_vpc_host" { - type = bool - description = "Makes this project a Shared VPC host if 'true' (default 'false')" - default = false -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_iap_ssh_ingress" { - type = bool - description = "Enable a firewall rule to allow SSH access using IAP tunnels" - default = true -} - -variable "enable_iap_rdp_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" - default = false -} - -variable "enable_iap_winrm_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" - default = false -} - -variable "enable_internal_traffic" { - type = bool - description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" - default = true -} - -variable "enable_cloud_router" { - type = bool - description = "Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview" - default = true -} - -variable "enable_cloud_nat" { - type = bool - description = "Enable the creation of Cloud NATs." - default = true -} - -variable "extra_iap_ports" { - type = list(string) - description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" - default = [] -} - -variable "allowed_ssh_ip_ranges" { - type = list(string) - description = "A list of CIDR IP ranges from which to allow ssh access" - default = [] - - validation { - condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) - error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." - } -} - -variable "firewall_rules" { - type = any - description = "List of firewall rules" - default = [] -} - -variable "firewall_log_config" { - type = string - description = "Firewall log configuration for Toolkit firewall rules (var.enable_iap_ssh_ingress and others)" - default = "DISABLE_LOGGING" - nullable = false - - validation { - condition = contains([ - "INCLUDE_ALL_METADATA", - "EXCLUDE_ALL_METADATA", - "DISABLE_LOGGING", - ], var.firewall_log_config) - error_message = "var.firewall_log_config must be set to \"DISABLE_LOGGING\", or enable logging with \"INCLUDE_ALL_METADATA\" or \"EXCLUDE_ALL_METADATA\"" - } -} - -variable "network_profile" { - type = string - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown - EOT - default = null -} diff --git a/deletion-test/build_script/modules/embedded/modules/network/vpc/versions.tf b/deletion-test/build_script/modules/embedded/modules/network/vpc/versions.tf deleted file mode 100644 index 71b7106734..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/network/vpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 0.15.0" -} diff --git a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/README.md b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/README.md deleted file mode 100644 index 192d7575a5..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/README.md +++ /dev/null @@ -1,320 +0,0 @@ -# Custom Images in the Cluster Toolkit (formerly HPC Toolkit) - -Please review the -[introduction to image building](../../../docs/image-building.md) for general -information on building custom images using the Toolkit. - -## Introduction - -This module uses [Packer](https://www.packer.io/) to create an image within an -Cluster Toolkit deployment. Packer operates by provisioning a short-lived VM in -Google Cloud on which it executes scripts to customize the boot disk for -repeated use. The VM's boot disk is specified from a source image that defaults -to the [HPC VM Image][hpcimage]. This Packer "template" supports customization -by the following approaches following a [recommended use](#recommended-use): - -- [startup-script metadata][startup-metadata] from [raw string][sss] or - [file][ssf] -- [Shell scripts][shell] uploaded from the Packer execution environment to the - VM -- [Ansible playbooks][ansible] uploaded from the Packer execution environment to - the VM - -They can be specified independently of one another, so that anywhere from 1 to 3 -solutions can be used simultaneously. In the case that 0 scripts are supplied, -the source boot disk is effectively copied to your project without -customization. This can be useful in scenarios where increased control over the -image maintenance lifecycle is desired or when policies restrict the use of -images to internal projects. - -## Minimum requirements - -### Outbound internet access - -Most customization scripts require access to resources on the public internet. -This can be achieved by one of the following 2 approaches: - -1. Using a public IP address on the VM - -- Set [var.omit_external_ip](#input_omit_external_ip) to `false` - -1. Configuring a VPC with a Cloud NAT in the region of the VM - -- Use the [vpc] module which automates NAT creation - -### Inbound internet access - -Read [order of execution](#order-of-execution) below for a discussion of VM -customization solutions and their requirements for inbound SSH access. -[Environments without SSH access](#environments-without-ssh-access) should use -the metadata-based startup-script solution. - -A simple way to enable inbound SSH access is to use the VPC module with -`allowed_ssh_ip_ranges` set to `0.0.0.0/0`. - -### User or service account executing Packer at command line - -The user or service account running Packer must have the permission to create -VMs in the selected VPC network and, if [use\_iap](#input_use_iap) is set, must -have the "IAP-Secured Tunnel User" role. Recommended roles are: - -- `roles/compute.instanceAdmin.v1` -- `roles/iap.tunnelResourceAccessor` - -### VM service account roles - -The service account attached to the temporary build VM created by Packer should -have the ability to write Cloud Logging entries so that you may inspect and -debug build logs. When using the metadata startup-script customization solution, -the service account attached to the temporary build VM created by Packer must -have the permission to modify its own metadata and to read from Cloud Storage -buckets. Recommended roles are: - -- `roles/compute.instanceAdmin.v1` -- `roles/iam.serviceAccountUser` -- `roles/logging.logWriter` -- `roles/monitoring.metricWriter` -- `roles/storage.objectViewer` - -It is recommended to create this service account as a separate step outside a -blueprint due to known delay in [IAM bindings propagation][iamprop]. - -## Example blueprints - -A recommended pattern for building images with this module is to use the -terraform based [startup-script] module along with this packer custom-image -module. Below you can find links to several examples of this pattern, including -usage instructions. - -### [Image Builder] - -The [Image Builder] blueprint demonstrates a solution that builds an image -using: - -- The [HPC VM Image][hpcimage] as a base upon which to customize -- A VPC network with firewall rules that allow IAP-based SSH tunnels -- A Toolkit runner that installs a custom script - -Please review the [examples README] for usage instructions. - -## Order of execution - -The startup script specified in metadata executes in parallel with the other -supported methods. However, the remaining methods execute in a well-defined -order relative to one another. - -1. All shell scripts will execute in the configured order -1. After shell scripts complete, all Ansible playbooks will execute in the - configured order - -> **_NOTE:_** if both [startup_script][sss] and [startup_script_file][ssf] are -> specified, then [startup_script_file][ssf] takes precedence. - -## Recommended use - -Because the [metadata startup script executes in parallel](#order-of-execution) -with the other solutions, conflicts can arise, especially when package managers -(`yum` or `apt`) lock their databases during package installation. Therefore, it -is recommended to choose one of the following approaches: - -1. Specify _either_ [startup_script][sss] _or_ [startup_script_file][ssf] and do - not specify [shell_scripts][shell] or [ansible_playbooks][ansible]. - - This can be especially useful in - [environments that restrict SSH access](#environments-without-ssh-access) -1. Specify any combination of [shell_scripts][shell] and - [ansible_playbooks][ansible] and do not specify [startup_script][sss] or - [startup_script_file][ssf]. - -If any of the startup script approaches fail by returning a code other than 0, -Packer will determine that the build has failed and refuse to save the image. - -## External access with SSH - -The [shell scripts][shell] and [Ansible playbooks][ansible] customization -solutions both require SSH access to the VM from the Packer execution -environment. SSH access can be enabled one of 2 ways: - -1. The VM is created without a public IP address and SSH tunnels are created - using [Identity-Aware Proxy (IAP)][iaptunnel]. - - Allow [use_iap](#input_use_iap) to take on its default value of `true` -1. The VM is created with an IP address on the public internet and firewall - rules allow SSH access from the Packer execution environment. - - Set `omit_external_ip = false` (or `omit_external_ip: false` in a - blueprint) - - Add firewall rules that open SSH to the VM - -The Packer template defaults to using to the 1st IAP-based solution because it -is more secure (no exposure to public internet) and because the [vpc] module -automatically sets up all necessary firewall rules for SSH tunneling and -outbound-only access to the internet through [Cloud NAT][cloudnat]. - -In either SSH solution, customization scripts should be supplied as files in the -[shell_scripts][shell] and [ansible_playbooks][ansible] settings. - -## Environments without SSH access - -Many network environments disallow SSH access to VMs. In these environments, the -[metadata-based startup scripts][startup-metadata] are appropriate because they -execute entirely independently of the Packer execution environment. - -In this scenario, a single scripts should be supplied in the form of a string to -the [startup_script][sss] input variable. This solution integrates well with -Toolkit runners. Runners operate by using a single startup script whose behavior -is extended by downloading and executing a customizable set of runners from -Cloud Storage at startup. - -> **_NOTE:_** Packer will attempt to use SSH if either [shell_scripts][shell] or -> [ansible_playbooks][ansible] are set to non-empty values. Leave them at their -> default, empty values to ensure access by SSH is disabled. - -## Supplying startup script as a string - -The [startup_script][sss] parameter accepts scripts formatted as strings. In -Packer and Terraform, multi-line strings can be specified using -[heredoc syntax](https://www.terraform.io/language/expressions/strings#heredoc-strings) -in an input [Packer variables file][pkrvars] (`*.pkrvars.hcl`) For example, the -following snippet defines a multi-line bash script followed by an integer -representing the size, in GiB, of the resulting image: - -```hcl -startup_script = <<-EOT - #!/bin/bash - yum install -y epel-release - yum install -y jq - EOT - -disk_size = 100 -``` - -In a blueprint, the equivalent syntax is: - -```yaml -... - settings: - startup_script: | - #!/bin/bash - yum install -y epel-release - yum install -y jq - disk_size: 100 -... -``` - -## Monitoring startup script execution - -When using startup script customization, Packer will print very limited output -to the console. For example: - -```text -==> example.googlecompute.toolkit_image: Waiting for any running startup script to finish... -==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... -==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... -==> example.googlecompute.toolkit_image: Startup script, if any, has finished running. -``` - -### Debugging startup-script failures - -> [!NOTE] -> There can be a delay in the propagation of the logs from the instance to -> Cloud Logging, so it may require waiting a few minutes to see the full logs. - -If the Packer image build fails, the module will output a `gcloud` command -that can be used directly to review startup-script execution. - -## License - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. You may obtain a copy of the -License at - -```text - http://www.apache.org/licenses/LICENSE-2.0 -``` - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. - - -## Requirements - -No requirements. - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_count](#input\_accelerator\_count) | Number of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `number` | `null` | no | -| [accelerator\_type](#input\_accelerator\_type) | Type of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `string` | `null` | no | -| [ansible\_playbooks](#input\_ansible\_playbooks) | A list of Ansible playbook configurations that will be uploaded to customize the VM image | 
list(object({
    playbook_file   = string
    galaxy_file     = string
    extra_arguments = list(string)
  }))
| `[]` | no | -| [communicator](#input\_communicator) | Communicator to use for provisioners that require access to VM ("ssh" or "winrm") | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name | `string` | n/a | yes | -| [disk\_size](#input\_disk\_size) | Size of disk image in GB | `number` | `null` | no | -| [disk\_type](#input\_disk\_type) | Type of persistent disk to provision | `string` | `"pd-balanced"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [image\_family](#input\_image\_family) | The family name of the image to be built. Defaults to `deployment_name` | `string` | `null` | no | -| [image\_name](#input\_image\_name) | The name of the image to be built. If not supplied, it will be set to image\_family-$ISO\_TIMESTAMP | `string` | `null` | no | -| [image\_storage\_locations](#input\_image\_storage\_locations) | Storage location, either regional or multi-regional, where snapshot content is to be stored and only accepts 1 value.
See https://developer.hashicorp.com/packer/plugins/builders/googlecompute#image_storage_locations | `list(string)` | `null` | no | -| [labels](#input\_labels) | Labels to apply to the short-lived VM | `map(string)` | `null` | no | -| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2-standard-4"` | no | -| [manifest\_file](#input\_manifest\_file) | File to which to write Packer build manifest | `string` | `"packer-manifest.json"` | no | -| [metadata](#input\_metadata) | Instance metadata for the builder VM (use var.startup\_script or var.startup\_script\_file to set startup-script metadata) | `map(string)` | `{}` | no | -| [network\_project\_id](#input\_network\_project\_id) | Project ID of Shared VPC network | `string` | `null` | no | -| [omit\_external\_ip](#input\_omit\_external\_ip) | Provision the image building VM without a public IP address | `bool` | `true` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which to create VM and image | `string` | n/a | yes | -| [scopes](#input\_scopes) | DEPRECATED: use var.service\_account\_scopes | `set(string)` | `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | The service account email to use. If null or 'default', then the default Compute Engine service account will be used. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Service account scopes to attach to the instance. See
https://cloud.google.com/compute/docs/access/service-accounts. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shell\_scripts](#input\_shell\_scripts) | A list of paths to local shell scripts which will be uploaded to customize the VM image | `list(string)` | `[]` | no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [source\_image](#input\_source\_image) | Source OS image to build from | `string` | `null` | no | -| [source\_image\_family](#input\_source\_image\_family) | Alternative to source\_image. Specify image family to build from latest image in family | `string` | `"hpc-rocky-linux-8"` | no | -| [source\_image\_project\_id](#input\_source\_image\_project\_id) | A list of project IDs to search for the source image. Packer will search the
first project ID in the list first, and fall back to the next in the list,
until it finds the source image. | `list(string)` | `null` | no | -| [ssh\_username](#input\_ssh\_username) | Username to use for SSH access to VM | `string` | `"hpc-toolkit-packer"` | no | -| [startup\_script](#input\_startup\_script) | Startup script (as raw string) used to build the custom Linux VM image (overridden by var.startup\_script\_file if both are set) | `string` | `null` | no | -| [startup\_script\_file](#input\_startup\_script\_file) | File path to local shell script that will be used to customize the Linux VM image (overrides var.startup\_script) | `string` | `null` | no | -| [state\_timeout](#input\_state\_timeout) | The time to wait for instance state changes, including image creation | `string` | `"10m"` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of subnetwork in which to provision image building VM | `string` | n/a | yes | -| [tags](#input\_tags) | Assign network tags to apply firewall rules to VM instance | `list(string)` | `null` | no | -| [use\_iap](#input\_use\_iap) | Use IAP proxy when connecting by SSH | `bool` | `true` | no | -| [use\_os\_login](#input\_use\_os\_login) | Use OS Login when connecting by SSH | `bool` | `false` | no | -| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | A list of strings containing PowerShell scripts which will customize a Windows VM image (requires WinRM communicator) | `list(string)` | `[]` | no | -| [wrap\_startup\_script](#input\_wrap\_startup\_script) | Wrap startup script with Packer-generated wrapper | `bool` | `true` | no | -| [zone](#input\_zone) | Cloud zone in which to provision image building VM | `string` | n/a | yes | - -## Outputs - -No outputs. - - -[ansible]: #input_ansible_playbooks -[cloudnat]: https://cloud.google.com/nat/docs/overview -[examples readme]: ../../../examples/README.md#image-builderyaml- -[hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[iamprop]: https://cloud.google.com/iam/docs/access-change-propagation -[iaptunnel]: https://cloud.google.com/iap/docs/using-tcp-forwarding -[image builder]: ../../../examples/image-builder.yaml -[logging-console]: https://console.cloud.google.com/logs/ -[logging-read-docs]: https://cloud.google.com/sdk/gcloud/reference/logging/read -[pkrvars]: https://www.packer.io/guides/hcl/variables#from-a-file -[shell]: #input_shell_scripts -[ssf]: #input_startup_script_file -[sss]: #input_startup_script -[startup-metadata]: https://cloud.google.com/compute/docs/instances/startup-scripts/linux -[startup-script]: ../../../modules/scripts/startup-script -[vpc]: ../../network/vpc/README.md diff --git a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/image.pkr.hcl b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/image.pkr.hcl deleted file mode 100644 index 9282cf7433..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/image.pkr.hcl +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "custom-image", ghpc_role = "packer" }) - - # construct a unique image name from the image family - image_family = var.image_family != null ? var.image_family : var.deployment_name - image_name_default = "${local.image_family}-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" - image_name = var.image_name != null ? var.image_name : local.image_name_default - - # construct vm image name for use when getting logs - instance_name = "packer-${substr(uuidv4(), 0, 6)}" - - # default to explicit var.communicator, otherwise in-order: ssh/winrm/none - shell_script_communicator = length(var.shell_scripts) > 0 ? "ssh" : "" - ansible_playbook_communicator = length(var.ansible_playbooks) > 0 ? "ssh" : "" - powershell_script_communicator = length(var.windows_startup_ps1) > 0 ? "winrm" : "" - communicator = coalesce( - var.communicator, - local.shell_script_communicator, - local.ansible_playbook_communicator, - local.powershell_script_communicator, - "none" - ) - - # must not enable IAP when no communicator is in use - use_iap = local.communicator == "none" ? false : var.use_iap - - # construct metadata from startup_script and metadata variables - startup_script_metadata = var.startup_script == null ? {} : { startup-script = var.startup_script } - - linux_user_metadata = { - block-project-ssh-keys = "TRUE" - shutdown-script = <<-EOT - #!/bin/bash - userdel -r ${var.ssh_username} - sed -i '/${var.ssh_username}/d' /var/lib/google/google_users - EOT - } - windows_packer_user = "packer_user" - windows_user_metadata = { - sysprep-specialize-script-cmd = "winrm quickconfig -quiet & net user /add ${local.windows_packer_user} & net localgroup administrators ${local.windows_packer_user} /add & winrm set winrm/config/service/auth @{Basic=\\\"true\\\"}" - windows-shutdown-script-cmd = <<-EOT - net user /delete ${local.windows_packer_user} - EOT - } - user_metadata = local.communicator == "winrm" ? local.windows_user_metadata : local.linux_user_metadata - - # merge metadata such that var.metadata always overrides user management - # metadata but always allow var.startup_script to override var.metadata - metadata = merge( - local.user_metadata, - var.metadata, - local.startup_script_metadata, - ) - - # determine best value for on_host_maintenance if not supplied by user - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - gpu_attached = contains(["a2", "g2"], local.machine_family) || var.accelerator_type != null - on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" - on_host_maintenance = ( - var.on_host_maintenance != null - ? var.on_host_maintenance - : local.on_host_maintenance_default - ) - - accelerator_type = var.accelerator_type == null ? null : "projects/${var.project_id}/zones/${var.zone}/acceleratorTypes/${var.accelerator_type}" - - winrm_username = local.communicator == "winrm" ? "packer_user" : null - winrm_insecure = local.communicator == "winrm" ? true : null - winrm_use_ssl = local.communicator == "winrm" ? true : null - - enable_integrity_monitoring = var.enable_shielded_vm && var.shielded_instance_config.enable_integrity_monitoring - enable_secure_boot = var.enable_shielded_vm && var.shielded_instance_config.enable_secure_boot - enable_vtpm = var.enable_shielded_vm && var.shielded_instance_config.enable_vtpm - - image_licenses = [ - "projects/click-to-deploy-images/global/licenses/hpc-toolkit-vm-image" - ] -} - -source "googlecompute" "toolkit_image" { - communicator = local.communicator - project_id = var.project_id - image_name = local.image_name - image_family = local.image_family - image_labels = local.labels - instance_name = local.instance_name - machine_type = var.machine_type - accelerator_type = local.accelerator_type - accelerator_count = var.accelerator_count - on_host_maintenance = local.on_host_maintenance - disk_size = var.disk_size - disk_type = var.disk_type - omit_external_ip = var.omit_external_ip - use_internal_ip = var.omit_external_ip - subnetwork = var.subnetwork_name - network_project_id = var.network_project_id - service_account_email = var.service_account_email - scopes = var.service_account_scopes - source_image = var.source_image - source_image_family = var.source_image_family - source_image_project_id = var.source_image_project_id - ssh_username = var.ssh_username - tags = var.tags - use_iap = local.use_iap - use_os_login = var.use_os_login - winrm_username = local.winrm_username - winrm_insecure = local.winrm_insecure - winrm_use_ssl = local.winrm_use_ssl - zone = var.zone - labels = local.labels - metadata = local.metadata - startup_script_file = var.startup_script_file - wrap_startup_script = var.wrap_startup_script - state_timeout = var.state_timeout - image_storage_locations = var.image_storage_locations - enable_secure_boot = local.enable_secure_boot - enable_vtpm = local.enable_vtpm - enable_integrity_monitoring = local.enable_integrity_monitoring - image_licenses = local.image_licenses -} - -build { - name = var.deployment_name - sources = ["sources.googlecompute.toolkit_image"] - - # using dynamic blocks to create provisioners ensures that there are no - # provisioner blocks when none are provided and we can use the none - # communicator when using startup-script - - # provisioner "shell" blocks - dynamic "provisioner" { - labels = ["shell"] - for_each = var.shell_scripts - content { - execute_command = "sudo -H sh -c '{{ .Vars }} {{ .Path }}'" - script = provisioner.value - } - } - - # provisioner "powershell" blocks - dynamic "provisioner" { - labels = ["powershell"] - for_each = var.windows_startup_ps1 - content { - inline = split("\n", provisioner.value) - } - } - - dynamic "provisioner" { - labels = ["powershell"] - for_each = length(var.windows_startup_ps1) > 0 ? [1] : [] - content { - inline = [ - "GCESysprep -no_shutdown" - ] - } - } - - # provisioner "ansible-local" blocks - # this installs custom roles/collections from ansible-galaxy in /home/packer - # which will be removed at the end; consider modifying /etc/ansible/ansible.cfg - dynamic "provisioner" { - labels = ["ansible-local"] - for_each = var.ansible_playbooks - content { - playbook_file = provisioner.value.playbook_file - galaxy_file = provisioner.value.galaxy_file - extra_arguments = provisioner.value.extra_arguments - } - } - - post-processor "manifest" { - output = var.manifest_file - strip_path = true - custom_data = { - built-by = "cloud-hpc-toolkit" - } - } - - # If there is an error during image creation, print out command for getting packer VM logs - error-cleanup-provisioner "shell-local" { - environment_vars = [ - "PRJ_ID=${var.project_id}", - "INST_NAME=${local.instance_name}", - "ZONE=${var.zone}", - ] - inline_shebang = "/bin/bash -e" - inline = [ - "type -P gcloud > /dev/null || exit 0", - "INST_ID=$(gcloud compute instances describe $INST_NAME --project $PRJ_ID --format=\"value(id)\" --zone=$ZONE)", - "echo 'Error building image try checking logs:'", - join(" ", ["echo \"gcloud logging --project $PRJ_ID read", - "'logName=(\\\"projects/$PRJ_ID/logs/GCEMetadataScripts\\\" OR \\\"projects/$PRJ_ID/logs/google_metadata_script_runner\\\") AND resource.labels.instance_id=$INST_ID'", - "--format=\\\"table(timestamp, resource.labels.instance_id, jsonPayload.message)\\\"", - "--order=asc\"" - ] - ) - ] - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/metadata.yaml deleted file mode 100644 index 23108c4e17..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - logging.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/variables.pkr.hcl b/deletion-test/build_script/modules/embedded/modules/packer/custom-image/variables.pkr.hcl deleted file mode 100644 index 3cede102ce..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/packer/custom-image/variables.pkr.hcl +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "deployment_name" { - description = "Cluster Toolkit deployment name" - type = string -} - -variable "project_id" { - description = "Project in which to create VM and image" - type = string -} - -variable "machine_type" { - description = "VM machine type on which to build new image" - type = string - default = "n2-standard-4" -} - -variable "disk_size" { - description = "Size of disk image in GB" - type = number - default = null -} - -variable "disk_type" { - description = "Type of persistent disk to provision" - type = string - default = "pd-balanced" -} - -variable "zone" { - description = "Cloud zone in which to provision image building VM" - type = string -} - -variable "network_project_id" { - description = "Project ID of Shared VPC network" - type = string - default = null -} - -variable "subnetwork_name" { - description = "Name of subnetwork in which to provision image building VM" - type = string -} - -variable "omit_external_ip" { - description = "Provision the image building VM without a public IP address" - type = bool - default = true -} - -variable "tags" { - description = "Assign network tags to apply firewall rules to VM instance" - type = list(string) - default = null -} - -variable "image_family" { - description = "The family name of the image to be built. Defaults to `deployment_name`" - type = string - default = null -} - -variable "image_name" { - description = "The name of the image to be built. If not supplied, it will be set to image_family-$ISO_TIMESTAMP" - type = string - default = null -} - -variable "source_image_project_id" { - description = < -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 4.0 | -| [local](#requirement\_local) | >= 2.0.0 | -| [null](#requirement\_null) | ~> 3.0 | -| [random](#requirement\_random) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.0 | -| [local](#provider\_local) | >= 2.0.0 | -| [null](#provider\_null) | ~> 3.0 | -| [random](#provider\_random) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance\_template](#module\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [local_file.submit_script](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [null_resource.submit_job](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.submit_job_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used for the job\_id | `string` | n/a | yes | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs | `bool` | `true` | no | -| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions. Valid inputs are `"alpha"`, `"beta"` and "" (empty string for default version) | `string` | `""` | no | -| [image](#input\_image) | DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided. | `any` | `null` | no | -| [instance\_image](#input\_instance\_image) | Google Cloud Batch compute node image. Ignored if `instance_template` is provided.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [instance\_template](#input\_instance\_template) | Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions. | `string` | `null` | no | -| [job\_filename](#input\_job\_filename) | The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified | `string` | `null` | no | -| [job\_id](#input\_job\_id) | An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs. | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided. | `map(string)` | n/a | yes | -| [log\_policy](#input\_log\_policy) | Create a block to define log policy.
When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging.
When set to `PATH`, path must be added to generated template.
When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. | `string` | `"CLOUD_LOGGING"` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided. | `string` | `"n2-standard-4"` | no | -| [mpi\_mode](#input\_mpi\_mode) | Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`. | `bool` | `false` | no | -| [native\_batch\_mounting](#input\_native\_batch\_mounting) | Batch can mount some fs\_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts. | `bool` | `true` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which to run the Google Cloud Batch job | `string` | n/a | yes | -| [runnable](#input\_runnable) | A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`. | `string` | `null` | no | -| [runnables](#input\_runnables) | A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template. | 
list(object({
    script = string
  }))
| `null` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/devstorage.read_only",
    "https://www.googleapis.com/auth/logging.write",
    "https://www.googleapis.com/auth/monitoring.write",
    "https://www.googleapis.com/auth/servicecontrol",
    "https://www.googleapis.com/auth/service.management.readonly",
    "https://www.googleapis.com/auth/trace.append"
  ]
}
| no | -| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided. | `string` | `null` | no | -| [submit](#input\_submit) | When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply. | `bool` | `false` | no | -| [subnetwork](#input\_subnetwork) | The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided. | `any` | `null` | no | -| [task\_count](#input\_task\_count) | Number of parallel tasks | `number` | `1` | no | -| [task\_count\_per\_node](#input\_task\_count\_per\_node) | Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value. | `number` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [gcloud\_version](#output\_gcloud\_version) | The version of gcloud to be used. | -| [instance\_template](#output\_instance\_template) | Instance template used by the Batch job. | -| [instructions](#output\_instructions) | Instructions for submitting the Batch job. | -| [job\_data](#output\_job\_data) | All data associated with the defined job, typically provided as input to clout-batch-login-node. | -| [network\_storage](#output\_network\_storage) | An array of network attached storage mounts used by the Batch job. | -| [startup\_script](#output\_startup\_script) | Startup script run before Google Cloud Batch job starts. | - diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/main.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/main.tf deleted file mode 100644 index 0d681536c9..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/main.tf +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "batch-job-template", ghpc_role = "scheduler" }) -} - -locals { - instance_template = coalesce(var.instance_template, module.instance_template.self_link) - - tasks_per_node = var.task_count_per_node != null ? var.task_count_per_node : (var.mpi_mode ? 1 : null) - - one_line_runnable = coalesce(var.runnable, "## Add your workload here ##") - runnables = coalesce(var.runnables, [{ script = local.one_line_runnable }]) - - job_template_contents = templatefile( - "${path.module}/templates/batch-job-base.yaml.tftpl", - { - synchronized = var.mpi_mode - runnables = local.runnables - task_count = var.task_count - tasks_per_node = local.tasks_per_node - require_hosts_file = var.mpi_mode - permissive_ssh = var.mpi_mode - log_policy = var.log_policy - instance_template = local.instance_template - nfs_volumes = local.native_batch_network_storage - labels = local.labels - } - ) - - submit_job_id = "${var.job_id}-${random_id.submit_job_suffix.hex}" - job_filename = coalesce(var.job_filename, "${var.job_id}.yaml") - job_template_output_path = "${path.root}/${local.job_filename}" - - submit_script_contents = templatefile( - "${path.module}/templates/batch-submit.sh.tftpl", - { - project = var.project_id - location = var.region - config = local_file.job_template.filename - submit_job_id = local.submit_job_id - } - ) - submit_script_output_path = "${path.root}/submit-${var.job_id}.sh" - - subnetwork_name = var.subnetwork != null ? var.subnetwork.name : "default" - subnetwork_project = var.subnetwork != null ? var.subnetwork.project : var.project_id - - # Filter network_storage for native Batch support - native_fstype = var.native_batch_mounting ? ["nfs"] : [] - native_batch_network_storage = [ - for ns in var.network_storage : - ns if contains(local.native_fstype, ns.fs_type) - ] - # other processing happens in startup_from_network_storage.tf - - # this code is similar to code in Packer and vm-instance modules - # it differs in that this module does not (yet) expose var.guest_acclerator - # for attaching GPUs to N1 VMs. For now, identify only A2 types. - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - gpu_attached = contains(["a2", "g2"], local.machine_family) - on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" - - on_host_maintenance = coalesce(var.on_host_maintenance, local.on_host_maintenance_default) - - network_storage_metadata = var.network_storage != null ? ({ network_storage = jsonencode(var.network_storage) }) : {} - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.network_storage_metadata, - local.disable_automatic_updates_metadata - ) -} - -module "instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = var.instance_template == null ? "${var.job_id}-instance-template" : "unused-template" - project_id = var.project_id - subnetwork = local.subnetwork_name - subnetwork_project = local.subnetwork_project - service_account = var.service_account - access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - labels = local.labels - - machine_type = var.machine_type - startup_script = local.startup_from_network_storage - metadata = local.metadata - source_image_family = data.google_compute_image.compute_image.family - source_image = data.google_compute_image.compute_image.name - source_image_project = data.google_compute_image.compute_image.project - on_host_maintenance = local.on_host_maintenance -} - -resource "local_file" "job_template" { - content = local.job_template_contents - filename = local.job_template_output_path - - lifecycle { - precondition { - condition = var.runnable == null || var.runnables == null - error_message = "var.runnable and var.runnables (plural) cannot both be set." - } - } -} - -resource "random_id" "submit_job_suffix" { - byte_length = 4 - keepers = { - always_run = timestamp() - } -} - -resource "local_file" "submit_script" { - content = local.submit_script_contents - filename = local.submit_script_output_path -} - -resource "null_resource" "submit_job" { - depends_on = [local_file.job_template, local_file.submit_script] - count = var.submit ? 1 : 0 - - # A new deployment should always submit a new job. Old finished jobs aren't persistent parts of - # Cloud infrastructure. - triggers = { - always_run = timestamp() - } - - provisioner "local-exec" { - command = local.submit_script_output_path - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml deleted file mode 100644 index 387e810962..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - batch.googleapis.com - - compute.googleapis.com -ghpc: - inject_module_id: job_id diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/outputs.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/outputs.tf deleted file mode 100644 index 0b1295975a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/outputs.tf +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - provided_instance_tpl_msg = "The Batch job template uses the existing VM instance template:" - generated_instance_tpl_msg = "The Batch job template uses a new VM instance template created matching the provided settings:" - submit_msg = <<-EOT - - The job has been submitted. See job status at: - https://console.cloud.google.com/batch/jobsDetail/regions/${var.region}/jobs/${local.submit_job_id}?project=${var.project_id} - EOT -} - -output "instructions" { - description = "Instructions for submitting the Batch job." - value = <<-EOT - - A Batch job template file has been created locally at: - ${abspath(local.job_template_output_path)} - - ${var.instance_template == null ? local.generated_instance_tpl_msg : local.provided_instance_tpl_msg} - ${local.instance_template} - ${var.submit ? local.submit_msg : ""} - - Use the following commands to: - Submit your job${var.submit ? " (Note: job has already been submitted)" : ""}: - gcloud ${var.gcloud_version} batch jobs submit ${local.submit_job_id} --config=${abspath(local.job_template_output_path)} --location=${var.region} --project=${var.project_id} - - Check status: - gcloud ${var.gcloud_version} batch jobs describe ${local.submit_job_id} --location=${var.region} --project=${var.project_id} | grep state: - - Delete job: - gcloud ${var.gcloud_version} batch jobs delete ${local.submit_job_id} --location=${var.region} --project=${var.project_id} - - List all jobs: - gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} - EOT -} - -output "job_data" { - description = "All data associated with the defined job, typically provided as input to clout-batch-login-node." - value = { - template_contents = local.job_template_contents, - filename = local.job_filename, - id = local.submit_job_id - } -} - -output "instance_template" { - description = "Instance template used by the Batch job." - value = local.instance_template -} - -output "network_storage" { - description = "An array of network attached storage mounts used by the Batch job." - value = var.network_storage -} - -output "startup_script" { - description = "Startup script run before Google Cloud Batch job starts." - value = var.startup_script -} - -output "gcloud_version" { - description = "The version of gcloud to be used." - value = var.gcloud_version -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf deleted file mode 100644 index 02bc58e4f7..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "inputs": -# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. -# var.network_storage : to be passed into metadata somewhere else (not here) -# var.startup_script : to be changed into a more complete file system with all the fs runners - -# "outputs": -# local.startup_from_network_storage : A full startup script with all the runners that are not supported -# natively and were included in the network_storage structure - -locals { - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - # Pull out runners to include in startup script - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - startup_script_runner = [{ - content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" - destination = "passed_startup_script.sh" - type = "shell" - }] - - full_runner_list = concat( - local.storage_client_install_runners, - local.mount_runners, - local.startup_script_runner - ) - - startup_from_network_storage = module.netstorage_startup_script.startup_script -} - -module "netstorage_startup_script" { - source = "../../scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.full_runner_list -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl deleted file mode 100644 index 83fccde53b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl +++ /dev/null @@ -1,53 +0,0 @@ -taskGroups: - - taskSpec: - runnables: - %{~ if synchronized ~} - - barrier: - name: "wait-for-node-startup" - %{~ endif ~} - %{~ for runnable in runnables ~} - - script: - text: ${indent(12, chomp(yamlencode(runnable.script)))} - %{~ if synchronized ~} - - barrier: - name: "wait-for-script-to-complete" - %{~ endif ~} - %{~ endfor ~} - %{~ if length(nfs_volumes) > 0 ~} - volumes: - %{~ for index, vol in nfs_volumes ~} - - nfs: - server: "${vol.server_ip}" - remotePath: "${vol.remote_mount}" - %{~ if vol.mount_options != "" && vol.mount_options != null ~} - mountOptions: "${vol.mount_options}" - %{~ endif ~} - mountPath: "${vol.local_mount}" - %{~ endfor ~} - %{~ endif ~} - taskCount: ${task_count} - %{~ if tasks_per_node != null ~} - taskCountPerNode: ${tasks_per_node} - %{~ endif ~} - requireHostsFile: ${require_hosts_file} - permissiveSsh: ${permissive_ssh} -%{~ if instance_template != null } -allocationPolicy: - instances: - - instanceTemplate: "${instance_template}" -%{~ endif } -%{~ if log_policy == "CLOUD_LOGGING" } -logsPolicy: - destination: "CLOUD_LOGGING" -%{ endif } -%{~ if log_policy == "PATH" } -logsPolicy: - destination: "PATH" - logsPath: ## Add logging path here -%{ endif } -%{~ if length(labels) > 0 ~} -labels: -%{ for k, v in labels ~} - ${k}: "${v}" -%{ endfor } -%{~ endif ~} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl deleted file mode 100644 index 25f89c3ceb..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e -o pipefail -GCLOUD_MAJOR_VERSION=$(gcloud --version | head -n 1 | awk '{print $NF}' | cut -f1 --delimiter=.) -if [ $((GCLOUD_MAJOR_VERSION >= 461)) ]; then - gcloud batch jobs submit ${submit_job_id} --project=${project} --location=${location} --config=${config} - echo "batch job ${submit_job_id} successfully submitted" -else - echo "gcloud must be updated to version 461.0.0 or later." - exit 1 -fi diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/variables.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/variables.tf deleted file mode 100644 index f65fbd111e..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/variables.tf +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "region" { - description = "The region in which to run the Google Cloud Batch job" - type = string -} - -variable "deployment_name" { - description = "Name of the deployment, used for the job_id" - type = string -} - -variable "labels" { - description = "Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided." - type = map(string) -} - -variable "job_id" { - description = "An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs." - type = string -} - -variable "job_filename" { - description = "The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified" - type = string - default = null -} - -variable "gcloud_version" { - description = "The version of the gcloud cli being used. Used for output instructions. Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default version)" - type = string - default = "" - - validation { - condition = contains(["alpha", "beta", ""], var.gcloud_version) - error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." - } -} - -variable "task_count" { - description = "Number of parallel tasks" - type = number - default = 1 -} - -variable "task_count_per_node" { - description = "Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value." - type = number - default = null -} - -variable "mpi_mode" { - description = "Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`." - type = bool - default = false -} - -variable "log_policy" { - description = <<-EOT - Create a block to define log policy. - When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging. - When set to `PATH`, path must be added to generated template. - When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. - EOT - type = string - default = "CLOUD_LOGGING" - - validation { - condition = contains(["CLOUD_LOGGING", "PATH", "DESTINATION_UNSPECIFIED"], var.log_policy) - error_message = "Allowed values for log_policy are 'CLOUD_LOGGING', 'PATH', or 'DESTINATION_UNSPECIFIED'." - } -} - -variable "runnables" { - description = "A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template." - type = list(object({ - script = string - })) - default = null -} - -variable "runnable" { - description = "A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`." - type = string - default = null -} - -variable "instance_template" { - description = "Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions." - type = string - default = null -} - -variable "subnetwork" { - description = "The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided." - type = any - default = null -} - -variable "enable_public_ips" { - description = "If set to true, instances will have public IPs" - type = bool - default = true -} - -variable "service_account" { - description = "Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided." - type = object({ - email = string, - scopes = set(string) - }) - default = { - email = null - scopes = [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write", - "https://www.googleapis.com/auth/servicecontrol", - "https://www.googleapis.com/auth/service.management.readonly", - "https://www.googleapis.com/auth/trace.append" - ] - } -} - -variable "machine_type" { - description = "Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided." - type = string - default = "n2-standard-4" -} - -variable "startup_script" { - description = "Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided." - type = string - default = null -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided." - type = list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "native_batch_mounting" { - description = "Batch can mount some fs_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts." - type = bool - default = true -} - -# Deprecated, replaced by instance_image -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided." - type = any - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - Google Cloud Batch compute node image. Ignored if `instance_template` is provided. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE`" - type = string - default = null - validation { - condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) - error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." - } -} - -variable "submit" { - description = "When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply." - type = bool - default = false -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/versions.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/versions.tf deleted file mode 100644 index a1161e1354..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-job-template/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - random = { - source = "hashicorp/random" - version = ">= 3.0" - } - google = { - source = "hashicorp/google" - version = ">= 4.0" - } - } - required_version = ">= 1.1" -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/README.md b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/README.md deleted file mode 100644 index c20ca7dbeb..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# Description - -This module creates a VM that acts as a login node to test and submit Google -Cloud Batch jobs. It is intended to be used along with the `batch-job-template` -module. - -This login node: - -- Uses the same VM settings as the first provided `batch-job-template`, such as - image, machine type, etc... -- Runs the same `startup-script` as the first provided `batch-job-template`. -- Has the same mounted file systems as the provided `batch-job-template`. -- Contains a folder with job templates generated by `batch-job-template` modules. - -Since the login node has the same mounted storage and is a homogeneous machine -to the Google Cloud Batch compute VMs, it can be used to inspect shared file -systems and test installed software before submitting a Google Cloud Batch job. - -## Example - -```yaml -- id: batch-job - source: modules/scheduler/batch-job-template - ... - -- id: batch-login - source: modules/scheduler/batch-login-node - use: [batch-job] - outputs: [instructions] -``` - -## Authentication - -To submit jobs from the login node, the service account attached to the VM needs -the `Batch Job Administrator` role. In most cases this service account will be -the Compute Engine default service account and will not be granted this role by -default. - -You can grant this role either by adding the `Batch Job Administrator` role to -the service account in the IAM page in the Google Cloud Console, or by running -the following command line: - -```bash -gcloud projects add-iam-policy-binding \ - --member=serviceAccount: \ - --role=roles/batch.jobsAdmin -``` - -## gcloud Batch Access - -Until the Google Cloud Batch API is generally available (GA), it may not be -available in all versions of the `gcloud` cli. You can test if the Google Cloud -Batch commands are available by running `gcloud [alpha|beta|] batch -h`. If the -Google Cloud Batch cli is not available it can generally be mitigated by either -updating `gcloud` by running `gcloud components update`, or using an image that -contains a more recent version of `gcloud`. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [login\_startup\_script](#module\_login\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_from_template.batch_login](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | -| [google_compute_instance_template.batch_instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [batch\_job\_directory](#input\_batch\_job\_directory) | The path of the directory on the login node in which to place the Google Cloud Batch job template | `string` | `"/home/batch-jobs"` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, also used for the job\_id | `string` | n/a | yes | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions.
Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default
version). Typically supplied by a batch-job-template module. If multiple
batch-job-template modules supply the gcloud\_version, only the first will be used. | `string` | `""` | no | -| [instance\_template](#input\_instance\_template) | Login VM instance template self-link. Typically supplied by a
batch-job-template module. If multiple batch-job-template modules supply the
instance\_template, the first will be used. | `string` | n/a | yes | -| [job\_data](#input\_job\_data) | List of jobs and supporting data for each, typically provided via "use" from the batch-job-template module. | 
list(object({
    template_contents = string,
    filename          = string,
    id                = string
  }))
| n/a | yes | -| [job\_filename](#input\_job\_filename) | Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [job\_id](#input\_job\_id) | Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions. | `string` | `null` | no | -| [job\_template\_contents](#input\_job\_template\_contents) | Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the login node. Key-value pairs | `map(string)` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which to create the login node | `string` | n/a | yes | -| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [zone](#input\_zone) | The zone in which to create the login node | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for accessing the login node and submitting Google Cloud Batch jobs | -| [login\_node\_name](#output\_login\_node\_name) | Name of the created VM | - diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/main.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/main.tf deleted file mode 100644 index 6f539af122..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/main.tf +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "batch-login-node", ghpc_role = "scheduler" }) -} - -data "google_compute_instance_template" "batch_instance_template" { - name = var.instance_template -} - -locals { - job_template_runners = [for job in var.job_data : { - content = job.template_contents - destination = "${var.batch_job_directory}/${job.filename}" - type = "data" - }] - - instance_template_metadata = data.google_compute_instance_template.batch_instance_template.metadata - startup_metadata = { startup-script = module.login_startup_script.startup_script } - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - login_metadata = merge(local.instance_template_metadata, local.startup_metadata, local.oslogin_metadata) - - batch_command_instructions = join("\n", [for job in var.job_data : <<-EOT - ## For job: ${job.id} ## - - Submit your job from login node: - gcloud ${var.gcloud_version} batch jobs submit ${job.id} --config=${var.batch_job_directory}/${job.filename} --location=${var.region} --project=${var.project_id} - - Check status: - gcloud ${var.gcloud_version} batch jobs describe ${job.id} --location=${var.region} --project=${var.project_id} | grep state: - - Delete job: - gcloud ${var.gcloud_version} batch jobs delete ${job.id} --location=${var.region} --project=${var.project_id} - - EOT - ]) - - list_all_jobs = <<-EOT - List all jobs: - gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} - EOT - - readme_contents = <<-EOT - # Batch Job Templates - - This folder contains Batch job templates created by the Cluster Toolkit. - These templates can be edited before submitting to Batch to capture more - complex workloads. - - Use the following commands to: - ${local.list_all_jobs} - - ${local.batch_command_instructions} - EOT - - # Construct startup script for network storage - storage_client_install_runners = [ - for i, ns in var.network_storage : merge(ns.client_install_runner, { - destination = "${i}-${ns.client_install_runner.destination}" - }) if ns.client_install_runner != null - ] - mount_runners = [ - for i, ns in var.network_storage : merge(ns.mount_runner, { - destination = "${i}-${ns.mount_runner.destination}" - }) if ns.mount_runner != null - ] - - startup_script_runner = { - content = var.startup_script != null ? var.startup_script : "echo 'Batch job template had no startup script'" - destination = "passed_startup_script.sh" - type = "shell" - } -} - -module "login_startup_script" { - source = "../../scripts/startup-script" - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = concat( - local.storage_client_install_runners, - local.mount_runners, - [local.startup_script_runner], - local.job_template_runners, - [ - { - content = local.readme_contents - destination = "${var.batch_job_directory}/README.md" - type = "data" - } - ] - ) -} - -resource "google_compute_instance_from_template" "batch_login" { - name = "${var.deployment_name}-batch-login" - source_instance_template = var.instance_template - project = var.project_id - zone = var.zone - metadata = local.login_metadata - - service_account { - scopes = ["https://www.googleapis.com/auth/cloud-platform"] - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml deleted file mode 100644 index 9af2319b4a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - batch.googleapis.com - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/outputs.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/outputs.tf deleted file mode 100644 index ea8eccf8d5..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/outputs.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "login_node_name" { - description = "Name of the created VM" - value = google_compute_instance_from_template.batch_login.name -} - -output "instructions" { - description = "Instructions for accessing the login node and submitting Google Cloud Batch jobs" - value = <<-EOT - - Batch job template files will be placed on the Batch login node in the following directory: - ${var.batch_job_directory} - - Use the following commands to: - SSH into the login node: - gcloud compute ssh --zone ${google_compute_instance_from_template.batch_login.zone} ${google_compute_instance_from_template.batch_login.name} --project ${google_compute_instance_from_template.batch_login.project} - - ${local.list_all_jobs} - - ${local.batch_command_instructions} - EOT -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/variables.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/variables.tf deleted file mode 100644 index 3b9caa7001..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/variables.tf +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "deployment_name" { - description = "Name of the deployment, also used for the job_id" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "region" { - description = "The region in which to create the login node" - type = string -} - -variable "zone" { - description = "The zone in which to create the login node" - type = string -} - -variable "labels" { - description = "Labels to add to the login node. Key-value pairs" - type = map(string) -} - -variable "instance_template" { - description = <<-EOT - Login VM instance template self-link. Typically supplied by a - batch-job-template module. If multiple batch-job-template modules supply the - instance_template, the first will be used. - EOT - type = string -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module." - type = list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "startup_script" { - description = "Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module." - type = string - default = null -} - -variable "job_data" { - description = "List of jobs and supporting data for each, typically provided via \"use\" from the batch-job-template module." - type = list(object({ - template_contents = string, - filename = string, - id = string - })) - validation { - condition = length(distinct([for job in var.job_data : job.filename])) == length(var.job_data) - error_message = "All filenames in var.job_data must be unique." - } - validation { - condition = length(distinct([for job in var.job_data : job.id])) == length(var.job_data) - error_message = "All job IDs in var.job_data must be unique." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_template_contents" { - description = "Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module." - type = string - default = null - validation { - condition = var.job_template_contents == null - error_message = "job_template_contents is deprecated. Please use `job_data` instead." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_filename" { - description = "Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module." - type = string - default = null - validation { - condition = var.job_filename == null - error_message = "job_filename is deprecated. Please use `job_data` instead." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_id" { - description = "Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions." - type = string - default = null - validation { - condition = var.job_id == null - error_message = "job_id is deprecated. Please use `job_data` instead." - } -} - -variable "gcloud_version" { - description = <<-EOT - The version of the gcloud cli being used. Used for output instructions. - Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default - version). Typically supplied by a batch-job-template module. If multiple - batch-job-template modules supply the gcloud_version, only the first will be used. - EOT - type = string - default = "" - - validation { - condition = contains(["alpha", "beta", ""], var.gcloud_version) - error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." - } -} - -variable "batch_job_directory" { - description = "The path of the directory on the login node in which to place the Google Cloud Batch job template" - type = string - default = "/home/batch-jobs" -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/versions.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/versions.tf deleted file mode 100644 index 15337a1d7b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/batch-login-node/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/README.md b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/README.md deleted file mode 100644 index dd4f7fdaa7..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/README.md +++ /dev/null @@ -1,220 +0,0 @@ -## Description - -This module creates a Google Kubernetes Engine -([GKE](https://cloud.google.com/kubernetes-engine)) cluster. - -### Example - -The following example creates a GKE cluster and a VPC designed to work with GKE. -See [VPC Network](#vpc-network) section for more information about network -requirements. - -```yaml - - id: network1 - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### VPC Network - -This module is configured to create a -[VPC-native cluster](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips). -This means that alias IPs are used and that the subnetwork requires secondary -ranges for pods and services. In the example shown above these secondary ranges -are created in the VPC module. By default the `gke-cluster` module will look for -ranges with the names `pods` and `services`. These names can be configured using -the `pods_ip_range_name` and `services_ip_range_name` settings. - -### Multi-networking - -To [enable Multi-networking](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment), pass multivpc module to gke-cluster module as described in example below. Passing a multivpc module enables multi networking and [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en) on the cluster. - -```yaml - - id: network - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: gke-cluster - source: modules/scheduler/gke-cluster - use: [network, multinetwork] ## enables multi networking and Dataplane V2 on cluster - settings: - cluster_name: $(vars.deployment_name) -``` - -Find an example of multi networking in GKE [here](../../../examples/gke-a3-megagpu.yaml). - -### Cluster Limitations - -The current implementations has the following limitations: - -- Autopilot is disabled -- Auto-provisioning of new node pools is disabled -- Network policies are not supported -- General addon configuration is not supported -- Only regional cluster is supported - -### GKE Inference Gateway - -Setting `enable_inference_gateway` to `true` will enable the `HttpLoadBalancing` -addon and deploy the Inference Gateway CRDs. This feature requires a subnet with -`purpose` set to `REGIONAL_MANAGED_PROXY` in the VPC. For more information, see -the [GKE Inference Gateway documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/serve-with-gke-inference-gateway). - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 7.2 | -| [google-beta](#requirement\_google-beta) | >= 7.2 | -| [kubernetes](#requirement\_kubernetes) | >= 2.36 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 7.2 | -| [google-beta](#provider\_google-beta) | >= 7.2 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | -| [workload\_identity](#module\_workload\_identity) | terraform-google-modules/kubernetes-engine/google//modules/workload-identity | >= 40.0 | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_cluster) | resource | -| [google-beta_google_container_node_pool.system_node_pools](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | -| [google-beta_google_container_engine_versions.version_prefix_filter](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/data-sources/google_container_engine_versions) | data source | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks enables multi networking and creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [authenticator\_security\_group](#input\_authenticator\_security\_group) | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| [autoscaling\_profile](#input\_autoscaling\_profile) | (Beta) Optimize for utilization or availability when deciding to remove nodes. Can be BALANCED or OPTIMIZE\_UTILIZATION. | `string` | `"OPTIMIZE_UTILIZATION"` | no | -| [cloud\_dns\_config](#input\_cloud\_dns\_config) | Configuration for Using Cloud DNS for GKE.

additive\_vpc\_scope\_dns\_domain: This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = "CLOUD\_DNS" and cluster\_dns\_scope = "CLUSTER\_SCOPE" must both be set as well.
cluster\_dns: Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS.
cluster\_dns\_scope: The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE.
cluster\_dns\_domain: The suffix used for all cluster service records. | 
object({
    additive_vpc_scope_dns_domain = optional(string)
    cluster_dns                   = optional(string, "PROVIDER_UNSPECIFIED")
    cluster_dns_scope             = optional(string, "DNS_SCOPE_UNSPECIFIED")
    cluster_dns_domain            = optional(string)
  })
| 
{
  "additive_vpc_scope_dns_domain": null,
  "cluster_dns": "PROVIDER_UNSPECIFIED",
  "cluster_dns_domain": null,
  "cluster_dns_scope": "DNS_SCOPE_UNSPECIFIED"
}
| no | -| [cluster\_availability\_type](#input\_cluster\_availability\_type) | Type of cluster availability. Possible values are: {REGIONAL, ZONAL} | `string` | `"REGIONAL"` | no | -| [cluster\_reference\_type](#input\_cluster\_reference\_type) | How the google\_container\_node\_pool.system\_node\_pools refers to the cluster. Possible values are: {SELF\_LINK, NAME} | `string` | `"SELF_LINK"` | no | -| [configure\_workload\_identity\_sa](#input\_configure\_workload\_identity\_sa) | When true, a kubernetes service account will be created and bound using workload identity to the service account used to create the cluster. | `bool` | `false` | no | -| [default\_max\_pods\_per\_node](#input\_default\_max\_pods\_per\_node) | The default maximum number of pods per node in this cluster. | `number` | `null` | no | -| [deletion\_protection](#input\_deletion\_protection) | "Determines if the cluster can be deleted by gcluster commands or not".
To delete a cluster provisioned with deletion\_protection set to true, you must first set it to false and apply the changes.
Then proceed with deletion as usual. | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`. | `string` | n/a | yes | -| [enable\_dataplane\_v2](#input\_enable\_dataplane\_v2) | Enables [Dataplane v2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2). This setting is immutable on clusters. If null, will default to false unless using multi-networking, in which case it will default to true | `bool` | `null` | no | -| [enable\_dcgm\_monitoring](#input\_enable\_dcgm\_monitoring) | Enable GKE to collect DCGM metrics | `bool` | `false` | no | -| [enable\_external\_dns\_endpoint](#input\_enable\_external\_dns\_endpoint) | Allow [DNS-based approach](https://cloud.google.com/kubernetes-engine/docs/concepts/network-isolation#dns-based_endpoint) for accessing the GKE control plane.
Refer this [dedicated blog](https://cloud.google.com/blog/products/containers-kubernetes/new-dns-based-endpoint-for-the-gke-control-plane) for more details. | `bool` | `false` | no | -| [enable\_filestore\_csi](#input\_enable\_filestore\_csi) | The status of the Filestore Container Storage Interface (CSI) driver addon, which allows the usage of filestore instance as volumes. | `bool` | `false` | no | -| [enable\_gcsfuse\_csi](#input\_enable\_gcsfuse\_csi) | The status of the GCSFuse Container Storage Interface (CSI) driver addon, which allows the usage of a GCS bucket as volumes. | `bool` | `false` | no | -| [enable\_inference\_gateway](#input\_enable\_inference\_gateway) | If true, enables GKE features required for Inference Gateway, including the HttpLoadBalancing addon, and installs required CRDs. | `bool` | `false` | no | -| [enable\_k8s\_beta\_apis](#input\_enable\_k8s\_beta\_apis) | List of Enabled Kubernetes Beta APIs. | `list(string)` | `null` | no | -| [enable\_managed\_lustre\_csi](#input\_enable\_managed\_lustre\_csi) | The status of the Google Compute Engine Managed Lustre Container Storage Interface (CSI) driver addon, which allows the usage of a lustre as volumes. | `bool` | `false` | no | -| [enable\_master\_global\_access](#input\_enable\_master\_global\_access) | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `false` | no | -| [enable\_multi\_networking](#input\_enable\_multi\_networking) | Enables [multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) (Requires GKE Enterprise). This setting is immutable on clusters and enables [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en). If null, will determine state based on if additional\_networks are passed in. | `bool` | `null` | no | -| [enable\_node\_local\_dns\_cache](#input\_enable\_node\_local\_dns\_cache) | Enable GKE NodeLocal DNSCache addon to improve DNS lookup latency | `bool` | `false` | no | -| [enable\_parallelstore\_csi](#input\_enable\_parallelstore\_csi) | The status of the Google Compute Engine Parallelstore Container Storage Interface (CSI) driver addon, which allows the usage of a parallelstore as volumes. | `bool` | `false` | no | -| [enable\_persistent\_disk\_csi](#input\_enable\_persistent\_disk\_csi) | The status of the Google Compute Engine Persistent Disk Container Storage Interface (CSI) driver addon, which allows the usage of a PD as volumes. | `bool` | `true` | no | -| [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | (Beta) Whether the master's internal IP address is used as the cluster endpoint. | `bool` | `true` | no | -| [enable\_private\_ipv6\_google\_access](#input\_enable\_private\_ipv6\_google\_access) | The private IPv6 google access type for the VMs in this subnet. | `bool` | `true` | no | -| [enable\_private\_nodes](#input\_enable\_private\_nodes) | (Beta) Whether nodes have internal IP addresses only. | `bool` | `true` | no | -| [enable\_ray\_operator](#input\_enable\_ray\_operator) | The status of the Ray operator addon, This feature enables Kubernetes APIs for managing and scaling Ray clusters and jobs. You control and are responsible for managing ray.io custom resources in your cluster. This feature is not compatible with GKE clusters that already have another Ray operator installed. Supports clusters on Kubernetes version 1.29.8-gke.1054000 or later. | `bool` | `false` | no | -| [gcp\_public\_cidrs\_access\_enabled](#input\_gcp\_public\_cidrs\_access\_enabled) | Whether the cluster master is accessible via all the Google Compute Engine Public IPs. To view this list of IP addresses look here https://cloud.google.com/compute/docs/faq#find_ip_range | `bool` | `false` | no | -| [k8s\_network\_names](#input\_k8s\_network\_names) | Kubernetes network names details for GKE. If starting index is not specified for gvnic or rdma, it would be set to the default values. | 
object({
    gvnic_prefix      = optional(string, "")
    gvnic_start_index = optional(number, 1)
    gvnic_postfix     = optional(string, "")
    rdma_prefix       = optional(string, "")
    rdma_start_index  = optional(number, 0)
    rdma_postfix      = optional(string, "")
  })
| 
{
  "gvnic_postfix": "",
  "gvnic_prefix": "gvnic-",
  "gvnic_start_index": 1,
  "rdma_postfix": "",
  "rdma_prefix": "rdma-",
  "rdma_start_index": 0
}
| no | -| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account name to use with the gke cluster | `string` | `"workload-identity-k8s-sa"` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [maintenance\_exclusions](#input\_maintenance\_exclusions) | List of maintenance exclusions. A cluster can have up to three. | 
list(object({
    name            = string
    start_time      = string
    end_time        = string
    exclusion_scope = string
  }))
| `[]` | no | -| [maintenance\_start\_time](#input\_maintenance\_start\_time) | Start time for daily maintenance operations. Specified in GMT with `HH:MM` format. | `string` | `"09:00"` | no | -| [master\_authorized\_networks](#input\_master\_authorized\_networks) | External network that can access Kubernetes master through HTTPS. Must be specified in CIDR notation. | 
list(object({
    cidr_block   = string
    display_name = string
  }))
| `[]` | no | -| [master\_ipv4\_cidr\_block](#input\_master\_ipv4\_cidr\_block) | (Beta) The IP range in CIDR notation to use for the hosted master network. | `string` | `"172.16.0.32/28"` | no | -| [min\_master\_version](#input\_min\_master\_version) | The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release. | `string` | `null` | no | -| [name\_suffix](#input\_name\_suffix) | Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`. | `string` | `""` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`. | `string` | n/a | yes | -| [networking\_mode](#input\_networking\_mode) | Determines whether alias IPs or routes will be used for pod IPs in the cluster. Options are VPC\_NATIVE or ROUTES. VPC\_NATIVE enables IP aliasing. The default is VPC\_NATIVE. | `string` | `"VPC_NATIVE"` | no | -| [pods\_ip\_range\_name](#input\_pods\_ip\_range\_name) | The name of the secondary subnet ip range to use for pods. | `string` | `"pods"` | no | -| [prefix\_with\_deployment\_name](#input\_prefix\_with\_deployment\_name) | If true, cluster name will be prefixed by `deployment_name` (ex: -). | `bool` | `true` | no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [region](#input\_region) | The region to host the cluster in. | `string` | n/a | yes | -| [release\_channel](#input\_release\_channel) | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. | `string` | `"UNSPECIFIED"` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the system node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the system node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [services\_ip\_range\_name](#input\_services\_ip\_range\_name) | The name of the secondary subnet range to use for services. | `string` | `"services"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to host the cluster in. | `string` | n/a | yes | -| [system\_node\_pool\_disk\_size\_gb](#input\_system\_node\_pool\_disk\_size\_gb) | Size of disk for each node of the system node pool. | `number` | `100` | no | -| [system\_node\_pool\_disk\_type](#input\_system\_node\_pool\_disk\_type) | Disk type for each node of the system node pool. | `string` | `null` | no | -| [system\_node\_pool\_enable\_secure\_boot](#input\_system\_node\_pool\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | -| [system\_node\_pool\_enabled](#input\_system\_node\_pool\_enabled) | Create a system node pool. | `bool` | `true` | no | -| [system\_node\_pool\_image\_type](#input\_system\_node\_pool\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | -| [system\_node\_pool\_kubernetes\_labels](#input\_system\_node\_pool\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | -| [system\_node\_pool\_machine\_type](#input\_system\_node\_pool\_machine\_type) | Machine type for the system node pool. | `string` | `"e2-standard-4"` | no | -| [system\_node\_pool\_name](#input\_system\_node\_pool\_name) | Name of the system node pool. | `string` | `"system"` | no | -| [system\_node\_pool\_node\_count](#input\_system\_node\_pool\_node\_count) | The total min and max nodes to be maintained in the system node pool. | 
object({
    total_min_nodes = number
    total_max_nodes = number
  })
| 
{
  "total_max_nodes": 10,
  "total_min_nodes": 2
}
| no | -| [system\_node\_pool\_taints](#input\_system\_node\_pool\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "components.gke.io/gke-managed-components",
    "value": true
  }
]
| no | -| [system\_node\_pool\_zones](#input\_system\_node\_pool\_zones) | The zones to use for the system node pool. If not specified, the cluster default node zone(s) will be used. | `list(string)` | `null` | no | -| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | -| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | -| [upgrade\_settings](#input\_upgrade\_settings) | Defines gke cluster upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | -| [version\_prefix](#input\_version\_prefix) | If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally. | `string` | `"1.31."` | no | -| [zone](#input\_zone) | Zone for a zonal cluster. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_id](#output\_cluster\_id) | An identifier for the resource with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | -| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | -| [gke\_version](#output\_gke\_version) | GKE cluster's version. | -| [instructions](#output\_instructions) | Instructions on how to connect to the created cluster. | -| [k8s\_service\_account\_name](#output\_k8s\_service\_account\_name) | Name of k8s service account. | - diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/main.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/main.tf deleted file mode 100644 index 6106f8d90f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/main.tf +++ /dev/null @@ -1,470 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-cluster", ghpc_role = "scheduler" }) -} - -locals { - upgrade_settings = { - strategy = var.upgrade_settings.strategy - max_surge = coalesce(var.upgrade_settings.max_surge, 0) - max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) - } -} - -locals { - dash = var.prefix_with_deployment_name && var.name_suffix != "" ? "-" : "" - prefix = var.prefix_with_deployment_name ? var.deployment_name : "" - name_maybe_empty = "${local.prefix}${local.dash}${var.name_suffix}" - name = local.name_maybe_empty != "" ? local.name_maybe_empty : "NO-NAME-GIVEN" - - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ - security_group = var.authenticator_security_group - }] - - default_sa_email = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" - sa_email = coalesce(var.service_account_email, local.default_sa_email) - - # additional VPCs enable multi networking - derived_enable_multi_networking = coalesce(var.enable_multi_networking, length(var.additional_networks) > 0) - - # multi networking needs enabled Dataplane v2 - derived_enable_dataplane_v2 = coalesce(var.enable_dataplane_v2, local.derived_enable_multi_networking) - - default_monitoring_component = [ - "SYSTEM_COMPONENTS", - "POD", - "DAEMONSET", - "DEPLOYMENT", - "STATEFULSET", - "STORAGE", - "HPA", - "CADVISOR", - "KUBELET" - ] - - default_logging_component = [ - "SYSTEM_COMPONENTS", - "WORKLOADS" - ] -} - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_container_engine_versions" "version_prefix_filter" { - provider = google-beta - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - version_prefix = var.version_prefix -} - -locals { - master_version = var.min_master_version != null ? var.min_master_version : data.google_container_engine_versions.version_prefix_filter.latest_master_version -} - -resource "google_container_cluster" "gke_cluster" { - provider = google-beta - - project = var.project_id - name = local.name - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - resource_labels = local.labels - networking_mode = var.networking_mode - # decouple node pool lifecycle from cluster life cycle - remove_default_node_pool = true - initial_node_count = 1 # must be set when remove_default_node_pool is set - node_locations = var.system_node_pool_zones - - deletion_protection = var.deletion_protection - - dynamic "enable_k8s_beta_apis" { - for_each = var.enable_k8s_beta_apis != null ? [1] : [] - content { - enabled_apis = var.enable_k8s_beta_apis - } - } - - network = var.network_id - subnetwork = var.subnetwork_self_link - - # Note: the existence of the "master_authorized_networks_config" block enables - # the master authorized networks even if it's empty. - master_authorized_networks_config { - dynamic "cidr_blocks" { - for_each = var.master_authorized_networks - content { - cidr_block = cidr_blocks.value.cidr_block - display_name = cidr_blocks.value.display_name - } - } - gcp_public_cidrs_access_enabled = var.gcp_public_cidrs_access_enabled - } - - private_ipv6_google_access = var.enable_private_ipv6_google_access ? "PRIVATE_IPV6_GOOGLE_ACCESS_TO_GOOGLE" : null - default_max_pods_per_node = var.default_max_pods_per_node - master_auth { - client_certificate_config { - issue_client_certificate = false - } - } - - enable_shielded_nodes = true - - cluster_autoscaling { - # Controls auto provisioning of node-pools - enabled = false - - # Controls autoscaling algorithm of node-pools - autoscaling_profile = var.autoscaling_profile - } - - datapath_provider = local.derived_enable_dataplane_v2 ? "ADVANCED_DATAPATH" : "LEGACY_DATAPATH" - - enable_multi_networking = local.derived_enable_multi_networking - - network_policy { - # Enabling NetworkPolicy for clusters with DatapathProvider=ADVANCED_DATAPATH - # is not allowed. Dataplane V2 will take care of network policy enforcement - # instead. - enabled = false - # GKE Dataplane V2 support. This must be set to PROVIDER_UNSPECIFIED in - # order to let the datapath_provider take effect. - # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/656#issuecomment-720398658 - provider = "PROVIDER_UNSPECIFIED" - } - - private_cluster_config { - enable_private_nodes = var.enable_private_nodes - enable_private_endpoint = var.enable_private_endpoint - master_ipv4_cidr_block = var.master_ipv4_cidr_block - master_global_access_config { - enabled = var.enable_master_global_access - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.pods_ip_range_name - services_secondary_range_name = var.services_ip_range_name - } - - workload_identity_config { - workload_pool = "${var.project_id}.svc.id.goog" - } - - dynamic "gateway_api_config" { - for_each = var.enable_inference_gateway ? [1] : [] - content { - channel = "CHANNEL_STANDARD" - } - } - - dynamic "authenticator_groups_config" { - for_each = local.cluster_authenticator_security_group - content { - security_group = authenticator_groups_config.value.security_group - } - } - - release_channel { - channel = var.release_channel - } - min_master_version = local.master_version - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - - dynamic "maintenance_exclusion" { - for_each = var.maintenance_exclusions - content { - exclusion_name = maintenance_exclusion.value.name - start_time = maintenance_exclusion.value.start_time - end_time = maintenance_exclusion.value.end_time - exclusion_options { - scope = maintenance_exclusion.value.exclusion_scope - } - } - } - } - - dynamic "dns_config" { - for_each = var.cloud_dns_config != null ? [1] : [] - content { - additive_vpc_scope_dns_domain = var.cloud_dns_config.additive_vpc_scope_dns_domain - cluster_dns = var.cloud_dns_config.cluster_dns - cluster_dns_scope = var.cloud_dns_config.cluster_dns_scope - cluster_dns_domain = var.cloud_dns_config.cluster_dns_domain - } - } - - addons_config { - gcp_filestore_csi_driver_config { - enabled = var.enable_filestore_csi - } - gcs_fuse_csi_driver_config { - enabled = var.enable_gcsfuse_csi - } - gce_persistent_disk_csi_driver_config { - enabled = var.enable_persistent_disk_csi - } - dns_cache_config { - enabled = var.enable_node_local_dns_cache - } - parallelstore_csi_driver_config { - enabled = var.enable_parallelstore_csi - } - ray_operator_config { - enabled = var.enable_ray_operator - } - lustre_csi_driver_config { - enabled = var.enable_managed_lustre_csi - } - dynamic "http_load_balancing" { - for_each = var.enable_inference_gateway ? [1] : [] - content { - disabled = false - } - } - } - - timeouts { - create = var.timeout_create - update = var.timeout_update - } - - node_config { - shielded_instance_config { - enable_secure_boot = var.system_node_pool_enable_secure_boot - enable_integrity_monitoring = true - } - } - - control_plane_endpoints_config { - dns_endpoint_config { - allow_external_traffic = var.enable_external_dns_endpoint - } - } - - lifecycle { - # Ignore all changes to the default node pool. It's being removed after creation. - ignore_changes = [ - node_config, - min_master_version, - ] - precondition { - condition = var.default_max_pods_per_node == null || var.networking_mode == "VPC_NATIVE" - error_message = "default_max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." - } - precondition { - condition = coalesce(var.enable_dataplane_v2, true) || !local.derived_enable_multi_networking - error_message = "'enable_dataplane_v2' cannot be false when enabling multi networking." - } - precondition { - condition = coalesce(var.enable_multi_networking, true) || length(var.additional_networks) == 0 - error_message = "'enable_multi_networking' cannot be false when using multivpc module, which passes additional_networks." - } - } - - monitoring_config { - enable_components = var.enable_dcgm_monitoring ? concat(local.default_monitoring_component, ["DCGM"]) : local.default_monitoring_component - managed_prometheus { - enabled = true - } - } - - logging_config { - enable_components = local.default_logging_component - } -} - -# We define explicit node pools, so that it can be modified without -# having to destroy the entire cluster. -resource "google_container_node_pool" "system_node_pools" { - provider = google-beta - count = var.system_node_pool_enabled ? 1 : 0 - - project = var.project_id - name = var.system_node_pool_name - cluster = var.cluster_reference_type == "NAME" ? google_container_cluster.gke_cluster.name : google_container_cluster.gke_cluster.self_link - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - node_locations = var.system_node_pool_zones - version = local.master_version - - autoscaling { - total_min_node_count = var.system_node_pool_node_count.total_min_nodes - total_max_node_count = var.system_node_pool_node_count.total_max_nodes - } - - upgrade_settings { - strategy = local.upgrade_settings.strategy - max_surge = local.upgrade_settings.max_surge - max_unavailable = local.upgrade_settings.max_unavailable - } - - management { - auto_repair = true - auto_upgrade = true - } - - node_config { - labels = var.system_node_pool_kubernetes_labels - resource_labels = local.labels - service_account = var.service_account_email - oauth_scopes = var.service_account_scopes - machine_type = var.system_node_pool_machine_type - disk_size_gb = var.system_node_pool_disk_size_gb - disk_type = var.system_node_pool_disk_type - - dynamic "taint" { - for_each = var.system_node_pool_taints - content { - key = taint.value.key - value = taint.value.value - effect = taint.value.effect - } - } - - # Forcing the use of the Container-optimized image, as it is the only - # image with the proper logging daemon installed. - # - # cos images use Shielded VMs since v1.13.6-gke.0. - # https://cloud.google.com/kubernetes-engine/docs/how-to/node-images - # - # We use COS_CONTAINERD to be compatible with (optional) gVisor. - # https://cloud.google.com/kubernetes-engine/docs/how-to/sandbox-pods - image_type = var.system_node_pool_image_type - - shielded_instance_config { - enable_secure_boot = var.system_node_pool_enable_secure_boot - enable_integrity_monitoring = true - } - - gvnic { - enabled = var.system_node_pool_image_type == "COS_CONTAINERD" - } - - # Implied by Workload Identity - workload_metadata_config { - mode = "GKE_METADATA" - } - # Implied by workload identity. - metadata = { - "disable-legacy-endpoints" = "true" - } - } - - lifecycle { - ignore_changes = [ - node_config[0].labels, - node_config[0].taint, - version, - ] - precondition { - condition = contains(["SURGE"], local.upgrade_settings.strategy) - error_message = "Only SURGE strategy is supported" - } - precondition { - condition = local.upgrade_settings.max_unavailable >= 0 - error_message = "max_unavailable should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_surge >= 0 - error_message = "max_surge should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 - error_message = "At least one of max_unavailable or max_surge must greater than 0" - } - } -} - -data "google_client_config" "default" {} - -provider "kubernetes" { - host = "https://${google_container_cluster.gke_cluster.endpoint}" - cluster_ca_certificate = base64decode(google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) - token = data.google_client_config.default.access_token -} - -module "workload_identity" { - count = var.configure_workload_identity_sa ? 1 : 0 - source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" - version = ">= 40.0" - - use_existing_gcp_sa = true - name = var.k8s_service_account_name - gcp_sa_name = local.sa_email - project_id = var.project_id - - # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1059 - depends_on = [ - data.google_project.project, - google_container_cluster.gke_cluster - ] -} - -locals { - k8s_service_account_name = one(module.workload_identity[*].k8s_service_account_name) -} - -locals { - # Separate gvnic and rdma networks and assign indexes - gvnic_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "GVNIC")] : - merge(net, { name = "${var.k8s_network_names.gvnic_prefix}${idx + var.k8s_network_names.gvnic_start_index}${var.k8s_network_names.gvnic_postfix}" }) - ] - - rdma_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "RDMA")] : - merge(net, { name = "${var.k8s_network_names.rdma_prefix}${idx + var.k8s_network_names.rdma_start_index}${var.k8s_network_names.rdma_postfix}" }) - ] - - all_networks = concat(local.gvnic_networks, local.rdma_networks) -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = google_container_cluster.gke_cluster.id - project_id = var.project_id - - apply_manifests = concat(flatten([ - for idx, network_info in local.all_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = network_info.name, - network_name = network_info.network - subnetwork_name = network_info.subnetwork, - device_mode = strcontains(upper(network_info.nic_type), "RDMA") ? "RDMA" : "NetDevice" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = network_info.name } - } - ] - ]), - var.enable_inference_gateway ? [ - { - source = "https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.0.0/manifests.yaml", - template_vars = {} - } - ] : [] - ) -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml deleted file mode 100644 index bd1517ce8f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/outputs.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/outputs.tf deleted file mode 100644 index 3326a5468e..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/outputs.tf +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cluster_id" { - description = "An identifier for the resource with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." - value = google_container_cluster.gke_cluster.id -} - -output "gke_cluster_exists" { - description = "A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations." - value = true - depends_on = [ - google_container_cluster.gke_cluster - ] -} - -locals { - private_endpoint_message = trimspace( - <<-EOT - This cluster was created with 'enable_private_endpoint: true'. - It cannot be accessed from a public IP addresses. - One way to access this cluster is from a VM created in the GKE cluster subnet. - EOT - ) - master_authorized_networks_message = length(var.master_authorized_networks) == 0 ? "" : trimspace( - <<-EOT - The following networks have been authorized to access this cluster: - ${join("\n", [for x in var.master_authorized_networks : " ${x.display_name}: ${x.cidr_block}"])}" - EOT - ) - public_endpoint_message = trimspace( - <<-EOT - To add authorized networks you can allowlist your IP with this command: - gcloud container clusters update ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} \ - --enable-master-authorized-networks \ - --master-authorized-networks /32 - EOT - ) - allowlist_your_ip_message = var.enable_private_endpoint ? local.private_endpoint_message : local.public_endpoint_message - kubernetes_service_account_message = local.k8s_service_account_name == null ? "" : trimspace( - <<-EOT - Use the following Kubernetes Service Account in the default namespace to run your workloads: - ${local.k8s_service_account_name} - The GCP Service Account mapped to this Kubernetes Service Account is: - ${local.sa_email} - EOT - ) - kubernetes_cluster_fetch_credential_message = var.enable_external_dns_endpoint ? trimspace( - <<-EOT - Use the following command to fetch credentials for the created cluster: - gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} \ - --dns-endpoint - EOT - ) : trimspace( - <<-EOT - Use the following command to fetch credentials for the created cluster: - gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} - EOT - ) -} - -output "instructions" { - description = "Instructions on how to connect to the created cluster." - value = trimspace( - <<-EOT - ${local.master_authorized_networks_message} - - ${local.allowlist_your_ip_message} - - ${local.kubernetes_cluster_fetch_credential_message} - - ${local.kubernetes_service_account_message} - EOT - ) -} - -output "k8s_service_account_name" { - description = "Name of k8s service account." - value = local.k8s_service_account_name -} - -output "gke_version" { - description = "GKE cluster's version." - value = google_container_cluster.gke_cluster.master_version -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl deleted file mode 100644 index d376a1a760..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: GKENetworkParamSet -metadata: - name: ${name} -spec: - vpc: ${network_name} - vpcSubnet: ${subnetwork_name} - deviceMode: ${device_mode} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl deleted file mode 100644 index 1571a92692..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: Network -metadata: - name: ${name} -spec: - parametersRef: - group: networking.gke.io - kind: GKENetworkParamSet - name: ${name} - type: Device diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/variables.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/variables.tf deleted file mode 100644 index 8d863b1730..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/gke-cluster/variables.tf +++ /dev/null @@ -1,533 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "name_suffix" { - description = "Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`." - type = string - default = "" -} - -variable "deployment_name" { - description = "Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`." - type = string -} - -variable "prefix_with_deployment_name" { - description = "If true, cluster name will be prefixed by `deployment_name` (ex: -)." - type = bool - default = true -} - -variable "region" { - description = "The region to host the cluster in." - type = string -} - -variable "zone" { - description = "Zone for a zonal cluster." - default = null - type = string -} - -variable "network_id" { - description = "The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`." - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to host the cluster in." - type = string -} - -variable "pods_ip_range_name" { - description = "The name of the secondary subnet ip range to use for pods." - type = string - default = "pods" -} - -variable "services_ip_range_name" { - description = "The name of the secondary subnet range to use for services." - type = string - default = "services" -} - -variable "enable_private_ipv6_google_access" { - description = "The private IPv6 google access type for the VMs in this subnet." - type = bool - default = true -} - -variable "release_channel" { - description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`." - type = string - default = "UNSPECIFIED" -} - -variable "min_master_version" { - description = "The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release." - type = string - default = null -} - -variable "version_prefix" { - description = "If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally." - type = string - default = "1.31." -} - -variable "maintenance_start_time" { - description = "Start time for daily maintenance operations. Specified in GMT with `HH:MM` format." - type = string - default = "09:00" -} - -variable "maintenance_exclusions" { - description = "List of maintenance exclusions. A cluster can have up to three." - type = list(object({ - name = string - start_time = string - end_time = string - exclusion_scope = string - })) - default = [] - validation { - condition = alltrue([ - for x in var.maintenance_exclusions : - contains(["NO_UPGRADES", "NO_MINOR_UPGRADES", "NO_MINOR_OR_NODE_UPGRADES"], x.exclusion_scope) - ]) - error_message = "`exclusion_scope` must be set to `NO_UPGRADES` OR `NO_MINOR_UPGRADES` OR `NO_MINOR_OR_NODE_UPGRADES`." - } -} - -variable "cloud_dns_config" { - description = < **_NOTE:_** The `project_id` and `region` settings would be inferred from the -> deployment variables of the same name, but they are included here for clarity. - -### Multi-networking - -To create network objects in GKE cluster, you can pass a multivpc module to a pre-existing-gke-cluster module instead of [applying a manifest manually](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment). - -```yaml - - id: network - source: modules/network/vpc - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: existing-gke-cluster ## multinetworking must be enabled in advance when cluster creation - source: modules/scheduler/pre-existing-gke-cluster - use: [multinetwork] - settings: - cluster_name: $(vars.deployment_name) -``` - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_container_cluster.existing_gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [cluster\_name](#input\_cluster\_name) | Name of the existing cluster | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project that hosts the existing cluster | `string` | n/a | yes | -| [rdma\_subnetwork\_name\_prefix](#input\_rdma\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | `string` | `null` | no | -| [region](#input\_region) | Region in which to search for the cluster | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | -| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster exists. | -| [gke\_version](#output\_gke\_version) | GKE cluster's version. | - diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf deleted file mode 100644 index 926d2be100..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_container_cluster" "existing_gke_cluster" { - name = var.cluster_name - project = var.project_id - location = var.region -} - -locals { - rdma_networks = [for network_info in var.additional_networks : network_info if strcontains(upper(network_info.nic_type), "RDMA")] - non_rdma_networks = [for network_info in var.additional_networks : network_info if !strcontains(upper(network_info.nic_type), "RDMA")] - apply_manifests_rdma_networks = flatten([ - for idx, network_info in local.rdma_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = "${var.rdma_subnetwork_name_prefix}-${idx}", - network_name = network_info.network - subnetwork_name = "${var.rdma_subnetwork_name_prefix}-${idx}", - device_mode = "RDMA" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = "${var.rdma_subnetwork_name_prefix}-${idx}" } - } - ] - ]) - - apply_manifests_non_rdma_networks = flatten([ - for idx, network_info in local.non_rdma_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = network_info.subnetwork - network_name = network_info.network - subnetwork_name = network_info.subnetwork - device_mode = "NetDevice" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = network_info.subnetwork } - } - ] - ]) -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = data.google_container_cluster.existing_gke_cluster.id - project_id = var.project_id - - apply_manifests = concat(local.apply_manifests_non_rdma_networks, local.apply_manifests_rdma_networks) -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf deleted file mode 100644 index 8884ee30b0..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cluster_id" { - description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." - value = data.google_container_cluster.existing_gke_cluster.id -} - -output "gke_cluster_exists" { - description = "A static flag that signals to downstream modules that a cluster exists." - value = true - depends_on = [ - data.google_container_cluster.existing_gke_cluster - ] -} - -output "gke_version" { - description = "GKE cluster's version." - value = data.google_container_cluster.existing_gke_cluster.master_version -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl deleted file mode 100644 index d376a1a760..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: GKENetworkParamSet -metadata: - name: ${name} -spec: - vpc: ${network_name} - vpcSubnet: ${subnetwork_name} - deviceMode: ${device_mode} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl deleted file mode 100644 index 1571a92692..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: Network -metadata: - name: ${name} -spec: - parametersRef: - group: networking.gke.io - kind: GKENetworkParamSet - name: ${name} - type: Device diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf deleted file mode 100644 index 9e9ed98ed3..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project that hosts the existing cluster" - type = string -} - -variable "cluster_name" { - description = "Name of the existing cluster" - type = string -} - -variable "region" { - description = "Region in which to search for the cluster" - type = string -} - -variable "additional_networks" { - description = "Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster." - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - })) -} - -variable "rdma_subnetwork_name_prefix" { - description = "Prefix of the RDMA subnetwork names" - default = null - type = string -} diff --git a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf b/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf deleted file mode 100644 index 562d8647b1..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = "> 5.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.74.0" - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/README.md b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/README.md deleted file mode 100644 index db9094909b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/README.md +++ /dev/null @@ -1,355 +0,0 @@ -## Description - -This module creates a startup script that will execute a list of runners in the -order they are specified. The runners are copied to a GCS bucket at deployment -time and then copied into the VM as they are executed after startup. - -Each runner receives the following attributes: - -- `destination`: (Required) The name of the file at the destination VM. If an - absolute path is provided, the file will be copied to that path, otherwise - the file will be created in a temporary folder and deleted once the startup - script runs. -- `type`: (Required) The type of the runner, one of the following: - - `shell`: The runner is a shell script and will be executed once copied to - the destination VM. - - `ansible-local`: The runner is an ansible playbook and will run on the VM - with the following command line flags: - - ```shell - ansible-playbook --connection=local --inventory=localhost, \ - --limit localhost <> - ``` - - - `data`: The data or file specified will be copied to `<>`. No - action will be performed after the data is staged. This data can be used by - subsequent runners or simply made available on the VM for later use. -- `content`: (Optional) Content to be uploaded and, if `type` is - either `shell` or `ansible-local`, executed. Must be defined if `source` is - not. -- `source`: (Optional) A path to the file or data you want to upload. Must be - defined if `content` is not. The source path is relative to the deployment - group directory. To ensure correctness of path use `ghpc_stage` function, that - would copy referenced file to the deployment group directory. For example: - - ```yaml - source: $(ghpc_stage("path/to/file")) - ``` - - For more examples with context, see the - [example blueprint snippet](#example). To reference any other source file, an - absolute path must be used. - -- `args`: (Optional) Arguments to be passed to `shell` or `ansible-local` - runners. For `shell` runners, these will be passed as arguments to the script - when it is executed. For `ansible-local` runners, they will be appended to - a list of default arguments that invoke `ansible-playbook` on the localhost. - Therefore`args` should not include any arguments that alter this behavior, - such as `--connection`, `--inventory`, or `--limit`. - -### Runner dependencies - -`ansible-local` runners require Ansible to be installed in the VM before -running. To support other playbook runners in the Cluster Toolkit, we install -version 2.11 of `ansible-core` as well as the larger package of collections -found in `ansible` version 4.10.0. - -If an `ansible-local` runner is found in the list supplied to this module, -a script to install Ansible will be prepended to the list of runners. This -behavior can be disabled by setting `var.prepend_ansible_installer` to `false`. -This script will do the following at VM startup: - -- Install system-wide python3 if not already installed using system package - managers (yum, apt-get, etc) -- Install `python3-distutils` system-wide in debian and ubuntu based - environments. This can be a missing dependency on system installations of - python3 for installing and upgrading pip. -- Install system-wide pip3 if not already installed and upgrade pip3 if the - version is not at least 18.0. -- Install and create a virtual environment located at `/usr/local/ghpc-venv`. -- Install ansible into this virtual environment if the current version of - ansible is not version 2.11 or higher. - -To use the virtual environment created by this script, you can activate it by -running the following command on the VM: - -```shell -source /usr/local/ghpc-venv/bin/activate -``` - -You may also need to provide the correct python interpreter as the python3 -binary in the virtual environment. This can be done by adding the following flag -when calling `ansible-playbook`: - -```shell --e ansible_python_interpreter=/usr/local/ghpc-venv/bin/activate -``` - -> **_NOTE:_** ansible-playbook and other ansible command line tools will only be -> accessible from the command line (and in your PATH variable) after activating -> this environment. - -### Staging the runners - -Runners will be uploaded to a -[GCS bucket](https://cloud.google.com/storage/docs/creating-buckets). This -bucket will be created by this module and named as -`${var.deployment_name}-startup-scripts-${random_id}`. VMs using the startup -script created by this module will pull the runners content from a GCS bucket -and therefore must have access to GCS. - -> **_NOTE:_** To ensure access to GCS, set the following OAuth scope on the -> instance using the startup scripts: -> `https://www.googleapis.com/auth/devstorage.read_only`. -> -> This is set as a default scope in the [vm-instance], -> [schedMD-slurm-on-gcp-login-node] and [schedMD-slurm-on-gcp-controller] -> modules - -[vm-instance]: ../../compute/vm-instance/README.md -[schedMD-slurm-on-gcp-login-node]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md -[schedMD-slurm-on-gcp-controller]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md - -### Tracking startup script execution - -For more information on how to use startup scripts on Google Cloud Platform, -please refer to -[this document](https://cloud.google.com/compute/docs/instances/startup-scripts/linux). - -To debug startup scripts from a Linux VM created with startup script generated -by this module: - -```shell -sudo DEBUG=1 google_metadata_script_runner startup -``` - -To view outputs from a Linux startup script, run: - -```shell -sudo journalctl -u google-startup-scripts.service -``` - -### Monitoring Agent Installation - -This `startup-script` module has several options for installing a Google -monitoring agent. There are two relevant settings: `install_stackdriver_agent` -and `install_cloud_ops_agent`. - -The _Stackdriver Agent_ also called the _Legacy Cloud Monitoring Agent_ provides -better performance under some HPC workloads. While official documentation -recommends using the _Cloud Ops Agent_, it is recommended to use -`install_stackdriver_agent` when performance is important. - -#### Stackdriver Agent Installation - -If an image or machine already has Cloud Ops Agent installed and you would like -to instead use the Stackdriver Agent, the following script will remove the Cloud -Ops Agent and install the Stackdriver Agent. - -```bash -# Remove Cloud Ops Agent -sudo systemctl stop google-cloud-ops-agent.service -sudo systemctl disable google-cloud-ops-agent.service -curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh -sudo bash add-google-cloud-ops-agent-repo.sh --uninstall -sudo bash add-google-cloud-ops-agent-repo.sh --remove-repo - -# Install Stackdriver Agent -curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh -sudo bash add-monitoring-agent-repo.sh --also-install -curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh -sudo bash add-logging-agent-repo.sh --also-install -sudo service stackdriver-agent start -sudo service google-fluentd restart -``` - -#### Cloud Ops Agent Installation - -If an image or machine already has the Stackdriver Agent installed and you would -like to instead use the Cloud Ops Agent, the following script will remove the -Stackdriver Agent and install the Cloud Ops Agent. - -```bash -# UnInstall Stackdriver Agent - -sudo systemctl stop stackdriver-agent.service -sudo systemctl disable stackdriver-agent.service -curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh -sudo dpkg --configure -a -sudo bash add-monitoring-agent-repo.sh --uninstall -sudo bash add-monitoring-agent-repo.sh --remove-repo -sudo systemctl stop google-fluentd.service -sudo systemctl disable google-fluentd.service -sudo dpkg --configure -a -curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh -sudo bash add-logging-agent-repo.sh --uninstall -sudo bash add-logging-agent-repo.sh --remove-repo - -# Install ops-agent - -curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh -sudo bash add-google-cloud-ops-agent-repo.sh --also-install -sudo service google-cloud-ops-agent start -``` - -As a reminder, this should be in a startup script, which should run on all -Compute nodes via the `compute_startup_script` on the controller. - -#### Testing Installation - -You can test if one of the agents is running using the following commands: - -```bash -# For Cloud Ops Agent -$ sudo systemctl is-active google-cloud-ops-agent"*" -active -active -active -active - -# For Legacy Monitoring and Logging Agents -$ sudo service stackdriver-agent status -stackdriver-agent is running [ OK ] -$ sudo service google-fluentd status -google-fluentd is running [ OK ] -``` - -For official documentation see troubleshooting docs: - -- [Cloud Ops Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-install-startup) -- [Legacy Monitoring Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/monitoring/troubleshooting) -- [Legacy Logging Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/logging/troubleshooting) - -### Example - -```yaml -- id: startup - source: modules/scripts/startup-script - settings: - runners: - # Some modules such as filestore have runners as outputs for convenience: - - $(homefs.install_nfs_client_runner) - # These runners can still be created manually: - # - type: shell - # destination: "modules/filestore/scripts/install_nfs_client.sh" - # source: "modules/filestore/scripts/install_nfs_client.sh" - - type: ansible-local - destination: "modules/filestore/scripts/mount.yaml" - source: "modules/filestore/scripts/mount.yaml" - - type: data - source: /tmp/foo.tgz - destination: /tmp/bar.tgz - - type: shell - destination: "decompress.sh" - content: | - #!/bin/sh - echo $2 - tar zxvf /tmp/$1 -C / - args: "bar.tgz 'Expanding file'" - -- id: compute-cluster - source: modules/compute/vm-instance - use: [homefs, startup] -``` - -In the above example, a new GCS bucket is created to upload the startup-scripts. -But in the case where the user wants to reuse existing GCS bucket or folder, -they are able to do so by using the `gcs_bucket_path` as shown in the below example - -```yaml -- id: startup - source: modules/scripts/startup-script - settings: - gcs_bucket_path: gs://user-test-bucket/folder1/folder2 - install_stackdriver_agent: true - -- id: compute-cluster - source: modules/compute/vm-instance - use: [startup] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 6.41 | -| [local](#requirement\_local) | >= 2.0.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.41 | -| [local](#provider\_local) | >= 2.0.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.configs_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | -| [google_storage_bucket_object.scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [local_file.debug_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [ansible\_virtualenv\_path](#input\_ansible\_virtualenv\_path) | Virtual environment path in which to install Ansible | `string` | `"/usr/local/ghpc-venv"` | no | -| [bucket\_viewers](#input\_bucket\_viewers) | Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account) | `list(string)` | `[]` | no | -| [configure\_ssh\_host\_patterns](#input\_configure\_ssh\_host\_patterns) | If specified, it will automate ssh configuration by:
- Defining a Host block for every element of this variable and setting StrictHostKeyChecking to 'No'.
Ex: "hpc*", "hpc01*", "ml*"
- The first time users log-in, it will create ssh keys that are added to the authorized keys list
This requires a shared /home filesystem and relies on specifying the right prefix. | `list(string)` | `[]` | no | -| [debug\_file](#input\_debug\_file) | Path to an optional local to be written with 'startup\_script'. | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used to name GCS bucket for startup scripts. | `string` | n/a | yes | -| [docker](#input\_docker) | Install and configure Docker | 
object({
    enabled        = optional(bool, false)
    world_writable = optional(bool, false)
    daemon_config  = optional(string, "")
  })
| 
{
  "enabled": false
}
| no | -| [enable\_docker\_world\_writable](#input\_enable\_docker\_world\_writable) | DEPRECATED: use var.docker | `bool` | `null` | no | -| [enable\_gpu\_network\_wait\_online](#input\_enable\_gpu\_network\_wait\_online) | Enable a SystemD unit that blocks execution of startup-scripts until after all network interfaces are online. (Works on reboots or boots of an image built using this solution) | `bool` | `false` | no | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | `null` | no | -| [http\_no\_proxy](#input\_http\_no\_proxy) | Domains for which to disable http\_proxy behavior. Honored only if var.http\_proxy is set | `string` | `".google.com,.googleapis.com,metadata.google.internal,localhost,127.0.0.1"` | no | -| [http\_proxy](#input\_http\_proxy) | Web (http and https) proxy configuration for pip, apt, and yum/dnf and interactive shells | `string` | `""` | no | -| [install\_ansible](#input\_install\_ansible) | Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used. | `bool` | `null` | no | -| [install\_cloud\_ops\_agent](#input\_install\_cloud\_ops\_agent) | Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true. | `bool` | `false` | no | -| [install\_cloud\_rdma\_drivers](#input\_install\_cloud\_rdma\_drivers) | If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8. Should not be enabled if using the HPC VM Image. | `bool` | `false` | no | -| [install\_docker](#input\_install\_docker) | DEPRECATED: use var.docker. | `bool` | `null` | no | -| [install\_stackdriver\_agent](#input\_install\_stackdriver\_agent) | Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance. | `bool` | `false` | no | -| [labels](#input\_labels) | Labels for the created GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_filesystem](#input\_local\_ssd\_filesystem) | Create and mount a filesystem from local SSD disks (data will be lost if VMs are powered down without enabling migration); enable by setting mountpoint field to a valid directory path. | 
object({
    fs_type     = optional(string, "ext4")
    mountpoint  = optional(string, "")
    permissions = optional(string, "0755")
  })
| 
{
  "fs_type": "ext4",
  "mountpoint": "",
  "permissions": "0755"
}
| no | -| [managed\_lustre](#input\_managed\_lustre) | Configure Managed Lustre (assumes driver already installed) | 
object({
    enabled = optional(bool, false)
    port    = optional(number, 988)
  })
| 
{
  "enabled": false,
  "port": 988
}
| no | -| [prepend\_ansible\_installer](#input\_prepend\_ansible\_installer) | DEPRECATED. Use `install_ansible=false` to prevent ansible installation. | `bool` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [runners](#input\_runners) | List of runners to run on remote VM.
Runners can be of type ansible-local, shell or data.
A runner must specify one of 'source' or 'content'.
All runners must specify 'destination'. If 'destination' does not include a
path, it will be copied in a temporary folder and deleted after running.
Runners may also pass 'args', which will be passed as argument to shell runners only. | `list(map(string))` | `[]` | no | -| [set\_ofi\_cloud\_rdma\_tunables](#input\_set\_ofi\_cloud\_rdma\_tunables) | Controls whether to enable specific OFI environment variables for workloads using Cloud RDMA networking. Should be false for non-RDMA workloads. | `bool` | `false` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [compute\_startup\_script](#output\_compute\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | -| [controller\_startup\_script](#output\_controller\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | -| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | - diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml deleted file mode 100644 index 02c449c7cb..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Configure ssh between nodes - become: true - hosts: localhost - vars: - ssh_config_path: "/etc/ssh/ssh_config" - bashrc: "{{ '/etc/bashrc' if ansible_facts['os_family'] == 'RedHat' else '/etc/bash.bashrc' }}" - setup_ssh_script: "/bin/bash /usr/local/ghpc/setup-ssh-keys.sh" - tasks: - - name: "Set StrictHostKeyChecking to no" - ansible.builtin.blockinfile: - path: "{{ ssh_config_path }}" - block: | - Host "{{ item }}" - StrictHostKeyChecking no - marker: "# {mark} ANSIBLE MANAGED BLOCK {{item}}" - loop: "{{ host_name_prefix }}" - - name: "Create ssh keys in .bashrc if not already done" - ansible.builtin.lineinfile: - path: "{{ bashrc }}" - regexp: '^{{ setup_ssh_script }}' - line: "{{ setup_ssh_script }}" diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh deleted file mode 100644 index 38c7ff9b5c..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -web_proxy="${1:-}" -if [ -z "$web_proxy" ]; then - echo "Error: must provide 1 argument identifying http/https proxy" - exit 1 -fi - -# configure pip to use proxy -PIP_CONF=/etc/pip.conf -if [ ! -f "$PIP_CONF" ]; then - cat <<-EOF >"$PIP_CONF" - [global] - proxy=$web_proxy - EOF -fi - -# configure yum or dnf to use proxy -if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - YUM_CONF="/etc/yum.conf" - if ! grep -q '^proxy=.*' "$YUM_CONF"; then - sed --follow-symlinks -i.bak "/^\[main]/a proxy=$web_proxy" "$YUM_CONF" - else - sed --follow-symlinks -i.bak "s,proxy=.*,proxy=$web_proxy," "$YUM_CONF" - fi -fi - -# configure apt to use proxy -if [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release 2>/dev/null || - grep -qi ubuntu /etc/os-release 2>/dev/null; then - APT_CONF_PROXY="/etc/apt/apt.conf.d/99proxy.conf" - if [ ! -f "$APT_CONF_PROXY" ]; then - cat <<-EOF >"$APT_CONF_PROXY" - Acquire::http::Proxy "$web_proxy"; - Acquire::https::Proxy "$web_proxy"; - EOF - fi -fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh deleted file mode 100644 index 682e1352a1..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script applies fixes to VMs that must occur early in boot. For example, -# when yum or apt repositories are misconfigured, preventing most package -# operations from completing successfully. - -source /etc/os-release - -if [[ "$PRETTY_NAME" == "CentOS Linux 7 (Core)" ]]; then - echo "Applying hotfixes for CentOS 7" - if grep -q '^mirrorlist' /etc/yum.repos.d/CentOS-Base.repo; then - echo "Removing mirrorlist from default CentOS 7 repositories" - sed -i '/^mirrorlist/d' /etc/yum.repos.d/CentOS-Base.repo - fi - if grep -q '^#baseurl=http://mirror.centos.org' /etc/yum.repos.d/CentOS-Base.repo; then - echo "Reconfiguring default CentOS 7 repositories to use CentOS Vault" - sed -i 's,^#baseurl=http://mirror.centos.org/,baseurl=http://vault.centos.org/,' /etc/yum.repos.d/CentOS-Base.repo - fi -fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh deleted file mode 100644 index 3a29ae808f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh +++ /dev/null @@ -1,73 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Given a url and filename, download an object to the vardir. When the installed -# version of gcloud is >=402.0.0 (Sept. 2022), then gcloud storage is used to -# fetch from the bucket. Otherwise gsutil is used. Note, the service account for -# the instance must be properly configured with a role having authorization to -# get objects from the bucket. -# -# This function is intended for single file downloads and no attempt is made to -# verify the checksum other than the default behavior of gcloud or gsutil. -# -# This function has no other platform dependencies other than gcloud / gsutil. - -# This code originated from: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 -stdlib::get_from_bucket() { - local OPTIND opt url fname dir="${VARDIR:-/var/lib/startup}" - while getopts ":u:f:d:" opt; do - case "${opt}" in - u) url="${OPTARG}" ;; - f) fname="${OPTARG}" ;; - d) dir="${OPTARG}" ;; - :) - stdlib::mandatory_argument -n stdlib::get_from_bucket -f "$OPTARG" - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error 'Usage: stdlib::get_from_bucket -u -f -d ' - stdlib::info 'For example: stdlib::get_from_bucket -u gs://mybucket/foo.tgz -d /var/tmp' - return "${E_UNKNOWN_ARG}" - ;; - esac - done - # Trivially compute the filename from the URL if unspecified. - if [[ -z ${fname} ]]; then - fname=${url##*/} - stdlib::debug "Computed filename='${fname}' given URL." - fi - [[ -d ${dir} ]] || mkdir "${dir}" - local attempt=0 - local max_retries=7 - # store gcs command as array and then split when called by stdlib::cmd - if stdlib::cmd gcloud help storage cp &>/dev/null; then - gcs_command=(gcloud storage cp --no-user-output-enabled) - else - gcs_command=(gsutil -q cp) - fi - while [[ $attempt -le $max_retries ]]; do - if [[ $attempt -gt 0 ]]; then - local wait=$((2 ** attempt)) - stdlib::error "Retry attempt ${attempt} of ${max_retries} with exponential backoff: ${wait} seconds." - sleep $wait - fi - if stdlib::cmd "${gcs_command[@]}" "${url}" "${dir}/${fname}"; then - break - else - stdlib::error "${gcs_command[*]} reported non-zero exit code fetching ${url}." - ((attempt++)) - fi - done -} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh deleted file mode 100644 index eac2b2e32a..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex -REQ_ANSIBLE_VERSION=2.15 -REQ_ANSIBLE_PIP_VERSION=8.7.0 -REQ_PIP_WHEEL_VERSION=0.45.1 -REQ_PIP_SETUPTOOLS_VERSION=80.8.0 -REQ_PIP_MAJOR_VERSION=25 -REQ_PYTHON3_VERSION=9 - -apt_wait() { - while fuser /var/lib/apt/lists/lock >/dev/null 2>&1; do - echo "Sleeping for apt lists lock" - sleep 3 - done -} - -# Installs any dependencies needed for python based on the OS -install_python_deps() { - # this file is present on both Debian and Ubuntu OSes - if [ -f /etc/debian_version ]; then - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3-setuptools python3-venv - fi -} - -# Gets the name of the python executable for python starting with python3, then -# checking python. Sets the variable to an empty string if neither are found. -get_python_path() { - python_path="" - if command -v python3 1>/dev/null; then - python_path=$(command -v python3) - elif command -v python 1>/dev/null; then - python_path=$(command -v python) - fi -} - -# Returns the python major version. If provided, it will use the first argument -# as the python executable, otherwise it will default to simply "python". -get_python_major_version() { - python_path=${1:-python} - python_major_version=$(${python_path} -c "import sys; print(sys.version_info.major)") -} - -# Returns the python minor version. If provided, it will use the first argument -# as the python executable, otherwise it will default to simply "python". -get_python_minor_version() { - python_path=${1:-python} - python_minor_version=$(${python_path} -c "import sys; print(sys.version_info.minor)") -} - -# Install python3 with the yum package manager. Updates python_path to the -# newly installed packaged. -install_python3_dnf() { - major_version=$(rpm -E "%{rhel}") - set -- "--disablerepo=*" "--enablerepo=baseos,appstream" - if grep -qi 'ID="rhel"' /etc/os-release; then - # Do not set --disablerepo / --enablerepo on RedHat, due to - # complex repo names; clear array - set -- - fi - # On Rocky Linux 9, Python 3.9 is installed by default but this - # has already been dropped by ansible-core for control nodes. - # https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix - # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where - # it is available as "python3*" but must be named explicitly on - # older releases. It also ensures longer support for Ansible. - if [ "${major_version}" -lt "10" ]; then - dnf install "$@" -y python3.12 python3.12-pip - python_path=$(command -v python3.12) - else - dnf install "$@" -y python3 python3-pip - python_path=$(command -v python3) - fi -} - -# Install python3 with the apt package manager. Updates python_path to the -# newly installed packaged. -install_python3_apt() { - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3 python3-setuptools python3-pip python3-venv - python_path=$(command -v python3) -} - -install_python3() { - if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || - [ -f /etc/system-release ]; then - install_python3_dnf - elif [ -f /etc/debian_version ]; then - install_python3_apt - else - echo "Error: Unsupported Distribution" - return 1 - fi -} - -# Install pip3 with the dnf package manager. Updates python_path to the -# newly installed packaged. -install_pip3_dnf() { - major_version=$(rpm -E "%{rhel}") - set -- "--disablerepo=*" "--enablerepo=baseos,appstream" - if grep -qi 'ID="rhel"' /etc/os-release; then - # Do not set --disablerepo / --enablerepo on RedHat, due to complex repo names - # clear array - set -- - fi - # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where - # it is available as "python3*" but must be named explicitly on - # older releases. It also ensures longer support for Ansible. - if [ "${major_version}" -lt "10" ]; then - dnf install "$@" -y python3.12-pip - else - dnf install "$@" -y python3-pip - fi -} - -# Install pip3 with the apt package manager. Updates python_path to the -# newly installed packaged. -install_pip3_apt() { - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3-pip -} - -install_pip3() { - if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || - [ -f /etc/system-release ]; then - install_pip3_dnf - elif [ -f /etc/debian_version ]; then - install_pip3_apt - else - echo "Error: Unsupported Distribution" - return 1 - fi -} - -main() { - if [ $# -gt 1 ]; then - echo "Error: provide only 1 optional argument identifying virtual environment path for Ansible" - return 1 - fi - - venv_path="${1:-/usr/local/ghpc-venv}" - - # Get the python3 executable, or install it if not found - get_python_path - get_python_major_version "${python_path}" - get_python_minor_version "${python_path}" - if [ "${python_path}" = "" ] || [ "${python_major_version}" = "2" ] || [ "${python_minor_version}" -lt "${REQ_PYTHON3_VERSION}" ]; then - if ! install_python3; then - return 1 - fi - get_python_major_version "${python_path}" - get_python_minor_version "${python_path}" - else - install_python_deps - fi - - # Install OS-packaged pip - if ! ${python_path} -m pip --version 2>/dev/null; then - if ! install_pip3; then - return 1 - fi - fi - - # Create pip virtual environment for Cluster Toolkit - ${python_path} -m venv "${venv_path}" --copies - venv_python_path=${venv_path}/bin/python3 - - # Upgrade pip if necessary - pip_version=$(${venv_python_path} -m pip --version | sed -nr 's/^pip ([0-9]+\.[0-9]+).*$/\1/p') - pip_major_version=$(echo "${pip_version}" | cut -d '.' -f 1) - if [ "${pip_major_version}" -lt "${REQ_PIP_MAJOR_VERSION}" ]; then - ${venv_python_path} -m pip install --upgrade pip - fi - - # upgrade wheel if necessary - wheel_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^wheel" || true) - if [ "$wheel_pkg" != "wheel==${REQ_PIP_WHEEL_VERSION}" ]; then - ${venv_python_path} -m pip install -U wheel==${REQ_PIP_WHEEL_VERSION} - fi - - # upgrade setuptools if necessary - setuptools_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^setuptools" || true) - if [ "$setuptools_pkg" != "setuptools==${REQ_PIP_SETUPTOOLS_VERSION}" ]; then - ${venv_python_path} -m pip install -U setuptools==${REQ_PIP_SETUPTOOLS_VERSION} - fi - - # configure ansible to always use correct Python binary - if [ ! -f /etc/ansible/ansible.cfg ]; then - mkdir /etc/ansible - cat <<-EOF >/etc/ansible/ansible.cfg - [defaults] - interpreter_python=${venv_python_path} - stdout_callback=debug - stderr_callback=debug - EOF - fi - - # Install ansible - ansible_version="" - if command -v ansible-playbook 1>/dev/null; then - ansible_version=$(ansible-playbook --version 2>/dev/null | sed -nr 's/^ansible-playbook.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p') - ansible_major_vers=$(echo "${ansible_version}" | cut -d '.' -f 1) - ansible_minor_vers=$(echo "${ansible_version}" | cut -d '.' -f 2) - ansible_req_major_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 1) - ansible_req_minor_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 2) - fi - if [ -z "${ansible_version}" ] || [ "${ansible_major_vers}" -ne "${ansible_req_major_vers}" ] || - [ "${ansible_minor_vers}" -lt "${ansible_req_minor_vers}" ]; then - ${venv_python_path} -m pip install ansible=="${REQ_ANSIBLE_PIP_VERSION}" - fi - while read -r cmd; do - if ! [ -L "/usr/bin/${cmd}" ]; then - ln -s "${venv_path}/bin/${cmd}" "/usr/bin/${cmd}" - fi - done <<-EOF - ansible - ansible-config - ansible-connection - ansible-console - ansible-doc - ansible-galaxy - ansible-inventory - ansible-playbook - ansible-pull - ansible-test - ansible-vault - EOF -} - -main "$@" diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh deleted file mode 100644 index 375792459b..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -OS_ID="$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g')" -OS_VERSION="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g')" -OS_VERSION_MAJOR="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//')" -REBOOT_FILE="/etc/.rdma_reboot" - -if { [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; } && { [ "${OS_VERSION_MAJOR}" = "8" ]; }; then - KMOD_VERSION="$(dnf list installed | awk '$1 ~ /^kmod-idpf-irdma(\.|$)/ {print $2}')" - - # For images that do not already have Cloud RDMA drivers installed - if [ -z "${KMOD_VERSION}" ] && [ -z "${REBOOT_FILE}" ]; then - sudo dnf update -y - sudo dnf install https://depot.ciq.com/public/files/gce-accelerator/irdma-kernel-modules-el8-x86_64/irdma-repos.rpm -y - sudo dnf install kmod-idpf-irdma rdma-core libibverbs-utils librdmacm-utils infiniband-diags perftest -y - sudo touch "${REBOOT_FILE}" - reboot - fi - echo "This image has IRDMA packages already installed, exiting." - exit 0 -else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. Cloud RDMA Drivers are only supported on Rocky Linux 8." - exit 1 -fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_docker.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_docker.yml deleted file mode 100644 index f9b0abeb14..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_docker.yml +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Install and configure Docker - hosts: all - become: true - vars: - docker_data_root: '' - docker_daemon_config: '' - enable_docker_world_writable: false - tasks: - - name: Check if docker is installed - ansible.builtin.stat: - path: /usr/bin/docker - register: docker_binary - - name: Download Docker Installer - ansible.builtin.get_url: - url: https://get.docker.com - dest: /tmp/get-docker.sh - owner: root - group: root - mode: '0644' - when: not docker_binary.stat.exists - - name: Install Docker - ansible.builtin.command: sh /tmp/get-docker.sh - register: docker_installed - changed_when: docker_installed.rc != 0 - when: not docker_binary.stat.exists - - name: Create Docker daemon configuration - ansible.builtin.copy: - dest: /etc/docker/daemon.json - mode: '0644' - content: '{{ docker_daemon_config }}' - validate: /usr/bin/dockerd --validate --config-file %s - when: docker_daemon_config - notify: - - Restart Docker - - name: Create Docker service override directory - ansible.builtin.file: - path: /etc/systemd/system/docker.service.d - state: directory - owner: root - group: root - mode: '0755' - - name: Create Docker service override configuration - ansible.builtin.copy: - dest: /etc/systemd/system/docker.service.d/data-root.conf - mode: '0644' - content: | - [Unit] - {% if docker_data_root %} - RequiresMountsFor={{ docker_data_root }} - {% endif %} - After=mount-localssd-raid.service - - name: Create Docker socket override directory - ansible.builtin.file: - path: /etc/systemd/system/docker.socket.d - state: directory - owner: root - group: root - mode: '0755' - when: enable_docker_world_writable - - name: Create Docker socket override configuration - ansible.builtin.copy: - dest: /etc/systemd/system/docker.socket.d/world-writable.conf - mode: '0644' - content: | - [Socket] - SocketMode=0666 - when: enable_docker_world_writable - notify: - - Reload SystemD - - Recreate Docker socket - - name: Delete Docker socket override configuration - ansible.builtin.file: - path: /etc/systemd/system/docker.socket.d/world-writable.conf - state: absent - when: not enable_docker_world_writable - notify: - - Reload SystemD - - Recreate Docker socket - - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - - name: Recreate Docker socket - ansible.builtin.service: - name: docker.socket - state: restarted - - name: Restart Docker - ansible.builtin.service: - name: docker.service - state: restarted - - post_tasks: - - name: Start Docker - ansible.builtin.service: - name: docker.service - state: started - enabled: true diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml deleted file mode 100644 index 9d295dfc7d..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Install network wait service for A3/A4 variants - hosts: all - become: true - tasks: - - - name: Create universal SystemD service for GPU networking delay - when: ansible_os_family == "Debian" - ansible.builtin.copy: - dest: /etc/systemd/system/delay-gpu-network.service - owner: root - group: root - mode: "0644" - content: | - [Unit] - Description=Delay boot on multi-NIC VMs until networks are routable - After=network-online.target - Wants=network-online.target - Before=google-startup-scripts.service - - [Service] - # This condition checks if the machine type is one of the supported A3/A4 variants. - # The service will only run if the machine type matches. - ExecCondition=/bin/bash -c "/usr/bin/curl -s -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/machine-type | grep -qE '(/a3-highgpu-8g|/a3-megagpu-8g|/a3-ultragpu-8g|/a4-highgpu-8g|/a4x-highgpu-4g)$'" - ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -o routable --timeout=180 - ExecStartPost=/bin/sleep 30 - - [Install] - WantedBy=multi-user.target - notify: - - Reload SystemD - - - name: Enable universal GPU network delay service - when: ansible_os_family == "Debian" - ansible.builtin.systemd_service: - name: delay-gpu-network.service - enabled: true - - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml deleted file mode 100644 index 94699471bb..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure Managed Lustre (assumes driver already installed) - hosts: all - become: true - vars: - default_lustre_port: 988 - managed_lustre_port: "{{ default_lustre_port }}" - tasks: - # Ideally changes to this file would also trigger an execution of lnetctl - # command to update accept_port but it is unclear if lnetctl supports this. - - name: Update lnet to use non-default port - when: managed_lustre_port | int != {{ default_lustre_port }} - ansible.builtin.copy: - owner: root - group: root - mode: '0644' - dest: /etc/modprobe.d/lnet.conf - content: | - options lnet accept_port={{ managed_lustre_port | int }} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh deleted file mode 100644 index eb4bf899b8..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -LEGACY_MONITORING_PACKAGE='stackdriver-agent' -LEGACY_MONITORING_SCRIPT_URL='https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh' -LEGACY_LOGGING_PACKAGE='google-fluentd' -LEGACY_LOGGING_SCRIPT_URL='https://dl.google.com/cloudagents/add-logging-agent-repo.sh' - -OPSAGENT_PACKAGE='google-cloud-ops-agent' -OPSAGENT_SCRIPT_URL='https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh' - -ops_or_legacy="${1:-legacy}" - -fail() { - echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" - exit 1 -} - -handle_debian() { - is_legacy_monitoring_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_MONITORING_PACKAGE} | - grep "${LEGACY_MONITORING_PACKAGE} is installed" - } - - is_legacy_logging_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_LOGGING_PACKAGE} | - grep "${LEGACY_LOGGING_PACKAGE} is installed" - } - - is_legacy_installed() { - is_legacy_monitoring_installed || is_legacy_logging_installed - } - - is_opsagent_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${OPSAGENT_PACKAGE} | - grep "${OPSAGENT_PACKAGE} is installed" - } - - install_with_retry() { - MAX_RETRY=50 - RETRY=0 - until [ ${RETRY} -eq ${MAX_RETRY} ] || curl -s "${1}" | bash -s -- --also-install; do - RETRY=$((RETRY + 1)) - echo "WARNING: Installation of ${1} failed on try ${RETRY} of ${MAX_RETRY}" - sleep 5 - done - if [ $RETRY -eq $MAX_RETRY ]; then - echo "ERROR: Installation of ${1} was not successful after ${MAX_RETRY} attempts." - exit 1 - fi - } - - install_opsagent() { - install_with_retry "${OPSAGENT_SCRIPT_URL}" - } - - install_stackdriver_agent() { - install_with_retry "${LEGACY_MONITORING_SCRIPT_URL}" - install_with_retry "${LEGACY_LOGGING_SCRIPT_URL}" - service stackdriver-agent start - service google-fluentd start - } -} - -handle_redhat() { - is_legacy_monitoring_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_MONITORING_PACKAGE} | - grep "${LEGACY_MONITORING_PACKAGE} is installed" - } - - is_legacy_logging_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_LOGGING_PACKAGE} | - grep "${LEGACY_LOGGING_PACKAGE} is installed" - } - - is_legacy_installed() { - is_legacy_monitoring_installed || is_legacy_logging_installed - } - - is_opsagent_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${OPSAGENT_PACKAGE} | - grep "${OPSAGENT_PACKAGE} is installed" - } - - install_opsagent() { - curl -s "${OPSAGENT_SCRIPT_URL}" | bash -s -- --also-install - } - - install_stackdriver_agent() { - curl -sS "${LEGACY_MONITORING_SCRIPT_URL}" | bash -s -- --also-install - curl -sS "${LEGACY_LOGGING_SCRIPT_URL}" | bash -s -- --also-install - service stackdriver-agent start - service google-fluentd start - } -} - -main() { - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - handle_redhat - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - handle_debian - else - fail "Unsupported platform." - fi - - # Handle cases that agent is already installed - if [[ -z "$(is_legacy_monitoring_installed)" && -n $(is_legacy_logging_installed) ]] || - [[ -n "$(is_legacy_monitoring_installed)" && -z $(is_legacy_logging_installed) ]]; then - fail "Bad state: legacy agent is partially installed" - elif [[ "${ops_or_legacy}" == "legacy" ]] && is_legacy_installed; then - echo "Legacy agent is already installed" - exit 0 - elif [[ "${ops_or_legacy}" != "legacy" ]] && is_opsagent_installed; then - echo "Ops agent is already installed" - exit 0 - elif is_legacy_installed || is_opsagent_installed; then - fail "Agent is already installed but does not match requested agent of ${ops_or_legacy}" - fi - - # install agent - if [[ "${ops_or_legacy}" == "legacy" ]]; then - echo "Installing legacy monitoring agent (stackdriver)" - install_stackdriver_agent - else - echo "Installing cloud ops agent" - echo "WARNING: cloud ops agent may have a performance impact. Consider using legacy monitoring agent (stackdriver)." - install_opsagent - fi -} - -main diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh deleted file mode 100644 index 738181aafb..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SCRIPT_COMPLETE_FILE="/run/startup_script_msg" - -# Ensure we're in an interactive terminal and not root -if [ -t 1 ] && [ "$(id -u)" -ne 0 ]; then - # Check if the file has contents otherwise skip - if [ -s "$SCRIPT_COMPLETE_FILE" ]; then - echo - cat "$SCRIPT_COMPLETE_FILE" - echo - fi -fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml deleted file mode 100644 index d94aac81fd..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Configure local SSDs - become: true - hosts: localhost - vars: - raid_name: localssd - array_dev: /dev/md/{{ raid_name }} - fstype: ext4 - interface: nvme - mode: '0755' - mountpoint: /mnt/{{ raid_name }} - tasks: - - name: Get local SSD devices - ansible.builtin.find: - file_type: link - path: /dev/disk/by-id - patterns: google-local-{{ "nvme-" if interface == "nvme" else "" }}ssd-* - register: local_ssd_devices - - - name: Exit if zero local ssd found - ansible.builtin.meta: end_play - when: local_ssd_devices.files | length == 0 - - - name: Install mdadm - ansible.builtin.package: - name: mdadm - state: present - - # this service will act during the play and upon reboots to ensure that local - # SSD volumes are always assembled into a RAID and re-formatted if necessary; - # there are many scenarios where a VM can be stopped or migrated during - # maintenance and the contents of local SSD will be discarded - - name: Install service to create local SSD RAID and format it - ansible.builtin.copy: - dest: /etc/systemd/system/create-localssd-raid.service - mode: 0644 - content: | - [Unit] - After=local-fs.target - Before=slurmd.service docker.service - ConditionPathExists=!{{ array_dev }} - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/usr/bin/bash -c "/usr/sbin/mdadm --create {{ array_dev }} --name={{ raid_name }} --homehost=any --level=0 --raid-devices={{ local_ssd_devices.files | length }} /dev/disk/by-id/google-local-nvme-ssd-*{{ " --force" if local_ssd_devices.files | length == 1 else "" }}" - ExecStartPost=/usr/sbin/mkfs -t {{ fstype }}{{ " -m 0" if fstype == "ext4" else "" }} {{ array_dev }} - - [Install] - WantedBy=slurmd.service docker.service - - - name: Create RAID array and format - ansible.builtin.systemd: - name: create-localssd-raid.service - state: started - enabled: true - daemon_reload: true - - - name: Install service to mount local SSD array - ansible.builtin.copy: - dest: /etc/systemd/system/mount-localssd-raid.service - mode: 0644 - content: | - [Unit] - After=local-fs.target create-localssd-raid.service - Before=slurmd.service docker.service - Wants=create-localssd-raid.service - ConditionPathIsMountPoint=!{{ mountpoint }} - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/usr/bin/systemd-mount -t {{ fstype }} -o discard,defaults,nofail {{ array_dev }} {{ mountpoint }} - ExecStartPost=/usr/bin/chmod {{ mode }} {{ mountpoint }} - ExecStop=/usr/bin/systemd-umount {{ mountpoint }} - - [Install] - WantedBy=slurmd.service docker.service - - - name: Mount RAID array and set permissions - ansible.builtin.systemd: - name: mount-localssd-raid.service - state: started - enabled: true - daemon_reload: true diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh deleted file mode 100644 index 1c8018fb01..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! -d ~/.ssh/ ]; then - source /usr/local/ghpc-venv/bin/activate - ansible-playbook /usr/local/ghpc/setup-ssh-keys.yml -fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml deleted file mode 100644 index 692896bb9c..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Setup SSH Keys for user - become: false - hosts: localhost - vars: - pub_key_path: "{{ ansible_env.HOME }}/.ssh" - pub_key_file: "{{ pub_key_path }}/id_rsa" - auth_key_file: "{{ pub_key_path }}/authorized_keys" - tasks: - - name: "Create .ssh folder" - ansible.builtin.file: - path: "{{ pub_key_path }}" - state: directory - mode: 0700 - owner: "{{ ansible_user_id }}" - - name: Create keys - community.crypto.openssh_keypair: - path: "{{ pub_key_file }}" - owner: "{{ ansible_user_id }}" - - name: Copy public key to authorized keys - ansible.builtin.copy: - src: "{{ pub_key_file }}.pub" - dest: "{{ auth_key_file }}" - owner: "{{ ansible_user_id }}" - mode: 0644 diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh deleted file mode 100644 index 8ca40bc73f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh +++ /dev/null @@ -1,39 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This code contains minor changes from the original: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 - -stdlib::main() { - DELETE_AT_EXIT="$(mktemp -d)" - readonly DELETE_AT_EXIT - - # Initialize state required by other functions, e.g. debug() - stdlib::init - stdlib::debug "Loaded startup-script-stdlib as an executable." - - stdlib::load_config_values - - stdlib::load_runners -} - -# if script is being executed and not sourced. -if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then - stdlib::finish() { - [[ -d ${DELETE_AT_EXIT:-} ]] && rm -rf "${DELETE_AT_EXIT}" - } - trap stdlib::finish EXIT - - stdlib::main "$@" -fi diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh deleted file mode 100644 index 589a3215ab..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh +++ /dev/null @@ -1,266 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This code contains minor changes from the original in: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 - -# Standard library of functions useful for startup scripts. - -# These are outside init_global_vars so logging functions work with the most -# basic case of `source startup-script-stdlib.sh` -readonly SYSLOG_DEBUG_PRIORITY="${SYSLOG_DEBUG_PRIORITY:-syslog.debug}" -readonly SYSLOG_INFO_PRIORITY="${SYSLOG_INFO_PRIORITY:-syslog.info}" -readonly SYSLOG_ERROR_PRIORITY="${SYSLOG_ERROR_PRIORITY:-syslog.error}" -# Global counter of how many times stdlib::init() has been called. -STARTUP_SCRIPT_STDLIB_INITIALIZED=0 - -# Error codes -readonly E_RUN_OR_DIE=5 -readonly E_MISSING_MANDATORY_ARG=9 -readonly E_UNKNOWN_ARG=10 - -SCRIPT_COMPLETE_FILE="/run/startup_script_msg" -SUCCESS_MESSAGE="* NOTICE **: The Cluster Toolkit startup scripts have finished running successfully." -readonly SUCCESS_MESSAGE -ERROR_MESSAGE="** ERROR **: The Cluster Toolkit startup scripts have finished running, but produced an error." -readonly ERROR_MESSAGE -WARNING_MESSAGE="** WARNING **: The Cluster Toolkit startup scripts are currently running." -readonly WARNING_MESSAGE - -stdlib::debug() { - [[ -z ${DEBUG:-} ]] && return 0 - local ds msg - msg="$*" - logger -p "${SYSLOG_DEBUG_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - [[ -n ${QUIET:-} ]] && return 0 - ds="$(date +"${DATE_FMT}") " - echo -e "${BLUE}${ds}Debug [$$]: ${msg}${NC}" >&2 -} - -stdlib::info() { - local ds msg - msg="$*" - logger -p "${SYSLOG_INFO_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - [[ -n ${QUIET:-} ]] && return 0 - ds="$(date +"${DATE_FMT}") " - echo -e "${GREEN}${ds}Info [$$]: ${msg}${NC}" >&2 -} - -stdlib::error() { - local ds msg - msg="$*" - ds="$(date +"${DATE_FMT}") " - logger -p "${SYSLOG_ERROR_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - echo -e "${RED}${ds}Error [$$]: ${msg}${NC}" >&2 -} - -stdlib::announce_runners_start() { - if [ -z "$recursive_proc" ]; then - wall -n "$WARNING_MESSAGE" - echo "$WARNING_MESSAGE" >"$SCRIPT_COMPLETE_FILE" - fi - export recursive_proc=$((${recursive_proc:=0} + 1)) -} - -stdlib::announce_runners_end() { - exit_code=$1 - export recursive_proc=$((${recursive_proc:=0} - 1)) - if [ "$recursive_proc" -le "0" ]; then - if [ "$exit_code" -ne "0" ]; then - wall -n "$ERROR_MESSAGE" - echo "$ERROR_MESSAGE" >"$SCRIPT_COMPLETE_FILE" - else - wall -n "$SUCCESS_MESSAGE" - echo -n "" >"$SCRIPT_COMPLETE_FILE" - fi - fi -} - -# The main initialization function of this library. This should be kept to the -# minimum amount of work required for all functions to operate cleanly. -stdlib::init() { - if [[ ${STARTUP_SCRIPT_STDLIB_INITIALIZED} -gt 0 ]]; then - stdlib::info 'stdlib::init()'" already initialized, no action taken." - return 0 - fi - ((STARTUP_SCRIPT_STDLIB_INITIALIZED++)) || true - stdlib::init_global_vars - stdlib::init_directories - stdlib::debug "stdlib::init(): startup-script-stdlib.sh initialized and ready" -} - -# Initialize global variables. -stdlib::init_global_vars() { - # The program name, used for logging. - readonly PROG="${PROG:-startup-script-stdlib}" - # Date format used for stderr logging. Passed to date + command. - readonly DATE_FMT="${DATE_FMT:-"%a %b %d %H:%M:%S %z %Y"}" - # var directory - readonly VARDIR="${VARDIR:-/var/lib/startup}" - # Override this with file://localhost/tmp/foo/bar in spec test context - readonly METADATA_BASE="${METADATA_BASE:-http://metadata.google.internal}" - - # Color variables - if [[ -n ${COLOR:-} ]]; then - readonly NC='\033[0m' # no color - readonly RED='\033[0;31m' # error - readonly GREEN='\033[0;32m' # info - readonly BLUE='\033[0;34m' # debug - else - readonly NC='' - readonly RED='' - readonly GREEN='' - readonly BLUE='' - fi - - return 0 -} - -stdlib::init_directories() { - if ! [[ -e ${VARDIR} ]]; then - install -d -m 0755 -o 0 -g 0 "${VARDIR}" - fi -} - -## -# Get a metadata key. When used without -o, this function is guaranteed to -# produce no output on STDOUT other than the retrieved value. This is intended -# to support the use case of -# FOO="$(stdlib::metadata_get -k instance/attributes/foo)" -# -# If the requested key does not exist, the error code will be 22 and zero bytes -# written to STDOUT. -stdlib::metadata_get() { - local OPTIND opt key outfile - local metadata="${METADATA_BASE%/}/computeMetadata/v1" - local exit_code - while getopts ":k:o:" opt; do - case "${opt}" in - k) key="${OPTARG}" ;; - o) outfile="${OPTARG}" ;; - :) - stdlib::error "Invalid option: -${OPTARG} requires an argument" - stdlib::metadata_get_usage - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error "Unknown option: -${opt}" - stdlib::metadata_get_usage - return "${E_UNKNOWN_ARG}" - ;; - esac - done - local url="${metadata}/${key#/}" - - stdlib::debug "Getting metadata resource url=${url}" - if [[ -z ${outfile:-} ]]; then - curl --location --silent --connect-timeout 1 --fail \ - -H 'Metadata-Flavor: Google' "$url" 2>/dev/null - exit_code=$? - else - stdlib::cmd curl --location \ - --silent \ - --connect-timeout 1 \ - --fail \ - --output "${outfile}" \ - -H 'Metadata-Flavor: Google' \ - "$url" - exit_code=$? - fi - case "${exit_code}" in - 22 | 37) - stdlib::debug "curl exit_code=${exit_code} for url=${url}" \ - "(Does not exist)" - ;; - esac - return "${exit_code}" -} - -stdlib::metadata_get_usage() { - stdlib::info 'Usage: stdlib::metadata_get -k ' - stdlib::info 'For example: stdlib::metadata_get -k instance/attributes/startup-config' -} - -# Load configuration values in the spirit of /etc/sysconfig defaults, but from -# metadata instead of the filesystem. -stdlib::load_config_values() { - local config_file - local key="instance/attributes/startup-script-config" - # shellcheck disable=SC2119 - config_file="$(stdlib::mktemp)" - stdlib::metadata_get -k "${key}" -o "${config_file}" - local status=$? - case "$status" in - 0) - stdlib::debug "SUCCESS: Configuration data sourced from $key" - ;; - 22 | 37) - stdlib::debug "no configuration data loaded from $key" - ;; - *) - stdlib::error "metadata_get -k $key returned unknown status=${status}" - ;; - esac - # shellcheck source=/dev/null - source "${config_file}" -} - -# Run a command logging the entry and exit. Intended for system level commands -# and operational debugging. Not intended for use with redirection. This is -# not named run() because bats uses a run() function. -stdlib::cmd() { - local exit_code argv=("$@") - stdlib::debug "BEGIN: stdlib::cmd() command=[${argv[*]}]" - "${argv[@]}" - exit_code=$? - stdlib::debug "END: stdlib::cmd() command=[${argv[*]}] exit_code=${exit_code}" - return $exit_code -} - -# Run a command successfully or exit the program with an error. -stdlib::run_or_die() { - if ! stdlib::cmd "$@"; then - stdlib::error "stdlib::run_or_die(): exiting with exit code ${E_RUN_OR_DIE}." - exit "${E_RUN_OR_DIE}" - fi -} - -# Intended to take advantage of automatic cleanup of startup script library -# temporary files without exporting a modified TMPDIR to child processes, which -# would cause the children to have their TMPDIR deleted out from under them. -# shellcheck disable=SC2120 -stdlib::mktemp() { - TMPDIR="${DELETE_AT_EXIT:-${TMPDIR}}" mktemp "$@" -} - -# Return a nice error message if a mandatory argument is missing. -stdlib::mandatory_argument() { - local OPTIND opt name flag - while getopts ":n:f:" opt; do - case "$opt" in - n) name="${OPTARG}" ;; - f) flag="${OPTARG}" ;; - :) - stdlib::error "Invalid argument: -${OPTARG} requires an argument to stdlib::mandatory_argument()" - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error "Unknown argument: -${OPTARG}" - stdlib::info "Usage: stdlib::mandatory_argument -n -f " - return "${E_UNKNOWN_ARG}" - ;; - esac - done - stdlib::error "Invalid argument: -${flag} requires an argument to ${name}()." -} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/main.tf b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/main.tf deleted file mode 100644 index 02124eeddc..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/main.tf +++ /dev/null @@ -1,306 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "startup-script", ghpc_role = "scripts" }) -} - -locals { - monitoring_agent_installer = ( - var.install_cloud_ops_agent || var.install_stackdriver_agent ? - [{ - type = "shell" - source = "${path.module}/files/install_monitoring_agent.sh" - destination = "install_monitoring_agent_automatic.sh" - args = var.install_cloud_ops_agent ? "ops" : "legacy" # install legacy (stackdriver) - }] : - [] - ) - - warnings = [ - { - type = "data" - content = file("${path.module}/files/running-script-warning.sh") - destination = "/etc/profile.d/99-running-script-warning.sh" - } - ] - - configure_ssh = length(var.configure_ssh_host_patterns) > 0 - host_args = { - host_name_prefix = var.configure_ssh_host_patterns - } - - prefix_file = "/tmp/prefix_file.json" - ansible_docker_settings_file = "/tmp/ansible_docker_settings.json" - - docker_config = try(jsondecode(var.docker.daemon_config), {}) - docker_data_root = try(local.docker_config.data-root, null) - - configure_ssh_runners = local.configure_ssh ? [ - { - type = "data" - source = "${path.module}/files/setup-ssh-keys.sh" - destination = "/usr/local/ghpc/setup-ssh-keys.sh" - }, - { - type = "data" - source = "${path.module}/files/setup-ssh-keys.yml" - destination = "/usr/local/ghpc/setup-ssh-keys.yml" - }, - { - type = "data" - content = jsonencode(local.host_args) - destination = local.prefix_file - }, - { - type = "ansible-local" - content = file("${path.module}/files/configure-ssh.yml") - destination = "configure-ssh.yml" - args = "-e @${local.prefix_file}" - } - ] : [] - - proxy_runner = var.http_proxy == "" ? [] : [ - { - type = "data" - destination = "/etc/profile.d/http_proxy.sh" - content = <<-EOT - #!/bin/bash - export http_proxy=${var.http_proxy} - export https_proxy=${var.http_proxy} - export NO_PROXY=${var.http_no_proxy} - EOT - }, - { - type = "shell" - source = "${path.module}/files/configure_proxy.sh" - destination = "configure_proxy.sh" - args = var.http_proxy - } - ] - - ofi_runner = !var.set_ofi_cloud_rdma_tunables ? [] : [ - { - type = "data" - destination = "/etc/profile.d/set_ofi_cloud_rdma_tunables.sh" - content = <<-EOT - #!/bin/bash - export FI_PROVIDER="verbs;ofi_rxm" - export FI_OFI_RXM_USE_RNDV_WRITE=0 - export FI_VERBS_INLINE_SIZE=39 - export I_MPI_FABRICS="shm:ofi" - export FI_UNIVERSE_SIZE=1024 - export I_MPI_ADJUST_ALLTOALL=1 - export I_MPI_ADJUST_IALLTOALL=1 - export I_MPI_ADJUST_BCAST=4 - export I_MPI_ADJUST_IBCAST=1 - EOT - }, - ] - - rdma_runner = !var.install_cloud_rdma_drivers ? [] : [ - { - type = "shell" - source = "${path.module}/files/install_cloud_rdma_drivers.sh" - destination = "install_cloud_rdma_drivers.sh" - } - ] - - docker_runner = !var.docker.enabled ? [] : [ - { - type = "data" - destination = local.ansible_docker_settings_file - content = jsonencode({ - enable_docker_world_writable = var.docker.world_writable - docker_daemon_config = var.docker.daemon_config - docker_data_root = local.docker_data_root - }) - }, - { - type = "ansible-local" - destination = "install_docker.yml" - content = file("${path.module}/files/install_docker.yml") - args = "-e \"@${local.ansible_docker_settings_file}\"" - }, - ] - - managed_lustre_runner = !var.managed_lustre.enabled ? [] : [ - { - type = "ansible-local" - destination = "install_managed_lustre.yml" - content = file("${path.module}/files/install_managed_lustre.yml") - args = "-e managed_lustre_port=${var.managed_lustre.port}" - }, - ] - - gpu_network_wait_online_runner = !var.enable_gpu_network_wait_online ? [] : [ - { - type = "ansible-local" - destination = "install_gpu_network_wait_online.yml" - content = file("${path.module}/files/install_gpu_network_wait_online.yml") - args = "" - }, - ] - - local_ssd_filesystem_enabled = can(coalesce(var.local_ssd_filesystem.mountpoint)) - raid_setup = !local.local_ssd_filesystem_enabled ? [] : [ - { - type = "ansible-local" - destination = "setup-raid.yml" - content = file("${path.module}/files/setup-raid.yml") - args = join(" ", [ - "-e mountpoint=${var.local_ssd_filesystem.mountpoint}", - "-e fs_type=${var.local_ssd_filesystem.fs_type}", - "-e mode=${var.local_ssd_filesystem.permissions}", - ]) - }, - ] - - supplied_ansible_runners = anytrue([for r in var.runners : r.type == "ansible-local"]) - has_ansible_runners = anytrue([ - local.supplied_ansible_runners, - local.configure_ssh, - var.docker.enabled, - var.managed_lustre.enabled, - var.enable_gpu_network_wait_online, - local.local_ssd_filesystem_enabled - ]) - - install_ansible = coalesce(var.install_ansible, local.has_ansible_runners) - ansible_installer = local.install_ansible ? [{ - type = "shell" - source = "${path.module}/files/install_ansible.sh" - destination = "install_ansible_automatic.sh" - args = var.ansible_virtualenv_path - }] : [] - - hotfix_runner = [{ - type = "shell" - source = "${path.module}/files/early_run_hotfixes.sh" - destination = "early_run_hotfixes.sh" - }] - - runners = concat( - local.warnings, - local.hotfix_runner, - local.proxy_runner, - local.ofi_runner, - local.rdma_runner, - local.monitoring_agent_installer, - local.ansible_installer, - local.raid_setup, # order RAID early to ensure filesystem is ready for subsequent runners - local.managed_lustre_runner, - local.configure_ssh_runners, - local.docker_runner, - local.gpu_network_wait_online_runner, - var.runners - ) - - bucket_regex = "^gs://([^/]*)/*(.*)" - gcs_bucket_path_trimmed = var.gcs_bucket_path == null ? null : trimsuffix(var.gcs_bucket_path, "/") - storage_folder_path = local.gcs_bucket_path_trimmed == null ? null : regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[1] - storage_folder_path_prefix = local.storage_folder_path == null || local.storage_folder_path == "" ? "" : "${local.storage_folder_path}/" - - user_provided_bucket_name = try(regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[0], null) - storage_bucket_name = coalesce(one(google_storage_bucket.configs_bucket[*].name), local.user_provided_bucket_name) - - load_runners = templatefile( - "${path.module}/templates/startup-script-custom.tftpl", - { - bucket = local.storage_bucket_name, - http_proxy = var.http_proxy, - no_proxy = var.http_no_proxy, - runners = [ - for runner in local.runners : { - object = google_storage_bucket_object.scripts[basename(runner["destination"])].output_name - type = runner["type"] - destination = runner["destination"] - args = contains(keys(runner), "args") ? runner["args"] : "" - } - ] - } - ) - - stdlib_head = file("${path.module}/files/startup-script-stdlib-head.sh") - get_from_bucket = file("${path.module}/files/get_from_bucket.sh") - stdlib_body = file("${path.module}/files/startup-script-stdlib-body.sh") - - # List representing complete content, to be concatenated together. - stdlib_list = [ - local.stdlib_head, - local.get_from_bucket, - local.load_runners, - local.stdlib_body, - ] - - # Final content output to the user - stdlib = join("", local.stdlib_list) - - runners_map = { for runner in local.runners : - basename(runner["destination"]) => { - content = lookup(runner, "content", null) - source = lookup(runner, "source", null) - } - } -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket" "configs_bucket" { - count = var.gcs_bucket_path == null ? 1 : 0 - project = var.project_id - name = "${var.deployment_name}-startup-scripts-${random_id.resource_name_suffix.hex}" - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -resource "google_storage_bucket_iam_binding" "viewers" { - bucket = local.storage_bucket_name - role = "roles/storage.objectViewer" - members = var.bucket_viewers -} - -resource "google_storage_bucket_object" "scripts" { - # this writes all scripts exactly once into GCS - for_each = local.runners_map - name = "${local.storage_folder_path_prefix}${each.key}-${substr(try(md5(each.value.content), filemd5(each.value.source)), 0, 4)}" - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) - bucket = local.storage_bucket_name - timeouts { - create = "10m" - update = "10m" - } - - lifecycle { - precondition { - condition = !(var.install_cloud_ops_agent && var.install_stackdriver_agent) - error_message = "Only one of var.install_stackdriver_agent or var.install_cloud_ops_agent can be set. Stackdriver is recommended for best performance." - } - } -} - -resource "local_file" "debug_file" { - for_each = toset(var.debug_file != null ? [var.debug_file] : []) - filename = var.debug_file - content = local.stdlib -} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/metadata.yaml b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/metadata.yaml deleted file mode 100644 index 2ada34471f..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - storage.googleapis.com diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/outputs.tf b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/outputs.tf deleted file mode 100644 index 6a15082814..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/outputs.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "script to load and run all runners, as a string value." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} - -output "compute_startup_script" { - description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} - -output "controller_startup_script" { - description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl deleted file mode 100644 index 3c894b00b0..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl +++ /dev/null @@ -1,65 +0,0 @@ - - -stdlib::run_playbook() { - if [ ! "$(which ansible-playbook)" ]; then - stdlib::error "ansible-playbook not found"\ - "Please install ansible before running ansible-local runners." - exit 1 - fi - ansible-playbook --connection=local --inventory=localhost, --limit localhost $1 $2 - ret_code=$? - return $${ret_code} -} - -stdlib::runner() { - - type=$1 - object=$2 - destination=$3 - tmpdir=$4 - args=$5 - - destpath="$(dirname $destination)" - filename="$(basename $destination)" - - if [ "$destpath" = "." ]; then - destpath=$tmpdir - fi - - stdlib::get_from_bucket -u "gs://${bucket}/$object" -d "$destpath" -f "$filename" - - stdlib::info "=== start executing runner: $object ===" - case "$1" in - ansible-local) stdlib::run_playbook "$destpath/$filename" "$args";; - shell) chmod u+x /$destpath/$filename && $destpath/$filename $args;; - esac - - exit_code=$? - stdlib::info "=== $object finished with exit_code=$exit_code ===" - if [ "$exit_code" -ne "0" ] ; then - stdlib::error "=== execution of $object failed, exiting ===" - stdlib::announce_runners_end "$exit_code" - exit $exit_code - fi -} - -stdlib::load_runners(){ - tmpdir="$(mktemp -d)" - - stdlib::debug "=== BEGIN Running runners ===" - stdlib::announce_runners_start - - %{if http_proxy != "" ~} - stdlib::info "=== Setting HTTP_PROXY,HTTPS_PROXY to ${http_proxy} ===" - export http_proxy=${http_proxy} - export https_proxy=${http_proxy} - export NO_PROXY=${no_proxy} - %{endif ~} - - %{for r in runners ~} - stdlib::runner "${r.type}" "${r.object}" "${r.destination}" $${tmpdir} "${r.args}" - %{endfor ~} - - stdlib::announce_runners_end "0" - stdlib::debug "=== END Running runners ===" -} diff --git a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/variables.tf b/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/variables.tf deleted file mode 100644 index 7080085ece..0000000000 --- a/deletion-test/build_script/modules/embedded/modules/scripts/startup-script/variables.tf +++ /dev/null @@ -1,298 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used to name GCS bucket for startup scripts." - type = string -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string - default = null -} - -variable "bucket_viewers" { - description = "Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account)" - type = list(string) - default = [] - - validation { - condition = alltrue([ - for u in var.bucket_viewers : length(regexall("^(allUsers$|allAuthenticatedUsers$|user:|group:|serviceAccount:|domain:)", u)) > 0 - ]) - error_message = "Bucket viewer members must begin with user/group/serviceAccount/domain following https://cloud.google.com/iam/docs/reference/rest/v1/Policy#Binding" - } -} - -variable "debug_file" { - description = "Path to an optional local to be written with 'startup_script'." - type = string - default = null -} - -variable "labels" { - description = "Labels for the created GCS bucket. Key-value pairs." - type = map(string) -} - -variable "runners" { - description = < 0 - error_message = "The POSIX permissions for the mountpoint must be represented as a 3 or 4-digit octal" - } - - default = { - fs_type = "ext4" - mountpoint = "" - permissions = "0755" - } - - nullable = false -} - -variable "install_cloud_ops_agent" { - description = "Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true." - type = bool - default = false -} - -variable "install_stackdriver_agent" { - description = "Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance." - type = bool - default = false -} - -variable "install_ansible" { - description = "Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used." - type = bool - default = null -} - -variable "configure_ssh_host_patterns" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 4.84 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [home\_pv](#module\_home\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | -| [kubectl\_apply](#module\_kubectl\_apply) | ../../../../modules/management/kubectl-apply | n/a | -| [slurm\_key\_pv](#module\_slurm\_key\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.gke_nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | -| [filestore\_id](#input\_filestore\_id) | An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `list(string)` | n/a | yes | -| [image](#input\_image) | The image for slurm daemon | `string` | n/a | yes | -| [instance\_templates](#input\_instance\_templates) | The URLs of Instance Templates | `list(string)` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| n/a | yes | -| [node\_count\_static](#input\_node\_count\_static) | The number of static nodes in node-pool | `number` | n/a | yes | -| [node\_pool\_names](#input\_node\_pool\_names) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `list(string)` | n/a | yes | -| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | -| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used in slurm controller | `string` | n/a | yes | -| [slurm\_controller\_instance](#input\_slurm\_controller\_instance) | Slurm cluster controller instance | `any` | n/a | yes | -| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | -| [subnetwork](#input\_subnetwork) | Primary subnetwork object | `any` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset\_name](#output\_nodeset\_name) | Name of the new Slinky nodset | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/main.tf deleted file mode 100644 index 8b2f1deeac..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/main.tf +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -### GKE NodeSet -locals { - manifest_path = "${path.module}/templates/nodeset-general.yaml.tftpl" -} - -module "kubectl_apply" { - source = "../../../../modules/management/kubectl-apply" - - cluster_id = var.cluster_id - project_id = var.project_id - - apply_manifests = [{ - source = local.manifest_path, - template_vars = { - slurm_namespace = var.slurm_namespace, - nodeset_name = "${var.slurm_cluster_name}-${var.nodeset_name}", - nodeset_cr_name = "${var.slurm_cluster_name}-${var.nodeset_name}", - controller_name = "${var.slurm_cluster_name}-controller", - node_pool_name = var.node_pool_names[0], - node_count = var.node_count_static, - image = var.image, - home_pvc = module.home_pv.pvc_name - slurm_key_pvc = module.slurm_key_pv.pvc_name - } - }] -} - -data "google_storage_bucket" "this" { - name = var.slurm_bucket[0].name - - depends_on = [var.slurm_bucket] -} - -### Slurm NodeSet -locals { - nodeset = { - gke_nodepool = var.node_pool_names[0] - nodeset_name = var.nodeset_name - node_count_static = var.node_count_static - subnetwork = "https://www.googleapis.com/compute/v1/projects/${var.project_id}/regions/${var.subnetwork.region}/subnetworks/${var.subnetwork.name}" - instance_template = var.instance_templates[0] - } -} - -resource "google_storage_bucket_object" "gke_nodeset_config" { - bucket = data.google_storage_bucket.this.name - name = "${var.slurm_bucket_dir}/nodeset_configs/${var.nodeset_name}.yaml" - content = yamlencode(local.nodeset) -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml deleted file mode 100644 index ea2cfc221e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/output.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/output.tf deleted file mode 100644 index 15970ff0b7..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/output.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset_name" { - description = "Name of the new Slinky nodset" - value = local.nodeset.nodeset_name -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf deleted file mode 100644 index 8a190c4019..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - slurm_key_storage = { - server_ip = var.slurm_controller_instance.network_interface[0].network_ip - remote_mount = "/slurm/key_distribution" # defined in /community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py - client_install_runner = {} - mount_runner = {} - fs_type = "" - local_mount = "" - mount_options = "" - } -} - -module "slurm_key_pv" { - source = "../../../../modules/file-system/gke-persistent-volume" - labels = {} - capacity_gib = 1 - cluster_id = var.cluster_id - filestore_id = "projects/empty/locations/empty/instances/empty" # this does not apply since this NFS is not a filestore - namespace = var.slurm_namespace - network_storage = local.slurm_key_storage - pv_name = "slurm-key-pv" - pvc_name = "slurm-key-pvc" -} - -# Assume the var.network_storage[0] will be home and only one home pv is accepted for now. -module "home_pv" { - source = "../../../../modules/file-system/gke-persistent-volume" - labels = {} - capacity_gib = 1024 - cluster_id = var.cluster_id - filestore_id = var.filestore_id[0] - network_storage = var.network_storage[0] - namespace = var.slurm_namespace - pv_name = "home-pv" - pvc_name = "home-pvc" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl deleted file mode 100644 index a5a4a5e7ac..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl +++ /dev/null @@ -1,203 +0,0 @@ -apiVersion: slinky.slurm.net/v1alpha1 -kind: NodeSet -metadata: - annotations: - meta.helm.sh/release-name: slurm - meta.helm.sh/release-namespace: ${slurm_namespace} - labels: - app.kubernetes.io/component: compute - app.kubernetes.io/instance: slurm - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: slurmd - app.kubernetes.io/part-of: slurm - app.kubernetes.io/version: "24.11" - helm.sh/chart: slurm-0.3.0 - nodeset.slinky.slurm.net/name: ${nodeset_name} - name: ${nodeset_name} - namespace: ${slurm_namespace} -spec: - clusterName: slurm - persistentVolumeClaimRetentionPolicy: - whenDeleted: Retain - whenScaled: Retain - replicas: ${node_count} - revisionHistoryLimit: 0 - selector: - matchLabels: - app.kubernetes.io/instance: slurm - app.kubernetes.io/name: slurmd - nodeset.slinky.slurm.net/name: ${nodeset_name} - serviceName: slurm-compute - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: slurmd - labels: - app.kubernetes.io/component: compute - app.kubernetes.io/instance: slurm - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: slurmd - app.kubernetes.io/part-of: slurm - app.kubernetes.io/version: "24.11" - helm.sh/chart: slurm-0.3.0 - nodeset.slinky.slurm.net/name: ${nodeset_name} - spec: - automountServiceAccountToken: false - containers: - - args: - - -g - - -- - - bash - - -c - - | - mkdir -p /usr/local/lib/slurm - ln -s /usr/lib/x86_64-linux-gnu/slurm/spank_pyxis.so /usr/local/lib/slurm/spank_pyxis.so - /usr/local/bin/entrypoint.sh -Z --conf-server ${controller_name}:6825 -N $NODE_NAME - command: - - tini - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_CPUS - value: "0" - - name: POD_MEMORY - value: "0" - image: ${image} - imagePullPolicy: IfNotPresent - name: slurmd - ports: - - containerPort: 6818 - name: slurmd - protocol: TCP - readinessProbe: - exec: - command: - - scontrol - - show - - slurmd - resources: {} - securityContext: - capabilities: - add: - - BPF - - NET_ADMIN - - SYS_ADMIN - - SYS_NICE - privileged: true - volumeMounts: - - mountPath: /etc/slurm - name: etc-slurm - - mountPath: /run - name: run - - mountPath: /var/spool/slurmd - name: slurm-spool - - mountPath: /var/log/slurm - name: slurm-log - - mountPath: /home - name: home-pvc - dnsConfig: - searches: - - ${controller_name} - hostNetwork: true - initContainers: - - command: - - tini - - -g - - -- - - bash - - -c - - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# - SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env - contains:\n# SLURM_USER - username or UID\n\nfunction init::common() {\n\tlocal - dir\n\n\tdir=/var/spool/slurmd\n\tmkdir -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" - \"$dir\"\n\tchmod -v 700 \"$dir\"\n\n\tdir=/var/spool/slurmctld\n\tmkdir - -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" \"$dir\"\n\tchmod - -v 700 \"$dir\"\n}\n\nfunction init::slurm() {\n\tSLURM_MOUNT=/mnt/slurm\n\tSLURM_DIR=/mnt/etc/slurm\n\n\t# - Workaround to ephemeral volumes not supporting securityContext\n\t# https://github.com/kubernetes/kubernetes/issues/81089\n\n\t# - Copy Slurm config files, secrets, and scripts\n\tmkdir -p \"$SLURM_DIR\"\n\tfind - \"$${SLURM_MOUNT}\" -type f -name \"*.conf\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind - \"$${SLURM_MOUNT}\" -type f -name \"*.key\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind - \"$${SLURM_MOUNT}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" - -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind \"$${SLURM_MOUNT}\" -type - f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 | xargs - -0r cp -vt \"$${SLURM_DIR}\"\n\n\t# Set general permissions and ownership\n\tfind - \"$${SLURM_DIR}\" -type f -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind - \"$${SLURM_DIR}\" -type f -name \"*.conf\" -print0 | xargs -0r chmod -v 644\n\tfind - \"$${SLURM_DIR}\" -type f -name \"*.key\" -print0 | xargs -0r chmod -v 600\n\tfind - \"$${SLURM_DIR}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" - -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind \"$${SLURM_DIR}\" - -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 - | xargs -0r chmod -v 755\n\n\t# Inject secrets into certain config files\n\tlocal - dbd_conf=\"slurmdbd.conf\"\n\tif [[ -f \"$${SLURM_MOUNT}/$${dbd_conf}\" ]]; - then\n\t\techo \"Injecting secrets from environment into: $${dbd_conf}\"\n\t\trm - -f \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tenvsubst <\"$${SLURM_MOUNT}/$${dbd_conf}\" - >\"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" - \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchmod -v 600 \"$${SLURM_DIR}/$${dbd_conf}\"\n\tfi\n\n\t# - Display Slurm directory files\n\tls -lAF \"$${SLURM_DIR}\"\n}\n\nfunction - main() {\n\tinit::common\n\tinit::slurm\n}\nmain\n" - env: - - name: SLURM_USER - value: slurm - image: ${image} - imagePullPolicy: IfNotPresent - name: init - resources: {} - volumeMounts: - - mountPath: /mnt/slurm - name: slurm-config - - mountPath: /mnt/etc/slurm - name: etc-slurm - - command: - - tini - - -g - - -- - - bash - - -c - - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# - SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env - contains:\n# SOCKET - Named socket to read from\n\nmkdir -v -p \"$(dirname - \"$SOCKET\")\"\nrm -f \"$SOCKET\"\nif ! [ -f \"$SOCKET\" ]; then\n\tmkfifo - -m 777 \"$SOCKET\"\nfi\nwhile IFS=\"\" read data; do\n\techo $data\ndone - <\"$SOCKET\"\n" - env: - - name: SOCKET - value: /var/log/slurm/slurmd.log - image: ghcr.io/slinkyproject/sackd:24.11-ubuntu24.04 - imagePullPolicy: IfNotPresent - name: logfile - resources: {} - restartPolicy: Always - volumeMounts: - - mountPath: /var/log/slurm - name: slurm-log - nodeSelector: - cloud.google.com/gke-nodepool: ${node_pool_name} - tolerations: - - effect: NoSchedule - key: nvidia.com/gpu - operator: Equal - value: present - volumes: - - emptyDir: - medium: Memory - name: etc-slurm - - emptyDir: {} - name: run - - name: slurm-config - persistentVolumeClaim: - claimName: ${slurm_key_pvc} - - emptyDir: - medium: Memory - name: slurm-spool - - emptyDir: - medium: Memory - name: slurm-log - - name: home-pvc - persistentVolumeClaim: - claimName: ${home_pvc} - updateStrategy: - rollingUpdate: - maxUnavailable: 20% - type: RollingUpdate diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/variables.tf deleted file mode 100644 index c091a0da86..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/variables.tf +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "cluster_id" { - description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" - type = string -} - -variable "slurm_cluster_name" { - type = string - description = "Cluster name, used in slurm controller" - - validation { - condition = var.slurm_cluster_name != null && can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) - error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." - } -} - -variable "slurm_controller_instance" { - type = any - description = "Slurm cluster controller instance" -} - -variable "image" { - description = "The image for slurm daemon" - type = string - nullable = false -} - -variable "node_pool_names" { - description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." - type = list(string) - nullable = false -} - -variable "node_count_static" { - description = "The number of static nodes in node-pool" - type = number -} - -variable "subnetwork" { - description = "Primary subnetwork object" - type = any -} - -variable "slurm_namespace" { - description = "slurm namespace for charts" - type = string - default = "slurm" -} - -variable "nodeset_name" { - description = "The nodeset name" - type = string - default = "gkenodeset" -} - -variable "slurm_bucket_dir" { - description = "Path directory within `bucket_name` for Slurm cluster file storage." - type = string - nullable = false -} - -variable "slurm_bucket" { - description = "GCS Bucket of Slurm cluster file storage." - type = any - nullable = true -} - -variable "instance_templates" { - description = "The URLs of Instance Templates" - type = list(string) - nullable = false -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured on nodes." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - - validation { - condition = length(var.network_storage) == 1 && var.network_storage[0].local_mount == "/home" - error_message = "The 'network_storage' variable must contain exactly one element, and that element's 'local_mount' attribute must be \"/home\"." - } -} - -variable "filestore_id" { - description = "An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." - type = list(string) - - validation { - condition = length(var.filestore_id) == 1 - error_message = "The 'filestore_id' variable must contain exactly one element." - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/versions.tf deleted file mode 100644 index 3d7237cb92..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-nodeset/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.84" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-nodeset/v1.51.0" - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/README.md deleted file mode 100644 index 2a7c363a87..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/README.md +++ /dev/null @@ -1,39 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 4.84 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [has\_tpu](#input\_has\_tpu) | If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu. | `bool` | `false` | no | -| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | -| [partition\_name](#input\_partition\_name) | The partition name | `string` | `"gke"` | no | -| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | -| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/main.tf deleted file mode 100644 index 2949fd6594..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/main.tf +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -data "google_storage_bucket" "this" { - name = var.slurm_bucket[0].name - - depends_on = [var.slurm_bucket] -} - -### Slurm Partition -locals { - partition_conf = { - "PowerDownOnIdle" = "NO" - "SuspendTime" = "INFINITE" - "SuspendTimeout" = var.has_tpu ? 240 : 120 - "ResumeTimeout" = var.has_tpu ? 600 : 300 - } - - partition = { - partition_name = var.partition_name - partition_conf = local.partition_conf - - partition_nodeset = [var.nodeset_name] - partition_nodeset_tpu = [] - partition_nodeset_dyn = [] - # Options - enable_job_exclusive = true - power_down_on_idle = false - } -} - -resource "google_storage_bucket_object" "parition_config" { - bucket = data.google_storage_bucket.this.name - name = "${var.slurm_bucket_dir}/partition_configs/${var.partition_name}.yaml" - content = yamlencode(local.partition) -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/metadata.yaml deleted file mode 100644 index 557e1fc2ae..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/variables.tf deleted file mode 100644 index 3aeed2e59a..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/variables.tf +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "has_tpu" { - description = "If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu." - type = bool - default = false -} - -variable "nodeset_name" { - description = "The nodeset name" - type = string - default = "gkenodeset" -} - -variable "partition_name" { - description = "The partition name" - type = string - default = "gke" -} - -variable "slurm_bucket_dir" { - description = "Path directory within `bucket_name` for Slurm cluster file storage." - type = string - nullable = false -} - -variable "slurm_bucket" { - description = "GCS Bucket of Slurm cluster file storage." - type = any - nullable = true -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/versions.tf deleted file mode 100644 index aede55263c..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/gke-partition/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.84" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-partition/v1.51.0" - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/README.md deleted file mode 100644 index 4f65411ddf..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/README.md +++ /dev/null @@ -1,271 +0,0 @@ -## Description - -This module performs the following tasks: - -- create an instance template from which execute points will be created -- create a managed instance group ([MIG][mig]) for execute points -- create a Toolkit runner to configure the autoscaler to scale the MIG - -It is expected to be used with the [htcondor-install] and [htcondor-setup] -modules. - -[htcondor-install]: ../../scripts/htcondor-install/README.md -[htcondor-setup]: ../../scheduler/htcondor-setup/README.md -[mig]: https://cloud.google.com/compute/docs/instance-groups/ - -### Known limitations - -This module may be used multiple times in a blueprint to create sets of -execute points in an HTCondor pool. If used more than 1 time, the setting -[name_prefix](#input_name_prefix) must be set to a value that is unique across -all uses of the htcondor-execute-point module. If you do not follow this -constraint, you will likely receive an error while running `terraform apply` -similar to that shown below. - -```text -Error: Invalid value for variable - - on modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf line 136, in module "startup_script": - 136: runners = local.all_runners - ├──────────────── - │ var.runners is list of map of string with 5 elements - -All startup-script runners must have a unique destination. -``` - -### How to configure jobs to select execute points - -HTCondor access points provisioned by the Toolkit are specially configured to -honor an attribute named `RequireId` in each [Job ClassAd][jobad]. This value -must be set to the ID of a MIG created by an instance of this module. The -[htcondor-access-point] module includes a setting `var.default_mig_id` that will -set this value automatically to the MIG ID corresponding to the module's -execute points. If this setting is left unset each job must specify `+RequireId` -explicitly. In all cases, the default value can be overridden explicitly as shown -below: - -```text -universe = vanilla -executable = /bin/echo -arguments = "Hello, World!" -output = out.$(ClusterId).$(ProcId) -error = err.$(ClusterId).$(ProcId) -log = log.$(ClusterId).$(ProcId) -request_cpus = 1 -request_memory = 100MB -+RequireId = "htcondor-pool-ep-mig" -queue -``` - -[htcondor-access-point]: ../../scheduler/htcondor-access-point/README.md -[jobad]: https://htcondor.readthedocs.io/en/latest/users-manual/matchmaking-with-classads.html - -### Example - -A full example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -The following code snippet creates a pool with 2 sets of HTCondor execute -points, one using On-demand pricing and the other using Spot pricing. They use -a startup script and network created in previous steps. - -```yaml -- id: htcondor_execute_point - source: community/modules/compute/htcondor-execute-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - min_idle: 2 - -- id: htcondor_execute_point_spot - source: community/modules/compute/htcondor-execute-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - spot: true - -- id: htcondor_access - source: community/modules/scheduler/htcondor-access-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - - htcondor_execute_point - - htcondor_execute_point_spot - settings: - default_mig_id: $(htcondor_execute_point.mig_id) - enable_public_ips: true - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - access_point_ips - - access_point_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## Behavior of Managed Instance Group (MIG) - -Regional [MIGs][mig] are used to provision Execute Points. By default, VMs -will be provisioned in any of the zones available in that region, however, it -can be constrained to run in fewer zones (or a single zone) using -[var.zones](#input_zones). - -When the configuration of an Execute Point is changed, the MIG can be configured -to [replace the VM][replacement] using a "proactive" or "opportunistic" policy. -By default, the policy is set to opportunistic. In practice, this means that -Execute Points will _NOT_ be automatically replaced by Terraform when changes to -the instance template / HTCondor configuration are made. We recommend leaving -this at the default value as it will allow the HTCondor autoscaler to replace -VMs when they become idle without disrupting running jobs. - -However, if it is desired [var.update_policy](#input_update_policy) can be set -to "PROACTIVE" to enable automatic replacement. This will disrupt running jobs -and send them back to the queue. Alternatively, one can leave the setting at -the default value of "OPPORTUNISTIC" and update: - -- intentionally by issuing an update via Cloud Console or using gcloud (below) -- VMs becomes unhealthy or are otherwise automatically replaced (e.g. regular - Google Cloud maintenance) - -For example, to manually update all instances in a MIG: - -```text -gcloud compute instance-groups managed update-instances \ - <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - -## Known Issues - -When using OS Login with "external users" (outside of the Google Cloud -organization), then Docker universe jobs will fail and cause the Docker daemon -to crash. This stems from the use of POSIX user ids (uid) outside the range -supported by Docker. Please consider disabling OS Login if this atypical -situation applies. - -```yaml -vars: - # add setting below to existing deployment variables - enable_oslogin: DISABLE -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 4.0 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.0 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [execute\_point\_instance\_template](#module\_execute\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [mig](#module\_mig) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.execute_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.execute_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Disk type for template | `string` | `"pd-balanced"` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape across zones for instance group managing execute points | `string` | `"ANY"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [execute\_point\_runner](#input\_execute\_point\_runner) | A list of Toolkit runners for configuring an HTCondor execute point | `list(map(string))` | `[]` | no | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account for HTCondor execute point (e-mail format) | `string` | n/a | yes | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | HTCondor execute point VM image

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to HTConodr execute points | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor execute points | `string` | `"n2-standard-4"` | no | -| [max\_size](#input\_max\_size) | Maximum size of the HTCondor execute point pool. | `number` | `5` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor execute points | `map(string)` | `{}` | no | -| [min\_idle](#input\_min\_idle) | Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max\_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly. | `number` | `0` | no | -| [name\_prefix](#input\_name\_prefix) | Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network HTCondor execute points will join | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HTCondor execute points will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which HTCondor execute points will be created | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork HTCondor execute points will join | `string` | `null` | no | -| [target\_size](#input\_target\_size) | Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size. | `number` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | -| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | Startup script to run at boot-time for Windows-based HTCondor execute points | `list(string)` | `[]` | no | -| [zones](#input\_zones) | Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [autoscaler\_runner](#output\_autoscaler\_runner) | Toolkit runner to configure the HTCondor autoscaler | -| [mig\_id](#output\_mig\_id) | ID of the managed instance group containing the execute points | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml deleted file mode 100644 index 375ae036cd..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Role - hosts: localhost - become: true - vars: - spool_dir: /var/lib/condor/spool - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - handlers: - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml deleted file mode 100644 index a85158fdfc..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This playbook makes the assumption that a virtual environment has been created -# with the autoscaler and its dependencies previously installed. A runner that -# does this is provided as an output of the htcondor-install module within the -# Cluster Toolkit at community/modules/scripts/htcondor-install. - ---- -- name: Configure HTCondor Autoscaler - hosts: all - vars: - python: /usr/local/htcondor/bin/python3 - autoscaler: /usr/local/htcondor/bin/autoscaler.py - systemd_override_path: /etc/systemd/system - become: true - tasks: - - name: User must supply HTCondor role - ansible.builtin.assert: - that: - - project_id is defined - - region is defined - - zone is defined - - mig_id is defined - - max_size is defined - - name: Create SystemD service for HTCondor autoscaler - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@.service" - mode: 0644 - content: | - [Unit] - Description=HTCondor Autoscaler MIG: %i - - [Service] - User=condor - Type=oneshot - ExecStart={{ python }} {{ autoscaler }} --p $PROJECT_ID --r $REGION --z $ZONE --mz --g %i --c $MAX_SIZE --i $MIN_IDLE - notify: - - Reload SystemD - - name: Create SystemD override directory for autoscaler configuration - ansible.builtin.file: - path: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d" - state: directory - owner: root - group: root - mode: 0755 - - name: Create autoscaler configuration - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d/miglimit.conf" - mode: 0644 - content: | - [Service] - Environment=PROJECT_ID={{ project_id }} - Environment=REGION={{ region }} - Environment=ZONE={{ zone }} - Environment=MAX_SIZE={{ max_size }} - Environment=MIN_IDLE={{ min_idle }} - notify: - - Reload SystemD - - name: Create SystemD timer for HTCondor autoscaler - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@.timer" - mode: 0644 - content: | - [Unit] - Description=Run HTCondor Autoscaler Periodically - - [Timer] - OnCalendar=minutely - AccuracySec=1us - RandomizedDelaySec=30 - # the directive below is ignored harmlessly on CentOS 7; this has impact - # that timing averages to 1 minute but is not precisely 1 minute; still - # useful to ensure that timers for different MIGs do not overlap - FixedRandomDelay=true - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - post_tasks: - - name: Activate HTCondor Autoscaler timer - ansible.builtin.systemd: - name: htcondor-autoscaler@{{ mig_id }}.timer - enabled: true - state: started diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf deleted file mode 100644 index 7b0df94987..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf +++ /dev/null @@ -1,218 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-execute-point", ghpc_role = "compute" }) -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - zones = coalescelist(var.zones, data.google_compute_zones.available.names) - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - windows_startup_ps1 = join("\n\n", flatten([var.windows_startup_ps1, local.execute_config_windows_startup_ps1])) - - is_windows_image = anytrue([for l in data.google_compute_image.compute_image.licenses : length(regexall("windows-cloud", l)) > 0]) - windows_startup_metadata = local.is_windows_image && local.windows_startup_ps1 != "" ? { - windows-startup-script-ps1 = local.windows_startup_ps1 - } : {} - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.windows_startup_metadata, - local.network_storage_metadata, - local.enable_oslogin, - local.disable_automatic_updates_metadata, - var.metadata - ) - - autoscaler_runner = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_configure_autoscaler.yml") - "destination" = "htcondor_configure_autoscaler_${module.mig.instance_group_manager.name}.yml" - "args" = join(" ", [ - "-e project_id=${var.project_id}", - "-e region=${var.region}", - "-e zone=${local.zones[0]}", # this value is required, but ignored by regional MIG autoscaler - "-e mig_id=${module.mig.instance_group_manager.name}", - "-e max_size=${var.max_size}", - "-e min_idle=${var.min_idle}", - ]) - } - - execute_config = templatefile("${path.module}/templates/condor_config.tftpl", { - htcondor_role = "get_htcondor_execute", - central_manager_ips = var.central_manager_ips, - guest_accelerator = local.guest_accelerator, - }) - - execute_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.execute_config.output_name}" - execute_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e htcondor_role=get_htcondor_execute", - "-e config_object=${local.execute_object}", - ]) - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.execute_point_runner, - [local.execute_runner], - ) - - execute_config_windows_startup_ps1 = templatefile( - "${path.module}/templates/download-condor-config.ps1.tftpl", - { - config_object = local.execute_object, - } - ) - - name_prefix = "${var.deployment_name}-${var.name_prefix}-ep" -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -resource "null_resource" "execute_config" { - triggers = { - config = local.execute_config - } -} - -resource "google_storage_bucket_object" "execute_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.execute_config.id), 0, 4)}" - content = local.execute_config - bucket = var.htcondor_bucket_name -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -module "execute_point_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.execute_point_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - gpu = one(local.guest_accelerator) - preemptible = var.spot - startup_script = local.is_windows_image ? null : module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.compute_image.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config -} - -module "mig" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = var.target_size - hostname = local.name_prefix - mig_name = local.name_prefix - instance_template = module.execute_point_instance_template.self_link - - health_check_name = "health-htcondor-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "SUBSTITUTE" - max_surge_fixed = length(local.zones) - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf deleted file mode 100644 index b31f40130f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "autoscaler_runner" { - value = local.autoscaler_runner - description = "Toolkit runner to configure the HTCondor autoscaler" -} - -output "mig_id" { - value = module.mig.instance_group_manager.name - description = "ID of the managed instance group containing the execute points" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl deleted file mode 100644 index c8f5ce31a8..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:${htcondor_role} -CONDOR_HOST = ${join(",", central_manager_ips)} - -# StartD configuration settings -%{ if length(guest_accelerator) > 0 ~} -use feature:GPUs -%{ endif ~} -use feature:PartitionableSlot -use feature:CommonCloudAttributesGoogle("-c created-by") -UPDATE_INTERVAL = 30 -TRUST_UID_DOMAIN = True -STARTER_ALLOW_RUNAS_OWNER = True -RUNBENCHMARKS = False diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl deleted file mode 100644 index 19789f122e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl +++ /dev/null @@ -1,34 +0,0 @@ -# create directory for local condor_config customizations -$config_dir = 'C:\Condor\config' -if(!(test-path -PathType container -Path $config_dir)) -{ - New-Item -ItemType Directory -Path $config_dir -} - -# update local condor_config if blueprint has changed -$config_file = "$config_dir\50-ghpc-managed" -if (Test-Path -Path $config_file -PathType Leaf) -{ - $local_hash = gcloud --format="value(md5_hash)" storage hash $config_file -} -else -{ - $local_hash = "INVALID-HASH" -} - -$remote_hash = gcloud --format="value(md5_hash)" storage hash ${config_object} -if ($local_hash -cne $remote_hash) -{ - Write-Output "Updating condor configuration" - gcloud storage cp ${config_object} $config_file - if ($LASTEXITCODE -ne 0) - { - throw "Could not download HTCondor configuration; exiting startup script" - } - Restart-Service condor -} - -# ignored if service is already running; must be here to handle case where -# machine is rebooted, but configuration has previously been downloaded -# and service is disabled from automatic start -Start-Service condor diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf deleted file mode 100644 index aab8a54c2d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf +++ /dev/null @@ -1,265 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HTCondor execute points will be created" - type = string -} - -variable "region" { - description = "The region in which HTCondor execute points will be created" - type = string -} - -variable "zones" { - description = "Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region." - type = list(string) - default = [] - nullable = false -} - -variable "distribution_policy_target_shape" { - description = "Target shape across zones for instance group managing execute points" - type = string - default = "ANY" -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to HTConodr execute points" - type = map(string) -} - -variable "machine_type" { - description = "Machine type to use for HTCondor execute points" - type = string - default = "n2-standard-4" -} - -variable "execute_point_runner" { - description = "A list of Toolkit runners for configuring an HTCondor execute point" - type = list(map(string)) - default = [] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "instance_image" { - description = <<-EOD - HTCondor execute point VM image - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "execute_point_service_account_email" { - description = "Service account for HTCondor execute point (e-mail format)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_self_link" { - description = "The self link of the network HTCondor execute points will join" - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork HTCondor execute points will join" - type = string - default = null -} - -variable "target_size" { - description = "Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size." - type = number - default = null -} - -variable "max_size" { - description = "Maximum size of the HTCondor execute point pool." - type = number - default = 5 -} - -variable "min_idle" { - description = "Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly." - type = number - default = 0 -} - -variable "metadata" { - description = "Metadata to add to HTCondor execute points" - type = map(string) - default = {} -} - -# this default is deliberately the opposite of vm-instance because of observed -# issues running HTCondor docker universe jobs with OS Login enabled and running -# jobs as a user with uid>2^31; these uids occur when users outside the GCP -# organization login to a VM and OS Login is enabled. -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "spot" { - description = "Provision VMs using discounted Spot pricing, allowing for preemption" - type = bool - default = false -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 100 -} - -variable "disk_type" { - description = "Disk type for template" - type = string - default = "pd-balanced" -} - -variable "windows_startup_ps1" { - description = "Startup script to run at boot-time for Windows-based HTCondor execute points" - type = list(string) - default = [] - nullable = false -} - -variable "central_manager_ips" { - description = "List of IP addresses of HTCondor Central Managers" - type = list(string) -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The HTCondor module supports 0 or 1 models of accelerator card on each execute point" - } -} - -variable "name_prefix" { - description = "Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module" - type = string - nullable = false - validation { - condition = length(var.name_prefix) > 0 - error_message = "var.name_prefix must be a set to a non-empty string and must also be unique across all instances of htcondor-execute-point" - } -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" - type = string - default = "OPPORTUNISTIC" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf deleted file mode 100644 index 729dc3cda5..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 1.1" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.74.0" - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/README.md deleted file mode 100644 index 278207b04a..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/README.md +++ /dev/null @@ -1,45 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_group_manager.mig](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_group_manager) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [base\_instance\_name](#input\_base\_instance\_name) | Base name for the instances in the MIG | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will be used to name MIG if `var.name` is not provided | `string` | n/a | yes | -| [ghpc\_module\_id](#input\_ghpc\_module\_id) | Internal GHPC field, do not set this value | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the MIG | `map(string)` | n/a | yes | -| [name](#input\_name) | Name of the MIG. If not provided, will be generated from `var.deployment_name` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the MIG will be created | `string` | n/a | yes | -| [target\_size](#input\_target\_size) | Target number of instances in the MIG | `number` | `0` | no | -| [versions](#input\_versions) | Application versions managed by this instance group. Each version deals with a specific instance template | 
list(object({
    name              = string
    instance_template = string
    target_size = optional(object({
      fixed   = optional(number)
      percent = optional(number)
    }))
  }))
| n/a | yes | -| [wait\_for\_instances](#input\_wait\_for\_instances) | Whether to wait for all instances to be created/updated before returning | `bool` | `false` | no | -| [zone](#input\_zone) | Compute Platform zone. Required, currently only zonal MIGs are supported | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [self\_link](#output\_self\_link) | The URL of the created MIG | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/main.tf deleted file mode 100644 index 0e7cf186c2..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/main.tf +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "mig", ghpc_role = "compute" }) -} - -locals { - sanitized_deploy_name = try(replace(lower(var.deployment_name), "/[^a-z0-9]/", ""), null) - sanitized_module_id = try(replace(lower(var.ghpc_module_id), "/[^a-z0-9]/", ""), null) - synth_mig_name = try("${local.sanitized_deploy_name}-${local.sanitized_module_id}", null) - - mig_name = var.name == null ? local.synth_mig_name : var.name - base_instance_name = var.base_instance_name == null ? local.mig_name : var.base_instance_name -} - -resource "google_compute_instance_group_manager" "mig" { - # REQUIRED - name = local.mig_name - base_instance_name = local.base_instance_name - zone = var.zone - - dynamic "version" { - for_each = var.versions - content { - name = version.value.name - instance_template = version.value.instance_template - dynamic "target_size" { - for_each = version.value.target_size != null ? [version.value.target_size] : [] - content { - fixed = target_size.value.fixed - percent = target_size.value.percent - } - } - } - } - - # OPTIONAL - project = var.project_id - target_size = var.target_size - wait_for_instances = var.wait_for_instances - - all_instances_config { - # TODO: validate that template metadata not getting wiped out - # TODO: validate that template labels not getting wiped out - labels = local.labels - } - - # OMITTED: - # * description - # * named_port - # * list_managed_instances_results - # * target_pools - specific for Load Balancers usage - # * wait_for_instances_status - # * auto_healing_policies - # * stateful_disk - # * stateful_internal_ip - # * update_policy - # * params - - - lifecycle { - precondition { - condition = local.mig_name != null - error_message = "Could not come up with a name for the MIG, specify `var.name`" - } - - precondition { - condition = local.base_instance_name != null - error_message = "Could not come up with a base_instance_name, specify `var.base_instance_name`" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/metadata.yaml deleted file mode 100644 index 97a4fa9a89..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com -ghpc: - inject_module_id: ghpc_module_id diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/outputs.tf deleted file mode 100644 index 23c66a3535..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "self_link" { - description = "The URL of the created MIG" - value = google_compute_instance_group_manager.mig.self_link -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/variables.tf deleted file mode 100644 index b6c3c0e78a..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/variables.tf +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "Project in which the MIG will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the deployment, will be used to name MIG if `var.name` is not provided" - type = string -} - -variable "labels" { - description = "Labels to add to the MIG" - type = map(string) -} - -variable "zone" { - description = "Compute Platform zone. Required, currently only zonal MIGs are supported" - type = string -} - - -variable "versions" { - description = <<-EOD - Application versions managed by this instance group. Each version deals with a specific instance template - EOD - type = list(object({ - name = string - instance_template = string - target_size = optional(object({ - fixed = optional(number) - percent = optional(number) - })) - })) - - validation { - condition = length(var.versions) > 0 - error_message = "At least one version must be provided" - } - -} - - -variable "ghpc_module_id" { - description = "Internal GHPC field, do not set this value" - type = string - default = null -} - -variable "name" { - description = "Name of the MIG. If not provided, will be generated from `var.deployment_name`" - type = string - default = null -} - -variable "base_instance_name" { - description = "Base name for the instances in the MIG" - type = string - default = null -} - - -variable "target_size" { - description = "Target number of instances in the MIG" - type = number - default = 0 -} - -variable "wait_for_instances" { - description = "Whether to wait for all instances to be created/updated before returning" - type = bool - default = false -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/mig/versions.tf deleted file mode 100644 index 4147447b44..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/mig/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - google = { - source = "hashicorp/google" - version = "> 5.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:mig/v1.74.0" - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/README.md deleted file mode 100644 index 1dcacc57e9..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# Description - -This module creates the Vertex AI Notebook, to be used in tutorials. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage - -This is a simple usage, using the default network: - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: - name_prefix: my-bucket - local_mount: /home/jupyter/my-bucket - - - id: notebook - source: community/modules/compute/notebook - use: [bucket] - settings: - name_prefix: notebook - machine_type: n1-standard-4 - -``` - -If the user wants do specify a custom subnetwork, or specific external IP restrictions, they can use the `network_interfaces` variable, here is an example on how to use a Shared VPC Subnet with an ephemeral external IP: - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: - name_prefix: my-bucket - local_mount: /home/jupyter/my-bucket - - - id: notebook - source: community/modules/compute/notebook - use: [bucket] - settings: - name_prefix: notebook - machine_type: n1-standard-4 - network_interfaces: - - network: "projects/HOST_PROJECT_ID/global/networks/SHARED_VPC_NAME" - subnet: "projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME" - nic_type: "VIRTIO_NET" -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 5.34 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 5.34 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.mount_script](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_workbench_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/workbench_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the notebook. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name, can be provided from the google-cloud-storage module | `string` | `null` | no | -| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "tf-latest-cpu",
  "name": null,
  "project": "deeplearning-platform-release"
}
| no | -| [labels](#input\_labels) | Labels to add to the resource Key-value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | The machine type to employ | `string` | n/a | yes | -| [mount\_runner](#input\_mount\_runner) | mount content from the google-cloud-storage module | `map(string)` | n/a | yes | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces for the VM instance. Each network interface is represented by an object with the following fields:

- network: (Optional) The name of the Virtual Private Cloud (VPC) network that this VM instance is connected to.

- subnet: (Optional) The name of the subnetwork within the specified VPC that this VM instance is connected to.

- nic\_type: (Optional) The type of vNIC to be used on this interface. Possible values are: `VIRTIO_NET`, `GVNIC`.

- access\_configs: (Optional) An array of access configurations for this network interface. The access\_config object contains:
* external\_ip: (Required) An external IP address associated with this instance. Specify an unused static external IP address available to the project or leave this field undefined to use an IP from a shared ephemeral IP address pool. If you specify a static external IP address, it must live in the same region as the zone of the instance. | 
list(object({
    network  = optional(string)
    subnet   = optional(string)
    nic_type = optional(string)
    access_configs = optional(list(object({
      external_ip = optional(string)
    })))
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | ID of project in which the notebook will be created. | `string` | n/a | yes | -| [service\_account\_email](#input\_service\_account\_email) | If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account | `string` | `null` | no | -| [zone](#input\_zone) | The zone to deploy to | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/main.tf deleted file mode 100644 index cd3ce3b4ea..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/main.tf +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "notebook", ghpc_role = "compute" }) -} - -locals { - suffix = random_id.resource_name_suffix.hex - #name = "thenotebook" - name = "notebook-${var.deployment_name}-${local.suffix}" - bucket = replace(var.gcs_bucket_path, "gs://", "") - post_script_filename = "mount-${local.suffix}.sh" - - # mount_runner_args is defined in the file: cluster-toolkit/modules/file-system/cloud-storage-bucket/outputs.tf - mount_args = split(" ", var.mount_runner.args) - - unused = local.mount_args[0] - remote_mount = local.mount_args[1] - local_mount = local.mount_args[2] - fs_type = local.mount_args[3] - # These options provide a "rw" mount of the GCS bucket - mount_options = "defaults,_netdev,allow_other,implicit_dirs,gid=1000,uid=1000" - - content0 = var.mount_runner.content - content1 = replace(local.content0, "$1", local.unused) - content2 = replace(local.content1, "$2", local.remote_mount) - content3 = replace(local.content2, "$3", local.local_mount) - content4 = replace(local.content3, "$4", local.fs_type) - content5 = replace(local.content4, "$5", local.mount_options) - -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket_object" "mount_script" { - name = local.post_script_filename - content = local.content5 - bucket = local.bucket -} - -resource "google_workbench_instance" "instance" { - name = local.name - location = var.zone - project = var.project_id - labels = local.labels - gce_setup { - machine_type = var.machine_type - metadata = { - post-startup-script = "${var.gcs_bucket_path}/${google_storage_bucket_object.mount_script.name}" - } - vm_image { - project = var.instance_image.project - family = var.instance_image.family - } - - dynamic "service_accounts" { - for_each = var.service_account_email == null ? [] : [1] - content { - email = var.service_account_email - } - } - - dynamic "network_interfaces" { - for_each = var.network_interfaces - content { - network = network_interfaces.value.network - subnet = network_interfaces.value.subnet - nic_type = network_interfaces.value.nic_type - - dynamic "access_configs" { - for_each = network_interfaces.value.access_configs != null ? network_interfaces.value.access_configs : [] - content { - external_ip = access_configs.value.external_ip - } - } - } - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/metadata.yaml deleted file mode 100644 index 4a7d5397ca..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - notebooks.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/variables.tf deleted file mode 100644 index 4359de8c10..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/notebook/variables.tf +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which the notebook will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment; used as part of name of the notebook." - type = string - # notebook name can have: lowercase letters, numbers, or hyphens (-) and cannot end with a hyphen - validation { - error_message = "The notebook name uses 'deployment_name' -- can only have: lowercase letters, numbers, or hyphens" - condition = can(regex("^[a-z0-9]+(?:-[a-z0-9]+)*$", var.deployment_name)) - } -} - -variable "zone" { - description = "The zone to deploy to" - type = string -} - -variable "machine_type" { - description = "The machine type to employ" - type = string -} - -variable "labels" { - description = "Labels to add to the resource Key-value pairs." - type = map(string) -} - -variable "instance_image" { - description = "Instance Image" - type = map(string) - default = { - project = "deeplearning-platform-release" - family = "tf-latest-cpu" - name = null - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "gcs_bucket_path" { - description = "Bucket name, can be provided from the google-cloud-storage module" - type = string - default = null -} - -variable "mount_runner" { - description = "mount content from the google-cloud-storage module" - type = map(string) - - validation { - condition = (length(split(" ", var.mount_runner.args)) == 5) - error_message = "There must be 5 elements in the Mount Runner Arguments: ${var.mount_runner.args} \n " - } -} - -variable "service_account_email" { - description = "If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account" - type = string - default = null -} - -variable "network_interfaces" { - type = list(object({ - network = optional(string) - subnet = optional(string) - nic_type = optional(string) - access_configs = optional(list(object({ - external_ip = optional(string) - }))) - })) - default = [] - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | -| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name    = string
    device_name  = string
    disk_size_gb = number
    disk_type    = string
    disk_labels  = map(string)
    auto_delete  = bool
    boot         = bool
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | -| [feature](#input\_feature) | The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used. | `string` | `null` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | Path to the Slurm bucket. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster. | `string` | n/a | yes | -| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_template\_self\_link](#output\_instance\_template\_self\_link) | The URI of the template. | -| [node\_name\_prefix](#output\_node\_name\_prefix) | The prefix to be used for the node names.

Make sure that nodes are named `-`
This temporary required for proper functioning of the nodes.
While Slurm scheduler uses "features" to bind node and nodeset,
the SlurmGCP relies on node names for this (to be switched to features as well). | -| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf deleted file mode 100644 index 31d9f14ae7..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset-dynamic", ghpc_role = "compute" }) -} - -module "instance_validation" { - source = "../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - nodeset_name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - feature = coalesce(var.feature, local.nodeset_name) - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - universe_domain = { "universe_domain" = var.universe_domain } - - metadata = merge( - local.disable_automatic_updates_metadata, - local.universe_domain, - { slurmd_feature = local.feature }, - var.metadata - ) - - nodeset = { - nodeset_name = local.nodeset_name - nodeset_feature : local.feature - startup_script = local.ghpc_startup_script - network_storage = var.network_storage - } - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - } - ] - - public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - ghpc_startup_script = [{ - filename = "ghpc_nodeset_startup.sh" - content = var.startup_script - }] - -} - -module "slurm_nodeset_template" { - source = "../../internal/slurm-gcp/instance_template" - - project_id = var.project_id - region = var.region - name_prefix = local.nodeset_name - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = "compute" - slurm_bucket_path = var.slurm_bucket_path - metadata = local.metadata - - additional_disks = local.additional_disks - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(local.labels, var.disk_labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - - bandwidth_tier = var.bandwidth_tier - can_ip_forward = var.can_ip_forward - - advanced_machine_features = var.advanced_machine_features - enable_confidential_vm = var.enable_confidential_vm - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - labels = local.labels - machine_type = var.machine_type - - min_cpu_platform = var.min_cpu_platform - on_host_maintenance = var.on_host_maintenance - termination_action = try(var.spot_instance_config.termination_action, null) - preemptible = var.preemptible - spot = var.enable_spot_vm - service_account = local.service_account - gpu = one(local.guest_accelerator) # requires gpu_definition.tf - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - subnetwork = var.subnetwork_self_link - additional_networks = var.additional_networks - access_config = local.access_config - tags = concat([var.slurm_cluster_name], var.tags) -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml deleted file mode 100644 index a99e59d09f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [compute.googleapis.com] -ghpc: - inject_module_id: name diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf deleted file mode 100644 index 2d2d1415cf..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset_dyn" { - description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset -} - -output "instance_template_self_link" { - description = "The URI of the template." - value = module.slurm_nodeset_template.self_link -} - -output "node_name_prefix" { - description = <<-EOD - The prefix to be used for the node names. - - Make sure that nodes are named `-` - This temporary required for proper functioning of the nodes. - While Slurm scheduler uses "features" to bind node and nodeset, - the SlurmGCP relies on node names for this (to be switched to features as well). - EOD - value = "${var.slurm_cluster_name}-${local.nodeset_name}" - -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf deleted file mode 100644 index ec6206e317..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf +++ /dev/null @@ -1,402 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "feature" { - type = string - description = "The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used." - default = null -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "slurm_cluster_name" { - description = "Name of the Slurm cluster." - type = string -} - -variable "slurm_bucket_path" { - description = "Path to the Slurm bucket." - type = string -} - - -variable "machine_type" { - description = "Compute Platform machine type to use for this partition compute nodes." - type = string - default = "c2-standard-60" -} - -variable "metadata" { - type = map(string) - description = "Metadata, provided as a map." - default = {} -} - -variable "instance_image" { - description = <<-EOD - Defines the image that will be used in the Slurm node group VM instances. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - - For more information on creating custom images that comply with Slurm on GCP - see the "Slurm on GCP Custom Images" section in docs/vm-images.md. - EOD - type = map(string) - default = { - family = "slurm-gcp-6-11-hpc-rocky-linux-8" - project = "schedmd-slurm-public" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations - description = <<-EOD - A flag that designates that the user is aware that they are requesting - to use a custom and potentially incompatible image for this Slurm on - GCP module. - - If the field is set to false, only the compatible families and project - names will be accepted. The deployment will fail with any other image - family or name. If set to true, no checks will be done. - - See: https://goo.gle/hpc-slurm-images - EOD - type = bool - default = false -} - - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "tags" { - type = list(string) - description = "Network tag list." - default = [] -} - -variable "disk_type" { - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - type = string - default = "pd-standard" -} - -variable "disk_size_gb" { - description = "Size of boot disk to create for the partition compute nodes." - type = number - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "additional_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = list(object({ - disk_name = string - device_name = string - disk_size_gb = number - disk_type = string - disk_labels = map(string) - auto_delete = bool - boot = bool - })) - default = [] -} - -variable "enable_confidential_vm" { - type = bool - description = "Enable the Confidential VM configuration. Note: the instance image must support option." - default = false -} - -variable "enable_shielded_vm" { - type = bool - description = "Enable the Shielded VM configuration. Note: the instance image must support option." - default = false -} - -variable "shielded_instance_config" { - type = object({ - enable_integrity_monitoring = bool - enable_secure_boot = bool - enable_vtpm = bool - }) - description = <<-EOD - Shielded VM configuration for the instance. Note: not used unless - enable_shielded_vm is 'true'. - - enable_integrity_monitoring : Compare the most recent boot measurements to the - integrity policy baseline and return a pair of pass/fail results depending on - whether they match or not. - - enable_secure_boot : Verify the digital signature of all boot components, and - halt the boot process if signature verification fails. - - enable_vtpm : Use a virtualized trusted platform module, which is a - specialized computer chip you can use to encrypt objects like keys and - certificates. - EOD - default = { - enable_integrity_monitoring = true - enable_secure_boot = true - enable_vtpm = true - } -} - - -variable "enable_oslogin" { - type = bool - description = <<-EOD - Enables Google Cloud os-login for user login and authentication for VMs. - See https://cloud.google.com/compute/docs/oslogin - EOD - default = true -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example." - type = bool - default = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "labels" { - description = "Labels to add to partition compute instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "min_cpu_platform" { - description = "The name of the minimum CPU platform that you want the instance to use." - type = string - default = null -} - -variable "on_host_maintenance" { - type = string - description = <<-EOD - Instance availability Policy. - - Note: Placement groups are not supported when on_host_maintenance is set to - "MIGRATE" and will be deactivated regardless of the value of - enable_placement. To support enable_placement, ensure on_host_maintenance is - set to "TERMINATE". - EOD - default = "TERMINATE" -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." - } -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - - -variable "service_account_email" { - description = "Service account e-mail address to attach to the compute instances." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the compute instances." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "enable_spot_vm" { - description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." - type = bool - default = false -} - -variable "spot_instance_config" { - description = "Configuration for spot VMs." - type = object({ - termination_action = string - }) - default = null -} - -variable "bandwidth_tier" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | -| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | -| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | -| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `null` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling worker nodes allowed in this partition.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of worker nodes to be statically created.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | -| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. | `string` | `""` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the TPU-vm. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the TPU-vm. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | -| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | `"2.14.0"` | no | -| [zone](#input\_zone) | Zone in which to create compute VMs. TPU partitions can only specify a single zone. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf deleted file mode 100644 index ac9b119702..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# locals { -# # This label allows for billing report tracking based on module. -# labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) -# } - -locals { - name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - nodeset_tpu = { - node_count_static = var.node_count_static - node_count_dynamic_max = var.node_count_dynamic_max - nodeset_name = local.name - node_type = var.node_type - - accelerator_config = var.accelerator_config - tf_version = var.tf_version - preemptible = var.preemptible - preserve_tpu = var.preserve_tpu - - data_disks = var.data_disks - docker_image = var.docker_image - - enable_public_ip = var.enable_public_ips - # TODO: rename to subnetwork_self_link, requires changes to the scripts - subnetwork = var.subnetwork_self_link - service_account = local.service_account - zone = var.zone - - project_id = var.project_id - reserved = var.reserved - network_storage = var.network_storage - } - - node_type_core_count = var.node_type == "" ? 0 : tonumber(regex("-(.*)", var.node_type)[0]) - - accelerator_core_list = var.accelerator_config.topology == "" ? [0, 0] : regexall("\\d+", var.accelerator_config.topology) - accelerator_core_count = length(local.accelerator_core_list) > 2 ? (local.accelerator_core_list[0] * local.accelerator_core_list[1] * local.accelerator_core_list[2]) * 2 : (local.accelerator_core_list[0] * local.accelerator_core_list[1]) * 2 - - tpu_core_count = local.accelerator_core_count == 0 ? local.node_type_core_count : local.accelerator_core_count -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml deleted file mode 100644 index 95b6d1c730..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name - has_to_be_used: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf deleted file mode 100644 index 8cb7b8663e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset_tpu" { - description = "Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset_tpu - - precondition { - condition = (var.node_type == "") != (var.accelerator_config == { topology : "", version : "" }) - error_message = "Either a node_type or an accelerator_config must be provided." - } - - precondition { - condition = ((local.tpu_core_count / 8) <= var.node_count_dynamic_max) || ((local.tpu_core_count / 8) <= var.node_count_static) - error_message = <<-EOD - When using TPUs there should be at least one node per every 8 cores. - Currently there are ${local.tpu_core_count} cores but only ${var.node_count_static} static nodes and ${var.node_count_dynamic_max} dynamic nodes. - EOD - } - - precondition { - condition = (var.node_count_dynamic_max % (local.tpu_core_count / 8) == 0) && (var.node_count_static % (local.tpu_core_count / 8) == 0) - error_message = <<-EOD - The number of worker nodes should be a multiple of ${local.tpu_core_count / 8}. - This is to ensure each node has a TPU machine for job scheduling. - EOD - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf deleted file mode 100644 index 367b0bee09..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "node_count_static" { - description = <<-EOD - Number of worker nodes to be statically created. - For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). - See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. - EOD - type = number - default = 0 -} - -variable "node_count_dynamic_max" { - description = <<-EOD - Maximum number of auto-scaling worker nodes allowed in this partition. - For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). - See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. - EOD - type = number - default = 0 -} - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "enable_public_ips" { - description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." - type = bool - default = false -} - -variable "disable_public_ips" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `enable_public_ips` instead." - type = bool - default = null - validation { - condition = var.disable_public_ips == null - error_message = "DEPRECATED: Use `enable_public_ips` instead." - } -} - -variable "node_type" { - description = "Specify a node type to base the vm configuration upon it." - type = string - default = "" -} - -variable "accelerator_config" { - description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." - type = object({ - topology = string - version = string - }) - default = { - topology = "" - version = "" - } - validation { - condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], var.accelerator_config.version) - error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" - } - validation { - condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) - error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." - } -} - -variable "tf_version" { - description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." - type = string - default = "2.14.0" -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - -variable "preserve_tpu" { - description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" - type = bool - default = false -} - -variable "zone" { - description = "Zone in which to create compute VMs. TPU partitions can only specify a single zone." - type = string -} - -variable "data_disks" { - description = "The data disks to include in the TPU node" - type = list(string) - default = [] -} - -variable "docker_image" { - description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" - type = string - default = null -} - -variable "subnetwork_self_link" { - type = string - description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." -} - -variable "service_account_email" { - description = "Service account e-mail address to attach to the TPU-vm." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the TPU-vm." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "service_account" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - } -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "reserved" { - description = "Specify whether TPU-vms in this nodeset are created under a reservation." - type = bool - default = false -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured on nodes." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - })) - default = [] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf deleted file mode 100644 index 398eeffdda..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.3" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.74.0" - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md deleted file mode 100644 index 7c9e32debf..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md +++ /dev/null @@ -1,227 +0,0 @@ -## Description - -This module creates a nodeset data structure intended to be input to the -[schedmd-slurm-gcp-v6-partition](../schedmd-slurm-gcp-v6-partition/) module. - -Nodesets allow adding heterogeneous node types to a partition, and hence -running jobs that mix multiple node characteristics. See the [heterogeneous jobs -section][hetjobs] of the SchedMD documentation for more information. - -To specify nodes from a specific nodesets in a partition, the [`--nodelist`] -(or `-w`) flag can be used, for example: - -```bash -srun -N 3 -p compute --nodelist cluster-compute-group-[0-2] hostname -``` - -Where the 3 nodes will be selected from the nodes `cluster-compute-group-[0-2]` -in the compute partition. - -Additionally, depending on how the nodes differ, a constraint can be added via -the [`--constraint`] (or `-C`) flag or other flags such as `--mincpus` can be -used to specify nodes with the desired characteristics. - -[`--nodelist`]: https://slurm.schedmd.com/srun.html#OPT_nodelist -[`--constraint`]: https://slurm.schedmd.com/srun.html#OPT_constraint -[hetjobs]: https://slurm.schedmd.com/heterogeneous_jobs.html - -### Example - -The following code snippet creates a partition module using the `nodeset` -module as input with: - -* a max node count of 200 -* VM machine type of `c2-standard-30` -* partition name of "compute" -* default nodeset name of "ghpc" -* connected to the `network` module via `use` -* nodes mounted to homefs via `use` - -```yaml -- id: nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - node_count_dynamic_max: 200 - machine_type: c2-standard-30 - -- id: compute_partition - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - use: - - homefs - - nodeset - settings: - partition_name: compute -``` - -## Custom Images - -For more information on creating valid custom images for the node group VM -instances or for custom instance templates, see our [vm-images.md] documentation -page. - -[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images - -## GPU Support - -More information on GPU support in Slurm on GCP and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) - -### Compute VM Zone Policies - -The Slurm on GCP nodeset module allows you to specify additional zones in -which to create VMs through [bulk creation][bulk]. This is valuable when -configuring partitions with popular VM families and you desire access to -more compute resources across zones. - -[bulk]: https://cloud.google.com/compute/docs/instances/multiple/about-bulk-creation -[networkpricing]: https://cloud.google.com/vpc/network-pricing - -> **_WARNING:_** Lenient zone policies can lead to additional egress costs when -> moving large amounts of data between zones in the same region. For example, -> traffic between VMs and traffic from VMs to shared filesystems such as -> Filestore. For more information on egress fees, see the -> [Network Pricing][networkpricing] Google Cloud documentation. -> -> To avoid egress charges, ensure your compute nodes are created in a single -> zone by setting var.zone and leaving var.zones to its default value of the -> empty list. -> -> **_NOTE:_** If a new zone is added to the region while the cluster is active, -> nodes in the partition may be created in that zone. In this case, the -> partition may need to be redeployed to ensure the newly added zone is denied. - -In the zonal example below, the nodeset's zone implicitly defaults to the -deployment variable `vars.zone`: - -```yaml -vars: - zone: us-central1-f - -- id: zonal-nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset -``` - -In the example below, we enable creation in additional zones: - -```yaml -vars: - zone: us-central1-f - -- id: multi-zonal-nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - settings: - zones: - - us-central1-a - - us-central1-b -``` - -## Support -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4 | -| [google](#requirement\_google) | >= 5.11 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 5.11 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.machine_type_zone_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [google_compute_machine_types.machine_types_by_zone](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | -| [google_compute_reservation.reservation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_topology](#input\_accelerator\_topology) | Specifies the shape of the Accelerator (GPU/TPU) slice. | `string` | `null` | no | -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = optional(string)
    subnetwork         = string
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    stack_type         = optional(string)
    queue_count        = optional(number)
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | -| [dws\_flex](#input\_dws\_flex) | If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes.
See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler
Options:
- enable: Enable DWS Flex Start
- max\_run\_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week).
- use\_job\_duration: Use the job duration to determine the max\_run\_duration, if job duration is not set, max\_run\_duration will be used.
- use\_bulk\_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances

Limitations:
- CAN NOT be used with reservations;
- CAN NOT be used with placement groups; | 
object({
    enabled          = optional(bool, true)
    max_run_duration = optional(number, 604800) # one week
    use_job_duration = optional(bool, false)
    use_bulk_insert  = optional(bool, false)
  })
| 
{
  "enabled": false
}
| no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_maintenance\_reservation](#input\_enable\_maintenance\_reservation) | Enables slurm reservation for scheduled maintenance. | `bool` | `false` | no | -| [enable\_opportunistic\_maintenance](#input\_enable\_opportunistic\_maintenance) | On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_placement](#input\_enable\_placement) | Use placement policy for VMs in this nodeset.
See: https://cloud.google.com/compute/docs/instances/placement-policies-overview
To set max\_distance of used policy, use `placement_max_distance` variable.

Enabled by default, reasons for users to disable it:
- If non-dense reservation is used, user can avoid extra-cost of creating placement policies;
- If user wants to avoid "all or nothing" VM provisioning behaviour;
- If user wants to intentionally have "spread" VMs (e.g. for reliability reasons) | `bool` | `true` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | -| [future\_reservation](#input\_future\_reservation) | If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT\_ID/zones/ZONE/futureReservations/FUTURE\_RESERVATION\_NAME'.
See https://cloud.google.com/compute/docs/instances/future-reservations-overview | `string` | `""` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_properties](#input\_instance\_properties) | Override the instance properties. Used to test features not supported by Slurm GCP,
recommended for advanced usage only.
See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert
If any sub-field (e.g. scheduling) is set, it will override the values computed by
SlurmGCP and ignoring values of provided vars. | `any` | `null` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for compute nodes. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | -| [maintenance\_interval](#input\_maintenance\_interval) | Sets the maintenance interval for instances in this nodeset.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. | `string` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_conf](#input\_node\_conf) | Map of Slurm node line configuration. | `map(any)` | `{}` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling nodes allowed in this partition. | `number` | `10` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | -| [placement\_max\_distance](#input\_placement\_max\_distance) | Maximum distance between nodes in the placement group. Requires enable\_placement to be true. Values must be supported by the chosen machine type. | `number` | `null` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | -| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]
- RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]

Must be a "SPECIFIC" reservation
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [zone](#input\_zone) | Zone in which to create compute VMs. Additional zones in the same region can be specified in var.zones. | `string` | n/a | yes | -| [zone\_target\_shape](#input\_zone\_target\_shape) | Strategy for distributing VMs across zones in a region.
ANY
GCE picks zones for creating VM instances to fulfill the requested number of VMs
within present resource constraints and to maximize utilization of unused zonal
reservations.
ANY\_SINGLE\_ZONE (default)
GCE always selects a single zone for all the VMs, optimizing for resource quotas,
available reservations and general capacity.
BALANCED
GCE prioritizes acquisition of resources, scheduling VMs in zones where resources
are available while distributing VMs as evenly as possible across allowed zones
to minimize the impact of zonal failure. | `string` | `"ANY_SINGLE_ZONE"` | no | -| [zones](#input\_zones) | Additional zones in which to allow creation of partition nodes. Google Cloud
will find zone based on availability, quota and reservations.
Should not be set if SPECIFIC reservation is used. | `set(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf deleted file mode 100644 index da6aae33ee..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) -} - -module "instance_validation" { - source = "../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata - ) - - name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.disk_resource_manager_tags - } - ] - - public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - ghpc_startup_script = [{ - filename = "ghpc_nodeset_startup.sh" - content = var.startup_script - }] - - termination_action = (var.dws_flex.enabled && !var.dws_flex.use_bulk_insert) ? "DELETE" : try(var.spot_instance_config.termination_action, null) - - nodeset = { - node_count_static = var.node_count_static - node_count_dynamic_max = var.node_count_dynamic_max - node_conf = var.node_conf - nodeset_name = local.name - dws_flex = var.dws_flex - - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(local.labels, var.disk_labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - - bandwidth_tier = var.bandwidth_tier - can_ip_forward = var.can_ip_forward - - enable_confidential_vm = var.enable_confidential_vm - enable_placement = var.enable_placement - placement_max_distance = var.placement_max_distance - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - gpu = one(local.guest_accelerator) - accelerator_topology = var.accelerator_topology - - labels = local.labels - machine_type = terraform_data.machine_type_zone_validation.output - advanced_machine_features = var.advanced_machine_features - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - region = var.region - resource_manager_tags = var.resource_manager_tags - service_account = local.service_account - shielded_instance_config = var.shielded_instance_config - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - subnetwork_self_link = var.subnetwork_self_link - additional_networks = var.additional_networks - access_config = local.access_config - tags = var.tags - spot = var.enable_spot_vm - termination_action = local.termination_action - reservation_name = local.reservation_name - future_reservation = local.future_reservation - maintenance_interval = var.maintenance_interval - instance_properties_json = jsonencode(var.instance_properties) - - zone_target_shape = var.zone_target_shape - zone_policy_allow = local.zones - zone_policy_deny = local.zones_deny - - startup_script = local.ghpc_startup_script - network_storage = var.network_storage - - enable_maintenance_reservation = var.enable_maintenance_reservation - enable_opportunistic_maintenance = var.enable_opportunistic_maintenance - } -} - -locals { - zones = setunion(var.zones, [var.zone]) - zones_deny = setsubtract(data.google_compute_zones.available.names, local.zones) -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region - - lifecycle { - postcondition { - condition = length(setsubtract(local.zones, self.names)) == 0 - error_message = <<-EOD - Invalid zones=${jsonencode(setsubtract(local.zones, self.names))} - Available zones=${jsonencode(self.names)} - EOD - } - } -} - -locals { - res_match = regex("^(?P(?Pprojects/(?P[a-z0-9-]+)/reservations/)?(?P[a-z0-9-]+)(?P/reservationBlocks/[a-z0-9-]+)?)?$", var.reservation_name) - - res_short_name = local.res_match.name - res_project = coalesce(local.res_match.project, var.project_id) - res_prefix = coalesce(local.res_match.prefix, "projects/${local.res_project}/reservations/") - res_suffix = local.res_match.suffix == null ? "" : local.res_match.suffix - - reservation_name = local.res_match.whole == null ? "" : "${local.res_prefix}${local.res_short_name}${local.res_suffix}" -} - -locals { - fr_match = regex("^(?Pprojects/(?P[a-z0-9-]+)/zones/(?P[a-z0-9-]+)/futureReservations/)?(?P[a-z0-9-]+)?$", var.future_reservation) - - fr_name = local.fr_match.name - fr_project = coalesce(local.fr_match.project, var.project_id) - fr_zone = coalesce(local.fr_match.zone, var.zone) - - future_reservation = var.future_reservation == "" ? "" : "projects/${local.fr_project}/zones/${local.fr_zone}/futureReservations/${local.fr_name}" -} - - -# tflint-ignore: terraform_unused_declarations -data "google_compute_reservation" "reservation" { - count = length(local.reservation_name) > 0 ? 1 : 0 - - name = local.res_short_name - project = local.res_project - zone = var.zone - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "Couldn't find the reservation ${var.reservation_name}" - } - - postcondition { - condition = coalesce(self.specific_reservation_required, true) - error_message = < 0] -} - -resource "terraform_data" "machine_type_zone_validation" { - input = var.machine_type - lifecycle { - precondition { - condition = length(local.zones_with_machine_type) > 0 - error_message = <<-EOT - machine type ${var.machine_type} is not available in any of the zones ${jsonencode(local.zones)}". To list zones in which it is available, run: - - gcloud compute machine-types list --filter="name=${var.machine_type}" - EOT - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml deleted file mode 100644 index 95b6d1c730..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name - has_to_be_used: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf deleted file mode 100644 index 18ed74e2d5..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset" { - description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset - - precondition { - condition = !contains([ - "c3-:pd-standard", - "h3-:pd-standard", - "h3-:pd-ssd", - ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") - error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." - } - - precondition { - condition = var.reservation_name == "" || length(var.zones) == 0 - error_message = <<-EOD - If a reservation is specified, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.accelerator_topology == null || var.enable_placement - error_message = "accelerator_topology requires enable_placement to be set to true." - } - - precondition { - condition = (var.accelerator_topology == null) || try(tonumber(split("x", var.accelerator_topology)[1]) % local.guest_accelerator[0].count == 0, false) - error_message = "accelerator_topology must be divisible by number of gpus in machine." - } - - precondition { - condition = var.placement_max_distance == null || var.enable_placement - error_message = "placement_max_distance requires enable_placement to be set to true." - } - - precondition { - condition = !(startswith(var.machine_type, "a3-") && var.placement_max_distance == 1) - error_message = "A3 machines do not support a placement_max_distance of 1." - } - - precondition { - condition = var.reservation_name == "" || !var.dws_flex.enabled - error_message = "Cannot use reservations with DWS Flex." - } - - precondition { - condition = !var.enable_placement || !var.dws_flex.enabled - error_message = "Cannot use DWS Flex with `enable_placement`." - } - - precondition { - condition = length(var.zones) == 0 || !var.dws_flex.enabled - error_message = <<-EOD - If a DWS Flex is enabled, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.on_host_maintenance == "TERMINATE" || !var.dws_flex.enabled - error_message = "If DWS Flex is used, `on_host_maintenance` should be set to 'TERMINATE'" - } - - precondition { - condition = !var.enable_spot_vm || !var.dws_flex.enabled - error_message = "Cannot use both Flex-Start and Spot VMs for provisioning." - } - - precondition { - condition = var.reservation_name == "" || var.future_reservation == "" - error_message = "Cannot use reservations and future reservations in the same nodeset" - } - - precondition { - condition = !var.enable_placement || var.future_reservation == "" - error_message = "Cannot use `enable_placement` with future reservations." - } - - precondition { - condition = var.future_reservation == "" || length(var.zones) == 0 - error_message = <<-EOD - If a future reservation is specified, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.future_reservation == "" || local.fr_zone == var.zone - error_message = <<-EOD - The zone of the deployment must match that of the future reservation - EOD - } - - precondition { - condition = var.node_count_dynamic_max > 0 || var.node_count_static > 0 - error_message = <<-EOD - This nodeset contains zero nodes, there should be at least one static or dynamic node - EOD - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf deleted file mode 100644 index 06ef5aac6f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf +++ /dev/null @@ -1,641 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "node_conf" { - description = "Map of Slurm node line configuration." - type = map(any) - default = {} - validation { - condition = lookup(var.node_conf, "Sockets", null) == null - error_message = <<-EOD - `Sockets` field is in conflict with `SocketsPerBoard` which is automatically generated by SlurmGCP. - Instead, you can override the following fields: `Boards`, `SocketsPerBoard`, `CoresPerSocket`, and `ThreadsPerCore`. - See: https://slurm.schedmd.com/slurm.conf.html#OPT_Boards and https://slurm.schedmd.com/slurm.conf.html#OPT_Sockets_1 - EOD - } -} - -variable "node_count_static" { - description = "Number of nodes to be statically created." - type = number - default = 0 -} - -variable "node_count_dynamic_max" { - description = "Maximum number of auto-scaling nodes allowed in this partition." - type = number - default = 10 -} - -## VM Definition -variable "instance_template" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Instance template can not be specified for compute nodes." - type = string - default = null - validation { - condition = var.instance_template == null - error_message = "DEPRECATED: Instance template can not be specified for compute nodes." - } -} - -variable "machine_type" { - description = "Compute Platform machine type to use for this partition compute nodes." - type = string - default = "c2-standard-60" -} - -variable "metadata" { - type = map(string) - description = "Metadata, provided as a map." - default = {} -} - -variable "instance_image" { - description = <<-EOD - Defines the image that will be used in the Slurm node group VM instances. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - - For more information on creating custom images that comply with Slurm on GCP - see the "Slurm on GCP Custom Images" section in docs/vm-images.md. - EOD - type = map(string) - default = { - family = "slurm-gcp-6-11-hpc-rocky-linux-8" - project = "schedmd-slurm-public" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations - description = <<-EOD - A flag that designates that the user is aware that they are requesting - to use a custom and potentially incompatible image for this Slurm on - GCP module. - - If the field is set to false, only the compatible families and project - names will be accepted. The deployment will fail with any other image - family or name. If set to true, no checks will be done. - - See: https://goo.gle/hpc-slurm-images - EOD - type = bool - default = false -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "tags" { - type = list(string) - description = "Network tag list." - default = [] -} - -variable "disk_type" { - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - type = string - default = "pd-standard" -} - -variable "disk_size_gb" { - description = "Size of boot disk to create for the partition compute nodes." - type = number - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string)) - auto_delete = optional(bool) - boot = optional(bool) - disk_resource_manager_tags = optional(map(string)) - })) - default = [] -} - -variable "enable_confidential_vm" { - type = bool - description = "Enable the Confidential VM configuration. Note: the instance image must support option." - default = false -} - -variable "enable_shielded_vm" { - type = bool - description = "Enable the Shielded VM configuration. Note: the instance image must support option." - default = false -} - -variable "shielded_instance_config" { - type = object({ - enable_integrity_monitoring = bool - enable_secure_boot = bool - enable_vtpm = bool - }) - description = <<-EOD - Shielded VM configuration for the instance. Note: not used unless - enable_shielded_vm is 'true'. - - enable_integrity_monitoring : Compare the most recent boot measurements to the - integrity policy baseline and return a pair of pass/fail results depending on - whether they match or not. - - enable_secure_boot : Verify the digital signature of all boot components, and - halt the boot process if signature verification fails. - - enable_vtpm : Use a virtualized trusted platform module, which is a - specialized computer chip you can use to encrypt objects like keys and - certificates. - EOD - default = { - enable_integrity_monitoring = true - enable_secure_boot = true - enable_vtpm = true - } -} - - -variable "enable_oslogin" { - type = bool - description = <<-EOD - Enables Google Cloud os-login for user login and authentication for VMs. - See https://cloud.google.com/compute/docs/oslogin - EOD - default = true -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example." - type = bool - default = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "labels" { - description = "Labels to add to partition compute instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "min_cpu_platform" { - description = "The name of the minimum CPU platform that you want the instance to use." - type = string - default = null -} - -variable "on_host_maintenance" { - type = string - description = <<-EOD - Instance availability Policy. - - Note: Placement groups are not supported when on_host_maintenance is set to - "MIGRATE" and will be deactivated regardless of the value of - enable_placement. To support enable_placement, ensure on_host_maintenance is - set to "TERMINATE". - EOD - default = "TERMINATE" -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." - } -} - -variable "accelerator_topology" { - type = string - description = "Specifies the shape of the Accelerator (GPU/TPU) slice." - nullable = true - default = null -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - - -variable "service_account_email" { - description = "Service account e-mail address to attach to the compute instances." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the compute instances." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "service_account" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - } -} - -variable "enable_spot_vm" { - description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." - type = bool - default = false -} - -variable "spot_instance_config" { - description = "Configuration for spot VMs." - type = object({ - termination_action = string - }) - default = null -} - -variable "bandwidth_tier" { - description = < 0 - error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME[/reservationBlocks/BLOCK_ID]', [...] are optional parts." - } -} - -variable "future_reservation" { - description = <<-EOD - If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zones/ZONE/futureReservations/FUTURE_RESERVATION_NAME'. - See https://cloud.google.com/compute/docs/instances/future-reservations-overview - EOD - type = string - default = "" - nullable = false - - validation { - condition = length(regexall("^(projects/([a-z0-9-]+)/zones/([a-z0-9-]+)/futureReservations/([a-z0-9-]+))?$", var.future_reservation)) > 0 || length(regexall("^([a-z0-9-]+)$", var.future_reservation)) > 0 - error_message = "Future reservation must be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zone/ZONE/futureReservations/FUTURE_RESERVATION_NAME'." - } -} - -variable "maintenance_interval" { - description = <<-EOD - Sets the maintenance interval for instances in this nodeset. - See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. - EOD - type = string - default = null -} - -variable "startup_script" { - description = "Startup script used by VMs in this nodeset" - type = string - default = "# no-op" -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured on nodes." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - })) - default = [] -} - - -variable "instance_properties" { - description = <<-EOD - Override the instance properties. Used to test features not supported by Slurm GCP, - recommended for advanced usage only. - See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert - If any sub-field (e.g. scheduling) is set, it will override the values computed by - SlurmGCP and ignoring values of provided vars. - EOD - type = any - default = null -} - - -variable "enable_maintenance_reservation" { - type = bool - description = "Enables slurm reservation for scheduled maintenance." - default = false -} - - -variable "enable_opportunistic_maintenance" { - type = bool - description = "On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle." - default = false -} - - -variable "dws_flex" { - description = <<-EOD - If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes. - See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler - Options: - - enable: Enable DWS Flex Start - - max_run_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week). - - use_job_duration: Use the job duration to determine the max_run_duration, if job duration is not set, max_run_duration will be used. - - use_bulk_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances - - Limitations: - - CAN NOT be used with reservations; - - CAN NOT be used with placement groups; - - EOD - - type = object({ - enabled = optional(bool, true) - max_run_duration = optional(number, 604800) # one week - use_job_duration = optional(bool, false) - use_bulk_insert = optional(bool, false) - }) - default = { - enabled = false - } - validation { - condition = var.dws_flex.max_run_duration >= 600 && var.dws_flex.max_run_duration <= 604800 - error_message = "Max duration must be at least than 10 minutes, and cannot be more than one week." - } -} - -variable "placement_max_distance" { - type = number - description = "Maximum distance between nodes in the placement group. Requires enable_placement to be true. Values must be supported by the chosen machine type." - nullable = true - default = null - - validation { - condition = coalesce(var.placement_max_distance, 1) >= 1 && coalesce(var.placement_max_distance, 3) <= 3 - error_message = "Invalid value for placement_max_distance. Valid values are null, 1, 2, or 3." - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf deleted file mode 100644 index e014c318e4..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.4" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 5.11" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.74.0" - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md deleted file mode 100644 index d3dbcd959e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md +++ /dev/null @@ -1,105 +0,0 @@ -## Description - -This module creates a compute partition that can be used as input to the -[schedmd-slurm-gcp-v6-controller](../../scheduler/schedmd-slurm-gcp-v6-controller/README.md). - -The partition module is designed to work alongside the -[schedmd-slurm-gcp-v6-nodeset](../schedmd-slurm-gcp-v6-nodeset/README.md) -module. A partition can be made up of one or -more nodesets, provided either through `use` (preferred) or defined manually -in the `nodeset` variable. - -### Example - -The following code snippet creates a partition module with: - -* 2 nodesets added via `use`. - * The first nodeset is made up of machines of type `c2-standard-30`. - * The second nodeset is made up of machines of type `c2-standard-60`. - * Both nodesets have a maximum count of 200 dynamically created nodes. -* partition name of "compute". -* connected to the `network` module via `use`. -* nodes mounted to homefs via `use`. - -```yaml -- id: nodeset_1 - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - name: c30 - node_count_dynamic_max: 200 - machine_type: c2-standard-30 - -- id: nodeset_2 - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - name: c60 - node_count_dynamic_max: 200 - machine_type: c2-standard-60 - -- id: compute_partition - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - use: - - homefs - - nodeset_1 - - nodeset_2 - settings: - partition_name: compute -``` - -## Support - -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [exclusive](#input\_exclusive) | Exclusive job access to nodes. When set to true nodes execute single job and are deleted
after job exits. If set to false, multiple jobs can be scheduled on one node. | `bool` | `true` | no | -| [is\_default](#input\_is\_default) | Sets this partition as the default partition by updating the partition\_conf.
If "Default" is already set in partition\_conf, this variable will have no effect. | `bool` | `false` | no | -| [network\_storage](#input\_network\_storage) | DEPRECATED | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | A list of nodesets.
For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip     = string
      remote_mount  = string
      local_mount   = string
      fs_type       = string
      mount_options = string
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | -| [partition\_conf](#input\_partition\_conf) | Slurm partition configuration as a map.
See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION | `map(string)` | `{}` | no | -| [partition\_name](#input\_partition\_name) | The name of the slurm partition. | `string` | n/a | yes | -| [resume\_timeout](#input\_resume\_timeout) | Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'ResumeTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. | `number` | `null` | no | -| [suspend\_time](#input\_suspend\_time) | Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram.
This sets 'SuspendTime' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details.
NOTE: use value -1 to exclude partition from suspend.
NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. | `number` | `300` | no | -| [suspend\_timeout](#input\_suspend\_timeout) | Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'SuspendTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. | `number` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Details of a nodesets in this partition | -| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of a dynamic nodesets in this partition | -| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of a TPU nodesets in this partition | -| [partitions](#output\_partitions) | Details of a slurm partition | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf deleted file mode 100644 index 1618c64280..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - use_static = [for ns in concat(var.nodeset, var.nodeset_tpu) : ns.nodeset_name if ns.node_count_static > 0] - - has_node = length(var.nodeset) > 0 - has_dyn = length(var.nodeset_dyn) > 0 - has_tpu = length(var.nodeset_tpu) > 0 - has_flex = length([for ns in var.nodeset : ns.dws_flex.enabled if ns.dws_flex.enabled]) > 0 -} - -locals { - partition_conf = merge({ - "Default" = var.is_default ? "YES" : null - "SuspendTime" = var.suspend_time < 0 ? "INFINITE" : var.suspend_time - "SuspendTimeout" = var.suspend_timeout != null ? var.suspend_timeout : (local.has_tpu ? 240 : 120) - }, var.partition_conf, { "ResumeTimeout" = local.has_flex ? 65535 : try(var.partition_conf["ResumeTimeout"], coalesce(var.resume_timeout, (local.has_tpu ? 600 : 300))) }) - - partition = { - partition_name = var.partition_name - partition_conf = local.partition_conf - - partition_nodeset = [for ns in var.nodeset : ns.nodeset_name] - partition_nodeset_tpu = [for ns in var.nodeset_tpu : ns.nodeset_name] - partition_nodeset_dyn = [for ns in var.nodeset_dyn : ns.nodeset_name] - # Options - enable_job_exclusive = var.exclusive - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml deleted file mode 100644 index 13ea127b3c..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - has_to_be_used: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf deleted file mode 100644 index 35dece64fb..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "partitions" { - description = "Details of a slurm partition" - - value = [local.partition] - - precondition { - condition = (length(local.use_static) == 0) || !var.exclusive - error_message = <<-EOD - Can't use static nodes within partition with `var.exclusive` set to `true`. - NOTE: Partition's `var.exclusive` is set to `true` by default. Set it to `false` explicitly to use static nodes. - EOD - } - - precondition { - # Can not mix TPU with other non-TPU nodesets due to SlurmGCP specific limitations; - # Can not mix dynamic with non-dynamic nodesets due to Slurms inability to - # turn off "power management" at nodeset level (can only do it at partition or node level). - condition = sum([for b in [local.has_node, local.has_dyn, local.has_tpu] : b ? 1 : 0]) == 1 - error_message = "Partition must contain exactly one type of nodeset." - } -} - -output "nodeset" { - description = "Details of a nodesets in this partition" - - value = var.nodeset -} - -output "nodeset_tpu" { - description = "Details of a TPU nodesets in this partition" - - value = var.nodeset_tpu -} - - -output "nodeset_dyn" { - description = "Details of a dynamic nodesets in this partition" - - value = var.nodeset_dyn -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf deleted file mode 100644 index a1c85adb90..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "partition_name" { - description = "The name of the slurm partition." - type = string - - validation { - condition = can(regex("^[a-z](?:[a-z0-9]*)$", var.partition_name)) - error_message = "Variable 'partition_name' must be a match of regex '^[a-z](?:[a-z0-9]*)$'." - } -} - -variable "partition_conf" { - description = <<-EOD - Slurm partition configuration as a map. - See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION - EOD - type = map(string) - default = {} -} - -variable "is_default" { - description = <<-EOD - Sets this partition as the default partition by updating the partition_conf. - If "Default" is already set in partition_conf, this variable will have no effect. - EOD - type = bool - default = false -} - -variable "exclusive" { - description = <<-EOD - Exclusive job access to nodes. When set to true nodes execute single job and are deleted - after job exits. If set to false, multiple jobs can be scheduled on one node. - EOD - type = bool - default = true -} - -variable "nodeset" { - description = <<-EOD - A list of nodesets. - For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset - EOD - type = list(object({ - node_count_static = optional(number, 0) - node_count_dynamic_max = optional(number, 1) - node_conf = optional(map(string), {}) - nodeset_name = string - additional_disks = optional(list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string), {}) - auto_delete = optional(bool, true) - boot = optional(bool, false) - disk_resource_manager_tags = optional(map(string), {}) - })), []) - bandwidth_tier = optional(string, "platform_default") - can_ip_forward = optional(bool, false) - disk_auto_delete = optional(bool, true) - disk_labels = optional(map(string), {}) - disk_resource_manager_tags = optional(map(string), {}) - disk_size_gb = optional(number) - disk_type = optional(string) - enable_confidential_vm = optional(bool, false) - enable_placement = optional(bool, false) - placement_max_distance = optional(number, null) - enable_oslogin = optional(bool, true) - enable_shielded_vm = optional(bool, false) - enable_maintenance_reservation = optional(bool, false) - enable_opportunistic_maintenance = optional(bool, false) - gpu = optional(object({ - count = number - type = string - })) - accelerator_topology = optional(string, null) - dws_flex = object({ - enabled = bool - max_run_duration = number - use_job_duration = bool - use_bulk_insert = bool - }) - labels = optional(map(string), {}) - machine_type = optional(string) - advanced_machine_features = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - maintenance_interval = optional(string) - instance_properties_json = string - metadata = optional(map(string), {}) - min_cpu_platform = optional(string) - network_tier = optional(string, "STANDARD") - network_storage = optional(list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = optional(map(string)) - mount_runner = optional(map(string)) - })), []) - on_host_maintenance = optional(string) - preemptible = optional(bool, false) - region = optional(string) - resource_manager_tags = optional(map(string), {}) - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - shielded_instance_config = optional(object({ - enable_integrity_monitoring = optional(bool, true) - enable_secure_boot = optional(bool, true) - enable_vtpm = optional(bool, true) - })) - source_image_family = optional(string) - source_image_project = optional(string) - source_image = optional(string) - subnetwork_self_link = string - additional_networks = optional(list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - }))) - access_config = optional(list(object({ - nat_ip = string - network_tier = string - }))) - spot = optional(bool, false) - tags = optional(list(string), []) - termination_action = optional(string) - reservation_name = optional(string) - future_reservation = string - startup_script = optional(list(object({ - filename = string - content = string })), []) - - zone_target_shape = string - zone_policy_allow = set(string) - zone_policy_deny = set(string) - })) - default = [] - - validation { - condition = length(distinct(var.nodeset[*].nodeset_name)) == length(var.nodeset) - error_message = "All nodesets must have a unique name." - } -} - -variable "nodeset_tpu" { - description = "Define TPU nodesets, as a list." - type = list(object({ - node_count_static = optional(number, 0) - node_count_dynamic_max = optional(number, 5) - nodeset_name = string - enable_public_ip = optional(bool, false) - node_type = string - accelerator_config = optional(object({ - topology = string - version = string - }), { - topology = "" - version = "" - }) - tf_version = string - preemptible = optional(bool, false) - preserve_tpu = optional(bool, false) - zone = string - data_disks = optional(list(string), []) - docker_image = optional(string, "") - network_storage = optional(list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - })), []) - subnetwork = string - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - project_id = string - reserved = optional(string, false) - })) - default = [] - - validation { - condition = length(distinct([for x in var.nodeset_tpu : x.nodeset_name])) == length(var.nodeset_tpu) - error_message = "All TPU nodesets must have a unique name." - } -} - -variable "nodeset_dyn" { - description = "Defines dynamic nodesets, as a list." - type = list(object({ - nodeset_name = string - nodeset_feature = string - })) - default = [] - - validation { - condition = length(distinct([for x in var.nodeset_dyn : x.nodeset_name])) == length(var.nodeset_dyn) - error_message = "All dynamic nodesets must have a unique name." - } -} - -variable "resume_timeout" { - description = <<-EOD - Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use. - If null is given, then a smart default will be chosen depending on nodesets in partition. - This sets 'ResumeTimeout' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. - EOD - type = number - default = null - - validation { - condition = var.resume_timeout == null ? true : var.resume_timeout > 0 && var.resume_timeout < 65536 - error_message = "Value must be > 0 and < 65536" - } -} - -variable "suspend_time" { - description = <<-EOD - Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram. - This sets 'SuspendTime' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details. - NOTE: use value -1 to exclude partition from suspend. - NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. - EOD - type = number - default = 300 - - validation { - condition = var.suspend_time >= -1 - error_message = "Value must be >= -1." - } -} - -variable "suspend_timeout" { - description = <<-EOD - Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown. - If null is given, then a smart default will be chosen depending on nodesets in partition. - This sets 'SuspendTimeout' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. - EOD - type = number - default = null - - validation { - condition = var.suspend_timeout == null ? true : var.suspend_timeout > 0 - error_message = "Value must be > 0." - } -} - - -# tflint-ignore: terraform_unused_declarations -variable "network_storage" { - description = "DEPRECATED" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] - validation { - condition = length(var.network_storage) == 0 - error_message = <<-EOD - network_storage in partition module is deprecated and should not be set. - To add network storage to compute nodes, use network_storage of nodeset module instead. - EOD - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf deleted file mode 100644 index d388f4bfdd..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.3" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.74.0" - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/README.md b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/README.md deleted file mode 100644 index 994f1500ba..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/README.md +++ /dev/null @@ -1,157 +0,0 @@ -## Description - -This module provides ways to create and manage Google Cloud Artifact Registry repositories. - -Currently this module is built to support repositories in Docker format although there are placeholder variables for other types which may work too. Remote repositories with pull-through cache functionality integrated with Google Secret Manager is currently supported. The aim of this module is to eventually offer feature parity with this [Terraform module](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository#nested_remote_repository_config), allowing creation of repositories in various formats, including Docker, Maven, NPM, Python, APT, YUM, and COMMON. - -This module is best suited for managing artifact repositories in HPC/AI containerized environments where artifacts need to be shared across distributed systems. It includes IAM role configurations and secret access handling for seamless integration with CI/CD pipelines and other services too. - -It is designed to help facilitate containerized workloads running in the Cluster Toolkit with SLURM leveraging [Enroot](https://github.com/NVIDIA/enroot) and [Pyxis](https://github.com/NVIDIA/pyxis). Docker repositories can store container images that are used in job submissions, enabling efficient and scalable execution of containerized HPC or AI based workloads. - -## Usage - -### Service Account / APIs - -You will need to enable the relevant APIs and create a Service Account for your cluster with the following Artifact Registry permissions. - -```yaml - - id: services-api - source: community/modules/project/service-enablement - settings: - gcp_service_list: - - secretmanager.googleapis.com - - cloudbuild.googleapis.com - - artifactregistry.googleapis.com - - - source: community/modules/project/service-account - kind: terraform - id: hpc_service_account - settings: - project_id: project_name - name: service_account_name - project_roles: - - artifactregistry.reader - - artifactregistry.writer - - secretmanager.secretAccessor -``` - -### Deployment - -Create a standard Docker repository. - -```yaml -- id: registry - source: community/modules/container/artifact-registry - settings: - repo_mode: STANDARD_REPOSITORY - format: DOCKER -``` - -Mirror of public Docker Hub repository. - -```yaml -- id: dockerhub_registry - source: community/modules/container/artifact-registry - settings: - repo_mode: REMOTE_REPOSITORY - format: DOCKER - repo_public_repository: DOCKER_HUB -``` - -Mirror of NVIDIA's [NGC Catalog](https://catalog.ngc.nvidia.com/containers). [API key](https://org.ngc.nvidia.com/setup/api-key) used in blueprint is stored in Secret Manager. - -```yaml -- id: ngc_registry - source: community/modules/container/artifact-registry - settings: - repo_mode: REMOTE_REPOSITORY - format: DOCKER - repo_mirror_url: "https://nvcr.io" - repo_username: $oauthtoken - repo_password: api_key_here - use_upstream_credentials: True -``` - -### Container Operations - -Retrieve `$REPOSITORY_NAME` from [Artifact Registry](https://console.cloud.google.com/artifacts) or by using `gcloud`. - -```yaml -gcloud artifacts repositories list --project="${PROJECT_ID}" -``` - -Pulling containers from your mirrored internal Artifact Repositories. - -Pull [Ubuntu](https://hub.docker.com/_/ubuntu) from Docker Hub mirror. - -```yaml -docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/library/ubuntu:latest -``` - -Pull [Pytorch](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch) from NGC Catalog mirror. - -```yaml -docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/nvidia/pytorch:24.11-py3 -``` - -Alternatively, proceed with running SLURM's [NVIDIA/pyxis](https://github.com/NVIDIA/pyxis) plugin, which will now be able to pull and use these containers directly from the mirrored repositories. - -Note: only Docker registries have been tested so far. Placeholders do exist for other registry types which may or may not work. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_artifact_registry_repository.artifact_registry](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository) | resource | -| [google_secret_manager_secret.repo_password_secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_version.repo_password_secret_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [random_password.repo_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | -| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment. | `string` | n/a | yes | -| [format](#input\_format) | Artifact Registry format (e.g., DOCKER). | `string` | `"DOCKER"` | no | -| [labels](#input\_labels) | Labels to add to the artifact registry. Key-value pairs. | `map(string)` | `{}` | no | -| [project\_id](#input\_project\_id) | Project ID where the artifact registry and secret are created. | `string` | n/a | yes | -| [region](#input\_region) | Region for the artifact registry. | `string` | n/a | yes | -| [repo\_mirror\_url](#input\_repo\_mirror\_url) | For REMOTE\_REPOSITORY, URL for a custom or common mirror. | `string` | `null` | no | -| [repo\_mode](#input\_repo\_mode) | Artifact Registry mode (STANDARD\_REPOSITORY, REMOTE\_REPOSITORY, etc.). | `string` | `"STANDARD_REPOSITORY"` | no | -| [repo\_password](#input\_repo\_password) | Optional password/API key. If null, one will be randomly generated. | `string` | `null` | no | -| [repo\_public\_repository](#input\_repo\_public\_repository) | For REMOTE\_REPOSITORY, name of a known public repo as per the Terraform module
(e.g., DOCKER\_HUB) or null for custom repo. | `string` | `null` | no | -| [repo\_username](#input\_repo\_username) | Username for external repository. | `string` | `null` | no | -| [repository\_base](#input\_repository\_base) | For APT/YUM public repos, repository\_base (e.g., 'DEBIAN', 'UBUNTU'). | `string` | `null` | no | -| [repository\_path](#input\_repository\_path) | For APT/YUM public repos, repository\_path (e.g., 'debian/dists/buster'). | `string` | `null` | no | -| [use\_upstream\_credentials](#input\_use\_upstream\_credentials) | Configure Service Account to use upstream credentials for REMOTE\_REPOSITORY:
If true, a username/password is used for the REMOTE\_REPOSITORY mirror.
If false (or if repo\_password == null), no password is created at all.
Note: Blueprint credentials will be stored in Secrets Manager. | `bool` | `false` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | (Optional) A list of objects to enable user-managed replication.
Each object can have:
location = string
kms\_key\_name = optional(string)
If empty, auto replication is used. | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [registry\_url](#output\_registry\_url) | The URL of the created artifact registry. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/main.tf b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/main.tf deleted file mode 100644 index c3406af607..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/main.tf +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "artifact-registry", ghpc_role = "container" }) -} - -locals { - # Auto (i.e., empty) vs user-managed replication - auto = length(var.user_managed_replication) == 0 ? true : false - - # For remote custom repositories, parse out host to create a base_component name - mirror_url_no_proto = var.repo_mirror_url != null ? replace(replace(var.repo_mirror_url, "https://", ""), "http://", "") : "" - mirror_host = local.mirror_url_no_proto != "" ? split("/", local.mirror_url_no_proto)[0] : "" - - base_component = replace( - replace( - replace( - lower( - local.mirror_host != "" - ? "${var.format}-${var.repo_mode}-${local.mirror_host}" - : "${var.format}-${var.repo_mode}-nohost" - ), - "\\.", "-" - ), - "/", "-" - ), - "_", "-" - ) - - repository_suffix = random_id.resource_name_suffix.hex - - # The final name for the artifact registry repository - repository_name = replace( - replace( - lower( - format("%s-%s", local.base_component, local.repository_suffix) - ), - ".", "-" - ), - "/", "-" - ) - - # The secret name is derived from the repository name - # with a suffix like "-secret". - derived_secret_name = format("%s-secret", local.repository_name) -} - -############################## -# PASSWORD / SECRET -############################## - -# Only create a random password if user didn't supply one -resource "random_password" "repo_password" { - count = var.use_upstream_credentials && var.repo_password == null ? 1 : 0 - length = 24 - special = true - override_special = "_-#=." -} - -resource "google_secret_manager_secret" "repo_password_secret" { - count = var.use_upstream_credentials ? 1 : 0 - project = var.project_id - - # Derive the secret ID from the repository name - secret_id = local.derived_secret_name - - labels = local.labels - - replication { - dynamic "auto" { - for_each = local.auto ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = local.auto ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = replicas.value.kms_key_name != null ? [1] : [] - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_version" "repo_password_secret_version" { - count = var.use_upstream_credentials ? 1 : 0 - secret = google_secret_manager_secret.repo_password_secret[0].id - - # If user provided a password, use it. Otherwise use the random password. - secret_data = var.repo_password != null ? var.repo_password : random_password.repo_password[0].result -} - -############################## -# IAM BINDINGS -############################## - -############################## -# ARTIFACT REGISTRY -############################## - -resource "random_id" "resource_name_suffix" { - byte_length = 2 -} - -resource "google_artifact_registry_repository" "artifact_registry" { - project = var.project_id - location = var.region - format = var.format - mode = var.repo_mode - description = var.deployment_name - labels = local.labels - repository_id = local.repository_name - - # Only create remote_repository_config if REMOTE_REPOSITORY - dynamic "remote_repository_config" { - for_each = var.repo_mode == "REMOTE_REPOSITORY" ? [1] : [] - content { - description = "Pull-through cache" - - dynamic "docker_repository" { - for_each = var.format == "DOCKER" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "docker_repository" { - for_each = var.format == "DOCKER" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "maven_repository" { - for_each = var.format == "MAVEN" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "maven_repository" { - for_each = var.format == "MAVEN" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "npm_repository" { - for_each = var.format == "NPM" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "npm_repository" { - for_each = var.format == "NPM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "python_repository" { - for_each = var.format == "PYTHON" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "python_repository" { - for_each = var.format == "PYTHON" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "apt_repository" { - for_each = var.format == "APT" && var.repo_public_repository != null ? [1] : [] - content { - public_repository { - repository_base = var.repository_base - repository_path = var.repository_path - } - } - } - - dynamic "apt_repository" { - for_each = var.format == "APT" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "yum_repository" { - for_each = var.format == "YUM" && var.repo_public_repository != null ? [1] : [] - content { - public_repository { - repository_base = var.repository_base - repository_path = var.repository_path - } - } - } - - dynamic "yum_repository" { - for_each = var.format == "YUM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "common_repository" { - for_each = var.format == "COMMON" ? [1] : [] - content { - uri = var.repo_mirror_url - } - } - - # Only enable upstream credentials if user wants it - dynamic "upstream_credentials" { - for_each = var.use_upstream_credentials ? [1] : [] - content { - username_password_credentials { - username = var.repo_username - password_secret_version = google_secret_manager_secret_version.repo_password_secret_version[0].name - } - } - } - } - } - - depends_on = [ - google_secret_manager_secret.repo_password_secret, - google_secret_manager_secret_version.repo_password_secret_version, - ] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/metadata.yaml deleted file mode 100644 index 6b68c98a54..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - secretmanager.googleapis.com - - artifactregistry.googleapis.com - - cloudbuild.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/outputs.tf deleted file mode 100644 index 92b6dbb165..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "registry_url" { - description = "The URL of the created artifact registry." - value = "${var.region}-docker.pkg.dev/${var.project_id}/${var.deployment_name}" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/validation.tf b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/validation.tf deleted file mode 100644 index a795060fb7..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/validation.tf +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "terraform_data" "input_validation" { - lifecycle { - precondition { - condition = ( - var.repo_password == null || - (var.use_upstream_credentials && var.repo_mode == "REMOTE_REPOSITORY") - ) - error_message = "repo_password may be set only when repo_mode=REMOTE_REPOSITORY and use_upstream_credentials=true." - } - - precondition { - condition = ( - !var.use_upstream_credentials || - var.repo_mode == "REMOTE_REPOSITORY" - ) - error_message = "use_upstream_credentials is allowed only when repo_mode is REMOTE_REPOSITORY." - } - - precondition { - condition = ( - var.repo_mode != "REMOTE_REPOSITORY" || - (var.repo_public_repository != null || var.repo_mirror_url != null) - ) - error_message = "For a REMOTE_REPOSITORY you must set repo_public_repository or repo_mirror_url." - } - - precondition { - condition = ( - !contains(["APT", "YUM"], var.format) || - (var.repository_base != null && var.repository_path != null) - ) - error_message = "APT/YUM formats require repository_base and repository_path." - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/variables.tf deleted file mode 100644 index 9a4eecb921..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/variables.tf +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "Project ID where the artifact registry and secret are created." - type = string -} - -variable "region" { - description = "Region for the artifact registry." - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment." - type = string -} - -variable "labels" { - description = "Labels to add to the artifact registry. Key-value pairs." - type = map(string) - default = {} -} - -variable "repo_password" { - description = "Optional password/API key. If null, one will be randomly generated." - type = string - default = null -} - -variable "user_managed_replication" { - description = <<-DOC - (Optional) A list of objects to enable user-managed replication. - Each object can have: - location = string - kms_key_name = optional(string) - If empty, auto replication is used. - DOC - type = list(object({ - location = string - kms_key_name = optional(string) - })) - default = [] -} - -variable "format" { - description = "Artifact Registry format (e.g., DOCKER)." - type = string - default = "DOCKER" -} - -variable "repo_mode" { - description = "Artifact Registry mode (STANDARD_REPOSITORY, REMOTE_REPOSITORY, etc.)." - type = string - default = "STANDARD_REPOSITORY" - - validation { - condition = can(regex("^(STANDARD_REPOSITORY|REMOTE_REPOSITORY|VIRTUAL_REPOSITORY)$", var.repo_mode)) - error_message = "repo_mode must be one of STANDARD_REPOSITORY, REMOTE_REPOSITORY, or VIRTUAL_REPOSITORY." - } -} - -variable "repo_public_repository" { - description = <<-DOC - For REMOTE_REPOSITORY, name of a known public repo as per the Terraform module - (e.g., DOCKER_HUB) or null for custom repo. - DOC - type = string - default = null - - # To Do: implement validation - # validation { - # condition = ((var.repo_mode != "REMOTE_REPOSITORY" && var.repo_public_repository == null) || (var.repo_mode == "REMOTE_REPOSITORY" && (var.repo_public_repository != null || var.repo_mirror_url != null))) - # error_message = "If repo_mode is REMOTE_REPOSITORY, you must set either repo_public_repository or repo_mirror_url. Otherwise, leave them null." - # } -} - -variable "repo_mirror_url" { - description = "For REMOTE_REPOSITORY, URL for a custom or common mirror." - type = string - default = null -} - -variable "use_upstream_credentials" { - description = <<-DOC - Configure Service Account to use upstream credentials for REMOTE_REPOSITORY: - If true, a username/password is used for the REMOTE_REPOSITORY mirror. - If false (or if repo_password == null), no password is created at all. - Note: Blueprint credentials will be stored in Secrets Manager. - DOC - type = bool - default = false -} - -variable "repo_username" { - description = "Username for external repository." - type = string - default = null -} - -variable "repository_base" { - description = "For APT/YUM public repos, repository_base (e.g., 'DEBIAN', 'UBUNTU')." - type = string - default = null -} - -variable "repository_path" { - description = "For APT/YUM public repos, repository_path (e.g., 'debian/dists/buster')." - type = string - default = null -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/versions.tf deleted file mode 100644 index 392a7131d2..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/container/artifact-registry/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/README.md b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/README.md deleted file mode 100644 index 23bf87398a..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/README.md +++ /dev/null @@ -1,76 +0,0 @@ -## Description - -Creates a BigQuery dataset. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage -This is a simple usage. - -```yaml - - id: bq-dataset - source: community/modules/database/bigquery-dataset - settings: - dataset_id: my_dataset -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_dataset.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | The name of the dataset to be created | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the dataset. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [dataset\_id](#output\_dataset\_id) | Name of the dataset that was created. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/main.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/main.tf deleted file mode 100644 index 1a9c4bba60..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/main.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-dataset", ghpc_role = "database" }) -} -locals { - dataset_id = var.dataset_id != null ? var.dataset_id : replace("${var.deployment_name}_dataset_${random_id.resource_name_suffix.hex}", "-", "_") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_bigquery_dataset" "pbsb" { - dataset_id = local.dataset_id - project = var.project_id - labels = local.labels -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml deleted file mode 100644 index 87ff9357e4..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigquery.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf deleted file mode 100644 index 9cd8e5df31..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "dataset_id" { - description = "Name of the dataset that was created." - value = google_bigquery_dataset.pbsb.dataset_id -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/variables.tf deleted file mode 100644 index 90c229af6b..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/variables.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "dataset_id" { - description = "The name of the dataset to be created" - type = string - default = null -} - -variable "labels" { - description = "Labels to add to the dataset. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/versions.tf deleted file mode 100644 index 12ddbe842d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-dataset/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/README.md b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/README.md deleted file mode 100644 index ef67cfef01..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/README.md +++ /dev/null @@ -1,87 +0,0 @@ -## Description - -Creates a BigQuery table with a specified schema. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage - -```yaml -id: bq-table - source: community/modules/database/bigquery-table - use: [bq-dataset] - settings: - table_schema: - ' - [ - { - "name": "id", "type": "STRING" - } - ] - ' -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_table.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Dataset name to be used to create the new BQ Table | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the tables. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [table\_id](#input\_table\_id) | Table name to be used to create the new BQ Table | `string` | `null` | no | -| [table\_schema](#input\_table\_schema) | Schema used to create the new BQ Table | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [dataset\_id](#output\_dataset\_id) | ID of BQ dataset | -| [table\_id](#output\_table\_id) | ID of created BQ table | -| [table\_name](#output\_table\_name) | Name of created BQ table | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/main.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/main.tf deleted file mode 100644 index 73f3923e00..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-table", ghpc_role = "database" }) -} - -locals { - table_id = var.table_id != null ? var.table_id : replace("${var.deployment_name}_table_${random_id.resource_name_suffix.hex}", "-", "_") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_bigquery_table" "pbsb" { - deletion_protection = false - project = var.project_id - table_id = local.table_id - dataset_id = var.dataset_id - schema = var.table_schema - labels = local.labels -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/metadata.yaml deleted file mode 100644 index 87ff9357e4..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigquery.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/outputs.tf deleted file mode 100644 index 4220ec1390..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/outputs.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "table_name" { - description = "Name of created BQ table" - value = google_bigquery_table.pbsb.friendly_name -} -output "table_id" { - description = "ID of created BQ table" - value = google_bigquery_table.pbsb.table_id -} -output "dataset_id" { - description = "ID of BQ dataset" - value = google_bigquery_table.pbsb.dataset_id -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/variables.tf deleted file mode 100644 index ec474b4e64..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/variables.tf +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "labels" { - description = "Labels to add to the tables. Key-value pairs." - type = map(string) -} - -variable "table_id" { - description = "Table name to be used to create the new BQ Table" - type = string - default = null -} - -variable "dataset_id" { - description = "Dataset name to be used to create the new BQ Table" - type = string -} - -variable "table_schema" { - description = "Schema used to create the new BQ Table" - type = string -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/versions.tf deleted file mode 100644 index 12ddbe842d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/bigquery-table/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md deleted file mode 100644 index 08364c175b..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md +++ /dev/null @@ -1,107 +0,0 @@ -## Description - -terraform-google-sql makes it easy to create a Google CloudSQL instance and -implement high availability settings. This module is meant for use with -Terraform 0.13+ and tested using Terraform 1.0+. - -The cloudsql created here is used to integrate with the slurm cluster to enable -accounting data storage. - -### Example - -```yaml -- id: cloudsql - source: community/modules/database/slurm-cloudsql-federation - use: [network] - settings: - sql_instance_name: slurm-sql6-demo - tier: "db-f1-micro" -``` - -This creates a cloud sql instance, including a database, user that would allow -the slurm cluster to use as an external DB. In addition, it will allow BigQuery -to run federated query through it. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | -| [google](#requirement\_google) | >= 3.83 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_connection.connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_connection) | resource | -| [google_compute_address.psc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_forwarding_rule.psc_consumer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_forwarding_rule) | resource | -| [google_sql_database.database](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database) | resource | -| [google_sql_database_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance) | resource | -| [google_sql_user.users](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [random_password.password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [authorized\_networks](#input\_authorized\_networks) | IP address ranges as authorized networks of the Cloud SQL for MySQL instances | `list(string)` | `[]` | no | -| [data\_cache\_enabled](#input\_data\_cache\_enabled) | Whether data cache is enabled for the instance. Can be used with ENTERPRISE\_PLUS edition. | `bool` | `false` | no | -| [database\_flags](#input\_database\_flags) | Database flags to set on instance. | `map(string)` | `{}` | no | -| [database\_version](#input\_database\_version) | The version of the database to be created. | `string` | `"MYSQL_8_0"` | no | -| [deletion\_protection](#input\_deletion\_protection) | Whether or not to allow Terraform to destroy the instance. | `string` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [disk\_autoresize](#input\_disk\_autoresize) | Set to false to disable automatic disk grow. | `bool` | `true` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of the database disk in GiB. | `number` | `null` | no | -| [edition](#input\_edition) | value | `string` | `"ENTERPRISE"` | no | -| [enable\_backups](#input\_enable\_backups) | Set true to enable backups | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is going to be created in.:
`projects//global/networks/`" | `string` | n/a | yes | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [query\_insights](#input\_query\_insights) | Query insights configuration. | 
object({
    enabled                 = optional(bool, false)
    query_plans_per_minute  = optional(number)
    query_string_length     = optional(number)
    record_application_tags = optional(bool)
    record_client_address   = optional(bool)
  })
| `{}` | no | -| [region](#input\_region) | The region where SQL instance will be configured | `string` | n/a | yes | -| [sql\_instance\_name](#input\_sql\_instance\_name) | name given to the sql instance for ease of identificaion | `string` | n/a | yes | -| [sql\_password](#input\_sql\_password) | Password for the SQL database. | `any` | `null` | no | -| [sql\_username](#input\_sql\_username) | Username for the SQL database | `string` | `"slurm"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self link of the network where Cloud SQL instance PSC endpoint will be created | `string` | `null` | no | -| [tier](#input\_tier) | The machine type to use for the SQL instance | `string` | n/a | yes | -| [use\_psc\_connection](#input\_use\_psc\_connection) | Create Private Service Connection instead of using Private Service Access peering | `bool` | `false` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cloudsql](#output\_cloudsql) | Describes the cloudsql instance. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf deleted file mode 100644 index 9b518a1b5f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "slurm-cloudsql-federation", ghpc_role = "database" }) -} - -locals { - user_managed_replication = var.user_managed_replication -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "random_password" "password" { - length = 12 - special = false -} - -locals { - sql_instance_name = var.sql_instance_name == null ? "${var.deployment_name}-sql-${random_id.resource_name_suffix.hex}" : var.sql_instance_name - sql_password = var.sql_password == null ? random_password.password.result : var.sql_password -} - - -resource "google_sql_database_instance" "instance" { - project = var.project_id - depends_on = [var.private_vpc_connection_peering] - name = local.sql_instance_name - region = var.region - deletion_protection = var.deletion_protection - database_version = var.database_version - - settings { - disk_size = var.disk_size_gb - disk_autoresize = var.disk_autoresize - edition = var.edition - tier = var.tier - user_labels = local.labels - - dynamic "data_cache_config" { - for_each = var.edition == "ENTERPRISE_PLUS" ? [""] : [] - content { - data_cache_enabled = var.data_cache_enabled - } - } - - dynamic "database_flags" { - for_each = var.database_flags - content { - name = database_flags.key - value = database_flags.value - } - } - - insights_config { - query_insights_enabled = var.query_insights.enabled - query_plans_per_minute = var.query_insights.query_plans_per_minute - query_string_length = var.query_insights.query_string_length - record_application_tags = var.query_insights.record_application_tags - record_client_address = var.query_insights.record_client_address - } - - ip_configuration { - ipv4_enabled = false - private_network = var.use_psc_connection ? null : var.network_id - enable_private_path_for_google_cloud_services = true - - dynamic "authorized_networks" { - for_each = var.use_psc_connection ? [] : var.authorized_networks - iterator = ip_range - - content { - value = ip_range.value - } - } - dynamic "psc_config" { - for_each = var.use_psc_connection ? [""] : [] - content { - psc_enabled = true - allowed_consumer_projects = [var.project_id] - } - } - } - - backup_configuration { - enabled = var.enable_backups - # to allow easy switching between ENTERPRISE and ENTERPRISE_PLUS - transaction_log_retention_days = 7 - } - } - lifecycle { - precondition { - condition = var.disk_autoresize && var.disk_size_gb == null || !var.disk_autoresize - error_message = "If setting disk_size_gb set disk_autorize to false to prevent re-provisioning of the instance after disk auto-expansion." - } - } -} - - - -resource "google_compute_address" "psc" { - count = var.use_psc_connection ? 1 : 0 - project = var.project_id - name = local.sql_instance_name - address_type = "INTERNAL" - region = var.region - subnetwork = var.subnetwork_self_link - labels = local.labels -} - -resource "google_compute_forwarding_rule" "psc_consumer" { - count = var.use_psc_connection ? 1 : 0 - name = local.sql_instance_name - project = var.project_id - region = var.region - subnetwork = var.subnetwork_self_link - ip_address = google_compute_address.psc[0].self_link - load_balancing_scheme = "" - recreate_closed_psc = true - target = google_sql_database_instance.instance.psc_service_attachment_link -} - -resource "google_sql_database" "database" { - project = var.project_id - name = "slurm_accounting" - instance = google_sql_database_instance.instance.name -} - -resource "google_sql_user" "users" { - project = var.project_id - name = var.sql_username - instance = google_sql_database_instance.instance.name - password = local.sql_password -} - -resource "google_bigquery_connection" "connection" { - provider = google - project = var.project_id - location = var.region - cloud_sql { - instance_id = google_sql_database_instance.instance.connection_name - database = google_sql_database.database.name - type = "MYSQL" - credential { - username = google_sql_user.users.name - password = google_sql_user.users.password - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml deleted file mode 100644 index fc0cae0859..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigqueryconnection.googleapis.com - - sqladmin.googleapis.com - - servicenetworking.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf deleted file mode 100644 index 0d05221cd8..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "cloudsql" { - description = "Describes the cloudsql instance." - sensitive = true - value = { - server_ip = var.use_psc_connection ? google_compute_address.psc[0].address : google_sql_database_instance.instance.ip_address[0].ip_address - user = google_sql_user.users.name - password = google_sql_user.users.password - db_name = google_sql_database.database.name - user_managed_replication = local.user_managed_replication - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf deleted file mode 100644 index a2f150419e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "authorized_networks" { - description = "IP address ranges as authorized networks of the Cloud SQL for MySQL instances" - type = list(string) - default = [] - nullable = false -} - -variable "database_version" { - description = "The version of the database to be created." - type = string - default = "MYSQL_8_0" - validation { - condition = contains(["MYSQL_5_7", "MYSQL_8_0", "MYSQL_8_4"], var.database_version) - error_message = "The database version must be either MYSQL_5_7, MYSQL_8_0 or MYSQL_8_4." - } -} - -variable "data_cache_enabled" { - description = "Whether data cache is enabled for the instance. Can be used with ENTERPRISE_PLUS edition." - type = bool - default = false -} - -variable "database_flags" { - description = "Database flags to set on instance." - type = map(string) - default = {} - nullable = false -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "disk_autoresize" { - description = "Set to false to disable automatic disk grow." - type = bool - default = true -} - -variable "disk_size_gb" { - description = "Size of the database disk in GiB." - type = number - default = null -} - -variable "edition" { - description = "value" - type = string - validation { - condition = contains(["ENTERPRISE", "ENTERPRISE_PLUS"], var.edition) - error_message = "The database edition must be either ENTERPRISE or ENTERPRISE_PLUS" - } - default = "ENTERPRISE" -} - -variable "enable_backups" { - description = "Set true to enable backups" - type = bool - default = false -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "query_insights" { - description = "Query insights configuration." - nullable = false - default = {} - type = object({ - enabled = optional(bool, false) - query_plans_per_minute = optional(number) - query_string_length = optional(number) - record_application_tags = optional(bool) - record_client_address = optional(bool) - }) -} - -variable "region" { - description = "The region where SQL instance will be configured" - type = string -} - -variable "tier" { - description = "The machine type to use for the SQL instance" - type = string -} - -variable "sql_instance_name" { - description = "name given to the sql instance for ease of identificaion" - type = string -} - -variable "deletion_protection" { - description = "Whether or not to allow Terraform to destroy the instance." - type = string - default = false -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} - -variable "sql_username" { - description = "Username for the SQL database" - type = string - default = "slurm" -} - -variable "sql_password" { - description = "Password for the SQL database." - type = any - default = null -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is going to be created in.: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "private_vpc_connection_peering" { - description = "The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation." - type = string - default = null -} - -variable "subnetwork_self_link" { - description = "Self link of the network where Cloud SQL instance PSC endpoint will be created" - type = string - default = null -} - -variable "user_managed_replication" { - type = list(object({ - location = string - kms_key_name = optional(string) - })) - description = "Replication parameters that will be used for defined secrets" - default = [] -} - -variable "use_psc_connection" { - description = "Create Private Service Connection instead of using Private Service Access peering" - type = bool - default = false -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf deleted file mode 100644 index 7e672858b6..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" - } - - required_version = ">= 0.13.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md deleted file mode 100644 index d39a58afe1..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md +++ /dev/null @@ -1,158 +0,0 @@ -> [!WARNING] -> This module is deprecated and will be removed on July 1, 2025. The -> recommended replacement is the -> [GCP Managed Lustre module](../../../../modules/file-system/managed-lustre/README.md) - -## Description -This module creates a DDN EXAScaler Cloud Lustre file system using code based on DDN's -[exascaler-cloud-terraform](https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp) (`scripts/2.2.2` is last release with GCP-specific module). - -More information about the architecture can be found at -[Overview of Lustre and EXAScaler Cloud][architecture]. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -> **Warning**: This file system has a license cost as described in the pricing -> section of the [DDN EXAScaler Cloud Marketplace Solution][marketplace]. -> -> **Note**: By default security.public_key is set to `null`, therefore the -> admin user is not created. To ensure the admin user is created, provide a -> public key via the security setting. -> -> **Note**: This module's instances require access to Google APIs and -> therefore, instances must have public IP address or it must be used in a -> subnetwork where [Private Google Access][private-google-access] is enabled. - -[private-google-access]: https://cloud.google.com/vpc/docs/configure-private-google-access -[marketplace]: https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud -[architecture]: https://cloud.google.com/architecture/parallel-file-systems-for-hpc#overview_of_lustre_and_exascaler_cloud - -## Mounting - -To mount the DDN EXAScaler Lustre file system you must first install the DDN -Lustre client and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -the [hpc-enterprise-slurm.yaml](../../../../examples/hpc-enterprise-slurm.yaml) for an -example of using this module with Slurm. - -If mounting is not automatically handled as described above, the DDN-EXAScaler -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - # This file system has an associated license cost. - # https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud - - id: lustrefs - source: community/modules/file-system/DDN-EXAScaler - use: [network1] - settings: {local_mount: /scratch} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(lustrefs.install_ddn_lustre_client_runner) - - $(lustrefs.mount_runner) - -``` - -See [additional documentation][ddn-install-docs] from DDN EXAScaler. - -[ddn-install-docs]: https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp#install-new-exascaler-cloud-clients -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## Support - -EXAScaler Cloud includes self-help support with access to publicly available -documents and videos. Premium support includes 24x7x365 access to DDN's experts, -along with support community access, automated notifications of updates and -other premium support features. For more information, visit -[EXAscaler Cloud on GCP][exa-gcp]. - -[exa-gcp]: https://console.cloud.google.com/marketplace/product/ddnstorage/exascaler-cloud - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [ddn\_exascaler](#module\_ddn\_exascaler) | github.com/DDNStorage/exascaler-cloud-terraform//gcp | a3355d50deebe45c0556b45bd599059b7c06988d | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [boot](#input\_boot) | Boot disk properties | 
object({
    disk_type   = string
    auto_delete = bool
    script_url  = string
  })
| 
{
  "auto_delete": true,
  "disk_type": "pd-standard",
  "script_url": null
}
| no | -| [cls](#input\_cls) | Compute client properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 0,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-2",
  "public_ip": true
}
| no | -| [clt](#input\_clt) | Compute client target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 0,
  "disk_size": 256,
  "disk_type": "pd-standard"
}
| no | -| [fsname](#input\_fsname) | EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long | `string` | `"exacloud"` | no | -| [image](#input\_image) | DEPRECATED: Source image properties | `any` | `null` | no | -| [instance\_image](#input\_instance\_image) | Source image properties

Expected Fields:
name: Unavailable with this module.
family: The image family to use.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp",
  "project": "ddn-public"
}
| no | -| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. Key-value pairs. | `map(string)` | `{}` | no | -| [local\_mount](#input\_local\_mount) | Mountpoint (at the client instances) for this EXAScaler system | `string` | `"/shared"` | no | -| [mds](#input\_mds) | Metadata server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | -| [mdt](#input\_mdt) | Metadata target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | -| [mgs](#input\_mgs) | Management server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | -| [mgt](#input\_mgt) | Management target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | -| [mnt](#input\_mnt) | Monitoring target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | -| [network\_properties](#input\_network\_properties) | Network options. 'network\_self\_link' or 'network\_properties' must be provided. | 
object({
    routing = string
    tier    = string
    id      = string
    auto    = bool
    mtu     = number
    new     = bool
    nat     = bool
  })
| `null` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self-link of the VPC network to where the system is connected. Ignored if 'network\_properties' is provided. 'network\_self\_link' or 'network\_properties' must be provided. | `string` | `null` | no | -| [oss](#input\_oss) | Object Storage server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 3,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-16",
  "public_ip": true
}
| no | -| [ost](#input\_ost) | Object Storage target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | -| [prefix](#input\_prefix) | EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud') | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Compute Platform project that will host the EXAScaler filesystem | `string` | n/a | yes | -| [security](#input\_security) | Security options | 
object({
    admin              = string
    public_key         = string
    block_project_keys = bool
    enable_os_login    = bool
    enable_local       = bool
    enable_ssh         = bool
    enable_http        = bool
    ssh_source_ranges  = list(string)
    http_source_ranges = list(string)
  })
| 
{
  "admin": "stack",
  "block_project_keys": false,
  "enable_http": false,
  "enable_local": false,
  "enable_os_login": true,
  "enable_ssh": false,
  "http_source_ranges": [
    "0.0.0.0/0"
  ],
  "public_key": null,
  "ssh_source_ranges": [
    "0.0.0.0/0"
  ]
}
| no | -| [service\_account](#input\_service\_account) | Service account name used by deploy application | 
object({
    new   = bool
    email = string
  })
| 
{
  "email": null,
  "new": false
}
| no | -| [subnetwork\_address](#input\_subnetwork\_address) | The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork\_properties' is provided. | `string` | `null` | no | -| [subnetwork\_properties](#input\_subnetwork\_properties) | Subnetwork properties. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | 
object({
    address = string
    private = bool
    id      = string
    new     = bool
  })
| `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork\_properties' is provided. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | `string` | `null` | no | -| [waiter](#input\_waiter) | Waiter to check progress and result for deployment. | `string` | `null` | no | -| [zone](#input\_zone) | Compute Platform zone where the servers will be located | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_config\_script](#output\_client\_config\_script) | Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler. | -| [http\_console](#output\_http\_console) | HTTP address to access the system web console. | -| [install\_ddn\_lustre\_client\_runner](#output\_install\_ddn\_lustre\_client\_runner) | Runner that encapsulates the `client_config_script` output on this module. | -| [mount\_command](#output\_mount\_command) | Command to mount the file system. `client_config_script` must be run first. | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the DDN EXAScaler Lustre file system | -| [network\_storage](#output\_network\_storage) | Describes a EXAScaler system to be mounted by other systems. | -| [private\_addresses](#output\_private\_addresses) | Private IP addresses for all instances. | -| [ssh\_console](#output\_ssh\_console) | Instructions to ssh into the instances. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf deleted file mode 100644 index 6a2fc4b702..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ddn-exascaler", ghpc_role = "file-system" }) -} - -locals { - - network_id = var.network_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.network_self_link)[0] : null - named_net = { - routing = "REGIONAL" - tier = "STANDARD" - id = local.network_id - auto = false - mtu = 1500 - new = false - nat = false - } - - subnetwork_id = var.subnetwork_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.subnetwork_self_link)[0] : null - named_subnet = { - address = var.subnetwork_address - private = true - id = local.subnetwork_id - new = false - } -} - -module "ddn_exascaler" { - source = "github.com/DDNStorage/exascaler-cloud-terraform//gcp?ref=a3355d50deebe45c0556b45bd599059b7c06988d" - fsname = var.fsname - zone = var.zone - project = var.project_id - prefix = var.prefix - labels = local.labels - security = var.security - service_account = var.service_account - waiter = var.waiter - network = var.network_properties == null ? local.named_net : var.network_properties - subnetwork = var.subnetwork_properties == null ? local.named_subnet : var.subnetwork_properties - boot = var.boot - image = var.instance_image - mgs = var.mgs - mgt = var.mgt - mnt = var.mnt - mds = var.mds - mdt = var.mdt - oss = var.oss - ost = var.ost - cls = var.cls - clt = var.clt -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml deleted file mode 100644 index b995bd4358..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - deploymentmanager.googleapis.com - - iam.googleapis.com - - runtimeconfig.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf deleted file mode 100644 index 2e9ae732ae..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -output "private_addresses" { - description = "Private IP addresses for all instances." - value = module.ddn_exascaler.private_addresses -} - -output "ssh_console" { - description = "Instructions to ssh into the instances." - value = module.ddn_exascaler.ssh_console -} - -output "client_config_script" { - description = "Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler." - value = module.ddn_exascaler.client_config -} - -output "install_ddn_lustre_client_runner" { - description = "Runner that encapsulates the `client_config_script` output on this module." - value = local.client_install_runner -} - -locals { - client_install_runner = { - "type" = "shell" - "content" = module.ddn_exascaler.client_config - "destination" = "install_ddn_lustre_client.sh" - } - - # Mount command provided by DDN does not support custom local mount - split_mount_cmd = split(" ", module.ddn_exascaler.mount_command) - split_mount_cmd_wo_mountpoint = slice(local.split_mount_cmd, 0, length(local.split_mount_cmd) - 1) - mount_cmd = "${join(" ", local.split_mount_cmd_wo_mountpoint)} ${var.local_mount}" - mount_cmd_w_mkdir = "mkdir -p ${var.local_mount} && ${local.mount_cmd}" - mount_runner = { - "type" = "shell" - "content" = local.mount_cmd_w_mkdir - "destination" = "mount-ddn-lustre.sh" - } -} - -output "mount_command" { - description = "Command to mount the file system. `client_config_script` must be run first." - value = local.mount_cmd_w_mkdir -} - -output "mount_runner" { - description = "Runner to mount the DDN EXAScaler Lustre file system" - value = local.mount_runner -} - -output "http_console" { - description = "HTTP address to access the system web console." - value = module.ddn_exascaler.http_console -} - -output "network_storage" { - description = "Describes a EXAScaler system to be mounted by other systems." - value = { - server_ip = split(":", split(" ", module.ddn_exascaler.mount_command)[3])[0] - remote_mount = length(regexall("^/.*", var.fsname)) > 0 ? var.fsname : format("/%s", var.fsname) - local_mount = var.local_mount != null ? var.local_mount : format("/mnt/%s", var.fsname) - fs_type = "lustre" - mount_options = "" - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } - depends_on = [ - module.ddn_exascaler - ] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf deleted file mode 100644 index 68bcc8a8ba..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf +++ /dev/null @@ -1,502 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -# EXAScaler filesystem name -# only alphanumeric characters are allowed, -# and the value must be 1-8 characters long -variable "fsname" { - description = "EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long" - type = string - default = "exacloud" -} - -# Project ID to manage resources -# https://cloud.google.com/resource-manager/docs/creating-managing-projects -variable "project_id" { - description = "Compute Platform project that will host the EXAScaler filesystem" - type = string -} - -# Zone name to manage resources -# https://cloud.google.com/compute/docs/regions-zones -variable "zone" { - description = "Compute Platform zone where the servers will be located" - type = string -} - -# Service account name used by deploy application -# https://cloud.google.com/iam/docs/service-accounts -# new: create a new custom service account or use an existing one: true or false -# email: existing service account email address, will be using if new is false -# set email = null to use the default compute service account -variable "service_account" { - description = "Service account name used by deploy application" - type = object({ - new = bool - email = string - }) - default = { - new = false - email = null - } -} - -# Waiter to check progress and result for deployment. -# To use Google Deployment Manager: -# waiter = "deploymentmanager" -# To use generic Google Cloud SDK command line: -# waiter = "sdk" -# If you don’t want to wait until the deployment is complete: -# waiter = null -# https://cloud.google.com/deployment-manager/runtime-configurator/creating-a-waiter -variable "waiter" { - description = "Waiter to check progress and result for deployment." - type = string - default = null -} - -# Security options -# admin: optional user name for remote SSH access -# Set admin = null to disable creation admin user -# public_key: path to the SSH public key on the local host -# Set public_key = null to disable creation admin user -# block_project_keys: true or false -# Block project-wide public SSH keys if you want to restrict -# deployment to only user with deployment-level public SSH key. -# https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys -# enable_os_login: true or false -# Enable or disable OS Login feature. -# Please note, enabling this option disables other security options: -# admin, public_key and block_project_keys. -# https://cloud.google.com/compute/docs/instances/managing-instance-access#enable_oslogin -# enable_local: true or false, enable or disable firewall rules for local access -# enable_ssh: true or false, enable or disable remote SSH access -# ssh_source_ranges: source IP ranges for remote SSH access in CIDR notation -# enable_http: true or false, enable or disable remote HTTP access -# http_source_ranges: source IP ranges for remote HTTP access in CIDR notation -variable "security" { - description = "Security options" - type = object({ - admin = string - public_key = string - block_project_keys = bool - enable_os_login = bool - enable_local = bool - enable_ssh = bool - enable_http = bool - ssh_source_ranges = list(string) - http_source_ranges = list(string) - }) - - default = { - admin = "stack" - public_key = null - block_project_keys = false - enable_os_login = true - enable_local = false - enable_ssh = false - enable_http = false - ssh_source_ranges = [ - "0.0.0.0/0" - ] - http_source_ranges = [ - "0.0.0.0/0" - ] - } -} - -variable "network_self_link" { - description = "The self-link of the VPC network to where the system is connected. Ignored if 'network_properties' is provided. 'network_self_link' or 'network_properties' must be provided." - type = string - default = null -} - -# Network properties -# https://cloud.google.com/vpc/docs/vpc -# routing: network-wide routing mode: REGIONAL or GLOBAL -# tier: networking tier for VM interfaces: STANDARD or PREMIUM -# id: existing network id, will be using if new is false -# auto: create subnets in each region automatically: false or true -# mtu: maximum transmission unit in bytes: 1460 - 1500 -# new: create a new network or use an existing one: true or false -# nat: allow instances without external IP to communicate with the outside world: true or false -variable "network_properties" { - description = "Network options. 'network_self_link' or 'network_properties' must be provided." - type = object({ - routing = string - tier = string - id = string - auto = bool - mtu = number - new = bool - nat = bool - }) - - default = null -} - -variable "subnetwork_self_link" { - description = "The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork_properties' is provided. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." - type = string - default = null -} - -variable "subnetwork_address" { - description = "The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork_properties' is provided." - type = string - default = null -} - -# Subnetwork properties -# https://cloud.google.com/vpc/docs/vpc -# address: IP range of internal addresses for a new subnetwork -# private: when enabled VMs in this subnetwork without external -# IP addresses can access Google APIs and services by using -# Private Google Access: true or false -# https://cloud.google.com/vpc/docs/private-access-options -# id: existing subnetwork id, will be using if new is false -# new: create a new subnetwork or use an existing one: true or false -variable "subnetwork_properties" { - description = "Subnetwork properties. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." - type = object({ - address = string - private = bool - id = string - new = bool - }) - default = null -} -# Boot disk properties -# disk_type: pd-standard, pd-ssd or pd-balanced -# auto_delete: true or false -# whether the disk will be auto-deleted when the instance is deleted -variable "boot" { - description = "Boot disk properties" - type = object({ - disk_type = string - auto_delete = bool - script_url = string - }) - default = { - disk_type = "pd-standard" - auto_delete = true - script_url = null - } -} - -# Source image properties -# project: project name -# family: image family name -# name: !!DEPRECATED!! - image name -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: Source image properties" - type = any - # Omitting type checking so validation can provide more useful error message - # type = object({ - # project = string - # family = string - # }) - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - Source image properties - - Expected Fields: - name: Unavailable with this module. - family: The image family to use. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "ddn-public" - family = "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp" - } - - validation { - condition = !can(coalesce(var.instance_image.name)) - error_message = "In var.instance_image, the \"name\" field is not used, please use the \"family\" setting." - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, the \"family\" field must be a string set to the image family." - } -} - -# Management server properties -# node_type: type of management server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of management servers -variable "mgs" { - description = "Management server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-32" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 1 - } -} - -# Management target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of management target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of management target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of management target in GB (scratch disk size must be exactly 375) -# disk_count: number of management targets -# disk_raid: create striped management target, true or false -variable "mgt" { - description = "Management target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 128 - disk_count = 1 - disk_raid = false - } -} - - -# Monitoring target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of monitoring target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of monitoring target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of monitoring target in GB (scratch disk size must be exactly 375) -# disk_count: number of monitoring targets -# disk_raid: create striped monitoring target, true or false -variable "mnt" { - description = "Monitoring target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 128 - disk_count = 1 - disk_raid = false - } -} - -# Metadata server properties -# node_type: type of metadata server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of metadata servers -variable "mds" { - description = "Metadata server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-32" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 1 - } -} - -# Metadata target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of metadata target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of metadata target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of metadata target in GB (scratch disk size must be exactly 375) -# disk_count: number of metadata targets -# disk_raid: create striped metadata target, true or false -variable "mdt" { - description = "Metadata target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-ssd" - disk_size = 3500 - disk_count = 1 - disk_raid = false - } -} - -# Object Storage server properties -# node_type: type of storage server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of storage servers -variable "oss" { - description = "Object Storage server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-16" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 3 - } -} - -# Object Storage target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of storage target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of storage target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of storage target in GB (scratch disk size must be exactly 375) -# disk_count: number of storage targets -# disk_raid: create striped storage target, true or false -variable "ost" { - description = "Object Storage target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-ssd" - disk_size = 3500 - disk_count = 1 - disk_raid = false - } -} - -# Compute client properties -# node_type: type of compute client -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of compute clients -variable "cls" { - description = "Compute client properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-2" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 0 - } -} -# Compute client target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of compute target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of compute target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of compute target in GB (scratch disk size must be exactly 375) -# disk_count: number of compute targets -variable "clt" { - description = "Compute client target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 256 - disk_count = 0 - } -} -variable "local_mount" { - description = "Mountpoint (at the client instances) for this EXAScaler system" - type = string - default = "/shared" -} - -variable "prefix" { - description = "EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud')" - type = string - default = null -} - -variable "labels" { - description = "Labels to add to EXAScaler Cloud deployment. Key-value pairs." - type = map(string) - default = {} -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf deleted file mode 100644 index 2981b4dd75..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/Intel-DAOS/README.md b/deletion-test/cluster/modules/embedded/community/modules/file-system/Intel-DAOS/README.md deleted file mode 100644 index 04db0acb8c..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/Intel-DAOS/README.md +++ /dev/null @@ -1 +0,0 @@ -> **_NOTE:_** Cluster Toolkit is dropping support for the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository. The DAOS example blueprints (`hpc-slurm-daos.yaml` and `pfs-daos.yaml`) have been removed from the Cluster Toolkit. We recommend migrating to the first-party [Parallelstore](../../../../modules/file-system/parallelstore/) module for similar functionality. To help with this transition, see the Parallelstore example blueprints ([pfs-parallelstore.yaml](../../../../examples/pfs-parallelstore.yaml) and [ps-slurm.yaml](../../../../examples/ps-slurm.yaml)). If the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository is necessary, we recommend using the last Cluster Toolkit [v1.41.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/releases/tag/v1.41.0). diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/README.md b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/README.md deleted file mode 100644 index 66aaaa46af..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/README.md +++ /dev/null @@ -1,152 +0,0 @@ -## Description - -This module creates a Network File Sharing (NFS) file system based on a VM -instance and [compute disk][disk]. This file system can share directories and -files with other clients over a network. `nfs-server` can be used by -[vm-instance](../../../../modules/compute/vm-instance/README.md) and SchedMD -community modules that create compute VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -If you are using Hyperdisk storage, check the possible disk size, IOPS, and throughput values for each disk type in the [Hyperdisk limits documentation](https://cloud.google.com/compute/docs/disks/hyperdisks#limits-disk). - -> **_WARNING:_** This module has only been tested against the HPC centos7 OS -> disk image (the default). Using other images may work, but have not been -> verified. - -[disk]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk - -### Example - -```yaml -- id: homefs - source: community/modules/file-system/nfs-server - use: [network1] -``` - -This creates a NFS on a virtual machine which allow other VMs to mount the -volume as an external file system. - -> **_NOTE:_** All disks are destroyed along with the instance, during a `gcluster destroy`/`terraform destroy` event. However, you can setup data retention with `create_boot_snapshot_before_destroy` (boot disk) and `create_snapshot_before_destroy` (data disk). - -## Mounting - -To mount the NFS Server you must first ensure that the NFS client has been -installed the and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -See the [hpc-centos-ss.yaml] test config for an example of using this module -with a `vm-instance` module. - -If mounting is not automatically handled as described above, the `nfs-server` -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - - id: nfs - source: community/modules/file-system/nfs-server - use: [network1] - settings: {local_mounts: [/mnt1]} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(nfs.install_nfs_client_runner) - - $(nfs.mount_runner) - -``` - -[hpc-centos-ss.yaml]: ../../../../tools/validate_configs/test_configs/hpc-centos-ss.yaml -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 6.14 | -| [null](#requirement\_null) | >= 3.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.14 | -| [null](#provider\_null) | >= 3.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_disk.attached_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_instance.compute_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource | -| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_default_service_account.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_default_service_account) | data source | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [auto\_delete\_disk](#input\_auto\_delete\_disk) | DEPRECATED: Whether or not the NFS disk should be auto-deleted | `string` | `null` | no | -| [boot\_disk\_size](#input\_boot\_disk\_size) | Storage size in GB for the boot disk | `number` | `null` | no | -| [boot\_disk\_type](#input\_boot\_disk\_type) | Storage type for the boot disk | `string` | `null` | no | -| [create\_boot\_snapshot\_before\_destroy](#input\_create\_boot\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the boot disk | `bool` | `false` | no | -| [create\_snapshot\_before\_destroy](#input\_create\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the NFS data disk | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the NFS instance if no name is specified. | `string` | n/a | yes | -| [disk\_size](#input\_disk\_size) | Storage size in GB for the NFS data disk | `number` | `"100"` | no | -| [image](#input\_image) | DEPRECATED: The VM image used by the NFS server | `string` | `null` | no | -| [instance\_image](#input\_instance\_image) | The VM image used by the NFS server.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to the NFS instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mounts](#input\_local\_mounts) | Mountpoint for this NFS compute instance | `list(string)` | 
[
  "/data"
]
| no | -| [machine\_type](#input\_machine\_type) | Type of the VM instance to use | `string` | `"n2d-standard-2"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the NFS VM. | `string` | `"default"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [provisioned\_iops](#input\_provisioned\_iops) | Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput | `number` | `null` | no | -| [provisioned\_throughput](#input\_provisioned\_throughput) | Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme | `number` | `null` | no | -| [scopes](#input\_scopes) | Scopes to apply to the controller | `list(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [service\_account](#input\_service\_account) | Service Account for the NFS server | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the NFS VM. | `string` | `null` | no | -| [type](#input\_type) | Storage type for the NFS data disk | `string` | `"pd-ssd"` | no | -| [zone](#input\_zone) | The zone name where the NFS instance located in. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | -| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | -| [network\_storage](#output\_network\_storage) | export of all desired folder directories | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/main.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/main.tf deleted file mode 100644 index a00d2681ba..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/main.tf +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "nfs-server", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - server_ip = google_compute_instance.compute_instance.network_interface[0].network_ip - fs_type = "nfs" - mount_options = "defaults,hard,intr" - install_nfs_client_runners = [for mount in var.local_mounts : - { - "type" = "shell" - "source" = "${path.module}/scripts/install-nfs-client.sh" - "destination" = "install-nfs${replace(mount, "/", "_")}.sh" - } - ] - mount_runners = [for mount in var.local_mounts : - { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"/exports${mount}\" \"${mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(mount, "/", "_")}.sh" - } - ] - ansible_mount_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/mount.yaml" - "destination" = "mount.yaml" - } -} - -data "google_compute_default_service_account" "default" {} - -resource "google_compute_disk" "attached_disk" { - project = var.project_id - name = "${local.name}-nfs-instance-disk" - size = var.disk_size - type = var.type - zone = var.zone - labels = local.labels - provisioned_iops = var.provisioned_iops - provisioned_throughput = var.provisioned_throughput - create_snapshot_before_destroy = var.create_snapshot_before_destroy -} - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project -} - -resource "null_resource" "image" { - triggers = { - name = try(var.instance_image.name, null), - family = try(var.instance_image.family, null), - project = var.instance_image.project - } -} - -resource "google_compute_disk" "boot_disk" { - project = var.project_id - - name = "${local.name}-boot-disk" - size = var.boot_disk_size - type = var.boot_disk_type - image = data.google_compute_image.compute_image.self_link - labels = local.labels - zone = var.zone - create_snapshot_before_destroy = var.create_boot_snapshot_before_destroy - - lifecycle { - replace_triggered_by = [null_resource.image] - ignore_changes = [ - image - ] - } -} - -resource "google_compute_instance" "compute_instance" { - project = var.project_id - name = "${local.name}-nfs-instance" - zone = var.zone - machine_type = var.machine_type - - boot_disk { - auto_delete = false - source = google_compute_disk.boot_disk.self_link - device_name = google_compute_disk.boot_disk.name - } - - attached_disk { - source = google_compute_disk.attached_disk.id - device_name = "attached_disk" - } - - network_interface { - network = var.network_self_link - subnetwork = var.subnetwork_self_link - } - - service_account { - email = var.service_account == null ? data.google_compute_default_service_account.default.email : var.service_account - scopes = var.scopes - } - - metadata = var.metadata - metadata_startup_script = templatefile("${path.module}/scripts/install-nfs-server.sh.tpl", { local_mounts = var.local_mounts }) - - labels = local.labels -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/outputs.tf deleted file mode 100644 index e23b94e2b2..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/outputs.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -# render the content for each folder -output "network_storage" { - description = "export of all desired folder directories" - value = [for i, mount in var.local_mounts : { - remote_mount = "/exports${mount}" - local_mount = mount - fs_type = local.fs_type - mount_options = local.mount_options - server_ip = local.server_ip - client_install_runner = local.install_nfs_client_runners[i] - mount_runner = local.mount_runners[i] - } - ] -} - -output "install_nfs_client" { - description = "Script for installing NFS client" - value = file("${path.module}/scripts/install-nfs-client.sh") -} - -output "install_nfs_client_runner" { - description = "Runner to install NFS client using the startup-script module" - value = local.install_nfs_client_runners[0] -} - -output "mount_runner" { - description = <<-EOT - Runner to mount the file-system using an ansible playbook. The startup-script - module will automatically handle installation of ansible. - - id: example-startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(your-fs-id.mount_runner) - ... - EOT - value = local.ansible_mount_runner -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl deleted file mode 100644 index 1b06a5f032..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -ex - -if [ ! -d "/exports" ]; then # first load, format and mount the disk - # See https://cloud.google.com/compute/docs/disks/add-persistent-disk - uuid=$(uuidgen) - mkfs.ext4 -F -m 0 -U "$uuid" -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/disk/by-id/google-attached_disk - - mkdir /exports - echo "UUID=$uuid /exports ext4 discard,defaults 0 0" >> /etc/fstab - mount --target /exports/ - - %{ for mount in local_mounts ~} - mkdir -p /exports${mount} - chmod 755 /exports${mount} - echo '/exports${mount} *(rw,sync,no_root_squash)' >> "/etc/exports" - %{ endfor ~} -fi - -systemctl start nfs-server rpcbind -systemctl enable nfs-server -exportfs -r diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml deleted file mode 100644 index f7fbe58d5e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Mounts the file systems specified in the metadata network_storage key - hosts: localhost - become: true - vars: - meta_key: "network_storage" - url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" - tasks: - - name: Read metadata network_storage information - ansible.builtin.uri: - url: "{{ url }}/{{ meta_key }}" - method: GET - headers: - Metadata-Flavor: "Google" - register: storage - - name: Mount file systems - ansible.posix.mount: - src: "{{ item.server_ip }}:/{{ item.remote_mount }}" - path: "{{ item.local_mount }}" - opts: "{{ item.mount_options }}" - boot: true - fstype: "{{ item.fs_type }}" - state: "mounted" - loop: "{{ storage.json }}" diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/variables.tf deleted file mode 100644 index 9a58da641e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/variables.tf +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the NFS instance if no name is specified." - type = string -} - -variable "name" { - description = "The resource name of the instance." - type = string - default = null -} - -variable "zone" { - description = "The zone name where the NFS instance located in." - type = string -} - -variable "boot_disk_size" { - description = "Storage size in GB for the boot disk" - type = number - default = null -} - -variable "boot_disk_type" { - description = "Storage type for the boot disk" - type = string - default = null -} - -variable "create_boot_snapshot_before_destroy" { - description = "Whether to create a snapshot before destroying the boot disk" - type = bool - default = false -} - -variable "disk_size" { - description = "Storage size in GB for the NFS data disk" - type = number - default = "100" -} - -variable "type" { - description = "Storage type for the NFS data disk" - type = string - default = "pd-ssd" -} - -variable "create_snapshot_before_destroy" { - description = "Whether to create a snapshot before destroying the NFS data disk" - type = bool - default = false -} - -variable "provisioned_iops" { - description = "Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput" - type = number - default = null -} - -variable "provisioned_throughput" { - description = "Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme" - type = number - default = null -} - -# Deprecated, replaced by instance_image -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: The VM image used by the NFS server" - type = string - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - The VM image used by the NFS server. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -# Deprecated, replaced by create_snapshot_before_destroy and create_boot_snapshot_before_destroy -# tflint-ignore: terraform_unused_declarations -variable "auto_delete_disk" { - description = "DEPRECATED: Whether or not the NFS disk should be auto-deleted" - type = string - default = null - - validation { - condition = var.auto_delete_disk == null - error_message = "The 'var.auto_delete_disk' setting is broken in Cluster Toolkit versions >1.25.0 and deprecated in versions >1.48.0, please use 'var.create_snapshot_before_destroy' and 'var.create_boot_snapshot_before_destroy' instead." - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the NFS VM." - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the NFS VM." - type = string - default = null -} - -variable "machine_type" { - description = "Type of the VM instance to use" - type = string - default = "n2d-standard-2" -} - -variable "labels" { - description = "Labels to add to the NFS instance. Key-value pairs." - type = map(string) -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "service_account" { - description = "Service Account for the NFS server" - type = string - default = null -} - -variable "scopes" { - description = "Scopes to apply to the controller" - type = list(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "local_mounts" { - description = "Mountpoint for this NFS compute instance" - type = list(string) - default = ["/data"] - - validation { - condition = alltrue([ - for m in var.local_mounts : substr(m, 0, 1) == "/" - ]) - error_message = "Local mountpoints have to start with '/'." - } - validation { - condition = length(var.local_mounts) > 0 - error_message = "At least one local mount must be specified in var.local_mounts." - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/versions.tf deleted file mode 100644 index 63443806b8..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/nfs-server/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.14" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/sycomp-scale/README.md b/deletion-test/cluster/modules/embedded/community/modules/file-system/sycomp-scale/README.md deleted file mode 100644 index 79ff12bc18..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/sycomp-scale/README.md +++ /dev/null @@ -1,35 +0,0 @@ -## Description - -This document provides information on how to deploy an instance of [Sycomp Intelligent Data Storage Platform](https://sycomp.com/solution/hpc/storage/) on Google Cloud Platform ([GCP](https://cloud.google.com/)) using the Google Cluster Toolkit. - -> **_NOTE:_** -> Sycomp Storage on GCP does not require an HPC Toolkit wrapper. -> Terraform modules are sourced directly from GitLab. - -Terraform modules for Sycomp Intelligent Data Storage Platform are downloaded on deployment using the Google Cloud Toolkit. - -The Terraform module parameters are documented in the `README.md` files in the respective module directories of the source GitLab repository. The main modules are: - -- `sycomp-scale` -- `sycomp-scale-expansion` - -## Examples - -The community examples folder (community/examples/sycomp/) contains four example blueprints that you can use to deploy or expand a Sycomp Storage cluster. - -- [community/examples/sycomp/sycomp-storage.yaml][sycomp-storage-yaml] - - Blueprint for deploying a Sycomp Storage cluster consisting of 3 storage servers. - -- [community/examples/sycomp/sycomp-storage-expansion.yaml][sycomp-storage-expansion-yaml] - - Blueprint for expanding the above created cluster from 3 to 4 storage servers. - -- [community/examples/sycomp/sycomp-storage-ece.yaml][sycomp-storage-ece-yaml] - - Blueprint for deploying a Sycomp Storage cluster consisting of 7 storage servers with ECE (Erasure Code Edition) software RAID. - -- [community/examples/sycomp/sycomp-storage-slurm.yaml][sycomp-storage-slurm-yaml] - - Blueprint for deploying a Slurm cluster and Sycomp Storage cluster with 3 servers. The Slurm compute nodes are configured as NFS clients and have the ability to use the Sycomp Storage filesystem. - -[sycomp-storage-yaml]: ../../../examples/sycomp/sycomp-storage.yaml -[sycomp-storage-expansion-yaml]: ../../../examples/sycomp/sycomp-storage-expansion.yaml -[sycomp-storage-ece-yaml]: ../../../examples/sycomp/sycomp-storage-ece.yaml -[sycomp-storage-slurm-yaml]: ../../../examples/sycomp/sycomp-storage-slurm.yaml diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/README.md b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/README.md deleted file mode 100644 index 0e2a936167..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/README.md +++ /dev/null @@ -1,182 +0,0 @@ -## Description - -This module provides scripts for client installation and mounting [WEKA] -filesystems. Client supports both UDP and DPDK modes and allows customization of -mount parameters using Compute VM instance metadata. - -For deploying Weka cluster please consult [WEKA installation on GCP]. - -[WEKA]: https://www.weka.io/ -[WEKA installation on GCP]: https://docs.weka.io/planning-and-installation/weka-installation-on-gcp - -## Prerequisites - -* up and running Weka cluster -* running on a [supported OS](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#operating-system) -* [open firewall](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#required-ports) - between WEKA backend servers and clients -* VPC peering configuration: - * if clients share VPCs created for WEKA cluster, no additional configuration - is necessary - * if dedicated VPCs are in use for clients, then WEKA VPCs needs to be peered - with VPCs that are used as: - * primary interface on client - * interfaces dedicated for DPDK client - * if dedicated VPCs are in use for clients, then those VPCs needs to be peered - with each other - -## Mounting -This example creates mount scripts that will mount `default` filesystem from -`10.0.0.3` WEKA backend: - -```yaml - - id: wekafs - source: community/modules/file-system/weka-client - settings: - local_mount: /scratch - server_ip: 10.0.0.3 - remote_mount: default - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: $(wekafs.runners) -``` - -If you need to add mount script along other runners, remember to add all 4 -runners provided by this script as shown in this example: - -```yaml - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(wekafs.client_install_runner) - - $(wekafs.mount_runner) - - type: shell - content: | - #!/bin/bash - - echo Sample - destination: sample-script.sh -``` - -To use the client within Slurm partition, with DPDK, remember to set additional -networks, and configure metadata. In this example, all four additional interfaces -are dedicated to WEKA DPDK - -```yaml - - id: c2_60_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - - mount-at-startup # as defined in previous examples - settings: - bandwidth_tier: virtio_enabled # Weka requires VirtIO, from WEKA 4.4.1, DPDK is also supported on gVNIC - additional_networks: - - subnetwork: weka-client-1 - nic_type: VIRTIO_NET - - subnetwork: weka-client-2 - nic_type: VIRTIO_NET - - subnetwork: weka-client-3 - nic_type: VIRTIO_NET - - subnetwork: weka-client-4 - nic_type: VIRTIO_NET - machine_type: c2-standard-60 - metadata: - weka-data_interfaces: 1,2,3,4 # allocate interfaces 1, 2, 3 and 4 to DPDK - weka-mode: dpdk - weka-options: num_cores=4,dpdk_base_memory_mb=16 - node_conf: - # From https://docs.weka.io/planning-and-installation/bare-metal/planning-a-weka-system-installation - # do not set RealMem as this is set automatically by Cluster Toolkit - CoreSpecCount: 4 - MemSpecLimit: 5120 -``` - -Due to the fact, that client installation takes ~6-7 minutes, if you use WEKA together with Slurm and do not bundle -client in the instance image, you may need to increase the timeout for startups scripts. - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - settings: - compute_startup_scripts_timeout: 600 - login_startup_scripts_timeout: 600 - ... - - id: compute_partition - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - settings: - resume_timeout: 600 - ... -``` - -## Supported VM metadata options -Client scripts do support following metadata keys: -* `weka-mode` - one of `udp` or `dpdk`. Defaults to `udp`. Sets client mode. -* `weka-data_interfaces` - comma separated list of interface identifiers, - specifying which interfaces are dedicated for data plane. Set to `1` to - dedicate second interface of instance for WEKA DPDK. Set to `2,5` to dedicate - third and sixth interface of instance for WEKA DPDK. -* `weka-mgmt_interface` - identifier of management interface, defaults to `0`, - which means to use primary interface as management interface. -* `weka-options` - additional [mount command options](https://docs.weka.io/weka-filesystems-and-object-stores/mounting-filesystems#mount-command-options) - to pass to `mount` command - -## Adding client to the OS image -To save time during the mount command install and precompile DPDK drivers in the -OS image. Following scripts compiles DPDK driver for currently running kernel. - -```shell -#!/bin/bash - -set -e -o pipefail - -echo Downloading and installing Weka client -curl --max-time 10 "{{ weka backend endpoint }}/dist/v1/install" | sh -WEKA_VERSION=$(weka -v | sed -e 's/^[^0-9]*//') -echo Installing Weka version: ${WEKA_VERSION} -weka version get "${WEKA_VERSION}" -weka version set "${WEKA_VERSION}" -# run setup for the second time, if it fails for the first time -weka local setup weka || weka local setup weka -weka version prepare "${WEKA_VERSION}" -weka local stop -weka local rm -f --all -``` - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | -| [mount\_options](#input\_mount\_options) | Mount options for filesystem shared by all clients. | `string` | `""` | no | -| [remote\_mount](#input\_remote\_mount) | Weka filesystem name. | `string` | n/a | yes | -| [server\_ip](#input\_server\_ip) | Weka backend IP address used for bootstrapping. | `string` | `""` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_install\_runner](#output\_client\_install\_runner) | Ansible runner that performs client installation needed to use file system. | -| [mount\_runner](#output\_mount\_runner) | Ansible runner that mounts the file system. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/metadata.yaml deleted file mode 100644 index 419bc3fe46..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/outputs.tf deleted file mode 100644 index 0bd9098d80..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/outputs.tf +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - template_args = { - local_mount = var.local_mount - mount_options = var.mount_options == "" ? "" : "-o ${var.mount_options}" - remote_mount = var.remote_mount - server_ip = var.server_ip - service_name = "weka-mount${replace(var.local_mount, "/", "-")}" - } - mount_script = templatefile("${path.module}/templates/mount-weka.sh.tftpl", local.template_args) - - mount_runner_ansible = { - type = "ansible-local" - content = templatefile( - "${path.module}/templates/mount-weka.yaml.tftpl", - merge( - local.template_args, - { mount_weka_script = local.mount_script } - ) - ) - destination = "mount_filesystem${replace(var.local_mount, "/", "_")}.yaml" - } - - client_install_runner = { - type = "ansible-local" - content = templatefile("${path.module}/templates/install-weka-client.yaml.tftpl", local.template_args) - destination = "install_filesystem${replace(var.local_mount, "/", "_")}.yaml" - } -} - -# currently WEKA mounts are not compatible with network_storage logic, as WEKA volumes needs to be mounted by -# systemd script and not /etc/fstab entry, as the mount command needs to have network configuration which may change -# between restarts -# -#output "network_storage" { -# description = "Describes a remote network storage to be mounted by fs-tab." -# value = { -# server_ip = var.server_ip -# remote_mount = var.remote_mount -# local_mount = var.local_mount -# fs_type = var.fs_type -# mount_options = var.mount_options -# client_install_runner = local.client_install_runner -# mount_runner = local.mount_runner -# } -#} -# -output "client_install_runner" { - description = "Ansible runner that performs client installation needed to use file system." - value = local.client_install_runner -} - -output "mount_runner" { - description = "Ansible runner that mounts the file system." - value = local.mount_runner_ansible -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl deleted file mode 100644 index ddc3acdb5d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Mounts the file systems specified in the metadata network_storage key - hosts: localhost - become: true - vars: - meta_key: "network_storage" - url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" - tasks: - - name: Check if weka is installed - ansible.builtin.stat: - path: /usr/bin/weka - register: weka_binary - - - name: Create temporary location for installation script - ansible.builtin.tempfile: - state: file - register: - install_script - when: not weka_binary.stat.exists - - - name: Download WEKA client - ansible.builtin.get_url: - url: http://${server_ip}:14000/dist/v1/install - dest: "{{ install_script.path }}" - mode: "700" - when: not weka_binary.stat.exists - - - name: Run WEKA installation script - ansible.builtin.shell: - cmd: "{{ install_script.path }}" - when: not weka_binary.stat.exists - register: weka_install_result - changed_when: weka_install_result.rc == 0 - - - name: Read metadata network_storage information - ansible.builtin.uri: - url: "{{ url }}/weka-version" - method: GET - headers: - Metadata-Flavor: "Google" - status_code: - - 200 - - 404 - register: get_weka_version - - - name: Set WEKA version from metadata server - ansible.builtin.set_fact: - weka_version: "{{ get_weka_version.body }}" - when: get_weka_version.status == 200 - - - name: Get version of WEKA installation client - ansible.builtin.shell: - cmd: weka -v | sed -e 's/^[^0-9.]*\([0-9.]*\)[^0-9.]*$/\1/' - register: get_weka_client_version - changed_when: get_weka_client_version.rc == 0 - - - name: Set WEKA version from WEKA installation client - ansible.builtin.set_fact: - weka_version: "{{ get_weka_client_version.stdout }}" - when: get_weka_version.status == 404 - - - name: Download user-defined WEKA version - ansible.builtin.shell: - cmd: weka version get {{ weka_version }} - register: result - changed_when: result.rc == 0 - - - name: Set user-defined WEKA version - ansible.builtin.shell: - cmd: weka version set {{ weka_version }} - register: result - changed_when: result.rc == 0 - - - name: Setup WEKA client - ansible.builtin.shell: - cmd: weka local setup weka - register: setup_1_result - changed_when: setup_1_result.rc == 0 - failed_when: false # ignore errors - - - name: Setup WEKA client (2nd try) - ansible.builtin.shell: - cmd: weka local setup weka - register: result - changed_when: result.rc == 0 - when: setup_1_result.rc != 0 - - - name: Prepare WEKA version - ansible.builtin.shell: - cmd: weka version prepare {{ weka_version }} - register: result - changed_when: result.rc == 0 - - - name: Stop WEKA client - ansible.builtin.shell: - cmd: weka local stop - async: 30 - poll: 10 - register: weka_stop - changed_when: weka_stop.get("rc") == 0 # when killed by async, rc is not defined - failed_when: false # ignore errors - - - name: Stop WEKA client (2nd try) - ansible.builtin.shell: - cmd: weka local stop - async: 30 - poll: 10 - register: result - changed_when: result.rc == 0 - failed_when: false # ignore errors - when: weka_stop.get("rc") != 0 - - - name: Remove WEKA containers - ansible.builtin.shell: - cmd: weka local rm -f --all - register: result - changed_when: result.rc == 0 - failed_when: false # ignore errors diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl deleted file mode 100644 index 19c6dc1fdc..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -# shellcheck disable=SC2034 -METADATA_BASE_URL="http://metadata.google.internal/computeMetadata/v1/instance" -# shellcheck disable=SC2034 -ATTR_URL="$${METADATA_BASE_URL}/attributes/weka-" -NET_URL="$${METADATA_BASE_URL}/network-interfaces" - -# shellcheck disable=SC1083 -WEKA_MODE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mode || echo -n udp) -# shellcheck disable=SC1083 -WEKA_DATA_INTERFACES=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}data_interfaces || exit 0) -# shellcheck disable=SC1083 -WEKA_MGMT_INTERFACE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mgmt_interface || echo -n 0) -# shellcheck disable=SC1083 -WEKA_OPTIONS=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}options || exit 0) - -WEKA_OPTIONS="$${WEKA_OPTIONS:+-o $WEKA_OPTIONS}" - -netmask_to_cidr () { - c=0 - # shellcheck disable=SC2086,SC1083 - x=0$( printf '%o' $${1//./ } ) - while [ "$x" -gt 0 ]; do - c=$(( c + x%2 )) - x=$(( x >> 1)) - done - echo $c ; -} - -# detect network interface naming scheme -if [[ -e /sys/class/net/eth0 ]] ; then - DEVICE_NAME="eth" - DEVICE_INDEX_BASE=0 -elif [[ -e /sys/class/net/ens4 ]] ; then - DEVICE_NAME="ens" - DEVICE_INDEX_BASE=4 -else - echo "Can't detect device names. Both /sys/class/net/eth0 and /sys/class/net/ens4 do not exists" - exit 1 -fi - -# ensure that /etc/hosts contains entry for hostname pointing to primary interface -NEW_IP=$(ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) | head -n 1 | sed -e 's/^.*inet \([0-9\.]\+\)\/.*$/\1/') -if [ -n "$NEW_IP" ] ; then - HOSTNAME=$(hostname) - sed -i -e "/$HOSTNAME/s/^[0-9\.]\+ $HOSTNAME/$NEW_IP $HOSTNAME/" /etc/hosts -else - echo "Failed to find primary interface address" - ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) - exit 1 -fi - -# shellcheck disable=SC2154 -echo "Mounting Weka ${server_ip}/${remote_mount} to ${local_mount}" -mkdir -p "${local_mount}" -service weka-agent start -if [[ $WEKA_MODE == "udp" ]] ; then - # shellcheck disable=SC2086,SC2154,SC2086 - mount -t wekafs ${mount_options} -o net=udp $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" - -elif [[ $WEKA_MODE == "dpdk" ]] ; then - declare -a DATA_INTERFACES - # split WEKA_DATA_INTERFACES by comma into array - # shellcheck disable=SC2034 - IFS=',' read -r -a DATA_INTERFACES <<< "$WEKA_DATA_INTERFACES" - - DATA_OPTIONS="" - # shellcheck disable=SC2066 - for interface in "$${DATA_INTERFACES[@]}" ; do - INTERFACE_IP=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/ip") - INTERFACE_MASK=$(netmask_to_cidr "$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/subnetmask")") - INTERFACE_GATEWAY=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/gateway") - - DATA_OPTIONS+="-o net=$DEVICE_NAME$((DEVICE_INDEX_BASE + interface))/$INTERFACE_IP/$INTERFACE_MASK/$INTERFACE_GATEWAY " - done - - # shellcheck disable=SC2086 - mount -t wekafs \ - ${mount_options} \ - -o mgmt_ip="$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$WEKA_MGMT_INTERFACE/ip")" \ - $DATA_OPTIONS $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" -else - echo "Unknown weka:mode metadata value: $${WEKA_MODE}. Allowed values: udp and dpdk" - exit 1 -fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl deleted file mode 100644 index 84587103a9..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Mount the WEKA file systems - hosts: localhost - become: true - vars: - local_mount: "${local_mount}" - remote_mount: "${remote_mount}" - server_ip: "${server_ip}" - service_name: "weka-mount-${replace(local_mount, "/", "_")}" - tasks: - - name: Create mount script - ansible.builtin.copy: - dest: "/etc/{{ service_name }}.sh" - mode: "0755" - content: | - ${indent(8, mount_weka_script)} - - - name: Create systemd service for weka mount - ansible.builtin.copy: - dest: "/etc/systemd/system/{{ service_name }}.service" - mode: "0644" - content: | - [Install] - WantedBy=multi-user.target - [Unit] - Description=Mount Weka {{ server_ip }}/{{ remote_mount }} at {{ local_mount }} - After=network-online.target - Wants=network-online.target - [Service] - RemainAfterExit=true - Type=oneshot - ExecStart=/bin/bash -c "/etc/{{ service_name }}.sh" - - - name: Enable and start weka mount service - ansible.builtin.systemd: - name: "{{ service_name }}" - daemon_reload: true - enabled: true - state: started diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/variables.tf deleted file mode 100644 index f07961d64c..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "local_mount" { - description = "The mount point where the contents of the device may be accessed after mounting." - type = string - default = "/mnt" -} - -variable "mount_options" { - description = "Mount options for filesystem shared by all clients." - type = string - default = "" - nullable = false -} - -variable "remote_mount" { - description = "Weka filesystem name." - type = string -} - -variable "server_ip" { - description = "Weka backend IP address used for bootstrapping." - type = string - default = "" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/versions.tf deleted file mode 100644 index 9e6af1fa7f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/file-system/weka-client/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 0.14.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb deleted file mode 100644 index f13726f691..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb +++ /dev/null @@ -1,125 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "project_id = \"${project_id}\"\n", - "dataset_id = \"${dataset_id}\"\n", - "table_id = \"${table_id}\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ONI1Xo0-KtAD", - "outputId": "fb9ca475-e4ec-4cd0-e0e6-14f409eefd7a" - }, - "outputs": [], - "source": [ - "from google.cloud import bigquery\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "\n", - "client = bigquery.Client(project=project_id)\n", - "\n", - "df = client.query(f'''\n", - "SELECT ticker, cast(price AS FLOAT64) AS price, CAST(OFFSET as INTEGER) AS offset, start_date, end_date, iteration\n", - "FROM `{project_id}.{dataset_id}.{table_id}`,\n", - "UNNEST(simulation_results) as NUMERIC with OFFSET\n", - "WHERE epoch_time IN\n", - " # Get the latest simulation runs for each Ticker Symbol\n", - "(SELECT MAX(epoch_time) FROM `{project_id}.{dataset_id}.{table_id}` GROUP BY ticker)\n", - "'''\n", - ").to_dataframe()\n", - "# Display the data\n", - "df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Define a function to plot the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "def plot_ticker(t,df):\n", - "\n", - " dtf = df[(df.ticker==t) &(df.offset == 250)].price.describe(include=[np.float64], percentiles=[.05, .01, .001])\n", - " cellText = []\n", - " for v in dtf.values:\n", - " cellText.append([v])\n", - " \n", - " pltf = df[df.ticker==t].pivot(index='offset', columns='iteration', values='price')\n", - " \n", - " fig = plt.figure(figsize=(10,5))\n", - " ax1 = fig.add_subplot(122)\n", - " pltf.plot(legend=False, ax=ax1, xlabel='Time(days)', ylabel='US$', title=f\"{ df[(df.ticker == t) & (df.offset == 0) & (df.iteration == 4)]}\")\n", - " ax2 = fig.add_subplot(121)\n", - " font_size=10\n", - " bbox=[0, 0, .5, 1]\n", - " ax2.axis('off')\n", - " mpl_table = ax2.table(cellText = cellText, rowLabels=dtf.index.values, bbox=bbox)\n", - " mpl_table.auto_set_font_size(False)\n", - " mpl_table.set_fontsize(font_size)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 808 - }, - "id": "jvBmb_KceX7z", - "outputId": "42a3ba9f-b68f-4c7b-d928-0fedeed9216c" - }, - "outputs": [], - "source": [ - "ticker_list = df.ticker.unique()\n", - "for t in ticker_list:\n", - " plot_ticker(t,df)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md deleted file mode 100644 index e54893a1bb..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md +++ /dev/null @@ -1,97 +0,0 @@ -## Description - -Copy files to a target GCS bucket. - -Primarily used for FSI - MonteCarlo Tutorial **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: -../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage -This copies the module files to the specified GCS bucket. It is expected that -the bucket will be mounted on the target VM. - -Some of the files are templates, and `main.tf` translates the files with the -passed variable values. This way the user does not have to change things like -pointing to the correct bigquery table or adding in the project_id. - -```yaml - - id: fsi_tutorial_files - source: community/modules/files/fsi-montecarlo-on-batch - use: [bq-dataset, bq-table, fsi_bucket, pubsub_topic] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 3.83 | -| [http](#requirement\_http) | ~> 3.0 | -| [random](#requirement\_random) | ~> 3.0 | -| [template](#requirement\_template) | ~> 2.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [http](#provider\_http) | ~> 3.0 | -| [random](#provider\_random) | ~> 3.0 | -| [template](#provider\_template) | ~> 2.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.get_iteration_sh](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.get_mc_reqs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.get_requirements](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.ipynb_obj_fsi](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.mc_obj_yaml](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.mc_run](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.run_batch_py](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [http_http.batch_py](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | -| [http_http.batch_requirements](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | -| [template_file.ipynb_fsi](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | -| [template_file.mc_run_py](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | -| [template_file.mc_run_yaml](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Bigquery dataset id | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to run project | `string` | n/a | yes | -| [table\_id](#input\_table\_id) | Bigquery table id | `string` | n/a | yes | -| [topic\_id](#input\_topic\_id) | Pubsub Topic Name | `string` | n/a | yes | -| [topic\_schema](#input\_topic\_schema) | Pubsub Topic schema | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh deleted file mode 100644 index 50aa865a31..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ticker=("GOOG" "AMZN" "MSFT" "NVDA" "META" "TSLA" "PEP" "COST") -echo "BI: $BATCH_TASK_INDEX" -echo "TI: ${ticker[$BATCH_TASK_INDEX]}" -python3 -m pip install -r /mnt/disks/fsi/mc_run_reqs.txt -python3 /mnt/disks/fsi/mc_run.py \ - --ticker "${ticker[$BATCH_TASK_INDEX]}" \ - --iterations 500 \ - --start_date 2022-01-01 diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf deleted file mode 100644 index 83dc7fe9cf..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - bucket = replace(var.gcs_bucket_path, "gs://", "") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -data "template_file" "mc_run_py" { - template = file("${path.module}/mc_run.tpl.py") - vars = { - project_id = var.project_id - topic_id = var.topic_id - topic_schema = var.topic_schema - dataset_id = var.dataset_id - table_id = var.table_id - } -} - -resource "google_storage_bucket_object" "mc_run" { - name = "mc_run.py" - content = data.template_file.mc_run_py.rendered - bucket = local.bucket -} - -data "template_file" "mc_run_yaml" { - template = file("${path.module}/mc_run.tpl.yaml") - vars = { - project_id = var.project_id - bucket_name = local.bucket - region = var.region - } -} - -resource "google_storage_bucket_object" "mc_obj_yaml" { - name = "mc_run.yaml" - content = data.template_file.mc_run_yaml.rendered - bucket = local.bucket -} - -data "template_file" "ipynb_fsi" { - template = file("${path.module}/FSI_MonteCarlo.ipynb") - vars = { - project_id = var.project_id - dataset_id = var.dataset_id - table_id = var.table_id - } -} -resource "google_storage_bucket_object" "ipynb_obj_fsi" { - name = "FSI_MonteCarlo.ipynb" - content = data.template_file.ipynb_fsi.rendered - bucket = local.bucket -} - -data "http" "batch_py" { - url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/batch.py" -} - -resource "google_storage_bucket_object" "run_batch_py" { - name = "batch.py" - content = data.http.batch_py.response_body - bucket = local.bucket -} - -data "http" "batch_requirements" { - url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/requirements.txt" -} - -resource "google_storage_bucket_object" "get_requirements" { - name = "requirements.txt" - content = data.http.batch_requirements.response_body - bucket = local.bucket -} - -resource "google_storage_bucket_object" "get_iteration_sh" { - name = "iteration.sh" - content = file("${path.module}/iteration.sh") - bucket = local.bucket -} - -resource "google_storage_bucket_object" "get_mc_reqs" { - name = "mc_run_reqs.txt" - content = file("${path.module}/mc_run_reqs.txt") - bucket = local.bucket -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py deleted file mode 100644 index 4e0a64e363..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Run MC simulation for VaR portfolio risk -""" - -import avro.schema -import io -import google.auth -import numpy -import time -import yfinance as yf - -from absl import app -from absl import flags -from avro.io import DatumWriter, BinaryEncoder, BinaryDecoder, DatumReader -from datetime import datetime -from datetime import timedelta -from google.cloud import pubsub_v1, bigquery -from google.cloud.pubsub import SchemaServiceClient - -PROJECT_ID = '${project_id}' -INCOMING_TOPIC_ID = '${topic_id}' -INCOMING_TOPIC_SCHEMA = '${topic_schema}' -DATASET_ID = '${dataset_id}' -TABLE_ID = '${table_id}' - - -FLAGS = flags.FLAGS - -flags.DEFINE_string("ticker", 'GOOG', "Nasdaq Stock Ticker to run, default GOOG") -flags.DEFINE_string("start_date", '2022-01-01' , "Start data for data query, default 2022-01-01") -flags.DEFINE_integer("calendar_days", 365 , "How many calendar days to include in the calculation") -flags.DEFINE_integer("epoch_time", f'{int(time.time())}' , "Epoch time, number of seconds since January 1st, 1970 at 00:00:00 UTC.") -flags.DEFINE_integer("iterations", 100 , "Number of iterations to run.") -flags.DEFINE_boolean("print_raw", False, "Dump raw data.") - -class VaRSimulator: - - def __init__(self): - pass - - def get_data(self): - self.get_historical_data_yahoo() - - def get_historical_data_yahoo(self): - - # get historical market data: https://pypi.org/project/yfinance/ - - self.raw_data = yf.Ticker(self.ticker).history(start=self.start_date, end=self.end_date ) - self.data = self.raw_data.Close - - def print_raw(self): - print(self.get_stats()) - print(type(self.raw_data)) - print(self.raw_data) - - def get_stats(self): - close = self.data - self.first = close[0] - self.last = close[-1] - self.trading_days = len(close) - self.cagr = (self.last / self.first) ** (365.0/self.calendar_days) -1.0 - self.volatility = self.data.pct_change().std() - return(self.first, self.last, self.trading_days, self.cagr, self.volatility) - - def run_simulation(self): - - returns = numpy.random.normal(self.cagr/self.trading_days, self.volatility, self.trading_days) + 1 - returns = numpy.insert(returns,0,1.0) - self.simulation_results = self.last * returns.cumprod() - return(self.simulation_results) - - def create_object(self): - self.object = { - "ticker": self.ticker, - "epoch_time": self.epoch_time, - "iteration": self.iteration, - "start_date": self.start_date, - "end_date": self.end_date, - "simulation_results": list(map(lambda x: {"price":x}, self.simulation_results)) - } - return(self.object) - - -class PubsubToBiquery: - - def __init__(self): - - the_time = int(time.time()) - - self.project_id = PROJECT_ID - - self.publisher_client = pubsub_v1.PublisherClient() - self.topic_path = self.publisher_client.topic_path(self.project_id, INCOMING_TOPIC_ID) - - self.schema_client = SchemaServiceClient() - self.schema_path = self.schema_client.schema_path(self.project_id, INCOMING_TOPIC_SCHEMA) - - pubsub_schema = self.schema_client.get_schema(request={"name": self.schema_path}) - avro_schema = avro.schema.parse(pubsub_schema.definition) - - self.writer = DatumWriter(avro_schema) - - - def publish_record(self,record): - - byte_stream = io.BytesIO() - encoder = BinaryEncoder(byte_stream) - self.writer.write(record, encoder) - data = byte_stream.getvalue() - byte_stream.flush() - future = self.publisher_client.publish(self.topic_path, data) - if(FLAGS.print_raw): - print(f"Published message ID: {future.result()}") - - -def main(argv): - - vr = VaRSimulator() - pbbq = PubsubToBiquery() - - vr.ticker =FLAGS.ticker - vr.start_date =FLAGS.start_date - vr.end_date =f'{(datetime.strptime(FLAGS.start_date,"%Y-%m-%d") + timedelta(days = FLAGS.calendar_days)).date()}' - vr.calendar_days = FLAGS.calendar_days - vr.epoch_time = FLAGS.epoch_time - vr.iteration = 1 - - vr.get_data() - vr.get_stats() - - for i in range(FLAGS.iterations): - vr.iteration = i - vr.run_simulation() - pbbq.publish_record(vr.create_object()) - - if(FLAGS.print_raw): - vr.print_raw() - - -if __name__ == "__main__": - """ This is executed when run from the command line """ - app.run(main) diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml deleted file mode 100644 index 7f7de4840b..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -project_id: "${project_id}" -region: "${region}" - -job_prefix: 'fsi-' -machine_type: "n2-standard-2" -volumes: -- {bucket_name: "${bucket_name}", gcs_path: "/mnt/disks/fsi"} - -container: - image_uri: "python" - entry_point: "/bin/bash" - commands: ["/mnt/disks/fsi/iteration.sh", "$BATCH_TASK_INDEX"] - -task_count: 8 #optional -parallelism: 4 #optional -task_count_per_node: 2 #optional -cpu_milli: 1000 #optional -memory_mib: 102400 #optional - - -labels: - env: "monte" - type: "carlo" diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt deleted file mode 100644 index 105ed70ad2..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt +++ /dev/null @@ -1,9 +0,0 @@ -absl-py -avro -google-auth -google-cloud -google-cloud-batch -google-cloud-pubsub -google-cloud-bigquery -yfinance -PyYAML diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml deleted file mode 100644 index 268c8faa9a..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [storage.googleapis.com] diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf deleted file mode 100644 index eddf3c9478..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which GCS bucket will be created." - type = string -} - -variable "gcs_bucket_path" { - description = "Bucket name" - type = string - default = null -} - -variable "topic_id" { - description = "Pubsub Topic Name" - type = string -} - -variable "topic_schema" { - description = "Pubsub Topic schema" - type = string -} - -variable "dataset_id" { - description = "Bigquery dataset id" - type = string -} - -variable "table_id" { - description = "Bigquery table id" - type = string -} - -variable "region" { - description = "Region to run project" - type = string -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf deleted file mode 100644 index 86dcb4dc52..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - http = { - source = "hashicorp/http" - version = "~> 3.0" - } - template = { - source = "hashicorp/template" - version = "~> 2.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md deleted file mode 100644 index ae8462d763..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md +++ /dev/null @@ -1,100 +0,0 @@ -# Module: Slurm Instance - - - -- [Module: Slurm Instance](#module-slurm-instance) - - [Overview](#overview) - - [Module API](#module-api) - - - -## Overview - -This module creates a [compute instance](../../../../docs/glossary.md#vm) from -[instance template](../../../../docs/glossary.md#instance-template) for a -[Slurm cluster](../slurm_cluster/README.md). - -> **NOTE:** This module is only intended to be used by Slurm modules. For -> general usage, please consider using: -> -> - [terraform-google-modules/vm/google//modules/compute_instance](https://registry.terraform.io/modules/terraform-google-modules/vm/google/latest/submodules/compute_instance). -> **WARNING:** The source image is not modified. Make sure to use a compatible -> source image. - -## Module API - -For the terraform module API reference, please see -[README_TF.md](./README_TF.md). - - -Copyright (C) SchedMD LLC. -Copyright 2018 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.0 | -| [google](#requirement\_google) | >= 3.43 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.43 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_from_template.slurm_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | -| [null_resource.replace_trigger](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_instance_template.base](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | -| [hostname](#input\_hostname) | Hostname of instances | `string` | n/a | yes | -| [instance\_template](#input\_instance\_template) | Instance template self\_link used to create compute instances | `string` | n/a | yes | -| [network](#input\_network) | Network to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | -| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | -| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | -| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | -| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances | `list(string)` | `[]` | no | -| [subnetwork](#input\_subnetwork) | Subnet to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The project that subnetwork belongs to | `string` | `null` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [available\_zones](#output\_available\_zones) | List of available zones in region | -| [instances\_details](#output\_instances\_details) | List of all details for compute instances | -| [instances\_self\_links](#output\_instances\_self\_links) | List of self-links for compute instances | -| [names](#output\_names) | List of available zones in region | -| [slurm\_instances](#output\_slurm\_instances) | List of all resource objects for compute instances | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf deleted file mode 100644 index 2af9008a0e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########## -# LOCALS # -########## - -locals { - num_instances = length(var.static_ips) == 0 ? var.num_instances : length(var.static_ips) - - # local.static_ips is the same as var.static_ips with a dummy element appended - # at the end of the list to work around "list does not have any elements so cannot - # determine type" error when var.static_ips is empty - static_ips = concat(var.static_ips, ["NOT_AN_IP"]) - - network_interfaces = [for index in range(local.num_instances) : - concat([ - { - access_config = var.access_config - alias_ip_range = [] - ipv6_access_config = [] - network = var.network - network_ip = length(var.static_ips) == 0 ? "" : element(local.static_ips, index) - nic_type = null - queue_count = null - stack_type = null - subnetwork = var.subnetwork - subnetwork_project = var.subnetwork_project - } - ], - var.additional_networks - ) - ] -} - -################ -# DATA SOURCES # -################ - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -data "google_compute_instance_template" "base" { - project = var.project_id - name = var.instance_template -} - -############# -# INSTANCES # -############# -resource "null_resource" "replace_trigger" { - triggers = { - trigger = var.replace_trigger - } -} - -# TODO: `internal/slurm-gcp/login` is ONLY user of `internal/slurm-gcp/instance` -# Remove this module, add functionality (+ prune generality) to the login module directly. -resource "google_compute_instance_from_template" "slurm_instance" { - count = local.num_instances - name = format("%s-%s", var.hostname, format("%03d", count.index + 1)) - project = var.project_id - zone = var.zone == null ? data.google_compute_zones.available.names[count.index % length(data.google_compute_zones.available.names)] : var.zone - - allow_stopping_for_update = true - - dynamic "network_interface" { - for_each = local.network_interfaces[count.index] - iterator = nic - content { - dynamic "access_config" { - for_each = nic.value.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = nic.value.alias_ip_range - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - dynamic "ipv6_access_config" { - for_each = nic.value.ipv6_access_config - iterator = access_config - content { - network_tier = access_config.value.network_tier - } - } - network = nic.value.network - network_ip = nic.value.network_ip - nic_type = nic.value.nic_type - queue_count = nic.value.queue_count - subnetwork = nic.value.subnetwork - subnetwork_project = nic.value.subnetwork_project - } - } - - source_instance_template = data.google_compute_instance_template.base.self_link - # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 - # we have to explicitly override instance labels instead of inheriting them from template. - labels = data.google_compute_instance_template.base.labels - - - lifecycle { - replace_triggered_by = [null_resource.replace_trigger.id] - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf deleted file mode 100644 index 4eba78a7e8..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "slurm_instances" { - description = "List of all resource objects for compute instances" - value = google_compute_instance_from_template.slurm_instance -} - -output "instances_self_links" { - description = "List of self-links for compute instances" - value = google_compute_instance_from_template.slurm_instance[*].self_link -} - -output "instances_details" { - description = "List of all details for compute instances" - value = google_compute_instance_from_template.slurm_instance[*] -} - -output "available_zones" { - description = "List of available zones in region" - value = data.google_compute_zones.available.names -} - -output "names" { - description = "List of available zones in region" - value = google_compute_instance_from_template.slurm_instance[*].name -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf deleted file mode 100644 index 11111a2c05..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "The GCP project ID" - default = null -} - -variable "network" { - description = "Network to deploy to. Only one of network or subnetwork should be specified." - type = string - default = "" -} - -variable "subnetwork" { - description = "Subnet to deploy to. Only one of network or subnetwork should be specified." - type = string - default = "" -} - -variable "subnetwork_project" { - description = "The project that subnetwork belongs to" - type = string - default = null -} - -variable "hostname" { - description = "Hostname of instances" - type = string -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })) - nullable = false -} - -variable "static_ips" { - description = "List of static IPs for VM instances" - type = list(string) - default = [] -} - -variable "access_config" { - description = "Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet." - type = list(object({ - nat_ip = string - network_tier = string - })) - default = [] -} - -variable "num_instances" { - description = "Number of instances to create. This value is ignored if static_ips is provided." - type = number - default = 1 -} - -variable "instance_template" { - description = "Instance template self_link used to create compute instances" - type = string -} - -variable "region" { - description = "Region where the instances should be created." - type = string - default = null -} - -variable "zone" { - description = "Zone where the instances should be created. If not specified, instances will be spread across available zones in the region." - type = string - default = null -} - -######### -# SLURM # -######### - -variable "replace_trigger" { - description = "Trigger value to replace the instances." - type = string - default = "" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf deleted file mode 100644 index a3e84c09bf..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = "~> 1.0" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.43" - } - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md deleted file mode 100644 index 87394bef6a..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md +++ /dev/null @@ -1,87 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.0 | -| [local](#requirement\_local) | ~> 2.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | ~> 2.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance\_template](#module\_instance\_template) | ../internal_instance_template | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.startup](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    disk_type                  = optional(string)
    disk_size_gb               = optional(number)
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `virtio_enabled` setting will only enable VirtioNet and will not enable TIER\_1.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard. | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See
https://cloud.google.com/compute/docs/gpus more details.
- type : the GPU type
- count : number of GPUs | 
object({
    type  = string
    count = number
  })
| `null` | no | -| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"n1-standard-1"` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Prefix for template resource. | `string` | `"default"` | no | -| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network
attribute for Legacy or Auto subnetted networks and subnetwork for custom
subnetted networks. | `string` | `null` | no | -| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | -| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the instances. See
'main.tf:local.service\_account' for the default. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming. | `string` | n/a | yes | -| [slurm\_instance\_role](#input\_slurm\_instance\_role) | Slurm instance type. Must be one of: controller; login; compute; or null. | `string` | n/a | yes | -| [source\_image](#input\_source\_image) | Source disk image. | `string` | `""` | no | -| [source\_image\_family](#input\_source\_image\_family) | Source image family. | `string` | `""` | no | -| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. If it is not provided, the provider project is used. | `string` | `""` | no | -| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must
exist in the same region this instance will be created in. Either network or
subnetwork must be provided. | `string` | `null` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [termination\_action](#input\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_template](#output\_instance\_template) | Instance template details | -| [labels](#output\_labels) | Labels attached to the instance template | -| [name](#output\_name) | Name of instance template | -| [self\_link](#output\_self\_link) | Self\_link of instance template | -| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | -| [tags](#output\_tags) | Tags that will be associated with instance(s) | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted deleted file mode 100644 index 2edaa942d2..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -SLURM_DIR=/slurm -FLAGFILE=$SLURM_DIR/slurm_configured_do_not_remove -SCRIPTS_DIR=$SLURM_DIR/scripts -if [[ -z "$HOME" ]]; then - # google-startup-scripts.service lacks environment variables - HOME="$(getent passwd "$(whoami)" | cut -d: -f6)" -fi - -# Temporary workaround for transition period when some of older images -# don't have "baked in" python yet. -# TODO: Remove -SLURM_PY="/slurm/python/venv/bin/python3.13" -SYSTEM_PY="/usr/bin/python3" -if [[ ! -e "$SLURM_PY" ]]; then - echo "Symlink $SLURM_PY does not exist. Creating symlink to $SYSTEM_PY" - mkdir -p /slurm/python/venv/bin - ln -s "$SYSTEM_PY" "$SLURM_PY" -fi - -METADATA_SERVER="metadata.google.internal" -URL="http://$METADATA_SERVER/computeMetadata/v1" -CURL="curl -sS --fail --header Metadata-Flavor:Google" - -PING_METADATA="ping -q -w1 -c1 $METADATA_SERVER" -echo "INFO: $PING_METADATA" -for i in $(seq 10); do - [ $i -gt 1 ] && sleep 5; - $PING_METADATA > /dev/null && s=0 && break || s=$?; - echo "ERROR: Failed to contact metadata server, will retry" -done -if [ $s -ne 0 ]; then - echo "ERROR: Unable to contact metadata server, aborting" - wall -n '*** Slurm setup failed in the startup script! see `journalctl -u google-startup-scripts` ***' - exit 1 -else - echo "INFO: Successfully contacted metadata server" -fi - -PING_GOOGLE="ping -q -w1 -c1 8.8.8.8" -echo "INFO: $PING_GOOGLE" -for i in $(seq 5); do - [ $i -gt 1 ] && sleep 2; - $PING_GOOGLE > /dev/null && s=0 && break || s=$?; - echo "failed to ping Google DNS, will retry" -done -if [ $s -ne 0 ]; then - echo "WARNING: No internet access detected" -else - echo "INFO: Internet access detected" -fi - -mkdir -p $SCRIPTS_DIR -UNIVERSE_DOMAIN="$($CURL $URL/instance/attributes/universe_domain)" -BUCKET="$($CURL $URL/instance/attributes/slurm_bucket_path)" -if [[ -z $BUCKET ]]; then - echo "ERROR: No bucket path detected." - exit 1 -fi - -SCRIPTS_ZIP="$HOME/slurm-gcp-scripts.zip" -export CLOUDSDK_CORE_UNIVERSE_DOMAIN="$UNIVERSE_DOMAIN" - -INSTANCE_ROLE="$($CURL $URL/instance/attributes/slurm_instance_role)" - -if [ "$INSTANCE_ROLE" == "controller" ]; then - DEVEL_ZIP="slurm-gcp-devel-controller.zip" -else - DEVEL_ZIP="slurm-gcp-devel.zip" -fi -until gcloud storage cp "$BUCKET/$DEVEL_ZIP" "$SCRIPTS_ZIP"; do - echo "WARN: Could not download SlurmGCP scripts, retrying in 5 seconds." - # Remove marker used to determine if gcloud is being used in a GCE VM. - # This can get mistakenly set to False in some cases. - rm -f /root/.config/gcloud/gce - sleep 5 -done -unzip -o "$SCRIPTS_ZIP" -d "$SCRIPTS_DIR" -rm -rf "$SCRIPTS_ZIP" - -#temporary hack to not make the script fail on TPU vm -chown slurm:slurm -R "$SCRIPTS_DIR" || true -chmod 700 -R "$SCRIPTS_DIR" - - -if [ -f $FLAGFILE ]; then - echo "WARNING: Slurm was previously configured, quitting" - exit 0 -fi -touch $FLAGFILE - -function tpu_setup { - #allow the following command to fail, as this attribute does not exist for regular nodes - docker_image=$($CURL $URL/instance/attributes/slurm_docker_image 2> /dev/null || true) - if [ -z $docker_image ]; then #Not a tpu node, do not do anything - return - fi - if [ "$OS_ENV" == "slurm_container" ]; then #Already inside the slurm container, we should continue starting - return - fi - - #given a input_string like "WORKER_0:Joseph;WORKER_1:richard;WORKER_2:edward;WORKER_3:john" and a number 1, this function will print richard - parse_metadata() { - local number=$1 - local input_string=$2 - local word=$(echo "$input_string" | awk -v n="$number" -F ':|;' '{ for (i = 1; i <= NF; i+=2) if ($(i) == "WORKER_"n) print $(i+1) }') - echo "$word" - } - - input_string=$($CURL $URL/instance/attributes/slurm_names) - worker_id=$($CURL $URL/instance/attributes/tpu-env | awk '/WORKER_ID/ {print $2}' | tr -d \') - real_name=$(parse_metadata $worker_id $input_string) - - #Prepare to docker pull with gcloud - mkdir -p /root/.docker - cat << EOF > /root/.docker/config.json -{ - "credHelpers": { - "gcr.io": "gcloud", - "us-docker.pkg.dev": "gcloud" - } -} -EOF - #cgroup detection - CGV=1 - CGROUP_FLAGS="-v /sys/fs/cgroup:/sys/fs/cgroup:rw" - if [ -f /sys/fs/cgroup/cgroup.controllers ]; then #CGV2 - CGV=2 - fi - if [ $CGV == 2 ]; then - CGROUP_FLAGS="--cgroup-parent=docker.slice --cgroupns=private --tmpfs /run --tmpfs /run/lock --tmpfs /tmp" - if [ ! -f /etc/systemd/system/docker.slice ]; then #In case that there is no slice prepared for hosting the containers create it - printf "[Unit]\nDescription=docker slice\nBefore=slices.target\n[Slice]\nCPUAccounting=true\nMemoryAccounting=true" > /etc/systemd/system/docker.slice - systemctl start docker.slice - fi - fi - #for the moment always use --privileged, as systemd might not work properly otherwise - TPU_FLAGS="--privileged" - # TPU_FLAGS="--cap-add SYS_RESOURCE --device /dev/accel0 --device /dev/accel1 --device /dev/accel2 --device /dev/accel3" - # if [ $CGV == 2 ]; then #In case that we are in CGV2 for systemd to work correctly for the moment we go with privileged - # TPU_FLAGS="--privileged" - # fi - - docker run -d $CGROUP_FLAGS $TPU_FLAGS --net=host --name=slurmd --hostname=$real_name --entrypoint=/usr/bin/systemd --restart unless-stopped $docker_image - exit 0 -} - -tpu_setup #will do nothing for normal nodes or the container spawned inside TPU - -echo "INFO: Running python cluster setup script" -SETUP_SCRIPT_FILE=$SCRIPTS_DIR/setup.py -chmod +x $SETUP_SCRIPT_FILE -exec $SETUP_SCRIPT_FILE diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf deleted file mode 100644 index c91bbc4fd1..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "instance_validation" { - source = "../../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -########## -# LOCALS # -########## - -locals { - additional_disks = [ - for disk in var.additional_disks : { - disk_name = disk.disk_name - device_name = disk.device_name - auto_delete = disk.auto_delete - source = disk.source - boot = disk.boot - disk_size_gb = disk.disk_size_gb - disk_type = disk.disk_type - disk_labels = merge( - disk.disk_labels, - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) - disk_resource_manager_tags = disk.disk_resource_manager_tags - } - ] - - service_account = { - email = try(var.service_account.email, null) - scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) - } - - source_image_family = ( - var.source_image_family != "" && var.source_image_family != null - ? var.source_image_family - : "slurm-gcp-6-11-hpc-rocky-linux-8" - ) - source_image_project = ( - var.source_image_project != "" && var.source_image_project != null - ? var.source_image_project - : "projects/schedmd-slurm-public/global/images/family" - ) - - source_image = ( - var.source_image != null - ? var.source_image - : "" - ) - - - name_prefix = "${var.slurm_cluster_name}-${var.slurm_instance_role}-${var.name_prefix}" - - total_egress_bandwidth_tier = var.bandwidth_tier == "tier_1_enabled" ? "TIER_1" : "DEFAULT" - - nic_type_map = { - platform_default = null - virtio_enabled = "VIRTIO_NET" - gvnic_enabled = "GVNIC" - tier_1_enabled = "GVNIC" - } - nic_type = lookup(local.nic_type_map, var.bandwidth_tier, null) - - labels = merge(var.labels, - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) -} - -######## -# DATA # -######## - -data "local_file" "startup" { - filename = "${path.module}/files/startup_sh_unlinted" -} - -############ -# TEMPLATE # -############ - -module "instance_template" { - source = "../internal_instance_template" - - project_id = var.project_id - - # Network - can_ip_forward = var.can_ip_forward - network_ip = var.network_ip - network = var.network - nic_type = local.nic_type - region = var.region - subnetwork_project = var.subnetwork_project - subnetwork = var.subnetwork - tags = var.tags - total_egress_bandwidth_tier = local.total_egress_bandwidth_tier - additional_networks = var.additional_networks - access_config = var.access_config - - # Instance - machine_type = var.machine_type - min_cpu_platform = var.min_cpu_platform - name_prefix = local.name_prefix - gpu = var.gpu - service_account = local.service_account - shielded_instance_config = var.shielded_instance_config - advanced_machine_features = var.advanced_machine_features - enable_confidential_vm = var.enable_confidential_vm - enable_shielded_vm = var.enable_shielded_vm - preemptible = var.preemptible - spot = var.spot - on_host_maintenance = var.on_host_maintenance - labels = local.labels - instance_termination_action = var.termination_action - resource_manager_tags = var.resource_manager_tags - - # Metadata - startup_script = coalesce(var.internal_startup_script, data.local_file.startup.content) - metadata = merge( - var.metadata, - { - enable-oslogin = upper(var.enable_oslogin) - slurm_bucket_path = var.slurm_bucket_path - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) - - # Image - source_image_project = local.source_image_project - source_image_family = local.source_image_family - source_image = local.source_image - - # Disk - disk_type = var.disk_type - disk_size_gb = var.disk_size_gb - auto_delete = var.disk_auto_delete - disk_labels = merge( - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - var.disk_labels, - ) - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - - max_run_duration = var.max_run_duration - provisioning_model = var.provisioning_model - reservation_affinity = var.reservation_affinity -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf deleted file mode 100644 index 65da41052e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "instance_template" { - description = "Instance template details" - value = module.instance_template -} - -output "self_link" { - description = "Self_link of instance template" - value = module.instance_template.self_link -} - -output "name" { - description = "Name of instance template" - value = module.instance_template.name -} - -output "tags" { - description = "Tags that will be associated with instance(s)" - value = module.instance_template.tags -} - -output "service_account" { - description = "Service account object, includes email and scopes." - value = module.instance_template.service_account -} - -output "labels" { - description = "Labels attached to the instance template" - value = local.labels -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf deleted file mode 100644 index 35dd9c376f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf +++ /dev/null @@ -1,431 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -########### -# GENERAL # -########### - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "on_host_maintenance" { - type = string - description = "Instance availability Policy" - default = "MIGRATE" -} - -variable "labels" { - type = map(string) - description = "Labels, provided as a map" - default = {} -} - -variable "enable_oslogin" { - type = bool - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >=0.13.0 | -| [google](#requirement\_google) | >= 3.88 | -| [google-beta](#requirement\_google-beta) | >= 6.13.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.88 | -| [google-beta](#provider\_google-beta) | >= 6.13.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance_template.tpl](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_template) | resource | -| [google_project.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    auto_delete                = bool
    boot                       = bool
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = map(string)
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | -| [alias\_ip\_range](#input\_alias\_ip\_range) | An array of alias IP ranges for this network interface. Can only be specified for network interfaces on subnet-mode networks.
ip\_cidr\_range: The IP CIDR range represented by this alias IP range. This IP CIDR range must belong to the specified subnetwork and cannot contain IP addresses reserved by system or used by other network interfaces. At the time of writing only a netmask (e.g. /24) may be supplied, with a CIDR format resulting in an API error.
subnetwork\_range\_name: The subnetwork secondary range name specifying the secondary range from which to allocate the IP CIDR range for this alias IP range. If left unspecified, the primary range of the subnetwork will be used. | 
object({
    ip_cidr_range         = string
    subnetwork_range_name = string
  })
| `null` | no | -| [auto\_delete](#input\_auto\_delete) | Whether or not the boot disk should be auto-deleted | `string` | `"true"` | no | -| [automatic\_restart](#input\_automatic\_restart) | (Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user). | `bool` | `true` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example | `string` | `"false"` | no | -| [disk\_encryption\_key](#input\_disk\_encryption\_key) | The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance | `string` | `null` | no | -| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `string` | `"100"` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Whether to enable the Confidential VM configuration on the instance. Note that the instance image must support Confidential VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Whether to enable the Shielded VM configuration on the instance. Note that the instance image must support Shielded VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | -| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See https://cloud.google.com/compute/docs/gpus more details | 
object({
    type  = string
    count = number
  })
| `null` | no | -| [instance\_termination\_action](#input\_instance\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | -| [ipv6\_access\_config](#input\_ipv6\_access\_config) | IPv6 access configurations. Currently a max of 1 IPv6 access configuration is supported. If not specified, the instance will have no external IPv6 Internet access. | 
list(object({
    network_tier = string
  }))
| `[]` | no | -| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create, e.g. n1-standard-1 | `string` | `"n1-standard-1"` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Name prefix for the instance template | `string` | n/a | yes | -| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks. | `string` | `""` | no | -| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | -| [nic\_type](#input\_nic\_type) | The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO\_NET. | `string` | `null` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | -| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | -| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#service_account. | 
object({
    email  = optional(string)
    scopes = set(string)
  })
| n/a | yes | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Not used unless enable\_shielded\_vm is true. Shielded VM configuration for the instance. | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [source\_image](#input\_source\_image) | Source disk image. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `""` | no | -| [source\_image\_family](#input\_source\_image\_family) | Source image family. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `"centos-7"` | no | -| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. The default project contains CentOS images. | `string` | `"centos-cloud"` | no | -| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | -| [stack\_type](#input\_stack\_type) | The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4\_ONLY. | `string` | `null` | no | -| [startup\_script](#input\_startup\_script) | User startup script to run when instances spin up | `string` | `""` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided. | `string` | `""` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [total\_egress\_bandwidth\_tier](#input\_total\_egress\_bandwidth\_tier) | Network bandwidth tier. Note: machine\_type must be a supported type. Values are 'TIER\_1' or 'DEFAULT'.
See https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration for details. | `string` | `"DEFAULT"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [name](#output\_name) | Name of instance template | -| [self\_link](#output\_self\_link) | Self-link of instance template | -| [service\_account](#output\_service\_account) | value | -| [tags](#output\_tags) | Tags that will be associated with instance(s) | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf deleted file mode 100644 index f8d2813ece..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "instance_validation" { - source = "../../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -######### -# Locals -######### - -locals { - source_image = var.source_image != "" ? var.source_image : "centos-7-v20201112" - source_image_family = var.source_image_family != "" ? var.source_image_family : "centos-7" - source_image_project = var.source_image_project != "" ? var.source_image_project : "centos-cloud" - - boot_disk = [ - { - source_image = var.source_image != "" ? format("${local.source_image_project}/${local.source_image}") : format("${local.source_image_project}/${local.source_image_family}") - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_labels = var.disk_labels - auto_delete = var.auto_delete - disk_resource_manager_tags = var.disk_resource_manager_tags - boot = "true" - }, - ] - - all_disks = concat(local.boot_disk, var.additional_disks) - - # NOTE: Even if all the shielded_instance_config or confidential_instance_config - # values are false, if the config block exists and an unsupported image is chosen, - # the apply will fail so we use a single-value array with the default value to - # initialize the block only if it is enabled. - shielded_vm_configs = var.enable_shielded_vm ? [true] : [] - - gpu_enabled = var.gpu != null - alias_ip_range_enabled = var.alias_ip_range != null - preemptible = var.preemptible || var.spot - on_host_maintenance = ( - local.preemptible || var.enable_confidential_vm || local.gpu_enabled - ? "TERMINATE" - : var.on_host_maintenance - ) - automatic_restart = ( - # must be false when preemptible is true - local.preemptible ? false : var.automatic_restart - ) - - nic_type = var.total_egress_bandwidth_tier == "TIER_1" ? "GVNIC" : var.nic_type - - - provisioning_model = coalesce(var.provisioning_model, local.preemptible ? "SPOT" : "STANDARD") -} - -data "google_project" "this" { - project_id = var.project_id -} - -#################### -# Instance Template -#################### -resource "google_compute_instance_template" "tpl" { - provider = google-beta - name_prefix = "${var.name_prefix}-" - project = var.project_id - machine_type = var.machine_type - labels = var.labels - metadata = var.metadata - tags = var.tags - can_ip_forward = var.can_ip_forward - metadata_startup_script = var.startup_script - region = var.region - min_cpu_platform = var.min_cpu_platform - resource_manager_tags = var.resource_manager_tags - - service_account { - email = coalesce(var.service_account.email, "${data.google_project.this.number}-compute@developer.gserviceaccount.com") - scopes = lookup(var.service_account, "scopes", null) - } - - dynamic "disk" { - for_each = local.all_disks - content { - auto_delete = lookup(disk.value, "auto_delete", null) - boot = lookup(disk.value, "boot", null) - device_name = lookup(disk.value, "device_name", null) - disk_name = lookup(disk.value, "disk_name", null) - disk_size_gb = lookup(disk.value, "disk_size_gb", lookup(disk.value, "disk_type", null) == "local-ssd" ? "375" : null) - disk_type = lookup(disk.value, "disk_type", null) - interface = lookup(disk.value, "interface", lookup(disk.value, "disk_type", null) == "local-ssd" ? "NVME" : null) - mode = lookup(disk.value, "mode", null) - source = lookup(disk.value, "source", null) - source_image = lookup(disk.value, "source_image", null) - type = lookup(disk.value, "disk_type", null) == "local-ssd" ? "SCRATCH" : "PERSISTENT" - labels = (lookup(disk.value, "source", null) != null || lookup(disk.value, "disk_type", null) == "local-ssd") ? null : lookup(disk.value, "disk_labels", null) - resource_manager_tags = lookup(disk.value, "disk_resource_manager_tags", {}) - - dynamic "disk_encryption_key" { - for_each = compact([var.disk_encryption_key == null ? null : 1]) - content { - kms_key_self_link = var.disk_encryption_key - } - } - } - } - - network_interface { - network = var.network - subnetwork = var.subnetwork - subnetwork_project = var.subnetwork_project - network_ip = try(coalesce(var.network_ip), null) - nic_type = local.nic_type - stack_type = var.stack_type - dynamic "access_config" { - for_each = var.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = var.ipv6_access_config - content { - network_tier = ipv6_access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = local.alias_ip_range_enabled ? [var.alias_ip_range] : [] - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - } - - dynamic "network_interface" { - for_each = var.additional_networks - content { - network = network_interface.value.network - subnetwork = network_interface.value.subnetwork - subnetwork_project = network_interface.value.subnetwork_project - network_ip = try(coalesce(network_interface.value.network_ip), null) - nic_type = try(coalesce(network_interface.value.nic_type), null) - dynamic "access_config" { - for_each = network_interface.value.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = network_interface.value.ipv6_access_config - content { - network_tier = ipv6_access_config.value.network_tier - } - } - } - } - - network_performance_config { - total_egress_bandwidth_tier = coalesce(var.total_egress_bandwidth_tier, "DEFAULT") - } - - lifecycle { - create_before_destroy = "true" - } - - scheduling { - preemptible = local.preemptible - provisioning_model = local.provisioning_model - automatic_restart = local.automatic_restart - on_host_maintenance = local.on_host_maintenance - instance_termination_action = var.instance_termination_action - - dynamic "max_run_duration" { - for_each = var.max_run_duration != null ? [var.max_run_duration] : [] - content { - seconds = max_run_duration.value - } - } - } - - dynamic "reservation_affinity" { - for_each = var.reservation_affinity != null ? [var.reservation_affinity] : [] - content { - type = reservation_affinity.value.type - } - } - - advanced_machine_features { - enable_nested_virtualization = var.advanced_machine_features.enable_nested_virtualization - threads_per_core = var.advanced_machine_features.threads_per_core - turbo_mode = var.advanced_machine_features.turbo_mode - visible_core_count = var.advanced_machine_features.visible_core_count - performance_monitoring_unit = var.advanced_machine_features.performance_monitoring_unit - enable_uefi_networking = var.advanced_machine_features.enable_uefi_networking - } - - dynamic "shielded_instance_config" { - for_each = local.shielded_vm_configs - content { - enable_secure_boot = lookup(var.shielded_instance_config, "enable_secure_boot", shielded_instance_config.value) - enable_vtpm = lookup(var.shielded_instance_config, "enable_vtpm", shielded_instance_config.value) - enable_integrity_monitoring = lookup(var.shielded_instance_config, "enable_integrity_monitoring", shielded_instance_config.value) - } - } - - confidential_instance_config { - enable_confidential_compute = var.enable_confidential_vm - } - - dynamic "guest_accelerator" { - for_each = local.gpu_enabled ? [var.gpu] : [] - content { - type = guest_accelerator.value.type - count = guest_accelerator.value.count - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf deleted file mode 100644 index 69f8d3b98c..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "self_link" { - description = "Self-link of instance template" - value = google_compute_instance_template.tpl.self_link -} - -output "name" { - description = "Name of instance template" - value = google_compute_instance_template.tpl.name -} - -output "tags" { - description = "Tags that will be associated with instance(s)" - value = google_compute_instance_template.tpl.tags -} - -output "service_account" { - description = "value" - value = google_compute_instance_template.tpl.service_account[0] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf deleted file mode 100644 index c285c3fea5..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf +++ /dev/null @@ -1,398 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - type = string - description = "The GCP project ID" - default = null -} - -variable "name_prefix" { - description = "Name prefix for the instance template" - type = string -} - -variable "machine_type" { - description = "Machine type to create, e.g. n1-standard-1" - type = string - default = "n1-standard-1" -} - -variable "min_cpu_platform" { - description = "Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform" - type = string - default = null -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example" - type = string - default = "false" -} - -variable "tags" { - type = list(string) - description = "Network tags, provided as a list" - default = [] -} - -variable "labels" { - type = map(string) - description = "Labels, provided as a map" - default = {} -} - -variable "preemptible" { - type = bool - description = "Allow the instance to be preempted" - default = false -} - -variable "spot" { - description = <<-EOD - Provision as a SPOT preemptible instance. - See https://cloud.google.com/compute/docs/instances/spot for more details. - EOD - type = bool - default = false -} - -variable "instance_termination_action" { - description = <<-EOD - Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'. - See https://cloud.google.com/compute/docs/instances/spot for more details. - EOD - type = string - default = null -} - -variable "automatic_restart" { - type = bool - description = "(Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user)." - default = true -} - -variable "on_host_maintenance" { - type = string - description = "Instance availability Policy" - default = "MIGRATE" -} - -variable "region" { - type = string - description = "Region where the instance template should be created." - nullable = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -####### -# disk -####### -variable "source_image" { - description = "Source disk image. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." - type = string - default = "" -} - -variable "source_image_family" { - description = "Source image family. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." - type = string - default = "centos-7" -} - -variable "source_image_project" { - description = "Project where the source image comes from. The default project contains CentOS images." - type = string - default = "centos-cloud" -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = string - default = "100" -} - -variable "disk_type" { - description = "Boot disk type, can be either pd-ssd, local-ssd, or pd-standard" - type = string - default = "pd-standard" -} - -variable "disk_labels" { - description = "Labels to be assigned to boot disk, provided as a map" - type = map(string) - default = {} -} - -variable "disk_encryption_key" { - description = "The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance" - type = string - default = null -} - -variable "auto_delete" { - description = "Whether or not the boot disk should be auto-deleted" - type = string - default = "true" -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - description = "List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name" - type = list(object({ - source = optional(string) - disk_name = optional(string) - device_name = string - auto_delete = bool - boot = bool - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = map(string) - disk_resource_manager_tags = map(string) - })) - default = [] -} - -#################### -# network_interface -#################### -variable "network" { - description = "The name or self_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks." - type = string - default = "" -} - -variable "nic_type" { - description = "The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO_NET." - type = string - default = null -} - -variable "subnetwork" { - description = "The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided." - type = string - default = "" -} - -variable "subnetwork_project" { - description = "The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used." - type = string - default = null -} - -variable "network_ip" { - description = "Private IP address to assign to the instance if desired." - type = string - default = "" -} - -variable "stack_type" { - description = "The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4_ONLY." - type = string - default = null -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - })) -} - -variable "total_egress_bandwidth_tier" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 6.41 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.41 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance](#module\_instance) | ../instance | n/a | -| [template](#module\_template) | ../instance_template | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | -| [login\_nodes](#input\_login\_nodes) | Slurm login instance definitions. | 
object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  })
| n/a | yes | -| [network\_storage](#input\_network\_storage) | Storage to mounted on login instances
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | -| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path to directory in the bucket for configs | `string` | n/a | yes | -| [slurm\_bucket\_name](#input\_slurm\_bucket\_name) | Name of the bucket for configs | `string` | n/a | yes | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name | `string` | n/a | yes | -| [startup\_scripts](#input\_startup\_scripts) | List of scripts to be ran on login VMs startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | -| [startup\_scripts\_timeout](#input\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script. If any script exceeds this timeout,
then the instance setup process is considered failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instances](#output\_instances) | VM instances of login nodes | -| [service\_account](#output\_service\_account) | Service Account used by login VMs | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf deleted file mode 100644 index 605461f7e6..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "template" { - source = "../instance_template" - - project_id = var.project_id - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = "login" - slurm_bucket_path = var.slurm_bucket_path - name_prefix = local.name - - additional_disks = var.login_nodes.additional_disks - bandwidth_tier = var.login_nodes.bandwidth_tier - can_ip_forward = var.login_nodes.can_ip_forward - advanced_machine_features = var.login_nodes.advanced_machine_features - disk_auto_delete = var.login_nodes.disk_auto_delete - disk_labels = var.login_nodes.disk_labels - disk_resource_manager_tags = var.login_nodes.disk_resource_manager_tags - disk_size_gb = var.login_nodes.disk_size_gb - disk_type = var.login_nodes.disk_type - enable_confidential_vm = var.login_nodes.enable_confidential_vm - enable_oslogin = var.login_nodes.enable_oslogin - enable_shielded_vm = var.login_nodes.enable_shielded_vm - gpu = var.login_nodes.gpu - labels = var.login_nodes.labels - machine_type = var.login_nodes.machine_type - metadata = merge(var.login_nodes.metadata, { - "universe_domain" = var.universe_domain, - "slurm_login_group" = local.name - }) - min_cpu_platform = var.login_nodes.min_cpu_platform - on_host_maintenance = var.login_nodes.on_host_maintenance - preemptible = var.login_nodes.preemptible - region = var.login_nodes.region - resource_manager_tags = var.login_nodes.resource_manager_tags - service_account = var.login_nodes.service_account - shielded_instance_config = var.login_nodes.shielded_instance_config - source_image_family = var.login_nodes.source_image_family - source_image_project = var.login_nodes.source_image_project - source_image = var.login_nodes.source_image - spot = var.login_nodes.spot - subnetwork = var.login_nodes.subnetwork - tags = concat([var.slurm_cluster_name], var.login_nodes.tags) - termination_action = var.login_nodes.termination_action - - internal_startup_script = var.internal_startup_script -} - -module "instance" { - source = "../instance" - - access_config = var.login_nodes.access_config - hostname = "${var.slurm_cluster_name}-${local.name}" - - project_id = var.project_id - - instance_template = module.template.self_link - num_instances = var.login_nodes.num_instances - - additional_networks = var.login_nodes.additional_networks - region = var.login_nodes.region - static_ips = var.login_nodes.static_ips - subnetwork = var.login_nodes.subnetwork - zone = var.login_nodes.zone - - replace_trigger = var.replace_trigger -} - -resource "google_storage_bucket_object" "startup_scripts" { - for_each = { - for s in var.startup_scripts : format( - "slurm-login-%s-script-%s", local.name, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_") - ) => s.content - } - - bucket = var.slurm_bucket_name - name = "${var.slurm_bucket_dir}/${each.key}" - content = each.value - source_md5hash = md5(each.value) -} - -locals { - name = var.login_nodes.group_name # short hand - - config = { - group_name = local.name - startup_scripts_timeout = var.startup_scripts_timeout - network_storage = var.network_storage - } -} - -resource "google_storage_bucket_object" "config" { - bucket = var.slurm_bucket_name - name = "${var.slurm_bucket_dir}/login_group_configs/${local.name}.yaml" - content = yamlencode(local.config) - source_md5hash = md5(yamlencode(local.config)) - - # To ensure that login group "is not ready" until all startup scripts are written down - depends_on = [google_storage_bucket_object.startup_scripts] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf deleted file mode 100644 index 04de18a188..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "service_account" { - value = module.template.service_account - description = "Service Account used by login VMs" -} - -output "instances" { - value = module.instance.slurm_instances - description = "VM instances of login nodes" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf deleted file mode 100644 index 3efd862942..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "slurm_cluster_name" { - type = string - description = "Cluster name" -} - -variable "slurm_bucket_path" { - type = string - description = "GCS Bucket URI of Slurm cluster file storage." -} - - -variable "slurm_bucket_name" { - type = string - description = "Name of the bucket for configs" -} - -variable "slurm_bucket_dir" { - type = string - description = "Path to directory in the bucket for configs" -} - - -variable "universe_domain" { - description = "Domain address for alternate API universe" - type = string - default = "googleapis.com" -} - -variable "login_nodes" { - description = "Slurm login instance definitions." - type = object({ - group_name = string - access_config = optional(list(object({ - nat_ip = string - network_tier = string - }))) - additional_disks = optional(list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string), {}) - auto_delete = optional(bool, true) - boot = optional(bool, false) - disk_resource_manager_tags = optional(map(string), {}) - })), []) - additional_networks = optional(list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })), []) - bandwidth_tier = optional(string, "platform_default") - can_ip_forward = optional(bool, false) - disk_auto_delete = optional(bool, true) - disk_labels = optional(map(string), {}) - disk_resource_manager_tags = optional(map(string), {}) - disk_size_gb = optional(number) - disk_type = optional(string, "n1-standard-1") - enable_confidential_vm = optional(bool, false) - enable_oslogin = optional(bool, true) - enable_shielded_vm = optional(bool, false) - gpu = optional(object({ - count = number - type = string - })) - labels = optional(map(string), {}) - machine_type = optional(string) - advanced_machine_features = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - metadata = optional(map(string), {}) - min_cpu_platform = optional(string) - num_instances = optional(number, 1) - on_host_maintenance = optional(string) - preemptible = optional(bool, false) - region = optional(string) - resource_manager_tags = optional(map(string), {}) - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - shielded_instance_config = optional(object({ - enable_integrity_monitoring = optional(bool, true) - enable_secure_boot = optional(bool, true) - enable_vtpm = optional(bool, true) - })) - source_image_family = optional(string) - source_image_project = optional(string) - source_image = optional(string) - static_ips = optional(list(string), []) - subnetwork = string - spot = optional(bool, false) - tags = optional(list(string), []) - zone = optional(string) - termination_action = optional(string) - }) -} - - -variable "startup_scripts" { - description = "List of scripts to be ran on login VMs startup." - type = list(object({ - filename = string - content = string - })) - default = [] -} - -variable "startup_scripts_timeout" { - description = < - -- [Module: Slurm Nodeset (TPU)](#module-slurm-nodeset-tpu) - - [Overview](#overview) - - [Module API](#module-api) - - - -## Overview - -This is a submodule of [slurm_cluster](../../../slurm_cluster/README.md). It -creates a Slurm TPU nodeset for [slurm_partition](../slurm_partition/README.md). - -## Module API - -For the terraform module API reference, please see -[README_TF.md](./README_TF.md). - - -Copyright (C) SchedMD LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.2 | -| [google](#requirement\_google) | >= 3.53 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.53 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.nodeset_tpu](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_subnetwork.nodeset_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | -| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | -| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `""` | no | -| [enable\_public\_ip](#input\_enable\_public\_ip) | Enables IP address to access the Internet. | `bool` | `false` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of nodes allowed in this partition to be created dynamically. | `number` | `0` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | -| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. Not needed if you use accelerator\_config | `string` | `null` | no | -| [nodeset\_name](#input\_nodeset\_name) | Name of Slurm nodeset. | `string` | n/a | yes | -| [preemptible](#input\_preemptible) | Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details. | `bool` | `false` | no | -| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `true` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the TPU-vm.
If none is given, the default service account and scopes will be used. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | -| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | n/a | yes | -| [zone](#input\_zone) | Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Nodeset details. | -| [nodeset\_name](#output\_nodeset\_name) | Nodeset name. | -| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf deleted file mode 100644 index 1a6a9cfba1..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########### -# NODESET # -########### - -locals { - node_conf_hw = { - Mem334CPU96 = { - CPUs = 96 - Boards = 1 - Sockets = 2 - CoresPerSocket = 24 - ThreadsPerCore = 2 - RealMemory = 307200 - } - Mem400CPU240 = { - CPUs = 240 - Boards = 1 - Sockets = 2 - CoresPerSocket = 60 - ThreadsPerCore = 2 - RealMemory = 400000 - } - } - node_conf_mappings = { - "v2" = local.node_conf_hw.Mem334CPU96 - "v3" = local.node_conf_hw.Mem334CPU96 - "v4" = local.node_conf_hw.Mem400CPU240 - } - simple_nodes = ["v2-8", "v3-8", "v4-8"] -} - -locals { - snetwork = data.google_compute_subnetwork.nodeset_subnetwork.name - region = join("-", slice(split("-", var.zone), 0, 2)) - tpu_fam = var.accelerator_config.version != "" ? lower(var.accelerator_config.version) : split("-", var.node_type)[0] - #If subnetwork is specified and it does not have private_ip_google_access, we need to have public IPs on the TPU - #if no subnetwork is specified, the default one will be used, this does not have private_ip_google_access so we need public IPs too - pub_need = !data.google_compute_subnetwork.nodeset_subnetwork.private_ip_google_access - can_preempt = var.node_type != null ? contains(local.simple_nodes, var.node_type) : false - nodeset_tpu = { - nodeset_name = var.nodeset_name - node_conf = local.node_conf_mappings[local.tpu_fam] - node_type = var.node_type - accelerator_config = var.accelerator_config - tf_version = var.tf_version - preemptible = local.can_preempt ? var.preemptible : false - reserved = var.reserved - node_count_dynamic_max = var.node_count_dynamic_max - node_count_static = var.node_count_static - enable_public_ip = var.enable_public_ip - zone = var.zone - service_account = var.service_account != null ? var.service_account : local.service_account - preserve_tpu = local.can_preempt ? var.preserve_tpu : false - data_disks = var.data_disks - docker_image = var.docker_image != "" ? var.docker_image : "us-docker.pkg.dev/schedmd-slurm-public/tpu/slurm-gcp-6-9:tf-${var.tf_version}" - subnetwork = local.snetwork - network_storage = var.network_storage - } - - service_account = { - email = try(var.service_account.email, null) - scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) - } -} - -data "google_compute_subnetwork" "nodeset_subnetwork" { - name = var.subnetwork - region = local.region - project = var.project_id - - self_link = ( - length(regexall("/projects/([^/]*)", var.subnetwork)) > 0 - && length(regexall("/regions/([^/]*)", var.subnetwork)) > 0 - ? var.subnetwork - : null - ) -} - -resource "null_resource" "nodeset_tpu" { - triggers = { - nodeset = sha256(jsonencode(local.nodeset_tpu)) - } - lifecycle { - precondition { - condition = sum([var.node_count_dynamic_max, var.node_count_static]) > 0 - error_message = "Sum of node_count_dynamic_max and node_count_static must be > 0." - } - precondition { - condition = !(var.preemptible && var.reserved) - error_message = "Nodeset cannot be preemptible and reserved at the same time." - } - precondition { - condition = !(var.subnetwork == null && !var.enable_public_ip) - error_message = "Using the default subnetwork for the TPU nodeset requires enable_public_ip set to true." - } - precondition { - condition = !(var.subnetwork != null && (local.pub_need && !var.enable_public_ip)) - error_message = "The subnetwork specified does not have Private Google Access enabled. This is required when enable_public_ip is set to false." - } - precondition { - condition = !(var.node_type == null && (var.accelerator_config.topology == "" && var.accelerator_config.version == "")) - error_message = "Either a node type or an accelerator_config must be provided." - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf deleted file mode 100644 index fce700d567..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "nodeset_name" { - description = "Nodeset name." - value = local.nodeset_tpu.nodeset_name -} - -output "nodeset" { - description = "Nodeset details." - value = local.nodeset_tpu -} - -output "service_account" { - description = "Service account object, includes email and scopes." - value = local.service_account -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf deleted file mode 100644 index a8c470dec9..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "nodeset_name" { - description = "Name of Slurm nodeset." - type = string - - validation { - condition = can(regex("^[a-z](?:[a-z0-9]{0,14})$", var.nodeset_name)) - error_message = "Variable 'nodeset_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,14})$'." - } -} - -variable "node_type" { - description = "Specify a node type to base the vm configuration upon it. Not needed if you use accelerator_config" - type = string - default = null -} - -variable "accelerator_config" { - description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." - type = object({ - topology = string - version = string - }) - default = { - topology = "" - version = "" - } - validation { - condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], upper(var.accelerator_config.version)) - error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" - } - validation { - condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) - error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." - } -} - -variable "docker_image" { - description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" - type = string - default = "" -} - -variable "tf_version" { - description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." - type = string -} - -variable "zone" { - description = "Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it." - type = string - - validation { - condition = can(coalesce(var.zone)) - error_message = "Zone cannot be null or empty." - } -} - -variable "preemptible" { - description = "Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details." - type = bool - default = false -} - -variable "reserved" { - description = "Specify whether TPU-vms in this nodeset are created under a reservation." - type = bool - default = false -} - -variable "preserve_tpu" { - description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" - type = bool - default = true -} - -variable "node_count_static" { - description = "Number of nodes to be statically created." - type = number - default = 0 - - validation { - condition = var.node_count_static >= 0 - error_message = "Value must be >= 0." - } -} - -variable "node_count_dynamic_max" { - description = "Maximum number of nodes allowed in this partition to be created dynamically." - type = number - default = 0 - - validation { - condition = var.node_count_dynamic_max >= 0 - error_message = "Value must be >= 0." - } -} - -variable "enable_public_ip" { - description = "Enables IP address to access the Internet." - type = bool - default = false -} - -variable "data_disks" { - type = list(string) - description = "The data disks to include in the TPU node" - default = [] -} - -variable "subnetwork" { - description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." - type = string -} - -variable "service_account" { - type = object({ - email = string - scopes = set(string) - }) - description = < -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | -| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | -| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | -| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | -| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | -| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "v0.7.2")
  })
| `{}` | no | -| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "v0.11.4")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | -| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md deleted file mode 100644 index 1957899617..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [helm](#provider\_helm) | ~> 2.17 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | -| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | -| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | -| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | -| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | -| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | -| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | -| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | -| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | -| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | -| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | -| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | -| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | -| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | -| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | -| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | -| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | -| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | -| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | -| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | -| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | -| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | -| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | -| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | -| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | -| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | -| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | -| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | -| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | -| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf deleted file mode 100644 index bd2383b772..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "helm_release" "apply_chart" { - # Required Identification - name = var.release_name - chart = var.chart_name - - # Chart Source & Version - repository = var.chart_repository - version = var.chart_version - devel = var.devel - - # Target Namespace - namespace = var.namespace - create_namespace = var.create_namespace - - # Values Configuration - values = var.values_yaml - - dynamic "set" { - for_each = var.set_values - content { - name = set.value.name - value = set.value.value - type = set.value.type - } - } - - # Installation/Upgrade Behavior - description = var.description - atomic = var.atomic - cleanup_on_fail = var.cleanup_on_fail - dependency_update = var.dependency_update - disable_crd_hooks = var.disable_crd_hooks - disable_openapi_validation = var.disable_openapi_validation - disable_webhooks = var.disable_webhooks - force_update = var.force_update - lint = var.lint - max_history = var.max_history - recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI - render_subchart_notes = var.render_subchart_notes - reset_values = var.reset_values - reuse_values = var.reuse_values - skip_crds = var.skip_crds - timeout = var.timeout - wait = var.wait - wait_for_jobs = var.wait_for_jobs - - # Verification & Credentials - keyring = var.keyring - pass_credentials = var.pass_credentials - verify = var.verify - - # Post Rendering - dynamic "postrender" { - # Only include the block if var.postrender is not null - for_each = var.postrender == null ? [] : [var.postrender] - content { - binary_path = postrender.value.binary_path - } - } - -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml deleted file mode 100644 index e18197e2b7..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf deleted file mode 100644 index 04e8e214fc..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -# --- Required --- -variable "release_name" { - description = "Name of the Helm release." - type = string -} - -variable "chart_name" { - description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." - type = string -} - -# --- Chart Location & Version --- -variable "chart_repository" { - description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." - type = string - default = null -} - -variable "chart_version" { - description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." - type = string - default = null -} - -variable "devel" { - description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." - type = bool - default = false -} - -# --- Namespace --- -variable "namespace" { - description = "Kubernetes namespace to install the Helm release into." - type = string - default = "default" -} - -variable "create_namespace" { - description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." - type = bool - default = true # Common convenience setting -} - -# --- Values Customization --- -variable "values_yaml" { - description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." - type = list(string) - default = [] -} - -variable "set_values" { - description = "List of objects defining values to set ('helm install --set')." - type = list(object({ - name = string # Path to the value (e.g., 'service.type', 'replicaCount') - value = string # The value to set - type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') - })) - default = [] -} - -# --- Installation/Upgrade Behavior --- -variable "description" { - description = "Set an optional description for the Helm release." - type = string - default = null -} - -variable "atomic" { - description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." - type = bool - default = false -} - -variable "wait" { - description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." - type = bool - default = true # Often a good default for dependencies -} - -variable "wait_for_jobs" { - description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." - type = bool - default = false # Helm CLI default is false -} - -variable "timeout" { - description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." - type = number - default = 300 # 5 minutes (Helm CLI default) -} - -variable "cleanup_on_fail" { - description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." - type = bool - default = false -} - -variable "dependency_update" { - description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." - type = bool - default = false -} - -variable "disable_crd_hooks" { - description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." - type = bool - default = false -} - -variable "disable_openapi_validation" { - description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." - type = bool - default = false -} - -variable "disable_webhooks" { - description = "Prevent hooks from running ('helm install --no-hooks')." - type = bool - default = false -} - -variable "force_update" { - description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." - type = bool - default = false -} - -variable "lint" { - description = "Run the helm chart linter during the plan ('helm lint')." - type = bool - default = false -} - -variable "max_history" { - description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." - type = number - default = null # Terraform provider defaults to Helm's default (usually 10) -} - -variable "recreate_pods" { - description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." - type = bool - default = false -} - -variable "render_subchart_notes" { - description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." - type = bool - default = false -} - -variable "reset_values" { - description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." - type = bool - default = false -} - -variable "reuse_values" { - description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." - type = bool - default = false # Helm CLI default is false -} - -variable "skip_crds" { - description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." - type = bool - default = false -} - -# --- Verification & Credentials --- -variable "keyring" { - description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." - type = string - default = null # Defaults to Helm's default keyring location -} - -variable "pass_credentials" { - description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." - type = bool - default = false -} - -variable "verify" { - description = "Verify the package before installing it ('helm install --verify')." - type = bool - default = false -} - -# --- Advanced Rendering --- -variable "postrender" { - description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." - type = object({ - binary_path = string # Path to the post-renderer executable - }) - default = null # Disabled by default -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf deleted file mode 100644 index 09d912e2c9..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md deleted file mode 100644 index 46bfe51a32..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md +++ /dev/null @@ -1,40 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.23 | - -## Providers - -| Name | Version | -|------|---------| -| [kubernetes](#provider\_kubernetes) | ~> 2.23 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubernetes_manifest.apply_manifests](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | -| [field\_manager](#input\_field\_manager) | (Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts. | 
object({
    name            = optional(string, null)
    force_conflicts = optional(bool, false)
  })
| `null` | no | -| [resource\_timeouts](#input\_resource\_timeouts) | (Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met. | 
object({
    create = optional(string, null)
    update = optional(string, null)
    delete = optional(string, null)
  })
| 
{
  "create": "15m",
  "delete": "5m",
  "update": "10m"
}
| no | -| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `""` | no | -| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | -| [wait\_for\_fields](#input\_wait\_for\_fields) | (Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern. | `map(string)` | `{}` | no | -| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf deleted file mode 100644 index f97f26038d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - yaml_separator = "\n---" - - # --- 1. Determine the primary source of YAML content --- - # Prioritize 'content' variable if provided - primary_content_body = var.content != "" ? var.content : null - - # --- 2. Handle 'source_path' based on its type (File vs. Directory) --- - - # Check if source_path is a directory (indicated by trailing slash) - is_directory = endswith(var.source_path, "/") - directory_absolute_path = local.is_directory ? abspath(var.source_path) : null - - # Check if source_path is a single yaml or tftpl file (only if not a directory) - is_single_file = !local.is_directory && ( - length(regexall("\\.yaml$", lower(var.source_path))) > 0 || - length(regexall("\\.tftpl$", lower(var.source_path))) > 0 - ) - single_file_raw_content = local.is_single_file ? ( - length(regexall("\\.tftpl$", lower(var.source_path))) > 0 ? - templatefile(abspath(var.source_path), var.template_vars) : - file(abspath(var.source_path)) - ) : null - - # Docs from primary_content_body - docs_from_primary_source = [ - for doc in split(local.yaml_separator, coalesce(local.primary_content_body, local.single_file_raw_content, "")) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - - # Docs from .yaml files in a directory - directory_yaml_files = local.is_directory ? fileset(local.directory_absolute_path, "*.yaml") : [] - docs_from_directory_yamls = flatten([ - for file_name in local.directory_yaml_files : - [ - for doc in split(local.yaml_separator, file(format("%s/%s", local.directory_absolute_path, file_name))) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - ]) - - # Docs from .tftpl files in a directory - directory_template_files = local.is_directory ? fileset(local.directory_absolute_path, "*.tftpl") : [] - docs_from_directory_templates = flatten([ - for file_name in local.directory_template_files : - [ - for doc in split(local.yaml_separator, templatefile(format("%s/%s", local.directory_absolute_path, file_name), var.template_vars)) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - ]) - - all_parsed_docs = concat( - local.docs_from_primary_source, - local.docs_from_directory_yamls, - local.docs_from_directory_templates - ) - - # --- 5. Create the final map for `for_each` (keys must be unique strings) --- - docs_map = tomap({ - for index, doc in local.all_parsed_docs : index => doc - if length(trimspace(doc)) > 0 - }) -} - -# Apply all manifest files dynamically -resource "kubernetes_manifest" "apply_manifests" { - for_each = local.docs_map - manifest = yamldecode(each.value) - timeouts { - create = var.resource_timeouts.create - update = var.resource_timeouts.update - delete = var.resource_timeouts.delete - } - - dynamic "wait" { - for_each = var.wait_for_rollout ? [1] : [] - content { - rollout = var.wait_for_rollout - fields = var.wait_for_fields - } - } - - # Configure the 'field_manager' block dynamically - dynamic "field_manager" { - for_each = var.field_manager != null ? [var.field_manager] : [] - content { - name = field_manager.value.name - force_conflicts = field_manager.value.force_conflicts - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml deleted file mode 100644 index e18197e2b7..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf deleted file mode 100644 index 0b846189ea..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -variable "content" { - description = "The YAML body to apply to gke cluster." - type = string - default = null -} - -variable "source_path" { - description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." - type = string - default = "" -} - -variable "template_vars" { - description = "The values to populate template file(s) with." - type = any - default = null -} - -variable "wait_for_rollout" { - description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." - type = bool - default = true -} - - -variable "wait_for_fields" { - description = "(Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern." - type = map(string) - default = {} -} - -variable "resource_timeouts" { - description = "(Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met." - type = object({ - create = optional(string, null) - update = optional(string, null) - delete = optional(string, null) - }) - default = { - create = "15m" # Default create timeout, also covers waiting for initial conditions - update = "10m" # Default update timeout, also covers waiting for update conditions - delete = "5m" # Default delete timeout - } -} - -variable "field_manager" { - description = "(Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts." - type = object({ - name = optional(string, null) - force_conflicts = optional(bool, false) - }) - default = null -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf deleted file mode 100644 index 61786b06de..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - # Defines the providers that this module depends on and their versions. - required_providers { - kubernetes = { - source = "hashicorp/kubernetes" - version = "~> 2.23" - } - } - required_version = ">= 1.3" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/main.tf b/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/main.tf deleted file mode 100644 index 8db4870452..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/management/dependencies-installer/main.tf +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] - project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] - - install_gpu_operator = try(var.gpu_operator.install, false) - install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) -} - -data "google_container_cluster" "gke_cluster" { - project = local.project_id - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -module "install_kueue" { - source = "./helm_install" - depends_on = [var.gke_cluster_exists] - - release_name = "kueue" - - chart_name = "oci://registry.k8s.io/kueue/charts/kueue" - chart_version = var.kueue.version # Specify your desired Kueue version - - create_namespace = true # Helm can also create the namespace - wait = true - timeout = 600 # seconds -} - -module "install_jobset" { - source = "./helm_install" - depends_on = [var.gke_cluster_exists, module.install_kueue] - release_name = "jobset-controller" # The release name for your JobSet installation - chart_name = "oci://registry.k8s.io/jobset/charts/jobset" # The Helm repository URL for nvidia charts - chart_version = var.jobset.version - create_namespace = true - namespace = "jobset-system" -} - -module "install_nvidia_dra_driver" { - count = local.install_nvidia_dra_driver ? 1 : 0 - depends_on = [var.gke_cluster_exists] - source = "./helm_install" - - release_name = "nvidia-dra-driver-gpu" # The release name - chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts - chart_name = "nvidia-dra-driver-gpu" # The chart name - chart_version = var.nvidia_dra_driver.version # The chart version - namespace = "nvidia-dra-driver-gpu" # The target namespace - create_namespace = true # Equivalent to --create-namespace - - # Use the 'values' argument to pass the YAML content - # This corresponds to the -f <(cat < -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.2 | -| [google](#requirement\_google) | >= 6.40 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.40 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_global_address.private_ip_alloc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_global_address) | resource | -| [google_compute_network_peering_routes_config.private_vpc_peering_routes_gcnv](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering_routes_config) | resource | -| [google_service_networking_connection.private_vpc_connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_networking_connection) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [address](#input\_address) | The IP address or beginning of the address range allocated for the Private Service Access. | `string` | `null` | no | -| [deletion\_policy](#input\_deletion\_policy) | The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to supporting resources. Key-value pairs. | `map(string)` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to configure Private Service Access:
`projects//global/networks/`" | `string` | n/a | yes | -| [prefix\_length](#input\_prefix\_length) | The prefix length of the IP range allocated for the Private Service Access. | `number` | `16` | no | -| [project\_id](#input\_project\_id) | ID of project in which Private Service Access will be created. | `string` | n/a | yes | -| [service\_name](#input\_service\_name) | The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'. | `string` | `"servicenetworking.googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cidr\_range](#output\_cidr\_range) | CIDR range of the created google\_compute\_global\_address | -| [connect\_mode](#output\_connect\_mode) | Services that use Private Service Access typically specify connect\_mode
"PRIVATE\_SERVICE\_ACCESS". This output value sets connect\_mode and additionally
blocks terraform actions until the VPC connection has been created. | -| [private\_vpc\_connection\_peering](#output\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection that was created by the service provider. | -| [reserved\_ip\_range](#output\_reserved\_ip\_range) | Named IP range to be used by services connected with Private Service Access. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/main.tf b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/main.tf deleted file mode 100644 index 429e4d93f0..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/main.tf +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "private-service-access", ghpc_role = "network" }) -} - -locals { - split_network_id = split("/", var.network_id) - network_name = local.split_network_id[4] - network_project = local.split_network_id[1] -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_compute_global_address" "private_ip_alloc" { - provider = google - name = "global-psconnect-ip-${random_id.resource_name_suffix.hex}" - project = var.project_id - purpose = "VPC_PEERING" - address_type = "INTERNAL" - network = var.network_id - prefix_length = var.prefix_length - labels = local.labels - address = var.address -} - -resource "google_service_networking_connection" "private_vpc_connection" { - network = var.network_id - service = var.service_name - reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name] - deletion_policy = var.deletion_policy - update_on_creation_fail = var.deletion_policy == "ABANDON" ? true : null -} - -# Google Cloud NetApp Volumes need enablement of custom_route import and export -resource "google_compute_network_peering_routes_config" "private_vpc_peering_routes_gcnv" { - count = var.service_name == "netapp.servicenetworking.goog" ? 1 : 0 - project = local.network_project - network = local.network_name - peering = google_service_networking_connection.private_vpc_connection.peering - - export_custom_routes = true - import_custom_routes = true -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/metadata.yaml deleted file mode 100644 index 93e8b3970e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - servicenetworking.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/outputs.tf deleted file mode 100644 index 296f2e9140..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "private_vpc_connection_peering" { - description = "The name of the VPC Network peering connection that was created by the service provider." - sensitive = true - value = google_service_networking_connection.private_vpc_connection.peering -} - -output "connect_mode" { - description = <<-EOT - Services that use Private Service Access typically specify connect_mode - "PRIVATE_SERVICE_ACCESS". This output value sets connect_mode and additionally - blocks terraform actions until the VPC connection has been created. - EOT - value = "PRIVATE_SERVICE_ACCESS" - depends_on = [ - google_service_networking_connection.private_vpc_connection, - ] -} - -output "reserved_ip_range" { - description = "Named IP range to be used by services connected with Private Service Access." - value = google_compute_global_address.private_ip_alloc.name -} - -output "cidr_range" { - description = "CIDR range of the created google_compute_global_address" - value = "${google_compute_global_address.private_ip_alloc.address}/${google_compute_global_address.private_ip_alloc.prefix_length}" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/variables.tf deleted file mode 100644 index 4b0a3e796f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/variables.tf +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "address" { - description = "The IP address or beginning of the address range allocated for the Private Service Access." - type = string - default = null -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to configure Private Service Access: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "labels" { - description = "Labels to add to supporting resources. Key-value pairs." - type = map(string) -} - -variable "prefix_length" { - description = "The prefix length of the IP range allocated for the Private Service Access." - type = number - default = 16 -} - -variable "project_id" { - description = "ID of project in which Private Service Access will be created." - type = string -} - -variable "service_name" { - description = "The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'." - type = string - default = "servicenetworking.googleapis.com" -} - -variable "deletion_policy" { - description = "The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON." - type = string - default = null -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/versions.tf deleted file mode 100644 index df2914cdb9..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/network/private-service-access/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.40" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" - } - - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" - } - - required_version = ">= 1.2" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/new-project/README.md b/deletion-test/cluster/modules/embedded/community/modules/project/new-project/README.md deleted file mode 100644 index 5e5cabe9d5..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/new-project/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module allows you to create opinionated Google Cloud Platform projects. It -creates projects and configures aspects like Shared VPC connectivity, IAM -access, Service Accounts, and API enablement to follow best practices. - -This module is meant for use with Terraform 0.13. - -**Note:** This module has been removed from the Cluster Toolkit. The upstream module (`terraform-google-project-factory`) is now the recommended way to create and manage GCP projects. - -### Example - -```yaml -- id: project - source: github.com/terraform-google-modules/terraform-google-project-factory?rev=v17.0.0&depth=1 -``` - -This creates a new project with pre-defined project ID, a designated folder and -organization and associated billing account which will be used to pay for -services consumed. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [project\_factory](#module\_project\_factory) | terraform-google-modules/project-factory/google | ~> 11.3 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [activate\_api\_identities](#input\_activate\_api\_identities) | The list of service identities (Google Managed service account for the API) to force-create for the project (e.g. in order to grant additional roles).
APIs in this list will automatically be appended to `activate_apis`.
Not including the API in this list will follow the default behaviour for identity creation (which is usually when the first resource using the API is created).
Any roles (e.g. service agent role) must be explicitly listed. See https://cloud.google.com/iam/docs/understanding-roles#service-agent-roles-roles for a list of related roles. | 
list(object({
    api   = string
    roles = list(string)
  }))
| `[]` | no | -| [activate\_apis](#input\_activate\_apis) | The list of apis to activate within the project | `list(string)` | 
[
  "compute.googleapis.com",
  "serviceusage.googleapis.com",
  "storage.googleapis.com"
]
| no | -| [auto\_create\_network](#input\_auto\_create\_network) | Create the default network | `bool` | `false` | no | -| [billing\_account](#input\_billing\_account) | The ID of the billing account to associate this project with | `string` | n/a | yes | -| [bucket\_force\_destroy](#input\_bucket\_force\_destroy) | Force the deletion of all objects within the GCS bucket when deleting the bucket (optional) | `bool` | `false` | no | -| [bucket\_labels](#input\_bucket\_labels) | A map of key/value label pairs to assign to the bucket (optional) | `map(string)` | `{}` | no | -| [bucket\_location](#input\_bucket\_location) | The location for a GCS bucket to create (optional) | `string` | `"US"` | no | -| [bucket\_name](#input\_bucket\_name) | A name for a GCS bucket to create (in the bucket\_project project), useful for Terraform state (optional) | `string` | `""` | no | -| [bucket\_project](#input\_bucket\_project) | A project to create a GCS bucket (bucket\_name) in, useful for Terraform state (optional) | `string` | `""` | no | -| [bucket\_ula](#input\_bucket\_ula) | Enable Uniform Bucket Level Access | `bool` | `true` | no | -| [bucket\_versioning](#input\_bucket\_versioning) | Enable versioning for a GCS bucket to create (optional) | `bool` | `false` | no | -| [budget\_alert\_pubsub\_topic](#input\_budget\_alert\_pubsub\_topic) | The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}` | `string` | `null` | no | -| [budget\_alert\_spent\_percents](#input\_budget\_alert\_spent\_percents) | A list of percentages of the budget to alert on when threshold is exceeded | `list(number)` | 
[
  0.5,
  0.7,
  1
]
| no | -| [budget\_amount](#input\_budget\_amount) | The amount to use for a budget alert | `number` | `null` | no | -| [budget\_display\_name](#input\_budget\_display\_name) | The display name of the budget. If not set defaults to `Budget For ` | `string` | `null` | no | -| [budget\_monitoring\_notification\_channels](#input\_budget\_monitoring\_notification\_channels) | A list of monitoring notification channels in the form `[projects/{project_id}/notificationChannels/{channel_id}]`. A maximum of 5 channels are allowed. | `list(string)` | `[]` | no | -| [consumer\_quotas](#input\_consumer\_quotas) | The quotas configuration you want to override for the project. | 
list(object({
    service = string,
    metric  = string,
    limit   = string,
    value   = string,
  }))
| `[]` | no | -| [create\_project\_sa](#input\_create\_project\_sa) | Whether the default service account for the project shall be created | `bool` | `true` | no | -| [default\_network\_tier](#input\_default\_network\_tier) | Default Network Service Tier for resources created in this project. If unset, the value will not be modified. See https://cloud.google.com/network-tiers/docs/using-network-service-tiers and https://cloud.google.com/network-tiers. | `string` | `""` | no | -| [default\_service\_account](#input\_default\_service\_account) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | `string` | `"keep"` | no | -| [disable\_dependent\_services](#input\_disable\_dependent\_services) | Whether services that are enabled and which depend on this service should also be disabled when this service is destroyed. | `bool` | `true` | no | -| [disable\_services\_on\_destroy](#input\_disable\_services\_on\_destroy) | Whether project services will be disabled when the resources are destroyed | `bool` | `true` | no | -| [domain](#input\_domain) | The domain name (optional). | `string` | `""` | no | -| [enable\_shared\_vpc\_host\_project](#input\_enable\_shared\_vpc\_host\_project) | If this project is a shared VPC host project. If true, you must *not* set svpc\_host\_project\_id variable. Default is false. | `bool` | `false` | no | -| [folder\_id](#input\_folder\_id) | The ID of a folder to host this project | `string` | `""` | no | -| [grant\_services\_network\_role](#input\_grant\_services\_network\_role) | Whether or not to grant service agents the network roles on the host project | `bool` | `true` | no | -| [grant\_services\_security\_admin\_role](#input\_grant\_services\_security\_admin\_role) | Whether or not to grant Kubernetes Engine Service Agent the Security Admin role on the host project so it can manage firewall rules | `bool` | `false` | no | -| [group\_name](#input\_group\_name) | A group to control the project by being assigned group\_role (defaults to project editor) | `string` | `""` | no | -| [group\_role](#input\_group\_role) | The role to give the controlling group (group\_name) over the project (defaults to project editor) | `string` | `"roles/editor"` | no | -| [labels](#input\_labels) | Map of labels for project | `map(string)` | `{}` | no | -| [lien](#input\_lien) | Add a lien on the project to prevent accidental deletion | `bool` | `false` | no | -| [name](#input\_name) | The name for the project | `string` | `null` | no | -| [org\_id](#input\_org\_id) | The organization ID. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The ID to give the project. If not provided, the `name` will be used. | `string` | `""` | no | -| [project\_sa\_name](#input\_project\_sa\_name) | Default service account name for the project. | `string` | `"project-service-account"` | no | -| [random\_project\_id](#input\_random\_project\_id) | Adds a suffix of 4 random characters to the `project_id` | `bool` | `false` | no | -| [sa\_role](#input\_sa\_role) | A role to give the default Service Account for the project (defaults to none) | `string` | `""` | no | -| [shared\_vpc\_subnets](#input\_shared\_vpc\_subnets) | List of subnets fully qualified subnet IDs (ie. projects/$project\_id/regions/$region/subnetworks/$subnet\_id) | `list(string)` | `[]` | no | -| [svpc\_host\_project\_id](#input\_svpc\_host\_project\_id) | The ID of the host project which hosts the shared VPC | `string` | `""` | no | -| [usage\_bucket\_name](#input\_usage\_bucket\_name) | Name of a GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | -| [usage\_bucket\_prefix](#input\_usage\_bucket\_prefix) | Prefix in the GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | -| [vpc\_service\_control\_attach\_enabled](#input\_vpc\_service\_control\_attach\_enabled) | Whether the project will be attached to a VPC Service Control Perimeter | `bool` | `false` | no | -| [vpc\_service\_control\_perimeter\_name](#input\_vpc\_service\_control\_perimeter\_name) | The name of a VPC Service Control Perimeter to add the created project to | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [api\_s\_account](#output\_api\_s\_account) | API service account email | -| [api\_s\_account\_fmt](#output\_api\_s\_account\_fmt) | API service account email formatted for terraform use | -| [budget\_name](#output\_budget\_name) | The name of the budget if created | -| [domain](#output\_domain) | The organization's domain | -| [enabled\_api\_identities](#output\_enabled\_api\_identities) | Enabled API identities in the project | -| [enabled\_apis](#output\_enabled\_apis) | Enabled APIs in the project | -| [group\_email](#output\_group\_email) | The email of the G Suite group with group\_name | -| [project\_bucket\_self\_link](#output\_project\_bucket\_self\_link) | Project's bucket selfLink | -| [project\_bucket\_url](#output\_project\_bucket\_url) | Project's bucket url | -| [project\_id](#output\_project\_id) | ID of the project that was created | -| [project\_name](#output\_project\_name) | Name of the project that was created | -| [project\_number](#output\_project\_number) | Number of the project that was created | -| [service\_account\_display\_name](#output\_service\_account\_display\_name) | The display name of the default service account | -| [service\_account\_email](#output\_service\_account\_email) | The email of the default service account | -| [service\_account\_id](#output\_service\_account\_id) | The id of the default service account | -| [service\_account\_name](#output\_service\_account\_name) | The fully-qualified name of the default service account | -| [service\_account\_unique\_id](#output\_service\_account\_unique\_id) | The unique id of the default service account | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/README.md b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/README.md deleted file mode 100644 index 0f5c10c7e4..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/README.md +++ /dev/null @@ -1,111 +0,0 @@ -## Description - -Allows creation of service accounts for a Google Cloud Platform project. - -### Example - -```yaml -- id: service_acct - source: community/modules/project/service-account - settings: - project_id: $(vars.project_id) - name: instance_acct - project_roles: - - logging.logWriter - - monitoring.metricWriter - - storage.objectViewer -``` - -This creates a service account in GCP project "project_id" with the name -"instance_acct". It will have the 3 roles listed for all resources within the -project. - -### Usage with startup-script module - -When this module is used in conjunction with the [startup-script] module, the -service account must be granted (at least) read access to the bucket. This can -be achieved by granting project-wide access as shown above or by specifying the -service account as a bucket viewer in the startup-script module: - -```yaml -- id: service_acct - source: community/modules/project/service-account - settings: - project_id: $(vars.project_id) - name: instance_acct - project_roles: - - logging.logWriter - - monitoring.metricWriter -- id: script - source: modules/scripts/startup-script - settings: - bucket_viewers: - - $(service_acct.service_account_iam_email) -``` - -[startup-script]: ../../../../modules/scripts/startup-script/README.md - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [service\_account](#module\_service\_account) | terraform-google-modules/service-accounts/google | ~> 4.2 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [billing\_account\_id](#input\_billing\_account\_id) | If assigning billing role, specify a billing account (default is to assign at the organizational level). | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment (will be prepended to service account name) | `string` | n/a | yes | -| [description](#input\_description) | Description of the created service account. | `string` | `"Service Account"` | no | -| [descriptions](#input\_descriptions) | Deprecated; create single service accounts using var.description. | `list(string)` | `null` | no | -| [display\_name](#input\_display\_name) | Display name of the created service account. | `string` | `"Service Account"` | no | -| [generate\_keys](#input\_generate\_keys) | Generate keys for service account. | `bool` | `false` | no | -| [grant\_billing\_role](#input\_grant\_billing\_role) | Grant billing user role. | `bool` | `false` | no | -| [grant\_xpn\_roles](#input\_grant\_xpn\_roles) | Grant roles for shared VPC management. | `bool` | `true` | no | -| [name](#input\_name) | Name of the service account to create. | `string` | n/a | yes | -| [names](#input\_names) | Deprecated; create single service accounts using var.name. | `list(string)` | `null` | no | -| [org\_id](#input\_org\_id) | Id of the organization for org-level roles. | `string` | `""` | no | -| [prefix](#input\_prefix) | Deprecated; prefix now set using var.deployment\_name | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | -| [project\_roles](#input\_project\_roles) | List of roles to grant to service account (e.g. "storage.objectViewer" or "compute.instanceAdmin.v1" | `list(string)` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [key](#output\_key) | Service account key (if creation was requested) | -| [service\_account\_email](#output\_service\_account\_email) | Service account e-mail address | -| [service\_account\_iam\_email](#output\_service\_account\_iam\_email) | Service account IAM binding format (serviceAccount:name@example.com) | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/main.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/main.tf deleted file mode 100644 index e8a69be642..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - display_name = "${var.display_name} (${var.deployment_name})" - description = "${var.description} (${var.deployment_name})" -} - -module "service_account" { - source = "terraform-google-modules/service-accounts/google" - version = "~> 4.2" - - billing_account_id = var.billing_account_id - description = local.description - display_name = local.display_name - generate_keys = var.generate_keys - grant_billing_role = var.grant_billing_role - grant_xpn_roles = var.grant_xpn_roles - names = [var.name] - org_id = var.org_id - prefix = var.deployment_name - project_id = var.project_id - project_roles = [for role in var.project_roles : "${var.project_id}=>roles/${role}"] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/metadata.yaml deleted file mode 100644 index c4dcdffdf4..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/outputs.tf deleted file mode 100644 index f9c9be05c8..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/outputs.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "key" { - description = "Service account key (if creation was requested)" - value = module.service_account.key -} - -output "service_account_email" { - description = "Service account e-mail address" - value = module.service_account.email - depends_on = [ - module.service_account, - ] -} - -output "service_account_iam_email" { - description = "Service account IAM binding format (serviceAccount:name@example.com)" - value = module.service_account.iam_email - depends_on = [ - module.service_account, - ] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/variables.tf deleted file mode 100644 index 53267f47e7..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/variables.tf +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "billing_account_id" { - description = "If assigning billing role, specify a billing account (default is to assign at the organizational level)." - type = string - default = "" -} - -variable "deployment_name" { - description = "Name of the deployment (will be prepended to service account name)" - type = string -} - -variable "description" { - description = "Description of the created service account." - type = string - default = "Service Account" -} - -# tflint-ignore: terraform_unused_declarations -variable "descriptions" { - description = "Deprecated; create single service accounts using var.description." - type = list(string) - default = null - - validation { - condition = var.descriptions == null - error_message = "var.descriptions has been deprecated in favor of creating single accounts with var.description" - } -} - -variable "display_name" { - description = "Display name of the created service account." - type = string - default = "Service Account" -} - -variable "generate_keys" { - description = "Generate keys for service account." - type = bool - default = false -} - -variable "grant_billing_role" { - description = "Grant billing user role." - type = bool - default = false -} - -variable "grant_xpn_roles" { - description = "Grant roles for shared VPC management." - type = bool - default = true -} - -variable "name" { - description = "Name of the service account to create." - type = string -} - -# tflint-ignore: terraform_unused_declarations -variable "names" { - description = "Deprecated; create single service accounts using var.name." - type = list(string) - default = null - - validation { - condition = var.names == null - error_message = "var.names has been deprecated in favor of creating single accounts with var.name" - } -} - -variable "org_id" { - description = "Id of the organization for org-level roles." - type = string - default = "" -} - -# tflint-ignore: terraform_unused_declarations -variable "prefix" { - description = "Deprecated; prefix now set using var.deployment_name" - type = string - default = null - - validation { - condition = var.prefix == null - error_message = "var.prefix has been deprecated in favor of setting prefix with var.deployment_name" - } -} - -variable "project_id" { - description = "ID of the project" - type = string -} - -variable "project_roles" { - description = "List of roles to grant to service account (e.g. \"storage.objectViewer\" or \"compute.instanceAdmin.v1\"" - type = list(string) -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-account/versions.tf deleted file mode 100644 index 38e6e71945..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/service-account/versions.tf +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/README.md b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/README.md deleted file mode 100644 index 266eac26ec..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/README.md +++ /dev/null @@ -1,70 +0,0 @@ -## Description - -Allows management of multiple API services for a Google Cloud Platform project. - -### Example - -```yaml -- id: services-api - source: community/modules/project/service-enablement - settings: - gcp_service_list: [ - "file.googleapis.com", - "compute.googleapis.com" - ] -``` - -This allows the project to enable both the filestore API as well as the compute API. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_project_service.gcp_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [disable\_on\_destroy](#input\_disable\_on\_destroy) | Disable services on destroy if they were enabled (or already enabled) during apply (default: false) | `bool` | `false` | no | -| [gcp\_service\_list](#input\_gcp\_service\_list) | list of APIs to be enabled for the project | `list(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/main.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/main.tf deleted file mode 100644 index 965e93c549..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/main.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -resource "google_project_service" "gcp_services" { - count = length(var.gcp_service_list) - project = var.project_id - service = var.gcp_service_list[count.index] - timeouts { - create = "30m" - update = "40m" - } - - disable_dependent_services = true - disable_on_destroy = var.disable_on_destroy -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/metadata.yaml deleted file mode 100644 index c594c8f819..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - serviceusage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/variables.tf deleted file mode 100644 index 08f13999fe..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/variables.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "ID of the project" - type = string -} - -variable "gcp_service_list" { - description = "list of APIs to be enabled for the project" - type = list(string) -} - -variable "disable_on_destroy" { - description = "Disable services on destroy if they were enabled (or already enabled) during apply (default: false)" - type = bool - default = false -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/versions.tf deleted file mode 100644 index 07f25fb045..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/project/service-enablement/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/README.md b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/README.md deleted file mode 100644 index 052e6aee23..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Description - -This module creates a Bigquery Pub/Sub Subscription. - -Primarily used for FSI - MonteCarlo Tutorial: -**[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Example - -The following example creates a Bigquery subscription using a Bigquery table and -Pub/Sub topic. - -```yaml - - id: bq_subscription - source: community/modules/pubsub/bigquery-sub - use: [bq-table, pubsub_topic] -``` - -Also see usages in this -[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_project_iam_member.editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_iam_member.viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_pubsub_subscription.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Name of the dataset that was created. Can be provided by the bigquery-table module | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [subscription\_id](#input\_subscription\_id) | The name of the pubsub subscription to be created | `string` | `null` | no | -| [table\_id](#input\_table\_id) | ID of created BQ table. Can be provided by the bigquery-table module | `string` | n/a | yes | -| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [subscription\_id](#output\_subscription\_id) | Name of the subscription that was created. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf deleted file mode 100644 index 8edbc6b24e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-sub", ghpc_role = "pubsub" }) -} - -locals { - subscription_id = var.subscription_id != null ? var.subscription_id : "${var.deployment_name}_subscription_${random_id.resource_name_suffix.hex}" -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} -data "google_project" "project" { - project_id = var.project_id -} - -resource "google_project_iam_member" "viewer" { - project = data.google_project.project.project_id - role = "roles/bigquery.metadataViewer" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" -} - -resource "google_project_iam_member" "editor" { - project = data.google_project.project.project_id - role = "roles/bigquery.dataEditor" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" -} - -resource "google_pubsub_subscription" "example" { - depends_on = [google_project_iam_member.editor, google_project_iam_member.viewer] - name = local.subscription_id - topic = var.topic_id - project = var.project_id - labels = local.labels - bigquery_config { - table = "${var.project_id}.${var.dataset_id}.${var.table_id}" - use_topic_schema = true - write_metadata = true - } - -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml deleted file mode 100644 index 9aedef48dc..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - pubsub.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf deleted file mode 100644 index fc81859503..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "subscription_id" { - description = "Name of the subscription that was created." - value = google_pubsub_subscription.example.name -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf deleted file mode 100644 index ee4dbbed8e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "topic_id" { - description = "The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module" - type = string -} - -variable "subscription_id" { - description = "The name of the pubsub subscription to be created" - type = string - default = null -} - -variable "dataset_id" { - description = "Name of the dataset that was created. Can be provided by the bigquery-table module" - type = string -} - -variable "table_id" { - description = "ID of created BQ table. Can be provided by the bigquery-table module" - type = string -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf deleted file mode 100644 index 46ad6e17c8..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" - } - required_version = ">= 1.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/README.md b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/README.md deleted file mode 100644 index 177f799dc6..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/README.md +++ /dev/null @@ -1,82 +0,0 @@ -## Description - -Creates a Pub/Sub topic - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -### Example - -The following example creates a Pub/Sub topic. - -```yaml - - id: pubsub_topic - source: community/modules/pubsub/topic -``` - -Also see usages in this -[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_pubsub_schema.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_schema) | resource | -| [google_pubsub_topic.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [schema\_id](#input\_schema\_id) | The name of the pubsub schema to be created | `string` | `null` | no | -| [schema\_json](#input\_schema\_json) | The JSON definition of the pubsub topic schema | `string` | `"{ \n \"name\" : \"Avro\", \n \"type\" : \"record\", \n \"fields\" : \n [\n {\"name\" : \"ticker\", \"type\" : \"string\"},\n {\"name\" : \"epoch_time\", \"type\" : \"int\"},\n {\"name\" : \"iteration\", \"type\" : \"int\"},\n {\"name\" : \"start_date\", \"type\" : \"string\"},\n {\"name\" : \"end_date\", \"type\" : \"string\"},\n {\n \"name\":\"simulation_results\",\n \"type\":{\n \"type\": \"array\", \n \"items\":{\n \"name\":\"Child\",\n \"type\":\"record\",\n \"fields\":[\n {\"name\":\"price\", \"type\":\"double\"}\n ]\n }\n }\n }\n ]\n }\n"` | no | -| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to be created | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [topic\_id](#output\_topic\_id) | Name of the topic that was created. | -| [topic\_schema](#output\_topic\_schema) | Name of the topic schema that was created. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/main.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/main.tf deleted file mode 100644 index 4ba68fb5d0..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/main.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "topic", ghpc_role = "pubsub" }) -} - -locals { - topic_id = var.topic_id != null ? var.topic_id : "${var.deployment_name}_topic_${random_id.resource_name_suffix.hex}" - schema_id = var.schema_id != null ? var.schema_id : "${var.deployment_name}_schema_${random_id.resource_name_suffix.hex}" -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_pubsub_topic" "example" { - name = local.topic_id - depends_on = [google_pubsub_schema.example] - project = var.project_id - labels = local.labels - schema_settings { - schema = "projects/${var.project_id}/schemas/${local.schema_id}" - encoding = "BINARY" - } -} - -resource "google_pubsub_schema" "example" { - name = local.schema_id - project = var.project_id - type = "AVRO" - - definition = var.schema_json -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/metadata.yaml deleted file mode 100644 index 9aedef48dc..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - pubsub.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/outputs.tf deleted file mode 100644 index 3ea9d951b2..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/outputs.tf +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "topic_id" { - description = "Name of the topic that was created." - value = google_pubsub_topic.example.name -} - - -output "topic_schema" { - description = "Name of the topic schema that was created." - value = local.schema_id -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/variables.tf deleted file mode 100644 index dca575d21d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/pubsub/topic/variables.tf +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "topic_id" { - description = "The name of the pubsub topic to be created" - type = string - default = null -} - -variable "schema_id" { - description = "The name of the pubsub schema to be created" - type = string - default = null -} - -variable "schema_json" { - description = "The JSON definition of the pubsub topic schema" - type = string - default = < **Note**: This is an experimental module. This module has only been tested in -> limited capacity with the Cluster Toolkit. The module interface may have undergo -> breaking changes in the future. - -### Example - -The following example will create a single GPU accelerated remote desktop. - -```yaml - - id: remote-desktop - source: community/modules/remote-desktop/chrome-remote-desktop - use: [network1] - settings: - install_nvidia_driver: true -``` - -### Setting up the Remote Desktop - -1. Once the remote desktop has been deployed, navigate to https://remotedesktop.google.com/headless. -1. Click through `Begin`, `Next`, & `Authorize`. -1. Copy the code snippet for `Debian Linux`. -1. SSH into the remote desktop machine. It will be listed under - [VM Instances](https://console.cloud.google.com/compute/instances) in the - Google Cloud web console. -1. Run the copied command and follow instructions to set up a PIN. -1. You should now see your machine listed on the - [Chrome Remote Desktop page](https://remotedesktop.google.com/access) under `Remote devices`. -1. Click on your machine and enter PIN if prompted. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.12.31 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [client\_startup\_script](#module\_client\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | -| [instances](#module\_instances) | ../../../../modules/compute/vm-instance | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | -| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image th
at supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-v
m-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. Cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | -| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-balanced"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs on the internet. | `bool` | `true` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required | 
list(object({
    type  = string,
    count = number
  }))
| 
[
  {
    "count": 1,
    "type": "nvidia-tesla-t4-vws"
  }
]
| no | -| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Installs the nvidia driver (true/false). For details, see https://cloud.google.com/compute/docs/gpus/install-drivers-gpu | `bool` | n/a | yes | -| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | -| [instance\_image](#input\_instance\_image) | Image used to build chrome remote desktop node. The default image is
name="debian-12-bookworm-v20250610" and project="debian-cloud".
NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues.

An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud".

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "name": "debian-12-bookworm-v20250610",
  "project": "debian-cloud"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation. Must be N1 family if GPU is used. | `string` | `"n1-standard-8"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface
**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.
Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "RDMA", "IRDMA", "MRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `"TERMINATE"` | no | -| [project\_id](#input\_project\_id) | Project in which Google Cloud resources will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/cloud-platform"
  ]
}
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core | `number` | `2` | no | -| [zone](#input\_zone) | Default zone for creating resources | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_name](#output\_instance\_name) | Name of the first instance created, if any. | -| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf deleted file mode 100644 index a5cf7c5d37..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "chrome-remote-desktop", ghpc_role = "remote-desktop" }) -} - -locals { - - user_startup_script_runners = var.startup_script == null ? [] : [ - { - type = "shell" - content = var.startup_script - destination = "user_startup_script.sh" - } - ] - - configure_nvidia_driver_runners = var.install_nvidia_driver == false ? [] : [ - { - type = "ansible-local" - content = file("${path.module}/scripts/configure-grid-drivers.yml") - destination = "/usr/local/ghpc/configure-grid-drivers.yml" - } - ] - - configure_chrome_remote_desktop_runners = [ - { - type = "ansible-local" - content = file("${path.module}/scripts/configure-chrome-desktop.yml") - destination = "/usr/local/ghpc/configure-chrome-desktop.yml" - } - ] - - disable_sleep = [ - { - type = "ansible-local" - content = file("${path.module}/scripts/disable-sleep.yml") - destination = "/usr/local/ghpc/disable-sleep.yml" - } - ] -} - -module "client_startup_script" { - source = "../../../../modules/scripts/startup-script" - - deployment_name = var.deployment_name - project_id = var.project_id - region = var.region - labels = local.labels - - runners = flatten([ - local.user_startup_script_runners, - local.configure_nvidia_driver_runners, - local.configure_chrome_remote_desktop_runners, - local.disable_sleep - ]) -} - -module "instances" { - source = "../../../../modules/compute/vm-instance" - - instance_count = var.instance_count - name_prefix = var.name_prefix - add_deployment_name_before_prefix = var.add_deployment_name_before_prefix - provisioning_model = var.spot ? "SPOT" : null - - deployment_name = var.deployment_name - project_id = var.project_id - region = var.region - zone = var.zone - labels = local.labels - - machine_type = var.machine_type - service_account_email = var.service_account.email - metadata = var.metadata - startup_script = module.client_startup_script.startup_script - enable_oslogin = var.enable_oslogin - - instance_image = var.instance_image - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - auto_delete_boot_disk = var.auto_delete_boot_disk - - disable_public_ips = !var.enable_public_ips - network_self_link = var.network_self_link - subnetwork_self_link = var.subnetwork_self_link - network_interfaces = var.network_interfaces - bandwidth_tier = var.bandwidth_tier - tags = var.tags - - threads_per_core = var.threads_per_core - guest_accelerator = var.guest_accelerator - on_host_maintenance = var.on_host_maintenance - - network_storage = var.network_storage - -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf deleted file mode 100644 index bcf8ece52d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "script to load and run all runners, as a string value." - value = module.client_startup_script.startup_script -} - -output "instance_name" { - description = "Name of the first instance created, if any." - value = var.instance_count > 0 ? module.instances.name[0] : null -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml deleted file mode 100644 index 391aa86433..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Ensure Desktop OS and Chrome Remote Desktop is installed - hosts: localhost - become: true - module_defaults: - ansible.builtin.apt: - update_cache: true - cache_valid_time: 3600 - tasks: - - name: Install desktop packages - ansible.builtin.apt: - name: - - xfce4 - - xfce4-goodies - state: present - register: apt_result - retries: 10 - delay: 30 - until: apt_result is success - - - name: Download and configure CRD - ansible.builtin.get_url: - url: https://dl.google.com/linux/direct/chrome-remote-desktop_current_amd64.deb - dest: /tmp/chrome-remote-desktop_current_amd64.deb - mode: "0755" - timeout: 30 - - - name: Install CRD - ansible.builtin.apt: - deb: /tmp/chrome-remote-desktop_current_amd64.deb - environment: - DEBIAN_FRONTEND: noninteractive - register: apt_result - retries: 10 - delay: 30 - until: apt_result is success - - - name: Configure CRD to use Xfce by default - ansible.builtin.copy: - dest: /etc/chrome-remote-desktop-session - content: "exec /etc/X11/Xsession /usr/bin/xfce4-session" - mode: 0644 - - - name: Start Chrome remote desktop - ansible.builtin.command: /etc/init.d/chrome-remote-desktop start - register: result - changed_when: result.rc == 0 diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml deleted file mode 100644 index daae08176d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Ensure nvidia grid drivers and other binaries are installed - hosts: localhost - become: true - vars: - dist_settings: - bullseye: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-510.85.02-grid.run - grid_ver: vGPU14.2 - bookworm: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-550.54.15-grid.run - grid_ver: vGPU17.1 - jammy: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - gcc-12 # must match compiler used to build kernel on latest Ubuntu 22 - - pkg-config # observed to be necessary for GRID driver installation on latest Ubuntu 22 - - libglvnd-dev # observed to be necessary for GRID driver installation on latest Ubuntu 22 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-525.125.06-grid.run - grid_ver: vGPU15.3 - tasks: - - name: Fail if using wrong OS - ansible.builtin.assert: - that: - - ansible_os_family in ["Debian", "Ubuntu"] - - ansible_distribution_release in dist_settings.keys() | list - fail_msg: "ansible_os_family: {{ ansible_os_family }} or ansible_distribution_release: {{ansible_distribution_release}} was not acceptable." - - - name: Check if GRID driver installed - ansible.builtin.command: which nvidia-smi - register: nvidiasmi_result - ignore_errors: true - changed_when: false - - - name: Install binaries for GRID drivers - ansible.builtin.apt: - name: '{{ dist_settings[ansible_distribution_release]["packages"] }}' - state: present - update_cache: true - register: apt_result - retries: 6 - delay: 10 - until: apt_result is success - - - name: Install GRID driver if not existing - when: nvidiasmi_result is failed - block: - - name: Download GPU driver - ansible.builtin.get_url: - url: https://storage.googleapis.com/nvidia-drivers-us-public/GRID/{{ dist_settings[ansible_distribution_release]["grid_ver"] }}/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} - dest: /tmp/ - mode: "0755" - timeout: 30 - - - name: Stop gdm service - ansible.builtin.systemd: - name: gdm - state: stopped - - - name: Install GPU driver - ansible.builtin.shell: | - #jinja2: trim_blocks: "True" - {% if ansible_distribution_release == "jammy" %} - CC=gcc-12 /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent - {% else %} - /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent - {% endif %} - register: result - changed_when: result.rc == 0 - - - name: Download VirtualGL driver - ansible.builtin.get_url: - url: https://sourceforge.net/projects/virtualgl/files/3.0.2/virtualgl_3.0.2_amd64.deb/download - dest: /tmp/virtualgl_3.0.2_amd64.deb - mode: "0755" - timeout: 30 - - - name: Install VirtualGL - ansible.builtin.command: gdebi /tmp/virtualgl_3.0.2_amd64.deb --non-interactive - register: result - changed_when: result.rc == 0 - - - name: Fix headless Nvidia issue - block: - - name: Lookup gpu info - ansible.builtin.command: nvidia-xconfig --query-gpu-info - register: gpu_info - failed_when: gpu_info.rc != 0 - changed_when: false - - - name: Extract PCI ID - ansible.builtin.shell: | - set -o pipefail - echo "{{ gpu_info.stdout }}" | grep "PCI BusID " | head -n 1 | cut -d':' -f2-99 | xargs - args: - executable: /bin/bash - register: pci_id - changed_when: false - - - name: Configure nvidia-xconfig - ansible.builtin.command: nvidia-xconfig -a --allow-empty-initial-configuration --enable-all-gpus --virtual=1920x1200 --busid={{ pci_id.stdout }} - register: result - changed_when: result.rc == 0 - - - name: Set HardDPMS to false - ansible.builtin.replace: - path: /etc/X11/xorg.conf - regexp: "Section \"Device\"" - replace: "Section \"Device\"\n Option \"HardDPMS\" \"false\"" - - - name: Configure VirtualGL for X - ansible.builtin.command: vglserver_config +glx +s +f -t - register: result - changed_when: result.rc == 0 - - - name: Configure gdm for X - block: - - name: Configure default display manager - ansible.builtin.copy: - dest: /etc/X11/default-display-manager - content: "/usr/sbin/gdm3" - mode: 0644 - - - name: Switch boot target to gui - ansible.builtin.command: systemctl set-default graphical.target - register: result - changed_when: result.rc == 0 - - - name: Start gdm service - ansible.builtin.systemd: - name: gdm - daemon_reload: true - state: started diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml deleted file mode 100644 index 6767b05fb2..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Mask sleep, suspend, hibernate, and hybrid-sleep targets - hosts: localhost - become: true - tasks: - - - name: Mask sleep target - ansible.builtin.systemd: - name: sleep.target - masked: true - - - name: Mask suspend target - ansible.builtin.systemd: - name: suspend.target - masked: true - - - name: Mask hibernate target - ansible.builtin.systemd: - name: hibernate.target - masked: true - - - name: Mask hybrid-sleep target - ansible.builtin.systemd: - name: hybrid-sleep.target - masked: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf deleted file mode 100644 index ac4c3b1869..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf +++ /dev/null @@ -1,277 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which Google Cloud resources will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. Cloud resource names will include this value." - type = string - #default = "chrome-remote-desktop" -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zone" { - description = "Default zone for creating resources" - type = string -} - -variable "instance_count" { - description = "Number of instances" - type = number - default = 1 -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "instance_image" { - description = <<-EOD - Image used to build chrome remote desktop node. The default image is - name="debian-12-bookworm-v20250610" and project="debian-cloud". - NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues. - - An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud". - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "debian-cloud" - name = "debian-12-bookworm-v20250610" - } -} - -variable "disk_size_gb" { - description = "Size of disk for instances." - type = number - default = 200 -} - -variable "disk_type" { - description = "Disk type for instances." - type = string - default = "pd-balanced" -} - -variable "auto_delete_boot_disk" { - description = "Controls if boot disk should be auto-deleted when instance is deleted." - type = bool - default = true -} - -variable "name_prefix" { - description = <<-EOT - An optional name for all VM and disk resources. - If not supplied, `deployment_name` will be used. - When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, - then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". - EOT - type = string - default = null -} - -variable "add_deployment_name_before_prefix" { - description = <<-EOT - If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. - See `name_prefix` for further details on resource naming behavior. - EOT - type = bool - default = false -} - -variable "enable_public_ips" { - description = "If set to true, instances will have public IPs on the internet." - type = bool - default = true -} - -variable "machine_type" { - description = "Machine type to use for the instance creation. Must be N1 family if GPU is used." - type = string - default = "n1-standard-8" -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "service_account" { - description = "Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account." - type = object({ - email = string, - scopes = set(string) - }) - default = { - email = null - scopes = [ - "https://www.googleapis.com/auth/cloud-platform", - ] - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the VM." - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the VM." - type = string - default = null -} - -variable "network_interfaces" { - description = <<-EOT - A list of network interfaces. The options match that of the terraform - network_interface block of google_compute_instance. For descriptions of the - subfields or more information see the documentation: - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - **_NOTE:_** If `network_interfaces` are set, `network_self_link` and - `subnetwork_self_link` will be ignored, even if they are provided through - the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply - to network interfaces defined in this variable. - Subfields: - network (string, required if subnetwork is not supplied) - subnetwork (string, required if network is not supplied) - subnetwork_project (string, optional) - network_ip (string, optional) - nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "RDMA", "IRDMA", "MRDMA"]) - stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) - queue_count (number, optional) - access_config (object, optional) - ipv6_access_config (object, optional) - alias_ip_range (list(object), optional) - EOT - type = list(object({ - network = string, - subnetwork = string, - subnetwork_project = string, - network_ip = string, - nic_type = string, - stack_type = string, - queue_count = number, - access_config = list(object({ - nat_ip = string, - public_ptr_domain_name = string, - network_tier = string - })), - ipv6_access_config = list(object({ - public_ptr_domain_name = string, - network_tier = string - })), - alias_ip_range = list(object({ - ip_cidr_range = string, - subnetwork_range_name = string - })) - })) - default = [] -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "startup_script" { - description = "Startup script used on the instance" - type = string - default = null -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required" - type = list(object({ - type = string, - count = number - })) - default = [{ - type = "nvidia-tesla-t4-vws" - count = 1 - }] -} - -variable "threads_per_core" { - description = "Sets the number of threads per physical core" - type = number - default = 2 -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" - type = string - default = "TERMINATE" -} - -variable "bandwidth_tier" { - description = <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -This mode can be switched to proactive (automatic) replacement by setting -[var.update_policy](#input_update_policy) to "PROACTIVE". In this case we -recommend the use of Filestore to store the job queue state ("spool") and -setting [var.spool_parent_dir][#input_spool_parent_dir] to its mount point: - -```yaml - - id: spoolfs - source: modules/file-system/filestore - use: - - network1 - settings: - filestore_tier: ENTERPRISE - local_mount: /shared - -... - - - id: htcondor_access - source: community/modules/scheduler/htcondor-access-point - use: - - network1 - - spoolfs - - htcondor_secrets - - htcondor_setup - - htcondor_cm - - htcondor_execute_point_group - settings: - spool_parent_dir: /shared -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 3.83 | -| [null](#requirement\_null) | >= 3.0 | -| [random](#requirement\_random) | ~> 3.6 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [null](#provider\_null) | >= 3.0 | -| [random](#provider\_random) | ~> 3.6 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [access\_point\_instance\_template](#module\_access\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [htcondor\_ap](#module\_htcondor\_ap) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_address.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_region_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_region_disk) | resource | -| [google_storage_bucket_object.ap_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.ap_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_shuffle.zones](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/shuffle) | resource | -| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_instance.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | -| [google_compute_region_instance_group.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_runner](#input\_access\_point\_runner) | A list of Toolkit runners for configuring an HTCondor access point | `list(map(string))` | `[]` | no | -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account for access point (e-mail format) | `string` | n/a | yes | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [autoscaler\_runner](#input\_autoscaler\_runner) | A list of Toolkit runners for configuring autoscaling daemons | `list(map(string))` | `[]` | no | -| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | -| [default\_mig\_id](#input\_default\_mig\_id) | Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | -| [disk\_type](#input\_disk\_type) | Boot disk size in GB | `string` | `"pd-balanced"` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape acoss zones for instance group managing high availability of access point | `string` | `"ANY_SINGLE_ZONE"` | no | -| [enable\_high\_availability](#input\_enable\_high\_availability) | Provision HTCondor access point in high availability mode | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | Enable Public IPs on the access points | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor and Toolkit support installed."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | -| [mig\_id](#input\_mig\_id) | List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module) | `list(string)` | `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spool\_disk\_size\_gb](#input\_spool\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | -| [spool\_disk\_type](#input\_spool\_disk\_type) | Boot disk size in GB | `string` | `"pd-ssd"` | no | -| [spool\_parent\_dir](#input\_spool\_parent\_dir) | HTCondor access point configuration SPOOL will be set to subdirectory named "spool" | `string` | `"/var/lib/condor"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | -| [zones](#input\_zones) | Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_ips](#output\_access\_point\_ips) | IP addresses of the access points provisioned by this module | -| [access\_point\_name](#output\_access\_point\_name) | Name of the access point provisioned by this module | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml deleted file mode 100644 index 6a2f50c831..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Access Point - hosts: localhost - become: true - vars: - spool_dir: /var/lib/condor/spool - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - htcondor_spool_disk_device: /dev/disk/by-id/google-htcondor-spool-disk - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - - name: Configure HTCondor SchedD - when: htcondor_role == 'get_htcondor_submit' - block: - - name: Format spool disk - community.general.filesystem: - fstype: ext4 - state: present - dev: "{{ htcondor_spool_disk_device }}" - # RUN TUNE2FS - - name: Mount spool (creates mount point) - ansible.posix.mount: - path: "{{ spool_dir }}" - src: "{{ htcondor_spool_disk_device }}" - fstype: ext4 - opts: defaults - state: mounted - - name: Ensure spool free space - ansible.builtin.command: tune2fs -r 0 {{ htcondor_spool_disk_device }} - - name: Setup spool directory - ansible.builtin.file: - path: "{{ spool_dir }}" - state: directory - owner: condor - group: condor - mode: 0755 - recurse: true - - name: Create SystemD override directory for HTCondor - ansible.builtin.file: - path: /etc/systemd/system/condor.service.d - state: directory - owner: root - group: root - mode: 0755 - - name: Ensure HTCondor starts after shared filesystem is mounted - ansible.builtin.copy: - dest: /etc/systemd/system/condor.service.d/mount-spool.conf - mode: 0644 - content: | - [Unit] - RequiresMountsFor={{ spool_dir }} - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor configuration complete; startup-script may still be executing ********" diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf deleted file mode 100644 index fdbcf5c32f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf +++ /dev/null @@ -1,338 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-access-point", ghpc_role = "scheduler" }) -} - -locals { - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - metadata = merge( - local.network_storage_metadata, - local.enable_oslogin_metadata, - local.disable_automatic_updates_metadata, - var.metadata - ) - - host_count = 1 - name_prefix = "${var.deployment_name}-ap" - - example_runner = { - type = "data" - destination = "/var/tmp/helloworld.sub" - content = <<-EOT - universe = vanilla - executable = /bin/sleep - arguments = 1000 - output = out.$(ClusterId).$(ProcId) - error = err.$(ClusterId).$(ProcId) - log = log.$(ClusterId).$(ProcId) - request_cpus = 1 - request_memory = 100MB - queue - EOT - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.access_point_runner, - [local.schedd_runner], - var.autoscaler_runner, - [local.example_runner] - ) - - ap_config = templatefile("${path.module}/templates/condor_config.tftpl", { - htcondor_role = "get_htcondor_submit", - central_manager_ips = var.central_manager_ips - spool_dir = "${var.spool_parent_dir}/spool", - mig_ids = var.mig_id, - default_mig_id = var.default_mig_id - }) - - ap_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.ap_config.output_name}" - schedd_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e htcondor_role=get_htcondor_submit", - "-e config_object=${local.ap_object}", - "-e spool_dir=${var.spool_parent_dir}/spool", - "-e htcondor_spool_disk_device=/dev/disk/by-id/google-${local.spool_disk_device_name}", - ]) - } - - access_point_ips = google_compute_address.ap.address - access_point_name = data.google_compute_instance.ap.name - - spool_disk_resource_name = "${var.deployment_name}-spool-disk" - spool_disk_device_name = "htcondor-spool-disk" - spool_disk_source = try(google_compute_disk.spool[0].name, google_compute_region_disk.spool[0].self_link) - - zones = coalescelist(var.zones, random_shuffle.zones.result) - - vm_family = split("-", var.machine_type)[0] - regional_pd_families = ["e2", "n1", "n2", "n2d"] -} - -data "google_compute_image" "htcondor" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - postcondition { - condition = self.disk_size_gb <= var.disk_size_gb - error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region - - lifecycle { - postcondition { - condition = alltrue([ - for z in var.zones : contains(self.names, z) - ]) - error_message = "Each entry in var.zones must be a zone in var.region: ${var.region}" - } - } -} - -resource "random_shuffle" "zones" { - input = data.google_compute_zones.available.names - result_count = var.enable_high_availability ? 2 : 1 -} - -data "google_compute_region_instance_group" "ap" { - self_link = module.htcondor_ap.self_link - lifecycle { - postcondition { - condition = length(self.instances) == local.host_count - error_message = "There should be ${local.host_count} access points found" - } - } -} - -data "google_compute_instance" "ap" { - self_link = data.google_compute_region_instance_group.ap.instances[0].instance -} - -resource "null_resource" "ap_config" { - triggers = { - config = local.ap_config - } -} - -resource "google_storage_bucket_object" "ap_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.ap_config.id), 0, 4)}" - content = local.ap_config - bucket = var.htcondor_bucket_name - - lifecycle { - precondition { - condition = var.default_mig_id == "" || contains(var.mig_id, var.default_mig_id) - error_message = "If set, var.default_mig_id must be an element in var.mig_id" - } - - # by construction, this precondition only fails when the user has set - # var.zones to a non-empty list of length not equal to 2 - precondition { - condition = !var.enable_high_availability || length(local.zones) == 2 - error_message = "When using HTCondor access point high availability, var.zones must be of length 2." - } - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -resource "google_compute_region_disk" "spool" { - count = var.enable_high_availability ? 1 : 0 - name = local.spool_disk_resource_name - labels = local.labels - type = var.spool_disk_type - region = var.region - size = var.spool_disk_size_gb - - replica_zones = local.zones - - lifecycle { - precondition { - condition = var.spool_disk_size_gb >= 200 - error_message = "When using HTCondor access point high availability, var.spool_disk_size_gb must be set to 200 or greater." - } - - precondition { - condition = contains(local.regional_pd_families, local.vm_family) - error_message = "When using HTCondor access point high availability, var.machine_type must be one of ${jsonencode(local.regional_pd_families)}." - } - } -} - -resource "google_compute_disk" "spool" { - count = var.enable_high_availability ? 0 : 1 - name = local.spool_disk_resource_name - labels = local.labels - type = var.spool_disk_type - zone = local.zones[0] - size = var.spool_disk_size_gb -} - -resource "google_compute_address" "ap" { - project = var.project_id - name = local.name_prefix - region = var.region - subnetwork = var.subnetwork_self_link - address_type = "INTERNAL" - purpose = "GCE_ENDPOINT" -} - -module "access_point_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.access_point_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - preemptible = false - startup_script = module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.htcondor.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - network_ip = google_compute_address.ap.id - - # spool disk - additional_disks = [ - { - source = local.spool_disk_source - device_name = local.spool_disk_device_name - } - ] -} - -module "htcondor_ap" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = local.host_count - hostname = local.name_prefix - instance_template = module.access_point_instance_template.self_link - - health_check_name = "health-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) - max_surge_fixed = 0 # must be 0 to preserve hostnames - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - - stateful_disks = [{ - device_name = local.spool_disk_device_name - delete_rule = "ON_PERMANENT_INSTANCE_DELETION" - }] - stateful_ips = var.enable_public_ips ? [{ - interface_name = "nic0" - delete_rule = "ON_PERMANENT_INSTANCE_DELETION" - is_external = true - }] : [] - - # the timeouts below are default for resource - wait_for_instances = true - mig_timeouts = { - create = "15m" - delete = "15m" - update = "15m" - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf deleted file mode 100644 index f7424c6d5d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "access_point_ips" { - description = "IP addresses of the access points provisioned by this module" - value = local.access_point_ips -} - -output "access_point_name" { - description = "Name of the access point provisioned by this module" - value = local.access_point_name -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl deleted file mode 100644 index 214fbc726f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:${htcondor_role} -CONDOR_HOST = ${join(",", central_manager_ips)} - -SPOOL = ${spool_dir} -SCHEDD_INTERVAL = 30 -TRUST_UID_DOMAIN = True -SUBMIT_ATTRS = RunAsOwner -RunAsOwner = True - -# When a job matches to a machine, add machine attributes to the job for -# condor_history (e.g. VM Instance ID) -use feature:JobsHaveInstanceIDs -SYSTEM_JOB_MACHINE_ATTRS = $(SYSTEM_JOB_MACHINE_ATTRS) \ - CloudVMType CloudZone CloudInterruptible -SYSTEM_JOB_MACHINE_ATTRS_HISTORY_LENGTH = 10 - -# Add Cloud attributes to SchedD ClassAd -use feature:ScheddCronOneShot(cloud, $(LIBEXEC)/common-cloud-attributes-google.py) -SCHEDD_CRON_cloud_PREFIX = Cloud - -# aid the user by automatically using RequireSpot in their Requirements, unless -# the user has explicitly used CloudInterruptible -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SPOT -JOB_TRANSFORM_SPOT @=end - REQUIREMENTS ! isUndefined(RequireSpot) && ! unresolved(Requirements, "^CloudInterruptible$") - SET Requirements ($(MY.Requirements)) && (CloudInterruptible is My.RequireSpot) -@end - -# help the user by enforcing that RequireSpot is undefined or a boolean -SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) SPOT -SUBMIT_REQUIREMENT_SPOT = isUndefined(RequireSpot) || isBoolean(RequireSpot) -SUBMIT_REQUIREMENT_SPOT_REASON = "If +RequireSpot is defined, it must be either True or False" - -%{ if length(mig_ids) > 0 ~} -MIG_IDS = "${join(" ", mig_ids)}" -MIG_ID_LIST = split($(MIG_IDS)) -%{ if default_mig_id != "" ~} -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) ID_DEFAULT -JOB_TRANSFORM_ID_DEFAULT @=end - DEFAULT RequireId "${default_mig_id}" -@end -%{ endif ~} -SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) MIGID -SUBMIT_REQUIREMENT_MIGID = !isUndefined(RequireId) && member(RequireId, $(MIG_ID_LIST)) -SUBMIT_REQUIREMENT_MIGID_REASON = strcat("Jobs must set +RequireId to one of following values surrounded by quotation marks:\n", $(MIG_IDS)) - -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) MIGID -JOB_TRANSFORM_MIGID @=end - REQUIREMENTS ! isUndefined(RequireId) && ! unresolved(Requirements, "^CloudCreatedBy$") - SET Requirements ($(MY.Requirements)) && regexp(strcat("/", My.RequireId, "$"), CloudCreatedBy) -@end -%{ endif ~} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf deleted file mode 100644 index f54a88ac2e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zones" { - description = "Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region." - type = list(string) - default = [] - nullable = false - - validation { - condition = length(var.zones) <= 2 - error_message = "Set var.zones to the empty list or up to 2 zones in var.region" - } -} - -variable "distribution_policy_target_shape" { - description = "Target shape acoss zones for instance group managing high availability of access point" - type = string - default = "ANY_SINGLE_ZONE" -} - -variable "network_self_link" { - description = "The self link of the network in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "access_point_service_account_email" { - description = "Service account for access point (e-mail format)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 32 - nullable = false -} - -variable "disk_type" { - description = "Boot disk size in GB" - type = string - default = "pd-balanced" - nullable = false -} - -variable "spool_disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 32 - nullable = false -} - -variable "spool_disk_type" { - description = "Boot disk size in GB" - type = string - default = "pd-ssd" - nullable = false -} - -variable "metadata" { - description = "Metadata to add to HTCondor central managers" - type = map(string) - default = {} -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - nullable = false - validation { - condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "enable_high_availability" { - description = "Provision HTCondor access point in high availability mode" - type = bool - default = false -} - -variable "instance_image" { - description = <<-EOD - Custom VM image with HTCondor and Toolkit support installed." - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "machine_type" { - description = "Machine type to use for HTCondor central managers" - type = string - default = "n2-standard-4" -} - -variable "access_point_runner" { - description = "A list of Toolkit runners for configuring an HTCondor access point" - type = list(map(string)) - default = [] -} - -variable "autoscaler_runner" { - description = "A list of Toolkit runners for configuring autoscaling daemons" - type = list(map(string)) - default = [] -} - -variable "spool_parent_dir" { - description = "HTCondor access point configuration SPOOL will be set to subdirectory named \"spool\"" - type = string - default = "/var/lib/condor" -} - -variable "central_manager_ips" { - description = "List of IP addresses of HTCondor Central Managers" - type = list(string) -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "enable_public_ips" { - description = "Enable Public IPs on the access points" - type = bool - default = false -} - -variable "mig_id" { - description = "List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module)" - type = list(string) - default = [] - nullable = false - - validation { - condition = length(var.mig_id) > 0 - error_message = "At least 1 MIG containing execute points must be provided to this module" - } -} - -variable "default_mig_id" { - description = "Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id" - type = string - default = "" - nullable = false -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" - type = string - default = "OPPORTUNISTIC" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf deleted file mode 100644 index 0d07e7abf1..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.6" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.74.0" - } - - required_version = ">= 1.1" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md deleted file mode 100644 index dfab563a55..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md +++ /dev/null @@ -1,159 +0,0 @@ -## Description - -This module provisions a highly available HTCondor central manager using a [Managed -Instance Group (MIG)][mig] with auto-healing. - -[mig]: https://cloud.google.com/compute/docs/instance-groups - -## Usage - -This module provisions an HTCondor central manager with a standard -configuration. For the node to function correctly, you must supply the input -variable described below: - -- [var.central_manager_runner](#input_central_manager_runner) - - Runner must download a POOL password / signing key and create an [IDTOKEN] - with no scopes (full authorization). - -A reference implementation is included in the Toolkit module -[htcondor-pool-secrets]. You may substitute implementations so long as they -duplicate the functionality in the references. Usage is demonstrated in the -[HTCondor example][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- -[htcondor-pool-secrets]: ../htcondor-pool-secrets/README.md -[IDTOKEN]: https://htcondor.readthedocs.io/en/latest/admin-manual/security.html#introducing-idtokens - -## Behavior of Managed Instance Group (MIG) - -A regional [MIG][mig] is used to provision the central manager, although only -1 node will ever be active at a time. By default, the node will be provisioned -in any of the zones available in that region, however, it can be constrained to -run in fewer zones (or a single zone) using [var.zones](#input_zones). - -When the configuration of the Central Manager is changed, the MIG can be -configured to [replace the VM][replacement] using a "proactive" or -"opportunistic" policy. By default, the Central Manager replacement policy is -set to proactive. In practice, this means that the Central Manager will be -replaced by Terraform when changes to the instance template / HTCondor -configuration are made. The Central Manager is safe to replace automatically as -it gathers its state information from periodic messages exchanged with the rest -of the HTCondor pool. - -This mode can be configured by setting [var.update_policy](#input_update_policy) -to either "PROACTIVE" (default) or "OPPORTUNISTIC". If set to opportunistic -replacement, the Central Manager will be replaced only when: - -- intentionally by issuing an update via Cloud Console or using gcloud (below) -- the VM becomes unhealthy or is otherwise automatically replaced (e.g. regular - Google Cloud maintenance) - -For example, to manually update all instances in a MIG: - -```text -gcloud compute instance-groups managed update-instances \ - <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - -## Limiting inter-zone egress - -Because all the elements of the HTCondor pool use regional MIGs, they may be -subject to [interzone egress fees][network-pricing]. The primary traffic between -nodes of an HTCondor pool running embarrassingly parallel jobs is expected to -be limited to API traffic for job scheduling and monitoring. Please review the -[network pricing][network-pricing] documentation and determine if this cost is -a concern. If it is, use [var.zones](#input_zones) to constrain each node within -your HTCondor pool to operate within a single zone. - -[network-pricing]: https://cloud.google.com/vpc/network-pricing - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1.0 | -| [google](#requirement\_google) | >= 3.83 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [central\_manager\_instance\_template](#module\_central\_manager\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [htcondor\_cm](#module\_htcondor\_cm) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_address.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_storage_bucket_object.cm_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.cm_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_instance.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | -| [google_compute_region_instance_group.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [central\_manager\_runner](#input\_central\_manager\_runner) | A list of Toolkit runners for configuring an HTCondor central manager | `list(map(string))` | `[]` | no | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for central manager (can be supplied by htcondor-setup module) | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `20` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape for instance group managing high availability of central manager | `string` | `"ANY_SINGLE_ZONE"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor installed using the htcondor-install module."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor central manager will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Central Manager ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle). | `string` | `"PROACTIVE"` | no | -| [zones](#input\_zones) | Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [central\_manager\_ips](#output\_central\_manager\_ips) | IP addresses of the central managers provisioned by this module | -| [central\_manager\_name](#output\_central\_manager\_name) | Name of the central managers provisioned by this module | -| [list\_instances\_command](#output\_list\_instances\_command) | Command to list central managers provisioned by this module | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml deleted file mode 100644 index 7408af6370..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor central manager - hosts: localhost - become: true - vars: - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - handlers: - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf deleted file mode 100644 index d288a91144..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-central-manager", ghpc_role = "scheduler" }) -} - -locals { - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - metadata = merge( - local.network_storage_metadata, - local.enable_oslogin_metadata, - local.disable_automatic_updates_metadata, - var.metadata - ) - - name_prefix = "${var.deployment_name}-cm" - - cm_config = templatefile("${path.module}/templates/condor_config.tftpl", {}) - - cm_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.cm_config.output_name}" - schedd_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e config_object=${local.cm_object}", - ]) - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.central_manager_runner, - [local.schedd_runner] - ) - - central_manager_ips = google_compute_address.cm.address - central_manager_name = data.google_compute_instance.cm.name - - list_instances_command = "gcloud compute instance-groups list-instances ${data.google_compute_region_instance_group.cm.name} --region ${var.region} --project ${var.project_id}" - - zones = coalescelist(var.zones, data.google_compute_zones.available.names) -} - -data "google_compute_image" "htcondor" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - postcondition { - condition = self.disk_size_gb <= var.disk_size_gb - error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -data "google_compute_region_instance_group" "cm" { - self_link = module.htcondor_cm.self_link - lifecycle { - postcondition { - condition = length(self.instances) == 1 - error_message = "There should only be 1 central manager found" - } - } -} - -data "google_compute_instance" "cm" { - self_link = data.google_compute_region_instance_group.cm.instances[0].instance -} - -resource "null_resource" "cm_config" { - triggers = { - config = local.cm_config - } -} - -resource "google_storage_bucket_object" "cm_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.cm_config.id), 0, 4)}" - content = local.cm_config - bucket = var.htcondor_bucket_name -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -resource "google_compute_address" "cm" { - project = var.project_id - name = local.name_prefix - region = var.region - subnetwork = var.subnetwork_self_link - address_type = "INTERNAL" - purpose = "GCE_ENDPOINT" -} - -module "central_manager_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.central_manager_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - preemptible = false - startup_script = module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.htcondor.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - network_ip = google_compute_address.cm.id -} - -module "htcondor_cm" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = 1 - hostname = local.name_prefix - instance_template = module.central_manager_instance_template.self_link - - health_check_name = "health-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) - max_surge_fixed = 0 # must be 0 to preserve hostnames - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - - # the timeouts below are default for resource - wait_for_instances = true - mig_timeouts = { - create = "15m" - delete = "15m" - update = "15m" - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf deleted file mode 100644 index a6272e7ca2..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "list_instances_command" { - description = "Command to list central managers provisioned by this module" - value = local.list_instances_command -} - -output "central_manager_ips" { - description = "IP addresses of the central managers provisioned by this module" - value = local.central_manager_ips -} - -output "central_manager_name" { - description = "Name of the central managers provisioned by this module" - value = local.central_manager_name -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl deleted file mode 100644 index 5b9676457e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:get_htcondor_central_manager -CONDOR_HOST = $(IPV4_ADDRESS) - -# Central Manager configuration settings -# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-collector-configuration-file-entries -# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-negotiator-configuration-file-entries -# set classad lifetime (expiration) to ~5x the update interval for all daemons -# defaults to 900s -CLASSAD_LIFETIME = 180 -COLLECTOR_UPDATE_INTERVAL = 30 -NEGOTIATOR_UPDATE_INTERVAL = 30 -NEGOTIATOR_DEPTH_FIRST = True -NEGOTIATOR_UPDATE_AFTER_CYCLE = True diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf deleted file mode 100644 index 7f85861c3f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf +++ /dev/null @@ -1,192 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor central manager will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zones" { - description = "Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region." - type = list(string) - default = [] - nullable = false -} - -variable "distribution_policy_target_shape" { - description = "Target shape for instance group managing high availability of central manager" - type = string - default = "ANY_SINGLE_ZONE" -} - -variable "network_self_link" { - description = "The self link of the network in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "central_manager_service_account_email" { - description = "Service account e-mail for central manager (can be supplied by htcondor-setup module)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 20 - nullable = false -} - -variable "metadata" { - description = "Metadata to add to HTCondor central managers" - type = map(string) - default = {} -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - nullable = false - validation { - condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "instance_image" { - description = <<-EOD - Custom VM image with HTCondor installed using the htcondor-install module." - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "machine_type" { - description = "Machine type to use for HTCondor central managers" - type = string - default = "n2-standard-4" -} - -variable "central_manager_runner" { - description = "A list of Toolkit runners for configuring an HTCondor central manager" - type = list(map(string)) - default = [] -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Central Manager (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)." - type = string - default = "PROACTIVE" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf deleted file mode 100644 index 4dee3adac7..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.74.0" - } - - required_version = ">= 1.1.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md deleted file mode 100644 index 7158e7bac6..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md +++ /dev/null @@ -1,172 +0,0 @@ -## Description - -This module is responsible for the following actions: - -- store an HTCondor Pool password in Google Cloud Secret Manager - - will generate a new password if one is not supplied -- create a secret in Google Cloud Secret Manager in which the HTCondor central - manager can place IDTOKENs (JWT Authorizations) for execute points to download -- create a Toolkit runner for the central manager - - download the POOL password / signing key - - create a local IDTOKEN for itself - - upload the execute point IDTOKEN secret -- create a Toolkit runner for access points - - download the POOL password / signing key - - create a local IDTOKEN for itself -- create a Toolkit runner for execute points - - Fetch the IDTOKEN secret generated by the central manager - -It is expected to be used with the [htcondor-install] and -[htcondor-execute-point] modules. - -[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[htcondor-install]: ../../scripts/htcondor-setup/README.md -[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md - -[htcrole]: https://htcondor.readthedocs.io/en/latest/getting-htcondor/admin-quick-start.html#what-get-htcondor-does-to-configure-a-role - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_setup - - - id: htcondor_startup_central_manager - source: modules/scripts/startup-script - settings: - runners: - - $(htcondor_install.install_htcondor_runner) - - $(htcondor_secrets.central_manager_runner) - - $(htcondor_setup.central_manager_runner) - -- id: htcondor_cm - source: modules/compute/vm-instance - use: - - network1 - - htcondor_startup_central_manager - settings: - name_prefix: cm0 - machine_type: c2-standard-4 - disable_public_ips: true - service_account: - email: $(htcondor_setup.central_manager_service_account) - scopes: - - cloud-platform - network_interfaces: - - network: null - subnetwork: $(network1.subnetwork_self_link) - subnetwork_project: $(vars.project_id) - network_ip: $(htcondor_setup.central_manager_internal_ip) - stack_type: null - access_config: [] - ipv6_access_config: [] - alias_ip_range: [] - nic_type: VIRTIO_NET - queue_count: null - outputs: - - internal_ip -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 4.84 | -| [random](#requirement\_random) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | -| [random](#provider\_random) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_secret_manager_secret.execute_point_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_iam_member.access_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.central_manager_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.central_manager_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.execute_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_version.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [random_password.pool](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [pool\_password](#input\_pool\_password) | HTCondor Pool Password | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [trust\_domain](#input\_trust\_domain) | Trust domain for HTCondor pool (if not supplied, will be set based on project\_id) | `string` | `""` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_runner](#output\_access\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor access point | -| [central\_manager\_runner](#output\_central\_manager\_runner) | Toolkit Runner to download pool secrets to an HTCondor central manager | -| [execute\_point\_runner](#output\_execute\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor execute point | -| [pool\_password\_secret\_id](#output\_pool\_password\_secret\_id) | Google Cloud Secret Manager ID containing HTCondor Pool Password | -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | PowerShell script to download pool secrets to an HTCondor execute point | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml deleted file mode 100644 index 538c809c2a..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Secrets - hosts: localhost - become: true - vars: - condor_config_root: /etc/condor - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - password_id is defined - - trust_domain is defined - - name: Set Pool Trust Domain - ansible.builtin.copy: - dest: "{{ condor_config_root }}/config.d/51-ghpc-trust-domain" - mode: 0644 - content: | - # these lines must appear AFTER any "use role:" settings - UID_DOMAIN = {{ trust_domain }} - TRUST_DOMAIN = {{ trust_domain }} - - name: Get HTCondor Pool password (token signing key) - when: htcondor_role != 'get_htcondor_execute' - ansible.builtin.shell: | - set -e -o pipefail +o history - POOL_PASSWORD=$(gcloud secrets versions access latest --secret={{ password_id }}) - echo -n "$POOL_PASSWORD" | sh -c "condor_store_cred add -c -i -" - args: - creates: "{{ condor_config_root }}/passwords.d/POOL" - executable: /bin/bash - - name: Configure HTCondor Central Manager - when: htcondor_role == 'get_htcondor_central_manager' - block: - - name: Create IDTOKEN for Central Manager - ansible.builtin.shell: | - umask 0077 - condor_token_create -identity condor@{{ trust_domain }} \ - -token condor@{{ trust_domain }} - args: - creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" - - name: Create IDTOKEN secret for Execute Points - when: xp_idtoken_secret_id | length > 0 - changed_when: true - ansible.builtin.shell: | - umask 0077 - TMPFILE=$(mktemp) - condor_token_create -authz READ -authz ADVERTISE_MASTER \ - -authz ADVERTISE_STARTD -identity condor@{{ trust_domain }} > "$TMPFILE" - gcloud secrets versions add --data-file "$TMPFILE" {{ xp_idtoken_secret_id }} - rm -f "$TMPFILE" - - name: Configure HTCondor SchedD - when: htcondor_role == 'get_htcondor_submit' - block: - - name: Create IDTOKEN to advertise access point - ansible.builtin.shell: | - umask 0077 - # DAEMON authorization can likely be removed in future when scopes - # needed to trigger a negotiation cycle are changed. Suggest review - # https://opensciencegrid.atlassian.net/jira/software/c/projects/HTCONDOR/issues/?filter=allissues - condor_token_create -authz READ -authz ADVERTISE_MASTER \ - -authz ADVERTISE_SCHEDD -authz DAEMON -identity condor@{{ trust_domain }} \ - -token condor@{{ trust_domain }} - args: - creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" - - name: Configure HTCondor StartD - when: htcondor_role == 'get_htcondor_execute' - block: - - name: Create SystemD override directory for HTCondor Execute Point - ansible.builtin.file: - path: /etc/systemd/system/condor.service.d - state: directory - owner: root - group: root - mode: 0755 - - name: Fetch IDTOKEN to advertise execute point - ansible.builtin.copy: - dest: "/etc/systemd/system/condor.service.d/htcondor-token-fetcher.conf" - mode: 0644 - content: | - [Service] - ExecStartPre=gcloud secrets versions access latest --secret {{ xp_idtoken_secret_id }} \ - --out-file {{ condor_config_root }}/tokens.d/condor@{{ trust_domain }} - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf deleted file mode 100644 index 1a7c761760..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-pool-secrets", ghpc_role = "scheduler" }) -} - -locals { - pool_password = coalesce(var.pool_password, random_password.pool.result) - auto = length(var.user_managed_replication) == 0 ? "" : "-user" - access_point_service_account_iam_email = "serviceAccount:${var.access_point_service_account_email}" - central_manager_service_account_iam_email = "serviceAccount:${var.central_manager_service_account_email}" - execute_point_service_account_iam_email = "serviceAccount:${var.execute_point_service_account_email}" - - trust_domain = coalesce(var.trust_domain, "c.${var.project_id}.internal") - - runner_cm = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_central_manager", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - - runner_access = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_submit", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - - runner_execute = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_execute", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - windows_startup_ps1 = templatefile( - "${path.module}/templates/fetch-idtoken.ps1.tftpl", - { - trust_domain = local.trust_domain, - xp_idtoken_secret_id = google_secret_manager_secret.execute_point_idtoken.secret_id, - } - ) -} - -resource "random_password" "pool" { - length = 24 - special = true - override_special = "_-#=." -} - -resource "google_secret_manager_secret" "pool_password" { - secret_id = "${var.deployment_name}-pool-password${local.auto}" - - labels = local.labels - - replication { - dynamic "auto" { - for_each = length(var.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_version" "pool_password" { - secret = google_secret_manager_secret.pool_password.id - secret_data = local.pool_password -} - -# this secret will be populated by the Central Manager -resource "google_secret_manager_secret" "execute_point_idtoken" { - secret_id = "${var.deployment_name}-execute-point-idtoken${local.auto}" - - labels = local.labels - - replication { - dynamic "auto" { - for_each = length(var.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_iam_member" "central_manager_password" { - secret_id = google_secret_manager_secret.pool_password.id - role = "roles/secretmanager.secretAccessor" - member = local.central_manager_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "central_manager_idtoken" { - secret_id = google_secret_manager_secret.execute_point_idtoken.id - role = "roles/secretmanager.secretVersionManager" - member = local.central_manager_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "access_point" { - secret_id = google_secret_manager_secret.pool_password.id - role = "roles/secretmanager.secretAccessor" - member = local.access_point_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "execute_point" { - secret_id = google_secret_manager_secret.execute_point_idtoken.id - role = "roles/secretmanager.secretAccessor" - member = local.execute_point_service_account_iam_email -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml deleted file mode 100644 index 4b0bdbd616..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com - - secretmanager.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf deleted file mode 100644 index 81c4986b16..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "pool_password_secret_id" { - description = "Google Cloud Secret Manager ID containing HTCondor Pool Password" - value = google_secret_manager_secret.pool_password.secret_id - sensitive = true -} - -output "central_manager_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor central manager" - value = local.runner_cm - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "access_point_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor access point" - value = local.runner_access - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "execute_point_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor execute point" - value = local.runner_execute - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "windows_startup_ps1" { - description = "PowerShell script to download pool secrets to an HTCondor execute point" - value = local.windows_startup_ps1 -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl deleted file mode 100644 index 04c96291ee..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl +++ /dev/null @@ -1,26 +0,0 @@ -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -$config_dir = 'C:\Condor\config' -if(!(test-path -PathType container -Path $config_dir)) -{ - New-Item -ItemType Directory -Path $config_dir -} -$config_file = "$config_dir\51-ghpc-trust-domain" - -$config_string = @' -# these lines must appear AFTER any "use role:" settings -UID_DOMAIN = ${trust_domain} -TRUST_DOMAIN = ${trust_domain} -'@ - -Set-Content -Path "$config_file" -Value "$config_string" - -# obtain IDTOKEN for authentication by StartD to Central Manager -gcloud secrets versions access latest --secret ${xp_idtoken_secret_id} ` - --out-file C:\condor\tokens.d\condor@${trust_domain} - -if ($LASTEXITCODE -ne 0) -{ - throw "Could not download HTCondor IDTOKEN; exiting startup script" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf deleted file mode 100644 index 22ef3644e8..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "access_point_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "central_manager_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "execute_point_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "pool_password" { - description = "HTCondor Pool Password" - type = string - sensitive = true - default = null -} - -variable "trust_domain" { - description = "Trust domain for HTCondor pool (if not supplied, will be set based on project_id)" - type = string - default = "" -} - -variable "user_managed_replication" { - type = list(object({ - location = string - kms_key_name = optional(string) - })) - description = "Replication parameters that will be used for defined secrets" - default = [] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf deleted file mode 100644 index d8a1d96f5f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.84" - } - random = { - source = "hashicorp/random" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md deleted file mode 100644 index 5a403c0a38..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module creates the service accounts for use by the primary elements of an -[HTCondor pool][pool]: - -- Central Managers -- Access Points -- Execute Points - -Each service account is assigned common roles necessary for the VM to function -properly. In particular, nearly every VM requires the ability to read from Cloud -Storage buckets and write Cloud Logging entries. These roles are configurable -as described below. - -[pool]: https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-admin-manual.html#the-different-roles-a-machine-can-play - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_service_accounts - source: community/modules/scheduler/htcondor-service-accounts - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - - htcondor_service_accounts - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_service_accounts - -- id: htcondor_cm - source: community/modules/scheduler/htcondor-central-manager - use: - - network1 - - htcondor_secrets - - htcondor_service_accounts - - htcondor_setup - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - central_manager_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [access\_point\_service\_account](#module\_access\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | -| [central\_manager\_service\_account](#module\_central\_manager\_service\_account) | ../../../../community/modules/project/service-account | n/a | -| [execute\_point\_service\_account](#module\_execute\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_roles](#input\_access\_point\_roles) | Project-wide roles for HTCondor Access Point service account | `list(string)` | 
[
  "compute.instanceAdmin.v1",
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [central\_manager\_roles](#input\_central\_manager\_roles) | Project-wide roles for HTCondor Central Manager service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_roles](#input\_execute\_point\_roles) | Project-wide roles for HTCondor Execute Point service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_service\_account\_email](#output\_access\_point\_service\_account\_email) | HTCondor Access Point Service Account (e-mail format) | -| [central\_manager\_service\_account\_email](#output\_central\_manager\_service\_account\_email) | HTCondor Central Manager Service Account (e-mail format) | -| [execute\_point\_service\_account\_email](#output\_execute\_point\_service\_account\_email) | HTCondor Execute Point Service Account (e-mail format) | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf deleted file mode 100644 index 9d97b18642..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# NB: the community/modules/project/service-account module will not output the -# service account e-mail address until all IAM bindings have been created; if -# underlying implementation changes, this module should declare explicit -# depends_on the IAM bindings to prevent race conditions for services that -# require them - -module "access_point_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Access Point" - deployment_name = var.deployment_name - name = "access" - project_roles = var.access_point_roles -} - -module "execute_point_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Execute Point" - deployment_name = var.deployment_name - name = "execute" - project_roles = var.execute_point_roles -} - -module "central_manager_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Central Manager" - deployment_name = var.deployment_name - name = "cm" - project_roles = var.central_manager_roles -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml deleted file mode 100644 index c4dcdffdf4..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf deleted file mode 100644 index 28f3a79457..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "access_point_service_account_email" { - description = "HTCondor Access Point Service Account (e-mail format)" - value = module.access_point_service_account.service_account_email -} - -output "central_manager_service_account_email" { - description = "HTCondor Central Manager Service Account (e-mail format)" - value = module.central_manager_service_account.service_account_email -} - -output "execute_point_service_account_email" { - description = "HTCondor Execute Point Service Account (e-mail format)" - value = module.execute_point_service_account.service_account_email -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf deleted file mode 100644 index ee186e0971..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "access_point_roles" { - description = "Project-wide roles for HTCondor Access Point service account" - type = list(string) - default = [ - "compute.instanceAdmin.v1", - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} - -variable "central_manager_roles" { - description = "Project-wide roles for HTCondor Central Manager service account" - type = list(string) - default = [ - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} - -variable "execute_point_roles" { - description = "Project-wide roles for HTCondor Execute Point service account" - type = list(string) - default = [ - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/README.md deleted file mode 100644 index 1722702ceb..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/README.md +++ /dev/null @@ -1,118 +0,0 @@ -## Description - -This module creates a bucket in which to store HTCondor configurations and -a firewall rule that allows Managed Instance Group health checks to probe the -health of HTCondor VMs. - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_service_accounts - source: community/modules/scheduler/htcondor-service-accounts - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - - htcondor_service_accounts - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_service_accounts - -- id: htcondor_cm - source: community/modules/scheduler/htcondor-central-manager - use: - - network1 - - htcondor_secrets - - htcondor_service_accounts - - htcondor_setup - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - central_manager_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [health\_check\_firewall\_rule](#module\_health\_check\_firewall\_rule) | ../../../../modules/network/firewall-rules | n/a | -| [htcondor\_bucket](#module\_htcondor\_bucket) | ../../../../modules/file-system/cloud-storage-bucket | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account e-mail for HTCondor Access Point | `string` | n/a | yes | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for HTCondor Central Manager | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account e-mail for HTCondor Execute Points | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which Central Managers will be placed. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [htcondor\_bucket\_name](#output\_htcondor\_bucket\_name) | Name of the HTCondor configuration bucket | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf deleted file mode 100644 index e048362663..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-setup", ghpc_role = "scheduler" }) -} - -locals { - service_account_iam_email = [ - "serviceAccount:${var.access_point_service_account_email}", - "serviceAccount:${var.central_manager_service_account_email}", - "serviceAccount:${var.execute_point_service_account_email}", - ] - service_account_email = [ - var.access_point_service_account_email, - var.central_manager_service_account_email, - var.execute_point_service_account_email, - ] -} - -module "health_check_firewall_rule" { - source = "../../../../modules/network/firewall-rules" - - subnetwork_self_link = var.subnetwork_self_link - - ingress_rules = [{ - name = "allow-health-check-${var.deployment_name}" - description = "Allow Managed Instance Group Health Checks for HTCondor VMs" - direction = "INGRESS" - source_ranges = [ - "130.211.0.0/22", - "35.191.0.0/16", - ] - target_service_accounts = local.service_account_email - allow = [{ - protocol = "tcp" - ports = ["9618"] - }] - }] -} - -module "htcondor_bucket" { - source = "../../../../modules/file-system/cloud-storage-bucket" - - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - name_prefix = "${var.deployment_name}-htcondor-config" - random_suffix = true - labels = local.labels - viewers = local.service_account_iam_email - - use_deployment_name_in_bucket_name = false -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml deleted file mode 100644 index 7b4918b962..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - iam.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf deleted file mode 100644 index a44223faee..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "htcondor_bucket_name" { - description = "Name of the HTCondor configuration bucket" - value = module.htcondor_bucket.gcs_bucket_name - - # ensure that all IAM bindings to the bucket and firewall rules are active - # before this modules output is allowed to propagate - depends_on = [ - module.htcondor_bucket, - module.health_check_firewall_rule - ] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf deleted file mode 100644 index 147a2ca88d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which Central Managers will be placed." - type = string -} - -variable "access_point_service_account_email" { - description = "Service account e-mail for HTCondor Access Point" - type = string -} - -variable "central_manager_service_account_email" { - description = "Service account e-mail for HTCondor Central Manager" - type = string -} - -variable "execute_point_service_account_email" { - description = "Service account e-mail for HTCondor Execute Points" - type = string -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md deleted file mode 100644 index 43254cbfa8..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md +++ /dev/null @@ -1,405 +0,0 @@ -## Description - -This module creates a slurm controller node via the internal -[slurm\_instance\_template] module. - -More information about Slurm On GCP can be found at the -[project's GitHub page][slurm-gcp] and in the -[Slurm on Google Cloud User Guide][slurm-ug]. - -The [user guide][slurm-ug] provides detailed instructions on customizing and -enhancing the Slurm on GCP cluster as well as recommendations on configuring the -controller for optimal performance at different scales. - -[slurm\_instance\_template]: /community/modules/internal/slurm-gcp/instance_template/README.md -[slurm-ug]: https://goo.gle/slurm-gcp-user-guide. -[enable\_cleanup\_compute]: #input\_enable\_cleanup\_compute -[enable\_cleanup\_subscriptions]: #input\_enable\_cleanup\_subscriptions -[enable\_reconfigure]: #input\_enable\_reconfigure - -### Example - -```yaml -- id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: - - network - - homefs - - compute_partition - settings: - machine_type: c2-standard-8 -``` - -This creates a controller node with the following attributes: - -* connected to the primary subnetwork of `network` -* the filesystem with the ID `homefs` (defined elsewhere in the blueprint) - mounted -* One partition with the ID `compute_partition` (defined elsewhere in the - blueprint) -* machine type upgraded from the default `c2-standard-4` to `c2-standard-8` - -### Live Cluster Reconfiguration - -The `schedmd-slurm-gcp-v6-controller` module supports the reconfiguration of -partitions and slurm configuration in a running, active cluster. - -To reconfigure a running cluster: - -1. Edit the blueprint with the desired configuration changes -2. Call `gcluster create -w` to overwrite the deployment directory -3. Follow instructions in terminal to deploy - -The following are examples of updates that can be made to a running cluster: - -* Add or remove a partition to the cluster -* Resize an existing partition -* Attach new network storage to an existing partition - -> **NOTE**: Changing the VM `machine_type` of a partition may not work. -> It is better to create a new partition and delete the old one. - -## Custom Images - -For more information on creating valid custom images for the controller VM -instance or for custom instance templates, see our [vm-images.md] documentation -page. - -[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images - -## GPU Support - -More information on GPU support in Slurm on GCP and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) - -## Reservation for Scheduled Maintenance - -A [maintenance event](https://cloud.google.com/compute/docs/instances/host-maintenance-overview#maintenanceevents) is when a compute engine stops a VM to perform a hardware or -software update which is determined by the host maintenance policy. This can -also affect the running jobs if the maintenance kicks in. Now, Customers can -protect jobs from getting terminated due to maintenance using the cluster -toolkit. You can enable creation of reservation for scheduled maintenance for -your compute nodeset and Slurm will reserve your node for maintenance during the -maintenance window. If you try to schedule any jobs which overlap with the -maintenance reservation, Slurm would not schedule any job. - -You can specify in your blueprint like - -```yaml - - id: compute_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network] - settings: - enable_maintenance_reservation: true -``` - -To enable creation of reservation for maintenance. - -While running job on slurm cluster, you can specify total run time of the job -using [-t flag](https://slurm.schedmd.com/srun.html#OPT_time).This would only -run the job outside of the maintenance window. - -```shell -srun -n1 -pcompute -t 10:00 -``` - -Currently upcoming maintenance notification is supported in ALPHA version of -compute API. You can update the API version from your blueprint, - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - settings: - endpoint_versions: - compute: "alpha" -``` - -## Opportunistic GCP maintenance in Slurm - -Customers can also enable running GCP maintenance as Slurm job opportunistically -to perform early maintenance. If a node is detected for maintenance, Slurm will -create a job to perform maintenance and put it in the job queue. - -If [backfill](https://slurm.schedmd.com/sched_config.html#backfill) scheduler is -used, Slurm will backfill maintenance job if it can find any empty time window. - -Customer can also choose builtin scheduler type. In this case, Slurm would run -maintenance job in strictly priority order. If the maintenance job doesn't kick -in, then forced maintenance will take place at scheduled window. - -Customer can enable this feature at nodeset level by, - -```yaml - - id: debug_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network] - settings: - enable_opportunistic_maintenance: true -``` - -## Placement Max Distance - -When using -[enable_placement](../../../../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md#input_enable_placement) -with Slurm, Google Compute Engine will attempt to place VMs as physically close -together as possible. Capacity constraints at the time of VM creation may still -force VMs to be spread across multiple racks. Google provides the `max-distance` -flag which can used to control the maximum spreading allowed. Read more about -`max-distance` in the -[official docs](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies -). - -You can use the `placement_max_distance` setting on the nodeset module to control the `max-distance` behavior. See the following example: - -```yaml - - id: nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [ network ] - settings: - machine_type: c2-standard-4 - node_count_dynamic_max: 30 - enable_placement: true - placement_max_distance: 1 - -> [!NOTE] -> `schedmd-slurm-gcp-v6-nodeset.settings.enable_placement: true` must also be -> set for placement_max_distance to take effect. - -In the above case using a value of 1 will restrict VM to be placed on the same -rack. You can confirm that the `max-distance` was applied by calling the -following command while jobs are running: - -```shell -gcloud beta compute resource-policies list \ - --format='yaml(name,groupPlacementPolicy.maxDistance)' -``` - -> [!WARNING] -> If a zone lacks capacity, using a lower `max-distance` value (such as 1) is -> more likely to cause VMs creation to fail. - -## TreeWidth and Node Communication - -Slurm uses a fan out mechanism to communicate large groups of nodes. The shape -of this fan out tree is determined by the -[TreeWidth](https://slurm.schedmd.com/slurm.conf.html#OPT_TreeWidth) -configuration variable. - -In the cloud, this fan out mechanism can become unstable when nodes restart with -new IP addresses. You can enforce that all nodes communicate directly with the -controller by setting TreeWidth to a value >= largest partition. - -If the largest partition was 200 nodes, configure the blueprint as follows: - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - ... - settings: - cloud_parameters: - tree_width: 200 -``` - -The default has been set to 128. Values above this have not been fully tested -and may cause congestion on the controller. A more scalable solution is under -way. - -## ResumeRate and Node Resumption - -The `ResumeRate` parameter in `slurm.conf` controls the maximum number of nodes -that Slurm attempts to resume (power up) per minute. This is particularly -important in cloud environments where auto-scaling can lead to a large number of -nodes starting concurrently. - -When many nodes start simultaneously, they can place a heavy load on shared -resources, especially shared filesystems, as they all try to mount filesystems -and access configuration files at the same time. By limiting the `ResumeRate`, -you can stagger the node startup process, reducing the peak load on these shared -resources and improving overall cluster stability during scaling events. - -For example, to limit the node resumption rate to 100 nodes per minute, -configure the blueprint as follows: - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - ... - settings: - cloud_parameters: - resume_rate: 100 -``` - -Adjust this value based on the capabilities of your shared filesystem and the -expected scaling behavior of your cluster. - -## Support -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 6.41 | -| [google-beta](#requirement\_google-beta) | >= 6.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.41 | -| [google-beta](#provider\_google-beta) | >= 6.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [bucket](#module\_bucket) | terraform-google-modules/cloud-storage/google | >= 6.1 | -| [daos\_network\_storage\_scripts](#module\_daos\_network\_storage\_scripts) | ../../../../modules/scripts/startup-script | n/a | -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [login](#module\_login) | ../../internal/slurm-gcp/login | n/a | -| [nodeset\_cleanup](#module\_nodeset\_cleanup) | ./modules/cleanup_compute | n/a | -| [nodeset\_cleanup\_tpu](#module\_nodeset\_cleanup\_tpu) | ./modules/cleanup_tpu | n/a | -| [slurm\_controller\_template](#module\_slurm\_controller\_template) | ../../internal/slurm-gcp/instance_template | n/a | -| [slurm\_files](#module\_slurm\_files) | ./modules/slurm_files | n/a | -| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | -| [slurm\_nodeset\_tpu](#module\_slurm\_nodeset\_tpu) | ../../internal/slurm-gcp/nodeset_tpu | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance_from_template.controller](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_from_template) | resource | -| [google_compute_disk.controller_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_secret_manager_secret.cloudsql](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_iam_member.cloudsql_secret_accessor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_version.cloudsql_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [google_storage_bucket_iam_member.legacy_readers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | -| [google_storage_bucket_iam_member.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | -| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_project.controller_project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = string
    device_name                = string
    disk_type                  = string
    disk_size_gb               = number
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. If not specified, then one will be chosen based on slurm\_cluster\_name. | `string` | `null` | no | -| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket.
Ignored when 'create\_bucket' is true. | `string` | `null` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | -| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Defaults inherited from [Slurm GCP repo](https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/terraform/slurm_cluster/modules/slurm_files/README_TF.md#input_cloud_parameters) | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | -| [cloudsql](#input\_cloudsql) | Use this database instead of the one on the controller.
server\_ip : Address of the database server.
user : The user to access the database as.
password : The password, given the user, to access the given database. (sensitive)
db\_name : The database to access.
user\_managed\_replication : The list of location and (optional) kms\_key\_name for secret | 
object({
    server_ip = string
    user      = string
    password  = string # sensitive
    db_name   = string
    user_managed_replication = optional(list(object({
      location     = string
      kms_key_name = optional(string)
    })), [])
  })
| `null` | no | -| [compute\_startup\_script](#input\_compute\_startup\_script) | DEPRECATED: `compute_startup_script` has been deprecated.
Use `startup_script` of nodeset module instead. | `any` | `null` | no | -| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script in compute nodes. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | -| [controller\_project\_id](#input\_controller\_project\_id) | Optionally. Provision controller and config bucket in the different project | `string` | `null` | no | -| [controller\_startup\_script](#input\_controller\_startup\_script) | Startup script used by the controller VM. | `string` | `"# no-op"` | no | -| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    type = string
    size = number
  })
| 
{
  "size": 50,
  "type": "pd-ssd"
}
| no | -| [create\_bucket](#input\_create\_bucket) | Create GCS bucket instead of using an existing one. | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment. | `string` | n/a | yes | -| [disable\_controller\_public\_ips](#input\_disable\_controller\_public\_ips) | DEPRECATED: Use `enable_controller_public_ips` instead. | `bool` | `null` | no | -| [disable\_default\_mounts](#input\_disable\_default\_mounts) | DEPRECATED: Use `enable_default_mounts` instead. | `bool` | `null` | no | -| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | -| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | `true` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_controller\_public\_ips](#input\_enable\_controller\_public\_ips) | If set to true. The controller will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. | `bool` | `false` | no | -| [enable\_default\_mounts](#input\_enable\_default\_mounts) | Enable default global network storage from the controller
- /home
- /opt/apps | `bool` | `true` | no | -| [enable\_devel](#input\_enable\_devel) | DEPRECATED: `enable_devel` is always on. | `bool` | `null` | no | -| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/tools/prologs-epilogs/README.md | `bool` | `null` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | -| [enable\_slurm\_gcp\_plugins](#input\_enable\_slurm\_gcp\_plugins) | DEPRECATED: Slurm GCP plugins have been deprecated.
Instead of 'max\_hops' plugin please use the 'placement\_max\_distance' nodeset property.
Instead of 'enable\_vpmu' plugin please use 'advanced\_machine\_features.performance\_monitoring\_unit' nodeset property. | `any` | `null` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": "beta"
}
| no | -| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for controller. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | -| [login\_network\_storage](#input\_login\_network\_storage) | An array of network attached storage mounts to be configured on all login nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [login\_nodes](#input\_login\_nodes) | List of slurm login instance definitions. | 
list(object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  }))
| `[]` | no | -| [login\_startup\_script](#input\_login\_startup\_script) | Startup script used by the login VMs. | `string` | `"# no-op"` | no | -| [login\_startup\_scripts\_timeout](#input\_login\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in login\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = optional(map(string))
    mount_runner          = optional(map(string))
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | Define nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | -| [partitions](#input\_partitions) | Cluster partitions as a list. See module slurm\_partition. | 
list(object({
    partition_name        = string
    partition_conf        = optional(map(string), {})
    partition_nodeset     = optional(list(string), [])
    partition_nodeset_dyn = optional(list(string), [])
    partition_nodeset_tpu = optional(list(string), [])
    enable_job_exclusive  = optional(bool, false)
  }))
| `[]` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [region](#input\_region) | The default region to place resources in. | `string` | n/a | yes | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the controller instance. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the controller instance. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming and slurm accounting.
If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). | `string` | `null` | no | -| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | -| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | -| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Post deployment instructions. | -| [slurm\_bucket](#output\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | -| [slurm\_bucket\_dir](#output\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | -| [slurm\_bucket\_name](#output\_slurm\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | -| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | Bucket path used by cluster. | -| [slurm\_cluster\_name](#output\_slurm\_cluster\_name) | Slurm cluster name. | -| [slurm\_controller\_instance](#output\_slurm\_controller\_instance) | Compute instance of controller node | -| [slurm\_login\_instances](#output\_slurm\_login\_instances) | Compute instances of login nodes | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf deleted file mode 100644 index 4a887b99cf..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.disk_resource_manager_tags - } - ] - - state_disk = var.controller_state_disk != null ? [{ - source = google_compute_disk.controller_disk[0].name - device_name = google_compute_disk.controller_disk[0].name - disk_labels = null - auto_delete = false - boot = false - }] : [] - - synth_def_sa_email = "${data.google_project.controller_project.number}-compute@developer.gserviceaccount.com" - - service_account = { - email = coalesce(var.service_account_email, local.synth_def_sa_email) - scopes = var.service_account_scopes - } - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata, - local.universe_domain - ) - - controller_project_id = coalesce(var.controller_project_id, var.project_id) -} - -data "google_project" "controller_project" { - project_id = local.controller_project_id -} - -resource "google_compute_disk" "controller_disk" { - count = var.controller_state_disk != null ? 1 : 0 - - project = local.controller_project_id - name = "${local.slurm_cluster_name}-controller-save" - type = var.controller_state_disk.type - size = var.controller_state_disk.size - zone = var.zone -} - -# INSTANCE TEMPLATE -module "slurm_controller_template" { - source = "../../internal/slurm-gcp/instance_template" - - project_id = local.controller_project_id - region = var.region - slurm_instance_role = "controller" - slurm_cluster_name = local.slurm_cluster_name - labels = local.labels - - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(var.disk_labels, local.labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = concat(local.additional_disks, local.state_disk) - - bandwidth_tier = var.bandwidth_tier - slurm_bucket_path = module.slurm_files.slurm_bucket_path - can_ip_forward = var.can_ip_forward - advanced_machine_features = var.advanced_machine_features - resource_manager_tags = var.resource_manager_tags - - enable_confidential_vm = var.enable_confidential_vm - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - gpu = one(module.gpu.guest_accelerator) - - machine_type = var.machine_type - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - service_account = local.service_account - - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - subnetwork = var.subnetwork_self_link - - tags = concat([local.slurm_cluster_name], var.tags) - # termination_action = TODO: add support for termination_action (?) -} - -# INSTANCE -resource "google_compute_instance_from_template" "controller" { - provider = google-beta - - name = "${local.slurm_cluster_name}-controller" - project = local.controller_project_id - zone = var.zone - source_instance_template = module.slurm_controller_template.self_link - # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 - # we have to explicitly override instance labels instead of inheriting them from template. - labels = module.slurm_controller_template.labels - - allow_stopping_for_update = true - - # Can't rely on template to specify nics due to usage of static_ip - network_interface { - dynamic "access_config" { - for_each = var.enable_controller_public_ips ? ["unit"] : [] - content { - nat_ip = null - network_tier = null - } - } - network_ip = length(var.static_ips) == 0 ? "" : var.static_ips[0] - subnetwork = var.subnetwork_self_link - } - - dynamic "network_interface" { - for_each = var.controller_network_attachment != null ? [1] : [] - content { - network_attachment = var.controller_network_attachment - } - } -} - -moved { - from = module.slurm_controller_instance.google_compute_instance_from_template.slurm_instance[0] - to = google_compute_instance_from_template.controller -} - -# SECRETS: CLOUDSQL -resource "google_secret_manager_secret" "cloudsql" { - count = var.cloudsql != null ? 1 : 0 - - secret_id = "${local.slurm_cluster_name}-slurm-secret-cloudsql" - project = var.project_id - - replication { - dynamic "auto" { - for_each = length(var.cloudsql.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.cloudsql.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = nonsensitive(var.cloudsql.user_managed_replication) - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } - - labels = { - slurm_cluster_name = local.slurm_cluster_name - } -} - -resource "google_secret_manager_secret_version" "cloudsql_version" { - count = var.cloudsql != null ? 1 : 0 - - secret = google_secret_manager_secret.cloudsql[0].id - secret_data = jsonencode(var.cloudsql) -} - -resource "google_secret_manager_secret_iam_member" "cloudsql_secret_accessor" { - count = var.cloudsql != null ? 1 : 0 - - secret_id = google_secret_manager_secret.cloudsql[0].id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${local.service_account.email}" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl deleted file mode 100644 index 219bdc5227..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl +++ /dev/null @@ -1,65 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/high_throughput.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxArraySize=10001 -MaxJobCount=500000 -MaxNodeCount=65536 -MinJobAge=60 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -SlurmctldDebug=error -SlurmdDebug=error - -# -# -# TIMERS -MessageTimeout=60 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=120 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl deleted file mode 100644 index 93ac47e341..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl +++ /dev/null @@ -1,34 +0,0 @@ -# slurmdbd.conf -# https://slurm.schedmd.com/slurmdbd.conf.html - -DebugLevel=info -PidFile=/var/run/slurm/slurmdbd.pid - -# https://slurm.schedmd.com/slurmdbd.conf.html#OPT_CommitDelay -CommitDelay=1 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -AuthType=auth/{auth_key} -AuthAltTypes=auth/jwt -AuthAltParameters=jwt_key={state_save}/jwt_hs256.key - -DbdHost={control_host} - -LogFile={slurmlog}/slurmdbd.log - -SlurmUser=slurm - -StorageLoc={db_name} - -StorageType=accounting_storage/mysql -StorageHost={db_host} -StoragePort={db_port} -StorageUser={db_user} -StoragePass={db_pass} - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl deleted file mode 100644 index d3f2615a68..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl +++ /dev/null @@ -1,71 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/slurm.conf.html -# https://slurm.schedmd.com/configurator.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxNodeCount=64000 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -AccountingStoreFlags=job_comment -JobAcctGatherFrequency=30 -JobAcctGatherType=jobacct_gather/cgroup -SlurmctldDebug=info -SlurmdDebug=info -DebugFlags=Power - -# -# -# TIMERS -MessageTimeout=600 -BatchStartTimeout=600 -PrologEpilogTimeout=600 -PrologFlags=Contain - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=600 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf deleted file mode 100644 index 21e915a125..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -locals { - # TODO: deprecate `var.login_[ startup_script, startup_scripts_timeout, network_storage]` - # in favour of vars defined in user-facing login module - ghpc_startup_login = [{ - filename = "ghpc_startup.sh" - content = var.login_startup_script - }] - - login_startup_scripts = concat(local.common_scripts, local.ghpc_startup_login) -} - -module "login" { - source = "../../internal/slurm-gcp/login" - for_each = { for x in var.login_nodes : x.group_name => x } - - project_id = var.project_id - - slurm_cluster_name = local.slurm_cluster_name - slurm_bucket_path = module.slurm_files.slurm_bucket_path - slurm_bucket_name = module.slurm_files.bucket_name - slurm_bucket_dir = module.slurm_files.bucket_dir - - login_nodes = each.value - - startup_scripts = local.login_startup_scripts - startup_scripts_timeout = var.login_startup_scripts_timeout - - network_storage = var.login_network_storage - - universe_domain = var.universe_domain - - # trigger replacement of login nodes when the controller instance is replaced - # Needed for re-mounting volumes hosted on controller - replace_trigger = google_compute_instance_from_template.controller.self_link -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf deleted file mode 100644 index 7622bdffef..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-controller", ghpc_role = "scheduler" }) -} - -locals { - # Since deployment name may be used to create a cluster name, we remove any invalid character from the beginning - # Also, slurm imposed a lot of restrictions to this name, so we format it to an acceptable string - tmp_cluster_name = substr(replace(lower(var.deployment_name), "/^[^a-z]*|[^a-z0-9]/", ""), 0, 10) - slurm_cluster_name = coalesce(var.slurm_cluster_name, local.tmp_cluster_name) - - universe_domain = { "universe_domain" = var.universe_domain } -} - -# See -# * slurm_files.tf -# * controller.tf -# * partition.tf -# * login.tf diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml deleted file mode 100644 index 7b4918b962..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - iam.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md deleted file mode 100644 index 002bf14145..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md +++ /dev/null @@ -1,42 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name         = string
    subnetwork_self_link = string
    additional_networks = list(object({
      subnetwork = string
    }))
  })
| n/a | yes | -| [nodeset\_template](#input\_nodeset\_template) | Self link of the nodeset template | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf deleted file mode 100644 index bd8773cf84..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - cleanup_dependencies_agg = flatten([ - var.nodeset.subnetwork_self_link, - var.nodeset.additional_networks[*].subnetwork, - var.nodeset_template]) -} - -# Can not use variadic list in `depends_on`, wrap it into a collection of `null_resource` -resource "null_resource" "dependencies" { - count = length(local.cleanup_dependencies_agg) -} - -resource "null_resource" "script" { - count = var.enable_cleanup_compute ? 1 : 0 - - triggers = { - project_id = var.project_id - cluster_name = var.slurm_cluster_name - nodeset_name = var.nodeset.nodeset_name - universe_domain = var.universe_domain - compute_endpoint_version = var.endpoint_versions.compute - gcloud_path_override = var.gcloud_path_override - } - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/cleanup_compute.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" - when = destroy - } - - # Ensure that clean up is done before attempt to delete the networks - depends_on = [null_resource.dependencies] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh deleted file mode 100644 index a98243d464..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -project="$1" -cluster_name="$2" -nodeset_name="$3" -universe_domain="$4" -compute_endpoint_version="$5" -gcloud_dir="$6" -MAX_ATTEMPTS=3 - -if [[ $# -ne 5 ]] && [[ $# -ne 6 ]]; then - echo "Usage: $0 []" - exit 1 -fi - -if [[ -n "${gcloud_dir}" ]]; then - export PATH="$gcloud_dir:$PATH" -fi - -export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" -export CLOUDSDK_CORE_PROJECT="${project}" - -if ! type -P gcloud 1>/dev/null; then - echo "gcloud is not available and your compute resources are not being cleaned up" - echo "https://console.cloud.google.com/compute/instances?project=${project}" - exit 1 -fi - -tmpfile=$(mktemp) # have to use a temp file, since `< <(gcloud ...)` doesn't work nicely with `head` -trap 'rm -f "$tmpfile"' EXIT - -echo "Deleting managed instance groups" -mig_filter="name:${cluster_name}-${nodeset_name}-*" -gcloud compute instance-groups managed list --format="value(self_link)" --filter="${mig_filter}" >"$tmpfile" -while batch="$(head -n 5)" && [[ ${#batch} -gt 0 ]]; do - groups=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line - # The lack of quotes around ${groups} is intentional and causes each new space-separated "word" to - # be treated as independent arguments. See PR#2523 - # shellcheck disable=SC2086 - for _ in $( #occasionally MIGs will fail to delete due to some active transformation happening, so let's retry - seq 1 $MAX_ATTEMPTS - ); do - if gcloud compute instance-groups managed delete --quiet ${groups}; then - break - fi - echo "MIG deletion failed, retrying" - done -done <"$tmpfile" -true >"$tmpfile" # Wipe contents of tmp file - -echo "Deleting compute nodes" -node_filter="name:${cluster_name}-${nodeset_name}-* labels.slurm_cluster_name=${cluster_name} AND labels.slurm_instance_role=compute" - -running_nodes_filter="${node_filter} AND status!=STOPPING" -# List all currently running instances and attempt to delete them -gcloud compute instances list --format="value(selfLink)" --filter="${running_nodes_filter}" >"$tmpfile" -# Do 500 instances at a time -while batch="$(head -n 500)" && [[ ${#batch} -gt 0 ]]; do - nodes=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line - # The lack of quotes around ${nodes} is intentional and causes each new space-separated "word" to - # be treated as independent arguments. See PR#2523 - # shellcheck disable=SC2086 - gcloud compute instances delete --quiet ${nodes} || echo "Failed to delete some instances" -done <"$tmpfile" - -# In case if controller tries to delete the nodes as well, -# wait until nodes in STOPPING state are deleted, before deleting the resource policies -stopping_nodes_filter="${node_filter} AND status=STOPPING" -while true; do - node=$(gcloud compute instances list --format="value(name)" --filter="${stopping_nodes_filter}" --limit=1) - if [[ -z "${node}" ]]; then - break - fi - echo "Waiting for instances to be deleted: ${node}" - sleep 5 -done - -echo "Deleting resource policies" -policies_filter="name:${cluster_name}-slurmgcp-managed-${nodeset_name}-*" -gcloud compute resource-policies list --format="value(selfLink)" --filter="${policies_filter}" | while read -r line; do - echo "Deleting resource policy: $line" - gcloud compute resource-policies delete --quiet "${line}" || { - echo "Failed to delete resource policy: $line" - } -done diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf deleted file mode 100644 index b6da69931c..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "Project ID" -} - - -variable "slurm_cluster_name" { - type = string - description = "Name of the Slurm cluster" -} - -variable "enable_cleanup_compute" { - description = < [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | 3.2.3 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf deleted file mode 100644 index ec86a03a24..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "null_resource" "script" { - count = var.enable_cleanup_compute ? 1 : 0 - - triggers = { - project_id = var.project_id - cluster_name = var.slurm_cluster_name - nodeset_name = var.nodeset.nodeset_name - zone = var.nodeset.zone - universe_domain = var.universe_domain - compute_endpoint_version = var.endpoint_versions.compute - gcloud_path_override = var.gcloud_path_override - } - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/cleanup_tpu.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.zone} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" - when = destroy - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh deleted file mode 100644 index c724e342c3..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -project="$1" -cluster_name="$2" -nodeset_name="$3" -zone="$4" -universe_domain="$5" -compute_endpoint_version="$6" -gcloud_dir="$7" - -if [[ $# -ne 6 ]] && [[ $# -ne 7 ]]; then - echo "Usage: $0 []" - exit 1 -fi - -if [[ -n "${gcloud_dir}" ]]; then - export PATH="$gcloud_dir:$PATH" -fi - -export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" -export CLOUDSDK_CORE_PROJECT="${project}" - -if ! type -P gcloud 1>/dev/null; then - echo "gcloud is not available and your compute resources are not being cleaned up" - echo "https://console.cloud.google.com/compute/instances?project=${project}" - exit 1 -fi - -echo "Deleting TPU nodes" -node_filter="name~${cluster_name}-${nodeset_name}" -running_nodes_filter="${node_filter} AND state!=DELETING" - -# List all currently running nodes and attempt to delete them -gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${running_nodes_filter}" | while read -r name; do - echo "Deleting TPU node: $name" - gcloud compute tpus tpu-vm delete --async --zone="${zone}" --quiet "${name}" || echo "Failed to delete $name" -done - -# Wait until nodes in DELETING state are deleted, before deleting the resource policies -deleting_nodes_filter="${node_filter} AND state=DELETING" -while true; do - node=$(gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${deleting_nodes_filter}" --limit=1) - if [[ -z "${node}" ]]; then - break - fi - echo "Waiting for nodes to be deleted: ${node}" - sleep 5 -done diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf deleted file mode 100644 index 1ac6f64b75..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright (C) Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "Project ID" -} - -variable "slurm_cluster_name" { - type = string - description = "Name of the Slurm cluster" -} - -variable "enable_cleanup_compute" { - description = < -Copyright (C) SchedMD LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.3 | -| [archive](#requirement\_archive) | ~> 2.0 | -| [google](#requirement\_google) | >= 6.41 | -| [local](#requirement\_local) | ~> 2.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [archive](#provider\_archive) | ~> 2.0 | -| [google](#provider\_google) | >= 6.41 | -| [local](#provider\_local) | ~> 2.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.controller_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.devel](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.devel_compute](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_dyn_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_tpu_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.task_epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.task_prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [random_uuid.cluster_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/uuid) | resource | -| [archive_file.slurm_gcp_devel_compute_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | -| [archive_file.slurm_gcp_devel_controller_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | -| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | -| [local_file.chs_gpu_health_check](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.external_epilog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.external_prolog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.setup_external](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. | `string` | `null` | no | -| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket to use. | `string` | n/a | yes | -| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | -| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Default behavior defined in scripts/conf.py | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | -| [cloudsql\_secret](#input\_cloudsql\_secret) | Secret URI to cloudsql secret. | `string` | `null` | no | -| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in compute\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | -| [controller\_startup\_scripts](#input\_controller\_startup\_scripts) | List of scripts to be ran on controller VM startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | -| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    device_name = string
  })
| 
{
  "device_name": null
}
| no | -| [disable\_default\_mounts](#input\_disable\_default\_mounts) | Disable default global network storage from the controller
- /home
- /apps | `bool` | `false` | no | -| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. Not for production use. | `bool` | `false` | no | -| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/v5/tools/prologs-epilogs/README.md | `bool` | `false` | no | -| [enable\_hybrid](#input\_enable\_hybrid) | Enables use of hybrid controller mode. When true, controller\_hybrid\_config will
be used instead of controller\_instance\_config and will disable login instances. | `bool` | `false` | no | -| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": null
}
| no | -| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | -| [google\_app\_cred\_path](#input\_google\_app\_cred\_path) | Path to Google Application Credentials. | `string` | `null` | no | -| [install\_dir](#input\_install\_dir) | Directory where the hybrid configuration directory will be installed on the
on-premise controller (e.g. /etc/slurm/hybrid). This updates the prefix path
for the resume and suspend scripts in the generated `cloud.conf` file.

This variable should be used when the TerraformHost and the SlurmctldHost
are different.

This will default to var.output\_dir if null. | `string` | `null` | no | -| [munge\_mount](#input\_munge\_mount) | Remote munge mount for compute and login nodes to acquire the munge.key.
By default, the munge mount server will be assumed to be the
`var.slurm_control_host` (or `var.slurm_control_addr` if non-null) when
`server_ip=null`. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| 
{
  "fs_type": "nfs",
  "mount_options": "",
  "remote_mount": "/etc/munge/",
  "server_ip": null
}
| no | -| [network\_storage](#input\_network\_storage) | Storage to mounted on all instances.
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | Cluster nodenets, as a list. | `list(any)` | `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Cluster nodenets (dynamic), as a list. | `list(any)` | `[]` | no | -| [nodeset\_startup\_scripts](#input\_nodeset\_startup\_scripts) | List of scripts to be ran on compute VM startup in the specific nodeset. | 
map(list(object({
    filename = string
    content  = string
  })))
| `{}` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Cluster nodenets (TPU), as a list. | `list(any)` | `[]` | no | -| [output\_dir](#input\_output\_dir) | Directory where this module will write its files to. These files include:
cloud.conf; cloud\_gres.conf; config.yaml; resume.py; suspend.py; and util.py. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The GCP project ID. | `string` | n/a | yes | -| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [slurm\_bin\_dir](#input\_slurm\_bin\_dir) | Path to directory of Slurm binary commands (e.g. scontrol, sinfo). If 'null',
then it will be assumed that binaries are in $PATH. | `string` | `null` | no | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | The cluster name, used for resource naming and slurm accounting. | `string` | n/a | yes | -| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | -| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | -| [slurm\_control\_addr](#input\_slurm\_control\_addr) | The IP address or a name by which the address can be identified.

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | -| [slurm\_control\_host](#input\_slurm\_control\_host) | The short, or long, hostname of the machine where Slurm control daemon is
executed (i.e. the name returned by the command "hostname -s").

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | -| [slurm\_control\_host\_port](#input\_slurm\_control\_host\_port) | The port number that the Slurm controller, slurmctld, listens to for work.

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldPort | `string` | `"6818"` | no | -| [slurm\_key\_mount](#input\_slurm\_key\_mount) | Remote mount for compute and login nodes to acquire the slurm.key. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| `null` | no | -| [slurm\_log\_dir](#input\_slurm\_log\_dir) | Directory where Slurm logs to. | `string` | `"/var/log/slurm"` | no | -| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | -| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [bucket\_dir](#output\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | -| [bucket\_name](#output\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | -| [config](#output\_config) | Cluster configuration. | -| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel-controller.zip b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel-controller.zip deleted file mode 100644 index 902c068c2a01a723f897028303de7b2985f2c32e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 84485 zcmagEQ;;r9)MiFWwr$(Cb;{OTwr$%!W!tu`{w6wNCSoG`@65Zr&5V_MuV*XD zfPkU`LH@V@{|n;30}}^(b9zUwDPKF6P0ocqk8fz&ZzEC$I-ZG}rIcD$w~XP8=UkI6 zTuqELnb*92q-9$15 zETdiL70W>zKy)P)SB;DKO^5y=W9qnTvxI*W>9DYhK8m#SylsSLm}7{wDJH*7P1gGU z{`dEHGI5g%kw$n4uxfZ2#ckXyeLRA`LOkNJiym`Hgz2wshbfqrEu>v0;M7%#S$V;S zn6!XqLQ|+2$p4mXwOhYLG_s_(3=sPwIgpa-JAWam8OKvg45Wmjopf|f#?>J^KI_1s zK)*PT|NeY*;c%K)H(uu;;OFh*?&j<(H6xJdacXg^P%A$(OF!`r@$hoR@erUevr`u0 z)ME~)cZdc32_?lH9*0g?Co@3;kukzlhMRl`oE8^9SC-I-s~^NoWUkUIFlsr0MOQ%9 zM9d5y3wuQjq&oLe?07ZQ6f0c# z77TX|yONW2&w(KL|0WktJXP&IYg%!cAA_=`rWo3L2!i%fKM`9Lnxn)sWZ0X*iNqt> z6Nz&5X8q;bN_@}8MN+3WPZS3RpV@*vkEBZq-yC}%+BOLqI`s9Fquz)KwkiYtk?$~1 z!B!w=ywDrR1!TpD7gaq*KQ2BR_)pTy3;a4er|S7}i;v6|B9&;`!MB*wGF6>zG?v@vx6N$nWEM>$59seN1eVbt;O=h90=T1p^ zZOT3#TW6*f(lv%jYcoyvs{`=!z>^q|kU)WMx`EG6r=RIIxBeOmQa1fW!9h{z5=1)` zgFMoBO)s%H3#l)ou-sX98+-dsvg`8(N^^UK`~mN?pdB2=OJ}It{+oZCS?{20^^{Y< zU7VW^6z(^R*5slp+5*>UJ2tX3^z%7-81O~5*a9OAB7eRX8a&;8etgC0 znJ#Dp$FjM0$BAh)lu?M$R+nxWO2q0PJQ<;YOeSYT_bjX>L)0dR4R zm6j)JV5~&vY!p=Qf-=Xt;58iUl~%x+D_JuJe^ag0*~pe@51*A|Y?K&Bo|e42r`!}u zt4gnNV-tph23>Pnxovf&*m5})4m{`NHZ6oW>`thjeG_mh3Yf9M{as+`?Sk9Ons~5b=xya{c~U%3DDs4KHz0R~rWtBMvMtcMV{uwv z#2K9%thRq&)fT<|=WBgqc8H%XS2j&0scG6_yLq&=w2Z3~(r4u8NR8Q$jn%-th%!w0 zImOev)wESJz{tW$nGzO>nWFl!e}qLr&XL&E+SVt-;rdOG87v5aUP-H2rU=I^Ub_W} z1%96E9Cn^sK*4@5o`12mzU$I5EM<4FE`EQ0I2+-5MHLo3N+qxNoscaDz~t#OgzVAA zkMA7`*!r`I&z&E!=faOLPc13I4$f5?^%Rsk3oqD@s`@PB!{?k0wTTaij$BAWzFtHO5e`Casym2BSZ@S2PZAXRumi#~e{&_dn=2}Te){c~J{Xw4-3!ylqSiwKQOF403&+RUCGHW z(1%d*vDmf(!3xm@q9mBupVbxTb>Ts_jA+^$MlNAt@3jhjcvt zQ|-^`sb9+;f!m)f^`?Dis;qU_bpt9d?^m>vL3Y_bRzC4RXw$3R?C%l;0#`m!@W>k^ zGD@ij_%?yo?v~aCDo-M6Ow(d!I*t^5D{37c{!S_1wB@wPCG~5FBh6v4iB<{|CF2em z>KL-dGSWLx6MJRnl1`T%EvYf+bI?7Kwq#hti)5tyP2rNH^>pbcg*DG47qjXXWm1yR zC=4`t;d2sm4ZFur# zM#g-7zKTM*R}lUJv{5A5xX?Q<$^QIVORykEk>C!S!{j87veUd>ii#Z21z%Z@gpOf1 zND7!0y~U^!P+(O?$$066rB6Q1BT~b@H@&z6tq#C1W`lU=y#UFMvC7y%&KvPw)H3Wr zL&XmAByj(Xc`zT4VcSmpyz5u(=G4fque=t3^j*>c zs{jw|4{0Y0-0j5n6n}2E89}^&ImTFCqX$ObM0Phf_l5Ktn2vh2b^276JFdJnc+tUG z_ljP|L=Y&B0*6t=I8k72>HZgIhR?YzSe0PGyaONB!+~U^X7xYSp$Ax*H&b}o_%>H` zFIoP^i$|C|bz}XBi+dJQ z@*@i8qxvYh??2VXDmb*FC?2wv?uiW!W>s%^LoC?gwle$%U{9y$f|8R&2OI~v6V-^X zZsZp+7{@S_g5)71x~A)Q!0K5I0JZ9so{x>tB6GaQy`%nXSIU{1k0g>Zn0fdiujg5e z{g)Fy9e-1_BP@)(qztvy{cl<3;2M!R5Bp-MtO_syzkAXssJ+ZuaF<8InU9k}-YSJy zq^dtpujm)B5rX2{E&)I>y>G9N2!o&|$YsX9YP9!w9R-5!hV@n3%F6_|ctQ1%*JS{1 z<@p2Lr@p#McYk?m65oegOzm)QE0jU1Ci~FZCI@T5siO~H9*q|ytn|c=cCW*Cs7x&)qykKd%^{QCOPC}wm6`kZ)-y7W9t8i@@4(Q7XDWEgO6+@y zX`C>YP-08&Pkt^ris{2_JxKmg^M{wd;2BqF+!<^qDTFghrO8A5h68f7b>Tz0SSBZ% zX2XXT-PsI=i;|6ca2}Roo|GcP7!KYj4pYvd)^GuzF_V-PEO$?ut_b{_T>LfWXhLF& zFI*e}%SMBQloS~>y8yDl&oP5N^UWEEkO6A}sTA#k;3%$YzS#2J)y&q#g)al^hHuQ2 z)Pm|)%K)g(Hk5jFCfRH@9cc^HyS*<@QKMSsg#R2wq?FP@M%oZm<`?fE`xE(GC%QC9 zkL~+xAwD7WH(%i#s^9`{5oDc4#)9wy8jf%jr&g`557eU3hyM6xjJ4b&TgYBq51vWb zv=+$`V)PpR-kLN+G5bJnjFw928D4dCA3od22@tL3l_@&CA7z7j zU6W30AnI6lw0Jlhxf`nteE!SdM(fKI9#GG#CyKQ|H}Aaj(ro(B>T>+s+i>fb?iBE_ zyW$l3fVN2V=(LWU-W2p`V&LshGsj=G5%mAS0 zCE6|cvoY{QU|I?=oZ((>MRev16yT_+`>_+zdQCQ@HRna(E!g2s)Rai^cj!n=;0-k{GKh93D^c8Od(=abHp#GWxDeHa^ zGOe!G*>B|%S>`0xm`0t)SVSK)A~ z?I^lQZej906b52#tb6m7_{3IIsk z&836jUOYSdEb?^Y*fs^fhxEmE+icq;7D2jtzBm7*smWRh*-XIT5n=W#Xe35E!u>=r z;$2mO;KB^UNh2jPE=py|RVFNrl8uo!Gz_JHQ@rlJy^^BFK27W@H)U%`Rm6i0oJBlr zJYwH083ZWX&L1#lH0<{W;M_HS$Mn=rLOmdOQs%yPxiZ|R3_lPihq{7wnz6p@+(lZ~ z4ixXCYsUHw{6fYB-x_hK;F0vPS4|KSKbXilYVFnFb1LLIv2AtRF!_H2XGNZ(+Y76B zc#70KlQ#m_svVb`Cj{HN{+f2+|9$J16ML#R^ay{(W8n1jl+9+NO}Z=!h7kA-)`;hj zB4v3%@iGfuQ(}%h7x*w}2*ygbur7tPJ|?@h|0a2nqfnyoc8Lyx#WTn>9=3n<3Y?uG z!>PydEgjC1)o6<#kbHk!B>(K@IFJ31NM&m+T8}Py=z;Ya+`QwH#M$B_GUzZIqcidHW#_U`h0@Rsl@2X#Rmkqy9cNBsNRTf_AjrIsk@ z7;PjVc8%m(BfQq1&pSr^f{KT7b{b>*h!os$snNCv7!s@LuZomc`aC;evk3+)hP1RW z<9Lil@v)Z`p(?cIO=d$qeLb!87p#>O3(Z9f{`$mcW+dKapt5b7yB<`}>GgEkRz>}r zRynaG+!A4#!B=sxFxWIUi|0KTZhi|f=`sGdcAi2ABc6;7Kw})~Gsw-$2nV&Ix zN-eF;_Hj$`+|`yKT#j4``iVetd5ME6K>k8$v6{j!<{Ug2xgUAks?Wiif{>ywhg_|X zL0p4)*xN7>YS28AZ^41%w^&w*Uq{fVLe^#nMq^@+?;oON!fDP6D&to##<&U<+T_9# zy3wE?ysh{<TAK$t!;rvF6~*0*hSbP~%#5zWRFzK$x(MCAE>2@78k z+%vo8--a!6+nC4UUnX>9LkY2p#Rj6TbI}$<0NP6s;#J$(zvRjIro-siP6vUS6h}Z7 z=q>wdp+eD}06SfgN|gme^lmz@HmPW+don^pH&3}AR*v8?PtTmX@YZOE&9L~$m!_~h z9ewmEY%A2L9%eYuMp;0|bmy29^MKA(f}kXkt93Vg$_52CJ>KgZJxazaTL3O4I~{bH zI-<{=A6)rz#pGlWBX7S=tP$e-vz;mxIU67by@KhulX092*)45=s?^TOVyjsGk6eP& z3co_%PTQ`Eaqa2OS8n$rw+l}e-g=|n*=5!b4gqezp*MEIXAz6)!7CnG??+LV`GQJ6 zVIh~HI_QVdK#*z|2+R`I5MHQrK!7jBLxBBYsR^n4`0DrG>`-rOv!4xFqB zlP!g7@2a|irIM*o%{E0&Z3O2d1Ds}ZX!i57B_TD=`dKMjbe-n1eVGPEURga;R4ug! z$BS>PnSE$UvCq`zGMij|y`}quI#w;wDJM{TMsWyjlC~Lw$RY2Bi)e52l-w<~&Sp-M zMtLbl4S6$d7YJfRKDD}~cB;S4U#Pea_bn5?J5$0osyD+uj_W%gMEZ%>XePU?Kg}*c z1GJ@ygZN%;_Ln-dELSL-BSIP-n#d!+0we}kwta-^@SIxP>ZbNKtF@IkF!?$vnPN8x zV4L z|3>(1Gi*(+bt4gNlJ#=^f1a4SxcfL~`E&9N57 zJ=Q!r@ecdWehfixbc9fJx52RzWFys2U~jG3tUJ}FpxDt9{YIs)C|pdQakBVIb?~2{ zwucY9Sm15gucY0zn$|x{NcRd{XztA>BJ|7g_-u>#jFzdpjp8?3Hu&!v1#n4x+OB{{ zdacGHiBBqVXuDw?P6pD9mK~m^M1%ee+zo<>Xw705z6~)ulm+Ab4#Z->HnFV763DN) zHFRX{+$>3AF(8@J^H|;HerT zI2++@%!Xr#9F}QV`WLcIE0j=W#LTRH0Er2 z(jv!wVh2ufnP68FhpJLog&-1i_6p|o9`3cYDT3zl4k#Ifp<{GLE*Icgw25dNdA#D= zer)7WjGyY*lZ>$*|_!g**Hzw0;iSjFtL)QbqXtxW3>o z`W9AsbkN^yb!M~Q%dkGd!>)zE^$^tSw9}I120W64U?!Qv5%L7I6QC7UovAW(FokCYEOZMHkI#dJdcHsD7(;oC$%` z;JO`<^%w|PdmZQIwDsZKdh-;)rM0Aks1u2*s5O@X1d{8u$MlnohTf42{IeV5pr8<5&lUh`4}?MkBIYfPRt&J)WkWZ zn-;VhxpG=NUrr<7tA}2u@lXm7Q*U~qgCLWrq%AX?Ok{1|$w>B6E(T9e>pQ^9@(xY+F{P^tU&j5*osr1Tq#_IllAL4=u`pbhOX#!N21j9gA)^$W)6Y`UPp95 z5Hb@4ws>;qls38+Km9_R?6`j#T>~!fUhZMy1o=25f5xnfZ{h6u(e5)=zPo`CpV$m< z7DRI$We~=lKYlk-YHB)#Z?GieRmEJYyf-R$WhipXTX26>e%Z}G0}R3xh|STbrh4s9 zjZU@XU>h48lqaf-rn+>9_~=#&M=4rNwi9a8KAF^@&inj(AywD;A?#sR)3-)|?7yD? zK~z6)vvNO7;uGv|6pi4GWPPI-%7_9o6fy@_dh@thKrDMaBsKY8R)-{+X9k+P2j7#>qq zM17fmT^R3Z$?w?zTqH1N`4qwo=ObA`wD0`aO;l7+W&iOVhTl1hp5B_TmSvIte`MKZ zKE)vb1p?ZF`k%;x{2y5?%v=pz9o-Dv?Mxip>|OuEONg4B!zL$6@1=(9aWoufV^(ij z_uW8Nao1?8JR%QWR3W0J73p*)u@<1wg!ZM6K~gGJwM`4p)|r!F*7t7zYPgtIFc5Pf*d)7WPLE)DqC~_vX_5Lo#?A|tD1e$5HwF?Qvg4= z?au4{sae-S;}yr#GiWpXdE2_nThn20pO1FcU~hK$!ya0)RS@p2uz84qErF)U0nZi@ z0*F})gc%y;A0_aNsD1-3Q5wWjkSIK~o75=VA`prd;~#l4E%Tlu;tER0a#dv=Lv9UO z9NX6vvYpzR${<2XreIt!*2*ZfMfWRza*n{uY;h2n^iQ^38ck0Pq%>!P ziu>V7J@UbJ_Pd@a0Bw9W$OBQX7&vBI2`;lj)VifZJ6~9&N9X@YU4pgA1jy&^a!f$z?a#* zo+;TNmkh-Mq%28svRS#3rX{hU3a|^3R8dXy->Nnthq8H4AwjI+21ByZhDrXILqKB0 zVg2Z(PmPh&jZgEEms(+c{7bTL==*&WQe1+qrbbI0S-W;oOsW6F9 z#iQBGhy88wxoN)2>dgfEjJQ>^(-$w`*X`%2@4#_y{(jHzey^K-87(>v1abrPuNvhu zBo2%Wj1dMmqrot~asLPz+Wl>U)_va3-`?%lwQnbPfD`hzTO9@y{q9)@@3SMqpB1;c0q{ogGGp-ykyiH6!x5+`^b(BPGI$-wKO0#cYJ%LN% zWndkC=k7!3J>_2`dEz3@btg_Z3J$-QyNlWp(lF5%5IZ=le7H@5pujQ2GPrLLcmgqFoc>GJ~1)5FKe*Y}cn`o+2m3tnQ`sO#2Mct8E+1Iyxj zaR$9YS5KgKsEXWYX66nZD_uOR6}{mNzS17$D7anpye#<*-dT#JVyb~|Ou;QjFocv9 zvU_%D?9KG9{&`-#ysc)a%r^M)>N|(Uy~BSL`rq|XRK=Q_`+wAuhW@`%%i6)%z{Snj z&dQbE*3F3C)zP+CL&srr6xn~Z4ik|xBAMiTm}`_P0=ShC7l$!(EzpbtK~AQ6yr3Pb zWAbXGQL(wNCtXiJwJ;f`5(9ZFBgetKx5LuS$>aORKc&03HyE=}z_rrT&K2PFY{A;m zA-9v)7l&C4`YZy$<)FOvprK)X|`V4;;i6dzimgI5j@jCH5{rSkT zT1ON%0!53i$YR#MuPjKe6r2#o+C1M6R;+yF+qBhCnH@}39VYwo-;jk&RprFxWmVPD z_h$5Fc-E&{BX@vLMgkM_10co`xKwbidaysAN|A^$l+ZK`J2s4t1I(0nR;hq-uDzfK zI6QBM{+8b1hR4VBxKBi*9UPoYDB0_Gy`!1VQyYjyk$4e&N*xeGRUZ_@cd+($w11L@ z&_-YnnC$A(ksdifg_qzy%z%j#Hc0^jkFP$$qFic0#imHwidxzyF$-e*CS8>Zb%vS`fM+yT5)ln|9YSOtE~3OVb6PpBKKetYw~*N+HzKPoTXgQs^jYrz zLs=-a@>LRM=No5I=Cg9~_3^px@SB^%HfxDazF7}<%wa7}#hqzE5?ZrTUK(GW?MJg8 z9JC%?b%_Dxq`v7kf}q}y`=w>!zjT*J)zj>ktj9i8 zTznIU4%~RJ5mKgG+{RlgYWQGh3S;@L+ShXHF_F<3QkU|k^NjqfkghEvakzd55(z7< zs}p)$Tj5$b#s8A`LbLKZ7!^It1bg6bbjE)J(^LjBcs(Yc{?fUn#A4~Fy$P)+vy>b| z6uT;!qdXyT!E@Dg*)}L?XqBWZ?P>RnOI>}^FA5I81m%*E+myl%2Z=YfY~7!Kg5KuG z$_3?rp|Bes=o)h+l<;$jRlI{Ea+W3U{;Ah;3}$K=soiXHixk~TPN|^F5K>q}O)=6< z*K<^05o~uyv7DS-a44~-b&`ABE+qNWseJ~n0oi3>g!j)W6Jr0920j~C%rWoUVI9Tw zp73<#tUkt$-Ix_(yl((?ndo!h6Re+;g64_${uA>0#(Ppu8_~1F;{Zu=ghRK!aVv0; z0lj4acBRXGW_D9Z3dsSKRW!t-sJO;$DdRRuXg_T@E0iy&Sc9OrTU#wDh%sIqi zdfx1*E-rIX5WK?=Bb)eh(F{eWZT*E^@pMr{{ym|4ZlrW1JV`|N2)Wg#U@q|KS_R z*1^csz}V@(da~Ht#^taT@Z#|k|s)f-Dkk>%E*iMS46 z6amHsYOe9@>GP`sz9XMVlDhW2i#UV97JOZdmAU=JyXaQE95OahSH|Tp^yJNU+3yR8 zh!24Db|j9DJ(x@he3iHj2)VPMO5+-Qf0x313c}BRL=7Ed??oSy@@SmWfG4YIujqQuyg&(!}xg z(Vm~Lso0HZoObXa1DQtpfGH24J`@S(&U@FYxFe*6T zcsqO4tjuCY^*`=f9#~}{6vQ|OBX69+@cZj0c4+UrP1l#nKEviuu|ua?urmk+f$*Gpk#SCOYomXb=O!6Kv^nLJPinl4g6!W#mC9 zsi&-dNHHVicci7MU@F*E4i!}f-l6X%+xn@zE|n@7r|F$Pd`k6f=B2dGuC!PzmT%ZJ z-p_cz-_wj7Q{P`A&7# z+gQN@r7U=l%t_`d%~Rj^644gmQllTUG=^HN?tZ(@K{lSlbc?wntTCdyl7xQr?(VxX0%LXJHATp0t9!qW>&A%XH}46rd*<)2c*iRq-lngyxvBrgy`j@(>0yE(If z^(SLz9vlN*3X8=@sF?nVQ7{!sSX^PwU%=dy_0WnK=8{0oLM2Rv32MBq6cWSy(@^aS zW6<|D9JmmIIZX(g&&j2;-4IHP6H>WShUf%^T%en;>pjsQ({t!6LZJx?YYP`GY)=0G zSJ1~;AsnJ#5CJ?qJfv`UC1bRo0P7-1qXZWd=b!?z{d|Bk9s~hDIfq6c8Np~%k^x}z zJYZfoK@#zaa9YMrk&L%WyoXO_SB>^)WfzQ((VFE-a?nyot~$Xia=>DdU^vRL=xi~> zbDZ~(OJCZBTqaR28DK3z@Qk*YNAMZ!89LjQItprh_F*X*O!!863N@RXl$^j)py_3D z&J$yVxRtxRt-EF>J5y^Vq;^g|#$?CnA*o|!aX-CesEJFZ%xZwA)alkJVuJFvtV@x3CAB<84%CC5E zrMrt&MIA{w^pFa_PmY2ugs%#Eb;ec5{p0QT_cxFMtLxJ1{rRX}o<3Uy?&^bF zS=b_PW|DHsJ*ww9i^^f4Cp0zQOX|+9tlX86aElM}P|BDM&8aKu=Ic47PpKKG`93i_ zzA~Z$4ffqdV5`ZKezCCGJ!ss@tn$>>Dh>;83+b)uTbwYE2~j5I*)DU?0%L_0+s*ep z?dz7SjrZ7t4aTOHei(X9d%5z6fZDL?H8->cBc0cOsIpk{H{CQ#2`@N$|1n}MpCS^Q zrn{MB6ZJl<+vtMkK=eC`HR?)E3w3>!)J5HLOYh-kPmQWSrKV8$TUzfHE;NOUjn-eT z0%YPF9XYz@iUy}k+nRa3^`wxs{8w~dBe|Q6TB#Og=*eO`gH);ZwA|8R>_N9QhFvB# z+V-9F^z#~#c_{mmj%A#Rt%A7Go+jC$in?W_EEU@tIJ7fC*kuC=`Kp`lcCZ;=ae{?8 zuynX`ho)Jzk+!pEqtqc`>%#t>Zkw+p @)wAQO`7^A}Dy}~JgKH-uricek%=|#H1 zc3SMAw3^JpCO>M85?q0I)+nyO5thX{{*C4Lt&t?%GaK1->+rJT-cWMTH@E%+t0L|REC zz@4Q7uN!T~y9ln$NwDdM)1*KuHBB*XLuz)r}LhKx>dA%xl?Ti#5}~@-9+$*MliAs=hXY5%`^{KlJRaR9lSvfv z3wS&~hm*6+(pIT7*D!-6>UKL8AF{Q981uJ^hLNzY3?>R$l|$QKZixa>wieLwt3Z$K zrB=>a2}hI{Rp)4PW>?y>uu;vy-EvetvKPk_=B&cj=FUuBGcPZfY)cpYp(o5x)R0eJ>a#iw$+)u~ul;yjlltnq;Tbq4g;fNGdmf1fU{!_cUwMSV|!T^(w*BCq= z<$n&;G;OFFC2=*$ee;jTyuot z)&X3KcqQa`@}hCGB%fMJDR;-f6|Qud+@{cvD1I&EKN@?30Ima3@ZUEOXEGP@0xpPv zfRMIT{m(1=-v_p74;6d3?j`c`zJkCHDyY>A6;7<-4;g<$A08xzwEy5z%$Dd2gj1*H zXzzTDekG%b5$EJ{IV*2Gi*7wLhg9_3E90%VR51^M`;|K$<2m3Xm6iTY)HL*HaF~D} zzR(bHj@p{_#?t|**Y(O-a%X3zZ{v^)8Jb3kV^rnUDHukTQu*o@wi%n0RWB|xFSvhg zu%9|>UV+14S)*ueQna2hdvzi&6BoX))&eZ}FM5O(T(ay{14G(D%spTDn*DTp2 zGKJvecAFMw;k2_GxpaoX5ab|bjDQq<=3@E@#y5HcZ!&{U2S%OF>FW|pFi zC!{L+@^Qj8&5;``FIRzqV)MG#FYWW;!nLvOu&PVT*)3I9mcG{Bf0tZ;SC$-O)rY$; z!?!zY>)Ph``C_+gy6Jr6=)!)*+8v&yyi_40#0HAwmFGm8zF3>Z1fkt6@SL)G7P%TW zg-8C6*bCejpXGAM3o_z~L1mqF(g;{}hLlFKFzJ3TOC8;s_F)Y%i@&K?BbGkSoUt9V zR;DHf7Y7%HCR`0fz}+9{(UhC1mOk*d#bxOvwWke|?U=ICUkHYe7DLeD&F2M#Umjdo zaP++L6!f@iZ%E%Umb~9>rLvyU2A**WI5Io(zkJfUf#Qkt+N+1n-(!qAPS={OjEb~k zf99rE&h3HB64_G3-Q2M^cj&hDK1Uk`=U2s_J?e77k!YP;aYLzb73t>pE$x+gk=ahS zmiux<=L$X!CJyUv=HC8vOTFfN&kgnSo88^%Zz@mnuNmltR4*W7G+Q)qEt(uKr@5qR zWVCLZuG@+v_3Bk9qQUq6=X~F$TIM6auHqcq7^0*!Qo6b_v!zD$)*P%Fj7u1K70??| zQNv%Hd3p=_zJ-mnWF!yEF=xH@E?OK$vW4 z0Ac=T2*DohPRj3 zD7c^UqiIw_+meK>0j*(6X6LSGO@)Gq&t5OQYd7=8iX&^wiAFco?WgOH!FfCHX~TBU zR(~od)qDc=BdM0mcA(={4*yy&SC^yRy`K8$0#0hp68t{q4MJdatN|J+U}&7^^ptzV zxAaHhIIM=>E|3hWBkZkAwMg~bZhFl>%kX`roBEIotj$kcq0@6Zv2h+c%el~>b~fc! zGiind6`oaB_$MS(xm-RU zg!f6(O2Xd3-OhC0@w z&SqYa!Hg`G`P^k7XRUKWnfz?ZAsU~<4{;Ffv*9a zfP>tTR>llFf#{&*EhpGClSgQSC_-eTZna$wT58rv{sdFB7SLB%1X%3YYHEK}iCu7q zQ?5>Av-`Lo?$jya!tZ$kg@@OFvfns3cDID!ooa=!)~ROoG$;27FGuV4L-c-GYHXQf z({Hd&{|aIbPRjSy?c;-zm$eN-VW{3{8lcs}3MthpF$MdL3?qL^AT13=yN|K0RYq0X zYQ&b>G+(3YXZ9LQyv;-m*(~_7aGCH7v$eT?y(i-WfJqFfC7yaamFt#ie-3feA$oOf9S4M@h zBNy42tKYbUdNDz7tDMNubY8*8YPE$rT{gMqMy&F^no-=-Hj2WELbY=atI?>V z#NSV>;EV|pLcFSZ0qOlaj2}l9=~tRg`d<)rkVn06X9()AeigvVunY`jpc&WB!{MOD zX0-aN`Z-5&ZjKrvJZ{_cCy93KkL&5D4S{Bvk&hE&mA>6C+y#M>pgDy!Zbz z1fXf>K+<*}(3fXSc=)4F^lTu@W?xa!nt}p6XAQkw>+s45O&)#LpHd)?B$A_)Bj%( zr*))A6$ieBZKU4l8atha^aU4-rae$~L^dqSSS|n;77#0MSVlv+2A}M%s0G@Fj?PSk zbVoI^5U7}NQJ5T5foepvwLzf5B2|d;)?r?t5eEeZtE_h;XyM0gI0I5@ajnaR72mdL zH^{ptn&nFiQh;ZmmusNQKtH2T#_*k*mn>>CuveGDIBwJeqMoW;Y)ghjx*5dY?;Jcn zU^UpqHVBmx-+wV>moab#+Fm~sp?3U|yqu5X9xq5^2o2r@rb`cgcWvv`fhk6DxBu`G z{;tq=XY}&(ef9D3U<~~D>d4>c@8jje)u-S8#gnf03oD?n--ibb-6jt5riwjS2M4K4 z;=Hi_#|c5O6ogR6n3_2;$dtNHGcMs$f>pX4SDH1n-c%Mw_WlgCm@>pd)>q)iLO3wMnmChMr-MyS%MU$xZ8M^ zJMkqT^oxM_Y|-J6_V_p$i`6*9#S@PxnX*AbP^lOy-gFo7=tZD{hx&_R1`CZqdD`Iv{&p_+fDzV(b5F+yZ@+&6O%`PPbvM(6ywTj{^3>|M_~? zVrow}Qu*#VwAINay7_V@eOK<4$o)#5%pGp^n0j

zZ2>8Loutw`d%}QuG^V?QKJi zEA9_vO_A$a1;Y<~8S{I+jr?9bBw00VbHIkA&jeDIRvffeI2Gpj0^TmUo#SRM+I>UO zA^?o(nxgdMvPB8S7CclL&5j)btdP$_+CmENQYA@#i2QacWez%-xnzB$#@nLFjTc{#SrL_6JPG z0%dj-9wlHH;_1HGJRVYvEQgXTNO47JpgCFc6)~|jO{O43hrTB%rn^X?J0Pjlk274O z?!l&}ezCRf4B+0sc;n^nm3`aax`lvCEurQL*re{iYvM8=?VH{+{8Esa%aKmQk&ReFtw13D-c1Pvsj+IzQA{e5M`A6-+9yqP*f7RlgiG` zwL7+SaC|u~EG+X4UcKok6uF;Ch`Zxop8H9ea0U8O)P%cAN;}!{OjCATcV!>4GMgSX z85$au;lRne1e z{I7si`yJv#WlCMcO0+V|w+m&nmevmSGI&^R)$VIB;du-6bW)fr!%$aH2`c<~kNT9y zZ+01uaij2=JhOz2Q>>bnnkCd;lLh13YQToqW#-VujA1nR=HVZmSrjxj61J|6eM@2c ze88y}xNNJ5#P!dHWl71n*gsOsjpK}E*4z04V!7TL;qbZg;=l-n62`R#TjR`o8dvz~ z*i_=}fu(RJ*IFJz4iRarNme@nxVW2&Cv3eMX4ViT=;}0jZVRm5 zY+Wx?@s4lV{m)ll5?5ENPd4DQk^-19=$-6UFd?m9J~fbJQI&70B{(@s2cI7$*9|Xn z+p$I!BzFJsM(Xds3m0+|vAhD#B*b<2{-8ReS-7KauoX+NLgd>m02?b90I^4jQ- zJ0@)npaIc2F%vSQQIUX=Bi!hm6X6&t#bvCk_y-l{?85iU zWTm2^Q~Oh29MBL30T`0@Kae0MPYzvQR>P1C;f3wj&v{9od`f?F8lVV9D*q6ehFt-J zcx5S>r_zq@ZmVlF0@zY$fv_g1~n3J_c2&&iG z7mVjb^k;8*&LnarieHxL5X{gXs_BhEq)iFUv04Z^O>YwT;x(j>vwv&(sl)x20Paf8d!Xi+XmQ`^&xA3+ipD7j$d zb9C<}_rAmdhu1zt)?6;~ZT=q1*mE~=-JrT13FhVu9_f`Gob?IGj>ls4rZd$VP3d_v zZTQ6$V(Ora?5dfpZy8vzDWfv?{14w#;zU*NvGJl6MMC1-C%2Z~JA&^nt9&4($zelMzT zdjMRPwzyyEWxZE$0dIDmlz-2~mCOn@RdnvZgSd!_#RqMp;H_A3KN6fnyb|8iK;6CU zI0Zg@CwM_S&ZT&#J%7=>^7oh_UmWoa5x$xKCv+X-4Y>0ASLzeR`cI)N^uN&6*3w+x z%*NE?U)Z{=`E9?|?)|IlKf;VTGQlA}gUW4iFUxKsne~*$6;85N2MvTz%bT>JP!d;i z*(vz*dO-NsY|L_L7wp=)AXfI@%1@mYiaFF&#h#8#4ptAT^9y^FsSqN}&czQIpv3Iw zlvBW(DNMnzuA>erjx3_K%9Gvy^DIaWR=*(-Z6dJ64?fTjK(JMXCD{s4LAMAtP}>ol z0Ri%tYH02wDXFRjF=rChbm(%j!6tHG419)#Xlad-GDxVl0H(sf0yfDReALv`meK3i ztzBhs8XA{Irw`h*(9fBVl|NN41*<<4azBQnSio3(Ri9!pYuE*-rtVW}LxBve7O*r3 z5t|vHPIRqqJ88=FS8Ug1jv9bqva3Cr+P1~4NR7>`f{_MFLd4e@xSnM7>PHb}9oTYcc={O5bL>Qp2M} z1sAMSg>+M1u+&qN(&jiDmD?{W5|#lGY|_!D>%3!~`%+9ZnD8nXcb%LsT54 zyceR1{Sg?Y(b6;WF8SsK{NNWh%-qYMB4qfR)aK1--qJAKm?~0HwExHA8DpLms1XXN z5K3ZG*D>%#2Nm?v0tbZrt`4mn{VaNFsd{OuP5d~2WCB0^7MkRKi7=+%_)n`v5Wx@* znr3Y-WwFkFd8iW_(~{U0q>sWB=v-OugXr1ZU?9EY!w7^`jP^h0RZZ$03xuf^=QhB_XwzXu+Fr*73 zrG`f@b7n;kqF=%3Cxg*9{zmVW(eKLW>$sj?4m-CNWWY7Ga0-yZ-Et%IhU7Le>8y&T zgwNKM#CVpFr*EyZxtWK&6GLZ!K|S$puE*!{7Bj4ad6e8^2E)yDx!W7OPN>QBz(0hc$T&G zm}xulxCt6x?m#?mns}>hoQ?D_zU%I85kIAOmoF@n(&@~|tG9GJ>oF^@HQ2|jwC$}1 zol0>bW-?aP#$Nyws9-A1IP}@W*#qC}FR*`&c^co_eR>-!%ah5EL&QVEK!)IuL%(d&4 z&mti~iu^rFPovr*38^h2v(qKC{+q=TjsM+Xkc8cq$}Xg1tO;qxXqjq_Ph<0B;*x9VPKeV*@*zXA~XP*Pb;qH!F|+>&wUwYq)J!Lv3}bCn8}m1(D97a^$6~$P)5o3v_ep zoM+rA2@1Dji*TDb)PtU6C0ffEcz-Ni}BvT)%#q+M)?pP1zu_*}~Y`jG&0|8~b| z9*sx+V015oIz8@h>DiY%QurNVZ!-I*Gxc$j_Uv$An@~G8k?lWg+9G)CHBeXd&ZtL* ztt>3}`%+Mk?k5}e=B6!P=gqPcw_b<=KcqY423nn~-E4I&h zE5El&YYXSyCG5=G&O!UhZ9s2hP0gsVWq3w8_|9$X>BEFWx9gvL{3QAZ;$$4iSeh+> zD{|kKl4neW>#kEiu$zwd)=HHti!Yvk!v6c(#gWv!D+SEo+}~As@}$z%b{I;ahunWHm>W?AVvjjO9&NSQX?6 z%yrr~1j5 zkCJcBVV2I1Z_kavWH49(hn%j62ywk0#Lh(-G$^ z{Ylih>ddb5!g%4OfXRy=&ce>&m8hwWDx0CeIo7>r)mhdbVjIY)ok{Rpi_LdL|Lcx* z+b~Nz&OBo}Q82;9cjLmf#3G;PLy#EFs45UxE@d1771plAL=q!4NyuwyMt6!LgJK)xS~7K{U1ST^K=qcmguJiDLG9` zE~;S^UuQ}?xGSkBS!e=?ZijYXI+5_n{Hy%@kvg_3+{LR{+|aUoH1Fiu5|l_xt}riFpB=yh^^iR$Sf$;#~>*Y z@58bUUvArruGvm+?~~6fH$w|<5$BRf5qNwV5AUT|!UNjU+UV{c+t2E;@cO>qa$vDw z!Ly0a=(K@6G6;FrL2`mmSqc8Rj(?84F4>{`IY?x7vYO)JNX72YnmzDOggVnXYf`n+PMoc`!RcCwZuhWqan#%>m##9B-*JAQ$EG->0DeIcHP%IWS?K@BxACj zuvV>3?x*m`;Lh6-_H;~7?NS!pc1Ht-v$iFe6nUpF4=L4pAQ_#ysJPl2&=k0 zk31Y4JluhjN%t~Oc;|En&{O|Gv(X0dE3NVyw7S+f(k?kUf^qmbt7x)6Cr3zZjo}QN zf)sGesR3R{SSba^eR_;KmF9xG2DsL!Y*4`Ojw~U1SR@jL$)C*tov`l! zOdt$@yqHR}v9Y79A{eG7rJR<-_^IeOo? zKi`L!Ln!SG@H&A0l_-Pss1PE=xVYO6yMP9iv;v)nnHo_rP-=Bnt%nUs@;a$SI)WoB zS{6Jqur<^0aD@ogh8czA!Lp>_;L@aP25hKtXjf8H!0AHSlsf7h5va;<1We4KOHR)F zTnsH$8H?L+vzZ5CH68eI`ObP6r(IX!L49xZeHNM@bD=u5A{~$Vr7!txHo9+fnKV#& z3t-n?sLb%88gD0jj8F3bgMWf7XIkuS@HX&h=0weWh;N}Y!m6=9Tp}#?nGvd$sQmWV z>SHOcExwM&M!_Y@HiUKg9(|(WSH6goMgd8=N zM~tbYW>N#e^~}cWj38B+ljkY2oUrDYl$(J3zMS6gE=E>n-p{t49}ka4XnCTG0zl(* zo6_HIcP1t#P{0Cq<>A7F&+f~UlaW#WjeNd-eVPE|9aOGf``SA`1ZK7oJNt>&DIv-aq=5CU&tL+IcLm#|@KTw~xk^C|oi-bO za^I}@T&rEtl;}FZWw?chgLCQ)L1nKj3PVv9i%K3t_V^SgHn`oSyt~OUlI$cClMCO# zDG}Q|JW&x&seCMBdFVY*C8Yc=M%N;-)yWQS?gE%E&Nf!E}Gq%pfX`uk`+}>4gKhDsXU`M>K zwmppa5>e*hj=z779#rG3=C}j_d$C{jCzBNQ-}Kx%TRbei+2%fSfwh7;CG&yv6b z;=Vxy`z(;A86gMOAgihLP9Nwapez>lkdW9eIDfJNI;k=aNFF88iV5P$A?@h?)-vAMFyW9+mcod;Vja#N z!yTH@M+V!&ZL!;P9qCM=>~=TiyUmQB0xMD9g0yYn%6w+KR*2J7+7n3`f>00Je<3T| z)Y8rvEZvkHE}#&Z%pcaSda?w*^L*{*BKk9FVkxfTz!9dCU_xjWFc!G(Vlu)7?C2}0 zTY!wbY?XwQZz4>OwQe8?=5Ko`Q`DK(GW|%R5h3Ta_p%@NW=gkk;e?up6E&xf_U<$Z z5a3;_4ayBOalz%u&}snFWrkUkV^Vu$GOr|gb*tb;)TA;)&4*a{RwCKov?3G*mcz6* zPDhl_zS-LwyR84f(yom2jl!1(&a+v&>|6rD5^MfwT0JF3F5V|$c)~5*yr0FRtCzQ( zDKgh@`TVi-E_;*YZ7h;U8h93bgTVlyPC|j-p4)&@o%8i$M1mrIYhqYpW<5lFZXkbjF?8r!#pE;84PGv=_e~HBnAVR;N~lC;hzWR(g~yu&16} z5m&Jo7aJg3!msbs@r~fI{U%XsIV7+K{@plR`NaXQc#MGwM>$I%UF(Nx-e&8MF@%8n zpd29hww2^<1&KWzp%6>>`ysL{km5LwVm=DSX&0qc+Xp+KpuT>>Iu@C)K^Ya~>IT|D z6AyFVNfaO7hfyb`CIJ3ZUvG&n$eeByX*SFRg}VG=ecUA6W;Y<(@n;qnGBR*I^o9e$ zLO48TCt0p`GM-|tXqmSn<4hIq1UHqtM z{~WKpz4JCA(2-#E`X58rufjH9|5oBjCat3klgDXsn{|wM5T~)b@5uU|N~&CLwI-KP z1J6)KRhP{+FOYi@^2{EL7@t#LOk)lnJ|3^wd%jV`FEM)k#_SNB%&$z1B+?XnCki+1 zbT3zC|I^bP#Lu3a&-tg~?2b$qxw%bIF^eOIh;gxj$TfO6d9{k9_eN%gLG#((39!v* zhJHi~W4y+^YhU0%@u^?x?G|J^hr;V!dXzF_0M-auZ z0<5qc{9RhDPd6bN1YYOBrei!q@Iv2id|zN2kqV4YKAU-ck)#VPlH#}%g7gAxx8>Aq|Fh$)@al{=LH#|l)*D@McMw;N1v=g?oCeW{ z{IJ;&A?&#B7OsqyrUs(VqQnTe0le|X3nJbpxMrt)tb(~rpbT=H zye==bUl?Y3vOPsmdcV@~WbJGDC2#1(5o`Kwrej;PNovpHaLco@jp56OHIr-*RF(75R zPPp!uo5gk*$wbpeqSvqs9$9`O>jPF4?rVKAc}X5!*Os1B`oKRtbrhj2);@b|SM_<30oJhZ|85d>7tEivPEy#ex z!5DrMTxeo-bKcS26GvjzbT$iy7yb9?bOtRg1i-iZ+ZKJs09!SRzT?nTEN=SZ^tTRpC^z=J+_C15_hMG|MZsh_ACKr9rCy7+^#@t)S3+AN^v^K~H|zWLD* zt=+p)w}Klm+`(r}Max0db_D*e-<`4kbLxlkB^~O!bT;vZeUeKXi`7CukAnB_CCP7C zZ{bM@^jH@9l;l{}FuA0DKs?m-K1JHXYFshxomt;_s4oK5>4uw)-EaHGHyu{MlkCm| zxFDvs^Bl_&u#OPqt6?oRlvwO~9^IbF6Q!|b86`^m%JeBAT_mP<@zM#2qmoM_wl1&2 zV*7|zH2E%pb>vG?n?SAo-GP7%650WRLha_et`O3EFfyk)+f&FTdA|bpm`}n%dzZa! zMjE@4HGI+EGnhSL0-oqa38j~}a4Fa{?1F-VR!TQ^@yx^1?&G10BpEgA^v*w+(iw_C z-*rP2%HA-*g%0swjXMK|<0ijtBCww?;e{{{Y@Gg;^I3u+tm|bUXoI0b(}xpfy=WAR zz%HaISuhjXMqlx&Nwn&c$yJN)=WeX}ce)TM1~zE9g5e*&mj$ws8aeYSZqBc^FvGW{ zm!>_Lt469H$A5z?)m4;G_!h=>A>5!OlUSG!%&L>D>}r1JWojk@o5jVH6NYZ~36t(R z3WN)M&1#om{lUapu7L_;a@0+!y~8@O{Na*G0!3~2GqtI2GBvNB0#iK?fO=vVZj#2? zplDbsI%UMlTZT>xN||`#6CtaZ9DVNjwWIQ2#K8Gl%>zl$rOgFO=%%-a8k>f2sS-yX zj>E-|`X7+xxkkTn#ugxF9#Y`2S`0-AXxS5-lL2khYk&Uo%k_h$tovBtsN%S~Gc5ws zUX4*9(lrF*<{GZ+4hvO|*jv!VqB61cP##A*C)HBASXFtsOe!VUiCM3`!nu0z7jgy+ zs<=9RUP%8ji|27m)#~Tc7)w! zHq4u&ja97JGt5mn;Teqn$qV*R90Im-VmbOH0s7+S63l4dFpIYGpv-ws6#Mm^}%2orHDY|HdZGa z&8-X4s2omR@l0W8e+xx|r*pE-*8;jpy_V;OK%R!1`5Q5XlO=TNItwQ)+@Ucr9-aTn z-k2`O3~E82kErg82A|$9Pf6k#)dNdi!FKyYimREO8xutx6vDoxYv`AF@A<^gpz5)k z@Z9mN5V&y#$<_kWaQTdl_9t1TvEJ>=;pZqlAGj8kPPFWT;PPO5bROKDC|n|-B=(6G zt8i(fUqI}ZtD+e&a5hOrL3AZqZhChAH)TsVg(i_)rrYp}Xu)P;sasF|!2mv4#Ijff z^%Nh+PDWMR^6H{b)!|jy-hn1$>I{)h2k&j=(3JcG0ffq$ZV4VMG%+H9fz(uVbnAf9g);p1+ zng3ZsO_`${8&xRH4RLB;T04d750W~S4piT*CbdR5b&ENnia@iPG{_70B5`9Y8`Y_r zN+P{k_P5Y0uIDk8BYG|XcWl3LmLNfRk$)h?yJ!>>+-kN==W4(2Atu<2gcUcIEe(X_|^B+HcFE z`2jmOkTW0aDc+BU={D*_@B<+WkCy~HwsmbF+zuf!kR7qZQL<0yjv^krgfT~$Lp0)@ zV+<<~eWJ9LpW{Gpq2`u?;qY6TxvbQ#VrQR!@nTRzh$ZD5y<VrYTP>$l z*I|3Q%G4{_YF;kXSZIf)yEpw)-L|c7bTD)cJW-aEKhz@HU(L~E4qkr&55XF?WJfA@ z4*T1Zc>`vQW?Brdp!@;Vq+z0qJZuOJU%5IzH_kmOEK)H; zZj{QFT{c=Jt8h5{V2f6z6*1_{IOe-`udK~pdix=OmQ#v9flH8tKKLoAYqxol%eSQ& z++e70MGkNFtpce1OisRF=J1e)Vq*z*AS4PzIJjxffGxuQEoZ(kLFFPQEK;g^RQOt} z3~oqfE?!&&eDDvx))&4?Id(zxQxT;Wdxf|$PGW$kc{K~xE<*%C!B`E;6hLr^HgnBJ zwud+&W{26}${KWIcU^^Q? z7Y}~yM5wme{nVqm`gZx}c9^_9C-Mb>4K+B~kYpZ1T{&&G{JL6299~$LpNr)jAWBQH zt1Q&HIIr%B(&buvVQDDsA$Ilr+w(e@tk`YlF{$drbPg#8CN<$~T9kZ5LE}beplOtq zt^!xE>q*vB&Q*m zxlp7WV1ND^4FJ<}+0EUKPY{;p3rgl^I;(guo;zb-)91_2EiX^cpOfeR;^qC_8kxC~ zeXha~i8%FJHmp_u<``;&pK~N2b>D`4FKwZKZW+)+L>^_zRgz{7enOr3$G_gr!kdMQ3cHp)Xj`7@jmTyW=!q zD$2Ll%8JWv@UE4TsiUkr+S-Ip(smta9{-fDQTTq0dZmFBFblPp(ahQD`$&bLQt11e zSSh}Ynnu>@7lz9Ki(mKawW5>*+<$dvZv~YNxLQBGoy_B!6-F;QsSTxf{F9czRfUB#wyPJ=@hxZr36&kRJ)%$(_1#N~) z`^HsA^>6K#J`47a8@IvP9g2{GL$n26ea!7HWHfnXYGOU*WXpLbH_gVPPP7Hq_PlkU z9op{;4xLwPxXdVtrAE{yan!eD22bQH)4gfE?lMoK&~cy`^#1SWcACe!^ku!5Y0bEm*Sn}T_L`DVHm#W80ThRUTwWyg zT8sMppjYdTwpZ(9Ra+g=$<68=X$tph!Hzr-$@Xw36&oFHFOvMWB=)o6+BIkeiUOE= z?T5C#^@l;TM(at7X)`Qahj$k&^~s(cs1{}#vUe!Rd|KxXwG*^80`@hy9@b&YG8ojuc(kl?9%$#R*wKXMtcB>7~{mev#`4t^U8 zA5Q7rxxDW+~d{J=4}3 zy6rtVp|!0(K>(e_nHBz*92TEv*Ub2my~J85V&~oGgv5+lt(bc#bFWk&S6liA4wsl~ zxA1~>YF4%t)2SSNF|#H*C*)8m#9q(5LR6~;^x^pPY89r7P=sINso8A=rr?V{F|=ca z9JSqwK9AikH#?y>0;#<1Okj`PUJtuJgt?zyLG_Ws>J_NE1XME|DE4% zGc$+R>to*@z9xw4djB|8%cBvTMfWg08`Iw)at^RR^0(b0b$G))q|KEag0Hc2V)2K~ zx%Hv0+^tYO*n>d!6ceeEr5zO-4?k@-Y5E+0-ul2s4 z{k%by5a4*{Gk%1Om!~a$6zCgQfjhwk?yvX!U<9|}w`cPD+qAE?{_}Bf^OyMQ?l976 zgci51?R~(T70J>hmdNgMe`U?#Oa6QS;0nl7!O}LrfZ`R~O=Z9=HltSaO{iJWaTNaD z9|E|o>pjFomtU_DbH5`aPh1%eZhwGaQI+KtDs$%Q`sCxb$77p-+vAZOY_$8sriG{j zXeWEn033_KU1)Hu(bP7EklC7Md3~KO)Zs22apECUK{#9|c7d^5ufV3u%PbM@*)~biPdaWE) zCwjvk{iqv+MT%*-MZ#7(O3}re1N;J2V+MhAX)Ctfi#>jmWJG-YuO`m$LQ2tkpR+hOz1~(n z!WJW3FDj0|mN?`1M7QCadgfQVwwB9I45oVa9R80?-HbLRYZdIp*eL`^DODom-|C+q6>!P#+v$a1p#=Z~A5)m};sd6OUE z^ZI7gnQLa#A`IP+I2$@<+7U~n3s=yC5WwL;LBB84jqze(=LTzD7_iZ%nm{wTOf={f zvK-!f7}O?><X@_|7ciIN5N|0^i`9y9{R00OW)7#74{?O> z5woEW01MNNb!$9sRgD|Om~)N@?!c7&b{y6_GR1zS&qe0q0jn=jL6koQXiA;z7o8Tc znpqOwFD=X#E!(1Byi84=@-TY6P5Kx=noH@dE%klT-MB11pPTa!7CtfBcSZ$>RzA(g zwk(3-vMIXbVMM`~db)A}XS(7UEDq_g>>f5|=wPB*nOx)@F?^^wmbZJ5VsXxv_Cz$M zisR-i8iOY~8aC|)-Mm?Nnqe0$r6ez8R)|C$ZG9(pHuVUcRs(?Be6uR$+QuiDe? zEq3x59~$sp{?X@tH*q(Cj*inh#(}k&`HhA+Ec$gXHrV~T%RW}AwJQ0LUp^#g+CfTNn(D0B`B4V$ zoaYY(*Hd|lm^3(cf>+zd-n0-eLLn}$lc9rysS};Eg-z0e-4+LWNc~WGMm}2*NJJ^YTYc+zQQZ;C zV5i&KpEih~{(@5COzo)SBo!Ru`!0G4@*S|c+hJxbzbxjS-%{(!rIvf_Rb$?mm>o_{ zd!RbH`k(l@(d8(H%$Uv#5wsHCR0~&0I749!Ek&DiTc^OjaZgK@x+LiwQ9ys(hbQ=& z@LbD<2;62!3tmE-5^+-0z&z+bfJ(&-v8qbvK@F*fOS(^1Ney_n`n1zr5S4@q7+t}Y zeyf5Za2$Nu2-o(OojZ2O3j<`ME0tUjzFZSxN0xkeI;814&%h=Hg{^i^R=2k`h zjtGonR7R(rirfKGIRQMSLu*U`NnLiLvJ|q(owP2o(sYP@b;`o$Nv{Ma#J~_clZ*4E zFrH8t2w-4pnpGexF{Z?yuCI(-lV|Eb%|#;z0J{Cvq{>Bw!`ss3hraAm=hB`Q-VPoe z=jJ%&{N<&+q+F{}fXlTw}!cnpUoOjvXLXY1!Yeu??x#iG~c4|3KGsZ$3>E0CUX za^4QMLf?-*pef$J5nc{pN+F7>JqFX3uIElXSnl@K2%oPJKu z7)?O0rjuzZP5ul5gK?3$k>Ff&;u$N!W2o$Z69bMNg-eufNb+^#x zzw)rnPBD=^*5Q1f|2($NhR4Ty$0?7VYl7+g@dy4N!BA8)Sk?g$0KoGPFYwGDbJGyKI+ZEJz|@ z7!HI1kkhEP;NLfh4XLy&A?MVln97T^uw%i3^$iw)u0L(689mdzS;cDp#65U>Nlz|To9RfH{$UJUIasys+C zQDBCNSg;CGN1am_X`eZ`7m<*{?17{&R05b_EtJe3OJ^`;y}nP&P*;!!vYaI~ZYdey zBPCfHz$EcrMn!2x8R%WLT81qy()8vo9#oG~2E$O&y$LiY%+Wu|P{VIsT2=l2_4C6w z8n%%UZKS6q!0G?@aU(kWBzhzHZLz?|fHV;a=uRHv%ey^vi*Yo#r1ZlTB*xcSOukT;m%nZ`55A+(le|TSSwS3*UF72K4blt^LS(N9Yk|SAw2Fhp0e1oY@@IizSR@Do;Yguv?=}>6_cjFy_g0Yau;=?@#=6 z(A7DEkaN*u$EZlMhWIUs;&E~mPi-XN3ZM~<721i!QV?eF!>sa7{KE2ZakpA@J)LVr zX{e}%nP>|VjyYwT)q~!03F>y3Im;ebrDH;~LA%T2l45RR$gFP!k)8?sBMyug6VpBm zd#38pfNJqiP=yu)bSd~MOEa)dxk;Nlj;CAm^cclzR*;h);Y7`poZDR;*eda{@t&WutpJJsl zGlRy%IVyb&8D{OFKt>Jd0%s+FFk5j@NmO)gE>xlaNHL?3#pXc;KOeVPH5g|Z>BL9$ zX--F525>YfyjpZkx|li{!zZ)CX>;bO!mgTl%h-8GpoDWWq-aIx)n5wVbPFhbUA@i3gvD0` zD|Q;RfM=%)RKfC2W`+E<^krl*%|Q>bxTH3MO*pkjaN>fnu>#7hFAGf0V8FEj_4IT* zIguGJOWT5xA~OT*fvyD9wAP@7#58pyamr$&bzoLxuR$`M5NY37QVY~4OZFNo(4185 znQj<@qZ-0_CL%*Xeh{A-z)+0yF#{v8n{7ndc7WGuT1Dl~#U{!~uJlAOpNwcj+zx;z zmg`lBAs*f~bMWnFYHnN~gSMDG=J}z%MWxj+A6ZSun78=V1&+|*k%!V-u(L09dOpp5 zcv*PWC)zV083P2DP0Jb=W3=d^<7x*2 zWVF(WPXMjg-)8SZ5Oe_25Ndg4($7y6;yaQXR{?_6bnhaIT9&%7WkLqHg~<)Ut({nX zZOZqD%BE1;XYQGbT{>A63RfX;rFipEfNFXf{3%U^b-r5yuOX#m{me4V76Ra2%yrWt z!I8u6qo~wLCMvsL_GNwKTFx>)taIZ#I{aDnt;|_7&`o!YTdb|>-g|Z!2j?T09mSN4 zB`+G9jRPzO9Wu+Ui{HeLlDwjHY;DgxMX?ni1Ep~%25SvH;>hLo zjVeKEa=Fch(jK){TCS2AylI&sxmWuTd@}<9leH=)!P73vXi2tb)CIjAGo;ff*V_dz z44;(OR>xvB_f&*3k+q3l3|Q-+N?UC%Iy5#pTA~@yVz7iPpd|Am)QWf*d^;tTV?p~h zB{Ab}7KFflCLfB?e$C`Q^T`)8V8e=WZh$wuCM097l_WZYC*r4L_z=^R?5d>vSLRmI;E+1g+y@nQVRc$A{9jjX{HQHdr&{=!4)@yV$$D1!`*dM|ih_DL9XI~f?@ zEj#d{B29#*mOYOmBX{BN)^*Axw}zdk$wGVLk=gC@YWQwvk-4Y?eH;;^oqP{HJz%Ks zz+ju(5ipk(KbM;*tmc=Rf}i%Z5v@gE($>O6R3hV;U|l0z$*s4F?M+g^nIgbx-7uI8 z1%5D1yFo^{x=(af56>JGV8gvF%-N@iV9N}z zJqwaV5{wzznByPw^J%PNNO{*3&yT(H!_o0U!SDC($n^WRZihY{-Iz;6=DL|%*P=Y` z&8M)6NnXT=zG;hqtZrSX*YB$N=&gsfN=?BE(%fCq8y@Q{TUOUrD76r5BweA_F_rrHx(Fna0s%bu=vj zfxl{>O938GQ`>QzE}#FB!oN=Hv;xA<2>dDGmPFUfK&)ffffq}6y^1nzmRg{r(4w32 z#ph9@zH|8S+l3SR^TO59;rDIy5Kd7mTFHvrk*tGVd0*Y)zg zbJS74J<}p6#PA81V4vr!eaKB$w@t{-g86+kGh@HH*T=!Vfd2(!S!<9s;K3KB2`5ip zteBYAJ)5OeL{8r0V_xXk>Ug>4ZEaQG1VdwxZSpXgC&?^I-=!6Uo-~uR zAtl|?ruSvsri3#r{<~=tOr$DwWhM#blF}>f2txjnJMTAs^OqM5n z6zC4ava(xY1*Qmg6c}(@4K__i5Ac$UX}NW}q_*k&!v4v2EM7 zZQHhO+qRRA^T)Q6j&ri_8T;{`_o|0_sj=o-bIxxbj87F?)qRyLAGp zBMR;{4hy!oAVf7P-2n)g$R-u~!R7fewR;AREDLV_TlHWclmV*eN-a-u`^-ubt@VWt zpF$Z^XgpC+GZx5M)ah{W1zY`@5|DgYBISFOv^Ta)O(BCA;X-`eYT01dGco|^Bv{>a z=X|9R3+QNQ-ko2x8CsZ@Qbfb1_uaPV#1(A$W7Z)pg97|}Xe+k*V)%4f1_q#GzNfhtd%1Z9x8BJkh zb|97X5nCXP{lh*=-|#k>_Hl_aTcW8^ z&I}TzRK^tFo@Xh_#%L>3Y|(+HeueVEv^u)WD=2NhLeddKn7z?%N+|J^!|v(u^gxt~ z7|iC!!rAR`oeJtnY1jn=vlk<9nhtqWF0IjJG#N58eNi#^MoJpHBuV?GyvSLCLv;o> zuwyJQ(*{3KR&fzOaBMVe8M1-y(2`|SQ8EU*W0GrS?ZJ|QpakeBvgM$F$mp8$b|a#d z1CPNTD$_QUWvByE$_L?i$vF$e2#y;!Dl~Sy$yYB#b;|5owqRTws~{M!LjfnCR8ae^ zjq(LV|5PCPikJo-oB~{28vT>c7&pXBkWNc>PV3;F0lnq3pJB0Hl> z$x#-ZSp>KvwUf%dijq=O8+T%Jad3q>vexGKSH>_F>*T^L^4gpia%`LXK@m&RgT>1l ziSNPtT|>7l@rXuGg)|L&MsSBOKtTS?Q>{tXxt7ppcOlh2O*q5I?HH`!m+N#&TMB+- z8v!u}-KWb;@l?k&dlt<-wjK;lZ{mS@mV7+{jk1)KEX3Z-$)#*RdKAA~!TJ-=nvE-z znwuG|`|-0or@hP!wfjmJEJ_RE`d<96Ri%lG>EL=2RACpUIo*b7`yvNJ4P-?TJuk+yczY$tuBc5;%|2FXiJ774a+SH+^<-Dl z{G6~&t1Ms(4Ji?@OV5jKURST1J#kJk4cINRBmONP*kW9>lWGYrxGSW@`mK{t=CB|p z+FLVs9IQd&GU?{%8w-uMCL}?72S;Be72j(xg#7Gtkpn+lHX(M|Bwg>uP`&l0zRTTW z+74nvtJxaNcRikd{HY^pWCyWfi!T~CF~&iWOSOo)0u6N|)7t5Rc zMjq6vq$s9A6wu4EsZffX)8YjgCDwZ1GCk=vHkOHc5rpQl(6UqEF5ZC8Vbn$k&n(j3 zZCt##nX#q}*P~eo+H5WB&J8`>$D~t^^`ZtD#Y-eLwPx06!sc$@=hc&mthVFMSHo_h zU4-sXZ|P?vdcc4r@1J>2;?`j~FzbRIZb;8v@%Tp=IDk-AFX*oCUgur*McXXsw=}?y z`~Ka>D+JgeTm5ThP+KL~Bq41D#K$TDy^F4a2TM+Np1;YD8DOtt^nO0nwK}xYDsjTQ~_N2-B@q4yOqbwXb^cFVL` z#;{0D%I=O?%uf=0r(JvCWFoNvwI=!fRvzZ)`Wk z*(I^PW4#o6TiMYf>~9qnm{EyisFKLdWA>-AQenRi9RrGKcOsKEV(AA5KC`?WWLkJ$ z2_C2(&2FO}T1)*W@zv1QTvx93=}%uXt-fR^^2u;9qQZ7D41P3F&IgzTR1fhWq~qa& zAOAHR%jpa^I4etP(-u}qxm zu=XIsTnqDbkMY0e9iszc1kT)##@yVG!dl;vMhvUzIZ#*^B!Y-mS5O-bfYQBa4G5Lx! zaBb=~a$x8my1_2I(ad;RWHDzw!{i>=Mpod1hWwSO_%HLnSx(eA06C|qR|54va__$c zr-xv~9W*wCmPQM*R?p5D-_?G-Im0){v9Q+^LEA1BRRvuQTR@)FF1Rhe0S%IUs~P0X zwO-F~!Hd@HhFzbx1xt2on7mAeQ;b2UDa3xXxGK@ry8+yWoS)7A+`&|N%uN|HV=dqy zAFL(~ccT+9rp%K#&_giX2%yxPZ%1J0MHQ|joM z1m_e_SuL(cI|Z#GohaqB)ImZt9=Z`fwlVu_y0_Ge4c0`%_$k`@ub{A_g~4^{1Z})d zjk})J0Y~#@wo5_u`}cph`LMPvP3h=BK=sQ1*EatLl=;8gd_7weXE!@XD?KM?J4XYv z|D=I0Idz;4*d1@)P<@t$;;}PV-2XibMAX{an$kIp7}>?}v~}G72QiY2m;i(TWJ68& z{ZAQmx2#ylPDPh)g4>-WziVM<%iR2{%Snf|lC$ogDfP%Japc$o3;Nglzlrx7vnCGj z%l$cucF zb<~wCte-$h2a0_H4FQRNza97j-`sJ18l18rdkL$NVt1c4FtaXTzFQJM-S{1@MFeE;BDQ(-=hVxAf za?=ObaU5%7_Af@Glb0RGo*?<5^+5s=+Uz~B`}24=b+<>GI6k~R{@JnZT<@MB==J%w z_jY}F!?5Y?9n3tHO?6FWU1+txGI_UuQGdK2J(Xd^XVFsipJ*a|D*}_V7|L>XiQq8G zi;u3*LFU_%O&v4P#1}oPs__39-+xcX9+FGjD7U3fz zqXsZuJdRZ&pI&S(T_oSeAYHaHHm`rJuaWyO-Ta<+%Cu9gJfe zrVj@uR8=wX>6$uMlq?ZuBaMZ%lK;9SUCm$cHHSWQy0reh&gF9yZLQOPz?h&kuc}zw zG8o7gwlHrP7JB~JG~v8pjx!Db&|y@^Rts8?IEx{`vNuPje8@NBxmyPmvI$VlAHHA! zKC&H>HZBQ7I?Cj#&=m)&%%*>}#9<72Ct9Xi5~Bk07Y~*VfWQQ96;!S0;v>@%TlS|3 zq~{BE)sI=LG0o}9mz0=WkhHHYvn(iP zQ_i!%n9mTYWPd|z6Ry_O7OJ}>uRjm*PiejRWV&+pgs6pB ze+W?ZZY~OOai#jiH;hsN&xy(s`4a0dS$41V&rZWljxbk>qtGbjah$ZE^8A*V#26QJ zzf*mc9WAdGOQ~O429H3$1qJ(5x^t%;ko`8e``oO={lf-_#hTO4G~c(r<|72 z3Tl?fo;d1=Bp943(QXjL^t=YHRV#5jw~phGI_}_>bXhvs7IHA z*sa%&Et@3Cc0;>paLg`YH*%0TQ_l$0K80j#92%OP{^sCVm^PUPM&bs`F710rpkqN! zxOWYpZa7c+Hqc+ep>)I`Inz2Tq%muU=gvgfMo2XB8z40s$FrV13s+e)2FMrhYby#6 zR=C-A)SMXKJ%Dd;w<4~Lw$u=1jbBWcJmM!?l+>qh=v8zSfoZY37xzbqpDla_>FPO7 zD2KSt8l(sgWg+xm^{bG&&6BzlExHheeGooxjYtYtjMKhr8;utJLWBLUF_lF zE5wMhOYgjJcOW1;tl+M$^P2S|>UgHrv1@!ayS+o#7H?P0Bz>9!h`%QMBLQ#aFD+g;|YZboc8C9o3r6iT2QS$t8P(-R3)Ti=suh~Wz-Yox z1Jxh1Q6gft7+$g=sD7dKTRVRQTc_-e(vrEO1QTWX0urrUujfizlO#u<;e zfKgL;wV!{2P`9O9M)v$O^$kzlS} z*VdpkM?Jn4A4 zy&gTxv~j5q#KFA4``!!!WQ!HK^fcf~5iIjoT<1ZniXQU5e6<=*k}-o+Y?v#4P7%-n zgVo#Qe53hN!=6vP8pCKyexHmQ>d-+HS7fpg>r}=bno=5}2`~x(?$8fVbEc7A`Lw0sntSLuCQl7L3!L4M|{b{>!IE?nnIhwJ?*?WSq z9P>kME~`G(R+w`{2Fnr*NTT2Fwv zbC~i{Mvih9hW{JxD%k8Z%xLl1Juw)gnW4Mk;Pn`_k2WwRZfMDY(EI{eW(XsvvPAU? zjlxIUI0C-lo1K6hEyGhs3fCc|$Wk^8n+4A|r@|gCuu)i{0ZbH%JJGE3YLoQcP|2YT zy`Q4n-YB;;Qr%FiM6m^GQX6;K9>FngtQHTQB0wO!rJQX^sUFqclv9lilXj~|=;i#R z#_1moI>hNB)o`QBV?9WLk+YtK2GfhtMbNczY7~AN4+n^2&ljT=d$Sr}^~2s<1zLvW z$!9s(I}&lPeTESlN5nL43IJ`3b&DB>OKuxZWd2D*`?(F~iDiLxKKSI5$eQI=OW<$eTFzvs zzQ5ld5cK@sHtxR+8LLHa=+T~eis^;E0g2`zdMEc<%0W?)4RdA753#UMg-nkGDfW+Fxv7>Wous;=ZL`5 zq?_k^AJ@otYrroXEzQG|v!E$m1s#v$UgGeT2X$Z83fvNSu+r+XNdIKd(Y>vmV_RPKxt)NVo%c%y zalaje(1O&q0qVe!D?KP(z(8SHV%!eIhh-wdMc7Hp2*&hBCrhV5$Ht-~I;pEQK6j6c zE}yy!0);Aa9r|C1U2$}**)`#pZ}|TzvO)U~H|ZZpK@-*grO197n}D3GT^wzkJZz2r zqr$jv+z#3ucAnIx7kbh+$D8TT!c=*54w`zrY+Y@}_^v(Y955rnC~GPGf%&H*3%`31 zK>fi;HCx^Od3ty}h3O!H9iy8vQ@t99ql2xu(Y%=v(++k zHG5wuv6G+D+10WWGd24!H268+P`=z;T!a7=M@jbhu}JDS%rL=-gknikwP0t+@pLkA zWfICpqJTf89=Io9j?et;wC3=rdrgAnGbo6Q&h?-sjMT<74#q(h_6!uJs0YY~xoV8) z!@OotDJ4EH6c{aeQmH54GgriRs^KB#ys}dF?BncN0$5T+QJBCFle`E5pIM2Dqn?I@ zP-)RAgR|154gybIRH=)N>1%B04cV;o2gT7L1cL(kVUMsK<;mL9QwKx|LhZFxiLiS< zgolW;)TCMMID_D3iKNvygM>P#%NjCC@Cu}LMZ#xB#O<1BCArnS>mFbH7I!466U;<$ zBo(BX`-laF$Hq3&e8Q9Vh1IDn3IY zrilb9(?(H3jpmcVG^lgz_M?i9q*?zanoIwYEF)M9jMyVuH>&(`)l>7-UMeJFwmFoQ zB$5a&UI=FHB&gc(mByI4JmH8OrSVJ`sm6kpOdVRIQcwT=ntQ~t=gWYhou?<3M(!6G zJ6mf2yMu#+16zm?R%W(JZWd-HM!vB8FKktePgqSXtqgs2VPx+KfkBjYur^vBzdQ0a z`c8v63Wf?wWhvC~91fzKl_Gse7z)sag%sVaE*@%YvInDRDwIz{YFNB(6*? zeM%7;R~2MxIdNZ~kNLo4r-5v#TbI9XbR@-r^c3yi*DS9Zuo}3_GO}FG=8G5^tT3U` z5!k6%3j|eq7aa|FB1qPZr5>?RR@1$8Jj*U6eN_<&`aVEPO6?1lZ*JBjMpjNYVlC^d z)aIlL;)0>fTGZng)U-jdCoNfxI-@<9sH+m{3#~4cZuq2%z%9D@8cRs#mC8;0`*vrn z(4D>oBWs%S;p~b#KxuSHf)*Xy10#!~oya=vE>KQb2elV98TFgcsCJ&0klyW4#E|Mn*}1`_+0 z3J2j^lS#t+mrA|o42Pr8O$D)`v^23WtEJ&7fULZGNJ7?Y=^n%cH(x0M1N(O!)wQY8c~&l=w()QQG2 zFnF)q?qTKzGZB95*~($yxvj-G=JE zB%ogF=Z7e6tHrL_oByhvi<=WuuNGIA1p1AJ%N4RLSP&gT;aFL$fCR6CNp37!K76E{ zAP=u{j%-1T$+n)Xx2;C_S3X=^wPi(J29eG*C9}6VYqRYYxb*(~H_DQ}7N(Otau^Mr zWa40Z7XMqdBA&X8QnJO`q^ALF+r}l9n zh8j6Xg31gTzCkr_@G3gn9!)Pf?=rQPS_xtlM3n@YPf$=SOY*HjYJH~UuMaY&^IKx( zr?lK^OFLsojKDH~58@z`5`}*zvyogikDh*=yEToX8Ul+HI`i(TB;(wg4z%6}j@3KZ zF3eO02*E?q4j>XV;q`VqYq!Lt4$u(Q+o4aG(Q;MkoWFYQWjg+Wsazjk}tBn zjLXRyC*M)5L|BcA@w9**bbEHQ+;3v8GH=F23ryFvl~~(GSC2DuQw(Q@*{T```;I^;n?*y&lVXbC>UbI5w(QVsNdE`uo5ZQe}astBAD?rhl zu`I(9;+)Z!B`{HsRedxP_}z0>oGLF)ithZvsIA5_Zb4&!1(Uv9LQ|R8KO{|XBqQR2 z2GdcRBp$tB9Szh;2iqHDw_&pp;3iB&qOr`$EU1rBn*Nh!#Ayh|o_o}GosFq zr7I8h?f8tQ+O=KZBr7eUVJN;9Ta`{^(qauUIbN^=H;F>XiD-@>{uE|iM0VPw@6K}Lva|iWvQu2cGQBa#f4)U$5X9O0m6&<&R|Y zHgdye^UQAG>@mqA$Kpj1Tgmasoh){q&5|=`wt91ys#dufv9Cp=e6~g4(ZWMj7Y|WU zNk-ZD&NTfPwk0s8BgG_h4R%zna}|?TtQSiwz$AN3fuo7V+pc$pKm#wXT+5fd)Y(%> zLkD?n!nLg=0OUvE;NB*ro*>6hA*UBiyt`ynvdRrqSFO5aJKxOkw65FG<&EPOEu%ZK zxw9x0K;Wcj@wvw~;{u=r;p$VCv5d8M{Sh8=-k(PSI!vj6kH~_piXE0R>d_;MXiMq4kph{5Yy})`xX~rkGl!mD%p#j23CvT@mGM5^ylA&V#PMs70OiY*n+f~6cD#CB{S%zOywgd4(_Os z&|@r_=q1!YxV#s8s*4^_od&%74cc&Gm{JpvUi8!lJ9%f8?`%84Wym$un^YC`&~&gG zbxoNMKmoV-Hj90=48V5raeUo-yu5uoRL^VLQkR>PEhVX$3OxSa707c}sYZXCJh(r; zl(tZpQuETeZ;|h(CWA^5(FQq4k-SQgbQj9cc}sswQj#4`PN|S|vq{UL|AA0bjeh&? zX1{Zsp|~2z{Z6$H8~{xv3yC&cCez4RUCCtjBUa!4t||cG4m8P5Qd!kT9kkN|>FSzC z(4<_UIi3U`9=zoZ29X2r9vEGbqj9Cd?l0JLB_)`}e2GkUo$fgeO?&(Dt2FN-TyM{; zCW*6(HJzDk%gF+UVG>Vs;l~%9aLkB3}d@iqL0~6IX6%6o)4IXw3g-7;GZh z?J`#QAwG4-Q_5@g@0Ln~NK$L8UVATmg9KdkEF5C>*8i@t2Ij#K4NB28Au~8!2WCoT z(4f?i>;7$1e$FBF2l#`U$n`rIpk+UG8gj%0*#h}-jQr7SdEE#1*6=0A!&!xHMBXwq zb?u>@YK@09`#CPd%f_X?+u@}%H7tLY8_4dO&xX{fd+C|dCQ{KHyWB|loV*Qr|G6K( zud-Hx2~L9o!ynh=pH6LOgoQ_*({wuAKNiU~Vc zHUBh@I?<34#U#wncT~&GQvTBSFi%H3@+kkYgKTpvH&4M+ajrlCFvTz%>W){`y+UPU z2gPa=N#d2v*(0C?#)cgWcRPEru|BQ)EUC?%ilz{&7P5k@6b;@eOJWst3-?^kLS$X$ zi)qi3ST^(g-LPFSK!OM0YkJkIPj5l4vMJ%O3$Jkjeiinq6-Qg|K=sPl$R2%jwtE!D zIszO(Wumuw_8wzr=eBcuyE-mCOez} zy~90zRUnQDn{rAlvqjlpqYfrcdk+*X&rzvCizHL;g89fOB)Sg0SbYY4=b`T?-fdld ze`tJo++4Q-!&9(EJ$CVx9e3nYdJKzQ=nc4r<&%%6u;;e+>k}te+-qH7Tx*JpPJc<0 zDPhE5%MN95Uij$PZ%xDXr6e^b)?-FA{y%)4xn&E)H7 z`8+*owOJEYr8RrfwerR5kva?3><>1RnL`PQ7Ri;7xBnEbgm4thN`Xo-qaix4t&EgQ zea--jpxZhUNx^2D-SJo)5-x*I{7Dp7LrkRaeBm2M>cB7t?TwyUX~qUQE_j;JHN&Oe39JD`=l=F4IM zKS1A=<}z(kZH!Ez(R(a|v)NQI-gvwED5tvU?#JNsuN!?kdU_pseSdJ{6GOkT+s?@q zSxqA+FXBxz)qX=1=E)lI20mD6W5!qt-*NTSOkCt$(906B8Vl1{xWE~5S$Ni}05tT1 zs!5K~F07rGR=IC%9j|RK)2Mzln6i>@fW4D`1rmr+jx*6uO3Om{zvnf6w z+ca-4>$}Y<P?cvw?Xie|7xE8sWM{>stzttbw&C zeCWiWcYahWaPHsDhjFO*Vhty;IDFKL!!0g|=T^f1*sAl}e)nws=i>SKgiut+7v0;t z;R5nmR#JL2pP)$#iEkB&m=wsT`@GN?xO95*lI6!WkF3Ii$G z4ke9J+H)e3)~l~UXh%`&RJLYo2`bLV&bM>hD%Zacq$H*Av+noJOS#O zcU#c$d!T_NxuCA@B^04n3+cX^T5WBhY}T5AW|}pfX^GtVvu1Oq3EHFpq?D=zSwVCPYN_Legvi40dE_L+>4P4f3_)8T$Bfm$d>d;?5vrzpe+rEC8(-ad*YaVKy! zp*OK)C-H#@Wv@G*GK%@P{D?I{{UoFFDt76N7*>#Hk8N*@|FC z5RVzY_71XKa2)~It<4Vj8=F8~R4eFKU35?2p014JIPO35xqO6^GJy?-+T`R~n;89$ z(a0L_PO92xQ=3fEYQ94}LF#CkcTw3ln(nj_-oV_(4qwQN(N+8u1l&`eKCB{?y!gGF_ks4ND7ym#Ylov1(0L z1hZ}`>P3h!tu9>wl{r7m73VY0+>Ncy`!m{$g z^yH$@mHnq5WbTEnEzZ!lK(&h=Y18qr% z{uKzW=hTBMBgu;7XR%U*YWtC18`swV5><2jYoRsDqAQ%)`X}qQi4X}(#C=f+X{SlN znL}}ZNjM6=$6)K5!NGm#QA2YCvV-v+WzEx<9}+E@D#j>7ti5Z6C@NA(6m!egHMXla z?0D|=?x^Yke0gv~`Hs{ZQwUV_MlhVGjO>z%8rO@i8q(7v9H6OBSOxji(}B)ULr~S~ zZu=o{R&PAvb8NOe&)vtz_h#gLzjN#M@AdLMzeyQC@S^jrCs1Yn0c+P}cSawxuJvQgUWSZ<c{QeHtcg_NamG* zIB1P11LH3y{)BTmu=wxAtOR7V>O^nQY*8N5SNnqSlE zL1%gv(oNblu3b9Q)eQ~GBZZEEo(>LQUzd(9tbVwC&04+T&@B)les&#$I4fTIGD>&R zHPpe8SNv4^KMl46;*D-)?R$mc{8KUOj3L{w|Ie_tBAvagHTER@AH&+mB5HP>_sgBL zOK)U1=7*>6!`b;1+=r*9kB_H^lW%Kam6f}n%j@39@yEF%xElW6<2~EU*~h&Q*tkpt z^lgKJai4PkL`-_poND-QP}aYg#n{s?p=R=o3eDC06o+UjyM_J`Dl;tLN}#_x*n%Qg zHyMm}xIt09dT+egP~>tbw=w2_{62!m8wZ-^&*A`QC04Pa-O*GL`Gof z2iBr_OqBIpBGd4_WxPM@tfN6HsaBBfT8c~pWs zR(JG24!p1bheeUfBkYO*JUj-$1e&s0fTizQYFxO;tthrK|JW_SLXJ$z7rh#;GJ(J2iO0*Th^emQ-SNfo>Dh9~1`r*faDOIDsc_}ovSc%F|m!G@=sY?7Ygq#dpKYi4)OziS4)pj8|0sC}YYU6Qj zlUzM;i9NX0)3>`JY`@IDgyg-O;oz82O*O4O>O81MB^J_=a_6||UyLp=BwdlFx?J-q zhwb8qKKOwP8z&Waj)^JdS3_i@R4>>nvbD-zggBr&iVNVPTG77E{En5?Zey81?yyNY z0$DM44F{<yD!w15^>j|bIKaYm;X&IyoWgKdNK0=zIIm$qI$Ka8^kJ& zY0pdZ(;I7HwEV2uVXg^^x_X>HU)k`rvTG$2vgTGJP|&)>=GeTRv%yj11Q)k(M*V_j zvq(WYXWwt*+9-tnqiPy7M#06klqVh$+a2VY6{QErJ^nb?TB)j(#~>)`U#>gVE;L6{ zm5mtUQaX|BpN(sTb1=YhD7I)X4r$slLZpLb*jZOyt`$@(w)ZqyX^b?NhFak;UiMlQ zzr_vCFK8)4;Nzm}I#0X|-V;BE(GCwV9|dMH!!fI7L5@$8nimn7rwbaSf_B;YaL)I^ zAejB-+dv02OI}`|M9CtN0)Q~U8WvU7%-Ngm$Qx$;pwpRB~%z%q<_9vevS-Jo*fq-f*72{ z_24$pxUFZGSG|4fAdyTFJx0NQ*4Bpf&q7iq*;FdUw%jFs^9U(6#a0gEm^8o6?M9|L14&M~=VS z`Z^pKrAv!KqGB6H#>>`aGfXi8a+PZ*!-8xEm+!Oozshyc(&0vb|59Qm3wi8}C1UQ4 z#m>_XS_UWF0o4~(ZpeOqu)fEh~XU3Vi%;|KD~WH8}Z2Eu{iecmLBLArTe4jH=l|Hke8*1+oZRy zudsh%y56P_l!nEEagw5?-`Ayo`Pj8+bB?B0&sNC3a3&2>XuKl5bS87Xah0ESOGai> zqO=+_ua8us3aFka7cqK?+bTyPnI}!HkPtV_o#f38uJz>UBRj-qOf%kUHD!G7hP9eQ za&b0~ax@fD3m`tT!!V!OM{pR+@O`W=k@hbs*xHcs%oWT+Wo7Py2@!m}dK0V6NGG0M z)x3AS}&Nr@GL{lyN#7cXr#6QHh<^RLB2XwBV3p0s%{k6h1X z#XCwb+xg=oPbAge_w#s@8dd%_R^&myxaf%_K)WM!_}X_ep0|nUTOQ?JD+?+fHijS{ zeSz^PiS=HEhA<9>IRHc9W_Q(2DC3w&zj9tq(<~1!9;jw?s7V>65d<#MX15Ele6$?a z6!ow?Q$QkLfur~&kASfX*(YDdtt45gkC9S=3Gh&zpq;ZgMdgrA`A38$<0k8u&NQzJ zy;yotO~Qi<7AvWc!ZhH-&sJ8Ir+RkMWaY)Zu!;cnR%Ndp7+2~2i@qnKdAR;*ePkX_hIN=$}`b3~h1l78KT%%%s9BZSlw{M-?+E zvh)=zr?Q~P_HC^O;%3pL!XdG!ZEArPP}L2gVky>XyyiXQ%JyQ;Yq0EnX)?GVz&Q1g z>_RRb=`jb+hqayQ69nZFvL_RvlJsKR&qIi}E_~4o>PRX3HdR(?m5@3q>~n6(>?+(Q zL>s{)A%db+TU_rnol0rT7qG!~i*=@$;uiWDW?s7ElpDA|$PIqrx z9{n!9@ca69BH-3w+lQYW|0nY3AiMazGZf?hLeeQ$qs=wLVXLl%X|+kc^Ihk?AAT+t z*6uoAr;Xc?HyYEtlkVWT6hTezS1@^hZM<1c`P0#sHQVZU>WtfdBt(Dvh5Fx7lM06j zgaH}|2<;~t@&9i@`gs7z$;HXu#Mby{&h%E>vOVZT>RnZmK8h0eCnEQn&qC4cmuch? zRWidi>rzJn7my;(z>YMQpo+yl+=5AbYgnCF5Rry6JAOY76Eb-&>)6Y&J+ZWFQ=HCI z;GScXe!S^BO6wC@fUParuv)REC0V7co}yL#ZV%|{y7(}E_`xdW?b#`^;uug);-F;* z?fl7K$oCzkiZfk+y;hGH)CbQgmk-(^qCuf;W4|=MrC$n_0M5II5ziLfv>Q3oE5y7s z`9-@G6pzLkq~A8pKf?~vn_N!A09;~?YNnhF$z|GxSL(JO{(Y)59#bL>6rxzkGyW!z zQ2rFW)zqS@Gjr0lMOT(p=G_A!{{{Tn9+=A#dHQGLJ^L78$^ zvU3kI`EbrjKXv-^kFA;Zu9J&d?{Q_D-E`YTVTOGS+opzu^<;SFiS#vH!HK z0%$MA)r%C|bjd;*{7SeOm?b78W(aAO22^%rX%U%)oVvPgd|KUr z2F{R&xmqNU#KngQrw+Nl-t`Y}gk+YmmKQ9<^rt#_nH>rX8aOjDiN^sFe~?u)zlQ&! z6)Dh0+Ufa zPglk|_QoBw%WX0pRZsSR^D{oX4q9|ol(Y2(#w2rd%+zWXTd*2{O;$t15}o;!-D1*} zhVC{_P;WZRAj>ULor$scGK4+9p`ElW(Y8orkG#rFkeQt!2_wj(;1g6I3Vf(OdqgcN zuo?d=Cxk`0#Txu7hw<6mOKu-EA}YS2#hT1uQkW`^6%%*QIqk)@m()clovP1!1}pi4 zxs>eSoWhYQEfkl2)(@>=00A`t)FO*pp`XbVRU`5D0IXm|k^D&=)ZOLT4s=ue16o(% zqAho!?#=_3EzieYF{5ZV)-Y-EP_CU#B9qf@J1HeHMZTNqN9`lsZA{bwOC1TiY|HIG zLUf$7W@p$CsT|Avr8t*MIExlkKq6;}`1U%gOZUUB`aU3^mV}0de@)`VKrP_Cj)Yyw z$a*w@3h6_*KO^C(*t|kC4Oh|8UqH8Ayek_B>caWz2WK)~8i#LqH-`Tw#T(4>DphK7 z3<9-AE@Vvf8>RaE?e4@KpCY!Q)Q3!KGFUF3`iVI(8$)tZN9?MVg5M3K99YS+^pYW{ z^g(m%qW`lhHKx?YT=K-pIs?<&U{goCv57Fj*YV|sTcL?;XE8PJr=Q5kbgURJJItUS zv+xj}#iAk~PSvN@hxSd^wFw|yfG6#siZ9s97Au;ww|2dl@p)ZQHhO+qTWgJ5$fh{PIrK z`~kP>-dm^EK6|foK1*0yLMD<@gdV}cnGx$FO3M>V3E_<>=eVH^>QX=6C4_cLFI)?) z*4_5o(IbYwEZvssPcO~wHf$czmOJvFApRZre8!PtmQU~hJ&=Hyv*)_{AxrN6%pCsv zDuMies>J^VlQfLmjoxEH9PECtc9EWeCJ{qCD^}6ejw)=HN6LU&pFeXDAj67_jksK! zGM)|l?u5Q56HW%)a&^DHP7ZVzxG$aRyon4D4Q;H%@3;L*kv8ex3;)|lM3a0k={QPi zCX1$_9V=)Gm6~WI_&f3M7uoTu=|mi5P$z*Ky>R}va74VBZ34L|%8Z+U>B@dj7+*E% zD$o+eNqi52p;-^eTe}43b63)Uq0}&o&RT<55%>vN20kbcZ@f2^V^}hL<5!VDtA*QK08; zumJen_$Z->Fi)EfK)=n zkm2|t-MRzZcE_dzw|@&0=W1 zb^SW9bYQLNW76{f%t(rej1y9kQX=cSJuOIpk$=C=#nV1!@N9avxtDw{xfygk3N37u z<&wuW8qR9%w|RcjZ@?f8qWbesm#H(urpUJD0W09k{o*b@Asfh^pG=nQ$XEE=YrAQ= zzk#_jNdCTuP#)AMR!d-ElN(C~&20ir7*2UoFN*>Psr%-H7{hBkjcAq_m&@Dm=w z7eX$L3F%88$1J4Q1Uiegdy#69xzV6ezE~#C&?HTmuacozwv@ANADnfr#k#(lN*P_M z5c6O^Mgn)!03jZi@_Im76fz1e;3pI}2c&`7M?K24KTDbb^%y2*zKHmzkknnmkyI1M zO-dOS405-o9seyWl2yCp&u#b`EVHO|QU^PUMCR1E1As>t@^q|p$Vd}C z%L8ow=bj=uVC8h@IUdrR-ejlPz9zS&ysxxm*V91Ujv&uixD+T7?Gug27*U@s?I6M{ zKvvbFO>9DjF_Ku(D?n|eLYH*Fjw4l|>zc|!zJ&~iX5e!ll_tF$pb4;9v~I3iw>tl_ z<#5N`Z1b1->|g@NJGPfNTnmkB=pXg#vlSr35OVft>^Lxx5$8J;=1FAmwe+=g_zFe< zYd@->B~Pyl^|Y;e4WjbY8e}_mPY}Lpx#bpr<-*|3c}I42)5Yt+Nw^xqjw09xf^>Bm zv&`0%Qzrh%6q}ktEt)==mpoYysnM#p^Y+MF)s#e4#MGM@x|>B?nwkR!6U&^8E;ocR zXK3M#n2=iF$B6!zh2xJ|HIue7F-|fsF^XumRB0-$ zD@~`BqOl#quW&_i`(6{*_VXvmE34wpUj%VXzWog2{U2E|?pUWx{&Qku`Rp7>CD*Ls z?rvkAghnhJ>5@n&lLl)G0`V^c11lqiYY$IZI!|5rijz|XioC3E&R5vUB3HUfZ9V1i zXUn>~@BsOSKnV<8oZ(&iClkwRNY?Och{w!I&y?yrhe<^i57rZe?~T9I-^gl1Lo4z; zYqH~I|xC40`(uCFcxbHynf zvQii#7~2jX#OzeBRU!82lo{8%9_#n$((+-Q=t9DO9yc7GDcgmuOr%FDz9IipIM)sp z4nHA)fEs^xV*h>N{4eD9|0WK_=) zk#YNJ_Gh_%)qR$iTY>ez ztK>9FCGmuG+V!_>WyDitCfWErXDTbona5 zR@Zdkz@@Q4A2YD|J8a3*vrTtY|7ylCaJ>2%ye|9v;$iFTY-`V(K4t43JieY@zWRCb zWE{2b+|u&u*F9}rCBx6LbijW;#OQULn<_$;QqMjM>@(-+BOusH=Lnx~xmXq^*Ghh; zvJU-|-~&%%>{BPq5HKr=r<_1epEefI>0LweOvNRGW2F`7dgX``Pk5z%Lh$Xol+YXt zlJNIM68-E;nD10W^H1@r6MkC>s86F)EWmHTVQa^()I?UxnpMFn#ts z_QCsBx&YM?_W4A~b~|x~fyAc-(PLJv(9S^`o8UF{H*t(gh8^0ICQE+u`RMxd^=AZC z1l9WXW`ABEkDlT}v#@m*7OMeBBbff+S{nC*LNlrfBmo*N9Ce7V_>Sw3FR9bN-N?f0 z!fi&1q6cFsHgT}IT!hxxlL#y$$|&nwoAyK0g{vd6H0x=SpX;N7`oRqDk<)9%NV^B2NiIZ(wWWv+RR@Aw^U5*VZUC zK^Y8)_Ju4(-rEkCibO86L}9j4Nsi?+*;sBu7dCM$C}o5=As|LAleLGp^#QHjTB_=*R!D(LB)-U&EaPuXRWQN?3*mt)2OL z2_>I;Hqdtfaw17e92k2G;@pMzRpY@0)KfDi-RCMDx9+MLZb+axGvdsItqLxQ^2Z30JVo zv4mJ;k_;0DD`#Do?z}CAJt{iPaG?=eMcaM9Ub(Dc1oA_QoB2$hFyUgSLGN2fZ zO@H!w%{3L8x|v92494ZcS)PhgRfrbn)g#ryg0s;t>r6bK+jp>g19bc~u7p}%#+W6* z-qA2Fz&k8)ltJ$zk)n=GT&Rq=;jNRwj+}S!JMJK~PTs?)&I5ib8WOq4jTF6v>PmnQ zR03NwQgRiGoUo)K4V;3~JRP@@N+*egcv&d=Lrq^0#AjT_Ej7ChxeKcImAIv9vVZH; zE-eg{j@^!5Zk&&!J55)0G+utdlo=3%pb=|d^U#=-WLNQos9b1BrV^GjfIcKpK`Rdr zz`S0Ppnr4&GlA|#OUeVbJMBm;U;7=qm>Yj(>OYH!ed`dq$}r?zG`?zw3U_1c^zL-K zi~hTqm3990^1?iH5RMN0eth-P@~^JJE%y8?`G+%0H?D61{ivT)qS42Y07GN~qlsi5 zF;JixP-_^2`UtGVi6iY&K%*-`d7(}|QG0Au`YC|}K3vX$h}&*|GP!V}Z>RG{oaXM; zwr2hW_V~oum?-FmUB31UOhbOf(*Z$3U$=s46AXM{spC{eRA$+wamM0Q`lNO{*Hho- zwlbl8T6Dj!zHSQMfVZZ5NL^3YG zefTw~sypTvTq*bJSe&ZUNK82 zaAwhv`bKgt^zve|#%MjQOQCsmsDFi_0pN!Z?$E$!ffa+|ZI1L21}OQkik~#TW~J!#HrxMSZkT zzM2FE&T^Cu2>a)oi+*%-YnhoZ_QHqfl~t5h7}{8w?8!?4xb`~Wgl(DhP&S}4+mqnT z^qbmf$uA%5jVf!S@5~+;s0kjr{t25uJpNh}zsA-@n-L-39>FGCU5=6_$Gi(FZjmjQ zn+llG6)FnOy0PxAHY7_}BSwiFWgW1g4UD0ROxXzV0j^>z-LRxE(OAH*gG$0L`xb2Q z4GfQ~bC23(yz2*=m*Kc3Qs22Z;S!7^<~f~5+#gR?#my&%u(eu}QtDZuD&qV!vpx#; zl&@BlO6&s|FD`ZF$>D3#@u{0}j6RF7He6I3Gxrng_1iE);7G0S^?m$K*TlXDj<_;X z5oC|W-42NrF zqI&uG%`U@YkgRwp*&(#|jF!vS2ONT>5kWG;*z3&ZR65|*?`$dQ1wWxfKdmo*AV%|i zZTRv%T0?t;W`_bS?HrG{7k~6FuZm;KYzRjFb46vMy(hZMu0S7d?<@L`C7IsJ4@$QqkVEl)vV&P10o-mqG5?7q zsj^3f>G=@>1l}GM6){V-u3byDpQ8M;i3W18ar`m6Iykfe`)qO7UucbE{cFxq zKW1<-4OfenvoU>2mP1T5KCB;KPGQ|-s8F3al^G!B+lwMD?#HJ-SZ$!0 z3qejU$@qN5{*@KE2AYU-0#=ALx9xkRAYSA0>N_BI>ZNgvvBX?9?Y8S+ zM5l%bUN9sFa}0Do{B(;+0TDYs>cf9N*VW3S;NkJ{@fD6zqziR^C*aTEdT@N&*g3h@ zebaF}P%P&O{o~tTPP3rVYXvV-8nthtL6Ud^QUKMo81Qo#0p~KcZdIG^9ZB@m!zOo^ zv-!uYyVO)%5yWZj2BZLl5`(MTcWMOk?e5{~#-4rATV1UB{rvg{?Zw^I!p76i!QC1# zjd8PkzdAf$K0j@eeDg}q#`6y6#l!v126^D*kA0IqH_*K>1;>V7P%k-v4@!NC#Q<&J zwa1AZ7Yk7pm*mYEXMT%;8 z4>t;p7b!4KYo%vnjol$FxsDFou>9=e3-6EMHqaB_Nc9Dj%+~KN3nw^Jsau+4Y{lzx z9Y%;H^37?Uw}J!Ci-zn2w@5dRbSZ{4R_ES&pW)(B68hA?sHLJPcIzbKJT2sg zliA9<{0$VJlWaV z9uMQP@indWzSFqsg9-OJKfK8;Qc;!I9hwZnNC_OE8Ln?4Rmym8h7xW3?6&7G_Sa;YB|=$ zH`M6jtE1h*oo%qDT2BJT{GH6?&$x;fRhOM_EJus)SOgEx^v~$?l~P*ul9}Lm*&A7r zJC43%gY}h-I}FT-BqQ96Oh6P&WYdyC$Q+%lMx`MZ#)k3gf?lh+5eG=?2cA7=UWyES zpIE(2(oJQz&K4#85s>uXNWU7D(7CuL2FU1w$eZ)ioP<_cW5{SB>j;~$B7=U?&XmAf& zLv-P3NvmG`GBUa8z~UnwS>YI!b(XiT52Z2+B6?Eza0^4499^8uYtrrUt>O~&sYn&~@)y2jnBm>v1*Edjv?4aaze*NI% zbh3zlqSnsbNy57VWw41m29i}LH7S~Areo{NhnBa5szJS@2%xE)7?tTIvrM!)LoX0d z!6OX{X>^kSD7=2&QHFA^w`6W`H}(UjiN|8rQd2KvX`mU$V)M6mWNu(9c)70-_w5(z z#k%Dwq%1ccQ!bm0}}q1ev9jwsDfq((~^vyD!LPYxfeU)^g>F7UpR@uS>pBXQue>r-zr%&)Q1@57B~}UN{?z-A5wtIGny>onG#SGA#F}{Z9*c2#D9PCAG?I+Nx=F1dVb~)F6b5ztrZ0w9V*Y6RhtF1+ zN{uIMzYkv|mf0FMde$YZE2guzu?l`fj3md*Z(ekp2!2rCFjapO<&r0&SRYn+-|sc2{yh!2x>7>;mj4X(Y#NG*$dC7iAbZ*qCT(uvTjMS(l0P99J((&A_+7}vC(fMU|rs+FH&E9E+>tyuYa-eW?J_D>@}X*IpiT89?mwkd|xg^awV-=7)NS&U7?# zy-CxlNFb!M^3S9}ZjHS_Byl$R+_jNy2U+;tLxkK+s;++Cx#Wgq0l8q1fbmZ=@yKEd z$L8#g0{7`6<%-$%CP=6BxP9@FO8?c^(B%u{!g^JRb%78&sf0xX0|TvTod^?wZH&L& z65K~c+4FU|XZxCWu7MLdUnN7lMHR&$sd#hCbFx1v2Bf4hi#GZgD+4ZWqo0pRZ*86s z7;j%M_dH~w)^DbQOssi@D$c%JvMVP^DqNCadt=sl7Vd*`NAocaRbTmdG%f!Ho_bWR zbS9dKog5>2=5#~&(5O1#KPV|z@9R zSx>-<(FM3K$S35=4I3~&l4hcv3`mGJVX;(4tF!67cv?sBf4kf|JggvMXGtTnd|$Wp zuVnUJhCRnwULFOD%6Du$Wwr>6|H$X{9O%YjXuir!Za$+9)(P{1`6q`mmLQJRG zbEojHT9b3ahV`<)wnawb?xd|1N>+cd0;k5;vhyp89z|iwP?VNIUTpUgF|;59po_Fb0Jzrj0kq9)?>YIhG@(j ze!Kx&gDkzqCQ+pCkH}J!k)hZN>llp6r;WZ#y|Xq-D*$GjfO8IfQ#%B%pnJbzS5afSY(NU@Dv) zssSr#ooVdq`+%yCj7P(sMbDTe?kyAQbL}TDoC$T-bmAgV=sH*L78a5b_Ml)L$2|9-1tTAPsxD<_kAEHsJJTkF>FEpUpoot*Aj` z&U86(*r&NRp7Wl}&(MSgXv;%x=zZTmrV&@h!l~Yl2$_Xw-DB_Hm_P@KCRwn8~lUy_#oU*0*)Dse; zVVfd|d!v0XfAqbxLlDOi1Vtr`OKSrwtjZ!6wWOWLcUGnMq!Kr_aJRNd;4DTj;YoBK zQvg8`@A@MxYJ7Gn7jERRn_nMo#kZ_2++62xaJHZCmM=u^Po*nW@I?Gji8!Sf@% zeJSg#ZX4}TM}EgRDoN`xU(V zF&EyWclP-P?iS0|JVh0j`6NUZ;fD~p-XoLRPMr<{c}{m6+`4#9`dej>wl1#-{uv-N zUln(4(X?dWT!f=HjrkcrIlDT<32_A{l4ZN1Oqer_Xh8>!;o*o3>q4jUk(;WjM_6kz zLaCCI?&#S52N)f7yMkS+ylxu4@9NyllRcX_W?|XqV#DwWU4CPkWG(3EK(G62kbfGk zMuC{Rsn-gcH`QUZr}57KT;Vb4Q3VT+Gfsn?*u$@l%#F@~W8Fc}Bu%eqv%3B;i~{-| z9PPF;$?+fU{SE+Z<8DQ@J^!{JzqTFyM)*&^_{VT~>f`6wOW2?P*AVZ=f&p@LFthw= z7&kh-oi|!yDc-tyz>~J#h1QfL?Ix;R|)i6RPiYZ=oHK#{t0izt2+l?moW?d{9U%kj7|nKrc> zl{GmVp6(tl4tKYv$A^XpWgcbJRHR4`%Spd9;$CS-(~As*d8#Q&?=1SjN0a^L;wY+a zJJF?T-BOW9G0r$J9L}*QQg9*6Vri>1pri>g}xP-B$u`7q6?QhttQ`0kP|M5N^f?`Rnt3{4!;(RAmXlLlF|T zHyzZxVL@Ths;C4g7&QQGd*s(LXPrP z%-hOLdnO@clZA;i*i~77QgF;v~@y{ zLs3{#Rd*Igxwf^^WUJzyLs!AK5Zy|fVu~8F1KCRDWHlX`e_X|Wv~dJy9*i+yPyR9${;XT-wQ<>~A3of64mvYH}v!Ah_=fYyS}RHy}A z?hSOrH%&*8zbfKkA4Z|Jbp$*gqM!75JWM7mthvx40^Kj@6Q8!IWk&HH>lSIH?w@(t zL8ZoH_zOWR-A$c^hE}bgv8((|lgeip@JWZh@Zf__vJ|=o&jybg&pfS>#)XRI*Bx*M zM^BqgaN3xLxNs8vDI#OHq-%ejAB;2|9O=GvPh0Q}x^QGVFnT5MD-DfSu-ut=9hV(% zb6e_*g=C@M^%MKNT`zje0h1t3PvBuUSyh@n#iGviBd6GrKgc$NFWt{kvhHm_=|tGNC++Y(|&=**wz)|xQb9^C6iQBDKPNE3_dO_`&a6V^_dTf%P_HCC|$%M zOT(`ofNxTz#uyr>fXHj!GB+AVI6YQVi0?{nG1a~I;WUc0YZ3Ek)jD+ zN=*y*&MT;q9ymYDdn?!7PItwWYhkLj>>DUjJ1!(DEPkFD(H4i%EZZBe8c?QlqVBrL zg_Gh1JhmlO@~b@F^aI*F>KehTMJU#w+Vohp7gf%h!aC|M<&+~l0=)uXO0;GXO;&Am z0t@L*t&W(uItdo%VUL8pwSS_DA5!h_AAG}748dQ{JJ3NwL(S&Q}T*mqb;Vl~Vt5+IS?vkj;b-v%6O2L$hQIl2XA$5G8@;;Qm zR==F@ufEUwXXhrUd)|Jo2Ft!>nrkRto!mS?1u#|}rm9cmS5{W?vQIMVKz$`Hi4b8u zg!!o`9fXDNyY(WmUxX-u6c-*nm7?Qm-n`4o*-I~mCEo~)i+4^KYCJ}SOB5Yo@PHG7 z&M}-ZBkNPY$;vH;rJ*K9Y;uP8USN3P(>~T6EiO~JI>bT*0{70&1;0r;#BzxQI=j{y zFfaBXvC^V*LIbT-m}-O?&Rj1!Ob%K#WjR;hRq3+_Mh>}FA7gYU`r-S@=41`wpXC02 zTR7vGkG-fdGwRMbCqV7jy*~O>@X9th?j`?RKL0K_H_Ym3OFnnJ#;8*EXS?$ireyCH z>sUe-2a7VFLf^0 zkd2?;)Sd|-SD`&R*JiYEmed*1R*_6x22X4c%xT>Uyk1e|Y^%kyG#hSc$k%j(5H|Dv zojTp4=1f{=N*3Px^GN z<3K8TG`CJ&dv}*kfnr{Ndnrn81F9Z^;WlZyyh{psJ8&1;f>(b7-LlaIbY+`+O%+-$ zCtRfKi-;>I2l!w9I{OQtB9Er?&rb8JR`2vkuGg5=kkvX!vQSC>;E1QfpqgL5EoGeW zNT+EoDzc|tp!Xr&EH^E9Jux_&PS9o3pIdic;q4!N^LMWvuu==56$`+7EqS*cT4EAt z#YyrX(=Y<>S81ZbKB-fBZ3}&*nt`;71$#6r)Qo$&*UfzZSd=0h>~ZC_&g7Q`RZ4IK z(Akku+O8k5ar{jh)%h(p34YorV#hz~*X)SpD2>&BBimX4U$&+82nn|KDXG)=AO=OZ zC;U)a#0p2-kX0dU5%sYaRfF659=I*~i+^2BSL@W#t0$U~^tUn~YEr{Nf)pUk&s+B6 zxmfC{Hpij4CRVm6;ipTG(&hLJGHSkuwAm3?yhmzGjZB0L{f+XYl(%QzDVN5c%_(N& z{x`>;T00em%G?Hpm2N2)KQSZ}R{W)wXc7REXV~L_#~Ep(HJF^)ND;MlP|?SYF;V%K zN76RF@60G6;wMF3S`HYpxQ+CVqngS~T1!*&A1rncjJWl?<@Q&qq>*#k6TkszhkXVt zd*mifu8Znhd0h6M#uhn6x1{;D4Mi|3hg|3;44ypr&qk`c+P-CbVWL^$K8qs2%h7X% z)0{v>(2A7VT^AB!!pv&$jMUf5tYWjgbXELZ{zSS_v>Bxdm@zk~zgZtL_mox9uadEo z&jhWUo?qkqc{UJ7f@FAN#P;x-Hpn5%N(01lmhpYbB3JZvm2Fq_C>;u1Vs7^Wiu1?j z(ky)6LtBRBH?}@jIJDO|(^9Y%79L8oE{0hmYXIc7&fg5DQFFL- z#;BW8`hYBbL{>L{9mq)SIeHPVs&cH$_B5DBIn)WvD1@hW#MvVMDR3_M^SAKd1}ry_ zO7hYU^g+GIjjm~K7MQE9pqJ;v(a!vO{l)y~t!|VIFTPRrb~h&*4pn;+t8DCF zq~#BeATy&w@FrAY({y&`co>B8n*{RYD>(pp_tIKWB`1Hb+s9S^`+U8Bd_AS>XI4fB zFX-*fV4@#OcR|6R;-2OfvfI2;g<^erU{H8wueW1CbKF>=4`NnV+9PRX1fyN1T=-ktshj z&KVa&;b2*-mKsykU@9zifRX((Ong&ZD*=_R@G=Mw4{ubV#EW*0! zGFk@uwyx_&$oXf#$Km)C8aJAkXy>;@utSjRq|fKnb=87Jygul+c~Rpw{N(g}IG*g@ zzI|zC1<`~SHGoY{#(uV0n*p_^i+CaX+A()r}hL>^R34Zf=f1D)Nw% z5pvFUtELoyjPTs6K@2_MF!*Y;J}_E<05yfo-0-U+g)&0rZl|rLptF$7l7lNG3)hL1 zK7;E!m1@ryDAte)S#cIRUD7R`xm9Dgm%Oq%Wrvw=owwWj`+hzDVy8d2c{mu-WcXUP zu75s%S@Z|*1qt9yA#ugJ1u}-WJV6#oq~HGk|$^*3_CX;>W5qHH8tI znfzaUT08Lb=JnYlN@UoS4W>3*Nl>QWrEADj33Sb&z0dRE>@4l6vXeOAo%lRuEJ9Db z+QH;@1+5Vjxj9b4CX-OBM?Pk?jNFOOe-->LjkSSAH0sC@U8$>FON>iRX0&NV=E0!4 z6wi7s;t4WXhCz3xV$T)p+-^6jLbKuS-fDkYZjM^xG}7fkJAg46Mn#d_iLb8sUYQ$0rLhtu_+(Zxq9FO`5qn$J0SWvSM zc`4~Y*Esz^MidBim+K6s6=u`a%vVSLuL3zh&G-C0#>}< zX*T=`+ndRFM$$y=eDNg@$cU*&Ys`*ObCo_yIH>a2tU5-e!EKe#6O$qz{}t*Lf^6el zA+A%awYH2tDL3uUk9q7AJxlIl9LB`V#SNpKOZIzrne*cjiIf6f6&EB64Hx+IlA$J? zux_LCe8jUmeCO~gGynawN_*BPb^>Z6dIXo>sNbTr7Y9&yyUSd~SI@c8E(=`zE>re` z8-ho&?ks1Vu4SQKi18k3f?4;E+nz#dqC6B`GCmHyqrmpz^kQ=AS`e?#UNcRN+u+Sm z>LCwuz`C9r|Ku?O_y)eES*c}wgB5tq~AKHd2)}|>j|q_$f{cO=yEdMOw0-l zsZv*)hNIM@v<7-%;+bBe2GY$Wq?VcVW)dY-e(bsLm8#C*`6bTPkt#q!sNc8!(a8@_ zi?7=}*Ir2&^vJKSs@?usXU%u_SA*ds3EV?`dpy%d<;sVX%sXceqL zH{E#PG^N8pI=@d{yA~Y2W_xBnJsRKhmy~qC5t)dT7Czwxj5diYzRx}YE3z(-at7~I zTaB{QR>8}yR4S+lR6rkKTPHetzFHDtU3G0*5+D}5Oq3(V^ZS&mT^8W&+we_!MN5X1 zF6#ra_XKC&Go~U*c^dQb4wa9zX|(AqO}rdobfrY&MS)h3S7RT<*0kcgQgmz`&4yYm zl`7O#tuyvCWz}kFZ?Z`r2T!amVk%hmV4 zluBP3bxs8;q5mqK;|9td=C3@%-O6i1l)y>w_{}jqbJ{#XIdLb4?6QNubVUH+WvS%s zOIB07bb!q}t~LZYyA`b?nU$yyuZh_*gMQ6A@`YfYwq$&W5JD~f^ou*sB~ez*J{4VD z%imI9<&thFqV4hmIx(IMkIcH)Ap@YZoS<4l>+S2Cb#Z~WArX&n&gNUdKRXwdy*hEGlqTq%-Y?^JIX*ne<08W2vcPR1_Hg@KLuG)3H~F?P}3|EhSz<#LBHKnKRjy94VNv(L~ZV^_;~VLS%^8XzbGrV zyRK|5U09Hm?yas<)=5{s*4sb-t*Qd+H`q1~Zha5e!NV*tv}T7<1Jxr2E=}v6mGU0D z#*ur%Fn4t)fb|gmbH4FI53LdnA4nrnNHS6^bjbzTJH8g9^QMMARR>er-BtUnv?geu z<)^P+`3+TfKGlY#MU9#2DQOaMC;8#IWh1K(4>&^0wwRcS3!-(y2~NOFzaU=u#|<}b zb#m=6`M${s_rf!?7irQ|%RFGiW-+{I!I^66Q|oVz2J0gKx75JW6J3~+k`jzv0eFi1 zJ0hxO#J`IAWz69Si+y*@6e4iz&ft1SX+{IzMSC;u9Bt4@HNO~5hPEDBXWU78Jk$cQ zV$O5y#M=0#QPI6$hwv9ygr`Oec?s^#Kx-999T>h^q?i^#BwZ~aanN!n{^`glm`{$mU<-QJOYsJb{mYq>*kTu_UZb!0df)F=m1^#iJE zF%N0NmsYjey*WY%la6HOZZgv9W;7M1G@=X!kl(fJKt6?%WRNNrN$Oa zwZU71T7{*6Ado%ke$QIci#_*AvJ@{2iyy!`*JuR#hll)Yz*gBvKSCKc9|p0=%_`0J z=-cwm(7qJ2|V$EonI` z6Zp2%)dof1H7u=a+bEPemY87}fTRqNr3|ofUJM5N21!06fIMH&cqlAbED)D5e z25eTM@wM~Q0CVFUOiNRUoPW`Ea3V9^guR)g_xHmXp@du=VEJWUW~j7mD*OHvPo1E%sL|qDyC|70A(B-}F-9a|dpg}I4jzkphleSgZJz%Q z$$`Uw#_LXUVX1N~jH5+Y!lW-n3#^|dst#)!bJV}->-U!qY6q!12&3t)E6O#RmVQf} zUlxNY9vSbS@4wc;hX@oPfN`9l{SH^f> z$j0O84Q_Z~y2Pjan>bRiGg+CrTN2~w!poW405;sjP6`gdq>Y@$VgfTegyXdjQ9^7KxHr1#=EX3G?CpN1t#&cJTE7zXo{m^70H$r$)AHWmEO z%0wh79&myz9Ju8qAzF+v&WE|jF$9W^HH6P7k%jPK8$+D9(cRn2{_cA>ZswQC)9Hgt zc!bNa8f#>#mmPq6=V`W;+1h4KJXnUe$VBiOb9`ZmeGgxW^8}oo$-_g5{&DDbr{C6; zd~HL0r)3IkH{rBldvyVa3o|3zL7K`Kc-pCuizVfiW;S;G&H9tGGc&a{24VmVHnM({ zTz-;9xl`oMK$uY#-C4wpUe9%Op)6A{25{VDjqiLHbeR)<2|s6?EWWDH`hJJt?PDAf za*AVPGaV65z|kiKYHI_vishuHJr}gp_DpAi=m#aiyYmgs!_~d`SiXbc<{4<*KH4HE zyB%7-voq%-k6L|dz>n6R_ZtnPVL<0!Lv}s#vkZ`|J(*eP5aWbCFdd%8(*z}nRqo2L zmPA{YYQumqnsXy?30d+Ip&I>fWfhvF)Z2mt+gh7g{**EZ{Nkjzor!KWY{jIlhs25W zeSg`d2j$yCwh;DmR^LC_4LMDxa%4kNO_F)+<@vTT#;g%Yq3~BdZh-K_#s=fH;*`Wr zb8Zw@C{I*81{|hybMDE6FM} zy0cy139V;)psdsK+J+w1eaYlD4s7JzwMncP-Hpbqkl7mcY|mK=D9O%j^nz~b{}@v& z1PL>IF%+Edo&)0f0Ruxu#ohs5Lbm6sNJ><^8?F>i4!7+iwtP&a#~75gP3K#(+{T}( zWoDPu(PKAfTQB1y6@r_ph+x$v?6A+OOaB9uKx@CP{wy9jolOGsCrnpVKXCo_$$Y=- zCmQUlFW@~fw=3F{;rRoBfK9-l3cK4kP`!pTxx6efjL&GH!5VA{r8vp2I4en1@^)T9 z;HE$8qdx58n(2*Bb?aqPw$@ZIssW0P;9#h)azzE1-;ytqs(($>nIJtjj(~$&=-uc#$%@cN&5tx^?s-1`E%W=F|iAb6Np`JErpXHZ4x2 zAm0sb-(44aY;E)LTy|bT2leg6pyGKic5~j@^>djidZXe54{TA9u;@Lv$@!j_5wSmlBXxoTP~ENsgRhvDEzEV~q>1!^V{P}zJDD<)WLVgI7%8d+ z(Wc`1+gKIVv>U6aruC|La%_l{_t8{IQjb%DDPRw|%~(B==Z%eKgAkZ7i4VSn8w#9_ zGQ^%;7~Gr;HLhi2tC+{=F5Y+}pM#9mOg`)R0|!26p_ujjs8~f)5{Zq6VGg{N$o*L8 z;bzxW`KFnKGXju($}HME13HpD!u?*s$2fmj^g*s4DgF@0d&Nh%-5w*l{-d)HhP$w) zmtMj|x;?goBA^lfOcgo)p!Mmu2OrN!P1Q9QmpiOhR4np>nqqinerS)fjT+BIi7q@R z$A?mLE31LlDUqDuwJ;~ghmf%blc4-*5A|hM-i#Z82kpx!F`gL7`Of|Yd1mh|`LehB zg>XT>?C*Z@nQRDjF#>h>j+Hm}tcmOb`XFmg;ntB&jOZ=1dR~sP}HLdoBC-ls@S<=h3iKbvh0}I=!H0-;1UFq z9=(Db!*`P)3s?&aduC=7R#3_$9kf8;JS9p}(tCyOB6!R9TvSOc_nCG-lCNA8xCuqWBChU)U#rT+zu7hM|f%(-o zwCN9yC6FE#OTatx@_|v&e@LQpQ~1K6FovI;0 zm9si-{7Hpvq-5kJ_IvDu?#{s4wPCzRtXYo<$uR~2wH}!Rt&X~QPe?lAR7*E;g4NTN zu9o54F%<^l49nrX0T|t^>3)vSnQCe0iv{8;jI`1hV?Nq}gz`U}n~E2%5x3Tlt;~I& zascA1Z-FDZE5;V4!|xNLYv1p$D~g|fJ0Tw=qUjIbp#>sD z&m?|b(-N=6GPe}MuGTw7k0Q($Owl@FO1Xn+-=jA0ZMPBZy*m&me}xPzhOZeliS4i?Pez#+^KFt#Oj5s18%lUyvX z&i5`%Xa-)thPgZANY}T!WWBwI7Bp)@aBmF%imAI~b8;Y-aMu*%3BpXQ?%RpI!wdbbK64yK-~XS7pBzJsPzGrcHO^omyAha6lXW^fS`?B}J5g3Y^BGs{M= zkAC{}vkw#22KV{+xx#*EIO*;ek(vu7b3h>O>3J-p42lX@&cSYgWjo)oq#a# zjff~rB%e3TI=_zLl#8Xi$3aXmG+V!mo%aI}lFXpw#QuhlY!R|1g{Cv0L1u{MX-OTZ zfX)^azy}g!dBgmAeiJuJ>nQa5M3t5hd? zyM%?~?oye^I1xuv|F)B;WD>dw3!}yY!OCcgVy`QSDJT_w>s?l(SH&_H<=~4~Gn$g< zf5bpEn1XUj{ztr~QszsVNN?oz-+el0VBI_pJ(q=I8aDYYEqRZTmAzwqEqSY~=pV-k zr|Xn#n1ZgHqp3#U5pkp-ctz<#7UD-GXudY9!B`Cm1)?d@!VhtNn8f~qHr}f^iKZi~ z-$>)Xwg2m8tSfe{i>AG`?MWl4C(CdYz41No4{6#XM(>s}zjyFBX!qX6LEcJ}Wo5;Sb!Sl;!w>0oqkdAC5ji!tCr7st4uK*)iW~49Z3fC(RUO7fA&moCzF7>`#saw4H6)Wk|b6(!UX%Ur*mXPKa zZ0QiZ&7k5nttvwpy$RDrCi=6qbeJfPHz_3UgjLW6QP=PzlY@CYqDIi>9E)A_p}P-e zje{QOQ3%rGf=Vf9lT3tE!r4fl5x98)lqn-_y35)N6SHO$#zPH1#N^Igojs8%FD~H% zfQc=9KmABXn#~+T|H!8%2Z!21v90d46uR9u2cVY)|owIHgHBlgWmDTjRVl|cC zM;ld@ZgoTSSCKaj6&L{BA)$MbI1jEXT0^>6yXE%=%Zqcw&@J%h?8e1tOW?V(V3ndG zHh#f3Q*z_aofOJ?drwm}A||941HwEe0s8bKR}qUYBb^5c?r%4zU>z`jeu%c7k~{s) zC9dyfq{Zx3JxaL(Bp7%5XUBhJfwaR9|Mfn>=)=F^2g?QK|L`L+1pFI*7~zcJtFTJP zFZK=6#Ke8X9u*Grs-oow`~`-l)F;vzqIz`GFhR?tmrzPCL|x(7i%S7(*49WF9Eo5@ zYqNyerOhiX+xah3@*uk8@0;-rza82XXr?+I0XpH2q4dzA>qe1F`Vfd1yHG&D>mbhh zH8+dulE@Y^ATQcV%NYr|AoMplq+8KyP2j!@462sKYDDCUUz1<>T#}s~y%Ht82JleY z#aBzks*02xlIDRW5L7FockP0$-KxyVf?kt<|4B~5gE(&qe^Td@DR`9DJ71~bq$5QD z)W3k@kgwNz)^FcwGEZQ-ZkvjpW3vt&mpiD~)-prsx9pcP+8V-^u(|(T=0=b=QcMyZY}8Y%3@@&{PWI1utB| zl!sl#SL6Ue6RDLOxDgHRqP4jZaotM5Y6>}ujQeh069~v)^)f|wI|(%vy}=l=iF0AI zzJW#Fs8;wMy*}DFJ#@!Iv6UNB>Bw<~=e9Peq6t0funTo~t2G@F95B18K#xZ%o8&>s zW+>Ul`hqq>kFcLg6Gmt{^H>1*6F>B|8N==PaZ2v+;l8^s!wmi?JSxOycvJ|4M}_G0 zJl|&6rHwLi_^9dzl8!A%kVB$t4w9sly|IL{#cIvVSbsE2Nss`=WX=e(W;_m4@wQk2 z%vlF`2)tK*WO=uF7&IMX%t6pfDrR`5AEkI^Vt)o};ak}8^dcX_+$*0X6=WOg_D@?T zrVwOc2Xav17#@R`6UD_)$u08NMmrW%0Abb~4)U=#Mk9=PCdFdv2rSy5K^yvGJwS#PsT?WZg=Lc7HV^ z`<>YCkwv#Md88fiS@2p3L3j;mP?6{LsuwWG7U;NJ|NZG&;CA7Qu_U&bLJ*t%(Wmcl;^U*BHgLN?b?|t@=Ay zr;4@pB`sr{8ZRaF?BK(3c6#(5+1ulH$7kI#fqoobQcw_YZG9=VNM7eRM6$1x)Xrw3 z^gLBmIN*o_PBTq_4r!Z!6RbB(R;-{gQz@yFKraKWf0*L4w-3bB&`{2tA#2R%w^>`8 z$Z<)3osy4cgJKAMNo}?)R+p@x-X&@!Uddv^jzKP_hWNB@*A$qqOAPJ-IlrX0!AP(U z#$`%ad85}&TY3Wb&iV>2_T@4q8vHrK>uVuvxURb4TE6dICMGv;U1qS>#(7HWl*p8b zJwNZU9#*(t3e(r;LXkL6QoR0x#C4M5j;~#lSSBez#I*MsYUd2)*;L7FMqW<)(^!&S zl9L@HJ@T5&NKT%UI;ggR`bIpJ+`EvWsMKU(Le$UsUkd?23(ST}W@E+TH68{EfU_wxGa)si-t?{>8YOfJW%| z&Hh>W+UYF9j8G{&X6;CH$L=+ed^pdDy_&;?UPB^qxrT%n*A(1Di;7E1XwBQ@%2m~j zI+t-N67r)Ny|CAFL33pAaLXDRz1j+6-aTsM3%&l`x#78gkQ5;q>@S;@Jufc#Vv&KX zPYgmQ4G$c!P4isf)!2A@{+=hdXbo~#k}pu?3uyTgr5Ic;rpOogfk6X$g7Sqa#3cD* znuM4X$ifHaB}u+O>tDzm1K{Bl3GPD4O@==E1Gg1jtjwMjgU@Vt@HPcaKdU!Os>r-m zn4OB7Y#v5Uift_;-V*XR5@U7y#xEXx=~{Ya6lPPZ2NO(c=GChi5-Y#~CI2gw#a?B5 zcrr-f2^I8iR0Y`KVxp1RGMtH!XTetI89Dm3MfnSU(XHY;SdK}e+S%%U2nP}PkG9kOWWSy(kq~c3gO@D9z zjkrc#p&L>VB#uiYHI~qhJI(mJ1xXiu{@k#@4%tt-2ylP<YlGG%=H7dVSgZ4D*ElcOoMpRL`;kp@H=4(6klzlI z(ODlz7aRu$Q@r^D6RctC_#mLrT*qtc5J4-{CKPus#}G8~>(56Aua8bYLe>NOdFM@) zUvY44!5onhrVzyZl#A%j(&l~_&3p)IaL*>>&&Jf@S;gky_bl}ut&W>)S<4>nK~U+yI`*Pt`*gN z;;(?PvmGDkMYM1Vhd;C?*RKW@ZP4s-5h&?mBx({Z=^khJHwy&pVJH@`l}SZh7u1fQ zV=(+L*$Uf5r||&bQ*s0b^}|~d29XKh4BStyVL{iqYDHdUdg~W!{xD$mw_I`NzYCf( zr9m5`H)_IU$(Br6cY9zpAHj2gc8!S%t})Su@c|G~{JDIQL@Crt?A2Y0L$*NJ^kN82 zXP$8RAN)xT|G&|y2U|B4UDI0O;>j1}iv{&1%(%|#D}G7I7kfyH+R*X45ARRU4o=Px z_H%Locj(Ek93JLcpty1Vk`gwd6T$^svKp=sDXFmP0Yol|D-#BxL9Q6RqEhdwB<4mI z6pj*5soK!g7jzNfYl^px1f6dLU2NQ#!{r9-E2N_elk~@v=YLG%&*k&vU%fp1Hd@NK zi=e&<1L5&QcTcGUZxZF#K^ph|b)`Z4Pm>bJRqtOu>dP?Qt@Fki38bG+z|ze)cvBfm z=~4|ECUL(yyo*+(-gRBw?H?9!(TWU?4tsW2YqV=c@Vai-=k>26?0p z4|f$QHQo9K$i8{e6z}f5bHR*e=rqrpG(_Ie$!g>DCjR)F855!>t8=&tGVy1U{aB3L zzgN{hFI7Wyi1DSbdm9(~=MkiN%Fq;YLT{8VJ~pqb^QyiWF?sfHZ0KKzp)sU&T*w5? zXj*HIb|M$Zr$K zvk-#lh2UB}=y==fh;)mhXq{hDFc#~T&s9YQ%B~}X-q`h~=kw`W=riv51w?*Be!h~r z9fn>)3%AhPRm(#3_QDo$-!0IgG;?il4QcF^a>TR6k+fY;XW&M&h`L*dEL!*7YKleg zEZqAjF%BT8mVhH(91NPP<)1x={Vi5gD%T@ls7aD#EI-l3KF%@uIibe^# zfsN<9?HJ8KfPvs{J&C4t4k4#-Aj@0yaD^(mIcO&}b|wF8Av8Vg zO@8QX5Bq!MIr-h5*<{aN?j;YIO`U{B&vT3Yzd{Fx0ZG2hS$GR- z=gI5ei}ZxsOg)bl&ry7sBE8oGG}~J9sVY`|wv*;D+9b-dPIwzew!OHt7xuZ8l|A1m_k_On>zg}Nmrnr5t z)s|+AE|2z2BVl`{f%G}Vi-Pj^nK;?}h+-VyNlmXathrLEan&uO%|gBlRb|73zv3eH zYfq1anvg~X2EahhSUC$iX4VWSZOpdDQwB*f^_kb3L_u5Kdq`5RpHP{HCsz9f@G|&y zLteq#IJxj!2sjhYaW^!HFlMBtHwt_Z;;v+GJA_NjI=_jr(vE33C3{I@UXBA($aca zIDR&+2Wg0WlKL2RV7n3NXlgeHNIJd0^|62YZRvpW*p2DLW)O#|!fH#A;Vd1yK|NI1 zQmi~8njXGWoscKpt4?sg^rS25iIqr?UsF#=|EhY@P0$h+eiW@+-N^xOjEjS2k+`~S zrX)fy;s~u&#%L_)5-tk}QqUT0;;Y7Hq@lVkehdy9+2%e>=8 z@Ly=5X1btx#@1$Ep<##o%v#&<`c%YqQ&af)X^mmo8U8oc;0XL>#pikTH28ghFfQP^ zXlJY|krGpD>g^xRZWkSK^QIBJ5ln9?V3%MQe>goQ=*#2adE4GdlQYun>b?eg*4V-` z7%<&pc#LU#Yc!RsIDpQMlWWP}qR@5w`0^pNQGi(4M&F1YB#O4nVS!OiBo7$>TJ`Igm}(M4h>7*w{?iOe3GS(XNgugHYHh)!(xcK8-@V4lOm=MkPt zMqvJqUcWy%ibmCTG-pJ(7~<~;rYAf{sHQ>laK;oxeIRDrIanALk?SiQv{Y2lyROmu zDR`vM?hL-$uJqY$L$@#PaV6-!>tEX!_hI`zO~@I0VIgu04zBU?0~Lo{&{)xRQ{@Vm z3RZ{rrP+^@bzZDkO)<7WwoSu@l9^l&ZFKuSU%L8vd8PM~*-u;4lg9=xu?ov66M~o*C=I zGG{O%=@wc(;kGa{tKr!#$fHNL77A6m@cdSn?&^8h=s{K}Od0htUr>`J%FV`uvg|G- zq5`uLHo1apr9R{eFYm=XYVZxR>`5wEUa_xk;sQ80q{@J18C?4G&=sSy)p#WnS7D={ zJX{}@mu2SR8M{p2`lI`h)eC3&nm(z8w^)PTDF)H{IF@dZh77s53>LEO2 zkyp7GLqjSNf!P{^p1}XW-FOu*1pi8FW2izFJB7nuV1?I^(8y@Oz6t{`PGZ8kFyzPFnr^E={drqmW~`P}cr(X90Nor?vJlxa1*w-{ z_s~{)BV^p&Ff&+nOBmcJTI7}9g|(6j&8tntXz8F;yj~iRi5ZZl;5H z*U`zxqi7T8BClv&=He;P`<(~yhX-$uPF^3JMvrmW8`w5^eD)g{Og6M&3sy`4OMInr zVJT_}2_I-(j=g@;2o{Q6(abCAl^V{JjpO#;LtI18f8%qr_?EzA9=U=Zg_q)>hd*Y7 zeW2hkd=9Qz4*@I9?Mz+~FtyW4<*wi(OmV_X`cb*-!sog4MjLYeObJ$#RN3Vax4Ulj zv#Ch%BUncuLKCDg$szN>L(5ySSo%wv9t@LZ)@5cL8QR4XM|FHOvj?ggj(?~+BnUPt zG&<6*P@DmKSO9iq=>?>AIC{y%I~EMab93{MuV^GGxyfF%kuIjJ)?yy(;qa!_Cw2%_ zq^?&RTJRIDa}nrf51{a*p23Z<-`+i_wh4a!nsWIV{6U3HknLIW27s7`GQ`^6NX!L^ zqJ->{$O4HT8KcmnaJ&yQoeHYEQr~vEmyAIVB2Ml0Nw=|K z7=k+o9~#_YhsUp99Bl%fCbfM34NYLG=|^WF7%TeVF&_+}S9#TfL2E^8a!s9wcPptm zhtBe*$qKy;HTDKnf}6-r`iv9|M~BM^t*@Bi^?CI3`}aS;J<1L~e0Y0&cyM<7{v zIpSY^{`ey4-arL(oqF~8`bF}0{TrPAUwuA5|6Ha!7cV|vB+0*`4a0VJlN*xlbmnk3 zi#`0V!&`-oe|?h+Wt@(5DbS_-6f~Z@5$;X`esr%bv)Li^Cdf4xm$?8R^Es{fHDRc) z{NHKSMqgRjb9+v)P~Y6Q+Z!f7yp{BotZs0-VLgu!ki9*{!Dn?z zeej)GhWQZ!Q55;*Bg4cyhHZDg{~-dmyzhSq@)K#+wFK|sm7GV&*pcV5tA(AeGU~ZH zNae7Iw4#iJzDJ^WnP6&@a|xU$SU~{sA??9?wp}dxbg#YpK>i8TH{b^3H}4;7!N(?G z-~i38!H{kJ3sF2xsqXH-=n=Z7Qh5c>eQ4gmZKyI<9F%=`2?vAwMNI$o>y81#U11bO zm}i7})y)v=p6Pd>jpnSDBzM=^@P)}!(zHHj8ic`<*6l64`*L!jE;ig1f&qWMY2^?qEDge_+Xnh^~-obC67Q8JazVM z65d2+GAp@Q=e5!WI_>ZA_7}nq&#-|95%Pyj`a`TK^zA_WdAx8A{b0A&@W;g@(k^t` z2my^&Scu8c!eg5l1vGF7z;_dC;0Zqxj7eRP9(Y?CbB(0L&2o%8ZgPn^i$g=<0JLd4 zEakJKx#j#iPROg-NZBE_q?gb1^EAUhsr*2?p)hO(dk{hotxPTn^4X*;Gw-9EpKjP4nTdKw=`xMfGKS|K| zkE789h^Sp&Zctku z7VzFGVIgdmQe$gN*si4NkyfMSB?50aui$Lh)a!N09)fP}R6n;T`ADk;saSofbpSgO z77iIv1|D2#+-}f9*nl8-J3pof_zey#PQo5+6+jiXcbHdtS6xN?s(wf^G6Nk-SRB~`gnB7RsWP098c zw%e(fi^2msu2T&*X(O1~RW`bF2pLl-ZlzGV?cDj*@1w9SxRM!T{x?uF#giYCJ+V7$ z8_p0D0lBB8w3mi}{D`$&ITMT;jDhKI&{KcVTo#P|aZMw-(-0u|F&FB7FcDpWb5p{U zBrm!w3-=cuaqtvbi^zPa|P^6Y4*Y~_&NgPbE?XC9Dq9zXd#vSXU+t`@cGb)|*~ zwqak8szGpf8pe8ilzyD8CgbcSGn0(PY8eK|gVY)fa6qWIV@P%%^%FLk29utX+mkA# z7G3hf;_Jil2;1;$Dl*LZ>RrMq?fI0WV{q@E*Co6SvoGx2Et={02F`vCcTGcN2K%NB zyY#`CLoJ`W?hqsJH*9OjrmYRsGdFkR6x(EkvG%4>7cULs$Hyk0?^54CD|uiS=F`>y z{uZM8%i#FvbiI7SUhuaPDDU2n@_Y$L%&VD1Jy*qN1S`OaJYjWFwI!XB!?*80y-vyD`*$DS z9-SSZ{G5{4rw7L;DS2~n{5B;Y-v9OJ^yqc=`u$%|(jGv9Uyo0IhR@+t0(2H_RbjKn z*samE#_qJY^RWe8C0p>tSiwwbdchf<>2j?10bE)@2a&O2EStwWD5~FK`6;I|<1zqNSh>+2J3^_TMzhCtCUa{O|GSk1vwHf0i$x z3--nKNPWFKr({Q#+v(?eJM!eReR34G(a6QROwDB?;IUzYjqSBU!>D5m1|s|h13E?O zgj9eqEk`+s)yR>jgggg0aEMher4}KAuK87W`Sl~E zOMYETNG;!Rp-d$HcvZ75AqY%`!%E#Jy(N>7##d5$#E!YgEKCiT18A;W6icVnrXXG%~oB@e0 zP4Seh^II~1kXM8G=Q?HVU#?I8P$zsYeykJMloZepV2ixICBL-gFKb$(?{)ynTV$TH zrRpVw@j45KovyFs)l38Y_AWz~%+3Y{DWvDTt;-y*0Jtk^6=YT$c7L<17>pO5lb*J< zVrzF&+DSfR3oiJwhE&#lWE-w%4L1s`nG7cV1oDlMaZPT&Q{$|uY*dLmHXnU6k2S4n zI*K*A^DP*y%3&NRUenBYeP%;4d-qRbh)?gGb*%n~5iGs0vRKiwtysMrd>JhdcLvJI zSgyBTN$?lxy|dmV-h2N9+=6rccWF{g2A>D0mU5Fw8aPeT+%^CientW)pMWvD<^<|eZfw%}N-aMp%jK@>)Ucw|Ts=h)F zUK><|E~PL=QA+Oa6SpTH?tgXdU%0NTVVIg0ir#SmZ*DlWaSfk4 z02i=*mElR%24=`0FrGIl@h`a}-gef)-x|m-qu;uTvO&omk!Zf5&Es4cu>joAW2+OR z*>dE}I1HanySM-^%-IiE#=j_vK2l0nN;Pu2yIWMeEp0N_g0F3w)ZLO7a#zp=6|^qs zuAoxwUhVICNXSj+CSBNRSJw167^gv;rWXk9)x|(!FL%#oIzft0wj3pojrT^=xIN66 zZX(ieCy;K&GQ!V-x6M1ezUc|{+E}$0alI*;n~l_+xxPL?!#)MGaN+((X!-3ut(a8! zvpjZa{L{Kl9_OFTGtqDC`2QJPQ~9glIPIP`wLk=IaDnNl)Cr>hW;wLz2TMIlr(umv zlgblAgG%GV$nuUKqf}z-(-Q~;PlM1tOSMgbhuF`~N5cA6_LqvFTi9aaTSg9feMN1wIA_=-);UmmBNru!ao~EW4CV_s8H`GQ7R~xEMqE{f(#(7b z4vHTNZycP@7yv=cVma!*1?kZwp7i%y52Yc~$Xi!UZ|^4rI&`ensmT0AOO& zltl0A>Xt}WFO6;unZz*P04+7TXg$k)f=2-|WQI7M?*$WU5a%Ga|K?-^#yP-`SLc7* zy?BuXiX(jYUYIiR?&Nv$$A8^U`s9r9a@*;Q^d0oah}_R-J8%{cz7jZVur~qv0u>jp z?Ym9w+;69Z-ZZq(lr)joOp_sBbB}N9`HUe$*6pAj~+~u|jrT;>lNBY-aSzYq$=mL_-2$budM6=Q9PliJ4 z(!%grEXrHO4+YH?EdzBC@+Al_l}%okM6XTLng9BHBY=zZy}pqJ&!vfv+Q4gQ3ji$>-3#@ zx(|DGXB)X6wm&KczC{dL*h4)$8EBIOnfBkNOj=b88FHwXo!a{dFM9PH56-&Bu!vtxVT+&Bg7z zdFnZL$|!^|w2qrr&!EveubH87+*@j(j+}aD9ASE}i(az-FpbZMtvf*DpJMLrfc*1J z*>;GSg1bn-{|pVw_A1n&&Ls=p7`wIKNiNzZCx!_!bxj5IQF1l)4LHXZmskUuwjm{FQm<7g{cO8*C z3KXqWe4RD9QX7Jnq0uPTOUfc&v+5Rtk9U$kVpRxFE}B-p`dpguL)qf~3vh{peXHD0mReaGhJMT&3 zWGEr*G!JE4xbMl=)Lv-e1WQPpLBz4;c!+YmgSE1|waZ(F8!EDpn+>;iq6oDDTd>c6 z-gtNiNC9?T7Geve^Ds@wGt#i)5`wuFOh~nx*S92S$=iCkun0@8g4YYf8R z;Evf?Xhm47KRjeMo~Umyj(~{XaKO$d9sq*xlR3`?tT1q+>AG?nKkupv<>QF z3Io{y5~>5CyKaNZn0yufohnz(aF)l&Fa}q(9LjZzaVXxP@x-%19-`@p9G`mkO_N zo>Nk?1(?Sb+%x8Zxn7}3!n$tIlEiefoRCW@YFZ%xm0wcAR5BrN1Ydi#^*%tbHLaCQ zNj~STBKyB1?|#x!f&Bav_{6nR!)&TnniAjd?U8psc~Iu9Mh=+L&3007a4+Qx^7BvI zazZ|V5$Zc#l_(Au$L0&#`ByFH-VAW^S+n8(-plXDhr{M{aTPL}iG zolZWejaq8xvOE^{^_Y0F+n-)y-3{^yT8A4)D~hO05TVFbfZtKL1vc5f^qLEGw9*wI zFuv#-sfM+KZO{@oSbA^eaEAMxcui{+VxQWa#nQ*h6VE-}GQdQR`F)(3x}#^x#M^;| zT5!MmEFeJu#8{x0Ueq%Rh0Vx?r*OX!#SKTAPr|s9PN-Cy^foN;rtrvaM%Rg0Z^{Bs z=5>h?lQt|EE+Mlp;fh+R%o@HX-i0x4>9njpinJ>QZOp=nGA!jU!(+-*+%KtB6TeT8 zdI`@Ex_e^5)v;^~YNhLfa|M@`|Iw>j(#k``)E%<&WZAT4Qn=16o7SUnIFBc?KMgx^ zF;P6j954gHWQ%m_QHJ)hU-;aYJoaa|1915e2_C;5-6)~AoWJlnF=mloHXqP^r-5xM z`Zqp5E@$?BPuogQg23z2Er6W{z3)fO3~yW;rgix@J}1YoJ1rKj&R9`E8oPIAak%e^ zy)o?1*qXv^TC@0%Q|QD_^2hjddH3=@nfCtQ)V(sll_^>AR>+jBS=}m_547NQDMJfb zxJkA7AT_(&Jdl0Spn=1_LqC8V(hn$o_WBzSzyV6b(_mv_ki&fpJ`c(myk{5$1kXOs z4-J>spU^7rZ}WP*1A!Uw`QJX*7Y{Meh%s^hAyiEHC!>M!AhF)dKFq~7MB&Mp_?4c9 z&ME!%gqKur{556BgMTS2XzRuiJ+G}SP z{(RW+=odcs9EBl8^1tzU;7sRT1t>?tgEWMwyDaXzy`lN`9FI17Lh~0sPmELL+}NCg z%LC)+bN#uFlJn`#{>8<{Ui`x6zWgn_h--Hw7WT*@@MyPKX(U~FADPT3=C130+>y_4 zT5h_PpvMj)$7%in;s}NC`ZmXzh6=+(ovO%6=4Dj<5F{rwjf7b>d8?XM*_4{#6kIYP zCsY-lC7{djn5{9UkculRGS+mB1##4&BT$&{G@m1ecXmaI)pPRXa5 z@%p@k#Ox;@-|4S-E3x?zoN)TLWlC3mT;Z<0Xa(0XG9$G#o*DLClt2vEzxoL;y*9nN zv#?aytlx%Dj4|>K=aK13Utu>DX6`9maNsI9ggyffsO5R&y&T*iBIuec+OOfi9aLFJ zbgt0_&2#^R1iQ!a+D<4Kya{h7*}&%Owq8ssvsh-i$3}QzPK=Zfm;&Sk6de|{tPs_U ze(VMrut8*wW@Ng`MVYc%i8QOJD(O`;YSJCaBn;6Em-P0V_@YRr1y776LM84#-x z{EA7&YgRAEDfguO30+Mf(gTfq0I;%K)pf;3kYWk5zv(!dH}v6ds`hE zOV25y_A*9cd#<1h`ODU9DQDhogw`(}XY9aJzHi0q`p_r7t!A$vwZHxFr-z2ZeSt$> zwuBQh1H&B>PlpT-ErpR%*0L5QQ}V3gQnGnE-NZI=vgTV=dIQk=$_N~r~#kd%({3F3zKMFYIGO%>!zW?t!eM-vcs77qeM(czSej zc6{=4v{B*(NdgB9;G~ibVKmn6Zt1VW>->p#XSDu11&EU5Rb>W7;;{;=ElgMTN`)mx#0ikjNo#;`AOz(&X$S(fqB6V46HqacsK(QRvMD}&oV-Od}YMd@x{ z0pn@K^D@H&ve;;9uUTQ%q3|RGZjh4sfVl+FLR~-C-%-9g9VK zU|pntblTQh2O;uSD!ML|p2L`+4iwBnRpmtDEeVE{qOIddwk3a3ozRxGp>3P_!b9F` zY!Tg6myKsq3XA=J0RRC1|4>T@2*jDpAT3t_07)$Y08mQ<1QY-W00;m80000000000 z000000000L0001YVRU0?Uu0!$WprgQ8TeIRC~B_2 zz45_^%O>fbv8i1&-u4zn;84;?;)NnvlCrB7`R<1yB|FZaqQISBBoajqA3ihl%y0td zO817>je|F*Z~h9uDOC%EtE=;95>4QW7bq>3(8&@FxCY@&v!cQAO9~&+Sgs_z$xb1m z3GwhLp8hSGK(9JzSr1Y<=q!S3xP^)f1pHc{cEBYRs?~xsSs?7VYy38Ykt~|PpMwdN zJH{n2D3tC&RS)L@a~?4L8t3%v_3PbkmoX1EQ>K0`!bJP}YH@zKyuLhnlb!k{e@KDW zg268xH&{a61JhdYg5?51upKA^tTu=xI7N7OhC43n6s)S;jv0(5P;%=G&pUTS)`*VV zqX|j^lMtU>!{R!IU(c==*J(6?PmA?GSAVSG)7gg)XUp~C@)}kj;C!{bSgaST zX}Kk5T41t_CLnmr9dmx<$)U0+s*GyE^_o`mRx9J+O!m?6p=6E~f?11p{JI}Xf!UH5 zE*d^`j*IbGsDg=&?sD7Z5FJMAn#(#8s)qUameuI4R9&M@0cr^O^E^>Zno`4={So1u?Lh?Mtws$Uz-l6kspFjwB*9C_}H%W<{etgYMezB zWlC=(YjFnF89(q%$p>#zmBZT$#?}Ncm@2V?&4x?vHk-tvsL}v_c0@(yS4`pL@35q; z{8IYSifrRxJ{);(Q+K)Gwi!$+Jp-crvvnpVKi#AUl+TzY#~BAo6Uj-eG#VcY>t16# zh4~!fz-Ii?5@IBtKK@F3M*G3mk~?2uIvGfwEB!g9hxjJ-Jo^D}emKOO{!AZJl8p?I zo97|olGfJKX%t1o(9@vUcn%ZK#}8a)&~-*WKO;HvTCnHCaKG&)?5lJl?)1zDe|DwR zU8hrMXD_kgG382$03nV8@$Zc)&?ni)cMy(pvUgVIrz|SikB}Bw$AtjXNuT#8P%6^r zhSKZF^ontTC7e*o0UDzW{683M)fPc%@R{z91A<%k>~^J+2q!Nudc42?vsS>iikI-E zL(>yZ>zM1Tmoj8ULx#94`IeU*6QWOjeew2GMO<)c9TgP|%m$kbtm;hRC4xdCg9{~J zxv+%?Mk+cigMR_9j^3=VVkEaHU9gzYTi3E&^aJ3vl(|J2KzhDUgsQEtlgtMCyh~EZJ4ffNl^ge_;-!39Xpu>q zg*^Mh;C^ze((t{_hD^Ut=|tay&3h`u$c!bfavTr&KvIS@ z^aq;&EACRz%As_eBH|Ih(TI+aWkWQPjIA#}yjv{K)|VIYe+!sfcwp^+4fJ}wdjEd0 z{Ou)}cUfSql4|2d!>1 z`Zvw+D;Ax@Bz4NF< z1>eOt@PBiNFILOT7d-dr`Pe#WHDT~FntWIK^Q&O@)y@6%r+vsPh`pS|Bqd581My#y zDqcOu1CbPR?+`{6m)tg3hK}7YpMl5bpI_bF56DnheE!fsy_f9@(xj^MsWNOln1AAk zqsqQ*$KtNEeNt>6H9gdB&W~Fz+vT<}Jj6_>;epKwyzhhnMLJDcN`kH`h;=6@LZOq6 z3&2*pK2D)kj&J=@XT$1WJ8?$~NYWpQZr;P~zB{-DgMy=_`$A>VHqtlx72G})V0)Vm zMjs6WGYA#BqVZRb0-@|EC#djyD^_%-wY@BEv%$i;L4%bt2)nU7P<^COataKd3`oZk zV!<`cLBlNMh!U7E$jTm;XXn3TNrC3?d!~~kM#~9^Yp@T=l@Dn>Mv}jzP#1r6m|H4M zj7sm8tIh&*j<^l5TU%S0llwBFKWW+5&4_RQm($bJDNKO=Q#vt|b7z?8L3V9UHyU4B zSo!bHQURsng;>%p>T$ddjTSNZU7(~!(oXNkX#M!^AR=}M&Qa6C2@j-kNeWgp=qnp< zwV}p{l2sPHUrCKfOY>-xL;4&EG5yN+2i@~8zqA>(In$#hBv$j|7Bq#jJxm%d=9i;b z9`E?Fj*hWGE*x8o?`_7*`!w`^T-Nb(SSWdYolR&HK^TU=q_(Lws7O<-QLK6p3PuqR zf<_c6iV>Sa#gh!Xo5@ez?5;bzCSF286j~38BKU`hda;OzZ6JqMw6TaFhy}4qK}w(^ zLcRD4YW-H5W_B5TbImi~@Xoh8JC}LCm4|P>tVEv`@0MaaFBGo^_MIJ~*WL`AKm6-- zC>JJ+N>3!(RCzQRs6X|g@MG$rng8C2d+h9NJQipygty-6!b|PjFf2G8^ZA>h85;5D z=w4)?uXhE&F5I*6FCHnnmc(7z3%%R&KD=fDz~n*zyiK1|5r;}aTp)FSeo&b(jEvQ{a@^0HEqpg2=}Nb)QCiKNb+^SP6ioIhD3KoFb{juw^VR zfG3pl$g&{7@AUwfRL6-?K>g<7zYA?U=!)svw6WaQZTgsLc$jB?R2knce;f`rKXT zZN_kUp8Jo{e8l%+t|~y^ST)u=W!M`n$}F$ld>*ZPjYpqXSuPUXi}T5eZoF&Uh5w7N z_2PVxh7+$?)AZ|V*rJ!=j|T2nP0RFr;D#6C-(AkBp{9J)Cq6^OWA%Th6oL>`@CgWZ I;OPYZ0Okmu$^ZZW diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel.zip b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/build/slurm-gcp-devel.zip deleted file mode 100644 index 5482eb34cdfba1ffb76279af28fa7b7eca38331a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 63549 zcmagFQ>-vd(51U=+qP}nwr$(CZQHhO+jzHaYre^3&c#X2f4c9wZaUq`lU1v#6r_Pc zPyitQ+yCzc``>|?rH!e+v7xbrDV>97i>j{uHXDlXdOb%%;4GLq9q6$jwl|R13X59(hG=qUxq~zYb^E@x6VWq6OsHW#mPu~aJg}hdJ zl7oN0OtK==Q13wr&WVMJ99ka{4a^a0u3p)DYz?y3A(?lqBZEY|{KfH?Nr)VKg(i3@ zFP8Xysslv`$pSL@5g|%PT`Y=>qY$+Z>M~BS$s`1RJmK>mbK$^1>qSAdG&ci2G@SlP zZ|1z2?Y0>WTDGP3oQeIp*<1C=u9x@QC7Cy_?jE(|+~Cq(Y(t9iejp|>sJXXfLQQC! zK`H6fp=(8Z>kzc6+@Z^%wizWRA^Mzlqy0BPjk4hOH_8~l5N$D3WFpT@3jqi6Vu*^k*v!&XiEM$f3x4`P4Kntd6;A$3y$xv}uimJkRA zC47c$Bp4`47ySh0zaj*jq{=2k(!!N^lT-y#m`$w?1{v_OM-vcAtHaEAc74`VQb|Ks z*!|1{TC3vPaUs{AR>Qs|Bmb_y)DJ4oUpLjGP@deF18~X73kp|FC{;40^bX$Krljv) zI_ah($)uPDlS^%Q=|n{>>5&vdE3+;J($@-6I64}8lIDrYgegf!K|0Js!zVN5@jWDN zg7sQ1Z?KHlV_wY_F8I13cg-Z_K-_wlYBxEyHoWlD)5da0y`O+qH5PfGx6idE3+&pm zz?WCHSicLD($!c_Tyn`FQfD~(&=S)ED}2(^ZAj8@J9amcXi7S!S2jlMNg`qG)+Fb zrFNMKzjWG09G-)`qr^$F2?)N-d1s%Z`HSQI7t8`TeL-H)Ii76rmU@aH^m{+N9>lb? z404}f35M&+g)})Y6t1dJq}KP~f$DiMBvFGLntuOU1)s$dsYi#6a%FE{Z z4Dk5qHgX3^8VuG`DwBTcw4knsf(Jnrm&IYM5f+p8rX-oEUw;7z@D zAOQdr{y)ir^dDKwOn+*?OguDONgqh{Wb@3-<7(|Ni-}+Q+8i@&;4L_ zN%vTr90E6OR1t!OCCO|Sk!DhpG0kf~y@X_%O1mbmjS~m{ywCmN`RvTSR64V3Qz%L^ zaMZAkc+qBm^j;ry5aN!M>rx6jTHV?DU+{kE4Cjy(2_g+AxW9=&v7qk&3zXO(1fN3C zkO5Sxgkmr)LRu6lXdAimd&pJ-iUaPf;Wf(8DMijVKgOV>nnDvYgk>QUL>mDxrXz@t zq=F^^U?dGHIT=z2@#avtWsb-oL>~*?@f;J<072p<2-DRSU^8CB7nJqttc@EF9`Wg* zn9>=u9(C4c`0CA%m$Sbs?riU5^MQrVzkTYZjG_GZAXx~I0DuN9pW;B(qZov z6Uk#`2Bv25GDZwRsWGc9RX2qiG~CR9@ zU9zDLw)@_wB%1gf;75Y~u9J2-1nA5n!K0SK>hiZ$jcbrt4cw@t|xs&4> zaZOkYz^A#6;LVQ6teiQl30SxnY;k)!n=q?e>{av;l%{``{Vbr$P@^a|gWskPx+bJU zoYLe=5HiFiDW(;Q8WuzX%7D&@lEt+x0oCn-_T`Hpf&!Q$jRvG+jnjNFM@fkhM-5|F z-nE8~x85x)o~lI+@xLXODz#{hEU}u-e0;?a^>tlsq04zXFK(bttq|>^^qY;D9oiL( zRzt$P$J^s)&VArx)cm)KN9AC{F1KE529N%5sg?~&r`9(ly^fb#!Sp8MPm%|q#NsBJ zqrE6dnMTC7TuD+ad&GSSjb&tZF8zCXgB%cdJ!;SxX!kGLz?TCYr2+fYW3kwQ zoGB9_mGoncbOAf^f?ZxXlZ|zu#Ex*>Ai-;!5`j0vg)DdyYGL^VqcI?5y8Kr{tAN^k zPCZ9Z2TFg2azw=(n~of?wATxS316i;+p1{WiLz2&ySy@-#;s68JC;N%(#i=V=g<_;RAG6kIc&-C7E>c z-MxWcp~|vfSy_9uEVS_~mUITUc!~!UV_>$?i!x-lxaX-B3TgT}F@<;R!QhgXNNzcy zvA47PdKdW(ayA;F(z{@*>mTgq5BC32=>JeFs&Yfs?LTTsLH++x%gWwJ-`Um3*3yN} z#?_F{#lfaTUE6+p49Rc39s_|RB8B*3gma8D0lBhZu_URJthqOb$9bNYI; zNui~`H$zu1ttbV$3LR-DGuPg%uhYWS(f#MvFSVzyFBqeU-=)gK)+NdD#hj(HQ+6-E zKM+GszmLoJrT?$r$CXQy#vquCu~inYT1Cq+?FH^W5?jJFEZP0c{cZAP_Unm#y`C^^ z6p{u_f!VajUl9n^$pUa^p2p`)-DFg$;c z?vBp>mfPFpq+eLQ0}PB*Fvat4v$KWPLkoaefoK_QMlC6ZvLPsl_ut0*@!@GYd^^5f z(sXyXw$$i96gY9NqfF>HA>&j4u=tu|Op29O6fE-Oov4*VV$&ejACh&+Fs@wg6iC55 zSpWbJxSLHU1jfU-i}ei5Xii9!Oz%^rKm*h{NLbJUGo1@ddpM>|-AU&pty9u%63o+? zG_Zdda$#Y>M=s89y{wyj+L(HB=gN0|E7K{fV^le1s%+q+ZF^TrWi=r|ku@R_ftImT zGMzeWO=5?k`IY9)3R+sq+RM`3H_V|l*?it}T8F33T2isgni2E*4d97PDlXHJld{%{ONiozn{F2e{+BgGULrjBbTHOD}dx=UHjvZFFuGQ}58j9(RgzZ6A+ zYu}||wmxyj<=$(T-=ANb_J4V~tn(IVWZMm}C+t>Ilw4Wn#G$oo6=m@?Ilk0~!9knR z)mP|{j%wSkqws12INzE^o@isaAOyd0{~rFy;$g))GND6vpq|8OWu>8J1x3d~O9iT^ ztcY-^KKyit_l3{s#G104=+vnNXV9ZL=p(4?3jFsW&KlalGTan^8^}XhOn{(lF9|Qcp3`pN z2RYvyInhAN^Kru!DJtPPI5I_nfd0;u?2)S#Ws9x%vMqOkzAW8KS-4~Fu6{qKhqq1v z_7O)LJD*@BhF;tl4LWdE=6C3z!1YN`k+eY;0A+1A5H^d89Vb{WgtiyuQiGRRqfA`? zYUtVCyQ;^@&fUMR!Y}#XG9s+pST8vr5_hH-3aTnDJ|Jg#y_G9b%5*gbBpR^J6qet` zpaQo(Y6X>Omv?bDiW@)K7{i$Vst+|?dyS>Fht(v#XgwkWMC7|X82z7U#ZvLhN7ZJ7-9bT8J_dqLN}KK58aH*WxRInC^B0(Xl+C3O0T5E5XP=c zyUvghJU$qZO7+5CjNO{=4;84|^4v2syVSsQ-%Wg|zg#*VMS+pH4K11#DVdjDG zy^`CG40eyZ5QzIa$12>z5<1C{_53zyIs`K|j@E5AyGDxaq@-5TW(vx0prjh=Wav7` zGYfRMAzMsOFWHyc(KyOJ?G_OOb!nZ0sY7(@8{$qmW`Q4mQN!inh&tro*l!}cJP@3% zoj1hTvKg^}PxKF>tP*}5dVmgakW)YNJbXdi+h~Vf~kMQyGHyw>)WtlK)UNYvL=IHnt0V7O?)?7WN z<2dYCOO5`G_STeNmrs)Mm`_rz4s`1lT6-E7&i{#L7@1@JWG=$z=3sSN-9PNEb&hms zb=A`aDRftPeM~wc!cf5v8|EJ?XHZT}u>qIj zrl1MZj)ulWh=rLTZ2$$<45~v1c7J2z z*oh%Zet-D*8vY^QesB2t`*Z#I`e+3B^ya|V@8|95&DpOv@Xei}`v=3Xr`L}Q2-PkI z{H}sER1XWGMC`P*3FHVbPzFq(ZA8VC7-T|KuMwAUCC(yMfg{Bd+F&9BE%R^=QbG}8 zJXgE=t)bfC@252dX7q<6qlAgfSyX2$iUXFjY9I5he2a^8q`;zG|gd z8|H4p!i^cPTuGgGya$`Cf_~JbwQ1#wFM0>04n0GlQ7)@sQkBZACBnstqCveTO zF~t)leyxnbBMs|SFnnO0nWkhf=`-op<&;c;)qm?o!#hvbUXX*D$p|1a%{0D z((nH}j@i?fyRVbiAL$4D{_}Lk?D2DJ)R^Jx-`L|Ljgmi4&=_|jbb`o2@`85PEbb=i z99l~i3R;UGqNp zY5`oF>Spax;mDf+KOhY3y+eyl((7$+BwA}97f&>*Xu=8sPN{65aNASNts8*?7V0O0 z5iB?gg%d!x=m8PwUNmzKv{OeS98t6~HYjsb^l5$)V&nH`)CzT+!KYj$y(DO(XeP|z6|6&I zH`monr015rl^+npB~|g)d512{xV(iE(Wj+dPZ$FKDqTb)*HW1Q9saEnQj4HeU-)nK z?_*3_t3ca&uZS!Ze5bOdSgX$WSTAE)OR#+YSfbRI46Lz}8}4w4X#?&61`D%QPPIuXp^xcS*XjSG54NW%Ca0_dNA;>?&%xtk4xa4^+qqu|w8 z{2xDE>@TRYIr97%9J2oi`13=HSv-U&X)Xn6kiwecU`vX`8$x1Rx^!WPHeGLWOi!_V zkAHHRFGsj~{iAhl!*W~ud6L_}iapP}(p@;^^CPAU`B_C;&^kE3vz32=YZ>>vW(J!} zq$?S5lgItPvy*6}kD&bCcf*f=yN{zr4A%iFnf!YrC^?WnAIdy9d$PYy{^u;42gS|# zb9`G=Uo0EqT58mk*Ii;;U|Wc(E9VB6PYOlPY_|gtTJTksqV<#9Nkd6pR^xG?`A~aR zW?Fyvu6FCT*CcK<>ztH*Q!`!SNM1FfdHFxD6&_kYv8bA#M%<+z->z;qr84C<*sf6t*%GSte{a zZc5&y<<`Bb($v(-BZ1TR3Dy-PjO}qtb`cArtxc7Pq^N21U{yDWzh@6yXJ)_)!KXt- z#c7LjL8N$ic;EhMc6&reN)$Q z;*@xc?hUC=KWx(M6Nce)`KAe5XPC9EwJRun#!E(bHA!2ZS6RcCa|Y31+ebj!^T?>I z#H`(&hZaJ11xaU~U@~pS;y1rr7Nw;VqEjRmTPK;TEO(2AM6!LgLg5P)C4mw0r3@R5 zHbz+w)Glzdv1vrRgDc^TE_K`lNEebJQ{b-3azOWrM+nqb#Oqy2I5^u1r>uSIrdHsk zXlm5Du1hRltlh6O@ec1f125O#;@8*f&(>h`68so3XkBd8&>?N#-n9^wOan=F2iNKQfub!9-Ya3j|BfDaXTN`Lu0P#?q76W6kLReXi_BKd zv6{rEJyB}z#-*&1sDZUlO$AMwX(b{2Rl-lZK`I{)rQjrG##-s-jDT@W2e{<-TT2KWDr<4w45W zR($f$!mNRUzcH6CQffu_wAVKoCfQv=tfR`Zv>x5~T%tggOb9*6o$g7m*FJ=Y6L|xY zrPx;1_q>fRC~EIjF~Uf;qFsS~M?jU%A;(oy#Wx2I3h^f~;Di3HTnXoFx?-O%vQ?&_ zSRvn~8Z1`p9xfUCW9Oo-g{nESy~jafvK-6bh7Ig}RG-Z%x0|q5l&~X$@P$hSMT?RD z^>tzJm(`p?4z1!5yfdIsxa5ZMBI}iUXwVc9CnYFx&lBxe+bT4an^0j;xke!86Ig1wy1a3U-6AC-WBj7PCA>u>I_E-fFcVmP_2=vOHFl z3)0;FbmK>mgat}282KFCx5a%Rame9y0FgDHi+r2E&ochpLtHncZbyQ-HH$}jWd~<{ zLbB_zRK4X)wN6ud-a;FGF^!lyYDzCwtX8m$>U=DH+Pevc$q(RJkn0+p}R|8}*Swt$mLv>W22k*bK28zK$h zFW9xNytvhmT__Vx9n+hdyC6`@-zi)?xFJ|MG%X?tg)&;S$|GoV5S)rIiO;zma=+sd4usXl6 zN0|yC!rXlPumMWUK~6aZteL_z4C@B!kmBePYMVUSJzDFb)KK+10?`%%Tl~-i{U8Ke zRala(02OqrU<0)s!5I)Bf2oG%0g{rcS`c$4QB9{VCmU=c2gcx6ScsO^7%78G&ye%<<22B)EMX>|IKJq!K3`FQzL^>VQKLm~HLIEn>~#dq~77PE$3 zfNJUir8X4E;93DogAlQq0qSJ;+K!W^%s|CXUFMhp2qwGQlc{Zc%&OG*>>3zppd>_m zoq-$bP(QxzY#rGk#NKS3fA{+l)*h~|9^P!ddAYiSBjh;xQ}X7^wUHkvdi|lrFW#>+ z8-FH%Q@0T8d*u>I|I)x@Oh)2t9YQ-@K3nki4~ z%_8#W9L!ApPCKIF80EbXRqU_87>$;mk$1@tFW@J?uwmwY1{ER0OH#Wxqj_t?NMou< zMbSZ%#WThND^Md8P$87Wmab#qs}3sYr3DTM`CT1aIr>@j^m6s`bi4R*{^%rr`Yklc z{W4)p!EsZYMG(O-4w`0dE@iRKL3yYX8q>1aHl&ZjH0XR;?uA`POFJ@x`|64&J)inz zUEP?&J+*^nIbf)lnze+11Owe>d1#&K_}tA=r?kL)uu!Ou$EZfwMV$}^l7lx*nIKS(_XBianEu2Fyps%lW3^l8}@pHJK2Zj zcLZBH8x~%4ZIcWQOqrA!8CJaaoDeEwdS}$of5n21Q_?);bKxnH$_5b!)}gu#l6AJG z#4wdn9WmkrfFh%qMF6X=Z4}; z<1rJ8Z1p`x4GihRNU4#r%bYpU!{|3~`l(>_&6nu?GWtCkeI3`+%Ms_cf(*FkR!#v@ zxLa;y-jLj8CY?3Wl<>K_k{Hht^3oBJQKzhtqTfxeP%iz^6lcIxY;s?JD7D829)<2j9)36JPxk9RGzrGHbvBL zVkz^gn}qK+1<$hfUNdbc9ydYb%Uy`)Efa5*&9l*7#t+@SZQ`f&p7O;NQaYVkdG*#F zXFX=+^#=Qx)%N|hpi?O>#7xGD+V~5A0u@Z9S%-dmID6oG{YAE)q3E-8qIT~cWyJH~ z$%0sZ>d0s*$UHkV8HiTA;d%!Q1rU^y>wp}6%gP&wP`0_XF-p8*5}EYHv@G)kg$^+g zaih<>{Njgqvrl>w=6P*n4*Hay^WfQ0xh(WX`4E+rv7%S9H$(}Z?EcwD@K&CVHxzuh zTkE{ecM-#T|42b$W&n)V*1y^78O^KuF_n-e>*&VzH5n}Fh%cGyF0Ww{YK3eI9B0Y0 z=p2<)HGGCv$XvTV`79C=q{zRc^fan%l91XWGCN&D>z6E+X#AfBgCy+sRCXa9V@*gi zM$1%dd>Wf46QAv*1{rjV`gd~~xb0RrZD&${cfw{;C;-r8NxQE7TNx{2E&0a)7l1~|mze$1X3Ku3qP{*J8K`8j#?KVL?ESi>E=8fvRcJrN0mX^3^|TZG%h;a>D4E74lU!24q(#{Q09TQ+`TC5E7@=;Y}0 zmyZko*m4GSG#OpQ+ff)c3v!gWEFpH(2oR|2e-RU3urv*hhzI0)ah|ArDxynNa1&ceaY-!&eX?A+H)g;?Lr;cM7Bwt zvrFJ@*FfFTyJH?1wz9C?AIm|#x?gPAV}I*~j4q$WV%8NpcXRYCPS`Di%!$`5LFHw+ zh&hF%9lN4PEduQ~zrxgtPF}+{NeN5mP~fiS!}p?ywquFvfm@4_kXrm^_LYxLRvYP3^;N zt2vOD(Wxl=4&8rpDY!WC_1r|8G;wEOQo35i^ehpFi$}2BG51K@x4ooK!rZ_B+DW3o z!H#*$^`3aK+``6$l6$e>Yj%?#UmP|@>eY7!vY-$DS=K(|LOzdK)acICtG}iXi1oQ_BlF#<61e9&#ELk&>xgrf{wC^Nb!FFiVZ8EEz~se`WMSva6Gwvkhj{ z&L;S+$L2etzqn)FHq6mZFwdG!7EE&S-MDb=`ro|#b>D&gZV5UVhu2QhUSG+_-c=q7 zuoRs#=MB+>vjQpET0w;-VZN=458S|mf%FSEKESWXP(>nS1S$s9wRCZbmzE9pQ=dZR zb!)l`G6xxx!pS7Y^V zole2Z68*L!C8uf0MKz4!>r870cPAAk3rzyi?b7bD(krf_5#<|Li{?z&+4h2;Adj~K zp-<0HxVN7m*wjJ3e94)htSmVRTA5FE2Z|NTt-FN+iZ(CF*yH#aXst3u2N!skFN2AH z8I}LYCL)LK7+DUPS)3z#))knr=_W=UpX5}$;{N0xG(8yMjLUj;r`s>i|BkbVQ4G9A zZ1*)lWHZ zII^a#-TlbnTU2&^`1j7kzp*|2UOPC4`pDb|7rrgKZns5pKW=ZVmbe6LG%UGiePq^%L_3{f%BT1dISUZ*D_>$=H3&Tih-Jx<7G1dgFkWp{^he zWF<>#!csE8M@q6ZfJx$`jEd5XGSItftqfaSr1{-lJg6R}42GekXA5Xvm}6jyp@!eO zw5s~!`}db`ENn9)+DK1JfYblwaWgvmBziOXeW}35fHV;a@3or5A-^_e|Udik0&3f z&}tS0K|WRC#~Dxs8u4VyOj?Kw#zd#7gc?aHr)=R!O@*Ui!GXI7!xU6S#D+p8xiXO1 z<~fj5S0NPj)3JfVgM|n?DmqjH1r-g+L%*~hN($;PdHQd*&U#eZY`vE%WFFFH8+FaM zyC^&CAnjgCvCciLibkz;7ts1WT6HVR#Dz&${}{5fD4774;YI8ta|n``)^ z?EarR_)Qx|Z(yr#xK7TTvO;-ujsl=FgExNh5dd+N%Ww1PgXM;9oA3^tjDwQ_^OcVm zfKr4C`4K;{Y2hDEDJU-=xBl<7lJ7J1Brf&KvxFevl>e@LvIvOB^M|9fkPyUI>MRl4 zA}+6|-{0f;_wV%cbTiuIBmL@azE3)Bey`W(i_g>h=a*X}qk`-fjD`VR1u@aNQIai# zNi-v_%eYglSW(OgHvlr#remsTGBuGz2*TW5WxN5#1-OFMjWG_+dRYr%NTrjN5P{Z8 zQ1ehY&KsB6wj@qvp;$cLz%!zii0k>egJgO4oWMDBd0Xf{%aoDF>v>{n`G?!fFy_g0 zdoewLuPJ^e=<1w7$hl~#b4(;zL;RLR@i;k(r#2FB70`&r3hhKOtcjV$DA_F>OpU%1a&9OoMoS@(lMdMpu^>HSur;;WX?B&NY4cR z5eLSLiRplaJyUgfP__6ss6vYYx)gkkr3Kig+@#$d$J4E4W}IRzD@fHO_{0J!nw++Pq9*&nL*>>9F;zX472v1Kt>JdB4;IlFk5j@NmO)gE>xlaXfdOZ#nxd3KOeVP zH5g|Z>EvhhX-;Q*25>YfyjpZ(-c9MbQL0pnI?zjHnoQ0jBiqa#|;)Np1=AWLr zjM25UE~j@2M?^(WZ5_>^x|li{!xyu{X-np*!k(IV>-c$RpoDWWq-aIx)eD7hx&@TJ zuHIH+!qOXp6*~=Dz_U{Ys$h8+vqJuQ`UFS9&6t zPe!yMZYRJK%k>(>Fb{9LIrz>qH8-x0L3_+T^TKfdlG0k3kF2I+%zOOWB1dTO=tJo( z*x9!_J)hkrSC>&AwBLUJzBc6A&E&4^(?ZOUDKQ;vA2-d@?` z8G`$3ed6q1s!=ZNekc8xK%2o#WQ{cM{uVMuB?eOR+Glsavcx(`udYwDv}75v-akt7|3UGF`fTfQYjB{bgn6}9sW(P_?1!u6?3uU{p*=tRSX0oAF0rF_j|G- zrj}p02~ze08Lf2U6F}SbkJ*P11Ra1hgj!yi^z##i_^#yURe)eM-G|7MmZdIinUDc) zVRA!oTNhS;yYl@%WmBk~GxtozZk?mzI^S)9w~$h@0cIIy z3juI1=DL}X;K&j8F;wa#6O}zL`?7v=EoT`Y*7=ED9saEPHs-8Z=;k}dZPqq*?|nOr z!}C$h&SFZ&l2^Wor608%re36&Y&)pQ#z7W?PMMXqr61x)NnTMpw)SV9qSy+M!P2-B zgY|}9apdy)MwK8nx!e{*X^+|}Emz45-n7h++^Yi!zS+Tmsah42;29TXv?SXz>Vm$` zS<)Gl>z#sEhA+x%t7Ea6dn!Vi$l63N2CNNGrR{bX9U7Y)Ezt~UF<3$tP?7}^YDGK@ zzTJ|_@t}j6l9&lM3qoK&lTXEHzZP%JYNF}EnPYJps*gjG%<01hDfrBTW%O}$#(!0HF#$4lPQ#hT^K;Sz*~YPF(Ymec zIDCCRk1J0RUCf1(aHm4FzRe7|n8Mjvs3S5#%#N}vwpv+{4;LVYTlO1)WF2bO^h_jvJKgOA`(=nhHl{r7}|RR-D_~+ zdaBrdSQhHZ4idmsRs6k?Z4Gu3pT=*DM=9#s$QoP`m1q*=uRN3)3gkEVL&cnLR#lh97nonM*p*#}P5w z$@kDRgNFJJ47RzQ0rOe$^SOz_YJRCH_-Rj@(OUE+?X65iB{Gf))-}SF+hicIY$Fjk!c* zu3Ncvt;!SLdehvN1Fo8n-r86{1d$7rPhiYC8%K0)_}GS%i9fDC zm#aW_dp*5+bp8H*rll=KHhvY5YB-G~LfobE;^P-L^#g17l~meLdeJ!|GO)wl+SoN+ zX&lW~M>7%-_-ppL6yO0hwVlW5^7*eR{2QcBt04T0z+Vz>NpyV-#5#tZc(HWXYbY~j zsRcR;t-2}Sd>%FGyZ;{kx^ZHEU%5Iv{eFxd!YOJ+D_LpjW?Kb?7{1^V?DKrJ|8djR?GUoFVE!D<&f2f-_j7PB;(x2r|9hiXNivu)g-a0*i^FkC;uQ9hk&PLJ>Q5Jj+ z+f1FX12=oq81QYzHvjYg{Ffq{?h_B^Pg@)IfU}q|&b$;; z=8Iflo^g53LKGbdWBNyWKDS^fwzVV&*?~EMPj$g&U)!F{T z?yPS(MLmJo6SPSnHVh`bvo5VUFmuB2jB5wptcA{K>z*5lQy^Ow!%sNOnBH&HR;RRq zB0{P{Cd(5(3UrrYMcJ*e0#gJ#3Jkct2Ad|M7kJslwA?yfQrmQ4Y5{~_4O`AqaOyE` zMz*IdRyl3F@A1}bLkNyGM-b_e+%^|}QUq>CJ`*}p3D*qa|MS zOTr z#O;D?8+Zz&ex_@o%9s^+EG+-QKgJ!)onOpf$ie^xctuIDrIg{>k26)sploK1+`Pfr zX=^EH@Q_j8YF5D`x^rcy?d~$)e2+{MF}6g}XGG z?_zB&@A!hIs3|9iO6G(;2-@N4A4>nOXv`4B;Mmf#-6*w0WRb(fmKs88(L4aXXJAf= zlpe=~WVs#D^cYtriE$gFDD6mbY1>KQNnws6Qw+8n!IiU{6@-s+kxWgZ(MVt%}Z2X<=|8R5aOY za9~tS?PZ5C(b|#cP%o8PJIX5L5h>-9NP^V7B|;?UojVm82j0|&H-ZLbP91v)E{=6D zwD+-~Ge8=!!|qnaB0@kKfI?+#BQH)NE-sD1*-xxHLKaY$6$h71NbjKj>cxLzu|X7B zpIF)j#Vi|XritVv8`eA$)QQ?z^-)z>xw)Musih>O(gI0)XW}<=1dDBIaSmx?-Ww^d z-Q%d3HTlW%eS^gB=bA}_E<`G*T8mBMm$Rb;zfq+IuT3Qb5aQ5t4ZU8-6 zz`bzug?Gc&4N~3RoYv#?)q~4Hc9z;>ts4fVm2mSg0eoF~@@giefdo~=m3dxo;@6Qn3?w8+ye(|h`3y)CFagj^RpRI$idOcPg&LP76iT^=Th{@-;Q0FLoQj* zrzuQ-v$_BJpoF%Q*vNXW7V}G=cL0C-L@HV> zU9+YVR>_w3mYMgiOhZZfz_WO?4u`DU@_w?nXc*C|OIVwDyl|c!g>c9-3*883HDWH1 z1DL9vCicZ&Qd#2$Z`jJpdN*`4PfBP%D|ub4xsApax<$ErVrCNQadJ=H_Rjsl%)E5vd2+5&fQy3dq?@3-X6nyGChuJym)dEJ* zkdpfg>Ga*|uAqq*wK_SPc?bpgx_ml}BKN#xQC69)!LM9jW`mt|vOyHSr97PRffs43pL>|PZu$KqI>p&vm85(w5IJ*z{!0)i@ zA?K=n4){-I+n?w0%hx*;&@e~icXsQVgxoB+#-DxAHaoZT{D<)OX!=*3jMTjL^$)n( zkmunqZIi(t&et~W7^GT@MO-sm8pV&FRY;=~7~5%Yba5;wEu_-mYN*pfc>1n#&*6o( zUegRto_`=fZZ)f=V*L_M$-TL=Zo)`MHN-SAeGSOhIuX5_u8|i@UT#64*`Hb>!`2>YYDb>gNDuCupSy`#wn$lUk zWyaEbbP#X+Al1b+siSkV40}(-$ufMYnhM0Y)G16^^zJ$5M@6}4K$ngI#jGcZSqGu) zn-iZ!K^`JKBEJ+5SfA#gNgu7X;fMHUcz3=#&*t2umzUN+Dh%mtqy#~6uLK%D1|atv zL=v)>cnHGjc+p?r7MAsV96td(@iW|YjsG`*LVB8u`1G<$gx|{i-Zo`Ouh)nlfAMYmkA;WQ4lbdcIjFfN64d&P z_`5+y#JY)WyxNG)5W{>c%S^AyTg$%j5ivYh-gi@8-gi-*-)IvjkI-rzy0Uq@Hq{5p z%-TBbrd0PruQ;)kLeb46GPe;O0i&t#3=~r5sey3?vd5s~pVQL02ln^7bC6v7T( zJ-=&i&9sbnHVp5KfOGrI(^Xv>ub&g;(V%n~H1h%qwZ+VEB4yIclux2_IcGEM{AJRJ z+Yu|q`$KeuEnu5|2QQ9QoZiDwo*4EiM3J!Abz^mPe2qJ1Bfpyrsbl5>SUYR{y)3RT zde@5Wsa`o-j`BpL)cTDz+c)E4QUX;X$qCK!=Ub~TsI~1&D-WJRrwNp$-aE&B!rP!l zy7%&JkQ#Wx=_ux<8A5(C5BXLy9V$ABw;2SCZZ-(}x3|rn3Hk}0>{DT1?WBEVR}k)| z{LMnEz;ntCSJ1}vebnIa8@k~Eobl{Lc~l8k1H;rI$X0gHlcvIrnFP4SQnoWS&bGXB z^ap_kll&LB(9AHjgrnw`@XA-Ys5 zv!f%;Y-DqhDo`$o)b)}Yv~%EU(#bL|D_ukcli@pwQ(N;Tv%{4>Y>;Le#vie^x5A>% zRtC3~Gqj0%b)E(`M;xuYxo$Yk@5*ojcGjB5+pAz?ot>Q-o#UvneJpQ#=VPY^!e$CW zA^<#qEj8WWn+otjd5NyQsvg}Wj|WLX_u~Gpg~e~Tvo2c|SN)qA_2>t2)c6xC`tR4< zkA{MtsV`XUx0tAlQNO*+-WnX&I3&77!Foq zF;KJUyF#-~^o<;hzhG%6ibEm|0ZG7sJ?J9;{Aoivtcnf15S9f?(54eCC_Umm=kOK) zdcx#?6Di1;L&)sf-ZA7tG{39K3D(0M0AQIlZv@eVI1I*^W&Ya6mnR<|%gf3`G|iDJ z*-uM=mrvEx@%i=k{Q0^h(9w^wDHW^~-`>Nhrwx0+)mKU8Ho8P9*N`PZZHf_`7?R(7 zly)5(BL$|0c^N~SI8Jr3|E|VjQdXVDUmyga41j{*+Z{Zy2lDxL^!CP@Ilp~8-|X4< zZx7Dk_51xg`ntb;pxN~gk7i%Wr@N=Kue3Wpn0-2aslUHYUdk~NvT3OX&a@E!lmN+D zjpVqxMR6DvB*qi!7>_{b3rvib8(@dT_==EpYzo9w9c9N>21NL>g(s=)G8(DDg-Dw< z3r#@H^pV->ttw%39Bs{*|3oiellR{6VVQ5thpBD&)18_Rjh6jvvGUwQp4%9ET+UW! zY^3|_;J+g?Yquv#Ca_8sGD<9DiWS-!WXjjZ7YuF(Fs5>_cpo%wX#(}7~^vbV%wWeZH3%~k8lB}KY zI=v$fuHglU_N|NmrCo%d!~l^&1j$5+lMp-DQncV8KDt+z+l7wIXk>2Lmo(wt1`{-x zpk#?bdMt{Csxk#gqgZM38DWS@SJcp}p@_#FU(HM=+AWlDqSa8w4lf&aR;R0NPo}Qc zp&FB9BnO}?++LDUvq{deg5HKC0b@IO{xKYX^liVo4BHj{lq>QUNqP1*kxLq;&`7J? z2DEI$6~69 zn*U7^$js&MY89YMv%_eUGt8CZFf>YK3@0_HEU!5x zA;tyW@5DfLN878}Qu?>H;lnRvA)#K?uAC_cL_giBr=jvZ+ES?$h)Pt@C3+U{AU(^% zr%6RBl45ffJ+KGbM`^=c1D{kt@)Ki3h6SuspIMMp-9iFN)d0pr>?$9UmsaseAx)QV z2;N@P+pfCmTJAd4e5vsS2_Bb+u}KgY2}I_s7~qBP&XWY;MwvMowZK z>fwbDcFUDx^Cn4>-QX@7EQ?FnwLApQ0hizd0BdrcH+79|^-{m$p4b zkkKF~+}nB(H=IWU8>p|~PZ>?z$9(wMb_GiM^~KZrC68^F~YM>C$h^Ou>^hDhh{ zYb%Q2R=8Pq)La-U0*fuz|U{&S}*RYv7qy#jf#J@AeE{S-f2~k@RW_BK#irlcF0y zbaXD83ZRDBjLtAX@sM~TBpuKQL(5-mdS^54qop-Kbl2q)*I$3P#>grhpV94dbC9&G z>vZMy_-C4VR+6{EA+chOF2ZIEt@p2v(9!LfB=PZUAb-R@W8i{gf>A?$B!8_xrb@9% z9F!*fBv9ia3ne0Ei{T{`VniKb;WWpZN9lHLgtjFlCf1g7z;zOHIRP3FtU57mm{7O^ z7Shes(eszp6p(YF=Gyiwdz=!{)ue8hR@7L5h|S}VwK*r=C1ZOW+gwBRG1Ue{`Af6K zWsLEV8wfR-Z;PTR>6AD0-7&yJ8}p6$)Ch4Qx7a1i+=E7N&LW9m4DNbU_W-vzn6)K1 zkpy$?JP%$e1|q7DH1&z@4>@-OW92h?)Z#S4>u6D7|M66)3`}_~gID%k!J(_QD-v*W zLIt-T+7;jWxFUm|QBQDc$`!Q_o+$@4&^{JWoW&>>gyne1X*plL+^OWby*}TE_}k$T z;8EAp?e*|}x|Lg_KMwi@&i8r%AXlWst*;4N0&khS;yMRXS$LoO<*QwPoP-&qYQs|g zvx9y0m zO2?9_!}4_q(*og#m*GkOlp)$YgHQ`jI5E|A*dfspk2PdbiHyzDdUTCgr-P+h#-W2% zYBahw#}gg>Yr_a=M)a?SeF!(3#X>LCG~(c3!2WTIqTl4qgvY|Iqp;C5$XN;hSCn%M4-U zRFoTh0A<{05 z2>t9a6f^n^V% zY=m#w@i`amrZSx7#v_P+V%W8zz7!d!$b8}#J;e$ps9S>F&#HZu-Vc6-M6xFN)nd3C z*ydANs_*Z&dw6}nw~f0mBgQJRYkIUN-XeP8Z$M-$?C(7Lm)|rzVDyZrRSh86)Df~< zuJoz z%8(Dk`j=ic+0ep!q0+BMYX&y-kS9A!R}3Nqrzyf?C}v_*abDl0rcxLB%%+*9mR|R? z9`+XIlg@c;Tm`k((lzJ&t*LIWr9`NPhI#O1O!P0t(a3)_@o;dTfMQ%lc;DX+Z}q9U z?wk>LT6E)nt6twJm&BW~0xam5f$K~lI)lic*KKZH_Vo@Cik@2oP= zgvwi0<8tNQREvQL3q^0&u!{hS6Vmjt*R!?@%N9SXOqIL6T6Q=xhZ?!LryIGm&-XN2 zu7)5Up0HWD5k6=pSSU@~D3{->l>0q}ObcTmCi&Mwq<*^ODW;RhcJAFRQe2M`ya*?= zt*N7mJNZ8a@=A#(2}asNdF|yk-e>*BU?U>*`0vw zo%ag{3BMiq(EOCve(J#COMOT^Kz~7LLfj7chh+l%dDwCDFviqJ2Wy95`^KUpI;pEQ zK2Nuc9>0bQJcSxkE&77wt^_*P%$mr{H{5?CvH|-KH<=&Xt46B-7b5#1VEuBkc5$?E z^058$9~8!O?Y7_Mu=A)sHQ$}KIo3pf8m7jpyWiODW$S7)%75iK>wp;vN?Akc4s@3B5?CIh07^aH|dW3EY&Kwl)+OvUfGHv$0EF_1yKYH`^smQU0dC~VyM5yu?#WDj4&QWT&E;5m_-Y)cp zc(1?!#TOmVzn0AXC>i}@)ZjN-Rj6jyuL+W+K>sW>;D!Pc0X$?SNeT)QYqexi8D!vl zAvwetAS5`~T{DnCSz{fAa6-dnV2z`hAPu7aGdn61){ULq*<-`#q0s2#_u?Fir=489 z>@3~vi%&Y9K2r835}{w@e$~)&N@+9%H5CmK|0E^zqL2GHhB(Pdz{XL!o%F^6kAx9u z67SOz`{9a0{&uA;H4xKC>k@JyV}zsyy19dus^fY4xx09>;#~B}m#*?>?dS~3hoz>5 znXQ(RtK0iRiXZ=INLEXa&D8C`(BNi$L-})ZaNz?`9Hlt^jz&_yVTK7s#1~1Kss}qm zjHQuDsE|-L5Ct%mc;Fs`Iz9<-(3-=c?llUPO`{+zI@f`iFj5=SI2Z?&+cQv@qV6LZ z<)|~F5Am5nCKvm>P+&CYN~auyOGDer6^p zjd&UnLZ(J156sAvI0!y=Ql%_5q^+@|*JrWK?H5Id5DW;P3{xL3%Au& zCcy0Z5FQ}RP?Ki1;S7M8C6HF(3=ry`ENjXp!YPv07K)tyA#T$`E6%CnTle@9u(%~j z8D}AiBPl1v{D+WVaAa&F!!I&nUr?38N-osv5e1(rg7D?vpys6}wWEtL~f7l+uxLhMjUsJO1QO7Sg&5tWArWe8vInhXDx60{YE-mzq9);s z-nZ>$KSD9zxU7#68f0pLBKwCihDZmuO7=~-u+?{DeFp+p|2_j0n?<*dbB{gwdz?g* z)}Qgp`h+4ht}@8fa{R6?5A&YcP7}#euQqS}&!IFY;$yUbZ+o_JyDvgy zu;RF8dtirB%`eE3+vsS(BSDf@EcLL3in`v7<7rk2>8q+}(Dy!KVoGnYLQ|7IF_KE! zAGXro3LP%0AZ}>N%td_xAuSsed(z_7s8ibg@!Cq^-q5N7nfgzv2;9Q!uhIAYskF%6;O7iO(B}4twi=K0F&}t(_NPur42-DZ#eKaHp-)z#)Ccz}tpehjhtb1ch ziGjerp~8Xx)?$|Q{;gW)InC)Pd|gg#BqKvC!e(i70w62v8kAJ?VLPHsw*#cl0x(!B z8##lWM&8}bu=o+mC_Snq3Z?|FLk72`4i#a^wYrFTD<`)t+iaylhILl|5~&dam1T}? z66!`{85+J=xdQ0=U$rPz3I<7Ma42!&%sks>0TDkizH zSlQ5_O1uJ`>KT#+EhhVVlK!?j;etZAgj(~8hAaY|X>vwSQ|4ytD^SVZ88XU}fi|X- z0#XIYB1j@{}8F-^fm3f(z(HIgwNEeBff zeaEUDY!?1(N$wCJQPD2VYV9i zpsog3%E{2l!@L1$bpD3p(($(a#+z5_jKnvz1@CGxNE!du$OEtbV#*l_8t{N$zkHN3 z*Zww++kL0}u&t7?CY%z1`!m*+Yg9#9z<8r}RcJCYuUn&~k1nm@>R-bnOGgQ|9_+mi zU5$WXmS9g&^h;=w7i{c&m5lu)D}u+qx~~JIl9#QD0cz0-fmg3}NA-a#4P12h1<~mj zmVQ2p*0g0QmN3_}fgFK}My%SyA3@~q)1njw2~u?D7e*a*)-ekjLoDdD$Zu>a(!Pv78I?mGw zQDjM025je}7{6V4N+OM~sLAnAl-YidQl)=`BsGmB)?zEuh)kNT!6(M@S70Yl2)Pi<;Uyl!tP9Cb z8dcwD!}}1eY=3G7ib#Byi#dQ?_g;>VL9$=vNy5KjP0Sd~AH16uJ+8mshT=G(y0Ozq(fa)gmaZJz zgHZNBreGsKWH!g*2F4MSBzh!K7_pV~H>rcw&a+8s`qWl`_Cn1nCq4GHaD?Bs5G-0` zu=4yqDk||$7QQo0ABJr)wCQjW$!xtHmFsNzgcaNQ(h3mCUSr@$LeaMCt zChPfLy3Z_|TE0mse~EUvG=xC-u6F_^8=POOJ9ne%8m6k=%m@pe>n4P0c87g~3%|(d zetyRY7-D8L7rk)%2y~U|!A=FO!Eav>BqaT*T_{%Uvz?*LRgTSwn~4E&^OLdzZYoqh z@)BTokJi*K{oSIYow7azyhy@!jNw*!sb#w`u`S-BFD>dC;P?_I%MchxHN zhl%~WqYD`e4QX{R-MeOmK58<^WKkX9{bZ@jWGQ#yyzIBMw?t*Rp&uhADL0$cYO$FU_%2pe8Ir-U|szqEAlk1G}wLld#1fC!witUw#$l zorLRc8C4{4R=)f%+y{zx7-TLf zh_Q$oUc%aRdB!ZiL4aUD)yMS;W^cC6o9ywR%P3orrNQyj%Q?wRVv7IhhFvfA=KpEa z)Jb9`@1yv5>32i`e!B=fS|dXU-E-P#->zhF7h=wm0d^u^1R0DlV3n6psc(>gBKl~^ zdo~I-k?L|8E%=a_yyY$7vwFUv(j=179<9^a3*R6C6FUtDU%m0aZK#I6H$sC{GL6p& z4%daAR2?uZG2*^^+f&d>kQt^jKc?!oJmi$@6km zqW>Xp9-O@L&`GhzL!9{>ljURQ*4S!Y|y*#Ol}n|Z;D-RAbd*N zhPeCu_xGRbS}`UV4GOeCT%&&)wcQ^qJQ8hq0IHX$947f#cF%V@gW6M0`nK!Rhy?{h zkZF58NHJgQ#Dp5LMnZCFZEe`DxG+J0R+$GJw4~_(d=knWb;rU3i;H&h4dg9PdHL!ay-C{ZE&u=C{o=vW#a!-AzqwG@E^atTD5g-{cKY zBbID>gwY5%@9QA7w`2G`gQ z(4Q&A?_5?tYaVu>Ats1RnxE~cm6;_k(08*;MLh7TFxf%0x|NwH z%4=UCv#^6=b%>t*qPbwoSx413lEcudF(-Ub#!vG(MoP z`L8J1-Cysp4`1a7qavnUlFRH-HrS{G2~*zvh0C*4>QJId)VrWQvWf|=gD+N}LEpLP zJ4&}(m){?nA0F3Ntw3-TY*7!L{G~_jd6XVQ;^+GPu3>rPW62yjEqw;WN#%Fiml#)C z5@J(dGGxjaG1zj0>0IYNy7pUBuzfhR_ZYo>hi2QdfN1ma`H}|z;y@<7>ftcR!iGFn zhj9g!lOFn=#eX-znTjmho5cm|w19x(HkG~A_rPs0H=BUE^tBp4rUvVy3_kc#)uxz0 zu|76-b8QgaVe$Dt{{DyBy7WbvmkCr8PXEAQzB#E~JjU{Fa6(os#WLCWPg_4*E~)wJ z!kXCBQ^F|Imn?hRA|Db9RDgJLKCrz(somY2_nLSrrl3l>Fane19-Xm+CV;TOmYdJb zI^S+Ywo(4==%NMfy}?EnZh7`O=W>5GxdQ$!R*fwaMloE@oVp%JxKVr9ZFe#9GPrUb z>2ZY&q*Hcz+>bQAExFd3dT{8&NYn} z!$3tyx}Iy@v|(v@hW$165RqJ5OF=|Uu+p&ENB2E!mg-c+?7rW|1j#l$%@Nmuq>MN3 zv_&#gsHNri^rY2cOHh;1>Q2+nlc+=N$X~PH-%Mf&B_vuTS3%llDp(2O%%714k!C?d za9&#(E|dP81`11LYBa-lvIupmWdl2WzLho=sTfXd@~T&*Lv5=^5f*u+Qfp!gMN1ZN+ue(YH03Wt|{$R zowVfq`V^ciZ5*ZAvX4r|{$f`8F%NcFc}(+h8Fx{2ug5#VJ>7}^Cbkt-Y3Ry3fy~{f zmAm54WChzt-<9DuZB%QBOs3I)D227zR5adryZk7lI`8Vk;Pk!wmKQBG|}-Q#1onM$RtG)}k~<*!#ONdz^;h!M-bpfq{Z1l-lUN))=*Qs}<^SbT#{bx=_1k{;YDh1r{wysnIh>2vqJ_Y>ibO~Z*ADP(u6z8`+{RpzHdqB(%e#64Yt%1HrxLhMXf1H4mUu&$t{ z7E6Z#7jK7>Mk(((5y|M+Rl~QTsCTGXvo{A7@GDz?sdwkDu+asuny^gW`uX++A};B1YQ_%*Y&%nfgm}juIeEarB)B=x}02Xsi$nxo`zzcF`aIX++kX?In@GbRQ#o! zp{EX+iOS*^6S!~ZQFZ#3JuQU4*uX3vpmjuUltWDo2eN(-;q(>6_wSltD2 zK|TX1?v7~%jE}TT4A4^t(sdFZTim7ZwPlgD%H%u>4S{oCZMhe@G_m_}ddFq#(kU^F z5bqx4G?RVKOsLjNn!RT4A0@l{6;{ei#7DaFrFQY?EgdV_4~^9(gh1MwYt`@tqDhrq z0Q2dJP+#5|->M&nmPD8sd#QDf7elk4n32&e? zGR!Ihe?!D~ysTb=RJcerg`u7NXoc&#SWg{lPa>wZU`Xo@Rg-a|i#1g5t7$xJVe0J1 z++qc}mLB~#S7L%3N5N62B19$Ch>0vTVbsB^!8P+fLZx=E@rU7( zX(jLzQqc({d-vZ+CLQ3UVvhJwp;btFm@l%LKIpF0-^ia@JCCOG)F5H0VQ}&lLA+Ki zNlN^r-sT5(@{iMD0HX*Jh!3snjDNAVn zN#0d!o}QrAjfFk%5vEloD-@ZY0Q^W5b5je9~s zX39cY(xHC^g6TVTB`>JK@ddA&QTc>rDPUsJv#_QVwYDtsdt%2h#fNkNV4L01du?iLBqG9avkc!C<<9xSs#V$ndiBvkLjY()2vr!5|>aU%$k4|6i+<|3r#7**QAv+u9kMIQ{HSs($QQ z0b{p*v+|67XYgq~9F=}&t{dM}|$dL2BH>5^Ol|G}JvRK1f(Hh-~uH%^fwvs)EJ?Ob%omwvmT(8&r z6YY!2M3GQOr|a2t8-~t3ROk;}^_XG+C88X5O7dE*$PF~rbg}>Htu%y0-*w8z&FnVJ zQ$t9`rJw{zwHO2GYTX_xFrVof#x5|ECOfJbk?@*>^2Q%7(t7rw9WGQy8w~V5g2ZXO z15=vcQyM|1`W7;cIyA1Gx>Hs4^(w;!j)9&I4qsmvjxKC|xV=rgKz;30l??E^R~ zUIwztx6#$q!I75&R0bhJQ~eSRZl!H|1>pjdG3$&W+c5t{SX+_FTGk$Y6!{Nf?PC!& ztJeGF*4d>evJ3OW)A#=LY!ddv)6>Vt)5FQPC9u-U-OuH9@8js>%n?i-fA8Ur{pIxI zP8f7dHUjFVUeUN$rEfeYEpb*Y92tad0ka5u>Lt`nfl;xkN`T@3EqS-VA6#{s6-*ff zxt%>Ia&?p8&khd=s#njo7dwi4HsvHz90)3MfLq(WXP^L_1p zOaLlLj}A?f6u?7A^Y~9@1)502%07NlhQ4hsv@IXKdW4g;a*IQXTOX$ zEQW z4AibW3@20SN^&Ym0~ymYC0QpAT$u?(hERp+PZMV)Nxd$bgkog>I|Zd$l`U1Yzn!%a zN63RR?2(3}|54yw-7^+N3a^MOJkZc6I5SA{W7lCzWrLdp?JvK=3L%rL5WHOt(1YsOayTia6l*=c=tXTb$pYT85~Pf25Q^G z)%A7$cB$vIjM_-YgJ~tcyxq26;HwTs3I0QuAS*c%DSz~8xav6mjwc=;n-Zpmu6O$v z=cjXX$E&-qL&}Grz=d?R#^#0a>|zBfLv3Et280^%#~@M?Ox@IB^D?o^w{+{d*f`AN zO^J=il}%Dr{{{BIR(J32hKT(##}cCVZn}eGdKJ}_&WQ7XI+b`xd-AR0rhgH-;Gj%- zs@ih(r#!Zc8~VTpE=-(s+!-dOv|lxmjdGn(i|E#>Kp{fE+6XRyn`%YpCgVF+PN$W1 z9I4$V`4D);+%+7y%G6jB5_Uhb2=uN%M_SZfm(D43I8OnYT4WDl$n|{s9&GKl0$A;G zO)rQ|2GgF829Zt2;?MGvR=c?t28yR9jcbE2`VXq9*Bk+p&{i3LKxlJNU{R9kC--1-uCY>6EsKFyGPqcG zs+n(!q$(XY!liT~`FA>|8P3T7%c<0?vpA?_%LtwZnr>%Zd9jvXrPS8lXr(#aR1#{1 z!+6nSRrD4&FgLHQ0*;T1uID`dGH^%y6h=GL&vF=;$pXuwmI*O7MQUD1WS%Bum;%yi z=fgGE3k`3!z`ubGXp*|PI*yV?3r zNg=rG@>iIoMb&z-KE`rK{yv3wT~N?yLRGfU)qc#-KEkHZOhqyFo@W{k?k9dSBdv7| zG2)ytkQk8{JTc-Aa+$2eNpo=l{Zs=WzOB_hiyKl>N>X!j-h4vP75B~6D9n=KDRAK8 zgA;?1xbEKs8n<@uvM70jXirko3E(Km8duF2Ub=d%YB$9Q1S2bQc@k}O8jMziNpgAy zvIPdNA2?#3^#LY;BxmglN)jEDdDiHvaa(8-%-~GjA{uaqEsjd0<lh*hdkX3 zrZT`q@vLcs1|!h5nLhJaw;L)%ae4B;hsIVc(z*&yf^o$%2d|80?{&~}-IS*t`#wLD zKC=DY*4JS{DP5Ws6O>vp(qFbNnxKo|kt$s~80O{DxqY8>o-5WtN`@Nz{Y!|MEflcR zmxy^b7CTPbX&Ibw`_*35cp%yvFC3<+`r5+tH)~N#>noPQmkEu1LLNGOd_jYBW0i3? zV&wT3YKvAqcQe125?&(QMqS5Oa7q}@@#xqZnB^oejuh+ePrlkioJ~PTFPjErlzV=} zm@TqNrWH&#=)M^JBi}@u2;rR3;^(B^K0Up{8-I}`VsY%>Ej`fBOa6_VU4JU&LtK;~ zY?I!&zQQa(cfL*SD-VeW<0MAQysyjr_OWZ$;TlP+nkkoi;Yu8!(0oOF=}6*!<1Rbx zl8VftL}@W*Ss$)I6;wM`DP;7LuvLjdG*6sdAtA1xJX5qq9B5Qn}S18Hh;ATi~c5rZ| zB*y=nUt!RV2K)ps`Yl$;|yu0_>%gxoW4!jaFVf^Q-WXZbw%5*jnIDU#NAl|GZt>Ao$*Q_g~Wq{l%&E&W@wSrD!F1wLOIB^t6>q=)i40FfZCw z+r%xFP{-`7XTjVN@AOzc^QahWV)Mt~}5smh>W;5b>n zc*ATBaSy|4HSM{nfA&Vi(!?~xx44Gp96MwbL6~I-CHUu-Btco6xdnxFCNZnB$XGmZ z$y3Elh%SA_%B#*Rv42~u1G`x?s&Yy$>X=$!1ypu{t6GY87_WIxyRyHS^BFFCUziNc z3o=gLC%KTzM0(7E@ndag_yj?^gzU*is3yMH_VE(pt&5!ZfH+dhy-k*uSjDGIi1?gY zvbc)03e!gLN{XUr*A&$`O{GxU@&{~i-(a08CA)=whMAY_IOPQX8{h#ud{4FMzM{MD za)m%)8;gFIm`A?484tKI-1ZUR!2gjvI>;@4?+nKHzmRmuSL<+3bJ}WXV_I!e?|j#K z{|i484{LLstJT4+&mDFHJ6E^ z)hFA)E2eCQZPuxQ0wyR;oQ@r7EJ+oMeXs?c@K(P%J})W*Yj*U06eeu)RNB6mZF_8K z*Qzv?tH?9UF7t5RdzjiQIuBD*ykWIsO-r&$Sv5(k_T3iH*?In9e!n<;n7e1E#D-%? zIe~+g6|}>YH=pM_LKSB^4|AmvF<=0eU8WGUMMQ%_+sbiad_%t!DhZT(2Q85$v}red zs$YP4Ve*@HE9fs8SCB#Ll)y9xa8FVh4FgcIHL96P5(KwtD_)7)zwqx9-LaTrnO`AF z6})3_3h-r*!CQ^ZYP!?Mom+IJIo5P8%*TTBQ`af0t(A|LucaRYncm1aG0-2qxQiP7 zrtm71Gg2LU5Ls6OmU%H^Ree&UZl&oa1>=2V8&_o`=HRHzYK^jHaS5z-e~u&sq4I$L z4P&L8V|;aJ(64H;1YJ*#?p*is4m=MJ4jMhSb?FX%g6Qnj+NQ)!zAs~4_4XM(f_(Mp zY#IAc*(!qcP+Y!9!%h{?r^2m-i-THXLSTlFR%$|KMV1u83K8cjLX%-BlxdslFv2P* ziZbt3nWSjZb3#(Z6=+SOUFdOItx%QIZZiI1)(H1mf-bW~Sq2Ytqst79Aj8_`o^-cA zPz%Z=aZEmFAXH)eXBjkNs@Im^vWeu~-_VF5qLxt2lF}$Rnpcp~arv;so_bfV4AP^z zRcEa!Y#B>G_&!6dIyO(_VH=^MIAj7cj3Ba|*PSRE#3fmHa{Cp}v-y>%Q;V1HKN%Gj zMG;9=)qf_Jv_xcx=JM~WZDk^_n7$|}yY4jEET$k3_#e;gpYN;3e1GD`CA)Ud2AIE{ zz9RF@Cj+uwqTnM+Hc^w?)m|E~Fns992Tg}Lyux~%AH=6(#6FR}^LsuDK3ZuEE7_a* zVi9??dI9xZA@{R2h`&@hM*Hw;ITxf zKBYI9bS0s?4dc|C&az1IOH`-g96bzSPj6_)%}cb+l3ByA^5bM?r-&l(3MlvlRR@9} zYEK?fi;C>V&t-%#C^uLGUuDofn|n!ZBY%jBu4%C*(wP+}i(?#3D=P8`U$u}fxf*=;8#Mb$E9sdJhL9n>@OP&qyycsh2&Unxx(4&;wu`r={Xv~LUwvTA#!F*x_3y@T&(eIs ztgq4~CP%=KYvjVlM93&rXK%O1?)Vh3^(8)J+7rR@dDM?Afms+*6T0G;EffN7z-2(n zmL(SqK_&NEqvw5}l_@bLHs(^tPS)v|-iDjHIt`74@xG2P*F1`iWIKx~xj+6ye@sV< z@Up@T>o5xr;8-on^I+9{YJ6zl^jsSOGWmEi4r=&9J?yb!@m^={>k#Ezdx%O|UYqxL zt?(BYFTZeAJ}zNC!baDV@*3Tp`R_0beB-9l&ii&z>==CUX$%P1U*>md+b=>Ts&L={ zzZ;(oN**Fdo87oZM_qns#fX&w2!8+qIefc#|D~_Ubf9ABr@7sMDInVRLShIN+=VM-9xG-0^!uNI1o(mz&(%+2%>K{J z;eYQEKh)1(&USXzPV{E>F0{@jPR?2Cx=tHXNdE~rRo@-%ssH8mJl-mS+TSp!2SW$i zo;e|__{V~zgvc~09WgDYvDe3n1PBTLbs>@VF^g}@r^Bo4bH&rB>rrHBvm&27q1kv| zXTQVelYSE#aTp~?aJE8|6(&u-y#PoFPXUs*^n`3EcX2vZx~ou$vEO0KdVdpRZJ7N1 zAADtKvqU|Cm3@9Z6%?--6k!zQNuxY6?8sh}C=&C4&riPI{gf=wLX)MtzCbv+EXMBu z`UF-HjTVr3%)N_rtL)7tt;*#JNv0NA!a}ty?TVGW9jCCIa~*cTdOBrXxzgVUrwJ0+ znN@e>a$?wZYp->+SU;9?yKrU6Un;rr+`{z-k<=+bP!{%{22A^LV`MbJ7 zO+4jDPn?Kg{RsPI@GJ}xjuf>B5hAnxN|ntD9TAi9NJ)wyMvx|F-9kVnLzl>~qC&t| zVHvd;WSm?~3o6J?Dya7Z5RaU?ajI1JP8M5!(aJUF z`c+}srcZze%6P2MwmZj5s(%uh&f>6nGTDf}CP?<9Ct1-%7^k<2ZDGrd!Z-TGZBK1} z%ZzM=rjwQcAmDeQZA}Z9j;pa$-i%FQ;^xNI4-ZTuQ0l|Eh+wcw2=Mv(_UfMFZM;9) zXoqTGt{Anv#G&P;c?g94Lkv7euR*g*%BS_PQb^=ZO}p%H>B66mRSubH;^z5)EE(=8 z;)2&scb^k~`_Y^2mO9ntw^a_5m+ko&Njee~n2MJFib4HEB{D@Yre@VQh5- z$$fUQfZ-lHNgZxPBs2|<2M#z05u%H@c{g_*7|BTr9E$QKGX+=&SUZ1(!P{;7e%K%C z_*u8gOBzz_7E2Npt?!5G2(Uh~!orPA*}vI;b38s_z}*BU!;|;awSWlBMu12B2`UvJ zp>=E+NkB}qNptfuwPtGvoD=H1Y4R89y#yj#G&u+j%`VBaPTx*Q_HORk=wY}Gx`B-F zC-0c`z>e~)ypNvdUCYX)`rrEV;nxD`<&k3f-GgJL!glp-GGSM^XxcSnp<{OG^g$}p z>q9*4s9yuCJ++5AOxzPhty^z}Y>``8x6F!6{ zY+RYrh{)4M8%sioFGE9XW5pW}PdR!|J$TB~Geye$>~C&YSgB%H`YIiLm2hXP`g?G8 zicKL>==#{Bd-P9c*0sOcqprapv#Wj5YVVwWm-y0|5|1?f#LnkX*6MGA5JF_e`MLR$j>VJA?GYAJJ(K>8hDvHk-QY&i;sc2RW zMF&w{F?9sAYy}A_K-BNUzSekDZ{u<>HH;vBY|?3#^I2g~eV^^+RtUhjPEMmzmPkma z+j!eiK|Did<~QthOeNgb(<2i)jum+Aq!wHeIb9%%O2Kb>u03L zc&G&@w@V;os~J|ZG0+vkhNxcwDrN)KI$jbnkNSz(Domb+q{g-4ETK;z74|-rZWC46 z5F&J&2Cv!F|BVX9@V&&ycD#RU**uZtKb=)OQlgD9vf4O#I_;g2V}iVXH;x4bl+)9? zp&T#_USSbEvpj$ht<~>QHi3*99(!BU6}zEU3Q-eHeMpH}Fh}UwZN3krsqM^#Lt~FN zVPwxZYR%WTLwD5pYQZ#gy#5&mkbi#hc5riZaN^ILaqtS81Y}mOe_lM9#sb{h+Fk>D zXKicb1i4lYev+)1eUJ0g#i-I6Ip=}=7hL>>gxl#{;0mpmDSLUX z$^c$M!!-6`8)$Em=+#WSv?ndrg5-;Fjpu-8cy)O7#?ICtet+-2 z(qfB902`al&~Fpy!LWK7uY+O>swqSv8Xas+@UO(K>yIy)Q^sB-QGjTNiL&^?M4Ej9 zOg;~xE!H$V+n6dcV0+7Hgt~ZrES?6CA^i!U6$6rEh#mJjHc?8LniI; zhLOI-oYv?ZX;PpI+o#|*p$ICVZ=d>(#slvSnA!#=2~w`*%d}Lc5`k)R9bAWEaG9^U z!3YJ8@<>>EhPJ`sd|t_7(GyBTstj8xL2CsyODLaI63S$3^e<@ukrHrd8k11Bj#9oV zNn-#VsfB3Z_qylEQBM`6-vKX;aD67V);WT3fwcE$q`Cw&OzaaU?k=E}y=#;V=HG_u zRL9ijHYkJ_y(RNHj(W>PN1b6JgT3~Nj^qZ;0$9Cs%CDqVIFS0eua^k&nP($I=U=WQ zDak_Efz5h# z8Y~52cGE0^`OaFpFUbmrxHm6J?$O}N+&cZR>&KMi>}H2V`Ip2;{buy|>nGwfkc%F0 z6Th)RI)Yt#14*jS@b{|GONMD;ZfU{NR|ZpM;?Q?{aMKX!l>`Sl14~ z^IY{6@8unYgfy(2=}ne zMGmcpM20#(d8s=3hQC1$D`wF-@VM*$@O4hnnFL!Dj&0k2tcjh;#I|kQ$;7s8+qR8~ zZQJI}eYmgpu^(!!uIgU9`6dSro*}LNOx^Lt*f5lX70E*=@*OP`$6DEmhO~TbB-LVW1404uT4k z0$kmH^wmcb6$dO?fw71hagMbQP07g)l}|{@MMmT*;kknt!+$F260XbwW|44iox)a`hJA`BRvpmbZ|q&(U2bU&ib`ZQmCPpv3Ni<33ujUv zg_AgOrdtYZawn=N(kUS6h>OlRC2}Hw&pieG$ z1>JBc(0+k!EU0`sAWH1-QBZA$MF=W$p3aQUD!(+zT%69B((d4S>fhW}CU#7Z8SvBB zO~=;u{f;|dg9tSJdjveu+ z)Riwsph^E*-JEOeS%kguqVkw^fK+e5ju`@1YJIQ&gD^uA=N>rn%2-M4Xt!y>J%ZEi zT+#CI?ajHip-~e5g^dlXwg|HRZ(Mmf8qM8?9M(kfwpPae(%LgH+44XlU z;-O@x(B3n8-tT_kP;8AzvKi(+R}Po5LGJ-qD@kvJNgalN`r-#-w9nT@FW+OebVuk8 zsKC;$3HW;nNAL2gxc00@U=%-x1|~btJvJ7z!p4^`wXZ#=cj7S#V}ft>#h92O*@0Q6z``xS;Y~-9oCZyp2(7`dR17S9}z(i z9MRB_vQ_Ilv{XBf{3p}B8_I$Ld*m)g$iLNTFyS;A2TIKSLAX^myDF*215{wK4@Kr@ z2l>(y2a;){ zz487}ltl%n!Bl=k`S$;c^3P8+kdvMBkNwcerER;<5r6%K-ZVb2<{JHD&y~>fv}(DU zFr;QX#l{f8`Tx!>s-FrIsu!m=w~hVwri_pOaSl&bv#fLRH7lu?*ek-<0xT+;MSGY3 zSZs(^6|p7owyxQMb}9f=C}|tCU}9-nxq2YzX{(!j2I1A9B?Cc{hBMV^$rE^Ah!$1u zABX!p23)U+UqcelQFiA`5XX#c>mhU8qWkrLn<#&L2qHY1WZPQxv;oY8U>CO(g8oFH z_}awDFTT})s>(bAO{BR$)=0Fs9eV%}?+JPJ9S{fgvUuh=QXabw`*kpqQzJxgSh9mT zCVF21`o-kH$Q@tx5ysDTwTfs2gr6`_g`-sIA|1j+f*HKOT%UFhF7EZ;^t?`#%lSfv zzxS8ZEot@Iz>Af}9Ghv8C7ys3Ks7A~{hda^dCY9u)c*C2CVA=MP{RDN+4?etm=S=Iw6f;OpSxZ43N|d9!=JI`R{o z_Rlos=9Pkj?;YNokN2Ge^1vkk=O$xruxDW!o&%$>L2{4)l;#wh3C6K|k26uK(z01Z zZNhWGKZ2EV0iE4Q3RDU$ftS*Ks7p3YJ8y2xsK^-Hr2;BLRl))`r?oCY{6*i$JEZwl zdHgXwqG?27(ng}&YGyiZlf_O`C5bL|Uq7Gbxg3%r@sEo};p;&p`%GsF_b=0Co- z!bD3omiS`J#+(C>C=ad-DW)}c*S-edk&-bohO}R3WumAK>ty15trSKT@TkJ#7Jwk# zEZysERU#ula`E^u(4NLg6hOtKpwQUW4 z|M1j@67O?=2!LAvQJLm_o)Hw*1_hs3gQ4Ylb&n?HRBv}o>(@iu8vk!P2%m52mv@4Z z@BK*Syv@;mUP?P~^o{{@o%A+#_%*3on3!HQ-jF*Pe9jzcesOoz3haw-sIkRYXNQG5 z`w&gF-bBp#JDI7Ub`>3(F6Zy@T&>^7BKY`bjAPGNO6fIA=7JODZ{)?ExcbhGHdl6@ zu&|?&%9_>SE9sWJ%tVhu9MH&s14 zTT~23K+?aW{A*Rh=Hj22AY%%nZq8406We4>AY=R--ejq6sLZJjk0DRhKJ*~hM@#ET?C3*iRFL|w(sQS{bzzAVAvcZZ0HE!Gv1!98d#$%U5{oqEa3 z=+veYo3D6OrE_%lS^l~{l*$;0=tmwCC@aTfBCTc6QERj?dGF1H+i{ z-<^q6LHVim`TOFjk%w7nic8>z$sJ~qhP?tMD%9%Zj~HL|zGQih)U<(CdpU?s3GDS3 zBJV)81wB|+t)Q*SseL4=KD2q84_F%A6&R~#ciCOmmS|; zNfT>9(sG8=<2)TTOkAA}B9HU6u9gw%V&f8$L2EYa8>m7KQ1Uv^KR_>?Y*K+mv@=h# zi0&X69MaCg6xAtB%I2AWaSav2%UePTbbSHoDE_TN2g_a4waFc?|c?n6FQ3XRZ&@JiC zO#i{AP;|}NIL19!tfXYeN>e5e zXqi9IWQzDL*23S|Q^ttmEDS)-dtANieR$RF4ay2{U!&49f86P{q_T+gP~f1LF;mjZ z>Xu-?j2*{3-s}d)*%ojPS;=i6A(g@%#IQw_`2-svg~W|j!v#AxLF`!`_Mg*Id#>Fj znFigwvXuq?yBMF3nu3!-E6F^fo7@Hxjx%~dX;3d<_F{A;7J$Bg_-u`()O5l@c=#f* z%+a{fyDni<`7dW1yYNTE$a2m7=S8K*P_U6#UKhPKnYsFTU;04MQe}1(QPqQ#W1STd%fwV?0{@)}NI< z#IXwJMT+)Mr>qZQWkK8|_D)zVO7}x!@}f`WY9zTgp|6qr<)P+BIA&zvB+w->PMM2A_5vc6 z9mM3nq!C-y#!gv?CU@;bzW2L_rP-Y5-bB3%n(kzZ$W!`%$Z`TzFyq1Sq!BV8d zZg8HjqPXUjkk%ZL68@tP`oXvLj~QI`r~Tf6gntF4A>(%Y)yN$(T|Dyt{yN9aO*~)F z7P&~~s5p>t@V1N}8}~V~n#i;o&v;`-=3{ch%i%CIHcybV3#IfJ89oug^mtcr~z}fR$kU;k}@k zl&dgm#QI2{iE%L?BiV$*RvoL!VesZ_8zmrgyLEb4LBh$FMq>NEZXH<3>c0$sj<>oz z3K5m>+<3}r{WI|+pVxDs8;4;9Dl>Tn%sO?f=zGPdd7Dri8X1VOUFOeS!q9c5=fsWc z<-hEUjm13yZIw#azp#U*C)RQbDlM?jJ+0w>Nf@Zl`~8p(*#%){=P1h+nQmX6$v(9m zenk=dgTQLqNd#r2F&b(W>n|GExn=p|dA2@U-^6@E!?ZPwJkPwKC)RkbBp8ZDGdw|l z%PTkx+9fk_ps;J%h!=Z0VSNXW;XF3PWwhr;vWyfN;#8x@e%}Jolr{2r1GWZPc8x=( z$S{zAkGz(I2_1M!=G{;VPQzTi#^q#cw>o1E_p$+qSTAukW(K;B?(io5>|5lwr{kS- ze8aI8AiO6(rtRNO$p~#-^X!V+p>Yl6T^9-L*j`;1fwtu(T??F!;Dl<#4qj&&zxqC) z?kDHdaAeanVT*sug8E$Zc8Xy^n>Cxf2ok!^)4PR(WDjXU&KuL@K$iEZn;&f>4)GKN zc=`dlHR_)?&k6)zQ#2am5bTzF43>1B8BRp_I>;kDSDF#6urwP{*!L@!wQr`X_6X3| zy4~%Jac)=8-Z$i$t|v@Rbt>x0gWq8`?lw|-+bSnOlA*%Z81;>lUMxH)-D;xLXm>)i z>6`%H)NKl^SoBLp*M_;vV8I%WaxE7g^c=t$Eq~L;w|zGmVcXDx$6e`j<8e;&?7Zf^ zSf62t3(=Q{Juvz^l!bzeT&W~C|AoiddWJ&kB$LAJu&;$+&y0>oM{#1Ein(n{_(7zK z^p>^}tEZd{WxaED3Yv3m1yi+jp;``1LcwMuga@NE87CIg=2F6)?%2unP#ck)gk<2q z;w6JO*t2cddTs!yGYzQVgrZ3^V>^EBW?7r63r+FRZg9(%>C;R~jD>HCAnlFyzcA|i z|k%=k{;H$a5)Q3 zkItz3lP|jcF66K`ceg@MzYSIex`;c9a~%5Oc)tG;lumJjnPUoy;JV@-T*_`rWTd>= zEN`;r&Fx?-xP?_=3n}7qG-R8)oQ$K`F{aW~#hPw@PUL8I%F^-b?sm+B|LBu*zJa&J zwlz;#jcqXnkxl#|M4|V{qPA17gGiCv6A!;Go}2Mj)vK+`FG6r;8&;r-x3*|jx^E%E z)tAotOpuaO6Y7Grf*ZxQU0E*79Zs^KgUx-Q}xJ8UEM3JH5I8;#Z7;7?D&K5 zjlNyMDN|lIi`aK}ZQ;wAO&Yhf>UXna`h=;tu}ZcPbarCUg&q?42Vbi|O4HnD4a1-2 zG}hb1I0#>K3^=M}<8#GrloNY^-pJbM3Ov>w0!`NRjxnzv2*)gB=*87;FPEJ7(cbSs z+ikq9==Ntq$BAqEv2Vox)r*WqBhw#08xg`8|6ddD$ASTJb~3m6DHu1pd|WqL<0#*H z_`s95-i6kbB=07FvMWlm%#7UgwKbo{9qwF=gG>@mY$8{{Ab}nS!mmtyAeDijVy(op zjkYO@4pnETv#PZ9jYPkFmo^zzQ#cH3X2|po8Fmu z5vPV3I$1*bs;l{Ll+JdHs|UcUT9}krik@K9M=fz8Er2VmCJobVL!G<~4p^3|%EbmX zs(+09C;{#}rZ#0A6YtB=?8o%xjHmgn_Fa`|QQXn7yu6%%7n@~Qw^3D_yW!>e*Ujnf z*6jGu2(kQcISn-c`C&O3S|k3IZY-nNK$x$Fvh2>XAABsue=eS~`nC&0s?H+~WeoF- z3)ATwn<`Doe@YV_X~JN{NX~betVZ1FCJxUha#R+QFowVQ=dN{Q{(XIX&11iw_nn^h zjr~NLir#%C;&t=8d;N9!_&OkU{|?5>{GfP!-cMMj%9E-pC3+}E#_^$tnl~yeDqa

CD$*CmoF<(j3m8~@B`NMlr42#cOJB;~rl`rV(M1Y=a~kpS8z^|%y;B~^82ah2=Z zs!X>k@40jp{E9HFwJE1*AUjd4RZdnjPy{Aa9LJhQap&=iOQFIsD>4_U^cDU&(#{`P z95%fPC}MLi?SrbzukDOly1PAnJ-$;RJ5AM4hAmhNmITsS(whmjV#vLLj{2qRC<;_Z zKJ3FP)U}O*7eEXEjwixp!oypNEF&@egFgxAid$zC@3C)@SLzKd$`7hE9wT0e+URfU zEj6_2{7u~DZ<iuD*>EQwU(!K2=HyFZE8Ne7-z^}BlS|M_0;`KZZ{4MQiFP4%;{?|{O z^A3F&ty9bBB2ExMrj>w)bFrq!DhtCFUJow`b1I#EjaI!djW~PqG}DPhveW~0_Tp>$FtS8%Sg~B9 z75-n2X_=TWQcus1D>c{K!>z5Yr6m<{Od~IKrrAM{IT>J`KQ20Ps5%)Ck0H=#;Tc+z zq@3KSbkfjW`U=J$DBk3nnwltKNl8+%*kz|t;m9DB{!u!8`vm<^zlOYpL>!sy^2vy; z1k#~FW9{pUaNR|yvs1`ws1=w5V22(TmIJEv#riFV#AR66FO)9gQKS)855PC6(_)QG zQbFW3Z&{m+B3&M9D8+Xrw^-`m`*E8pwpVI=&7QHn+igSn(vV|_Udqgh_RcG5kRP}| zEc&X}Jx+JUlY2L(>gCCDlLD;jB1O+YL@R!R1YfCyU=uBvY@J}il1|eSqJJXX-R|x`yt$4Md=_sV&9_= zne!r438bXx=&1|?U-RZ&R?bm+F+Anw&xCl_q>;vBWQ0WV0VW?fG3XrA87qoD&6}*; zVt6`QQsgFgMBfFbHv!#a{n6qwwYyUsL=bS_>|Ds3q*EM^NRX?0odN4&FETqF1~&}Q zN~M`bn9VXt+{be z<~b4CfbR9tr-FBm>2V*$=kob?;ki+EZ+ptQ^EGC*asbDjpD-0?k67mtiXe9);#o7P zR=3F_os<|+vQj>2T#^x>q<%}~AU#5IV>;&|13aWMuC>=`uAc0vz)^8o|nsgE;NcE{;Qi;hqu@ z=b?V~t+IXU6LGz}dG^jEkzn2nQ*3K{Cou=S6HPp3CGwObH{ZFy>_)1s2T9ExU#kI6 zKK=7ixf)LFl=5WgVG!NIVA#D>Bh-l_^cHOtk$Cb`k5a^bJKCnB-!h5>NYB}*DLtjK(K{+ts z68h|yEj2|9eLzmSe~o(A-;@T8Sq)jOgJesUln<^1YE0_+_1iM$$-f!2Eycx-bPEi= zfXxcC!q*dnvww-Y9Qt$Xt}FZlV{ZYTHG|e_p>$$__^+k!_QOjof7)=91I9Is!3R{D zX>m^Km0sJ!9;s&_9pb?S@58TMM6(W>xsXao1t$j-BG(Ctx(eDXAlohkX(=}#S z3R^~htVP%2wY>*zi$V*mtLbW;I(zrVFarpcZKJ0&oFo9Y!UFu|Kc0(~o@z@xx_eSp zs}ezm1b{x*cZgZ@J+$3{wDLVlV|sKlbof`aKb5>A>rRC<&TMW8Gq2&CKw8~&G#YC= z6n2J{T*BnAP`=F%*B-$P%96ELwk zRycIjIOUz#^fIvU>l?iSVbbp2cb-5n61k%PS0L$ut1UHl=c4v_mWpClIw|2$ji*egOEjKzeg3EBz-DSQFJDE`A4A8o<0A&~< zDW!uZ9n(iX`GT<6yLK|-H6{`z2H9bbu`_@FKh-@{sh#m!iX>(NtUxoi-h3aw5%}-| z>WjKb@THy+Efx-zwd!cFL=C3H(*~J2Kf}d0#kCUAC@gif(ghpSDS!5#f`RG>^X2B@ z2Z>jNgM~Os;Sn<%@&yd07sEePHJg5K zr!RBg3;+jT{Y;WOJ>F3G?u@IaDT?FG8FKuVq^8_mM++p5FAE=JA)Emat;A%H?d^rH`1)ITgQ^(&rrUTK=-Z~g2Qk;s zagWRSDJ*_0Kgq#=i)e=^&qe>YOZQbPHtG71|K>%l$Hq#73v6V9);Ou2&1 z91__S0?3oygDMBdOtVx)2a4;cZg7rr1Y9d;42Ob!H9%CMxuuIRt821Wg^<136%32K z;$juQ%V%hT7Ip|?+j@X$EHl?Jx8iDWhzyq%H2+TZDCgtxfK^rXf~b;fFYCTJB@nJ4=B!v z8bxsyCPUIAgSAa#w~wN#CUu9Eex1L^=lg!W;9_SWq-7)o(sblnwtir~U|IABxCCj- zpDv&mfdH0Jy9~|#%}3Di#n~RiM=j6=VmF9&cGldMIO@-?G(C+OmzDD0I;{ivdGq@0 z5j84&+73&bqck|n|I$75sT8L6(9!q#aCVmNRM|z`_MP-Rbv#l}yvE7&b_Kl&6s09z z!Y+$gt5-gDwVcA`cfcz6T{?Rsn`rcr5r$HCg_anPn#@@9ip+ySO&Pw;TI3UChzyhN zOy!ct-p%-N01AU>FVRHhp>yGCv> zATVI0^Ml;7P#Y4LXbH)B8tPFiB?vvj_4vtPCU`#e6J`fj+hak^ew3x;16`Ah0~t{u z&|RK0*f!WrGjl&3`CkeYwrVD~!7!Ujt&V?=y&aa5pF3Ptl{T&oVt$aV#E*N&|KJw} ztMC9>Efa3YUhy86aN*|&m>&eC9yhyg5}ikpMzyE`w${6KO^@3e3XjPQDM}|)f#r7c za#YNYpe&Ww&GM|w( zk-A>|C;~HM8_=6_V%6NGj}i~6|87)nnQA|LGd55CdxmJqn)M>3PV@%2Y z3lP9Mc8Qs#a5D*KVdde4)y^X)++F7Wcti$JBBK{ZN!Y?2^bGpmi6J<7TxZ$R{w6`-sq48E_s(Jf58jICtG)wGs)1h zR4>AO4>QH8H{^Asl$xvv!;nmf$LK7ye>lCEn!Xmq?|0NpSK~E!Gm?79hg|uXpk$+5 z7h?xZmeA?OTwkDFK|8yTtd({bF!XAw=*FukQRT5q90YBF_3;%aInAnok&|T8sJSzy zNx9TKxo*0Bn`P@w8W}~QTF<{8a^sulhFr`UdZ5oZyjL{kB zg^Oo-iyBC`kO8c+7|bO~sr@!X?yrU-$P#&nfA9IqU{uoIm!FwslJ!wy^`xoPwxUH|qqD_Gi>*q9rn+s$k+!@> zZOq|$OorRPqQu(6U?+gr4CO_jHLPMuW7oZN_*JJwpkCNFW^1|T9$KmFrAg=XPZi8B zrE|O>xx<2$XZTxrO^8x>2|oWhre|)uCny)*l+ayH@R#mLApC5V+CdFR!} zU{{agb!78W^^r9(dsfh|S?AxOSf{O-A0ot1i$C?^&U0zBb&GFhH_!676j+6%2dZd? z{2!fIFQ!LU-Rsan&{=L!Eur;}_09VDe#0$inSTnO-=A#I#Tf-y9C7y=G(gcVRQ3$a zu1hlsHlmb`0HLtd{?b7w1Ac&wIwPcS=qf_&=lrQA*{h$LM`nbY*%URU#X`xL$_N#4 zcsoR7b(rdDt->Y>DhVYo++p)<+f3^zaUpQVTjN7-zsDZY+UznBevOrGfnFNI=I=;62C=EJd~`jYBxQP4DmgJTbKBrG zq@c0b`hxpu!QSa+J@%fm%7NQ{9DgE|0Fxbm>pj*zVNFaA*t?1}y4nf?5!=Dqpkj8N z4z@i*k!kL+%4ZU(`@q7UV1LYr^R5&4tWlS4lW~^m_&9%ucy{=z2`nFS0ayr(ihxuU zMJ0rfY$HwcELeWekw*OvEB%P*H4i+FSW~sRx02(jZ)G9Ypn>A-xSsm*xeQ@J0R3Bi zm#mAfe4S4~!CQ4DHX+zHE?z?~&%whiFpOrWaU=C3CLV3up0)BGr^b(1bMM`=cbz)gEI{v3VCShb)8U52g!MrXoBdLqmcsdCP1{KUrOrb*Fr zK!+HbC(=u!m7)~yX0WXqq#lewElNy_D2h0@Sk`cLnw8i@9&7{VYfVc=huL0<#@_N} zai-1ak4$g^TsY7J@>@E@a3p7Y_+SID`hoM9(1n2b8|pRQp9BGgGe|lLV4#6&Y#;qD zAW6DwX@&$y>49TRFx|e<0jTgA_wyga}AxUG71>YVNdkaP_#^k+3nr_taDrrF`IL9N13LJ-Lw z^}J^<>BXJRYg(nPRpKCM$8RDZr585jm>qjcX6~H1DdswIYeOx!Q0eY(YFibyA ztamVmMKI>1tX;pRY0^=K7^qg-Hdl3aNE0j$LBj_=@{5E`mfCz9LR9BE>1TP5b%|Pw zRs2=a(X;RM2%V0_s>5cWbZ(80ka&MG4-?}<*(Q0)v~)~1Az#e9b<}8Q#m!JLbU?SB zVebq=!Ke4B!NTMP4R0{yrzEM%*4iAEmTZ~saRP`E-YM8sZUGhSEa2NNR~wZ5*Kl;I z?a*Yj5@Z%_tW_zh|J1oT(#-^m9N3;h0y7YBzdL4zG>E40Bv=%G@=)ge z^yHF*Y0I1LW3Gx4#)@^B@8!U0oG!8iZ1tQ1PQWM@h?yP^)KM)`t+yPR55Kh}uUz}(BPx>uwepw8*WOQO+e&AY% z03zrQ2^@N!P)%%b(efeU@0e&QkMdfNzI|b%1<)J8r<6A;0cFhhg&cgozL3TTmP-Pv zUrD2dJ5yCzyQQ(tZv5PNjbJ0qoB(iJEV`)qyf~3U9kD7w3u$vvTTM15#ggkPcX3nC z51+4_fDrD=00Lq`B>`)vjQ|AVWx=Z1ue2?-E-J+hI5FK>Pc&Y;li%>Y?N5V8KyDaQ zW#e59@2;-^01WMKZYEth&Dl13{XI;|=#e9jdDVH#JJTXYZs4hiByA}=HO(!@W9(Kn z0qp%`!)|VhEV_Gpfg`$#@k!qpRS8xq_w0xwsj>Uxl;yWE!?<)EK?OvPs%J%~LAH`% zp?!P`s?$3avc8Mw*ex^L`*dtc3MNjo#co#@Ze?+Yq?k1z+aITq4Ywq=>I7w7uUU`uLhehMwZu+5L-*AmZwm z0=2h;TE%uz)1C`nYJaA;MDmA{;NSTM=i}*Fe5}|(^zaHY=@@Gjl-&-i*x8x$l}D>N zH4s4W$R|X{Y#h`vY|Lpud6of^btE?r8)lx=2d2l@c$%alv(8%?(UNG-R&5*;{E)7{RGnBQm6Jd zD-`_5Y~)e}JP|ukfatG%7%qZU5zJjqe<^uo?A);K*oYbcN%h-W_-fjL%a#vV^epSh zT1V5t$Xz|0h^>akL!Ufyl_ISWPXJolS)_ugxqCFCQ%-AR`e7PZyyuh3W+CU($*J1p|J7kWABgb>L0&0pY2ZNwTh9PsRr66&pAEtuq-E&}q zfbHP0afwghmyrFrDzXwa|AsrIi_>k#sQqsiz%eFOUGw>tEU(F@YPtC(P0aYs+1AU% zXrQdDgEdghiG{`-Ll&TWH-RINT-=1>{oQn#E9o}CpG;0aFub_Uu z(!IGb9qa6JUBS$4owUCq|C#0W;aEWzag5b)(1$1}v2H{aTQ-Sv3*BBlo)foNs+`Gkq%rCV;P0ObAYQj}s1 zl9nYc;@SSGOG_rs{nhV)-m5_Tz*1;(Kh9`|cHLOhPhKlz){uGHS=cbjw!d2G%{xd} zM)zB2(RehqKp&TxTMAd!iSSVB1yYpps6>JD$gw0+%lu95YN)m?T~Gp%ZAxRGKrV%G)rXJ26FwS#Qn}pO zH==oaZ-c+Elc^_xMC+jX%;#pST^KdH>IK=gU?yBN#L_45Q2pbcnh@~*)2mBw11uvo zf0U&cqCpN-018IDqHTEVZEZ{sE9?7Pt)FKM z`FJ^$yXVf};<&Z?=$J<3QLqPwXeA$4d%WtRR+%>v&z{QIfDq>>M8GCF_L`l7T*vQE z>g*39w^e%aws`MtT@8J)5x1hW__5s0 z!74QIc~(hWAx6=kN)NGZN{u(}I4a5{04bF*46!>97bY``X)s$GYaI-Sab4avxlYq4StDpfXxTCf>MWYm6wW{4FnS(}kq`w(nhL)qs<_OZCdQx3 z*gA-W+z)faM`M+(NL)Y2%L&R~*;r$*(_i53Cf9VY^)2r~b^_fO*WR~jR!~mAK%AeW zlh~gt!!AJHh%9Axm*S)&Pv{yvbfW7H8Z(*&gWYcmOo3PBa+#e1!T&+i`7+nj{5DlrNTC`*G0C*Q^aUMECGgOFZcbOpZu(k zSiYm;OMf0?a8BldF?I%_$WxG@9-+eCBZ+%%F2<&Sz8n@od2TveWv(*Bgn+91P_H1| z#dmOsUgs|;BLVO#*s~QlSEHlE+>A=W;E_r|z-0$Mq_%fb1DDN*c#w6W(E;Ge^CIAC zx_7Nfzf-hQJqCL07X)wN!?W za3QW~X|$a~#ebL9sF3*xiC%;Qu+&BSI#m2 zs0CBfoK?IdT*I-!c{rU$gtl!II+PZ|tzid=snl%AWp2Z|bK_C2eLvN1+J?Z8oDG=* z3xdm*w{h=_Ew^+S zb#;Ycuvq@FvxWN(FY7RJbRbH$TShy8?w6{(S`X_QfP;>9ZrV#aEKWW?Yij^SD9)}h zZB!KL4k%V)?xIwB2v^w(!P6o(u|r0^I4%nJpXyLzo)D>t@qT+~Ktx@Fcz%63n0miD z4_BF@WSV>j40|D z5s+eJ(d>!HdIm*`pUgcN{0fe-LarwdDrU|VFbO?07soI4oyI`xg+wuOANT>lqW&~Y zF!`-kor@iBJkpul;DESmcDKX0>$o>g(UHIhX3)`q>$kR zW|;;-hf{?1A90v9{bRRQnP%rJI?d^vqs~^pIdW2Xa_kX?)hSZvRPG*Y6}2E%7xcU^ z`CCJyQz~UNI~Cbgpz;ia{PGQHLV!C@zR1iGUWE2TUI(H{Q~<$=Jk;F2zXX+8f>fYV z|EgFHa;rOVA{D7k`@|Y7YJ&;tupm7D;K8(37H=Fi1N|j#%xa#z)i5s+|PH5yI#V zcIv@kTN;ftJg}EPHQYD~t_G0ySq(Pmtnx(eNsABd2CPY2qPWUQHqL?%s&VY_=vY9T z#Ce`^9EjC48iMU=9f!{l6A_MsGjumc7sEqlk~?kndP(^tY~FqBiVpk#ve3poJY^MG zNyrN^LHl{gf!`iTRgoj;NwXlKuWZ>%^za#dyhEF_~^NSi6zC6}1v03{UWLgM(g!1n3C&<@tw@ zAK<~o3lwwM`5sR5i~3%>#q!ocSP+6 z*BQ6)J@{*AX2o0tz+AgAc@H!B=OS^EZ%Cn79df^W&xxkrSlJ`a;zu!dfq<6Yu;mVVTErbrjbrW=7wBobyyHEgK5I1sbL#mQP<>85VKZG^_k`hh6 z>!p9yX8GipzfOYBx42Ay0EaEKkHT>Rm0RKxw{F=^A*nFXLXYoWY|XOAlI^xRldN95 zO-6Vl*+VduBPnVh3jU-g|G#^n24K~}F~PsHuAG;4f$#A@)5vBy=aILlidDIwiE^(f zJ^^`hrK=m=9{W(4jV`!xUJgy_c)NzPqM_uNQG(e6XVB_P4ACA^Jpd9wEla{}MTWo1 zZ)!uf-<0UJfZ+^}e7Dg-g&Ng+WD<7ehGs?Y>cgrJbJTC>_7mPJRe2%k;#=e3bs7<@ zOTtw=9kmZSZe}aZhvq%2g>`nTV8utqW9%sZ9V1GeaUd{eLb*Ep=Fj#grZz_cRm6Uz-3a&WNmi5m0LeCMEx7rYhZD?qdLZh0 zLTwkKQe|3VMMJ55_j}c+~8Qd0VGq5hE;Z<>a?KK*vgF>|=DWaX-8-XkFmnOX(gAK2< zau|=p7hVRH#!0JG-A=~Q^w;w(SJhgwodVN`zvYA0ozu2VpM#WJiP`zq&s?TYcN@Y( zj!7uI%rOF2D(mQ+`_ds%%-4AjS_)nR+~@5evX{sLse~9QYb2z z{zoVc4F_Na)kHV@>4w!2G}KlaoWqu$r0TIQtLGG9>Q(Vn?kU@~&K*xEa%mu&iSS|q z21LS4O?AvdJ`T?*;u=n@bWLJ#;XLl9F74q7B8*+A0TZkX1|QB!j5Z}gU3^$c-?LV(;)Gk6+zat<)NeGP z1>E4x<>HPX%P|>8F-}@Fzduhk4x%0!YESByThSxgO-YD3xCiOd;<|RK2zWWhpKf6! z)S7-@m+{K>?6~Yk{Lp(-{~%?2&X!!;9pX>9O7gg_32Z|YMLjr1vShy{KX9>(23USq zazta!W*MKoqj`dct4d$7T)vGpUF1|!S{rB^d~nAf4n9cN) zR@=61yQkgL)8@2o+qTW=X&ci$ZQHhO+xD$}?mhSX&W`_a;7BofRMoNKD1zs#VWF_`pYIUDfO&M;(QL5fGYl_a(7QgB+Qw@s- zd_!)eW&{Lki*E{K>gO20D81k#WFTIc?dmBSg=Epu&fepCPec!<6;vF_RY8U zHi?lIM&ofu#)yL7=SajNrsM)$5>v7q$R!x9^2*6vWu<&e*aP)a+7gHrue|DiA{!nk z7>qJ-nJ6BB0elBn&wHYkwLP~Xvkle&UwmCf1!OxLADRXU-6O6awN=07Cru0P`Bap- zx5L5Y*kt_?Ei(Mk%DH4z%X3+jXiAfWn7}IhQg`c`koxDR6(%!Dtj3R;G_*2?9VGp- z65~vR4557_Z5ruSQ_6EUDJ&^;M-2O7g@KBX&-J!T=&4oG@RuOpx(b_mB&rbyO{CO4 z3YKKGJVBo(Utt*^4_GO1-@o;pS&>*ZSGMkGc3$aHQgYepm{u~0cR!Wybs1))|VU=>EwyhubR$;g=996frcDeU(Nri7lhsmdFjnqESQY0JqSK|Drf1# z)yQ0Z{#EZIt`h#Dlb95Q54_Xo^Swx8X^SAoxJn+T1i{Pe4LMFa1w*U$+y3f;Cnq|3 z>jZxg-?8isDe8?bYa5!E>wpLF--4;xpCNIgk?8l!H(N?!iF`Q2GJKd-hgWwFm5{zv=Sri4+3_mMDlR(?%T>4hgG zFFh)?cBxl#g9pU?H`Zp0P&)&ogD$G1`cyZ4VmU;euVM7B19AL6V* ztO`Du+upJc?I;`*)?W7cn|j+9?{3`=XL&2sXTQ;(X0)VpXO7tVmpVjGPwWU;d_Wo- z8eCpKKQw@^MmNQM-)ZP+g9l6qntmC&3lV&uC zB(U(IAeGu*z|hJKPM?W$r?rQy&X9%a<|)zLn1+zqZ_lR0p{nBK1ivqWy?GqTChIxa z+w|ZkpcEtbzyFFlo}4}O#hUPjwzQw3+@Bhp-_aO2+&%zc?Fu&v|>4*Afhg$kJfvjx8{Uuht`)M6>BYvOr^W%bFd|InoEaM%Tx&Q>Ih zSHNZ2YdrE>M#IfZpM$8!$=@|50;ZB>CiDgb`R99!cft>vd8zXSok)>Jq2+r+Xj!vt z`xl~jNrbPinZHn4To^iYnmfBASo={B4;16glc#jPk|S7N5>czuH*H8)@>7U%lgN)w zjsUmgkxG2RL(SY~htEV*&OEeU2(4P&N@SYS>Qj&(j1hEbka@zD*d6(bSAWS5#W2xh zQ58y8r{}SpQFg2%n_|GT?7(%b*HilrAkXOv3t*G(hmwx`WOk+pFUlRgx}?pP={5n^aM`Fm^Ui!2612tbL6^bC%D~}aHLNZ z5pXFTx_H{7R0}sCLX;WTFQ@<2IC;O&L$$}Ma_^q<^Em?tEuX5)sTr2_#w_teJ$k)M zZ9#xZMZ`J$rl`sb59{p+rO;z^8r|``-7iJ~&Z@Gzzn$1D9*=g`$9my;EQK!G!{8I< znpUx{q*%j$(#-aR81vf>KM_Zty`D)vKX{JZ)=oG*$1zx2x}cSU#Y=CB(Uha=pptRw zkOPlHwl(>W-Pg{-oul&2v-A|1G0&2xR74?0E6IX4mmi9c?oKq1*)Croe)x*`b^^ua zLv_EyHBGOojD5dQrj1;}1 zUw@-!K_o8u?N6qQ51(Fkki#GL+1&AlUO>(2lFV!tnoY)NPKq67UP11}XK0kQ;Leaz z4TTQa%r97j;38%yz>mpS;wQFSNzdB@Fq9G*B7caFseE>0;rhbx#g>q=U!z^132sf7t#U{4772W;E&M-)3FlKinJ9@DzF520=SW=_T7O7Ue)!p?s# z>w9dC^yany?PVi-#du2hk^(7BHY-?c$|>rj+jCX^I?gAo4PCpFod-Dy%oWGM^Hw%S z!PgV&Ya{7`*RzY?kuO3#DV-Xryzf)m`S9EghDNYvFwOuf`z1)}bViZ`M6z>S!yV4@ zXyj}w>OOc)hCdE->Zf-qX~z;6`2jJ?&7O?X(y%-T z42>`P{Ze#t=|Qv_u7^2o`w3iJz=i^rlOYmUa?C~T-0Pp9x<0`@64JDUj}CtJV(u<4 zDvmaRdqtyVBxA99Mr=JwB}bFE$;w_F!!gPyPhtqVtN`e%FOXb{Dhs!_fR&K8Eze~V z>!H6(+hUG076`bSp-;xAmWPw7hF6*a0W`ONXX`jHC1h8+g}gfcdLXbIf^a`2GdSN& zr2I*B2yu?UCNL6F;i1an?HPTeX(`|m{qi@oPJ~=u>C)BH+0Xu1EA;q0#c`)Zuwy3( zox_y$V^oJ3S`GY+R9;<$|J3MqF`t&1ut~LwZDhdKlld5JU8BdkB;NddAlB$>E^hXz zTKH`m%^3G6OJ?9z9B}`wKPvF{DUEyOJ=_p~OH#Iqt-M9?ufqaE!Nm7YI2Dt=;I3WL zM=SZb;PiMg2*_?BVCEtmkLe;jwas|rpc14c%R|~y#t&hw4KE4BsvL#1Fe=qIh-=no z7ZKf_HSU&cEog4ENe3?W2xim5VlG&3Qth02D-o-sQywMIQdZY<%f}r-HtUp5>sXIVDNhcC00LG=Uy`6&J*i_;qzPFn`ul zgWH$063QanJMk6UMZXQ@#N3h#N`j}O>!HT))#k+ib~)yx-xN8lwAWHP0jTQMCd{wA ztXR%RnoiiMloe04Q6^{Kbzew#8j&m!r>7(@@%7hpR(2W{(-*x$Yhwh&E;=g6G@-fG zx|sYs(=K#{@}N2WF)AY*wBj_b&2HJC={}q_B%-AP>k4Sp3$;p0Op?*-o>x><|5*Fp zeaE}H7)8HBx$Ypz|tD)+l6DLubf#2BpLd~z*rL%*909CJc0zhCF88uw zGjABWM_YJVd*OKp8z=`>u)OIGoQ_pi9fi+`GM%>+{8mHlEnocaKes*8)5lXg{ znEq$1ZkOPy?#f+A3$> zXxs~@F5x@ZD(_cpM4Ff`9Vm`F&-B)?o($gjWruOR3`iW|5G!wP!7#$Kr`2a%ln!81 zU{%AOidN#1Ty)FTWmy`98fE)J64SJiY@FW&^jcp%Jc^zY4|s#`LJylG%K$hP`(5j+ ztFLH-$^i1IDX&MU%%D7{mvLk48r_Auvz?Z%Tte~V_{Wl8Jq_eJ%uES(TTyJ+H*$o ze9$^n&Xz!{B7uEarmF!y%WH(%b4glt-I^-fT~KcB!cl)y9PQWpX^eu}1$PQGHG_g2 zK5e4p6qCapoHm*5*MhhB1yp*dzSgrwMS6e#1k;Y!O&1Sy zwlp&`RA7#FOj>2GbGS=0{gwrF9R_6=C#Or4DHi9jjJM3`;r)n*EyG!_52q>vBJEw| zXh35L14Cr46WifC3ym46Gc!(inv6U7D`xL;OGH~_%5Tr9?)i7OF=0y}VMxkuZkV8H zuutwblQ&|e1`-I}#D$D53UkAp)IsGonT$iP^y|--v)-eYdoeu6PZ=21B;}O@lM{$SS8mUdeqK!YK_{oX_$qU9nwAmW`UJ+}@5JM!t9=y>a9)wl13QcuXM*3?04Z1+4u{MTS))ZP}cs{O^I=c zi2UI5#9`0~LY#u>0{ixX?4_M1p>g9*Y=466D97PBxGdPeZHim2ttrvMNThNXkCII_ z2H1hTP(--v24O>0diXY=S(FXlUkI}Le^12CNU-T%w^@kb^VB>R0)?$atsG^u!rbUe zzlBZS7LUMoz;ir8DhFl^%e_XU!;6QF!ThUi;lbnn?(Ipii+>bRz}p>I5pEBx%qHjM zG(&HBIa$Ea01O7PV_zwPo9y1q7 zpX)4*&=(ZYpXe_KWPSa-p*C)vUnHPS?N@%#w`dlN4d7Qh$(iEA%LzxdWg%IWtfwtJ z5(m}c_4(1ko#Od!Y5JB)PB58N>cW5XaZpMCr*_?%9<;&-tJnLt0K8(2wQN#(M@wq(O zpl<#!WuR%tOCJBncp}SVb`(vwbwm^NJkcd~s)a;M{j8Fwy+=OgeCAd4Cr1f`@`VD0%uC0O= zA?EcatpSPkZ$%?3>E+78(*)Im0}4<(?y7C;<@qz05yC**OJ%_(suzheh*EW*QWpR( zDkdV%Pe?*-i*?)S@5}zg?!1D<(9Z~nbaK6fcH|zRpIC7mj{r1*Za*I+2qh!ZwU!nu zHWo%o5rN^TKvi@_YLiH=_T8~Hl$Zjg2TpZ8n`Xy2wuaB$K1s2qItrB*MN(gND%p`o z?8ddyl&D@Xo=PPIP7AWs6&21_CukfcYTF`(){FI*g{y*UhUP>Zqfagcwt*y~B>oy+ zE4)c$G?11*MfEA6{_o)p+TYNkaiarH?536crnNDnKOAJqTIZ07pINX@eVI{o0-Th3 ze~~k_(!)Jr5jtF>YeWruPejVpRw;*)r>6(`GU265GM9!P`VO5l^TW1l!KV{z$MZwY zf7z~uf)fUwQ}pG*vEU*~{xsse=eRvL|6_d+zZsNCrfWtR??fp0!e48Yep-pyaOzSN z3f*X+7ynn--VJxq)9aUSW(DQmIR#r%^nfJ}|VVO=~ z>IRo9W*P=B>znRsAap3St_Re6WGp;;aI#Mv%b)hyo}u$zrsvX}_``3;KJW91gmxfVlv*d2{;L~(8VJDn*!mnsPsZS+k*<}q%Nja$=OyuO4 z;8|W!_t%%}pca^Acdik_Xr@geUkAxW$rLbHyN=U8eLc4pfM{f=5ZHY|Xb?DoF#MpV z!jO7bqwVnqdOg0}A&xxw%CTKIR24HczRlBFD6Jz5bFGakYglCBa0N8fj5 z$?eYarGGy$w8{<7l<)*6({*P0L+GRU{N*s)?eCjoScWeasH)e+kz8<(l+%AR+h<Q`5e9gUTp zl-<~1XyvB3Dk)$D$M{?us=Vl9j8|!nzy({L22c3`ejAhEAZy#Dv{zngtsKG2N#w&8 z$5COy6Uotf$ADciYJg9zJ2&p`V=BVx^(1c?E`OEfQvMl^Lf=tUt3Fd(q27V-jl+QC z201#SUfHB8d5v|_QMXCH>GcJ^9_aMpW}zqpnFCdFa8py(8Q#4|2)eOfh?m}`EBz#? zIfwT^qBqOtD|se|r>XC4wrsOy$>B9!D2gbjTL5+0_q*yyC(2`|@qrVvT%`Ss%x+jF zC2N1ID0fVI=9VEzC!7>uf1|%Vf%BP??rmNtT9iogV3NWd&q(sY=ZIh%{-hN*>Y({-_T3wM~QV5>EvR(OOU!MrnXVow{p2CzPD zREAct;BOEE;h))q0^CULc z%s-xLOFC;u6>mGp+B+54G{M{6c>;64s@{O*&_R=)5i_6F(kUR+Cje1rzw;?>{&K-z zk*fS$$LDJPKG!Noc$72FHe+}G{Dq^f^isCXeUZ)CAOvn>A8=AK50n38`mO#LN}6Bl zk3o!V(kPlpkknr9@NFznfwJNG92zvZe9*Ljbc1X_*z(k*c<-y~V@hPM*0SNQmKdUk zk7!6DwuL(!Nx1!oMQ1T9g>!A@vneQ;+~s9o&W?7=+GtkbqSQVHJ!-CzA9GDqo8F1` zekA|8;q;`s>6L~48*^i&TP;U1BrMU`FossX!rGs z5cfQQbQo*?D(?!HT}1kz)|@qk+iaP9@tcAt85Tu^!?tG${2?4SPPCLC=EN(Ln6^sj z?6@}!Vk8l)9e(B%Hb8jXkpDWIh#M8T&-T*L4o+;>&MmE>2C>DrWjjIsceXpg0JtaVU4z6$F|U@)2?4*P$`k;J5BL zwIjc^4)+A`NHXsy-Bi|_%YBdAA-@f|*Zt@l$F-B_dxHC+@bxJ(UVBWWPE>BIYA%ph zPzoo9y+|zt|I&WD;$}Ly-PleO(-1)1SYN^t2UR_-04Qk!l*~6zDPE zqZ@(l4tqx{?#kcgk@1#;W&AEkI3V5_uG+L-X}%Za)_`Q0$T%9U4xk$Ujl#8{yS}oz zpN8g4YQY$}U=$5Ij!1N$W*180R$y<;k8}?XPpnk~>nja3m6k5TggO(6!k$q8z6j~>amzJAaMQ}p#cQ~Y&FPebSS>=-n01p3!m_K&)m z6Zz2ZXS2Q68uX)D14s}P7z|)8D7O^QOv$pkDL<-Y=hD7jy~+He?m9XXAg2EOa08*? z!TI8Ls(A!kVH?cZn#=|}0s7kJ`~8p*!@g0vFkXd5gQdOlnvKfUO^m70jH^4>^ZN;z zk$-ld<L8fR8lU zI#eBQA~BQ}N1YCL!~WsOj9j-vr)K07pYImD{YMHn*gFUYYY^sqPv|@G6)>1fz*!-) z1k`-FX!AJz<~B+l%v>Lvp*od2!&{Y>=diA2nH!twX@+m$SskTY`^6kH6<)zBw3lOe zdFlUtC$r)jo*LLD(ftLFF!o40pEpSXw~5`>uqDZ^WAs;9C%-N>2XE{-WbGJ&Ic(4c z4AX~_2Rw{)S(1^OeHh;CE1LB@vgh<0y=lv~c2EoVQW0aTrIzr^mSt(9jdjcP6{MQoXuU%)mIeL_tFpXnDqT)y`M#^y>o-oxV60 zBHU)E^^#lNwND7P0;!y^_xL1EX!#E)MdG>V9$hZ=jQh2cF0TbYYUbJpZ*?=-r2-Y8 zT{Q2N_ZF~kolWRV7 z1E^N@l;5ViJCp>7s}80iT;UzF>@eGG^)6~)3#N=jL&j1Qx%tSNXLB{QbYL}Y(V{91IktO+SsQPcgR!{;zZnqCej{ks(*YW@r;fnaXG1lufX;ResDkvZi|opGRx#lH9qqZlL9x&JuWCB#*40IVo1llnVTBlC>gu;fdG_KUY%rWM697hxYZXmhNDe*`&*5=ozKm&tD&$yWb5)x#0ZXTCW-eA zLJzs$cRN{f0wpn;mdCYxn2!&s?zL+=8TKsHSi@Nsodr5Uq$GdMlxfb6iuBJD)ZQAe)SD_cPts>B7T9yg zQPm0}@H(*k zXxisbfDP>{mw+In*`u%gnv{HC_t`9ie->oNmG}g_oi<*v%L|tP)ld~L17_u<`cEO; zHa9be=BtPfmh-9Q^j;gyy5~5h<9@l@p=T$Lr(1!v$Iq?#4%V(@8jTy(;a}w%8?>oN z_1i8qB5>->k_|7!9BpfS@jf%T0ezh!g8ulkNYp-$uXsq^AX0sE`bLKFeLnC4NTb;8 zX4v51Qv#Dn2q{ZB)3jH1cb@HF0XR26sevUdB$nxL7KbkqWtkVMf`A+1n$0nR@d3^s zJ0q#j$=RWkG#?MaDav;5945m-!7ymQeBlg((n(y!+TZbVhRQQ{z~wfOr_M!AZJOFT zC)br$t;{e#y_au}u>_v*Lr6q#yKX$TWsY;j=r7wk@?IYC<2XIP-(d;J zM=KqyFJ}ghf$beTeYC_T-knT-4h#(HzV{w=zB9CiTe*>AJ`R&!n>VRjkn4C5rHxY` zR-XB;CrsIE+O{U=4k3~aS$u-*LWf+xat@`1!uI{h`a3LjXrlaqOislf6Rg4EQI@W{ z@+*BCU7MAJN*N=YB6sXMwSuKMK9Y`>b_xRGfDTDY}(NO zyIfXQZsbq;&1-vb|2Y-W)=8B+UCj$hru}P^iIkeJ_D*GoIm8kRD_H}3HqN8YFCqA) zu2&eQ4$h9&x@zU+@~u$$XqH>Vql6PFJp(Qb(@Pp{`)yk8U;!}1k^_(Z4*Ydu(&Kcx z)vBE?0%Z*e=gJv~38$}M@V23wRXJ+xb{IRVmF~l{MqR}?Ud~&8BgZN+ZqzeI_YSp6 zJ?q^WDe13Gya~*St8*`&@BDlpq1NhCoA$!At>aNz`DI!;wJenW^FBe3=D-}bQ=K+$ zOqi1<9ON>T_ejY>D^;c;ny12d7#&&SS+E!{Ky|Iwg3EyTUP2$ejXV*<8A9_6%5*Zm zC!mwmON#(kdWt6hkyXsVt1=hvWo*%!#%$9jc#(A|uLr8++Vl5_x(rx*UVy%aZ^lkd}U-GM1KVN&)doA_4 zgBN60?SzG&8|ilNTl+2%590&$P$3TttTQnWtgX{z-!kKNBv#(=f_h+rxL?VBJyuNm zho4{KOMmx@EY@E>R)QKC_=cf5XvD`|=G6_X0GKRrVLCT*CLSY(p8&Iry9_8V*aBk} z$EwqCcK39)clEw?s{gqQAqffvzg?J!V8Fh3X?s%Dd-fjTW>){n3lW~UuVMlzwp#;R z#vr5ixc{*-^Xs~`bwMjHFfx60eX#n6);E$l+rO;4G-kM;&gqWA$Rk)8dAz%L3MALJ_m|%QXrt|{N(o6 zK9ad(T4z?eIRI9^S0d#0k5AJvVjd>pK;_~fH`z#WUwkJ87DR3PuFurSpQAhjrdG9B zg*OanHVL9cz4Ubo{hU`9HK9muRcTupe`WWj<|3kxe^)v9vC9Ax`YXR;Poj~+Pcduy zcSD{{#!Iirrj<3y1~8&(rU)DL;Ro{H3cLMjxcC$fiMj-fZ>G8I(Yu-klQ!lgv9v!A$+@QdkA)2*iny`2+Xp z{`}!=!#0HliBqZAm!bE^+pDF6Z%cb8pGQ*GD}|p#z+zwz69Jd{_f4=l&FjDeP~mC< zYOr$7a0+fwcehYp#xwNtzw5C$(Ub~~XYf~`Sd2i8DA)H^BB_9tL@Y*J0q5AL8;z2{h?q;qwF6`V zv9t6s9@M{~FVDXhjw_XQ3M7kvvIkIDIYh(%VQ0#JIQ`hgKSaJtrAwEM041<(ybwF# zj{K~F`lT%9Bl1lz1MyD+QJ{SNes9(Da^?LYf##THrYO3JlpVR11LPu(;I-@sl$~O7 z-x}QDA~;hjZB5`rsPrXgMt^?EEWp?Ghv=c2%+CH*bz#N3;_FPL^`5=}zK5NXTg!GM zjtnh7SpIvR3nhi%Mj_O%sZ@#~94INY5y^6PTq4*rT_e7b7yR9B)Aw0aL-K5aPbn|4 zx1PmBUOjyt9EohYk8j0c&y){*Zy=%W48bGggz zh-gzoNZO{;E@A?LWZ!NztB?BZyDlit%bZCs@5iD1T#M#){wSH7G1cs_97sg~NMgZm zu3~7XsaA&EkYgQd-Bb_%(96_6!BOb7REI!`(els-`GHi$ZkAUW zk~dN7RHsgBE!Aq@gFPLv+jnYLFqu^sJN~KPWB9^+rcMP97+*?^go&ck)VO@cWXJ^n z)gp=`}r-aXyGbd_N{R+ubZ{g_O4>UweQsu1VR5usJ zfhD<Gz2i%-W!&oq`<7`kE%g4{f&K|7#{`W zabJp3<(>05T}h$9D_2dt|3ZRlq*d7LmFNX6k=Smo~m@Q%!WatbLTup7NZT3!Ajump6+D`YW@lyy$^ z(AW*8oPeRj9KDToq+tbh@6Tz$j+6`7*}}hfrL5L`seZfR1*4rdL%WsrK8n^v9Q^aP zlW|&%YX^sZ8!TaI(eHRf&vSQ-oBembIx+TVyp|G>w9G&iV-1zu|Z?B=HIBrg65r5 zQ#LbuiMDfkpBIKskb!%GaI8uoCE3-{6Qhoh!4504W5 zHaqGvk_Fe1uIB{fT9D?0notDzp%x6ghvDp z0Ib6R06<~LasPk~4gg?dVWwwl zZQ}kPi14-zb*WeYK(#X9|C_@9<~CCR08S>(F8{j6{QFpY?cd#Gfb&~N3jkpMH{rj& zLxE;?|Haa?HF0*cbF|WPa<+3cF#At4o@?6|1OgnMxDEgS9Q!|jEe&kN+Qrev$-~y@ zKlD$FPYTKmoGVOV=l^?|m5hN*J4a_dTRUSDr~g2OoOR;61TNDakOyqxALz6NQeB+v zO>B+-1Mi>y)xh=m2lkzTetdRz)=o@j_AU(0CQi=(K3)GbTm6^A_5Xmt`@gyW@VowZ z`af*}|2Mrn;D6EoUw_trr~lI>;$Pl@{{!K$|8^4mcg{cWy}+&Z4>%Hh&%yGT(0 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl deleted file mode 100644 index ffeb167cfc..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl +++ /dev/null @@ -1,7 +0,0 @@ -# cgroup.conf -# https://slurm.schedmd.com/cgroup.conf.html - -ConstrainCores=yes -ConstrainRamSpace=yes -ConstrainSwapSpace=no -ConstrainDevices=yes diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl deleted file mode 100644 index 4951289842..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl +++ /dev/null @@ -1,67 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/slurm.conf.html -# https://slurm.schedmd.com/configurator.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxNodeCount=64000 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -AccountingStoreFlags=job_comment -JobAcctGatherFrequency=30 -JobAcctGatherType=jobacct_gather/cgroup -SlurmctldDebug=info -SlurmdDebug=info -DebugFlags=Power - -# -# -# TIMERS -MessageTimeout=60 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=120 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl deleted file mode 100644 index 8c90a9dfbe..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl +++ /dev/null @@ -1,31 +0,0 @@ -# slurmdbd.conf -# https://slurm.schedmd.com/slurmdbd.conf.html - -DebugLevel=info -PidFile=/var/run/slurm/slurmdbd.pid - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -AuthType=auth/{auth_key} -AuthAltTypes=auth/jwt -AuthAltParameters=jwt_key={state_save}/jwt_hs256.key - -DbdHost={control_host} - -LogFile={slurmlog}/slurmdbd.log - -SlurmUser=slurm - -StorageLoc={db_name} - -StorageType=accounting_storage/mysql -StorageHost={db_host} -StoragePort={db_port} -StorageUser={db_user} -StoragePass={db_pass} - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh deleted file mode 100644 index db514fc9e5..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -x /opt/apps/adm/slurm/slurm_epilog ]]; then - exec /opt/apps/adm/slurm/slurm_epilog -fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh deleted file mode 100644 index 37a91bb1ea..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -x /opt/apps/adm/slurm/slurm_prolog ]]; then - exec /opt/apps/adm/slurm/slurm_prolog -fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh deleted file mode 100644 index 0877ff3b19..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SLURM_EXTERNAL_ROOT="/opt/apps/adm/slurm" -SLURM_MUX_FILE="slurm_mux" - -mkdir -p "${SLURM_EXTERNAL_ROOT}" -mkdir -p "${SLURM_EXTERNAL_ROOT}/logs" -mkdir -p "${SLURM_EXTERNAL_ROOT}/etc" - -# create common prolog / epilog "multiplex" script -if [ ! -f "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" ]; then - # indentation matters in EOT below; do not blindly edit! - cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -CMD="${0##*/}" -# Locate script -BASE=$(readlink -f $0) -BASE=${BASE%/*} - -export CLUSTER_ADM_BASE=${BASE} - -# Source config file if it exists for extra DEBUG settings -# used below -SLURM_MUX_CONF=${CLUSTER_ADM_BASE}/etc/slurm_mux.conf -if [[ -r ${SLURM_MUX_CONF} ]]; then - source ${SLURM_MUX_CONF} -fi - -# Setup logging if configured and directory exists -LOGFILE="/dev/null" -if [[ -d ${DEBUG_SLURM_MUX_LOG_DIR} && ${DEBUG_SLURM_MUX_ENABLE_LOG} == "yes" ]]; then - LOGFILE="${DEBUG_SLURM_MUX_LOG_DIR}/${CMD}-${SLURM_SCRIPT_CONTEXT}-job-${SLURMD_NODENAME}.log" - exec >>${LOGFILE} 2>&1 -fi - -# Global scriptlets -for SCRIPTLET in ${BASE}/${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do - if [[ -x ${SCRIPTLET} ]]; then - echo "Running ${SCRIPTLET}" - ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 - echo "Running ${SCRIPTLET} returned $?" - fi -done - -# Per partition scriptlets -for SCRIPTLET in ${BASE}/partition-${SLURM_JOB_PARTITION}-${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do - if [[ -x ${SCRIPTLET} ]]; then - echo "Running ${SCRIPTLET}" - ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 - echo "Running ${SCRIPTLET} returned $?" - fi -done -EOT -fi - -# ensure proper permissions on slurm_mux script -chmod 0755 "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" - -# create default slurm_mux configuration file -if [ ! -f "${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" ]; then - cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" -# these settings are intended for temporary debugging purposes only; leaving -# them enabled will write files for each job to a shared NFS directory without -# any automated cleanup -DEBUG_SLURM_MUX_LOG_DIR=/opt/apps/adm/slurm/logs -DEBUG_SLURM_MUX_ENABLE_LOG=no -EOT -fi - -# create epilog symbolic link -if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_epilog" ]; then - cd ${SLURM_EXTERNAL_ROOT} - # delete existing file if necessary - rm -f slurm_epilog - ln -s ${SLURM_MUX_FILE} slurm_epilog - cd - >/dev/null -fi - -# create prolog symbolic link -if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_prolog" ]; then - cd ${SLURM_EXTERNAL_ROOT} - # delete existing file if necessary - rm -f slurm_prolog - ln -s ${SLURM_MUX_FILE} slurm_prolog - cd - >/dev/null -fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf deleted file mode 100644 index e63b2d1100..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf +++ /dev/null @@ -1,406 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - scripts_dir = abspath("${path.module}/scripts") - - bucket_dir = coalesce(var.bucket_dir, format("%s-files", var.slurm_cluster_name)) -} - -######## -# DATA # -######## - -data "google_storage_bucket" "this" { - name = var.bucket_name -} - -########## -# RANDOM # -########## - -resource "random_uuid" "cluster_id" { -} - -################## -# CLUSTER CONFIG # -################## - -locals { - config = { - enable_bigquery_load = var.enable_bigquery_load - cloudsql_secret = var.cloudsql_secret - cluster_id = random_uuid.cluster_id.result - project = var.project_id - slurm_cluster_name = var.slurm_cluster_name - enable_slurm_auth = var.enable_slurm_auth - bucket_path = local.bucket_path - enable_debug_logging = var.enable_debug_logging - extra_logging_flags = var.extra_logging_flags - controller_state_disk = var.controller_state_disk - - # storage - disable_default_mounts = var.disable_default_mounts - network_storage = var.network_storage - - # timeouts - controller_startup_scripts_timeout = var.controller_startup_scripts_timeout - compute_startup_scripts_timeout = var.compute_startup_scripts_timeout - - munge_mount = local.munge_mount - slurm_key_mount = var.slurm_key_mount - - # slurm conf - prolog_scripts = [for k, v in google_storage_bucket_object.prolog_scripts : k] - epilog_scripts = [for k, v in google_storage_bucket_object.epilog_scripts : k] - task_prolog_scripts = [for k, v in google_storage_bucket_object.task_prolog_scripts : k] - task_epilog_scripts = [for k, v in google_storage_bucket_object.task_epilog_scripts : k] - cloud_parameters = var.cloud_parameters - - # hybrid - hybrid = var.enable_hybrid - google_app_cred_path = var.enable_hybrid ? local.google_app_cred_path : null - output_dir = var.enable_hybrid ? local.output_dir : null - install_dir = var.enable_hybrid ? local.install_dir : null - slurm_control_host = var.enable_hybrid ? var.slurm_control_host : null - slurm_control_host_port = var.enable_hybrid ? local.slurm_control_host_port : null - slurm_control_addr = var.enable_hybrid ? var.slurm_control_addr : null - slurm_bin_dir = var.enable_hybrid ? local.slurm_bin_dir : null - slurm_log_dir = var.enable_hybrid ? local.slurm_log_dir : null - controller_network_attachment = var.controller_network_attachment - - - # config files templates - slurmdbd_conf_tpl = file(coalesce(var.slurmdbd_conf_tpl, "${local.etc_dir}/slurmdbd.conf.tpl")) - slurm_conf_tpl = var.slurm_conf_template != null ? var.slurm_conf_template : file(coalesce(var.slurm_conf_tpl, "${local.etc_dir}/slurm.conf.tpl")) - cgroup_conf_tpl = file(coalesce(var.cgroup_conf_tpl, "${local.etc_dir}/cgroup.conf.tpl")) - - # Providers - endpoint_versions = var.endpoint_versions - } - - x_nodeset = toset(var.nodeset[*].nodeset_name) - x_nodeset_dyn = toset(var.nodeset_dyn[*].nodeset_name) - x_nodeset_tpu = toset(var.nodeset_tpu[*].nodeset.nodeset_name) - x_nodeset_overlap = setintersection([], local.x_nodeset, local.x_nodeset_dyn, local.x_nodeset_tpu) - - etc_dir = abspath("${path.module}/etc") - - bucket_path = format("%s/%s", data.google_storage_bucket.this.url, local.bucket_dir) - - slurm_control_host_port = coalesce(var.slurm_control_host_port, "6818") - - google_app_cred_path = var.google_app_cred_path != null ? abspath(var.google_app_cred_path) : null - slurm_bin_dir = var.slurm_bin_dir != null ? abspath(var.slurm_bin_dir) : null - slurm_log_dir = var.slurm_log_dir != null ? abspath(var.slurm_log_dir) : null - - munge_mount = var.enable_hybrid ? { - server_ip = lookup(var.munge_mount, "server_ip", coalesce(var.slurm_control_addr, var.slurm_control_host)) - remote_mount = lookup(var.munge_mount, "remote_mount", "/etc/munge/") - fs_type = lookup(var.munge_mount, "fs_type", "nfs") - mount_options = lookup(var.munge_mount, "mount_options", "") - } : null - - output_dir = can(coalesce(var.output_dir)) ? abspath(var.output_dir) : abspath(".") - install_dir = can(coalesce(var.install_dir)) ? abspath(var.install_dir) : local.output_dir -} - -resource "google_storage_bucket_object" "config" { - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/config.yaml" - content = yamlencode(local.config) - source_md5hash = md5(yamlencode(local.config)) - - # Take dependency on all other "config artifacts" so creation of `config.yaml` - # can be used as a signal for setup.py that "everything is ready". - # Some of following files, particularly mount scripts for new NFSes, can take a while to be created. - depends_on = [ - google_storage_bucket_object.controller_startup_scripts, - google_storage_bucket_object.nodeset_startup_scripts, - google_storage_bucket_object.prolog_scripts, - google_storage_bucket_object.epilog_scripts, - google_storage_bucket_object.task_prolog_scripts, - google_storage_bucket_object.task_epilog_scripts - ] -} - -resource "google_storage_bucket_object" "nodeset_config" { - for_each = { for ns in var.nodeset : ns.nodeset_name => merge(ns, { - instance_properties = jsondecode(ns.instance_properties_json) - }) } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -resource "google_storage_bucket_object" "nodeset_dyn_config" { - for_each = { for ns in var.nodeset_dyn : ns.nodeset_name => ns } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_dyn_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -resource "google_storage_bucket_object" "nodeset_tpu_config" { - for_each = { for n in var.nodeset_tpu[*].nodeset : n.nodeset_name => n } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_tpu_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -######### -# DEVEL # -######### - -locals { - build_dir = abspath("${path.module}/build") - - slurm_gcp_devel_controller_zip = "slurm-gcp-devel-controller.zip" - slurm_gcp_devel_compute_zip = "slurm-gcp-devel.zip" - slurm_gcp_devel_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_controller_zip) - slurm_gcp_devel_compute_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_compute_zip) - - controller_files = [ - "tools/gpu-test", - "tools/task-epilog", - "tools/task-prolog", - "conf.py", - "file_cache.py", - "get_tpu_vmcount.py", - "job_submit.lua.tpl", - "load_bq.py", - "local_pubsub.py", - "mig_flex.py", - "resume_wrapper.sh", - "resume.py", - "setup_network_storage.py", - "setup.py", - "slurmsync.py", - "sort_nodes.py", - "suspend_wrapper.sh", - "suspend.py", - "tpu.py", - "util.py", - "watch_delete_vm_op.py", - ] - - compute_files = [ - "tools/gpu-test", - "tools/task-epilog", - "tools/task-prolog", - "file_cache.py", - "get_tpu_vmcount.py", - "job_submit.lua.tpl", - "local_pubsub.py", - "mig_flex.py", - "setup_network_storage.py", - "setup.py", - "slurmsync.py", - "sort_nodes.py", - "suspend.py", - "tpu.py", - "util.py", - "watch_delete_vm_op.py", - ] -} - -data "archive_file" "slurm_gcp_devel_controller_zip" { - output_path = "${local.build_dir}/${local.slurm_gcp_devel_controller_zip}" - type = "zip" - - dynamic "source" { - for_each = local.controller_files - content { - content = file("${local.scripts_dir}/${source.value}") - filename = source.value - } - } -} - -data "archive_file" "slurm_gcp_devel_compute_zip" { - output_path = "${local.build_dir}/${local.slurm_gcp_devel_compute_zip}" - type = "zip" - - dynamic "source" { - for_each = local.compute_files - content { - content = file("${local.scripts_dir}/${source.value}") - filename = source.value - } - } -} - -resource "google_storage_bucket_object" "devel" { - bucket = var.bucket_name - name = local.slurm_gcp_devel_zip_bucket - source = data.archive_file.slurm_gcp_devel_controller_zip.output_path - source_md5hash = data.archive_file.slurm_gcp_devel_controller_zip.output_md5 -} - -resource "google_storage_bucket_object" "devel_compute" { - bucket = var.bucket_name - name = local.slurm_gcp_devel_compute_zip_bucket - source = data.archive_file.slurm_gcp_devel_compute_zip.output_path - source_md5hash = data.archive_file.slurm_gcp_devel_compute_zip.output_md5 -} - -########### -# SCRIPTS # -########### - -resource "google_storage_bucket_object" "controller_startup_scripts" { - for_each = { - for x in local.controller_startup_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-controller-script-%s", local.bucket_dir, each.key) - content = each.value.content - source_md5hash = md5(each.value.content) -} - -resource "google_storage_bucket_object" "nodeset_startup_scripts" { - for_each = { for x in flatten([ - for nodeset, scripts in var.nodeset_startup_scripts - : [for s in scripts - : { - content = s.content, - name = format("slurm-nodeset-%s-script-%s", nodeset, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_")) } - ]]) : x.name => x.content } - - bucket = var.bucket_name - name = format("%s/%s", local.bucket_dir, each.key) - content = each.value - source_md5hash = md5(each.value) -} - -resource "google_storage_bucket_object" "prolog_scripts" { - for_each = { - for x in local.prolog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-prolog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -resource "google_storage_bucket_object" "epilog_scripts" { - for_each = { - for x in local.epilog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-epilog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -resource "google_storage_bucket_object" "task_prolog_scripts" { - for_each = { - for x in local.task_prolog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-task_prolog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -resource "google_storage_bucket_object" "task_epilog_scripts" { - for_each = { - for x in local.task_epilog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-task_epilog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -############################ -# DATA: CHS GPU HEALTH CHECK -############################ - -data "local_file" "chs_gpu_health_check" { - filename = "${path.module}/scripts/tools/gpu-test" -} - -################################ -# DATA: EXTERNAL PROLOG/EPILOG # -################################ - -data "local_file" "external_epilog" { - filename = "${path.module}/files/external_epilog.sh" -} - -data "local_file" "external_prolog" { - filename = "${path.module}/files/external_prolog.sh" -} - -data "local_file" "setup_external" { - filename = "${path.module}/files/setup_external.sh" -} - -locals { - external_epilog = [{ - filename = "z_external_epilog.sh" - content = data.local_file.external_epilog.content - source = null - }] - external_prolog = [{ - filename = "z_external_prolog.sh" - content = data.local_file.external_prolog.content - source = null - }] - setup_external = [{ - filename = "z_setup_external.sh" - content = data.local_file.setup_external.content - }] - chs_gpu_health_check = [{ - filename = "a_chs_gpu_health_check.sh" - content = data.local_file.chs_gpu_health_check.content - source = null - }] - - chs_prolog = var.enable_chs_gpu_health_check_prolog ? local.chs_gpu_health_check : [] - ext_prolog = var.enable_external_prolog_epilog ? local.external_prolog : [] - prolog_scripts = concat(local.chs_prolog, local.ext_prolog, var.prolog_scripts) - task_prolog_scripts = var.task_prolog_scripts - - chs_epilog = var.enable_chs_gpu_health_check_epilog ? local.chs_gpu_health_check : [] - ext_epilog = var.enable_external_prolog_epilog ? local.external_epilog : [] - epilog_scripts = concat(local.chs_epilog, local.ext_epilog, var.epilog_scripts) - task_epilog_scripts = var.task_epilog_scripts - - controller_startup_scripts = var.enable_external_prolog_epilog ? concat(local.setup_external, var.controller_startup_scripts) : var.controller_startup_scripts - - -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf deleted file mode 100644 index 111c997d62..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "slurm_bucket_path" { - description = "GCS Bucket URI of Slurm cluster file storage." - value = local.bucket_path -} - -output "bucket_name" { - description = "GCS Bucket name of Slurm cluster file storage." - value = data.google_storage_bucket.this.name -} - -output "bucket_dir" { - description = "Path directory within `bucket_name` for Slurm cluster file storage." - value = local.bucket_dir -} - -output "config" { - description = "Cluster configuration." - value = local.config - - precondition { - condition = var.enable_hybrid ? can(coalesce(var.slurm_control_host)) : true - error_message = "Input slurm_control_host is required." - } - - precondition { - condition = length(local.x_nodeset_overlap) == 0 - error_message = "All nodeset names must be unique among all nodeset types." - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py deleted file mode 100644 index 89ceefa3df..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py +++ /dev/null @@ -1,658 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional, Iterable, Dict, Set, Tuple -from itertools import chain -from collections import defaultdict -import json -from pathlib import Path -import util -from util import dirs, slurmdirs -import tpu -from addict import Dict as NSDict # type: ignore - -FILE_PREAMBLE = """ -# Warning: -# This file is managed by a script. Manual modifications will be overwritten. -""" - - - -def dict_to_conf(conf, delim=" ") -> str: - """convert dict to delimited slurm-style key-value pairs""" - - def filter_conf(pair): - k, v = pair - if isinstance(v, list): - v = ",".join(str(el) for el in v if el is not None) - return k, (v if bool(v) or v == 0 else None) - - return delim.join( - f"{k}={v}" for k, v in map(filter_conf, conf.items()) if v is not None - ) - - -TOPOLOGY_PLUGIN_TREE = "topology/tree" - -def topology_plugin(lkp: util.Lookup) -> str: - """ - Returns configured topology plugin, defaults to `topology/tree`. - """ - cp, key = lkp.cfg.cloud_parameters, "topology_plugin" - if key not in cp or cp[key] is None: - return TOPOLOGY_PLUGIN_TREE - return cp[key] - -def conflines(lkp: util.Lookup) -> str: - params = lkp.cfg.cloud_parameters - def get(key, default): - """ - Returns the value of the key in params if it exists and is not None, - otherwise returns supplied default. - We can't rely on the `dict.get` method because the value could be `None` as - well as empty NSDict, depending on type of the `cfg.cloud_parameters`. - TODO: Simplify once NSDict is removed from the codebase. - """ - if key not in params or params[key] is None: - return default - return params[key] - - no_comma_params = get("no_comma_params", False) - - any_gpus = any( - lkp.template_info(nodeset.instance_template).gpu - for nodeset in lkp.cfg.nodeset.values() - ) - - any_tpu = any( - tpu_nodeset is not None - for part in lkp.cfg.partitions.values() - for tpu_nodeset in part.partition_nodeset_tpu - ) - - any_gke = any( - lkp.nodeset_is_gke(nodeset) - for nodeset in lkp.cfg.nodeset.values() - ) - - any_dynamic = any(bool(p.partition_feature) for p in lkp.cfg.partitions.values()) - comma_params = { - "LaunchParameters": [ - "enable_nss_slurm", - "use_interactive_step", - ], - "SlurmctldParameters": [ - "cloud_reg_addrs" if any_dynamic or any_tpu or any_gke else "cloud_dns", - "enable_configless", - "idle_on_node_suspend", - ], - "GresTypes": [ - "gpu" if any_gpus else None, - ], - } - - scripts_dir = lkp.cfg.install_dir or dirs.scripts - prolog_path = Path(dirs.custom_scripts / "prolog.d") - epilog_path = Path(dirs.custom_scripts / "epilog.d") - task_prolog_path = Path(dirs.custom_scripts / "task_prolog.d") - task_epilog_path = Path(dirs.custom_scripts / "task_epilog.d") - default_tree_width = 65533 if any_dynamic else 128 - - conf_options = { - **(comma_params if not no_comma_params else {}), - "Prolog": f"{prolog_path}/*" if lkp.cfg.prolog_scripts else None, - "Epilog": f"{epilog_path}/*" if lkp.cfg.epilog_scripts else None, - "TaskProlog": f"{task_prolog_path}/task-prolog" if lkp.cfg.task_prolog_scripts else None, - "TaskEpilog": f"{task_epilog_path}/task-epilog" if lkp.cfg.task_epilog_scripts else None, - "PrologFlags": get("prolog_flags", None), - "SwitchType": get("switch_type", None), - "PrivateData": get("private_data", []), - "SchedulerParameters": get("scheduler_parameters", [ - "bf_continue", - "salloc_wait_nodes", - "ignore_prefer_validation", - ]), - "ResumeProgram": f"{scripts_dir}/resume_wrapper.sh", - "ResumeFailProgram": f"{scripts_dir}/suspend_wrapper.sh", - "ResumeRate": get("resume_rate", 0), - "ResumeTimeout": get("resume_timeout", 300), - "SuspendProgram": f"{scripts_dir}/suspend_wrapper.sh", - "SuspendRate": get("suspend_rate", 0), - "SuspendTimeout": get("suspend_timeout", 300), - "SlurmdTimeout": get("slurmd_timeout", 300), - "UnkillableStepTimeout": get("unkillable_step_timeout", 300), - "TreeWidth": get("tree_width", default_tree_width), - "JobSubmitPlugins": "lua" if any_tpu else None, - "TopologyPlugin": topology_plugin(lkp), - "TopologyParam": get("topology_param", "SwitchAsNodeRank"), - } - return dict_to_conf(conf_options, delim="\n") - - - - -def nodeset_lines(nodeset, lkp: util.Lookup) -> str: - template_info = lkp.template_info(nodeset.instance_template) - machine_conf = lkp.template_machine_conf(nodeset.instance_template) - - # follow https://slurm.schedmd.com/slurm.conf.html#OPT_Boards - # by setting Boards, SocketsPerBoard, CoresPerSocket, and ThreadsPerCore - gres = f"gpu:{template_info.gpu.count}" if template_info.gpu else None - node_conf = { - "RealMemory": machine_conf.memory, - "Boards": machine_conf.boards, - "SocketsPerBoard": machine_conf.sockets_per_board, - "CoresPerSocket": machine_conf.cores_per_socket, - "ThreadsPerCore": machine_conf.threads_per_core, - "CPUs": machine_conf.cpus, - "Gres": gres, - **nodeset.node_conf, - } - nodelist = lkp.nodelist(nodeset) - - return "\n".join( - map( - dict_to_conf, - [ - {"NodeName": nodelist, "State": "CLOUD", **node_conf}, - {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, - ], - ) - ) - - -def nodeset_tpu_lines(nodeset, lkp: util.Lookup) -> str: - nodelist = lkp.nodelist(nodeset) - return "\n".join( - map( - dict_to_conf, - [ - {"NodeName": nodelist, "State": "CLOUD", **nodeset.node_conf}, - {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, - ], - ) - ) - - -def nodeset_dyn_lines(nodeset): - """generate slurm NodeSet definition for dynamic nodeset""" - return dict_to_conf( - {"NodeSet": nodeset.nodeset_name, "Feature": nodeset.nodeset_feature} - ) - - -def partitionlines(partition, lkp: util.Lookup) -> str: - """Make a partition line for the slurm.conf""" - MIN_MEM_PER_CPU = 100 - - def defmempercpu(nodeset_name: str) -> int: - nodeset = lkp.cfg.nodeset.get(nodeset_name) - template = nodeset.instance_template - machine = lkp.template_machine_conf(template) - mem_spec_limit = int(nodeset.node_conf.get("MemSpecLimit", 0)) - return max(MIN_MEM_PER_CPU, (machine.memory - mem_spec_limit) // machine.cpus) - - defmem = min( - map(defmempercpu, partition.partition_nodeset), default=MIN_MEM_PER_CPU - ) - - nodesets = list( - chain( - partition.partition_nodeset, - partition.partition_nodeset_dyn, - partition.partition_nodeset_tpu, - ) - ) - - is_tpu = len(partition.partition_nodeset_tpu) > 0 - is_dyn = len(partition.partition_nodeset_dyn) > 0 - - oversub_exlusive = partition.enable_job_exclusive or is_tpu - power_down_on_idle = partition.enable_job_exclusive and not is_dyn - - line_elements = { - "PartitionName": partition.partition_name, - "Nodes": ",".join(nodesets), - "State": "UP", - "DefMemPerCPU": defmem, - "SuspendTime": 300, - "Oversubscribe": "Exclusive" if oversub_exlusive else None, - "PowerDownOnIdle": "YES" if power_down_on_idle else None, - **partition.partition_conf, - } - - return dict_to_conf(line_elements) - - -def suspend_exc_lines(lkp: util.Lookup) -> Iterable[str]: - static_nodelists = [] - for ns in lkp.power_managed_nodesets(): - if ns.node_count_static: - nodelist = lkp.nodelist_range(ns.nodeset_name, 0, ns.node_count_static) - static_nodelists.append(nodelist) - suspend_exc_nodes = {"SuspendExcNodes": static_nodelists} - - dyn_parts = [ - p.partition_name - for p in lkp.cfg.partitions.values() - if len(p.partition_nodeset_dyn) > 0 - ] - suspend_exc_parts = {"SuspendExcParts": [*dyn_parts]} - - return filter( - None, - [ - dict_to_conf(suspend_exc_nodes) if static_nodelists else None, - dict_to_conf(suspend_exc_parts), - ], - ) - - -def make_cloud_conf(lkp: util.Lookup) -> str: - """generate cloud.conf snippet""" - lines = [ - FILE_PREAMBLE, - conflines(lkp), - *(nodeset_lines(n, lkp) for n in lkp.cfg.nodeset.values()), - *(nodeset_dyn_lines(n) for n in lkp.cfg.nodeset_dyn.values()), - *(nodeset_tpu_lines(n, lkp) for n in lkp.cfg.nodeset_tpu.values()), - *(partitionlines(p, lkp) for p in lkp.cfg.partitions.values()), - *(suspend_exc_lines(lkp)), - ] - return "\n\n".join(filter(None, lines)) - - -def gen_cloud_conf(lkp: util.Lookup) -> None: - content = make_cloud_conf(lkp) - - conf_file = lkp.etc_dir / "cloud.conf" - conf_file.write_text(content) - util.chown_slurm(conf_file, mode=0o644) - - -def install_slurm_conf(lkp: util.Lookup) -> None: - """install slurm.conf""" - if lkp.cfg.ompi_version: - mpi_default = "pmi2" - else: - mpi_default = "none" - - conf_options = { - "name": lkp.cfg.slurm_cluster_name, - "control_addr": lkp.control_addr if lkp.control_addr else lkp.hostname_fqdn, - "control_host": lkp.control_host, - "accounting_storage_host": lkp.control_addr if lkp.cfg.controller_network_attachment else lkp.control_host, - "control_host_port": lkp.control_host_port, - "scripts": dirs.scripts, - "slurmlog": dirs.log, - "state_save": slurmdirs.state, - "mpi_default": mpi_default, - "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", - } - - conf = lkp.cfg.slurm_conf_tpl.format(**conf_options) - - conf_file = lkp.etc_dir / "slurm.conf" - conf_file.write_text(conf) - util.chown_slurm(conf_file, mode=0o644) - - -def install_slurmdbd_conf(lkp: util.Lookup) -> None: - """install slurmdbd.conf""" - conf_options = { - "control_host": lkp.control_host, - "slurmlog": dirs.log, - "state_save": slurmdirs.state, - "db_name": "slurm_acct_db", - "db_user": "slurm", - "db_pass": '""', - "db_host": "localhost", - "db_port": "3306", - "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", - } - - if lkp.cfg.cloudsql_secret: - secret_name = f"{lkp.cfg.slurm_cluster_name}-slurm-secret-cloudsql" - payload = json.loads(util.access_secret_version(lkp.project, secret_name)) - - if payload["db_name"] and payload["db_name"] != "": - conf_options["db_name"] = payload["db_name"] - if payload["user"] and payload["user"] != "": - conf_options["db_user"] = payload["user"] - if payload["password"] and payload["password"] != "": - conf_options["db_pass"] = payload["password"] - - db_host_str = payload["server_ip"].split(":") - if db_host_str[0]: - conf_options["db_host"] = db_host_str[0] - conf_options["db_port"] = ( - db_host_str[1] if len(db_host_str) >= 2 else "3306" - ) - - conf = lkp.cfg.slurmdbd_conf_tpl.format(**conf_options) - - conf_file = lkp.etc_dir / "slurmdbd.conf" - conf_file.write_text(conf) - util.chown_slurm(conf_file, 0o600) - - -def install_cgroup_conf(lkp: util.Lookup) -> None: - """install cgroup.conf""" - conf_file = lkp.etc_dir / "cgroup.conf" - conf_file.write_text(lkp.cfg.cgroup_conf_tpl) - util.chown_slurm(conf_file, mode=0o600) - - -def install_jobsubmit_lua(lkp: util.Lookup) -> None: - """install job_submit.lua if there are tpu nodes in the cluster""" - if not any( - tpu_nodeset is not None - for part in lkp.cfg.partitions.values() - for tpu_nodeset in part.partition_nodeset_tpu - ): - return # No TPU partitions, no need for job_submit.lua - - scripts_dir = lkp.cfg.slurm_scripts_dir or dirs.scripts - tpl = (scripts_dir / "job_submit.lua.tpl").read_text() - conf = tpl.format(scripts_dir=scripts_dir) - - conf_file = lkp.etc_dir / "job_submit.lua" - conf_file.write_text(conf) - util.chown_slurm(conf_file, 0o600) - - -def gen_cloud_gres_conf_lines(lkp: util.Lookup) -> str: - """generate cloud_gres.conf's content""" - - gpu_nodes = defaultdict(list) - for nodeset in lkp.cfg.nodeset.values(): - ti = lkp.template_info(nodeset.instance_template) - gpu_count = ti.gpu.count if ti.gpu else 0 - gpu_type = ti.gpu.type if ti.gpu else None - if gpu_count: - gpu_nodes[(gpu_count, gpu_type)].append(lkp.nodelist(nodeset)) - - lines = [ - dict_to_conf( - { - "NodeName": names, - "Name": "gpu", - "Type": gpu_type, - "File": "/dev/nvidia{}".format(f"[0-{gpu_count-1}]" if gpu_count > 1 else "0"), - } - ) - for (gpu_count, gpu_type), names in gpu_nodes.items() - ] - lines.append("\n") - return "\n".join(lines) - - -def gen_cloud_gres_conf(lkp: util.Lookup) -> None: - """create cloud_gres.conf file""" - - content = FILE_PREAMBLE + gen_cloud_gres_conf_lines(lkp) - - conf_file = lkp.etc_dir / "cloud_gres.conf" - conf_file.write_text(content) - util.chown_slurm(conf_file, mode=0o600) - - -def install_gres_conf(lkp: util.Lookup) -> None: - conf_file = lkp.etc_dir / "cloud_gres.conf" - gres_conf = lkp.etc_dir / "gres.conf" - if not gres_conf.exists(): - gres_conf.symlink_to(conf_file) - util.chown_slurm(gres_conf, mode=0o600) - - -class Switch: - """ - Represents a switch in the topology.conf file. - NOTE: It's class user job to make sure that there is no leaf-less Switches in the tree - """ - - def __init__( - self, - name: str, - nodes: Optional[Iterable[str]] = None, - switches: Optional[Dict[str, "Switch"]] = None, - ): - self.name = name - self.nodes = nodes or [] - self.switches = switches or {} - - def conf_line(self) -> str: - d = {"SwitchName": self.name} - if self.nodes: - d["Nodes"] = util.to_hostlist(self.nodes) - if self.switches: - d["Switches"] = util.to_hostlist(self.switches.keys()) - return dict_to_conf(d) - - def render_conf_lines(self) -> Iterable[str]: - yield self.conf_line() - for s in sorted(self.switches.values(), key=lambda s: s.name): - yield from s.render_conf_lines() - -class TopologySummary: - """ - Represents a summary of the topology, to make judgements about changes. - To be stored in JSON file along side of topology.conf to simplify parsing. - """ - def __init__( - self, - physical_host: Optional[Dict[str, str]] = None, - down_nodes: Optional[Iterable[str]] = None, - tpu_nodes: Optional[Iterable[str]] = None, - ) -> None: - self.physical_host = physical_host or {} - self.down_nodes = set(down_nodes or []) - self.tpu_nodes = set(tpu_nodes or []) - - - @classmethod - def path(cls, lkp: util.Lookup) -> Path: - return lkp.etc_dir / "cloud_topology.summary.json" - - @classmethod - def loads(cls, s: str) -> "TopologySummary": - d = json.loads(s) - return cls( - physical_host=d.get("physical_host"), - down_nodes=d.get("down_nodes"), - tpu_nodes=d.get("tpu_nodes"), - ) - - @classmethod - def load(cls, lkp: util.Lookup) -> "TopologySummary": - p = cls.path(lkp) - if not p.exists(): - return cls() # Return empty instance - return cls.loads(p.read_text()) - - def dumps(self) -> str: - return json.dumps( - { - "physical_host": self.physical_host, - "down_nodes": list(self.down_nodes), - "tpu_nodes": list(self.tpu_nodes), - }, - indent=2) - - def dump(self, lkp: util.Lookup) -> None: - TopologySummary.path(lkp).write_text(self.dumps()) - - def _nodenames(self) -> Set[str]: - return set(self.physical_host) | self.down_nodes | self.tpu_nodes - - def requires_reconfigure(self, prev: "TopologySummary") -> bool: - """ - Reconfigure IFF one of the following occurs: - * A node is added - * A node get a non-empty physicalHost - """ - if len(self._nodenames() - prev._nodenames()) > 0: - return True - for n, ph in self.physical_host.items(): - if ph and ph != prev.physical_host.get(n): - return True - return False - -class TopologyBuilder: - def __init__(self) -> None: - self._r = Switch("") # fake root, not part of the tree - self.summary = TopologySummary() - - def add(self, path: List[str], nodes: Iterable[str]) -> None: - n = self._r - assert path - for p in path: - n = n.switches.setdefault(p, Switch(p)) - n.nodes = [*n.nodes, *nodes] - - def render_conf_lines(self) -> Iterable[str]: - if not self._r.switches: - return [] # type: ignore - for s in sorted(self._r.switches.values(), key=lambda s: s.name): - yield from s.render_conf_lines() - - def compress(self) -> "TopologyBuilder": - compressed = TopologyBuilder() - compressed.summary = self.summary - def _walk( - u: Switch, c: Switch - ): # u: uncompressed node, c: its counterpart in compressed tree - pref = f"{c.name}_" if c != compressed._r else "s" - for i, us in enumerate(sorted(u.switches.values(), key=lambda s: s.name)): - cs = Switch(f"{pref}{i}", nodes=us.nodes) - c.switches[cs.name] = cs - _walk(us, cs) - - _walk(self._r, compressed._r) - return compressed - - -def add_tpu_nodeset_topology(nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup): - tpuobj = tpu.TPU.make(nodeset.nodeset_name, lkp) - static, dynamic = lkp.nodenames(nodeset) - - pref = ["tpu-root", f"ns_{nodeset.nodeset_name}"] - if tpuobj.vmcount == 1: # Put all nodes in one switch - all_nodes = list(chain(static, dynamic)) - bldr.add(pref, all_nodes) - bldr.summary.tpu_nodes.update(all_nodes) - return - - # Chunk nodes into sub-switches of size `vmcount` - chunk_num = 0 - for nodenames in (static, dynamic): - for nodeschunk in util.chunked(nodenames, n=tpuobj.vmcount): - chunk_name = f"{nodeset.nodeset_name}-{chunk_num}" - chunk_num += 1 - bldr.add([*pref, chunk_name], nodeschunk) - bldr.summary.tpu_nodes.update(nodeschunk) - -_SLURM_TOPO_ROOT = "slurm-root" - -def _make_physical_path(physical_host: str) -> List[str]: - assert physical_host.startswith("/"), f"Unexpected physicalHost: {physical_host}" - parts = physical_host[1:].split("/") - # Due to issues with Slurm's topology plugin, we can not use all components of `physicalHost`, - # trim it down to `cluster/rack`. - short_path = parts[:2] - return [_SLURM_TOPO_ROOT, *short_path] - -def add_nodeset_topology( - nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup -) -> None: - up_nodes = set() - default_path = [_SLURM_TOPO_ROOT, f"ns_{nodeset.nodeset_name}"] - - for inst in lkp.instances().values(): - try: - if lkp.node_nodeset_name(inst.name) != nodeset.nodeset_name: - continue - except Exception: - continue - - phys_host = inst.resource_status.physical_host or "" - bldr.summary.physical_host[inst.name] = phys_host - up_nodes.add(inst.name) - - if phys_host: - bldr.add(_make_physical_path(phys_host), [inst.name]) - else: - bldr.add(default_path, [inst.name]) - - down_nodes = [] - for node in chain(*lkp.nodenames(nodeset)): - if node not in up_nodes: - down_nodes.append(node) - if down_nodes: - bldr.add(default_path, down_nodes) - bldr.summary.down_nodes.update(down_nodes) - -def gen_topology(lkp: util.Lookup) -> TopologyBuilder: - bldr = TopologyBuilder() - for ns in lkp.cfg.nodeset_tpu.values(): - add_tpu_nodeset_topology(ns, bldr, lkp) - for ns in lkp.cfg.nodeset.values(): - add_nodeset_topology(ns, bldr, lkp) - return bldr - -def gen_topology_conf(lkp: util.Lookup) -> Tuple[bool, TopologySummary]: - """ - Generates slurm topology.conf. - Returns whether the topology.conf got updated. - """ - topo = gen_topology(lkp).compress() - conf_file = lkp.etc_dir / "cloud_topology.conf" - - with open(conf_file, "w") as f: - f.writelines(FILE_PREAMBLE + "\n") - for line in topo.render_conf_lines(): - f.write(line) - f.write("\n") - f.write("\n") - - prev_summary = TopologySummary.load(lkp) - return topo.summary.requires_reconfigure(prev_summary), topo.summary - -def install_topology_conf(lkp: util.Lookup) -> None: - conf_file = lkp.etc_dir / "cloud_topology.conf" - summary_file = lkp.etc_dir / "cloud_topology.summary.json" - topo_conf = lkp.etc_dir / "topology.conf" - - if not topo_conf.exists(): - topo_conf.symlink_to(conf_file) - - util.chown_slurm(conf_file, mode=0o600) - util.chown_slurm(summary_file, mode=0o600) - - -def gen_controller_configs(lkp: util.Lookup) -> None: - install_slurm_conf(lkp) - install_slurmdbd_conf(lkp) - gen_cloud_conf(lkp) - gen_cloud_gres_conf(lkp) - install_gres_conf(lkp) - install_cgroup_conf(lkp) - install_jobsubmit_lua(lkp) - - if topology_plugin(lkp) == TOPOLOGY_PLUGIN_TREE: - _, summary = gen_topology_conf(lkp) - summary.dump(lkp) - install_topology_conf(lkp) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py deleted file mode 100644 index cd2e41e5af..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any -from pathlib import Path -import shutil -import pickle - -import logging -log = logging.getLogger() - -# Can't reuse tool from util.py to avoid circular dependencies -# TODO: break down util.py for better modularity. -def _chown_slurm(path: Path) -> None: - shutil.chown(path, user="slurm", group="slurm") - -class FileCache: - def __init__(self, path: Path): - self.path = path - - def get(self, key: str) -> Any | None: - p = self.path / key - if not p.exists(): - return None - - try: - with p.open("rb") as f: - return pickle.load(f) - - except Exception as e: - log.warning(f"Failed to read cached value at {p}: {e}") - return None - - def set(self, key: str, data: Any) -> None: - p = self.path / key - - try: - # Create & chown before writing to minimize chances - # of ending up with root-owned corrupted file that can't be cleaned up - # TODO: restrict usage of cache by root to avoid all this complexity - # or have a cache per user. - p.touch(exist_ok=True) - _chown_slurm(p) - with p.open("wb") as f: - pickle.dump(data, f) - - except Exception as e: - log.warning(f"Failed to write cached value at {p}: {e}") - - -class NoCache: - def get(self, key: str) -> Any: - log.warning("No cache used") - return None - - def set(self, key: str, data: Any) -> None: - log.warning("No cache used") - - -def cache(name: str) -> FileCache | NoCache: - try: - path = Path("/tmp/slurm_gcp_cache/") / name - if not path.exists(): - path.mkdir(exist_ok=True, parents=True) - _chown_slurm(path) - return FileCache(path) - except: - log.exception(f"Failed to create cache, fallback to NoCache") - return NoCache() diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py deleted file mode 100644 index df0fd8ebe0..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright 2024 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import util -import tpu - - -def get_vmcount_of_tpu_part(part): - res = 0 - lkp = util.lookup() - for ns in lkp.cfg.partitions[part].partition_nodeset_tpu: - tpu_obj = tpu.TPU.make(ns, lkp) - if res == 0: - res = tpu_obj.vmcount - else: - if res != tpu_obj.vmcount: - # this should not happen, that in the same partition there are different vmcount nodesets - return -1 - return res - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--partitions", - "-p", - help="The partition(s) to retrieve the TPU vmcount value for.", - ) - args = parser.parse_args() - if not args.partitions: - exit(0) - - # useful exit code - # partition does not exists in config.yaml, thus do not exist in slurm - PART_INVALID = -1 - # in the same partition there are nodesets with different vmcounts - DIFF_VMCOUNTS_SAME_PART = -2 - # partition is a list of partitions in which at least two of them have different vmcount - DIFF_PART_DIFFERENT_VMCOUNTS = -3 - vmcounts = [] - # valid equals to 0 means that we are ok, otherwise it will be set to one of the previously defined exit codes - valid = 0 - for part in args.partitions.split(","): - if part not in util.lookup().cfg.partitions: - valid = PART_INVALID - break - else: - if util.lookup().partition_is_tpu(part): - vmcount = get_vmcount_of_tpu_part(part) - if vmcount == -1: - valid = DIFF_VMCOUNTS_SAME_PART - break - vmcounts.append(vmcount) - else: - vmcounts.append(0) - # this means that there are different vmcounts for these partitions - if valid == 0 and len(set(vmcounts)) != 1: - valid = DIFF_PART_DIFFERENT_VMCOUNTS - if valid != 0: - print(f"VMCOUNT:{valid}") - else: - print(f"VMCOUNT:{vmcounts[0]}") diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl deleted file mode 100644 index 810a0742b0..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl +++ /dev/null @@ -1,103 +0,0 @@ -SCRIPTS_DIR = "{scripts_dir}" -NO_VAL = 4294967294 --- get_tpu_vmcount.py exit code -PART_INVALID = -1 -- partition does not exists in config.yaml, thus do not exist in slurm -DIFF_VMCOUNTS_SAME_PART = -2 -- in the same partition there are nodesets with different vmcounts -DIFF_PART_DIFFERENT_VMCOUNTS = -3 -- partition is a list of partitions in which at least two of them have different vmcount -UNKWOWN_ERROR = -4 -- get_tpu_vmcount.py did not return a valid response - -function get_part(job_desc, part_list) - if job_desc.partition then - return job_desc.partition - end - for name, val in pairs(part_list) do - if val.flag_default == 1 then - return name - end - end - return nil -end - -function os.capture(cmd, raw) - local handle = assert(io.popen(cmd, 'r')) - local output = assert(handle:read('*a')) - handle:close() - return output -end - -function get_vmcount(part) - local cmd = SCRIPTS_DIR .. "/get_tpu_vmcount.py -p " .. part - local out = os.capture(cmd, true) - for line in out:gmatch("(.-)\r?\n") do - local tag, val = line:match("([^:]+):([^:]+)") - if tag == "VMCOUNT" then - return tonumber(val) - end - end - return UNKWOWN_ERROR -end - -function slurm_job_submit(job_desc, part_list, submit_uid) - local part = get_part(job_desc, part_list) - local vmcount = get_vmcount(part) - -- Only do something if the job is in a TPU partition, if vmcount is 0, it implies that the partition(s) specified are not TPU ones - if vmcount == 0 then - return slurm.SUCCESS - end - -- This is a TPU job, but as the vmcount is 1 it can he handled the same way - if vmcount == 1 then - return slurm.SUCCESS - end - -- Check for errors - if vmcount == PART_INVALID then - slurm.log_user("Invalid partition specified " .. part) - return slurm.FAILURE - end - if vmcount == DIFF_VMCOUNTS_SAME_PART then - slurm.log_user("In partition(s) " .. part .. - " there are more than one tpu nodeset vmcount, this should not happen.") - return slurm.ERROR - end - if vmcount == DIFF_PART_DIFFERENT_VMCOUNTS then - slurm.log_user("In partition list " .. part .. - " there are more than one TPU types, cannot determine which is the correct vmcount to use, please retry with only one partition.") - return slurm.FAILURE - end - if vmcount == UNKWOWN_ERROR then - slurm.log_user("Something went wrong while executing get_tpu_vmcount.py.") - return slurm.ERROR - end - -- This is surely a TPU node - if vmcount > 1 then - local min_nodes = job_desc.min_nodes - local max_nodes = job_desc.max_nodes - -- if not specified assume it is one, this should be improved taking into account the cpus, mem, and other factors - if min_nodes == NO_VAL then - min_nodes = 1 - max_nodes = 1 - end - -- as max_nodes can be higher than the nodes in the partition, we are not able to calculate with certainty the nodes that this job will have if this value is set to something - -- different than min_nodes - if min_nodes ~= max_nodes then - slurm.log_user("Max nodes cannot be set different than min nodes for the TPU partitions.") - return slurm.ERROR - end - -- Set the number of switches to the number of nodes originally requested by the job, as the job requests "TPU groups" - job_desc.req_switch = min_nodes - - -- Apply the node increase into the job description. - job_desc.min_nodes = min_nodes * vmcount - job_desc.max_nodes = max_nodes * vmcount - -- if job_desc.features then - -- slurm.log_user("Features: %s",job_desc.features) - -- end - end - - return slurm.SUCCESS -end - -function slurm_job_modify(job_desc, job_rec, part_list, modify_uid) - return slurm.SUCCESS -end - -return slurm.SUCCESS diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py deleted file mode 100644 index cabd6e3e9f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py +++ /dev/null @@ -1,352 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, Callable, Any -import argparse -import os -import shelve -import uuid -from collections import namedtuple -from datetime import datetime, timedelta, timezone -from pathlib import Path -from pprint import pprint - -import util -from google.api_core import exceptions, retry -from google.cloud import bigquery as bq -from google.cloud.bigquery import SchemaField # type: ignore -from util import lookup, run - -SACCT = "sacct" -script = Path(__file__).resolve() - -DEFAULT_TIMESTAMP_FILE = script.parent / "bq_timestamp" -timestamp_file = Path(os.environ.get("TIMESTAMP_FILE", DEFAULT_TIMESTAMP_FILE)) -# The maximum request to insert_rows is 10MB, each sacct row is about 1200 bytes or ~ 8000 rows. -# Set to 5000 for a little wiggle room. -BQ_ROW_BATCH_SIZE = 5000 - -# cluster_id_file = script.parent / 'cluster_uuid' -# try: -# cluster_id = cluster_id_file.read_text().rstrip() -# except FileNotFoundError: -# cluster_id = uuid.uuid4().hex -# cluster_id_file.write_text(cluster_id) - -job_idx_cache_path = script.parent / "bq_job_idx_cache" - -SLURM_TIME_FORMAT = r"%Y-%m-%dT%H:%M:%S" - - -def make_datetime(time_string): - if time_string == "None": - return None - return datetime.strptime(time_string, SLURM_TIME_FORMAT).replace( - tzinfo=timezone.utc - ) - - -def make_time_interval(seconds): - sign = 1 - if seconds < 0: - sign = -1 - seconds = abs(seconds) - d, r = divmod(seconds, 60 * 60 * 24) - h, r = divmod(r, 60 * 60) - m, s = divmod(r, 60) - d *= sign - h *= sign - return f"{d}D {h:02}:{m:02}:{s}" - - -converters: Dict[str, Callable[[Any], Any]] = { - "DATETIME": make_datetime, - "INTERVAL": make_time_interval, - "STRING": str, - "INT64": lambda n: int(n or 0), -} - - -def schema_field(field_name, data_type, description, required=False): - return SchemaField( - field_name, - data_type, - description=description, - mode="REQUIRED" if required else "NULLABLE", - ) - - -schema_fields = [ - schema_field("cluster_name", "STRING", "cluster name", required=True), - schema_field("cluster_id", "STRING", "UUID for the cluster", required=True), - schema_field("entry_uuid", "STRING", "entry UUID for the job row", required=True), - schema_field( - "job_db_uuid", "STRING", "job db index from the slurm database", required=True - ), - schema_field("job_id_raw", "INT64", "raw job id", required=True), - schema_field("job_id", "STRING", "job id", required=True), - schema_field("state", "STRING", "final job state", required=True), - schema_field("job_name", "STRING", "job name"), - schema_field("partition", "STRING", "job partition"), - schema_field("submit_time", "DATETIME", "job submit time"), - schema_field("start_time", "DATETIME", "job start time"), - schema_field("end_time", "DATETIME", "job end time"), - schema_field("elapsed_raw", "INT64", "STRING", "job run time in seconds"), - # schema_field("elapsed_time", "INTERVAL", "STRING", "job run time interval"), - schema_field("timelimit_raw", "STRING", "job timelimit in minutes"), - schema_field("timelimit", "STRING", "job timelimit"), - # schema_field("num_tasks", "INT64", "number of allocated tasks in job"), - schema_field("nodelist", "STRING", "names of nodes allocated to job"), - schema_field("user", "STRING", "user responsible for job"), - schema_field("uid", "INT64", "uid of job user"), - schema_field("group", "STRING", "group of job user"), - schema_field("gid", "INT64", "gid of job user"), - schema_field("wckey", "STRING", "job wckey"), - schema_field("qos", "STRING", "job qos"), - schema_field("comment", "STRING", "job comment"), - schema_field("admin_comment", "STRING", "job admin comment"), - # extra will be added in 23.02 - # schema_field("extra", "STRING", "job extra field"), - schema_field("exitcode", "STRING", "job exit code"), - schema_field("alloc_cpus", "INT64", "count of allocated CPUs"), - schema_field("alloc_nodes", "INT64", "number of nodes allocated to job"), - schema_field("alloc_tres", "STRING", "allocated trackable resources (TRES)"), - # schema_field("system_cpu", "INTERVAL", "cpu time used by parent processes"), - # schema_field("cpu_time", "INTERVAL", "CPU time used (elapsed * cpu count)"), - schema_field("cpu_time_raw", "INT64", "CPU time used (elapsed * cpu count)"), - # schema_field("ave_cpu", "INT64", "Average CPU time of all tasks in job"), - # schema_field( - # "tres_usage_tot", - # "STRING", - # "Tres total usage by all tasks in job", - # ), -] - - -slurm_field_map = { - "job_db_uuid": "DBIndex", - "job_id_raw": "JobIDRaw", - "job_id": "JobID", - "state": "State", - "job_name": "JobName", - "partition": "Partition", - "submit_time": "Submit", - "start_time": "Start", - "end_time": "End", - "elapsed_raw": "ElapsedRaw", - "elapsed_time": "Elapsed", - "timelimit_raw": "TimelimitRaw", - "timelimit": "Timelimit", - "num_tasks": "NTasks", - "nodelist": "Nodelist", - "user": "User", - "uid": "Uid", - "group": "Group", - "gid": "Gid", - "wckey": "Wckey", - "qos": "Qos", - "comment": "Comment", - "admin_comment": "AdminComment", - # "extra": "Extra", - "exit_code": "ExitCode", - "alloc_cpus": "AllocCPUs", - "alloc_nodes": "AllocNodes", - "alloc_tres": "AllocTres", - "system_cpu": "SystemCPU", - "cpu_time": "CPUTime", - "cpu_time_raw": "CPUTimeRaw", - "ave_cpu": "AveCPU", - "tres_usage_tot": "TresUsageInTot", -} - -# new field name is the key for job_schema. Used to lookup the datatype when -# creating the job rows -job_schema = {field.name: field for field in schema_fields} -# Order is important here, as that is how they are parsed from sacct output -Job = namedtuple("Job", job_schema.keys()) # type: ignore -# ... see https://github.com/python/mypy/issues/848 - -client = bq.Client( - project=lookup().cfg.project, - credentials=util.default_credentials(), - client_options=util.create_client_options(util.ApiEndpoint.BQ), -) -dataset_id = f"{lookup().cfg.slurm_cluster_name}_job_data" -dataset = bq.DatasetReference(project=lookup().project, dataset_id=dataset_id) -table = bq.Table( - bq.TableReference(dataset, f"jobs_{lookup().cfg.slurm_cluster_name}"), schema_fields -) - - -class JobInsertionFailed(Exception): - pass - - -def make_job_row(job): - job_row = { - field_name: converters[field.field_type](job[field_name]) - for field_name, field in job_schema.items() - if field_name in job - } - job_row["entry_uuid"] = uuid.uuid4().hex - job_row["cluster_id"] = lookup().cfg.cluster_id - job_row["cluster_name"] = lookup().cfg.slurm_cluster_name - return job_row - - -def load_slurm_jobs(start, end): - states = ",".join( - ( - "BOOT_FAIL", - "CANCELLED", - "COMPLETED", - "DEADLINE", - "FAILED", - "NODE_FAIL", - "OUT_OF_MEMORY", - "PREEMPTED", - "REQUEUED", - "REVOKED", - "TIMEOUT", - ) - ) - start_iso = start.isoformat(timespec="seconds") - end_iso = end.isoformat(timespec="seconds") - # slurm_fields and bq_fields will be in matching order - slurm_fields = ",".join(slurm_field_map.values()) - bq_fields = slurm_field_map.keys() - cmd = ( - f"{SACCT} --start {start_iso} --end {end_iso} -X -D --format={slurm_fields} " - f"--state={states} --parsable2 --noheader --allusers --duplicates" - ) - text = run(cmd).stdout.splitlines() - # zip pairs bq_fields with the value from sacct - jobs = [dict(zip(bq_fields, line.split("|"))) for line in text] - - # The job index cache allows us to avoid sending duplicate jobs. This avoids a race condition with updating the database. - with shelve.open(str(job_idx_cache_path), flag="r") as job_idx_cache: - job_rows = [ - make_job_row(job) - for job in jobs - if str(job["job_db_uuid"]) not in job_idx_cache - ] - return job_rows - - -def init_table(): - global dataset - global table - dataset = client.create_dataset(dataset, exists_ok=True) # type: ignore - table = client.create_table(table, exists_ok=True) - until_found = retry.Retry(predicate=retry.if_exception_type(exceptions.NotFound)) - table = client.get_table(table, retry=until_found) - # cannot add required fields to an existing schema - table.schema = schema_fields - table = client.update_table(table, ["schema"]) - - -def purge_job_idx_cache(): - purge_time = datetime.now() - timedelta(minutes=30) - with shelve.open(str(job_idx_cache_path), writeback=True) as cache: - to_delete = [] - for idx, stamp in cache.items(): - if stamp < purge_time: - to_delete.append(idx) - for idx in to_delete: - del cache[idx] - - -def bq_submit(jobs): - try: - result = client.insert_rows(table, jobs) - except exceptions.NotFound as e: - print(f"failed to upload job data, table not yet found: {e}") - raise e - except Exception as e: - print(f"failed to upload job data: {e}") - raise e - if result: - pprint(jobs) - pprint(result) - raise JobInsertionFailed("failed to upload job data to big query") - print(f"successfully loaded {len(jobs)} jobs") - - -def get_time_window(): - if not timestamp_file.is_file(): - timestamp_file.touch() - try: - timestamp = datetime.strptime( - timestamp_file.read_text().rstrip(), SLURM_TIME_FORMAT - ) - # time window will overlap the previous by 10 minutes. Duplicates will be filtered out by the job_idx_cache - start = timestamp - timedelta(minutes=10) - except ValueError: - # timestamp 1 is 1 second after the epoch; timestamp 0 is special for sacct - start = datetime.fromtimestamp(1) - # end is now() truncated to the last second - end = datetime.now().replace(microsecond=0) - return start, end - - -def write_timestamp(time): - timestamp_file.write_text(time.isoformat(timespec="seconds")) - - -def update_job_idx_cache(jobs, timestamp): - with shelve.open(str(job_idx_cache_path), writeback=True) as job_idx_cache: - for job in jobs: - job_idx = str(job["job_db_uuid"]) - job_idx_cache[job_idx] = timestamp - - -def main(): - if not lookup().cfg.enable_bigquery_load: - print("bigquery load is not currently enabled") - exit(0) - init_table() - - start, end = get_time_window() - jobs = load_slurm_jobs(start, end) - # on failure, an exception will cause the timestamp not to be rewritten. So - # it will try again next time. If some writes succeed, we don't currently - # have a way to not submit duplicates next time. - if jobs: - num_batches = (len(jobs) - 1) // BQ_ROW_BATCH_SIZE + 1 - print( - f"loading {num_batches} batches of BigQuery data in batches of size : {BQ_ROW_BATCH_SIZE}" - ) - for batch_indx, job_indx in enumerate(range(0, len(jobs), BQ_ROW_BATCH_SIZE)): - print(f"loading BigQuery data batch {batch_indx} of {num_batches}") - bq_submit(jobs[job_indx : job_indx + BQ_ROW_BATCH_SIZE]) - write_timestamp(end) - update_job_idx_cache(jobs, end) - - -parser = argparse.ArgumentParser(description="submit slurm job data to big query") -parser.add_argument( - "timestamp_file", - nargs="?", - action="store", - type=Path, - help="specify timestamp file for reading and writing the time window start. Precedence over TIMESTAMP_FILE env var.", -) - -purge_job_idx_cache() -if __name__ == "__main__": - args = parser.parse_args() - if args.timestamp_file: - timestamp_file = args.timestamp_file.resolve() - main() diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py deleted file mode 100644 index d4a4477f83..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -Implementation of message queue that mimics interface of GCP (PubSub)[https://cloud.google.com/pubsub] - -Messages are stored on controller state disk (to survive controller re-creation) with following layout: - -// -├- -| └- -└- .staging - └- - └- - -One message is one immutable file, that will be deleted after acknowledgement. -NOTE: Implementation assumes that both `` and `.staging/` are on the same disk device, -so it can rely on atomic "move / rename" operation. -""" -from typing import Any -import util -import json -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path -import os -import uuid - -import logging -log = logging.getLogger() - - -@dataclass(frozen=True) -class Message: - id: str - created: datetime - data: Any - - def to_json(self) -> dict[str, str]: - return dict( - id=self.id, - created=self.created.isoformat(), - data=self.data) - - @classmethod - def from_json(cls, data: dict[str, str]) -> 'Message': - return cls( - id=data['id'], - created=datetime.fromisoformat(data['created']), - data=data['data']) - -class Topic: - """ - Acts as PubSub topic (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics). - We can have multiple instances of - """ - def __init__(self, path: Path, staging: Path) -> None: - self._path = path - self._staging = staging - - def _gen_id(self, created: datetime) -> str: - ts = created.strftime("%Y_%m_%d-%H_%M_%S") - suf = str(uuid.uuid4())[:8] - return f"{ts}-{suf}" - - def publish(self, data: Any) -> None: - created = util.now() - id = self._gen_id(created) - msg = Message(id=id, created=created, data=data) - - staged = self._staging / msg.id - dst = self._path / msg.id - - # Write to stagin area first then perform atomic move - # to prevent "reads of partial writes" - staged.write_text(json.dumps(msg.to_json())) - util.chown_slurm(staged) - staged.rename(dst) - - -class Subscription: - """ - Acts as PubSub subscription (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) - with following settings: - - ``` - ackDeadlineSeconds = +Inf # don't resend message that was already being delivered but not acked yet - retainAckedMessages = False # don't persist messages that were already acked - enableMessageOrdering = True # delivers messages in chronoligical order - messageRetentionDuration = +Inf # don't expire messages - deadLetterPolicy = None # "deadlettering" is disabled, subscriber should take care of any poisonous messages - retryPolicy = { # NACKed message will be re-delievered after some time - minimumBackoff = 30s # NOTE: Practically there is no timer, but Subscription instance will not try to re-deliver NACKed messages. - maximumBackoff = 30s # Assumes that slurmsync runs every 30+ sec. - } - ``` - - IMPORTANT: Should only be run as part of slurmsync, - this is our way to ensure that at most one instance exists at a time. - There is no concurancy safeguards in place, avoid multithreaded `pull`, - while multithreaded `ack` & `modify_ack_deadline` are OK. - """ - - def __init__(self, path: Path) -> None: - self._path: Path = path - # contains ALL messages pulled by this subscription instance - # both acked, nacked, and still being processed - # used to prevent double delivery within lifetime of subscription (slurmsync) - self._pulled: set[str] = set() - - def _delete(self, id: str) -> None: - log.debug(f"removing {id}") - try: - os.unlink(self._path / id) - except: - log.exception(f"Failed to remove message {id}") - - def _read_msg(self, id: str) -> Message | None: - try: - with open(self._path / id, 'r') as f: - content = json.loads(f.read()) - return Message.from_json(content) - except Exception: - log.exception(f"Failed to read message {id}") - self._delete(id) # delete message to reduce "deadlettering" - return None - - def pull(self, max_messages: int) -> list[Message]: - if not self._path.exists(): - log.warning(f"Topic {self._path} does not exist") - return [] - res = [] - ls = sorted(os.listdir(self._path)) - for name in ls: - msg = self._read_msg(name) - if msg is not None and msg.id not in self._pulled: - self._pulled.add(msg.id) - res.append(msg) - - if len(res) >= max_messages: - break - return res - - - def ack(self, ids: list[str]) -> None: - for id in ids: - self._delete(id) - - - def modify_ack_deadline(self, ids: list[str], deadline: int) -> None: - """ - Modifies the ack deadline for a specific message. - IMPORTANT: Only accepts deadline=0, which is a way to NACK - Any other values are also meaningless due to ackDeadlineSeconds==+Inf - """ - assert deadline == 0 # no op, next subscriber (slurmsync) will pick this up - - -# Topics and Subscriptions are singletons -# TODO: consider making thread-safe -_topics = {} -_subscriptions = {} - -def _make_path(name: str) -> Path: - p = util.slurmdirs.state / "pubsub" / name - p.mkdir(parents=True, exist_ok=True) - util.chown_slurm(p) - return p - -def _make_staging_path(name: str) -> Path: - p = util.slurmdirs.state / "pubsub" / ".staging" / name - p.mkdir(parents=True, exist_ok=True) - util.chown_slurm(p) - return p - -def topic(name: str) -> Topic: - if name not in _topics: - _topics[name] = Topic(_make_path(name), _make_staging_path(name)) - return _topics[name] - -def subscription(name: str) -> Subscription: - if name not in _subscriptions: - _subscriptions[name] = Subscription(_make_path(name)) - return _subscriptions[name] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py deleted file mode 100644 index 8ea3d0657e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional - -import util -import uuid -from addict import Dict as NSDict # type: ignore -from datetime import datetime, timedelta -from collections import defaultdict -import logging -from time import sleep - -log = logging.getLogger() - -DWS_EOL_RESERVATION_DURATION = 10 # minutes - -def _duration(flex_options: NSDict, job_id: Optional[int], lkp: util.Lookup) -> int: - dur = flex_options.max_run_duration - if not job_id or not flex_options.use_job_duration: - return dur - - job = lkp.job(job_id) - if not job or not job.duration: - return dur - - if timedelta(minutes=10) <= job.duration <= timedelta(weeks=1): - return int(job.duration.total_seconds()) - - log.info("Job TimeLimit cannot be less than 10 minutes or exceed one week") - return dur - -def _create_slurm_reservation(node_name: str, boot_time: datetime, run_duration: int, lkp: util.Lookup): - """ - Create a Slurm reservation starting at EOL - buffer time. - """ - eol = boot_time + timedelta(seconds=run_duration) - start_str = eol.strftime("%Y-%m-%dT%H:%M:%S") - reservation_name = f"dws-eol-{node_name}" - log.debug(f"creating slurm reservation for {node_name}") - try: - util.run(f"{lkp.scontrol} create reservation user=slurm starttime={start_str} duration={DWS_EOL_RESERVATION_DURATION} nodes={node_name} reservationname={reservation_name} flags=maint,ignore_jobs") - except Exception as e: - log.error(f"Failed to create reservation for {node_name}: {e}") - -def _delete_slurm_reservation(node_name: str, lkp: util.Lookup): - """ - Delete the Slurm reservation for the given node. - """ - reservation_name = f"dws-eol-{node_name}" - try: - util.run(f"{lkp.scontrol} delete reservation {reservation_name}") - log.debug(f"Deleted Slurm reservation {reservation_name} for {node_name}") - except Exception as e: - log.error(f"Failed to delete reservation for {node_name}: {e}") - -def resume_flex_chunk(nodes: List[str], job_id: Optional[int], lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - assert len(nodeset.zone_policy_allow) > 0 - region = lkp.node_region(model) - - assert nodeset.dws_flex.enabled - - uid = str(uuid.uuid4())[:8] - if job_id: - mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-job-{job_id}-{uid}" - else: - mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-{uid}" - - # Create MIG - req = lkp.compute.regionInstanceGroupManagers().insert( - project=lkp.project, - region=region, - body=dict( - name=mig_name, - versions=[dict(instanceTemplate=nodeset.instance_template)], - targetSize=0, - distributionPolicy=dict( - zones=[ - dict(zone=f"zones/{z}") for z in nodeset.zone_policy_allow - ], - targetShape="ANY_SINGLE_ZONE" ), - updatePolicy = dict(instanceRedistributionType = "NONE" ), - instanceLifecyclePolicy=dict(defaultActionOnFailure= "DO_NOTHING" ), # TODO(FLEX): Not supported yet, migrate once supported - ) - ) - util.log_api_request(req) - op = req.execute() - res = util.wait_for_operation(op) - assert "error" not in res, f"{res}" - - # Create resize request - duration_seconds = _duration(nodeset.dws_flex, job_id, lkp) - req = lkp.compute.regionInstanceGroupManagerResizeRequests().insert( - project=lkp.project, - region=region, - instanceGroupManager=mig_name, - body=dict( - name="initial-resize", - instances=[dict(name=n) for n in nodes], - requested_run_duration=dict( - seconds=duration_seconds - ) - ) - ) - util.log_api_request(req) - op = req.execute() - res = util.wait_for_operation(op) - - # Create Slurm reservations if use_job_duration is set - if nodeset.dws_flex.use_job_duration: - # Get run duration (seconds) - run_duration = duration_seconds - for node_name in nodes: - # Fetch instance creation time from GCP instance (via util.py) - instance = lkp.instance(node_name) - if(instance and instance.creation_timestamp): - log.debug("creating with creation_timestamp") - boot_time = instance.creation_timestamp # Already a datetime object - else: - boot_time = datetime.utcnow() - log.debug("creating with utcnow time: {boot_time}") - _create_slurm_reservation(node_name, boot_time, run_duration, lkp) - - assert "error" not in res, f"{res}" - -def _suspend_flex_mig(mig_self_link: str, nodes: List[str], lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - assert len(nodeset.zone_policy_allow) > 0 - region = lkp.node_region(model) - project=lkp.project - instanceGroupManager=util.trim_self_link(mig_self_link) - - links = [ - f"zones/{inst.zone}/instances/{inst.name}" - for inst in [ - lkp.instance(node) for node in nodes - ] if inst - ] - - target_mig=lkp.get_mig(lkp.project, region, instanceGroupManager) - assert target_mig - - # TODO(FLEX): This will not work if MIG didn't obtain capacity yet. - # The request will fail and MIG will continue provisioning. - # Instead whole MIG should be deleted. - # + All other instances in MIG are not provisioned also, safe to delete - # - Need to come up will clear test to differentiate non-provisioned MIG and single VM being down; - # Particularly CRITICAL due to ActionOnFailure=DO_NOTHING - # - Need to `down_nodes_notify_jobs` for all nodes in MIG, make sure that it doesn't interfere with Slurm suspend-flow. - - if target_mig["targetSize"] == len(nodes): #We can just delete the whole MIG in this case - req = lkp.compute.regionInstanceGroupManagers().delete( - project=project, - region=region, - instanceGroupManager=instanceGroupManager, - ) - else: - req = lkp.compute.regionInstanceGroupManagers().deleteInstances( - project=project, - region=region, - instanceGroupManager=instanceGroupManager, - body=dict( - instances=links, - skipInstancesOnValidationError=True, - ) - ) - - util.log_api_request(req) - op = req.execute() - - res = util.wait_for_operation(op) - - # Delete Slurm reservations for nodes being deprovisioned - for node_name in nodes: - log.info("delete dws reservation") - _delete_slurm_reservation(node_name, lkp) - - assert "error" not in res, f"{res}" - -def _suspend_provisioning_inst(nodes:List[str], node_template:str, lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - assert len(nodeset.zone_policy_allow) > 0 - region = lkp.node_region(model) - - mig_list=lkp.get_mig_list(lkp.project, region) - - # FLEX (#TODO): If we enter this conditional it's likely this was called so early that MIG creation hasn't started - # Consider potentially retrying? No natural mechanism for retry currently but we could - # perhaps use slurmsync and then try it again to ensure it wasn't a case of being too early. - # This is important since we're now enabling long ResumeTimeout (Slurm won't call suspend on node within reasonable timeframe) - # so until we do this is slurmsync this is a temporary workaround. - - if not mig_list or not mig_list.get("items"): - log.info("No matching MIG found to delete! Retrying...") - sleep(5) - mig_list=lkp.get_mig_list(lkp.project, region) - if not mig_list or not mig_list.get("items"): - return - - for mig in mig_list["items"]: - if mig["instanceTemplate"] == node_template: - if mig["currentActions"]["creating"] > 0 and mig["targetSize"] == mig["currentActions"]["creating"]: - req = lkp.compute.regionInstanceGroupManagers().delete( - project=lkp.project, - region=region, - instanceGroupManager=util.trim_self_link(mig["selfLink"]), - ) - - util.log_api_request(req) - op = req.execute() - - res = util.wait_for_operation(op) - assert "error" not in res, f"{res}" - return - - log.info("No matching MIG found to delete!") - -def suspend_flex_nodes(nodes: List[str], lkp: util.Lookup) -> None: - by_mig = defaultdict(list) - not_provisioned = defaultdict(list) - for node in nodes: - inst = lkp.instance(node) - if not inst: - not_provisioned[lkp.node_template(node)].append(node) - else: - mig = inst.metadata.get("created-by") - if not mig: - log.error(f"Can not suspend {node}, can not find associated MIG") - continue - by_mig[mig].append(node) - - for mig, nodes in by_mig.items(): - _suspend_flex_mig(mig, nodes, lkp) - - for node_template, nodes in not_provisioned.items(): - _suspend_provisioning_inst(nodes, node_template, lkp) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt deleted file mode 100644 index 2ab3162ccf..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ -pytest -pytest-mock -pytest_unordered -mock - -types-mock -types-httplib2 -types-requests -types-PyYAML diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt deleted file mode 100644 index e923e53dbf..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -addict==2.4.0 -google-api-core==2.19.0 -google-api-python-client==2.93.0 -google-auth==2.40.3 -google-auth-httplib2==0.1.0 -google-cloud-bigquery==3.11.3 -google-cloud-core==2.3.3 -google-cloud-secret-manager~=2.22 -google-cloud-storage==2.10.0 -google-cloud-tpu==1.10.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.1 -grpcio==1.60.0 -grpcio-status==1.60.0 -httplib2==0.22.0 -more-executors==2.11.4 -pyyaml==6.0.2 -requests==2.32.4 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py deleted file mode 100644 index ea0012a0b1..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py +++ /dev/null @@ -1,703 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional, Dict, Any -import argparse -from datetime import timedelta -import shlex -import json -import logging -import os -import yaml -import collections -from pathlib import Path -from dataclasses import dataclass -from addict import Dict as NSDict # type: ignore - -import util -from util import ( - chunked, - ensure_execute, - execute_with_futures, - log_api_request, - map_with_futures, - run, - separate, - to_hostlist, - trim_self_link, - wait_for_operation, -) -from util import lookup, ReservationDetails -import tpu -import mig_flex - -log = logging.getLogger() - -PLACEMENT_MAX_CNT = 1500 -# Placement group needs to be the same for an entire bulk_insert hence -# if placement is used the actual BULK_INSERT_LIMIT will be -# max([1000, PLACEMENT_MAX_CNT]) -BULK_INSERT_LIMIT = 5000 - -# https://cloud.google.com/compute/docs/instance-groups#types_of_managed_instance_groups -ZONAL_MIG_SIZE_LIMIT = 1000 - - -@dataclass(frozen=True) -class ResumeJobData: - job_id: int - partition: str - nodes_alloc: List[str] - -@dataclass(frozen=True) -class ResumeData: - jobs: List[ResumeJobData] - - -def get_resume_file_data() -> Optional[ResumeData]: - if not (path := os.getenv("SLURM_RESUME_FILE")): - log.error("SLURM_RESUME_FILE was not in environment. Cannot get detailed job, node, partition allocation data.") - return None - blob = Path(path).read_text() - log.debug(f"Resume data: {blob}") - data = json.loads(blob) - - jobs = [] - for jo in data.get("jobs", []): - job = ResumeJobData( - job_id = jo.get("job_id"), - partition = jo.get("partition"), - nodes_alloc = util.to_hostnames(jo.get("nodes_alloc")), - ) - jobs.append(job) - return ResumeData(jobs=jobs) - -def instance_properties(nodeset: NSDict, model:str, placement_group:Optional[str], labels:Optional[dict], job_id:Optional[int]): - props = NSDict() - - if labels: # merge in extra labels on instance and disks - template_link = lookup().node_template(model) - template_info = lookup().template_info(template_link) - - props.labels = {**template_info.labels, **labels} - - for disk in template_info.disks: - if disk.initializeParams.get("diskType", "local-ssd") == "local-ssd": - continue # do not label local ssd - disk.initializeParams.labels.update(labels) - props.disks = template_info.disks - - if placement_group: - props.resourcePolicies = [placement_group] - - if reservation := lookup().nodeset_reservation(nodeset): - update_reservation_props(reservation, props, placement_group, reservation.calendar) - - if (fr := lookup().future_reservation(nodeset)) and fr.specific: - assert fr.active_reservation - update_reservation_props(fr.active_reservation, props, placement_group, fr.calendar) - - if props.resourcePolicies: - props.scheduling.onHostMaintenance = "TERMINATE" - - if nodeset.maintenance_interval: - props.scheduling.maintenanceInterval = nodeset.maintenance_interval - - if nodeset.dws_flex.enabled and nodeset.dws_flex.use_bulk_insert: - update_props_dws(props, nodeset.dws_flex, job_id) - - # Override with properties explicit specified in the nodeset - props.update(nodeset.get("instance_properties") or {}) - return props - -def update_reservation_props(reservation:ReservationDetails, props:NSDict, placement_group:Optional[str], calendar_mode:bool) -> None: - props.reservationAffinity = { - "consumeReservationType": "SPECIFIC_RESERVATION", - "key": f"compute.{util.universe_domain()}/reservation-name", - "values": [reservation.bulk_insert_name], - } - - if reservation.dense or calendar_mode: - props.scheduling.provisioningModel = "RESERVATION_BOUND" - - # Figure out `resourcePolicies` - if reservation.policies: # use ones already attached to reservations - props.resourcePolicies = reservation.policies - elif reservation.dense and placement_group: # use once created by Slurm - props.resourcePolicies = [placement_group] - else: # vanilla reservations don't support external policies - props.resourcePolicies = [] - log.info( - f"reservation {reservation.bulk_insert_name} is being used with resourcePolicies: {props.resourcePolicies}") - -def update_props_dws(props: NSDict, dws_flex: NSDict, job_id: Optional[int]) -> None: - props.scheduling.onHostMaintenance = "TERMINATE" - props.scheduling.instanceTerminationAction = "DELETE" - props.reservationAffinity['consumeReservationType'] = "NO_RESERVATION" - props.scheduling.maxRunDuration['seconds'] = dws_flex_duration(dws_flex, job_id) - -def dws_flex_duration(dws_flex: NSDict, job_id: Optional[int]) -> int: - max_duration = dws_flex.max_run_duration - if dws_flex.use_job_duration and job_id is not None and (job := lookup().job(job_id)) and job.duration: - if timedelta(seconds=30) <= job.duration <= timedelta(weeks=1): - max_duration = int(job.duration.total_seconds()) - else: - log.info("Job TimeLimit cannot be less than 30 seconds or exceed one week") - return max_duration - -def create_instances_request(nodes: List[str], placement_group: Optional[str], excl_job_id: Optional[int]): - """Call regionInstances.bulkInsert to create instances""" - assert 0 < len(nodes) <= BULK_INSERT_LIMIT - - # model here indicates any node that can be used to describe the rest - model = next(iter(nodes)) - log.debug(f"create_instances_request: {model} placement: {placement_group}") - - nodeset = lookup().node_nodeset(model) - template = lookup().node_template(model) - labels = {"slurm_job_id": excl_job_id} if excl_job_id else None - - body = dict( - count = len(nodes), - sourceInstanceTemplate = template, - # key is instance name, value overwrites properties (no overwrites) - perInstanceProperties = {k: {} for k in nodes}, - instanceProperties = instance_properties( - nodeset, model, placement_group, labels, excl_job_id - ), - ) - - if placement_group and excl_job_id is not None: - pass # do not set minCount to force "all or nothing" behavior - else: - body["minCount"] = 1 - - zone_allow = nodeset.zone_policy_allow or [] - zone_deny = nodeset.zone_policy_deny or [] - - if len(zone_allow) == 1: # if only one zone is used, use zonal BulkInsert API, as less prone to errors - api_method = lookup().compute.instances().bulkInsert - method_args = {"zone": zone_allow[0]} - else: - api_method = lookup().compute.regionInstances().bulkInsert - method_args = {"region": lookup().node_region(model)} - - body["locationPolicy"] = dict( - locations = { - **{ f"zones/{z}": {"preference": "ALLOW"} for z in zone_allow }, - **{ f"zones/{z}": {"preference": "DENY"} for z in zone_deny }}, - targetShape = nodeset.zone_target_shape, - ) - - req = api_method( - project=lookup().project, - body=body, - **method_args) - log.debug(f"new request: endpoint={req.methodId} nodes={to_hostlist(nodes)}") - log_api_request(req) - return req - -@dataclass() -class PlacementAndNodes: - placement: Optional[str] - nodes: List[str] - -@dataclass(frozen=True) -class BulkChunk: - nodes: List[str] - prefix: str # - - chunk_idx: int - excl_job_id: Optional[int] - placement_group: Optional[str] = None - - @property - def name(self): - if self.placement_group is not None: - return f"{self.prefix}:job{self.excl_job_id}:{self.placement_group}:{self.chunk_idx}" - if self.excl_job_id is not None: - return f"{self.prefix}:job{self.excl_job_id}:{self.chunk_idx}" - return f"{self.prefix}:{self.chunk_idx}" - - -def group_nodes_bulk(nodes: List[str], resume_data: Optional[ResumeData], lkp: util.Lookup): - """group nodes by nodeset, placement_group, exclusive_job_id if any""" - if resume_data is None: # all nodes will be considered jobless - resume_data = ResumeData(jobs=[]) - - nodes_set = set(nodes) # turn into set to simplify intersection - non_excl = nodes_set.copy() - groups : Dict[Optional[int], List[PlacementAndNodes]] = {} # excl_job_id|none -> PlacementAndNodes - - # expand all exclusive job nodelists - for job in resume_data.jobs: - if not lkp.cfg.partitions[job.partition].enable_job_exclusive: - continue - - groups[job.job_id] = [] - # placement group assignment is based on all allocated nodes, ... - for pn in create_placements(job.nodes_alloc, job.job_id, lkp): - groups[job.job_id].append( - PlacementAndNodes( - placement=pn.placement, - #... but we only want to handle nodes in nodes_resume in this run. - nodes = sorted(set(pn.nodes) & nodes_set) - )) - non_excl.difference_update(job.nodes_alloc) - - groups[None] = create_placements(sorted(non_excl), excl_job_id=None, lkp=lkp) - - def chunk_nodes(nodes: List[str]): - if not nodes: - return [] - - model = nodes[0] - - if lkp.is_flex_node(model): - chunk_size = ZONAL_MIG_SIZE_LIMIT - elif lkp.node_is_tpu(model): - ns_name = lkp.node_nodeset_name(model) - chunk_size = tpu.TPU.make(ns_name, lkp).vmcount - else: - chunk_size = BULK_INSERT_LIMIT - - return chunked(nodes, n=chunk_size) - - chunks = [ - BulkChunk( - nodes=nodes_chunk, - prefix=lkp.node_prefix(nodes_chunk[0]), # - - excl_job_id = job_id, - placement_group=pn.placement, - chunk_idx=i) - - for job_id, placements in groups.items() - for pn in placements if pn.nodes - for i, nodes_chunk in enumerate(chunk_nodes(pn.nodes)) - ] - return {chunk.name: chunk for chunk in chunks} - - -def resume_nodes(nodes: List[str], resume_data: Optional[ResumeData]): - """resume nodes in nodelist""" - lkp = lookup() - # Prevent dormant nodes associated with a reservation from being resumed - nodes, dormant_res_nodes = util.separate(lkp.is_dormant_res_node, nodes) - - if dormant_res_nodes: - log.warning(f"Resume was unable to resume reservation nodes={dormant_res_nodes}") - down_nodes_notify_jobs(dormant_res_nodes, "Reservation is not active, nodes cannot be resumed", resume_data) - - nodes, flex_managed = util.separate(lkp.is_provisioning_flex_node, nodes) - if flex_managed: - log.warning(f"Resume was unable to resume nodes={flex_managed} already managed by MIGs") - down_nodes_notify_jobs(flex_managed, "VM is managed MIG, can not be resumed", resume_data) - - if not nodes: - log.info("No nodes to resume") - return - - nodes = sorted(nodes, key=lkp.node_prefix) - grouped_nodes = group_nodes_bulk(nodes, resume_data, lkp) - - if log.isEnabledFor(logging.DEBUG): - grouped_nodelists = { - group: to_hostlist(chunk.nodes) for group, chunk in grouped_nodes.items() - } - log.debug( - "node bulk groups: \n{}".format(yaml.safe_dump(grouped_nodelists).rstrip()) - ) - - tpu_chunks, flex_chunks = [], [] - bi_inserts = {} - - for group, chunk in grouped_nodes.items(): - model = chunk.nodes[0] - - if lkp.node_is_tpu(model): - tpu_chunks.append(chunk.nodes) - elif lkp.is_flex_node(model): - flex_chunks.append(chunk) - else: - bi_inserts[group] = create_instances_request( - chunk.nodes, chunk.placement_group, chunk.excl_job_id - ) - - for chunk in flex_chunks: - mig_flex.resume_flex_chunk(chunk.nodes, chunk.excl_job_id, lkp) - - # execute all bulkInsert requests with batch - bulk_ops = dict( - zip(bi_inserts.keys(), map_with_futures(ensure_execute, bi_inserts.values())) - ) - log.debug(f"bulk_ops={yaml.safe_dump(bulk_ops)}") - started = { - group: op for group, op in bulk_ops.items() if not isinstance(op, Exception) - } - failed = { - group: err for group, err in bulk_ops.items() if isinstance(err, Exception) - } - if failed: - failed_reqs = [str(e) for e in failed.items()] - log.error("bulkInsert API failures: {}".format("; ".join(failed_reqs))) - for ident, exc in failed.items(): - down_nodes_notify_jobs(grouped_nodes[ident].nodes, f"GCP Error: {exc._get_reason()}", resume_data) # type: ignore - - if log.isEnabledFor(logging.DEBUG): - for group, op in started.items(): - group_nodes = grouped_nodelists[group] - name = op["name"] - gid = op["operationGroupId"] - log.debug( - f"new bulkInsert operation started: group={group} nodes={group_nodes} name={name} operationGroupId={gid}" - ) - # wait for all bulkInserts to complete and log any errors - bulk_operations = {group: wait_for_operation(op) for group, op in started.items()} - - # Start TPU after regular nodes so that regular nodes are not affected by the slower TPU nodes - execute_with_futures(tpu.start_tpu, tpu_chunks) - - for group, op in bulk_operations.items(): - _handle_bulk_insert_op(op, grouped_nodes[group].nodes, resume_data) - - -def _get_failed_zonal_instance_inserts(bulk_op: Any, zone: str, lkp: util.Lookup) -> list[Any]: - group_id = bulk_op["operationGroupId"] - user = bulk_op["user"] - started = bulk_op["startTime"] - ended = bulk_op["endTime"] - - fltr = f'(user eq "{user}") AND (operationType eq "insert") AND (creationTimestamp > "{started}") AND (creationTimestamp < "{ended}")' - act = lkp.compute.zoneOperations() - req = act.list(project=lkp.project, zone=zone, filter=fltr) - ops = [] - while req is not None: - result = util.ensure_execute(req) - for op in result.get("items", []): - if op.get("operationGroupId") == group_id and "error" in op: - ops.append(op) - req = act.list_next(req, result) - return ops - - -def _get_failed_instance_inserts(bulk_op: Any, lkp: util.Lookup) -> list[Any]: - zones = set() # gather zones that had failed inserts - for loc, stat in bulk_op.get("instancesBulkInsertOperationMetadata", {}).get("perLocationStatus", {}).items(): - pref, zone = loc.split("/", 1) - if not pref == "zones": - log.error(f"Unexpected location: {loc} in operation {bulk_op['name']}") - continue - if stat.get("targetVmCount", 0) != stat.get("createdVmCount", 0): - zones.add(zone) - - res = [] - for zone in zones: - res.extend(_get_failed_zonal_instance_inserts(bulk_op, zone, lkp)) - return res - -def _handle_bulk_insert_op(op: Dict, nodes: List[str], resume_data: Optional[ResumeData]) -> None: - """ - Handles **DONE** BulkInsert operations - """ - assert op["operationType"] == "bulkInsert" and op["status"] == "DONE", f"unexpected op: {op}" - - group_id = op["operationGroupId"] - if "error" in op: - error = op["error"]["errors"][0] - log.error( - f"bulkInsert operation error: {error['code']} name={op['name']} operationGroupId={group_id} nodes={to_hostlist(nodes)}" - ) - - created = 0 - for status in op["instancesBulkInsertOperationMetadata"]["perLocationStatus"].values(): - created += status.get("createdVmCount", 0) - if created == len(nodes): - log.info(f"created {len(nodes)} instances: nodes={to_hostlist(nodes)}") - return # no need to gather status of insert-operations. - - # TODO: don't gather insert-operations per bulkInsert request, instead aggregate it - # across all bulkInserts (goes one level above this function) - failed = _get_failed_instance_inserts(op, util.lookup()) - - # Multiple errors are possible, group by all of them (joined string codes) - by_error_inserts = util.groupby_unsorted( - failed, - lambda op: "+".join(err["code"] for err in op["error"]["errors"]), - ) - for code, failed_ops in by_error_inserts: - failed_ops = list(failed_ops) - failed_nodes = [trim_self_link(op["targetLink"]) for op in failed_ops] - hostlist = util.to_hostlist(failed_nodes) - log.error( - f"{len(failed_nodes)} instances failed to start: {code} ({hostlist}) operationGroupId={group_id}" - ) - - msg = "; ".join( - f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" - for err in failed_ops[0]["error"]["errors"] - ) - if code != "RESOURCE_ALREADY_EXISTS": - down_nodes_notify_jobs(failed_nodes, f"GCP Error: {msg}", resume_data) - log.error( - f"errors from insert for node '{failed_nodes[0]}' ({failed_ops[0]['name']}): {msg}" - ) - - -def down_nodes_notify_jobs(nodes: List[str], reason: str, resume_data: Optional[ResumeData]) -> None: - """set nodes down with reason""" - nodes_set = set(nodes) # turn into set to speed up intersection - jobs = resume_data.jobs if resume_data else [] - reason_quoted = shlex.quote(reason) - - for job in jobs: - if not (set(job.nodes_alloc) & nodes_set): - continue - run(f"{lookup().scontrol} update jobid={job.job_id} admincomment={reason_quoted}", check=False) - run(f"{lookup().scontrol} notify {job.job_id} {reason_quoted}", check=False) - - nodelist = util.to_hostlist(nodes) - log.error(f"Marking nodes {nodelist} as DOWN, reason: {reason}") - run(f"{lookup().scontrol} update nodename={nodelist} state=down reason={reason_quoted}", check=False) - - - - -def create_placement_request(pg_name: str, region: str, max_distance: Optional[int], accelerator_topology: Optional[str]): - config = { - "name": pg_name, - "region": region, - "groupPlacementPolicy": { - "collocation": "COLLOCATED", - "maxDistance": max_distance, - "gpuTopology": accelerator_topology, - }, - } - - request = lookup().compute.resourcePolicies().insert( - project=lookup().project, region=region, body=config - ) - log_api_request(request) - return request - - -def create_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - nodeset_map = collections.defaultdict(list) - for node in nodes: # split nodes on nodesets - nodeset_map[lkp.node_nodeset_name(node)].append(node) - - placements = [] - for _, ns_nodes in nodeset_map.items(): - placements.extend(create_nodeset_placements(ns_nodes, excl_job_id, lkp)) - return placements - - -def _allocate_nodes_to_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - # canned result for no placement policies created - no_pp = [PlacementAndNodes(placement=None, nodes=nodes)] - - model = nodes[0] - nodeset = lkp.node_nodeset(model) - - is_slice = bool(getattr(nodeset, 'accelerator_topology', None)) - - excl_job_placement = (excl_job_id is not None) and (not is_slice) - - if excl_job_placement and len(nodes) < 2: - return no_pp # don't create placement_policy for just one node - - if lkp.is_flex_node(model): - return no_pp # TODO(FLEX): Add support for workload policies - if lkp.node_is_tpu(model): - return no_pp - if not (nodeset.enable_placement and valid_placement_node(model)): - return no_pp - - max_count = calculate_chunk_size(nodeset, lkp) - - name_prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-{nodeset.nodeset_name}" - - if excl_job_placement: # simply chunk given nodes by max size of placement - return [ - PlacementAndNodes(placement=f"{name_prefix}-{excl_job_id}-{i}", nodes=chunk) - for i, chunk in enumerate(chunked(nodes, n=max_count)) - ] - - # split whole nodeset (not only nodes to resume) into chunks of max size of placement - # create placements (most likely already exists) placements for requested nodes - chunks = collections.defaultdict(list) # chunk_id -> nodes - invalid = [] - - for node in nodes: - try: - chunk = lkp.node_index(node) // max_count - chunks[chunk].append(node) - except: - invalid.append(node) - - placements = [ - # NOTE: use 0 instead of job_id for consistency with previous SlurmGCP behavior - PlacementAndNodes(placement=f"{name_prefix}-0-{c_id}", nodes=c_nodes) - for c_id, c_nodes in chunks.items() - ] - - if invalid: - placements.append(PlacementAndNodes(placement=None, nodes=invalid)) - log.error(f"Could not find placement for nodes with unexpected names: {to_hostlist(invalid)}") - - return placements - -def calculate_hosts_per_topo(accelerator_topology: str, machine_type: NSDict) -> int: - # Calculate total number of hosts per topology (Assumes format: '1x72') - try: - top_split = [int(x) for x in accelerator_topology.split("x")] - except Exception as e: - log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") - raise e - - if len(machine_type.accelerators) == 0: - gpus_per_machine = 0 - else: - gpus_per_machine = machine_type.accelerators[0].count - - if len(top_split) != 2: - log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") - elif top_split[0] <= 0 or top_split[1] <= 0: - log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") - elif gpus_per_machine <= 0: - log.error(f"The machine type has no accelerators. Cannot use accelerator topology {accelerator_topology}.") - elif top_split[1] % gpus_per_machine: - log.error(f"The GPU count {gpus_per_machine} per node is not a factor of the accelerator topology {accelerator_topology}") - - return (top_split[0] * top_split[1]) // gpus_per_machine - -def calculate_chunk_size(nodeset: NSDict, lkp: util.Lookup) -> int: - # Calculates the chunk size based on max distance value received or accelerator topology - # Assuming nodeset is not tpu - machine_type = lkp.template_info(nodeset.instance_template).machine_type - max_distance = nodeset.placement_max_distance - accelerator_topology = nodeset.accelerator_topology - - # Look for accelerator topology first - if accelerator_topology: - hosts_per_topo = calculate_hosts_per_topo(accelerator_topology, machine_type) - return hosts_per_topo - - if max_distance == 1: - return 22 - elif max_distance == 2: - if machine_type.family.startswith("a3"): - return 256 - else: - return 150 - elif max_distance == 3: - return 1500 - else: - return PLACEMENT_MAX_CNT - -def create_nodeset_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - placements = _allocate_nodes_to_placements(nodes, excl_job_id, lkp) - region = lkp.node_region(nodes[0]) - max_distance = lkp.node_nodeset(nodes[0]).get('placement_max_distance') - accelerator_topology = lkp.nodeset_accelerator_topology(lkp.node_nodeset_name(nodes[0])) - - if log.isEnabledFor(logging.DEBUG): - debug_p = {p.placement: to_hostlist(p.nodes) for p in placements} - log.debug( - f"creating {len(placements)} placement groups: \n{yaml.safe_dump(debug_p).rstrip()}" - ) - - requests = { - p.placement: create_placement_request(p.placement, region, max_distance, accelerator_topology) for p in placements if p.placement - } - if not requests: - return placements - # TODO: aggregate all requests for whole resume and execute them at once (don't limit to nodeset/job) - ops = dict( - zip(requests.keys(), map_with_futures(ensure_execute, requests.values())) - ) - - def classify_result(item): - op = item[1] - if not isinstance(op, Exception): - return "submitted" - if all(e.get("reason") == "alreadyExists" for e in op.error_details): # type: ignore - return "redundant" - return "failed" - - grouped_ops = dict(util.groupby_unsorted(list(ops.items()), classify_result)) - submitted, redundant, failed = ( - dict(grouped_ops.get(key, {})) for key in ("submitted", "redundant", "failed") - ) - if redundant: - log.warning( - "placement policies already exist: {}".format(",".join(redundant.keys())) - ) - if failed: - reqs = [f"{e}" for _, e in failed.values()] - log.fatal("failed to create placement policies: {}".format("; ".join(reqs))) - operations = {group: wait_for_operation(op) for group, op in submitted.items()} - for group, op in operations.items(): - if "error" in op: - msg = "; ".join( - f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" - for err in op["error"]["errors"] - ) - log.error( - f"placement group failed to create: '{group}' ({op['name']}): {msg}" - ) - - log.info( - f"created {len(operations)} placement groups ({to_hostlist(operations.keys())})" - ) - return placements - - -def valid_placement_node(node: str) -> bool: - invalid_types = frozenset(["e2", "t2d", "n1", "t2a", "m1", "m2", "m3"]) - mt = lookup().node_template_info(node).machineType - if mt.split("-")[0] in invalid_types: - log.warn(f"Unsupported machine type for placement policy: {mt}.") - log.warn( - f"Please do not use any the following machine types with placement policy: ({','.join(invalid_types)})" - ) - return False - return True - - -def main(nodelist: str) -> None: - """main called when run as script""" - log.debug(f"ResumeProgram {nodelist}") - # Filter out nodes not in config.yaml - other_nodes, nodes = separate( - lookup().is_power_managed_node, util.to_hostnames(nodelist) - ) - if other_nodes: - log.error( - f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" - ) - - if not nodes: - log.info("No nodes to resume") - return - resume_data = get_resume_file_data() - log.info(f"resume {util.to_hostlist(nodes)}") - resume_nodes(nodes, resume_data) - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("nodelist", help="list of nodes to resume") - args = util.init_log_and_parse(parser) - main(args.nodelist) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh deleted file mode 100644 index 023d246f01..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PYTHON_SCRIPT="${SCRIPT_DIR}/resume.py" - -# Capture all arguments passed by Slurm (the nodelist). -ALL_ARGS=("$@") - -# This array will hold extra argument for resume.py, like the resume data file. -UNIQUE_RESUME_FILE="" - -# Handle SLURM_RESUME_FILE if provided -if [ -n "${SLURM_RESUME_FILE-}" ] && [ -f "$SLURM_RESUME_FILE" ]; then - SAFE_DIR="/tmp/slurm_resume_data" - mkdir -p "$SAFE_DIR" - - UNIQUE_RESUME_FILE="${SAFE_DIR}/resumedata.$$.json" - cp "$SLURM_RESUME_FILE" "$UNIQUE_RESUME_FILE" -fi - -SLURM_RESUME_FILE="${UNIQUE_RESUME_FILE}" -setsid "${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & - -exit 0 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py deleted file mode 100644 index 846524adf2..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py +++ /dev/null @@ -1,660 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import logging -import os -import shutil -import subprocess -import stat -import time -import yaml -from pathlib import Path -import functools - -import util -from util import ( - lookup, - dirs, - slurmdirs, - run, - install_custom_scripts, -) -import conf -import slurmsync - -from setup_network_storage import ( - setup_network_storage, - setup_nfs_exports, -) - - -log = logging.getLogger() - - -MOTD_HEADER = """ - SSSSSSS - SSSSSSSSS - SSSSSSSSS - SSSSSSSSS - SSSS SSSSSSS SSSS - SSSSSS SSSSSS - SSSSSS SSSSSSS SSSSSS - SSSS SSSSSSSSS SSSS - SSS SSSSSSSSS SSS - SSSSS SSSS SSSSSSSSS SSSS SSSSS - SSS SSSSSS SSSSSSSSS SSSSSS SSS - SSSSSS SSSSSSS SSSSSS - SSS SSSSSS SSSSSS SSS - SSSSS SSSS SSSSSSS SSSS SSSSS - S SSS SSSSSSSSS SSS S - SSS SSSS SSSSSSSSS SSSS SSS - S SSS SSSSSS SSSSSSSSS SSSSSS SSS S - SSSSS SSSSSS SSSSSSSSS SSSSSS SSSSS - S SSSSS SSSS SSSSSSS SSSS SSSSS S - S SSS SSS SSS SSS S - S S S S - SSS - SSS - SSS - SSS - SSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSS SSSSSSSSSSSSSSSSSSSS -SSSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSSS SSSSSSSSSSSSSSSSSSSSSS -SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSSS SSS SSSSSSSSSSSSSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSS SSS SSSSSSSSSSSSS SSSS SSSS SSSS SSSS - -""" -_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" - -def start_motd(): - """advise in motd that slurm is currently configuring""" - wall_msg = "*** Slurm is currently being configured in the background. ***" - motd_msg = MOTD_HEADER + wall_msg + "\n\n" - Path("/etc/motd").write_text(motd_msg) - util.run(f"wall -n '{wall_msg}'", timeout=30) - - -def end_motd(broadcast=True): - """modify motd to signal that setup is complete""" - Path("/etc/motd").write_text(MOTD_HEADER) - - if not broadcast: - return - - run( - "wall -n '*** Slurm {} setup complete ***'".format(lookup().instance_role), - timeout=30, - ) - if not lookup().is_controller: - run( - """wall -n ' -/home on the controller was mounted over the existing /home. -Log back in to ensure your home directory is correct. -'""", - timeout=30, - ) - - -def failed_motd(): - """modify motd to signal that setup is failed""" - wall_msg = f"*** Slurm setup failed! Please view log: {util.get_log_path()} ***" - motd_msg = MOTD_HEADER + wall_msg + "\n\n" - Path("/etc/motd").write_text(motd_msg) - util.run(f"wall -n '{wall_msg}'", timeout=30) - - -def _startup_script_timeout(lkp: util.Lookup) -> int: - if lkp.is_controller: - return lkp.cfg.get("controller_startup_scripts_timeout", 300) - elif lkp.instance_role == "compute": - return lkp.cfg.get("compute_startup_scripts_timeout", 300) - elif lkp.is_login_node: - return lkp.cfg.login_groups[util.instance_login_group()].get("startup_scripts_timeout", 300) - return 300 - - -def run_custom_scripts(): - """run custom scripts based on instance_role""" - custom_dir = dirs.custom_scripts - if lookup().is_controller: - # controller has all scripts, but only runs controller.d - custom_dirs = [custom_dir / "controller.d"] - elif lookup().instance_role == "compute": - # compute setup with nodeset.d - custom_dirs = [custom_dir / "nodeset.d"] - elif lookup().is_login_node: - # login setup with only login.d - custom_dirs = [custom_dir / "login.d"] - else: - # Unknown role: run nothing - custom_dirs = [] - - timeout = _startup_script_timeout(lookup()) - - custom_scripts = [ - p - for d in custom_dirs - for p in d.rglob("*") - if p.is_file() and not p.name.endswith(".disabled") - ] - print_scripts = ",".join(str(s.relative_to(custom_dir)) for s in custom_scripts) - log.debug(f"custom scripts to run: {custom_dir}/({print_scripts})") - - try: - for script in custom_scripts: - log.info(f"running script {script.name} with timeout={timeout}") - result = run(str(script), timeout=timeout, check=False, shell=True) - runlog = ( - f"{script.name} returncode={result.returncode}\n" - f"stdout={result.stdout}stderr={result.stderr}" - ) - log.info(runlog) - result.check_returncode() - except OSError as e: - log.error(f"script {script} is not executable") - raise e - except subprocess.TimeoutExpired as e: - log.error(f"script {script} did not complete within timeout={timeout}") - raise e - except Exception as e: - log.exception(f"script {script} encountered an exception") - raise e - -def mount_save_state_disk(): - disk_name = f"/dev/disk/by-id/google-{lookup().cfg.controller_state_disk.device_name}" - mount_point = util.slurmdirs.state - fs_type = "ext4" - - rdevice = util.run(f"realpath {disk_name}").stdout.strip() - file_output = util.run(f"file -s {rdevice}").stdout.strip() - if "filesystem" not in file_output: - util.run(f"mkfs -t {fs_type} -q {rdevice}") - - fstab_entry = f"{disk_name} {mount_point} {fs_type}" - with open("/etc/fstab", "r") as f: - fstab = f.readlines() - if fstab_entry not in fstab: - with open("/etc/fstab", "a") as f: - f.write(f"{fstab_entry} defaults 0 0\n") - - util.run(f"systemctl daemon-reload") - - os.makedirs(mount_point, exist_ok=True) - util.run(f"mount {mount_point}") - - util.chown_slurm(mount_point) - - -def setup_jwt_key(): - jwt_key = Path(slurmdirs.state / "jwt_hs256.key") - - if jwt_key.exists(): - log.info("JWT key already exists. Skipping key generation.") - else: - run("dd if=/dev/urandom bs=32 count=1 > " + str(jwt_key), shell=True) - - util.chown_slurm(jwt_key, mode=0o400) - - -def _generate_key(p: Path) -> None: - run(f"dd if=/dev/random of={p} bs=1024 count=1") - - -def setup_key(lkp: util.Lookup) -> None: - file_name = "munge.key" - dir = dirs.munge - - if lkp.cfg.enable_slurm_auth: - file_name = "slurm.key" - dir = slurmdirs.etc - - dst = Path(dir / file_name) - - if lkp.cfg.controller_state_disk.device_name: - # Copy key from persistent state disk - persist = slurmdirs.state / file_name - if not persist.exists(): - _generate_key(persist) - - shutil.copyfile(persist, dst) - if lkp.cfg.enable_slurm_auth: - util.chown_slurm(dst, mode=0o400) - util.chown_slurm(persist, mode=0o400) - else: - shutil.chown(dst, user="munge", group="munge") - os.chmod(dst, stat.S_IRUSR) - else: - if dst.exists(): - log.info("key already exists. Skipping key generation.") - else: - _generate_key(dst) - if lkp.cfg.enable_slurm_auth: - util.chown_slurm(dst, mode=0o400) - else: - shutil.chown(dst, user="munge", group="munge") - os.chmod(dst, stat.S_IRUSR) - - if lkp.cfg.enable_slurm_auth: - # Put key into shared volume for distribution - distributed = util.slurmdirs.key_distribution / file_name - shutil.copyfile(dst, distributed) - util.chown_slurm(distributed, mode=0o400) - # Munge is distributed from /etc/munge. - else: - run("systemctl restart munge", timeout=30) - - -def setup_nss_slurm(): - """install and configure nss_slurm""" - # setup nss_slurm - util.mkdirp(Path("/var/spool/slurmd")) - run( - "ln -s {}/lib/libnss_slurm.so.2 /usr/lib64/libnss_slurm.so.2".format( - slurmdirs.prefix - ), - check=False, - ) - run(r"sed -i 's/\(^\(passwd\|group\):\s\+\)/\1slurm /g' /etc/nsswitch.conf") - - -def setup_sudoers(): - content = """ -# Allow SlurmUser to manage the slurm daemons -slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmd.service -slurm ALL= NOPASSWD: /usr/bin/systemctl restart sackd.service -slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmctld.service -""" - sudoers_file = Path("/etc/sudoers.d/slurm") - sudoers_file.write_text(content) - sudoers_file.chmod(0o0440) - - -def setup_maintenance_script(): - perform_maintenance = """#!/bin/bash - -#SBATCH --priority=low -#SBATCH --time=180 - -VM_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") -ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" | cut -d '/' -f 4) - -gcloud compute instances perform-maintenance $VM_NAME \ - --zone=$ZONE -""" - - - with open(_MAINTENANCE_SBATCH_SCRIPT_PATH, "w") as f: - f.write(perform_maintenance) - - util.chown_slurm(_MAINTENANCE_SBATCH_SCRIPT_PATH, mode=0o755) - - -def update_system_config(file, content): - """Add system defaults options for service files""" - sysconfig = Path("/etc/sysconfig") - default = Path("/etc/default") - - if sysconfig.exists(): - conf_dir = sysconfig - elif default.exists(): - conf_dir = default - else: - raise Exception("Cannot determine system configuration directory.") - - slurmd_file = Path(conf_dir, file) - slurmd_file.write_text(content) - -def _symlink_mysql_datadir(lkp: util.Lookup) -> None: - """ Symlink /var/lib/mysql to controller state disk if needed. """ - if not lkp.cfg.controller_state_disk.device_name: - return - - datadir = Path("/var/lib/mysql") - dst = slurmdirs.state / "mysql" - - if dst.exists(): - run(f"rm -rf {datadir}") - else: - shutil.move(datadir, dst) - - datadir.symlink_to(dst, target_is_directory=True) - shutil.chown(datadir, user="mysql", group="mysql") - run(f"chown -R mysql:mysql {dst}") - -def configure_mysql(lkp: util.Lookup) -> None: - cnfdir = Path("/etc/my.cnf.d") - if not cnfdir.exists(): - cnfdir = Path("/etc/mysql/conf.d") - if not (cnfdir / "mysql_slurm.cnf").exists(): - (cnfdir / "mysql_slurm.cnf").write_text( - """ -[mysqld] -bind-address=127.0.0.1 -innodb_buffer_pool_size=1024M -innodb_log_file_size=64M -innodb_lock_wait_timeout=900 -""" - ) - - run("systemctl stop mariadb", timeout=30) - _symlink_mysql_datadir(lkp) - - run("systemctl enable mariadb", timeout=30) - run("systemctl restart mariadb", timeout=30) - - db_name = "slurm_acct_db" - - - cmd = "mysql -u root -e" - for host in ("localhost", lkp.control_host): - run(f"""{cmd} "drop user if exists 'slurm'@'{host}'";""", timeout=30) - run(f"""{cmd} "create user 'slurm'@'{host}'";""", timeout=30) - run(f"""{cmd} "grant all on {db_name}.* TO 'slurm'@'{host}'";""", timeout=30) - - -def configure_dirs(): - for p in dirs.values(): - util.mkdirp(p) - - for p in (dirs.slurm, dirs.scripts, dirs.custom_scripts): - util.chown_slurm(p) - - for p in slurmdirs.values(): - util.mkdirp(p) - util.chown_slurm(p) - - for sl, tgt in ( # create symlinks - (Path("/etc/slurm"), slurmdirs.etc), - (dirs.scripts / "etc", slurmdirs.etc), - (dirs.scripts / "log", dirs.log), - ): - if sl.exists() and sl.is_symlink(): - sl.unlink() - sl.symlink_to(tgt) - - # copy auxiliary scripts - for dst_folder, src_file in ((lookup().cfg.slurm_bin_dir, - Path("sort_nodes.py")), - (dirs.custom_scripts / "task_prolog.d", - Path("tools/task-prolog")), - (dirs.custom_scripts / "task_epilog.d", - Path("tools/task-epilog"))): - dst = Path(dst_folder) / src_file.name - util.mkdirp(dst.parent) - shutil.copyfile(util.scripts_dir / src_file, dst) - os.chmod(dst, 0o755) - - -def self_report_controller_address(lkp: util.Lookup) -> None: - if not lkp.cfg.controller_network_attachment: - return # only self report address if network attachment is used - data = { "slurm_control_addr": lkp.cfg.slurm_control_addr } - bucket, prefix = util._get_bucket_and_common_prefix() - blob = util.storage_client().bucket(bucket).blob(f"{prefix}/controller_addr.yaml") - with blob.open('w') as f: - f.write(yaml.dump(data)) - -def setup_controller(): - """Run controller setup""" - log.info("Setting up controller") - lkp = util.lookup() - util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) - install_custom_scripts() - conf.gen_controller_configs(lkp) - - if lkp.cfg.controller_state_disk.device_name != None: - mount_save_state_disk() - - setup_jwt_key() - setup_key(lkp) - - setup_sudoers() - setup_network_storage() - - run_custom_scripts() - - if not lkp.cfg.cloudsql_secret: - configure_mysql(lkp) - - run("systemctl enable slurmdbd", timeout=30) - run("systemctl restart slurmdbd", timeout=30) - - # Wait for slurmdbd to come up - time.sleep(5) - - sacctmgr = f"{slurmdirs.prefix}/bin/sacctmgr -i" - result = run( - f"{sacctmgr} add cluster {lkp.cfg.slurm_cluster_name}", timeout=30, check=False - ) - if "already exists" in result.stdout: - log.info(result.stdout) - elif result.returncode > 1: - result.check_returncode() # will raise error - - run("systemctl enable slurmctld", timeout=30) - run("systemctl restart slurmctld", timeout=30) - - run("systemctl enable slurmrestd", timeout=30) - run("systemctl restart slurmrestd", timeout=30) - - # Export at the end to signal that everything is up - run("systemctl enable nfs-server", timeout=30) - run("systemctl start nfs-server", timeout=30) - - setup_nfs_exports() - run("systemctl enable --now slurmcmd.timer", timeout=30) - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status slurmdbd", timeout=30) - run("systemctl status slurmctld", timeout=30) - run("systemctl status slurmrestd", timeout=30) - - try: - slurmsync.sync_instances() - except Exception: - log.exception("Failed to sync instances, will try next time.") - - run("systemctl enable slurm_load_bq.timer", timeout=30) - run("systemctl start slurm_load_bq.timer", timeout=30) - run("systemctl status slurm_load_bq.timer", timeout=30) - - # Add script to perform maintenance - setup_maintenance_script() - - self_report_controller_address(lkp) - - log.info("Done setting up controller") - pass - - -def setup_login(): - """run login node setup""" - log.info("Setting up login") - - lkp = lookup() - slurmctld_host = f"{lkp.control_host}" - if lkp.control_addr: - slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" - sackd_options = [ - f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', - ] - sysconf = f"""SACKD_OPTIONS='{" ".join(sackd_options)}'""" - update_system_config("sackd", sysconf) - install_custom_scripts() - - setup_network_storage() - setup_sudoers() - if not lkp.cfg.enable_slurm_auth: - run("systemctl restart munge", timeout=30) - run("systemctl enable sackd", timeout=30) - run("systemctl restart sackd", timeout=30) - run("systemctl enable --now slurmcmd.timer", timeout=30) - - run_custom_scripts() - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status sackd", timeout=30) - - log.info("Done setting up login") - - -def setup_compute(): - """run compute node setup""" - log.info("Setting up compute") - - lkp = lookup() - util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) - slurmctld_host = f"{lkp.control_host}" - if lkp.control_addr: - slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" - slurmd_options = [ - f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', - ] - - try: - slurmd_feature = util.instance_metadata("attributes/slurmd_feature", silent=True) - except util.MetadataNotFoundError: - slurmd_feature = None - - if slurmd_feature is not None: - slurmd_options.append(f'--conf="Feature={slurmd_feature}"') - slurmd_options.append("-Z") - - sysconf = f"""SLURMD_OPTIONS='{" ".join(slurmd_options)}'""" - update_system_config("slurmd", sysconf) - install_custom_scripts() - - setup_nss_slurm() - setup_network_storage() - - has_gpu = run("lspci | grep --ignore-case 'NVIDIA' | wc -l", shell=True).returncode - if has_gpu: - run("nvidia-smi") - - run_custom_scripts() - - setup_sudoers() - if not lkp.cfg.enable_slurm_auth: - run("systemctl restart munge", timeout=30) - run("systemctl enable slurmd", timeout=30) - run("systemctl restart slurmd", timeout=30) - run("systemctl enable --now slurmcmd.timer", timeout=30) - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status slurmd", timeout=30) - - log.info("Done setting up compute") - -def setup_cloud_ops() -> None: - """Add health checks, deployment info, and updated setup path to cloud ops config.""" - cloudOpsStatus = run( - "systemctl is-active --quiet google-cloud-ops-agent.service", check=False - ).returncode - - if cloudOpsStatus != 0: - return - - with open("/etc/google-cloud-ops-agent/config.yaml", "r") as f: - file = yaml.safe_load(f) - - # Update setup receiver path - file["logging"]["receivers"]["setup"]["include_paths"] = ["/var/log/slurm/setup.log"] - - cluster_info = { - 'type':'modify_fields', - 'fields': { - 'labels."cluster_name"':{ - 'static_value':f"{lookup().cfg.slurm_cluster_name}" - }, - 'labels."hostname"':{ - 'static_value': f"{lookup().hostname}" - } - } - } - - file["logging"]["processors"]["add_cluster_info"] = cluster_info - file["logging"]["service"]["pipelines"]["slurmlog_pipeline"]["processors"].append("add_cluster_info") - file["logging"]["service"]["pipelines"]["slurmlog2_pipeline"]["processors"].append("add_cluster_info") - - with open("/etc/google-cloud-ops-agent/config.yaml", "w") as f: - yaml.safe_dump(file, f, sort_keys=False) - - retries = 2 - for _ in range(retries): - try: - run("systemctl restart google-cloud-ops-agent.service", timeout=120) - break - except subprocess.TimeoutExpired: - log.error("google-cloud-ops-agent.service did not restart within 120s.") - result=run("cat /var/log/google-cloud-ops-agent/subagents/logging-module.log", timeout=120, shell=True) - if result.stdout: - log.error(f"Logs for google-cloud-ops-agent (logging-module.log file):\n{result.stdout}") - raise - - -def main(): - start_motd() - - log.info("Starting setup, fetching config") - sleep_seconds = 5 - while True: - try: - _, cfg = util.fetch_config() - util.update_config(cfg) - break - except util.DeffetiveStoredConfigError as e: - log.warning(f"config is not ready yet: {e}, sleeping for {sleep_seconds}s") - except Exception as e: - log.exception(f"unexpected error while fetching config, sleeping for {sleep_seconds}s") - time.sleep(sleep_seconds) - log.info("Config fetched") - setup_cloud_ops() - configure_dirs() - # call the setup function for the instance type - { - "controller": setup_controller, - "compute": setup_compute, - "login": setup_login, - }.get( - lookup().instance_role, - lambda: log.fatal(f"Unknown node role: {lookup().instance_role}"))() - - end_motd() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--slurmd-feature", dest="slurmd_feature", help="Unused, to be removed.") - _ = util.init_log_and_parse(parser) - - try: - main() - except Exception: - log.exception("Aborting setup...") - failed_motd() diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py deleted file mode 100644 index 095f42e758..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py +++ /dev/null @@ -1,327 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List - -import os -import sys -import stat -import time -import logging -import uuid - -import shutil -from pathlib import Path -from concurrent.futures import as_completed -from addict import Dict as NSDict # type: ignore - -import util -from util import NSMount, lookup, run, dirs, separate -from more_executors import Executors, ExceptionRetryPolicy - - -log = logging.getLogger() - -def mounts_by_local(mounts: list[NSMount]) -> dict[str, NSMount]: - """convert list of mounts to dict of mounts, local_mount as key""" - return {str(m.local_mount.resolve()): m for m in mounts} - - -def _get_default_mounts(lkp: util.Lookup) -> list[NSMount]: - if lkp.cfg.disable_default_mounts: - return [] - return [ - NSMount( - server_ip=lkp.controller_mount_server_ip(), - remote_mount=path, - local_mount=path, - fs_type="nfs", - mount_options="defaults,hard,intr", - ) - for path in ( - dirs.home, - dirs.apps, - ) - ] - -def get_slurm_bucket_mount() -> NSMount: - bucket, path = util._get_bucket_and_common_prefix() - return NSMount( - fs_type="gcsfuse", - server_ip="", - remote_mount=Path(bucket), - local_mount=dirs.slurm_bucket_mount, - mount_options=f"defaults,_netdev,implicit_dirs,only_dir={path}", - ) - -def resolve_network_storage() -> List[NSMount]: - """Combine appropriate network_storage fields to a single list""" - lkp = lookup() - - # create dict of mounts, local_mount: mount_info - mounts = mounts_by_local(_get_default_mounts(lkp)) - - if lkp.is_controller and util.should_mount_slurm_bucket(): - mounts.update(mounts_by_local([get_slurm_bucket_mount()])) - - # On non-controller instances, entries in network_storage could overwrite - # default exports from the controller. Be careful, of course - common = [lkp.normalize_ns_mount(m) for m in lkp.cfg.network_storage] - mounts.update(mounts_by_local(common)) - - if lkp.is_login_node: - login_group = lkp.cfg.login_groups[util.instance_login_group()] - login_ns = [lkp.normalize_ns_mount(m) for m in login_group.network_storage] - mounts.update(mounts_by_local(login_ns)) - - if lkp.instance_role == "compute": - try: - nodeset = lkp.node_nodeset() - except Exception: - pass # external nodename, skip lookup - else: - nodeset_ns = [lkp.normalize_ns_mount(m) for m in nodeset.network_storage] - mounts.update(mounts_by_local(nodeset_ns)) - - return list(mounts.values()) - - -def is_controller_mount(mount) -> bool: - # NOTE: Valid Lustre server_ip can take the form of '@tcp' - server_ip = mount.server_ip.split("@")[0] - mount_addr = util.host_lookup(server_ip) - return mount_addr == lookup().control_host_addr - -def setup_network_storage(): - """prepare network fs mounts and add them to fstab""" - log.info("Set up network storage") - - all_mounts = resolve_network_storage() - if lookup().is_controller: - mounts, _ = separate(is_controller_mount, all_mounts) - else: - mounts = all_mounts - - # Determine fstab entries and write them out - fstab_entries = [] - for mount in mounts: - local_mount = mount.local_mount - fs_type = mount.fs_type - server_ip = mount.server_ip or "" - src = mount.remote_mount if fs_type == "gcsfuse" else f"{server_ip}:{mount.remote_mount}" - - log.info(f"Setting up mount ({fs_type}) {src} to {local_mount}") - util.mkdirp(local_mount) - - mount_options = mount.mount_options.split(",") if mount.mount_options else [] - if "_netdev" not in mount_options: - mount_options += ["_netdev"] - options_line = ",".join(mount_options) - - - fstab_entries.append(f"{src} {local_mount} {fs_type} {options_line} 0 0") - - fstab = Path("/etc/fstab") - if not Path(fstab.with_suffix(".bak")).is_file(): - shutil.copy2(fstab, fstab.with_suffix(".bak")) - shutil.copy2(fstab.with_suffix(".bak"), fstab) - with open(fstab, "a") as f: - f.write("\n") - for entry in fstab_entries: - f.write(entry) - f.write("\n") - - mount_fstab(mounts, log) - if lookup().cfg.enable_slurm_auth: - slurm_key_mount_handler() - else: - munge_mount_handler() - - -def mount_fstab(mounts: list[NSMount], log): - """Wait on each mount, then make sure all fstab is mounted""" - def mount_path(path: Path): - log.info(f"Waiting for '{path}' to be mounted...") - try: - run(f"mount {path}", timeout=120) - except Exception as e: - exc_type, _, _ = sys.exc_info() - log.error(f"mount of path '{path}' failed: {exc_type}: {e}") - raise e - log.info(f"Mount point '{path}' was mounted.") - - MAX_MOUNT_TIMEOUT = 60 * 5 - future_list = [] - retry_policy = ExceptionRetryPolicy( - max_attempts=120, exponent=1.6, sleep=1.0, max_sleep=16.0 - ) - with Executors.thread_pool().with_timeout(MAX_MOUNT_TIMEOUT).with_retry( - retry_policy=retry_policy - ) as exe: - for m in mounts: - future = exe.submit(mount_path, m.local_mount) - future_list.append(future) - - # Iterate over futures, checking for exceptions - for future in as_completed(future_list): - try: - future.result() - except Exception as e: - raise e - - -def munge_mount_handler(): - if lookup().is_controller: - return - mnt = lookup().munge_mount - - log.info(f"Mounting munge share to: {mnt.local_mount}") - mnt.local_mount.mkdir() - if mnt.fs_type == "gcsfuse": - cmd = [ - "gcsfuse", - f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, - mnt.server_ip, - str(mnt.local_mount), - ] - else: - cmd = [ - "mount", - f"--types={mnt.fs_type}", - f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, - f"{mnt.server_ip}:{mnt.remote_mount}", - str(mnt.local_mount), - ] - # wait max 240s for munge mount - timeout = 240 - for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): - try: - run(cmd, timeout=timeout) - break - except Exception as e: - log.error( - f"munge mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" - ) - time.sleep(wait) - err = e - continue - else: - raise err - - munge_key = Path(dirs.munge / "munge.key") - log.info(f"Copy munge.key from: {mnt.local_mount}") - shutil.copy2(Path(mnt.local_mount / "munge.key"), munge_key) - - log.info("Restrict permissions of munge.key") - shutil.chown(munge_key, user="munge", group="munge") - os.chmod(munge_key, stat.S_IRUSR) - - log.info(f"Unmount {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - run(f"fusermount -u {mnt.local_mount}", timeout=120) - else: - run(f"umount {mnt.local_mount}", timeout=120) - shutil.rmtree(mnt.local_mount) - -def slurm_key_mount_handler(): - if lookup().is_controller: - return - mnt = lookup().slurm_key_mount - - log.info(f"Mounting slurm_key share to: {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - cmd = [ - "gcsfuse", - f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, - mnt.server_ip, - str(mnt.local_mount), - ] - else: - cmd = [ - "mount", - f"--types={mnt.fs_type}", - f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, - f"{mnt.server_ip}:{mnt.remote_mount}", - str(mnt.local_mount), - ] - timeout = 120 # wait max 120s to mount - for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): - try: - run(cmd, timeout=timeout) - break - except Exception as e: - log.error( - f"slurm key mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" - ) - time.sleep(wait) - err = e - continue - else: - raise err - - file_name = "slurm.key" - dst = Path(util.slurmdirs.etc / file_name) - log.info(f"Copy slurm.key from: {mnt.local_mount}") - shutil.copy2(mnt.local_mount / file_name, dst) - - log.info("Restrict permissions of slurm.key") - util.chown_slurm(dst, mode=0o400) - - log.info(f"Unmount {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - run(f"fusermount -u {mnt.local_mount}", timeout=120) - else: - run(f"umount {mnt.local_mount}", timeout=120) - shutil.rmtree(mnt.local_mount) - - -def setup_nfs_exports(): - """nfs export all needed directories""" - lkp = util.lookup() - assert lkp.is_controller - - # The controller only needs to set up exports for cluster-internal mounts - exported_mounts = [m for m in resolve_network_storage() if is_controller_mount(m)] - - # key by remote mount path since that is what needs exporting - to_export = {m.remote_mount: "*(rw,no_subtree_check,no_root_squash)" for m in exported_mounts} - - key_mount = lkp.slurm_key_mount if lkp.cfg.enable_slurm_auth else lkp.munge_mount - if is_controller_mount(key_mount): - # Export key mount as read-only - to_export[key_mount.remote_mount] = "*(ro,no_subtree_check,no_root_squash)" - - if util.should_mount_slurm_bucket(): - mnt = get_slurm_bucket_mount() - # FSID is required for virtual filesystem that is not based on a device - # Also export it as read-only - fsid=str(uuid.uuid4()) - to_export[mnt.local_mount] = f"*(ro,no_subtree_check,no_root_squash,fsid={fsid})" - - # export path if corresponding selector boolean is True - lines = [] - for path,options in to_export.items(): - util.mkdirp(Path(path)) - run(rf"sed -i '\#{path}#d' /etc/exports", timeout=30) - lines.append(f"{path} {options}") - - exportsd = Path("/etc/exports.d") - util.mkdirp(exportsd) - with (exportsd / "slurm.exports").open("w") as f: - f.write("\n") - f.write("\n".join(lines)) - run("exportfs -a", timeout=30) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py deleted file mode 100644 index 1bfdd5acce..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py +++ /dev/null @@ -1,679 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import fcntl -import json -import logging -import re -import sys -import shlex -from datetime import datetime, timedelta -from itertools import chain -from pathlib import Path -from dataclasses import dataclass -from typing import Dict, Tuple, List, Optional, Protocol, Any -from functools import lru_cache - -import util -from util import ( - batch_execute, - ensure_execute, - execute_with_futures, - FutureReservation, - install_custom_scripts, - run, - separate, - to_hostlist, - NodeState, - chunked, - dirs, -) -from util import lookup -from suspend import delete_instances -import tpu -import conf -import watch_delete_vm_op - -log = logging.getLogger() - -TOT_REQ_CNT = 1000 -_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" - -class NodeAction(Protocol): - def apply(self, nodes:List[str]) -> None: - ... - - def __hash__(self): - ... - -@dataclass(frozen=True) -class NodeActionPowerUp(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to resume ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=power_up") - -@dataclass(frozen=True) -class NodeActionIdle(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} nodes to idle ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=resume") - -@dataclass(frozen=True) -class NodeActionPowerDown(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to power down ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=power_down") - - -@dataclass(frozen=True) -class NodeActionPowerDownForce(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to power down ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=power_down_force") - - -@dataclass(frozen=True) -class NodeActionDelete(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to delete ({hostlist})") - delete_instances(nodes) - -@dataclass(frozen=True) -class NodeActionPrempt(): - def apply(self, nodes:List[str]) -> None: - NodeActionDown(reason="Preempted instance").apply(nodes) - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances restarted ({hostlist})") - start_instances(nodes) - -@dataclass(frozen=True) -class NodeActionUnchanged(): - def apply(self, nodes:List[str]) -> None: - pass - -@dataclass(frozen=True) -class NodeActionDown(): - reason: str - - def apply(self, nodes: List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} nodes set down ({hostlist}) with reason={self.reason}") - run(f"{lookup().scontrol} update nodename={hostlist} state=down reason={shlex.quote(self.reason)}") - -@dataclass(frozen=True) -class NodeActionUnknown(): - slurm_state: Optional[NodeState] - instance_state: Optional[str] - - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.error(f"{len(nodes)} nodes have unexpected {self.slurm_state} and instance state:{self.instance_state}, ({hostlist})") - -def start_instance_op(node: str) -> Any: - inst = lookup().instance(node) - assert inst - - return lookup().compute.instances().start( - project=lookup().project, - zone=inst.zone, - instance=inst.name, - ) - - -def start_instances(node_list): - log.info("{} instances to start ({})".format(len(node_list), ",".join(node_list))) - lkp = lookup() - # TODO: use code from resume.py to assign proper placement - normal, tpu_nodes = separate(lkp.node_is_tpu, node_list) - ops = {node: start_instance_op(node) for node in normal} - - done, failed = batch_execute(ops) - - tpu_start_data = [] - for ns, nodes in util.groupby_unsorted(tpu_nodes, lkp.node_nodeset_name): - tpuobj = tpu.TPU.make(ns, lkp) - for snodes in chunked(nodes, n=tpuobj.vmcount): - tpu_start_data.append({"tpu": tpuobj, "node": snodes}) - execute_with_futures(tpu.start_tpu, tpu_start_data) - - -def _find_dynamic_node_status() -> NodeAction: - # TODO: cover more cases: - # * delete dead dynamic nodes - # * delete orhpaned instances - return NodeActionUnchanged() # don't touch dynamic nodes - -def get_fr_action(fr: FutureReservation, state:Optional[NodeState]) -> Optional[NodeAction]: - now = util.now() - if state is None: - return None # handle like any other node - if fr.start_time < now < fr.end_time: - return None # handle like any other node - - if state.base == "DOWN": - return NodeActionUnchanged() - if fr.start_time >= now: - msg = f"Waiting for reservation:{fr.name} to start at {fr.start_time}" - else: - msg = f"Reservation:{fr.name} is after its end-time" - return NodeActionDown(reason=msg) - -def _find_tpu_node_action(nodename, state) -> NodeAction: - lkp = lookup() - tpuobj = tpu.TPU.make(lkp.node_nodeset_name(nodename), lkp) - inst = tpuobj.get_node(nodename) - # If we do not find the node but it is from a Tpu that has multiple vms look for the master node - if inst is None and tpuobj.vmcount > 1: - # Get the tpu slurm nodelist of the nodes in the same tpu group as nodename - nodelist = run( - f"{lkp.scontrol} show topo {nodename}" - + " | awk -F'=' '/Level=0/ { print $NF }'", - shell=True, - ).stdout - l_nodelist = util.to_hostnames(nodelist) - group_names = set(l_nodelist) - # get the list of all the existing tpus in the nodeset - tpus_list = set(tpuobj.list_node_names()) - # In the intersection there must be only one node that is the master - tpus_int = list(group_names.intersection(tpus_list)) - if len(tpus_int) > 1: - log.error( - f"More than one cloud tpu node for tpu group {nodelist}, there should be only one that should be {l_nodelist[0]}, but we have found {tpus_int}" - ) - return NodeActionUnknown(slurm_state=state, instance_state=None) - if len(tpus_int) == 1: - inst = tpuobj.get_node(tpus_int[0]) - # if len(tpus_int ==0) this case is not relevant as this would be the case always that a TPU group is not running - if inst is None: - if state.base == "DOWN" and "POWERED_DOWN" in state.flags: - return NodeActionIdle() - if "POWERING_DOWN" in state.flags: - return NodeActionIdle() - if "COMPLETING" in state.flags: - return NodeActionDown(reason="Unbacked instance") - if state.base != "DOWN" and not ( - set(("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN")) - & state.flags - ): - return NodeActionDown(reason="Unbacked instance") - if lkp.is_static_node(nodename): - return NodeActionPowerUp() - elif ( - state is not None - and "POWERED_DOWN" not in state.flags - and "POWERING_DOWN" not in state.flags - and inst.state == tpu.TPU.State.STOPPED - ): - if tpuobj.preemptible: - return NodeActionPrempt() - if state.base != "DOWN": - return NodeActionDown(reason="Instance terminated") - elif ( - state is None or "POWERED_DOWN" in state.flags - ) and inst.state == tpu.TPU.State.READY: - return NodeActionDelete() - elif state is None: - # if state is None here, the instance exists but it's not in Slurm - return NodeActionUnknown(slurm_state=state, instance_state=inst.status) - - return NodeActionUnchanged() - -def get_node_action(nodename: str) -> NodeAction: - """Determine node/instance status that requires action""" - lkp = lookup() - state = lkp.node_state(nodename) - - if lkp.node_is_gke(nodename): - return NodeActionUnchanged() - - if lkp.node_is_fr(nodename): - fr = lkp.future_reservation(lkp.node_nodeset(nodename)) - assert fr - if action := get_fr_action(fr, state): - return action - - if lkp.node_is_dyn(nodename): - return _find_dynamic_node_status() - - if lkp.node_is_tpu(nodename): - return _find_tpu_node_action(nodename, state) - - # split below is workaround for VMs whose hostname is FQDN - inst = lkp.instance(nodename.split(".")[0]) - power_flags = frozenset( - ("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN") - ) & (state.flags if state is not None else set()) - - if (state is None) and (inst is None): - # Should never happen - return NodeActionUnknown(None, None) - if inst is None: - assert state is not None # to keep type-checker happy - if "POWERING_UP" in state.flags: - return NodeActionUnchanged() - if state.base == "DOWN" and "POWERED_DOWN" in state.flags: - return NodeActionIdle() - if "POWERING_DOWN" in state.flags: - return NodeActionIdle() - if "COMPLETING" in state.flags: - return NodeActionDown(reason="Unbacked instance") - if state.base != "DOWN" and not power_flags: - return NodeActionDown(reason="Unbacked instance") - if state.base == "DOWN" and not power_flags: - return NodeActionPowerDown() - if "NOT_RESPONDING" in state.flags: - return NodeActionPowerDown() - if "POWERED_DOWN" in state.flags and lkp.is_static_node(nodename): - return NodeActionPowerUp() - elif ( - state is not None - and "POWERED_DOWN" not in state.flags - and "POWERING_DOWN" not in state.flags - and inst.status == "TERMINATED" - ): - if inst.scheduling.preemptible: - return NodeActionPrempt() - if state.base != "DOWN": - return NodeActionDown(reason="Instance terminated") - elif (state is None or "POWERED_DOWN" in state.flags) and inst.status == "RUNNING": - log.info("%s is potential orphan node", nodename) - threshold = timedelta(seconds=90) - age = util.now() - inst.creation_timestamp - log.info(f"{nodename} state: {state}, age: {age}") - if age < threshold: - log.info(f"{nodename} not marked as orphan, it started less than {threshold.seconds}s ago ({age.seconds}s)") - return NodeActionUnchanged() - return NodeActionDelete() - elif state is None: - # if state is None here, the instance exists but it's not in Slurm - return NodeActionUnknown(slurm_state=state, instance_state=inst.status) - elif lkp.is_flex_node(nodename) and "POWERING_UP" in state.flags: - threshold = timedelta(seconds=int(lkp.cfg.compute_startup_scripts_timeout) * 2) #extra buffer for unexpectedly long startup scripts - if util.now() - inst.creation_timestamp > threshold: - log.info(f"{nodename} was unable to join the cluster after {threshold.seconds}s, potential failure on VM startup. Powering down...") - return NodeActionPowerDownForce() - return NodeActionUnchanged() - - -def delete_resource_policies(links: list[str], lkp: util.Lookup) -> None: - requests = {} - for link in links: - name = util.trim_self_link(link) - region = util.parse_self_link(link).region - requests[name] = lkp.compute.resourcePolicies().delete(project=lkp.project, region=region, resourcePolicy=name) - - def swallow_err(_: str) -> None: - pass - - done, failed = batch_execute(requests, log_err=swallow_err) - if failed: - # Filter out resourceInUseByAnotherResource errors , they are expected to happen - def ignore_err(e) -> bool: - return "resourceInUseByAnotherResource" in str(e) - - failures = [f"{n}: {e}" for n, (_, e) in failed.items() if not ignore_err(e)] - if failures: - log.error(f"some placement groups failed to delete: {failures}") - log.info( - f"deleted {len(done)} of {len(links)} placement groups ({to_hostlist(done.keys())})" - ) - - - -@lru_cache -def _get_resource_policies_in_region(lkp: util.Lookup, region: str) -> list[Any]: - res = [] - act = lkp.compute.resourcePolicies() - op = act.list(project=lkp.project, region=region) - prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-" - while op is not None: - result = ensure_execute(op) - res.extend([p for p in result.get("items", []) if p.get("name", "").startswith(prefix)]) - op = act.list_next(op, result) - return res - - -@lru_cache -def _get_resource_policies(lkp: util.Lookup) -> list[Any]: - res = [] - for region in lkp.cluster_regions(): - res.extend(_get_resource_policies_in_region(lkp, region)) - return res - -def sync_placement_groups(): - """Delete placement policies that are for jobs that have completed/terminated""" - keep_states = frozenset( - [ - "RUNNING", - "CONFIGURING", - "STOPPED", - "SUSPENDED", - "COMPLETING", - "PENDING", - ] - ) - - lkp = lookup() - keep_jobs = { - str(job.id) - for job in lkp.get_jobs() - if job.job_state in keep_states - } - keep_jobs.add("0") # Job 0 is a placeholder for static node placement - - to_delete = [] - pg_regex = re.compile( - rf"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-(?P[^\s\-]+)-(?P\d+)-(?P\d+)" - ) - - for pg in _get_resource_policies(lkp): - name = pg["name"] - - if (mtch := pg_regex.match(name)) is None: - log.warning(f"Unexpected resource policy {name=}") - continue - if mtch.group("job_id") not in keep_jobs: - to_delete.append(pg["selfLink"]) - - if to_delete: - delete_resource_policies(to_delete, lkp) - - -def sync_instances(): - compute_instances = { - name for name, inst in lookup().instances().items() if inst.role == "compute" - } - slurm_nodes = set(lookup().slurm_nodes().keys()) - log.debug(f"reconciling {len(compute_instances)} GCP instances and {len(slurm_nodes)} Slurm nodes.") - - for action, nodes in util.groupby_unsorted(list(compute_instances | slurm_nodes), get_node_action): - action.apply(list(nodes)) - - -def reconfigure_slurm(): - update_msg = "*** slurm configuration was updated ***" - if lookup().cfg.hybrid: - # terraform handles generating the config.yaml, don't do it here - return - - upd, cfg_new = util.fetch_config() - if not upd: - log.debug("No changes in config detected.") - return - log.debug("Changes in config detected. Reconfiguring Slurm now.") - util.update_config(cfg_new) - - if lookup().is_controller: - conf.gen_controller_configs(lookup()) - log.info("Restarting slurmctld to make changes take effect.") - try: - # TODO: consider removing "restart" since "reconfigure" should restart slurmctld as well - run("sudo systemctl restart slurmctld.service", check=False) - util.scontrol_reconfigure(lookup()) - except Exception: - log.exception("failed to reconfigure slurmctld") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - elif lookup().instance_role_safe == "compute": - log.info("Restarting slurmd to make changes take effect.") - run("systemctl restart slurmd") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - elif lookup().is_login_node: - log.info("Restarting sackd to make changes take effect.") - run("systemctl restart sackd") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - - -def update_topology(lkp: util.Lookup) -> None: - if conf.topology_plugin(lkp) != conf.TOPOLOGY_PLUGIN_TREE: - return - updated, summary = conf.gen_topology_conf(lkp) - if updated: - log.info("Topology configuration updated. Reconfiguring Slurm.") - util.scontrol_reconfigure(lkp) - # Safe summary only after Slurm got reconfigured, so summary reflects Slurm POV - summary.dump(lkp) - - -def delete_reservation(lkp: util.Lookup, reservation_name: str) -> None: - util.run(f"{lkp.scontrol} delete reservation {reservation_name}") - - -def create_reservation(lkp: util.Lookup, reservation_name: str, node: str, start_time: datetime) -> None: - # Format time to be compatible with slurm reservation. - formatted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S') - - util.run(f"{lkp.scontrol} create reservation user=slurm starttime={formatted_start_time} duration=180 nodes={node} reservationname={reservation_name} flags=maint,ignore_jobs") - - -def get_slurm_reservation_maintenance(lkp: util.Lookup) -> Dict[str, datetime]: - res = util.run(f"{lkp.scontrol} show reservation --json") - all_reservations = json.loads(res.stdout) - reservation_map = {} - - for reservation in all_reservations['reservations']: - name = reservation.get('name') - nodes = reservation.get('node_list') - time_epoch = reservation.get('start_time', {}).get('number') - - if name is None or nodes is None or time_epoch is None: - continue - - if reservation.get('node_count') != 1: - continue - - if name != f"{nodes}_maintenance": - continue - - reservation_map[name] = datetime.fromtimestamp(time_epoch) - - return reservation_map - -@lru_cache -def get_upcoming_maintenance(lkp: util.Lookup) -> Dict[str, Tuple[str, datetime]]: - upc_maint_map = {} - - for node, inst in lkp.instances().items(): - if inst.resource_status.upcoming_maintenance: - upc_maint_map[node + "_maintenance"] = (node, inst.resource_status.upcoming_maintenance.window_start_time) - - return upc_maint_map - - -def sync_maintenance_reservation(lkp: util.Lookup) -> None: - upc_maint_map = get_upcoming_maintenance(lkp) # map reservation_name -> (node_name, time) - log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") - - curr_reservation_map = get_slurm_reservation_maintenance(lkp) # map reservation_name -> time - log.debug(f"curr-reservation-map: {curr_reservation_map}") - - del_reservation = set(curr_reservation_map.keys() - upc_maint_map.keys()) - create_reservation_map = {} - - for res_name, (node, start_time) in upc_maint_map.items(): - try: - enabled = lkp.node_nodeset(node).enable_maintenance_reservation - except Exception: - enabled = False - - if not enabled: - if res_name in curr_reservation_map: - del_reservation.add(res_name) - continue - - if res_name in curr_reservation_map: - diff = curr_reservation_map[res_name] - start_time - if abs(diff) <= timedelta(seconds=1): - continue - else: - del_reservation.add(res_name) - create_reservation_map[res_name] = (node, start_time) - else: - create_reservation_map[res_name] = (node, start_time) - - log.debug(f"del-reservation: {del_reservation}") - for res_name in del_reservation: - delete_reservation(lkp, res_name) - - log.debug(f"create-reservation-map: {create_reservation_map}") - for res_name, (node, start_time) in create_reservation_map.items(): - create_reservation(lkp, res_name, node, start_time) - - -def delete_maintenance_job(job_name: str) -> None: - util.run(f"scancel --name={job_name}") - - -def create_maintenance_job(job_name: str, node: str) -> None: - util.run(f"sbatch --job-name={job_name} --nodelist={node} {_MAINTENANCE_SBATCH_SCRIPT_PATH}") - - -def get_slurm_maintenance_job(lkp: util.Lookup) -> Dict[str, str]: - jobs = {} - - for job in lkp.get_jobs(): - if job.name is None or job.required_nodes is None or job.job_state is None: - continue - - if job.name != f"{job.required_nodes}_maintenance": - continue - - if job.job_state != "PENDING": - continue - - jobs[job.name] = job.required_nodes - - return jobs - - -def sync_opportunistic_maintenance(lkp: util.Lookup) -> None: - upc_maint_map = get_upcoming_maintenance(lkp) # map job_name -> (node_name, time) - log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") - - curr_jobs = get_slurm_maintenance_job(lkp) # map job_name -> node. - log.debug(f"curr-maintenance-job-map: {curr_jobs}") - - del_jobs = set(curr_jobs.keys() - upc_maint_map.keys()) - create_jobs = {} - - for job_name, (node, _) in upc_maint_map.items(): - try: - enabled = lkp.node_nodeset(node).enable_opportunistic_maintenance - except Exception: - enabled = False - - if not enabled: - if job_name in curr_jobs: - del_jobs.add(job_name) - continue - - if job_name not in curr_jobs: - create_jobs[job_name] = node - - log.debug(f"del-maintenance-job: {del_jobs}") - for job_name in del_jobs: - delete_maintenance_job(job_name) - - log.debug(f"create-maintenance-job: {create_jobs}") - for job_name, node in create_jobs.items(): - create_maintenance_job(job_name, node) - - - -def sync_flex_migs(lkp: util.Lookup) -> None: - pass - - -def process_messages(lkp: util.Lookup) -> None: - try: - watch_delete_vm_op.watch_vm_delete_ops(lkp) - except: - log.exception("failed during watching delete VM operations") - - -def main(): - lkp = lookup() - if util.should_mount_slurm_bucket() and not lkp.is_controller: - return - try: - reconfigure_slurm() - except Exception: - log.exception("failed to reconfigure slurm") - if lkp.is_controller: - try: - process_messages(lkp) - except: - log.exception("failed to process messages") - - try: - sync_instances() - except Exception: - log.exception("failed to sync instances") - - try: - sync_flex_migs(lkp) - except Exception: - log.exception("failed to sync DWS Flex MIGs") - - try: - sync_placement_groups() - except Exception: - log.exception("failed to sync placement groups") - - try: - update_topology(lkp) - except Exception: - log.exception("failed to update topology") - - try: - sync_maintenance_reservation(lkp) - except Exception: - log.exception("failed to sync slurm reservation for scheduled maintenance") - - try: - sync_opportunistic_maintenance(lkp) - except Exception: - log.exception("failed to sync opportunistic reservation for scheduled maintenance") - - - try: - # TODO: it performs 1 to 4 GCS list requests, - # use cached version, combine with `_list_config_blobs` - install_custom_scripts(check_hash=True) - except Exception: - log.exception("failed to sync custom scripts") - - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - _ = util.init_log_and_parse(parser) - - pid_file = (Path("/tmp") / Path(__file__).name).with_suffix(".pid") - with pid_file.open("w") as fp: - try: - fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) - main() - except BlockingIOError: - sys.exit(0) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py deleted file mode 100644 index ae36c54222..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This script sorts nodes based on their `physicalHost`. - -See https://cloud.google.com/compute/docs/instances/use-compact-placement-policies - -You can reduce latency in tightly coupled HPC workloads (including distributed ML training) -by deploying them to machines that are located close together. -For example, if you deploy your workload on a single physical rack, you can expect lower latency -than if your workload is spread across multiple racks. -Sending data across multiple rack requires sending data through additional network switches. - -Example usage: -``` my_sbatch.sh -#SBATCH --ntasks-per-node=8 -#SBATCH --nodes=64 - -export SLURM_HOSTFILE=$(sort_nodes.py) - -srun -l hostname | sort -``` -""" -import os -import subprocess -import uuid -from typing import List, Optional, Dict -from collections import OrderedDict - -def order(paths: List[List[str]]) -> List[str]: - """ - Orders the leaves of the tree in a way that minimizes the sum of distance in between - each pair of neighboring nodes in the resulting order. - The resulting order will always start from the first node in the input list. - The ordering is "stable" with respect to the input order of the leaves i.e. - given a choice between two nodes (identical in other ways) it will select "nodelist-smallest" one. - - Returns a list of nodenames, ordered as described above. - """ - if not paths: return [] - class Vert: - "Represents a vertex in a *network* tree." - def __init__(self, name: str, parent: Optional["Vert"]): - self.name = name - self.parent = parent - # Use `OrderedDict` to preserve insertion order - # TODO: once we move to Python 3.7+ use regular `dict` since it has the same guarantee - self.children: OrderedDict = OrderedDict() - - # build a tree, children are ordered by insertion order - root = Vert("", None) - for path in paths: - n = root - for v in path: - if v not in n.children: - n.children[v] = Vert(v, n) - n = n.children[v] - - # walk the tree in insertion order, gather leaves - result = [] - def gather_nodes(v: Vert) -> None: - if not v.children: # this is a Slurm node - result.append(v.name) - for u in v.children.values(): - gather_nodes(u) - gather_nodes(root) - return result - - -class Instance: - def __init__(self, name: str, zone: str, physical_host: Optional[str]): - self.name = name - self.zone = zone - self.physical_host = physical_host - - -def make_path(node_name: str, inst: Optional[Instance]) -> List[str]: - if not inst: # node with unknown instance (e.g. hybrid cluster) - return ["unknown", node_name] - zone = f"zone_{inst.zone}" - if not inst.physical_host: # node without physical host info (e.g. no placement policy) - return [zone, "unknown", node_name] - - assert inst.physical_host.startswith("/"), f"Unexpected physicalHost: {inst.physical_host}" - parts = inst.physical_host[1:].split("/") - if len(parts) >= 4: - return [*parts, node_name] - return [zone, *parts, node_name] - - -def to_hostnames(nodelist: str) -> List[str]: - cmd = ["scontrol", "show", "hostnames", nodelist] - out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout - return [n.decode("utf-8") for n in out.splitlines()] - - -def get_instances(node_names: List[str]) -> Dict[str, Optional[Instance]]: - fmt = ( - "--format=csv[no-heading,separator=','](zone,resourceStatus.physicalHost,name)" - ) - cmd = ["gcloud", "compute", "instances", "list", fmt] - - scp = os.path.commonprefix(node_names) - if scp: - cmd.append(f"--filter=name~'{scp}.*'") - out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout - d = {} - for line in out.splitlines(): - zone, physical_host, name = line.decode("utf-8").split(",") - d[name] = Instance(name, zone, physical_host) - return {n: d.get(n) for n in node_names} - - -def main(args) -> None: - nodelist = args.nodelist or os.getenv("SLURM_NODELIST") - if not nodelist: - raise ValueError("nodelist is not provided and SLURM_NODELIST is not set") - - if args.ntasks_per_node is None: - args.ntasks_per_node = int(os.getenv("SLURM_NTASKS_PER_NODE", "") or 1) - assert args.ntasks_per_node > 0 - - output = args.output or f"hosts.{uuid.uuid4()}" - - node_names = to_hostnames(nodelist) - instannces = get_instances(node_names) - paths = [make_path(n, instannces[n]) for n in node_names] - ordered = order(paths) - - with open(output, "w") as f: - for node in ordered: - for _ in range(args.ntasks_per_node): - f.write(node) - f.write("\n") - print(output) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument( - "--nodelist", - type=str, - help="Slurm 'hostlist expression' of nodes to sort, if not set the value of SLURM_NODELIST environment variable will be used", - ) - parser.add_argument( - "--ntasks-per-node", - type=int, - help="""Number of times to repeat each node in resulting sorted list. -If not set, the value of SLURM_NTASKS_PER_NODE environment variable will be used, -if neither is set, defaults to 1""", - ) - parser.add_argument( - "--output", type=str, help="Output file to write, defaults to 'hosts.'" - ) - args = parser.parse_args() - main(args) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py deleted file mode 100644 index ecef70f1cc..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Any -import argparse -import logging - -import util -from util import ( - log_api_request, - batch_execute, - to_hostlist, - separate, -) -from util import lookup -import tpu -import mig_flex -import watch_delete_vm_op - -log = logging.getLogger() - -TOT_REQ_CNT = 1000 - - -def truncate_iter(iterable, max_count): - end = "..." - _iter = iter(iterable) - for i, el in enumerate(_iter, start=1): - if i >= max_count: - yield end - break - yield el - - -def delete_instance_request(name: str) -> Any: - inst = lookup().instance(name) - assert inst - - request = lookup().compute.instances().delete( - project=lookup().project, - zone=inst.zone, - instance=name, - ) - log_api_request(request) - return request - - -def delete_instances(instances): - """delete instances individually""" - invalid, valid = separate(lambda inst: bool(lookup().instance(inst)), instances) - if len(invalid) > 0: - log.debug("instances do not exist: {}".format(",".join(invalid))) - if len(valid) == 0: - log.debug("No instances to delete") - return - - requests = {inst: delete_instance_request(inst) for inst in valid} - - log.info(f"to delete {len(valid)} instances ({to_hostlist(valid)})") - ops, failed = batch_execute(requests) - for node, (_, err) in failed.items(): - log.error(f"instance {node} failed to delete: {err}") - - log.info(f"deleting {len(ops)} instances {to_hostlist(ops.keys())}") - - topic = watch_delete_vm_op.watch_delete_vm_op_topic() - for node, op in ops.items(): - topic.publish(op, node) - - - - -def suspend_nodes(nodes: List[str]) -> None: - lkp = lookup() - other_nodes, tpu_nodes = util.separate(lkp.node_is_tpu, nodes) - bulk_nodes, flex_nodes = util.separate(lkp.is_flex_node, other_nodes) - - mig_flex.suspend_flex_nodes(flex_nodes, lkp) - delete_instances(bulk_nodes) - tpu.delete_tpu_instances(tpu_nodes) - - -def main(nodelist): - """main called when run as script""" - log.debug(f"SuspendProgram {nodelist}") - - # Filter out nodes not in config.yaml - other_nodes, pm_nodes = separate( - lookup().is_power_managed_node, util.to_hostnames(nodelist) - ) - if other_nodes: - log.debug( - f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" - ) - if pm_nodes: - log.debug(f"Suspending nodes '{to_hostlist(pm_nodes)}' from '{nodelist}'") - else: - log.debug("No cloud nodes to suspend") - return - - log.info(f"suspend {nodelist}") - suspend_nodes(pm_nodes) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument("nodelist", help="list of nodes to suspend") - args = util.init_log_and_parse(parser) - - main(args.nodelist) diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh deleted file mode 100644 index 9079e4e4b0..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PYTHON_SCRIPT="${SCRIPT_DIR}/suspend.py" - -# Capture all arguments passed by Slurm (the nodelist). -ALL_ARGS=("$@") - -"${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & -disown - -exit 0 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py deleted file mode 100644 index 0ce7fb5ec4..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Any -import sys -from dataclasses import dataclass, field -from datetime import datetime - -SCRIPTS_DIR = "community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts" -if SCRIPTS_DIR not in sys.path: - sys.path.append(SCRIPTS_DIR) # TODO: make this more robust - -import util - - -SOME_TS = datetime.fromisoformat("2018-09-03T20:56:35.450686+00:00") -# TODO: use "real" classes once they are defined (instead of NSDict) - -@dataclass -class Placeholder: - pass - -@dataclass -class TstNodeset: - nodeset_name: str = "cantor" - node_count_static: int = 0 - node_count_dynamic_max: int = 0 - node_conf: dict[str, Any] = field(default_factory=dict) - instance_template: Optional[str] = None - reservation_name: Optional[str] = "" - zone_policy_allow: Optional[list[str]] = field(default_factory=list) - enable_placement: bool = True - placement_max_distance: Optional[int] = None - accelerator_topology: Optional[str] = "" - future_reservation: Optional[str] = "" - -@dataclass -class TstPartition: - partition_name: str = "euler" - partition_nodeset: list[str] = field(default_factory=list) - partition_nodeset_tpu: list[str] = field(default_factory=list) - enable_job_exclusive: bool = False - -@dataclass -class TstCfg: - slurm_cluster_name: str = "m22" - cloud_parameters: dict[str, Any] = field(default_factory=dict) - - partitions: dict[str, TstPartition] = field(default_factory=dict) - nodeset: dict[str, TstNodeset] = field(default_factory=dict) - nodeset_tpu: dict[str, TstNodeset] = field(default_factory=dict) - nodeset_dyn: dict[str, TstNodeset] = field(default_factory=dict) - - install_dir: Optional[str] = None - output_dir: Optional[str] = None - - prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - task_prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - task_epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - - -@dataclass -class TstTPU: # to prevent client initialization durint "TPU.__init__" - vmcount: int - -@dataclass -class TstMachineConf: - cpus: int - memory: int - sockets: int - sockets_per_board: int - cores_per_socket: int - boards: int - threads_per_core: int - - -@dataclass -class TstTemplateInfo: - gpu: Optional[util.AcceleratorInfo] - -def tstInstance(name: str, physical_host: Optional[str] = None): - return util.Instance( - name=name, - zone="anorien", - status="RUNNING", - creation_timestamp=SOME_TS, - resource_status=util.InstanceResourceStatus( - physical_host=physical_host, - upcoming_maintenance=None, - ), - scheduling=util.NSDict(), - role="compute", - metadata={}, - ) - -def make_to_hostnames_mock(tbl: Optional[dict[str, list[str]]]): - tbl = tbl or {} - - def se(k: str) -> list[str]: - if k not in tbl: - raise AssertionError(f"to_hostnames mock: unexpected nodelist: '{k}'") - return tbl[k] - - return se diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py deleted file mode 100644 index 6bd6762748..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from mock import Mock -from common import TstNodeset, TstCfg, TstMachineConf, TstTemplateInfo, Placeholder - -import addict # type: ignore -import conf -import util - - -def test_nodeset_tpu_lines(): - nodeset = TstNodeset( - "turbo", - node_count_static=2, - node_count_dynamic_max=3, - node_conf={"red": "velvet"}, - ) - assert conf.nodeset_tpu_lines(nodeset, util.Lookup(TstCfg())) == "\n".join( - [ - "NodeName=m22-turbo-[0-4] State=CLOUD red=velvet", - "NodeSet=turbo Nodes=m22-turbo-[0-4]", - ] - ) - - -def test_nodeset_lines(): - nodeset = TstNodeset( - "turbo", - node_count_static=2, - node_count_dynamic_max=3, - node_conf={"red": "velvet", "CPUs": 55}, - ) - lkp = util.Lookup(TstCfg()) - lkp.template_info = Mock(return_value=TstTemplateInfo( - gpu=util.AcceleratorInfo(type="Popov", count=33) - )) - mc = TstMachineConf( - cpus=5, - memory=6, - sockets=7, - sockets_per_board=8, - boards=9, - threads_per_core=10, - cores_per_socket=11, - ) - lkp.template_machine_conf = Mock(return_value=mc) # type: ignore[method-assign] - assert conf.nodeset_lines(nodeset, lkp) == "\n".join( - [ - "NodeName=m22-turbo-[0-4] State=CLOUD RealMemory=6 Boards=9 SocketsPerBoard=8 CoresPerSocket=11 ThreadsPerCore=10 CPUs=55 Gres=gpu:33 red=velvet", - "NodeSet=turbo Nodes=m22-turbo-[0-4]", - ] - ) - - -@pytest.mark.parametrize( - "value,want", - [ - ({"a": 1}, "a=1"), - ({"a": "two"}, "a=two"), - ({"a": [3, 4]}, "a=3,4"), - ({"a": ["five", "six"]}, "a=five,six"), - ({"a": None}, ""), - ({"a": ["seven", None, 8]}, "a=seven,8"), - ({"a": 1, "b": "two"}, "a=1 b=two"), - ({"a": 1, "b": None, "c": "three"}, "a=1 c=three"), - ({"a": 0, "b": None, "c": 0.0, "e": ""}, "a=0 c=0.0"), - ({"a": [0, 0.0, None, "X", "", "Y"]}, "a=0,0.0,X,,Y"), - ]) -def test_dict_to_conf(value: dict, want: str): - assert conf.dict_to_conf(value) == want - - - -@pytest.mark.parametrize( - "cfg,want", - [ - (TstCfg( - install_dir="ukulele", - ), - """LaunchParameters=enable_nss_slurm,use_interactive_step -SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend -SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -SlurmdTimeout=300 -UnkillableStepTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - (TstCfg( - install_dir="ukulele", - cloud_parameters={ - "no_comma_params": True, - "private_data": None, - "scheduler_parameters": None, - "resume_rate": None, - "resume_timeout": None, - "suspend_rate": None, - "suspend_timeout": None, - "unkillable_step_timeout": None, - "slurmd_timeout": None, - "topology_plugin": None, - "topology_param": None, - "tree_width": None, - }, - ), - """SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -SlurmdTimeout=300 -UnkillableStepTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - (TstCfg( - install_dir="ukulele", - cloud_parameters={ - "no_comma_params": True, - "private_data": [ - "events", - "jobs", - ], - "scheduler_parameters": [ - "bf_busy_nodes", - "bf_continue", - "ignore_prefer_validation", - "nohold_on_prolog_fail", - ], - "resume_rate": 1, - "resume_timeout": 2, - "suspend_rate": 3, - "suspend_timeout": 4, - "slurmd_timeout": 5, - "unkillable_step_timeout": 6, - "tree_width": 7, - "topology_plugin": "guess", - "topology_param": "yellow", - }, - ), - """PrivateData=events,jobs -SchedulerParameters=bf_busy_nodes,bf_continue,ignore_prefer_validation,nohold_on_prolog_fail -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=1 -ResumeTimeout=2 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=3 -SuspendTimeout=4 -SlurmdTimeout=5 -UnkillableStepTimeout=6 -TreeWidth=7 -TopologyPlugin=guess -TopologyParam=yellow"""), - (TstCfg( - install_dir="ukulele", - task_prolog_scripts=[Placeholder()], - task_epilog_scripts=[Placeholder()], - ), - """LaunchParameters=enable_nss_slurm,use_interactive_step -SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend -TaskProlog=/slurm/custom_scripts/task_prolog.d/task-prolog -TaskEpilog=/slurm/custom_scripts/task_epilog.d/task-epilog -SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -SlurmdTimeout=300 -UnkillableStepTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - ]) -def test_conflines(cfg, want): - assert conf.conflines(util.Lookup(cfg)) == want - - cfg.cloud_parameters = addict.Dict(cfg.cloud_parameters) - assert conf.conflines(util.Lookup(cfg)) == want - - -@pytest.mark.parametrize( - "cfg,gputype,gpucount,want", - [ - (TstCfg(), - "", - 0, - "\n"), - (TstCfg( - nodeset={"turbo": TstNodeset("turbo")} - ), - "Popov", - 8, - "Name=gpu Type=Popov File=/dev/nvidia[0-7]\n\n"), - ]) -def test_gen_cloud_gres_conf_lines(cfg, gputype, gpucount, want): - lkp = util.Lookup(cfg) - lkp.template_info = Mock(return_value=TstTemplateInfo( - gpu=util.AcceleratorInfo(type=gputype, count=gpucount) - )) - assert conf.gen_cloud_gres_conf_lines(lkp) == want diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py deleted file mode 100644 index 77f1229605..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional - -import os -import pytest -import unittest.mock -import unittest -import tempfile - -from common import TstCfg, TstNodeset, TstPartition, TstTPU # needed to import util -import util -import resume -from resume import ResumeData, ResumeJobData, BulkChunk, PlacementAndNodes - -def test_get_resume_file_data_no_env(): - with unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": ""}): - assert resume.get_resume_file_data() is None - - -def test_get_resume_file_data(): - with tempfile.NamedTemporaryFile() as f: - f.write(b"""{ - "jobs": [ - { - "extra": null, - "job_id": 1, - "features": null, - "nodes_alloc": "green-[0-2]", - "nodes_resume": "green-[0-1]", - "oversubscribe": "OK", - "partition": "red", - "reservation": null - } - ], - "all_nodes_resume": "green-[0-1]" -}""") - f.flush() - with ( - unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": f.name}), - unittest.mock.patch("util.to_hostnames") as mock_to_hostnames, - ): - mock_to_hostnames.return_value = ["green-0", "green-1", "green-2"] - assert resume.get_resume_file_data() == ResumeData(jobs=[ - ResumeJobData( - job_id = 1, - partition="red", - nodes_alloc=["green-0", "green-1", "green-2"], - ) - ]) - mock_to_hostnames.assert_called_once_with("green-[0-2]") - - -@unittest.mock.patch("tpu.TPU.make") -@unittest.mock.patch("resume.create_placements") -def test_group_nodes_bulk(mock_create_placements, mock_tpu): - cfg = TstCfg( - nodeset={ - "n": TstNodeset(nodeset_name="n"), - }, - nodeset_tpu={ - "t": TstNodeset(nodeset_name="t"), - }, - partitions={ - "p1": TstPartition( - partition_name="p1", - enable_job_exclusive=True, - ), - "p2": TstPartition( - partition_name="p2", - partition_nodeset_tpu=["t"], - enable_job_exclusive=True, - ) - } - ) - lkp = util.Lookup(cfg) - - def mock_create_placements_se(nodes, excl_job_id, lkp): - args = (set(nodes), excl_job_id) - if ({'c-n-1', 'c-n-2', 'c-t-8', 'c-t-9'}, None) == args: - return [ - PlacementAndNodes("g0", ["c-n-1", "c-n-2"]), - PlacementAndNodes(None, ['c-t-8', 'c-t-9']), - ] - if ({"c-n-0", "c-n-8"}, 1) == args: - return [ - PlacementAndNodes("g10", ["c-n-0"]), - PlacementAndNodes("g11", ["c-n-8"]), - ] - if ({'c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5'}, 2) == args: - return [ - PlacementAndNodes(None, ['c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5']) - ] - raise AssertionError(f"unexpected invocation: '{args}'") - mock_create_placements.side_effect = mock_create_placements_se - - def mock_tpu_se(ns: str, lkp) -> TstTPU: - if ns == "t": - return TstTPU(vmcount=2) - raise AssertionError(f"unexpected invocation: '{ns}'") - mock_tpu.side_effect = mock_tpu_se - - got = resume.group_nodes_bulk( - ["c-n-0", "c-n-1", "c-n-2", "c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-8", "c-t-9"], - ResumeData(jobs=[ - ResumeJobData(job_id=1, partition="p1", nodes_alloc=["c-n-0", "c-n-8"]), - ResumeJobData(job_id=2, partition="p2", nodes_alloc=["c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-4", "c-t-5"]), - ]), lkp) - mock_create_placements.assert_called() - assert got == { - "c-n:jobNone:g0:0": BulkChunk( - nodes=["c-n-1", "c-n-2"], prefix="c-n", chunk_idx=0, excl_job_id=None, placement_group="g0"), - "c-n:job1:g10:0": BulkChunk( - nodes=["c-n-0"], prefix="c-n", chunk_idx=0, excl_job_id=1, placement_group="g10"), - "c-t:0": BulkChunk( - nodes=["c-t-8", "c-t-9"], prefix="c-t", chunk_idx=0, excl_job_id=None, placement_group=None), - "c-t:job2:0": BulkChunk( - nodes=["c-t-0", "c-t-1"], prefix="c-t", chunk_idx=0, excl_job_id=2, placement_group=None), - "c-t:job2:1": BulkChunk( - nodes=["c-t-2", "c-t-3"], prefix="c-t", chunk_idx=1, excl_job_id=2, placement_group=None), - } - - -@pytest.mark.parametrize( - "nodes,excl_job_id,expected", - [ - ( # TPU - no placements - ["c-t-0", "c-t-2"], 4, [PlacementAndNodes(None, ["c-t-0", "c-t-2"])] - ), - ( # disabled placements - no placemens - ["c-x-0", "c-x-2"], 4, [PlacementAndNodes(None, ["c-x-0", "c-x-2"])] - ), - ( # excl_job - ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], 4, [ - PlacementAndNodes("c-slurmgcp-managed-n-4-0", ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"]) - ] - ), - ( # no excl_job - ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], None, [ - PlacementAndNodes("c-slurmgcp-managed-n-0-0", ["c-n-0", "c-n-2"]), - PlacementAndNodes('c-slurmgcp-managed-n-0-1', ['c-n-2011']), - PlacementAndNodes(None, ["c-n-uno"]), - ] - ), - ], -) -def test_allocate_nodes_to_placements(nodes: list[str], excl_job_id: Optional[int], expected: list[PlacementAndNodes]): - cfg = TstCfg( - slurm_cluster_name="c", - nodeset={ - "n": TstNodeset(nodeset_name="n", enable_placement=True), - "x": TstNodeset(nodeset_name="x", enable_placement=False) - }, - nodeset_tpu={ - "t": TstNodeset(nodeset_name="t") - }) - lkp = util.Lookup(cfg) - - with unittest.mock.patch("resume.valid_placement_node") as mock_valid_placement_node: - mock_valid_placement_node.return_value = True - lkp.template_info = unittest.mock.Mock(return_value=unittest.mock.Mock(machine_type=unittest.mock.Mock(family="n1"))) - - assert resume._allocate_nodes_to_placements(nodes, excl_job_id, lkp) == expected diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py deleted file mode 100644 index df9f3a0137..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -import json -import mock -from pytest_unordered import unordered -from common import TstCfg, TstNodeset, TstTPU, tstInstance -import sort_nodes - -import util -import conf -import tempfile - -PRELUDE = """ -# Warning: -# This file is managed by a script. Manual modifications will be overwritten. - -""" - -def test_gen_topology_conf_empty(): - out_dir = tempfile.mkdtemp() - cfg = TstCfg(output_dir=out_dir) - conf.gen_topology_conf(util.Lookup(cfg)) - assert open(out_dir + "/cloud_topology.conf").read() == PRELUDE + "\n" - - -@mock.patch("tpu.TPU.make") -def test_gen_topology_conf(tpu_mock): - output_dir = tempfile.mkdtemp() - cfg = TstCfg( - nodeset_tpu={ - "a": TstNodeset("bold", node_count_static=4, node_count_dynamic_max=5), - "b": TstNodeset("slim", node_count_dynamic_max=3), - }, - nodeset={ - "c": TstNodeset("green", node_count_static=2, node_count_dynamic_max=3), - "d": TstNodeset("blue", node_count_static=7), - "e": TstNodeset("pink", node_count_dynamic_max=4), - }, - output_dir=output_dir, - ) - - def tpu_se(ns: str, lkp) -> TstTPU: - if ns == "bold": - return TstTPU(vmcount=3) - if ns == "slim": - return TstTPU(vmcount=1) - raise AssertionError(f"unexpected TPU name: '{ns}'") - - tpu_mock.side_effect = tpu_se - - lkp = util.Lookup(cfg) - lkp.instances = lambda: { n.name: n for n in [ # type: ignore[assignment] - # nodeset blue - tstInstance("m22-blue-0"), # no physicalHost - tstInstance("m22-blue-0", physical_host="/a/a/a"), - tstInstance("m22-blue-1", physical_host="/a/a/b"), - tstInstance("m22-blue-2", physical_host="/a/b/a"), - tstInstance("m22-blue-3", physical_host="/b/a/a"), - # nodeset green - tstInstance("m22-green-3", physical_host="/a/a/c"), - ]} - - uncompressed = conf.gen_topology(lkp) - want_uncompressed = [ - #NOTE: the switch names are not unique, it's not valid content for topology.conf - # The uniquefication and compression of names are done in the compress() method - "SwitchName=slurm-root Switches=a,b,ns_blue,ns_green,ns_pink", - # "physical" topology - 'SwitchName=a Switches=a,b', - 'SwitchName=a Nodes=m22-blue-[0-1],m22-green-3', - 'SwitchName=b Nodes=m22-blue-2', - 'SwitchName=b Switches=a', - 'SwitchName=a Nodes=m22-blue-3', - # topology "by nodeset" - "SwitchName=ns_blue Nodes=m22-blue-[4-6]", - "SwitchName=ns_green Nodes=m22-green-[0-2,4]", - "SwitchName=ns_pink Nodes=m22-pink-[0-3]", - # TPU topology - "SwitchName=tpu-root Switches=ns_bold,ns_slim", - "SwitchName=ns_bold Switches=bold-[0-3]", - "SwitchName=bold-0 Nodes=m22-bold-[0-2]", - "SwitchName=bold-1 Nodes=m22-bold-3", - "SwitchName=bold-2 Nodes=m22-bold-[4-6]", - "SwitchName=bold-3 Nodes=m22-bold-[7-8]", - "SwitchName=ns_slim Nodes=m22-slim-[0-2]"] - assert list(uncompressed.render_conf_lines()) == want_uncompressed - - compressed = uncompressed.compress() - want_compressed = [ - "SwitchName=s0 Switches=s0_[0-4]", # root - # "physical" topology - 'SwitchName=s0_0 Switches=s0_0_[0-1]', # /a - 'SwitchName=s0_0_0 Nodes=m22-blue-[0-1],m22-green-3', # /a/a - 'SwitchName=s0_0_1 Nodes=m22-blue-2', # /a/b - 'SwitchName=s0_1 Switches=s0_1_0', # /b - 'SwitchName=s0_1_0 Nodes=m22-blue-3', # /b/a - # topology "by nodeset" - "SwitchName=s0_2 Nodes=m22-blue-[4-6]", - "SwitchName=s0_3 Nodes=m22-green-[0-2,4]", - "SwitchName=s0_4 Nodes=m22-pink-[0-3]", - # TPU topology - "SwitchName=s1 Switches=s1_[0-1]", - "SwitchName=s1_0 Switches=s1_0_[0-3]", - "SwitchName=s1_0_0 Nodes=m22-bold-[0-2]", - "SwitchName=s1_0_1 Nodes=m22-bold-3", - "SwitchName=s1_0_2 Nodes=m22-bold-[4-6]", - "SwitchName=s1_0_3 Nodes=m22-bold-[7-8]", - "SwitchName=s1_1 Nodes=m22-slim-[0-2]"] - assert list(compressed.render_conf_lines()) == want_compressed - - upd, summary = conf.gen_topology_conf(lkp) - assert upd == True - want_written = PRELUDE + "\n".join(want_compressed) + "\n\n" - assert open(output_dir + "/cloud_topology.conf").read() == want_written - - summary.dump(lkp) - summary_got = json.loads(open(output_dir + "/cloud_topology.summary.json").read()) - - assert summary_got == { - "down_nodes": unordered( - [f"m22-blue-{i}" for i in (4,5,6)] + - [f"m22-green-{i}" for i in (0,1,2,4)] + - [f"m22-pink-{i}" for i in range(4)]), - "tpu_nodes": unordered( - [f"m22-bold-{i}" for i in range(9)] + - [f"m22-slim-{i}" for i in range(3)]), - 'physical_host': { - 'm22-blue-0': '/a/a/a', - 'm22-blue-1': '/a/a/b', - 'm22-blue-2': '/a/b/a', - 'm22-blue-3': '/b/a/a', - 'm22-green-3': '/a/a/c'}, - } - - - -def test_gen_topology_conf_update(): - cfg = TstCfg( - nodeset={ - "c": TstNodeset("green", node_count_static=2), - }, - output_dir=tempfile.mkdtemp(), - ) - lkp = util.Lookup(cfg) - lkp.instances = lambda: { # type: ignore[assignment] - # no instances - } - - # initial generation - reconfigure - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # add node: node_count_static 2 -> 3 - reconfigure - lkp.cfg.nodeset["c"].node_count_static = 3 - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # remove node: node_count_static 3 -> 2 - no reconfigure - lkp.cfg.nodeset["c"].node_count_static = 2 - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - # set empty physicalHost - no reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - # set physicalHost - reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/c")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # change physicalHost - reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/z")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # shut down node - no reconfigure - lkp.instances = lambda: {} # type: ignore[assignment] - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - -@pytest.mark.parametrize( - "paths,expected", - [ - (["z/n-0", "z/n-1", "z/n-2", "z/n-3", "z/n-4", "z/n-10"], ['n-0', 'n-1', 'n-2', 'n-3', 'n-4', 'n-10']), - (["y/n-0", "z/n-1", "x/n-2", "x/n-3", "y/n-4", "g/n-10"], ['n-0', 'n-4', 'n-1', 'n-2', 'n-3', 'n-10']), - ]) -def test_sort_nodes_order(paths: list[str], expected: list[str]) -> None: - paths_expanded = [l.split("/") for l in paths] - assert sort_nodes.order(paths_expanded) == expected diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py deleted file mode 100644 index 69617d0301..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py +++ /dev/null @@ -1,668 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Type - -import pytest -from mock import Mock -from datetime import datetime, timezone, timedelta -import unittest - -from common import TstNodeset, TstCfg # needed to import util -import util -from util import NodeState, MachineType, AcceleratorInfo, UpcomingMaintenance, InstanceResourceStatus, FutureReservation, ReservationDetails -from google.api_core.client_options import ClientOptions # noqa: E402 -from addict import Dict as NSDict # type: ignore - -# Note: need to install pytest-mock - -@pytest.mark.parametrize( - "name,expected", - [ - ( - "az-buka-23", - { - "cluster": "az", - "nodeset": "buka", - "node": "23", - "prefix": "az-buka", - "range": None, - "suffix": "23", - }, - ), - ( - "az-buka-xyzf", - { - "cluster": "az", - "nodeset": "buka", - "node": "xyzf", - "prefix": "az-buka", - "range": None, - "suffix": "xyzf", - }, - ), - ( - "az-buka-[2-3]", - { - "cluster": "az", - "nodeset": "buka", - "node": "[2-3]", - "prefix": "az-buka", - "range": "[2-3]", - "suffix": None, - }, - ), - ], -) -def test_node_desc(name, expected): - assert util.lookup()._node_desc(name) == expected - - -@pytest.mark.parametrize( - "name,expected", - [ - ("az-buka-23", 23), - ("az-buka-0", 0), - ("az-buka", Exception), - ("az-buka-xyzf", ValueError), - ("az-buka-[2-3]", ValueError), - ], -) -def test_node_index(name, expected): - if type(expected) is type and issubclass(expected, Exception): - with pytest.raises(expected): - util.lookup().node_index(name) - else: - assert util.lookup().node_index(name) == expected - - -@pytest.mark.parametrize( - "name", - [ - "az-buka", - ], -) -def test_node_desc_fail(name): - with pytest.raises(Exception): - util.lookup()._node_desc(name) - - -@pytest.mark.parametrize( - "names,expected", - [ - ("pedro,pedro-1,pedro-2,pedro-01,pedro-02", "pedro,pedro-[1-2,01-02]"), - ("pedro,,pedro-1,,pedro-2", "pedro,pedro-[1-2]"), - ("pedro-8,pedro-9,pedro-10,pedro-11", "pedro-[8-9,10-11]"), - ("pedro-08,pedro-09,pedro-10,pedro-11", "pedro-[08-11]"), - ("pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-09]"), - ("pedro-10,pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-10]"), - ("pedro-8,pedro-9,juan-10,juan-11", "juan-[10-11],pedro-[8-9]"), - ("az,buki,vedi", "az,buki,vedi"), - ("a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12", "a[0-9,10-12]"), - ("a0,a2,a4,a6,a7,a8,a11,a12", "a[0,2,4,6-8,11-12]"), - ("seas7-0,seas7-1", "seas7-[0-1]"), - ], -) -def test_to_hostlist(names, expected): - assert util.to_hostlist(names.split(",")) == expected - - -@pytest.mark.parametrize( - "api,ep_ver,expected", - [ - ( - util.ApiEndpoint.BQ, - "v1", - ClientOptions(api_endpoint="https://bq.googleapis.com/v1/"), - ), - ( - util.ApiEndpoint.COMPUTE, - "staging_v1", - ClientOptions(api_endpoint="https://compute.googleapis.com/staging_v1/"), - ), - ( - util.ApiEndpoint.SECRET, - "v1", - ClientOptions(api_endpoint="https://secret_manager.googleapis.com/v1/"), - ), - ( - util.ApiEndpoint.STORAGE, - "beta", - ClientOptions(api_endpoint="https://storage.googleapis.com/beta/"), - ), - ( - util.ApiEndpoint.TPU, - "alpha", - ClientOptions(api_endpoint="https://tpu.googleapis.com/alpha/"), - ), - ], -) -def test_create_client_options( - api: util.ApiEndpoint, ep_ver: str, expected: ClientOptions, mocker -): - ud_mock = mocker.patch("util.universe_domain") - ep_mock = mocker.patch("util.endpoint_version") - ud_mock.return_value = "googleapis.com" - ep_mock.return_value = ep_ver - assert util.create_client_options(api).__repr__() == expected.__repr__() - - - -@pytest.mark.parametrize( - "nodeset,err", - [ - (TstNodeset(reservation_name="projects/x/reservations/y"), AssertionError), # no zones - (TstNodeset( - reservation_name="projects/x/reservations/y", - zone_policy_allow=["eine", "zwei"]), AssertionError), # multiples zones - (TstNodeset( - reservation_name="robin", - zone_policy_allow=["eine"]), ValueError), # invalid name - (TstNodeset( - reservation_name="projects/reservations/y", - zone_policy_allow=["eine"]), ValueError), # invalid name - (TstNodeset( - reservation_name="projects/x/zones/z/reservations/y", - zone_policy_allow=["eine"]), ValueError), # invalid name - ] -) -def test_nodeset_reservation_err(nodeset, err): - lkp = util.Lookup(TstCfg()) - lkp._get_reservation = Mock() - with pytest.raises(err): - lkp.nodeset_reservation(nodeset) - lkp._get_reservation.assert_not_called() # type: ignore - -@pytest.mark.parametrize( - "nodeset,policies,expected", - [ - (TstNodeset(), [], None), # no reservation - (TstNodeset( - reservation_name="projects/bobin/reservations/robin", - zone_policy_allow=["eine"]), - [], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=[], - deployment_type=None, - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/bobin/reservations/robin")), - (TstNodeset( - reservation_name="projects/bobin/reservations/robin", - zone_policy_allow=["eine"]), - ["seven/wanders", "five/red/apples", "yum"], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=["wanders", "apples", "yum"], - deployment_type=None, - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/bobin/reservations/robin")), - (TstNodeset( - reservation_name="projects/bobin/reservations/robin/snek/cheese-brie-6", - zone_policy_allow=["eine"]), - [], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=[], - deployment_type=None, - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/bobin/reservations/robin/snek/cheese-brie-6")), - - ]) - -def test_nodeset_reservation_ok(nodeset, policies, expected): - lkp = util.Lookup(TstCfg()) - lkp._get_reservation = Mock() - - if not expected: - assert lkp.nodeset_reservation(nodeset) is None - lkp._get_reservation.assert_not_called() # type: ignore - return - - lkp._get_reservation.return_value = { # type: ignore - "resourcePolicies": {i: p for i, p in enumerate(policies)}, - } - assert lkp.nodeset_reservation(nodeset) == expected - lkp._get_reservation.assert_called_once_with(expected.project, expected.zone, expected.name) # type: ignore - -@pytest.mark.parametrize( - "job_info,expected_job", - [ - ( - """JobId=123 - TimeLimit=02:00:00 - JobName=myjob - JobState=PENDING - ReqNodeList=node-[1-10]""", - util.Job( - id=123, - duration=timedelta(days=0, hours=2, minutes=0, seconds=0), - name="myjob", - job_state="PENDING", - required_nodes="node-[1-10]" - ), - ), - ( - """JobId=456 - JobName=anotherjob - JobState=PENDING - ReqNodeList=node-group1""", - util.Job( - id=456, - duration=None, - name="anotherjob", - job_state="PENDING", - required_nodes="node-group1" - ), - ), - ( - """JobId=789 - TimeLimit=00:30:00 - JobState=COMPLETED""", - util.Job( - id=789, - duration=timedelta(minutes=30), - name=None, - job_state="COMPLETED", - required_nodes=None - ), - ), - ( - """JobId=101112 - TimeLimit=1-00:30:00 - JobState=COMPLETED, - ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", - util.Job( - id=101112, - duration=timedelta(days=1, hours=0, minutes=30, seconds=0), - name=None, - job_state="COMPLETED", - required_nodes="node-[1-10],grob-pop-[2,1,44-77]" - ), - ), - ( - """JobId=131415 - TimeLimit=1-00:30:00 - JobName=mynode-1_maintenance - JobState=COMPLETED, - ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", - util.Job( - id=131415, - duration=timedelta(days=1, hours=0, minutes=30, seconds=0), - name="mynode-1_maintenance", - job_state="COMPLETED", - required_nodes="node-[1-10],grob-pop-[2,1,44-77]" - ), - ), - ], -) -def test_parse_job_info(job_info, expected_job): - lkp = util.Lookup(TstCfg()) - assert lkp._parse_job_info(job_info) == expected_job - - - -@pytest.mark.parametrize( - "node,state,want", - [ - ("c-n-2", NodeState("DOWN", frozenset([])), NodeState("DOWN", frozenset([]))), # happy scenario - ("c-d-vodoo", None, None), # dynamic nodeset - ("c-x-44", None, None), # unknown(removed) nodeset - ("c-n-7", None, None), # Out of bounds: c-n-[0-4] - downsized nodeset - ("c-t-7", None, None), # Out of bounds: c-t-[0-4] - downsized nodeset TPU - ("c-n-2", None, RuntimeError), # something is wrong - ("c-t-2", None, RuntimeError), # something is wrong, but TPU - - # Check boundaries match [0-5) - ("c-n-5", None, None), # out of boundaries - ("c-n-4", None, RuntimeError), # within boundaries - ]) -def test_node_state(node: str, state: Optional[NodeState], want: NodeState | None | Type[Exception]): - cfg = TstCfg( - slurm_cluster_name="c", - nodeset={ - "n": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, - nodeset_tpu={ - "t": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, - nodeset_dyn={ - "d": TstNodeset()}, - ) - lkp = util.Lookup(cfg) - lkp.slurm_nodes = lambda: {node: state} if state else {} # type: ignore[assignment] - # ... see https://github.com/python/typeshed/issues/6347 - - if type(want) is type and issubclass(want, Exception): - with pytest.raises(want): - lkp.node_state(node) - else: - assert lkp.node_state(node) == want - - - -@pytest.mark.parametrize( - "jo,want", - [ - ({ - "accelerators": [ { "guestAcceleratorCount": 1, "guestAcceleratorType": "nvidia-tesla-a100" } ], - "creationTimestamp": "1969-12-31T16:00:00.000-08:00", - "description": "Accelerator Optimized: 1 NVIDIA Tesla A100 GPU, 12 vCPUs, 85GB RAM", - "guestCpus": 12, - "id": "1000012", - "imageSpaceGb": 0, - "isSharedCpu": False, - "kind": "compute#machineType", - "maximumPersistentDisks": 128, - "maximumPersistentDisksSizeGb": "263168", - "memoryMb": 87040, - "name": "a2-highgpu-1g", - "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/us-central1-a/machineTypes/a2-highgpu-1g", - "zone": "us-central1-a" - }, MachineType( - name="a2-highgpu-1g", - guest_cpus=12, - memory_mb=87040, - accelerators=[ - AcceleratorInfo(type="nvidia-tesla-a100", count=1) - ] - )), - ({ - "architecture": "X86_64", - "creationTimestamp": "1969-12-31T16:00:00.000-08:00", - "description": "8 vCPUs, 32 GB RAM", - "guestCpus": 8, - "id": "1210008", - "imageSpaceGb": 0, - "isSharedCpu": False, - "kind": "compute#machineType", - "maximumPersistentDisks": 128, - "maximumPersistentDisksSizeGb": "263168", - "memoryMb": 32768, - "name": "t2d-standard-8", - "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/europe-north2-b/machineTypes/t2d-standard-8", - "zone": "europe-north2-b" - }, MachineType( - name="t2d-standard-8", - guest_cpus=8, - memory_mb=32768, - accelerators=[] - )), - ]) -def test_MachineType_from_json(jo: dict, want: MachineType): - assert MachineType.from_json(jo) == want - - -@pytest.mark.parametrize( - "template,expected", - [ - ( - NSDict({ - "machine_type": MachineType( - name="e2", - guest_cpus=12, - memory_mb=87040, - accelerators=[]), - }), - None - ), - ( - NSDict({ - "machine_type": MachineType( - name="tpu-machine", - guest_cpus=12, - memory_mb=87040, - accelerators=[ - AcceleratorInfo(type="tpu-v6", count=1) - ]), - }), - None - ), - ( - NSDict({ - "machine_type": MachineType( - name="a2-highgpu-1g", - guest_cpus=12, - memory_mb=87040, - accelerators=[AcceleratorInfo(type="nvidia-tesla-a100", count=1)] - ), - }), - AcceleratorInfo(type="nvidia-tesla-a100", count=1) - ), - ( - NSDict({ - "machine_type": MachineType( - name="a2-highgpu-1g", - guest_cpus=12, - memory_mb=87040, - accelerators=[]), - "guestAccelerators":[ { "acceleratorCount": 1, "acceleratorType": "nvidia-tesla-a100" } ], - }), - AcceleratorInfo(type="nvidia-tesla-a100", count=1) - ), - ], -) -def test_get_template_gpu(template, expected): - assert util.get_template_gpu(template) == expected - - -UTC, PST = timezone.utc, timezone(timedelta(hours=-8)) - -@pytest.mark.parametrize( - "got,want", - [ - # from instance.creationTimestamp: - ("2024-11-30T12:47:51.676-08:00", datetime(2024, 11, 30, 12, 47, 51, 676000, tzinfo=PST)), - # from futureReservation.creationTimestamp - ("2024-11-05T15:23:33.702-08:00", datetime(2024, 11, 5, 15, 23, 33, 702000, tzinfo=PST)), - # from futureReservation.timeWindow.endTime - ("2025-01-15T00:00:00Z", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), - # fallback to UTC if no tz is specified - ("2025-01-15T00:00:00", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), - ]) -def test_parse_gcp_timestamp(got: str, want: datetime): - assert util.parse_gcp_timestamp(got) == want - - -@pytest.mark.parametrize( - "got,want", - [ - (None, None), - (dict( - windowStartTime="2025-01-15T00:00:00Z", - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - (dict( - startTimeWindow=dict( - earliest="2025-01-15T00:00:00Z"), - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - (dict( - windowStartTime="2025-01-15T00:00:00Z", - startTimeWindow=dict( - earliest="2025-01-25T00:00:00Z"), # ignored - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - ]) -def tests_parse_UpcomingMaintenance_OK(got: dict, want: Optional[UpcomingMaintenance]): - assert UpcomingMaintenance.from_json(got) == want - - -@pytest.mark.parametrize( - "got", - [ - {}, - dict( - windowStartTime=dict( - earliest="2025-01-15T00:00:00Z")), - ]) -def tests_parse_UpcomingMaintenance_FAIL(got: dict): - with pytest.raises(ValueError): - UpcomingMaintenance.from_json(got) - - -@pytest.mark.parametrize( - "got,want", - [ - (None, InstanceResourceStatus( - physical_host=None, - upcoming_maintenance=None)), - ({}, InstanceResourceStatus( - physical_host=None, - upcoming_maintenance=None)), - (dict( - physicalHost="/aaa/bbb/ccc"), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=None)), - (dict( # invalid upcomingMaintenance field to be ignored - physicalHost="/aaa/bbb/ccc", - upcomingMaintenance="maintenance is upon us"), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=None)), - (dict( - physicalHost="/aaa/bbb/ccc", - upcomingMaintenance=dict(windowStartTime="2025-01-15T00:00:00Z")), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=UpcomingMaintenance( - window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC)))), - ]) -def test_parse_InstanceResourceStatus(got: dict, want: Optional[InstanceResourceStatus]): - assert InstanceResourceStatus.from_json(got) == want - - -@pytest.mark.parametrize( - "link,component_name,expected", - [ - ( - "mylink/regions/us-cental1/other", - "regions", - "us-cental1" - ), - ( - "mylink/global/other", - "regions", - None - ), - ], -) -def test_get_self_link_component(link, component_name, expected): - assert util.get_self_link_component(link, component_name) == expected - - -def test_future_reservation_none(): - lkp = util.Lookup(TstCfg()) - assert lkp.future_reservation(TstNodeset()) == None - - -def test_future_reservation_declined(): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-03T23:30:00Z" }, - status = {"procurementStatus": "DECLINED"}, - reservationMode = "CALENDAR", - specificReservationRequired = True, - )) - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 3, 23, 30, tzinfo=timezone.utc), - reservation_mode="CALENDAR", - active_reservation=None) - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - -@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 13, 0, 0, tzinfo=timezone.utc)) -def test_future_reservation_active(_): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, - status = { - "procurementStatus": "FULFILLED", - "autoCreatedReservations": [ - "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" - ], - }, - specificReservationRequired = True, - )) - lkp._get_reservation = Mock(return_value=dict()) - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), - reservation_mode=None, - active_reservation=ReservationDetails( - project='manhattan', - zone='danger', - name='melon', - policies=[], - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/manhattan/reservations/melon", - deployment_type=None)) - - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - lkp._get_reservation.assert_called_once_with("manhattan", "danger", "melon") - -@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 28, 0, 0, tzinfo=timezone.utc)) -def test_future_reservation_inactive(_): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, - status = { - "procurementStatus": "FULFILLED", - "autoCreatedReservations": [ - "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" - ], - }, - reservationMode = "DEFAULT", - specificReservationRequired = True, - )) - lkp._get_reservation = Mock() - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), - reservation_mode="DEFAULT", - active_reservation=None) - - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - lkp._get_reservation.assert_not_called() diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test deleted file mode 100644 index a583642015..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -unset CUDA_VISIBLE_DEVICES - -LOG_FILE="/var/log/slurm/chs_health_check.log" -TMP_DCGM_OUT="/tmp/dcgm.out" -TMP_ECC_ERRORS_OUT="/tmp/ecc_errors.out" - -log_step() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" -} - -# Fail gracefully if nvidia-smi or dcgmi doesn't exist -if ! type -P nvidia-smi 1>/dev/null; then - log_step "nvidia-smi not found - this script requires nvidia-smi to function" - exit 0 -fi - -if ! type -P dcgmi 1>/dev/null; then - log_step "dcgmi not found - this script requires dcgmi to function" - exit 0 -fi - -if ! type -P nv-hostengine 1>/dev/null; then - log_step "nv-hostengine not found - this script requires nv-hostengine to function" - exit 0 -fi - -################################################### -# Disable running health checks -################################################### -# Check if the environment variable '$SLURM_JOB_EXTRA' is set and contains the -# substring 'healthchecks_prolog=off' -if [[ -n "$SLURM_JOB_EXTRA" ]]; then - log_step "Environment variable SLURM_JOB_EXTRA is set. Checking if it contains healthchecks_prolog=off." - # Check if the value of the variable matches the string "healthchecks_prolog=off" - if [[ "$SLURM_JOB_EXTRA" == *"healthchecks_prolog=off"* ]]; then - log_step "Environment variable SLURM_JOB_EXTRA matches substring healthchecks_prolog=off. Skipping health checks." - exit 0 - else - log_step "Environment variable SLURM_JOB_EXTRA does NOT match substring healthchecks_prolog=off. Attempting to run health checks." - fi -else - log_step "Environment variable SLURM_JOB_EXTRA is NOT set. Attempting to run health checks." -fi - -# Exit if GPU isn't H/B 100/200 -GPU_MODEL=$(nvidia-smi --query-gpu=name --format=csv,noheader) -if ! [[ "$GPU_MODEL" =~ [BH][1-2]00 ]]; then - log_step "No Supported GPU detected" - exit 0 -fi - -NUMGPUS=$(nvidia-smi -L | wc -l) - -# Check that all GPUs are healthy via DCGM and check for ECC errors -if [ $NUMGPUS -gt 0 ]; then - log_step "Execute DCGM health check, ECC error check, and NVLink error check for GPUs" - GPULIST=$(nvidia-smi --query-gpu=index --format=csv,noheader | tr '\n' ',' | sed 's/,$//') - rm -f $TMP_DCGM_OUT - rm -f $TMP_ECC_ERRORS_OUT - - # Run DCGM checks - START_HOSTENGINE=false - if ! pidof nv-hostengine > /dev/null; then - log_step "Starting nv-hostengine..." - nv-hostengine >> "$LOG_FILE" 2>&1 - sleep 1 # Give it a moment to start up - START_HOSTENGINE=true - fi - GROUPID=$(dcgmi group -c gpuinfo | awk '{print $NF}' | tr -d ' ') - dcgmi group -g $GROUPID -a $GPULIST >> "$LOG_FILE" 2>&1 - dcgmi diag -g $GROUPID -r 1 > "$TMP_DCGM_OUT" 2>&1 - cat "$TMP_DCGM_OUT" >> "$LOG_FILE" - dcgmi group -d $GROUPID >> "$LOG_FILE" 2>&1 - - # Terminate the host engine if it was manually started - if [ "$START_HOSTENGINE" = true ]; then - log_step "Terminating nv-hostengine..." - nv-hostengine -t >> "$LOG_FILE" 2>&1 - fi - - # Check for DCGM failures - DCGM_FAILED=0 - if grep -i fail "$TMP_DCGM_OUT" > /dev/null; then - DCGM_FAILED=1 - fi - - # Check for ECC errors - nvidia-smi --query-gpu=ecc.errors.uncorrected.volatile.total --format=csv,noheader > "$TMP_ECC_ERRORS_OUT" - cat "$TMP_ECC_ERRORS_OUT" >> "$LOG_FILE" - ECC_ERRORS=$(awk -F', ' '{sum += $2} END {print sum}' "$TMP_ECC_ERRORS_OUT") - log_step "ECC Errors: $ECC_ERRORS" - - # Check for NVLink errors - NVLINK_ERRORS=$(nvidia-smi nvlink -sc 0bz -i 0 2>/dev/null | grep -i "Error Count" | awk '{sum += $3} END {print sum}') - # Set to 0 if empty/null - NVLINK_ERRORS=${NVLINK_ERRORS:-0} - log_step "NVLink Errors: $NVLINK_ERRORS" - - if [ $DCGM_FAILED -eq 1 ] || \ - [ $ECC_ERRORS -gt 0 ] || \ - [ $NVLINK_ERRORS -gt 0 ]; then - REASON="GPU issues detected: " - if [ $DCGM_FAILED -eq 1 ]; then - REASON+="DCGM test failed, " - fi - if [ $ECC_ERRORS -gt 0 ]; then - REASON+="ECC errors found ($ECC_ERRORS double-bit errors), " - fi - if [ $NVLINK_ERRORS -gt 0 ]; then - REASON+="NVLink errors detected ($NVLINK_ERRORS errors), " - fi - REASON+="see $LOG_FILE" - log_step "$REASON" - exit 1 - fi -fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog deleted file mode 100644 index a22ddea9e5..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Main TaskEpilog Script -# This script executes all *.sh scripts found in /slurm/custom_scripts/task_epilog.d/ -# -# slurm.conf configuration: -# TaskEpilog=/slurm/scripts/tools/task-epilog - -# Directory containing the individual task epilog scripts -EPILOG_D_DIR="/slurm/custom_scripts/task_epilog.d" - -# --- Output Handling for TaskEpilog --- -# The stdout and stderr of this script (and the sub-scripts it calls) -# are typically captured by Slurm and written to the job's output/error file -# or a separate Slurm log, depending on configuration. -# Unlike TaskProlog, stdout is not typically parsed for special commands -# like 'export' or 'print' to affect the (now finished) task's environment. -# -# --- Error Handling --- -# If any script in EPILOG_D_DIR exits with a non-zero status, -# this main script will also exit with a non-zero status. -# Slurm will log this. Depending on Slurm's configuration, -# frequent epilog failures might lead to node issues or alerts. -set -e # Exit immediately if a command exits with a non-zero status. - -# Check if the directory exists -if [[ ! -d "$EPILOG_D_DIR" ]]; then - # Log in task stdout and exit if the directory is missing. This likely indicates a configuration error. - echo "print TaskEpilog Error: Directory '$EPILOG_D_DIR' not found. Check Slurm configuration." - exit 1 -fi - -# Find and execute all *.sh scripts in the directory -# Scripts will be executed in reverse alphabetical order of their filenames. -find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do - if [[ -x "$script" ]]; then - # Execute the script. Its stdout will be captured by this wrapper. - # Its stderr will also be passed through. - # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. - "$script" - else - # Log in task stdout a warning if a *.sh file is found but is not executable - echo "print TaskEpilog Warning: Script '$script' is not executable and will be skipped." - fi -done - -# Check if any scripts were found and executed -if [[ $(find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then - # Log in task stdout if no scripts were found to execute - echo "print TaskEpilog Info: No executable *.sh scripts found in $EPILOG_D_DIR." -fi - -# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) -exit 0 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog deleted file mode 100644 index feddb23209..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Main TaskProlog Script -# This script executes all *.sh scripts found in /slurm/custom_scripts/task_prolog.d/ -# -# slurm.conf configuration: -# TaskProlog=/slurm/scripts/tools/task-prolog - -# Directory containing the individual task prolog scripts -PROLOG_D_DIR="/slurm/custom_scripts/task_prolog.d" - -# --- Output Handling for TaskProlog --- -# Slurm's TaskProlog can interpret specific stdout lines: -# - "export NAME=value" : Sets an environment variable for the task. -# - "unset NAME" : Unsets an environment variable for the task. -# - "print message" : Prints a message to the task's standard output. -# -# This wrapper script will concatenate the stdout of all sub-scripts. -# If sub-scripts need to set/unset environment variables or print messages -# for the task, they should output the appropriate "export", "unset", or "print" -# commands to their own stdout. - -# --- Error Handling --- -# If any script in PROLOG_D_DIR exits with a non-zero status, -# this main script will also exit with a non-zero status. -# This will typically cause the task to fail. -set -e # Exit immediately if a command exits with a non-zero status. - -# Check if the directory exists -if [[ ! -d "$PROLOG_D_DIR" ]]; then - # Log in task stdout and exit if the directory is missing. All jobs will be failed. - echo "print TaskProlog Error: Directory '$PROLOG_D_DIR' not found. Check Slurm configuration." - exit 1 -fi - -# Find and execute all *.sh scripts in the directory -# Scripts will be executed in reverse alphabetical order of their filenames. -find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do - if [[ -x "$script" ]]; then - # Execute the script. Its stdout will be captured by this wrapper. - # Its stderr will also be passed through. - # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. - "$script" - else - # Log a warning in task stdout if a *.sh file is found but is not executable - echo "print TaskProlog Warning: Script '$script' is not executable and will be skipped." - fi -done - -# Check if any scripts were found and executed -if [[ $(find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then - # Log in task stdout if no scripts were found to execute - echo "print TaskProlog Info: No executable *.sh scripts found in $PROLOG_D_DIR." -fi - -# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) -exit 0 diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py deleted file mode 100644 index 531f0348dc..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py +++ /dev/null @@ -1,331 +0,0 @@ -# mypy: ignore-errors -# This implementation of TPU integration is to be deprecated - -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List - -import socket -import logging -from pathlib import Path -import yaml - -import util -from util import create_client_options, ApiEndpoint - -from google.cloud import tpu_v2 as tpu # noqa: E402 -import google.api_core.exceptions as gExceptions # noqa: E402 - -log = logging.getLogger() - -_tpu_cache = {} - -class TPU: - """Class for handling the TPU-vm nodes""" - - State = tpu.types.cloud_tpu.Node.State - TPUS_PER_VM = 4 - __expected_states = { - "create": State.READY, - "start": State.READY, - "stop": State.STOPPED, - } - - __tpu_version_mapping = { - "V2": tpu.AcceleratorConfig().Type.V2, - "V3": tpu.AcceleratorConfig().Type.V3, - "V4": tpu.AcceleratorConfig().Type.V4, - } - - @classmethod - def make(cls, nodeset_name: str, lkp: util.Lookup) -> "TPU": - key = (id(lkp), nodeset_name) - if key not in _tpu_cache: - nodeset = lkp.cfg.nodeset_tpu[nodeset_name] - _tpu_cache[key] = cls(nodeset, lkp) - return _tpu_cache[key] - - - def __init__(self, nodeset: object, lkp: util.Lookup): - self._nodeset = nodeset - self.lkp = lkp - self._parent = f"projects/{lkp.project}/locations/{nodeset.zone}" - co = create_client_options(ApiEndpoint.TPU) - self._client = tpu.TpuClient(client_options=co) - self.data_disks = [] - for data_disk in nodeset.data_disks: - ad = tpu.AttachedDisk() - ad.source_disk = data_disk - ad.mode = tpu.AttachedDisk.DiskMode.DISK_MODE_UNSPECIFIED - self.data_disks.append(ad) - ns_ac = nodeset.accelerator_config - if ns_ac.topology != "" and ns_ac.version != "": - ac = tpu.AcceleratorConfig() - ac.topology = ns_ac.topology - ac.type_ = self.__tpu_version_mapping[ns_ac.version] - self.ac = ac - else: - req = tpu.GetAcceleratorTypeRequest( - name=f"{self._parent}/acceleratorTypes/{nodeset.node_type}" - ) - self.ac = self._client.get_accelerator_type(req).accelerator_configs[0] - self.vmcount = self.__calc_vm_from_topology(self.ac.topology) - - @property - def nodeset(self): - return self._nodeset - - @property - def preserve_tpu(self): - return self._nodeset.preserve_tpu - - @property - def node_type(self): - return self._nodeset.node_type - - @property - def tf_version(self): - return self._nodeset.tf_version - - @property - def enable_public_ip(self): - return self._nodeset.enable_public_ip - - @property - def preemptible(self): - return self._nodeset.preemptible - - @property - def reserved(self): - return self._nodeset.reserved - - @property - def service_account(self): - return self._nodeset.service_account - - @property - def zone(self): - return self._nodeset.zone - - def check_node_type(self): - if self.node_type is None: - return False - try: - request = tpu.GetAcceleratorTypeRequest( - name=f"{self._parent}/acceleratorTypes/{self.node_type}" - ) - return self._client.get_accelerator_type(request=request) is not None - except Exception: - return False - - def check_tf_version(self): - try: - request = tpu.GetRuntimeVersionRequest( - name=f"{self._parent}/runtimeVersions/{self.tf_version}" - ) - return self._client.get_runtime_version(request=request) is not None - except Exception: - return False - - def __calc_vm_from_topology(self, topology): - topo = topology.split("x") - tot = 1 - for num in topo: - tot = tot * int(num) - return tot // self.TPUS_PER_VM - - def __check_resp(self, response, op_name): - des_state = self.__expected_states.get(op_name) - # If the state is not in the table just print the response - if des_state is None: - return False - if response.__class__.__name__ != "Node": # If the response is not a node fail - return False - if response.state == des_state: - return True - return False - - def list_nodes(self): - try: - request = tpu.ListNodesRequest(parent=self._parent) - res = self._client.list_nodes(request=request) - except gExceptions.NotFound: - res = None - return res - - def list_node_names(self): - return [node.name.split("/")[-1] for node in self.list_nodes()] - - def start_node(self, nodename): - request = tpu.StartNodeRequest(name=f"{self._parent}/nodes/{nodename}") - resp = self._client.start_node(request=request).result() - return self.__check_resp(resp, "start") - - def stop_node(self, nodename): - request = tpu.StopNodeRequest(name=f"{self._parent}/nodes/{nodename}") - resp = self._client.stop_node(request=request).result() - return self.__check_resp(resp, "stop") - - def get_node(self, nodename): - try: - request = tpu.GetNodeRequest(name=f"{self._parent}/nodes/{nodename}") - res = self._client.get_node(request=request) - except gExceptions.NotFound: - res = None - return res - - def _register_node(self, nodename, ip_addr): - dns_name = socket.getnameinfo((ip_addr, 0), 0)[0] - util.run( - f"{self.lkp.scontrol} update nodename={nodename} nodeaddr={ip_addr} nodehostname={dns_name}" - ) - - def create_node(self, nodename): - if self.vmcount > 1 and not isinstance(nodename, list): - log.error( - f"Tried to create a {self.vmcount} node TPU on nodeset {self._nodeset.nodeset_name} but only received one nodename {nodename}" - ) - return False - if self.vmcount > 1 and ( - isinstance(nodename, list) and len(nodename) != self.vmcount - ): - log.error( - f"Expected to receive a list of {self.vmcount} nodenames for TPU node creation in nodeset {self._nodeset.nodeset_name}, but received this list {nodename}" - ) - return False - - node = tpu.Node() - node.accelerator_config = self.ac - node.runtime_version = f"tpu-vm-tf-{self.tf_version}" - startup_script = """ - #!/bin/bash - echo "startup script not found > /var/log/startup_error.log" - """ - with open( - Path(self.lkp.cfg.slurm_scripts_dir or util.dirs.scripts) / "startup.sh", "r" - ) as script: - startup_script = script.read() - if isinstance(nodename, list): - node_id = nodename[0] - slurm_names = [] - wid = 0 - for node_wid in nodename: - slurm_names.append(f"WORKER_{wid}:{node_wid}") - wid += 1 - else: - node_id = nodename - slurm_names = [f"WORKER_0:{nodename}"] - node.metadata = { - "slurm_docker_image": self.nodeset.docker_image, - "startup-script": startup_script, - "slurm_instance_role": "compute", - "slurm_cluster_name": self.lkp.cfg.slurm_cluster_name, - "slurm_bucket_path": self.lkp.cfg.bucket_path, - "slurm_names": ";".join(slurm_names), - "universe_domain": util.universe_domain(), - } - node.tags = [self.lkp.cfg.slurm_cluster_name] - if self.nodeset.service_account: - node.service_account.email = self.nodeset.service_account.email - node.service_account.scope = self.nodeset.service_account.scopes - node.scheduling_config.preemptible = self.preemptible - node.scheduling_config.reserved = self.reserved - node.network_config.subnetwork = self.nodeset.subnetwork - node.network_config.enable_external_ips = self.enable_public_ip - if self.data_disks: - node.data_disks = self.data_disks - - request = tpu.CreateNodeRequest(parent=self._parent, node=node, node_id=node_id) - resp = self._client.create_node(request=request).result() - if not self.__check_resp(resp, "create"): - return False - if isinstance(nodename, list): - for node_id, net_endpoint in zip(nodename, resp.network_endpoints): - self._register_node(node_id, net_endpoint.ip_address) - else: - ip_add = resp.network_endpoints[0].ip_address - self._register_node(nodename, ip_add) - return True - - def delete_node(self, nodename): - request = tpu.DeleteNodeRequest(name=f"{self._parent}/nodes/{nodename}") - try: - resp = self._client.delete_node(request=request).result() - if resp: - return self.get_node(nodename=nodename) is None - return False - except gExceptions.NotFound: - # log only error if vmcount is 1 as for other tpu vm count, this could be "phantom" nodes - if self.vmcount == 1: - log.error(f"Tpu single node {nodename} not found") - else: - # for the TPU nodes that consist in more than one vm, only the first node of the TPU a.k.a. the master node will - # exist as real TPU nodes, so the other ones are expected to not be found, check the hostname of the node that has - # not been found, and if it ends in 0, it means that is the master node and it should have been found, and in consequence - # log an error - nodehostname = yaml.safe_load( - util.run(f"{self.lkp.scontrol} --yaml show node {nodename}").stdout.rstrip() - )["nodes"][0]["hostname"] - if nodehostname.split("-")[-1] == "0": - log.error(f"TPU master node {nodename} not found") - else: - log.info(f"Deleted TPU 'phantom' node {nodename}") - # If the node is not found it is tecnichally deleted, so return success. - return True - -def _stop_tpu(node: str) -> None: - lkp = util.lookup() - tpuobj = TPU.make(lkp.node_nodeset_name(node), lkp) - if tpuobj.nodeset.preserve_tpu and tpuobj.vmcount == 1: - log.info(f"stopping node {node}") - if tpuobj.stop_node(node): - return - log.error("Error stopping node {node} will delete instead") - log.info(f"deleting node {node}") - if not tpuobj.delete_node(node): - log.error("Error deleting node {node}") - - -def delete_tpu_instances(instances: List[str]) -> None: - util.execute_with_futures(_stop_tpu, instances) - - -def start_tpu(node: List[str]): - lkp = util.lookup() - tpuobj = TPU.make(lkp.node_nodeset_name(node[0]), lkp) - - if len(node) == 1: - node = node[0] - log.debug( - f"Will create a TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node}" - ) - tpunode = tpuobj.get_node(node) - if tpunode is None: - if not tpuobj.create_node(nodename=node): - log.error("Error creating tpu node {node}") - else: - if tpuobj.preserve_tpu: - if not tpuobj.start_node(nodename=node): - log.error("Error starting tpu node {node}") - else: - log.info( - f"Tpu node {node} is already created, but will not start it because nodeset does not have preserve_tpu option active." - ) - else: - log.debug( - f"Will create a multi-vm TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node[0]}" - ) - if not tpuobj.create_node(nodename=node): - log.error("Error creating tpu node {node}") diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py deleted file mode 100644 index 217fd0bca2..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py +++ /dev/null @@ -1,2224 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Iterable, List, Tuple, Optional, Any, Dict, Sequence, Type, Callable, Union -import argparse -import base64 -from dataclasses import dataclass, field -from datetime import timedelta, datetime, timezone -import hashlib -import inspect -import json -import logging -import logging.config -import logging.handlers -import math -import os -import re -import shlex -import shutil -import socket -import subprocess -import sys -from enum import Enum -from collections import defaultdict -from concurrent.futures import ThreadPoolExecutor, as_completed -from contextlib import contextmanager -from functools import lru_cache, reduce, wraps -from itertools import chain, islice -from pathlib import Path -from time import sleep, time - -# TODO: remove "type: ignore" once moved to newer version of libraries -from google.cloud import secretmanager -from google.cloud import storage # type: ignore - -import google.auth # type: ignore -from google.oauth2 import service_account # type: ignore -import googleapiclient.discovery # type: ignore -import google_auth_httplib2 # type: ignore -from googleapiclient.http import set_user_agent # type: ignore -from google.api_core.client_options import ClientOptions -import httplib2 - -import google.api_core.exceptions as gExceptions - -import requests as requests_lib - -import yaml -from addict import Dict as NSDict # type: ignore -import file_cache - -USER_AGENT = "Slurm_GCP_Scripts/1.5 (GPN:SchedMD)" -ENV_CONFIG_YAML = os.getenv("SLURM_CONFIG_YAML") -if ENV_CONFIG_YAML: - CONFIG_FILE = Path(ENV_CONFIG_YAML) -else: - CONFIG_FILE = Path(__file__).with_name("config.yaml") -API_REQ_LIMIT = 2000 - - -def mkdirp(path: Path) -> None: - path.mkdir(parents=True, exist_ok=True) - - -scripts_dir = next( - p for p in (Path(__file__).parent, Path("/slurm/scripts")) if p.is_dir() -) - - -# load all directories as Paths into a dict-like namespace -dirs = NSDict( - home = Path("/home"), - apps = Path("/opt/apps"), - slurm = Path("/slurm"), - scripts = scripts_dir, - custom_scripts = Path("/slurm/custom_scripts"), - munge = Path("/etc/munge"), - secdisk = Path("/mnt/disks/sec"), - log = Path("/var/log/slurm"), - slurm_bucket_mount = Path("/slurm/bucket"), -) - -slurmdirs = NSDict( - prefix = Path("/usr/local"), - etc = Path("/usr/local/etc/slurm"), - state = Path("/var/spool/slurm"), - key_distribution = Path("/slurm/key_distribution"), -) - - -# TODO: Remove this hack (relies on undocumented behavior of PyYAML) -# No need to represent NSDict and Path once we move to properly typed & serializable config. -yaml.SafeDumper.yaml_representers[ - None # type: ignore -] = lambda self, data: yaml.representer.SafeRepresenter.represent_str(self, str(data)) # type: ignore - - -class ApiEndpoint(Enum): - COMPUTE = "compute" - BQ = "bq" - STORAGE = "storage" - TPU = "tpu" - SECRET = "secret_manager" - - -@dataclass(frozen=True) -class AcceleratorInfo: - type: str - count: int - - @classmethod - def from_json(cls, jo: dict) -> "AcceleratorInfo": - return cls( - type=jo["guestAcceleratorType"], - count=jo["guestAcceleratorCount"]) - -@dataclass(frozen=True) -class MachineType: - name: str - guest_cpus: int - memory_mb: int - accelerators: List[AcceleratorInfo] - - @classmethod - def from_json(cls, jo: dict) -> "MachineType": - return cls( - name=jo["name"], - guest_cpus=jo["guestCpus"], - memory_mb=jo["memoryMb"], - accelerators=[ - AcceleratorInfo.from_json(a) for a in jo.get("accelerators", [])], - ) - - @property - def family(self) -> str: - # TODO: doesn't work with N1 custom machine types - # See https://cloud.google.com/compute/docs/instances/creating-instance-with-custom-machine-type#create - return self.name.split("-")[0] - - @property - def supports_smt(self) -> bool: - # https://cloud.google.com/compute/docs/cpu-platforms - if self.family in ("t2a", "t2d", "h3", "c4a", "h4d",): - return False - if self.guest_cpus == 1: - return False - return True - - @property - def sockets(self) -> int: - return { - "h3": 2, - "h4d": 2, - "c2d": 2 if self.guest_cpus > 56 else 1, - "a3": 2, - "c2": 2 if self.guest_cpus > 30 else 1, - "c3": 2 if self.guest_cpus > 88 else 1, - "c3d": 2 if self.guest_cpus > 180 else 1, - "c4": 2 if self.guest_cpus > 96 else 1, - "c4d": 2 if self.guest_cpus > 192 else 1, - }.get( - self.family, - 1, # assume 1 socket for all other families - ) - - -@dataclass(frozen=True) -class UpcomingMaintenance: - window_start_time: datetime - - @classmethod - def from_json(cls, jo: Optional[dict]) -> Optional["UpcomingMaintenance"]: - if jo is None: - return None - try: - if "windowStartTime" in jo: - ts = parse_gcp_timestamp(jo["windowStartTime"]) - elif "startTimeWindow" in jo: - ts = parse_gcp_timestamp(jo["startTimeWindow"]["earliest"]) - else: - raise Exception("Neither windowStartTime nor startTimeWindow are found") - except BaseException as e: - raise ValueError(f"Unexpected format for upcomingMaintenance: {jo}") from e - return cls(window_start_time=ts) - -@dataclass(frozen=True) -class InstanceResourceStatus: - physical_host: Optional[str] - upcoming_maintenance: Optional[UpcomingMaintenance] - - @classmethod - def from_json(cls, jo: Optional[dict]) -> "InstanceResourceStatus": - if not jo: - return cls( - physical_host=None, - upcoming_maintenance=None, - ) - - try: - maint = UpcomingMaintenance.from_json(jo.get("upcomingMaintenance")) - except ValueError as e: - log.exception("Failed to parse upcomingMaintenance, ignoring") - maint = None # intentionally swallow exception - - return cls( - physical_host=jo.get("physicalHost"), - upcoming_maintenance=maint, - ) - - -@dataclass(frozen=True) -class Instance: - name: str - zone: str - status: str - creation_timestamp: datetime - role: Optional[str] - resource_status: InstanceResourceStatus - metadata: Dict[str, str] - # TODO: use proper InstanceScheduling class - scheduling: NSDict - - @classmethod - def from_json(cls, jo: dict) -> "Instance": - return cls( - name=jo["name"], - zone=trim_self_link(jo["zone"]), - status=jo["status"], - creation_timestamp=parse_gcp_timestamp(jo["creationTimestamp"]), - resource_status=InstanceResourceStatus.from_json(jo.get("resourceStatus")), - scheduling=NSDict(jo.get("scheduling")), - role = jo.get("labels", {}).get("slurm_instance_role"), - metadata = {k["key"]: k["value"] for k in jo.get("metadata", {}).get("items", [])} - ) - - -@dataclass(frozen=True) -class NSMount: - server_ip: str - local_mount: Path - remote_mount: Path - fs_type: str - mount_options: str - -@lru_cache(maxsize=1) -def default_credentials(): - return google.auth.default()[0] - - -@lru_cache(maxsize=1) -def authentication_project(): - return google.auth.default()[1] - - -DEFAULT_UNIVERSE_DOMAIN = "googleapis.com" - - -def now() -> datetime: - """ - Return current time as timezone-aware datetime. - - IMPORTANT: DO NOT use `datetime.now()`, unless you explicitly need to have tz-naive datetime. - Otherwise there is a risk of getting: "cannot compare naive and aware datetimes" error, - since all timetstamps we receive from GCP API are tz-aware. - - Another motivation for this function is to allow to mock time in tests. - """ - return datetime.now(timezone.utc) - -def parse_gcp_timestamp(s: str) -> datetime: - """ - Parse timestamp strings returned by GCP API into datetime. - Works with both Zulu and non-Zulu timestamps. - NOTE: It always return tz-aware datetime (fallbacks to UTC and logs error). - """ - # Requires Python >= 3.7 - # TODO: Remove this "hack" of trimming the Z from timestamps once we move to Python 3.11 - # (context: https://discuss.python.org/t/parse-z-timezone-suffix-in-datetime/2220/30) - ts = datetime.fromisoformat(s.replace('Z', '+00:00')) - if ts.tzinfo is None: # fallback to UTC - log.error(f"Received timestamp without timezone info: {s}") - ts = ts.replace(tzinfo=timezone.utc) - return ts - - -def universe_domain() -> str: - try: - return instance_metadata("attributes/universe_domain") - except MetadataNotFoundError: - return DEFAULT_UNIVERSE_DOMAIN - - -def endpoint_version(api: ApiEndpoint) -> Optional[str]: - return lookup().endpoint_versions.get(api.value, None) - - -@lru_cache(maxsize=1) -def get_credentials() -> Optional[service_account.Credentials]: - """Get credentials for service account""" - key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") - if key_path is not None: - credentials = service_account.Credentials.from_service_account_file( - key_path, scopes=[f"https://www.{universe_domain()}/auth/cloud-platform"] - ) - else: - credentials = default_credentials() - - return credentials - - -@lru_cache(maxsize=1) -def get_dev_key() -> Optional[str]: - """Get dev key for project (uses json or yaml format)""" - try: - with open("/etc/slurm/slurm_vars.yaml", 'r') as file: - data = yaml.safe_load(file) - return data['google_developer_key'] - except: - return None - - -def create_client_options(api: ApiEndpoint) -> ClientOptions: - """Create client options for cloud endpoints""" - ver = endpoint_version(api) - ud = universe_domain() - options = {} - if ud and ud != DEFAULT_UNIVERSE_DOMAIN: - options["universe_domain"] = ud - if ver: - options["api_endpoint"] = f"https://{api.value}.{ud}/{ver}/" - co = ClientOptions(**options) - log.debug(f"Using ClientOptions = {co} for API: {api.value}") - return co - -log = logging.getLogger() - - -def access_secret_version(project_id, secret_id, version_id="latest"): - """ - Access the payload for the given secret version if one exists. The version - can be a version number as a string (e.g. "5") or an alias (e.g. "latest"). - """ - co = create_client_options(ApiEndpoint.SECRET) - client = secretmanager.SecretManagerServiceClient(client_options=co) - name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" - try: - response = client.access_secret_version(request={"name": name}) - log.debug(f"Secret '{name}' was found.") - payload = response.payload.data.decode("UTF-8") - except gExceptions.NotFound: - log.debug(f"Secret '{name}' was not found!") - payload = None - - return payload - - -def parse_self_link(self_link: str): - """Parse a selfLink url, extracting all useful values - https://.../v1/projects//regions//... - {'project': , 'region': , ...} - can also extract zone, instance (name), image, etc - """ - link_patt = re.compile(r"(?P[^\/\s]+)s\/(?P[^\s\/]+)") - return NSDict(link_patt.findall(self_link)) - - -def parse_bucket_uri(uri: str): - """ - Parse a bucket url - E.g. gs:/// - """ - pattern = re.compile(r"gs://(?P[^/\s]+)/(?P([^/\s]+)(/[^/\s]+)*)") - matches = pattern.match(uri) - assert matches, f"Unexpected bucker URI: '{uri}'" - return matches.group("bucket"), matches.group("path") - - -def get_template_gpu(template): - """get gpu info from machine type or guest accelerators""" - gpu_keyword = "nvidia" - gpu = None - if template.machine_type.accelerators: - tma = template.machine_type.accelerators[0] - if gpu_keyword in tma.type.lower(): - gpu = tma - elif template.guestAccelerators: - tga = template.guestAccelerators[0] - if gpu_keyword in tga.acceleratorType.lower(): - gpu = AcceleratorInfo( - type=tga.acceleratorType, - count=tga.acceleratorCount) - return gpu - - -def trim_self_link(link: str): - """get resource name from self link url, eg. - https://.../v1/projects//regions/ - -> - """ - try: - return link[link.rindex("/") + 1 :] - except ValueError: - raise Exception(f"'/' not found, not a self link: '{link}' ") - - -def get_self_link_component(link: str, component_name: str): - """ - Extracts a component (e.g., 'region', 'project') from a self-link URL. - Args: - link: The self-link URL string. - component_name: The name of the component to extract (e.g., 'regions', 'projects'). - Returns: - The extracted component value (e.g., '', ''), - or None if the component is not found in the link. - """ - search_string = f"/{component_name}/" - start_index = link.rfind(search_string) - - if start_index == -1: - return None - - start_index += len(search_string) - end_index = link.find("/", start_index) - - if end_index == -1: - # If no further slash, the rest of the string is the component - return link[start_index:] - else: - return link[start_index:end_index] - - -def execute_with_futures(func, seq): - with ThreadPoolExecutor() as exe: - futures = [] - for i in seq: - future = exe.submit(func, i) - futures.append(future) - for future in as_completed(futures): - result = future.exception() - if result is not None: - raise result - - -def map_with_futures(func, seq): - with ThreadPoolExecutor() as exe: - futures = [] - for i in seq: - future = exe.submit(func, i) - futures.append(future) - for future in futures: - # Will be result or raise Exception - res = None - try: - res = future.result() - except Exception as e: - res = e - yield res - -def should_mount_slurm_bucket() -> bool: - try: - return instance_metadata("attributes/slurm_bucket_mount", silent=True).lower() == "true" - except MetadataNotFoundError: - return False - - -def _get_bucket_and_common_prefix() -> Tuple[str, str]: - uri = instance_metadata("attributes/slurm_bucket_path") - return parse_bucket_uri(uri) - -def blob_get(file): - bucket_name, path = _get_bucket_and_common_prefix() - blob_name = f"{path}/{file}" - return storage_client().get_bucket(bucket_name).blob(blob_name) - - -def blob_list(prefix="", delimiter=None): - bucket_name, path = _get_bucket_and_common_prefix() - blob_prefix = f"{path}/{prefix}" - # Note: The call returns a response only when the iterator is consumed. - blobs = storage_client().list_blobs( - bucket_name, prefix=blob_prefix, delimiter=delimiter - ) - return [blob for blob in blobs] - -def file_list(prefix="", subpath="") -> List[os.DirEntry]: - path = dirs.slurm_bucket_mount - file_prefix = f"{path}/{subpath}" - try: - files = os.scandir(file_prefix) - return [file for file in files if file.name.startswith(prefix)] - except: - return [] - # Not considering lack of file's existence as fatal (we may check for files we know don't exist). - # Responsibility of callee to determine if it is fatal or not, blob_list returns empty iterator in similar cases. - -def hash_file(fullpath: Path) -> str: - with open(fullpath, "rb") as f: - file_hash = hashlib.md5() - chunk = f.read(8192) - while chunk: - file_hash.update(chunk) - chunk = f.read(8192) - return base64.b64encode(file_hash.digest()).decode("utf-8") - - -def install_custom_scripts(check_hash:bool=False): - """download custom scripts from gcs bucket""" - role, tokens = lookup().instance_role, [] - - mounted_scripts=False - if should_mount_slurm_bucket() and role != "controller": - mounted_scripts=True - - all_prolog_tokens = ["prolog", "epilog", "task_prolog", "task_epilog"] - if role == "controller": - tokens = ["controller"] + all_prolog_tokens - elif role == "compute": - tokens = [f"nodeset-{lookup().node_nodeset_name()}"] + all_prolog_tokens - elif role == "login": - tokens = [f"login-{instance_login_group()}"] - - prefixes = [f"slurm-{tok}-script" for tok in tokens] - - # TODO: use single `blob_list`, to reduce ~4x number of GCS requests - if mounted_scripts: - source_collection = list(chain.from_iterable(file_list(prefix=p) for p in prefixes)) - else: - source_collection = list(chain.from_iterable(blob_list(prefix=p) for p in prefixes)) - - script_pattern = re.compile(r"^slurm-(?P\S+)-script-(?P\S+)") - for source in source_collection: - if mounted_scripts: - m = script_pattern.match(source.name) - else: - m = script_pattern.match(Path(source.name).name) - - if not m: - log.warning(f"found blob that doesn't match expected pattern: {source.name}") - continue - path_parts = m["path"].split("-") - path_parts[0] += ".d" - stem, _, ext = m["name"].rpartition("_") - filename = ".".join((stem, ext)) - - path = Path(*path_parts, filename) - fullpath = (dirs.custom_scripts / path).resolve() - mkdirp(fullpath.parent) - - for par in path.parents: - chown_slurm(dirs.custom_scripts / par) - need_update = True - - if check_hash and fullpath.exists() and isinstance(source,storage.Blob): - # TODO: MD5 reported by gcloud may differ from the one calculated here (e.g. if blob got gzipped), - # consider using gCRC32C - need_update = hash_file(fullpath) != source.md5_hash - - log.info(f"installing custom script: {path} from {source.name}") - - if isinstance(source,os.DirEntry): - shutil.copy(source.path, fullpath) #Needs to be copied since mounted nfs is read-only - chown_slurm(fullpath, mode=0o755) - - elif need_update: - with fullpath.open("wb") as f: - source.download_to_file(f) - chown_slurm(fullpath, mode=0o755) - -def compute_service(version="beta"): - """Make thread-safe compute service handle - creates a new Http for each request - """ - credentials = get_credentials() - dev_key = get_dev_key() - - def build_request(http, *args, **kwargs): - new_http = set_user_agent(httplib2.Http(), USER_AGENT) - if credentials is not None: - new_http = google_auth_httplib2.AuthorizedHttp(credentials, http=new_http) - return googleapiclient.http.HttpRequest(new_http, *args, **kwargs) - - ver = endpoint_version(ApiEndpoint.COMPUTE) - disc_url = googleapiclient.discovery.DISCOVERY_URI - if ver: - version = ver - disc_url = disc_url.replace(DEFAULT_UNIVERSE_DOMAIN, universe_domain()) - - log.debug(f"Using version={version} of Google Compute Engine API") - return googleapiclient.discovery.build( - "compute", - version, - requestBuilder=build_request, - credentials=credentials, - developerKey=dev_key, - discoveryServiceUrl=disc_url, - cache_discovery=False, # See https://github.com/googleapis/google-api-python-client/issues/299 - ) - -def storage_client() -> storage.Client: - """ - Config-independent storage client - """ - ud = universe_domain() - co = {} - if ud and ud != DEFAULT_UNIVERSE_DOMAIN: - co["universe_domain"] = ud - return storage.Client(client_options=ClientOptions(**co)) - - -class DeffetiveStoredConfigError(Exception): - """ - Raised when config can not be loaded and assembled from bucket - """ - pass - - -def _fill_cfg_defaults(cfg: NSDict) -> NSDict: - if not cfg.slurm_log_dir: - cfg.slurm_log_dir = dirs.log - if not cfg.slurm_bin_dir: - cfg.slurm_bin_dir = slurmdirs.prefix / "bin" - if not cfg.slurm_control_host: - try: - control_dns_name = instance_metadata("attributes/slurm_control_dns", silent=True) - cfg.slurm_control_host = control_dns_name - except MetadataNotFoundError: - cfg.slurm_control_host = f"{cfg.slurm_cluster_name}-controller" - if not cfg.slurm_control_host_port: - cfg.slurm_control_host_port = "6820-6830" - return cfg - -@dataclass -class _ConfigBlobs: - """ - "Private" class that represent a collection of GCS blobs for configuration - """ - core: storage.Blob - controller_addr: Optional[storage.Blob] - partition: List[storage.Blob] = field(default_factory=list) - nodeset: List[storage.Blob] = field(default_factory=list) - nodeset_dyn: List[storage.Blob] = field(default_factory=list) - nodeset_tpu: List[storage.Blob] = field(default_factory=list) - login_group: List[storage.Blob] = field(default_factory=list) - - @property - def hash(self) -> str: - h = hashlib.md5() - all = [self.core] + self.partition + self.nodeset + self.nodeset_dyn + self.nodeset_tpu - if self.controller_addr: - all.append(self.controller_addr) - - # sort blobs so hash is consistent - for blob in sorted(all, key=lambda b: b.name): - h.update(blob.md5_hash.encode("utf-8")) - return h.hexdigest() - -@dataclass -class _ConfigFiles: - """ - "Private" class that represent a collection of files for configuration - """ - core: Path - controller_addr: Optional[Path] - partition: List[Path] = field(default_factory=list) - nodeset: List[Path] = field(default_factory=list) - nodeset_dyn: List[Path] = field(default_factory=list) - nodeset_tpu: List[Path] = field(default_factory=list) - login_group: List[Path] = field(default_factory=list) - -def _list_config_blobs() -> _ConfigBlobs: - _, common_prefix = _get_bucket_and_common_prefix() - - core: Optional[storage.Blob] = None - controller_addr: Optional[storage.Blob] = None - rest: Dict[str, List[storage.Blob]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} - - is_controller = instance_role() == "controller" - - for blob in blob_list(prefix=""): - if blob.name == f"{common_prefix}/config.yaml": - core = blob - if blob.name == f"{common_prefix}/controller_addr.yaml" and not is_controller: - # Don't add this config blobs for controller to avoid "double reconfiguration": - # Initially this file doesn't exist and produce later by `setup_controller`; - # Appearance of this blob would trigger change in combined hash of config files; - # Ignore existence of this file for controller, assume that - # no other instance nodes will proceed with configuration until this file is created. - controller_addr = blob - for key in rest.keys(): - if blob.name.startswith(f"{common_prefix}/{key}_configs/"): - rest[key].append(blob) - - if core is None: - raise DeffetiveStoredConfigError(f"{common_prefix}/config.yaml not found in bucket") - - return _ConfigBlobs(core=core, controller_addr=controller_addr, **rest) - -def _list_config_files() -> _ConfigFiles: - file_dir = dirs.slurm_bucket_mount - core: Optional[Path] = None - controller_addr: Optional[Path] = None - rest: Dict[str, List[Path]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} - - if Path(f"{file_dir}/config.yaml").exists(): - core = Path(f"{file_dir}/config.yaml") - - for key in rest.keys(): - for f in file_list(subpath=f"{key}_configs"): - rest[key].append(f.path) - - if core is None: - raise Exception(f"config.yaml was not found in mounted folder: {dirs.slurm_bucket_mount}") #Intentionally not using DeffetiveStoredConfigError as this is considered a fatal error - - return _ConfigFiles(core=core, controller_addr=None, **rest) - -def _fetch_config(old_hash: Optional[str]) -> Optional[Tuple[NSDict, str]]: - """Fetch config from bucket, returns None if no changes are detected.""" - blobs = _list_config_blobs() - if old_hash == blobs.hash: - return None - - def _download(bs) -> List[Any]: - return [yaml.safe_load(b.download_as_text()) for b in bs] - - return _assemble_config( - core=_download([blobs.core])[0], - controller_addr=_download([blobs.controller_addr])[0] if blobs.controller_addr else None, - partitions=_download(blobs.partition), - nodesets=_download(blobs.nodeset), - nodesets_dyn=_download(blobs.nodeset_dyn), - nodesets_tpu=_download(blobs.nodeset_tpu), - login_groups=_download(blobs.login_group), - ), blobs.hash - -def _fetch_mounted_config() -> Optional[Tuple[NSDict, str]]: - if not dirs.slurm_bucket_mount.is_mount(): - raise Exception(f"{dirs.slurm_bucket_mount} is not mounted") - - files = _list_config_files() - - def _load(files) -> List[Any]: - file_yaml=[] - for file in files: - with open(file, "r") as f: - file_yaml.append(yaml.safe_load(f)) - return file_yaml - - return _assemble_config( - core=_load([files.core])[0], - controller_addr=None, - partitions=_load(files.partition), - nodesets=_load(files.nodeset), - nodesets_dyn=_load(files.nodeset_dyn), - nodesets_tpu=_load(files.nodeset_tpu), - login_groups=_load(files.login_group), - ) - -def controller_lookup_self_ip() -> str: - assert instance_role() == "controller" - # Get IP of LAST network-interface - # TODO: Consider change order of NICs definition, so right NIC is always @0. - idx = instance_metadata("network-interfaces").split()[-1] # either `0/` or `1/` - return instance_metadata(f"network-interfaces/{idx}ip") - -def _assemble_config( - core: Any, - controller_addr: Optional[Any], - partitions: List[Any], - nodesets: List[Any], - nodesets_dyn: List[Any], - nodesets_tpu: List[Any], - login_groups: List[Any], - ) -> NSDict: - cfg = NSDict(core) - - if cfg.controller_network_attachment: - # lookup controller address - if instance_role() == "controller": - # ignore stored value of `controller_addr`, it will be overwritten during `setup_controller` - cfg.slurm_control_addr = controller_lookup_self_ip() - else: - if not controller_addr: - raise DeffetiveStoredConfigError("controller_addr.yaml not found in bucket") - cfg.slurm_control_addr = controller_addr["slurm_control_addr"] - - # add partition configs - for p_yaml in partitions: - p_cfg = NSDict(p_yaml) - assert p_cfg.get("partition_name"), "partition_name is required" - p_name = p_cfg.partition_name - assert p_name not in cfg.partitions, f"partition {p_name} already defined" - cfg.partitions[p_name] = p_cfg - - # add nodeset configs - ns_names = set() - def _add_nodesets(yamls: List[Any], target: dict): - for ns_yaml in yamls: - ns_cfg = NSDict(ns_yaml) - assert ns_cfg.get("nodeset_name"), "nodeset_name is required" - ns_name = ns_cfg.nodeset_name - assert ns_name not in ns_names, f"nodeset {ns_name} already defined" - target[ns_name] = ns_cfg - ns_names.add(ns_name) - - _add_nodesets(nodesets, cfg.nodeset) - _add_nodesets(nodesets_dyn, cfg.nodeset_dyn) - _add_nodesets(nodesets_tpu, cfg.nodeset_tpu) - - # validate that configs for all referenced nodesets are present - for p in cfg.partitions.values(): - for ns_name in chain(p.partition_nodeset, p.partition_nodeset_dyn, p.partition_nodeset_tpu): - if ns_name not in ns_names: - raise DeffetiveStoredConfigError(f"nodeset {ns_name} not defined in config") - - for lg_yaml in login_groups: - lg_cfg = NSDict(lg_yaml) - assert lg_cfg.get("group_name"), "group_name is required" - lg_name = lg_cfg.group_name - assert lg_name not in cfg.login_groups - cfg.login_groups[lg_name] = lg_cfg - - if instance_role() == "login": - group = instance_login_group() - if group not in cfg.login_groups: - raise DeffetiveStoredConfigError(f"login group '{group}' does not exist in config") - - return _fill_cfg_defaults(cfg) - -def fetch_config() -> Tuple[bool, NSDict]: - """ - Fetches config from bucket and saves it locally - Returns True if new (updated) config was fetched - """ - hash_file = Path("/slurm/scripts/.config.hash") - old_hash = hash_file.read_text() if hash_file.exists() else None - - if should_mount_slurm_bucket() and instance_role() != "controller": - cfg = _fetch_mounted_config() - CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) - chown_slurm(CONFIG_FILE) - return False, cfg - - cfg_and_hash = _fetch_config(old_hash=old_hash) - - if not cfg_and_hash: - return False, _load_config() - - cfg, hash = cfg_and_hash - hash_file.write_text(hash) - chown_slurm(hash_file) - CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) - chown_slurm(CONFIG_FILE) - return True, cfg - -def owned_file_handler(filename): - """create file handler""" - chown_slurm(filename) - return logging.handlers.WatchedFileHandler(filename, delay=True) - -def get_log_path() -> Path: - """ - Returns path to log file for the current script. - e.g. resume.py -> /var/log/slurm/resume.log - """ - cfg_log_dir = lookup().cfg.slurm_log_dir - log_dir = Path(cfg_log_dir) if cfg_log_dir else dirs.log - return (log_dir / Path(sys.argv[0]).name).with_suffix(".log") - -def init_log_and_parse(parser: argparse.ArgumentParser) -> argparse.Namespace: - parser.add_argument( - "--debug", - "-d", - dest="loglevel", - action="store_const", - const=logging.DEBUG, - default=logging.INFO, - help="Enable debugging output", - ) - parser.add_argument( - "--trace-api", - "-t", - action="store_true", - help="Enable detailed api request output", - ) - args = parser.parse_args() - loglevel = args.loglevel - if lookup().cfg.enable_debug_logging: - loglevel = logging.DEBUG - if args.trace_api: - lookup().cfg.extra_logging_flags["trace_api"] = True - # Configure root logger - logging.config.dictConfig({ - "version": 1, - "disable_existing_loggers": True, - "formatters": { - "standard": { - "format": "%(levelname)s: %(message)s", - }, - "stamp": { - "format": "%(asctime)s %(levelname)s: %(message)s", - }, - }, - "handlers": { - "stdout_handler": { - "level": logging.DEBUG, - "formatter": "standard", - "class": "logging.StreamHandler", - "stream": sys.stdout, - }, - "file_handler": { - "()": owned_file_handler, - "level": logging.DEBUG, - "formatter": "stamp", - "filename": get_log_path(), - }, - }, - "root": { - "handlers": ["stdout_handler", "file_handler"], - "level": loglevel, - }, - }) - - sys.excepthook = _handle_exception - - return args - - -def log_api_request(request): - """log.trace info about a compute API request""" - if not lookup().cfg.extra_logging_flags.get("trace_api"): - return - # output the whole request object as pretty yaml - # the body is nested json, so load it as well - rep = json.loads(request.to_json()) - if rep.get("body", None) is not None: - rep["body"] = json.loads(rep["body"]) - pretty_req = yaml.safe_dump(rep).rstrip() - # label log message with the calling function - log.debug(f"{inspect.stack()[1].function}:\n{pretty_req}") - - -def _handle_exception(exc_type, exc_value, exc_trace): - """log exceptions other than KeyboardInterrupt""" - if not issubclass(exc_type, KeyboardInterrupt): - log.exception("Fatal exception", exc_info=(exc_type, exc_value, exc_trace)) - sys.__excepthook__(exc_type, exc_value, exc_trace) - - -def run( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False, - timeout=None, - check=True, - universal_newlines=True, - **kwargs, -): - """Wrapper for subprocess.run() with convenient defaults""" - if isinstance(args, list): - args = list(filter(lambda x: x is not None, args)) - args = " ".join(args) - if not shell and isinstance(args, str): - args = shlex.split(args) - log.debug(f"run: {args}") - try: - result = subprocess.run( - args, - stdout=stdout, - stderr=stderr, - shell=shell, - timeout=timeout, - check=check, - universal_newlines=universal_newlines, - **kwargs, - ) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - log_subprocess(e) - raise - log_subprocess(result) - return result - -def log_subprocess(subj: subprocess.CalledProcessError | subprocess.TimeoutExpired | subprocess.CompletedProcess) -> None: - match subj: - case subprocess.CompletedProcess(returncode=0): - # Do not log successful runs, to not overwhelm logs (e.g. scontrol show jobs --json) - # TODO: consider still doing it in DEBUG or trim output to few KBs. - return - case subprocess.CompletedProcess(): # non-zero returncode - log.error(f"Command '{subj.args}' returned exit status {subj.returncode}.") - case subprocess.CalledProcessError() | subprocess.TimeoutExpired(): - log.error(str(subj)) - - - def normalize(out: None | str | bytes) -> None | str: - """ - Turns stderr and stdout into string: - > A bytes sequence, or a string if run() was called with an encoding, errors, or text=True. None if was not captured. - """ - match out: - case None: - return None - case str(): - return out.strip() - case bytes(): - return out.decode().strip() - case _: - return repr(out) - - if stdout := normalize(subj.stdout): - log.error(f"stdout: {stdout}") - if stderr := normalize(subj.stderr): - log.error(f"stderr: {stderr}") - - -def chown_slurm(path: Path, mode=None) -> None: - if path.exists(): - if mode: - path.chmod(mode) - else: - mkdirp(path.parent) - if mode: - path.touch(mode=mode) - else: - path.touch() - try: - shutil.chown(path, user="slurm", group="slurm") - except LookupError: - log.warning(f"User 'slurm' does not exist. Cannot 'chown slurm:slurm {path}'.") - except PermissionError: - log.warning(f"Not authorized to 'chown slurm:slurm {path}'.") - except Exception as err: - log.error(err) - - -@contextmanager -def cd(path): - """Change working directory for context""" - prev = Path.cwd() - os.chdir(path) - try: - yield - finally: - os.chdir(prev) - - -def cached_property(f): - return property(lru_cache()(f)) - - -def retry(max_retries: int, init_wait_time: float, warn_msg: str, exc_type: Type[Exception]): - """Retries functions that raises the exception exc_type. - Retry time is increased by a factor of two for every iteration. - - Args: - max_retries (int): Maximum number of retries - init_wait_time (float): Initial wait time in secs - warn_msg (str): Message to print during retries - exc_type (Exception): Exception type to check for - """ - - if max_retries <= 0: - raise ValueError("Incorrect value for max_retries, must be >= 1") - if init_wait_time <= 0.0: - raise ValueError("Invalid value for init_wait_time, must be > 0.0") - - def decorator(f): - @wraps(f) - def wrapper(*args, **kwargs): - retry = 0 - secs = init_wait_time - captured_exc: Optional[BaseException] = None - while retry < max_retries: - try: - return f(*args, **kwargs) - except exc_type as e: - captured_exc = e - log.warn(f"{warn_msg}, retrying in {secs}") - sleep(secs) - retry += 1 - secs *= 2 - assert captured_exc - raise captured_exc - - return wrapper - - return decorator - - -def separate(pred: Callable[[Any], bool], coll: Iterable[Any]) -> Tuple[List[Any], List[Any]]: - """filter into 2 lists based on pred returning True or False - returns ([False], [True]) - """ - res: Tuple[List[Any], List[Any]] = ([],[]) - for el in coll: - res[pred(el)].append(el) - return res - - -def chunked(iterable, n=API_REQ_LIMIT): - """group iterator into chunks of max size n""" - it = iter(iterable) - while True: - chunk = list(islice(it, n)) - if not chunk: - return - yield chunk - -def groupby_unsorted(seq: Sequence[Any], key): - indices = defaultdict(list) - for i, el in enumerate(seq): - indices[key(el)].append(i) - for k, idxs in indices.items(): - yield k, (seq[i] for i in idxs) - - -@lru_cache(maxsize=32) -def find_ratio(a, n, s, r0=None): - """given the start (a), count (n), and sum (s), find the ratio required""" - if n == 2: - return s / a - 1 - an = a * n - if n == 1 or s == an: - return 1 - if r0 is None: - # we only need to know which side of 1 to guess, and the iteration will work - r0 = 1.1 if an < s else 0.9 - - # geometric sum formula - def f(r): - return a * (1 - r**n) / (1 - r) - s - - # derivative of f - def df(r): - rm1 = r - 1 - rn = r**n - return (a * (rn * (n * rm1 - r) + r)) / (r * rm1**2) - - MIN_DR = 0.0001 # negligible change - r = r0 - # print(f"r(0)={r0}") - MAX_TRIES = 64 - for i in range(1, MAX_TRIES + 1): - try: - dr = f(r) / df(r) - except ZeroDivisionError: - log.error(f"Failed to find ratio due to zero division! Returning r={r0}") - return r0 - r = r - dr - # print(f"r({i})={r}") - # if the change in r is small, we are close enough - if abs(dr) < MIN_DR: - break - else: - log.error(f"Could not find ratio after {MAX_TRIES}! Returning r={r0}") - return r0 - return r - - -def backoff_delay(start, timeout=None, ratio=None, count: int = 0): - """generates `count` waits starting at `start` - sum of waits is `timeout` or each one is `ratio` bigger than the last - the last wait is always 0""" - # timeout or ratio must be set but not both - assert (timeout is None) ^ (ratio is None) - assert ratio is None or ratio > 0 - assert timeout is None or timeout >= start - assert (count > 1 or timeout is not None) and isinstance(count, int) - assert start > 0 - - if count == 0: - # Equation for auto-count is tuned to have a max of - # ~int(timeout) counts with a start wait of <0.01. - # Increasing start wait decreases count eg. - # backoff_delay(10, timeout=60) -> count = 5 - count = int( - (timeout / ((start + 0.05) ** (1 / 2)) + 2) // math.log(timeout + 2) - ) - - yield start - # if ratio is set: - # timeout = start * (1 - ratio**(count - 1)) / (1 - ratio) - if ratio is None: - ratio = find_ratio(start, count - 1, timeout) - - wait = start - # we have start and 0, so we only need to generate count - 2 - for _ in range(count - 2): - wait *= ratio - yield wait - yield 0 - return - - -ROOT_URL = "http://metadata.google.internal/computeMetadata/v1" - -class MetadataNotFoundError(Exception): - pass - -def get_metadata(path:str, silent=False) -> str: - """Get metadata relative to metadata/computeMetadata/v1""" - HEADERS = {"Metadata-Flavor": "Google"} - url = f"{ROOT_URL}/{path}" - try: - resp = requests_lib.get(url, headers=HEADERS) - resp.raise_for_status() - return resp.text - except requests_lib.exceptions.HTTPError: - if not silent: - log.warning(f"metadata not found ({url})") - raise MetadataNotFoundError(f"failed to get_metadata from {url}") - - -@lru_cache(maxsize=None) -def instance_metadata(path: str, silent:bool=False) -> str: - return get_metadata(f"instance/{path}", silent=silent) - -def instance_role(): - return instance_metadata("attributes/slurm_instance_role") - - -def instance_login_group(): - return instance_metadata("attributes/slurm_login_group") - - -def natural_sort(text): - def atoi(text): - return int(text) if text.isdigit() else text - - return [atoi(w) for w in re.split(r"(\d+)", text)] - - -def to_hostlist(names: Iterable[str]) -> str: - """ - Fast implementation of `hostlist` that doesn't invoke `scontrol` - IMPORTANT: - * Acts as `scontrol show hostlistsorted`, i.e. original order is not preserved - * Achieves worse compression than `scontrol show hostlist` for some cases - """ - pref = defaultdict(list) - tokenizer = re.compile(r"^(.*?)(\d*)$") - for name in filter(None, names): - matches = tokenizer.match(name) - assert matches, name - p, s = matches.groups() - pref[p].append(s) - - def _compress_suffixes(ss: List[str]) -> List[str]: - cur, res = None, [] - - def cur_repr(): - assert cur - nums, strs = cur - if nums[0] == nums[1]: - return strs[0] - return f"{strs[0]}-{strs[1]}" - - for s in sorted(ss, key=int): - n = int(s) - if cur is None: - cur = ((n, n), (s, s)) - continue - - nums, strs = cur - if n == nums[1] + 1: - cur = ((nums[0], n), (strs[0], s)) - else: - res.append(cur_repr()) - cur = ((n, n), (s, s)) - if cur: - res.append(cur_repr()) - return res - - res = [] - for p in sorted(pref.keys()): - sl = defaultdict(list) - for s in pref[p]: - sl[len(s)].append(s) - cs = [] - for ln in sorted(sl.keys()): - if ln == 0: - res.append(p) - else: - cs.extend(_compress_suffixes(sl[ln])) - if not cs: - continue - if len(cs) == 1 and "-" not in cs[0]: - res.append(f"{p}{cs[0]}") - else: - res.append(f"{p}[{','.join(cs)}]") - return ",".join(res) - -@lru_cache(maxsize=None) -def to_hostnames(nodelist: str) -> List[str]: - """make list of hostnames from hostlist expression""" - if not nodelist: - return [] # avoid degenerate invocation of scontrol - if isinstance(nodelist, str): - hostlist = nodelist - else: - hostlist = ",".join(nodelist) - hostnames = run(f"{lookup().scontrol} show hostnames {hostlist}").stdout.splitlines() - return hostnames - - -def retry_exception(exc) -> bool: - """return true for exceptions that should always be retried""" - msg = str(exc) - retry_errors = ( - "Rate Limit Exceeded", - "Quota Exceeded", - "Quota exceeded", - ) - return any(err in msg for err in retry_errors) - - -def ensure_execute(request): - """Handle rate limits and socket time outs""" - - for retry, wait in enumerate(backoff_delay(0.5, timeout=10 * 60, count=20)): - try: - return request.execute() - except googleapiclient.errors.HttpError as e: - if retry_exception(e): - log.error(f"retry:{retry} '{e}'") - sleep(wait) - continue - raise - - except socket.timeout as e: - # socket timed out, try again - log.debug(e) - - except Exception as e: - log.error(e, exc_info=True) - raise - - break - - -def batch_execute(requests, retry_cb=None, log_err=log.error): - """execute list or dict as batch requests - retry if retry_cb returns true - """ - BATCH_LIMIT = 1000 - if not isinstance(requests, dict): - requests = {str(k): v for k, v in enumerate(requests)} # rid generated here - done = {} - failed = {} - timestamps: List[float] = [] - rate_limited = False - - def batch_callback(rid, resp, exc): - nonlocal rate_limited - if exc is not None: - log_err(f"compute request exception {rid}: {exc}") - if retry_exception(exc): - rate_limited = True - else: - req = requests.pop(rid) - failed[rid] = (req, exc) - else: - # if retry_cb is set, don't move to done until it returns false - if retry_cb is None or not retry_cb(resp): - requests.pop(rid) - done[rid] = resp - - def batch_request(reqs): - batch = lookup().compute.new_batch_http_request(callback=batch_callback) - for rid, req in reqs: - batch.add(req, request_id=rid) - return batch - - while requests: - if timestamps: - timestamps = [stamp for stamp in timestamps if stamp > time()] - if rate_limited and timestamps: - stamp = next(iter(timestamps)) - sleep(max(stamp - time(), 0)) - rate_limited = False - # up to API_REQ_LIMIT (2000) requests - # in chunks of up to BATCH_LIMIT (1000) - batches = [ - batch_request(chunk) - for chunk in chunked(islice(requests.items(), API_REQ_LIMIT), BATCH_LIMIT) - ] - timestamps.append(time() + 100) - with ThreadPoolExecutor() as exe: - futures = [] - for batch in batches: - future = exe.submit(ensure_execute, batch) - futures.append(future) - for future in futures: - result = future.exception() - if result is not None: - raise result - - return done, failed - - -def get_operation_req(lkp: "Lookup", name: str, region: Optional[str]=None, zone: Optional[str]=None) -> Any: - if zone: - return lkp.compute.zoneOperations().get(project=lkp.project, zone=zone, operation=name) - elif region: - return lkp.compute.regionOperations().get(project=lkp.project, region=region, operation=name) - return lkp.compute.globalOperations().get(project=lkp.project, operation=name) - -def wait_request(operation, project: str): - """makes the appropriate wait request for a given operation""" - if "zone" in operation: - req = lookup().compute.zoneOperations().wait( - project=project, - zone=trim_self_link(operation["zone"]), - operation=operation["name"], - ) - elif "region" in operation: - req = lookup().compute.regionOperations().wait( - project=project, - region=trim_self_link(operation["region"]), - operation=operation["name"], - ) - else: - req = lookup().compute.globalOperations().wait( - project=project, operation=operation["name"] - ) - return req - - -def wait_for_operation(operation) -> Dict[str, Any]: - """wait for given operation""" - project = parse_self_link(operation["selfLink"]).project - wait_req = wait_request(operation, project=project) - - while True: - result = ensure_execute(wait_req) - if result["status"] == "DONE": - log_errors = " with errors" if "error" in result else "" - log.debug( - f"operation complete{log_errors}: type={result['operationType']}, name={result['name']}" - ) - return result - - - -def getThreadsPerCore(template) -> int: - if not template.machine_type.supports_smt: - return 1 - return template.advancedMachineFeatures.threadsPerCore or 2 - - -@retry( - max_retries=9, - init_wait_time=1, - warn_msg="Temporary failure in name resolution", - exc_type=socket.gaierror, -) -def host_lookup(host_name: str) -> str: - return socket.gethostbyname(host_name) - - -class Dumper(yaml.SafeDumper): - """Add representers for pathlib.Path and NSDict for yaml serialization""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.add_representer(NSDict, self.represent_nsdict) - self.add_multi_representer(Path, self.represent_path) - - @staticmethod - def represent_nsdict(dumper, data): - return dumper.represent_mapping("tag:yaml.org,2002:map", data.items()) - - @staticmethod - def represent_path(dumper, path): - return dumper.represent_scalar("tag:yaml.org,2002:str", str(path)) - - -@dataclass(frozen=True) -class ReservationDetails: - project: str - zone: str - name: str - policies: List[str] # names (not URLs) of resource policies - bulk_insert_name: str # name in format suitable for bulk insert (currently identical to user supplied name in long format) - deployment_type: Optional[str] - reservation_mode: Optional[str] - assured_count: int - delete_at_time: Optional[datetime] - - @property - def dense(self) -> bool: - return self.deployment_type == "DENSE" - - @property - def calendar(self) -> bool: - return self.reservation_mode == "CALENDAR" - -@dataclass(frozen=True) -class FutureReservation: - project: str - zone: str - name: str - specific: bool - start_time: datetime - end_time: datetime - reservation_mode: Optional[str] - active_reservation: Optional[ReservationDetails] - - @property - def calendar(self) -> bool: - return self.reservation_mode == "CALENDAR" - -@dataclass -class Job: - id: int - name: Optional[str] = None - required_nodes: Optional[str] = None - job_state: Optional[str] = None - duration: Optional[timedelta] = None - -@dataclass(frozen=True) -class NodeState: - base: str - flags: frozenset - -class Lookup: - """Wrapper class for cached data access""" - - def __init__(self, cfg): - self._cfg = cfg - - @property - def cfg(self): - return self._cfg - - @property - def project(self): - return self.cfg.project or authentication_project() - - @cached_property - def control_addr(self) -> Optional[str]: - return self.cfg.get("slurm_control_addr", None) - - @property - def control_host(self): - return self.cfg.slurm_control_host - - @cached_property - def control_host_addr(self): - return self.control_addr or host_lookup(self.cfg.slurm_control_host) - - @property - def control_host_port(self): - return self.cfg.slurm_control_host_port - - @property - def endpoint_versions(self): - return self.cfg.endpoint_versions - - @property - def scontrol(self): - return Path(self.cfg.slurm_bin_dir or "") / "scontrol" - - @cached_property - def instance_role(self): - return instance_role() - - @cached_property - def instance_role_safe(self): - try: - role = self.instance_role - except Exception as e: - log.error(e) - role = None - return role - - @property - def is_controller(self): - return self.instance_role_safe == "controller" - - @property - def is_login_node(self): - return self.instance_role_safe == "login" - - @cached_property - def compute(self): - # TODO evaluate when we need to use google_app_cred_path - if self.cfg.google_app_cred_path: - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.cfg.google_app_cred_path - return compute_service() - - @cached_property - def hostname(self): - return socket.gethostname() - - @cached_property - def hostname_fqdn(self): - return socket.getfqdn() - - @cached_property - def zone(self): - return instance_metadata("zone") - - node_desc_regex = re.compile( - r"^(?P(?P[^\s\-]+)-(?P\S+))-(?P(?P\w+)|(?P\[[\d,-]+\]))$" - ) - - @lru_cache(maxsize=None) - def _node_desc(self, node_name): - """Get parts from node name""" - if not node_name: - node_name = self.hostname - # workaround below is for VMs whose hostname is FQDN - node_name_short = node_name.split(".")[0] - m = self.node_desc_regex.match(node_name_short) - if not m: - raise Exception(f"node name {node_name} is not valid") - return m.groupdict() - - def node_prefix(self, node_name=None): - return self._node_desc(node_name)["prefix"] - - def node_index(self, node: str) -> int: - """ node_index("cluster-nodeset-45") == 45 """ - suff = self._node_desc(node)["suffix"] - - if suff is None: - raise ValueError(f"Node {node} name does not end with numeric index") - return int(suff) - - def node_nodeset_name(self, node_name=None): - return self._node_desc(node_name)["nodeset"] - - def node_nodeset(self, node_name=None): - nodeset_name = self.node_nodeset_name(node_name) - if nodeset_name in self.cfg.nodeset_tpu: - return self.cfg.nodeset_tpu[nodeset_name] - - return self.cfg.nodeset[nodeset_name] - - def partition_is_tpu(self, part: str) -> bool: - """check if partition with name part contains a nodeset of type tpu""" - return len(self.cfg.partitions[part].partition_nodeset_tpu) > 0 - - - def node_is_tpu(self, node_name=None): - nodeset_name = self.node_nodeset_name(node_name) - return self.cfg.nodeset_tpu.get(nodeset_name) is not None - - def nodeset_is_tpu(self, nodeset_name=None) -> bool: - return self.cfg.nodeset_tpu.get(nodeset_name) is not None - - def node_is_fr(self, node_name:str) -> bool: - return bool(self.node_nodeset(node_name).future_reservation) - - def is_dormant_res_node(self, node_name:str) -> bool: - fr = self.future_reservation(self.node_nodeset(node_name)) - res = self.nodeset_reservation(self.node_nodeset(node_name)) - - if fr is None and res is None: - return False - - if fr: - return fr.active_reservation is None - - if res: - if res.calendar: - # If reservation is calendar based, check if it is past the delete_at_time - if res.delete_at_time is not None and now() >= res.delete_at_time: - log.debug(f"DWS calendar reservation {res.bulk_insert_name} is past deletion time {res.delete_at_time}, skipping resume.") - return True - - # If assured_count is 0 do not resume nodes as they are not active yet - if res.delete_at_time is not None and res.assured_count <= 0: - log.debug(f"DWS calendar reservation {res.bulk_insert_name} is not active yet, skipping resume.") - return True - - return False - - def node_is_dyn(self, node_name=None) -> bool: - nodeset = self.node_nodeset_name(node_name) - return self.cfg.nodeset_dyn.get(nodeset) is not None - - def node_is_gke(self, node_name=None) -> bool: - return self.nodeset_is_gke(self.node_nodeset(node_name)) - - def nodeset_is_gke(self, nodeset=None) -> bool: - return "gke_nodepool" in nodeset - - def node_template(self, node_name=None) -> str: - """ Self link of nodeset template """ - return self.node_nodeset(node_name).instance_template - - def node_template_info(self, node_name=None): - return self.template_info(self.node_template(node_name)) - - def node_region(self, node_name=None): - nodeset = self.node_nodeset(node_name) - return parse_self_link(nodeset.subnetwork).region - - def nodeset_accelerator_topology(self, nodeset_name: str) -> Optional[str]: - if not self.nodeset_is_tpu(nodeset_name): - return getattr(self.cfg.nodeset[nodeset_name], 'accelerator_topology', None) - return None - - def nodeset_prefix(self, nodeset_name): - return f"{self.cfg.slurm_cluster_name}-{nodeset_name}" - - def nodelist_range(self, nodeset_name: str, start: int, count: int) -> str: - assert 0 <= start and 0 < count - pref = self.nodeset_prefix(nodeset_name) - if count == 1: - return f"{pref}-{start}" - return f"{pref}-[{start}-{start + count - 1}]" - - def static_dynamic_sizes(self, nodeset: NSDict) -> Tuple[int, int]: - return (nodeset.node_count_static or 0, nodeset.node_count_dynamic_max or 0) - - def nodelist(self, nodeset) -> str: - cnt = sum(self.static_dynamic_sizes(nodeset)) - if cnt == 0: - return "" - return self.nodelist_range(nodeset.nodeset_name, 0, cnt) - - def nodenames(self, nodeset) -> Tuple[Iterable[str], Iterable[str]]: - pref = self.nodeset_prefix(nodeset.nodeset_name) - s_count, d_count = self.static_dynamic_sizes(nodeset) - return ( - (f"{pref}-{i}" for i in range(s_count)), - (f"{pref}-{i}" for i in range(s_count, s_count + d_count)), - ) - - def power_managed_nodesets(self) -> Iterable[NSDict]: - return chain(self.cfg.nodeset.values(), self.cfg.nodeset_tpu.values()) - - def is_power_managed_node(self, node_name: str) -> bool: - try: - ns = self.node_nodeset(node_name) - if ns is None: - return False - idx = int(self._node_desc(node_name)["suffix"]) - return idx < sum(self.static_dynamic_sizes(ns)) - except Exception: - return False - - def is_static_node(self, node_name: str) -> bool: - if not self.is_power_managed_node(node_name): - return False - idx = int(self._node_desc(node_name)["suffix"]) - return idx < self.node_nodeset(node_name).node_count_static - - @lru_cache(maxsize=None) - def slurm_nodes(self) -> Dict[str, NodeState]: - def parse_line(node_line) -> Tuple[str, NodeState]: - """turn node,state line to (node, NodeState)""" - # state flags include: CLOUD, COMPLETING, DRAIN, FAIL, POWERED_DOWN, - # POWERING_DOWN - node, fullstate = node_line.split(",") - state = fullstate.split("+") - state_tuple = NodeState(base=state[0], flags=frozenset(state[1:])) - return (node, state_tuple) - - cmd = ( - f"{self.scontrol} show nodes | " - r"grep -oP '^NodeName=\K(\S+)|\s+State=\K(\S+)' | " - r"paste -sd',\n'" - ) - node_lines = run(cmd, shell=True).stdout.rstrip().splitlines() - nodes = { - node: state - for node, state in map(parse_line, node_lines) - if "CLOUD" in state.flags or "DYNAMIC_NORM" in state.flags - } - return nodes - - def node_state(self, nodename: str) -> Optional[NodeState]: - state = self.slurm_nodes().get(nodename) - if state is not None: - return state - - # state is None => Slurm doesn't know this node, - # there are two reasons: - # * happy: - # * node belongs to removed nodeset - # * node belongs to downsized portion of nodeset - # * dynamic node that didn't register itself - # * unhappy: - # * there is a drift in Slurm and SlurmGCP configurations - # * `slurm_nodes` function failed to handle `scontrol show nodes`, - # TODO: make `slurm_nodes` robust by using `scontrol show nodes --json` - # In either of "unhappy" cases it's too dangerous to proceed - abort slurmsync. - try: - ns = self.node_nodeset(nodename) - except: - log.info(f"Unknown node {nodename}, belongs to unknown nodeset") - return None # Can't find nodeset, may be belongs to removed nodeset - - if self.node_is_dyn(nodename): - log.info(f"Unknown node {nodename}, belongs to dynamic nodeset") - return None # we can't make any judjment for dynamic nodes - - cnt = sum(self.static_dynamic_sizes(ns)) - if self.node_index(nodename) >= cnt: - log.info(f"Unknown node {nodename}, out of nodeset size boundaries ({cnt})") - return None # node belongs to downsized nodeset - - raise RuntimeError(f"Slurm does not recognize node {nodename}, potential misconfiguration.") - - - @lru_cache(maxsize=1) - def instances(self) -> Dict[str, Instance]: - instance_information_fields = [ - "creationTimestamp", - "name", - "resourceStatus", - "scheduling", - "status", - "labels.slurm_instance_role", - "zone", - "metadata", - ] - - instance_fields = ",".join(sorted(instance_information_fields)) - fields = f"items.zones.instances({instance_fields}),nextPageToken" - flt = f"labels.slurm_cluster_name={self.cfg.slurm_cluster_name} AND name:{self.cfg.slurm_cluster_name}-*" - act = self.compute.instances() - op = act.aggregatedList(project=self.project, fields=fields, filter=flt) - - instances = {} - while op is not None: - result = ensure_execute(op) - for zone in result.get("items", {}).values(): - for jo in zone.get("instances", []): - inst = Instance.from_json(jo) - if inst.name in instances: - log.error(f"Duplicate VM name {inst.name} across multiple zones") - instances[inst.name] = inst - op = act.aggregatedList_next(op, result) - return instances - - def instance(self, instance_name: str) -> Optional[Instance]: - return self.instances().get(instance_name) - - @lru_cache() - def _get_reservation(self, project: str, zone: str, name: str) -> Any: - """See https://cloud.google.com/compute/docs/reference/rest/v1/reservations""" - return self.compute.reservations().get( - project=project, zone=zone, reservation=name).execute() - - @lru_cache() - def get_mig(self, project: str, region: str, self_link:str) -> Any: - """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" - return self.compute.regionInstanceGroupManagers().get(project=project, region=region, instanceGroupManager=self_link).execute() - - @lru_cache - def get_mig_instances(self, project: str, region: str, self_link:str) -> Any: - return self.compute.regionInstanceGroupManagers().listManagedInstances(project=project, region=region, instanceGroupManager=self_link).execute() - - @lru_cache() - def get_mig_list(self, project: str, region: str) -> Any: - """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" - return self.compute.regionInstanceGroupManagers().list(project=project, region=region).execute() - - @lru_cache() - def _get_future_reservation(self, project:str, zone:str, name: str) -> Any: - """See https://cloud.google.com/compute/docs/reference/rest/v1/futureReservations""" - return self.compute.futureReservations().get(project=project, zone=zone, futureReservation=name).execute() - - def get_reservation_details(self, project:str, zone:str, name:str, bulk_insert_name:str) -> ReservationDetails: - reservation = self._get_reservation(project, zone, name) - - # Converts policy URLs to names, e.g.: - # projects/111111/regions/us-central1/resourcePolicies/zebra -> zebra - policies = [u.split("/")[-1] for u in reservation.get("resourcePolicies", {}).values()] - - return ReservationDetails( - project=project, - zone=zone, - name=name, - policies=policies, - deployment_type=reservation.get("deploymentType"), - reservation_mode=reservation.get("reservationMode"), - assured_count=int(reservation.get("specificReservation", {}).get("assuredCount", 0)), - delete_at_time=parse_gcp_timestamp(reservation.get("deleteAtTime")) if reservation.get("deleteAtTime") else None, - bulk_insert_name=bulk_insert_name) - - def nodeset_reservation(self, nodeset: NSDict) -> Optional[ReservationDetails]: - if not nodeset.reservation_name: - return None - - zones = list(nodeset.zone_policy_allow or []) - assert len(zones) == 1, "Only single zone is supported if using a reservation" - zone = zones[0] - - regex = re.compile(r'^projects/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$') - if not (match := regex.match(nodeset.reservation_name)): - raise ValueError( - f"Invalid reservation name: '{nodeset.reservation_name}', expected format is 'projects/PROJECT/reservations/NAME'" - ) - - project, name = match.group("project", "reservation") - return self.get_reservation_details(project, zone, name, nodeset.reservation_name) - - def future_reservation(self, nodeset: NSDict) -> Optional[FutureReservation]: - if not nodeset.future_reservation: - return None - - active_reservation = None - match = re.search(r'^projects/(?P[^/]+)/zones/(?P[^/]+)/futureReservations/(?P[^/]+)(/.*)?$', nodeset.future_reservation) - assert match, f"Invalid future reservation name '{nodeset.future_reservation}'" - project, zone, name = match.group("project","zone","name") - fr = self._get_future_reservation(project,zone,name) - - start_time = parse_gcp_timestamp(fr["timeWindow"]["startTime"]) - end_time = parse_gcp_timestamp(fr["timeWindow"]["endTime"]) - - if "autoCreatedReservations" in fr["status"] and (res:=fr["status"]["autoCreatedReservations"][0]): - if start_time <= now() <=end_time: - match = re.search(r'projects/(?P[^/]+)/zones/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$',res) - assert match, f"Unexpected reservation name '{res}'" - res_name = match.group("name") - bulk_insert_name = f"projects/{project}/reservations/{res_name}" - active_reservation = self.get_reservation_details(project, zone, res_name, bulk_insert_name) - - return FutureReservation( - project=project, - zone=zone, - name=name, - specific=fr["specificReservationRequired"], - start_time=start_time, - end_time=end_time, - reservation_mode=fr.get("reservationMode"), - active_reservation=active_reservation - ) - - @lru_cache(maxsize=1) - def machine_types(self): - field_names = "name,zone,guestCpus,memoryMb,accelerators" - fields = f"items.zones.machineTypes({field_names}),nextPageToken" - - machines: Dict[str, Dict[str, Any]] = defaultdict(dict) - act = self.compute.machineTypes() - op = act.aggregatedList(project=self.project, fields=fields) - while op is not None: - result = ensure_execute(op) - machine_iter = chain.from_iterable( - scope.get("machineTypes", []) for scope in result["items"].values() - ) - for machine in machine_iter: - name = machine["name"] - zone = machine["zone"] - machines[name][zone] = machine - - op = act.aggregatedList_next(op, result) - return machines - - def machine_type(self, name: str) -> MachineType: - custom_patt = re.compile( - r"((?P\w+)-)?custom-(?P\d+)-(?P\d+)" - ) - if match := custom_patt.match(name): - return MachineType( - name=name, - guest_cpus=int(match.group("cpus")), - memory_mb=int(match.group("mem")), - accelerators=[], - ) - - machines = self.machine_types() - if name not in machines: - raise Exception(f"machine type {name} not found") - per_zone = machines[name] - assert per_zone - return MachineType.from_json( - next(iter(per_zone.values())) # pick the first/any zone - ) - - def template_machine_conf(self, template_link): - template = self.template_info(template_link) - machine = template.machine_type - - machine_conf = NSDict() - machine_conf.boards = 1 # No information, assume 1 - machine_conf.sockets = machine.sockets - # the value below for SocketsPerBoard must be type int - machine_conf.sockets_per_board = machine_conf.sockets // machine_conf.boards - machine_conf.threads_per_core = 1 - _div = 2 if getThreadsPerCore(template) == 1 else 1 - machine_conf.cpus = ( - int(machine.guest_cpus / _div) if machine.supports_smt else machine.guest_cpus - ) - machine_conf.cores_per_socket = int(machine_conf.cpus / machine_conf.sockets) - # Because the actual memory on the host will be different than - # what is configured (e.g. kernel will take it). From - # experiments, about 16 MB per GB are used (plus about 400 MB - # buffer for the first couple of GB's. Using 30 MB to be safe. - gb = machine.memory_mb // 1024 - machine_conf.memory = machine.memory_mb - (400 + (30 * gb)) - return machine_conf - - @lru_cache(maxsize=None) - def template_info(self, template_link): - template_name = trim_self_link(template_link) - cache = file_cache.cache("template_cache") - - if cached := cache.get(template_name): - return NSDict(cached) - - region = get_self_link_component(template_link, "regions") - - template = ensure_execute( - self.compute.instanceTemplates().get( - project=self.project, instanceTemplate=template_name - ) if region is None else - self.compute.regionInstanceTemplates().get( - project=self.project, region=region, instanceTemplate=template_name - ) - ).get("properties") - template = NSDict(template) - # name and link are not in properties, so stick them in - template.name = template_name - template.link = template_link - template.machine_type = self.machine_type(template.machineType) - # TODO delete metadata to reduce memory footprint? - # del template.metadata - - template.gpu = get_template_gpu(template) - - cache.set(template_name, template.to_dict()) - return template - - def _parse_job_info(self, job_info: str) -> Job: - """Extract job details""" - if match:= re.search(r"JobId=(\d+)", job_info): - job_id = int(match.group(1)) - else: - raise ValueError(f"Job ID not found in the job info: {job_info}") - - if match:= re.search(r"TimeLimit=(?:(\d+)-)?(\d{2}):(\d{2}):(\d{2})", job_info): - days, hours, minutes, seconds = match.groups() - duration = timedelta( - days=int(days) if days else 0, - hours=int(hours), - minutes=int(minutes), - seconds=int(seconds) - ) - else: - duration = None - - if match := re.search(r"JobName=([^\n]+)", job_info): - name = match.group(1) - else: - name = None - - if match := re.search(r"JobState=(\w+)", job_info): - job_state = match.group(1) - else: - job_state = None - - if match := re.search(r"ReqNodeList=([^ ]+)", job_info): - required_nodes = match.group(1) - else: - required_nodes = None - - return Job(id=job_id, duration=duration, name=name, job_state=job_state, required_nodes=required_nodes) - - @lru_cache - def get_jobs(self) -> List[Job]: - res = run(f"{self.scontrol} show jobs", timeout=30) - - return [self._parse_job_info(job) for job in res.stdout.split("\n\n")[:-1]] - - @lru_cache - def job(self, job_id: int) -> Optional[Job]: - job_info = run(f"{self.scontrol} show jobid {job_id}", check=False).stdout.rstrip() - if not job_info: - return None - - return self._parse_job_info(job_info=job_info) - - @property - def etc_dir(self) -> Path: - return Path(self.cfg.output_dir or slurmdirs.etc) - - def controller_mount_server_ip(self) -> str: - return self.control_addr or self.control_host - - def normalize_ns_mount(self, ns: Union[dict, NSMount]) -> NSMount: - if isinstance(ns, NSMount): - return ns - - server_ip = ns.get("server_ip") or "$controller" - if server_ip == "$controller": - server_ip = self.controller_mount_server_ip() - - return NSMount( - server_ip=server_ip, - local_mount=Path(ns["local_mount"]), - remote_mount=Path(ns["remote_mount"]), - fs_type=ns["fs_type"], - mount_options=ns["mount_options"], - ) - - @property - def munge_mount(self) -> NSMount: - if self.cfg.munge_mount: - mnt = self.cfg.munge_mount - mnt.local_mount = mnt.local_mount or "/mnt/munge" - return self.normalize_ns_mount(mnt) - else: - return NSMount( - server_ip=self.controller_mount_server_ip(), - local_mount=Path("/mnt/munge"), - remote_mount=dirs.munge, - fs_type="nfs", - mount_options="defaults,hard,intr,_netdev", - ) - - @property - def slurm_key_mount(self) -> NSMount: - if self.cfg.slurm_key_mount: - mnt = self.cfg.slurm_key_mount - mnt.local_mount = mnt.local_mount or slurmdirs.key_distribution - return self.normalize_ns_mount(mnt) - else: - return NSMount( - server_ip=self.controller_mount_server_ip(), - local_mount=slurmdirs.key_distribution, - remote_mount=slurmdirs.key_distribution, - fs_type="nfs", - mount_options="defaults,hard,intr,_netdev", - ) - - def is_flex_node(self, node: str) -> bool: - try: - nodeset = self.node_nodeset(node) - if nodeset.dws_flex.use_bulk_insert: - return False #For legacy flex support - return bool(nodeset.dws_flex.enabled) - except: - return False - - def is_provisioning_flex_node(self, node:str) -> bool: - if not self.is_flex_node(node): - return False - if self.instance(node) is not None: - return True - - nodeset = self.node_nodeset(node) - zones = nodeset.zone_policy_allow - assert len(zones) > 0 - region = self.node_region(node) - - potential_migs=[] - mig_list=self.get_mig_list(self.project, region) - - if not mig_list or not mig_list.get("items"): - return False - - for mig in mig_list["items"]: - if not mig.get("instanceTemplate"): #possibly an old MIG - return False - if mig["instanceTemplate"] == self.node_template(node) and mig["currentActions"]["creating"] > 0: - potential_migs.append(self.get_mig_instances(self.project, region, trim_self_link(mig["selfLink"]))) - - if not potential_migs: - return False - - for instance_collection in potential_migs[0]["managedInstances"]: - if node in instance_collection["name"] and instance_collection["currentAction"]=="CREATING": - return True - return False - - def cluster_regions(self) -> list[str]: - """ - Returns all regions used in cluster - NOTE: only concerned with normal nodesets, - neither TPU, nor dynamic, nor login node, nor controller node are considered - """ - res = set() - for nodeset in self.cfg.nodeset.values(): - res.add(parse_self_link(nodeset.subnetwork).region) - return list(res) - - - -_lkp: Optional[Lookup] = None - -def _load_config() -> NSDict: - return NSDict(yaml.safe_load(CONFIG_FILE.read_text())) - -def lookup() -> Lookup: - global _lkp - if _lkp is None: - try: - cfg = _load_config() - except FileNotFoundError: - log.error(f"config file not found: {CONFIG_FILE}") - cfg = NSDict() # TODO: fail here, once all code paths are covered (mainly init_logging) - _lkp = Lookup(cfg) - return _lkp - -def update_config(cfg: NSDict) -> None: - global _lkp - _lkp = Lookup(cfg) - -def scontrol_reconfigure(lkp: Lookup) -> None: - log.info("Running systemctl restart slurmctld.service") - run("sudo systemctl restart slurmctld.service", timeout=30) - log.info("Running scontrol reconfigure") - run(f"{lkp.scontrol} reconfigure") diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py deleted file mode 100644 index d1d77a1833..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any - - -from dataclasses import dataclass, asdict -import util -import local_pubsub - -import logging -log = logging.getLogger() - -# Name of the topic -TOPIC = "watch_delete_vm_op" - -@dataclass(frozen=True) -class WatchDeleteVmOp_Message: - op_name: str - zone: str - node: str - -class WatchDeleteVmOp_Topic: - def __init__(self, topic: local_pubsub.Topic) -> None: - self._t = topic - - def publish(self, op: dict[str, Any], node: str) -> None: - assert op.get("operationType") == "delete" - assert op.get("zone") - assert node - - msg = WatchDeleteVmOp_Message(op_name=op["name"], zone=op["zone"], node=node) - self._t.publish(data=asdict(msg)) - - -def watch_delete_vm_op_topic() -> WatchDeleteVmOp_Topic: - return WatchDeleteVmOp_Topic(local_pubsub.topic(TOPIC)) - - -def _watch_op(lkp: util.Lookup, m: WatchDeleteVmOp_Message) -> bool: - """ - Processes VM delete-operation. - If operation is still running - do nothing - If operation failed - log error & remove op from watch list - If operation is done - remove op from watch list do nothing - - To avoid querying status for each op individually, use list of VM instances as - a source of data. Don't query op for instance X if instance X is not present - (presumably deleted). - NOTE: This optimization can lead to false-positives - - absence of error-logs in case op failed, but VM got deleted by other means. - - Returns True if message should be marked as processed (ack). - """ - - inst = lkp.instance(m.node) - - if not inst: - log.debug(f"Stop watching op {m.op_name}, VM {m.node} appears to be deleted") - return True # ack, potentially false-positive - - if inst.status == "TERMINATED": - log.debug(f"Stop watching op {m.op_name}, VM {m.node} is TERMINATED") - return True # ack, potentially false-positive - - if inst.status == "STOPPING": - log.debug(f"Skipping op {m.op_name}, VM {m.node} is STOPPING") - return False # try later - - try: - op = util.get_operation_req(lkp, m.op_name, zone=m.zone).execute() - except: - # TODO: consider less conservative handling, but be careful not to cause deadlettering. - log.exception(f"Failed to get operation {m.op_name}, will not retry") - return True # ack (remove) - - if op["status"] != "DONE": - log.debug(f"Watching op {m.op_name} is still not done ({op['status']})") - return False # try later - - if "error" in op: - log.error(f"Operation {m.op_name} to delete {m.node} finished with error: {op['error']}") - else: - log.debug(f"Operation {m.op_name} to delete {m.node} successfully finished") - return True # ack - - -def watch_vm_delete_ops(lkp: util.Lookup) -> None: - sub = local_pubsub.subscription(TOPIC) - - # Pull once instead of "pulling until empty", motivation: - # Bulk of cases processed by `_watch_op` relies on freshness of `lkp.instances`, - # `lkp.instances` are fetched once during run of `slurmsync`. - # Therefore we shouldn't try to re-process messages that has been already NACKed in this run, - # since they will be handled with the same `lkp.instance` as a previous attempt. - msgs = sub.pull(max_messages=1000) # 1000 is arbitrary number to be adjusted if needed. - log.debug(f"Processing {len(msgs)} delete VM operations") - # TODO: handle messages in butches to improve latency - for m in msgs: - try: - dm = WatchDeleteVmOp_Message(**m.data) - ack = _watch_op(lkp, dm) - except Exception: - log.exception(f"Failed to process the message {m.id}, removing") - ack = True - if ack: - sub.ack([m.id]) - else: - sub.modify_ack_deadline([m.id], deadline=0) # NACK - - - - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf deleted file mode 100644 index 71905a0342..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf +++ /dev/null @@ -1,504 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "bucket_name" { - description = <<-EOD - Name of GCS bucket to use. - EOD - type = string -} - -variable "bucket_dir" { - description = "Bucket directory for cluster files to be put into." - type = string - default = null -} - -variable "enable_debug_logging" { - type = bool - description = "Enables debug logging mode. Not for production use." - default = false -} - -variable "extra_logging_flags" { - type = map(bool) - description = "The only available flag is `trace_api`" - default = {} -} - -variable "project_id" { - description = "The GCP project ID." - type = string -} - -variable "enable_slurm_auth" { - description = < x... } - nodeset_map = { for k, vs in local.nodeset_map_ell : k => vs[0] } - - nodeset_tpu_map_ell = { for x in var.nodeset_tpu : x.nodeset_name => x... } - nodeset_tpu_map = { for k, vs in local.nodeset_tpu_map_ell : k => vs[0] } - - nodeset_dyn_map_ell = { for x in var.nodeset_dyn : x.nodeset_name => x... } - nodeset_dyn_map = { for k, vs in local.nodeset_dyn_map_ell : k => vs[0] } - - - no_reservation_affinity = { type : "NO_RESERVATION" } -} - -# NODESET -module "slurm_nodeset_template" { - source = "../../internal/slurm-gcp/instance_template" - for_each = local.nodeset_map - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - slurm_instance_role = "compute" - slurm_bucket_path = module.slurm_files.slurm_bucket_path - - additional_disks = each.value.additional_disks - bandwidth_tier = each.value.bandwidth_tier - can_ip_forward = each.value.can_ip_forward - advanced_machine_features = each.value.advanced_machine_features - disk_auto_delete = each.value.disk_auto_delete - disk_labels = each.value.disk_labels - disk_resource_manager_tags = each.value.disk_resource_manager_tags - disk_size_gb = each.value.disk_size_gb - disk_type = each.value.disk_type - enable_confidential_vm = each.value.enable_confidential_vm - enable_oslogin = each.value.enable_oslogin - enable_shielded_vm = each.value.enable_shielded_vm - gpu = each.value.gpu - labels = merge(each.value.labels, { slurm_nodeset = each.value.nodeset_name }) - machine_type = each.value.machine_type - metadata = merge(each.value.metadata, local.universe_domain) - min_cpu_platform = each.value.min_cpu_platform - name_prefix = each.value.nodeset_name - on_host_maintenance = each.value.on_host_maintenance - preemptible = each.value.preemptible - region = each.value.region - resource_manager_tags = each.value.resource_manager_tags - spot = each.value.spot - termination_action = each.value.termination_action - service_account = each.value.service_account - shielded_instance_config = each.value.shielded_instance_config - source_image_family = each.value.source_image_family - source_image_project = each.value.source_image_project - source_image = each.value.source_image - subnetwork = each.value.subnetwork_self_link - additional_networks = each.value.additional_networks - access_config = each.value.access_config - tags = concat([local.slurm_cluster_name], each.value.tags) - - max_run_duration = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? each.value.dws_flex.max_run_duration : null - provisioning_model = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? "FLEX_START" : null - reservation_affinity = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? local.no_reservation_affinity : null -} - -module "nodeset_cleanup" { - source = "./modules/cleanup_compute" - for_each = local.nodeset_map - - nodeset = each.value - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - enable_cleanup_compute = var.enable_cleanup_compute - universe_domain = var.universe_domain - endpoint_versions = var.endpoint_versions - gcloud_path_override = var.gcloud_path_override - nodeset_template = module.slurm_nodeset_template[each.value.nodeset_name].self_link -} - -locals { - nodesets = [for name, ns in local.nodeset_map : { - nodeset_name = ns.nodeset_name - node_conf = ns.node_conf - dws_flex = ns.dws_flex - instance_template = module.slurm_nodeset_template[ns.nodeset_name].self_link - node_count_dynamic_max = ns.node_count_dynamic_max - node_count_static = ns.node_count_static - subnetwork = ns.subnetwork_self_link - reservation_name = ns.reservation_name - future_reservation = ns.future_reservation - maintenance_interval = ns.maintenance_interval - instance_properties_json = ns.instance_properties_json - enable_placement = ns.enable_placement - placement_max_distance = ns.placement_max_distance - network_storage = ns.network_storage - zone_target_shape = ns.zone_target_shape - zone_policy_allow = ns.zone_policy_allow - zone_policy_deny = ns.zone_policy_deny - enable_maintenance_reservation = ns.enable_maintenance_reservation - enable_opportunistic_maintenance = ns.enable_opportunistic_maintenance - accelerator_topology = ns.accelerator_topology - }] -} - -# NODESET TPU -module "slurm_nodeset_tpu" { - source = "../../internal/slurm-gcp/nodeset_tpu" - for_each = local.nodeset_tpu_map - - project_id = var.project_id - node_count_dynamic_max = each.value.node_count_dynamic_max - node_count_static = each.value.node_count_static - nodeset_name = each.value.nodeset_name - zone = each.value.zone - node_type = each.value.node_type - accelerator_config = each.value.accelerator_config - tf_version = each.value.tf_version - preemptible = each.value.preemptible - preserve_tpu = each.value.preserve_tpu - enable_public_ip = each.value.enable_public_ip - service_account = each.value.service_account - data_disks = each.value.data_disks - docker_image = each.value.docker_image - subnetwork = each.value.subnetwork -} - -module "nodeset_cleanup_tpu" { - source = "./modules/cleanup_tpu" - for_each = local.nodeset_tpu_map - - nodeset = { - nodeset_name = each.value.nodeset_name - zone = each.value.zone - } - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - enable_cleanup_compute = var.enable_cleanup_compute - universe_domain = var.universe_domain - endpoint_versions = var.endpoint_versions - gcloud_path_override = var.gcloud_path_override - - depends_on = [ - # Depend on controller network, as a best effort to avoid - # subnetwork resourceInUseByAnotherResource error - var.subnetwork_self_link - ] -} - -resource "google_storage_bucket_object" "parition_config" { - for_each = { for p in var.partitions : p.partition_name => p } - - bucket = module.slurm_files.bucket_name - name = "${module.slurm_files.bucket_dir}/partition_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -moved { - from = module.slurm_files.google_storage_bucket_object.parition_config - to = google_storage_bucket_object.parition_config -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf deleted file mode 100644 index 218c36e392..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# BUCKET - -locals { - synt_suffix = substr(md5("${local.controller_project_id}${var.deployment_name}"), 0, 5) - synth_bucket_name = "${local.slurm_cluster_name}${local.synt_suffix}" - - bucket_name = var.create_bucket ? module.bucket[0].name : var.bucket_name -} - -module "bucket" { - source = "terraform-google-modules/cloud-storage/google" - version = ">= 6.1" - - count = var.create_bucket ? 1 : 0 - - location = var.region - names = [local.synth_bucket_name] - prefix = "slurm" - project_id = local.controller_project_id - - force_destroy = { - (local.synth_bucket_name) = true - } - - labels = merge(local.labels, { - slurm_cluster_name = local.slurm_cluster_name - }) -} - -# BUCKET IAMs -locals { - compute_sa = toset(flatten([for x in module.slurm_nodeset_template : x.service_account])) - compute_tpu_sa = toset(flatten([for x in module.slurm_nodeset_tpu : x.service_account])) - login_sa = toset(flatten([for x in module.login : x.service_account])) - - viewers = toset(flatten([ - "serviceAccount:${module.slurm_controller_template.service_account.email}", - formatlist("serviceAccount:%s", [for x in local.compute_sa : x.email]), - formatlist("serviceAccount:%s", [for x in local.compute_tpu_sa : x.email if x.email != null]), - formatlist("serviceAccount:%s", [for x in local.login_sa : x.email]), - ])) -} - - -resource "google_storage_bucket_iam_member" "viewers" { - for_each = local.viewers - bucket = local.bucket_name - role = "roles/storage.objectViewer" - member = each.value -} - -resource "google_storage_bucket_iam_member" "legacy_readers" { - for_each = local.viewers - bucket = local.bucket_name - role = "roles/storage.legacyBucketReader" - member = each.value -} - -locals { - daos_ns = [ - for ns in var.network_storage : - ns if ns.fs_type == "daos" - ] - - daos_client_install_runners = [ - for ns in local.daos_ns : - ns.client_install_runner if ns.client_install_runner != null - ] - - daos_mount_runners = [ - for ns in local.daos_ns : - ns.mount_runner if ns.mount_runner != null - ] - - daos_network_storage_runners = concat( - local.daos_client_install_runners, - local.daos_mount_runners, - ) - - daos_install_mount_script = { - filename = "ghpc_daos_mount.sh" - content = length(local.daos_ns) > 0 ? module.daos_network_storage_scripts[0].startup_script : "" - } - - common_scripts = length(local.daos_ns) > 0 ? [local.daos_install_mount_script] : [] -} - -# SLURM FILES -locals { - ghpc_startup_script_controller = concat( - local.common_scripts, - [{ - filename = "ghpc_startup.sh" - content = var.controller_startup_script - }]) - - controller_state_disk = { - device_name : try(google_compute_disk.controller_disk[0].name, null) - } - - - nodeset_startup_scripts = { for k, v in local.nodeset_map : k => concat(local.common_scripts, v.startup_script) } -} - -module "daos_network_storage_scripts" { - count = length(local.daos_ns) > 0 ? 1 : 0 - - source = "../../../../modules/scripts/startup-script" - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.daos_network_storage_runners -} - -module "slurm_files" { - source = "./modules/slurm_files" - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - bucket_dir = var.bucket_dir - bucket_name = local.bucket_name - controller_network_attachment = var.controller_network_attachment - - slurmdbd_conf_tpl = var.slurmdbd_conf_tpl - slurm_conf_tpl = var.slurm_conf_tpl - slurm_conf_template = var.slurm_conf_template - cgroup_conf_tpl = var.cgroup_conf_tpl - cloud_parameters = var.cloud_parameters - cloudsql_secret = try( - one(google_secret_manager_secret_version.cloudsql_version[*].id), - null) - - controller_startup_scripts = local.ghpc_startup_script_controller - controller_startup_scripts_timeout = var.controller_startup_scripts_timeout - nodeset_startup_scripts = local.nodeset_startup_scripts - compute_startup_scripts_timeout = var.compute_startup_scripts_timeout - controller_state_disk = local.controller_state_disk - - enable_debug_logging = var.enable_debug_logging - extra_logging_flags = var.extra_logging_flags - - enable_slurm_auth = var.enable_slurm_auth - - enable_bigquery_load = var.enable_bigquery_load - enable_external_prolog_epilog = var.enable_external_prolog_epilog - enable_chs_gpu_health_check_prolog = var.enable_chs_gpu_health_check_prolog - enable_chs_gpu_health_check_epilog = var.enable_chs_gpu_health_check_epilog - epilog_scripts = var.epilog_scripts - prolog_scripts = var.prolog_scripts - task_epilog_scripts = var.task_epilog_scripts - task_prolog_scripts = var.task_prolog_scripts - - disable_default_mounts = !var.enable_default_mounts - network_storage = [ - for storage in var.network_storage : { - server_ip = storage.server_ip, - remote_mount = storage.remote_mount, - local_mount = storage.local_mount, - fs_type = storage.fs_type, - mount_options = storage.mount_options - } - if storage.fs_type != "daos" - ] - - nodeset = local.nodesets - nodeset_dyn = values(local.nodeset_dyn_map) - # Use legacy format for now - nodeset_tpu = values(module.slurm_nodeset_tpu)[*] - - - depends_on = [module.bucket] - - # Providers - endpoint_versions = var.endpoint_versions -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf deleted file mode 100644 index 85ad10fa21..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf +++ /dev/null @@ -1,814 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########### -# GENERAL # -########### - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "deployment_name" { - description = "Name of the deployment." - type = string -} - -variable "slurm_cluster_name" { - type = string - description = <<-EOD - Cluster name, used for resource naming and slurm accounting. - If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). - EOD - default = null - - validation { - condition = var.slurm_cluster_name == null || can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) - error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." - } -} - -variable "region" { - type = string - description = "The default region to place resources in." -} - -variable "zone" { - type = string - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 

list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disable\_login\_public\_ips](#input\_disable\_login\_public\_ips) | DEPRECATED: Use `enable_login_public_ips` instead. | `bool` | `null` | no | -| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_login\_public\_ips](#input\_enable\_login\_public\_ips) | If set to true. The login node will have a random public IP assigned to it. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for login nodes. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Unique name prefix for login nodes. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all login groups. | `string` | n/a | yes | -| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the login instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the login instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [login\_nodes](#output\_login\_nodes) | Slurm login instance definition. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf deleted file mode 100644 index 6ebe5902dc..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-login", ghpc_role = "scheduler" }) -} - -module "instance_validation" { - source = "../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata - ) - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.disk_resource_manager_tags - } - ] - - public_access_config = [{ nat_ip = null, network_tier = null }] - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - # lower, replace `_` with `-`, and remove any non-alphanumeric characters - group_name = replace( - replace( - lower(var.name_prefix), - "_", "-"), - "/[^-a-z0-9]/", "") - - - login_node = { - group_name = local.group_name - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(var.disk_labels, local.labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - additional_networks = var.additional_networks - - can_ip_forward = var.can_ip_forward - advanced_machine_features = var.advanced_machine_features - - enable_confidential_vm = var.enable_confidential_vm - access_config = var.enable_login_public_ips ? local.public_access_config : [] - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - gpu = one(local.guest_accelerator) - labels = local.labels - machine_type = var.machine_type - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - num_instances = var.num_instances - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - region = var.region - resource_manager_tags = var.resource_manager_tags - zone = var.zone - - service_account = local.service_account - - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - static_ips = var.static_ips - bandwidth_tier = var.bandwidth_tier - - subnetwork = var.subnetwork_self_link - tags = var.tags - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml deleted file mode 100644 index 47f003258e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name_prefix - has_to_be_used: true diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf deleted file mode 100644 index e700542794..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "login_nodes" { - description = "Slurm login instance definition." - value = [local.login_node] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf deleted file mode 100644 index 7c1a2e06b5..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf +++ /dev/null @@ -1,419 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -variable "project_id" { # tflint-ignore: terraform_unused_declarations - type = string - description = "Project ID to create resources in." -} - -variable "region" { - type = string - description = "Region where the instances should be created." - default = null -} - -variable "zone" { - type = string - description = <<-EOD - Zone where the instances should be created. If not specified, instances will be - spread across available zones in the region. - EOD - default = null -} - -variable "name_prefix" { - type = string - description = <<-EOD - Unique name prefix for login nodes. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all login groups. - EOD -} - -variable "num_instances" { - type = number - description = "Number of instances to create. This value is ignored if static_ips is provided." - default = 1 -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} -} - -variable "disk_type" { - type = string - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - default = "pd-ssd" -} - -variable "disk_size_gb" { - type = number - description = "Boot disk size in GB." - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - type = list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string)) - auto_delete = optional(bool) - boot = optional(bool) - disk_resource_manager_tags = optional(map(string)) - })) - description = "List of maps of disks." - default = [] -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })) - nullable = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "disable_smt" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - type = bool - default = null - validation { - condition = var.disable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "static_ips" { - type = list(string) - description = "List of static IPs for VM instances." - default = [] -} - -variable "bandwidth_tier" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 6.16 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.16 | -| [helm](#provider\_helm) | ~> 2.17 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.cert_manager](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.prometheus](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.slurm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.slurm_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cert\_manager\_chart\_version](#input\_cert\_manager\_chart\_version) | Version of the Cert Manager chart to install. | `string` | `"v1.18.2"` | no | -| [cert\_manager\_values](#input\_cert\_manager\_values) | Value overrides for the Cert Manager release | `any` | 
{
  "crds": {
    "enabled": true
  }
}
| no | -| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | -| [install\_kube\_prometheus\_stack](#input\_install\_kube\_prometheus\_stack) | Install the Kube Prometheus Stack. | `bool` | `false` | no | -| [install\_slurm\_chart](#input\_install\_slurm\_chart) | Install slurm-operator chart. | `bool` | `true` | no | -| [install\_slurm\_operator\_chart](#input\_install\_slurm\_operator\_chart) | Install slurm-operator chart. | `bool` | `true` | no | -| [node\_pool\_names](#input\_node\_pool\_names) | Names of node pools, for use in node affinities (Slinky system components). | `list(string)` | `null` | no | -| [project\_id](#input\_project\_id) | The project ID that hosts the GKE cluster. | `string` | n/a | yes | -| [prometheus\_chart\_version](#input\_prometheus\_chart\_version) | Version of the Kube Prometheus Stack chart to install. | `string` | `"77.0.1"` | no | -| [prometheus\_values](#input\_prometheus\_values) | Value overrides for the Prometheus release | `any` | 
{
  "installCRDs": true
}
| no | -| [slurm\_chart\_version](#input\_slurm\_chart\_version) | Version of the Slurm chart to install. | `string` | `"0.3.1"` | no | -| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | -| [slurm\_operator\_chart\_version](#input\_slurm\_operator\_chart\_version) | Version of the Slurm Operator chart to install. | `string` | `"0.3.1"` | no | -| [slurm\_operator\_namespace](#input\_slurm\_operator\_namespace) | slurm namespace for charts | `string` | `"slinky"` | no | -| [slurm\_operator\_repository](#input\_slurm\_operator\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | -| [slurm\_operator\_values](#input\_slurm\_operator\_values) | Value overrides for the Slinky release | `any` | `{}` | no | -| [slurm\_repository](#input\_slurm\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | -| [slurm\_values](#input\_slurm\_values) | Value overrides for the Slurm release | `any` | `{}` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [slurm\_namespace](#output\_slurm\_namespace) | namespace for the slurm chart | -| [slurm\_operator\_namespace](#output\_slurm\_operator\_namespace) | namespace for the slinky operator chart | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/main.tf deleted file mode 100644 index aff33b73a0..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/main.tf +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] - project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] - - # Define affinity settings when node pools are specified - node_pool_affinity = var.node_pool_names != null ? { - nodeAffinity = { - requiredDuringSchedulingIgnoredDuringExecution = { - nodeSelectorTerms = [{ - matchExpressions = [{ - key = "cloud.google.com/gke-nodepool" - operator = "In" - values = var.node_pool_names - }] - }] - } - } - } : {} -} - -data "google_client_config" "default" {} - -data "google_container_cluster" "gke_cluster" { - project = local.project_id - name = local.cluster_name - location = local.cluster_location -} - -resource "helm_release" "cert_manager" { - name = "cert-manager" - chart = "cert-manager" - repository = "https://charts.jetstack.io" - version = var.cert_manager_chart_version - namespace = "cert-manager" - create_namespace = true - - values = concat( - [yamlencode({ - affinity = local.node_pool_affinity - webhook = { - affinity = local.node_pool_affinity - } - cainjector = { - affinity = local.node_pool_affinity - } - startupapicheck = { - affinity = local.node_pool_affinity - } - })], - [yamlencode(var.cert_manager_values)] - ) -} - -resource "helm_release" "slurm_operator" { - count = var.install_slurm_operator_chart ? 1 : 0 - name = "slurm-operator" - chart = "slurm-operator" - repository = var.slurm_operator_repository - version = var.slurm_operator_chart_version - namespace = var.slurm_operator_namespace - create_namespace = true - - # The Cert Manager webhook deployment must be running to provision the Operator - depends_on = [ - helm_release.cert_manager - ] - - values = concat( - [yamlencode({ - operator = { - affinity = local.node_pool_affinity - } - webhook = { - affinity = local.node_pool_affinity - } - })], - [yamlencode(var.slurm_operator_values)] - ) -} - -resource "helm_release" "slurm" { - count = var.install_slurm_chart ? 1 : 0 - name = "slurm" - chart = "slurm" - repository = var.slurm_repository - version = var.slurm_chart_version - namespace = var.slurm_namespace - create_namespace = true - - # The Slurm Operator must be running to provision Slurm clusters/nodesets - depends_on = [ - helm_release.slurm_operator - ] - - values = concat( - [yamlencode({ - controller = { - affinity = local.node_pool_affinity - } - accounting = { - affinity = local.node_pool_affinity - } - mariadb = { - primary = { - affinity = local.node_pool_affinity - } - secondary = { - affinity = local.node_pool_affinity - } - } - restapi = { - affinity = local.node_pool_affinity - } - slurm-exporter = { - exporter = { - affinity = local.node_pool_affinity - } - } - })], - [yamlencode(var.slurm_values)] - ) -} - -resource "helm_release" "prometheus" { - count = var.install_kube_prometheus_stack ? 1 : 0 - name = "prometheus" - chart = "kube-prometheus-stack" - repository = "https://prometheus-community.github.io/helm-charts" - version = var.prometheus_chart_version - namespace = "prometheus" - create_namespace = true - - values = concat( - [yamlencode({ - crds = { - upgradeJob = { - affinity = local.node_pool_affinity - } - } - alertmanager = { - alertmanagerSpec = { - affinity = local.node_pool_affinity - } - } - prometheusOperator = { - admissionWebhooks = { - deployment = { - affinity = local.node_pool_affinity - } - patch = { - affinity = local.node_pool_affinity - } - } - affinity = local.node_pool_affinity - } - prometheus = { - prometheusSpec = { - affinity = local.node_pool_affinity - } - } - thanosRuler = { - thanosRulerSpec = { - affinity = local.node_pool_affinity - } - } - kube-state-metrics = { - affinity = local.node_pool_affinity - } - grafana = { - affinity = local.node_pool_affinity - imageRenderer = { - affinity = local.node_pool_affinity - } - } - prometheus-windows-exporter = { - affinity = local.node_pool_affinity - } - })], - [yamlencode(var.prometheus_values)] - ) -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/metadata.yaml deleted file mode 100644 index e18197e2b7..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/outputs.tf deleted file mode 100644 index 8ea6385905..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "slurm_namespace" { - description = "namespace for the slurm chart" - value = var.slurm_namespace -} - -output "slurm_operator_namespace" { - description = "namespace for the slinky operator chart" - value = var.slurm_operator_namespace -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/providers.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/providers.tf deleted file mode 100644 index 313d6dc58e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/providers.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -provider "helm" { - kubernetes { - host = "https://${data.google_container_cluster.gke_cluster.endpoint}" - token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode( - data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate, - ) - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/variables.tf deleted file mode 100644 index 8acaf78562..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/variables.tf +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "The project ID that hosts the GKE cluster." - type = string -} - -variable "cluster_id" { - description = "An identifier for the GKE cluster resource with format projects//locations//clusters/." - type = string - nullable = false -} - -variable "node_pool_names" { - description = "Names of node pools, for use in node affinities (Slinky system components)." - type = list(string) - default = null -} - -variable "cert_manager_chart_version" { - description = "Version of the Cert Manager chart to install." - type = string - default = "v1.18.2" -} - -variable "cert_manager_values" { - description = "Value overrides for the Cert Manager release" - type = any - default = { - crds = { - enabled = true - } - } -} - -variable "slurm_operator_chart_version" { - description = "Version of the Slurm Operator chart to install." - type = string - default = "0.3.1" -} - -variable "slurm_operator_values" { - description = "Value overrides for the Slinky release" - type = any - default = {} -} - -variable "slurm_chart_version" { - description = "Version of the Slurm chart to install." - type = string - default = "0.3.1" -} - -variable "slurm_values" { - description = "Value overrides for the Slurm release" - type = any - default = {} -} - -variable "install_kube_prometheus_stack" { - # Components detailed at https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack - description = "Install the Kube Prometheus Stack." - type = bool - default = false -} - -variable "prometheus_chart_version" { - description = "Version of the Kube Prometheus Stack chart to install." - type = string - default = "77.0.1" -} - -variable "prometheus_values" { - description = "Value overrides for the Prometheus release" - type = any - default = { - installCRDs = true - } -} - -variable "slurm_namespace" { - description = "slurm namespace for charts" - type = string - default = "slurm" -} - -variable "slurm_operator_namespace" { - description = "slurm namespace for charts" - type = string - default = "slinky" -} - -variable "install_slurm_chart" { - description = "Install slurm-operator chart." - type = bool - default = true -} - -variable "install_slurm_operator_chart" { - description = "Install slurm-operator chart." - type = bool - default = true -} - -variable "slurm_repository" { - description = "Value overrides for the Slinky release" - type = string - default = "oci://ghcr.io/slinkyproject/charts" -} - -variable "slurm_operator_repository" { - description = "Value overrides for the Slinky release" - type = string - default = "oci://ghcr.io/slinkyproject/charts" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/versions.tf deleted file mode 100644 index ae4327aeef..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scheduler/slinky/versions.tf +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - google = { - source = "hashicorp/google" - version = ">= 6.16" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/README.md deleted file mode 100644 index 71a862fd6c..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/README.md +++ /dev/null @@ -1,149 +0,0 @@ -## Description - -This module creates a Toolkit runner that will install HTCondor on RedHat 7 or -8 and its derivative operating systems. These include the CentOS 7 and Rocky -Linux 8 releases of the [HPC VM Image][hpcvmimage]. It may also function on -RedHat 9 and derivatives, however it is not yet supported. Please report any -[issues] on these 3 distributions or open a [discussion] to request support on -Debian or Ubuntu distributions. - -[issues]: https://github.com/GoogleCloudPlatform/hpc-toolkit/issues -[discussion]: https://github.com/GoogleCloudPlatform/hpc-toolkit/discussions - -It also exports a list of Google Cloud APIs which must be enabled prior to -provisioning an HTCondor Pool. - -It is expected to be used with the [htcondor-setup] and -[htcondor-execute-point] modules. - -[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[htcondor-setup]: ../../scheduler/htcondor-setup/README.md -[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md - -### Example - -The following code snippet uses this module to create startup scripts that -install the HTCondor software into a custom VM image. - -```yaml -deployment_groups: -- group: primary - modules: - - id: network1 - source: modules/network/vpc - outputs: - - network_name - - - id: htcondor_install - source: community/modules/scripts/htcondor-install - - - id: htcondor_install_script - source: modules/scripts/startup-script - use: - - htcondor_install - -- group: packer - modules: - - id: custom-image - source: modules/packer/custom-image - kind: packer - use: - - network1 - - htcondor_install_script - settings: - disk_size: 50 - source_image_family: hpc-rocky-linux-8 - image_family: "htcondor-10x" -``` - -A full example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -## Important note - -All POSIX users and HTCondor jobs can act as the service account attached to -VMs within the pool. This enables the use of IAM restrictions via service -accounts but also allows users to access services to which system daemons need -access (e.g. to create Cloud Logging entries). If this is undesirable, one can -restrict access to the instance metadata server to the `root` and `condor` -users. This will allow system services to use the service account, but not -other POSIX users or HTCondor jobs. The firewall example below is appropriate -for CentOS 7. - -```shell -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 1 \ - -m owner --uid-owner root -p tcp -d metadata.google.internal --dport 80 -j ACCEPT -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 2 \ - -m owner --uid-owner condor -p tcp -d metadata.google.internal --dport 80 -j ACCEPT -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 3 \ - -p tcp -d metadata.google.internal --dport 80 -j DROP -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 4 \ - -p tcp -d metadata.google.internal --dport 8080 -j DROP -firewall-cmd --permanent --zone=public --add-port=9618/tcp -firewall-cmd --reload -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [condor\_version](#input\_condor\_version) | Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: "23.0.0","23.*")) | `string` | `"23.*"` | no | -| [enable\_docker](#input\_enable\_docker) | Install and enable docker daemon alongside HTCondor | `bool` | `true` | no | -| [http\_proxy](#input\_http\_proxy) | Set system default web (http and https) proxy for Windows HTCondor installation | `string` | `""` | no | -| [python\_windows\_installer\_url](#input\_python\_windows\_installer\_url) | URL of Python installer for Windows | `string` | `"https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [gcp\_service\_list](#output\_gcp\_service\_list) | Google Cloud APIs required by HTCondor | -| [runners](#output\_runners) | Runner to install HTCondor using startup-scripts | -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | Windows PowerShell script to install HTCondor | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py deleted file mode 100644 index 77bafa0310..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py +++ /dev/null @@ -1,417 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -# Copyright 2018 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script for resizing managed instance group (MIG) cluster size based -# on the number of jobs in the Condor Queue. - -from absl import app -from absl import flags -from collections import OrderedDict -from datetime import datetime -from pprint import pprint -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials - -import argparse -import os -import math -import time -import htcondor -import classad - -parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) -parser.add_argument("--p", required=True, help="Project id", type=str) -parser.add_argument( - "--z", - required=True, - help="Name of GCP zone where the managed instance group is located", - type=str, -) -parser.add_argument( - "--r", - required=True, - help="Name of GCP region where the managed instance group is located", - type=str, -) -parser.add_argument( - "--mz", - required=False, - help="Enabled multizone (regional) managed instance group", - action="store_true", -) -parser.add_argument( - "--g", required=True, help="Name of the managed instance group", type=str -) -parser.add_argument( - "--i", - default=0, - help="Minimum number of idle compute instances", - type=int -) -parser.add_argument( - "--c", required=True, help="Maximum number of compute instances", type=int -) -parser.add_argument( - "--v", - default=0, - help="Increase output verbosity. 1-show basic debug info. 2-show detail debug info", - type=int, - choices=[0, 1, 2], -) -parser.add_argument( - "--d", - default=0, - help="Dry Run, default=0, if 1, then no scaling actions", - type=int, - choices=[0, 1], -) - -args = parser.parse_args() - -class AutoScaler: - def __init__(self, multizone=False): - - self.multizone = multizone - # Obtain credentials - self.credentials = GoogleCredentials.get_application_default() - self.service = discovery.build("compute", "v1", credentials=self.credentials) - - if self.multizone: - self.instanceGroupManagers = self.service.regionInstanceGroupManagers() - else: - self.instanceGroupManagers = self.service.instanceGroupManagers() - - # Remove specified instances from MIG and decrease MIG size - def deleteFromMig(self, node_self_links): - requestDelInstance = self.instanceGroupManagers.deleteInstances( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - body={ "instances": node_self_links }, - ) - - # execute if not a dry-run - if not self.dryrun: - response = requestDelInstance.execute() - if self.debug > 0: - pprint(response) - return response - return "Dry Run" - - def getInstanceTemplateInfo(self): - requestTemplateName = self.instanceGroupManagers.get( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - fields="instanceTemplate", - ) - responseTemplateName = requestTemplateName.execute() - template_name = "" - - if self.debug > 1: - print("Request for the template name") - pprint(responseTemplateName) - - if len(responseTemplateName) > 0: - template_url = responseTemplateName.get("instanceTemplate") - template_url_partitioned = template_url.split("/") - template_name = template_url_partitioned[len(template_url_partitioned) - 1] - - requestInstanceTemplate = self.service.instanceTemplates().get( - project=self.project, instanceTemplate=template_name, fields="properties" - ) - responseInstanceTemplateInfo = requestInstanceTemplate.execute() - - if self.debug > 1: - print("Template information") - pprint(responseInstanceTemplateInfo["properties"]) - - machine_type = responseInstanceTemplateInfo["properties"]["machineType"] - is_spot = responseInstanceTemplateInfo["properties"]["scheduling"][ - "preemptible" - ] - if self.debug > 0: - print("Machine Type: " + machine_type) - print("Is spot: " + str(is_spot)) - request = self.service.machineTypes().get( - project=self.project, zone=self.zone, machineType=machine_type - ) - response = request.execute() - guest_cpus = response["guestCpus"] - if self.debug > 1: - print("Machine information") - pprint(responseInstanceTemplateInfo["properties"]) - if self.debug > 0: - print("Guest CPUs: " + str(guest_cpus)) - - instanceTemplateInfo = { - "machine_type": machine_type, - "is_spot": is_spot, - "guest_cpus": guest_cpus, - } - return instanceTemplateInfo - - def scale(self): - # diagnosis - if self.debug > 1: - print("Launching autoscaler.py with the following arguments:") - print("project_id: " + self.project) - print("zone: " + self.zone) - print("region: " + self.region) - print(f"multizone: {self.multizone}") - print("group_manager: " + self.instance_group_manager) - print("computeinstancelimit: " + str(self.compute_instance_limit)) - print("debuglevel: " + str(self.debug)) - - if self.multizone: - self.zoneargs = {"region": self.region} - else: - self.zoneargs = {"zone": self.zone} - - # Each HTCondor scheduler (SchedD), maintains a list of jobs under its - # stewardship. A full list of Job ClassAd attributes can be found at - # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html - schedd = htcondor.Schedd() - # encourage the job queue to start a new negotiation cycle; there are - # internal unconfigurable rate limits so not guaranteed; this is not - # strictly required for success, but may reduce latency of autoscaling - schedd.reschedule() - REQUEST_CPUS_ATTRIBUTE = "RequestCpus" - REQUEST_GPUS_ATTRIBUTE = "RequestGpus" - REQUEST_MEMORY_ATTRIBUTE = "RequestMemory" - job_attributes = [ - REQUEST_CPUS_ATTRIBUTE, - REQUEST_GPUS_ATTRIBUTE, - REQUEST_MEMORY_ATTRIBUTE, - ] - - instanceTemplateInfo = self.getInstanceTemplateInfo() - self.is_spot = instanceTemplateInfo["is_spot"] - self.cores_per_node = instanceTemplateInfo["guest_cpus"] - print(f"MIG is configured for Spot pricing: {self.is_spot}") - print("Number of CPU per compute node: " + str(self.cores_per_node)) - - # this query will constrain the search for jobs to those that either - # require spot VMs or do not require Spot VMs based on whether the - # VM instance template is configured for Spot pricing - spot_query = classad.ExprTree(f"RequireId == \"{self.instance_group_manager}\"") - - # For purpose of scaling a Managed Instance Group, count only jobs that - # are idle and likely participated in a negotiation cycle (there does - # not appear to be a single classad attribute for this). - # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html#JobStatus - LAST_CYCLE_ATTRIBUTE = "LastNegotiationCycleTime0" - coll = htcondor.Collector() - negotiator_ad = coll.query(htcondor.AdTypes.Negotiator, projection=[LAST_CYCLE_ATTRIBUTE]) - if len(negotiator_ad) != 1: - print(f"There should be exactly 1 negotiator in the pool. There is {len(negotiator_ad)}") - exit() - last_negotiation_cycle_time = negotiator_ad[0].get(LAST_CYCLE_ATTRIBUTE) - if not last_negotiation_cycle_time: - print(f"The negotiator has not yet started a match cycle. Exiting auto-scaling.") - exit() - - print(f"Last negotiation cycle occurred at: {datetime.fromtimestamp(last_negotiation_cycle_time)}") - idle_job_query = classad.ExprTree(f"JobStatus == 1 && QDate < {last_negotiation_cycle_time}") - idle_job_ads = schedd.query(constraint=idle_job_query.and_(spot_query), - projection=job_attributes) - - total_idle_request_cpus = sum(j[REQUEST_CPUS_ATTRIBUTE] for j in idle_job_ads) - print(f"Total CPUs requested by idle jobs: {total_idle_request_cpus}") - - if self.debug > 1: - print("Information about the compute instance template") - pprint(instanceTemplateInfo) - - # Calculate the minimum number of instances that, for fully packed - # execute points, could satisfy current job queue - min_hosts_for_idle_jobs = math.ceil(total_idle_request_cpus / self.cores_per_node) - if self.debug > 0: - print(f"Minimum hosts needed: {total_idle_request_cpus} / {self.cores_per_node} = {min_hosts_for_idle_jobs}") - - # Get current number of instances in the MIG - requestGroupInfo = self.instanceGroupManagers.get( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - ) - responseGroupInfo = requestGroupInfo.execute() - current_target = responseGroupInfo["targetSize"] - print(f"Current MIG target size: {current_target}") - - # Find instances that are being modified by the MIG (currentAction is - # any value other than "NONE"). A common reason an instance is modified - # is it because it has failed a health check. - reqModifyingInstances = self.instanceGroupManagers.listManagedInstances( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - filter="currentAction != \"NONE\"", - orderBy="creationTimestamp desc" - ) - respModifyingInstances = reqModifyingInstances.execute() - - # Find VMs that are idle (no dynamic slots created from partitionable - # slots) in the MIG handled by this autoscaler - filter_idle_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots==0") - filter_claimed_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots>0") - filter_mig = classad.ExprTree(f"regexp(\".*/{self.instance_group_manager}$\", CloudCreatedBy)") - # A full list of Machine (StartD) ClassAd attributes can be found at - # https://htcondor.readthedocs.io/en/latest/classad-attributes/machine-classad-attributes.html - idle_node_ads = coll.query(htcondor.AdTypes.Startd, - constraint=filter_idle_vms.and_(filter_mig), - projection=["Machine", "CloudZone"]) - - NODENAME_ATTRIBUTE = "Machine" - claimed_node_ads = coll.query(htcondor.AdTypes.Startd, - constraint=filter_claimed_vms.and_(filter_mig), - projection=[NODENAME_ATTRIBUTE]) - claimed_nodes = [ ad[NODENAME_ATTRIBUTE].split(".")[0] for ad in claimed_node_ads] - - # treat OrderedDict as a set by ignoring key values; this set will - # contain VMs we would consider deleting, in inverse order of - # their readiness to join pool (creating, unhealthy, healthy+idle) - idle_nodes = OrderedDict() - try: - modifyingInstances = respModifyingInstances["managedInstances"] - except KeyError: - modifyingInstances = [] - - print(f"There are {len(modifyingInstances)} VMs being modified by the managed instance group") - - # there is potential for nodes in MIG health check "VERIFYING" state - # to have already joined the pool and be running jobs - for instance in modifyingInstances: - self_link = instance["instance"] - node_name = self_link.rsplit("/", 1)[-1] - if node_name not in claimed_nodes: - idle_nodes[self_link] = "modifying" - - for ad in idle_node_ads: - node = ad["Machine"].split(".")[0] - zone = ad["CloudZone"] - self_link = "https://www.googleapis.com/compute/v1/projects/" + \ - self.project + "/zones/" + zone + "/instances/" + node - # there is potential for nodes in MIG health check "VERIFYING" state - # to have already joined the pool and be idle; delete them last - if self_link in idle_nodes: - idle_nodes.move_to_end(self_link) - idle_nodes[self_link] = "idle" - n_idle = len(idle_nodes) - - print(f"There are {n_idle} VMs being modified or idle in the pool") - if self.debug > 1: - print("Listing idle nodes:") - pprint(idle_nodes) - - # always keep size tending toward the minimum idle VMs requested - new_target = current_target + self.compute_instance_min_idle - n_idle + min_hosts_for_idle_jobs - if new_target > self.compute_instance_limit: - self.size = self.compute_instance_limit - print(f"MIG target size will be limited by {self.compute_instance_limit}") - else: - self.size = new_target - - print(f"New MIG target size: {self.size}") - - if self.debug > 1: - print("MIG Information:") - print(responseGroupInfo) - - if self.size == current_target: - if current_target == 0: - print("Queue is empty") - print("Running correct number of VMs to handle queue") - exit() - - if self.size < current_target: - print("Scaling down. Looking for nodes that can be shut down") - - if self.debug > 1: - print("Compute node busy status:") - for node in idle_nodes: - print(node) - - # Shut down idle nodes up to our calculated limit - nodes_to_delete = list(idle_nodes.keys())[0:current_target-self.size] - for node in nodes_to_delete: - print(f"Attempting to delete: {node.rsplit('/',1)[-1]}") - respDel = self.deleteFromMig(nodes_to_delete) - - if self.debug > 1: - print("Scaling down complete") - - if self.size > current_target: - print( - "Scaling up. Need to increase number of instances to " + str(self.size) - ) - # Request to resize - request = self.instanceGroupManagers.resize( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - size=self.size, - ) - response = request.execute() - if self.debug > 1: - print("Requesting to increase MIG size") - pprint(response) - print("Scaling up complete") - - -def main(): - - scaler = AutoScaler(args.mz) - - # Project ID - scaler.project = args.p # Ex:'slurm-var-demo' - - # Name of the zone where the managed instance group is located - scaler.zone = args.z # Ex: 'us-central1-f' - - # Name of the region where the managed instance group is located - scaler.region = args.r # Ex: 'us-central1' - - # The name of the managed instance group. - scaler.instance_group_manager = args.g # Ex: 'condor-compute-igm' - - # Default number of cores per instance, will be replaced with actual value - scaler.cores_per_node = 4 - - # Default number of running instances that the managed instance group should maintain at any given time. This number will go up and down based on the load (number of jobs in the queue) - scaler.size = 0 - - scaler.compute_instance_min_idle = args.i - - # Dry run: : 0, run scaling; 1, only provide info. - scaler.dryrun = args.d > 0 - - # Debug level: 1-print debug information, 2 - print detail debug information - scaler.debug = 0 - if args.v: - scaler.debug = args.v - - # Limit for the maximum number of compute instance. If zero (default setting), no limit will be enforced by the script - scaler.compute_instance_limit = 0 - if args.c: - scaler.compute_instance_limit = abs(args.c) - - scaler.scale() - - -if __name__ == "__main__": - main() diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml deleted file mode 100644 index db989f9d40..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Install but do not activate HTCondor autoscaler - become: true - hosts: localhost - tasks: - - name: Install Python 3 pip - ansible.builtin.package: - name: python3-pip - state: present - - name: Create virtual environment for HTCondor autoscaler - ansible.builtin.pip: - name: pip - version: 21.3.1 # last Python 3.6-compatible release - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv - - name: Install latest setuptools - ansible.builtin.pip: - name: setuptools - version: 59.6.0 # last Python 3.6-compatible release - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv - - name: Install HTCondor autoscaler dependencies - with_items: - - oauth2client - - google-api-python-client - - absl-py - - htcondor - ansible.builtin.pip: - name: "{{ item }}" - state: present # rely on pip resolver to pick latest compatible releases - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml deleted file mode 100644 index 4d3abbbfd6..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The instructions for installing HTCondor may change with time, although we -# anticipate that they will stay fixed for the 23.0 releases. Find up-to-date -# recommendations at: -## https://htcondor.readthedocs.io/en/latest/getting-htcondor/from-our-repositories.html - ---- -- name: Ensure HTCondor is installed - hosts: all - vars: - enable_docker: true - htcondor_key: https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-23.0-Key - docker_key: https://download.docker.com/linux/centos/gpg - become: true - module_defaults: - ansible.builtin.yum: - lock_timeout: 300 - tasks: - - name: Enable EPEL repository - ansible.builtin.yum: - name: - - epel-release - - name: Directly install RPM verification keys - ansible.builtin.rpm_key: - state: present - key: "{{ item }}" - loop: - - "{{ htcondor_key }}" - - "{{ docker_key }}" - register: key_install - retries: 10 - delay: 60 - until: key_install is success - - name: Enable HTCondor LTS Release repository - ansible.builtin.yum_repository: - name: htcondor-feature - description: HTCondor LTS Release (23.0) - file: htcondor - baseurl: https://research.cs.wisc.edu/htcondor/repo/23.0/el$releasever/$basearch/release - gpgkey: "{{ htcondor_key }}" - gpgcheck: true - repo_gpgcheck: true - priority: "90" - - name: Install HTCondor - ansible.builtin.yum: - name: condor-{{ condor_version | default("23.*") | string }} - state: present - - name: Ensure token directory - ansible.builtin.file: - path: /etc/condor/tokens.d - mode: 0700 - owner: root - group: root - - name: Install Docker and configure HTCondor to use it - when: enable_docker | bool # allows string to be passed at CLI - block: - - name: Setup Docker repo - ansible.builtin.yum_repository: - name: docker-ce-stable - description: Docker CE Stable - $basearch - baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable - enabled: yes - gpgcheck: yes - gpgkey: "{{ docker_key }}" - - name: Install Docker - ansible.builtin.yum: - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-compose-plugin - - name: Enable Docker - ansible.builtin.service: - name: docker - state: started - enabled: true - - name: Add condor to docker group - ansible.builtin.user: - name: condor - groups: docker - append: yes diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/main.tf deleted file mode 100644 index 0853e035f4..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/main.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - runners = [ - { - "type" = "ansible-local" - "source" = "${path.module}/files/install-htcondor.yaml" - "destination" = "install-htcondor.yaml" - "args" = join(" ", [ - "-e enable_docker=${var.enable_docker}", - "-e condor_version=${var.condor_version}", - ]) - }, - { - "type" = "ansible-local" - "content" = file("${path.module}/files/install-htcondor-autoscaler-deps.yml") - "destination" = "install-htcondor-autoscaler-deps.yml" - }, - { - "type" = "data" - "content" = file("${path.module}/files/autoscaler.py") - "destination" = "/usr/local/htcondor/bin/autoscaler.py" - }, - ] - - install_htcondor_ps1 = templatefile( - "${path.module}/templates/install-htcondor.ps1.tftpl", { - condor_version = var.condor_version, - http_proxy = var.http_proxy, - python_windows_installer_url = var.python_windows_installer_url, - }) - - required_apis = [ - "compute.googleapis.com", - "secretmanager.googleapis.com", - ] -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf deleted file mode 100644 index c7951737ff..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "runners" { - description = "Runner to install HTCondor using startup-scripts" - value = local.runners -} - -output "windows_startup_ps1" { - description = "Windows PowerShell script to install HTCondor" - value = local.install_htcondor_ps1 -} - -output "gcp_service_list" { - description = "Google Cloud APIs required by HTCondor" - value = local.required_apis -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl deleted file mode 100644 index 7492da3c12..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl +++ /dev/null @@ -1,59 +0,0 @@ -#Requires -RunAsAdministrator - -# Windows 2016 needs forced upgrade to TLS 1.2 -[Net.ServicePointManager]::SecurityProtocol = 'Tls12' - -# important for catching exception in Invoke-WebRequest -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -%{ if http_proxy != "" ~} -[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") -%{ endif ~} - -# do not show progress bar when running Invoke-WebRequest -$ProgressPreference = 'SilentlyContinue' - -# download C Runtime DLL necessary for HTCondor installer -$runtime_installer = 'C:\vc_redist.x64.exe' -Invoke-WebRequest https://aka.ms/vs/17/release/vc_redist.x64.exe -OutFile "$runtime_installer" -Start-Process -FilePath "$runtime_installer" -Wait -ArgumentList "/norestart /quiet /log c:\vc_redist_log.txt" -Remove-Item "$runtime_installer" - -# download HTCondor installer -$htcondor_installer = 'C:\htcondor.msi' -%{ if condor_version == "23.*" } -Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/current/condor-Windows-x64.msi -OutFile "$htcondor_installer" -%{ else ~} -Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/${condor_version}/release/condor-${condor_version}-Windows-x64.msi -OutFile "$htcondor_installer" -%{ endif ~} -$args='/qn /l* condor-install-log.txt /i' -$args=$args + " $htcondor_installer" -$args=$args + ' NEWPOOL="N"' -$args=$args + ' RUNJOBS="N"' -$args=$args + ' SUBMITJOBS="N"' -$args=$args + ' INSTALLDIR="C:\Condor"' -Start-Process "msiexec.exe" -Wait -ArgumentList "$args" -Remove-Item "$htcondor_installer" - -# do not start HTCondor on boot by default. Allow startup script to download -# configuration first and then start HTCondor -Set-Service -StartupType Manual condor - -# remove settings from condor_config that we want to override in configuration step -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^CONDOR_HOST' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^INSTALL_USER' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^DAEMON_LIST' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^use SECURITY' -NotMatch) - -# install Python so that custom ClassAd hooks can execute -$python_installer = 'C:\python-installer.exe' -Invoke-WebRequest -Uri "${python_windows_installer_url}" -OutFile "$python_installer" -Start-Process -FilePath "$python_installer" -Wait -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1 Include_test=0' -%{ if http_proxy == "" ~} -Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --no-warn-script-location requests" -%{ else ~} -Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --proxy ${http_proxy} --no-warn-script-location requests" -%{ endif ~} -Invoke-WebRequest -Uri "https://raw.githubusercontent.com/htcondor/htcondor/main/src/condor_scripts/common-cloud-attributes-google.py" -OutFile "C:\Condor\bin\common-cloud-attributes-google.py" -Remove-Item "$python_installer" diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/variables.tf deleted file mode 100644 index 1afdf4e0eb..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "enable_docker" { - description = "Install and enable docker daemon alongside HTCondor" - type = bool - default = true -} - -variable "condor_version" { - description = "Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: \"23.0.0\",\"23.*\"))" - type = string - default = "23.*" - - validation { - error_message = "var.condor_version must be set to \"23.*\" for latest 23.0 release or to a specific \"23.0.y\" release." - condition = var.condor_version == "23.*" || ( - length(split(".", var.condor_version)) == 3 && alltrue([ - for v in split(".", var.condor_version) : can(tonumber(v)) - ]) && split(".", var.condor_version)[0] == "23" - && split(".", var.condor_version)[1] == "0" - ) - } -} - -variable "http_proxy" { - description = "Set system default web (http and https) proxy for Windows HTCondor installation" - type = string - default = "" - nullable = false -} - -variable "python_windows_installer_url" { - description = "URL of Python installer for Windows" - type = string - default = "https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe" - nullable = false -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/htcondor-install/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/README.md deleted file mode 100644 index 55c2fc7e4e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/README.md +++ /dev/null @@ -1,116 +0,0 @@ -## Description - -This module will create a startup-script runner that will execute Ramble commands. - -Ramble is a multi-platform experimentation framework capable of driving -software installation, acquiring input files, configuring experiments, and -extracting results. For more information about Ramble, see: -https://github.com/GoogleCloudPlatform/ramble - -This module outputs a startup script runner, which can be combined with other -startup script runners to execute a set of Ramble commands. - -Ramble makes extensive use of Spack. It must be installed with a Toolkit runner -generated by the [spack-setup module](../spack-setup/README.md) following the -[basic example](#basic-example) below. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -# Examples - -## Basic Example - -Below is a basic example of using this module. - -```yaml - - id: spack - source: community/modules/scripts/spack-setup - - - id: ramble-setup - source: community/modules/scripts/ramble-setup - - - id: ramble-execute - source: community/modules/scripts/ramble-execute - use: [spack, ramble-setup] - settings: - commands: - - ramble list -``` - -This example shows installing Spack and Ramble with their own modules -(spack-setup and ramble-setup respectively). Then the ramble-execute module -is added to simply list all applications Ramble knows about. - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | -| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [log\_file](#input\_log\_file) | Log file to write output from Ramble execute steps into | `string` | `"/var/log/ramble-execute.log"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by an instance of ramble-setup.
Can be defined explicitly, or by chaining an instance of a ramble-setup module
through a `use` setting. | `string` | n/a | yes | -| [ramble\_runner](#input\_ramble\_runner) | Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | -| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting.
Defaults to /etc/profile.d/spack.sh if not set. | `string` | `"/etc/profile.d/spack.sh"` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the ramble-setup module. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble startup script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for ramble, to be reused by ramble-execute module. | -| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | -| [ramble\_runner](#output\_ramble\_runner) | Runner to execute Ramble commands using an ansible playbook. The startup-script module
will automatically handle installation of ansible. | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to Spack profile script. | -| [startup\_script](#output\_startup\_script) | Ramble startup script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/main.tf deleted file mode 100644 index 7ef0b029e3..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/main.tf +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ramble-execute", ghpc_role = "scripts" }) -} - -locals { - commands_content = var.commands == null ? "echo 'no ramble commands provided'" : indent(4, yamlencode(var.commands)) - - execute_contents = templatefile( - "${path.module}/templates/ramble_execute.yml.tpl", - { - pre_script = "if [ -f ${var.spack_profile_script_path} ]; then . ${var.spack_profile_script_path}; fi; . ${var.ramble_profile_script_path}" - log_file = var.log_file - commands = local.commands_content - system_user_name = var.system_user_name - } - ) - - data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] - - execute_md5 = substr(md5(local.execute_contents), 0, 4) - execute_runner = { - type = "ansible-local" - content = local.execute_contents - destination = "ramble_execute_${local.execute_md5}.yml" - } - - previous_runners = var.ramble_runner != null ? [var.ramble_runner] : [] - runners = concat(local.previous_runners, local.data_runners, [local.execute_runner]) - - # Destinations should be unique while also being known at time of apply - combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) - combined_md5 = substr(md5(local.combined_unique_string), 0, 4) - combined_runner = { - type = "shell" - content = module.startup_script.startup_script - destination = "combined_install_ramble_${local.combined_md5}.sh" - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = var.gcs_bucket_path -} - -resource "local_file" "debug_file_ansible_execute" { - content = local.execute_contents - filename = "${path.module}/debug_execute_${local.execute_md5}.yml" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf deleted file mode 100644 index 4e6c3a44d8..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Ramble startup script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Ramble startup script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "ramble_runner" { - description = <<-EOT - Runner to execute Ramble commands using an ansible playbook. The startup-script module - will automatically handle installation of ansible. - EOT - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for ramble, to be reused by ramble-execute module." - value = var.gcs_bucket_path -} - -output "spack_profile_script_path" { - description = "Path to Spack profile script." - value = var.spack_profile_script_path -} - -output "ramble_profile_script_path" { - description = "Path to Ramble profile script." - value = var.ramble_profile_script_path -} - -output "system_user_name" { - description = "The system user used to execute commands." - value = var.system_user_name -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl deleted file mode 100644 index 0e98f3aa2c..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Execute Commands - hosts: localhost - vars: - pre_script: ${pre_script} - log_file: ${log_file} - commands: ${commands} - system_user_name: ${system_user_name} - tasks: - - name: Execute command block - block: - - name: Print commands to be executed - ansible.builtin.debug: - msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" - - - name: Streaming log info - ansible.builtin.debug: - msg: | - Logs from commands will not be printed here until success (or failure) - Streaming logs can be found at {{ log_file }} - - - name: Ensure user can write to log file - ansible.builtin.file: - path: "{{ log_file }}" - state: touch - owner: "{{ system_user_name }}" - - - name: Execute commands - ansible.builtin.shell: | - set -eo pipefail - { - {{ pre_script }} - echo " === Starting commands ===" - {{ commands }} - echo " === Finished commands ===" - } 2>&1 | tee -a {{ log_file }} - args: - executable: /bin/bash - register: output - become: true - become_user: "{{ system_user_name }}" - - always: - - name: Print commands output - ansible.builtin.debug: - var: output.stdout_lines diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/variables.tf deleted file mode 100644 index ec67228df5..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/variables.tf +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing spack scripts." - type = string -} - -variable "region" { - description = "Region to place bucket containing spack scripts." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "log_file" { - description = "Log file to write output from Ramble execute steps into" - default = "/var/log/ramble-execute.log" - type = string -} - -variable "data_files" { - description = <<-EOT - A list of files to be transferred prior to running commands. - It must specify one of 'source' (absolute local file path) or 'content' (string). - It must specify a 'destination' with absolute path where file should be placed. - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) - error_message = "All destinations must be absolute paths and start with '/'." - } - validation { - condition = alltrue([ - for r in var.data_files : - can(r["content"]) != can(r["source"]) - ]) - error_message = "A data_file must specify either 'content' or 'source', but never both." - } - validation { - condition = alltrue([ - for r in var.data_files : - lookup(r, "content", lookup(r, "source", null)) != null - ]) - error_message = "A data_file must specify a non-null 'content' or 'source'." - } -} - -variable "commands" { - description = "String of commands to run within this module" - default = null - type = string -} - -variable "ramble_runner" { - description = "Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module." - type = object({ - type = string - content = string - destination = string - }) -} - -variable "system_user_name" { - description = "Name of the system user used to execute commands. Generally passed from the ramble-setup module." - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string -} - -variable "spack_profile_script_path" { - description = <<-EOT - Path to the Spack profile.d script. - Can be defined explicitly, or by chaining an instance of a spack-setup module - through a `use` setting. - Defaults to /etc/profile.d/spack.sh if not set. - EOT - type = string - default = "/etc/profile.d/spack.sh" -} - -variable "ramble_profile_script_path" { - description = <<-EOT - Path to the Ramble profile.d script. Created by an instance of ramble-setup. - Can be defined explicitly, or by chaining an instance of a ramble-setup module - through a `use` setting. - EOT - type = string -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/versions.tf deleted file mode 100644 index 9b23317323..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-execute/versions.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/README.md deleted file mode 100644 index 9891088105..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module will create a set of startup-script runners that will setup Ramble, -and install Ramble’s dependencies. - -Ramble is a multi-platform experimentation framework capable of driving -software installation, acquiring input files, configuring experiments, and -extracting results. For more information about ramble, see: -https://github.com/GoogleCloudPlatform/ramble - -This module outputs two startup script runners, which can be added to startup -scripts to setup, ramble and its dependencies. - -For this module to be completely functional, it depends on a spack -installation. For more information, see Cluster-Toolkit’s Spack module. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -# Examples - -## Basic Example - -```yaml -- id: ramble-setup - source: community/modules/scripts/ramble-setup -``` - -This example simply installs ramble on a VM. - -## Full Example - -```yaml -- id: ramble-setup - source: community/modules/scripts/ramble-setup - settings: - install_dir: /ramble - ramble_url: https://github.com/GoogleCloudPlatform/ramble - ramble_ref: v0.2.1 - log_file: /var/log/ramble.log - chown_owner: “owner” - chgrp_group: “user_group” - chmod_mode: “a+r” -``` - -This example simply installs ramble into a VM at the location `/ramble`, checks -out the v0.2.1 tag, changes the owner and group to “owner” and “user_group”, -and chmod’s the clone to make it world readable. - -Also see a more complete [Ramble example blueprint](../../../examples/ramble.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 4.42 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [chmod\_mode](#input\_chmod\_mode) | Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify).
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | -| [install\_dir](#input\_install\_dir) | Destination directory of installation of Ramble. | `string` | `"/apps/ramble"` | no | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by this module | `string` | `"/etc/profile.d/ramble.sh"` | no | -| [ramble\_ref](#input\_ramble\_ref) | Git ref to checkout for Ramble. | `string` | `"develop"` | no | -| [ramble\_url](#input\_ramble\_url) | URL for Ramble repository to clone. | `string` | `"https://github.com/GoogleCloudPlatform/ramble"` | no | -| [ramble\_virtualenv\_path](#input\_ramble\_virtualenv\_path) | Virtual environment path in which to install Ramble Python interpreter and other dependencies | `string` | `"/usr/local/ramble-python"` | no | -| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | -| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Ramble. It will be created if it does not exist. | `string` | `"ramble"` | no | -| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble installation script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module. | -| [ramble\_path](#output\_ramble\_path) | Location ramble is installed into. | -| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | -| [ramble\_ref](#output\_ramble\_ref) | Git ref the ramble install is checked out to use | -| [ramble\_runner](#output\_ramble\_runner) | Runner to be used with startup-script module or passed to ramble-execute module.
- installs Ramble dependencies
- installs Ramble
- generates profile.d script to enable access to Ramble
This is safe to run in parallel by multiple machines. | -| [startup\_script](#output\_startup\_script) | Ramble installation script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/main.tf deleted file mode 100644 index 4389af7d33..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/main.tf +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ramble-setup", ghpc_role = "scripts" }) -} - -locals { - profile_script = <<-EOF - if [ -f ${var.install_dir}/share/ramble/setup-env.sh ]; then - test -t 1 && echo "** Ramble's python virtualenv (/usr/local/ramble-python) is activated. Call 'deactivate' to deactivate." - VIRTUAL_ENV_DISABLE_PROMPT=1 . ${var.ramble_virtualenv_path}/bin/activate - . ${var.install_dir}/share/ramble/setup-env.sh - fi - EOF - - script_content = templatefile( - "${path.module}/templates/ramble_setup.yml.tftpl", - { - sw_name = "ramble" - profile_script = indent(4, yamlencode(local.profile_script)) - install_dir = var.install_dir - git_url = var.ramble_url - git_ref = var.ramble_ref - chmod_mode = var.chmod_mode - system_user_name = var.system_user_name - system_user_uid = var.system_user_uid - system_user_gid = var.system_user_gid - finalize_setup_script = "echo 'no finalize setup script'" - profile_script_path = var.ramble_profile_script_path - } - ) - - install_ramble_deps_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/install_ramble_deps.yml" - "destination" = "install_ramble_deps.yml" - "args" = "-e virtualenv_path=${var.ramble_virtualenv_path}" - } - - python_reqs_content = templatefile( - "${path.module}/templates/install_ramble_python_deps.yml.tftpl", - { - install_dir = var.install_dir - virtualenv_path = var.ramble_virtualenv_path - } - ) - - python_reqs_runner = { - "type" = "ansible-local" - "content" = local.python_reqs_content - "destination" = "install_ramble_reqs.yml" - } - - install_ramble_runner = { - "type" = "ansible-local" - "content" = local.script_content - "destination" = "install_ramble.yml" - } - - bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}"), 0, 8) - # Max bucket name length is 63, so truncate deployment_name if necessary. - # The string "-ramble-scripts-" is 16 characters and bucket_md5 is 8 characters, - # leaving 63-16-8=39 chars for deployment_name. - bucket_name = "${substr(var.deployment_name, 0, 39)}-ramble-scripts-${local.bucket_md5}" - runners = [local.install_ramble_deps_runner, local.install_ramble_runner, local.python_reqs_runner] - - combined_runner = { - "type" = "shell" - "content" = module.startup_script.startup_script - "destination" = "ramble-install-and-setup.sh" - } - -} - -resource "google_storage_bucket" "bucket" { - project = var.project_id - name = local.bucket_name - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" -} - -resource "local_file" "debug_file_shell_install" { - content = local.script_content - filename = "${path.module}/debug_install.yml" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf deleted file mode 100644 index e587470eac..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Ramble installation script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Ramble installation script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "ramble_runner" { - description = <<-EOT - Runner to be used with startup-script module or passed to ramble-execute module. - - installs Ramble dependencies - - installs Ramble - - generates profile.d script to enable access to Ramble - This is safe to run in parallel by multiple machines. - EOT - value = local.combined_runner -} - -output "ramble_path" { - description = "Location ramble is installed into." - value = var.install_dir -} - -output "ramble_ref" { - description = "Git ref the ramble install is checked out to use" - value = var.ramble_ref -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module." - value = "gs://${google_storage_bucket.bucket.name}" -} - -output "ramble_profile_script_path" { - description = "Path to Ramble profile script." - value = var.ramble_profile_script_path -} - -output "system_user_name" { - description = "The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands." - value = var.system_user_name -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml deleted file mode 100644 index b7905bbe9e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Create python virtual env for a tool - become: yes - hosts: localhost - vars: - virtualenv_path: ${virtualenv_path} - tasks: - - name: Install dependencies through system package manager - ansible.builtin.package: - name: - - python3 - - python3-pip - - git - register: package - changed_when: package.changed - retries: 5 - delay: 10 - until: package is success - - - name: Create virtualenv for tool - # Python 3.6 is minimum we wish to support due to ease of installation on - # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip - # supported by 3.6. Additionally, recent versions of pip are necessary for - # proper dependency resolution of real-world problems with google-cloud-* - # (and third-party) Python packages (20.3+ probably effective minimum). - ansible.builtin.pip: - name: pip>=21.3.1 - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv - - - name: Add google-cloud-storage to virtualenv - ansible.builtin.pip: - name: google-cloud-storage - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl deleted file mode 100644 index ea14780a58..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Install Python Requirements - hosts: localhost - vars: - install_dir: ${install_dir} - virtualenv_path: ${virtualenv_path} - tasks: - - - name: Install dependencies - ansible.builtin.pip: - requirements: "{{ install_dir }}/requirements.txt" - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl deleted file mode 100644 index ca48a5afa0..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Install Software - hosts: localhost - vars: - sw_name: ${sw_name} - profile_script: ${profile_script} - install_dir: ${install_dir} - git_url: ${git_url} - git_ref: ${git_ref} - chmod_mode: ${chmod_mode} - system_user_name: ${system_user_name} - system_user_uid: ${system_user_uid} - system_user_gid: ${system_user_gid} - finalize_setup_script: ${finalize_setup_script} - profile_script_path: ${profile_script_path} - tasks: - - name: Print software name - ansible.builtin.debug: - msg: "Running installation for software: {{ sw_name }}" - - - name: Add profile script for software - ansible.builtin.copy: - dest: "{{ profile_script_path }}" - mode: '0644' - content: "{{ profile_script }}" - when: profile_script - - - name: Look up user to use for install - block: - - - name: Check if user already exists - ansible.builtin.getent: - database: passwd - key: "{{ system_user_name }}" - - - name: Look up existing user details - ansible.builtin.user: - name: "{{ system_user_name }}" - register: system_user - - rescue: - - name: User did not exist, create group for system user - ansible.builtin.group: - name: "{{ system_user_name }}" - gid: "{{ system_user_gid }}" - system: true - register: system_group - - - name: Create system user - ansible.builtin.user: - name: "{{ system_user_name }}" - comment: "{{ sw_name }} installation" - uid: "{{ system_user_uid }}" - group: "{{ system_group.name }}" - system: true - register: system_user - - - name: Create parent of install directory - ansible.builtin.file: - path: "{{ install_dir | dirname }}" - state: directory - - - name: Set lock dir - ansible.builtin.set_fact: - lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" - - - name: Acquire lock - ansible.builtin.command: - mkdir "{{ lock_dir }}" - register: lock_out - changed_when: lock_out.rc == 0 - failed_when: false - - - name: Add hostname to lock_dir - ansible.builtin.file: - path: "{{ lock_dir }}/{{ ansible_hostname }}" - state: touch - when: lock_out.rc == 0 - - - name: Clone branch or tag into installation directory - ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} - failed_when: false - register: clone_res - when: lock_out.rc == 0 - - - name: Clone commit hash into installation directory - ansible.builtin.command: "{{ item }}" - with_items: - - git clone {{ git_url }} {{ install_dir }} - - git -C {{ install_dir }} checkout {{ git_ref }} - when: lock_out.rc == 0 and clone_res.rc != 0 - - - name: Transfer ownership to system user - ansible.builtin.file: - path: "{{ install_dir }}" - owner: "{{ system_user.name }}" - group: "{{ system_user.group }}" - recurse: true - follow: false - when: lock_out.rc == 0 - - - name: Finalize setup - ansible.builtin.shell: "{{ finalize_setup_script }}" - when: lock_out.rc == 0 and finalize_setup_script - become: true - become_user: "{{ system_user.name }}" - - - name: Apply chmod - ansible.builtin.file: - path: "{{ install_dir }}" - mode: "{{ chmod_mode | default(omit, true) }}" - recurse: true - follow: false - when: (lock_out.rc == 0) and (chmod_mode != None) - - - name: Release lock - ansible.builtin.file: - path: "{{ lock_dir }}/done" - state: touch - when: lock_out.rc == 0 - - - name: Wait for lock - block: - - name: Wait for lock - ansible.builtin.wait_for: - path: "{{ lock_dir }}/done" - state: present - timeout: 600 - sleep: 10 - when: lock_out.rc != 0 - - rescue: - - name: Timed out on waiting for lock, get lock directory contents - ansible.builtin.find: - paths: "{{ lock_dir }}" - register: lock_dir_contents - - - name: Print lock directory contents, it should contain name of host that is holding lock - ansible.builtin.debug: - msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" - - - name: Failed to get lock - ansible.builtin.fail: - msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/variables.tf deleted file mode 100644 index 0d3a8eed05..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/variables.tf +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "install_dir" { - description = "Destination directory of installation of Ramble." - default = "/apps/ramble" - type = string -} - -variable "ramble_url" { - description = "URL for Ramble repository to clone." - default = "https://github.com/GoogleCloudPlatform/ramble" - type = string -} - -variable "ramble_ref" { - description = "Git ref to checkout for Ramble." - default = "develop" - type = string -} - -variable "chmod_mode" { - description = <<-EOT - Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify). - For usage information see: - https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode - EOT - default = "" - type = string - nullable = false -} - -variable "system_user_name" { - description = "Name of system user that will perform installation of Ramble. It will be created if it does not exist." - default = "ramble" - type = string - nullable = false -} - -variable "system_user_uid" { - description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." - default = 1104762904 - type = number - nullable = false -} - -variable "system_user_gid" { - description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." - default = 1104762904 - type = number - nullable = false -} - -variable "ramble_virtualenv_path" { - description = "Virtual environment path in which to install Ramble Python interpreter and other dependencies" - default = "/usr/local/ramble-python" - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing startup script." - type = string -} - -variable "region" { - description = "Region to place bucket containing startup script." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "ramble_profile_script_path" { - description = "Path to the Ramble profile.d script. Created by this module" - type = string - default = "/etc/profile.d/ramble.sh" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/versions.tf deleted file mode 100644 index 936b4a5b80..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/ramble-setup/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/README.md deleted file mode 100644 index 8cbb75fb42..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/README.md +++ /dev/null @@ -1,141 +0,0 @@ -## Description - -This module creates a script that defines a software build using Spack and -performs any additional customization to a Spack installation. - -There are two main variable inputs that can be used to define a Spack build: -`data_files` and `commands`. - -- `data_files`: Any files specified will be transferred to the machine running - outputted script. Data file `content` can be defined inline in the blueprint - or can point to a `source`, an absolute local path of a file. This can be used - to transfer environment definition files, config definition files, GPG keys, - or software licenses. `data_files` are transferred before `commands` are run. -- `commands`: A script that is run. This can be used to perform actions such as - installation of compilers & packages, environment creation, adding a build - cache, and modifying the spack configuration. - -## Example - -The `spack-execute` module should `use` a `spack-setup` module. This will -prepend the installation of Spack and its dependencies to the build. Then -`spack-execute` can be used by a module that takes `startup-script` as an input. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: builder-vm - source: modules/compute/vm-instance - use: [network1, spack-build] -``` - -To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. - -[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml - -### Using with `startup-script` module - -The `spack-runner` output can be used by the `startup-script` module. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(spack-build.spack-runner) - - type: shell - destination: "my-script.sh" - content: echo 'hello world' - - - id: workstation - source: modules/compute/vm-instance - use: [network1, startup-script] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | -| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [log\_file](#input\_log\_file) | Defines the logfile that script output will be written to | `string` | `"/var/log/spack.log"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by an instance of spack-setup.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting. | `string` | n/a | yes | -| [spack\_runner](#input\_spack\_runner) | Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | -| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the spack-setup module. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack startup script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | -| [spack\_runner](#output\_spack\_runner) | Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules. | -| [startup\_script](#output\_startup\_script) | Spack startup script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/main.tf deleted file mode 100644 index 04ebcf7d49..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "spack-execute", ghpc_role = "scripts" }) -} - -locals { - commands_content = var.commands == null ? "echo 'no spack commands provided'" : indent(4, yamlencode(var.commands)) - - execute_contents = templatefile( - "${path.module}/templates/execute_commands.yml.tpl", - { - pre_script = ". ${var.spack_profile_script_path}" - log_file = var.log_file - commands = local.commands_content - system_user_name = var.system_user_name - } - ) - - data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] - - execute_md5 = substr(md5(local.execute_contents), 0, 4) - execute_runner = { - type = "ansible-local" - content = local.execute_contents - destination = "spack_execute_${local.execute_md5}.yml" - } - - runners = concat([var.spack_runner], local.data_runners, [local.execute_runner]) - - # Destinations should be unique while also being known at time of apply - combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) - combined_md5 = substr(md5(local.combined_unique_string), 0, 4) - combined_runner = { - type = "shell" - content = module.startup_script.startup_script - destination = "combined_install_spack_${local.combined_md5}.sh" - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = var.gcs_bucket_path -} - -resource "local_file" "debug_file_ansible_execute" { - content = local.execute_contents - filename = "${path.module}/debug_execute_${local.execute_md5}.yml" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/outputs.tf deleted file mode 100644 index 4a52532d51..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/outputs.tf +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Spack startup script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Spack startup script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "spack_runner" { - description = "Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules." - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." - value = var.gcs_bucket_path -} - -output "spack_profile_script_path" { - description = "Path to the Spack profile.d script." - value = var.spack_profile_script_path -} - -output "system_user_name" { - description = "The system user used to execute commands." - value = var.system_user_name -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl deleted file mode 100644 index 0e98f3aa2c..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Execute Commands - hosts: localhost - vars: - pre_script: ${pre_script} - log_file: ${log_file} - commands: ${commands} - system_user_name: ${system_user_name} - tasks: - - name: Execute command block - block: - - name: Print commands to be executed - ansible.builtin.debug: - msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" - - - name: Streaming log info - ansible.builtin.debug: - msg: | - Logs from commands will not be printed here until success (or failure) - Streaming logs can be found at {{ log_file }} - - - name: Ensure user can write to log file - ansible.builtin.file: - path: "{{ log_file }}" - state: touch - owner: "{{ system_user_name }}" - - - name: Execute commands - ansible.builtin.shell: | - set -eo pipefail - { - {{ pre_script }} - echo " === Starting commands ===" - {{ commands }} - echo " === Finished commands ===" - } 2>&1 | tee -a {{ log_file }} - args: - executable: /bin/bash - register: output - become: true - become_user: "{{ system_user_name }}" - - always: - - name: Print commands output - ansible.builtin.debug: - var: output.stdout_lines diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/variables.tf deleted file mode 100644 index 851cd1aed8..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/variables.tf +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing spack scripts." - type = string -} - -variable "region" { - description = "Region to place bucket containing spack scripts." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "log_file" { - description = "Defines the logfile that script output will be written to" - default = "/var/log/spack.log" - type = string -} - -variable "data_files" { - description = <<-EOT - A list of files to be transferred prior to running commands. - It must specify one of 'source' (absolute local file path) or 'content' (string). - It must specify a 'destination' with absolute path where file should be placed. - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) - error_message = "All destinations must be absolute paths and start with '/'." - } - validation { - condition = alltrue([ - for r in var.data_files : - can(r["content"]) != can(r["source"]) - ]) - error_message = "A data_file must specify either 'content' or 'source', but never both." - } - validation { - condition = alltrue([ - for r in var.data_files : - lookup(r, "content", lookup(r, "source", null)) != null - ]) - error_message = "A data_file must specify a non-null 'content' or 'source'." - } -} - -variable "commands" { - description = "String of commands to run within this module" - type = string - default = null -} - -variable "spack_runner" { - description = "Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module." - type = object({ - type = string - content = string - destination = string - }) -} - -variable "system_user_name" { - description = "Name of the system user used to execute commands. Generally passed from the spack-setup module." - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string -} - -variable "spack_profile_script_path" { - description = <<-EOT - Path to the Spack profile.d script. Created by an instance of spack-setup. - Can be defined explicitly, or by chaining an instance of a spack-setup module - through a `use` setting. - EOT - type = string -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/versions.tf deleted file mode 100644 index 09583c3d43..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-execute/versions.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 1.0.0" - required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/README.md deleted file mode 100644 index 01d3e6d389..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/README.md +++ /dev/null @@ -1,382 +0,0 @@ -## Description - -This module can be used to setup and install Spack on a VM. To actually run -Spack commands to install other software use the -[spack-execute](../spack-execute/) module. - -This module generates a script that performs the following: - -1. Install system dependencies needed for Spack -1. Clone Spack into a predefined directory -1. Check out a specific version of Spack - -There are several options on how to consume the outputs of this module: - -> [!IMPORTANT] -> Breaking changes between after v1.21.0. `spack-install` module replaced by -> `spack-setup` and `spack-execute` modules. -> [Details Below](#deprecations-and-breaking-changes) - -## Examples - -### `use` `spack-setup` with `spack-execute` - -This will prepend the `spack-setup` script to the `spack-execute` commands. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: builder - source: modules/compute/vm-instance - use: [network1, spack-build] -``` - -### `use` `spack-setup` with `vm-instance` or Slurm module - -This will run `spack-setup` scripts on the downstream compute resource. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-installer - source: modules/compute/vm-instance - use: [network1, spack-setup] -``` - -OR - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: [network1, partition1, spack-setup] -``` - -### Build `starup-script` with `spack-runner` output - -This will use the generated `spack-setup` script as one step in `startup-script`. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(spack-setup.spack-runner) - - type: shell - destination: "my-script.sh" - content: echo 'hello world' - - - id: workstation - source: modules/compute/vm-instance - use: [network1, startup-script] -``` - -To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. - -[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml - -## Environment Setup - -### Activating Spack - -[Spack installation] produces a setup script that adds `spack` to your `PATH` as -well as some other command-line integration tools. This script can be found at -`/share/spack/setup-env.sh`. This script will be automatically -added to bash startup by any machine that runs the `spack_runner`. - -If you have multiple machines that all want to use the same shared Spack -installation you can just have both machines run the `spack_runner`. - -[Spack installation]: https://spack-tutorial.readthedocs.io/en/latest/tutorial_basics.html#installing-spack - -### Managing Spack Python dependencies - -Spack is configured with [SPACK_PYTHON] to ensure that Spack itself uses a -Python virtual environment with a supported copy of Python with the package -`google-cloud-storage` pre-installed. This enables Spack to use mirrors and -[build caches][builds] on Google Cloud Storage. It does not configure Python -packages *inside* Spack virtual environments. If you need to add more Python -dependencies for Spack itself, use the `spack python` command: - -```shell -sudo -i spack python -m pip install package-name -``` - -[SPACK_PYTHON]: https://spack.readthedocs.io/en/latest/getting_started.html#shell-support -[builds]: https://spack.readthedocs.io/en/latest/binary_caches.html - -## Spack Permissions - -### System `spack` user is created - Default - -By default this module will create a `spack` linux user and group with -consistent UID and GID. This user and group will own the Spack installation. To -allow a user to manually add Spack packages to the system Spack installation, -you can add the user to the spack group: - -```sh -sudo usermod -a -G spack -``` - -Log out and back in so the group change will take effect, then `` will -be able to call `spack install `. - -> [!NOTE] -> A background persistent SSH connections may prevent the group change from -> taking effect. - -You can use the `system_user_name`, `system_user_uid`, and `system_user_gid` to -customize the name and ids of the system user. While unlikely, it is possible -that the default `system_user_uid` or `system_user_gid` could conflict with -existing UIDs. - -### Use and existing user - -Alternatively, if `system_user_name` is a user already on the system, then this -existing user will be used for Spack installation. - -#### OS Login User - -If OS Login is enabled (default for most Cluster Toolkit modules) then you can -provide an OS Login user name: - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - system_user_name: username_company_com -``` - -This will work even if the user has not yet logged onto the machine. When the -specified user does log on to the machine they will be able to call -`spack install` without any further configuration. - -#### Pre-configured user - -You can also use a startup script to configure a user: - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - system_user_name: special-user - - - id: startup - source: modules/scripts/startup-script - settings: - runners: - - type: shell - destination: "create_user.sh" - content: | - #!/bin/bash - sudo useradd -u 799 special-user - sudo groupadd -g 922 org-group - sudo usermod -g org-group special-user - - $(spack-setup.spack_runner) - - - id: spack-vms - source: modules/compute/vm-instance - use: [network1, startup] - settings: - name_prefix: spack-vm - machine_type: n2d-standard-2 - instance_count: 5 -``` - -### Chaining spack installations - -If there is a need to have a non-root user to install spack packages it is -recommended to create a separate installation for that user and chain Spack installations -([Spack docs](https://spack.readthedocs.io/en/latest/chain.html#chaining-spack-installations)). - -Steps to chain Spack installations: - -1. Get the version of the system Spack: - - ```sh - $ spack --version - - 0.20.0 (e493ab31c6f81a9e415a4b0e0e2263374c61e758) - # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - # Note commit hash and use in next step - ``` - -1. Clone a new spack installation: - - ```sh - git clone -c feature.manyFiles=true https://github.com/spack/spack.git /spack - git -C /spack checkout - ``` - -1. Point the new Spack installation to the system Spack installation. Create a - file at `/spack/etc/spack/upstreams.yaml` with the following - contents: - - ```yaml - upstreams: - spack-instance-1: - install_tree: /sw/spack/opt/spack/ - ``` - -1. Add the following line to your `.bashrc` to make sure the new `spack` is in - your `PATH`. - - ```sh - . /spack/share/spack/setup-env.sh - ``` - -## Deprecations and Breaking Changes - -The old `spack-install` module has been replaced by the `spack-setup` and -`spack-execute` modules. Generally this change strives to allow for a more -flexible definition of a Spack build by using native Spack commands. - -For every deprecated variable from `spack-install` there is documentation on how -to perform the equivalent action using `commands` and `data_files`. The -documentation can be found on the [inputs table](#inputs) below. - -Below is a simple example of the same functionality shown before and after the -breaking changes. - -```yaml - # Before - - id: spack-install - source: community/modules/scripts/spack-install - settings: - install_dir: /sw/spack - compilers: - - gcc@10.3.0 target=x86_64 - packages: - - intel-mpi@2018.4.274%gcc@10.3.0 - -- id: spack-startup - source: modules/scripts/startup-script - settings: - runners: - - $(spack.install_spack_deps_runner) - - $(spack.install_spack_runner) -``` - -```yaml - # After - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - install_dir: /sw/spack - - - id: spack-execute - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - spack load gcc@10.3.0 target=x86_64 - spack compiler find --scope site - spack install intel-mpi@2018.4.274%gcc@10.3.0 - -- id: spack-startup - source: modules/scripts/startup-script - settings: - runners: - - $(spack-execute.spack-runner) -``` - -Although the old `spack-install` module will no longer be maintained, it is -still possible to use the old module in a blueprint by referencing an old -version from GitHub. Note the source line in the following example. - -```yaml - - id: spack-install - source: github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/spack-install?ref=v1.22.1&depth=1 -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 4.42 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [chmod\_mode](#input\_chmod\_mode) | `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification.
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `"g+w"` | no | -| [configure\_for\_google](#input\_configure\_for\_google) | When true, the spack installation will be configured to pull from Google's Spack binary cache. | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | -| [install\_dir](#input\_install\_dir) | Directory to install spack into. | `string` | `"/sw/spack"` | no | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by this module | `string` | `"/etc/profile.d/spack.sh"` | no | -| [spack\_ref](#input\_spack\_ref) | Git ref to checkout for spack. | `string` | `"v0.20.0"` | no | -| [spack\_url](#input\_spack\_url) | URL to clone the spack repo from. | `string` | `"https://github.com/spack/spack"` | no | -| [spack\_virtualenv\_path](#input\_spack\_virtualenv\_path) | Virtual environment path in which to install Spack Python interpreter and other dependencies | `string` | `"/usr/local/spack-python"` | no | -| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Spack. It will be created if it does not exist. | `string` | `"spack"` | no | -| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack installation script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | -| [spack\_path](#output\_spack\_path) | Path to the root of the spack installation | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | -| [spack\_runner](#output\_spack\_runner) | Runner to be used with startup-script module or passed to spack-execute module.
- installs Spack dependencies
- installs Spack
- generates profile.d script to enable access to Spack
This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. | -| [startup\_script](#output\_startup\_script) | Spack installation script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Spack. It can be reused by spack-execute module to install spack packages. | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/main.tf deleted file mode 100644 index d45f5d1be3..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/main.tf +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "spack-setup", ghpc_role = "scripts" }) -} - -locals { - profile_script = <<-EOF - SPACK_PYTHON=${var.spack_virtualenv_path}/bin/python3 - if [ -f ${var.install_dir}/share/spack/setup-env.sh ]; then - test -t 1 && echo "Running Spack setup, this may take a moment on first login." - . ${var.install_dir}/share/spack/setup-env.sh - fi - EOF - - supported_cache_versions = ["v0.19.0", "v0.20.0"] - cache_version = contains(local.supported_cache_versions, var.spack_ref) ? var.spack_ref : "latest" - add_google_mirror_script = !var.configure_for_google ? "" : <<-EOF - if ! spack mirror list | grep -q google_binary_cache; then - spack mirror add --scope site google_binary_cache gs://spack/${local.cache_version} - spack buildcache keys --install --trust - fi - EOF - - finalize_setup_script = <<-EOF - set -e - . ${var.spack_profile_script_path} - spack config --scope site add 'packages:all:permissions:read:world' - spack config --scope site add 'packages:all:permissions:write:group' - spack gpg init - spack compiler find --scope site - ${local.add_google_mirror_script} - # perform fast install to make sure Spack is fully initialized - spack install xz - spack uninstall --yes-to-all xz - EOF - - script_content = templatefile( - "${path.module}/templates/spack_setup.yml.tftpl", - { - sw_name = "spack" - profile_script = indent(4, yamlencode(local.profile_script)) - install_dir = var.install_dir - git_url = var.spack_url - git_ref = var.spack_ref - chmod_mode = var.chmod_mode - system_user_name = var.system_user_name - system_user_uid = var.system_user_uid - system_user_gid = var.system_user_gid - finalize_setup_script = indent(4, yamlencode(local.finalize_setup_script)) - profile_script_path = var.spack_profile_script_path - } - ) - - install_spack_deps_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/install_spack_deps.yml" - "destination" = "install_spack_deps.yml" - "args" = "-e virtualenv_path=${var.spack_virtualenv_path}" - } - install_spack_runner = { - "type" = "ansible-local" - "content" = local.script_content - "destination" = "install_spack.yml" - } - - bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}.${local.script_content}"), 0, 8) - # Max bucket name length is 63, so truncate deployment_name if necessary. - # The string "-spack-scripts-" is 15 characters and bucket_md5 is 8 characters, - # leaving 63-15-8=40 chars for deployment_name. Using 39 so it has the same prefix as the - # ramble-setup module's GCS bucket. - bucket_name = "${substr(var.deployment_name, 0, 39)}-spack-scripts-${local.bucket_md5}" - runners = [local.install_spack_deps_runner, local.install_spack_runner] - - combined_runner = { - "type" = "shell" - "content" = module.startup_script.startup_script - "destination" = "spack-install-and-setup.sh" - } -} - -resource "google_storage_bucket" "bucket" { - project = var.project_id - name = local.bucket_name - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" -} - -resource "local_file" "debug_file_shell_install" { - content = local.script_content - filename = "${path.module}/debug_install.yml" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml deleted file mode 100644 index 2ada34471f..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/outputs.tf deleted file mode 100644 index d94b9757db..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/outputs.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Spack installation script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Spack installation script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "spack_path" { - description = "Path to the root of the spack installation" - value = var.install_dir -} - -output "spack_runner" { - description = <<-EOT - Runner to be used with startup-script module or passed to spack-execute module. - - installs Spack dependencies - - installs Spack - - generates profile.d script to enable access to Spack - This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. - EOT - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." - value = "gs://${google_storage_bucket.bucket.name}" -} - -output "spack_profile_script_path" { - description = "Path to the Spack profile.d script." - value = var.spack_profile_script_path -} - -output "system_user_name" { - description = "The system user used to install Spack. It can be reused by spack-execute module to install spack packages." - value = var.system_user_name -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml deleted file mode 100644 index b7905bbe9e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Create python virtual env for a tool - become: yes - hosts: localhost - vars: - virtualenv_path: ${virtualenv_path} - tasks: - - name: Install dependencies through system package manager - ansible.builtin.package: - name: - - python3 - - python3-pip - - git - register: package - changed_when: package.changed - retries: 5 - delay: 10 - until: package is success - - - name: Create virtualenv for tool - # Python 3.6 is minimum we wish to support due to ease of installation on - # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip - # supported by 3.6. Additionally, recent versions of pip are necessary for - # proper dependency resolution of real-world problems with google-cloud-* - # (and third-party) Python packages (20.3+ probably effective minimum). - ansible.builtin.pip: - name: pip>=21.3.1 - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv - - - name: Add google-cloud-storage to virtualenv - ansible.builtin.pip: - name: google-cloud-storage - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl deleted file mode 100644 index ca48a5afa0..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Install Software - hosts: localhost - vars: - sw_name: ${sw_name} - profile_script: ${profile_script} - install_dir: ${install_dir} - git_url: ${git_url} - git_ref: ${git_ref} - chmod_mode: ${chmod_mode} - system_user_name: ${system_user_name} - system_user_uid: ${system_user_uid} - system_user_gid: ${system_user_gid} - finalize_setup_script: ${finalize_setup_script} - profile_script_path: ${profile_script_path} - tasks: - - name: Print software name - ansible.builtin.debug: - msg: "Running installation for software: {{ sw_name }}" - - - name: Add profile script for software - ansible.builtin.copy: - dest: "{{ profile_script_path }}" - mode: '0644' - content: "{{ profile_script }}" - when: profile_script - - - name: Look up user to use for install - block: - - - name: Check if user already exists - ansible.builtin.getent: - database: passwd - key: "{{ system_user_name }}" - - - name: Look up existing user details - ansible.builtin.user: - name: "{{ system_user_name }}" - register: system_user - - rescue: - - name: User did not exist, create group for system user - ansible.builtin.group: - name: "{{ system_user_name }}" - gid: "{{ system_user_gid }}" - system: true - register: system_group - - - name: Create system user - ansible.builtin.user: - name: "{{ system_user_name }}" - comment: "{{ sw_name }} installation" - uid: "{{ system_user_uid }}" - group: "{{ system_group.name }}" - system: true - register: system_user - - - name: Create parent of install directory - ansible.builtin.file: - path: "{{ install_dir | dirname }}" - state: directory - - - name: Set lock dir - ansible.builtin.set_fact: - lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" - - - name: Acquire lock - ansible.builtin.command: - mkdir "{{ lock_dir }}" - register: lock_out - changed_when: lock_out.rc == 0 - failed_when: false - - - name: Add hostname to lock_dir - ansible.builtin.file: - path: "{{ lock_dir }}/{{ ansible_hostname }}" - state: touch - when: lock_out.rc == 0 - - - name: Clone branch or tag into installation directory - ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} - failed_when: false - register: clone_res - when: lock_out.rc == 0 - - - name: Clone commit hash into installation directory - ansible.builtin.command: "{{ item }}" - with_items: - - git clone {{ git_url }} {{ install_dir }} - - git -C {{ install_dir }} checkout {{ git_ref }} - when: lock_out.rc == 0 and clone_res.rc != 0 - - - name: Transfer ownership to system user - ansible.builtin.file: - path: "{{ install_dir }}" - owner: "{{ system_user.name }}" - group: "{{ system_user.group }}" - recurse: true - follow: false - when: lock_out.rc == 0 - - - name: Finalize setup - ansible.builtin.shell: "{{ finalize_setup_script }}" - when: lock_out.rc == 0 and finalize_setup_script - become: true - become_user: "{{ system_user.name }}" - - - name: Apply chmod - ansible.builtin.file: - path: "{{ install_dir }}" - mode: "{{ chmod_mode | default(omit, true) }}" - recurse: true - follow: false - when: (lock_out.rc == 0) and (chmod_mode != None) - - - name: Release lock - ansible.builtin.file: - path: "{{ lock_dir }}/done" - state: touch - when: lock_out.rc == 0 - - - name: Wait for lock - block: - - name: Wait for lock - ansible.builtin.wait_for: - path: "{{ lock_dir }}/done" - state: present - timeout: 600 - sleep: 10 - when: lock_out.rc != 0 - - rescue: - - name: Timed out on waiting for lock, get lock directory contents - ansible.builtin.find: - paths: "{{ lock_dir }}" - register: lock_dir_contents - - - name: Print lock directory contents, it should contain name of host that is holding lock - ansible.builtin.debug: - msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" - - - name: Failed to get lock - ansible.builtin.fail: - msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/variables.tf deleted file mode 100644 index 85baeec401..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/variables.tf +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -# spack-setup variables - -variable "install_dir" { - description = "Directory to install spack into." - type = string - default = "/sw/spack" -} - -variable "spack_url" { - description = "URL to clone the spack repo from." - type = string - default = "https://github.com/spack/spack" -} - -variable "spack_ref" { - description = "Git ref to checkout for spack." - type = string - default = "v0.20.0" -} - -variable "configure_for_google" { - description = "When true, the spack installation will be configured to pull from Google's Spack binary cache." - type = bool - default = true -} - - -variable "chmod_mode" { - description = <<-EOT - `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification. - For usage information see: - https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode - EOT - default = "g+w" - type = string - nullable = false -} - -variable "system_user_name" { - description = "Name of system user that will perform installation of Spack. It will be created if it does not exist." - default = "spack" - type = string - nullable = false -} - -variable "system_user_uid" { - description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." - default = 1104762903 - type = number - nullable = false -} - -variable "system_user_gid" { - description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." - default = 1104762903 - type = number - nullable = false -} - -variable "spack_virtualenv_path" { - description = "Virtual environment path in which to install Spack Python interpreter and other dependencies" - default = "/usr/local/spack-python" - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing startup script." - type = string -} - -variable "region" { - description = "Region to place bucket containing startup script." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "spack_profile_script_path" { - description = "Path to the Spack profile.d script. Created by this module" - type = string - default = "/etc/profile.d/spack.sh" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/versions.tf deleted file mode 100644 index ff1180fc1b..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/spack-setup/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/README.md deleted file mode 100644 index ee9c057c39..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/README.md +++ /dev/null @@ -1,87 +0,0 @@ -## Description - -This module will insert a dependency on the completion of the startup script -for one or more specified compute VMs and report back if it fails. This can be useful when running -post-boot installation scripts that require the startup script to finish setting up a node. - -> **_WARNING:_**: this module is experimental and not fully supported. - -### Additional Dependencies - -* [**gcloud**](https://cloud.google.com/sdk/gcloud) must be present in the path - of the machine where `terraform apply` is run. - -### Example - -```yaml -- id: workstation - source: modules/compute/vm-instance - use: - - network1 - - my-startup-script - settings: - instance_count: 4 - -# Wait for all instances of the above VM to finish running startup scripts. -- id: wait - source: community/modules/scripts/wait-for-startup - settings: - instance_names: $(workstation.name) -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.validate_instance_names](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.wait_for_startup](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | -| [instance\_name](#input\_instance\_name) | Name of the instance we are waiting for (can be null if 'instance\_names' is not empty) | `string` | `null` | no | -| [instance\_names](#input\_instance\_names) | A list of instance names we are waiting for, in addition to the one mentioned in 'instance\_name' (if any) | `list(string)` | `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [timeout](#input\_timeout) | Timeout in seconds | `number` | `1200` | no | -| [zone](#input\_zone) | The GCP zone where the instance is running | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/main.tf deleted file mode 100644 index 3f6b416251..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/main.tf +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - combined_instance_names = concat(var.instance_names, [var.instance_name]) -} - -resource "null_resource" "validate_instance_names" { - lifecycle { - precondition { - condition = var.instance_name != null || length(var.instance_names) > 0 - error_message = "At least one instance name must be provided" - } - } -} - -resource "null_resource" "wait_for_startup" { - count = length(local.combined_instance_names) - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/wait-for-startup-status.sh" - environment = { - INSTANCE_NAME = self.triggers.instance_name - ZONE = var.zone - PROJECT_ID = var.project_id - TIMEOUT = var.timeout - GCLOUD_PATH = var.gcloud_path_override - } - } - - triggers = { - instance_name = local.combined_instance_names[count.index] - } -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf deleted file mode 100644 index 11a2ddf118..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh deleted file mode 100644 index fae5833121..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -z "${INSTANCE_NAME}" ]]; then - echo "INSTANCE_NAME is unset... exiting" - exit 0 -fi -if [[ -z "${ZONE}" ]]; then - echo "ZONE is unset" - exit 1 -fi -if [[ -z "${PROJECT_ID}" ]]; then - echo "PROJECT_ID is unset" - exit 1 -fi -if [[ -z "${TIMEOUT}" ]]; then - echo "TIMEOUT is unset" - exit 1 -fi - -if [[ -n "${GCLOUD_PATH}" ]]; then - export PATH="$GCLOUD_PATH:$PATH" -fi - -echo "Waiting for startup: instance_name='${INSTANCE_NAME}', zone='${ZONE}', project_id='${PROJECT_ID}', timeout_seconds='${TIMEOUT}'" - -# Wrapper around grep that swallows the error status code 1 -c1grep() { grep "$@" || test $? = 1; } - -now=$(date +%s) - -# If VM was created more than 30 days ago, serial port logs may no longer exist. -# Exit without errors if the instance is older than 30 days. -logsExpiryDays=30 -createdTimestampIso=$(gcloud compute instances describe "${INSTANCE_NAME}" --project "${PROJECT_ID}" --zone "${ZONE}" --format "value(creationTimestamp)") -earliestAllowedCreatedTimestamp=$(date -d "${createdTimestampIso} +${logsExpiryDays} day" +%s) -if [[ "$earliestAllowedCreatedTimestamp" -lt "$now" ]]; then - echo "Instance was created more than 30 days ago - serial port 1 logs are likely expired... exiting" - exit 0 -fi - -deadline=$((now + TIMEOUT)) -error_file=$(mktemp) -fetch_cmd="gcloud compute instances get-serial-port-output ${INSTANCE_NAME} --port 1 --zone ${ZONE} --project ${PROJECT_ID}" -# Match string for all finish types of the old guest agent and successful -# finishes on the new guest agent -FINISH_LINE="startup-script exit status" -# Match string for failures on the new guest agent -FINISH_LINE_ERR="Script \"startup-script\" failed with error:" - -# NEW: Accept also these finish lines as success. -STARTUP_SCRIPT_SUCCEEDED_LINE="google-startup-scripts.service: Succeeded." -STARTUP_SCRIPT_FINISHED_LINE="Finished Google Compute Engine Startup Scripts." -STARTUP_SCRIPT_SERVICE_FINISHED_LINE="Finished google-startup-scripts.service - Google Compute Engine Startup Scripts." - -NON_FATAL_ERRORS=( - "Internal error" -) - -until [[ now -gt deadline ]]; do - ser_log=$( - set -o pipefail - ${fetch_cmd} 2>"${error_file}" | - c1grep "${FINISH_LINE}\|${FINISH_LINE_ERR}\|${STARTUP_SCRIPT_SUCCEEDED_LINE}\|${STARTUP_SCRIPT_FINISHED_LINE}\|${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}" - ) || { - err=$(cat "${error_file}") - echo "$err" - fatal_error="true" - for e in "${NON_FATAL_ERRORS[@]}"; do - if [[ $err = *"$e"* ]]; then - fatal_error="false" - break - fi - done - - if [[ $fatal_error = "true" ]]; then - exit 1 - fi - } - if [[ -n "${ser_log}" ]]; then break; fi - sleep 5 - now=$(date +%s) -done - -# This line checks for an exit code - the assumption is that there is a number -# at the end of the line and it is an exit code. -# Modified to correctly extract the last numeric exit status from the relevant log line. -LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=Script \"startup-script\" failed with error: exit status )[0-9]+" | tail -n 1) -if [[ -z "${LAST_EXIT_STATUS}" ]]; then - LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=startup-script exit status )[0-9]+" | tail -n 1) -fi - -# This specific text is monitored for in tests, do not change. -INSPECT_OUTPUT_TEXT="To inspect the startup script output, please run:" - -# --- Prioritize explicit failure from the script itself --- -if [[ "${LAST_EXIT_STATUS}" == 1 ]]; then - echo "startup-script finished with errors, ${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" - exit 1 -# --- Then explicit success from the script itself --- -elif [[ "${LAST_EXIT_STATUS}" == 0 ]]; then - echo "startup-script finished successfully" - exit 0 -elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SUCCEEDED_LINE}"; then - echo "startup-script finished successfully (startup script succeeded line detected)" - exit 0 -elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_FINISHED_LINE}"; then - echo "startup-script finished successfully (startup script finished line detected)" - exit 0 -elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}"; then - echo "startup-script finished successfully (startup script service finished line detected)" - exit 0 -# --- If we reached deadline, it's a timeout --- -elif [[ now -ge deadline ]]; then - echo "startup-script timed out after ${TIMEOUT} seconds" - echo "${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" - exit 1 -# --- All other cases are considered failure or invalid state --- -else - echo "Invalid or undetermined startup script status. Last detected exit status: '${LAST_EXIT_STATUS}'" - echo "${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" - exit "${LAST_EXIT_STATUS}" -fi diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf deleted file mode 100644 index fe6410a920..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "instance_name" { - description = "Name of the instance we are waiting for (can be null if 'instance_names' is not empty)" - type = string - default = null -} - -variable "instance_names" { - description = "A list of instance names we are waiting for, in addition to the one mentioned in 'instance_name' (if any)" - type = list(string) - default = [] -} - -variable "zone" { - description = "The GCP zone where the instance is running" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "timeout" { - description = "Timeout in seconds" - type = number - default = 1200 - validation { - condition = var.timeout >= 0 - error_message = "The timeout should be non-negative" - } -} - -variable "gcloud_path_override" { - description = "Directory of the gcloud executable to be used during cleanup" - type = string - default = "" - nullable = false -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf deleted file mode 100644 index 8cd43b944e..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/README.md b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/README.md deleted file mode 100644 index fc25bc0a55..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/README.md +++ /dev/null @@ -1,109 +0,0 @@ -## Description - -This module contains a set of scripts to be used in customizing Windows VMs at -boot or during image building. Please note that the installation of NVIDIA GPU -drivers takes, at minimum, 30-60 minutes. It is therefore recommended to build -a custom image and reuse it as shown below, rather than install GPU drivers at -boot time. - -> NOTE: the output `windows_startup_ps1` must be passed explicitly as shown -> below when used with Packer modules. This is due to a limitation in the `use` -> keyword and inputs of type `list` in Packer modules; this does not impact -> Terraform modules - -### NVIDIA Drivers and CUDA Toolkit - -Many Google Cloud VM families include or can have NVIDIA GPUs attached to them. -This module supports GPU applications by enabling you to easily install -a compatible release of NVIDIA drivers and of the CUDA Toolkit. The script is -the [solution recommended by our documentation][docs] and is [directly sourced -from GitHub][script-src]. - -[docs]: https://cloud.google.com/compute/docs/gpus/install-drivers-gpu#windows -[script-src]: https://github.com/GoogleCloudPlatform/compute-gpu-installation/blob/24dac3004360e0696c49560f2da2cd60fcb80107/windows/install_gpu_driver.ps1 - -```yaml -- group: primary - modules: - - id: network1 - source: modules/network/vpc - settings: - enable_iap_rdp_ingress: true - enable_iap_winrm_ingress: true - - - id: windows_startup - source: community/modules/scripts/windows-startup-script - settings: - install_nvidia_driver: true - -- group: packer - modules: - - id: image - source: modules/packer/custom-image - kind: packer - use: - - network1 - - windows_startup - settings: - source_image_family: windows-2016 - machine_type: n1-standard-8 - accelerator_count: 1 - accelerator_type: nvidia-tesla-t4 - disk_size: 75 - disk_type: pd-ssd - omit_external_ip: false - state_timeout: 15m -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [http\_proxy](#input\_http\_proxy) | Set http and https proxy for use by Invoke-WebRequest commands | `string` | `""` | no | -| [http\_proxy\_set\_environment](#input\_http\_proxy\_set\_environment) | Set system default environment variables http\_proxy and https\_proxy for all commands | `bool` | `false` | no | -| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install\_nvidia\_driver\_script | `bool` | `false` | no | -| [install\_nvidia\_driver\_args](#input\_install\_nvidia\_driver\_args) | Arguments to supply to NVIDIA driver install script | `string` | `"/s /n"` | no | -| [install\_nvidia\_driver\_script](#input\_install\_nvidia\_driver\_script) | Install script for NVIDIA drivers specified by http/https URL | `string` | `"https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe"` | no | -| [no\_proxy](#input\_no\_proxy) | Environment variables no\_proxy (only used if var.http\_proxy\_set\_environment is enabled) | `string` | `"169.254.169.254,metadata,metadata.google.internal,.googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | A string list of scripts selected by this module | - diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/main.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/main.tf deleted file mode 100644 index 5e6bc8b94d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/main.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - setx_http_proxy_ps1 = !var.http_proxy_set_environment ? [] : [ - templatefile("${path.module}/templates/setx_http_proxy.ps1", { - "http_proxy" : var.http_proxy, - "no_proxy" : var.no_proxy, - }) - ] - - nvidia_ps1 = !var.install_nvidia_driver ? [] : [ - templatefile("${path.module}/templates/install_gpu_driver.ps1.tftpl", { - "url" : var.install_nvidia_driver_script - "args" : var.install_nvidia_driver_args - "http_proxy" : var.http_proxy, - }) - ] - - startup_ps1 = concat(local.setx_http_proxy_ps1, local.nvidia_ps1) -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf deleted file mode 100644 index 006ea312ad..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "windows_startup_ps1" { - description = "A string list of scripts selected by this module" - value = local.startup_ps1 -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl deleted file mode 100644 index 55c4a2a3cd..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl +++ /dev/null @@ -1,38 +0,0 @@ -#Requires -RunAsAdministrator - -# Windows 2016 needs forced upgrade to TLS 1.2 -[Net.ServicePointManager]::SecurityProtocol = 'Tls12' - -# important for catching exception in Invoke-WebRequest -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -%{ if http_proxy != "" } -[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") -%{ endif } - -# Create the folder for the driver download -$file_dir = 'C:\NVIDIA-Driver\nvidia_installer_windows.exe' -if (!(Test-Path -Path 'C:\NVIDIA-Driver')) { - New-Item -Path 'C:\' -Name 'NVIDIA-Driver' -ItemType 'directory' | Out-Null -} - -# Download the file to a specified directory -Write-Output "Downloading ${url} to $file_dir" -# Disabling progress bar has surprising large (10-100x) impact on speed -$ProgressPreference = 'SilentlyContinue' -try { - Invoke-WebRequest -Uri "${url}" -OutFile "$file_dir" -} catch { - Write-Output "$_" - throw "Failed to download ${url}; exiting startup script" -} - -# Install the file with the specified path from earlier as well as the RunAs admin option -Write-Output "Executing $file_dir with arguments '${args}'" -try { - Start-Process -FilePath "$file_dir" -ArgumentList '${args}' -Wait -} catch { - Write-Output "$_" - throw "Could not install NVIDIA driver; exiting startup script" -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 deleted file mode 100644 index ca4d13f98b..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 +++ /dev/null @@ -1,21 +0,0 @@ -<# - Copyright 2025 "Google LLC" - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -#> - -#Requires -RunAsAdministrator - -setx http_proxy ${http_proxy} /m -setx https_proxy ${http_proxy} /m -setx no_proxy ${no_proxy} /m diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf deleted file mode 100644 index 9e4fb9e67d..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "install_nvidia_driver" { - description = "Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install_nvidia_driver_script" - type = bool - default = false -} - -variable "install_nvidia_driver_script" { - description = "Install script for NVIDIA drivers specified by http/https URL" - type = string - default = "https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe" -} - -variable "install_nvidia_driver_args" { - description = "Arguments to supply to NVIDIA driver install script" - type = string - default = "/s /n" -} - -variable "http_proxy" { - description = "Set http and https proxy for use by Invoke-WebRequest commands" - type = string - default = "" - nullable = false -} - -variable "http_proxy_set_environment" { - description = "Set system default environment variables http_proxy and https_proxy for all commands" - type = bool - default = false - nullable = false -} - -variable "no_proxy" { - description = "Environment variables no_proxy (only used if var.http_proxy_set_environment is enabled)" - type = string - default = "169.254.169.254,metadata,metadata.google.internal,.googleapis.com" - nullable = false -} diff --git a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf b/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf deleted file mode 100644 index dfeeac34f8..0000000000 --- a/deletion-test/cluster/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/cluster/modules/embedded/modules/README.md b/deletion-test/cluster/modules/embedded/modules/README.md deleted file mode 100644 index 6886b3f330..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/README.md +++ /dev/null @@ -1,554 +0,0 @@ -# Modules - -This directory contains a set of core modules built for the Cluster Toolkit. Modules -describe the building blocks of an AI/ML and HPC deployment. The expected fields in a -module are listed in more detail [below](#module-fields). Blueprints can be -extended in functionality by incorporating [modules from GitHub -repositories][ghmods]. - -[ghmods]: #github-modules - -## All Modules - -Modules from various sources are all listed here for visibility. Badges are used -to indicate the source and status of many of these resources. - -Modules listed below with the ![core-badge] badge are located in this -folder and are tested and maintained by the Cluster Toolkit team. - -Modules labeled with the ![community-badge] badge are contributed by -the community (including the Cluster Toolkit team, partners, etc.). Community modules -are located in the [community folder](../community/modules/README.md). - -Modules labeled with the ![deprecated-badge] badge are now deprecated and may be -removed in the future. Customers are advised to transition to alternatives. - -Modules that are still in development and less stable are labeled with the -![experimental-badge] badge. - -[core-badge]: https://img.shields.io/badge/-core-blue?style=plastic -[community-badge]: https://img.shields.io/badge/-community-%23b8def4?style=plastic -[stable-badge]: https://img.shields.io/badge/-stable-lightgrey?style=plastic -[experimental-badge]: https://img.shields.io/badge/-experimental-%23febfa2?style=plastic -[deprecated-badge]: https://img.shields.io/badge/-deprecated-%23fea2a2?style=plastic - -### Compute - -* **[vm-instance]** ![core-badge] : Creates one or more VM instances. -* **[schedmd-slurm-gcp-v6-partition]** ![core-badge] : - Creates a partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. -* **[schedmd-slurm-gcp-v6-nodeset]** ![core-badge] : - Creates a nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. -* **[schedmd-slurm-gcp-v6-nodeset-tpu]** ![core-badge] : - Creates a TPU nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. -* **[schedmd-slurm-gcp-v6-nodeset-dynamic]** ![core-badge] ![experimental-badge]: - Creates a dynamic nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module and instance template. -* **[gke-node-pool]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes node pool using GKE. -* **[resource-policy]** ![core-badge] ![experimental-badge] : Create a resource policy for compute engines that can be applied to gke-node-pool's nodes. -* **[gke-job-template]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes job file to be used with a [gke-node-pool]. -* **[htcondor-execute-point]** ![community-badge] ![experimental-badge] : - Manages a group of execute points for use in an [HTCondor - pool][htcondor-setup]. -* **[mig]** ![community-badge] ![experimental-badge] : Creates a Managed Instance Group. -* **[notebook]** ![community-badge] ![experimental-badge] : Creates a Vertex AI - Notebook. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. -* **[gke-nodeset]** ![community-badge] ![experimental-badge] : Create a slinky nodeset to be used by the [gke-partition] module. -* **[gke-partition]** ![community-badge] ![experimental-badge] : Creates a slinky partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. - -[vm-instance]: compute/vm-instance/README.md -[gke-node-pool]: ../modules/compute/gke-node-pool/README.md -[resource-policy]: ../modules/compute/resource-policy/README.md -[gke-job-template]: ../modules/compute/gke-job-template/README.md -[schedmd-slurm-gcp-v6-partition]: ../community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md -[schedmd-slurm-gcp-v6-nodeset]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md -[schedmd-slurm-gcp-v6-nodeset-tpu]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md -[schedmd-slurm-gcp-v6-nodeset-dynamic]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md -[htcondor-execute-point]: ../community/modules/compute/htcondor-execute-point/README.md -[mig]: ../community/modules/compute/mig/README.md -[notebook]: ../community/modules/compute/notebook/README.md -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -### Database - -* **[slurm-cloudsql-federation]** ![community-badge] ![experimental-badge] : - Creates a [Google SQL Instance](https://cloud.google.com/sql/) meant to be - integrated with a [slurm-controller][schedmd-slurm-gcp-v6-controller]. -* **[bigquery-dataset]** ![community-badge] ![experimental-badge] : Creates a BQ - dataset. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. -* **[bigquery-table]** ![community-badge] ![experimental-badge] : Creates a BQ - table. Primarily used for - [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. - -[slurm-cloudsql-federation]: ../community/modules/database/slurm-cloudsql-federation/README.md -[bigquery-dataset]: ../community/modules/database/bigquery-dataset/README.md -[bigquery-table]: ../community/modules/database/bigquery-table/README.md -[fsi-montecarlo-on-batch]: ../community/modules/files/fsi-montecarlo-on-batch/README.md - -### File System - -* **[filestore]** ![core-badge] : Creates a - [filestore](https://cloud.google.com/filestore) file system. -* **[parallelstore]** ![core-badge] ![experimental-badge]: Creates a - [parallelstore](https://cloud.google.com/parallelstore) file system. -* **[pre-existing-network-storage]** ![core-badge] : Specifies a - pre-existing file system that can be mounted on a VM. -* **[managed-lustre]** ![core-badge] ![experimental-badge]: Creates a - [managed-lustred](https://cloud.google.com/managed-lustre) file system. -* **[DDN-EXAScaler]** ![community-badge] ![deprecated-badge] : Creates - a [DDN EXAscaler lustre](https://www.ddn.com/partners/google-cloud-platform/) - file system. This module is deprecated and will be removed by July 1, 2025. Consider migrating to managed-lustre. -* **[cloud-storage-bucket]** ![core-badge] : Creates a Google Cloud Storage (GCS) bucket. -* **[gke-persistent-volume]** ![core-badge] ![experimental-badge] : Creates - persistent volumes and persistent volume claims for shared storage. -* **[nfs-server]** ![community-badge] ![experimental-badge] : Creates a VM and - configures an NFS server that can be mounted by other VM. -* **[weka-client]** ![community-badge] ![experimental-badge] : Installs client - and mounts [WEKA](https://www.weka.io/) filesystems. - -[filestore]: file-system/filestore/README.md -[parallelstore]: file-system/parallelstore/README.md -[pre-existing-network-storage]: file-system/pre-existing-network-storage/README.md -[managed-lustre]: file-system/managed-lustre/README.md -[ddn-exascaler]: ../community/modules/file-system/DDN-EXAScaler/README.md -[nfs-server]: ../community/modules/file-system/nfs-server/README.md -[cloud-storage-bucket]: file-system/cloud-storage-bucket/README.md -[gke-persistent-volume]: file-system/gke-persistent-volume/README.md -[weka-client]: ../community/modules/file-system/weka-client/README.md - -### Monitoring - -* **[dashboard]** ![core-badge] : Creates a - [monitoring dashboard](https://cloud.google.com/monitoring/dashboards) for - visually tracking a Cluster Toolkit deployment. - -[dashboard]: monitoring/dashboard/README.md - -### Network - -* **[vpc]** ![core-badge] : Creates a - [Virtual Private Cloud (VPC)](https://cloud.google.com/vpc) network with - regional subnetworks and firewall rules. -* **[multivpc]** ![core-badge] ![experimental-badge]: Creates a variable - number of VPC networks using the [vpc] module. -* **[pre-existing-vpc]** ![core-badge] : Used to connect newly - built components to a pre-existing VPC network. -* **[firewall-rules]** ![core-badge] ![experimental-badge] : Add custom firewall - rules to existing networks (commonly used with [pre-existing-vpc]). -* **[private-service-access]** ![community-badge] ![experimental-badge] : - Configures Private Services Access for a VPC network (commonly used with [filestore] and [slurm-cloudsql-federation]). - -[vpc]: network/vpc/README.md -[multivpc]: network/multivpc/README.md -[pre-existing-vpc]: network/pre-existing-vpc/README.md -[firewall-rules]: network/firewall-rules/README.md -[private-service-access]: ../community/modules/network/private-service-access/README.md - -### Packer - -* **[custom-image]** ![core-badge] : Creates a custom VM Image - based on the GCP HPC VM image. - -[custom-image]: packer/custom-image/README.md - -### Project - -* **[service-account]** ![community-badge] ![experimental-badge] : Creates [service - accounts](https://cloud.google.com/iam/docs/service-accounts) for a GCP - project. -* **[service-enablement]** ![community-badge] ![experimental-badge] : Allows enabling - various APIs for a Google Cloud Project. - -[service-account]: ../community/modules/project/service-account/README.md -[service-enablement]: ../community/modules/project/service-enablement/README.md - -### Pub/Sub - -* **[topic]** ![community-badge] ![experimental-badge] : Creates a -Pub/Sub topic. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. -* **[bigquery-sub]** ![community-badge] ![experimental-badge] : Creates a -Pub/Sub subscription. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. - -[topic]: ../community/modules/pubsub/topic/README.md -[bigquery-sub]: ../community/modules/pubsub/bigquery-sub/README.md - -### Remote Desktop - -* **[chrome-remote-desktop]** ![community-badge] ![experimental-badge] : Creates - a GPU accelerated Chrome Remote Desktop. - -[chrome-remote-desktop]: ../community/modules/remote-desktop/chrome-remote-desktop/README.md - -### Scheduler - -* **[batch-job-template]** ![core-badge] : Creates a Google Cloud Batch job - template that works with other Toolkit modules. -* **[batch-login-node]** ![core-badge] : Creates a VM that can be used for - submission of Google Cloud Batch jobs. -* **[gke-cluster]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes cluster using GKE. -* **[pre-existing-gke-cluster]** ![core-badge] ![experimental-badge] : Retrieves an existing GKE cluster. Substitute for ([gke-cluster]) module. -* **[schedmd-slurm-gcp-v6-controller]** ![core-badge] : - Creates a Slurm controller node. -* **[schedmd-slurm-gcp-v6-login]** ![core-badge] : - Creates a Slurm login node. -* **[htcondor-setup]** ![community-badge] ![experimental-badge] : Creates the - base infrastructure for an HTCondor pool (service accounts and Cloud Storage bucket). -* **[htcondor-pool-secrets]** ![community-badge] ![experimental-badge] : Creates - and manages access to the secrets necessary for secure operation of an - HTCondor pool. -* **[htcondor-access-point]** ![community-badge] ![experimental-badge] : Creates - a regional instance group managing a highly available HTCondor access point - (login node). - -[batch-job-template]: ../modules/scheduler/batch-job-template/README.md -[batch-login-node]: ../modules/scheduler/batch-login-node/README.md -[gke-cluster]: ../modules/scheduler/gke-cluster/README.md -[pre-existing-gke-cluster]: ../modules/scheduler/pre-existing-gke-cluster/README.md -[htcondor-setup]: ../community/modules/scheduler/htcondor-setup/README.md -[htcondor-pool-secrets]: ../community/modules/scheduler/htcondor-pool-secrets/README.md -[htcondor-access-point]: ../community/modules/scheduler/htcondor-access-point/README.md -[schedmd-slurm-gcp-v6-controller]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md -[schedmd-slurm-gcp-v6-login]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md - -### Scripts - -* **[startup-script]** ![core-badge] : Creates a customizable startup script - that can be fed into compute VMs. -* **[windows-startup-script]** ![community-badge] ![experimental-badge]: Creates - Windows PowerShell (PS1) scripts that can be used to customize Windows VMs - and VM images. -* **[htcondor-install]** ![community-badge] ![experimental-badge] : Creates - a startup script to install HTCondor and exports a list of required APIs -* **[ramble-execute]** ![community-badge] ![experimental-badge] : Creates a - startup script to execute - [Ramble](https://github.com/GoogleCloudPlatform/ramble) commands on a target - VM -* **[ramble-setup]** ![community-badge] ![experimental-badge] : Creates a - startup script to install - [Ramble](https://github.com/GoogleCloudPlatform/ramble) on an instance or a - slurm login or controller. -* **[spack-setup]** ![community-badge] ![experimental-badge] : Creates a startup - script to install [Spack](https://github.com/spack/spack) on an instance or a - slurm login or controller. -* **[spack-execute]** ![community-badge] ![experimental-badge] : Defines a - software build using [Spack](https://github.com/spack/spack). -* **[wait-for-startup]** ![community-badge] ![experimental-badge] : Waits for - successful completion of a startup script on a compute VM. - -[startup-script]: scripts/startup-script/README.md -[windows-startup-script]: ../community/modules/scripts/windows-startup-script/README.md -[htcondor-install]: ../community/modules/scripts/htcondor-install/README.md -[kubernetes-operations]: ../community/modules/scripts/kubernetes-operations/README.md -[ramble-execute]: ../community/modules/scripts/ramble-execute/README.md -[ramble-setup]: ../community/modules/scripts/ramble-setup/README.md -[spack-setup]: ../community/modules/scripts/spack-setup/README.md -[spack-execute]: ../community/modules/scripts/spack-execute/README.md -[wait-for-startup]: ../community/modules/scripts/wait-for-startup/README.md - -## Module Fields - -### ID (Required) - -The `id` field is used to uniquely identify and reference a defined module. -ID's are used in [variables](../examples/README.md#variables) and become the -name of each module when writing the terraform `main.tf` file. They are also -used in the [use](#use-optional) and [outputs](#outputs-optional) lists -described below. - -For terraform modules, the ID will be rendered into the terraform module label -at the top level main.tf file. - -### Source (Required) - -The source is a path or URL that points to the source files for Packer or -Terraform modules. A source can either be a filesystem path or a URL to a git -repository: - -* Filesystem paths - * modules embedded in the `gcluster` executable - * modules in the local filesystem -* Remote modules using [Terraform URL syntax](https://developer.hashicorp.com/terraform/language/modules/sources) - * Hosted on [GitHub](https://developer.hashicorp.com/terraform/language/modules/sources#github) - * Google Cloud Storage [Buckets](https://developer.hashicorp.com/terraform/language/modules/sources#gcs-bucket) - * Generic [git repositories](https://developer.hashicorp.com/terraform/language/modules/sources#generic-git-repository) - - when modules are in a subdirectory of the git repository, a special - double-slash `//` notation can be required as described below - -An important distinction is that those URLs are natively supported by Terraform so -they are not copied to your deployment directory. Packer does not have native -support for git-hosted modules so the Toolkit will copy these modules into the -deployment folder on your behalf. - -#### Embedded Modules - -Embedded modules are added to the gcluster binary during compilation and cannot -be edited. To refer to embedded modules, set the source path to -`modules/<>` or `community/modules/<>`. - -The paths match the modules in the repository structure for [core modules](./) -and [community modules](../community/modules/). Because the modules are embedded -during compilation, your local copies may differ unless you recompile gcluster. - -For example, this example snippet uses the embedded pre-existing-vpc module: - -```yaml - - id: network1 - source: modules/network/pre-existing-vpc -``` - -#### Local Modules - -Local modules point to a module in the file system and can easily be edited. -They are very useful during module development. To use a local module, set -the source to a path starting with `/`, `./`, or `../`. For instance, the -following module definition refers the local pre-existing-vpc modules. - -```yaml - - id: network1 - source: modules/network/pre-existing-vpc -``` - -> **_NOTE:_** Relative paths (beginning with `.` or `..` must be relative to the -> working directory from which `gcluster` is executed. This example would have to be -> run from a local copy of the Cluster Toolkit repository. An alternative is to use -> absolute paths to modules. - -#### GitHub-hosted Modules and Packages - -To use a Terraform module available on GitHub, set the source to a path starting -with `github.com` (HTTPS) or `git@github.com` (SSH). For instance, the following -module definition sources the Toolkit vpc module: - -```yaml - - id: network1 - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc -``` - -This example uses the [double-slash notation][tfsubdir] (`//`) to indicate that -the Toolkit is a "package" of multiple modules whose root directory is the root -of the git repository. The remainder of the path indicates the sub-directory of -the vpc module. - -The example above uses the default `main` branch of the Toolkit. Specific -[revisions][tfrev] can be selected with any valid [git reference][gitref]. -(git branch, commit hash or tag). If the git reference is a tag or branch, we -recommend setting `&depth=1` to reduce the data transferred over the network. -This option cannot be set when the reference is a commit hash. The following -examples select the vpc module on the active `develop` branch and also an older -release of the filestore module: - -```yaml - - id: network1 - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc?ref=develop - ... - - id: homefs - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/file-system/filestore?ref=v1.22.1&depth=1 -``` - -Because Terraform modules natively support this syntax, gcluster will not copy -GitHub-hosted modules into your deployment folder. Terraform will download them -into a hidden folder when you run `terraform init`. - -[tfrev]: https://www.terraform.io/language/modules/sources#selecting-a-revision -[gitref]: https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection#_single_revisions -[tfsubdir]: https://www.terraform.io/language/modules/sources#modules-in-package-sub-directories - -##### GitHub-hosted Packer modules - -Packer does not natively support GitHub-hosted modules so `gcluster create` will -copy modules into your deployment folder. - -If the module uses `//` package notation, `gcluster create` will copy the entire -repository to the module path: `deployment_name/group_name/module_id`. However, -when `gcluster deploy` is invoked, it will run Packer from the subdirectory -`deployment_name/group_name/module_id/subdirectory/after/double_slash`. - -If the module does not use `//` package notation, `gcluster create` will copy -only the final directory in the path to `deployment_name/group_name/module_id`. - -In all cases, `gcluster create` will remove the `.git` directory from the packer -module to ensure that you can manage the entire deployment directory with its -own git versioning. - -##### GitHub over SSH - -Get module from GitHub over SSH: - -```yaml - - id: network1 - source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc -``` - -Specific versions can be selected as for HTTPS: - -```yaml - - id: network1 - source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc?ref=v1.22.1&depth=1 -``` - -##### Generic Git Modules - -To use a Terraform module available in a non-GitHub git repository such as -gitlab, set the source to a path starting `git::`. Two Standard git protocols -are supported, `git::https://` for HTTPS or `git::git@github.com` for SSH. - -Additional formatting and features after `git::` are identical to that of the -[GitHub Modules](#github-modules) described above. - -#### Google Cloud Storage Modules - -To use a Terraform module available in a Google Cloud Storage bucket, set the source -to a URL with the special `gcs::` prefix, followed by a [GCS bucket object URL](https://cloud.google.com/storage/docs/request-endpoints#typical). - -For example: `gcs::https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH_TO_MODULE` - -### Kind (May be Required) - -`kind` refers to the way in which a module is deployed. Currently, `kind` can be -either `terraform` or `packer`. It must be specified for modules of type -`packer`. If omitted, it will default to `terraform`. - -### Settings (May Be Required) - -The settings field is a map that supplies any user-defined variables for each -module. Settings values can be simple strings, numbers or booleans, but can -also support complex data types like maps and lists of variable depth. These -settings will become the values for the variables defined in either the -`variables.tf` file for Terraform or `variable.pkr.hcl` file for Packer. - -For some modules, there are mandatory variables that must be set, -therefore `settings` is a required field in that case. In many situations, a -combination of sensible defaults, deployment variables and used modules can -populated all required settings and therefore the settings field can be omitted. - -### Use (Optional) - -The `use` field is a powerful way of linking a module to one or more other -modules. When a module "uses" another module, the outputs of the used -module are compared to the settings of the current module. If they have -matching names and the setting has no explicit value, then it will be set to -the used module's output. For example, see the following blueprint snippet: - -```yaml -modules: -- id: network1 - source: modules/network/vpc - -- id: workstation - source: modules/compute/vm-instance - use: [network1] - settings: - ... -``` - -In this snippet, the VM instance `workstation` uses the outputs of vpc -`network1`. - -In this case both `network_self_link` and `subnetwork_self_link` in the -[workstation settings](compute/vm-instance/README.md#Inputs) will be set -to `$(network1.network_self_link)` and `$(network1.subnetwork_self_link)` which -refer to the [network1 outputs](network/vpc/README#Outputs) -of the same names. - -The order of precedence that `gcluster` uses in determining when to infer a setting -value is in the following priority order: - -1. Explicitly set in the blueprint using the `settings` field -1. Output from a used module, taken in the order provided in the `use` list -1. Deployment variable (`vars`) of the same name -1. Default value for the setting - -> **_NOTE:_** See the -> [network storage documentation](./../docs/network_storage.md) for more -> information about mounting network storage file systems via the `use` field. - -### Outputs (Optional) - -The `outputs` field adds the output of individual Terraform modules to the -output of its deployment group. This enables the value to be available via -`terraform output`. This can useful for displaying the IP of a login node or -printing instructions on how to use a module, as we have in the -[monitoring dashboard module](monitoring/dashboard/README.md#Outputs). - -The outputs field is a lists that it can be in either of two formats: a string -equal to the name of the module output, or a map specifying the `name`, -`description`, and whether the value is `sensitive` and should be suppressed -from the standard output of Terraform commands. An example is shown below -that displays the internal and public IP addresses of a VM created by the -vm-instance module: - -```yaml - - id: vm - source: modules/compute/vm-instance - use: - - network1 - settings: - machine_type: e2-medium - outputs: - - internal_ip - - name: external_ip - description: "External IP of VM" - sensitive: true -``` - -The outputs shown after running Terraform apply will resemble: - -```text -Apply complete! Resources: 7 added, 0 changed, 0 destroyed. - -Outputs: - -external_ip_simplevm = -internal_ip_simplevm = [ - "10.128.0.19", -] -``` - -### Required Services (APIs) (optional) - -Each Toolkit module depends upon Google Cloud services ("APIs") being enabled -in the project used by the AI/ML and HPC environment. For example, the [creation of -VMs](compute/vm-instance/) requires the Compute Engine API -(compute.googleapis.com). The [startup-script](scripts/startup-script/) module -requires the Cloud Storage API (storage.googleapis.com) for storage of the -scripts themselves. Each module included in the Toolkit source code describes -its required APIs internally. The Toolkit will merge the requirements from all -modules and [automatically validate](../README.md#blueprint-validation) that all -APIs are enabled in the project specified by `$(vars.project_id)`. - -## Common Settings - -The following common naming conventions should be used to decrease the verbosity -needed to define a blueprint. This is intentional to allow multiple -modules to share inferred settings from deployment variables or from other -modules listed under the `use` field. - -For example, if all modules are to be created in a single region, that region -can be defined as a deployment variable named `region`, which is shared between -all modules without an explicit setting. Similarly, if many modules need to be -connected to the same VPC network, they all can add the vpc module ID to their -`use` list so that `network_self_link` would be inferred from that vpc module rather -than having to set it manually. - -* **project_id**: The GCP project ID in which to create the GCP resources. -* **deployment_name**: The name of the current deployment of a blueprint. This - can help to avoid naming conflicts of modules when multiple deployments are - created from the same blueprint. -* **region**: The GCP - [region](https://cloud.google.com/compute/docs/regions-zones) the module - will be created in. -* **zone**: The GCP [zone](https://cloud.google.com/compute/docs/regions-zones) - the module will be created in. -* **labels**: - [Labels](https://cloud.google.com/resource-manager/docs/creating-managing-labels) - added to the module. In order to include any module in advanced - monitoring, labels must be exposed. We strongly recommend that all modules - expose this variable. - -## Writing Custom Cluster Toolkit Modules - -Modules are flexible by design, however we define some [best practices](../docs/module-guidelines.md) when -creating a new module meant to be used with the Cluster Toolkit. diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/README.md b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/README.md deleted file mode 100644 index f807cd727e..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/README.md +++ /dev/null @@ -1,133 +0,0 @@ -## Description - -This module is used to create a Kubernetes job template file. - -The job template file can be submitted as is or used as a template for further -customization. Add the `instructions` output to a blueprint (as shown below) to -get instructions on how to use `kubectl` to submit the job. - -This module is designed to `use` one or more `gke-node-pool` modules. The job -will be configured to run on any of the specified node pools. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a GKE job template file. - -```yaml - - id: job-template - source: modules/compute/gke-job-template - use: [compute_pool] - settings: - node_count: 3 - outputs: [instructions] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### Storage Options - -This module natively supports: - -* Filestore as a shared file system between pods/nodes. -* Pod level ephemeral storage options: - * memory backed emptyDir - * local SSD backed emptyDir - * SSD persistent disk backed ephemeral volume - * balanced persistent disk backed ephemeral volume - -See the [storage-gke.yaml blueprint](../../../examples/storage-gke.yaml) and the -associated [documentation](../../../../examples/README.md#storage-gkeyaml--) for -examples of how to use Filestore and ephemeral storage with this module. - -### Requested Resources - -When one or more `gke-node-pool` modules are referenced with the `use` field. -The requested resources will be populated to achieve a 1 pod per node packing -while still leaving some headroom for required system pods. - -This functionality can be overridden by specifying the desired cpu requirement -using the `requested_cpu_per_pod` setting. - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.2 | -| [local](#requirement\_local) | >= 2.0.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allocatable\_cpu\_per\_node](#input\_allocatable\_cpu\_per\_node) | The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | -| [allocatable\_gpu\_per\_node](#input\_allocatable\_gpu\_per\_node) | The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | -| [backoff\_limit](#input\_backoff\_limit) | Controls the number of retries before considering a Job as failed. Set to zero for shared fate. | `number` | `0` | no | -| [command](#input\_command) | The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes. | `list(string)` | 
[
  "hostname"
]
| no | -| [completion\_mode](#input\_completion\_mode) | Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information | `string` | `"Indexed"` | no | -| [ephemeral\_volumes](#input\_ephemeral\_volumes) | Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB. | 
list(object({
    type       = string
    mount_path = string
    size_gb    = number
  }))
| `[]` | no | -| [has\_gpu](#input\_has\_gpu) | Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module. | `list(bool)` | 
[
  false
]
| no | -| [image](#input\_image) | The container image the job should use. | `string` | `"debian"` | no | -| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account to run the job as. If null then no service account is specified. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the GKE job template. Key-value pairs. | `map(string)` | n/a | yes | -| [machine\_family](#input\_machine\_family) | The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria. | `string` | `null` | no | -| [name](#input\_name) | The name of the job. | `string` | `"my-job"` | no | -| [node\_count](#input\_node\_count) | How many nodes the job should run in parallel. | `number` | `1` | no | -| [node\_pool\_names](#input\_node\_pool\_names) | A list of node pool names on which to run the job. Can be populated via `use` field. | `list(string)` | `[]` | no | -| [node\_selectors](#input\_node\_selectors) | A list of node selectors to use to place the job. | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | -| [persistent\_volume\_claims](#input\_persistent\_volume\_claims) | A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module. | 
list(object({
    name          = string
    namespace     = string
    mount_path    = string
    mount_options = string
    storage_type  = string
  }))
| `[]` | no | -| [random\_name\_sufix](#input\_random\_name\_sufix) | Appends a random suffix to the job name to avoid clashes. | `bool` | `true` | no | -| [requested\_cpu\_per\_pod](#input\_requested\_cpu\_per\_pod) | The requested cpu per pod. If null, allocatable\_cpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_cpu\_per\_node. | `number` | `-1` | no | -| [requested\_gpu\_per\_pod](#input\_requested\_gpu\_per\_pod) | The requested gpu per pod. If null, allocatable\_gpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_gpu\_per\_node. | `number` | `-1` | no | -| [restart\_policy](#input\_restart\_policy) | Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed. | `string` | `"Never"` | no | -| [security\_context](#input\_security\_context) | The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | -| [tolerations](#input\_tolerations) | Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field. | 
list(object({
    key      = string
    operator = string
    value    = string
    effect   = string
  }))
| 
[
  {
    "effect": "NoSchedule",
    "key": "user-workload",
    "operator": "Equal",
    "value": "true"
  }
]
| no | -| [tpu\_accelerator\_type](#input\_tpu\_accelerator\_type) | The TPU accelerator type label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | -| [tpu\_chips\_per\_node](#input\_tpu\_chips\_per\_node) | The number of TPU chips per node. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | -| [tpu\_topology](#input\_tpu\_topology) | The TPU topology label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for submitting the GKE job. | - diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/main.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/main.tf deleted file mode 100644 index e84138bb3f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/main.tf +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-job-template", ghpc_role = "compute" }) -} - -locals { - tpu_accelerator_node_selector = var.tpu_accelerator_type[0] != null ? [{ - key = "cloud.google.com/gke-tpu-accelerator" - value = var.tpu_accelerator_type[0] - }] : [] - - tpu_topology_node_selector = var.tpu_topology[0] != null ? [{ - key = "cloud.google.com/gke-tpu-topology" - value = var.tpu_topology[0] - }] : [] -} - -locals { - # Start with the minimum cpu available of used node pools - min_allocatable_cpu = min(var.allocatable_cpu_per_node...) - full_node_cpu_request = ( - local.min_allocatable_cpu > 2 ? # if large enough - local.min_allocatable_cpu - 1 : # leave headroom for 1 cpu - local.min_allocatable_cpu / 2 + 0.1 # else take just over half - ) - (local.any_gcs ? 0.25 : 0) # save room for gcs side car - - cpu_request = ( - var.requested_cpu_per_pod >= 0 ? # if user supplied requested cpu - var.requested_cpu_per_pod : # then honor it - ( # else - local.min_allocatable_cpu >= 0 ? # if allocatable cpu was supplied - local.full_node_cpu_request : # then claim the full node - -1 # else do not set a limit - ) - ) - millicpu = floor(local.cpu_request * 1000) - cpu_request_string = local.millicpu >= 0 ? "${local.millicpu}m" : null - full_node_request = local.min_allocatable_cpu >= 0 && var.requested_cpu_per_pod < 0 - - memory_request_value = try(sum([for ed in var.ephemeral_volumes : - ed.size_gb - if ed.type == "memory" - ]), 0) - memory_request_string = local.memory_request_value > 0 ? "${local.memory_request_value}Gi" : null - - ephemeral_request_value = try(sum([for ed in var.ephemeral_volumes : - ed.size_gb - if ed.type == "local-ssd" - ]), 0) - ephemeral_request_string = local.ephemeral_request_value > 0 ? "${local.ephemeral_request_value}Gi" : null - - uses_local_ssd = anytrue([for ed in var.ephemeral_volumes : - ed.type == "local-ssd" - ]) - local_ssd_node_selector = local.uses_local_ssd ? [{ - key = "cloud.google.com/gke-ephemeral-storage-local-ssd" - value = "true" - }] : [] - - # Setup limit for GPUs per pod - min_allocatable_gpu = min(var.allocatable_gpu_per_node...) - min_allocatable_gpu_per_pod = local.min_allocatable_gpu > 0 ? local.min_allocatable_gpu : null - gpu_limit_per_pod = var.requested_gpu_per_pod > 0 ? var.requested_gpu_per_pod : local.min_allocatable_gpu_per_pod - gpu_limit_string = alltrue(var.has_gpu) ? tostring(local.gpu_limit_per_pod) : null - - empty_dir_volumes = [for ed in var.ephemeral_volumes : - { - name = replace(trim(ed.mount_path, "/"), "/", "-") - mount_path = ed.mount_path - size_limit = "${ed.size_gb}Gi" - in_memory = ed.type == "memory" - } - if contains(["memory", "local-ssd"], ed.type) - ] - - ephemeral_pd_volumes = [for pd in var.ephemeral_volumes : - { - name = replace(trim(pd.mount_path, "/"), "/", "-") - mount_path = pd.mount_path - storage_class_name = pd.type == "pd-ssd" ? "premium-rwo" : "standard-rwo" - storage = "${pd.size_gb}Gi" - } - if contains(["pd-balanced", "pd-ssd"], pd.type) - ] - - pvc_volumes = [for pvc in var.persistent_volume_claims : - { - name = replace(trim(pvc.mount_path, "/"), "/", "-") - mount_path = pvc.mount_path - claim_name = pvc.name - } - ] - - volume_mounts = [for v in concat(local.empty_dir_volumes, local.ephemeral_pd_volumes, local.pvc_volumes) : - { - name = v.name - mount_path = v.mount_path - } - ] - - suffix = var.random_name_sufix ? "-${random_id.resource_name_suffix.hex}" : "" - machine_family_node_selector = var.machine_family != null ? [{ - key = "cloud.google.com/machine-family" - value = var.machine_family - }] : [] - node_selectors = concat(local.machine_family_node_selector, local.local_ssd_node_selector, local.tpu_accelerator_node_selector, local.tpu_topology_node_selector, var.node_selectors) - - any_gcs = anytrue([for pvc in var.persistent_volume_claims : - pvc.storage_type == "gcs" - ]) - - job_template_contents = templatefile( - "${path.module}/templates/gke-job-base.yaml.tftpl", - { - name = var.name - suffix = local.suffix - image = var.image - command = var.command - node_count = var.node_count - completion_mode = var.completion_mode - k8s_service_account_name = var.k8s_service_account_name - node_pool_names = var.node_pool_names - node_selectors = local.node_selectors - tpu_limit = var.tpu_chips_per_node[0] - full_node_request = local.full_node_request - cpu_request = local.cpu_request_string - gpu_limit = local.gpu_limit_string - restart_policy = var.restart_policy - backoff_limit = var.backoff_limit - tolerations = distinct(var.tolerations) - security_context = var.security_context - labels = local.labels - - empty_dir_volumes = local.empty_dir_volumes - ephemeral_pd_volumes = local.ephemeral_pd_volumes - pvc_volumes = local.pvc_volumes - volume_mounts = local.volume_mounts - memory_request = local.memory_request_string - ephemeral_request = local.ephemeral_request_string - gcs_annotation = local.any_gcs - } - ) - - job_template_output_path = "${path.root}/${var.name}${local.suffix}.yaml" - -} - -resource "random_id" "resource_name_suffix" { - byte_length = 2 - keepers = { - timestamp = timestamp() - } -} - -resource "local_file" "job_template" { - content = local.job_template_contents - filename = local.job_template_output_path - - lifecycle { - precondition { - condition = local.any_gcs ? var.k8s_service_account_name != null : true - error_message = "When using GCS, a kubernetes service account with workload identity is required. gke-cluster module will perform this setup when var.configure_workload_identity_sa is set to true." - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/outputs.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/outputs.tf deleted file mode 100644 index adf78e936d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "instructions" { - description = "Instructions for submitting the GKE job." - value = <<-EOT - A GKE job file has been created locally at: - ${abspath(local.job_template_output_path)} - - Use the following commands to: - Submit your job: - kubectl create -f ${abspath(local.job_template_output_path)} - EOT -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl deleted file mode 100644 index 11df39ce2c..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl +++ /dev/null @@ -1,128 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: ${name}${suffix} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - parallelism: ${node_count} - completions: ${node_count} - completionMode: ${completion_mode} - template: - %{~ if gcs_annotation ~} - metadata: - annotations: - gke-gcsfuse/volumes: "true" - %{~ endif ~} - spec: - %{~ if length(security_context) > 0 ~} - securityContext: - %{~ for context in security_context ~} - ${context.key}: ${context.value} - %{~ endfor ~} - %{~ endif ~} - %{~ if k8s_service_account_name != null ~} - serviceAccountName: ${k8s_service_account_name} - %{~ endif ~} - %{~ if length(node_pool_names) > 0 ~} - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.google.com/gke-nodepool - operator: In - values: - %{~ for node_pool in node_pool_names ~} - - ${node_pool} - %{~ endfor ~} - %{~ endif ~} - %{~ if length(node_selectors) > 0 ~} - nodeSelector: - %{~ for selector in node_selectors ~} - ${selector.key}: "${selector.value}" - %{~ endfor ~} - %{~ endif ~} - tolerations: - %{~ for toleration in tolerations ~} - - key: ${toleration.key} - operator: ${toleration.operator} - value: "${toleration.value}" - effect: ${toleration.effect} - %{~ endfor ~} - containers: - - name: ${name}-container - image: ${image} - command: - %{for s in command}- ${indent(8, yamlencode(s))}%{~ endfor } - %{~ if gpu_limit != null || cpu_request != null || tpu_limit != null ~} - resources: - %{~ if gpu_limit != null || tpu_limit != null ~} - limits: - %{~ if gpu_limit != null ~} - # GPUs should only be specified as limits - # https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/ - nvidia.com/gpu: ${gpu_limit} - %{~ endif ~} - %{~ if tpu_limit != null ~} - google.com/tpu: ${tpu_limit} - %{~ endif ~} - %{~ endif ~} - %{~ if cpu_request != null || memory_request != null || ephemeral_request != null || tpu_limit != null ~} - requests: - %{~ if full_node_request ~} - # cpu request attempts full node per pod - %{~ endif ~} - %{~ if cpu_request != null ~} - cpu: ${cpu_request} - %{~ endif ~} - %{~ if tpu_limit != null ~} - google.com/tpu: ${tpu_limit} - %{~ endif ~} - %{~ if memory_request != null ~} - memory: ${memory_request} - %{~ endif ~} - %{~ if ephemeral_request != null ~} - ephemeral-storage: ${ephemeral_request} - %{~ endif ~} - %{~ endif ~} - %{~ endif ~} - %{~ if length(volume_mounts) > 0 ~} - volumeMounts: - %{~ for v in volume_mounts ~} - - name: ${v.name} - mountPath: ${v.mount_path} - %{~ endfor ~} - %{~ endif ~} - %{~ if length(volume_mounts) > 0 ~} - volumes: - %{~ for ed in empty_dir_volumes ~} - - name: ${ed.name} - emptyDir: - sizeLimit: ${ed.size_limit} - %{~ if ed.in_memory ~} - medium: "Memory" - %{~ endif ~} - %{~ endfor ~} - %{~ for pd in ephemeral_pd_volumes ~} - - name: ${pd.name} - ephemeral: - volumeClaimTemplate: - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: ${pd.storage_class_name} - resources: - requests: - storage: ${pd.storage} - %{~ endfor ~} - %{~ for pvc in pvc_volumes ~} - - name: ${pvc.name} - persistentVolumeClaim: - claimName: ${pvc.claim_name} - %{~ endfor ~} - %{~ endif ~} - restartPolicy: ${restart_policy} - backoffLimit: ${backoff_limit} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/variables.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/variables.tf deleted file mode 100644 index fd83f2b692..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/variables.tf +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "name" { - description = "The name of the job." - type = string - default = "my-job" -} - -variable "node_count" { - description = "How many nodes the job should run in parallel." - type = number - default = 1 -} - -variable "completion_mode" { - description = "Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information" - type = string - default = "Indexed" -} - -variable "command" { - description = "The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes." - type = list(string) - default = ["hostname"] -} - -variable "image" { - description = "The container image the job should use." - type = string - default = "debian" -} - -variable "k8s_service_account_name" { - description = "Kubernetes service account to run the job as. If null then no service account is specified." - type = string - default = null -} - -variable "node_pool_names" { - description = "A list of node pool names on which to run the job. Can be populated via `use` field." - type = list(string) - default = [] -} - -variable "allocatable_cpu_per_node" { - description = "The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." - type = list(number) - default = [-1] -} - -variable "has_gpu" { - description = "Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module." - type = list(bool) - default = [false] -} - -variable "requested_cpu_per_pod" { - description = "The requested cpu per pod. If null, allocatable_cpu_per_node will be used to claim whole nodes. If provided will override allocatable_cpu_per_node." - type = number - default = -1 -} - -variable "allocatable_gpu_per_node" { - description = "The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." - type = list(number) - default = [-1] -} - -variable "requested_gpu_per_pod" { - description = "The requested gpu per pod. If null, allocatable_gpu_per_node will be used to claim whole nodes. If provided will override allocatable_gpu_per_node." - type = number - default = -1 -} - -variable "tolerations" { - description = "Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field." - type = list(object({ - key = string - operator = string - value = string - effect = string - })) - default = [ - { - key = "user-workload" - operator = "Equal" - value = "true" - effect = "NoSchedule" - } - ] -} - -variable "security_context" { - description = "The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/" - type = list(object({ - key = string - value = string - })) - default = [] -} - -variable "machine_family" { - description = "The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria." - type = string - default = null -} - -variable "node_selectors" { - description = "A list of node selectors to use to place the job." - type = list(object({ - key = string - value = string - })) - default = [] -} - -variable "restart_policy" { - description = "Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed." - type = string - default = "Never" -} - -variable "backoff_limit" { - description = "Controls the number of retries before considering a Job as failed. Set to zero for shared fate." - type = number - default = 0 -} - -variable "random_name_sufix" { - description = "Appends a random suffix to the job name to avoid clashes." - type = bool - default = true -} - -variable "persistent_volume_claims" { - description = "A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module." - type = list(object({ - name = string - namespace = string - mount_path = string - mount_options = string - storage_type = string - })) - default = [] -} - -variable "ephemeral_volumes" { - description = "Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB." - type = list(object({ - type = string - mount_path = string - size_gb = number - })) - default = [] - validation { - condition = alltrue([ - for v in var.ephemeral_volumes : - contains(["pd-balanced", "pd-ssd", "memory", "local-ssd"], v.type) - ]) - error_message = "Type must be one of 'pd-balanced', 'pd-ssd', 'memory', 'local-ssd'." - } - validation { - condition = alltrue([ - for v in var.ephemeral_volumes : - substr(v.mount_path, 0, 1) == "/" - ]) - error_message = "Mount path must start with the '/' character." - } -} - -variable "labels" { - description = "Labels to add to the GKE job template. Key-value pairs." - type = map(string) -} - -variable "tpu_accelerator_type" { - description = "The TPU accelerator type label. Populated from gke-node-pool via `use` field." - type = list(string) - default = [null] -} - -variable "tpu_topology" { - description = "The TPU topology label. Populated from gke-node-pool via `use` field." - type = list(string) - default = [null] -} - -variable "tpu_chips_per_node" { - description = "The number of TPU chips per node. Populated from gke-node-pool via `use` field." - type = list(string) - default = [null] -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/versions.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/versions.tf deleted file mode 100644 index 0f902ac8c5..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-job-template/versions.tf +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.2" - - required_providers { - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/README.md b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/README.md deleted file mode 100644 index b25d905252..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/README.md +++ /dev/null @@ -1,388 +0,0 @@ -## Description - -This module creates a Google Kubernetes Engine -([GKE](https://cloud.google.com/kubernetes-engine)) node pool. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a GKE node group. - -```yaml - - id: compute_pool - source: modules/compute/gke-node-pool - use: [gke_cluster] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### Taints and Tolerations - -By default node pools created with this module will be tainted with -`user-workload=true:NoSchedule` to prevent system pods from being scheduled. -User jobs targeting the node pool should include this toleration. This behavior -can be overridden using the `taints` setting. See -[docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-taints) for -more info. - -### Local SSD Storage -GKE offers two options for managing locally attached SSDs. - -The first, and recommended, option is for GKE to manage the ephemeral storage -space on the node, which will then be automatically attached to pods which -request an `emptyDir` volume. This can be accomplished using the -[`local_ssd_count_ephemeral_storage`] variable. - -The second, more complex, option is for GCP to attach these nodes as raw block -storage. In this case, the cluster administrator is responsible for software -RAID settings, partitioning, formatting and mounting these disks on the host -OS. Still, this may be desired behavior in use cases which aren't supported -by an `emptyDir` volume (for example, a `ReadOnlyMany` or `ReadWriteMany` PV). -This can be accomplished using the [`local_ssd_count_nvme_block`] variable. - -The [`local_ssd_count_ephemeral_storage`] and [`local_ssd_count_nvme_block`] -variables are mutually exclusive and cannot be mixed together. - -Also, the number of SSDs which can be attached to a node depends on the -[machine type](https://cloud.google.com/compute/docs/disks#local_ssd_machine_type_restrictions). - -See [docs](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd) -for more info. - -[`local_ssd_count_ephemeral_storage`]: #input\_local\_ssd\_count\_ephemeral\_storage -[`local_ssd_count_nvme_block`]: #input\_local\_ssd\_count\_nvme\_block - -### Considerations with GPUs - -When a GPU is attached to a node an additional taint is automatically added: -`nvidia.com/gpu=present:NoSchedule`. For jobs to get placed on these nodes, the -equivalent toleration is required. The `gke-job-template` module will -automatically apply this toleration when using a node pool with GPUs. - -Nvidia GPU drivers must be installed. The recommended approach for GKE to install -GPU dirvers is by applying a DaemonSet to the cluster. See -[these instructions](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus#cos). - -However, in some cases it may be desired to compile a different driver (such as -a desire to install a newer version, compatibility with the -[Nvidia GPU-operator](https://github.com/NVIDIA/gpu-operator) or other -use-cases). In this case, ensure that you turn off the -[enable_secure_boot](#input\_enable\_secure\_boot) option to allow unsigned -kernel modules to be loaded. - -#### Maximize GPU network bandwidth with GPUDirect and multi-networking -For A3 Series machines to achieve optimal performance , GKE provide two networking stacks for remote direct memory access (RDMA): - -- A3 High machine types (a3-highgpu-8g): utilize GPUDirect-TCPX to reduce the overhead required to transfer packet payloads to and from GPUs, which significantly improves throughput at scale compared to GPUs that don't use GPUDirect. -- A3 Mega machine types (a3-megagpu-8g): utilize GPUDirect-TCPXO to improve GPU to GPU communication, and further improves GPU to VM communication. - -To achieve this, when creating nodepools with A3 Series machine type, pass in a multivpc module to the gke-node-pool module, and the gke-node-pool module would detect the eligible machine type and enable GPUDirect for it. More specifically, the below components will be installed in the nodepool for enabling GPUDirect. - -- Install NCCL plugin for GPUDirect [TCPX](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpx) or [TCPXO](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpxo) -- Install [NRI](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nri_device_injector) device injector plugin -- Provide support for injecting GPUDirect required components(annotations, volumes, rxdm sidecar etc.) into the user workload in the form of Kubernetes Job. - - Provide sample workload to showcase how it will be updated with the required components injected, and how it can be deployed. - - Allow user to use the provided script to update their own workload and deploy. - -The GPUDirect supports included in the Cluster Toolkit aim to automate the [GPUDirect User Guid](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#install-gpudirect-tcpx-nccl) and provide better usability. - -> **_NOTE:_** You must [enable multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) feature when creating the GKE cluster. When gke-cluster depends on multivpc (with the use keyword), multi networking will be automatically enabled on the cluster creation. -> When gke-cluster or pre-existing-gke-cluster depends on multivpc (with the use keyword), the [network objects](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment) required for multi networking will be created on the cluster. - -### GPUs Examples - -There are several ways to add GPUs to a GKE node pool. See -[docs](https://cloud.google.com/compute/docs/gpus) for more info on GPUs. - -The following is a node pool that uses `a2`, `a3` or `g2` machine types which has a -fixed number of attached GPUs, let's call these machine types as "pre-defined gpu machine families": - -```yaml - - id: simple-a2-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g -``` - -> **Note**: It is not necessary to define the [`guest_accelerator`] setting when -> using pre-defined gpu machine families as information about GPUs, such as type, count and -> `gpu_driver_installation_config`, is automatically inferred from the machine type. -> Optional fields such as `gpu_partition_size` need to be specified only if they have -> non-default values. - -The following scenarios require the [`guest_accelerator`] block is specified: - -- To partition an A100 GPU into multiple GPUs on an A2 family machine. -- To specify a time sharing configuration on a GPUs. -- To attach a GPU to an N1 family machine. - -The following is an example of -[partitioning](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus-multi) -an A100 GPU: - -> **Note**: In the following example, `type`, `count` and `gpu_driver_installation_config` are picked up automatically. - -```yaml - - id: multi-instance-gpu-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g - guest_accelerator: - - gpu_partition_size: 1g.5gb -``` - -[`guest_accelerator`]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_guest_accelerator - -The following is an example of -[GPU time sharing](https://cloud.google.com/kubernetes-engine/docs/concepts/timesharing-gpus) -(with partitioned GPUs): - -```yaml - - id: time-sharing-gpu-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g - guest_accelerator: - - gpu_partition_size: 1g.5gb - gpu_sharing_config: - gpu_sharing_strategy: TIME_SHARING - max_shared_clients_per_gpu: 3 -``` - -Following is an example of using a GPU attached to an `n1` machine: - -```yaml - - id: t4-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: n1-standard-16 - guest_accelerator: - - type: nvidia-tesla-t4 - count: 2 -``` - -The following is an example of using a GPU (with sharing config) attached to an `n1` machine: - -```yaml - - id: n1-t4-pool - source: community/modules/compute/gke-node-pool - use: [gke_cluster] - settings: - name: n1-t4-pool - machine_type: n1-standard-1 - guest_accelerator: - - type: nvidia-tesla-t4 - count: 2 - gpu_driver_installation_config: - gpu_driver_version: "LATEST" - gpu_sharing_config: - max_shared_clients_per_gpu: 2 - gpu_sharing_strategy: "TIME_SHARING" -``` - -Finally, the following is adding multivpc to a node pool: - -```yaml - - id: network - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: gke-cluster - source: modules/scheduler/gke-cluster - use: [network, multinetwork] - settings: - cluster_name: $(vars.deployment_name) - - - id: a3-megagpu_pool - source: modules/compute/gke-node-pool - use: [gke-cluster, multinetwork] - settings: - machine_type: a3-megagpu-8g - ... -``` - -## Using GCE Reservations -You can reserve Google Compute Engine instances in a specific zone to ensure resources are available for their workloads when needed. For more details on how to manage reservations, see [Reserving Compute Engine zonal resources](https://cloud.google.com/compute/docs/instances/reserving-zonal-resources). - -After creating a reservation, you can consume the reserved GCE VM instances in GKE. GKE clusters deployed using Cluster Toolkit support the same consumption modes as Compute Engine: NO_RESERVATION(default), ANY_RESERVATION, SPECIFIC_RESERVATION. - -This can be accomplished using [`reservation_affinity`](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/compute/gke-node-pool/README.md#input_reservation_affinity). - -```yaml -# Target any reservation -reservation_affinity: - consume_reservation_type: ANY_RESERVATION - -# Target a specific reservation -reservation_affinity: - consume_reservation_type: SPECIFIC_RESERVATION - specific_reservations: - - name: specific-reservation-1 -``` - -The following requirements need to be satisfied for the node pool nodes to be able to use a specific reservation: -1. A reservation with the name must exist in the specified project(`var.project_id`) and one of the specified zones(`var.zones`). -2. Its consumption type must be `specific`. -3. Its GCE VM Properties must match with those of the Node Pool; Machine type, Accelerators (GPU Type and count), Local SSD disk type and count. - -If you want to utilise a shared reservation, the owner project of the shared reservation needs to be explicitly specified like the following. Note that a shared reservation can be used by the project that hosts the reservation (owner project) and by the projects the reservation is shared with (consumer projects). See how to [create and use a shared reservation](https://cloud.google.com/compute/docs/instances/reservations-shared). - -```yaml -reservation_affinity: - consume_reservation_type: SPECIFIC_RESERVATION - specific_reservations: - - name: specific-reservation-shared - project: shared_reservation_owner_project_id -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 7.2 | -| [google-beta](#requirement\_google-beta) | >= 7.2 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 7.2 | -| [google-beta](#provider\_google-beta) | >= 7.2 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | -| [tpu](#module\_tpu) | ../../internal/tpu-definition | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_container_node_pool.node_pool](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | -| [null_resource.enable_tcpx_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.enable_tcpxo_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.install_dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_machine_types.machine_info](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | -| [google_compute_region_instance_template.instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_template) | data source | -| [google_compute_reservation.specific_reservations](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [auto\_repair](#input\_auto\_repair) | Whether the nodes will be automatically repaired. | `bool` | `true` | no | -| [auto\_upgrade](#input\_auto\_upgrade) | Whether the nodes will be automatically upgraded. | `bool` | `false` | no | -| [autoscaling\_total\_max\_nodes](#input\_autoscaling\_total\_max\_nodes) | Total maximum number of nodes in the NodePool. | `number` | `1000` | no | -| [autoscaling\_total\_min\_nodes](#input\_autoscaling\_total\_min\_nodes) | Total minimum number of nodes in the NodePool. | `number` | `0` | no | -| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | -| [compact\_placement](#input\_compact\_placement) | DEPRECATED: Use `placement_policy` | `bool` | `null` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for each node. | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Disk type for each node. | `string` | `null` | no | -| [enable\_flex\_start](#input\_enable\_flex\_start) | If true, start the node pool with Flex Start provisioning model.
To learn more about flex-start mode, please refer to
https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and
https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest | `bool` | `false` | no | -| [enable\_gcfs](#input\_enable\_gcfs) | Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config). | `bool` | `false` | no | -| [enable\_numa\_aware\_scheduling](#input\_enable\_numa\_aware\_scheduling) | Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling. | `bool` | `false` | no | -| [enable\_private\_nodes](#input\_enable\_private\_nodes) | Whether nodes have internal IP addresses only. | `bool` | `true` | no | -| [enable\_queued\_provisioning](#input\_enable\_queued\_provisioning) | If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool. | `bool` | `false` | no | -| [enable\_secure\_boot](#input\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | -| [gke\_version](#input\_gke\_version) | GKE version | `string` | n/a | yes | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = optional(string)
    count = optional(number, 0)
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | -| [host\_maintenance\_interval](#input\_host\_maintenance\_interval) | Specifies the frequency of planned maintenance events. | `string` | `""` | no | -| [image\_type](#input\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | -| [initial\_node\_count](#input\_initial\_node\_count) | The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static\_node\_count and must be set to a value between autoscaling\_total\_min\_nodes and autoscaling\_total\_max\_nodes. | `number` | `null` | no | -| [internal\_ghpc\_module\_id](#input\_internal\_ghpc\_module\_id) | DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide). | `string` | n/a | yes | -| [is\_reservation\_active](#input\_is\_reservation\_active) | Whether the specified reservation is already created. | `bool` | `true` | no | -| [kubernetes\_labels](#input\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_count\_ephemeral\_storage](#input\_local\_ssd\_count\_ephemeral\_storage) | The number of local SSDs to attach to each node to back ephemeral storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | -| [local\_ssd\_count\_nvme\_block](#input\_local\_ssd\_count\_nvme\_block) | The number of local SSDs to attach to each node to back block storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | -| [machine\_type](#input\_machine\_type) | The name of a Google Compute Engine machine type. | `string` | `"c2-standard-60"` | no | -| [max\_pods\_per\_node](#input\_max\_pods\_per\_node) | The maximum number of pods per node in this node pool. This will force replacement. | `number` | `null` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [name](#input\_name) | The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix.
If setting manually, ensure a unique value across all gke-node-pools. | `string` | `null` | no | -| [num\_node\_pools](#input\_num\_node\_pools) | Number of node pools to create. This is same as num\_slices. | `number` | `1` | no | -| [num\_slices](#input\_num\_slices) | Number of TPUs slices to create. This is same as num\_node\_pools. | `number` | `1` | no | -| [placement\_policy](#input\_placement\_policy) | Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| 
{
  "name": null,
  "tpu_topology": null,
  "type": null
}
| no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Reservation resource to consume. When targeting SPECIFIC\_RESERVATION, specific\_reservations needs be specified.
Even though specific\_reservations is a list, only one reservation is allowed by the NodePool API.
It is assumed that the specified reservation exists and has available capacity.
For a shared reservation, specify the project\_id as well in which it was created.
To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared | 
object({
    consume_reservation_type = string
    specific_reservations = optional(list(object({
      name    = string
      project = optional(string)
    })))
  })
| 
{
  "consume_reservation_type": "NO_RESERVATION",
  "specific_reservations": []
}
| no | -| [run\_workload\_script](#input\_run\_workload\_script) | Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only. | `bool` | `true` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [static\_node\_count](#input\_static\_node\_count) | The static number of nodes in the node pool. If set, autoscaling will be disabled. | `number` | `null` | no | -| [taints](#input\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | -| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | -| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | -| [total\_max\_nodes](#input\_total\_max\_nodes) | DEPRECATED: Use autoscaling\_total\_max\_nodes. | `number` | `null` | no | -| [total\_min\_nodes](#input\_total\_min\_nodes) | DEPRECATED: Use autoscaling\_total\_min\_nodes. | `number` | `null` | no | -| [upgrade\_settings](#input\_upgrade\_settings) | Defines node pool upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | -| [zones](#input\_zones) | A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable. | `list(string)` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [allocatable\_cpu\_per\_node](#output\_allocatable\_cpu\_per\_node) | Number of CPUs available for scheduling pods on each node. | -| [allocatable\_gpu\_per\_node](#output\_allocatable\_gpu\_per\_node) | Number of GPUs available for scheduling pods on each node. | -| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | -| [guest\_accelerator](#output\_guest\_accelerator) | The accelerator type of the nodes. | -| [has\_gpu](#output\_has\_gpu) | Boolean value indicating whether nodes in the pool are configured with GPUs. | -| [instance\_templates](#output\_instance\_templates) | The URLs of Instance Templates | -| [instructions](#output\_instructions) | Instructions for submitting the sample GPUDirect enabled job. | -| [machine\_type](#output\_machine\_type) | Machine Type | -| [node\_count\_static](#output\_node\_count\_static) | The number of static nodes in node-pool. | -| [node\_pool\_names](#output\_node\_pool\_names) | Names of the node pools. | -| [static\_gpu\_count](#output\_static\_gpu\_count) | Total number of GPUs in the node pool. Available only for static node pools. | -| [tolerations](#output\_tolerations) | Tolerations needed for a pod to be scheduled on this node pool. | -| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | -| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | -| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | - diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf deleted file mode 100644 index 0c1c255255..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -## Required variables: -# local_ssd_count_ephemeral_storage -# local_ssd_count_nvme_block -# machine_type - -locals { - - local_ssd_machines = { - "a3-highgpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, - "a3-megagpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, - "a3-ultragpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, - "a4-highgpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, - } - - generated_local_ssd_config = lookup(local.local_ssd_machines, var.machine_type, { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = null }) - - # Select in priority order: - # (1) var.local_ssd_count_ephemeral_storage and var.local_ssd_count_nvme_block if any is not null - # (2) local.local_ssd_machines if not empty - # (3) default to null value for both local_ssd_count_ephemeral_storage and local_ssd_count_nvme_block - local_ssd_config = (var.local_ssd_count_ephemeral_storage == null && var.local_ssd_count_nvme_block == null) ? local.generated_local_ssd_config : { local_ssd_count_ephemeral_storage = var.local_ssd_count_ephemeral_storage, local_ssd_count_nvme_block = var.local_ssd_count_nvme_block } -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml deleted file mode 100644 index 1106f63479..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: batch/v1 -kind: Job -metadata: - name: my-sample-job -spec: - parallelism: 2 - completions: 2 - completionMode: Indexed - template: - spec: - containers: - - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx-dev:v3.1.9 - imagePullPolicy: Always - command: - - /bin/sh - - -c - - | - service ssh restart; - sleep infinity; - env: - - name: LD_LIBRARY_PATH - value: /usr/local/nvidia/lib64 - volumeMounts: - - name: config-volume - mountPath: /configs - resources: - limits: - nvidia.com/gpu: 8 - volumes: - - name: config-volume - configMap: - name: nccl-configmap - defaultMode: 0777 - restartPolicy: Never - backoffLimit: 0 diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml deleted file mode 100644 index bce6720681..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: batch/v1 -kind: Job -metadata: - name: my-sample-job -spec: - parallelism: 2 - completions: 2 - completionMode: Indexed - template: - spec: - hostname: host1 - subdomain: nccl-host-1 - containers: - - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.14 - imagePullPolicy: Always - command: - - /bin/sh - - -c - - | - set -ex - chmod 755 /scripts/demo-run-nccl-test-tcpxo-via-mpi.sh - cat >/scripts/allgather.sh < 0: - container["env"].extend(env_vars) - container["volumeMounts"].extend(volume_mounts) - -if __name__ == "__main__": - main() diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py deleted file mode 100644 index db9fb3e7ff..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import yaml -import argparse -import os - -def main(): - parser = argparse.ArgumentParser(description="TCPXO Job Manifest Generator") - parser.add_argument("-f", "--file", required=True, help="Path to your job template YAML file") - parser.add_argument("-r", "--rxdm", required=True, help="RxDM version") - - args = parser.parse_args() - - # Get the YAML file from the user - if not args.file: - args.file = input("Please provide the path to your job template YAML file: ") - - # Get component versions from user - if not args.rxdm: - args.rxdm = input("Enter the RxDM version: ") - - # Load and modify the YAML - with open(args.file, "r") as file: - job_manifest = yaml.load(file, Loader=yaml.BaseLoader) - - # Update annotations - add_annotations(job_manifest) - - # Update volumes - add_volumes(job_manifest) - - # Update tolerations - add_tolerations(job_manifest) - - # Add tcpxo-daemon container - add_tcpxo_daemon_container(job_manifest, args.rxdm) - - # Update environment variables and volumeMounts for GPU containers - update_gpu_containers(job_manifest) - - # Generate the new YAML file - updated_job = str(yaml.dump(job_manifest, default_flow_style=False, width=1000, default_style="|", sort_keys=False)).replace("|-", "") - - new_file_name = args.file.replace(".yaml", "-tcpxo.yaml") - with open(new_file_name, "w", encoding="utf-8") as file: - file.write(updated_job) - - # Step 7: Provide instructions to the user - print("\nA new manifest has been generated and updated to have TCPXO enabled based on the provided workload") - print("It can be found in {path}".format(path=os.path.abspath(new_file_name))) - print("You can use the following commands to submit the sample job:") - print(" kubectl create -f {path}".format(path=os.path.abspath(new_file_name))) - -def add_annotations(job_manifest): - annotations = { - 'devices.gke.io/container.tcpxo-daemon':"""|+ -- path: /dev/nvidia0 -- path: /dev/nvidia1 -- path: /dev/nvidia2 -- path: /dev/nvidia3 -- path: /dev/nvidia4 -- path: /dev/nvidia5 -- path: /dev/nvidia6 -- path: /dev/nvidia7 -- path: /dev/nvidiactl -- path: /dev/nvidia-uvm -- path: /dev/dmabuf_import_helper""", - "networking.gke.io/default-interface": "eth0", - "networking.gke.io/interfaces": """| -[ - {"interfaceName":"eth0","network":"default"}, - {"interfaceName":"eth1","network":"vpc1"}, - {"interfaceName":"eth2","network":"vpc2"}, - {"interfaceName":"eth3","network":"vpc3"}, - {"interfaceName":"eth4","network":"vpc4"}, - {"interfaceName":"eth5","network":"vpc5"}, - {"interfaceName":"eth6","network":"vpc6"}, - {"interfaceName":"eth7","network":"vpc7"}, - {"interfaceName":"eth8","network":"vpc8"} -]""", - } - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("metadata", {}) - - # Add/update annotations - pod_template_spec = job_manifest["spec"]["template"]["metadata"] - if "annotations" in pod_template_spec: - pod_template_spec["annotations"].update(annotations) - else: - pod_template_spec["annotations"] = annotations - -def add_tolerations(job_manifest): - tolerations = [ - {"key": "user-workload", "operator": "Equal", "value": """\"true\"""", "effect": "NoSchedule"}, - ] - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add tolerations - pod_spec = job_manifest["spec"]["template"]["spec"] - if "tolerations" in pod_spec: - pod_spec["tolerations"].extend(tolerations) - else: - pod_spec["tolerations"] = tolerations - -def add_volumes(job_manifest): - volumes = [ - {"name": "nvidia-install-dir-host", "hostPath": {"path": "/home/kubernetes/bin/nvidia"}}, - {"name": "sys", "hostPath": {"path": "/sys"}}, - {"name": "proc-sys", "hostPath": {"path": "/proc/sys"}}, - {"name": "aperture-devices", "hostPath": {"path": "/dev/aperture_devices"}}, - ] - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add volumes - pod_spec = job_manifest["spec"]["template"]["spec"] - if "volumes" in pod_spec: - pod_spec["volumes"].extend(volumes) - else: - pod_spec["volumes"] = volumes - - -def add_tcpxo_daemon_container(job_template, rxdm_version): - tcpxo_daemon_container = { - "name": "tcpxo-daemon", - "image": f"us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:{rxdm_version}", # Use provided RxDM version - "imagePullPolicy": "Always", - "command": ["/bin/sh", "-c"], - "args": [ - """| - set -ex - chmod 755 /fts/entrypoint_rxdm_container.sh - /fts/entrypoint_rxdm_container.sh --num_hops=2 --num_nics=8 --uid= --alsologtostderr""" - ], - "securityContext": { - "capabilities": {"add": ["NET_ADMIN", "NET_BIND_SERVICE"]} - }, - "volumeMounts": [ - {"name": "nvidia-install-dir-host", "mountPath": "/usr/local/nvidia"}, - {"name": "sys", "mountPath": "/hostsysfs"}, - {"name": "proc-sys", "mountPath": "/hostprocsysfs"}, - ], - "env": [{"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}], - } - - # Create path if it doesn't exist - job_template.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add container - pod_spec = job_template["spec"]["template"]["spec"] - pod_spec.setdefault("containers", []).insert(0, tcpxo_daemon_container) - -def update_gpu_containers(job_manifest): - env_vars = [ - {"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}, - {"name": "NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY", "value": "/dev/aperture_devices"}, - ] - volume_mounts = [{"name": "aperture-devices", "mountPath": "/dev/aperture_devices"}] - - pod_spec = job_manifest.get("spec", {}).get("template", {}).get("spec", {}) - for container in pod_spec.get("containers", []): - # Create path if it doesn't exist - container.setdefault("env", []) - container.setdefault("volumeMounts", []) - if int(container.get("resources", {}).get("limits", {}).get("nvidia.com/gpu", 0)) > 0: - container["env"].extend(env_vars) - container["volumeMounts"].extend(volume_mounts) - -if __name__ == "__main__": - main() diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf deleted file mode 100644 index d23d050986..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# Enable GPUDirect for A3 and A3Mega VMs, this involve multiple kubectl steps to integrate with the created cluster -# 1. Install NCCL plugin daemonset -# 2. Install NRI plugin daemonset -# 3. Update provided workload to inject rxdm sidecar and other required annotation, volume etc. -locals { - workload_path_tcpx = "${path.module}/gpu-direct-workload/sample-tcpx-workload-job.yaml" - workload_path_tcpxo = "${path.module}/gpu-direct-workload/sample-tcpxo-workload-job.yaml" - - gpu_direct_settings = { - "a3-highgpu-8g" = { - # Manifest to be installed for enabling TCPX on a3-highgpu-8g machines - gpu_direct_manifests = [ - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-tcpx-installer.yaml", # nccl_plugin v3.1.9 for tcpx - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-config.yaml", # nccl_configmap - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/nri_device_injector/nri-device-injector.yaml", # nri_plugin - ] - updated_workload_path = replace(local.workload_path_tcpx, ".yaml", "-tcpx.yaml") - rxdm_version = "v2.0.12" # matching nccl-tcpx-installer version v3.1.9 - min_additional_networks = 4 - major_minor_version_acceptable_map = { - "1.27" = "1.27.7-gke.1121000" - "1.28" = "1.28.8-gke.1095000" - "1.29" = "1.29.3-gke.1093000" - "1.30" = "1.30.2-gke.1023000" - } - } - "a3-megagpu-8g" = { - # Manifest to be installed for enabling TCPXO on a3-megagpu-8g machines - gpu_direct_manifests = [ - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/gpudirect-tcpxo/nccl-tcpxo-installer.yaml", # nccl_plugin v1.0.14 for tcpxo - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/nri_device_injector/nri-device-injector.yaml", # nri_plugin - ] - updated_workload_path = replace(local.workload_path_tcpxo, ".yaml", "-tcpxo.yaml") - rxdm_version = "v1.0.20" # matching nccl-tcpxo-installer version v1.0.14 - min_additional_networks = 8 - major_minor_version_acceptable_map = { - "1.28" = "1.28.9-gke.1250000" - "1.29" = "1.29.4-gke.1542000" - "1.30" = "1.30.4-gke.1129000" - "1.31" = "1.31.1-gke.2008000" - "1.32" = "1.32.2-gke.1489001" - } - } - } - - min_additional_networks = try(local.gpu_direct_settings[var.machine_type].min_additional_networks, 0) - - gke_version_regex = "(\\d+\\.\\d+)\\.(\\d+)-gke\\.(\\d+)" # GKE version format: 1.X.Y-gke.Z , regex output: ["1.X" , "Y", "Z"] - - gke_version_parts = regex(local.gke_version_regex, var.gke_version) - gke_version_major = local.gke_version_parts[0] - - major_minor_version_acceptable_map = try(local.gpu_direct_setting[var.machine_type].major_minor_version_acceptable_map, null) - minor_version_acceptable = try(contains(keys(local.major_minor_version_acceptable_map), local.gke_version_major), false) ? local.major_minor_version_acceptable_map[local.gke_version_major] : "1.0.0-gke.0" - minor_version_acceptable_parts = regex(local.gke_version_regex, local.minor_version_acceptable) - gke_gpudirect_compatible = local.gke_version_parts[1] > local.minor_version_acceptable_parts[1] || (local.gke_version_parts[1] == local.minor_version_acceptable_parts[1] && local.gke_version_parts[2] >= local.minor_version_acceptable_parts[2]) -} - -check "gpu_direct_check_multi_vpc" { - assert { - condition = length(var.additional_networks) >= local.min_additional_networks - error_message = "To achieve optimal performance for ${var.machine_type} machine, at least ${local.min_additional_networks} additional vpc is recommended. You could configure it in the blueprint through modules/network/multivpc with network_count set as ${local.min_additional_networks}" - } -} - -check "gke_version_requirements" { - assert { - condition = local.gke_gpudirect_compatible - error_message = "GPUDirect is not supported on GKE version ${var.gke_version} for ${var.machine_type} machine. For supported version details visit https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#requirements" - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf deleted file mode 100644 index 1ddc7ba8c3..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -data "google_compute_machine_types" "machine_info" { - for_each = var.zones == null ? toset([]) : toset(var.zones) - - project = var.project_id - zone = each.key - filter = "name = \"${var.machine_type}\"" -} - -locals { - valid_machine_info = { - for zone, data in data.google_compute_machine_types.machine_info : - zone => data.machine_types if length(data.machine_types) > 0 - } - - guest_cpus = try(local.valid_machine_info[0].guest_cpus, 0) -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/main.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/main.tf deleted file mode 100644 index 05314497fc..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/main.tf +++ /dev/null @@ -1,482 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-node-pool", ghpc_role = "compute" }) -} - -locals { - upgrade_settings = { - strategy = var.upgrade_settings.strategy - max_surge = coalesce(var.upgrade_settings.max_surge, 0) - max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) - } -} - -module "gpu" { - source = "../../internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - has_gpu = length(local.guest_accelerator) > 0 - allocatable_gpu_per_node = local.has_gpu ? max(local.guest_accelerator[*].count...) : -1 - is_static_node_pool_with_gpus = var.static_node_count != null && local.allocatable_gpu_per_node != -1 - static_gpu_count = local.is_static_node_pool_with_gpus ? var.static_node_count * local.allocatable_gpu_per_node : 0 - gpu_taint = local.has_gpu ? [{ - key = "nvidia.com/gpu" - value = "present" - effect = "NO_SCHEDULE" - }] : [] - - autoscale_set = var.autoscaling_total_min_nodes != 0 || var.autoscaling_total_max_nodes != 1000 - static_node_set = var.static_node_count != null - initial_node_set = try(var.initial_node_count > 0, false) - - module_unique_id = replace(lower(var.internal_ghpc_module_id), "/[^a-z0-9\\-]/", "") -} - - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] -} - -module "tpu" { - source = "../../internal/tpu-definition" - - machine_type = var.machine_type - placement_policy = var.placement_policy -} - - -data "google_container_cluster" "gke_cluster" { - name = local.cluster_name - location = local.cluster_location -} - -resource "google_container_node_pool" "node_pool" { - provider = google-beta - - count = max(var.num_node_pools, var.num_slices) - - name = (max(var.num_node_pools, var.num_slices) == 1) ? coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])) : join("-", [coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])), count.index]) - cluster = var.cluster_id - node_locations = var.zones - - node_count = var.static_node_count - dynamic "autoscaling" { - for_each = local.static_node_set ? [] : [1] - content { - total_min_node_count = var.autoscaling_total_min_nodes - total_max_node_count = var.autoscaling_total_max_nodes - location_policy = "ANY" - } - } - - initial_node_count = var.initial_node_count - - max_pods_per_node = var.max_pods_per_node - - management { - auto_repair = var.auto_repair - auto_upgrade = var.auto_upgrade - } - - upgrade_settings { - strategy = local.upgrade_settings.strategy - max_surge = local.upgrade_settings.max_surge - max_unavailable = local.upgrade_settings.max_unavailable - } - - dynamic "placement_policy" { - for_each = var.placement_policy.type != null ? [1] : [] - content { - type = var.placement_policy.type - policy_name = var.placement_policy.name - tpu_topology = module.tpu.is_tpu ? var.placement_policy.tpu_topology : null - } - } - - dynamic "queued_provisioning" { - for_each = var.enable_queued_provisioning ? [1] : [] - content { - enabled = true - } - } - - node_config { - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - resource_labels = local.labels - labels = var.kubernetes_labels - service_account = var.service_account_email - oauth_scopes = var.service_account_scopes - machine_type = var.machine_type - spot = var.spot - image_type = var.image_type - flex_start = var.enable_flex_start - max_run_duration = var.max_run_duration != null ? "${var.max_run_duration}s" : null - - dynamic "guest_accelerator" { - for_each = local.guest_accelerator - iterator = ga - content { - type = coalesce(ga.value.type, try(local.generated_guest_accelerator[0].type, "")) - count = coalesce(try(ga.value.count, 0) > 0 ? ga.value.count : try(local.generated_guest_accelerator[0].count, "0")) - - gpu_partition_size = try(ga.value.gpu_partition_size, null) - - dynamic "gpu_driver_installation_config" { - # in case user did not specify guest_accelerator settings, we need a try to default to [] - for_each = try([ga.value.gpu_driver_installation_config], [{ gpu_driver_version = "DEFAULT" }]) - iterator = gdic - content { - gpu_driver_version = gdic.value.gpu_driver_version - } - } - - dynamic "gpu_sharing_config" { - for_each = try(ga.value.gpu_sharing_config == null, true) ? [] : [ga.value.gpu_sharing_config] - iterator = gsc - content { - gpu_sharing_strategy = gsc.value.gpu_sharing_strategy - max_shared_clients_per_gpu = gsc.value.max_shared_clients_per_gpu - } - } - } - } - - dynamic "taint" { - for_each = concat(var.taints, local.gpu_taint, module.tpu.tpu_taint) - content { - key = taint.value.key - value = taint.value.value - effect = taint.value.effect - } - } - - dynamic "ephemeral_storage_local_ssd_config" { - for_each = local.local_ssd_config.local_ssd_count_ephemeral_storage != null ? [1] : [] - content { - local_ssd_count = local.local_ssd_config.local_ssd_count_ephemeral_storage - } - } - - dynamic "local_nvme_ssd_block_config" { - for_each = local.local_ssd_config.local_ssd_count_nvme_block != null ? [1] : [] - content { - local_ssd_count = local.local_ssd_config.local_ssd_count_nvme_block - } - } - - shielded_instance_config { - enable_secure_boot = var.enable_secure_boot - enable_integrity_monitoring = true - } - - dynamic "gcfs_config" { - for_each = var.enable_gcfs ? [1] : [] - content { - enabled = true - } - } - - gvnic { - enabled = var.image_type == "COS_CONTAINERD" - } - - dynamic "advanced_machine_features" { - for_each = local.set_threads_per_core ? [1] : [] - content { - threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf - } - } - - # Implied by Workload Identity - workload_metadata_config { - mode = "GKE_METADATA" - } - # Implied by workload identity. - metadata = { - "disable-legacy-endpoints" = "true" - } - - linux_node_config { - sysctls = { - "net.ipv4.tcp_rmem" = "4096 87380 16777216" - "net.ipv4.tcp_wmem" = "4096 16384 16777216" - } - } - - reservation_affinity { - consume_reservation_type = var.reservation_affinity.consume_reservation_type - key = local.is_valid_reservation ? local.reservation_resource_api_label : null - values = local.is_valid_reservation ? (var.is_reservation_active ? local.active_reservation_values : local.default_reservation_values) : null - } - - dynamic "host_maintenance_policy" { - for_each = var.host_maintenance_interval != "" ? [1] : [] - content { - maintenance_interval = var.host_maintenance_interval - } - } - - kubelet_config { - cpu_manager_policy = var.enable_numa_aware_scheduling ? "static" : null - dynamic "topology_manager" { - for_each = var.enable_numa_aware_scheduling ? [1] : [] - content { - policy = "restricted" - } - } - dynamic "memory_manager" { - for_each = var.enable_numa_aware_scheduling ? [1] : [] - content { - policy = "Static" - } - } - } - } - - network_config { - dynamic "additional_node_network_configs" { - for_each = var.additional_networks - - content { - network = additional_node_network_configs.value.network - subnetwork = additional_node_network_configs.value.subnetwork - } - } - - enable_private_nodes = var.enable_private_nodes - } - - timeouts { - create = var.timeout_create - update = var.timeout_update - } - - lifecycle { - ignore_changes = [ - node_config[0].labels, - initial_node_count, - # Ignore local/ephemeral ssd configs as they are tied to machine types. - node_config[0].ephemeral_storage_local_ssd_config, - node_config[0].local_nvme_ssd_block_config, - ] - precondition { - condition = (var.max_pods_per_node == null) || (data.google_container_cluster.gke_cluster.networking_mode == "VPC_NATIVE") - error_message = "max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." - } - precondition { - condition = !local.static_node_set || !local.autoscale_set - error_message = "static_node_count cannot be set with either autoscaling_total_min_nodes or autoscaling_total_max_nodes." - } - precondition { - condition = !local.static_node_set || !local.initial_node_set - error_message = "initial_node_count cannot be set with static_node_count." - } - precondition { - condition = !local.initial_node_set || (coalesce(var.initial_node_count, 0) >= var.autoscaling_total_min_nodes && coalesce(var.initial_node_count, 0) <= var.autoscaling_total_max_nodes) - error_message = "initial_node_count must be between autoscaling_total_min_nodes and autoscaling_total_max_nodes included." - } - precondition { - condition = !(coalesce(local.local_ssd_config.local_ssd_count_ephemeral_storage, 0) > 0 && coalesce(local.local_ssd_config.local_ssd_count_nvme_block, 0) > 0) - error_message = "Only one of local_ssd_count_ephemeral_storage or local_ssd_count_nvme_block can be set to a non-zero value." - } - precondition { - condition = ( - (var.reservation_affinity.consume_reservation_type != "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 0) || - (var.reservation_affinity.consume_reservation_type == "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 1) - ) - error_message = <<-EOT - When using NO_RESERVATION or ANY_RESERVATION as the `consume_reservation_type`, `specific_reservations` cannot be set. - On the other hand, with SPECIFIC_RESERVATION you must set `specific_reservations`. - EOT - } - precondition { - condition = ( - (local.input_specific_reservations_count == 0) || - ((length(local.verified_specific_reservations) == 1 || !var.is_reservation_active) && - length(local.specific_reservation_requirement_violations) == 0) - ) - error_message = <<-EOT - Check if your reservation is configured correctly: - - A reservation with the name must exist in the specified project and one of the specified zones - - - Its consumption type must be "specific" - %{for property in local.specific_reservation_requirement_violations} - - ${local.specific_reservation_requirement_violation_messages[property]} - %{endfor} - EOT - } - precondition { - condition = ( - (local.input_specific_reservations_count == 0) || - (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) == 0) || - (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) > 0 && try(local.input_reservation_projects[0], var.project_id) == var.project_id) - ) - error_message = "Shared extended reservations are not supported by GKE." - } - precondition { - condition = contains(["SURGE"], local.upgrade_settings.strategy) - error_message = "Only SURGE strategy is supported" - } - precondition { - condition = local.upgrade_settings.max_unavailable >= 0 - error_message = "max_unavailable should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_surge >= 0 - error_message = "max_surge should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 - error_message = "At least one of max_unavailable or max_surge must greater than 0" - } - precondition { - condition = var.placement_policy.type != "COMPACT" || (var.zones != null ? (length(var.zones) == 1) : false) - error_message = "Compact placement is only available for node pools operating in a single zone." - } - precondition { - condition = var.placement_policy.type != "COMPACT" || local.upgrade_settings.strategy != "BLUE_GREEN" - error_message = "Compact placement is not supported with blue-green upgrades." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.placement_policy.type == "COMPACT") - error_message = "placement_policy cannot be COMPACT when enable_queued_provisioning is true." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.reservation_affinity.consume_reservation_type != "NO_RESERVATION") - error_message = "reservation_affinity should be NO_RESERVATION when enable_queued_provisioning is true." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.autoscaling_total_min_nodes != 0) - error_message = "autoscaling_total_min_nodes should be 0 when enable_queued_provisioning is true." - } - precondition { - condition = !(var.num_node_pools > 1 && var.num_slices > 1) - error_message = "num_node_pools is for CPUs and GPUS, and num_slices is for TPUs. Both cannot be set at the same time to create a group of identical nodepools / slices." - } - precondition { - condition = !(var.num_node_pools == 0 && var.num_slices == 0) - error_message = "Either num_node_pools (for CPUs and GPUS) or num_slices (for TPUs) should be set to a positive integer value." - } - precondition { - condition = !(var.num_node_pools < 0 || var.num_slices < 0) - error_message = "Negative integer value of num_node_pools or num_slices is not valid. Please use a positive integer value to set num_node_pools for CPUs and GPUS, and num_slices for TPUs." - } - precondition { - condition = var.enable_flex_start == true ? (var.auto_repair == false) : true - error_message = "enable_flex_start needs node auto_repair set to false." - } - precondition { - condition = var.enable_flex_start == true ? (var.static_node_count == null) : true - error_message = "enable_flex_start does not work with static_node_count. static_node_count should be set to null." - } - precondition { - condition = var.enable_flex_start == true ? (var.reservation_affinity.consume_reservation_type == "NO_RESERVATION") : true - error_message = "enable_flex_start only works with reservation_affinity consume_reservation_type NO_RESERVATION." - } - precondition { - condition = var.enable_flex_start == true ? (var.spot == false) : true - error_message = "Both enable_flex_start and spot consumption option cannot be set to true at the same time." - } - } -} - -locals { - supported_machine_types_for_install_dependencies = ["a3-highgpu-8g", "a3-megagpu-8g"] -} - -# Replicates GKE's naming logic for its instance templates. The full -# pattern is "gke-{cluster_name}-{nodepool_name}-{hash}". -# -# This code builds the "{cluster_name}-{nodepool_name}" prefix, which is -# capped at 32 characters plus a dash '-' in between, by truncating names if needed: -# - If both names > 16 chars, both are cut to 16. -# - If one name > 16, it's shortened so the combined name length is 32. -data "google_compute_region_instance_template" "instance_template" { - for_each = { for idx, np in google_container_node_pool.node_pool : idx => np } - project = var.project_id - filter = "name: gke-${ - (length(local.cluster_name) <= 16 && length(each.value.name) <= 16) ? "${local.cluster_name}-${each.value.name}" : - (length(local.cluster_name) > 16 && length(each.value.name) > 16) ? "${substr(local.cluster_name, 0, 16)}-${substr(each.value.name, 0, 16)}" : - (length(local.cluster_name) > 16) ? "${substr(local.cluster_name, 0, 32 - length(each.value.name))}-${each.value.name}" : - "${local.cluster_name}-${substr(each.value.name, 0, 32 - length(local.cluster_name))}" - }*" - most_recent = true -} - -resource "null_resource" "install_dependencies" { - count = var.run_workload_script && contains(local.supported_machine_types_for_install_dependencies, var.machine_type) ? 1 : 0 - provisioner "local-exec" { - command = "pip3 install pyyaml" - } -} - -locals { - gpu_direct_setting = lookup(local.gpu_direct_settings, var.machine_type, { gpu_direct_manifests = [], updated_workload_path = "", rxdm_version = "" }) -} - -# execute script to inject rxdm sidecar into workload to enable tcpx for a3-highgpu-8g VM workload -resource "null_resource" "enable_tcpx_in_workload" { - count = var.run_workload_script && var.machine_type == "a3-highgpu-8g" ? 1 : 0 - triggers = { - always_run = timestamp() - } - provisioner "local-exec" { - command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpx-in-workload.py --file ${local.workload_path_tcpx} --rxdm ${local.gpu_direct_setting.rxdm_version}" - } - - depends_on = [null_resource.install_dependencies] -} - -# execute script to inject rxdm sidecar into workload to enable tcpxo for a3-megagpu-8g VM workload -resource "null_resource" "enable_tcpxo_in_workload" { - count = var.run_workload_script && var.machine_type == "a3-megagpu-8g" ? 1 : 0 - triggers = { - always_run = timestamp() - } - provisioner "local-exec" { - command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py --file ${local.workload_path_tcpxo} --rxdm ${local.gpu_direct_setting.rxdm_version}" - } - - depends_on = [null_resource.install_dependencies] -} - -# apply manifest to enable tcpx -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = var.cluster_id - project_id = var.project_id - - apply_manifests = flatten([ - for manifest in local.gpu_direct_setting.gpu_direct_manifests : [ - { - source = manifest - } - ] - ]) -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/metadata.yaml deleted file mode 100644 index e980d595a2..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com -ghpc: - inject_module_id: internal_ghpc_module_id diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/outputs.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/outputs.tf deleted file mode 100644 index 44e1c3d971..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/outputs.tf +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "node_pool_names" { - description = "Names of the node pools." - value = google_container_node_pool.node_pool[*].name -} - -locals { - # Shared core machines only have 1 cpu allocatable, even if they have 2 cpu capacity - vcpu = local.machine_shared_core ? 1 : local.guest_cpus - useable_cpu = local.set_threads_per_core ? local.threads_per_core * local.vcpu / 2 : local.vcpu - - # allocatable resource definition: https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes#cpu_reservations - second_core = local.useable_cpu > 1 ? 1 : 0 - third_fourth_core = local.useable_cpu == 3 ? 1 : local.useable_cpu > 3 ? 2 : 0 - cores_above_four = local.useable_cpu > 4 ? local.useable_cpu - 4 : 0 - - allocatable_cpu = 0.94 + (0.99 * local.second_core) + (0.995 * local.third_fourth_core) + (0.9975 * local.cores_above_four) -} - -output "allocatable_cpu_per_node" { - description = "Number of CPUs available for scheduling pods on each node." - value = local.allocatable_cpu -} - -output "has_gpu" { - description = "Boolean value indicating whether nodes in the pool are configured with GPUs." - value = local.has_gpu -} - -output "allocatable_gpu_per_node" { - description = "Number of GPUs available for scheduling pods on each node." - value = local.allocatable_gpu_per_node -} - -output "static_gpu_count" { - description = "Total number of GPUs in the node pool. Available only for static node pools." - value = local.static_gpu_count -} - -locals { - translate_toleration = { - PREFER_NO_SCHEDULE = "PreferNoSchedule" - NO_SCHEDULE = "NoSchedule" - NO_EXECUTE = "NoExecute" - } - taints = google_container_node_pool.node_pool[0].node_config[0].taint - tolerations = [for taint in local.taints : { - key = taint.key - operator = "Equal" - value = taint.value - effect = lookup(local.translate_toleration, taint.effect, null) - }] -} - -output "tolerations" { - description = "Tolerations needed for a pod to be scheduled on this node pool." - value = local.tolerations -} - -locals { - gpu_direct_enabled = var.machine_type == "a3-highgpu-8g" || var.machine_type == "a3-megagpu-8g" - script_path = { - a3-highgpu-8g = "enable-tcpx-in-workload.py", - a3-megagpu-8g = "enable-tcpxo-in-workload.py" - } - nccl_path = var.machine_type == "a3-highgpu-8g" ? "configs" : "scripts" - gpu_direct_instruction = <<-EOT - Since you are using ${var.machine_type} machine type that has GPUDirect support, your nodepool had been configured with the required plugins. - To fully utilize GPUDirect you will need to add some components into your workload manifest. Details below: - - A sample GKE job that has GPUDirect enabled and NCCL test included has been generated locally at: - ${abspath(local.gpu_direct_setting.updated_workload_path)} - - You can use the following commands to submit the sample job: - kubectl create -f ${abspath(local.gpu_direct_setting.updated_workload_path)} - After submitting the sample job, you can validate the GPU performance by initiating NCCL test included in the sample workload: - NCCL test can be initiated from any one of the sample job Pods and coordinate with the peer Pods: - export POD_NAME=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | head -n 1) - export PEER_POD_IPS=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.status.podIP}}{{" "}}{{end}}') - kubectl exec --stdin --tty --container=nccl-test $POD_NAME -- /${local.nccl_path}/allgather.sh $PEER_POD_IPS - - If you would like to enable GPUDirect for your own workload, please follow the below steps: - export WORKLOAD_PATH=<> - python3 ${abspath("${path.module}/gpu-direct-workload/scripts/${lookup(local.script_path, var.machine_type, "")}")} --file $WORKLOAD_PATH --rxdm ${local.gpu_direct_setting.rxdm_version} - **WARNING** - The "--rxdm" version is tied to the nccl-tcpx/o-installer that had been deployed to your cluster, changing it to other value might have impact on performance - **WARNING** - - Or you can also follow our GPUDirect user guide to update your workload - https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#add-gpudirect-manifests - EOT -} - -output "instructions" { - description = "Instructions for submitting the sample GPUDirect enabled job." - value = local.gpu_direct_enabled ? local.gpu_direct_instruction : null -} - -output "node_count_static" { - description = "The number of static nodes in node-pool." - value = coalesce(var.static_node_count, var.initial_node_count, 0) -} - -output "guest_accelerator" { - description = "The accelerator type of the nodes." - value = local.guest_accelerator -} - -output "cluster_id" { - description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." - value = var.cluster_id -} - -output "machine_type" { - description = "Machine Type" - value = var.machine_type -} - -output "instance_templates" { - description = "The URLs of Instance Templates" - value = [for key, template in data.google_compute_region_instance_template.instance_template : template.self_link] -} - -output "tpu_accelerator_type" { - description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." - value = module.tpu.is_tpu ? module.tpu.tpu_accelerator_type : null -} - -output "tpu_topology" { - description = "The topology of the TPU slice (e.g., '4x4')." - value = module.tpu.is_tpu ? module.tpu.tpu_topology : null -} - -output "tpu_chips_per_node" { - description = "The number of TPU chips on each node in the pool." - value = module.tpu.is_tpu ? module.tpu.tpu_chips_per_node : null -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf deleted file mode 100644 index 7c29e3902a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# Split the input into three different lists where the details of a given reservation are at the same index across these lists. -locals { - # Specific block of an extended reservation can be targeted with exr-one/reservationBlocks/exr-one-block-1 - # Data source needs to be queried with the reservation name only. So, we extract the reservation name - input_reservation_names = [for r in try(var.reservation_affinity.specific_reservations, []) : split("/", r.name)[0]] - input_reservation_projects = [for r in try(var.reservation_affinity.specific_reservations, []) : coalesce(r.project, var.project_id)] - # We, also, remember the suffix "/reservationBlocks/exr-one-block-1" for use elsewhere afterwards - input_reservation_suffixes = [for r in try(var.reservation_affinity.specific_reservations, []) : substr(r.name, length(split("/", r.name)[0]), -1)] - # Adding this variable to by-pass the machine-type validation for TPUs - is_tpu = var.placement_policy.tpu_topology != null -} - -data "google_compute_reservation" "specific_reservations" { - for_each = ( - local.input_specific_reservations_count == 0 ? - {} : - { - for pair in flatten([ - for zone in try(var.zones, []) : [ - for i, reservation_name in try(local.input_reservation_names, []) : { - key : "${local.input_reservation_projects[i]}/${zone}/${reservation_name}" - zone : zone - reservation_name : reservation_name - project : local.input_reservation_projects[i] - } - ] - ]) : - pair.key => pair - } - ) - name = each.value.reservation_name - zone = each.value.zone - project = each.value.project -} - -locals { - generated_guest_accelerator = module.gpu.machine_type_guest_accelerator - reservation_resource_api_label = "compute.googleapis.com/reservation-name" - input_specific_reservations_count = try(length(var.reservation_affinity.specific_reservations), 0) - - # Filter specific reservations - verified_specific_reservations = [for k, v in data.google_compute_reservation.specific_reservations : v if(v.specific_reservation != null && v.specific_reservation_required == true)] - - # Build two maps to be used to compare the VM properties between reservations and the node pool - # Validation of only machine-type for CPUs and and both machine-type and guest-accelerators for GPUs - # Skip this for TPUs ( returns an empty list to skip the machine-type validation for aggregate TPU reservations) - reservation_vm_properties = local.is_tpu ? [] : [for reservation in local.verified_specific_reservations : { - "machine_type" : try(reservation.specific_reservation[0].instance_properties[0].machine_type, "") - "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs - { for acc in try(reservation.specific_reservation[0].instance_properties[0].guest_accelerators, []) : acc.accelerator_type => acc.accelerator_count } - ) : {} # If no GPUs, it's an empty map {} - }] - - nodepool_vm_properties = { - "machine_type" : var.machine_type - "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs - { for acc in try(local.guest_accelerator, []) : coalesce(acc.type, try(local.generated_guest_accelerator[0].type, "")) => coalesce(acc.count, try(local.generated_guest_accelerator[0].count, 0)) } - ) : {} # If no GPUs, it's an empty map {} - } - - # Compare two maps by counting the keys that mismatch. - # Know that in map comparison the order of keys does not matter. That is {NVME: x, SCSI: y} and {SCSI: y, NVME: x} are equal - # As of this writing, there is only one reservation supported by the Node Pool API. So, directly accessing it from the list - specific_reservation_requirement_violations = length(local.reservation_vm_properties) == 0 ? [] : [for k, v in local.nodepool_vm_properties : k if v != local.reservation_vm_properties[0][k]] - - specific_reservation_requirement_violation_messages = { - "machine_type" : <<-EOT - The reservation has "${try(local.reservation_vm_properties[0].machine_type, "")}" machine type and the node pool has "${local.nodepool_vm_properties.machine_type}". Check the relevant node pool setting: "machine_type" - EOT - "guest_accelerators" : <<-EOT - The reservation has ${jsonencode(try(local.reservation_vm_properties[0].guest_accelerators, {}))} accelerators and the node pool has ${jsonencode(try(local.nodepool_vm_properties.guest_accelerators, {}))}. Check the relevant node pool setting: "guest_accelerator". When unspecified, for the machine_type=${var.machine_type}, the default is guest_accelerator=${jsonencode(try(local.generated_guest_accelerator, [{}]))}. - EOT - } -} - -locals { - # Check if reservation is valid, that is, if it exists, there should be only 1 verified specific reservation or the reservation doesn't exist - is_valid_reservation = length(local.verified_specific_reservations) == 1 || !var.is_reservation_active - - # Build the list of reservation names when var.is_reservation_active is true - active_reservation_values = [ - for i, r in local.verified_specific_reservations : - length(local.input_reservation_suffixes[i]) > 0 ? - format("%s%s", r.name, local.input_reservation_suffixes[i]) : - "projects/${r.project}/reservations/${r.name}" - ] - - # Define a default reservation value if no specific reservations are present - specific_reservation_name = length(local.input_reservation_names) > 0 ? local.input_reservation_names[0] : "" - default_reservation_values = ["projects/${var.project_id}/reservations/${local.specific_reservation_name}"] -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf deleted file mode 100644 index e582db33da..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' - -# "inputs": -# var.machine_type: Machine type for the instance being evaluated. -# var.threads_per_core : Sets the number of threads per physical core, where 0 -# has behavior described in description. - -# "outputs": -# local.set_threads_per_core: bool that tells if threads per core should be set, -# to be used with a dynamic block. -# local.threads_per_core: actual threads_per_core to be used. - -locals { - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - machine_shared_core = length(local.machine_vals) <= 2 - machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) - - smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) - smt_capable_vcpu = local.machine_vcpus >= 2 - - smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core - set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) - threads_per_core = var.threads_per_core == 2 ? 2 : 1 -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/variables.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/variables.tf deleted file mode 100644 index b44ea28d57..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/variables.tf +++ /dev/null @@ -1,487 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "cluster_id" { - description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" - type = string -} - -variable "zones" { - description = "A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable." - type = list(string) - default = null -} - -variable "name" { - description = <<-EOD - The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix. - If setting manually, ensure a unique value across all gke-node-pools. - EOD - type = string - default = null - - validation { - # Check if the variable is null OR if it matches the GCP resource naming regex. - condition = var.name == null || can(regex("^[a-z]([-a-z0-9]{0,34}[a-z0-9])?$", var.name)) - error_message = <<-EOD - If provided, the node pool name must be between 1 and 36 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. - Underscores are not allowed. A shorter length is enforced to accommodate a suffix when creating multiple node pools. - EOD - } -} - -variable "internal_ghpc_module_id" { - description = "DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide)." - type = string -} - -variable "machine_type" { - description = "The name of a Google Compute Engine machine type." - type = string - default = "c2-standard-60" -} - -variable "disk_size_gb" { - description = "Size of disk for each node." - type = number - default = 100 -} - -variable "disk_type" { - description = "Disk type for each node." - type = string - default = null -} - -variable "enable_gcfs" { - description = "Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config)." - type = bool - default = false -} - -variable "enable_secure_boot" { - description = "Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info." - type = bool - default = true -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = optional(string) - count = optional(number, 0) - gpu_driver_installation_config = optional(object({ - gpu_driver_version = string - }), { gpu_driver_version = "DEFAULT" }) - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = string - max_shared_clients_per_gpu = number - })) - })) - default = [] - nullable = false - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.count != null]) - error_message = "var.guest_accelerator[*].count cannot be null" - } - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.count >= 0]) - error_message = "var.guest_accelerator[*].count must never be negative" - } - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.gpu_driver_installation_config != null]) - error_message = "var.guest_accelerator[*].gpu_driver_installation_config must not be null; leave unset to enable GKE to select default GPU driver installation" - } -} - -variable "image_type" { - description = "The default image type used by NAP once a new node pool is being created. Use either COS_CONTAINERD or UBUNTU_CONTAINERD." - type = string - default = "COS_CONTAINERD" -} - -variable "local_ssd_count_ephemeral_storage" { - description = <<-EOT - The number of local SSDs to attach to each node to back ephemeral storage. - Uses NVMe interfaces. Must be supported by `machine_type`. - When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. - [See above](#local-ssd-storage) for more info. - EOT - type = number - default = null -} - -variable "local_ssd_count_nvme_block" { - description = <<-EOT - The number of local SSDs to attach to each node to back block storage. - Uses NVMe interfaces. Must be supported by `machine_type`. - When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. - [See above](#local-ssd-storage) for more info. - - EOT - type = number - default = null -} - -variable "autoscaling_total_min_nodes" { - description = "Total minimum number of nodes in the NodePool." - type = number - default = 0 -} - -variable "autoscaling_total_max_nodes" { - description = "Total maximum number of nodes in the NodePool." - type = number - default = 1000 -} - -variable "static_node_count" { - description = "The static number of nodes in the node pool. If set, autoscaling will be disabled." - type = number - default = null -} - -variable "is_reservation_active" { - description = "Whether the specified reservation is already created." - type = bool - default = true -} - -variable "auto_repair" { - description = "Whether the nodes will be automatically repaired." - type = bool - default = true -} - -variable "auto_upgrade" { - description = "Whether the nodes will be automatically upgraded." - type = bool - default = false -} - -variable "threads_per_core" { - description = <<-EOT - Sets the number of threads per physical core. By setting threads_per_core - to 2, Simultaneous Multithreading (SMT) is enabled extending the total number - of virtual cores. For example, a machine of type c2-standard-60 will have 60 - virtual cores with threads_per_core equal to 2. With threads_per_core equal - to 1 (SMT turned off), only the 30 physical cores will be available on the VM. - - The default value of \"0\" will turn off SMT for supported machine types, and - will fall back to GCE defaults for unsupported machine types (t2d, shared-core - instances, or instances with less than 2 vCPU). - - Disabling SMT can be more performant in many HPC workloads, therefore it is - disabled by default where compatible. - - null = SMT configuration will use the GCE defaults for the machine type - 0 = SMT will be disabled where compatible (default) - 1 = SMT will always be disabled (will fail on incompatible machine types) - 2 = SMT will always be enabled (will fail on incompatible machine types) - EOT - type = number - default = 0 - - validation { - condition = var.threads_per_core == null || try(var.threads_per_core >= 0, false) && try(var.threads_per_core <= 2, false) - error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." - } -} - -variable "spot" { - description = "Provision VMs using discounted Spot pricing, allowing for preemption" - type = bool - default = false -} - -# tflint-ignore: terraform_unused_declarations -variable "compact_placement" { - description = "DEPRECATED: Use `placement_policy`" - type = bool - default = null - validation { - condition = var.compact_placement == null - error_message = "`compact_placement` is deprecated. Use `placement_policy` instead" - } -} - -variable "placement_policy" { - description = <<-EOT - Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool. - It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - EOT - - type = object({ - type = string - name = optional(string) - tpu_topology = optional(string) - }) - default = { - type = null - name = null - tpu_topology = null - } - validation { - condition = var.placement_policy.type == null || try(contains(["COMPACT"], var.placement_policy.type), false) - error_message = "`COMPACT` is the only supported value for `placement_policy.type`." - } -} - -variable "service_account_email" { - description = "Service account e-mail address to use with the node pool" - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to to use with the node pool." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "taints" { - description = "Taints to be applied to the system node pool." - type = list(object({ - key = string - value = any - effect = string - })) - default = [] -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} - -variable "kubernetes_labels" { - description = <<-EOT - Kubernetes labels to be applied to each node in the node group. Key-value pairs. - (The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) - EOT - type = map(string) - default = null -} - -variable "timeout_create" { - description = "Timeout for creating a node pool" - type = string - default = null -} - -variable "timeout_update" { - description = "Timeout for updating a node pool" - type = string - default = null -} - -# Deprecated - -# tflint-ignore: terraform_unused_declarations -variable "total_min_nodes" { - description = "DEPRECATED: Use autoscaling_total_min_nodes." - type = number - default = null - validation { - condition = var.total_min_nodes == null - error_message = "total_min_nodes was renamed to autoscaling_total_min_nodes and is deprecated; use autoscaling_total_min_nodes" - } -} - -# tflint-ignore: terraform_unused_declarations -variable "total_max_nodes" { - description = "DEPRECATED: Use autoscaling_total_max_nodes." - type = number - default = null - validation { - condition = var.total_max_nodes == null - error_message = "total_max_nodes was renamed to autoscaling_total_max_nodes and is deprecated; use autoscaling_total_max_nodes" - } -} - -# tflint-ignore: terraform_unused_declarations -variable "service_account" { - description = "DEPRECATED: use service_account_email and scopes." - type = object({ - email = string, - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "service_account is deprecated and replaced with service_account_email and scopes." - } -} - -variable "additional_networks" { - description = "Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool" - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - })) - nullable = false -} - -variable "reservation_affinity" { - description = <<-EOT - Reservation resource to consume. When targeting SPECIFIC_RESERVATION, specific_reservations needs be specified. - Even though specific_reservations is a list, only one reservation is allowed by the NodePool API. - It is assumed that the specified reservation exists and has available capacity. - For a shared reservation, specify the project_id as well in which it was created. - To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared - EOT - type = object({ - consume_reservation_type = string - specific_reservations = optional(list(object({ - name = string - project = optional(string) - }))) - }) - default = { - consume_reservation_type = "NO_RESERVATION" - specific_reservations = [] - } - validation { - condition = contains(["NO_RESERVATION", "ANY_RESERVATION", "SPECIFIC_RESERVATION"], var.reservation_affinity.consume_reservation_type) - error_message = "Accepted values are: {NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION}" - } -} - -variable "host_maintenance_interval" { - description = "Specifies the frequency of planned maintenance events." - type = string - default = "" - nullable = false - validation { - condition = contains(["", "PERIODIC", "AS_NEEDED"], var.host_maintenance_interval) - error_message = "Invalid host_maintenance_interval value. Must be PERIODIC, AS_NEEDED or the empty string" - } -} - -variable "initial_node_count" { - description = "The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static_node_count and must be set to a value between autoscaling_total_min_nodes and autoscaling_total_max_nodes." - type = number - default = null -} - -variable "gke_version" { - description = "GKE version" - type = string -} - -variable "max_pods_per_node" { - description = "The maximum number of pods per node in this node pool. This will force replacement." - type = number - default = null -} - -variable "upgrade_settings" { - description = <<-EOT - Defines node pool upgrade settings. It is highly recommended that you define all max_surge and max_unavailable. - If max_surge is not specified, it would be set to a default value of 0. - If max_unavailable is not specified, it would be set to a default value of 1. - EOT - type = object({ - strategy = string - max_surge = optional(number) - max_unavailable = optional(number) - }) - default = { - strategy = "SURGE" - max_surge = 0 - max_unavailable = 1 - } -} - -variable "run_workload_script" { - description = "Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only." - type = bool - default = true -} - -variable "enable_queued_provisioning" { - description = "If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool." - type = bool - default = false -} - -variable "enable_flex_start" { - description = <<-EOT - If true, start the node pool with Flex Start provisioning model. - To learn more about flex-start mode, please refer to - https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and - https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest - EOT - type = bool - default = false -} - -variable "max_run_duration" { - description = "The duration (in whole seconds) of the instance. Instance will run and be terminated after then." - type = number - default = null -} - -variable "enable_private_nodes" { - description = "Whether nodes have internal IP addresses only." - type = bool - default = true -} - -variable "num_node_pools" { - description = "Number of node pools to create. This is same as num_slices." - type = number - default = 1 -} - -variable "num_slices" { - description = "Number of TPUs slices to create. This is same as num_node_pools." - type = number - default = 1 -} - -variable "enable_numa_aware_scheduling" { - description = "Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling." - type = bool - default = false -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/versions.tf b/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/versions.tf deleted file mode 100644 index f018d04fc5..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/gke-node-pool/versions.tf +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.5" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 7.2" - } - google-beta = { - source = "hashicorp/google-beta" - version = ">= 7.2" - } - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/README.md b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/README.md deleted file mode 100644 index 3b769e8761..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/README.md +++ /dev/null @@ -1,82 +0,0 @@ -## Description - -This modules create a [resource policy for compute engines](https://cloud.google.com/compute/docs/instances/placement-policies-overview). This policy can be passed to a gke-node-pool module to apply the policy on the node-pool's nodes. - -Note: By default, you can't apply compact placement policies with a max distance value to A3 VMs. To request access to this feature, contact your [Technical Account Manager (TAM)](https://cloud.google.com/tam) or the [Sales team](https://cloud.google.com/contact). - -### Example - -The following example creates a group placement resource policy and applies it to a gke-node-pool. - -```yaml - - id: group_placement_1 - source: modules/compute/resource-policy - settings: - name: gp-np-1 - group_placement_max_distance: 2 - - - id: node_pool_1 - source: modules/compute/gke-node-pool - use: [group_placement_1] - settings: - machine_type: e2-standard-8 - outputs: [instructions] -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google-beta](#requirement\_google-beta) | >= 6.29.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google-beta](#provider\_google-beta) | >= 6.29.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_resource_policy.policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [group\_placement\_max\_distance](#input\_group\_placement\_max\_distance) | The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | `number` | `0` | no | -| [name](#input\_name) | The resource policy's name. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The project ID for the resource policy. | `string` | n/a | yes | -| [region](#input\_region) | The region for the the resource policy. | `string` | n/a | yes | -| [workload\_policy](#input\_workload\_policy) | Describes the workload policy | 
object({
    type                  = optional(string, null)
    max_topology_distance = optional(string, null)
    accelerator_topology  = optional(string, null)
  })
| 
{
  "accelerator_topology": null,
  "max_topology_distance": null,
  "type": null
}
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [placement\_policy](#output\_placement\_policy) | Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions.
The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. | - diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/main.tf b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/main.tf deleted file mode 100644 index 906424ca7c..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/main.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - name = "${var.name}-${random_id.resource_name_suffix.hex}" -} - -resource "google_compute_resource_policy" "policy" { - name = local.name - region = var.region - project = var.project_id - provider = google-beta - - dynamic "workload_policy" { - for_each = var.workload_policy.type != null ? [1] : [] - - content { - type = var.workload_policy.type - max_topology_distance = var.workload_policy.max_topology_distance - accelerator_topology = var.workload_policy.accelerator_topology - } - } - - dynamic "group_placement_policy" { - for_each = var.group_placement_max_distance > 0 ? [1] : [] - - content { - collocation = "COLLOCATED" - max_distance = var.group_placement_max_distance - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/outputs.tf b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/outputs.tf deleted file mode 100644 index c1dc65bcbb..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "placement_policy" { - description = <<-EOT - Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. - It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. - EOT - - value = { - type = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? "COMPACT" : null - name = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? local.name : null - tpu_topology = (var.workload_policy.type != null) ? var.workload_policy.accelerator_topology : null - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/variables.tf b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/variables.tf deleted file mode 100644 index 92434326ca..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/variables.tf +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "The project ID for the resource policy." - type = string -} - -variable "region" { - description = "The region for the the resource policy." - type = string -} - -variable "name" { - description = "The resource policy's name." - type = string - - validation { - # Check if the variable matches the GCP resource naming regex. - condition = can(regex("^[a-z]([-a-z0-9]{0,52}[a-z0-9])?$", var.name)) - error_message = <<-EOD - The resource policy name must be between 1 and 54 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. - Underscores are not allowed. A shorter length is enforced to accommodate a random suffix. - EOD - } -} - -variable "group_placement_max_distance" { - description = <<-EOT - The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - EOT - - type = number - default = 0 -} - -variable "workload_policy" { - description = "Describes the workload policy" - type = object({ - type = optional(string, null) - max_topology_distance = optional(string, null) - accelerator_topology = optional(string, null) - }) - default = { - type = null - max_topology_distance = null - accelerator_topology = null - } - nullable = false -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/versions.tf b/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/versions.tf deleted file mode 100644 index f235fbade3..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/resource-policy/versions.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.29.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:resource-policy/v1.37.2" - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/README.md b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/README.md deleted file mode 100644 index 0c4737e0d9..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/README.md +++ /dev/null @@ -1,257 +0,0 @@ -## Description - -This module creates one or more -[compute VM instances](https://cloud.google.com/compute/docs/instances). - -### Example - -```yaml -- id: compute - source: modules/compute/vm-instance - use: [network1] - settings: - instance_count: 8 - name_prefix: compute - machine_type: c2-standard-60 -``` - -This creates a cluster of 8 compute VMs that are: - -* named `compute-[0-7]` -* on the network defined by the `network1` module -* of type c2-standard-60 - -> **_NOTE:_** Simultaneous Multithreading (SMT) is deactivated by default -> (threads_per_core=1), which means only the physical cores are visible on the -> VM. With SMT disabled, a machine of type c2-standard-60 will only have the 30 -> physical cores visible. To change this, set `threads_per_core=2` under -> settings. - -### VPC Networks - -There are two methods for adding network connectivity to the `vm-instance` -module. The first is shown in the example above, where a `vpc` module or -`pre-existing-vpc` module is used by the `vm-instance` module. When this -happens, the `network_self_link` and `subnetwork_self_link` outputs from the -network are provided as input to the `vm-instance` and a network interface is -defined based on that. This can also be done updating the `network_self_link` and -`subnetwork_self_link` settings directly. - -The alternative option can be used when more than one network needs to be added -to the `vm-instance` or further customization is needed beyond what is provided -via other variables. For this option, the `network_interfaces` variable can be -used to set up one or more network interfaces on the VM instance. The format is -consistent with the terraform `google_compute_instance` `network_interface` -block, and more information can be found in the -[terraform docs][network-interface-tf]. - -> **_NOTE:_** When supplying the `network_interfaces` variable, networks -> associated with the `vm-instance` via use will be ignored in favor of the -> networks added in `network_interfaces`. In addition, `bandwidth_tier` and -> `disable_public_ips` will not apply to networks defined in -> `network_interfaces`. - -[network-interface-tf]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - -### SSH key metadata - -This module will ignore all changes to the `ssh-keys` metadata field that are -typically set by [external Google Cloud tools that automate SSH access][gcpssh] -when not using OS Login. For example, clicking on the Google Cloud Console SSH -button next to VMs in the VM Instances list will temporarily modify VM metadata -to include a dynamically-generated SSH public key. - -[gcpssh]: https://cloud.google.com/compute/docs/connect/add-ssh-keys#metadata - -### Placement - -The `placement_policy` variable can be used to control where your VM instances -are physically located relative to each other within a zone. See the official -placement [guide][guide-link] and [api][api-link] documentation. - -[guide-link]: https://cloud.google.com/compute/docs/instances/define-instance-placement -[api-link]: https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement - -Use the following settings for compact placement: - -```yaml - ... - settings: - instance_count: 4 - machine_type: c2-standard-60 - placement_policy: - collocation: "COLLOCATED" -``` - -By default the above placement policy will always result in the most compact set -of VMs available. If you would like that provisioning failed if some level of -compactness is not obtainable, you can enforce this with the [`max_distance` -setting](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies): - -```yaml - ... - settings: - instance_count: 4 - machine_type: c2-standard-60 - placement_policy: - collocation: "COLLOCATED" - max_distance: 1 -``` - -Use the following settings for spread placement: - -```yaml - ... - settings: - instance_count: 4 - machine_type: n2-standard-4 - placement_policy: - availability_domain_count: 2 -``` - -When `vm_count` is not set, as shown in the examples above, then the VMs will be -added to the placement policy incrementally. This is the **recommended way** to -use placement policies. - -If `vm_count` is specified then VMs will stay in pending state until the -specified number of VMs are created. See the warning below if using this field. - -> [!WARNING] -> When creating a compact placement using `vm_count` with more than 10 VMs, you -> must add `-parallelism=` argument on apply. For example if you have 15 VMs -> in a placement group: `terraform apply -parallelism=15`. This is because -> terraform self limits to 10 parallel requests by default but the create -> instance requests will not succeed until all VMs in the placement group have -> been requested, forming a deadlock. - -### GPU Support - -More information on GPU support in `vm-instance` and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../docs/gpu-support.md) - -## Lifecycle - -The `vm-instance` module will be replaced when the `instance_image` variable is -changed and `terraform apply` is run on the deployment group folder or -`gcluster deploy` is run. However, it will not be automatically replaced if a new -image is created in a family. - -To selectively replace the vm-instance(s), consider running terraform -`apply -replace` such as: - -> See https://developer.hashicorp.com/terraform/cli/commands/plan#replace-address for precise syntax terraform apply -replace=ADDRESS - -```shell -terraform state list -# search for the module ID and resource -terraform apply -replace="address" -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 4.73.0 | -| [google-beta](#requirement\_google-beta) | >= 6.13.0 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.73.0 | -| [google-beta](#provider\_google-beta) | >= 6.13.0 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | -| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance) | resource | -| [google-beta_google_compute_resource_policy.placement_policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | -| [google_compute_address.compute_ip](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_disk.additional_disks](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.replace_vm_trigger_from_placement](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | -| [additional\_persistent\_disks](#input\_additional\_persistent\_disks) | Configurations of additional disks to be included on the partition nodes. | 
object({
    count = optional(number, 0)
    type  = optional(string, "pd-balanced")
    size  = optional(number, 200)
  })
| `{}` | no | -| [allocate\_ip](#input\_allocate\_ip) | If not null, allocate IPs with the given configuration. See details at
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address | 
object({
    address_type = optional(string, "INTERNAL")
    purpose      = optional(string),
    network_tier = optional(string),
    ip_version   = optional(string, "IPV4"),
  })
| `null` | no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | -| [automatic\_restart](#input\_automatic\_restart) | Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user). | `bool` | `null` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will optionally be used name resources according to `name_prefix` | `string` | n/a | yes | -| [disable\_public\_ips](#input\_disable\_public\_ips) | If set to true, instances will not have public IPs | `bool` | `false` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | -| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-standard"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | -| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_count](#input\_local\_ssd\_count) | The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd. | `number` | `0` | no | -| [local\_ssd\_interface](#input\_local\_ssd\_interface) | Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set. | `string` | `"NVME"` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface

**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply
to network interfaces defined in this variable.

Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "MRDMA", "IRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. Can use "default" for the default network. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `null` | no | -| [placement\_policy](#input\_placement\_policy) | Control where your VM instances are physically located relative to each other within a zone.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy#nested_group_placement_policy | `any` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [provisioning\_model](#input\_provisioning\_model) | Provisioning model for cloud instance. | `string` | `null` | no | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME
- RESERVATION\_NAME

Must be a "SPECIFIC\_RESERVATION"
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | -| [service\_account](#input\_service\_account) | DEPRECATED - Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [spot](#input\_spot) | DEPRECATED - Use `provisioning_model` instead. | `bool` | `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | -| [zone](#input\_zone) | Compute Platform zone | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [external\_ip](#output\_external\_ip) | External IP of the instances (if enabled) | -| [instructions](#output\_instructions) | Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions. | -| [internal\_ip](#output\_internal\_ip) | Internal IP of the instances | -| [name](#output\_name) | Names of instances created | -| [self\_link](#output\_self\_link) | The tuple URIs of the created instances | - diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/compute_image.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/main.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/main.tf deleted file mode 100644 index 0a8c7d354e..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/main.tf +++ /dev/null @@ -1,334 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "vm-instance", ghpc_role = "compute" }) -} - -module "gpu" { - source = "../../internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - native_fstype = [] - startup_script = local.startup_from_network_storage != null ? ( - { startup-script = local.startup_from_network_storage }) : {} - network_storage = var.network_storage != null ? ( - { network_storage = jsonencode(var.network_storage) }) : {} - - prefix_optional_deployment_name = var.name_prefix != null ? var.name_prefix : var.deployment_name - prefix_always_deployment_name = var.name_prefix != null ? "${var.deployment_name}-${var.name_prefix}" : var.deployment_name - resource_prefix = var.add_deployment_name_before_prefix ? local.prefix_always_deployment_name : local.prefix_optional_deployment_name - - enable_gvnic = var.bandwidth_tier != "not_enabled" - enable_tier_1 = var.bandwidth_tier == "tier_1_enabled" - - provisioning_model = var.provisioning_model - - spot = var.provisioning_model == "SPOT" - - # compact_placement : true when placement policy is provided and collocation set; false if unset - compact_placement = try(var.placement_policy.collocation, null) != null - - gpu_attached = contains(["a2", "g2"], local.machine_family) || length(local.guest_accelerator) > 0 - - # both of these must be false if either compact placement or preemptible/spot instances are used - # automatic restart is tolerant of GPUs while on host maintenance is not - automatic_restart_default = local.compact_placement || local.spot ? false : null - on_host_maintenance_default = local.compact_placement || local.spot || local.gpu_attached ? "TERMINATE" : "MIGRATE" - - automatic_restart = ( - var.automatic_restart != null - ? var.automatic_restart - : local.automatic_restart_default - ) - - on_host_maintenance = ( - var.on_host_maintenance != null - ? var.on_host_maintenance - : local.on_host_maintenance_default - ) - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - # Network Interfaces - # Support for `use` input and base network parameters like `network_self_link` and `subnetwork_self_link` - empty_access_config = { - nat_ip = null, - public_ptr_domain_name = null, - network_tier = null - } - default_network_interface = { - network = var.network_self_link - subnetwork = var.subnetwork_self_link - subnetwork_project = null # will populate from subnetwork_self_link - network_ip = null - nic_type = local.enable_gvnic ? "GVNIC" : null - stack_type = null - queue_count = null - access_config = var.disable_public_ips ? [] : [local.empty_access_config] - ipv6_access_config = [] - alias_ip_range = [] - } - network_interfaces = coalescelist(var.network_interfaces, [local.default_network_interface]) - network_interfaces_with_ips = var.allocate_ip == null ? local.network_interfaces : [ - for i, interface in local.network_interfaces : - merge(interface, { - network_ip = google_compute_address.compute_ip[i].address - }) - ] -} - -resource "null_resource" "image" { - triggers = { - name = try(var.instance_image.name, null), - family = try(var.instance_image.family, null), - project = try(var.instance_image.project, null) - } -} - -resource "google_compute_disk" "additional_disks" { - project = var.project_id - - count = var.instance_count * var.additional_persistent_disks.count - - # NB: this resource array must be sliced accounting for var.instance_count - name = "${local.resource_prefix}-disk-${count.index}" - type = var.additional_persistent_disks.type - size = var.additional_persistent_disks.size - labels = local.labels - zone = var.zone -} - -resource "google_compute_resource_policy" "placement_policy" { - project = var.project_id - provider = google-beta - - count = var.placement_policy != null ? 1 : 0 - name = "${local.resource_prefix}-vm-instance-placement" - group_placement_policy { - vm_count = try(var.placement_policy.vm_count, null) - availability_domain_count = try(var.placement_policy.availability_domain_count, null) - collocation = try(var.placement_policy.collocation, null) - max_distance = try(var.placement_policy.max_distance, null) - } -} - -resource "null_resource" "replace_vm_trigger_from_placement" { - triggers = { - vm_count = try(tostring(var.placement_policy.vm_count), "") - availability_domain_count = try(tostring(var.placement_policy.availability_domain_count), "") - max_distance = try(tostring(var.placement_policy.max_distance), "") - collocation = try(var.placement_policy.collocation, "") - } -} - -resource "google_compute_address" "compute_ip" { - project = var.project_id - - count = var.allocate_ip != null ? length(local.network_interfaces) : 0 - - name = "${local.resource_prefix}-${count.index}" - - address = local.network_interfaces[count.index].network_ip - region = var.region - network = can(coalesce(local.network_interfaces[count.index].subnetwork)) ? null : local.network_interfaces[count.index].network - subnetwork = local.network_interfaces[count.index].subnetwork - address_type = var.allocate_ip.address_type - purpose = var.allocate_ip.purpose - network_tier = var.allocate_ip.network_tier - ip_version = var.allocate_ip.ip_version -} - -resource "google_compute_instance" "compute_vm" { - project = var.project_id - provider = google-beta - - count = var.instance_count - - depends_on = [var.network_self_link, var.network_storage] - - name = "${local.resource_prefix}-${count.index}" - min_cpu_platform = var.min_cpu_platform - machine_type = var.machine_type - zone = var.zone - - resource_policies = google_compute_resource_policy.placement_policy[*].self_link - - tags = var.tags - labels = local.labels - - boot_disk { - initialize_params { - image = data.google_compute_image.compute_image.self_link - size = var.disk_size_gb - type = var.disk_type - labels = local.labels - } - - device_name = "${local.resource_prefix}-boot-disk-${count.index}" - auto_delete = var.auto_delete_boot_disk - } - - dynamic "attached_disk" { - for_each = slice( - google_compute_disk.additional_disks, - var.additional_persistent_disks.count * count.index, - var.additional_persistent_disks.count * count.index + var.additional_persistent_disks.count, - ) - - content { - source = attached_disk.value.self_link - device_name = "additional-disk-${attached_disk.key}" - mode = "READ_WRITE" - } - } - - dynamic "scratch_disk" { - for_each = range(var.local_ssd_count) - content { - interface = var.local_ssd_interface - } - } - - dynamic "network_interface" { - for_each = local.network_interfaces_with_ips - - content { - network = network_interface.value.network - subnetwork = network_interface.value.subnetwork - subnetwork_project = network_interface.value.subnetwork_project - network_ip = network_interface.value.network_ip - nic_type = network_interface.value.nic_type - stack_type = network_interface.value.stack_type - queue_count = network_interface.value.queue_count - dynamic "access_config" { - for_each = network_interface.value.access_config - content { - nat_ip = access_config.value.nat_ip - public_ptr_domain_name = access_config.value.public_ptr_domain_name - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = network_interface.value.ipv6_access_config - content { - public_ptr_domain_name = ipv6_access_config.value.public_ptr_domain_name - network_tier = ipv6_access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = network_interface.value.alias_ip_range - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - } - } - - network_performance_config { - total_egress_bandwidth_tier = local.enable_tier_1 ? "TIER_1" : "DEFAULT" - } - - service_account { - email = var.service_account_email - scopes = var.service_account_scopes - } - - dynamic "guest_accelerator" { - for_each = local.guest_accelerator - content { - count = guest_accelerator.value.count - type = guest_accelerator.value.type - } - } - - scheduling { - on_host_maintenance = local.on_host_maintenance - automatic_restart = local.automatic_restart - preemptible = local.spot - provisioning_model = local.provisioning_model - } - - dynamic "advanced_machine_features" { - for_each = local.set_threads_per_core ? [1] : [] - content { - threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf - } - } - - dynamic "reservation_affinity" { - for_each = var.reservation_name == "" ? [] : [1] - content { - type = "SPECIFIC_RESERVATION" - specific_reservation { - key = "compute.googleapis.com/reservation-name" - values = [var.reservation_name] - } - } - } - - metadata = merge( - local.network_storage, - local.startup_script, - local.enable_oslogin, - local.disable_automatic_updates_metadata, - var.metadata - ) - - lifecycle { - ignore_changes = [ - metadata["ssh-keys"], - ] - - replace_triggered_by = [ - null_resource.replace_vm_trigger_from_placement - ] - - precondition { - condition = (length(var.network_interfaces) == 0) != (var.network_self_link == null && var.subnetwork_self_link == null) - error_message = "Exactly one of network_interfaces or network_self_link/subnetwork_self_link must be specified." - } - precondition { - condition = alltrue([for interface in var.network_interfaces : interface.network_ip == null]) || var.instance_count == 1 - error_message = <<-EOT - The network_ip cannot be statically set on vm-instance when the VM instance_count is greater than 1. - Either set the network_ip to null to allow it to be set dynamically for all instances, or create modules for each VM instance with its own network interface. - EOT - } - precondition { - condition = !contains([ - "c3-:pd-standard", - "h3-:pd-standard", - "h3-:pd-ssd", - ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") - error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/outputs.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/outputs.tf deleted file mode 100644 index eab8cb56bd..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "name" { - description = "Names of instances created" - value = google_compute_instance.compute_vm[*].name -} - -output "self_link" { - description = "The tuple URIs of the created instances" - value = google_compute_instance.compute_vm[*].self_link -} - -output "external_ip" { - description = "External IP of the instances (if enabled)" - value = try(google_compute_instance.compute_vm[*].network_interface[0].access_config[0].nat_ip, []) -} - -output "internal_ip" { - description = "Internal IP of the instances" - value = google_compute_instance.compute_vm[*].network_interface[0].network_ip -} - -locals { - first_instance_link = try(google_compute_instance.compute_vm[0].self_link, "no-instance") - ssh_instructions = <<-EOT - Use the following commands to SSH into the first VM created: - gcloud compute ssh ${local.first_instance_link} --project ${var.project_id} - If not accessible from the public internet, use an SSH tunnel through IAP: - gcloud compute ssh ${local.first_instance_link} --tunnel-through-iap --project ${var.project_id} - EOT -} - -output "instructions" { - description = "Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions." - value = var.instance_count > 0 ? local.ssh_instructions : "No instances were created." -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf deleted file mode 100644 index 02bc58e4f7..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "inputs": -# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. -# var.network_storage : to be passed into metadata somewhere else (not here) -# var.startup_script : to be changed into a more complete file system with all the fs runners - -# "outputs": -# local.startup_from_network_storage : A full startup script with all the runners that are not supported -# natively and were included in the network_storage structure - -locals { - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - # Pull out runners to include in startup script - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - startup_script_runner = [{ - content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" - destination = "passed_startup_script.sh" - type = "shell" - }] - - full_runner_list = concat( - local.storage_client_install_runners, - local.mount_runners, - local.startup_script_runner - ) - - startup_from_network_storage = module.netstorage_startup_script.startup_script -} - -module "netstorage_startup_script" { - source = "../../scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.full_runner_list -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf deleted file mode 100644 index e582db33da..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' - -# "inputs": -# var.machine_type: Machine type for the instance being evaluated. -# var.threads_per_core : Sets the number of threads per physical core, where 0 -# has behavior described in description. - -# "outputs": -# local.set_threads_per_core: bool that tells if threads per core should be set, -# to be used with a dynamic block. -# local.threads_per_core: actual threads_per_core to be used. - -locals { - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - machine_shared_core = length(local.machine_vals) <= 2 - machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) - - smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) - smt_capable_vcpu = local.machine_vcpus >= 2 - - smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core - set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) - threads_per_core = var.threads_per_core == 2 ? 2 : 1 -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/variables.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/variables.tf deleted file mode 100644 index 5519b8cd40..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/variables.tf +++ /dev/null @@ -1,452 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "instance_count" { - description = "Number of instances" - type = number - default = 1 -} - -variable "instance_image" { - description = "Instance Image" - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "disk_size_gb" { - description = "Size of disk for instances." - type = number - default = 200 -} - -variable "disk_type" { - description = "Disk type for instances." - type = string - default = "pd-standard" -} - -variable "auto_delete_boot_disk" { - description = "Controls if boot disk should be auto-deleted when instance is deleted." - type = bool - default = true -} - -variable "local_ssd_count" { - description = "The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd." - type = number - default = 0 -} - -variable "local_ssd_interface" { - description = "Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set." - type = string - default = "NVME" -} - -variable "additional_persistent_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = object({ - count = optional(number, 0) - type = optional(string, "pd-balanced") - size = optional(number, 200) - }) - default = {} -} - -variable "name_prefix" { - description = <<-EOT - An optional name for all VM and disk resources. - If not supplied, `deployment_name` will be used. - When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, - then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". - EOT - type = string - default = null -} - -variable "add_deployment_name_before_prefix" { - description = <<-EOT - If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. - See `name_prefix` for further details on resource naming behavior. - EOT - type = bool - default = false -} - -variable "disable_public_ips" { - description = "If set to true, instances will not have public IPs" - type = bool - default = false -} - -variable "machine_type" { - description = "Machine type to use for the instance creation" - type = string - default = "c2-standard-60" -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "deployment_name" { - description = "Name of the deployment, will optionally be used name resources according to `name_prefix`" - type = string -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} - -variable "service_account_email" { - description = "Service account e-mail address to use with the node pool" - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to to use with the node pool." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -# tflint-ignore: terraform_unused_declarations -variable "service_account" { - description = "DEPRECATED - Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string, - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "The 'service_account' setting is deprecated, please use 'var.service_account_email' and 'var.service_account_scopes' instead." - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the VM. Can use \"default\" for the default network." - type = string - default = null -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the VM." - type = string - default = null -} - -variable "network_interfaces" { - description = <<-EOT - A list of network interfaces. The options match that of the terraform - network_interface block of google_compute_instance. For descriptions of the - subfields or more information see the documentation: - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - - **_NOTE:_** If `network_interfaces` are set, `network_self_link` and - `subnetwork_self_link` will be ignored, even if they are provided through - the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply - to network interfaces defined in this variable. - - Subfields: - network (string, required if subnetwork is not supplied) - subnetwork (string, required if network is not supplied) - subnetwork_project (string, optional) - network_ip (string, optional) - nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "MRDMA", "IRDMA"]) - stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) - queue_count (number, optional) - access_config (object, optional) - ipv6_access_config (object, optional) - alias_ip_range (list(object), optional) - EOT - type = list(object({ - network = string, - subnetwork = string, - subnetwork_project = string, - network_ip = string, - nic_type = string, - stack_type = string, - queue_count = number, - access_config = list(object({ - nat_ip = string, - public_ptr_domain_name = string, - network_tier = string - })), - ipv6_access_config = list(object({ - public_ptr_domain_name = string, - network_tier = string - })), - alias_ip_range = list(object({ - ip_cidr_range = string, - subnetwork_range_name = string - })) - })) - default = [] - validation { - condition = alltrue([ - for ni in var.network_interfaces : (ni.network == null) != (ni.subnetwork == null) - ]) - error_message = "All additional network interfaces must define exactly one of \"network\" or \"subnetwork\"." - } - validation { - condition = alltrue([ - for ni in var.network_interfaces : ni.nic_type == "GVNIC" || ni.nic_type == "VIRTIO_NET" || ni.nic_type == "MRDMA" || ni.nic_type == "IRDMA" || ni.nic_type == null - ]) - error_message = "In the variable network_interfaces, field \"nic_type\" must be \"GVNIC\", \"VIRTIO_NET\", \"MRDMA\", \"IRDMA\", or null." - } - validation { - condition = alltrue([ - for ni in var.network_interfaces : ni.stack_type == "IPV4_ONLY" || ni.stack_type == "IPV4_IPV6" || ni.stack_type == null - ]) - error_message = "In the variable network_interfaces, field \"stack_type\" must be either \"IPV4_ONLY\", \"IPV4_IPV6\" or null." - } -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "zone" { - description = "Compute Platform zone" - type = string -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "startup_script" { - description = "Startup script used on the instance" - type = string - default = null -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false -} - -variable "automatic_restart" { - description = "Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user)." - type = bool - default = null -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" - type = string - default = null - validation { - condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) - error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." - } -} - -variable "bandwidth_tier" { - description = <= 0, false) && try(var.threads_per_core <= 2, false) - error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." - } - -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "allocate_ip" { - description = <<-EOT - If not null, allocate IPs with the given configuration. See details at - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address - EOT - type = object({ - address_type = optional(string, "INTERNAL") - purpose = optional(string), - network_tier = optional(string), - ip_version = optional(string, "IPV4"), - }) - default = null -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "reservation_name" { - description = <<-EOD - Name of the reservation to use for VM resources, should be in one of the following formats: - - projects/PROJECT_ID/reservations/RESERVATION_NAME - - RESERVATION_NAME - - Must be a "SPECIFIC_RESERVATION" - Set to empty string if using no reservation or automatically-consumed reservations - EOD - type = string - default = "" - nullable = false - - validation { - condition = length(regexall("^((projects/([a-z0-9-]+)/reservations/)?([a-z0-9-]+))?$", var.reservation_name)) > 0 - error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME', [...] is an optional part." - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/versions.tf b/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/versions.tf deleted file mode 100644 index 0429782c6d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/compute/vm-instance/versions.tf +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.73.0" - } - - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.13.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/README.md deleted file mode 100644 index 285a20bde2..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/README.md +++ /dev/null @@ -1,170 +0,0 @@ -## Description - -This module creates a [Google Cloud Storage (GCS) bucket](https://cloud.google.com/storage). - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -### Example - -The following example will create a bucket named `simulation-results-xxxxxxxx`, -where `xxxxxxxx` is a randomly generated id. - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: - name_prefix: simulation-results - random_suffix: true -``` - -> **_NOTE:_** Use of `random_suffix` may cause the following error when used -> with other modules: -> `value depends on resource attributes that cannot be determined until apply`. -> To resolve this set `random_suffix` to `false` (default). - - - -> **_NOTE:_** Bucket namespace is shared by all users of Google Cloud so it is -> possible to have a bucket name clash with an existing bucket that is not in -> your project. To resolve this try to use a more unique name, or set the -> `random_suffix` variable to `true`. - -## Naming of Bucket - -There are potentially three parts to the bucket name. Each of these parts are -configurable in the blueprint. - -1. A **custom prefix**, provided by the user in the blueprint \ -Provide the custom prefix using the `name_prefix` setting. - -1. The **deployment name**, included by default \ -The deployment name can be excluded by setting `use_deployment_name_in_bucket_name: false`. - -1. A **random id** suffix, excluded by default \ -The random id can be included by setting `random_suffix: true`. - -If none of these are provided (no `name_prefix`, -`use_deployment_name_in_bucket_name: false`, & `random_suffix: false`), then the -bucket name will default to `no-bucket-name-provided`. - -Since bucket namespace is shared by all users of Google Cloud, it is more likely -to experience naming clashes than with other resources. In many cases, adding -the `random_suffix` will resolve the naming clash issue. - -> **Warning**: If a bucket is created with a `random_suffix` and then used as -> the bucket for a startup script in the same deployment group this will cause a -> `not known at apply time` error in terraform. The solution is to either create -> the bucket in a separate deployment group or to remove the random suffix. - -## Mounting - -To mount the Cloud Storage bucket you must first ensure that the GCS Fuse client -has been installed and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. - -If mounting is not automatically handled as described above, the -`cloud-storage-bucket` module outputs runners that can be used with the -`startup-script` module to install the client and mount the file system. See the -following example: - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: {local_mount: /data} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(bucket.client_install_runner) - - $(bucket.mount_runner) -``` - -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | -| [google-beta](#requirement\_google-beta) | >= 6.9.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [google-beta](#provider\_google-beta) | >= 6.9.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource | -| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [autoclass](#input\_autoclass) | Configure bucket autoclass setup

The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern.

The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time.
Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE')

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/autoclass | 
object({
    enabled                = optional(bool, false)
    terminal_storage_class = optional(string, null)
  })
| 
{
  "enabled": false
}
| no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the GCS bucket. | `string` | n/a | yes | -| [enable\_hierarchical\_namespace](#input\_enable\_hierarchical\_namespace) | If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket. | `bool` | `false` | no | -| [enable\_object\_retention](#input\_enable\_object\_retention) | If true, enables retention policy at per object level for the bucket.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/object-lock | `bool` | `false` | no | -| [enable\_versioning](#input\_enable\_versioning) | If true, enables versioning for the bucket. | `bool` | `false` | no | -| [force\_destroy](#input\_force\_destroy) | If true will destroy bucket with all objects stored within. | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to the GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | -| [lifecycle\_rules](#input\_lifecycle\_rules) | List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule | 
list(object({
    # Object with keys:
    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.
    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.
    action = object({
      type          = string
      storage_class = optional(string)
    })

    # Object with keys:
    # - age - (Optional) Minimum age of an object in days to satisfy this condition.
    # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field.
    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.
    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".
    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY.
    # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition.
    # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition.
    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.
    # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition.
    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.
    # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object.
    # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent.
    condition = object({
      age                        = optional(number)
      send_age_if_zero           = optional(bool)
      created_before             = optional(string)
      with_state                 = optional(string)
      matches_storage_class      = optional(string)
      matches_prefix             = optional(string)
      matches_suffix             = optional(string)
      num_newer_versions         = optional(number)
      custom_time_before         = optional(string)
      days_since_custom_time     = optional(number)
      days_since_noncurrent_time = optional(number)
      noncurrent_time_before     = optional(string)
    })
  }))
| `[]` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | -| [mount\_options](#input\_mount\_options) | Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories) | `string` | `"defaults,_netdev,implicit_dirs"` | no | -| [name\_prefix](#input\_name\_prefix) | Name Prefix. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | -| [public\_access\_prevention](#input\_public\_access\_prevention) | Bucket public access can be controlled by setting a value of either `inherited` or `enforced`.
When set to `enforced`, public access to the bucket is blocked.
If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/public-access-prevention | `string` | `null` | no | -| [random\_suffix](#input\_random\_suffix) | If true, a random id will be appended to the suffix of the bucket name. | `bool` | `false` | no | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [retention\_policy\_period](#input\_retention\_policy\_period) | If defined, this will configure retention\_policy with retention\_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/bucket-lock | `number` | `null` | no | -| [soft\_delete\_retention\_duration](#input\_soft\_delete\_retention\_duration) | If defined, this will configure soft\_delete\_policy with retention\_duration\_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days).
Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/soft-delete | `number` | `null` | no | -| [storage\_class](#input\_storage\_class) | The storage class of the GCS bucket. | `string` | `"REGIONAL"` | no | -| [uniform\_bucket\_level\_access](#input\_uniform\_bucket\_level\_access) | Allow uniform control access to the bucket. | `bool` | `true` | no | -| [use\_deployment\_name\_in\_bucket\_name](#input\_use\_deployment\_name\_in\_bucket\_name) | If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments. | `bool` | `true` | no | -| [viewers](#input\_viewers) | A list of additional accounts that can read packages from this bucket | `set(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use gcs fuse. | -| [gcs\_bucket\_name](#output\_gcs\_bucket\_name) | Bucket name. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | The gsutil bucket path with format of `gs://`. | -| [mount\_runner](#output\_mount\_runner) | Runner that mounts the cloud storage bucket with gcs fuse. | -| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | - diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf deleted file mode 100644 index 81ba0ca6a9..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "cloud-storage-bucket", ghpc_role = "file-system" }) -} - -locals { - prefix = var.name_prefix != null ? var.name_prefix : "" - deployment = var.use_deployment_name_in_bucket_name ? var.deployment_name : "" - suffix = var.random_suffix ? random_id.resource_name_suffix.hex : "" - first_dash = (local.prefix != "" && (local.deployment != "" || local.suffix != "")) ? "-" : "" - second_dash = local.deployment != "" && local.suffix != "" ? "-" : "" - composite_name = "${local.prefix}${local.first_dash}${local.deployment}${local.second_dash}${local.suffix}" - name = local.composite_name == "" ? "no-bucket-name-provided" : local.composite_name -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket" "bucket" { - provider = google-beta - project = var.project_id - name = local.name - uniform_bucket_level_access = var.uniform_bucket_level_access - location = var.region - storage_class = var.storage_class - labels = local.labels - force_destroy = var.force_destroy - public_access_prevention = var.public_access_prevention - enable_object_retention = var.enable_object_retention - hierarchical_namespace { - enabled = var.enable_hierarchical_namespace - } - - dynamic "autoclass" { - for_each = var.autoclass.enabled ? [1] : [] - content { - enabled = var.autoclass.enabled - terminal_storage_class = var.autoclass.terminal_storage_class - } - } - - dynamic "soft_delete_policy" { - for_each = var.soft_delete_retention_duration == null ? [] : [1] - content { - retention_duration_seconds = var.soft_delete_retention_duration - } - } - - dynamic "retention_policy" { - for_each = var.retention_policy_period == null ? [] : [1] - content { - retention_period = var.retention_policy_period - } - } - - dynamic "versioning" { - for_each = var.enable_versioning ? [1] : [] - content { - enabled = var.enable_versioning - } - } - - dynamic "lifecycle_rule" { - for_each = var.lifecycle_rules - content { - action { - type = lifecycle_rule.value.action.type - storage_class = lookup(lifecycle_rule.value.action, "storage_class", null) - } - condition { - age = lookup(lifecycle_rule.value.condition, "age", null) - send_age_if_zero = lookup(lifecycle_rule.value.condition, "send_age_if_zero", null) - created_before = lookup(lifecycle_rule.value.condition, "created_before", null) - with_state = lookup(lifecycle_rule.value.condition, "with_state", contains(keys(lifecycle_rule.value.condition), "is_live") ? (lifecycle_rule.value.condition["is_live"] ? "LIVE" : null) : null) - matches_storage_class = lifecycle_rule.value.condition["matches_storage_class"] != null ? split(",", lifecycle_rule.value.condition["matches_storage_class"]) : null - matches_prefix = lifecycle_rule.value.condition["matches_prefix"] != null ? split(",", lifecycle_rule.value.condition["matches_prefix"]) : null - matches_suffix = lifecycle_rule.value.condition["matches_suffix"] != null ? split(",", lifecycle_rule.value.condition["matches_suffix"]) : null - num_newer_versions = lookup(lifecycle_rule.value.condition, "num_newer_versions", null) - custom_time_before = lookup(lifecycle_rule.value.condition, "custom_time_before", null) - days_since_custom_time = lookup(lifecycle_rule.value.condition, "days_since_custom_time", null) - days_since_noncurrent_time = lookup(lifecycle_rule.value.condition, "days_since_noncurrent_time", null) - noncurrent_time_before = lookup(lifecycle_rule.value.condition, "noncurrent_time_before", null) - } - } - } - - lifecycle { - precondition { - condition = !var.autoclass.enabled || !var.enable_hierarchical_namespace - error_message = "Hierarchical namespace is not compatible with Autoclass enabled." - } - - precondition { - condition = !var.enable_hierarchical_namespace || var.uniform_bucket_level_access - error_message = "Hierarchical namespace is not compatible with Uniform bucket level access disabled." - } - - precondition { - condition = !var.enable_versioning || !var.enable_hierarchical_namespace - error_message = "Hierarchical namespace is not compatible with Object versioning enabled." - } - } -} - -resource "google_storage_bucket_iam_binding" "viewers" { - bucket = google_storage_bucket.bucket.name - role = "roles/storage.objectViewer" - members = var.viewers -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf deleted file mode 100644 index 29ddfef2d2..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a remote network storage to be mounted by fs-tab." - value = { - remote_mount = local.name - local_mount = var.local_mount - fs_type = "gcsfuse" - mount_options = var.mount_options - server_ip = "" - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } -} - -locals { - client_install_runner = { - "type" = "shell" - "content" = file("${path.module}/scripts/install-gcs-fuse.sh") - "destination" = "install-gcsfuse${replace(var.local_mount, "/", "_")}.sh" - } - - mount_runner = { - "type" = "shell" - "destination" = "mount_gcs${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"not-used\" \"${local.name}\" \"${var.local_mount}\" \"gcsfuse\" \"${var.mount_options}\"" - "content" = file("${path.module}/scripts/mount.sh") - } -} - -output "client_install_runner" { - description = "Runner that performs client installation needed to use gcs fuse." - value = local.client_install_runner -} - -output "mount_runner" { - description = "Runner that mounts the cloud storage bucket with gcs fuse." - value = local.mount_runner -} - -output "gcs_bucket_path" { - description = "The gsutil bucket path with format of `gs://`." - # cannot use resource attribute, will cause lookup failure in startup-script - value = "gs://${local.name}" - - # needed to make sure bucket contents are deleted before bucket - depends_on = [ - google_storage_bucket.bucket - ] -} - -output "gcs_bucket_name" { - description = "Bucket name." - value = google_storage_bucket.bucket.name -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh deleted file mode 100644 index f8a990260b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -if [ ! "$(which gcsfuse)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then - tee /etc/yum.repos.d/gcsfuse.repo >/dev/null </dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf deleted file mode 100644 index 9804e4b268..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf +++ /dev/null @@ -1,254 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which GCS bucket will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment; used as part of name of the GCS bucket." - type = string -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "labels" { - description = "Labels to add to the GCS bucket. Key-value pairs." - type = map(string) -} - -variable "local_mount" { - description = "The mount point where the contents of the device may be accessed after mounting." - type = string - default = "/mnt" -} - -variable "mount_options" { - description = "Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories)" - type = string - default = "defaults,_netdev,implicit_dirs" -} - -variable "name_prefix" { - description = "Name Prefix." - type = string - default = null -} - -variable "use_deployment_name_in_bucket_name" { - description = "If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments." - type = bool - default = true -} - -variable "random_suffix" { - description = "If true, a random id will be appended to the suffix of the bucket name." - type = bool - default = false -} - -variable "force_destroy" { - description = "If true will destroy bucket with all objects stored within." - type = bool - default = false -} - -variable "viewers" { - description = "A list of additional accounts that can read packages from this bucket" - type = set(string) - default = [] - - validation { - error_message = "All bucket viewers must be in IAM style: user:user@example.com, serviceAccount:sa@example.com, or group:group@example.com." - condition = alltrue([ - for viewer in var.viewers : length(regexall("^(user|serviceAccount|group):", viewer)) > 0 - ]) - } -} - -variable "enable_hierarchical_namespace" { - description = "If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket." - type = bool - default = false -} - -variable "uniform_bucket_level_access" { - description = "Allow uniform control access to the bucket." - type = bool - default = true -} - -variable "storage_class" { - description = "The storage class of the GCS bucket." - type = string - default = "REGIONAL" - validation { - condition = contains([ - "STANDARD", - "MULTI_REGIONAL", - "REGIONAL", - "NEARLINE", - "COLDLINE", - "ARCHIVE" - ], var.storage_class) - error_message = "Allowed values for GCS storage_class are 'STANDARD', 'MULTI_REGIONAL', 'REGIONAL', 'NEARLINE', 'COLDLINE', 'ARCHIVE'.\nhttps://cloud.google.com/storage/docs/storage-classes" - } -} - -variable "autoclass" { - description = <<-EOT - Configure bucket autoclass setup - - The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern. - - The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time. - Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE') - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/autoclass - EOT - type = object({ - enabled = optional(bool, false) - terminal_storage_class = optional(string, null) - }) - default = { - enabled = false - } - nullable = false - validation { - condition = !can(coalesce(var.autoclass.terminal_storage_class)) || var.autoclass.enabled - error_message = "Cannot set bucket var.autoclass.terminal_storage_class unless var.autoclass.enabled is true" - } -} - -variable "public_access_prevention" { - description = <<-EOT - Bucket public access can be controlled by setting a value of either `inherited` or `enforced`. - When set to `enforced`, public access to the bucket is blocked. - If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/public-access-prevention - EOT - type = string - default = null - validation { - condition = var.public_access_prevention == null ? true : contains([ - "inherited", - "enforced" - ], var.public_access_prevention) - error_message = "Allowed values for public_access_prevention are 'inherited', 'enforced'.\n" - } -} - -variable "soft_delete_retention_duration" { - description = <<-EOT - If defined, this will configure soft_delete_policy with retention_duration_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days). - Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/soft-delete - EOT - type = number - default = null - validation { - condition = var.soft_delete_retention_duration == null ? true : var.soft_delete_retention_duration == 0 || var.soft_delete_retention_duration >= 604800 && var.soft_delete_retention_duration <= 7776000 - error_message = "var.soft_delete_retention_duration value can be 0 or in between 604800(7 days) and 7776000(90 days)." - } -} - -variable "enable_versioning" { - description = "If true, enables versioning for the bucket." - type = bool - default = false -} - -variable "lifecycle_rules" { - description = "List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule" - type = list(object({ - # Object with keys: - # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass. - # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule. - action = object({ - type = string - storage_class = optional(string) - }) - - # Object with keys: - # - age - (Optional) Minimum age of an object in days to satisfy this condition. - # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field. - # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition. - # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY". - # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY. - # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition. - # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition. - # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition. - # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition. - # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true. - # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object. - # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent. - condition = object({ - age = optional(number) - send_age_if_zero = optional(bool) - created_before = optional(string) - with_state = optional(string) - matches_storage_class = optional(string) - matches_prefix = optional(string) - matches_suffix = optional(string) - num_newer_versions = optional(number) - custom_time_before = optional(string) - days_since_custom_time = optional(number) - days_since_noncurrent_time = optional(number) - noncurrent_time_before = optional(string) - }) - })) - default = [] -} - -variable "retention_policy_period" { - description = <<-EOT - If defined, this will configure retention_policy with retention_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/bucket-lock - EOT - type = number - default = null - validation { - condition = var.retention_policy_period == null ? true : var.retention_policy_period > 0 && var.retention_policy_period <= 3155760000 - error_message = "var.soft_delete_policy_retention_duration value must be in between 1 and 3155760000(100 years) seconds." - } -} - -variable "enable_object_retention" { - description = <<-EOT - If true, enables retention policy at per object level for the bucket. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/object-lock - EOT - type = bool - default = false -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf deleted file mode 100644 index 217ee2f3a2..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.9.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" - } - required_version = ">= 0.14.0" -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/README.md deleted file mode 100644 index 3bf251828e..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/README.md +++ /dev/null @@ -1,248 +0,0 @@ -## Description - -This module creates a [filestore](https://cloud.google.com/filestore) -instance. Filestore is a high performance network file system that can be -mounted to one or more compute VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -### Deletion protection - -We recommend considering enabling [Filestore deletion protection][fdp]. Deletion -protection will prevent unintentional deletion of an entire Filestore instance. -It does not prevent deletion of files within the Filestore instance when mounted -by a VM. It is not available on some [tiers](#filestore-tiers), including the -default BASIC\_HDD tier or BASIC\_SSD tier. Follow the documentation link for -up to date details. - -Usage can be enabled in a blueprint with, for example: - -```yaml - - id: homefs - source: modules/file-system/filestore - use: [network] - settings: - deletion_protection: - enabled: true - reason: Avoid data loss - filestore_tier: ZONAL - local_mount: /home - size_gb: 1024 -``` - -[fdp]: https://cloud.google.com/filestore/docs/deletion-protection - -### Filestore tiers - -At the time of writing, Filestore supports 5 [tiers of service][tiers] that are -specified in the Toolkit using the following names: - -- Basic HDD: "BASIC\_HDD" ([preferred][tierapi]) or "STANDARD" (deprecated) -- Basic SSD: "BASIC\_SSD" ([preferred][tierapi]) or "PREMIUM" (deprecated) -- Zonal: "ZONAL" -- Enterprise: "ENTERPRISE" -- Regional: "REGIONAL" - -[tierapi]: https://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier - -**Please review the minimum storage requirements for each tier**. The Terraform -module can only enforce the minimum value of the `size_gb` parameter for the -lowest tier of service. If you supply a value that is too low, Filestore -creation will fail when you run `terraform apply`. - -[tiers]: https://cloud.google.com/filestore/docs/service-tiers - -### Filestore protocols and mount options -After Filestore instance is created, you can mount this to the compute node -using different mount options. Toolkit uses [default mount options](https://linux.die.net/man/8/mount) -for all tier services. Filestore has recommended mount options for different -service tiers which may overall improve performance. These can be found here: -[recommended mount options.](https://cloud.google.com/filestore/docs/mounting-fileshares) -While creating filestore module, you can overwrite these mount options as -mentioned below. - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /homefs - mount_options: defaults,hard,timeo=600,retrans=3,_netdev -``` - -Filestore supports NFS protocols `NFS_V3` (default) and `NFS_V4_1`. Protocol support depends on the selected tier: -- `NFS_V3`: Supported on all tiers (`BASIC_HDD`, `BASIC_SSD`, `HIGH_SCALE_SSD`, `ZONAL`, `ENTERPRISE`). -- `NFS_V4_1`: Supported only on `HIGH_SCALE_SSD`, `ZONAL`, `REGIONAL`, and `ENTERPRISE`. -This can be specified at creation time via the `protocol` variable. By default, `NFS_V3` is used for compatibility. -See the example below and [this page](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#protocol-1) for more information. - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /homefs - protocol: NFS_V4_1 - filestore_tier: ZONAL -``` - -### Filestore quota - -Your project must have unused quota for Cloud Filestore in the region you will -provision the storage. This can be found by browsing to the [Quota tab within IAM -& Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. -Please note that there are separate quota limits for HDD and SSD storage. - -All projects begin with 0 available quota for High Scale SSD tier. To use this -tier, [make a request and wait for it to be approved][hs-ssd-quota]. - -[hs-ssd-quota]: https://cloud.google.com/filestore/docs/high-scale - -### Example - Basic HDD - -The Filestore instance defined below will have the following attributes: - -- (default) `BASIC_HDD` tier -- (default) 1TiB capacity -- `homefs` module ID -- mount point at `/home` -- connected to the network defined in the `network1` module - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /home -``` - -### Example - High Scale SSD - -The Filestore instance defined below will have the following attributes: - -- `HIGH_SCALE_SSD` tier -- 10TiB capacity -- `highscale` module ID -- mount point at `/projects` -- connected to the VPC network defined in the `network1` module - -```yaml -- id: highscale - source: modules/file-system/filestore - use: [network1] - settings: - filestore_tier: HIGH_SCALE_SSD - size_gb: 10240 - local_mount: /projects -``` - -## Mounting - -To mount the Filestore instance you must first ensure that the NFS client has -been installed and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -See the [hpc-slurm](../../../examples/hpc-slurm.yaml) for -an example of using this module with Slurm. - -If mounting is not automatically handled as described above, the `filestore` -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - - id: filestore - source: modules/file-system/filestore - use: [network1] - settings: {local_mount: /scratch} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(filestore.install_nfs_client_runner) - - $(filestore.mount_runner) - -``` - -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 6.4 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.4 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_filestore_instance.filestore_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [connect\_mode](#input\_connect\_mode) | Used to select mode - supported values DIRECT\_PEERING and PRIVATE\_SERVICE\_ACCESS. | `string` | `"DIRECT_PEERING"` | no | -| [deletion\_protection](#input\_deletion\_protection) | Configure Filestore instance deletion protection | 
object({
    enabled = optional(bool, false)
    reason  = optional(string)
  })
| 
{
  "enabled": false
}
| no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the filestore instance if no name is specified. | `string` | n/a | yes | -| [description](#input\_description) | A description of the filestore instance. | `string` | `""` | no | -| [filestore\_share\_name](#input\_filestore\_share\_name) | Name of the file system share on the instance. | `string` | `"nfsshare"` | no | -| [filestore\_tier](#input\_filestore\_tier) | The service tier of the instance. | `string` | `"BASIC_HDD"` | no | -| [labels](#input\_labels) | Labels to add to the filestore instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mount](#input\_local\_mount) | Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md)). | `string` | `"/shared"` | no | -| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"defaults,_netdev"` | no | -| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [nfs\_export\_options](#input\_nfs\_export\_options) | Define NFS export options. | 
list(object({
    access_mode = optional(string)
    ip_ranges   = optional(list(string))
    squash_mode = optional(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | ID of project in which Filestore instance will be created. | `string` | n/a | yes | -| [protocol](#input\_protocol) | NFS protocol version. Default is NFS\_V3. NFS\_V4\_1 is only supported with HIGH\_SCALE\_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers. | `string` | `"NFS_V3"` | no | -| [region](#input\_region) | Location for Filestore instances at Enterprise tier. | `string` | n/a | yes | -| [reserved\_ip\_range](#input\_reserved\_ip\_range) | Reserved IP range for Filestore instance. Users are encouraged to set to null
for automatic selection. If supplied, it must be:

CIDR format when var.connect\_mode == "DIRECT\_PEERING"
Named IP Range when var.connect\_mode == "PRIVATE\_SERVICE\_ACCESS"

See Cloud documentation for more details:

https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range | `string` | `null` | no | -| [size\_gb](#input\_size\_gb) | Storage size of the filestore instance in GB. | `number` | `1024` | no | -| [zone](#input\_zone) | Location for Filestore instances below Enterprise tier. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | -| [filestore\_id](#output\_filestore\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | -| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | -| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | -| [network\_storage](#output\_network\_storage) | Describes a filestore instance. | - diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/main.tf deleted file mode 100644 index ce035dbb2b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/main.tf +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "filestore", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - is_high_capacity_tier = contains(["HIGH_SCALE_SSD", "ZONAL", "REGIONAL"], var.filestore_tier) && var.size_gb >= 10240 && var.size_gb <= 102400 - - timeouts = local.is_high_capacity_tier ? [1] : [] - server_ip = google_filestore_instance.filestore_instance.networks[0].ip_addresses[0] - remote_mount = format("/%s", google_filestore_instance.filestore_instance.file_shares[0].name) - fs_type = "nfs" - mount_options = var.mount_options - - install_nfs_client_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-nfs-client.sh" - "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" - } - mount_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" - } - - # id format: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network#id - split_network_id = split("/", var.network_id) - network_name = local.split_network_id[4] - network_project = local.split_network_id[1] - shared_vpc = local.network_project != var.project_id -} - -resource "google_filestore_instance" "filestore_instance" { - project = var.project_id - - name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - description = var.description - location = contains(["ENTERPRISE", "REGIONAL"], var.filestore_tier) ? var.region : var.zone - tier = var.filestore_tier - protocol = var.protocol - - deletion_protection_enabled = var.deletion_protection.enabled - deletion_protection_reason = var.deletion_protection.reason - - file_shares { - capacity_gb = var.size_gb - name = var.filestore_share_name - dynamic "nfs_export_options" { - for_each = var.nfs_export_options - content { - access_mode = nfs_export_options.value.access_mode - ip_ranges = nfs_export_options.value.ip_ranges - squash_mode = nfs_export_options.value.squash_mode - } - } - } - - labels = local.labels - - networks { - network = local.shared_vpc ? var.network_id : local.network_name - connect_mode = var.connect_mode - modes = ["MODE_IPV4"] - reserved_ip_range = var.reserved_ip_range - } - - dynamic "timeouts" { - for_each = local.timeouts - content { - create = "1h" - update = "1h" - delete = "1h" - } - } - - lifecycle { - precondition { - condition = ( - var.reserved_ip_range == null || - var.connect_mode == "PRIVATE_SERVICE_ACCESS" || - var.connect_mode == "DIRECT_PEERING" && can(cidrhost(var.reserved_ip_range, 0)) && contains(["24", "29"], try(split("/", var.reserved_ip_range)[1], "")) - ) - error_message = <<-EOT - If connect_mode is set to DIRECT_PEERING and reserved_ip_range is - specified then it must be a CIDR IP range with suffix range size 29 for - BASIC_HDD or BASIC_SSD tiers. Otherwise the range size must be 24. - EOT - } - - precondition { - condition = !startswith(var.filestore_tier, "BASIC") || var.protocol != "NFS_V4_1" - error_message = "NFS_V4_1 is not supported on BASIC Filestore tiers." - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/metadata.yaml deleted file mode 100644 index 5298336f09..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - file.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/outputs.tf deleted file mode 100644 index 9bdb3bdc7b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/outputs.tf +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a filestore instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = local.mount_options - client_install_runner = local.install_nfs_client_runner - mount_runner = local.mount_runner - } -} - -output "install_nfs_client" { - description = "Script for installing NFS client" - value = file("${path.module}/scripts/install-nfs-client.sh") -} - -output "install_nfs_client_runner" { - description = "Runner to install NFS client using the startup-script module" - value = local.install_nfs_client_runner -} - -output "mount_runner" { - description = <<-EOT - Runner to mount the file-system using an ansible playbook. The startup-script - module will automatically handle installation of ansible. - - id: example-startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(your-fs-id.mount_runner) - ... - EOT - value = local.mount_runner -} - -output "filestore_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" - value = google_filestore_instance.filestore_instance.id -} - -output "capacity_gib" { - description = "File share capacity in GiB." - value = google_filestore_instance.filestore_instance.file_shares[0].capacity_gb -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/mount.sh b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/variables.tf deleted file mode 100644 index 2d7e9258c0..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/variables.tf +++ /dev/null @@ -1,189 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which Filestore instance will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the filestore instance if no name is specified." - type = string -} - -variable "zone" { - description = "Location for Filestore instances below Enterprise tier." - type = string -} - -variable "region" { - description = "Location for Filestore instances at Enterprise tier." - type = string -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is connected given in the format: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "name" { - description = "The resource name of the instance." - type = string - default = null -} - -variable "filestore_share_name" { - description = "Name of the file system share on the instance." - type = string - default = "nfsshare" -} - -variable "local_mount" { - description = "Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md))." - type = string - default = "/shared" -} - -variable "size_gb" { - description = "Storage size of the filestore instance in GB." - type = number - default = 1024 - validation { - condition = var.size_gb >= 1024 - error_message = "No Filestore tier supports less than 1024GiB.\nSee https://cloud.google.com/filestore/docs/service-tiers." - } -} - -variable "filestore_tier" { - description = "The service tier of the instance." - type = string - default = "BASIC_HDD" - validation { - condition = var.filestore_tier != "STANDARD" - error_message = "The preferred name for STANDARD tier is now BASIC_HDD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." - } - validation { - condition = var.filestore_tier != "PREMIUM" - error_message = "The preferred name for PREMIUM tier is now BASIC_SSD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." - } - validation { - condition = contains([ - "BASIC_HDD", - "BASIC_SSD", - "HIGH_SCALE_SSD", - "ZONAL", - "REGIONAL", - "ENTERPRISE" - ], var.filestore_tier) - # Avoid adding the legacy tier name in error_message, for e.g. 'HIGH_SCALE_SSD', 'ENTERPRISE'. - # As we want to steer the customer to new one's, but also support the legacy ones for older customers. - error_message = "Allowed values for filestore_tier are 'BASIC_HDD','BASIC_SSD','ZONAL','REGIONAL'.\nhttps://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#tier\nhttps://cloud.google.com/filestore/docs/reference/rest/v1/Tier." - } -} - -variable "labels" { - description = "Labels to add to the filestore instance. Key-value pairs." - type = map(string) -} - -variable "connect_mode" { - description = "Used to select mode - supported values DIRECT_PEERING and PRIVATE_SERVICE_ACCESS." - type = string - default = "DIRECT_PEERING" - nullable = false - validation { - condition = contains(["DIRECT_PEERING", "PRIVATE_SERVICE_ACCESS"], var.connect_mode) - error_message = "Allowed values for connect_mode are \"DIRECT_PEERING\" or \"PRIVATE_SERVICE_ACCESS\"." - } -} - -variable "nfs_export_options" { - description = "Define NFS export options." - type = list(object({ - access_mode = optional(string) - ip_ranges = optional(list(string)) - squash_mode = optional(string) - })) - default = [] - nullable = false -} - -variable "reserved_ip_range" { - description = <<-EOT - Reserved IP range for Filestore instance. Users are encouraged to set to null - for automatic selection. If supplied, it must be: - - CIDR format when var.connect_mode == "DIRECT_PEERING" - Named IP Range when var.connect_mode == "PRIVATE_SERVICE_ACCESS" - - See Cloud documentation for more details: - - https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range - EOT - type = string - default = null - nullable = true -} - -variable "mount_options" { - description = "NFS mount options to mount file system." - type = string - default = "defaults,_netdev" -} - -variable "deletion_protection" { - description = "Configure Filestore instance deletion protection" - type = object({ - enabled = optional(bool, false) - reason = optional(string) - }) - default = { - enabled = false - } - nullable = false - - validation { - condition = !can(coalesce(var.deletion_protection.reason)) || var.deletion_protection.enabled - error_message = "Cannot set Filestore var.deletion_protection.reason unless var.deletion_protection.enabled is true" - } -} - -variable "protocol" { - description = "NFS protocol version. Default is NFS_V3. NFS_V4_1 is only supported with HIGH_SCALE_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers." - type = string - default = "NFS_V3" - validation { - condition = contains(["NFS_V3", "NFS_V4_1"], var.protocol) - error_message = "Allowed values for protocol are 'NFS_V3' or 'NFS_V4_1'." - } -} - -variable "description" { - description = "A description of the filestore instance." - type = string - default = "" - validation { - condition = length(var.description) <= 2048 - error_message = "Filestore description must be 2048 characters or fewer" - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/filestore/versions.tf deleted file mode 100644 index 1ba0e7967e..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/filestore/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.4" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/README.md deleted file mode 100644 index 88ae4511e3..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/README.md +++ /dev/null @@ -1,200 +0,0 @@ -## Description - -This module creates Kubernetes Persistent Volumes (PV) and Persistent Volume -Claims (PVC) that can be used by a [gke-job-template]. - -`gke-persistent-volume` works with Filestore, Google Cloud Storage and Managed Lustre. Each -`gke-persistent-volume` can only be used with a single file system so if multiple -shared file systems are used then multiple `gke-persistent-volume` modules are -needed in the blueprint. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a Filestore and then uses the -`gke-persistent-volume` module to use the Filestore as shared storage in a -`gke-job-template`. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] - settings: - master_authorized_networks: - - display_name: deployment-machine - cidr_block: /32 - - - id: datafs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /data - - - id: datafs-pv - source: modules/file-system/gke-persistent-volume - use: [datafs, gke_cluster] - - - id: job-template - source: modules/compute/gke-job-template - use: [datafs-pv, compute_pool, gke_cluster] -``` - -The following example creates a GCS bucket and then uses the -`gke-persistent-volume` module to use the bucket as shared storage in a -`gke-job-template`. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] - settings: - master_authorized_networks: - - display_name: deployment-machine - cidr_block: /32 - - - id: data-bucket - source: modules/file-system/cloud-storage-bucket - settings: - local_mount: /data - - - id: datagcs-pv - source: modules/file-system/gke-persistent-volume - use: [data-bucket, gke_cluster] - - - id: job-template - source: modules/compute/gke-job-template - use: [datagcs-pv, compute_pool, gke_cluster] -``` - -The following example creates a Managed Lustre and then uses the -`gke-persistent-volume` module to use the Lustre as shared storage in a -`gke-job-template`. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] - settings: - master_authorized_networks: - - display_name: deployment-machine - cidr_block: /32 - - - id: data-managedlustre - source: modules/file-system/managed-lustre - settings: - local_mount: /data - - - id: datalustre-pv - source: modules/file-system/gke-persistent-volume - use: [data-managedlustre, gke_cluster] - - - id: job-template - source: modules/compute/gke-job-template - use: [datalustre-pv, compute_pool, gke_cluster] -``` - -See example -[storage-gke.yaml](../../../../examples/README.md#storage-gkeyaml--) blueprint -for a complete example. - -### Authorized Network - -Since the `gke-persistent-volume` module is making calls to the Kubernetes API -to create Kubernetes entities, the machine performing the deployment must be -authorized to connect to the Kubernetes API. You can add the -`master_authorized_networks` settings block, as shown in the example above, with -the IP address of the machine performing the deployment. This will ensure that -the deploying machine can connect to the cluster. - -### Connecting Via Use - -The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit -modules. For example the `gke-persistent-volume` module can `use` a -`gke-cluster` module and a `filestore` module, as shown in the example above. - -```mermaid - graph TD; - vpc--> |OneToMany| gke-cluster; - gke-cluster--> |OneToMany| gke-node-pool; - gke-node-pool--> |ManyToMany| gke-job-template; - gke-cluster--> |OneToMany| gke-persistent-volume; - gke-persistent-volume--> |ManyToMany| gke-job-template; - vpc--> |OneToMany| filestore; - vpc--> |OneToMany| gcs; - vpc--> |OneToMany| managed-lustre; - filestore--> |OneToOne| gke-persistent-volume; - gcs--> |OneToOne| gke-persistent-volume; - managed-lustre--> |OneToOne| gke-persistent-volume; - ``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [kubectl](#provider\_kubectl) | >= 1.7.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubectl_manifest.pv](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.pvc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.pvc_namespace](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [capacity\_gib](#input\_capacity\_gib) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | -| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | -| [filestore\_id](#input\_filestore\_id) | An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | -| [gcs\_bucket\_name](#input\_gcs\_bucket\_name) | The gcs bucket to be used with the persistent volume. | `string` | `null` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [lustre\_id](#input\_lustre\_id) | An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | Network attached storage mount to be configured. | 
object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  })
| n/a | yes | -| [pv\_name](#input\_pv\_name) | The name for PV. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | -| [pvc\_name](#input\_pvc\_name) | The name for PVC. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object describing the Kubernetes PersistentVolumeClaim created by this module. | -| [pvc\_name](#output\_pvc\_name) | The name of the Kubernetes PVC created by this module. | - diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/main.tf deleted file mode 100644 index 818ebaf595..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/main.tf +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-persistent-volume", ghpc_role = "file-system" }) -} - -locals { - # Flags indicating which storage type is active based on input variables. - storage_type_active = { - gcs = var.gcs_bucket_name != null - lustre = var.lustre_id != null - filestore = var.filestore_id != null - } - - # Determine the active storage type name. - active_types = [for type, is_active in local.storage_type_active : type if is_active] - - # The precondition in kubectl_manifest.pv ensures exactly one type is active. - storage_type = length(local.active_types) > 0 ? local.active_types[0] : "unknown" - - # Map containing the base name derivation logic for each storage type. - base_name_map = { - gcs = var.gcs_bucket_name - lustre = var.lustre_id != null ? split("/", var.lustre_id)[5] : null - filestore = var.filestore_id != null ? split("/", var.filestore_id)[5] : null - } - # Retrieve the base name for the active storage type. - base_name = local.base_name_map[local.storage_type] - - # PV and PVC names - pv_name = var.pv_name != null ? var.pv_name : "${local.base_name}-pv" - pvc_name = var.pvc_name != null ? var.pvc_name : "${local.base_name}-pvc" - - # Template file paths - pv_templates = { - gcs = "${path.module}/templates/gcs-pv.yaml.tftpl" - lustre = "${path.module}/templates/managed-lustre-pv.yaml.tftpl" - filestore = "${path.module}/templates/filestore-pv.yaml.tftpl" - } - pvc_templates = { - gcs = "${path.module}/templates/gcs-pvc.yaml.tftpl" - lustre = "${path.module}/templates/managed-lustre-pvc.yaml.tftpl" - filestore = "${path.module}/templates/filestore-pvc.yaml.tftpl" - } - - # Common variables for all PVC templates - common_pvc_vars = { - pv_name = local.pv_name - pvc_name = local.pvc_name - labels = local.labels - capacity = "${var.capacity_gib}Gi" - namespace = var.namespace - } - - # Common variables for all PV templates - common_pv_vars = { - pv_name = local.pv_name - capacity = "${var.capacity_gib}Gi" - labels = local.labels - } - - # Variables for PV templates, merging common vars with type-specific ones. - pv_template_vars = { - gcs = merge(local.common_pv_vars, { - mount_options = var.gcs_bucket_name != null ? split(",", var.network_storage.mount_options) : [] - bucket_name = var.gcs_bucket_name - namespace = var.namespace - pvc_name = local.pvc_name - }) - lustre = merge(local.common_pv_vars, { - location = var.lustre_id != null ? split("/", var.lustre_id)[3] : null - project = split("/", var.cluster_id)[1] - instance_name = local.base_name - server_ip = var.lustre_id != null ? split("@", var.network_storage.server_ip)[0] : null - filesystem_name = var.network_storage.remote_mount - pvc_name = local.pvc_name - namespace = var.namespace - }) - filestore = merge(local.common_pv_vars, { - location = var.filestore_id != null ? split("/", var.filestore_id)[3] : null - filestore_name = local.base_name - share_name = trimprefix(var.network_storage.remote_mount, "/") - ip_address = var.network_storage.server_ip - pvc_name = local.pvc_name - namespace = var.namespace - }) - } - - # Rendered YAML contents - pv_content = templatefile( - local.pv_templates[local.storage_type], - local.pv_template_vars[local.storage_type] - ) - pvc_content = templatefile( - local.pvc_templates[local.storage_type], - local.common_pvc_vars - ) - - # GKE Cluster details - cluster_name = split("/", var.cluster_id)[5] - cluster_location = split("/", var.cluster_id)[3] -} - -data "google_container_cluster" "gke_cluster" { - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -provider "kubectl" { - host = "https://${data.google_container_cluster.gke_cluster.endpoint}" - cluster_ca_certificate = base64decode(data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) - token = data.google_client_config.default.access_token - load_config_file = false -} - -resource "kubectl_manifest" "pvc_namespace" { - count = var.namespace != "default" ? 1 : 0 - - yaml_body = templatefile("${path.module}/templates/namespace.yaml.tftpl", { - namespace = var.namespace - }) -} - -resource "kubectl_manifest" "pv" { - yaml_body = local.pv_content - - lifecycle { - precondition { - condition = length(local.active_types) == 1 - error_message = "Exactly one of gcs_bucket_name, filestore_id, or lustre_id must be set." - } - } -} - -resource "kubectl_manifest" "pvc" { - yaml_body = local.pvc_content - depends_on = [kubectl_manifest.pv, kubectl_manifest.pvc_namespace] -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf deleted file mode 100644 index 60cf2dbe0f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "persistent_volume_claims" { - description = "An object describing the Kubernetes PersistentVolumeClaim created by this module." - value = { - name = local.pvc_name - namespace = var.namespace - mount_path = var.network_storage.local_mount - mount_options = var.network_storage.mount_options - storage_type = local.storage_type - } -} - -output "pvc_name" { - description = "The name of the Kubernetes PVC created by this module." - value = local.pvc_name -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl deleted file mode 100644 index 06a1276c1e..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ${pv_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - storageClassName: "" - capacity: - storage: ${capacity} - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - volumeMode: Filesystem - csi: - driver: filestore.csi.storage.gke.io - volumeHandle: "modeInstance/${location}/${filestore_name}/${share_name}" - volumeAttributes: - ip: ${ip_address} - volume: ${share_name} - claimRef: - name: ${pvc_name} - namespace: ${namespace} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl deleted file mode 100644 index 83cfb3bc8c..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl +++ /dev/null @@ -1,18 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - volumeName: ${pv_name} - resources: - requests: - storage: ${capacity} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl deleted file mode 100644 index aa0e570a8b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ${pv_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - storageClassName: "" - capacity: - storage: ${capacity} - accessModes: - - ReadWriteMany - %{~ if mount_options != null ~} - mountOptions: - %{~ for key in mount_options ~} - - ${key} - %{~ endfor ~} - %{~ endif ~} - csi: - driver: gcsfuse.csi.storage.gke.io - volumeHandle: ${bucket_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl deleted file mode 100644 index 4d02c85629..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl +++ /dev/null @@ -1,21 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - volumeName: ${pv_name} - resources: - requests: - storage: ${capacity} - claimRef: - name: ${pvc_name} - namespace: ${namespace} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl deleted file mode 100644 index 2b3b5e7738..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ${pv_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - storageClassName: "" - capacity: - storage: ${capacity} - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - volumeMode: Filesystem - claimRef: - namespace: ${namespace} - name: ${pvc_name} - csi: - driver: lustre.csi.storage.gke.io - volumeHandle: "${project}/${location}/${instance_name}/default-pool/default-container" - volumeAttributes: - ip: ${server_ip} - filesystem: ${filesystem_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl deleted file mode 100644 index 83cfb3bc8c..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl +++ /dev/null @@ -1,18 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - volumeName: ${pv_name} - resources: - requests: - storage: ${capacity} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl deleted file mode 100644 index fa7647e33f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf deleted file mode 100644 index fd281756e7..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "cluster_id" { - description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" - type = string -} - -variable "network_storage" { - description = "Network attached storage mount to be configured." - type = object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - }) -} - -variable "filestore_id" { - description = "An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." - type = string - default = null - validation { - condition = ( - var.filestore_id == null || - try(length(split("/", var.filestore_id)), 0) == 6 - ) - error_message = "filestore_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." - } -} - -variable "lustre_id" { - description = "An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." - type = string - default = null - validation { - condition = ( - var.lustre_id == null || - try(length(split("/", var.lustre_id)), 0) == 6 - ) - error_message = "lustre_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." - } -} - -variable "gcs_bucket_name" { - description = "The gcs bucket to be used with the persistent volume." - type = string - default = null -} - -variable "capacity_gib" { - description = "The storage capacity with which to create the persistent volume." - type = number -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} - -variable "namespace" { - description = "Kubernetes namespace to deploy the storage PVC/PV" - type = string - default = "default" -} - -variable "pv_name" { - description = "The name for PV. IF not set, a name will be generated based on the storage name." - type = string - default = null -} - -variable "pvc_name" { - description = "The name for PVC. IF not set, a name will be generated based on the storage name." - type = string - default = null -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf deleted file mode 100644 index fa1c3e2b3f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.7.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.74.0" - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/README.md deleted file mode 100644 index 78ef5402aa..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/README.md +++ /dev/null @@ -1,134 +0,0 @@ -## Description - -This module creates Kubernetes Storage Class (SC) that can be used by a Persistent Volume Claim (PVC) -to dynamically provision GCP storage resources like Parallelstore. - -### Example - -The following example uses the `gke-storage` module to creates a Parallelstore Storage Class and Persistent Volume Claim, -then use them in a `gke-job-template` to dynamically provision the resource. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network] - settings: - enable_parallelstore_csi: true - - # Private Service Access (PSA) requires the compute.networkAdmin role which is - # included in the Owner role, but not Editor. - # PSA is required for all Parallelstore functionality. - # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: gke_storage - source: modules/file-system/gke-storage - use: [ gke_cluster, private_service_access ] - settings: - storage_type: Parallelstore - access_mode: ReadWriteMany - sc_volume_binding_mode: Immediate - sc_reclaim_policy: Delete - sc_topology_zones: [$(vars.zone)] - pvc_count: 2 - capacity_gb: 12000 - - - id: job_template - source: modules/compute/gke-job-template - use: [gke_storage, compute_pool] -``` - -See example -[gke-managed-parallelstore.yaml](../../../examples/README.md#gke-managed-parallelstoreyaml--) blueprint -for a complete example. - -### Authorized Network - -Since the `gke-storage` module is making calls to the Kubernetes API -to create Kubernetes entities, the machine performing the deployment must be -authorized to connect to the Kubernetes API. You can add the -`master_authorized_networks` settings block, as shown in the example above, with -the IP address of the machine performing the deployment. This will ensure that -the deploying machine can connect to the cluster. - -### Connecting Via Use - -The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit -modules. For example the `gke-storage` module can `use` a -`gke-cluster` module and a `private_service_access` module, as shown in the example above. - -```mermaid -graph TD; - vpc-->|OneToMany|gke-cluster; - gke-cluster-->|OneToMany|gke-node-pool; - gke-node-pool-->|ManyToMany|gke-job-template; - gke-cluster-->|OneToMany|gke-storage; - gke-storage-->|ManyToMany|gke-job-template; -``` - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_mode](#input\_access\_mode) | The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)
Valid access modes:
- ReadWriteOnce
- ReadOnlyMany
- ReadWriteMany
- ReadWriteOncePod | `string` | n/a | yes | -| [capacity\_gb](#input\_capacity\_gb) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | -| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [mount\_options](#input\_mount\_options) | Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class. | `string` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [pv\_mount\_path](#input\_pv\_mount\_path) | Path within the container at which the volume should be mounted. Must not contain ':'. | `string` | `"/data"` | no | -| [pvc\_count](#input\_pvc\_count) | How many PersistentVolumeClaims that will be created | `number` | `1` | no | -| [sc\_reclaim\_policy](#input\_sc\_reclaim\_policy) | Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted.
[More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming)
Supported value:
- Retain
- Delete | `string` | n/a | yes | -| [sc\_topology\_zones](#input\_sc\_topology\_zones) | Zone location that allow the volumes to be dynamically provisioned. | `list(string)` | `null` | no | -| [sc\_volume\_binding\_mode](#input\_sc\_volume\_binding\_mode) | Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound.
Supported value:
- Immediate
- WaitForFirstConsumer | `string` | `"WaitForFirstConsumer"` | no | -| [storage\_type](#input\_storage\_type) | The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview)
to used. This module currently support dynamic provisioning for the below storage options
- Parallelstore
- Hyperdisk-balanced
- Hyperdisk-throughput
- Hyperdisk-extreme | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object that describes a k8s PVC created by this module. | - diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/main.tf deleted file mode 100644 index 9c9a641f79..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/main.tf +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-storage", ghpc_role = "file-system" }) -} - -locals { - storage_type = lower(var.storage_type) - storage_class_name = "${local.storage_type}-sc" - pvc_name_prefix = "${local.storage_type}-pvc" -} - -check "private_vpc_connection_peering" { - assert { - condition = lower(var.storage_type) != "parallelstore" ? true : var.private_vpc_connection_peering != null - error_message = <<-EOT - Parallelstore must be run within the same VPC as the GKE cluster and have private services access enabled. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access. - If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). - EOT - } -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = var.cluster_id - project_id = var.project_id - - # count = var.pvc_count - apply_manifests = flatten( - [ - # create StorageClass in the cluster - { - content = templatefile( - "${path.module}/storage-class/${local.storage_class_name}.yaml.tftpl", - { - name = local.storage_class_name - labels = local.labels - volume_binding_mode = var.sc_volume_binding_mode - reclaim_policy = var.sc_reclaim_policy - topology_zones = var.sc_topology_zones - }) - }, - var.namespace != "default" ? [{ - content = templatefile( - "${path.module}/persistent-volume-claim/namespace.yaml.tftpl", - { - namespace = var.namespace - }) - }] : [], - # create PersistentVolumeClaim in the cluster - flatten([ - for idx in range(var.pvc_count) : [ - { - content = templatefile( - "${path.module}/persistent-volume-claim/${(local.pvc_name_prefix)}.yaml.tftpl", - { - pvc_name = "${local.pvc_name_prefix}-${idx}" - labels = local.labels - capacity = "${var.capacity_gb}Gi" - access_mode = var.access_mode - storage_class_name = local.storage_class_name - namespace = var.namespace - } - ) - } - ] - ]) - ]) -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/metadata.yaml deleted file mode 100644 index 8722823274..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/outputs.tf deleted file mode 100644 index ce80cdb266..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/outputs.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "persistent_volume_claims" { - description = "An object that describes a k8s PVC created by this module." - value = flatten([ - for idx in range(var.pvc_count) : [{ - name = "${local.pvc_name_prefix}-${idx}" - namespace = var.namespace - mount_path = "${var.pv_mount_path}/${local.pvc_name_prefix}-${idx}" - mount_options = var.mount_options - storage_type = local.storage_type - }] - ]) -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl deleted file mode 100644 index fa7647e33f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl deleted file mode 100644 index 46e1f023d3..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - type: hyperdisk-balanced - provisioned-throughput-on-create: "250Mi" - provisioned-iops-on-create: "7000" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl deleted file mode 100644 index 445020d001..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - %{~ endfor ~} - type: hyperdisk-extreme - provisioned-iops-on-create: "50000" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl deleted file mode 100644 index ec404aec45..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - type: hyperdisk-throughput - provisioned-throughput-on-create: "250Mi" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl deleted file mode 100644 index e6b8ea8d3e..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: parallelstore.csi.storage.gke.io -parameters: -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/variables.tf deleted file mode 100644 index dba1c33b77..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/variables.tf +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "cluster_id" { - description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" - type = string -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} - -variable "storage_type" { - description = <<-EOT - The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview) - to used. This module currently support dynamic provisioning for the below storage options - - Parallelstore - - Hyperdisk-balanced - - Hyperdisk-throughput - - Hyperdisk-extreme - EOT - type = string - nullable = false - validation { - condition = var.storage_type == null ? false : contains(["parallelstore", "hyperdisk-balanced", "hyperdisk-throughput", "hyperdisk-extreme"], lower(var.storage_type)) - error_message = "Allowed string values for var.storage_type are \"Parallelstore\", \"Hyperdisk-balanced\", \"Hyperdisk-throughput\", \"Hyperdisk-extreme\"." - } -} - -variable "access_mode" { - description = <<-EOT - The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) - Valid access modes: - - ReadWriteOnce - - ReadOnlyMany - - ReadWriteMany - - ReadWriteOncePod - EOT - type = string - nullable = false - validation { - condition = var.access_mode == null ? false : contains(["readwriteonce", "readonlymany", "readwritemany", "readwriteoncepod"], lower(var.access_mode)) - error_message = "Allowed string values for var.access_mode are \"ReadWriteOnce\", \"ReadOnlyMany\", \"ReadWriteMany\", \"ReadWriteOncePod\"." - } -} - -variable "sc_volume_binding_mode" { - description = <<-EOT - Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound. - Supported value: - - Immediate - - WaitForFirstConsumer - EOT - type = string - default = "WaitForFirstConsumer" - validation { - condition = var.sc_volume_binding_mode == null ? true : contains(["immediate", "waitforfirstconsumer"], lower(var.sc_volume_binding_mode)) - error_message = "Allowed string values for var.sc_volume_binding_mode are \"Immediate\", \"WaitForFirstConsumer\"." - } -} - -variable "sc_reclaim_policy" { - description = <<-EOT - Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted. - [More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming) - Supported value: - - Retain - - Delete - EOT - type = string - nullable = false - validation { - condition = var.sc_reclaim_policy == null ? true : contains(["retain", "delete"], lower(var.sc_reclaim_policy)) - error_message = "Allowed string values for var.sc_reclaim_policy are \"Retain\", \"Delete\"." - } -} - -variable "sc_topology_zones" { - description = "Zone location that allow the volumes to be dynamically provisioned." - type = list(string) - default = null -} - -variable "pvc_count" { - description = "How many PersistentVolumeClaims that will be created" - type = number - default = 1 -} - -variable "pv_mount_path" { - description = "Path within the container at which the volume should be mounted. Must not contain ':'." - type = string - default = "/data" - validation { - condition = var.pv_mount_path == null ? true : !strcontains(var.pv_mount_path, ":") - error_message = "pv_mount_path must not contain ':', please correct it and retry" - } -} - -variable "mount_options" { - description = "Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class." - type = string - default = null -} - -variable "capacity_gb" { - description = "The storage capacity with which to create the persistent volume." - type = number -} - -variable "private_vpc_connection_peering" { - description = <<-EOT - The name of the VPC Network peering connection. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access and - If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). - EOT - type = string - default = null -} - -variable "namespace" { - description = "Kubernetes namespace to deploy the storage PVC/PV" - type = string - default = "default" -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/versions.tf deleted file mode 100644 index bcc803e41e..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/gke-storage/versions.tf +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.5" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.74.0" - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/README.md deleted file mode 100644 index 28530a379f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/README.md +++ /dev/null @@ -1,289 +0,0 @@ -## Description - -This module creates a [Managed Lustre](https://cloud.google.com/managed-lustre) -instance. Managed Lustre is a high performance network file system that can be -mounted to one or more VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -### Supported Operating Systems - -A Managed Lustre instance can be used with Slurm cluster or compute -VM running Ubuntu 20.04, 22.04 or Rocky Linux 8 (including the HPC flavor). - -### Managed Lustre Access - -Managed Lustre must be enabled for your project by Google staff. Please contact -your sales representative for further steps. - -### Example - New VPC - -For Managed Lustre instance, the snippet below creates new VPC and configures -private-service-access for this newly created network. Both items are required -to be passed to the Lustre module to ensure that they're built in order and -that the correct subnetwork has private service access. - -```yaml - - id: network - source: modules/network/vpc - - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] -``` - -### Example - Slurm - -When using Slurm you must take into consideration whether or not you are using -an official image from the `schedmd-slurm-public` project or building your own. -The Lustre client modules are pre-installed in the official images. With the -official images, Lustre can be used as follows: - -```yaml -- id: managed_lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - -# Other modules: nodesets, partitions, login, etc. - -- id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: - - network - - lustre_partition - - managed_lustre - - slurm_login - settings: - machine_type: n2-standard-4 - enable_controller_public_ips: true -``` - -For custom images you must install the modules during the image build as the -Slurm cluster will not run the installation script like it does for the -standard VMs. - -Assuming you have a startup script for the Slurm image building, you can add -this Ansible playbook to correctly install the Lustre drivers into the image -(for Slurm-GCP versions greater than 6.10.0): - -```yaml -- type: data - destination: /var/tmp/slurm_vars.json - content: | - { - "reboot": false, - "install_cuda": false, - "install_gcsfuse": true, - "install_lustre": false, - "install_managed_lustre": true, - "install_nvidia_repo": true, - "install_ompi": true, - "allow_kernel_upgrades": false, - "monitoring_agent": "cloud-ops", - } -``` - -The `install_managed_lustre: true` line specifies that slurm-gcp should install -the correct modules within the slurm image. This runner should be placed -ahead of the script that calls the ansible build of the slurm-gcp image. - -### Example - Existing VPC - -If you want to use existing network with private-service-access configured, you need -to manually provide `private_vpc_connection_peering` to the Managed Lustre module. -You can get this details from the Google Cloud Console UI in `VPC network peering` -section. Below is the example of using existing network and creating Managed Lustre. -If existing network is not configured with private-service-access, you can follow -[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) -to set it up. - -```yaml - - id: network - source: modules/network/pre-existing-vpc - settings: - network_name: // Add network name - subnetwork_name: // Add subnetwork name - - - id: lustre - source: modules/file-system/managed-lustre - use: [network] - settings: - private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" -``` - -### Example - GKE compatibility - -By default the Managed Lustre instance that is deployed is not compatible with -GKE. To enable the compatibility use the `gke_support_enabled: true` option. -This creates a file `/etc/modprobe/lnet.conf` that changes the listening port -to 6988. - -```yaml - - id: managed-lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - gke_support_enabled: true -``` - -> [!WARNING] -> -> 1. VMs cannot connect to both GKE compatible and GKE incompatible lustre -> instances at the same time as they connect to different ports. Lustre can -> only listen to one port at a time. -> -> 2. Setting `gke_support_enabled: true` will not affect Slurm nodes, GKE -> compatibility must be built into the Slurm image. - -### Example - Importing data from GSC Bucket - -One option with the Managed Lustre instance is to import data from a GSC bucket -upon the lustre instance creation. To do this, use the `import_gcs_bucket_uri` -variable to dictate the bucket to pull data from. The data will be imported -under the directory specified by `local_mount` (`/shared` if unspecified). - -> [!NOTE] -> -> 1. This is a one way operation. Once the data has been copied to the lustre -> instance it will not be updated with any changes made to the GCS bucket. -> -> 2. Once the lustre instance has been created in Terraform, the copy process -> will proceed in the background. Data may not be appear in the mounted -> directory for a period of time after the deployment has completed (see below). - -```yaml -- id: managed_lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - import_gcs_bucket_uri: gs:// -``` - -> [!WARNING] -> Please follow [this guide](https://cloud.google.com/managed-lustre/docs/transfer-data#required_permissions) -> to set up the correct IAM permissions for importing data from GCS to lustre. -> Without this, the copy process may fail silently leaving an empty lustre -> instance. - -If an import is requested, gcluster will output a json response similar to: - -```json -{ - "name": "projects//locations//operations/", - "metadata": { - "@type": "type.googleapis.com/google.cloud.lustre.v1.ImportDataMetadata", - "createTime": "", - "target": "projects//locations//instances/", - "requestedCancellation": false, - "apiVersion": "v1" - }, - "done": false -} -``` - -You can retrieve more information about the transfer using the following -command, substituting with values from the json response above: - -```bash -gcloud lustre operations describe --location --project -``` - -This will provide information on if the transfer is complete or if any errors -have occurred. See more at -[Get operation](https://cloud.google.com/managed-lustre/docs/transfer-data#get_operation). - -## License - - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 6.27.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.27.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_lustre_instance.lustre_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/lustre_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | -| [google_storage_bucket.lustre_import_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the Lustre instance if no name is specified. | `string` | n/a | yes | -| [description](#input\_description) | Description of the created Lustre instance. | `string` | `"Lustre Instance"` | no | -| [gke\_support\_enabled](#input\_gke\_support\_enabled) | Set to true to create Managed Lustre instance with GKE compatibility.
Note: This does not work with Slurm, the Slurm image must be built with
the correct compatibility. | `bool` | `false` | no | -| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to managed lustre. Data will
be imported to the local\_mount directory. Changing this value will not
trigger a redeployment, to prevent data deletion. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the Managed Lustre instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mount](#input\_local\_mount) | Local mount point for the Managed Lustre instance. | `string` | `"/shared"` | no | -| [mount\_options](#input\_mount\_options) | Mounting options for the file system. | `string` | `"defaults,_netdev"` | no | -| [name](#input\_name) | Name of the Lustre instance | `string` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | Network self-link this instance will be on, required for checking private service access | `string` | n/a | yes | -| [per\_unit\_storage\_throughput](#input\_per\_unit\_storage\_throughput) | Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000. | `number` | `500` | no | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | ID of project in which Lustre instance will be created. | `string` | n/a | yes | -| [remote\_mount](#input\_remote\_mount) | Remote mount point of the Managed Lustre instance | `string` | n/a | yes | -| [size\_gib](#input\_size\_gib) | Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations | `number` | `36000` | no | -| [zone](#input\_zone) | Location for the Lustre instance. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | -| [install\_managed\_lustre\_client](#output\_install\_managed\_lustre\_client) | Script for installing Managed Lustre client | -| [lustre\_id](#output\_lustre\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | -| [network\_storage](#output\_network\_storage) | Describes a Managed Lustre instance. | - diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/main.tf deleted file mode 100644 index a969c53673..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/main.tf +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "managed-lustre", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -data "google_compute_network_peering" "private_peering" { - name = var.private_vpc_connection_peering - network = var.network_self_link -} - -locals { - server_ip = split(":", google_lustre_instance.lustre_instance.mount_point)[0] - remote_mount = split(":", google_lustre_instance.lustre_instance.mount_point)[1] - fs_type = "lustre" - mount_options = var.mount_options - instance_id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - destination_path = "/" - - install_managed_lustre_client_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-managed-lustre-client.sh" - "destination" = "install-managed-lustre-client${replace(var.local_mount, "/", "_")}.sh" - "args" = var.gke_support_enabled ? "1" : "0" - } - mount_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" - } - - bucket_count = try(length(data.google_storage_bucket.lustre_import_bucket), 0) -} - -data "google_storage_bucket" "lustre_import_bucket" { - count = try(length(var.import_gcs_bucket_uri) > 0, false) ? 1 : 0 - - name = split("//", var.import_gcs_bucket_uri)[1] -} - -resource "google_lustre_instance" "lustre_instance" { - project = var.project_id - - description = var.description - instance_id = local.instance_id - location = var.zone - - filesystem = var.remote_mount - capacity_gib = var.size_gib - per_unit_storage_throughput = var.per_unit_storage_throughput - - labels = local.labels - network = var.network_id - - gke_support_enabled = var.gke_support_enabled - - timeouts { - create = "1h" - update = "1h" - delete = "1h" - } - - depends_on = [var.private_vpc_connection_peering, data.google_storage_bucket.lustre_import_bucket] - - lifecycle { - precondition { - condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" - error_message = "The subnetwork that the lustre instance is hosted on must have private service access." - } - } - - provisioner "local-exec" { - command = < 0 ]]; then - curl -X POST \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $(gcloud auth print-access-token)" \ - -d '{"gcsPath": {"uri":"${coalesce(var.import_gcs_bucket_uri, "gs://")}"}, "lustrePath": {"path":"${local.destination_path}"}}' \ - https://lustre.googleapis.com/v1/projects/${var.project_id}/locations/${var.zone}/instances/${local.instance_id}:importData - fi - EOF - interpreter = ["bash", "-c"] - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/metadata.yaml deleted file mode 100644 index 66da9827b6..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - lustre.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/outputs.tf deleted file mode 100644 index 6de815524a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a Managed Lustre instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = local.mount_options - client_install_runner = local.install_managed_lustre_client_runner - mount_runner = local.mount_runner - } -} - -output "install_managed_lustre_client" { - description = "Script for installing Managed Lustre client" - value = file("${path.module}/scripts/install-managed-lustre-client.sh") -} - -output "lustre_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" - value = google_lustre_instance.lustre_instance.id -} - -output "capacity_gib" { - description = "File share capacity in GiB." - value = google_lustre_instance.lustre_instance.capacity_gib -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh deleted file mode 100644 index 878130ab47..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Install Managed Lustre client modules -# Based on these instructions: https://cloud.google.com/managed-lustre/docs/connect-from-compute-engine - -# The client modules currently only support Rocky 8, and Ubuntu 20.04/22.04 - -set -e - -GKE_ENABLED=$1 - -# Update lnet to enable GKE supported Lustre instance -if [[ $GKE_ENABLED == "1" ]]; then - if [[ -f "/etc/modprobe.d/lnet.conf" ]] && grep -Fq "options lnet accept_port" /etc/modprobe.d/lnet.conf; then - echo "Lnet accept port already set, continuing without updating /etc/modprobe.d/lnet.conf" - else - echo "options lnet accept_port=6988" >>/etc/modprobe.d/lnet.conf - fi -fi - -if grep -q lustre /proc/filesystems; then - echo "Skipping managed lustre client install as it is already supported" - exit 0 -fi - -# Get distro information -. /etc/os-release -DIST="NA" -if [[ $NAME == *"Ubuntu"* ]]; then - if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then - DIST="Ubuntu" - fi -elif [[ $NAME == *"Rocky"* ]]; then - if [[ $VERSION_ID == "8"* ]]; then - DIST="Rocky" - fi -fi - -if [[ ${DIST} == "Ubuntu" ]]; then - KEY_LOC=/etc/apt/keyrings - KEY_NAME=gcp-ar-repo.gpg - # Download new repo key - mkdir -p "${KEY_LOC}" - wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null - - # Set up apt repo - echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list - - # Install modules - apt update - apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) -elif [[ ${DIST} == "Rocky" ]]; then - # Set up yum repo - touch /etc/yum.repos.d/artifact-registry.repo - tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF - [lustre-client-rocky-8] - name=lustre-client-rocky-8 - baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - # Install modules - yum makecache - yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client -fi - -if [[ $DIST != "NA" ]]; then - # Load the new lustre client module - modprobe lustre -fi diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/variables.tf deleted file mode 100644 index 65607af66d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/variables.tf +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which Lustre instance will be created." - type = string -} - -variable "description" { - description = "Description of the created Lustre instance." - type = string - default = "Lustre Instance" -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the Lustre instance if no name is specified." - type = string -} - -variable "zone" { - description = "Location for the Lustre instance." - type = string -} - -variable "name" { - description = "Name of the Lustre instance" - type = string -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is connected given in the format: - `projects//global/networks/`" - EOT - type = string - nullable = false - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "network_self_link" { - description = "Network self-link this instance will be on, required for checking private service access" - type = string - nullable = false -} - -variable "remote_mount" { - description = "Remote mount point of the Managed Lustre instance" - type = string - nullable = false -} - -variable "local_mount" { - description = "Local mount point for the Managed Lustre instance." - type = string - default = "/shared" -} - -variable "size_gib" { - description = "Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations" - type = number - default = 36000 -} - -variable "per_unit_storage_throughput" { - description = "Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000." - type = number - default = 500 -} - -variable "labels" { - description = "Labels to add to the Managed Lustre instance. Key-value pairs." - type = map(string) -} - -variable "mount_options" { - description = "Mounting options for the file system." - type = string - default = "defaults,_netdev" -} - -variable "private_vpc_connection_peering" { - description = <<-EOT - The name of the VPC Network peering connection. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access and - If using existing VPC with private-service-access enabled, set this manually." - EOT - type = string - nullable = false -} - -variable "gke_support_enabled" { - description = <<-EOT - Set to true to create Managed Lustre instance with GKE compatibility. - Note: This does not work with Slurm, the Slurm image must be built with - the correct compatibility. - EOT - type = bool - nullable = false - default = false -} - -variable "import_gcs_bucket_uri" { - description = <<-EOT - The name of the GCS bucket to import data from to managed lustre. Data will - be imported to the local_mount directory. Changing this value will not - trigger a redeployment, to prevent data deletion. - EOT - type = string - default = null - - validation { - condition = startswith(coalesce(var.import_gcs_bucket_uri, "gs://"), "gs://") - error_message = "The GCS bucket uri must start with 'gs://'" - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/versions.tf deleted file mode 100644 index 2322c9a8fd..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/managed-lustre/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.27.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/README.md deleted file mode 100644 index 82332f3406..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/README.md +++ /dev/null @@ -1,193 +0,0 @@ -## Description - -This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) -storage pool. - -NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. -NetApp Volume provides: - -- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x -- a [rich feature set][service-levels] -- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) -- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data -- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost - -Support for NetApp Volumes is split into two modules. - -- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. -- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -### NetApp storage pool service levels - -The netapp-storage-pool module currently supports the following NetApp Volumes [service levels][service-levels]: - -- Standard: 16 KiBps throughput per provisioned KiB of volume capacity. -- Premium: 64 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. -- Extreme: 128 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. - -Check the [service level matrix][service-levels] for additional information on capability differences between service levels. Flex service levels are currently not supported, but you can connect to existing Flex volumes using the [pre-existing-network-storage module][pre-existing]. - -### On-boarding NetApp Volumes -NetApp Volumes uses [Private Service Access](https://cloud.google.com/vpc/docs/private-services-access) (PSA) to connect volumes to your network. Before you create a storage pool, make sure to [connect NetApp Volumes to your network](https://cloud.google.com/netapp/volumes/docs/get-started/configure-access/networking). - -Example of creating a storage pool using a new network: - -```yaml -deployment_groups: -- group: primary - modules: - - id: network - source: modules/network/vpc - settings: - region: $(vars.region) - - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - service_name: "netapp.servicenetworking.goog" - deletion_policy: "ABANDON" - - - id: netapp_pool - source: modules/file-system/netapp-storage-pool - use: [network, private_service_access] - settings: - pool_name: $(vars.deployment_name)-eda-pool - capacity_gib: 20000 - service_level: "EXTREME" - region: $(vars.region) -``` - -Example of creating a storage pool using an existing network which was already PSA-peered with NetApp Volume: - -```yaml -deployment_groups: - - group: primary - modules: - - id: network - source: modules/network/pre-existing-vpc - settings: - project_id: $(vars.project_id) - region: $(vars.region) - network_name: $(vars.network) - - - id: netapp_pool - source: modules/file-system/netapp-storage-pool - use: [network] - settings: - pool_name: "eda-pool" - capacity_gib: 20000 - service_level: "EXTREME" - region: $(vars.region) -``` - -### Storage pool example - -The following example shows all available parameters in use: - -```yaml - - id: netapp_pool - source: modules/file-system/netapp-storage-pool - use: [network, private_service_access] - settings: - pool_name: "mypool" - region: "us-west4" - capacity_gib: 2048 - service_level: "EXTREME" - active_directory_policy: "projects/myproject/locations/us-east4/activeDirectories/my-ad" - cmek_policy: "projects/myproject/locations/us-east4/kmsConfigs/my-cmek-policy" - ldap_enabled: false - allow_auto_tiering: false - description: "Demo storage pool" - labels: - owner: bob -``` - -### NetApp Volumes quota - -Your project must have unused quota for NetApp Volumes in the region you will -provision the storage pool. This can be found by browsing to the [Quota tab within IAM & Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. -Please note that there are separate quota limits for Standard and Premium/Extreme service levels. - -See also NetApp Volumes [default quotas](https://cloud.google.com/netapp/volumes/docs/quotas#netapp-volumes-default-quotas). - -[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels -[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering -[pre-existing]: ../pre-existing-network-storage/README.md -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5.7 | -| [google](#requirement\_google) | >= 6.45.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.45.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_netapp_storage_pool.netapp_storage_pool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_storage_pool) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [active\_directory\_policy](#input\_active\_directory\_policy) | The ID of the Active Directory policy to apply to the storage pool in the format:
`projects//locations//activeDirectoryPolicies/` | `string` | `null` | no | -| [allow\_auto\_tiering](#input\_allow\_auto\_tiering) | Whether to allow automatic tiering for the storage pool. | `bool` | `false` | no | -| [capacity\_gib](#input\_capacity\_gib) | The capacity of the storage pool in GiB. | `number` | `2048` | no | -| [cmek\_policy](#input\_cmek\_policy) | The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format:
`projects//locations//kmsConfigs/` | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used as name of the NetApp storage pool if no name is specified. | `string` | n/a | yes | -| [description](#input\_description) | A description of the NetApp storage pool. | `string` | `""` | no | -| [labels](#input\_labels) | Labels to add to the NetApp storage pool. Key-value pairs. | `map(string)` | n/a | yes | -| [ldap\_enabled](#input\_ldap\_enabled) | Whether to enable LDAP for the storage pool. | `bool` | `false` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | Network self-link the pool will be on, required for checking private service access | `string` | n/a | yes | -| [pool\_name](#input\_pool\_name) | The name of the storage pool. Leave empty to generate name based on deployment name. | `string` | `null` | no | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the private VPC connection peering. | `string` | `"sn-netapp-prod"` | no | -| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | -| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | -| [service\_level](#input\_service\_level) | The service level of the storage pool. | `string` | `"PREMIUM"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gb](#output\_capacity\_gb) | Storage pool capacity in GiB. | -| [netapp\_storage\_pool\_id](#output\_netapp\_storage\_pool\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}` | - diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/main.tf deleted file mode 100644 index b9d63c11c3..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/main.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "netapp-storage-pool", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -data "google_compute_network_peering" "private_peering" { - name = var.private_vpc_connection_peering - network = var.network_self_link -} - -resource "google_netapp_storage_pool" "netapp_storage_pool" { - project = var.project_id - - name = var.pool_name != null ? var.pool_name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - location = var.region - network = var.network_id - service_level = var.service_level - capacity_gib = var.capacity_gib - - active_directory = var.active_directory_policy - kms_config = var.cmek_policy - ldap_enabled = var.ldap_enabled - allow_auto_tiering = var.allow_auto_tiering - - description = var.description - labels = local.labels - - depends_on = [data.google_compute_network_peering.private_peering] - - lifecycle { - precondition { - condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" - error_message = "The network for the storage pool must have private service access." - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml deleted file mode 100644 index 7a5291f9d5..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - netapp.googleapis.com - - servicenetworking.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf deleted file mode 100644 index 91379631c6..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "netapp_storage_pool_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}`" - value = google_netapp_storage_pool.netapp_storage_pool.id -} - -output "capacity_gb" { - description = "Storage pool capacity in GiB." - value = google_netapp_storage_pool.netapp_storage_pool.capacity_gib -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf deleted file mode 100644 index 04f19fd3fb..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which the NetApp storage pool will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the deployment, used as name of the NetApp storage pool if no name is specified." - type = string -} - -variable "region" { - description = "Location for NetApp storage pool." - type = string -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "network_self_link" { - description = "Network self-link the pool will be on, required for checking private service access" - type = string - nullable = false -} - -variable "private_vpc_connection_peering" { - description = "The name of the private VPC connection peering." - type = string - default = "sn-netapp-prod" -} - -variable "pool_name" { - description = "The name of the storage pool. Leave empty to generate name based on deployment name." - type = string - default = null -} - -variable "service_level" { - description = "The service level of the storage pool." - type = string - default = "PREMIUM" - validation { - condition = contains(["STANDARD", "PREMIUM", "EXTREME"], var.service_level) - error_message = "Allowed values for service_level are 'STANDARD', 'PREMIUM', or 'EXTREME'." - } -} - -variable "capacity_gib" { - description = "The capacity of the storage pool in GiB." - type = number - default = 2048 - validation { - condition = var.capacity_gib >= 2048 - error_message = "The minimum capacity for the storage pool is 2048 GiB." - } -} - -variable "active_directory_policy" { - description = <<-EOT - The ID of the Active Directory policy to apply to the storage pool in the format: - `projects//locations//activeDirectoryPolicies/` - EOT - type = string - default = null - validation { - condition = var.active_directory_policy == null ? true : length(split("/", var.active_directory_policy)) == 6 - error_message = "The active directory policy must be provided in the following format: projects//locations//activeDirectoryPolicies/." - } -} - -variable "cmek_policy" { - description = <<-EOT - The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format: - `projects//locations//kmsConfigs/` - EOT - type = string - default = null - validation { - condition = var.cmek_policy == null ? true : length(split("/", var.cmek_policy)) == 6 - error_message = "The CMEK policy must be provided in the following format: projects//locations//kmsConfigs/." - } -} - -variable "ldap_enabled" { - description = "Whether to enable LDAP for the storage pool." - type = bool - default = false -} - -variable "allow_auto_tiering" { - description = "Whether to allow automatic tiering for the storage pool." - type = bool - default = false -} - -variable "description" { - description = "A description of the NetApp storage pool." - type = string - default = "" - validation { - condition = length(var.description) <= 2048 - error_message = "NetApp storage pool description must be 2048 characters or fewer" - } -} - -variable "labels" { - description = "Labels to add to the NetApp storage pool. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf deleted file mode 100644 index f6501116cd..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.45.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" - } - - required_version = ">= 1.5.7" -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/README.md deleted file mode 100644 index 6aaaf0cb05..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/README.md +++ /dev/null @@ -1,201 +0,0 @@ -## Description - -This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) -volume. - -NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. -NetApp Volume provides: - -- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x -- a [rich feature set][service-levels] -- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) -- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data -- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost - -Support for NetApp Volumes is split into two modules. - -- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. -- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -## Deletion protection -The netapp-volume module currently doesn't implement volume deletion protection. If you create a volume with Cluster Toolkit by using this module, Cluster Toolkit will also delete it when you run `gcluster destroy`. All the data in the volume will be gone. If you want to retain the volume instead, it is advised to [use existing volumes not created by Cluster Toolkit](#using-existing-volumes-not-created-by-cluster-toolkit). - -## Volumes overview -Volumes are filesystem containers which can be shared using NFS or SMB filesharing protocols. Volumes *live* inside of [storage pools](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview), which can be provisioned using the [netapp-storage-pool] module. Volumes inherit fundamental settings from the pool. They *consume* capacity provided by the pool. You can create one or multiple volumes *inside* a pool. - -[netapp-storage-pool]: ../netapp-storage-pool/README.md -[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels -[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering -[pre-existing]: ../pre-existing-network-storage/README.md -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## Volume examples -The following examples show the use of netapp-volume. They builds on top of an storage pool which can be provisioned using the [netapp-storage-pool][netapp-storage-pool] module. - -### Example with minimal parameters - -```yaml - - id: home_volume - source: modules/file-system/netapp-volume - use: [netapp_pool] # Create this pool using the netapp-storage-pool module - settings: - volume_name: "eda-home" - capacity_gib: 1024 # Size up to available capacity in the pool - local_mount: "/eda-home" # Mount point at client when client uses USE directive - protocols: ["NFSV3"] - region: $(vars.region) - # Default export policy exports to "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" and no_root_squash -``` - -### Example with all parameters - -```yaml - - id: shared_volume - source: modules/file-system/netapp-volume - use: [netapp_pool] # Create this pool using the netapp-storage-pool module - settings: - volume_name: "eda-shared" - capacity_gib: 25000 # Size up to available capacity in the pool - large_capacity: true - local_mount: "/shared" # Mount point at client when client uses USE directive - mount_options: "rw" # Allows customizing mount options for special workloads - protocols: ["NFSV3","NFSV4"] # List of protocols. ["NFSV3], ["NFSv4] or ["NFSV3, "NFSV4"] - region: $(vars.region) - unix_permissions: "0777" # Specify default permissions for roo inode owned by root:root - # If no export policy is specified, a permissive default policy will be applied, which is: - # allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" # RFC1918 - # has_root_access = true # no_root_squash enabled - # access_type = "READ_WRITE" - export_policy: - - allowed_clients: "10.10.20.8,10.10.20.9" - has_root_access: true # no_root_squash enabled - access_type: "READ_WRITE" - nfsv3: false # allow only NFSv4 for these hosts - nfsv4: true - - allowed_clients: "10.0.0.0/8" - has_root_access: false # no_root_squash disabled - access_type: "READ_WRITE" - nfsv3: true # allow only NFSv3 for these hosts - nfsv4: false - tiering_policy: # Enable auto-tiering. Requires auto-tiering enabled storage pool - tier_action: "ENABLED" - cooling_threshold_days: 31 # tier data blocks which have not been touched for 31 days - - description: "Shared volume for EDA job" - labels: - owner: bob -``` - -## Protocol support -Since Cluster Toolkit is currently built to provision Linux-based compute clusters, this module supports NFSv3 and NFSv4.1 only. SMB is blocked. - -## Large volumes -Volumes larger than 15 TiB can be created as [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). Such volumes can grow up to 3 PiB and can scale read performance up to 29 GiBps. They provide six IP addresses to the volume. They are exported via the `server_ips` output. When connecting a large volume to a client using the USE directive, cluster toolkit currently uses the first IP only. This will be improved in the future. - -This feature is allow-listed GA. To request allow-listing, see [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). - -## Auto-tiering support -For auto-tiering enabled storage pools you can enable auto-tiering on the volume. For more information, see [manage auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering). - -## Using existing volumes not created by Cluster Toolkit -NetApp Volumes volumes are regular NFS exports. You can use the [pre-existing-network-storage] module to integrate them into Cluster Toolkit. - -Example code: - -```yaml -- id: homefs - source: modules/file-system/pre-existing-network-storage - settings: - server_ip: ## Set server IP here ## - remote_mount: nfsshare - local_mount: /home - fs_type: nfs -``` - -This creates a resource in Cluster Toolkit which references the specified NFS export, which will be mounted at `/home` by clients which mount if via USE directive. - -Note that the `server_ip` must be known before deployment and this module does not allow -to specify a list of IPs for large volumes. - -[pre-existing-network-storage]: ../pre-existing-network-storage/README.md - -## FlexCache support -NetApp FlexCache technology accelerates data access, reduces WAN latency and lowers WAN bandwidth costs for read-intensive workloads, especially where clients need to access the same data repeatedly. When you create a FlexCache volume, you create a remote cache of an already existing (origin) volume that contains only the actively accessed data (hot data) of the origin volume. - -The FlexCache support in Google Cloud NetApp Volumes allows you to provision a cache volume in your Google network to improve performance for hybrid cloud environments. A FlexCache volume can help you transition workloads to the hybrid cloud by caching data from an on-premises data center to cloud. - -Deploying FlexCache volumes requires manual steps on the ONTAP origin side, which are not automated. Therefore this module has no support to deploy FlexCache volumes today. Deploy them manually and use the [pre-existing-network-storage](#using-existing-volumes-not-created-by-cluster-toolkit) instead. - -## License - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5.7 | -| [google](#requirement\_google) | >= 6.45.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.45.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_netapp_volume.netapp_volume](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_volume) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [capacity\_gib](#input\_capacity\_gib) | The capacity of the volume in GiB. | `number` | `1024` | no | -| [description](#input\_description) | A description of the NetApp volume. | `string` | `""` | no | -| [export\_policy\_rules](#input\_export\_policy\_rules) | Define NFS export policy. | 
list(object({
    allowed_clients = optional(string)
    has_root_access = optional(bool, false)
    access_type     = optional(string, "READ_WRITE")
    nfsv3           = optional(bool)
    nfsv4           = optional(bool)
  }))
| 
[
  {
    "access_type": "READ_WRITE",
    "allowed_clients": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
    "has_root_access": true
  }
]
| no | -| [labels](#input\_labels) | Labels to add to the NetApp volume. Key-value pairs. | `map(string)` | n/a | yes | -| [large\_capacity](#input\_large\_capacity) | If true, the volume will be created with large capacity.
Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. | `bool` | `false` | no | -| [local\_mount](#input\_local\_mount) | Mountpoint for this volume. | `string` | `"/shared"` | no | -| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"rw,hard,rsize=65536,wsize=65536,tcp"` | no | -| [netapp\_storage\_pool\_id](#input\_netapp\_storage\_pool\_id) | The ID of the NetApp storage pool to use for the volume. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | -| [protocols](#input\_protocols) | The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported. | `list(string)` | 
[
  "NFSV3"
]
| no | -| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | -| [tiering\_policy](#input\_tiering\_policy) | Define the tiering policy for the NetApp volume. | 
object({
    tier_action            = optional(string)
    cooling_threshold_days = optional(number)
  })
| `null` | no | -| [unix\_permissions](#input\_unix\_permissions) | UNIX permissions for root inode in the volume. | `string` | `"0777"` | no | -| [volume\_name](#input\_volume\_name) | The name of the volume. Needs to be unique within the storage pool. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gb](#output\_capacity\_gb) | Volume capacity in GiB. | -| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | -| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | -| [netapp\_volume\_id](#output\_netapp\_volume\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}` | -| [network\_storage](#output\_network\_storage) | Describes a NetApp Volumes volume. | -| [server\_ips](#output\_server\_ips) | List of IP addresses of the volume. | - diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/main.tf deleted file mode 100644 index d8345bf347..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/main.tf +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "netapp-volume", ghpc_role = "file-system" }) -} - -# resource "random_id" "resource_name_suffix" { -# byte_length = 4 -# } - -locals { - full_path = split(":", google_netapp_volume.netapp_volume.mount_options[0].export_full) - server_ip = local.full_path[0] - remote_mount = local.full_path[1] - # Large volumes will have 6 IPs - server_ips = [for ip in google_netapp_volume.netapp_volume.mount_options[*].export_full : split(":", ip)[0]] - fs_type = "nfs" - mount_options = var.mount_options - - install_nfs_client_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-nfs-client.sh" - "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" - } - mount_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${join(",", local.server_ips)}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" - } - - split_pool_id = split("/", var.netapp_storage_pool_id) - pool_name = local.split_pool_id[5] -} - -resource "google_netapp_volume" "netapp_volume" { - project = var.project_id - - name = var.volume_name - share_name = var.volume_name - location = var.region - protocols = var.protocols - capacity_gib = var.capacity_gib - large_capacity = var.large_capacity - multiple_endpoints = var.large_capacity == true ? true : null - storage_pool = local.pool_name - unix_permissions = var.unix_permissions - - dynamic "tiering_policy" { - for_each = var.tiering_policy == null ? [] : [0] - content { - cooling_threshold_days = lookup(var.tiering_policy, "cooling_threshold_days", null) - tier_action = lookup(var.tiering_policy, "tier_action", null) - } - } - - description = var.description - labels = local.labels - - dynamic "export_policy" { - for_each = var.export_policy_rules == null ? [] : [0] - content { - dynamic "rules" { - for_each = var.export_policy_rules - content { - access_type = rules.value.access_type - allowed_clients = rules.value.allowed_clients - has_root_access = rules.value.has_root_access - nfsv3 = rules.value.nfsv3 == null ? contains([for p in var.protocols : lower(p)], "nfsv3") : rules.value.nfsv3 - nfsv4 = rules.value.nfsv4 == null ? contains([for p in var.protocols : lower(p)], "nfsv4") : rules.value.nfsv4 - } - } - } - } - - depends_on = [var.netapp_storage_pool_id] -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/metadata.yaml deleted file mode 100644 index e4a7aaaa14..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - netapp.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/outputs.tf deleted file mode 100644 index 641eae007a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/outputs.tf +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -output "network_storage" { - description = "Describes a NetApp Volumes volume." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = local.mount_options - client_install_runner = local.install_nfs_client_runner - mount_runner = local.mount_runner - } -} - -output "install_nfs_client" { - description = "Script for installing NFS client" - value = file("${path.module}/scripts/install-nfs-client.sh") -} - -output "install_nfs_client_runner" { - description = "Runner to install NFS client using the startup-script module" - value = local.install_nfs_client_runner -} - -output "mount_runner" { - description = <<-EOT - Runner to mount the file-system using an ansible playbook. The startup-script - module will automatically handle installation of ansible. - - id: example-startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(your-fs-id.mount_runner) - ... - EOT - value = local.mount_runner -} - -output "netapp_volume_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}`" - value = google_netapp_volume.netapp_volume.id -} - -output "capacity_gb" { - description = "Volume capacity in GiB." - value = google_netapp_volume.netapp_volume.capacity_gib -} - -output "server_ips" { - description = "List of IP addresses of the volume." - value = local.server_ips -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh deleted file mode 100644 index 1b1595e5a4..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsupported distribution' - return 1 - fi -fi diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh deleted file mode 100644 index 8253d40a24..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IPS=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -# accept a list of colon-separated IPs and randomly pick one to enable load balancing -# In recent changes cluster toolkit doesn't seem to use this file anymore, -# which makes all mounts use the first IP in the list. Needs to be investigated in future. -IFS="," read -r -a arrIPS <<<"${SERVER_IPS}" -rand1=$(od -vAn -t d -N1 /dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/variables.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/variables.tf deleted file mode 100644 index 272558ff77..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/variables.tf +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which the NetApp storage pool will be created." - type = string -} - -variable "netapp_storage_pool_id" { - description = "The ID of the NetApp storage pool to use for the volume." - type = string - validation { - condition = length(split("/", var.netapp_storage_pool_id)) == 6 - error_message = "The storage pool id must be provided in the following format: projects//locations//storagePools/." - } -} - -variable "region" { - description = "Location for NetApp storage pool." - type = string -} - -variable "volume_name" { - description = "The name of the volume. Needs to be unique within the storage pool." - type = string - default = null -} - -variable "capacity_gib" { - description = "The capacity of the volume in GiB." - type = number - default = 1024 - validation { - condition = var.capacity_gib >= 100 - error_message = "The minimum capacity for the volume is 100 GiB." - } -} - -variable "protocols" { - description = "The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported." - type = list(string) - default = ["NFSV3"] - validation { - condition = alltrue([for p in var.protocols : contains(["NFSV3", "NFSV4"], p)]) - error_message = "Allowed values for protocols are 'NFSV3' or 'NFSV4'." - } -} - -variable "description" { - description = "A description of the NetApp volume." - type = string - default = "" - validation { - condition = length(var.description) <= 2048 - error_message = "NetApp volume description must be 2048 characters or fewer" - } -} - -variable "labels" { - description = "Labels to add to the NetApp volume. Key-value pairs." - type = map(string) -} - -variable "local_mount" { - description = "Mountpoint for this volume." - type = string - default = "/shared" -} - -variable "mount_options" { - description = "NFS mount options to mount file system." - type = string - default = "rw,hard,rsize=65536,wsize=65536,tcp" -} - -variable "large_capacity" { - description = <<-EOT - If true, the volume will be created with large capacity. - Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. - EOT - type = bool - default = false -} - -variable "unix_permissions" { - description = "UNIX permissions for root inode in the volume." - type = string - default = "0777" - validation { - condition = length(var.unix_permissions) <= 4 - error_message = "UNIX permissions must be a 4-digit octal number." - } -} - -variable "tiering_policy" { - description = "Define the tiering policy for the NetApp volume." - type = object({ - tier_action = optional(string) - cooling_threshold_days = optional(number) - }) - default = null -} - -variable "export_policy_rules" { - description = "Define NFS export policy." - type = list(object({ - allowed_clients = optional(string) - has_root_access = optional(bool, false) - access_type = optional(string, "READ_WRITE") - nfsv3 = optional(bool) - nfsv4 = optional(bool) - })) - # Permissive default if user does not specify nfs_export_options. Allow all RFC1918 CIDRS with no_root_squash - default = [{ - allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", - has_root_access = true, - access_type = "READ_WRITE", - }] - nullable = true -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/versions.tf deleted file mode 100644 index c624d5100b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/netapp-volume/versions.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.45.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" - } - - required_version = ">= 1.5.7" -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/README.md deleted file mode 100644 index 0b942f067f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/README.md +++ /dev/null @@ -1,196 +0,0 @@ -## Description - -This module creates [parallelstore](https://cloud.google.com/parallelstore) -instance. Parallelstore is Google Cloud's first party parallel file system -service based on [Intel DAOS](https://docs.daos.io/v2.2/) - -### Supported Operating Systems - -A parallelstore instance can be used with Slurm cluster or compute -VM running Ubuntu 22.04, debian 12 or HPC Rocky Linux 8. - -### Parallelstore Quota - -To get access to a private preview of Parallelstore APIs, your project needs to -be allowlisted. To set this up, please work with your account representative. - -### Parallelstore mount options - -After parallelstore instance is created, you can specify mount options depending -upon your workload. DAOS is configured to deliver the best user experience for -interactive workloads with aggressive caching. If you are running parallel -workloads concurrently accessing the sane files from multiple client nodes, it -is recommended to disable the writeback cache to avoid cross-client consistency -issues. You can specify different mount options as follows, - -```yaml - - id: parallelstore - source: modules/file-system/parallelstore - use: [network, ps_connect] - settings: - mount_options: "disable-wb-cache,thread-count=20,eq-count=8" -``` - -### Example - New VPC - -For parallelstore instance, Below snippet creates new VPC and configures private-service-access -for this newly created network. - -```yaml - - id: network - source: modules/network/vpc - - # Private Service Access (PSA) requires the compute.networkAdmin role which is - # included in the Owner role, but not Editor. - # PSA is required for all Parallelstore functionality. - # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: parallelstore - source: modules/file-system/parallelstore - use: [network, private_service_access] -``` - -### Example - Existing VPC - -If you want to use existing network with private-service-access configured, you need -to manually provide `private_vpc_connection_peering` to the parallelstore module. -You can get this details from the Google Cloud Console UI in `VPC network peering` -section. Below is the example of using existing network and creating parallelstore. -If existing network is not configured with private-service-access, you can follow -[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) -to set it up. - -```yaml - - id: network - source: modules/network/pre-existing-vpc - settings: - network_name: // Add network name - subnetwork_name: // Add subnetwork name - - - id: parallelstore - source: modules/file-system/parallelstore - use: [network] - settings: - private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" -``` - -### Import data from GCS bucket - -You can import data from your GCS bucket to parallelstore instance. Important to -note that data may not be available to the instance immediately. This depends on -latency and size of data. Below is the example of importing data from bucket. - -```yaml - - id: parallelstore - source: modules/file-system/parallelstore - use: [network] - settings: - import_gcs_bucket_uri: gs://gcs-bucket/folder-path - import_destination_path: /gcs/import/ -``` - -Here you can replace `import_gcs_bucket_uri` with the uri of sub folder within GCS -bucket and `import_destination_path` with local directory within parallelstore -instance. - -### Additional configuration for DAOS agent and dfuse -Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - daos_agent_config: | - credential_config: - cache_expiration: 1m -``` - -Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/parallelstore - settings: - dfuse_environment: - D_LOG_FILE: /tmp/client.log - D_APPEND_PID_TO_LOG: 1 - D_LOG_MASK: debug -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13 | -| [google](#requirement\_google) | >= 6.13.0 | -| [null](#requirement\_null) | ~> 3.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.13.0 | -| [null](#provider\_null) | ~> 3.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_parallelstore_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/parallelstore_instance) | resource | -| [null_resource.hydration](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [daos\_agent\_config](#input\_daos\_agent\_config) | Additional configuration to be added to daos\_config.yml | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. | `string` | n/a | yes | -| [dfuse\_environment](#input\_dfuse\_environment) | Additional environment variables for DFuse process | `map(string)` | `{}` | no | -| [directory\_stripe](#input\_directory\_stripe) | The parallelstore stripe level for directories. | `string` | `null` | no | -| [file\_stripe](#input\_file\_stripe) | The parallelstore stripe level for files. | `string` | `null` | no | -| [import\_destination\_path](#input\_import\_destination\_path) | The name of local path to import data on parallelstore instance from GCS bucket. | `string` | `null` | no | -| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to parallelstore. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to parallel store instance. | `map(string)` | `{}` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/parallelstore"` | no | -| [mount\_options](#input\_mount\_options) | Options describing various aspects of the parallelstore instance. | `string` | `"disable-wb-cache,thread-count=16,eq-count=8"` | no | -| [name](#input\_name) | Name of parallelstore instance. | `string` | `null` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [size\_gb](#input\_size\_gb) | Storage size of the parallelstore instance in GB. | `number` | `12000` | no | -| [zone](#input\_zone) | Location for parallelstore instance. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions to monitor import-data operation from GCS bucket to parallelstore. | -| [network\_storage](#output\_network\_storage) | Describes a parallelstore instance. | - diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/main.tf b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/main.tf deleted file mode 100644 index acc2a0551e..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/main.tf +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "parallelstore", ghpc_role = "file-system" }) -} - -locals { - fs_type = "daos" - server_ip = "" - remote_mount = "" - id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - access_points = jsonencode(google_parallelstore_instance.instance.access_points) - destination_path = var.import_destination_path == null ? "/" : var.import_destination_path - - client_install_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-daos-client.sh" - "destination" = "install_daos_client.sh" - } - - mount_runner = { - "type" = "shell" - "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { - access_points = local.access_points - daos_agent_config = var.daos_agent_config - dfuse_environment = var.dfuse_environment - local_mount = var.local_mount - mount_options = join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) - }) - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - } -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_parallelstore_instance" "instance" { - project = var.project_id - instance_id = local.id - location = var.zone - capacity_gib = var.size_gb - network = var.network_id - file_stripe_level = var.file_stripe - directory_stripe_level = var.directory_stripe - - labels = local.labels - - depends_on = [var.private_vpc_connection_peering] -} - -resource "null_resource" "hydration" { - count = var.import_gcs_bucket_uri != null ? 1 : 0 - - depends_on = [resource.google_parallelstore_instance.instance] - provisioner "local-exec" { - command = "curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -d '{\"source_gcs_bucket\": {\"uri\":\"${var.import_gcs_bucket_uri}\"}, \"destination_parallelstore\": {\"path\":\"${local.destination_path}\"}}' https://parallelstore.googleapis.com/v1beta/projects/${var.project_id}/locations/${var.zone}/instances/${local.id}:importData" - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/metadata.yaml deleted file mode 100644 index c0994d15bb..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - parallelstore.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/outputs.tf deleted file mode 100644 index f6e817ac8a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/outputs.tf +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - operation_instructions = <<-EOT - Data is being imported from GCS bucket to parallelstore instance. It may - not be available immediately. - EOT -} - -output "network_storage" { - description = "Describes a parallelstore instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = var.mount_options - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } - - precondition { - condition = var.import_gcs_bucket_uri != null || var.import_destination_path == null - error_message = <<-EOD - Please specify import_gcs_bucket_uri to import data to parallelstore instance. - EOD - } -} - -output "instructions" { - description = "Instructions to monitor import-data operation from GCS bucket to parallelstore." - value = var.import_gcs_bucket_uri != null ? local.operation_instructions : "Data is not imported from GCS bucket." -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh deleted file mode 100644 index e96eadb56a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || - { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 -fi - -if [ -x /bin/daos ]; then - echo "DAOS already installed" - daos version -else - # Install the DAOS client library - # The following commands should be executed on each client vm. - ## For Rocky linux 8 / RedHat 8. - if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then - # 1) Add the Parallelstore package repository - cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF - [parallelstore-v2-6-el${OS_VERSION_MAJOR}] - name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 - baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - - ## TODO: Remove disable automatic update script after issue is fixed. - if [ -x /usr/bin/google_disable_automatic_updates ]; then - /usr/bin/google_disable_automatic_updates - fi - dnf clean all - dnf makecache - - # 2) Install daos-client - dnf install -y epel-release # needed for capstone - dnf install -y daos-client - - # 3) Upgrade libfabric - dnf upgrade -y libfabric - - # For Ubuntu 22.04 and debian 12, - elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then - # shellcheck disable=SC2034 - DEBIAN_FRONTEND=noninteractive - - # 1) Add the Parallelstore package repository - curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg - echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list - - apt-get update - - # 2) Install daos-client - apt-get install -y daos-client - - # 3) Create daos_agent.service (comes pre-installed with RedHat) - if ! getent passwd daos_agent >/dev/null 2>&1; then - useradd daos_agent - fi - cat >/etc/systemd/system/daos_agent.service <<-EOF - [Unit] - Description=DAOS Agent - StartLimitIntervalSec=60 - Wants=network-online.target - After=network-online.target - - [Service] - Type=notify - User=daos_agent - Group=daos_agent - RuntimeDirectory=daos_agent - RuntimeDirectoryMode=0755 - ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml - StandardOutput=journal - StandardError=journal - Restart=always - RestartSec=10 - LimitMEMLOCK=infinity - LimitCORE=infinity - StartLimitBurst=5 - - [Install] - WantedBy=multi-user.target - EOF - else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - fi -fi - -exit 0 diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl deleted file mode 100644 index c6f5d53660..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; } && { [[ "$${OS_VERSION_MAJOR}" = "8" ]] || [[ "$${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "$${OS_ID}" = "ubuntu" ]] && [[ "$${OS_VERSION}" = "22.04" ]]; } || - { [[ "$${OS_ID}" = "debian" ]] && [[ "$${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system $${OS_ID} $${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - -fi - -# Edit agent config -daos_config=/etc/daos/daos_agent.yml - -# rewrite $daos_config from scratch -mv $${daos_config} $${daos_config}.orig - -exclude_fabric_ifaces="" -# Get names of network interfaces not in first PCI slot -# The first PCI slot is a standard network adapter while remaining interfaces -# are typically network cards dedicated to GPU or workload communication -if [[ "$${OS_ID}" == "debian" ]] || [[ "$${OS_ID}" = "ubuntu" ]]; then - extra_interfaces=$(find /sys/class/net/ -not -name 'enp0s*' -regextype posix-extended -regex '.*/enp[0-9]+s.*' -printf '"%f"\n' | paste -s -d ',') -elif [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; then - extra_interfaces=$(find /sys/class/net/ -not -name eth0 -regextype posix-extended -regex '.*/eth[0-9]+' -printf '"%f"\n' | paste -s -d ',') -fi - -cat > $daos_config </etc/systemd/system/"$${service_name}" </global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "import_gcs_bucket_uri" { - description = "The name of the GCS bucket to import data from to parallelstore." - type = string - default = null -} - -variable "import_destination_path" { - description = "The name of local path to import data on parallelstore instance from GCS bucket." - type = string - default = null -} - -variable "file_stripe" { - description = "The parallelstore stripe level for files." - type = string - default = null - validation { - condition = var.file_stripe == null ? true : contains([ - "FILE_STRIPE_LEVEL_UNSPECIFIED", - "FILE_STRIPE_LEVEL_MIN", - "FILE_STRIPE_LEVEL_BALANCED", - "FILE_STRIPE_LEVEL_MAX", - ], var.file_stripe) - error_message = "var.file_stripe must be set to \"FILE_STRIPE_LEVEL_UNSPECIFIED\", \"FILE_STRIPE_LEVEL_MIN\", \"FILE_STRIPE_LEVEL_BALANCED\", or \"FILE_STRIPE_LEVEL_MAX\"" - } -} - -variable "directory_stripe" { - description = "The parallelstore stripe level for directories." - type = string - default = null - validation { - condition = var.directory_stripe == null ? true : contains([ - "DIRECTORY_STRIPE_LEVEL_UNSPECIFIED", - "DIRECTORY_STRIPE_LEVEL_MIN", - "DIRECTORY_STRIPE_LEVEL_BALANCED", - "DIRECTORY_STRIPE_LEVEL_MAX", - ], var.directory_stripe) - error_message = "var.directory_stripe must be set to \"DIRECTORY_STRIPE_LEVEL_UNSPECIFIED\", \"DIRECTORY_STRIPE_LEVEL_MIN\", \"DIRECTORY_STRIPE_LEVEL_BALANCED\", or \"DIRECTORY_STRIPE_LEVEL_MAX\"" - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/versions.tf b/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/versions.tf deleted file mode 100644 index 174b5281e4..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/parallelstore/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.13.0" - } - - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/README.md b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/README.md deleted file mode 100644 index 47cf1518a1..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/README.md +++ /dev/null @@ -1,192 +0,0 @@ -## Description - -This module defines a file-system that already exists (i.e. it does not create -a new file system) in a way that can be shared with other modules. This allows -a compute VM to mount a filesystem that is not part of the current deployment -group. - -The pre-existing network storage can be referenced in the same way as any Cluster -Toolkit supported file-system such as [filestore](../filestore/README.md). - -For more information on network storage options in the Cluster Toolkit, see -the extended [Network Storage documentation](../../../docs/network_storage.md). - -### Example - -```yaml -- id: homefs - source: modules/file-system/pre-existing-network-storage - settings: - server_ip: ## Set server IP here ## - remote_mount: nfsshare - local_mount: /home - fs_type: nfs -``` - -This creates a pre-existing-network-storage module in terraform at the -provided IP in `server_ip` of type nfs that will be mounted at `/home`. Note -that the `server_ip` must be known before deployment. - -The following is an example of using `pre-existing-network-storage` with a GCS -bucket: - -```yaml -- id: data-bucket - source: modules/file-system/pre-existing-network-storage - settings: - remote_mount: my-bucket-name - local_mount: /data - fs_type: gcsfuse - mount_options: defaults,_netdev,implicit_dirs -``` - -The `implicit_dirs` mount option allows object paths to be treated as if they -were directories. This is important when working with files that were created by -another source, but there may have performance impacts. The `_netdev` mount option -denotes that the storage device requires network access. - -The following is an example of using `pre-existing-network-storage` with the `lustre` -filesystem: - -```yaml -- id: lustrefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: lustre - server_ip: 192.168.227.11@tcp - local_mount: /scratch - remote_mount: /exacloud -``` - -Note the use of the MGS NID (Network ID) in the `server_ip` field - in -particular, note the `@tcp` suffix. - -The following is an example of using `pre-existing-network-storage` with the -`managed_lustre` filesystem: - -```yaml -- id: lustrefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: managed_lustre - server_ip: 192.168.227.11@tcp - local_mount: /scratch - remote_mount: /mg_lustre -``` - -This is similar to the `lustre` filesystem, with the exception that it connects -with a managed Lustre instance hosted by GCP. Currently only Rocky 8 and -Ubuntu 20.04 and Ubuntu 22.04 are supported. - -The following is an example of using `pre-existing-network-storage` with the `daos` -filesystem. In order to use existing `parallelstore` instance, `fs_type` needs to be -explicitly mentioned in blueprint. The `remote_mount` option refers to `access_points` -for `parallelstore` instance. - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 -``` - -Parallelstore supports additional options for its mountpoints under `parallelstore_options` setting. -Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 - parallelstore_options: - daos_agent_config: | - credential_config: - cache_expiration: 1m -``` - -Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 - parallelstore_options: - dfuse_environment: - D_LOG_FILE: /tmp/client.log - D_APPEND_PID_TO_LOG: 1 - D_LOG_MASK: debug -``` - -### Mounting - -For the `fs_type` listed below, this module will provide `client_install_runner` -and `mount_runner` outputs. These can be used to create a startup script to -mount the network storage system. - -Supported `fs_type`: - -- nfs -- lustre -- managed_lustre -- gcsfuse -- daos - -[scripts/mount.sh](./scripts/mount.sh) is used as the contents of -`mount_runner`. This script will update `/etc/fstab` and mount the network -storage. This script will fail if the specified `local_mount` is already being -used by another entry in `/etc/fstab`. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. - -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## License - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [fs\_type](#input\_fs\_type) | Type of file system to be mounted (e.g., nfs, lustre) | `string` | `"nfs"` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | -| [managed\_lustre\_options](#input\_managed\_lustre\_options) | Managed Lustre specific options:
gke\_support\_enabled (bool, default = false)
Note: gke\_support\_enabled does not work with Slurm, the Slurm image must be built with
the correct compatibility. | 
object({
    gke_support_enabled = optional(bool, false)
  })
| `{}` | no | -| [mount\_options](#input\_mount\_options) | Options describing various aspects of the file system. Consider adding setting to 'defaults,\_netdev,implicit\_dirs' when using gcsfuse. | `string` | `"defaults,_netdev"` | no | -| [parallelstore\_options](#input\_parallelstore\_options) | Parallelstore specific options | 
object({
    daos_agent_config = optional(string, "")
    dfuse_environment = optional(map(string), {})
  })
| `{}` | no | -| [remote\_mount](#input\_remote\_mount) | Remote FS name or export. This is the exported directory for nfs, fs name for lustre, and bucket name (without gs://) for gcsfuse. | `string` | n/a | yes | -| [server\_ip](#input\_server\_ip) | The device name as supplied to fs-tab, excluding remote fs-name(for nfs, that is the server IP, for lustre [:]). This can be omitted for gcsfuse. | `string` | `""` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use file system. | -| [mount\_runner](#output\_mount\_runner) | Runner that mounts the file system. | -| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | - diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf deleted file mode 100644 index 203b6dfdac..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a remote network storage to be mounted by fs-tab." - value = { - server_ip = var.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = var.mount_options - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } -} - -locals { - # Update remote mount to include a slash if the fs_type requires one to exist - remote_mount_with_slash = length(regexall("^/.*", var.remote_mount)) > 0 ? ( - var.remote_mount - ) : format("/%s", var.remote_mount) - remote_mount = contains(local.mount_vanilla_supported_fstype, local.fs_type) ? ( - local.remote_mount_with_slash - ) : var.remote_mount - - ml_gke_support_enabled = coalesce(try(var.managed_lustre_options.gke_support_enabled, false), false) - - # Collapse fs_type lustre and managed lustre for most uses, only needs to be - # different for client installation - fs_type = strcontains(var.fs_type, "lustre") ? "lustre" : var.fs_type - - # Client Install - ddn_lustre_client_install_script = templatefile( - "${path.module}/templates/ddn_exascaler_luster_client_install.tftpl", - { - server_ip = split("@", var.server_ip)[0] - remote_mount = local.remote_mount - local_mount = var.local_mount - } - ) - managed_lustre_client_install_script = file("${path.module}/scripts/install-managed-lustre-client.sh") - nfs_client_install_script = file("${path.module}/scripts/install-nfs-client.sh") - gcs_fuse_install_script = file("${path.module}/scripts/install-gcs-fuse.sh") - daos_client_install_script = file("${path.module}/scripts/install-daos-client.sh") - - install_scripts = { - "lustre" = local.ddn_lustre_client_install_script - "managed_lustre" = local.managed_lustre_client_install_script - "nfs" = local.nfs_client_install_script - "gcsfuse" = local.gcs_fuse_install_script - "daos" = local.daos_client_install_script - } - - client_install_runner = { - "type" = "shell" - "content" = lookup(local.install_scripts, var.fs_type, "echo 'skipping: client_install_runner not yet supported for ${var.fs_type}'") - "destination" = "install_filesystem_client${replace(var.local_mount, "/", "_")}.sh" - "args" = local.ml_gke_support_enabled ? "1" : "" - } - - mount_vanilla_supported_fstype = ["lustre", "nfs"] - mount_runner_vanilla = { - "type" = "shell" - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"${var.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" - "content" = ( - contains(local.mount_vanilla_supported_fstype, local.fs_type) ? - file("${path.module}/scripts/mount.sh") : - "echo 'skipping: mount_runner not yet supported for ${var.fs_type}'" - ) - } - gcsbucket = trimprefix(var.remote_mount, "gs://") - mount_runner_gcsfuse = { - "type" = "shell" - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"not-used\" \"${local.gcsbucket}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" - "content" = file("${path.module}/scripts/mount.sh") - } - - mount_runner_daos = { - "type" = "shell" - "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { - access_points = var.remote_mount - daos_agent_config = var.parallelstore_options.daos_agent_config - dfuse_environment = var.parallelstore_options.dfuse_environment - local_mount = var.local_mount - # avoid passing "--" as mount option to dfuse - mount_options = length(var.mount_options) == 0 ? "" : join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) - }) - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - } - - mount_scripts = { - "lustre" = local.mount_runner_vanilla - "nfs" = local.mount_runner_vanilla - "gcsfuse" = local.mount_runner_gcsfuse - "daos" = local.mount_runner_daos - } - - mount_runner = lookup(local.mount_scripts, local.fs_type, local.mount_runner_vanilla) -} - -output "client_install_runner" { - description = "Runner that performs client installation needed to use file system." - value = local.client_install_runner -} - -output "mount_runner" { - description = "Runner that mounts the file system." - value = local.mount_runner -} diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh deleted file mode 100644 index e96eadb56a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || - { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 -fi - -if [ -x /bin/daos ]; then - echo "DAOS already installed" - daos version -else - # Install the DAOS client library - # The following commands should be executed on each client vm. - ## For Rocky linux 8 / RedHat 8. - if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then - # 1) Add the Parallelstore package repository - cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF - [parallelstore-v2-6-el${OS_VERSION_MAJOR}] - name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 - baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - - ## TODO: Remove disable automatic update script after issue is fixed. - if [ -x /usr/bin/google_disable_automatic_updates ]; then - /usr/bin/google_disable_automatic_updates - fi - dnf clean all - dnf makecache - - # 2) Install daos-client - dnf install -y epel-release # needed for capstone - dnf install -y daos-client - - # 3) Upgrade libfabric - dnf upgrade -y libfabric - - # For Ubuntu 22.04 and debian 12, - elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then - # shellcheck disable=SC2034 - DEBIAN_FRONTEND=noninteractive - - # 1) Add the Parallelstore package repository - curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg - echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list - - apt-get update - - # 2) Install daos-client - apt-get install -y daos-client - - # 3) Create daos_agent.service (comes pre-installed with RedHat) - if ! getent passwd daos_agent >/dev/null 2>&1; then - useradd daos_agent - fi - cat >/etc/systemd/system/daos_agent.service <<-EOF - [Unit] - Description=DAOS Agent - StartLimitIntervalSec=60 - Wants=network-online.target - After=network-online.target - - [Service] - Type=notify - User=daos_agent - Group=daos_agent - RuntimeDirectory=daos_agent - RuntimeDirectoryMode=0755 - ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml - StandardOutput=journal - StandardError=journal - Restart=always - RestartSec=10 - LimitMEMLOCK=infinity - LimitCORE=infinity - StartLimitBurst=5 - - [Install] - WantedBy=multi-user.target - EOF - else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - fi -fi - -exit 0 diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh deleted file mode 100644 index f8a990260b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -if [ ! "$(which gcsfuse)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then - tee /etc/yum.repos.d/gcsfuse.repo >/dev/null <>/etc/modprobe.d/lnet.conf - fi -fi - -if grep -q lustre /proc/filesystems; then - echo "Skipping managed lustre client install as it is already supported" - exit 0 -fi - -# Get distro information -. /etc/os-release -DIST="NA" -if [[ $NAME == *"Ubuntu"* ]]; then - if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then - DIST="Ubuntu" - fi -elif [[ $NAME == *"Rocky"* ]]; then - if [[ $VERSION_ID == "8"* ]]; then - DIST="Rocky" - fi -fi - -if [[ ${DIST} == "Ubuntu" ]]; then - KEY_LOC=/etc/apt/keyrings - KEY_NAME=gcp-ar-repo.gpg - # Download new repo key - mkdir -p "${KEY_LOC}" - wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null - - # Set up apt repo - echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list - - # Install modules - apt update - apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) -elif [[ ${DIST} == "Rocky" ]]; then - # Set up yum repo - touch /etc/yum.repos.d/artifact-registry.repo - tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF - [lustre-client-rocky-8] - name=lustre-client-rocky-8 - baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - # Install modules - yum makecache - yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client -fi - -if [[ $DIST != "NA" ]]; then - # Load the new lustre client module - modprobe lustre -fi diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl b/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl deleted file mode 100644 index f5f0291e85..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh - -# Copyright 2022 DataDirect Networks -# Modifications Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Prior Art: https://github.com/DDNStorage/exascaler-cloud-terraform/blob/78deadbb2c1fa7e4603cf9605b0f7d1782117954/gcp/templates/client-script.tftpl - -# install new EXAScaler Cloud clients: -# all instances must be in the same zone -# and connected to the same network and subnet -# to set up EXAScaler Cloud filesystem on a new client instance, -# run the following commands on the client with root privileges: -set -e -if [[ ! -z $(cat /proc/filesystems | grep lustre) ]]; then - echo "Skipping lustre client install as it is already supported" - exit 0 -fi - -cat >/etc/esc-client.conf< $daos_config </etc/systemd/system/"$${service_name}" < -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string
    count = number
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [guest\_accelerator](#output\_guest\_accelerator) | Sanitized list of the type and count of accelerator cards attached to the instance. | -| [machine\_type\_guest\_accelerator](#output\_machine\_type\_guest\_accelerator) | List of the type and count of accelerator cards attached to the specified machine type. | - diff --git a/deletion-test/cluster/modules/embedded/modules/internal/gpu-definition/main.tf b/deletion-test/cluster/modules/embedded/modules/internal/gpu-definition/main.tf deleted file mode 100644 index f0861cddc9..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/gpu-definition/main.tf +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "machine_type" { - description = "Machine type to use for the instance creation" - type = string -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string - count = number - gpu_driver_installation_config = optional(object({ - gpu_driver_version = string - }), { gpu_driver_version = "DEFAULT" }) - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = string - max_shared_clients_per_gpu = number - })) - })) - default = [] - nullable = false -} - -locals { - # example state; terraform will ignore diffs if last element of URL matches - # guest_accelerator = [ - # { - # count = 1 - # type = "https://www.googleapis.com/compute/beta/projects/PROJECT/zones/ZONE/acceleratorTypes/nvidia-tesla-a100" - # }, - # ] - accelerator_machines = { - "a2-highgpu-1g" = { type = "nvidia-tesla-a100", count = 1 }, - "a2-highgpu-2g" = { type = "nvidia-tesla-a100", count = 2 }, - "a2-highgpu-4g" = { type = "nvidia-tesla-a100", count = 4 }, - "a2-highgpu-8g" = { type = "nvidia-tesla-a100", count = 8 }, - "a2-megagpu-16g" = { type = "nvidia-tesla-a100", count = 16 }, - "a2-ultragpu-1g" = { type = "nvidia-a100-80gb", count = 1 }, - "a2-ultragpu-2g" = { type = "nvidia-a100-80gb", count = 2 }, - "a2-ultragpu-4g" = { type = "nvidia-a100-80gb", count = 4 }, - "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, - "a3-highgpu-1g" = { type = "nvidia-h100-80gb", count = 1 }, - "a3-highgpu-2g" = { type = "nvidia-h100-80gb", count = 2 }, - "a3-highgpu-4g" = { type = "nvidia-h100-80gb", count = 4 }, - "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, - "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, - "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, - "a4-highgpu-8g-lowmem" = { type = "nvidia-b200", count = 8 }, - "a4-highgpu-8g" = { type = "nvidia-b200", count = 8 }, - "a4x-highgpu-4g" = { type = "nvidia-gb200", count = 4 }, - "a4x-highgpu-4g-nolssd" = { type = "nvidia-gb200", count = 4 }, - "g2-standard-4" = { type = "nvidia-l4", count = 1 }, - "g2-standard-8" = { type = "nvidia-l4", count = 1 }, - "g2-standard-12" = { type = "nvidia-l4", count = 1 }, - "g2-standard-16" = { type = "nvidia-l4", count = 1 }, - "g2-standard-24" = { type = "nvidia-l4", count = 2 }, - "g2-standard-32" = { type = "nvidia-l4", count = 1 }, - "g2-standard-48" = { type = "nvidia-l4", count = 4 }, - "g2-standard-96" = { type = "nvidia-l4", count = 8 }, - } - generated_guest_accelerator = try([local.accelerator_machines[var.machine_type]], []) - - # Select in priority order: - # (1) var.guest_accelerator if not empty - # (2) local.generated_guest_accelerator if not empty - # (3) default to empty list if both are empty - guest_accelerator = try(coalescelist(var.guest_accelerator, local.generated_guest_accelerator), []) -} - -output "guest_accelerator" { - description = "Sanitized list of the type and count of accelerator cards attached to the instance." - value = local.guest_accelerator -} - -output "machine_type_guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the specified machine type." - value = local.generated_guest_accelerator -} - -terraform { - required_version = ">= 1.3" -} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/README.md b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/README.md deleted file mode 100644 index 21746fe0d8..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/README.md +++ /dev/null @@ -1,30 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [disk\_type](#input\_disk\_type) | The disk type to validate. | `string` | n/a | yes | -| [machine\_type](#input\_machine\_type) | The machine type to validate. | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/main.tf b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/main.tf deleted file mode 100644 index d89d7edfec..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/main.tf +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -check "disk_type_c4_compatibility" { - assert { - condition = !(can(regex("^c4-", var.machine_type)) && var.disk_type == "pd-ssd") - error_message = "The C4 machine series does not support pd-ssd. Please use hyperdisk-balanced or another compatible disk type." - } -} - - -check "disk_type_c2_compatibility" { - assert { - condition = !(can(regex("^c2-", var.machine_type)) && can(regex("hyperdisk", var.disk_type))) - error_message = "The C2 machine series does not support Hyperdisk as a boot disk. Please use a compatible disk type like pd-ssd, pd-standard, or pd-balanced." - } -} - - -check "disk_type_pd_extreme_compatibility" { - assert { - condition = var.disk_type != "pd-extreme" || can(regex("^(m1-|m2-|m3-|n2-|n2d-)", var.machine_type)) - error_message = "pd-extreme disks are only supported for M1, M2, M3, N2, and N2D machine series." - } -} - - -check "disk_type_hyperdisk_extreme_compatibility" { - assert { - condition = var.disk_type != "hyperdisk-extreme" || can(regex("^(c3-|m1-|m3-|n2-)", var.machine_type)) - error_message = "hyperdisk-extreme disks are only supported for C3, M1, M3, and N2 machine series." - } -} - - -check "disk_type_hyperdisk_throughput_compatibility" { - assert { - condition = var.disk_type != "hyperdisk-throughput" || can(regex("^(c3-|c3d-|n4-|n2-|n2d-|n1-|t2d-|m1-)", var.machine_type)) - error_message = "hyperdisk-throughput disks are only supported for C3, C3D, N4, N2, N2D, N1, T2D, and M1 machine series." - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/variables.tf b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/variables.tf deleted file mode 100644 index 23478051b3..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/variables.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "machine_type" { - type = string - description = "The machine type to validate." -} - -variable "disk_type" { - type = string - description = "The disk type to validate." -} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/versions.tf b/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/versions.tf deleted file mode 100644 index 4702005614..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/instance_validations/versions.tf +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 0.15.0" -} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/README.md b/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/README.md deleted file mode 100644 index 8aa9270a0a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/README.md +++ /dev/null @@ -1,54 +0,0 @@ - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | -| [google-beta](#requirement\_google-beta) | >= 6.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google-beta](#provider\_google-beta) | >= 6.0.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_network_attachment.self](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_network_attachment) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [connection\_preference](#input\_connection\_preference) | The connection preference of service attachment. | `string` | `"ACCEPT_AUTOMATIC"` | no | -| [name](#input\_name) | Name of the resource. Provided by the client when the resource is created | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The ID of the project in which the resource belongs. | `string` | n/a | yes | -| [region](#input\_region) | Region where the network attachment resides | `string` | n/a | yes | -| [subnetwork\_self\_links](#input\_subnetwork\_self\_links) | An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment. | `list(string)` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [self\_link](#output\_self\_link) | Server-defined URL for the resource. | - diff --git a/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/main.tf b/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/main.tf deleted file mode 100644 index bbbece7085..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -variable "connection_preference" { - type = string - description = "The connection preference of service attachment." - default = "ACCEPT_AUTOMATIC" -} - -variable "subnetwork_self_links" { - type = list(string) - description = " An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment." -} - -variable "name" { - type = string - description = "Name of the resource. Provided by the client when the resource is created" -} - -variable "project_id" { - type = string - description = "The ID of the project in which the resource belongs." -} - -variable "region" { - type = string - description = "Region where the network attachment resides" -} - - -resource "google_compute_network_attachment" "self" { - provider = google-beta - - project = var.project_id - region = var.region - name = var.name - connection_preference = var.connection_preference - subnetworks = var.subnetwork_self_links -} - - -output "self_link" { - value = google_compute_network_attachment.self.self_link - description = "Server-defined URL for the resource." -} - -terraform { - required_version = ">= 0.15.0" - - required_providers { - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.0.0" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/metadata.yaml deleted file mode 100644 index e80fc96b9c..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/network-attachment/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/README.md b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/README.md deleted file mode 100644 index 610d82c1b9..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/README.md +++ /dev/null @@ -1,85 +0,0 @@ -## Description - -This is an internal helper module designed to encapsulate and centralize all hardware-specific logic for Google Cloud TPUs. It is intended to be called by parent modules like `gke-node-pool` to determine if a node pool is TPU-based and to retrieve its specific attributes. - -This module's primary responsibilities are: - -* Reliably detect if a node pool is for TPUs by checking its `placement_policy`. -* Determine the correct GKE `tpu-accelerator` label based on the machine type family. -* Determine the `number of chips per node` based on the specific machine type. -* Generate the standard **Kubernetes taint** that should be applied to TPU nodes. - -This follows the same design pattern as the `gpu-definition` internal module, promoting a clean separation of concerns within the gke-node-pool module. - -## Usage - -This module is not intended for direct use in a blueprint. It should be called from a parent module like `gke-node-pool`. - -```yaml -module "tpu" { - source = "../../internal/tpu-definition" - - # Pass the parent module's variables to this module - machine_type = var.machine_type - placement_policy = var.placement_policy -} - -# Example of consuming the module's outputs in the parent module -locals { - # The tpu_taint is then used in the node_config's dynamic "taint" block - tpu_taint = module.tpu.tpu_taint -} -``` - -## License - - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [machine\_type](#input\_machine\_type) | The machine type of the node pool. | `string` | n/a | yes | -| [placement\_policy](#input\_placement\_policy) | The placement policy for the node pool. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [is\_tpu](#output\_is\_tpu) | Boolean value indicating if the node pool is for TPUs. | -| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | -| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | -| [tpu\_taint](#output\_tpu\_taint) | A list containing the standard TPU taint object if the node pool is for TPUs. | -| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | - diff --git a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/main.tf b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/main.tf deleted file mode 100644 index c8ee417d71..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/main.tf +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # Determine if this is a TPU node pool by checking if the machine_type exists in our authoritative map of TPU machine types. - is_tpu = contains(keys(local.tpu_chip_count_map), var.machine_type) - - tpu_taint = local.is_tpu ? [{ - key = "google.com/tpu" - value = "present" - effect = "NO_SCHEDULE" - }] : [] - - # Map of machine prefixes to GKE accelerator labels. - tpu_accelerator_map = { - "ct4p" = "tpu-v4-podslice" # TPU v4 - "ct5lp" = "tpu-v5-lite-podslice" # TPU v5e - "ct5p" = "tpu-v5p-slice" # TPU v5p - "ct6e" = "tpu-v6e-slice" # TPU v6e - "tpu7x" = "tpu7x" # TPU v7x - } - - # Map specific GCE machine types to the number of TPU chips per node (VM). - # The machine-type map must be updated to reflect new TPU releases with reference to public documentation: https://docs.cloud.google.com/tpu/docs/intro-to-tpu - tpu_chip_count_map = { - # v4 - ct4p - "ct4p-hightpu-4t" = 4 - - # v5e - ct5lp - "ct5lp-hightpu-1t" = 1 - "ct5lp-hightpu-4t" = 4 - "ct5lp-hightpu-8t" = 8 - - # v5p - ct5p - "ct5p-hightpu-1t" = 1 - "ct5p-hightpu-2t" = 2 - "ct5p-hightpu-4t" = 4 - - # v6e - ct6e - "ct6e-standard-1t" = 1 - "ct6e-standard-4t" = 4 - "ct6e-standard-8t" = 8 - - # v7x - tpu7x - "tpu7x-standard-4t" = 4 - } - - # Robustly extract the machine family prefix (e.g., "ct6e"). - tpu_machine_family = local.is_tpu ? element(split("-", var.machine_type), 0) : "" - tpu_accelerator_type = local.is_tpu ? lookup(local.tpu_accelerator_map, local.tpu_machine_family, null) : null - tpu_chips_per_node = local.is_tpu ? lookup(local.tpu_chip_count_map, var.machine_type, null) : null -} - -terraform { - required_version = ">= 1.3" -} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/outputs.tf b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/outputs.tf deleted file mode 100644 index fa3c21fa34..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/outputs.tf +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "is_tpu" { - description = "Boolean value indicating if the node pool is for TPUs." - value = local.is_tpu -} - -output "tpu_accelerator_type" { - description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." - value = local.tpu_accelerator_type -} - -output "tpu_topology" { - description = "The topology of the TPU slice (e.g., '4x4')." - value = local.is_tpu ? var.placement_policy.tpu_topology : null -} - -output "tpu_chips_per_node" { - description = "The number of TPU chips on each node in the pool." - value = local.tpu_chips_per_node -} - -output "tpu_taint" { - description = "A list containing the standard TPU taint object if the node pool is for TPUs." - value = local.tpu_taint -} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/variables.tf b/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/variables.tf deleted file mode 100644 index 254488c02d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/tpu-definition/variables.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "machine_type" { - description = "The machine type of the node pool." - type = string -} - -variable "placement_policy" { - description = "The placement policy for the node pool." - type = object({ - type = string - name = optional(string) - tpu_topology = optional(string) - }) -} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/README.md b/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/README.md deleted file mode 100644 index aefac9d187..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/README.md +++ /dev/null @@ -1,56 +0,0 @@ - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_network_peering.peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [export\_custom\_routes](#input\_export\_custom\_routes) | (Optional) Whether to export the custom routes to the peer network. Defaults to false. | `bool` | `null` | no | -| [import\_custom\_routes](#input\_import\_custom\_routes) | (Optional) Whether to import the custom routes from the peer network. Defaults to false. | `bool` | `null` | no | -| [import\_subnet\_routes\_with\_public\_ip](#input\_import\_subnet\_routes\_with\_public\_ip) | (Optional) Whether subnet routes with public IP range are imported. | `bool` | `null` | no | -| [name](#input\_name) | Name of the peering. | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | The primary network of the peering. | `string` | n/a | yes | -| [peer\_network\_self\_link](#input\_peer\_network\_self\_link) | The peer network in the peering. The peer network may belong to a different project. | `string` | n/a | yes | -| [stack\_type](#input\_stack\_type) | (Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [peering\_name](#output\_peering\_name) | Name of the peering. | - diff --git a/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/main.tf b/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/main.tf deleted file mode 100644 index 386fa9377b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/main.tf +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "name" { - type = string - description = "Name of the peering." -} - -variable "network_self_link" { - type = string - description = "The primary network of the peering." -} - -variable "peer_network_self_link" { - type = string - description = "The peer network in the peering. The peer network may belong to a different project." -} - -variable "export_custom_routes" { - type = bool - description = "(Optional) Whether to export the custom routes to the peer network. Defaults to false." - default = null -} - -variable "import_custom_routes" { - type = bool - description = "(Optional) Whether to import the custom routes from the peer network. Defaults to false." - default = null -} - -variable "import_subnet_routes_with_public_ip" { - type = bool - description = "(Optional) Whether subnet routes with public IP range are imported. " - default = null -} - -variable "stack_type" { - type = string - description = "(Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. " - default = null -} - -resource "google_compute_network_peering" "peering" { - name = var.name - network = var.network_self_link - peer_network = var.peer_network_self_link - export_custom_routes = var.export_custom_routes - import_custom_routes = var.import_custom_routes - import_subnet_routes_with_public_ip = var.import_subnet_routes_with_public_ip - stack_type = var.stack_type -} - -output "peering_name" { - value = google_compute_network_peering.peering.name - description = "Name of the peering." -} - -terraform { - required_version = ">= 0.15.0" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/metadata.yaml deleted file mode 100644 index e80fc96b9c..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/internal/vpc_peering/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/README.md b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/README.md deleted file mode 100644 index d7054eb725..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/README.md +++ /dev/null @@ -1,244 +0,0 @@ -## Description - -This module simplifies the following functionality: - -* Applying Kubernetes manifests to GKE clusters: It provides flexible options for specifying manifests, allowing you to either directly embed them as strings content or reference them from URLs, files, templates, or entire .yaml and .tftpl files in directories. -* Deploying commonly used infrastructure like [Kueue](https://kueue.sigs.k8s.io/docs/) or [Jobset](https://jobset.sigs.k8s.io/docs/). - -> Note: Kueue can work with a variety of frameworks out of the box, find them [here](https://kueue.sigs.k8s.io/docs/tasks/run/) - -### Explanation - -* **Manifest:** - * **Raw String:** Specify manifests directly within the module configuration using the `content: manifest_body` format. - * **File/Template/Directory Reference:** Set `source` to the path to: - * A single URL to a manifest file. Ex.: `https://github.com/.../myrepo/manifest.yaml`. - - > **Note:** Applying from a URL has important limitations. Please review the [Considerations & Callouts for Applying from URLs](#applying-manifests-from-urls-considerations--callouts) section below. - * A single local YAML manifest file (`.yaml`). Ex.: `./manifest.yaml`. - * A template file (`.tftpl`) to generate a manifest. Ex.: `./template.yaml.tftpl`. You can pass the variables to format the template file in `template_vars`. - * A directory containing multiple YAML or template files. Ex: `./manifests/`. You can pass the variables to format the template files in `template_vars`. - -#### Manifest Example - -```yaml -- id: existing-gke-cluster - source: modules/scheduler/pre-existing-gke-cluster - settings: - project_id: $(vars.project_id) - cluster_name: my-gke-cluster - region: us-central1 - -- id: kubectl-apply - source: modules/management/kubectl-apply - use: [existing-gke-cluster] - settings: - - content: | - apiVersion: v1 - kind: Namespace - metadata: - name: my-namespace - - source: "https://github.com/kubernetes-sigs/jobset/releases/download/v0.6.0/manifests.yaml" - - source: $(ghpc_stage("manifests/configmap1.yaml")) - - source: $(ghpc_stage("manifests/configmap2.yaml.tftpl")) - template_vars: {name: "dev-config", public: "false"} - - source: $(ghpc_stage("manifests"))/ - template_vars: {name: "dev-config", public: "false"} -``` - -#### Pre-build infrastructure Example - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml")) - jobset: - install: true -``` - -The `config_path` field in `kueue` installation accepts a template file, too. You will need to provide variables for the template using `config_template_vars` field. - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) - config_template_vars: {name: "dev-config", public: "false"} - jobset: - install: true -``` - -You can specify a particular kueue version that you would like to use using the `version` flag. By default, we recommend customers to [use v0.10.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L68). You can find the list of supported kueue versions [here](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L18). - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - version: v0.10.0 - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) - config_template_vars: {name: "dev-config", public: "false"} - jobset: - install: true -``` - -> **_NOTE:_** -> -> The `project_id` and `region` settings would be inferred from the deployment variables of the same name, but they are included here for clarity. -> -> Terraform may apply resources in parallel, leading to potential dependency issues. If a resource's dependencies aren't ready, it will be applied again up to 15 times. - -## Callouts - -### Applying Manifests from URLs: Considerations & Callouts - -While this module supports applying manifests directly from remote `http://` or `https://` URLs, this method introduces complexities not present when using local files. For production environments, we recommend sourcing manifests from local paths or a version-controlled Git repository. Moreover, this method will be deprecated soon. Hence we recommend to use other methods to source manifests. - -If you choose to use the URL method, be aware of the following potential issues and their solutions. - -#### **1. Apply Order and Race Conditions** - -The module applies manifests from the `apply_manifests` list in parallel. This can create a **race condition** if one manifest depends on another. The most common example is applying a manifest with custom resources (like a `ClusterQueue`) at the same time as the manifest that defines it (the `CustomResourceDefinition` or CRD). - -There is **no guarantee** that the CRD will be applied before the resource that uses it. This can lead to non-deterministic deployment failures with errors like: - -```Error: resource [kueue.x-k8s.io/v1beta1/ClusterQueue] isn't valid for cluster``` - -##### **Recommended Workaround: Two-Stage Apply** - -To ensure a reliable deployment, you must manually enforce the correct order of operations. - -1. **Initial Deployment:** In your blueprint, include **only** the manifest(s) containing the `CustomResourceDefinition` (CRD) resources in the `apply_manifests` list. - - *Example `settings` for the first run:* - - ```yaml - settings: - apply_manifests: - # This manifest contains the CRDs for Kueue - - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" - server_side_apply: true - ``` - -2. **Run the deployment** (`gcluster deploy` or `terraform apply`). - -3. **Second Deployment:** Once the first apply is successful, **add** the manifests containing your custom resources (like `ClusterQueue`, `LocalQueue`) to the list. - - *Example `settings` for the second run:* - - ```yaml - settings: - apply_manifests: - # The CRD manifest is still present - - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" - server_side_apply: true - - # Now, add your configuration manifest - - source: "https://gist.githubusercontent.com/YourUser/..." # Your configuration URL - server_side_apply: true - ``` - -4. **Run the deployment command again.** Since the CRDs are now guaranteed to exist in the cluster, this second apply will succeed reliably. - -#### **2. Large Manifests (CRDs)** - -* **Issue:** Applying very large manifests can fail with a `metadata.annotations: Too long` error. -* **Solution:** Enable Server-Side Apply by setting `server_side_apply: true` for the manifest entry. - -#### **3. Conflicts on Re-application** - -* **Issue:** Re-running a deployment after a partial failure can cause server-side apply field manager `conflicts`. -* **Solution:** Forcibly take ownership of the resource fields by setting `force_conflicts: true`. - -#### **4. Terraform Template Files (`.tftpl`)** - -* **Limitation:** This module **cannot** render a template file (`.tftpl`) when sourced from a remote URL. -* **Workaround:** You must render the template into a pure YAML file locally, host that rendered file at a URL, and provide the URL of the rendered file in your blueprint. - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 7.2 | -| [helm](#requirement\_helm) | ~> 2.17 | -| [http](#requirement\_http) | ~> 3.0 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 7.2 | -| [http](#provider\_http) | ~> 3.0 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [configure\_kueue](#module\_configure\_kueue) | ./kubectl | n/a | -| [install\_gib](#module\_install\_gib) | ./kubectl | n/a | -| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | -| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | -| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | -| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | -| [kubectl\_apply\_manifests](#module\_kubectl\_apply\_manifests) | ./kubectl | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.gib_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.initial_gib_version](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.jobset_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.kueue_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | -| [http_http.manifest_from_url](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [apply\_manifests](#input\_apply\_manifests) | A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md).
NOTE: The `enable` input acts as a FF to apply a manifest or not. By default it is always set to `true`. | 
list(object({
    enable            = optional(bool, true)
    content           = optional(string, null)
    source            = optional(string, null)
    template_vars     = optional(map(any), null)
    server_side_apply = optional(bool, false)
    wait_for_rollout  = optional(bool, true)
  }))
| `[]` | no | -| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | -| [gib](#input\_gib) | Install the NCCL gIB plugin | 
object({
    install = bool
    path    = string
    template_vars = object({
      image   = optional(string, "us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib")
      version = string
      node_affinity = optional(any, {
        requiredDuringSchedulingIgnoredDuringExecution = {
          nodeSelectorTerms = [{
            matchExpressions = [{
              key      = "cloud.google.com/gke-gpu",
              operator = "In",
              values   = ["true"]
            }]
          }]
        }
      })
      accelerator_count = number
      max_unavailable   = optional(string, "50%")
    })
  })
| 
{
  "install": false,
  "path": "",
  "template_vars": {
    "accelerator_count": 0,
    "version": ""
  }
}
| no | -| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | -| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "0.10.1")
  })
| `{}` | no | -| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "0.13.3")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | -| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | -| [target\_architecture](#input\_target\_architecture) | The target architecture for the GKE nodes and gIB plugin (e.g., 'x86\_64' or 'arm64'). | `string` | `"x86_64"` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/README.md b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/README.md deleted file mode 100644 index 1957899617..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [helm](#provider\_helm) | ~> 2.17 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | -| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | -| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | -| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | -| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | -| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | -| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | -| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | -| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | -| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | -| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | -| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | -| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | -| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | -| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | -| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | -| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | -| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | -| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | -| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | -| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | -| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | -| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | -| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | -| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | -| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | -| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | -| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | -| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | -| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf deleted file mode 100644 index 8cc09bd3e2..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "helm_release" "apply_chart" { - # Required Identification - name = var.release_name - chart = var.chart_name - - # Chart Source & Version - repository = var.chart_repository - version = var.chart_version - devel = var.devel - - # Target Namespace - namespace = var.namespace - create_namespace = var.create_namespace - - # Values Configuration - values = var.values_yaml - - dynamic "set" { - for_each = var.set_values - content { - name = set.value.name - value = set.value.value - type = set.value.type - } - } - - # Installation/Upgrade Behavior - description = var.description - atomic = var.atomic - cleanup_on_fail = var.cleanup_on_fail - dependency_update = var.dependency_update - disable_crd_hooks = var.disable_crd_hooks - disable_openapi_validation = var.disable_openapi_validation - disable_webhooks = var.disable_webhooks - force_update = var.force_update - lint = var.lint - max_history = var.max_history - recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI - render_subchart_notes = var.render_subchart_notes - reset_values = var.reset_values - reuse_values = var.reuse_values - skip_crds = var.skip_crds - timeout = var.timeout - wait = var.wait - wait_for_jobs = var.wait_for_jobs - - # Verification & Credentials - keyring = var.keyring - pass_credentials = var.pass_credentials - verify = var.verify - - # Post Rendering - dynamic "postrender" { - # Only include the block if var.postrender is not null - for_each = var.postrender == null ? [] : [var.postrender] - content { - binary_path = postrender.value.binary_path - } - } - - # Lifecycle block (optional - generally avoid complex lifecycle in generic modules) - # lifecycle { - # ignore_changes = [] - # } -} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf deleted file mode 100644 index 04e8e214fc..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -# --- Required --- -variable "release_name" { - description = "Name of the Helm release." - type = string -} - -variable "chart_name" { - description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." - type = string -} - -# --- Chart Location & Version --- -variable "chart_repository" { - description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." - type = string - default = null -} - -variable "chart_version" { - description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." - type = string - default = null -} - -variable "devel" { - description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." - type = bool - default = false -} - -# --- Namespace --- -variable "namespace" { - description = "Kubernetes namespace to install the Helm release into." - type = string - default = "default" -} - -variable "create_namespace" { - description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." - type = bool - default = true # Common convenience setting -} - -# --- Values Customization --- -variable "values_yaml" { - description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." - type = list(string) - default = [] -} - -variable "set_values" { - description = "List of objects defining values to set ('helm install --set')." - type = list(object({ - name = string # Path to the value (e.g., 'service.type', 'replicaCount') - value = string # The value to set - type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') - })) - default = [] -} - -# --- Installation/Upgrade Behavior --- -variable "description" { - description = "Set an optional description for the Helm release." - type = string - default = null -} - -variable "atomic" { - description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." - type = bool - default = false -} - -variable "wait" { - description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." - type = bool - default = true # Often a good default for dependencies -} - -variable "wait_for_jobs" { - description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." - type = bool - default = false # Helm CLI default is false -} - -variable "timeout" { - description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." - type = number - default = 300 # 5 minutes (Helm CLI default) -} - -variable "cleanup_on_fail" { - description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." - type = bool - default = false -} - -variable "dependency_update" { - description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." - type = bool - default = false -} - -variable "disable_crd_hooks" { - description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." - type = bool - default = false -} - -variable "disable_openapi_validation" { - description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." - type = bool - default = false -} - -variable "disable_webhooks" { - description = "Prevent hooks from running ('helm install --no-hooks')." - type = bool - default = false -} - -variable "force_update" { - description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." - type = bool - default = false -} - -variable "lint" { - description = "Run the helm chart linter during the plan ('helm lint')." - type = bool - default = false -} - -variable "max_history" { - description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." - type = number - default = null # Terraform provider defaults to Helm's default (usually 10) -} - -variable "recreate_pods" { - description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." - type = bool - default = false -} - -variable "render_subchart_notes" { - description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." - type = bool - default = false -} - -variable "reset_values" { - description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." - type = bool - default = false -} - -variable "reuse_values" { - description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." - type = bool - default = false # Helm CLI default is false -} - -variable "skip_crds" { - description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." - type = bool - default = false -} - -# --- Verification & Credentials --- -variable "keyring" { - description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." - type = string - default = null # Defaults to Helm's default keyring location -} - -variable "pass_credentials" { - description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." - type = bool - default = false -} - -variable "verify" { - description = "Verify the package before installing it ('helm install --verify')." - type = bool - default = false -} - -# --- Advanced Rendering --- -variable "postrender" { - description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." - type = object({ - binary_path = string # Path to the post-renderer executable - }) - default = null # Disabled by default -} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf deleted file mode 100644 index 09d912e2c9..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml deleted file mode 100644 index 92fc1bca22..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# For referencing the original jobset helm chart values, pull the latest jobset chart version -# `helm pull oci://registry.k8s.io/jobset/charts/jobset --version=0.10.1` (latest helm chart version) - -controller: - # It ensures the Jobset pod(s) can be scheduled on GKE clusters where the - # system node pool uses the default "gke-managed-components" taint. - tolerations: - - key: "components.gke.io/gke-managed-components" - operator: "Equal" - value: "true" - effect: "NoSchedule" diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/README.md b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/README.md deleted file mode 100644 index 691f4dc34a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/README.md +++ /dev/null @@ -1,55 +0,0 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | - -## Providers - -| Name | Version | -|------|---------| -| [kubectl](#provider\_kubectl) | >= 1.7.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubectl_manifest.apply_doc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_path_documents.templates](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | -| [kubectl_path_documents.yamls](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | -| [force\_conflicts](#input\_force\_conflicts) | The force\_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller) | `bool` | `false` | no | -| [server\_side\_apply](#input\_server\_side\_apply) | Allow using kubectl server-side apply method. | `bool` | `false` | no | -| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `null` | no | -| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | -| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf deleted file mode 100644 index acf1d3c908..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - yaml_separator = "\n---" - - # This locals block processes manifest inputs from one of four methods, - # evaluated in order of precedence using coalesce. - - # --- METHOD 1: Direct Content Input --- - # Used when manifest content is passed directly as a string. - content_yaml_body = var.content - - # Fallback for safe path checking in subsequent methods. - null_safe_source = coalesce(var.source_path, " ") - - # --- METHOD 2: Single Local YAML File --- - # Used when var.source_path points to a local .yaml file. - yaml_file = length(regexall("\\.yaml(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null - yaml_file_content = local.yaml_file != null ? file(local.yaml_file) : null - - # --- METHOD 3: Single Local Template File --- - # Used when var.source_path points to a local .tftpl file. - template_file = length(regexall("\\.tftpl(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null - template_file_content = local.template_file != null ? templatefile(local.template_file, var.template_vars) : null - - # --- CONSOLIDATE & PROCESS --- - # Coalesce finds the first non-null content from the methods above. - yaml_body = coalesce(local.content_yaml_body, local.yaml_file_content, local.template_file_content, " ") - # Ensure only valid YAML is processed - # It explicitly tests if the content can be decoded before including it. - yaml_body_docs = compact(flatten([ - for doc in split(local.yaml_separator, local.yaml_body) : [ - for content in [trimspace(doc)] : ( - # Use a temporary local variable and can() to test for successful YAML decoding. - # This handles malformed documents (like comment blocks) which cause yamldecode() to fail. - can(yamldecode(content)) && length(yamldecode(content)) > 0 ? content : null - ) - ] - ])) - - # --- METHOD 4: Directory of Files --- - # If no content was found via the methods above AND the source path looks like a directory, - # we assume this is the desired method. The data blocks below will handle it. - directory = length(local.yaml_body_docs) == 0 && endswith(local.null_safe_source, "/") ? abspath(var.source_path) : null - - # --- FINAL AGGREGATION --- - # Combine documents from single-source methods and directory-scan methods into one list. - docs_list = concat(try(local.yaml_body_docs, []), try(data.kubectl_path_documents.yamls[0].documents, []), try(data.kubectl_path_documents.templates[0].documents, [])) - docs_map = tomap({ - for index, doc in local.docs_list : index => doc - }) -} - -data "kubectl_path_documents" "yamls" { - count = local.directory != null ? 1 : 0 - pattern = "${local.directory}/*.yaml" -} - -data "kubectl_path_documents" "templates" { - count = local.directory != null ? 1 : 0 - pattern = "${local.directory}/*.tftpl" - vars = var.template_vars -} - -resource "kubectl_manifest" "apply_doc" { - for_each = local.docs_map - yaml_body = each.value - server_side_apply = var.server_side_apply - wait_for_rollout = var.wait_for_rollout - force_conflicts = var.force_conflicts - - lifecycle { - precondition { - condition = !var.force_conflicts || var.server_side_apply - error_message = "The 'force_conflicts' variable can only be set to true when 'server_side_apply' is also true." - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf deleted file mode 100644 index 7bf34e089c..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "content" { - description = "The YAML body to apply to gke cluster." - type = string - default = null -} - -variable "source_path" { - description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." - type = string - default = null -} - -variable "template_vars" { - description = "The values to populate template file(s) with." - type = any - default = null -} - -variable "server_side_apply" { - description = "Allow using kubectl server-side apply method." - type = bool - default = false -} - -variable "wait_for_rollout" { - description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." - type = bool - default = true -} - -variable "force_conflicts" { - description = "The force_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller)" - type = bool - default = false -} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf deleted file mode 100644 index cce452239f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.7.0" - } - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml deleted file mode 100644 index 7c0bef7013..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# For referencing the original Kueue helm chart values, pull the latest helm chart version -# `helm pull oci://registry.k8s.io/kueue/charts/kueue --version=0.13.3` (latest helm chart version) - -controllerManager: - # -- Enables the Topology-Aware Scheduling feature gate. - featureGates: - - name: TopologyAwareScheduling - enabled: true - - # It ensures the Kueue pod can schedule on GKE clusters where the - # system node pool uses the default "gke-managed-components" taint. - tolerations: - - key: "components.gke.io/gke-managed-components" - operator: "Equal" - value: "true" - effect: "NoSchedule" diff --git a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/main.tf b/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/main.tf deleted file mode 100644 index 73a15ad1ab..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/management/kubectl-apply/main.tf +++ /dev/null @@ -1,271 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] - project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] - - # 1. First, Identify manifests that are explicitly enabled. - enabled_manifests = { - for index, manifest in var.apply_manifests : index => manifest - if try(manifest.enable, true) - } - - # 2. Identify URL-based manifests - url_manifests = { - for index, manifest in local.enabled_manifests : index => manifest - if try(manifest.source, null) != null && (startswith(manifest.source, "http://") || startswith(manifest.source, "https://")) - } - - # 3. Rebuild the map by populating the 'content' field for URLs based manifest - processed_apply_manifests_map = tomap({ - for index, manifest in local.enabled_manifests : tostring(index) => { - # If this manifest was a URL, its content is the body from the HTTP call. - content = contains(keys(local.url_manifests), tostring(index)) ? data.http.manifest_from_url[tostring(index)].body : manifest.content - - # If this was a URL, its source path is now null. Otherwise, use original. - source = contains(keys(local.url_manifests), tostring(index)) ? null : manifest.source - - # Pass other vars - template_vars = manifest.template_vars - server_side_apply = manifest.server_side_apply - wait_for_rollout = manifest.wait_for_rollout - } - }) - - install_kueue = try(var.kueue.install, false) - install_jobset = try(var.jobset.install, false) - install_gpu_operator = try(var.gpu_operator.install, false) - install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) - install_gib = try(var.gib.install, false) -} - -data "http" "manifest_from_url" { - for_each = local.url_manifests - url = each.value.source -} - -data "google_container_cluster" "gke_cluster" { - project = local.project_id - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -module "kubectl_apply_manifests" { - for_each = local.processed_apply_manifests_map - source = "./kubectl" - depends_on = [var.gke_cluster_exists] - - content = each.value.content - source_path = each.value.source - template_vars = each.value.template_vars - server_side_apply = each.value.server_side_apply - wait_for_rollout = each.value.wait_for_rollout - - providers = { - kubectl = kubectl - } -} - -module "install_kueue" { - source = "./helm_install" - count = local.install_kueue ? 1 : 0 - wait = false - timeout = 1200 - release_name = "kueue" - chart_repository = "oci://registry.k8s.io/kueue/charts" - chart_name = "kueue" - chart_version = var.kueue.version - namespace = "kueue-system" - create_namespace = true - values_yaml = [ - file("${path.module}/kueue/kueue-helm-values.yaml") - ] - - depends_on = [var.gke_cluster_exists] -} - -module "configure_kueue" { - source = "./kubectl" - source_path = local.install_kueue ? try(var.kueue.config_path, "") : null - template_vars = local.install_kueue ? try(var.kueue.config_template_vars, null) : null - depends_on = [module.install_kueue] - - server_side_apply = true - wait_for_rollout = true - - providers = { - kubectl = kubectl - } -} - -module "install_jobset" { - source = "./helm_install" - count = local.install_jobset ? 1 : 0 - wait = false - timeout = 1200 - release_name = "jobset" - chart_repository = "oci://registry.k8s.io/jobset/charts" - chart_name = "jobset" - chart_version = var.jobset.version - namespace = "jobset-system" - create_namespace = true - values_yaml = [ - file("${path.module}/jobset/jobset-helm-values.yaml") - ] - depends_on = [var.gke_cluster_exists, module.configure_kueue] -} - -module "install_nvidia_dra_driver" { - count = local.install_nvidia_dra_driver ? 1 : 0 - depends_on = [module.kubectl_apply_manifests, var.gke_cluster_exists, module.configure_kueue] - source = "./helm_install" - - release_name = "nvidia-dra-driver-gpu" # The release name - chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts - chart_name = "nvidia-dra-driver-gpu" # The chart name - chart_version = var.nvidia_dra_driver.version # The chart version - namespace = "nvidia-dra-driver-gpu" # The target namespace - create_namespace = true # Equivalent to --create-namespace - - # Use the 'values' argument to pass the YAML content - # This corresponds to the -f <(cat < -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_monitoring_dashboard.dashboard](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_dashboard) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [base\_dashboard](#input\_base\_dashboard) | Baseline dashboard template, select from HPC or Empty | `string` | `"HPC"` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the monitoring dashboard instance. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [title](#input\_title) | Title of the created dashboard | `string` | `"Cluster Toolkit Dashboard"` | no | -| [widgets](#input\_widgets) | List of additional widgets to add to the base dashboard. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for accessing the monitoring dashboard | - diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl deleted file mode 100644 index f25cbbd2c6..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl +++ /dev/null @@ -1,17 +0,0 @@ -{ - "displayName": "${title}: ${deployment_name}", - "gridLayout": { - "columns": 2, - "widgets": [ - { - "text": { - "content": "Metrics from the ${deployment_name} deployment of the Cluster Toolkit.", - "format": "MARKDOWN" - }, - "title": "${title}" - }%{ for widget in widgets ~}, - ${widget} - %{endfor ~} - ] - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl deleted file mode 100644 index 5b20435a9a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl +++ /dev/null @@ -1,595 +0,0 @@ -{ - "displayName": "${title}: ${deployment_name}", - "labels": ${jsonencode(labels)}, - "gridLayout": { - "columns": 2, - "widgets": [ - { - "text": { - "content": "HPC metrics from the ${deployment_name} deployment of the Cluster Toolkit.", - "format": "MARKDOWN" - }, - "title": "${title}" - }, - { - "title": "VM Instance - Memory utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/memory/percent_used\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - CPU Utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", - "pickTimeSeriesFilter": { - "direction": "TOP", - "numTimeSeries": 20, - "rankingMethod": "METHOD_MEAN" - } - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - CPU utilization (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - }, - "unitOverride": "%" - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk read operations", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/read_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk write operations", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/write_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk Read Bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/disk/read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk Write Bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/disk/write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Throttled read bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Throttled write bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Received packets", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/received_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - Sent packets", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - Received bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/received_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Sent bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_MEAN", - "groupByFields": [ - "metric.label.\"instance_name\"", - "metric.label.\"loadbalanced\"", - "resource.label.\"project_id\"", - "resource.label.\"instance_id\"", - "resource.label.\"zone\"" - ], - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - } - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Network Traffic Bytes (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/interface/traffic\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Network Packets (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/interface/packets\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "TCP connections", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/network/tcp_connections\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - }, - "unitOverride": "1" - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - CPU utilization for steal", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "STACKED_BAR", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MAX" - }, - "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\" metric.label.\"cpu_state\"=\"steal\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - CPU utilization [MEAN]", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }%{ for widget in widgets ~}, - ${widget} - %{endfor ~} - ] - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/main.tf b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/main.tf deleted file mode 100644 index df3c5c36b0..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/main.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "dashboard", ghpc_role = "monitoring" }) -} - -locals { - dash_path = "${path.module}/dashboards/${var.base_dashboard}.json.tpl" -} - -resource "google_monitoring_dashboard" "dashboard" { - dashboard_json = templatefile(local.dash_path, { - widgets = var.widgets - deployment_name = var.deployment_name - title = var.title - labels = local.labels - } - ) - project = var.project_id -} diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/metadata.yaml deleted file mode 100644 index de1a10f57d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - stackdriver.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/outputs.tf b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/outputs.tf deleted file mode 100644 index b7ff35fb0e..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "instructions" { - description = "Instructions for accessing the monitoring dashboard" - value = <<-EOT - A monitoring dashboard has been created. To view, navigate to the following URL: - https://console.cloud.google.com/monitoring/dashboards/builder${regex("/[0-9a-z-]*$", google_monitoring_dashboard.dashboard.id)} - EOT -} diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/variables.tf b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/variables.tf deleted file mode 100644 index 8194f8b73a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/variables.tf +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "base_dashboard" { - description = "Baseline dashboard template, select from HPC or Empty" - type = string - default = "HPC" - validation { - condition = contains(["HPC", "Empty"], var.base_dashboard) - error_message = "Must set var.base_dashboard to either \"HPC\" or \"Empty\"." - } -} - -variable "title" { - description = "Title of the created dashboard" - type = string - default = "Cluster Toolkit Dashboard" -} - -variable "widgets" { - description = "List of additional widgets to add to the base dashboard." - type = list(string) - default = [] -} - -variable "labels" { - description = "Labels to add to the monitoring dashboard instance. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/versions.tf b/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/versions.tf deleted file mode 100644 index 2717fe79f6..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/monitoring/dashboard/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/README.md b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/README.md deleted file mode 100644 index 057f4b649d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/README.md +++ /dev/null @@ -1,111 +0,0 @@ -## Description - -This module facilitates the creation of custom firewall rules for existing -networks. - -## Example usage - -This module can be used by other Toolkit modules to create application-specific -firewall rules or in conjunction with the [pre-existing-vpc] module to enable -traffic in existing networks. The snippet below is drawn from the -[ml-slurm.yaml] example: - -```yaml -- group: primary - modules: - - id: network - source: modules/network/pre-existing-vpc - - # this example anticipates that the VPC default network has internal traffic - # allowed and IAP tunneling for SSH connections - - id: firewall_rule - source: modules/network/firewall-rules - use: - - network - settings: - ingress_rules: - - name: $(vars.deployment_name)-allow-internal-traffic - description: Allow internal traffic - destination_ranges: - - $(network.subnetwork_address) - source_ranges: - - $(network.subnetwork_address) - allow: - - protocol: tcp - ports: - - 0-65535 - - protocol: udp - ports: - - 0-65535 - - protocol: icmp - - name: $(vars.deployment_name)-allow-iap-ssh - description: Allow IAP-tunneled SSH connections - destination_ranges: - - $(network.subnetwork_address) - source_ranges: - - 35.235.240.0/20 - allow: - - protocol: tcp - ports: - - 22 -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [firewall\_rule](#module\_firewall\_rule) | terraform-google-modules/network/google//modules/firewall-rules | ~> 12.0 | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.pga_check](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [google_compute_subnetwork.subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [egress\_rules](#input\_egress\_rules) | List of egress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | -| [ingress\_rules](#input\_ingress\_rules) | List of ingress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | -| [network\_name](#input\_network\_name) | The name of the network to create firewall rules in | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The project ID to host the network in | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork whose global network firewall rules will be modified. | `string` | n/a | yes | - -## Outputs - -No outputs. - - -[pre-existing-vpc]: ../pre-existing-vpc/README.md -[ml-slurm.yaml]: ../../../examples/ml-slurm.yaml diff --git a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/main.tf b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/main.tf deleted file mode 100644 index 05241278ad..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/main.tf +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - use_subnetwork_data = (var.project_id == null || var.network_name == null) && var.subnetwork_self_link != null -} - -# the google_compute_network data source does not allow identification by -# self_link, which uniquely identifies subnet, project, and network -data "google_compute_subnetwork" "subnetwork" { - # Only instantiate this data source if needed - count = local.use_subnetwork_data ? 1 : 0 - self_link = var.subnetwork_self_link -} - -locals { - # Derived values from data source, null if data source is not used - derived_project_id = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].project : null - derived_network_name = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].network : null - - # Effective values: Use var if provided, otherwise use derived value - effective_project_id = coalesce(var.project_id, local.derived_project_id) - effective_network_name = coalesce(var.network_name, local.derived_network_name) -} - -# Module-level check for Private Google Access on the subnetwork. -# This check is only relevant if subnetwork_self_link was provided and used. -resource "terraform_data" "pga_check" { - count = local.use_subnetwork_data ? 1 : 0 - - lifecycle { - precondition { - condition = data.google_compute_subnetwork.subnetwork[0].private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.subnetwork[0].name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } - } -} - -module "firewall_rule" { - source = "terraform-google-modules/network/google//modules/firewall-rules" - version = "~> 12.0" - project_id = local.effective_project_id - network_name = local.effective_network_name - - ingress_rules = var.ingress_rules - egress_rules = var.egress_rules -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/variables.tf b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/variables.tf deleted file mode 100644 index 05e9be4425..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/variables.tf +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork whose global network firewall rules will be modified." - type = string -} - -variable "project_id" { - description = "The project ID to host the network in" - type = string - default = null -} - -variable "network_name" { - description = "The name of the network to create firewall rules in" - type = string - default = null -} - -variable "ingress_rules" { - description = "List of ingress rules" - default = [] - type = list(object({ - name = string - description = optional(string, null) - disabled = optional(bool, null) - priority = optional(number, null) - destination_ranges = optional(list(string), []) - source_ranges = optional(list(string), []) - source_tags = optional(list(string)) - source_service_accounts = optional(list(string)) - target_tags = optional(list(string)) - target_service_accounts = optional(list(string)) - - allow = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - deny = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - log_config = optional(object({ - metadata = string - })) - })) -} - -variable "egress_rules" { - description = "List of egress rules" - default = [] - type = list(object({ - name = string - description = optional(string, null) - disabled = optional(bool, null) - priority = optional(number, null) - destination_ranges = optional(list(string), []) - source_ranges = optional(list(string), []) - source_tags = optional(list(string)) - source_service_accounts = optional(list(string)) - target_tags = optional(list(string)) - target_service_accounts = optional(list(string)) - - allow = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - deny = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - log_config = optional(object({ - metadata = string - })) - })) -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/versions.tf deleted file mode 100644 index 9061dd3ae5..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/firewall-rules/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.74.0" - } - - required_version = ">= 1.5" -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/README.md b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/README.md deleted file mode 100644 index abbfe3b97b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/README.md +++ /dev/null @@ -1,143 +0,0 @@ -## Description - -This module accomplishes the following: - -* Creates one [VPC network][cft-network] - * Each VPC contains a variable number of subnetworks as specified in the - `subnetworks_template` variable - * Each subnetwork contains distinct IP address ranges -* Outputs the following unique parameters - * `subnetwork_interfaces` which is compatible with Slurm and vm-instance - modules - * `subnetwork_interfaces_gke` which is compatible with GKE modules - -This module is a simplified version of the VPC module and its main difference -is the variable `subnetwork_template` which is the template for all subnetworks -created within the network. This template contains the following values: - -1. `count`: The number of subnetworks to be created -1. `name_prefix`: The prefix for the subnetwork names -1. `ip_range`: [CIDR-formatted IP range][cidr] -1. `region`: The region where the subnetwork will be deployed - -> [!WARNING] -> The `ip_range` should be always be large enough to split into `count` -> subnetworks and the number of required connections within. - -[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v10.0.0 -[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation - -### Example - -This snippet uses the gpu-vpc module to create a new VPC network named -`test-rdma-net` with 8 subnetworks named `test-mrdma-sub-#` where # ranges from -0 to 7. The subnetworks will split the `ip_range` evenly, starting from bit 16 -(0 indexed). The networks are ingested by the Slurm nodeset within the -`additional_networks` setting. - -```yaml - - id: rdma-net - source: modules/network/gpu-rdma-vpc - settings: - network_name: test-rdma-net - network_profile: https://www.googleapis.com/compute/beta/projects/$(vars.project_id)/global/networkProfiles/$(vars.zone)-vpc-roce - network_routing_mode: REGIONAL - subnetworks_template: - name_prefix: test-mrdma-sub - count: 8 - ip_range: 192.168.0.0/16 - region: $(vars.region) - - - id: a3_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network0] - settings: - machine_type: a3-ultragpu-8g - additional_networks: - $(concat( - [{ - network=null, - subnetwork=network1.subnetwork_self_link, - subnetwork_project=vars.project_id, - nic_type="GVNIC", - queue_count=null, - network_ip="", - stack_type=null, - access_config=[], - ipv6_access_config=[], - alias_ip_range=[] - }], - rdma-net.subnetwork_interfaces - )) - ... -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | DEPRECATED: enable\_internal\_traffic can not be specified for gpu-rdma-vpc. | `bool` | `null` | no | -| [firewall\_log\_config](#input\_firewall\_log\_config) | DEPRECATED: firewall\_log\_config can not be specified for gpu-rdma-vpc. | `string` | `null` | no | -| [firewall\_rules](#input\_firewall\_rules) | DEPRECATED: firewall\_rules can not be specified for gpu-rdma-vpc. | `any` | `null` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}} | `string` | n/a | yes | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "REGIONAL") | `string` | `"REGIONAL"` | no | -| [nic\_type](#input\_nic\_type) | NIC type for use in modules that use the output | `string` | `"MRDMA"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | -| [subnetworks\_template](#input\_subnetworks\_template) | Specifications for the subnetworks that will be created within this VPC.

count (number, required, number of subnets to create, default is 8)
name\_prefix (string, required, subnet name prefix, default is deployment name)
ip\_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16)
region (string, optional, region to deploy subnets to, defaults to vars.region) | 
object({
    count       = number
    name_prefix = string
    ip_range    = string
    region      = optional(string)
  })
| 
{
  "count": 8,
  "ip_range": "192.168.0.0/16",
  "name_prefix": null,
  "region": null
}
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [network\_id](#output\_network\_id) | ID of the new VPC network | -| [network\_name](#output\_network\_name) | Name of the new VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | -| [subnetwork\_interfaces](#output\_subnetwork\_interfaces) | Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules) | -| [subnetwork\_interfaces\_gke](#output\_subnetwork\_interfaces\_gke) | Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool) | -| [subnetwork\_name\_prefix](#output\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | -| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | - diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/main.tf b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/main.tf deleted file mode 100644 index e37db01976..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/main.tf +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - autoname = replace(var.deployment_name, "_", "-") - network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name - subnet_prefix = var.subnetworks_template.name_prefix == null ? "${local.autoname}-subnet" : var.subnetworks_template.name_prefix - - new_bits = ceil(log(var.subnetworks_template.count, 2)) - template_subnetworks = [for i in range(var.subnetworks_template.count) : - { - subnet_name = "${local.subnet_prefix}-${i}" - subnet_region = try(var.subnetworks_template.region, var.region) - subnet_ip = cidrsubnet(var.subnetworks_template.ip_range, local.new_bits, i) - } - ] - - firewall_rules = [] - - output_subnets = [ - for subnet in module.vpc.subnets : { - network = null - subnetwork = subnet.self_link - subnetwork_project = null # will populate from subnetwork_self_link - network_ip = null - nic_type = var.nic_type - stack_type = null - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } - ] - - output_subnets_gke = [ - for i in range(length(module.vpc.subnets)) : { - network = local.network_name - subnetwork = local.template_subnetworks[i].subnet_name - subnetwork_project = var.project_id - network_ip = null - nic_type = var.nic_type - stack_type = null - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } - ] -} - -module "vpc" { - source = "terraform-google-modules/network/google" - version = "~> 12.0" - - network_name = local.network_name - project_id = var.project_id - auto_create_subnetworks = false - subnets = local.template_subnetworks - routing_mode = var.network_routing_mode - mtu = var.mtu - description = var.network_description - shared_vpc_host = var.shared_vpc_host - delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes - firewall_rules = local.firewall_rules - network_profile = var.network_profile -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf deleted file mode 100644 index 0a21f1d3f2..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the new VPC network" - value = module.vpc.network_name - depends_on = [module.vpc] -} - -output "network_id" { - description = "ID of the new VPC network" - value = module.vpc.network_id - depends_on = [module.vpc] -} - -output "network_self_link" { - description = "Self link of the new VPC network" - value = module.vpc.network_self_link - depends_on = [module.vpc] -} - -output "subnetworks" { - description = "Full list of subnetwork objects belonging to the new VPC network" - value = module.vpc.subnets - depends_on = [module.vpc] -} - -output "subnetwork_interfaces" { - description = "Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules)" - value = local.output_subnets - depends_on = [module.vpc] -} - -# The output subnetwork_interfaces is compatible with vm-instance module but not with gke-node-pool -# See https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/99493df21cecf6a092c45298bf7a45e0343cf622/modules/compute/vm-instance/variables.tf#L220 -# So, we need a separate output that makes the network and subnetwork names available -output "subnetwork_interfaces_gke" { - description = "Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool)" - value = local.output_subnets_gke - depends_on = [module.vpc] -} - -output "subnetwork_name_prefix" { - description = "Prefix of the RDMA subnetwork names" - value = var.subnetworks_template.name_prefix -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf deleted file mode 100644 index a30fb50e7d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "network_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" - type = string - default = null -} - -variable "region" { - description = "The default region for Cloud resources" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "subnetworks_template" { - description = <<-EOT - Specifications for the subnetworks that will be created within this VPC. - - count (number, required, number of subnets to create, default is 8) - name_prefix (string, required, subnet name prefix, default is deployment name) - ip_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16) - region (string, optional, region to deploy subnets to, defaults to vars.region) - EOT - nullable = false - type = object({ - count = number - name_prefix = string - ip_range = string - region = optional(string) - }) - default = { - count = 8 - name_prefix = null - ip_range = "192.168.0.0/16" - region = null - } - - validation { - condition = var.subnetworks_template.count > 0 - error_message = "Number of subnetworks must be greater than 0" - } - - validation { - condition = can(cidrhost(var.subnetworks_template.ip_range, 0)) - error_message = "IP address range must be in CIDR format." - } -} - -variable "network_routing_mode" { - type = string - default = "REGIONAL" - description = "The network routing mode (default \"REGIONAL\")" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "shared_vpc_host" { - type = bool - description = "Makes this project a Shared VPC host if 'true' (default 'false')" - default = false -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_internal_traffic" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." - type = bool - default = null - validation { - condition = var.enable_internal_traffic == null - error_message = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." - } -} - -variable "firewall_rules" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." - type = any - default = null - validation { - condition = var.firewall_rules == null - error_message = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." - } -} - -variable "firewall_log_config" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." - type = string - default = null - validation { - condition = var.firewall_log_config == null - error_message = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." - } -} - -variable "network_profile" { - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - EOT - type = string - nullable = false - - validation { - condition = can(coalesce(var.network_profile)) - error_message = "var.network_profile must be specified and not an empty string" - } -} - -variable "nic_type" { - description = "NIC type for use in modules that use the output" - type = string - nullable = true - default = "MRDMA" - - validation { - condition = contains(["MRDMA"], var.nic_type) - error_message = "The nic_type must be \"MRDMA\"." - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf deleted file mode 100644 index 71b7106734..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 0.15.0" -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/multivpc/README.md b/deletion-test/cluster/modules/embedded/modules/network/multivpc/README.md deleted file mode 100644 index 973e6b32c9..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/multivpc/README.md +++ /dev/null @@ -1,136 +0,0 @@ -## Description - -This module accomplishes the following: - -* Creates 2 to 8 [VPC networks][vpc] - * Each VPC contains exactly 1 subnetwork - * Each subnetwork contains distinct IP address ranges -* Outputs the `additional_networks` parameter, which is compatible with Slurm - modules - -There are 4 variables that differentiate this module from the standard VPC -module. - -1. `network_prefix`: The name prefix of the VPCs to be created. All - networks and subnetworks will start with this and end with a unique number. -1. `network_count`: The number of VPCs to be created. -1. `global_ip_address_range`: [CIDR-formatted IP range][cidr] -1. `network_cidr_suffix`: The CIDR suffix that defines the address - space that the individual VPCs will cover. - -> [!WARNING] -> The `network_cidr_suffix` should be always be larger than the CIDR suffix on -> `global_ip_address_range`. The difference between these two suffixes should -> be large enough to accommodate the number of VPCs that are being deployed -> (e.g. CIDR suffix bit difference <= `ceil(log2(network_count)))`). - -> [!NOTE] -> For deployments that need multiple VPCs that do not meet this use-case, users -> should deploy multiple individual VPC modules. - -[vpc]: ../vpc/README.md -[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation - -### Example - -This snippet uses the multivpc module to create 8 new VPC networks named -`multivpc-net-#` where # ranges from 0 to 7. Additionally, it creates 1 -subnetwork in each VPC. - -```yaml - - id: network - source: modules/network/vpc - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: a3_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network, multinetwork] - settings: - machine_type: a3-highgpu-8g - ... -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | - -## Providers - -| Name | Version | -|------|---------| -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [vpcs](#module\_vpcs) | ../vpc | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.global_ip_cidr_suffix](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | -| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | -| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | -| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | -| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | -| [global\_ip\_address\_range](#input\_global\_ip\_address\_range) | IP address range (CIDR) that will span entire set of VPC networks | `string` | `"172.16.0.0/12"` | no | -| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT) | `number` | `2` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_count](#input\_network\_count) | The number of vpc nettworks to create | `number` | `4` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_interface\_defaults](#input\_network\_interface\_defaults) | The template of the network settings to be used on all vpcs. | 
object({
    network            = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string, "GVNIC")
    stack_type         = optional(string, "IPV4_ONLY")
    queue_count        = optional(string)
    access_config = optional(list(object({
      nat_ip                 = string
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  })
| 
{
  "access_config": [],
  "alias_ip_range": [],
  "ipv6_access_config": [],
  "network": null,
  "network_ip": "",
  "nic_type": "GVNIC",
  "queue_count": null,
  "stack_type": "IPV4_ONLY",
  "subnetwork": null,
  "subnetwork_project": null
}
| no | -| [network\_name\_prefix](#input\_network\_name\_prefix) | The base name of the vpcs and their subnets, will be appended with a sequence number | `string` | `""` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network dynamic routing mode | `string` | `"REGIONAL"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [subnetwork\_cidr\_suffix](#input\_subnetwork\_cidr\_suffix) | The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network. | `number` | `16` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [additional\_networks](#output\_additional\_networks) | Network interfaces for each subnetwork created by this module | -| [network\_ids](#output\_network\_ids) | IDs of the new VPC network | -| [network\_names](#output\_network\_names) | Names of the new VPC networks | -| [network\_self\_links](#output\_network\_self\_links) | Self link of the new VPC network | -| [subnetwork\_addresses](#output\_subnetwork\_addresses) | IP address range of the primary subnetwork | -| [subnetwork\_names](#output\_subnetwork\_names) | Names of the subnetwork created in each network | -| [subnetwork\_self\_links](#output\_subnetwork\_self\_links) | Self link of the primary subnetwork | - diff --git a/deletion-test/cluster/modules/embedded/modules/network/multivpc/main.tf b/deletion-test/cluster/modules/embedded/modules/network/multivpc/main.tf deleted file mode 100644 index ad06e793c1..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/multivpc/main.tf +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # this input variable is validated to be in CIDR format - network_name = coalesce(replace(var.network_name_prefix, "_", "-"), replace(var.deployment_name, "_", "-")) - global_ip_cidr_prefix = split("/", var.global_ip_address_range)[0] - global_ip_cidr_suffix = split("/", var.global_ip_address_range)[1] - global_ip_cidr_valid = "${local.global_ip_cidr_prefix}/${terraform_data.global_ip_cidr_suffix.output}" - subnetwork_new_bits = var.subnetwork_cidr_suffix - local.global_ip_cidr_suffix - maximum_subnetworks = pow(2, local.subnetwork_new_bits) - additional_networks = [ - for vpc in module.vpcs : - merge(var.network_interface_defaults, { - network = vpc.network_name - subnetwork = vpc.subnetwork_name - subnetwork_project = var.project_id - }) - ] -} - -resource "terraform_data" "global_ip_cidr_suffix" { - input = local.global_ip_cidr_suffix - lifecycle { - precondition { - condition = local.maximum_subnetworks >= var.network_count - error_message = < 1 - error_message = "The minimum VPCs able to be created by this module is 2. Use the standard Toolkit module at modules/network/vpc for count = 1" - } - validation { - condition = var.network_count <= 8 - error_message = "The maximum VPCs able to be created by this module is 8" - } -} - -variable "global_ip_address_range" { - description = "IP address range (CIDR) that will span entire set of VPC networks" - type = string - default = "172.16.0.0/12" - - validation { - condition = can(cidrhost(var.global_ip_address_range, 0)) - error_message = "var.global_ip_address_range must be an IPv4 CIDR range (e.g. \"172.16.0.0/12\")." - } -} - -variable "subnetwork_cidr_suffix" { - description = "The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network." - type = number - default = 16 -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "network_routing_mode" { - type = string - default = "REGIONAL" - description = "The network dynamic routing mode" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "ips_per_nat" { - type = number - description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT)" - default = 2 -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_iap_ssh_ingress" { - type = bool - description = "Enable a firewall rule to allow SSH access using IAP tunnels" - default = true -} - -variable "enable_iap_rdp_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" - default = false -} - -variable "enable_iap_winrm_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" - default = false -} - -variable "enable_internal_traffic" { - type = bool - description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" - default = true -} - -variable "extra_iap_ports" { - type = list(string) - description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" - default = [] -} - -variable "allowed_ssh_ip_ranges" { - type = list(string) - description = "A list of CIDR IP ranges from which to allow ssh access" - default = [] - - validation { - condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) - error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." - } -} - -variable "firewall_rules" { - type = any - description = "List of firewall rules" - default = [] -} - -variable "network_interface_defaults" { - type = object({ - network = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - network_ip = optional(string, "") - nic_type = optional(string, "GVNIC") - stack_type = optional(string, "IPV4_ONLY") - queue_count = optional(string) - access_config = optional(list(object({ - nat_ip = string - network_tier = string - public_ptr_domain_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - public_ptr_domain_name = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - }) - description = "The template of the network settings to be used on all vpcs." - default = { - network = null - subnetwork = null - subnetwork_project = null - network_ip = "" - nic_type = "GVNIC" - stack_type = "IPV4_ONLY" - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } -} - -variable "network_profile" { - type = string - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown - EOT - default = null -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/multivpc/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/multivpc/versions.tf deleted file mode 100644 index e75a67f7b6..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/multivpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.4.0" -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/README.md b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/README.md deleted file mode 100644 index 4d63b17091..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/README.md +++ /dev/null @@ -1,94 +0,0 @@ -## Description - -This module discovers a subnetwork that already exists in Google Cloud and -outputs subnetwork attributes that uniquely identify it for use by other modules. - -For example, the blueprint below discovers the referred to subnetwork. -With the `use` keyword, the [vm-instance] module accepts the `subnetwork_self_link` -input variables that uniquely identify the subnetwork in which the VM will be created. - -[vpc]: ../vpc/README.md -[vm-instance]: ../../compute/vm-instance/README.md - -> **_NOTE:_** Additional IAM work is needed for this to work correctly. - -### Example - -```yaml -- id: network - source: modules/network/pre-existing-subnetwork - settings: - subnetwork_self_link: https://www.googleapis.com/compute/v1/projects/name-of-host-project/regions/REGION/subnetworks/SUBNETNAME - -- id: example_vm - source: modules/compute/vm-instance - use: - - network - settings: - name_prefix: example - machine_type: c2-standard-4 -``` - -As described in documentation: -[https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork] - -If subnetwork_self_link is provided then name,region,project is ignored. - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [project](#input\_project) | Name of the project that owns the subnetwork | `string` | `null` | no | -| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | `null` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self-link of the subnet in the VPC | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | -| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | - diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/main.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/main.tf deleted file mode 100644 index 9fb206f969..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/main.tf +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -data "google_compute_subnetwork" "primary_subnetwork" { - name = var.subnetwork_name - region = var.region - project = var.project - self_link = var.subnetwork_self_link - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The subnetwork: ${coalesce(var.subnetwork_name, var.subnetwork_self_link)} could not be found." - } - } -} - -# Module-level check for Private Google Access on the subnetwork -check "private_google_access_enabled_subnetwork" { - assert { - condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml deleted file mode 100644 index 6a6f1e5757..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com -ghpc: - has_to_be_used: true diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf deleted file mode 100644 index 868708dc6b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "subnetwork" { - description = "Full subnetwork object in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork -} - -output "subnetwork_name" { - description = "Name of the subnetwork in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.name -} - -output "subnetwork_self_link" { - description = "Subnetwork self-link in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.self_link -} - -output "subnetwork_address" { - description = "Subnetwork IP range in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf deleted file mode 100644 index d5191843e8..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "subnetwork_self_link" { - description = "Self-link of the subnet in the VPC" - type = string - default = null -} - -variable "project" { - description = "Name of the project that owns the subnetwork" - type = string - default = null -} - -variable "subnetwork_name" { - description = "Name of the pre-existing VPC subnetwork" - type = string - default = null -} - -variable "region" { - description = "Region in which to search for primary subnetwork" - type = string - default = null -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf deleted file mode 100644 index 917d948433..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.74.0" - } - - required_version = ">= 1.5" -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/README.md b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/README.md deleted file mode 100644 index 38a1840c2d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/README.md +++ /dev/null @@ -1,110 +0,0 @@ -## Description - -This module discovers a VPC network that already exists in Google Cloud and -outputs network attributes that uniquely identify it for use by other modules. -The module outputs are aligned with the [vpc module][vpc] so that it can be used -as a drop-in substitute when a VPC already exists. - -For example, the blueprint below discovers the "default" global network and the -"default" regional subnetwork in us-central1. With the `use` keyword, the -[vm-instance] module accepts the `network_self_link` and `subnetwork_self_link` -input variables that uniquely identify the network and subnetwork in which the -VM will be created. - -[vpc]: ../vpc/README.md -[vm-instance]: ../../compute/vm-instance/README.md - -### Example - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - settings: - project_id: $(vars.project_id) - region: us-central1 - -- id: example_vm - source: modules/compute/vm-instance - use: - - network1 - settings: - name_prefix: example - machine_type: c2-standard-4 -``` - -> **_NOTE:_** The `project_id` and `region` settings would be inferred from the -> deployment variables of the same name, but they are included here for clarity. - -### Use shared-vpc - -If a network is created in different project, this module can be used to -reference the network. To use a network from a different project first make sure -you have a [cloud nat][cloudnat] and [IAP][iap] forwarding. For more details, -refer [shared-vpc][shared-vpc-doc] - -[cloudnat]: https://cloud.google.com/nat/docs/overview -[iap]: https://cloud.google.com/iap/docs/using-tcp-forwarding -[shared-vpc-doc]: ../../../examples/README.md#hpc-slurm-sharedvpcyaml-community-badge-experimental-badge - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_network.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network) | data source | -| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [network\_name](#input\_network\_name) | Name of the existing VPC network | `string` | `"default"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | n/a | yes | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork; defaults to var.network\_name if set to null. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [network\_id](#output\_network\_id) | ID of the existing VPC network | -| [network\_name](#output\_network\_name) | Name of the existing VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the existing VPC network | -| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | -| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | - diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/main.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/main.tf deleted file mode 100644 index ed332bab72..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/main.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -data "google_compute_network" "vpc" { - name = var.network_name - project = var.project_id - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The network: ${var.network_name} could not be found in project: ${var.project_id}." - } - } -} - -locals { - subnetwork_name = var.subnetwork_name != null ? var.subnetwork_name : var.network_name -} - -data "google_compute_subnetwork" "primary_subnetwork" { - name = local.subnetwork_name - region = var.region - project = var.project_id - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The subnetwork: ${local.subnetwork_name} could not be found in project: ${var.project_id} and region: ${var.region}." - } - } -} - -# Module-level check for Private Google Access on the subnetwork -check "private_google_access_enabled_subnetwork" { - assert { - condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/outputs.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/outputs.tf deleted file mode 100644 index 00861af5ca..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the existing VPC network" - value = data.google_compute_network.vpc.name -} - -output "network_id" { - description = "ID of the existing VPC network" - value = data.google_compute_network.vpc.id -} - -output "network_self_link" { - description = "Self link of the existing VPC network" - value = data.google_compute_network.vpc.self_link -} - -output "subnetwork" { - description = "Full subnetwork object in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork -} - -output "subnetwork_name" { - description = "Name of the subnetwork in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.name -} - -output "subnetwork_self_link" { - description = "Subnetwork self-link in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.self_link -} - -output "subnetwork_address" { - description = "Subnetwork IP range in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/variables.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/variables.tf deleted file mode 100644 index 291a81604a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/variables.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "network_name" { - description = "Name of the existing VPC network" - type = string - default = "default" -} - -variable "subnetwork_name" { - description = "Name of the pre-existing VPC subnetwork; defaults to var.network_name if set to null." - type = string - default = null -} - -variable "region" { - description = "Region in which to search for primary subnetwork" - type = string -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/versions.tf deleted file mode 100644 index 81fe5aeff3..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/pre-existing-vpc/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.74.0" - } - - required_version = ">= 1.5" -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/README.md b/deletion-test/cluster/modules/embedded/modules/network/vpc/README.md deleted file mode 100644 index 2c2b1aa1a3..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/vpc/README.md +++ /dev/null @@ -1,237 +0,0 @@ -## Description - -This module creates a new [VPC network][vpc] with 1 or more subnetworks and -a [Cloud Router][router] for every region with a subnetwork. By default, it will -create: - -* A [Cloud NAT][nat] to enable outbound access to the public internet for VMs - without public IP addresses; VMs with public IP addresses bypass the NAT to - directly access the public internet -* A firewall rule that enables inbound SSH access from [Identity-Aware - Proxy][iap] -* A firewall rule that enables all traffic internal to the network - -This behavior is optional and can be configured as [described below](#inputs). -This module is based on networking support in the [Cloud Foundation -Toolkit][cft]. We recommend following the [documentation for the network -module][cft-network] and [submodules][cft-network-submodules] for more details. -In particular, the detailed structure of input variables can be found for: - -* [var.firewall\_rules](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/firewall-rules#inputs) -* [var.secondary\_ranges](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets#inputs) - -[vpc]: https://cloud.google.com/vpc -[router]: https://github.com/terraform-google-modules/terraform-google-cloud-router -[nat]: https://github.com/terraform-google-modules/terraform-google-cloud-nat -[iap]: https://cloud.google.com/iap -[cft]: https://cloud.google.com/foundation-toolkit -[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0 -[cft-network-submodules]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules - -Additionally, [Google Private Access][gpa] is enabled by default on all -subnetworks unless it is explicitly disabled. This setting ensures that all VMs -can use Google services such as [Cloud Storage][gcs] even if they do not have -public IP addresses or Cloud NAT is disabled. - -[gpa]: https://cloud.google.com/vpc/docs/private-google-access -[gcs]: https://cloud.google.com/storage - -### Example - -This creates a new VPC network named `cluster-net`. - -```yaml - - id: network1 - source: modules/network/vpc - settings: - network_name: cluster-net -``` - -### Deprecation warning - -The variables listed below have been deprecated and will be removed in a future -release. Until they are removed,You may continue to use them in Toolkit -blueprints with the same functionality as documented in the [Toolkit 1.0 -release][vpc1.0]. - -* Deprecated variables - * `var.primary_subnetwork` - * `var.additional_subnetworks` - * `var.subnetwork_size` - -[vpc1.0]: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/v1.0.0/modules/network/vpc/README.md - -The following variables have been added to support explicit IP ranges for -subnetworks while retaining existing functionality. We advise adopting them even -if not using explicit IP ranges . The Toolkit ***does not support*** mixing -deprecated variables with the new replacements. The new functionality is -described in [more detail below](#subnetworks). - -* New variables to adopt - * `var.subnetworks` - * A value for this can be generated by merging `var.primary_subnetwork` and - `var.additional_subnetworks` into a single list - * `var.default_primary_subnetwork_size` - * This variable has been renamed for clarity; its value can be directly - copied from an explicit setting for `var.subnetwork_size`; if your blueprint - does not have an explicit setting, the default values are the same - -### Subnetworks - -This module will always provision at least 1 "primary" subnetwork in which most -resources are expected to be provisioned. This primary subnetwork is determined -by - -1. The first element of [var.subnetworks](#input_subnetworks) if it is not the - empty list -2. A default subnetwork automatically calculated from - * [var.subnetwork_name](#input_subnetwork_name) - * [var.region](#input_region) - * [var.network_address_range](#input_network_address_range) - * [var.default_primary_subnetwork_size](#input_default_primary_subnetwork_size) - -If `var.subnetworks` is provided then the primary subnetwork name is taken -explicitly from it and `var.subnetwork_name` is ignored. - -`var.subnetworks` behaves identically to the [Cloud Foundation Toolkit subnets -module][cftsubnets] with the lone exception that one can provide ***one*** of -the following settings for each subnetwork: - -* `new_bits` -* `subnet_ip` - -If each subnetwork defines `subnet_ip` then these are taken to be their explicit -CIDR IP ranges. If each subnetwork defines `new_bits`, then these are taken to -be the size of the CIDR subnetwork (in bits). IP ranges for each subnetwork are -calculated using `var.network_address_range` as the base IP, producing the most -compact set of subnetworks possible. - -> **_NOTE:_** we do not presently support the modification of individual subnetworks -> when using this module to provision more than 1 subnetwork using automatically -> calculated IP ranges based upon `new_bits`. Doing so will cause IP ranges to be -> recalculated for each subnetwork. We advise appending new subnetworks to the end -> of `var.subnetworks`. - -[cftsubnets]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets - -### SSH Access - -By default a firewall rule is created to allow inbound SSH access from -[Identity-Aware Proxy][iap]. A user must have the `IAP-Secured Tunnel User` -(`roles/iap.tunnelResourceAccessor`) IAM role to be able to SSH over IAP. - -To allow regular SSH access from a known IP address you can add the following -`firewall_rules` setting to the `vpc` module: - -```yaml - - id: network1 - source: modules/network/vpc - settings: - firewall_rules: - - name: ssh-my-machine - direction: INGRESS - ranges: [/32] - allow: - - protocol: tcp - ports: [22] -``` - -> **Note**: You must populate the above example with the source IP address from -> which you plan to SSH from. You can use a service like -> [whatismyip.com](https://whatismyip.com) to determine your IP address. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | - -## Providers - -| Name | Version | -|------|---------| -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 7.3 | -| [nat\_ip\_addresses](#module\_nat\_ip\_addresses) | terraform-google-modules/address/google | ~> 4.1 | -| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.cloud_nat_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.network_profile_firewall_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.secondary_ranges_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_subnetworks](#input\_additional\_subnetworks) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `list(map(string))` | `null` | no | -| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | -| [default\_primary\_subnetwork\_size](#input\_default\_primary\_subnetwork\_size) | The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks | `number` | `15` | no | -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_cloud\_nat](#input\_enable\_cloud\_nat) | Enable the creation of Cloud NATs. | `bool` | `true` | no | -| [enable\_cloud\_router](#input\_enable\_cloud\_router) | Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview | `bool` | `true` | no | -| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | -| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | -| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | -| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | -| [firewall\_log\_config](#input\_firewall\_log\_config) | Firewall log configuration for Toolkit firewall rules (var.enable\_iap\_ssh\_ingress and others) | `string` | `"DISABLE_LOGGING"` | no | -| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | -| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples | `number` | `2` | no | -| [labels](#input\_labels) | Labels to add to network resources that support labels. Key-value pairs of strings. | `map(string)` | `{}` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_address\_range](#input\_network\_address\_range) | IP address range (CIDR) for global network | `string` | `"10.0.0.0/9"` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "GLOBAL") | `string` | `"GLOBAL"` | no | -| [primary\_subnetwork](#input\_primary\_subnetwork) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `map(string)` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [secondary\_ranges](#input\_secondary\_ranges) | "Secondary ranges associated with the subnets.
This will be deprecated in favour of secondary\_ranges\_list at a later date.
Please migrate to using the same." | `map(list(object({ range_name = string, ip_cidr_range = string })))` | `{}` | no | -| [secondary\_ranges\_list](#input\_secondary\_ranges\_list) | "List of secondary ranges associated with the subnetworks.
Each subnetwork must be specified at most once in this list." | 
list(object({
    subnetwork_name = string,
    ranges = list(object({
      range_name    = string,
      ip_cidr_range = string
    }))
  }))
| `[]` | no | -| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-primary-subnet") | `string` | `null` | no | -| [subnetwork\_size](#input\_subnetwork\_size) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `number` | `null` | no | -| [subnetworks](#input\_subnetworks) | List of subnetworks to create within the VPC. If left empty, it will be
replaced by a single, default subnetwork constructed from other parameters
(e.g. var.region). In all cases, the first subnetwork in the list is identified
by outputs as a "primary" subnetwork.

subnet\_name (string, required, name of subnet)
subnet\_region (string, required, region of subnet)
subnet\_ip (string, mutually exclusive with new\_bits, CIDR-formatted IP range for subnetwork)
new\_bits (number, mutually exclusive with subnet\_ip, CIDR bits used to calculate subnetwork range)
subnet\_private\_access (bool, optional, Enable Private Access on subnetwork)
subnet\_flow\_logs (map(string), optional, Configure Flow Logs see terraform-google-network module)
description (string, optional, Description of Network)
purpose (string, optional, related to Load Balancing)
role (string, optional, related to Load Balancing) | `list(map(string))` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nat\_ips](#output\_nat\_ips) | External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used) | -| [network\_id](#output\_network\_id) | ID of the new VPC network | -| [network\_name](#output\_network\_name) | Name of the new VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | -| [subnetwork](#output\_subnetwork) | Primary subnetwork object | -| [subnetwork\_address](#output\_subnetwork\_address) | IP address range of the primary subnetwork | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the primary subnetwork | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Self link of the primary subnetwork | -| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | - diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/main.tf b/deletion-test/cluster/modules/embedded/modules/network/vpc/main.tf deleted file mode 100644 index 24c8eb22bd..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/vpc/main.tf +++ /dev/null @@ -1,256 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -resource "terraform_data" "secondary_ranges_validation" { - lifecycle { - precondition { - condition = !(length(var.secondary_ranges) > 0 && length(var.secondary_ranges_list) > 0) - error_message = "Only one of var.secondary_ranges or var.secondary_ranges_list should be specified" - } - } -} - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "vpc", ghpc_role = "network" }) -} - -locals { - autoname = replace(var.deployment_name, "_", "-") - network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name - subnetwork_name = var.subnetwork_name == null ? "${local.autoname}-primary-subnet" : var.subnetwork_name - - # define a default subnetwork for cases in which no explicit subnetworks are - # defined in var.subnetworks - default_primary_subnetwork_cidr_block = cidrsubnet(var.network_address_range, var.default_primary_subnetwork_size, 0) - default_primary_subnetwork = { - subnet_name = local.subnetwork_name - subnet_ip = local.default_primary_subnetwork_cidr_block - subnet_region = var.region - subnet_private_access = true - subnet_flow_logs = false - description = "primary subnetwork in ${local.network_name}" - purpose = null - role = null - } - - # Identify user-supplied primary subnetwork - # (1) explicit var.subnetworks[0] - # (2) implicit local default subnetwork - input_primary_subnetwork = coalesce(try(var.subnetworks[0], null), local.default_primary_subnetwork) - - # Identify user-supplied additional subnetworks - # (1) explicit var.subnetworks[1:end] - # (2) empty list - input_additional_subnetworks = try(slice(var.subnetworks, 1, length(var.subnetworks)), []) - - # at this point we have constructed a list of subnetworks but need to extract - # user-provided CIDR blocks or calculate them from user-provided new_bits - # after we complete deprecation, local.all_subnetworks can be replaced with - # var.subnetworks (or local.default_primary_subnetwork if that is null) - input_subnetworks = concat([local.input_primary_subnetwork], local.input_additional_subnetworks) - subnetworks_cidr_blocks = try( - local.input_subnetworks[*]["subnet_ip"], - cidrsubnets(var.network_address_range, local.input_subnetworks[*]["new_bits"]...) - ) - - # merge in the CIDR blocks (even when already there) and remove new_bits - subnetworks = [for i, subnet in local.input_subnetworks : - merge({ for k, v in subnet : k => v if k != "new_bits" }, { "subnet_ip" = local.subnetworks_cidr_blocks[i] }) - ] - - # gather the unique regions for purposes of creating Router/NAT - cloud_router_regions = var.enable_cloud_router ? distinct([for subnet in local.subnetworks : subnet.subnet_region]) : [] - cloud_nat_regions = var.enable_cloud_nat ? local.cloud_router_regions : [] - - # this comprehension should have 1 and only 1 match - output_primary_subnetwork = one([for k, v in module.vpc.subnets : v if k == "${local.subnetworks[0].subnet_region}/${local.subnetworks[0].subnet_name}"]) - output_primary_subnetwork_name = local.output_primary_subnetwork.name - output_primary_subnetwork_self_link = local.output_primary_subnetwork.self_link - output_primary_subnetwork_ip_cidr_range = local.output_primary_subnetwork.ip_cidr_range - - iap_ports = distinct(concat(compact([ - var.enable_iap_rdp_ingress ? "3389" : "", - var.enable_iap_ssh_ingress ? "22" : "", - var.enable_iap_winrm_ingress ? "5986" : "", - ]), var.extra_iap_ports)) - - firewall_log_api_values = { - "DISABLE_LOGGING" = null - "INCLUDE_ALL_METADATA" = { metadata = "INCLUDE_ALL_METADATA" }, - "EXCLUDE_ALL_METADATA" = { metadata = "EXCLUDE_ALL_METADATA" }, - } - firewall_log_config = lookup(local.firewall_log_api_values, var.firewall_log_config, null) - - allow_iap_ingress = { - name = "${local.network_name}-fw-allow-iap-ingress" - description = "allow TCP access via Identity-Aware Proxy" - direction = "INGRESS" - priority = null - ranges = ["35.235.240.0/20"] - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = local.iap_ports - }] - deny = [] - log_config = local.firewall_log_config - } - - allow_ssh_ingress = { - name = "${local.network_name}-fw-allow-ssh-ingress" - description = "allow SSH access" - direction = "INGRESS" - priority = null - ranges = var.allowed_ssh_ip_ranges - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = ["22"] - }] - deny = [] - log_config = local.firewall_log_config - } - - allow_internal_traffic = { - name = "${local.network_name}-fw-allow-internal-traffic" - priority = null - description = "allow traffic between nodes of this VPC" - direction = "INGRESS" - ranges = [var.network_address_range] - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = ["0-65535"] - }, { - protocol = "udp" - ports = ["0-65535"] - }, { - protocol = "icmp" - ports = null - }, - ] - deny = [] - log_config = local.firewall_log_config - } - - firewall_rules = concat( - var.firewall_rules, - length(var.allowed_ssh_ip_ranges) > 0 ? [local.allow_ssh_ingress] : [], - var.enable_internal_traffic ? [local.allow_internal_traffic] : [], - length(local.iap_ports) > 0 ? [local.allow_iap_ingress] : [] - ) - - secondary_ranges_map = { - for secondary_range in var.secondary_ranges_list : - secondary_range.subnetwork_name => secondary_range.ranges - } -} - -resource "terraform_data" "network_profile_firewall_validation" { - lifecycle { - precondition { - condition = !(try(strcontains(var.network_profile, "roce"), false) && length(local.firewall_rules) > 0) - error_message = "If var.network_profile contains 'roce', var.firewall_rules must be empty and var.enable_internal_traffic must be false, please see: https://cloud.google.com/vpc/docs/rdma-network-profiles#additional_features_that_dont_apply_to_traffic_from_rdma_nics" - } - } -} - -module "vpc" { - source = "terraform-google-modules/network/google" - version = "~> 12.0" - - depends_on = [terraform_data.network_profile_firewall_validation] - - network_name = local.network_name - project_id = var.project_id - auto_create_subnetworks = false - subnets = local.subnetworks - secondary_ranges = length(local.secondary_ranges_map) > 0 ? local.secondary_ranges_map : var.secondary_ranges - routing_mode = var.network_routing_mode - mtu = var.mtu - description = var.network_description - shared_vpc_host = var.shared_vpc_host - delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes - firewall_rules = local.firewall_rules - network_profile = var.network_profile -} - -resource "terraform_data" "cloud_nat_validation" { - lifecycle { - precondition { - condition = var.enable_cloud_router == true || var.enable_cloud_nat == false - error_message = <<-EOD - "Cannot have Cloud NAT without a Cloud Router. If you desire Cloud NAT functionality please set `enable_cloud_router` to true." - EOD - } - } -} - -# This use of the module may appear odd when var.ips_per_nat = 0. The module -# will be called for all regions with subnetworks but names will be set to the -# empty list. This is a perfectly valid value (the default!). In this scenario, -# no IP addresses are created and all module outputs are empty lists. -# -# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/variables.tf#L27 -# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/outputs.tf -module "nat_ip_addresses" { - source = "terraform-google-modules/address/google" - version = "~> 4.1" - - depends_on = [terraform_data.cloud_nat_validation] - - for_each = toset(local.cloud_nat_regions) - - project_id = var.project_id - region = each.value - # an external, regional (not global) IP address is suited for a regional NAT - address_type = "EXTERNAL" - global = false - labels = local.labels - names = [for idx in range(var.ips_per_nat) : "${local.network_name}-nat-ips-${each.value}-${idx}"] -} - -module "cloud_router" { - source = "terraform-google-modules/cloud-router/google" - version = "~> 7.3" - - depends_on = [terraform_data.cloud_nat_validation] - - for_each = toset(local.cloud_router_regions) - - project = var.project_id - name = "${local.network_name}-router" - region = each.value - network = module.vpc.network_name - # in scenario with no NAT IPs, no NAT is created even if router is created - # https://github.com/terraform-google-modules/terraform-google-cloud-router/blob/v2.0.0/nat.tf#L18-L20 - nats = length(module.nat_ip_addresses[each.value].self_links) == 0 ? [] : [ - { - name : "cloud-nat-${each.value}", - nat_ips : module.nat_ip_addresses[each.value].self_links - }, - ] -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/network/vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/outputs.tf b/deletion-test/cluster/modules/embedded/modules/network/vpc/outputs.tf deleted file mode 100644 index c2ee6bdf6b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/vpc/outputs.tf +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the new VPC network" - value = module.vpc.network_name - depends_on = [module.vpc, module.cloud_router] -} - -output "network_id" { - description = "ID of the new VPC network" - value = module.vpc.network_id - depends_on = [module.vpc, module.cloud_router] -} - -output "network_self_link" { - description = "Self link of the new VPC network" - value = module.vpc.network_self_link - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetworks" { - description = "Full list of subnetwork objects belonging to the new VPC network" - value = module.vpc.subnets - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork" { - description = "Primary subnetwork object" - value = local.output_primary_subnetwork - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_name" { - description = "Name of the primary subnetwork" - value = local.output_primary_subnetwork_name - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_self_link" { - description = "Self link of the primary subnetwork" - value = local.output_primary_subnetwork_self_link - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_address" { - description = "IP address range of the primary subnetwork" - value = local.output_primary_subnetwork_ip_cidr_range - depends_on = [module.vpc, module.cloud_router] -} - -output "nat_ips" { - description = "External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used)" - value = flatten([for ipmod in module.nat_ip_addresses : ipmod.addresses]) -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/variables.tf b/deletion-test/cluster/modules/embedded/modules/network/vpc/variables.tf deleted file mode 100644 index e036189404..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/vpc/variables.tf +++ /dev/null @@ -1,301 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "labels" { - description = "Labels to add to network resources that support labels. Key-value pairs of strings." - type = map(string) - default = {} - nullable = false -} - -variable "network_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" - type = string - default = null -} - -variable "subnetwork_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-primary-subnet\")" - type = string - default = null -} - -# tflint-ignore: terraform_unused_declarations -variable "subnetwork_size" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = number - default = null - validation { - condition = var.subnetwork_size == null - error_message = "subnetwork_size is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -variable "default_primary_subnetwork_size" { - description = "The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks" - type = number - default = 15 -} - -variable "region" { - description = "The default region for Cloud resources" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "network_address_range" { - description = "IP address range (CIDR) for global network" - type = string - default = "10.0.0.0/9" - - validation { - condition = can(cidrhost(var.network_address_range, 0)) - error_message = "IP address range must be in CIDR format." - } -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "subnetworks" { - description = <<-EOT - List of subnetworks to create within the VPC. If left empty, it will be - replaced by a single, default subnetwork constructed from other parameters - (e.g. var.region). In all cases, the first subnetwork in the list is identified - by outputs as a "primary" subnetwork. - - subnet_name (string, required, name of subnet) - subnet_region (string, required, region of subnet) - subnet_ip (string, mutually exclusive with new_bits, CIDR-formatted IP range for subnetwork) - new_bits (number, mutually exclusive with subnet_ip, CIDR bits used to calculate subnetwork range) - subnet_private_access (bool, optional, Enable Private Access on subnetwork) - subnet_flow_logs (map(string), optional, Configure Flow Logs see terraform-google-network module) - description (string, optional, Description of Network) - purpose (string, optional, related to Load Balancing) - role (string, optional, related to Load Balancing) - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_name"]) - ]) - error_message = "All subnetworks must define \"subnet_name\"." - } - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_region"]) - ]) - error_message = "All subnetworks must define \"subnet_region\"." - } - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_ip"]) != can(s["new_bits"]) - ]) - error_message = "All subnetworks must define exactly one of \"subnet_ip\" or \"new_bits\"." - } - validation { - condition = alltrue([for s in var.subnetworks : can(s["subnet_ip"])]) || alltrue([for s in var.subnetworks : can(s["new_bits"])]) - error_message = "All subnetworks must make same choice of \"subnet_ip\" or \"new_bits\"." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "primary_subnetwork" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = map(string) - default = null - validation { - condition = var.primary_subnetwork == null - error_message = "primary_subnetwork is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "additional_subnetworks" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = list(map(string)) - default = null - validation { - condition = var.additional_subnetworks == null - error_message = "additional_subnetworks is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -variable "secondary_ranges" { - type = map(list(object({ range_name = string, ip_cidr_range = string }))) - description = <<-EOT - "Secondary ranges associated with the subnets. - This will be deprecated in favour of secondary_ranges_list at a later date. - Please migrate to using the same." - EOT - default = {} -} - -variable "secondary_ranges_list" { - type = list(object({ - subnetwork_name = string, - ranges = list(object({ - range_name = string, - ip_cidr_range = string - })) - })) - description = <<-EOT - "List of secondary ranges associated with the subnetworks. - Each subnetwork must be specified at most once in this list." - EOT - default = [] - validation { - condition = (length(var.secondary_ranges_list[*].subnetwork_name) == - length(distinct(var.secondary_ranges_list[*].subnetwork_name))) - error_message = "Each subnetwork should be specified at most once in this list. Remove any duplicates." - } -} - -variable "network_routing_mode" { - type = string - default = "GLOBAL" - description = "The network routing mode (default \"GLOBAL\")" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "ips_per_nat" { - type = number - description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples" - default = 2 -} - -variable "shared_vpc_host" { - type = bool - description = "Makes this project a Shared VPC host if 'true' (default 'false')" - default = false -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_iap_ssh_ingress" { - type = bool - description = "Enable a firewall rule to allow SSH access using IAP tunnels" - default = true -} - -variable "enable_iap_rdp_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" - default = false -} - -variable "enable_iap_winrm_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" - default = false -} - -variable "enable_internal_traffic" { - type = bool - description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" - default = true -} - -variable "enable_cloud_router" { - type = bool - description = "Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview" - default = true -} - -variable "enable_cloud_nat" { - type = bool - description = "Enable the creation of Cloud NATs." - default = true -} - -variable "extra_iap_ports" { - type = list(string) - description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" - default = [] -} - -variable "allowed_ssh_ip_ranges" { - type = list(string) - description = "A list of CIDR IP ranges from which to allow ssh access" - default = [] - - validation { - condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) - error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." - } -} - -variable "firewall_rules" { - type = any - description = "List of firewall rules" - default = [] -} - -variable "firewall_log_config" { - type = string - description = "Firewall log configuration for Toolkit firewall rules (var.enable_iap_ssh_ingress and others)" - default = "DISABLE_LOGGING" - nullable = false - - validation { - condition = contains([ - "INCLUDE_ALL_METADATA", - "EXCLUDE_ALL_METADATA", - "DISABLE_LOGGING", - ], var.firewall_log_config) - error_message = "var.firewall_log_config must be set to \"DISABLE_LOGGING\", or enable logging with \"INCLUDE_ALL_METADATA\" or \"EXCLUDE_ALL_METADATA\"" - } -} - -variable "network_profile" { - type = string - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown - EOT - default = null -} diff --git a/deletion-test/cluster/modules/embedded/modules/network/vpc/versions.tf b/deletion-test/cluster/modules/embedded/modules/network/vpc/versions.tf deleted file mode 100644 index 71b7106734..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/network/vpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 0.15.0" -} diff --git a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/README.md b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/README.md deleted file mode 100644 index 192d7575a5..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/README.md +++ /dev/null @@ -1,320 +0,0 @@ -# Custom Images in the Cluster Toolkit (formerly HPC Toolkit) - -Please review the -[introduction to image building](../../../docs/image-building.md) for general -information on building custom images using the Toolkit. - -## Introduction - -This module uses [Packer](https://www.packer.io/) to create an image within an -Cluster Toolkit deployment. Packer operates by provisioning a short-lived VM in -Google Cloud on which it executes scripts to customize the boot disk for -repeated use. The VM's boot disk is specified from a source image that defaults -to the [HPC VM Image][hpcimage]. This Packer "template" supports customization -by the following approaches following a [recommended use](#recommended-use): - -- [startup-script metadata][startup-metadata] from [raw string][sss] or - [file][ssf] -- [Shell scripts][shell] uploaded from the Packer execution environment to the - VM -- [Ansible playbooks][ansible] uploaded from the Packer execution environment to - the VM - -They can be specified independently of one another, so that anywhere from 1 to 3 -solutions can be used simultaneously. In the case that 0 scripts are supplied, -the source boot disk is effectively copied to your project without -customization. This can be useful in scenarios where increased control over the -image maintenance lifecycle is desired or when policies restrict the use of -images to internal projects. - -## Minimum requirements - -### Outbound internet access - -Most customization scripts require access to resources on the public internet. -This can be achieved by one of the following 2 approaches: - -1. Using a public IP address on the VM - -- Set [var.omit_external_ip](#input_omit_external_ip) to `false` - -1. Configuring a VPC with a Cloud NAT in the region of the VM - -- Use the [vpc] module which automates NAT creation - -### Inbound internet access - -Read [order of execution](#order-of-execution) below for a discussion of VM -customization solutions and their requirements for inbound SSH access. -[Environments without SSH access](#environments-without-ssh-access) should use -the metadata-based startup-script solution. - -A simple way to enable inbound SSH access is to use the VPC module with -`allowed_ssh_ip_ranges` set to `0.0.0.0/0`. - -### User or service account executing Packer at command line - -The user or service account running Packer must have the permission to create -VMs in the selected VPC network and, if [use\_iap](#input_use_iap) is set, must -have the "IAP-Secured Tunnel User" role. Recommended roles are: - -- `roles/compute.instanceAdmin.v1` -- `roles/iap.tunnelResourceAccessor` - -### VM service account roles - -The service account attached to the temporary build VM created by Packer should -have the ability to write Cloud Logging entries so that you may inspect and -debug build logs. When using the metadata startup-script customization solution, -the service account attached to the temporary build VM created by Packer must -have the permission to modify its own metadata and to read from Cloud Storage -buckets. Recommended roles are: - -- `roles/compute.instanceAdmin.v1` -- `roles/iam.serviceAccountUser` -- `roles/logging.logWriter` -- `roles/monitoring.metricWriter` -- `roles/storage.objectViewer` - -It is recommended to create this service account as a separate step outside a -blueprint due to known delay in [IAM bindings propagation][iamprop]. - -## Example blueprints - -A recommended pattern for building images with this module is to use the -terraform based [startup-script] module along with this packer custom-image -module. Below you can find links to several examples of this pattern, including -usage instructions. - -### [Image Builder] - -The [Image Builder] blueprint demonstrates a solution that builds an image -using: - -- The [HPC VM Image][hpcimage] as a base upon which to customize -- A VPC network with firewall rules that allow IAP-based SSH tunnels -- A Toolkit runner that installs a custom script - -Please review the [examples README] for usage instructions. - -## Order of execution - -The startup script specified in metadata executes in parallel with the other -supported methods. However, the remaining methods execute in a well-defined -order relative to one another. - -1. All shell scripts will execute in the configured order -1. After shell scripts complete, all Ansible playbooks will execute in the - configured order - -> **_NOTE:_** if both [startup_script][sss] and [startup_script_file][ssf] are -> specified, then [startup_script_file][ssf] takes precedence. - -## Recommended use - -Because the [metadata startup script executes in parallel](#order-of-execution) -with the other solutions, conflicts can arise, especially when package managers -(`yum` or `apt`) lock their databases during package installation. Therefore, it -is recommended to choose one of the following approaches: - -1. Specify _either_ [startup_script][sss] _or_ [startup_script_file][ssf] and do - not specify [shell_scripts][shell] or [ansible_playbooks][ansible]. - - This can be especially useful in - [environments that restrict SSH access](#environments-without-ssh-access) -1. Specify any combination of [shell_scripts][shell] and - [ansible_playbooks][ansible] and do not specify [startup_script][sss] or - [startup_script_file][ssf]. - -If any of the startup script approaches fail by returning a code other than 0, -Packer will determine that the build has failed and refuse to save the image. - -## External access with SSH - -The [shell scripts][shell] and [Ansible playbooks][ansible] customization -solutions both require SSH access to the VM from the Packer execution -environment. SSH access can be enabled one of 2 ways: - -1. The VM is created without a public IP address and SSH tunnels are created - using [Identity-Aware Proxy (IAP)][iaptunnel]. - - Allow [use_iap](#input_use_iap) to take on its default value of `true` -1. The VM is created with an IP address on the public internet and firewall - rules allow SSH access from the Packer execution environment. - - Set `omit_external_ip = false` (or `omit_external_ip: false` in a - blueprint) - - Add firewall rules that open SSH to the VM - -The Packer template defaults to using to the 1st IAP-based solution because it -is more secure (no exposure to public internet) and because the [vpc] module -automatically sets up all necessary firewall rules for SSH tunneling and -outbound-only access to the internet through [Cloud NAT][cloudnat]. - -In either SSH solution, customization scripts should be supplied as files in the -[shell_scripts][shell] and [ansible_playbooks][ansible] settings. - -## Environments without SSH access - -Many network environments disallow SSH access to VMs. In these environments, the -[metadata-based startup scripts][startup-metadata] are appropriate because they -execute entirely independently of the Packer execution environment. - -In this scenario, a single scripts should be supplied in the form of a string to -the [startup_script][sss] input variable. This solution integrates well with -Toolkit runners. Runners operate by using a single startup script whose behavior -is extended by downloading and executing a customizable set of runners from -Cloud Storage at startup. - -> **_NOTE:_** Packer will attempt to use SSH if either [shell_scripts][shell] or -> [ansible_playbooks][ansible] are set to non-empty values. Leave them at their -> default, empty values to ensure access by SSH is disabled. - -## Supplying startup script as a string - -The [startup_script][sss] parameter accepts scripts formatted as strings. In -Packer and Terraform, multi-line strings can be specified using -[heredoc syntax](https://www.terraform.io/language/expressions/strings#heredoc-strings) -in an input [Packer variables file][pkrvars] (`*.pkrvars.hcl`) For example, the -following snippet defines a multi-line bash script followed by an integer -representing the size, in GiB, of the resulting image: - -```hcl -startup_script = <<-EOT - #!/bin/bash - yum install -y epel-release - yum install -y jq - EOT - -disk_size = 100 -``` - -In a blueprint, the equivalent syntax is: - -```yaml -... - settings: - startup_script: | - #!/bin/bash - yum install -y epel-release - yum install -y jq - disk_size: 100 -... -``` - -## Monitoring startup script execution - -When using startup script customization, Packer will print very limited output -to the console. For example: - -```text -==> example.googlecompute.toolkit_image: Waiting for any running startup script to finish... -==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... -==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... -==> example.googlecompute.toolkit_image: Startup script, if any, has finished running. -``` - -### Debugging startup-script failures - -> [!NOTE] -> There can be a delay in the propagation of the logs from the instance to -> Cloud Logging, so it may require waiting a few minutes to see the full logs. - -If the Packer image build fails, the module will output a `gcloud` command -that can be used directly to review startup-script execution. - -## License - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. You may obtain a copy of the -License at - -```text - http://www.apache.org/licenses/LICENSE-2.0 -``` - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. - - -## Requirements - -No requirements. - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_count](#input\_accelerator\_count) | Number of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `number` | `null` | no | -| [accelerator\_type](#input\_accelerator\_type) | Type of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `string` | `null` | no | -| [ansible\_playbooks](#input\_ansible\_playbooks) | A list of Ansible playbook configurations that will be uploaded to customize the VM image | 
list(object({
    playbook_file   = string
    galaxy_file     = string
    extra_arguments = list(string)
  }))
| `[]` | no | -| [communicator](#input\_communicator) | Communicator to use for provisioners that require access to VM ("ssh" or "winrm") | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name | `string` | n/a | yes | -| [disk\_size](#input\_disk\_size) | Size of disk image in GB | `number` | `null` | no | -| [disk\_type](#input\_disk\_type) | Type of persistent disk to provision | `string` | `"pd-balanced"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [image\_family](#input\_image\_family) | The family name of the image to be built. Defaults to `deployment_name` | `string` | `null` | no | -| [image\_name](#input\_image\_name) | The name of the image to be built. If not supplied, it will be set to image\_family-$ISO\_TIMESTAMP | `string` | `null` | no | -| [image\_storage\_locations](#input\_image\_storage\_locations) | Storage location, either regional or multi-regional, where snapshot content is to be stored and only accepts 1 value.
See https://developer.hashicorp.com/packer/plugins/builders/googlecompute#image_storage_locations | `list(string)` | `null` | no | -| [labels](#input\_labels) | Labels to apply to the short-lived VM | `map(string)` | `null` | no | -| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2-standard-4"` | no | -| [manifest\_file](#input\_manifest\_file) | File to which to write Packer build manifest | `string` | `"packer-manifest.json"` | no | -| [metadata](#input\_metadata) | Instance metadata for the builder VM (use var.startup\_script or var.startup\_script\_file to set startup-script metadata) | `map(string)` | `{}` | no | -| [network\_project\_id](#input\_network\_project\_id) | Project ID of Shared VPC network | `string` | `null` | no | -| [omit\_external\_ip](#input\_omit\_external\_ip) | Provision the image building VM without a public IP address | `bool` | `true` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which to create VM and image | `string` | n/a | yes | -| [scopes](#input\_scopes) | DEPRECATED: use var.service\_account\_scopes | `set(string)` | `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | The service account email to use. If null or 'default', then the default Compute Engine service account will be used. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Service account scopes to attach to the instance. See
https://cloud.google.com/compute/docs/access/service-accounts. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shell\_scripts](#input\_shell\_scripts) | A list of paths to local shell scripts which will be uploaded to customize the VM image | `list(string)` | `[]` | no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [source\_image](#input\_source\_image) | Source OS image to build from | `string` | `null` | no | -| [source\_image\_family](#input\_source\_image\_family) | Alternative to source\_image. Specify image family to build from latest image in family | `string` | `"hpc-rocky-linux-8"` | no | -| [source\_image\_project\_id](#input\_source\_image\_project\_id) | A list of project IDs to search for the source image. Packer will search the
first project ID in the list first, and fall back to the next in the list,
until it finds the source image. | `list(string)` | `null` | no | -| [ssh\_username](#input\_ssh\_username) | Username to use for SSH access to VM | `string` | `"hpc-toolkit-packer"` | no | -| [startup\_script](#input\_startup\_script) | Startup script (as raw string) used to build the custom Linux VM image (overridden by var.startup\_script\_file if both are set) | `string` | `null` | no | -| [startup\_script\_file](#input\_startup\_script\_file) | File path to local shell script that will be used to customize the Linux VM image (overrides var.startup\_script) | `string` | `null` | no | -| [state\_timeout](#input\_state\_timeout) | The time to wait for instance state changes, including image creation | `string` | `"10m"` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of subnetwork in which to provision image building VM | `string` | n/a | yes | -| [tags](#input\_tags) | Assign network tags to apply firewall rules to VM instance | `list(string)` | `null` | no | -| [use\_iap](#input\_use\_iap) | Use IAP proxy when connecting by SSH | `bool` | `true` | no | -| [use\_os\_login](#input\_use\_os\_login) | Use OS Login when connecting by SSH | `bool` | `false` | no | -| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | A list of strings containing PowerShell scripts which will customize a Windows VM image (requires WinRM communicator) | `list(string)` | `[]` | no | -| [wrap\_startup\_script](#input\_wrap\_startup\_script) | Wrap startup script with Packer-generated wrapper | `bool` | `true` | no | -| [zone](#input\_zone) | Cloud zone in which to provision image building VM | `string` | n/a | yes | - -## Outputs - -No outputs. - - -[ansible]: #input_ansible_playbooks -[cloudnat]: https://cloud.google.com/nat/docs/overview -[examples readme]: ../../../examples/README.md#image-builderyaml- -[hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[iamprop]: https://cloud.google.com/iam/docs/access-change-propagation -[iaptunnel]: https://cloud.google.com/iap/docs/using-tcp-forwarding -[image builder]: ../../../examples/image-builder.yaml -[logging-console]: https://console.cloud.google.com/logs/ -[logging-read-docs]: https://cloud.google.com/sdk/gcloud/reference/logging/read -[pkrvars]: https://www.packer.io/guides/hcl/variables#from-a-file -[shell]: #input_shell_scripts -[ssf]: #input_startup_script_file -[sss]: #input_startup_script -[startup-metadata]: https://cloud.google.com/compute/docs/instances/startup-scripts/linux -[startup-script]: ../../../modules/scripts/startup-script -[vpc]: ../../network/vpc/README.md diff --git a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/image.pkr.hcl b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/image.pkr.hcl deleted file mode 100644 index 9282cf7433..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/image.pkr.hcl +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "custom-image", ghpc_role = "packer" }) - - # construct a unique image name from the image family - image_family = var.image_family != null ? var.image_family : var.deployment_name - image_name_default = "${local.image_family}-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" - image_name = var.image_name != null ? var.image_name : local.image_name_default - - # construct vm image name for use when getting logs - instance_name = "packer-${substr(uuidv4(), 0, 6)}" - - # default to explicit var.communicator, otherwise in-order: ssh/winrm/none - shell_script_communicator = length(var.shell_scripts) > 0 ? "ssh" : "" - ansible_playbook_communicator = length(var.ansible_playbooks) > 0 ? "ssh" : "" - powershell_script_communicator = length(var.windows_startup_ps1) > 0 ? "winrm" : "" - communicator = coalesce( - var.communicator, - local.shell_script_communicator, - local.ansible_playbook_communicator, - local.powershell_script_communicator, - "none" - ) - - # must not enable IAP when no communicator is in use - use_iap = local.communicator == "none" ? false : var.use_iap - - # construct metadata from startup_script and metadata variables - startup_script_metadata = var.startup_script == null ? {} : { startup-script = var.startup_script } - - linux_user_metadata = { - block-project-ssh-keys = "TRUE" - shutdown-script = <<-EOT - #!/bin/bash - userdel -r ${var.ssh_username} - sed -i '/${var.ssh_username}/d' /var/lib/google/google_users - EOT - } - windows_packer_user = "packer_user" - windows_user_metadata = { - sysprep-specialize-script-cmd = "winrm quickconfig -quiet & net user /add ${local.windows_packer_user} & net localgroup administrators ${local.windows_packer_user} /add & winrm set winrm/config/service/auth @{Basic=\\\"true\\\"}" - windows-shutdown-script-cmd = <<-EOT - net user /delete ${local.windows_packer_user} - EOT - } - user_metadata = local.communicator == "winrm" ? local.windows_user_metadata : local.linux_user_metadata - - # merge metadata such that var.metadata always overrides user management - # metadata but always allow var.startup_script to override var.metadata - metadata = merge( - local.user_metadata, - var.metadata, - local.startup_script_metadata, - ) - - # determine best value for on_host_maintenance if not supplied by user - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - gpu_attached = contains(["a2", "g2"], local.machine_family) || var.accelerator_type != null - on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" - on_host_maintenance = ( - var.on_host_maintenance != null - ? var.on_host_maintenance - : local.on_host_maintenance_default - ) - - accelerator_type = var.accelerator_type == null ? null : "projects/${var.project_id}/zones/${var.zone}/acceleratorTypes/${var.accelerator_type}" - - winrm_username = local.communicator == "winrm" ? "packer_user" : null - winrm_insecure = local.communicator == "winrm" ? true : null - winrm_use_ssl = local.communicator == "winrm" ? true : null - - enable_integrity_monitoring = var.enable_shielded_vm && var.shielded_instance_config.enable_integrity_monitoring - enable_secure_boot = var.enable_shielded_vm && var.shielded_instance_config.enable_secure_boot - enable_vtpm = var.enable_shielded_vm && var.shielded_instance_config.enable_vtpm - - image_licenses = [ - "projects/click-to-deploy-images/global/licenses/hpc-toolkit-vm-image" - ] -} - -source "googlecompute" "toolkit_image" { - communicator = local.communicator - project_id = var.project_id - image_name = local.image_name - image_family = local.image_family - image_labels = local.labels - instance_name = local.instance_name - machine_type = var.machine_type - accelerator_type = local.accelerator_type - accelerator_count = var.accelerator_count - on_host_maintenance = local.on_host_maintenance - disk_size = var.disk_size - disk_type = var.disk_type - omit_external_ip = var.omit_external_ip - use_internal_ip = var.omit_external_ip - subnetwork = var.subnetwork_name - network_project_id = var.network_project_id - service_account_email = var.service_account_email - scopes = var.service_account_scopes - source_image = var.source_image - source_image_family = var.source_image_family - source_image_project_id = var.source_image_project_id - ssh_username = var.ssh_username - tags = var.tags - use_iap = local.use_iap - use_os_login = var.use_os_login - winrm_username = local.winrm_username - winrm_insecure = local.winrm_insecure - winrm_use_ssl = local.winrm_use_ssl - zone = var.zone - labels = local.labels - metadata = local.metadata - startup_script_file = var.startup_script_file - wrap_startup_script = var.wrap_startup_script - state_timeout = var.state_timeout - image_storage_locations = var.image_storage_locations - enable_secure_boot = local.enable_secure_boot - enable_vtpm = local.enable_vtpm - enable_integrity_monitoring = local.enable_integrity_monitoring - image_licenses = local.image_licenses -} - -build { - name = var.deployment_name - sources = ["sources.googlecompute.toolkit_image"] - - # using dynamic blocks to create provisioners ensures that there are no - # provisioner blocks when none are provided and we can use the none - # communicator when using startup-script - - # provisioner "shell" blocks - dynamic "provisioner" { - labels = ["shell"] - for_each = var.shell_scripts - content { - execute_command = "sudo -H sh -c '{{ .Vars }} {{ .Path }}'" - script = provisioner.value - } - } - - # provisioner "powershell" blocks - dynamic "provisioner" { - labels = ["powershell"] - for_each = var.windows_startup_ps1 - content { - inline = split("\n", provisioner.value) - } - } - - dynamic "provisioner" { - labels = ["powershell"] - for_each = length(var.windows_startup_ps1) > 0 ? [1] : [] - content { - inline = [ - "GCESysprep -no_shutdown" - ] - } - } - - # provisioner "ansible-local" blocks - # this installs custom roles/collections from ansible-galaxy in /home/packer - # which will be removed at the end; consider modifying /etc/ansible/ansible.cfg - dynamic "provisioner" { - labels = ["ansible-local"] - for_each = var.ansible_playbooks - content { - playbook_file = provisioner.value.playbook_file - galaxy_file = provisioner.value.galaxy_file - extra_arguments = provisioner.value.extra_arguments - } - } - - post-processor "manifest" { - output = var.manifest_file - strip_path = true - custom_data = { - built-by = "cloud-hpc-toolkit" - } - } - - # If there is an error during image creation, print out command for getting packer VM logs - error-cleanup-provisioner "shell-local" { - environment_vars = [ - "PRJ_ID=${var.project_id}", - "INST_NAME=${local.instance_name}", - "ZONE=${var.zone}", - ] - inline_shebang = "/bin/bash -e" - inline = [ - "type -P gcloud > /dev/null || exit 0", - "INST_ID=$(gcloud compute instances describe $INST_NAME --project $PRJ_ID --format=\"value(id)\" --zone=$ZONE)", - "echo 'Error building image try checking logs:'", - join(" ", ["echo \"gcloud logging --project $PRJ_ID read", - "'logName=(\\\"projects/$PRJ_ID/logs/GCEMetadataScripts\\\" OR \\\"projects/$PRJ_ID/logs/google_metadata_script_runner\\\") AND resource.labels.instance_id=$INST_ID'", - "--format=\\\"table(timestamp, resource.labels.instance_id, jsonPayload.message)\\\"", - "--order=asc\"" - ] - ) - ] - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/metadata.yaml deleted file mode 100644 index 23108c4e17..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - logging.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/variables.pkr.hcl b/deletion-test/cluster/modules/embedded/modules/packer/custom-image/variables.pkr.hcl deleted file mode 100644 index 3cede102ce..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/packer/custom-image/variables.pkr.hcl +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "deployment_name" { - description = "Cluster Toolkit deployment name" - type = string -} - -variable "project_id" { - description = "Project in which to create VM and image" - type = string -} - -variable "machine_type" { - description = "VM machine type on which to build new image" - type = string - default = "n2-standard-4" -} - -variable "disk_size" { - description = "Size of disk image in GB" - type = number - default = null -} - -variable "disk_type" { - description = "Type of persistent disk to provision" - type = string - default = "pd-balanced" -} - -variable "zone" { - description = "Cloud zone in which to provision image building VM" - type = string -} - -variable "network_project_id" { - description = "Project ID of Shared VPC network" - type = string - default = null -} - -variable "subnetwork_name" { - description = "Name of subnetwork in which to provision image building VM" - type = string -} - -variable "omit_external_ip" { - description = "Provision the image building VM without a public IP address" - type = bool - default = true -} - -variable "tags" { - description = "Assign network tags to apply firewall rules to VM instance" - type = list(string) - default = null -} - -variable "image_family" { - description = "The family name of the image to be built. Defaults to `deployment_name`" - type = string - default = null -} - -variable "image_name" { - description = "The name of the image to be built. If not supplied, it will be set to image_family-$ISO_TIMESTAMP" - type = string - default = null -} - -variable "source_image_project_id" { - description = < -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 4.0 | -| [local](#requirement\_local) | >= 2.0.0 | -| [null](#requirement\_null) | ~> 3.0 | -| [random](#requirement\_random) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.0 | -| [local](#provider\_local) | >= 2.0.0 | -| [null](#provider\_null) | ~> 3.0 | -| [random](#provider\_random) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance\_template](#module\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [local_file.submit_script](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [null_resource.submit_job](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.submit_job_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used for the job\_id | `string` | n/a | yes | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs | `bool` | `true` | no | -| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions. Valid inputs are `"alpha"`, `"beta"` and "" (empty string for default version) | `string` | `""` | no | -| [image](#input\_image) | DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided. | `any` | `null` | no | -| [instance\_image](#input\_instance\_image) | Google Cloud Batch compute node image. Ignored if `instance_template` is provided.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [instance\_template](#input\_instance\_template) | Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions. | `string` | `null` | no | -| [job\_filename](#input\_job\_filename) | The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified | `string` | `null` | no | -| [job\_id](#input\_job\_id) | An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs. | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided. | `map(string)` | n/a | yes | -| [log\_policy](#input\_log\_policy) | Create a block to define log policy.
When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging.
When set to `PATH`, path must be added to generated template.
When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. | `string` | `"CLOUD_LOGGING"` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided. | `string` | `"n2-standard-4"` | no | -| [mpi\_mode](#input\_mpi\_mode) | Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`. | `bool` | `false` | no | -| [native\_batch\_mounting](#input\_native\_batch\_mounting) | Batch can mount some fs\_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts. | `bool` | `true` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which to run the Google Cloud Batch job | `string` | n/a | yes | -| [runnable](#input\_runnable) | A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`. | `string` | `null` | no | -| [runnables](#input\_runnables) | A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template. | 
list(object({
    script = string
  }))
| `null` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/devstorage.read_only",
    "https://www.googleapis.com/auth/logging.write",
    "https://www.googleapis.com/auth/monitoring.write",
    "https://www.googleapis.com/auth/servicecontrol",
    "https://www.googleapis.com/auth/service.management.readonly",
    "https://www.googleapis.com/auth/trace.append"
  ]
}
| no | -| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided. | `string` | `null` | no | -| [submit](#input\_submit) | When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply. | `bool` | `false` | no | -| [subnetwork](#input\_subnetwork) | The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided. | `any` | `null` | no | -| [task\_count](#input\_task\_count) | Number of parallel tasks | `number` | `1` | no | -| [task\_count\_per\_node](#input\_task\_count\_per\_node) | Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value. | `number` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [gcloud\_version](#output\_gcloud\_version) | The version of gcloud to be used. | -| [instance\_template](#output\_instance\_template) | Instance template used by the Batch job. | -| [instructions](#output\_instructions) | Instructions for submitting the Batch job. | -| [job\_data](#output\_job\_data) | All data associated with the defined job, typically provided as input to clout-batch-login-node. | -| [network\_storage](#output\_network\_storage) | An array of network attached storage mounts used by the Batch job. | -| [startup\_script](#output\_startup\_script) | Startup script run before Google Cloud Batch job starts. | - diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/main.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/main.tf deleted file mode 100644 index 0d681536c9..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/main.tf +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "batch-job-template", ghpc_role = "scheduler" }) -} - -locals { - instance_template = coalesce(var.instance_template, module.instance_template.self_link) - - tasks_per_node = var.task_count_per_node != null ? var.task_count_per_node : (var.mpi_mode ? 1 : null) - - one_line_runnable = coalesce(var.runnable, "## Add your workload here ##") - runnables = coalesce(var.runnables, [{ script = local.one_line_runnable }]) - - job_template_contents = templatefile( - "${path.module}/templates/batch-job-base.yaml.tftpl", - { - synchronized = var.mpi_mode - runnables = local.runnables - task_count = var.task_count - tasks_per_node = local.tasks_per_node - require_hosts_file = var.mpi_mode - permissive_ssh = var.mpi_mode - log_policy = var.log_policy - instance_template = local.instance_template - nfs_volumes = local.native_batch_network_storage - labels = local.labels - } - ) - - submit_job_id = "${var.job_id}-${random_id.submit_job_suffix.hex}" - job_filename = coalesce(var.job_filename, "${var.job_id}.yaml") - job_template_output_path = "${path.root}/${local.job_filename}" - - submit_script_contents = templatefile( - "${path.module}/templates/batch-submit.sh.tftpl", - { - project = var.project_id - location = var.region - config = local_file.job_template.filename - submit_job_id = local.submit_job_id - } - ) - submit_script_output_path = "${path.root}/submit-${var.job_id}.sh" - - subnetwork_name = var.subnetwork != null ? var.subnetwork.name : "default" - subnetwork_project = var.subnetwork != null ? var.subnetwork.project : var.project_id - - # Filter network_storage for native Batch support - native_fstype = var.native_batch_mounting ? ["nfs"] : [] - native_batch_network_storage = [ - for ns in var.network_storage : - ns if contains(local.native_fstype, ns.fs_type) - ] - # other processing happens in startup_from_network_storage.tf - - # this code is similar to code in Packer and vm-instance modules - # it differs in that this module does not (yet) expose var.guest_acclerator - # for attaching GPUs to N1 VMs. For now, identify only A2 types. - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - gpu_attached = contains(["a2", "g2"], local.machine_family) - on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" - - on_host_maintenance = coalesce(var.on_host_maintenance, local.on_host_maintenance_default) - - network_storage_metadata = var.network_storage != null ? ({ network_storage = jsonencode(var.network_storage) }) : {} - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.network_storage_metadata, - local.disable_automatic_updates_metadata - ) -} - -module "instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = var.instance_template == null ? "${var.job_id}-instance-template" : "unused-template" - project_id = var.project_id - subnetwork = local.subnetwork_name - subnetwork_project = local.subnetwork_project - service_account = var.service_account - access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - labels = local.labels - - machine_type = var.machine_type - startup_script = local.startup_from_network_storage - metadata = local.metadata - source_image_family = data.google_compute_image.compute_image.family - source_image = data.google_compute_image.compute_image.name - source_image_project = data.google_compute_image.compute_image.project - on_host_maintenance = local.on_host_maintenance -} - -resource "local_file" "job_template" { - content = local.job_template_contents - filename = local.job_template_output_path - - lifecycle { - precondition { - condition = var.runnable == null || var.runnables == null - error_message = "var.runnable and var.runnables (plural) cannot both be set." - } - } -} - -resource "random_id" "submit_job_suffix" { - byte_length = 4 - keepers = { - always_run = timestamp() - } -} - -resource "local_file" "submit_script" { - content = local.submit_script_contents - filename = local.submit_script_output_path -} - -resource "null_resource" "submit_job" { - depends_on = [local_file.job_template, local_file.submit_script] - count = var.submit ? 1 : 0 - - # A new deployment should always submit a new job. Old finished jobs aren't persistent parts of - # Cloud infrastructure. - triggers = { - always_run = timestamp() - } - - provisioner "local-exec" { - command = local.submit_script_output_path - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml deleted file mode 100644 index 387e810962..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - batch.googleapis.com - - compute.googleapis.com -ghpc: - inject_module_id: job_id diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/outputs.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/outputs.tf deleted file mode 100644 index 0b1295975a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/outputs.tf +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - provided_instance_tpl_msg = "The Batch job template uses the existing VM instance template:" - generated_instance_tpl_msg = "The Batch job template uses a new VM instance template created matching the provided settings:" - submit_msg = <<-EOT - - The job has been submitted. See job status at: - https://console.cloud.google.com/batch/jobsDetail/regions/${var.region}/jobs/${local.submit_job_id}?project=${var.project_id} - EOT -} - -output "instructions" { - description = "Instructions for submitting the Batch job." - value = <<-EOT - - A Batch job template file has been created locally at: - ${abspath(local.job_template_output_path)} - - ${var.instance_template == null ? local.generated_instance_tpl_msg : local.provided_instance_tpl_msg} - ${local.instance_template} - ${var.submit ? local.submit_msg : ""} - - Use the following commands to: - Submit your job${var.submit ? " (Note: job has already been submitted)" : ""}: - gcloud ${var.gcloud_version} batch jobs submit ${local.submit_job_id} --config=${abspath(local.job_template_output_path)} --location=${var.region} --project=${var.project_id} - - Check status: - gcloud ${var.gcloud_version} batch jobs describe ${local.submit_job_id} --location=${var.region} --project=${var.project_id} | grep state: - - Delete job: - gcloud ${var.gcloud_version} batch jobs delete ${local.submit_job_id} --location=${var.region} --project=${var.project_id} - - List all jobs: - gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} - EOT -} - -output "job_data" { - description = "All data associated with the defined job, typically provided as input to clout-batch-login-node." - value = { - template_contents = local.job_template_contents, - filename = local.job_filename, - id = local.submit_job_id - } -} - -output "instance_template" { - description = "Instance template used by the Batch job." - value = local.instance_template -} - -output "network_storage" { - description = "An array of network attached storage mounts used by the Batch job." - value = var.network_storage -} - -output "startup_script" { - description = "Startup script run before Google Cloud Batch job starts." - value = var.startup_script -} - -output "gcloud_version" { - description = "The version of gcloud to be used." - value = var.gcloud_version -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf deleted file mode 100644 index 02bc58e4f7..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "inputs": -# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. -# var.network_storage : to be passed into metadata somewhere else (not here) -# var.startup_script : to be changed into a more complete file system with all the fs runners - -# "outputs": -# local.startup_from_network_storage : A full startup script with all the runners that are not supported -# natively and were included in the network_storage structure - -locals { - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - # Pull out runners to include in startup script - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - startup_script_runner = [{ - content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" - destination = "passed_startup_script.sh" - type = "shell" - }] - - full_runner_list = concat( - local.storage_client_install_runners, - local.mount_runners, - local.startup_script_runner - ) - - startup_from_network_storage = module.netstorage_startup_script.startup_script -} - -module "netstorage_startup_script" { - source = "../../scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.full_runner_list -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl deleted file mode 100644 index 83fccde53b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl +++ /dev/null @@ -1,53 +0,0 @@ -taskGroups: - - taskSpec: - runnables: - %{~ if synchronized ~} - - barrier: - name: "wait-for-node-startup" - %{~ endif ~} - %{~ for runnable in runnables ~} - - script: - text: ${indent(12, chomp(yamlencode(runnable.script)))} - %{~ if synchronized ~} - - barrier: - name: "wait-for-script-to-complete" - %{~ endif ~} - %{~ endfor ~} - %{~ if length(nfs_volumes) > 0 ~} - volumes: - %{~ for index, vol in nfs_volumes ~} - - nfs: - server: "${vol.server_ip}" - remotePath: "${vol.remote_mount}" - %{~ if vol.mount_options != "" && vol.mount_options != null ~} - mountOptions: "${vol.mount_options}" - %{~ endif ~} - mountPath: "${vol.local_mount}" - %{~ endfor ~} - %{~ endif ~} - taskCount: ${task_count} - %{~ if tasks_per_node != null ~} - taskCountPerNode: ${tasks_per_node} - %{~ endif ~} - requireHostsFile: ${require_hosts_file} - permissiveSsh: ${permissive_ssh} -%{~ if instance_template != null } -allocationPolicy: - instances: - - instanceTemplate: "${instance_template}" -%{~ endif } -%{~ if log_policy == "CLOUD_LOGGING" } -logsPolicy: - destination: "CLOUD_LOGGING" -%{ endif } -%{~ if log_policy == "PATH" } -logsPolicy: - destination: "PATH" - logsPath: ## Add logging path here -%{ endif } -%{~ if length(labels) > 0 ~} -labels: -%{ for k, v in labels ~} - ${k}: "${v}" -%{ endfor } -%{~ endif ~} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl deleted file mode 100644 index 25f89c3ceb..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e -o pipefail -GCLOUD_MAJOR_VERSION=$(gcloud --version | head -n 1 | awk '{print $NF}' | cut -f1 --delimiter=.) -if [ $((GCLOUD_MAJOR_VERSION >= 461)) ]; then - gcloud batch jobs submit ${submit_job_id} --project=${project} --location=${location} --config=${config} - echo "batch job ${submit_job_id} successfully submitted" -else - echo "gcloud must be updated to version 461.0.0 or later." - exit 1 -fi diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/variables.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/variables.tf deleted file mode 100644 index f65fbd111e..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/variables.tf +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "region" { - description = "The region in which to run the Google Cloud Batch job" - type = string -} - -variable "deployment_name" { - description = "Name of the deployment, used for the job_id" - type = string -} - -variable "labels" { - description = "Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided." - type = map(string) -} - -variable "job_id" { - description = "An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs." - type = string -} - -variable "job_filename" { - description = "The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified" - type = string - default = null -} - -variable "gcloud_version" { - description = "The version of the gcloud cli being used. Used for output instructions. Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default version)" - type = string - default = "" - - validation { - condition = contains(["alpha", "beta", ""], var.gcloud_version) - error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." - } -} - -variable "task_count" { - description = "Number of parallel tasks" - type = number - default = 1 -} - -variable "task_count_per_node" { - description = "Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value." - type = number - default = null -} - -variable "mpi_mode" { - description = "Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`." - type = bool - default = false -} - -variable "log_policy" { - description = <<-EOT - Create a block to define log policy. - When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging. - When set to `PATH`, path must be added to generated template. - When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. - EOT - type = string - default = "CLOUD_LOGGING" - - validation { - condition = contains(["CLOUD_LOGGING", "PATH", "DESTINATION_UNSPECIFIED"], var.log_policy) - error_message = "Allowed values for log_policy are 'CLOUD_LOGGING', 'PATH', or 'DESTINATION_UNSPECIFIED'." - } -} - -variable "runnables" { - description = "A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template." - type = list(object({ - script = string - })) - default = null -} - -variable "runnable" { - description = "A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`." - type = string - default = null -} - -variable "instance_template" { - description = "Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions." - type = string - default = null -} - -variable "subnetwork" { - description = "The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided." - type = any - default = null -} - -variable "enable_public_ips" { - description = "If set to true, instances will have public IPs" - type = bool - default = true -} - -variable "service_account" { - description = "Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided." - type = object({ - email = string, - scopes = set(string) - }) - default = { - email = null - scopes = [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write", - "https://www.googleapis.com/auth/servicecontrol", - "https://www.googleapis.com/auth/service.management.readonly", - "https://www.googleapis.com/auth/trace.append" - ] - } -} - -variable "machine_type" { - description = "Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided." - type = string - default = "n2-standard-4" -} - -variable "startup_script" { - description = "Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided." - type = string - default = null -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided." - type = list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "native_batch_mounting" { - description = "Batch can mount some fs_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts." - type = bool - default = true -} - -# Deprecated, replaced by instance_image -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided." - type = any - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - Google Cloud Batch compute node image. Ignored if `instance_template` is provided. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE`" - type = string - default = null - validation { - condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) - error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." - } -} - -variable "submit" { - description = "When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply." - type = bool - default = false -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/versions.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/versions.tf deleted file mode 100644 index a1161e1354..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-job-template/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - random = { - source = "hashicorp/random" - version = ">= 3.0" - } - google = { - source = "hashicorp/google" - version = ">= 4.0" - } - } - required_version = ">= 1.1" -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/README.md b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/README.md deleted file mode 100644 index c20ca7dbeb..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# Description - -This module creates a VM that acts as a login node to test and submit Google -Cloud Batch jobs. It is intended to be used along with the `batch-job-template` -module. - -This login node: - -- Uses the same VM settings as the first provided `batch-job-template`, such as - image, machine type, etc... -- Runs the same `startup-script` as the first provided `batch-job-template`. -- Has the same mounted file systems as the provided `batch-job-template`. -- Contains a folder with job templates generated by `batch-job-template` modules. - -Since the login node has the same mounted storage and is a homogeneous machine -to the Google Cloud Batch compute VMs, it can be used to inspect shared file -systems and test installed software before submitting a Google Cloud Batch job. - -## Example - -```yaml -- id: batch-job - source: modules/scheduler/batch-job-template - ... - -- id: batch-login - source: modules/scheduler/batch-login-node - use: [batch-job] - outputs: [instructions] -``` - -## Authentication - -To submit jobs from the login node, the service account attached to the VM needs -the `Batch Job Administrator` role. In most cases this service account will be -the Compute Engine default service account and will not be granted this role by -default. - -You can grant this role either by adding the `Batch Job Administrator` role to -the service account in the IAM page in the Google Cloud Console, or by running -the following command line: - -```bash -gcloud projects add-iam-policy-binding \ - --member=serviceAccount: \ - --role=roles/batch.jobsAdmin -``` - -## gcloud Batch Access - -Until the Google Cloud Batch API is generally available (GA), it may not be -available in all versions of the `gcloud` cli. You can test if the Google Cloud -Batch commands are available by running `gcloud [alpha|beta|] batch -h`. If the -Google Cloud Batch cli is not available it can generally be mitigated by either -updating `gcloud` by running `gcloud components update`, or using an image that -contains a more recent version of `gcloud`. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [login\_startup\_script](#module\_login\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_from_template.batch_login](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | -| [google_compute_instance_template.batch_instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [batch\_job\_directory](#input\_batch\_job\_directory) | The path of the directory on the login node in which to place the Google Cloud Batch job template | `string` | `"/home/batch-jobs"` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, also used for the job\_id | `string` | n/a | yes | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions.
Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default
version). Typically supplied by a batch-job-template module. If multiple
batch-job-template modules supply the gcloud\_version, only the first will be used. | `string` | `""` | no | -| [instance\_template](#input\_instance\_template) | Login VM instance template self-link. Typically supplied by a
batch-job-template module. If multiple batch-job-template modules supply the
instance\_template, the first will be used. | `string` | n/a | yes | -| [job\_data](#input\_job\_data) | List of jobs and supporting data for each, typically provided via "use" from the batch-job-template module. | 
list(object({
    template_contents = string,
    filename          = string,
    id                = string
  }))
| n/a | yes | -| [job\_filename](#input\_job\_filename) | Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [job\_id](#input\_job\_id) | Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions. | `string` | `null` | no | -| [job\_template\_contents](#input\_job\_template\_contents) | Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the login node. Key-value pairs | `map(string)` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which to create the login node | `string` | n/a | yes | -| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [zone](#input\_zone) | The zone in which to create the login node | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for accessing the login node and submitting Google Cloud Batch jobs | -| [login\_node\_name](#output\_login\_node\_name) | Name of the created VM | - diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/main.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/main.tf deleted file mode 100644 index 6f539af122..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/main.tf +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "batch-login-node", ghpc_role = "scheduler" }) -} - -data "google_compute_instance_template" "batch_instance_template" { - name = var.instance_template -} - -locals { - job_template_runners = [for job in var.job_data : { - content = job.template_contents - destination = "${var.batch_job_directory}/${job.filename}" - type = "data" - }] - - instance_template_metadata = data.google_compute_instance_template.batch_instance_template.metadata - startup_metadata = { startup-script = module.login_startup_script.startup_script } - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - login_metadata = merge(local.instance_template_metadata, local.startup_metadata, local.oslogin_metadata) - - batch_command_instructions = join("\n", [for job in var.job_data : <<-EOT - ## For job: ${job.id} ## - - Submit your job from login node: - gcloud ${var.gcloud_version} batch jobs submit ${job.id} --config=${var.batch_job_directory}/${job.filename} --location=${var.region} --project=${var.project_id} - - Check status: - gcloud ${var.gcloud_version} batch jobs describe ${job.id} --location=${var.region} --project=${var.project_id} | grep state: - - Delete job: - gcloud ${var.gcloud_version} batch jobs delete ${job.id} --location=${var.region} --project=${var.project_id} - - EOT - ]) - - list_all_jobs = <<-EOT - List all jobs: - gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} - EOT - - readme_contents = <<-EOT - # Batch Job Templates - - This folder contains Batch job templates created by the Cluster Toolkit. - These templates can be edited before submitting to Batch to capture more - complex workloads. - - Use the following commands to: - ${local.list_all_jobs} - - ${local.batch_command_instructions} - EOT - - # Construct startup script for network storage - storage_client_install_runners = [ - for i, ns in var.network_storage : merge(ns.client_install_runner, { - destination = "${i}-${ns.client_install_runner.destination}" - }) if ns.client_install_runner != null - ] - mount_runners = [ - for i, ns in var.network_storage : merge(ns.mount_runner, { - destination = "${i}-${ns.mount_runner.destination}" - }) if ns.mount_runner != null - ] - - startup_script_runner = { - content = var.startup_script != null ? var.startup_script : "echo 'Batch job template had no startup script'" - destination = "passed_startup_script.sh" - type = "shell" - } -} - -module "login_startup_script" { - source = "../../scripts/startup-script" - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = concat( - local.storage_client_install_runners, - local.mount_runners, - [local.startup_script_runner], - local.job_template_runners, - [ - { - content = local.readme_contents - destination = "${var.batch_job_directory}/README.md" - type = "data" - } - ] - ) -} - -resource "google_compute_instance_from_template" "batch_login" { - name = "${var.deployment_name}-batch-login" - source_instance_template = var.instance_template - project = var.project_id - zone = var.zone - metadata = local.login_metadata - - service_account { - scopes = ["https://www.googleapis.com/auth/cloud-platform"] - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml deleted file mode 100644 index 9af2319b4a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - batch.googleapis.com - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/outputs.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/outputs.tf deleted file mode 100644 index ea8eccf8d5..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/outputs.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "login_node_name" { - description = "Name of the created VM" - value = google_compute_instance_from_template.batch_login.name -} - -output "instructions" { - description = "Instructions for accessing the login node and submitting Google Cloud Batch jobs" - value = <<-EOT - - Batch job template files will be placed on the Batch login node in the following directory: - ${var.batch_job_directory} - - Use the following commands to: - SSH into the login node: - gcloud compute ssh --zone ${google_compute_instance_from_template.batch_login.zone} ${google_compute_instance_from_template.batch_login.name} --project ${google_compute_instance_from_template.batch_login.project} - - ${local.list_all_jobs} - - ${local.batch_command_instructions} - EOT -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/variables.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/variables.tf deleted file mode 100644 index 3b9caa7001..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/variables.tf +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "deployment_name" { - description = "Name of the deployment, also used for the job_id" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "region" { - description = "The region in which to create the login node" - type = string -} - -variable "zone" { - description = "The zone in which to create the login node" - type = string -} - -variable "labels" { - description = "Labels to add to the login node. Key-value pairs" - type = map(string) -} - -variable "instance_template" { - description = <<-EOT - Login VM instance template self-link. Typically supplied by a - batch-job-template module. If multiple batch-job-template modules supply the - instance_template, the first will be used. - EOT - type = string -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module." - type = list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "startup_script" { - description = "Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module." - type = string - default = null -} - -variable "job_data" { - description = "List of jobs and supporting data for each, typically provided via \"use\" from the batch-job-template module." - type = list(object({ - template_contents = string, - filename = string, - id = string - })) - validation { - condition = length(distinct([for job in var.job_data : job.filename])) == length(var.job_data) - error_message = "All filenames in var.job_data must be unique." - } - validation { - condition = length(distinct([for job in var.job_data : job.id])) == length(var.job_data) - error_message = "All job IDs in var.job_data must be unique." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_template_contents" { - description = "Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module." - type = string - default = null - validation { - condition = var.job_template_contents == null - error_message = "job_template_contents is deprecated. Please use `job_data` instead." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_filename" { - description = "Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module." - type = string - default = null - validation { - condition = var.job_filename == null - error_message = "job_filename is deprecated. Please use `job_data` instead." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_id" { - description = "Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions." - type = string - default = null - validation { - condition = var.job_id == null - error_message = "job_id is deprecated. Please use `job_data` instead." - } -} - -variable "gcloud_version" { - description = <<-EOT - The version of the gcloud cli being used. Used for output instructions. - Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default - version). Typically supplied by a batch-job-template module. If multiple - batch-job-template modules supply the gcloud_version, only the first will be used. - EOT - type = string - default = "" - - validation { - condition = contains(["alpha", "beta", ""], var.gcloud_version) - error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." - } -} - -variable "batch_job_directory" { - description = "The path of the directory on the login node in which to place the Google Cloud Batch job template" - type = string - default = "/home/batch-jobs" -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/versions.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/versions.tf deleted file mode 100644 index 15337a1d7b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/batch-login-node/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/README.md b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/README.md deleted file mode 100644 index dd4f7fdaa7..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/README.md +++ /dev/null @@ -1,220 +0,0 @@ -## Description - -This module creates a Google Kubernetes Engine -([GKE](https://cloud.google.com/kubernetes-engine)) cluster. - -### Example - -The following example creates a GKE cluster and a VPC designed to work with GKE. -See [VPC Network](#vpc-network) section for more information about network -requirements. - -```yaml - - id: network1 - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### VPC Network - -This module is configured to create a -[VPC-native cluster](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips). -This means that alias IPs are used and that the subnetwork requires secondary -ranges for pods and services. In the example shown above these secondary ranges -are created in the VPC module. By default the `gke-cluster` module will look for -ranges with the names `pods` and `services`. These names can be configured using -the `pods_ip_range_name` and `services_ip_range_name` settings. - -### Multi-networking - -To [enable Multi-networking](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment), pass multivpc module to gke-cluster module as described in example below. Passing a multivpc module enables multi networking and [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en) on the cluster. - -```yaml - - id: network - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: gke-cluster - source: modules/scheduler/gke-cluster - use: [network, multinetwork] ## enables multi networking and Dataplane V2 on cluster - settings: - cluster_name: $(vars.deployment_name) -``` - -Find an example of multi networking in GKE [here](../../../examples/gke-a3-megagpu.yaml). - -### Cluster Limitations - -The current implementations has the following limitations: - -- Autopilot is disabled -- Auto-provisioning of new node pools is disabled -- Network policies are not supported -- General addon configuration is not supported -- Only regional cluster is supported - -### GKE Inference Gateway - -Setting `enable_inference_gateway` to `true` will enable the `HttpLoadBalancing` -addon and deploy the Inference Gateway CRDs. This feature requires a subnet with -`purpose` set to `REGIONAL_MANAGED_PROXY` in the VPC. For more information, see -the [GKE Inference Gateway documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/serve-with-gke-inference-gateway). - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 7.2 | -| [google-beta](#requirement\_google-beta) | >= 7.2 | -| [kubernetes](#requirement\_kubernetes) | >= 2.36 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 7.2 | -| [google-beta](#provider\_google-beta) | >= 7.2 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | -| [workload\_identity](#module\_workload\_identity) | terraform-google-modules/kubernetes-engine/google//modules/workload-identity | >= 40.0 | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_cluster) | resource | -| [google-beta_google_container_node_pool.system_node_pools](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | -| [google-beta_google_container_engine_versions.version_prefix_filter](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/data-sources/google_container_engine_versions) | data source | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks enables multi networking and creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [authenticator\_security\_group](#input\_authenticator\_security\_group) | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| [autoscaling\_profile](#input\_autoscaling\_profile) | (Beta) Optimize for utilization or availability when deciding to remove nodes. Can be BALANCED or OPTIMIZE\_UTILIZATION. | `string` | `"OPTIMIZE_UTILIZATION"` | no | -| [cloud\_dns\_config](#input\_cloud\_dns\_config) | Configuration for Using Cloud DNS for GKE.

additive\_vpc\_scope\_dns\_domain: This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = "CLOUD\_DNS" and cluster\_dns\_scope = "CLUSTER\_SCOPE" must both be set as well.
cluster\_dns: Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS.
cluster\_dns\_scope: The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE.
cluster\_dns\_domain: The suffix used for all cluster service records. | 
object({
    additive_vpc_scope_dns_domain = optional(string)
    cluster_dns                   = optional(string, "PROVIDER_UNSPECIFIED")
    cluster_dns_scope             = optional(string, "DNS_SCOPE_UNSPECIFIED")
    cluster_dns_domain            = optional(string)
  })
| 
{
  "additive_vpc_scope_dns_domain": null,
  "cluster_dns": "PROVIDER_UNSPECIFIED",
  "cluster_dns_domain": null,
  "cluster_dns_scope": "DNS_SCOPE_UNSPECIFIED"
}
| no | -| [cluster\_availability\_type](#input\_cluster\_availability\_type) | Type of cluster availability. Possible values are: {REGIONAL, ZONAL} | `string` | `"REGIONAL"` | no | -| [cluster\_reference\_type](#input\_cluster\_reference\_type) | How the google\_container\_node\_pool.system\_node\_pools refers to the cluster. Possible values are: {SELF\_LINK, NAME} | `string` | `"SELF_LINK"` | no | -| [configure\_workload\_identity\_sa](#input\_configure\_workload\_identity\_sa) | When true, a kubernetes service account will be created and bound using workload identity to the service account used to create the cluster. | `bool` | `false` | no | -| [default\_max\_pods\_per\_node](#input\_default\_max\_pods\_per\_node) | The default maximum number of pods per node in this cluster. | `number` | `null` | no | -| [deletion\_protection](#input\_deletion\_protection) | "Determines if the cluster can be deleted by gcluster commands or not".
To delete a cluster provisioned with deletion\_protection set to true, you must first set it to false and apply the changes.
Then proceed with deletion as usual. | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`. | `string` | n/a | yes | -| [enable\_dataplane\_v2](#input\_enable\_dataplane\_v2) | Enables [Dataplane v2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2). This setting is immutable on clusters. If null, will default to false unless using multi-networking, in which case it will default to true | `bool` | `null` | no | -| [enable\_dcgm\_monitoring](#input\_enable\_dcgm\_monitoring) | Enable GKE to collect DCGM metrics | `bool` | `false` | no | -| [enable\_external\_dns\_endpoint](#input\_enable\_external\_dns\_endpoint) | Allow [DNS-based approach](https://cloud.google.com/kubernetes-engine/docs/concepts/network-isolation#dns-based_endpoint) for accessing the GKE control plane.
Refer this [dedicated blog](https://cloud.google.com/blog/products/containers-kubernetes/new-dns-based-endpoint-for-the-gke-control-plane) for more details. | `bool` | `false` | no | -| [enable\_filestore\_csi](#input\_enable\_filestore\_csi) | The status of the Filestore Container Storage Interface (CSI) driver addon, which allows the usage of filestore instance as volumes. | `bool` | `false` | no | -| [enable\_gcsfuse\_csi](#input\_enable\_gcsfuse\_csi) | The status of the GCSFuse Container Storage Interface (CSI) driver addon, which allows the usage of a GCS bucket as volumes. | `bool` | `false` | no | -| [enable\_inference\_gateway](#input\_enable\_inference\_gateway) | If true, enables GKE features required for Inference Gateway, including the HttpLoadBalancing addon, and installs required CRDs. | `bool` | `false` | no | -| [enable\_k8s\_beta\_apis](#input\_enable\_k8s\_beta\_apis) | List of Enabled Kubernetes Beta APIs. | `list(string)` | `null` | no | -| [enable\_managed\_lustre\_csi](#input\_enable\_managed\_lustre\_csi) | The status of the Google Compute Engine Managed Lustre Container Storage Interface (CSI) driver addon, which allows the usage of a lustre as volumes. | `bool` | `false` | no | -| [enable\_master\_global\_access](#input\_enable\_master\_global\_access) | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `false` | no | -| [enable\_multi\_networking](#input\_enable\_multi\_networking) | Enables [multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) (Requires GKE Enterprise). This setting is immutable on clusters and enables [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en). If null, will determine state based on if additional\_networks are passed in. | `bool` | `null` | no | -| [enable\_node\_local\_dns\_cache](#input\_enable\_node\_local\_dns\_cache) | Enable GKE NodeLocal DNSCache addon to improve DNS lookup latency | `bool` | `false` | no | -| [enable\_parallelstore\_csi](#input\_enable\_parallelstore\_csi) | The status of the Google Compute Engine Parallelstore Container Storage Interface (CSI) driver addon, which allows the usage of a parallelstore as volumes. | `bool` | `false` | no | -| [enable\_persistent\_disk\_csi](#input\_enable\_persistent\_disk\_csi) | The status of the Google Compute Engine Persistent Disk Container Storage Interface (CSI) driver addon, which allows the usage of a PD as volumes. | `bool` | `true` | no | -| [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | (Beta) Whether the master's internal IP address is used as the cluster endpoint. | `bool` | `true` | no | -| [enable\_private\_ipv6\_google\_access](#input\_enable\_private\_ipv6\_google\_access) | The private IPv6 google access type for the VMs in this subnet. | `bool` | `true` | no | -| [enable\_private\_nodes](#input\_enable\_private\_nodes) | (Beta) Whether nodes have internal IP addresses only. | `bool` | `true` | no | -| [enable\_ray\_operator](#input\_enable\_ray\_operator) | The status of the Ray operator addon, This feature enables Kubernetes APIs for managing and scaling Ray clusters and jobs. You control and are responsible for managing ray.io custom resources in your cluster. This feature is not compatible with GKE clusters that already have another Ray operator installed. Supports clusters on Kubernetes version 1.29.8-gke.1054000 or later. | `bool` | `false` | no | -| [gcp\_public\_cidrs\_access\_enabled](#input\_gcp\_public\_cidrs\_access\_enabled) | Whether the cluster master is accessible via all the Google Compute Engine Public IPs. To view this list of IP addresses look here https://cloud.google.com/compute/docs/faq#find_ip_range | `bool` | `false` | no | -| [k8s\_network\_names](#input\_k8s\_network\_names) | Kubernetes network names details for GKE. If starting index is not specified for gvnic or rdma, it would be set to the default values. | 
object({
    gvnic_prefix      = optional(string, "")
    gvnic_start_index = optional(number, 1)
    gvnic_postfix     = optional(string, "")
    rdma_prefix       = optional(string, "")
    rdma_start_index  = optional(number, 0)
    rdma_postfix      = optional(string, "")
  })
| 
{
  "gvnic_postfix": "",
  "gvnic_prefix": "gvnic-",
  "gvnic_start_index": 1,
  "rdma_postfix": "",
  "rdma_prefix": "rdma-",
  "rdma_start_index": 0
}
| no | -| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account name to use with the gke cluster | `string` | `"workload-identity-k8s-sa"` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [maintenance\_exclusions](#input\_maintenance\_exclusions) | List of maintenance exclusions. A cluster can have up to three. | 
list(object({
    name            = string
    start_time      = string
    end_time        = string
    exclusion_scope = string
  }))
| `[]` | no | -| [maintenance\_start\_time](#input\_maintenance\_start\_time) | Start time for daily maintenance operations. Specified in GMT with `HH:MM` format. | `string` | `"09:00"` | no | -| [master\_authorized\_networks](#input\_master\_authorized\_networks) | External network that can access Kubernetes master through HTTPS. Must be specified in CIDR notation. | 
list(object({
    cidr_block   = string
    display_name = string
  }))
| `[]` | no | -| [master\_ipv4\_cidr\_block](#input\_master\_ipv4\_cidr\_block) | (Beta) The IP range in CIDR notation to use for the hosted master network. | `string` | `"172.16.0.32/28"` | no | -| [min\_master\_version](#input\_min\_master\_version) | The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release. | `string` | `null` | no | -| [name\_suffix](#input\_name\_suffix) | Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`. | `string` | `""` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`. | `string` | n/a | yes | -| [networking\_mode](#input\_networking\_mode) | Determines whether alias IPs or routes will be used for pod IPs in the cluster. Options are VPC\_NATIVE or ROUTES. VPC\_NATIVE enables IP aliasing. The default is VPC\_NATIVE. | `string` | `"VPC_NATIVE"` | no | -| [pods\_ip\_range\_name](#input\_pods\_ip\_range\_name) | The name of the secondary subnet ip range to use for pods. | `string` | `"pods"` | no | -| [prefix\_with\_deployment\_name](#input\_prefix\_with\_deployment\_name) | If true, cluster name will be prefixed by `deployment_name` (ex: -). | `bool` | `true` | no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [region](#input\_region) | The region to host the cluster in. | `string` | n/a | yes | -| [release\_channel](#input\_release\_channel) | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. | `string` | `"UNSPECIFIED"` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the system node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the system node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [services\_ip\_range\_name](#input\_services\_ip\_range\_name) | The name of the secondary subnet range to use for services. | `string` | `"services"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to host the cluster in. | `string` | n/a | yes | -| [system\_node\_pool\_disk\_size\_gb](#input\_system\_node\_pool\_disk\_size\_gb) | Size of disk for each node of the system node pool. | `number` | `100` | no | -| [system\_node\_pool\_disk\_type](#input\_system\_node\_pool\_disk\_type) | Disk type for each node of the system node pool. | `string` | `null` | no | -| [system\_node\_pool\_enable\_secure\_boot](#input\_system\_node\_pool\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | -| [system\_node\_pool\_enabled](#input\_system\_node\_pool\_enabled) | Create a system node pool. | `bool` | `true` | no | -| [system\_node\_pool\_image\_type](#input\_system\_node\_pool\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | -| [system\_node\_pool\_kubernetes\_labels](#input\_system\_node\_pool\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | -| [system\_node\_pool\_machine\_type](#input\_system\_node\_pool\_machine\_type) | Machine type for the system node pool. | `string` | `"e2-standard-4"` | no | -| [system\_node\_pool\_name](#input\_system\_node\_pool\_name) | Name of the system node pool. | `string` | `"system"` | no | -| [system\_node\_pool\_node\_count](#input\_system\_node\_pool\_node\_count) | The total min and max nodes to be maintained in the system node pool. | 
object({
    total_min_nodes = number
    total_max_nodes = number
  })
| 
{
  "total_max_nodes": 10,
  "total_min_nodes": 2
}
| no | -| [system\_node\_pool\_taints](#input\_system\_node\_pool\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "components.gke.io/gke-managed-components",
    "value": true
  }
]
| no | -| [system\_node\_pool\_zones](#input\_system\_node\_pool\_zones) | The zones to use for the system node pool. If not specified, the cluster default node zone(s) will be used. | `list(string)` | `null` | no | -| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | -| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | -| [upgrade\_settings](#input\_upgrade\_settings) | Defines gke cluster upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | -| [version\_prefix](#input\_version\_prefix) | If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally. | `string` | `"1.31."` | no | -| [zone](#input\_zone) | Zone for a zonal cluster. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_id](#output\_cluster\_id) | An identifier for the resource with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | -| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | -| [gke\_version](#output\_gke\_version) | GKE cluster's version. | -| [instructions](#output\_instructions) | Instructions on how to connect to the created cluster. | -| [k8s\_service\_account\_name](#output\_k8s\_service\_account\_name) | Name of k8s service account. | - diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/main.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/main.tf deleted file mode 100644 index 6106f8d90f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/main.tf +++ /dev/null @@ -1,470 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-cluster", ghpc_role = "scheduler" }) -} - -locals { - upgrade_settings = { - strategy = var.upgrade_settings.strategy - max_surge = coalesce(var.upgrade_settings.max_surge, 0) - max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) - } -} - -locals { - dash = var.prefix_with_deployment_name && var.name_suffix != "" ? "-" : "" - prefix = var.prefix_with_deployment_name ? var.deployment_name : "" - name_maybe_empty = "${local.prefix}${local.dash}${var.name_suffix}" - name = local.name_maybe_empty != "" ? local.name_maybe_empty : "NO-NAME-GIVEN" - - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ - security_group = var.authenticator_security_group - }] - - default_sa_email = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" - sa_email = coalesce(var.service_account_email, local.default_sa_email) - - # additional VPCs enable multi networking - derived_enable_multi_networking = coalesce(var.enable_multi_networking, length(var.additional_networks) > 0) - - # multi networking needs enabled Dataplane v2 - derived_enable_dataplane_v2 = coalesce(var.enable_dataplane_v2, local.derived_enable_multi_networking) - - default_monitoring_component = [ - "SYSTEM_COMPONENTS", - "POD", - "DAEMONSET", - "DEPLOYMENT", - "STATEFULSET", - "STORAGE", - "HPA", - "CADVISOR", - "KUBELET" - ] - - default_logging_component = [ - "SYSTEM_COMPONENTS", - "WORKLOADS" - ] -} - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_container_engine_versions" "version_prefix_filter" { - provider = google-beta - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - version_prefix = var.version_prefix -} - -locals { - master_version = var.min_master_version != null ? var.min_master_version : data.google_container_engine_versions.version_prefix_filter.latest_master_version -} - -resource "google_container_cluster" "gke_cluster" { - provider = google-beta - - project = var.project_id - name = local.name - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - resource_labels = local.labels - networking_mode = var.networking_mode - # decouple node pool lifecycle from cluster life cycle - remove_default_node_pool = true - initial_node_count = 1 # must be set when remove_default_node_pool is set - node_locations = var.system_node_pool_zones - - deletion_protection = var.deletion_protection - - dynamic "enable_k8s_beta_apis" { - for_each = var.enable_k8s_beta_apis != null ? [1] : [] - content { - enabled_apis = var.enable_k8s_beta_apis - } - } - - network = var.network_id - subnetwork = var.subnetwork_self_link - - # Note: the existence of the "master_authorized_networks_config" block enables - # the master authorized networks even if it's empty. - master_authorized_networks_config { - dynamic "cidr_blocks" { - for_each = var.master_authorized_networks - content { - cidr_block = cidr_blocks.value.cidr_block - display_name = cidr_blocks.value.display_name - } - } - gcp_public_cidrs_access_enabled = var.gcp_public_cidrs_access_enabled - } - - private_ipv6_google_access = var.enable_private_ipv6_google_access ? "PRIVATE_IPV6_GOOGLE_ACCESS_TO_GOOGLE" : null - default_max_pods_per_node = var.default_max_pods_per_node - master_auth { - client_certificate_config { - issue_client_certificate = false - } - } - - enable_shielded_nodes = true - - cluster_autoscaling { - # Controls auto provisioning of node-pools - enabled = false - - # Controls autoscaling algorithm of node-pools - autoscaling_profile = var.autoscaling_profile - } - - datapath_provider = local.derived_enable_dataplane_v2 ? "ADVANCED_DATAPATH" : "LEGACY_DATAPATH" - - enable_multi_networking = local.derived_enable_multi_networking - - network_policy { - # Enabling NetworkPolicy for clusters with DatapathProvider=ADVANCED_DATAPATH - # is not allowed. Dataplane V2 will take care of network policy enforcement - # instead. - enabled = false - # GKE Dataplane V2 support. This must be set to PROVIDER_UNSPECIFIED in - # order to let the datapath_provider take effect. - # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/656#issuecomment-720398658 - provider = "PROVIDER_UNSPECIFIED" - } - - private_cluster_config { - enable_private_nodes = var.enable_private_nodes - enable_private_endpoint = var.enable_private_endpoint - master_ipv4_cidr_block = var.master_ipv4_cidr_block - master_global_access_config { - enabled = var.enable_master_global_access - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.pods_ip_range_name - services_secondary_range_name = var.services_ip_range_name - } - - workload_identity_config { - workload_pool = "${var.project_id}.svc.id.goog" - } - - dynamic "gateway_api_config" { - for_each = var.enable_inference_gateway ? [1] : [] - content { - channel = "CHANNEL_STANDARD" - } - } - - dynamic "authenticator_groups_config" { - for_each = local.cluster_authenticator_security_group - content { - security_group = authenticator_groups_config.value.security_group - } - } - - release_channel { - channel = var.release_channel - } - min_master_version = local.master_version - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - - dynamic "maintenance_exclusion" { - for_each = var.maintenance_exclusions - content { - exclusion_name = maintenance_exclusion.value.name - start_time = maintenance_exclusion.value.start_time - end_time = maintenance_exclusion.value.end_time - exclusion_options { - scope = maintenance_exclusion.value.exclusion_scope - } - } - } - } - - dynamic "dns_config" { - for_each = var.cloud_dns_config != null ? [1] : [] - content { - additive_vpc_scope_dns_domain = var.cloud_dns_config.additive_vpc_scope_dns_domain - cluster_dns = var.cloud_dns_config.cluster_dns - cluster_dns_scope = var.cloud_dns_config.cluster_dns_scope - cluster_dns_domain = var.cloud_dns_config.cluster_dns_domain - } - } - - addons_config { - gcp_filestore_csi_driver_config { - enabled = var.enable_filestore_csi - } - gcs_fuse_csi_driver_config { - enabled = var.enable_gcsfuse_csi - } - gce_persistent_disk_csi_driver_config { - enabled = var.enable_persistent_disk_csi - } - dns_cache_config { - enabled = var.enable_node_local_dns_cache - } - parallelstore_csi_driver_config { - enabled = var.enable_parallelstore_csi - } - ray_operator_config { - enabled = var.enable_ray_operator - } - lustre_csi_driver_config { - enabled = var.enable_managed_lustre_csi - } - dynamic "http_load_balancing" { - for_each = var.enable_inference_gateway ? [1] : [] - content { - disabled = false - } - } - } - - timeouts { - create = var.timeout_create - update = var.timeout_update - } - - node_config { - shielded_instance_config { - enable_secure_boot = var.system_node_pool_enable_secure_boot - enable_integrity_monitoring = true - } - } - - control_plane_endpoints_config { - dns_endpoint_config { - allow_external_traffic = var.enable_external_dns_endpoint - } - } - - lifecycle { - # Ignore all changes to the default node pool. It's being removed after creation. - ignore_changes = [ - node_config, - min_master_version, - ] - precondition { - condition = var.default_max_pods_per_node == null || var.networking_mode == "VPC_NATIVE" - error_message = "default_max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." - } - precondition { - condition = coalesce(var.enable_dataplane_v2, true) || !local.derived_enable_multi_networking - error_message = "'enable_dataplane_v2' cannot be false when enabling multi networking." - } - precondition { - condition = coalesce(var.enable_multi_networking, true) || length(var.additional_networks) == 0 - error_message = "'enable_multi_networking' cannot be false when using multivpc module, which passes additional_networks." - } - } - - monitoring_config { - enable_components = var.enable_dcgm_monitoring ? concat(local.default_monitoring_component, ["DCGM"]) : local.default_monitoring_component - managed_prometheus { - enabled = true - } - } - - logging_config { - enable_components = local.default_logging_component - } -} - -# We define explicit node pools, so that it can be modified without -# having to destroy the entire cluster. -resource "google_container_node_pool" "system_node_pools" { - provider = google-beta - count = var.system_node_pool_enabled ? 1 : 0 - - project = var.project_id - name = var.system_node_pool_name - cluster = var.cluster_reference_type == "NAME" ? google_container_cluster.gke_cluster.name : google_container_cluster.gke_cluster.self_link - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - node_locations = var.system_node_pool_zones - version = local.master_version - - autoscaling { - total_min_node_count = var.system_node_pool_node_count.total_min_nodes - total_max_node_count = var.system_node_pool_node_count.total_max_nodes - } - - upgrade_settings { - strategy = local.upgrade_settings.strategy - max_surge = local.upgrade_settings.max_surge - max_unavailable = local.upgrade_settings.max_unavailable - } - - management { - auto_repair = true - auto_upgrade = true - } - - node_config { - labels = var.system_node_pool_kubernetes_labels - resource_labels = local.labels - service_account = var.service_account_email - oauth_scopes = var.service_account_scopes - machine_type = var.system_node_pool_machine_type - disk_size_gb = var.system_node_pool_disk_size_gb - disk_type = var.system_node_pool_disk_type - - dynamic "taint" { - for_each = var.system_node_pool_taints - content { - key = taint.value.key - value = taint.value.value - effect = taint.value.effect - } - } - - # Forcing the use of the Container-optimized image, as it is the only - # image with the proper logging daemon installed. - # - # cos images use Shielded VMs since v1.13.6-gke.0. - # https://cloud.google.com/kubernetes-engine/docs/how-to/node-images - # - # We use COS_CONTAINERD to be compatible with (optional) gVisor. - # https://cloud.google.com/kubernetes-engine/docs/how-to/sandbox-pods - image_type = var.system_node_pool_image_type - - shielded_instance_config { - enable_secure_boot = var.system_node_pool_enable_secure_boot - enable_integrity_monitoring = true - } - - gvnic { - enabled = var.system_node_pool_image_type == "COS_CONTAINERD" - } - - # Implied by Workload Identity - workload_metadata_config { - mode = "GKE_METADATA" - } - # Implied by workload identity. - metadata = { - "disable-legacy-endpoints" = "true" - } - } - - lifecycle { - ignore_changes = [ - node_config[0].labels, - node_config[0].taint, - version, - ] - precondition { - condition = contains(["SURGE"], local.upgrade_settings.strategy) - error_message = "Only SURGE strategy is supported" - } - precondition { - condition = local.upgrade_settings.max_unavailable >= 0 - error_message = "max_unavailable should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_surge >= 0 - error_message = "max_surge should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 - error_message = "At least one of max_unavailable or max_surge must greater than 0" - } - } -} - -data "google_client_config" "default" {} - -provider "kubernetes" { - host = "https://${google_container_cluster.gke_cluster.endpoint}" - cluster_ca_certificate = base64decode(google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) - token = data.google_client_config.default.access_token -} - -module "workload_identity" { - count = var.configure_workload_identity_sa ? 1 : 0 - source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" - version = ">= 40.0" - - use_existing_gcp_sa = true - name = var.k8s_service_account_name - gcp_sa_name = local.sa_email - project_id = var.project_id - - # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1059 - depends_on = [ - data.google_project.project, - google_container_cluster.gke_cluster - ] -} - -locals { - k8s_service_account_name = one(module.workload_identity[*].k8s_service_account_name) -} - -locals { - # Separate gvnic and rdma networks and assign indexes - gvnic_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "GVNIC")] : - merge(net, { name = "${var.k8s_network_names.gvnic_prefix}${idx + var.k8s_network_names.gvnic_start_index}${var.k8s_network_names.gvnic_postfix}" }) - ] - - rdma_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "RDMA")] : - merge(net, { name = "${var.k8s_network_names.rdma_prefix}${idx + var.k8s_network_names.rdma_start_index}${var.k8s_network_names.rdma_postfix}" }) - ] - - all_networks = concat(local.gvnic_networks, local.rdma_networks) -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = google_container_cluster.gke_cluster.id - project_id = var.project_id - - apply_manifests = concat(flatten([ - for idx, network_info in local.all_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = network_info.name, - network_name = network_info.network - subnetwork_name = network_info.subnetwork, - device_mode = strcontains(upper(network_info.nic_type), "RDMA") ? "RDMA" : "NetDevice" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = network_info.name } - } - ] - ]), - var.enable_inference_gateway ? [ - { - source = "https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.0.0/manifests.yaml", - template_vars = {} - } - ] : [] - ) -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml deleted file mode 100644 index bd1517ce8f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/outputs.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/outputs.tf deleted file mode 100644 index 3326a5468e..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/outputs.tf +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cluster_id" { - description = "An identifier for the resource with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." - value = google_container_cluster.gke_cluster.id -} - -output "gke_cluster_exists" { - description = "A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations." - value = true - depends_on = [ - google_container_cluster.gke_cluster - ] -} - -locals { - private_endpoint_message = trimspace( - <<-EOT - This cluster was created with 'enable_private_endpoint: true'. - It cannot be accessed from a public IP addresses. - One way to access this cluster is from a VM created in the GKE cluster subnet. - EOT - ) - master_authorized_networks_message = length(var.master_authorized_networks) == 0 ? "" : trimspace( - <<-EOT - The following networks have been authorized to access this cluster: - ${join("\n", [for x in var.master_authorized_networks : " ${x.display_name}: ${x.cidr_block}"])}" - EOT - ) - public_endpoint_message = trimspace( - <<-EOT - To add authorized networks you can allowlist your IP with this command: - gcloud container clusters update ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} \ - --enable-master-authorized-networks \ - --master-authorized-networks /32 - EOT - ) - allowlist_your_ip_message = var.enable_private_endpoint ? local.private_endpoint_message : local.public_endpoint_message - kubernetes_service_account_message = local.k8s_service_account_name == null ? "" : trimspace( - <<-EOT - Use the following Kubernetes Service Account in the default namespace to run your workloads: - ${local.k8s_service_account_name} - The GCP Service Account mapped to this Kubernetes Service Account is: - ${local.sa_email} - EOT - ) - kubernetes_cluster_fetch_credential_message = var.enable_external_dns_endpoint ? trimspace( - <<-EOT - Use the following command to fetch credentials for the created cluster: - gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} \ - --dns-endpoint - EOT - ) : trimspace( - <<-EOT - Use the following command to fetch credentials for the created cluster: - gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} - EOT - ) -} - -output "instructions" { - description = "Instructions on how to connect to the created cluster." - value = trimspace( - <<-EOT - ${local.master_authorized_networks_message} - - ${local.allowlist_your_ip_message} - - ${local.kubernetes_cluster_fetch_credential_message} - - ${local.kubernetes_service_account_message} - EOT - ) -} - -output "k8s_service_account_name" { - description = "Name of k8s service account." - value = local.k8s_service_account_name -} - -output "gke_version" { - description = "GKE cluster's version." - value = google_container_cluster.gke_cluster.master_version -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl deleted file mode 100644 index d376a1a760..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: GKENetworkParamSet -metadata: - name: ${name} -spec: - vpc: ${network_name} - vpcSubnet: ${subnetwork_name} - deviceMode: ${device_mode} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl deleted file mode 100644 index 1571a92692..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: Network -metadata: - name: ${name} -spec: - parametersRef: - group: networking.gke.io - kind: GKENetworkParamSet - name: ${name} - type: Device diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/variables.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/variables.tf deleted file mode 100644 index 8d863b1730..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/gke-cluster/variables.tf +++ /dev/null @@ -1,533 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "name_suffix" { - description = "Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`." - type = string - default = "" -} - -variable "deployment_name" { - description = "Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`." - type = string -} - -variable "prefix_with_deployment_name" { - description = "If true, cluster name will be prefixed by `deployment_name` (ex: -)." - type = bool - default = true -} - -variable "region" { - description = "The region to host the cluster in." - type = string -} - -variable "zone" { - description = "Zone for a zonal cluster." - default = null - type = string -} - -variable "network_id" { - description = "The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`." - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to host the cluster in." - type = string -} - -variable "pods_ip_range_name" { - description = "The name of the secondary subnet ip range to use for pods." - type = string - default = "pods" -} - -variable "services_ip_range_name" { - description = "The name of the secondary subnet range to use for services." - type = string - default = "services" -} - -variable "enable_private_ipv6_google_access" { - description = "The private IPv6 google access type for the VMs in this subnet." - type = bool - default = true -} - -variable "release_channel" { - description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`." - type = string - default = "UNSPECIFIED" -} - -variable "min_master_version" { - description = "The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release." - type = string - default = null -} - -variable "version_prefix" { - description = "If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally." - type = string - default = "1.31." -} - -variable "maintenance_start_time" { - description = "Start time for daily maintenance operations. Specified in GMT with `HH:MM` format." - type = string - default = "09:00" -} - -variable "maintenance_exclusions" { - description = "List of maintenance exclusions. A cluster can have up to three." - type = list(object({ - name = string - start_time = string - end_time = string - exclusion_scope = string - })) - default = [] - validation { - condition = alltrue([ - for x in var.maintenance_exclusions : - contains(["NO_UPGRADES", "NO_MINOR_UPGRADES", "NO_MINOR_OR_NODE_UPGRADES"], x.exclusion_scope) - ]) - error_message = "`exclusion_scope` must be set to `NO_UPGRADES` OR `NO_MINOR_UPGRADES` OR `NO_MINOR_OR_NODE_UPGRADES`." - } -} - -variable "cloud_dns_config" { - description = < **_NOTE:_** The `project_id` and `region` settings would be inferred from the -> deployment variables of the same name, but they are included here for clarity. - -### Multi-networking - -To create network objects in GKE cluster, you can pass a multivpc module to a pre-existing-gke-cluster module instead of [applying a manifest manually](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment). - -```yaml - - id: network - source: modules/network/vpc - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: existing-gke-cluster ## multinetworking must be enabled in advance when cluster creation - source: modules/scheduler/pre-existing-gke-cluster - use: [multinetwork] - settings: - cluster_name: $(vars.deployment_name) -``` - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_container_cluster.existing_gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [cluster\_name](#input\_cluster\_name) | Name of the existing cluster | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project that hosts the existing cluster | `string` | n/a | yes | -| [rdma\_subnetwork\_name\_prefix](#input\_rdma\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | `string` | `null` | no | -| [region](#input\_region) | Region in which to search for the cluster | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | -| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster exists. | -| [gke\_version](#output\_gke\_version) | GKE cluster's version. | - diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf deleted file mode 100644 index 926d2be100..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_container_cluster" "existing_gke_cluster" { - name = var.cluster_name - project = var.project_id - location = var.region -} - -locals { - rdma_networks = [for network_info in var.additional_networks : network_info if strcontains(upper(network_info.nic_type), "RDMA")] - non_rdma_networks = [for network_info in var.additional_networks : network_info if !strcontains(upper(network_info.nic_type), "RDMA")] - apply_manifests_rdma_networks = flatten([ - for idx, network_info in local.rdma_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = "${var.rdma_subnetwork_name_prefix}-${idx}", - network_name = network_info.network - subnetwork_name = "${var.rdma_subnetwork_name_prefix}-${idx}", - device_mode = "RDMA" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = "${var.rdma_subnetwork_name_prefix}-${idx}" } - } - ] - ]) - - apply_manifests_non_rdma_networks = flatten([ - for idx, network_info in local.non_rdma_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = network_info.subnetwork - network_name = network_info.network - subnetwork_name = network_info.subnetwork - device_mode = "NetDevice" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = network_info.subnetwork } - } - ] - ]) -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = data.google_container_cluster.existing_gke_cluster.id - project_id = var.project_id - - apply_manifests = concat(local.apply_manifests_non_rdma_networks, local.apply_manifests_rdma_networks) -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf deleted file mode 100644 index 8884ee30b0..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cluster_id" { - description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." - value = data.google_container_cluster.existing_gke_cluster.id -} - -output "gke_cluster_exists" { - description = "A static flag that signals to downstream modules that a cluster exists." - value = true - depends_on = [ - data.google_container_cluster.existing_gke_cluster - ] -} - -output "gke_version" { - description = "GKE cluster's version." - value = data.google_container_cluster.existing_gke_cluster.master_version -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl deleted file mode 100644 index d376a1a760..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: GKENetworkParamSet -metadata: - name: ${name} -spec: - vpc: ${network_name} - vpcSubnet: ${subnetwork_name} - deviceMode: ${device_mode} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl deleted file mode 100644 index 1571a92692..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: Network -metadata: - name: ${name} -spec: - parametersRef: - group: networking.gke.io - kind: GKENetworkParamSet - name: ${name} - type: Device diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf deleted file mode 100644 index 9e9ed98ed3..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project that hosts the existing cluster" - type = string -} - -variable "cluster_name" { - description = "Name of the existing cluster" - type = string -} - -variable "region" { - description = "Region in which to search for the cluster" - type = string -} - -variable "additional_networks" { - description = "Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster." - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - })) -} - -variable "rdma_subnetwork_name_prefix" { - description = "Prefix of the RDMA subnetwork names" - default = null - type = string -} diff --git a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf b/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf deleted file mode 100644 index 562d8647b1..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = "> 5.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.74.0" - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/README.md b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/README.md deleted file mode 100644 index db9094909b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/README.md +++ /dev/null @@ -1,355 +0,0 @@ -## Description - -This module creates a startup script that will execute a list of runners in the -order they are specified. The runners are copied to a GCS bucket at deployment -time and then copied into the VM as they are executed after startup. - -Each runner receives the following attributes: - -- `destination`: (Required) The name of the file at the destination VM. If an - absolute path is provided, the file will be copied to that path, otherwise - the file will be created in a temporary folder and deleted once the startup - script runs. -- `type`: (Required) The type of the runner, one of the following: - - `shell`: The runner is a shell script and will be executed once copied to - the destination VM. - - `ansible-local`: The runner is an ansible playbook and will run on the VM - with the following command line flags: - - ```shell - ansible-playbook --connection=local --inventory=localhost, \ - --limit localhost <> - ``` - - - `data`: The data or file specified will be copied to `<>`. No - action will be performed after the data is staged. This data can be used by - subsequent runners or simply made available on the VM for later use. -- `content`: (Optional) Content to be uploaded and, if `type` is - either `shell` or `ansible-local`, executed. Must be defined if `source` is - not. -- `source`: (Optional) A path to the file or data you want to upload. Must be - defined if `content` is not. The source path is relative to the deployment - group directory. To ensure correctness of path use `ghpc_stage` function, that - would copy referenced file to the deployment group directory. For example: - - ```yaml - source: $(ghpc_stage("path/to/file")) - ``` - - For more examples with context, see the - [example blueprint snippet](#example). To reference any other source file, an - absolute path must be used. - -- `args`: (Optional) Arguments to be passed to `shell` or `ansible-local` - runners. For `shell` runners, these will be passed as arguments to the script - when it is executed. For `ansible-local` runners, they will be appended to - a list of default arguments that invoke `ansible-playbook` on the localhost. - Therefore`args` should not include any arguments that alter this behavior, - such as `--connection`, `--inventory`, or `--limit`. - -### Runner dependencies - -`ansible-local` runners require Ansible to be installed in the VM before -running. To support other playbook runners in the Cluster Toolkit, we install -version 2.11 of `ansible-core` as well as the larger package of collections -found in `ansible` version 4.10.0. - -If an `ansible-local` runner is found in the list supplied to this module, -a script to install Ansible will be prepended to the list of runners. This -behavior can be disabled by setting `var.prepend_ansible_installer` to `false`. -This script will do the following at VM startup: - -- Install system-wide python3 if not already installed using system package - managers (yum, apt-get, etc) -- Install `python3-distutils` system-wide in debian and ubuntu based - environments. This can be a missing dependency on system installations of - python3 for installing and upgrading pip. -- Install system-wide pip3 if not already installed and upgrade pip3 if the - version is not at least 18.0. -- Install and create a virtual environment located at `/usr/local/ghpc-venv`. -- Install ansible into this virtual environment if the current version of - ansible is not version 2.11 or higher. - -To use the virtual environment created by this script, you can activate it by -running the following command on the VM: - -```shell -source /usr/local/ghpc-venv/bin/activate -``` - -You may also need to provide the correct python interpreter as the python3 -binary in the virtual environment. This can be done by adding the following flag -when calling `ansible-playbook`: - -```shell --e ansible_python_interpreter=/usr/local/ghpc-venv/bin/activate -``` - -> **_NOTE:_** ansible-playbook and other ansible command line tools will only be -> accessible from the command line (and in your PATH variable) after activating -> this environment. - -### Staging the runners - -Runners will be uploaded to a -[GCS bucket](https://cloud.google.com/storage/docs/creating-buckets). This -bucket will be created by this module and named as -`${var.deployment_name}-startup-scripts-${random_id}`. VMs using the startup -script created by this module will pull the runners content from a GCS bucket -and therefore must have access to GCS. - -> **_NOTE:_** To ensure access to GCS, set the following OAuth scope on the -> instance using the startup scripts: -> `https://www.googleapis.com/auth/devstorage.read_only`. -> -> This is set as a default scope in the [vm-instance], -> [schedMD-slurm-on-gcp-login-node] and [schedMD-slurm-on-gcp-controller] -> modules - -[vm-instance]: ../../compute/vm-instance/README.md -[schedMD-slurm-on-gcp-login-node]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md -[schedMD-slurm-on-gcp-controller]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md - -### Tracking startup script execution - -For more information on how to use startup scripts on Google Cloud Platform, -please refer to -[this document](https://cloud.google.com/compute/docs/instances/startup-scripts/linux). - -To debug startup scripts from a Linux VM created with startup script generated -by this module: - -```shell -sudo DEBUG=1 google_metadata_script_runner startup -``` - -To view outputs from a Linux startup script, run: - -```shell -sudo journalctl -u google-startup-scripts.service -``` - -### Monitoring Agent Installation - -This `startup-script` module has several options for installing a Google -monitoring agent. There are two relevant settings: `install_stackdriver_agent` -and `install_cloud_ops_agent`. - -The _Stackdriver Agent_ also called the _Legacy Cloud Monitoring Agent_ provides -better performance under some HPC workloads. While official documentation -recommends using the _Cloud Ops Agent_, it is recommended to use -`install_stackdriver_agent` when performance is important. - -#### Stackdriver Agent Installation - -If an image or machine already has Cloud Ops Agent installed and you would like -to instead use the Stackdriver Agent, the following script will remove the Cloud -Ops Agent and install the Stackdriver Agent. - -```bash -# Remove Cloud Ops Agent -sudo systemctl stop google-cloud-ops-agent.service -sudo systemctl disable google-cloud-ops-agent.service -curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh -sudo bash add-google-cloud-ops-agent-repo.sh --uninstall -sudo bash add-google-cloud-ops-agent-repo.sh --remove-repo - -# Install Stackdriver Agent -curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh -sudo bash add-monitoring-agent-repo.sh --also-install -curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh -sudo bash add-logging-agent-repo.sh --also-install -sudo service stackdriver-agent start -sudo service google-fluentd restart -``` - -#### Cloud Ops Agent Installation - -If an image or machine already has the Stackdriver Agent installed and you would -like to instead use the Cloud Ops Agent, the following script will remove the -Stackdriver Agent and install the Cloud Ops Agent. - -```bash -# UnInstall Stackdriver Agent - -sudo systemctl stop stackdriver-agent.service -sudo systemctl disable stackdriver-agent.service -curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh -sudo dpkg --configure -a -sudo bash add-monitoring-agent-repo.sh --uninstall -sudo bash add-monitoring-agent-repo.sh --remove-repo -sudo systemctl stop google-fluentd.service -sudo systemctl disable google-fluentd.service -sudo dpkg --configure -a -curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh -sudo bash add-logging-agent-repo.sh --uninstall -sudo bash add-logging-agent-repo.sh --remove-repo - -# Install ops-agent - -curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh -sudo bash add-google-cloud-ops-agent-repo.sh --also-install -sudo service google-cloud-ops-agent start -``` - -As a reminder, this should be in a startup script, which should run on all -Compute nodes via the `compute_startup_script` on the controller. - -#### Testing Installation - -You can test if one of the agents is running using the following commands: - -```bash -# For Cloud Ops Agent -$ sudo systemctl is-active google-cloud-ops-agent"*" -active -active -active -active - -# For Legacy Monitoring and Logging Agents -$ sudo service stackdriver-agent status -stackdriver-agent is running [ OK ] -$ sudo service google-fluentd status -google-fluentd is running [ OK ] -``` - -For official documentation see troubleshooting docs: - -- [Cloud Ops Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-install-startup) -- [Legacy Monitoring Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/monitoring/troubleshooting) -- [Legacy Logging Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/logging/troubleshooting) - -### Example - -```yaml -- id: startup - source: modules/scripts/startup-script - settings: - runners: - # Some modules such as filestore have runners as outputs for convenience: - - $(homefs.install_nfs_client_runner) - # These runners can still be created manually: - # - type: shell - # destination: "modules/filestore/scripts/install_nfs_client.sh" - # source: "modules/filestore/scripts/install_nfs_client.sh" - - type: ansible-local - destination: "modules/filestore/scripts/mount.yaml" - source: "modules/filestore/scripts/mount.yaml" - - type: data - source: /tmp/foo.tgz - destination: /tmp/bar.tgz - - type: shell - destination: "decompress.sh" - content: | - #!/bin/sh - echo $2 - tar zxvf /tmp/$1 -C / - args: "bar.tgz 'Expanding file'" - -- id: compute-cluster - source: modules/compute/vm-instance - use: [homefs, startup] -``` - -In the above example, a new GCS bucket is created to upload the startup-scripts. -But in the case where the user wants to reuse existing GCS bucket or folder, -they are able to do so by using the `gcs_bucket_path` as shown in the below example - -```yaml -- id: startup - source: modules/scripts/startup-script - settings: - gcs_bucket_path: gs://user-test-bucket/folder1/folder2 - install_stackdriver_agent: true - -- id: compute-cluster - source: modules/compute/vm-instance - use: [startup] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 6.41 | -| [local](#requirement\_local) | >= 2.0.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.41 | -| [local](#provider\_local) | >= 2.0.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.configs_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | -| [google_storage_bucket_object.scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [local_file.debug_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [ansible\_virtualenv\_path](#input\_ansible\_virtualenv\_path) | Virtual environment path in which to install Ansible | `string` | `"/usr/local/ghpc-venv"` | no | -| [bucket\_viewers](#input\_bucket\_viewers) | Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account) | `list(string)` | `[]` | no | -| [configure\_ssh\_host\_patterns](#input\_configure\_ssh\_host\_patterns) | If specified, it will automate ssh configuration by:
- Defining a Host block for every element of this variable and setting StrictHostKeyChecking to 'No'.
Ex: "hpc*", "hpc01*", "ml*"
- The first time users log-in, it will create ssh keys that are added to the authorized keys list
This requires a shared /home filesystem and relies on specifying the right prefix. | `list(string)` | `[]` | no | -| [debug\_file](#input\_debug\_file) | Path to an optional local to be written with 'startup\_script'. | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used to name GCS bucket for startup scripts. | `string` | n/a | yes | -| [docker](#input\_docker) | Install and configure Docker | 
object({
    enabled        = optional(bool, false)
    world_writable = optional(bool, false)
    daemon_config  = optional(string, "")
  })
| 
{
  "enabled": false
}
| no | -| [enable\_docker\_world\_writable](#input\_enable\_docker\_world\_writable) | DEPRECATED: use var.docker | `bool` | `null` | no | -| [enable\_gpu\_network\_wait\_online](#input\_enable\_gpu\_network\_wait\_online) | Enable a SystemD unit that blocks execution of startup-scripts until after all network interfaces are online. (Works on reboots or boots of an image built using this solution) | `bool` | `false` | no | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | `null` | no | -| [http\_no\_proxy](#input\_http\_no\_proxy) | Domains for which to disable http\_proxy behavior. Honored only if var.http\_proxy is set | `string` | `".google.com,.googleapis.com,metadata.google.internal,localhost,127.0.0.1"` | no | -| [http\_proxy](#input\_http\_proxy) | Web (http and https) proxy configuration for pip, apt, and yum/dnf and interactive shells | `string` | `""` | no | -| [install\_ansible](#input\_install\_ansible) | Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used. | `bool` | `null` | no | -| [install\_cloud\_ops\_agent](#input\_install\_cloud\_ops\_agent) | Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true. | `bool` | `false` | no | -| [install\_cloud\_rdma\_drivers](#input\_install\_cloud\_rdma\_drivers) | If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8. Should not be enabled if using the HPC VM Image. | `bool` | `false` | no | -| [install\_docker](#input\_install\_docker) | DEPRECATED: use var.docker. | `bool` | `null` | no | -| [install\_stackdriver\_agent](#input\_install\_stackdriver\_agent) | Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance. | `bool` | `false` | no | -| [labels](#input\_labels) | Labels for the created GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_filesystem](#input\_local\_ssd\_filesystem) | Create and mount a filesystem from local SSD disks (data will be lost if VMs are powered down without enabling migration); enable by setting mountpoint field to a valid directory path. | 
object({
    fs_type     = optional(string, "ext4")
    mountpoint  = optional(string, "")
    permissions = optional(string, "0755")
  })
| 
{
  "fs_type": "ext4",
  "mountpoint": "",
  "permissions": "0755"
}
| no | -| [managed\_lustre](#input\_managed\_lustre) | Configure Managed Lustre (assumes driver already installed) | 
object({
    enabled = optional(bool, false)
    port    = optional(number, 988)
  })
| 
{
  "enabled": false,
  "port": 988
}
| no | -| [prepend\_ansible\_installer](#input\_prepend\_ansible\_installer) | DEPRECATED. Use `install_ansible=false` to prevent ansible installation. | `bool` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [runners](#input\_runners) | List of runners to run on remote VM.
Runners can be of type ansible-local, shell or data.
A runner must specify one of 'source' or 'content'.
All runners must specify 'destination'. If 'destination' does not include a
path, it will be copied in a temporary folder and deleted after running.
Runners may also pass 'args', which will be passed as argument to shell runners only. | `list(map(string))` | `[]` | no | -| [set\_ofi\_cloud\_rdma\_tunables](#input\_set\_ofi\_cloud\_rdma\_tunables) | Controls whether to enable specific OFI environment variables for workloads using Cloud RDMA networking. Should be false for non-RDMA workloads. | `bool` | `false` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [compute\_startup\_script](#output\_compute\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | -| [controller\_startup\_script](#output\_controller\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | -| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | - diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml deleted file mode 100644 index 02c449c7cb..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Configure ssh between nodes - become: true - hosts: localhost - vars: - ssh_config_path: "/etc/ssh/ssh_config" - bashrc: "{{ '/etc/bashrc' if ansible_facts['os_family'] == 'RedHat' else '/etc/bash.bashrc' }}" - setup_ssh_script: "/bin/bash /usr/local/ghpc/setup-ssh-keys.sh" - tasks: - - name: "Set StrictHostKeyChecking to no" - ansible.builtin.blockinfile: - path: "{{ ssh_config_path }}" - block: | - Host "{{ item }}" - StrictHostKeyChecking no - marker: "# {mark} ANSIBLE MANAGED BLOCK {{item}}" - loop: "{{ host_name_prefix }}" - - name: "Create ssh keys in .bashrc if not already done" - ansible.builtin.lineinfile: - path: "{{ bashrc }}" - regexp: '^{{ setup_ssh_script }}' - line: "{{ setup_ssh_script }}" diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh deleted file mode 100644 index 38c7ff9b5c..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -web_proxy="${1:-}" -if [ -z "$web_proxy" ]; then - echo "Error: must provide 1 argument identifying http/https proxy" - exit 1 -fi - -# configure pip to use proxy -PIP_CONF=/etc/pip.conf -if [ ! -f "$PIP_CONF" ]; then - cat <<-EOF >"$PIP_CONF" - [global] - proxy=$web_proxy - EOF -fi - -# configure yum or dnf to use proxy -if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - YUM_CONF="/etc/yum.conf" - if ! grep -q '^proxy=.*' "$YUM_CONF"; then - sed --follow-symlinks -i.bak "/^\[main]/a proxy=$web_proxy" "$YUM_CONF" - else - sed --follow-symlinks -i.bak "s,proxy=.*,proxy=$web_proxy," "$YUM_CONF" - fi -fi - -# configure apt to use proxy -if [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release 2>/dev/null || - grep -qi ubuntu /etc/os-release 2>/dev/null; then - APT_CONF_PROXY="/etc/apt/apt.conf.d/99proxy.conf" - if [ ! -f "$APT_CONF_PROXY" ]; then - cat <<-EOF >"$APT_CONF_PROXY" - Acquire::http::Proxy "$web_proxy"; - Acquire::https::Proxy "$web_proxy"; - EOF - fi -fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh deleted file mode 100644 index 682e1352a1..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script applies fixes to VMs that must occur early in boot. For example, -# when yum or apt repositories are misconfigured, preventing most package -# operations from completing successfully. - -source /etc/os-release - -if [[ "$PRETTY_NAME" == "CentOS Linux 7 (Core)" ]]; then - echo "Applying hotfixes for CentOS 7" - if grep -q '^mirrorlist' /etc/yum.repos.d/CentOS-Base.repo; then - echo "Removing mirrorlist from default CentOS 7 repositories" - sed -i '/^mirrorlist/d' /etc/yum.repos.d/CentOS-Base.repo - fi - if grep -q '^#baseurl=http://mirror.centos.org' /etc/yum.repos.d/CentOS-Base.repo; then - echo "Reconfiguring default CentOS 7 repositories to use CentOS Vault" - sed -i 's,^#baseurl=http://mirror.centos.org/,baseurl=http://vault.centos.org/,' /etc/yum.repos.d/CentOS-Base.repo - fi -fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh deleted file mode 100644 index 3a29ae808f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh +++ /dev/null @@ -1,73 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Given a url and filename, download an object to the vardir. When the installed -# version of gcloud is >=402.0.0 (Sept. 2022), then gcloud storage is used to -# fetch from the bucket. Otherwise gsutil is used. Note, the service account for -# the instance must be properly configured with a role having authorization to -# get objects from the bucket. -# -# This function is intended for single file downloads and no attempt is made to -# verify the checksum other than the default behavior of gcloud or gsutil. -# -# This function has no other platform dependencies other than gcloud / gsutil. - -# This code originated from: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 -stdlib::get_from_bucket() { - local OPTIND opt url fname dir="${VARDIR:-/var/lib/startup}" - while getopts ":u:f:d:" opt; do - case "${opt}" in - u) url="${OPTARG}" ;; - f) fname="${OPTARG}" ;; - d) dir="${OPTARG}" ;; - :) - stdlib::mandatory_argument -n stdlib::get_from_bucket -f "$OPTARG" - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error 'Usage: stdlib::get_from_bucket -u -f -d ' - stdlib::info 'For example: stdlib::get_from_bucket -u gs://mybucket/foo.tgz -d /var/tmp' - return "${E_UNKNOWN_ARG}" - ;; - esac - done - # Trivially compute the filename from the URL if unspecified. - if [[ -z ${fname} ]]; then - fname=${url##*/} - stdlib::debug "Computed filename='${fname}' given URL." - fi - [[ -d ${dir} ]] || mkdir "${dir}" - local attempt=0 - local max_retries=7 - # store gcs command as array and then split when called by stdlib::cmd - if stdlib::cmd gcloud help storage cp &>/dev/null; then - gcs_command=(gcloud storage cp --no-user-output-enabled) - else - gcs_command=(gsutil -q cp) - fi - while [[ $attempt -le $max_retries ]]; do - if [[ $attempt -gt 0 ]]; then - local wait=$((2 ** attempt)) - stdlib::error "Retry attempt ${attempt} of ${max_retries} with exponential backoff: ${wait} seconds." - sleep $wait - fi - if stdlib::cmd "${gcs_command[@]}" "${url}" "${dir}/${fname}"; then - break - else - stdlib::error "${gcs_command[*]} reported non-zero exit code fetching ${url}." - ((attempt++)) - fi - done -} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh deleted file mode 100644 index eac2b2e32a..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex -REQ_ANSIBLE_VERSION=2.15 -REQ_ANSIBLE_PIP_VERSION=8.7.0 -REQ_PIP_WHEEL_VERSION=0.45.1 -REQ_PIP_SETUPTOOLS_VERSION=80.8.0 -REQ_PIP_MAJOR_VERSION=25 -REQ_PYTHON3_VERSION=9 - -apt_wait() { - while fuser /var/lib/apt/lists/lock >/dev/null 2>&1; do - echo "Sleeping for apt lists lock" - sleep 3 - done -} - -# Installs any dependencies needed for python based on the OS -install_python_deps() { - # this file is present on both Debian and Ubuntu OSes - if [ -f /etc/debian_version ]; then - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3-setuptools python3-venv - fi -} - -# Gets the name of the python executable for python starting with python3, then -# checking python. Sets the variable to an empty string if neither are found. -get_python_path() { - python_path="" - if command -v python3 1>/dev/null; then - python_path=$(command -v python3) - elif command -v python 1>/dev/null; then - python_path=$(command -v python) - fi -} - -# Returns the python major version. If provided, it will use the first argument -# as the python executable, otherwise it will default to simply "python". -get_python_major_version() { - python_path=${1:-python} - python_major_version=$(${python_path} -c "import sys; print(sys.version_info.major)") -} - -# Returns the python minor version. If provided, it will use the first argument -# as the python executable, otherwise it will default to simply "python". -get_python_minor_version() { - python_path=${1:-python} - python_minor_version=$(${python_path} -c "import sys; print(sys.version_info.minor)") -} - -# Install python3 with the yum package manager. Updates python_path to the -# newly installed packaged. -install_python3_dnf() { - major_version=$(rpm -E "%{rhel}") - set -- "--disablerepo=*" "--enablerepo=baseos,appstream" - if grep -qi 'ID="rhel"' /etc/os-release; then - # Do not set --disablerepo / --enablerepo on RedHat, due to - # complex repo names; clear array - set -- - fi - # On Rocky Linux 9, Python 3.9 is installed by default but this - # has already been dropped by ansible-core for control nodes. - # https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix - # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where - # it is available as "python3*" but must be named explicitly on - # older releases. It also ensures longer support for Ansible. - if [ "${major_version}" -lt "10" ]; then - dnf install "$@" -y python3.12 python3.12-pip - python_path=$(command -v python3.12) - else - dnf install "$@" -y python3 python3-pip - python_path=$(command -v python3) - fi -} - -# Install python3 with the apt package manager. Updates python_path to the -# newly installed packaged. -install_python3_apt() { - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3 python3-setuptools python3-pip python3-venv - python_path=$(command -v python3) -} - -install_python3() { - if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || - [ -f /etc/system-release ]; then - install_python3_dnf - elif [ -f /etc/debian_version ]; then - install_python3_apt - else - echo "Error: Unsupported Distribution" - return 1 - fi -} - -# Install pip3 with the dnf package manager. Updates python_path to the -# newly installed packaged. -install_pip3_dnf() { - major_version=$(rpm -E "%{rhel}") - set -- "--disablerepo=*" "--enablerepo=baseos,appstream" - if grep -qi 'ID="rhel"' /etc/os-release; then - # Do not set --disablerepo / --enablerepo on RedHat, due to complex repo names - # clear array - set -- - fi - # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where - # it is available as "python3*" but must be named explicitly on - # older releases. It also ensures longer support for Ansible. - if [ "${major_version}" -lt "10" ]; then - dnf install "$@" -y python3.12-pip - else - dnf install "$@" -y python3-pip - fi -} - -# Install pip3 with the apt package manager. Updates python_path to the -# newly installed packaged. -install_pip3_apt() { - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3-pip -} - -install_pip3() { - if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || - [ -f /etc/system-release ]; then - install_pip3_dnf - elif [ -f /etc/debian_version ]; then - install_pip3_apt - else - echo "Error: Unsupported Distribution" - return 1 - fi -} - -main() { - if [ $# -gt 1 ]; then - echo "Error: provide only 1 optional argument identifying virtual environment path for Ansible" - return 1 - fi - - venv_path="${1:-/usr/local/ghpc-venv}" - - # Get the python3 executable, or install it if not found - get_python_path - get_python_major_version "${python_path}" - get_python_minor_version "${python_path}" - if [ "${python_path}" = "" ] || [ "${python_major_version}" = "2" ] || [ "${python_minor_version}" -lt "${REQ_PYTHON3_VERSION}" ]; then - if ! install_python3; then - return 1 - fi - get_python_major_version "${python_path}" - get_python_minor_version "${python_path}" - else - install_python_deps - fi - - # Install OS-packaged pip - if ! ${python_path} -m pip --version 2>/dev/null; then - if ! install_pip3; then - return 1 - fi - fi - - # Create pip virtual environment for Cluster Toolkit - ${python_path} -m venv "${venv_path}" --copies - venv_python_path=${venv_path}/bin/python3 - - # Upgrade pip if necessary - pip_version=$(${venv_python_path} -m pip --version | sed -nr 's/^pip ([0-9]+\.[0-9]+).*$/\1/p') - pip_major_version=$(echo "${pip_version}" | cut -d '.' -f 1) - if [ "${pip_major_version}" -lt "${REQ_PIP_MAJOR_VERSION}" ]; then - ${venv_python_path} -m pip install --upgrade pip - fi - - # upgrade wheel if necessary - wheel_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^wheel" || true) - if [ "$wheel_pkg" != "wheel==${REQ_PIP_WHEEL_VERSION}" ]; then - ${venv_python_path} -m pip install -U wheel==${REQ_PIP_WHEEL_VERSION} - fi - - # upgrade setuptools if necessary - setuptools_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^setuptools" || true) - if [ "$setuptools_pkg" != "setuptools==${REQ_PIP_SETUPTOOLS_VERSION}" ]; then - ${venv_python_path} -m pip install -U setuptools==${REQ_PIP_SETUPTOOLS_VERSION} - fi - - # configure ansible to always use correct Python binary - if [ ! -f /etc/ansible/ansible.cfg ]; then - mkdir /etc/ansible - cat <<-EOF >/etc/ansible/ansible.cfg - [defaults] - interpreter_python=${venv_python_path} - stdout_callback=debug - stderr_callback=debug - EOF - fi - - # Install ansible - ansible_version="" - if command -v ansible-playbook 1>/dev/null; then - ansible_version=$(ansible-playbook --version 2>/dev/null | sed -nr 's/^ansible-playbook.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p') - ansible_major_vers=$(echo "${ansible_version}" | cut -d '.' -f 1) - ansible_minor_vers=$(echo "${ansible_version}" | cut -d '.' -f 2) - ansible_req_major_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 1) - ansible_req_minor_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 2) - fi - if [ -z "${ansible_version}" ] || [ "${ansible_major_vers}" -ne "${ansible_req_major_vers}" ] || - [ "${ansible_minor_vers}" -lt "${ansible_req_minor_vers}" ]; then - ${venv_python_path} -m pip install ansible=="${REQ_ANSIBLE_PIP_VERSION}" - fi - while read -r cmd; do - if ! [ -L "/usr/bin/${cmd}" ]; then - ln -s "${venv_path}/bin/${cmd}" "/usr/bin/${cmd}" - fi - done <<-EOF - ansible - ansible-config - ansible-connection - ansible-console - ansible-doc - ansible-galaxy - ansible-inventory - ansible-playbook - ansible-pull - ansible-test - ansible-vault - EOF -} - -main "$@" diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh deleted file mode 100644 index 375792459b..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -OS_ID="$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g')" -OS_VERSION="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g')" -OS_VERSION_MAJOR="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//')" -REBOOT_FILE="/etc/.rdma_reboot" - -if { [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; } && { [ "${OS_VERSION_MAJOR}" = "8" ]; }; then - KMOD_VERSION="$(dnf list installed | awk '$1 ~ /^kmod-idpf-irdma(\.|$)/ {print $2}')" - - # For images that do not already have Cloud RDMA drivers installed - if [ -z "${KMOD_VERSION}" ] && [ -z "${REBOOT_FILE}" ]; then - sudo dnf update -y - sudo dnf install https://depot.ciq.com/public/files/gce-accelerator/irdma-kernel-modules-el8-x86_64/irdma-repos.rpm -y - sudo dnf install kmod-idpf-irdma rdma-core libibverbs-utils librdmacm-utils infiniband-diags perftest -y - sudo touch "${REBOOT_FILE}" - reboot - fi - echo "This image has IRDMA packages already installed, exiting." - exit 0 -else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. Cloud RDMA Drivers are only supported on Rocky Linux 8." - exit 1 -fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_docker.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_docker.yml deleted file mode 100644 index f9b0abeb14..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_docker.yml +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Install and configure Docker - hosts: all - become: true - vars: - docker_data_root: '' - docker_daemon_config: '' - enable_docker_world_writable: false - tasks: - - name: Check if docker is installed - ansible.builtin.stat: - path: /usr/bin/docker - register: docker_binary - - name: Download Docker Installer - ansible.builtin.get_url: - url: https://get.docker.com - dest: /tmp/get-docker.sh - owner: root - group: root - mode: '0644' - when: not docker_binary.stat.exists - - name: Install Docker - ansible.builtin.command: sh /tmp/get-docker.sh - register: docker_installed - changed_when: docker_installed.rc != 0 - when: not docker_binary.stat.exists - - name: Create Docker daemon configuration - ansible.builtin.copy: - dest: /etc/docker/daemon.json - mode: '0644' - content: '{{ docker_daemon_config }}' - validate: /usr/bin/dockerd --validate --config-file %s - when: docker_daemon_config - notify: - - Restart Docker - - name: Create Docker service override directory - ansible.builtin.file: - path: /etc/systemd/system/docker.service.d - state: directory - owner: root - group: root - mode: '0755' - - name: Create Docker service override configuration - ansible.builtin.copy: - dest: /etc/systemd/system/docker.service.d/data-root.conf - mode: '0644' - content: | - [Unit] - {% if docker_data_root %} - RequiresMountsFor={{ docker_data_root }} - {% endif %} - After=mount-localssd-raid.service - - name: Create Docker socket override directory - ansible.builtin.file: - path: /etc/systemd/system/docker.socket.d - state: directory - owner: root - group: root - mode: '0755' - when: enable_docker_world_writable - - name: Create Docker socket override configuration - ansible.builtin.copy: - dest: /etc/systemd/system/docker.socket.d/world-writable.conf - mode: '0644' - content: | - [Socket] - SocketMode=0666 - when: enable_docker_world_writable - notify: - - Reload SystemD - - Recreate Docker socket - - name: Delete Docker socket override configuration - ansible.builtin.file: - path: /etc/systemd/system/docker.socket.d/world-writable.conf - state: absent - when: not enable_docker_world_writable - notify: - - Reload SystemD - - Recreate Docker socket - - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - - name: Recreate Docker socket - ansible.builtin.service: - name: docker.socket - state: restarted - - name: Restart Docker - ansible.builtin.service: - name: docker.service - state: restarted - - post_tasks: - - name: Start Docker - ansible.builtin.service: - name: docker.service - state: started - enabled: true diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml deleted file mode 100644 index 9d295dfc7d..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Install network wait service for A3/A4 variants - hosts: all - become: true - tasks: - - - name: Create universal SystemD service for GPU networking delay - when: ansible_os_family == "Debian" - ansible.builtin.copy: - dest: /etc/systemd/system/delay-gpu-network.service - owner: root - group: root - mode: "0644" - content: | - [Unit] - Description=Delay boot on multi-NIC VMs until networks are routable - After=network-online.target - Wants=network-online.target - Before=google-startup-scripts.service - - [Service] - # This condition checks if the machine type is one of the supported A3/A4 variants. - # The service will only run if the machine type matches. - ExecCondition=/bin/bash -c "/usr/bin/curl -s -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/machine-type | grep -qE '(/a3-highgpu-8g|/a3-megagpu-8g|/a3-ultragpu-8g|/a4-highgpu-8g|/a4x-highgpu-4g)$'" - ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -o routable --timeout=180 - ExecStartPost=/bin/sleep 30 - - [Install] - WantedBy=multi-user.target - notify: - - Reload SystemD - - - name: Enable universal GPU network delay service - when: ansible_os_family == "Debian" - ansible.builtin.systemd_service: - name: delay-gpu-network.service - enabled: true - - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml deleted file mode 100644 index 94699471bb..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure Managed Lustre (assumes driver already installed) - hosts: all - become: true - vars: - default_lustre_port: 988 - managed_lustre_port: "{{ default_lustre_port }}" - tasks: - # Ideally changes to this file would also trigger an execution of lnetctl - # command to update accept_port but it is unclear if lnetctl supports this. - - name: Update lnet to use non-default port - when: managed_lustre_port | int != {{ default_lustre_port }} - ansible.builtin.copy: - owner: root - group: root - mode: '0644' - dest: /etc/modprobe.d/lnet.conf - content: | - options lnet accept_port={{ managed_lustre_port | int }} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh deleted file mode 100644 index eb4bf899b8..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -LEGACY_MONITORING_PACKAGE='stackdriver-agent' -LEGACY_MONITORING_SCRIPT_URL='https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh' -LEGACY_LOGGING_PACKAGE='google-fluentd' -LEGACY_LOGGING_SCRIPT_URL='https://dl.google.com/cloudagents/add-logging-agent-repo.sh' - -OPSAGENT_PACKAGE='google-cloud-ops-agent' -OPSAGENT_SCRIPT_URL='https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh' - -ops_or_legacy="${1:-legacy}" - -fail() { - echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" - exit 1 -} - -handle_debian() { - is_legacy_monitoring_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_MONITORING_PACKAGE} | - grep "${LEGACY_MONITORING_PACKAGE} is installed" - } - - is_legacy_logging_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_LOGGING_PACKAGE} | - grep "${LEGACY_LOGGING_PACKAGE} is installed" - } - - is_legacy_installed() { - is_legacy_monitoring_installed || is_legacy_logging_installed - } - - is_opsagent_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${OPSAGENT_PACKAGE} | - grep "${OPSAGENT_PACKAGE} is installed" - } - - install_with_retry() { - MAX_RETRY=50 - RETRY=0 - until [ ${RETRY} -eq ${MAX_RETRY} ] || curl -s "${1}" | bash -s -- --also-install; do - RETRY=$((RETRY + 1)) - echo "WARNING: Installation of ${1} failed on try ${RETRY} of ${MAX_RETRY}" - sleep 5 - done - if [ $RETRY -eq $MAX_RETRY ]; then - echo "ERROR: Installation of ${1} was not successful after ${MAX_RETRY} attempts." - exit 1 - fi - } - - install_opsagent() { - install_with_retry "${OPSAGENT_SCRIPT_URL}" - } - - install_stackdriver_agent() { - install_with_retry "${LEGACY_MONITORING_SCRIPT_URL}" - install_with_retry "${LEGACY_LOGGING_SCRIPT_URL}" - service stackdriver-agent start - service google-fluentd start - } -} - -handle_redhat() { - is_legacy_monitoring_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_MONITORING_PACKAGE} | - grep "${LEGACY_MONITORING_PACKAGE} is installed" - } - - is_legacy_logging_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_LOGGING_PACKAGE} | - grep "${LEGACY_LOGGING_PACKAGE} is installed" - } - - is_legacy_installed() { - is_legacy_monitoring_installed || is_legacy_logging_installed - } - - is_opsagent_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${OPSAGENT_PACKAGE} | - grep "${OPSAGENT_PACKAGE} is installed" - } - - install_opsagent() { - curl -s "${OPSAGENT_SCRIPT_URL}" | bash -s -- --also-install - } - - install_stackdriver_agent() { - curl -sS "${LEGACY_MONITORING_SCRIPT_URL}" | bash -s -- --also-install - curl -sS "${LEGACY_LOGGING_SCRIPT_URL}" | bash -s -- --also-install - service stackdriver-agent start - service google-fluentd start - } -} - -main() { - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - handle_redhat - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - handle_debian - else - fail "Unsupported platform." - fi - - # Handle cases that agent is already installed - if [[ -z "$(is_legacy_monitoring_installed)" && -n $(is_legacy_logging_installed) ]] || - [[ -n "$(is_legacy_monitoring_installed)" && -z $(is_legacy_logging_installed) ]]; then - fail "Bad state: legacy agent is partially installed" - elif [[ "${ops_or_legacy}" == "legacy" ]] && is_legacy_installed; then - echo "Legacy agent is already installed" - exit 0 - elif [[ "${ops_or_legacy}" != "legacy" ]] && is_opsagent_installed; then - echo "Ops agent is already installed" - exit 0 - elif is_legacy_installed || is_opsagent_installed; then - fail "Agent is already installed but does not match requested agent of ${ops_or_legacy}" - fi - - # install agent - if [[ "${ops_or_legacy}" == "legacy" ]]; then - echo "Installing legacy monitoring agent (stackdriver)" - install_stackdriver_agent - else - echo "Installing cloud ops agent" - echo "WARNING: cloud ops agent may have a performance impact. Consider using legacy monitoring agent (stackdriver)." - install_opsagent - fi -} - -main diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh deleted file mode 100644 index 738181aafb..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SCRIPT_COMPLETE_FILE="/run/startup_script_msg" - -# Ensure we're in an interactive terminal and not root -if [ -t 1 ] && [ "$(id -u)" -ne 0 ]; then - # Check if the file has contents otherwise skip - if [ -s "$SCRIPT_COMPLETE_FILE" ]; then - echo - cat "$SCRIPT_COMPLETE_FILE" - echo - fi -fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml deleted file mode 100644 index d94aac81fd..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Configure local SSDs - become: true - hosts: localhost - vars: - raid_name: localssd - array_dev: /dev/md/{{ raid_name }} - fstype: ext4 - interface: nvme - mode: '0755' - mountpoint: /mnt/{{ raid_name }} - tasks: - - name: Get local SSD devices - ansible.builtin.find: - file_type: link - path: /dev/disk/by-id - patterns: google-local-{{ "nvme-" if interface == "nvme" else "" }}ssd-* - register: local_ssd_devices - - - name: Exit if zero local ssd found - ansible.builtin.meta: end_play - when: local_ssd_devices.files | length == 0 - - - name: Install mdadm - ansible.builtin.package: - name: mdadm - state: present - - # this service will act during the play and upon reboots to ensure that local - # SSD volumes are always assembled into a RAID and re-formatted if necessary; - # there are many scenarios where a VM can be stopped or migrated during - # maintenance and the contents of local SSD will be discarded - - name: Install service to create local SSD RAID and format it - ansible.builtin.copy: - dest: /etc/systemd/system/create-localssd-raid.service - mode: 0644 - content: | - [Unit] - After=local-fs.target - Before=slurmd.service docker.service - ConditionPathExists=!{{ array_dev }} - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/usr/bin/bash -c "/usr/sbin/mdadm --create {{ array_dev }} --name={{ raid_name }} --homehost=any --level=0 --raid-devices={{ local_ssd_devices.files | length }} /dev/disk/by-id/google-local-nvme-ssd-*{{ " --force" if local_ssd_devices.files | length == 1 else "" }}" - ExecStartPost=/usr/sbin/mkfs -t {{ fstype }}{{ " -m 0" if fstype == "ext4" else "" }} {{ array_dev }} - - [Install] - WantedBy=slurmd.service docker.service - - - name: Create RAID array and format - ansible.builtin.systemd: - name: create-localssd-raid.service - state: started - enabled: true - daemon_reload: true - - - name: Install service to mount local SSD array - ansible.builtin.copy: - dest: /etc/systemd/system/mount-localssd-raid.service - mode: 0644 - content: | - [Unit] - After=local-fs.target create-localssd-raid.service - Before=slurmd.service docker.service - Wants=create-localssd-raid.service - ConditionPathIsMountPoint=!{{ mountpoint }} - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/usr/bin/systemd-mount -t {{ fstype }} -o discard,defaults,nofail {{ array_dev }} {{ mountpoint }} - ExecStartPost=/usr/bin/chmod {{ mode }} {{ mountpoint }} - ExecStop=/usr/bin/systemd-umount {{ mountpoint }} - - [Install] - WantedBy=slurmd.service docker.service - - - name: Mount RAID array and set permissions - ansible.builtin.systemd: - name: mount-localssd-raid.service - state: started - enabled: true - daemon_reload: true diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh deleted file mode 100644 index 1c8018fb01..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! -d ~/.ssh/ ]; then - source /usr/local/ghpc-venv/bin/activate - ansible-playbook /usr/local/ghpc/setup-ssh-keys.yml -fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml deleted file mode 100644 index 692896bb9c..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Setup SSH Keys for user - become: false - hosts: localhost - vars: - pub_key_path: "{{ ansible_env.HOME }}/.ssh" - pub_key_file: "{{ pub_key_path }}/id_rsa" - auth_key_file: "{{ pub_key_path }}/authorized_keys" - tasks: - - name: "Create .ssh folder" - ansible.builtin.file: - path: "{{ pub_key_path }}" - state: directory - mode: 0700 - owner: "{{ ansible_user_id }}" - - name: Create keys - community.crypto.openssh_keypair: - path: "{{ pub_key_file }}" - owner: "{{ ansible_user_id }}" - - name: Copy public key to authorized keys - ansible.builtin.copy: - src: "{{ pub_key_file }}.pub" - dest: "{{ auth_key_file }}" - owner: "{{ ansible_user_id }}" - mode: 0644 diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh deleted file mode 100644 index 8ca40bc73f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh +++ /dev/null @@ -1,39 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This code contains minor changes from the original: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 - -stdlib::main() { - DELETE_AT_EXIT="$(mktemp -d)" - readonly DELETE_AT_EXIT - - # Initialize state required by other functions, e.g. debug() - stdlib::init - stdlib::debug "Loaded startup-script-stdlib as an executable." - - stdlib::load_config_values - - stdlib::load_runners -} - -# if script is being executed and not sourced. -if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then - stdlib::finish() { - [[ -d ${DELETE_AT_EXIT:-} ]] && rm -rf "${DELETE_AT_EXIT}" - } - trap stdlib::finish EXIT - - stdlib::main "$@" -fi diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh deleted file mode 100644 index 589a3215ab..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh +++ /dev/null @@ -1,266 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This code contains minor changes from the original in: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 - -# Standard library of functions useful for startup scripts. - -# These are outside init_global_vars so logging functions work with the most -# basic case of `source startup-script-stdlib.sh` -readonly SYSLOG_DEBUG_PRIORITY="${SYSLOG_DEBUG_PRIORITY:-syslog.debug}" -readonly SYSLOG_INFO_PRIORITY="${SYSLOG_INFO_PRIORITY:-syslog.info}" -readonly SYSLOG_ERROR_PRIORITY="${SYSLOG_ERROR_PRIORITY:-syslog.error}" -# Global counter of how many times stdlib::init() has been called. -STARTUP_SCRIPT_STDLIB_INITIALIZED=0 - -# Error codes -readonly E_RUN_OR_DIE=5 -readonly E_MISSING_MANDATORY_ARG=9 -readonly E_UNKNOWN_ARG=10 - -SCRIPT_COMPLETE_FILE="/run/startup_script_msg" -SUCCESS_MESSAGE="* NOTICE **: The Cluster Toolkit startup scripts have finished running successfully." -readonly SUCCESS_MESSAGE -ERROR_MESSAGE="** ERROR **: The Cluster Toolkit startup scripts have finished running, but produced an error." -readonly ERROR_MESSAGE -WARNING_MESSAGE="** WARNING **: The Cluster Toolkit startup scripts are currently running." -readonly WARNING_MESSAGE - -stdlib::debug() { - [[ -z ${DEBUG:-} ]] && return 0 - local ds msg - msg="$*" - logger -p "${SYSLOG_DEBUG_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - [[ -n ${QUIET:-} ]] && return 0 - ds="$(date +"${DATE_FMT}") " - echo -e "${BLUE}${ds}Debug [$$]: ${msg}${NC}" >&2 -} - -stdlib::info() { - local ds msg - msg="$*" - logger -p "${SYSLOG_INFO_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - [[ -n ${QUIET:-} ]] && return 0 - ds="$(date +"${DATE_FMT}") " - echo -e "${GREEN}${ds}Info [$$]: ${msg}${NC}" >&2 -} - -stdlib::error() { - local ds msg - msg="$*" - ds="$(date +"${DATE_FMT}") " - logger -p "${SYSLOG_ERROR_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - echo -e "${RED}${ds}Error [$$]: ${msg}${NC}" >&2 -} - -stdlib::announce_runners_start() { - if [ -z "$recursive_proc" ]; then - wall -n "$WARNING_MESSAGE" - echo "$WARNING_MESSAGE" >"$SCRIPT_COMPLETE_FILE" - fi - export recursive_proc=$((${recursive_proc:=0} + 1)) -} - -stdlib::announce_runners_end() { - exit_code=$1 - export recursive_proc=$((${recursive_proc:=0} - 1)) - if [ "$recursive_proc" -le "0" ]; then - if [ "$exit_code" -ne "0" ]; then - wall -n "$ERROR_MESSAGE" - echo "$ERROR_MESSAGE" >"$SCRIPT_COMPLETE_FILE" - else - wall -n "$SUCCESS_MESSAGE" - echo -n "" >"$SCRIPT_COMPLETE_FILE" - fi - fi -} - -# The main initialization function of this library. This should be kept to the -# minimum amount of work required for all functions to operate cleanly. -stdlib::init() { - if [[ ${STARTUP_SCRIPT_STDLIB_INITIALIZED} -gt 0 ]]; then - stdlib::info 'stdlib::init()'" already initialized, no action taken." - return 0 - fi - ((STARTUP_SCRIPT_STDLIB_INITIALIZED++)) || true - stdlib::init_global_vars - stdlib::init_directories - stdlib::debug "stdlib::init(): startup-script-stdlib.sh initialized and ready" -} - -# Initialize global variables. -stdlib::init_global_vars() { - # The program name, used for logging. - readonly PROG="${PROG:-startup-script-stdlib}" - # Date format used for stderr logging. Passed to date + command. - readonly DATE_FMT="${DATE_FMT:-"%a %b %d %H:%M:%S %z %Y"}" - # var directory - readonly VARDIR="${VARDIR:-/var/lib/startup}" - # Override this with file://localhost/tmp/foo/bar in spec test context - readonly METADATA_BASE="${METADATA_BASE:-http://metadata.google.internal}" - - # Color variables - if [[ -n ${COLOR:-} ]]; then - readonly NC='\033[0m' # no color - readonly RED='\033[0;31m' # error - readonly GREEN='\033[0;32m' # info - readonly BLUE='\033[0;34m' # debug - else - readonly NC='' - readonly RED='' - readonly GREEN='' - readonly BLUE='' - fi - - return 0 -} - -stdlib::init_directories() { - if ! [[ -e ${VARDIR} ]]; then - install -d -m 0755 -o 0 -g 0 "${VARDIR}" - fi -} - -## -# Get a metadata key. When used without -o, this function is guaranteed to -# produce no output on STDOUT other than the retrieved value. This is intended -# to support the use case of -# FOO="$(stdlib::metadata_get -k instance/attributes/foo)" -# -# If the requested key does not exist, the error code will be 22 and zero bytes -# written to STDOUT. -stdlib::metadata_get() { - local OPTIND opt key outfile - local metadata="${METADATA_BASE%/}/computeMetadata/v1" - local exit_code - while getopts ":k:o:" opt; do - case "${opt}" in - k) key="${OPTARG}" ;; - o) outfile="${OPTARG}" ;; - :) - stdlib::error "Invalid option: -${OPTARG} requires an argument" - stdlib::metadata_get_usage - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error "Unknown option: -${opt}" - stdlib::metadata_get_usage - return "${E_UNKNOWN_ARG}" - ;; - esac - done - local url="${metadata}/${key#/}" - - stdlib::debug "Getting metadata resource url=${url}" - if [[ -z ${outfile:-} ]]; then - curl --location --silent --connect-timeout 1 --fail \ - -H 'Metadata-Flavor: Google' "$url" 2>/dev/null - exit_code=$? - else - stdlib::cmd curl --location \ - --silent \ - --connect-timeout 1 \ - --fail \ - --output "${outfile}" \ - -H 'Metadata-Flavor: Google' \ - "$url" - exit_code=$? - fi - case "${exit_code}" in - 22 | 37) - stdlib::debug "curl exit_code=${exit_code} for url=${url}" \ - "(Does not exist)" - ;; - esac - return "${exit_code}" -} - -stdlib::metadata_get_usage() { - stdlib::info 'Usage: stdlib::metadata_get -k ' - stdlib::info 'For example: stdlib::metadata_get -k instance/attributes/startup-config' -} - -# Load configuration values in the spirit of /etc/sysconfig defaults, but from -# metadata instead of the filesystem. -stdlib::load_config_values() { - local config_file - local key="instance/attributes/startup-script-config" - # shellcheck disable=SC2119 - config_file="$(stdlib::mktemp)" - stdlib::metadata_get -k "${key}" -o "${config_file}" - local status=$? - case "$status" in - 0) - stdlib::debug "SUCCESS: Configuration data sourced from $key" - ;; - 22 | 37) - stdlib::debug "no configuration data loaded from $key" - ;; - *) - stdlib::error "metadata_get -k $key returned unknown status=${status}" - ;; - esac - # shellcheck source=/dev/null - source "${config_file}" -} - -# Run a command logging the entry and exit. Intended for system level commands -# and operational debugging. Not intended for use with redirection. This is -# not named run() because bats uses a run() function. -stdlib::cmd() { - local exit_code argv=("$@") - stdlib::debug "BEGIN: stdlib::cmd() command=[${argv[*]}]" - "${argv[@]}" - exit_code=$? - stdlib::debug "END: stdlib::cmd() command=[${argv[*]}] exit_code=${exit_code}" - return $exit_code -} - -# Run a command successfully or exit the program with an error. -stdlib::run_or_die() { - if ! stdlib::cmd "$@"; then - stdlib::error "stdlib::run_or_die(): exiting with exit code ${E_RUN_OR_DIE}." - exit "${E_RUN_OR_DIE}" - fi -} - -# Intended to take advantage of automatic cleanup of startup script library -# temporary files without exporting a modified TMPDIR to child processes, which -# would cause the children to have their TMPDIR deleted out from under them. -# shellcheck disable=SC2120 -stdlib::mktemp() { - TMPDIR="${DELETE_AT_EXIT:-${TMPDIR}}" mktemp "$@" -} - -# Return a nice error message if a mandatory argument is missing. -stdlib::mandatory_argument() { - local OPTIND opt name flag - while getopts ":n:f:" opt; do - case "$opt" in - n) name="${OPTARG}" ;; - f) flag="${OPTARG}" ;; - :) - stdlib::error "Invalid argument: -${OPTARG} requires an argument to stdlib::mandatory_argument()" - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error "Unknown argument: -${OPTARG}" - stdlib::info "Usage: stdlib::mandatory_argument -n -f " - return "${E_UNKNOWN_ARG}" - ;; - esac - done - stdlib::error "Invalid argument: -${flag} requires an argument to ${name}()." -} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/main.tf b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/main.tf deleted file mode 100644 index 02124eeddc..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/main.tf +++ /dev/null @@ -1,306 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "startup-script", ghpc_role = "scripts" }) -} - -locals { - monitoring_agent_installer = ( - var.install_cloud_ops_agent || var.install_stackdriver_agent ? - [{ - type = "shell" - source = "${path.module}/files/install_monitoring_agent.sh" - destination = "install_monitoring_agent_automatic.sh" - args = var.install_cloud_ops_agent ? "ops" : "legacy" # install legacy (stackdriver) - }] : - [] - ) - - warnings = [ - { - type = "data" - content = file("${path.module}/files/running-script-warning.sh") - destination = "/etc/profile.d/99-running-script-warning.sh" - } - ] - - configure_ssh = length(var.configure_ssh_host_patterns) > 0 - host_args = { - host_name_prefix = var.configure_ssh_host_patterns - } - - prefix_file = "/tmp/prefix_file.json" - ansible_docker_settings_file = "/tmp/ansible_docker_settings.json" - - docker_config = try(jsondecode(var.docker.daemon_config), {}) - docker_data_root = try(local.docker_config.data-root, null) - - configure_ssh_runners = local.configure_ssh ? [ - { - type = "data" - source = "${path.module}/files/setup-ssh-keys.sh" - destination = "/usr/local/ghpc/setup-ssh-keys.sh" - }, - { - type = "data" - source = "${path.module}/files/setup-ssh-keys.yml" - destination = "/usr/local/ghpc/setup-ssh-keys.yml" - }, - { - type = "data" - content = jsonencode(local.host_args) - destination = local.prefix_file - }, - { - type = "ansible-local" - content = file("${path.module}/files/configure-ssh.yml") - destination = "configure-ssh.yml" - args = "-e @${local.prefix_file}" - } - ] : [] - - proxy_runner = var.http_proxy == "" ? [] : [ - { - type = "data" - destination = "/etc/profile.d/http_proxy.sh" - content = <<-EOT - #!/bin/bash - export http_proxy=${var.http_proxy} - export https_proxy=${var.http_proxy} - export NO_PROXY=${var.http_no_proxy} - EOT - }, - { - type = "shell" - source = "${path.module}/files/configure_proxy.sh" - destination = "configure_proxy.sh" - args = var.http_proxy - } - ] - - ofi_runner = !var.set_ofi_cloud_rdma_tunables ? [] : [ - { - type = "data" - destination = "/etc/profile.d/set_ofi_cloud_rdma_tunables.sh" - content = <<-EOT - #!/bin/bash - export FI_PROVIDER="verbs;ofi_rxm" - export FI_OFI_RXM_USE_RNDV_WRITE=0 - export FI_VERBS_INLINE_SIZE=39 - export I_MPI_FABRICS="shm:ofi" - export FI_UNIVERSE_SIZE=1024 - export I_MPI_ADJUST_ALLTOALL=1 - export I_MPI_ADJUST_IALLTOALL=1 - export I_MPI_ADJUST_BCAST=4 - export I_MPI_ADJUST_IBCAST=1 - EOT - }, - ] - - rdma_runner = !var.install_cloud_rdma_drivers ? [] : [ - { - type = "shell" - source = "${path.module}/files/install_cloud_rdma_drivers.sh" - destination = "install_cloud_rdma_drivers.sh" - } - ] - - docker_runner = !var.docker.enabled ? [] : [ - { - type = "data" - destination = local.ansible_docker_settings_file - content = jsonencode({ - enable_docker_world_writable = var.docker.world_writable - docker_daemon_config = var.docker.daemon_config - docker_data_root = local.docker_data_root - }) - }, - { - type = "ansible-local" - destination = "install_docker.yml" - content = file("${path.module}/files/install_docker.yml") - args = "-e \"@${local.ansible_docker_settings_file}\"" - }, - ] - - managed_lustre_runner = !var.managed_lustre.enabled ? [] : [ - { - type = "ansible-local" - destination = "install_managed_lustre.yml" - content = file("${path.module}/files/install_managed_lustre.yml") - args = "-e managed_lustre_port=${var.managed_lustre.port}" - }, - ] - - gpu_network_wait_online_runner = !var.enable_gpu_network_wait_online ? [] : [ - { - type = "ansible-local" - destination = "install_gpu_network_wait_online.yml" - content = file("${path.module}/files/install_gpu_network_wait_online.yml") - args = "" - }, - ] - - local_ssd_filesystem_enabled = can(coalesce(var.local_ssd_filesystem.mountpoint)) - raid_setup = !local.local_ssd_filesystem_enabled ? [] : [ - { - type = "ansible-local" - destination = "setup-raid.yml" - content = file("${path.module}/files/setup-raid.yml") - args = join(" ", [ - "-e mountpoint=${var.local_ssd_filesystem.mountpoint}", - "-e fs_type=${var.local_ssd_filesystem.fs_type}", - "-e mode=${var.local_ssd_filesystem.permissions}", - ]) - }, - ] - - supplied_ansible_runners = anytrue([for r in var.runners : r.type == "ansible-local"]) - has_ansible_runners = anytrue([ - local.supplied_ansible_runners, - local.configure_ssh, - var.docker.enabled, - var.managed_lustre.enabled, - var.enable_gpu_network_wait_online, - local.local_ssd_filesystem_enabled - ]) - - install_ansible = coalesce(var.install_ansible, local.has_ansible_runners) - ansible_installer = local.install_ansible ? [{ - type = "shell" - source = "${path.module}/files/install_ansible.sh" - destination = "install_ansible_automatic.sh" - args = var.ansible_virtualenv_path - }] : [] - - hotfix_runner = [{ - type = "shell" - source = "${path.module}/files/early_run_hotfixes.sh" - destination = "early_run_hotfixes.sh" - }] - - runners = concat( - local.warnings, - local.hotfix_runner, - local.proxy_runner, - local.ofi_runner, - local.rdma_runner, - local.monitoring_agent_installer, - local.ansible_installer, - local.raid_setup, # order RAID early to ensure filesystem is ready for subsequent runners - local.managed_lustre_runner, - local.configure_ssh_runners, - local.docker_runner, - local.gpu_network_wait_online_runner, - var.runners - ) - - bucket_regex = "^gs://([^/]*)/*(.*)" - gcs_bucket_path_trimmed = var.gcs_bucket_path == null ? null : trimsuffix(var.gcs_bucket_path, "/") - storage_folder_path = local.gcs_bucket_path_trimmed == null ? null : regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[1] - storage_folder_path_prefix = local.storage_folder_path == null || local.storage_folder_path == "" ? "" : "${local.storage_folder_path}/" - - user_provided_bucket_name = try(regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[0], null) - storage_bucket_name = coalesce(one(google_storage_bucket.configs_bucket[*].name), local.user_provided_bucket_name) - - load_runners = templatefile( - "${path.module}/templates/startup-script-custom.tftpl", - { - bucket = local.storage_bucket_name, - http_proxy = var.http_proxy, - no_proxy = var.http_no_proxy, - runners = [ - for runner in local.runners : { - object = google_storage_bucket_object.scripts[basename(runner["destination"])].output_name - type = runner["type"] - destination = runner["destination"] - args = contains(keys(runner), "args") ? runner["args"] : "" - } - ] - } - ) - - stdlib_head = file("${path.module}/files/startup-script-stdlib-head.sh") - get_from_bucket = file("${path.module}/files/get_from_bucket.sh") - stdlib_body = file("${path.module}/files/startup-script-stdlib-body.sh") - - # List representing complete content, to be concatenated together. - stdlib_list = [ - local.stdlib_head, - local.get_from_bucket, - local.load_runners, - local.stdlib_body, - ] - - # Final content output to the user - stdlib = join("", local.stdlib_list) - - runners_map = { for runner in local.runners : - basename(runner["destination"]) => { - content = lookup(runner, "content", null) - source = lookup(runner, "source", null) - } - } -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket" "configs_bucket" { - count = var.gcs_bucket_path == null ? 1 : 0 - project = var.project_id - name = "${var.deployment_name}-startup-scripts-${random_id.resource_name_suffix.hex}" - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -resource "google_storage_bucket_iam_binding" "viewers" { - bucket = local.storage_bucket_name - role = "roles/storage.objectViewer" - members = var.bucket_viewers -} - -resource "google_storage_bucket_object" "scripts" { - # this writes all scripts exactly once into GCS - for_each = local.runners_map - name = "${local.storage_folder_path_prefix}${each.key}-${substr(try(md5(each.value.content), filemd5(each.value.source)), 0, 4)}" - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) - bucket = local.storage_bucket_name - timeouts { - create = "10m" - update = "10m" - } - - lifecycle { - precondition { - condition = !(var.install_cloud_ops_agent && var.install_stackdriver_agent) - error_message = "Only one of var.install_stackdriver_agent or var.install_cloud_ops_agent can be set. Stackdriver is recommended for best performance." - } - } -} - -resource "local_file" "debug_file" { - for_each = toset(var.debug_file != null ? [var.debug_file] : []) - filename = var.debug_file - content = local.stdlib -} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/metadata.yaml b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/metadata.yaml deleted file mode 100644 index 2ada34471f..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - storage.googleapis.com diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/outputs.tf b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/outputs.tf deleted file mode 100644 index 6a15082814..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/outputs.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "script to load and run all runners, as a string value." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} - -output "compute_startup_script" { - description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} - -output "controller_startup_script" { - description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl deleted file mode 100644 index 3c894b00b0..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl +++ /dev/null @@ -1,65 +0,0 @@ - - -stdlib::run_playbook() { - if [ ! "$(which ansible-playbook)" ]; then - stdlib::error "ansible-playbook not found"\ - "Please install ansible before running ansible-local runners." - exit 1 - fi - ansible-playbook --connection=local --inventory=localhost, --limit localhost $1 $2 - ret_code=$? - return $${ret_code} -} - -stdlib::runner() { - - type=$1 - object=$2 - destination=$3 - tmpdir=$4 - args=$5 - - destpath="$(dirname $destination)" - filename="$(basename $destination)" - - if [ "$destpath" = "." ]; then - destpath=$tmpdir - fi - - stdlib::get_from_bucket -u "gs://${bucket}/$object" -d "$destpath" -f "$filename" - - stdlib::info "=== start executing runner: $object ===" - case "$1" in - ansible-local) stdlib::run_playbook "$destpath/$filename" "$args";; - shell) chmod u+x /$destpath/$filename && $destpath/$filename $args;; - esac - - exit_code=$? - stdlib::info "=== $object finished with exit_code=$exit_code ===" - if [ "$exit_code" -ne "0" ] ; then - stdlib::error "=== execution of $object failed, exiting ===" - stdlib::announce_runners_end "$exit_code" - exit $exit_code - fi -} - -stdlib::load_runners(){ - tmpdir="$(mktemp -d)" - - stdlib::debug "=== BEGIN Running runners ===" - stdlib::announce_runners_start - - %{if http_proxy != "" ~} - stdlib::info "=== Setting HTTP_PROXY,HTTPS_PROXY to ${http_proxy} ===" - export http_proxy=${http_proxy} - export https_proxy=${http_proxy} - export NO_PROXY=${no_proxy} - %{endif ~} - - %{for r in runners ~} - stdlib::runner "${r.type}" "${r.object}" "${r.destination}" $${tmpdir} "${r.args}" - %{endfor ~} - - stdlib::announce_runners_end "0" - stdlib::debug "=== END Running runners ===" -} diff --git a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/variables.tf b/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/variables.tf deleted file mode 100644 index 7080085ece..0000000000 --- a/deletion-test/cluster/modules/embedded/modules/scripts/startup-script/variables.tf +++ /dev/null @@ -1,298 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used to name GCS bucket for startup scripts." - type = string -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string - default = null -} - -variable "bucket_viewers" { - description = "Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account)" - type = list(string) - default = [] - - validation { - condition = alltrue([ - for u in var.bucket_viewers : length(regexall("^(allUsers$|allAuthenticatedUsers$|user:|group:|serviceAccount:|domain:)", u)) > 0 - ]) - error_message = "Bucket viewer members must begin with user/group/serviceAccount/domain following https://cloud.google.com/iam/docs/reference/rest/v1/Policy#Binding" - } -} - -variable "debug_file" { - description = "Path to an optional local to be written with 'startup_script'." - type = string - default = null -} - -variable "labels" { - description = "Labels for the created GCS bucket. Key-value pairs." - type = map(string) -} - -variable "runners" { - description = < 0 - error_message = "The POSIX permissions for the mountpoint must be represented as a 3 or 4-digit octal" - } - - default = { - fs_type = "ext4" - mountpoint = "" - permissions = "0755" - } - - nullable = false -} - -variable "install_cloud_ops_agent" { - description = "Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true." - type = bool - default = false -} - -variable "install_stackdriver_agent" { - description = "Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance." - type = bool - default = false -} - -variable "install_ansible" { - description = "Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used." - type = bool - default = null -} - -variable "configure_ssh_host_patterns" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 4.84 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [home\_pv](#module\_home\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | -| [kubectl\_apply](#module\_kubectl\_apply) | ../../../../modules/management/kubectl-apply | n/a | -| [slurm\_key\_pv](#module\_slurm\_key\_pv) | ../../../../modules/file-system/gke-persistent-volume | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.gke_nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | -| [filestore\_id](#input\_filestore\_id) | An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `list(string)` | n/a | yes | -| [image](#input\_image) | The image for slurm daemon | `string` | n/a | yes | -| [instance\_templates](#input\_instance\_templates) | The URLs of Instance Templates | `list(string)` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| n/a | yes | -| [node\_count\_static](#input\_node\_count\_static) | The number of static nodes in node-pool | `number` | n/a | yes | -| [node\_pool\_names](#input\_node\_pool\_names) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `list(string)` | n/a | yes | -| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | -| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used in slurm controller | `string` | n/a | yes | -| [slurm\_controller\_instance](#input\_slurm\_controller\_instance) | Slurm cluster controller instance | `any` | n/a | yes | -| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | -| [subnetwork](#input\_subnetwork) | Primary subnetwork object | `any` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset\_name](#output\_nodeset\_name) | Name of the new Slinky nodset | - diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/main.tf deleted file mode 100644 index 8b2f1deeac..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/main.tf +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -### GKE NodeSet -locals { - manifest_path = "${path.module}/templates/nodeset-general.yaml.tftpl" -} - -module "kubectl_apply" { - source = "../../../../modules/management/kubectl-apply" - - cluster_id = var.cluster_id - project_id = var.project_id - - apply_manifests = [{ - source = local.manifest_path, - template_vars = { - slurm_namespace = var.slurm_namespace, - nodeset_name = "${var.slurm_cluster_name}-${var.nodeset_name}", - nodeset_cr_name = "${var.slurm_cluster_name}-${var.nodeset_name}", - controller_name = "${var.slurm_cluster_name}-controller", - node_pool_name = var.node_pool_names[0], - node_count = var.node_count_static, - image = var.image, - home_pvc = module.home_pv.pvc_name - slurm_key_pvc = module.slurm_key_pv.pvc_name - } - }] -} - -data "google_storage_bucket" "this" { - name = var.slurm_bucket[0].name - - depends_on = [var.slurm_bucket] -} - -### Slurm NodeSet -locals { - nodeset = { - gke_nodepool = var.node_pool_names[0] - nodeset_name = var.nodeset_name - node_count_static = var.node_count_static - subnetwork = "https://www.googleapis.com/compute/v1/projects/${var.project_id}/regions/${var.subnetwork.region}/subnetworks/${var.subnetwork.name}" - instance_template = var.instance_templates[0] - } -} - -resource "google_storage_bucket_object" "gke_nodeset_config" { - bucket = data.google_storage_bucket.this.name - name = "${var.slurm_bucket_dir}/nodeset_configs/${var.nodeset_name}.yaml" - content = yamlencode(local.nodeset) -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml deleted file mode 100644 index ea2cfc221e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/output.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/output.tf deleted file mode 100644 index 15970ff0b7..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/output.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset_name" { - description = "Name of the new Slinky nodset" - value = local.nodeset.nodeset_name -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf deleted file mode 100644 index 8a190c4019..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/persistent_volumes.tf +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - slurm_key_storage = { - server_ip = var.slurm_controller_instance.network_interface[0].network_ip - remote_mount = "/slurm/key_distribution" # defined in /community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py - client_install_runner = {} - mount_runner = {} - fs_type = "" - local_mount = "" - mount_options = "" - } -} - -module "slurm_key_pv" { - source = "../../../../modules/file-system/gke-persistent-volume" - labels = {} - capacity_gib = 1 - cluster_id = var.cluster_id - filestore_id = "projects/empty/locations/empty/instances/empty" # this does not apply since this NFS is not a filestore - namespace = var.slurm_namespace - network_storage = local.slurm_key_storage - pv_name = "slurm-key-pv" - pvc_name = "slurm-key-pvc" -} - -# Assume the var.network_storage[0] will be home and only one home pv is accepted for now. -module "home_pv" { - source = "../../../../modules/file-system/gke-persistent-volume" - labels = {} - capacity_gib = 1024 - cluster_id = var.cluster_id - filestore_id = var.filestore_id[0] - network_storage = var.network_storage[0] - namespace = var.slurm_namespace - pv_name = "home-pv" - pvc_name = "home-pvc" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl deleted file mode 100644 index a5a4a5e7ac..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/templates/nodeset-general.yaml.tftpl +++ /dev/null @@ -1,203 +0,0 @@ -apiVersion: slinky.slurm.net/v1alpha1 -kind: NodeSet -metadata: - annotations: - meta.helm.sh/release-name: slurm - meta.helm.sh/release-namespace: ${slurm_namespace} - labels: - app.kubernetes.io/component: compute - app.kubernetes.io/instance: slurm - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: slurmd - app.kubernetes.io/part-of: slurm - app.kubernetes.io/version: "24.11" - helm.sh/chart: slurm-0.3.0 - nodeset.slinky.slurm.net/name: ${nodeset_name} - name: ${nodeset_name} - namespace: ${slurm_namespace} -spec: - clusterName: slurm - persistentVolumeClaimRetentionPolicy: - whenDeleted: Retain - whenScaled: Retain - replicas: ${node_count} - revisionHistoryLimit: 0 - selector: - matchLabels: - app.kubernetes.io/instance: slurm - app.kubernetes.io/name: slurmd - nodeset.slinky.slurm.net/name: ${nodeset_name} - serviceName: slurm-compute - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: slurmd - labels: - app.kubernetes.io/component: compute - app.kubernetes.io/instance: slurm - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: slurmd - app.kubernetes.io/part-of: slurm - app.kubernetes.io/version: "24.11" - helm.sh/chart: slurm-0.3.0 - nodeset.slinky.slurm.net/name: ${nodeset_name} - spec: - automountServiceAccountToken: false - containers: - - args: - - -g - - -- - - bash - - -c - - | - mkdir -p /usr/local/lib/slurm - ln -s /usr/lib/x86_64-linux-gnu/slurm/spank_pyxis.so /usr/local/lib/slurm/spank_pyxis.so - /usr/local/bin/entrypoint.sh -Z --conf-server ${controller_name}:6825 -N $NODE_NAME - command: - - tini - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_CPUS - value: "0" - - name: POD_MEMORY - value: "0" - image: ${image} - imagePullPolicy: IfNotPresent - name: slurmd - ports: - - containerPort: 6818 - name: slurmd - protocol: TCP - readinessProbe: - exec: - command: - - scontrol - - show - - slurmd - resources: {} - securityContext: - capabilities: - add: - - BPF - - NET_ADMIN - - SYS_ADMIN - - SYS_NICE - privileged: true - volumeMounts: - - mountPath: /etc/slurm - name: etc-slurm - - mountPath: /run - name: run - - mountPath: /var/spool/slurmd - name: slurm-spool - - mountPath: /var/log/slurm - name: slurm-log - - mountPath: /home - name: home-pvc - dnsConfig: - searches: - - ${controller_name} - hostNetwork: true - initContainers: - - command: - - tini - - -g - - -- - - bash - - -c - - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# - SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env - contains:\n# SLURM_USER - username or UID\n\nfunction init::common() {\n\tlocal - dir\n\n\tdir=/var/spool/slurmd\n\tmkdir -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" - \"$dir\"\n\tchmod -v 700 \"$dir\"\n\n\tdir=/var/spool/slurmctld\n\tmkdir - -p \"$dir\"\n\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" \"$dir\"\n\tchmod - -v 700 \"$dir\"\n}\n\nfunction init::slurm() {\n\tSLURM_MOUNT=/mnt/slurm\n\tSLURM_DIR=/mnt/etc/slurm\n\n\t# - Workaround to ephemeral volumes not supporting securityContext\n\t# https://github.com/kubernetes/kubernetes/issues/81089\n\n\t# - Copy Slurm config files, secrets, and scripts\n\tmkdir -p \"$SLURM_DIR\"\n\tfind - \"$${SLURM_MOUNT}\" -type f -name \"*.conf\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind - \"$${SLURM_MOUNT}\" -type f -name \"*.key\" -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind - \"$${SLURM_MOUNT}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" - -print0 | xargs -0r cp -vt \"$${SLURM_DIR}\"\n\tfind \"$${SLURM_MOUNT}\" -type - f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 | xargs - -0r cp -vt \"$${SLURM_DIR}\"\n\n\t# Set general permissions and ownership\n\tfind - \"$${SLURM_DIR}\" -type f -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind - \"$${SLURM_DIR}\" -type f -name \"*.conf\" -print0 | xargs -0r chmod -v 644\n\tfind - \"$${SLURM_DIR}\" -type f -name \"*.key\" -print0 | xargs -0r chmod -v 600\n\tfind - \"$${SLURM_DIR}\" -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" - -print0 | xargs -0r chown -v \"$${SLURM_USER}:$${SLURM_USER}\"\n\tfind \"$${SLURM_DIR}\" - -type f -regextype posix-extended -regex \"^.*/(pro|epi)log-.*$\" -print0 - | xargs -0r chmod -v 755\n\n\t# Inject secrets into certain config files\n\tlocal - dbd_conf=\"slurmdbd.conf\"\n\tif [[ -f \"$${SLURM_MOUNT}/$${dbd_conf}\" ]]; - then\n\t\techo \"Injecting secrets from environment into: $${dbd_conf}\"\n\t\trm - -f \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tenvsubst <\"$${SLURM_MOUNT}/$${dbd_conf}\" - >\"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchown -v \"$${SLURM_USER}:$${SLURM_USER}\" - \"$${SLURM_DIR}/$${dbd_conf}\"\n\t\tchmod -v 600 \"$${SLURM_DIR}/$${dbd_conf}\"\n\tfi\n\n\t# - Display Slurm directory files\n\tls -lAF \"$${SLURM_DIR}\"\n}\n\nfunction - main() {\n\tinit::common\n\tinit::slurm\n}\nmain\n" - env: - - name: SLURM_USER - value: slurm - image: ${image} - imagePullPolicy: IfNotPresent - name: init - resources: {} - volumeMounts: - - mountPath: /mnt/slurm - name: slurm-config - - mountPath: /mnt/etc/slurm - name: etc-slurm - - command: - - tini - - -g - - -- - - bash - - -c - - "#!/usr/bin/env bash\n# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.\n# - SPDX-License-Identifier: Apache-2.0\n\nset -euo pipefail\n\n# Assume env - contains:\n# SOCKET - Named socket to read from\n\nmkdir -v -p \"$(dirname - \"$SOCKET\")\"\nrm -f \"$SOCKET\"\nif ! [ -f \"$SOCKET\" ]; then\n\tmkfifo - -m 777 \"$SOCKET\"\nfi\nwhile IFS=\"\" read data; do\n\techo $data\ndone - <\"$SOCKET\"\n" - env: - - name: SOCKET - value: /var/log/slurm/slurmd.log - image: ghcr.io/slinkyproject/sackd:24.11-ubuntu24.04 - imagePullPolicy: IfNotPresent - name: logfile - resources: {} - restartPolicy: Always - volumeMounts: - - mountPath: /var/log/slurm - name: slurm-log - nodeSelector: - cloud.google.com/gke-nodepool: ${node_pool_name} - tolerations: - - effect: NoSchedule - key: nvidia.com/gpu - operator: Equal - value: present - volumes: - - emptyDir: - medium: Memory - name: etc-slurm - - emptyDir: {} - name: run - - name: slurm-config - persistentVolumeClaim: - claimName: ${slurm_key_pvc} - - emptyDir: - medium: Memory - name: slurm-spool - - emptyDir: - medium: Memory - name: slurm-log - - name: home-pvc - persistentVolumeClaim: - claimName: ${home_pvc} - updateStrategy: - rollingUpdate: - maxUnavailable: 20% - type: RollingUpdate diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/variables.tf deleted file mode 100644 index c091a0da86..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/variables.tf +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "cluster_id" { - description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" - type = string -} - -variable "slurm_cluster_name" { - type = string - description = "Cluster name, used in slurm controller" - - validation { - condition = var.slurm_cluster_name != null && can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) - error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." - } -} - -variable "slurm_controller_instance" { - type = any - description = "Slurm cluster controller instance" -} - -variable "image" { - description = "The image for slurm daemon" - type = string - nullable = false -} - -variable "node_pool_names" { - description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." - type = list(string) - nullable = false -} - -variable "node_count_static" { - description = "The number of static nodes in node-pool" - type = number -} - -variable "subnetwork" { - description = "Primary subnetwork object" - type = any -} - -variable "slurm_namespace" { - description = "slurm namespace for charts" - type = string - default = "slurm" -} - -variable "nodeset_name" { - description = "The nodeset name" - type = string - default = "gkenodeset" -} - -variable "slurm_bucket_dir" { - description = "Path directory within `bucket_name` for Slurm cluster file storage." - type = string - nullable = false -} - -variable "slurm_bucket" { - description = "GCS Bucket of Slurm cluster file storage." - type = any - nullable = true -} - -variable "instance_templates" { - description = "The URLs of Instance Templates" - type = list(string) - nullable = false -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured on nodes." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - - validation { - condition = length(var.network_storage) == 1 && var.network_storage[0].local_mount == "/home" - error_message = "The 'network_storage' variable must contain exactly one element, and that element's 'local_mount' attribute must be \"/home\"." - } -} - -variable "filestore_id" { - description = "An array of identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." - type = list(string) - - validation { - condition = length(var.filestore_id) == 1 - error_message = "The 'filestore_id' variable must contain exactly one element." - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/versions.tf deleted file mode 100644 index 3d7237cb92..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-nodeset/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.84" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-nodeset/v1.51.0" - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/README.md deleted file mode 100644 index 2a7c363a87..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/README.md +++ /dev/null @@ -1,39 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 4.84 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [has\_tpu](#input\_has\_tpu) | If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu. | `bool` | `false` | no | -| [nodeset\_name](#input\_nodeset\_name) | The nodeset name | `string` | `"gkenodeset"` | no | -| [partition\_name](#input\_partition\_name) | The partition name | `string` | `"gke"` | no | -| [slurm\_bucket](#input\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | `any` | n/a | yes | -| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/main.tf deleted file mode 100644 index 2949fd6594..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/main.tf +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -data "google_storage_bucket" "this" { - name = var.slurm_bucket[0].name - - depends_on = [var.slurm_bucket] -} - -### Slurm Partition -locals { - partition_conf = { - "PowerDownOnIdle" = "NO" - "SuspendTime" = "INFINITE" - "SuspendTimeout" = var.has_tpu ? 240 : 120 - "ResumeTimeout" = var.has_tpu ? 600 : 300 - } - - partition = { - partition_name = var.partition_name - partition_conf = local.partition_conf - - partition_nodeset = [var.nodeset_name] - partition_nodeset_tpu = [] - partition_nodeset_dyn = [] - # Options - enable_job_exclusive = true - power_down_on_idle = false - } -} - -resource "google_storage_bucket_object" "parition_config" { - bucket = data.google_storage_bucket.this.name - name = "${var.slurm_bucket_dir}/partition_configs/${var.partition_name}.yaml" - content = yamlencode(local.partition) -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/metadata.yaml deleted file mode 100644 index 557e1fc2ae..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/variables.tf deleted file mode 100644 index 3aeed2e59a..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/variables.tf +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "has_tpu" { - description = "If set to true, the nodeset template's Pod spec will contain request/limit for TPU resource, open port 8740 for TPU communication and add toleration for google.com/tpu." - type = bool - default = false -} - -variable "nodeset_name" { - description = "The nodeset name" - type = string - default = "gkenodeset" -} - -variable "partition_name" { - description = "The partition name" - type = string - default = "gke" -} - -variable "slurm_bucket_dir" { - description = "Path directory within `bucket_name` for Slurm cluster file storage." - type = string - nullable = false -} - -variable "slurm_bucket" { - description = "GCS Bucket of Slurm cluster file storage." - type = any - nullable = true -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/versions.tf deleted file mode 100644 index aede55263c..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/gke-partition/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.84" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-partition/v1.51.0" - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md deleted file mode 100644 index 4f65411ddf..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md +++ /dev/null @@ -1,271 +0,0 @@ -## Description - -This module performs the following tasks: - -- create an instance template from which execute points will be created -- create a managed instance group ([MIG][mig]) for execute points -- create a Toolkit runner to configure the autoscaler to scale the MIG - -It is expected to be used with the [htcondor-install] and [htcondor-setup] -modules. - -[htcondor-install]: ../../scripts/htcondor-install/README.md -[htcondor-setup]: ../../scheduler/htcondor-setup/README.md -[mig]: https://cloud.google.com/compute/docs/instance-groups/ - -### Known limitations - -This module may be used multiple times in a blueprint to create sets of -execute points in an HTCondor pool. If used more than 1 time, the setting -[name_prefix](#input_name_prefix) must be set to a value that is unique across -all uses of the htcondor-execute-point module. If you do not follow this -constraint, you will likely receive an error while running `terraform apply` -similar to that shown below. - -```text -Error: Invalid value for variable - - on modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf line 136, in module "startup_script": - 136: runners = local.all_runners - ├──────────────── - │ var.runners is list of map of string with 5 elements - -All startup-script runners must have a unique destination. -``` - -### How to configure jobs to select execute points - -HTCondor access points provisioned by the Toolkit are specially configured to -honor an attribute named `RequireId` in each [Job ClassAd][jobad]. This value -must be set to the ID of a MIG created by an instance of this module. The -[htcondor-access-point] module includes a setting `var.default_mig_id` that will -set this value automatically to the MIG ID corresponding to the module's -execute points. If this setting is left unset each job must specify `+RequireId` -explicitly. In all cases, the default value can be overridden explicitly as shown -below: - -```text -universe = vanilla -executable = /bin/echo -arguments = "Hello, World!" -output = out.$(ClusterId).$(ProcId) -error = err.$(ClusterId).$(ProcId) -log = log.$(ClusterId).$(ProcId) -request_cpus = 1 -request_memory = 100MB -+RequireId = "htcondor-pool-ep-mig" -queue -``` - -[htcondor-access-point]: ../../scheduler/htcondor-access-point/README.md -[jobad]: https://htcondor.readthedocs.io/en/latest/users-manual/matchmaking-with-classads.html - -### Example - -A full example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -The following code snippet creates a pool with 2 sets of HTCondor execute -points, one using On-demand pricing and the other using Spot pricing. They use -a startup script and network created in previous steps. - -```yaml -- id: htcondor_execute_point - source: community/modules/compute/htcondor-execute-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - min_idle: 2 - -- id: htcondor_execute_point_spot - source: community/modules/compute/htcondor-execute-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - spot: true - -- id: htcondor_access - source: community/modules/scheduler/htcondor-access-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - - htcondor_execute_point - - htcondor_execute_point_spot - settings: - default_mig_id: $(htcondor_execute_point.mig_id) - enable_public_ips: true - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - access_point_ips - - access_point_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## Behavior of Managed Instance Group (MIG) - -Regional [MIGs][mig] are used to provision Execute Points. By default, VMs -will be provisioned in any of the zones available in that region, however, it -can be constrained to run in fewer zones (or a single zone) using -[var.zones](#input_zones). - -When the configuration of an Execute Point is changed, the MIG can be configured -to [replace the VM][replacement] using a "proactive" or "opportunistic" policy. -By default, the policy is set to opportunistic. In practice, this means that -Execute Points will _NOT_ be automatically replaced by Terraform when changes to -the instance template / HTCondor configuration are made. We recommend leaving -this at the default value as it will allow the HTCondor autoscaler to replace -VMs when they become idle without disrupting running jobs. - -However, if it is desired [var.update_policy](#input_update_policy) can be set -to "PROACTIVE" to enable automatic replacement. This will disrupt running jobs -and send them back to the queue. Alternatively, one can leave the setting at -the default value of "OPPORTUNISTIC" and update: - -- intentionally by issuing an update via Cloud Console or using gcloud (below) -- VMs becomes unhealthy or are otherwise automatically replaced (e.g. regular - Google Cloud maintenance) - -For example, to manually update all instances in a MIG: - -```text -gcloud compute instance-groups managed update-instances \ - <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - -## Known Issues - -When using OS Login with "external users" (outside of the Google Cloud -organization), then Docker universe jobs will fail and cause the Docker daemon -to crash. This stems from the use of POSIX user ids (uid) outside the range -supported by Docker. Please consider disabling OS Login if this atypical -situation applies. - -```yaml -vars: - # add setting below to existing deployment variables - enable_oslogin: DISABLE -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 4.0 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.0 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [execute\_point\_instance\_template](#module\_execute\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [mig](#module\_mig) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.execute_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.execute_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Disk type for template | `string` | `"pd-balanced"` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape across zones for instance group managing execute points | `string` | `"ANY"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [execute\_point\_runner](#input\_execute\_point\_runner) | A list of Toolkit runners for configuring an HTCondor execute point | `list(map(string))` | `[]` | no | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account for HTCondor execute point (e-mail format) | `string` | n/a | yes | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | HTCondor execute point VM image

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to HTConodr execute points | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor execute points | `string` | `"n2-standard-4"` | no | -| [max\_size](#input\_max\_size) | Maximum size of the HTCondor execute point pool. | `number` | `5` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor execute points | `map(string)` | `{}` | no | -| [min\_idle](#input\_min\_idle) | Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max\_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly. | `number` | `0` | no | -| [name\_prefix](#input\_name\_prefix) | Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network HTCondor execute points will join | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HTCondor execute points will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which HTCondor execute points will be created | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork HTCondor execute points will join | `string` | `null` | no | -| [target\_size](#input\_target\_size) | Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size. | `number` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | -| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | Startup script to run at boot-time for Windows-based HTCondor execute points | `list(string)` | `[]` | no | -| [zones](#input\_zones) | Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [autoscaler\_runner](#output\_autoscaler\_runner) | Toolkit runner to configure the HTCondor autoscaler | -| [mig\_id](#output\_mig\_id) | ID of the managed instance group containing the execute points | - diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml deleted file mode 100644 index 375ae036cd..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Role - hosts: localhost - become: true - vars: - spool_dir: /var/lib/condor/spool - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - handlers: - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml deleted file mode 100644 index a85158fdfc..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This playbook makes the assumption that a virtual environment has been created -# with the autoscaler and its dependencies previously installed. A runner that -# does this is provided as an output of the htcondor-install module within the -# Cluster Toolkit at community/modules/scripts/htcondor-install. - ---- -- name: Configure HTCondor Autoscaler - hosts: all - vars: - python: /usr/local/htcondor/bin/python3 - autoscaler: /usr/local/htcondor/bin/autoscaler.py - systemd_override_path: /etc/systemd/system - become: true - tasks: - - name: User must supply HTCondor role - ansible.builtin.assert: - that: - - project_id is defined - - region is defined - - zone is defined - - mig_id is defined - - max_size is defined - - name: Create SystemD service for HTCondor autoscaler - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@.service" - mode: 0644 - content: | - [Unit] - Description=HTCondor Autoscaler MIG: %i - - [Service] - User=condor - Type=oneshot - ExecStart={{ python }} {{ autoscaler }} --p $PROJECT_ID --r $REGION --z $ZONE --mz --g %i --c $MAX_SIZE --i $MIN_IDLE - notify: - - Reload SystemD - - name: Create SystemD override directory for autoscaler configuration - ansible.builtin.file: - path: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d" - state: directory - owner: root - group: root - mode: 0755 - - name: Create autoscaler configuration - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d/miglimit.conf" - mode: 0644 - content: | - [Service] - Environment=PROJECT_ID={{ project_id }} - Environment=REGION={{ region }} - Environment=ZONE={{ zone }} - Environment=MAX_SIZE={{ max_size }} - Environment=MIN_IDLE={{ min_idle }} - notify: - - Reload SystemD - - name: Create SystemD timer for HTCondor autoscaler - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@.timer" - mode: 0644 - content: | - [Unit] - Description=Run HTCondor Autoscaler Periodically - - [Timer] - OnCalendar=minutely - AccuracySec=1us - RandomizedDelaySec=30 - # the directive below is ignored harmlessly on CentOS 7; this has impact - # that timing averages to 1 minute but is not precisely 1 minute; still - # useful to ensure that timers for different MIGs do not overlap - FixedRandomDelay=true - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - post_tasks: - - name: Activate HTCondor Autoscaler timer - ansible.builtin.systemd: - name: htcondor-autoscaler@{{ mig_id }}.timer - enabled: true - state: started diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf deleted file mode 100644 index 7b0df94987..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf +++ /dev/null @@ -1,218 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-execute-point", ghpc_role = "compute" }) -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - zones = coalescelist(var.zones, data.google_compute_zones.available.names) - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - windows_startup_ps1 = join("\n\n", flatten([var.windows_startup_ps1, local.execute_config_windows_startup_ps1])) - - is_windows_image = anytrue([for l in data.google_compute_image.compute_image.licenses : length(regexall("windows-cloud", l)) > 0]) - windows_startup_metadata = local.is_windows_image && local.windows_startup_ps1 != "" ? { - windows-startup-script-ps1 = local.windows_startup_ps1 - } : {} - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.windows_startup_metadata, - local.network_storage_metadata, - local.enable_oslogin, - local.disable_automatic_updates_metadata, - var.metadata - ) - - autoscaler_runner = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_configure_autoscaler.yml") - "destination" = "htcondor_configure_autoscaler_${module.mig.instance_group_manager.name}.yml" - "args" = join(" ", [ - "-e project_id=${var.project_id}", - "-e region=${var.region}", - "-e zone=${local.zones[0]}", # this value is required, but ignored by regional MIG autoscaler - "-e mig_id=${module.mig.instance_group_manager.name}", - "-e max_size=${var.max_size}", - "-e min_idle=${var.min_idle}", - ]) - } - - execute_config = templatefile("${path.module}/templates/condor_config.tftpl", { - htcondor_role = "get_htcondor_execute", - central_manager_ips = var.central_manager_ips, - guest_accelerator = local.guest_accelerator, - }) - - execute_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.execute_config.output_name}" - execute_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e htcondor_role=get_htcondor_execute", - "-e config_object=${local.execute_object}", - ]) - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.execute_point_runner, - [local.execute_runner], - ) - - execute_config_windows_startup_ps1 = templatefile( - "${path.module}/templates/download-condor-config.ps1.tftpl", - { - config_object = local.execute_object, - } - ) - - name_prefix = "${var.deployment_name}-${var.name_prefix}-ep" -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -resource "null_resource" "execute_config" { - triggers = { - config = local.execute_config - } -} - -resource "google_storage_bucket_object" "execute_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.execute_config.id), 0, 4)}" - content = local.execute_config - bucket = var.htcondor_bucket_name -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -module "execute_point_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.execute_point_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - gpu = one(local.guest_accelerator) - preemptible = var.spot - startup_script = local.is_windows_image ? null : module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.compute_image.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config -} - -module "mig" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = var.target_size - hostname = local.name_prefix - mig_name = local.name_prefix - instance_template = module.execute_point_instance_template.self_link - - health_check_name = "health-htcondor-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "SUBSTITUTE" - max_surge_fixed = length(local.zones) - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf deleted file mode 100644 index b31f40130f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "autoscaler_runner" { - value = local.autoscaler_runner - description = "Toolkit runner to configure the HTCondor autoscaler" -} - -output "mig_id" { - value = module.mig.instance_group_manager.name - description = "ID of the managed instance group containing the execute points" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl deleted file mode 100644 index c8f5ce31a8..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:${htcondor_role} -CONDOR_HOST = ${join(",", central_manager_ips)} - -# StartD configuration settings -%{ if length(guest_accelerator) > 0 ~} -use feature:GPUs -%{ endif ~} -use feature:PartitionableSlot -use feature:CommonCloudAttributesGoogle("-c created-by") -UPDATE_INTERVAL = 30 -TRUST_UID_DOMAIN = True -STARTER_ALLOW_RUNAS_OWNER = True -RUNBENCHMARKS = False diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl deleted file mode 100644 index 19789f122e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl +++ /dev/null @@ -1,34 +0,0 @@ -# create directory for local condor_config customizations -$config_dir = 'C:\Condor\config' -if(!(test-path -PathType container -Path $config_dir)) -{ - New-Item -ItemType Directory -Path $config_dir -} - -# update local condor_config if blueprint has changed -$config_file = "$config_dir\50-ghpc-managed" -if (Test-Path -Path $config_file -PathType Leaf) -{ - $local_hash = gcloud --format="value(md5_hash)" storage hash $config_file -} -else -{ - $local_hash = "INVALID-HASH" -} - -$remote_hash = gcloud --format="value(md5_hash)" storage hash ${config_object} -if ($local_hash -cne $remote_hash) -{ - Write-Output "Updating condor configuration" - gcloud storage cp ${config_object} $config_file - if ($LASTEXITCODE -ne 0) - { - throw "Could not download HTCondor configuration; exiting startup script" - } - Restart-Service condor -} - -# ignored if service is already running; must be here to handle case where -# machine is rebooted, but configuration has previously been downloaded -# and service is disabled from automatic start -Start-Service condor diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf deleted file mode 100644 index aab8a54c2d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf +++ /dev/null @@ -1,265 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HTCondor execute points will be created" - type = string -} - -variable "region" { - description = "The region in which HTCondor execute points will be created" - type = string -} - -variable "zones" { - description = "Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region." - type = list(string) - default = [] - nullable = false -} - -variable "distribution_policy_target_shape" { - description = "Target shape across zones for instance group managing execute points" - type = string - default = "ANY" -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to HTConodr execute points" - type = map(string) -} - -variable "machine_type" { - description = "Machine type to use for HTCondor execute points" - type = string - default = "n2-standard-4" -} - -variable "execute_point_runner" { - description = "A list of Toolkit runners for configuring an HTCondor execute point" - type = list(map(string)) - default = [] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "instance_image" { - description = <<-EOD - HTCondor execute point VM image - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "execute_point_service_account_email" { - description = "Service account for HTCondor execute point (e-mail format)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_self_link" { - description = "The self link of the network HTCondor execute points will join" - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork HTCondor execute points will join" - type = string - default = null -} - -variable "target_size" { - description = "Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size." - type = number - default = null -} - -variable "max_size" { - description = "Maximum size of the HTCondor execute point pool." - type = number - default = 5 -} - -variable "min_idle" { - description = "Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly." - type = number - default = 0 -} - -variable "metadata" { - description = "Metadata to add to HTCondor execute points" - type = map(string) - default = {} -} - -# this default is deliberately the opposite of vm-instance because of observed -# issues running HTCondor docker universe jobs with OS Login enabled and running -# jobs as a user with uid>2^31; these uids occur when users outside the GCP -# organization login to a VM and OS Login is enabled. -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "spot" { - description = "Provision VMs using discounted Spot pricing, allowing for preemption" - type = bool - default = false -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 100 -} - -variable "disk_type" { - description = "Disk type for template" - type = string - default = "pd-balanced" -} - -variable "windows_startup_ps1" { - description = "Startup script to run at boot-time for Windows-based HTCondor execute points" - type = list(string) - default = [] - nullable = false -} - -variable "central_manager_ips" { - description = "List of IP addresses of HTCondor Central Managers" - type = list(string) -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The HTCondor module supports 0 or 1 models of accelerator card on each execute point" - } -} - -variable "name_prefix" { - description = "Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module" - type = string - nullable = false - validation { - condition = length(var.name_prefix) > 0 - error_message = "var.name_prefix must be a set to a non-empty string and must also be unique across all instances of htcondor-execute-point" - } -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" - type = string - default = "OPPORTUNISTIC" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf deleted file mode 100644 index 729dc3cda5..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 1.1" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.74.0" - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/mig/README.md deleted file mode 100644 index 278207b04a..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/mig/README.md +++ /dev/null @@ -1,45 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_group_manager.mig](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_group_manager) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [base\_instance\_name](#input\_base\_instance\_name) | Base name for the instances in the MIG | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will be used to name MIG if `var.name` is not provided | `string` | n/a | yes | -| [ghpc\_module\_id](#input\_ghpc\_module\_id) | Internal GHPC field, do not set this value | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the MIG | `map(string)` | n/a | yes | -| [name](#input\_name) | Name of the MIG. If not provided, will be generated from `var.deployment_name` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the MIG will be created | `string` | n/a | yes | -| [target\_size](#input\_target\_size) | Target number of instances in the MIG | `number` | `0` | no | -| [versions](#input\_versions) | Application versions managed by this instance group. Each version deals with a specific instance template | 
list(object({
    name              = string
    instance_template = string
    target_size = optional(object({
      fixed   = optional(number)
      percent = optional(number)
    }))
  }))
| n/a | yes | -| [wait\_for\_instances](#input\_wait\_for\_instances) | Whether to wait for all instances to be created/updated before returning | `bool` | `false` | no | -| [zone](#input\_zone) | Compute Platform zone. Required, currently only zonal MIGs are supported | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [self\_link](#output\_self\_link) | The URL of the created MIG | - diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/mig/main.tf deleted file mode 100644 index 0e7cf186c2..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/mig/main.tf +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "mig", ghpc_role = "compute" }) -} - -locals { - sanitized_deploy_name = try(replace(lower(var.deployment_name), "/[^a-z0-9]/", ""), null) - sanitized_module_id = try(replace(lower(var.ghpc_module_id), "/[^a-z0-9]/", ""), null) - synth_mig_name = try("${local.sanitized_deploy_name}-${local.sanitized_module_id}", null) - - mig_name = var.name == null ? local.synth_mig_name : var.name - base_instance_name = var.base_instance_name == null ? local.mig_name : var.base_instance_name -} - -resource "google_compute_instance_group_manager" "mig" { - # REQUIRED - name = local.mig_name - base_instance_name = local.base_instance_name - zone = var.zone - - dynamic "version" { - for_each = var.versions - content { - name = version.value.name - instance_template = version.value.instance_template - dynamic "target_size" { - for_each = version.value.target_size != null ? [version.value.target_size] : [] - content { - fixed = target_size.value.fixed - percent = target_size.value.percent - } - } - } - } - - # OPTIONAL - project = var.project_id - target_size = var.target_size - wait_for_instances = var.wait_for_instances - - all_instances_config { - # TODO: validate that template metadata not getting wiped out - # TODO: validate that template labels not getting wiped out - labels = local.labels - } - - # OMITTED: - # * description - # * named_port - # * list_managed_instances_results - # * target_pools - specific for Load Balancers usage - # * wait_for_instances_status - # * auto_healing_policies - # * stateful_disk - # * stateful_internal_ip - # * update_policy - # * params - - - lifecycle { - precondition { - condition = local.mig_name != null - error_message = "Could not come up with a name for the MIG, specify `var.name`" - } - - precondition { - condition = local.base_instance_name != null - error_message = "Could not come up with a base_instance_name, specify `var.base_instance_name`" - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/mig/metadata.yaml deleted file mode 100644 index 97a4fa9a89..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/mig/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com -ghpc: - inject_module_id: ghpc_module_id diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/mig/outputs.tf deleted file mode 100644 index 23c66a3535..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/mig/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "self_link" { - description = "The URL of the created MIG" - value = google_compute_instance_group_manager.mig.self_link -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/mig/variables.tf deleted file mode 100644 index b6c3c0e78a..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/mig/variables.tf +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "Project in which the MIG will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the deployment, will be used to name MIG if `var.name` is not provided" - type = string -} - -variable "labels" { - description = "Labels to add to the MIG" - type = map(string) -} - -variable "zone" { - description = "Compute Platform zone. Required, currently only zonal MIGs are supported" - type = string -} - - -variable "versions" { - description = <<-EOD - Application versions managed by this instance group. Each version deals with a specific instance template - EOD - type = list(object({ - name = string - instance_template = string - target_size = optional(object({ - fixed = optional(number) - percent = optional(number) - })) - })) - - validation { - condition = length(var.versions) > 0 - error_message = "At least one version must be provided" - } - -} - - -variable "ghpc_module_id" { - description = "Internal GHPC field, do not set this value" - type = string - default = null -} - -variable "name" { - description = "Name of the MIG. If not provided, will be generated from `var.deployment_name`" - type = string - default = null -} - -variable "base_instance_name" { - description = "Base name for the instances in the MIG" - type = string - default = null -} - - -variable "target_size" { - description = "Target number of instances in the MIG" - type = number - default = 0 -} - -variable "wait_for_instances" { - description = "Whether to wait for all instances to be created/updated before returning" - type = bool - default = false -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/mig/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/mig/versions.tf deleted file mode 100644 index 4147447b44..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/mig/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - google = { - source = "hashicorp/google" - version = "> 5.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:mig/v1.74.0" - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/README.md deleted file mode 100644 index 1dcacc57e9..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# Description - -This module creates the Vertex AI Notebook, to be used in tutorials. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage - -This is a simple usage, using the default network: - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: - name_prefix: my-bucket - local_mount: /home/jupyter/my-bucket - - - id: notebook - source: community/modules/compute/notebook - use: [bucket] - settings: - name_prefix: notebook - machine_type: n1-standard-4 - -``` - -If the user wants do specify a custom subnetwork, or specific external IP restrictions, they can use the `network_interfaces` variable, here is an example on how to use a Shared VPC Subnet with an ephemeral external IP: - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: - name_prefix: my-bucket - local_mount: /home/jupyter/my-bucket - - - id: notebook - source: community/modules/compute/notebook - use: [bucket] - settings: - name_prefix: notebook - machine_type: n1-standard-4 - network_interfaces: - - network: "projects/HOST_PROJECT_ID/global/networks/SHARED_VPC_NAME" - subnet: "projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME" - nic_type: "VIRTIO_NET" -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 5.34 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 5.34 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.mount_script](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_workbench_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/workbench_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the notebook. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name, can be provided from the google-cloud-storage module | `string` | `null` | no | -| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "tf-latest-cpu",
  "name": null,
  "project": "deeplearning-platform-release"
}
| no | -| [labels](#input\_labels) | Labels to add to the resource Key-value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | The machine type to employ | `string` | n/a | yes | -| [mount\_runner](#input\_mount\_runner) | mount content from the google-cloud-storage module | `map(string)` | n/a | yes | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces for the VM instance. Each network interface is represented by an object with the following fields:

- network: (Optional) The name of the Virtual Private Cloud (VPC) network that this VM instance is connected to.

- subnet: (Optional) The name of the subnetwork within the specified VPC that this VM instance is connected to.

- nic\_type: (Optional) The type of vNIC to be used on this interface. Possible values are: `VIRTIO_NET`, `GVNIC`.

- access\_configs: (Optional) An array of access configurations for this network interface. The access\_config object contains:
* external\_ip: (Required) An external IP address associated with this instance. Specify an unused static external IP address available to the project or leave this field undefined to use an IP from a shared ephemeral IP address pool. If you specify a static external IP address, it must live in the same region as the zone of the instance. | 
list(object({
    network  = optional(string)
    subnet   = optional(string)
    nic_type = optional(string)
    access_configs = optional(list(object({
      external_ip = optional(string)
    })))
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | ID of project in which the notebook will be created. | `string` | n/a | yes | -| [service\_account\_email](#input\_service\_account\_email) | If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account | `string` | `null` | no | -| [zone](#input\_zone) | The zone to deploy to | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/main.tf deleted file mode 100644 index cd3ce3b4ea..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/main.tf +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "notebook", ghpc_role = "compute" }) -} - -locals { - suffix = random_id.resource_name_suffix.hex - #name = "thenotebook" - name = "notebook-${var.deployment_name}-${local.suffix}" - bucket = replace(var.gcs_bucket_path, "gs://", "") - post_script_filename = "mount-${local.suffix}.sh" - - # mount_runner_args is defined in the file: cluster-toolkit/modules/file-system/cloud-storage-bucket/outputs.tf - mount_args = split(" ", var.mount_runner.args) - - unused = local.mount_args[0] - remote_mount = local.mount_args[1] - local_mount = local.mount_args[2] - fs_type = local.mount_args[3] - # These options provide a "rw" mount of the GCS bucket - mount_options = "defaults,_netdev,allow_other,implicit_dirs,gid=1000,uid=1000" - - content0 = var.mount_runner.content - content1 = replace(local.content0, "$1", local.unused) - content2 = replace(local.content1, "$2", local.remote_mount) - content3 = replace(local.content2, "$3", local.local_mount) - content4 = replace(local.content3, "$4", local.fs_type) - content5 = replace(local.content4, "$5", local.mount_options) - -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket_object" "mount_script" { - name = local.post_script_filename - content = local.content5 - bucket = local.bucket -} - -resource "google_workbench_instance" "instance" { - name = local.name - location = var.zone - project = var.project_id - labels = local.labels - gce_setup { - machine_type = var.machine_type - metadata = { - post-startup-script = "${var.gcs_bucket_path}/${google_storage_bucket_object.mount_script.name}" - } - vm_image { - project = var.instance_image.project - family = var.instance_image.family - } - - dynamic "service_accounts" { - for_each = var.service_account_email == null ? [] : [1] - content { - email = var.service_account_email - } - } - - dynamic "network_interfaces" { - for_each = var.network_interfaces - content { - network = network_interfaces.value.network - subnet = network_interfaces.value.subnet - nic_type = network_interfaces.value.nic_type - - dynamic "access_configs" { - for_each = network_interfaces.value.access_configs != null ? network_interfaces.value.access_configs : [] - content { - external_ip = access_configs.value.external_ip - } - } - } - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml deleted file mode 100644 index 4a7d5397ca..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - notebooks.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/notebook/variables.tf deleted file mode 100644 index 4359de8c10..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/notebook/variables.tf +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which the notebook will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment; used as part of name of the notebook." - type = string - # notebook name can have: lowercase letters, numbers, or hyphens (-) and cannot end with a hyphen - validation { - error_message = "The notebook name uses 'deployment_name' -- can only have: lowercase letters, numbers, or hyphens" - condition = can(regex("^[a-z0-9]+(?:-[a-z0-9]+)*$", var.deployment_name)) - } -} - -variable "zone" { - description = "The zone to deploy to" - type = string -} - -variable "machine_type" { - description = "The machine type to employ" - type = string -} - -variable "labels" { - description = "Labels to add to the resource Key-value pairs." - type = map(string) -} - -variable "instance_image" { - description = "Instance Image" - type = map(string) - default = { - project = "deeplearning-platform-release" - family = "tf-latest-cpu" - name = null - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "gcs_bucket_path" { - description = "Bucket name, can be provided from the google-cloud-storage module" - type = string - default = null -} - -variable "mount_runner" { - description = "mount content from the google-cloud-storage module" - type = map(string) - - validation { - condition = (length(split(" ", var.mount_runner.args)) == 5) - error_message = "There must be 5 elements in the Mount Runner Arguments: ${var.mount_runner.args} \n " - } -} - -variable "service_account_email" { - description = "If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account" - type = string - default = null -} - -variable "network_interfaces" { - type = list(object({ - network = optional(string) - subnet = optional(string) - nic_type = optional(string) - access_configs = optional(list(object({ - external_ip = optional(string) - }))) - })) - default = [] - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | -| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name    = string
    device_name  = string
    disk_size_gb = number
    disk_type    = string
    disk_labels  = map(string)
    auto_delete  = bool
    boot         = bool
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | -| [feature](#input\_feature) | The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used. | `string` | `null` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | Path to the Slurm bucket. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster. | `string` | n/a | yes | -| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_template\_self\_link](#output\_instance\_template\_self\_link) | The URI of the template. | -| [node\_name\_prefix](#output\_node\_name\_prefix) | The prefix to be used for the node names.

Make sure that nodes are named `-`
This temporary required for proper functioning of the nodes.
While Slurm scheduler uses "features" to bind node and nodeset,
the SlurmGCP relies on node names for this (to be switched to features as well). | -| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf deleted file mode 100644 index 31d9f14ae7..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset-dynamic", ghpc_role = "compute" }) -} - -module "instance_validation" { - source = "../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - nodeset_name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - feature = coalesce(var.feature, local.nodeset_name) - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - universe_domain = { "universe_domain" = var.universe_domain } - - metadata = merge( - local.disable_automatic_updates_metadata, - local.universe_domain, - { slurmd_feature = local.feature }, - var.metadata - ) - - nodeset = { - nodeset_name = local.nodeset_name - nodeset_feature : local.feature - startup_script = local.ghpc_startup_script - network_storage = var.network_storage - } - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - } - ] - - public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - ghpc_startup_script = [{ - filename = "ghpc_nodeset_startup.sh" - content = var.startup_script - }] - -} - -module "slurm_nodeset_template" { - source = "../../internal/slurm-gcp/instance_template" - - project_id = var.project_id - region = var.region - name_prefix = local.nodeset_name - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = "compute" - slurm_bucket_path = var.slurm_bucket_path - metadata = local.metadata - - additional_disks = local.additional_disks - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(local.labels, var.disk_labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - - bandwidth_tier = var.bandwidth_tier - can_ip_forward = var.can_ip_forward - - advanced_machine_features = var.advanced_machine_features - enable_confidential_vm = var.enable_confidential_vm - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - labels = local.labels - machine_type = var.machine_type - - min_cpu_platform = var.min_cpu_platform - on_host_maintenance = var.on_host_maintenance - termination_action = try(var.spot_instance_config.termination_action, null) - preemptible = var.preemptible - spot = var.enable_spot_vm - service_account = local.service_account - gpu = one(local.guest_accelerator) # requires gpu_definition.tf - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - subnetwork = var.subnetwork_self_link - additional_networks = var.additional_networks - access_config = local.access_config - tags = concat([var.slurm_cluster_name], var.tags) -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml deleted file mode 100644 index a99e59d09f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [compute.googleapis.com] -ghpc: - inject_module_id: name diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf deleted file mode 100644 index 2d2d1415cf..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset_dyn" { - description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset -} - -output "instance_template_self_link" { - description = "The URI of the template." - value = module.slurm_nodeset_template.self_link -} - -output "node_name_prefix" { - description = <<-EOD - The prefix to be used for the node names. - - Make sure that nodes are named `-` - This temporary required for proper functioning of the nodes. - While Slurm scheduler uses "features" to bind node and nodeset, - the SlurmGCP relies on node names for this (to be switched to features as well). - EOD - value = "${var.slurm_cluster_name}-${local.nodeset_name}" - -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf deleted file mode 100644 index ec6206e317..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf +++ /dev/null @@ -1,402 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "feature" { - type = string - description = "The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used." - default = null -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "slurm_cluster_name" { - description = "Name of the Slurm cluster." - type = string -} - -variable "slurm_bucket_path" { - description = "Path to the Slurm bucket." - type = string -} - - -variable "machine_type" { - description = "Compute Platform machine type to use for this partition compute nodes." - type = string - default = "c2-standard-60" -} - -variable "metadata" { - type = map(string) - description = "Metadata, provided as a map." - default = {} -} - -variable "instance_image" { - description = <<-EOD - Defines the image that will be used in the Slurm node group VM instances. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - - For more information on creating custom images that comply with Slurm on GCP - see the "Slurm on GCP Custom Images" section in docs/vm-images.md. - EOD - type = map(string) - default = { - family = "slurm-gcp-6-11-hpc-rocky-linux-8" - project = "schedmd-slurm-public" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations - description = <<-EOD - A flag that designates that the user is aware that they are requesting - to use a custom and potentially incompatible image for this Slurm on - GCP module. - - If the field is set to false, only the compatible families and project - names will be accepted. The deployment will fail with any other image - family or name. If set to true, no checks will be done. - - See: https://goo.gle/hpc-slurm-images - EOD - type = bool - default = false -} - - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "tags" { - type = list(string) - description = "Network tag list." - default = [] -} - -variable "disk_type" { - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - type = string - default = "pd-standard" -} - -variable "disk_size_gb" { - description = "Size of boot disk to create for the partition compute nodes." - type = number - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "additional_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = list(object({ - disk_name = string - device_name = string - disk_size_gb = number - disk_type = string - disk_labels = map(string) - auto_delete = bool - boot = bool - })) - default = [] -} - -variable "enable_confidential_vm" { - type = bool - description = "Enable the Confidential VM configuration. Note: the instance image must support option." - default = false -} - -variable "enable_shielded_vm" { - type = bool - description = "Enable the Shielded VM configuration. Note: the instance image must support option." - default = false -} - -variable "shielded_instance_config" { - type = object({ - enable_integrity_monitoring = bool - enable_secure_boot = bool - enable_vtpm = bool - }) - description = <<-EOD - Shielded VM configuration for the instance. Note: not used unless - enable_shielded_vm is 'true'. - - enable_integrity_monitoring : Compare the most recent boot measurements to the - integrity policy baseline and return a pair of pass/fail results depending on - whether they match or not. - - enable_secure_boot : Verify the digital signature of all boot components, and - halt the boot process if signature verification fails. - - enable_vtpm : Use a virtualized trusted platform module, which is a - specialized computer chip you can use to encrypt objects like keys and - certificates. - EOD - default = { - enable_integrity_monitoring = true - enable_secure_boot = true - enable_vtpm = true - } -} - - -variable "enable_oslogin" { - type = bool - description = <<-EOD - Enables Google Cloud os-login for user login and authentication for VMs. - See https://cloud.google.com/compute/docs/oslogin - EOD - default = true -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example." - type = bool - default = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "labels" { - description = "Labels to add to partition compute instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "min_cpu_platform" { - description = "The name of the minimum CPU platform that you want the instance to use." - type = string - default = null -} - -variable "on_host_maintenance" { - type = string - description = <<-EOD - Instance availability Policy. - - Note: Placement groups are not supported when on_host_maintenance is set to - "MIGRATE" and will be deactivated regardless of the value of - enable_placement. To support enable_placement, ensure on_host_maintenance is - set to "TERMINATE". - EOD - default = "TERMINATE" -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." - } -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - - -variable "service_account_email" { - description = "Service account e-mail address to attach to the compute instances." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the compute instances." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "enable_spot_vm" { - description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." - type = bool - default = false -} - -variable "spot_instance_config" { - description = "Configuration for spot VMs." - type = object({ - termination_action = string - }) - default = null -} - -variable "bandwidth_tier" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | -| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | -| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | -| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `null` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling worker nodes allowed in this partition.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of worker nodes to be statically created.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | -| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. | `string` | `""` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the TPU-vm. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the TPU-vm. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | -| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | `"2.14.0"` | no | -| [zone](#input\_zone) | Zone in which to create compute VMs. TPU partitions can only specify a single zone. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf deleted file mode 100644 index ac9b119702..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# locals { -# # This label allows for billing report tracking based on module. -# labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) -# } - -locals { - name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - nodeset_tpu = { - node_count_static = var.node_count_static - node_count_dynamic_max = var.node_count_dynamic_max - nodeset_name = local.name - node_type = var.node_type - - accelerator_config = var.accelerator_config - tf_version = var.tf_version - preemptible = var.preemptible - preserve_tpu = var.preserve_tpu - - data_disks = var.data_disks - docker_image = var.docker_image - - enable_public_ip = var.enable_public_ips - # TODO: rename to subnetwork_self_link, requires changes to the scripts - subnetwork = var.subnetwork_self_link - service_account = local.service_account - zone = var.zone - - project_id = var.project_id - reserved = var.reserved - network_storage = var.network_storage - } - - node_type_core_count = var.node_type == "" ? 0 : tonumber(regex("-(.*)", var.node_type)[0]) - - accelerator_core_list = var.accelerator_config.topology == "" ? [0, 0] : regexall("\\d+", var.accelerator_config.topology) - accelerator_core_count = length(local.accelerator_core_list) > 2 ? (local.accelerator_core_list[0] * local.accelerator_core_list[1] * local.accelerator_core_list[2]) * 2 : (local.accelerator_core_list[0] * local.accelerator_core_list[1]) * 2 - - tpu_core_count = local.accelerator_core_count == 0 ? local.node_type_core_count : local.accelerator_core_count -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml deleted file mode 100644 index 95b6d1c730..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name - has_to_be_used: true diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf deleted file mode 100644 index 8cb7b8663e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset_tpu" { - description = "Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset_tpu - - precondition { - condition = (var.node_type == "") != (var.accelerator_config == { topology : "", version : "" }) - error_message = "Either a node_type or an accelerator_config must be provided." - } - - precondition { - condition = ((local.tpu_core_count / 8) <= var.node_count_dynamic_max) || ((local.tpu_core_count / 8) <= var.node_count_static) - error_message = <<-EOD - When using TPUs there should be at least one node per every 8 cores. - Currently there are ${local.tpu_core_count} cores but only ${var.node_count_static} static nodes and ${var.node_count_dynamic_max} dynamic nodes. - EOD - } - - precondition { - condition = (var.node_count_dynamic_max % (local.tpu_core_count / 8) == 0) && (var.node_count_static % (local.tpu_core_count / 8) == 0) - error_message = <<-EOD - The number of worker nodes should be a multiple of ${local.tpu_core_count / 8}. - This is to ensure each node has a TPU machine for job scheduling. - EOD - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf deleted file mode 100644 index 367b0bee09..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "node_count_static" { - description = <<-EOD - Number of worker nodes to be statically created. - For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). - See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. - EOD - type = number - default = 0 -} - -variable "node_count_dynamic_max" { - description = <<-EOD - Maximum number of auto-scaling worker nodes allowed in this partition. - For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). - See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. - EOD - type = number - default = 0 -} - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "enable_public_ips" { - description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." - type = bool - default = false -} - -variable "disable_public_ips" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `enable_public_ips` instead." - type = bool - default = null - validation { - condition = var.disable_public_ips == null - error_message = "DEPRECATED: Use `enable_public_ips` instead." - } -} - -variable "node_type" { - description = "Specify a node type to base the vm configuration upon it." - type = string - default = "" -} - -variable "accelerator_config" { - description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." - type = object({ - topology = string - version = string - }) - default = { - topology = "" - version = "" - } - validation { - condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], var.accelerator_config.version) - error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" - } - validation { - condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) - error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." - } -} - -variable "tf_version" { - description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." - type = string - default = "2.14.0" -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - -variable "preserve_tpu" { - description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" - type = bool - default = false -} - -variable "zone" { - description = "Zone in which to create compute VMs. TPU partitions can only specify a single zone." - type = string -} - -variable "data_disks" { - description = "The data disks to include in the TPU node" - type = list(string) - default = [] -} - -variable "docker_image" { - description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" - type = string - default = null -} - -variable "subnetwork_self_link" { - type = string - description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." -} - -variable "service_account_email" { - description = "Service account e-mail address to attach to the TPU-vm." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the TPU-vm." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "service_account" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - } -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "reserved" { - description = "Specify whether TPU-vms in this nodeset are created under a reservation." - type = bool - default = false -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured on nodes." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - })) - default = [] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf deleted file mode 100644 index 398eeffdda..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.3" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.74.0" - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md deleted file mode 100644 index 7c9e32debf..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md +++ /dev/null @@ -1,227 +0,0 @@ -## Description - -This module creates a nodeset data structure intended to be input to the -[schedmd-slurm-gcp-v6-partition](../schedmd-slurm-gcp-v6-partition/) module. - -Nodesets allow adding heterogeneous node types to a partition, and hence -running jobs that mix multiple node characteristics. See the [heterogeneous jobs -section][hetjobs] of the SchedMD documentation for more information. - -To specify nodes from a specific nodesets in a partition, the [`--nodelist`] -(or `-w`) flag can be used, for example: - -```bash -srun -N 3 -p compute --nodelist cluster-compute-group-[0-2] hostname -``` - -Where the 3 nodes will be selected from the nodes `cluster-compute-group-[0-2]` -in the compute partition. - -Additionally, depending on how the nodes differ, a constraint can be added via -the [`--constraint`] (or `-C`) flag or other flags such as `--mincpus` can be -used to specify nodes with the desired characteristics. - -[`--nodelist`]: https://slurm.schedmd.com/srun.html#OPT_nodelist -[`--constraint`]: https://slurm.schedmd.com/srun.html#OPT_constraint -[hetjobs]: https://slurm.schedmd.com/heterogeneous_jobs.html - -### Example - -The following code snippet creates a partition module using the `nodeset` -module as input with: - -* a max node count of 200 -* VM machine type of `c2-standard-30` -* partition name of "compute" -* default nodeset name of "ghpc" -* connected to the `network` module via `use` -* nodes mounted to homefs via `use` - -```yaml -- id: nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - node_count_dynamic_max: 200 - machine_type: c2-standard-30 - -- id: compute_partition - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - use: - - homefs - - nodeset - settings: - partition_name: compute -``` - -## Custom Images - -For more information on creating valid custom images for the node group VM -instances or for custom instance templates, see our [vm-images.md] documentation -page. - -[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images - -## GPU Support - -More information on GPU support in Slurm on GCP and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) - -### Compute VM Zone Policies - -The Slurm on GCP nodeset module allows you to specify additional zones in -which to create VMs through [bulk creation][bulk]. This is valuable when -configuring partitions with popular VM families and you desire access to -more compute resources across zones. - -[bulk]: https://cloud.google.com/compute/docs/instances/multiple/about-bulk-creation -[networkpricing]: https://cloud.google.com/vpc/network-pricing - -> **_WARNING:_** Lenient zone policies can lead to additional egress costs when -> moving large amounts of data between zones in the same region. For example, -> traffic between VMs and traffic from VMs to shared filesystems such as -> Filestore. For more information on egress fees, see the -> [Network Pricing][networkpricing] Google Cloud documentation. -> -> To avoid egress charges, ensure your compute nodes are created in a single -> zone by setting var.zone and leaving var.zones to its default value of the -> empty list. -> -> **_NOTE:_** If a new zone is added to the region while the cluster is active, -> nodes in the partition may be created in that zone. In this case, the -> partition may need to be redeployed to ensure the newly added zone is denied. - -In the zonal example below, the nodeset's zone implicitly defaults to the -deployment variable `vars.zone`: - -```yaml -vars: - zone: us-central1-f - -- id: zonal-nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset -``` - -In the example below, we enable creation in additional zones: - -```yaml -vars: - zone: us-central1-f - -- id: multi-zonal-nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - settings: - zones: - - us-central1-a - - us-central1-b -``` - -## Support -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4 | -| [google](#requirement\_google) | >= 5.11 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 5.11 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.machine_type_zone_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [google_compute_machine_types.machine_types_by_zone](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | -| [google_compute_reservation.reservation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_topology](#input\_accelerator\_topology) | Specifies the shape of the Accelerator (GPU/TPU) slice. | `string` | `null` | no | -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = optional(string)
    subnetwork         = string
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    stack_type         = optional(string)
    queue_count        = optional(number)
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | -| [dws\_flex](#input\_dws\_flex) | If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes.
See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler
Options:
- enable: Enable DWS Flex Start
- max\_run\_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week).
- use\_job\_duration: Use the job duration to determine the max\_run\_duration, if job duration is not set, max\_run\_duration will be used.
- use\_bulk\_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances

Limitations:
- CAN NOT be used with reservations;
- CAN NOT be used with placement groups; | 
object({
    enabled          = optional(bool, true)
    max_run_duration = optional(number, 604800) # one week
    use_job_duration = optional(bool, false)
    use_bulk_insert  = optional(bool, false)
  })
| 
{
  "enabled": false
}
| no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_maintenance\_reservation](#input\_enable\_maintenance\_reservation) | Enables slurm reservation for scheduled maintenance. | `bool` | `false` | no | -| [enable\_opportunistic\_maintenance](#input\_enable\_opportunistic\_maintenance) | On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_placement](#input\_enable\_placement) | Use placement policy for VMs in this nodeset.
See: https://cloud.google.com/compute/docs/instances/placement-policies-overview
To set max\_distance of used policy, use `placement_max_distance` variable.

Enabled by default, reasons for users to disable it:
- If non-dense reservation is used, user can avoid extra-cost of creating placement policies;
- If user wants to avoid "all or nothing" VM provisioning behaviour;
- If user wants to intentionally have "spread" VMs (e.g. for reliability reasons) | `bool` | `true` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | -| [future\_reservation](#input\_future\_reservation) | If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT\_ID/zones/ZONE/futureReservations/FUTURE\_RESERVATION\_NAME'.
See https://cloud.google.com/compute/docs/instances/future-reservations-overview | `string` | `""` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_properties](#input\_instance\_properties) | Override the instance properties. Used to test features not supported by Slurm GCP,
recommended for advanced usage only.
See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert
If any sub-field (e.g. scheduling) is set, it will override the values computed by
SlurmGCP and ignoring values of provided vars. | `any` | `null` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for compute nodes. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | -| [maintenance\_interval](#input\_maintenance\_interval) | Sets the maintenance interval for instances in this nodeset.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. | `string` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_conf](#input\_node\_conf) | Map of Slurm node line configuration. | `map(any)` | `{}` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling nodes allowed in this partition. | `number` | `10` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | -| [placement\_max\_distance](#input\_placement\_max\_distance) | Maximum distance between nodes in the placement group. Requires enable\_placement to be true. Values must be supported by the chosen machine type. | `number` | `null` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | -| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]
- RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]

Must be a "SPECIFIC" reservation
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [zone](#input\_zone) | Zone in which to create compute VMs. Additional zones in the same region can be specified in var.zones. | `string` | n/a | yes | -| [zone\_target\_shape](#input\_zone\_target\_shape) | Strategy for distributing VMs across zones in a region.
ANY
GCE picks zones for creating VM instances to fulfill the requested number of VMs
within present resource constraints and to maximize utilization of unused zonal
reservations.
ANY\_SINGLE\_ZONE (default)
GCE always selects a single zone for all the VMs, optimizing for resource quotas,
available reservations and general capacity.
BALANCED
GCE prioritizes acquisition of resources, scheduling VMs in zones where resources
are available while distributing VMs as evenly as possible across allowed zones
to minimize the impact of zonal failure. | `string` | `"ANY_SINGLE_ZONE"` | no | -| [zones](#input\_zones) | Additional zones in which to allow creation of partition nodes. Google Cloud
will find zone based on availability, quota and reservations.
Should not be set if SPECIFIC reservation is used. | `set(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf deleted file mode 100644 index da6aae33ee..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) -} - -module "instance_validation" { - source = "../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata - ) - - name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.disk_resource_manager_tags - } - ] - - public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - ghpc_startup_script = [{ - filename = "ghpc_nodeset_startup.sh" - content = var.startup_script - }] - - termination_action = (var.dws_flex.enabled && !var.dws_flex.use_bulk_insert) ? "DELETE" : try(var.spot_instance_config.termination_action, null) - - nodeset = { - node_count_static = var.node_count_static - node_count_dynamic_max = var.node_count_dynamic_max - node_conf = var.node_conf - nodeset_name = local.name - dws_flex = var.dws_flex - - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(local.labels, var.disk_labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - - bandwidth_tier = var.bandwidth_tier - can_ip_forward = var.can_ip_forward - - enable_confidential_vm = var.enable_confidential_vm - enable_placement = var.enable_placement - placement_max_distance = var.placement_max_distance - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - gpu = one(local.guest_accelerator) - accelerator_topology = var.accelerator_topology - - labels = local.labels - machine_type = terraform_data.machine_type_zone_validation.output - advanced_machine_features = var.advanced_machine_features - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - region = var.region - resource_manager_tags = var.resource_manager_tags - service_account = local.service_account - shielded_instance_config = var.shielded_instance_config - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - subnetwork_self_link = var.subnetwork_self_link - additional_networks = var.additional_networks - access_config = local.access_config - tags = var.tags - spot = var.enable_spot_vm - termination_action = local.termination_action - reservation_name = local.reservation_name - future_reservation = local.future_reservation - maintenance_interval = var.maintenance_interval - instance_properties_json = jsonencode(var.instance_properties) - - zone_target_shape = var.zone_target_shape - zone_policy_allow = local.zones - zone_policy_deny = local.zones_deny - - startup_script = local.ghpc_startup_script - network_storage = var.network_storage - - enable_maintenance_reservation = var.enable_maintenance_reservation - enable_opportunistic_maintenance = var.enable_opportunistic_maintenance - } -} - -locals { - zones = setunion(var.zones, [var.zone]) - zones_deny = setsubtract(data.google_compute_zones.available.names, local.zones) -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region - - lifecycle { - postcondition { - condition = length(setsubtract(local.zones, self.names)) == 0 - error_message = <<-EOD - Invalid zones=${jsonencode(setsubtract(local.zones, self.names))} - Available zones=${jsonencode(self.names)} - EOD - } - } -} - -locals { - res_match = regex("^(?P(?Pprojects/(?P[a-z0-9-]+)/reservations/)?(?P[a-z0-9-]+)(?P/reservationBlocks/[a-z0-9-]+)?)?$", var.reservation_name) - - res_short_name = local.res_match.name - res_project = coalesce(local.res_match.project, var.project_id) - res_prefix = coalesce(local.res_match.prefix, "projects/${local.res_project}/reservations/") - res_suffix = local.res_match.suffix == null ? "" : local.res_match.suffix - - reservation_name = local.res_match.whole == null ? "" : "${local.res_prefix}${local.res_short_name}${local.res_suffix}" -} - -locals { - fr_match = regex("^(?Pprojects/(?P[a-z0-9-]+)/zones/(?P[a-z0-9-]+)/futureReservations/)?(?P[a-z0-9-]+)?$", var.future_reservation) - - fr_name = local.fr_match.name - fr_project = coalesce(local.fr_match.project, var.project_id) - fr_zone = coalesce(local.fr_match.zone, var.zone) - - future_reservation = var.future_reservation == "" ? "" : "projects/${local.fr_project}/zones/${local.fr_zone}/futureReservations/${local.fr_name}" -} - - -# tflint-ignore: terraform_unused_declarations -data "google_compute_reservation" "reservation" { - count = length(local.reservation_name) > 0 ? 1 : 0 - - name = local.res_short_name - project = local.res_project - zone = var.zone - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "Couldn't find the reservation ${var.reservation_name}" - } - - postcondition { - condition = coalesce(self.specific_reservation_required, true) - error_message = < 0] -} - -resource "terraform_data" "machine_type_zone_validation" { - input = var.machine_type - lifecycle { - precondition { - condition = length(local.zones_with_machine_type) > 0 - error_message = <<-EOT - machine type ${var.machine_type} is not available in any of the zones ${jsonencode(local.zones)}". To list zones in which it is available, run: - - gcloud compute machine-types list --filter="name=${var.machine_type}" - EOT - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml deleted file mode 100644 index 95b6d1c730..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name - has_to_be_used: true diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf deleted file mode 100644 index 18ed74e2d5..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset" { - description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset - - precondition { - condition = !contains([ - "c3-:pd-standard", - "h3-:pd-standard", - "h3-:pd-ssd", - ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") - error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." - } - - precondition { - condition = var.reservation_name == "" || length(var.zones) == 0 - error_message = <<-EOD - If a reservation is specified, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.accelerator_topology == null || var.enable_placement - error_message = "accelerator_topology requires enable_placement to be set to true." - } - - precondition { - condition = (var.accelerator_topology == null) || try(tonumber(split("x", var.accelerator_topology)[1]) % local.guest_accelerator[0].count == 0, false) - error_message = "accelerator_topology must be divisible by number of gpus in machine." - } - - precondition { - condition = var.placement_max_distance == null || var.enable_placement - error_message = "placement_max_distance requires enable_placement to be set to true." - } - - precondition { - condition = !(startswith(var.machine_type, "a3-") && var.placement_max_distance == 1) - error_message = "A3 machines do not support a placement_max_distance of 1." - } - - precondition { - condition = var.reservation_name == "" || !var.dws_flex.enabled - error_message = "Cannot use reservations with DWS Flex." - } - - precondition { - condition = !var.enable_placement || !var.dws_flex.enabled - error_message = "Cannot use DWS Flex with `enable_placement`." - } - - precondition { - condition = length(var.zones) == 0 || !var.dws_flex.enabled - error_message = <<-EOD - If a DWS Flex is enabled, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.on_host_maintenance == "TERMINATE" || !var.dws_flex.enabled - error_message = "If DWS Flex is used, `on_host_maintenance` should be set to 'TERMINATE'" - } - - precondition { - condition = !var.enable_spot_vm || !var.dws_flex.enabled - error_message = "Cannot use both Flex-Start and Spot VMs for provisioning." - } - - precondition { - condition = var.reservation_name == "" || var.future_reservation == "" - error_message = "Cannot use reservations and future reservations in the same nodeset" - } - - precondition { - condition = !var.enable_placement || var.future_reservation == "" - error_message = "Cannot use `enable_placement` with future reservations." - } - - precondition { - condition = var.future_reservation == "" || length(var.zones) == 0 - error_message = <<-EOD - If a future reservation is specified, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.future_reservation == "" || local.fr_zone == var.zone - error_message = <<-EOD - The zone of the deployment must match that of the future reservation - EOD - } - - precondition { - condition = var.node_count_dynamic_max > 0 || var.node_count_static > 0 - error_message = <<-EOD - This nodeset contains zero nodes, there should be at least one static or dynamic node - EOD - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf deleted file mode 100644 index 06ef5aac6f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf +++ /dev/null @@ -1,641 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "node_conf" { - description = "Map of Slurm node line configuration." - type = map(any) - default = {} - validation { - condition = lookup(var.node_conf, "Sockets", null) == null - error_message = <<-EOD - `Sockets` field is in conflict with `SocketsPerBoard` which is automatically generated by SlurmGCP. - Instead, you can override the following fields: `Boards`, `SocketsPerBoard`, `CoresPerSocket`, and `ThreadsPerCore`. - See: https://slurm.schedmd.com/slurm.conf.html#OPT_Boards and https://slurm.schedmd.com/slurm.conf.html#OPT_Sockets_1 - EOD - } -} - -variable "node_count_static" { - description = "Number of nodes to be statically created." - type = number - default = 0 -} - -variable "node_count_dynamic_max" { - description = "Maximum number of auto-scaling nodes allowed in this partition." - type = number - default = 10 -} - -## VM Definition -variable "instance_template" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Instance template can not be specified for compute nodes." - type = string - default = null - validation { - condition = var.instance_template == null - error_message = "DEPRECATED: Instance template can not be specified for compute nodes." - } -} - -variable "machine_type" { - description = "Compute Platform machine type to use for this partition compute nodes." - type = string - default = "c2-standard-60" -} - -variable "metadata" { - type = map(string) - description = "Metadata, provided as a map." - default = {} -} - -variable "instance_image" { - description = <<-EOD - Defines the image that will be used in the Slurm node group VM instances. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - - For more information on creating custom images that comply with Slurm on GCP - see the "Slurm on GCP Custom Images" section in docs/vm-images.md. - EOD - type = map(string) - default = { - family = "slurm-gcp-6-11-hpc-rocky-linux-8" - project = "schedmd-slurm-public" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "instance_image_custom" { # tflint-ignore: terraform_unused_declarations - description = <<-EOD - A flag that designates that the user is aware that they are requesting - to use a custom and potentially incompatible image for this Slurm on - GCP module. - - If the field is set to false, only the compatible families and project - names will be accepted. The deployment will fail with any other image - family or name. If set to true, no checks will be done. - - See: https://goo.gle/hpc-slurm-images - EOD - type = bool - default = false -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "tags" { - type = list(string) - description = "Network tag list." - default = [] -} - -variable "disk_type" { - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - type = string - default = "pd-standard" -} - -variable "disk_size_gb" { - description = "Size of boot disk to create for the partition compute nodes." - type = number - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string)) - auto_delete = optional(bool) - boot = optional(bool) - disk_resource_manager_tags = optional(map(string)) - })) - default = [] -} - -variable "enable_confidential_vm" { - type = bool - description = "Enable the Confidential VM configuration. Note: the instance image must support option." - default = false -} - -variable "enable_shielded_vm" { - type = bool - description = "Enable the Shielded VM configuration. Note: the instance image must support option." - default = false -} - -variable "shielded_instance_config" { - type = object({ - enable_integrity_monitoring = bool - enable_secure_boot = bool - enable_vtpm = bool - }) - description = <<-EOD - Shielded VM configuration for the instance. Note: not used unless - enable_shielded_vm is 'true'. - - enable_integrity_monitoring : Compare the most recent boot measurements to the - integrity policy baseline and return a pair of pass/fail results depending on - whether they match or not. - - enable_secure_boot : Verify the digital signature of all boot components, and - halt the boot process if signature verification fails. - - enable_vtpm : Use a virtualized trusted platform module, which is a - specialized computer chip you can use to encrypt objects like keys and - certificates. - EOD - default = { - enable_integrity_monitoring = true - enable_secure_boot = true - enable_vtpm = true - } -} - - -variable "enable_oslogin" { - type = bool - description = <<-EOD - Enables Google Cloud os-login for user login and authentication for VMs. - See https://cloud.google.com/compute/docs/oslogin - EOD - default = true -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example." - type = bool - default = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "labels" { - description = "Labels to add to partition compute instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "min_cpu_platform" { - description = "The name of the minimum CPU platform that you want the instance to use." - type = string - default = null -} - -variable "on_host_maintenance" { - type = string - description = <<-EOD - Instance availability Policy. - - Note: Placement groups are not supported when on_host_maintenance is set to - "MIGRATE" and will be deactivated regardless of the value of - enable_placement. To support enable_placement, ensure on_host_maintenance is - set to "TERMINATE". - EOD - default = "TERMINATE" -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." - } -} - -variable "accelerator_topology" { - type = string - description = "Specifies the shape of the Accelerator (GPU/TPU) slice." - nullable = true - default = null -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - - -variable "service_account_email" { - description = "Service account e-mail address to attach to the compute instances." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the compute instances." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "service_account" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - } -} - -variable "enable_spot_vm" { - description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." - type = bool - default = false -} - -variable "spot_instance_config" { - description = "Configuration for spot VMs." - type = object({ - termination_action = string - }) - default = null -} - -variable "bandwidth_tier" { - description = < 0 - error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME[/reservationBlocks/BLOCK_ID]', [...] are optional parts." - } -} - -variable "future_reservation" { - description = <<-EOD - If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zones/ZONE/futureReservations/FUTURE_RESERVATION_NAME'. - See https://cloud.google.com/compute/docs/instances/future-reservations-overview - EOD - type = string - default = "" - nullable = false - - validation { - condition = length(regexall("^(projects/([a-z0-9-]+)/zones/([a-z0-9-]+)/futureReservations/([a-z0-9-]+))?$", var.future_reservation)) > 0 || length(regexall("^([a-z0-9-]+)$", var.future_reservation)) > 0 - error_message = "Future reservation must be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zone/ZONE/futureReservations/FUTURE_RESERVATION_NAME'." - } -} - -variable "maintenance_interval" { - description = <<-EOD - Sets the maintenance interval for instances in this nodeset. - See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. - EOD - type = string - default = null -} - -variable "startup_script" { - description = "Startup script used by VMs in this nodeset" - type = string - default = "# no-op" -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured on nodes." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - })) - default = [] -} - - -variable "instance_properties" { - description = <<-EOD - Override the instance properties. Used to test features not supported by Slurm GCP, - recommended for advanced usage only. - See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert - If any sub-field (e.g. scheduling) is set, it will override the values computed by - SlurmGCP and ignoring values of provided vars. - EOD - type = any - default = null -} - - -variable "enable_maintenance_reservation" { - type = bool - description = "Enables slurm reservation for scheduled maintenance." - default = false -} - - -variable "enable_opportunistic_maintenance" { - type = bool - description = "On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle." - default = false -} - - -variable "dws_flex" { - description = <<-EOD - If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes. - See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler - Options: - - enable: Enable DWS Flex Start - - max_run_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week). - - use_job_duration: Use the job duration to determine the max_run_duration, if job duration is not set, max_run_duration will be used. - - use_bulk_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances - - Limitations: - - CAN NOT be used with reservations; - - CAN NOT be used with placement groups; - - EOD - - type = object({ - enabled = optional(bool, true) - max_run_duration = optional(number, 604800) # one week - use_job_duration = optional(bool, false) - use_bulk_insert = optional(bool, false) - }) - default = { - enabled = false - } - validation { - condition = var.dws_flex.max_run_duration >= 600 && var.dws_flex.max_run_duration <= 604800 - error_message = "Max duration must be at least than 10 minutes, and cannot be more than one week." - } -} - -variable "placement_max_distance" { - type = number - description = "Maximum distance between nodes in the placement group. Requires enable_placement to be true. Values must be supported by the chosen machine type." - nullable = true - default = null - - validation { - condition = coalesce(var.placement_max_distance, 1) >= 1 && coalesce(var.placement_max_distance, 3) <= 3 - error_message = "Invalid value for placement_max_distance. Valid values are null, 1, 2, or 3." - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf deleted file mode 100644 index e014c318e4..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.4" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 5.11" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.74.0" - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md deleted file mode 100644 index d3dbcd959e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md +++ /dev/null @@ -1,105 +0,0 @@ -## Description - -This module creates a compute partition that can be used as input to the -[schedmd-slurm-gcp-v6-controller](../../scheduler/schedmd-slurm-gcp-v6-controller/README.md). - -The partition module is designed to work alongside the -[schedmd-slurm-gcp-v6-nodeset](../schedmd-slurm-gcp-v6-nodeset/README.md) -module. A partition can be made up of one or -more nodesets, provided either through `use` (preferred) or defined manually -in the `nodeset` variable. - -### Example - -The following code snippet creates a partition module with: - -* 2 nodesets added via `use`. - * The first nodeset is made up of machines of type `c2-standard-30`. - * The second nodeset is made up of machines of type `c2-standard-60`. - * Both nodesets have a maximum count of 200 dynamically created nodes. -* partition name of "compute". -* connected to the `network` module via `use`. -* nodes mounted to homefs via `use`. - -```yaml -- id: nodeset_1 - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - name: c30 - node_count_dynamic_max: 200 - machine_type: c2-standard-30 - -- id: nodeset_2 - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - name: c60 - node_count_dynamic_max: 200 - machine_type: c2-standard-60 - -- id: compute_partition - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - use: - - homefs - - nodeset_1 - - nodeset_2 - settings: - partition_name: compute -``` - -## Support - -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [exclusive](#input\_exclusive) | Exclusive job access to nodes. When set to true nodes execute single job and are deleted
after job exits. If set to false, multiple jobs can be scheduled on one node. | `bool` | `true` | no | -| [is\_default](#input\_is\_default) | Sets this partition as the default partition by updating the partition\_conf.
If "Default" is already set in partition\_conf, this variable will have no effect. | `bool` | `false` | no | -| [network\_storage](#input\_network\_storage) | DEPRECATED | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | A list of nodesets.
For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip     = string
      remote_mount  = string
      local_mount   = string
      fs_type       = string
      mount_options = string
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | -| [partition\_conf](#input\_partition\_conf) | Slurm partition configuration as a map.
See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION | `map(string)` | `{}` | no | -| [partition\_name](#input\_partition\_name) | The name of the slurm partition. | `string` | n/a | yes | -| [resume\_timeout](#input\_resume\_timeout) | Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'ResumeTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. | `number` | `null` | no | -| [suspend\_time](#input\_suspend\_time) | Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram.
This sets 'SuspendTime' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details.
NOTE: use value -1 to exclude partition from suspend.
NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. | `number` | `300` | no | -| [suspend\_timeout](#input\_suspend\_timeout) | Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'SuspendTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. | `number` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Details of a nodesets in this partition | -| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of a dynamic nodesets in this partition | -| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of a TPU nodesets in this partition | -| [partitions](#output\_partitions) | Details of a slurm partition | - diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf deleted file mode 100644 index 1618c64280..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - use_static = [for ns in concat(var.nodeset, var.nodeset_tpu) : ns.nodeset_name if ns.node_count_static > 0] - - has_node = length(var.nodeset) > 0 - has_dyn = length(var.nodeset_dyn) > 0 - has_tpu = length(var.nodeset_tpu) > 0 - has_flex = length([for ns in var.nodeset : ns.dws_flex.enabled if ns.dws_flex.enabled]) > 0 -} - -locals { - partition_conf = merge({ - "Default" = var.is_default ? "YES" : null - "SuspendTime" = var.suspend_time < 0 ? "INFINITE" : var.suspend_time - "SuspendTimeout" = var.suspend_timeout != null ? var.suspend_timeout : (local.has_tpu ? 240 : 120) - }, var.partition_conf, { "ResumeTimeout" = local.has_flex ? 65535 : try(var.partition_conf["ResumeTimeout"], coalesce(var.resume_timeout, (local.has_tpu ? 600 : 300))) }) - - partition = { - partition_name = var.partition_name - partition_conf = local.partition_conf - - partition_nodeset = [for ns in var.nodeset : ns.nodeset_name] - partition_nodeset_tpu = [for ns in var.nodeset_tpu : ns.nodeset_name] - partition_nodeset_dyn = [for ns in var.nodeset_dyn : ns.nodeset_name] - # Options - enable_job_exclusive = var.exclusive - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml deleted file mode 100644 index 13ea127b3c..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - has_to_be_used: true diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf deleted file mode 100644 index 35dece64fb..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "partitions" { - description = "Details of a slurm partition" - - value = [local.partition] - - precondition { - condition = (length(local.use_static) == 0) || !var.exclusive - error_message = <<-EOD - Can't use static nodes within partition with `var.exclusive` set to `true`. - NOTE: Partition's `var.exclusive` is set to `true` by default. Set it to `false` explicitly to use static nodes. - EOD - } - - precondition { - # Can not mix TPU with other non-TPU nodesets due to SlurmGCP specific limitations; - # Can not mix dynamic with non-dynamic nodesets due to Slurms inability to - # turn off "power management" at nodeset level (can only do it at partition or node level). - condition = sum([for b in [local.has_node, local.has_dyn, local.has_tpu] : b ? 1 : 0]) == 1 - error_message = "Partition must contain exactly one type of nodeset." - } -} - -output "nodeset" { - description = "Details of a nodesets in this partition" - - value = var.nodeset -} - -output "nodeset_tpu" { - description = "Details of a TPU nodesets in this partition" - - value = var.nodeset_tpu -} - - -output "nodeset_dyn" { - description = "Details of a dynamic nodesets in this partition" - - value = var.nodeset_dyn -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf deleted file mode 100644 index a1c85adb90..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "partition_name" { - description = "The name of the slurm partition." - type = string - - validation { - condition = can(regex("^[a-z](?:[a-z0-9]*)$", var.partition_name)) - error_message = "Variable 'partition_name' must be a match of regex '^[a-z](?:[a-z0-9]*)$'." - } -} - -variable "partition_conf" { - description = <<-EOD - Slurm partition configuration as a map. - See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION - EOD - type = map(string) - default = {} -} - -variable "is_default" { - description = <<-EOD - Sets this partition as the default partition by updating the partition_conf. - If "Default" is already set in partition_conf, this variable will have no effect. - EOD - type = bool - default = false -} - -variable "exclusive" { - description = <<-EOD - Exclusive job access to nodes. When set to true nodes execute single job and are deleted - after job exits. If set to false, multiple jobs can be scheduled on one node. - EOD - type = bool - default = true -} - -variable "nodeset" { - description = <<-EOD - A list of nodesets. - For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset - EOD - type = list(object({ - node_count_static = optional(number, 0) - node_count_dynamic_max = optional(number, 1) - node_conf = optional(map(string), {}) - nodeset_name = string - additional_disks = optional(list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string), {}) - auto_delete = optional(bool, true) - boot = optional(bool, false) - disk_resource_manager_tags = optional(map(string), {}) - })), []) - bandwidth_tier = optional(string, "platform_default") - can_ip_forward = optional(bool, false) - disk_auto_delete = optional(bool, true) - disk_labels = optional(map(string), {}) - disk_resource_manager_tags = optional(map(string), {}) - disk_size_gb = optional(number) - disk_type = optional(string) - enable_confidential_vm = optional(bool, false) - enable_placement = optional(bool, false) - placement_max_distance = optional(number, null) - enable_oslogin = optional(bool, true) - enable_shielded_vm = optional(bool, false) - enable_maintenance_reservation = optional(bool, false) - enable_opportunistic_maintenance = optional(bool, false) - gpu = optional(object({ - count = number - type = string - })) - accelerator_topology = optional(string, null) - dws_flex = object({ - enabled = bool - max_run_duration = number - use_job_duration = bool - use_bulk_insert = bool - }) - labels = optional(map(string), {}) - machine_type = optional(string) - advanced_machine_features = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - maintenance_interval = optional(string) - instance_properties_json = string - metadata = optional(map(string), {}) - min_cpu_platform = optional(string) - network_tier = optional(string, "STANDARD") - network_storage = optional(list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = optional(map(string)) - mount_runner = optional(map(string)) - })), []) - on_host_maintenance = optional(string) - preemptible = optional(bool, false) - region = optional(string) - resource_manager_tags = optional(map(string), {}) - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - shielded_instance_config = optional(object({ - enable_integrity_monitoring = optional(bool, true) - enable_secure_boot = optional(bool, true) - enable_vtpm = optional(bool, true) - })) - source_image_family = optional(string) - source_image_project = optional(string) - source_image = optional(string) - subnetwork_self_link = string - additional_networks = optional(list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - }))) - access_config = optional(list(object({ - nat_ip = string - network_tier = string - }))) - spot = optional(bool, false) - tags = optional(list(string), []) - termination_action = optional(string) - reservation_name = optional(string) - future_reservation = string - startup_script = optional(list(object({ - filename = string - content = string })), []) - - zone_target_shape = string - zone_policy_allow = set(string) - zone_policy_deny = set(string) - })) - default = [] - - validation { - condition = length(distinct(var.nodeset[*].nodeset_name)) == length(var.nodeset) - error_message = "All nodesets must have a unique name." - } -} - -variable "nodeset_tpu" { - description = "Define TPU nodesets, as a list." - type = list(object({ - node_count_static = optional(number, 0) - node_count_dynamic_max = optional(number, 5) - nodeset_name = string - enable_public_ip = optional(bool, false) - node_type = string - accelerator_config = optional(object({ - topology = string - version = string - }), { - topology = "" - version = "" - }) - tf_version = string - preemptible = optional(bool, false) - preserve_tpu = optional(bool, false) - zone = string - data_disks = optional(list(string), []) - docker_image = optional(string, "") - network_storage = optional(list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - })), []) - subnetwork = string - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - project_id = string - reserved = optional(string, false) - })) - default = [] - - validation { - condition = length(distinct([for x in var.nodeset_tpu : x.nodeset_name])) == length(var.nodeset_tpu) - error_message = "All TPU nodesets must have a unique name." - } -} - -variable "nodeset_dyn" { - description = "Defines dynamic nodesets, as a list." - type = list(object({ - nodeset_name = string - nodeset_feature = string - })) - default = [] - - validation { - condition = length(distinct([for x in var.nodeset_dyn : x.nodeset_name])) == length(var.nodeset_dyn) - error_message = "All dynamic nodesets must have a unique name." - } -} - -variable "resume_timeout" { - description = <<-EOD - Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use. - If null is given, then a smart default will be chosen depending on nodesets in partition. - This sets 'ResumeTimeout' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. - EOD - type = number - default = null - - validation { - condition = var.resume_timeout == null ? true : var.resume_timeout > 0 && var.resume_timeout < 65536 - error_message = "Value must be > 0 and < 65536" - } -} - -variable "suspend_time" { - description = <<-EOD - Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram. - This sets 'SuspendTime' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details. - NOTE: use value -1 to exclude partition from suspend. - NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. - EOD - type = number - default = 300 - - validation { - condition = var.suspend_time >= -1 - error_message = "Value must be >= -1." - } -} - -variable "suspend_timeout" { - description = <<-EOD - Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown. - If null is given, then a smart default will be chosen depending on nodesets in partition. - This sets 'SuspendTimeout' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. - EOD - type = number - default = null - - validation { - condition = var.suspend_timeout == null ? true : var.suspend_timeout > 0 - error_message = "Value must be > 0." - } -} - - -# tflint-ignore: terraform_unused_declarations -variable "network_storage" { - description = "DEPRECATED" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] - validation { - condition = length(var.network_storage) == 0 - error_message = <<-EOD - network_storage in partition module is deprecated and should not be set. - To add network storage to compute nodes, use network_storage of nodeset module instead. - EOD - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf b/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf deleted file mode 100644 index d388f4bfdd..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.3" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.74.0" - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/README.md b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/README.md deleted file mode 100644 index 994f1500ba..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/README.md +++ /dev/null @@ -1,157 +0,0 @@ -## Description - -This module provides ways to create and manage Google Cloud Artifact Registry repositories. - -Currently this module is built to support repositories in Docker format although there are placeholder variables for other types which may work too. Remote repositories with pull-through cache functionality integrated with Google Secret Manager is currently supported. The aim of this module is to eventually offer feature parity with this [Terraform module](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository#nested_remote_repository_config), allowing creation of repositories in various formats, including Docker, Maven, NPM, Python, APT, YUM, and COMMON. - -This module is best suited for managing artifact repositories in HPC/AI containerized environments where artifacts need to be shared across distributed systems. It includes IAM role configurations and secret access handling for seamless integration with CI/CD pipelines and other services too. - -It is designed to help facilitate containerized workloads running in the Cluster Toolkit with SLURM leveraging [Enroot](https://github.com/NVIDIA/enroot) and [Pyxis](https://github.com/NVIDIA/pyxis). Docker repositories can store container images that are used in job submissions, enabling efficient and scalable execution of containerized HPC or AI based workloads. - -## Usage - -### Service Account / APIs - -You will need to enable the relevant APIs and create a Service Account for your cluster with the following Artifact Registry permissions. - -```yaml - - id: services-api - source: community/modules/project/service-enablement - settings: - gcp_service_list: - - secretmanager.googleapis.com - - cloudbuild.googleapis.com - - artifactregistry.googleapis.com - - - source: community/modules/project/service-account - kind: terraform - id: hpc_service_account - settings: - project_id: project_name - name: service_account_name - project_roles: - - artifactregistry.reader - - artifactregistry.writer - - secretmanager.secretAccessor -``` - -### Deployment - -Create a standard Docker repository. - -```yaml -- id: registry - source: community/modules/container/artifact-registry - settings: - repo_mode: STANDARD_REPOSITORY - format: DOCKER -``` - -Mirror of public Docker Hub repository. - -```yaml -- id: dockerhub_registry - source: community/modules/container/artifact-registry - settings: - repo_mode: REMOTE_REPOSITORY - format: DOCKER - repo_public_repository: DOCKER_HUB -``` - -Mirror of NVIDIA's [NGC Catalog](https://catalog.ngc.nvidia.com/containers). [API key](https://org.ngc.nvidia.com/setup/api-key) used in blueprint is stored in Secret Manager. - -```yaml -- id: ngc_registry - source: community/modules/container/artifact-registry - settings: - repo_mode: REMOTE_REPOSITORY - format: DOCKER - repo_mirror_url: "https://nvcr.io" - repo_username: $oauthtoken - repo_password: api_key_here - use_upstream_credentials: True -``` - -### Container Operations - -Retrieve `$REPOSITORY_NAME` from [Artifact Registry](https://console.cloud.google.com/artifacts) or by using `gcloud`. - -```yaml -gcloud artifacts repositories list --project="${PROJECT_ID}" -``` - -Pulling containers from your mirrored internal Artifact Repositories. - -Pull [Ubuntu](https://hub.docker.com/_/ubuntu) from Docker Hub mirror. - -```yaml -docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/library/ubuntu:latest -``` - -Pull [Pytorch](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch) from NGC Catalog mirror. - -```yaml -docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/nvidia/pytorch:24.11-py3 -``` - -Alternatively, proceed with running SLURM's [NVIDIA/pyxis](https://github.com/NVIDIA/pyxis) plugin, which will now be able to pull and use these containers directly from the mirrored repositories. - -Note: only Docker registries have been tested so far. Placeholders do exist for other registry types which may or may not work. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_artifact_registry_repository.artifact_registry](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository) | resource | -| [google_secret_manager_secret.repo_password_secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_version.repo_password_secret_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [random_password.repo_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | -| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment. | `string` | n/a | yes | -| [format](#input\_format) | Artifact Registry format (e.g., DOCKER). | `string` | `"DOCKER"` | no | -| [labels](#input\_labels) | Labels to add to the artifact registry. Key-value pairs. | `map(string)` | `{}` | no | -| [project\_id](#input\_project\_id) | Project ID where the artifact registry and secret are created. | `string` | n/a | yes | -| [region](#input\_region) | Region for the artifact registry. | `string` | n/a | yes | -| [repo\_mirror\_url](#input\_repo\_mirror\_url) | For REMOTE\_REPOSITORY, URL for a custom or common mirror. | `string` | `null` | no | -| [repo\_mode](#input\_repo\_mode) | Artifact Registry mode (STANDARD\_REPOSITORY, REMOTE\_REPOSITORY, etc.). | `string` | `"STANDARD_REPOSITORY"` | no | -| [repo\_password](#input\_repo\_password) | Optional password/API key. If null, one will be randomly generated. | `string` | `null` | no | -| [repo\_public\_repository](#input\_repo\_public\_repository) | For REMOTE\_REPOSITORY, name of a known public repo as per the Terraform module
(e.g., DOCKER\_HUB) or null for custom repo. | `string` | `null` | no | -| [repo\_username](#input\_repo\_username) | Username for external repository. | `string` | `null` | no | -| [repository\_base](#input\_repository\_base) | For APT/YUM public repos, repository\_base (e.g., 'DEBIAN', 'UBUNTU'). | `string` | `null` | no | -| [repository\_path](#input\_repository\_path) | For APT/YUM public repos, repository\_path (e.g., 'debian/dists/buster'). | `string` | `null` | no | -| [use\_upstream\_credentials](#input\_use\_upstream\_credentials) | Configure Service Account to use upstream credentials for REMOTE\_REPOSITORY:
If true, a username/password is used for the REMOTE\_REPOSITORY mirror.
If false (or if repo\_password == null), no password is created at all.
Note: Blueprint credentials will be stored in Secrets Manager. | `bool` | `false` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | (Optional) A list of objects to enable user-managed replication.
Each object can have:
location = string
kms\_key\_name = optional(string)
If empty, auto replication is used. | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [registry\_url](#output\_registry\_url) | The URL of the created artifact registry. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/main.tf b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/main.tf deleted file mode 100644 index c3406af607..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/main.tf +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "artifact-registry", ghpc_role = "container" }) -} - -locals { - # Auto (i.e., empty) vs user-managed replication - auto = length(var.user_managed_replication) == 0 ? true : false - - # For remote custom repositories, parse out host to create a base_component name - mirror_url_no_proto = var.repo_mirror_url != null ? replace(replace(var.repo_mirror_url, "https://", ""), "http://", "") : "" - mirror_host = local.mirror_url_no_proto != "" ? split("/", local.mirror_url_no_proto)[0] : "" - - base_component = replace( - replace( - replace( - lower( - local.mirror_host != "" - ? "${var.format}-${var.repo_mode}-${local.mirror_host}" - : "${var.format}-${var.repo_mode}-nohost" - ), - "\\.", "-" - ), - "/", "-" - ), - "_", "-" - ) - - repository_suffix = random_id.resource_name_suffix.hex - - # The final name for the artifact registry repository - repository_name = replace( - replace( - lower( - format("%s-%s", local.base_component, local.repository_suffix) - ), - ".", "-" - ), - "/", "-" - ) - - # The secret name is derived from the repository name - # with a suffix like "-secret". - derived_secret_name = format("%s-secret", local.repository_name) -} - -############################## -# PASSWORD / SECRET -############################## - -# Only create a random password if user didn't supply one -resource "random_password" "repo_password" { - count = var.use_upstream_credentials && var.repo_password == null ? 1 : 0 - length = 24 - special = true - override_special = "_-#=." -} - -resource "google_secret_manager_secret" "repo_password_secret" { - count = var.use_upstream_credentials ? 1 : 0 - project = var.project_id - - # Derive the secret ID from the repository name - secret_id = local.derived_secret_name - - labels = local.labels - - replication { - dynamic "auto" { - for_each = local.auto ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = local.auto ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = replicas.value.kms_key_name != null ? [1] : [] - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_version" "repo_password_secret_version" { - count = var.use_upstream_credentials ? 1 : 0 - secret = google_secret_manager_secret.repo_password_secret[0].id - - # If user provided a password, use it. Otherwise use the random password. - secret_data = var.repo_password != null ? var.repo_password : random_password.repo_password[0].result -} - -############################## -# IAM BINDINGS -############################## - -############################## -# ARTIFACT REGISTRY -############################## - -resource "random_id" "resource_name_suffix" { - byte_length = 2 -} - -resource "google_artifact_registry_repository" "artifact_registry" { - project = var.project_id - location = var.region - format = var.format - mode = var.repo_mode - description = var.deployment_name - labels = local.labels - repository_id = local.repository_name - - # Only create remote_repository_config if REMOTE_REPOSITORY - dynamic "remote_repository_config" { - for_each = var.repo_mode == "REMOTE_REPOSITORY" ? [1] : [] - content { - description = "Pull-through cache" - - dynamic "docker_repository" { - for_each = var.format == "DOCKER" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "docker_repository" { - for_each = var.format == "DOCKER" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "maven_repository" { - for_each = var.format == "MAVEN" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "maven_repository" { - for_each = var.format == "MAVEN" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "npm_repository" { - for_each = var.format == "NPM" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "npm_repository" { - for_each = var.format == "NPM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "python_repository" { - for_each = var.format == "PYTHON" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "python_repository" { - for_each = var.format == "PYTHON" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "apt_repository" { - for_each = var.format == "APT" && var.repo_public_repository != null ? [1] : [] - content { - public_repository { - repository_base = var.repository_base - repository_path = var.repository_path - } - } - } - - dynamic "apt_repository" { - for_each = var.format == "APT" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "yum_repository" { - for_each = var.format == "YUM" && var.repo_public_repository != null ? [1] : [] - content { - public_repository { - repository_base = var.repository_base - repository_path = var.repository_path - } - } - } - - dynamic "yum_repository" { - for_each = var.format == "YUM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "common_repository" { - for_each = var.format == "COMMON" ? [1] : [] - content { - uri = var.repo_mirror_url - } - } - - # Only enable upstream credentials if user wants it - dynamic "upstream_credentials" { - for_each = var.use_upstream_credentials ? [1] : [] - content { - username_password_credentials { - username = var.repo_username - password_secret_version = google_secret_manager_secret_version.repo_password_secret_version[0].name - } - } - } - } - } - - depends_on = [ - google_secret_manager_secret.repo_password_secret, - google_secret_manager_secret_version.repo_password_secret_version, - ] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml deleted file mode 100644 index 6b68c98a54..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - secretmanager.googleapis.com - - artifactregistry.googleapis.com - - cloudbuild.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf deleted file mode 100644 index 92b6dbb165..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "registry_url" { - description = "The URL of the created artifact registry." - value = "${var.region}-docker.pkg.dev/${var.project_id}/${var.deployment_name}" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf deleted file mode 100644 index a795060fb7..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "terraform_data" "input_validation" { - lifecycle { - precondition { - condition = ( - var.repo_password == null || - (var.use_upstream_credentials && var.repo_mode == "REMOTE_REPOSITORY") - ) - error_message = "repo_password may be set only when repo_mode=REMOTE_REPOSITORY and use_upstream_credentials=true." - } - - precondition { - condition = ( - !var.use_upstream_credentials || - var.repo_mode == "REMOTE_REPOSITORY" - ) - error_message = "use_upstream_credentials is allowed only when repo_mode is REMOTE_REPOSITORY." - } - - precondition { - condition = ( - var.repo_mode != "REMOTE_REPOSITORY" || - (var.repo_public_repository != null || var.repo_mirror_url != null) - ) - error_message = "For a REMOTE_REPOSITORY you must set repo_public_repository or repo_mirror_url." - } - - precondition { - condition = ( - !contains(["APT", "YUM"], var.format) || - (var.repository_base != null && var.repository_path != null) - ) - error_message = "APT/YUM formats require repository_base and repository_path." - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf deleted file mode 100644 index 9a4eecb921..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "Project ID where the artifact registry and secret are created." - type = string -} - -variable "region" { - description = "Region for the artifact registry." - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment." - type = string -} - -variable "labels" { - description = "Labels to add to the artifact registry. Key-value pairs." - type = map(string) - default = {} -} - -variable "repo_password" { - description = "Optional password/API key. If null, one will be randomly generated." - type = string - default = null -} - -variable "user_managed_replication" { - description = <<-DOC - (Optional) A list of objects to enable user-managed replication. - Each object can have: - location = string - kms_key_name = optional(string) - If empty, auto replication is used. - DOC - type = list(object({ - location = string - kms_key_name = optional(string) - })) - default = [] -} - -variable "format" { - description = "Artifact Registry format (e.g., DOCKER)." - type = string - default = "DOCKER" -} - -variable "repo_mode" { - description = "Artifact Registry mode (STANDARD_REPOSITORY, REMOTE_REPOSITORY, etc.)." - type = string - default = "STANDARD_REPOSITORY" - - validation { - condition = can(regex("^(STANDARD_REPOSITORY|REMOTE_REPOSITORY|VIRTUAL_REPOSITORY)$", var.repo_mode)) - error_message = "repo_mode must be one of STANDARD_REPOSITORY, REMOTE_REPOSITORY, or VIRTUAL_REPOSITORY." - } -} - -variable "repo_public_repository" { - description = <<-DOC - For REMOTE_REPOSITORY, name of a known public repo as per the Terraform module - (e.g., DOCKER_HUB) or null for custom repo. - DOC - type = string - default = null - - # To Do: implement validation - # validation { - # condition = ((var.repo_mode != "REMOTE_REPOSITORY" && var.repo_public_repository == null) || (var.repo_mode == "REMOTE_REPOSITORY" && (var.repo_public_repository != null || var.repo_mirror_url != null))) - # error_message = "If repo_mode is REMOTE_REPOSITORY, you must set either repo_public_repository or repo_mirror_url. Otherwise, leave them null." - # } -} - -variable "repo_mirror_url" { - description = "For REMOTE_REPOSITORY, URL for a custom or common mirror." - type = string - default = null -} - -variable "use_upstream_credentials" { - description = <<-DOC - Configure Service Account to use upstream credentials for REMOTE_REPOSITORY: - If true, a username/password is used for the REMOTE_REPOSITORY mirror. - If false (or if repo_password == null), no password is created at all. - Note: Blueprint credentials will be stored in Secrets Manager. - DOC - type = bool - default = false -} - -variable "repo_username" { - description = "Username for external repository." - type = string - default = null -} - -variable "repository_base" { - description = "For APT/YUM public repos, repository_base (e.g., 'DEBIAN', 'UBUNTU')." - type = string - default = null -} - -variable "repository_path" { - description = "For APT/YUM public repos, repository_path (e.g., 'debian/dists/buster')." - type = string - default = null -} diff --git a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf b/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf deleted file mode 100644 index 392a7131d2..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md deleted file mode 100644 index 23bf87398a..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md +++ /dev/null @@ -1,76 +0,0 @@ -## Description - -Creates a BigQuery dataset. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage -This is a simple usage. - -```yaml - - id: bq-dataset - source: community/modules/database/bigquery-dataset - settings: - dataset_id: my_dataset -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_dataset.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | The name of the dataset to be created | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the dataset. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [dataset\_id](#output\_dataset\_id) | Name of the dataset that was created. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf deleted file mode 100644 index 1a9c4bba60..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-dataset", ghpc_role = "database" }) -} -locals { - dataset_id = var.dataset_id != null ? var.dataset_id : replace("${var.deployment_name}_dataset_${random_id.resource_name_suffix.hex}", "-", "_") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_bigquery_dataset" "pbsb" { - dataset_id = local.dataset_id - project = var.project_id - labels = local.labels -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml deleted file mode 100644 index 87ff9357e4..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigquery.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf deleted file mode 100644 index 9cd8e5df31..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "dataset_id" { - description = "Name of the dataset that was created." - value = google_bigquery_dataset.pbsb.dataset_id -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf deleted file mode 100644 index 90c229af6b..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "dataset_id" { - description = "The name of the dataset to be created" - type = string - default = null -} - -variable "labels" { - description = "Labels to add to the dataset. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf deleted file mode 100644 index 12ddbe842d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/README.md b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/README.md deleted file mode 100644 index ef67cfef01..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/README.md +++ /dev/null @@ -1,87 +0,0 @@ -## Description - -Creates a BigQuery table with a specified schema. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage - -```yaml -id: bq-table - source: community/modules/database/bigquery-table - use: [bq-dataset] - settings: - table_schema: - ' - [ - { - "name": "id", "type": "STRING" - } - ] - ' -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_table.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Dataset name to be used to create the new BQ Table | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the tables. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [table\_id](#input\_table\_id) | Table name to be used to create the new BQ Table | `string` | `null` | no | -| [table\_schema](#input\_table\_schema) | Schema used to create the new BQ Table | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [dataset\_id](#output\_dataset\_id) | ID of BQ dataset | -| [table\_id](#output\_table\_id) | ID of created BQ table | -| [table\_name](#output\_table\_name) | Name of created BQ table | - diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/main.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/main.tf deleted file mode 100644 index 73f3923e00..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-table", ghpc_role = "database" }) -} - -locals { - table_id = var.table_id != null ? var.table_id : replace("${var.deployment_name}_table_${random_id.resource_name_suffix.hex}", "-", "_") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_bigquery_table" "pbsb" { - deletion_protection = false - project = var.project_id - table_id = local.table_id - dataset_id = var.dataset_id - schema = var.table_schema - labels = local.labels -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml deleted file mode 100644 index 87ff9357e4..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigquery.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf deleted file mode 100644 index 4220ec1390..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "table_name" { - description = "Name of created BQ table" - value = google_bigquery_table.pbsb.friendly_name -} -output "table_id" { - description = "ID of created BQ table" - value = google_bigquery_table.pbsb.table_id -} -output "dataset_id" { - description = "ID of BQ dataset" - value = google_bigquery_table.pbsb.dataset_id -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf deleted file mode 100644 index ec474b4e64..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "labels" { - description = "Labels to add to the tables. Key-value pairs." - type = map(string) -} - -variable "table_id" { - description = "Table name to be used to create the new BQ Table" - type = string - default = null -} - -variable "dataset_id" { - description = "Dataset name to be used to create the new BQ Table" - type = string -} - -variable "table_schema" { - description = "Schema used to create the new BQ Table" - type = string -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf b/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf deleted file mode 100644 index 12ddbe842d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md deleted file mode 100644 index 08364c175b..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md +++ /dev/null @@ -1,107 +0,0 @@ -## Description - -terraform-google-sql makes it easy to create a Google CloudSQL instance and -implement high availability settings. This module is meant for use with -Terraform 0.13+ and tested using Terraform 1.0+. - -The cloudsql created here is used to integrate with the slurm cluster to enable -accounting data storage. - -### Example - -```yaml -- id: cloudsql - source: community/modules/database/slurm-cloudsql-federation - use: [network] - settings: - sql_instance_name: slurm-sql6-demo - tier: "db-f1-micro" -``` - -This creates a cloud sql instance, including a database, user that would allow -the slurm cluster to use as an external DB. In addition, it will allow BigQuery -to run federated query through it. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | -| [google](#requirement\_google) | >= 3.83 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_connection.connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_connection) | resource | -| [google_compute_address.psc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_forwarding_rule.psc_consumer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_forwarding_rule) | resource | -| [google_sql_database.database](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database) | resource | -| [google_sql_database_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance) | resource | -| [google_sql_user.users](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [random_password.password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [authorized\_networks](#input\_authorized\_networks) | IP address ranges as authorized networks of the Cloud SQL for MySQL instances | `list(string)` | `[]` | no | -| [data\_cache\_enabled](#input\_data\_cache\_enabled) | Whether data cache is enabled for the instance. Can be used with ENTERPRISE\_PLUS edition. | `bool` | `false` | no | -| [database\_flags](#input\_database\_flags) | Database flags to set on instance. | `map(string)` | `{}` | no | -| [database\_version](#input\_database\_version) | The version of the database to be created. | `string` | `"MYSQL_8_0"` | no | -| [deletion\_protection](#input\_deletion\_protection) | Whether or not to allow Terraform to destroy the instance. | `string` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [disk\_autoresize](#input\_disk\_autoresize) | Set to false to disable automatic disk grow. | `bool` | `true` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of the database disk in GiB. | `number` | `null` | no | -| [edition](#input\_edition) | value | `string` | `"ENTERPRISE"` | no | -| [enable\_backups](#input\_enable\_backups) | Set true to enable backups | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is going to be created in.:
`projects//global/networks/`" | `string` | n/a | yes | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [query\_insights](#input\_query\_insights) | Query insights configuration. | 
object({
    enabled                 = optional(bool, false)
    query_plans_per_minute  = optional(number)
    query_string_length     = optional(number)
    record_application_tags = optional(bool)
    record_client_address   = optional(bool)
  })
| `{}` | no | -| [region](#input\_region) | The region where SQL instance will be configured | `string` | n/a | yes | -| [sql\_instance\_name](#input\_sql\_instance\_name) | name given to the sql instance for ease of identificaion | `string` | n/a | yes | -| [sql\_password](#input\_sql\_password) | Password for the SQL database. | `any` | `null` | no | -| [sql\_username](#input\_sql\_username) | Username for the SQL database | `string` | `"slurm"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self link of the network where Cloud SQL instance PSC endpoint will be created | `string` | `null` | no | -| [tier](#input\_tier) | The machine type to use for the SQL instance | `string` | n/a | yes | -| [use\_psc\_connection](#input\_use\_psc\_connection) | Create Private Service Connection instead of using Private Service Access peering | `bool` | `false` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cloudsql](#output\_cloudsql) | Describes the cloudsql instance. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf deleted file mode 100644 index 9b518a1b5f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "slurm-cloudsql-federation", ghpc_role = "database" }) -} - -locals { - user_managed_replication = var.user_managed_replication -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "random_password" "password" { - length = 12 - special = false -} - -locals { - sql_instance_name = var.sql_instance_name == null ? "${var.deployment_name}-sql-${random_id.resource_name_suffix.hex}" : var.sql_instance_name - sql_password = var.sql_password == null ? random_password.password.result : var.sql_password -} - - -resource "google_sql_database_instance" "instance" { - project = var.project_id - depends_on = [var.private_vpc_connection_peering] - name = local.sql_instance_name - region = var.region - deletion_protection = var.deletion_protection - database_version = var.database_version - - settings { - disk_size = var.disk_size_gb - disk_autoresize = var.disk_autoresize - edition = var.edition - tier = var.tier - user_labels = local.labels - - dynamic "data_cache_config" { - for_each = var.edition == "ENTERPRISE_PLUS" ? [""] : [] - content { - data_cache_enabled = var.data_cache_enabled - } - } - - dynamic "database_flags" { - for_each = var.database_flags - content { - name = database_flags.key - value = database_flags.value - } - } - - insights_config { - query_insights_enabled = var.query_insights.enabled - query_plans_per_minute = var.query_insights.query_plans_per_minute - query_string_length = var.query_insights.query_string_length - record_application_tags = var.query_insights.record_application_tags - record_client_address = var.query_insights.record_client_address - } - - ip_configuration { - ipv4_enabled = false - private_network = var.use_psc_connection ? null : var.network_id - enable_private_path_for_google_cloud_services = true - - dynamic "authorized_networks" { - for_each = var.use_psc_connection ? [] : var.authorized_networks - iterator = ip_range - - content { - value = ip_range.value - } - } - dynamic "psc_config" { - for_each = var.use_psc_connection ? [""] : [] - content { - psc_enabled = true - allowed_consumer_projects = [var.project_id] - } - } - } - - backup_configuration { - enabled = var.enable_backups - # to allow easy switching between ENTERPRISE and ENTERPRISE_PLUS - transaction_log_retention_days = 7 - } - } - lifecycle { - precondition { - condition = var.disk_autoresize && var.disk_size_gb == null || !var.disk_autoresize - error_message = "If setting disk_size_gb set disk_autorize to false to prevent re-provisioning of the instance after disk auto-expansion." - } - } -} - - - -resource "google_compute_address" "psc" { - count = var.use_psc_connection ? 1 : 0 - project = var.project_id - name = local.sql_instance_name - address_type = "INTERNAL" - region = var.region - subnetwork = var.subnetwork_self_link - labels = local.labels -} - -resource "google_compute_forwarding_rule" "psc_consumer" { - count = var.use_psc_connection ? 1 : 0 - name = local.sql_instance_name - project = var.project_id - region = var.region - subnetwork = var.subnetwork_self_link - ip_address = google_compute_address.psc[0].self_link - load_balancing_scheme = "" - recreate_closed_psc = true - target = google_sql_database_instance.instance.psc_service_attachment_link -} - -resource "google_sql_database" "database" { - project = var.project_id - name = "slurm_accounting" - instance = google_sql_database_instance.instance.name -} - -resource "google_sql_user" "users" { - project = var.project_id - name = var.sql_username - instance = google_sql_database_instance.instance.name - password = local.sql_password -} - -resource "google_bigquery_connection" "connection" { - provider = google - project = var.project_id - location = var.region - cloud_sql { - instance_id = google_sql_database_instance.instance.connection_name - database = google_sql_database.database.name - type = "MYSQL" - credential { - username = google_sql_user.users.name - password = google_sql_user.users.password - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml deleted file mode 100644 index fc0cae0859..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigqueryconnection.googleapis.com - - sqladmin.googleapis.com - - servicenetworking.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf deleted file mode 100644 index 0d05221cd8..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "cloudsql" { - description = "Describes the cloudsql instance." - sensitive = true - value = { - server_ip = var.use_psc_connection ? google_compute_address.psc[0].address : google_sql_database_instance.instance.ip_address[0].ip_address - user = google_sql_user.users.name - password = google_sql_user.users.password - db_name = google_sql_database.database.name - user_managed_replication = local.user_managed_replication - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf deleted file mode 100644 index a2f150419e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "authorized_networks" { - description = "IP address ranges as authorized networks of the Cloud SQL for MySQL instances" - type = list(string) - default = [] - nullable = false -} - -variable "database_version" { - description = "The version of the database to be created." - type = string - default = "MYSQL_8_0" - validation { - condition = contains(["MYSQL_5_7", "MYSQL_8_0", "MYSQL_8_4"], var.database_version) - error_message = "The database version must be either MYSQL_5_7, MYSQL_8_0 or MYSQL_8_4." - } -} - -variable "data_cache_enabled" { - description = "Whether data cache is enabled for the instance. Can be used with ENTERPRISE_PLUS edition." - type = bool - default = false -} - -variable "database_flags" { - description = "Database flags to set on instance." - type = map(string) - default = {} - nullable = false -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "disk_autoresize" { - description = "Set to false to disable automatic disk grow." - type = bool - default = true -} - -variable "disk_size_gb" { - description = "Size of the database disk in GiB." - type = number - default = null -} - -variable "edition" { - description = "value" - type = string - validation { - condition = contains(["ENTERPRISE", "ENTERPRISE_PLUS"], var.edition) - error_message = "The database edition must be either ENTERPRISE or ENTERPRISE_PLUS" - } - default = "ENTERPRISE" -} - -variable "enable_backups" { - description = "Set true to enable backups" - type = bool - default = false -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "query_insights" { - description = "Query insights configuration." - nullable = false - default = {} - type = object({ - enabled = optional(bool, false) - query_plans_per_minute = optional(number) - query_string_length = optional(number) - record_application_tags = optional(bool) - record_client_address = optional(bool) - }) -} - -variable "region" { - description = "The region where SQL instance will be configured" - type = string -} - -variable "tier" { - description = "The machine type to use for the SQL instance" - type = string -} - -variable "sql_instance_name" { - description = "name given to the sql instance for ease of identificaion" - type = string -} - -variable "deletion_protection" { - description = "Whether or not to allow Terraform to destroy the instance." - type = string - default = false -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} - -variable "sql_username" { - description = "Username for the SQL database" - type = string - default = "slurm" -} - -variable "sql_password" { - description = "Password for the SQL database." - type = any - default = null -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is going to be created in.: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "private_vpc_connection_peering" { - description = "The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation." - type = string - default = null -} - -variable "subnetwork_self_link" { - description = "Self link of the network where Cloud SQL instance PSC endpoint will be created" - type = string - default = null -} - -variable "user_managed_replication" { - type = list(object({ - location = string - kms_key_name = optional(string) - })) - description = "Replication parameters that will be used for defined secrets" - default = [] -} - -variable "use_psc_connection" { - description = "Create Private Service Connection instead of using Private Service Access peering" - type = bool - default = false -} diff --git a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf b/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf deleted file mode 100644 index 7e672858b6..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.74.0" - } - - required_version = ">= 0.13.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md deleted file mode 100644 index d39a58afe1..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md +++ /dev/null @@ -1,158 +0,0 @@ -> [!WARNING] -> This module is deprecated and will be removed on July 1, 2025. The -> recommended replacement is the -> [GCP Managed Lustre module](../../../../modules/file-system/managed-lustre/README.md) - -## Description -This module creates a DDN EXAScaler Cloud Lustre file system using code based on DDN's -[exascaler-cloud-terraform](https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp) (`scripts/2.2.2` is last release with GCP-specific module). - -More information about the architecture can be found at -[Overview of Lustre and EXAScaler Cloud][architecture]. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -> **Warning**: This file system has a license cost as described in the pricing -> section of the [DDN EXAScaler Cloud Marketplace Solution][marketplace]. -> -> **Note**: By default security.public_key is set to `null`, therefore the -> admin user is not created. To ensure the admin user is created, provide a -> public key via the security setting. -> -> **Note**: This module's instances require access to Google APIs and -> therefore, instances must have public IP address or it must be used in a -> subnetwork where [Private Google Access][private-google-access] is enabled. - -[private-google-access]: https://cloud.google.com/vpc/docs/configure-private-google-access -[marketplace]: https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud -[architecture]: https://cloud.google.com/architecture/parallel-file-systems-for-hpc#overview_of_lustre_and_exascaler_cloud - -## Mounting - -To mount the DDN EXAScaler Lustre file system you must first install the DDN -Lustre client and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -the [hpc-enterprise-slurm.yaml](../../../../examples/hpc-enterprise-slurm.yaml) for an -example of using this module with Slurm. - -If mounting is not automatically handled as described above, the DDN-EXAScaler -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - # This file system has an associated license cost. - # https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud - - id: lustrefs - source: community/modules/file-system/DDN-EXAScaler - use: [network1] - settings: {local_mount: /scratch} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(lustrefs.install_ddn_lustre_client_runner) - - $(lustrefs.mount_runner) - -``` - -See [additional documentation][ddn-install-docs] from DDN EXAScaler. - -[ddn-install-docs]: https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp#install-new-exascaler-cloud-clients -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## Support - -EXAScaler Cloud includes self-help support with access to publicly available -documents and videos. Premium support includes 24x7x365 access to DDN's experts, -along with support community access, automated notifications of updates and -other premium support features. For more information, visit -[EXAscaler Cloud on GCP][exa-gcp]. - -[exa-gcp]: https://console.cloud.google.com/marketplace/product/ddnstorage/exascaler-cloud - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [ddn\_exascaler](#module\_ddn\_exascaler) | github.com/DDNStorage/exascaler-cloud-terraform//gcp | a3355d50deebe45c0556b45bd599059b7c06988d | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [boot](#input\_boot) | Boot disk properties | 
object({
    disk_type   = string
    auto_delete = bool
    script_url  = string
  })
| 
{
  "auto_delete": true,
  "disk_type": "pd-standard",
  "script_url": null
}
| no | -| [cls](#input\_cls) | Compute client properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 0,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-2",
  "public_ip": true
}
| no | -| [clt](#input\_clt) | Compute client target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 0,
  "disk_size": 256,
  "disk_type": "pd-standard"
}
| no | -| [fsname](#input\_fsname) | EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long | `string` | `"exacloud"` | no | -| [image](#input\_image) | DEPRECATED: Source image properties | `any` | `null` | no | -| [instance\_image](#input\_instance\_image) | Source image properties

Expected Fields:
name: Unavailable with this module.
family: The image family to use.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp",
  "project": "ddn-public"
}
| no | -| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. Key-value pairs. | `map(string)` | `{}` | no | -| [local\_mount](#input\_local\_mount) | Mountpoint (at the client instances) for this EXAScaler system | `string` | `"/shared"` | no | -| [mds](#input\_mds) | Metadata server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | -| [mdt](#input\_mdt) | Metadata target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | -| [mgs](#input\_mgs) | Management server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | -| [mgt](#input\_mgt) | Management target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | -| [mnt](#input\_mnt) | Monitoring target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | -| [network\_properties](#input\_network\_properties) | Network options. 'network\_self\_link' or 'network\_properties' must be provided. | 
object({
    routing = string
    tier    = string
    id      = string
    auto    = bool
    mtu     = number
    new     = bool
    nat     = bool
  })
| `null` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self-link of the VPC network to where the system is connected. Ignored if 'network\_properties' is provided. 'network\_self\_link' or 'network\_properties' must be provided. | `string` | `null` | no | -| [oss](#input\_oss) | Object Storage server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 3,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-16",
  "public_ip": true
}
| no | -| [ost](#input\_ost) | Object Storage target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | -| [prefix](#input\_prefix) | EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud') | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Compute Platform project that will host the EXAScaler filesystem | `string` | n/a | yes | -| [security](#input\_security) | Security options | 
object({
    admin              = string
    public_key         = string
    block_project_keys = bool
    enable_os_login    = bool
    enable_local       = bool
    enable_ssh         = bool
    enable_http        = bool
    ssh_source_ranges  = list(string)
    http_source_ranges = list(string)
  })
| 
{
  "admin": "stack",
  "block_project_keys": false,
  "enable_http": false,
  "enable_local": false,
  "enable_os_login": true,
  "enable_ssh": false,
  "http_source_ranges": [
    "0.0.0.0/0"
  ],
  "public_key": null,
  "ssh_source_ranges": [
    "0.0.0.0/0"
  ]
}
| no | -| [service\_account](#input\_service\_account) | Service account name used by deploy application | 
object({
    new   = bool
    email = string
  })
| 
{
  "email": null,
  "new": false
}
| no | -| [subnetwork\_address](#input\_subnetwork\_address) | The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork\_properties' is provided. | `string` | `null` | no | -| [subnetwork\_properties](#input\_subnetwork\_properties) | Subnetwork properties. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | 
object({
    address = string
    private = bool
    id      = string
    new     = bool
  })
| `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork\_properties' is provided. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | `string` | `null` | no | -| [waiter](#input\_waiter) | Waiter to check progress and result for deployment. | `string` | `null` | no | -| [zone](#input\_zone) | Compute Platform zone where the servers will be located | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_config\_script](#output\_client\_config\_script) | Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler. | -| [http\_console](#output\_http\_console) | HTTP address to access the system web console. | -| [install\_ddn\_lustre\_client\_runner](#output\_install\_ddn\_lustre\_client\_runner) | Runner that encapsulates the `client_config_script` output on this module. | -| [mount\_command](#output\_mount\_command) | Command to mount the file system. `client_config_script` must be run first. | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the DDN EXAScaler Lustre file system | -| [network\_storage](#output\_network\_storage) | Describes a EXAScaler system to be mounted by other systems. | -| [private\_addresses](#output\_private\_addresses) | Private IP addresses for all instances. | -| [ssh\_console](#output\_ssh\_console) | Instructions to ssh into the instances. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf deleted file mode 100644 index 6a2fc4b702..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ddn-exascaler", ghpc_role = "file-system" }) -} - -locals { - - network_id = var.network_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.network_self_link)[0] : null - named_net = { - routing = "REGIONAL" - tier = "STANDARD" - id = local.network_id - auto = false - mtu = 1500 - new = false - nat = false - } - - subnetwork_id = var.subnetwork_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.subnetwork_self_link)[0] : null - named_subnet = { - address = var.subnetwork_address - private = true - id = local.subnetwork_id - new = false - } -} - -module "ddn_exascaler" { - source = "github.com/DDNStorage/exascaler-cloud-terraform//gcp?ref=a3355d50deebe45c0556b45bd599059b7c06988d" - fsname = var.fsname - zone = var.zone - project = var.project_id - prefix = var.prefix - labels = local.labels - security = var.security - service_account = var.service_account - waiter = var.waiter - network = var.network_properties == null ? local.named_net : var.network_properties - subnetwork = var.subnetwork_properties == null ? local.named_subnet : var.subnetwork_properties - boot = var.boot - image = var.instance_image - mgs = var.mgs - mgt = var.mgt - mnt = var.mnt - mds = var.mds - mdt = var.mdt - oss = var.oss - ost = var.ost - cls = var.cls - clt = var.clt -} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml deleted file mode 100644 index b995bd4358..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - deploymentmanager.googleapis.com - - iam.googleapis.com - - runtimeconfig.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf deleted file mode 100644 index 2e9ae732ae..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -output "private_addresses" { - description = "Private IP addresses for all instances." - value = module.ddn_exascaler.private_addresses -} - -output "ssh_console" { - description = "Instructions to ssh into the instances." - value = module.ddn_exascaler.ssh_console -} - -output "client_config_script" { - description = "Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler." - value = module.ddn_exascaler.client_config -} - -output "install_ddn_lustre_client_runner" { - description = "Runner that encapsulates the `client_config_script` output on this module." - value = local.client_install_runner -} - -locals { - client_install_runner = { - "type" = "shell" - "content" = module.ddn_exascaler.client_config - "destination" = "install_ddn_lustre_client.sh" - } - - # Mount command provided by DDN does not support custom local mount - split_mount_cmd = split(" ", module.ddn_exascaler.mount_command) - split_mount_cmd_wo_mountpoint = slice(local.split_mount_cmd, 0, length(local.split_mount_cmd) - 1) - mount_cmd = "${join(" ", local.split_mount_cmd_wo_mountpoint)} ${var.local_mount}" - mount_cmd_w_mkdir = "mkdir -p ${var.local_mount} && ${local.mount_cmd}" - mount_runner = { - "type" = "shell" - "content" = local.mount_cmd_w_mkdir - "destination" = "mount-ddn-lustre.sh" - } -} - -output "mount_command" { - description = "Command to mount the file system. `client_config_script` must be run first." - value = local.mount_cmd_w_mkdir -} - -output "mount_runner" { - description = "Runner to mount the DDN EXAScaler Lustre file system" - value = local.mount_runner -} - -output "http_console" { - description = "HTTP address to access the system web console." - value = module.ddn_exascaler.http_console -} - -output "network_storage" { - description = "Describes a EXAScaler system to be mounted by other systems." - value = { - server_ip = split(":", split(" ", module.ddn_exascaler.mount_command)[3])[0] - remote_mount = length(regexall("^/.*", var.fsname)) > 0 ? var.fsname : format("/%s", var.fsname) - local_mount = var.local_mount != null ? var.local_mount : format("/mnt/%s", var.fsname) - fs_type = "lustre" - mount_options = "" - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } - depends_on = [ - module.ddn_exascaler - ] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf deleted file mode 100644 index 68bcc8a8ba..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf +++ /dev/null @@ -1,502 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -# EXAScaler filesystem name -# only alphanumeric characters are allowed, -# and the value must be 1-8 characters long -variable "fsname" { - description = "EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long" - type = string - default = "exacloud" -} - -# Project ID to manage resources -# https://cloud.google.com/resource-manager/docs/creating-managing-projects -variable "project_id" { - description = "Compute Platform project that will host the EXAScaler filesystem" - type = string -} - -# Zone name to manage resources -# https://cloud.google.com/compute/docs/regions-zones -variable "zone" { - description = "Compute Platform zone where the servers will be located" - type = string -} - -# Service account name used by deploy application -# https://cloud.google.com/iam/docs/service-accounts -# new: create a new custom service account or use an existing one: true or false -# email: existing service account email address, will be using if new is false -# set email = null to use the default compute service account -variable "service_account" { - description = "Service account name used by deploy application" - type = object({ - new = bool - email = string - }) - default = { - new = false - email = null - } -} - -# Waiter to check progress and result for deployment. -# To use Google Deployment Manager: -# waiter = "deploymentmanager" -# To use generic Google Cloud SDK command line: -# waiter = "sdk" -# If you don’t want to wait until the deployment is complete: -# waiter = null -# https://cloud.google.com/deployment-manager/runtime-configurator/creating-a-waiter -variable "waiter" { - description = "Waiter to check progress and result for deployment." - type = string - default = null -} - -# Security options -# admin: optional user name for remote SSH access -# Set admin = null to disable creation admin user -# public_key: path to the SSH public key on the local host -# Set public_key = null to disable creation admin user -# block_project_keys: true or false -# Block project-wide public SSH keys if you want to restrict -# deployment to only user with deployment-level public SSH key. -# https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys -# enable_os_login: true or false -# Enable or disable OS Login feature. -# Please note, enabling this option disables other security options: -# admin, public_key and block_project_keys. -# https://cloud.google.com/compute/docs/instances/managing-instance-access#enable_oslogin -# enable_local: true or false, enable or disable firewall rules for local access -# enable_ssh: true or false, enable or disable remote SSH access -# ssh_source_ranges: source IP ranges for remote SSH access in CIDR notation -# enable_http: true or false, enable or disable remote HTTP access -# http_source_ranges: source IP ranges for remote HTTP access in CIDR notation -variable "security" { - description = "Security options" - type = object({ - admin = string - public_key = string - block_project_keys = bool - enable_os_login = bool - enable_local = bool - enable_ssh = bool - enable_http = bool - ssh_source_ranges = list(string) - http_source_ranges = list(string) - }) - - default = { - admin = "stack" - public_key = null - block_project_keys = false - enable_os_login = true - enable_local = false - enable_ssh = false - enable_http = false - ssh_source_ranges = [ - "0.0.0.0/0" - ] - http_source_ranges = [ - "0.0.0.0/0" - ] - } -} - -variable "network_self_link" { - description = "The self-link of the VPC network to where the system is connected. Ignored if 'network_properties' is provided. 'network_self_link' or 'network_properties' must be provided." - type = string - default = null -} - -# Network properties -# https://cloud.google.com/vpc/docs/vpc -# routing: network-wide routing mode: REGIONAL or GLOBAL -# tier: networking tier for VM interfaces: STANDARD or PREMIUM -# id: existing network id, will be using if new is false -# auto: create subnets in each region automatically: false or true -# mtu: maximum transmission unit in bytes: 1460 - 1500 -# new: create a new network or use an existing one: true or false -# nat: allow instances without external IP to communicate with the outside world: true or false -variable "network_properties" { - description = "Network options. 'network_self_link' or 'network_properties' must be provided." - type = object({ - routing = string - tier = string - id = string - auto = bool - mtu = number - new = bool - nat = bool - }) - - default = null -} - -variable "subnetwork_self_link" { - description = "The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork_properties' is provided. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." - type = string - default = null -} - -variable "subnetwork_address" { - description = "The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork_properties' is provided." - type = string - default = null -} - -# Subnetwork properties -# https://cloud.google.com/vpc/docs/vpc -# address: IP range of internal addresses for a new subnetwork -# private: when enabled VMs in this subnetwork without external -# IP addresses can access Google APIs and services by using -# Private Google Access: true or false -# https://cloud.google.com/vpc/docs/private-access-options -# id: existing subnetwork id, will be using if new is false -# new: create a new subnetwork or use an existing one: true or false -variable "subnetwork_properties" { - description = "Subnetwork properties. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." - type = object({ - address = string - private = bool - id = string - new = bool - }) - default = null -} -# Boot disk properties -# disk_type: pd-standard, pd-ssd or pd-balanced -# auto_delete: true or false -# whether the disk will be auto-deleted when the instance is deleted -variable "boot" { - description = "Boot disk properties" - type = object({ - disk_type = string - auto_delete = bool - script_url = string - }) - default = { - disk_type = "pd-standard" - auto_delete = true - script_url = null - } -} - -# Source image properties -# project: project name -# family: image family name -# name: !!DEPRECATED!! - image name -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: Source image properties" - type = any - # Omitting type checking so validation can provide more useful error message - # type = object({ - # project = string - # family = string - # }) - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - Source image properties - - Expected Fields: - name: Unavailable with this module. - family: The image family to use. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "ddn-public" - family = "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp" - } - - validation { - condition = !can(coalesce(var.instance_image.name)) - error_message = "In var.instance_image, the \"name\" field is not used, please use the \"family\" setting." - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, the \"family\" field must be a string set to the image family." - } -} - -# Management server properties -# node_type: type of management server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of management servers -variable "mgs" { - description = "Management server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-32" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 1 - } -} - -# Management target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of management target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of management target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of management target in GB (scratch disk size must be exactly 375) -# disk_count: number of management targets -# disk_raid: create striped management target, true or false -variable "mgt" { - description = "Management target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 128 - disk_count = 1 - disk_raid = false - } -} - - -# Monitoring target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of monitoring target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of monitoring target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of monitoring target in GB (scratch disk size must be exactly 375) -# disk_count: number of monitoring targets -# disk_raid: create striped monitoring target, true or false -variable "mnt" { - description = "Monitoring target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 128 - disk_count = 1 - disk_raid = false - } -} - -# Metadata server properties -# node_type: type of metadata server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of metadata servers -variable "mds" { - description = "Metadata server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-32" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 1 - } -} - -# Metadata target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of metadata target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of metadata target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of metadata target in GB (scratch disk size must be exactly 375) -# disk_count: number of metadata targets -# disk_raid: create striped metadata target, true or false -variable "mdt" { - description = "Metadata target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-ssd" - disk_size = 3500 - disk_count = 1 - disk_raid = false - } -} - -# Object Storage server properties -# node_type: type of storage server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of storage servers -variable "oss" { - description = "Object Storage server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-16" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 3 - } -} - -# Object Storage target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of storage target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of storage target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of storage target in GB (scratch disk size must be exactly 375) -# disk_count: number of storage targets -# disk_raid: create striped storage target, true or false -variable "ost" { - description = "Object Storage target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-ssd" - disk_size = 3500 - disk_count = 1 - disk_raid = false - } -} - -# Compute client properties -# node_type: type of compute client -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of compute clients -variable "cls" { - description = "Compute client properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-2" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 0 - } -} -# Compute client target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of compute target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of compute target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of compute target in GB (scratch disk size must be exactly 375) -# disk_count: number of compute targets -variable "clt" { - description = "Compute client target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 256 - disk_count = 0 - } -} -variable "local_mount" { - description = "Mountpoint (at the client instances) for this EXAScaler system" - type = string - default = "/shared" -} - -variable "prefix" { - description = "EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud')" - type = string - default = null -} - -variable "labels" { - description = "Labels to add to EXAScaler Cloud deployment. Key-value pairs." - type = map(string) - default = {} -} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf deleted file mode 100644 index 2981b4dd75..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md b/deletion-test/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md deleted file mode 100644 index 04db0acb8c..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md +++ /dev/null @@ -1 +0,0 @@ -> **_NOTE:_** Cluster Toolkit is dropping support for the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository. The DAOS example blueprints (`hpc-slurm-daos.yaml` and `pfs-daos.yaml`) have been removed from the Cluster Toolkit. We recommend migrating to the first-party [Parallelstore](../../../../modules/file-system/parallelstore/) module for similar functionality. To help with this transition, see the Parallelstore example blueprints ([pfs-parallelstore.yaml](../../../../examples/pfs-parallelstore.yaml) and [ps-slurm.yaml](../../../../examples/ps-slurm.yaml)). If the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository is necessary, we recommend using the last Cluster Toolkit [v1.41.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/releases/tag/v1.41.0). diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/README.md b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/README.md deleted file mode 100644 index 66aaaa46af..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/README.md +++ /dev/null @@ -1,152 +0,0 @@ -## Description - -This module creates a Network File Sharing (NFS) file system based on a VM -instance and [compute disk][disk]. This file system can share directories and -files with other clients over a network. `nfs-server` can be used by -[vm-instance](../../../../modules/compute/vm-instance/README.md) and SchedMD -community modules that create compute VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -If you are using Hyperdisk storage, check the possible disk size, IOPS, and throughput values for each disk type in the [Hyperdisk limits documentation](https://cloud.google.com/compute/docs/disks/hyperdisks#limits-disk). - -> **_WARNING:_** This module has only been tested against the HPC centos7 OS -> disk image (the default). Using other images may work, but have not been -> verified. - -[disk]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk - -### Example - -```yaml -- id: homefs - source: community/modules/file-system/nfs-server - use: [network1] -``` - -This creates a NFS on a virtual machine which allow other VMs to mount the -volume as an external file system. - -> **_NOTE:_** All disks are destroyed along with the instance, during a `gcluster destroy`/`terraform destroy` event. However, you can setup data retention with `create_boot_snapshot_before_destroy` (boot disk) and `create_snapshot_before_destroy` (data disk). - -## Mounting - -To mount the NFS Server you must first ensure that the NFS client has been -installed the and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -See the [hpc-centos-ss.yaml] test config for an example of using this module -with a `vm-instance` module. - -If mounting is not automatically handled as described above, the `nfs-server` -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - - id: nfs - source: community/modules/file-system/nfs-server - use: [network1] - settings: {local_mounts: [/mnt1]} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(nfs.install_nfs_client_runner) - - $(nfs.mount_runner) - -``` - -[hpc-centos-ss.yaml]: ../../../../tools/validate_configs/test_configs/hpc-centos-ss.yaml -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 6.14 | -| [null](#requirement\_null) | >= 3.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.14 | -| [null](#provider\_null) | >= 3.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_disk.attached_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_instance.compute_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource | -| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_default_service_account.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_default_service_account) | data source | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [auto\_delete\_disk](#input\_auto\_delete\_disk) | DEPRECATED: Whether or not the NFS disk should be auto-deleted | `string` | `null` | no | -| [boot\_disk\_size](#input\_boot\_disk\_size) | Storage size in GB for the boot disk | `number` | `null` | no | -| [boot\_disk\_type](#input\_boot\_disk\_type) | Storage type for the boot disk | `string` | `null` | no | -| [create\_boot\_snapshot\_before\_destroy](#input\_create\_boot\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the boot disk | `bool` | `false` | no | -| [create\_snapshot\_before\_destroy](#input\_create\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the NFS data disk | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the NFS instance if no name is specified. | `string` | n/a | yes | -| [disk\_size](#input\_disk\_size) | Storage size in GB for the NFS data disk | `number` | `"100"` | no | -| [image](#input\_image) | DEPRECATED: The VM image used by the NFS server | `string` | `null` | no | -| [instance\_image](#input\_instance\_image) | The VM image used by the NFS server.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to the NFS instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mounts](#input\_local\_mounts) | Mountpoint for this NFS compute instance | `list(string)` | 
[
  "/data"
]
| no | -| [machine\_type](#input\_machine\_type) | Type of the VM instance to use | `string` | `"n2d-standard-2"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the NFS VM. | `string` | `"default"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [provisioned\_iops](#input\_provisioned\_iops) | Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput | `number` | `null` | no | -| [provisioned\_throughput](#input\_provisioned\_throughput) | Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme | `number` | `null` | no | -| [scopes](#input\_scopes) | Scopes to apply to the controller | `list(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [service\_account](#input\_service\_account) | Service Account for the NFS server | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the NFS VM. | `string` | `null` | no | -| [type](#input\_type) | Storage type for the NFS data disk | `string` | `"pd-ssd"` | no | -| [zone](#input\_zone) | The zone name where the NFS instance located in. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | -| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | -| [network\_storage](#output\_network\_storage) | export of all desired folder directories | - diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf deleted file mode 100644 index a00d2681ba..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "nfs-server", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - server_ip = google_compute_instance.compute_instance.network_interface[0].network_ip - fs_type = "nfs" - mount_options = "defaults,hard,intr" - install_nfs_client_runners = [for mount in var.local_mounts : - { - "type" = "shell" - "source" = "${path.module}/scripts/install-nfs-client.sh" - "destination" = "install-nfs${replace(mount, "/", "_")}.sh" - } - ] - mount_runners = [for mount in var.local_mounts : - { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"/exports${mount}\" \"${mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(mount, "/", "_")}.sh" - } - ] - ansible_mount_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/mount.yaml" - "destination" = "mount.yaml" - } -} - -data "google_compute_default_service_account" "default" {} - -resource "google_compute_disk" "attached_disk" { - project = var.project_id - name = "${local.name}-nfs-instance-disk" - size = var.disk_size - type = var.type - zone = var.zone - labels = local.labels - provisioned_iops = var.provisioned_iops - provisioned_throughput = var.provisioned_throughput - create_snapshot_before_destroy = var.create_snapshot_before_destroy -} - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project -} - -resource "null_resource" "image" { - triggers = { - name = try(var.instance_image.name, null), - family = try(var.instance_image.family, null), - project = var.instance_image.project - } -} - -resource "google_compute_disk" "boot_disk" { - project = var.project_id - - name = "${local.name}-boot-disk" - size = var.boot_disk_size - type = var.boot_disk_type - image = data.google_compute_image.compute_image.self_link - labels = local.labels - zone = var.zone - create_snapshot_before_destroy = var.create_boot_snapshot_before_destroy - - lifecycle { - replace_triggered_by = [null_resource.image] - ignore_changes = [ - image - ] - } -} - -resource "google_compute_instance" "compute_instance" { - project = var.project_id - name = "${local.name}-nfs-instance" - zone = var.zone - machine_type = var.machine_type - - boot_disk { - auto_delete = false - source = google_compute_disk.boot_disk.self_link - device_name = google_compute_disk.boot_disk.name - } - - attached_disk { - source = google_compute_disk.attached_disk.id - device_name = "attached_disk" - } - - network_interface { - network = var.network_self_link - subnetwork = var.subnetwork_self_link - } - - service_account { - email = var.service_account == null ? data.google_compute_default_service_account.default.email : var.service_account - scopes = var.scopes - } - - metadata = var.metadata - metadata_startup_script = templatefile("${path.module}/scripts/install-nfs-server.sh.tpl", { local_mounts = var.local_mounts }) - - labels = local.labels -} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf deleted file mode 100644 index e23b94e2b2..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -# render the content for each folder -output "network_storage" { - description = "export of all desired folder directories" - value = [for i, mount in var.local_mounts : { - remote_mount = "/exports${mount}" - local_mount = mount - fs_type = local.fs_type - mount_options = local.mount_options - server_ip = local.server_ip - client_install_runner = local.install_nfs_client_runners[i] - mount_runner = local.mount_runners[i] - } - ] -} - -output "install_nfs_client" { - description = "Script for installing NFS client" - value = file("${path.module}/scripts/install-nfs-client.sh") -} - -output "install_nfs_client_runner" { - description = "Runner to install NFS client using the startup-script module" - value = local.install_nfs_client_runners[0] -} - -output "mount_runner" { - description = <<-EOT - Runner to mount the file-system using an ansible playbook. The startup-script - module will automatically handle installation of ansible. - - id: example-startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(your-fs-id.mount_runner) - ... - EOT - value = local.ansible_mount_runner -} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl deleted file mode 100644 index 1b06a5f032..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -ex - -if [ ! -d "/exports" ]; then # first load, format and mount the disk - # See https://cloud.google.com/compute/docs/disks/add-persistent-disk - uuid=$(uuidgen) - mkfs.ext4 -F -m 0 -U "$uuid" -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/disk/by-id/google-attached_disk - - mkdir /exports - echo "UUID=$uuid /exports ext4 discard,defaults 0 0" >> /etc/fstab - mount --target /exports/ - - %{ for mount in local_mounts ~} - mkdir -p /exports${mount} - chmod 755 /exports${mount} - echo '/exports${mount} *(rw,sync,no_root_squash)' >> "/etc/exports" - %{ endfor ~} -fi - -systemctl start nfs-server rpcbind -systemctl enable nfs-server -exportfs -r diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml deleted file mode 100644 index f7fbe58d5e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Mounts the file systems specified in the metadata network_storage key - hosts: localhost - become: true - vars: - meta_key: "network_storage" - url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" - tasks: - - name: Read metadata network_storage information - ansible.builtin.uri: - url: "{{ url }}/{{ meta_key }}" - method: GET - headers: - Metadata-Flavor: "Google" - register: storage - - name: Mount file systems - ansible.posix.mount: - src: "{{ item.server_ip }}:/{{ item.remote_mount }}" - path: "{{ item.local_mount }}" - opts: "{{ item.mount_options }}" - boot: true - fstype: "{{ item.fs_type }}" - state: "mounted" - loop: "{{ storage.json }}" diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf deleted file mode 100644 index 9a58da641e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the NFS instance if no name is specified." - type = string -} - -variable "name" { - description = "The resource name of the instance." - type = string - default = null -} - -variable "zone" { - description = "The zone name where the NFS instance located in." - type = string -} - -variable "boot_disk_size" { - description = "Storage size in GB for the boot disk" - type = number - default = null -} - -variable "boot_disk_type" { - description = "Storage type for the boot disk" - type = string - default = null -} - -variable "create_boot_snapshot_before_destroy" { - description = "Whether to create a snapshot before destroying the boot disk" - type = bool - default = false -} - -variable "disk_size" { - description = "Storage size in GB for the NFS data disk" - type = number - default = "100" -} - -variable "type" { - description = "Storage type for the NFS data disk" - type = string - default = "pd-ssd" -} - -variable "create_snapshot_before_destroy" { - description = "Whether to create a snapshot before destroying the NFS data disk" - type = bool - default = false -} - -variable "provisioned_iops" { - description = "Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput" - type = number - default = null -} - -variable "provisioned_throughput" { - description = "Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme" - type = number - default = null -} - -# Deprecated, replaced by instance_image -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: The VM image used by the NFS server" - type = string - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - The VM image used by the NFS server. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -# Deprecated, replaced by create_snapshot_before_destroy and create_boot_snapshot_before_destroy -# tflint-ignore: terraform_unused_declarations -variable "auto_delete_disk" { - description = "DEPRECATED: Whether or not the NFS disk should be auto-deleted" - type = string - default = null - - validation { - condition = var.auto_delete_disk == null - error_message = "The 'var.auto_delete_disk' setting is broken in Cluster Toolkit versions >1.25.0 and deprecated in versions >1.48.0, please use 'var.create_snapshot_before_destroy' and 'var.create_boot_snapshot_before_destroy' instead." - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the NFS VM." - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the NFS VM." - type = string - default = null -} - -variable "machine_type" { - description = "Type of the VM instance to use" - type = string - default = "n2d-standard-2" -} - -variable "labels" { - description = "Labels to add to the NFS instance. Key-value pairs." - type = map(string) -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "service_account" { - description = "Service Account for the NFS server" - type = string - default = null -} - -variable "scopes" { - description = "Scopes to apply to the controller" - type = list(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "local_mounts" { - description = "Mountpoint for this NFS compute instance" - type = list(string) - default = ["/data"] - - validation { - condition = alltrue([ - for m in var.local_mounts : substr(m, 0, 1) == "/" - ]) - error_message = "Local mountpoints have to start with '/'." - } - validation { - condition = length(var.local_mounts) > 0 - error_message = "At least one local mount must be specified in var.local_mounts." - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf deleted file mode 100644 index 63443806b8..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.14" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/sycomp-scale/README.md b/deletion-test/primary/modules/embedded/community/modules/file-system/sycomp-scale/README.md deleted file mode 100644 index 79ff12bc18..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/sycomp-scale/README.md +++ /dev/null @@ -1,35 +0,0 @@ -## Description - -This document provides information on how to deploy an instance of [Sycomp Intelligent Data Storage Platform](https://sycomp.com/solution/hpc/storage/) on Google Cloud Platform ([GCP](https://cloud.google.com/)) using the Google Cluster Toolkit. - -> **_NOTE:_** -> Sycomp Storage on GCP does not require an HPC Toolkit wrapper. -> Terraform modules are sourced directly from GitLab. - -Terraform modules for Sycomp Intelligent Data Storage Platform are downloaded on deployment using the Google Cloud Toolkit. - -The Terraform module parameters are documented in the `README.md` files in the respective module directories of the source GitLab repository. The main modules are: - -- `sycomp-scale` -- `sycomp-scale-expansion` - -## Examples - -The community examples folder (community/examples/sycomp/) contains four example blueprints that you can use to deploy or expand a Sycomp Storage cluster. - -- [community/examples/sycomp/sycomp-storage.yaml][sycomp-storage-yaml] - - Blueprint for deploying a Sycomp Storage cluster consisting of 3 storage servers. - -- [community/examples/sycomp/sycomp-storage-expansion.yaml][sycomp-storage-expansion-yaml] - - Blueprint for expanding the above created cluster from 3 to 4 storage servers. - -- [community/examples/sycomp/sycomp-storage-ece.yaml][sycomp-storage-ece-yaml] - - Blueprint for deploying a Sycomp Storage cluster consisting of 7 storage servers with ECE (Erasure Code Edition) software RAID. - -- [community/examples/sycomp/sycomp-storage-slurm.yaml][sycomp-storage-slurm-yaml] - - Blueprint for deploying a Slurm cluster and Sycomp Storage cluster with 3 servers. The Slurm compute nodes are configured as NFS clients and have the ability to use the Sycomp Storage filesystem. - -[sycomp-storage-yaml]: ../../../examples/sycomp/sycomp-storage.yaml -[sycomp-storage-expansion-yaml]: ../../../examples/sycomp/sycomp-storage-expansion.yaml -[sycomp-storage-ece-yaml]: ../../../examples/sycomp/sycomp-storage-ece.yaml -[sycomp-storage-slurm-yaml]: ../../../examples/sycomp/sycomp-storage-slurm.yaml diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/README.md b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/README.md deleted file mode 100644 index 0e2a936167..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/README.md +++ /dev/null @@ -1,182 +0,0 @@ -## Description - -This module provides scripts for client installation and mounting [WEKA] -filesystems. Client supports both UDP and DPDK modes and allows customization of -mount parameters using Compute VM instance metadata. - -For deploying Weka cluster please consult [WEKA installation on GCP]. - -[WEKA]: https://www.weka.io/ -[WEKA installation on GCP]: https://docs.weka.io/planning-and-installation/weka-installation-on-gcp - -## Prerequisites - -* up and running Weka cluster -* running on a [supported OS](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#operating-system) -* [open firewall](https://docs.weka.io/planning-and-installation/prerequisites-and-compatibility#required-ports) - between WEKA backend servers and clients -* VPC peering configuration: - * if clients share VPCs created for WEKA cluster, no additional configuration - is necessary - * if dedicated VPCs are in use for clients, then WEKA VPCs needs to be peered - with VPCs that are used as: - * primary interface on client - * interfaces dedicated for DPDK client - * if dedicated VPCs are in use for clients, then those VPCs needs to be peered - with each other - -## Mounting -This example creates mount scripts that will mount `default` filesystem from -`10.0.0.3` WEKA backend: - -```yaml - - id: wekafs - source: community/modules/file-system/weka-client - settings: - local_mount: /scratch - server_ip: 10.0.0.3 - remote_mount: default - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: $(wekafs.runners) -``` - -If you need to add mount script along other runners, remember to add all 4 -runners provided by this script as shown in this example: - -```yaml - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(wekafs.client_install_runner) - - $(wekafs.mount_runner) - - type: shell - content: | - #!/bin/bash - - echo Sample - destination: sample-script.sh -``` - -To use the client within Slurm partition, with DPDK, remember to set additional -networks, and configure metadata. In this example, all four additional interfaces -are dedicated to WEKA DPDK - -```yaml - - id: c2_60_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - - mount-at-startup # as defined in previous examples - settings: - bandwidth_tier: virtio_enabled # Weka requires VirtIO, from WEKA 4.4.1, DPDK is also supported on gVNIC - additional_networks: - - subnetwork: weka-client-1 - nic_type: VIRTIO_NET - - subnetwork: weka-client-2 - nic_type: VIRTIO_NET - - subnetwork: weka-client-3 - nic_type: VIRTIO_NET - - subnetwork: weka-client-4 - nic_type: VIRTIO_NET - machine_type: c2-standard-60 - metadata: - weka-data_interfaces: 1,2,3,4 # allocate interfaces 1, 2, 3 and 4 to DPDK - weka-mode: dpdk - weka-options: num_cores=4,dpdk_base_memory_mb=16 - node_conf: - # From https://docs.weka.io/planning-and-installation/bare-metal/planning-a-weka-system-installation - # do not set RealMem as this is set automatically by Cluster Toolkit - CoreSpecCount: 4 - MemSpecLimit: 5120 -``` - -Due to the fact, that client installation takes ~6-7 minutes, if you use WEKA together with Slurm and do not bundle -client in the instance image, you may need to increase the timeout for startups scripts. - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - settings: - compute_startup_scripts_timeout: 600 - login_startup_scripts_timeout: 600 - ... - - id: compute_partition - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - settings: - resume_timeout: 600 - ... -``` - -## Supported VM metadata options -Client scripts do support following metadata keys: -* `weka-mode` - one of `udp` or `dpdk`. Defaults to `udp`. Sets client mode. -* `weka-data_interfaces` - comma separated list of interface identifiers, - specifying which interfaces are dedicated for data plane. Set to `1` to - dedicate second interface of instance for WEKA DPDK. Set to `2,5` to dedicate - third and sixth interface of instance for WEKA DPDK. -* `weka-mgmt_interface` - identifier of management interface, defaults to `0`, - which means to use primary interface as management interface. -* `weka-options` - additional [mount command options](https://docs.weka.io/weka-filesystems-and-object-stores/mounting-filesystems#mount-command-options) - to pass to `mount` command - -## Adding client to the OS image -To save time during the mount command install and precompile DPDK drivers in the -OS image. Following scripts compiles DPDK driver for currently running kernel. - -```shell -#!/bin/bash - -set -e -o pipefail - -echo Downloading and installing Weka client -curl --max-time 10 "{{ weka backend endpoint }}/dist/v1/install" | sh -WEKA_VERSION=$(weka -v | sed -e 's/^[^0-9]*//') -echo Installing Weka version: ${WEKA_VERSION} -weka version get "${WEKA_VERSION}" -weka version set "${WEKA_VERSION}" -# run setup for the second time, if it fails for the first time -weka local setup weka || weka local setup weka -weka version prepare "${WEKA_VERSION}" -weka local stop -weka local rm -f --all -``` - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | -| [mount\_options](#input\_mount\_options) | Mount options for filesystem shared by all clients. | `string` | `""` | no | -| [remote\_mount](#input\_remote\_mount) | Weka filesystem name. | `string` | n/a | yes | -| [server\_ip](#input\_server\_ip) | Weka backend IP address used for bootstrapping. | `string` | `""` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_install\_runner](#output\_client\_install\_runner) | Ansible runner that performs client installation needed to use file system. | -| [mount\_runner](#output\_mount\_runner) | Ansible runner that mounts the file system. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/metadata.yaml deleted file mode 100644 index 419bc3fe46..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/outputs.tf deleted file mode 100644 index 0bd9098d80..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/outputs.tf +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - template_args = { - local_mount = var.local_mount - mount_options = var.mount_options == "" ? "" : "-o ${var.mount_options}" - remote_mount = var.remote_mount - server_ip = var.server_ip - service_name = "weka-mount${replace(var.local_mount, "/", "-")}" - } - mount_script = templatefile("${path.module}/templates/mount-weka.sh.tftpl", local.template_args) - - mount_runner_ansible = { - type = "ansible-local" - content = templatefile( - "${path.module}/templates/mount-weka.yaml.tftpl", - merge( - local.template_args, - { mount_weka_script = local.mount_script } - ) - ) - destination = "mount_filesystem${replace(var.local_mount, "/", "_")}.yaml" - } - - client_install_runner = { - type = "ansible-local" - content = templatefile("${path.module}/templates/install-weka-client.yaml.tftpl", local.template_args) - destination = "install_filesystem${replace(var.local_mount, "/", "_")}.yaml" - } -} - -# currently WEKA mounts are not compatible with network_storage logic, as WEKA volumes needs to be mounted by -# systemd script and not /etc/fstab entry, as the mount command needs to have network configuration which may change -# between restarts -# -#output "network_storage" { -# description = "Describes a remote network storage to be mounted by fs-tab." -# value = { -# server_ip = var.server_ip -# remote_mount = var.remote_mount -# local_mount = var.local_mount -# fs_type = var.fs_type -# mount_options = var.mount_options -# client_install_runner = local.client_install_runner -# mount_runner = local.mount_runner -# } -#} -# -output "client_install_runner" { - description = "Ansible runner that performs client installation needed to use file system." - value = local.client_install_runner -} - -output "mount_runner" { - description = "Ansible runner that mounts the file system." - value = local.mount_runner_ansible -} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl deleted file mode 100644 index ddc3acdb5d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/install-weka-client.yaml.tftpl +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Mounts the file systems specified in the metadata network_storage key - hosts: localhost - become: true - vars: - meta_key: "network_storage" - url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" - tasks: - - name: Check if weka is installed - ansible.builtin.stat: - path: /usr/bin/weka - register: weka_binary - - - name: Create temporary location for installation script - ansible.builtin.tempfile: - state: file - register: - install_script - when: not weka_binary.stat.exists - - - name: Download WEKA client - ansible.builtin.get_url: - url: http://${server_ip}:14000/dist/v1/install - dest: "{{ install_script.path }}" - mode: "700" - when: not weka_binary.stat.exists - - - name: Run WEKA installation script - ansible.builtin.shell: - cmd: "{{ install_script.path }}" - when: not weka_binary.stat.exists - register: weka_install_result - changed_when: weka_install_result.rc == 0 - - - name: Read metadata network_storage information - ansible.builtin.uri: - url: "{{ url }}/weka-version" - method: GET - headers: - Metadata-Flavor: "Google" - status_code: - - 200 - - 404 - register: get_weka_version - - - name: Set WEKA version from metadata server - ansible.builtin.set_fact: - weka_version: "{{ get_weka_version.body }}" - when: get_weka_version.status == 200 - - - name: Get version of WEKA installation client - ansible.builtin.shell: - cmd: weka -v | sed -e 's/^[^0-9.]*\([0-9.]*\)[^0-9.]*$/\1/' - register: get_weka_client_version - changed_when: get_weka_client_version.rc == 0 - - - name: Set WEKA version from WEKA installation client - ansible.builtin.set_fact: - weka_version: "{{ get_weka_client_version.stdout }}" - when: get_weka_version.status == 404 - - - name: Download user-defined WEKA version - ansible.builtin.shell: - cmd: weka version get {{ weka_version }} - register: result - changed_when: result.rc == 0 - - - name: Set user-defined WEKA version - ansible.builtin.shell: - cmd: weka version set {{ weka_version }} - register: result - changed_when: result.rc == 0 - - - name: Setup WEKA client - ansible.builtin.shell: - cmd: weka local setup weka - register: setup_1_result - changed_when: setup_1_result.rc == 0 - failed_when: false # ignore errors - - - name: Setup WEKA client (2nd try) - ansible.builtin.shell: - cmd: weka local setup weka - register: result - changed_when: result.rc == 0 - when: setup_1_result.rc != 0 - - - name: Prepare WEKA version - ansible.builtin.shell: - cmd: weka version prepare {{ weka_version }} - register: result - changed_when: result.rc == 0 - - - name: Stop WEKA client - ansible.builtin.shell: - cmd: weka local stop - async: 30 - poll: 10 - register: weka_stop - changed_when: weka_stop.get("rc") == 0 # when killed by async, rc is not defined - failed_when: false # ignore errors - - - name: Stop WEKA client (2nd try) - ansible.builtin.shell: - cmd: weka local stop - async: 30 - poll: 10 - register: result - changed_when: result.rc == 0 - failed_when: false # ignore errors - when: weka_stop.get("rc") != 0 - - - name: Remove WEKA containers - ansible.builtin.shell: - cmd: weka local rm -f --all - register: result - changed_when: result.rc == 0 - failed_when: false # ignore errors diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl deleted file mode 100644 index 19c6dc1fdc..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.sh.tftpl +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -# shellcheck disable=SC2034 -METADATA_BASE_URL="http://metadata.google.internal/computeMetadata/v1/instance" -# shellcheck disable=SC2034 -ATTR_URL="$${METADATA_BASE_URL}/attributes/weka-" -NET_URL="$${METADATA_BASE_URL}/network-interfaces" - -# shellcheck disable=SC1083 -WEKA_MODE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mode || echo -n udp) -# shellcheck disable=SC1083 -WEKA_DATA_INTERFACES=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}data_interfaces || exit 0) -# shellcheck disable=SC1083 -WEKA_MGMT_INTERFACE=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}mgmt_interface || echo -n 0) -# shellcheck disable=SC1083 -WEKA_OPTIONS=$(curl --fail -s -H "Metadata-Flavor: Google" $${ATTR_URL}options || exit 0) - -WEKA_OPTIONS="$${WEKA_OPTIONS:+-o $WEKA_OPTIONS}" - -netmask_to_cidr () { - c=0 - # shellcheck disable=SC2086,SC1083 - x=0$( printf '%o' $${1//./ } ) - while [ "$x" -gt 0 ]; do - c=$(( c + x%2 )) - x=$(( x >> 1)) - done - echo $c ; -} - -# detect network interface naming scheme -if [[ -e /sys/class/net/eth0 ]] ; then - DEVICE_NAME="eth" - DEVICE_INDEX_BASE=0 -elif [[ -e /sys/class/net/ens4 ]] ; then - DEVICE_NAME="ens" - DEVICE_INDEX_BASE=4 -else - echo "Can't detect device names. Both /sys/class/net/eth0 and /sys/class/net/ens4 do not exists" - exit 1 -fi - -# ensure that /etc/hosts contains entry for hostname pointing to primary interface -NEW_IP=$(ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) | head -n 1 | sed -e 's/^.*inet \([0-9\.]\+\)\/.*$/\1/') -if [ -n "$NEW_IP" ] ; then - HOSTNAME=$(hostname) - sed -i -e "/$HOSTNAME/s/^[0-9\.]\+ $HOSTNAME/$NEW_IP $HOSTNAME/" /etc/hosts -else - echo "Failed to find primary interface address" - ip -4 -o addr show dev $DEVICE_NAME$(( DEVICE_INDEX_BASE + WEKA_MGMT_INTERFACE )) - exit 1 -fi - -# shellcheck disable=SC2154 -echo "Mounting Weka ${server_ip}/${remote_mount} to ${local_mount}" -mkdir -p "${local_mount}" -service weka-agent start -if [[ $WEKA_MODE == "udp" ]] ; then - # shellcheck disable=SC2086,SC2154,SC2086 - mount -t wekafs ${mount_options} -o net=udp $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" - -elif [[ $WEKA_MODE == "dpdk" ]] ; then - declare -a DATA_INTERFACES - # split WEKA_DATA_INTERFACES by comma into array - # shellcheck disable=SC2034 - IFS=',' read -r -a DATA_INTERFACES <<< "$WEKA_DATA_INTERFACES" - - DATA_OPTIONS="" - # shellcheck disable=SC2066 - for interface in "$${DATA_INTERFACES[@]}" ; do - INTERFACE_IP=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/ip") - INTERFACE_MASK=$(netmask_to_cidr "$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/subnetmask")") - INTERFACE_GATEWAY=$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$interface/gateway") - - DATA_OPTIONS+="-o net=$DEVICE_NAME$((DEVICE_INDEX_BASE + interface))/$INTERFACE_IP/$INTERFACE_MASK/$INTERFACE_GATEWAY " - done - - # shellcheck disable=SC2086 - mount -t wekafs \ - ${mount_options} \ - -o mgmt_ip="$(curl --fail -s -H "Metadata-Flavor: Google" "$NET_URL/$WEKA_MGMT_INTERFACE/ip")" \ - $DATA_OPTIONS $WEKA_OPTIONS "${server_ip}/${remote_mount}" "${local_mount}" -else - echo "Unknown weka:mode metadata value: $${WEKA_MODE}. Allowed values: udp and dpdk" - exit 1 -fi diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl deleted file mode 100644 index 84587103a9..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/templates/mount-weka.yaml.tftpl +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Mount the WEKA file systems - hosts: localhost - become: true - vars: - local_mount: "${local_mount}" - remote_mount: "${remote_mount}" - server_ip: "${server_ip}" - service_name: "weka-mount-${replace(local_mount, "/", "_")}" - tasks: - - name: Create mount script - ansible.builtin.copy: - dest: "/etc/{{ service_name }}.sh" - mode: "0755" - content: | - ${indent(8, mount_weka_script)} - - - name: Create systemd service for weka mount - ansible.builtin.copy: - dest: "/etc/systemd/system/{{ service_name }}.service" - mode: "0644" - content: | - [Install] - WantedBy=multi-user.target - [Unit] - Description=Mount Weka {{ server_ip }}/{{ remote_mount }} at {{ local_mount }} - After=network-online.target - Wants=network-online.target - [Service] - RemainAfterExit=true - Type=oneshot - ExecStart=/bin/bash -c "/etc/{{ service_name }}.sh" - - - name: Enable and start weka mount service - ansible.builtin.systemd: - name: "{{ service_name }}" - daemon_reload: true - enabled: true - state: started diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/variables.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/variables.tf deleted file mode 100644 index f07961d64c..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "local_mount" { - description = "The mount point where the contents of the device may be accessed after mounting." - type = string - default = "/mnt" -} - -variable "mount_options" { - description = "Mount options for filesystem shared by all clients." - type = string - default = "" - nullable = false -} - -variable "remote_mount" { - description = "Weka filesystem name." - type = string -} - -variable "server_ip" { - description = "Weka backend IP address used for bootstrapping." - type = string - default = "" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/versions.tf b/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/versions.tf deleted file mode 100644 index 9e6af1fa7f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/file-system/weka-client/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 0.14.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb deleted file mode 100644 index f13726f691..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb +++ /dev/null @@ -1,125 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "project_id = \"${project_id}\"\n", - "dataset_id = \"${dataset_id}\"\n", - "table_id = \"${table_id}\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ONI1Xo0-KtAD", - "outputId": "fb9ca475-e4ec-4cd0-e0e6-14f409eefd7a" - }, - "outputs": [], - "source": [ - "from google.cloud import bigquery\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "\n", - "client = bigquery.Client(project=project_id)\n", - "\n", - "df = client.query(f'''\n", - "SELECT ticker, cast(price AS FLOAT64) AS price, CAST(OFFSET as INTEGER) AS offset, start_date, end_date, iteration\n", - "FROM `{project_id}.{dataset_id}.{table_id}`,\n", - "UNNEST(simulation_results) as NUMERIC with OFFSET\n", - "WHERE epoch_time IN\n", - " # Get the latest simulation runs for each Ticker Symbol\n", - "(SELECT MAX(epoch_time) FROM `{project_id}.{dataset_id}.{table_id}` GROUP BY ticker)\n", - "'''\n", - ").to_dataframe()\n", - "# Display the data\n", - "df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Define a function to plot the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "def plot_ticker(t,df):\n", - "\n", - " dtf = df[(df.ticker==t) &(df.offset == 250)].price.describe(include=[np.float64], percentiles=[.05, .01, .001])\n", - " cellText = []\n", - " for v in dtf.values:\n", - " cellText.append([v])\n", - " \n", - " pltf = df[df.ticker==t].pivot(index='offset', columns='iteration', values='price')\n", - " \n", - " fig = plt.figure(figsize=(10,5))\n", - " ax1 = fig.add_subplot(122)\n", - " pltf.plot(legend=False, ax=ax1, xlabel='Time(days)', ylabel='US$', title=f\"{ df[(df.ticker == t) & (df.offset == 0) & (df.iteration == 4)]}\")\n", - " ax2 = fig.add_subplot(121)\n", - " font_size=10\n", - " bbox=[0, 0, .5, 1]\n", - " ax2.axis('off')\n", - " mpl_table = ax2.table(cellText = cellText, rowLabels=dtf.index.values, bbox=bbox)\n", - " mpl_table.auto_set_font_size(False)\n", - " mpl_table.set_fontsize(font_size)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 808 - }, - "id": "jvBmb_KceX7z", - "outputId": "42a3ba9f-b68f-4c7b-d928-0fedeed9216c" - }, - "outputs": [], - "source": [ - "ticker_list = df.ticker.unique()\n", - "for t in ticker_list:\n", - " plot_ticker(t,df)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md deleted file mode 100644 index e54893a1bb..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md +++ /dev/null @@ -1,97 +0,0 @@ -## Description - -Copy files to a target GCS bucket. - -Primarily used for FSI - MonteCarlo Tutorial **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: -../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage -This copies the module files to the specified GCS bucket. It is expected that -the bucket will be mounted on the target VM. - -Some of the files are templates, and `main.tf` translates the files with the -passed variable values. This way the user does not have to change things like -pointing to the correct bigquery table or adding in the project_id. - -```yaml - - id: fsi_tutorial_files - source: community/modules/files/fsi-montecarlo-on-batch - use: [bq-dataset, bq-table, fsi_bucket, pubsub_topic] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 3.83 | -| [http](#requirement\_http) | ~> 3.0 | -| [random](#requirement\_random) | ~> 3.0 | -| [template](#requirement\_template) | ~> 2.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [http](#provider\_http) | ~> 3.0 | -| [random](#provider\_random) | ~> 3.0 | -| [template](#provider\_template) | ~> 2.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.get_iteration_sh](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.get_mc_reqs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.get_requirements](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.ipynb_obj_fsi](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.mc_obj_yaml](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.mc_run](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.run_batch_py](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [http_http.batch_py](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | -| [http_http.batch_requirements](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | -| [template_file.ipynb_fsi](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | -| [template_file.mc_run_py](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | -| [template_file.mc_run_yaml](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Bigquery dataset id | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to run project | `string` | n/a | yes | -| [table\_id](#input\_table\_id) | Bigquery table id | `string` | n/a | yes | -| [topic\_id](#input\_topic\_id) | Pubsub Topic Name | `string` | n/a | yes | -| [topic\_schema](#input\_topic\_schema) | Pubsub Topic schema | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh deleted file mode 100644 index 50aa865a31..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ticker=("GOOG" "AMZN" "MSFT" "NVDA" "META" "TSLA" "PEP" "COST") -echo "BI: $BATCH_TASK_INDEX" -echo "TI: ${ticker[$BATCH_TASK_INDEX]}" -python3 -m pip install -r /mnt/disks/fsi/mc_run_reqs.txt -python3 /mnt/disks/fsi/mc_run.py \ - --ticker "${ticker[$BATCH_TASK_INDEX]}" \ - --iterations 500 \ - --start_date 2022-01-01 diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf deleted file mode 100644 index 83dc7fe9cf..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - bucket = replace(var.gcs_bucket_path, "gs://", "") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -data "template_file" "mc_run_py" { - template = file("${path.module}/mc_run.tpl.py") - vars = { - project_id = var.project_id - topic_id = var.topic_id - topic_schema = var.topic_schema - dataset_id = var.dataset_id - table_id = var.table_id - } -} - -resource "google_storage_bucket_object" "mc_run" { - name = "mc_run.py" - content = data.template_file.mc_run_py.rendered - bucket = local.bucket -} - -data "template_file" "mc_run_yaml" { - template = file("${path.module}/mc_run.tpl.yaml") - vars = { - project_id = var.project_id - bucket_name = local.bucket - region = var.region - } -} - -resource "google_storage_bucket_object" "mc_obj_yaml" { - name = "mc_run.yaml" - content = data.template_file.mc_run_yaml.rendered - bucket = local.bucket -} - -data "template_file" "ipynb_fsi" { - template = file("${path.module}/FSI_MonteCarlo.ipynb") - vars = { - project_id = var.project_id - dataset_id = var.dataset_id - table_id = var.table_id - } -} -resource "google_storage_bucket_object" "ipynb_obj_fsi" { - name = "FSI_MonteCarlo.ipynb" - content = data.template_file.ipynb_fsi.rendered - bucket = local.bucket -} - -data "http" "batch_py" { - url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/batch.py" -} - -resource "google_storage_bucket_object" "run_batch_py" { - name = "batch.py" - content = data.http.batch_py.response_body - bucket = local.bucket -} - -data "http" "batch_requirements" { - url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/requirements.txt" -} - -resource "google_storage_bucket_object" "get_requirements" { - name = "requirements.txt" - content = data.http.batch_requirements.response_body - bucket = local.bucket -} - -resource "google_storage_bucket_object" "get_iteration_sh" { - name = "iteration.sh" - content = file("${path.module}/iteration.sh") - bucket = local.bucket -} - -resource "google_storage_bucket_object" "get_mc_reqs" { - name = "mc_run_reqs.txt" - content = file("${path.module}/mc_run_reqs.txt") - bucket = local.bucket -} diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py deleted file mode 100644 index 4e0a64e363..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Run MC simulation for VaR portfolio risk -""" - -import avro.schema -import io -import google.auth -import numpy -import time -import yfinance as yf - -from absl import app -from absl import flags -from avro.io import DatumWriter, BinaryEncoder, BinaryDecoder, DatumReader -from datetime import datetime -from datetime import timedelta -from google.cloud import pubsub_v1, bigquery -from google.cloud.pubsub import SchemaServiceClient - -PROJECT_ID = '${project_id}' -INCOMING_TOPIC_ID = '${topic_id}' -INCOMING_TOPIC_SCHEMA = '${topic_schema}' -DATASET_ID = '${dataset_id}' -TABLE_ID = '${table_id}' - - -FLAGS = flags.FLAGS - -flags.DEFINE_string("ticker", 'GOOG', "Nasdaq Stock Ticker to run, default GOOG") -flags.DEFINE_string("start_date", '2022-01-01' , "Start data for data query, default 2022-01-01") -flags.DEFINE_integer("calendar_days", 365 , "How many calendar days to include in the calculation") -flags.DEFINE_integer("epoch_time", f'{int(time.time())}' , "Epoch time, number of seconds since January 1st, 1970 at 00:00:00 UTC.") -flags.DEFINE_integer("iterations", 100 , "Number of iterations to run.") -flags.DEFINE_boolean("print_raw", False, "Dump raw data.") - -class VaRSimulator: - - def __init__(self): - pass - - def get_data(self): - self.get_historical_data_yahoo() - - def get_historical_data_yahoo(self): - - # get historical market data: https://pypi.org/project/yfinance/ - - self.raw_data = yf.Ticker(self.ticker).history(start=self.start_date, end=self.end_date ) - self.data = self.raw_data.Close - - def print_raw(self): - print(self.get_stats()) - print(type(self.raw_data)) - print(self.raw_data) - - def get_stats(self): - close = self.data - self.first = close[0] - self.last = close[-1] - self.trading_days = len(close) - self.cagr = (self.last / self.first) ** (365.0/self.calendar_days) -1.0 - self.volatility = self.data.pct_change().std() - return(self.first, self.last, self.trading_days, self.cagr, self.volatility) - - def run_simulation(self): - - returns = numpy.random.normal(self.cagr/self.trading_days, self.volatility, self.trading_days) + 1 - returns = numpy.insert(returns,0,1.0) - self.simulation_results = self.last * returns.cumprod() - return(self.simulation_results) - - def create_object(self): - self.object = { - "ticker": self.ticker, - "epoch_time": self.epoch_time, - "iteration": self.iteration, - "start_date": self.start_date, - "end_date": self.end_date, - "simulation_results": list(map(lambda x: {"price":x}, self.simulation_results)) - } - return(self.object) - - -class PubsubToBiquery: - - def __init__(self): - - the_time = int(time.time()) - - self.project_id = PROJECT_ID - - self.publisher_client = pubsub_v1.PublisherClient() - self.topic_path = self.publisher_client.topic_path(self.project_id, INCOMING_TOPIC_ID) - - self.schema_client = SchemaServiceClient() - self.schema_path = self.schema_client.schema_path(self.project_id, INCOMING_TOPIC_SCHEMA) - - pubsub_schema = self.schema_client.get_schema(request={"name": self.schema_path}) - avro_schema = avro.schema.parse(pubsub_schema.definition) - - self.writer = DatumWriter(avro_schema) - - - def publish_record(self,record): - - byte_stream = io.BytesIO() - encoder = BinaryEncoder(byte_stream) - self.writer.write(record, encoder) - data = byte_stream.getvalue() - byte_stream.flush() - future = self.publisher_client.publish(self.topic_path, data) - if(FLAGS.print_raw): - print(f"Published message ID: {future.result()}") - - -def main(argv): - - vr = VaRSimulator() - pbbq = PubsubToBiquery() - - vr.ticker =FLAGS.ticker - vr.start_date =FLAGS.start_date - vr.end_date =f'{(datetime.strptime(FLAGS.start_date,"%Y-%m-%d") + timedelta(days = FLAGS.calendar_days)).date()}' - vr.calendar_days = FLAGS.calendar_days - vr.epoch_time = FLAGS.epoch_time - vr.iteration = 1 - - vr.get_data() - vr.get_stats() - - for i in range(FLAGS.iterations): - vr.iteration = i - vr.run_simulation() - pbbq.publish_record(vr.create_object()) - - if(FLAGS.print_raw): - vr.print_raw() - - -if __name__ == "__main__": - """ This is executed when run from the command line """ - app.run(main) diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml deleted file mode 100644 index 7f7de4840b..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -project_id: "${project_id}" -region: "${region}" - -job_prefix: 'fsi-' -machine_type: "n2-standard-2" -volumes: -- {bucket_name: "${bucket_name}", gcs_path: "/mnt/disks/fsi"} - -container: - image_uri: "python" - entry_point: "/bin/bash" - commands: ["/mnt/disks/fsi/iteration.sh", "$BATCH_TASK_INDEX"] - -task_count: 8 #optional -parallelism: 4 #optional -task_count_per_node: 2 #optional -cpu_milli: 1000 #optional -memory_mib: 102400 #optional - - -labels: - env: "monte" - type: "carlo" diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt deleted file mode 100644 index 105ed70ad2..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt +++ /dev/null @@ -1,9 +0,0 @@ -absl-py -avro -google-auth -google-cloud -google-cloud-batch -google-cloud-pubsub -google-cloud-bigquery -yfinance -PyYAML diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml deleted file mode 100644 index 268c8faa9a..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [storage.googleapis.com] diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf deleted file mode 100644 index eddf3c9478..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which GCS bucket will be created." - type = string -} - -variable "gcs_bucket_path" { - description = "Bucket name" - type = string - default = null -} - -variable "topic_id" { - description = "Pubsub Topic Name" - type = string -} - -variable "topic_schema" { - description = "Pubsub Topic schema" - type = string -} - -variable "dataset_id" { - description = "Bigquery dataset id" - type = string -} - -variable "table_id" { - description = "Bigquery table id" - type = string -} - -variable "region" { - description = "Region to run project" - type = string -} diff --git a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf b/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf deleted file mode 100644 index 86dcb4dc52..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - http = { - source = "hashicorp/http" - version = "~> 3.0" - } - template = { - source = "hashicorp/template" - version = "~> 2.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.74.0" - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md deleted file mode 100644 index ae8462d763..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md +++ /dev/null @@ -1,100 +0,0 @@ -# Module: Slurm Instance - - - -- [Module: Slurm Instance](#module-slurm-instance) - - [Overview](#overview) - - [Module API](#module-api) - - - -## Overview - -This module creates a [compute instance](../../../../docs/glossary.md#vm) from -[instance template](../../../../docs/glossary.md#instance-template) for a -[Slurm cluster](../slurm_cluster/README.md). - -> **NOTE:** This module is only intended to be used by Slurm modules. For -> general usage, please consider using: -> -> - [terraform-google-modules/vm/google//modules/compute_instance](https://registry.terraform.io/modules/terraform-google-modules/vm/google/latest/submodules/compute_instance). -> **WARNING:** The source image is not modified. Make sure to use a compatible -> source image. - -## Module API - -For the terraform module API reference, please see -[README_TF.md](./README_TF.md). - - -Copyright (C) SchedMD LLC. -Copyright 2018 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.0 | -| [google](#requirement\_google) | >= 3.43 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.43 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_from_template.slurm_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | -| [null_resource.replace_trigger](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_instance_template.base](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | -| [hostname](#input\_hostname) | Hostname of instances | `string` | n/a | yes | -| [instance\_template](#input\_instance\_template) | Instance template self\_link used to create compute instances | `string` | n/a | yes | -| [network](#input\_network) | Network to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | -| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | -| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | -| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | -| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances | `list(string)` | `[]` | no | -| [subnetwork](#input\_subnetwork) | Subnet to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The project that subnetwork belongs to | `string` | `null` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [available\_zones](#output\_available\_zones) | List of available zones in region | -| [instances\_details](#output\_instances\_details) | List of all details for compute instances | -| [instances\_self\_links](#output\_instances\_self\_links) | List of self-links for compute instances | -| [names](#output\_names) | List of available zones in region | -| [slurm\_instances](#output\_slurm\_instances) | List of all resource objects for compute instances | - diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf deleted file mode 100644 index 2af9008a0e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########## -# LOCALS # -########## - -locals { - num_instances = length(var.static_ips) == 0 ? var.num_instances : length(var.static_ips) - - # local.static_ips is the same as var.static_ips with a dummy element appended - # at the end of the list to work around "list does not have any elements so cannot - # determine type" error when var.static_ips is empty - static_ips = concat(var.static_ips, ["NOT_AN_IP"]) - - network_interfaces = [for index in range(local.num_instances) : - concat([ - { - access_config = var.access_config - alias_ip_range = [] - ipv6_access_config = [] - network = var.network - network_ip = length(var.static_ips) == 0 ? "" : element(local.static_ips, index) - nic_type = null - queue_count = null - stack_type = null - subnetwork = var.subnetwork - subnetwork_project = var.subnetwork_project - } - ], - var.additional_networks - ) - ] -} - -################ -# DATA SOURCES # -################ - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -data "google_compute_instance_template" "base" { - project = var.project_id - name = var.instance_template -} - -############# -# INSTANCES # -############# -resource "null_resource" "replace_trigger" { - triggers = { - trigger = var.replace_trigger - } -} - -# TODO: `internal/slurm-gcp/login` is ONLY user of `internal/slurm-gcp/instance` -# Remove this module, add functionality (+ prune generality) to the login module directly. -resource "google_compute_instance_from_template" "slurm_instance" { - count = local.num_instances - name = format("%s-%s", var.hostname, format("%03d", count.index + 1)) - project = var.project_id - zone = var.zone == null ? data.google_compute_zones.available.names[count.index % length(data.google_compute_zones.available.names)] : var.zone - - allow_stopping_for_update = true - - dynamic "network_interface" { - for_each = local.network_interfaces[count.index] - iterator = nic - content { - dynamic "access_config" { - for_each = nic.value.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = nic.value.alias_ip_range - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - dynamic "ipv6_access_config" { - for_each = nic.value.ipv6_access_config - iterator = access_config - content { - network_tier = access_config.value.network_tier - } - } - network = nic.value.network - network_ip = nic.value.network_ip - nic_type = nic.value.nic_type - queue_count = nic.value.queue_count - subnetwork = nic.value.subnetwork - subnetwork_project = nic.value.subnetwork_project - } - } - - source_instance_template = data.google_compute_instance_template.base.self_link - # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 - # we have to explicitly override instance labels instead of inheriting them from template. - labels = data.google_compute_instance_template.base.labels - - - lifecycle { - replace_triggered_by = [null_resource.replace_trigger.id] - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf deleted file mode 100644 index 4eba78a7e8..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "slurm_instances" { - description = "List of all resource objects for compute instances" - value = google_compute_instance_from_template.slurm_instance -} - -output "instances_self_links" { - description = "List of self-links for compute instances" - value = google_compute_instance_from_template.slurm_instance[*].self_link -} - -output "instances_details" { - description = "List of all details for compute instances" - value = google_compute_instance_from_template.slurm_instance[*] -} - -output "available_zones" { - description = "List of available zones in region" - value = data.google_compute_zones.available.names -} - -output "names" { - description = "List of available zones in region" - value = google_compute_instance_from_template.slurm_instance[*].name -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf deleted file mode 100644 index 11111a2c05..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "The GCP project ID" - default = null -} - -variable "network" { - description = "Network to deploy to. Only one of network or subnetwork should be specified." - type = string - default = "" -} - -variable "subnetwork" { - description = "Subnet to deploy to. Only one of network or subnetwork should be specified." - type = string - default = "" -} - -variable "subnetwork_project" { - description = "The project that subnetwork belongs to" - type = string - default = null -} - -variable "hostname" { - description = "Hostname of instances" - type = string -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })) - nullable = false -} - -variable "static_ips" { - description = "List of static IPs for VM instances" - type = list(string) - default = [] -} - -variable "access_config" { - description = "Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet." - type = list(object({ - nat_ip = string - network_tier = string - })) - default = [] -} - -variable "num_instances" { - description = "Number of instances to create. This value is ignored if static_ips is provided." - type = number - default = 1 -} - -variable "instance_template" { - description = "Instance template self_link used to create compute instances" - type = string -} - -variable "region" { - description = "Region where the instances should be created." - type = string - default = null -} - -variable "zone" { - description = "Zone where the instances should be created. If not specified, instances will be spread across available zones in the region." - type = string - default = null -} - -######### -# SLURM # -######### - -variable "replace_trigger" { - description = "Trigger value to replace the instances." - type = string - default = "" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf deleted file mode 100644 index a3e84c09bf..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = "~> 1.0" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.43" - } - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md deleted file mode 100644 index 87394bef6a..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md +++ /dev/null @@ -1,87 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.0 | -| [local](#requirement\_local) | ~> 2.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | ~> 2.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance\_template](#module\_instance\_template) | ../internal_instance_template | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.startup](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    disk_type                  = optional(string)
    disk_size_gb               = optional(number)
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `virtio_enabled` setting will only enable VirtioNet and will not enable TIER\_1.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard. | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See
https://cloud.google.com/compute/docs/gpus more details.
- type : the GPU type
- count : number of GPUs | 
object({
    type  = string
    count = number
  })
| `null` | no | -| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"n1-standard-1"` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Prefix for template resource. | `string` | `"default"` | no | -| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network
attribute for Legacy or Auto subnetted networks and subnetwork for custom
subnetted networks. | `string` | `null` | no | -| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | -| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the instances. See
'main.tf:local.service\_account' for the default. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming. | `string` | n/a | yes | -| [slurm\_instance\_role](#input\_slurm\_instance\_role) | Slurm instance type. Must be one of: controller; login; compute; or null. | `string` | n/a | yes | -| [source\_image](#input\_source\_image) | Source disk image. | `string` | `""` | no | -| [source\_image\_family](#input\_source\_image\_family) | Source image family. | `string` | `""` | no | -| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. If it is not provided, the provider project is used. | `string` | `""` | no | -| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must
exist in the same region this instance will be created in. Either network or
subnetwork must be provided. | `string` | `null` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [termination\_action](#input\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_template](#output\_instance\_template) | Instance template details | -| [labels](#output\_labels) | Labels attached to the instance template | -| [name](#output\_name) | Name of instance template | -| [self\_link](#output\_self\_link) | Self\_link of instance template | -| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | -| [tags](#output\_tags) | Tags that will be associated with instance(s) | - diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted deleted file mode 100644 index 2edaa942d2..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -SLURM_DIR=/slurm -FLAGFILE=$SLURM_DIR/slurm_configured_do_not_remove -SCRIPTS_DIR=$SLURM_DIR/scripts -if [[ -z "$HOME" ]]; then - # google-startup-scripts.service lacks environment variables - HOME="$(getent passwd "$(whoami)" | cut -d: -f6)" -fi - -# Temporary workaround for transition period when some of older images -# don't have "baked in" python yet. -# TODO: Remove -SLURM_PY="/slurm/python/venv/bin/python3.13" -SYSTEM_PY="/usr/bin/python3" -if [[ ! -e "$SLURM_PY" ]]; then - echo "Symlink $SLURM_PY does not exist. Creating symlink to $SYSTEM_PY" - mkdir -p /slurm/python/venv/bin - ln -s "$SYSTEM_PY" "$SLURM_PY" -fi - -METADATA_SERVER="metadata.google.internal" -URL="http://$METADATA_SERVER/computeMetadata/v1" -CURL="curl -sS --fail --header Metadata-Flavor:Google" - -PING_METADATA="ping -q -w1 -c1 $METADATA_SERVER" -echo "INFO: $PING_METADATA" -for i in $(seq 10); do - [ $i -gt 1 ] && sleep 5; - $PING_METADATA > /dev/null && s=0 && break || s=$?; - echo "ERROR: Failed to contact metadata server, will retry" -done -if [ $s -ne 0 ]; then - echo "ERROR: Unable to contact metadata server, aborting" - wall -n '*** Slurm setup failed in the startup script! see `journalctl -u google-startup-scripts` ***' - exit 1 -else - echo "INFO: Successfully contacted metadata server" -fi - -PING_GOOGLE="ping -q -w1 -c1 8.8.8.8" -echo "INFO: $PING_GOOGLE" -for i in $(seq 5); do - [ $i -gt 1 ] && sleep 2; - $PING_GOOGLE > /dev/null && s=0 && break || s=$?; - echo "failed to ping Google DNS, will retry" -done -if [ $s -ne 0 ]; then - echo "WARNING: No internet access detected" -else - echo "INFO: Internet access detected" -fi - -mkdir -p $SCRIPTS_DIR -UNIVERSE_DOMAIN="$($CURL $URL/instance/attributes/universe_domain)" -BUCKET="$($CURL $URL/instance/attributes/slurm_bucket_path)" -if [[ -z $BUCKET ]]; then - echo "ERROR: No bucket path detected." - exit 1 -fi - -SCRIPTS_ZIP="$HOME/slurm-gcp-scripts.zip" -export CLOUDSDK_CORE_UNIVERSE_DOMAIN="$UNIVERSE_DOMAIN" - -INSTANCE_ROLE="$($CURL $URL/instance/attributes/slurm_instance_role)" - -if [ "$INSTANCE_ROLE" == "controller" ]; then - DEVEL_ZIP="slurm-gcp-devel-controller.zip" -else - DEVEL_ZIP="slurm-gcp-devel.zip" -fi -until gcloud storage cp "$BUCKET/$DEVEL_ZIP" "$SCRIPTS_ZIP"; do - echo "WARN: Could not download SlurmGCP scripts, retrying in 5 seconds." - # Remove marker used to determine if gcloud is being used in a GCE VM. - # This can get mistakenly set to False in some cases. - rm -f /root/.config/gcloud/gce - sleep 5 -done -unzip -o "$SCRIPTS_ZIP" -d "$SCRIPTS_DIR" -rm -rf "$SCRIPTS_ZIP" - -#temporary hack to not make the script fail on TPU vm -chown slurm:slurm -R "$SCRIPTS_DIR" || true -chmod 700 -R "$SCRIPTS_DIR" - - -if [ -f $FLAGFILE ]; then - echo "WARNING: Slurm was previously configured, quitting" - exit 0 -fi -touch $FLAGFILE - -function tpu_setup { - #allow the following command to fail, as this attribute does not exist for regular nodes - docker_image=$($CURL $URL/instance/attributes/slurm_docker_image 2> /dev/null || true) - if [ -z $docker_image ]; then #Not a tpu node, do not do anything - return - fi - if [ "$OS_ENV" == "slurm_container" ]; then #Already inside the slurm container, we should continue starting - return - fi - - #given a input_string like "WORKER_0:Joseph;WORKER_1:richard;WORKER_2:edward;WORKER_3:john" and a number 1, this function will print richard - parse_metadata() { - local number=$1 - local input_string=$2 - local word=$(echo "$input_string" | awk -v n="$number" -F ':|;' '{ for (i = 1; i <= NF; i+=2) if ($(i) == "WORKER_"n) print $(i+1) }') - echo "$word" - } - - input_string=$($CURL $URL/instance/attributes/slurm_names) - worker_id=$($CURL $URL/instance/attributes/tpu-env | awk '/WORKER_ID/ {print $2}' | tr -d \') - real_name=$(parse_metadata $worker_id $input_string) - - #Prepare to docker pull with gcloud - mkdir -p /root/.docker - cat << EOF > /root/.docker/config.json -{ - "credHelpers": { - "gcr.io": "gcloud", - "us-docker.pkg.dev": "gcloud" - } -} -EOF - #cgroup detection - CGV=1 - CGROUP_FLAGS="-v /sys/fs/cgroup:/sys/fs/cgroup:rw" - if [ -f /sys/fs/cgroup/cgroup.controllers ]; then #CGV2 - CGV=2 - fi - if [ $CGV == 2 ]; then - CGROUP_FLAGS="--cgroup-parent=docker.slice --cgroupns=private --tmpfs /run --tmpfs /run/lock --tmpfs /tmp" - if [ ! -f /etc/systemd/system/docker.slice ]; then #In case that there is no slice prepared for hosting the containers create it - printf "[Unit]\nDescription=docker slice\nBefore=slices.target\n[Slice]\nCPUAccounting=true\nMemoryAccounting=true" > /etc/systemd/system/docker.slice - systemctl start docker.slice - fi - fi - #for the moment always use --privileged, as systemd might not work properly otherwise - TPU_FLAGS="--privileged" - # TPU_FLAGS="--cap-add SYS_RESOURCE --device /dev/accel0 --device /dev/accel1 --device /dev/accel2 --device /dev/accel3" - # if [ $CGV == 2 ]; then #In case that we are in CGV2 for systemd to work correctly for the moment we go with privileged - # TPU_FLAGS="--privileged" - # fi - - docker run -d $CGROUP_FLAGS $TPU_FLAGS --net=host --name=slurmd --hostname=$real_name --entrypoint=/usr/bin/systemd --restart unless-stopped $docker_image - exit 0 -} - -tpu_setup #will do nothing for normal nodes or the container spawned inside TPU - -echo "INFO: Running python cluster setup script" -SETUP_SCRIPT_FILE=$SCRIPTS_DIR/setup.py -chmod +x $SETUP_SCRIPT_FILE -exec $SETUP_SCRIPT_FILE diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf deleted file mode 100644 index c91bbc4fd1..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "instance_validation" { - source = "../../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -########## -# LOCALS # -########## - -locals { - additional_disks = [ - for disk in var.additional_disks : { - disk_name = disk.disk_name - device_name = disk.device_name - auto_delete = disk.auto_delete - source = disk.source - boot = disk.boot - disk_size_gb = disk.disk_size_gb - disk_type = disk.disk_type - disk_labels = merge( - disk.disk_labels, - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) - disk_resource_manager_tags = disk.disk_resource_manager_tags - } - ] - - service_account = { - email = try(var.service_account.email, null) - scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) - } - - source_image_family = ( - var.source_image_family != "" && var.source_image_family != null - ? var.source_image_family - : "slurm-gcp-6-11-hpc-rocky-linux-8" - ) - source_image_project = ( - var.source_image_project != "" && var.source_image_project != null - ? var.source_image_project - : "projects/schedmd-slurm-public/global/images/family" - ) - - source_image = ( - var.source_image != null - ? var.source_image - : "" - ) - - - name_prefix = "${var.slurm_cluster_name}-${var.slurm_instance_role}-${var.name_prefix}" - - total_egress_bandwidth_tier = var.bandwidth_tier == "tier_1_enabled" ? "TIER_1" : "DEFAULT" - - nic_type_map = { - platform_default = null - virtio_enabled = "VIRTIO_NET" - gvnic_enabled = "GVNIC" - tier_1_enabled = "GVNIC" - } - nic_type = lookup(local.nic_type_map, var.bandwidth_tier, null) - - labels = merge(var.labels, - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) -} - -######## -# DATA # -######## - -data "local_file" "startup" { - filename = "${path.module}/files/startup_sh_unlinted" -} - -############ -# TEMPLATE # -############ - -module "instance_template" { - source = "../internal_instance_template" - - project_id = var.project_id - - # Network - can_ip_forward = var.can_ip_forward - network_ip = var.network_ip - network = var.network - nic_type = local.nic_type - region = var.region - subnetwork_project = var.subnetwork_project - subnetwork = var.subnetwork - tags = var.tags - total_egress_bandwidth_tier = local.total_egress_bandwidth_tier - additional_networks = var.additional_networks - access_config = var.access_config - - # Instance - machine_type = var.machine_type - min_cpu_platform = var.min_cpu_platform - name_prefix = local.name_prefix - gpu = var.gpu - service_account = local.service_account - shielded_instance_config = var.shielded_instance_config - advanced_machine_features = var.advanced_machine_features - enable_confidential_vm = var.enable_confidential_vm - enable_shielded_vm = var.enable_shielded_vm - preemptible = var.preemptible - spot = var.spot - on_host_maintenance = var.on_host_maintenance - labels = local.labels - instance_termination_action = var.termination_action - resource_manager_tags = var.resource_manager_tags - - # Metadata - startup_script = coalesce(var.internal_startup_script, data.local_file.startup.content) - metadata = merge( - var.metadata, - { - enable-oslogin = upper(var.enable_oslogin) - slurm_bucket_path = var.slurm_bucket_path - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) - - # Image - source_image_project = local.source_image_project - source_image_family = local.source_image_family - source_image = local.source_image - - # Disk - disk_type = var.disk_type - disk_size_gb = var.disk_size_gb - auto_delete = var.disk_auto_delete - disk_labels = merge( - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - var.disk_labels, - ) - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - - max_run_duration = var.max_run_duration - provisioning_model = var.provisioning_model - reservation_affinity = var.reservation_affinity -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf deleted file mode 100644 index 65da41052e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "instance_template" { - description = "Instance template details" - value = module.instance_template -} - -output "self_link" { - description = "Self_link of instance template" - value = module.instance_template.self_link -} - -output "name" { - description = "Name of instance template" - value = module.instance_template.name -} - -output "tags" { - description = "Tags that will be associated with instance(s)" - value = module.instance_template.tags -} - -output "service_account" { - description = "Service account object, includes email and scopes." - value = module.instance_template.service_account -} - -output "labels" { - description = "Labels attached to the instance template" - value = local.labels -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf deleted file mode 100644 index 35dd9c376f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf +++ /dev/null @@ -1,431 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -########### -# GENERAL # -########### - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "on_host_maintenance" { - type = string - description = "Instance availability Policy" - default = "MIGRATE" -} - -variable "labels" { - type = map(string) - description = "Labels, provided as a map" - default = {} -} - -variable "enable_oslogin" { - type = bool - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >=0.13.0 | -| [google](#requirement\_google) | >= 3.88 | -| [google-beta](#requirement\_google-beta) | >= 6.13.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.88 | -| [google-beta](#provider\_google-beta) | >= 6.13.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance\_validation](#module\_instance\_validation) | ../../../../../modules/internal/instance_validations | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance_template.tpl](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_template) | resource | -| [google_project.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    auto_delete                = bool
    boot                       = bool
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = map(string)
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | -| [alias\_ip\_range](#input\_alias\_ip\_range) | An array of alias IP ranges for this network interface. Can only be specified for network interfaces on subnet-mode networks.
ip\_cidr\_range: The IP CIDR range represented by this alias IP range. This IP CIDR range must belong to the specified subnetwork and cannot contain IP addresses reserved by system or used by other network interfaces. At the time of writing only a netmask (e.g. /24) may be supplied, with a CIDR format resulting in an API error.
subnetwork\_range\_name: The subnetwork secondary range name specifying the secondary range from which to allocate the IP CIDR range for this alias IP range. If left unspecified, the primary range of the subnetwork will be used. | 
object({
    ip_cidr_range         = string
    subnetwork_range_name = string
  })
| `null` | no | -| [auto\_delete](#input\_auto\_delete) | Whether or not the boot disk should be auto-deleted | `string` | `"true"` | no | -| [automatic\_restart](#input\_automatic\_restart) | (Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user). | `bool` | `true` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example | `string` | `"false"` | no | -| [disk\_encryption\_key](#input\_disk\_encryption\_key) | The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance | `string` | `null` | no | -| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `string` | `"100"` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Whether to enable the Confidential VM configuration on the instance. Note that the instance image must support Confidential VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Whether to enable the Shielded VM configuration on the instance. Note that the instance image must support Shielded VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | -| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See https://cloud.google.com/compute/docs/gpus more details | 
object({
    type  = string
    count = number
  })
| `null` | no | -| [instance\_termination\_action](#input\_instance\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | -| [ipv6\_access\_config](#input\_ipv6\_access\_config) | IPv6 access configurations. Currently a max of 1 IPv6 access configuration is supported. If not specified, the instance will have no external IPv6 Internet access. | 
list(object({
    network_tier = string
  }))
| `[]` | no | -| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create, e.g. n1-standard-1 | `string` | `"n1-standard-1"` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Name prefix for the instance template | `string` | n/a | yes | -| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks. | `string` | `""` | no | -| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | -| [nic\_type](#input\_nic\_type) | The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO\_NET. | `string` | `null` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | -| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | -| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#service_account. | 
object({
    email  = optional(string)
    scopes = set(string)
  })
| n/a | yes | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Not used unless enable\_shielded\_vm is true. Shielded VM configuration for the instance. | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [source\_image](#input\_source\_image) | Source disk image. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `""` | no | -| [source\_image\_family](#input\_source\_image\_family) | Source image family. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `"centos-7"` | no | -| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. The default project contains CentOS images. | `string` | `"centos-cloud"` | no | -| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | -| [stack\_type](#input\_stack\_type) | The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4\_ONLY. | `string` | `null` | no | -| [startup\_script](#input\_startup\_script) | User startup script to run when instances spin up | `string` | `""` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided. | `string` | `""` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [total\_egress\_bandwidth\_tier](#input\_total\_egress\_bandwidth\_tier) | Network bandwidth tier. Note: machine\_type must be a supported type. Values are 'TIER\_1' or 'DEFAULT'.
See https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration for details. | `string` | `"DEFAULT"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [name](#output\_name) | Name of instance template | -| [self\_link](#output\_self\_link) | Self-link of instance template | -| [service\_account](#output\_service\_account) | value | -| [tags](#output\_tags) | Tags that will be associated with instance(s) | - diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf deleted file mode 100644 index f8d2813ece..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "instance_validation" { - source = "../../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -######### -# Locals -######### - -locals { - source_image = var.source_image != "" ? var.source_image : "centos-7-v20201112" - source_image_family = var.source_image_family != "" ? var.source_image_family : "centos-7" - source_image_project = var.source_image_project != "" ? var.source_image_project : "centos-cloud" - - boot_disk = [ - { - source_image = var.source_image != "" ? format("${local.source_image_project}/${local.source_image}") : format("${local.source_image_project}/${local.source_image_family}") - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_labels = var.disk_labels - auto_delete = var.auto_delete - disk_resource_manager_tags = var.disk_resource_manager_tags - boot = "true" - }, - ] - - all_disks = concat(local.boot_disk, var.additional_disks) - - # NOTE: Even if all the shielded_instance_config or confidential_instance_config - # values are false, if the config block exists and an unsupported image is chosen, - # the apply will fail so we use a single-value array with the default value to - # initialize the block only if it is enabled. - shielded_vm_configs = var.enable_shielded_vm ? [true] : [] - - gpu_enabled = var.gpu != null - alias_ip_range_enabled = var.alias_ip_range != null - preemptible = var.preemptible || var.spot - on_host_maintenance = ( - local.preemptible || var.enable_confidential_vm || local.gpu_enabled - ? "TERMINATE" - : var.on_host_maintenance - ) - automatic_restart = ( - # must be false when preemptible is true - local.preemptible ? false : var.automatic_restart - ) - - nic_type = var.total_egress_bandwidth_tier == "TIER_1" ? "GVNIC" : var.nic_type - - - provisioning_model = coalesce(var.provisioning_model, local.preemptible ? "SPOT" : "STANDARD") -} - -data "google_project" "this" { - project_id = var.project_id -} - -#################### -# Instance Template -#################### -resource "google_compute_instance_template" "tpl" { - provider = google-beta - name_prefix = "${var.name_prefix}-" - project = var.project_id - machine_type = var.machine_type - labels = var.labels - metadata = var.metadata - tags = var.tags - can_ip_forward = var.can_ip_forward - metadata_startup_script = var.startup_script - region = var.region - min_cpu_platform = var.min_cpu_platform - resource_manager_tags = var.resource_manager_tags - - service_account { - email = coalesce(var.service_account.email, "${data.google_project.this.number}-compute@developer.gserviceaccount.com") - scopes = lookup(var.service_account, "scopes", null) - } - - dynamic "disk" { - for_each = local.all_disks - content { - auto_delete = lookup(disk.value, "auto_delete", null) - boot = lookup(disk.value, "boot", null) - device_name = lookup(disk.value, "device_name", null) - disk_name = lookup(disk.value, "disk_name", null) - disk_size_gb = lookup(disk.value, "disk_size_gb", lookup(disk.value, "disk_type", null) == "local-ssd" ? "375" : null) - disk_type = lookup(disk.value, "disk_type", null) - interface = lookup(disk.value, "interface", lookup(disk.value, "disk_type", null) == "local-ssd" ? "NVME" : null) - mode = lookup(disk.value, "mode", null) - source = lookup(disk.value, "source", null) - source_image = lookup(disk.value, "source_image", null) - type = lookup(disk.value, "disk_type", null) == "local-ssd" ? "SCRATCH" : "PERSISTENT" - labels = (lookup(disk.value, "source", null) != null || lookup(disk.value, "disk_type", null) == "local-ssd") ? null : lookup(disk.value, "disk_labels", null) - resource_manager_tags = lookup(disk.value, "disk_resource_manager_tags", {}) - - dynamic "disk_encryption_key" { - for_each = compact([var.disk_encryption_key == null ? null : 1]) - content { - kms_key_self_link = var.disk_encryption_key - } - } - } - } - - network_interface { - network = var.network - subnetwork = var.subnetwork - subnetwork_project = var.subnetwork_project - network_ip = try(coalesce(var.network_ip), null) - nic_type = local.nic_type - stack_type = var.stack_type - dynamic "access_config" { - for_each = var.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = var.ipv6_access_config - content { - network_tier = ipv6_access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = local.alias_ip_range_enabled ? [var.alias_ip_range] : [] - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - } - - dynamic "network_interface" { - for_each = var.additional_networks - content { - network = network_interface.value.network - subnetwork = network_interface.value.subnetwork - subnetwork_project = network_interface.value.subnetwork_project - network_ip = try(coalesce(network_interface.value.network_ip), null) - nic_type = try(coalesce(network_interface.value.nic_type), null) - dynamic "access_config" { - for_each = network_interface.value.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = network_interface.value.ipv6_access_config - content { - network_tier = ipv6_access_config.value.network_tier - } - } - } - } - - network_performance_config { - total_egress_bandwidth_tier = coalesce(var.total_egress_bandwidth_tier, "DEFAULT") - } - - lifecycle { - create_before_destroy = "true" - } - - scheduling { - preemptible = local.preemptible - provisioning_model = local.provisioning_model - automatic_restart = local.automatic_restart - on_host_maintenance = local.on_host_maintenance - instance_termination_action = var.instance_termination_action - - dynamic "max_run_duration" { - for_each = var.max_run_duration != null ? [var.max_run_duration] : [] - content { - seconds = max_run_duration.value - } - } - } - - dynamic "reservation_affinity" { - for_each = var.reservation_affinity != null ? [var.reservation_affinity] : [] - content { - type = reservation_affinity.value.type - } - } - - advanced_machine_features { - enable_nested_virtualization = var.advanced_machine_features.enable_nested_virtualization - threads_per_core = var.advanced_machine_features.threads_per_core - turbo_mode = var.advanced_machine_features.turbo_mode - visible_core_count = var.advanced_machine_features.visible_core_count - performance_monitoring_unit = var.advanced_machine_features.performance_monitoring_unit - enable_uefi_networking = var.advanced_machine_features.enable_uefi_networking - } - - dynamic "shielded_instance_config" { - for_each = local.shielded_vm_configs - content { - enable_secure_boot = lookup(var.shielded_instance_config, "enable_secure_boot", shielded_instance_config.value) - enable_vtpm = lookup(var.shielded_instance_config, "enable_vtpm", shielded_instance_config.value) - enable_integrity_monitoring = lookup(var.shielded_instance_config, "enable_integrity_monitoring", shielded_instance_config.value) - } - } - - confidential_instance_config { - enable_confidential_compute = var.enable_confidential_vm - } - - dynamic "guest_accelerator" { - for_each = local.gpu_enabled ? [var.gpu] : [] - content { - type = guest_accelerator.value.type - count = guest_accelerator.value.count - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf deleted file mode 100644 index 69f8d3b98c..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "self_link" { - description = "Self-link of instance template" - value = google_compute_instance_template.tpl.self_link -} - -output "name" { - description = "Name of instance template" - value = google_compute_instance_template.tpl.name -} - -output "tags" { - description = "Tags that will be associated with instance(s)" - value = google_compute_instance_template.tpl.tags -} - -output "service_account" { - description = "value" - value = google_compute_instance_template.tpl.service_account[0] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf deleted file mode 100644 index c285c3fea5..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf +++ /dev/null @@ -1,398 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - type = string - description = "The GCP project ID" - default = null -} - -variable "name_prefix" { - description = "Name prefix for the instance template" - type = string -} - -variable "machine_type" { - description = "Machine type to create, e.g. n1-standard-1" - type = string - default = "n1-standard-1" -} - -variable "min_cpu_platform" { - description = "Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform" - type = string - default = null -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example" - type = string - default = "false" -} - -variable "tags" { - type = list(string) - description = "Network tags, provided as a list" - default = [] -} - -variable "labels" { - type = map(string) - description = "Labels, provided as a map" - default = {} -} - -variable "preemptible" { - type = bool - description = "Allow the instance to be preempted" - default = false -} - -variable "spot" { - description = <<-EOD - Provision as a SPOT preemptible instance. - See https://cloud.google.com/compute/docs/instances/spot for more details. - EOD - type = bool - default = false -} - -variable "instance_termination_action" { - description = <<-EOD - Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'. - See https://cloud.google.com/compute/docs/instances/spot for more details. - EOD - type = string - default = null -} - -variable "automatic_restart" { - type = bool - description = "(Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user)." - default = true -} - -variable "on_host_maintenance" { - type = string - description = "Instance availability Policy" - default = "MIGRATE" -} - -variable "region" { - type = string - description = "Region where the instance template should be created." - nullable = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -####### -# disk -####### -variable "source_image" { - description = "Source disk image. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." - type = string - default = "" -} - -variable "source_image_family" { - description = "Source image family. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." - type = string - default = "centos-7" -} - -variable "source_image_project" { - description = "Project where the source image comes from. The default project contains CentOS images." - type = string - default = "centos-cloud" -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = string - default = "100" -} - -variable "disk_type" { - description = "Boot disk type, can be either pd-ssd, local-ssd, or pd-standard" - type = string - default = "pd-standard" -} - -variable "disk_labels" { - description = "Labels to be assigned to boot disk, provided as a map" - type = map(string) - default = {} -} - -variable "disk_encryption_key" { - description = "The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance" - type = string - default = null -} - -variable "auto_delete" { - description = "Whether or not the boot disk should be auto-deleted" - type = string - default = "true" -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - description = "List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name" - type = list(object({ - source = optional(string) - disk_name = optional(string) - device_name = string - auto_delete = bool - boot = bool - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = map(string) - disk_resource_manager_tags = map(string) - })) - default = [] -} - -#################### -# network_interface -#################### -variable "network" { - description = "The name or self_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks." - type = string - default = "" -} - -variable "nic_type" { - description = "The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO_NET." - type = string - default = null -} - -variable "subnetwork" { - description = "The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided." - type = string - default = "" -} - -variable "subnetwork_project" { - description = "The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used." - type = string - default = null -} - -variable "network_ip" { - description = "Private IP address to assign to the instance if desired." - type = string - default = "" -} - -variable "stack_type" { - description = "The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4_ONLY." - type = string - default = null -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - })) -} - -variable "total_egress_bandwidth_tier" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 6.41 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.41 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance](#module\_instance) | ../instance | n/a | -| [template](#module\_template) | ../instance_template | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | -| [login\_nodes](#input\_login\_nodes) | Slurm login instance definitions. | 
object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  })
| n/a | yes | -| [network\_storage](#input\_network\_storage) | Storage to mounted on login instances
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | -| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path to directory in the bucket for configs | `string` | n/a | yes | -| [slurm\_bucket\_name](#input\_slurm\_bucket\_name) | Name of the bucket for configs | `string` | n/a | yes | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name | `string` | n/a | yes | -| [startup\_scripts](#input\_startup\_scripts) | List of scripts to be ran on login VMs startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | -| [startup\_scripts\_timeout](#input\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script. If any script exceeds this timeout,
then the instance setup process is considered failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instances](#output\_instances) | VM instances of login nodes | -| [service\_account](#output\_service\_account) | Service Account used by login VMs | - diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf deleted file mode 100644 index 605461f7e6..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "template" { - source = "../instance_template" - - project_id = var.project_id - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = "login" - slurm_bucket_path = var.slurm_bucket_path - name_prefix = local.name - - additional_disks = var.login_nodes.additional_disks - bandwidth_tier = var.login_nodes.bandwidth_tier - can_ip_forward = var.login_nodes.can_ip_forward - advanced_machine_features = var.login_nodes.advanced_machine_features - disk_auto_delete = var.login_nodes.disk_auto_delete - disk_labels = var.login_nodes.disk_labels - disk_resource_manager_tags = var.login_nodes.disk_resource_manager_tags - disk_size_gb = var.login_nodes.disk_size_gb - disk_type = var.login_nodes.disk_type - enable_confidential_vm = var.login_nodes.enable_confidential_vm - enable_oslogin = var.login_nodes.enable_oslogin - enable_shielded_vm = var.login_nodes.enable_shielded_vm - gpu = var.login_nodes.gpu - labels = var.login_nodes.labels - machine_type = var.login_nodes.machine_type - metadata = merge(var.login_nodes.metadata, { - "universe_domain" = var.universe_domain, - "slurm_login_group" = local.name - }) - min_cpu_platform = var.login_nodes.min_cpu_platform - on_host_maintenance = var.login_nodes.on_host_maintenance - preemptible = var.login_nodes.preemptible - region = var.login_nodes.region - resource_manager_tags = var.login_nodes.resource_manager_tags - service_account = var.login_nodes.service_account - shielded_instance_config = var.login_nodes.shielded_instance_config - source_image_family = var.login_nodes.source_image_family - source_image_project = var.login_nodes.source_image_project - source_image = var.login_nodes.source_image - spot = var.login_nodes.spot - subnetwork = var.login_nodes.subnetwork - tags = concat([var.slurm_cluster_name], var.login_nodes.tags) - termination_action = var.login_nodes.termination_action - - internal_startup_script = var.internal_startup_script -} - -module "instance" { - source = "../instance" - - access_config = var.login_nodes.access_config - hostname = "${var.slurm_cluster_name}-${local.name}" - - project_id = var.project_id - - instance_template = module.template.self_link - num_instances = var.login_nodes.num_instances - - additional_networks = var.login_nodes.additional_networks - region = var.login_nodes.region - static_ips = var.login_nodes.static_ips - subnetwork = var.login_nodes.subnetwork - zone = var.login_nodes.zone - - replace_trigger = var.replace_trigger -} - -resource "google_storage_bucket_object" "startup_scripts" { - for_each = { - for s in var.startup_scripts : format( - "slurm-login-%s-script-%s", local.name, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_") - ) => s.content - } - - bucket = var.slurm_bucket_name - name = "${var.slurm_bucket_dir}/${each.key}" - content = each.value - source_md5hash = md5(each.value) -} - -locals { - name = var.login_nodes.group_name # short hand - - config = { - group_name = local.name - startup_scripts_timeout = var.startup_scripts_timeout - network_storage = var.network_storage - } -} - -resource "google_storage_bucket_object" "config" { - bucket = var.slurm_bucket_name - name = "${var.slurm_bucket_dir}/login_group_configs/${local.name}.yaml" - content = yamlencode(local.config) - source_md5hash = md5(yamlencode(local.config)) - - # To ensure that login group "is not ready" until all startup scripts are written down - depends_on = [google_storage_bucket_object.startup_scripts] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf deleted file mode 100644 index 04de18a188..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "service_account" { - value = module.template.service_account - description = "Service Account used by login VMs" -} - -output "instances" { - value = module.instance.slurm_instances - description = "VM instances of login nodes" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf deleted file mode 100644 index 3efd862942..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "slurm_cluster_name" { - type = string - description = "Cluster name" -} - -variable "slurm_bucket_path" { - type = string - description = "GCS Bucket URI of Slurm cluster file storage." -} - - -variable "slurm_bucket_name" { - type = string - description = "Name of the bucket for configs" -} - -variable "slurm_bucket_dir" { - type = string - description = "Path to directory in the bucket for configs" -} - - -variable "universe_domain" { - description = "Domain address for alternate API universe" - type = string - default = "googleapis.com" -} - -variable "login_nodes" { - description = "Slurm login instance definitions." - type = object({ - group_name = string - access_config = optional(list(object({ - nat_ip = string - network_tier = string - }))) - additional_disks = optional(list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string), {}) - auto_delete = optional(bool, true) - boot = optional(bool, false) - disk_resource_manager_tags = optional(map(string), {}) - })), []) - additional_networks = optional(list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })), []) - bandwidth_tier = optional(string, "platform_default") - can_ip_forward = optional(bool, false) - disk_auto_delete = optional(bool, true) - disk_labels = optional(map(string), {}) - disk_resource_manager_tags = optional(map(string), {}) - disk_size_gb = optional(number) - disk_type = optional(string, "n1-standard-1") - enable_confidential_vm = optional(bool, false) - enable_oslogin = optional(bool, true) - enable_shielded_vm = optional(bool, false) - gpu = optional(object({ - count = number - type = string - })) - labels = optional(map(string), {}) - machine_type = optional(string) - advanced_machine_features = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - metadata = optional(map(string), {}) - min_cpu_platform = optional(string) - num_instances = optional(number, 1) - on_host_maintenance = optional(string) - preemptible = optional(bool, false) - region = optional(string) - resource_manager_tags = optional(map(string), {}) - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - shielded_instance_config = optional(object({ - enable_integrity_monitoring = optional(bool, true) - enable_secure_boot = optional(bool, true) - enable_vtpm = optional(bool, true) - })) - source_image_family = optional(string) - source_image_project = optional(string) - source_image = optional(string) - static_ips = optional(list(string), []) - subnetwork = string - spot = optional(bool, false) - tags = optional(list(string), []) - zone = optional(string) - termination_action = optional(string) - }) -} - - -variable "startup_scripts" { - description = "List of scripts to be ran on login VMs startup." - type = list(object({ - filename = string - content = string - })) - default = [] -} - -variable "startup_scripts_timeout" { - description = < - -- [Module: Slurm Nodeset (TPU)](#module-slurm-nodeset-tpu) - - [Overview](#overview) - - [Module API](#module-api) - - - -## Overview - -This is a submodule of [slurm_cluster](../../../slurm_cluster/README.md). It -creates a Slurm TPU nodeset for [slurm_partition](../slurm_partition/README.md). - -## Module API - -For the terraform module API reference, please see -[README_TF.md](./README_TF.md). - - -Copyright (C) SchedMD LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.2 | -| [google](#requirement\_google) | >= 3.53 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.53 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.nodeset_tpu](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_subnetwork.nodeset_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | -| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | -| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `""` | no | -| [enable\_public\_ip](#input\_enable\_public\_ip) | Enables IP address to access the Internet. | `bool` | `false` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of nodes allowed in this partition to be created dynamically. | `number` | `0` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | -| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. Not needed if you use accelerator\_config | `string` | `null` | no | -| [nodeset\_name](#input\_nodeset\_name) | Name of Slurm nodeset. | `string` | n/a | yes | -| [preemptible](#input\_preemptible) | Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details. | `bool` | `false` | no | -| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `true` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the TPU-vm.
If none is given, the default service account and scopes will be used. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | -| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | n/a | yes | -| [zone](#input\_zone) | Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Nodeset details. | -| [nodeset\_name](#output\_nodeset\_name) | Nodeset name. | -| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf deleted file mode 100644 index 1a6a9cfba1..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########### -# NODESET # -########### - -locals { - node_conf_hw = { - Mem334CPU96 = { - CPUs = 96 - Boards = 1 - Sockets = 2 - CoresPerSocket = 24 - ThreadsPerCore = 2 - RealMemory = 307200 - } - Mem400CPU240 = { - CPUs = 240 - Boards = 1 - Sockets = 2 - CoresPerSocket = 60 - ThreadsPerCore = 2 - RealMemory = 400000 - } - } - node_conf_mappings = { - "v2" = local.node_conf_hw.Mem334CPU96 - "v3" = local.node_conf_hw.Mem334CPU96 - "v4" = local.node_conf_hw.Mem400CPU240 - } - simple_nodes = ["v2-8", "v3-8", "v4-8"] -} - -locals { - snetwork = data.google_compute_subnetwork.nodeset_subnetwork.name - region = join("-", slice(split("-", var.zone), 0, 2)) - tpu_fam = var.accelerator_config.version != "" ? lower(var.accelerator_config.version) : split("-", var.node_type)[0] - #If subnetwork is specified and it does not have private_ip_google_access, we need to have public IPs on the TPU - #if no subnetwork is specified, the default one will be used, this does not have private_ip_google_access so we need public IPs too - pub_need = !data.google_compute_subnetwork.nodeset_subnetwork.private_ip_google_access - can_preempt = var.node_type != null ? contains(local.simple_nodes, var.node_type) : false - nodeset_tpu = { - nodeset_name = var.nodeset_name - node_conf = local.node_conf_mappings[local.tpu_fam] - node_type = var.node_type - accelerator_config = var.accelerator_config - tf_version = var.tf_version - preemptible = local.can_preempt ? var.preemptible : false - reserved = var.reserved - node_count_dynamic_max = var.node_count_dynamic_max - node_count_static = var.node_count_static - enable_public_ip = var.enable_public_ip - zone = var.zone - service_account = var.service_account != null ? var.service_account : local.service_account - preserve_tpu = local.can_preempt ? var.preserve_tpu : false - data_disks = var.data_disks - docker_image = var.docker_image != "" ? var.docker_image : "us-docker.pkg.dev/schedmd-slurm-public/tpu/slurm-gcp-6-9:tf-${var.tf_version}" - subnetwork = local.snetwork - network_storage = var.network_storage - } - - service_account = { - email = try(var.service_account.email, null) - scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) - } -} - -data "google_compute_subnetwork" "nodeset_subnetwork" { - name = var.subnetwork - region = local.region - project = var.project_id - - self_link = ( - length(regexall("/projects/([^/]*)", var.subnetwork)) > 0 - && length(regexall("/regions/([^/]*)", var.subnetwork)) > 0 - ? var.subnetwork - : null - ) -} - -resource "null_resource" "nodeset_tpu" { - triggers = { - nodeset = sha256(jsonencode(local.nodeset_tpu)) - } - lifecycle { - precondition { - condition = sum([var.node_count_dynamic_max, var.node_count_static]) > 0 - error_message = "Sum of node_count_dynamic_max and node_count_static must be > 0." - } - precondition { - condition = !(var.preemptible && var.reserved) - error_message = "Nodeset cannot be preemptible and reserved at the same time." - } - precondition { - condition = !(var.subnetwork == null && !var.enable_public_ip) - error_message = "Using the default subnetwork for the TPU nodeset requires enable_public_ip set to true." - } - precondition { - condition = !(var.subnetwork != null && (local.pub_need && !var.enable_public_ip)) - error_message = "The subnetwork specified does not have Private Google Access enabled. This is required when enable_public_ip is set to false." - } - precondition { - condition = !(var.node_type == null && (var.accelerator_config.topology == "" && var.accelerator_config.version == "")) - error_message = "Either a node type or an accelerator_config must be provided." - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf deleted file mode 100644 index fce700d567..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "nodeset_name" { - description = "Nodeset name." - value = local.nodeset_tpu.nodeset_name -} - -output "nodeset" { - description = "Nodeset details." - value = local.nodeset_tpu -} - -output "service_account" { - description = "Service account object, includes email and scopes." - value = local.service_account -} diff --git a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf b/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf deleted file mode 100644 index a8c470dec9..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "nodeset_name" { - description = "Name of Slurm nodeset." - type = string - - validation { - condition = can(regex("^[a-z](?:[a-z0-9]{0,14})$", var.nodeset_name)) - error_message = "Variable 'nodeset_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,14})$'." - } -} - -variable "node_type" { - description = "Specify a node type to base the vm configuration upon it. Not needed if you use accelerator_config" - type = string - default = null -} - -variable "accelerator_config" { - description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." - type = object({ - topology = string - version = string - }) - default = { - topology = "" - version = "" - } - validation { - condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], upper(var.accelerator_config.version)) - error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" - } - validation { - condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) - error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." - } -} - -variable "docker_image" { - description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" - type = string - default = "" -} - -variable "tf_version" { - description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." - type = string -} - -variable "zone" { - description = "Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it." - type = string - - validation { - condition = can(coalesce(var.zone)) - error_message = "Zone cannot be null or empty." - } -} - -variable "preemptible" { - description = "Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details." - type = bool - default = false -} - -variable "reserved" { - description = "Specify whether TPU-vms in this nodeset are created under a reservation." - type = bool - default = false -} - -variable "preserve_tpu" { - description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" - type = bool - default = true -} - -variable "node_count_static" { - description = "Number of nodes to be statically created." - type = number - default = 0 - - validation { - condition = var.node_count_static >= 0 - error_message = "Value must be >= 0." - } -} - -variable "node_count_dynamic_max" { - description = "Maximum number of nodes allowed in this partition to be created dynamically." - type = number - default = 0 - - validation { - condition = var.node_count_dynamic_max >= 0 - error_message = "Value must be >= 0." - } -} - -variable "enable_public_ip" { - description = "Enables IP address to access the Internet." - type = bool - default = false -} - -variable "data_disks" { - type = list(string) - description = "The data disks to include in the TPU node" - default = [] -} - -variable "subnetwork" { - description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." - type = string -} - -variable "service_account" { - type = object({ - email = string - scopes = set(string) - }) - description = < -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | -| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | -| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | -| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | -| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | -| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "v0.7.2")
  })
| `{}` | no | -| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "v0.11.4")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | -| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md deleted file mode 100644 index 1957899617..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [helm](#provider\_helm) | ~> 2.17 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | -| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | -| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | -| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | -| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | -| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | -| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | -| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | -| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | -| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | -| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | -| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | -| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | -| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | -| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | -| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | -| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | -| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | -| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | -| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | -| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | -| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | -| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | -| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | -| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | -| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | -| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | -| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | -| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | -| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf deleted file mode 100644 index bd2383b772..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "helm_release" "apply_chart" { - # Required Identification - name = var.release_name - chart = var.chart_name - - # Chart Source & Version - repository = var.chart_repository - version = var.chart_version - devel = var.devel - - # Target Namespace - namespace = var.namespace - create_namespace = var.create_namespace - - # Values Configuration - values = var.values_yaml - - dynamic "set" { - for_each = var.set_values - content { - name = set.value.name - value = set.value.value - type = set.value.type - } - } - - # Installation/Upgrade Behavior - description = var.description - atomic = var.atomic - cleanup_on_fail = var.cleanup_on_fail - dependency_update = var.dependency_update - disable_crd_hooks = var.disable_crd_hooks - disable_openapi_validation = var.disable_openapi_validation - disable_webhooks = var.disable_webhooks - force_update = var.force_update - lint = var.lint - max_history = var.max_history - recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI - render_subchart_notes = var.render_subchart_notes - reset_values = var.reset_values - reuse_values = var.reuse_values - skip_crds = var.skip_crds - timeout = var.timeout - wait = var.wait - wait_for_jobs = var.wait_for_jobs - - # Verification & Credentials - keyring = var.keyring - pass_credentials = var.pass_credentials - verify = var.verify - - # Post Rendering - dynamic "postrender" { - # Only include the block if var.postrender is not null - for_each = var.postrender == null ? [] : [var.postrender] - content { - binary_path = postrender.value.binary_path - } - } - -} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml deleted file mode 100644 index e18197e2b7..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf deleted file mode 100644 index 04e8e214fc..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -# --- Required --- -variable "release_name" { - description = "Name of the Helm release." - type = string -} - -variable "chart_name" { - description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." - type = string -} - -# --- Chart Location & Version --- -variable "chart_repository" { - description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." - type = string - default = null -} - -variable "chart_version" { - description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." - type = string - default = null -} - -variable "devel" { - description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." - type = bool - default = false -} - -# --- Namespace --- -variable "namespace" { - description = "Kubernetes namespace to install the Helm release into." - type = string - default = "default" -} - -variable "create_namespace" { - description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." - type = bool - default = true # Common convenience setting -} - -# --- Values Customization --- -variable "values_yaml" { - description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." - type = list(string) - default = [] -} - -variable "set_values" { - description = "List of objects defining values to set ('helm install --set')." - type = list(object({ - name = string # Path to the value (e.g., 'service.type', 'replicaCount') - value = string # The value to set - type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') - })) - default = [] -} - -# --- Installation/Upgrade Behavior --- -variable "description" { - description = "Set an optional description for the Helm release." - type = string - default = null -} - -variable "atomic" { - description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." - type = bool - default = false -} - -variable "wait" { - description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." - type = bool - default = true # Often a good default for dependencies -} - -variable "wait_for_jobs" { - description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." - type = bool - default = false # Helm CLI default is false -} - -variable "timeout" { - description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." - type = number - default = 300 # 5 minutes (Helm CLI default) -} - -variable "cleanup_on_fail" { - description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." - type = bool - default = false -} - -variable "dependency_update" { - description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." - type = bool - default = false -} - -variable "disable_crd_hooks" { - description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." - type = bool - default = false -} - -variable "disable_openapi_validation" { - description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." - type = bool - default = false -} - -variable "disable_webhooks" { - description = "Prevent hooks from running ('helm install --no-hooks')." - type = bool - default = false -} - -variable "force_update" { - description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." - type = bool - default = false -} - -variable "lint" { - description = "Run the helm chart linter during the plan ('helm lint')." - type = bool - default = false -} - -variable "max_history" { - description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." - type = number - default = null # Terraform provider defaults to Helm's default (usually 10) -} - -variable "recreate_pods" { - description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." - type = bool - default = false -} - -variable "render_subchart_notes" { - description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." - type = bool - default = false -} - -variable "reset_values" { - description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." - type = bool - default = false -} - -variable "reuse_values" { - description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." - type = bool - default = false # Helm CLI default is false -} - -variable "skip_crds" { - description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." - type = bool - default = false -} - -# --- Verification & Credentials --- -variable "keyring" { - description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." - type = string - default = null # Defaults to Helm's default keyring location -} - -variable "pass_credentials" { - description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." - type = bool - default = false -} - -variable "verify" { - description = "Verify the package before installing it ('helm install --verify')." - type = bool - default = false -} - -# --- Advanced Rendering --- -variable "postrender" { - description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." - type = object({ - binary_path = string # Path to the post-renderer executable - }) - default = null # Disabled by default -} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf deleted file mode 100644 index 09d912e2c9..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md deleted file mode 100644 index 46bfe51a32..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md +++ /dev/null @@ -1,40 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.23 | - -## Providers - -| Name | Version | -|------|---------| -| [kubernetes](#provider\_kubernetes) | ~> 2.23 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubernetes_manifest.apply_manifests](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | -| [field\_manager](#input\_field\_manager) | (Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts. | 
object({
    name            = optional(string, null)
    force_conflicts = optional(bool, false)
  })
| `null` | no | -| [resource\_timeouts](#input\_resource\_timeouts) | (Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met. | 
object({
    create = optional(string, null)
    update = optional(string, null)
    delete = optional(string, null)
  })
| 
{
  "create": "15m",
  "delete": "5m",
  "update": "10m"
}
| no | -| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `""` | no | -| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | -| [wait\_for\_fields](#input\_wait\_for\_fields) | (Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern. | `map(string)` | `{}` | no | -| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf deleted file mode 100644 index f97f26038d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - yaml_separator = "\n---" - - # --- 1. Determine the primary source of YAML content --- - # Prioritize 'content' variable if provided - primary_content_body = var.content != "" ? var.content : null - - # --- 2. Handle 'source_path' based on its type (File vs. Directory) --- - - # Check if source_path is a directory (indicated by trailing slash) - is_directory = endswith(var.source_path, "/") - directory_absolute_path = local.is_directory ? abspath(var.source_path) : null - - # Check if source_path is a single yaml or tftpl file (only if not a directory) - is_single_file = !local.is_directory && ( - length(regexall("\\.yaml$", lower(var.source_path))) > 0 || - length(regexall("\\.tftpl$", lower(var.source_path))) > 0 - ) - single_file_raw_content = local.is_single_file ? ( - length(regexall("\\.tftpl$", lower(var.source_path))) > 0 ? - templatefile(abspath(var.source_path), var.template_vars) : - file(abspath(var.source_path)) - ) : null - - # Docs from primary_content_body - docs_from_primary_source = [ - for doc in split(local.yaml_separator, coalesce(local.primary_content_body, local.single_file_raw_content, "")) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - - # Docs from .yaml files in a directory - directory_yaml_files = local.is_directory ? fileset(local.directory_absolute_path, "*.yaml") : [] - docs_from_directory_yamls = flatten([ - for file_name in local.directory_yaml_files : - [ - for doc in split(local.yaml_separator, file(format("%s/%s", local.directory_absolute_path, file_name))) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - ]) - - # Docs from .tftpl files in a directory - directory_template_files = local.is_directory ? fileset(local.directory_absolute_path, "*.tftpl") : [] - docs_from_directory_templates = flatten([ - for file_name in local.directory_template_files : - [ - for doc in split(local.yaml_separator, templatefile(format("%s/%s", local.directory_absolute_path, file_name), var.template_vars)) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - ]) - - all_parsed_docs = concat( - local.docs_from_primary_source, - local.docs_from_directory_yamls, - local.docs_from_directory_templates - ) - - # --- 5. Create the final map for `for_each` (keys must be unique strings) --- - docs_map = tomap({ - for index, doc in local.all_parsed_docs : index => doc - if length(trimspace(doc)) > 0 - }) -} - -# Apply all manifest files dynamically -resource "kubernetes_manifest" "apply_manifests" { - for_each = local.docs_map - manifest = yamldecode(each.value) - timeouts { - create = var.resource_timeouts.create - update = var.resource_timeouts.update - delete = var.resource_timeouts.delete - } - - dynamic "wait" { - for_each = var.wait_for_rollout ? [1] : [] - content { - rollout = var.wait_for_rollout - fields = var.wait_for_fields - } - } - - # Configure the 'field_manager' block dynamically - dynamic "field_manager" { - for_each = var.field_manager != null ? [var.field_manager] : [] - content { - name = field_manager.value.name - force_conflicts = field_manager.value.force_conflicts - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml deleted file mode 100644 index e18197e2b7..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf deleted file mode 100644 index 0b846189ea..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -variable "content" { - description = "The YAML body to apply to gke cluster." - type = string - default = null -} - -variable "source_path" { - description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." - type = string - default = "" -} - -variable "template_vars" { - description = "The values to populate template file(s) with." - type = any - default = null -} - -variable "wait_for_rollout" { - description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." - type = bool - default = true -} - - -variable "wait_for_fields" { - description = "(Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern." - type = map(string) - default = {} -} - -variable "resource_timeouts" { - description = "(Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met." - type = object({ - create = optional(string, null) - update = optional(string, null) - delete = optional(string, null) - }) - default = { - create = "15m" # Default create timeout, also covers waiting for initial conditions - update = "10m" # Default update timeout, also covers waiting for update conditions - delete = "5m" # Default delete timeout - } -} - -variable "field_manager" { - description = "(Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts." - type = object({ - name = optional(string, null) - force_conflicts = optional(bool, false) - }) - default = null -} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf deleted file mode 100644 index 61786b06de..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - # Defines the providers that this module depends on and their versions. - required_providers { - kubernetes = { - source = "hashicorp/kubernetes" - version = "~> 2.23" - } - } - required_version = ">= 1.3" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf b/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf deleted file mode 100644 index 8db4870452..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] - project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] - - install_gpu_operator = try(var.gpu_operator.install, false) - install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) -} - -data "google_container_cluster" "gke_cluster" { - project = local.project_id - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -module "install_kueue" { - source = "./helm_install" - depends_on = [var.gke_cluster_exists] - - release_name = "kueue" - - chart_name = "oci://registry.k8s.io/kueue/charts/kueue" - chart_version = var.kueue.version # Specify your desired Kueue version - - create_namespace = true # Helm can also create the namespace - wait = true - timeout = 600 # seconds -} - -module "install_jobset" { - source = "./helm_install" - depends_on = [var.gke_cluster_exists, module.install_kueue] - release_name = "jobset-controller" # The release name for your JobSet installation - chart_name = "oci://registry.k8s.io/jobset/charts/jobset" # The Helm repository URL for nvidia charts - chart_version = var.jobset.version - create_namespace = true - namespace = "jobset-system" -} - -module "install_nvidia_dra_driver" { - count = local.install_nvidia_dra_driver ? 1 : 0 - depends_on = [var.gke_cluster_exists] - source = "./helm_install" - - release_name = "nvidia-dra-driver-gpu" # The release name - chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts - chart_name = "nvidia-dra-driver-gpu" # The chart name - chart_version = var.nvidia_dra_driver.version # The chart version - namespace = "nvidia-dra-driver-gpu" # The target namespace - create_namespace = true # Equivalent to --create-namespace - - # Use the 'values' argument to pass the YAML content - # This corresponds to the -f <(cat < -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.2 | -| [google](#requirement\_google) | >= 6.40 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.40 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_global_address.private_ip_alloc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_global_address) | resource | -| [google_compute_network_peering_routes_config.private_vpc_peering_routes_gcnv](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering_routes_config) | resource | -| [google_service_networking_connection.private_vpc_connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_networking_connection) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [address](#input\_address) | The IP address or beginning of the address range allocated for the Private Service Access. | `string` | `null` | no | -| [deletion\_policy](#input\_deletion\_policy) | The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to supporting resources. Key-value pairs. | `map(string)` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to configure Private Service Access:
`projects//global/networks/`" | `string` | n/a | yes | -| [prefix\_length](#input\_prefix\_length) | The prefix length of the IP range allocated for the Private Service Access. | `number` | `16` | no | -| [project\_id](#input\_project\_id) | ID of project in which Private Service Access will be created. | `string` | n/a | yes | -| [service\_name](#input\_service\_name) | The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'. | `string` | `"servicenetworking.googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cidr\_range](#output\_cidr\_range) | CIDR range of the created google\_compute\_global\_address | -| [connect\_mode](#output\_connect\_mode) | Services that use Private Service Access typically specify connect\_mode
"PRIVATE\_SERVICE\_ACCESS". This output value sets connect\_mode and additionally
blocks terraform actions until the VPC connection has been created. | -| [private\_vpc\_connection\_peering](#output\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection that was created by the service provider. | -| [reserved\_ip\_range](#output\_reserved\_ip\_range) | Named IP range to be used by services connected with Private Service Access. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/main.tf b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/main.tf deleted file mode 100644 index 429e4d93f0..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/main.tf +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "private-service-access", ghpc_role = "network" }) -} - -locals { - split_network_id = split("/", var.network_id) - network_name = local.split_network_id[4] - network_project = local.split_network_id[1] -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_compute_global_address" "private_ip_alloc" { - provider = google - name = "global-psconnect-ip-${random_id.resource_name_suffix.hex}" - project = var.project_id - purpose = "VPC_PEERING" - address_type = "INTERNAL" - network = var.network_id - prefix_length = var.prefix_length - labels = local.labels - address = var.address -} - -resource "google_service_networking_connection" "private_vpc_connection" { - network = var.network_id - service = var.service_name - reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name] - deletion_policy = var.deletion_policy - update_on_creation_fail = var.deletion_policy == "ABANDON" ? true : null -} - -# Google Cloud NetApp Volumes need enablement of custom_route import and export -resource "google_compute_network_peering_routes_config" "private_vpc_peering_routes_gcnv" { - count = var.service_name == "netapp.servicenetworking.goog" ? 1 : 0 - project = local.network_project - network = local.network_name - peering = google_service_networking_connection.private_vpc_connection.peering - - export_custom_routes = true - import_custom_routes = true -} diff --git a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml deleted file mode 100644 index 93e8b3970e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - servicenetworking.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf deleted file mode 100644 index 296f2e9140..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "private_vpc_connection_peering" { - description = "The name of the VPC Network peering connection that was created by the service provider." - sensitive = true - value = google_service_networking_connection.private_vpc_connection.peering -} - -output "connect_mode" { - description = <<-EOT - Services that use Private Service Access typically specify connect_mode - "PRIVATE_SERVICE_ACCESS". This output value sets connect_mode and additionally - blocks terraform actions until the VPC connection has been created. - EOT - value = "PRIVATE_SERVICE_ACCESS" - depends_on = [ - google_service_networking_connection.private_vpc_connection, - ] -} - -output "reserved_ip_range" { - description = "Named IP range to be used by services connected with Private Service Access." - value = google_compute_global_address.private_ip_alloc.name -} - -output "cidr_range" { - description = "CIDR range of the created google_compute_global_address" - value = "${google_compute_global_address.private_ip_alloc.address}/${google_compute_global_address.private_ip_alloc.prefix_length}" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/variables.tf b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/variables.tf deleted file mode 100644 index 4b0a3e796f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/variables.tf +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "address" { - description = "The IP address or beginning of the address range allocated for the Private Service Access." - type = string - default = null -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to configure Private Service Access: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "labels" { - description = "Labels to add to supporting resources. Key-value pairs." - type = map(string) -} - -variable "prefix_length" { - description = "The prefix length of the IP range allocated for the Private Service Access." - type = number - default = 16 -} - -variable "project_id" { - description = "ID of project in which Private Service Access will be created." - type = string -} - -variable "service_name" { - description = "The name of the service to connect. Defaults to 'servicenetworking.googleapis.com'." - type = string - default = "servicenetworking.googleapis.com" -} - -variable "deletion_policy" { - description = "The policy to apply when deleting the Private Service Access. Leave empty or use ABANDON." - type = string - default = null -} diff --git a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/versions.tf b/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/versions.tf deleted file mode 100644 index df2914cdb9..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/network/private-service-access/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.40" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" - } - - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.74.0" - } - - required_version = ">= 1.2" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/new-project/README.md b/deletion-test/primary/modules/embedded/community/modules/project/new-project/README.md deleted file mode 100644 index 5e5cabe9d5..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/new-project/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module allows you to create opinionated Google Cloud Platform projects. It -creates projects and configures aspects like Shared VPC connectivity, IAM -access, Service Accounts, and API enablement to follow best practices. - -This module is meant for use with Terraform 0.13. - -**Note:** This module has been removed from the Cluster Toolkit. The upstream module (`terraform-google-project-factory`) is now the recommended way to create and manage GCP projects. - -### Example - -```yaml -- id: project - source: github.com/terraform-google-modules/terraform-google-project-factory?rev=v17.0.0&depth=1 -``` - -This creates a new project with pre-defined project ID, a designated folder and -organization and associated billing account which will be used to pay for -services consumed. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [project\_factory](#module\_project\_factory) | terraform-google-modules/project-factory/google | ~> 11.3 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [activate\_api\_identities](#input\_activate\_api\_identities) | The list of service identities (Google Managed service account for the API) to force-create for the project (e.g. in order to grant additional roles).
APIs in this list will automatically be appended to `activate_apis`.
Not including the API in this list will follow the default behaviour for identity creation (which is usually when the first resource using the API is created).
Any roles (e.g. service agent role) must be explicitly listed. See https://cloud.google.com/iam/docs/understanding-roles#service-agent-roles-roles for a list of related roles. | 
list(object({
    api   = string
    roles = list(string)
  }))
| `[]` | no | -| [activate\_apis](#input\_activate\_apis) | The list of apis to activate within the project | `list(string)` | 
[
  "compute.googleapis.com",
  "serviceusage.googleapis.com",
  "storage.googleapis.com"
]
| no | -| [auto\_create\_network](#input\_auto\_create\_network) | Create the default network | `bool` | `false` | no | -| [billing\_account](#input\_billing\_account) | The ID of the billing account to associate this project with | `string` | n/a | yes | -| [bucket\_force\_destroy](#input\_bucket\_force\_destroy) | Force the deletion of all objects within the GCS bucket when deleting the bucket (optional) | `bool` | `false` | no | -| [bucket\_labels](#input\_bucket\_labels) | A map of key/value label pairs to assign to the bucket (optional) | `map(string)` | `{}` | no | -| [bucket\_location](#input\_bucket\_location) | The location for a GCS bucket to create (optional) | `string` | `"US"` | no | -| [bucket\_name](#input\_bucket\_name) | A name for a GCS bucket to create (in the bucket\_project project), useful for Terraform state (optional) | `string` | `""` | no | -| [bucket\_project](#input\_bucket\_project) | A project to create a GCS bucket (bucket\_name) in, useful for Terraform state (optional) | `string` | `""` | no | -| [bucket\_ula](#input\_bucket\_ula) | Enable Uniform Bucket Level Access | `bool` | `true` | no | -| [bucket\_versioning](#input\_bucket\_versioning) | Enable versioning for a GCS bucket to create (optional) | `bool` | `false` | no | -| [budget\_alert\_pubsub\_topic](#input\_budget\_alert\_pubsub\_topic) | The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}` | `string` | `null` | no | -| [budget\_alert\_spent\_percents](#input\_budget\_alert\_spent\_percents) | A list of percentages of the budget to alert on when threshold is exceeded | `list(number)` | 
[
  0.5,
  0.7,
  1
]
| no | -| [budget\_amount](#input\_budget\_amount) | The amount to use for a budget alert | `number` | `null` | no | -| [budget\_display\_name](#input\_budget\_display\_name) | The display name of the budget. If not set defaults to `Budget For ` | `string` | `null` | no | -| [budget\_monitoring\_notification\_channels](#input\_budget\_monitoring\_notification\_channels) | A list of monitoring notification channels in the form `[projects/{project_id}/notificationChannels/{channel_id}]`. A maximum of 5 channels are allowed. | `list(string)` | `[]` | no | -| [consumer\_quotas](#input\_consumer\_quotas) | The quotas configuration you want to override for the project. | 
list(object({
    service = string,
    metric  = string,
    limit   = string,
    value   = string,
  }))
| `[]` | no | -| [create\_project\_sa](#input\_create\_project\_sa) | Whether the default service account for the project shall be created | `bool` | `true` | no | -| [default\_network\_tier](#input\_default\_network\_tier) | Default Network Service Tier for resources created in this project. If unset, the value will not be modified. See https://cloud.google.com/network-tiers/docs/using-network-service-tiers and https://cloud.google.com/network-tiers. | `string` | `""` | no | -| [default\_service\_account](#input\_default\_service\_account) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | `string` | `"keep"` | no | -| [disable\_dependent\_services](#input\_disable\_dependent\_services) | Whether services that are enabled and which depend on this service should also be disabled when this service is destroyed. | `bool` | `true` | no | -| [disable\_services\_on\_destroy](#input\_disable\_services\_on\_destroy) | Whether project services will be disabled when the resources are destroyed | `bool` | `true` | no | -| [domain](#input\_domain) | The domain name (optional). | `string` | `""` | no | -| [enable\_shared\_vpc\_host\_project](#input\_enable\_shared\_vpc\_host\_project) | If this project is a shared VPC host project. If true, you must *not* set svpc\_host\_project\_id variable. Default is false. | `bool` | `false` | no | -| [folder\_id](#input\_folder\_id) | The ID of a folder to host this project | `string` | `""` | no | -| [grant\_services\_network\_role](#input\_grant\_services\_network\_role) | Whether or not to grant service agents the network roles on the host project | `bool` | `true` | no | -| [grant\_services\_security\_admin\_role](#input\_grant\_services\_security\_admin\_role) | Whether or not to grant Kubernetes Engine Service Agent the Security Admin role on the host project so it can manage firewall rules | `bool` | `false` | no | -| [group\_name](#input\_group\_name) | A group to control the project by being assigned group\_role (defaults to project editor) | `string` | `""` | no | -| [group\_role](#input\_group\_role) | The role to give the controlling group (group\_name) over the project (defaults to project editor) | `string` | `"roles/editor"` | no | -| [labels](#input\_labels) | Map of labels for project | `map(string)` | `{}` | no | -| [lien](#input\_lien) | Add a lien on the project to prevent accidental deletion | `bool` | `false` | no | -| [name](#input\_name) | The name for the project | `string` | `null` | no | -| [org\_id](#input\_org\_id) | The organization ID. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The ID to give the project. If not provided, the `name` will be used. | `string` | `""` | no | -| [project\_sa\_name](#input\_project\_sa\_name) | Default service account name for the project. | `string` | `"project-service-account"` | no | -| [random\_project\_id](#input\_random\_project\_id) | Adds a suffix of 4 random characters to the `project_id` | `bool` | `false` | no | -| [sa\_role](#input\_sa\_role) | A role to give the default Service Account for the project (defaults to none) | `string` | `""` | no | -| [shared\_vpc\_subnets](#input\_shared\_vpc\_subnets) | List of subnets fully qualified subnet IDs (ie. projects/$project\_id/regions/$region/subnetworks/$subnet\_id) | `list(string)` | `[]` | no | -| [svpc\_host\_project\_id](#input\_svpc\_host\_project\_id) | The ID of the host project which hosts the shared VPC | `string` | `""` | no | -| [usage\_bucket\_name](#input\_usage\_bucket\_name) | Name of a GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | -| [usage\_bucket\_prefix](#input\_usage\_bucket\_prefix) | Prefix in the GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | -| [vpc\_service\_control\_attach\_enabled](#input\_vpc\_service\_control\_attach\_enabled) | Whether the project will be attached to a VPC Service Control Perimeter | `bool` | `false` | no | -| [vpc\_service\_control\_perimeter\_name](#input\_vpc\_service\_control\_perimeter\_name) | The name of a VPC Service Control Perimeter to add the created project to | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [api\_s\_account](#output\_api\_s\_account) | API service account email | -| [api\_s\_account\_fmt](#output\_api\_s\_account\_fmt) | API service account email formatted for terraform use | -| [budget\_name](#output\_budget\_name) | The name of the budget if created | -| [domain](#output\_domain) | The organization's domain | -| [enabled\_api\_identities](#output\_enabled\_api\_identities) | Enabled API identities in the project | -| [enabled\_apis](#output\_enabled\_apis) | Enabled APIs in the project | -| [group\_email](#output\_group\_email) | The email of the G Suite group with group\_name | -| [project\_bucket\_self\_link](#output\_project\_bucket\_self\_link) | Project's bucket selfLink | -| [project\_bucket\_url](#output\_project\_bucket\_url) | Project's bucket url | -| [project\_id](#output\_project\_id) | ID of the project that was created | -| [project\_name](#output\_project\_name) | Name of the project that was created | -| [project\_number](#output\_project\_number) | Number of the project that was created | -| [service\_account\_display\_name](#output\_service\_account\_display\_name) | The display name of the default service account | -| [service\_account\_email](#output\_service\_account\_email) | The email of the default service account | -| [service\_account\_id](#output\_service\_account\_id) | The id of the default service account | -| [service\_account\_name](#output\_service\_account\_name) | The fully-qualified name of the default service account | -| [service\_account\_unique\_id](#output\_service\_account\_unique\_id) | The unique id of the default service account | - diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/README.md b/deletion-test/primary/modules/embedded/community/modules/project/service-account/README.md deleted file mode 100644 index 0f5c10c7e4..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/service-account/README.md +++ /dev/null @@ -1,111 +0,0 @@ -## Description - -Allows creation of service accounts for a Google Cloud Platform project. - -### Example - -```yaml -- id: service_acct - source: community/modules/project/service-account - settings: - project_id: $(vars.project_id) - name: instance_acct - project_roles: - - logging.logWriter - - monitoring.metricWriter - - storage.objectViewer -``` - -This creates a service account in GCP project "project_id" with the name -"instance_acct". It will have the 3 roles listed for all resources within the -project. - -### Usage with startup-script module - -When this module is used in conjunction with the [startup-script] module, the -service account must be granted (at least) read access to the bucket. This can -be achieved by granting project-wide access as shown above or by specifying the -service account as a bucket viewer in the startup-script module: - -```yaml -- id: service_acct - source: community/modules/project/service-account - settings: - project_id: $(vars.project_id) - name: instance_acct - project_roles: - - logging.logWriter - - monitoring.metricWriter -- id: script - source: modules/scripts/startup-script - settings: - bucket_viewers: - - $(service_acct.service_account_iam_email) -``` - -[startup-script]: ../../../../modules/scripts/startup-script/README.md - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [service\_account](#module\_service\_account) | terraform-google-modules/service-accounts/google | ~> 4.2 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [billing\_account\_id](#input\_billing\_account\_id) | If assigning billing role, specify a billing account (default is to assign at the organizational level). | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment (will be prepended to service account name) | `string` | n/a | yes | -| [description](#input\_description) | Description of the created service account. | `string` | `"Service Account"` | no | -| [descriptions](#input\_descriptions) | Deprecated; create single service accounts using var.description. | `list(string)` | `null` | no | -| [display\_name](#input\_display\_name) | Display name of the created service account. | `string` | `"Service Account"` | no | -| [generate\_keys](#input\_generate\_keys) | Generate keys for service account. | `bool` | `false` | no | -| [grant\_billing\_role](#input\_grant\_billing\_role) | Grant billing user role. | `bool` | `false` | no | -| [grant\_xpn\_roles](#input\_grant\_xpn\_roles) | Grant roles for shared VPC management. | `bool` | `true` | no | -| [name](#input\_name) | Name of the service account to create. | `string` | n/a | yes | -| [names](#input\_names) | Deprecated; create single service accounts using var.name. | `list(string)` | `null` | no | -| [org\_id](#input\_org\_id) | Id of the organization for org-level roles. | `string` | `""` | no | -| [prefix](#input\_prefix) | Deprecated; prefix now set using var.deployment\_name | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | -| [project\_roles](#input\_project\_roles) | List of roles to grant to service account (e.g. "storage.objectViewer" or "compute.instanceAdmin.v1" | `list(string)` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [key](#output\_key) | Service account key (if creation was requested) | -| [service\_account\_email](#output\_service\_account\_email) | Service account e-mail address | -| [service\_account\_iam\_email](#output\_service\_account\_iam\_email) | Service account IAM binding format (serviceAccount:name@example.com) | - diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/main.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-account/main.tf deleted file mode 100644 index e8a69be642..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/service-account/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - display_name = "${var.display_name} (${var.deployment_name})" - description = "${var.description} (${var.deployment_name})" -} - -module "service_account" { - source = "terraform-google-modules/service-accounts/google" - version = "~> 4.2" - - billing_account_id = var.billing_account_id - description = local.description - display_name = local.display_name - generate_keys = var.generate_keys - grant_billing_role = var.grant_billing_role - grant_xpn_roles = var.grant_xpn_roles - names = [var.name] - org_id = var.org_id - prefix = var.deployment_name - project_id = var.project_id - project_roles = [for role in var.project_roles : "${var.project_id}=>roles/${role}"] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/project/service-account/metadata.yaml deleted file mode 100644 index c4dcdffdf4..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/service-account/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-account/outputs.tf deleted file mode 100644 index f9c9be05c8..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/service-account/outputs.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "key" { - description = "Service account key (if creation was requested)" - value = module.service_account.key -} - -output "service_account_email" { - description = "Service account e-mail address" - value = module.service_account.email - depends_on = [ - module.service_account, - ] -} - -output "service_account_iam_email" { - description = "Service account IAM binding format (serviceAccount:name@example.com)" - value = module.service_account.iam_email - depends_on = [ - module.service_account, - ] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/variables.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-account/variables.tf deleted file mode 100644 index 53267f47e7..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/service-account/variables.tf +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "billing_account_id" { - description = "If assigning billing role, specify a billing account (default is to assign at the organizational level)." - type = string - default = "" -} - -variable "deployment_name" { - description = "Name of the deployment (will be prepended to service account name)" - type = string -} - -variable "description" { - description = "Description of the created service account." - type = string - default = "Service Account" -} - -# tflint-ignore: terraform_unused_declarations -variable "descriptions" { - description = "Deprecated; create single service accounts using var.description." - type = list(string) - default = null - - validation { - condition = var.descriptions == null - error_message = "var.descriptions has been deprecated in favor of creating single accounts with var.description" - } -} - -variable "display_name" { - description = "Display name of the created service account." - type = string - default = "Service Account" -} - -variable "generate_keys" { - description = "Generate keys for service account." - type = bool - default = false -} - -variable "grant_billing_role" { - description = "Grant billing user role." - type = bool - default = false -} - -variable "grant_xpn_roles" { - description = "Grant roles for shared VPC management." - type = bool - default = true -} - -variable "name" { - description = "Name of the service account to create." - type = string -} - -# tflint-ignore: terraform_unused_declarations -variable "names" { - description = "Deprecated; create single service accounts using var.name." - type = list(string) - default = null - - validation { - condition = var.names == null - error_message = "var.names has been deprecated in favor of creating single accounts with var.name" - } -} - -variable "org_id" { - description = "Id of the organization for org-level roles." - type = string - default = "" -} - -# tflint-ignore: terraform_unused_declarations -variable "prefix" { - description = "Deprecated; prefix now set using var.deployment_name" - type = string - default = null - - validation { - condition = var.prefix == null - error_message = "var.prefix has been deprecated in favor of setting prefix with var.deployment_name" - } -} - -variable "project_id" { - description = "ID of the project" - type = string -} - -variable "project_roles" { - description = "List of roles to grant to service account (e.g. \"storage.objectViewer\" or \"compute.instanceAdmin.v1\"" - type = list(string) -} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-account/versions.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-account/versions.tf deleted file mode 100644 index 38e6e71945..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/service-account/versions.tf +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/README.md b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/README.md deleted file mode 100644 index 266eac26ec..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/README.md +++ /dev/null @@ -1,70 +0,0 @@ -## Description - -Allows management of multiple API services for a Google Cloud Platform project. - -### Example - -```yaml -- id: services-api - source: community/modules/project/service-enablement - settings: - gcp_service_list: [ - "file.googleapis.com", - "compute.googleapis.com" - ] -``` - -This allows the project to enable both the filestore API as well as the compute API. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_project_service.gcp_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [disable\_on\_destroy](#input\_disable\_on\_destroy) | Disable services on destroy if they were enabled (or already enabled) during apply (default: false) | `bool` | `false` | no | -| [gcp\_service\_list](#input\_gcp\_service\_list) | list of APIs to be enabled for the project | `list(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/main.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/main.tf deleted file mode 100644 index 965e93c549..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/main.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -resource "google_project_service" "gcp_services" { - count = length(var.gcp_service_list) - project = var.project_id - service = var.gcp_service_list[count.index] - timeouts { - create = "30m" - update = "40m" - } - - disable_dependent_services = true - disable_on_destroy = var.disable_on_destroy -} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml deleted file mode 100644 index c594c8f819..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - serviceusage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/variables.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/variables.tf deleted file mode 100644 index 08f13999fe..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/variables.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "ID of the project" - type = string -} - -variable "gcp_service_list" { - description = "list of APIs to be enabled for the project" - type = list(string) -} - -variable "disable_on_destroy" { - description = "Disable services on destroy if they were enabled (or already enabled) during apply (default: false)" - type = bool - default = false -} diff --git a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/versions.tf b/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/versions.tf deleted file mode 100644 index 07f25fb045..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/project/service-enablement/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md deleted file mode 100644 index 052e6aee23..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Description - -This module creates a Bigquery Pub/Sub Subscription. - -Primarily used for FSI - MonteCarlo Tutorial: -**[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Example - -The following example creates a Bigquery subscription using a Bigquery table and -Pub/Sub topic. - -```yaml - - id: bq_subscription - source: community/modules/pubsub/bigquery-sub - use: [bq-table, pubsub_topic] -``` - -Also see usages in this -[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_project_iam_member.editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_iam_member.viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_pubsub_subscription.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Name of the dataset that was created. Can be provided by the bigquery-table module | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [subscription\_id](#input\_subscription\_id) | The name of the pubsub subscription to be created | `string` | `null` | no | -| [table\_id](#input\_table\_id) | ID of created BQ table. Can be provided by the bigquery-table module | `string` | n/a | yes | -| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [subscription\_id](#output\_subscription\_id) | Name of the subscription that was created. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf deleted file mode 100644 index 8edbc6b24e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-sub", ghpc_role = "pubsub" }) -} - -locals { - subscription_id = var.subscription_id != null ? var.subscription_id : "${var.deployment_name}_subscription_${random_id.resource_name_suffix.hex}" -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} -data "google_project" "project" { - project_id = var.project_id -} - -resource "google_project_iam_member" "viewer" { - project = data.google_project.project.project_id - role = "roles/bigquery.metadataViewer" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" -} - -resource "google_project_iam_member" "editor" { - project = data.google_project.project.project_id - role = "roles/bigquery.dataEditor" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" -} - -resource "google_pubsub_subscription" "example" { - depends_on = [google_project_iam_member.editor, google_project_iam_member.viewer] - name = local.subscription_id - topic = var.topic_id - project = var.project_id - labels = local.labels - bigquery_config { - table = "${var.project_id}.${var.dataset_id}.${var.table_id}" - use_topic_schema = true - write_metadata = true - } - -} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml deleted file mode 100644 index 9aedef48dc..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - pubsub.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf deleted file mode 100644 index fc81859503..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "subscription_id" { - description = "Name of the subscription that was created." - value = google_pubsub_subscription.example.name -} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf deleted file mode 100644 index ee4dbbed8e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "topic_id" { - description = "The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module" - type = string -} - -variable "subscription_id" { - description = "The name of the pubsub subscription to be created" - type = string - default = null -} - -variable "dataset_id" { - description = "Name of the dataset that was created. Can be provided by the bigquery-table module" - type = string -} - -variable "table_id" { - description = "ID of created BQ table. Can be provided by the bigquery-table module" - type = string -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf deleted file mode 100644 index 46ad6e17c8..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.74.0" - } - required_version = ">= 1.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/README.md b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/README.md deleted file mode 100644 index 177f799dc6..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/README.md +++ /dev/null @@ -1,82 +0,0 @@ -## Description - -Creates a Pub/Sub topic - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -### Example - -The following example creates a Pub/Sub topic. - -```yaml - - id: pubsub_topic - source: community/modules/pubsub/topic -``` - -Also see usages in this -[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_pubsub_schema.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_schema) | resource | -| [google_pubsub_topic.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [schema\_id](#input\_schema\_id) | The name of the pubsub schema to be created | `string` | `null` | no | -| [schema\_json](#input\_schema\_json) | The JSON definition of the pubsub topic schema | `string` | `"{ \n \"name\" : \"Avro\", \n \"type\" : \"record\", \n \"fields\" : \n [\n {\"name\" : \"ticker\", \"type\" : \"string\"},\n {\"name\" : \"epoch_time\", \"type\" : \"int\"},\n {\"name\" : \"iteration\", \"type\" : \"int\"},\n {\"name\" : \"start_date\", \"type\" : \"string\"},\n {\"name\" : \"end_date\", \"type\" : \"string\"},\n {\n \"name\":\"simulation_results\",\n \"type\":{\n \"type\": \"array\", \n \"items\":{\n \"name\":\"Child\",\n \"type\":\"record\",\n \"fields\":[\n {\"name\":\"price\", \"type\":\"double\"}\n ]\n }\n }\n }\n ]\n }\n"` | no | -| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to be created | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [topic\_id](#output\_topic\_id) | Name of the topic that was created. | -| [topic\_schema](#output\_topic\_schema) | Name of the topic schema that was created. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/main.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/main.tf deleted file mode 100644 index 4ba68fb5d0..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/main.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "topic", ghpc_role = "pubsub" }) -} - -locals { - topic_id = var.topic_id != null ? var.topic_id : "${var.deployment_name}_topic_${random_id.resource_name_suffix.hex}" - schema_id = var.schema_id != null ? var.schema_id : "${var.deployment_name}_schema_${random_id.resource_name_suffix.hex}" -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_pubsub_topic" "example" { - name = local.topic_id - depends_on = [google_pubsub_schema.example] - project = var.project_id - labels = local.labels - schema_settings { - schema = "projects/${var.project_id}/schemas/${local.schema_id}" - encoding = "BINARY" - } -} - -resource "google_pubsub_schema" "example" { - name = local.schema_id - project = var.project_id - type = "AVRO" - - definition = var.schema_json -} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml deleted file mode 100644 index 9aedef48dc..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - pubsub.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf deleted file mode 100644 index 3ea9d951b2..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "topic_id" { - description = "Name of the topic that was created." - value = google_pubsub_topic.example.name -} - - -output "topic_schema" { - description = "Name of the topic schema that was created." - value = local.schema_id -} diff --git a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/variables.tf b/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/variables.tf deleted file mode 100644 index dca575d21d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/pubsub/topic/variables.tf +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "topic_id" { - description = "The name of the pubsub topic to be created" - type = string - default = null -} - -variable "schema_id" { - description = "The name of the pubsub schema to be created" - type = string - default = null -} - -variable "schema_json" { - description = "The JSON definition of the pubsub topic schema" - type = string - default = < **Note**: This is an experimental module. This module has only been tested in -> limited capacity with the Cluster Toolkit. The module interface may have undergo -> breaking changes in the future. - -### Example - -The following example will create a single GPU accelerated remote desktop. - -```yaml - - id: remote-desktop - source: community/modules/remote-desktop/chrome-remote-desktop - use: [network1] - settings: - install_nvidia_driver: true -``` - -### Setting up the Remote Desktop - -1. Once the remote desktop has been deployed, navigate to https://remotedesktop.google.com/headless. -1. Click through `Begin`, `Next`, & `Authorize`. -1. Copy the code snippet for `Debian Linux`. -1. SSH into the remote desktop machine. It will be listed under - [VM Instances](https://console.cloud.google.com/compute/instances) in the - Google Cloud web console. -1. Run the copied command and follow instructions to set up a PIN. -1. You should now see your machine listed on the - [Chrome Remote Desktop page](https://remotedesktop.google.com/access) under `Remote devices`. -1. Click on your machine and enter PIN if prompted. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.12.31 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [client\_startup\_script](#module\_client\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | -| [instances](#module\_instances) | ../../../../modules/compute/vm-instance | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | -| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image th
at supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-v
m-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. Cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | -| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-balanced"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs on the internet. | `bool` | `true` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required | 
list(object({
    type  = string,
    count = number
  }))
| 
[
  {
    "count": 1,
    "type": "nvidia-tesla-t4-vws"
  }
]
| no | -| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Installs the nvidia driver (true/false). For details, see https://cloud.google.com/compute/docs/gpus/install-drivers-gpu | `bool` | n/a | yes | -| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | -| [instance\_image](#input\_instance\_image) | Image used to build chrome remote desktop node. The default image is
name="debian-12-bookworm-v20250610" and project="debian-cloud".
NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues.

An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud".

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "name": "debian-12-bookworm-v20250610",
  "project": "debian-cloud"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation. Must be N1 family if GPU is used. | `string` | `"n1-standard-8"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface
**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.
Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "RDMA", "IRDMA", "MRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `"TERMINATE"` | no | -| [project\_id](#input\_project\_id) | Project in which Google Cloud resources will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/cloud-platform"
  ]
}
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core | `number` | `2` | no | -| [zone](#input\_zone) | Default zone for creating resources | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_name](#output\_instance\_name) | Name of the first instance created, if any. | -| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf deleted file mode 100644 index a5cf7c5d37..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "chrome-remote-desktop", ghpc_role = "remote-desktop" }) -} - -locals { - - user_startup_script_runners = var.startup_script == null ? [] : [ - { - type = "shell" - content = var.startup_script - destination = "user_startup_script.sh" - } - ] - - configure_nvidia_driver_runners = var.install_nvidia_driver == false ? [] : [ - { - type = "ansible-local" - content = file("${path.module}/scripts/configure-grid-drivers.yml") - destination = "/usr/local/ghpc/configure-grid-drivers.yml" - } - ] - - configure_chrome_remote_desktop_runners = [ - { - type = "ansible-local" - content = file("${path.module}/scripts/configure-chrome-desktop.yml") - destination = "/usr/local/ghpc/configure-chrome-desktop.yml" - } - ] - - disable_sleep = [ - { - type = "ansible-local" - content = file("${path.module}/scripts/disable-sleep.yml") - destination = "/usr/local/ghpc/disable-sleep.yml" - } - ] -} - -module "client_startup_script" { - source = "../../../../modules/scripts/startup-script" - - deployment_name = var.deployment_name - project_id = var.project_id - region = var.region - labels = local.labels - - runners = flatten([ - local.user_startup_script_runners, - local.configure_nvidia_driver_runners, - local.configure_chrome_remote_desktop_runners, - local.disable_sleep - ]) -} - -module "instances" { - source = "../../../../modules/compute/vm-instance" - - instance_count = var.instance_count - name_prefix = var.name_prefix - add_deployment_name_before_prefix = var.add_deployment_name_before_prefix - provisioning_model = var.spot ? "SPOT" : null - - deployment_name = var.deployment_name - project_id = var.project_id - region = var.region - zone = var.zone - labels = local.labels - - machine_type = var.machine_type - service_account_email = var.service_account.email - metadata = var.metadata - startup_script = module.client_startup_script.startup_script - enable_oslogin = var.enable_oslogin - - instance_image = var.instance_image - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - auto_delete_boot_disk = var.auto_delete_boot_disk - - disable_public_ips = !var.enable_public_ips - network_self_link = var.network_self_link - subnetwork_self_link = var.subnetwork_self_link - network_interfaces = var.network_interfaces - bandwidth_tier = var.bandwidth_tier - tags = var.tags - - threads_per_core = var.threads_per_core - guest_accelerator = var.guest_accelerator - on_host_maintenance = var.on_host_maintenance - - network_storage = var.network_storage - -} diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf deleted file mode 100644 index bcf8ece52d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "script to load and run all runners, as a string value." - value = module.client_startup_script.startup_script -} - -output "instance_name" { - description = "Name of the first instance created, if any." - value = var.instance_count > 0 ? module.instances.name[0] : null -} diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml deleted file mode 100644 index 391aa86433..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Ensure Desktop OS and Chrome Remote Desktop is installed - hosts: localhost - become: true - module_defaults: - ansible.builtin.apt: - update_cache: true - cache_valid_time: 3600 - tasks: - - name: Install desktop packages - ansible.builtin.apt: - name: - - xfce4 - - xfce4-goodies - state: present - register: apt_result - retries: 10 - delay: 30 - until: apt_result is success - - - name: Download and configure CRD - ansible.builtin.get_url: - url: https://dl.google.com/linux/direct/chrome-remote-desktop_current_amd64.deb - dest: /tmp/chrome-remote-desktop_current_amd64.deb - mode: "0755" - timeout: 30 - - - name: Install CRD - ansible.builtin.apt: - deb: /tmp/chrome-remote-desktop_current_amd64.deb - environment: - DEBIAN_FRONTEND: noninteractive - register: apt_result - retries: 10 - delay: 30 - until: apt_result is success - - - name: Configure CRD to use Xfce by default - ansible.builtin.copy: - dest: /etc/chrome-remote-desktop-session - content: "exec /etc/X11/Xsession /usr/bin/xfce4-session" - mode: 0644 - - - name: Start Chrome remote desktop - ansible.builtin.command: /etc/init.d/chrome-remote-desktop start - register: result - changed_when: result.rc == 0 diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml deleted file mode 100644 index daae08176d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Ensure nvidia grid drivers and other binaries are installed - hosts: localhost - become: true - vars: - dist_settings: - bullseye: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-510.85.02-grid.run - grid_ver: vGPU14.2 - bookworm: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-550.54.15-grid.run - grid_ver: vGPU17.1 - jammy: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - gcc-12 # must match compiler used to build kernel on latest Ubuntu 22 - - pkg-config # observed to be necessary for GRID driver installation on latest Ubuntu 22 - - libglvnd-dev # observed to be necessary for GRID driver installation on latest Ubuntu 22 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-525.125.06-grid.run - grid_ver: vGPU15.3 - tasks: - - name: Fail if using wrong OS - ansible.builtin.assert: - that: - - ansible_os_family in ["Debian", "Ubuntu"] - - ansible_distribution_release in dist_settings.keys() | list - fail_msg: "ansible_os_family: {{ ansible_os_family }} or ansible_distribution_release: {{ansible_distribution_release}} was not acceptable." - - - name: Check if GRID driver installed - ansible.builtin.command: which nvidia-smi - register: nvidiasmi_result - ignore_errors: true - changed_when: false - - - name: Install binaries for GRID drivers - ansible.builtin.apt: - name: '{{ dist_settings[ansible_distribution_release]["packages"] }}' - state: present - update_cache: true - register: apt_result - retries: 6 - delay: 10 - until: apt_result is success - - - name: Install GRID driver if not existing - when: nvidiasmi_result is failed - block: - - name: Download GPU driver - ansible.builtin.get_url: - url: https://storage.googleapis.com/nvidia-drivers-us-public/GRID/{{ dist_settings[ansible_distribution_release]["grid_ver"] }}/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} - dest: /tmp/ - mode: "0755" - timeout: 30 - - - name: Stop gdm service - ansible.builtin.systemd: - name: gdm - state: stopped - - - name: Install GPU driver - ansible.builtin.shell: | - #jinja2: trim_blocks: "True" - {% if ansible_distribution_release == "jammy" %} - CC=gcc-12 /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent - {% else %} - /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent - {% endif %} - register: result - changed_when: result.rc == 0 - - - name: Download VirtualGL driver - ansible.builtin.get_url: - url: https://sourceforge.net/projects/virtualgl/files/3.0.2/virtualgl_3.0.2_amd64.deb/download - dest: /tmp/virtualgl_3.0.2_amd64.deb - mode: "0755" - timeout: 30 - - - name: Install VirtualGL - ansible.builtin.command: gdebi /tmp/virtualgl_3.0.2_amd64.deb --non-interactive - register: result - changed_when: result.rc == 0 - - - name: Fix headless Nvidia issue - block: - - name: Lookup gpu info - ansible.builtin.command: nvidia-xconfig --query-gpu-info - register: gpu_info - failed_when: gpu_info.rc != 0 - changed_when: false - - - name: Extract PCI ID - ansible.builtin.shell: | - set -o pipefail - echo "{{ gpu_info.stdout }}" | grep "PCI BusID " | head -n 1 | cut -d':' -f2-99 | xargs - args: - executable: /bin/bash - register: pci_id - changed_when: false - - - name: Configure nvidia-xconfig - ansible.builtin.command: nvidia-xconfig -a --allow-empty-initial-configuration --enable-all-gpus --virtual=1920x1200 --busid={{ pci_id.stdout }} - register: result - changed_when: result.rc == 0 - - - name: Set HardDPMS to false - ansible.builtin.replace: - path: /etc/X11/xorg.conf - regexp: "Section \"Device\"" - replace: "Section \"Device\"\n Option \"HardDPMS\" \"false\"" - - - name: Configure VirtualGL for X - ansible.builtin.command: vglserver_config +glx +s +f -t - register: result - changed_when: result.rc == 0 - - - name: Configure gdm for X - block: - - name: Configure default display manager - ansible.builtin.copy: - dest: /etc/X11/default-display-manager - content: "/usr/sbin/gdm3" - mode: 0644 - - - name: Switch boot target to gui - ansible.builtin.command: systemctl set-default graphical.target - register: result - changed_when: result.rc == 0 - - - name: Start gdm service - ansible.builtin.systemd: - name: gdm - daemon_reload: true - state: started diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml deleted file mode 100644 index 6767b05fb2..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Mask sleep, suspend, hibernate, and hybrid-sleep targets - hosts: localhost - become: true - tasks: - - - name: Mask sleep target - ansible.builtin.systemd: - name: sleep.target - masked: true - - - name: Mask suspend target - ansible.builtin.systemd: - name: suspend.target - masked: true - - - name: Mask hibernate target - ansible.builtin.systemd: - name: hibernate.target - masked: true - - - name: Mask hybrid-sleep target - ansible.builtin.systemd: - name: hybrid-sleep.target - masked: true diff --git a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf b/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf deleted file mode 100644 index ac4c3b1869..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf +++ /dev/null @@ -1,277 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which Google Cloud resources will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. Cloud resource names will include this value." - type = string - #default = "chrome-remote-desktop" -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zone" { - description = "Default zone for creating resources" - type = string -} - -variable "instance_count" { - description = "Number of instances" - type = number - default = 1 -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "instance_image" { - description = <<-EOD - Image used to build chrome remote desktop node. The default image is - name="debian-12-bookworm-v20250610" and project="debian-cloud". - NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues. - - An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud". - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "debian-cloud" - name = "debian-12-bookworm-v20250610" - } -} - -variable "disk_size_gb" { - description = "Size of disk for instances." - type = number - default = 200 -} - -variable "disk_type" { - description = "Disk type for instances." - type = string - default = "pd-balanced" -} - -variable "auto_delete_boot_disk" { - description = "Controls if boot disk should be auto-deleted when instance is deleted." - type = bool - default = true -} - -variable "name_prefix" { - description = <<-EOT - An optional name for all VM and disk resources. - If not supplied, `deployment_name` will be used. - When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, - then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". - EOT - type = string - default = null -} - -variable "add_deployment_name_before_prefix" { - description = <<-EOT - If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. - See `name_prefix` for further details on resource naming behavior. - EOT - type = bool - default = false -} - -variable "enable_public_ips" { - description = "If set to true, instances will have public IPs on the internet." - type = bool - default = true -} - -variable "machine_type" { - description = "Machine type to use for the instance creation. Must be N1 family if GPU is used." - type = string - default = "n1-standard-8" -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "service_account" { - description = "Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account." - type = object({ - email = string, - scopes = set(string) - }) - default = { - email = null - scopes = [ - "https://www.googleapis.com/auth/cloud-platform", - ] - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the VM." - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the VM." - type = string - default = null -} - -variable "network_interfaces" { - description = <<-EOT - A list of network interfaces. The options match that of the terraform - network_interface block of google_compute_instance. For descriptions of the - subfields or more information see the documentation: - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - **_NOTE:_** If `network_interfaces` are set, `network_self_link` and - `subnetwork_self_link` will be ignored, even if they are provided through - the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply - to network interfaces defined in this variable. - Subfields: - network (string, required if subnetwork is not supplied) - subnetwork (string, required if network is not supplied) - subnetwork_project (string, optional) - network_ip (string, optional) - nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "RDMA", "IRDMA", "MRDMA"]) - stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) - queue_count (number, optional) - access_config (object, optional) - ipv6_access_config (object, optional) - alias_ip_range (list(object), optional) - EOT - type = list(object({ - network = string, - subnetwork = string, - subnetwork_project = string, - network_ip = string, - nic_type = string, - stack_type = string, - queue_count = number, - access_config = list(object({ - nat_ip = string, - public_ptr_domain_name = string, - network_tier = string - })), - ipv6_access_config = list(object({ - public_ptr_domain_name = string, - network_tier = string - })), - alias_ip_range = list(object({ - ip_cidr_range = string, - subnetwork_range_name = string - })) - })) - default = [] -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "startup_script" { - description = "Startup script used on the instance" - type = string - default = null -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required" - type = list(object({ - type = string, - count = number - })) - default = [{ - type = "nvidia-tesla-t4-vws" - count = 1 - }] -} - -variable "threads_per_core" { - description = "Sets the number of threads per physical core" - type = number - default = 2 -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" - type = string - default = "TERMINATE" -} - -variable "bandwidth_tier" { - description = <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -This mode can be switched to proactive (automatic) replacement by setting -[var.update_policy](#input_update_policy) to "PROACTIVE". In this case we -recommend the use of Filestore to store the job queue state ("spool") and -setting [var.spool_parent_dir][#input_spool_parent_dir] to its mount point: - -```yaml - - id: spoolfs - source: modules/file-system/filestore - use: - - network1 - settings: - filestore_tier: ENTERPRISE - local_mount: /shared - -... - - - id: htcondor_access - source: community/modules/scheduler/htcondor-access-point - use: - - network1 - - spoolfs - - htcondor_secrets - - htcondor_setup - - htcondor_cm - - htcondor_execute_point_group - settings: - spool_parent_dir: /shared -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 3.83 | -| [null](#requirement\_null) | >= 3.0 | -| [random](#requirement\_random) | ~> 3.6 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [null](#provider\_null) | >= 3.0 | -| [random](#provider\_random) | ~> 3.6 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [access\_point\_instance\_template](#module\_access\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [htcondor\_ap](#module\_htcondor\_ap) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_address.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_region_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_region_disk) | resource | -| [google_storage_bucket_object.ap_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.ap_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_shuffle.zones](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/shuffle) | resource | -| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_instance.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | -| [google_compute_region_instance_group.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_runner](#input\_access\_point\_runner) | A list of Toolkit runners for configuring an HTCondor access point | `list(map(string))` | `[]` | no | -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account for access point (e-mail format) | `string` | n/a | yes | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [autoscaler\_runner](#input\_autoscaler\_runner) | A list of Toolkit runners for configuring autoscaling daemons | `list(map(string))` | `[]` | no | -| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | -| [default\_mig\_id](#input\_default\_mig\_id) | Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | -| [disk\_type](#input\_disk\_type) | Boot disk size in GB | `string` | `"pd-balanced"` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape acoss zones for instance group managing high availability of access point | `string` | `"ANY_SINGLE_ZONE"` | no | -| [enable\_high\_availability](#input\_enable\_high\_availability) | Provision HTCondor access point in high availability mode | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | Enable Public IPs on the access points | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor and Toolkit support installed."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | -| [mig\_id](#input\_mig\_id) | List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module) | `list(string)` | `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spool\_disk\_size\_gb](#input\_spool\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | -| [spool\_disk\_type](#input\_spool\_disk\_type) | Boot disk size in GB | `string` | `"pd-ssd"` | no | -| [spool\_parent\_dir](#input\_spool\_parent\_dir) | HTCondor access point configuration SPOOL will be set to subdirectory named "spool" | `string` | `"/var/lib/condor"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | -| [zones](#input\_zones) | Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_ips](#output\_access\_point\_ips) | IP addresses of the access points provisioned by this module | -| [access\_point\_name](#output\_access\_point\_name) | Name of the access point provisioned by this module | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml deleted file mode 100644 index 6a2f50c831..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Access Point - hosts: localhost - become: true - vars: - spool_dir: /var/lib/condor/spool - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - htcondor_spool_disk_device: /dev/disk/by-id/google-htcondor-spool-disk - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - - name: Configure HTCondor SchedD - when: htcondor_role == 'get_htcondor_submit' - block: - - name: Format spool disk - community.general.filesystem: - fstype: ext4 - state: present - dev: "{{ htcondor_spool_disk_device }}" - # RUN TUNE2FS - - name: Mount spool (creates mount point) - ansible.posix.mount: - path: "{{ spool_dir }}" - src: "{{ htcondor_spool_disk_device }}" - fstype: ext4 - opts: defaults - state: mounted - - name: Ensure spool free space - ansible.builtin.command: tune2fs -r 0 {{ htcondor_spool_disk_device }} - - name: Setup spool directory - ansible.builtin.file: - path: "{{ spool_dir }}" - state: directory - owner: condor - group: condor - mode: 0755 - recurse: true - - name: Create SystemD override directory for HTCondor - ansible.builtin.file: - path: /etc/systemd/system/condor.service.d - state: directory - owner: root - group: root - mode: 0755 - - name: Ensure HTCondor starts after shared filesystem is mounted - ansible.builtin.copy: - dest: /etc/systemd/system/condor.service.d/mount-spool.conf - mode: 0644 - content: | - [Unit] - RequiresMountsFor={{ spool_dir }} - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor configuration complete; startup-script may still be executing ********" diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf deleted file mode 100644 index fdbcf5c32f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf +++ /dev/null @@ -1,338 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-access-point", ghpc_role = "scheduler" }) -} - -locals { - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - metadata = merge( - local.network_storage_metadata, - local.enable_oslogin_metadata, - local.disable_automatic_updates_metadata, - var.metadata - ) - - host_count = 1 - name_prefix = "${var.deployment_name}-ap" - - example_runner = { - type = "data" - destination = "/var/tmp/helloworld.sub" - content = <<-EOT - universe = vanilla - executable = /bin/sleep - arguments = 1000 - output = out.$(ClusterId).$(ProcId) - error = err.$(ClusterId).$(ProcId) - log = log.$(ClusterId).$(ProcId) - request_cpus = 1 - request_memory = 100MB - queue - EOT - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.access_point_runner, - [local.schedd_runner], - var.autoscaler_runner, - [local.example_runner] - ) - - ap_config = templatefile("${path.module}/templates/condor_config.tftpl", { - htcondor_role = "get_htcondor_submit", - central_manager_ips = var.central_manager_ips - spool_dir = "${var.spool_parent_dir}/spool", - mig_ids = var.mig_id, - default_mig_id = var.default_mig_id - }) - - ap_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.ap_config.output_name}" - schedd_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e htcondor_role=get_htcondor_submit", - "-e config_object=${local.ap_object}", - "-e spool_dir=${var.spool_parent_dir}/spool", - "-e htcondor_spool_disk_device=/dev/disk/by-id/google-${local.spool_disk_device_name}", - ]) - } - - access_point_ips = google_compute_address.ap.address - access_point_name = data.google_compute_instance.ap.name - - spool_disk_resource_name = "${var.deployment_name}-spool-disk" - spool_disk_device_name = "htcondor-spool-disk" - spool_disk_source = try(google_compute_disk.spool[0].name, google_compute_region_disk.spool[0].self_link) - - zones = coalescelist(var.zones, random_shuffle.zones.result) - - vm_family = split("-", var.machine_type)[0] - regional_pd_families = ["e2", "n1", "n2", "n2d"] -} - -data "google_compute_image" "htcondor" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - postcondition { - condition = self.disk_size_gb <= var.disk_size_gb - error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region - - lifecycle { - postcondition { - condition = alltrue([ - for z in var.zones : contains(self.names, z) - ]) - error_message = "Each entry in var.zones must be a zone in var.region: ${var.region}" - } - } -} - -resource "random_shuffle" "zones" { - input = data.google_compute_zones.available.names - result_count = var.enable_high_availability ? 2 : 1 -} - -data "google_compute_region_instance_group" "ap" { - self_link = module.htcondor_ap.self_link - lifecycle { - postcondition { - condition = length(self.instances) == local.host_count - error_message = "There should be ${local.host_count} access points found" - } - } -} - -data "google_compute_instance" "ap" { - self_link = data.google_compute_region_instance_group.ap.instances[0].instance -} - -resource "null_resource" "ap_config" { - triggers = { - config = local.ap_config - } -} - -resource "google_storage_bucket_object" "ap_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.ap_config.id), 0, 4)}" - content = local.ap_config - bucket = var.htcondor_bucket_name - - lifecycle { - precondition { - condition = var.default_mig_id == "" || contains(var.mig_id, var.default_mig_id) - error_message = "If set, var.default_mig_id must be an element in var.mig_id" - } - - # by construction, this precondition only fails when the user has set - # var.zones to a non-empty list of length not equal to 2 - precondition { - condition = !var.enable_high_availability || length(local.zones) == 2 - error_message = "When using HTCondor access point high availability, var.zones must be of length 2." - } - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -resource "google_compute_region_disk" "spool" { - count = var.enable_high_availability ? 1 : 0 - name = local.spool_disk_resource_name - labels = local.labels - type = var.spool_disk_type - region = var.region - size = var.spool_disk_size_gb - - replica_zones = local.zones - - lifecycle { - precondition { - condition = var.spool_disk_size_gb >= 200 - error_message = "When using HTCondor access point high availability, var.spool_disk_size_gb must be set to 200 or greater." - } - - precondition { - condition = contains(local.regional_pd_families, local.vm_family) - error_message = "When using HTCondor access point high availability, var.machine_type must be one of ${jsonencode(local.regional_pd_families)}." - } - } -} - -resource "google_compute_disk" "spool" { - count = var.enable_high_availability ? 0 : 1 - name = local.spool_disk_resource_name - labels = local.labels - type = var.spool_disk_type - zone = local.zones[0] - size = var.spool_disk_size_gb -} - -resource "google_compute_address" "ap" { - project = var.project_id - name = local.name_prefix - region = var.region - subnetwork = var.subnetwork_self_link - address_type = "INTERNAL" - purpose = "GCE_ENDPOINT" -} - -module "access_point_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.access_point_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - preemptible = false - startup_script = module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.htcondor.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - network_ip = google_compute_address.ap.id - - # spool disk - additional_disks = [ - { - source = local.spool_disk_source - device_name = local.spool_disk_device_name - } - ] -} - -module "htcondor_ap" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = local.host_count - hostname = local.name_prefix - instance_template = module.access_point_instance_template.self_link - - health_check_name = "health-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) - max_surge_fixed = 0 # must be 0 to preserve hostnames - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - - stateful_disks = [{ - device_name = local.spool_disk_device_name - delete_rule = "ON_PERMANENT_INSTANCE_DELETION" - }] - stateful_ips = var.enable_public_ips ? [{ - interface_name = "nic0" - delete_rule = "ON_PERMANENT_INSTANCE_DELETION" - is_external = true - }] : [] - - # the timeouts below are default for resource - wait_for_instances = true - mig_timeouts = { - create = "15m" - delete = "15m" - update = "15m" - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf deleted file mode 100644 index f7424c6d5d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "access_point_ips" { - description = "IP addresses of the access points provisioned by this module" - value = local.access_point_ips -} - -output "access_point_name" { - description = "Name of the access point provisioned by this module" - value = local.access_point_name -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl deleted file mode 100644 index 214fbc726f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:${htcondor_role} -CONDOR_HOST = ${join(",", central_manager_ips)} - -SPOOL = ${spool_dir} -SCHEDD_INTERVAL = 30 -TRUST_UID_DOMAIN = True -SUBMIT_ATTRS = RunAsOwner -RunAsOwner = True - -# When a job matches to a machine, add machine attributes to the job for -# condor_history (e.g. VM Instance ID) -use feature:JobsHaveInstanceIDs -SYSTEM_JOB_MACHINE_ATTRS = $(SYSTEM_JOB_MACHINE_ATTRS) \ - CloudVMType CloudZone CloudInterruptible -SYSTEM_JOB_MACHINE_ATTRS_HISTORY_LENGTH = 10 - -# Add Cloud attributes to SchedD ClassAd -use feature:ScheddCronOneShot(cloud, $(LIBEXEC)/common-cloud-attributes-google.py) -SCHEDD_CRON_cloud_PREFIX = Cloud - -# aid the user by automatically using RequireSpot in their Requirements, unless -# the user has explicitly used CloudInterruptible -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SPOT -JOB_TRANSFORM_SPOT @=end - REQUIREMENTS ! isUndefined(RequireSpot) && ! unresolved(Requirements, "^CloudInterruptible$") - SET Requirements ($(MY.Requirements)) && (CloudInterruptible is My.RequireSpot) -@end - -# help the user by enforcing that RequireSpot is undefined or a boolean -SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) SPOT -SUBMIT_REQUIREMENT_SPOT = isUndefined(RequireSpot) || isBoolean(RequireSpot) -SUBMIT_REQUIREMENT_SPOT_REASON = "If +RequireSpot is defined, it must be either True or False" - -%{ if length(mig_ids) > 0 ~} -MIG_IDS = "${join(" ", mig_ids)}" -MIG_ID_LIST = split($(MIG_IDS)) -%{ if default_mig_id != "" ~} -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) ID_DEFAULT -JOB_TRANSFORM_ID_DEFAULT @=end - DEFAULT RequireId "${default_mig_id}" -@end -%{ endif ~} -SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) MIGID -SUBMIT_REQUIREMENT_MIGID = !isUndefined(RequireId) && member(RequireId, $(MIG_ID_LIST)) -SUBMIT_REQUIREMENT_MIGID_REASON = strcat("Jobs must set +RequireId to one of following values surrounded by quotation marks:\n", $(MIG_IDS)) - -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) MIGID -JOB_TRANSFORM_MIGID @=end - REQUIREMENTS ! isUndefined(RequireId) && ! unresolved(Requirements, "^CloudCreatedBy$") - SET Requirements ($(MY.Requirements)) && regexp(strcat("/", My.RequireId, "$"), CloudCreatedBy) -@end -%{ endif ~} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf deleted file mode 100644 index f54a88ac2e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zones" { - description = "Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region." - type = list(string) - default = [] - nullable = false - - validation { - condition = length(var.zones) <= 2 - error_message = "Set var.zones to the empty list or up to 2 zones in var.region" - } -} - -variable "distribution_policy_target_shape" { - description = "Target shape acoss zones for instance group managing high availability of access point" - type = string - default = "ANY_SINGLE_ZONE" -} - -variable "network_self_link" { - description = "The self link of the network in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "access_point_service_account_email" { - description = "Service account for access point (e-mail format)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 32 - nullable = false -} - -variable "disk_type" { - description = "Boot disk size in GB" - type = string - default = "pd-balanced" - nullable = false -} - -variable "spool_disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 32 - nullable = false -} - -variable "spool_disk_type" { - description = "Boot disk size in GB" - type = string - default = "pd-ssd" - nullable = false -} - -variable "metadata" { - description = "Metadata to add to HTCondor central managers" - type = map(string) - default = {} -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - nullable = false - validation { - condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "enable_high_availability" { - description = "Provision HTCondor access point in high availability mode" - type = bool - default = false -} - -variable "instance_image" { - description = <<-EOD - Custom VM image with HTCondor and Toolkit support installed." - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "machine_type" { - description = "Machine type to use for HTCondor central managers" - type = string - default = "n2-standard-4" -} - -variable "access_point_runner" { - description = "A list of Toolkit runners for configuring an HTCondor access point" - type = list(map(string)) - default = [] -} - -variable "autoscaler_runner" { - description = "A list of Toolkit runners for configuring autoscaling daemons" - type = list(map(string)) - default = [] -} - -variable "spool_parent_dir" { - description = "HTCondor access point configuration SPOOL will be set to subdirectory named \"spool\"" - type = string - default = "/var/lib/condor" -} - -variable "central_manager_ips" { - description = "List of IP addresses of HTCondor Central Managers" - type = list(string) -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "enable_public_ips" { - description = "Enable Public IPs on the access points" - type = bool - default = false -} - -variable "mig_id" { - description = "List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module)" - type = list(string) - default = [] - nullable = false - - validation { - condition = length(var.mig_id) > 0 - error_message = "At least 1 MIG containing execute points must be provided to this module" - } -} - -variable "default_mig_id" { - description = "Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id" - type = string - default = "" - nullable = false -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" - type = string - default = "OPPORTUNISTIC" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf deleted file mode 100644 index 0d07e7abf1..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.6" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.74.0" - } - - required_version = ">= 1.1" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md deleted file mode 100644 index dfab563a55..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md +++ /dev/null @@ -1,159 +0,0 @@ -## Description - -This module provisions a highly available HTCondor central manager using a [Managed -Instance Group (MIG)][mig] with auto-healing. - -[mig]: https://cloud.google.com/compute/docs/instance-groups - -## Usage - -This module provisions an HTCondor central manager with a standard -configuration. For the node to function correctly, you must supply the input -variable described below: - -- [var.central_manager_runner](#input_central_manager_runner) - - Runner must download a POOL password / signing key and create an [IDTOKEN] - with no scopes (full authorization). - -A reference implementation is included in the Toolkit module -[htcondor-pool-secrets]. You may substitute implementations so long as they -duplicate the functionality in the references. Usage is demonstrated in the -[HTCondor example][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- -[htcondor-pool-secrets]: ../htcondor-pool-secrets/README.md -[IDTOKEN]: https://htcondor.readthedocs.io/en/latest/admin-manual/security.html#introducing-idtokens - -## Behavior of Managed Instance Group (MIG) - -A regional [MIG][mig] is used to provision the central manager, although only -1 node will ever be active at a time. By default, the node will be provisioned -in any of the zones available in that region, however, it can be constrained to -run in fewer zones (or a single zone) using [var.zones](#input_zones). - -When the configuration of the Central Manager is changed, the MIG can be -configured to [replace the VM][replacement] using a "proactive" or -"opportunistic" policy. By default, the Central Manager replacement policy is -set to proactive. In practice, this means that the Central Manager will be -replaced by Terraform when changes to the instance template / HTCondor -configuration are made. The Central Manager is safe to replace automatically as -it gathers its state information from periodic messages exchanged with the rest -of the HTCondor pool. - -This mode can be configured by setting [var.update_policy](#input_update_policy) -to either "PROACTIVE" (default) or "OPPORTUNISTIC". If set to opportunistic -replacement, the Central Manager will be replaced only when: - -- intentionally by issuing an update via Cloud Console or using gcloud (below) -- the VM becomes unhealthy or is otherwise automatically replaced (e.g. regular - Google Cloud maintenance) - -For example, to manually update all instances in a MIG: - -```text -gcloud compute instance-groups managed update-instances \ - <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - -## Limiting inter-zone egress - -Because all the elements of the HTCondor pool use regional MIGs, they may be -subject to [interzone egress fees][network-pricing]. The primary traffic between -nodes of an HTCondor pool running embarrassingly parallel jobs is expected to -be limited to API traffic for job scheduling and monitoring. Please review the -[network pricing][network-pricing] documentation and determine if this cost is -a concern. If it is, use [var.zones](#input_zones) to constrain each node within -your HTCondor pool to operate within a single zone. - -[network-pricing]: https://cloud.google.com/vpc/network-pricing - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1.0 | -| [google](#requirement\_google) | >= 3.83 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [central\_manager\_instance\_template](#module\_central\_manager\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [htcondor\_cm](#module\_htcondor\_cm) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_address.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_storage_bucket_object.cm_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.cm_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_instance.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | -| [google_compute_region_instance_group.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [central\_manager\_runner](#input\_central\_manager\_runner) | A list of Toolkit runners for configuring an HTCondor central manager | `list(map(string))` | `[]` | no | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for central manager (can be supplied by htcondor-setup module) | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `20` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape for instance group managing high availability of central manager | `string` | `"ANY_SINGLE_ZONE"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor installed using the htcondor-install module."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor central manager will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Central Manager ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle). | `string` | `"PROACTIVE"` | no | -| [zones](#input\_zones) | Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [central\_manager\_ips](#output\_central\_manager\_ips) | IP addresses of the central managers provisioned by this module | -| [central\_manager\_name](#output\_central\_manager\_name) | Name of the central managers provisioned by this module | -| [list\_instances\_command](#output\_list\_instances\_command) | Command to list central managers provisioned by this module | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml deleted file mode 100644 index 7408af6370..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor central manager - hosts: localhost - become: true - vars: - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - handlers: - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor system configuration complete ********" diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf deleted file mode 100644 index d288a91144..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-central-manager", ghpc_role = "scheduler" }) -} - -locals { - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - metadata = merge( - local.network_storage_metadata, - local.enable_oslogin_metadata, - local.disable_automatic_updates_metadata, - var.metadata - ) - - name_prefix = "${var.deployment_name}-cm" - - cm_config = templatefile("${path.module}/templates/condor_config.tftpl", {}) - - cm_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.cm_config.output_name}" - schedd_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e config_object=${local.cm_object}", - ]) - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.central_manager_runner, - [local.schedd_runner] - ) - - central_manager_ips = google_compute_address.cm.address - central_manager_name = data.google_compute_instance.cm.name - - list_instances_command = "gcloud compute instance-groups list-instances ${data.google_compute_region_instance_group.cm.name} --region ${var.region} --project ${var.project_id}" - - zones = coalescelist(var.zones, data.google_compute_zones.available.names) -} - -data "google_compute_image" "htcondor" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - postcondition { - condition = self.disk_size_gb <= var.disk_size_gb - error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -data "google_compute_region_instance_group" "cm" { - self_link = module.htcondor_cm.self_link - lifecycle { - postcondition { - condition = length(self.instances) == 1 - error_message = "There should only be 1 central manager found" - } - } -} - -data "google_compute_instance" "cm" { - self_link = data.google_compute_region_instance_group.cm.instances[0].instance -} - -resource "null_resource" "cm_config" { - triggers = { - config = local.cm_config - } -} - -resource "google_storage_bucket_object" "cm_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.cm_config.id), 0, 4)}" - content = local.cm_config - bucket = var.htcondor_bucket_name -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -resource "google_compute_address" "cm" { - project = var.project_id - name = local.name_prefix - region = var.region - subnetwork = var.subnetwork_self_link - address_type = "INTERNAL" - purpose = "GCE_ENDPOINT" -} - -module "central_manager_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.central_manager_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - preemptible = false - startup_script = module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.htcondor.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - network_ip = google_compute_address.cm.id -} - -module "htcondor_cm" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = 1 - hostname = local.name_prefix - instance_template = module.central_manager_instance_template.self_link - - health_check_name = "health-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) - max_surge_fixed = 0 # must be 0 to preserve hostnames - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - - # the timeouts below are default for resource - wait_for_instances = true - mig_timeouts = { - create = "15m" - delete = "15m" - update = "15m" - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf deleted file mode 100644 index a6272e7ca2..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "list_instances_command" { - description = "Command to list central managers provisioned by this module" - value = local.list_instances_command -} - -output "central_manager_ips" { - description = "IP addresses of the central managers provisioned by this module" - value = local.central_manager_ips -} - -output "central_manager_name" { - description = "Name of the central managers provisioned by this module" - value = local.central_manager_name -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl deleted file mode 100644 index 5b9676457e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:get_htcondor_central_manager -CONDOR_HOST = $(IPV4_ADDRESS) - -# Central Manager configuration settings -# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-collector-configuration-file-entries -# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-negotiator-configuration-file-entries -# set classad lifetime (expiration) to ~5x the update interval for all daemons -# defaults to 900s -CLASSAD_LIFETIME = 180 -COLLECTOR_UPDATE_INTERVAL = 30 -NEGOTIATOR_UPDATE_INTERVAL = 30 -NEGOTIATOR_DEPTH_FIRST = True -NEGOTIATOR_UPDATE_AFTER_CYCLE = True diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf deleted file mode 100644 index 7f85861c3f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf +++ /dev/null @@ -1,192 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor central manager will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zones" { - description = "Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region." - type = list(string) - default = [] - nullable = false -} - -variable "distribution_policy_target_shape" { - description = "Target shape for instance group managing high availability of central manager" - type = string - default = "ANY_SINGLE_ZONE" -} - -variable "network_self_link" { - description = "The self link of the network in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "central_manager_service_account_email" { - description = "Service account e-mail for central manager (can be supplied by htcondor-setup module)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 20 - nullable = false -} - -variable "metadata" { - description = "Metadata to add to HTCondor central managers" - type = map(string) - default = {} -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - nullable = false - validation { - condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "instance_image" { - description = <<-EOD - Custom VM image with HTCondor installed using the htcondor-install module." - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "machine_type" { - description = "Machine type to use for HTCondor central managers" - type = string - default = "n2-standard-4" -} - -variable "central_manager_runner" { - description = "A list of Toolkit runners for configuring an HTCondor central manager" - type = list(map(string)) - default = [] -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Central Manager (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)." - type = string - default = "PROACTIVE" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf deleted file mode 100644 index 4dee3adac7..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.74.0" - } - - required_version = ">= 1.1.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md deleted file mode 100644 index 7158e7bac6..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md +++ /dev/null @@ -1,172 +0,0 @@ -## Description - -This module is responsible for the following actions: - -- store an HTCondor Pool password in Google Cloud Secret Manager - - will generate a new password if one is not supplied -- create a secret in Google Cloud Secret Manager in which the HTCondor central - manager can place IDTOKENs (JWT Authorizations) for execute points to download -- create a Toolkit runner for the central manager - - download the POOL password / signing key - - create a local IDTOKEN for itself - - upload the execute point IDTOKEN secret -- create a Toolkit runner for access points - - download the POOL password / signing key - - create a local IDTOKEN for itself -- create a Toolkit runner for execute points - - Fetch the IDTOKEN secret generated by the central manager - -It is expected to be used with the [htcondor-install] and -[htcondor-execute-point] modules. - -[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[htcondor-install]: ../../scripts/htcondor-setup/README.md -[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md - -[htcrole]: https://htcondor.readthedocs.io/en/latest/getting-htcondor/admin-quick-start.html#what-get-htcondor-does-to-configure-a-role - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_setup - - - id: htcondor_startup_central_manager - source: modules/scripts/startup-script - settings: - runners: - - $(htcondor_install.install_htcondor_runner) - - $(htcondor_secrets.central_manager_runner) - - $(htcondor_setup.central_manager_runner) - -- id: htcondor_cm - source: modules/compute/vm-instance - use: - - network1 - - htcondor_startup_central_manager - settings: - name_prefix: cm0 - machine_type: c2-standard-4 - disable_public_ips: true - service_account: - email: $(htcondor_setup.central_manager_service_account) - scopes: - - cloud-platform - network_interfaces: - - network: null - subnetwork: $(network1.subnetwork_self_link) - subnetwork_project: $(vars.project_id) - network_ip: $(htcondor_setup.central_manager_internal_ip) - stack_type: null - access_config: [] - ipv6_access_config: [] - alias_ip_range: [] - nic_type: VIRTIO_NET - queue_count: null - outputs: - - internal_ip -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 4.84 | -| [random](#requirement\_random) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | -| [random](#provider\_random) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_secret_manager_secret.execute_point_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_iam_member.access_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.central_manager_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.central_manager_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.execute_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_version.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [random_password.pool](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [pool\_password](#input\_pool\_password) | HTCondor Pool Password | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [trust\_domain](#input\_trust\_domain) | Trust domain for HTCondor pool (if not supplied, will be set based on project\_id) | `string` | `""` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_runner](#output\_access\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor access point | -| [central\_manager\_runner](#output\_central\_manager\_runner) | Toolkit Runner to download pool secrets to an HTCondor central manager | -| [execute\_point\_runner](#output\_execute\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor execute point | -| [pool\_password\_secret\_id](#output\_pool\_password\_secret\_id) | Google Cloud Secret Manager ID containing HTCondor Pool Password | -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | PowerShell script to download pool secrets to an HTCondor execute point | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml deleted file mode 100644 index 538c809c2a..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Secrets - hosts: localhost - become: true - vars: - condor_config_root: /etc/condor - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - password_id is defined - - trust_domain is defined - - name: Set Pool Trust Domain - ansible.builtin.copy: - dest: "{{ condor_config_root }}/config.d/51-ghpc-trust-domain" - mode: 0644 - content: | - # these lines must appear AFTER any "use role:" settings - UID_DOMAIN = {{ trust_domain }} - TRUST_DOMAIN = {{ trust_domain }} - - name: Get HTCondor Pool password (token signing key) - when: htcondor_role != 'get_htcondor_execute' - ansible.builtin.shell: | - set -e -o pipefail +o history - POOL_PASSWORD=$(gcloud secrets versions access latest --secret={{ password_id }}) - echo -n "$POOL_PASSWORD" | sh -c "condor_store_cred add -c -i -" - args: - creates: "{{ condor_config_root }}/passwords.d/POOL" - executable: /bin/bash - - name: Configure HTCondor Central Manager - when: htcondor_role == 'get_htcondor_central_manager' - block: - - name: Create IDTOKEN for Central Manager - ansible.builtin.shell: | - umask 0077 - condor_token_create -identity condor@{{ trust_domain }} \ - -token condor@{{ trust_domain }} - args: - creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" - - name: Create IDTOKEN secret for Execute Points - when: xp_idtoken_secret_id | length > 0 - changed_when: true - ansible.builtin.shell: | - umask 0077 - TMPFILE=$(mktemp) - condor_token_create -authz READ -authz ADVERTISE_MASTER \ - -authz ADVERTISE_STARTD -identity condor@{{ trust_domain }} > "$TMPFILE" - gcloud secrets versions add --data-file "$TMPFILE" {{ xp_idtoken_secret_id }} - rm -f "$TMPFILE" - - name: Configure HTCondor SchedD - when: htcondor_role == 'get_htcondor_submit' - block: - - name: Create IDTOKEN to advertise access point - ansible.builtin.shell: | - umask 0077 - # DAEMON authorization can likely be removed in future when scopes - # needed to trigger a negotiation cycle are changed. Suggest review - # https://opensciencegrid.atlassian.net/jira/software/c/projects/HTCONDOR/issues/?filter=allissues - condor_token_create -authz READ -authz ADVERTISE_MASTER \ - -authz ADVERTISE_SCHEDD -authz DAEMON -identity condor@{{ trust_domain }} \ - -token condor@{{ trust_domain }} - args: - creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" - - name: Configure HTCondor StartD - when: htcondor_role == 'get_htcondor_execute' - block: - - name: Create SystemD override directory for HTCondor Execute Point - ansible.builtin.file: - path: /etc/systemd/system/condor.service.d - state: directory - owner: root - group: root - mode: 0755 - - name: Fetch IDTOKEN to advertise execute point - ansible.builtin.copy: - dest: "/etc/systemd/system/condor.service.d/htcondor-token-fetcher.conf" - mode: 0644 - content: | - [Service] - ExecStartPre=gcloud secrets versions access latest --secret {{ xp_idtoken_secret_id }} \ - --out-file {{ condor_config_root }}/tokens.d/condor@{{ trust_domain }} - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf deleted file mode 100644 index 1a7c761760..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-pool-secrets", ghpc_role = "scheduler" }) -} - -locals { - pool_password = coalesce(var.pool_password, random_password.pool.result) - auto = length(var.user_managed_replication) == 0 ? "" : "-user" - access_point_service_account_iam_email = "serviceAccount:${var.access_point_service_account_email}" - central_manager_service_account_iam_email = "serviceAccount:${var.central_manager_service_account_email}" - execute_point_service_account_iam_email = "serviceAccount:${var.execute_point_service_account_email}" - - trust_domain = coalesce(var.trust_domain, "c.${var.project_id}.internal") - - runner_cm = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_central_manager", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - - runner_access = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_submit", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - - runner_execute = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_execute", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - windows_startup_ps1 = templatefile( - "${path.module}/templates/fetch-idtoken.ps1.tftpl", - { - trust_domain = local.trust_domain, - xp_idtoken_secret_id = google_secret_manager_secret.execute_point_idtoken.secret_id, - } - ) -} - -resource "random_password" "pool" { - length = 24 - special = true - override_special = "_-#=." -} - -resource "google_secret_manager_secret" "pool_password" { - secret_id = "${var.deployment_name}-pool-password${local.auto}" - - labels = local.labels - - replication { - dynamic "auto" { - for_each = length(var.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_version" "pool_password" { - secret = google_secret_manager_secret.pool_password.id - secret_data = local.pool_password -} - -# this secret will be populated by the Central Manager -resource "google_secret_manager_secret" "execute_point_idtoken" { - secret_id = "${var.deployment_name}-execute-point-idtoken${local.auto}" - - labels = local.labels - - replication { - dynamic "auto" { - for_each = length(var.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_iam_member" "central_manager_password" { - secret_id = google_secret_manager_secret.pool_password.id - role = "roles/secretmanager.secretAccessor" - member = local.central_manager_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "central_manager_idtoken" { - secret_id = google_secret_manager_secret.execute_point_idtoken.id - role = "roles/secretmanager.secretVersionManager" - member = local.central_manager_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "access_point" { - secret_id = google_secret_manager_secret.pool_password.id - role = "roles/secretmanager.secretAccessor" - member = local.access_point_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "execute_point" { - secret_id = google_secret_manager_secret.execute_point_idtoken.id - role = "roles/secretmanager.secretAccessor" - member = local.execute_point_service_account_iam_email -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml deleted file mode 100644 index 4b0bdbd616..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com - - secretmanager.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf deleted file mode 100644 index 81c4986b16..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "pool_password_secret_id" { - description = "Google Cloud Secret Manager ID containing HTCondor Pool Password" - value = google_secret_manager_secret.pool_password.secret_id - sensitive = true -} - -output "central_manager_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor central manager" - value = local.runner_cm - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "access_point_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor access point" - value = local.runner_access - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "execute_point_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor execute point" - value = local.runner_execute - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "windows_startup_ps1" { - description = "PowerShell script to download pool secrets to an HTCondor execute point" - value = local.windows_startup_ps1 -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl deleted file mode 100644 index 04c96291ee..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl +++ /dev/null @@ -1,26 +0,0 @@ -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -$config_dir = 'C:\Condor\config' -if(!(test-path -PathType container -Path $config_dir)) -{ - New-Item -ItemType Directory -Path $config_dir -} -$config_file = "$config_dir\51-ghpc-trust-domain" - -$config_string = @' -# these lines must appear AFTER any "use role:" settings -UID_DOMAIN = ${trust_domain} -TRUST_DOMAIN = ${trust_domain} -'@ - -Set-Content -Path "$config_file" -Value "$config_string" - -# obtain IDTOKEN for authentication by StartD to Central Manager -gcloud secrets versions access latest --secret ${xp_idtoken_secret_id} ` - --out-file C:\condor\tokens.d\condor@${trust_domain} - -if ($LASTEXITCODE -ne 0) -{ - throw "Could not download HTCondor IDTOKEN; exiting startup script" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf deleted file mode 100644 index 22ef3644e8..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "access_point_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "central_manager_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "execute_point_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "pool_password" { - description = "HTCondor Pool Password" - type = string - sensitive = true - default = null -} - -variable "trust_domain" { - description = "Trust domain for HTCondor pool (if not supplied, will be set based on project_id)" - type = string - default = "" -} - -variable "user_managed_replication" { - type = list(object({ - location = string - kms_key_name = optional(string) - })) - description = "Replication parameters that will be used for defined secrets" - default = [] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf deleted file mode 100644 index d8a1d96f5f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.84" - } - random = { - source = "hashicorp/random" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md deleted file mode 100644 index 5a403c0a38..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module creates the service accounts for use by the primary elements of an -[HTCondor pool][pool]: - -- Central Managers -- Access Points -- Execute Points - -Each service account is assigned common roles necessary for the VM to function -properly. In particular, nearly every VM requires the ability to read from Cloud -Storage buckets and write Cloud Logging entries. These roles are configurable -as described below. - -[pool]: https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-admin-manual.html#the-different-roles-a-machine-can-play - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_service_accounts - source: community/modules/scheduler/htcondor-service-accounts - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - - htcondor_service_accounts - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_service_accounts - -- id: htcondor_cm - source: community/modules/scheduler/htcondor-central-manager - use: - - network1 - - htcondor_secrets - - htcondor_service_accounts - - htcondor_setup - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - central_manager_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [access\_point\_service\_account](#module\_access\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | -| [central\_manager\_service\_account](#module\_central\_manager\_service\_account) | ../../../../community/modules/project/service-account | n/a | -| [execute\_point\_service\_account](#module\_execute\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_roles](#input\_access\_point\_roles) | Project-wide roles for HTCondor Access Point service account | `list(string)` | 
[
  "compute.instanceAdmin.v1",
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [central\_manager\_roles](#input\_central\_manager\_roles) | Project-wide roles for HTCondor Central Manager service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_roles](#input\_execute\_point\_roles) | Project-wide roles for HTCondor Execute Point service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_service\_account\_email](#output\_access\_point\_service\_account\_email) | HTCondor Access Point Service Account (e-mail format) | -| [central\_manager\_service\_account\_email](#output\_central\_manager\_service\_account\_email) | HTCondor Central Manager Service Account (e-mail format) | -| [execute\_point\_service\_account\_email](#output\_execute\_point\_service\_account\_email) | HTCondor Execute Point Service Account (e-mail format) | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf deleted file mode 100644 index 9d97b18642..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# NB: the community/modules/project/service-account module will not output the -# service account e-mail address until all IAM bindings have been created; if -# underlying implementation changes, this module should declare explicit -# depends_on the IAM bindings to prevent race conditions for services that -# require them - -module "access_point_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Access Point" - deployment_name = var.deployment_name - name = "access" - project_roles = var.access_point_roles -} - -module "execute_point_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Execute Point" - deployment_name = var.deployment_name - name = "execute" - project_roles = var.execute_point_roles -} - -module "central_manager_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Central Manager" - deployment_name = var.deployment_name - name = "cm" - project_roles = var.central_manager_roles -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml deleted file mode 100644 index c4dcdffdf4..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf deleted file mode 100644 index 28f3a79457..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "access_point_service_account_email" { - description = "HTCondor Access Point Service Account (e-mail format)" - value = module.access_point_service_account.service_account_email -} - -output "central_manager_service_account_email" { - description = "HTCondor Central Manager Service Account (e-mail format)" - value = module.central_manager_service_account.service_account_email -} - -output "execute_point_service_account_email" { - description = "HTCondor Execute Point Service Account (e-mail format)" - value = module.execute_point_service_account.service_account_email -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf deleted file mode 100644 index ee186e0971..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "access_point_roles" { - description = "Project-wide roles for HTCondor Access Point service account" - type = list(string) - default = [ - "compute.instanceAdmin.v1", - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} - -variable "central_manager_roles" { - description = "Project-wide roles for HTCondor Central Manager service account" - type = list(string) - default = [ - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} - -variable "execute_point_roles" { - description = "Project-wide roles for HTCondor Execute Point service account" - type = list(string) - default = [ - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md deleted file mode 100644 index 1722702ceb..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md +++ /dev/null @@ -1,118 +0,0 @@ -## Description - -This module creates a bucket in which to store HTCondor configurations and -a firewall rule that allows Managed Instance Group health checks to probe the -health of HTCondor VMs. - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_service_accounts - source: community/modules/scheduler/htcondor-service-accounts - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - - htcondor_service_accounts - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_service_accounts - -- id: htcondor_cm - source: community/modules/scheduler/htcondor-central-manager - use: - - network1 - - htcondor_secrets - - htcondor_service_accounts - - htcondor_setup - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - central_manager_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [health\_check\_firewall\_rule](#module\_health\_check\_firewall\_rule) | ../../../../modules/network/firewall-rules | n/a | -| [htcondor\_bucket](#module\_htcondor\_bucket) | ../../../../modules/file-system/cloud-storage-bucket | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account e-mail for HTCondor Access Point | `string` | n/a | yes | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for HTCondor Central Manager | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account e-mail for HTCondor Execute Points | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which Central Managers will be placed. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [htcondor\_bucket\_name](#output\_htcondor\_bucket\_name) | Name of the HTCondor configuration bucket | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf deleted file mode 100644 index e048362663..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-setup", ghpc_role = "scheduler" }) -} - -locals { - service_account_iam_email = [ - "serviceAccount:${var.access_point_service_account_email}", - "serviceAccount:${var.central_manager_service_account_email}", - "serviceAccount:${var.execute_point_service_account_email}", - ] - service_account_email = [ - var.access_point_service_account_email, - var.central_manager_service_account_email, - var.execute_point_service_account_email, - ] -} - -module "health_check_firewall_rule" { - source = "../../../../modules/network/firewall-rules" - - subnetwork_self_link = var.subnetwork_self_link - - ingress_rules = [{ - name = "allow-health-check-${var.deployment_name}" - description = "Allow Managed Instance Group Health Checks for HTCondor VMs" - direction = "INGRESS" - source_ranges = [ - "130.211.0.0/22", - "35.191.0.0/16", - ] - target_service_accounts = local.service_account_email - allow = [{ - protocol = "tcp" - ports = ["9618"] - }] - }] -} - -module "htcondor_bucket" { - source = "../../../../modules/file-system/cloud-storage-bucket" - - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - name_prefix = "${var.deployment_name}-htcondor-config" - random_suffix = true - labels = local.labels - viewers = local.service_account_iam_email - - use_deployment_name_in_bucket_name = false -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml deleted file mode 100644 index 7b4918b962..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - iam.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf deleted file mode 100644 index a44223faee..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "htcondor_bucket_name" { - description = "Name of the HTCondor configuration bucket" - value = module.htcondor_bucket.gcs_bucket_name - - # ensure that all IAM bindings to the bucket and firewall rules are active - # before this modules output is allowed to propagate - depends_on = [ - module.htcondor_bucket, - module.health_check_firewall_rule - ] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf deleted file mode 100644 index 147a2ca88d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which Central Managers will be placed." - type = string -} - -variable "access_point_service_account_email" { - description = "Service account e-mail for HTCondor Access Point" - type = string -} - -variable "central_manager_service_account_email" { - description = "Service account e-mail for HTCondor Central Manager" - type = string -} - -variable "execute_point_service_account_email" { - description = "Service account e-mail for HTCondor Execute Points" - type = string -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md deleted file mode 100644 index 43254cbfa8..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md +++ /dev/null @@ -1,405 +0,0 @@ -## Description - -This module creates a slurm controller node via the internal -[slurm\_instance\_template] module. - -More information about Slurm On GCP can be found at the -[project's GitHub page][slurm-gcp] and in the -[Slurm on Google Cloud User Guide][slurm-ug]. - -The [user guide][slurm-ug] provides detailed instructions on customizing and -enhancing the Slurm on GCP cluster as well as recommendations on configuring the -controller for optimal performance at different scales. - -[slurm\_instance\_template]: /community/modules/internal/slurm-gcp/instance_template/README.md -[slurm-ug]: https://goo.gle/slurm-gcp-user-guide. -[enable\_cleanup\_compute]: #input\_enable\_cleanup\_compute -[enable\_cleanup\_subscriptions]: #input\_enable\_cleanup\_subscriptions -[enable\_reconfigure]: #input\_enable\_reconfigure - -### Example - -```yaml -- id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: - - network - - homefs - - compute_partition - settings: - machine_type: c2-standard-8 -``` - -This creates a controller node with the following attributes: - -* connected to the primary subnetwork of `network` -* the filesystem with the ID `homefs` (defined elsewhere in the blueprint) - mounted -* One partition with the ID `compute_partition` (defined elsewhere in the - blueprint) -* machine type upgraded from the default `c2-standard-4` to `c2-standard-8` - -### Live Cluster Reconfiguration - -The `schedmd-slurm-gcp-v6-controller` module supports the reconfiguration of -partitions and slurm configuration in a running, active cluster. - -To reconfigure a running cluster: - -1. Edit the blueprint with the desired configuration changes -2. Call `gcluster create -w` to overwrite the deployment directory -3. Follow instructions in terminal to deploy - -The following are examples of updates that can be made to a running cluster: - -* Add or remove a partition to the cluster -* Resize an existing partition -* Attach new network storage to an existing partition - -> **NOTE**: Changing the VM `machine_type` of a partition may not work. -> It is better to create a new partition and delete the old one. - -## Custom Images - -For more information on creating valid custom images for the controller VM -instance or for custom instance templates, see our [vm-images.md] documentation -page. - -[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images - -## GPU Support - -More information on GPU support in Slurm on GCP and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) - -## Reservation for Scheduled Maintenance - -A [maintenance event](https://cloud.google.com/compute/docs/instances/host-maintenance-overview#maintenanceevents) is when a compute engine stops a VM to perform a hardware or -software update which is determined by the host maintenance policy. This can -also affect the running jobs if the maintenance kicks in. Now, Customers can -protect jobs from getting terminated due to maintenance using the cluster -toolkit. You can enable creation of reservation for scheduled maintenance for -your compute nodeset and Slurm will reserve your node for maintenance during the -maintenance window. If you try to schedule any jobs which overlap with the -maintenance reservation, Slurm would not schedule any job. - -You can specify in your blueprint like - -```yaml - - id: compute_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network] - settings: - enable_maintenance_reservation: true -``` - -To enable creation of reservation for maintenance. - -While running job on slurm cluster, you can specify total run time of the job -using [-t flag](https://slurm.schedmd.com/srun.html#OPT_time).This would only -run the job outside of the maintenance window. - -```shell -srun -n1 -pcompute -t 10:00 -``` - -Currently upcoming maintenance notification is supported in ALPHA version of -compute API. You can update the API version from your blueprint, - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - settings: - endpoint_versions: - compute: "alpha" -``` - -## Opportunistic GCP maintenance in Slurm - -Customers can also enable running GCP maintenance as Slurm job opportunistically -to perform early maintenance. If a node is detected for maintenance, Slurm will -create a job to perform maintenance and put it in the job queue. - -If [backfill](https://slurm.schedmd.com/sched_config.html#backfill) scheduler is -used, Slurm will backfill maintenance job if it can find any empty time window. - -Customer can also choose builtin scheduler type. In this case, Slurm would run -maintenance job in strictly priority order. If the maintenance job doesn't kick -in, then forced maintenance will take place at scheduled window. - -Customer can enable this feature at nodeset level by, - -```yaml - - id: debug_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network] - settings: - enable_opportunistic_maintenance: true -``` - -## Placement Max Distance - -When using -[enable_placement](../../../../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md#input_enable_placement) -with Slurm, Google Compute Engine will attempt to place VMs as physically close -together as possible. Capacity constraints at the time of VM creation may still -force VMs to be spread across multiple racks. Google provides the `max-distance` -flag which can used to control the maximum spreading allowed. Read more about -`max-distance` in the -[official docs](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies -). - -You can use the `placement_max_distance` setting on the nodeset module to control the `max-distance` behavior. See the following example: - -```yaml - - id: nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [ network ] - settings: - machine_type: c2-standard-4 - node_count_dynamic_max: 30 - enable_placement: true - placement_max_distance: 1 - -> [!NOTE] -> `schedmd-slurm-gcp-v6-nodeset.settings.enable_placement: true` must also be -> set for placement_max_distance to take effect. - -In the above case using a value of 1 will restrict VM to be placed on the same -rack. You can confirm that the `max-distance` was applied by calling the -following command while jobs are running: - -```shell -gcloud beta compute resource-policies list \ - --format='yaml(name,groupPlacementPolicy.maxDistance)' -``` - -> [!WARNING] -> If a zone lacks capacity, using a lower `max-distance` value (such as 1) is -> more likely to cause VMs creation to fail. - -## TreeWidth and Node Communication - -Slurm uses a fan out mechanism to communicate large groups of nodes. The shape -of this fan out tree is determined by the -[TreeWidth](https://slurm.schedmd.com/slurm.conf.html#OPT_TreeWidth) -configuration variable. - -In the cloud, this fan out mechanism can become unstable when nodes restart with -new IP addresses. You can enforce that all nodes communicate directly with the -controller by setting TreeWidth to a value >= largest partition. - -If the largest partition was 200 nodes, configure the blueprint as follows: - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - ... - settings: - cloud_parameters: - tree_width: 200 -``` - -The default has been set to 128. Values above this have not been fully tested -and may cause congestion on the controller. A more scalable solution is under -way. - -## ResumeRate and Node Resumption - -The `ResumeRate` parameter in `slurm.conf` controls the maximum number of nodes -that Slurm attempts to resume (power up) per minute. This is particularly -important in cloud environments where auto-scaling can lead to a large number of -nodes starting concurrently. - -When many nodes start simultaneously, they can place a heavy load on shared -resources, especially shared filesystems, as they all try to mount filesystems -and access configuration files at the same time. By limiting the `ResumeRate`, -you can stagger the node startup process, reducing the peak load on these shared -resources and improving overall cluster stability during scaling events. - -For example, to limit the node resumption rate to 100 nodes per minute, -configure the blueprint as follows: - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - ... - settings: - cloud_parameters: - resume_rate: 100 -``` - -Adjust this value based on the capabilities of your shared filesystem and the -expected scaling behavior of your cluster. - -## Support -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 6.41 | -| [google-beta](#requirement\_google-beta) | >= 6.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.41 | -| [google-beta](#provider\_google-beta) | >= 6.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [bucket](#module\_bucket) | terraform-google-modules/cloud-storage/google | >= 6.1 | -| [daos\_network\_storage\_scripts](#module\_daos\_network\_storage\_scripts) | ../../../../modules/scripts/startup-script | n/a | -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [login](#module\_login) | ../../internal/slurm-gcp/login | n/a | -| [nodeset\_cleanup](#module\_nodeset\_cleanup) | ./modules/cleanup_compute | n/a | -| [nodeset\_cleanup\_tpu](#module\_nodeset\_cleanup\_tpu) | ./modules/cleanup_tpu | n/a | -| [slurm\_controller\_template](#module\_slurm\_controller\_template) | ../../internal/slurm-gcp/instance_template | n/a | -| [slurm\_files](#module\_slurm\_files) | ./modules/slurm_files | n/a | -| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | -| [slurm\_nodeset\_tpu](#module\_slurm\_nodeset\_tpu) | ../../internal/slurm-gcp/nodeset_tpu | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance_from_template.controller](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_from_template) | resource | -| [google_compute_disk.controller_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_secret_manager_secret.cloudsql](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_iam_member.cloudsql_secret_accessor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_version.cloudsql_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [google_storage_bucket_iam_member.legacy_readers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | -| [google_storage_bucket_iam_member.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | -| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_project.controller_project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = string
    device_name                = string
    disk_type                  = string
    disk_size_gb               = number
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. If not specified, then one will be chosen based on slurm\_cluster\_name. | `string` | `null` | no | -| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket.
Ignored when 'create\_bucket' is true. | `string` | `null` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | -| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Defaults inherited from [Slurm GCP repo](https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/terraform/slurm_cluster/modules/slurm_files/README_TF.md#input_cloud_parameters) | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | -| [cloudsql](#input\_cloudsql) | Use this database instead of the one on the controller.
server\_ip : Address of the database server.
user : The user to access the database as.
password : The password, given the user, to access the given database. (sensitive)
db\_name : The database to access.
user\_managed\_replication : The list of location and (optional) kms\_key\_name for secret | 
object({
    server_ip = string
    user      = string
    password  = string # sensitive
    db_name   = string
    user_managed_replication = optional(list(object({
      location     = string
      kms_key_name = optional(string)
    })), [])
  })
| `null` | no | -| [compute\_startup\_script](#input\_compute\_startup\_script) | DEPRECATED: `compute_startup_script` has been deprecated.
Use `startup_script` of nodeset module instead. | `any` | `null` | no | -| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script in compute nodes. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | -| [controller\_project\_id](#input\_controller\_project\_id) | Optionally. Provision controller and config bucket in the different project | `string` | `null` | no | -| [controller\_startup\_script](#input\_controller\_startup\_script) | Startup script used by the controller VM. | `string` | `"# no-op"` | no | -| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    type = string
    size = number
  })
| 
{
  "size": 50,
  "type": "pd-ssd"
}
| no | -| [create\_bucket](#input\_create\_bucket) | Create GCS bucket instead of using an existing one. | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment. | `string` | n/a | yes | -| [disable\_controller\_public\_ips](#input\_disable\_controller\_public\_ips) | DEPRECATED: Use `enable_controller_public_ips` instead. | `bool` | `null` | no | -| [disable\_default\_mounts](#input\_disable\_default\_mounts) | DEPRECATED: Use `enable_default_mounts` instead. | `bool` | `null` | no | -| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | -| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | `true` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_controller\_public\_ips](#input\_enable\_controller\_public\_ips) | If set to true. The controller will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. | `bool` | `false` | no | -| [enable\_default\_mounts](#input\_enable\_default\_mounts) | Enable default global network storage from the controller
- /home
- /opt/apps | `bool` | `true` | no | -| [enable\_devel](#input\_enable\_devel) | DEPRECATED: `enable_devel` is always on. | `bool` | `null` | no | -| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/tools/prologs-epilogs/README.md | `bool` | `null` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | -| [enable\_slurm\_gcp\_plugins](#input\_enable\_slurm\_gcp\_plugins) | DEPRECATED: Slurm GCP plugins have been deprecated.
Instead of 'max\_hops' plugin please use the 'placement\_max\_distance' nodeset property.
Instead of 'enable\_vpmu' plugin please use 'advanced\_machine\_features.performance\_monitoring\_unit' nodeset property. | `any` | `null` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": "beta"
}
| no | -| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for controller. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | -| [login\_network\_storage](#input\_login\_network\_storage) | An array of network attached storage mounts to be configured on all login nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [login\_nodes](#input\_login\_nodes) | List of slurm login instance definitions. | 
list(object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  }))
| `[]` | no | -| [login\_startup\_script](#input\_login\_startup\_script) | Startup script used by the login VMs. | `string` | `"# no-op"` | no | -| [login\_startup\_scripts\_timeout](#input\_login\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in login\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = optional(map(string))
    mount_runner          = optional(map(string))
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | Define nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    accelerator_topology = optional(string, null)
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | -| [partitions](#input\_partitions) | Cluster partitions as a list. See module slurm\_partition. | 
list(object({
    partition_name        = string
    partition_conf        = optional(map(string), {})
    partition_nodeset     = optional(list(string), [])
    partition_nodeset_dyn = optional(list(string), [])
    partition_nodeset_tpu = optional(list(string), [])
    enable_job_exclusive  = optional(bool, false)
  }))
| `[]` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [region](#input\_region) | The default region to place resources in. | `string` | n/a | yes | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the controller instance. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the controller instance. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming and slurm accounting.
If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). | `string` | `null` | no | -| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | -| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | -| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Post deployment instructions. | -| [slurm\_bucket](#output\_slurm\_bucket) | GCS Bucket of Slurm cluster file storage. | -| [slurm\_bucket\_dir](#output\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | -| [slurm\_bucket\_name](#output\_slurm\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | -| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | Bucket path used by cluster. | -| [slurm\_cluster\_name](#output\_slurm\_cluster\_name) | Slurm cluster name. | -| [slurm\_controller\_instance](#output\_slurm\_controller\_instance) | Compute instance of controller node | -| [slurm\_login\_instances](#output\_slurm\_login\_instances) | Compute instances of login nodes | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf deleted file mode 100644 index 4a887b99cf..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.disk_resource_manager_tags - } - ] - - state_disk = var.controller_state_disk != null ? [{ - source = google_compute_disk.controller_disk[0].name - device_name = google_compute_disk.controller_disk[0].name - disk_labels = null - auto_delete = false - boot = false - }] : [] - - synth_def_sa_email = "${data.google_project.controller_project.number}-compute@developer.gserviceaccount.com" - - service_account = { - email = coalesce(var.service_account_email, local.synth_def_sa_email) - scopes = var.service_account_scopes - } - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata, - local.universe_domain - ) - - controller_project_id = coalesce(var.controller_project_id, var.project_id) -} - -data "google_project" "controller_project" { - project_id = local.controller_project_id -} - -resource "google_compute_disk" "controller_disk" { - count = var.controller_state_disk != null ? 1 : 0 - - project = local.controller_project_id - name = "${local.slurm_cluster_name}-controller-save" - type = var.controller_state_disk.type - size = var.controller_state_disk.size - zone = var.zone -} - -# INSTANCE TEMPLATE -module "slurm_controller_template" { - source = "../../internal/slurm-gcp/instance_template" - - project_id = local.controller_project_id - region = var.region - slurm_instance_role = "controller" - slurm_cluster_name = local.slurm_cluster_name - labels = local.labels - - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(var.disk_labels, local.labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = concat(local.additional_disks, local.state_disk) - - bandwidth_tier = var.bandwidth_tier - slurm_bucket_path = module.slurm_files.slurm_bucket_path - can_ip_forward = var.can_ip_forward - advanced_machine_features = var.advanced_machine_features - resource_manager_tags = var.resource_manager_tags - - enable_confidential_vm = var.enable_confidential_vm - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - gpu = one(module.gpu.guest_accelerator) - - machine_type = var.machine_type - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - service_account = local.service_account - - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - subnetwork = var.subnetwork_self_link - - tags = concat([local.slurm_cluster_name], var.tags) - # termination_action = TODO: add support for termination_action (?) -} - -# INSTANCE -resource "google_compute_instance_from_template" "controller" { - provider = google-beta - - name = "${local.slurm_cluster_name}-controller" - project = local.controller_project_id - zone = var.zone - source_instance_template = module.slurm_controller_template.self_link - # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 - # we have to explicitly override instance labels instead of inheriting them from template. - labels = module.slurm_controller_template.labels - - allow_stopping_for_update = true - - # Can't rely on template to specify nics due to usage of static_ip - network_interface { - dynamic "access_config" { - for_each = var.enable_controller_public_ips ? ["unit"] : [] - content { - nat_ip = null - network_tier = null - } - } - network_ip = length(var.static_ips) == 0 ? "" : var.static_ips[0] - subnetwork = var.subnetwork_self_link - } - - dynamic "network_interface" { - for_each = var.controller_network_attachment != null ? [1] : [] - content { - network_attachment = var.controller_network_attachment - } - } -} - -moved { - from = module.slurm_controller_instance.google_compute_instance_from_template.slurm_instance[0] - to = google_compute_instance_from_template.controller -} - -# SECRETS: CLOUDSQL -resource "google_secret_manager_secret" "cloudsql" { - count = var.cloudsql != null ? 1 : 0 - - secret_id = "${local.slurm_cluster_name}-slurm-secret-cloudsql" - project = var.project_id - - replication { - dynamic "auto" { - for_each = length(var.cloudsql.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.cloudsql.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = nonsensitive(var.cloudsql.user_managed_replication) - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } - - labels = { - slurm_cluster_name = local.slurm_cluster_name - } -} - -resource "google_secret_manager_secret_version" "cloudsql_version" { - count = var.cloudsql != null ? 1 : 0 - - secret = google_secret_manager_secret.cloudsql[0].id - secret_data = jsonencode(var.cloudsql) -} - -resource "google_secret_manager_secret_iam_member" "cloudsql_secret_accessor" { - count = var.cloudsql != null ? 1 : 0 - - secret_id = google_secret_manager_secret.cloudsql[0].id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${local.service_account.email}" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl deleted file mode 100644 index 219bdc5227..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl +++ /dev/null @@ -1,65 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/high_throughput.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxArraySize=10001 -MaxJobCount=500000 -MaxNodeCount=65536 -MinJobAge=60 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -SlurmctldDebug=error -SlurmdDebug=error - -# -# -# TIMERS -MessageTimeout=60 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=120 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl deleted file mode 100644 index 93ac47e341..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl +++ /dev/null @@ -1,34 +0,0 @@ -# slurmdbd.conf -# https://slurm.schedmd.com/slurmdbd.conf.html - -DebugLevel=info -PidFile=/var/run/slurm/slurmdbd.pid - -# https://slurm.schedmd.com/slurmdbd.conf.html#OPT_CommitDelay -CommitDelay=1 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -AuthType=auth/{auth_key} -AuthAltTypes=auth/jwt -AuthAltParameters=jwt_key={state_save}/jwt_hs256.key - -DbdHost={control_host} - -LogFile={slurmlog}/slurmdbd.log - -SlurmUser=slurm - -StorageLoc={db_name} - -StorageType=accounting_storage/mysql -StorageHost={db_host} -StoragePort={db_port} -StorageUser={db_user} -StoragePass={db_pass} - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl deleted file mode 100644 index d3f2615a68..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl +++ /dev/null @@ -1,71 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/slurm.conf.html -# https://slurm.schedmd.com/configurator.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxNodeCount=64000 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -AccountingStoreFlags=job_comment -JobAcctGatherFrequency=30 -JobAcctGatherType=jobacct_gather/cgroup -SlurmctldDebug=info -SlurmdDebug=info -DebugFlags=Power - -# -# -# TIMERS -MessageTimeout=600 -BatchStartTimeout=600 -PrologEpilogTimeout=600 -PrologFlags=Contain - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=600 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf deleted file mode 100644 index 21e915a125..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -locals { - # TODO: deprecate `var.login_[ startup_script, startup_scripts_timeout, network_storage]` - # in favour of vars defined in user-facing login module - ghpc_startup_login = [{ - filename = "ghpc_startup.sh" - content = var.login_startup_script - }] - - login_startup_scripts = concat(local.common_scripts, local.ghpc_startup_login) -} - -module "login" { - source = "../../internal/slurm-gcp/login" - for_each = { for x in var.login_nodes : x.group_name => x } - - project_id = var.project_id - - slurm_cluster_name = local.slurm_cluster_name - slurm_bucket_path = module.slurm_files.slurm_bucket_path - slurm_bucket_name = module.slurm_files.bucket_name - slurm_bucket_dir = module.slurm_files.bucket_dir - - login_nodes = each.value - - startup_scripts = local.login_startup_scripts - startup_scripts_timeout = var.login_startup_scripts_timeout - - network_storage = var.login_network_storage - - universe_domain = var.universe_domain - - # trigger replacement of login nodes when the controller instance is replaced - # Needed for re-mounting volumes hosted on controller - replace_trigger = google_compute_instance_from_template.controller.self_link -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf deleted file mode 100644 index 7622bdffef..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-controller", ghpc_role = "scheduler" }) -} - -locals { - # Since deployment name may be used to create a cluster name, we remove any invalid character from the beginning - # Also, slurm imposed a lot of restrictions to this name, so we format it to an acceptable string - tmp_cluster_name = substr(replace(lower(var.deployment_name), "/^[^a-z]*|[^a-z0-9]/", ""), 0, 10) - slurm_cluster_name = coalesce(var.slurm_cluster_name, local.tmp_cluster_name) - - universe_domain = { "universe_domain" = var.universe_domain } -} - -# See -# * slurm_files.tf -# * controller.tf -# * partition.tf -# * login.tf diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml deleted file mode 100644 index 7b4918b962..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - iam.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md deleted file mode 100644 index 002bf14145..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md +++ /dev/null @@ -1,42 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name         = string
    subnetwork_self_link = string
    additional_networks = list(object({
      subnetwork = string
    }))
  })
| n/a | yes | -| [nodeset\_template](#input\_nodeset\_template) | Self link of the nodeset template | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf deleted file mode 100644 index bd8773cf84..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - cleanup_dependencies_agg = flatten([ - var.nodeset.subnetwork_self_link, - var.nodeset.additional_networks[*].subnetwork, - var.nodeset_template]) -} - -# Can not use variadic list in `depends_on`, wrap it into a collection of `null_resource` -resource "null_resource" "dependencies" { - count = length(local.cleanup_dependencies_agg) -} - -resource "null_resource" "script" { - count = var.enable_cleanup_compute ? 1 : 0 - - triggers = { - project_id = var.project_id - cluster_name = var.slurm_cluster_name - nodeset_name = var.nodeset.nodeset_name - universe_domain = var.universe_domain - compute_endpoint_version = var.endpoint_versions.compute - gcloud_path_override = var.gcloud_path_override - } - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/cleanup_compute.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" - when = destroy - } - - # Ensure that clean up is done before attempt to delete the networks - depends_on = [null_resource.dependencies] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh deleted file mode 100644 index a98243d464..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -project="$1" -cluster_name="$2" -nodeset_name="$3" -universe_domain="$4" -compute_endpoint_version="$5" -gcloud_dir="$6" -MAX_ATTEMPTS=3 - -if [[ $# -ne 5 ]] && [[ $# -ne 6 ]]; then - echo "Usage: $0 []" - exit 1 -fi - -if [[ -n "${gcloud_dir}" ]]; then - export PATH="$gcloud_dir:$PATH" -fi - -export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" -export CLOUDSDK_CORE_PROJECT="${project}" - -if ! type -P gcloud 1>/dev/null; then - echo "gcloud is not available and your compute resources are not being cleaned up" - echo "https://console.cloud.google.com/compute/instances?project=${project}" - exit 1 -fi - -tmpfile=$(mktemp) # have to use a temp file, since `< <(gcloud ...)` doesn't work nicely with `head` -trap 'rm -f "$tmpfile"' EXIT - -echo "Deleting managed instance groups" -mig_filter="name:${cluster_name}-${nodeset_name}-*" -gcloud compute instance-groups managed list --format="value(self_link)" --filter="${mig_filter}" >"$tmpfile" -while batch="$(head -n 5)" && [[ ${#batch} -gt 0 ]]; do - groups=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line - # The lack of quotes around ${groups} is intentional and causes each new space-separated "word" to - # be treated as independent arguments. See PR#2523 - # shellcheck disable=SC2086 - for _ in $( #occasionally MIGs will fail to delete due to some active transformation happening, so let's retry - seq 1 $MAX_ATTEMPTS - ); do - if gcloud compute instance-groups managed delete --quiet ${groups}; then - break - fi - echo "MIG deletion failed, retrying" - done -done <"$tmpfile" -true >"$tmpfile" # Wipe contents of tmp file - -echo "Deleting compute nodes" -node_filter="name:${cluster_name}-${nodeset_name}-* labels.slurm_cluster_name=${cluster_name} AND labels.slurm_instance_role=compute" - -running_nodes_filter="${node_filter} AND status!=STOPPING" -# List all currently running instances and attempt to delete them -gcloud compute instances list --format="value(selfLink)" --filter="${running_nodes_filter}" >"$tmpfile" -# Do 500 instances at a time -while batch="$(head -n 500)" && [[ ${#batch} -gt 0 ]]; do - nodes=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line - # The lack of quotes around ${nodes} is intentional and causes each new space-separated "word" to - # be treated as independent arguments. See PR#2523 - # shellcheck disable=SC2086 - gcloud compute instances delete --quiet ${nodes} || echo "Failed to delete some instances" -done <"$tmpfile" - -# In case if controller tries to delete the nodes as well, -# wait until nodes in STOPPING state are deleted, before deleting the resource policies -stopping_nodes_filter="${node_filter} AND status=STOPPING" -while true; do - node=$(gcloud compute instances list --format="value(name)" --filter="${stopping_nodes_filter}" --limit=1) - if [[ -z "${node}" ]]; then - break - fi - echo "Waiting for instances to be deleted: ${node}" - sleep 5 -done - -echo "Deleting resource policies" -policies_filter="name:${cluster_name}-slurmgcp-managed-${nodeset_name}-*" -gcloud compute resource-policies list --format="value(selfLink)" --filter="${policies_filter}" | while read -r line; do - echo "Deleting resource policy: $line" - gcloud compute resource-policies delete --quiet "${line}" || { - echo "Failed to delete resource policy: $line" - } -done diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf deleted file mode 100644 index b6da69931c..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "Project ID" -} - - -variable "slurm_cluster_name" { - type = string - description = "Name of the Slurm cluster" -} - -variable "enable_cleanup_compute" { - description = < [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | 3.2.3 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf deleted file mode 100644 index ec86a03a24..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "null_resource" "script" { - count = var.enable_cleanup_compute ? 1 : 0 - - triggers = { - project_id = var.project_id - cluster_name = var.slurm_cluster_name - nodeset_name = var.nodeset.nodeset_name - zone = var.nodeset.zone - universe_domain = var.universe_domain - compute_endpoint_version = var.endpoint_versions.compute - gcloud_path_override = var.gcloud_path_override - } - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/cleanup_tpu.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.zone} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" - when = destroy - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh deleted file mode 100644 index c724e342c3..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -project="$1" -cluster_name="$2" -nodeset_name="$3" -zone="$4" -universe_domain="$5" -compute_endpoint_version="$6" -gcloud_dir="$7" - -if [[ $# -ne 6 ]] && [[ $# -ne 7 ]]; then - echo "Usage: $0 []" - exit 1 -fi - -if [[ -n "${gcloud_dir}" ]]; then - export PATH="$gcloud_dir:$PATH" -fi - -export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" -export CLOUDSDK_CORE_PROJECT="${project}" - -if ! type -P gcloud 1>/dev/null; then - echo "gcloud is not available and your compute resources are not being cleaned up" - echo "https://console.cloud.google.com/compute/instances?project=${project}" - exit 1 -fi - -echo "Deleting TPU nodes" -node_filter="name~${cluster_name}-${nodeset_name}" -running_nodes_filter="${node_filter} AND state!=DELETING" - -# List all currently running nodes and attempt to delete them -gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${running_nodes_filter}" | while read -r name; do - echo "Deleting TPU node: $name" - gcloud compute tpus tpu-vm delete --async --zone="${zone}" --quiet "${name}" || echo "Failed to delete $name" -done - -# Wait until nodes in DELETING state are deleted, before deleting the resource policies -deleting_nodes_filter="${node_filter} AND state=DELETING" -while true; do - node=$(gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${deleting_nodes_filter}" --limit=1) - if [[ -z "${node}" ]]; then - break - fi - echo "Waiting for nodes to be deleted: ${node}" - sleep 5 -done diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf deleted file mode 100644 index 1ac6f64b75..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright (C) Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "Project ID" -} - -variable "slurm_cluster_name" { - type = string - description = "Name of the Slurm cluster" -} - -variable "enable_cleanup_compute" { - description = < -Copyright (C) SchedMD LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.3 | -| [archive](#requirement\_archive) | ~> 2.0 | -| [google](#requirement\_google) | >= 6.41 | -| [local](#requirement\_local) | ~> 2.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [archive](#provider\_archive) | ~> 2.0 | -| [google](#provider\_google) | >= 6.41 | -| [local](#provider\_local) | ~> 2.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.controller_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.devel](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.devel_compute](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_dyn_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_tpu_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.task_epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.task_prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [random_uuid.cluster_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/uuid) | resource | -| [archive_file.slurm_gcp_devel_compute_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | -| [archive_file.slurm_gcp_devel_controller_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | -| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | -| [local_file.chs_gpu_health_check](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.external_epilog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.external_prolog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.setup_external](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. | `string` | `null` | no | -| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket to use. | `string` | n/a | yes | -| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | -| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Default behavior defined in scripts/conf.py | 
object({
    no_comma_params         = optional(bool, false)
    private_data            = optional(list(string))
    scheduler_parameters    = optional(list(string))
    resume_rate             = optional(number)
    resume_timeout          = optional(number)
    suspend_rate            = optional(number)
    suspend_timeout         = optional(number)
    slurmd_timeout          = optional(number)
    unkillable_step_timeout = optional(number)
    topology_plugin         = optional(string)
    topology_param          = optional(string)
    tree_width              = optional(number)
    prolog_flags            = optional(string)
    switch_type             = optional(string)
  })
| `{}` | no | -| [cloudsql\_secret](#input\_cloudsql\_secret) | Secret URI to cloudsql secret. | `string` | `null` | no | -| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in compute\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | -| [controller\_startup\_scripts](#input\_controller\_startup\_scripts) | List of scripts to be ran on controller VM startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | -| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    device_name = string
  })
| 
{
  "device_name": null
}
| no | -| [disable\_default\_mounts](#input\_disable\_default\_mounts) | Disable default global network storage from the controller
- /home
- /apps | `bool` | `false` | no | -| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. Not for production use. | `bool` | `false` | no | -| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/v5/tools/prologs-epilogs/README.md | `bool` | `false` | no | -| [enable\_hybrid](#input\_enable\_hybrid) | Enables use of hybrid controller mode. When true, controller\_hybrid\_config will
be used instead of controller\_instance\_config and will disable login instances. | `bool` | `false` | no | -| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": null
}
| no | -| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | -| [google\_app\_cred\_path](#input\_google\_app\_cred\_path) | Path to Google Application Credentials. | `string` | `null` | no | -| [install\_dir](#input\_install\_dir) | Directory where the hybrid configuration directory will be installed on the
on-premise controller (e.g. /etc/slurm/hybrid). This updates the prefix path
for the resume and suspend scripts in the generated `cloud.conf` file.

This variable should be used when the TerraformHost and the SlurmctldHost
are different.

This will default to var.output\_dir if null. | `string` | `null` | no | -| [munge\_mount](#input\_munge\_mount) | Remote munge mount for compute and login nodes to acquire the munge.key.
By default, the munge mount server will be assumed to be the
`var.slurm_control_host` (or `var.slurm_control_addr` if non-null) when
`server_ip=null`. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| 
{
  "fs_type": "nfs",
  "mount_options": "",
  "remote_mount": "/etc/munge/",
  "server_ip": null
}
| no | -| [network\_storage](#input\_network\_storage) | Storage to mounted on all instances.
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | Cluster nodenets, as a list. | `list(any)` | `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Cluster nodenets (dynamic), as a list. | `list(any)` | `[]` | no | -| [nodeset\_startup\_scripts](#input\_nodeset\_startup\_scripts) | List of scripts to be ran on compute VM startup in the specific nodeset. | 
map(list(object({
    filename = string
    content  = string
  })))
| `{}` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Cluster nodenets (TPU), as a list. | `list(any)` | `[]` | no | -| [output\_dir](#input\_output\_dir) | Directory where this module will write its files to. These files include:
cloud.conf; cloud\_gres.conf; config.yaml; resume.py; suspend.py; and util.py. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The GCP project ID. | `string` | n/a | yes | -| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [slurm\_bin\_dir](#input\_slurm\_bin\_dir) | Path to directory of Slurm binary commands (e.g. scontrol, sinfo). If 'null',
then it will be assumed that binaries are in $PATH. | `string` | `null` | no | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | The cluster name, used for resource naming and slurm accounting. | `string` | n/a | yes | -| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | -| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | -| [slurm\_control\_addr](#input\_slurm\_control\_addr) | The IP address or a name by which the address can be identified.

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | -| [slurm\_control\_host](#input\_slurm\_control\_host) | The short, or long, hostname of the machine where Slurm control daemon is
executed (i.e. the name returned by the command "hostname -s").

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | -| [slurm\_control\_host\_port](#input\_slurm\_control\_host\_port) | The port number that the Slurm controller, slurmctld, listens to for work.

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldPort | `string` | `"6818"` | no | -| [slurm\_key\_mount](#input\_slurm\_key\_mount) | Remote mount for compute and login nodes to acquire the slurm.key. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| `null` | no | -| [slurm\_log\_dir](#input\_slurm\_log\_dir) | Directory where Slurm logs to. | `string` | `"/var/log/slurm"` | no | -| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | -| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [bucket\_dir](#output\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | -| [bucket\_name](#output\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | -| [config](#output\_config) | Cluster configuration. | -| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl deleted file mode 100644 index ffeb167cfc..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl +++ /dev/null @@ -1,7 +0,0 @@ -# cgroup.conf -# https://slurm.schedmd.com/cgroup.conf.html - -ConstrainCores=yes -ConstrainRamSpace=yes -ConstrainSwapSpace=no -ConstrainDevices=yes diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl deleted file mode 100644 index 4951289842..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl +++ /dev/null @@ -1,67 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/slurm.conf.html -# https://slurm.schedmd.com/configurator.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxNodeCount=64000 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -AccountingStoreFlags=job_comment -JobAcctGatherFrequency=30 -JobAcctGatherType=jobacct_gather/cgroup -SlurmctldDebug=info -SlurmdDebug=info -DebugFlags=Power - -# -# -# TIMERS -MessageTimeout=60 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=120 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl deleted file mode 100644 index 8c90a9dfbe..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl +++ /dev/null @@ -1,31 +0,0 @@ -# slurmdbd.conf -# https://slurm.schedmd.com/slurmdbd.conf.html - -DebugLevel=info -PidFile=/var/run/slurm/slurmdbd.pid - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -AuthType=auth/{auth_key} -AuthAltTypes=auth/jwt -AuthAltParameters=jwt_key={state_save}/jwt_hs256.key - -DbdHost={control_host} - -LogFile={slurmlog}/slurmdbd.log - -SlurmUser=slurm - -StorageLoc={db_name} - -StorageType=accounting_storage/mysql -StorageHost={db_host} -StoragePort={db_port} -StorageUser={db_user} -StoragePass={db_pass} - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh deleted file mode 100644 index db514fc9e5..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -x /opt/apps/adm/slurm/slurm_epilog ]]; then - exec /opt/apps/adm/slurm/slurm_epilog -fi diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh deleted file mode 100644 index 37a91bb1ea..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -x /opt/apps/adm/slurm/slurm_prolog ]]; then - exec /opt/apps/adm/slurm/slurm_prolog -fi diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh deleted file mode 100644 index 0877ff3b19..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SLURM_EXTERNAL_ROOT="/opt/apps/adm/slurm" -SLURM_MUX_FILE="slurm_mux" - -mkdir -p "${SLURM_EXTERNAL_ROOT}" -mkdir -p "${SLURM_EXTERNAL_ROOT}/logs" -mkdir -p "${SLURM_EXTERNAL_ROOT}/etc" - -# create common prolog / epilog "multiplex" script -if [ ! -f "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" ]; then - # indentation matters in EOT below; do not blindly edit! - cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -CMD="${0##*/}" -# Locate script -BASE=$(readlink -f $0) -BASE=${BASE%/*} - -export CLUSTER_ADM_BASE=${BASE} - -# Source config file if it exists for extra DEBUG settings -# used below -SLURM_MUX_CONF=${CLUSTER_ADM_BASE}/etc/slurm_mux.conf -if [[ -r ${SLURM_MUX_CONF} ]]; then - source ${SLURM_MUX_CONF} -fi - -# Setup logging if configured and directory exists -LOGFILE="/dev/null" -if [[ -d ${DEBUG_SLURM_MUX_LOG_DIR} && ${DEBUG_SLURM_MUX_ENABLE_LOG} == "yes" ]]; then - LOGFILE="${DEBUG_SLURM_MUX_LOG_DIR}/${CMD}-${SLURM_SCRIPT_CONTEXT}-job-${SLURMD_NODENAME}.log" - exec >>${LOGFILE} 2>&1 -fi - -# Global scriptlets -for SCRIPTLET in ${BASE}/${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do - if [[ -x ${SCRIPTLET} ]]; then - echo "Running ${SCRIPTLET}" - ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 - echo "Running ${SCRIPTLET} returned $?" - fi -done - -# Per partition scriptlets -for SCRIPTLET in ${BASE}/partition-${SLURM_JOB_PARTITION}-${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do - if [[ -x ${SCRIPTLET} ]]; then - echo "Running ${SCRIPTLET}" - ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 - echo "Running ${SCRIPTLET} returned $?" - fi -done -EOT -fi - -# ensure proper permissions on slurm_mux script -chmod 0755 "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" - -# create default slurm_mux configuration file -if [ ! -f "${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" ]; then - cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" -# these settings are intended for temporary debugging purposes only; leaving -# them enabled will write files for each job to a shared NFS directory without -# any automated cleanup -DEBUG_SLURM_MUX_LOG_DIR=/opt/apps/adm/slurm/logs -DEBUG_SLURM_MUX_ENABLE_LOG=no -EOT -fi - -# create epilog symbolic link -if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_epilog" ]; then - cd ${SLURM_EXTERNAL_ROOT} - # delete existing file if necessary - rm -f slurm_epilog - ln -s ${SLURM_MUX_FILE} slurm_epilog - cd - >/dev/null -fi - -# create prolog symbolic link -if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_prolog" ]; then - cd ${SLURM_EXTERNAL_ROOT} - # delete existing file if necessary - rm -f slurm_prolog - ln -s ${SLURM_MUX_FILE} slurm_prolog - cd - >/dev/null -fi diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf deleted file mode 100644 index e63b2d1100..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf +++ /dev/null @@ -1,406 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - scripts_dir = abspath("${path.module}/scripts") - - bucket_dir = coalesce(var.bucket_dir, format("%s-files", var.slurm_cluster_name)) -} - -######## -# DATA # -######## - -data "google_storage_bucket" "this" { - name = var.bucket_name -} - -########## -# RANDOM # -########## - -resource "random_uuid" "cluster_id" { -} - -################## -# CLUSTER CONFIG # -################## - -locals { - config = { - enable_bigquery_load = var.enable_bigquery_load - cloudsql_secret = var.cloudsql_secret - cluster_id = random_uuid.cluster_id.result - project = var.project_id - slurm_cluster_name = var.slurm_cluster_name - enable_slurm_auth = var.enable_slurm_auth - bucket_path = local.bucket_path - enable_debug_logging = var.enable_debug_logging - extra_logging_flags = var.extra_logging_flags - controller_state_disk = var.controller_state_disk - - # storage - disable_default_mounts = var.disable_default_mounts - network_storage = var.network_storage - - # timeouts - controller_startup_scripts_timeout = var.controller_startup_scripts_timeout - compute_startup_scripts_timeout = var.compute_startup_scripts_timeout - - munge_mount = local.munge_mount - slurm_key_mount = var.slurm_key_mount - - # slurm conf - prolog_scripts = [for k, v in google_storage_bucket_object.prolog_scripts : k] - epilog_scripts = [for k, v in google_storage_bucket_object.epilog_scripts : k] - task_prolog_scripts = [for k, v in google_storage_bucket_object.task_prolog_scripts : k] - task_epilog_scripts = [for k, v in google_storage_bucket_object.task_epilog_scripts : k] - cloud_parameters = var.cloud_parameters - - # hybrid - hybrid = var.enable_hybrid - google_app_cred_path = var.enable_hybrid ? local.google_app_cred_path : null - output_dir = var.enable_hybrid ? local.output_dir : null - install_dir = var.enable_hybrid ? local.install_dir : null - slurm_control_host = var.enable_hybrid ? var.slurm_control_host : null - slurm_control_host_port = var.enable_hybrid ? local.slurm_control_host_port : null - slurm_control_addr = var.enable_hybrid ? var.slurm_control_addr : null - slurm_bin_dir = var.enable_hybrid ? local.slurm_bin_dir : null - slurm_log_dir = var.enable_hybrid ? local.slurm_log_dir : null - controller_network_attachment = var.controller_network_attachment - - - # config files templates - slurmdbd_conf_tpl = file(coalesce(var.slurmdbd_conf_tpl, "${local.etc_dir}/slurmdbd.conf.tpl")) - slurm_conf_tpl = var.slurm_conf_template != null ? var.slurm_conf_template : file(coalesce(var.slurm_conf_tpl, "${local.etc_dir}/slurm.conf.tpl")) - cgroup_conf_tpl = file(coalesce(var.cgroup_conf_tpl, "${local.etc_dir}/cgroup.conf.tpl")) - - # Providers - endpoint_versions = var.endpoint_versions - } - - x_nodeset = toset(var.nodeset[*].nodeset_name) - x_nodeset_dyn = toset(var.nodeset_dyn[*].nodeset_name) - x_nodeset_tpu = toset(var.nodeset_tpu[*].nodeset.nodeset_name) - x_nodeset_overlap = setintersection([], local.x_nodeset, local.x_nodeset_dyn, local.x_nodeset_tpu) - - etc_dir = abspath("${path.module}/etc") - - bucket_path = format("%s/%s", data.google_storage_bucket.this.url, local.bucket_dir) - - slurm_control_host_port = coalesce(var.slurm_control_host_port, "6818") - - google_app_cred_path = var.google_app_cred_path != null ? abspath(var.google_app_cred_path) : null - slurm_bin_dir = var.slurm_bin_dir != null ? abspath(var.slurm_bin_dir) : null - slurm_log_dir = var.slurm_log_dir != null ? abspath(var.slurm_log_dir) : null - - munge_mount = var.enable_hybrid ? { - server_ip = lookup(var.munge_mount, "server_ip", coalesce(var.slurm_control_addr, var.slurm_control_host)) - remote_mount = lookup(var.munge_mount, "remote_mount", "/etc/munge/") - fs_type = lookup(var.munge_mount, "fs_type", "nfs") - mount_options = lookup(var.munge_mount, "mount_options", "") - } : null - - output_dir = can(coalesce(var.output_dir)) ? abspath(var.output_dir) : abspath(".") - install_dir = can(coalesce(var.install_dir)) ? abspath(var.install_dir) : local.output_dir -} - -resource "google_storage_bucket_object" "config" { - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/config.yaml" - content = yamlencode(local.config) - source_md5hash = md5(yamlencode(local.config)) - - # Take dependency on all other "config artifacts" so creation of `config.yaml` - # can be used as a signal for setup.py that "everything is ready". - # Some of following files, particularly mount scripts for new NFSes, can take a while to be created. - depends_on = [ - google_storage_bucket_object.controller_startup_scripts, - google_storage_bucket_object.nodeset_startup_scripts, - google_storage_bucket_object.prolog_scripts, - google_storage_bucket_object.epilog_scripts, - google_storage_bucket_object.task_prolog_scripts, - google_storage_bucket_object.task_epilog_scripts - ] -} - -resource "google_storage_bucket_object" "nodeset_config" { - for_each = { for ns in var.nodeset : ns.nodeset_name => merge(ns, { - instance_properties = jsondecode(ns.instance_properties_json) - }) } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -resource "google_storage_bucket_object" "nodeset_dyn_config" { - for_each = { for ns in var.nodeset_dyn : ns.nodeset_name => ns } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_dyn_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -resource "google_storage_bucket_object" "nodeset_tpu_config" { - for_each = { for n in var.nodeset_tpu[*].nodeset : n.nodeset_name => n } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_tpu_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -######### -# DEVEL # -######### - -locals { - build_dir = abspath("${path.module}/build") - - slurm_gcp_devel_controller_zip = "slurm-gcp-devel-controller.zip" - slurm_gcp_devel_compute_zip = "slurm-gcp-devel.zip" - slurm_gcp_devel_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_controller_zip) - slurm_gcp_devel_compute_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_compute_zip) - - controller_files = [ - "tools/gpu-test", - "tools/task-epilog", - "tools/task-prolog", - "conf.py", - "file_cache.py", - "get_tpu_vmcount.py", - "job_submit.lua.tpl", - "load_bq.py", - "local_pubsub.py", - "mig_flex.py", - "resume_wrapper.sh", - "resume.py", - "setup_network_storage.py", - "setup.py", - "slurmsync.py", - "sort_nodes.py", - "suspend_wrapper.sh", - "suspend.py", - "tpu.py", - "util.py", - "watch_delete_vm_op.py", - ] - - compute_files = [ - "tools/gpu-test", - "tools/task-epilog", - "tools/task-prolog", - "file_cache.py", - "get_tpu_vmcount.py", - "job_submit.lua.tpl", - "local_pubsub.py", - "mig_flex.py", - "setup_network_storage.py", - "setup.py", - "slurmsync.py", - "sort_nodes.py", - "suspend.py", - "tpu.py", - "util.py", - "watch_delete_vm_op.py", - ] -} - -data "archive_file" "slurm_gcp_devel_controller_zip" { - output_path = "${local.build_dir}/${local.slurm_gcp_devel_controller_zip}" - type = "zip" - - dynamic "source" { - for_each = local.controller_files - content { - content = file("${local.scripts_dir}/${source.value}") - filename = source.value - } - } -} - -data "archive_file" "slurm_gcp_devel_compute_zip" { - output_path = "${local.build_dir}/${local.slurm_gcp_devel_compute_zip}" - type = "zip" - - dynamic "source" { - for_each = local.compute_files - content { - content = file("${local.scripts_dir}/${source.value}") - filename = source.value - } - } -} - -resource "google_storage_bucket_object" "devel" { - bucket = var.bucket_name - name = local.slurm_gcp_devel_zip_bucket - source = data.archive_file.slurm_gcp_devel_controller_zip.output_path - source_md5hash = data.archive_file.slurm_gcp_devel_controller_zip.output_md5 -} - -resource "google_storage_bucket_object" "devel_compute" { - bucket = var.bucket_name - name = local.slurm_gcp_devel_compute_zip_bucket - source = data.archive_file.slurm_gcp_devel_compute_zip.output_path - source_md5hash = data.archive_file.slurm_gcp_devel_compute_zip.output_md5 -} - -########### -# SCRIPTS # -########### - -resource "google_storage_bucket_object" "controller_startup_scripts" { - for_each = { - for x in local.controller_startup_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-controller-script-%s", local.bucket_dir, each.key) - content = each.value.content - source_md5hash = md5(each.value.content) -} - -resource "google_storage_bucket_object" "nodeset_startup_scripts" { - for_each = { for x in flatten([ - for nodeset, scripts in var.nodeset_startup_scripts - : [for s in scripts - : { - content = s.content, - name = format("slurm-nodeset-%s-script-%s", nodeset, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_")) } - ]]) : x.name => x.content } - - bucket = var.bucket_name - name = format("%s/%s", local.bucket_dir, each.key) - content = each.value - source_md5hash = md5(each.value) -} - -resource "google_storage_bucket_object" "prolog_scripts" { - for_each = { - for x in local.prolog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-prolog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -resource "google_storage_bucket_object" "epilog_scripts" { - for_each = { - for x in local.epilog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-epilog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -resource "google_storage_bucket_object" "task_prolog_scripts" { - for_each = { - for x in local.task_prolog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-task_prolog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -resource "google_storage_bucket_object" "task_epilog_scripts" { - for_each = { - for x in local.task_epilog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-task_epilog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) -} - -############################ -# DATA: CHS GPU HEALTH CHECK -############################ - -data "local_file" "chs_gpu_health_check" { - filename = "${path.module}/scripts/tools/gpu-test" -} - -################################ -# DATA: EXTERNAL PROLOG/EPILOG # -################################ - -data "local_file" "external_epilog" { - filename = "${path.module}/files/external_epilog.sh" -} - -data "local_file" "external_prolog" { - filename = "${path.module}/files/external_prolog.sh" -} - -data "local_file" "setup_external" { - filename = "${path.module}/files/setup_external.sh" -} - -locals { - external_epilog = [{ - filename = "z_external_epilog.sh" - content = data.local_file.external_epilog.content - source = null - }] - external_prolog = [{ - filename = "z_external_prolog.sh" - content = data.local_file.external_prolog.content - source = null - }] - setup_external = [{ - filename = "z_setup_external.sh" - content = data.local_file.setup_external.content - }] - chs_gpu_health_check = [{ - filename = "a_chs_gpu_health_check.sh" - content = data.local_file.chs_gpu_health_check.content - source = null - }] - - chs_prolog = var.enable_chs_gpu_health_check_prolog ? local.chs_gpu_health_check : [] - ext_prolog = var.enable_external_prolog_epilog ? local.external_prolog : [] - prolog_scripts = concat(local.chs_prolog, local.ext_prolog, var.prolog_scripts) - task_prolog_scripts = var.task_prolog_scripts - - chs_epilog = var.enable_chs_gpu_health_check_epilog ? local.chs_gpu_health_check : [] - ext_epilog = var.enable_external_prolog_epilog ? local.external_epilog : [] - epilog_scripts = concat(local.chs_epilog, local.ext_epilog, var.epilog_scripts) - task_epilog_scripts = var.task_epilog_scripts - - controller_startup_scripts = var.enable_external_prolog_epilog ? concat(local.setup_external, var.controller_startup_scripts) : var.controller_startup_scripts - - -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf deleted file mode 100644 index 111c997d62..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "slurm_bucket_path" { - description = "GCS Bucket URI of Slurm cluster file storage." - value = local.bucket_path -} - -output "bucket_name" { - description = "GCS Bucket name of Slurm cluster file storage." - value = data.google_storage_bucket.this.name -} - -output "bucket_dir" { - description = "Path directory within `bucket_name` for Slurm cluster file storage." - value = local.bucket_dir -} - -output "config" { - description = "Cluster configuration." - value = local.config - - precondition { - condition = var.enable_hybrid ? can(coalesce(var.slurm_control_host)) : true - error_message = "Input slurm_control_host is required." - } - - precondition { - condition = length(local.x_nodeset_overlap) == 0 - error_message = "All nodeset names must be unique among all nodeset types." - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py deleted file mode 100644 index 89ceefa3df..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py +++ /dev/null @@ -1,658 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional, Iterable, Dict, Set, Tuple -from itertools import chain -from collections import defaultdict -import json -from pathlib import Path -import util -from util import dirs, slurmdirs -import tpu -from addict import Dict as NSDict # type: ignore - -FILE_PREAMBLE = """ -# Warning: -# This file is managed by a script. Manual modifications will be overwritten. -""" - - - -def dict_to_conf(conf, delim=" ") -> str: - """convert dict to delimited slurm-style key-value pairs""" - - def filter_conf(pair): - k, v = pair - if isinstance(v, list): - v = ",".join(str(el) for el in v if el is not None) - return k, (v if bool(v) or v == 0 else None) - - return delim.join( - f"{k}={v}" for k, v in map(filter_conf, conf.items()) if v is not None - ) - - -TOPOLOGY_PLUGIN_TREE = "topology/tree" - -def topology_plugin(lkp: util.Lookup) -> str: - """ - Returns configured topology plugin, defaults to `topology/tree`. - """ - cp, key = lkp.cfg.cloud_parameters, "topology_plugin" - if key not in cp or cp[key] is None: - return TOPOLOGY_PLUGIN_TREE - return cp[key] - -def conflines(lkp: util.Lookup) -> str: - params = lkp.cfg.cloud_parameters - def get(key, default): - """ - Returns the value of the key in params if it exists and is not None, - otherwise returns supplied default. - We can't rely on the `dict.get` method because the value could be `None` as - well as empty NSDict, depending on type of the `cfg.cloud_parameters`. - TODO: Simplify once NSDict is removed from the codebase. - """ - if key not in params or params[key] is None: - return default - return params[key] - - no_comma_params = get("no_comma_params", False) - - any_gpus = any( - lkp.template_info(nodeset.instance_template).gpu - for nodeset in lkp.cfg.nodeset.values() - ) - - any_tpu = any( - tpu_nodeset is not None - for part in lkp.cfg.partitions.values() - for tpu_nodeset in part.partition_nodeset_tpu - ) - - any_gke = any( - lkp.nodeset_is_gke(nodeset) - for nodeset in lkp.cfg.nodeset.values() - ) - - any_dynamic = any(bool(p.partition_feature) for p in lkp.cfg.partitions.values()) - comma_params = { - "LaunchParameters": [ - "enable_nss_slurm", - "use_interactive_step", - ], - "SlurmctldParameters": [ - "cloud_reg_addrs" if any_dynamic or any_tpu or any_gke else "cloud_dns", - "enable_configless", - "idle_on_node_suspend", - ], - "GresTypes": [ - "gpu" if any_gpus else None, - ], - } - - scripts_dir = lkp.cfg.install_dir or dirs.scripts - prolog_path = Path(dirs.custom_scripts / "prolog.d") - epilog_path = Path(dirs.custom_scripts / "epilog.d") - task_prolog_path = Path(dirs.custom_scripts / "task_prolog.d") - task_epilog_path = Path(dirs.custom_scripts / "task_epilog.d") - default_tree_width = 65533 if any_dynamic else 128 - - conf_options = { - **(comma_params if not no_comma_params else {}), - "Prolog": f"{prolog_path}/*" if lkp.cfg.prolog_scripts else None, - "Epilog": f"{epilog_path}/*" if lkp.cfg.epilog_scripts else None, - "TaskProlog": f"{task_prolog_path}/task-prolog" if lkp.cfg.task_prolog_scripts else None, - "TaskEpilog": f"{task_epilog_path}/task-epilog" if lkp.cfg.task_epilog_scripts else None, - "PrologFlags": get("prolog_flags", None), - "SwitchType": get("switch_type", None), - "PrivateData": get("private_data", []), - "SchedulerParameters": get("scheduler_parameters", [ - "bf_continue", - "salloc_wait_nodes", - "ignore_prefer_validation", - ]), - "ResumeProgram": f"{scripts_dir}/resume_wrapper.sh", - "ResumeFailProgram": f"{scripts_dir}/suspend_wrapper.sh", - "ResumeRate": get("resume_rate", 0), - "ResumeTimeout": get("resume_timeout", 300), - "SuspendProgram": f"{scripts_dir}/suspend_wrapper.sh", - "SuspendRate": get("suspend_rate", 0), - "SuspendTimeout": get("suspend_timeout", 300), - "SlurmdTimeout": get("slurmd_timeout", 300), - "UnkillableStepTimeout": get("unkillable_step_timeout", 300), - "TreeWidth": get("tree_width", default_tree_width), - "JobSubmitPlugins": "lua" if any_tpu else None, - "TopologyPlugin": topology_plugin(lkp), - "TopologyParam": get("topology_param", "SwitchAsNodeRank"), - } - return dict_to_conf(conf_options, delim="\n") - - - - -def nodeset_lines(nodeset, lkp: util.Lookup) -> str: - template_info = lkp.template_info(nodeset.instance_template) - machine_conf = lkp.template_machine_conf(nodeset.instance_template) - - # follow https://slurm.schedmd.com/slurm.conf.html#OPT_Boards - # by setting Boards, SocketsPerBoard, CoresPerSocket, and ThreadsPerCore - gres = f"gpu:{template_info.gpu.count}" if template_info.gpu else None - node_conf = { - "RealMemory": machine_conf.memory, - "Boards": machine_conf.boards, - "SocketsPerBoard": machine_conf.sockets_per_board, - "CoresPerSocket": machine_conf.cores_per_socket, - "ThreadsPerCore": machine_conf.threads_per_core, - "CPUs": machine_conf.cpus, - "Gres": gres, - **nodeset.node_conf, - } - nodelist = lkp.nodelist(nodeset) - - return "\n".join( - map( - dict_to_conf, - [ - {"NodeName": nodelist, "State": "CLOUD", **node_conf}, - {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, - ], - ) - ) - - -def nodeset_tpu_lines(nodeset, lkp: util.Lookup) -> str: - nodelist = lkp.nodelist(nodeset) - return "\n".join( - map( - dict_to_conf, - [ - {"NodeName": nodelist, "State": "CLOUD", **nodeset.node_conf}, - {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, - ], - ) - ) - - -def nodeset_dyn_lines(nodeset): - """generate slurm NodeSet definition for dynamic nodeset""" - return dict_to_conf( - {"NodeSet": nodeset.nodeset_name, "Feature": nodeset.nodeset_feature} - ) - - -def partitionlines(partition, lkp: util.Lookup) -> str: - """Make a partition line for the slurm.conf""" - MIN_MEM_PER_CPU = 100 - - def defmempercpu(nodeset_name: str) -> int: - nodeset = lkp.cfg.nodeset.get(nodeset_name) - template = nodeset.instance_template - machine = lkp.template_machine_conf(template) - mem_spec_limit = int(nodeset.node_conf.get("MemSpecLimit", 0)) - return max(MIN_MEM_PER_CPU, (machine.memory - mem_spec_limit) // machine.cpus) - - defmem = min( - map(defmempercpu, partition.partition_nodeset), default=MIN_MEM_PER_CPU - ) - - nodesets = list( - chain( - partition.partition_nodeset, - partition.partition_nodeset_dyn, - partition.partition_nodeset_tpu, - ) - ) - - is_tpu = len(partition.partition_nodeset_tpu) > 0 - is_dyn = len(partition.partition_nodeset_dyn) > 0 - - oversub_exlusive = partition.enable_job_exclusive or is_tpu - power_down_on_idle = partition.enable_job_exclusive and not is_dyn - - line_elements = { - "PartitionName": partition.partition_name, - "Nodes": ",".join(nodesets), - "State": "UP", - "DefMemPerCPU": defmem, - "SuspendTime": 300, - "Oversubscribe": "Exclusive" if oversub_exlusive else None, - "PowerDownOnIdle": "YES" if power_down_on_idle else None, - **partition.partition_conf, - } - - return dict_to_conf(line_elements) - - -def suspend_exc_lines(lkp: util.Lookup) -> Iterable[str]: - static_nodelists = [] - for ns in lkp.power_managed_nodesets(): - if ns.node_count_static: - nodelist = lkp.nodelist_range(ns.nodeset_name, 0, ns.node_count_static) - static_nodelists.append(nodelist) - suspend_exc_nodes = {"SuspendExcNodes": static_nodelists} - - dyn_parts = [ - p.partition_name - for p in lkp.cfg.partitions.values() - if len(p.partition_nodeset_dyn) > 0 - ] - suspend_exc_parts = {"SuspendExcParts": [*dyn_parts]} - - return filter( - None, - [ - dict_to_conf(suspend_exc_nodes) if static_nodelists else None, - dict_to_conf(suspend_exc_parts), - ], - ) - - -def make_cloud_conf(lkp: util.Lookup) -> str: - """generate cloud.conf snippet""" - lines = [ - FILE_PREAMBLE, - conflines(lkp), - *(nodeset_lines(n, lkp) for n in lkp.cfg.nodeset.values()), - *(nodeset_dyn_lines(n) for n in lkp.cfg.nodeset_dyn.values()), - *(nodeset_tpu_lines(n, lkp) for n in lkp.cfg.nodeset_tpu.values()), - *(partitionlines(p, lkp) for p in lkp.cfg.partitions.values()), - *(suspend_exc_lines(lkp)), - ] - return "\n\n".join(filter(None, lines)) - - -def gen_cloud_conf(lkp: util.Lookup) -> None: - content = make_cloud_conf(lkp) - - conf_file = lkp.etc_dir / "cloud.conf" - conf_file.write_text(content) - util.chown_slurm(conf_file, mode=0o644) - - -def install_slurm_conf(lkp: util.Lookup) -> None: - """install slurm.conf""" - if lkp.cfg.ompi_version: - mpi_default = "pmi2" - else: - mpi_default = "none" - - conf_options = { - "name": lkp.cfg.slurm_cluster_name, - "control_addr": lkp.control_addr if lkp.control_addr else lkp.hostname_fqdn, - "control_host": lkp.control_host, - "accounting_storage_host": lkp.control_addr if lkp.cfg.controller_network_attachment else lkp.control_host, - "control_host_port": lkp.control_host_port, - "scripts": dirs.scripts, - "slurmlog": dirs.log, - "state_save": slurmdirs.state, - "mpi_default": mpi_default, - "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", - } - - conf = lkp.cfg.slurm_conf_tpl.format(**conf_options) - - conf_file = lkp.etc_dir / "slurm.conf" - conf_file.write_text(conf) - util.chown_slurm(conf_file, mode=0o644) - - -def install_slurmdbd_conf(lkp: util.Lookup) -> None: - """install slurmdbd.conf""" - conf_options = { - "control_host": lkp.control_host, - "slurmlog": dirs.log, - "state_save": slurmdirs.state, - "db_name": "slurm_acct_db", - "db_user": "slurm", - "db_pass": '""', - "db_host": "localhost", - "db_port": "3306", - "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", - } - - if lkp.cfg.cloudsql_secret: - secret_name = f"{lkp.cfg.slurm_cluster_name}-slurm-secret-cloudsql" - payload = json.loads(util.access_secret_version(lkp.project, secret_name)) - - if payload["db_name"] and payload["db_name"] != "": - conf_options["db_name"] = payload["db_name"] - if payload["user"] and payload["user"] != "": - conf_options["db_user"] = payload["user"] - if payload["password"] and payload["password"] != "": - conf_options["db_pass"] = payload["password"] - - db_host_str = payload["server_ip"].split(":") - if db_host_str[0]: - conf_options["db_host"] = db_host_str[0] - conf_options["db_port"] = ( - db_host_str[1] if len(db_host_str) >= 2 else "3306" - ) - - conf = lkp.cfg.slurmdbd_conf_tpl.format(**conf_options) - - conf_file = lkp.etc_dir / "slurmdbd.conf" - conf_file.write_text(conf) - util.chown_slurm(conf_file, 0o600) - - -def install_cgroup_conf(lkp: util.Lookup) -> None: - """install cgroup.conf""" - conf_file = lkp.etc_dir / "cgroup.conf" - conf_file.write_text(lkp.cfg.cgroup_conf_tpl) - util.chown_slurm(conf_file, mode=0o600) - - -def install_jobsubmit_lua(lkp: util.Lookup) -> None: - """install job_submit.lua if there are tpu nodes in the cluster""" - if not any( - tpu_nodeset is not None - for part in lkp.cfg.partitions.values() - for tpu_nodeset in part.partition_nodeset_tpu - ): - return # No TPU partitions, no need for job_submit.lua - - scripts_dir = lkp.cfg.slurm_scripts_dir or dirs.scripts - tpl = (scripts_dir / "job_submit.lua.tpl").read_text() - conf = tpl.format(scripts_dir=scripts_dir) - - conf_file = lkp.etc_dir / "job_submit.lua" - conf_file.write_text(conf) - util.chown_slurm(conf_file, 0o600) - - -def gen_cloud_gres_conf_lines(lkp: util.Lookup) -> str: - """generate cloud_gres.conf's content""" - - gpu_nodes = defaultdict(list) - for nodeset in lkp.cfg.nodeset.values(): - ti = lkp.template_info(nodeset.instance_template) - gpu_count = ti.gpu.count if ti.gpu else 0 - gpu_type = ti.gpu.type if ti.gpu else None - if gpu_count: - gpu_nodes[(gpu_count, gpu_type)].append(lkp.nodelist(nodeset)) - - lines = [ - dict_to_conf( - { - "NodeName": names, - "Name": "gpu", - "Type": gpu_type, - "File": "/dev/nvidia{}".format(f"[0-{gpu_count-1}]" if gpu_count > 1 else "0"), - } - ) - for (gpu_count, gpu_type), names in gpu_nodes.items() - ] - lines.append("\n") - return "\n".join(lines) - - -def gen_cloud_gres_conf(lkp: util.Lookup) -> None: - """create cloud_gres.conf file""" - - content = FILE_PREAMBLE + gen_cloud_gres_conf_lines(lkp) - - conf_file = lkp.etc_dir / "cloud_gres.conf" - conf_file.write_text(content) - util.chown_slurm(conf_file, mode=0o600) - - -def install_gres_conf(lkp: util.Lookup) -> None: - conf_file = lkp.etc_dir / "cloud_gres.conf" - gres_conf = lkp.etc_dir / "gres.conf" - if not gres_conf.exists(): - gres_conf.symlink_to(conf_file) - util.chown_slurm(gres_conf, mode=0o600) - - -class Switch: - """ - Represents a switch in the topology.conf file. - NOTE: It's class user job to make sure that there is no leaf-less Switches in the tree - """ - - def __init__( - self, - name: str, - nodes: Optional[Iterable[str]] = None, - switches: Optional[Dict[str, "Switch"]] = None, - ): - self.name = name - self.nodes = nodes or [] - self.switches = switches or {} - - def conf_line(self) -> str: - d = {"SwitchName": self.name} - if self.nodes: - d["Nodes"] = util.to_hostlist(self.nodes) - if self.switches: - d["Switches"] = util.to_hostlist(self.switches.keys()) - return dict_to_conf(d) - - def render_conf_lines(self) -> Iterable[str]: - yield self.conf_line() - for s in sorted(self.switches.values(), key=lambda s: s.name): - yield from s.render_conf_lines() - -class TopologySummary: - """ - Represents a summary of the topology, to make judgements about changes. - To be stored in JSON file along side of topology.conf to simplify parsing. - """ - def __init__( - self, - physical_host: Optional[Dict[str, str]] = None, - down_nodes: Optional[Iterable[str]] = None, - tpu_nodes: Optional[Iterable[str]] = None, - ) -> None: - self.physical_host = physical_host or {} - self.down_nodes = set(down_nodes or []) - self.tpu_nodes = set(tpu_nodes or []) - - - @classmethod - def path(cls, lkp: util.Lookup) -> Path: - return lkp.etc_dir / "cloud_topology.summary.json" - - @classmethod - def loads(cls, s: str) -> "TopologySummary": - d = json.loads(s) - return cls( - physical_host=d.get("physical_host"), - down_nodes=d.get("down_nodes"), - tpu_nodes=d.get("tpu_nodes"), - ) - - @classmethod - def load(cls, lkp: util.Lookup) -> "TopologySummary": - p = cls.path(lkp) - if not p.exists(): - return cls() # Return empty instance - return cls.loads(p.read_text()) - - def dumps(self) -> str: - return json.dumps( - { - "physical_host": self.physical_host, - "down_nodes": list(self.down_nodes), - "tpu_nodes": list(self.tpu_nodes), - }, - indent=2) - - def dump(self, lkp: util.Lookup) -> None: - TopologySummary.path(lkp).write_text(self.dumps()) - - def _nodenames(self) -> Set[str]: - return set(self.physical_host) | self.down_nodes | self.tpu_nodes - - def requires_reconfigure(self, prev: "TopologySummary") -> bool: - """ - Reconfigure IFF one of the following occurs: - * A node is added - * A node get a non-empty physicalHost - """ - if len(self._nodenames() - prev._nodenames()) > 0: - return True - for n, ph in self.physical_host.items(): - if ph and ph != prev.physical_host.get(n): - return True - return False - -class TopologyBuilder: - def __init__(self) -> None: - self._r = Switch("") # fake root, not part of the tree - self.summary = TopologySummary() - - def add(self, path: List[str], nodes: Iterable[str]) -> None: - n = self._r - assert path - for p in path: - n = n.switches.setdefault(p, Switch(p)) - n.nodes = [*n.nodes, *nodes] - - def render_conf_lines(self) -> Iterable[str]: - if not self._r.switches: - return [] # type: ignore - for s in sorted(self._r.switches.values(), key=lambda s: s.name): - yield from s.render_conf_lines() - - def compress(self) -> "TopologyBuilder": - compressed = TopologyBuilder() - compressed.summary = self.summary - def _walk( - u: Switch, c: Switch - ): # u: uncompressed node, c: its counterpart in compressed tree - pref = f"{c.name}_" if c != compressed._r else "s" - for i, us in enumerate(sorted(u.switches.values(), key=lambda s: s.name)): - cs = Switch(f"{pref}{i}", nodes=us.nodes) - c.switches[cs.name] = cs - _walk(us, cs) - - _walk(self._r, compressed._r) - return compressed - - -def add_tpu_nodeset_topology(nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup): - tpuobj = tpu.TPU.make(nodeset.nodeset_name, lkp) - static, dynamic = lkp.nodenames(nodeset) - - pref = ["tpu-root", f"ns_{nodeset.nodeset_name}"] - if tpuobj.vmcount == 1: # Put all nodes in one switch - all_nodes = list(chain(static, dynamic)) - bldr.add(pref, all_nodes) - bldr.summary.tpu_nodes.update(all_nodes) - return - - # Chunk nodes into sub-switches of size `vmcount` - chunk_num = 0 - for nodenames in (static, dynamic): - for nodeschunk in util.chunked(nodenames, n=tpuobj.vmcount): - chunk_name = f"{nodeset.nodeset_name}-{chunk_num}" - chunk_num += 1 - bldr.add([*pref, chunk_name], nodeschunk) - bldr.summary.tpu_nodes.update(nodeschunk) - -_SLURM_TOPO_ROOT = "slurm-root" - -def _make_physical_path(physical_host: str) -> List[str]: - assert physical_host.startswith("/"), f"Unexpected physicalHost: {physical_host}" - parts = physical_host[1:].split("/") - # Due to issues with Slurm's topology plugin, we can not use all components of `physicalHost`, - # trim it down to `cluster/rack`. - short_path = parts[:2] - return [_SLURM_TOPO_ROOT, *short_path] - -def add_nodeset_topology( - nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup -) -> None: - up_nodes = set() - default_path = [_SLURM_TOPO_ROOT, f"ns_{nodeset.nodeset_name}"] - - for inst in lkp.instances().values(): - try: - if lkp.node_nodeset_name(inst.name) != nodeset.nodeset_name: - continue - except Exception: - continue - - phys_host = inst.resource_status.physical_host or "" - bldr.summary.physical_host[inst.name] = phys_host - up_nodes.add(inst.name) - - if phys_host: - bldr.add(_make_physical_path(phys_host), [inst.name]) - else: - bldr.add(default_path, [inst.name]) - - down_nodes = [] - for node in chain(*lkp.nodenames(nodeset)): - if node not in up_nodes: - down_nodes.append(node) - if down_nodes: - bldr.add(default_path, down_nodes) - bldr.summary.down_nodes.update(down_nodes) - -def gen_topology(lkp: util.Lookup) -> TopologyBuilder: - bldr = TopologyBuilder() - for ns in lkp.cfg.nodeset_tpu.values(): - add_tpu_nodeset_topology(ns, bldr, lkp) - for ns in lkp.cfg.nodeset.values(): - add_nodeset_topology(ns, bldr, lkp) - return bldr - -def gen_topology_conf(lkp: util.Lookup) -> Tuple[bool, TopologySummary]: - """ - Generates slurm topology.conf. - Returns whether the topology.conf got updated. - """ - topo = gen_topology(lkp).compress() - conf_file = lkp.etc_dir / "cloud_topology.conf" - - with open(conf_file, "w") as f: - f.writelines(FILE_PREAMBLE + "\n") - for line in topo.render_conf_lines(): - f.write(line) - f.write("\n") - f.write("\n") - - prev_summary = TopologySummary.load(lkp) - return topo.summary.requires_reconfigure(prev_summary), topo.summary - -def install_topology_conf(lkp: util.Lookup) -> None: - conf_file = lkp.etc_dir / "cloud_topology.conf" - summary_file = lkp.etc_dir / "cloud_topology.summary.json" - topo_conf = lkp.etc_dir / "topology.conf" - - if not topo_conf.exists(): - topo_conf.symlink_to(conf_file) - - util.chown_slurm(conf_file, mode=0o600) - util.chown_slurm(summary_file, mode=0o600) - - -def gen_controller_configs(lkp: util.Lookup) -> None: - install_slurm_conf(lkp) - install_slurmdbd_conf(lkp) - gen_cloud_conf(lkp) - gen_cloud_gres_conf(lkp) - install_gres_conf(lkp) - install_cgroup_conf(lkp) - install_jobsubmit_lua(lkp) - - if topology_plugin(lkp) == TOPOLOGY_PLUGIN_TREE: - _, summary = gen_topology_conf(lkp) - summary.dump(lkp) - install_topology_conf(lkp) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py deleted file mode 100644 index cd2e41e5af..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any -from pathlib import Path -import shutil -import pickle - -import logging -log = logging.getLogger() - -# Can't reuse tool from util.py to avoid circular dependencies -# TODO: break down util.py for better modularity. -def _chown_slurm(path: Path) -> None: - shutil.chown(path, user="slurm", group="slurm") - -class FileCache: - def __init__(self, path: Path): - self.path = path - - def get(self, key: str) -> Any | None: - p = self.path / key - if not p.exists(): - return None - - try: - with p.open("rb") as f: - return pickle.load(f) - - except Exception as e: - log.warning(f"Failed to read cached value at {p}: {e}") - return None - - def set(self, key: str, data: Any) -> None: - p = self.path / key - - try: - # Create & chown before writing to minimize chances - # of ending up with root-owned corrupted file that can't be cleaned up - # TODO: restrict usage of cache by root to avoid all this complexity - # or have a cache per user. - p.touch(exist_ok=True) - _chown_slurm(p) - with p.open("wb") as f: - pickle.dump(data, f) - - except Exception as e: - log.warning(f"Failed to write cached value at {p}: {e}") - - -class NoCache: - def get(self, key: str) -> Any: - log.warning("No cache used") - return None - - def set(self, key: str, data: Any) -> None: - log.warning("No cache used") - - -def cache(name: str) -> FileCache | NoCache: - try: - path = Path("/tmp/slurm_gcp_cache/") / name - if not path.exists(): - path.mkdir(exist_ok=True, parents=True) - _chown_slurm(path) - return FileCache(path) - except: - log.exception(f"Failed to create cache, fallback to NoCache") - return NoCache() diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py deleted file mode 100644 index df0fd8ebe0..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright 2024 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import util -import tpu - - -def get_vmcount_of_tpu_part(part): - res = 0 - lkp = util.lookup() - for ns in lkp.cfg.partitions[part].partition_nodeset_tpu: - tpu_obj = tpu.TPU.make(ns, lkp) - if res == 0: - res = tpu_obj.vmcount - else: - if res != tpu_obj.vmcount: - # this should not happen, that in the same partition there are different vmcount nodesets - return -1 - return res - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--partitions", - "-p", - help="The partition(s) to retrieve the TPU vmcount value for.", - ) - args = parser.parse_args() - if not args.partitions: - exit(0) - - # useful exit code - # partition does not exists in config.yaml, thus do not exist in slurm - PART_INVALID = -1 - # in the same partition there are nodesets with different vmcounts - DIFF_VMCOUNTS_SAME_PART = -2 - # partition is a list of partitions in which at least two of them have different vmcount - DIFF_PART_DIFFERENT_VMCOUNTS = -3 - vmcounts = [] - # valid equals to 0 means that we are ok, otherwise it will be set to one of the previously defined exit codes - valid = 0 - for part in args.partitions.split(","): - if part not in util.lookup().cfg.partitions: - valid = PART_INVALID - break - else: - if util.lookup().partition_is_tpu(part): - vmcount = get_vmcount_of_tpu_part(part) - if vmcount == -1: - valid = DIFF_VMCOUNTS_SAME_PART - break - vmcounts.append(vmcount) - else: - vmcounts.append(0) - # this means that there are different vmcounts for these partitions - if valid == 0 and len(set(vmcounts)) != 1: - valid = DIFF_PART_DIFFERENT_VMCOUNTS - if valid != 0: - print(f"VMCOUNT:{valid}") - else: - print(f"VMCOUNT:{vmcounts[0]}") diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl deleted file mode 100644 index 810a0742b0..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl +++ /dev/null @@ -1,103 +0,0 @@ -SCRIPTS_DIR = "{scripts_dir}" -NO_VAL = 4294967294 --- get_tpu_vmcount.py exit code -PART_INVALID = -1 -- partition does not exists in config.yaml, thus do not exist in slurm -DIFF_VMCOUNTS_SAME_PART = -2 -- in the same partition there are nodesets with different vmcounts -DIFF_PART_DIFFERENT_VMCOUNTS = -3 -- partition is a list of partitions in which at least two of them have different vmcount -UNKWOWN_ERROR = -4 -- get_tpu_vmcount.py did not return a valid response - -function get_part(job_desc, part_list) - if job_desc.partition then - return job_desc.partition - end - for name, val in pairs(part_list) do - if val.flag_default == 1 then - return name - end - end - return nil -end - -function os.capture(cmd, raw) - local handle = assert(io.popen(cmd, 'r')) - local output = assert(handle:read('*a')) - handle:close() - return output -end - -function get_vmcount(part) - local cmd = SCRIPTS_DIR .. "/get_tpu_vmcount.py -p " .. part - local out = os.capture(cmd, true) - for line in out:gmatch("(.-)\r?\n") do - local tag, val = line:match("([^:]+):([^:]+)") - if tag == "VMCOUNT" then - return tonumber(val) - end - end - return UNKWOWN_ERROR -end - -function slurm_job_submit(job_desc, part_list, submit_uid) - local part = get_part(job_desc, part_list) - local vmcount = get_vmcount(part) - -- Only do something if the job is in a TPU partition, if vmcount is 0, it implies that the partition(s) specified are not TPU ones - if vmcount == 0 then - return slurm.SUCCESS - end - -- This is a TPU job, but as the vmcount is 1 it can he handled the same way - if vmcount == 1 then - return slurm.SUCCESS - end - -- Check for errors - if vmcount == PART_INVALID then - slurm.log_user("Invalid partition specified " .. part) - return slurm.FAILURE - end - if vmcount == DIFF_VMCOUNTS_SAME_PART then - slurm.log_user("In partition(s) " .. part .. - " there are more than one tpu nodeset vmcount, this should not happen.") - return slurm.ERROR - end - if vmcount == DIFF_PART_DIFFERENT_VMCOUNTS then - slurm.log_user("In partition list " .. part .. - " there are more than one TPU types, cannot determine which is the correct vmcount to use, please retry with only one partition.") - return slurm.FAILURE - end - if vmcount == UNKWOWN_ERROR then - slurm.log_user("Something went wrong while executing get_tpu_vmcount.py.") - return slurm.ERROR - end - -- This is surely a TPU node - if vmcount > 1 then - local min_nodes = job_desc.min_nodes - local max_nodes = job_desc.max_nodes - -- if not specified assume it is one, this should be improved taking into account the cpus, mem, and other factors - if min_nodes == NO_VAL then - min_nodes = 1 - max_nodes = 1 - end - -- as max_nodes can be higher than the nodes in the partition, we are not able to calculate with certainty the nodes that this job will have if this value is set to something - -- different than min_nodes - if min_nodes ~= max_nodes then - slurm.log_user("Max nodes cannot be set different than min nodes for the TPU partitions.") - return slurm.ERROR - end - -- Set the number of switches to the number of nodes originally requested by the job, as the job requests "TPU groups" - job_desc.req_switch = min_nodes - - -- Apply the node increase into the job description. - job_desc.min_nodes = min_nodes * vmcount - job_desc.max_nodes = max_nodes * vmcount - -- if job_desc.features then - -- slurm.log_user("Features: %s",job_desc.features) - -- end - end - - return slurm.SUCCESS -end - -function slurm_job_modify(job_desc, job_rec, part_list, modify_uid) - return slurm.SUCCESS -end - -return slurm.SUCCESS diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py deleted file mode 100644 index cabd6e3e9f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py +++ /dev/null @@ -1,352 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, Callable, Any -import argparse -import os -import shelve -import uuid -from collections import namedtuple -from datetime import datetime, timedelta, timezone -from pathlib import Path -from pprint import pprint - -import util -from google.api_core import exceptions, retry -from google.cloud import bigquery as bq -from google.cloud.bigquery import SchemaField # type: ignore -from util import lookup, run - -SACCT = "sacct" -script = Path(__file__).resolve() - -DEFAULT_TIMESTAMP_FILE = script.parent / "bq_timestamp" -timestamp_file = Path(os.environ.get("TIMESTAMP_FILE", DEFAULT_TIMESTAMP_FILE)) -# The maximum request to insert_rows is 10MB, each sacct row is about 1200 bytes or ~ 8000 rows. -# Set to 5000 for a little wiggle room. -BQ_ROW_BATCH_SIZE = 5000 - -# cluster_id_file = script.parent / 'cluster_uuid' -# try: -# cluster_id = cluster_id_file.read_text().rstrip() -# except FileNotFoundError: -# cluster_id = uuid.uuid4().hex -# cluster_id_file.write_text(cluster_id) - -job_idx_cache_path = script.parent / "bq_job_idx_cache" - -SLURM_TIME_FORMAT = r"%Y-%m-%dT%H:%M:%S" - - -def make_datetime(time_string): - if time_string == "None": - return None - return datetime.strptime(time_string, SLURM_TIME_FORMAT).replace( - tzinfo=timezone.utc - ) - - -def make_time_interval(seconds): - sign = 1 - if seconds < 0: - sign = -1 - seconds = abs(seconds) - d, r = divmod(seconds, 60 * 60 * 24) - h, r = divmod(r, 60 * 60) - m, s = divmod(r, 60) - d *= sign - h *= sign - return f"{d}D {h:02}:{m:02}:{s}" - - -converters: Dict[str, Callable[[Any], Any]] = { - "DATETIME": make_datetime, - "INTERVAL": make_time_interval, - "STRING": str, - "INT64": lambda n: int(n or 0), -} - - -def schema_field(field_name, data_type, description, required=False): - return SchemaField( - field_name, - data_type, - description=description, - mode="REQUIRED" if required else "NULLABLE", - ) - - -schema_fields = [ - schema_field("cluster_name", "STRING", "cluster name", required=True), - schema_field("cluster_id", "STRING", "UUID for the cluster", required=True), - schema_field("entry_uuid", "STRING", "entry UUID for the job row", required=True), - schema_field( - "job_db_uuid", "STRING", "job db index from the slurm database", required=True - ), - schema_field("job_id_raw", "INT64", "raw job id", required=True), - schema_field("job_id", "STRING", "job id", required=True), - schema_field("state", "STRING", "final job state", required=True), - schema_field("job_name", "STRING", "job name"), - schema_field("partition", "STRING", "job partition"), - schema_field("submit_time", "DATETIME", "job submit time"), - schema_field("start_time", "DATETIME", "job start time"), - schema_field("end_time", "DATETIME", "job end time"), - schema_field("elapsed_raw", "INT64", "STRING", "job run time in seconds"), - # schema_field("elapsed_time", "INTERVAL", "STRING", "job run time interval"), - schema_field("timelimit_raw", "STRING", "job timelimit in minutes"), - schema_field("timelimit", "STRING", "job timelimit"), - # schema_field("num_tasks", "INT64", "number of allocated tasks in job"), - schema_field("nodelist", "STRING", "names of nodes allocated to job"), - schema_field("user", "STRING", "user responsible for job"), - schema_field("uid", "INT64", "uid of job user"), - schema_field("group", "STRING", "group of job user"), - schema_field("gid", "INT64", "gid of job user"), - schema_field("wckey", "STRING", "job wckey"), - schema_field("qos", "STRING", "job qos"), - schema_field("comment", "STRING", "job comment"), - schema_field("admin_comment", "STRING", "job admin comment"), - # extra will be added in 23.02 - # schema_field("extra", "STRING", "job extra field"), - schema_field("exitcode", "STRING", "job exit code"), - schema_field("alloc_cpus", "INT64", "count of allocated CPUs"), - schema_field("alloc_nodes", "INT64", "number of nodes allocated to job"), - schema_field("alloc_tres", "STRING", "allocated trackable resources (TRES)"), - # schema_field("system_cpu", "INTERVAL", "cpu time used by parent processes"), - # schema_field("cpu_time", "INTERVAL", "CPU time used (elapsed * cpu count)"), - schema_field("cpu_time_raw", "INT64", "CPU time used (elapsed * cpu count)"), - # schema_field("ave_cpu", "INT64", "Average CPU time of all tasks in job"), - # schema_field( - # "tres_usage_tot", - # "STRING", - # "Tres total usage by all tasks in job", - # ), -] - - -slurm_field_map = { - "job_db_uuid": "DBIndex", - "job_id_raw": "JobIDRaw", - "job_id": "JobID", - "state": "State", - "job_name": "JobName", - "partition": "Partition", - "submit_time": "Submit", - "start_time": "Start", - "end_time": "End", - "elapsed_raw": "ElapsedRaw", - "elapsed_time": "Elapsed", - "timelimit_raw": "TimelimitRaw", - "timelimit": "Timelimit", - "num_tasks": "NTasks", - "nodelist": "Nodelist", - "user": "User", - "uid": "Uid", - "group": "Group", - "gid": "Gid", - "wckey": "Wckey", - "qos": "Qos", - "comment": "Comment", - "admin_comment": "AdminComment", - # "extra": "Extra", - "exit_code": "ExitCode", - "alloc_cpus": "AllocCPUs", - "alloc_nodes": "AllocNodes", - "alloc_tres": "AllocTres", - "system_cpu": "SystemCPU", - "cpu_time": "CPUTime", - "cpu_time_raw": "CPUTimeRaw", - "ave_cpu": "AveCPU", - "tres_usage_tot": "TresUsageInTot", -} - -# new field name is the key for job_schema. Used to lookup the datatype when -# creating the job rows -job_schema = {field.name: field for field in schema_fields} -# Order is important here, as that is how they are parsed from sacct output -Job = namedtuple("Job", job_schema.keys()) # type: ignore -# ... see https://github.com/python/mypy/issues/848 - -client = bq.Client( - project=lookup().cfg.project, - credentials=util.default_credentials(), - client_options=util.create_client_options(util.ApiEndpoint.BQ), -) -dataset_id = f"{lookup().cfg.slurm_cluster_name}_job_data" -dataset = bq.DatasetReference(project=lookup().project, dataset_id=dataset_id) -table = bq.Table( - bq.TableReference(dataset, f"jobs_{lookup().cfg.slurm_cluster_name}"), schema_fields -) - - -class JobInsertionFailed(Exception): - pass - - -def make_job_row(job): - job_row = { - field_name: converters[field.field_type](job[field_name]) - for field_name, field in job_schema.items() - if field_name in job - } - job_row["entry_uuid"] = uuid.uuid4().hex - job_row["cluster_id"] = lookup().cfg.cluster_id - job_row["cluster_name"] = lookup().cfg.slurm_cluster_name - return job_row - - -def load_slurm_jobs(start, end): - states = ",".join( - ( - "BOOT_FAIL", - "CANCELLED", - "COMPLETED", - "DEADLINE", - "FAILED", - "NODE_FAIL", - "OUT_OF_MEMORY", - "PREEMPTED", - "REQUEUED", - "REVOKED", - "TIMEOUT", - ) - ) - start_iso = start.isoformat(timespec="seconds") - end_iso = end.isoformat(timespec="seconds") - # slurm_fields and bq_fields will be in matching order - slurm_fields = ",".join(slurm_field_map.values()) - bq_fields = slurm_field_map.keys() - cmd = ( - f"{SACCT} --start {start_iso} --end {end_iso} -X -D --format={slurm_fields} " - f"--state={states} --parsable2 --noheader --allusers --duplicates" - ) - text = run(cmd).stdout.splitlines() - # zip pairs bq_fields with the value from sacct - jobs = [dict(zip(bq_fields, line.split("|"))) for line in text] - - # The job index cache allows us to avoid sending duplicate jobs. This avoids a race condition with updating the database. - with shelve.open(str(job_idx_cache_path), flag="r") as job_idx_cache: - job_rows = [ - make_job_row(job) - for job in jobs - if str(job["job_db_uuid"]) not in job_idx_cache - ] - return job_rows - - -def init_table(): - global dataset - global table - dataset = client.create_dataset(dataset, exists_ok=True) # type: ignore - table = client.create_table(table, exists_ok=True) - until_found = retry.Retry(predicate=retry.if_exception_type(exceptions.NotFound)) - table = client.get_table(table, retry=until_found) - # cannot add required fields to an existing schema - table.schema = schema_fields - table = client.update_table(table, ["schema"]) - - -def purge_job_idx_cache(): - purge_time = datetime.now() - timedelta(minutes=30) - with shelve.open(str(job_idx_cache_path), writeback=True) as cache: - to_delete = [] - for idx, stamp in cache.items(): - if stamp < purge_time: - to_delete.append(idx) - for idx in to_delete: - del cache[idx] - - -def bq_submit(jobs): - try: - result = client.insert_rows(table, jobs) - except exceptions.NotFound as e: - print(f"failed to upload job data, table not yet found: {e}") - raise e - except Exception as e: - print(f"failed to upload job data: {e}") - raise e - if result: - pprint(jobs) - pprint(result) - raise JobInsertionFailed("failed to upload job data to big query") - print(f"successfully loaded {len(jobs)} jobs") - - -def get_time_window(): - if not timestamp_file.is_file(): - timestamp_file.touch() - try: - timestamp = datetime.strptime( - timestamp_file.read_text().rstrip(), SLURM_TIME_FORMAT - ) - # time window will overlap the previous by 10 minutes. Duplicates will be filtered out by the job_idx_cache - start = timestamp - timedelta(minutes=10) - except ValueError: - # timestamp 1 is 1 second after the epoch; timestamp 0 is special for sacct - start = datetime.fromtimestamp(1) - # end is now() truncated to the last second - end = datetime.now().replace(microsecond=0) - return start, end - - -def write_timestamp(time): - timestamp_file.write_text(time.isoformat(timespec="seconds")) - - -def update_job_idx_cache(jobs, timestamp): - with shelve.open(str(job_idx_cache_path), writeback=True) as job_idx_cache: - for job in jobs: - job_idx = str(job["job_db_uuid"]) - job_idx_cache[job_idx] = timestamp - - -def main(): - if not lookup().cfg.enable_bigquery_load: - print("bigquery load is not currently enabled") - exit(0) - init_table() - - start, end = get_time_window() - jobs = load_slurm_jobs(start, end) - # on failure, an exception will cause the timestamp not to be rewritten. So - # it will try again next time. If some writes succeed, we don't currently - # have a way to not submit duplicates next time. - if jobs: - num_batches = (len(jobs) - 1) // BQ_ROW_BATCH_SIZE + 1 - print( - f"loading {num_batches} batches of BigQuery data in batches of size : {BQ_ROW_BATCH_SIZE}" - ) - for batch_indx, job_indx in enumerate(range(0, len(jobs), BQ_ROW_BATCH_SIZE)): - print(f"loading BigQuery data batch {batch_indx} of {num_batches}") - bq_submit(jobs[job_indx : job_indx + BQ_ROW_BATCH_SIZE]) - write_timestamp(end) - update_job_idx_cache(jobs, end) - - -parser = argparse.ArgumentParser(description="submit slurm job data to big query") -parser.add_argument( - "timestamp_file", - nargs="?", - action="store", - type=Path, - help="specify timestamp file for reading and writing the time window start. Precedence over TIMESTAMP_FILE env var.", -) - -purge_job_idx_cache() -if __name__ == "__main__": - args = parser.parse_args() - if args.timestamp_file: - timestamp_file = args.timestamp_file.resolve() - main() diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py deleted file mode 100644 index d4a4477f83..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -Implementation of message queue that mimics interface of GCP (PubSub)[https://cloud.google.com/pubsub] - -Messages are stored on controller state disk (to survive controller re-creation) with following layout: - -// -├- -| └- -└- .staging - └- - └- - -One message is one immutable file, that will be deleted after acknowledgement. -NOTE: Implementation assumes that both `` and `.staging/` are on the same disk device, -so it can rely on atomic "move / rename" operation. -""" -from typing import Any -import util -import json -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path -import os -import uuid - -import logging -log = logging.getLogger() - - -@dataclass(frozen=True) -class Message: - id: str - created: datetime - data: Any - - def to_json(self) -> dict[str, str]: - return dict( - id=self.id, - created=self.created.isoformat(), - data=self.data) - - @classmethod - def from_json(cls, data: dict[str, str]) -> 'Message': - return cls( - id=data['id'], - created=datetime.fromisoformat(data['created']), - data=data['data']) - -class Topic: - """ - Acts as PubSub topic (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics). - We can have multiple instances of - """ - def __init__(self, path: Path, staging: Path) -> None: - self._path = path - self._staging = staging - - def _gen_id(self, created: datetime) -> str: - ts = created.strftime("%Y_%m_%d-%H_%M_%S") - suf = str(uuid.uuid4())[:8] - return f"{ts}-{suf}" - - def publish(self, data: Any) -> None: - created = util.now() - id = self._gen_id(created) - msg = Message(id=id, created=created, data=data) - - staged = self._staging / msg.id - dst = self._path / msg.id - - # Write to stagin area first then perform atomic move - # to prevent "reads of partial writes" - staged.write_text(json.dumps(msg.to_json())) - util.chown_slurm(staged) - staged.rename(dst) - - -class Subscription: - """ - Acts as PubSub subscription (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) - with following settings: - - ``` - ackDeadlineSeconds = +Inf # don't resend message that was already being delivered but not acked yet - retainAckedMessages = False # don't persist messages that were already acked - enableMessageOrdering = True # delivers messages in chronoligical order - messageRetentionDuration = +Inf # don't expire messages - deadLetterPolicy = None # "deadlettering" is disabled, subscriber should take care of any poisonous messages - retryPolicy = { # NACKed message will be re-delievered after some time - minimumBackoff = 30s # NOTE: Practically there is no timer, but Subscription instance will not try to re-deliver NACKed messages. - maximumBackoff = 30s # Assumes that slurmsync runs every 30+ sec. - } - ``` - - IMPORTANT: Should only be run as part of slurmsync, - this is our way to ensure that at most one instance exists at a time. - There is no concurancy safeguards in place, avoid multithreaded `pull`, - while multithreaded `ack` & `modify_ack_deadline` are OK. - """ - - def __init__(self, path: Path) -> None: - self._path: Path = path - # contains ALL messages pulled by this subscription instance - # both acked, nacked, and still being processed - # used to prevent double delivery within lifetime of subscription (slurmsync) - self._pulled: set[str] = set() - - def _delete(self, id: str) -> None: - log.debug(f"removing {id}") - try: - os.unlink(self._path / id) - except: - log.exception(f"Failed to remove message {id}") - - def _read_msg(self, id: str) -> Message | None: - try: - with open(self._path / id, 'r') as f: - content = json.loads(f.read()) - return Message.from_json(content) - except Exception: - log.exception(f"Failed to read message {id}") - self._delete(id) # delete message to reduce "deadlettering" - return None - - def pull(self, max_messages: int) -> list[Message]: - if not self._path.exists(): - log.warning(f"Topic {self._path} does not exist") - return [] - res = [] - ls = sorted(os.listdir(self._path)) - for name in ls: - msg = self._read_msg(name) - if msg is not None and msg.id not in self._pulled: - self._pulled.add(msg.id) - res.append(msg) - - if len(res) >= max_messages: - break - return res - - - def ack(self, ids: list[str]) -> None: - for id in ids: - self._delete(id) - - - def modify_ack_deadline(self, ids: list[str], deadline: int) -> None: - """ - Modifies the ack deadline for a specific message. - IMPORTANT: Only accepts deadline=0, which is a way to NACK - Any other values are also meaningless due to ackDeadlineSeconds==+Inf - """ - assert deadline == 0 # no op, next subscriber (slurmsync) will pick this up - - -# Topics and Subscriptions are singletons -# TODO: consider making thread-safe -_topics = {} -_subscriptions = {} - -def _make_path(name: str) -> Path: - p = util.slurmdirs.state / "pubsub" / name - p.mkdir(parents=True, exist_ok=True) - util.chown_slurm(p) - return p - -def _make_staging_path(name: str) -> Path: - p = util.slurmdirs.state / "pubsub" / ".staging" / name - p.mkdir(parents=True, exist_ok=True) - util.chown_slurm(p) - return p - -def topic(name: str) -> Topic: - if name not in _topics: - _topics[name] = Topic(_make_path(name), _make_staging_path(name)) - return _topics[name] - -def subscription(name: str) -> Subscription: - if name not in _subscriptions: - _subscriptions[name] = Subscription(_make_path(name)) - return _subscriptions[name] diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py deleted file mode 100644 index 8ea3d0657e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional - -import util -import uuid -from addict import Dict as NSDict # type: ignore -from datetime import datetime, timedelta -from collections import defaultdict -import logging -from time import sleep - -log = logging.getLogger() - -DWS_EOL_RESERVATION_DURATION = 10 # minutes - -def _duration(flex_options: NSDict, job_id: Optional[int], lkp: util.Lookup) -> int: - dur = flex_options.max_run_duration - if not job_id or not flex_options.use_job_duration: - return dur - - job = lkp.job(job_id) - if not job or not job.duration: - return dur - - if timedelta(minutes=10) <= job.duration <= timedelta(weeks=1): - return int(job.duration.total_seconds()) - - log.info("Job TimeLimit cannot be less than 10 minutes or exceed one week") - return dur - -def _create_slurm_reservation(node_name: str, boot_time: datetime, run_duration: int, lkp: util.Lookup): - """ - Create a Slurm reservation starting at EOL - buffer time. - """ - eol = boot_time + timedelta(seconds=run_duration) - start_str = eol.strftime("%Y-%m-%dT%H:%M:%S") - reservation_name = f"dws-eol-{node_name}" - log.debug(f"creating slurm reservation for {node_name}") - try: - util.run(f"{lkp.scontrol} create reservation user=slurm starttime={start_str} duration={DWS_EOL_RESERVATION_DURATION} nodes={node_name} reservationname={reservation_name} flags=maint,ignore_jobs") - except Exception as e: - log.error(f"Failed to create reservation for {node_name}: {e}") - -def _delete_slurm_reservation(node_name: str, lkp: util.Lookup): - """ - Delete the Slurm reservation for the given node. - """ - reservation_name = f"dws-eol-{node_name}" - try: - util.run(f"{lkp.scontrol} delete reservation {reservation_name}") - log.debug(f"Deleted Slurm reservation {reservation_name} for {node_name}") - except Exception as e: - log.error(f"Failed to delete reservation for {node_name}: {e}") - -def resume_flex_chunk(nodes: List[str], job_id: Optional[int], lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - assert len(nodeset.zone_policy_allow) > 0 - region = lkp.node_region(model) - - assert nodeset.dws_flex.enabled - - uid = str(uuid.uuid4())[:8] - if job_id: - mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-job-{job_id}-{uid}" - else: - mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-{uid}" - - # Create MIG - req = lkp.compute.regionInstanceGroupManagers().insert( - project=lkp.project, - region=region, - body=dict( - name=mig_name, - versions=[dict(instanceTemplate=nodeset.instance_template)], - targetSize=0, - distributionPolicy=dict( - zones=[ - dict(zone=f"zones/{z}") for z in nodeset.zone_policy_allow - ], - targetShape="ANY_SINGLE_ZONE" ), - updatePolicy = dict(instanceRedistributionType = "NONE" ), - instanceLifecyclePolicy=dict(defaultActionOnFailure= "DO_NOTHING" ), # TODO(FLEX): Not supported yet, migrate once supported - ) - ) - util.log_api_request(req) - op = req.execute() - res = util.wait_for_operation(op) - assert "error" not in res, f"{res}" - - # Create resize request - duration_seconds = _duration(nodeset.dws_flex, job_id, lkp) - req = lkp.compute.regionInstanceGroupManagerResizeRequests().insert( - project=lkp.project, - region=region, - instanceGroupManager=mig_name, - body=dict( - name="initial-resize", - instances=[dict(name=n) for n in nodes], - requested_run_duration=dict( - seconds=duration_seconds - ) - ) - ) - util.log_api_request(req) - op = req.execute() - res = util.wait_for_operation(op) - - # Create Slurm reservations if use_job_duration is set - if nodeset.dws_flex.use_job_duration: - # Get run duration (seconds) - run_duration = duration_seconds - for node_name in nodes: - # Fetch instance creation time from GCP instance (via util.py) - instance = lkp.instance(node_name) - if(instance and instance.creation_timestamp): - log.debug("creating with creation_timestamp") - boot_time = instance.creation_timestamp # Already a datetime object - else: - boot_time = datetime.utcnow() - log.debug("creating with utcnow time: {boot_time}") - _create_slurm_reservation(node_name, boot_time, run_duration, lkp) - - assert "error" not in res, f"{res}" - -def _suspend_flex_mig(mig_self_link: str, nodes: List[str], lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - assert len(nodeset.zone_policy_allow) > 0 - region = lkp.node_region(model) - project=lkp.project - instanceGroupManager=util.trim_self_link(mig_self_link) - - links = [ - f"zones/{inst.zone}/instances/{inst.name}" - for inst in [ - lkp.instance(node) for node in nodes - ] if inst - ] - - target_mig=lkp.get_mig(lkp.project, region, instanceGroupManager) - assert target_mig - - # TODO(FLEX): This will not work if MIG didn't obtain capacity yet. - # The request will fail and MIG will continue provisioning. - # Instead whole MIG should be deleted. - # + All other instances in MIG are not provisioned also, safe to delete - # - Need to come up will clear test to differentiate non-provisioned MIG and single VM being down; - # Particularly CRITICAL due to ActionOnFailure=DO_NOTHING - # - Need to `down_nodes_notify_jobs` for all nodes in MIG, make sure that it doesn't interfere with Slurm suspend-flow. - - if target_mig["targetSize"] == len(nodes): #We can just delete the whole MIG in this case - req = lkp.compute.regionInstanceGroupManagers().delete( - project=project, - region=region, - instanceGroupManager=instanceGroupManager, - ) - else: - req = lkp.compute.regionInstanceGroupManagers().deleteInstances( - project=project, - region=region, - instanceGroupManager=instanceGroupManager, - body=dict( - instances=links, - skipInstancesOnValidationError=True, - ) - ) - - util.log_api_request(req) - op = req.execute() - - res = util.wait_for_operation(op) - - # Delete Slurm reservations for nodes being deprovisioned - for node_name in nodes: - log.info("delete dws reservation") - _delete_slurm_reservation(node_name, lkp) - - assert "error" not in res, f"{res}" - -def _suspend_provisioning_inst(nodes:List[str], node_template:str, lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - assert len(nodeset.zone_policy_allow) > 0 - region = lkp.node_region(model) - - mig_list=lkp.get_mig_list(lkp.project, region) - - # FLEX (#TODO): If we enter this conditional it's likely this was called so early that MIG creation hasn't started - # Consider potentially retrying? No natural mechanism for retry currently but we could - # perhaps use slurmsync and then try it again to ensure it wasn't a case of being too early. - # This is important since we're now enabling long ResumeTimeout (Slurm won't call suspend on node within reasonable timeframe) - # so until we do this is slurmsync this is a temporary workaround. - - if not mig_list or not mig_list.get("items"): - log.info("No matching MIG found to delete! Retrying...") - sleep(5) - mig_list=lkp.get_mig_list(lkp.project, region) - if not mig_list or not mig_list.get("items"): - return - - for mig in mig_list["items"]: - if mig["instanceTemplate"] == node_template: - if mig["currentActions"]["creating"] > 0 and mig["targetSize"] == mig["currentActions"]["creating"]: - req = lkp.compute.regionInstanceGroupManagers().delete( - project=lkp.project, - region=region, - instanceGroupManager=util.trim_self_link(mig["selfLink"]), - ) - - util.log_api_request(req) - op = req.execute() - - res = util.wait_for_operation(op) - assert "error" not in res, f"{res}" - return - - log.info("No matching MIG found to delete!") - -def suspend_flex_nodes(nodes: List[str], lkp: util.Lookup) -> None: - by_mig = defaultdict(list) - not_provisioned = defaultdict(list) - for node in nodes: - inst = lkp.instance(node) - if not inst: - not_provisioned[lkp.node_template(node)].append(node) - else: - mig = inst.metadata.get("created-by") - if not mig: - log.error(f"Can not suspend {node}, can not find associated MIG") - continue - by_mig[mig].append(node) - - for mig, nodes in by_mig.items(): - _suspend_flex_mig(mig, nodes, lkp) - - for node_template, nodes in not_provisioned.items(): - _suspend_provisioning_inst(nodes, node_template, lkp) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt deleted file mode 100644 index 2ab3162ccf..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ -pytest -pytest-mock -pytest_unordered -mock - -types-mock -types-httplib2 -types-requests -types-PyYAML diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt deleted file mode 100644 index e923e53dbf..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -addict==2.4.0 -google-api-core==2.19.0 -google-api-python-client==2.93.0 -google-auth==2.40.3 -google-auth-httplib2==0.1.0 -google-cloud-bigquery==3.11.3 -google-cloud-core==2.3.3 -google-cloud-secret-manager~=2.22 -google-cloud-storage==2.10.0 -google-cloud-tpu==1.10.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.1 -grpcio==1.60.0 -grpcio-status==1.60.0 -httplib2==0.22.0 -more-executors==2.11.4 -pyyaml==6.0.2 -requests==2.32.4 diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py deleted file mode 100644 index ea0012a0b1..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py +++ /dev/null @@ -1,703 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional, Dict, Any -import argparse -from datetime import timedelta -import shlex -import json -import logging -import os -import yaml -import collections -from pathlib import Path -from dataclasses import dataclass -from addict import Dict as NSDict # type: ignore - -import util -from util import ( - chunked, - ensure_execute, - execute_with_futures, - log_api_request, - map_with_futures, - run, - separate, - to_hostlist, - trim_self_link, - wait_for_operation, -) -from util import lookup, ReservationDetails -import tpu -import mig_flex - -log = logging.getLogger() - -PLACEMENT_MAX_CNT = 1500 -# Placement group needs to be the same for an entire bulk_insert hence -# if placement is used the actual BULK_INSERT_LIMIT will be -# max([1000, PLACEMENT_MAX_CNT]) -BULK_INSERT_LIMIT = 5000 - -# https://cloud.google.com/compute/docs/instance-groups#types_of_managed_instance_groups -ZONAL_MIG_SIZE_LIMIT = 1000 - - -@dataclass(frozen=True) -class ResumeJobData: - job_id: int - partition: str - nodes_alloc: List[str] - -@dataclass(frozen=True) -class ResumeData: - jobs: List[ResumeJobData] - - -def get_resume_file_data() -> Optional[ResumeData]: - if not (path := os.getenv("SLURM_RESUME_FILE")): - log.error("SLURM_RESUME_FILE was not in environment. Cannot get detailed job, node, partition allocation data.") - return None - blob = Path(path).read_text() - log.debug(f"Resume data: {blob}") - data = json.loads(blob) - - jobs = [] - for jo in data.get("jobs", []): - job = ResumeJobData( - job_id = jo.get("job_id"), - partition = jo.get("partition"), - nodes_alloc = util.to_hostnames(jo.get("nodes_alloc")), - ) - jobs.append(job) - return ResumeData(jobs=jobs) - -def instance_properties(nodeset: NSDict, model:str, placement_group:Optional[str], labels:Optional[dict], job_id:Optional[int]): - props = NSDict() - - if labels: # merge in extra labels on instance and disks - template_link = lookup().node_template(model) - template_info = lookup().template_info(template_link) - - props.labels = {**template_info.labels, **labels} - - for disk in template_info.disks: - if disk.initializeParams.get("diskType", "local-ssd") == "local-ssd": - continue # do not label local ssd - disk.initializeParams.labels.update(labels) - props.disks = template_info.disks - - if placement_group: - props.resourcePolicies = [placement_group] - - if reservation := lookup().nodeset_reservation(nodeset): - update_reservation_props(reservation, props, placement_group, reservation.calendar) - - if (fr := lookup().future_reservation(nodeset)) and fr.specific: - assert fr.active_reservation - update_reservation_props(fr.active_reservation, props, placement_group, fr.calendar) - - if props.resourcePolicies: - props.scheduling.onHostMaintenance = "TERMINATE" - - if nodeset.maintenance_interval: - props.scheduling.maintenanceInterval = nodeset.maintenance_interval - - if nodeset.dws_flex.enabled and nodeset.dws_flex.use_bulk_insert: - update_props_dws(props, nodeset.dws_flex, job_id) - - # Override with properties explicit specified in the nodeset - props.update(nodeset.get("instance_properties") or {}) - return props - -def update_reservation_props(reservation:ReservationDetails, props:NSDict, placement_group:Optional[str], calendar_mode:bool) -> None: - props.reservationAffinity = { - "consumeReservationType": "SPECIFIC_RESERVATION", - "key": f"compute.{util.universe_domain()}/reservation-name", - "values": [reservation.bulk_insert_name], - } - - if reservation.dense or calendar_mode: - props.scheduling.provisioningModel = "RESERVATION_BOUND" - - # Figure out `resourcePolicies` - if reservation.policies: # use ones already attached to reservations - props.resourcePolicies = reservation.policies - elif reservation.dense and placement_group: # use once created by Slurm - props.resourcePolicies = [placement_group] - else: # vanilla reservations don't support external policies - props.resourcePolicies = [] - log.info( - f"reservation {reservation.bulk_insert_name} is being used with resourcePolicies: {props.resourcePolicies}") - -def update_props_dws(props: NSDict, dws_flex: NSDict, job_id: Optional[int]) -> None: - props.scheduling.onHostMaintenance = "TERMINATE" - props.scheduling.instanceTerminationAction = "DELETE" - props.reservationAffinity['consumeReservationType'] = "NO_RESERVATION" - props.scheduling.maxRunDuration['seconds'] = dws_flex_duration(dws_flex, job_id) - -def dws_flex_duration(dws_flex: NSDict, job_id: Optional[int]) -> int: - max_duration = dws_flex.max_run_duration - if dws_flex.use_job_duration and job_id is not None and (job := lookup().job(job_id)) and job.duration: - if timedelta(seconds=30) <= job.duration <= timedelta(weeks=1): - max_duration = int(job.duration.total_seconds()) - else: - log.info("Job TimeLimit cannot be less than 30 seconds or exceed one week") - return max_duration - -def create_instances_request(nodes: List[str], placement_group: Optional[str], excl_job_id: Optional[int]): - """Call regionInstances.bulkInsert to create instances""" - assert 0 < len(nodes) <= BULK_INSERT_LIMIT - - # model here indicates any node that can be used to describe the rest - model = next(iter(nodes)) - log.debug(f"create_instances_request: {model} placement: {placement_group}") - - nodeset = lookup().node_nodeset(model) - template = lookup().node_template(model) - labels = {"slurm_job_id": excl_job_id} if excl_job_id else None - - body = dict( - count = len(nodes), - sourceInstanceTemplate = template, - # key is instance name, value overwrites properties (no overwrites) - perInstanceProperties = {k: {} for k in nodes}, - instanceProperties = instance_properties( - nodeset, model, placement_group, labels, excl_job_id - ), - ) - - if placement_group and excl_job_id is not None: - pass # do not set minCount to force "all or nothing" behavior - else: - body["minCount"] = 1 - - zone_allow = nodeset.zone_policy_allow or [] - zone_deny = nodeset.zone_policy_deny or [] - - if len(zone_allow) == 1: # if only one zone is used, use zonal BulkInsert API, as less prone to errors - api_method = lookup().compute.instances().bulkInsert - method_args = {"zone": zone_allow[0]} - else: - api_method = lookup().compute.regionInstances().bulkInsert - method_args = {"region": lookup().node_region(model)} - - body["locationPolicy"] = dict( - locations = { - **{ f"zones/{z}": {"preference": "ALLOW"} for z in zone_allow }, - **{ f"zones/{z}": {"preference": "DENY"} for z in zone_deny }}, - targetShape = nodeset.zone_target_shape, - ) - - req = api_method( - project=lookup().project, - body=body, - **method_args) - log.debug(f"new request: endpoint={req.methodId} nodes={to_hostlist(nodes)}") - log_api_request(req) - return req - -@dataclass() -class PlacementAndNodes: - placement: Optional[str] - nodes: List[str] - -@dataclass(frozen=True) -class BulkChunk: - nodes: List[str] - prefix: str # - - chunk_idx: int - excl_job_id: Optional[int] - placement_group: Optional[str] = None - - @property - def name(self): - if self.placement_group is not None: - return f"{self.prefix}:job{self.excl_job_id}:{self.placement_group}:{self.chunk_idx}" - if self.excl_job_id is not None: - return f"{self.prefix}:job{self.excl_job_id}:{self.chunk_idx}" - return f"{self.prefix}:{self.chunk_idx}" - - -def group_nodes_bulk(nodes: List[str], resume_data: Optional[ResumeData], lkp: util.Lookup): - """group nodes by nodeset, placement_group, exclusive_job_id if any""" - if resume_data is None: # all nodes will be considered jobless - resume_data = ResumeData(jobs=[]) - - nodes_set = set(nodes) # turn into set to simplify intersection - non_excl = nodes_set.copy() - groups : Dict[Optional[int], List[PlacementAndNodes]] = {} # excl_job_id|none -> PlacementAndNodes - - # expand all exclusive job nodelists - for job in resume_data.jobs: - if not lkp.cfg.partitions[job.partition].enable_job_exclusive: - continue - - groups[job.job_id] = [] - # placement group assignment is based on all allocated nodes, ... - for pn in create_placements(job.nodes_alloc, job.job_id, lkp): - groups[job.job_id].append( - PlacementAndNodes( - placement=pn.placement, - #... but we only want to handle nodes in nodes_resume in this run. - nodes = sorted(set(pn.nodes) & nodes_set) - )) - non_excl.difference_update(job.nodes_alloc) - - groups[None] = create_placements(sorted(non_excl), excl_job_id=None, lkp=lkp) - - def chunk_nodes(nodes: List[str]): - if not nodes: - return [] - - model = nodes[0] - - if lkp.is_flex_node(model): - chunk_size = ZONAL_MIG_SIZE_LIMIT - elif lkp.node_is_tpu(model): - ns_name = lkp.node_nodeset_name(model) - chunk_size = tpu.TPU.make(ns_name, lkp).vmcount - else: - chunk_size = BULK_INSERT_LIMIT - - return chunked(nodes, n=chunk_size) - - chunks = [ - BulkChunk( - nodes=nodes_chunk, - prefix=lkp.node_prefix(nodes_chunk[0]), # - - excl_job_id = job_id, - placement_group=pn.placement, - chunk_idx=i) - - for job_id, placements in groups.items() - for pn in placements if pn.nodes - for i, nodes_chunk in enumerate(chunk_nodes(pn.nodes)) - ] - return {chunk.name: chunk for chunk in chunks} - - -def resume_nodes(nodes: List[str], resume_data: Optional[ResumeData]): - """resume nodes in nodelist""" - lkp = lookup() - # Prevent dormant nodes associated with a reservation from being resumed - nodes, dormant_res_nodes = util.separate(lkp.is_dormant_res_node, nodes) - - if dormant_res_nodes: - log.warning(f"Resume was unable to resume reservation nodes={dormant_res_nodes}") - down_nodes_notify_jobs(dormant_res_nodes, "Reservation is not active, nodes cannot be resumed", resume_data) - - nodes, flex_managed = util.separate(lkp.is_provisioning_flex_node, nodes) - if flex_managed: - log.warning(f"Resume was unable to resume nodes={flex_managed} already managed by MIGs") - down_nodes_notify_jobs(flex_managed, "VM is managed MIG, can not be resumed", resume_data) - - if not nodes: - log.info("No nodes to resume") - return - - nodes = sorted(nodes, key=lkp.node_prefix) - grouped_nodes = group_nodes_bulk(nodes, resume_data, lkp) - - if log.isEnabledFor(logging.DEBUG): - grouped_nodelists = { - group: to_hostlist(chunk.nodes) for group, chunk in grouped_nodes.items() - } - log.debug( - "node bulk groups: \n{}".format(yaml.safe_dump(grouped_nodelists).rstrip()) - ) - - tpu_chunks, flex_chunks = [], [] - bi_inserts = {} - - for group, chunk in grouped_nodes.items(): - model = chunk.nodes[0] - - if lkp.node_is_tpu(model): - tpu_chunks.append(chunk.nodes) - elif lkp.is_flex_node(model): - flex_chunks.append(chunk) - else: - bi_inserts[group] = create_instances_request( - chunk.nodes, chunk.placement_group, chunk.excl_job_id - ) - - for chunk in flex_chunks: - mig_flex.resume_flex_chunk(chunk.nodes, chunk.excl_job_id, lkp) - - # execute all bulkInsert requests with batch - bulk_ops = dict( - zip(bi_inserts.keys(), map_with_futures(ensure_execute, bi_inserts.values())) - ) - log.debug(f"bulk_ops={yaml.safe_dump(bulk_ops)}") - started = { - group: op for group, op in bulk_ops.items() if not isinstance(op, Exception) - } - failed = { - group: err for group, err in bulk_ops.items() if isinstance(err, Exception) - } - if failed: - failed_reqs = [str(e) for e in failed.items()] - log.error("bulkInsert API failures: {}".format("; ".join(failed_reqs))) - for ident, exc in failed.items(): - down_nodes_notify_jobs(grouped_nodes[ident].nodes, f"GCP Error: {exc._get_reason()}", resume_data) # type: ignore - - if log.isEnabledFor(logging.DEBUG): - for group, op in started.items(): - group_nodes = grouped_nodelists[group] - name = op["name"] - gid = op["operationGroupId"] - log.debug( - f"new bulkInsert operation started: group={group} nodes={group_nodes} name={name} operationGroupId={gid}" - ) - # wait for all bulkInserts to complete and log any errors - bulk_operations = {group: wait_for_operation(op) for group, op in started.items()} - - # Start TPU after regular nodes so that regular nodes are not affected by the slower TPU nodes - execute_with_futures(tpu.start_tpu, tpu_chunks) - - for group, op in bulk_operations.items(): - _handle_bulk_insert_op(op, grouped_nodes[group].nodes, resume_data) - - -def _get_failed_zonal_instance_inserts(bulk_op: Any, zone: str, lkp: util.Lookup) -> list[Any]: - group_id = bulk_op["operationGroupId"] - user = bulk_op["user"] - started = bulk_op["startTime"] - ended = bulk_op["endTime"] - - fltr = f'(user eq "{user}") AND (operationType eq "insert") AND (creationTimestamp > "{started}") AND (creationTimestamp < "{ended}")' - act = lkp.compute.zoneOperations() - req = act.list(project=lkp.project, zone=zone, filter=fltr) - ops = [] - while req is not None: - result = util.ensure_execute(req) - for op in result.get("items", []): - if op.get("operationGroupId") == group_id and "error" in op: - ops.append(op) - req = act.list_next(req, result) - return ops - - -def _get_failed_instance_inserts(bulk_op: Any, lkp: util.Lookup) -> list[Any]: - zones = set() # gather zones that had failed inserts - for loc, stat in bulk_op.get("instancesBulkInsertOperationMetadata", {}).get("perLocationStatus", {}).items(): - pref, zone = loc.split("/", 1) - if not pref == "zones": - log.error(f"Unexpected location: {loc} in operation {bulk_op['name']}") - continue - if stat.get("targetVmCount", 0) != stat.get("createdVmCount", 0): - zones.add(zone) - - res = [] - for zone in zones: - res.extend(_get_failed_zonal_instance_inserts(bulk_op, zone, lkp)) - return res - -def _handle_bulk_insert_op(op: Dict, nodes: List[str], resume_data: Optional[ResumeData]) -> None: - """ - Handles **DONE** BulkInsert operations - """ - assert op["operationType"] == "bulkInsert" and op["status"] == "DONE", f"unexpected op: {op}" - - group_id = op["operationGroupId"] - if "error" in op: - error = op["error"]["errors"][0] - log.error( - f"bulkInsert operation error: {error['code']} name={op['name']} operationGroupId={group_id} nodes={to_hostlist(nodes)}" - ) - - created = 0 - for status in op["instancesBulkInsertOperationMetadata"]["perLocationStatus"].values(): - created += status.get("createdVmCount", 0) - if created == len(nodes): - log.info(f"created {len(nodes)} instances: nodes={to_hostlist(nodes)}") - return # no need to gather status of insert-operations. - - # TODO: don't gather insert-operations per bulkInsert request, instead aggregate it - # across all bulkInserts (goes one level above this function) - failed = _get_failed_instance_inserts(op, util.lookup()) - - # Multiple errors are possible, group by all of them (joined string codes) - by_error_inserts = util.groupby_unsorted( - failed, - lambda op: "+".join(err["code"] for err in op["error"]["errors"]), - ) - for code, failed_ops in by_error_inserts: - failed_ops = list(failed_ops) - failed_nodes = [trim_self_link(op["targetLink"]) for op in failed_ops] - hostlist = util.to_hostlist(failed_nodes) - log.error( - f"{len(failed_nodes)} instances failed to start: {code} ({hostlist}) operationGroupId={group_id}" - ) - - msg = "; ".join( - f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" - for err in failed_ops[0]["error"]["errors"] - ) - if code != "RESOURCE_ALREADY_EXISTS": - down_nodes_notify_jobs(failed_nodes, f"GCP Error: {msg}", resume_data) - log.error( - f"errors from insert for node '{failed_nodes[0]}' ({failed_ops[0]['name']}): {msg}" - ) - - -def down_nodes_notify_jobs(nodes: List[str], reason: str, resume_data: Optional[ResumeData]) -> None: - """set nodes down with reason""" - nodes_set = set(nodes) # turn into set to speed up intersection - jobs = resume_data.jobs if resume_data else [] - reason_quoted = shlex.quote(reason) - - for job in jobs: - if not (set(job.nodes_alloc) & nodes_set): - continue - run(f"{lookup().scontrol} update jobid={job.job_id} admincomment={reason_quoted}", check=False) - run(f"{lookup().scontrol} notify {job.job_id} {reason_quoted}", check=False) - - nodelist = util.to_hostlist(nodes) - log.error(f"Marking nodes {nodelist} as DOWN, reason: {reason}") - run(f"{lookup().scontrol} update nodename={nodelist} state=down reason={reason_quoted}", check=False) - - - - -def create_placement_request(pg_name: str, region: str, max_distance: Optional[int], accelerator_topology: Optional[str]): - config = { - "name": pg_name, - "region": region, - "groupPlacementPolicy": { - "collocation": "COLLOCATED", - "maxDistance": max_distance, - "gpuTopology": accelerator_topology, - }, - } - - request = lookup().compute.resourcePolicies().insert( - project=lookup().project, region=region, body=config - ) - log_api_request(request) - return request - - -def create_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - nodeset_map = collections.defaultdict(list) - for node in nodes: # split nodes on nodesets - nodeset_map[lkp.node_nodeset_name(node)].append(node) - - placements = [] - for _, ns_nodes in nodeset_map.items(): - placements.extend(create_nodeset_placements(ns_nodes, excl_job_id, lkp)) - return placements - - -def _allocate_nodes_to_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - # canned result for no placement policies created - no_pp = [PlacementAndNodes(placement=None, nodes=nodes)] - - model = nodes[0] - nodeset = lkp.node_nodeset(model) - - is_slice = bool(getattr(nodeset, 'accelerator_topology', None)) - - excl_job_placement = (excl_job_id is not None) and (not is_slice) - - if excl_job_placement and len(nodes) < 2: - return no_pp # don't create placement_policy for just one node - - if lkp.is_flex_node(model): - return no_pp # TODO(FLEX): Add support for workload policies - if lkp.node_is_tpu(model): - return no_pp - if not (nodeset.enable_placement and valid_placement_node(model)): - return no_pp - - max_count = calculate_chunk_size(nodeset, lkp) - - name_prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-{nodeset.nodeset_name}" - - if excl_job_placement: # simply chunk given nodes by max size of placement - return [ - PlacementAndNodes(placement=f"{name_prefix}-{excl_job_id}-{i}", nodes=chunk) - for i, chunk in enumerate(chunked(nodes, n=max_count)) - ] - - # split whole nodeset (not only nodes to resume) into chunks of max size of placement - # create placements (most likely already exists) placements for requested nodes - chunks = collections.defaultdict(list) # chunk_id -> nodes - invalid = [] - - for node in nodes: - try: - chunk = lkp.node_index(node) // max_count - chunks[chunk].append(node) - except: - invalid.append(node) - - placements = [ - # NOTE: use 0 instead of job_id for consistency with previous SlurmGCP behavior - PlacementAndNodes(placement=f"{name_prefix}-0-{c_id}", nodes=c_nodes) - for c_id, c_nodes in chunks.items() - ] - - if invalid: - placements.append(PlacementAndNodes(placement=None, nodes=invalid)) - log.error(f"Could not find placement for nodes with unexpected names: {to_hostlist(invalid)}") - - return placements - -def calculate_hosts_per_topo(accelerator_topology: str, machine_type: NSDict) -> int: - # Calculate total number of hosts per topology (Assumes format: '1x72') - try: - top_split = [int(x) for x in accelerator_topology.split("x")] - except Exception as e: - log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") - raise e - - if len(machine_type.accelerators) == 0: - gpus_per_machine = 0 - else: - gpus_per_machine = machine_type.accelerators[0].count - - if len(top_split) != 2: - log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") - elif top_split[0] <= 0 or top_split[1] <= 0: - log.error(f"Accelerator topology {accelerator_topology} is formatted incorrectly.") - elif gpus_per_machine <= 0: - log.error(f"The machine type has no accelerators. Cannot use accelerator topology {accelerator_topology}.") - elif top_split[1] % gpus_per_machine: - log.error(f"The GPU count {gpus_per_machine} per node is not a factor of the accelerator topology {accelerator_topology}") - - return (top_split[0] * top_split[1]) // gpus_per_machine - -def calculate_chunk_size(nodeset: NSDict, lkp: util.Lookup) -> int: - # Calculates the chunk size based on max distance value received or accelerator topology - # Assuming nodeset is not tpu - machine_type = lkp.template_info(nodeset.instance_template).machine_type - max_distance = nodeset.placement_max_distance - accelerator_topology = nodeset.accelerator_topology - - # Look for accelerator topology first - if accelerator_topology: - hosts_per_topo = calculate_hosts_per_topo(accelerator_topology, machine_type) - return hosts_per_topo - - if max_distance == 1: - return 22 - elif max_distance == 2: - if machine_type.family.startswith("a3"): - return 256 - else: - return 150 - elif max_distance == 3: - return 1500 - else: - return PLACEMENT_MAX_CNT - -def create_nodeset_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - placements = _allocate_nodes_to_placements(nodes, excl_job_id, lkp) - region = lkp.node_region(nodes[0]) - max_distance = lkp.node_nodeset(nodes[0]).get('placement_max_distance') - accelerator_topology = lkp.nodeset_accelerator_topology(lkp.node_nodeset_name(nodes[0])) - - if log.isEnabledFor(logging.DEBUG): - debug_p = {p.placement: to_hostlist(p.nodes) for p in placements} - log.debug( - f"creating {len(placements)} placement groups: \n{yaml.safe_dump(debug_p).rstrip()}" - ) - - requests = { - p.placement: create_placement_request(p.placement, region, max_distance, accelerator_topology) for p in placements if p.placement - } - if not requests: - return placements - # TODO: aggregate all requests for whole resume and execute them at once (don't limit to nodeset/job) - ops = dict( - zip(requests.keys(), map_with_futures(ensure_execute, requests.values())) - ) - - def classify_result(item): - op = item[1] - if not isinstance(op, Exception): - return "submitted" - if all(e.get("reason") == "alreadyExists" for e in op.error_details): # type: ignore - return "redundant" - return "failed" - - grouped_ops = dict(util.groupby_unsorted(list(ops.items()), classify_result)) - submitted, redundant, failed = ( - dict(grouped_ops.get(key, {})) for key in ("submitted", "redundant", "failed") - ) - if redundant: - log.warning( - "placement policies already exist: {}".format(",".join(redundant.keys())) - ) - if failed: - reqs = [f"{e}" for _, e in failed.values()] - log.fatal("failed to create placement policies: {}".format("; ".join(reqs))) - operations = {group: wait_for_operation(op) for group, op in submitted.items()} - for group, op in operations.items(): - if "error" in op: - msg = "; ".join( - f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" - for err in op["error"]["errors"] - ) - log.error( - f"placement group failed to create: '{group}' ({op['name']}): {msg}" - ) - - log.info( - f"created {len(operations)} placement groups ({to_hostlist(operations.keys())})" - ) - return placements - - -def valid_placement_node(node: str) -> bool: - invalid_types = frozenset(["e2", "t2d", "n1", "t2a", "m1", "m2", "m3"]) - mt = lookup().node_template_info(node).machineType - if mt.split("-")[0] in invalid_types: - log.warn(f"Unsupported machine type for placement policy: {mt}.") - log.warn( - f"Please do not use any the following machine types with placement policy: ({','.join(invalid_types)})" - ) - return False - return True - - -def main(nodelist: str) -> None: - """main called when run as script""" - log.debug(f"ResumeProgram {nodelist}") - # Filter out nodes not in config.yaml - other_nodes, nodes = separate( - lookup().is_power_managed_node, util.to_hostnames(nodelist) - ) - if other_nodes: - log.error( - f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" - ) - - if not nodes: - log.info("No nodes to resume") - return - resume_data = get_resume_file_data() - log.info(f"resume {util.to_hostlist(nodes)}") - resume_nodes(nodes, resume_data) - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("nodelist", help="list of nodes to resume") - args = util.init_log_and_parse(parser) - main(args.nodelist) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh deleted file mode 100644 index 023d246f01..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PYTHON_SCRIPT="${SCRIPT_DIR}/resume.py" - -# Capture all arguments passed by Slurm (the nodelist). -ALL_ARGS=("$@") - -# This array will hold extra argument for resume.py, like the resume data file. -UNIQUE_RESUME_FILE="" - -# Handle SLURM_RESUME_FILE if provided -if [ -n "${SLURM_RESUME_FILE-}" ] && [ -f "$SLURM_RESUME_FILE" ]; then - SAFE_DIR="/tmp/slurm_resume_data" - mkdir -p "$SAFE_DIR" - - UNIQUE_RESUME_FILE="${SAFE_DIR}/resumedata.$$.json" - cp "$SLURM_RESUME_FILE" "$UNIQUE_RESUME_FILE" -fi - -SLURM_RESUME_FILE="${UNIQUE_RESUME_FILE}" -setsid "${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & - -exit 0 diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py deleted file mode 100644 index 846524adf2..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py +++ /dev/null @@ -1,660 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import logging -import os -import shutil -import subprocess -import stat -import time -import yaml -from pathlib import Path -import functools - -import util -from util import ( - lookup, - dirs, - slurmdirs, - run, - install_custom_scripts, -) -import conf -import slurmsync - -from setup_network_storage import ( - setup_network_storage, - setup_nfs_exports, -) - - -log = logging.getLogger() - - -MOTD_HEADER = """ - SSSSSSS - SSSSSSSSS - SSSSSSSSS - SSSSSSSSS - SSSS SSSSSSS SSSS - SSSSSS SSSSSS - SSSSSS SSSSSSS SSSSSS - SSSS SSSSSSSSS SSSS - SSS SSSSSSSSS SSS - SSSSS SSSS SSSSSSSSS SSSS SSSSS - SSS SSSSSS SSSSSSSSS SSSSSS SSS - SSSSSS SSSSSSS SSSSSS - SSS SSSSSS SSSSSS SSS - SSSSS SSSS SSSSSSS SSSS SSSSS - S SSS SSSSSSSSS SSS S - SSS SSSS SSSSSSSSS SSSS SSS - S SSS SSSSSS SSSSSSSSS SSSSSS SSS S - SSSSS SSSSSS SSSSSSSSS SSSSSS SSSSS - S SSSSS SSSS SSSSSSS SSSS SSSSS S - S SSS SSS SSS SSS S - S S S S - SSS - SSS - SSS - SSS - SSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSS SSSSSSSSSSSSSSSSSSSS -SSSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSSS SSSSSSSSSSSSSSSSSSSSSS -SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSSS SSS SSSSSSSSSSSSSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSS SSS SSSSSSSSSSSSS SSSS SSSS SSSS SSSS - -""" -_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" - -def start_motd(): - """advise in motd that slurm is currently configuring""" - wall_msg = "*** Slurm is currently being configured in the background. ***" - motd_msg = MOTD_HEADER + wall_msg + "\n\n" - Path("/etc/motd").write_text(motd_msg) - util.run(f"wall -n '{wall_msg}'", timeout=30) - - -def end_motd(broadcast=True): - """modify motd to signal that setup is complete""" - Path("/etc/motd").write_text(MOTD_HEADER) - - if not broadcast: - return - - run( - "wall -n '*** Slurm {} setup complete ***'".format(lookup().instance_role), - timeout=30, - ) - if not lookup().is_controller: - run( - """wall -n ' -/home on the controller was mounted over the existing /home. -Log back in to ensure your home directory is correct. -'""", - timeout=30, - ) - - -def failed_motd(): - """modify motd to signal that setup is failed""" - wall_msg = f"*** Slurm setup failed! Please view log: {util.get_log_path()} ***" - motd_msg = MOTD_HEADER + wall_msg + "\n\n" - Path("/etc/motd").write_text(motd_msg) - util.run(f"wall -n '{wall_msg}'", timeout=30) - - -def _startup_script_timeout(lkp: util.Lookup) -> int: - if lkp.is_controller: - return lkp.cfg.get("controller_startup_scripts_timeout", 300) - elif lkp.instance_role == "compute": - return lkp.cfg.get("compute_startup_scripts_timeout", 300) - elif lkp.is_login_node: - return lkp.cfg.login_groups[util.instance_login_group()].get("startup_scripts_timeout", 300) - return 300 - - -def run_custom_scripts(): - """run custom scripts based on instance_role""" - custom_dir = dirs.custom_scripts - if lookup().is_controller: - # controller has all scripts, but only runs controller.d - custom_dirs = [custom_dir / "controller.d"] - elif lookup().instance_role == "compute": - # compute setup with nodeset.d - custom_dirs = [custom_dir / "nodeset.d"] - elif lookup().is_login_node: - # login setup with only login.d - custom_dirs = [custom_dir / "login.d"] - else: - # Unknown role: run nothing - custom_dirs = [] - - timeout = _startup_script_timeout(lookup()) - - custom_scripts = [ - p - for d in custom_dirs - for p in d.rglob("*") - if p.is_file() and not p.name.endswith(".disabled") - ] - print_scripts = ",".join(str(s.relative_to(custom_dir)) for s in custom_scripts) - log.debug(f"custom scripts to run: {custom_dir}/({print_scripts})") - - try: - for script in custom_scripts: - log.info(f"running script {script.name} with timeout={timeout}") - result = run(str(script), timeout=timeout, check=False, shell=True) - runlog = ( - f"{script.name} returncode={result.returncode}\n" - f"stdout={result.stdout}stderr={result.stderr}" - ) - log.info(runlog) - result.check_returncode() - except OSError as e: - log.error(f"script {script} is not executable") - raise e - except subprocess.TimeoutExpired as e: - log.error(f"script {script} did not complete within timeout={timeout}") - raise e - except Exception as e: - log.exception(f"script {script} encountered an exception") - raise e - -def mount_save_state_disk(): - disk_name = f"/dev/disk/by-id/google-{lookup().cfg.controller_state_disk.device_name}" - mount_point = util.slurmdirs.state - fs_type = "ext4" - - rdevice = util.run(f"realpath {disk_name}").stdout.strip() - file_output = util.run(f"file -s {rdevice}").stdout.strip() - if "filesystem" not in file_output: - util.run(f"mkfs -t {fs_type} -q {rdevice}") - - fstab_entry = f"{disk_name} {mount_point} {fs_type}" - with open("/etc/fstab", "r") as f: - fstab = f.readlines() - if fstab_entry not in fstab: - with open("/etc/fstab", "a") as f: - f.write(f"{fstab_entry} defaults 0 0\n") - - util.run(f"systemctl daemon-reload") - - os.makedirs(mount_point, exist_ok=True) - util.run(f"mount {mount_point}") - - util.chown_slurm(mount_point) - - -def setup_jwt_key(): - jwt_key = Path(slurmdirs.state / "jwt_hs256.key") - - if jwt_key.exists(): - log.info("JWT key already exists. Skipping key generation.") - else: - run("dd if=/dev/urandom bs=32 count=1 > " + str(jwt_key), shell=True) - - util.chown_slurm(jwt_key, mode=0o400) - - -def _generate_key(p: Path) -> None: - run(f"dd if=/dev/random of={p} bs=1024 count=1") - - -def setup_key(lkp: util.Lookup) -> None: - file_name = "munge.key" - dir = dirs.munge - - if lkp.cfg.enable_slurm_auth: - file_name = "slurm.key" - dir = slurmdirs.etc - - dst = Path(dir / file_name) - - if lkp.cfg.controller_state_disk.device_name: - # Copy key from persistent state disk - persist = slurmdirs.state / file_name - if not persist.exists(): - _generate_key(persist) - - shutil.copyfile(persist, dst) - if lkp.cfg.enable_slurm_auth: - util.chown_slurm(dst, mode=0o400) - util.chown_slurm(persist, mode=0o400) - else: - shutil.chown(dst, user="munge", group="munge") - os.chmod(dst, stat.S_IRUSR) - else: - if dst.exists(): - log.info("key already exists. Skipping key generation.") - else: - _generate_key(dst) - if lkp.cfg.enable_slurm_auth: - util.chown_slurm(dst, mode=0o400) - else: - shutil.chown(dst, user="munge", group="munge") - os.chmod(dst, stat.S_IRUSR) - - if lkp.cfg.enable_slurm_auth: - # Put key into shared volume for distribution - distributed = util.slurmdirs.key_distribution / file_name - shutil.copyfile(dst, distributed) - util.chown_slurm(distributed, mode=0o400) - # Munge is distributed from /etc/munge. - else: - run("systemctl restart munge", timeout=30) - - -def setup_nss_slurm(): - """install and configure nss_slurm""" - # setup nss_slurm - util.mkdirp(Path("/var/spool/slurmd")) - run( - "ln -s {}/lib/libnss_slurm.so.2 /usr/lib64/libnss_slurm.so.2".format( - slurmdirs.prefix - ), - check=False, - ) - run(r"sed -i 's/\(^\(passwd\|group\):\s\+\)/\1slurm /g' /etc/nsswitch.conf") - - -def setup_sudoers(): - content = """ -# Allow SlurmUser to manage the slurm daemons -slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmd.service -slurm ALL= NOPASSWD: /usr/bin/systemctl restart sackd.service -slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmctld.service -""" - sudoers_file = Path("/etc/sudoers.d/slurm") - sudoers_file.write_text(content) - sudoers_file.chmod(0o0440) - - -def setup_maintenance_script(): - perform_maintenance = """#!/bin/bash - -#SBATCH --priority=low -#SBATCH --time=180 - -VM_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") -ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" | cut -d '/' -f 4) - -gcloud compute instances perform-maintenance $VM_NAME \ - --zone=$ZONE -""" - - - with open(_MAINTENANCE_SBATCH_SCRIPT_PATH, "w") as f: - f.write(perform_maintenance) - - util.chown_slurm(_MAINTENANCE_SBATCH_SCRIPT_PATH, mode=0o755) - - -def update_system_config(file, content): - """Add system defaults options for service files""" - sysconfig = Path("/etc/sysconfig") - default = Path("/etc/default") - - if sysconfig.exists(): - conf_dir = sysconfig - elif default.exists(): - conf_dir = default - else: - raise Exception("Cannot determine system configuration directory.") - - slurmd_file = Path(conf_dir, file) - slurmd_file.write_text(content) - -def _symlink_mysql_datadir(lkp: util.Lookup) -> None: - """ Symlink /var/lib/mysql to controller state disk if needed. """ - if not lkp.cfg.controller_state_disk.device_name: - return - - datadir = Path("/var/lib/mysql") - dst = slurmdirs.state / "mysql" - - if dst.exists(): - run(f"rm -rf {datadir}") - else: - shutil.move(datadir, dst) - - datadir.symlink_to(dst, target_is_directory=True) - shutil.chown(datadir, user="mysql", group="mysql") - run(f"chown -R mysql:mysql {dst}") - -def configure_mysql(lkp: util.Lookup) -> None: - cnfdir = Path("/etc/my.cnf.d") - if not cnfdir.exists(): - cnfdir = Path("/etc/mysql/conf.d") - if not (cnfdir / "mysql_slurm.cnf").exists(): - (cnfdir / "mysql_slurm.cnf").write_text( - """ -[mysqld] -bind-address=127.0.0.1 -innodb_buffer_pool_size=1024M -innodb_log_file_size=64M -innodb_lock_wait_timeout=900 -""" - ) - - run("systemctl stop mariadb", timeout=30) - _symlink_mysql_datadir(lkp) - - run("systemctl enable mariadb", timeout=30) - run("systemctl restart mariadb", timeout=30) - - db_name = "slurm_acct_db" - - - cmd = "mysql -u root -e" - for host in ("localhost", lkp.control_host): - run(f"""{cmd} "drop user if exists 'slurm'@'{host}'";""", timeout=30) - run(f"""{cmd} "create user 'slurm'@'{host}'";""", timeout=30) - run(f"""{cmd} "grant all on {db_name}.* TO 'slurm'@'{host}'";""", timeout=30) - - -def configure_dirs(): - for p in dirs.values(): - util.mkdirp(p) - - for p in (dirs.slurm, dirs.scripts, dirs.custom_scripts): - util.chown_slurm(p) - - for p in slurmdirs.values(): - util.mkdirp(p) - util.chown_slurm(p) - - for sl, tgt in ( # create symlinks - (Path("/etc/slurm"), slurmdirs.etc), - (dirs.scripts / "etc", slurmdirs.etc), - (dirs.scripts / "log", dirs.log), - ): - if sl.exists() and sl.is_symlink(): - sl.unlink() - sl.symlink_to(tgt) - - # copy auxiliary scripts - for dst_folder, src_file in ((lookup().cfg.slurm_bin_dir, - Path("sort_nodes.py")), - (dirs.custom_scripts / "task_prolog.d", - Path("tools/task-prolog")), - (dirs.custom_scripts / "task_epilog.d", - Path("tools/task-epilog"))): - dst = Path(dst_folder) / src_file.name - util.mkdirp(dst.parent) - shutil.copyfile(util.scripts_dir / src_file, dst) - os.chmod(dst, 0o755) - - -def self_report_controller_address(lkp: util.Lookup) -> None: - if not lkp.cfg.controller_network_attachment: - return # only self report address if network attachment is used - data = { "slurm_control_addr": lkp.cfg.slurm_control_addr } - bucket, prefix = util._get_bucket_and_common_prefix() - blob = util.storage_client().bucket(bucket).blob(f"{prefix}/controller_addr.yaml") - with blob.open('w') as f: - f.write(yaml.dump(data)) - -def setup_controller(): - """Run controller setup""" - log.info("Setting up controller") - lkp = util.lookup() - util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) - install_custom_scripts() - conf.gen_controller_configs(lkp) - - if lkp.cfg.controller_state_disk.device_name != None: - mount_save_state_disk() - - setup_jwt_key() - setup_key(lkp) - - setup_sudoers() - setup_network_storage() - - run_custom_scripts() - - if not lkp.cfg.cloudsql_secret: - configure_mysql(lkp) - - run("systemctl enable slurmdbd", timeout=30) - run("systemctl restart slurmdbd", timeout=30) - - # Wait for slurmdbd to come up - time.sleep(5) - - sacctmgr = f"{slurmdirs.prefix}/bin/sacctmgr -i" - result = run( - f"{sacctmgr} add cluster {lkp.cfg.slurm_cluster_name}", timeout=30, check=False - ) - if "already exists" in result.stdout: - log.info(result.stdout) - elif result.returncode > 1: - result.check_returncode() # will raise error - - run("systemctl enable slurmctld", timeout=30) - run("systemctl restart slurmctld", timeout=30) - - run("systemctl enable slurmrestd", timeout=30) - run("systemctl restart slurmrestd", timeout=30) - - # Export at the end to signal that everything is up - run("systemctl enable nfs-server", timeout=30) - run("systemctl start nfs-server", timeout=30) - - setup_nfs_exports() - run("systemctl enable --now slurmcmd.timer", timeout=30) - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status slurmdbd", timeout=30) - run("systemctl status slurmctld", timeout=30) - run("systemctl status slurmrestd", timeout=30) - - try: - slurmsync.sync_instances() - except Exception: - log.exception("Failed to sync instances, will try next time.") - - run("systemctl enable slurm_load_bq.timer", timeout=30) - run("systemctl start slurm_load_bq.timer", timeout=30) - run("systemctl status slurm_load_bq.timer", timeout=30) - - # Add script to perform maintenance - setup_maintenance_script() - - self_report_controller_address(lkp) - - log.info("Done setting up controller") - pass - - -def setup_login(): - """run login node setup""" - log.info("Setting up login") - - lkp = lookup() - slurmctld_host = f"{lkp.control_host}" - if lkp.control_addr: - slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" - sackd_options = [ - f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', - ] - sysconf = f"""SACKD_OPTIONS='{" ".join(sackd_options)}'""" - update_system_config("sackd", sysconf) - install_custom_scripts() - - setup_network_storage() - setup_sudoers() - if not lkp.cfg.enable_slurm_auth: - run("systemctl restart munge", timeout=30) - run("systemctl enable sackd", timeout=30) - run("systemctl restart sackd", timeout=30) - run("systemctl enable --now slurmcmd.timer", timeout=30) - - run_custom_scripts() - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status sackd", timeout=30) - - log.info("Done setting up login") - - -def setup_compute(): - """run compute node setup""" - log.info("Setting up compute") - - lkp = lookup() - util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) - slurmctld_host = f"{lkp.control_host}" - if lkp.control_addr: - slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" - slurmd_options = [ - f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', - ] - - try: - slurmd_feature = util.instance_metadata("attributes/slurmd_feature", silent=True) - except util.MetadataNotFoundError: - slurmd_feature = None - - if slurmd_feature is not None: - slurmd_options.append(f'--conf="Feature={slurmd_feature}"') - slurmd_options.append("-Z") - - sysconf = f"""SLURMD_OPTIONS='{" ".join(slurmd_options)}'""" - update_system_config("slurmd", sysconf) - install_custom_scripts() - - setup_nss_slurm() - setup_network_storage() - - has_gpu = run("lspci | grep --ignore-case 'NVIDIA' | wc -l", shell=True).returncode - if has_gpu: - run("nvidia-smi") - - run_custom_scripts() - - setup_sudoers() - if not lkp.cfg.enable_slurm_auth: - run("systemctl restart munge", timeout=30) - run("systemctl enable slurmd", timeout=30) - run("systemctl restart slurmd", timeout=30) - run("systemctl enable --now slurmcmd.timer", timeout=30) - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status slurmd", timeout=30) - - log.info("Done setting up compute") - -def setup_cloud_ops() -> None: - """Add health checks, deployment info, and updated setup path to cloud ops config.""" - cloudOpsStatus = run( - "systemctl is-active --quiet google-cloud-ops-agent.service", check=False - ).returncode - - if cloudOpsStatus != 0: - return - - with open("/etc/google-cloud-ops-agent/config.yaml", "r") as f: - file = yaml.safe_load(f) - - # Update setup receiver path - file["logging"]["receivers"]["setup"]["include_paths"] = ["/var/log/slurm/setup.log"] - - cluster_info = { - 'type':'modify_fields', - 'fields': { - 'labels."cluster_name"':{ - 'static_value':f"{lookup().cfg.slurm_cluster_name}" - }, - 'labels."hostname"':{ - 'static_value': f"{lookup().hostname}" - } - } - } - - file["logging"]["processors"]["add_cluster_info"] = cluster_info - file["logging"]["service"]["pipelines"]["slurmlog_pipeline"]["processors"].append("add_cluster_info") - file["logging"]["service"]["pipelines"]["slurmlog2_pipeline"]["processors"].append("add_cluster_info") - - with open("/etc/google-cloud-ops-agent/config.yaml", "w") as f: - yaml.safe_dump(file, f, sort_keys=False) - - retries = 2 - for _ in range(retries): - try: - run("systemctl restart google-cloud-ops-agent.service", timeout=120) - break - except subprocess.TimeoutExpired: - log.error("google-cloud-ops-agent.service did not restart within 120s.") - result=run("cat /var/log/google-cloud-ops-agent/subagents/logging-module.log", timeout=120, shell=True) - if result.stdout: - log.error(f"Logs for google-cloud-ops-agent (logging-module.log file):\n{result.stdout}") - raise - - -def main(): - start_motd() - - log.info("Starting setup, fetching config") - sleep_seconds = 5 - while True: - try: - _, cfg = util.fetch_config() - util.update_config(cfg) - break - except util.DeffetiveStoredConfigError as e: - log.warning(f"config is not ready yet: {e}, sleeping for {sleep_seconds}s") - except Exception as e: - log.exception(f"unexpected error while fetching config, sleeping for {sleep_seconds}s") - time.sleep(sleep_seconds) - log.info("Config fetched") - setup_cloud_ops() - configure_dirs() - # call the setup function for the instance type - { - "controller": setup_controller, - "compute": setup_compute, - "login": setup_login, - }.get( - lookup().instance_role, - lambda: log.fatal(f"Unknown node role: {lookup().instance_role}"))() - - end_motd() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--slurmd-feature", dest="slurmd_feature", help="Unused, to be removed.") - _ = util.init_log_and_parse(parser) - - try: - main() - except Exception: - log.exception("Aborting setup...") - failed_motd() diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py deleted file mode 100644 index 095f42e758..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py +++ /dev/null @@ -1,327 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List - -import os -import sys -import stat -import time -import logging -import uuid - -import shutil -from pathlib import Path -from concurrent.futures import as_completed -from addict import Dict as NSDict # type: ignore - -import util -from util import NSMount, lookup, run, dirs, separate -from more_executors import Executors, ExceptionRetryPolicy - - -log = logging.getLogger() - -def mounts_by_local(mounts: list[NSMount]) -> dict[str, NSMount]: - """convert list of mounts to dict of mounts, local_mount as key""" - return {str(m.local_mount.resolve()): m for m in mounts} - - -def _get_default_mounts(lkp: util.Lookup) -> list[NSMount]: - if lkp.cfg.disable_default_mounts: - return [] - return [ - NSMount( - server_ip=lkp.controller_mount_server_ip(), - remote_mount=path, - local_mount=path, - fs_type="nfs", - mount_options="defaults,hard,intr", - ) - for path in ( - dirs.home, - dirs.apps, - ) - ] - -def get_slurm_bucket_mount() -> NSMount: - bucket, path = util._get_bucket_and_common_prefix() - return NSMount( - fs_type="gcsfuse", - server_ip="", - remote_mount=Path(bucket), - local_mount=dirs.slurm_bucket_mount, - mount_options=f"defaults,_netdev,implicit_dirs,only_dir={path}", - ) - -def resolve_network_storage() -> List[NSMount]: - """Combine appropriate network_storage fields to a single list""" - lkp = lookup() - - # create dict of mounts, local_mount: mount_info - mounts = mounts_by_local(_get_default_mounts(lkp)) - - if lkp.is_controller and util.should_mount_slurm_bucket(): - mounts.update(mounts_by_local([get_slurm_bucket_mount()])) - - # On non-controller instances, entries in network_storage could overwrite - # default exports from the controller. Be careful, of course - common = [lkp.normalize_ns_mount(m) for m in lkp.cfg.network_storage] - mounts.update(mounts_by_local(common)) - - if lkp.is_login_node: - login_group = lkp.cfg.login_groups[util.instance_login_group()] - login_ns = [lkp.normalize_ns_mount(m) for m in login_group.network_storage] - mounts.update(mounts_by_local(login_ns)) - - if lkp.instance_role == "compute": - try: - nodeset = lkp.node_nodeset() - except Exception: - pass # external nodename, skip lookup - else: - nodeset_ns = [lkp.normalize_ns_mount(m) for m in nodeset.network_storage] - mounts.update(mounts_by_local(nodeset_ns)) - - return list(mounts.values()) - - -def is_controller_mount(mount) -> bool: - # NOTE: Valid Lustre server_ip can take the form of '@tcp' - server_ip = mount.server_ip.split("@")[0] - mount_addr = util.host_lookup(server_ip) - return mount_addr == lookup().control_host_addr - -def setup_network_storage(): - """prepare network fs mounts and add them to fstab""" - log.info("Set up network storage") - - all_mounts = resolve_network_storage() - if lookup().is_controller: - mounts, _ = separate(is_controller_mount, all_mounts) - else: - mounts = all_mounts - - # Determine fstab entries and write them out - fstab_entries = [] - for mount in mounts: - local_mount = mount.local_mount - fs_type = mount.fs_type - server_ip = mount.server_ip or "" - src = mount.remote_mount if fs_type == "gcsfuse" else f"{server_ip}:{mount.remote_mount}" - - log.info(f"Setting up mount ({fs_type}) {src} to {local_mount}") - util.mkdirp(local_mount) - - mount_options = mount.mount_options.split(",") if mount.mount_options else [] - if "_netdev" not in mount_options: - mount_options += ["_netdev"] - options_line = ",".join(mount_options) - - - fstab_entries.append(f"{src} {local_mount} {fs_type} {options_line} 0 0") - - fstab = Path("/etc/fstab") - if not Path(fstab.with_suffix(".bak")).is_file(): - shutil.copy2(fstab, fstab.with_suffix(".bak")) - shutil.copy2(fstab.with_suffix(".bak"), fstab) - with open(fstab, "a") as f: - f.write("\n") - for entry in fstab_entries: - f.write(entry) - f.write("\n") - - mount_fstab(mounts, log) - if lookup().cfg.enable_slurm_auth: - slurm_key_mount_handler() - else: - munge_mount_handler() - - -def mount_fstab(mounts: list[NSMount], log): - """Wait on each mount, then make sure all fstab is mounted""" - def mount_path(path: Path): - log.info(f"Waiting for '{path}' to be mounted...") - try: - run(f"mount {path}", timeout=120) - except Exception as e: - exc_type, _, _ = sys.exc_info() - log.error(f"mount of path '{path}' failed: {exc_type}: {e}") - raise e - log.info(f"Mount point '{path}' was mounted.") - - MAX_MOUNT_TIMEOUT = 60 * 5 - future_list = [] - retry_policy = ExceptionRetryPolicy( - max_attempts=120, exponent=1.6, sleep=1.0, max_sleep=16.0 - ) - with Executors.thread_pool().with_timeout(MAX_MOUNT_TIMEOUT).with_retry( - retry_policy=retry_policy - ) as exe: - for m in mounts: - future = exe.submit(mount_path, m.local_mount) - future_list.append(future) - - # Iterate over futures, checking for exceptions - for future in as_completed(future_list): - try: - future.result() - except Exception as e: - raise e - - -def munge_mount_handler(): - if lookup().is_controller: - return - mnt = lookup().munge_mount - - log.info(f"Mounting munge share to: {mnt.local_mount}") - mnt.local_mount.mkdir() - if mnt.fs_type == "gcsfuse": - cmd = [ - "gcsfuse", - f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, - mnt.server_ip, - str(mnt.local_mount), - ] - else: - cmd = [ - "mount", - f"--types={mnt.fs_type}", - f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, - f"{mnt.server_ip}:{mnt.remote_mount}", - str(mnt.local_mount), - ] - # wait max 240s for munge mount - timeout = 240 - for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): - try: - run(cmd, timeout=timeout) - break - except Exception as e: - log.error( - f"munge mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" - ) - time.sleep(wait) - err = e - continue - else: - raise err - - munge_key = Path(dirs.munge / "munge.key") - log.info(f"Copy munge.key from: {mnt.local_mount}") - shutil.copy2(Path(mnt.local_mount / "munge.key"), munge_key) - - log.info("Restrict permissions of munge.key") - shutil.chown(munge_key, user="munge", group="munge") - os.chmod(munge_key, stat.S_IRUSR) - - log.info(f"Unmount {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - run(f"fusermount -u {mnt.local_mount}", timeout=120) - else: - run(f"umount {mnt.local_mount}", timeout=120) - shutil.rmtree(mnt.local_mount) - -def slurm_key_mount_handler(): - if lookup().is_controller: - return - mnt = lookup().slurm_key_mount - - log.info(f"Mounting slurm_key share to: {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - cmd = [ - "gcsfuse", - f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, - mnt.server_ip, - str(mnt.local_mount), - ] - else: - cmd = [ - "mount", - f"--types={mnt.fs_type}", - f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, - f"{mnt.server_ip}:{mnt.remote_mount}", - str(mnt.local_mount), - ] - timeout = 120 # wait max 120s to mount - for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): - try: - run(cmd, timeout=timeout) - break - except Exception as e: - log.error( - f"slurm key mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" - ) - time.sleep(wait) - err = e - continue - else: - raise err - - file_name = "slurm.key" - dst = Path(util.slurmdirs.etc / file_name) - log.info(f"Copy slurm.key from: {mnt.local_mount}") - shutil.copy2(mnt.local_mount / file_name, dst) - - log.info("Restrict permissions of slurm.key") - util.chown_slurm(dst, mode=0o400) - - log.info(f"Unmount {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - run(f"fusermount -u {mnt.local_mount}", timeout=120) - else: - run(f"umount {mnt.local_mount}", timeout=120) - shutil.rmtree(mnt.local_mount) - - -def setup_nfs_exports(): - """nfs export all needed directories""" - lkp = util.lookup() - assert lkp.is_controller - - # The controller only needs to set up exports for cluster-internal mounts - exported_mounts = [m for m in resolve_network_storage() if is_controller_mount(m)] - - # key by remote mount path since that is what needs exporting - to_export = {m.remote_mount: "*(rw,no_subtree_check,no_root_squash)" for m in exported_mounts} - - key_mount = lkp.slurm_key_mount if lkp.cfg.enable_slurm_auth else lkp.munge_mount - if is_controller_mount(key_mount): - # Export key mount as read-only - to_export[key_mount.remote_mount] = "*(ro,no_subtree_check,no_root_squash)" - - if util.should_mount_slurm_bucket(): - mnt = get_slurm_bucket_mount() - # FSID is required for virtual filesystem that is not based on a device - # Also export it as read-only - fsid=str(uuid.uuid4()) - to_export[mnt.local_mount] = f"*(ro,no_subtree_check,no_root_squash,fsid={fsid})" - - # export path if corresponding selector boolean is True - lines = [] - for path,options in to_export.items(): - util.mkdirp(Path(path)) - run(rf"sed -i '\#{path}#d' /etc/exports", timeout=30) - lines.append(f"{path} {options}") - - exportsd = Path("/etc/exports.d") - util.mkdirp(exportsd) - with (exportsd / "slurm.exports").open("w") as f: - f.write("\n") - f.write("\n".join(lines)) - run("exportfs -a", timeout=30) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py deleted file mode 100644 index 1bfdd5acce..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py +++ /dev/null @@ -1,679 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import fcntl -import json -import logging -import re -import sys -import shlex -from datetime import datetime, timedelta -from itertools import chain -from pathlib import Path -from dataclasses import dataclass -from typing import Dict, Tuple, List, Optional, Protocol, Any -from functools import lru_cache - -import util -from util import ( - batch_execute, - ensure_execute, - execute_with_futures, - FutureReservation, - install_custom_scripts, - run, - separate, - to_hostlist, - NodeState, - chunked, - dirs, -) -from util import lookup -from suspend import delete_instances -import tpu -import conf -import watch_delete_vm_op - -log = logging.getLogger() - -TOT_REQ_CNT = 1000 -_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" - -class NodeAction(Protocol): - def apply(self, nodes:List[str]) -> None: - ... - - def __hash__(self): - ... - -@dataclass(frozen=True) -class NodeActionPowerUp(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to resume ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=power_up") - -@dataclass(frozen=True) -class NodeActionIdle(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} nodes to idle ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=resume") - -@dataclass(frozen=True) -class NodeActionPowerDown(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to power down ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=power_down") - - -@dataclass(frozen=True) -class NodeActionPowerDownForce(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to power down ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=power_down_force") - - -@dataclass(frozen=True) -class NodeActionDelete(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to delete ({hostlist})") - delete_instances(nodes) - -@dataclass(frozen=True) -class NodeActionPrempt(): - def apply(self, nodes:List[str]) -> None: - NodeActionDown(reason="Preempted instance").apply(nodes) - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances restarted ({hostlist})") - start_instances(nodes) - -@dataclass(frozen=True) -class NodeActionUnchanged(): - def apply(self, nodes:List[str]) -> None: - pass - -@dataclass(frozen=True) -class NodeActionDown(): - reason: str - - def apply(self, nodes: List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} nodes set down ({hostlist}) with reason={self.reason}") - run(f"{lookup().scontrol} update nodename={hostlist} state=down reason={shlex.quote(self.reason)}") - -@dataclass(frozen=True) -class NodeActionUnknown(): - slurm_state: Optional[NodeState] - instance_state: Optional[str] - - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.error(f"{len(nodes)} nodes have unexpected {self.slurm_state} and instance state:{self.instance_state}, ({hostlist})") - -def start_instance_op(node: str) -> Any: - inst = lookup().instance(node) - assert inst - - return lookup().compute.instances().start( - project=lookup().project, - zone=inst.zone, - instance=inst.name, - ) - - -def start_instances(node_list): - log.info("{} instances to start ({})".format(len(node_list), ",".join(node_list))) - lkp = lookup() - # TODO: use code from resume.py to assign proper placement - normal, tpu_nodes = separate(lkp.node_is_tpu, node_list) - ops = {node: start_instance_op(node) for node in normal} - - done, failed = batch_execute(ops) - - tpu_start_data = [] - for ns, nodes in util.groupby_unsorted(tpu_nodes, lkp.node_nodeset_name): - tpuobj = tpu.TPU.make(ns, lkp) - for snodes in chunked(nodes, n=tpuobj.vmcount): - tpu_start_data.append({"tpu": tpuobj, "node": snodes}) - execute_with_futures(tpu.start_tpu, tpu_start_data) - - -def _find_dynamic_node_status() -> NodeAction: - # TODO: cover more cases: - # * delete dead dynamic nodes - # * delete orhpaned instances - return NodeActionUnchanged() # don't touch dynamic nodes - -def get_fr_action(fr: FutureReservation, state:Optional[NodeState]) -> Optional[NodeAction]: - now = util.now() - if state is None: - return None # handle like any other node - if fr.start_time < now < fr.end_time: - return None # handle like any other node - - if state.base == "DOWN": - return NodeActionUnchanged() - if fr.start_time >= now: - msg = f"Waiting for reservation:{fr.name} to start at {fr.start_time}" - else: - msg = f"Reservation:{fr.name} is after its end-time" - return NodeActionDown(reason=msg) - -def _find_tpu_node_action(nodename, state) -> NodeAction: - lkp = lookup() - tpuobj = tpu.TPU.make(lkp.node_nodeset_name(nodename), lkp) - inst = tpuobj.get_node(nodename) - # If we do not find the node but it is from a Tpu that has multiple vms look for the master node - if inst is None and tpuobj.vmcount > 1: - # Get the tpu slurm nodelist of the nodes in the same tpu group as nodename - nodelist = run( - f"{lkp.scontrol} show topo {nodename}" - + " | awk -F'=' '/Level=0/ { print $NF }'", - shell=True, - ).stdout - l_nodelist = util.to_hostnames(nodelist) - group_names = set(l_nodelist) - # get the list of all the existing tpus in the nodeset - tpus_list = set(tpuobj.list_node_names()) - # In the intersection there must be only one node that is the master - tpus_int = list(group_names.intersection(tpus_list)) - if len(tpus_int) > 1: - log.error( - f"More than one cloud tpu node for tpu group {nodelist}, there should be only one that should be {l_nodelist[0]}, but we have found {tpus_int}" - ) - return NodeActionUnknown(slurm_state=state, instance_state=None) - if len(tpus_int) == 1: - inst = tpuobj.get_node(tpus_int[0]) - # if len(tpus_int ==0) this case is not relevant as this would be the case always that a TPU group is not running - if inst is None: - if state.base == "DOWN" and "POWERED_DOWN" in state.flags: - return NodeActionIdle() - if "POWERING_DOWN" in state.flags: - return NodeActionIdle() - if "COMPLETING" in state.flags: - return NodeActionDown(reason="Unbacked instance") - if state.base != "DOWN" and not ( - set(("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN")) - & state.flags - ): - return NodeActionDown(reason="Unbacked instance") - if lkp.is_static_node(nodename): - return NodeActionPowerUp() - elif ( - state is not None - and "POWERED_DOWN" not in state.flags - and "POWERING_DOWN" not in state.flags - and inst.state == tpu.TPU.State.STOPPED - ): - if tpuobj.preemptible: - return NodeActionPrempt() - if state.base != "DOWN": - return NodeActionDown(reason="Instance terminated") - elif ( - state is None or "POWERED_DOWN" in state.flags - ) and inst.state == tpu.TPU.State.READY: - return NodeActionDelete() - elif state is None: - # if state is None here, the instance exists but it's not in Slurm - return NodeActionUnknown(slurm_state=state, instance_state=inst.status) - - return NodeActionUnchanged() - -def get_node_action(nodename: str) -> NodeAction: - """Determine node/instance status that requires action""" - lkp = lookup() - state = lkp.node_state(nodename) - - if lkp.node_is_gke(nodename): - return NodeActionUnchanged() - - if lkp.node_is_fr(nodename): - fr = lkp.future_reservation(lkp.node_nodeset(nodename)) - assert fr - if action := get_fr_action(fr, state): - return action - - if lkp.node_is_dyn(nodename): - return _find_dynamic_node_status() - - if lkp.node_is_tpu(nodename): - return _find_tpu_node_action(nodename, state) - - # split below is workaround for VMs whose hostname is FQDN - inst = lkp.instance(nodename.split(".")[0]) - power_flags = frozenset( - ("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN") - ) & (state.flags if state is not None else set()) - - if (state is None) and (inst is None): - # Should never happen - return NodeActionUnknown(None, None) - if inst is None: - assert state is not None # to keep type-checker happy - if "POWERING_UP" in state.flags: - return NodeActionUnchanged() - if state.base == "DOWN" and "POWERED_DOWN" in state.flags: - return NodeActionIdle() - if "POWERING_DOWN" in state.flags: - return NodeActionIdle() - if "COMPLETING" in state.flags: - return NodeActionDown(reason="Unbacked instance") - if state.base != "DOWN" and not power_flags: - return NodeActionDown(reason="Unbacked instance") - if state.base == "DOWN" and not power_flags: - return NodeActionPowerDown() - if "NOT_RESPONDING" in state.flags: - return NodeActionPowerDown() - if "POWERED_DOWN" in state.flags and lkp.is_static_node(nodename): - return NodeActionPowerUp() - elif ( - state is not None - and "POWERED_DOWN" not in state.flags - and "POWERING_DOWN" not in state.flags - and inst.status == "TERMINATED" - ): - if inst.scheduling.preemptible: - return NodeActionPrempt() - if state.base != "DOWN": - return NodeActionDown(reason="Instance terminated") - elif (state is None or "POWERED_DOWN" in state.flags) and inst.status == "RUNNING": - log.info("%s is potential orphan node", nodename) - threshold = timedelta(seconds=90) - age = util.now() - inst.creation_timestamp - log.info(f"{nodename} state: {state}, age: {age}") - if age < threshold: - log.info(f"{nodename} not marked as orphan, it started less than {threshold.seconds}s ago ({age.seconds}s)") - return NodeActionUnchanged() - return NodeActionDelete() - elif state is None: - # if state is None here, the instance exists but it's not in Slurm - return NodeActionUnknown(slurm_state=state, instance_state=inst.status) - elif lkp.is_flex_node(nodename) and "POWERING_UP" in state.flags: - threshold = timedelta(seconds=int(lkp.cfg.compute_startup_scripts_timeout) * 2) #extra buffer for unexpectedly long startup scripts - if util.now() - inst.creation_timestamp > threshold: - log.info(f"{nodename} was unable to join the cluster after {threshold.seconds}s, potential failure on VM startup. Powering down...") - return NodeActionPowerDownForce() - return NodeActionUnchanged() - - -def delete_resource_policies(links: list[str], lkp: util.Lookup) -> None: - requests = {} - for link in links: - name = util.trim_self_link(link) - region = util.parse_self_link(link).region - requests[name] = lkp.compute.resourcePolicies().delete(project=lkp.project, region=region, resourcePolicy=name) - - def swallow_err(_: str) -> None: - pass - - done, failed = batch_execute(requests, log_err=swallow_err) - if failed: - # Filter out resourceInUseByAnotherResource errors , they are expected to happen - def ignore_err(e) -> bool: - return "resourceInUseByAnotherResource" in str(e) - - failures = [f"{n}: {e}" for n, (_, e) in failed.items() if not ignore_err(e)] - if failures: - log.error(f"some placement groups failed to delete: {failures}") - log.info( - f"deleted {len(done)} of {len(links)} placement groups ({to_hostlist(done.keys())})" - ) - - - -@lru_cache -def _get_resource_policies_in_region(lkp: util.Lookup, region: str) -> list[Any]: - res = [] - act = lkp.compute.resourcePolicies() - op = act.list(project=lkp.project, region=region) - prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-" - while op is not None: - result = ensure_execute(op) - res.extend([p for p in result.get("items", []) if p.get("name", "").startswith(prefix)]) - op = act.list_next(op, result) - return res - - -@lru_cache -def _get_resource_policies(lkp: util.Lookup) -> list[Any]: - res = [] - for region in lkp.cluster_regions(): - res.extend(_get_resource_policies_in_region(lkp, region)) - return res - -def sync_placement_groups(): - """Delete placement policies that are for jobs that have completed/terminated""" - keep_states = frozenset( - [ - "RUNNING", - "CONFIGURING", - "STOPPED", - "SUSPENDED", - "COMPLETING", - "PENDING", - ] - ) - - lkp = lookup() - keep_jobs = { - str(job.id) - for job in lkp.get_jobs() - if job.job_state in keep_states - } - keep_jobs.add("0") # Job 0 is a placeholder for static node placement - - to_delete = [] - pg_regex = re.compile( - rf"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-(?P[^\s\-]+)-(?P\d+)-(?P\d+)" - ) - - for pg in _get_resource_policies(lkp): - name = pg["name"] - - if (mtch := pg_regex.match(name)) is None: - log.warning(f"Unexpected resource policy {name=}") - continue - if mtch.group("job_id") not in keep_jobs: - to_delete.append(pg["selfLink"]) - - if to_delete: - delete_resource_policies(to_delete, lkp) - - -def sync_instances(): - compute_instances = { - name for name, inst in lookup().instances().items() if inst.role == "compute" - } - slurm_nodes = set(lookup().slurm_nodes().keys()) - log.debug(f"reconciling {len(compute_instances)} GCP instances and {len(slurm_nodes)} Slurm nodes.") - - for action, nodes in util.groupby_unsorted(list(compute_instances | slurm_nodes), get_node_action): - action.apply(list(nodes)) - - -def reconfigure_slurm(): - update_msg = "*** slurm configuration was updated ***" - if lookup().cfg.hybrid: - # terraform handles generating the config.yaml, don't do it here - return - - upd, cfg_new = util.fetch_config() - if not upd: - log.debug("No changes in config detected.") - return - log.debug("Changes in config detected. Reconfiguring Slurm now.") - util.update_config(cfg_new) - - if lookup().is_controller: - conf.gen_controller_configs(lookup()) - log.info("Restarting slurmctld to make changes take effect.") - try: - # TODO: consider removing "restart" since "reconfigure" should restart slurmctld as well - run("sudo systemctl restart slurmctld.service", check=False) - util.scontrol_reconfigure(lookup()) - except Exception: - log.exception("failed to reconfigure slurmctld") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - elif lookup().instance_role_safe == "compute": - log.info("Restarting slurmd to make changes take effect.") - run("systemctl restart slurmd") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - elif lookup().is_login_node: - log.info("Restarting sackd to make changes take effect.") - run("systemctl restart sackd") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - - -def update_topology(lkp: util.Lookup) -> None: - if conf.topology_plugin(lkp) != conf.TOPOLOGY_PLUGIN_TREE: - return - updated, summary = conf.gen_topology_conf(lkp) - if updated: - log.info("Topology configuration updated. Reconfiguring Slurm.") - util.scontrol_reconfigure(lkp) - # Safe summary only after Slurm got reconfigured, so summary reflects Slurm POV - summary.dump(lkp) - - -def delete_reservation(lkp: util.Lookup, reservation_name: str) -> None: - util.run(f"{lkp.scontrol} delete reservation {reservation_name}") - - -def create_reservation(lkp: util.Lookup, reservation_name: str, node: str, start_time: datetime) -> None: - # Format time to be compatible with slurm reservation. - formatted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S') - - util.run(f"{lkp.scontrol} create reservation user=slurm starttime={formatted_start_time} duration=180 nodes={node} reservationname={reservation_name} flags=maint,ignore_jobs") - - -def get_slurm_reservation_maintenance(lkp: util.Lookup) -> Dict[str, datetime]: - res = util.run(f"{lkp.scontrol} show reservation --json") - all_reservations = json.loads(res.stdout) - reservation_map = {} - - for reservation in all_reservations['reservations']: - name = reservation.get('name') - nodes = reservation.get('node_list') - time_epoch = reservation.get('start_time', {}).get('number') - - if name is None or nodes is None or time_epoch is None: - continue - - if reservation.get('node_count') != 1: - continue - - if name != f"{nodes}_maintenance": - continue - - reservation_map[name] = datetime.fromtimestamp(time_epoch) - - return reservation_map - -@lru_cache -def get_upcoming_maintenance(lkp: util.Lookup) -> Dict[str, Tuple[str, datetime]]: - upc_maint_map = {} - - for node, inst in lkp.instances().items(): - if inst.resource_status.upcoming_maintenance: - upc_maint_map[node + "_maintenance"] = (node, inst.resource_status.upcoming_maintenance.window_start_time) - - return upc_maint_map - - -def sync_maintenance_reservation(lkp: util.Lookup) -> None: - upc_maint_map = get_upcoming_maintenance(lkp) # map reservation_name -> (node_name, time) - log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") - - curr_reservation_map = get_slurm_reservation_maintenance(lkp) # map reservation_name -> time - log.debug(f"curr-reservation-map: {curr_reservation_map}") - - del_reservation = set(curr_reservation_map.keys() - upc_maint_map.keys()) - create_reservation_map = {} - - for res_name, (node, start_time) in upc_maint_map.items(): - try: - enabled = lkp.node_nodeset(node).enable_maintenance_reservation - except Exception: - enabled = False - - if not enabled: - if res_name in curr_reservation_map: - del_reservation.add(res_name) - continue - - if res_name in curr_reservation_map: - diff = curr_reservation_map[res_name] - start_time - if abs(diff) <= timedelta(seconds=1): - continue - else: - del_reservation.add(res_name) - create_reservation_map[res_name] = (node, start_time) - else: - create_reservation_map[res_name] = (node, start_time) - - log.debug(f"del-reservation: {del_reservation}") - for res_name in del_reservation: - delete_reservation(lkp, res_name) - - log.debug(f"create-reservation-map: {create_reservation_map}") - for res_name, (node, start_time) in create_reservation_map.items(): - create_reservation(lkp, res_name, node, start_time) - - -def delete_maintenance_job(job_name: str) -> None: - util.run(f"scancel --name={job_name}") - - -def create_maintenance_job(job_name: str, node: str) -> None: - util.run(f"sbatch --job-name={job_name} --nodelist={node} {_MAINTENANCE_SBATCH_SCRIPT_PATH}") - - -def get_slurm_maintenance_job(lkp: util.Lookup) -> Dict[str, str]: - jobs = {} - - for job in lkp.get_jobs(): - if job.name is None or job.required_nodes is None or job.job_state is None: - continue - - if job.name != f"{job.required_nodes}_maintenance": - continue - - if job.job_state != "PENDING": - continue - - jobs[job.name] = job.required_nodes - - return jobs - - -def sync_opportunistic_maintenance(lkp: util.Lookup) -> None: - upc_maint_map = get_upcoming_maintenance(lkp) # map job_name -> (node_name, time) - log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") - - curr_jobs = get_slurm_maintenance_job(lkp) # map job_name -> node. - log.debug(f"curr-maintenance-job-map: {curr_jobs}") - - del_jobs = set(curr_jobs.keys() - upc_maint_map.keys()) - create_jobs = {} - - for job_name, (node, _) in upc_maint_map.items(): - try: - enabled = lkp.node_nodeset(node).enable_opportunistic_maintenance - except Exception: - enabled = False - - if not enabled: - if job_name in curr_jobs: - del_jobs.add(job_name) - continue - - if job_name not in curr_jobs: - create_jobs[job_name] = node - - log.debug(f"del-maintenance-job: {del_jobs}") - for job_name in del_jobs: - delete_maintenance_job(job_name) - - log.debug(f"create-maintenance-job: {create_jobs}") - for job_name, node in create_jobs.items(): - create_maintenance_job(job_name, node) - - - -def sync_flex_migs(lkp: util.Lookup) -> None: - pass - - -def process_messages(lkp: util.Lookup) -> None: - try: - watch_delete_vm_op.watch_vm_delete_ops(lkp) - except: - log.exception("failed during watching delete VM operations") - - -def main(): - lkp = lookup() - if util.should_mount_slurm_bucket() and not lkp.is_controller: - return - try: - reconfigure_slurm() - except Exception: - log.exception("failed to reconfigure slurm") - if lkp.is_controller: - try: - process_messages(lkp) - except: - log.exception("failed to process messages") - - try: - sync_instances() - except Exception: - log.exception("failed to sync instances") - - try: - sync_flex_migs(lkp) - except Exception: - log.exception("failed to sync DWS Flex MIGs") - - try: - sync_placement_groups() - except Exception: - log.exception("failed to sync placement groups") - - try: - update_topology(lkp) - except Exception: - log.exception("failed to update topology") - - try: - sync_maintenance_reservation(lkp) - except Exception: - log.exception("failed to sync slurm reservation for scheduled maintenance") - - try: - sync_opportunistic_maintenance(lkp) - except Exception: - log.exception("failed to sync opportunistic reservation for scheduled maintenance") - - - try: - # TODO: it performs 1 to 4 GCS list requests, - # use cached version, combine with `_list_config_blobs` - install_custom_scripts(check_hash=True) - except Exception: - log.exception("failed to sync custom scripts") - - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - _ = util.init_log_and_parse(parser) - - pid_file = (Path("/tmp") / Path(__file__).name).with_suffix(".pid") - with pid_file.open("w") as fp: - try: - fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) - main() - except BlockingIOError: - sys.exit(0) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py deleted file mode 100644 index ae36c54222..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This script sorts nodes based on their `physicalHost`. - -See https://cloud.google.com/compute/docs/instances/use-compact-placement-policies - -You can reduce latency in tightly coupled HPC workloads (including distributed ML training) -by deploying them to machines that are located close together. -For example, if you deploy your workload on a single physical rack, you can expect lower latency -than if your workload is spread across multiple racks. -Sending data across multiple rack requires sending data through additional network switches. - -Example usage: -``` my_sbatch.sh -#SBATCH --ntasks-per-node=8 -#SBATCH --nodes=64 - -export SLURM_HOSTFILE=$(sort_nodes.py) - -srun -l hostname | sort -``` -""" -import os -import subprocess -import uuid -from typing import List, Optional, Dict -from collections import OrderedDict - -def order(paths: List[List[str]]) -> List[str]: - """ - Orders the leaves of the tree in a way that minimizes the sum of distance in between - each pair of neighboring nodes in the resulting order. - The resulting order will always start from the first node in the input list. - The ordering is "stable" with respect to the input order of the leaves i.e. - given a choice between two nodes (identical in other ways) it will select "nodelist-smallest" one. - - Returns a list of nodenames, ordered as described above. - """ - if not paths: return [] - class Vert: - "Represents a vertex in a *network* tree." - def __init__(self, name: str, parent: Optional["Vert"]): - self.name = name - self.parent = parent - # Use `OrderedDict` to preserve insertion order - # TODO: once we move to Python 3.7+ use regular `dict` since it has the same guarantee - self.children: OrderedDict = OrderedDict() - - # build a tree, children are ordered by insertion order - root = Vert("", None) - for path in paths: - n = root - for v in path: - if v not in n.children: - n.children[v] = Vert(v, n) - n = n.children[v] - - # walk the tree in insertion order, gather leaves - result = [] - def gather_nodes(v: Vert) -> None: - if not v.children: # this is a Slurm node - result.append(v.name) - for u in v.children.values(): - gather_nodes(u) - gather_nodes(root) - return result - - -class Instance: - def __init__(self, name: str, zone: str, physical_host: Optional[str]): - self.name = name - self.zone = zone - self.physical_host = physical_host - - -def make_path(node_name: str, inst: Optional[Instance]) -> List[str]: - if not inst: # node with unknown instance (e.g. hybrid cluster) - return ["unknown", node_name] - zone = f"zone_{inst.zone}" - if not inst.physical_host: # node without physical host info (e.g. no placement policy) - return [zone, "unknown", node_name] - - assert inst.physical_host.startswith("/"), f"Unexpected physicalHost: {inst.physical_host}" - parts = inst.physical_host[1:].split("/") - if len(parts) >= 4: - return [*parts, node_name] - return [zone, *parts, node_name] - - -def to_hostnames(nodelist: str) -> List[str]: - cmd = ["scontrol", "show", "hostnames", nodelist] - out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout - return [n.decode("utf-8") for n in out.splitlines()] - - -def get_instances(node_names: List[str]) -> Dict[str, Optional[Instance]]: - fmt = ( - "--format=csv[no-heading,separator=','](zone,resourceStatus.physicalHost,name)" - ) - cmd = ["gcloud", "compute", "instances", "list", fmt] - - scp = os.path.commonprefix(node_names) - if scp: - cmd.append(f"--filter=name~'{scp}.*'") - out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout - d = {} - for line in out.splitlines(): - zone, physical_host, name = line.decode("utf-8").split(",") - d[name] = Instance(name, zone, physical_host) - return {n: d.get(n) for n in node_names} - - -def main(args) -> None: - nodelist = args.nodelist or os.getenv("SLURM_NODELIST") - if not nodelist: - raise ValueError("nodelist is not provided and SLURM_NODELIST is not set") - - if args.ntasks_per_node is None: - args.ntasks_per_node = int(os.getenv("SLURM_NTASKS_PER_NODE", "") or 1) - assert args.ntasks_per_node > 0 - - output = args.output or f"hosts.{uuid.uuid4()}" - - node_names = to_hostnames(nodelist) - instannces = get_instances(node_names) - paths = [make_path(n, instannces[n]) for n in node_names] - ordered = order(paths) - - with open(output, "w") as f: - for node in ordered: - for _ in range(args.ntasks_per_node): - f.write(node) - f.write("\n") - print(output) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument( - "--nodelist", - type=str, - help="Slurm 'hostlist expression' of nodes to sort, if not set the value of SLURM_NODELIST environment variable will be used", - ) - parser.add_argument( - "--ntasks-per-node", - type=int, - help="""Number of times to repeat each node in resulting sorted list. -If not set, the value of SLURM_NTASKS_PER_NODE environment variable will be used, -if neither is set, defaults to 1""", - ) - parser.add_argument( - "--output", type=str, help="Output file to write, defaults to 'hosts.'" - ) - args = parser.parse_args() - main(args) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py deleted file mode 100644 index ecef70f1cc..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Any -import argparse -import logging - -import util -from util import ( - log_api_request, - batch_execute, - to_hostlist, - separate, -) -from util import lookup -import tpu -import mig_flex -import watch_delete_vm_op - -log = logging.getLogger() - -TOT_REQ_CNT = 1000 - - -def truncate_iter(iterable, max_count): - end = "..." - _iter = iter(iterable) - for i, el in enumerate(_iter, start=1): - if i >= max_count: - yield end - break - yield el - - -def delete_instance_request(name: str) -> Any: - inst = lookup().instance(name) - assert inst - - request = lookup().compute.instances().delete( - project=lookup().project, - zone=inst.zone, - instance=name, - ) - log_api_request(request) - return request - - -def delete_instances(instances): - """delete instances individually""" - invalid, valid = separate(lambda inst: bool(lookup().instance(inst)), instances) - if len(invalid) > 0: - log.debug("instances do not exist: {}".format(",".join(invalid))) - if len(valid) == 0: - log.debug("No instances to delete") - return - - requests = {inst: delete_instance_request(inst) for inst in valid} - - log.info(f"to delete {len(valid)} instances ({to_hostlist(valid)})") - ops, failed = batch_execute(requests) - for node, (_, err) in failed.items(): - log.error(f"instance {node} failed to delete: {err}") - - log.info(f"deleting {len(ops)} instances {to_hostlist(ops.keys())}") - - topic = watch_delete_vm_op.watch_delete_vm_op_topic() - for node, op in ops.items(): - topic.publish(op, node) - - - - -def suspend_nodes(nodes: List[str]) -> None: - lkp = lookup() - other_nodes, tpu_nodes = util.separate(lkp.node_is_tpu, nodes) - bulk_nodes, flex_nodes = util.separate(lkp.is_flex_node, other_nodes) - - mig_flex.suspend_flex_nodes(flex_nodes, lkp) - delete_instances(bulk_nodes) - tpu.delete_tpu_instances(tpu_nodes) - - -def main(nodelist): - """main called when run as script""" - log.debug(f"SuspendProgram {nodelist}") - - # Filter out nodes not in config.yaml - other_nodes, pm_nodes = separate( - lookup().is_power_managed_node, util.to_hostnames(nodelist) - ) - if other_nodes: - log.debug( - f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" - ) - if pm_nodes: - log.debug(f"Suspending nodes '{to_hostlist(pm_nodes)}' from '{nodelist}'") - else: - log.debug("No cloud nodes to suspend") - return - - log.info(f"suspend {nodelist}") - suspend_nodes(pm_nodes) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument("nodelist", help="list of nodes to suspend") - args = util.init_log_and_parse(parser) - - main(args.nodelist) diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh deleted file mode 100644 index 9079e4e4b0..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PYTHON_SCRIPT="${SCRIPT_DIR}/suspend.py" - -# Capture all arguments passed by Slurm (the nodelist). -ALL_ARGS=("$@") - -"${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & -disown - -exit 0 diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py deleted file mode 100644 index 0ce7fb5ec4..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Any -import sys -from dataclasses import dataclass, field -from datetime import datetime - -SCRIPTS_DIR = "community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts" -if SCRIPTS_DIR not in sys.path: - sys.path.append(SCRIPTS_DIR) # TODO: make this more robust - -import util - - -SOME_TS = datetime.fromisoformat("2018-09-03T20:56:35.450686+00:00") -# TODO: use "real" classes once they are defined (instead of NSDict) - -@dataclass -class Placeholder: - pass - -@dataclass -class TstNodeset: - nodeset_name: str = "cantor" - node_count_static: int = 0 - node_count_dynamic_max: int = 0 - node_conf: dict[str, Any] = field(default_factory=dict) - instance_template: Optional[str] = None - reservation_name: Optional[str] = "" - zone_policy_allow: Optional[list[str]] = field(default_factory=list) - enable_placement: bool = True - placement_max_distance: Optional[int] = None - accelerator_topology: Optional[str] = "" - future_reservation: Optional[str] = "" - -@dataclass -class TstPartition: - partition_name: str = "euler" - partition_nodeset: list[str] = field(default_factory=list) - partition_nodeset_tpu: list[str] = field(default_factory=list) - enable_job_exclusive: bool = False - -@dataclass -class TstCfg: - slurm_cluster_name: str = "m22" - cloud_parameters: dict[str, Any] = field(default_factory=dict) - - partitions: dict[str, TstPartition] = field(default_factory=dict) - nodeset: dict[str, TstNodeset] = field(default_factory=dict) - nodeset_tpu: dict[str, TstNodeset] = field(default_factory=dict) - nodeset_dyn: dict[str, TstNodeset] = field(default_factory=dict) - - install_dir: Optional[str] = None - output_dir: Optional[str] = None - - prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - task_prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - task_epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - - -@dataclass -class TstTPU: # to prevent client initialization durint "TPU.__init__" - vmcount: int - -@dataclass -class TstMachineConf: - cpus: int - memory: int - sockets: int - sockets_per_board: int - cores_per_socket: int - boards: int - threads_per_core: int - - -@dataclass -class TstTemplateInfo: - gpu: Optional[util.AcceleratorInfo] - -def tstInstance(name: str, physical_host: Optional[str] = None): - return util.Instance( - name=name, - zone="anorien", - status="RUNNING", - creation_timestamp=SOME_TS, - resource_status=util.InstanceResourceStatus( - physical_host=physical_host, - upcoming_maintenance=None, - ), - scheduling=util.NSDict(), - role="compute", - metadata={}, - ) - -def make_to_hostnames_mock(tbl: Optional[dict[str, list[str]]]): - tbl = tbl or {} - - def se(k: str) -> list[str]: - if k not in tbl: - raise AssertionError(f"to_hostnames mock: unexpected nodelist: '{k}'") - return tbl[k] - - return se diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py deleted file mode 100644 index 6bd6762748..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from mock import Mock -from common import TstNodeset, TstCfg, TstMachineConf, TstTemplateInfo, Placeholder - -import addict # type: ignore -import conf -import util - - -def test_nodeset_tpu_lines(): - nodeset = TstNodeset( - "turbo", - node_count_static=2, - node_count_dynamic_max=3, - node_conf={"red": "velvet"}, - ) - assert conf.nodeset_tpu_lines(nodeset, util.Lookup(TstCfg())) == "\n".join( - [ - "NodeName=m22-turbo-[0-4] State=CLOUD red=velvet", - "NodeSet=turbo Nodes=m22-turbo-[0-4]", - ] - ) - - -def test_nodeset_lines(): - nodeset = TstNodeset( - "turbo", - node_count_static=2, - node_count_dynamic_max=3, - node_conf={"red": "velvet", "CPUs": 55}, - ) - lkp = util.Lookup(TstCfg()) - lkp.template_info = Mock(return_value=TstTemplateInfo( - gpu=util.AcceleratorInfo(type="Popov", count=33) - )) - mc = TstMachineConf( - cpus=5, - memory=6, - sockets=7, - sockets_per_board=8, - boards=9, - threads_per_core=10, - cores_per_socket=11, - ) - lkp.template_machine_conf = Mock(return_value=mc) # type: ignore[method-assign] - assert conf.nodeset_lines(nodeset, lkp) == "\n".join( - [ - "NodeName=m22-turbo-[0-4] State=CLOUD RealMemory=6 Boards=9 SocketsPerBoard=8 CoresPerSocket=11 ThreadsPerCore=10 CPUs=55 Gres=gpu:33 red=velvet", - "NodeSet=turbo Nodes=m22-turbo-[0-4]", - ] - ) - - -@pytest.mark.parametrize( - "value,want", - [ - ({"a": 1}, "a=1"), - ({"a": "two"}, "a=two"), - ({"a": [3, 4]}, "a=3,4"), - ({"a": ["five", "six"]}, "a=five,six"), - ({"a": None}, ""), - ({"a": ["seven", None, 8]}, "a=seven,8"), - ({"a": 1, "b": "two"}, "a=1 b=two"), - ({"a": 1, "b": None, "c": "three"}, "a=1 c=three"), - ({"a": 0, "b": None, "c": 0.0, "e": ""}, "a=0 c=0.0"), - ({"a": [0, 0.0, None, "X", "", "Y"]}, "a=0,0.0,X,,Y"), - ]) -def test_dict_to_conf(value: dict, want: str): - assert conf.dict_to_conf(value) == want - - - -@pytest.mark.parametrize( - "cfg,want", - [ - (TstCfg( - install_dir="ukulele", - ), - """LaunchParameters=enable_nss_slurm,use_interactive_step -SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend -SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -SlurmdTimeout=300 -UnkillableStepTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - (TstCfg( - install_dir="ukulele", - cloud_parameters={ - "no_comma_params": True, - "private_data": None, - "scheduler_parameters": None, - "resume_rate": None, - "resume_timeout": None, - "suspend_rate": None, - "suspend_timeout": None, - "unkillable_step_timeout": None, - "slurmd_timeout": None, - "topology_plugin": None, - "topology_param": None, - "tree_width": None, - }, - ), - """SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -SlurmdTimeout=300 -UnkillableStepTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - (TstCfg( - install_dir="ukulele", - cloud_parameters={ - "no_comma_params": True, - "private_data": [ - "events", - "jobs", - ], - "scheduler_parameters": [ - "bf_busy_nodes", - "bf_continue", - "ignore_prefer_validation", - "nohold_on_prolog_fail", - ], - "resume_rate": 1, - "resume_timeout": 2, - "suspend_rate": 3, - "suspend_timeout": 4, - "slurmd_timeout": 5, - "unkillable_step_timeout": 6, - "tree_width": 7, - "topology_plugin": "guess", - "topology_param": "yellow", - }, - ), - """PrivateData=events,jobs -SchedulerParameters=bf_busy_nodes,bf_continue,ignore_prefer_validation,nohold_on_prolog_fail -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=1 -ResumeTimeout=2 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=3 -SuspendTimeout=4 -SlurmdTimeout=5 -UnkillableStepTimeout=6 -TreeWidth=7 -TopologyPlugin=guess -TopologyParam=yellow"""), - (TstCfg( - install_dir="ukulele", - task_prolog_scripts=[Placeholder()], - task_epilog_scripts=[Placeholder()], - ), - """LaunchParameters=enable_nss_slurm,use_interactive_step -SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend -TaskProlog=/slurm/custom_scripts/task_prolog.d/task-prolog -TaskEpilog=/slurm/custom_scripts/task_epilog.d/task-epilog -SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -SlurmdTimeout=300 -UnkillableStepTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - ]) -def test_conflines(cfg, want): - assert conf.conflines(util.Lookup(cfg)) == want - - cfg.cloud_parameters = addict.Dict(cfg.cloud_parameters) - assert conf.conflines(util.Lookup(cfg)) == want - - -@pytest.mark.parametrize( - "cfg,gputype,gpucount,want", - [ - (TstCfg(), - "", - 0, - "\n"), - (TstCfg( - nodeset={"turbo": TstNodeset("turbo")} - ), - "Popov", - 8, - "Name=gpu Type=Popov File=/dev/nvidia[0-7]\n\n"), - ]) -def test_gen_cloud_gres_conf_lines(cfg, gputype, gpucount, want): - lkp = util.Lookup(cfg) - lkp.template_info = Mock(return_value=TstTemplateInfo( - gpu=util.AcceleratorInfo(type=gputype, count=gpucount) - )) - assert conf.gen_cloud_gres_conf_lines(lkp) == want diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py deleted file mode 100644 index 77f1229605..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional - -import os -import pytest -import unittest.mock -import unittest -import tempfile - -from common import TstCfg, TstNodeset, TstPartition, TstTPU # needed to import util -import util -import resume -from resume import ResumeData, ResumeJobData, BulkChunk, PlacementAndNodes - -def test_get_resume_file_data_no_env(): - with unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": ""}): - assert resume.get_resume_file_data() is None - - -def test_get_resume_file_data(): - with tempfile.NamedTemporaryFile() as f: - f.write(b"""{ - "jobs": [ - { - "extra": null, - "job_id": 1, - "features": null, - "nodes_alloc": "green-[0-2]", - "nodes_resume": "green-[0-1]", - "oversubscribe": "OK", - "partition": "red", - "reservation": null - } - ], - "all_nodes_resume": "green-[0-1]" -}""") - f.flush() - with ( - unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": f.name}), - unittest.mock.patch("util.to_hostnames") as mock_to_hostnames, - ): - mock_to_hostnames.return_value = ["green-0", "green-1", "green-2"] - assert resume.get_resume_file_data() == ResumeData(jobs=[ - ResumeJobData( - job_id = 1, - partition="red", - nodes_alloc=["green-0", "green-1", "green-2"], - ) - ]) - mock_to_hostnames.assert_called_once_with("green-[0-2]") - - -@unittest.mock.patch("tpu.TPU.make") -@unittest.mock.patch("resume.create_placements") -def test_group_nodes_bulk(mock_create_placements, mock_tpu): - cfg = TstCfg( - nodeset={ - "n": TstNodeset(nodeset_name="n"), - }, - nodeset_tpu={ - "t": TstNodeset(nodeset_name="t"), - }, - partitions={ - "p1": TstPartition( - partition_name="p1", - enable_job_exclusive=True, - ), - "p2": TstPartition( - partition_name="p2", - partition_nodeset_tpu=["t"], - enable_job_exclusive=True, - ) - } - ) - lkp = util.Lookup(cfg) - - def mock_create_placements_se(nodes, excl_job_id, lkp): - args = (set(nodes), excl_job_id) - if ({'c-n-1', 'c-n-2', 'c-t-8', 'c-t-9'}, None) == args: - return [ - PlacementAndNodes("g0", ["c-n-1", "c-n-2"]), - PlacementAndNodes(None, ['c-t-8', 'c-t-9']), - ] - if ({"c-n-0", "c-n-8"}, 1) == args: - return [ - PlacementAndNodes("g10", ["c-n-0"]), - PlacementAndNodes("g11", ["c-n-8"]), - ] - if ({'c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5'}, 2) == args: - return [ - PlacementAndNodes(None, ['c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5']) - ] - raise AssertionError(f"unexpected invocation: '{args}'") - mock_create_placements.side_effect = mock_create_placements_se - - def mock_tpu_se(ns: str, lkp) -> TstTPU: - if ns == "t": - return TstTPU(vmcount=2) - raise AssertionError(f"unexpected invocation: '{ns}'") - mock_tpu.side_effect = mock_tpu_se - - got = resume.group_nodes_bulk( - ["c-n-0", "c-n-1", "c-n-2", "c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-8", "c-t-9"], - ResumeData(jobs=[ - ResumeJobData(job_id=1, partition="p1", nodes_alloc=["c-n-0", "c-n-8"]), - ResumeJobData(job_id=2, partition="p2", nodes_alloc=["c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-4", "c-t-5"]), - ]), lkp) - mock_create_placements.assert_called() - assert got == { - "c-n:jobNone:g0:0": BulkChunk( - nodes=["c-n-1", "c-n-2"], prefix="c-n", chunk_idx=0, excl_job_id=None, placement_group="g0"), - "c-n:job1:g10:0": BulkChunk( - nodes=["c-n-0"], prefix="c-n", chunk_idx=0, excl_job_id=1, placement_group="g10"), - "c-t:0": BulkChunk( - nodes=["c-t-8", "c-t-9"], prefix="c-t", chunk_idx=0, excl_job_id=None, placement_group=None), - "c-t:job2:0": BulkChunk( - nodes=["c-t-0", "c-t-1"], prefix="c-t", chunk_idx=0, excl_job_id=2, placement_group=None), - "c-t:job2:1": BulkChunk( - nodes=["c-t-2", "c-t-3"], prefix="c-t", chunk_idx=1, excl_job_id=2, placement_group=None), - } - - -@pytest.mark.parametrize( - "nodes,excl_job_id,expected", - [ - ( # TPU - no placements - ["c-t-0", "c-t-2"], 4, [PlacementAndNodes(None, ["c-t-0", "c-t-2"])] - ), - ( # disabled placements - no placemens - ["c-x-0", "c-x-2"], 4, [PlacementAndNodes(None, ["c-x-0", "c-x-2"])] - ), - ( # excl_job - ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], 4, [ - PlacementAndNodes("c-slurmgcp-managed-n-4-0", ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"]) - ] - ), - ( # no excl_job - ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], None, [ - PlacementAndNodes("c-slurmgcp-managed-n-0-0", ["c-n-0", "c-n-2"]), - PlacementAndNodes('c-slurmgcp-managed-n-0-1', ['c-n-2011']), - PlacementAndNodes(None, ["c-n-uno"]), - ] - ), - ], -) -def test_allocate_nodes_to_placements(nodes: list[str], excl_job_id: Optional[int], expected: list[PlacementAndNodes]): - cfg = TstCfg( - slurm_cluster_name="c", - nodeset={ - "n": TstNodeset(nodeset_name="n", enable_placement=True), - "x": TstNodeset(nodeset_name="x", enable_placement=False) - }, - nodeset_tpu={ - "t": TstNodeset(nodeset_name="t") - }) - lkp = util.Lookup(cfg) - - with unittest.mock.patch("resume.valid_placement_node") as mock_valid_placement_node: - mock_valid_placement_node.return_value = True - lkp.template_info = unittest.mock.Mock(return_value=unittest.mock.Mock(machine_type=unittest.mock.Mock(family="n1"))) - - assert resume._allocate_nodes_to_placements(nodes, excl_job_id, lkp) == expected diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py deleted file mode 100644 index df9f3a0137..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -import json -import mock -from pytest_unordered import unordered -from common import TstCfg, TstNodeset, TstTPU, tstInstance -import sort_nodes - -import util -import conf -import tempfile - -PRELUDE = """ -# Warning: -# This file is managed by a script. Manual modifications will be overwritten. - -""" - -def test_gen_topology_conf_empty(): - out_dir = tempfile.mkdtemp() - cfg = TstCfg(output_dir=out_dir) - conf.gen_topology_conf(util.Lookup(cfg)) - assert open(out_dir + "/cloud_topology.conf").read() == PRELUDE + "\n" - - -@mock.patch("tpu.TPU.make") -def test_gen_topology_conf(tpu_mock): - output_dir = tempfile.mkdtemp() - cfg = TstCfg( - nodeset_tpu={ - "a": TstNodeset("bold", node_count_static=4, node_count_dynamic_max=5), - "b": TstNodeset("slim", node_count_dynamic_max=3), - }, - nodeset={ - "c": TstNodeset("green", node_count_static=2, node_count_dynamic_max=3), - "d": TstNodeset("blue", node_count_static=7), - "e": TstNodeset("pink", node_count_dynamic_max=4), - }, - output_dir=output_dir, - ) - - def tpu_se(ns: str, lkp) -> TstTPU: - if ns == "bold": - return TstTPU(vmcount=3) - if ns == "slim": - return TstTPU(vmcount=1) - raise AssertionError(f"unexpected TPU name: '{ns}'") - - tpu_mock.side_effect = tpu_se - - lkp = util.Lookup(cfg) - lkp.instances = lambda: { n.name: n for n in [ # type: ignore[assignment] - # nodeset blue - tstInstance("m22-blue-0"), # no physicalHost - tstInstance("m22-blue-0", physical_host="/a/a/a"), - tstInstance("m22-blue-1", physical_host="/a/a/b"), - tstInstance("m22-blue-2", physical_host="/a/b/a"), - tstInstance("m22-blue-3", physical_host="/b/a/a"), - # nodeset green - tstInstance("m22-green-3", physical_host="/a/a/c"), - ]} - - uncompressed = conf.gen_topology(lkp) - want_uncompressed = [ - #NOTE: the switch names are not unique, it's not valid content for topology.conf - # The uniquefication and compression of names are done in the compress() method - "SwitchName=slurm-root Switches=a,b,ns_blue,ns_green,ns_pink", - # "physical" topology - 'SwitchName=a Switches=a,b', - 'SwitchName=a Nodes=m22-blue-[0-1],m22-green-3', - 'SwitchName=b Nodes=m22-blue-2', - 'SwitchName=b Switches=a', - 'SwitchName=a Nodes=m22-blue-3', - # topology "by nodeset" - "SwitchName=ns_blue Nodes=m22-blue-[4-6]", - "SwitchName=ns_green Nodes=m22-green-[0-2,4]", - "SwitchName=ns_pink Nodes=m22-pink-[0-3]", - # TPU topology - "SwitchName=tpu-root Switches=ns_bold,ns_slim", - "SwitchName=ns_bold Switches=bold-[0-3]", - "SwitchName=bold-0 Nodes=m22-bold-[0-2]", - "SwitchName=bold-1 Nodes=m22-bold-3", - "SwitchName=bold-2 Nodes=m22-bold-[4-6]", - "SwitchName=bold-3 Nodes=m22-bold-[7-8]", - "SwitchName=ns_slim Nodes=m22-slim-[0-2]"] - assert list(uncompressed.render_conf_lines()) == want_uncompressed - - compressed = uncompressed.compress() - want_compressed = [ - "SwitchName=s0 Switches=s0_[0-4]", # root - # "physical" topology - 'SwitchName=s0_0 Switches=s0_0_[0-1]', # /a - 'SwitchName=s0_0_0 Nodes=m22-blue-[0-1],m22-green-3', # /a/a - 'SwitchName=s0_0_1 Nodes=m22-blue-2', # /a/b - 'SwitchName=s0_1 Switches=s0_1_0', # /b - 'SwitchName=s0_1_0 Nodes=m22-blue-3', # /b/a - # topology "by nodeset" - "SwitchName=s0_2 Nodes=m22-blue-[4-6]", - "SwitchName=s0_3 Nodes=m22-green-[0-2,4]", - "SwitchName=s0_4 Nodes=m22-pink-[0-3]", - # TPU topology - "SwitchName=s1 Switches=s1_[0-1]", - "SwitchName=s1_0 Switches=s1_0_[0-3]", - "SwitchName=s1_0_0 Nodes=m22-bold-[0-2]", - "SwitchName=s1_0_1 Nodes=m22-bold-3", - "SwitchName=s1_0_2 Nodes=m22-bold-[4-6]", - "SwitchName=s1_0_3 Nodes=m22-bold-[7-8]", - "SwitchName=s1_1 Nodes=m22-slim-[0-2]"] - assert list(compressed.render_conf_lines()) == want_compressed - - upd, summary = conf.gen_topology_conf(lkp) - assert upd == True - want_written = PRELUDE + "\n".join(want_compressed) + "\n\n" - assert open(output_dir + "/cloud_topology.conf").read() == want_written - - summary.dump(lkp) - summary_got = json.loads(open(output_dir + "/cloud_topology.summary.json").read()) - - assert summary_got == { - "down_nodes": unordered( - [f"m22-blue-{i}" for i in (4,5,6)] + - [f"m22-green-{i}" for i in (0,1,2,4)] + - [f"m22-pink-{i}" for i in range(4)]), - "tpu_nodes": unordered( - [f"m22-bold-{i}" for i in range(9)] + - [f"m22-slim-{i}" for i in range(3)]), - 'physical_host': { - 'm22-blue-0': '/a/a/a', - 'm22-blue-1': '/a/a/b', - 'm22-blue-2': '/a/b/a', - 'm22-blue-3': '/b/a/a', - 'm22-green-3': '/a/a/c'}, - } - - - -def test_gen_topology_conf_update(): - cfg = TstCfg( - nodeset={ - "c": TstNodeset("green", node_count_static=2), - }, - output_dir=tempfile.mkdtemp(), - ) - lkp = util.Lookup(cfg) - lkp.instances = lambda: { # type: ignore[assignment] - # no instances - } - - # initial generation - reconfigure - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # add node: node_count_static 2 -> 3 - reconfigure - lkp.cfg.nodeset["c"].node_count_static = 3 - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # remove node: node_count_static 3 -> 2 - no reconfigure - lkp.cfg.nodeset["c"].node_count_static = 2 - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - # set empty physicalHost - no reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - # set physicalHost - reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/c")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # change physicalHost - reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/z")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # shut down node - no reconfigure - lkp.instances = lambda: {} # type: ignore[assignment] - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - -@pytest.mark.parametrize( - "paths,expected", - [ - (["z/n-0", "z/n-1", "z/n-2", "z/n-3", "z/n-4", "z/n-10"], ['n-0', 'n-1', 'n-2', 'n-3', 'n-4', 'n-10']), - (["y/n-0", "z/n-1", "x/n-2", "x/n-3", "y/n-4", "g/n-10"], ['n-0', 'n-4', 'n-1', 'n-2', 'n-3', 'n-10']), - ]) -def test_sort_nodes_order(paths: list[str], expected: list[str]) -> None: - paths_expanded = [l.split("/") for l in paths] - assert sort_nodes.order(paths_expanded) == expected diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py deleted file mode 100644 index 69617d0301..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py +++ /dev/null @@ -1,668 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Type - -import pytest -from mock import Mock -from datetime import datetime, timezone, timedelta -import unittest - -from common import TstNodeset, TstCfg # needed to import util -import util -from util import NodeState, MachineType, AcceleratorInfo, UpcomingMaintenance, InstanceResourceStatus, FutureReservation, ReservationDetails -from google.api_core.client_options import ClientOptions # noqa: E402 -from addict import Dict as NSDict # type: ignore - -# Note: need to install pytest-mock - -@pytest.mark.parametrize( - "name,expected", - [ - ( - "az-buka-23", - { - "cluster": "az", - "nodeset": "buka", - "node": "23", - "prefix": "az-buka", - "range": None, - "suffix": "23", - }, - ), - ( - "az-buka-xyzf", - { - "cluster": "az", - "nodeset": "buka", - "node": "xyzf", - "prefix": "az-buka", - "range": None, - "suffix": "xyzf", - }, - ), - ( - "az-buka-[2-3]", - { - "cluster": "az", - "nodeset": "buka", - "node": "[2-3]", - "prefix": "az-buka", - "range": "[2-3]", - "suffix": None, - }, - ), - ], -) -def test_node_desc(name, expected): - assert util.lookup()._node_desc(name) == expected - - -@pytest.mark.parametrize( - "name,expected", - [ - ("az-buka-23", 23), - ("az-buka-0", 0), - ("az-buka", Exception), - ("az-buka-xyzf", ValueError), - ("az-buka-[2-3]", ValueError), - ], -) -def test_node_index(name, expected): - if type(expected) is type and issubclass(expected, Exception): - with pytest.raises(expected): - util.lookup().node_index(name) - else: - assert util.lookup().node_index(name) == expected - - -@pytest.mark.parametrize( - "name", - [ - "az-buka", - ], -) -def test_node_desc_fail(name): - with pytest.raises(Exception): - util.lookup()._node_desc(name) - - -@pytest.mark.parametrize( - "names,expected", - [ - ("pedro,pedro-1,pedro-2,pedro-01,pedro-02", "pedro,pedro-[1-2,01-02]"), - ("pedro,,pedro-1,,pedro-2", "pedro,pedro-[1-2]"), - ("pedro-8,pedro-9,pedro-10,pedro-11", "pedro-[8-9,10-11]"), - ("pedro-08,pedro-09,pedro-10,pedro-11", "pedro-[08-11]"), - ("pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-09]"), - ("pedro-10,pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-10]"), - ("pedro-8,pedro-9,juan-10,juan-11", "juan-[10-11],pedro-[8-9]"), - ("az,buki,vedi", "az,buki,vedi"), - ("a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12", "a[0-9,10-12]"), - ("a0,a2,a4,a6,a7,a8,a11,a12", "a[0,2,4,6-8,11-12]"), - ("seas7-0,seas7-1", "seas7-[0-1]"), - ], -) -def test_to_hostlist(names, expected): - assert util.to_hostlist(names.split(",")) == expected - - -@pytest.mark.parametrize( - "api,ep_ver,expected", - [ - ( - util.ApiEndpoint.BQ, - "v1", - ClientOptions(api_endpoint="https://bq.googleapis.com/v1/"), - ), - ( - util.ApiEndpoint.COMPUTE, - "staging_v1", - ClientOptions(api_endpoint="https://compute.googleapis.com/staging_v1/"), - ), - ( - util.ApiEndpoint.SECRET, - "v1", - ClientOptions(api_endpoint="https://secret_manager.googleapis.com/v1/"), - ), - ( - util.ApiEndpoint.STORAGE, - "beta", - ClientOptions(api_endpoint="https://storage.googleapis.com/beta/"), - ), - ( - util.ApiEndpoint.TPU, - "alpha", - ClientOptions(api_endpoint="https://tpu.googleapis.com/alpha/"), - ), - ], -) -def test_create_client_options( - api: util.ApiEndpoint, ep_ver: str, expected: ClientOptions, mocker -): - ud_mock = mocker.patch("util.universe_domain") - ep_mock = mocker.patch("util.endpoint_version") - ud_mock.return_value = "googleapis.com" - ep_mock.return_value = ep_ver - assert util.create_client_options(api).__repr__() == expected.__repr__() - - - -@pytest.mark.parametrize( - "nodeset,err", - [ - (TstNodeset(reservation_name="projects/x/reservations/y"), AssertionError), # no zones - (TstNodeset( - reservation_name="projects/x/reservations/y", - zone_policy_allow=["eine", "zwei"]), AssertionError), # multiples zones - (TstNodeset( - reservation_name="robin", - zone_policy_allow=["eine"]), ValueError), # invalid name - (TstNodeset( - reservation_name="projects/reservations/y", - zone_policy_allow=["eine"]), ValueError), # invalid name - (TstNodeset( - reservation_name="projects/x/zones/z/reservations/y", - zone_policy_allow=["eine"]), ValueError), # invalid name - ] -) -def test_nodeset_reservation_err(nodeset, err): - lkp = util.Lookup(TstCfg()) - lkp._get_reservation = Mock() - with pytest.raises(err): - lkp.nodeset_reservation(nodeset) - lkp._get_reservation.assert_not_called() # type: ignore - -@pytest.mark.parametrize( - "nodeset,policies,expected", - [ - (TstNodeset(), [], None), # no reservation - (TstNodeset( - reservation_name="projects/bobin/reservations/robin", - zone_policy_allow=["eine"]), - [], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=[], - deployment_type=None, - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/bobin/reservations/robin")), - (TstNodeset( - reservation_name="projects/bobin/reservations/robin", - zone_policy_allow=["eine"]), - ["seven/wanders", "five/red/apples", "yum"], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=["wanders", "apples", "yum"], - deployment_type=None, - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/bobin/reservations/robin")), - (TstNodeset( - reservation_name="projects/bobin/reservations/robin/snek/cheese-brie-6", - zone_policy_allow=["eine"]), - [], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=[], - deployment_type=None, - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/bobin/reservations/robin/snek/cheese-brie-6")), - - ]) - -def test_nodeset_reservation_ok(nodeset, policies, expected): - lkp = util.Lookup(TstCfg()) - lkp._get_reservation = Mock() - - if not expected: - assert lkp.nodeset_reservation(nodeset) is None - lkp._get_reservation.assert_not_called() # type: ignore - return - - lkp._get_reservation.return_value = { # type: ignore - "resourcePolicies": {i: p for i, p in enumerate(policies)}, - } - assert lkp.nodeset_reservation(nodeset) == expected - lkp._get_reservation.assert_called_once_with(expected.project, expected.zone, expected.name) # type: ignore - -@pytest.mark.parametrize( - "job_info,expected_job", - [ - ( - """JobId=123 - TimeLimit=02:00:00 - JobName=myjob - JobState=PENDING - ReqNodeList=node-[1-10]""", - util.Job( - id=123, - duration=timedelta(days=0, hours=2, minutes=0, seconds=0), - name="myjob", - job_state="PENDING", - required_nodes="node-[1-10]" - ), - ), - ( - """JobId=456 - JobName=anotherjob - JobState=PENDING - ReqNodeList=node-group1""", - util.Job( - id=456, - duration=None, - name="anotherjob", - job_state="PENDING", - required_nodes="node-group1" - ), - ), - ( - """JobId=789 - TimeLimit=00:30:00 - JobState=COMPLETED""", - util.Job( - id=789, - duration=timedelta(minutes=30), - name=None, - job_state="COMPLETED", - required_nodes=None - ), - ), - ( - """JobId=101112 - TimeLimit=1-00:30:00 - JobState=COMPLETED, - ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", - util.Job( - id=101112, - duration=timedelta(days=1, hours=0, minutes=30, seconds=0), - name=None, - job_state="COMPLETED", - required_nodes="node-[1-10],grob-pop-[2,1,44-77]" - ), - ), - ( - """JobId=131415 - TimeLimit=1-00:30:00 - JobName=mynode-1_maintenance - JobState=COMPLETED, - ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", - util.Job( - id=131415, - duration=timedelta(days=1, hours=0, minutes=30, seconds=0), - name="mynode-1_maintenance", - job_state="COMPLETED", - required_nodes="node-[1-10],grob-pop-[2,1,44-77]" - ), - ), - ], -) -def test_parse_job_info(job_info, expected_job): - lkp = util.Lookup(TstCfg()) - assert lkp._parse_job_info(job_info) == expected_job - - - -@pytest.mark.parametrize( - "node,state,want", - [ - ("c-n-2", NodeState("DOWN", frozenset([])), NodeState("DOWN", frozenset([]))), # happy scenario - ("c-d-vodoo", None, None), # dynamic nodeset - ("c-x-44", None, None), # unknown(removed) nodeset - ("c-n-7", None, None), # Out of bounds: c-n-[0-4] - downsized nodeset - ("c-t-7", None, None), # Out of bounds: c-t-[0-4] - downsized nodeset TPU - ("c-n-2", None, RuntimeError), # something is wrong - ("c-t-2", None, RuntimeError), # something is wrong, but TPU - - # Check boundaries match [0-5) - ("c-n-5", None, None), # out of boundaries - ("c-n-4", None, RuntimeError), # within boundaries - ]) -def test_node_state(node: str, state: Optional[NodeState], want: NodeState | None | Type[Exception]): - cfg = TstCfg( - slurm_cluster_name="c", - nodeset={ - "n": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, - nodeset_tpu={ - "t": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, - nodeset_dyn={ - "d": TstNodeset()}, - ) - lkp = util.Lookup(cfg) - lkp.slurm_nodes = lambda: {node: state} if state else {} # type: ignore[assignment] - # ... see https://github.com/python/typeshed/issues/6347 - - if type(want) is type and issubclass(want, Exception): - with pytest.raises(want): - lkp.node_state(node) - else: - assert lkp.node_state(node) == want - - - -@pytest.mark.parametrize( - "jo,want", - [ - ({ - "accelerators": [ { "guestAcceleratorCount": 1, "guestAcceleratorType": "nvidia-tesla-a100" } ], - "creationTimestamp": "1969-12-31T16:00:00.000-08:00", - "description": "Accelerator Optimized: 1 NVIDIA Tesla A100 GPU, 12 vCPUs, 85GB RAM", - "guestCpus": 12, - "id": "1000012", - "imageSpaceGb": 0, - "isSharedCpu": False, - "kind": "compute#machineType", - "maximumPersistentDisks": 128, - "maximumPersistentDisksSizeGb": "263168", - "memoryMb": 87040, - "name": "a2-highgpu-1g", - "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/us-central1-a/machineTypes/a2-highgpu-1g", - "zone": "us-central1-a" - }, MachineType( - name="a2-highgpu-1g", - guest_cpus=12, - memory_mb=87040, - accelerators=[ - AcceleratorInfo(type="nvidia-tesla-a100", count=1) - ] - )), - ({ - "architecture": "X86_64", - "creationTimestamp": "1969-12-31T16:00:00.000-08:00", - "description": "8 vCPUs, 32 GB RAM", - "guestCpus": 8, - "id": "1210008", - "imageSpaceGb": 0, - "isSharedCpu": False, - "kind": "compute#machineType", - "maximumPersistentDisks": 128, - "maximumPersistentDisksSizeGb": "263168", - "memoryMb": 32768, - "name": "t2d-standard-8", - "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/europe-north2-b/machineTypes/t2d-standard-8", - "zone": "europe-north2-b" - }, MachineType( - name="t2d-standard-8", - guest_cpus=8, - memory_mb=32768, - accelerators=[] - )), - ]) -def test_MachineType_from_json(jo: dict, want: MachineType): - assert MachineType.from_json(jo) == want - - -@pytest.mark.parametrize( - "template,expected", - [ - ( - NSDict({ - "machine_type": MachineType( - name="e2", - guest_cpus=12, - memory_mb=87040, - accelerators=[]), - }), - None - ), - ( - NSDict({ - "machine_type": MachineType( - name="tpu-machine", - guest_cpus=12, - memory_mb=87040, - accelerators=[ - AcceleratorInfo(type="tpu-v6", count=1) - ]), - }), - None - ), - ( - NSDict({ - "machine_type": MachineType( - name="a2-highgpu-1g", - guest_cpus=12, - memory_mb=87040, - accelerators=[AcceleratorInfo(type="nvidia-tesla-a100", count=1)] - ), - }), - AcceleratorInfo(type="nvidia-tesla-a100", count=1) - ), - ( - NSDict({ - "machine_type": MachineType( - name="a2-highgpu-1g", - guest_cpus=12, - memory_mb=87040, - accelerators=[]), - "guestAccelerators":[ { "acceleratorCount": 1, "acceleratorType": "nvidia-tesla-a100" } ], - }), - AcceleratorInfo(type="nvidia-tesla-a100", count=1) - ), - ], -) -def test_get_template_gpu(template, expected): - assert util.get_template_gpu(template) == expected - - -UTC, PST = timezone.utc, timezone(timedelta(hours=-8)) - -@pytest.mark.parametrize( - "got,want", - [ - # from instance.creationTimestamp: - ("2024-11-30T12:47:51.676-08:00", datetime(2024, 11, 30, 12, 47, 51, 676000, tzinfo=PST)), - # from futureReservation.creationTimestamp - ("2024-11-05T15:23:33.702-08:00", datetime(2024, 11, 5, 15, 23, 33, 702000, tzinfo=PST)), - # from futureReservation.timeWindow.endTime - ("2025-01-15T00:00:00Z", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), - # fallback to UTC if no tz is specified - ("2025-01-15T00:00:00", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), - ]) -def test_parse_gcp_timestamp(got: str, want: datetime): - assert util.parse_gcp_timestamp(got) == want - - -@pytest.mark.parametrize( - "got,want", - [ - (None, None), - (dict( - windowStartTime="2025-01-15T00:00:00Z", - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - (dict( - startTimeWindow=dict( - earliest="2025-01-15T00:00:00Z"), - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - (dict( - windowStartTime="2025-01-15T00:00:00Z", - startTimeWindow=dict( - earliest="2025-01-25T00:00:00Z"), # ignored - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - ]) -def tests_parse_UpcomingMaintenance_OK(got: dict, want: Optional[UpcomingMaintenance]): - assert UpcomingMaintenance.from_json(got) == want - - -@pytest.mark.parametrize( - "got", - [ - {}, - dict( - windowStartTime=dict( - earliest="2025-01-15T00:00:00Z")), - ]) -def tests_parse_UpcomingMaintenance_FAIL(got: dict): - with pytest.raises(ValueError): - UpcomingMaintenance.from_json(got) - - -@pytest.mark.parametrize( - "got,want", - [ - (None, InstanceResourceStatus( - physical_host=None, - upcoming_maintenance=None)), - ({}, InstanceResourceStatus( - physical_host=None, - upcoming_maintenance=None)), - (dict( - physicalHost="/aaa/bbb/ccc"), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=None)), - (dict( # invalid upcomingMaintenance field to be ignored - physicalHost="/aaa/bbb/ccc", - upcomingMaintenance="maintenance is upon us"), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=None)), - (dict( - physicalHost="/aaa/bbb/ccc", - upcomingMaintenance=dict(windowStartTime="2025-01-15T00:00:00Z")), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=UpcomingMaintenance( - window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC)))), - ]) -def test_parse_InstanceResourceStatus(got: dict, want: Optional[InstanceResourceStatus]): - assert InstanceResourceStatus.from_json(got) == want - - -@pytest.mark.parametrize( - "link,component_name,expected", - [ - ( - "mylink/regions/us-cental1/other", - "regions", - "us-cental1" - ), - ( - "mylink/global/other", - "regions", - None - ), - ], -) -def test_get_self_link_component(link, component_name, expected): - assert util.get_self_link_component(link, component_name) == expected - - -def test_future_reservation_none(): - lkp = util.Lookup(TstCfg()) - assert lkp.future_reservation(TstNodeset()) == None - - -def test_future_reservation_declined(): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-03T23:30:00Z" }, - status = {"procurementStatus": "DECLINED"}, - reservationMode = "CALENDAR", - specificReservationRequired = True, - )) - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 3, 23, 30, tzinfo=timezone.utc), - reservation_mode="CALENDAR", - active_reservation=None) - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - -@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 13, 0, 0, tzinfo=timezone.utc)) -def test_future_reservation_active(_): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, - status = { - "procurementStatus": "FULFILLED", - "autoCreatedReservations": [ - "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" - ], - }, - specificReservationRequired = True, - )) - lkp._get_reservation = Mock(return_value=dict()) - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), - reservation_mode=None, - active_reservation=ReservationDetails( - project='manhattan', - zone='danger', - name='melon', - policies=[], - reservation_mode=None, - assured_count=0, - delete_at_time=None, - bulk_insert_name="projects/manhattan/reservations/melon", - deployment_type=None)) - - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - lkp._get_reservation.assert_called_once_with("manhattan", "danger", "melon") - -@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 28, 0, 0, tzinfo=timezone.utc)) -def test_future_reservation_inactive(_): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, - status = { - "procurementStatus": "FULFILLED", - "autoCreatedReservations": [ - "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" - ], - }, - reservationMode = "DEFAULT", - specificReservationRequired = True, - )) - lkp._get_reservation = Mock() - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), - reservation_mode="DEFAULT", - active_reservation=None) - - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - lkp._get_reservation.assert_not_called() diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test deleted file mode 100644 index a583642015..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -unset CUDA_VISIBLE_DEVICES - -LOG_FILE="/var/log/slurm/chs_health_check.log" -TMP_DCGM_OUT="/tmp/dcgm.out" -TMP_ECC_ERRORS_OUT="/tmp/ecc_errors.out" - -log_step() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" -} - -# Fail gracefully if nvidia-smi or dcgmi doesn't exist -if ! type -P nvidia-smi 1>/dev/null; then - log_step "nvidia-smi not found - this script requires nvidia-smi to function" - exit 0 -fi - -if ! type -P dcgmi 1>/dev/null; then - log_step "dcgmi not found - this script requires dcgmi to function" - exit 0 -fi - -if ! type -P nv-hostengine 1>/dev/null; then - log_step "nv-hostengine not found - this script requires nv-hostengine to function" - exit 0 -fi - -################################################### -# Disable running health checks -################################################### -# Check if the environment variable '$SLURM_JOB_EXTRA' is set and contains the -# substring 'healthchecks_prolog=off' -if [[ -n "$SLURM_JOB_EXTRA" ]]; then - log_step "Environment variable SLURM_JOB_EXTRA is set. Checking if it contains healthchecks_prolog=off." - # Check if the value of the variable matches the string "healthchecks_prolog=off" - if [[ "$SLURM_JOB_EXTRA" == *"healthchecks_prolog=off"* ]]; then - log_step "Environment variable SLURM_JOB_EXTRA matches substring healthchecks_prolog=off. Skipping health checks." - exit 0 - else - log_step "Environment variable SLURM_JOB_EXTRA does NOT match substring healthchecks_prolog=off. Attempting to run health checks." - fi -else - log_step "Environment variable SLURM_JOB_EXTRA is NOT set. Attempting to run health checks." -fi - -# Exit if GPU isn't H/B 100/200 -GPU_MODEL=$(nvidia-smi --query-gpu=name --format=csv,noheader) -if ! [[ "$GPU_MODEL" =~ [BH][1-2]00 ]]; then - log_step "No Supported GPU detected" - exit 0 -fi - -NUMGPUS=$(nvidia-smi -L | wc -l) - -# Check that all GPUs are healthy via DCGM and check for ECC errors -if [ $NUMGPUS -gt 0 ]; then - log_step "Execute DCGM health check, ECC error check, and NVLink error check for GPUs" - GPULIST=$(nvidia-smi --query-gpu=index --format=csv,noheader | tr '\n' ',' | sed 's/,$//') - rm -f $TMP_DCGM_OUT - rm -f $TMP_ECC_ERRORS_OUT - - # Run DCGM checks - START_HOSTENGINE=false - if ! pidof nv-hostengine > /dev/null; then - log_step "Starting nv-hostengine..." - nv-hostengine >> "$LOG_FILE" 2>&1 - sleep 1 # Give it a moment to start up - START_HOSTENGINE=true - fi - GROUPID=$(dcgmi group -c gpuinfo | awk '{print $NF}' | tr -d ' ') - dcgmi group -g $GROUPID -a $GPULIST >> "$LOG_FILE" 2>&1 - dcgmi diag -g $GROUPID -r 1 > "$TMP_DCGM_OUT" 2>&1 - cat "$TMP_DCGM_OUT" >> "$LOG_FILE" - dcgmi group -d $GROUPID >> "$LOG_FILE" 2>&1 - - # Terminate the host engine if it was manually started - if [ "$START_HOSTENGINE" = true ]; then - log_step "Terminating nv-hostengine..." - nv-hostengine -t >> "$LOG_FILE" 2>&1 - fi - - # Check for DCGM failures - DCGM_FAILED=0 - if grep -i fail "$TMP_DCGM_OUT" > /dev/null; then - DCGM_FAILED=1 - fi - - # Check for ECC errors - nvidia-smi --query-gpu=ecc.errors.uncorrected.volatile.total --format=csv,noheader > "$TMP_ECC_ERRORS_OUT" - cat "$TMP_ECC_ERRORS_OUT" >> "$LOG_FILE" - ECC_ERRORS=$(awk -F', ' '{sum += $2} END {print sum}' "$TMP_ECC_ERRORS_OUT") - log_step "ECC Errors: $ECC_ERRORS" - - # Check for NVLink errors - NVLINK_ERRORS=$(nvidia-smi nvlink -sc 0bz -i 0 2>/dev/null | grep -i "Error Count" | awk '{sum += $3} END {print sum}') - # Set to 0 if empty/null - NVLINK_ERRORS=${NVLINK_ERRORS:-0} - log_step "NVLink Errors: $NVLINK_ERRORS" - - if [ $DCGM_FAILED -eq 1 ] || \ - [ $ECC_ERRORS -gt 0 ] || \ - [ $NVLINK_ERRORS -gt 0 ]; then - REASON="GPU issues detected: " - if [ $DCGM_FAILED -eq 1 ]; then - REASON+="DCGM test failed, " - fi - if [ $ECC_ERRORS -gt 0 ]; then - REASON+="ECC errors found ($ECC_ERRORS double-bit errors), " - fi - if [ $NVLINK_ERRORS -gt 0 ]; then - REASON+="NVLink errors detected ($NVLINK_ERRORS errors), " - fi - REASON+="see $LOG_FILE" - log_step "$REASON" - exit 1 - fi -fi diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog deleted file mode 100644 index a22ddea9e5..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Main TaskEpilog Script -# This script executes all *.sh scripts found in /slurm/custom_scripts/task_epilog.d/ -# -# slurm.conf configuration: -# TaskEpilog=/slurm/scripts/tools/task-epilog - -# Directory containing the individual task epilog scripts -EPILOG_D_DIR="/slurm/custom_scripts/task_epilog.d" - -# --- Output Handling for TaskEpilog --- -# The stdout and stderr of this script (and the sub-scripts it calls) -# are typically captured by Slurm and written to the job's output/error file -# or a separate Slurm log, depending on configuration. -# Unlike TaskProlog, stdout is not typically parsed for special commands -# like 'export' or 'print' to affect the (now finished) task's environment. -# -# --- Error Handling --- -# If any script in EPILOG_D_DIR exits with a non-zero status, -# this main script will also exit with a non-zero status. -# Slurm will log this. Depending on Slurm's configuration, -# frequent epilog failures might lead to node issues or alerts. -set -e # Exit immediately if a command exits with a non-zero status. - -# Check if the directory exists -if [[ ! -d "$EPILOG_D_DIR" ]]; then - # Log in task stdout and exit if the directory is missing. This likely indicates a configuration error. - echo "print TaskEpilog Error: Directory '$EPILOG_D_DIR' not found. Check Slurm configuration." - exit 1 -fi - -# Find and execute all *.sh scripts in the directory -# Scripts will be executed in reverse alphabetical order of their filenames. -find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do - if [[ -x "$script" ]]; then - # Execute the script. Its stdout will be captured by this wrapper. - # Its stderr will also be passed through. - # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. - "$script" - else - # Log in task stdout a warning if a *.sh file is found but is not executable - echo "print TaskEpilog Warning: Script '$script' is not executable and will be skipped." - fi -done - -# Check if any scripts were found and executed -if [[ $(find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then - # Log in task stdout if no scripts were found to execute - echo "print TaskEpilog Info: No executable *.sh scripts found in $EPILOG_D_DIR." -fi - -# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) -exit 0 diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog deleted file mode 100644 index feddb23209..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Main TaskProlog Script -# This script executes all *.sh scripts found in /slurm/custom_scripts/task_prolog.d/ -# -# slurm.conf configuration: -# TaskProlog=/slurm/scripts/tools/task-prolog - -# Directory containing the individual task prolog scripts -PROLOG_D_DIR="/slurm/custom_scripts/task_prolog.d" - -# --- Output Handling for TaskProlog --- -# Slurm's TaskProlog can interpret specific stdout lines: -# - "export NAME=value" : Sets an environment variable for the task. -# - "unset NAME" : Unsets an environment variable for the task. -# - "print message" : Prints a message to the task's standard output. -# -# This wrapper script will concatenate the stdout of all sub-scripts. -# If sub-scripts need to set/unset environment variables or print messages -# for the task, they should output the appropriate "export", "unset", or "print" -# commands to their own stdout. - -# --- Error Handling --- -# If any script in PROLOG_D_DIR exits with a non-zero status, -# this main script will also exit with a non-zero status. -# This will typically cause the task to fail. -set -e # Exit immediately if a command exits with a non-zero status. - -# Check if the directory exists -if [[ ! -d "$PROLOG_D_DIR" ]]; then - # Log in task stdout and exit if the directory is missing. All jobs will be failed. - echo "print TaskProlog Error: Directory '$PROLOG_D_DIR' not found. Check Slurm configuration." - exit 1 -fi - -# Find and execute all *.sh scripts in the directory -# Scripts will be executed in reverse alphabetical order of their filenames. -find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do - if [[ -x "$script" ]]; then - # Execute the script. Its stdout will be captured by this wrapper. - # Its stderr will also be passed through. - # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. - "$script" - else - # Log a warning in task stdout if a *.sh file is found but is not executable - echo "print TaskProlog Warning: Script '$script' is not executable and will be skipped." - fi -done - -# Check if any scripts were found and executed -if [[ $(find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then - # Log in task stdout if no scripts were found to execute - echo "print TaskProlog Info: No executable *.sh scripts found in $PROLOG_D_DIR." -fi - -# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) -exit 0 diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py deleted file mode 100644 index 531f0348dc..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py +++ /dev/null @@ -1,331 +0,0 @@ -# mypy: ignore-errors -# This implementation of TPU integration is to be deprecated - -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List - -import socket -import logging -from pathlib import Path -import yaml - -import util -from util import create_client_options, ApiEndpoint - -from google.cloud import tpu_v2 as tpu # noqa: E402 -import google.api_core.exceptions as gExceptions # noqa: E402 - -log = logging.getLogger() - -_tpu_cache = {} - -class TPU: - """Class for handling the TPU-vm nodes""" - - State = tpu.types.cloud_tpu.Node.State - TPUS_PER_VM = 4 - __expected_states = { - "create": State.READY, - "start": State.READY, - "stop": State.STOPPED, - } - - __tpu_version_mapping = { - "V2": tpu.AcceleratorConfig().Type.V2, - "V3": tpu.AcceleratorConfig().Type.V3, - "V4": tpu.AcceleratorConfig().Type.V4, - } - - @classmethod - def make(cls, nodeset_name: str, lkp: util.Lookup) -> "TPU": - key = (id(lkp), nodeset_name) - if key not in _tpu_cache: - nodeset = lkp.cfg.nodeset_tpu[nodeset_name] - _tpu_cache[key] = cls(nodeset, lkp) - return _tpu_cache[key] - - - def __init__(self, nodeset: object, lkp: util.Lookup): - self._nodeset = nodeset - self.lkp = lkp - self._parent = f"projects/{lkp.project}/locations/{nodeset.zone}" - co = create_client_options(ApiEndpoint.TPU) - self._client = tpu.TpuClient(client_options=co) - self.data_disks = [] - for data_disk in nodeset.data_disks: - ad = tpu.AttachedDisk() - ad.source_disk = data_disk - ad.mode = tpu.AttachedDisk.DiskMode.DISK_MODE_UNSPECIFIED - self.data_disks.append(ad) - ns_ac = nodeset.accelerator_config - if ns_ac.topology != "" and ns_ac.version != "": - ac = tpu.AcceleratorConfig() - ac.topology = ns_ac.topology - ac.type_ = self.__tpu_version_mapping[ns_ac.version] - self.ac = ac - else: - req = tpu.GetAcceleratorTypeRequest( - name=f"{self._parent}/acceleratorTypes/{nodeset.node_type}" - ) - self.ac = self._client.get_accelerator_type(req).accelerator_configs[0] - self.vmcount = self.__calc_vm_from_topology(self.ac.topology) - - @property - def nodeset(self): - return self._nodeset - - @property - def preserve_tpu(self): - return self._nodeset.preserve_tpu - - @property - def node_type(self): - return self._nodeset.node_type - - @property - def tf_version(self): - return self._nodeset.tf_version - - @property - def enable_public_ip(self): - return self._nodeset.enable_public_ip - - @property - def preemptible(self): - return self._nodeset.preemptible - - @property - def reserved(self): - return self._nodeset.reserved - - @property - def service_account(self): - return self._nodeset.service_account - - @property - def zone(self): - return self._nodeset.zone - - def check_node_type(self): - if self.node_type is None: - return False - try: - request = tpu.GetAcceleratorTypeRequest( - name=f"{self._parent}/acceleratorTypes/{self.node_type}" - ) - return self._client.get_accelerator_type(request=request) is not None - except Exception: - return False - - def check_tf_version(self): - try: - request = tpu.GetRuntimeVersionRequest( - name=f"{self._parent}/runtimeVersions/{self.tf_version}" - ) - return self._client.get_runtime_version(request=request) is not None - except Exception: - return False - - def __calc_vm_from_topology(self, topology): - topo = topology.split("x") - tot = 1 - for num in topo: - tot = tot * int(num) - return tot // self.TPUS_PER_VM - - def __check_resp(self, response, op_name): - des_state = self.__expected_states.get(op_name) - # If the state is not in the table just print the response - if des_state is None: - return False - if response.__class__.__name__ != "Node": # If the response is not a node fail - return False - if response.state == des_state: - return True - return False - - def list_nodes(self): - try: - request = tpu.ListNodesRequest(parent=self._parent) - res = self._client.list_nodes(request=request) - except gExceptions.NotFound: - res = None - return res - - def list_node_names(self): - return [node.name.split("/")[-1] for node in self.list_nodes()] - - def start_node(self, nodename): - request = tpu.StartNodeRequest(name=f"{self._parent}/nodes/{nodename}") - resp = self._client.start_node(request=request).result() - return self.__check_resp(resp, "start") - - def stop_node(self, nodename): - request = tpu.StopNodeRequest(name=f"{self._parent}/nodes/{nodename}") - resp = self._client.stop_node(request=request).result() - return self.__check_resp(resp, "stop") - - def get_node(self, nodename): - try: - request = tpu.GetNodeRequest(name=f"{self._parent}/nodes/{nodename}") - res = self._client.get_node(request=request) - except gExceptions.NotFound: - res = None - return res - - def _register_node(self, nodename, ip_addr): - dns_name = socket.getnameinfo((ip_addr, 0), 0)[0] - util.run( - f"{self.lkp.scontrol} update nodename={nodename} nodeaddr={ip_addr} nodehostname={dns_name}" - ) - - def create_node(self, nodename): - if self.vmcount > 1 and not isinstance(nodename, list): - log.error( - f"Tried to create a {self.vmcount} node TPU on nodeset {self._nodeset.nodeset_name} but only received one nodename {nodename}" - ) - return False - if self.vmcount > 1 and ( - isinstance(nodename, list) and len(nodename) != self.vmcount - ): - log.error( - f"Expected to receive a list of {self.vmcount} nodenames for TPU node creation in nodeset {self._nodeset.nodeset_name}, but received this list {nodename}" - ) - return False - - node = tpu.Node() - node.accelerator_config = self.ac - node.runtime_version = f"tpu-vm-tf-{self.tf_version}" - startup_script = """ - #!/bin/bash - echo "startup script not found > /var/log/startup_error.log" - """ - with open( - Path(self.lkp.cfg.slurm_scripts_dir or util.dirs.scripts) / "startup.sh", "r" - ) as script: - startup_script = script.read() - if isinstance(nodename, list): - node_id = nodename[0] - slurm_names = [] - wid = 0 - for node_wid in nodename: - slurm_names.append(f"WORKER_{wid}:{node_wid}") - wid += 1 - else: - node_id = nodename - slurm_names = [f"WORKER_0:{nodename}"] - node.metadata = { - "slurm_docker_image": self.nodeset.docker_image, - "startup-script": startup_script, - "slurm_instance_role": "compute", - "slurm_cluster_name": self.lkp.cfg.slurm_cluster_name, - "slurm_bucket_path": self.lkp.cfg.bucket_path, - "slurm_names": ";".join(slurm_names), - "universe_domain": util.universe_domain(), - } - node.tags = [self.lkp.cfg.slurm_cluster_name] - if self.nodeset.service_account: - node.service_account.email = self.nodeset.service_account.email - node.service_account.scope = self.nodeset.service_account.scopes - node.scheduling_config.preemptible = self.preemptible - node.scheduling_config.reserved = self.reserved - node.network_config.subnetwork = self.nodeset.subnetwork - node.network_config.enable_external_ips = self.enable_public_ip - if self.data_disks: - node.data_disks = self.data_disks - - request = tpu.CreateNodeRequest(parent=self._parent, node=node, node_id=node_id) - resp = self._client.create_node(request=request).result() - if not self.__check_resp(resp, "create"): - return False - if isinstance(nodename, list): - for node_id, net_endpoint in zip(nodename, resp.network_endpoints): - self._register_node(node_id, net_endpoint.ip_address) - else: - ip_add = resp.network_endpoints[0].ip_address - self._register_node(nodename, ip_add) - return True - - def delete_node(self, nodename): - request = tpu.DeleteNodeRequest(name=f"{self._parent}/nodes/{nodename}") - try: - resp = self._client.delete_node(request=request).result() - if resp: - return self.get_node(nodename=nodename) is None - return False - except gExceptions.NotFound: - # log only error if vmcount is 1 as for other tpu vm count, this could be "phantom" nodes - if self.vmcount == 1: - log.error(f"Tpu single node {nodename} not found") - else: - # for the TPU nodes that consist in more than one vm, only the first node of the TPU a.k.a. the master node will - # exist as real TPU nodes, so the other ones are expected to not be found, check the hostname of the node that has - # not been found, and if it ends in 0, it means that is the master node and it should have been found, and in consequence - # log an error - nodehostname = yaml.safe_load( - util.run(f"{self.lkp.scontrol} --yaml show node {nodename}").stdout.rstrip() - )["nodes"][0]["hostname"] - if nodehostname.split("-")[-1] == "0": - log.error(f"TPU master node {nodename} not found") - else: - log.info(f"Deleted TPU 'phantom' node {nodename}") - # If the node is not found it is tecnichally deleted, so return success. - return True - -def _stop_tpu(node: str) -> None: - lkp = util.lookup() - tpuobj = TPU.make(lkp.node_nodeset_name(node), lkp) - if tpuobj.nodeset.preserve_tpu and tpuobj.vmcount == 1: - log.info(f"stopping node {node}") - if tpuobj.stop_node(node): - return - log.error("Error stopping node {node} will delete instead") - log.info(f"deleting node {node}") - if not tpuobj.delete_node(node): - log.error("Error deleting node {node}") - - -def delete_tpu_instances(instances: List[str]) -> None: - util.execute_with_futures(_stop_tpu, instances) - - -def start_tpu(node: List[str]): - lkp = util.lookup() - tpuobj = TPU.make(lkp.node_nodeset_name(node[0]), lkp) - - if len(node) == 1: - node = node[0] - log.debug( - f"Will create a TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node}" - ) - tpunode = tpuobj.get_node(node) - if tpunode is None: - if not tpuobj.create_node(nodename=node): - log.error("Error creating tpu node {node}") - else: - if tpuobj.preserve_tpu: - if not tpuobj.start_node(nodename=node): - log.error("Error starting tpu node {node}") - else: - log.info( - f"Tpu node {node} is already created, but will not start it because nodeset does not have preserve_tpu option active." - ) - else: - log.debug( - f"Will create a multi-vm TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node[0]}" - ) - if not tpuobj.create_node(nodename=node): - log.error("Error creating tpu node {node}") diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py deleted file mode 100644 index 217fd0bca2..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py +++ /dev/null @@ -1,2224 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Iterable, List, Tuple, Optional, Any, Dict, Sequence, Type, Callable, Union -import argparse -import base64 -from dataclasses import dataclass, field -from datetime import timedelta, datetime, timezone -import hashlib -import inspect -import json -import logging -import logging.config -import logging.handlers -import math -import os -import re -import shlex -import shutil -import socket -import subprocess -import sys -from enum import Enum -from collections import defaultdict -from concurrent.futures import ThreadPoolExecutor, as_completed -from contextlib import contextmanager -from functools import lru_cache, reduce, wraps -from itertools import chain, islice -from pathlib import Path -from time import sleep, time - -# TODO: remove "type: ignore" once moved to newer version of libraries -from google.cloud import secretmanager -from google.cloud import storage # type: ignore - -import google.auth # type: ignore -from google.oauth2 import service_account # type: ignore -import googleapiclient.discovery # type: ignore -import google_auth_httplib2 # type: ignore -from googleapiclient.http import set_user_agent # type: ignore -from google.api_core.client_options import ClientOptions -import httplib2 - -import google.api_core.exceptions as gExceptions - -import requests as requests_lib - -import yaml -from addict import Dict as NSDict # type: ignore -import file_cache - -USER_AGENT = "Slurm_GCP_Scripts/1.5 (GPN:SchedMD)" -ENV_CONFIG_YAML = os.getenv("SLURM_CONFIG_YAML") -if ENV_CONFIG_YAML: - CONFIG_FILE = Path(ENV_CONFIG_YAML) -else: - CONFIG_FILE = Path(__file__).with_name("config.yaml") -API_REQ_LIMIT = 2000 - - -def mkdirp(path: Path) -> None: - path.mkdir(parents=True, exist_ok=True) - - -scripts_dir = next( - p for p in (Path(__file__).parent, Path("/slurm/scripts")) if p.is_dir() -) - - -# load all directories as Paths into a dict-like namespace -dirs = NSDict( - home = Path("/home"), - apps = Path("/opt/apps"), - slurm = Path("/slurm"), - scripts = scripts_dir, - custom_scripts = Path("/slurm/custom_scripts"), - munge = Path("/etc/munge"), - secdisk = Path("/mnt/disks/sec"), - log = Path("/var/log/slurm"), - slurm_bucket_mount = Path("/slurm/bucket"), -) - -slurmdirs = NSDict( - prefix = Path("/usr/local"), - etc = Path("/usr/local/etc/slurm"), - state = Path("/var/spool/slurm"), - key_distribution = Path("/slurm/key_distribution"), -) - - -# TODO: Remove this hack (relies on undocumented behavior of PyYAML) -# No need to represent NSDict and Path once we move to properly typed & serializable config. -yaml.SafeDumper.yaml_representers[ - None # type: ignore -] = lambda self, data: yaml.representer.SafeRepresenter.represent_str(self, str(data)) # type: ignore - - -class ApiEndpoint(Enum): - COMPUTE = "compute" - BQ = "bq" - STORAGE = "storage" - TPU = "tpu" - SECRET = "secret_manager" - - -@dataclass(frozen=True) -class AcceleratorInfo: - type: str - count: int - - @classmethod - def from_json(cls, jo: dict) -> "AcceleratorInfo": - return cls( - type=jo["guestAcceleratorType"], - count=jo["guestAcceleratorCount"]) - -@dataclass(frozen=True) -class MachineType: - name: str - guest_cpus: int - memory_mb: int - accelerators: List[AcceleratorInfo] - - @classmethod - def from_json(cls, jo: dict) -> "MachineType": - return cls( - name=jo["name"], - guest_cpus=jo["guestCpus"], - memory_mb=jo["memoryMb"], - accelerators=[ - AcceleratorInfo.from_json(a) for a in jo.get("accelerators", [])], - ) - - @property - def family(self) -> str: - # TODO: doesn't work with N1 custom machine types - # See https://cloud.google.com/compute/docs/instances/creating-instance-with-custom-machine-type#create - return self.name.split("-")[0] - - @property - def supports_smt(self) -> bool: - # https://cloud.google.com/compute/docs/cpu-platforms - if self.family in ("t2a", "t2d", "h3", "c4a", "h4d",): - return False - if self.guest_cpus == 1: - return False - return True - - @property - def sockets(self) -> int: - return { - "h3": 2, - "h4d": 2, - "c2d": 2 if self.guest_cpus > 56 else 1, - "a3": 2, - "c2": 2 if self.guest_cpus > 30 else 1, - "c3": 2 if self.guest_cpus > 88 else 1, - "c3d": 2 if self.guest_cpus > 180 else 1, - "c4": 2 if self.guest_cpus > 96 else 1, - "c4d": 2 if self.guest_cpus > 192 else 1, - }.get( - self.family, - 1, # assume 1 socket for all other families - ) - - -@dataclass(frozen=True) -class UpcomingMaintenance: - window_start_time: datetime - - @classmethod - def from_json(cls, jo: Optional[dict]) -> Optional["UpcomingMaintenance"]: - if jo is None: - return None - try: - if "windowStartTime" in jo: - ts = parse_gcp_timestamp(jo["windowStartTime"]) - elif "startTimeWindow" in jo: - ts = parse_gcp_timestamp(jo["startTimeWindow"]["earliest"]) - else: - raise Exception("Neither windowStartTime nor startTimeWindow are found") - except BaseException as e: - raise ValueError(f"Unexpected format for upcomingMaintenance: {jo}") from e - return cls(window_start_time=ts) - -@dataclass(frozen=True) -class InstanceResourceStatus: - physical_host: Optional[str] - upcoming_maintenance: Optional[UpcomingMaintenance] - - @classmethod - def from_json(cls, jo: Optional[dict]) -> "InstanceResourceStatus": - if not jo: - return cls( - physical_host=None, - upcoming_maintenance=None, - ) - - try: - maint = UpcomingMaintenance.from_json(jo.get("upcomingMaintenance")) - except ValueError as e: - log.exception("Failed to parse upcomingMaintenance, ignoring") - maint = None # intentionally swallow exception - - return cls( - physical_host=jo.get("physicalHost"), - upcoming_maintenance=maint, - ) - - -@dataclass(frozen=True) -class Instance: - name: str - zone: str - status: str - creation_timestamp: datetime - role: Optional[str] - resource_status: InstanceResourceStatus - metadata: Dict[str, str] - # TODO: use proper InstanceScheduling class - scheduling: NSDict - - @classmethod - def from_json(cls, jo: dict) -> "Instance": - return cls( - name=jo["name"], - zone=trim_self_link(jo["zone"]), - status=jo["status"], - creation_timestamp=parse_gcp_timestamp(jo["creationTimestamp"]), - resource_status=InstanceResourceStatus.from_json(jo.get("resourceStatus")), - scheduling=NSDict(jo.get("scheduling")), - role = jo.get("labels", {}).get("slurm_instance_role"), - metadata = {k["key"]: k["value"] for k in jo.get("metadata", {}).get("items", [])} - ) - - -@dataclass(frozen=True) -class NSMount: - server_ip: str - local_mount: Path - remote_mount: Path - fs_type: str - mount_options: str - -@lru_cache(maxsize=1) -def default_credentials(): - return google.auth.default()[0] - - -@lru_cache(maxsize=1) -def authentication_project(): - return google.auth.default()[1] - - -DEFAULT_UNIVERSE_DOMAIN = "googleapis.com" - - -def now() -> datetime: - """ - Return current time as timezone-aware datetime. - - IMPORTANT: DO NOT use `datetime.now()`, unless you explicitly need to have tz-naive datetime. - Otherwise there is a risk of getting: "cannot compare naive and aware datetimes" error, - since all timetstamps we receive from GCP API are tz-aware. - - Another motivation for this function is to allow to mock time in tests. - """ - return datetime.now(timezone.utc) - -def parse_gcp_timestamp(s: str) -> datetime: - """ - Parse timestamp strings returned by GCP API into datetime. - Works with both Zulu and non-Zulu timestamps. - NOTE: It always return tz-aware datetime (fallbacks to UTC and logs error). - """ - # Requires Python >= 3.7 - # TODO: Remove this "hack" of trimming the Z from timestamps once we move to Python 3.11 - # (context: https://discuss.python.org/t/parse-z-timezone-suffix-in-datetime/2220/30) - ts = datetime.fromisoformat(s.replace('Z', '+00:00')) - if ts.tzinfo is None: # fallback to UTC - log.error(f"Received timestamp without timezone info: {s}") - ts = ts.replace(tzinfo=timezone.utc) - return ts - - -def universe_domain() -> str: - try: - return instance_metadata("attributes/universe_domain") - except MetadataNotFoundError: - return DEFAULT_UNIVERSE_DOMAIN - - -def endpoint_version(api: ApiEndpoint) -> Optional[str]: - return lookup().endpoint_versions.get(api.value, None) - - -@lru_cache(maxsize=1) -def get_credentials() -> Optional[service_account.Credentials]: - """Get credentials for service account""" - key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") - if key_path is not None: - credentials = service_account.Credentials.from_service_account_file( - key_path, scopes=[f"https://www.{universe_domain()}/auth/cloud-platform"] - ) - else: - credentials = default_credentials() - - return credentials - - -@lru_cache(maxsize=1) -def get_dev_key() -> Optional[str]: - """Get dev key for project (uses json or yaml format)""" - try: - with open("/etc/slurm/slurm_vars.yaml", 'r') as file: - data = yaml.safe_load(file) - return data['google_developer_key'] - except: - return None - - -def create_client_options(api: ApiEndpoint) -> ClientOptions: - """Create client options for cloud endpoints""" - ver = endpoint_version(api) - ud = universe_domain() - options = {} - if ud and ud != DEFAULT_UNIVERSE_DOMAIN: - options["universe_domain"] = ud - if ver: - options["api_endpoint"] = f"https://{api.value}.{ud}/{ver}/" - co = ClientOptions(**options) - log.debug(f"Using ClientOptions = {co} for API: {api.value}") - return co - -log = logging.getLogger() - - -def access_secret_version(project_id, secret_id, version_id="latest"): - """ - Access the payload for the given secret version if one exists. The version - can be a version number as a string (e.g. "5") or an alias (e.g. "latest"). - """ - co = create_client_options(ApiEndpoint.SECRET) - client = secretmanager.SecretManagerServiceClient(client_options=co) - name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" - try: - response = client.access_secret_version(request={"name": name}) - log.debug(f"Secret '{name}' was found.") - payload = response.payload.data.decode("UTF-8") - except gExceptions.NotFound: - log.debug(f"Secret '{name}' was not found!") - payload = None - - return payload - - -def parse_self_link(self_link: str): - """Parse a selfLink url, extracting all useful values - https://.../v1/projects//regions//... - {'project': , 'region': , ...} - can also extract zone, instance (name), image, etc - """ - link_patt = re.compile(r"(?P[^\/\s]+)s\/(?P[^\s\/]+)") - return NSDict(link_patt.findall(self_link)) - - -def parse_bucket_uri(uri: str): - """ - Parse a bucket url - E.g. gs:/// - """ - pattern = re.compile(r"gs://(?P[^/\s]+)/(?P([^/\s]+)(/[^/\s]+)*)") - matches = pattern.match(uri) - assert matches, f"Unexpected bucker URI: '{uri}'" - return matches.group("bucket"), matches.group("path") - - -def get_template_gpu(template): - """get gpu info from machine type or guest accelerators""" - gpu_keyword = "nvidia" - gpu = None - if template.machine_type.accelerators: - tma = template.machine_type.accelerators[0] - if gpu_keyword in tma.type.lower(): - gpu = tma - elif template.guestAccelerators: - tga = template.guestAccelerators[0] - if gpu_keyword in tga.acceleratorType.lower(): - gpu = AcceleratorInfo( - type=tga.acceleratorType, - count=tga.acceleratorCount) - return gpu - - -def trim_self_link(link: str): - """get resource name from self link url, eg. - https://.../v1/projects//regions/ - -> - """ - try: - return link[link.rindex("/") + 1 :] - except ValueError: - raise Exception(f"'/' not found, not a self link: '{link}' ") - - -def get_self_link_component(link: str, component_name: str): - """ - Extracts a component (e.g., 'region', 'project') from a self-link URL. - Args: - link: The self-link URL string. - component_name: The name of the component to extract (e.g., 'regions', 'projects'). - Returns: - The extracted component value (e.g., '', ''), - or None if the component is not found in the link. - """ - search_string = f"/{component_name}/" - start_index = link.rfind(search_string) - - if start_index == -1: - return None - - start_index += len(search_string) - end_index = link.find("/", start_index) - - if end_index == -1: - # If no further slash, the rest of the string is the component - return link[start_index:] - else: - return link[start_index:end_index] - - -def execute_with_futures(func, seq): - with ThreadPoolExecutor() as exe: - futures = [] - for i in seq: - future = exe.submit(func, i) - futures.append(future) - for future in as_completed(futures): - result = future.exception() - if result is not None: - raise result - - -def map_with_futures(func, seq): - with ThreadPoolExecutor() as exe: - futures = [] - for i in seq: - future = exe.submit(func, i) - futures.append(future) - for future in futures: - # Will be result or raise Exception - res = None - try: - res = future.result() - except Exception as e: - res = e - yield res - -def should_mount_slurm_bucket() -> bool: - try: - return instance_metadata("attributes/slurm_bucket_mount", silent=True).lower() == "true" - except MetadataNotFoundError: - return False - - -def _get_bucket_and_common_prefix() -> Tuple[str, str]: - uri = instance_metadata("attributes/slurm_bucket_path") - return parse_bucket_uri(uri) - -def blob_get(file): - bucket_name, path = _get_bucket_and_common_prefix() - blob_name = f"{path}/{file}" - return storage_client().get_bucket(bucket_name).blob(blob_name) - - -def blob_list(prefix="", delimiter=None): - bucket_name, path = _get_bucket_and_common_prefix() - blob_prefix = f"{path}/{prefix}" - # Note: The call returns a response only when the iterator is consumed. - blobs = storage_client().list_blobs( - bucket_name, prefix=blob_prefix, delimiter=delimiter - ) - return [blob for blob in blobs] - -def file_list(prefix="", subpath="") -> List[os.DirEntry]: - path = dirs.slurm_bucket_mount - file_prefix = f"{path}/{subpath}" - try: - files = os.scandir(file_prefix) - return [file for file in files if file.name.startswith(prefix)] - except: - return [] - # Not considering lack of file's existence as fatal (we may check for files we know don't exist). - # Responsibility of callee to determine if it is fatal or not, blob_list returns empty iterator in similar cases. - -def hash_file(fullpath: Path) -> str: - with open(fullpath, "rb") as f: - file_hash = hashlib.md5() - chunk = f.read(8192) - while chunk: - file_hash.update(chunk) - chunk = f.read(8192) - return base64.b64encode(file_hash.digest()).decode("utf-8") - - -def install_custom_scripts(check_hash:bool=False): - """download custom scripts from gcs bucket""" - role, tokens = lookup().instance_role, [] - - mounted_scripts=False - if should_mount_slurm_bucket() and role != "controller": - mounted_scripts=True - - all_prolog_tokens = ["prolog", "epilog", "task_prolog", "task_epilog"] - if role == "controller": - tokens = ["controller"] + all_prolog_tokens - elif role == "compute": - tokens = [f"nodeset-{lookup().node_nodeset_name()}"] + all_prolog_tokens - elif role == "login": - tokens = [f"login-{instance_login_group()}"] - - prefixes = [f"slurm-{tok}-script" for tok in tokens] - - # TODO: use single `blob_list`, to reduce ~4x number of GCS requests - if mounted_scripts: - source_collection = list(chain.from_iterable(file_list(prefix=p) for p in prefixes)) - else: - source_collection = list(chain.from_iterable(blob_list(prefix=p) for p in prefixes)) - - script_pattern = re.compile(r"^slurm-(?P\S+)-script-(?P\S+)") - for source in source_collection: - if mounted_scripts: - m = script_pattern.match(source.name) - else: - m = script_pattern.match(Path(source.name).name) - - if not m: - log.warning(f"found blob that doesn't match expected pattern: {source.name}") - continue - path_parts = m["path"].split("-") - path_parts[0] += ".d" - stem, _, ext = m["name"].rpartition("_") - filename = ".".join((stem, ext)) - - path = Path(*path_parts, filename) - fullpath = (dirs.custom_scripts / path).resolve() - mkdirp(fullpath.parent) - - for par in path.parents: - chown_slurm(dirs.custom_scripts / par) - need_update = True - - if check_hash and fullpath.exists() and isinstance(source,storage.Blob): - # TODO: MD5 reported by gcloud may differ from the one calculated here (e.g. if blob got gzipped), - # consider using gCRC32C - need_update = hash_file(fullpath) != source.md5_hash - - log.info(f"installing custom script: {path} from {source.name}") - - if isinstance(source,os.DirEntry): - shutil.copy(source.path, fullpath) #Needs to be copied since mounted nfs is read-only - chown_slurm(fullpath, mode=0o755) - - elif need_update: - with fullpath.open("wb") as f: - source.download_to_file(f) - chown_slurm(fullpath, mode=0o755) - -def compute_service(version="beta"): - """Make thread-safe compute service handle - creates a new Http for each request - """ - credentials = get_credentials() - dev_key = get_dev_key() - - def build_request(http, *args, **kwargs): - new_http = set_user_agent(httplib2.Http(), USER_AGENT) - if credentials is not None: - new_http = google_auth_httplib2.AuthorizedHttp(credentials, http=new_http) - return googleapiclient.http.HttpRequest(new_http, *args, **kwargs) - - ver = endpoint_version(ApiEndpoint.COMPUTE) - disc_url = googleapiclient.discovery.DISCOVERY_URI - if ver: - version = ver - disc_url = disc_url.replace(DEFAULT_UNIVERSE_DOMAIN, universe_domain()) - - log.debug(f"Using version={version} of Google Compute Engine API") - return googleapiclient.discovery.build( - "compute", - version, - requestBuilder=build_request, - credentials=credentials, - developerKey=dev_key, - discoveryServiceUrl=disc_url, - cache_discovery=False, # See https://github.com/googleapis/google-api-python-client/issues/299 - ) - -def storage_client() -> storage.Client: - """ - Config-independent storage client - """ - ud = universe_domain() - co = {} - if ud and ud != DEFAULT_UNIVERSE_DOMAIN: - co["universe_domain"] = ud - return storage.Client(client_options=ClientOptions(**co)) - - -class DeffetiveStoredConfigError(Exception): - """ - Raised when config can not be loaded and assembled from bucket - """ - pass - - -def _fill_cfg_defaults(cfg: NSDict) -> NSDict: - if not cfg.slurm_log_dir: - cfg.slurm_log_dir = dirs.log - if not cfg.slurm_bin_dir: - cfg.slurm_bin_dir = slurmdirs.prefix / "bin" - if not cfg.slurm_control_host: - try: - control_dns_name = instance_metadata("attributes/slurm_control_dns", silent=True) - cfg.slurm_control_host = control_dns_name - except MetadataNotFoundError: - cfg.slurm_control_host = f"{cfg.slurm_cluster_name}-controller" - if not cfg.slurm_control_host_port: - cfg.slurm_control_host_port = "6820-6830" - return cfg - -@dataclass -class _ConfigBlobs: - """ - "Private" class that represent a collection of GCS blobs for configuration - """ - core: storage.Blob - controller_addr: Optional[storage.Blob] - partition: List[storage.Blob] = field(default_factory=list) - nodeset: List[storage.Blob] = field(default_factory=list) - nodeset_dyn: List[storage.Blob] = field(default_factory=list) - nodeset_tpu: List[storage.Blob] = field(default_factory=list) - login_group: List[storage.Blob] = field(default_factory=list) - - @property - def hash(self) -> str: - h = hashlib.md5() - all = [self.core] + self.partition + self.nodeset + self.nodeset_dyn + self.nodeset_tpu - if self.controller_addr: - all.append(self.controller_addr) - - # sort blobs so hash is consistent - for blob in sorted(all, key=lambda b: b.name): - h.update(blob.md5_hash.encode("utf-8")) - return h.hexdigest() - -@dataclass -class _ConfigFiles: - """ - "Private" class that represent a collection of files for configuration - """ - core: Path - controller_addr: Optional[Path] - partition: List[Path] = field(default_factory=list) - nodeset: List[Path] = field(default_factory=list) - nodeset_dyn: List[Path] = field(default_factory=list) - nodeset_tpu: List[Path] = field(default_factory=list) - login_group: List[Path] = field(default_factory=list) - -def _list_config_blobs() -> _ConfigBlobs: - _, common_prefix = _get_bucket_and_common_prefix() - - core: Optional[storage.Blob] = None - controller_addr: Optional[storage.Blob] = None - rest: Dict[str, List[storage.Blob]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} - - is_controller = instance_role() == "controller" - - for blob in blob_list(prefix=""): - if blob.name == f"{common_prefix}/config.yaml": - core = blob - if blob.name == f"{common_prefix}/controller_addr.yaml" and not is_controller: - # Don't add this config blobs for controller to avoid "double reconfiguration": - # Initially this file doesn't exist and produce later by `setup_controller`; - # Appearance of this blob would trigger change in combined hash of config files; - # Ignore existence of this file for controller, assume that - # no other instance nodes will proceed with configuration until this file is created. - controller_addr = blob - for key in rest.keys(): - if blob.name.startswith(f"{common_prefix}/{key}_configs/"): - rest[key].append(blob) - - if core is None: - raise DeffetiveStoredConfigError(f"{common_prefix}/config.yaml not found in bucket") - - return _ConfigBlobs(core=core, controller_addr=controller_addr, **rest) - -def _list_config_files() -> _ConfigFiles: - file_dir = dirs.slurm_bucket_mount - core: Optional[Path] = None - controller_addr: Optional[Path] = None - rest: Dict[str, List[Path]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} - - if Path(f"{file_dir}/config.yaml").exists(): - core = Path(f"{file_dir}/config.yaml") - - for key in rest.keys(): - for f in file_list(subpath=f"{key}_configs"): - rest[key].append(f.path) - - if core is None: - raise Exception(f"config.yaml was not found in mounted folder: {dirs.slurm_bucket_mount}") #Intentionally not using DeffetiveStoredConfigError as this is considered a fatal error - - return _ConfigFiles(core=core, controller_addr=None, **rest) - -def _fetch_config(old_hash: Optional[str]) -> Optional[Tuple[NSDict, str]]: - """Fetch config from bucket, returns None if no changes are detected.""" - blobs = _list_config_blobs() - if old_hash == blobs.hash: - return None - - def _download(bs) -> List[Any]: - return [yaml.safe_load(b.download_as_text()) for b in bs] - - return _assemble_config( - core=_download([blobs.core])[0], - controller_addr=_download([blobs.controller_addr])[0] if blobs.controller_addr else None, - partitions=_download(blobs.partition), - nodesets=_download(blobs.nodeset), - nodesets_dyn=_download(blobs.nodeset_dyn), - nodesets_tpu=_download(blobs.nodeset_tpu), - login_groups=_download(blobs.login_group), - ), blobs.hash - -def _fetch_mounted_config() -> Optional[Tuple[NSDict, str]]: - if not dirs.slurm_bucket_mount.is_mount(): - raise Exception(f"{dirs.slurm_bucket_mount} is not mounted") - - files = _list_config_files() - - def _load(files) -> List[Any]: - file_yaml=[] - for file in files: - with open(file, "r") as f: - file_yaml.append(yaml.safe_load(f)) - return file_yaml - - return _assemble_config( - core=_load([files.core])[0], - controller_addr=None, - partitions=_load(files.partition), - nodesets=_load(files.nodeset), - nodesets_dyn=_load(files.nodeset_dyn), - nodesets_tpu=_load(files.nodeset_tpu), - login_groups=_load(files.login_group), - ) - -def controller_lookup_self_ip() -> str: - assert instance_role() == "controller" - # Get IP of LAST network-interface - # TODO: Consider change order of NICs definition, so right NIC is always @0. - idx = instance_metadata("network-interfaces").split()[-1] # either `0/` or `1/` - return instance_metadata(f"network-interfaces/{idx}ip") - -def _assemble_config( - core: Any, - controller_addr: Optional[Any], - partitions: List[Any], - nodesets: List[Any], - nodesets_dyn: List[Any], - nodesets_tpu: List[Any], - login_groups: List[Any], - ) -> NSDict: - cfg = NSDict(core) - - if cfg.controller_network_attachment: - # lookup controller address - if instance_role() == "controller": - # ignore stored value of `controller_addr`, it will be overwritten during `setup_controller` - cfg.slurm_control_addr = controller_lookup_self_ip() - else: - if not controller_addr: - raise DeffetiveStoredConfigError("controller_addr.yaml not found in bucket") - cfg.slurm_control_addr = controller_addr["slurm_control_addr"] - - # add partition configs - for p_yaml in partitions: - p_cfg = NSDict(p_yaml) - assert p_cfg.get("partition_name"), "partition_name is required" - p_name = p_cfg.partition_name - assert p_name not in cfg.partitions, f"partition {p_name} already defined" - cfg.partitions[p_name] = p_cfg - - # add nodeset configs - ns_names = set() - def _add_nodesets(yamls: List[Any], target: dict): - for ns_yaml in yamls: - ns_cfg = NSDict(ns_yaml) - assert ns_cfg.get("nodeset_name"), "nodeset_name is required" - ns_name = ns_cfg.nodeset_name - assert ns_name not in ns_names, f"nodeset {ns_name} already defined" - target[ns_name] = ns_cfg - ns_names.add(ns_name) - - _add_nodesets(nodesets, cfg.nodeset) - _add_nodesets(nodesets_dyn, cfg.nodeset_dyn) - _add_nodesets(nodesets_tpu, cfg.nodeset_tpu) - - # validate that configs for all referenced nodesets are present - for p in cfg.partitions.values(): - for ns_name in chain(p.partition_nodeset, p.partition_nodeset_dyn, p.partition_nodeset_tpu): - if ns_name not in ns_names: - raise DeffetiveStoredConfigError(f"nodeset {ns_name} not defined in config") - - for lg_yaml in login_groups: - lg_cfg = NSDict(lg_yaml) - assert lg_cfg.get("group_name"), "group_name is required" - lg_name = lg_cfg.group_name - assert lg_name not in cfg.login_groups - cfg.login_groups[lg_name] = lg_cfg - - if instance_role() == "login": - group = instance_login_group() - if group not in cfg.login_groups: - raise DeffetiveStoredConfigError(f"login group '{group}' does not exist in config") - - return _fill_cfg_defaults(cfg) - -def fetch_config() -> Tuple[bool, NSDict]: - """ - Fetches config from bucket and saves it locally - Returns True if new (updated) config was fetched - """ - hash_file = Path("/slurm/scripts/.config.hash") - old_hash = hash_file.read_text() if hash_file.exists() else None - - if should_mount_slurm_bucket() and instance_role() != "controller": - cfg = _fetch_mounted_config() - CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) - chown_slurm(CONFIG_FILE) - return False, cfg - - cfg_and_hash = _fetch_config(old_hash=old_hash) - - if not cfg_and_hash: - return False, _load_config() - - cfg, hash = cfg_and_hash - hash_file.write_text(hash) - chown_slurm(hash_file) - CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) - chown_slurm(CONFIG_FILE) - return True, cfg - -def owned_file_handler(filename): - """create file handler""" - chown_slurm(filename) - return logging.handlers.WatchedFileHandler(filename, delay=True) - -def get_log_path() -> Path: - """ - Returns path to log file for the current script. - e.g. resume.py -> /var/log/slurm/resume.log - """ - cfg_log_dir = lookup().cfg.slurm_log_dir - log_dir = Path(cfg_log_dir) if cfg_log_dir else dirs.log - return (log_dir / Path(sys.argv[0]).name).with_suffix(".log") - -def init_log_and_parse(parser: argparse.ArgumentParser) -> argparse.Namespace: - parser.add_argument( - "--debug", - "-d", - dest="loglevel", - action="store_const", - const=logging.DEBUG, - default=logging.INFO, - help="Enable debugging output", - ) - parser.add_argument( - "--trace-api", - "-t", - action="store_true", - help="Enable detailed api request output", - ) - args = parser.parse_args() - loglevel = args.loglevel - if lookup().cfg.enable_debug_logging: - loglevel = logging.DEBUG - if args.trace_api: - lookup().cfg.extra_logging_flags["trace_api"] = True - # Configure root logger - logging.config.dictConfig({ - "version": 1, - "disable_existing_loggers": True, - "formatters": { - "standard": { - "format": "%(levelname)s: %(message)s", - }, - "stamp": { - "format": "%(asctime)s %(levelname)s: %(message)s", - }, - }, - "handlers": { - "stdout_handler": { - "level": logging.DEBUG, - "formatter": "standard", - "class": "logging.StreamHandler", - "stream": sys.stdout, - }, - "file_handler": { - "()": owned_file_handler, - "level": logging.DEBUG, - "formatter": "stamp", - "filename": get_log_path(), - }, - }, - "root": { - "handlers": ["stdout_handler", "file_handler"], - "level": loglevel, - }, - }) - - sys.excepthook = _handle_exception - - return args - - -def log_api_request(request): - """log.trace info about a compute API request""" - if not lookup().cfg.extra_logging_flags.get("trace_api"): - return - # output the whole request object as pretty yaml - # the body is nested json, so load it as well - rep = json.loads(request.to_json()) - if rep.get("body", None) is not None: - rep["body"] = json.loads(rep["body"]) - pretty_req = yaml.safe_dump(rep).rstrip() - # label log message with the calling function - log.debug(f"{inspect.stack()[1].function}:\n{pretty_req}") - - -def _handle_exception(exc_type, exc_value, exc_trace): - """log exceptions other than KeyboardInterrupt""" - if not issubclass(exc_type, KeyboardInterrupt): - log.exception("Fatal exception", exc_info=(exc_type, exc_value, exc_trace)) - sys.__excepthook__(exc_type, exc_value, exc_trace) - - -def run( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False, - timeout=None, - check=True, - universal_newlines=True, - **kwargs, -): - """Wrapper for subprocess.run() with convenient defaults""" - if isinstance(args, list): - args = list(filter(lambda x: x is not None, args)) - args = " ".join(args) - if not shell and isinstance(args, str): - args = shlex.split(args) - log.debug(f"run: {args}") - try: - result = subprocess.run( - args, - stdout=stdout, - stderr=stderr, - shell=shell, - timeout=timeout, - check=check, - universal_newlines=universal_newlines, - **kwargs, - ) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - log_subprocess(e) - raise - log_subprocess(result) - return result - -def log_subprocess(subj: subprocess.CalledProcessError | subprocess.TimeoutExpired | subprocess.CompletedProcess) -> None: - match subj: - case subprocess.CompletedProcess(returncode=0): - # Do not log successful runs, to not overwhelm logs (e.g. scontrol show jobs --json) - # TODO: consider still doing it in DEBUG or trim output to few KBs. - return - case subprocess.CompletedProcess(): # non-zero returncode - log.error(f"Command '{subj.args}' returned exit status {subj.returncode}.") - case subprocess.CalledProcessError() | subprocess.TimeoutExpired(): - log.error(str(subj)) - - - def normalize(out: None | str | bytes) -> None | str: - """ - Turns stderr and stdout into string: - > A bytes sequence, or a string if run() was called with an encoding, errors, or text=True. None if was not captured. - """ - match out: - case None: - return None - case str(): - return out.strip() - case bytes(): - return out.decode().strip() - case _: - return repr(out) - - if stdout := normalize(subj.stdout): - log.error(f"stdout: {stdout}") - if stderr := normalize(subj.stderr): - log.error(f"stderr: {stderr}") - - -def chown_slurm(path: Path, mode=None) -> None: - if path.exists(): - if mode: - path.chmod(mode) - else: - mkdirp(path.parent) - if mode: - path.touch(mode=mode) - else: - path.touch() - try: - shutil.chown(path, user="slurm", group="slurm") - except LookupError: - log.warning(f"User 'slurm' does not exist. Cannot 'chown slurm:slurm {path}'.") - except PermissionError: - log.warning(f"Not authorized to 'chown slurm:slurm {path}'.") - except Exception as err: - log.error(err) - - -@contextmanager -def cd(path): - """Change working directory for context""" - prev = Path.cwd() - os.chdir(path) - try: - yield - finally: - os.chdir(prev) - - -def cached_property(f): - return property(lru_cache()(f)) - - -def retry(max_retries: int, init_wait_time: float, warn_msg: str, exc_type: Type[Exception]): - """Retries functions that raises the exception exc_type. - Retry time is increased by a factor of two for every iteration. - - Args: - max_retries (int): Maximum number of retries - init_wait_time (float): Initial wait time in secs - warn_msg (str): Message to print during retries - exc_type (Exception): Exception type to check for - """ - - if max_retries <= 0: - raise ValueError("Incorrect value for max_retries, must be >= 1") - if init_wait_time <= 0.0: - raise ValueError("Invalid value for init_wait_time, must be > 0.0") - - def decorator(f): - @wraps(f) - def wrapper(*args, **kwargs): - retry = 0 - secs = init_wait_time - captured_exc: Optional[BaseException] = None - while retry < max_retries: - try: - return f(*args, **kwargs) - except exc_type as e: - captured_exc = e - log.warn(f"{warn_msg}, retrying in {secs}") - sleep(secs) - retry += 1 - secs *= 2 - assert captured_exc - raise captured_exc - - return wrapper - - return decorator - - -def separate(pred: Callable[[Any], bool], coll: Iterable[Any]) -> Tuple[List[Any], List[Any]]: - """filter into 2 lists based on pred returning True or False - returns ([False], [True]) - """ - res: Tuple[List[Any], List[Any]] = ([],[]) - for el in coll: - res[pred(el)].append(el) - return res - - -def chunked(iterable, n=API_REQ_LIMIT): - """group iterator into chunks of max size n""" - it = iter(iterable) - while True: - chunk = list(islice(it, n)) - if not chunk: - return - yield chunk - -def groupby_unsorted(seq: Sequence[Any], key): - indices = defaultdict(list) - for i, el in enumerate(seq): - indices[key(el)].append(i) - for k, idxs in indices.items(): - yield k, (seq[i] for i in idxs) - - -@lru_cache(maxsize=32) -def find_ratio(a, n, s, r0=None): - """given the start (a), count (n), and sum (s), find the ratio required""" - if n == 2: - return s / a - 1 - an = a * n - if n == 1 or s == an: - return 1 - if r0 is None: - # we only need to know which side of 1 to guess, and the iteration will work - r0 = 1.1 if an < s else 0.9 - - # geometric sum formula - def f(r): - return a * (1 - r**n) / (1 - r) - s - - # derivative of f - def df(r): - rm1 = r - 1 - rn = r**n - return (a * (rn * (n * rm1 - r) + r)) / (r * rm1**2) - - MIN_DR = 0.0001 # negligible change - r = r0 - # print(f"r(0)={r0}") - MAX_TRIES = 64 - for i in range(1, MAX_TRIES + 1): - try: - dr = f(r) / df(r) - except ZeroDivisionError: - log.error(f"Failed to find ratio due to zero division! Returning r={r0}") - return r0 - r = r - dr - # print(f"r({i})={r}") - # if the change in r is small, we are close enough - if abs(dr) < MIN_DR: - break - else: - log.error(f"Could not find ratio after {MAX_TRIES}! Returning r={r0}") - return r0 - return r - - -def backoff_delay(start, timeout=None, ratio=None, count: int = 0): - """generates `count` waits starting at `start` - sum of waits is `timeout` or each one is `ratio` bigger than the last - the last wait is always 0""" - # timeout or ratio must be set but not both - assert (timeout is None) ^ (ratio is None) - assert ratio is None or ratio > 0 - assert timeout is None or timeout >= start - assert (count > 1 or timeout is not None) and isinstance(count, int) - assert start > 0 - - if count == 0: - # Equation for auto-count is tuned to have a max of - # ~int(timeout) counts with a start wait of <0.01. - # Increasing start wait decreases count eg. - # backoff_delay(10, timeout=60) -> count = 5 - count = int( - (timeout / ((start + 0.05) ** (1 / 2)) + 2) // math.log(timeout + 2) - ) - - yield start - # if ratio is set: - # timeout = start * (1 - ratio**(count - 1)) / (1 - ratio) - if ratio is None: - ratio = find_ratio(start, count - 1, timeout) - - wait = start - # we have start and 0, so we only need to generate count - 2 - for _ in range(count - 2): - wait *= ratio - yield wait - yield 0 - return - - -ROOT_URL = "http://metadata.google.internal/computeMetadata/v1" - -class MetadataNotFoundError(Exception): - pass - -def get_metadata(path:str, silent=False) -> str: - """Get metadata relative to metadata/computeMetadata/v1""" - HEADERS = {"Metadata-Flavor": "Google"} - url = f"{ROOT_URL}/{path}" - try: - resp = requests_lib.get(url, headers=HEADERS) - resp.raise_for_status() - return resp.text - except requests_lib.exceptions.HTTPError: - if not silent: - log.warning(f"metadata not found ({url})") - raise MetadataNotFoundError(f"failed to get_metadata from {url}") - - -@lru_cache(maxsize=None) -def instance_metadata(path: str, silent:bool=False) -> str: - return get_metadata(f"instance/{path}", silent=silent) - -def instance_role(): - return instance_metadata("attributes/slurm_instance_role") - - -def instance_login_group(): - return instance_metadata("attributes/slurm_login_group") - - -def natural_sort(text): - def atoi(text): - return int(text) if text.isdigit() else text - - return [atoi(w) for w in re.split(r"(\d+)", text)] - - -def to_hostlist(names: Iterable[str]) -> str: - """ - Fast implementation of `hostlist` that doesn't invoke `scontrol` - IMPORTANT: - * Acts as `scontrol show hostlistsorted`, i.e. original order is not preserved - * Achieves worse compression than `scontrol show hostlist` for some cases - """ - pref = defaultdict(list) - tokenizer = re.compile(r"^(.*?)(\d*)$") - for name in filter(None, names): - matches = tokenizer.match(name) - assert matches, name - p, s = matches.groups() - pref[p].append(s) - - def _compress_suffixes(ss: List[str]) -> List[str]: - cur, res = None, [] - - def cur_repr(): - assert cur - nums, strs = cur - if nums[0] == nums[1]: - return strs[0] - return f"{strs[0]}-{strs[1]}" - - for s in sorted(ss, key=int): - n = int(s) - if cur is None: - cur = ((n, n), (s, s)) - continue - - nums, strs = cur - if n == nums[1] + 1: - cur = ((nums[0], n), (strs[0], s)) - else: - res.append(cur_repr()) - cur = ((n, n), (s, s)) - if cur: - res.append(cur_repr()) - return res - - res = [] - for p in sorted(pref.keys()): - sl = defaultdict(list) - for s in pref[p]: - sl[len(s)].append(s) - cs = [] - for ln in sorted(sl.keys()): - if ln == 0: - res.append(p) - else: - cs.extend(_compress_suffixes(sl[ln])) - if not cs: - continue - if len(cs) == 1 and "-" not in cs[0]: - res.append(f"{p}{cs[0]}") - else: - res.append(f"{p}[{','.join(cs)}]") - return ",".join(res) - -@lru_cache(maxsize=None) -def to_hostnames(nodelist: str) -> List[str]: - """make list of hostnames from hostlist expression""" - if not nodelist: - return [] # avoid degenerate invocation of scontrol - if isinstance(nodelist, str): - hostlist = nodelist - else: - hostlist = ",".join(nodelist) - hostnames = run(f"{lookup().scontrol} show hostnames {hostlist}").stdout.splitlines() - return hostnames - - -def retry_exception(exc) -> bool: - """return true for exceptions that should always be retried""" - msg = str(exc) - retry_errors = ( - "Rate Limit Exceeded", - "Quota Exceeded", - "Quota exceeded", - ) - return any(err in msg for err in retry_errors) - - -def ensure_execute(request): - """Handle rate limits and socket time outs""" - - for retry, wait in enumerate(backoff_delay(0.5, timeout=10 * 60, count=20)): - try: - return request.execute() - except googleapiclient.errors.HttpError as e: - if retry_exception(e): - log.error(f"retry:{retry} '{e}'") - sleep(wait) - continue - raise - - except socket.timeout as e: - # socket timed out, try again - log.debug(e) - - except Exception as e: - log.error(e, exc_info=True) - raise - - break - - -def batch_execute(requests, retry_cb=None, log_err=log.error): - """execute list or dict as batch requests - retry if retry_cb returns true - """ - BATCH_LIMIT = 1000 - if not isinstance(requests, dict): - requests = {str(k): v for k, v in enumerate(requests)} # rid generated here - done = {} - failed = {} - timestamps: List[float] = [] - rate_limited = False - - def batch_callback(rid, resp, exc): - nonlocal rate_limited - if exc is not None: - log_err(f"compute request exception {rid}: {exc}") - if retry_exception(exc): - rate_limited = True - else: - req = requests.pop(rid) - failed[rid] = (req, exc) - else: - # if retry_cb is set, don't move to done until it returns false - if retry_cb is None or not retry_cb(resp): - requests.pop(rid) - done[rid] = resp - - def batch_request(reqs): - batch = lookup().compute.new_batch_http_request(callback=batch_callback) - for rid, req in reqs: - batch.add(req, request_id=rid) - return batch - - while requests: - if timestamps: - timestamps = [stamp for stamp in timestamps if stamp > time()] - if rate_limited and timestamps: - stamp = next(iter(timestamps)) - sleep(max(stamp - time(), 0)) - rate_limited = False - # up to API_REQ_LIMIT (2000) requests - # in chunks of up to BATCH_LIMIT (1000) - batches = [ - batch_request(chunk) - for chunk in chunked(islice(requests.items(), API_REQ_LIMIT), BATCH_LIMIT) - ] - timestamps.append(time() + 100) - with ThreadPoolExecutor() as exe: - futures = [] - for batch in batches: - future = exe.submit(ensure_execute, batch) - futures.append(future) - for future in futures: - result = future.exception() - if result is not None: - raise result - - return done, failed - - -def get_operation_req(lkp: "Lookup", name: str, region: Optional[str]=None, zone: Optional[str]=None) -> Any: - if zone: - return lkp.compute.zoneOperations().get(project=lkp.project, zone=zone, operation=name) - elif region: - return lkp.compute.regionOperations().get(project=lkp.project, region=region, operation=name) - return lkp.compute.globalOperations().get(project=lkp.project, operation=name) - -def wait_request(operation, project: str): - """makes the appropriate wait request for a given operation""" - if "zone" in operation: - req = lookup().compute.zoneOperations().wait( - project=project, - zone=trim_self_link(operation["zone"]), - operation=operation["name"], - ) - elif "region" in operation: - req = lookup().compute.regionOperations().wait( - project=project, - region=trim_self_link(operation["region"]), - operation=operation["name"], - ) - else: - req = lookup().compute.globalOperations().wait( - project=project, operation=operation["name"] - ) - return req - - -def wait_for_operation(operation) -> Dict[str, Any]: - """wait for given operation""" - project = parse_self_link(operation["selfLink"]).project - wait_req = wait_request(operation, project=project) - - while True: - result = ensure_execute(wait_req) - if result["status"] == "DONE": - log_errors = " with errors" if "error" in result else "" - log.debug( - f"operation complete{log_errors}: type={result['operationType']}, name={result['name']}" - ) - return result - - - -def getThreadsPerCore(template) -> int: - if not template.machine_type.supports_smt: - return 1 - return template.advancedMachineFeatures.threadsPerCore or 2 - - -@retry( - max_retries=9, - init_wait_time=1, - warn_msg="Temporary failure in name resolution", - exc_type=socket.gaierror, -) -def host_lookup(host_name: str) -> str: - return socket.gethostbyname(host_name) - - -class Dumper(yaml.SafeDumper): - """Add representers for pathlib.Path and NSDict for yaml serialization""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.add_representer(NSDict, self.represent_nsdict) - self.add_multi_representer(Path, self.represent_path) - - @staticmethod - def represent_nsdict(dumper, data): - return dumper.represent_mapping("tag:yaml.org,2002:map", data.items()) - - @staticmethod - def represent_path(dumper, path): - return dumper.represent_scalar("tag:yaml.org,2002:str", str(path)) - - -@dataclass(frozen=True) -class ReservationDetails: - project: str - zone: str - name: str - policies: List[str] # names (not URLs) of resource policies - bulk_insert_name: str # name in format suitable for bulk insert (currently identical to user supplied name in long format) - deployment_type: Optional[str] - reservation_mode: Optional[str] - assured_count: int - delete_at_time: Optional[datetime] - - @property - def dense(self) -> bool: - return self.deployment_type == "DENSE" - - @property - def calendar(self) -> bool: - return self.reservation_mode == "CALENDAR" - -@dataclass(frozen=True) -class FutureReservation: - project: str - zone: str - name: str - specific: bool - start_time: datetime - end_time: datetime - reservation_mode: Optional[str] - active_reservation: Optional[ReservationDetails] - - @property - def calendar(self) -> bool: - return self.reservation_mode == "CALENDAR" - -@dataclass -class Job: - id: int - name: Optional[str] = None - required_nodes: Optional[str] = None - job_state: Optional[str] = None - duration: Optional[timedelta] = None - -@dataclass(frozen=True) -class NodeState: - base: str - flags: frozenset - -class Lookup: - """Wrapper class for cached data access""" - - def __init__(self, cfg): - self._cfg = cfg - - @property - def cfg(self): - return self._cfg - - @property - def project(self): - return self.cfg.project or authentication_project() - - @cached_property - def control_addr(self) -> Optional[str]: - return self.cfg.get("slurm_control_addr", None) - - @property - def control_host(self): - return self.cfg.slurm_control_host - - @cached_property - def control_host_addr(self): - return self.control_addr or host_lookup(self.cfg.slurm_control_host) - - @property - def control_host_port(self): - return self.cfg.slurm_control_host_port - - @property - def endpoint_versions(self): - return self.cfg.endpoint_versions - - @property - def scontrol(self): - return Path(self.cfg.slurm_bin_dir or "") / "scontrol" - - @cached_property - def instance_role(self): - return instance_role() - - @cached_property - def instance_role_safe(self): - try: - role = self.instance_role - except Exception as e: - log.error(e) - role = None - return role - - @property - def is_controller(self): - return self.instance_role_safe == "controller" - - @property - def is_login_node(self): - return self.instance_role_safe == "login" - - @cached_property - def compute(self): - # TODO evaluate when we need to use google_app_cred_path - if self.cfg.google_app_cred_path: - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.cfg.google_app_cred_path - return compute_service() - - @cached_property - def hostname(self): - return socket.gethostname() - - @cached_property - def hostname_fqdn(self): - return socket.getfqdn() - - @cached_property - def zone(self): - return instance_metadata("zone") - - node_desc_regex = re.compile( - r"^(?P(?P[^\s\-]+)-(?P\S+))-(?P(?P\w+)|(?P\[[\d,-]+\]))$" - ) - - @lru_cache(maxsize=None) - def _node_desc(self, node_name): - """Get parts from node name""" - if not node_name: - node_name = self.hostname - # workaround below is for VMs whose hostname is FQDN - node_name_short = node_name.split(".")[0] - m = self.node_desc_regex.match(node_name_short) - if not m: - raise Exception(f"node name {node_name} is not valid") - return m.groupdict() - - def node_prefix(self, node_name=None): - return self._node_desc(node_name)["prefix"] - - def node_index(self, node: str) -> int: - """ node_index("cluster-nodeset-45") == 45 """ - suff = self._node_desc(node)["suffix"] - - if suff is None: - raise ValueError(f"Node {node} name does not end with numeric index") - return int(suff) - - def node_nodeset_name(self, node_name=None): - return self._node_desc(node_name)["nodeset"] - - def node_nodeset(self, node_name=None): - nodeset_name = self.node_nodeset_name(node_name) - if nodeset_name in self.cfg.nodeset_tpu: - return self.cfg.nodeset_tpu[nodeset_name] - - return self.cfg.nodeset[nodeset_name] - - def partition_is_tpu(self, part: str) -> bool: - """check if partition with name part contains a nodeset of type tpu""" - return len(self.cfg.partitions[part].partition_nodeset_tpu) > 0 - - - def node_is_tpu(self, node_name=None): - nodeset_name = self.node_nodeset_name(node_name) - return self.cfg.nodeset_tpu.get(nodeset_name) is not None - - def nodeset_is_tpu(self, nodeset_name=None) -> bool: - return self.cfg.nodeset_tpu.get(nodeset_name) is not None - - def node_is_fr(self, node_name:str) -> bool: - return bool(self.node_nodeset(node_name).future_reservation) - - def is_dormant_res_node(self, node_name:str) -> bool: - fr = self.future_reservation(self.node_nodeset(node_name)) - res = self.nodeset_reservation(self.node_nodeset(node_name)) - - if fr is None and res is None: - return False - - if fr: - return fr.active_reservation is None - - if res: - if res.calendar: - # If reservation is calendar based, check if it is past the delete_at_time - if res.delete_at_time is not None and now() >= res.delete_at_time: - log.debug(f"DWS calendar reservation {res.bulk_insert_name} is past deletion time {res.delete_at_time}, skipping resume.") - return True - - # If assured_count is 0 do not resume nodes as they are not active yet - if res.delete_at_time is not None and res.assured_count <= 0: - log.debug(f"DWS calendar reservation {res.bulk_insert_name} is not active yet, skipping resume.") - return True - - return False - - def node_is_dyn(self, node_name=None) -> bool: - nodeset = self.node_nodeset_name(node_name) - return self.cfg.nodeset_dyn.get(nodeset) is not None - - def node_is_gke(self, node_name=None) -> bool: - return self.nodeset_is_gke(self.node_nodeset(node_name)) - - def nodeset_is_gke(self, nodeset=None) -> bool: - return "gke_nodepool" in nodeset - - def node_template(self, node_name=None) -> str: - """ Self link of nodeset template """ - return self.node_nodeset(node_name).instance_template - - def node_template_info(self, node_name=None): - return self.template_info(self.node_template(node_name)) - - def node_region(self, node_name=None): - nodeset = self.node_nodeset(node_name) - return parse_self_link(nodeset.subnetwork).region - - def nodeset_accelerator_topology(self, nodeset_name: str) -> Optional[str]: - if not self.nodeset_is_tpu(nodeset_name): - return getattr(self.cfg.nodeset[nodeset_name], 'accelerator_topology', None) - return None - - def nodeset_prefix(self, nodeset_name): - return f"{self.cfg.slurm_cluster_name}-{nodeset_name}" - - def nodelist_range(self, nodeset_name: str, start: int, count: int) -> str: - assert 0 <= start and 0 < count - pref = self.nodeset_prefix(nodeset_name) - if count == 1: - return f"{pref}-{start}" - return f"{pref}-[{start}-{start + count - 1}]" - - def static_dynamic_sizes(self, nodeset: NSDict) -> Tuple[int, int]: - return (nodeset.node_count_static or 0, nodeset.node_count_dynamic_max or 0) - - def nodelist(self, nodeset) -> str: - cnt = sum(self.static_dynamic_sizes(nodeset)) - if cnt == 0: - return "" - return self.nodelist_range(nodeset.nodeset_name, 0, cnt) - - def nodenames(self, nodeset) -> Tuple[Iterable[str], Iterable[str]]: - pref = self.nodeset_prefix(nodeset.nodeset_name) - s_count, d_count = self.static_dynamic_sizes(nodeset) - return ( - (f"{pref}-{i}" for i in range(s_count)), - (f"{pref}-{i}" for i in range(s_count, s_count + d_count)), - ) - - def power_managed_nodesets(self) -> Iterable[NSDict]: - return chain(self.cfg.nodeset.values(), self.cfg.nodeset_tpu.values()) - - def is_power_managed_node(self, node_name: str) -> bool: - try: - ns = self.node_nodeset(node_name) - if ns is None: - return False - idx = int(self._node_desc(node_name)["suffix"]) - return idx < sum(self.static_dynamic_sizes(ns)) - except Exception: - return False - - def is_static_node(self, node_name: str) -> bool: - if not self.is_power_managed_node(node_name): - return False - idx = int(self._node_desc(node_name)["suffix"]) - return idx < self.node_nodeset(node_name).node_count_static - - @lru_cache(maxsize=None) - def slurm_nodes(self) -> Dict[str, NodeState]: - def parse_line(node_line) -> Tuple[str, NodeState]: - """turn node,state line to (node, NodeState)""" - # state flags include: CLOUD, COMPLETING, DRAIN, FAIL, POWERED_DOWN, - # POWERING_DOWN - node, fullstate = node_line.split(",") - state = fullstate.split("+") - state_tuple = NodeState(base=state[0], flags=frozenset(state[1:])) - return (node, state_tuple) - - cmd = ( - f"{self.scontrol} show nodes | " - r"grep -oP '^NodeName=\K(\S+)|\s+State=\K(\S+)' | " - r"paste -sd',\n'" - ) - node_lines = run(cmd, shell=True).stdout.rstrip().splitlines() - nodes = { - node: state - for node, state in map(parse_line, node_lines) - if "CLOUD" in state.flags or "DYNAMIC_NORM" in state.flags - } - return nodes - - def node_state(self, nodename: str) -> Optional[NodeState]: - state = self.slurm_nodes().get(nodename) - if state is not None: - return state - - # state is None => Slurm doesn't know this node, - # there are two reasons: - # * happy: - # * node belongs to removed nodeset - # * node belongs to downsized portion of nodeset - # * dynamic node that didn't register itself - # * unhappy: - # * there is a drift in Slurm and SlurmGCP configurations - # * `slurm_nodes` function failed to handle `scontrol show nodes`, - # TODO: make `slurm_nodes` robust by using `scontrol show nodes --json` - # In either of "unhappy" cases it's too dangerous to proceed - abort slurmsync. - try: - ns = self.node_nodeset(nodename) - except: - log.info(f"Unknown node {nodename}, belongs to unknown nodeset") - return None # Can't find nodeset, may be belongs to removed nodeset - - if self.node_is_dyn(nodename): - log.info(f"Unknown node {nodename}, belongs to dynamic nodeset") - return None # we can't make any judjment for dynamic nodes - - cnt = sum(self.static_dynamic_sizes(ns)) - if self.node_index(nodename) >= cnt: - log.info(f"Unknown node {nodename}, out of nodeset size boundaries ({cnt})") - return None # node belongs to downsized nodeset - - raise RuntimeError(f"Slurm does not recognize node {nodename}, potential misconfiguration.") - - - @lru_cache(maxsize=1) - def instances(self) -> Dict[str, Instance]: - instance_information_fields = [ - "creationTimestamp", - "name", - "resourceStatus", - "scheduling", - "status", - "labels.slurm_instance_role", - "zone", - "metadata", - ] - - instance_fields = ",".join(sorted(instance_information_fields)) - fields = f"items.zones.instances({instance_fields}),nextPageToken" - flt = f"labels.slurm_cluster_name={self.cfg.slurm_cluster_name} AND name:{self.cfg.slurm_cluster_name}-*" - act = self.compute.instances() - op = act.aggregatedList(project=self.project, fields=fields, filter=flt) - - instances = {} - while op is not None: - result = ensure_execute(op) - for zone in result.get("items", {}).values(): - for jo in zone.get("instances", []): - inst = Instance.from_json(jo) - if inst.name in instances: - log.error(f"Duplicate VM name {inst.name} across multiple zones") - instances[inst.name] = inst - op = act.aggregatedList_next(op, result) - return instances - - def instance(self, instance_name: str) -> Optional[Instance]: - return self.instances().get(instance_name) - - @lru_cache() - def _get_reservation(self, project: str, zone: str, name: str) -> Any: - """See https://cloud.google.com/compute/docs/reference/rest/v1/reservations""" - return self.compute.reservations().get( - project=project, zone=zone, reservation=name).execute() - - @lru_cache() - def get_mig(self, project: str, region: str, self_link:str) -> Any: - """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" - return self.compute.regionInstanceGroupManagers().get(project=project, region=region, instanceGroupManager=self_link).execute() - - @lru_cache - def get_mig_instances(self, project: str, region: str, self_link:str) -> Any: - return self.compute.regionInstanceGroupManagers().listManagedInstances(project=project, region=region, instanceGroupManager=self_link).execute() - - @lru_cache() - def get_mig_list(self, project: str, region: str) -> Any: - """https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers""" - return self.compute.regionInstanceGroupManagers().list(project=project, region=region).execute() - - @lru_cache() - def _get_future_reservation(self, project:str, zone:str, name: str) -> Any: - """See https://cloud.google.com/compute/docs/reference/rest/v1/futureReservations""" - return self.compute.futureReservations().get(project=project, zone=zone, futureReservation=name).execute() - - def get_reservation_details(self, project:str, zone:str, name:str, bulk_insert_name:str) -> ReservationDetails: - reservation = self._get_reservation(project, zone, name) - - # Converts policy URLs to names, e.g.: - # projects/111111/regions/us-central1/resourcePolicies/zebra -> zebra - policies = [u.split("/")[-1] for u in reservation.get("resourcePolicies", {}).values()] - - return ReservationDetails( - project=project, - zone=zone, - name=name, - policies=policies, - deployment_type=reservation.get("deploymentType"), - reservation_mode=reservation.get("reservationMode"), - assured_count=int(reservation.get("specificReservation", {}).get("assuredCount", 0)), - delete_at_time=parse_gcp_timestamp(reservation.get("deleteAtTime")) if reservation.get("deleteAtTime") else None, - bulk_insert_name=bulk_insert_name) - - def nodeset_reservation(self, nodeset: NSDict) -> Optional[ReservationDetails]: - if not nodeset.reservation_name: - return None - - zones = list(nodeset.zone_policy_allow or []) - assert len(zones) == 1, "Only single zone is supported if using a reservation" - zone = zones[0] - - regex = re.compile(r'^projects/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$') - if not (match := regex.match(nodeset.reservation_name)): - raise ValueError( - f"Invalid reservation name: '{nodeset.reservation_name}', expected format is 'projects/PROJECT/reservations/NAME'" - ) - - project, name = match.group("project", "reservation") - return self.get_reservation_details(project, zone, name, nodeset.reservation_name) - - def future_reservation(self, nodeset: NSDict) -> Optional[FutureReservation]: - if not nodeset.future_reservation: - return None - - active_reservation = None - match = re.search(r'^projects/(?P[^/]+)/zones/(?P[^/]+)/futureReservations/(?P[^/]+)(/.*)?$', nodeset.future_reservation) - assert match, f"Invalid future reservation name '{nodeset.future_reservation}'" - project, zone, name = match.group("project","zone","name") - fr = self._get_future_reservation(project,zone,name) - - start_time = parse_gcp_timestamp(fr["timeWindow"]["startTime"]) - end_time = parse_gcp_timestamp(fr["timeWindow"]["endTime"]) - - if "autoCreatedReservations" in fr["status"] and (res:=fr["status"]["autoCreatedReservations"][0]): - if start_time <= now() <=end_time: - match = re.search(r'projects/(?P[^/]+)/zones/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$',res) - assert match, f"Unexpected reservation name '{res}'" - res_name = match.group("name") - bulk_insert_name = f"projects/{project}/reservations/{res_name}" - active_reservation = self.get_reservation_details(project, zone, res_name, bulk_insert_name) - - return FutureReservation( - project=project, - zone=zone, - name=name, - specific=fr["specificReservationRequired"], - start_time=start_time, - end_time=end_time, - reservation_mode=fr.get("reservationMode"), - active_reservation=active_reservation - ) - - @lru_cache(maxsize=1) - def machine_types(self): - field_names = "name,zone,guestCpus,memoryMb,accelerators" - fields = f"items.zones.machineTypes({field_names}),nextPageToken" - - machines: Dict[str, Dict[str, Any]] = defaultdict(dict) - act = self.compute.machineTypes() - op = act.aggregatedList(project=self.project, fields=fields) - while op is not None: - result = ensure_execute(op) - machine_iter = chain.from_iterable( - scope.get("machineTypes", []) for scope in result["items"].values() - ) - for machine in machine_iter: - name = machine["name"] - zone = machine["zone"] - machines[name][zone] = machine - - op = act.aggregatedList_next(op, result) - return machines - - def machine_type(self, name: str) -> MachineType: - custom_patt = re.compile( - r"((?P\w+)-)?custom-(?P\d+)-(?P\d+)" - ) - if match := custom_patt.match(name): - return MachineType( - name=name, - guest_cpus=int(match.group("cpus")), - memory_mb=int(match.group("mem")), - accelerators=[], - ) - - machines = self.machine_types() - if name not in machines: - raise Exception(f"machine type {name} not found") - per_zone = machines[name] - assert per_zone - return MachineType.from_json( - next(iter(per_zone.values())) # pick the first/any zone - ) - - def template_machine_conf(self, template_link): - template = self.template_info(template_link) - machine = template.machine_type - - machine_conf = NSDict() - machine_conf.boards = 1 # No information, assume 1 - machine_conf.sockets = machine.sockets - # the value below for SocketsPerBoard must be type int - machine_conf.sockets_per_board = machine_conf.sockets // machine_conf.boards - machine_conf.threads_per_core = 1 - _div = 2 if getThreadsPerCore(template) == 1 else 1 - machine_conf.cpus = ( - int(machine.guest_cpus / _div) if machine.supports_smt else machine.guest_cpus - ) - machine_conf.cores_per_socket = int(machine_conf.cpus / machine_conf.sockets) - # Because the actual memory on the host will be different than - # what is configured (e.g. kernel will take it). From - # experiments, about 16 MB per GB are used (plus about 400 MB - # buffer for the first couple of GB's. Using 30 MB to be safe. - gb = machine.memory_mb // 1024 - machine_conf.memory = machine.memory_mb - (400 + (30 * gb)) - return machine_conf - - @lru_cache(maxsize=None) - def template_info(self, template_link): - template_name = trim_self_link(template_link) - cache = file_cache.cache("template_cache") - - if cached := cache.get(template_name): - return NSDict(cached) - - region = get_self_link_component(template_link, "regions") - - template = ensure_execute( - self.compute.instanceTemplates().get( - project=self.project, instanceTemplate=template_name - ) if region is None else - self.compute.regionInstanceTemplates().get( - project=self.project, region=region, instanceTemplate=template_name - ) - ).get("properties") - template = NSDict(template) - # name and link are not in properties, so stick them in - template.name = template_name - template.link = template_link - template.machine_type = self.machine_type(template.machineType) - # TODO delete metadata to reduce memory footprint? - # del template.metadata - - template.gpu = get_template_gpu(template) - - cache.set(template_name, template.to_dict()) - return template - - def _parse_job_info(self, job_info: str) -> Job: - """Extract job details""" - if match:= re.search(r"JobId=(\d+)", job_info): - job_id = int(match.group(1)) - else: - raise ValueError(f"Job ID not found in the job info: {job_info}") - - if match:= re.search(r"TimeLimit=(?:(\d+)-)?(\d{2}):(\d{2}):(\d{2})", job_info): - days, hours, minutes, seconds = match.groups() - duration = timedelta( - days=int(days) if days else 0, - hours=int(hours), - minutes=int(minutes), - seconds=int(seconds) - ) - else: - duration = None - - if match := re.search(r"JobName=([^\n]+)", job_info): - name = match.group(1) - else: - name = None - - if match := re.search(r"JobState=(\w+)", job_info): - job_state = match.group(1) - else: - job_state = None - - if match := re.search(r"ReqNodeList=([^ ]+)", job_info): - required_nodes = match.group(1) - else: - required_nodes = None - - return Job(id=job_id, duration=duration, name=name, job_state=job_state, required_nodes=required_nodes) - - @lru_cache - def get_jobs(self) -> List[Job]: - res = run(f"{self.scontrol} show jobs", timeout=30) - - return [self._parse_job_info(job) for job in res.stdout.split("\n\n")[:-1]] - - @lru_cache - def job(self, job_id: int) -> Optional[Job]: - job_info = run(f"{self.scontrol} show jobid {job_id}", check=False).stdout.rstrip() - if not job_info: - return None - - return self._parse_job_info(job_info=job_info) - - @property - def etc_dir(self) -> Path: - return Path(self.cfg.output_dir or slurmdirs.etc) - - def controller_mount_server_ip(self) -> str: - return self.control_addr or self.control_host - - def normalize_ns_mount(self, ns: Union[dict, NSMount]) -> NSMount: - if isinstance(ns, NSMount): - return ns - - server_ip = ns.get("server_ip") or "$controller" - if server_ip == "$controller": - server_ip = self.controller_mount_server_ip() - - return NSMount( - server_ip=server_ip, - local_mount=Path(ns["local_mount"]), - remote_mount=Path(ns["remote_mount"]), - fs_type=ns["fs_type"], - mount_options=ns["mount_options"], - ) - - @property - def munge_mount(self) -> NSMount: - if self.cfg.munge_mount: - mnt = self.cfg.munge_mount - mnt.local_mount = mnt.local_mount or "/mnt/munge" - return self.normalize_ns_mount(mnt) - else: - return NSMount( - server_ip=self.controller_mount_server_ip(), - local_mount=Path("/mnt/munge"), - remote_mount=dirs.munge, - fs_type="nfs", - mount_options="defaults,hard,intr,_netdev", - ) - - @property - def slurm_key_mount(self) -> NSMount: - if self.cfg.slurm_key_mount: - mnt = self.cfg.slurm_key_mount - mnt.local_mount = mnt.local_mount or slurmdirs.key_distribution - return self.normalize_ns_mount(mnt) - else: - return NSMount( - server_ip=self.controller_mount_server_ip(), - local_mount=slurmdirs.key_distribution, - remote_mount=slurmdirs.key_distribution, - fs_type="nfs", - mount_options="defaults,hard,intr,_netdev", - ) - - def is_flex_node(self, node: str) -> bool: - try: - nodeset = self.node_nodeset(node) - if nodeset.dws_flex.use_bulk_insert: - return False #For legacy flex support - return bool(nodeset.dws_flex.enabled) - except: - return False - - def is_provisioning_flex_node(self, node:str) -> bool: - if not self.is_flex_node(node): - return False - if self.instance(node) is not None: - return True - - nodeset = self.node_nodeset(node) - zones = nodeset.zone_policy_allow - assert len(zones) > 0 - region = self.node_region(node) - - potential_migs=[] - mig_list=self.get_mig_list(self.project, region) - - if not mig_list or not mig_list.get("items"): - return False - - for mig in mig_list["items"]: - if not mig.get("instanceTemplate"): #possibly an old MIG - return False - if mig["instanceTemplate"] == self.node_template(node) and mig["currentActions"]["creating"] > 0: - potential_migs.append(self.get_mig_instances(self.project, region, trim_self_link(mig["selfLink"]))) - - if not potential_migs: - return False - - for instance_collection in potential_migs[0]["managedInstances"]: - if node in instance_collection["name"] and instance_collection["currentAction"]=="CREATING": - return True - return False - - def cluster_regions(self) -> list[str]: - """ - Returns all regions used in cluster - NOTE: only concerned with normal nodesets, - neither TPU, nor dynamic, nor login node, nor controller node are considered - """ - res = set() - for nodeset in self.cfg.nodeset.values(): - res.add(parse_self_link(nodeset.subnetwork).region) - return list(res) - - - -_lkp: Optional[Lookup] = None - -def _load_config() -> NSDict: - return NSDict(yaml.safe_load(CONFIG_FILE.read_text())) - -def lookup() -> Lookup: - global _lkp - if _lkp is None: - try: - cfg = _load_config() - except FileNotFoundError: - log.error(f"config file not found: {CONFIG_FILE}") - cfg = NSDict() # TODO: fail here, once all code paths are covered (mainly init_logging) - _lkp = Lookup(cfg) - return _lkp - -def update_config(cfg: NSDict) -> None: - global _lkp - _lkp = Lookup(cfg) - -def scontrol_reconfigure(lkp: Lookup) -> None: - log.info("Running systemctl restart slurmctld.service") - run("sudo systemctl restart slurmctld.service", timeout=30) - log.info("Running scontrol reconfigure") - run(f"{lkp.scontrol} reconfigure") diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py deleted file mode 100644 index d1d77a1833..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any - - -from dataclasses import dataclass, asdict -import util -import local_pubsub - -import logging -log = logging.getLogger() - -# Name of the topic -TOPIC = "watch_delete_vm_op" - -@dataclass(frozen=True) -class WatchDeleteVmOp_Message: - op_name: str - zone: str - node: str - -class WatchDeleteVmOp_Topic: - def __init__(self, topic: local_pubsub.Topic) -> None: - self._t = topic - - def publish(self, op: dict[str, Any], node: str) -> None: - assert op.get("operationType") == "delete" - assert op.get("zone") - assert node - - msg = WatchDeleteVmOp_Message(op_name=op["name"], zone=op["zone"], node=node) - self._t.publish(data=asdict(msg)) - - -def watch_delete_vm_op_topic() -> WatchDeleteVmOp_Topic: - return WatchDeleteVmOp_Topic(local_pubsub.topic(TOPIC)) - - -def _watch_op(lkp: util.Lookup, m: WatchDeleteVmOp_Message) -> bool: - """ - Processes VM delete-operation. - If operation is still running - do nothing - If operation failed - log error & remove op from watch list - If operation is done - remove op from watch list do nothing - - To avoid querying status for each op individually, use list of VM instances as - a source of data. Don't query op for instance X if instance X is not present - (presumably deleted). - NOTE: This optimization can lead to false-positives - - absence of error-logs in case op failed, but VM got deleted by other means. - - Returns True if message should be marked as processed (ack). - """ - - inst = lkp.instance(m.node) - - if not inst: - log.debug(f"Stop watching op {m.op_name}, VM {m.node} appears to be deleted") - return True # ack, potentially false-positive - - if inst.status == "TERMINATED": - log.debug(f"Stop watching op {m.op_name}, VM {m.node} is TERMINATED") - return True # ack, potentially false-positive - - if inst.status == "STOPPING": - log.debug(f"Skipping op {m.op_name}, VM {m.node} is STOPPING") - return False # try later - - try: - op = util.get_operation_req(lkp, m.op_name, zone=m.zone).execute() - except: - # TODO: consider less conservative handling, but be careful not to cause deadlettering. - log.exception(f"Failed to get operation {m.op_name}, will not retry") - return True # ack (remove) - - if op["status"] != "DONE": - log.debug(f"Watching op {m.op_name} is still not done ({op['status']})") - return False # try later - - if "error" in op: - log.error(f"Operation {m.op_name} to delete {m.node} finished with error: {op['error']}") - else: - log.debug(f"Operation {m.op_name} to delete {m.node} successfully finished") - return True # ack - - -def watch_vm_delete_ops(lkp: util.Lookup) -> None: - sub = local_pubsub.subscription(TOPIC) - - # Pull once instead of "pulling until empty", motivation: - # Bulk of cases processed by `_watch_op` relies on freshness of `lkp.instances`, - # `lkp.instances` are fetched once during run of `slurmsync`. - # Therefore we shouldn't try to re-process messages that has been already NACKed in this run, - # since they will be handled with the same `lkp.instance` as a previous attempt. - msgs = sub.pull(max_messages=1000) # 1000 is arbitrary number to be adjusted if needed. - log.debug(f"Processing {len(msgs)} delete VM operations") - # TODO: handle messages in butches to improve latency - for m in msgs: - try: - dm = WatchDeleteVmOp_Message(**m.data) - ack = _watch_op(lkp, dm) - except Exception: - log.exception(f"Failed to process the message {m.id}, removing") - ack = True - if ack: - sub.ack([m.id]) - else: - sub.modify_ack_deadline([m.id], deadline=0) # NACK - - - - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf deleted file mode 100644 index 71905a0342..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf +++ /dev/null @@ -1,504 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "bucket_name" { - description = <<-EOD - Name of GCS bucket to use. - EOD - type = string -} - -variable "bucket_dir" { - description = "Bucket directory for cluster files to be put into." - type = string - default = null -} - -variable "enable_debug_logging" { - type = bool - description = "Enables debug logging mode. Not for production use." - default = false -} - -variable "extra_logging_flags" { - type = map(bool) - description = "The only available flag is `trace_api`" - default = {} -} - -variable "project_id" { - description = "The GCP project ID." - type = string -} - -variable "enable_slurm_auth" { - description = < x... } - nodeset_map = { for k, vs in local.nodeset_map_ell : k => vs[0] } - - nodeset_tpu_map_ell = { for x in var.nodeset_tpu : x.nodeset_name => x... } - nodeset_tpu_map = { for k, vs in local.nodeset_tpu_map_ell : k => vs[0] } - - nodeset_dyn_map_ell = { for x in var.nodeset_dyn : x.nodeset_name => x... } - nodeset_dyn_map = { for k, vs in local.nodeset_dyn_map_ell : k => vs[0] } - - - no_reservation_affinity = { type : "NO_RESERVATION" } -} - -# NODESET -module "slurm_nodeset_template" { - source = "../../internal/slurm-gcp/instance_template" - for_each = local.nodeset_map - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - slurm_instance_role = "compute" - slurm_bucket_path = module.slurm_files.slurm_bucket_path - - additional_disks = each.value.additional_disks - bandwidth_tier = each.value.bandwidth_tier - can_ip_forward = each.value.can_ip_forward - advanced_machine_features = each.value.advanced_machine_features - disk_auto_delete = each.value.disk_auto_delete - disk_labels = each.value.disk_labels - disk_resource_manager_tags = each.value.disk_resource_manager_tags - disk_size_gb = each.value.disk_size_gb - disk_type = each.value.disk_type - enable_confidential_vm = each.value.enable_confidential_vm - enable_oslogin = each.value.enable_oslogin - enable_shielded_vm = each.value.enable_shielded_vm - gpu = each.value.gpu - labels = merge(each.value.labels, { slurm_nodeset = each.value.nodeset_name }) - machine_type = each.value.machine_type - metadata = merge(each.value.metadata, local.universe_domain) - min_cpu_platform = each.value.min_cpu_platform - name_prefix = each.value.nodeset_name - on_host_maintenance = each.value.on_host_maintenance - preemptible = each.value.preemptible - region = each.value.region - resource_manager_tags = each.value.resource_manager_tags - spot = each.value.spot - termination_action = each.value.termination_action - service_account = each.value.service_account - shielded_instance_config = each.value.shielded_instance_config - source_image_family = each.value.source_image_family - source_image_project = each.value.source_image_project - source_image = each.value.source_image - subnetwork = each.value.subnetwork_self_link - additional_networks = each.value.additional_networks - access_config = each.value.access_config - tags = concat([local.slurm_cluster_name], each.value.tags) - - max_run_duration = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? each.value.dws_flex.max_run_duration : null - provisioning_model = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? "FLEX_START" : null - reservation_affinity = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? local.no_reservation_affinity : null -} - -module "nodeset_cleanup" { - source = "./modules/cleanup_compute" - for_each = local.nodeset_map - - nodeset = each.value - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - enable_cleanup_compute = var.enable_cleanup_compute - universe_domain = var.universe_domain - endpoint_versions = var.endpoint_versions - gcloud_path_override = var.gcloud_path_override - nodeset_template = module.slurm_nodeset_template[each.value.nodeset_name].self_link -} - -locals { - nodesets = [for name, ns in local.nodeset_map : { - nodeset_name = ns.nodeset_name - node_conf = ns.node_conf - dws_flex = ns.dws_flex - instance_template = module.slurm_nodeset_template[ns.nodeset_name].self_link - node_count_dynamic_max = ns.node_count_dynamic_max - node_count_static = ns.node_count_static - subnetwork = ns.subnetwork_self_link - reservation_name = ns.reservation_name - future_reservation = ns.future_reservation - maintenance_interval = ns.maintenance_interval - instance_properties_json = ns.instance_properties_json - enable_placement = ns.enable_placement - placement_max_distance = ns.placement_max_distance - network_storage = ns.network_storage - zone_target_shape = ns.zone_target_shape - zone_policy_allow = ns.zone_policy_allow - zone_policy_deny = ns.zone_policy_deny - enable_maintenance_reservation = ns.enable_maintenance_reservation - enable_opportunistic_maintenance = ns.enable_opportunistic_maintenance - accelerator_topology = ns.accelerator_topology - }] -} - -# NODESET TPU -module "slurm_nodeset_tpu" { - source = "../../internal/slurm-gcp/nodeset_tpu" - for_each = local.nodeset_tpu_map - - project_id = var.project_id - node_count_dynamic_max = each.value.node_count_dynamic_max - node_count_static = each.value.node_count_static - nodeset_name = each.value.nodeset_name - zone = each.value.zone - node_type = each.value.node_type - accelerator_config = each.value.accelerator_config - tf_version = each.value.tf_version - preemptible = each.value.preemptible - preserve_tpu = each.value.preserve_tpu - enable_public_ip = each.value.enable_public_ip - service_account = each.value.service_account - data_disks = each.value.data_disks - docker_image = each.value.docker_image - subnetwork = each.value.subnetwork -} - -module "nodeset_cleanup_tpu" { - source = "./modules/cleanup_tpu" - for_each = local.nodeset_tpu_map - - nodeset = { - nodeset_name = each.value.nodeset_name - zone = each.value.zone - } - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - enable_cleanup_compute = var.enable_cleanup_compute - universe_domain = var.universe_domain - endpoint_versions = var.endpoint_versions - gcloud_path_override = var.gcloud_path_override - - depends_on = [ - # Depend on controller network, as a best effort to avoid - # subnetwork resourceInUseByAnotherResource error - var.subnetwork_self_link - ] -} - -resource "google_storage_bucket_object" "parition_config" { - for_each = { for p in var.partitions : p.partition_name => p } - - bucket = module.slurm_files.bucket_name - name = "${module.slurm_files.bucket_dir}/partition_configs/${each.key}.yaml" - content = yamlencode(each.value) - source_md5hash = md5(yamlencode(each.value)) -} - -moved { - from = module.slurm_files.google_storage_bucket_object.parition_config - to = google_storage_bucket_object.parition_config -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf deleted file mode 100644 index 218c36e392..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# BUCKET - -locals { - synt_suffix = substr(md5("${local.controller_project_id}${var.deployment_name}"), 0, 5) - synth_bucket_name = "${local.slurm_cluster_name}${local.synt_suffix}" - - bucket_name = var.create_bucket ? module.bucket[0].name : var.bucket_name -} - -module "bucket" { - source = "terraform-google-modules/cloud-storage/google" - version = ">= 6.1" - - count = var.create_bucket ? 1 : 0 - - location = var.region - names = [local.synth_bucket_name] - prefix = "slurm" - project_id = local.controller_project_id - - force_destroy = { - (local.synth_bucket_name) = true - } - - labels = merge(local.labels, { - slurm_cluster_name = local.slurm_cluster_name - }) -} - -# BUCKET IAMs -locals { - compute_sa = toset(flatten([for x in module.slurm_nodeset_template : x.service_account])) - compute_tpu_sa = toset(flatten([for x in module.slurm_nodeset_tpu : x.service_account])) - login_sa = toset(flatten([for x in module.login : x.service_account])) - - viewers = toset(flatten([ - "serviceAccount:${module.slurm_controller_template.service_account.email}", - formatlist("serviceAccount:%s", [for x in local.compute_sa : x.email]), - formatlist("serviceAccount:%s", [for x in local.compute_tpu_sa : x.email if x.email != null]), - formatlist("serviceAccount:%s", [for x in local.login_sa : x.email]), - ])) -} - - -resource "google_storage_bucket_iam_member" "viewers" { - for_each = local.viewers - bucket = local.bucket_name - role = "roles/storage.objectViewer" - member = each.value -} - -resource "google_storage_bucket_iam_member" "legacy_readers" { - for_each = local.viewers - bucket = local.bucket_name - role = "roles/storage.legacyBucketReader" - member = each.value -} - -locals { - daos_ns = [ - for ns in var.network_storage : - ns if ns.fs_type == "daos" - ] - - daos_client_install_runners = [ - for ns in local.daos_ns : - ns.client_install_runner if ns.client_install_runner != null - ] - - daos_mount_runners = [ - for ns in local.daos_ns : - ns.mount_runner if ns.mount_runner != null - ] - - daos_network_storage_runners = concat( - local.daos_client_install_runners, - local.daos_mount_runners, - ) - - daos_install_mount_script = { - filename = "ghpc_daos_mount.sh" - content = length(local.daos_ns) > 0 ? module.daos_network_storage_scripts[0].startup_script : "" - } - - common_scripts = length(local.daos_ns) > 0 ? [local.daos_install_mount_script] : [] -} - -# SLURM FILES -locals { - ghpc_startup_script_controller = concat( - local.common_scripts, - [{ - filename = "ghpc_startup.sh" - content = var.controller_startup_script - }]) - - controller_state_disk = { - device_name : try(google_compute_disk.controller_disk[0].name, null) - } - - - nodeset_startup_scripts = { for k, v in local.nodeset_map : k => concat(local.common_scripts, v.startup_script) } -} - -module "daos_network_storage_scripts" { - count = length(local.daos_ns) > 0 ? 1 : 0 - - source = "../../../../modules/scripts/startup-script" - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.daos_network_storage_runners -} - -module "slurm_files" { - source = "./modules/slurm_files" - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - bucket_dir = var.bucket_dir - bucket_name = local.bucket_name - controller_network_attachment = var.controller_network_attachment - - slurmdbd_conf_tpl = var.slurmdbd_conf_tpl - slurm_conf_tpl = var.slurm_conf_tpl - slurm_conf_template = var.slurm_conf_template - cgroup_conf_tpl = var.cgroup_conf_tpl - cloud_parameters = var.cloud_parameters - cloudsql_secret = try( - one(google_secret_manager_secret_version.cloudsql_version[*].id), - null) - - controller_startup_scripts = local.ghpc_startup_script_controller - controller_startup_scripts_timeout = var.controller_startup_scripts_timeout - nodeset_startup_scripts = local.nodeset_startup_scripts - compute_startup_scripts_timeout = var.compute_startup_scripts_timeout - controller_state_disk = local.controller_state_disk - - enable_debug_logging = var.enable_debug_logging - extra_logging_flags = var.extra_logging_flags - - enable_slurm_auth = var.enable_slurm_auth - - enable_bigquery_load = var.enable_bigquery_load - enable_external_prolog_epilog = var.enable_external_prolog_epilog - enable_chs_gpu_health_check_prolog = var.enable_chs_gpu_health_check_prolog - enable_chs_gpu_health_check_epilog = var.enable_chs_gpu_health_check_epilog - epilog_scripts = var.epilog_scripts - prolog_scripts = var.prolog_scripts - task_epilog_scripts = var.task_epilog_scripts - task_prolog_scripts = var.task_prolog_scripts - - disable_default_mounts = !var.enable_default_mounts - network_storage = [ - for storage in var.network_storage : { - server_ip = storage.server_ip, - remote_mount = storage.remote_mount, - local_mount = storage.local_mount, - fs_type = storage.fs_type, - mount_options = storage.mount_options - } - if storage.fs_type != "daos" - ] - - nodeset = local.nodesets - nodeset_dyn = values(local.nodeset_dyn_map) - # Use legacy format for now - nodeset_tpu = values(module.slurm_nodeset_tpu)[*] - - - depends_on = [module.bucket] - - # Providers - endpoint_versions = var.endpoint_versions -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf deleted file mode 100644 index 85ad10fa21..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf +++ /dev/null @@ -1,814 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########### -# GENERAL # -########### - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "deployment_name" { - description = "Name of the deployment." - type = string -} - -variable "slurm_cluster_name" { - type = string - description = <<-EOD - Cluster name, used for resource naming and slurm accounting. - If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). - EOD - default = null - - validation { - condition = var.slurm_cluster_name == null || can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) - error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." - } -} - -variable "region" { - type = string - description = "The default region to place resources in." -} - -variable "zone" { - type = string - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [instance\_validation](#module\_instance\_validation) | ../../../../modules/internal/instance_validations | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disable\_login\_public\_ips](#input\_disable\_login\_public\_ips) | DEPRECATED: Use `enable_login_public_ips` instead. | `bool` | `null` | no | -| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_login\_public\_ips](#input\_enable\_login\_public\_ips) | If set to true. The login node will have a random public IP assigned to it. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-11-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for login nodes. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Unique name prefix for login nodes. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all login groups. | `string` | n/a | yes | -| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the login instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the login instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [login\_nodes](#output\_login\_nodes) | Slurm login instance definition. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf deleted file mode 100644 index 6ebe5902dc..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-login", ghpc_role = "scheduler" }) -} - -module "instance_validation" { - source = "../../../../modules/internal/instance_validations" - - machine_type = var.machine_type - disk_type = var.disk_type -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata - ) - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.disk_resource_manager_tags - } - ] - - public_access_config = [{ nat_ip = null, network_tier = null }] - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - # lower, replace `_` with `-`, and remove any non-alphanumeric characters - group_name = replace( - replace( - lower(var.name_prefix), - "_", "-"), - "/[^-a-z0-9]/", "") - - - login_node = { - group_name = local.group_name - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(var.disk_labels, local.labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - additional_networks = var.additional_networks - - can_ip_forward = var.can_ip_forward - advanced_machine_features = var.advanced_machine_features - - enable_confidential_vm = var.enable_confidential_vm - access_config = var.enable_login_public_ips ? local.public_access_config : [] - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - gpu = one(local.guest_accelerator) - labels = local.labels - machine_type = var.machine_type - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - num_instances = var.num_instances - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - region = var.region - resource_manager_tags = var.resource_manager_tags - zone = var.zone - - service_account = local.service_account - - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - static_ips = var.static_ips - bandwidth_tier = var.bandwidth_tier - - subnetwork = var.subnetwork_self_link - tags = var.tags - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml deleted file mode 100644 index 47f003258e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name_prefix - has_to_be_used: true diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf deleted file mode 100644 index e700542794..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "login_nodes" { - description = "Slurm login instance definition." - value = [local.login_node] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf deleted file mode 100644 index db6cfc1318..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${var.instance_image.project}/global/images/family" : - "projects/${var.instance_image.project}/global/images" - ) - source_image_family = try(var.instance_image.family, "") - source_image = try(var.instance_image.name, "") -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf deleted file mode 100644 index 7c1a2e06b5..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf +++ /dev/null @@ -1,419 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -variable "project_id" { # tflint-ignore: terraform_unused_declarations - type = string - description = "Project ID to create resources in." -} - -variable "region" { - type = string - description = "Region where the instances should be created." - default = null -} - -variable "zone" { - type = string - description = <<-EOD - Zone where the instances should be created. If not specified, instances will be - spread across available zones in the region. - EOD - default = null -} - -variable "name_prefix" { - type = string - description = <<-EOD - Unique name prefix for login nodes. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all login groups. - EOD -} - -variable "num_instances" { - type = number - description = "Number of instances to create. This value is ignored if static_ips is provided." - default = 1 -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} -} - -variable "disk_type" { - type = string - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - default = "pd-ssd" -} - -variable "disk_size_gb" { - type = number - description = "Boot disk size in GB." - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - type = list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string)) - auto_delete = optional(bool) - boot = optional(bool) - disk_resource_manager_tags = optional(map(string)) - })) - description = "List of maps of disks." - default = [] -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })) - nullable = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "disable_smt" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - type = bool - default = null - validation { - condition = var.disable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "static_ips" { - type = list(string) - description = "List of static IPs for VM instances." - default = [] -} - -variable "bandwidth_tier" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 6.16 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.16 | -| [helm](#provider\_helm) | ~> 2.17 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.cert_manager](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.prometheus](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.slurm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.slurm_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cert\_manager\_chart\_version](#input\_cert\_manager\_chart\_version) | Version of the Cert Manager chart to install. | `string` | `"v1.18.2"` | no | -| [cert\_manager\_values](#input\_cert\_manager\_values) | Value overrides for the Cert Manager release | `any` | 
{
  "crds": {
    "enabled": true
  }
}
| no | -| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | -| [install\_kube\_prometheus\_stack](#input\_install\_kube\_prometheus\_stack) | Install the Kube Prometheus Stack. | `bool` | `false` | no | -| [install\_slurm\_chart](#input\_install\_slurm\_chart) | Install slurm-operator chart. | `bool` | `true` | no | -| [install\_slurm\_operator\_chart](#input\_install\_slurm\_operator\_chart) | Install slurm-operator chart. | `bool` | `true` | no | -| [node\_pool\_names](#input\_node\_pool\_names) | Names of node pools, for use in node affinities (Slinky system components). | `list(string)` | `null` | no | -| [project\_id](#input\_project\_id) | The project ID that hosts the GKE cluster. | `string` | n/a | yes | -| [prometheus\_chart\_version](#input\_prometheus\_chart\_version) | Version of the Kube Prometheus Stack chart to install. | `string` | `"77.0.1"` | no | -| [prometheus\_values](#input\_prometheus\_values) | Value overrides for the Prometheus release | `any` | 
{
  "installCRDs": true
}
| no | -| [slurm\_chart\_version](#input\_slurm\_chart\_version) | Version of the Slurm chart to install. | `string` | `"0.3.1"` | no | -| [slurm\_namespace](#input\_slurm\_namespace) | slurm namespace for charts | `string` | `"slurm"` | no | -| [slurm\_operator\_chart\_version](#input\_slurm\_operator\_chart\_version) | Version of the Slurm Operator chart to install. | `string` | `"0.3.1"` | no | -| [slurm\_operator\_namespace](#input\_slurm\_operator\_namespace) | slurm namespace for charts | `string` | `"slinky"` | no | -| [slurm\_operator\_repository](#input\_slurm\_operator\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | -| [slurm\_operator\_values](#input\_slurm\_operator\_values) | Value overrides for the Slinky release | `any` | `{}` | no | -| [slurm\_repository](#input\_slurm\_repository) | Value overrides for the Slinky release | `string` | `"oci://ghcr.io/slinkyproject/charts"` | no | -| [slurm\_values](#input\_slurm\_values) | Value overrides for the Slurm release | `any` | `{}` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [slurm\_namespace](#output\_slurm\_namespace) | namespace for the slurm chart | -| [slurm\_operator\_namespace](#output\_slurm\_operator\_namespace) | namespace for the slinky operator chart | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/main.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/main.tf deleted file mode 100644 index aff33b73a0..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/main.tf +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] - project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] - - # Define affinity settings when node pools are specified - node_pool_affinity = var.node_pool_names != null ? { - nodeAffinity = { - requiredDuringSchedulingIgnoredDuringExecution = { - nodeSelectorTerms = [{ - matchExpressions = [{ - key = "cloud.google.com/gke-nodepool" - operator = "In" - values = var.node_pool_names - }] - }] - } - } - } : {} -} - -data "google_client_config" "default" {} - -data "google_container_cluster" "gke_cluster" { - project = local.project_id - name = local.cluster_name - location = local.cluster_location -} - -resource "helm_release" "cert_manager" { - name = "cert-manager" - chart = "cert-manager" - repository = "https://charts.jetstack.io" - version = var.cert_manager_chart_version - namespace = "cert-manager" - create_namespace = true - - values = concat( - [yamlencode({ - affinity = local.node_pool_affinity - webhook = { - affinity = local.node_pool_affinity - } - cainjector = { - affinity = local.node_pool_affinity - } - startupapicheck = { - affinity = local.node_pool_affinity - } - })], - [yamlencode(var.cert_manager_values)] - ) -} - -resource "helm_release" "slurm_operator" { - count = var.install_slurm_operator_chart ? 1 : 0 - name = "slurm-operator" - chart = "slurm-operator" - repository = var.slurm_operator_repository - version = var.slurm_operator_chart_version - namespace = var.slurm_operator_namespace - create_namespace = true - - # The Cert Manager webhook deployment must be running to provision the Operator - depends_on = [ - helm_release.cert_manager - ] - - values = concat( - [yamlencode({ - operator = { - affinity = local.node_pool_affinity - } - webhook = { - affinity = local.node_pool_affinity - } - })], - [yamlencode(var.slurm_operator_values)] - ) -} - -resource "helm_release" "slurm" { - count = var.install_slurm_chart ? 1 : 0 - name = "slurm" - chart = "slurm" - repository = var.slurm_repository - version = var.slurm_chart_version - namespace = var.slurm_namespace - create_namespace = true - - # The Slurm Operator must be running to provision Slurm clusters/nodesets - depends_on = [ - helm_release.slurm_operator - ] - - values = concat( - [yamlencode({ - controller = { - affinity = local.node_pool_affinity - } - accounting = { - affinity = local.node_pool_affinity - } - mariadb = { - primary = { - affinity = local.node_pool_affinity - } - secondary = { - affinity = local.node_pool_affinity - } - } - restapi = { - affinity = local.node_pool_affinity - } - slurm-exporter = { - exporter = { - affinity = local.node_pool_affinity - } - } - })], - [yamlencode(var.slurm_values)] - ) -} - -resource "helm_release" "prometheus" { - count = var.install_kube_prometheus_stack ? 1 : 0 - name = "prometheus" - chart = "kube-prometheus-stack" - repository = "https://prometheus-community.github.io/helm-charts" - version = var.prometheus_chart_version - namespace = "prometheus" - create_namespace = true - - values = concat( - [yamlencode({ - crds = { - upgradeJob = { - affinity = local.node_pool_affinity - } - } - alertmanager = { - alertmanagerSpec = { - affinity = local.node_pool_affinity - } - } - prometheusOperator = { - admissionWebhooks = { - deployment = { - affinity = local.node_pool_affinity - } - patch = { - affinity = local.node_pool_affinity - } - } - affinity = local.node_pool_affinity - } - prometheus = { - prometheusSpec = { - affinity = local.node_pool_affinity - } - } - thanosRuler = { - thanosRulerSpec = { - affinity = local.node_pool_affinity - } - } - kube-state-metrics = { - affinity = local.node_pool_affinity - } - grafana = { - affinity = local.node_pool_affinity - imageRenderer = { - affinity = local.node_pool_affinity - } - } - prometheus-windows-exporter = { - affinity = local.node_pool_affinity - } - })], - [yamlencode(var.prometheus_values)] - ) -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/metadata.yaml deleted file mode 100644 index e18197e2b7..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/outputs.tf deleted file mode 100644 index 8ea6385905..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "slurm_namespace" { - description = "namespace for the slurm chart" - value = var.slurm_namespace -} - -output "slurm_operator_namespace" { - description = "namespace for the slinky operator chart" - value = var.slurm_operator_namespace -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/providers.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/providers.tf deleted file mode 100644 index 313d6dc58e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/providers.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -provider "helm" { - kubernetes { - host = "https://${data.google_container_cluster.gke_cluster.endpoint}" - token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode( - data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate, - ) - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/variables.tf deleted file mode 100644 index 8acaf78562..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/variables.tf +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "The project ID that hosts the GKE cluster." - type = string -} - -variable "cluster_id" { - description = "An identifier for the GKE cluster resource with format projects//locations//clusters/." - type = string - nullable = false -} - -variable "node_pool_names" { - description = "Names of node pools, for use in node affinities (Slinky system components)." - type = list(string) - default = null -} - -variable "cert_manager_chart_version" { - description = "Version of the Cert Manager chart to install." - type = string - default = "v1.18.2" -} - -variable "cert_manager_values" { - description = "Value overrides for the Cert Manager release" - type = any - default = { - crds = { - enabled = true - } - } -} - -variable "slurm_operator_chart_version" { - description = "Version of the Slurm Operator chart to install." - type = string - default = "0.3.1" -} - -variable "slurm_operator_values" { - description = "Value overrides for the Slinky release" - type = any - default = {} -} - -variable "slurm_chart_version" { - description = "Version of the Slurm chart to install." - type = string - default = "0.3.1" -} - -variable "slurm_values" { - description = "Value overrides for the Slurm release" - type = any - default = {} -} - -variable "install_kube_prometheus_stack" { - # Components detailed at https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack - description = "Install the Kube Prometheus Stack." - type = bool - default = false -} - -variable "prometheus_chart_version" { - description = "Version of the Kube Prometheus Stack chart to install." - type = string - default = "77.0.1" -} - -variable "prometheus_values" { - description = "Value overrides for the Prometheus release" - type = any - default = { - installCRDs = true - } -} - -variable "slurm_namespace" { - description = "slurm namespace for charts" - type = string - default = "slurm" -} - -variable "slurm_operator_namespace" { - description = "slurm namespace for charts" - type = string - default = "slinky" -} - -variable "install_slurm_chart" { - description = "Install slurm-operator chart." - type = bool - default = true -} - -variable "install_slurm_operator_chart" { - description = "Install slurm-operator chart." - type = bool - default = true -} - -variable "slurm_repository" { - description = "Value overrides for the Slinky release" - type = string - default = "oci://ghcr.io/slinkyproject/charts" -} - -variable "slurm_operator_repository" { - description = "Value overrides for the Slinky release" - type = string - default = "oci://ghcr.io/slinkyproject/charts" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/versions.tf deleted file mode 100644 index ae4327aeef..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scheduler/slinky/versions.tf +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - google = { - source = "hashicorp/google" - version = ">= 6.16" - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/README.md deleted file mode 100644 index 71a862fd6c..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/README.md +++ /dev/null @@ -1,149 +0,0 @@ -## Description - -This module creates a Toolkit runner that will install HTCondor on RedHat 7 or -8 and its derivative operating systems. These include the CentOS 7 and Rocky -Linux 8 releases of the [HPC VM Image][hpcvmimage]. It may also function on -RedHat 9 and derivatives, however it is not yet supported. Please report any -[issues] on these 3 distributions or open a [discussion] to request support on -Debian or Ubuntu distributions. - -[issues]: https://github.com/GoogleCloudPlatform/hpc-toolkit/issues -[discussion]: https://github.com/GoogleCloudPlatform/hpc-toolkit/discussions - -It also exports a list of Google Cloud APIs which must be enabled prior to -provisioning an HTCondor Pool. - -It is expected to be used with the [htcondor-setup] and -[htcondor-execute-point] modules. - -[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[htcondor-setup]: ../../scheduler/htcondor-setup/README.md -[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md - -### Example - -The following code snippet uses this module to create startup scripts that -install the HTCondor software into a custom VM image. - -```yaml -deployment_groups: -- group: primary - modules: - - id: network1 - source: modules/network/vpc - outputs: - - network_name - - - id: htcondor_install - source: community/modules/scripts/htcondor-install - - - id: htcondor_install_script - source: modules/scripts/startup-script - use: - - htcondor_install - -- group: packer - modules: - - id: custom-image - source: modules/packer/custom-image - kind: packer - use: - - network1 - - htcondor_install_script - settings: - disk_size: 50 - source_image_family: hpc-rocky-linux-8 - image_family: "htcondor-10x" -``` - -A full example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -## Important note - -All POSIX users and HTCondor jobs can act as the service account attached to -VMs within the pool. This enables the use of IAM restrictions via service -accounts but also allows users to access services to which system daemons need -access (e.g. to create Cloud Logging entries). If this is undesirable, one can -restrict access to the instance metadata server to the `root` and `condor` -users. This will allow system services to use the service account, but not -other POSIX users or HTCondor jobs. The firewall example below is appropriate -for CentOS 7. - -```shell -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 1 \ - -m owner --uid-owner root -p tcp -d metadata.google.internal --dport 80 -j ACCEPT -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 2 \ - -m owner --uid-owner condor -p tcp -d metadata.google.internal --dport 80 -j ACCEPT -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 3 \ - -p tcp -d metadata.google.internal --dport 80 -j DROP -firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT_direct 4 \ - -p tcp -d metadata.google.internal --dport 8080 -j DROP -firewall-cmd --permanent --zone=public --add-port=9618/tcp -firewall-cmd --reload -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [condor\_version](#input\_condor\_version) | Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: "23.0.0","23.*")) | `string` | `"23.*"` | no | -| [enable\_docker](#input\_enable\_docker) | Install and enable docker daemon alongside HTCondor | `bool` | `true` | no | -| [http\_proxy](#input\_http\_proxy) | Set system default web (http and https) proxy for Windows HTCondor installation | `string` | `""` | no | -| [python\_windows\_installer\_url](#input\_python\_windows\_installer\_url) | URL of Python installer for Windows | `string` | `"https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [gcp\_service\_list](#output\_gcp\_service\_list) | Google Cloud APIs required by HTCondor | -| [runners](#output\_runners) | Runner to install HTCondor using startup-scripts | -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | Windows PowerShell script to install HTCondor | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py deleted file mode 100644 index 77bafa0310..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py +++ /dev/null @@ -1,417 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -# Copyright 2018 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script for resizing managed instance group (MIG) cluster size based -# on the number of jobs in the Condor Queue. - -from absl import app -from absl import flags -from collections import OrderedDict -from datetime import datetime -from pprint import pprint -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials - -import argparse -import os -import math -import time -import htcondor -import classad - -parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) -parser.add_argument("--p", required=True, help="Project id", type=str) -parser.add_argument( - "--z", - required=True, - help="Name of GCP zone where the managed instance group is located", - type=str, -) -parser.add_argument( - "--r", - required=True, - help="Name of GCP region where the managed instance group is located", - type=str, -) -parser.add_argument( - "--mz", - required=False, - help="Enabled multizone (regional) managed instance group", - action="store_true", -) -parser.add_argument( - "--g", required=True, help="Name of the managed instance group", type=str -) -parser.add_argument( - "--i", - default=0, - help="Minimum number of idle compute instances", - type=int -) -parser.add_argument( - "--c", required=True, help="Maximum number of compute instances", type=int -) -parser.add_argument( - "--v", - default=0, - help="Increase output verbosity. 1-show basic debug info. 2-show detail debug info", - type=int, - choices=[0, 1, 2], -) -parser.add_argument( - "--d", - default=0, - help="Dry Run, default=0, if 1, then no scaling actions", - type=int, - choices=[0, 1], -) - -args = parser.parse_args() - -class AutoScaler: - def __init__(self, multizone=False): - - self.multizone = multizone - # Obtain credentials - self.credentials = GoogleCredentials.get_application_default() - self.service = discovery.build("compute", "v1", credentials=self.credentials) - - if self.multizone: - self.instanceGroupManagers = self.service.regionInstanceGroupManagers() - else: - self.instanceGroupManagers = self.service.instanceGroupManagers() - - # Remove specified instances from MIG and decrease MIG size - def deleteFromMig(self, node_self_links): - requestDelInstance = self.instanceGroupManagers.deleteInstances( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - body={ "instances": node_self_links }, - ) - - # execute if not a dry-run - if not self.dryrun: - response = requestDelInstance.execute() - if self.debug > 0: - pprint(response) - return response - return "Dry Run" - - def getInstanceTemplateInfo(self): - requestTemplateName = self.instanceGroupManagers.get( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - fields="instanceTemplate", - ) - responseTemplateName = requestTemplateName.execute() - template_name = "" - - if self.debug > 1: - print("Request for the template name") - pprint(responseTemplateName) - - if len(responseTemplateName) > 0: - template_url = responseTemplateName.get("instanceTemplate") - template_url_partitioned = template_url.split("/") - template_name = template_url_partitioned[len(template_url_partitioned) - 1] - - requestInstanceTemplate = self.service.instanceTemplates().get( - project=self.project, instanceTemplate=template_name, fields="properties" - ) - responseInstanceTemplateInfo = requestInstanceTemplate.execute() - - if self.debug > 1: - print("Template information") - pprint(responseInstanceTemplateInfo["properties"]) - - machine_type = responseInstanceTemplateInfo["properties"]["machineType"] - is_spot = responseInstanceTemplateInfo["properties"]["scheduling"][ - "preemptible" - ] - if self.debug > 0: - print("Machine Type: " + machine_type) - print("Is spot: " + str(is_spot)) - request = self.service.machineTypes().get( - project=self.project, zone=self.zone, machineType=machine_type - ) - response = request.execute() - guest_cpus = response["guestCpus"] - if self.debug > 1: - print("Machine information") - pprint(responseInstanceTemplateInfo["properties"]) - if self.debug > 0: - print("Guest CPUs: " + str(guest_cpus)) - - instanceTemplateInfo = { - "machine_type": machine_type, - "is_spot": is_spot, - "guest_cpus": guest_cpus, - } - return instanceTemplateInfo - - def scale(self): - # diagnosis - if self.debug > 1: - print("Launching autoscaler.py with the following arguments:") - print("project_id: " + self.project) - print("zone: " + self.zone) - print("region: " + self.region) - print(f"multizone: {self.multizone}") - print("group_manager: " + self.instance_group_manager) - print("computeinstancelimit: " + str(self.compute_instance_limit)) - print("debuglevel: " + str(self.debug)) - - if self.multizone: - self.zoneargs = {"region": self.region} - else: - self.zoneargs = {"zone": self.zone} - - # Each HTCondor scheduler (SchedD), maintains a list of jobs under its - # stewardship. A full list of Job ClassAd attributes can be found at - # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html - schedd = htcondor.Schedd() - # encourage the job queue to start a new negotiation cycle; there are - # internal unconfigurable rate limits so not guaranteed; this is not - # strictly required for success, but may reduce latency of autoscaling - schedd.reschedule() - REQUEST_CPUS_ATTRIBUTE = "RequestCpus" - REQUEST_GPUS_ATTRIBUTE = "RequestGpus" - REQUEST_MEMORY_ATTRIBUTE = "RequestMemory" - job_attributes = [ - REQUEST_CPUS_ATTRIBUTE, - REQUEST_GPUS_ATTRIBUTE, - REQUEST_MEMORY_ATTRIBUTE, - ] - - instanceTemplateInfo = self.getInstanceTemplateInfo() - self.is_spot = instanceTemplateInfo["is_spot"] - self.cores_per_node = instanceTemplateInfo["guest_cpus"] - print(f"MIG is configured for Spot pricing: {self.is_spot}") - print("Number of CPU per compute node: " + str(self.cores_per_node)) - - # this query will constrain the search for jobs to those that either - # require spot VMs or do not require Spot VMs based on whether the - # VM instance template is configured for Spot pricing - spot_query = classad.ExprTree(f"RequireId == \"{self.instance_group_manager}\"") - - # For purpose of scaling a Managed Instance Group, count only jobs that - # are idle and likely participated in a negotiation cycle (there does - # not appear to be a single classad attribute for this). - # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html#JobStatus - LAST_CYCLE_ATTRIBUTE = "LastNegotiationCycleTime0" - coll = htcondor.Collector() - negotiator_ad = coll.query(htcondor.AdTypes.Negotiator, projection=[LAST_CYCLE_ATTRIBUTE]) - if len(negotiator_ad) != 1: - print(f"There should be exactly 1 negotiator in the pool. There is {len(negotiator_ad)}") - exit() - last_negotiation_cycle_time = negotiator_ad[0].get(LAST_CYCLE_ATTRIBUTE) - if not last_negotiation_cycle_time: - print(f"The negotiator has not yet started a match cycle. Exiting auto-scaling.") - exit() - - print(f"Last negotiation cycle occurred at: {datetime.fromtimestamp(last_negotiation_cycle_time)}") - idle_job_query = classad.ExprTree(f"JobStatus == 1 && QDate < {last_negotiation_cycle_time}") - idle_job_ads = schedd.query(constraint=idle_job_query.and_(spot_query), - projection=job_attributes) - - total_idle_request_cpus = sum(j[REQUEST_CPUS_ATTRIBUTE] for j in idle_job_ads) - print(f"Total CPUs requested by idle jobs: {total_idle_request_cpus}") - - if self.debug > 1: - print("Information about the compute instance template") - pprint(instanceTemplateInfo) - - # Calculate the minimum number of instances that, for fully packed - # execute points, could satisfy current job queue - min_hosts_for_idle_jobs = math.ceil(total_idle_request_cpus / self.cores_per_node) - if self.debug > 0: - print(f"Minimum hosts needed: {total_idle_request_cpus} / {self.cores_per_node} = {min_hosts_for_idle_jobs}") - - # Get current number of instances in the MIG - requestGroupInfo = self.instanceGroupManagers.get( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - ) - responseGroupInfo = requestGroupInfo.execute() - current_target = responseGroupInfo["targetSize"] - print(f"Current MIG target size: {current_target}") - - # Find instances that are being modified by the MIG (currentAction is - # any value other than "NONE"). A common reason an instance is modified - # is it because it has failed a health check. - reqModifyingInstances = self.instanceGroupManagers.listManagedInstances( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - filter="currentAction != \"NONE\"", - orderBy="creationTimestamp desc" - ) - respModifyingInstances = reqModifyingInstances.execute() - - # Find VMs that are idle (no dynamic slots created from partitionable - # slots) in the MIG handled by this autoscaler - filter_idle_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots==0") - filter_claimed_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots>0") - filter_mig = classad.ExprTree(f"regexp(\".*/{self.instance_group_manager}$\", CloudCreatedBy)") - # A full list of Machine (StartD) ClassAd attributes can be found at - # https://htcondor.readthedocs.io/en/latest/classad-attributes/machine-classad-attributes.html - idle_node_ads = coll.query(htcondor.AdTypes.Startd, - constraint=filter_idle_vms.and_(filter_mig), - projection=["Machine", "CloudZone"]) - - NODENAME_ATTRIBUTE = "Machine" - claimed_node_ads = coll.query(htcondor.AdTypes.Startd, - constraint=filter_claimed_vms.and_(filter_mig), - projection=[NODENAME_ATTRIBUTE]) - claimed_nodes = [ ad[NODENAME_ATTRIBUTE].split(".")[0] for ad in claimed_node_ads] - - # treat OrderedDict as a set by ignoring key values; this set will - # contain VMs we would consider deleting, in inverse order of - # their readiness to join pool (creating, unhealthy, healthy+idle) - idle_nodes = OrderedDict() - try: - modifyingInstances = respModifyingInstances["managedInstances"] - except KeyError: - modifyingInstances = [] - - print(f"There are {len(modifyingInstances)} VMs being modified by the managed instance group") - - # there is potential for nodes in MIG health check "VERIFYING" state - # to have already joined the pool and be running jobs - for instance in modifyingInstances: - self_link = instance["instance"] - node_name = self_link.rsplit("/", 1)[-1] - if node_name not in claimed_nodes: - idle_nodes[self_link] = "modifying" - - for ad in idle_node_ads: - node = ad["Machine"].split(".")[0] - zone = ad["CloudZone"] - self_link = "https://www.googleapis.com/compute/v1/projects/" + \ - self.project + "/zones/" + zone + "/instances/" + node - # there is potential for nodes in MIG health check "VERIFYING" state - # to have already joined the pool and be idle; delete them last - if self_link in idle_nodes: - idle_nodes.move_to_end(self_link) - idle_nodes[self_link] = "idle" - n_idle = len(idle_nodes) - - print(f"There are {n_idle} VMs being modified or idle in the pool") - if self.debug > 1: - print("Listing idle nodes:") - pprint(idle_nodes) - - # always keep size tending toward the minimum idle VMs requested - new_target = current_target + self.compute_instance_min_idle - n_idle + min_hosts_for_idle_jobs - if new_target > self.compute_instance_limit: - self.size = self.compute_instance_limit - print(f"MIG target size will be limited by {self.compute_instance_limit}") - else: - self.size = new_target - - print(f"New MIG target size: {self.size}") - - if self.debug > 1: - print("MIG Information:") - print(responseGroupInfo) - - if self.size == current_target: - if current_target == 0: - print("Queue is empty") - print("Running correct number of VMs to handle queue") - exit() - - if self.size < current_target: - print("Scaling down. Looking for nodes that can be shut down") - - if self.debug > 1: - print("Compute node busy status:") - for node in idle_nodes: - print(node) - - # Shut down idle nodes up to our calculated limit - nodes_to_delete = list(idle_nodes.keys())[0:current_target-self.size] - for node in nodes_to_delete: - print(f"Attempting to delete: {node.rsplit('/',1)[-1]}") - respDel = self.deleteFromMig(nodes_to_delete) - - if self.debug > 1: - print("Scaling down complete") - - if self.size > current_target: - print( - "Scaling up. Need to increase number of instances to " + str(self.size) - ) - # Request to resize - request = self.instanceGroupManagers.resize( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - size=self.size, - ) - response = request.execute() - if self.debug > 1: - print("Requesting to increase MIG size") - pprint(response) - print("Scaling up complete") - - -def main(): - - scaler = AutoScaler(args.mz) - - # Project ID - scaler.project = args.p # Ex:'slurm-var-demo' - - # Name of the zone where the managed instance group is located - scaler.zone = args.z # Ex: 'us-central1-f' - - # Name of the region where the managed instance group is located - scaler.region = args.r # Ex: 'us-central1' - - # The name of the managed instance group. - scaler.instance_group_manager = args.g # Ex: 'condor-compute-igm' - - # Default number of cores per instance, will be replaced with actual value - scaler.cores_per_node = 4 - - # Default number of running instances that the managed instance group should maintain at any given time. This number will go up and down based on the load (number of jobs in the queue) - scaler.size = 0 - - scaler.compute_instance_min_idle = args.i - - # Dry run: : 0, run scaling; 1, only provide info. - scaler.dryrun = args.d > 0 - - # Debug level: 1-print debug information, 2 - print detail debug information - scaler.debug = 0 - if args.v: - scaler.debug = args.v - - # Limit for the maximum number of compute instance. If zero (default setting), no limit will be enforced by the script - scaler.compute_instance_limit = 0 - if args.c: - scaler.compute_instance_limit = abs(args.c) - - scaler.scale() - - -if __name__ == "__main__": - main() diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml deleted file mode 100644 index db989f9d40..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Install but do not activate HTCondor autoscaler - become: true - hosts: localhost - tasks: - - name: Install Python 3 pip - ansible.builtin.package: - name: python3-pip - state: present - - name: Create virtual environment for HTCondor autoscaler - ansible.builtin.pip: - name: pip - version: 21.3.1 # last Python 3.6-compatible release - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv - - name: Install latest setuptools - ansible.builtin.pip: - name: setuptools - version: 59.6.0 # last Python 3.6-compatible release - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv - - name: Install HTCondor autoscaler dependencies - with_items: - - oauth2client - - google-api-python-client - - absl-py - - htcondor - ansible.builtin.pip: - name: "{{ item }}" - state: present # rely on pip resolver to pick latest compatible releases - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml deleted file mode 100644 index 4d3abbbfd6..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The instructions for installing HTCondor may change with time, although we -# anticipate that they will stay fixed for the 23.0 releases. Find up-to-date -# recommendations at: -## https://htcondor.readthedocs.io/en/latest/getting-htcondor/from-our-repositories.html - ---- -- name: Ensure HTCondor is installed - hosts: all - vars: - enable_docker: true - htcondor_key: https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-23.0-Key - docker_key: https://download.docker.com/linux/centos/gpg - become: true - module_defaults: - ansible.builtin.yum: - lock_timeout: 300 - tasks: - - name: Enable EPEL repository - ansible.builtin.yum: - name: - - epel-release - - name: Directly install RPM verification keys - ansible.builtin.rpm_key: - state: present - key: "{{ item }}" - loop: - - "{{ htcondor_key }}" - - "{{ docker_key }}" - register: key_install - retries: 10 - delay: 60 - until: key_install is success - - name: Enable HTCondor LTS Release repository - ansible.builtin.yum_repository: - name: htcondor-feature - description: HTCondor LTS Release (23.0) - file: htcondor - baseurl: https://research.cs.wisc.edu/htcondor/repo/23.0/el$releasever/$basearch/release - gpgkey: "{{ htcondor_key }}" - gpgcheck: true - repo_gpgcheck: true - priority: "90" - - name: Install HTCondor - ansible.builtin.yum: - name: condor-{{ condor_version | default("23.*") | string }} - state: present - - name: Ensure token directory - ansible.builtin.file: - path: /etc/condor/tokens.d - mode: 0700 - owner: root - group: root - - name: Install Docker and configure HTCondor to use it - when: enable_docker | bool # allows string to be passed at CLI - block: - - name: Setup Docker repo - ansible.builtin.yum_repository: - name: docker-ce-stable - description: Docker CE Stable - $basearch - baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable - enabled: yes - gpgcheck: yes - gpgkey: "{{ docker_key }}" - - name: Install Docker - ansible.builtin.yum: - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-compose-plugin - - name: Enable Docker - ansible.builtin.service: - name: docker - state: started - enabled: true - - name: Add condor to docker group - ansible.builtin.user: - name: condor - groups: docker - append: yes diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf deleted file mode 100644 index 0853e035f4..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - runners = [ - { - "type" = "ansible-local" - "source" = "${path.module}/files/install-htcondor.yaml" - "destination" = "install-htcondor.yaml" - "args" = join(" ", [ - "-e enable_docker=${var.enable_docker}", - "-e condor_version=${var.condor_version}", - ]) - }, - { - "type" = "ansible-local" - "content" = file("${path.module}/files/install-htcondor-autoscaler-deps.yml") - "destination" = "install-htcondor-autoscaler-deps.yml" - }, - { - "type" = "data" - "content" = file("${path.module}/files/autoscaler.py") - "destination" = "/usr/local/htcondor/bin/autoscaler.py" - }, - ] - - install_htcondor_ps1 = templatefile( - "${path.module}/templates/install-htcondor.ps1.tftpl", { - condor_version = var.condor_version, - http_proxy = var.http_proxy, - python_windows_installer_url = var.python_windows_installer_url, - }) - - required_apis = [ - "compute.googleapis.com", - "secretmanager.googleapis.com", - ] -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf deleted file mode 100644 index c7951737ff..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "runners" { - description = "Runner to install HTCondor using startup-scripts" - value = local.runners -} - -output "windows_startup_ps1" { - description = "Windows PowerShell script to install HTCondor" - value = local.install_htcondor_ps1 -} - -output "gcp_service_list" { - description = "Google Cloud APIs required by HTCondor" - value = local.required_apis -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl deleted file mode 100644 index 7492da3c12..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl +++ /dev/null @@ -1,59 +0,0 @@ -#Requires -RunAsAdministrator - -# Windows 2016 needs forced upgrade to TLS 1.2 -[Net.ServicePointManager]::SecurityProtocol = 'Tls12' - -# important for catching exception in Invoke-WebRequest -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -%{ if http_proxy != "" ~} -[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") -%{ endif ~} - -# do not show progress bar when running Invoke-WebRequest -$ProgressPreference = 'SilentlyContinue' - -# download C Runtime DLL necessary for HTCondor installer -$runtime_installer = 'C:\vc_redist.x64.exe' -Invoke-WebRequest https://aka.ms/vs/17/release/vc_redist.x64.exe -OutFile "$runtime_installer" -Start-Process -FilePath "$runtime_installer" -Wait -ArgumentList "/norestart /quiet /log c:\vc_redist_log.txt" -Remove-Item "$runtime_installer" - -# download HTCondor installer -$htcondor_installer = 'C:\htcondor.msi' -%{ if condor_version == "23.*" } -Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/current/condor-Windows-x64.msi -OutFile "$htcondor_installer" -%{ else ~} -Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/${condor_version}/release/condor-${condor_version}-Windows-x64.msi -OutFile "$htcondor_installer" -%{ endif ~} -$args='/qn /l* condor-install-log.txt /i' -$args=$args + " $htcondor_installer" -$args=$args + ' NEWPOOL="N"' -$args=$args + ' RUNJOBS="N"' -$args=$args + ' SUBMITJOBS="N"' -$args=$args + ' INSTALLDIR="C:\Condor"' -Start-Process "msiexec.exe" -Wait -ArgumentList "$args" -Remove-Item "$htcondor_installer" - -# do not start HTCondor on boot by default. Allow startup script to download -# configuration first and then start HTCondor -Set-Service -StartupType Manual condor - -# remove settings from condor_config that we want to override in configuration step -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^CONDOR_HOST' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^INSTALL_USER' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^DAEMON_LIST' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^use SECURITY' -NotMatch) - -# install Python so that custom ClassAd hooks can execute -$python_installer = 'C:\python-installer.exe' -Invoke-WebRequest -Uri "${python_windows_installer_url}" -OutFile "$python_installer" -Start-Process -FilePath "$python_installer" -Wait -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1 Include_test=0' -%{ if http_proxy == "" ~} -Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --no-warn-script-location requests" -%{ else ~} -Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --proxy ${http_proxy} --no-warn-script-location requests" -%{ endif ~} -Invoke-WebRequest -Uri "https://raw.githubusercontent.com/htcondor/htcondor/main/src/condor_scripts/common-cloud-attributes-google.py" -OutFile "C:\Condor\bin\common-cloud-attributes-google.py" -Remove-Item "$python_installer" diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf deleted file mode 100644 index 1afdf4e0eb..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "enable_docker" { - description = "Install and enable docker daemon alongside HTCondor" - type = bool - default = true -} - -variable "condor_version" { - description = "Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: \"23.0.0\",\"23.*\"))" - type = string - default = "23.*" - - validation { - error_message = "var.condor_version must be set to \"23.*\" for latest 23.0 release or to a specific \"23.0.y\" release." - condition = var.condor_version == "23.*" || ( - length(split(".", var.condor_version)) == 3 && alltrue([ - for v in split(".", var.condor_version) : can(tonumber(v)) - ]) && split(".", var.condor_version)[0] == "23" - && split(".", var.condor_version)[1] == "0" - ) - } -} - -variable "http_proxy" { - description = "Set system default web (http and https) proxy for Windows HTCondor installation" - type = string - default = "" - nullable = false -} - -variable "python_windows_installer_url" { - description = "URL of Python installer for Windows" - type = string - default = "https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe" - nullable = false -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md deleted file mode 100644 index 55c2fc7e4e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md +++ /dev/null @@ -1,116 +0,0 @@ -## Description - -This module will create a startup-script runner that will execute Ramble commands. - -Ramble is a multi-platform experimentation framework capable of driving -software installation, acquiring input files, configuring experiments, and -extracting results. For more information about Ramble, see: -https://github.com/GoogleCloudPlatform/ramble - -This module outputs a startup script runner, which can be combined with other -startup script runners to execute a set of Ramble commands. - -Ramble makes extensive use of Spack. It must be installed with a Toolkit runner -generated by the [spack-setup module](../spack-setup/README.md) following the -[basic example](#basic-example) below. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -# Examples - -## Basic Example - -Below is a basic example of using this module. - -```yaml - - id: spack - source: community/modules/scripts/spack-setup - - - id: ramble-setup - source: community/modules/scripts/ramble-setup - - - id: ramble-execute - source: community/modules/scripts/ramble-execute - use: [spack, ramble-setup] - settings: - commands: - - ramble list -``` - -This example shows installing Spack and Ramble with their own modules -(spack-setup and ramble-setup respectively). Then the ramble-execute module -is added to simply list all applications Ramble knows about. - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | -| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [log\_file](#input\_log\_file) | Log file to write output from Ramble execute steps into | `string` | `"/var/log/ramble-execute.log"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by an instance of ramble-setup.
Can be defined explicitly, or by chaining an instance of a ramble-setup module
through a `use` setting. | `string` | n/a | yes | -| [ramble\_runner](#input\_ramble\_runner) | Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | -| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting.
Defaults to /etc/profile.d/spack.sh if not set. | `string` | `"/etc/profile.d/spack.sh"` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the ramble-setup module. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble startup script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for ramble, to be reused by ramble-execute module. | -| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | -| [ramble\_runner](#output\_ramble\_runner) | Runner to execute Ramble commands using an ansible playbook. The startup-script module
will automatically handle installation of ansible. | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to Spack profile script. | -| [startup\_script](#output\_startup\_script) | Ramble startup script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf deleted file mode 100644 index 7ef0b029e3..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ramble-execute", ghpc_role = "scripts" }) -} - -locals { - commands_content = var.commands == null ? "echo 'no ramble commands provided'" : indent(4, yamlencode(var.commands)) - - execute_contents = templatefile( - "${path.module}/templates/ramble_execute.yml.tpl", - { - pre_script = "if [ -f ${var.spack_profile_script_path} ]; then . ${var.spack_profile_script_path}; fi; . ${var.ramble_profile_script_path}" - log_file = var.log_file - commands = local.commands_content - system_user_name = var.system_user_name - } - ) - - data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] - - execute_md5 = substr(md5(local.execute_contents), 0, 4) - execute_runner = { - type = "ansible-local" - content = local.execute_contents - destination = "ramble_execute_${local.execute_md5}.yml" - } - - previous_runners = var.ramble_runner != null ? [var.ramble_runner] : [] - runners = concat(local.previous_runners, local.data_runners, [local.execute_runner]) - - # Destinations should be unique while also being known at time of apply - combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) - combined_md5 = substr(md5(local.combined_unique_string), 0, 4) - combined_runner = { - type = "shell" - content = module.startup_script.startup_script - destination = "combined_install_ramble_${local.combined_md5}.sh" - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = var.gcs_bucket_path -} - -resource "local_file" "debug_file_ansible_execute" { - content = local.execute_contents - filename = "${path.module}/debug_execute_${local.execute_md5}.yml" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf deleted file mode 100644 index 4e6c3a44d8..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Ramble startup script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Ramble startup script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "ramble_runner" { - description = <<-EOT - Runner to execute Ramble commands using an ansible playbook. The startup-script module - will automatically handle installation of ansible. - EOT - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for ramble, to be reused by ramble-execute module." - value = var.gcs_bucket_path -} - -output "spack_profile_script_path" { - description = "Path to Spack profile script." - value = var.spack_profile_script_path -} - -output "ramble_profile_script_path" { - description = "Path to Ramble profile script." - value = var.ramble_profile_script_path -} - -output "system_user_name" { - description = "The system user used to execute commands." - value = var.system_user_name -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl deleted file mode 100644 index 0e98f3aa2c..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Execute Commands - hosts: localhost - vars: - pre_script: ${pre_script} - log_file: ${log_file} - commands: ${commands} - system_user_name: ${system_user_name} - tasks: - - name: Execute command block - block: - - name: Print commands to be executed - ansible.builtin.debug: - msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" - - - name: Streaming log info - ansible.builtin.debug: - msg: | - Logs from commands will not be printed here until success (or failure) - Streaming logs can be found at {{ log_file }} - - - name: Ensure user can write to log file - ansible.builtin.file: - path: "{{ log_file }}" - state: touch - owner: "{{ system_user_name }}" - - - name: Execute commands - ansible.builtin.shell: | - set -eo pipefail - { - {{ pre_script }} - echo " === Starting commands ===" - {{ commands }} - echo " === Finished commands ===" - } 2>&1 | tee -a {{ log_file }} - args: - executable: /bin/bash - register: output - become: true - become_user: "{{ system_user_name }}" - - always: - - name: Print commands output - ansible.builtin.debug: - var: output.stdout_lines diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf deleted file mode 100644 index ec67228df5..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing spack scripts." - type = string -} - -variable "region" { - description = "Region to place bucket containing spack scripts." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "log_file" { - description = "Log file to write output from Ramble execute steps into" - default = "/var/log/ramble-execute.log" - type = string -} - -variable "data_files" { - description = <<-EOT - A list of files to be transferred prior to running commands. - It must specify one of 'source' (absolute local file path) or 'content' (string). - It must specify a 'destination' with absolute path where file should be placed. - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) - error_message = "All destinations must be absolute paths and start with '/'." - } - validation { - condition = alltrue([ - for r in var.data_files : - can(r["content"]) != can(r["source"]) - ]) - error_message = "A data_file must specify either 'content' or 'source', but never both." - } - validation { - condition = alltrue([ - for r in var.data_files : - lookup(r, "content", lookup(r, "source", null)) != null - ]) - error_message = "A data_file must specify a non-null 'content' or 'source'." - } -} - -variable "commands" { - description = "String of commands to run within this module" - default = null - type = string -} - -variable "ramble_runner" { - description = "Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module." - type = object({ - type = string - content = string - destination = string - }) -} - -variable "system_user_name" { - description = "Name of the system user used to execute commands. Generally passed from the ramble-setup module." - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string -} - -variable "spack_profile_script_path" { - description = <<-EOT - Path to the Spack profile.d script. - Can be defined explicitly, or by chaining an instance of a spack-setup module - through a `use` setting. - Defaults to /etc/profile.d/spack.sh if not set. - EOT - type = string - default = "/etc/profile.d/spack.sh" -} - -variable "ramble_profile_script_path" { - description = <<-EOT - Path to the Ramble profile.d script. Created by an instance of ramble-setup. - Can be defined explicitly, or by chaining an instance of a ramble-setup module - through a `use` setting. - EOT - type = string -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf deleted file mode 100644 index 9b23317323..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md deleted file mode 100644 index 9891088105..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module will create a set of startup-script runners that will setup Ramble, -and install Ramble’s dependencies. - -Ramble is a multi-platform experimentation framework capable of driving -software installation, acquiring input files, configuring experiments, and -extracting results. For more information about ramble, see: -https://github.com/GoogleCloudPlatform/ramble - -This module outputs two startup script runners, which can be added to startup -scripts to setup, ramble and its dependencies. - -For this module to be completely functional, it depends on a spack -installation. For more information, see Cluster-Toolkit’s Spack module. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -# Examples - -## Basic Example - -```yaml -- id: ramble-setup - source: community/modules/scripts/ramble-setup -``` - -This example simply installs ramble on a VM. - -## Full Example - -```yaml -- id: ramble-setup - source: community/modules/scripts/ramble-setup - settings: - install_dir: /ramble - ramble_url: https://github.com/GoogleCloudPlatform/ramble - ramble_ref: v0.2.1 - log_file: /var/log/ramble.log - chown_owner: “owner” - chgrp_group: “user_group” - chmod_mode: “a+r” -``` - -This example simply installs ramble into a VM at the location `/ramble`, checks -out the v0.2.1 tag, changes the owner and group to “owner” and “user_group”, -and chmod’s the clone to make it world readable. - -Also see a more complete [Ramble example blueprint](../../../examples/ramble.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 4.42 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [chmod\_mode](#input\_chmod\_mode) | Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify).
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | -| [install\_dir](#input\_install\_dir) | Destination directory of installation of Ramble. | `string` | `"/apps/ramble"` | no | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by this module | `string` | `"/etc/profile.d/ramble.sh"` | no | -| [ramble\_ref](#input\_ramble\_ref) | Git ref to checkout for Ramble. | `string` | `"develop"` | no | -| [ramble\_url](#input\_ramble\_url) | URL for Ramble repository to clone. | `string` | `"https://github.com/GoogleCloudPlatform/ramble"` | no | -| [ramble\_virtualenv\_path](#input\_ramble\_virtualenv\_path) | Virtual environment path in which to install Ramble Python interpreter and other dependencies | `string` | `"/usr/local/ramble-python"` | no | -| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | -| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Ramble. It will be created if it does not exist. | `string` | `"ramble"` | no | -| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble installation script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module. | -| [ramble\_path](#output\_ramble\_path) | Location ramble is installed into. | -| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | -| [ramble\_ref](#output\_ramble\_ref) | Git ref the ramble install is checked out to use | -| [ramble\_runner](#output\_ramble\_runner) | Runner to be used with startup-script module or passed to ramble-execute module.
- installs Ramble dependencies
- installs Ramble
- generates profile.d script to enable access to Ramble
This is safe to run in parallel by multiple machines. | -| [startup\_script](#output\_startup\_script) | Ramble installation script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf deleted file mode 100644 index 4389af7d33..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ramble-setup", ghpc_role = "scripts" }) -} - -locals { - profile_script = <<-EOF - if [ -f ${var.install_dir}/share/ramble/setup-env.sh ]; then - test -t 1 && echo "** Ramble's python virtualenv (/usr/local/ramble-python) is activated. Call 'deactivate' to deactivate." - VIRTUAL_ENV_DISABLE_PROMPT=1 . ${var.ramble_virtualenv_path}/bin/activate - . ${var.install_dir}/share/ramble/setup-env.sh - fi - EOF - - script_content = templatefile( - "${path.module}/templates/ramble_setup.yml.tftpl", - { - sw_name = "ramble" - profile_script = indent(4, yamlencode(local.profile_script)) - install_dir = var.install_dir - git_url = var.ramble_url - git_ref = var.ramble_ref - chmod_mode = var.chmod_mode - system_user_name = var.system_user_name - system_user_uid = var.system_user_uid - system_user_gid = var.system_user_gid - finalize_setup_script = "echo 'no finalize setup script'" - profile_script_path = var.ramble_profile_script_path - } - ) - - install_ramble_deps_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/install_ramble_deps.yml" - "destination" = "install_ramble_deps.yml" - "args" = "-e virtualenv_path=${var.ramble_virtualenv_path}" - } - - python_reqs_content = templatefile( - "${path.module}/templates/install_ramble_python_deps.yml.tftpl", - { - install_dir = var.install_dir - virtualenv_path = var.ramble_virtualenv_path - } - ) - - python_reqs_runner = { - "type" = "ansible-local" - "content" = local.python_reqs_content - "destination" = "install_ramble_reqs.yml" - } - - install_ramble_runner = { - "type" = "ansible-local" - "content" = local.script_content - "destination" = "install_ramble.yml" - } - - bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}"), 0, 8) - # Max bucket name length is 63, so truncate deployment_name if necessary. - # The string "-ramble-scripts-" is 16 characters and bucket_md5 is 8 characters, - # leaving 63-16-8=39 chars for deployment_name. - bucket_name = "${substr(var.deployment_name, 0, 39)}-ramble-scripts-${local.bucket_md5}" - runners = [local.install_ramble_deps_runner, local.install_ramble_runner, local.python_reqs_runner] - - combined_runner = { - "type" = "shell" - "content" = module.startup_script.startup_script - "destination" = "ramble-install-and-setup.sh" - } - -} - -resource "google_storage_bucket" "bucket" { - project = var.project_id - name = local.bucket_name - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" -} - -resource "local_file" "debug_file_shell_install" { - content = local.script_content - filename = "${path.module}/debug_install.yml" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf deleted file mode 100644 index e587470eac..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Ramble installation script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Ramble installation script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "ramble_runner" { - description = <<-EOT - Runner to be used with startup-script module or passed to ramble-execute module. - - installs Ramble dependencies - - installs Ramble - - generates profile.d script to enable access to Ramble - This is safe to run in parallel by multiple machines. - EOT - value = local.combined_runner -} - -output "ramble_path" { - description = "Location ramble is installed into." - value = var.install_dir -} - -output "ramble_ref" { - description = "Git ref the ramble install is checked out to use" - value = var.ramble_ref -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module." - value = "gs://${google_storage_bucket.bucket.name}" -} - -output "ramble_profile_script_path" { - description = "Path to Ramble profile script." - value = var.ramble_profile_script_path -} - -output "system_user_name" { - description = "The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands." - value = var.system_user_name -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml deleted file mode 100644 index b7905bbe9e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Create python virtual env for a tool - become: yes - hosts: localhost - vars: - virtualenv_path: ${virtualenv_path} - tasks: - - name: Install dependencies through system package manager - ansible.builtin.package: - name: - - python3 - - python3-pip - - git - register: package - changed_when: package.changed - retries: 5 - delay: 10 - until: package is success - - - name: Create virtualenv for tool - # Python 3.6 is minimum we wish to support due to ease of installation on - # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip - # supported by 3.6. Additionally, recent versions of pip are necessary for - # proper dependency resolution of real-world problems with google-cloud-* - # (and third-party) Python packages (20.3+ probably effective minimum). - ansible.builtin.pip: - name: pip>=21.3.1 - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv - - - name: Add google-cloud-storage to virtualenv - ansible.builtin.pip: - name: google-cloud-storage - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl deleted file mode 100644 index ea14780a58..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Install Python Requirements - hosts: localhost - vars: - install_dir: ${install_dir} - virtualenv_path: ${virtualenv_path} - tasks: - - - name: Install dependencies - ansible.builtin.pip: - requirements: "{{ install_dir }}/requirements.txt" - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl deleted file mode 100644 index ca48a5afa0..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Install Software - hosts: localhost - vars: - sw_name: ${sw_name} - profile_script: ${profile_script} - install_dir: ${install_dir} - git_url: ${git_url} - git_ref: ${git_ref} - chmod_mode: ${chmod_mode} - system_user_name: ${system_user_name} - system_user_uid: ${system_user_uid} - system_user_gid: ${system_user_gid} - finalize_setup_script: ${finalize_setup_script} - profile_script_path: ${profile_script_path} - tasks: - - name: Print software name - ansible.builtin.debug: - msg: "Running installation for software: {{ sw_name }}" - - - name: Add profile script for software - ansible.builtin.copy: - dest: "{{ profile_script_path }}" - mode: '0644' - content: "{{ profile_script }}" - when: profile_script - - - name: Look up user to use for install - block: - - - name: Check if user already exists - ansible.builtin.getent: - database: passwd - key: "{{ system_user_name }}" - - - name: Look up existing user details - ansible.builtin.user: - name: "{{ system_user_name }}" - register: system_user - - rescue: - - name: User did not exist, create group for system user - ansible.builtin.group: - name: "{{ system_user_name }}" - gid: "{{ system_user_gid }}" - system: true - register: system_group - - - name: Create system user - ansible.builtin.user: - name: "{{ system_user_name }}" - comment: "{{ sw_name }} installation" - uid: "{{ system_user_uid }}" - group: "{{ system_group.name }}" - system: true - register: system_user - - - name: Create parent of install directory - ansible.builtin.file: - path: "{{ install_dir | dirname }}" - state: directory - - - name: Set lock dir - ansible.builtin.set_fact: - lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" - - - name: Acquire lock - ansible.builtin.command: - mkdir "{{ lock_dir }}" - register: lock_out - changed_when: lock_out.rc == 0 - failed_when: false - - - name: Add hostname to lock_dir - ansible.builtin.file: - path: "{{ lock_dir }}/{{ ansible_hostname }}" - state: touch - when: lock_out.rc == 0 - - - name: Clone branch or tag into installation directory - ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} - failed_when: false - register: clone_res - when: lock_out.rc == 0 - - - name: Clone commit hash into installation directory - ansible.builtin.command: "{{ item }}" - with_items: - - git clone {{ git_url }} {{ install_dir }} - - git -C {{ install_dir }} checkout {{ git_ref }} - when: lock_out.rc == 0 and clone_res.rc != 0 - - - name: Transfer ownership to system user - ansible.builtin.file: - path: "{{ install_dir }}" - owner: "{{ system_user.name }}" - group: "{{ system_user.group }}" - recurse: true - follow: false - when: lock_out.rc == 0 - - - name: Finalize setup - ansible.builtin.shell: "{{ finalize_setup_script }}" - when: lock_out.rc == 0 and finalize_setup_script - become: true - become_user: "{{ system_user.name }}" - - - name: Apply chmod - ansible.builtin.file: - path: "{{ install_dir }}" - mode: "{{ chmod_mode | default(omit, true) }}" - recurse: true - follow: false - when: (lock_out.rc == 0) and (chmod_mode != None) - - - name: Release lock - ansible.builtin.file: - path: "{{ lock_dir }}/done" - state: touch - when: lock_out.rc == 0 - - - name: Wait for lock - block: - - name: Wait for lock - ansible.builtin.wait_for: - path: "{{ lock_dir }}/done" - state: present - timeout: 600 - sleep: 10 - when: lock_out.rc != 0 - - rescue: - - name: Timed out on waiting for lock, get lock directory contents - ansible.builtin.find: - paths: "{{ lock_dir }}" - register: lock_dir_contents - - - name: Print lock directory contents, it should contain name of host that is holding lock - ansible.builtin.debug: - msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" - - - name: Failed to get lock - ansible.builtin.fail: - msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf deleted file mode 100644 index 0d3a8eed05..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "install_dir" { - description = "Destination directory of installation of Ramble." - default = "/apps/ramble" - type = string -} - -variable "ramble_url" { - description = "URL for Ramble repository to clone." - default = "https://github.com/GoogleCloudPlatform/ramble" - type = string -} - -variable "ramble_ref" { - description = "Git ref to checkout for Ramble." - default = "develop" - type = string -} - -variable "chmod_mode" { - description = <<-EOT - Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify). - For usage information see: - https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode - EOT - default = "" - type = string - nullable = false -} - -variable "system_user_name" { - description = "Name of system user that will perform installation of Ramble. It will be created if it does not exist." - default = "ramble" - type = string - nullable = false -} - -variable "system_user_uid" { - description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." - default = 1104762904 - type = number - nullable = false -} - -variable "system_user_gid" { - description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." - default = 1104762904 - type = number - nullable = false -} - -variable "ramble_virtualenv_path" { - description = "Virtual environment path in which to install Ramble Python interpreter and other dependencies" - default = "/usr/local/ramble-python" - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing startup script." - type = string -} - -variable "region" { - description = "Region to place bucket containing startup script." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "ramble_profile_script_path" { - description = "Path to the Ramble profile.d script. Created by this module" - type = string - default = "/etc/profile.d/ramble.sh" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf deleted file mode 100644 index 936b4a5b80..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/README.md deleted file mode 100644 index 8cbb75fb42..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/README.md +++ /dev/null @@ -1,141 +0,0 @@ -## Description - -This module creates a script that defines a software build using Spack and -performs any additional customization to a Spack installation. - -There are two main variable inputs that can be used to define a Spack build: -`data_files` and `commands`. - -- `data_files`: Any files specified will be transferred to the machine running - outputted script. Data file `content` can be defined inline in the blueprint - or can point to a `source`, an absolute local path of a file. This can be used - to transfer environment definition files, config definition files, GPG keys, - or software licenses. `data_files` are transferred before `commands` are run. -- `commands`: A script that is run. This can be used to perform actions such as - installation of compilers & packages, environment creation, adding a build - cache, and modifying the spack configuration. - -## Example - -The `spack-execute` module should `use` a `spack-setup` module. This will -prepend the installation of Spack and its dependencies to the build. Then -`spack-execute` can be used by a module that takes `startup-script` as an input. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: builder-vm - source: modules/compute/vm-instance - use: [network1, spack-build] -``` - -To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. - -[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml - -### Using with `startup-script` module - -The `spack-runner` output can be used by the `startup-script` module. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(spack-build.spack-runner) - - type: shell - destination: "my-script.sh" - content: echo 'hello world' - - - id: workstation - source: modules/compute/vm-instance - use: [network1, startup-script] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | -| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [log\_file](#input\_log\_file) | Defines the logfile that script output will be written to | `string` | `"/var/log/spack.log"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by an instance of spack-setup.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting. | `string` | n/a | yes | -| [spack\_runner](#input\_spack\_runner) | Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | -| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the spack-setup module. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack startup script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | -| [spack\_runner](#output\_spack\_runner) | Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules. | -| [startup\_script](#output\_startup\_script) | Spack startup script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf deleted file mode 100644 index 04ebcf7d49..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "spack-execute", ghpc_role = "scripts" }) -} - -locals { - commands_content = var.commands == null ? "echo 'no spack commands provided'" : indent(4, yamlencode(var.commands)) - - execute_contents = templatefile( - "${path.module}/templates/execute_commands.yml.tpl", - { - pre_script = ". ${var.spack_profile_script_path}" - log_file = var.log_file - commands = local.commands_content - system_user_name = var.system_user_name - } - ) - - data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] - - execute_md5 = substr(md5(local.execute_contents), 0, 4) - execute_runner = { - type = "ansible-local" - content = local.execute_contents - destination = "spack_execute_${local.execute_md5}.yml" - } - - runners = concat([var.spack_runner], local.data_runners, [local.execute_runner]) - - # Destinations should be unique while also being known at time of apply - combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) - combined_md5 = substr(md5(local.combined_unique_string), 0, 4) - combined_runner = { - type = "shell" - content = module.startup_script.startup_script - destination = "combined_install_spack_${local.combined_md5}.sh" - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = var.gcs_bucket_path -} - -resource "local_file" "debug_file_ansible_execute" { - content = local.execute_contents - filename = "${path.module}/debug_execute_${local.execute_md5}.yml" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf deleted file mode 100644 index 4a52532d51..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Spack startup script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Spack startup script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "spack_runner" { - description = "Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules." - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." - value = var.gcs_bucket_path -} - -output "spack_profile_script_path" { - description = "Path to the Spack profile.d script." - value = var.spack_profile_script_path -} - -output "system_user_name" { - description = "The system user used to execute commands." - value = var.system_user_name -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl deleted file mode 100644 index 0e98f3aa2c..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Execute Commands - hosts: localhost - vars: - pre_script: ${pre_script} - log_file: ${log_file} - commands: ${commands} - system_user_name: ${system_user_name} - tasks: - - name: Execute command block - block: - - name: Print commands to be executed - ansible.builtin.debug: - msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" - - - name: Streaming log info - ansible.builtin.debug: - msg: | - Logs from commands will not be printed here until success (or failure) - Streaming logs can be found at {{ log_file }} - - - name: Ensure user can write to log file - ansible.builtin.file: - path: "{{ log_file }}" - state: touch - owner: "{{ system_user_name }}" - - - name: Execute commands - ansible.builtin.shell: | - set -eo pipefail - { - {{ pre_script }} - echo " === Starting commands ===" - {{ commands }} - echo " === Finished commands ===" - } 2>&1 | tee -a {{ log_file }} - args: - executable: /bin/bash - register: output - become: true - become_user: "{{ system_user_name }}" - - always: - - name: Print commands output - ansible.builtin.debug: - var: output.stdout_lines diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf deleted file mode 100644 index 851cd1aed8..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing spack scripts." - type = string -} - -variable "region" { - description = "Region to place bucket containing spack scripts." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "log_file" { - description = "Defines the logfile that script output will be written to" - default = "/var/log/spack.log" - type = string -} - -variable "data_files" { - description = <<-EOT - A list of files to be transferred prior to running commands. - It must specify one of 'source' (absolute local file path) or 'content' (string). - It must specify a 'destination' with absolute path where file should be placed. - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) - error_message = "All destinations must be absolute paths and start with '/'." - } - validation { - condition = alltrue([ - for r in var.data_files : - can(r["content"]) != can(r["source"]) - ]) - error_message = "A data_file must specify either 'content' or 'source', but never both." - } - validation { - condition = alltrue([ - for r in var.data_files : - lookup(r, "content", lookup(r, "source", null)) != null - ]) - error_message = "A data_file must specify a non-null 'content' or 'source'." - } -} - -variable "commands" { - description = "String of commands to run within this module" - type = string - default = null -} - -variable "spack_runner" { - description = "Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module." - type = object({ - type = string - content = string - destination = string - }) -} - -variable "system_user_name" { - description = "Name of the system user used to execute commands. Generally passed from the spack-setup module." - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string -} - -variable "spack_profile_script_path" { - description = <<-EOT - Path to the Spack profile.d script. Created by an instance of spack-setup. - Can be defined explicitly, or by chaining an instance of a spack-setup module - through a `use` setting. - EOT - type = string -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf deleted file mode 100644 index 09583c3d43..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 1.0.0" - required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/README.md deleted file mode 100644 index 01d3e6d389..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/README.md +++ /dev/null @@ -1,382 +0,0 @@ -## Description - -This module can be used to setup and install Spack on a VM. To actually run -Spack commands to install other software use the -[spack-execute](../spack-execute/) module. - -This module generates a script that performs the following: - -1. Install system dependencies needed for Spack -1. Clone Spack into a predefined directory -1. Check out a specific version of Spack - -There are several options on how to consume the outputs of this module: - -> [!IMPORTANT] -> Breaking changes between after v1.21.0. `spack-install` module replaced by -> `spack-setup` and `spack-execute` modules. -> [Details Below](#deprecations-and-breaking-changes) - -## Examples - -### `use` `spack-setup` with `spack-execute` - -This will prepend the `spack-setup` script to the `spack-execute` commands. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: builder - source: modules/compute/vm-instance - use: [network1, spack-build] -``` - -### `use` `spack-setup` with `vm-instance` or Slurm module - -This will run `spack-setup` scripts on the downstream compute resource. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-installer - source: modules/compute/vm-instance - use: [network1, spack-setup] -``` - -OR - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: [network1, partition1, spack-setup] -``` - -### Build `starup-script` with `spack-runner` output - -This will use the generated `spack-setup` script as one step in `startup-script`. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(spack-setup.spack-runner) - - type: shell - destination: "my-script.sh" - content: echo 'hello world' - - - id: workstation - source: modules/compute/vm-instance - use: [network1, startup-script] -``` - -To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. - -[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml - -## Environment Setup - -### Activating Spack - -[Spack installation] produces a setup script that adds `spack` to your `PATH` as -well as some other command-line integration tools. This script can be found at -`/share/spack/setup-env.sh`. This script will be automatically -added to bash startup by any machine that runs the `spack_runner`. - -If you have multiple machines that all want to use the same shared Spack -installation you can just have both machines run the `spack_runner`. - -[Spack installation]: https://spack-tutorial.readthedocs.io/en/latest/tutorial_basics.html#installing-spack - -### Managing Spack Python dependencies - -Spack is configured with [SPACK_PYTHON] to ensure that Spack itself uses a -Python virtual environment with a supported copy of Python with the package -`google-cloud-storage` pre-installed. This enables Spack to use mirrors and -[build caches][builds] on Google Cloud Storage. It does not configure Python -packages *inside* Spack virtual environments. If you need to add more Python -dependencies for Spack itself, use the `spack python` command: - -```shell -sudo -i spack python -m pip install package-name -``` - -[SPACK_PYTHON]: https://spack.readthedocs.io/en/latest/getting_started.html#shell-support -[builds]: https://spack.readthedocs.io/en/latest/binary_caches.html - -## Spack Permissions - -### System `spack` user is created - Default - -By default this module will create a `spack` linux user and group with -consistent UID and GID. This user and group will own the Spack installation. To -allow a user to manually add Spack packages to the system Spack installation, -you can add the user to the spack group: - -```sh -sudo usermod -a -G spack -``` - -Log out and back in so the group change will take effect, then `` will -be able to call `spack install `. - -> [!NOTE] -> A background persistent SSH connections may prevent the group change from -> taking effect. - -You can use the `system_user_name`, `system_user_uid`, and `system_user_gid` to -customize the name and ids of the system user. While unlikely, it is possible -that the default `system_user_uid` or `system_user_gid` could conflict with -existing UIDs. - -### Use and existing user - -Alternatively, if `system_user_name` is a user already on the system, then this -existing user will be used for Spack installation. - -#### OS Login User - -If OS Login is enabled (default for most Cluster Toolkit modules) then you can -provide an OS Login user name: - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - system_user_name: username_company_com -``` - -This will work even if the user has not yet logged onto the machine. When the -specified user does log on to the machine they will be able to call -`spack install` without any further configuration. - -#### Pre-configured user - -You can also use a startup script to configure a user: - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - system_user_name: special-user - - - id: startup - source: modules/scripts/startup-script - settings: - runners: - - type: shell - destination: "create_user.sh" - content: | - #!/bin/bash - sudo useradd -u 799 special-user - sudo groupadd -g 922 org-group - sudo usermod -g org-group special-user - - $(spack-setup.spack_runner) - - - id: spack-vms - source: modules/compute/vm-instance - use: [network1, startup] - settings: - name_prefix: spack-vm - machine_type: n2d-standard-2 - instance_count: 5 -``` - -### Chaining spack installations - -If there is a need to have a non-root user to install spack packages it is -recommended to create a separate installation for that user and chain Spack installations -([Spack docs](https://spack.readthedocs.io/en/latest/chain.html#chaining-spack-installations)). - -Steps to chain Spack installations: - -1. Get the version of the system Spack: - - ```sh - $ spack --version - - 0.20.0 (e493ab31c6f81a9e415a4b0e0e2263374c61e758) - # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - # Note commit hash and use in next step - ``` - -1. Clone a new spack installation: - - ```sh - git clone -c feature.manyFiles=true https://github.com/spack/spack.git /spack - git -C /spack checkout - ``` - -1. Point the new Spack installation to the system Spack installation. Create a - file at `/spack/etc/spack/upstreams.yaml` with the following - contents: - - ```yaml - upstreams: - spack-instance-1: - install_tree: /sw/spack/opt/spack/ - ``` - -1. Add the following line to your `.bashrc` to make sure the new `spack` is in - your `PATH`. - - ```sh - . /spack/share/spack/setup-env.sh - ``` - -## Deprecations and Breaking Changes - -The old `spack-install` module has been replaced by the `spack-setup` and -`spack-execute` modules. Generally this change strives to allow for a more -flexible definition of a Spack build by using native Spack commands. - -For every deprecated variable from `spack-install` there is documentation on how -to perform the equivalent action using `commands` and `data_files`. The -documentation can be found on the [inputs table](#inputs) below. - -Below is a simple example of the same functionality shown before and after the -breaking changes. - -```yaml - # Before - - id: spack-install - source: community/modules/scripts/spack-install - settings: - install_dir: /sw/spack - compilers: - - gcc@10.3.0 target=x86_64 - packages: - - intel-mpi@2018.4.274%gcc@10.3.0 - -- id: spack-startup - source: modules/scripts/startup-script - settings: - runners: - - $(spack.install_spack_deps_runner) - - $(spack.install_spack_runner) -``` - -```yaml - # After - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - install_dir: /sw/spack - - - id: spack-execute - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - spack load gcc@10.3.0 target=x86_64 - spack compiler find --scope site - spack install intel-mpi@2018.4.274%gcc@10.3.0 - -- id: spack-startup - source: modules/scripts/startup-script - settings: - runners: - - $(spack-execute.spack-runner) -``` - -Although the old `spack-install` module will no longer be maintained, it is -still possible to use the old module in a blueprint by referencing an old -version from GitHub. Note the source line in the following example. - -```yaml - - id: spack-install - source: github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/spack-install?ref=v1.22.1&depth=1 -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 4.42 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [chmod\_mode](#input\_chmod\_mode) | `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification.
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `"g+w"` | no | -| [configure\_for\_google](#input\_configure\_for\_google) | When true, the spack installation will be configured to pull from Google's Spack binary cache. | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | -| [install\_dir](#input\_install\_dir) | Directory to install spack into. | `string` | `"/sw/spack"` | no | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by this module | `string` | `"/etc/profile.d/spack.sh"` | no | -| [spack\_ref](#input\_spack\_ref) | Git ref to checkout for spack. | `string` | `"v0.20.0"` | no | -| [spack\_url](#input\_spack\_url) | URL to clone the spack repo from. | `string` | `"https://github.com/spack/spack"` | no | -| [spack\_virtualenv\_path](#input\_spack\_virtualenv\_path) | Virtual environment path in which to install Spack Python interpreter and other dependencies | `string` | `"/usr/local/spack-python"` | no | -| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Spack. It will be created if it does not exist. | `string` | `"spack"` | no | -| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack installation script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | -| [spack\_path](#output\_spack\_path) | Path to the root of the spack installation | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | -| [spack\_runner](#output\_spack\_runner) | Runner to be used with startup-script module or passed to spack-execute module.
- installs Spack dependencies
- installs Spack
- generates profile.d script to enable access to Spack
This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. | -| [startup\_script](#output\_startup\_script) | Spack installation script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Spack. It can be reused by spack-execute module to install spack packages. | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf deleted file mode 100644 index d45f5d1be3..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "spack-setup", ghpc_role = "scripts" }) -} - -locals { - profile_script = <<-EOF - SPACK_PYTHON=${var.spack_virtualenv_path}/bin/python3 - if [ -f ${var.install_dir}/share/spack/setup-env.sh ]; then - test -t 1 && echo "Running Spack setup, this may take a moment on first login." - . ${var.install_dir}/share/spack/setup-env.sh - fi - EOF - - supported_cache_versions = ["v0.19.0", "v0.20.0"] - cache_version = contains(local.supported_cache_versions, var.spack_ref) ? var.spack_ref : "latest" - add_google_mirror_script = !var.configure_for_google ? "" : <<-EOF - if ! spack mirror list | grep -q google_binary_cache; then - spack mirror add --scope site google_binary_cache gs://spack/${local.cache_version} - spack buildcache keys --install --trust - fi - EOF - - finalize_setup_script = <<-EOF - set -e - . ${var.spack_profile_script_path} - spack config --scope site add 'packages:all:permissions:read:world' - spack config --scope site add 'packages:all:permissions:write:group' - spack gpg init - spack compiler find --scope site - ${local.add_google_mirror_script} - # perform fast install to make sure Spack is fully initialized - spack install xz - spack uninstall --yes-to-all xz - EOF - - script_content = templatefile( - "${path.module}/templates/spack_setup.yml.tftpl", - { - sw_name = "spack" - profile_script = indent(4, yamlencode(local.profile_script)) - install_dir = var.install_dir - git_url = var.spack_url - git_ref = var.spack_ref - chmod_mode = var.chmod_mode - system_user_name = var.system_user_name - system_user_uid = var.system_user_uid - system_user_gid = var.system_user_gid - finalize_setup_script = indent(4, yamlencode(local.finalize_setup_script)) - profile_script_path = var.spack_profile_script_path - } - ) - - install_spack_deps_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/install_spack_deps.yml" - "destination" = "install_spack_deps.yml" - "args" = "-e virtualenv_path=${var.spack_virtualenv_path}" - } - install_spack_runner = { - "type" = "ansible-local" - "content" = local.script_content - "destination" = "install_spack.yml" - } - - bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}.${local.script_content}"), 0, 8) - # Max bucket name length is 63, so truncate deployment_name if necessary. - # The string "-spack-scripts-" is 15 characters and bucket_md5 is 8 characters, - # leaving 63-15-8=40 chars for deployment_name. Using 39 so it has the same prefix as the - # ramble-setup module's GCS bucket. - bucket_name = "${substr(var.deployment_name, 0, 39)}-spack-scripts-${local.bucket_md5}" - runners = [local.install_spack_deps_runner, local.install_spack_runner] - - combined_runner = { - "type" = "shell" - "content" = module.startup_script.startup_script - "destination" = "spack-install-and-setup.sh" - } -} - -resource "google_storage_bucket" "bucket" { - project = var.project_id - name = local.bucket_name - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" -} - -resource "local_file" "debug_file_shell_install" { - content = local.script_content - filename = "${path.module}/debug_install.yml" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml deleted file mode 100644 index 2ada34471f..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf deleted file mode 100644 index d94b9757db..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Spack installation script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Spack installation script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "spack_path" { - description = "Path to the root of the spack installation" - value = var.install_dir -} - -output "spack_runner" { - description = <<-EOT - Runner to be used with startup-script module or passed to spack-execute module. - - installs Spack dependencies - - installs Spack - - generates profile.d script to enable access to Spack - This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. - EOT - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." - value = "gs://${google_storage_bucket.bucket.name}" -} - -output "spack_profile_script_path" { - description = "Path to the Spack profile.d script." - value = var.spack_profile_script_path -} - -output "system_user_name" { - description = "The system user used to install Spack. It can be reused by spack-execute module to install spack packages." - value = var.system_user_name -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml deleted file mode 100644 index b7905bbe9e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Create python virtual env for a tool - become: yes - hosts: localhost - vars: - virtualenv_path: ${virtualenv_path} - tasks: - - name: Install dependencies through system package manager - ansible.builtin.package: - name: - - python3 - - python3-pip - - git - register: package - changed_when: package.changed - retries: 5 - delay: 10 - until: package is success - - - name: Create virtualenv for tool - # Python 3.6 is minimum we wish to support due to ease of installation on - # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip - # supported by 3.6. Additionally, recent versions of pip are necessary for - # proper dependency resolution of real-world problems with google-cloud-* - # (and third-party) Python packages (20.3+ probably effective minimum). - ansible.builtin.pip: - name: pip>=21.3.1 - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv - - - name: Add google-cloud-storage to virtualenv - ansible.builtin.pip: - name: google-cloud-storage - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl deleted file mode 100644 index ca48a5afa0..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Install Software - hosts: localhost - vars: - sw_name: ${sw_name} - profile_script: ${profile_script} - install_dir: ${install_dir} - git_url: ${git_url} - git_ref: ${git_ref} - chmod_mode: ${chmod_mode} - system_user_name: ${system_user_name} - system_user_uid: ${system_user_uid} - system_user_gid: ${system_user_gid} - finalize_setup_script: ${finalize_setup_script} - profile_script_path: ${profile_script_path} - tasks: - - name: Print software name - ansible.builtin.debug: - msg: "Running installation for software: {{ sw_name }}" - - - name: Add profile script for software - ansible.builtin.copy: - dest: "{{ profile_script_path }}" - mode: '0644' - content: "{{ profile_script }}" - when: profile_script - - - name: Look up user to use for install - block: - - - name: Check if user already exists - ansible.builtin.getent: - database: passwd - key: "{{ system_user_name }}" - - - name: Look up existing user details - ansible.builtin.user: - name: "{{ system_user_name }}" - register: system_user - - rescue: - - name: User did not exist, create group for system user - ansible.builtin.group: - name: "{{ system_user_name }}" - gid: "{{ system_user_gid }}" - system: true - register: system_group - - - name: Create system user - ansible.builtin.user: - name: "{{ system_user_name }}" - comment: "{{ sw_name }} installation" - uid: "{{ system_user_uid }}" - group: "{{ system_group.name }}" - system: true - register: system_user - - - name: Create parent of install directory - ansible.builtin.file: - path: "{{ install_dir | dirname }}" - state: directory - - - name: Set lock dir - ansible.builtin.set_fact: - lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" - - - name: Acquire lock - ansible.builtin.command: - mkdir "{{ lock_dir }}" - register: lock_out - changed_when: lock_out.rc == 0 - failed_when: false - - - name: Add hostname to lock_dir - ansible.builtin.file: - path: "{{ lock_dir }}/{{ ansible_hostname }}" - state: touch - when: lock_out.rc == 0 - - - name: Clone branch or tag into installation directory - ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} - failed_when: false - register: clone_res - when: lock_out.rc == 0 - - - name: Clone commit hash into installation directory - ansible.builtin.command: "{{ item }}" - with_items: - - git clone {{ git_url }} {{ install_dir }} - - git -C {{ install_dir }} checkout {{ git_ref }} - when: lock_out.rc == 0 and clone_res.rc != 0 - - - name: Transfer ownership to system user - ansible.builtin.file: - path: "{{ install_dir }}" - owner: "{{ system_user.name }}" - group: "{{ system_user.group }}" - recurse: true - follow: false - when: lock_out.rc == 0 - - - name: Finalize setup - ansible.builtin.shell: "{{ finalize_setup_script }}" - when: lock_out.rc == 0 and finalize_setup_script - become: true - become_user: "{{ system_user.name }}" - - - name: Apply chmod - ansible.builtin.file: - path: "{{ install_dir }}" - mode: "{{ chmod_mode | default(omit, true) }}" - recurse: true - follow: false - when: (lock_out.rc == 0) and (chmod_mode != None) - - - name: Release lock - ansible.builtin.file: - path: "{{ lock_dir }}/done" - state: touch - when: lock_out.rc == 0 - - - name: Wait for lock - block: - - name: Wait for lock - ansible.builtin.wait_for: - path: "{{ lock_dir }}/done" - state: present - timeout: 600 - sleep: 10 - when: lock_out.rc != 0 - - rescue: - - name: Timed out on waiting for lock, get lock directory contents - ansible.builtin.find: - paths: "{{ lock_dir }}" - register: lock_dir_contents - - - name: Print lock directory contents, it should contain name of host that is holding lock - ansible.builtin.debug: - msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" - - - name: Failed to get lock - ansible.builtin.fail: - msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf deleted file mode 100644 index 85baeec401..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -# spack-setup variables - -variable "install_dir" { - description = "Directory to install spack into." - type = string - default = "/sw/spack" -} - -variable "spack_url" { - description = "URL to clone the spack repo from." - type = string - default = "https://github.com/spack/spack" -} - -variable "spack_ref" { - description = "Git ref to checkout for spack." - type = string - default = "v0.20.0" -} - -variable "configure_for_google" { - description = "When true, the spack installation will be configured to pull from Google's Spack binary cache." - type = bool - default = true -} - - -variable "chmod_mode" { - description = <<-EOT - `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification. - For usage information see: - https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode - EOT - default = "g+w" - type = string - nullable = false -} - -variable "system_user_name" { - description = "Name of system user that will perform installation of Spack. It will be created if it does not exist." - default = "spack" - type = string - nullable = false -} - -variable "system_user_uid" { - description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." - default = 1104762903 - type = number - nullable = false -} - -variable "system_user_gid" { - description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." - default = 1104762903 - type = number - nullable = false -} - -variable "spack_virtualenv_path" { - description = "Virtual environment path in which to install Spack Python interpreter and other dependencies" - default = "/usr/local/spack-python" - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing startup script." - type = string -} - -variable "region" { - description = "Region to place bucket containing startup script." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "spack_profile_script_path" { - description = "Path to the Spack profile.d script. Created by this module" - type = string - default = "/etc/profile.d/spack.sh" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf deleted file mode 100644 index ff1180fc1b..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md deleted file mode 100644 index ee9c057c39..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md +++ /dev/null @@ -1,87 +0,0 @@ -## Description - -This module will insert a dependency on the completion of the startup script -for one or more specified compute VMs and report back if it fails. This can be useful when running -post-boot installation scripts that require the startup script to finish setting up a node. - -> **_WARNING:_**: this module is experimental and not fully supported. - -### Additional Dependencies - -* [**gcloud**](https://cloud.google.com/sdk/gcloud) must be present in the path - of the machine where `terraform apply` is run. - -### Example - -```yaml -- id: workstation - source: modules/compute/vm-instance - use: - - network1 - - my-startup-script - settings: - instance_count: 4 - -# Wait for all instances of the above VM to finish running startup scripts. -- id: wait - source: community/modules/scripts/wait-for-startup - settings: - instance_names: $(workstation.name) -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.validate_instance_names](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.wait_for_startup](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | -| [instance\_name](#input\_instance\_name) | Name of the instance we are waiting for (can be null if 'instance\_names' is not empty) | `string` | `null` | no | -| [instance\_names](#input\_instance\_names) | A list of instance names we are waiting for, in addition to the one mentioned in 'instance\_name' (if any) | `list(string)` | `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [timeout](#input\_timeout) | Timeout in seconds | `number` | `1200` | no | -| [zone](#input\_zone) | The GCP zone where the instance is running | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf deleted file mode 100644 index 3f6b416251..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - combined_instance_names = concat(var.instance_names, [var.instance_name]) -} - -resource "null_resource" "validate_instance_names" { - lifecycle { - precondition { - condition = var.instance_name != null || length(var.instance_names) > 0 - error_message = "At least one instance name must be provided" - } - } -} - -resource "null_resource" "wait_for_startup" { - count = length(local.combined_instance_names) - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/wait-for-startup-status.sh" - environment = { - INSTANCE_NAME = self.triggers.instance_name - ZONE = var.zone - PROJECT_ID = var.project_id - TIMEOUT = var.timeout - GCLOUD_PATH = var.gcloud_path_override - } - } - - triggers = { - instance_name = local.combined_instance_names[count.index] - } -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf deleted file mode 100644 index 11a2ddf118..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh deleted file mode 100644 index fae5833121..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -z "${INSTANCE_NAME}" ]]; then - echo "INSTANCE_NAME is unset... exiting" - exit 0 -fi -if [[ -z "${ZONE}" ]]; then - echo "ZONE is unset" - exit 1 -fi -if [[ -z "${PROJECT_ID}" ]]; then - echo "PROJECT_ID is unset" - exit 1 -fi -if [[ -z "${TIMEOUT}" ]]; then - echo "TIMEOUT is unset" - exit 1 -fi - -if [[ -n "${GCLOUD_PATH}" ]]; then - export PATH="$GCLOUD_PATH:$PATH" -fi - -echo "Waiting for startup: instance_name='${INSTANCE_NAME}', zone='${ZONE}', project_id='${PROJECT_ID}', timeout_seconds='${TIMEOUT}'" - -# Wrapper around grep that swallows the error status code 1 -c1grep() { grep "$@" || test $? = 1; } - -now=$(date +%s) - -# If VM was created more than 30 days ago, serial port logs may no longer exist. -# Exit without errors if the instance is older than 30 days. -logsExpiryDays=30 -createdTimestampIso=$(gcloud compute instances describe "${INSTANCE_NAME}" --project "${PROJECT_ID}" --zone "${ZONE}" --format "value(creationTimestamp)") -earliestAllowedCreatedTimestamp=$(date -d "${createdTimestampIso} +${logsExpiryDays} day" +%s) -if [[ "$earliestAllowedCreatedTimestamp" -lt "$now" ]]; then - echo "Instance was created more than 30 days ago - serial port 1 logs are likely expired... exiting" - exit 0 -fi - -deadline=$((now + TIMEOUT)) -error_file=$(mktemp) -fetch_cmd="gcloud compute instances get-serial-port-output ${INSTANCE_NAME} --port 1 --zone ${ZONE} --project ${PROJECT_ID}" -# Match string for all finish types of the old guest agent and successful -# finishes on the new guest agent -FINISH_LINE="startup-script exit status" -# Match string for failures on the new guest agent -FINISH_LINE_ERR="Script \"startup-script\" failed with error:" - -# NEW: Accept also these finish lines as success. -STARTUP_SCRIPT_SUCCEEDED_LINE="google-startup-scripts.service: Succeeded." -STARTUP_SCRIPT_FINISHED_LINE="Finished Google Compute Engine Startup Scripts." -STARTUP_SCRIPT_SERVICE_FINISHED_LINE="Finished google-startup-scripts.service - Google Compute Engine Startup Scripts." - -NON_FATAL_ERRORS=( - "Internal error" -) - -until [[ now -gt deadline ]]; do - ser_log=$( - set -o pipefail - ${fetch_cmd} 2>"${error_file}" | - c1grep "${FINISH_LINE}\|${FINISH_LINE_ERR}\|${STARTUP_SCRIPT_SUCCEEDED_LINE}\|${STARTUP_SCRIPT_FINISHED_LINE}\|${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}" - ) || { - err=$(cat "${error_file}") - echo "$err" - fatal_error="true" - for e in "${NON_FATAL_ERRORS[@]}"; do - if [[ $err = *"$e"* ]]; then - fatal_error="false" - break - fi - done - - if [[ $fatal_error = "true" ]]; then - exit 1 - fi - } - if [[ -n "${ser_log}" ]]; then break; fi - sleep 5 - now=$(date +%s) -done - -# This line checks for an exit code - the assumption is that there is a number -# at the end of the line and it is an exit code. -# Modified to correctly extract the last numeric exit status from the relevant log line. -LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=Script \"startup-script\" failed with error: exit status )[0-9]+" | tail -n 1) -if [[ -z "${LAST_EXIT_STATUS}" ]]; then - LAST_EXIT_STATUS=$(echo "${ser_log}" | grep -oP "(?<=startup-script exit status )[0-9]+" | tail -n 1) -fi - -# This specific text is monitored for in tests, do not change. -INSPECT_OUTPUT_TEXT="To inspect the startup script output, please run:" - -# --- Prioritize explicit failure from the script itself --- -if [[ "${LAST_EXIT_STATUS}" == 1 ]]; then - echo "startup-script finished with errors, ${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" - exit 1 -# --- Then explicit success from the script itself --- -elif [[ "${LAST_EXIT_STATUS}" == 0 ]]; then - echo "startup-script finished successfully" - exit 0 -elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SUCCEEDED_LINE}"; then - echo "startup-script finished successfully (startup script succeeded line detected)" - exit 0 -elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_FINISHED_LINE}"; then - echo "startup-script finished successfully (startup script finished line detected)" - exit 0 -elif echo "${ser_log}" | grep -qE "${STARTUP_SCRIPT_SERVICE_FINISHED_LINE}"; then - echo "startup-script finished successfully (startup script service finished line detected)" - exit 0 -# --- If we reached deadline, it's a timeout --- -elif [[ now -ge deadline ]]; then - echo "startup-script timed out after ${TIMEOUT} seconds" - echo "${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" - exit 1 -# --- All other cases are considered failure or invalid state --- -else - echo "Invalid or undetermined startup script status. Last detected exit status: '${LAST_EXIT_STATUS}'" - echo "${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" - exit "${LAST_EXIT_STATUS}" -fi diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf deleted file mode 100644 index fe6410a920..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "instance_name" { - description = "Name of the instance we are waiting for (can be null if 'instance_names' is not empty)" - type = string - default = null -} - -variable "instance_names" { - description = "A list of instance names we are waiting for, in addition to the one mentioned in 'instance_name' (if any)" - type = list(string) - default = [] -} - -variable "zone" { - description = "The GCP zone where the instance is running" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "timeout" { - description = "Timeout in seconds" - type = number - default = 1200 - validation { - condition = var.timeout >= 0 - error_message = "The timeout should be non-negative" - } -} - -variable "gcloud_path_override" { - description = "Directory of the gcloud executable to be used during cleanup" - type = string - default = "" - nullable = false -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf deleted file mode 100644 index 8cd43b944e..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md deleted file mode 100644 index fc25bc0a55..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md +++ /dev/null @@ -1,109 +0,0 @@ -## Description - -This module contains a set of scripts to be used in customizing Windows VMs at -boot or during image building. Please note that the installation of NVIDIA GPU -drivers takes, at minimum, 30-60 minutes. It is therefore recommended to build -a custom image and reuse it as shown below, rather than install GPU drivers at -boot time. - -> NOTE: the output `windows_startup_ps1` must be passed explicitly as shown -> below when used with Packer modules. This is due to a limitation in the `use` -> keyword and inputs of type `list` in Packer modules; this does not impact -> Terraform modules - -### NVIDIA Drivers and CUDA Toolkit - -Many Google Cloud VM families include or can have NVIDIA GPUs attached to them. -This module supports GPU applications by enabling you to easily install -a compatible release of NVIDIA drivers and of the CUDA Toolkit. The script is -the [solution recommended by our documentation][docs] and is [directly sourced -from GitHub][script-src]. - -[docs]: https://cloud.google.com/compute/docs/gpus/install-drivers-gpu#windows -[script-src]: https://github.com/GoogleCloudPlatform/compute-gpu-installation/blob/24dac3004360e0696c49560f2da2cd60fcb80107/windows/install_gpu_driver.ps1 - -```yaml -- group: primary - modules: - - id: network1 - source: modules/network/vpc - settings: - enable_iap_rdp_ingress: true - enable_iap_winrm_ingress: true - - - id: windows_startup - source: community/modules/scripts/windows-startup-script - settings: - install_nvidia_driver: true - -- group: packer - modules: - - id: image - source: modules/packer/custom-image - kind: packer - use: - - network1 - - windows_startup - settings: - source_image_family: windows-2016 - machine_type: n1-standard-8 - accelerator_count: 1 - accelerator_type: nvidia-tesla-t4 - disk_size: 75 - disk_type: pd-ssd - omit_external_ip: false - state_timeout: 15m -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [http\_proxy](#input\_http\_proxy) | Set http and https proxy for use by Invoke-WebRequest commands | `string` | `""` | no | -| [http\_proxy\_set\_environment](#input\_http\_proxy\_set\_environment) | Set system default environment variables http\_proxy and https\_proxy for all commands | `bool` | `false` | no | -| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install\_nvidia\_driver\_script | `bool` | `false` | no | -| [install\_nvidia\_driver\_args](#input\_install\_nvidia\_driver\_args) | Arguments to supply to NVIDIA driver install script | `string` | `"/s /n"` | no | -| [install\_nvidia\_driver\_script](#input\_install\_nvidia\_driver\_script) | Install script for NVIDIA drivers specified by http/https URL | `string` | `"https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe"` | no | -| [no\_proxy](#input\_no\_proxy) | Environment variables no\_proxy (only used if var.http\_proxy\_set\_environment is enabled) | `string` | `"169.254.169.254,metadata,metadata.google.internal,.googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | A string list of scripts selected by this module | - diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf deleted file mode 100644 index 5e6bc8b94d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - setx_http_proxy_ps1 = !var.http_proxy_set_environment ? [] : [ - templatefile("${path.module}/templates/setx_http_proxy.ps1", { - "http_proxy" : var.http_proxy, - "no_proxy" : var.no_proxy, - }) - ] - - nvidia_ps1 = !var.install_nvidia_driver ? [] : [ - templatefile("${path.module}/templates/install_gpu_driver.ps1.tftpl", { - "url" : var.install_nvidia_driver_script - "args" : var.install_nvidia_driver_args - "http_proxy" : var.http_proxy, - }) - ] - - startup_ps1 = concat(local.setx_http_proxy_ps1, local.nvidia_ps1) -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf deleted file mode 100644 index 006ea312ad..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "windows_startup_ps1" { - description = "A string list of scripts selected by this module" - value = local.startup_ps1 -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl deleted file mode 100644 index 55c4a2a3cd..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl +++ /dev/null @@ -1,38 +0,0 @@ -#Requires -RunAsAdministrator - -# Windows 2016 needs forced upgrade to TLS 1.2 -[Net.ServicePointManager]::SecurityProtocol = 'Tls12' - -# important for catching exception in Invoke-WebRequest -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -%{ if http_proxy != "" } -[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") -%{ endif } - -# Create the folder for the driver download -$file_dir = 'C:\NVIDIA-Driver\nvidia_installer_windows.exe' -if (!(Test-Path -Path 'C:\NVIDIA-Driver')) { - New-Item -Path 'C:\' -Name 'NVIDIA-Driver' -ItemType 'directory' | Out-Null -} - -# Download the file to a specified directory -Write-Output "Downloading ${url} to $file_dir" -# Disabling progress bar has surprising large (10-100x) impact on speed -$ProgressPreference = 'SilentlyContinue' -try { - Invoke-WebRequest -Uri "${url}" -OutFile "$file_dir" -} catch { - Write-Output "$_" - throw "Failed to download ${url}; exiting startup script" -} - -# Install the file with the specified path from earlier as well as the RunAs admin option -Write-Output "Executing $file_dir with arguments '${args}'" -try { - Start-Process -FilePath "$file_dir" -ArgumentList '${args}' -Wait -} catch { - Write-Output "$_" - throw "Could not install NVIDIA driver; exiting startup script" -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 deleted file mode 100644 index ca4d13f98b..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 +++ /dev/null @@ -1,21 +0,0 @@ -<# - Copyright 2025 "Google LLC" - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -#> - -#Requires -RunAsAdministrator - -setx http_proxy ${http_proxy} /m -setx https_proxy ${http_proxy} /m -setx no_proxy ${no_proxy} /m diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf deleted file mode 100644 index 9e4fb9e67d..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "install_nvidia_driver" { - description = "Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install_nvidia_driver_script" - type = bool - default = false -} - -variable "install_nvidia_driver_script" { - description = "Install script for NVIDIA drivers specified by http/https URL" - type = string - default = "https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe" -} - -variable "install_nvidia_driver_args" { - description = "Arguments to supply to NVIDIA driver install script" - type = string - default = "/s /n" -} - -variable "http_proxy" { - description = "Set http and https proxy for use by Invoke-WebRequest commands" - type = string - default = "" - nullable = false -} - -variable "http_proxy_set_environment" { - description = "Set system default environment variables http_proxy and https_proxy for all commands" - type = bool - default = false - nullable = false -} - -variable "no_proxy" { - description = "Environment variables no_proxy (only used if var.http_proxy_set_environment is enabled)" - type = string - default = "169.254.169.254,metadata,metadata.google.internal,.googleapis.com" - nullable = false -} diff --git a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf b/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf deleted file mode 100644 index dfeeac34f8..0000000000 --- a/deletion-test/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/primary/modules/embedded/modules/README.md b/deletion-test/primary/modules/embedded/modules/README.md deleted file mode 100644 index 6886b3f330..0000000000 --- a/deletion-test/primary/modules/embedded/modules/README.md +++ /dev/null @@ -1,554 +0,0 @@ -# Modules - -This directory contains a set of core modules built for the Cluster Toolkit. Modules -describe the building blocks of an AI/ML and HPC deployment. The expected fields in a -module are listed in more detail [below](#module-fields). Blueprints can be -extended in functionality by incorporating [modules from GitHub -repositories][ghmods]. - -[ghmods]: #github-modules - -## All Modules - -Modules from various sources are all listed here for visibility. Badges are used -to indicate the source and status of many of these resources. - -Modules listed below with the ![core-badge] badge are located in this -folder and are tested and maintained by the Cluster Toolkit team. - -Modules labeled with the ![community-badge] badge are contributed by -the community (including the Cluster Toolkit team, partners, etc.). Community modules -are located in the [community folder](../community/modules/README.md). - -Modules labeled with the ![deprecated-badge] badge are now deprecated and may be -removed in the future. Customers are advised to transition to alternatives. - -Modules that are still in development and less stable are labeled with the -![experimental-badge] badge. - -[core-badge]: https://img.shields.io/badge/-core-blue?style=plastic -[community-badge]: https://img.shields.io/badge/-community-%23b8def4?style=plastic -[stable-badge]: https://img.shields.io/badge/-stable-lightgrey?style=plastic -[experimental-badge]: https://img.shields.io/badge/-experimental-%23febfa2?style=plastic -[deprecated-badge]: https://img.shields.io/badge/-deprecated-%23fea2a2?style=plastic - -### Compute - -* **[vm-instance]** ![core-badge] : Creates one or more VM instances. -* **[schedmd-slurm-gcp-v6-partition]** ![core-badge] : - Creates a partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. -* **[schedmd-slurm-gcp-v6-nodeset]** ![core-badge] : - Creates a nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. -* **[schedmd-slurm-gcp-v6-nodeset-tpu]** ![core-badge] : - Creates a TPU nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. -* **[schedmd-slurm-gcp-v6-nodeset-dynamic]** ![core-badge] ![experimental-badge]: - Creates a dynamic nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module and instance template. -* **[gke-node-pool]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes node pool using GKE. -* **[resource-policy]** ![core-badge] ![experimental-badge] : Create a resource policy for compute engines that can be applied to gke-node-pool's nodes. -* **[gke-job-template]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes job file to be used with a [gke-node-pool]. -* **[htcondor-execute-point]** ![community-badge] ![experimental-badge] : - Manages a group of execute points for use in an [HTCondor - pool][htcondor-setup]. -* **[mig]** ![community-badge] ![experimental-badge] : Creates a Managed Instance Group. -* **[notebook]** ![community-badge] ![experimental-badge] : Creates a Vertex AI - Notebook. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. -* **[gke-nodeset]** ![community-badge] ![experimental-badge] : Create a slinky nodeset to be used by the [gke-partition] module. -* **[gke-partition]** ![community-badge] ![experimental-badge] : Creates a slinky partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. - -[vm-instance]: compute/vm-instance/README.md -[gke-node-pool]: ../modules/compute/gke-node-pool/README.md -[resource-policy]: ../modules/compute/resource-policy/README.md -[gke-job-template]: ../modules/compute/gke-job-template/README.md -[schedmd-slurm-gcp-v6-partition]: ../community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md -[schedmd-slurm-gcp-v6-nodeset]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md -[schedmd-slurm-gcp-v6-nodeset-tpu]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md -[schedmd-slurm-gcp-v6-nodeset-dynamic]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md -[htcondor-execute-point]: ../community/modules/compute/htcondor-execute-point/README.md -[mig]: ../community/modules/compute/mig/README.md -[notebook]: ../community/modules/compute/notebook/README.md -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -### Database - -* **[slurm-cloudsql-federation]** ![community-badge] ![experimental-badge] : - Creates a [Google SQL Instance](https://cloud.google.com/sql/) meant to be - integrated with a [slurm-controller][schedmd-slurm-gcp-v6-controller]. -* **[bigquery-dataset]** ![community-badge] ![experimental-badge] : Creates a BQ - dataset. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. -* **[bigquery-table]** ![community-badge] ![experimental-badge] : Creates a BQ - table. Primarily used for - [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. - -[slurm-cloudsql-federation]: ../community/modules/database/slurm-cloudsql-federation/README.md -[bigquery-dataset]: ../community/modules/database/bigquery-dataset/README.md -[bigquery-table]: ../community/modules/database/bigquery-table/README.md -[fsi-montecarlo-on-batch]: ../community/modules/files/fsi-montecarlo-on-batch/README.md - -### File System - -* **[filestore]** ![core-badge] : Creates a - [filestore](https://cloud.google.com/filestore) file system. -* **[parallelstore]** ![core-badge] ![experimental-badge]: Creates a - [parallelstore](https://cloud.google.com/parallelstore) file system. -* **[pre-existing-network-storage]** ![core-badge] : Specifies a - pre-existing file system that can be mounted on a VM. -* **[managed-lustre]** ![core-badge] ![experimental-badge]: Creates a - [managed-lustred](https://cloud.google.com/managed-lustre) file system. -* **[DDN-EXAScaler]** ![community-badge] ![deprecated-badge] : Creates - a [DDN EXAscaler lustre](https://www.ddn.com/partners/google-cloud-platform/) - file system. This module is deprecated and will be removed by July 1, 2025. Consider migrating to managed-lustre. -* **[cloud-storage-bucket]** ![core-badge] : Creates a Google Cloud Storage (GCS) bucket. -* **[gke-persistent-volume]** ![core-badge] ![experimental-badge] : Creates - persistent volumes and persistent volume claims for shared storage. -* **[nfs-server]** ![community-badge] ![experimental-badge] : Creates a VM and - configures an NFS server that can be mounted by other VM. -* **[weka-client]** ![community-badge] ![experimental-badge] : Installs client - and mounts [WEKA](https://www.weka.io/) filesystems. - -[filestore]: file-system/filestore/README.md -[parallelstore]: file-system/parallelstore/README.md -[pre-existing-network-storage]: file-system/pre-existing-network-storage/README.md -[managed-lustre]: file-system/managed-lustre/README.md -[ddn-exascaler]: ../community/modules/file-system/DDN-EXAScaler/README.md -[nfs-server]: ../community/modules/file-system/nfs-server/README.md -[cloud-storage-bucket]: file-system/cloud-storage-bucket/README.md -[gke-persistent-volume]: file-system/gke-persistent-volume/README.md -[weka-client]: ../community/modules/file-system/weka-client/README.md - -### Monitoring - -* **[dashboard]** ![core-badge] : Creates a - [monitoring dashboard](https://cloud.google.com/monitoring/dashboards) for - visually tracking a Cluster Toolkit deployment. - -[dashboard]: monitoring/dashboard/README.md - -### Network - -* **[vpc]** ![core-badge] : Creates a - [Virtual Private Cloud (VPC)](https://cloud.google.com/vpc) network with - regional subnetworks and firewall rules. -* **[multivpc]** ![core-badge] ![experimental-badge]: Creates a variable - number of VPC networks using the [vpc] module. -* **[pre-existing-vpc]** ![core-badge] : Used to connect newly - built components to a pre-existing VPC network. -* **[firewall-rules]** ![core-badge] ![experimental-badge] : Add custom firewall - rules to existing networks (commonly used with [pre-existing-vpc]). -* **[private-service-access]** ![community-badge] ![experimental-badge] : - Configures Private Services Access for a VPC network (commonly used with [filestore] and [slurm-cloudsql-federation]). - -[vpc]: network/vpc/README.md -[multivpc]: network/multivpc/README.md -[pre-existing-vpc]: network/pre-existing-vpc/README.md -[firewall-rules]: network/firewall-rules/README.md -[private-service-access]: ../community/modules/network/private-service-access/README.md - -### Packer - -* **[custom-image]** ![core-badge] : Creates a custom VM Image - based on the GCP HPC VM image. - -[custom-image]: packer/custom-image/README.md - -### Project - -* **[service-account]** ![community-badge] ![experimental-badge] : Creates [service - accounts](https://cloud.google.com/iam/docs/service-accounts) for a GCP - project. -* **[service-enablement]** ![community-badge] ![experimental-badge] : Allows enabling - various APIs for a Google Cloud Project. - -[service-account]: ../community/modules/project/service-account/README.md -[service-enablement]: ../community/modules/project/service-enablement/README.md - -### Pub/Sub - -* **[topic]** ![community-badge] ![experimental-badge] : Creates a -Pub/Sub topic. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. -* **[bigquery-sub]** ![community-badge] ![experimental-badge] : Creates a -Pub/Sub subscription. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. - -[topic]: ../community/modules/pubsub/topic/README.md -[bigquery-sub]: ../community/modules/pubsub/bigquery-sub/README.md - -### Remote Desktop - -* **[chrome-remote-desktop]** ![community-badge] ![experimental-badge] : Creates - a GPU accelerated Chrome Remote Desktop. - -[chrome-remote-desktop]: ../community/modules/remote-desktop/chrome-remote-desktop/README.md - -### Scheduler - -* **[batch-job-template]** ![core-badge] : Creates a Google Cloud Batch job - template that works with other Toolkit modules. -* **[batch-login-node]** ![core-badge] : Creates a VM that can be used for - submission of Google Cloud Batch jobs. -* **[gke-cluster]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes cluster using GKE. -* **[pre-existing-gke-cluster]** ![core-badge] ![experimental-badge] : Retrieves an existing GKE cluster. Substitute for ([gke-cluster]) module. -* **[schedmd-slurm-gcp-v6-controller]** ![core-badge] : - Creates a Slurm controller node. -* **[schedmd-slurm-gcp-v6-login]** ![core-badge] : - Creates a Slurm login node. -* **[htcondor-setup]** ![community-badge] ![experimental-badge] : Creates the - base infrastructure for an HTCondor pool (service accounts and Cloud Storage bucket). -* **[htcondor-pool-secrets]** ![community-badge] ![experimental-badge] : Creates - and manages access to the secrets necessary for secure operation of an - HTCondor pool. -* **[htcondor-access-point]** ![community-badge] ![experimental-badge] : Creates - a regional instance group managing a highly available HTCondor access point - (login node). - -[batch-job-template]: ../modules/scheduler/batch-job-template/README.md -[batch-login-node]: ../modules/scheduler/batch-login-node/README.md -[gke-cluster]: ../modules/scheduler/gke-cluster/README.md -[pre-existing-gke-cluster]: ../modules/scheduler/pre-existing-gke-cluster/README.md -[htcondor-setup]: ../community/modules/scheduler/htcondor-setup/README.md -[htcondor-pool-secrets]: ../community/modules/scheduler/htcondor-pool-secrets/README.md -[htcondor-access-point]: ../community/modules/scheduler/htcondor-access-point/README.md -[schedmd-slurm-gcp-v6-controller]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md -[schedmd-slurm-gcp-v6-login]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md - -### Scripts - -* **[startup-script]** ![core-badge] : Creates a customizable startup script - that can be fed into compute VMs. -* **[windows-startup-script]** ![community-badge] ![experimental-badge]: Creates - Windows PowerShell (PS1) scripts that can be used to customize Windows VMs - and VM images. -* **[htcondor-install]** ![community-badge] ![experimental-badge] : Creates - a startup script to install HTCondor and exports a list of required APIs -* **[ramble-execute]** ![community-badge] ![experimental-badge] : Creates a - startup script to execute - [Ramble](https://github.com/GoogleCloudPlatform/ramble) commands on a target - VM -* **[ramble-setup]** ![community-badge] ![experimental-badge] : Creates a - startup script to install - [Ramble](https://github.com/GoogleCloudPlatform/ramble) on an instance or a - slurm login or controller. -* **[spack-setup]** ![community-badge] ![experimental-badge] : Creates a startup - script to install [Spack](https://github.com/spack/spack) on an instance or a - slurm login or controller. -* **[spack-execute]** ![community-badge] ![experimental-badge] : Defines a - software build using [Spack](https://github.com/spack/spack). -* **[wait-for-startup]** ![community-badge] ![experimental-badge] : Waits for - successful completion of a startup script on a compute VM. - -[startup-script]: scripts/startup-script/README.md -[windows-startup-script]: ../community/modules/scripts/windows-startup-script/README.md -[htcondor-install]: ../community/modules/scripts/htcondor-install/README.md -[kubernetes-operations]: ../community/modules/scripts/kubernetes-operations/README.md -[ramble-execute]: ../community/modules/scripts/ramble-execute/README.md -[ramble-setup]: ../community/modules/scripts/ramble-setup/README.md -[spack-setup]: ../community/modules/scripts/spack-setup/README.md -[spack-execute]: ../community/modules/scripts/spack-execute/README.md -[wait-for-startup]: ../community/modules/scripts/wait-for-startup/README.md - -## Module Fields - -### ID (Required) - -The `id` field is used to uniquely identify and reference a defined module. -ID's are used in [variables](../examples/README.md#variables) and become the -name of each module when writing the terraform `main.tf` file. They are also -used in the [use](#use-optional) and [outputs](#outputs-optional) lists -described below. - -For terraform modules, the ID will be rendered into the terraform module label -at the top level main.tf file. - -### Source (Required) - -The source is a path or URL that points to the source files for Packer or -Terraform modules. A source can either be a filesystem path or a URL to a git -repository: - -* Filesystem paths - * modules embedded in the `gcluster` executable - * modules in the local filesystem -* Remote modules using [Terraform URL syntax](https://developer.hashicorp.com/terraform/language/modules/sources) - * Hosted on [GitHub](https://developer.hashicorp.com/terraform/language/modules/sources#github) - * Google Cloud Storage [Buckets](https://developer.hashicorp.com/terraform/language/modules/sources#gcs-bucket) - * Generic [git repositories](https://developer.hashicorp.com/terraform/language/modules/sources#generic-git-repository) - - when modules are in a subdirectory of the git repository, a special - double-slash `//` notation can be required as described below - -An important distinction is that those URLs are natively supported by Terraform so -they are not copied to your deployment directory. Packer does not have native -support for git-hosted modules so the Toolkit will copy these modules into the -deployment folder on your behalf. - -#### Embedded Modules - -Embedded modules are added to the gcluster binary during compilation and cannot -be edited. To refer to embedded modules, set the source path to -`modules/<>` or `community/modules/<>`. - -The paths match the modules in the repository structure for [core modules](./) -and [community modules](../community/modules/). Because the modules are embedded -during compilation, your local copies may differ unless you recompile gcluster. - -For example, this example snippet uses the embedded pre-existing-vpc module: - -```yaml - - id: network1 - source: modules/network/pre-existing-vpc -``` - -#### Local Modules - -Local modules point to a module in the file system and can easily be edited. -They are very useful during module development. To use a local module, set -the source to a path starting with `/`, `./`, or `../`. For instance, the -following module definition refers the local pre-existing-vpc modules. - -```yaml - - id: network1 - source: modules/network/pre-existing-vpc -``` - -> **_NOTE:_** Relative paths (beginning with `.` or `..` must be relative to the -> working directory from which `gcluster` is executed. This example would have to be -> run from a local copy of the Cluster Toolkit repository. An alternative is to use -> absolute paths to modules. - -#### GitHub-hosted Modules and Packages - -To use a Terraform module available on GitHub, set the source to a path starting -with `github.com` (HTTPS) or `git@github.com` (SSH). For instance, the following -module definition sources the Toolkit vpc module: - -```yaml - - id: network1 - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc -``` - -This example uses the [double-slash notation][tfsubdir] (`//`) to indicate that -the Toolkit is a "package" of multiple modules whose root directory is the root -of the git repository. The remainder of the path indicates the sub-directory of -the vpc module. - -The example above uses the default `main` branch of the Toolkit. Specific -[revisions][tfrev] can be selected with any valid [git reference][gitref]. -(git branch, commit hash or tag). If the git reference is a tag or branch, we -recommend setting `&depth=1` to reduce the data transferred over the network. -This option cannot be set when the reference is a commit hash. The following -examples select the vpc module on the active `develop` branch and also an older -release of the filestore module: - -```yaml - - id: network1 - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc?ref=develop - ... - - id: homefs - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/file-system/filestore?ref=v1.22.1&depth=1 -``` - -Because Terraform modules natively support this syntax, gcluster will not copy -GitHub-hosted modules into your deployment folder. Terraform will download them -into a hidden folder when you run `terraform init`. - -[tfrev]: https://www.terraform.io/language/modules/sources#selecting-a-revision -[gitref]: https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection#_single_revisions -[tfsubdir]: https://www.terraform.io/language/modules/sources#modules-in-package-sub-directories - -##### GitHub-hosted Packer modules - -Packer does not natively support GitHub-hosted modules so `gcluster create` will -copy modules into your deployment folder. - -If the module uses `//` package notation, `gcluster create` will copy the entire -repository to the module path: `deployment_name/group_name/module_id`. However, -when `gcluster deploy` is invoked, it will run Packer from the subdirectory -`deployment_name/group_name/module_id/subdirectory/after/double_slash`. - -If the module does not use `//` package notation, `gcluster create` will copy -only the final directory in the path to `deployment_name/group_name/module_id`. - -In all cases, `gcluster create` will remove the `.git` directory from the packer -module to ensure that you can manage the entire deployment directory with its -own git versioning. - -##### GitHub over SSH - -Get module from GitHub over SSH: - -```yaml - - id: network1 - source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc -``` - -Specific versions can be selected as for HTTPS: - -```yaml - - id: network1 - source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc?ref=v1.22.1&depth=1 -``` - -##### Generic Git Modules - -To use a Terraform module available in a non-GitHub git repository such as -gitlab, set the source to a path starting `git::`. Two Standard git protocols -are supported, `git::https://` for HTTPS or `git::git@github.com` for SSH. - -Additional formatting and features after `git::` are identical to that of the -[GitHub Modules](#github-modules) described above. - -#### Google Cloud Storage Modules - -To use a Terraform module available in a Google Cloud Storage bucket, set the source -to a URL with the special `gcs::` prefix, followed by a [GCS bucket object URL](https://cloud.google.com/storage/docs/request-endpoints#typical). - -For example: `gcs::https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH_TO_MODULE` - -### Kind (May be Required) - -`kind` refers to the way in which a module is deployed. Currently, `kind` can be -either `terraform` or `packer`. It must be specified for modules of type -`packer`. If omitted, it will default to `terraform`. - -### Settings (May Be Required) - -The settings field is a map that supplies any user-defined variables for each -module. Settings values can be simple strings, numbers or booleans, but can -also support complex data types like maps and lists of variable depth. These -settings will become the values for the variables defined in either the -`variables.tf` file for Terraform or `variable.pkr.hcl` file for Packer. - -For some modules, there are mandatory variables that must be set, -therefore `settings` is a required field in that case. In many situations, a -combination of sensible defaults, deployment variables and used modules can -populated all required settings and therefore the settings field can be omitted. - -### Use (Optional) - -The `use` field is a powerful way of linking a module to one or more other -modules. When a module "uses" another module, the outputs of the used -module are compared to the settings of the current module. If they have -matching names and the setting has no explicit value, then it will be set to -the used module's output. For example, see the following blueprint snippet: - -```yaml -modules: -- id: network1 - source: modules/network/vpc - -- id: workstation - source: modules/compute/vm-instance - use: [network1] - settings: - ... -``` - -In this snippet, the VM instance `workstation` uses the outputs of vpc -`network1`. - -In this case both `network_self_link` and `subnetwork_self_link` in the -[workstation settings](compute/vm-instance/README.md#Inputs) will be set -to `$(network1.network_self_link)` and `$(network1.subnetwork_self_link)` which -refer to the [network1 outputs](network/vpc/README#Outputs) -of the same names. - -The order of precedence that `gcluster` uses in determining when to infer a setting -value is in the following priority order: - -1. Explicitly set in the blueprint using the `settings` field -1. Output from a used module, taken in the order provided in the `use` list -1. Deployment variable (`vars`) of the same name -1. Default value for the setting - -> **_NOTE:_** See the -> [network storage documentation](./../docs/network_storage.md) for more -> information about mounting network storage file systems via the `use` field. - -### Outputs (Optional) - -The `outputs` field adds the output of individual Terraform modules to the -output of its deployment group. This enables the value to be available via -`terraform output`. This can useful for displaying the IP of a login node or -printing instructions on how to use a module, as we have in the -[monitoring dashboard module](monitoring/dashboard/README.md#Outputs). - -The outputs field is a lists that it can be in either of two formats: a string -equal to the name of the module output, or a map specifying the `name`, -`description`, and whether the value is `sensitive` and should be suppressed -from the standard output of Terraform commands. An example is shown below -that displays the internal and public IP addresses of a VM created by the -vm-instance module: - -```yaml - - id: vm - source: modules/compute/vm-instance - use: - - network1 - settings: - machine_type: e2-medium - outputs: - - internal_ip - - name: external_ip - description: "External IP of VM" - sensitive: true -``` - -The outputs shown after running Terraform apply will resemble: - -```text -Apply complete! Resources: 7 added, 0 changed, 0 destroyed. - -Outputs: - -external_ip_simplevm = -internal_ip_simplevm = [ - "10.128.0.19", -] -``` - -### Required Services (APIs) (optional) - -Each Toolkit module depends upon Google Cloud services ("APIs") being enabled -in the project used by the AI/ML and HPC environment. For example, the [creation of -VMs](compute/vm-instance/) requires the Compute Engine API -(compute.googleapis.com). The [startup-script](scripts/startup-script/) module -requires the Cloud Storage API (storage.googleapis.com) for storage of the -scripts themselves. Each module included in the Toolkit source code describes -its required APIs internally. The Toolkit will merge the requirements from all -modules and [automatically validate](../README.md#blueprint-validation) that all -APIs are enabled in the project specified by `$(vars.project_id)`. - -## Common Settings - -The following common naming conventions should be used to decrease the verbosity -needed to define a blueprint. This is intentional to allow multiple -modules to share inferred settings from deployment variables or from other -modules listed under the `use` field. - -For example, if all modules are to be created in a single region, that region -can be defined as a deployment variable named `region`, which is shared between -all modules without an explicit setting. Similarly, if many modules need to be -connected to the same VPC network, they all can add the vpc module ID to their -`use` list so that `network_self_link` would be inferred from that vpc module rather -than having to set it manually. - -* **project_id**: The GCP project ID in which to create the GCP resources. -* **deployment_name**: The name of the current deployment of a blueprint. This - can help to avoid naming conflicts of modules when multiple deployments are - created from the same blueprint. -* **region**: The GCP - [region](https://cloud.google.com/compute/docs/regions-zones) the module - will be created in. -* **zone**: The GCP [zone](https://cloud.google.com/compute/docs/regions-zones) - the module will be created in. -* **labels**: - [Labels](https://cloud.google.com/resource-manager/docs/creating-managing-labels) - added to the module. In order to include any module in advanced - monitoring, labels must be exposed. We strongly recommend that all modules - expose this variable. - -## Writing Custom Cluster Toolkit Modules - -Modules are flexible by design, however we define some [best practices](../docs/module-guidelines.md) when -creating a new module meant to be used with the Cluster Toolkit. diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/README.md b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/README.md deleted file mode 100644 index f807cd727e..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/README.md +++ /dev/null @@ -1,133 +0,0 @@ -## Description - -This module is used to create a Kubernetes job template file. - -The job template file can be submitted as is or used as a template for further -customization. Add the `instructions` output to a blueprint (as shown below) to -get instructions on how to use `kubectl` to submit the job. - -This module is designed to `use` one or more `gke-node-pool` modules. The job -will be configured to run on any of the specified node pools. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a GKE job template file. - -```yaml - - id: job-template - source: modules/compute/gke-job-template - use: [compute_pool] - settings: - node_count: 3 - outputs: [instructions] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### Storage Options - -This module natively supports: - -* Filestore as a shared file system between pods/nodes. -* Pod level ephemeral storage options: - * memory backed emptyDir - * local SSD backed emptyDir - * SSD persistent disk backed ephemeral volume - * balanced persistent disk backed ephemeral volume - -See the [storage-gke.yaml blueprint](../../../examples/storage-gke.yaml) and the -associated [documentation](../../../../examples/README.md#storage-gkeyaml--) for -examples of how to use Filestore and ephemeral storage with this module. - -### Requested Resources - -When one or more `gke-node-pool` modules are referenced with the `use` field. -The requested resources will be populated to achieve a 1 pod per node packing -while still leaving some headroom for required system pods. - -This functionality can be overridden by specifying the desired cpu requirement -using the `requested_cpu_per_pod` setting. - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.2 | -| [local](#requirement\_local) | >= 2.0.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allocatable\_cpu\_per\_node](#input\_allocatable\_cpu\_per\_node) | The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | -| [allocatable\_gpu\_per\_node](#input\_allocatable\_gpu\_per\_node) | The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | -| [backoff\_limit](#input\_backoff\_limit) | Controls the number of retries before considering a Job as failed. Set to zero for shared fate. | `number` | `0` | no | -| [command](#input\_command) | The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes. | `list(string)` | 
[
  "hostname"
]
| no | -| [completion\_mode](#input\_completion\_mode) | Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information | `string` | `"Indexed"` | no | -| [ephemeral\_volumes](#input\_ephemeral\_volumes) | Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB. | 
list(object({
    type       = string
    mount_path = string
    size_gb    = number
  }))
| `[]` | no | -| [has\_gpu](#input\_has\_gpu) | Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module. | `list(bool)` | 
[
  false
]
| no | -| [image](#input\_image) | The container image the job should use. | `string` | `"debian"` | no | -| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account to run the job as. If null then no service account is specified. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the GKE job template. Key-value pairs. | `map(string)` | n/a | yes | -| [machine\_family](#input\_machine\_family) | The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria. | `string` | `null` | no | -| [name](#input\_name) | The name of the job. | `string` | `"my-job"` | no | -| [node\_count](#input\_node\_count) | How many nodes the job should run in parallel. | `number` | `1` | no | -| [node\_pool\_names](#input\_node\_pool\_names) | A list of node pool names on which to run the job. Can be populated via `use` field. | `list(string)` | `[]` | no | -| [node\_selectors](#input\_node\_selectors) | A list of node selectors to use to place the job. | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | -| [persistent\_volume\_claims](#input\_persistent\_volume\_claims) | A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module. | 
list(object({
    name          = string
    namespace     = string
    mount_path    = string
    mount_options = string
    storage_type  = string
  }))
| `[]` | no | -| [random\_name\_sufix](#input\_random\_name\_sufix) | Appends a random suffix to the job name to avoid clashes. | `bool` | `true` | no | -| [requested\_cpu\_per\_pod](#input\_requested\_cpu\_per\_pod) | The requested cpu per pod. If null, allocatable\_cpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_cpu\_per\_node. | `number` | `-1` | no | -| [requested\_gpu\_per\_pod](#input\_requested\_gpu\_per\_pod) | The requested gpu per pod. If null, allocatable\_gpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_gpu\_per\_node. | `number` | `-1` | no | -| [restart\_policy](#input\_restart\_policy) | Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed. | `string` | `"Never"` | no | -| [security\_context](#input\_security\_context) | The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | -| [tolerations](#input\_tolerations) | Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field. | 
list(object({
    key      = string
    operator = string
    value    = string
    effect   = string
  }))
| 
[
  {
    "effect": "NoSchedule",
    "key": "user-workload",
    "operator": "Equal",
    "value": "true"
  }
]
| no | -| [tpu\_accelerator\_type](#input\_tpu\_accelerator\_type) | The TPU accelerator type label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | -| [tpu\_chips\_per\_node](#input\_tpu\_chips\_per\_node) | The number of TPU chips per node. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | -| [tpu\_topology](#input\_tpu\_topology) | The TPU topology label. Populated from gke-node-pool via `use` field. | `list(string)` | 
[
  null
]
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for submitting the GKE job. | - diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/main.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/main.tf deleted file mode 100644 index e84138bb3f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/main.tf +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-job-template", ghpc_role = "compute" }) -} - -locals { - tpu_accelerator_node_selector = var.tpu_accelerator_type[0] != null ? [{ - key = "cloud.google.com/gke-tpu-accelerator" - value = var.tpu_accelerator_type[0] - }] : [] - - tpu_topology_node_selector = var.tpu_topology[0] != null ? [{ - key = "cloud.google.com/gke-tpu-topology" - value = var.tpu_topology[0] - }] : [] -} - -locals { - # Start with the minimum cpu available of used node pools - min_allocatable_cpu = min(var.allocatable_cpu_per_node...) - full_node_cpu_request = ( - local.min_allocatable_cpu > 2 ? # if large enough - local.min_allocatable_cpu - 1 : # leave headroom for 1 cpu - local.min_allocatable_cpu / 2 + 0.1 # else take just over half - ) - (local.any_gcs ? 0.25 : 0) # save room for gcs side car - - cpu_request = ( - var.requested_cpu_per_pod >= 0 ? # if user supplied requested cpu - var.requested_cpu_per_pod : # then honor it - ( # else - local.min_allocatable_cpu >= 0 ? # if allocatable cpu was supplied - local.full_node_cpu_request : # then claim the full node - -1 # else do not set a limit - ) - ) - millicpu = floor(local.cpu_request * 1000) - cpu_request_string = local.millicpu >= 0 ? "${local.millicpu}m" : null - full_node_request = local.min_allocatable_cpu >= 0 && var.requested_cpu_per_pod < 0 - - memory_request_value = try(sum([for ed in var.ephemeral_volumes : - ed.size_gb - if ed.type == "memory" - ]), 0) - memory_request_string = local.memory_request_value > 0 ? "${local.memory_request_value}Gi" : null - - ephemeral_request_value = try(sum([for ed in var.ephemeral_volumes : - ed.size_gb - if ed.type == "local-ssd" - ]), 0) - ephemeral_request_string = local.ephemeral_request_value > 0 ? "${local.ephemeral_request_value}Gi" : null - - uses_local_ssd = anytrue([for ed in var.ephemeral_volumes : - ed.type == "local-ssd" - ]) - local_ssd_node_selector = local.uses_local_ssd ? [{ - key = "cloud.google.com/gke-ephemeral-storage-local-ssd" - value = "true" - }] : [] - - # Setup limit for GPUs per pod - min_allocatable_gpu = min(var.allocatable_gpu_per_node...) - min_allocatable_gpu_per_pod = local.min_allocatable_gpu > 0 ? local.min_allocatable_gpu : null - gpu_limit_per_pod = var.requested_gpu_per_pod > 0 ? var.requested_gpu_per_pod : local.min_allocatable_gpu_per_pod - gpu_limit_string = alltrue(var.has_gpu) ? tostring(local.gpu_limit_per_pod) : null - - empty_dir_volumes = [for ed in var.ephemeral_volumes : - { - name = replace(trim(ed.mount_path, "/"), "/", "-") - mount_path = ed.mount_path - size_limit = "${ed.size_gb}Gi" - in_memory = ed.type == "memory" - } - if contains(["memory", "local-ssd"], ed.type) - ] - - ephemeral_pd_volumes = [for pd in var.ephemeral_volumes : - { - name = replace(trim(pd.mount_path, "/"), "/", "-") - mount_path = pd.mount_path - storage_class_name = pd.type == "pd-ssd" ? "premium-rwo" : "standard-rwo" - storage = "${pd.size_gb}Gi" - } - if contains(["pd-balanced", "pd-ssd"], pd.type) - ] - - pvc_volumes = [for pvc in var.persistent_volume_claims : - { - name = replace(trim(pvc.mount_path, "/"), "/", "-") - mount_path = pvc.mount_path - claim_name = pvc.name - } - ] - - volume_mounts = [for v in concat(local.empty_dir_volumes, local.ephemeral_pd_volumes, local.pvc_volumes) : - { - name = v.name - mount_path = v.mount_path - } - ] - - suffix = var.random_name_sufix ? "-${random_id.resource_name_suffix.hex}" : "" - machine_family_node_selector = var.machine_family != null ? [{ - key = "cloud.google.com/machine-family" - value = var.machine_family - }] : [] - node_selectors = concat(local.machine_family_node_selector, local.local_ssd_node_selector, local.tpu_accelerator_node_selector, local.tpu_topology_node_selector, var.node_selectors) - - any_gcs = anytrue([for pvc in var.persistent_volume_claims : - pvc.storage_type == "gcs" - ]) - - job_template_contents = templatefile( - "${path.module}/templates/gke-job-base.yaml.tftpl", - { - name = var.name - suffix = local.suffix - image = var.image - command = var.command - node_count = var.node_count - completion_mode = var.completion_mode - k8s_service_account_name = var.k8s_service_account_name - node_pool_names = var.node_pool_names - node_selectors = local.node_selectors - tpu_limit = var.tpu_chips_per_node[0] - full_node_request = local.full_node_request - cpu_request = local.cpu_request_string - gpu_limit = local.gpu_limit_string - restart_policy = var.restart_policy - backoff_limit = var.backoff_limit - tolerations = distinct(var.tolerations) - security_context = var.security_context - labels = local.labels - - empty_dir_volumes = local.empty_dir_volumes - ephemeral_pd_volumes = local.ephemeral_pd_volumes - pvc_volumes = local.pvc_volumes - volume_mounts = local.volume_mounts - memory_request = local.memory_request_string - ephemeral_request = local.ephemeral_request_string - gcs_annotation = local.any_gcs - } - ) - - job_template_output_path = "${path.root}/${var.name}${local.suffix}.yaml" - -} - -resource "random_id" "resource_name_suffix" { - byte_length = 2 - keepers = { - timestamp = timestamp() - } -} - -resource "local_file" "job_template" { - content = local.job_template_contents - filename = local.job_template_output_path - - lifecycle { - precondition { - condition = local.any_gcs ? var.k8s_service_account_name != null : true - error_message = "When using GCS, a kubernetes service account with workload identity is required. gke-cluster module will perform this setup when var.configure_workload_identity_sa is set to true." - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf deleted file mode 100644 index adf78e936d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "instructions" { - description = "Instructions for submitting the GKE job." - value = <<-EOT - A GKE job file has been created locally at: - ${abspath(local.job_template_output_path)} - - Use the following commands to: - Submit your job: - kubectl create -f ${abspath(local.job_template_output_path)} - EOT -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl deleted file mode 100644 index 11df39ce2c..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl +++ /dev/null @@ -1,128 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: ${name}${suffix} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - parallelism: ${node_count} - completions: ${node_count} - completionMode: ${completion_mode} - template: - %{~ if gcs_annotation ~} - metadata: - annotations: - gke-gcsfuse/volumes: "true" - %{~ endif ~} - spec: - %{~ if length(security_context) > 0 ~} - securityContext: - %{~ for context in security_context ~} - ${context.key}: ${context.value} - %{~ endfor ~} - %{~ endif ~} - %{~ if k8s_service_account_name != null ~} - serviceAccountName: ${k8s_service_account_name} - %{~ endif ~} - %{~ if length(node_pool_names) > 0 ~} - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.google.com/gke-nodepool - operator: In - values: - %{~ for node_pool in node_pool_names ~} - - ${node_pool} - %{~ endfor ~} - %{~ endif ~} - %{~ if length(node_selectors) > 0 ~} - nodeSelector: - %{~ for selector in node_selectors ~} - ${selector.key}: "${selector.value}" - %{~ endfor ~} - %{~ endif ~} - tolerations: - %{~ for toleration in tolerations ~} - - key: ${toleration.key} - operator: ${toleration.operator} - value: "${toleration.value}" - effect: ${toleration.effect} - %{~ endfor ~} - containers: - - name: ${name}-container - image: ${image} - command: - %{for s in command}- ${indent(8, yamlencode(s))}%{~ endfor } - %{~ if gpu_limit != null || cpu_request != null || tpu_limit != null ~} - resources: - %{~ if gpu_limit != null || tpu_limit != null ~} - limits: - %{~ if gpu_limit != null ~} - # GPUs should only be specified as limits - # https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/ - nvidia.com/gpu: ${gpu_limit} - %{~ endif ~} - %{~ if tpu_limit != null ~} - google.com/tpu: ${tpu_limit} - %{~ endif ~} - %{~ endif ~} - %{~ if cpu_request != null || memory_request != null || ephemeral_request != null || tpu_limit != null ~} - requests: - %{~ if full_node_request ~} - # cpu request attempts full node per pod - %{~ endif ~} - %{~ if cpu_request != null ~} - cpu: ${cpu_request} - %{~ endif ~} - %{~ if tpu_limit != null ~} - google.com/tpu: ${tpu_limit} - %{~ endif ~} - %{~ if memory_request != null ~} - memory: ${memory_request} - %{~ endif ~} - %{~ if ephemeral_request != null ~} - ephemeral-storage: ${ephemeral_request} - %{~ endif ~} - %{~ endif ~} - %{~ endif ~} - %{~ if length(volume_mounts) > 0 ~} - volumeMounts: - %{~ for v in volume_mounts ~} - - name: ${v.name} - mountPath: ${v.mount_path} - %{~ endfor ~} - %{~ endif ~} - %{~ if length(volume_mounts) > 0 ~} - volumes: - %{~ for ed in empty_dir_volumes ~} - - name: ${ed.name} - emptyDir: - sizeLimit: ${ed.size_limit} - %{~ if ed.in_memory ~} - medium: "Memory" - %{~ endif ~} - %{~ endfor ~} - %{~ for pd in ephemeral_pd_volumes ~} - - name: ${pd.name} - ephemeral: - volumeClaimTemplate: - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: ${pd.storage_class_name} - resources: - requests: - storage: ${pd.storage} - %{~ endfor ~} - %{~ for pvc in pvc_volumes ~} - - name: ${pvc.name} - persistentVolumeClaim: - claimName: ${pvc.claim_name} - %{~ endfor ~} - %{~ endif ~} - restartPolicy: ${restart_policy} - backoffLimit: ${backoff_limit} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/variables.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/variables.tf deleted file mode 100644 index fd83f2b692..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/variables.tf +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "name" { - description = "The name of the job." - type = string - default = "my-job" -} - -variable "node_count" { - description = "How many nodes the job should run in parallel." - type = number - default = 1 -} - -variable "completion_mode" { - description = "Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information" - type = string - default = "Indexed" -} - -variable "command" { - description = "The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes." - type = list(string) - default = ["hostname"] -} - -variable "image" { - description = "The container image the job should use." - type = string - default = "debian" -} - -variable "k8s_service_account_name" { - description = "Kubernetes service account to run the job as. If null then no service account is specified." - type = string - default = null -} - -variable "node_pool_names" { - description = "A list of node pool names on which to run the job. Can be populated via `use` field." - type = list(string) - default = [] -} - -variable "allocatable_cpu_per_node" { - description = "The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." - type = list(number) - default = [-1] -} - -variable "has_gpu" { - description = "Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module." - type = list(bool) - default = [false] -} - -variable "requested_cpu_per_pod" { - description = "The requested cpu per pod. If null, allocatable_cpu_per_node will be used to claim whole nodes. If provided will override allocatable_cpu_per_node." - type = number - default = -1 -} - -variable "allocatable_gpu_per_node" { - description = "The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." - type = list(number) - default = [-1] -} - -variable "requested_gpu_per_pod" { - description = "The requested gpu per pod. If null, allocatable_gpu_per_node will be used to claim whole nodes. If provided will override allocatable_gpu_per_node." - type = number - default = -1 -} - -variable "tolerations" { - description = "Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field." - type = list(object({ - key = string - operator = string - value = string - effect = string - })) - default = [ - { - key = "user-workload" - operator = "Equal" - value = "true" - effect = "NoSchedule" - } - ] -} - -variable "security_context" { - description = "The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/" - type = list(object({ - key = string - value = string - })) - default = [] -} - -variable "machine_family" { - description = "The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria." - type = string - default = null -} - -variable "node_selectors" { - description = "A list of node selectors to use to place the job." - type = list(object({ - key = string - value = string - })) - default = [] -} - -variable "restart_policy" { - description = "Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed." - type = string - default = "Never" -} - -variable "backoff_limit" { - description = "Controls the number of retries before considering a Job as failed. Set to zero for shared fate." - type = number - default = 0 -} - -variable "random_name_sufix" { - description = "Appends a random suffix to the job name to avoid clashes." - type = bool - default = true -} - -variable "persistent_volume_claims" { - description = "A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module." - type = list(object({ - name = string - namespace = string - mount_path = string - mount_options = string - storage_type = string - })) - default = [] -} - -variable "ephemeral_volumes" { - description = "Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB." - type = list(object({ - type = string - mount_path = string - size_gb = number - })) - default = [] - validation { - condition = alltrue([ - for v in var.ephemeral_volumes : - contains(["pd-balanced", "pd-ssd", "memory", "local-ssd"], v.type) - ]) - error_message = "Type must be one of 'pd-balanced', 'pd-ssd', 'memory', 'local-ssd'." - } - validation { - condition = alltrue([ - for v in var.ephemeral_volumes : - substr(v.mount_path, 0, 1) == "/" - ]) - error_message = "Mount path must start with the '/' character." - } -} - -variable "labels" { - description = "Labels to add to the GKE job template. Key-value pairs." - type = map(string) -} - -variable "tpu_accelerator_type" { - description = "The TPU accelerator type label. Populated from gke-node-pool via `use` field." - type = list(string) - default = [null] -} - -variable "tpu_topology" { - description = "The TPU topology label. Populated from gke-node-pool via `use` field." - type = list(string) - default = [null] -} - -variable "tpu_chips_per_node" { - description = "The number of TPU chips per node. Populated from gke-node-pool via `use` field." - type = list(string) - default = [null] -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/versions.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/versions.tf deleted file mode 100644 index 0f902ac8c5..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-job-template/versions.tf +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.2" - - required_providers { - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/README.md b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/README.md deleted file mode 100644 index b25d905252..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/README.md +++ /dev/null @@ -1,388 +0,0 @@ -## Description - -This module creates a Google Kubernetes Engine -([GKE](https://cloud.google.com/kubernetes-engine)) node pool. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a GKE node group. - -```yaml - - id: compute_pool - source: modules/compute/gke-node-pool - use: [gke_cluster] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### Taints and Tolerations - -By default node pools created with this module will be tainted with -`user-workload=true:NoSchedule` to prevent system pods from being scheduled. -User jobs targeting the node pool should include this toleration. This behavior -can be overridden using the `taints` setting. See -[docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-taints) for -more info. - -### Local SSD Storage -GKE offers two options for managing locally attached SSDs. - -The first, and recommended, option is for GKE to manage the ephemeral storage -space on the node, which will then be automatically attached to pods which -request an `emptyDir` volume. This can be accomplished using the -[`local_ssd_count_ephemeral_storage`] variable. - -The second, more complex, option is for GCP to attach these nodes as raw block -storage. In this case, the cluster administrator is responsible for software -RAID settings, partitioning, formatting and mounting these disks on the host -OS. Still, this may be desired behavior in use cases which aren't supported -by an `emptyDir` volume (for example, a `ReadOnlyMany` or `ReadWriteMany` PV). -This can be accomplished using the [`local_ssd_count_nvme_block`] variable. - -The [`local_ssd_count_ephemeral_storage`] and [`local_ssd_count_nvme_block`] -variables are mutually exclusive and cannot be mixed together. - -Also, the number of SSDs which can be attached to a node depends on the -[machine type](https://cloud.google.com/compute/docs/disks#local_ssd_machine_type_restrictions). - -See [docs](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd) -for more info. - -[`local_ssd_count_ephemeral_storage`]: #input\_local\_ssd\_count\_ephemeral\_storage -[`local_ssd_count_nvme_block`]: #input\_local\_ssd\_count\_nvme\_block - -### Considerations with GPUs - -When a GPU is attached to a node an additional taint is automatically added: -`nvidia.com/gpu=present:NoSchedule`. For jobs to get placed on these nodes, the -equivalent toleration is required. The `gke-job-template` module will -automatically apply this toleration when using a node pool with GPUs. - -Nvidia GPU drivers must be installed. The recommended approach for GKE to install -GPU dirvers is by applying a DaemonSet to the cluster. See -[these instructions](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus#cos). - -However, in some cases it may be desired to compile a different driver (such as -a desire to install a newer version, compatibility with the -[Nvidia GPU-operator](https://github.com/NVIDIA/gpu-operator) or other -use-cases). In this case, ensure that you turn off the -[enable_secure_boot](#input\_enable\_secure\_boot) option to allow unsigned -kernel modules to be loaded. - -#### Maximize GPU network bandwidth with GPUDirect and multi-networking -For A3 Series machines to achieve optimal performance , GKE provide two networking stacks for remote direct memory access (RDMA): - -- A3 High machine types (a3-highgpu-8g): utilize GPUDirect-TCPX to reduce the overhead required to transfer packet payloads to and from GPUs, which significantly improves throughput at scale compared to GPUs that don't use GPUDirect. -- A3 Mega machine types (a3-megagpu-8g): utilize GPUDirect-TCPXO to improve GPU to GPU communication, and further improves GPU to VM communication. - -To achieve this, when creating nodepools with A3 Series machine type, pass in a multivpc module to the gke-node-pool module, and the gke-node-pool module would detect the eligible machine type and enable GPUDirect for it. More specifically, the below components will be installed in the nodepool for enabling GPUDirect. - -- Install NCCL plugin for GPUDirect [TCPX](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpx) or [TCPXO](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpxo) -- Install [NRI](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nri_device_injector) device injector plugin -- Provide support for injecting GPUDirect required components(annotations, volumes, rxdm sidecar etc.) into the user workload in the form of Kubernetes Job. - - Provide sample workload to showcase how it will be updated with the required components injected, and how it can be deployed. - - Allow user to use the provided script to update their own workload and deploy. - -The GPUDirect supports included in the Cluster Toolkit aim to automate the [GPUDirect User Guid](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#install-gpudirect-tcpx-nccl) and provide better usability. - -> **_NOTE:_** You must [enable multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) feature when creating the GKE cluster. When gke-cluster depends on multivpc (with the use keyword), multi networking will be automatically enabled on the cluster creation. -> When gke-cluster or pre-existing-gke-cluster depends on multivpc (with the use keyword), the [network objects](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment) required for multi networking will be created on the cluster. - -### GPUs Examples - -There are several ways to add GPUs to a GKE node pool. See -[docs](https://cloud.google.com/compute/docs/gpus) for more info on GPUs. - -The following is a node pool that uses `a2`, `a3` or `g2` machine types which has a -fixed number of attached GPUs, let's call these machine types as "pre-defined gpu machine families": - -```yaml - - id: simple-a2-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g -``` - -> **Note**: It is not necessary to define the [`guest_accelerator`] setting when -> using pre-defined gpu machine families as information about GPUs, such as type, count and -> `gpu_driver_installation_config`, is automatically inferred from the machine type. -> Optional fields such as `gpu_partition_size` need to be specified only if they have -> non-default values. - -The following scenarios require the [`guest_accelerator`] block is specified: - -- To partition an A100 GPU into multiple GPUs on an A2 family machine. -- To specify a time sharing configuration on a GPUs. -- To attach a GPU to an N1 family machine. - -The following is an example of -[partitioning](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus-multi) -an A100 GPU: - -> **Note**: In the following example, `type`, `count` and `gpu_driver_installation_config` are picked up automatically. - -```yaml - - id: multi-instance-gpu-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g - guest_accelerator: - - gpu_partition_size: 1g.5gb -``` - -[`guest_accelerator`]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_guest_accelerator - -The following is an example of -[GPU time sharing](https://cloud.google.com/kubernetes-engine/docs/concepts/timesharing-gpus) -(with partitioned GPUs): - -```yaml - - id: time-sharing-gpu-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g - guest_accelerator: - - gpu_partition_size: 1g.5gb - gpu_sharing_config: - gpu_sharing_strategy: TIME_SHARING - max_shared_clients_per_gpu: 3 -``` - -Following is an example of using a GPU attached to an `n1` machine: - -```yaml - - id: t4-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: n1-standard-16 - guest_accelerator: - - type: nvidia-tesla-t4 - count: 2 -``` - -The following is an example of using a GPU (with sharing config) attached to an `n1` machine: - -```yaml - - id: n1-t4-pool - source: community/modules/compute/gke-node-pool - use: [gke_cluster] - settings: - name: n1-t4-pool - machine_type: n1-standard-1 - guest_accelerator: - - type: nvidia-tesla-t4 - count: 2 - gpu_driver_installation_config: - gpu_driver_version: "LATEST" - gpu_sharing_config: - max_shared_clients_per_gpu: 2 - gpu_sharing_strategy: "TIME_SHARING" -``` - -Finally, the following is adding multivpc to a node pool: - -```yaml - - id: network - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: gke-cluster - source: modules/scheduler/gke-cluster - use: [network, multinetwork] - settings: - cluster_name: $(vars.deployment_name) - - - id: a3-megagpu_pool - source: modules/compute/gke-node-pool - use: [gke-cluster, multinetwork] - settings: - machine_type: a3-megagpu-8g - ... -``` - -## Using GCE Reservations -You can reserve Google Compute Engine instances in a specific zone to ensure resources are available for their workloads when needed. For more details on how to manage reservations, see [Reserving Compute Engine zonal resources](https://cloud.google.com/compute/docs/instances/reserving-zonal-resources). - -After creating a reservation, you can consume the reserved GCE VM instances in GKE. GKE clusters deployed using Cluster Toolkit support the same consumption modes as Compute Engine: NO_RESERVATION(default), ANY_RESERVATION, SPECIFIC_RESERVATION. - -This can be accomplished using [`reservation_affinity`](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/compute/gke-node-pool/README.md#input_reservation_affinity). - -```yaml -# Target any reservation -reservation_affinity: - consume_reservation_type: ANY_RESERVATION - -# Target a specific reservation -reservation_affinity: - consume_reservation_type: SPECIFIC_RESERVATION - specific_reservations: - - name: specific-reservation-1 -``` - -The following requirements need to be satisfied for the node pool nodes to be able to use a specific reservation: -1. A reservation with the name must exist in the specified project(`var.project_id`) and one of the specified zones(`var.zones`). -2. Its consumption type must be `specific`. -3. Its GCE VM Properties must match with those of the Node Pool; Machine type, Accelerators (GPU Type and count), Local SSD disk type and count. - -If you want to utilise a shared reservation, the owner project of the shared reservation needs to be explicitly specified like the following. Note that a shared reservation can be used by the project that hosts the reservation (owner project) and by the projects the reservation is shared with (consumer projects). See how to [create and use a shared reservation](https://cloud.google.com/compute/docs/instances/reservations-shared). - -```yaml -reservation_affinity: - consume_reservation_type: SPECIFIC_RESERVATION - specific_reservations: - - name: specific-reservation-shared - project: shared_reservation_owner_project_id -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 7.2 | -| [google-beta](#requirement\_google-beta) | >= 7.2 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 7.2 | -| [google-beta](#provider\_google-beta) | >= 7.2 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | -| [tpu](#module\_tpu) | ../../internal/tpu-definition | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_container_node_pool.node_pool](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | -| [null_resource.enable_tcpx_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.enable_tcpxo_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.install_dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_machine_types.machine_info](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | -| [google_compute_region_instance_template.instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_template) | data source | -| [google_compute_reservation.specific_reservations](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [auto\_repair](#input\_auto\_repair) | Whether the nodes will be automatically repaired. | `bool` | `true` | no | -| [auto\_upgrade](#input\_auto\_upgrade) | Whether the nodes will be automatically upgraded. | `bool` | `false` | no | -| [autoscaling\_total\_max\_nodes](#input\_autoscaling\_total\_max\_nodes) | Total maximum number of nodes in the NodePool. | `number` | `1000` | no | -| [autoscaling\_total\_min\_nodes](#input\_autoscaling\_total\_min\_nodes) | Total minimum number of nodes in the NodePool. | `number` | `0` | no | -| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | -| [compact\_placement](#input\_compact\_placement) | DEPRECATED: Use `placement_policy` | `bool` | `null` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for each node. | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Disk type for each node. | `string` | `null` | no | -| [enable\_flex\_start](#input\_enable\_flex\_start) | If true, start the node pool with Flex Start provisioning model.
To learn more about flex-start mode, please refer to
https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and
https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest | `bool` | `false` | no | -| [enable\_gcfs](#input\_enable\_gcfs) | Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config). | `bool` | `false` | no | -| [enable\_numa\_aware\_scheduling](#input\_enable\_numa\_aware\_scheduling) | Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling. | `bool` | `false` | no | -| [enable\_private\_nodes](#input\_enable\_private\_nodes) | Whether nodes have internal IP addresses only. | `bool` | `true` | no | -| [enable\_queued\_provisioning](#input\_enable\_queued\_provisioning) | If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool. | `bool` | `false` | no | -| [enable\_secure\_boot](#input\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | -| [gke\_version](#input\_gke\_version) | GKE version | `string` | n/a | yes | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = optional(string)
    count = optional(number, 0)
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | -| [host\_maintenance\_interval](#input\_host\_maintenance\_interval) | Specifies the frequency of planned maintenance events. | `string` | `""` | no | -| [image\_type](#input\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | -| [initial\_node\_count](#input\_initial\_node\_count) | The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static\_node\_count and must be set to a value between autoscaling\_total\_min\_nodes and autoscaling\_total\_max\_nodes. | `number` | `null` | no | -| [internal\_ghpc\_module\_id](#input\_internal\_ghpc\_module\_id) | DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide). | `string` | n/a | yes | -| [is\_reservation\_active](#input\_is\_reservation\_active) | Whether the specified reservation is already created. | `bool` | `true` | no | -| [kubernetes\_labels](#input\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_count\_ephemeral\_storage](#input\_local\_ssd\_count\_ephemeral\_storage) | The number of local SSDs to attach to each node to back ephemeral storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | -| [local\_ssd\_count\_nvme\_block](#input\_local\_ssd\_count\_nvme\_block) | The number of local SSDs to attach to each node to back block storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | -| [machine\_type](#input\_machine\_type) | The name of a Google Compute Engine machine type. | `string` | `"c2-standard-60"` | no | -| [max\_pods\_per\_node](#input\_max\_pods\_per\_node) | The maximum number of pods per node in this node pool. This will force replacement. | `number` | `null` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [name](#input\_name) | The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix.
If setting manually, ensure a unique value across all gke-node-pools. | `string` | `null` | no | -| [num\_node\_pools](#input\_num\_node\_pools) | Number of node pools to create. This is same as num\_slices. | `number` | `1` | no | -| [num\_slices](#input\_num\_slices) | Number of TPUs slices to create. This is same as num\_node\_pools. | `number` | `1` | no | -| [placement\_policy](#input\_placement\_policy) | Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| 
{
  "name": null,
  "tpu_topology": null,
  "type": null
}
| no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Reservation resource to consume. When targeting SPECIFIC\_RESERVATION, specific\_reservations needs be specified.
Even though specific\_reservations is a list, only one reservation is allowed by the NodePool API.
It is assumed that the specified reservation exists and has available capacity.
For a shared reservation, specify the project\_id as well in which it was created.
To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared | 
object({
    consume_reservation_type = string
    specific_reservations = optional(list(object({
      name    = string
      project = optional(string)
    })))
  })
| 
{
  "consume_reservation_type": "NO_RESERVATION",
  "specific_reservations": []
}
| no | -| [run\_workload\_script](#input\_run\_workload\_script) | Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only. | `bool` | `true` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [static\_node\_count](#input\_static\_node\_count) | The static number of nodes in the node pool. If set, autoscaling will be disabled. | `number` | `null` | no | -| [taints](#input\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | -| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | -| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | -| [total\_max\_nodes](#input\_total\_max\_nodes) | DEPRECATED: Use autoscaling\_total\_max\_nodes. | `number` | `null` | no | -| [total\_min\_nodes](#input\_total\_min\_nodes) | DEPRECATED: Use autoscaling\_total\_min\_nodes. | `number` | `null` | no | -| [upgrade\_settings](#input\_upgrade\_settings) | Defines node pool upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | -| [zones](#input\_zones) | A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable. | `list(string)` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [allocatable\_cpu\_per\_node](#output\_allocatable\_cpu\_per\_node) | Number of CPUs available for scheduling pods on each node. | -| [allocatable\_gpu\_per\_node](#output\_allocatable\_gpu\_per\_node) | Number of GPUs available for scheduling pods on each node. | -| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | -| [guest\_accelerator](#output\_guest\_accelerator) | The accelerator type of the nodes. | -| [has\_gpu](#output\_has\_gpu) | Boolean value indicating whether nodes in the pool are configured with GPUs. | -| [instance\_templates](#output\_instance\_templates) | The URLs of Instance Templates | -| [instructions](#output\_instructions) | Instructions for submitting the sample GPUDirect enabled job. | -| [machine\_type](#output\_machine\_type) | Machine Type | -| [node\_count\_static](#output\_node\_count\_static) | The number of static nodes in node-pool. | -| [node\_pool\_names](#output\_node\_pool\_names) | Names of the node pools. | -| [static\_gpu\_count](#output\_static\_gpu\_count) | Total number of GPUs in the node pool. Available only for static node pools. | -| [tolerations](#output\_tolerations) | Tolerations needed for a pod to be scheduled on this node pool. | -| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | -| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | -| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | - diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf deleted file mode 100644 index 0c1c255255..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -## Required variables: -# local_ssd_count_ephemeral_storage -# local_ssd_count_nvme_block -# machine_type - -locals { - - local_ssd_machines = { - "a3-highgpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, - "a3-megagpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, - "a3-ultragpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, - "a4-highgpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, - } - - generated_local_ssd_config = lookup(local.local_ssd_machines, var.machine_type, { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = null }) - - # Select in priority order: - # (1) var.local_ssd_count_ephemeral_storage and var.local_ssd_count_nvme_block if any is not null - # (2) local.local_ssd_machines if not empty - # (3) default to null value for both local_ssd_count_ephemeral_storage and local_ssd_count_nvme_block - local_ssd_config = (var.local_ssd_count_ephemeral_storage == null && var.local_ssd_count_nvme_block == null) ? local.generated_local_ssd_config : { local_ssd_count_ephemeral_storage = var.local_ssd_count_ephemeral_storage, local_ssd_count_nvme_block = var.local_ssd_count_nvme_block } -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml deleted file mode 100644 index 1106f63479..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: batch/v1 -kind: Job -metadata: - name: my-sample-job -spec: - parallelism: 2 - completions: 2 - completionMode: Indexed - template: - spec: - containers: - - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx-dev:v3.1.9 - imagePullPolicy: Always - command: - - /bin/sh - - -c - - | - service ssh restart; - sleep infinity; - env: - - name: LD_LIBRARY_PATH - value: /usr/local/nvidia/lib64 - volumeMounts: - - name: config-volume - mountPath: /configs - resources: - limits: - nvidia.com/gpu: 8 - volumes: - - name: config-volume - configMap: - name: nccl-configmap - defaultMode: 0777 - restartPolicy: Never - backoffLimit: 0 diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml deleted file mode 100644 index bce6720681..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: batch/v1 -kind: Job -metadata: - name: my-sample-job -spec: - parallelism: 2 - completions: 2 - completionMode: Indexed - template: - spec: - hostname: host1 - subdomain: nccl-host-1 - containers: - - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.14 - imagePullPolicy: Always - command: - - /bin/sh - - -c - - | - set -ex - chmod 755 /scripts/demo-run-nccl-test-tcpxo-via-mpi.sh - cat >/scripts/allgather.sh < 0: - container["env"].extend(env_vars) - container["volumeMounts"].extend(volume_mounts) - -if __name__ == "__main__": - main() diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py deleted file mode 100644 index db9fb3e7ff..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import yaml -import argparse -import os - -def main(): - parser = argparse.ArgumentParser(description="TCPXO Job Manifest Generator") - parser.add_argument("-f", "--file", required=True, help="Path to your job template YAML file") - parser.add_argument("-r", "--rxdm", required=True, help="RxDM version") - - args = parser.parse_args() - - # Get the YAML file from the user - if not args.file: - args.file = input("Please provide the path to your job template YAML file: ") - - # Get component versions from user - if not args.rxdm: - args.rxdm = input("Enter the RxDM version: ") - - # Load and modify the YAML - with open(args.file, "r") as file: - job_manifest = yaml.load(file, Loader=yaml.BaseLoader) - - # Update annotations - add_annotations(job_manifest) - - # Update volumes - add_volumes(job_manifest) - - # Update tolerations - add_tolerations(job_manifest) - - # Add tcpxo-daemon container - add_tcpxo_daemon_container(job_manifest, args.rxdm) - - # Update environment variables and volumeMounts for GPU containers - update_gpu_containers(job_manifest) - - # Generate the new YAML file - updated_job = str(yaml.dump(job_manifest, default_flow_style=False, width=1000, default_style="|", sort_keys=False)).replace("|-", "") - - new_file_name = args.file.replace(".yaml", "-tcpxo.yaml") - with open(new_file_name, "w", encoding="utf-8") as file: - file.write(updated_job) - - # Step 7: Provide instructions to the user - print("\nA new manifest has been generated and updated to have TCPXO enabled based on the provided workload") - print("It can be found in {path}".format(path=os.path.abspath(new_file_name))) - print("You can use the following commands to submit the sample job:") - print(" kubectl create -f {path}".format(path=os.path.abspath(new_file_name))) - -def add_annotations(job_manifest): - annotations = { - 'devices.gke.io/container.tcpxo-daemon':"""|+ -- path: /dev/nvidia0 -- path: /dev/nvidia1 -- path: /dev/nvidia2 -- path: /dev/nvidia3 -- path: /dev/nvidia4 -- path: /dev/nvidia5 -- path: /dev/nvidia6 -- path: /dev/nvidia7 -- path: /dev/nvidiactl -- path: /dev/nvidia-uvm -- path: /dev/dmabuf_import_helper""", - "networking.gke.io/default-interface": "eth0", - "networking.gke.io/interfaces": """| -[ - {"interfaceName":"eth0","network":"default"}, - {"interfaceName":"eth1","network":"vpc1"}, - {"interfaceName":"eth2","network":"vpc2"}, - {"interfaceName":"eth3","network":"vpc3"}, - {"interfaceName":"eth4","network":"vpc4"}, - {"interfaceName":"eth5","network":"vpc5"}, - {"interfaceName":"eth6","network":"vpc6"}, - {"interfaceName":"eth7","network":"vpc7"}, - {"interfaceName":"eth8","network":"vpc8"} -]""", - } - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("metadata", {}) - - # Add/update annotations - pod_template_spec = job_manifest["spec"]["template"]["metadata"] - if "annotations" in pod_template_spec: - pod_template_spec["annotations"].update(annotations) - else: - pod_template_spec["annotations"] = annotations - -def add_tolerations(job_manifest): - tolerations = [ - {"key": "user-workload", "operator": "Equal", "value": """\"true\"""", "effect": "NoSchedule"}, - ] - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add tolerations - pod_spec = job_manifest["spec"]["template"]["spec"] - if "tolerations" in pod_spec: - pod_spec["tolerations"].extend(tolerations) - else: - pod_spec["tolerations"] = tolerations - -def add_volumes(job_manifest): - volumes = [ - {"name": "nvidia-install-dir-host", "hostPath": {"path": "/home/kubernetes/bin/nvidia"}}, - {"name": "sys", "hostPath": {"path": "/sys"}}, - {"name": "proc-sys", "hostPath": {"path": "/proc/sys"}}, - {"name": "aperture-devices", "hostPath": {"path": "/dev/aperture_devices"}}, - ] - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add volumes - pod_spec = job_manifest["spec"]["template"]["spec"] - if "volumes" in pod_spec: - pod_spec["volumes"].extend(volumes) - else: - pod_spec["volumes"] = volumes - - -def add_tcpxo_daemon_container(job_template, rxdm_version): - tcpxo_daemon_container = { - "name": "tcpxo-daemon", - "image": f"us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:{rxdm_version}", # Use provided RxDM version - "imagePullPolicy": "Always", - "command": ["/bin/sh", "-c"], - "args": [ - """| - set -ex - chmod 755 /fts/entrypoint_rxdm_container.sh - /fts/entrypoint_rxdm_container.sh --num_hops=2 --num_nics=8 --uid= --alsologtostderr""" - ], - "securityContext": { - "capabilities": {"add": ["NET_ADMIN", "NET_BIND_SERVICE"]} - }, - "volumeMounts": [ - {"name": "nvidia-install-dir-host", "mountPath": "/usr/local/nvidia"}, - {"name": "sys", "mountPath": "/hostsysfs"}, - {"name": "proc-sys", "mountPath": "/hostprocsysfs"}, - ], - "env": [{"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}], - } - - # Create path if it doesn't exist - job_template.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add container - pod_spec = job_template["spec"]["template"]["spec"] - pod_spec.setdefault("containers", []).insert(0, tcpxo_daemon_container) - -def update_gpu_containers(job_manifest): - env_vars = [ - {"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}, - {"name": "NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY", "value": "/dev/aperture_devices"}, - ] - volume_mounts = [{"name": "aperture-devices", "mountPath": "/dev/aperture_devices"}] - - pod_spec = job_manifest.get("spec", {}).get("template", {}).get("spec", {}) - for container in pod_spec.get("containers", []): - # Create path if it doesn't exist - container.setdefault("env", []) - container.setdefault("volumeMounts", []) - if int(container.get("resources", {}).get("limits", {}).get("nvidia.com/gpu", 0)) > 0: - container["env"].extend(env_vars) - container["volumeMounts"].extend(volume_mounts) - -if __name__ == "__main__": - main() diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf deleted file mode 100644 index d23d050986..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# Enable GPUDirect for A3 and A3Mega VMs, this involve multiple kubectl steps to integrate with the created cluster -# 1. Install NCCL plugin daemonset -# 2. Install NRI plugin daemonset -# 3. Update provided workload to inject rxdm sidecar and other required annotation, volume etc. -locals { - workload_path_tcpx = "${path.module}/gpu-direct-workload/sample-tcpx-workload-job.yaml" - workload_path_tcpxo = "${path.module}/gpu-direct-workload/sample-tcpxo-workload-job.yaml" - - gpu_direct_settings = { - "a3-highgpu-8g" = { - # Manifest to be installed for enabling TCPX on a3-highgpu-8g machines - gpu_direct_manifests = [ - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-tcpx-installer.yaml", # nccl_plugin v3.1.9 for tcpx - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-config.yaml", # nccl_configmap - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/nri_device_injector/nri-device-injector.yaml", # nri_plugin - ] - updated_workload_path = replace(local.workload_path_tcpx, ".yaml", "-tcpx.yaml") - rxdm_version = "v2.0.12" # matching nccl-tcpx-installer version v3.1.9 - min_additional_networks = 4 - major_minor_version_acceptable_map = { - "1.27" = "1.27.7-gke.1121000" - "1.28" = "1.28.8-gke.1095000" - "1.29" = "1.29.3-gke.1093000" - "1.30" = "1.30.2-gke.1023000" - } - } - "a3-megagpu-8g" = { - # Manifest to be installed for enabling TCPXO on a3-megagpu-8g machines - gpu_direct_manifests = [ - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/gpudirect-tcpxo/nccl-tcpxo-installer.yaml", # nccl_plugin v1.0.14 for tcpxo - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/bd4a7491672b48dfec28f3679b679a614f6cbbc7/nri_device_injector/nri-device-injector.yaml", # nri_plugin - ] - updated_workload_path = replace(local.workload_path_tcpxo, ".yaml", "-tcpxo.yaml") - rxdm_version = "v1.0.20" # matching nccl-tcpxo-installer version v1.0.14 - min_additional_networks = 8 - major_minor_version_acceptable_map = { - "1.28" = "1.28.9-gke.1250000" - "1.29" = "1.29.4-gke.1542000" - "1.30" = "1.30.4-gke.1129000" - "1.31" = "1.31.1-gke.2008000" - "1.32" = "1.32.2-gke.1489001" - } - } - } - - min_additional_networks = try(local.gpu_direct_settings[var.machine_type].min_additional_networks, 0) - - gke_version_regex = "(\\d+\\.\\d+)\\.(\\d+)-gke\\.(\\d+)" # GKE version format: 1.X.Y-gke.Z , regex output: ["1.X" , "Y", "Z"] - - gke_version_parts = regex(local.gke_version_regex, var.gke_version) - gke_version_major = local.gke_version_parts[0] - - major_minor_version_acceptable_map = try(local.gpu_direct_setting[var.machine_type].major_minor_version_acceptable_map, null) - minor_version_acceptable = try(contains(keys(local.major_minor_version_acceptable_map), local.gke_version_major), false) ? local.major_minor_version_acceptable_map[local.gke_version_major] : "1.0.0-gke.0" - minor_version_acceptable_parts = regex(local.gke_version_regex, local.minor_version_acceptable) - gke_gpudirect_compatible = local.gke_version_parts[1] > local.minor_version_acceptable_parts[1] || (local.gke_version_parts[1] == local.minor_version_acceptable_parts[1] && local.gke_version_parts[2] >= local.minor_version_acceptable_parts[2]) -} - -check "gpu_direct_check_multi_vpc" { - assert { - condition = length(var.additional_networks) >= local.min_additional_networks - error_message = "To achieve optimal performance for ${var.machine_type} machine, at least ${local.min_additional_networks} additional vpc is recommended. You could configure it in the blueprint through modules/network/multivpc with network_count set as ${local.min_additional_networks}" - } -} - -check "gke_version_requirements" { - assert { - condition = local.gke_gpudirect_compatible - error_message = "GPUDirect is not supported on GKE version ${var.gke_version} for ${var.machine_type} machine. For supported version details visit https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#requirements" - } -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf deleted file mode 100644 index 1ddc7ba8c3..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -data "google_compute_machine_types" "machine_info" { - for_each = var.zones == null ? toset([]) : toset(var.zones) - - project = var.project_id - zone = each.key - filter = "name = \"${var.machine_type}\"" -} - -locals { - valid_machine_info = { - for zone, data in data.google_compute_machine_types.machine_info : - zone => data.machine_types if length(data.machine_types) > 0 - } - - guest_cpus = try(local.valid_machine_info[0].guest_cpus, 0) -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/main.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/main.tf deleted file mode 100644 index 05314497fc..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/main.tf +++ /dev/null @@ -1,482 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-node-pool", ghpc_role = "compute" }) -} - -locals { - upgrade_settings = { - strategy = var.upgrade_settings.strategy - max_surge = coalesce(var.upgrade_settings.max_surge, 0) - max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) - } -} - -module "gpu" { - source = "../../internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - has_gpu = length(local.guest_accelerator) > 0 - allocatable_gpu_per_node = local.has_gpu ? max(local.guest_accelerator[*].count...) : -1 - is_static_node_pool_with_gpus = var.static_node_count != null && local.allocatable_gpu_per_node != -1 - static_gpu_count = local.is_static_node_pool_with_gpus ? var.static_node_count * local.allocatable_gpu_per_node : 0 - gpu_taint = local.has_gpu ? [{ - key = "nvidia.com/gpu" - value = "present" - effect = "NO_SCHEDULE" - }] : [] - - autoscale_set = var.autoscaling_total_min_nodes != 0 || var.autoscaling_total_max_nodes != 1000 - static_node_set = var.static_node_count != null - initial_node_set = try(var.initial_node_count > 0, false) - - module_unique_id = replace(lower(var.internal_ghpc_module_id), "/[^a-z0-9\\-]/", "") -} - - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] -} - -module "tpu" { - source = "../../internal/tpu-definition" - - machine_type = var.machine_type - placement_policy = var.placement_policy -} - - -data "google_container_cluster" "gke_cluster" { - name = local.cluster_name - location = local.cluster_location -} - -resource "google_container_node_pool" "node_pool" { - provider = google-beta - - count = max(var.num_node_pools, var.num_slices) - - name = (max(var.num_node_pools, var.num_slices) == 1) ? coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])) : join("-", [coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])), count.index]) - cluster = var.cluster_id - node_locations = var.zones - - node_count = var.static_node_count - dynamic "autoscaling" { - for_each = local.static_node_set ? [] : [1] - content { - total_min_node_count = var.autoscaling_total_min_nodes - total_max_node_count = var.autoscaling_total_max_nodes - location_policy = "ANY" - } - } - - initial_node_count = var.initial_node_count - - max_pods_per_node = var.max_pods_per_node - - management { - auto_repair = var.auto_repair - auto_upgrade = var.auto_upgrade - } - - upgrade_settings { - strategy = local.upgrade_settings.strategy - max_surge = local.upgrade_settings.max_surge - max_unavailable = local.upgrade_settings.max_unavailable - } - - dynamic "placement_policy" { - for_each = var.placement_policy.type != null ? [1] : [] - content { - type = var.placement_policy.type - policy_name = var.placement_policy.name - tpu_topology = module.tpu.is_tpu ? var.placement_policy.tpu_topology : null - } - } - - dynamic "queued_provisioning" { - for_each = var.enable_queued_provisioning ? [1] : [] - content { - enabled = true - } - } - - node_config { - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - resource_labels = local.labels - labels = var.kubernetes_labels - service_account = var.service_account_email - oauth_scopes = var.service_account_scopes - machine_type = var.machine_type - spot = var.spot - image_type = var.image_type - flex_start = var.enable_flex_start - max_run_duration = var.max_run_duration != null ? "${var.max_run_duration}s" : null - - dynamic "guest_accelerator" { - for_each = local.guest_accelerator - iterator = ga - content { - type = coalesce(ga.value.type, try(local.generated_guest_accelerator[0].type, "")) - count = coalesce(try(ga.value.count, 0) > 0 ? ga.value.count : try(local.generated_guest_accelerator[0].count, "0")) - - gpu_partition_size = try(ga.value.gpu_partition_size, null) - - dynamic "gpu_driver_installation_config" { - # in case user did not specify guest_accelerator settings, we need a try to default to [] - for_each = try([ga.value.gpu_driver_installation_config], [{ gpu_driver_version = "DEFAULT" }]) - iterator = gdic - content { - gpu_driver_version = gdic.value.gpu_driver_version - } - } - - dynamic "gpu_sharing_config" { - for_each = try(ga.value.gpu_sharing_config == null, true) ? [] : [ga.value.gpu_sharing_config] - iterator = gsc - content { - gpu_sharing_strategy = gsc.value.gpu_sharing_strategy - max_shared_clients_per_gpu = gsc.value.max_shared_clients_per_gpu - } - } - } - } - - dynamic "taint" { - for_each = concat(var.taints, local.gpu_taint, module.tpu.tpu_taint) - content { - key = taint.value.key - value = taint.value.value - effect = taint.value.effect - } - } - - dynamic "ephemeral_storage_local_ssd_config" { - for_each = local.local_ssd_config.local_ssd_count_ephemeral_storage != null ? [1] : [] - content { - local_ssd_count = local.local_ssd_config.local_ssd_count_ephemeral_storage - } - } - - dynamic "local_nvme_ssd_block_config" { - for_each = local.local_ssd_config.local_ssd_count_nvme_block != null ? [1] : [] - content { - local_ssd_count = local.local_ssd_config.local_ssd_count_nvme_block - } - } - - shielded_instance_config { - enable_secure_boot = var.enable_secure_boot - enable_integrity_monitoring = true - } - - dynamic "gcfs_config" { - for_each = var.enable_gcfs ? [1] : [] - content { - enabled = true - } - } - - gvnic { - enabled = var.image_type == "COS_CONTAINERD" - } - - dynamic "advanced_machine_features" { - for_each = local.set_threads_per_core ? [1] : [] - content { - threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf - } - } - - # Implied by Workload Identity - workload_metadata_config { - mode = "GKE_METADATA" - } - # Implied by workload identity. - metadata = { - "disable-legacy-endpoints" = "true" - } - - linux_node_config { - sysctls = { - "net.ipv4.tcp_rmem" = "4096 87380 16777216" - "net.ipv4.tcp_wmem" = "4096 16384 16777216" - } - } - - reservation_affinity { - consume_reservation_type = var.reservation_affinity.consume_reservation_type - key = local.is_valid_reservation ? local.reservation_resource_api_label : null - values = local.is_valid_reservation ? (var.is_reservation_active ? local.active_reservation_values : local.default_reservation_values) : null - } - - dynamic "host_maintenance_policy" { - for_each = var.host_maintenance_interval != "" ? [1] : [] - content { - maintenance_interval = var.host_maintenance_interval - } - } - - kubelet_config { - cpu_manager_policy = var.enable_numa_aware_scheduling ? "static" : null - dynamic "topology_manager" { - for_each = var.enable_numa_aware_scheduling ? [1] : [] - content { - policy = "restricted" - } - } - dynamic "memory_manager" { - for_each = var.enable_numa_aware_scheduling ? [1] : [] - content { - policy = "Static" - } - } - } - } - - network_config { - dynamic "additional_node_network_configs" { - for_each = var.additional_networks - - content { - network = additional_node_network_configs.value.network - subnetwork = additional_node_network_configs.value.subnetwork - } - } - - enable_private_nodes = var.enable_private_nodes - } - - timeouts { - create = var.timeout_create - update = var.timeout_update - } - - lifecycle { - ignore_changes = [ - node_config[0].labels, - initial_node_count, - # Ignore local/ephemeral ssd configs as they are tied to machine types. - node_config[0].ephemeral_storage_local_ssd_config, - node_config[0].local_nvme_ssd_block_config, - ] - precondition { - condition = (var.max_pods_per_node == null) || (data.google_container_cluster.gke_cluster.networking_mode == "VPC_NATIVE") - error_message = "max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." - } - precondition { - condition = !local.static_node_set || !local.autoscale_set - error_message = "static_node_count cannot be set with either autoscaling_total_min_nodes or autoscaling_total_max_nodes." - } - precondition { - condition = !local.static_node_set || !local.initial_node_set - error_message = "initial_node_count cannot be set with static_node_count." - } - precondition { - condition = !local.initial_node_set || (coalesce(var.initial_node_count, 0) >= var.autoscaling_total_min_nodes && coalesce(var.initial_node_count, 0) <= var.autoscaling_total_max_nodes) - error_message = "initial_node_count must be between autoscaling_total_min_nodes and autoscaling_total_max_nodes included." - } - precondition { - condition = !(coalesce(local.local_ssd_config.local_ssd_count_ephemeral_storage, 0) > 0 && coalesce(local.local_ssd_config.local_ssd_count_nvme_block, 0) > 0) - error_message = "Only one of local_ssd_count_ephemeral_storage or local_ssd_count_nvme_block can be set to a non-zero value." - } - precondition { - condition = ( - (var.reservation_affinity.consume_reservation_type != "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 0) || - (var.reservation_affinity.consume_reservation_type == "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 1) - ) - error_message = <<-EOT - When using NO_RESERVATION or ANY_RESERVATION as the `consume_reservation_type`, `specific_reservations` cannot be set. - On the other hand, with SPECIFIC_RESERVATION you must set `specific_reservations`. - EOT - } - precondition { - condition = ( - (local.input_specific_reservations_count == 0) || - ((length(local.verified_specific_reservations) == 1 || !var.is_reservation_active) && - length(local.specific_reservation_requirement_violations) == 0) - ) - error_message = <<-EOT - Check if your reservation is configured correctly: - - A reservation with the name must exist in the specified project and one of the specified zones - - - Its consumption type must be "specific" - %{for property in local.specific_reservation_requirement_violations} - - ${local.specific_reservation_requirement_violation_messages[property]} - %{endfor} - EOT - } - precondition { - condition = ( - (local.input_specific_reservations_count == 0) || - (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) == 0) || - (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) > 0 && try(local.input_reservation_projects[0], var.project_id) == var.project_id) - ) - error_message = "Shared extended reservations are not supported by GKE." - } - precondition { - condition = contains(["SURGE"], local.upgrade_settings.strategy) - error_message = "Only SURGE strategy is supported" - } - precondition { - condition = local.upgrade_settings.max_unavailable >= 0 - error_message = "max_unavailable should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_surge >= 0 - error_message = "max_surge should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 - error_message = "At least one of max_unavailable or max_surge must greater than 0" - } - precondition { - condition = var.placement_policy.type != "COMPACT" || (var.zones != null ? (length(var.zones) == 1) : false) - error_message = "Compact placement is only available for node pools operating in a single zone." - } - precondition { - condition = var.placement_policy.type != "COMPACT" || local.upgrade_settings.strategy != "BLUE_GREEN" - error_message = "Compact placement is not supported with blue-green upgrades." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.placement_policy.type == "COMPACT") - error_message = "placement_policy cannot be COMPACT when enable_queued_provisioning is true." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.reservation_affinity.consume_reservation_type != "NO_RESERVATION") - error_message = "reservation_affinity should be NO_RESERVATION when enable_queued_provisioning is true." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.autoscaling_total_min_nodes != 0) - error_message = "autoscaling_total_min_nodes should be 0 when enable_queued_provisioning is true." - } - precondition { - condition = !(var.num_node_pools > 1 && var.num_slices > 1) - error_message = "num_node_pools is for CPUs and GPUS, and num_slices is for TPUs. Both cannot be set at the same time to create a group of identical nodepools / slices." - } - precondition { - condition = !(var.num_node_pools == 0 && var.num_slices == 0) - error_message = "Either num_node_pools (for CPUs and GPUS) or num_slices (for TPUs) should be set to a positive integer value." - } - precondition { - condition = !(var.num_node_pools < 0 || var.num_slices < 0) - error_message = "Negative integer value of num_node_pools or num_slices is not valid. Please use a positive integer value to set num_node_pools for CPUs and GPUS, and num_slices for TPUs." - } - precondition { - condition = var.enable_flex_start == true ? (var.auto_repair == false) : true - error_message = "enable_flex_start needs node auto_repair set to false." - } - precondition { - condition = var.enable_flex_start == true ? (var.static_node_count == null) : true - error_message = "enable_flex_start does not work with static_node_count. static_node_count should be set to null." - } - precondition { - condition = var.enable_flex_start == true ? (var.reservation_affinity.consume_reservation_type == "NO_RESERVATION") : true - error_message = "enable_flex_start only works with reservation_affinity consume_reservation_type NO_RESERVATION." - } - precondition { - condition = var.enable_flex_start == true ? (var.spot == false) : true - error_message = "Both enable_flex_start and spot consumption option cannot be set to true at the same time." - } - } -} - -locals { - supported_machine_types_for_install_dependencies = ["a3-highgpu-8g", "a3-megagpu-8g"] -} - -# Replicates GKE's naming logic for its instance templates. The full -# pattern is "gke-{cluster_name}-{nodepool_name}-{hash}". -# -# This code builds the "{cluster_name}-{nodepool_name}" prefix, which is -# capped at 32 characters plus a dash '-' in between, by truncating names if needed: -# - If both names > 16 chars, both are cut to 16. -# - If one name > 16, it's shortened so the combined name length is 32. -data "google_compute_region_instance_template" "instance_template" { - for_each = { for idx, np in google_container_node_pool.node_pool : idx => np } - project = var.project_id - filter = "name: gke-${ - (length(local.cluster_name) <= 16 && length(each.value.name) <= 16) ? "${local.cluster_name}-${each.value.name}" : - (length(local.cluster_name) > 16 && length(each.value.name) > 16) ? "${substr(local.cluster_name, 0, 16)}-${substr(each.value.name, 0, 16)}" : - (length(local.cluster_name) > 16) ? "${substr(local.cluster_name, 0, 32 - length(each.value.name))}-${each.value.name}" : - "${local.cluster_name}-${substr(each.value.name, 0, 32 - length(local.cluster_name))}" - }*" - most_recent = true -} - -resource "null_resource" "install_dependencies" { - count = var.run_workload_script && contains(local.supported_machine_types_for_install_dependencies, var.machine_type) ? 1 : 0 - provisioner "local-exec" { - command = "pip3 install pyyaml" - } -} - -locals { - gpu_direct_setting = lookup(local.gpu_direct_settings, var.machine_type, { gpu_direct_manifests = [], updated_workload_path = "", rxdm_version = "" }) -} - -# execute script to inject rxdm sidecar into workload to enable tcpx for a3-highgpu-8g VM workload -resource "null_resource" "enable_tcpx_in_workload" { - count = var.run_workload_script && var.machine_type == "a3-highgpu-8g" ? 1 : 0 - triggers = { - always_run = timestamp() - } - provisioner "local-exec" { - command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpx-in-workload.py --file ${local.workload_path_tcpx} --rxdm ${local.gpu_direct_setting.rxdm_version}" - } - - depends_on = [null_resource.install_dependencies] -} - -# execute script to inject rxdm sidecar into workload to enable tcpxo for a3-megagpu-8g VM workload -resource "null_resource" "enable_tcpxo_in_workload" { - count = var.run_workload_script && var.machine_type == "a3-megagpu-8g" ? 1 : 0 - triggers = { - always_run = timestamp() - } - provisioner "local-exec" { - command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py --file ${local.workload_path_tcpxo} --rxdm ${local.gpu_direct_setting.rxdm_version}" - } - - depends_on = [null_resource.install_dependencies] -} - -# apply manifest to enable tcpx -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = var.cluster_id - project_id = var.project_id - - apply_manifests = flatten([ - for manifest in local.gpu_direct_setting.gpu_direct_manifests : [ - { - source = manifest - } - ] - ]) -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml deleted file mode 100644 index e980d595a2..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com -ghpc: - inject_module_id: internal_ghpc_module_id diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf deleted file mode 100644 index 44e1c3d971..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "node_pool_names" { - description = "Names of the node pools." - value = google_container_node_pool.node_pool[*].name -} - -locals { - # Shared core machines only have 1 cpu allocatable, even if they have 2 cpu capacity - vcpu = local.machine_shared_core ? 1 : local.guest_cpus - useable_cpu = local.set_threads_per_core ? local.threads_per_core * local.vcpu / 2 : local.vcpu - - # allocatable resource definition: https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes#cpu_reservations - second_core = local.useable_cpu > 1 ? 1 : 0 - third_fourth_core = local.useable_cpu == 3 ? 1 : local.useable_cpu > 3 ? 2 : 0 - cores_above_four = local.useable_cpu > 4 ? local.useable_cpu - 4 : 0 - - allocatable_cpu = 0.94 + (0.99 * local.second_core) + (0.995 * local.third_fourth_core) + (0.9975 * local.cores_above_four) -} - -output "allocatable_cpu_per_node" { - description = "Number of CPUs available for scheduling pods on each node." - value = local.allocatable_cpu -} - -output "has_gpu" { - description = "Boolean value indicating whether nodes in the pool are configured with GPUs." - value = local.has_gpu -} - -output "allocatable_gpu_per_node" { - description = "Number of GPUs available for scheduling pods on each node." - value = local.allocatable_gpu_per_node -} - -output "static_gpu_count" { - description = "Total number of GPUs in the node pool. Available only for static node pools." - value = local.static_gpu_count -} - -locals { - translate_toleration = { - PREFER_NO_SCHEDULE = "PreferNoSchedule" - NO_SCHEDULE = "NoSchedule" - NO_EXECUTE = "NoExecute" - } - taints = google_container_node_pool.node_pool[0].node_config[0].taint - tolerations = [for taint in local.taints : { - key = taint.key - operator = "Equal" - value = taint.value - effect = lookup(local.translate_toleration, taint.effect, null) - }] -} - -output "tolerations" { - description = "Tolerations needed for a pod to be scheduled on this node pool." - value = local.tolerations -} - -locals { - gpu_direct_enabled = var.machine_type == "a3-highgpu-8g" || var.machine_type == "a3-megagpu-8g" - script_path = { - a3-highgpu-8g = "enable-tcpx-in-workload.py", - a3-megagpu-8g = "enable-tcpxo-in-workload.py" - } - nccl_path = var.machine_type == "a3-highgpu-8g" ? "configs" : "scripts" - gpu_direct_instruction = <<-EOT - Since you are using ${var.machine_type} machine type that has GPUDirect support, your nodepool had been configured with the required plugins. - To fully utilize GPUDirect you will need to add some components into your workload manifest. Details below: - - A sample GKE job that has GPUDirect enabled and NCCL test included has been generated locally at: - ${abspath(local.gpu_direct_setting.updated_workload_path)} - - You can use the following commands to submit the sample job: - kubectl create -f ${abspath(local.gpu_direct_setting.updated_workload_path)} - After submitting the sample job, you can validate the GPU performance by initiating NCCL test included in the sample workload: - NCCL test can be initiated from any one of the sample job Pods and coordinate with the peer Pods: - export POD_NAME=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | head -n 1) - export PEER_POD_IPS=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.status.podIP}}{{" "}}{{end}}') - kubectl exec --stdin --tty --container=nccl-test $POD_NAME -- /${local.nccl_path}/allgather.sh $PEER_POD_IPS - - If you would like to enable GPUDirect for your own workload, please follow the below steps: - export WORKLOAD_PATH=<> - python3 ${abspath("${path.module}/gpu-direct-workload/scripts/${lookup(local.script_path, var.machine_type, "")}")} --file $WORKLOAD_PATH --rxdm ${local.gpu_direct_setting.rxdm_version} - **WARNING** - The "--rxdm" version is tied to the nccl-tcpx/o-installer that had been deployed to your cluster, changing it to other value might have impact on performance - **WARNING** - - Or you can also follow our GPUDirect user guide to update your workload - https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#add-gpudirect-manifests - EOT -} - -output "instructions" { - description = "Instructions for submitting the sample GPUDirect enabled job." - value = local.gpu_direct_enabled ? local.gpu_direct_instruction : null -} - -output "node_count_static" { - description = "The number of static nodes in node-pool." - value = coalesce(var.static_node_count, var.initial_node_count, 0) -} - -output "guest_accelerator" { - description = "The accelerator type of the nodes." - value = local.guest_accelerator -} - -output "cluster_id" { - description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." - value = var.cluster_id -} - -output "machine_type" { - description = "Machine Type" - value = var.machine_type -} - -output "instance_templates" { - description = "The URLs of Instance Templates" - value = [for key, template in data.google_compute_region_instance_template.instance_template : template.self_link] -} - -output "tpu_accelerator_type" { - description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." - value = module.tpu.is_tpu ? module.tpu.tpu_accelerator_type : null -} - -output "tpu_topology" { - description = "The topology of the TPU slice (e.g., '4x4')." - value = module.tpu.is_tpu ? module.tpu.tpu_topology : null -} - -output "tpu_chips_per_node" { - description = "The number of TPU chips on each node in the pool." - value = module.tpu.is_tpu ? module.tpu.tpu_chips_per_node : null -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf deleted file mode 100644 index 7c29e3902a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# Split the input into three different lists where the details of a given reservation are at the same index across these lists. -locals { - # Specific block of an extended reservation can be targeted with exr-one/reservationBlocks/exr-one-block-1 - # Data source needs to be queried with the reservation name only. So, we extract the reservation name - input_reservation_names = [for r in try(var.reservation_affinity.specific_reservations, []) : split("/", r.name)[0]] - input_reservation_projects = [for r in try(var.reservation_affinity.specific_reservations, []) : coalesce(r.project, var.project_id)] - # We, also, remember the suffix "/reservationBlocks/exr-one-block-1" for use elsewhere afterwards - input_reservation_suffixes = [for r in try(var.reservation_affinity.specific_reservations, []) : substr(r.name, length(split("/", r.name)[0]), -1)] - # Adding this variable to by-pass the machine-type validation for TPUs - is_tpu = var.placement_policy.tpu_topology != null -} - -data "google_compute_reservation" "specific_reservations" { - for_each = ( - local.input_specific_reservations_count == 0 ? - {} : - { - for pair in flatten([ - for zone in try(var.zones, []) : [ - for i, reservation_name in try(local.input_reservation_names, []) : { - key : "${local.input_reservation_projects[i]}/${zone}/${reservation_name}" - zone : zone - reservation_name : reservation_name - project : local.input_reservation_projects[i] - } - ] - ]) : - pair.key => pair - } - ) - name = each.value.reservation_name - zone = each.value.zone - project = each.value.project -} - -locals { - generated_guest_accelerator = module.gpu.machine_type_guest_accelerator - reservation_resource_api_label = "compute.googleapis.com/reservation-name" - input_specific_reservations_count = try(length(var.reservation_affinity.specific_reservations), 0) - - # Filter specific reservations - verified_specific_reservations = [for k, v in data.google_compute_reservation.specific_reservations : v if(v.specific_reservation != null && v.specific_reservation_required == true)] - - # Build two maps to be used to compare the VM properties between reservations and the node pool - # Validation of only machine-type for CPUs and and both machine-type and guest-accelerators for GPUs - # Skip this for TPUs ( returns an empty list to skip the machine-type validation for aggregate TPU reservations) - reservation_vm_properties = local.is_tpu ? [] : [for reservation in local.verified_specific_reservations : { - "machine_type" : try(reservation.specific_reservation[0].instance_properties[0].machine_type, "") - "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs - { for acc in try(reservation.specific_reservation[0].instance_properties[0].guest_accelerators, []) : acc.accelerator_type => acc.accelerator_count } - ) : {} # If no GPUs, it's an empty map {} - }] - - nodepool_vm_properties = { - "machine_type" : var.machine_type - "guest_accelerators" : local.has_gpu ? ( # Conditional check for GPUs - { for acc in try(local.guest_accelerator, []) : coalesce(acc.type, try(local.generated_guest_accelerator[0].type, "")) => coalesce(acc.count, try(local.generated_guest_accelerator[0].count, 0)) } - ) : {} # If no GPUs, it's an empty map {} - } - - # Compare two maps by counting the keys that mismatch. - # Know that in map comparison the order of keys does not matter. That is {NVME: x, SCSI: y} and {SCSI: y, NVME: x} are equal - # As of this writing, there is only one reservation supported by the Node Pool API. So, directly accessing it from the list - specific_reservation_requirement_violations = length(local.reservation_vm_properties) == 0 ? [] : [for k, v in local.nodepool_vm_properties : k if v != local.reservation_vm_properties[0][k]] - - specific_reservation_requirement_violation_messages = { - "machine_type" : <<-EOT - The reservation has "${try(local.reservation_vm_properties[0].machine_type, "")}" machine type and the node pool has "${local.nodepool_vm_properties.machine_type}". Check the relevant node pool setting: "machine_type" - EOT - "guest_accelerators" : <<-EOT - The reservation has ${jsonencode(try(local.reservation_vm_properties[0].guest_accelerators, {}))} accelerators and the node pool has ${jsonencode(try(local.nodepool_vm_properties.guest_accelerators, {}))}. Check the relevant node pool setting: "guest_accelerator". When unspecified, for the machine_type=${var.machine_type}, the default is guest_accelerator=${jsonencode(try(local.generated_guest_accelerator, [{}]))}. - EOT - } -} - -locals { - # Check if reservation is valid, that is, if it exists, there should be only 1 verified specific reservation or the reservation doesn't exist - is_valid_reservation = length(local.verified_specific_reservations) == 1 || !var.is_reservation_active - - # Build the list of reservation names when var.is_reservation_active is true - active_reservation_values = [ - for i, r in local.verified_specific_reservations : - length(local.input_reservation_suffixes[i]) > 0 ? - format("%s%s", r.name, local.input_reservation_suffixes[i]) : - "projects/${r.project}/reservations/${r.name}" - ] - - # Define a default reservation value if no specific reservations are present - specific_reservation_name = length(local.input_reservation_names) > 0 ? local.input_reservation_names[0] : "" - default_reservation_values = ["projects/${var.project_id}/reservations/${local.specific_reservation_name}"] -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf deleted file mode 100644 index e582db33da..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' - -# "inputs": -# var.machine_type: Machine type for the instance being evaluated. -# var.threads_per_core : Sets the number of threads per physical core, where 0 -# has behavior described in description. - -# "outputs": -# local.set_threads_per_core: bool that tells if threads per core should be set, -# to be used with a dynamic block. -# local.threads_per_core: actual threads_per_core to be used. - -locals { - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - machine_shared_core = length(local.machine_vals) <= 2 - machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) - - smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) - smt_capable_vcpu = local.machine_vcpus >= 2 - - smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core - set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) - threads_per_core = var.threads_per_core == 2 ? 2 : 1 -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf deleted file mode 100644 index b44ea28d57..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf +++ /dev/null @@ -1,487 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "cluster_id" { - description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" - type = string -} - -variable "zones" { - description = "A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable." - type = list(string) - default = null -} - -variable "name" { - description = <<-EOD - The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix. - If setting manually, ensure a unique value across all gke-node-pools. - EOD - type = string - default = null - - validation { - # Check if the variable is null OR if it matches the GCP resource naming regex. - condition = var.name == null || can(regex("^[a-z]([-a-z0-9]{0,34}[a-z0-9])?$", var.name)) - error_message = <<-EOD - If provided, the node pool name must be between 1 and 36 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. - Underscores are not allowed. A shorter length is enforced to accommodate a suffix when creating multiple node pools. - EOD - } -} - -variable "internal_ghpc_module_id" { - description = "DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide)." - type = string -} - -variable "machine_type" { - description = "The name of a Google Compute Engine machine type." - type = string - default = "c2-standard-60" -} - -variable "disk_size_gb" { - description = "Size of disk for each node." - type = number - default = 100 -} - -variable "disk_type" { - description = "Disk type for each node." - type = string - default = null -} - -variable "enable_gcfs" { - description = "Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config)." - type = bool - default = false -} - -variable "enable_secure_boot" { - description = "Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info." - type = bool - default = true -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = optional(string) - count = optional(number, 0) - gpu_driver_installation_config = optional(object({ - gpu_driver_version = string - }), { gpu_driver_version = "DEFAULT" }) - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = string - max_shared_clients_per_gpu = number - })) - })) - default = [] - nullable = false - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.count != null]) - error_message = "var.guest_accelerator[*].count cannot be null" - } - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.count >= 0]) - error_message = "var.guest_accelerator[*].count must never be negative" - } - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.gpu_driver_installation_config != null]) - error_message = "var.guest_accelerator[*].gpu_driver_installation_config must not be null; leave unset to enable GKE to select default GPU driver installation" - } -} - -variable "image_type" { - description = "The default image type used by NAP once a new node pool is being created. Use either COS_CONTAINERD or UBUNTU_CONTAINERD." - type = string - default = "COS_CONTAINERD" -} - -variable "local_ssd_count_ephemeral_storage" { - description = <<-EOT - The number of local SSDs to attach to each node to back ephemeral storage. - Uses NVMe interfaces. Must be supported by `machine_type`. - When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. - [See above](#local-ssd-storage) for more info. - EOT - type = number - default = null -} - -variable "local_ssd_count_nvme_block" { - description = <<-EOT - The number of local SSDs to attach to each node to back block storage. - Uses NVMe interfaces. Must be supported by `machine_type`. - When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. - [See above](#local-ssd-storage) for more info. - - EOT - type = number - default = null -} - -variable "autoscaling_total_min_nodes" { - description = "Total minimum number of nodes in the NodePool." - type = number - default = 0 -} - -variable "autoscaling_total_max_nodes" { - description = "Total maximum number of nodes in the NodePool." - type = number - default = 1000 -} - -variable "static_node_count" { - description = "The static number of nodes in the node pool. If set, autoscaling will be disabled." - type = number - default = null -} - -variable "is_reservation_active" { - description = "Whether the specified reservation is already created." - type = bool - default = true -} - -variable "auto_repair" { - description = "Whether the nodes will be automatically repaired." - type = bool - default = true -} - -variable "auto_upgrade" { - description = "Whether the nodes will be automatically upgraded." - type = bool - default = false -} - -variable "threads_per_core" { - description = <<-EOT - Sets the number of threads per physical core. By setting threads_per_core - to 2, Simultaneous Multithreading (SMT) is enabled extending the total number - of virtual cores. For example, a machine of type c2-standard-60 will have 60 - virtual cores with threads_per_core equal to 2. With threads_per_core equal - to 1 (SMT turned off), only the 30 physical cores will be available on the VM. - - The default value of \"0\" will turn off SMT for supported machine types, and - will fall back to GCE defaults for unsupported machine types (t2d, shared-core - instances, or instances with less than 2 vCPU). - - Disabling SMT can be more performant in many HPC workloads, therefore it is - disabled by default where compatible. - - null = SMT configuration will use the GCE defaults for the machine type - 0 = SMT will be disabled where compatible (default) - 1 = SMT will always be disabled (will fail on incompatible machine types) - 2 = SMT will always be enabled (will fail on incompatible machine types) - EOT - type = number - default = 0 - - validation { - condition = var.threads_per_core == null || try(var.threads_per_core >= 0, false) && try(var.threads_per_core <= 2, false) - error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." - } -} - -variable "spot" { - description = "Provision VMs using discounted Spot pricing, allowing for preemption" - type = bool - default = false -} - -# tflint-ignore: terraform_unused_declarations -variable "compact_placement" { - description = "DEPRECATED: Use `placement_policy`" - type = bool - default = null - validation { - condition = var.compact_placement == null - error_message = "`compact_placement` is deprecated. Use `placement_policy` instead" - } -} - -variable "placement_policy" { - description = <<-EOT - Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool. - It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - EOT - - type = object({ - type = string - name = optional(string) - tpu_topology = optional(string) - }) - default = { - type = null - name = null - tpu_topology = null - } - validation { - condition = var.placement_policy.type == null || try(contains(["COMPACT"], var.placement_policy.type), false) - error_message = "`COMPACT` is the only supported value for `placement_policy.type`." - } -} - -variable "service_account_email" { - description = "Service account e-mail address to use with the node pool" - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to to use with the node pool." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "taints" { - description = "Taints to be applied to the system node pool." - type = list(object({ - key = string - value = any - effect = string - })) - default = [] -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} - -variable "kubernetes_labels" { - description = <<-EOT - Kubernetes labels to be applied to each node in the node group. Key-value pairs. - (The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) - EOT - type = map(string) - default = null -} - -variable "timeout_create" { - description = "Timeout for creating a node pool" - type = string - default = null -} - -variable "timeout_update" { - description = "Timeout for updating a node pool" - type = string - default = null -} - -# Deprecated - -# tflint-ignore: terraform_unused_declarations -variable "total_min_nodes" { - description = "DEPRECATED: Use autoscaling_total_min_nodes." - type = number - default = null - validation { - condition = var.total_min_nodes == null - error_message = "total_min_nodes was renamed to autoscaling_total_min_nodes and is deprecated; use autoscaling_total_min_nodes" - } -} - -# tflint-ignore: terraform_unused_declarations -variable "total_max_nodes" { - description = "DEPRECATED: Use autoscaling_total_max_nodes." - type = number - default = null - validation { - condition = var.total_max_nodes == null - error_message = "total_max_nodes was renamed to autoscaling_total_max_nodes and is deprecated; use autoscaling_total_max_nodes" - } -} - -# tflint-ignore: terraform_unused_declarations -variable "service_account" { - description = "DEPRECATED: use service_account_email and scopes." - type = object({ - email = string, - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "service_account is deprecated and replaced with service_account_email and scopes." - } -} - -variable "additional_networks" { - description = "Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool" - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - })) - nullable = false -} - -variable "reservation_affinity" { - description = <<-EOT - Reservation resource to consume. When targeting SPECIFIC_RESERVATION, specific_reservations needs be specified. - Even though specific_reservations is a list, only one reservation is allowed by the NodePool API. - It is assumed that the specified reservation exists and has available capacity. - For a shared reservation, specify the project_id as well in which it was created. - To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared - EOT - type = object({ - consume_reservation_type = string - specific_reservations = optional(list(object({ - name = string - project = optional(string) - }))) - }) - default = { - consume_reservation_type = "NO_RESERVATION" - specific_reservations = [] - } - validation { - condition = contains(["NO_RESERVATION", "ANY_RESERVATION", "SPECIFIC_RESERVATION"], var.reservation_affinity.consume_reservation_type) - error_message = "Accepted values are: {NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION}" - } -} - -variable "host_maintenance_interval" { - description = "Specifies the frequency of planned maintenance events." - type = string - default = "" - nullable = false - validation { - condition = contains(["", "PERIODIC", "AS_NEEDED"], var.host_maintenance_interval) - error_message = "Invalid host_maintenance_interval value. Must be PERIODIC, AS_NEEDED or the empty string" - } -} - -variable "initial_node_count" { - description = "The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static_node_count and must be set to a value between autoscaling_total_min_nodes and autoscaling_total_max_nodes." - type = number - default = null -} - -variable "gke_version" { - description = "GKE version" - type = string -} - -variable "max_pods_per_node" { - description = "The maximum number of pods per node in this node pool. This will force replacement." - type = number - default = null -} - -variable "upgrade_settings" { - description = <<-EOT - Defines node pool upgrade settings. It is highly recommended that you define all max_surge and max_unavailable. - If max_surge is not specified, it would be set to a default value of 0. - If max_unavailable is not specified, it would be set to a default value of 1. - EOT - type = object({ - strategy = string - max_surge = optional(number) - max_unavailable = optional(number) - }) - default = { - strategy = "SURGE" - max_surge = 0 - max_unavailable = 1 - } -} - -variable "run_workload_script" { - description = "Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only." - type = bool - default = true -} - -variable "enable_queued_provisioning" { - description = "If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool." - type = bool - default = false -} - -variable "enable_flex_start" { - description = <<-EOT - If true, start the node pool with Flex Start provisioning model. - To learn more about flex-start mode, please refer to - https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and - https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest - EOT - type = bool - default = false -} - -variable "max_run_duration" { - description = "The duration (in whole seconds) of the instance. Instance will run and be terminated after then." - type = number - default = null -} - -variable "enable_private_nodes" { - description = "Whether nodes have internal IP addresses only." - type = bool - default = true -} - -variable "num_node_pools" { - description = "Number of node pools to create. This is same as num_slices." - type = number - default = 1 -} - -variable "num_slices" { - description = "Number of TPUs slices to create. This is same as num_node_pools." - type = number - default = 1 -} - -variable "enable_numa_aware_scheduling" { - description = "Enable [NUMA-aware](https://cloud.google.com/kubernetes-engine/distributed-cloud/bare-metal/docs/vm-runtime/numa) scheduling." - type = bool - default = false -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf b/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf deleted file mode 100644 index f018d04fc5..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.5" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 7.2" - } - google-beta = { - source = "hashicorp/google-beta" - version = ">= 7.2" - } - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.74.0" - } -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/README.md b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/README.md deleted file mode 100644 index 3b769e8761..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/README.md +++ /dev/null @@ -1,82 +0,0 @@ -## Description - -This modules create a [resource policy for compute engines](https://cloud.google.com/compute/docs/instances/placement-policies-overview). This policy can be passed to a gke-node-pool module to apply the policy on the node-pool's nodes. - -Note: By default, you can't apply compact placement policies with a max distance value to A3 VMs. To request access to this feature, contact your [Technical Account Manager (TAM)](https://cloud.google.com/tam) or the [Sales team](https://cloud.google.com/contact). - -### Example - -The following example creates a group placement resource policy and applies it to a gke-node-pool. - -```yaml - - id: group_placement_1 - source: modules/compute/resource-policy - settings: - name: gp-np-1 - group_placement_max_distance: 2 - - - id: node_pool_1 - source: modules/compute/gke-node-pool - use: [group_placement_1] - settings: - machine_type: e2-standard-8 - outputs: [instructions] -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google-beta](#requirement\_google-beta) | >= 6.29.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google-beta](#provider\_google-beta) | >= 6.29.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_resource_policy.policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [group\_placement\_max\_distance](#input\_group\_placement\_max\_distance) | The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | `number` | `0` | no | -| [name](#input\_name) | The resource policy's name. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The project ID for the resource policy. | `string` | n/a | yes | -| [region](#input\_region) | The region for the the resource policy. | `string` | n/a | yes | -| [workload\_policy](#input\_workload\_policy) | Describes the workload policy | 
object({
    type                  = optional(string, null)
    max_topology_distance = optional(string, null)
    accelerator_topology  = optional(string, null)
  })
| 
{
  "accelerator_topology": null,
  "max_topology_distance": null,
  "type": null
}
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [placement\_policy](#output\_placement\_policy) | Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions.
The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. | - diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/main.tf b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/main.tf deleted file mode 100644 index 906424ca7c..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/main.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - name = "${var.name}-${random_id.resource_name_suffix.hex}" -} - -resource "google_compute_resource_policy" "policy" { - name = local.name - region = var.region - project = var.project_id - provider = google-beta - - dynamic "workload_policy" { - for_each = var.workload_policy.type != null ? [1] : [] - - content { - type = var.workload_policy.type - max_topology_distance = var.workload_policy.max_topology_distance - accelerator_topology = var.workload_policy.accelerator_topology - } - } - - dynamic "group_placement_policy" { - for_each = var.group_placement_max_distance > 0 ? [1] : [] - - content { - collocation = "COLLOCATED" - max_distance = var.group_placement_max_distance - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/outputs.tf b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/outputs.tf deleted file mode 100644 index c1dc65bcbb..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "placement_policy" { - description = <<-EOT - Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. - It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - The value `tpu_topology` is only used for TPU node pools. The `gke-node-pool` module ensures it is configured appropriately for only TPUs during placement policy mapping. - EOT - - value = { - type = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? "COMPACT" : null - name = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? local.name : null - tpu_topology = (var.workload_policy.type != null) ? var.workload_policy.accelerator_topology : null - } -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/variables.tf b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/variables.tf deleted file mode 100644 index 92434326ca..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/variables.tf +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "The project ID for the resource policy." - type = string -} - -variable "region" { - description = "The region for the the resource policy." - type = string -} - -variable "name" { - description = "The resource policy's name." - type = string - - validation { - # Check if the variable matches the GCP resource naming regex. - condition = can(regex("^[a-z]([-a-z0-9]{0,52}[a-z0-9])?$", var.name)) - error_message = <<-EOD - The resource policy name must be between 1 and 54 characters, start with a lowercase letter, end with an alphanumeric, and contain only lowercase letters, numbers, and hyphens. - Underscores are not allowed. A shorter length is enforced to accommodate a random suffix. - EOD - } -} - -variable "group_placement_max_distance" { - description = <<-EOT - The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - EOT - - type = number - default = 0 -} - -variable "workload_policy" { - description = "Describes the workload policy" - type = object({ - type = optional(string, null) - max_topology_distance = optional(string, null) - accelerator_topology = optional(string, null) - }) - default = { - type = null - max_topology_distance = null - accelerator_topology = null - } - nullable = false -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/versions.tf b/deletion-test/primary/modules/embedded/modules/compute/resource-policy/versions.tf deleted file mode 100644 index f235fbade3..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/resource-policy/versions.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.29.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:resource-policy/v1.37.2" - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/README.md b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/README.md deleted file mode 100644 index 0c4737e0d9..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/README.md +++ /dev/null @@ -1,257 +0,0 @@ -## Description - -This module creates one or more -[compute VM instances](https://cloud.google.com/compute/docs/instances). - -### Example - -```yaml -- id: compute - source: modules/compute/vm-instance - use: [network1] - settings: - instance_count: 8 - name_prefix: compute - machine_type: c2-standard-60 -``` - -This creates a cluster of 8 compute VMs that are: - -* named `compute-[0-7]` -* on the network defined by the `network1` module -* of type c2-standard-60 - -> **_NOTE:_** Simultaneous Multithreading (SMT) is deactivated by default -> (threads_per_core=1), which means only the physical cores are visible on the -> VM. With SMT disabled, a machine of type c2-standard-60 will only have the 30 -> physical cores visible. To change this, set `threads_per_core=2` under -> settings. - -### VPC Networks - -There are two methods for adding network connectivity to the `vm-instance` -module. The first is shown in the example above, where a `vpc` module or -`pre-existing-vpc` module is used by the `vm-instance` module. When this -happens, the `network_self_link` and `subnetwork_self_link` outputs from the -network are provided as input to the `vm-instance` and a network interface is -defined based on that. This can also be done updating the `network_self_link` and -`subnetwork_self_link` settings directly. - -The alternative option can be used when more than one network needs to be added -to the `vm-instance` or further customization is needed beyond what is provided -via other variables. For this option, the `network_interfaces` variable can be -used to set up one or more network interfaces on the VM instance. The format is -consistent with the terraform `google_compute_instance` `network_interface` -block, and more information can be found in the -[terraform docs][network-interface-tf]. - -> **_NOTE:_** When supplying the `network_interfaces` variable, networks -> associated with the `vm-instance` via use will be ignored in favor of the -> networks added in `network_interfaces`. In addition, `bandwidth_tier` and -> `disable_public_ips` will not apply to networks defined in -> `network_interfaces`. - -[network-interface-tf]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - -### SSH key metadata - -This module will ignore all changes to the `ssh-keys` metadata field that are -typically set by [external Google Cloud tools that automate SSH access][gcpssh] -when not using OS Login. For example, clicking on the Google Cloud Console SSH -button next to VMs in the VM Instances list will temporarily modify VM metadata -to include a dynamically-generated SSH public key. - -[gcpssh]: https://cloud.google.com/compute/docs/connect/add-ssh-keys#metadata - -### Placement - -The `placement_policy` variable can be used to control where your VM instances -are physically located relative to each other within a zone. See the official -placement [guide][guide-link] and [api][api-link] documentation. - -[guide-link]: https://cloud.google.com/compute/docs/instances/define-instance-placement -[api-link]: https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement - -Use the following settings for compact placement: - -```yaml - ... - settings: - instance_count: 4 - machine_type: c2-standard-60 - placement_policy: - collocation: "COLLOCATED" -``` - -By default the above placement policy will always result in the most compact set -of VMs available. If you would like that provisioning failed if some level of -compactness is not obtainable, you can enforce this with the [`max_distance` -setting](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies): - -```yaml - ... - settings: - instance_count: 4 - machine_type: c2-standard-60 - placement_policy: - collocation: "COLLOCATED" - max_distance: 1 -``` - -Use the following settings for spread placement: - -```yaml - ... - settings: - instance_count: 4 - machine_type: n2-standard-4 - placement_policy: - availability_domain_count: 2 -``` - -When `vm_count` is not set, as shown in the examples above, then the VMs will be -added to the placement policy incrementally. This is the **recommended way** to -use placement policies. - -If `vm_count` is specified then VMs will stay in pending state until the -specified number of VMs are created. See the warning below if using this field. - -> [!WARNING] -> When creating a compact placement using `vm_count` with more than 10 VMs, you -> must add `-parallelism=` argument on apply. For example if you have 15 VMs -> in a placement group: `terraform apply -parallelism=15`. This is because -> terraform self limits to 10 parallel requests by default but the create -> instance requests will not succeed until all VMs in the placement group have -> been requested, forming a deadlock. - -### GPU Support - -More information on GPU support in `vm-instance` and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../docs/gpu-support.md) - -## Lifecycle - -The `vm-instance` module will be replaced when the `instance_image` variable is -changed and `terraform apply` is run on the deployment group folder or -`gcluster deploy` is run. However, it will not be automatically replaced if a new -image is created in a family. - -To selectively replace the vm-instance(s), consider running terraform -`apply -replace` such as: - -> See https://developer.hashicorp.com/terraform/cli/commands/plan#replace-address for precise syntax terraform apply -replace=ADDRESS - -```shell -terraform state list -# search for the module ID and resource -terraform apply -replace="address" -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 4.73.0 | -| [google-beta](#requirement\_google-beta) | >= 6.13.0 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.73.0 | -| [google-beta](#provider\_google-beta) | >= 6.13.0 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | -| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance) | resource | -| [google-beta_google_compute_resource_policy.placement_policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | -| [google_compute_address.compute_ip](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_disk.additional_disks](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.replace_vm_trigger_from_placement](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | -| [additional\_persistent\_disks](#input\_additional\_persistent\_disks) | Configurations of additional disks to be included on the partition nodes. | 
object({
    count = optional(number, 0)
    type  = optional(string, "pd-balanced")
    size  = optional(number, 200)
  })
| `{}` | no | -| [allocate\_ip](#input\_allocate\_ip) | If not null, allocate IPs with the given configuration. See details at
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address | 
object({
    address_type = optional(string, "INTERNAL")
    purpose      = optional(string),
    network_tier = optional(string),
    ip_version   = optional(string, "IPV4"),
  })
| `null` | no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | -| [automatic\_restart](#input\_automatic\_restart) | Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user). | `bool` | `null` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will optionally be used name resources according to `name_prefix` | `string` | n/a | yes | -| [disable\_public\_ips](#input\_disable\_public\_ips) | If set to true, instances will not have public IPs | `bool` | `false` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | -| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-standard"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | -| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_count](#input\_local\_ssd\_count) | The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd. | `number` | `0` | no | -| [local\_ssd\_interface](#input\_local\_ssd\_interface) | Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set. | `string` | `"NVME"` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface

**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply
to network interfaces defined in this variable.

Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "MRDMA", "IRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. Can use "default" for the default network. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `null` | no | -| [placement\_policy](#input\_placement\_policy) | Control where your VM instances are physically located relative to each other within a zone.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy#nested_group_placement_policy | `any` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [provisioning\_model](#input\_provisioning\_model) | Provisioning model for cloud instance. | `string` | `null` | no | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME
- RESERVATION\_NAME

Must be a "SPECIFIC\_RESERVATION"
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | -| [service\_account](#input\_service\_account) | DEPRECATED - Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [spot](#input\_spot) | DEPRECATED - Use `provisioning_model` instead. | `bool` | `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | -| [zone](#input\_zone) | Compute Platform zone | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [external\_ip](#output\_external\_ip) | External IP of the instances (if enabled) | -| [instructions](#output\_instructions) | Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions. | -| [internal\_ip](#output\_internal\_ip) | Internal IP of the instances | -| [name](#output\_name) | Names of instances created | -| [self\_link](#output\_self\_link) | The tuple URIs of the created instances | - diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/main.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/main.tf deleted file mode 100644 index 0a8c7d354e..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/main.tf +++ /dev/null @@ -1,334 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "vm-instance", ghpc_role = "compute" }) -} - -module "gpu" { - source = "../../internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - native_fstype = [] - startup_script = local.startup_from_network_storage != null ? ( - { startup-script = local.startup_from_network_storage }) : {} - network_storage = var.network_storage != null ? ( - { network_storage = jsonencode(var.network_storage) }) : {} - - prefix_optional_deployment_name = var.name_prefix != null ? var.name_prefix : var.deployment_name - prefix_always_deployment_name = var.name_prefix != null ? "${var.deployment_name}-${var.name_prefix}" : var.deployment_name - resource_prefix = var.add_deployment_name_before_prefix ? local.prefix_always_deployment_name : local.prefix_optional_deployment_name - - enable_gvnic = var.bandwidth_tier != "not_enabled" - enable_tier_1 = var.bandwidth_tier == "tier_1_enabled" - - provisioning_model = var.provisioning_model - - spot = var.provisioning_model == "SPOT" - - # compact_placement : true when placement policy is provided and collocation set; false if unset - compact_placement = try(var.placement_policy.collocation, null) != null - - gpu_attached = contains(["a2", "g2"], local.machine_family) || length(local.guest_accelerator) > 0 - - # both of these must be false if either compact placement or preemptible/spot instances are used - # automatic restart is tolerant of GPUs while on host maintenance is not - automatic_restart_default = local.compact_placement || local.spot ? false : null - on_host_maintenance_default = local.compact_placement || local.spot || local.gpu_attached ? "TERMINATE" : "MIGRATE" - - automatic_restart = ( - var.automatic_restart != null - ? var.automatic_restart - : local.automatic_restart_default - ) - - on_host_maintenance = ( - var.on_host_maintenance != null - ? var.on_host_maintenance - : local.on_host_maintenance_default - ) - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - # Network Interfaces - # Support for `use` input and base network parameters like `network_self_link` and `subnetwork_self_link` - empty_access_config = { - nat_ip = null, - public_ptr_domain_name = null, - network_tier = null - } - default_network_interface = { - network = var.network_self_link - subnetwork = var.subnetwork_self_link - subnetwork_project = null # will populate from subnetwork_self_link - network_ip = null - nic_type = local.enable_gvnic ? "GVNIC" : null - stack_type = null - queue_count = null - access_config = var.disable_public_ips ? [] : [local.empty_access_config] - ipv6_access_config = [] - alias_ip_range = [] - } - network_interfaces = coalescelist(var.network_interfaces, [local.default_network_interface]) - network_interfaces_with_ips = var.allocate_ip == null ? local.network_interfaces : [ - for i, interface in local.network_interfaces : - merge(interface, { - network_ip = google_compute_address.compute_ip[i].address - }) - ] -} - -resource "null_resource" "image" { - triggers = { - name = try(var.instance_image.name, null), - family = try(var.instance_image.family, null), - project = try(var.instance_image.project, null) - } -} - -resource "google_compute_disk" "additional_disks" { - project = var.project_id - - count = var.instance_count * var.additional_persistent_disks.count - - # NB: this resource array must be sliced accounting for var.instance_count - name = "${local.resource_prefix}-disk-${count.index}" - type = var.additional_persistent_disks.type - size = var.additional_persistent_disks.size - labels = local.labels - zone = var.zone -} - -resource "google_compute_resource_policy" "placement_policy" { - project = var.project_id - provider = google-beta - - count = var.placement_policy != null ? 1 : 0 - name = "${local.resource_prefix}-vm-instance-placement" - group_placement_policy { - vm_count = try(var.placement_policy.vm_count, null) - availability_domain_count = try(var.placement_policy.availability_domain_count, null) - collocation = try(var.placement_policy.collocation, null) - max_distance = try(var.placement_policy.max_distance, null) - } -} - -resource "null_resource" "replace_vm_trigger_from_placement" { - triggers = { - vm_count = try(tostring(var.placement_policy.vm_count), "") - availability_domain_count = try(tostring(var.placement_policy.availability_domain_count), "") - max_distance = try(tostring(var.placement_policy.max_distance), "") - collocation = try(var.placement_policy.collocation, "") - } -} - -resource "google_compute_address" "compute_ip" { - project = var.project_id - - count = var.allocate_ip != null ? length(local.network_interfaces) : 0 - - name = "${local.resource_prefix}-${count.index}" - - address = local.network_interfaces[count.index].network_ip - region = var.region - network = can(coalesce(local.network_interfaces[count.index].subnetwork)) ? null : local.network_interfaces[count.index].network - subnetwork = local.network_interfaces[count.index].subnetwork - address_type = var.allocate_ip.address_type - purpose = var.allocate_ip.purpose - network_tier = var.allocate_ip.network_tier - ip_version = var.allocate_ip.ip_version -} - -resource "google_compute_instance" "compute_vm" { - project = var.project_id - provider = google-beta - - count = var.instance_count - - depends_on = [var.network_self_link, var.network_storage] - - name = "${local.resource_prefix}-${count.index}" - min_cpu_platform = var.min_cpu_platform - machine_type = var.machine_type - zone = var.zone - - resource_policies = google_compute_resource_policy.placement_policy[*].self_link - - tags = var.tags - labels = local.labels - - boot_disk { - initialize_params { - image = data.google_compute_image.compute_image.self_link - size = var.disk_size_gb - type = var.disk_type - labels = local.labels - } - - device_name = "${local.resource_prefix}-boot-disk-${count.index}" - auto_delete = var.auto_delete_boot_disk - } - - dynamic "attached_disk" { - for_each = slice( - google_compute_disk.additional_disks, - var.additional_persistent_disks.count * count.index, - var.additional_persistent_disks.count * count.index + var.additional_persistent_disks.count, - ) - - content { - source = attached_disk.value.self_link - device_name = "additional-disk-${attached_disk.key}" - mode = "READ_WRITE" - } - } - - dynamic "scratch_disk" { - for_each = range(var.local_ssd_count) - content { - interface = var.local_ssd_interface - } - } - - dynamic "network_interface" { - for_each = local.network_interfaces_with_ips - - content { - network = network_interface.value.network - subnetwork = network_interface.value.subnetwork - subnetwork_project = network_interface.value.subnetwork_project - network_ip = network_interface.value.network_ip - nic_type = network_interface.value.nic_type - stack_type = network_interface.value.stack_type - queue_count = network_interface.value.queue_count - dynamic "access_config" { - for_each = network_interface.value.access_config - content { - nat_ip = access_config.value.nat_ip - public_ptr_domain_name = access_config.value.public_ptr_domain_name - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = network_interface.value.ipv6_access_config - content { - public_ptr_domain_name = ipv6_access_config.value.public_ptr_domain_name - network_tier = ipv6_access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = network_interface.value.alias_ip_range - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - } - } - - network_performance_config { - total_egress_bandwidth_tier = local.enable_tier_1 ? "TIER_1" : "DEFAULT" - } - - service_account { - email = var.service_account_email - scopes = var.service_account_scopes - } - - dynamic "guest_accelerator" { - for_each = local.guest_accelerator - content { - count = guest_accelerator.value.count - type = guest_accelerator.value.type - } - } - - scheduling { - on_host_maintenance = local.on_host_maintenance - automatic_restart = local.automatic_restart - preemptible = local.spot - provisioning_model = local.provisioning_model - } - - dynamic "advanced_machine_features" { - for_each = local.set_threads_per_core ? [1] : [] - content { - threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf - } - } - - dynamic "reservation_affinity" { - for_each = var.reservation_name == "" ? [] : [1] - content { - type = "SPECIFIC_RESERVATION" - specific_reservation { - key = "compute.googleapis.com/reservation-name" - values = [var.reservation_name] - } - } - } - - metadata = merge( - local.network_storage, - local.startup_script, - local.enable_oslogin, - local.disable_automatic_updates_metadata, - var.metadata - ) - - lifecycle { - ignore_changes = [ - metadata["ssh-keys"], - ] - - replace_triggered_by = [ - null_resource.replace_vm_trigger_from_placement - ] - - precondition { - condition = (length(var.network_interfaces) == 0) != (var.network_self_link == null && var.subnetwork_self_link == null) - error_message = "Exactly one of network_interfaces or network_self_link/subnetwork_self_link must be specified." - } - precondition { - condition = alltrue([for interface in var.network_interfaces : interface.network_ip == null]) || var.instance_count == 1 - error_message = <<-EOT - The network_ip cannot be statically set on vm-instance when the VM instance_count is greater than 1. - Either set the network_ip to null to allow it to be set dynamically for all instances, or create modules for each VM instance with its own network interface. - EOT - } - precondition { - condition = !contains([ - "c3-:pd-standard", - "h3-:pd-standard", - "h3-:pd-ssd", - ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") - error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/outputs.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/outputs.tf deleted file mode 100644 index eab8cb56bd..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "name" { - description = "Names of instances created" - value = google_compute_instance.compute_vm[*].name -} - -output "self_link" { - description = "The tuple URIs of the created instances" - value = google_compute_instance.compute_vm[*].self_link -} - -output "external_ip" { - description = "External IP of the instances (if enabled)" - value = try(google_compute_instance.compute_vm[*].network_interface[0].access_config[0].nat_ip, []) -} - -output "internal_ip" { - description = "Internal IP of the instances" - value = google_compute_instance.compute_vm[*].network_interface[0].network_ip -} - -locals { - first_instance_link = try(google_compute_instance.compute_vm[0].self_link, "no-instance") - ssh_instructions = <<-EOT - Use the following commands to SSH into the first VM created: - gcloud compute ssh ${local.first_instance_link} --project ${var.project_id} - If not accessible from the public internet, use an SSH tunnel through IAP: - gcloud compute ssh ${local.first_instance_link} --tunnel-through-iap --project ${var.project_id} - EOT -} - -output "instructions" { - description = "Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions." - value = var.instance_count > 0 ? local.ssh_instructions : "No instances were created." -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf deleted file mode 100644 index 02bc58e4f7..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "inputs": -# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. -# var.network_storage : to be passed into metadata somewhere else (not here) -# var.startup_script : to be changed into a more complete file system with all the fs runners - -# "outputs": -# local.startup_from_network_storage : A full startup script with all the runners that are not supported -# natively and were included in the network_storage structure - -locals { - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - # Pull out runners to include in startup script - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - startup_script_runner = [{ - content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" - destination = "passed_startup_script.sh" - type = "shell" - }] - - full_runner_list = concat( - local.storage_client_install_runners, - local.mount_runners, - local.startup_script_runner - ) - - startup_from_network_storage = module.netstorage_startup_script.startup_script -} - -module "netstorage_startup_script" { - source = "../../scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.full_runner_list -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf deleted file mode 100644 index e582db33da..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' - -# "inputs": -# var.machine_type: Machine type for the instance being evaluated. -# var.threads_per_core : Sets the number of threads per physical core, where 0 -# has behavior described in description. - -# "outputs": -# local.set_threads_per_core: bool that tells if threads per core should be set, -# to be used with a dynamic block. -# local.threads_per_core: actual threads_per_core to be used. - -locals { - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - machine_shared_core = length(local.machine_vals) <= 2 - machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) - - smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) - smt_capable_vcpu = local.machine_vcpus >= 2 - - smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core - set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) - threads_per_core = var.threads_per_core == 2 ? 2 : 1 -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/variables.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/variables.tf deleted file mode 100644 index 5519b8cd40..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/variables.tf +++ /dev/null @@ -1,452 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "instance_count" { - description = "Number of instances" - type = number - default = 1 -} - -variable "instance_image" { - description = "Instance Image" - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "disk_size_gb" { - description = "Size of disk for instances." - type = number - default = 200 -} - -variable "disk_type" { - description = "Disk type for instances." - type = string - default = "pd-standard" -} - -variable "auto_delete_boot_disk" { - description = "Controls if boot disk should be auto-deleted when instance is deleted." - type = bool - default = true -} - -variable "local_ssd_count" { - description = "The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd." - type = number - default = 0 -} - -variable "local_ssd_interface" { - description = "Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set." - type = string - default = "NVME" -} - -variable "additional_persistent_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = object({ - count = optional(number, 0) - type = optional(string, "pd-balanced") - size = optional(number, 200) - }) - default = {} -} - -variable "name_prefix" { - description = <<-EOT - An optional name for all VM and disk resources. - If not supplied, `deployment_name` will be used. - When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, - then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". - EOT - type = string - default = null -} - -variable "add_deployment_name_before_prefix" { - description = <<-EOT - If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. - See `name_prefix` for further details on resource naming behavior. - EOT - type = bool - default = false -} - -variable "disable_public_ips" { - description = "If set to true, instances will not have public IPs" - type = bool - default = false -} - -variable "machine_type" { - description = "Machine type to use for the instance creation" - type = string - default = "c2-standard-60" -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "deployment_name" { - description = "Name of the deployment, will optionally be used name resources according to `name_prefix`" - type = string -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} - -variable "service_account_email" { - description = "Service account e-mail address to use with the node pool" - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to to use with the node pool." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -# tflint-ignore: terraform_unused_declarations -variable "service_account" { - description = "DEPRECATED - Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string, - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "The 'service_account' setting is deprecated, please use 'var.service_account_email' and 'var.service_account_scopes' instead." - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the VM. Can use \"default\" for the default network." - type = string - default = null -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the VM." - type = string - default = null -} - -variable "network_interfaces" { - description = <<-EOT - A list of network interfaces. The options match that of the terraform - network_interface block of google_compute_instance. For descriptions of the - subfields or more information see the documentation: - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - - **_NOTE:_** If `network_interfaces` are set, `network_self_link` and - `subnetwork_self_link` will be ignored, even if they are provided through - the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply - to network interfaces defined in this variable. - - Subfields: - network (string, required if subnetwork is not supplied) - subnetwork (string, required if network is not supplied) - subnetwork_project (string, optional) - network_ip (string, optional) - nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "MRDMA", "IRDMA"]) - stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) - queue_count (number, optional) - access_config (object, optional) - ipv6_access_config (object, optional) - alias_ip_range (list(object), optional) - EOT - type = list(object({ - network = string, - subnetwork = string, - subnetwork_project = string, - network_ip = string, - nic_type = string, - stack_type = string, - queue_count = number, - access_config = list(object({ - nat_ip = string, - public_ptr_domain_name = string, - network_tier = string - })), - ipv6_access_config = list(object({ - public_ptr_domain_name = string, - network_tier = string - })), - alias_ip_range = list(object({ - ip_cidr_range = string, - subnetwork_range_name = string - })) - })) - default = [] - validation { - condition = alltrue([ - for ni in var.network_interfaces : (ni.network == null) != (ni.subnetwork == null) - ]) - error_message = "All additional network interfaces must define exactly one of \"network\" or \"subnetwork\"." - } - validation { - condition = alltrue([ - for ni in var.network_interfaces : ni.nic_type == "GVNIC" || ni.nic_type == "VIRTIO_NET" || ni.nic_type == "MRDMA" || ni.nic_type == "IRDMA" || ni.nic_type == null - ]) - error_message = "In the variable network_interfaces, field \"nic_type\" must be \"GVNIC\", \"VIRTIO_NET\", \"MRDMA\", \"IRDMA\", or null." - } - validation { - condition = alltrue([ - for ni in var.network_interfaces : ni.stack_type == "IPV4_ONLY" || ni.stack_type == "IPV4_IPV6" || ni.stack_type == null - ]) - error_message = "In the variable network_interfaces, field \"stack_type\" must be either \"IPV4_ONLY\", \"IPV4_IPV6\" or null." - } -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "zone" { - description = "Compute Platform zone" - type = string -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "startup_script" { - description = "Startup script used on the instance" - type = string - default = null -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false -} - -variable "automatic_restart" { - description = "Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user)." - type = bool - default = null -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" - type = string - default = null - validation { - condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) - error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." - } -} - -variable "bandwidth_tier" { - description = <= 0, false) && try(var.threads_per_core <= 2, false) - error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." - } - -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "allocate_ip" { - description = <<-EOT - If not null, allocate IPs with the given configuration. See details at - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address - EOT - type = object({ - address_type = optional(string, "INTERNAL") - purpose = optional(string), - network_tier = optional(string), - ip_version = optional(string, "IPV4"), - }) - default = null -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "reservation_name" { - description = <<-EOD - Name of the reservation to use for VM resources, should be in one of the following formats: - - projects/PROJECT_ID/reservations/RESERVATION_NAME - - RESERVATION_NAME - - Must be a "SPECIFIC_RESERVATION" - Set to empty string if using no reservation or automatically-consumed reservations - EOD - type = string - default = "" - nullable = false - - validation { - condition = length(regexall("^((projects/([a-z0-9-]+)/reservations/)?([a-z0-9-]+))?$", var.reservation_name)) > 0 - error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME', [...] is an optional part." - } -} diff --git a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/versions.tf b/deletion-test/primary/modules/embedded/modules/compute/vm-instance/versions.tf deleted file mode 100644 index 0429782c6d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/compute/vm-instance/versions.tf +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.73.0" - } - - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.13.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/README.md b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/README.md deleted file mode 100644 index 285a20bde2..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/README.md +++ /dev/null @@ -1,170 +0,0 @@ -## Description - -This module creates a [Google Cloud Storage (GCS) bucket](https://cloud.google.com/storage). - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -### Example - -The following example will create a bucket named `simulation-results-xxxxxxxx`, -where `xxxxxxxx` is a randomly generated id. - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: - name_prefix: simulation-results - random_suffix: true -``` - -> **_NOTE:_** Use of `random_suffix` may cause the following error when used -> with other modules: -> `value depends on resource attributes that cannot be determined until apply`. -> To resolve this set `random_suffix` to `false` (default). - - - -> **_NOTE:_** Bucket namespace is shared by all users of Google Cloud so it is -> possible to have a bucket name clash with an existing bucket that is not in -> your project. To resolve this try to use a more unique name, or set the -> `random_suffix` variable to `true`. - -## Naming of Bucket - -There are potentially three parts to the bucket name. Each of these parts are -configurable in the blueprint. - -1. A **custom prefix**, provided by the user in the blueprint \ -Provide the custom prefix using the `name_prefix` setting. - -1. The **deployment name**, included by default \ -The deployment name can be excluded by setting `use_deployment_name_in_bucket_name: false`. - -1. A **random id** suffix, excluded by default \ -The random id can be included by setting `random_suffix: true`. - -If none of these are provided (no `name_prefix`, -`use_deployment_name_in_bucket_name: false`, & `random_suffix: false`), then the -bucket name will default to `no-bucket-name-provided`. - -Since bucket namespace is shared by all users of Google Cloud, it is more likely -to experience naming clashes than with other resources. In many cases, adding -the `random_suffix` will resolve the naming clash issue. - -> **Warning**: If a bucket is created with a `random_suffix` and then used as -> the bucket for a startup script in the same deployment group this will cause a -> `not known at apply time` error in terraform. The solution is to either create -> the bucket in a separate deployment group or to remove the random suffix. - -## Mounting - -To mount the Cloud Storage bucket you must first ensure that the GCS Fuse client -has been installed and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. - -If mounting is not automatically handled as described above, the -`cloud-storage-bucket` module outputs runners that can be used with the -`startup-script` module to install the client and mount the file system. See the -following example: - -```yaml - - id: bucket - source: modules/file-system/cloud-storage-bucket - settings: {local_mount: /data} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(bucket.client_install_runner) - - $(bucket.mount_runner) -``` - -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | -| [google-beta](#requirement\_google-beta) | >= 6.9.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [google-beta](#provider\_google-beta) | >= 6.9.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource | -| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [autoclass](#input\_autoclass) | Configure bucket autoclass setup

The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern.

The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time.
Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE')

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/autoclass | 
object({
    enabled                = optional(bool, false)
    terminal_storage_class = optional(string, null)
  })
| 
{
  "enabled": false
}
| no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the GCS bucket. | `string` | n/a | yes | -| [enable\_hierarchical\_namespace](#input\_enable\_hierarchical\_namespace) | If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket. | `bool` | `false` | no | -| [enable\_object\_retention](#input\_enable\_object\_retention) | If true, enables retention policy at per object level for the bucket.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/object-lock | `bool` | `false` | no | -| [enable\_versioning](#input\_enable\_versioning) | If true, enables versioning for the bucket. | `bool` | `false` | no | -| [force\_destroy](#input\_force\_destroy) | If true will destroy bucket with all objects stored within. | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to the GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | -| [lifecycle\_rules](#input\_lifecycle\_rules) | List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule | 
list(object({
    # Object with keys:
    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.
    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.
    action = object({
      type          = string
      storage_class = optional(string)
    })

    # Object with keys:
    # - age - (Optional) Minimum age of an object in days to satisfy this condition.
    # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field.
    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.
    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".
    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY.
    # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition.
    # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition.
    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.
    # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition.
    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.
    # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object.
    # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent.
    condition = object({
      age                        = optional(number)
      send_age_if_zero           = optional(bool)
      created_before             = optional(string)
      with_state                 = optional(string)
      matches_storage_class      = optional(string)
      matches_prefix             = optional(string)
      matches_suffix             = optional(string)
      num_newer_versions         = optional(number)
      custom_time_before         = optional(string)
      days_since_custom_time     = optional(number)
      days_since_noncurrent_time = optional(number)
      noncurrent_time_before     = optional(string)
    })
  }))
| `[]` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | -| [mount\_options](#input\_mount\_options) | Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories) | `string` | `"defaults,_netdev,implicit_dirs"` | no | -| [name\_prefix](#input\_name\_prefix) | Name Prefix. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | -| [public\_access\_prevention](#input\_public\_access\_prevention) | Bucket public access can be controlled by setting a value of either `inherited` or `enforced`.
When set to `enforced`, public access to the bucket is blocked.
If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/public-access-prevention | `string` | `null` | no | -| [random\_suffix](#input\_random\_suffix) | If true, a random id will be appended to the suffix of the bucket name. | `bool` | `false` | no | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [retention\_policy\_period](#input\_retention\_policy\_period) | If defined, this will configure retention\_policy with retention\_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/bucket-lock | `number` | `null` | no | -| [soft\_delete\_retention\_duration](#input\_soft\_delete\_retention\_duration) | If defined, this will configure soft\_delete\_policy with retention\_duration\_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days).
Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/soft-delete | `number` | `null` | no | -| [storage\_class](#input\_storage\_class) | The storage class of the GCS bucket. | `string` | `"REGIONAL"` | no | -| [uniform\_bucket\_level\_access](#input\_uniform\_bucket\_level\_access) | Allow uniform control access to the bucket. | `bool` | `true` | no | -| [use\_deployment\_name\_in\_bucket\_name](#input\_use\_deployment\_name\_in\_bucket\_name) | If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments. | `bool` | `true` | no | -| [viewers](#input\_viewers) | A list of additional accounts that can read packages from this bucket | `set(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use gcs fuse. | -| [gcs\_bucket\_name](#output\_gcs\_bucket\_name) | Bucket name. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | The gsutil bucket path with format of `gs://`. | -| [mount\_runner](#output\_mount\_runner) | Runner that mounts the cloud storage bucket with gcs fuse. | -| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | - diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf deleted file mode 100644 index 81ba0ca6a9..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/main.tf +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "cloud-storage-bucket", ghpc_role = "file-system" }) -} - -locals { - prefix = var.name_prefix != null ? var.name_prefix : "" - deployment = var.use_deployment_name_in_bucket_name ? var.deployment_name : "" - suffix = var.random_suffix ? random_id.resource_name_suffix.hex : "" - first_dash = (local.prefix != "" && (local.deployment != "" || local.suffix != "")) ? "-" : "" - second_dash = local.deployment != "" && local.suffix != "" ? "-" : "" - composite_name = "${local.prefix}${local.first_dash}${local.deployment}${local.second_dash}${local.suffix}" - name = local.composite_name == "" ? "no-bucket-name-provided" : local.composite_name -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket" "bucket" { - provider = google-beta - project = var.project_id - name = local.name - uniform_bucket_level_access = var.uniform_bucket_level_access - location = var.region - storage_class = var.storage_class - labels = local.labels - force_destroy = var.force_destroy - public_access_prevention = var.public_access_prevention - enable_object_retention = var.enable_object_retention - hierarchical_namespace { - enabled = var.enable_hierarchical_namespace - } - - dynamic "autoclass" { - for_each = var.autoclass.enabled ? [1] : [] - content { - enabled = var.autoclass.enabled - terminal_storage_class = var.autoclass.terminal_storage_class - } - } - - dynamic "soft_delete_policy" { - for_each = var.soft_delete_retention_duration == null ? [] : [1] - content { - retention_duration_seconds = var.soft_delete_retention_duration - } - } - - dynamic "retention_policy" { - for_each = var.retention_policy_period == null ? [] : [1] - content { - retention_period = var.retention_policy_period - } - } - - dynamic "versioning" { - for_each = var.enable_versioning ? [1] : [] - content { - enabled = var.enable_versioning - } - } - - dynamic "lifecycle_rule" { - for_each = var.lifecycle_rules - content { - action { - type = lifecycle_rule.value.action.type - storage_class = lookup(lifecycle_rule.value.action, "storage_class", null) - } - condition { - age = lookup(lifecycle_rule.value.condition, "age", null) - send_age_if_zero = lookup(lifecycle_rule.value.condition, "send_age_if_zero", null) - created_before = lookup(lifecycle_rule.value.condition, "created_before", null) - with_state = lookup(lifecycle_rule.value.condition, "with_state", contains(keys(lifecycle_rule.value.condition), "is_live") ? (lifecycle_rule.value.condition["is_live"] ? "LIVE" : null) : null) - matches_storage_class = lifecycle_rule.value.condition["matches_storage_class"] != null ? split(",", lifecycle_rule.value.condition["matches_storage_class"]) : null - matches_prefix = lifecycle_rule.value.condition["matches_prefix"] != null ? split(",", lifecycle_rule.value.condition["matches_prefix"]) : null - matches_suffix = lifecycle_rule.value.condition["matches_suffix"] != null ? split(",", lifecycle_rule.value.condition["matches_suffix"]) : null - num_newer_versions = lookup(lifecycle_rule.value.condition, "num_newer_versions", null) - custom_time_before = lookup(lifecycle_rule.value.condition, "custom_time_before", null) - days_since_custom_time = lookup(lifecycle_rule.value.condition, "days_since_custom_time", null) - days_since_noncurrent_time = lookup(lifecycle_rule.value.condition, "days_since_noncurrent_time", null) - noncurrent_time_before = lookup(lifecycle_rule.value.condition, "noncurrent_time_before", null) - } - } - } - - lifecycle { - precondition { - condition = !var.autoclass.enabled || !var.enable_hierarchical_namespace - error_message = "Hierarchical namespace is not compatible with Autoclass enabled." - } - - precondition { - condition = !var.enable_hierarchical_namespace || var.uniform_bucket_level_access - error_message = "Hierarchical namespace is not compatible with Uniform bucket level access disabled." - } - - precondition { - condition = !var.enable_versioning || !var.enable_hierarchical_namespace - error_message = "Hierarchical namespace is not compatible with Object versioning enabled." - } - } -} - -resource "google_storage_bucket_iam_binding" "viewers" { - bucket = google_storage_bucket.bucket.name - role = "roles/storage.objectViewer" - members = var.viewers -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf deleted file mode 100644 index 29ddfef2d2..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/outputs.tf +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a remote network storage to be mounted by fs-tab." - value = { - remote_mount = local.name - local_mount = var.local_mount - fs_type = "gcsfuse" - mount_options = var.mount_options - server_ip = "" - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } -} - -locals { - client_install_runner = { - "type" = "shell" - "content" = file("${path.module}/scripts/install-gcs-fuse.sh") - "destination" = "install-gcsfuse${replace(var.local_mount, "/", "_")}.sh" - } - - mount_runner = { - "type" = "shell" - "destination" = "mount_gcs${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"not-used\" \"${local.name}\" \"${var.local_mount}\" \"gcsfuse\" \"${var.mount_options}\"" - "content" = file("${path.module}/scripts/mount.sh") - } -} - -output "client_install_runner" { - description = "Runner that performs client installation needed to use gcs fuse." - value = local.client_install_runner -} - -output "mount_runner" { - description = "Runner that mounts the cloud storage bucket with gcs fuse." - value = local.mount_runner -} - -output "gcs_bucket_path" { - description = "The gsutil bucket path with format of `gs://`." - # cannot use resource attribute, will cause lookup failure in startup-script - value = "gs://${local.name}" - - # needed to make sure bucket contents are deleted before bucket - depends_on = [ - google_storage_bucket.bucket - ] -} - -output "gcs_bucket_name" { - description = "Bucket name." - value = google_storage_bucket.bucket.name -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh deleted file mode 100644 index f8a990260b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -if [ ! "$(which gcsfuse)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then - tee /etc/yum.repos.d/gcsfuse.repo >/dev/null </dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf deleted file mode 100644 index 9804e4b268..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/variables.tf +++ /dev/null @@ -1,254 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which GCS bucket will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment; used as part of name of the GCS bucket." - type = string -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "labels" { - description = "Labels to add to the GCS bucket. Key-value pairs." - type = map(string) -} - -variable "local_mount" { - description = "The mount point where the contents of the device may be accessed after mounting." - type = string - default = "/mnt" -} - -variable "mount_options" { - description = "Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories)" - type = string - default = "defaults,_netdev,implicit_dirs" -} - -variable "name_prefix" { - description = "Name Prefix." - type = string - default = null -} - -variable "use_deployment_name_in_bucket_name" { - description = "If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments." - type = bool - default = true -} - -variable "random_suffix" { - description = "If true, a random id will be appended to the suffix of the bucket name." - type = bool - default = false -} - -variable "force_destroy" { - description = "If true will destroy bucket with all objects stored within." - type = bool - default = false -} - -variable "viewers" { - description = "A list of additional accounts that can read packages from this bucket" - type = set(string) - default = [] - - validation { - error_message = "All bucket viewers must be in IAM style: user:user@example.com, serviceAccount:sa@example.com, or group:group@example.com." - condition = alltrue([ - for viewer in var.viewers : length(regexall("^(user|serviceAccount|group):", viewer)) > 0 - ]) - } -} - -variable "enable_hierarchical_namespace" { - description = "If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket." - type = bool - default = false -} - -variable "uniform_bucket_level_access" { - description = "Allow uniform control access to the bucket." - type = bool - default = true -} - -variable "storage_class" { - description = "The storage class of the GCS bucket." - type = string - default = "REGIONAL" - validation { - condition = contains([ - "STANDARD", - "MULTI_REGIONAL", - "REGIONAL", - "NEARLINE", - "COLDLINE", - "ARCHIVE" - ], var.storage_class) - error_message = "Allowed values for GCS storage_class are 'STANDARD', 'MULTI_REGIONAL', 'REGIONAL', 'NEARLINE', 'COLDLINE', 'ARCHIVE'.\nhttps://cloud.google.com/storage/docs/storage-classes" - } -} - -variable "autoclass" { - description = <<-EOT - Configure bucket autoclass setup - - The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern. - - The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time. - Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE') - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/autoclass - EOT - type = object({ - enabled = optional(bool, false) - terminal_storage_class = optional(string, null) - }) - default = { - enabled = false - } - nullable = false - validation { - condition = !can(coalesce(var.autoclass.terminal_storage_class)) || var.autoclass.enabled - error_message = "Cannot set bucket var.autoclass.terminal_storage_class unless var.autoclass.enabled is true" - } -} - -variable "public_access_prevention" { - description = <<-EOT - Bucket public access can be controlled by setting a value of either `inherited` or `enforced`. - When set to `enforced`, public access to the bucket is blocked. - If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/public-access-prevention - EOT - type = string - default = null - validation { - condition = var.public_access_prevention == null ? true : contains([ - "inherited", - "enforced" - ], var.public_access_prevention) - error_message = "Allowed values for public_access_prevention are 'inherited', 'enforced'.\n" - } -} - -variable "soft_delete_retention_duration" { - description = <<-EOT - If defined, this will configure soft_delete_policy with retention_duration_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days). - Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/soft-delete - EOT - type = number - default = null - validation { - condition = var.soft_delete_retention_duration == null ? true : var.soft_delete_retention_duration == 0 || var.soft_delete_retention_duration >= 604800 && var.soft_delete_retention_duration <= 7776000 - error_message = "var.soft_delete_retention_duration value can be 0 or in between 604800(7 days) and 7776000(90 days)." - } -} - -variable "enable_versioning" { - description = "If true, enables versioning for the bucket." - type = bool - default = false -} - -variable "lifecycle_rules" { - description = "List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule" - type = list(object({ - # Object with keys: - # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass. - # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule. - action = object({ - type = string - storage_class = optional(string) - }) - - # Object with keys: - # - age - (Optional) Minimum age of an object in days to satisfy this condition. - # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field. - # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition. - # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY". - # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY. - # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition. - # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition. - # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition. - # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition. - # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true. - # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object. - # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent. - condition = object({ - age = optional(number) - send_age_if_zero = optional(bool) - created_before = optional(string) - with_state = optional(string) - matches_storage_class = optional(string) - matches_prefix = optional(string) - matches_suffix = optional(string) - num_newer_versions = optional(number) - custom_time_before = optional(string) - days_since_custom_time = optional(number) - days_since_noncurrent_time = optional(number) - noncurrent_time_before = optional(string) - }) - })) - default = [] -} - -variable "retention_policy_period" { - description = <<-EOT - If defined, this will configure retention_policy with retention_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/bucket-lock - EOT - type = number - default = null - validation { - condition = var.retention_policy_period == null ? true : var.retention_policy_period > 0 && var.retention_policy_period <= 3155760000 - error_message = "var.soft_delete_policy_retention_duration value must be in between 1 and 3155760000(100 years) seconds." - } -} - -variable "enable_object_retention" { - description = <<-EOT - If true, enables retention policy at per object level for the bucket. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/object-lock - EOT - type = bool - default = false -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf deleted file mode 100644 index 217ee2f3a2..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/cloud-storage-bucket/versions.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.9.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.74.0" - } - required_version = ">= 0.14.0" -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/README.md b/deletion-test/primary/modules/embedded/modules/file-system/filestore/README.md deleted file mode 100644 index 3bf251828e..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/filestore/README.md +++ /dev/null @@ -1,248 +0,0 @@ -## Description - -This module creates a [filestore](https://cloud.google.com/filestore) -instance. Filestore is a high performance network file system that can be -mounted to one or more compute VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -### Deletion protection - -We recommend considering enabling [Filestore deletion protection][fdp]. Deletion -protection will prevent unintentional deletion of an entire Filestore instance. -It does not prevent deletion of files within the Filestore instance when mounted -by a VM. It is not available on some [tiers](#filestore-tiers), including the -default BASIC\_HDD tier or BASIC\_SSD tier. Follow the documentation link for -up to date details. - -Usage can be enabled in a blueprint with, for example: - -```yaml - - id: homefs - source: modules/file-system/filestore - use: [network] - settings: - deletion_protection: - enabled: true - reason: Avoid data loss - filestore_tier: ZONAL - local_mount: /home - size_gb: 1024 -``` - -[fdp]: https://cloud.google.com/filestore/docs/deletion-protection - -### Filestore tiers - -At the time of writing, Filestore supports 5 [tiers of service][tiers] that are -specified in the Toolkit using the following names: - -- Basic HDD: "BASIC\_HDD" ([preferred][tierapi]) or "STANDARD" (deprecated) -- Basic SSD: "BASIC\_SSD" ([preferred][tierapi]) or "PREMIUM" (deprecated) -- Zonal: "ZONAL" -- Enterprise: "ENTERPRISE" -- Regional: "REGIONAL" - -[tierapi]: https://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier - -**Please review the minimum storage requirements for each tier**. The Terraform -module can only enforce the minimum value of the `size_gb` parameter for the -lowest tier of service. If you supply a value that is too low, Filestore -creation will fail when you run `terraform apply`. - -[tiers]: https://cloud.google.com/filestore/docs/service-tiers - -### Filestore protocols and mount options -After Filestore instance is created, you can mount this to the compute node -using different mount options. Toolkit uses [default mount options](https://linux.die.net/man/8/mount) -for all tier services. Filestore has recommended mount options for different -service tiers which may overall improve performance. These can be found here: -[recommended mount options.](https://cloud.google.com/filestore/docs/mounting-fileshares) -While creating filestore module, you can overwrite these mount options as -mentioned below. - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /homefs - mount_options: defaults,hard,timeo=600,retrans=3,_netdev -``` - -Filestore supports NFS protocols `NFS_V3` (default) and `NFS_V4_1`. Protocol support depends on the selected tier: -- `NFS_V3`: Supported on all tiers (`BASIC_HDD`, `BASIC_SSD`, `HIGH_SCALE_SSD`, `ZONAL`, `ENTERPRISE`). -- `NFS_V4_1`: Supported only on `HIGH_SCALE_SSD`, `ZONAL`, `REGIONAL`, and `ENTERPRISE`. -This can be specified at creation time via the `protocol` variable. By default, `NFS_V3` is used for compatibility. -See the example below and [this page](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#protocol-1) for more information. - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /homefs - protocol: NFS_V4_1 - filestore_tier: ZONAL -``` - -### Filestore quota - -Your project must have unused quota for Cloud Filestore in the region you will -provision the storage. This can be found by browsing to the [Quota tab within IAM -& Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. -Please note that there are separate quota limits for HDD and SSD storage. - -All projects begin with 0 available quota for High Scale SSD tier. To use this -tier, [make a request and wait for it to be approved][hs-ssd-quota]. - -[hs-ssd-quota]: https://cloud.google.com/filestore/docs/high-scale - -### Example - Basic HDD - -The Filestore instance defined below will have the following attributes: - -- (default) `BASIC_HDD` tier -- (default) 1TiB capacity -- `homefs` module ID -- mount point at `/home` -- connected to the network defined in the `network1` module - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /home -``` - -### Example - High Scale SSD - -The Filestore instance defined below will have the following attributes: - -- `HIGH_SCALE_SSD` tier -- 10TiB capacity -- `highscale` module ID -- mount point at `/projects` -- connected to the VPC network defined in the `network1` module - -```yaml -- id: highscale - source: modules/file-system/filestore - use: [network1] - settings: - filestore_tier: HIGH_SCALE_SSD - size_gb: 10240 - local_mount: /projects -``` - -## Mounting - -To mount the Filestore instance you must first ensure that the NFS client has -been installed and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -See the [hpc-slurm](../../../examples/hpc-slurm.yaml) for -an example of using this module with Slurm. - -If mounting is not automatically handled as described above, the `filestore` -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - - id: filestore - source: modules/file-system/filestore - use: [network1] - settings: {local_mount: /scratch} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(filestore.install_nfs_client_runner) - - $(filestore.mount_runner) - -``` - -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 6.4 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.4 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_filestore_instance.filestore_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [connect\_mode](#input\_connect\_mode) | Used to select mode - supported values DIRECT\_PEERING and PRIVATE\_SERVICE\_ACCESS. | `string` | `"DIRECT_PEERING"` | no | -| [deletion\_protection](#input\_deletion\_protection) | Configure Filestore instance deletion protection | 
object({
    enabled = optional(bool, false)
    reason  = optional(string)
  })
| 
{
  "enabled": false
}
| no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the filestore instance if no name is specified. | `string` | n/a | yes | -| [description](#input\_description) | A description of the filestore instance. | `string` | `""` | no | -| [filestore\_share\_name](#input\_filestore\_share\_name) | Name of the file system share on the instance. | `string` | `"nfsshare"` | no | -| [filestore\_tier](#input\_filestore\_tier) | The service tier of the instance. | `string` | `"BASIC_HDD"` | no | -| [labels](#input\_labels) | Labels to add to the filestore instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mount](#input\_local\_mount) | Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md)). | `string` | `"/shared"` | no | -| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"defaults,_netdev"` | no | -| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [nfs\_export\_options](#input\_nfs\_export\_options) | Define NFS export options. | 
list(object({
    access_mode = optional(string)
    ip_ranges   = optional(list(string))
    squash_mode = optional(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | ID of project in which Filestore instance will be created. | `string` | n/a | yes | -| [protocol](#input\_protocol) | NFS protocol version. Default is NFS\_V3. NFS\_V4\_1 is only supported with HIGH\_SCALE\_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers. | `string` | `"NFS_V3"` | no | -| [region](#input\_region) | Location for Filestore instances at Enterprise tier. | `string` | n/a | yes | -| [reserved\_ip\_range](#input\_reserved\_ip\_range) | Reserved IP range for Filestore instance. Users are encouraged to set to null
for automatic selection. If supplied, it must be:

CIDR format when var.connect\_mode == "DIRECT\_PEERING"
Named IP Range when var.connect\_mode == "PRIVATE\_SERVICE\_ACCESS"

See Cloud documentation for more details:

https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range | `string` | `null` | no | -| [size\_gb](#input\_size\_gb) | Storage size of the filestore instance in GB. | `number` | `1024` | no | -| [zone](#input\_zone) | Location for Filestore instances below Enterprise tier. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | -| [filestore\_id](#output\_filestore\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | -| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | -| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | -| [network\_storage](#output\_network\_storage) | Describes a filestore instance. | - diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/filestore/main.tf deleted file mode 100644 index ce035dbb2b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/filestore/main.tf +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "filestore", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - is_high_capacity_tier = contains(["HIGH_SCALE_SSD", "ZONAL", "REGIONAL"], var.filestore_tier) && var.size_gb >= 10240 && var.size_gb <= 102400 - - timeouts = local.is_high_capacity_tier ? [1] : [] - server_ip = google_filestore_instance.filestore_instance.networks[0].ip_addresses[0] - remote_mount = format("/%s", google_filestore_instance.filestore_instance.file_shares[0].name) - fs_type = "nfs" - mount_options = var.mount_options - - install_nfs_client_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-nfs-client.sh" - "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" - } - mount_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" - } - - # id format: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network#id - split_network_id = split("/", var.network_id) - network_name = local.split_network_id[4] - network_project = local.split_network_id[1] - shared_vpc = local.network_project != var.project_id -} - -resource "google_filestore_instance" "filestore_instance" { - project = var.project_id - - name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - description = var.description - location = contains(["ENTERPRISE", "REGIONAL"], var.filestore_tier) ? var.region : var.zone - tier = var.filestore_tier - protocol = var.protocol - - deletion_protection_enabled = var.deletion_protection.enabled - deletion_protection_reason = var.deletion_protection.reason - - file_shares { - capacity_gb = var.size_gb - name = var.filestore_share_name - dynamic "nfs_export_options" { - for_each = var.nfs_export_options - content { - access_mode = nfs_export_options.value.access_mode - ip_ranges = nfs_export_options.value.ip_ranges - squash_mode = nfs_export_options.value.squash_mode - } - } - } - - labels = local.labels - - networks { - network = local.shared_vpc ? var.network_id : local.network_name - connect_mode = var.connect_mode - modes = ["MODE_IPV4"] - reserved_ip_range = var.reserved_ip_range - } - - dynamic "timeouts" { - for_each = local.timeouts - content { - create = "1h" - update = "1h" - delete = "1h" - } - } - - lifecycle { - precondition { - condition = ( - var.reserved_ip_range == null || - var.connect_mode == "PRIVATE_SERVICE_ACCESS" || - var.connect_mode == "DIRECT_PEERING" && can(cidrhost(var.reserved_ip_range, 0)) && contains(["24", "29"], try(split("/", var.reserved_ip_range)[1], "")) - ) - error_message = <<-EOT - If connect_mode is set to DIRECT_PEERING and reserved_ip_range is - specified then it must be a CIDR IP range with suffix range size 29 for - BASIC_HDD or BASIC_SSD tiers. Otherwise the range size must be 24. - EOT - } - - precondition { - condition = !startswith(var.filestore_tier, "BASIC") || var.protocol != "NFS_V4_1" - error_message = "NFS_V4_1 is not supported on BASIC Filestore tiers." - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/filestore/metadata.yaml deleted file mode 100644 index 5298336f09..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/filestore/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - file.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/filestore/outputs.tf deleted file mode 100644 index 9bdb3bdc7b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/filestore/outputs.tf +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a filestore instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = local.mount_options - client_install_runner = local.install_nfs_client_runner - mount_runner = local.mount_runner - } -} - -output "install_nfs_client" { - description = "Script for installing NFS client" - value = file("${path.module}/scripts/install-nfs-client.sh") -} - -output "install_nfs_client_runner" { - description = "Runner to install NFS client using the startup-script module" - value = local.install_nfs_client_runner -} - -output "mount_runner" { - description = <<-EOT - Runner to mount the file-system using an ansible playbook. The startup-script - module will automatically handle installation of ansible. - - id: example-startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(your-fs-id.mount_runner) - ... - EOT - value = local.mount_runner -} - -output "filestore_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" - value = google_filestore_instance.filestore_instance.id -} - -output "capacity_gib" { - description = "File share capacity in GiB." - value = google_filestore_instance.filestore_instance.file_shares[0].capacity_gb -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh b/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/filestore/variables.tf deleted file mode 100644 index 2d7e9258c0..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/filestore/variables.tf +++ /dev/null @@ -1,189 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which Filestore instance will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the filestore instance if no name is specified." - type = string -} - -variable "zone" { - description = "Location for Filestore instances below Enterprise tier." - type = string -} - -variable "region" { - description = "Location for Filestore instances at Enterprise tier." - type = string -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is connected given in the format: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "name" { - description = "The resource name of the instance." - type = string - default = null -} - -variable "filestore_share_name" { - description = "Name of the file system share on the instance." - type = string - default = "nfsshare" -} - -variable "local_mount" { - description = "Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md))." - type = string - default = "/shared" -} - -variable "size_gb" { - description = "Storage size of the filestore instance in GB." - type = number - default = 1024 - validation { - condition = var.size_gb >= 1024 - error_message = "No Filestore tier supports less than 1024GiB.\nSee https://cloud.google.com/filestore/docs/service-tiers." - } -} - -variable "filestore_tier" { - description = "The service tier of the instance." - type = string - default = "BASIC_HDD" - validation { - condition = var.filestore_tier != "STANDARD" - error_message = "The preferred name for STANDARD tier is now BASIC_HDD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." - } - validation { - condition = var.filestore_tier != "PREMIUM" - error_message = "The preferred name for PREMIUM tier is now BASIC_SSD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." - } - validation { - condition = contains([ - "BASIC_HDD", - "BASIC_SSD", - "HIGH_SCALE_SSD", - "ZONAL", - "REGIONAL", - "ENTERPRISE" - ], var.filestore_tier) - # Avoid adding the legacy tier name in error_message, for e.g. 'HIGH_SCALE_SSD', 'ENTERPRISE'. - # As we want to steer the customer to new one's, but also support the legacy ones for older customers. - error_message = "Allowed values for filestore_tier are 'BASIC_HDD','BASIC_SSD','ZONAL','REGIONAL'.\nhttps://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#tier\nhttps://cloud.google.com/filestore/docs/reference/rest/v1/Tier." - } -} - -variable "labels" { - description = "Labels to add to the filestore instance. Key-value pairs." - type = map(string) -} - -variable "connect_mode" { - description = "Used to select mode - supported values DIRECT_PEERING and PRIVATE_SERVICE_ACCESS." - type = string - default = "DIRECT_PEERING" - nullable = false - validation { - condition = contains(["DIRECT_PEERING", "PRIVATE_SERVICE_ACCESS"], var.connect_mode) - error_message = "Allowed values for connect_mode are \"DIRECT_PEERING\" or \"PRIVATE_SERVICE_ACCESS\"." - } -} - -variable "nfs_export_options" { - description = "Define NFS export options." - type = list(object({ - access_mode = optional(string) - ip_ranges = optional(list(string)) - squash_mode = optional(string) - })) - default = [] - nullable = false -} - -variable "reserved_ip_range" { - description = <<-EOT - Reserved IP range for Filestore instance. Users are encouraged to set to null - for automatic selection. If supplied, it must be: - - CIDR format when var.connect_mode == "DIRECT_PEERING" - Named IP Range when var.connect_mode == "PRIVATE_SERVICE_ACCESS" - - See Cloud documentation for more details: - - https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range - EOT - type = string - default = null - nullable = true -} - -variable "mount_options" { - description = "NFS mount options to mount file system." - type = string - default = "defaults,_netdev" -} - -variable "deletion_protection" { - description = "Configure Filestore instance deletion protection" - type = object({ - enabled = optional(bool, false) - reason = optional(string) - }) - default = { - enabled = false - } - nullable = false - - validation { - condition = !can(coalesce(var.deletion_protection.reason)) || var.deletion_protection.enabled - error_message = "Cannot set Filestore var.deletion_protection.reason unless var.deletion_protection.enabled is true" - } -} - -variable "protocol" { - description = "NFS protocol version. Default is NFS_V3. NFS_V4_1 is only supported with HIGH_SCALE_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers." - type = string - default = "NFS_V3" - validation { - condition = contains(["NFS_V3", "NFS_V4_1"], var.protocol) - error_message = "Allowed values for protocol are 'NFS_V3' or 'NFS_V4_1'." - } -} - -variable "description" { - description = "A description of the filestore instance." - type = string - default = "" - validation { - condition = length(var.description) <= 2048 - error_message = "Filestore description must be 2048 characters or fewer" - } -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/filestore/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/filestore/versions.tf deleted file mode 100644 index 1ba0e7967e..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/filestore/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.4" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md deleted file mode 100644 index 88ae4511e3..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md +++ /dev/null @@ -1,200 +0,0 @@ -## Description - -This module creates Kubernetes Persistent Volumes (PV) and Persistent Volume -Claims (PVC) that can be used by a [gke-job-template]. - -`gke-persistent-volume` works with Filestore, Google Cloud Storage and Managed Lustre. Each -`gke-persistent-volume` can only be used with a single file system so if multiple -shared file systems are used then multiple `gke-persistent-volume` modules are -needed in the blueprint. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a Filestore and then uses the -`gke-persistent-volume` module to use the Filestore as shared storage in a -`gke-job-template`. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] - settings: - master_authorized_networks: - - display_name: deployment-machine - cidr_block: /32 - - - id: datafs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /data - - - id: datafs-pv - source: modules/file-system/gke-persistent-volume - use: [datafs, gke_cluster] - - - id: job-template - source: modules/compute/gke-job-template - use: [datafs-pv, compute_pool, gke_cluster] -``` - -The following example creates a GCS bucket and then uses the -`gke-persistent-volume` module to use the bucket as shared storage in a -`gke-job-template`. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] - settings: - master_authorized_networks: - - display_name: deployment-machine - cidr_block: /32 - - - id: data-bucket - source: modules/file-system/cloud-storage-bucket - settings: - local_mount: /data - - - id: datagcs-pv - source: modules/file-system/gke-persistent-volume - use: [data-bucket, gke_cluster] - - - id: job-template - source: modules/compute/gke-job-template - use: [datagcs-pv, compute_pool, gke_cluster] -``` - -The following example creates a Managed Lustre and then uses the -`gke-persistent-volume` module to use the Lustre as shared storage in a -`gke-job-template`. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] - settings: - master_authorized_networks: - - display_name: deployment-machine - cidr_block: /32 - - - id: data-managedlustre - source: modules/file-system/managed-lustre - settings: - local_mount: /data - - - id: datalustre-pv - source: modules/file-system/gke-persistent-volume - use: [data-managedlustre, gke_cluster] - - - id: job-template - source: modules/compute/gke-job-template - use: [datalustre-pv, compute_pool, gke_cluster] -``` - -See example -[storage-gke.yaml](../../../../examples/README.md#storage-gkeyaml--) blueprint -for a complete example. - -### Authorized Network - -Since the `gke-persistent-volume` module is making calls to the Kubernetes API -to create Kubernetes entities, the machine performing the deployment must be -authorized to connect to the Kubernetes API. You can add the -`master_authorized_networks` settings block, as shown in the example above, with -the IP address of the machine performing the deployment. This will ensure that -the deploying machine can connect to the cluster. - -### Connecting Via Use - -The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit -modules. For example the `gke-persistent-volume` module can `use` a -`gke-cluster` module and a `filestore` module, as shown in the example above. - -```mermaid - graph TD; - vpc--> |OneToMany| gke-cluster; - gke-cluster--> |OneToMany| gke-node-pool; - gke-node-pool--> |ManyToMany| gke-job-template; - gke-cluster--> |OneToMany| gke-persistent-volume; - gke-persistent-volume--> |ManyToMany| gke-job-template; - vpc--> |OneToMany| filestore; - vpc--> |OneToMany| gcs; - vpc--> |OneToMany| managed-lustre; - filestore--> |OneToOne| gke-persistent-volume; - gcs--> |OneToOne| gke-persistent-volume; - managed-lustre--> |OneToOne| gke-persistent-volume; - ``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [kubectl](#provider\_kubectl) | >= 1.7.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubectl_manifest.pv](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.pvc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.pvc_namespace](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [capacity\_gib](#input\_capacity\_gib) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | -| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | -| [filestore\_id](#input\_filestore\_id) | An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | -| [gcs\_bucket\_name](#input\_gcs\_bucket\_name) | The gcs bucket to be used with the persistent volume. | `string` | `null` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [lustre\_id](#input\_lustre\_id) | An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | Network attached storage mount to be configured. | 
object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  })
| n/a | yes | -| [pv\_name](#input\_pv\_name) | The name for PV. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | -| [pvc\_name](#input\_pvc\_name) | The name for PVC. IF not set, a name will be generated based on the storage name. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object describing the Kubernetes PersistentVolumeClaim created by this module. | -| [pvc\_name](#output\_pvc\_name) | The name of the Kubernetes PVC created by this module. | - diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf deleted file mode 100644 index 818ebaf595..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-persistent-volume", ghpc_role = "file-system" }) -} - -locals { - # Flags indicating which storage type is active based on input variables. - storage_type_active = { - gcs = var.gcs_bucket_name != null - lustre = var.lustre_id != null - filestore = var.filestore_id != null - } - - # Determine the active storage type name. - active_types = [for type, is_active in local.storage_type_active : type if is_active] - - # The precondition in kubectl_manifest.pv ensures exactly one type is active. - storage_type = length(local.active_types) > 0 ? local.active_types[0] : "unknown" - - # Map containing the base name derivation logic for each storage type. - base_name_map = { - gcs = var.gcs_bucket_name - lustre = var.lustre_id != null ? split("/", var.lustre_id)[5] : null - filestore = var.filestore_id != null ? split("/", var.filestore_id)[5] : null - } - # Retrieve the base name for the active storage type. - base_name = local.base_name_map[local.storage_type] - - # PV and PVC names - pv_name = var.pv_name != null ? var.pv_name : "${local.base_name}-pv" - pvc_name = var.pvc_name != null ? var.pvc_name : "${local.base_name}-pvc" - - # Template file paths - pv_templates = { - gcs = "${path.module}/templates/gcs-pv.yaml.tftpl" - lustre = "${path.module}/templates/managed-lustre-pv.yaml.tftpl" - filestore = "${path.module}/templates/filestore-pv.yaml.tftpl" - } - pvc_templates = { - gcs = "${path.module}/templates/gcs-pvc.yaml.tftpl" - lustre = "${path.module}/templates/managed-lustre-pvc.yaml.tftpl" - filestore = "${path.module}/templates/filestore-pvc.yaml.tftpl" - } - - # Common variables for all PVC templates - common_pvc_vars = { - pv_name = local.pv_name - pvc_name = local.pvc_name - labels = local.labels - capacity = "${var.capacity_gib}Gi" - namespace = var.namespace - } - - # Common variables for all PV templates - common_pv_vars = { - pv_name = local.pv_name - capacity = "${var.capacity_gib}Gi" - labels = local.labels - } - - # Variables for PV templates, merging common vars with type-specific ones. - pv_template_vars = { - gcs = merge(local.common_pv_vars, { - mount_options = var.gcs_bucket_name != null ? split(",", var.network_storage.mount_options) : [] - bucket_name = var.gcs_bucket_name - namespace = var.namespace - pvc_name = local.pvc_name - }) - lustre = merge(local.common_pv_vars, { - location = var.lustre_id != null ? split("/", var.lustre_id)[3] : null - project = split("/", var.cluster_id)[1] - instance_name = local.base_name - server_ip = var.lustre_id != null ? split("@", var.network_storage.server_ip)[0] : null - filesystem_name = var.network_storage.remote_mount - pvc_name = local.pvc_name - namespace = var.namespace - }) - filestore = merge(local.common_pv_vars, { - location = var.filestore_id != null ? split("/", var.filestore_id)[3] : null - filestore_name = local.base_name - share_name = trimprefix(var.network_storage.remote_mount, "/") - ip_address = var.network_storage.server_ip - pvc_name = local.pvc_name - namespace = var.namespace - }) - } - - # Rendered YAML contents - pv_content = templatefile( - local.pv_templates[local.storage_type], - local.pv_template_vars[local.storage_type] - ) - pvc_content = templatefile( - local.pvc_templates[local.storage_type], - local.common_pvc_vars - ) - - # GKE Cluster details - cluster_name = split("/", var.cluster_id)[5] - cluster_location = split("/", var.cluster_id)[3] -} - -data "google_container_cluster" "gke_cluster" { - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -provider "kubectl" { - host = "https://${data.google_container_cluster.gke_cluster.endpoint}" - cluster_ca_certificate = base64decode(data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) - token = data.google_client_config.default.access_token - load_config_file = false -} - -resource "kubectl_manifest" "pvc_namespace" { - count = var.namespace != "default" ? 1 : 0 - - yaml_body = templatefile("${path.module}/templates/namespace.yaml.tftpl", { - namespace = var.namespace - }) -} - -resource "kubectl_manifest" "pv" { - yaml_body = local.pv_content - - lifecycle { - precondition { - condition = length(local.active_types) == 1 - error_message = "Exactly one of gcs_bucket_name, filestore_id, or lustre_id must be set." - } - } -} - -resource "kubectl_manifest" "pvc" { - yaml_body = local.pvc_content - depends_on = [kubectl_manifest.pv, kubectl_manifest.pvc_namespace] -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf deleted file mode 100644 index 60cf2dbe0f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "persistent_volume_claims" { - description = "An object describing the Kubernetes PersistentVolumeClaim created by this module." - value = { - name = local.pvc_name - namespace = var.namespace - mount_path = var.network_storage.local_mount - mount_options = var.network_storage.mount_options - storage_type = local.storage_type - } -} - -output "pvc_name" { - description = "The name of the Kubernetes PVC created by this module." - value = local.pvc_name -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl deleted file mode 100644 index 06a1276c1e..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ${pv_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - storageClassName: "" - capacity: - storage: ${capacity} - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - volumeMode: Filesystem - csi: - driver: filestore.csi.storage.gke.io - volumeHandle: "modeInstance/${location}/${filestore_name}/${share_name}" - volumeAttributes: - ip: ${ip_address} - volume: ${share_name} - claimRef: - name: ${pvc_name} - namespace: ${namespace} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl deleted file mode 100644 index 83cfb3bc8c..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl +++ /dev/null @@ -1,18 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - volumeName: ${pv_name} - resources: - requests: - storage: ${capacity} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl deleted file mode 100644 index aa0e570a8b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ${pv_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - storageClassName: "" - capacity: - storage: ${capacity} - accessModes: - - ReadWriteMany - %{~ if mount_options != null ~} - mountOptions: - %{~ for key in mount_options ~} - - ${key} - %{~ endfor ~} - %{~ endif ~} - csi: - driver: gcsfuse.csi.storage.gke.io - volumeHandle: ${bucket_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl deleted file mode 100644 index 4d02c85629..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl +++ /dev/null @@ -1,21 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - volumeName: ${pv_name} - resources: - requests: - storage: ${capacity} - claimRef: - name: ${pvc_name} - namespace: ${namespace} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl deleted file mode 100644 index 2b3b5e7738..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pv.yaml.tftpl +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ${pv_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - storageClassName: "" - capacity: - storage: ${capacity} - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - volumeMode: Filesystem - claimRef: - namespace: ${namespace} - name: ${pvc_name} - csi: - driver: lustre.csi.storage.gke.io - volumeHandle: "${project}/${location}/${instance_name}/default-pool/default-container" - volumeAttributes: - ip: ${server_ip} - filesystem: ${filesystem_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl deleted file mode 100644 index 83cfb3bc8c..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/managed-lustre-pvc.yaml.tftpl +++ /dev/null @@ -1,18 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - volumeName: ${pv_name} - resources: - requests: - storage: ${capacity} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl deleted file mode 100644 index fa7647e33f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/namespace.yaml.tftpl +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf deleted file mode 100644 index fd281756e7..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "cluster_id" { - description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" - type = string -} - -variable "network_storage" { - description = "Network attached storage mount to be configured." - type = object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - }) -} - -variable "filestore_id" { - description = "An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." - type = string - default = null - validation { - condition = ( - var.filestore_id == null || - try(length(split("/", var.filestore_id)), 0) == 6 - ) - error_message = "filestore_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." - } -} - -variable "lustre_id" { - description = "An identifier for a lustre with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." - type = string - default = null - validation { - condition = ( - var.lustre_id == null || - try(length(split("/", var.lustre_id)), 0) == 6 - ) - error_message = "lustre_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." - } -} - -variable "gcs_bucket_name" { - description = "The gcs bucket to be used with the persistent volume." - type = string - default = null -} - -variable "capacity_gib" { - description = "The storage capacity with which to create the persistent volume." - type = number -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} - -variable "namespace" { - description = "Kubernetes namespace to deploy the storage PVC/PV" - type = string - default = "default" -} - -variable "pv_name" { - description = "The name for PV. IF not set, a name will be generated based on the storage name." - type = string - default = null -} - -variable "pvc_name" { - description = "The name for PVC. IF not set, a name will be generated based on the storage name." - type = string - default = null -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf deleted file mode 100644 index fa1c3e2b3f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.7.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.74.0" - } -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/README.md b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/README.md deleted file mode 100644 index 78ef5402aa..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/README.md +++ /dev/null @@ -1,134 +0,0 @@ -## Description - -This module creates Kubernetes Storage Class (SC) that can be used by a Persistent Volume Claim (PVC) -to dynamically provision GCP storage resources like Parallelstore. - -### Example - -The following example uses the `gke-storage` module to creates a Parallelstore Storage Class and Persistent Volume Claim, -then use them in a `gke-job-template` to dynamically provision the resource. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network] - settings: - enable_parallelstore_csi: true - - # Private Service Access (PSA) requires the compute.networkAdmin role which is - # included in the Owner role, but not Editor. - # PSA is required for all Parallelstore functionality. - # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: gke_storage - source: modules/file-system/gke-storage - use: [ gke_cluster, private_service_access ] - settings: - storage_type: Parallelstore - access_mode: ReadWriteMany - sc_volume_binding_mode: Immediate - sc_reclaim_policy: Delete - sc_topology_zones: [$(vars.zone)] - pvc_count: 2 - capacity_gb: 12000 - - - id: job_template - source: modules/compute/gke-job-template - use: [gke_storage, compute_pool] -``` - -See example -[gke-managed-parallelstore.yaml](../../../examples/README.md#gke-managed-parallelstoreyaml--) blueprint -for a complete example. - -### Authorized Network - -Since the `gke-storage` module is making calls to the Kubernetes API -to create Kubernetes entities, the machine performing the deployment must be -authorized to connect to the Kubernetes API. You can add the -`master_authorized_networks` settings block, as shown in the example above, with -the IP address of the machine performing the deployment. This will ensure that -the deploying machine can connect to the cluster. - -### Connecting Via Use - -The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit -modules. For example the `gke-storage` module can `use` a -`gke-cluster` module and a `private_service_access` module, as shown in the example above. - -```mermaid -graph TD; - vpc-->|OneToMany|gke-cluster; - gke-cluster-->|OneToMany|gke-node-pool; - gke-node-pool-->|ManyToMany|gke-job-template; - gke-cluster-->|OneToMany|gke-storage; - gke-storage-->|ManyToMany|gke-job-template; -``` - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_mode](#input\_access\_mode) | The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)
Valid access modes:
- ReadWriteOnce
- ReadOnlyMany
- ReadWriteMany
- ReadWriteOncePod | `string` | n/a | yes | -| [capacity\_gb](#input\_capacity\_gb) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | -| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [mount\_options](#input\_mount\_options) | Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class. | `string` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to deploy the storage PVC/PV | `string` | `"default"` | no | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [pv\_mount\_path](#input\_pv\_mount\_path) | Path within the container at which the volume should be mounted. Must not contain ':'. | `string` | `"/data"` | no | -| [pvc\_count](#input\_pvc\_count) | How many PersistentVolumeClaims that will be created | `number` | `1` | no | -| [sc\_reclaim\_policy](#input\_sc\_reclaim\_policy) | Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted.
[More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming)
Supported value:
- Retain
- Delete | `string` | n/a | yes | -| [sc\_topology\_zones](#input\_sc\_topology\_zones) | Zone location that allow the volumes to be dynamically provisioned. | `list(string)` | `null` | no | -| [sc\_volume\_binding\_mode](#input\_sc\_volume\_binding\_mode) | Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound.
Supported value:
- Immediate
- WaitForFirstConsumer | `string` | `"WaitForFirstConsumer"` | no | -| [storage\_type](#input\_storage\_type) | The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview)
to used. This module currently support dynamic provisioning for the below storage options
- Parallelstore
- Hyperdisk-balanced
- Hyperdisk-throughput
- Hyperdisk-extreme | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object that describes a k8s PVC created by this module. | - diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/main.tf deleted file mode 100644 index 9c9a641f79..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/main.tf +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-storage", ghpc_role = "file-system" }) -} - -locals { - storage_type = lower(var.storage_type) - storage_class_name = "${local.storage_type}-sc" - pvc_name_prefix = "${local.storage_type}-pvc" -} - -check "private_vpc_connection_peering" { - assert { - condition = lower(var.storage_type) != "parallelstore" ? true : var.private_vpc_connection_peering != null - error_message = <<-EOT - Parallelstore must be run within the same VPC as the GKE cluster and have private services access enabled. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access. - If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). - EOT - } -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = var.cluster_id - project_id = var.project_id - - # count = var.pvc_count - apply_manifests = flatten( - [ - # create StorageClass in the cluster - { - content = templatefile( - "${path.module}/storage-class/${local.storage_class_name}.yaml.tftpl", - { - name = local.storage_class_name - labels = local.labels - volume_binding_mode = var.sc_volume_binding_mode - reclaim_policy = var.sc_reclaim_policy - topology_zones = var.sc_topology_zones - }) - }, - var.namespace != "default" ? [{ - content = templatefile( - "${path.module}/persistent-volume-claim/namespace.yaml.tftpl", - { - namespace = var.namespace - }) - }] : [], - # create PersistentVolumeClaim in the cluster - flatten([ - for idx in range(var.pvc_count) : [ - { - content = templatefile( - "${path.module}/persistent-volume-claim/${(local.pvc_name_prefix)}.yaml.tftpl", - { - pvc_name = "${local.pvc_name_prefix}-${idx}" - labels = local.labels - capacity = "${var.capacity_gb}Gi" - access_mode = var.access_mode - storage_class_name = local.storage_class_name - namespace = var.namespace - } - ) - } - ] - ]) - ]) -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml deleted file mode 100644 index 8722823274..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf deleted file mode 100644 index ce80cdb266..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "persistent_volume_claims" { - description = "An object that describes a k8s PVC created by this module." - value = flatten([ - for idx in range(var.pvc_count) : [{ - name = "${local.pvc_name_prefix}-${idx}" - namespace = var.namespace - mount_path = "${var.pv_mount_path}/${local.pvc_name_prefix}-${idx}" - mount_options = var.mount_options - storage_type = local.storage_type - }] - ]) -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl deleted file mode 100644 index fa7647e33f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/namespace.yaml.tftpl +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl deleted file mode 100644 index 893b5e7103..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - namespace: ${namespace} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl deleted file mode 100644 index 46e1f023d3..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - type: hyperdisk-balanced - provisioned-throughput-on-create: "250Mi" - provisioned-iops-on-create: "7000" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl deleted file mode 100644 index 445020d001..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - %{~ endfor ~} - type: hyperdisk-extreme - provisioned-iops-on-create: "50000" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl deleted file mode 100644 index ec404aec45..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - type: hyperdisk-throughput - provisioned-throughput-on-create: "250Mi" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl deleted file mode 100644 index e6b8ea8d3e..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: parallelstore.csi.storage.gke.io -parameters: -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/variables.tf deleted file mode 100644 index dba1c33b77..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/variables.tf +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "cluster_id" { - description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" - type = string -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} - -variable "storage_type" { - description = <<-EOT - The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview) - to used. This module currently support dynamic provisioning for the below storage options - - Parallelstore - - Hyperdisk-balanced - - Hyperdisk-throughput - - Hyperdisk-extreme - EOT - type = string - nullable = false - validation { - condition = var.storage_type == null ? false : contains(["parallelstore", "hyperdisk-balanced", "hyperdisk-throughput", "hyperdisk-extreme"], lower(var.storage_type)) - error_message = "Allowed string values for var.storage_type are \"Parallelstore\", \"Hyperdisk-balanced\", \"Hyperdisk-throughput\", \"Hyperdisk-extreme\"." - } -} - -variable "access_mode" { - description = <<-EOT - The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) - Valid access modes: - - ReadWriteOnce - - ReadOnlyMany - - ReadWriteMany - - ReadWriteOncePod - EOT - type = string - nullable = false - validation { - condition = var.access_mode == null ? false : contains(["readwriteonce", "readonlymany", "readwritemany", "readwriteoncepod"], lower(var.access_mode)) - error_message = "Allowed string values for var.access_mode are \"ReadWriteOnce\", \"ReadOnlyMany\", \"ReadWriteMany\", \"ReadWriteOncePod\"." - } -} - -variable "sc_volume_binding_mode" { - description = <<-EOT - Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound. - Supported value: - - Immediate - - WaitForFirstConsumer - EOT - type = string - default = "WaitForFirstConsumer" - validation { - condition = var.sc_volume_binding_mode == null ? true : contains(["immediate", "waitforfirstconsumer"], lower(var.sc_volume_binding_mode)) - error_message = "Allowed string values for var.sc_volume_binding_mode are \"Immediate\", \"WaitForFirstConsumer\"." - } -} - -variable "sc_reclaim_policy" { - description = <<-EOT - Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted. - [More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming) - Supported value: - - Retain - - Delete - EOT - type = string - nullable = false - validation { - condition = var.sc_reclaim_policy == null ? true : contains(["retain", "delete"], lower(var.sc_reclaim_policy)) - error_message = "Allowed string values for var.sc_reclaim_policy are \"Retain\", \"Delete\"." - } -} - -variable "sc_topology_zones" { - description = "Zone location that allow the volumes to be dynamically provisioned." - type = list(string) - default = null -} - -variable "pvc_count" { - description = "How many PersistentVolumeClaims that will be created" - type = number - default = 1 -} - -variable "pv_mount_path" { - description = "Path within the container at which the volume should be mounted. Must not contain ':'." - type = string - default = "/data" - validation { - condition = var.pv_mount_path == null ? true : !strcontains(var.pv_mount_path, ":") - error_message = "pv_mount_path must not contain ':', please correct it and retry" - } -} - -variable "mount_options" { - description = "Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class." - type = string - default = null -} - -variable "capacity_gb" { - description = "The storage capacity with which to create the persistent volume." - type = number -} - -variable "private_vpc_connection_peering" { - description = <<-EOT - The name of the VPC Network peering connection. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access and - If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). - EOT - type = string - default = null -} - -variable "namespace" { - description = "Kubernetes namespace to deploy the storage PVC/PV" - type = string - default = "default" -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/versions.tf deleted file mode 100644 index bcc803e41e..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/gke-storage/versions.tf +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.5" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.74.0" - } -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/README.md b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/README.md deleted file mode 100644 index 28530a379f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/README.md +++ /dev/null @@ -1,289 +0,0 @@ -## Description - -This module creates a [Managed Lustre](https://cloud.google.com/managed-lustre) -instance. Managed Lustre is a high performance network file system that can be -mounted to one or more VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -### Supported Operating Systems - -A Managed Lustre instance can be used with Slurm cluster or compute -VM running Ubuntu 20.04, 22.04 or Rocky Linux 8 (including the HPC flavor). - -### Managed Lustre Access - -Managed Lustre must be enabled for your project by Google staff. Please contact -your sales representative for further steps. - -### Example - New VPC - -For Managed Lustre instance, the snippet below creates new VPC and configures -private-service-access for this newly created network. Both items are required -to be passed to the Lustre module to ensure that they're built in order and -that the correct subnetwork has private service access. - -```yaml - - id: network - source: modules/network/vpc - - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] -``` - -### Example - Slurm - -When using Slurm you must take into consideration whether or not you are using -an official image from the `schedmd-slurm-public` project or building your own. -The Lustre client modules are pre-installed in the official images. With the -official images, Lustre can be used as follows: - -```yaml -- id: managed_lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - -# Other modules: nodesets, partitions, login, etc. - -- id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: - - network - - lustre_partition - - managed_lustre - - slurm_login - settings: - machine_type: n2-standard-4 - enable_controller_public_ips: true -``` - -For custom images you must install the modules during the image build as the -Slurm cluster will not run the installation script like it does for the -standard VMs. - -Assuming you have a startup script for the Slurm image building, you can add -this Ansible playbook to correctly install the Lustre drivers into the image -(for Slurm-GCP versions greater than 6.10.0): - -```yaml -- type: data - destination: /var/tmp/slurm_vars.json - content: | - { - "reboot": false, - "install_cuda": false, - "install_gcsfuse": true, - "install_lustre": false, - "install_managed_lustre": true, - "install_nvidia_repo": true, - "install_ompi": true, - "allow_kernel_upgrades": false, - "monitoring_agent": "cloud-ops", - } -``` - -The `install_managed_lustre: true` line specifies that slurm-gcp should install -the correct modules within the slurm image. This runner should be placed -ahead of the script that calls the ansible build of the slurm-gcp image. - -### Example - Existing VPC - -If you want to use existing network with private-service-access configured, you need -to manually provide `private_vpc_connection_peering` to the Managed Lustre module. -You can get this details from the Google Cloud Console UI in `VPC network peering` -section. Below is the example of using existing network and creating Managed Lustre. -If existing network is not configured with private-service-access, you can follow -[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) -to set it up. - -```yaml - - id: network - source: modules/network/pre-existing-vpc - settings: - network_name: // Add network name - subnetwork_name: // Add subnetwork name - - - id: lustre - source: modules/file-system/managed-lustre - use: [network] - settings: - private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" -``` - -### Example - GKE compatibility - -By default the Managed Lustre instance that is deployed is not compatible with -GKE. To enable the compatibility use the `gke_support_enabled: true` option. -This creates a file `/etc/modprobe/lnet.conf` that changes the listening port -to 6988. - -```yaml - - id: managed-lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - gke_support_enabled: true -``` - -> [!WARNING] -> -> 1. VMs cannot connect to both GKE compatible and GKE incompatible lustre -> instances at the same time as they connect to different ports. Lustre can -> only listen to one port at a time. -> -> 2. Setting `gke_support_enabled: true` will not affect Slurm nodes, GKE -> compatibility must be built into the Slurm image. - -### Example - Importing data from GSC Bucket - -One option with the Managed Lustre instance is to import data from a GSC bucket -upon the lustre instance creation. To do this, use the `import_gcs_bucket_uri` -variable to dictate the bucket to pull data from. The data will be imported -under the directory specified by `local_mount` (`/shared` if unspecified). - -> [!NOTE] -> -> 1. This is a one way operation. Once the data has been copied to the lustre -> instance it will not be updated with any changes made to the GCS bucket. -> -> 2. Once the lustre instance has been created in Terraform, the copy process -> will proceed in the background. Data may not be appear in the mounted -> directory for a period of time after the deployment has completed (see below). - -```yaml -- id: managed_lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - import_gcs_bucket_uri: gs:// -``` - -> [!WARNING] -> Please follow [this guide](https://cloud.google.com/managed-lustre/docs/transfer-data#required_permissions) -> to set up the correct IAM permissions for importing data from GCS to lustre. -> Without this, the copy process may fail silently leaving an empty lustre -> instance. - -If an import is requested, gcluster will output a json response similar to: - -```json -{ - "name": "projects//locations//operations/", - "metadata": { - "@type": "type.googleapis.com/google.cloud.lustre.v1.ImportDataMetadata", - "createTime": "", - "target": "projects//locations//instances/", - "requestedCancellation": false, - "apiVersion": "v1" - }, - "done": false -} -``` - -You can retrieve more information about the transfer using the following -command, substituting with values from the json response above: - -```bash -gcloud lustre operations describe --location --project -``` - -This will provide information on if the transfer is complete or if any errors -have occurred. See more at -[Get operation](https://cloud.google.com/managed-lustre/docs/transfer-data#get_operation). - -## License - - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 6.27.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.27.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_lustre_instance.lustre_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/lustre_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | -| [google_storage_bucket.lustre_import_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the Lustre instance if no name is specified. | `string` | n/a | yes | -| [description](#input\_description) | Description of the created Lustre instance. | `string` | `"Lustre Instance"` | no | -| [gke\_support\_enabled](#input\_gke\_support\_enabled) | Set to true to create Managed Lustre instance with GKE compatibility.
Note: This does not work with Slurm, the Slurm image must be built with
the correct compatibility. | `bool` | `false` | no | -| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to managed lustre. Data will
be imported to the local\_mount directory. Changing this value will not
trigger a redeployment, to prevent data deletion. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the Managed Lustre instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mount](#input\_local\_mount) | Local mount point for the Managed Lustre instance. | `string` | `"/shared"` | no | -| [mount\_options](#input\_mount\_options) | Mounting options for the file system. | `string` | `"defaults,_netdev"` | no | -| [name](#input\_name) | Name of the Lustre instance | `string` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | Network self-link this instance will be on, required for checking private service access | `string` | n/a | yes | -| [per\_unit\_storage\_throughput](#input\_per\_unit\_storage\_throughput) | Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000. | `number` | `500` | no | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | ID of project in which Lustre instance will be created. | `string` | n/a | yes | -| [remote\_mount](#input\_remote\_mount) | Remote mount point of the Managed Lustre instance | `string` | n/a | yes | -| [size\_gib](#input\_size\_gib) | Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations | `number` | `36000` | no | -| [zone](#input\_zone) | Location for the Lustre instance. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | -| [install\_managed\_lustre\_client](#output\_install\_managed\_lustre\_client) | Script for installing Managed Lustre client | -| [lustre\_id](#output\_lustre\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | -| [network\_storage](#output\_network\_storage) | Describes a Managed Lustre instance. | - diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/main.tf deleted file mode 100644 index a969c53673..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/main.tf +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "managed-lustre", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -data "google_compute_network_peering" "private_peering" { - name = var.private_vpc_connection_peering - network = var.network_self_link -} - -locals { - server_ip = split(":", google_lustre_instance.lustre_instance.mount_point)[0] - remote_mount = split(":", google_lustre_instance.lustre_instance.mount_point)[1] - fs_type = "lustre" - mount_options = var.mount_options - instance_id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - destination_path = "/" - - install_managed_lustre_client_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-managed-lustre-client.sh" - "destination" = "install-managed-lustre-client${replace(var.local_mount, "/", "_")}.sh" - "args" = var.gke_support_enabled ? "1" : "0" - } - mount_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" - } - - bucket_count = try(length(data.google_storage_bucket.lustre_import_bucket), 0) -} - -data "google_storage_bucket" "lustre_import_bucket" { - count = try(length(var.import_gcs_bucket_uri) > 0, false) ? 1 : 0 - - name = split("//", var.import_gcs_bucket_uri)[1] -} - -resource "google_lustre_instance" "lustre_instance" { - project = var.project_id - - description = var.description - instance_id = local.instance_id - location = var.zone - - filesystem = var.remote_mount - capacity_gib = var.size_gib - per_unit_storage_throughput = var.per_unit_storage_throughput - - labels = local.labels - network = var.network_id - - gke_support_enabled = var.gke_support_enabled - - timeouts { - create = "1h" - update = "1h" - delete = "1h" - } - - depends_on = [var.private_vpc_connection_peering, data.google_storage_bucket.lustre_import_bucket] - - lifecycle { - precondition { - condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" - error_message = "The subnetwork that the lustre instance is hosted on must have private service access." - } - } - - provisioner "local-exec" { - command = < 0 ]]; then - curl -X POST \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $(gcloud auth print-access-token)" \ - -d '{"gcsPath": {"uri":"${coalesce(var.import_gcs_bucket_uri, "gs://")}"}, "lustrePath": {"path":"${local.destination_path}"}}' \ - https://lustre.googleapis.com/v1/projects/${var.project_id}/locations/${var.zone}/instances/${local.instance_id}:importData - fi - EOF - interpreter = ["bash", "-c"] - } -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml deleted file mode 100644 index 66da9827b6..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - lustre.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf deleted file mode 100644 index 6de815524a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a Managed Lustre instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = local.mount_options - client_install_runner = local.install_managed_lustre_client_runner - mount_runner = local.mount_runner - } -} - -output "install_managed_lustre_client" { - description = "Script for installing Managed Lustre client" - value = file("${path.module}/scripts/install-managed-lustre-client.sh") -} - -output "lustre_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" - value = google_lustre_instance.lustre_instance.id -} - -output "capacity_gib" { - description = "File share capacity in GiB." - value = google_lustre_instance.lustre_instance.capacity_gib -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh deleted file mode 100644 index 878130ab47..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Install Managed Lustre client modules -# Based on these instructions: https://cloud.google.com/managed-lustre/docs/connect-from-compute-engine - -# The client modules currently only support Rocky 8, and Ubuntu 20.04/22.04 - -set -e - -GKE_ENABLED=$1 - -# Update lnet to enable GKE supported Lustre instance -if [[ $GKE_ENABLED == "1" ]]; then - if [[ -f "/etc/modprobe.d/lnet.conf" ]] && grep -Fq "options lnet accept_port" /etc/modprobe.d/lnet.conf; then - echo "Lnet accept port already set, continuing without updating /etc/modprobe.d/lnet.conf" - else - echo "options lnet accept_port=6988" >>/etc/modprobe.d/lnet.conf - fi -fi - -if grep -q lustre /proc/filesystems; then - echo "Skipping managed lustre client install as it is already supported" - exit 0 -fi - -# Get distro information -. /etc/os-release -DIST="NA" -if [[ $NAME == *"Ubuntu"* ]]; then - if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then - DIST="Ubuntu" - fi -elif [[ $NAME == *"Rocky"* ]]; then - if [[ $VERSION_ID == "8"* ]]; then - DIST="Rocky" - fi -fi - -if [[ ${DIST} == "Ubuntu" ]]; then - KEY_LOC=/etc/apt/keyrings - KEY_NAME=gcp-ar-repo.gpg - # Download new repo key - mkdir -p "${KEY_LOC}" - wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null - - # Set up apt repo - echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list - - # Install modules - apt update - apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) -elif [[ ${DIST} == "Rocky" ]]; then - # Set up yum repo - touch /etc/yum.repos.d/artifact-registry.repo - tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF - [lustre-client-rocky-8] - name=lustre-client-rocky-8 - baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - # Install modules - yum makecache - yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client -fi - -if [[ $DIST != "NA" ]]; then - # Load the new lustre client module - modprobe lustre -fi diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf deleted file mode 100644 index 65607af66d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which Lustre instance will be created." - type = string -} - -variable "description" { - description = "Description of the created Lustre instance." - type = string - default = "Lustre Instance" -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the Lustre instance if no name is specified." - type = string -} - -variable "zone" { - description = "Location for the Lustre instance." - type = string -} - -variable "name" { - description = "Name of the Lustre instance" - type = string -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is connected given in the format: - `projects//global/networks/`" - EOT - type = string - nullable = false - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "network_self_link" { - description = "Network self-link this instance will be on, required for checking private service access" - type = string - nullable = false -} - -variable "remote_mount" { - description = "Remote mount point of the Managed Lustre instance" - type = string - nullable = false -} - -variable "local_mount" { - description = "Local mount point for the Managed Lustre instance." - type = string - default = "/shared" -} - -variable "size_gib" { - description = "Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations" - type = number - default = 36000 -} - -variable "per_unit_storage_throughput" { - description = "Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000." - type = number - default = 500 -} - -variable "labels" { - description = "Labels to add to the Managed Lustre instance. Key-value pairs." - type = map(string) -} - -variable "mount_options" { - description = "Mounting options for the file system." - type = string - default = "defaults,_netdev" -} - -variable "private_vpc_connection_peering" { - description = <<-EOT - The name of the VPC Network peering connection. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access and - If using existing VPC with private-service-access enabled, set this manually." - EOT - type = string - nullable = false -} - -variable "gke_support_enabled" { - description = <<-EOT - Set to true to create Managed Lustre instance with GKE compatibility. - Note: This does not work with Slurm, the Slurm image must be built with - the correct compatibility. - EOT - type = bool - nullable = false - default = false -} - -variable "import_gcs_bucket_uri" { - description = <<-EOT - The name of the GCS bucket to import data from to managed lustre. Data will - be imported to the local_mount directory. Changing this value will not - trigger a redeployment, to prevent data deletion. - EOT - type = string - default = null - - validation { - condition = startswith(coalesce(var.import_gcs_bucket_uri, "gs://"), "gs://") - error_message = "The GCS bucket uri must start with 'gs://'" - } -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf deleted file mode 100644 index 2322c9a8fd..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.27.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.74.0" - } - - required_version = ">= 1.3.0" -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/README.md b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/README.md deleted file mode 100644 index 82332f3406..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/README.md +++ /dev/null @@ -1,193 +0,0 @@ -## Description - -This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) -storage pool. - -NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. -NetApp Volume provides: - -- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x -- a [rich feature set][service-levels] -- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) -- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data -- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost - -Support for NetApp Volumes is split into two modules. - -- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. -- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -### NetApp storage pool service levels - -The netapp-storage-pool module currently supports the following NetApp Volumes [service levels][service-levels]: - -- Standard: 16 KiBps throughput per provisioned KiB of volume capacity. -- Premium: 64 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. -- Extreme: 128 KiBps throughput per provisioned KiB of volume capacity. Optional [auto-tiering]. - -Check the [service level matrix][service-levels] for additional information on capability differences between service levels. Flex service levels are currently not supported, but you can connect to existing Flex volumes using the [pre-existing-network-storage module][pre-existing]. - -### On-boarding NetApp Volumes -NetApp Volumes uses [Private Service Access](https://cloud.google.com/vpc/docs/private-services-access) (PSA) to connect volumes to your network. Before you create a storage pool, make sure to [connect NetApp Volumes to your network](https://cloud.google.com/netapp/volumes/docs/get-started/configure-access/networking). - -Example of creating a storage pool using a new network: - -```yaml -deployment_groups: -- group: primary - modules: - - id: network - source: modules/network/vpc - settings: - region: $(vars.region) - - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - service_name: "netapp.servicenetworking.goog" - deletion_policy: "ABANDON" - - - id: netapp_pool - source: modules/file-system/netapp-storage-pool - use: [network, private_service_access] - settings: - pool_name: $(vars.deployment_name)-eda-pool - capacity_gib: 20000 - service_level: "EXTREME" - region: $(vars.region) -``` - -Example of creating a storage pool using an existing network which was already PSA-peered with NetApp Volume: - -```yaml -deployment_groups: - - group: primary - modules: - - id: network - source: modules/network/pre-existing-vpc - settings: - project_id: $(vars.project_id) - region: $(vars.region) - network_name: $(vars.network) - - - id: netapp_pool - source: modules/file-system/netapp-storage-pool - use: [network] - settings: - pool_name: "eda-pool" - capacity_gib: 20000 - service_level: "EXTREME" - region: $(vars.region) -``` - -### Storage pool example - -The following example shows all available parameters in use: - -```yaml - - id: netapp_pool - source: modules/file-system/netapp-storage-pool - use: [network, private_service_access] - settings: - pool_name: "mypool" - region: "us-west4" - capacity_gib: 2048 - service_level: "EXTREME" - active_directory_policy: "projects/myproject/locations/us-east4/activeDirectories/my-ad" - cmek_policy: "projects/myproject/locations/us-east4/kmsConfigs/my-cmek-policy" - ldap_enabled: false - allow_auto_tiering: false - description: "Demo storage pool" - labels: - owner: bob -``` - -### NetApp Volumes quota - -Your project must have unused quota for NetApp Volumes in the region you will -provision the storage pool. This can be found by browsing to the [Quota tab within IAM & Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. -Please note that there are separate quota limits for Standard and Premium/Extreme service levels. - -See also NetApp Volumes [default quotas](https://cloud.google.com/netapp/volumes/docs/quotas#netapp-volumes-default-quotas). - -[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels -[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering -[pre-existing]: ../pre-existing-network-storage/README.md -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5.7 | -| [google](#requirement\_google) | >= 6.45.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.45.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_netapp_storage_pool.netapp_storage_pool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_storage_pool) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [active\_directory\_policy](#input\_active\_directory\_policy) | The ID of the Active Directory policy to apply to the storage pool in the format:
`projects//locations//activeDirectoryPolicies/` | `string` | `null` | no | -| [allow\_auto\_tiering](#input\_allow\_auto\_tiering) | Whether to allow automatic tiering for the storage pool. | `bool` | `false` | no | -| [capacity\_gib](#input\_capacity\_gib) | The capacity of the storage pool in GiB. | `number` | `2048` | no | -| [cmek\_policy](#input\_cmek\_policy) | The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format:
`projects//locations//kmsConfigs/` | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used as name of the NetApp storage pool if no name is specified. | `string` | n/a | yes | -| [description](#input\_description) | A description of the NetApp storage pool. | `string` | `""` | no | -| [labels](#input\_labels) | Labels to add to the NetApp storage pool. Key-value pairs. | `map(string)` | n/a | yes | -| [ldap\_enabled](#input\_ldap\_enabled) | Whether to enable LDAP for the storage pool. | `bool` | `false` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | Network self-link the pool will be on, required for checking private service access | `string` | n/a | yes | -| [pool\_name](#input\_pool\_name) | The name of the storage pool. Leave empty to generate name based on deployment name. | `string` | `null` | no | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the private VPC connection peering. | `string` | `"sn-netapp-prod"` | no | -| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | -| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | -| [service\_level](#input\_service\_level) | The service level of the storage pool. | `string` | `"PREMIUM"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gb](#output\_capacity\_gb) | Storage pool capacity in GiB. | -| [netapp\_storage\_pool\_id](#output\_netapp\_storage\_pool\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}` | - diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/main.tf deleted file mode 100644 index b9d63c11c3..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/main.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "netapp-storage-pool", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -data "google_compute_network_peering" "private_peering" { - name = var.private_vpc_connection_peering - network = var.network_self_link -} - -resource "google_netapp_storage_pool" "netapp_storage_pool" { - project = var.project_id - - name = var.pool_name != null ? var.pool_name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - location = var.region - network = var.network_id - service_level = var.service_level - capacity_gib = var.capacity_gib - - active_directory = var.active_directory_policy - kms_config = var.cmek_policy - ldap_enabled = var.ldap_enabled - allow_auto_tiering = var.allow_auto_tiering - - description = var.description - labels = local.labels - - depends_on = [data.google_compute_network_peering.private_peering] - - lifecycle { - precondition { - condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" - error_message = "The network for the storage pool must have private service access." - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml deleted file mode 100644 index 7a5291f9d5..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - netapp.googleapis.com - - servicenetworking.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf deleted file mode 100644 index 91379631c6..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "netapp_storage_pool_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/storagePools/{{name}}`" - value = google_netapp_storage_pool.netapp_storage_pool.id -} - -output "capacity_gb" { - description = "Storage pool capacity in GiB." - value = google_netapp_storage_pool.netapp_storage_pool.capacity_gib -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf deleted file mode 100644 index 04f19fd3fb..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/variables.tf +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which the NetApp storage pool will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the deployment, used as name of the NetApp storage pool if no name is specified." - type = string -} - -variable "region" { - description = "Location for NetApp storage pool." - type = string -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the NetApp storage pool is connected given in the format: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "network_self_link" { - description = "Network self-link the pool will be on, required for checking private service access" - type = string - nullable = false -} - -variable "private_vpc_connection_peering" { - description = "The name of the private VPC connection peering." - type = string - default = "sn-netapp-prod" -} - -variable "pool_name" { - description = "The name of the storage pool. Leave empty to generate name based on deployment name." - type = string - default = null -} - -variable "service_level" { - description = "The service level of the storage pool." - type = string - default = "PREMIUM" - validation { - condition = contains(["STANDARD", "PREMIUM", "EXTREME"], var.service_level) - error_message = "Allowed values for service_level are 'STANDARD', 'PREMIUM', or 'EXTREME'." - } -} - -variable "capacity_gib" { - description = "The capacity of the storage pool in GiB." - type = number - default = 2048 - validation { - condition = var.capacity_gib >= 2048 - error_message = "The minimum capacity for the storage pool is 2048 GiB." - } -} - -variable "active_directory_policy" { - description = <<-EOT - The ID of the Active Directory policy to apply to the storage pool in the format: - `projects//locations//activeDirectoryPolicies/` - EOT - type = string - default = null - validation { - condition = var.active_directory_policy == null ? true : length(split("/", var.active_directory_policy)) == 6 - error_message = "The active directory policy must be provided in the following format: projects//locations//activeDirectoryPolicies/." - } -} - -variable "cmek_policy" { - description = <<-EOT - The ID of the Customer Managed Encryption Key (CMEK) policy to apply to the storage pool in the format: - `projects//locations//kmsConfigs/` - EOT - type = string - default = null - validation { - condition = var.cmek_policy == null ? true : length(split("/", var.cmek_policy)) == 6 - error_message = "The CMEK policy must be provided in the following format: projects//locations//kmsConfigs/." - } -} - -variable "ldap_enabled" { - description = "Whether to enable LDAP for the storage pool." - type = bool - default = false -} - -variable "allow_auto_tiering" { - description = "Whether to allow automatic tiering for the storage pool." - type = bool - default = false -} - -variable "description" { - description = "A description of the NetApp storage pool." - type = string - default = "" - validation { - condition = length(var.description) <= 2048 - error_message = "NetApp storage pool description must be 2048 characters or fewer" - } -} - -variable "labels" { - description = "Labels to add to the NetApp storage pool. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf deleted file mode 100644 index f6501116cd..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-storage-pool/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.45.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-storage-pool/v1.70.0" - } - - required_version = ">= 1.5.7" -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/README.md b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/README.md deleted file mode 100644 index 6aaaf0cb05..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/README.md +++ /dev/null @@ -1,201 +0,0 @@ -## Description - -This module creates a [Google Cloud NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) -volume. - -NetApp Volumes is a first-party Google service that provides NFS and/or SMB shared file-systems to VMs. It offers advanced data management capabilities and highly scalable capacity and performance. -NetApp Volume provides: - -- robust support for NFSv3, NFSv4.x and SMB 2.1 and 3.x -- a [rich feature set][service-levels] -- scalable [performance](https://cloud.google.com/netapp/volumes/docs/performance/performance-benchmarks) -- FlexCache: Caching of ONTAP-based volumes to provide high-throughput and low latency read access to compute clusters of on-premises data -- [Auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering) of unused data to optimse cost - -Support for NetApp Volumes is split into two modules. - -- **netapp-storage-pool** provisions a [storage pool](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview). Storage pools are pre-provisioned storage capacity containers which host volumes. A pool also defines fundamental properties of all the volumes within, like the region, the attached network, the [service level][service-levels], CMEK encryption, Active Directory and LDAP settings. -- **netapp-volume** provisions a [volume](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview) inside an existing storage pool. A volume file-system container which is shared using NFS or SMB. It provides advanced data management capabilities. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -## Deletion protection -The netapp-volume module currently doesn't implement volume deletion protection. If you create a volume with Cluster Toolkit by using this module, Cluster Toolkit will also delete it when you run `gcluster destroy`. All the data in the volume will be gone. If you want to retain the volume instead, it is advised to [use existing volumes not created by Cluster Toolkit](#using-existing-volumes-not-created-by-cluster-toolkit). - -## Volumes overview -Volumes are filesystem containers which can be shared using NFS or SMB filesharing protocols. Volumes *live* inside of [storage pools](https://cloud.google.com/netapp/volumes/docs/configure-and-use/storage-pools/overview), which can be provisioned using the [netapp-storage-pool] module. Volumes inherit fundamental settings from the pool. They *consume* capacity provided by the pool. You can create one or multiple volumes *inside* a pool. - -[netapp-storage-pool]: ../netapp-storage-pool/README.md -[service-levels]: https://cloud.google.com/netapp/volumes/docs/discover/service-levels -[auto-tiering]: https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering -[pre-existing]: ../pre-existing-network-storage/README.md -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## Volume examples -The following examples show the use of netapp-volume. They builds on top of an storage pool which can be provisioned using the [netapp-storage-pool][netapp-storage-pool] module. - -### Example with minimal parameters - -```yaml - - id: home_volume - source: modules/file-system/netapp-volume - use: [netapp_pool] # Create this pool using the netapp-storage-pool module - settings: - volume_name: "eda-home" - capacity_gib: 1024 # Size up to available capacity in the pool - local_mount: "/eda-home" # Mount point at client when client uses USE directive - protocols: ["NFSV3"] - region: $(vars.region) - # Default export policy exports to "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" and no_root_squash -``` - -### Example with all parameters - -```yaml - - id: shared_volume - source: modules/file-system/netapp-volume - use: [netapp_pool] # Create this pool using the netapp-storage-pool module - settings: - volume_name: "eda-shared" - capacity_gib: 25000 # Size up to available capacity in the pool - large_capacity: true - local_mount: "/shared" # Mount point at client when client uses USE directive - mount_options: "rw" # Allows customizing mount options for special workloads - protocols: ["NFSV3","NFSV4"] # List of protocols. ["NFSV3], ["NFSv4] or ["NFSV3, "NFSV4"] - region: $(vars.region) - unix_permissions: "0777" # Specify default permissions for roo inode owned by root:root - # If no export policy is specified, a permissive default policy will be applied, which is: - # allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" # RFC1918 - # has_root_access = true # no_root_squash enabled - # access_type = "READ_WRITE" - export_policy: - - allowed_clients: "10.10.20.8,10.10.20.9" - has_root_access: true # no_root_squash enabled - access_type: "READ_WRITE" - nfsv3: false # allow only NFSv4 for these hosts - nfsv4: true - - allowed_clients: "10.0.0.0/8" - has_root_access: false # no_root_squash disabled - access_type: "READ_WRITE" - nfsv3: true # allow only NFSv3 for these hosts - nfsv4: false - tiering_policy: # Enable auto-tiering. Requires auto-tiering enabled storage pool - tier_action: "ENABLED" - cooling_threshold_days: 31 # tier data blocks which have not been touched for 31 days - - description: "Shared volume for EDA job" - labels: - owner: bob -``` - -## Protocol support -Since Cluster Toolkit is currently built to provision Linux-based compute clusters, this module supports NFSv3 and NFSv4.1 only. SMB is blocked. - -## Large volumes -Volumes larger than 15 TiB can be created as [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). Such volumes can grow up to 3 PiB and can scale read performance up to 29 GiBps. They provide six IP addresses to the volume. They are exported via the `server_ips` output. When connecting a large volume to a client using the USE directive, cluster toolkit currently uses the first IP only. This will be improved in the future. - -This feature is allow-listed GA. To request allow-listing, see [Large Volumes](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/overview#large-capacity-volumes). - -## Auto-tiering support -For auto-tiering enabled storage pools you can enable auto-tiering on the volume. For more information, see [manage auto-tiering](https://cloud.google.com/netapp/volumes/docs/configure-and-use/volumes/manage-auto-tiering). - -## Using existing volumes not created by Cluster Toolkit -NetApp Volumes volumes are regular NFS exports. You can use the [pre-existing-network-storage] module to integrate them into Cluster Toolkit. - -Example code: - -```yaml -- id: homefs - source: modules/file-system/pre-existing-network-storage - settings: - server_ip: ## Set server IP here ## - remote_mount: nfsshare - local_mount: /home - fs_type: nfs -``` - -This creates a resource in Cluster Toolkit which references the specified NFS export, which will be mounted at `/home` by clients which mount if via USE directive. - -Note that the `server_ip` must be known before deployment and this module does not allow -to specify a list of IPs for large volumes. - -[pre-existing-network-storage]: ../pre-existing-network-storage/README.md - -## FlexCache support -NetApp FlexCache technology accelerates data access, reduces WAN latency and lowers WAN bandwidth costs for read-intensive workloads, especially where clients need to access the same data repeatedly. When you create a FlexCache volume, you create a remote cache of an already existing (origin) volume that contains only the actively accessed data (hot data) of the origin volume. - -The FlexCache support in Google Cloud NetApp Volumes allows you to provision a cache volume in your Google network to improve performance for hybrid cloud environments. A FlexCache volume can help you transition workloads to the hybrid cloud by caching data from an on-premises data center to cloud. - -Deploying FlexCache volumes requires manual steps on the ONTAP origin side, which are not automated. Therefore this module has no support to deploy FlexCache volumes today. Deploy them manually and use the [pre-existing-network-storage](#using-existing-volumes-not-created-by-cluster-toolkit) instead. - -## License - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5.7 | -| [google](#requirement\_google) | >= 6.45.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.45.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_netapp_volume.netapp_volume](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_volume) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [capacity\_gib](#input\_capacity\_gib) | The capacity of the volume in GiB. | `number` | `1024` | no | -| [description](#input\_description) | A description of the NetApp volume. | `string` | `""` | no | -| [export\_policy\_rules](#input\_export\_policy\_rules) | Define NFS export policy. | 
list(object({
    allowed_clients = optional(string)
    has_root_access = optional(bool, false)
    access_type     = optional(string, "READ_WRITE")
    nfsv3           = optional(bool)
    nfsv4           = optional(bool)
  }))
| 
[
  {
    "access_type": "READ_WRITE",
    "allowed_clients": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
    "has_root_access": true
  }
]
| no | -| [labels](#input\_labels) | Labels to add to the NetApp volume. Key-value pairs. | `map(string)` | n/a | yes | -| [large\_capacity](#input\_large\_capacity) | If true, the volume will be created with large capacity.
Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. | `bool` | `false` | no | -| [local\_mount](#input\_local\_mount) | Mountpoint for this volume. | `string` | `"/shared"` | no | -| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"rw,hard,rsize=65536,wsize=65536,tcp"` | no | -| [netapp\_storage\_pool\_id](#input\_netapp\_storage\_pool\_id) | The ID of the NetApp storage pool to use for the volume. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | ID of project in which the NetApp storage pool will be created. | `string` | n/a | yes | -| [protocols](#input\_protocols) | The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported. | `list(string)` | 
[
  "NFSV3"
]
| no | -| [region](#input\_region) | Location for NetApp storage pool. | `string` | n/a | yes | -| [tiering\_policy](#input\_tiering\_policy) | Define the tiering policy for the NetApp volume. | 
object({
    tier_action            = optional(string)
    cooling_threshold_days = optional(number)
  })
| `null` | no | -| [unix\_permissions](#input\_unix\_permissions) | UNIX permissions for root inode in the volume. | `string` | `"0777"` | no | -| [volume\_name](#input\_volume\_name) | The name of the volume. Needs to be unique within the storage pool. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gb](#output\_capacity\_gb) | Volume capacity in GiB. | -| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | -| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | -| [netapp\_volume\_id](#output\_netapp\_volume\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}` | -| [network\_storage](#output\_network\_storage) | Describes a NetApp Volumes volume. | -| [server\_ips](#output\_server\_ips) | List of IP addresses of the volume. | - diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/main.tf deleted file mode 100644 index d8345bf347..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/main.tf +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "netapp-volume", ghpc_role = "file-system" }) -} - -# resource "random_id" "resource_name_suffix" { -# byte_length = 4 -# } - -locals { - full_path = split(":", google_netapp_volume.netapp_volume.mount_options[0].export_full) - server_ip = local.full_path[0] - remote_mount = local.full_path[1] - # Large volumes will have 6 IPs - server_ips = [for ip in google_netapp_volume.netapp_volume.mount_options[*].export_full : split(":", ip)[0]] - fs_type = "nfs" - mount_options = var.mount_options - - install_nfs_client_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-nfs-client.sh" - "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" - } - mount_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${join(",", local.server_ips)}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" - } - - split_pool_id = split("/", var.netapp_storage_pool_id) - pool_name = local.split_pool_id[5] -} - -resource "google_netapp_volume" "netapp_volume" { - project = var.project_id - - name = var.volume_name - share_name = var.volume_name - location = var.region - protocols = var.protocols - capacity_gib = var.capacity_gib - large_capacity = var.large_capacity - multiple_endpoints = var.large_capacity == true ? true : null - storage_pool = local.pool_name - unix_permissions = var.unix_permissions - - dynamic "tiering_policy" { - for_each = var.tiering_policy == null ? [] : [0] - content { - cooling_threshold_days = lookup(var.tiering_policy, "cooling_threshold_days", null) - tier_action = lookup(var.tiering_policy, "tier_action", null) - } - } - - description = var.description - labels = local.labels - - dynamic "export_policy" { - for_each = var.export_policy_rules == null ? [] : [0] - content { - dynamic "rules" { - for_each = var.export_policy_rules - content { - access_type = rules.value.access_type - allowed_clients = rules.value.allowed_clients - has_root_access = rules.value.has_root_access - nfsv3 = rules.value.nfsv3 == null ? contains([for p in var.protocols : lower(p)], "nfsv3") : rules.value.nfsv3 - nfsv4 = rules.value.nfsv4 == null ? contains([for p in var.protocols : lower(p)], "nfsv4") : rules.value.nfsv4 - } - } - } - } - - depends_on = [var.netapp_storage_pool_id] -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/metadata.yaml deleted file mode 100644 index e4a7aaaa14..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - netapp.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/outputs.tf deleted file mode 100644 index 641eae007a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/outputs.tf +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -output "network_storage" { - description = "Describes a NetApp Volumes volume." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = local.mount_options - client_install_runner = local.install_nfs_client_runner - mount_runner = local.mount_runner - } -} - -output "install_nfs_client" { - description = "Script for installing NFS client" - value = file("${path.module}/scripts/install-nfs-client.sh") -} - -output "install_nfs_client_runner" { - description = "Runner to install NFS client using the startup-script module" - value = local.install_nfs_client_runner -} - -output "mount_runner" { - description = <<-EOT - Runner to mount the file-system using an ansible playbook. The startup-script - module will automatically handle installation of ansible. - - id: example-startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(your-fs-id.mount_runner) - ... - EOT - value = local.mount_runner -} - -output "netapp_volume_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/volumes/{{name}}`" - value = google_netapp_volume.netapp_volume.id -} - -output "capacity_gb" { - description = "Volume capacity in GiB." - value = google_netapp_volume.netapp_volume.capacity_gib -} - -output "server_ips" { - description = "List of IP addresses of the volume." - value = local.server_ips -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh deleted file mode 100644 index 1b1595e5a4..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsupported distribution' - return 1 - fi -fi diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh deleted file mode 100644 index 8253d40a24..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/scripts/mount.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IPS=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -# accept a list of colon-separated IPs and randomly pick one to enable load balancing -# In recent changes cluster toolkit doesn't seem to use this file anymore, -# which makes all mounts use the first IP in the list. Needs to be investigated in future. -IFS="," read -r -a arrIPS <<<"${SERVER_IPS}" -rand1=$(od -vAn -t d -N1 /dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/variables.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/variables.tf deleted file mode 100644 index 272558ff77..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/variables.tf +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which the NetApp storage pool will be created." - type = string -} - -variable "netapp_storage_pool_id" { - description = "The ID of the NetApp storage pool to use for the volume." - type = string - validation { - condition = length(split("/", var.netapp_storage_pool_id)) == 6 - error_message = "The storage pool id must be provided in the following format: projects//locations//storagePools/." - } -} - -variable "region" { - description = "Location for NetApp storage pool." - type = string -} - -variable "volume_name" { - description = "The name of the volume. Needs to be unique within the storage pool." - type = string - default = null -} - -variable "capacity_gib" { - description = "The capacity of the volume in GiB." - type = number - default = 1024 - validation { - condition = var.capacity_gib >= 100 - error_message = "The minimum capacity for the volume is 100 GiB." - } -} - -variable "protocols" { - description = "The protocols that the volume supports. Currently, only NFSv3 and NFSv4 is supported." - type = list(string) - default = ["NFSV3"] - validation { - condition = alltrue([for p in var.protocols : contains(["NFSV3", "NFSV4"], p)]) - error_message = "Allowed values for protocols are 'NFSV3' or 'NFSV4'." - } -} - -variable "description" { - description = "A description of the NetApp volume." - type = string - default = "" - validation { - condition = length(var.description) <= 2048 - error_message = "NetApp volume description must be 2048 characters or fewer" - } -} - -variable "labels" { - description = "Labels to add to the NetApp volume. Key-value pairs." - type = map(string) -} - -variable "local_mount" { - description = "Mountpoint for this volume." - type = string - default = "/shared" -} - -variable "mount_options" { - description = "NFS mount options to mount file system." - type = string - default = "rw,hard,rsize=65536,wsize=65536,tcp" -} - -variable "large_capacity" { - description = <<-EOT - If true, the volume will be created with large capacity. - Large capacity volumes have 6 IP addresses and a minimal size of 15 TiB. - EOT - type = bool - default = false -} - -variable "unix_permissions" { - description = "UNIX permissions for root inode in the volume." - type = string - default = "0777" - validation { - condition = length(var.unix_permissions) <= 4 - error_message = "UNIX permissions must be a 4-digit octal number." - } -} - -variable "tiering_policy" { - description = "Define the tiering policy for the NetApp volume." - type = object({ - tier_action = optional(string) - cooling_threshold_days = optional(number) - }) - default = null -} - -variable "export_policy_rules" { - description = "Define NFS export policy." - type = list(object({ - allowed_clients = optional(string) - has_root_access = optional(bool, false) - access_type = optional(string, "READ_WRITE") - nfsv3 = optional(bool) - nfsv4 = optional(bool) - })) - # Permissive default if user does not specify nfs_export_options. Allow all RFC1918 CIDRS with no_root_squash - default = [{ - allowed_clients = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", - has_root_access = true, - access_type = "READ_WRITE", - }] - nullable = true -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/versions.tf deleted file mode 100644 index c624d5100b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/netapp-volume/versions.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.45.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:netapp-volume/v1.70.0" - } - - required_version = ">= 1.5.7" -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/README.md b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/README.md deleted file mode 100644 index 0b942f067f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/README.md +++ /dev/null @@ -1,196 +0,0 @@ -## Description - -This module creates [parallelstore](https://cloud.google.com/parallelstore) -instance. Parallelstore is Google Cloud's first party parallel file system -service based on [Intel DAOS](https://docs.daos.io/v2.2/) - -### Supported Operating Systems - -A parallelstore instance can be used with Slurm cluster or compute -VM running Ubuntu 22.04, debian 12 or HPC Rocky Linux 8. - -### Parallelstore Quota - -To get access to a private preview of Parallelstore APIs, your project needs to -be allowlisted. To set this up, please work with your account representative. - -### Parallelstore mount options - -After parallelstore instance is created, you can specify mount options depending -upon your workload. DAOS is configured to deliver the best user experience for -interactive workloads with aggressive caching. If you are running parallel -workloads concurrently accessing the sane files from multiple client nodes, it -is recommended to disable the writeback cache to avoid cross-client consistency -issues. You can specify different mount options as follows, - -```yaml - - id: parallelstore - source: modules/file-system/parallelstore - use: [network, ps_connect] - settings: - mount_options: "disable-wb-cache,thread-count=20,eq-count=8" -``` - -### Example - New VPC - -For parallelstore instance, Below snippet creates new VPC and configures private-service-access -for this newly created network. - -```yaml - - id: network - source: modules/network/vpc - - # Private Service Access (PSA) requires the compute.networkAdmin role which is - # included in the Owner role, but not Editor. - # PSA is required for all Parallelstore functionality. - # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: parallelstore - source: modules/file-system/parallelstore - use: [network, private_service_access] -``` - -### Example - Existing VPC - -If you want to use existing network with private-service-access configured, you need -to manually provide `private_vpc_connection_peering` to the parallelstore module. -You can get this details from the Google Cloud Console UI in `VPC network peering` -section. Below is the example of using existing network and creating parallelstore. -If existing network is not configured with private-service-access, you can follow -[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) -to set it up. - -```yaml - - id: network - source: modules/network/pre-existing-vpc - settings: - network_name: // Add network name - subnetwork_name: // Add subnetwork name - - - id: parallelstore - source: modules/file-system/parallelstore - use: [network] - settings: - private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" -``` - -### Import data from GCS bucket - -You can import data from your GCS bucket to parallelstore instance. Important to -note that data may not be available to the instance immediately. This depends on -latency and size of data. Below is the example of importing data from bucket. - -```yaml - - id: parallelstore - source: modules/file-system/parallelstore - use: [network] - settings: - import_gcs_bucket_uri: gs://gcs-bucket/folder-path - import_destination_path: /gcs/import/ -``` - -Here you can replace `import_gcs_bucket_uri` with the uri of sub folder within GCS -bucket and `import_destination_path` with local directory within parallelstore -instance. - -### Additional configuration for DAOS agent and dfuse -Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - daos_agent_config: | - credential_config: - cache_expiration: 1m -``` - -Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/parallelstore - settings: - dfuse_environment: - D_LOG_FILE: /tmp/client.log - D_APPEND_PID_TO_LOG: 1 - D_LOG_MASK: debug -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13 | -| [google](#requirement\_google) | >= 6.13.0 | -| [null](#requirement\_null) | ~> 3.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.13.0 | -| [null](#provider\_null) | ~> 3.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_parallelstore_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/parallelstore_instance) | resource | -| [null_resource.hydration](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [daos\_agent\_config](#input\_daos\_agent\_config) | Additional configuration to be added to daos\_config.yml | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. | `string` | n/a | yes | -| [dfuse\_environment](#input\_dfuse\_environment) | Additional environment variables for DFuse process | `map(string)` | `{}` | no | -| [directory\_stripe](#input\_directory\_stripe) | The parallelstore stripe level for directories. | `string` | `null` | no | -| [file\_stripe](#input\_file\_stripe) | The parallelstore stripe level for files. | `string` | `null` | no | -| [import\_destination\_path](#input\_import\_destination\_path) | The name of local path to import data on parallelstore instance from GCS bucket. | `string` | `null` | no | -| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to parallelstore. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to parallel store instance. | `map(string)` | `{}` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/parallelstore"` | no | -| [mount\_options](#input\_mount\_options) | Options describing various aspects of the parallelstore instance. | `string` | `"disable-wb-cache,thread-count=16,eq-count=8"` | no | -| [name](#input\_name) | Name of parallelstore instance. | `string` | `null` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [size\_gb](#input\_size\_gb) | Storage size of the parallelstore instance in GB. | `number` | `12000` | no | -| [zone](#input\_zone) | Location for parallelstore instance. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions to monitor import-data operation from GCS bucket to parallelstore. | -| [network\_storage](#output\_network\_storage) | Describes a parallelstore instance. | - diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/main.tf b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/main.tf deleted file mode 100644 index acc2a0551e..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/main.tf +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "parallelstore", ghpc_role = "file-system" }) -} - -locals { - fs_type = "daos" - server_ip = "" - remote_mount = "" - id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - access_points = jsonencode(google_parallelstore_instance.instance.access_points) - destination_path = var.import_destination_path == null ? "/" : var.import_destination_path - - client_install_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-daos-client.sh" - "destination" = "install_daos_client.sh" - } - - mount_runner = { - "type" = "shell" - "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { - access_points = local.access_points - daos_agent_config = var.daos_agent_config - dfuse_environment = var.dfuse_environment - local_mount = var.local_mount - mount_options = join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) - }) - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - } -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_parallelstore_instance" "instance" { - project = var.project_id - instance_id = local.id - location = var.zone - capacity_gib = var.size_gb - network = var.network_id - file_stripe_level = var.file_stripe - directory_stripe_level = var.directory_stripe - - labels = local.labels - - depends_on = [var.private_vpc_connection_peering] -} - -resource "null_resource" "hydration" { - count = var.import_gcs_bucket_uri != null ? 1 : 0 - - depends_on = [resource.google_parallelstore_instance.instance] - provisioner "local-exec" { - command = "curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -d '{\"source_gcs_bucket\": {\"uri\":\"${var.import_gcs_bucket_uri}\"}, \"destination_parallelstore\": {\"path\":\"${local.destination_path}\"}}' https://parallelstore.googleapis.com/v1beta/projects/${var.project_id}/locations/${var.zone}/instances/${local.id}:importData" - } -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml deleted file mode 100644 index c0994d15bb..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - parallelstore.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf deleted file mode 100644 index f6e817ac8a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - operation_instructions = <<-EOT - Data is being imported from GCS bucket to parallelstore instance. It may - not be available immediately. - EOT -} - -output "network_storage" { - description = "Describes a parallelstore instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = var.mount_options - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } - - precondition { - condition = var.import_gcs_bucket_uri != null || var.import_destination_path == null - error_message = <<-EOD - Please specify import_gcs_bucket_uri to import data to parallelstore instance. - EOD - } -} - -output "instructions" { - description = "Instructions to monitor import-data operation from GCS bucket to parallelstore." - value = var.import_gcs_bucket_uri != null ? local.operation_instructions : "Data is not imported from GCS bucket." -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh deleted file mode 100644 index e96eadb56a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || - { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 -fi - -if [ -x /bin/daos ]; then - echo "DAOS already installed" - daos version -else - # Install the DAOS client library - # The following commands should be executed on each client vm. - ## For Rocky linux 8 / RedHat 8. - if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then - # 1) Add the Parallelstore package repository - cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF - [parallelstore-v2-6-el${OS_VERSION_MAJOR}] - name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 - baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - - ## TODO: Remove disable automatic update script after issue is fixed. - if [ -x /usr/bin/google_disable_automatic_updates ]; then - /usr/bin/google_disable_automatic_updates - fi - dnf clean all - dnf makecache - - # 2) Install daos-client - dnf install -y epel-release # needed for capstone - dnf install -y daos-client - - # 3) Upgrade libfabric - dnf upgrade -y libfabric - - # For Ubuntu 22.04 and debian 12, - elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then - # shellcheck disable=SC2034 - DEBIAN_FRONTEND=noninteractive - - # 1) Add the Parallelstore package repository - curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg - echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list - - apt-get update - - # 2) Install daos-client - apt-get install -y daos-client - - # 3) Create daos_agent.service (comes pre-installed with RedHat) - if ! getent passwd daos_agent >/dev/null 2>&1; then - useradd daos_agent - fi - cat >/etc/systemd/system/daos_agent.service <<-EOF - [Unit] - Description=DAOS Agent - StartLimitIntervalSec=60 - Wants=network-online.target - After=network-online.target - - [Service] - Type=notify - User=daos_agent - Group=daos_agent - RuntimeDirectory=daos_agent - RuntimeDirectoryMode=0755 - ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml - StandardOutput=journal - StandardError=journal - Restart=always - RestartSec=10 - LimitMEMLOCK=infinity - LimitCORE=infinity - StartLimitBurst=5 - - [Install] - WantedBy=multi-user.target - EOF - else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - fi -fi - -exit 0 diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl deleted file mode 100644 index c6f5d53660..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; } && { [[ "$${OS_VERSION_MAJOR}" = "8" ]] || [[ "$${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "$${OS_ID}" = "ubuntu" ]] && [[ "$${OS_VERSION}" = "22.04" ]]; } || - { [[ "$${OS_ID}" = "debian" ]] && [[ "$${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system $${OS_ID} $${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - -fi - -# Edit agent config -daos_config=/etc/daos/daos_agent.yml - -# rewrite $daos_config from scratch -mv $${daos_config} $${daos_config}.orig - -exclude_fabric_ifaces="" -# Get names of network interfaces not in first PCI slot -# The first PCI slot is a standard network adapter while remaining interfaces -# are typically network cards dedicated to GPU or workload communication -if [[ "$${OS_ID}" == "debian" ]] || [[ "$${OS_ID}" = "ubuntu" ]]; then - extra_interfaces=$(find /sys/class/net/ -not -name 'enp0s*' -regextype posix-extended -regex '.*/enp[0-9]+s.*' -printf '"%f"\n' | paste -s -d ',') -elif [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; then - extra_interfaces=$(find /sys/class/net/ -not -name eth0 -regextype posix-extended -regex '.*/eth[0-9]+' -printf '"%f"\n' | paste -s -d ',') -fi - -cat > $daos_config </etc/systemd/system/"$${service_name}" </global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "import_gcs_bucket_uri" { - description = "The name of the GCS bucket to import data from to parallelstore." - type = string - default = null -} - -variable "import_destination_path" { - description = "The name of local path to import data on parallelstore instance from GCS bucket." - type = string - default = null -} - -variable "file_stripe" { - description = "The parallelstore stripe level for files." - type = string - default = null - validation { - condition = var.file_stripe == null ? true : contains([ - "FILE_STRIPE_LEVEL_UNSPECIFIED", - "FILE_STRIPE_LEVEL_MIN", - "FILE_STRIPE_LEVEL_BALANCED", - "FILE_STRIPE_LEVEL_MAX", - ], var.file_stripe) - error_message = "var.file_stripe must be set to \"FILE_STRIPE_LEVEL_UNSPECIFIED\", \"FILE_STRIPE_LEVEL_MIN\", \"FILE_STRIPE_LEVEL_BALANCED\", or \"FILE_STRIPE_LEVEL_MAX\"" - } -} - -variable "directory_stripe" { - description = "The parallelstore stripe level for directories." - type = string - default = null - validation { - condition = var.directory_stripe == null ? true : contains([ - "DIRECTORY_STRIPE_LEVEL_UNSPECIFIED", - "DIRECTORY_STRIPE_LEVEL_MIN", - "DIRECTORY_STRIPE_LEVEL_BALANCED", - "DIRECTORY_STRIPE_LEVEL_MAX", - ], var.directory_stripe) - error_message = "var.directory_stripe must be set to \"DIRECTORY_STRIPE_LEVEL_UNSPECIFIED\", \"DIRECTORY_STRIPE_LEVEL_MIN\", \"DIRECTORY_STRIPE_LEVEL_BALANCED\", or \"DIRECTORY_STRIPE_LEVEL_MAX\"" - } -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/versions.tf b/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/versions.tf deleted file mode 100644 index 174b5281e4..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/parallelstore/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.13.0" - } - - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md deleted file mode 100644 index 47cf1518a1..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md +++ /dev/null @@ -1,192 +0,0 @@ -## Description - -This module defines a file-system that already exists (i.e. it does not create -a new file system) in a way that can be shared with other modules. This allows -a compute VM to mount a filesystem that is not part of the current deployment -group. - -The pre-existing network storage can be referenced in the same way as any Cluster -Toolkit supported file-system such as [filestore](../filestore/README.md). - -For more information on network storage options in the Cluster Toolkit, see -the extended [Network Storage documentation](../../../docs/network_storage.md). - -### Example - -```yaml -- id: homefs - source: modules/file-system/pre-existing-network-storage - settings: - server_ip: ## Set server IP here ## - remote_mount: nfsshare - local_mount: /home - fs_type: nfs -``` - -This creates a pre-existing-network-storage module in terraform at the -provided IP in `server_ip` of type nfs that will be mounted at `/home`. Note -that the `server_ip` must be known before deployment. - -The following is an example of using `pre-existing-network-storage` with a GCS -bucket: - -```yaml -- id: data-bucket - source: modules/file-system/pre-existing-network-storage - settings: - remote_mount: my-bucket-name - local_mount: /data - fs_type: gcsfuse - mount_options: defaults,_netdev,implicit_dirs -``` - -The `implicit_dirs` mount option allows object paths to be treated as if they -were directories. This is important when working with files that were created by -another source, but there may have performance impacts. The `_netdev` mount option -denotes that the storage device requires network access. - -The following is an example of using `pre-existing-network-storage` with the `lustre` -filesystem: - -```yaml -- id: lustrefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: lustre - server_ip: 192.168.227.11@tcp - local_mount: /scratch - remote_mount: /exacloud -``` - -Note the use of the MGS NID (Network ID) in the `server_ip` field - in -particular, note the `@tcp` suffix. - -The following is an example of using `pre-existing-network-storage` with the -`managed_lustre` filesystem: - -```yaml -- id: lustrefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: managed_lustre - server_ip: 192.168.227.11@tcp - local_mount: /scratch - remote_mount: /mg_lustre -``` - -This is similar to the `lustre` filesystem, with the exception that it connects -with a managed Lustre instance hosted by GCP. Currently only Rocky 8 and -Ubuntu 20.04 and Ubuntu 22.04 are supported. - -The following is an example of using `pre-existing-network-storage` with the `daos` -filesystem. In order to use existing `parallelstore` instance, `fs_type` needs to be -explicitly mentioned in blueprint. The `remote_mount` option refers to `access_points` -for `parallelstore` instance. - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 -``` - -Parallelstore supports additional options for its mountpoints under `parallelstore_options` setting. -Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 - parallelstore_options: - daos_agent_config: | - credential_config: - cache_expiration: 1m -``` - -Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 - parallelstore_options: - dfuse_environment: - D_LOG_FILE: /tmp/client.log - D_APPEND_PID_TO_LOG: 1 - D_LOG_MASK: debug -``` - -### Mounting - -For the `fs_type` listed below, this module will provide `client_install_runner` -and `mount_runner` outputs. These can be used to create a startup script to -mount the network storage system. - -Supported `fs_type`: - -- nfs -- lustre -- managed_lustre -- gcsfuse -- daos - -[scripts/mount.sh](./scripts/mount.sh) is used as the contents of -`mount_runner`. This script will update `/etc/fstab` and mount the network -storage. This script will fail if the specified `local_mount` is already being -used by another entry in `/etc/fstab`. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. - -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## License - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [fs\_type](#input\_fs\_type) | Type of file system to be mounted (e.g., nfs, lustre) | `string` | `"nfs"` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | -| [managed\_lustre\_options](#input\_managed\_lustre\_options) | Managed Lustre specific options:
gke\_support\_enabled (bool, default = false)
Note: gke\_support\_enabled does not work with Slurm, the Slurm image must be built with
the correct compatibility. | 
object({
    gke_support_enabled = optional(bool, false)
  })
| `{}` | no | -| [mount\_options](#input\_mount\_options) | Options describing various aspects of the file system. Consider adding setting to 'defaults,\_netdev,implicit\_dirs' when using gcsfuse. | `string` | `"defaults,_netdev"` | no | -| [parallelstore\_options](#input\_parallelstore\_options) | Parallelstore specific options | 
object({
    daos_agent_config = optional(string, "")
    dfuse_environment = optional(map(string), {})
  })
| `{}` | no | -| [remote\_mount](#input\_remote\_mount) | Remote FS name or export. This is the exported directory for nfs, fs name for lustre, and bucket name (without gs://) for gcsfuse. | `string` | n/a | yes | -| [server\_ip](#input\_server\_ip) | The device name as supplied to fs-tab, excluding remote fs-name(for nfs, that is the server IP, for lustre [:]). This can be omitted for gcsfuse. | `string` | `""` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use file system. | -| [mount\_runner](#output\_mount\_runner) | Runner that mounts the file system. | -| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | - diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf deleted file mode 100644 index 203b6dfdac..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a remote network storage to be mounted by fs-tab." - value = { - server_ip = var.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = var.mount_options - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } -} - -locals { - # Update remote mount to include a slash if the fs_type requires one to exist - remote_mount_with_slash = length(regexall("^/.*", var.remote_mount)) > 0 ? ( - var.remote_mount - ) : format("/%s", var.remote_mount) - remote_mount = contains(local.mount_vanilla_supported_fstype, local.fs_type) ? ( - local.remote_mount_with_slash - ) : var.remote_mount - - ml_gke_support_enabled = coalesce(try(var.managed_lustre_options.gke_support_enabled, false), false) - - # Collapse fs_type lustre and managed lustre for most uses, only needs to be - # different for client installation - fs_type = strcontains(var.fs_type, "lustre") ? "lustre" : var.fs_type - - # Client Install - ddn_lustre_client_install_script = templatefile( - "${path.module}/templates/ddn_exascaler_luster_client_install.tftpl", - { - server_ip = split("@", var.server_ip)[0] - remote_mount = local.remote_mount - local_mount = var.local_mount - } - ) - managed_lustre_client_install_script = file("${path.module}/scripts/install-managed-lustre-client.sh") - nfs_client_install_script = file("${path.module}/scripts/install-nfs-client.sh") - gcs_fuse_install_script = file("${path.module}/scripts/install-gcs-fuse.sh") - daos_client_install_script = file("${path.module}/scripts/install-daos-client.sh") - - install_scripts = { - "lustre" = local.ddn_lustre_client_install_script - "managed_lustre" = local.managed_lustre_client_install_script - "nfs" = local.nfs_client_install_script - "gcsfuse" = local.gcs_fuse_install_script - "daos" = local.daos_client_install_script - } - - client_install_runner = { - "type" = "shell" - "content" = lookup(local.install_scripts, var.fs_type, "echo 'skipping: client_install_runner not yet supported for ${var.fs_type}'") - "destination" = "install_filesystem_client${replace(var.local_mount, "/", "_")}.sh" - "args" = local.ml_gke_support_enabled ? "1" : "" - } - - mount_vanilla_supported_fstype = ["lustre", "nfs"] - mount_runner_vanilla = { - "type" = "shell" - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"${var.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" - "content" = ( - contains(local.mount_vanilla_supported_fstype, local.fs_type) ? - file("${path.module}/scripts/mount.sh") : - "echo 'skipping: mount_runner not yet supported for ${var.fs_type}'" - ) - } - gcsbucket = trimprefix(var.remote_mount, "gs://") - mount_runner_gcsfuse = { - "type" = "shell" - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"not-used\" \"${local.gcsbucket}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" - "content" = file("${path.module}/scripts/mount.sh") - } - - mount_runner_daos = { - "type" = "shell" - "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { - access_points = var.remote_mount - daos_agent_config = var.parallelstore_options.daos_agent_config - dfuse_environment = var.parallelstore_options.dfuse_environment - local_mount = var.local_mount - # avoid passing "--" as mount option to dfuse - mount_options = length(var.mount_options) == 0 ? "" : join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) - }) - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - } - - mount_scripts = { - "lustre" = local.mount_runner_vanilla - "nfs" = local.mount_runner_vanilla - "gcsfuse" = local.mount_runner_gcsfuse - "daos" = local.mount_runner_daos - } - - mount_runner = lookup(local.mount_scripts, local.fs_type, local.mount_runner_vanilla) -} - -output "client_install_runner" { - description = "Runner that performs client installation needed to use file system." - value = local.client_install_runner -} - -output "mount_runner" { - description = "Runner that mounts the file system." - value = local.mount_runner -} diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh deleted file mode 100644 index e96eadb56a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || - { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 -fi - -if [ -x /bin/daos ]; then - echo "DAOS already installed" - daos version -else - # Install the DAOS client library - # The following commands should be executed on each client vm. - ## For Rocky linux 8 / RedHat 8. - if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then - # 1) Add the Parallelstore package repository - cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF - [parallelstore-v2-6-el${OS_VERSION_MAJOR}] - name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 - baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - - ## TODO: Remove disable automatic update script after issue is fixed. - if [ -x /usr/bin/google_disable_automatic_updates ]; then - /usr/bin/google_disable_automatic_updates - fi - dnf clean all - dnf makecache - - # 2) Install daos-client - dnf install -y epel-release # needed for capstone - dnf install -y daos-client - - # 3) Upgrade libfabric - dnf upgrade -y libfabric - - # For Ubuntu 22.04 and debian 12, - elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then - # shellcheck disable=SC2034 - DEBIAN_FRONTEND=noninteractive - - # 1) Add the Parallelstore package repository - curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg - echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list - - apt-get update - - # 2) Install daos-client - apt-get install -y daos-client - - # 3) Create daos_agent.service (comes pre-installed with RedHat) - if ! getent passwd daos_agent >/dev/null 2>&1; then - useradd daos_agent - fi - cat >/etc/systemd/system/daos_agent.service <<-EOF - [Unit] - Description=DAOS Agent - StartLimitIntervalSec=60 - Wants=network-online.target - After=network-online.target - - [Service] - Type=notify - User=daos_agent - Group=daos_agent - RuntimeDirectory=daos_agent - RuntimeDirectoryMode=0755 - ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml - StandardOutput=journal - StandardError=journal - Restart=always - RestartSec=10 - LimitMEMLOCK=infinity - LimitCORE=infinity - StartLimitBurst=5 - - [Install] - WantedBy=multi-user.target - EOF - else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - fi -fi - -exit 0 diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh deleted file mode 100644 index f8a990260b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -if [ ! "$(which gcsfuse)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then - tee /etc/yum.repos.d/gcsfuse.repo >/dev/null <>/etc/modprobe.d/lnet.conf - fi -fi - -if grep -q lustre /proc/filesystems; then - echo "Skipping managed lustre client install as it is already supported" - exit 0 -fi - -# Get distro information -. /etc/os-release -DIST="NA" -if [[ $NAME == *"Ubuntu"* ]]; then - if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then - DIST="Ubuntu" - fi -elif [[ $NAME == *"Rocky"* ]]; then - if [[ $VERSION_ID == "8"* ]]; then - DIST="Rocky" - fi -fi - -if [[ ${DIST} == "Ubuntu" ]]; then - KEY_LOC=/etc/apt/keyrings - KEY_NAME=gcp-ar-repo.gpg - # Download new repo key - mkdir -p "${KEY_LOC}" - wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null - - # Set up apt repo - echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list - - # Install modules - apt update - apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) -elif [[ ${DIST} == "Rocky" ]]; then - # Set up yum repo - touch /etc/yum.repos.d/artifact-registry.repo - tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF - [lustre-client-rocky-8] - name=lustre-client-rocky-8 - baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - # Install modules - yum makecache - yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client -fi - -if [[ $DIST != "NA" ]]; then - # Load the new lustre client module - modprobe lustre -fi diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl b/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl deleted file mode 100644 index f5f0291e85..0000000000 --- a/deletion-test/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh - -# Copyright 2022 DataDirect Networks -# Modifications Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Prior Art: https://github.com/DDNStorage/exascaler-cloud-terraform/blob/78deadbb2c1fa7e4603cf9605b0f7d1782117954/gcp/templates/client-script.tftpl - -# install new EXAScaler Cloud clients: -# all instances must be in the same zone -# and connected to the same network and subnet -# to set up EXAScaler Cloud filesystem on a new client instance, -# run the following commands on the client with root privileges: -set -e -if [[ ! -z $(cat /proc/filesystems | grep lustre) ]]; then - echo "Skipping lustre client install as it is already supported" - exit 0 -fi - -cat >/etc/esc-client.conf< $daos_config </etc/systemd/system/"$${service_name}" < -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string
    count = number
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [guest\_accelerator](#output\_guest\_accelerator) | Sanitized list of the type and count of accelerator cards attached to the instance. | -| [machine\_type\_guest\_accelerator](#output\_machine\_type\_guest\_accelerator) | List of the type and count of accelerator cards attached to the specified machine type. | - diff --git a/deletion-test/primary/modules/embedded/modules/internal/gpu-definition/main.tf b/deletion-test/primary/modules/embedded/modules/internal/gpu-definition/main.tf deleted file mode 100644 index f0861cddc9..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/gpu-definition/main.tf +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "machine_type" { - description = "Machine type to use for the instance creation" - type = string -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string - count = number - gpu_driver_installation_config = optional(object({ - gpu_driver_version = string - }), { gpu_driver_version = "DEFAULT" }) - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = string - max_shared_clients_per_gpu = number - })) - })) - default = [] - nullable = false -} - -locals { - # example state; terraform will ignore diffs if last element of URL matches - # guest_accelerator = [ - # { - # count = 1 - # type = "https://www.googleapis.com/compute/beta/projects/PROJECT/zones/ZONE/acceleratorTypes/nvidia-tesla-a100" - # }, - # ] - accelerator_machines = { - "a2-highgpu-1g" = { type = "nvidia-tesla-a100", count = 1 }, - "a2-highgpu-2g" = { type = "nvidia-tesla-a100", count = 2 }, - "a2-highgpu-4g" = { type = "nvidia-tesla-a100", count = 4 }, - "a2-highgpu-8g" = { type = "nvidia-tesla-a100", count = 8 }, - "a2-megagpu-16g" = { type = "nvidia-tesla-a100", count = 16 }, - "a2-ultragpu-1g" = { type = "nvidia-a100-80gb", count = 1 }, - "a2-ultragpu-2g" = { type = "nvidia-a100-80gb", count = 2 }, - "a2-ultragpu-4g" = { type = "nvidia-a100-80gb", count = 4 }, - "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, - "a3-highgpu-1g" = { type = "nvidia-h100-80gb", count = 1 }, - "a3-highgpu-2g" = { type = "nvidia-h100-80gb", count = 2 }, - "a3-highgpu-4g" = { type = "nvidia-h100-80gb", count = 4 }, - "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, - "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, - "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, - "a4-highgpu-8g-lowmem" = { type = "nvidia-b200", count = 8 }, - "a4-highgpu-8g" = { type = "nvidia-b200", count = 8 }, - "a4x-highgpu-4g" = { type = "nvidia-gb200", count = 4 }, - "a4x-highgpu-4g-nolssd" = { type = "nvidia-gb200", count = 4 }, - "g2-standard-4" = { type = "nvidia-l4", count = 1 }, - "g2-standard-8" = { type = "nvidia-l4", count = 1 }, - "g2-standard-12" = { type = "nvidia-l4", count = 1 }, - "g2-standard-16" = { type = "nvidia-l4", count = 1 }, - "g2-standard-24" = { type = "nvidia-l4", count = 2 }, - "g2-standard-32" = { type = "nvidia-l4", count = 1 }, - "g2-standard-48" = { type = "nvidia-l4", count = 4 }, - "g2-standard-96" = { type = "nvidia-l4", count = 8 }, - } - generated_guest_accelerator = try([local.accelerator_machines[var.machine_type]], []) - - # Select in priority order: - # (1) var.guest_accelerator if not empty - # (2) local.generated_guest_accelerator if not empty - # (3) default to empty list if both are empty - guest_accelerator = try(coalescelist(var.guest_accelerator, local.generated_guest_accelerator), []) -} - -output "guest_accelerator" { - description = "Sanitized list of the type and count of accelerator cards attached to the instance." - value = local.guest_accelerator -} - -output "machine_type_guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the specified machine type." - value = local.generated_guest_accelerator -} - -terraform { - required_version = ">= 1.3" -} diff --git a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/README.md b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/README.md deleted file mode 100644 index 21746fe0d8..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/README.md +++ /dev/null @@ -1,30 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [disk\_type](#input\_disk\_type) | The disk type to validate. | `string` | n/a | yes | -| [machine\_type](#input\_machine\_type) | The machine type to validate. | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/main.tf b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/main.tf deleted file mode 100644 index d89d7edfec..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/main.tf +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -check "disk_type_c4_compatibility" { - assert { - condition = !(can(regex("^c4-", var.machine_type)) && var.disk_type == "pd-ssd") - error_message = "The C4 machine series does not support pd-ssd. Please use hyperdisk-balanced or another compatible disk type." - } -} - - -check "disk_type_c2_compatibility" { - assert { - condition = !(can(regex("^c2-", var.machine_type)) && can(regex("hyperdisk", var.disk_type))) - error_message = "The C2 machine series does not support Hyperdisk as a boot disk. Please use a compatible disk type like pd-ssd, pd-standard, or pd-balanced." - } -} - - -check "disk_type_pd_extreme_compatibility" { - assert { - condition = var.disk_type != "pd-extreme" || can(regex("^(m1-|m2-|m3-|n2-|n2d-)", var.machine_type)) - error_message = "pd-extreme disks are only supported for M1, M2, M3, N2, and N2D machine series." - } -} - - -check "disk_type_hyperdisk_extreme_compatibility" { - assert { - condition = var.disk_type != "hyperdisk-extreme" || can(regex("^(c3-|m1-|m3-|n2-)", var.machine_type)) - error_message = "hyperdisk-extreme disks are only supported for C3, M1, M3, and N2 machine series." - } -} - - -check "disk_type_hyperdisk_throughput_compatibility" { - assert { - condition = var.disk_type != "hyperdisk-throughput" || can(regex("^(c3-|c3d-|n4-|n2-|n2d-|n1-|t2d-|m1-)", var.machine_type)) - error_message = "hyperdisk-throughput disks are only supported for C3, C3D, N4, N2, N2D, N1, T2D, and M1 machine series." - } -} diff --git a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/variables.tf b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/variables.tf deleted file mode 100644 index 23478051b3..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/variables.tf +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "machine_type" { - type = string - description = "The machine type to validate." -} - -variable "disk_type" { - type = string - description = "The disk type to validate." -} diff --git a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/versions.tf b/deletion-test/primary/modules/embedded/modules/internal/instance_validations/versions.tf deleted file mode 100644 index 4702005614..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/instance_validations/versions.tf +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 0.15.0" -} diff --git a/deletion-test/primary/modules/embedded/modules/internal/network-attachment/README.md b/deletion-test/primary/modules/embedded/modules/internal/network-attachment/README.md deleted file mode 100644 index 8aa9270a0a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/network-attachment/README.md +++ /dev/null @@ -1,54 +0,0 @@ - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | -| [google-beta](#requirement\_google-beta) | >= 6.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google-beta](#provider\_google-beta) | >= 6.0.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_network_attachment.self](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_network_attachment) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [connection\_preference](#input\_connection\_preference) | The connection preference of service attachment. | `string` | `"ACCEPT_AUTOMATIC"` | no | -| [name](#input\_name) | Name of the resource. Provided by the client when the resource is created | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The ID of the project in which the resource belongs. | `string` | n/a | yes | -| [region](#input\_region) | Region where the network attachment resides | `string` | n/a | yes | -| [subnetwork\_self\_links](#input\_subnetwork\_self\_links) | An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment. | `list(string)` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [self\_link](#output\_self\_link) | Server-defined URL for the resource. | - diff --git a/deletion-test/primary/modules/embedded/modules/internal/network-attachment/main.tf b/deletion-test/primary/modules/embedded/modules/internal/network-attachment/main.tf deleted file mode 100644 index bbbece7085..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/network-attachment/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -variable "connection_preference" { - type = string - description = "The connection preference of service attachment." - default = "ACCEPT_AUTOMATIC" -} - -variable "subnetwork_self_links" { - type = list(string) - description = " An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment." -} - -variable "name" { - type = string - description = "Name of the resource. Provided by the client when the resource is created" -} - -variable "project_id" { - type = string - description = "The ID of the project in which the resource belongs." -} - -variable "region" { - type = string - description = "Region where the network attachment resides" -} - - -resource "google_compute_network_attachment" "self" { - provider = google-beta - - project = var.project_id - region = var.region - name = var.name - connection_preference = var.connection_preference - subnetworks = var.subnetwork_self_links -} - - -output "self_link" { - value = google_compute_network_attachment.self.self_link - description = "Server-defined URL for the resource." -} - -terraform { - required_version = ">= 0.15.0" - - required_providers { - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.0.0" - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml b/deletion-test/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml deleted file mode 100644 index e80fc96b9c..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/README.md b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/README.md deleted file mode 100644 index 610d82c1b9..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/README.md +++ /dev/null @@ -1,85 +0,0 @@ -## Description - -This is an internal helper module designed to encapsulate and centralize all hardware-specific logic for Google Cloud TPUs. It is intended to be called by parent modules like `gke-node-pool` to determine if a node pool is TPU-based and to retrieve its specific attributes. - -This module's primary responsibilities are: - -* Reliably detect if a node pool is for TPUs by checking its `placement_policy`. -* Determine the correct GKE `tpu-accelerator` label based on the machine type family. -* Determine the `number of chips per node` based on the specific machine type. -* Generate the standard **Kubernetes taint** that should be applied to TPU nodes. - -This follows the same design pattern as the `gpu-definition` internal module, promoting a clean separation of concerns within the gke-node-pool module. - -## Usage - -This module is not intended for direct use in a blueprint. It should be called from a parent module like `gke-node-pool`. - -```yaml -module "tpu" { - source = "../../internal/tpu-definition" - - # Pass the parent module's variables to this module - machine_type = var.machine_type - placement_policy = var.placement_policy -} - -# Example of consuming the module's outputs in the parent module -locals { - # The tpu_taint is then used in the node_config's dynamic "taint" block - tpu_taint = module.tpu.tpu_taint -} -``` - -## License - - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [machine\_type](#input\_machine\_type) | The machine type of the node pool. | `string` | n/a | yes | -| [placement\_policy](#input\_placement\_policy) | The placement policy for the node pool. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [is\_tpu](#output\_is\_tpu) | Boolean value indicating if the node pool is for TPUs. | -| [tpu\_accelerator\_type](#output\_tpu\_accelerator\_type) | The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice'). | -| [tpu\_chips\_per\_node](#output\_tpu\_chips\_per\_node) | The number of TPU chips on each node in the pool. | -| [tpu\_taint](#output\_tpu\_taint) | A list containing the standard TPU taint object if the node pool is for TPUs. | -| [tpu\_topology](#output\_tpu\_topology) | The topology of the TPU slice (e.g., '4x4'). | - diff --git a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/main.tf b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/main.tf deleted file mode 100644 index c8ee417d71..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/main.tf +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # Determine if this is a TPU node pool by checking if the machine_type exists in our authoritative map of TPU machine types. - is_tpu = contains(keys(local.tpu_chip_count_map), var.machine_type) - - tpu_taint = local.is_tpu ? [{ - key = "google.com/tpu" - value = "present" - effect = "NO_SCHEDULE" - }] : [] - - # Map of machine prefixes to GKE accelerator labels. - tpu_accelerator_map = { - "ct4p" = "tpu-v4-podslice" # TPU v4 - "ct5lp" = "tpu-v5-lite-podslice" # TPU v5e - "ct5p" = "tpu-v5p-slice" # TPU v5p - "ct6e" = "tpu-v6e-slice" # TPU v6e - "tpu7x" = "tpu7x" # TPU v7x - } - - # Map specific GCE machine types to the number of TPU chips per node (VM). - # The machine-type map must be updated to reflect new TPU releases with reference to public documentation: https://docs.cloud.google.com/tpu/docs/intro-to-tpu - tpu_chip_count_map = { - # v4 - ct4p - "ct4p-hightpu-4t" = 4 - - # v5e - ct5lp - "ct5lp-hightpu-1t" = 1 - "ct5lp-hightpu-4t" = 4 - "ct5lp-hightpu-8t" = 8 - - # v5p - ct5p - "ct5p-hightpu-1t" = 1 - "ct5p-hightpu-2t" = 2 - "ct5p-hightpu-4t" = 4 - - # v6e - ct6e - "ct6e-standard-1t" = 1 - "ct6e-standard-4t" = 4 - "ct6e-standard-8t" = 8 - - # v7x - tpu7x - "tpu7x-standard-4t" = 4 - } - - # Robustly extract the machine family prefix (e.g., "ct6e"). - tpu_machine_family = local.is_tpu ? element(split("-", var.machine_type), 0) : "" - tpu_accelerator_type = local.is_tpu ? lookup(local.tpu_accelerator_map, local.tpu_machine_family, null) : null - tpu_chips_per_node = local.is_tpu ? lookup(local.tpu_chip_count_map, var.machine_type, null) : null -} - -terraform { - required_version = ">= 1.3" -} diff --git a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/outputs.tf b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/outputs.tf deleted file mode 100644 index fa3c21fa34..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/outputs.tf +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "is_tpu" { - description = "Boolean value indicating if the node pool is for TPUs." - value = local.is_tpu -} - -output "tpu_accelerator_type" { - description = "The label value for the TPU accelerator type (e.g., 'tpu-v6e-slice')." - value = local.tpu_accelerator_type -} - -output "tpu_topology" { - description = "The topology of the TPU slice (e.g., '4x4')." - value = local.is_tpu ? var.placement_policy.tpu_topology : null -} - -output "tpu_chips_per_node" { - description = "The number of TPU chips on each node in the pool." - value = local.tpu_chips_per_node -} - -output "tpu_taint" { - description = "A list containing the standard TPU taint object if the node pool is for TPUs." - value = local.tpu_taint -} diff --git a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/variables.tf b/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/variables.tf deleted file mode 100644 index 254488c02d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/tpu-definition/variables.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "machine_type" { - description = "The machine type of the node pool." - type = string -} - -variable "placement_policy" { - description = "The placement policy for the node pool." - type = object({ - type = string - name = optional(string) - tpu_topology = optional(string) - }) -} diff --git a/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/README.md b/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/README.md deleted file mode 100644 index aefac9d187..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/README.md +++ /dev/null @@ -1,56 +0,0 @@ - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_network_peering.peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [export\_custom\_routes](#input\_export\_custom\_routes) | (Optional) Whether to export the custom routes to the peer network. Defaults to false. | `bool` | `null` | no | -| [import\_custom\_routes](#input\_import\_custom\_routes) | (Optional) Whether to import the custom routes from the peer network. Defaults to false. | `bool` | `null` | no | -| [import\_subnet\_routes\_with\_public\_ip](#input\_import\_subnet\_routes\_with\_public\_ip) | (Optional) Whether subnet routes with public IP range are imported. | `bool` | `null` | no | -| [name](#input\_name) | Name of the peering. | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | The primary network of the peering. | `string` | n/a | yes | -| [peer\_network\_self\_link](#input\_peer\_network\_self\_link) | The peer network in the peering. The peer network may belong to a different project. | `string` | n/a | yes | -| [stack\_type](#input\_stack\_type) | (Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [peering\_name](#output\_peering\_name) | Name of the peering. | - diff --git a/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/main.tf b/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/main.tf deleted file mode 100644 index 386fa9377b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/main.tf +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "name" { - type = string - description = "Name of the peering." -} - -variable "network_self_link" { - type = string - description = "The primary network of the peering." -} - -variable "peer_network_self_link" { - type = string - description = "The peer network in the peering. The peer network may belong to a different project." -} - -variable "export_custom_routes" { - type = bool - description = "(Optional) Whether to export the custom routes to the peer network. Defaults to false." - default = null -} - -variable "import_custom_routes" { - type = bool - description = "(Optional) Whether to import the custom routes from the peer network. Defaults to false." - default = null -} - -variable "import_subnet_routes_with_public_ip" { - type = bool - description = "(Optional) Whether subnet routes with public IP range are imported. " - default = null -} - -variable "stack_type" { - type = string - description = "(Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. " - default = null -} - -resource "google_compute_network_peering" "peering" { - name = var.name - network = var.network_self_link - peer_network = var.peer_network_self_link - export_custom_routes = var.export_custom_routes - import_custom_routes = var.import_custom_routes - import_subnet_routes_with_public_ip = var.import_subnet_routes_with_public_ip - stack_type = var.stack_type -} - -output "peering_name" { - value = google_compute_network_peering.peering.name - description = "Name of the peering." -} - -terraform { - required_version = ">= 0.15.0" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml b/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml deleted file mode 100644 index e80fc96b9c..0000000000 --- a/deletion-test/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/README.md b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/README.md deleted file mode 100644 index d7054eb725..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/README.md +++ /dev/null @@ -1,244 +0,0 @@ -## Description - -This module simplifies the following functionality: - -* Applying Kubernetes manifests to GKE clusters: It provides flexible options for specifying manifests, allowing you to either directly embed them as strings content or reference them from URLs, files, templates, or entire .yaml and .tftpl files in directories. -* Deploying commonly used infrastructure like [Kueue](https://kueue.sigs.k8s.io/docs/) or [Jobset](https://jobset.sigs.k8s.io/docs/). - -> Note: Kueue can work with a variety of frameworks out of the box, find them [here](https://kueue.sigs.k8s.io/docs/tasks/run/) - -### Explanation - -* **Manifest:** - * **Raw String:** Specify manifests directly within the module configuration using the `content: manifest_body` format. - * **File/Template/Directory Reference:** Set `source` to the path to: - * A single URL to a manifest file. Ex.: `https://github.com/.../myrepo/manifest.yaml`. - - > **Note:** Applying from a URL has important limitations. Please review the [Considerations & Callouts for Applying from URLs](#applying-manifests-from-urls-considerations--callouts) section below. - * A single local YAML manifest file (`.yaml`). Ex.: `./manifest.yaml`. - * A template file (`.tftpl`) to generate a manifest. Ex.: `./template.yaml.tftpl`. You can pass the variables to format the template file in `template_vars`. - * A directory containing multiple YAML or template files. Ex: `./manifests/`. You can pass the variables to format the template files in `template_vars`. - -#### Manifest Example - -```yaml -- id: existing-gke-cluster - source: modules/scheduler/pre-existing-gke-cluster - settings: - project_id: $(vars.project_id) - cluster_name: my-gke-cluster - region: us-central1 - -- id: kubectl-apply - source: modules/management/kubectl-apply - use: [existing-gke-cluster] - settings: - - content: | - apiVersion: v1 - kind: Namespace - metadata: - name: my-namespace - - source: "https://github.com/kubernetes-sigs/jobset/releases/download/v0.6.0/manifests.yaml" - - source: $(ghpc_stage("manifests/configmap1.yaml")) - - source: $(ghpc_stage("manifests/configmap2.yaml.tftpl")) - template_vars: {name: "dev-config", public: "false"} - - source: $(ghpc_stage("manifests"))/ - template_vars: {name: "dev-config", public: "false"} -``` - -#### Pre-build infrastructure Example - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml")) - jobset: - install: true -``` - -The `config_path` field in `kueue` installation accepts a template file, too. You will need to provide variables for the template using `config_template_vars` field. - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) - config_template_vars: {name: "dev-config", public: "false"} - jobset: - install: true -``` - -You can specify a particular kueue version that you would like to use using the `version` flag. By default, we recommend customers to [use v0.10.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L68). You can find the list of supported kueue versions [here](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L18). - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - version: v0.10.0 - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) - config_template_vars: {name: "dev-config", public: "false"} - jobset: - install: true -``` - -> **_NOTE:_** -> -> The `project_id` and `region` settings would be inferred from the deployment variables of the same name, but they are included here for clarity. -> -> Terraform may apply resources in parallel, leading to potential dependency issues. If a resource's dependencies aren't ready, it will be applied again up to 15 times. - -## Callouts - -### Applying Manifests from URLs: Considerations & Callouts - -While this module supports applying manifests directly from remote `http://` or `https://` URLs, this method introduces complexities not present when using local files. For production environments, we recommend sourcing manifests from local paths or a version-controlled Git repository. Moreover, this method will be deprecated soon. Hence we recommend to use other methods to source manifests. - -If you choose to use the URL method, be aware of the following potential issues and their solutions. - -#### **1. Apply Order and Race Conditions** - -The module applies manifests from the `apply_manifests` list in parallel. This can create a **race condition** if one manifest depends on another. The most common example is applying a manifest with custom resources (like a `ClusterQueue`) at the same time as the manifest that defines it (the `CustomResourceDefinition` or CRD). - -There is **no guarantee** that the CRD will be applied before the resource that uses it. This can lead to non-deterministic deployment failures with errors like: - -```Error: resource [kueue.x-k8s.io/v1beta1/ClusterQueue] isn't valid for cluster``` - -##### **Recommended Workaround: Two-Stage Apply** - -To ensure a reliable deployment, you must manually enforce the correct order of operations. - -1. **Initial Deployment:** In your blueprint, include **only** the manifest(s) containing the `CustomResourceDefinition` (CRD) resources in the `apply_manifests` list. - - *Example `settings` for the first run:* - - ```yaml - settings: - apply_manifests: - # This manifest contains the CRDs for Kueue - - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" - server_side_apply: true - ``` - -2. **Run the deployment** (`gcluster deploy` or `terraform apply`). - -3. **Second Deployment:** Once the first apply is successful, **add** the manifests containing your custom resources (like `ClusterQueue`, `LocalQueue`) to the list. - - *Example `settings` for the second run:* - - ```yaml - settings: - apply_manifests: - # The CRD manifest is still present - - source: "https://raw.githubusercontent.com/GoogleCloudPlatform/cluster-toolkit/refs/heads/develop/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml" - server_side_apply: true - - # Now, add your configuration manifest - - source: "https://gist.githubusercontent.com/YourUser/..." # Your configuration URL - server_side_apply: true - ``` - -4. **Run the deployment command again.** Since the CRDs are now guaranteed to exist in the cluster, this second apply will succeed reliably. - -#### **2. Large Manifests (CRDs)** - -* **Issue:** Applying very large manifests can fail with a `metadata.annotations: Too long` error. -* **Solution:** Enable Server-Side Apply by setting `server_side_apply: true` for the manifest entry. - -#### **3. Conflicts on Re-application** - -* **Issue:** Re-running a deployment after a partial failure can cause server-side apply field manager `conflicts`. -* **Solution:** Forcibly take ownership of the resource fields by setting `force_conflicts: true`. - -#### **4. Terraform Template Files (`.tftpl`)** - -* **Limitation:** This module **cannot** render a template file (`.tftpl`) when sourced from a remote URL. -* **Workaround:** You must render the template into a pure YAML file locally, host that rendered file at a URL, and provide the URL of the rendered file in your blueprint. - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 7.2 | -| [helm](#requirement\_helm) | ~> 2.17 | -| [http](#requirement\_http) | ~> 3.0 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 7.2 | -| [http](#provider\_http) | ~> 3.0 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [configure\_kueue](#module\_configure\_kueue) | ./kubectl | n/a | -| [install\_gib](#module\_install\_gib) | ./kubectl | n/a | -| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | -| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | -| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | -| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | -| [kubectl\_apply\_manifests](#module\_kubectl\_apply\_manifests) | ./kubectl | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.gib_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.initial_gib_version](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.jobset_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.kueue_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | -| [http_http.manifest_from_url](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [apply\_manifests](#input\_apply\_manifests) | A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md).
NOTE: The `enable` input acts as a FF to apply a manifest or not. By default it is always set to `true`. | 
list(object({
    enable            = optional(bool, true)
    content           = optional(string, null)
    source            = optional(string, null)
    template_vars     = optional(map(any), null)
    server_side_apply = optional(bool, false)
    wait_for_rollout  = optional(bool, true)
  }))
| `[]` | no | -| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | -| [gib](#input\_gib) | Install the NCCL gIB plugin | 
object({
    install = bool
    path    = string
    template_vars = object({
      image   = optional(string, "us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib")
      version = string
      node_affinity = optional(any, {
        requiredDuringSchedulingIgnoredDuringExecution = {
          nodeSelectorTerms = [{
            matchExpressions = [{
              key      = "cloud.google.com/gke-gpu",
              operator = "In",
              values   = ["true"]
            }]
          }]
        }
      })
      accelerator_count = number
      max_unavailable   = optional(string, "50%")
    })
  })
| 
{
  "install": false,
  "path": "",
  "template_vars": {
    "accelerator_count": 0,
    "version": ""
  }
}
| no | -| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | -| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "0.10.1")
  })
| `{}` | no | -| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "0.13.3")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | -| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | -| [target\_architecture](#input\_target\_architecture) | The target architecture for the GKE nodes and gIB plugin (e.g., 'x86\_64' or 'arm64'). | `string` | `"x86_64"` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md deleted file mode 100644 index 1957899617..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [helm](#provider\_helm) | ~> 2.17 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | -| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | -| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | -| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | -| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | -| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | -| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | -| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | -| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | -| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | -| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | -| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | -| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | -| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | -| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | -| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | -| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | -| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | -| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | -| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | -| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | -| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | -| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | -| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | -| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | -| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | -| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | -| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | -| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | -| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf deleted file mode 100644 index 8cc09bd3e2..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "helm_release" "apply_chart" { - # Required Identification - name = var.release_name - chart = var.chart_name - - # Chart Source & Version - repository = var.chart_repository - version = var.chart_version - devel = var.devel - - # Target Namespace - namespace = var.namespace - create_namespace = var.create_namespace - - # Values Configuration - values = var.values_yaml - - dynamic "set" { - for_each = var.set_values - content { - name = set.value.name - value = set.value.value - type = set.value.type - } - } - - # Installation/Upgrade Behavior - description = var.description - atomic = var.atomic - cleanup_on_fail = var.cleanup_on_fail - dependency_update = var.dependency_update - disable_crd_hooks = var.disable_crd_hooks - disable_openapi_validation = var.disable_openapi_validation - disable_webhooks = var.disable_webhooks - force_update = var.force_update - lint = var.lint - max_history = var.max_history - recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI - render_subchart_notes = var.render_subchart_notes - reset_values = var.reset_values - reuse_values = var.reuse_values - skip_crds = var.skip_crds - timeout = var.timeout - wait = var.wait - wait_for_jobs = var.wait_for_jobs - - # Verification & Credentials - keyring = var.keyring - pass_credentials = var.pass_credentials - verify = var.verify - - # Post Rendering - dynamic "postrender" { - # Only include the block if var.postrender is not null - for_each = var.postrender == null ? [] : [var.postrender] - content { - binary_path = postrender.value.binary_path - } - } - - # Lifecycle block (optional - generally avoid complex lifecycle in generic modules) - # lifecycle { - # ignore_changes = [] - # } -} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf deleted file mode 100644 index 04e8e214fc..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -# --- Required --- -variable "release_name" { - description = "Name of the Helm release." - type = string -} - -variable "chart_name" { - description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." - type = string -} - -# --- Chart Location & Version --- -variable "chart_repository" { - description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." - type = string - default = null -} - -variable "chart_version" { - description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." - type = string - default = null -} - -variable "devel" { - description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." - type = bool - default = false -} - -# --- Namespace --- -variable "namespace" { - description = "Kubernetes namespace to install the Helm release into." - type = string - default = "default" -} - -variable "create_namespace" { - description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." - type = bool - default = true # Common convenience setting -} - -# --- Values Customization --- -variable "values_yaml" { - description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." - type = list(string) - default = [] -} - -variable "set_values" { - description = "List of objects defining values to set ('helm install --set')." - type = list(object({ - name = string # Path to the value (e.g., 'service.type', 'replicaCount') - value = string # The value to set - type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') - })) - default = [] -} - -# --- Installation/Upgrade Behavior --- -variable "description" { - description = "Set an optional description for the Helm release." - type = string - default = null -} - -variable "atomic" { - description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." - type = bool - default = false -} - -variable "wait" { - description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." - type = bool - default = true # Often a good default for dependencies -} - -variable "wait_for_jobs" { - description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." - type = bool - default = false # Helm CLI default is false -} - -variable "timeout" { - description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." - type = number - default = 300 # 5 minutes (Helm CLI default) -} - -variable "cleanup_on_fail" { - description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." - type = bool - default = false -} - -variable "dependency_update" { - description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." - type = bool - default = false -} - -variable "disable_crd_hooks" { - description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." - type = bool - default = false -} - -variable "disable_openapi_validation" { - description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." - type = bool - default = false -} - -variable "disable_webhooks" { - description = "Prevent hooks from running ('helm install --no-hooks')." - type = bool - default = false -} - -variable "force_update" { - description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." - type = bool - default = false -} - -variable "lint" { - description = "Run the helm chart linter during the plan ('helm lint')." - type = bool - default = false -} - -variable "max_history" { - description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." - type = number - default = null # Terraform provider defaults to Helm's default (usually 10) -} - -variable "recreate_pods" { - description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." - type = bool - default = false -} - -variable "render_subchart_notes" { - description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." - type = bool - default = false -} - -variable "reset_values" { - description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." - type = bool - default = false -} - -variable "reuse_values" { - description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." - type = bool - default = false # Helm CLI default is false -} - -variable "skip_crds" { - description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." - type = bool - default = false -} - -# --- Verification & Credentials --- -variable "keyring" { - description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." - type = string - default = null # Defaults to Helm's default keyring location -} - -variable "pass_credentials" { - description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." - type = bool - default = false -} - -variable "verify" { - description = "Verify the package before installing it ('helm install --verify')." - type = bool - default = false -} - -# --- Advanced Rendering --- -variable "postrender" { - description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." - type = object({ - binary_path = string # Path to the post-renderer executable - }) - default = null # Disabled by default -} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf deleted file mode 100644 index 09d912e2c9..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml deleted file mode 100644 index 92fc1bca22..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/jobset/jobset-helm-values.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# For referencing the original jobset helm chart values, pull the latest jobset chart version -# `helm pull oci://registry.k8s.io/jobset/charts/jobset --version=0.10.1` (latest helm chart version) - -controller: - # It ensures the Jobset pod(s) can be scheduled on GKE clusters where the - # system node pool uses the default "gke-managed-components" taint. - tolerations: - - key: "components.gke.io/gke-managed-components" - operator: "Equal" - value: "true" - effect: "NoSchedule" diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md deleted file mode 100644 index 691f4dc34a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md +++ /dev/null @@ -1,55 +0,0 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | - -## Providers - -| Name | Version | -|------|---------| -| [kubectl](#provider\_kubectl) | >= 1.7.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubectl_manifest.apply_doc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_path_documents.templates](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | -| [kubectl_path_documents.yamls](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | -| [force\_conflicts](#input\_force\_conflicts) | The force\_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller) | `bool` | `false` | no | -| [server\_side\_apply](#input\_server\_side\_apply) | Allow using kubectl server-side apply method. | `bool` | `false` | no | -| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `null` | no | -| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | -| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | - -## Outputs - -No outputs. - diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf deleted file mode 100644 index acf1d3c908..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - yaml_separator = "\n---" - - # This locals block processes manifest inputs from one of four methods, - # evaluated in order of precedence using coalesce. - - # --- METHOD 1: Direct Content Input --- - # Used when manifest content is passed directly as a string. - content_yaml_body = var.content - - # Fallback for safe path checking in subsequent methods. - null_safe_source = coalesce(var.source_path, " ") - - # --- METHOD 2: Single Local YAML File --- - # Used when var.source_path points to a local .yaml file. - yaml_file = length(regexall("\\.yaml(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null - yaml_file_content = local.yaml_file != null ? file(local.yaml_file) : null - - # --- METHOD 3: Single Local Template File --- - # Used when var.source_path points to a local .tftpl file. - template_file = length(regexall("\\.tftpl(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null - template_file_content = local.template_file != null ? templatefile(local.template_file, var.template_vars) : null - - # --- CONSOLIDATE & PROCESS --- - # Coalesce finds the first non-null content from the methods above. - yaml_body = coalesce(local.content_yaml_body, local.yaml_file_content, local.template_file_content, " ") - # Ensure only valid YAML is processed - # It explicitly tests if the content can be decoded before including it. - yaml_body_docs = compact(flatten([ - for doc in split(local.yaml_separator, local.yaml_body) : [ - for content in [trimspace(doc)] : ( - # Use a temporary local variable and can() to test for successful YAML decoding. - # This handles malformed documents (like comment blocks) which cause yamldecode() to fail. - can(yamldecode(content)) && length(yamldecode(content)) > 0 ? content : null - ) - ] - ])) - - # --- METHOD 4: Directory of Files --- - # If no content was found via the methods above AND the source path looks like a directory, - # we assume this is the desired method. The data blocks below will handle it. - directory = length(local.yaml_body_docs) == 0 && endswith(local.null_safe_source, "/") ? abspath(var.source_path) : null - - # --- FINAL AGGREGATION --- - # Combine documents from single-source methods and directory-scan methods into one list. - docs_list = concat(try(local.yaml_body_docs, []), try(data.kubectl_path_documents.yamls[0].documents, []), try(data.kubectl_path_documents.templates[0].documents, [])) - docs_map = tomap({ - for index, doc in local.docs_list : index => doc - }) -} - -data "kubectl_path_documents" "yamls" { - count = local.directory != null ? 1 : 0 - pattern = "${local.directory}/*.yaml" -} - -data "kubectl_path_documents" "templates" { - count = local.directory != null ? 1 : 0 - pattern = "${local.directory}/*.tftpl" - vars = var.template_vars -} - -resource "kubectl_manifest" "apply_doc" { - for_each = local.docs_map - yaml_body = each.value - server_side_apply = var.server_side_apply - wait_for_rollout = var.wait_for_rollout - force_conflicts = var.force_conflicts - - lifecycle { - precondition { - condition = !var.force_conflicts || var.server_side_apply - error_message = "The 'force_conflicts' variable can only be set to true when 'server_side_apply' is also true." - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf deleted file mode 100644 index 7bf34e089c..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "content" { - description = "The YAML body to apply to gke cluster." - type = string - default = null -} - -variable "source_path" { - description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." - type = string - default = null -} - -variable "template_vars" { - description = "The values to populate template file(s) with." - type = any - default = null -} - -variable "server_side_apply" { - description = "Allow using kubectl server-side apply method." - type = bool - default = false -} - -variable "wait_for_rollout" { - description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." - type = bool - default = true -} - -variable "force_conflicts" { - description = "The force_conflicts boolean, when true, compels kubectl apply (in server-side apply mode) to forcefully take ownership and override any resource fields managed by a different entity. For more information, see [Using Server-Side Apply in a controller](https://kubernetes.io/docs/reference/using-api/server-side-apply/#using-server-side-apply-in-a-controller)" - type = bool - default = false -} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf deleted file mode 100644 index cce452239f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.7.0" - } - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml deleted file mode 100644 index 7c0bef7013..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/kueue/kueue-helm-values.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# For referencing the original Kueue helm chart values, pull the latest helm chart version -# `helm pull oci://registry.k8s.io/kueue/charts/kueue --version=0.13.3` (latest helm chart version) - -controllerManager: - # -- Enables the Topology-Aware Scheduling feature gate. - featureGates: - - name: TopologyAwareScheduling - enabled: true - - # It ensures the Kueue pod can schedule on GKE clusters where the - # system node pool uses the default "gke-managed-components" taint. - tolerations: - - key: "components.gke.io/gke-managed-components" - operator: "Equal" - value: "true" - effect: "NoSchedule" diff --git a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/main.tf b/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/main.tf deleted file mode 100644 index 73a15ad1ab..0000000000 --- a/deletion-test/primary/modules/embedded/modules/management/kubectl-apply/main.tf +++ /dev/null @@ -1,271 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] - project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] - - # 1. First, Identify manifests that are explicitly enabled. - enabled_manifests = { - for index, manifest in var.apply_manifests : index => manifest - if try(manifest.enable, true) - } - - # 2. Identify URL-based manifests - url_manifests = { - for index, manifest in local.enabled_manifests : index => manifest - if try(manifest.source, null) != null && (startswith(manifest.source, "http://") || startswith(manifest.source, "https://")) - } - - # 3. Rebuild the map by populating the 'content' field for URLs based manifest - processed_apply_manifests_map = tomap({ - for index, manifest in local.enabled_manifests : tostring(index) => { - # If this manifest was a URL, its content is the body from the HTTP call. - content = contains(keys(local.url_manifests), tostring(index)) ? data.http.manifest_from_url[tostring(index)].body : manifest.content - - # If this was a URL, its source path is now null. Otherwise, use original. - source = contains(keys(local.url_manifests), tostring(index)) ? null : manifest.source - - # Pass other vars - template_vars = manifest.template_vars - server_side_apply = manifest.server_side_apply - wait_for_rollout = manifest.wait_for_rollout - } - }) - - install_kueue = try(var.kueue.install, false) - install_jobset = try(var.jobset.install, false) - install_gpu_operator = try(var.gpu_operator.install, false) - install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) - install_gib = try(var.gib.install, false) -} - -data "http" "manifest_from_url" { - for_each = local.url_manifests - url = each.value.source -} - -data "google_container_cluster" "gke_cluster" { - project = local.project_id - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -module "kubectl_apply_manifests" { - for_each = local.processed_apply_manifests_map - source = "./kubectl" - depends_on = [var.gke_cluster_exists] - - content = each.value.content - source_path = each.value.source - template_vars = each.value.template_vars - server_side_apply = each.value.server_side_apply - wait_for_rollout = each.value.wait_for_rollout - - providers = { - kubectl = kubectl - } -} - -module "install_kueue" { - source = "./helm_install" - count = local.install_kueue ? 1 : 0 - wait = false - timeout = 1200 - release_name = "kueue" - chart_repository = "oci://registry.k8s.io/kueue/charts" - chart_name = "kueue" - chart_version = var.kueue.version - namespace = "kueue-system" - create_namespace = true - values_yaml = [ - file("${path.module}/kueue/kueue-helm-values.yaml") - ] - - depends_on = [var.gke_cluster_exists] -} - -module "configure_kueue" { - source = "./kubectl" - source_path = local.install_kueue ? try(var.kueue.config_path, "") : null - template_vars = local.install_kueue ? try(var.kueue.config_template_vars, null) : null - depends_on = [module.install_kueue] - - server_side_apply = true - wait_for_rollout = true - - providers = { - kubectl = kubectl - } -} - -module "install_jobset" { - source = "./helm_install" - count = local.install_jobset ? 1 : 0 - wait = false - timeout = 1200 - release_name = "jobset" - chart_repository = "oci://registry.k8s.io/jobset/charts" - chart_name = "jobset" - chart_version = var.jobset.version - namespace = "jobset-system" - create_namespace = true - values_yaml = [ - file("${path.module}/jobset/jobset-helm-values.yaml") - ] - depends_on = [var.gke_cluster_exists, module.configure_kueue] -} - -module "install_nvidia_dra_driver" { - count = local.install_nvidia_dra_driver ? 1 : 0 - depends_on = [module.kubectl_apply_manifests, var.gke_cluster_exists, module.configure_kueue] - source = "./helm_install" - - release_name = "nvidia-dra-driver-gpu" # The release name - chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts - chart_name = "nvidia-dra-driver-gpu" # The chart name - chart_version = var.nvidia_dra_driver.version # The chart version - namespace = "nvidia-dra-driver-gpu" # The target namespace - create_namespace = true # Equivalent to --create-namespace - - # Use the 'values' argument to pass the YAML content - # This corresponds to the -f <(cat < -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_monitoring_dashboard.dashboard](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_dashboard) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [base\_dashboard](#input\_base\_dashboard) | Baseline dashboard template, select from HPC or Empty | `string` | `"HPC"` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the monitoring dashboard instance. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [title](#input\_title) | Title of the created dashboard | `string` | `"Cluster Toolkit Dashboard"` | no | -| [widgets](#input\_widgets) | List of additional widgets to add to the base dashboard. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for accessing the monitoring dashboard | - diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl deleted file mode 100644 index f25cbbd2c6..0000000000 --- a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl +++ /dev/null @@ -1,17 +0,0 @@ -{ - "displayName": "${title}: ${deployment_name}", - "gridLayout": { - "columns": 2, - "widgets": [ - { - "text": { - "content": "Metrics from the ${deployment_name} deployment of the Cluster Toolkit.", - "format": "MARKDOWN" - }, - "title": "${title}" - }%{ for widget in widgets ~}, - ${widget} - %{endfor ~} - ] - } -} diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl deleted file mode 100644 index 5b20435a9a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl +++ /dev/null @@ -1,595 +0,0 @@ -{ - "displayName": "${title}: ${deployment_name}", - "labels": ${jsonencode(labels)}, - "gridLayout": { - "columns": 2, - "widgets": [ - { - "text": { - "content": "HPC metrics from the ${deployment_name} deployment of the Cluster Toolkit.", - "format": "MARKDOWN" - }, - "title": "${title}" - }, - { - "title": "VM Instance - Memory utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/memory/percent_used\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - CPU Utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", - "pickTimeSeriesFilter": { - "direction": "TOP", - "numTimeSeries": 20, - "rankingMethod": "METHOD_MEAN" - } - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - CPU utilization (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - }, - "unitOverride": "%" - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk read operations", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/read_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk write operations", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/write_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk Read Bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/disk/read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk Write Bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/disk/write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Throttled read bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Throttled write bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Received packets", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/received_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - Sent packets", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - Received bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/received_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Sent bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_MEAN", - "groupByFields": [ - "metric.label.\"instance_name\"", - "metric.label.\"loadbalanced\"", - "resource.label.\"project_id\"", - "resource.label.\"instance_id\"", - "resource.label.\"zone\"" - ], - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - } - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Network Traffic Bytes (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/interface/traffic\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Network Packets (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/interface/packets\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "TCP connections", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/network/tcp_connections\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - }, - "unitOverride": "1" - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - CPU utilization for steal", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "STACKED_BAR", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MAX" - }, - "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\" metric.label.\"cpu_state\"=\"steal\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - CPU utilization [MEAN]", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }%{ for widget in widgets ~}, - ${widget} - %{endfor ~} - ] - } -} diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/main.tf b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/main.tf deleted file mode 100644 index df3c5c36b0..0000000000 --- a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/main.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "dashboard", ghpc_role = "monitoring" }) -} - -locals { - dash_path = "${path.module}/dashboards/${var.base_dashboard}.json.tpl" -} - -resource "google_monitoring_dashboard" "dashboard" { - dashboard_json = templatefile(local.dash_path, { - widgets = var.widgets - deployment_name = var.deployment_name - title = var.title - labels = local.labels - } - ) - project = var.project_id -} diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml deleted file mode 100644 index de1a10f57d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - stackdriver.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf deleted file mode 100644 index b7ff35fb0e..0000000000 --- a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "instructions" { - description = "Instructions for accessing the monitoring dashboard" - value = <<-EOT - A monitoring dashboard has been created. To view, navigate to the following URL: - https://console.cloud.google.com/monitoring/dashboards/builder${regex("/[0-9a-z-]*$", google_monitoring_dashboard.dashboard.id)} - EOT -} diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/variables.tf b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/variables.tf deleted file mode 100644 index 8194f8b73a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/variables.tf +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "base_dashboard" { - description = "Baseline dashboard template, select from HPC or Empty" - type = string - default = "HPC" - validation { - condition = contains(["HPC", "Empty"], var.base_dashboard) - error_message = "Must set var.base_dashboard to either \"HPC\" or \"Empty\"." - } -} - -variable "title" { - description = "Title of the created dashboard" - type = string - default = "Cluster Toolkit Dashboard" -} - -variable "widgets" { - description = "List of additional widgets to add to the base dashboard." - type = list(string) - default = [] -} - -variable "labels" { - description = "Labels to add to the monitoring dashboard instance. Key-value pairs." - type = map(string) -} diff --git a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/versions.tf b/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/versions.tf deleted file mode 100644 index 2717fe79f6..0000000000 --- a/deletion-test/primary/modules/embedded/modules/monitoring/dashboard/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/README.md b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/README.md deleted file mode 100644 index 057f4b649d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/README.md +++ /dev/null @@ -1,111 +0,0 @@ -## Description - -This module facilitates the creation of custom firewall rules for existing -networks. - -## Example usage - -This module can be used by other Toolkit modules to create application-specific -firewall rules or in conjunction with the [pre-existing-vpc] module to enable -traffic in existing networks. The snippet below is drawn from the -[ml-slurm.yaml] example: - -```yaml -- group: primary - modules: - - id: network - source: modules/network/pre-existing-vpc - - # this example anticipates that the VPC default network has internal traffic - # allowed and IAP tunneling for SSH connections - - id: firewall_rule - source: modules/network/firewall-rules - use: - - network - settings: - ingress_rules: - - name: $(vars.deployment_name)-allow-internal-traffic - description: Allow internal traffic - destination_ranges: - - $(network.subnetwork_address) - source_ranges: - - $(network.subnetwork_address) - allow: - - protocol: tcp - ports: - - 0-65535 - - protocol: udp - ports: - - 0-65535 - - protocol: icmp - - name: $(vars.deployment_name)-allow-iap-ssh - description: Allow IAP-tunneled SSH connections - destination_ranges: - - $(network.subnetwork_address) - source_ranges: - - 35.235.240.0/20 - allow: - - protocol: tcp - ports: - - 22 -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [firewall\_rule](#module\_firewall\_rule) | terraform-google-modules/network/google//modules/firewall-rules | ~> 12.0 | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.pga_check](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [google_compute_subnetwork.subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [egress\_rules](#input\_egress\_rules) | List of egress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | -| [ingress\_rules](#input\_ingress\_rules) | List of ingress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | -| [network\_name](#input\_network\_name) | The name of the network to create firewall rules in | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The project ID to host the network in | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork whose global network firewall rules will be modified. | `string` | n/a | yes | - -## Outputs - -No outputs. - - -[pre-existing-vpc]: ../pre-existing-vpc/README.md -[ml-slurm.yaml]: ../../../examples/ml-slurm.yaml diff --git a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/main.tf b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/main.tf deleted file mode 100644 index 05241278ad..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/main.tf +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - use_subnetwork_data = (var.project_id == null || var.network_name == null) && var.subnetwork_self_link != null -} - -# the google_compute_network data source does not allow identification by -# self_link, which uniquely identifies subnet, project, and network -data "google_compute_subnetwork" "subnetwork" { - # Only instantiate this data source if needed - count = local.use_subnetwork_data ? 1 : 0 - self_link = var.subnetwork_self_link -} - -locals { - # Derived values from data source, null if data source is not used - derived_project_id = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].project : null - derived_network_name = local.use_subnetwork_data ? data.google_compute_subnetwork.subnetwork[0].network : null - - # Effective values: Use var if provided, otherwise use derived value - effective_project_id = coalesce(var.project_id, local.derived_project_id) - effective_network_name = coalesce(var.network_name, local.derived_network_name) -} - -# Module-level check for Private Google Access on the subnetwork. -# This check is only relevant if subnetwork_self_link was provided and used. -resource "terraform_data" "pga_check" { - count = local.use_subnetwork_data ? 1 : 0 - - lifecycle { - precondition { - condition = data.google_compute_subnetwork.subnetwork[0].private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.subnetwork[0].name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } - } -} - -module "firewall_rule" { - source = "terraform-google-modules/network/google//modules/firewall-rules" - version = "~> 12.0" - project_id = local.effective_project_id - network_name = local.effective_network_name - - ingress_rules = var.ingress_rules - egress_rules = var.egress_rules -} diff --git a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/variables.tf b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/variables.tf deleted file mode 100644 index 05e9be4425..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/variables.tf +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork whose global network firewall rules will be modified." - type = string -} - -variable "project_id" { - description = "The project ID to host the network in" - type = string - default = null -} - -variable "network_name" { - description = "The name of the network to create firewall rules in" - type = string - default = null -} - -variable "ingress_rules" { - description = "List of ingress rules" - default = [] - type = list(object({ - name = string - description = optional(string, null) - disabled = optional(bool, null) - priority = optional(number, null) - destination_ranges = optional(list(string), []) - source_ranges = optional(list(string), []) - source_tags = optional(list(string)) - source_service_accounts = optional(list(string)) - target_tags = optional(list(string)) - target_service_accounts = optional(list(string)) - - allow = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - deny = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - log_config = optional(object({ - metadata = string - })) - })) -} - -variable "egress_rules" { - description = "List of egress rules" - default = [] - type = list(object({ - name = string - description = optional(string, null) - disabled = optional(bool, null) - priority = optional(number, null) - destination_ranges = optional(list(string), []) - source_ranges = optional(list(string), []) - source_tags = optional(list(string)) - source_service_accounts = optional(list(string)) - target_tags = optional(list(string)) - target_service_accounts = optional(list(string)) - - allow = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - deny = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - log_config = optional(object({ - metadata = string - })) - })) -} diff --git a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/versions.tf b/deletion-test/primary/modules/embedded/modules/network/firewall-rules/versions.tf deleted file mode 100644 index 9061dd3ae5..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/firewall-rules/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.74.0" - } - - required_version = ">= 1.5" -} diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md deleted file mode 100644 index abbfe3b97b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md +++ /dev/null @@ -1,143 +0,0 @@ -## Description - -This module accomplishes the following: - -* Creates one [VPC network][cft-network] - * Each VPC contains a variable number of subnetworks as specified in the - `subnetworks_template` variable - * Each subnetwork contains distinct IP address ranges -* Outputs the following unique parameters - * `subnetwork_interfaces` which is compatible with Slurm and vm-instance - modules - * `subnetwork_interfaces_gke` which is compatible with GKE modules - -This module is a simplified version of the VPC module and its main difference -is the variable `subnetwork_template` which is the template for all subnetworks -created within the network. This template contains the following values: - -1. `count`: The number of subnetworks to be created -1. `name_prefix`: The prefix for the subnetwork names -1. `ip_range`: [CIDR-formatted IP range][cidr] -1. `region`: The region where the subnetwork will be deployed - -> [!WARNING] -> The `ip_range` should be always be large enough to split into `count` -> subnetworks and the number of required connections within. - -[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v10.0.0 -[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation - -### Example - -This snippet uses the gpu-vpc module to create a new VPC network named -`test-rdma-net` with 8 subnetworks named `test-mrdma-sub-#` where # ranges from -0 to 7. The subnetworks will split the `ip_range` evenly, starting from bit 16 -(0 indexed). The networks are ingested by the Slurm nodeset within the -`additional_networks` setting. - -```yaml - - id: rdma-net - source: modules/network/gpu-rdma-vpc - settings: - network_name: test-rdma-net - network_profile: https://www.googleapis.com/compute/beta/projects/$(vars.project_id)/global/networkProfiles/$(vars.zone)-vpc-roce - network_routing_mode: REGIONAL - subnetworks_template: - name_prefix: test-mrdma-sub - count: 8 - ip_range: 192.168.0.0/16 - region: $(vars.region) - - - id: a3_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network0] - settings: - machine_type: a3-ultragpu-8g - additional_networks: - $(concat( - [{ - network=null, - subnetwork=network1.subnetwork_self_link, - subnetwork_project=vars.project_id, - nic_type="GVNIC", - queue_count=null, - network_ip="", - stack_type=null, - access_config=[], - ipv6_access_config=[], - alias_ip_range=[] - }], - rdma-net.subnetwork_interfaces - )) - ... -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | DEPRECATED: enable\_internal\_traffic can not be specified for gpu-rdma-vpc. | `bool` | `null` | no | -| [firewall\_log\_config](#input\_firewall\_log\_config) | DEPRECATED: firewall\_log\_config can not be specified for gpu-rdma-vpc. | `string` | `null` | no | -| [firewall\_rules](#input\_firewall\_rules) | DEPRECATED: firewall\_rules can not be specified for gpu-rdma-vpc. | `any` | `null` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}} | `string` | n/a | yes | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "REGIONAL") | `string` | `"REGIONAL"` | no | -| [nic\_type](#input\_nic\_type) | NIC type for use in modules that use the output | `string` | `"MRDMA"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | -| [subnetworks\_template](#input\_subnetworks\_template) | Specifications for the subnetworks that will be created within this VPC.

count (number, required, number of subnets to create, default is 8)
name\_prefix (string, required, subnet name prefix, default is deployment name)
ip\_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16)
region (string, optional, region to deploy subnets to, defaults to vars.region) | 
object({
    count       = number
    name_prefix = string
    ip_range    = string
    region      = optional(string)
  })
| 
{
  "count": 8,
  "ip_range": "192.168.0.0/16",
  "name_prefix": null,
  "region": null
}
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [network\_id](#output\_network\_id) | ID of the new VPC network | -| [network\_name](#output\_network\_name) | Name of the new VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | -| [subnetwork\_interfaces](#output\_subnetwork\_interfaces) | Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules) | -| [subnetwork\_interfaces\_gke](#output\_subnetwork\_interfaces\_gke) | Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool) | -| [subnetwork\_name\_prefix](#output\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | -| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | - diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf deleted file mode 100644 index e37db01976..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - autoname = replace(var.deployment_name, "_", "-") - network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name - subnet_prefix = var.subnetworks_template.name_prefix == null ? "${local.autoname}-subnet" : var.subnetworks_template.name_prefix - - new_bits = ceil(log(var.subnetworks_template.count, 2)) - template_subnetworks = [for i in range(var.subnetworks_template.count) : - { - subnet_name = "${local.subnet_prefix}-${i}" - subnet_region = try(var.subnetworks_template.region, var.region) - subnet_ip = cidrsubnet(var.subnetworks_template.ip_range, local.new_bits, i) - } - ] - - firewall_rules = [] - - output_subnets = [ - for subnet in module.vpc.subnets : { - network = null - subnetwork = subnet.self_link - subnetwork_project = null # will populate from subnetwork_self_link - network_ip = null - nic_type = var.nic_type - stack_type = null - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } - ] - - output_subnets_gke = [ - for i in range(length(module.vpc.subnets)) : { - network = local.network_name - subnetwork = local.template_subnetworks[i].subnet_name - subnetwork_project = var.project_id - network_ip = null - nic_type = var.nic_type - stack_type = null - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } - ] -} - -module "vpc" { - source = "terraform-google-modules/network/google" - version = "~> 12.0" - - network_name = local.network_name - project_id = var.project_id - auto_create_subnetworks = false - subnets = local.template_subnetworks - routing_mode = var.network_routing_mode - mtu = var.mtu - description = var.network_description - shared_vpc_host = var.shared_vpc_host - delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes - firewall_rules = local.firewall_rules - network_profile = var.network_profile -} diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf deleted file mode 100644 index 0a21f1d3f2..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the new VPC network" - value = module.vpc.network_name - depends_on = [module.vpc] -} - -output "network_id" { - description = "ID of the new VPC network" - value = module.vpc.network_id - depends_on = [module.vpc] -} - -output "network_self_link" { - description = "Self link of the new VPC network" - value = module.vpc.network_self_link - depends_on = [module.vpc] -} - -output "subnetworks" { - description = "Full list of subnetwork objects belonging to the new VPC network" - value = module.vpc.subnets - depends_on = [module.vpc] -} - -output "subnetwork_interfaces" { - description = "Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules)" - value = local.output_subnets - depends_on = [module.vpc] -} - -# The output subnetwork_interfaces is compatible with vm-instance module but not with gke-node-pool -# See https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/99493df21cecf6a092c45298bf7a45e0343cf622/modules/compute/vm-instance/variables.tf#L220 -# So, we need a separate output that makes the network and subnetwork names available -output "subnetwork_interfaces_gke" { - description = "Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool)" - value = local.output_subnets_gke - depends_on = [module.vpc] -} - -output "subnetwork_name_prefix" { - description = "Prefix of the RDMA subnetwork names" - value = var.subnetworks_template.name_prefix -} diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf deleted file mode 100644 index a30fb50e7d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "network_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" - type = string - default = null -} - -variable "region" { - description = "The default region for Cloud resources" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "subnetworks_template" { - description = <<-EOT - Specifications for the subnetworks that will be created within this VPC. - - count (number, required, number of subnets to create, default is 8) - name_prefix (string, required, subnet name prefix, default is deployment name) - ip_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16) - region (string, optional, region to deploy subnets to, defaults to vars.region) - EOT - nullable = false - type = object({ - count = number - name_prefix = string - ip_range = string - region = optional(string) - }) - default = { - count = 8 - name_prefix = null - ip_range = "192.168.0.0/16" - region = null - } - - validation { - condition = var.subnetworks_template.count > 0 - error_message = "Number of subnetworks must be greater than 0" - } - - validation { - condition = can(cidrhost(var.subnetworks_template.ip_range, 0)) - error_message = "IP address range must be in CIDR format." - } -} - -variable "network_routing_mode" { - type = string - default = "REGIONAL" - description = "The network routing mode (default \"REGIONAL\")" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "shared_vpc_host" { - type = bool - description = "Makes this project a Shared VPC host if 'true' (default 'false')" - default = false -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_internal_traffic" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." - type = bool - default = null - validation { - condition = var.enable_internal_traffic == null - error_message = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." - } -} - -variable "firewall_rules" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." - type = any - default = null - validation { - condition = var.firewall_rules == null - error_message = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." - } -} - -variable "firewall_log_config" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." - type = string - default = null - validation { - condition = var.firewall_log_config == null - error_message = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." - } -} - -variable "network_profile" { - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - EOT - type = string - nullable = false - - validation { - condition = can(coalesce(var.network_profile)) - error_message = "var.network_profile must be specified and not an empty string" - } -} - -variable "nic_type" { - description = "NIC type for use in modules that use the output" - type = string - nullable = true - default = "MRDMA" - - validation { - condition = contains(["MRDMA"], var.nic_type) - error_message = "The nic_type must be \"MRDMA\"." - } -} diff --git a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf b/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf deleted file mode 100644 index 71b7106734..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 0.15.0" -} diff --git a/deletion-test/primary/modules/embedded/modules/network/multivpc/README.md b/deletion-test/primary/modules/embedded/modules/network/multivpc/README.md deleted file mode 100644 index 973e6b32c9..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/multivpc/README.md +++ /dev/null @@ -1,136 +0,0 @@ -## Description - -This module accomplishes the following: - -* Creates 2 to 8 [VPC networks][vpc] - * Each VPC contains exactly 1 subnetwork - * Each subnetwork contains distinct IP address ranges -* Outputs the `additional_networks` parameter, which is compatible with Slurm - modules - -There are 4 variables that differentiate this module from the standard VPC -module. - -1. `network_prefix`: The name prefix of the VPCs to be created. All - networks and subnetworks will start with this and end with a unique number. -1. `network_count`: The number of VPCs to be created. -1. `global_ip_address_range`: [CIDR-formatted IP range][cidr] -1. `network_cidr_suffix`: The CIDR suffix that defines the address - space that the individual VPCs will cover. - -> [!WARNING] -> The `network_cidr_suffix` should be always be larger than the CIDR suffix on -> `global_ip_address_range`. The difference between these two suffixes should -> be large enough to accommodate the number of VPCs that are being deployed -> (e.g. CIDR suffix bit difference <= `ceil(log2(network_count)))`). - -> [!NOTE] -> For deployments that need multiple VPCs that do not meet this use-case, users -> should deploy multiple individual VPC modules. - -[vpc]: ../vpc/README.md -[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation - -### Example - -This snippet uses the multivpc module to create 8 new VPC networks named -`multivpc-net-#` where # ranges from 0 to 7. Additionally, it creates 1 -subnetwork in each VPC. - -```yaml - - id: network - source: modules/network/vpc - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: a3_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network, multinetwork] - settings: - machine_type: a3-highgpu-8g - ... -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | - -## Providers - -| Name | Version | -|------|---------| -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [vpcs](#module\_vpcs) | ../vpc | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.global_ip_cidr_suffix](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | -| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | -| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | -| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | -| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | -| [global\_ip\_address\_range](#input\_global\_ip\_address\_range) | IP address range (CIDR) that will span entire set of VPC networks | `string` | `"172.16.0.0/12"` | no | -| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT) | `number` | `2` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_count](#input\_network\_count) | The number of vpc nettworks to create | `number` | `4` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_interface\_defaults](#input\_network\_interface\_defaults) | The template of the network settings to be used on all vpcs. | 
object({
    network            = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string, "GVNIC")
    stack_type         = optional(string, "IPV4_ONLY")
    queue_count        = optional(string)
    access_config = optional(list(object({
      nat_ip                 = string
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  })
| 
{
  "access_config": [],
  "alias_ip_range": [],
  "ipv6_access_config": [],
  "network": null,
  "network_ip": "",
  "nic_type": "GVNIC",
  "queue_count": null,
  "stack_type": "IPV4_ONLY",
  "subnetwork": null,
  "subnetwork_project": null
}
| no | -| [network\_name\_prefix](#input\_network\_name\_prefix) | The base name of the vpcs and their subnets, will be appended with a sequence number | `string` | `""` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network dynamic routing mode | `string` | `"REGIONAL"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [subnetwork\_cidr\_suffix](#input\_subnetwork\_cidr\_suffix) | The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network. | `number` | `16` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [additional\_networks](#output\_additional\_networks) | Network interfaces for each subnetwork created by this module | -| [network\_ids](#output\_network\_ids) | IDs of the new VPC network | -| [network\_names](#output\_network\_names) | Names of the new VPC networks | -| [network\_self\_links](#output\_network\_self\_links) | Self link of the new VPC network | -| [subnetwork\_addresses](#output\_subnetwork\_addresses) | IP address range of the primary subnetwork | -| [subnetwork\_names](#output\_subnetwork\_names) | Names of the subnetwork created in each network | -| [subnetwork\_self\_links](#output\_subnetwork\_self\_links) | Self link of the primary subnetwork | - diff --git a/deletion-test/primary/modules/embedded/modules/network/multivpc/main.tf b/deletion-test/primary/modules/embedded/modules/network/multivpc/main.tf deleted file mode 100644 index ad06e793c1..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/multivpc/main.tf +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # this input variable is validated to be in CIDR format - network_name = coalesce(replace(var.network_name_prefix, "_", "-"), replace(var.deployment_name, "_", "-")) - global_ip_cidr_prefix = split("/", var.global_ip_address_range)[0] - global_ip_cidr_suffix = split("/", var.global_ip_address_range)[1] - global_ip_cidr_valid = "${local.global_ip_cidr_prefix}/${terraform_data.global_ip_cidr_suffix.output}" - subnetwork_new_bits = var.subnetwork_cidr_suffix - local.global_ip_cidr_suffix - maximum_subnetworks = pow(2, local.subnetwork_new_bits) - additional_networks = [ - for vpc in module.vpcs : - merge(var.network_interface_defaults, { - network = vpc.network_name - subnetwork = vpc.subnetwork_name - subnetwork_project = var.project_id - }) - ] -} - -resource "terraform_data" "global_ip_cidr_suffix" { - input = local.global_ip_cidr_suffix - lifecycle { - precondition { - condition = local.maximum_subnetworks >= var.network_count - error_message = < 1 - error_message = "The minimum VPCs able to be created by this module is 2. Use the standard Toolkit module at modules/network/vpc for count = 1" - } - validation { - condition = var.network_count <= 8 - error_message = "The maximum VPCs able to be created by this module is 8" - } -} - -variable "global_ip_address_range" { - description = "IP address range (CIDR) that will span entire set of VPC networks" - type = string - default = "172.16.0.0/12" - - validation { - condition = can(cidrhost(var.global_ip_address_range, 0)) - error_message = "var.global_ip_address_range must be an IPv4 CIDR range (e.g. \"172.16.0.0/12\")." - } -} - -variable "subnetwork_cidr_suffix" { - description = "The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network." - type = number - default = 16 -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "network_routing_mode" { - type = string - default = "REGIONAL" - description = "The network dynamic routing mode" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "ips_per_nat" { - type = number - description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT)" - default = 2 -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_iap_ssh_ingress" { - type = bool - description = "Enable a firewall rule to allow SSH access using IAP tunnels" - default = true -} - -variable "enable_iap_rdp_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" - default = false -} - -variable "enable_iap_winrm_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" - default = false -} - -variable "enable_internal_traffic" { - type = bool - description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" - default = true -} - -variable "extra_iap_ports" { - type = list(string) - description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" - default = [] -} - -variable "allowed_ssh_ip_ranges" { - type = list(string) - description = "A list of CIDR IP ranges from which to allow ssh access" - default = [] - - validation { - condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) - error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." - } -} - -variable "firewall_rules" { - type = any - description = "List of firewall rules" - default = [] -} - -variable "network_interface_defaults" { - type = object({ - network = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - network_ip = optional(string, "") - nic_type = optional(string, "GVNIC") - stack_type = optional(string, "IPV4_ONLY") - queue_count = optional(string) - access_config = optional(list(object({ - nat_ip = string - network_tier = string - public_ptr_domain_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - public_ptr_domain_name = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - }) - description = "The template of the network settings to be used on all vpcs." - default = { - network = null - subnetwork = null - subnetwork_project = null - network_ip = "" - nic_type = "GVNIC" - stack_type = "IPV4_ONLY" - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } -} - -variable "network_profile" { - type = string - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown - EOT - default = null -} diff --git a/deletion-test/primary/modules/embedded/modules/network/multivpc/versions.tf b/deletion-test/primary/modules/embedded/modules/network/multivpc/versions.tf deleted file mode 100644 index e75a67f7b6..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/multivpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.4.0" -} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md deleted file mode 100644 index 4d63b17091..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md +++ /dev/null @@ -1,94 +0,0 @@ -## Description - -This module discovers a subnetwork that already exists in Google Cloud and -outputs subnetwork attributes that uniquely identify it for use by other modules. - -For example, the blueprint below discovers the referred to subnetwork. -With the `use` keyword, the [vm-instance] module accepts the `subnetwork_self_link` -input variables that uniquely identify the subnetwork in which the VM will be created. - -[vpc]: ../vpc/README.md -[vm-instance]: ../../compute/vm-instance/README.md - -> **_NOTE:_** Additional IAM work is needed for this to work correctly. - -### Example - -```yaml -- id: network - source: modules/network/pre-existing-subnetwork - settings: - subnetwork_self_link: https://www.googleapis.com/compute/v1/projects/name-of-host-project/regions/REGION/subnetworks/SUBNETNAME - -- id: example_vm - source: modules/compute/vm-instance - use: - - network - settings: - name_prefix: example - machine_type: c2-standard-4 -``` - -As described in documentation: -[https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork] - -If subnetwork_self_link is provided then name,region,project is ignored. - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [project](#input\_project) | Name of the project that owns the subnetwork | `string` | `null` | no | -| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | `null` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self-link of the subnet in the VPC | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | -| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | - diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf deleted file mode 100644 index 9fb206f969..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -data "google_compute_subnetwork" "primary_subnetwork" { - name = var.subnetwork_name - region = var.region - project = var.project - self_link = var.subnetwork_self_link - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The subnetwork: ${coalesce(var.subnetwork_name, var.subnetwork_self_link)} could not be found." - } - } -} - -# Module-level check for Private Google Access on the subnetwork -check "private_google_access_enabled_subnetwork" { - assert { - condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } -} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml deleted file mode 100644 index 6a6f1e5757..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com -ghpc: - has_to_be_used: true diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf deleted file mode 100644 index 868708dc6b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "subnetwork" { - description = "Full subnetwork object in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork -} - -output "subnetwork_name" { - description = "Name of the subnetwork in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.name -} - -output "subnetwork_self_link" { - description = "Subnetwork self-link in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.self_link -} - -output "subnetwork_address" { - description = "Subnetwork IP range in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range -} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf deleted file mode 100644 index d5191843e8..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "subnetwork_self_link" { - description = "Self-link of the subnet in the VPC" - type = string - default = null -} - -variable "project" { - description = "Name of the project that owns the subnetwork" - type = string - default = null -} - -variable "subnetwork_name" { - description = "Name of the pre-existing VPC subnetwork" - type = string - default = null -} - -variable "region" { - description = "Region in which to search for primary subnetwork" - type = string - default = null -} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf deleted file mode 100644 index 917d948433..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.74.0" - } - - required_version = ">= 1.5" -} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/README.md b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/README.md deleted file mode 100644 index 38a1840c2d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/README.md +++ /dev/null @@ -1,110 +0,0 @@ -## Description - -This module discovers a VPC network that already exists in Google Cloud and -outputs network attributes that uniquely identify it for use by other modules. -The module outputs are aligned with the [vpc module][vpc] so that it can be used -as a drop-in substitute when a VPC already exists. - -For example, the blueprint below discovers the "default" global network and the -"default" regional subnetwork in us-central1. With the `use` keyword, the -[vm-instance] module accepts the `network_self_link` and `subnetwork_self_link` -input variables that uniquely identify the network and subnetwork in which the -VM will be created. - -[vpc]: ../vpc/README.md -[vm-instance]: ../../compute/vm-instance/README.md - -### Example - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - settings: - project_id: $(vars.project_id) - region: us-central1 - -- id: example_vm - source: modules/compute/vm-instance - use: - - network1 - settings: - name_prefix: example - machine_type: c2-standard-4 -``` - -> **_NOTE:_** The `project_id` and `region` settings would be inferred from the -> deployment variables of the same name, but they are included here for clarity. - -### Use shared-vpc - -If a network is created in different project, this module can be used to -reference the network. To use a network from a different project first make sure -you have a [cloud nat][cloudnat] and [IAP][iap] forwarding. For more details, -refer [shared-vpc][shared-vpc-doc] - -[cloudnat]: https://cloud.google.com/nat/docs/overview -[iap]: https://cloud.google.com/iap/docs/using-tcp-forwarding -[shared-vpc-doc]: ../../../examples/README.md#hpc-slurm-sharedvpcyaml-community-badge-experimental-badge - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_network.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network) | data source | -| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [network\_name](#input\_network\_name) | Name of the existing VPC network | `string` | `"default"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | n/a | yes | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork; defaults to var.network\_name if set to null. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [network\_id](#output\_network\_id) | ID of the existing VPC network | -| [network\_name](#output\_network\_name) | Name of the existing VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the existing VPC network | -| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | -| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | - diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf deleted file mode 100644 index ed332bab72..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -data "google_compute_network" "vpc" { - name = var.network_name - project = var.project_id - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The network: ${var.network_name} could not be found in project: ${var.project_id}." - } - } -} - -locals { - subnetwork_name = var.subnetwork_name != null ? var.subnetwork_name : var.network_name -} - -data "google_compute_subnetwork" "primary_subnetwork" { - name = local.subnetwork_name - region = var.region - project = var.project_id - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The subnetwork: ${local.subnetwork_name} could not be found in project: ${var.project_id} and region: ${var.region}." - } - } -} - -# Module-level check for Private Google Access on the subnetwork -check "private_google_access_enabled_subnetwork" { - assert { - condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } -} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf deleted file mode 100644 index 00861af5ca..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the existing VPC network" - value = data.google_compute_network.vpc.name -} - -output "network_id" { - description = "ID of the existing VPC network" - value = data.google_compute_network.vpc.id -} - -output "network_self_link" { - description = "Self link of the existing VPC network" - value = data.google_compute_network.vpc.self_link -} - -output "subnetwork" { - description = "Full subnetwork object in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork -} - -output "subnetwork_name" { - description = "Name of the subnetwork in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.name -} - -output "subnetwork_self_link" { - description = "Subnetwork self-link in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.self_link -} - -output "subnetwork_address" { - description = "Subnetwork IP range in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range -} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf deleted file mode 100644 index 291a81604a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "network_name" { - description = "Name of the existing VPC network" - type = string - default = "default" -} - -variable "subnetwork_name" { - description = "Name of the pre-existing VPC subnetwork; defaults to var.network_name if set to null." - type = string - default = null -} - -variable "region" { - description = "Region in which to search for primary subnetwork" - type = string -} diff --git a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf b/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf deleted file mode 100644 index 81fe5aeff3..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.74.0" - } - - required_version = ">= 1.5" -} diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/README.md b/deletion-test/primary/modules/embedded/modules/network/vpc/README.md deleted file mode 100644 index 2c2b1aa1a3..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/vpc/README.md +++ /dev/null @@ -1,237 +0,0 @@ -## Description - -This module creates a new [VPC network][vpc] with 1 or more subnetworks and -a [Cloud Router][router] for every region with a subnetwork. By default, it will -create: - -* A [Cloud NAT][nat] to enable outbound access to the public internet for VMs - without public IP addresses; VMs with public IP addresses bypass the NAT to - directly access the public internet -* A firewall rule that enables inbound SSH access from [Identity-Aware - Proxy][iap] -* A firewall rule that enables all traffic internal to the network - -This behavior is optional and can be configured as [described below](#inputs). -This module is based on networking support in the [Cloud Foundation -Toolkit][cft]. We recommend following the [documentation for the network -module][cft-network] and [submodules][cft-network-submodules] for more details. -In particular, the detailed structure of input variables can be found for: - -* [var.firewall\_rules](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/firewall-rules#inputs) -* [var.secondary\_ranges](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets#inputs) - -[vpc]: https://cloud.google.com/vpc -[router]: https://github.com/terraform-google-modules/terraform-google-cloud-router -[nat]: https://github.com/terraform-google-modules/terraform-google-cloud-nat -[iap]: https://cloud.google.com/iap -[cft]: https://cloud.google.com/foundation-toolkit -[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0 -[cft-network-submodules]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules - -Additionally, [Google Private Access][gpa] is enabled by default on all -subnetworks unless it is explicitly disabled. This setting ensures that all VMs -can use Google services such as [Cloud Storage][gcs] even if they do not have -public IP addresses or Cloud NAT is disabled. - -[gpa]: https://cloud.google.com/vpc/docs/private-google-access -[gcs]: https://cloud.google.com/storage - -### Example - -This creates a new VPC network named `cluster-net`. - -```yaml - - id: network1 - source: modules/network/vpc - settings: - network_name: cluster-net -``` - -### Deprecation warning - -The variables listed below have been deprecated and will be removed in a future -release. Until they are removed,You may continue to use them in Toolkit -blueprints with the same functionality as documented in the [Toolkit 1.0 -release][vpc1.0]. - -* Deprecated variables - * `var.primary_subnetwork` - * `var.additional_subnetworks` - * `var.subnetwork_size` - -[vpc1.0]: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/v1.0.0/modules/network/vpc/README.md - -The following variables have been added to support explicit IP ranges for -subnetworks while retaining existing functionality. We advise adopting them even -if not using explicit IP ranges . The Toolkit ***does not support*** mixing -deprecated variables with the new replacements. The new functionality is -described in [more detail below](#subnetworks). - -* New variables to adopt - * `var.subnetworks` - * A value for this can be generated by merging `var.primary_subnetwork` and - `var.additional_subnetworks` into a single list - * `var.default_primary_subnetwork_size` - * This variable has been renamed for clarity; its value can be directly - copied from an explicit setting for `var.subnetwork_size`; if your blueprint - does not have an explicit setting, the default values are the same - -### Subnetworks - -This module will always provision at least 1 "primary" subnetwork in which most -resources are expected to be provisioned. This primary subnetwork is determined -by - -1. The first element of [var.subnetworks](#input_subnetworks) if it is not the - empty list -2. A default subnetwork automatically calculated from - * [var.subnetwork_name](#input_subnetwork_name) - * [var.region](#input_region) - * [var.network_address_range](#input_network_address_range) - * [var.default_primary_subnetwork_size](#input_default_primary_subnetwork_size) - -If `var.subnetworks` is provided then the primary subnetwork name is taken -explicitly from it and `var.subnetwork_name` is ignored. - -`var.subnetworks` behaves identically to the [Cloud Foundation Toolkit subnets -module][cftsubnets] with the lone exception that one can provide ***one*** of -the following settings for each subnetwork: - -* `new_bits` -* `subnet_ip` - -If each subnetwork defines `subnet_ip` then these are taken to be their explicit -CIDR IP ranges. If each subnetwork defines `new_bits`, then these are taken to -be the size of the CIDR subnetwork (in bits). IP ranges for each subnetwork are -calculated using `var.network_address_range` as the base IP, producing the most -compact set of subnetworks possible. - -> **_NOTE:_** we do not presently support the modification of individual subnetworks -> when using this module to provision more than 1 subnetwork using automatically -> calculated IP ranges based upon `new_bits`. Doing so will cause IP ranges to be -> recalculated for each subnetwork. We advise appending new subnetworks to the end -> of `var.subnetworks`. - -[cftsubnets]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets - -### SSH Access - -By default a firewall rule is created to allow inbound SSH access from -[Identity-Aware Proxy][iap]. A user must have the `IAP-Secured Tunnel User` -(`roles/iap.tunnelResourceAccessor`) IAM role to be able to SSH over IAP. - -To allow regular SSH access from a known IP address you can add the following -`firewall_rules` setting to the `vpc` module: - -```yaml - - id: network1 - source: modules/network/vpc - settings: - firewall_rules: - - name: ssh-my-machine - direction: INGRESS - ranges: [/32] - allow: - - protocol: tcp - ports: [22] -``` - -> **Note**: You must populate the above example with the source IP address from -> which you plan to SSH from. You can use a service like -> [whatismyip.com](https://whatismyip.com) to determine your IP address. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | - -## Providers - -| Name | Version | -|------|---------| -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 7.3 | -| [nat\_ip\_addresses](#module\_nat\_ip\_addresses) | terraform-google-modules/address/google | ~> 4.1 | -| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 12.0 | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.cloud_nat_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.network_profile_firewall_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.secondary_ranges_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_subnetworks](#input\_additional\_subnetworks) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `list(map(string))` | `null` | no | -| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | -| [default\_primary\_subnetwork\_size](#input\_default\_primary\_subnetwork\_size) | The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks | `number` | `15` | no | -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_cloud\_nat](#input\_enable\_cloud\_nat) | Enable the creation of Cloud NATs. | `bool` | `true` | no | -| [enable\_cloud\_router](#input\_enable\_cloud\_router) | Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview | `bool` | `true` | no | -| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | -| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | -| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | -| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | -| [firewall\_log\_config](#input\_firewall\_log\_config) | Firewall log configuration for Toolkit firewall rules (var.enable\_iap\_ssh\_ingress and others) | `string` | `"DISABLE_LOGGING"` | no | -| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | -| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples | `number` | `2` | no | -| [labels](#input\_labels) | Labels to add to network resources that support labels. Key-value pairs of strings. | `map(string)` | `{}` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_address\_range](#input\_network\_address\_range) | IP address range (CIDR) for global network | `string` | `"10.0.0.0/9"` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "GLOBAL") | `string` | `"GLOBAL"` | no | -| [primary\_subnetwork](#input\_primary\_subnetwork) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `map(string)` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [secondary\_ranges](#input\_secondary\_ranges) | "Secondary ranges associated with the subnets.
This will be deprecated in favour of secondary\_ranges\_list at a later date.
Please migrate to using the same." | `map(list(object({ range_name = string, ip_cidr_range = string })))` | `{}` | no | -| [secondary\_ranges\_list](#input\_secondary\_ranges\_list) | "List of secondary ranges associated with the subnetworks.
Each subnetwork must be specified at most once in this list." | 
list(object({
    subnetwork_name = string,
    ranges = list(object({
      range_name    = string,
      ip_cidr_range = string
    }))
  }))
| `[]` | no | -| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-primary-subnet") | `string` | `null` | no | -| [subnetwork\_size](#input\_subnetwork\_size) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `number` | `null` | no | -| [subnetworks](#input\_subnetworks) | List of subnetworks to create within the VPC. If left empty, it will be
replaced by a single, default subnetwork constructed from other parameters
(e.g. var.region). In all cases, the first subnetwork in the list is identified
by outputs as a "primary" subnetwork.

subnet\_name (string, required, name of subnet)
subnet\_region (string, required, region of subnet)
subnet\_ip (string, mutually exclusive with new\_bits, CIDR-formatted IP range for subnetwork)
new\_bits (number, mutually exclusive with subnet\_ip, CIDR bits used to calculate subnetwork range)
subnet\_private\_access (bool, optional, Enable Private Access on subnetwork)
subnet\_flow\_logs (map(string), optional, Configure Flow Logs see terraform-google-network module)
description (string, optional, Description of Network)
purpose (string, optional, related to Load Balancing)
role (string, optional, related to Load Balancing) | `list(map(string))` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nat\_ips](#output\_nat\_ips) | External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used) | -| [network\_id](#output\_network\_id) | ID of the new VPC network | -| [network\_name](#output\_network\_name) | Name of the new VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | -| [subnetwork](#output\_subnetwork) | Primary subnetwork object | -| [subnetwork\_address](#output\_subnetwork\_address) | IP address range of the primary subnetwork | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the primary subnetwork | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Self link of the primary subnetwork | -| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | - diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/main.tf b/deletion-test/primary/modules/embedded/modules/network/vpc/main.tf deleted file mode 100644 index 24c8eb22bd..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/vpc/main.tf +++ /dev/null @@ -1,256 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -resource "terraform_data" "secondary_ranges_validation" { - lifecycle { - precondition { - condition = !(length(var.secondary_ranges) > 0 && length(var.secondary_ranges_list) > 0) - error_message = "Only one of var.secondary_ranges or var.secondary_ranges_list should be specified" - } - } -} - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "vpc", ghpc_role = "network" }) -} - -locals { - autoname = replace(var.deployment_name, "_", "-") - network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name - subnetwork_name = var.subnetwork_name == null ? "${local.autoname}-primary-subnet" : var.subnetwork_name - - # define a default subnetwork for cases in which no explicit subnetworks are - # defined in var.subnetworks - default_primary_subnetwork_cidr_block = cidrsubnet(var.network_address_range, var.default_primary_subnetwork_size, 0) - default_primary_subnetwork = { - subnet_name = local.subnetwork_name - subnet_ip = local.default_primary_subnetwork_cidr_block - subnet_region = var.region - subnet_private_access = true - subnet_flow_logs = false - description = "primary subnetwork in ${local.network_name}" - purpose = null - role = null - } - - # Identify user-supplied primary subnetwork - # (1) explicit var.subnetworks[0] - # (2) implicit local default subnetwork - input_primary_subnetwork = coalesce(try(var.subnetworks[0], null), local.default_primary_subnetwork) - - # Identify user-supplied additional subnetworks - # (1) explicit var.subnetworks[1:end] - # (2) empty list - input_additional_subnetworks = try(slice(var.subnetworks, 1, length(var.subnetworks)), []) - - # at this point we have constructed a list of subnetworks but need to extract - # user-provided CIDR blocks or calculate them from user-provided new_bits - # after we complete deprecation, local.all_subnetworks can be replaced with - # var.subnetworks (or local.default_primary_subnetwork if that is null) - input_subnetworks = concat([local.input_primary_subnetwork], local.input_additional_subnetworks) - subnetworks_cidr_blocks = try( - local.input_subnetworks[*]["subnet_ip"], - cidrsubnets(var.network_address_range, local.input_subnetworks[*]["new_bits"]...) - ) - - # merge in the CIDR blocks (even when already there) and remove new_bits - subnetworks = [for i, subnet in local.input_subnetworks : - merge({ for k, v in subnet : k => v if k != "new_bits" }, { "subnet_ip" = local.subnetworks_cidr_blocks[i] }) - ] - - # gather the unique regions for purposes of creating Router/NAT - cloud_router_regions = var.enable_cloud_router ? distinct([for subnet in local.subnetworks : subnet.subnet_region]) : [] - cloud_nat_regions = var.enable_cloud_nat ? local.cloud_router_regions : [] - - # this comprehension should have 1 and only 1 match - output_primary_subnetwork = one([for k, v in module.vpc.subnets : v if k == "${local.subnetworks[0].subnet_region}/${local.subnetworks[0].subnet_name}"]) - output_primary_subnetwork_name = local.output_primary_subnetwork.name - output_primary_subnetwork_self_link = local.output_primary_subnetwork.self_link - output_primary_subnetwork_ip_cidr_range = local.output_primary_subnetwork.ip_cidr_range - - iap_ports = distinct(concat(compact([ - var.enable_iap_rdp_ingress ? "3389" : "", - var.enable_iap_ssh_ingress ? "22" : "", - var.enable_iap_winrm_ingress ? "5986" : "", - ]), var.extra_iap_ports)) - - firewall_log_api_values = { - "DISABLE_LOGGING" = null - "INCLUDE_ALL_METADATA" = { metadata = "INCLUDE_ALL_METADATA" }, - "EXCLUDE_ALL_METADATA" = { metadata = "EXCLUDE_ALL_METADATA" }, - } - firewall_log_config = lookup(local.firewall_log_api_values, var.firewall_log_config, null) - - allow_iap_ingress = { - name = "${local.network_name}-fw-allow-iap-ingress" - description = "allow TCP access via Identity-Aware Proxy" - direction = "INGRESS" - priority = null - ranges = ["35.235.240.0/20"] - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = local.iap_ports - }] - deny = [] - log_config = local.firewall_log_config - } - - allow_ssh_ingress = { - name = "${local.network_name}-fw-allow-ssh-ingress" - description = "allow SSH access" - direction = "INGRESS" - priority = null - ranges = var.allowed_ssh_ip_ranges - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = ["22"] - }] - deny = [] - log_config = local.firewall_log_config - } - - allow_internal_traffic = { - name = "${local.network_name}-fw-allow-internal-traffic" - priority = null - description = "allow traffic between nodes of this VPC" - direction = "INGRESS" - ranges = [var.network_address_range] - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = ["0-65535"] - }, { - protocol = "udp" - ports = ["0-65535"] - }, { - protocol = "icmp" - ports = null - }, - ] - deny = [] - log_config = local.firewall_log_config - } - - firewall_rules = concat( - var.firewall_rules, - length(var.allowed_ssh_ip_ranges) > 0 ? [local.allow_ssh_ingress] : [], - var.enable_internal_traffic ? [local.allow_internal_traffic] : [], - length(local.iap_ports) > 0 ? [local.allow_iap_ingress] : [] - ) - - secondary_ranges_map = { - for secondary_range in var.secondary_ranges_list : - secondary_range.subnetwork_name => secondary_range.ranges - } -} - -resource "terraform_data" "network_profile_firewall_validation" { - lifecycle { - precondition { - condition = !(try(strcontains(var.network_profile, "roce"), false) && length(local.firewall_rules) > 0) - error_message = "If var.network_profile contains 'roce', var.firewall_rules must be empty and var.enable_internal_traffic must be false, please see: https://cloud.google.com/vpc/docs/rdma-network-profiles#additional_features_that_dont_apply_to_traffic_from_rdma_nics" - } - } -} - -module "vpc" { - source = "terraform-google-modules/network/google" - version = "~> 12.0" - - depends_on = [terraform_data.network_profile_firewall_validation] - - network_name = local.network_name - project_id = var.project_id - auto_create_subnetworks = false - subnets = local.subnetworks - secondary_ranges = length(local.secondary_ranges_map) > 0 ? local.secondary_ranges_map : var.secondary_ranges - routing_mode = var.network_routing_mode - mtu = var.mtu - description = var.network_description - shared_vpc_host = var.shared_vpc_host - delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes - firewall_rules = local.firewall_rules - network_profile = var.network_profile -} - -resource "terraform_data" "cloud_nat_validation" { - lifecycle { - precondition { - condition = var.enable_cloud_router == true || var.enable_cloud_nat == false - error_message = <<-EOD - "Cannot have Cloud NAT without a Cloud Router. If you desire Cloud NAT functionality please set `enable_cloud_router` to true." - EOD - } - } -} - -# This use of the module may appear odd when var.ips_per_nat = 0. The module -# will be called for all regions with subnetworks but names will be set to the -# empty list. This is a perfectly valid value (the default!). In this scenario, -# no IP addresses are created and all module outputs are empty lists. -# -# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/variables.tf#L27 -# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/outputs.tf -module "nat_ip_addresses" { - source = "terraform-google-modules/address/google" - version = "~> 4.1" - - depends_on = [terraform_data.cloud_nat_validation] - - for_each = toset(local.cloud_nat_regions) - - project_id = var.project_id - region = each.value - # an external, regional (not global) IP address is suited for a regional NAT - address_type = "EXTERNAL" - global = false - labels = local.labels - names = [for idx in range(var.ips_per_nat) : "${local.network_name}-nat-ips-${each.value}-${idx}"] -} - -module "cloud_router" { - source = "terraform-google-modules/cloud-router/google" - version = "~> 7.3" - - depends_on = [terraform_data.cloud_nat_validation] - - for_each = toset(local.cloud_router_regions) - - project = var.project_id - name = "${local.network_name}-router" - region = each.value - network = module.vpc.network_name - # in scenario with no NAT IPs, no NAT is created even if router is created - # https://github.com/terraform-google-modules/terraform-google-cloud-router/blob/v2.0.0/nat.tf#L18-L20 - nats = length(module.nat_ip_addresses[each.value].self_links) == 0 ? [] : [ - { - name : "cloud-nat-${each.value}", - nat_ips : module.nat_ip_addresses[each.value].self_links - }, - ] -} diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/metadata.yaml b/deletion-test/primary/modules/embedded/modules/network/vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/outputs.tf b/deletion-test/primary/modules/embedded/modules/network/vpc/outputs.tf deleted file mode 100644 index c2ee6bdf6b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/vpc/outputs.tf +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the new VPC network" - value = module.vpc.network_name - depends_on = [module.vpc, module.cloud_router] -} - -output "network_id" { - description = "ID of the new VPC network" - value = module.vpc.network_id - depends_on = [module.vpc, module.cloud_router] -} - -output "network_self_link" { - description = "Self link of the new VPC network" - value = module.vpc.network_self_link - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetworks" { - description = "Full list of subnetwork objects belonging to the new VPC network" - value = module.vpc.subnets - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork" { - description = "Primary subnetwork object" - value = local.output_primary_subnetwork - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_name" { - description = "Name of the primary subnetwork" - value = local.output_primary_subnetwork_name - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_self_link" { - description = "Self link of the primary subnetwork" - value = local.output_primary_subnetwork_self_link - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_address" { - description = "IP address range of the primary subnetwork" - value = local.output_primary_subnetwork_ip_cidr_range - depends_on = [module.vpc, module.cloud_router] -} - -output "nat_ips" { - description = "External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used)" - value = flatten([for ipmod in module.nat_ip_addresses : ipmod.addresses]) -} diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/variables.tf b/deletion-test/primary/modules/embedded/modules/network/vpc/variables.tf deleted file mode 100644 index e036189404..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/vpc/variables.tf +++ /dev/null @@ -1,301 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "labels" { - description = "Labels to add to network resources that support labels. Key-value pairs of strings." - type = map(string) - default = {} - nullable = false -} - -variable "network_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" - type = string - default = null -} - -variable "subnetwork_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-primary-subnet\")" - type = string - default = null -} - -# tflint-ignore: terraform_unused_declarations -variable "subnetwork_size" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = number - default = null - validation { - condition = var.subnetwork_size == null - error_message = "subnetwork_size is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -variable "default_primary_subnetwork_size" { - description = "The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks" - type = number - default = 15 -} - -variable "region" { - description = "The default region for Cloud resources" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "network_address_range" { - description = "IP address range (CIDR) for global network" - type = string - default = "10.0.0.0/9" - - validation { - condition = can(cidrhost(var.network_address_range, 0)) - error_message = "IP address range must be in CIDR format." - } -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "subnetworks" { - description = <<-EOT - List of subnetworks to create within the VPC. If left empty, it will be - replaced by a single, default subnetwork constructed from other parameters - (e.g. var.region). In all cases, the first subnetwork in the list is identified - by outputs as a "primary" subnetwork. - - subnet_name (string, required, name of subnet) - subnet_region (string, required, region of subnet) - subnet_ip (string, mutually exclusive with new_bits, CIDR-formatted IP range for subnetwork) - new_bits (number, mutually exclusive with subnet_ip, CIDR bits used to calculate subnetwork range) - subnet_private_access (bool, optional, Enable Private Access on subnetwork) - subnet_flow_logs (map(string), optional, Configure Flow Logs see terraform-google-network module) - description (string, optional, Description of Network) - purpose (string, optional, related to Load Balancing) - role (string, optional, related to Load Balancing) - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_name"]) - ]) - error_message = "All subnetworks must define \"subnet_name\"." - } - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_region"]) - ]) - error_message = "All subnetworks must define \"subnet_region\"." - } - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_ip"]) != can(s["new_bits"]) - ]) - error_message = "All subnetworks must define exactly one of \"subnet_ip\" or \"new_bits\"." - } - validation { - condition = alltrue([for s in var.subnetworks : can(s["subnet_ip"])]) || alltrue([for s in var.subnetworks : can(s["new_bits"])]) - error_message = "All subnetworks must make same choice of \"subnet_ip\" or \"new_bits\"." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "primary_subnetwork" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = map(string) - default = null - validation { - condition = var.primary_subnetwork == null - error_message = "primary_subnetwork is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "additional_subnetworks" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = list(map(string)) - default = null - validation { - condition = var.additional_subnetworks == null - error_message = "additional_subnetworks is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -variable "secondary_ranges" { - type = map(list(object({ range_name = string, ip_cidr_range = string }))) - description = <<-EOT - "Secondary ranges associated with the subnets. - This will be deprecated in favour of secondary_ranges_list at a later date. - Please migrate to using the same." - EOT - default = {} -} - -variable "secondary_ranges_list" { - type = list(object({ - subnetwork_name = string, - ranges = list(object({ - range_name = string, - ip_cidr_range = string - })) - })) - description = <<-EOT - "List of secondary ranges associated with the subnetworks. - Each subnetwork must be specified at most once in this list." - EOT - default = [] - validation { - condition = (length(var.secondary_ranges_list[*].subnetwork_name) == - length(distinct(var.secondary_ranges_list[*].subnetwork_name))) - error_message = "Each subnetwork should be specified at most once in this list. Remove any duplicates." - } -} - -variable "network_routing_mode" { - type = string - default = "GLOBAL" - description = "The network routing mode (default \"GLOBAL\")" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "ips_per_nat" { - type = number - description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT). The number of NAT IPs depend on the port reservation allocated for each node and the number of ports that a single NAT IP can serve. Refer this documentation for more details: https://cloud.google.com/nat/docs/ports-and-addresses#port-reservation-examples" - default = 2 -} - -variable "shared_vpc_host" { - type = bool - description = "Makes this project a Shared VPC host if 'true' (default 'false')" - default = false -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_iap_ssh_ingress" { - type = bool - description = "Enable a firewall rule to allow SSH access using IAP tunnels" - default = true -} - -variable "enable_iap_rdp_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" - default = false -} - -variable "enable_iap_winrm_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" - default = false -} - -variable "enable_internal_traffic" { - type = bool - description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" - default = true -} - -variable "enable_cloud_router" { - type = bool - description = "Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview" - default = true -} - -variable "enable_cloud_nat" { - type = bool - description = "Enable the creation of Cloud NATs." - default = true -} - -variable "extra_iap_ports" { - type = list(string) - description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" - default = [] -} - -variable "allowed_ssh_ip_ranges" { - type = list(string) - description = "A list of CIDR IP ranges from which to allow ssh access" - default = [] - - validation { - condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) - error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." - } -} - -variable "firewall_rules" { - type = any - description = "List of firewall rules" - default = [] -} - -variable "firewall_log_config" { - type = string - description = "Firewall log configuration for Toolkit firewall rules (var.enable_iap_ssh_ingress and others)" - default = "DISABLE_LOGGING" - nullable = false - - validation { - condition = contains([ - "INCLUDE_ALL_METADATA", - "EXCLUDE_ALL_METADATA", - "DISABLE_LOGGING", - ], var.firewall_log_config) - error_message = "var.firewall_log_config must be set to \"DISABLE_LOGGING\", or enable logging with \"INCLUDE_ALL_METADATA\" or \"EXCLUDE_ALL_METADATA\"" - } -} - -variable "network_profile" { - type = string - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown - EOT - default = null -} diff --git a/deletion-test/primary/modules/embedded/modules/network/vpc/versions.tf b/deletion-test/primary/modules/embedded/modules/network/vpc/versions.tf deleted file mode 100644 index 71b7106734..0000000000 --- a/deletion-test/primary/modules/embedded/modules/network/vpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 0.15.0" -} diff --git a/deletion-test/primary/modules/embedded/modules/packer/custom-image/README.md b/deletion-test/primary/modules/embedded/modules/packer/custom-image/README.md deleted file mode 100644 index 192d7575a5..0000000000 --- a/deletion-test/primary/modules/embedded/modules/packer/custom-image/README.md +++ /dev/null @@ -1,320 +0,0 @@ -# Custom Images in the Cluster Toolkit (formerly HPC Toolkit) - -Please review the -[introduction to image building](../../../docs/image-building.md) for general -information on building custom images using the Toolkit. - -## Introduction - -This module uses [Packer](https://www.packer.io/) to create an image within an -Cluster Toolkit deployment. Packer operates by provisioning a short-lived VM in -Google Cloud on which it executes scripts to customize the boot disk for -repeated use. The VM's boot disk is specified from a source image that defaults -to the [HPC VM Image][hpcimage]. This Packer "template" supports customization -by the following approaches following a [recommended use](#recommended-use): - -- [startup-script metadata][startup-metadata] from [raw string][sss] or - [file][ssf] -- [Shell scripts][shell] uploaded from the Packer execution environment to the - VM -- [Ansible playbooks][ansible] uploaded from the Packer execution environment to - the VM - -They can be specified independently of one another, so that anywhere from 1 to 3 -solutions can be used simultaneously. In the case that 0 scripts are supplied, -the source boot disk is effectively copied to your project without -customization. This can be useful in scenarios where increased control over the -image maintenance lifecycle is desired or when policies restrict the use of -images to internal projects. - -## Minimum requirements - -### Outbound internet access - -Most customization scripts require access to resources on the public internet. -This can be achieved by one of the following 2 approaches: - -1. Using a public IP address on the VM - -- Set [var.omit_external_ip](#input_omit_external_ip) to `false` - -1. Configuring a VPC with a Cloud NAT in the region of the VM - -- Use the [vpc] module which automates NAT creation - -### Inbound internet access - -Read [order of execution](#order-of-execution) below for a discussion of VM -customization solutions and their requirements for inbound SSH access. -[Environments without SSH access](#environments-without-ssh-access) should use -the metadata-based startup-script solution. - -A simple way to enable inbound SSH access is to use the VPC module with -`allowed_ssh_ip_ranges` set to `0.0.0.0/0`. - -### User or service account executing Packer at command line - -The user or service account running Packer must have the permission to create -VMs in the selected VPC network and, if [use\_iap](#input_use_iap) is set, must -have the "IAP-Secured Tunnel User" role. Recommended roles are: - -- `roles/compute.instanceAdmin.v1` -- `roles/iap.tunnelResourceAccessor` - -### VM service account roles - -The service account attached to the temporary build VM created by Packer should -have the ability to write Cloud Logging entries so that you may inspect and -debug build logs. When using the metadata startup-script customization solution, -the service account attached to the temporary build VM created by Packer must -have the permission to modify its own metadata and to read from Cloud Storage -buckets. Recommended roles are: - -- `roles/compute.instanceAdmin.v1` -- `roles/iam.serviceAccountUser` -- `roles/logging.logWriter` -- `roles/monitoring.metricWriter` -- `roles/storage.objectViewer` - -It is recommended to create this service account as a separate step outside a -blueprint due to known delay in [IAM bindings propagation][iamprop]. - -## Example blueprints - -A recommended pattern for building images with this module is to use the -terraform based [startup-script] module along with this packer custom-image -module. Below you can find links to several examples of this pattern, including -usage instructions. - -### [Image Builder] - -The [Image Builder] blueprint demonstrates a solution that builds an image -using: - -- The [HPC VM Image][hpcimage] as a base upon which to customize -- A VPC network with firewall rules that allow IAP-based SSH tunnels -- A Toolkit runner that installs a custom script - -Please review the [examples README] for usage instructions. - -## Order of execution - -The startup script specified in metadata executes in parallel with the other -supported methods. However, the remaining methods execute in a well-defined -order relative to one another. - -1. All shell scripts will execute in the configured order -1. After shell scripts complete, all Ansible playbooks will execute in the - configured order - -> **_NOTE:_** if both [startup_script][sss] and [startup_script_file][ssf] are -> specified, then [startup_script_file][ssf] takes precedence. - -## Recommended use - -Because the [metadata startup script executes in parallel](#order-of-execution) -with the other solutions, conflicts can arise, especially when package managers -(`yum` or `apt`) lock their databases during package installation. Therefore, it -is recommended to choose one of the following approaches: - -1. Specify _either_ [startup_script][sss] _or_ [startup_script_file][ssf] and do - not specify [shell_scripts][shell] or [ansible_playbooks][ansible]. - - This can be especially useful in - [environments that restrict SSH access](#environments-without-ssh-access) -1. Specify any combination of [shell_scripts][shell] and - [ansible_playbooks][ansible] and do not specify [startup_script][sss] or - [startup_script_file][ssf]. - -If any of the startup script approaches fail by returning a code other than 0, -Packer will determine that the build has failed and refuse to save the image. - -## External access with SSH - -The [shell scripts][shell] and [Ansible playbooks][ansible] customization -solutions both require SSH access to the VM from the Packer execution -environment. SSH access can be enabled one of 2 ways: - -1. The VM is created without a public IP address and SSH tunnels are created - using [Identity-Aware Proxy (IAP)][iaptunnel]. - - Allow [use_iap](#input_use_iap) to take on its default value of `true` -1. The VM is created with an IP address on the public internet and firewall - rules allow SSH access from the Packer execution environment. - - Set `omit_external_ip = false` (or `omit_external_ip: false` in a - blueprint) - - Add firewall rules that open SSH to the VM - -The Packer template defaults to using to the 1st IAP-based solution because it -is more secure (no exposure to public internet) and because the [vpc] module -automatically sets up all necessary firewall rules for SSH tunneling and -outbound-only access to the internet through [Cloud NAT][cloudnat]. - -In either SSH solution, customization scripts should be supplied as files in the -[shell_scripts][shell] and [ansible_playbooks][ansible] settings. - -## Environments without SSH access - -Many network environments disallow SSH access to VMs. In these environments, the -[metadata-based startup scripts][startup-metadata] are appropriate because they -execute entirely independently of the Packer execution environment. - -In this scenario, a single scripts should be supplied in the form of a string to -the [startup_script][sss] input variable. This solution integrates well with -Toolkit runners. Runners operate by using a single startup script whose behavior -is extended by downloading and executing a customizable set of runners from -Cloud Storage at startup. - -> **_NOTE:_** Packer will attempt to use SSH if either [shell_scripts][shell] or -> [ansible_playbooks][ansible] are set to non-empty values. Leave them at their -> default, empty values to ensure access by SSH is disabled. - -## Supplying startup script as a string - -The [startup_script][sss] parameter accepts scripts formatted as strings. In -Packer and Terraform, multi-line strings can be specified using -[heredoc syntax](https://www.terraform.io/language/expressions/strings#heredoc-strings) -in an input [Packer variables file][pkrvars] (`*.pkrvars.hcl`) For example, the -following snippet defines a multi-line bash script followed by an integer -representing the size, in GiB, of the resulting image: - -```hcl -startup_script = <<-EOT - #!/bin/bash - yum install -y epel-release - yum install -y jq - EOT - -disk_size = 100 -``` - -In a blueprint, the equivalent syntax is: - -```yaml -... - settings: - startup_script: | - #!/bin/bash - yum install -y epel-release - yum install -y jq - disk_size: 100 -... -``` - -## Monitoring startup script execution - -When using startup script customization, Packer will print very limited output -to the console. For example: - -```text -==> example.googlecompute.toolkit_image: Waiting for any running startup script to finish... -==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... -==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... -==> example.googlecompute.toolkit_image: Startup script, if any, has finished running. -``` - -### Debugging startup-script failures - -> [!NOTE] -> There can be a delay in the propagation of the logs from the instance to -> Cloud Logging, so it may require waiting a few minutes to see the full logs. - -If the Packer image build fails, the module will output a `gcloud` command -that can be used directly to review startup-script execution. - -## License - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. You may obtain a copy of the -License at - -```text - http://www.apache.org/licenses/LICENSE-2.0 -``` - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. - - -## Requirements - -No requirements. - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_count](#input\_accelerator\_count) | Number of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `number` | `null` | no | -| [accelerator\_type](#input\_accelerator\_type) | Type of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `string` | `null` | no | -| [ansible\_playbooks](#input\_ansible\_playbooks) | A list of Ansible playbook configurations that will be uploaded to customize the VM image | 
list(object({
    playbook_file   = string
    galaxy_file     = string
    extra_arguments = list(string)
  }))
| `[]` | no | -| [communicator](#input\_communicator) | Communicator to use for provisioners that require access to VM ("ssh" or "winrm") | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name | `string` | n/a | yes | -| [disk\_size](#input\_disk\_size) | Size of disk image in GB | `number` | `null` | no | -| [disk\_type](#input\_disk\_type) | Type of persistent disk to provision | `string` | `"pd-balanced"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [image\_family](#input\_image\_family) | The family name of the image to be built. Defaults to `deployment_name` | `string` | `null` | no | -| [image\_name](#input\_image\_name) | The name of the image to be built. If not supplied, it will be set to image\_family-$ISO\_TIMESTAMP | `string` | `null` | no | -| [image\_storage\_locations](#input\_image\_storage\_locations) | Storage location, either regional or multi-regional, where snapshot content is to be stored and only accepts 1 value.
See https://developer.hashicorp.com/packer/plugins/builders/googlecompute#image_storage_locations | `list(string)` | `null` | no | -| [labels](#input\_labels) | Labels to apply to the short-lived VM | `map(string)` | `null` | no | -| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2-standard-4"` | no | -| [manifest\_file](#input\_manifest\_file) | File to which to write Packer build manifest | `string` | `"packer-manifest.json"` | no | -| [metadata](#input\_metadata) | Instance metadata for the builder VM (use var.startup\_script or var.startup\_script\_file to set startup-script metadata) | `map(string)` | `{}` | no | -| [network\_project\_id](#input\_network\_project\_id) | Project ID of Shared VPC network | `string` | `null` | no | -| [omit\_external\_ip](#input\_omit\_external\_ip) | Provision the image building VM without a public IP address | `bool` | `true` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which to create VM and image | `string` | n/a | yes | -| [scopes](#input\_scopes) | DEPRECATED: use var.service\_account\_scopes | `set(string)` | `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | The service account email to use. If null or 'default', then the default Compute Engine service account will be used. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Service account scopes to attach to the instance. See
https://cloud.google.com/compute/docs/access/service-accounts. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shell\_scripts](#input\_shell\_scripts) | A list of paths to local shell scripts which will be uploaded to customize the VM image | `list(string)` | `[]` | no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [source\_image](#input\_source\_image) | Source OS image to build from | `string` | `null` | no | -| [source\_image\_family](#input\_source\_image\_family) | Alternative to source\_image. Specify image family to build from latest image in family | `string` | `"hpc-rocky-linux-8"` | no | -| [source\_image\_project\_id](#input\_source\_image\_project\_id) | A list of project IDs to search for the source image. Packer will search the
first project ID in the list first, and fall back to the next in the list,
until it finds the source image. | `list(string)` | `null` | no | -| [ssh\_username](#input\_ssh\_username) | Username to use for SSH access to VM | `string` | `"hpc-toolkit-packer"` | no | -| [startup\_script](#input\_startup\_script) | Startup script (as raw string) used to build the custom Linux VM image (overridden by var.startup\_script\_file if both are set) | `string` | `null` | no | -| [startup\_script\_file](#input\_startup\_script\_file) | File path to local shell script that will be used to customize the Linux VM image (overrides var.startup\_script) | `string` | `null` | no | -| [state\_timeout](#input\_state\_timeout) | The time to wait for instance state changes, including image creation | `string` | `"10m"` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of subnetwork in which to provision image building VM | `string` | n/a | yes | -| [tags](#input\_tags) | Assign network tags to apply firewall rules to VM instance | `list(string)` | `null` | no | -| [use\_iap](#input\_use\_iap) | Use IAP proxy when connecting by SSH | `bool` | `true` | no | -| [use\_os\_login](#input\_use\_os\_login) | Use OS Login when connecting by SSH | `bool` | `false` | no | -| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | A list of strings containing PowerShell scripts which will customize a Windows VM image (requires WinRM communicator) | `list(string)` | `[]` | no | -| [wrap\_startup\_script](#input\_wrap\_startup\_script) | Wrap startup script with Packer-generated wrapper | `bool` | `true` | no | -| [zone](#input\_zone) | Cloud zone in which to provision image building VM | `string` | n/a | yes | - -## Outputs - -No outputs. - - -[ansible]: #input_ansible_playbooks -[cloudnat]: https://cloud.google.com/nat/docs/overview -[examples readme]: ../../../examples/README.md#image-builderyaml- -[hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[iamprop]: https://cloud.google.com/iam/docs/access-change-propagation -[iaptunnel]: https://cloud.google.com/iap/docs/using-tcp-forwarding -[image builder]: ../../../examples/image-builder.yaml -[logging-console]: https://console.cloud.google.com/logs/ -[logging-read-docs]: https://cloud.google.com/sdk/gcloud/reference/logging/read -[pkrvars]: https://www.packer.io/guides/hcl/variables#from-a-file -[shell]: #input_shell_scripts -[ssf]: #input_startup_script_file -[sss]: #input_startup_script -[startup-metadata]: https://cloud.google.com/compute/docs/instances/startup-scripts/linux -[startup-script]: ../../../modules/scripts/startup-script -[vpc]: ../../network/vpc/README.md diff --git a/deletion-test/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl b/deletion-test/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl deleted file mode 100644 index 9282cf7433..0000000000 --- a/deletion-test/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "custom-image", ghpc_role = "packer" }) - - # construct a unique image name from the image family - image_family = var.image_family != null ? var.image_family : var.deployment_name - image_name_default = "${local.image_family}-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" - image_name = var.image_name != null ? var.image_name : local.image_name_default - - # construct vm image name for use when getting logs - instance_name = "packer-${substr(uuidv4(), 0, 6)}" - - # default to explicit var.communicator, otherwise in-order: ssh/winrm/none - shell_script_communicator = length(var.shell_scripts) > 0 ? "ssh" : "" - ansible_playbook_communicator = length(var.ansible_playbooks) > 0 ? "ssh" : "" - powershell_script_communicator = length(var.windows_startup_ps1) > 0 ? "winrm" : "" - communicator = coalesce( - var.communicator, - local.shell_script_communicator, - local.ansible_playbook_communicator, - local.powershell_script_communicator, - "none" - ) - - # must not enable IAP when no communicator is in use - use_iap = local.communicator == "none" ? false : var.use_iap - - # construct metadata from startup_script and metadata variables - startup_script_metadata = var.startup_script == null ? {} : { startup-script = var.startup_script } - - linux_user_metadata = { - block-project-ssh-keys = "TRUE" - shutdown-script = <<-EOT - #!/bin/bash - userdel -r ${var.ssh_username} - sed -i '/${var.ssh_username}/d' /var/lib/google/google_users - EOT - } - windows_packer_user = "packer_user" - windows_user_metadata = { - sysprep-specialize-script-cmd = "winrm quickconfig -quiet & net user /add ${local.windows_packer_user} & net localgroup administrators ${local.windows_packer_user} /add & winrm set winrm/config/service/auth @{Basic=\\\"true\\\"}" - windows-shutdown-script-cmd = <<-EOT - net user /delete ${local.windows_packer_user} - EOT - } - user_metadata = local.communicator == "winrm" ? local.windows_user_metadata : local.linux_user_metadata - - # merge metadata such that var.metadata always overrides user management - # metadata but always allow var.startup_script to override var.metadata - metadata = merge( - local.user_metadata, - var.metadata, - local.startup_script_metadata, - ) - - # determine best value for on_host_maintenance if not supplied by user - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - gpu_attached = contains(["a2", "g2"], local.machine_family) || var.accelerator_type != null - on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" - on_host_maintenance = ( - var.on_host_maintenance != null - ? var.on_host_maintenance - : local.on_host_maintenance_default - ) - - accelerator_type = var.accelerator_type == null ? null : "projects/${var.project_id}/zones/${var.zone}/acceleratorTypes/${var.accelerator_type}" - - winrm_username = local.communicator == "winrm" ? "packer_user" : null - winrm_insecure = local.communicator == "winrm" ? true : null - winrm_use_ssl = local.communicator == "winrm" ? true : null - - enable_integrity_monitoring = var.enable_shielded_vm && var.shielded_instance_config.enable_integrity_monitoring - enable_secure_boot = var.enable_shielded_vm && var.shielded_instance_config.enable_secure_boot - enable_vtpm = var.enable_shielded_vm && var.shielded_instance_config.enable_vtpm - - image_licenses = [ - "projects/click-to-deploy-images/global/licenses/hpc-toolkit-vm-image" - ] -} - -source "googlecompute" "toolkit_image" { - communicator = local.communicator - project_id = var.project_id - image_name = local.image_name - image_family = local.image_family - image_labels = local.labels - instance_name = local.instance_name - machine_type = var.machine_type - accelerator_type = local.accelerator_type - accelerator_count = var.accelerator_count - on_host_maintenance = local.on_host_maintenance - disk_size = var.disk_size - disk_type = var.disk_type - omit_external_ip = var.omit_external_ip - use_internal_ip = var.omit_external_ip - subnetwork = var.subnetwork_name - network_project_id = var.network_project_id - service_account_email = var.service_account_email - scopes = var.service_account_scopes - source_image = var.source_image - source_image_family = var.source_image_family - source_image_project_id = var.source_image_project_id - ssh_username = var.ssh_username - tags = var.tags - use_iap = local.use_iap - use_os_login = var.use_os_login - winrm_username = local.winrm_username - winrm_insecure = local.winrm_insecure - winrm_use_ssl = local.winrm_use_ssl - zone = var.zone - labels = local.labels - metadata = local.metadata - startup_script_file = var.startup_script_file - wrap_startup_script = var.wrap_startup_script - state_timeout = var.state_timeout - image_storage_locations = var.image_storage_locations - enable_secure_boot = local.enable_secure_boot - enable_vtpm = local.enable_vtpm - enable_integrity_monitoring = local.enable_integrity_monitoring - image_licenses = local.image_licenses -} - -build { - name = var.deployment_name - sources = ["sources.googlecompute.toolkit_image"] - - # using dynamic blocks to create provisioners ensures that there are no - # provisioner blocks when none are provided and we can use the none - # communicator when using startup-script - - # provisioner "shell" blocks - dynamic "provisioner" { - labels = ["shell"] - for_each = var.shell_scripts - content { - execute_command = "sudo -H sh -c '{{ .Vars }} {{ .Path }}'" - script = provisioner.value - } - } - - # provisioner "powershell" blocks - dynamic "provisioner" { - labels = ["powershell"] - for_each = var.windows_startup_ps1 - content { - inline = split("\n", provisioner.value) - } - } - - dynamic "provisioner" { - labels = ["powershell"] - for_each = length(var.windows_startup_ps1) > 0 ? [1] : [] - content { - inline = [ - "GCESysprep -no_shutdown" - ] - } - } - - # provisioner "ansible-local" blocks - # this installs custom roles/collections from ansible-galaxy in /home/packer - # which will be removed at the end; consider modifying /etc/ansible/ansible.cfg - dynamic "provisioner" { - labels = ["ansible-local"] - for_each = var.ansible_playbooks - content { - playbook_file = provisioner.value.playbook_file - galaxy_file = provisioner.value.galaxy_file - extra_arguments = provisioner.value.extra_arguments - } - } - - post-processor "manifest" { - output = var.manifest_file - strip_path = true - custom_data = { - built-by = "cloud-hpc-toolkit" - } - } - - # If there is an error during image creation, print out command for getting packer VM logs - error-cleanup-provisioner "shell-local" { - environment_vars = [ - "PRJ_ID=${var.project_id}", - "INST_NAME=${local.instance_name}", - "ZONE=${var.zone}", - ] - inline_shebang = "/bin/bash -e" - inline = [ - "type -P gcloud > /dev/null || exit 0", - "INST_ID=$(gcloud compute instances describe $INST_NAME --project $PRJ_ID --format=\"value(id)\" --zone=$ZONE)", - "echo 'Error building image try checking logs:'", - join(" ", ["echo \"gcloud logging --project $PRJ_ID read", - "'logName=(\\\"projects/$PRJ_ID/logs/GCEMetadataScripts\\\" OR \\\"projects/$PRJ_ID/logs/google_metadata_script_runner\\\") AND resource.labels.instance_id=$INST_ID'", - "--format=\\\"table(timestamp, resource.labels.instance_id, jsonPayload.message)\\\"", - "--order=asc\"" - ] - ) - ] - } -} diff --git a/deletion-test/primary/modules/embedded/modules/packer/custom-image/metadata.yaml b/deletion-test/primary/modules/embedded/modules/packer/custom-image/metadata.yaml deleted file mode 100644 index 23108c4e17..0000000000 --- a/deletion-test/primary/modules/embedded/modules/packer/custom-image/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - logging.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl b/deletion-test/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl deleted file mode 100644 index 3cede102ce..0000000000 --- a/deletion-test/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "deployment_name" { - description = "Cluster Toolkit deployment name" - type = string -} - -variable "project_id" { - description = "Project in which to create VM and image" - type = string -} - -variable "machine_type" { - description = "VM machine type on which to build new image" - type = string - default = "n2-standard-4" -} - -variable "disk_size" { - description = "Size of disk image in GB" - type = number - default = null -} - -variable "disk_type" { - description = "Type of persistent disk to provision" - type = string - default = "pd-balanced" -} - -variable "zone" { - description = "Cloud zone in which to provision image building VM" - type = string -} - -variable "network_project_id" { - description = "Project ID of Shared VPC network" - type = string - default = null -} - -variable "subnetwork_name" { - description = "Name of subnetwork in which to provision image building VM" - type = string -} - -variable "omit_external_ip" { - description = "Provision the image building VM without a public IP address" - type = bool - default = true -} - -variable "tags" { - description = "Assign network tags to apply firewall rules to VM instance" - type = list(string) - default = null -} - -variable "image_family" { - description = "The family name of the image to be built. Defaults to `deployment_name`" - type = string - default = null -} - -variable "image_name" { - description = "The name of the image to be built. If not supplied, it will be set to image_family-$ISO_TIMESTAMP" - type = string - default = null -} - -variable "source_image_project_id" { - description = < -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 4.0 | -| [local](#requirement\_local) | >= 2.0.0 | -| [null](#requirement\_null) | ~> 3.0 | -| [random](#requirement\_random) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.0 | -| [local](#provider\_local) | >= 2.0.0 | -| [null](#provider\_null) | ~> 3.0 | -| [random](#provider\_random) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance\_template](#module\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [local_file.submit_script](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [null_resource.submit_job](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.submit_job_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used for the job\_id | `string` | n/a | yes | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs | `bool` | `true` | no | -| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions. Valid inputs are `"alpha"`, `"beta"` and "" (empty string for default version) | `string` | `""` | no | -| [image](#input\_image) | DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided. | `any` | `null` | no | -| [instance\_image](#input\_instance\_image) | Google Cloud Batch compute node image. Ignored if `instance_template` is provided.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [instance\_template](#input\_instance\_template) | Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions. | `string` | `null` | no | -| [job\_filename](#input\_job\_filename) | The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified | `string` | `null` | no | -| [job\_id](#input\_job\_id) | An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs. | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided. | `map(string)` | n/a | yes | -| [log\_policy](#input\_log\_policy) | Create a block to define log policy.
When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging.
When set to `PATH`, path must be added to generated template.
When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. | `string` | `"CLOUD_LOGGING"` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided. | `string` | `"n2-standard-4"` | no | -| [mpi\_mode](#input\_mpi\_mode) | Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`. | `bool` | `false` | no | -| [native\_batch\_mounting](#input\_native\_batch\_mounting) | Batch can mount some fs\_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts. | `bool` | `true` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which to run the Google Cloud Batch job | `string` | n/a | yes | -| [runnable](#input\_runnable) | A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`. | `string` | `null` | no | -| [runnables](#input\_runnables) | A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template. | 
list(object({
    script = string
  }))
| `null` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/devstorage.read_only",
    "https://www.googleapis.com/auth/logging.write",
    "https://www.googleapis.com/auth/monitoring.write",
    "https://www.googleapis.com/auth/servicecontrol",
    "https://www.googleapis.com/auth/service.management.readonly",
    "https://www.googleapis.com/auth/trace.append"
  ]
}
| no | -| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided. | `string` | `null` | no | -| [submit](#input\_submit) | When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply. | `bool` | `false` | no | -| [subnetwork](#input\_subnetwork) | The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided. | `any` | `null` | no | -| [task\_count](#input\_task\_count) | Number of parallel tasks | `number` | `1` | no | -| [task\_count\_per\_node](#input\_task\_count\_per\_node) | Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value. | `number` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [gcloud\_version](#output\_gcloud\_version) | The version of gcloud to be used. | -| [instance\_template](#output\_instance\_template) | Instance template used by the Batch job. | -| [instructions](#output\_instructions) | Instructions for submitting the Batch job. | -| [job\_data](#output\_job\_data) | All data associated with the defined job, typically provided as input to clout-batch-login-node. | -| [network\_storage](#output\_network\_storage) | An array of network attached storage mounts used by the Batch job. | -| [startup\_script](#output\_startup\_script) | Startup script run before Google Cloud Batch job starts. | - diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf deleted file mode 100644 index 0d681536c9..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "batch-job-template", ghpc_role = "scheduler" }) -} - -locals { - instance_template = coalesce(var.instance_template, module.instance_template.self_link) - - tasks_per_node = var.task_count_per_node != null ? var.task_count_per_node : (var.mpi_mode ? 1 : null) - - one_line_runnable = coalesce(var.runnable, "## Add your workload here ##") - runnables = coalesce(var.runnables, [{ script = local.one_line_runnable }]) - - job_template_contents = templatefile( - "${path.module}/templates/batch-job-base.yaml.tftpl", - { - synchronized = var.mpi_mode - runnables = local.runnables - task_count = var.task_count - tasks_per_node = local.tasks_per_node - require_hosts_file = var.mpi_mode - permissive_ssh = var.mpi_mode - log_policy = var.log_policy - instance_template = local.instance_template - nfs_volumes = local.native_batch_network_storage - labels = local.labels - } - ) - - submit_job_id = "${var.job_id}-${random_id.submit_job_suffix.hex}" - job_filename = coalesce(var.job_filename, "${var.job_id}.yaml") - job_template_output_path = "${path.root}/${local.job_filename}" - - submit_script_contents = templatefile( - "${path.module}/templates/batch-submit.sh.tftpl", - { - project = var.project_id - location = var.region - config = local_file.job_template.filename - submit_job_id = local.submit_job_id - } - ) - submit_script_output_path = "${path.root}/submit-${var.job_id}.sh" - - subnetwork_name = var.subnetwork != null ? var.subnetwork.name : "default" - subnetwork_project = var.subnetwork != null ? var.subnetwork.project : var.project_id - - # Filter network_storage for native Batch support - native_fstype = var.native_batch_mounting ? ["nfs"] : [] - native_batch_network_storage = [ - for ns in var.network_storage : - ns if contains(local.native_fstype, ns.fs_type) - ] - # other processing happens in startup_from_network_storage.tf - - # this code is similar to code in Packer and vm-instance modules - # it differs in that this module does not (yet) expose var.guest_acclerator - # for attaching GPUs to N1 VMs. For now, identify only A2 types. - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - gpu_attached = contains(["a2", "g2"], local.machine_family) - on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" - - on_host_maintenance = coalesce(var.on_host_maintenance, local.on_host_maintenance_default) - - network_storage_metadata = var.network_storage != null ? ({ network_storage = jsonencode(var.network_storage) }) : {} - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.network_storage_metadata, - local.disable_automatic_updates_metadata - ) -} - -module "instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = var.instance_template == null ? "${var.job_id}-instance-template" : "unused-template" - project_id = var.project_id - subnetwork = local.subnetwork_name - subnetwork_project = local.subnetwork_project - service_account = var.service_account - access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - labels = local.labels - - machine_type = var.machine_type - startup_script = local.startup_from_network_storage - metadata = local.metadata - source_image_family = data.google_compute_image.compute_image.family - source_image = data.google_compute_image.compute_image.name - source_image_project = data.google_compute_image.compute_image.project - on_host_maintenance = local.on_host_maintenance -} - -resource "local_file" "job_template" { - content = local.job_template_contents - filename = local.job_template_output_path - - lifecycle { - precondition { - condition = var.runnable == null || var.runnables == null - error_message = "var.runnable and var.runnables (plural) cannot both be set." - } - } -} - -resource "random_id" "submit_job_suffix" { - byte_length = 4 - keepers = { - always_run = timestamp() - } -} - -resource "local_file" "submit_script" { - content = local.submit_script_contents - filename = local.submit_script_output_path -} - -resource "null_resource" "submit_job" { - depends_on = [local_file.job_template, local_file.submit_script] - count = var.submit ? 1 : 0 - - # A new deployment should always submit a new job. Old finished jobs aren't persistent parts of - # Cloud infrastructure. - triggers = { - always_run = timestamp() - } - - provisioner "local-exec" { - command = local.submit_script_output_path - } -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml deleted file mode 100644 index 387e810962..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - batch.googleapis.com - - compute.googleapis.com -ghpc: - inject_module_id: job_id diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf deleted file mode 100644 index 0b1295975a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - provided_instance_tpl_msg = "The Batch job template uses the existing VM instance template:" - generated_instance_tpl_msg = "The Batch job template uses a new VM instance template created matching the provided settings:" - submit_msg = <<-EOT - - The job has been submitted. See job status at: - https://console.cloud.google.com/batch/jobsDetail/regions/${var.region}/jobs/${local.submit_job_id}?project=${var.project_id} - EOT -} - -output "instructions" { - description = "Instructions for submitting the Batch job." - value = <<-EOT - - A Batch job template file has been created locally at: - ${abspath(local.job_template_output_path)} - - ${var.instance_template == null ? local.generated_instance_tpl_msg : local.provided_instance_tpl_msg} - ${local.instance_template} - ${var.submit ? local.submit_msg : ""} - - Use the following commands to: - Submit your job${var.submit ? " (Note: job has already been submitted)" : ""}: - gcloud ${var.gcloud_version} batch jobs submit ${local.submit_job_id} --config=${abspath(local.job_template_output_path)} --location=${var.region} --project=${var.project_id} - - Check status: - gcloud ${var.gcloud_version} batch jobs describe ${local.submit_job_id} --location=${var.region} --project=${var.project_id} | grep state: - - Delete job: - gcloud ${var.gcloud_version} batch jobs delete ${local.submit_job_id} --location=${var.region} --project=${var.project_id} - - List all jobs: - gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} - EOT -} - -output "job_data" { - description = "All data associated with the defined job, typically provided as input to clout-batch-login-node." - value = { - template_contents = local.job_template_contents, - filename = local.job_filename, - id = local.submit_job_id - } -} - -output "instance_template" { - description = "Instance template used by the Batch job." - value = local.instance_template -} - -output "network_storage" { - description = "An array of network attached storage mounts used by the Batch job." - value = var.network_storage -} - -output "startup_script" { - description = "Startup script run before Google Cloud Batch job starts." - value = var.startup_script -} - -output "gcloud_version" { - description = "The version of gcloud to be used." - value = var.gcloud_version -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf deleted file mode 100644 index 02bc58e4f7..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "inputs": -# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. -# var.network_storage : to be passed into metadata somewhere else (not here) -# var.startup_script : to be changed into a more complete file system with all the fs runners - -# "outputs": -# local.startup_from_network_storage : A full startup script with all the runners that are not supported -# natively and were included in the network_storage structure - -locals { - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - # Pull out runners to include in startup script - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - startup_script_runner = [{ - content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" - destination = "passed_startup_script.sh" - type = "shell" - }] - - full_runner_list = concat( - local.storage_client_install_runners, - local.mount_runners, - local.startup_script_runner - ) - - startup_from_network_storage = module.netstorage_startup_script.startup_script -} - -module "netstorage_startup_script" { - source = "../../scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.full_runner_list -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl deleted file mode 100644 index 83fccde53b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl +++ /dev/null @@ -1,53 +0,0 @@ -taskGroups: - - taskSpec: - runnables: - %{~ if synchronized ~} - - barrier: - name: "wait-for-node-startup" - %{~ endif ~} - %{~ for runnable in runnables ~} - - script: - text: ${indent(12, chomp(yamlencode(runnable.script)))} - %{~ if synchronized ~} - - barrier: - name: "wait-for-script-to-complete" - %{~ endif ~} - %{~ endfor ~} - %{~ if length(nfs_volumes) > 0 ~} - volumes: - %{~ for index, vol in nfs_volumes ~} - - nfs: - server: "${vol.server_ip}" - remotePath: "${vol.remote_mount}" - %{~ if vol.mount_options != "" && vol.mount_options != null ~} - mountOptions: "${vol.mount_options}" - %{~ endif ~} - mountPath: "${vol.local_mount}" - %{~ endfor ~} - %{~ endif ~} - taskCount: ${task_count} - %{~ if tasks_per_node != null ~} - taskCountPerNode: ${tasks_per_node} - %{~ endif ~} - requireHostsFile: ${require_hosts_file} - permissiveSsh: ${permissive_ssh} -%{~ if instance_template != null } -allocationPolicy: - instances: - - instanceTemplate: "${instance_template}" -%{~ endif } -%{~ if log_policy == "CLOUD_LOGGING" } -logsPolicy: - destination: "CLOUD_LOGGING" -%{ endif } -%{~ if log_policy == "PATH" } -logsPolicy: - destination: "PATH" - logsPath: ## Add logging path here -%{ endif } -%{~ if length(labels) > 0 ~} -labels: -%{ for k, v in labels ~} - ${k}: "${v}" -%{ endfor } -%{~ endif ~} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl deleted file mode 100644 index 25f89c3ceb..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e -o pipefail -GCLOUD_MAJOR_VERSION=$(gcloud --version | head -n 1 | awk '{print $NF}' | cut -f1 --delimiter=.) -if [ $((GCLOUD_MAJOR_VERSION >= 461)) ]; then - gcloud batch jobs submit ${submit_job_id} --project=${project} --location=${location} --config=${config} - echo "batch job ${submit_job_id} successfully submitted" -else - echo "gcloud must be updated to version 461.0.0 or later." - exit 1 -fi diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf deleted file mode 100644 index f65fbd111e..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "region" { - description = "The region in which to run the Google Cloud Batch job" - type = string -} - -variable "deployment_name" { - description = "Name of the deployment, used for the job_id" - type = string -} - -variable "labels" { - description = "Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided." - type = map(string) -} - -variable "job_id" { - description = "An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs." - type = string -} - -variable "job_filename" { - description = "The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified" - type = string - default = null -} - -variable "gcloud_version" { - description = "The version of the gcloud cli being used. Used for output instructions. Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default version)" - type = string - default = "" - - validation { - condition = contains(["alpha", "beta", ""], var.gcloud_version) - error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." - } -} - -variable "task_count" { - description = "Number of parallel tasks" - type = number - default = 1 -} - -variable "task_count_per_node" { - description = "Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value." - type = number - default = null -} - -variable "mpi_mode" { - description = "Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`." - type = bool - default = false -} - -variable "log_policy" { - description = <<-EOT - Create a block to define log policy. - When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging. - When set to `PATH`, path must be added to generated template. - When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. - EOT - type = string - default = "CLOUD_LOGGING" - - validation { - condition = contains(["CLOUD_LOGGING", "PATH", "DESTINATION_UNSPECIFIED"], var.log_policy) - error_message = "Allowed values for log_policy are 'CLOUD_LOGGING', 'PATH', or 'DESTINATION_UNSPECIFIED'." - } -} - -variable "runnables" { - description = "A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template." - type = list(object({ - script = string - })) - default = null -} - -variable "runnable" { - description = "A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`." - type = string - default = null -} - -variable "instance_template" { - description = "Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions." - type = string - default = null -} - -variable "subnetwork" { - description = "The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided." - type = any - default = null -} - -variable "enable_public_ips" { - description = "If set to true, instances will have public IPs" - type = bool - default = true -} - -variable "service_account" { - description = "Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided." - type = object({ - email = string, - scopes = set(string) - }) - default = { - email = null - scopes = [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write", - "https://www.googleapis.com/auth/servicecontrol", - "https://www.googleapis.com/auth/service.management.readonly", - "https://www.googleapis.com/auth/trace.append" - ] - } -} - -variable "machine_type" { - description = "Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided." - type = string - default = "n2-standard-4" -} - -variable "startup_script" { - description = "Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided." - type = string - default = null -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided." - type = list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "native_batch_mounting" { - description = "Batch can mount some fs_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts." - type = bool - default = true -} - -# Deprecated, replaced by instance_image -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided." - type = any - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - Google Cloud Batch compute node image. Ignored if `instance_template` is provided. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE`" - type = string - default = null - validation { - condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) - error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." - } -} - -variable "submit" { - description = "When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply." - type = bool - default = false -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf deleted file mode 100644 index a1161e1354..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - random = { - source = "hashicorp/random" - version = ">= 3.0" - } - google = { - source = "hashicorp/google" - version = ">= 4.0" - } - } - required_version = ">= 1.1" -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/README.md b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/README.md deleted file mode 100644 index c20ca7dbeb..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# Description - -This module creates a VM that acts as a login node to test and submit Google -Cloud Batch jobs. It is intended to be used along with the `batch-job-template` -module. - -This login node: - -- Uses the same VM settings as the first provided `batch-job-template`, such as - image, machine type, etc... -- Runs the same `startup-script` as the first provided `batch-job-template`. -- Has the same mounted file systems as the provided `batch-job-template`. -- Contains a folder with job templates generated by `batch-job-template` modules. - -Since the login node has the same mounted storage and is a homogeneous machine -to the Google Cloud Batch compute VMs, it can be used to inspect shared file -systems and test installed software before submitting a Google Cloud Batch job. - -## Example - -```yaml -- id: batch-job - source: modules/scheduler/batch-job-template - ... - -- id: batch-login - source: modules/scheduler/batch-login-node - use: [batch-job] - outputs: [instructions] -``` - -## Authentication - -To submit jobs from the login node, the service account attached to the VM needs -the `Batch Job Administrator` role. In most cases this service account will be -the Compute Engine default service account and will not be granted this role by -default. - -You can grant this role either by adding the `Batch Job Administrator` role to -the service account in the IAM page in the Google Cloud Console, or by running -the following command line: - -```bash -gcloud projects add-iam-policy-binding \ - --member=serviceAccount: \ - --role=roles/batch.jobsAdmin -``` - -## gcloud Batch Access - -Until the Google Cloud Batch API is generally available (GA), it may not be -available in all versions of the `gcloud` cli. You can test if the Google Cloud -Batch commands are available by running `gcloud [alpha|beta|] batch -h`. If the -Google Cloud Batch cli is not available it can generally be mitigated by either -updating `gcloud` by running `gcloud components update`, or using an image that -contains a more recent version of `gcloud`. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [login\_startup\_script](#module\_login\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_from_template.batch_login](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | -| [google_compute_instance_template.batch_instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [batch\_job\_directory](#input\_batch\_job\_directory) | The path of the directory on the login node in which to place the Google Cloud Batch job template | `string` | `"/home/batch-jobs"` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, also used for the job\_id | `string` | n/a | yes | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions.
Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default
version). Typically supplied by a batch-job-template module. If multiple
batch-job-template modules supply the gcloud\_version, only the first will be used. | `string` | `""` | no | -| [instance\_template](#input\_instance\_template) | Login VM instance template self-link. Typically supplied by a
batch-job-template module. If multiple batch-job-template modules supply the
instance\_template, the first will be used. | `string` | n/a | yes | -| [job\_data](#input\_job\_data) | List of jobs and supporting data for each, typically provided via "use" from the batch-job-template module. | 
list(object({
    template_contents = string,
    filename          = string,
    id                = string
  }))
| n/a | yes | -| [job\_filename](#input\_job\_filename) | Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [job\_id](#input\_job\_id) | Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions. | `string` | `null` | no | -| [job\_template\_contents](#input\_job\_template\_contents) | Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the login node. Key-value pairs | `map(string)` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which to create the login node | `string` | n/a | yes | -| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [zone](#input\_zone) | The zone in which to create the login node | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for accessing the login node and submitting Google Cloud Batch jobs | -| [login\_node\_name](#output\_login\_node\_name) | Name of the created VM | - diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf deleted file mode 100644 index 6f539af122..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "batch-login-node", ghpc_role = "scheduler" }) -} - -data "google_compute_instance_template" "batch_instance_template" { - name = var.instance_template -} - -locals { - job_template_runners = [for job in var.job_data : { - content = job.template_contents - destination = "${var.batch_job_directory}/${job.filename}" - type = "data" - }] - - instance_template_metadata = data.google_compute_instance_template.batch_instance_template.metadata - startup_metadata = { startup-script = module.login_startup_script.startup_script } - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - login_metadata = merge(local.instance_template_metadata, local.startup_metadata, local.oslogin_metadata) - - batch_command_instructions = join("\n", [for job in var.job_data : <<-EOT - ## For job: ${job.id} ## - - Submit your job from login node: - gcloud ${var.gcloud_version} batch jobs submit ${job.id} --config=${var.batch_job_directory}/${job.filename} --location=${var.region} --project=${var.project_id} - - Check status: - gcloud ${var.gcloud_version} batch jobs describe ${job.id} --location=${var.region} --project=${var.project_id} | grep state: - - Delete job: - gcloud ${var.gcloud_version} batch jobs delete ${job.id} --location=${var.region} --project=${var.project_id} - - EOT - ]) - - list_all_jobs = <<-EOT - List all jobs: - gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} - EOT - - readme_contents = <<-EOT - # Batch Job Templates - - This folder contains Batch job templates created by the Cluster Toolkit. - These templates can be edited before submitting to Batch to capture more - complex workloads. - - Use the following commands to: - ${local.list_all_jobs} - - ${local.batch_command_instructions} - EOT - - # Construct startup script for network storage - storage_client_install_runners = [ - for i, ns in var.network_storage : merge(ns.client_install_runner, { - destination = "${i}-${ns.client_install_runner.destination}" - }) if ns.client_install_runner != null - ] - mount_runners = [ - for i, ns in var.network_storage : merge(ns.mount_runner, { - destination = "${i}-${ns.mount_runner.destination}" - }) if ns.mount_runner != null - ] - - startup_script_runner = { - content = var.startup_script != null ? var.startup_script : "echo 'Batch job template had no startup script'" - destination = "passed_startup_script.sh" - type = "shell" - } -} - -module "login_startup_script" { - source = "../../scripts/startup-script" - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = concat( - local.storage_client_install_runners, - local.mount_runners, - [local.startup_script_runner], - local.job_template_runners, - [ - { - content = local.readme_contents - destination = "${var.batch_job_directory}/README.md" - type = "data" - } - ] - ) -} - -resource "google_compute_instance_from_template" "batch_login" { - name = "${var.deployment_name}-batch-login" - source_instance_template = var.instance_template - project = var.project_id - zone = var.zone - metadata = local.login_metadata - - service_account { - scopes = ["https://www.googleapis.com/auth/cloud-platform"] - } -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml deleted file mode 100644 index 9af2319b4a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - batch.googleapis.com - - compute.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf deleted file mode 100644 index ea8eccf8d5..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "login_node_name" { - description = "Name of the created VM" - value = google_compute_instance_from_template.batch_login.name -} - -output "instructions" { - description = "Instructions for accessing the login node and submitting Google Cloud Batch jobs" - value = <<-EOT - - Batch job template files will be placed on the Batch login node in the following directory: - ${var.batch_job_directory} - - Use the following commands to: - SSH into the login node: - gcloud compute ssh --zone ${google_compute_instance_from_template.batch_login.zone} ${google_compute_instance_from_template.batch_login.name} --project ${google_compute_instance_from_template.batch_login.project} - - ${local.list_all_jobs} - - ${local.batch_command_instructions} - EOT -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf deleted file mode 100644 index 3b9caa7001..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "deployment_name" { - description = "Name of the deployment, also used for the job_id" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "region" { - description = "The region in which to create the login node" - type = string -} - -variable "zone" { - description = "The zone in which to create the login node" - type = string -} - -variable "labels" { - description = "Labels to add to the login node. Key-value pairs" - type = map(string) -} - -variable "instance_template" { - description = <<-EOT - Login VM instance template self-link. Typically supplied by a - batch-job-template module. If multiple batch-job-template modules supply the - instance_template, the first will be used. - EOT - type = string -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module." - type = list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "startup_script" { - description = "Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module." - type = string - default = null -} - -variable "job_data" { - description = "List of jobs and supporting data for each, typically provided via \"use\" from the batch-job-template module." - type = list(object({ - template_contents = string, - filename = string, - id = string - })) - validation { - condition = length(distinct([for job in var.job_data : job.filename])) == length(var.job_data) - error_message = "All filenames in var.job_data must be unique." - } - validation { - condition = length(distinct([for job in var.job_data : job.id])) == length(var.job_data) - error_message = "All job IDs in var.job_data must be unique." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_template_contents" { - description = "Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module." - type = string - default = null - validation { - condition = var.job_template_contents == null - error_message = "job_template_contents is deprecated. Please use `job_data` instead." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_filename" { - description = "Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module." - type = string - default = null - validation { - condition = var.job_filename == null - error_message = "job_filename is deprecated. Please use `job_data` instead." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_id" { - description = "Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions." - type = string - default = null - validation { - condition = var.job_id == null - error_message = "job_id is deprecated. Please use `job_data` instead." - } -} - -variable "gcloud_version" { - description = <<-EOT - The version of the gcloud cli being used. Used for output instructions. - Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default - version). Typically supplied by a batch-job-template module. If multiple - batch-job-template modules supply the gcloud_version, only the first will be used. - EOT - type = string - default = "" - - validation { - condition = contains(["alpha", "beta", ""], var.gcloud_version) - error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." - } -} - -variable "batch_job_directory" { - description = "The path of the directory on the login node in which to place the Google Cloud Batch job template" - type = string - default = "/home/batch-jobs" -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf b/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf deleted file mode 100644 index 15337a1d7b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.74.0" - } - - required_version = ">= 0.14.0" -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/README.md b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/README.md deleted file mode 100644 index dd4f7fdaa7..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/README.md +++ /dev/null @@ -1,220 +0,0 @@ -## Description - -This module creates a Google Kubernetes Engine -([GKE](https://cloud.google.com/kubernetes-engine)) cluster. - -### Example - -The following example creates a GKE cluster and a VPC designed to work with GKE. -See [VPC Network](#vpc-network) section for more information about network -requirements. - -```yaml - - id: network1 - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### VPC Network - -This module is configured to create a -[VPC-native cluster](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips). -This means that alias IPs are used and that the subnetwork requires secondary -ranges for pods and services. In the example shown above these secondary ranges -are created in the VPC module. By default the `gke-cluster` module will look for -ranges with the names `pods` and `services`. These names can be configured using -the `pods_ip_range_name` and `services_ip_range_name` settings. - -### Multi-networking - -To [enable Multi-networking](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment), pass multivpc module to gke-cluster module as described in example below. Passing a multivpc module enables multi networking and [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en) on the cluster. - -```yaml - - id: network - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: gke-cluster - source: modules/scheduler/gke-cluster - use: [network, multinetwork] ## enables multi networking and Dataplane V2 on cluster - settings: - cluster_name: $(vars.deployment_name) -``` - -Find an example of multi networking in GKE [here](../../../examples/gke-a3-megagpu.yaml). - -### Cluster Limitations - -The current implementations has the following limitations: - -- Autopilot is disabled -- Auto-provisioning of new node pools is disabled -- Network policies are not supported -- General addon configuration is not supported -- Only regional cluster is supported - -### GKE Inference Gateway - -Setting `enable_inference_gateway` to `true` will enable the `HttpLoadBalancing` -addon and deploy the Inference Gateway CRDs. This feature requires a subnet with -`purpose` set to `REGIONAL_MANAGED_PROXY` in the VPC. For more information, see -the [GKE Inference Gateway documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/serve-with-gke-inference-gateway). - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 7.2 | -| [google-beta](#requirement\_google-beta) | >= 7.2 | -| [kubernetes](#requirement\_kubernetes) | >= 2.36 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 7.2 | -| [google-beta](#provider\_google-beta) | >= 7.2 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | -| [workload\_identity](#module\_workload\_identity) | terraform-google-modules/kubernetes-engine/google//modules/workload-identity | >= 40.0 | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_cluster) | resource | -| [google-beta_google_container_node_pool.system_node_pools](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | -| [google-beta_google_container_engine_versions.version_prefix_filter](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/data-sources/google_container_engine_versions) | data source | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks enables multi networking and creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [authenticator\_security\_group](#input\_authenticator\_security\_group) | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| [autoscaling\_profile](#input\_autoscaling\_profile) | (Beta) Optimize for utilization or availability when deciding to remove nodes. Can be BALANCED or OPTIMIZE\_UTILIZATION. | `string` | `"OPTIMIZE_UTILIZATION"` | no | -| [cloud\_dns\_config](#input\_cloud\_dns\_config) | Configuration for Using Cloud DNS for GKE.

additive\_vpc\_scope\_dns\_domain: This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = "CLOUD\_DNS" and cluster\_dns\_scope = "CLUSTER\_SCOPE" must both be set as well.
cluster\_dns: Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS.
cluster\_dns\_scope: The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE.
cluster\_dns\_domain: The suffix used for all cluster service records. | 
object({
    additive_vpc_scope_dns_domain = optional(string)
    cluster_dns                   = optional(string, "PROVIDER_UNSPECIFIED")
    cluster_dns_scope             = optional(string, "DNS_SCOPE_UNSPECIFIED")
    cluster_dns_domain            = optional(string)
  })
| 
{
  "additive_vpc_scope_dns_domain": null,
  "cluster_dns": "PROVIDER_UNSPECIFIED",
  "cluster_dns_domain": null,
  "cluster_dns_scope": "DNS_SCOPE_UNSPECIFIED"
}
| no | -| [cluster\_availability\_type](#input\_cluster\_availability\_type) | Type of cluster availability. Possible values are: {REGIONAL, ZONAL} | `string` | `"REGIONAL"` | no | -| [cluster\_reference\_type](#input\_cluster\_reference\_type) | How the google\_container\_node\_pool.system\_node\_pools refers to the cluster. Possible values are: {SELF\_LINK, NAME} | `string` | `"SELF_LINK"` | no | -| [configure\_workload\_identity\_sa](#input\_configure\_workload\_identity\_sa) | When true, a kubernetes service account will be created and bound using workload identity to the service account used to create the cluster. | `bool` | `false` | no | -| [default\_max\_pods\_per\_node](#input\_default\_max\_pods\_per\_node) | The default maximum number of pods per node in this cluster. | `number` | `null` | no | -| [deletion\_protection](#input\_deletion\_protection) | "Determines if the cluster can be deleted by gcluster commands or not".
To delete a cluster provisioned with deletion\_protection set to true, you must first set it to false and apply the changes.
Then proceed with deletion as usual. | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`. | `string` | n/a | yes | -| [enable\_dataplane\_v2](#input\_enable\_dataplane\_v2) | Enables [Dataplane v2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2). This setting is immutable on clusters. If null, will default to false unless using multi-networking, in which case it will default to true | `bool` | `null` | no | -| [enable\_dcgm\_monitoring](#input\_enable\_dcgm\_monitoring) | Enable GKE to collect DCGM metrics | `bool` | `false` | no | -| [enable\_external\_dns\_endpoint](#input\_enable\_external\_dns\_endpoint) | Allow [DNS-based approach](https://cloud.google.com/kubernetes-engine/docs/concepts/network-isolation#dns-based_endpoint) for accessing the GKE control plane.
Refer this [dedicated blog](https://cloud.google.com/blog/products/containers-kubernetes/new-dns-based-endpoint-for-the-gke-control-plane) for more details. | `bool` | `false` | no | -| [enable\_filestore\_csi](#input\_enable\_filestore\_csi) | The status of the Filestore Container Storage Interface (CSI) driver addon, which allows the usage of filestore instance as volumes. | `bool` | `false` | no | -| [enable\_gcsfuse\_csi](#input\_enable\_gcsfuse\_csi) | The status of the GCSFuse Container Storage Interface (CSI) driver addon, which allows the usage of a GCS bucket as volumes. | `bool` | `false` | no | -| [enable\_inference\_gateway](#input\_enable\_inference\_gateway) | If true, enables GKE features required for Inference Gateway, including the HttpLoadBalancing addon, and installs required CRDs. | `bool` | `false` | no | -| [enable\_k8s\_beta\_apis](#input\_enable\_k8s\_beta\_apis) | List of Enabled Kubernetes Beta APIs. | `list(string)` | `null` | no | -| [enable\_managed\_lustre\_csi](#input\_enable\_managed\_lustre\_csi) | The status of the Google Compute Engine Managed Lustre Container Storage Interface (CSI) driver addon, which allows the usage of a lustre as volumes. | `bool` | `false` | no | -| [enable\_master\_global\_access](#input\_enable\_master\_global\_access) | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `false` | no | -| [enable\_multi\_networking](#input\_enable\_multi\_networking) | Enables [multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) (Requires GKE Enterprise). This setting is immutable on clusters and enables [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en). If null, will determine state based on if additional\_networks are passed in. | `bool` | `null` | no | -| [enable\_node\_local\_dns\_cache](#input\_enable\_node\_local\_dns\_cache) | Enable GKE NodeLocal DNSCache addon to improve DNS lookup latency | `bool` | `false` | no | -| [enable\_parallelstore\_csi](#input\_enable\_parallelstore\_csi) | The status of the Google Compute Engine Parallelstore Container Storage Interface (CSI) driver addon, which allows the usage of a parallelstore as volumes. | `bool` | `false` | no | -| [enable\_persistent\_disk\_csi](#input\_enable\_persistent\_disk\_csi) | The status of the Google Compute Engine Persistent Disk Container Storage Interface (CSI) driver addon, which allows the usage of a PD as volumes. | `bool` | `true` | no | -| [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | (Beta) Whether the master's internal IP address is used as the cluster endpoint. | `bool` | `true` | no | -| [enable\_private\_ipv6\_google\_access](#input\_enable\_private\_ipv6\_google\_access) | The private IPv6 google access type for the VMs in this subnet. | `bool` | `true` | no | -| [enable\_private\_nodes](#input\_enable\_private\_nodes) | (Beta) Whether nodes have internal IP addresses only. | `bool` | `true` | no | -| [enable\_ray\_operator](#input\_enable\_ray\_operator) | The status of the Ray operator addon, This feature enables Kubernetes APIs for managing and scaling Ray clusters and jobs. You control and are responsible for managing ray.io custom resources in your cluster. This feature is not compatible with GKE clusters that already have another Ray operator installed. Supports clusters on Kubernetes version 1.29.8-gke.1054000 or later. | `bool` | `false` | no | -| [gcp\_public\_cidrs\_access\_enabled](#input\_gcp\_public\_cidrs\_access\_enabled) | Whether the cluster master is accessible via all the Google Compute Engine Public IPs. To view this list of IP addresses look here https://cloud.google.com/compute/docs/faq#find_ip_range | `bool` | `false` | no | -| [k8s\_network\_names](#input\_k8s\_network\_names) | Kubernetes network names details for GKE. If starting index is not specified for gvnic or rdma, it would be set to the default values. | 
object({
    gvnic_prefix      = optional(string, "")
    gvnic_start_index = optional(number, 1)
    gvnic_postfix     = optional(string, "")
    rdma_prefix       = optional(string, "")
    rdma_start_index  = optional(number, 0)
    rdma_postfix      = optional(string, "")
  })
| 
{
  "gvnic_postfix": "",
  "gvnic_prefix": "gvnic-",
  "gvnic_start_index": 1,
  "rdma_postfix": "",
  "rdma_prefix": "rdma-",
  "rdma_start_index": 0
}
| no | -| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account name to use with the gke cluster | `string` | `"workload-identity-k8s-sa"` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [maintenance\_exclusions](#input\_maintenance\_exclusions) | List of maintenance exclusions. A cluster can have up to three. | 
list(object({
    name            = string
    start_time      = string
    end_time        = string
    exclusion_scope = string
  }))
| `[]` | no | -| [maintenance\_start\_time](#input\_maintenance\_start\_time) | Start time for daily maintenance operations. Specified in GMT with `HH:MM` format. | `string` | `"09:00"` | no | -| [master\_authorized\_networks](#input\_master\_authorized\_networks) | External network that can access Kubernetes master through HTTPS. Must be specified in CIDR notation. | 
list(object({
    cidr_block   = string
    display_name = string
  }))
| `[]` | no | -| [master\_ipv4\_cidr\_block](#input\_master\_ipv4\_cidr\_block) | (Beta) The IP range in CIDR notation to use for the hosted master network. | `string` | `"172.16.0.32/28"` | no | -| [min\_master\_version](#input\_min\_master\_version) | The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release. | `string` | `null` | no | -| [name\_suffix](#input\_name\_suffix) | Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`. | `string` | `""` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`. | `string` | n/a | yes | -| [networking\_mode](#input\_networking\_mode) | Determines whether alias IPs or routes will be used for pod IPs in the cluster. Options are VPC\_NATIVE or ROUTES. VPC\_NATIVE enables IP aliasing. The default is VPC\_NATIVE. | `string` | `"VPC_NATIVE"` | no | -| [pods\_ip\_range\_name](#input\_pods\_ip\_range\_name) | The name of the secondary subnet ip range to use for pods. | `string` | `"pods"` | no | -| [prefix\_with\_deployment\_name](#input\_prefix\_with\_deployment\_name) | If true, cluster name will be prefixed by `deployment_name` (ex: -). | `bool` | `true` | no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [region](#input\_region) | The region to host the cluster in. | `string` | n/a | yes | -| [release\_channel](#input\_release\_channel) | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. | `string` | `"UNSPECIFIED"` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the system node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the system node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [services\_ip\_range\_name](#input\_services\_ip\_range\_name) | The name of the secondary subnet range to use for services. | `string` | `"services"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to host the cluster in. | `string` | n/a | yes | -| [system\_node\_pool\_disk\_size\_gb](#input\_system\_node\_pool\_disk\_size\_gb) | Size of disk for each node of the system node pool. | `number` | `100` | no | -| [system\_node\_pool\_disk\_type](#input\_system\_node\_pool\_disk\_type) | Disk type for each node of the system node pool. | `string` | `null` | no | -| [system\_node\_pool\_enable\_secure\_boot](#input\_system\_node\_pool\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | -| [system\_node\_pool\_enabled](#input\_system\_node\_pool\_enabled) | Create a system node pool. | `bool` | `true` | no | -| [system\_node\_pool\_image\_type](#input\_system\_node\_pool\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | -| [system\_node\_pool\_kubernetes\_labels](#input\_system\_node\_pool\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | -| [system\_node\_pool\_machine\_type](#input\_system\_node\_pool\_machine\_type) | Machine type for the system node pool. | `string` | `"e2-standard-4"` | no | -| [system\_node\_pool\_name](#input\_system\_node\_pool\_name) | Name of the system node pool. | `string` | `"system"` | no | -| [system\_node\_pool\_node\_count](#input\_system\_node\_pool\_node\_count) | The total min and max nodes to be maintained in the system node pool. | 
object({
    total_min_nodes = number
    total_max_nodes = number
  })
| 
{
  "total_max_nodes": 10,
  "total_min_nodes": 2
}
| no | -| [system\_node\_pool\_taints](#input\_system\_node\_pool\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "components.gke.io/gke-managed-components",
    "value": true
  }
]
| no | -| [system\_node\_pool\_zones](#input\_system\_node\_pool\_zones) | The zones to use for the system node pool. If not specified, the cluster default node zone(s) will be used. | `list(string)` | `null` | no | -| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | -| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | -| [upgrade\_settings](#input\_upgrade\_settings) | Defines gke cluster upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | -| [version\_prefix](#input\_version\_prefix) | If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally. | `string` | `"1.31."` | no | -| [zone](#input\_zone) | Zone for a zonal cluster. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_id](#output\_cluster\_id) | An identifier for the resource with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | -| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | -| [gke\_version](#output\_gke\_version) | GKE cluster's version. | -| [instructions](#output\_instructions) | Instructions on how to connect to the created cluster. | -| [k8s\_service\_account\_name](#output\_k8s\_service\_account\_name) | Name of k8s service account. | - diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf deleted file mode 100644 index 6106f8d90f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf +++ /dev/null @@ -1,470 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-cluster", ghpc_role = "scheduler" }) -} - -locals { - upgrade_settings = { - strategy = var.upgrade_settings.strategy - max_surge = coalesce(var.upgrade_settings.max_surge, 0) - max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) - } -} - -locals { - dash = var.prefix_with_deployment_name && var.name_suffix != "" ? "-" : "" - prefix = var.prefix_with_deployment_name ? var.deployment_name : "" - name_maybe_empty = "${local.prefix}${local.dash}${var.name_suffix}" - name = local.name_maybe_empty != "" ? local.name_maybe_empty : "NO-NAME-GIVEN" - - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ - security_group = var.authenticator_security_group - }] - - default_sa_email = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" - sa_email = coalesce(var.service_account_email, local.default_sa_email) - - # additional VPCs enable multi networking - derived_enable_multi_networking = coalesce(var.enable_multi_networking, length(var.additional_networks) > 0) - - # multi networking needs enabled Dataplane v2 - derived_enable_dataplane_v2 = coalesce(var.enable_dataplane_v2, local.derived_enable_multi_networking) - - default_monitoring_component = [ - "SYSTEM_COMPONENTS", - "POD", - "DAEMONSET", - "DEPLOYMENT", - "STATEFULSET", - "STORAGE", - "HPA", - "CADVISOR", - "KUBELET" - ] - - default_logging_component = [ - "SYSTEM_COMPONENTS", - "WORKLOADS" - ] -} - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_container_engine_versions" "version_prefix_filter" { - provider = google-beta - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - version_prefix = var.version_prefix -} - -locals { - master_version = var.min_master_version != null ? var.min_master_version : data.google_container_engine_versions.version_prefix_filter.latest_master_version -} - -resource "google_container_cluster" "gke_cluster" { - provider = google-beta - - project = var.project_id - name = local.name - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - resource_labels = local.labels - networking_mode = var.networking_mode - # decouple node pool lifecycle from cluster life cycle - remove_default_node_pool = true - initial_node_count = 1 # must be set when remove_default_node_pool is set - node_locations = var.system_node_pool_zones - - deletion_protection = var.deletion_protection - - dynamic "enable_k8s_beta_apis" { - for_each = var.enable_k8s_beta_apis != null ? [1] : [] - content { - enabled_apis = var.enable_k8s_beta_apis - } - } - - network = var.network_id - subnetwork = var.subnetwork_self_link - - # Note: the existence of the "master_authorized_networks_config" block enables - # the master authorized networks even if it's empty. - master_authorized_networks_config { - dynamic "cidr_blocks" { - for_each = var.master_authorized_networks - content { - cidr_block = cidr_blocks.value.cidr_block - display_name = cidr_blocks.value.display_name - } - } - gcp_public_cidrs_access_enabled = var.gcp_public_cidrs_access_enabled - } - - private_ipv6_google_access = var.enable_private_ipv6_google_access ? "PRIVATE_IPV6_GOOGLE_ACCESS_TO_GOOGLE" : null - default_max_pods_per_node = var.default_max_pods_per_node - master_auth { - client_certificate_config { - issue_client_certificate = false - } - } - - enable_shielded_nodes = true - - cluster_autoscaling { - # Controls auto provisioning of node-pools - enabled = false - - # Controls autoscaling algorithm of node-pools - autoscaling_profile = var.autoscaling_profile - } - - datapath_provider = local.derived_enable_dataplane_v2 ? "ADVANCED_DATAPATH" : "LEGACY_DATAPATH" - - enable_multi_networking = local.derived_enable_multi_networking - - network_policy { - # Enabling NetworkPolicy for clusters with DatapathProvider=ADVANCED_DATAPATH - # is not allowed. Dataplane V2 will take care of network policy enforcement - # instead. - enabled = false - # GKE Dataplane V2 support. This must be set to PROVIDER_UNSPECIFIED in - # order to let the datapath_provider take effect. - # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/656#issuecomment-720398658 - provider = "PROVIDER_UNSPECIFIED" - } - - private_cluster_config { - enable_private_nodes = var.enable_private_nodes - enable_private_endpoint = var.enable_private_endpoint - master_ipv4_cidr_block = var.master_ipv4_cidr_block - master_global_access_config { - enabled = var.enable_master_global_access - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.pods_ip_range_name - services_secondary_range_name = var.services_ip_range_name - } - - workload_identity_config { - workload_pool = "${var.project_id}.svc.id.goog" - } - - dynamic "gateway_api_config" { - for_each = var.enable_inference_gateway ? [1] : [] - content { - channel = "CHANNEL_STANDARD" - } - } - - dynamic "authenticator_groups_config" { - for_each = local.cluster_authenticator_security_group - content { - security_group = authenticator_groups_config.value.security_group - } - } - - release_channel { - channel = var.release_channel - } - min_master_version = local.master_version - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - - dynamic "maintenance_exclusion" { - for_each = var.maintenance_exclusions - content { - exclusion_name = maintenance_exclusion.value.name - start_time = maintenance_exclusion.value.start_time - end_time = maintenance_exclusion.value.end_time - exclusion_options { - scope = maintenance_exclusion.value.exclusion_scope - } - } - } - } - - dynamic "dns_config" { - for_each = var.cloud_dns_config != null ? [1] : [] - content { - additive_vpc_scope_dns_domain = var.cloud_dns_config.additive_vpc_scope_dns_domain - cluster_dns = var.cloud_dns_config.cluster_dns - cluster_dns_scope = var.cloud_dns_config.cluster_dns_scope - cluster_dns_domain = var.cloud_dns_config.cluster_dns_domain - } - } - - addons_config { - gcp_filestore_csi_driver_config { - enabled = var.enable_filestore_csi - } - gcs_fuse_csi_driver_config { - enabled = var.enable_gcsfuse_csi - } - gce_persistent_disk_csi_driver_config { - enabled = var.enable_persistent_disk_csi - } - dns_cache_config { - enabled = var.enable_node_local_dns_cache - } - parallelstore_csi_driver_config { - enabled = var.enable_parallelstore_csi - } - ray_operator_config { - enabled = var.enable_ray_operator - } - lustre_csi_driver_config { - enabled = var.enable_managed_lustre_csi - } - dynamic "http_load_balancing" { - for_each = var.enable_inference_gateway ? [1] : [] - content { - disabled = false - } - } - } - - timeouts { - create = var.timeout_create - update = var.timeout_update - } - - node_config { - shielded_instance_config { - enable_secure_boot = var.system_node_pool_enable_secure_boot - enable_integrity_monitoring = true - } - } - - control_plane_endpoints_config { - dns_endpoint_config { - allow_external_traffic = var.enable_external_dns_endpoint - } - } - - lifecycle { - # Ignore all changes to the default node pool. It's being removed after creation. - ignore_changes = [ - node_config, - min_master_version, - ] - precondition { - condition = var.default_max_pods_per_node == null || var.networking_mode == "VPC_NATIVE" - error_message = "default_max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." - } - precondition { - condition = coalesce(var.enable_dataplane_v2, true) || !local.derived_enable_multi_networking - error_message = "'enable_dataplane_v2' cannot be false when enabling multi networking." - } - precondition { - condition = coalesce(var.enable_multi_networking, true) || length(var.additional_networks) == 0 - error_message = "'enable_multi_networking' cannot be false when using multivpc module, which passes additional_networks." - } - } - - monitoring_config { - enable_components = var.enable_dcgm_monitoring ? concat(local.default_monitoring_component, ["DCGM"]) : local.default_monitoring_component - managed_prometheus { - enabled = true - } - } - - logging_config { - enable_components = local.default_logging_component - } -} - -# We define explicit node pools, so that it can be modified without -# having to destroy the entire cluster. -resource "google_container_node_pool" "system_node_pools" { - provider = google-beta - count = var.system_node_pool_enabled ? 1 : 0 - - project = var.project_id - name = var.system_node_pool_name - cluster = var.cluster_reference_type == "NAME" ? google_container_cluster.gke_cluster.name : google_container_cluster.gke_cluster.self_link - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - node_locations = var.system_node_pool_zones - version = local.master_version - - autoscaling { - total_min_node_count = var.system_node_pool_node_count.total_min_nodes - total_max_node_count = var.system_node_pool_node_count.total_max_nodes - } - - upgrade_settings { - strategy = local.upgrade_settings.strategy - max_surge = local.upgrade_settings.max_surge - max_unavailable = local.upgrade_settings.max_unavailable - } - - management { - auto_repair = true - auto_upgrade = true - } - - node_config { - labels = var.system_node_pool_kubernetes_labels - resource_labels = local.labels - service_account = var.service_account_email - oauth_scopes = var.service_account_scopes - machine_type = var.system_node_pool_machine_type - disk_size_gb = var.system_node_pool_disk_size_gb - disk_type = var.system_node_pool_disk_type - - dynamic "taint" { - for_each = var.system_node_pool_taints - content { - key = taint.value.key - value = taint.value.value - effect = taint.value.effect - } - } - - # Forcing the use of the Container-optimized image, as it is the only - # image with the proper logging daemon installed. - # - # cos images use Shielded VMs since v1.13.6-gke.0. - # https://cloud.google.com/kubernetes-engine/docs/how-to/node-images - # - # We use COS_CONTAINERD to be compatible with (optional) gVisor. - # https://cloud.google.com/kubernetes-engine/docs/how-to/sandbox-pods - image_type = var.system_node_pool_image_type - - shielded_instance_config { - enable_secure_boot = var.system_node_pool_enable_secure_boot - enable_integrity_monitoring = true - } - - gvnic { - enabled = var.system_node_pool_image_type == "COS_CONTAINERD" - } - - # Implied by Workload Identity - workload_metadata_config { - mode = "GKE_METADATA" - } - # Implied by workload identity. - metadata = { - "disable-legacy-endpoints" = "true" - } - } - - lifecycle { - ignore_changes = [ - node_config[0].labels, - node_config[0].taint, - version, - ] - precondition { - condition = contains(["SURGE"], local.upgrade_settings.strategy) - error_message = "Only SURGE strategy is supported" - } - precondition { - condition = local.upgrade_settings.max_unavailable >= 0 - error_message = "max_unavailable should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_surge >= 0 - error_message = "max_surge should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 - error_message = "At least one of max_unavailable or max_surge must greater than 0" - } - } -} - -data "google_client_config" "default" {} - -provider "kubernetes" { - host = "https://${google_container_cluster.gke_cluster.endpoint}" - cluster_ca_certificate = base64decode(google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) - token = data.google_client_config.default.access_token -} - -module "workload_identity" { - count = var.configure_workload_identity_sa ? 1 : 0 - source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" - version = ">= 40.0" - - use_existing_gcp_sa = true - name = var.k8s_service_account_name - gcp_sa_name = local.sa_email - project_id = var.project_id - - # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1059 - depends_on = [ - data.google_project.project, - google_container_cluster.gke_cluster - ] -} - -locals { - k8s_service_account_name = one(module.workload_identity[*].k8s_service_account_name) -} - -locals { - # Separate gvnic and rdma networks and assign indexes - gvnic_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "GVNIC")] : - merge(net, { name = "${var.k8s_network_names.gvnic_prefix}${idx + var.k8s_network_names.gvnic_start_index}${var.k8s_network_names.gvnic_postfix}" }) - ] - - rdma_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "RDMA")] : - merge(net, { name = "${var.k8s_network_names.rdma_prefix}${idx + var.k8s_network_names.rdma_start_index}${var.k8s_network_names.rdma_postfix}" }) - ] - - all_networks = concat(local.gvnic_networks, local.rdma_networks) -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = google_container_cluster.gke_cluster.id - project_id = var.project_id - - apply_manifests = concat(flatten([ - for idx, network_info in local.all_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = network_info.name, - network_name = network_info.network - subnetwork_name = network_info.subnetwork, - device_mode = strcontains(upper(network_info.nic_type), "RDMA") ? "RDMA" : "NetDevice" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = network_info.name } - } - ] - ]), - var.enable_inference_gateway ? [ - { - source = "https://github.com/kubernetes-sigs/gateway-api-inference-extension/releases/download/v1.0.0/manifests.yaml", - template_vars = {} - } - ] : [] - ) -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml deleted file mode 100644 index bd1517ce8f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf deleted file mode 100644 index 3326a5468e..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cluster_id" { - description = "An identifier for the resource with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." - value = google_container_cluster.gke_cluster.id -} - -output "gke_cluster_exists" { - description = "A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations." - value = true - depends_on = [ - google_container_cluster.gke_cluster - ] -} - -locals { - private_endpoint_message = trimspace( - <<-EOT - This cluster was created with 'enable_private_endpoint: true'. - It cannot be accessed from a public IP addresses. - One way to access this cluster is from a VM created in the GKE cluster subnet. - EOT - ) - master_authorized_networks_message = length(var.master_authorized_networks) == 0 ? "" : trimspace( - <<-EOT - The following networks have been authorized to access this cluster: - ${join("\n", [for x in var.master_authorized_networks : " ${x.display_name}: ${x.cidr_block}"])}" - EOT - ) - public_endpoint_message = trimspace( - <<-EOT - To add authorized networks you can allowlist your IP with this command: - gcloud container clusters update ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} \ - --enable-master-authorized-networks \ - --master-authorized-networks /32 - EOT - ) - allowlist_your_ip_message = var.enable_private_endpoint ? local.private_endpoint_message : local.public_endpoint_message - kubernetes_service_account_message = local.k8s_service_account_name == null ? "" : trimspace( - <<-EOT - Use the following Kubernetes Service Account in the default namespace to run your workloads: - ${local.k8s_service_account_name} - The GCP Service Account mapped to this Kubernetes Service Account is: - ${local.sa_email} - EOT - ) - kubernetes_cluster_fetch_credential_message = var.enable_external_dns_endpoint ? trimspace( - <<-EOT - Use the following command to fetch credentials for the created cluster: - gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} \ - --dns-endpoint - EOT - ) : trimspace( - <<-EOT - Use the following command to fetch credentials for the created cluster: - gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} - EOT - ) -} - -output "instructions" { - description = "Instructions on how to connect to the created cluster." - value = trimspace( - <<-EOT - ${local.master_authorized_networks_message} - - ${local.allowlist_your_ip_message} - - ${local.kubernetes_cluster_fetch_credential_message} - - ${local.kubernetes_service_account_message} - EOT - ) -} - -output "k8s_service_account_name" { - description = "Name of k8s service account." - value = local.k8s_service_account_name -} - -output "gke_version" { - description = "GKE cluster's version." - value = google_container_cluster.gke_cluster.master_version -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl deleted file mode 100644 index d376a1a760..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: GKENetworkParamSet -metadata: - name: ${name} -spec: - vpc: ${network_name} - vpcSubnet: ${subnetwork_name} - deviceMode: ${device_mode} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl deleted file mode 100644 index 1571a92692..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: Network -metadata: - name: ${name} -spec: - parametersRef: - group: networking.gke.io - kind: GKENetworkParamSet - name: ${name} - type: Device diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf b/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf deleted file mode 100644 index 8d863b1730..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf +++ /dev/null @@ -1,533 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "name_suffix" { - description = "Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`." - type = string - default = "" -} - -variable "deployment_name" { - description = "Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`." - type = string -} - -variable "prefix_with_deployment_name" { - description = "If true, cluster name will be prefixed by `deployment_name` (ex: -)." - type = bool - default = true -} - -variable "region" { - description = "The region to host the cluster in." - type = string -} - -variable "zone" { - description = "Zone for a zonal cluster." - default = null - type = string -} - -variable "network_id" { - description = "The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`." - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to host the cluster in." - type = string -} - -variable "pods_ip_range_name" { - description = "The name of the secondary subnet ip range to use for pods." - type = string - default = "pods" -} - -variable "services_ip_range_name" { - description = "The name of the secondary subnet range to use for services." - type = string - default = "services" -} - -variable "enable_private_ipv6_google_access" { - description = "The private IPv6 google access type for the VMs in this subnet." - type = bool - default = true -} - -variable "release_channel" { - description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`." - type = string - default = "UNSPECIFIED" -} - -variable "min_master_version" { - description = "The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release." - type = string - default = null -} - -variable "version_prefix" { - description = "If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally." - type = string - default = "1.31." -} - -variable "maintenance_start_time" { - description = "Start time for daily maintenance operations. Specified in GMT with `HH:MM` format." - type = string - default = "09:00" -} - -variable "maintenance_exclusions" { - description = "List of maintenance exclusions. A cluster can have up to three." - type = list(object({ - name = string - start_time = string - end_time = string - exclusion_scope = string - })) - default = [] - validation { - condition = alltrue([ - for x in var.maintenance_exclusions : - contains(["NO_UPGRADES", "NO_MINOR_UPGRADES", "NO_MINOR_OR_NODE_UPGRADES"], x.exclusion_scope) - ]) - error_message = "`exclusion_scope` must be set to `NO_UPGRADES` OR `NO_MINOR_UPGRADES` OR `NO_MINOR_OR_NODE_UPGRADES`." - } -} - -variable "cloud_dns_config" { - description = < **_NOTE:_** The `project_id` and `region` settings would be inferred from the -> deployment variables of the same name, but they are included here for clarity. - -### Multi-networking - -To create network objects in GKE cluster, you can pass a multivpc module to a pre-existing-gke-cluster module instead of [applying a manifest manually](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment). - -```yaml - - id: network - source: modules/network/vpc - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: existing-gke-cluster ## multinetworking must be enabled in advance when cluster creation - source: modules/scheduler/pre-existing-gke-cluster - use: [multinetwork] - settings: - cluster_name: $(vars.deployment_name) -``` - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_container_cluster.existing_gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [cluster\_name](#input\_cluster\_name) | Name of the existing cluster | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project that hosts the existing cluster | `string` | n/a | yes | -| [rdma\_subnetwork\_name\_prefix](#input\_rdma\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | `string` | `null` | no | -| [region](#input\_region) | Region in which to search for the cluster | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | -| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster exists. | -| [gke\_version](#output\_gke\_version) | GKE cluster's version. | - diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf deleted file mode 100644 index 926d2be100..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_container_cluster" "existing_gke_cluster" { - name = var.cluster_name - project = var.project_id - location = var.region -} - -locals { - rdma_networks = [for network_info in var.additional_networks : network_info if strcontains(upper(network_info.nic_type), "RDMA")] - non_rdma_networks = [for network_info in var.additional_networks : network_info if !strcontains(upper(network_info.nic_type), "RDMA")] - apply_manifests_rdma_networks = flatten([ - for idx, network_info in local.rdma_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = "${var.rdma_subnetwork_name_prefix}-${idx}", - network_name = network_info.network - subnetwork_name = "${var.rdma_subnetwork_name_prefix}-${idx}", - device_mode = "RDMA" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = "${var.rdma_subnetwork_name_prefix}-${idx}" } - } - ] - ]) - - apply_manifests_non_rdma_networks = flatten([ - for idx, network_info in local.non_rdma_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = network_info.subnetwork - network_name = network_info.network - subnetwork_name = network_info.subnetwork - device_mode = "NetDevice" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = network_info.subnetwork } - } - ] - ]) -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = data.google_container_cluster.existing_gke_cluster.id - project_id = var.project_id - - apply_manifests = concat(local.apply_manifests_non_rdma_networks, local.apply_manifests_rdma_networks) -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf deleted file mode 100644 index 8884ee30b0..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cluster_id" { - description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." - value = data.google_container_cluster.existing_gke_cluster.id -} - -output "gke_cluster_exists" { - description = "A static flag that signals to downstream modules that a cluster exists." - value = true - depends_on = [ - data.google_container_cluster.existing_gke_cluster - ] -} - -output "gke_version" { - description = "GKE cluster's version." - value = data.google_container_cluster.existing_gke_cluster.master_version -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl deleted file mode 100644 index d376a1a760..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: GKENetworkParamSet -metadata: - name: ${name} -spec: - vpc: ${network_name} - vpcSubnet: ${subnetwork_name} - deviceMode: ${device_mode} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl deleted file mode 100644 index 1571a92692..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: Network -metadata: - name: ${name} -spec: - parametersRef: - group: networking.gke.io - kind: GKENetworkParamSet - name: ${name} - type: Device diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf deleted file mode 100644 index 9e9ed98ed3..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project that hosts the existing cluster" - type = string -} - -variable "cluster_name" { - description = "Name of the existing cluster" - type = string -} - -variable "region" { - description = "Region in which to search for the cluster" - type = string -} - -variable "additional_networks" { - description = "Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster." - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - })) -} - -variable "rdma_subnetwork_name_prefix" { - description = "Prefix of the RDMA subnetwork names" - default = null - type = string -} diff --git a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf b/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf deleted file mode 100644 index 562d8647b1..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = "> 5.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.74.0" - } - - required_version = ">= 1.3" -} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/README.md b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/README.md deleted file mode 100644 index db9094909b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/README.md +++ /dev/null @@ -1,355 +0,0 @@ -## Description - -This module creates a startup script that will execute a list of runners in the -order they are specified. The runners are copied to a GCS bucket at deployment -time and then copied into the VM as they are executed after startup. - -Each runner receives the following attributes: - -- `destination`: (Required) The name of the file at the destination VM. If an - absolute path is provided, the file will be copied to that path, otherwise - the file will be created in a temporary folder and deleted once the startup - script runs. -- `type`: (Required) The type of the runner, one of the following: - - `shell`: The runner is a shell script and will be executed once copied to - the destination VM. - - `ansible-local`: The runner is an ansible playbook and will run on the VM - with the following command line flags: - - ```shell - ansible-playbook --connection=local --inventory=localhost, \ - --limit localhost <> - ``` - - - `data`: The data or file specified will be copied to `<>`. No - action will be performed after the data is staged. This data can be used by - subsequent runners or simply made available on the VM for later use. -- `content`: (Optional) Content to be uploaded and, if `type` is - either `shell` or `ansible-local`, executed. Must be defined if `source` is - not. -- `source`: (Optional) A path to the file or data you want to upload. Must be - defined if `content` is not. The source path is relative to the deployment - group directory. To ensure correctness of path use `ghpc_stage` function, that - would copy referenced file to the deployment group directory. For example: - - ```yaml - source: $(ghpc_stage("path/to/file")) - ``` - - For more examples with context, see the - [example blueprint snippet](#example). To reference any other source file, an - absolute path must be used. - -- `args`: (Optional) Arguments to be passed to `shell` or `ansible-local` - runners. For `shell` runners, these will be passed as arguments to the script - when it is executed. For `ansible-local` runners, they will be appended to - a list of default arguments that invoke `ansible-playbook` on the localhost. - Therefore`args` should not include any arguments that alter this behavior, - such as `--connection`, `--inventory`, or `--limit`. - -### Runner dependencies - -`ansible-local` runners require Ansible to be installed in the VM before -running. To support other playbook runners in the Cluster Toolkit, we install -version 2.11 of `ansible-core` as well as the larger package of collections -found in `ansible` version 4.10.0. - -If an `ansible-local` runner is found in the list supplied to this module, -a script to install Ansible will be prepended to the list of runners. This -behavior can be disabled by setting `var.prepend_ansible_installer` to `false`. -This script will do the following at VM startup: - -- Install system-wide python3 if not already installed using system package - managers (yum, apt-get, etc) -- Install `python3-distutils` system-wide in debian and ubuntu based - environments. This can be a missing dependency on system installations of - python3 for installing and upgrading pip. -- Install system-wide pip3 if not already installed and upgrade pip3 if the - version is not at least 18.0. -- Install and create a virtual environment located at `/usr/local/ghpc-venv`. -- Install ansible into this virtual environment if the current version of - ansible is not version 2.11 or higher. - -To use the virtual environment created by this script, you can activate it by -running the following command on the VM: - -```shell -source /usr/local/ghpc-venv/bin/activate -``` - -You may also need to provide the correct python interpreter as the python3 -binary in the virtual environment. This can be done by adding the following flag -when calling `ansible-playbook`: - -```shell --e ansible_python_interpreter=/usr/local/ghpc-venv/bin/activate -``` - -> **_NOTE:_** ansible-playbook and other ansible command line tools will only be -> accessible from the command line (and in your PATH variable) after activating -> this environment. - -### Staging the runners - -Runners will be uploaded to a -[GCS bucket](https://cloud.google.com/storage/docs/creating-buckets). This -bucket will be created by this module and named as -`${var.deployment_name}-startup-scripts-${random_id}`. VMs using the startup -script created by this module will pull the runners content from a GCS bucket -and therefore must have access to GCS. - -> **_NOTE:_** To ensure access to GCS, set the following OAuth scope on the -> instance using the startup scripts: -> `https://www.googleapis.com/auth/devstorage.read_only`. -> -> This is set as a default scope in the [vm-instance], -> [schedMD-slurm-on-gcp-login-node] and [schedMD-slurm-on-gcp-controller] -> modules - -[vm-instance]: ../../compute/vm-instance/README.md -[schedMD-slurm-on-gcp-login-node]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md -[schedMD-slurm-on-gcp-controller]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md - -### Tracking startup script execution - -For more information on how to use startup scripts on Google Cloud Platform, -please refer to -[this document](https://cloud.google.com/compute/docs/instances/startup-scripts/linux). - -To debug startup scripts from a Linux VM created with startup script generated -by this module: - -```shell -sudo DEBUG=1 google_metadata_script_runner startup -``` - -To view outputs from a Linux startup script, run: - -```shell -sudo journalctl -u google-startup-scripts.service -``` - -### Monitoring Agent Installation - -This `startup-script` module has several options for installing a Google -monitoring agent. There are two relevant settings: `install_stackdriver_agent` -and `install_cloud_ops_agent`. - -The _Stackdriver Agent_ also called the _Legacy Cloud Monitoring Agent_ provides -better performance under some HPC workloads. While official documentation -recommends using the _Cloud Ops Agent_, it is recommended to use -`install_stackdriver_agent` when performance is important. - -#### Stackdriver Agent Installation - -If an image or machine already has Cloud Ops Agent installed and you would like -to instead use the Stackdriver Agent, the following script will remove the Cloud -Ops Agent and install the Stackdriver Agent. - -```bash -# Remove Cloud Ops Agent -sudo systemctl stop google-cloud-ops-agent.service -sudo systemctl disable google-cloud-ops-agent.service -curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh -sudo bash add-google-cloud-ops-agent-repo.sh --uninstall -sudo bash add-google-cloud-ops-agent-repo.sh --remove-repo - -# Install Stackdriver Agent -curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh -sudo bash add-monitoring-agent-repo.sh --also-install -curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh -sudo bash add-logging-agent-repo.sh --also-install -sudo service stackdriver-agent start -sudo service google-fluentd restart -``` - -#### Cloud Ops Agent Installation - -If an image or machine already has the Stackdriver Agent installed and you would -like to instead use the Cloud Ops Agent, the following script will remove the -Stackdriver Agent and install the Cloud Ops Agent. - -```bash -# UnInstall Stackdriver Agent - -sudo systemctl stop stackdriver-agent.service -sudo systemctl disable stackdriver-agent.service -curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh -sudo dpkg --configure -a -sudo bash add-monitoring-agent-repo.sh --uninstall -sudo bash add-monitoring-agent-repo.sh --remove-repo -sudo systemctl stop google-fluentd.service -sudo systemctl disable google-fluentd.service -sudo dpkg --configure -a -curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh -sudo bash add-logging-agent-repo.sh --uninstall -sudo bash add-logging-agent-repo.sh --remove-repo - -# Install ops-agent - -curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh -sudo bash add-google-cloud-ops-agent-repo.sh --also-install -sudo service google-cloud-ops-agent start -``` - -As a reminder, this should be in a startup script, which should run on all -Compute nodes via the `compute_startup_script` on the controller. - -#### Testing Installation - -You can test if one of the agents is running using the following commands: - -```bash -# For Cloud Ops Agent -$ sudo systemctl is-active google-cloud-ops-agent"*" -active -active -active -active - -# For Legacy Monitoring and Logging Agents -$ sudo service stackdriver-agent status -stackdriver-agent is running [ OK ] -$ sudo service google-fluentd status -google-fluentd is running [ OK ] -``` - -For official documentation see troubleshooting docs: - -- [Cloud Ops Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-install-startup) -- [Legacy Monitoring Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/monitoring/troubleshooting) -- [Legacy Logging Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/logging/troubleshooting) - -### Example - -```yaml -- id: startup - source: modules/scripts/startup-script - settings: - runners: - # Some modules such as filestore have runners as outputs for convenience: - - $(homefs.install_nfs_client_runner) - # These runners can still be created manually: - # - type: shell - # destination: "modules/filestore/scripts/install_nfs_client.sh" - # source: "modules/filestore/scripts/install_nfs_client.sh" - - type: ansible-local - destination: "modules/filestore/scripts/mount.yaml" - source: "modules/filestore/scripts/mount.yaml" - - type: data - source: /tmp/foo.tgz - destination: /tmp/bar.tgz - - type: shell - destination: "decompress.sh" - content: | - #!/bin/sh - echo $2 - tar zxvf /tmp/$1 -C / - args: "bar.tgz 'Expanding file'" - -- id: compute-cluster - source: modules/compute/vm-instance - use: [homefs, startup] -``` - -In the above example, a new GCS bucket is created to upload the startup-scripts. -But in the case where the user wants to reuse existing GCS bucket or folder, -they are able to do so by using the `gcs_bucket_path` as shown in the below example - -```yaml -- id: startup - source: modules/scripts/startup-script - settings: - gcs_bucket_path: gs://user-test-bucket/folder1/folder2 - install_stackdriver_agent: true - -- id: compute-cluster - source: modules/compute/vm-instance - use: [startup] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 6.41 | -| [local](#requirement\_local) | >= 2.0.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.41 | -| [local](#provider\_local) | >= 2.0.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.configs_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | -| [google_storage_bucket_object.scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [local_file.debug_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [ansible\_virtualenv\_path](#input\_ansible\_virtualenv\_path) | Virtual environment path in which to install Ansible | `string` | `"/usr/local/ghpc-venv"` | no | -| [bucket\_viewers](#input\_bucket\_viewers) | Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account) | `list(string)` | `[]` | no | -| [configure\_ssh\_host\_patterns](#input\_configure\_ssh\_host\_patterns) | If specified, it will automate ssh configuration by:
- Defining a Host block for every element of this variable and setting StrictHostKeyChecking to 'No'.
Ex: "hpc*", "hpc01*", "ml*"
- The first time users log-in, it will create ssh keys that are added to the authorized keys list
This requires a shared /home filesystem and relies on specifying the right prefix. | `list(string)` | `[]` | no | -| [debug\_file](#input\_debug\_file) | Path to an optional local to be written with 'startup\_script'. | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used to name GCS bucket for startup scripts. | `string` | n/a | yes | -| [docker](#input\_docker) | Install and configure Docker | 
object({
    enabled        = optional(bool, false)
    world_writable = optional(bool, false)
    daemon_config  = optional(string, "")
  })
| 
{
  "enabled": false
}
| no | -| [enable\_docker\_world\_writable](#input\_enable\_docker\_world\_writable) | DEPRECATED: use var.docker | `bool` | `null` | no | -| [enable\_gpu\_network\_wait\_online](#input\_enable\_gpu\_network\_wait\_online) | Enable a SystemD unit that blocks execution of startup-scripts until after all network interfaces are online. (Works on reboots or boots of an image built using this solution) | `bool` | `false` | no | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | `null` | no | -| [http\_no\_proxy](#input\_http\_no\_proxy) | Domains for which to disable http\_proxy behavior. Honored only if var.http\_proxy is set | `string` | `".google.com,.googleapis.com,metadata.google.internal,localhost,127.0.0.1"` | no | -| [http\_proxy](#input\_http\_proxy) | Web (http and https) proxy configuration for pip, apt, and yum/dnf and interactive shells | `string` | `""` | no | -| [install\_ansible](#input\_install\_ansible) | Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used. | `bool` | `null` | no | -| [install\_cloud\_ops\_agent](#input\_install\_cloud\_ops\_agent) | Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true. | `bool` | `false` | no | -| [install\_cloud\_rdma\_drivers](#input\_install\_cloud\_rdma\_drivers) | If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8. Should not be enabled if using the HPC VM Image. | `bool` | `false` | no | -| [install\_docker](#input\_install\_docker) | DEPRECATED: use var.docker. | `bool` | `null` | no | -| [install\_stackdriver\_agent](#input\_install\_stackdriver\_agent) | Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance. | `bool` | `false` | no | -| [labels](#input\_labels) | Labels for the created GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_filesystem](#input\_local\_ssd\_filesystem) | Create and mount a filesystem from local SSD disks (data will be lost if VMs are powered down without enabling migration); enable by setting mountpoint field to a valid directory path. | 
object({
    fs_type     = optional(string, "ext4")
    mountpoint  = optional(string, "")
    permissions = optional(string, "0755")
  })
| 
{
  "fs_type": "ext4",
  "mountpoint": "",
  "permissions": "0755"
}
| no | -| [managed\_lustre](#input\_managed\_lustre) | Configure Managed Lustre (assumes driver already installed) | 
object({
    enabled = optional(bool, false)
    port    = optional(number, 988)
  })
| 
{
  "enabled": false,
  "port": 988
}
| no | -| [prepend\_ansible\_installer](#input\_prepend\_ansible\_installer) | DEPRECATED. Use `install_ansible=false` to prevent ansible installation. | `bool` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [runners](#input\_runners) | List of runners to run on remote VM.
Runners can be of type ansible-local, shell or data.
A runner must specify one of 'source' or 'content'.
All runners must specify 'destination'. If 'destination' does not include a
path, it will be copied in a temporary folder and deleted after running.
Runners may also pass 'args', which will be passed as argument to shell runners only. | `list(map(string))` | `[]` | no | -| [set\_ofi\_cloud\_rdma\_tunables](#input\_set\_ofi\_cloud\_rdma\_tunables) | Controls whether to enable specific OFI environment variables for workloads using Cloud RDMA networking. Should be false for non-RDMA workloads. | `bool` | `false` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [compute\_startup\_script](#output\_compute\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | -| [controller\_startup\_script](#output\_controller\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | -| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | - diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml deleted file mode 100644 index 02c449c7cb..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Configure ssh between nodes - become: true - hosts: localhost - vars: - ssh_config_path: "/etc/ssh/ssh_config" - bashrc: "{{ '/etc/bashrc' if ansible_facts['os_family'] == 'RedHat' else '/etc/bash.bashrc' }}" - setup_ssh_script: "/bin/bash /usr/local/ghpc/setup-ssh-keys.sh" - tasks: - - name: "Set StrictHostKeyChecking to no" - ansible.builtin.blockinfile: - path: "{{ ssh_config_path }}" - block: | - Host "{{ item }}" - StrictHostKeyChecking no - marker: "# {mark} ANSIBLE MANAGED BLOCK {{item}}" - loop: "{{ host_name_prefix }}" - - name: "Create ssh keys in .bashrc if not already done" - ansible.builtin.lineinfile: - path: "{{ bashrc }}" - regexp: '^{{ setup_ssh_script }}' - line: "{{ setup_ssh_script }}" diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh deleted file mode 100644 index 38c7ff9b5c..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -web_proxy="${1:-}" -if [ -z "$web_proxy" ]; then - echo "Error: must provide 1 argument identifying http/https proxy" - exit 1 -fi - -# configure pip to use proxy -PIP_CONF=/etc/pip.conf -if [ ! -f "$PIP_CONF" ]; then - cat <<-EOF >"$PIP_CONF" - [global] - proxy=$web_proxy - EOF -fi - -# configure yum or dnf to use proxy -if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - YUM_CONF="/etc/yum.conf" - if ! grep -q '^proxy=.*' "$YUM_CONF"; then - sed --follow-symlinks -i.bak "/^\[main]/a proxy=$web_proxy" "$YUM_CONF" - else - sed --follow-symlinks -i.bak "s,proxy=.*,proxy=$web_proxy," "$YUM_CONF" - fi -fi - -# configure apt to use proxy -if [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release 2>/dev/null || - grep -qi ubuntu /etc/os-release 2>/dev/null; then - APT_CONF_PROXY="/etc/apt/apt.conf.d/99proxy.conf" - if [ ! -f "$APT_CONF_PROXY" ]; then - cat <<-EOF >"$APT_CONF_PROXY" - Acquire::http::Proxy "$web_proxy"; - Acquire::https::Proxy "$web_proxy"; - EOF - fi -fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh deleted file mode 100644 index 682e1352a1..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script applies fixes to VMs that must occur early in boot. For example, -# when yum or apt repositories are misconfigured, preventing most package -# operations from completing successfully. - -source /etc/os-release - -if [[ "$PRETTY_NAME" == "CentOS Linux 7 (Core)" ]]; then - echo "Applying hotfixes for CentOS 7" - if grep -q '^mirrorlist' /etc/yum.repos.d/CentOS-Base.repo; then - echo "Removing mirrorlist from default CentOS 7 repositories" - sed -i '/^mirrorlist/d' /etc/yum.repos.d/CentOS-Base.repo - fi - if grep -q '^#baseurl=http://mirror.centos.org' /etc/yum.repos.d/CentOS-Base.repo; then - echo "Reconfiguring default CentOS 7 repositories to use CentOS Vault" - sed -i 's,^#baseurl=http://mirror.centos.org/,baseurl=http://vault.centos.org/,' /etc/yum.repos.d/CentOS-Base.repo - fi -fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh deleted file mode 100644 index 3a29ae808f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh +++ /dev/null @@ -1,73 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Given a url and filename, download an object to the vardir. When the installed -# version of gcloud is >=402.0.0 (Sept. 2022), then gcloud storage is used to -# fetch from the bucket. Otherwise gsutil is used. Note, the service account for -# the instance must be properly configured with a role having authorization to -# get objects from the bucket. -# -# This function is intended for single file downloads and no attempt is made to -# verify the checksum other than the default behavior of gcloud or gsutil. -# -# This function has no other platform dependencies other than gcloud / gsutil. - -# This code originated from: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 -stdlib::get_from_bucket() { - local OPTIND opt url fname dir="${VARDIR:-/var/lib/startup}" - while getopts ":u:f:d:" opt; do - case "${opt}" in - u) url="${OPTARG}" ;; - f) fname="${OPTARG}" ;; - d) dir="${OPTARG}" ;; - :) - stdlib::mandatory_argument -n stdlib::get_from_bucket -f "$OPTARG" - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error 'Usage: stdlib::get_from_bucket -u -f -d ' - stdlib::info 'For example: stdlib::get_from_bucket -u gs://mybucket/foo.tgz -d /var/tmp' - return "${E_UNKNOWN_ARG}" - ;; - esac - done - # Trivially compute the filename from the URL if unspecified. - if [[ -z ${fname} ]]; then - fname=${url##*/} - stdlib::debug "Computed filename='${fname}' given URL." - fi - [[ -d ${dir} ]] || mkdir "${dir}" - local attempt=0 - local max_retries=7 - # store gcs command as array and then split when called by stdlib::cmd - if stdlib::cmd gcloud help storage cp &>/dev/null; then - gcs_command=(gcloud storage cp --no-user-output-enabled) - else - gcs_command=(gsutil -q cp) - fi - while [[ $attempt -le $max_retries ]]; do - if [[ $attempt -gt 0 ]]; then - local wait=$((2 ** attempt)) - stdlib::error "Retry attempt ${attempt} of ${max_retries} with exponential backoff: ${wait} seconds." - sleep $wait - fi - if stdlib::cmd "${gcs_command[@]}" "${url}" "${dir}/${fname}"; then - break - else - stdlib::error "${gcs_command[*]} reported non-zero exit code fetching ${url}." - ((attempt++)) - fi - done -} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh deleted file mode 100644 index eac2b2e32a..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex -REQ_ANSIBLE_VERSION=2.15 -REQ_ANSIBLE_PIP_VERSION=8.7.0 -REQ_PIP_WHEEL_VERSION=0.45.1 -REQ_PIP_SETUPTOOLS_VERSION=80.8.0 -REQ_PIP_MAJOR_VERSION=25 -REQ_PYTHON3_VERSION=9 - -apt_wait() { - while fuser /var/lib/apt/lists/lock >/dev/null 2>&1; do - echo "Sleeping for apt lists lock" - sleep 3 - done -} - -# Installs any dependencies needed for python based on the OS -install_python_deps() { - # this file is present on both Debian and Ubuntu OSes - if [ -f /etc/debian_version ]; then - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3-setuptools python3-venv - fi -} - -# Gets the name of the python executable for python starting with python3, then -# checking python. Sets the variable to an empty string if neither are found. -get_python_path() { - python_path="" - if command -v python3 1>/dev/null; then - python_path=$(command -v python3) - elif command -v python 1>/dev/null; then - python_path=$(command -v python) - fi -} - -# Returns the python major version. If provided, it will use the first argument -# as the python executable, otherwise it will default to simply "python". -get_python_major_version() { - python_path=${1:-python} - python_major_version=$(${python_path} -c "import sys; print(sys.version_info.major)") -} - -# Returns the python minor version. If provided, it will use the first argument -# as the python executable, otherwise it will default to simply "python". -get_python_minor_version() { - python_path=${1:-python} - python_minor_version=$(${python_path} -c "import sys; print(sys.version_info.minor)") -} - -# Install python3 with the yum package manager. Updates python_path to the -# newly installed packaged. -install_python3_dnf() { - major_version=$(rpm -E "%{rhel}") - set -- "--disablerepo=*" "--enablerepo=baseos,appstream" - if grep -qi 'ID="rhel"' /etc/os-release; then - # Do not set --disablerepo / --enablerepo on RedHat, due to - # complex repo names; clear array - set -- - fi - # On Rocky Linux 9, Python 3.9 is installed by default but this - # has already been dropped by ansible-core for control nodes. - # https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix - # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where - # it is available as "python3*" but must be named explicitly on - # older releases. It also ensures longer support for Ansible. - if [ "${major_version}" -lt "10" ]; then - dnf install "$@" -y python3.12 python3.12-pip - python_path=$(command -v python3.12) - else - dnf install "$@" -y python3 python3-pip - python_path=$(command -v python3) - fi -} - -# Install python3 with the apt package manager. Updates python_path to the -# newly installed packaged. -install_python3_apt() { - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3 python3-setuptools python3-pip python3-venv - python_path=$(command -v python3) -} - -install_python3() { - if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || - [ -f /etc/system-release ]; then - install_python3_dnf - elif [ -f /etc/debian_version ]; then - install_python3_apt - else - echo "Error: Unsupported Distribution" - return 1 - fi -} - -# Install pip3 with the dnf package manager. Updates python_path to the -# newly installed packaged. -install_pip3_dnf() { - major_version=$(rpm -E "%{rhel}") - set -- "--disablerepo=*" "--enablerepo=baseos,appstream" - if grep -qi 'ID="rhel"' /etc/os-release; then - # Do not set --disablerepo / --enablerepo on RedHat, due to complex repo names - # clear array - set -- - fi - # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where - # it is available as "python3*" but must be named explicitly on - # older releases. It also ensures longer support for Ansible. - if [ "${major_version}" -lt "10" ]; then - dnf install "$@" -y python3.12-pip - else - dnf install "$@" -y python3-pip - fi -} - -# Install pip3 with the apt package manager. Updates python_path to the -# newly installed packaged. -install_pip3_apt() { - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3-pip -} - -install_pip3() { - if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || - [ -f /etc/system-release ]; then - install_pip3_dnf - elif [ -f /etc/debian_version ]; then - install_pip3_apt - else - echo "Error: Unsupported Distribution" - return 1 - fi -} - -main() { - if [ $# -gt 1 ]; then - echo "Error: provide only 1 optional argument identifying virtual environment path for Ansible" - return 1 - fi - - venv_path="${1:-/usr/local/ghpc-venv}" - - # Get the python3 executable, or install it if not found - get_python_path - get_python_major_version "${python_path}" - get_python_minor_version "${python_path}" - if [ "${python_path}" = "" ] || [ "${python_major_version}" = "2" ] || [ "${python_minor_version}" -lt "${REQ_PYTHON3_VERSION}" ]; then - if ! install_python3; then - return 1 - fi - get_python_major_version "${python_path}" - get_python_minor_version "${python_path}" - else - install_python_deps - fi - - # Install OS-packaged pip - if ! ${python_path} -m pip --version 2>/dev/null; then - if ! install_pip3; then - return 1 - fi - fi - - # Create pip virtual environment for Cluster Toolkit - ${python_path} -m venv "${venv_path}" --copies - venv_python_path=${venv_path}/bin/python3 - - # Upgrade pip if necessary - pip_version=$(${venv_python_path} -m pip --version | sed -nr 's/^pip ([0-9]+\.[0-9]+).*$/\1/p') - pip_major_version=$(echo "${pip_version}" | cut -d '.' -f 1) - if [ "${pip_major_version}" -lt "${REQ_PIP_MAJOR_VERSION}" ]; then - ${venv_python_path} -m pip install --upgrade pip - fi - - # upgrade wheel if necessary - wheel_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^wheel" || true) - if [ "$wheel_pkg" != "wheel==${REQ_PIP_WHEEL_VERSION}" ]; then - ${venv_python_path} -m pip install -U wheel==${REQ_PIP_WHEEL_VERSION} - fi - - # upgrade setuptools if necessary - setuptools_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^setuptools" || true) - if [ "$setuptools_pkg" != "setuptools==${REQ_PIP_SETUPTOOLS_VERSION}" ]; then - ${venv_python_path} -m pip install -U setuptools==${REQ_PIP_SETUPTOOLS_VERSION} - fi - - # configure ansible to always use correct Python binary - if [ ! -f /etc/ansible/ansible.cfg ]; then - mkdir /etc/ansible - cat <<-EOF >/etc/ansible/ansible.cfg - [defaults] - interpreter_python=${venv_python_path} - stdout_callback=debug - stderr_callback=debug - EOF - fi - - # Install ansible - ansible_version="" - if command -v ansible-playbook 1>/dev/null; then - ansible_version=$(ansible-playbook --version 2>/dev/null | sed -nr 's/^ansible-playbook.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p') - ansible_major_vers=$(echo "${ansible_version}" | cut -d '.' -f 1) - ansible_minor_vers=$(echo "${ansible_version}" | cut -d '.' -f 2) - ansible_req_major_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 1) - ansible_req_minor_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 2) - fi - if [ -z "${ansible_version}" ] || [ "${ansible_major_vers}" -ne "${ansible_req_major_vers}" ] || - [ "${ansible_minor_vers}" -lt "${ansible_req_minor_vers}" ]; then - ${venv_python_path} -m pip install ansible=="${REQ_ANSIBLE_PIP_VERSION}" - fi - while read -r cmd; do - if ! [ -L "/usr/bin/${cmd}" ]; then - ln -s "${venv_path}/bin/${cmd}" "/usr/bin/${cmd}" - fi - done <<-EOF - ansible - ansible-config - ansible-connection - ansible-console - ansible-doc - ansible-galaxy - ansible-inventory - ansible-playbook - ansible-pull - ansible-test - ansible-vault - EOF -} - -main "$@" diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh deleted file mode 100644 index 375792459b..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -OS_ID="$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g')" -OS_VERSION="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g')" -OS_VERSION_MAJOR="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//')" -REBOOT_FILE="/etc/.rdma_reboot" - -if { [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; } && { [ "${OS_VERSION_MAJOR}" = "8" ]; }; then - KMOD_VERSION="$(dnf list installed | awk '$1 ~ /^kmod-idpf-irdma(\.|$)/ {print $2}')" - - # For images that do not already have Cloud RDMA drivers installed - if [ -z "${KMOD_VERSION}" ] && [ -z "${REBOOT_FILE}" ]; then - sudo dnf update -y - sudo dnf install https://depot.ciq.com/public/files/gce-accelerator/irdma-kernel-modules-el8-x86_64/irdma-repos.rpm -y - sudo dnf install kmod-idpf-irdma rdma-core libibverbs-utils librdmacm-utils infiniband-diags perftest -y - sudo touch "${REBOOT_FILE}" - reboot - fi - echo "This image has IRDMA packages already installed, exiting." - exit 0 -else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. Cloud RDMA Drivers are only supported on Rocky Linux 8." - exit 1 -fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml deleted file mode 100644 index f9b0abeb14..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Install and configure Docker - hosts: all - become: true - vars: - docker_data_root: '' - docker_daemon_config: '' - enable_docker_world_writable: false - tasks: - - name: Check if docker is installed - ansible.builtin.stat: - path: /usr/bin/docker - register: docker_binary - - name: Download Docker Installer - ansible.builtin.get_url: - url: https://get.docker.com - dest: /tmp/get-docker.sh - owner: root - group: root - mode: '0644' - when: not docker_binary.stat.exists - - name: Install Docker - ansible.builtin.command: sh /tmp/get-docker.sh - register: docker_installed - changed_when: docker_installed.rc != 0 - when: not docker_binary.stat.exists - - name: Create Docker daemon configuration - ansible.builtin.copy: - dest: /etc/docker/daemon.json - mode: '0644' - content: '{{ docker_daemon_config }}' - validate: /usr/bin/dockerd --validate --config-file %s - when: docker_daemon_config - notify: - - Restart Docker - - name: Create Docker service override directory - ansible.builtin.file: - path: /etc/systemd/system/docker.service.d - state: directory - owner: root - group: root - mode: '0755' - - name: Create Docker service override configuration - ansible.builtin.copy: - dest: /etc/systemd/system/docker.service.d/data-root.conf - mode: '0644' - content: | - [Unit] - {% if docker_data_root %} - RequiresMountsFor={{ docker_data_root }} - {% endif %} - After=mount-localssd-raid.service - - name: Create Docker socket override directory - ansible.builtin.file: - path: /etc/systemd/system/docker.socket.d - state: directory - owner: root - group: root - mode: '0755' - when: enable_docker_world_writable - - name: Create Docker socket override configuration - ansible.builtin.copy: - dest: /etc/systemd/system/docker.socket.d/world-writable.conf - mode: '0644' - content: | - [Socket] - SocketMode=0666 - when: enable_docker_world_writable - notify: - - Reload SystemD - - Recreate Docker socket - - name: Delete Docker socket override configuration - ansible.builtin.file: - path: /etc/systemd/system/docker.socket.d/world-writable.conf - state: absent - when: not enable_docker_world_writable - notify: - - Reload SystemD - - Recreate Docker socket - - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - - name: Recreate Docker socket - ansible.builtin.service: - name: docker.socket - state: restarted - - name: Restart Docker - ansible.builtin.service: - name: docker.service - state: restarted - - post_tasks: - - name: Start Docker - ansible.builtin.service: - name: docker.service - state: started - enabled: true diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml deleted file mode 100644 index 9d295dfc7d..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Install network wait service for A3/A4 variants - hosts: all - become: true - tasks: - - - name: Create universal SystemD service for GPU networking delay - when: ansible_os_family == "Debian" - ansible.builtin.copy: - dest: /etc/systemd/system/delay-gpu-network.service - owner: root - group: root - mode: "0644" - content: | - [Unit] - Description=Delay boot on multi-NIC VMs until networks are routable - After=network-online.target - Wants=network-online.target - Before=google-startup-scripts.service - - [Service] - # This condition checks if the machine type is one of the supported A3/A4 variants. - # The service will only run if the machine type matches. - ExecCondition=/bin/bash -c "/usr/bin/curl -s -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/machine-type | grep -qE '(/a3-highgpu-8g|/a3-megagpu-8g|/a3-ultragpu-8g|/a4-highgpu-8g|/a4x-highgpu-4g)$'" - ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -o routable --timeout=180 - ExecStartPost=/bin/sleep 30 - - [Install] - WantedBy=multi-user.target - notify: - - Reload SystemD - - - name: Enable universal GPU network delay service - when: ansible_os_family == "Debian" - ansible.builtin.systemd_service: - name: delay-gpu-network.service - enabled: true - - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml deleted file mode 100644 index 94699471bb..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure Managed Lustre (assumes driver already installed) - hosts: all - become: true - vars: - default_lustre_port: 988 - managed_lustre_port: "{{ default_lustre_port }}" - tasks: - # Ideally changes to this file would also trigger an execution of lnetctl - # command to update accept_port but it is unclear if lnetctl supports this. - - name: Update lnet to use non-default port - when: managed_lustre_port | int != {{ default_lustre_port }} - ansible.builtin.copy: - owner: root - group: root - mode: '0644' - dest: /etc/modprobe.d/lnet.conf - content: | - options lnet accept_port={{ managed_lustre_port | int }} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh deleted file mode 100644 index eb4bf899b8..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -LEGACY_MONITORING_PACKAGE='stackdriver-agent' -LEGACY_MONITORING_SCRIPT_URL='https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh' -LEGACY_LOGGING_PACKAGE='google-fluentd' -LEGACY_LOGGING_SCRIPT_URL='https://dl.google.com/cloudagents/add-logging-agent-repo.sh' - -OPSAGENT_PACKAGE='google-cloud-ops-agent' -OPSAGENT_SCRIPT_URL='https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh' - -ops_or_legacy="${1:-legacy}" - -fail() { - echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" - exit 1 -} - -handle_debian() { - is_legacy_monitoring_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_MONITORING_PACKAGE} | - grep "${LEGACY_MONITORING_PACKAGE} is installed" - } - - is_legacy_logging_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_LOGGING_PACKAGE} | - grep "${LEGACY_LOGGING_PACKAGE} is installed" - } - - is_legacy_installed() { - is_legacy_monitoring_installed || is_legacy_logging_installed - } - - is_opsagent_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${OPSAGENT_PACKAGE} | - grep "${OPSAGENT_PACKAGE} is installed" - } - - install_with_retry() { - MAX_RETRY=50 - RETRY=0 - until [ ${RETRY} -eq ${MAX_RETRY} ] || curl -s "${1}" | bash -s -- --also-install; do - RETRY=$((RETRY + 1)) - echo "WARNING: Installation of ${1} failed on try ${RETRY} of ${MAX_RETRY}" - sleep 5 - done - if [ $RETRY -eq $MAX_RETRY ]; then - echo "ERROR: Installation of ${1} was not successful after ${MAX_RETRY} attempts." - exit 1 - fi - } - - install_opsagent() { - install_with_retry "${OPSAGENT_SCRIPT_URL}" - } - - install_stackdriver_agent() { - install_with_retry "${LEGACY_MONITORING_SCRIPT_URL}" - install_with_retry "${LEGACY_LOGGING_SCRIPT_URL}" - service stackdriver-agent start - service google-fluentd start - } -} - -handle_redhat() { - is_legacy_monitoring_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_MONITORING_PACKAGE} | - grep "${LEGACY_MONITORING_PACKAGE} is installed" - } - - is_legacy_logging_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_LOGGING_PACKAGE} | - grep "${LEGACY_LOGGING_PACKAGE} is installed" - } - - is_legacy_installed() { - is_legacy_monitoring_installed || is_legacy_logging_installed - } - - is_opsagent_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${OPSAGENT_PACKAGE} | - grep "${OPSAGENT_PACKAGE} is installed" - } - - install_opsagent() { - curl -s "${OPSAGENT_SCRIPT_URL}" | bash -s -- --also-install - } - - install_stackdriver_agent() { - curl -sS "${LEGACY_MONITORING_SCRIPT_URL}" | bash -s -- --also-install - curl -sS "${LEGACY_LOGGING_SCRIPT_URL}" | bash -s -- --also-install - service stackdriver-agent start - service google-fluentd start - } -} - -main() { - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - handle_redhat - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - handle_debian - else - fail "Unsupported platform." - fi - - # Handle cases that agent is already installed - if [[ -z "$(is_legacy_monitoring_installed)" && -n $(is_legacy_logging_installed) ]] || - [[ -n "$(is_legacy_monitoring_installed)" && -z $(is_legacy_logging_installed) ]]; then - fail "Bad state: legacy agent is partially installed" - elif [[ "${ops_or_legacy}" == "legacy" ]] && is_legacy_installed; then - echo "Legacy agent is already installed" - exit 0 - elif [[ "${ops_or_legacy}" != "legacy" ]] && is_opsagent_installed; then - echo "Ops agent is already installed" - exit 0 - elif is_legacy_installed || is_opsagent_installed; then - fail "Agent is already installed but does not match requested agent of ${ops_or_legacy}" - fi - - # install agent - if [[ "${ops_or_legacy}" == "legacy" ]]; then - echo "Installing legacy monitoring agent (stackdriver)" - install_stackdriver_agent - else - echo "Installing cloud ops agent" - echo "WARNING: cloud ops agent may have a performance impact. Consider using legacy monitoring agent (stackdriver)." - install_opsagent - fi -} - -main diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh deleted file mode 100644 index 738181aafb..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SCRIPT_COMPLETE_FILE="/run/startup_script_msg" - -# Ensure we're in an interactive terminal and not root -if [ -t 1 ] && [ "$(id -u)" -ne 0 ]; then - # Check if the file has contents otherwise skip - if [ -s "$SCRIPT_COMPLETE_FILE" ]; then - echo - cat "$SCRIPT_COMPLETE_FILE" - echo - fi -fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml deleted file mode 100644 index d94aac81fd..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Configure local SSDs - become: true - hosts: localhost - vars: - raid_name: localssd - array_dev: /dev/md/{{ raid_name }} - fstype: ext4 - interface: nvme - mode: '0755' - mountpoint: /mnt/{{ raid_name }} - tasks: - - name: Get local SSD devices - ansible.builtin.find: - file_type: link - path: /dev/disk/by-id - patterns: google-local-{{ "nvme-" if interface == "nvme" else "" }}ssd-* - register: local_ssd_devices - - - name: Exit if zero local ssd found - ansible.builtin.meta: end_play - when: local_ssd_devices.files | length == 0 - - - name: Install mdadm - ansible.builtin.package: - name: mdadm - state: present - - # this service will act during the play and upon reboots to ensure that local - # SSD volumes are always assembled into a RAID and re-formatted if necessary; - # there are many scenarios where a VM can be stopped or migrated during - # maintenance and the contents of local SSD will be discarded - - name: Install service to create local SSD RAID and format it - ansible.builtin.copy: - dest: /etc/systemd/system/create-localssd-raid.service - mode: 0644 - content: | - [Unit] - After=local-fs.target - Before=slurmd.service docker.service - ConditionPathExists=!{{ array_dev }} - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/usr/bin/bash -c "/usr/sbin/mdadm --create {{ array_dev }} --name={{ raid_name }} --homehost=any --level=0 --raid-devices={{ local_ssd_devices.files | length }} /dev/disk/by-id/google-local-nvme-ssd-*{{ " --force" if local_ssd_devices.files | length == 1 else "" }}" - ExecStartPost=/usr/sbin/mkfs -t {{ fstype }}{{ " -m 0" if fstype == "ext4" else "" }} {{ array_dev }} - - [Install] - WantedBy=slurmd.service docker.service - - - name: Create RAID array and format - ansible.builtin.systemd: - name: create-localssd-raid.service - state: started - enabled: true - daemon_reload: true - - - name: Install service to mount local SSD array - ansible.builtin.copy: - dest: /etc/systemd/system/mount-localssd-raid.service - mode: 0644 - content: | - [Unit] - After=local-fs.target create-localssd-raid.service - Before=slurmd.service docker.service - Wants=create-localssd-raid.service - ConditionPathIsMountPoint=!{{ mountpoint }} - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/usr/bin/systemd-mount -t {{ fstype }} -o discard,defaults,nofail {{ array_dev }} {{ mountpoint }} - ExecStartPost=/usr/bin/chmod {{ mode }} {{ mountpoint }} - ExecStop=/usr/bin/systemd-umount {{ mountpoint }} - - [Install] - WantedBy=slurmd.service docker.service - - - name: Mount RAID array and set permissions - ansible.builtin.systemd: - name: mount-localssd-raid.service - state: started - enabled: true - daemon_reload: true diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh deleted file mode 100644 index 1c8018fb01..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! -d ~/.ssh/ ]; then - source /usr/local/ghpc-venv/bin/activate - ansible-playbook /usr/local/ghpc/setup-ssh-keys.yml -fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml deleted file mode 100644 index 692896bb9c..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Setup SSH Keys for user - become: false - hosts: localhost - vars: - pub_key_path: "{{ ansible_env.HOME }}/.ssh" - pub_key_file: "{{ pub_key_path }}/id_rsa" - auth_key_file: "{{ pub_key_path }}/authorized_keys" - tasks: - - name: "Create .ssh folder" - ansible.builtin.file: - path: "{{ pub_key_path }}" - state: directory - mode: 0700 - owner: "{{ ansible_user_id }}" - - name: Create keys - community.crypto.openssh_keypair: - path: "{{ pub_key_file }}" - owner: "{{ ansible_user_id }}" - - name: Copy public key to authorized keys - ansible.builtin.copy: - src: "{{ pub_key_file }}.pub" - dest: "{{ auth_key_file }}" - owner: "{{ ansible_user_id }}" - mode: 0644 diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh deleted file mode 100644 index 8ca40bc73f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh +++ /dev/null @@ -1,39 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This code contains minor changes from the original: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 - -stdlib::main() { - DELETE_AT_EXIT="$(mktemp -d)" - readonly DELETE_AT_EXIT - - # Initialize state required by other functions, e.g. debug() - stdlib::init - stdlib::debug "Loaded startup-script-stdlib as an executable." - - stdlib::load_config_values - - stdlib::load_runners -} - -# if script is being executed and not sourced. -if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then - stdlib::finish() { - [[ -d ${DELETE_AT_EXIT:-} ]] && rm -rf "${DELETE_AT_EXIT}" - } - trap stdlib::finish EXIT - - stdlib::main "$@" -fi diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh deleted file mode 100644 index 589a3215ab..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh +++ /dev/null @@ -1,266 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This code contains minor changes from the original in: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 - -# Standard library of functions useful for startup scripts. - -# These are outside init_global_vars so logging functions work with the most -# basic case of `source startup-script-stdlib.sh` -readonly SYSLOG_DEBUG_PRIORITY="${SYSLOG_DEBUG_PRIORITY:-syslog.debug}" -readonly SYSLOG_INFO_PRIORITY="${SYSLOG_INFO_PRIORITY:-syslog.info}" -readonly SYSLOG_ERROR_PRIORITY="${SYSLOG_ERROR_PRIORITY:-syslog.error}" -# Global counter of how many times stdlib::init() has been called. -STARTUP_SCRIPT_STDLIB_INITIALIZED=0 - -# Error codes -readonly E_RUN_OR_DIE=5 -readonly E_MISSING_MANDATORY_ARG=9 -readonly E_UNKNOWN_ARG=10 - -SCRIPT_COMPLETE_FILE="/run/startup_script_msg" -SUCCESS_MESSAGE="* NOTICE **: The Cluster Toolkit startup scripts have finished running successfully." -readonly SUCCESS_MESSAGE -ERROR_MESSAGE="** ERROR **: The Cluster Toolkit startup scripts have finished running, but produced an error." -readonly ERROR_MESSAGE -WARNING_MESSAGE="** WARNING **: The Cluster Toolkit startup scripts are currently running." -readonly WARNING_MESSAGE - -stdlib::debug() { - [[ -z ${DEBUG:-} ]] && return 0 - local ds msg - msg="$*" - logger -p "${SYSLOG_DEBUG_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - [[ -n ${QUIET:-} ]] && return 0 - ds="$(date +"${DATE_FMT}") " - echo -e "${BLUE}${ds}Debug [$$]: ${msg}${NC}" >&2 -} - -stdlib::info() { - local ds msg - msg="$*" - logger -p "${SYSLOG_INFO_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - [[ -n ${QUIET:-} ]] && return 0 - ds="$(date +"${DATE_FMT}") " - echo -e "${GREEN}${ds}Info [$$]: ${msg}${NC}" >&2 -} - -stdlib::error() { - local ds msg - msg="$*" - ds="$(date +"${DATE_FMT}") " - logger -p "${SYSLOG_ERROR_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - echo -e "${RED}${ds}Error [$$]: ${msg}${NC}" >&2 -} - -stdlib::announce_runners_start() { - if [ -z "$recursive_proc" ]; then - wall -n "$WARNING_MESSAGE" - echo "$WARNING_MESSAGE" >"$SCRIPT_COMPLETE_FILE" - fi - export recursive_proc=$((${recursive_proc:=0} + 1)) -} - -stdlib::announce_runners_end() { - exit_code=$1 - export recursive_proc=$((${recursive_proc:=0} - 1)) - if [ "$recursive_proc" -le "0" ]; then - if [ "$exit_code" -ne "0" ]; then - wall -n "$ERROR_MESSAGE" - echo "$ERROR_MESSAGE" >"$SCRIPT_COMPLETE_FILE" - else - wall -n "$SUCCESS_MESSAGE" - echo -n "" >"$SCRIPT_COMPLETE_FILE" - fi - fi -} - -# The main initialization function of this library. This should be kept to the -# minimum amount of work required for all functions to operate cleanly. -stdlib::init() { - if [[ ${STARTUP_SCRIPT_STDLIB_INITIALIZED} -gt 0 ]]; then - stdlib::info 'stdlib::init()'" already initialized, no action taken." - return 0 - fi - ((STARTUP_SCRIPT_STDLIB_INITIALIZED++)) || true - stdlib::init_global_vars - stdlib::init_directories - stdlib::debug "stdlib::init(): startup-script-stdlib.sh initialized and ready" -} - -# Initialize global variables. -stdlib::init_global_vars() { - # The program name, used for logging. - readonly PROG="${PROG:-startup-script-stdlib}" - # Date format used for stderr logging. Passed to date + command. - readonly DATE_FMT="${DATE_FMT:-"%a %b %d %H:%M:%S %z %Y"}" - # var directory - readonly VARDIR="${VARDIR:-/var/lib/startup}" - # Override this with file://localhost/tmp/foo/bar in spec test context - readonly METADATA_BASE="${METADATA_BASE:-http://metadata.google.internal}" - - # Color variables - if [[ -n ${COLOR:-} ]]; then - readonly NC='\033[0m' # no color - readonly RED='\033[0;31m' # error - readonly GREEN='\033[0;32m' # info - readonly BLUE='\033[0;34m' # debug - else - readonly NC='' - readonly RED='' - readonly GREEN='' - readonly BLUE='' - fi - - return 0 -} - -stdlib::init_directories() { - if ! [[ -e ${VARDIR} ]]; then - install -d -m 0755 -o 0 -g 0 "${VARDIR}" - fi -} - -## -# Get a metadata key. When used without -o, this function is guaranteed to -# produce no output on STDOUT other than the retrieved value. This is intended -# to support the use case of -# FOO="$(stdlib::metadata_get -k instance/attributes/foo)" -# -# If the requested key does not exist, the error code will be 22 and zero bytes -# written to STDOUT. -stdlib::metadata_get() { - local OPTIND opt key outfile - local metadata="${METADATA_BASE%/}/computeMetadata/v1" - local exit_code - while getopts ":k:o:" opt; do - case "${opt}" in - k) key="${OPTARG}" ;; - o) outfile="${OPTARG}" ;; - :) - stdlib::error "Invalid option: -${OPTARG} requires an argument" - stdlib::metadata_get_usage - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error "Unknown option: -${opt}" - stdlib::metadata_get_usage - return "${E_UNKNOWN_ARG}" - ;; - esac - done - local url="${metadata}/${key#/}" - - stdlib::debug "Getting metadata resource url=${url}" - if [[ -z ${outfile:-} ]]; then - curl --location --silent --connect-timeout 1 --fail \ - -H 'Metadata-Flavor: Google' "$url" 2>/dev/null - exit_code=$? - else - stdlib::cmd curl --location \ - --silent \ - --connect-timeout 1 \ - --fail \ - --output "${outfile}" \ - -H 'Metadata-Flavor: Google' \ - "$url" - exit_code=$? - fi - case "${exit_code}" in - 22 | 37) - stdlib::debug "curl exit_code=${exit_code} for url=${url}" \ - "(Does not exist)" - ;; - esac - return "${exit_code}" -} - -stdlib::metadata_get_usage() { - stdlib::info 'Usage: stdlib::metadata_get -k ' - stdlib::info 'For example: stdlib::metadata_get -k instance/attributes/startup-config' -} - -# Load configuration values in the spirit of /etc/sysconfig defaults, but from -# metadata instead of the filesystem. -stdlib::load_config_values() { - local config_file - local key="instance/attributes/startup-script-config" - # shellcheck disable=SC2119 - config_file="$(stdlib::mktemp)" - stdlib::metadata_get -k "${key}" -o "${config_file}" - local status=$? - case "$status" in - 0) - stdlib::debug "SUCCESS: Configuration data sourced from $key" - ;; - 22 | 37) - stdlib::debug "no configuration data loaded from $key" - ;; - *) - stdlib::error "metadata_get -k $key returned unknown status=${status}" - ;; - esac - # shellcheck source=/dev/null - source "${config_file}" -} - -# Run a command logging the entry and exit. Intended for system level commands -# and operational debugging. Not intended for use with redirection. This is -# not named run() because bats uses a run() function. -stdlib::cmd() { - local exit_code argv=("$@") - stdlib::debug "BEGIN: stdlib::cmd() command=[${argv[*]}]" - "${argv[@]}" - exit_code=$? - stdlib::debug "END: stdlib::cmd() command=[${argv[*]}] exit_code=${exit_code}" - return $exit_code -} - -# Run a command successfully or exit the program with an error. -stdlib::run_or_die() { - if ! stdlib::cmd "$@"; then - stdlib::error "stdlib::run_or_die(): exiting with exit code ${E_RUN_OR_DIE}." - exit "${E_RUN_OR_DIE}" - fi -} - -# Intended to take advantage of automatic cleanup of startup script library -# temporary files without exporting a modified TMPDIR to child processes, which -# would cause the children to have their TMPDIR deleted out from under them. -# shellcheck disable=SC2120 -stdlib::mktemp() { - TMPDIR="${DELETE_AT_EXIT:-${TMPDIR}}" mktemp "$@" -} - -# Return a nice error message if a mandatory argument is missing. -stdlib::mandatory_argument() { - local OPTIND opt name flag - while getopts ":n:f:" opt; do - case "$opt" in - n) name="${OPTARG}" ;; - f) flag="${OPTARG}" ;; - :) - stdlib::error "Invalid argument: -${OPTARG} requires an argument to stdlib::mandatory_argument()" - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error "Unknown argument: -${OPTARG}" - stdlib::info "Usage: stdlib::mandatory_argument -n -f " - return "${E_UNKNOWN_ARG}" - ;; - esac - done - stdlib::error "Invalid argument: -${flag} requires an argument to ${name}()." -} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/main.tf b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/main.tf deleted file mode 100644 index 02124eeddc..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/main.tf +++ /dev/null @@ -1,306 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "startup-script", ghpc_role = "scripts" }) -} - -locals { - monitoring_agent_installer = ( - var.install_cloud_ops_agent || var.install_stackdriver_agent ? - [{ - type = "shell" - source = "${path.module}/files/install_monitoring_agent.sh" - destination = "install_monitoring_agent_automatic.sh" - args = var.install_cloud_ops_agent ? "ops" : "legacy" # install legacy (stackdriver) - }] : - [] - ) - - warnings = [ - { - type = "data" - content = file("${path.module}/files/running-script-warning.sh") - destination = "/etc/profile.d/99-running-script-warning.sh" - } - ] - - configure_ssh = length(var.configure_ssh_host_patterns) > 0 - host_args = { - host_name_prefix = var.configure_ssh_host_patterns - } - - prefix_file = "/tmp/prefix_file.json" - ansible_docker_settings_file = "/tmp/ansible_docker_settings.json" - - docker_config = try(jsondecode(var.docker.daemon_config), {}) - docker_data_root = try(local.docker_config.data-root, null) - - configure_ssh_runners = local.configure_ssh ? [ - { - type = "data" - source = "${path.module}/files/setup-ssh-keys.sh" - destination = "/usr/local/ghpc/setup-ssh-keys.sh" - }, - { - type = "data" - source = "${path.module}/files/setup-ssh-keys.yml" - destination = "/usr/local/ghpc/setup-ssh-keys.yml" - }, - { - type = "data" - content = jsonencode(local.host_args) - destination = local.prefix_file - }, - { - type = "ansible-local" - content = file("${path.module}/files/configure-ssh.yml") - destination = "configure-ssh.yml" - args = "-e @${local.prefix_file}" - } - ] : [] - - proxy_runner = var.http_proxy == "" ? [] : [ - { - type = "data" - destination = "/etc/profile.d/http_proxy.sh" - content = <<-EOT - #!/bin/bash - export http_proxy=${var.http_proxy} - export https_proxy=${var.http_proxy} - export NO_PROXY=${var.http_no_proxy} - EOT - }, - { - type = "shell" - source = "${path.module}/files/configure_proxy.sh" - destination = "configure_proxy.sh" - args = var.http_proxy - } - ] - - ofi_runner = !var.set_ofi_cloud_rdma_tunables ? [] : [ - { - type = "data" - destination = "/etc/profile.d/set_ofi_cloud_rdma_tunables.sh" - content = <<-EOT - #!/bin/bash - export FI_PROVIDER="verbs;ofi_rxm" - export FI_OFI_RXM_USE_RNDV_WRITE=0 - export FI_VERBS_INLINE_SIZE=39 - export I_MPI_FABRICS="shm:ofi" - export FI_UNIVERSE_SIZE=1024 - export I_MPI_ADJUST_ALLTOALL=1 - export I_MPI_ADJUST_IALLTOALL=1 - export I_MPI_ADJUST_BCAST=4 - export I_MPI_ADJUST_IBCAST=1 - EOT - }, - ] - - rdma_runner = !var.install_cloud_rdma_drivers ? [] : [ - { - type = "shell" - source = "${path.module}/files/install_cloud_rdma_drivers.sh" - destination = "install_cloud_rdma_drivers.sh" - } - ] - - docker_runner = !var.docker.enabled ? [] : [ - { - type = "data" - destination = local.ansible_docker_settings_file - content = jsonencode({ - enable_docker_world_writable = var.docker.world_writable - docker_daemon_config = var.docker.daemon_config - docker_data_root = local.docker_data_root - }) - }, - { - type = "ansible-local" - destination = "install_docker.yml" - content = file("${path.module}/files/install_docker.yml") - args = "-e \"@${local.ansible_docker_settings_file}\"" - }, - ] - - managed_lustre_runner = !var.managed_lustre.enabled ? [] : [ - { - type = "ansible-local" - destination = "install_managed_lustre.yml" - content = file("${path.module}/files/install_managed_lustre.yml") - args = "-e managed_lustre_port=${var.managed_lustre.port}" - }, - ] - - gpu_network_wait_online_runner = !var.enable_gpu_network_wait_online ? [] : [ - { - type = "ansible-local" - destination = "install_gpu_network_wait_online.yml" - content = file("${path.module}/files/install_gpu_network_wait_online.yml") - args = "" - }, - ] - - local_ssd_filesystem_enabled = can(coalesce(var.local_ssd_filesystem.mountpoint)) - raid_setup = !local.local_ssd_filesystem_enabled ? [] : [ - { - type = "ansible-local" - destination = "setup-raid.yml" - content = file("${path.module}/files/setup-raid.yml") - args = join(" ", [ - "-e mountpoint=${var.local_ssd_filesystem.mountpoint}", - "-e fs_type=${var.local_ssd_filesystem.fs_type}", - "-e mode=${var.local_ssd_filesystem.permissions}", - ]) - }, - ] - - supplied_ansible_runners = anytrue([for r in var.runners : r.type == "ansible-local"]) - has_ansible_runners = anytrue([ - local.supplied_ansible_runners, - local.configure_ssh, - var.docker.enabled, - var.managed_lustre.enabled, - var.enable_gpu_network_wait_online, - local.local_ssd_filesystem_enabled - ]) - - install_ansible = coalesce(var.install_ansible, local.has_ansible_runners) - ansible_installer = local.install_ansible ? [{ - type = "shell" - source = "${path.module}/files/install_ansible.sh" - destination = "install_ansible_automatic.sh" - args = var.ansible_virtualenv_path - }] : [] - - hotfix_runner = [{ - type = "shell" - source = "${path.module}/files/early_run_hotfixes.sh" - destination = "early_run_hotfixes.sh" - }] - - runners = concat( - local.warnings, - local.hotfix_runner, - local.proxy_runner, - local.ofi_runner, - local.rdma_runner, - local.monitoring_agent_installer, - local.ansible_installer, - local.raid_setup, # order RAID early to ensure filesystem is ready for subsequent runners - local.managed_lustre_runner, - local.configure_ssh_runners, - local.docker_runner, - local.gpu_network_wait_online_runner, - var.runners - ) - - bucket_regex = "^gs://([^/]*)/*(.*)" - gcs_bucket_path_trimmed = var.gcs_bucket_path == null ? null : trimsuffix(var.gcs_bucket_path, "/") - storage_folder_path = local.gcs_bucket_path_trimmed == null ? null : regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[1] - storage_folder_path_prefix = local.storage_folder_path == null || local.storage_folder_path == "" ? "" : "${local.storage_folder_path}/" - - user_provided_bucket_name = try(regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[0], null) - storage_bucket_name = coalesce(one(google_storage_bucket.configs_bucket[*].name), local.user_provided_bucket_name) - - load_runners = templatefile( - "${path.module}/templates/startup-script-custom.tftpl", - { - bucket = local.storage_bucket_name, - http_proxy = var.http_proxy, - no_proxy = var.http_no_proxy, - runners = [ - for runner in local.runners : { - object = google_storage_bucket_object.scripts[basename(runner["destination"])].output_name - type = runner["type"] - destination = runner["destination"] - args = contains(keys(runner), "args") ? runner["args"] : "" - } - ] - } - ) - - stdlib_head = file("${path.module}/files/startup-script-stdlib-head.sh") - get_from_bucket = file("${path.module}/files/get_from_bucket.sh") - stdlib_body = file("${path.module}/files/startup-script-stdlib-body.sh") - - # List representing complete content, to be concatenated together. - stdlib_list = [ - local.stdlib_head, - local.get_from_bucket, - local.load_runners, - local.stdlib_body, - ] - - # Final content output to the user - stdlib = join("", local.stdlib_list) - - runners_map = { for runner in local.runners : - basename(runner["destination"]) => { - content = lookup(runner, "content", null) - source = lookup(runner, "source", null) - } - } -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket" "configs_bucket" { - count = var.gcs_bucket_path == null ? 1 : 0 - project = var.project_id - name = "${var.deployment_name}-startup-scripts-${random_id.resource_name_suffix.hex}" - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -resource "google_storage_bucket_iam_binding" "viewers" { - bucket = local.storage_bucket_name - role = "roles/storage.objectViewer" - members = var.bucket_viewers -} - -resource "google_storage_bucket_object" "scripts" { - # this writes all scripts exactly once into GCS - for_each = local.runners_map - name = "${local.storage_folder_path_prefix}${each.key}-${substr(try(md5(each.value.content), filemd5(each.value.source)), 0, 4)}" - content = each.value.content - source = each.value.source - source_md5hash = each.value.content != null && each.value.content != "" ? md5(each.value.content) : filemd5(each.value.source) - bucket = local.storage_bucket_name - timeouts { - create = "10m" - update = "10m" - } - - lifecycle { - precondition { - condition = !(var.install_cloud_ops_agent && var.install_stackdriver_agent) - error_message = "Only one of var.install_stackdriver_agent or var.install_cloud_ops_agent can be set. Stackdriver is recommended for best performance." - } - } -} - -resource "local_file" "debug_file" { - for_each = toset(var.debug_file != null ? [var.debug_file] : []) - filename = var.debug_file - content = local.stdlib -} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml deleted file mode 100644 index 2ada34471f..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - storage.googleapis.com diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/outputs.tf b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/outputs.tf deleted file mode 100644 index 6a15082814..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/outputs.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "script to load and run all runners, as a string value." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} - -output "compute_startup_script" { - description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} - -output "controller_startup_script" { - description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl deleted file mode 100644 index 3c894b00b0..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl +++ /dev/null @@ -1,65 +0,0 @@ - - -stdlib::run_playbook() { - if [ ! "$(which ansible-playbook)" ]; then - stdlib::error "ansible-playbook not found"\ - "Please install ansible before running ansible-local runners." - exit 1 - fi - ansible-playbook --connection=local --inventory=localhost, --limit localhost $1 $2 - ret_code=$? - return $${ret_code} -} - -stdlib::runner() { - - type=$1 - object=$2 - destination=$3 - tmpdir=$4 - args=$5 - - destpath="$(dirname $destination)" - filename="$(basename $destination)" - - if [ "$destpath" = "." ]; then - destpath=$tmpdir - fi - - stdlib::get_from_bucket -u "gs://${bucket}/$object" -d "$destpath" -f "$filename" - - stdlib::info "=== start executing runner: $object ===" - case "$1" in - ansible-local) stdlib::run_playbook "$destpath/$filename" "$args";; - shell) chmod u+x /$destpath/$filename && $destpath/$filename $args;; - esac - - exit_code=$? - stdlib::info "=== $object finished with exit_code=$exit_code ===" - if [ "$exit_code" -ne "0" ] ; then - stdlib::error "=== execution of $object failed, exiting ===" - stdlib::announce_runners_end "$exit_code" - exit $exit_code - fi -} - -stdlib::load_runners(){ - tmpdir="$(mktemp -d)" - - stdlib::debug "=== BEGIN Running runners ===" - stdlib::announce_runners_start - - %{if http_proxy != "" ~} - stdlib::info "=== Setting HTTP_PROXY,HTTPS_PROXY to ${http_proxy} ===" - export http_proxy=${http_proxy} - export https_proxy=${http_proxy} - export NO_PROXY=${no_proxy} - %{endif ~} - - %{for r in runners ~} - stdlib::runner "${r.type}" "${r.object}" "${r.destination}" $${tmpdir} "${r.args}" - %{endfor ~} - - stdlib::announce_runners_end "0" - stdlib::debug "=== END Running runners ===" -} diff --git a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/variables.tf b/deletion-test/primary/modules/embedded/modules/scripts/startup-script/variables.tf deleted file mode 100644 index 7080085ece..0000000000 --- a/deletion-test/primary/modules/embedded/modules/scripts/startup-script/variables.tf +++ /dev/null @@ -1,298 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used to name GCS bucket for startup scripts." - type = string -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string - default = null -} - -variable "bucket_viewers" { - description = "Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account)" - type = list(string) - default = [] - - validation { - condition = alltrue([ - for u in var.bucket_viewers : length(regexall("^(allUsers$|allAuthenticatedUsers$|user:|group:|serviceAccount:|domain:)", u)) > 0 - ]) - error_message = "Bucket viewer members must begin with user/group/serviceAccount/domain following https://cloud.google.com/iam/docs/reference/rest/v1/Policy#Binding" - } -} - -variable "debug_file" { - description = "Path to an optional local to be written with 'startup_script'." - type = string - default = null -} - -variable "labels" { - description = "Labels for the created GCS bucket. Key-value pairs." - type = map(string) -} - -variable "runners" { - description = < 0 - error_message = "The POSIX permissions for the mountpoint must be represented as a 3 or 4-digit octal" - } - - default = { - fs_type = "ext4" - mountpoint = "" - permissions = "0755" - } - - nullable = false -} - -variable "install_cloud_ops_agent" { - description = "Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true." - type = bool - default = false -} - -variable "install_stackdriver_agent" { - description = "Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance." - type = bool - default = false -} - -variable "install_ansible" { - description = "Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used." - type = bool - default = null -} - -variable "configure_ssh_host_patterns" { - description = < **_NOTE:_** if both [startup_script][sss] and [startup_script_file][ssf] are -> specified, then [startup_script_file][ssf] takes precedence. - -## Recommended use - -Because the [metadata startup script executes in parallel](#order-of-execution) -with the other solutions, conflicts can arise, especially when package managers -(`yum` or `apt`) lock their databases during package installation. Therefore, it -is recommended to choose one of the following approaches: - -1. Specify _either_ [startup_script][sss] _or_ [startup_script_file][ssf] and do - not specify [shell_scripts][shell] or [ansible_playbooks][ansible]. - - This can be especially useful in - [environments that restrict SSH access](#environments-without-ssh-access) -1. Specify any combination of [shell_scripts][shell] and - [ansible_playbooks][ansible] and do not specify [startup_script][sss] or - [startup_script_file][ssf]. - -If any of the startup script approaches fail by returning a code other than 0, -Packer will determine that the build has failed and refuse to save the image. - -## External access with SSH - -The [shell scripts][shell] and [Ansible playbooks][ansible] customization -solutions both require SSH access to the VM from the Packer execution -environment. SSH access can be enabled one of 2 ways: - -1. The VM is created without a public IP address and SSH tunnels are created - using [Identity-Aware Proxy (IAP)][iaptunnel]. - - Allow [use_iap](#input_use_iap) to take on its default value of `true` -1. The VM is created with an IP address on the public internet and firewall - rules allow SSH access from the Packer execution environment. - - Set `omit_external_ip = false` (or `omit_external_ip: false` in a - blueprint) - - Add firewall rules that open SSH to the VM - -The Packer template defaults to using to the 1st IAP-based solution because it -is more secure (no exposure to public internet) and because the [vpc] module -automatically sets up all necessary firewall rules for SSH tunneling and -outbound-only access to the internet through [Cloud NAT][cloudnat]. - -In either SSH solution, customization scripts should be supplied as files in the -[shell_scripts][shell] and [ansible_playbooks][ansible] settings. - -## Environments without SSH access - -Many network environments disallow SSH access to VMs. In these environments, the -[metadata-based startup scripts][startup-metadata] are appropriate because they -execute entirely independently of the Packer execution environment. - -In this scenario, a single scripts should be supplied in the form of a string to -the [startup_script][sss] input variable. This solution integrates well with -Toolkit runners. Runners operate by using a single startup script whose behavior -is extended by downloading and executing a customizable set of runners from -Cloud Storage at startup. - -> **_NOTE:_** Packer will attempt to use SSH if either [shell_scripts][shell] or -> [ansible_playbooks][ansible] are set to non-empty values. Leave them at their -> default, empty values to ensure access by SSH is disabled. - -## Supplying startup script as a string - -The [startup_script][sss] parameter accepts scripts formatted as strings. In -Packer and Terraform, multi-line strings can be specified using -[heredoc syntax](https://www.terraform.io/language/expressions/strings#heredoc-strings) -in an input [Packer variables file][pkrvars] (`*.pkrvars.hcl`) For example, the -following snippet defines a multi-line bash script followed by an integer -representing the size, in GiB, of the resulting image: - -```hcl -startup_script = <<-EOT - #!/bin/bash - yum install -y epel-release - yum install -y jq - EOT - -disk_size = 100 -``` - -In a blueprint, the equivalent syntax is: - -```yaml -... - settings: - startup_script: | - #!/bin/bash - yum install -y epel-release - yum install -y jq - disk_size: 100 -... -``` - -## Monitoring startup script execution - -When using startup script customization, Packer will print very limited output -to the console. For example: - -```text -==> example.googlecompute.toolkit_image: Waiting for any running startup script to finish... -==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... -==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... -==> example.googlecompute.toolkit_image: Startup script, if any, has finished running. -``` - -### Debugging startup-script failures - -> [!NOTE] -> There can be a delay in the propagation of the logs from the instance to -> Cloud Logging, so it may require waiting a few minutes to see the full logs. - -If the Packer image build fails, the module will output a `gcloud` command -that can be used directly to review startup-script execution. - -## License - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. You may obtain a copy of the -License at - -```text - http://www.apache.org/licenses/LICENSE-2.0 -``` - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. - - -## Requirements - -No requirements. - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_count](#input\_accelerator\_count) | Number of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `number` | `null` | no | -| [accelerator\_type](#input\_accelerator\_type) | Type of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `string` | `null` | no | -| [ansible\_playbooks](#input\_ansible\_playbooks) | A list of Ansible playbook configurations that will be uploaded to customize the VM image | 
list(object({
    playbook_file   = string
    galaxy_file     = string
    extra_arguments = list(string)
  }))
| `[]` | no | -| [communicator](#input\_communicator) | Communicator to use for provisioners that require access to VM ("ssh" or "winrm") | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name | `string` | n/a | yes | -| [disk\_size](#input\_disk\_size) | Size of disk image in GB | `number` | `null` | no | -| [disk\_type](#input\_disk\_type) | Type of persistent disk to provision | `string` | `"pd-balanced"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [image\_family](#input\_image\_family) | The family name of the image to be built. Defaults to `deployment_name` | `string` | `null` | no | -| [image\_name](#input\_image\_name) | The name of the image to be built. If not supplied, it will be set to image\_family-$ISO\_TIMESTAMP | `string` | `null` | no | -| [image\_storage\_locations](#input\_image\_storage\_locations) | Storage location, either regional or multi-regional, where snapshot content is to be stored and only accepts 1 value.
See https://developer.hashicorp.com/packer/plugins/builders/googlecompute#image_storage_locations | `list(string)` | `null` | no | -| [labels](#input\_labels) | Labels to apply to the short-lived VM | `map(string)` | `null` | no | -| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2-standard-4"` | no | -| [manifest\_file](#input\_manifest\_file) | File to which to write Packer build manifest | `string` | `"packer-manifest.json"` | no | -| [metadata](#input\_metadata) | Instance metadata for the builder VM (use var.startup\_script or var.startup\_script\_file to set startup-script metadata) | `map(string)` | `{}` | no | -| [network\_project\_id](#input\_network\_project\_id) | Project ID of Shared VPC network | `string` | `null` | no | -| [omit\_external\_ip](#input\_omit\_external\_ip) | Provision the image building VM without a public IP address | `bool` | `true` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which to create VM and image | `string` | n/a | yes | -| [scopes](#input\_scopes) | DEPRECATED: use var.service\_account\_scopes | `set(string)` | `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | The service account email to use. If null or 'default', then the default Compute Engine service account will be used. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Service account scopes to attach to the instance. See
https://cloud.google.com/compute/docs/access/service-accounts. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shell\_scripts](#input\_shell\_scripts) | A list of paths to local shell scripts which will be uploaded to customize the VM image | `list(string)` | `[]` | no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [source\_image](#input\_source\_image) | Source OS image to build from | `string` | `null` | no | -| [source\_image\_family](#input\_source\_image\_family) | Alternative to source\_image. Specify image family to build from latest image in family | `string` | `"hpc-rocky-linux-8"` | no | -| [source\_image\_project\_id](#input\_source\_image\_project\_id) | A list of project IDs to search for the source image. Packer will search the
first project ID in the list first, and fall back to the next in the list,
until it finds the source image. | `list(string)` | `null` | no | -| [ssh\_username](#input\_ssh\_username) | Username to use for SSH access to VM | `string` | `"hpc-toolkit-packer"` | no | -| [startup\_script](#input\_startup\_script) | Startup script (as raw string) used to build the custom Linux VM image (overridden by var.startup\_script\_file if both are set) | `string` | `null` | no | -| [startup\_script\_file](#input\_startup\_script\_file) | File path to local shell script that will be used to customize the Linux VM image (overrides var.startup\_script) | `string` | `null` | no | -| [state\_timeout](#input\_state\_timeout) | The time to wait for instance state changes, including image creation | `string` | `"10m"` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of subnetwork in which to provision image building VM | `string` | n/a | yes | -| [tags](#input\_tags) | Assign network tags to apply firewall rules to VM instance | `list(string)` | `null` | no | -| [use\_iap](#input\_use\_iap) | Use IAP proxy when connecting by SSH | `bool` | `true` | no | -| [use\_os\_login](#input\_use\_os\_login) | Use OS Login when connecting by SSH | `bool` | `false` | no | -| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | A list of strings containing PowerShell scripts which will customize a Windows VM image (requires WinRM communicator) | `list(string)` | `[]` | no | -| [wrap\_startup\_script](#input\_wrap\_startup\_script) | Wrap startup script with Packer-generated wrapper | `bool` | `true` | no | -| [zone](#input\_zone) | Cloud zone in which to provision image building VM | `string` | n/a | yes | - -## Outputs - -No outputs. - - -[ansible]: #input_ansible_playbooks -[cloudnat]: https://cloud.google.com/nat/docs/overview -[examples readme]: ../../../examples/README.md#image-builderyaml- -[hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[iamprop]: https://cloud.google.com/iam/docs/access-change-propagation -[iaptunnel]: https://cloud.google.com/iap/docs/using-tcp-forwarding -[image builder]: ../../../examples/image-builder.yaml -[logging-console]: https://console.cloud.google.com/logs/ -[logging-read-docs]: https://cloud.google.com/sdk/gcloud/reference/logging/read -[pkrvars]: https://www.packer.io/guides/hcl/variables#from-a-file -[shell]: #input_shell_scripts -[ssf]: #input_startup_script_file -[sss]: #input_startup_script -[startup-metadata]: https://cloud.google.com/compute/docs/instances/startup-scripts/linux -[startup-script]: ../../../modules/scripts/startup-script -[vpc]: ../../network/vpc/README.md diff --git a/deletion-test/slurm-build/slurm-image/image.pkr.hcl b/deletion-test/slurm-build/slurm-image/image.pkr.hcl deleted file mode 100644 index 9282cf7433..0000000000 --- a/deletion-test/slurm-build/slurm-image/image.pkr.hcl +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "custom-image", ghpc_role = "packer" }) - - # construct a unique image name from the image family - image_family = var.image_family != null ? var.image_family : var.deployment_name - image_name_default = "${local.image_family}-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" - image_name = var.image_name != null ? var.image_name : local.image_name_default - - # construct vm image name for use when getting logs - instance_name = "packer-${substr(uuidv4(), 0, 6)}" - - # default to explicit var.communicator, otherwise in-order: ssh/winrm/none - shell_script_communicator = length(var.shell_scripts) > 0 ? "ssh" : "" - ansible_playbook_communicator = length(var.ansible_playbooks) > 0 ? "ssh" : "" - powershell_script_communicator = length(var.windows_startup_ps1) > 0 ? "winrm" : "" - communicator = coalesce( - var.communicator, - local.shell_script_communicator, - local.ansible_playbook_communicator, - local.powershell_script_communicator, - "none" - ) - - # must not enable IAP when no communicator is in use - use_iap = local.communicator == "none" ? false : var.use_iap - - # construct metadata from startup_script and metadata variables - startup_script_metadata = var.startup_script == null ? {} : { startup-script = var.startup_script } - - linux_user_metadata = { - block-project-ssh-keys = "TRUE" - shutdown-script = <<-EOT - #!/bin/bash - userdel -r ${var.ssh_username} - sed -i '/${var.ssh_username}/d' /var/lib/google/google_users - EOT - } - windows_packer_user = "packer_user" - windows_user_metadata = { - sysprep-specialize-script-cmd = "winrm quickconfig -quiet & net user /add ${local.windows_packer_user} & net localgroup administrators ${local.windows_packer_user} /add & winrm set winrm/config/service/auth @{Basic=\\\"true\\\"}" - windows-shutdown-script-cmd = <<-EOT - net user /delete ${local.windows_packer_user} - EOT - } - user_metadata = local.communicator == "winrm" ? local.windows_user_metadata : local.linux_user_metadata - - # merge metadata such that var.metadata always overrides user management - # metadata but always allow var.startup_script to override var.metadata - metadata = merge( - local.user_metadata, - var.metadata, - local.startup_script_metadata, - ) - - # determine best value for on_host_maintenance if not supplied by user - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - gpu_attached = contains(["a2", "g2"], local.machine_family) || var.accelerator_type != null - on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" - on_host_maintenance = ( - var.on_host_maintenance != null - ? var.on_host_maintenance - : local.on_host_maintenance_default - ) - - accelerator_type = var.accelerator_type == null ? null : "projects/${var.project_id}/zones/${var.zone}/acceleratorTypes/${var.accelerator_type}" - - winrm_username = local.communicator == "winrm" ? "packer_user" : null - winrm_insecure = local.communicator == "winrm" ? true : null - winrm_use_ssl = local.communicator == "winrm" ? true : null - - enable_integrity_monitoring = var.enable_shielded_vm && var.shielded_instance_config.enable_integrity_monitoring - enable_secure_boot = var.enable_shielded_vm && var.shielded_instance_config.enable_secure_boot - enable_vtpm = var.enable_shielded_vm && var.shielded_instance_config.enable_vtpm - - image_licenses = [ - "projects/click-to-deploy-images/global/licenses/hpc-toolkit-vm-image" - ] -} - -source "googlecompute" "toolkit_image" { - communicator = local.communicator - project_id = var.project_id - image_name = local.image_name - image_family = local.image_family - image_labels = local.labels - instance_name = local.instance_name - machine_type = var.machine_type - accelerator_type = local.accelerator_type - accelerator_count = var.accelerator_count - on_host_maintenance = local.on_host_maintenance - disk_size = var.disk_size - disk_type = var.disk_type - omit_external_ip = var.omit_external_ip - use_internal_ip = var.omit_external_ip - subnetwork = var.subnetwork_name - network_project_id = var.network_project_id - service_account_email = var.service_account_email - scopes = var.service_account_scopes - source_image = var.source_image - source_image_family = var.source_image_family - source_image_project_id = var.source_image_project_id - ssh_username = var.ssh_username - tags = var.tags - use_iap = local.use_iap - use_os_login = var.use_os_login - winrm_username = local.winrm_username - winrm_insecure = local.winrm_insecure - winrm_use_ssl = local.winrm_use_ssl - zone = var.zone - labels = local.labels - metadata = local.metadata - startup_script_file = var.startup_script_file - wrap_startup_script = var.wrap_startup_script - state_timeout = var.state_timeout - image_storage_locations = var.image_storage_locations - enable_secure_boot = local.enable_secure_boot - enable_vtpm = local.enable_vtpm - enable_integrity_monitoring = local.enable_integrity_monitoring - image_licenses = local.image_licenses -} - -build { - name = var.deployment_name - sources = ["sources.googlecompute.toolkit_image"] - - # using dynamic blocks to create provisioners ensures that there are no - # provisioner blocks when none are provided and we can use the none - # communicator when using startup-script - - # provisioner "shell" blocks - dynamic "provisioner" { - labels = ["shell"] - for_each = var.shell_scripts - content { - execute_command = "sudo -H sh -c '{{ .Vars }} {{ .Path }}'" - script = provisioner.value - } - } - - # provisioner "powershell" blocks - dynamic "provisioner" { - labels = ["powershell"] - for_each = var.windows_startup_ps1 - content { - inline = split("\n", provisioner.value) - } - } - - dynamic "provisioner" { - labels = ["powershell"] - for_each = length(var.windows_startup_ps1) > 0 ? [1] : [] - content { - inline = [ - "GCESysprep -no_shutdown" - ] - } - } - - # provisioner "ansible-local" blocks - # this installs custom roles/collections from ansible-galaxy in /home/packer - # which will be removed at the end; consider modifying /etc/ansible/ansible.cfg - dynamic "provisioner" { - labels = ["ansible-local"] - for_each = var.ansible_playbooks - content { - playbook_file = provisioner.value.playbook_file - galaxy_file = provisioner.value.galaxy_file - extra_arguments = provisioner.value.extra_arguments - } - } - - post-processor "manifest" { - output = var.manifest_file - strip_path = true - custom_data = { - built-by = "cloud-hpc-toolkit" - } - } - - # If there is an error during image creation, print out command for getting packer VM logs - error-cleanup-provisioner "shell-local" { - environment_vars = [ - "PRJ_ID=${var.project_id}", - "INST_NAME=${local.instance_name}", - "ZONE=${var.zone}", - ] - inline_shebang = "/bin/bash -e" - inline = [ - "type -P gcloud > /dev/null || exit 0", - "INST_ID=$(gcloud compute instances describe $INST_NAME --project $PRJ_ID --format=\"value(id)\" --zone=$ZONE)", - "echo 'Error building image try checking logs:'", - join(" ", ["echo \"gcloud logging --project $PRJ_ID read", - "'logName=(\\\"projects/$PRJ_ID/logs/GCEMetadataScripts\\\" OR \\\"projects/$PRJ_ID/logs/google_metadata_script_runner\\\") AND resource.labels.instance_id=$INST_ID'", - "--format=\\\"table(timestamp, resource.labels.instance_id, jsonPayload.message)\\\"", - "--order=asc\"" - ] - ) - ] - } -} diff --git a/deletion-test/slurm-build/slurm-image/metadata.yaml b/deletion-test/slurm-build/slurm-image/metadata.yaml deleted file mode 100644 index 23108c4e17..0000000000 --- a/deletion-test/slurm-build/slurm-image/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - logging.googleapis.com - - storage.googleapis.com diff --git a/deletion-test/slurm-build/slurm-image/variables.pkr.hcl b/deletion-test/slurm-build/slurm-image/variables.pkr.hcl deleted file mode 100644 index 3cede102ce..0000000000 --- a/deletion-test/slurm-build/slurm-image/variables.pkr.hcl +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "deployment_name" { - description = "Cluster Toolkit deployment name" - type = string -} - -variable "project_id" { - description = "Project in which to create VM and image" - type = string -} - -variable "machine_type" { - description = "VM machine type on which to build new image" - type = string - default = "n2-standard-4" -} - -variable "disk_size" { - description = "Size of disk image in GB" - type = number - default = null -} - -variable "disk_type" { - description = "Type of persistent disk to provision" - type = string - default = "pd-balanced" -} - -variable "zone" { - description = "Cloud zone in which to provision image building VM" - type = string -} - -variable "network_project_id" { - description = "Project ID of Shared VPC network" - type = string - default = null -} - -variable "subnetwork_name" { - description = "Name of subnetwork in which to provision image building VM" - type = string -} - -variable "omit_external_ip" { - description = "Provision the image building VM without a public IP address" - type = bool - default = true -} - -variable "tags" { - description = "Assign network tags to apply firewall rules to VM instance" - type = list(string) - default = null -} - -variable "image_family" { - description = "The family name of the image to be built. Defaults to `deployment_name`" - type = string - default = null -} - -variable "image_name" { - description = "The name of the image to be built. If not supplied, it will be set to image_family-$ISO_TIMESTAMP" - type = string - default = null -} - -variable "source_image_project_id" { - description = < /dev/null; then log "ERROR" "Missing required dependency: $cmd" @@ -37,28 +35,50 @@ check_dependencies() { } load_exclusions() { - if [[ ! -f "$EXCLUSION_FILE" ]]; then - log "ERROR" "Exclusion file not found: $EXCLUSION_FILE." - exit 1 - fi - log "INFO" "Loading exclusions from $EXCLUSION_FILE..." - while IFS= read -r line || [[ -n "$line" ]]; do + + local line_count=0 + # Helper function to process each line from the exclusion source + process_line() { + local line="$1" local trimmed_line trimmed_line=$(echo "$line" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') if [[ -n "$trimmed_line" ]] && [[ "$trimmed_line" != \#* ]]; then - EXCLUSION_MAP["$trimmed_line"]=1 + if [[ -z "${EXCLUSION_MAP[${trimmed_line}]:-}" ]]; then + EXCLUSION_MAP["${trimmed_line}"]=1 + ((line_count++)) + fi fi - done < "$EXCLUSION_FILE" + } + + log "INFO" "Exclusion file is a GCS path. Streaming content..." + # Preliminary check to see if the GCS object exists and is accessible + if ! gcloud storage ls "$EXCLUSION_FILE" > /dev/null 2>&1; then + log "ERROR" "Cannot access GCS exclusion file: $EXCLUSION_FILE. Please check the path and permissions." + exit 1 + fi + while IFS= read -r line || [[ -n "$line" ]]; do + process_line "$line" + done < <(gcloud storage cat "$EXCLUSION_FILE") + + if [[ ${#EXCLUSION_MAP[@]} -eq 0 ]]; then + log "ERROR" "No valid exclusion entries loaded from $EXCLUSION_FILE. Exiting to prevent accidental deletion." + exit 1 + else + log "INFO" "Loaded ${#EXCLUSION_MAP[@]} unique exclusion entries." + fi } + +# Returns 0 if EXCLUDED (do NOT delete) +# Returns 1 if NOT excluded (OK to delete) is_excluded() { local resource_name="$1" - local labels_str="${2:-}" + local labels_str="${2:-}" # Expected format: key1=value1;key2=value2 - if [[ -n "${EXCLUSION_MAP[$resource_name]:-}" ]]; then - log "SKIP" "$resource_name (In Exclusion List)" - return 0 + if [[ -n "${EXCLUSION_MAP[${resource_name}]:-}" ]]; then + log "SKIP" "$resource_name (In Exclusion Map)" + return 0 # Excluded fi if [[ -n "$labels_str" ]]; then @@ -70,26 +90,29 @@ is_excluded() { if [[ "$KEY" == "do-not-delete" ]]; then if [[ "$VAL" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then local exp_seconds - if ! exp_seconds=$(date -d "$VAL + 1 day" +%s 2>/dev/null); then + if ! exp_seconds=$(date -d "$VAL + 1 day" -u +%s 2>/dev/null); then log "WARNING" "$resource_name (Label: do-not-delete invalid date value: $VAL)" + return 1 # Not excluded else local current_seconds - current_seconds=$(date +%s) + current_seconds=$(date -u +%s) if [[ "$exp_seconds" -gt "$current_seconds" ]]; then - log "SKIP" "$resource_name (Label: do-not-delete=$VAL, valid until end of day)" - return 0 + log "SKIP" "$resource_name (Label: do-not-delete=$VAL, valid)" + return 0 # Excluded else log "INFO" "$resource_name (Label: do-not-delete=$VAL expired)" + return 1 # Not excluded fi fi else log "WARNING" "$resource_name (Label: do-not-delete invalid date format: $VAL, expected YYYY-MM-DD)" + return 1 # Not excluded fi break fi done fi - return 1 + return 1 # Not excluded } execute_delete() { @@ -111,9 +134,110 @@ execute_delete() { fi } -# ============================================================================== +populate_protected_resources() { + log "INFO" "Identifying protected instances and their associated resources..." + local instances_data + if ! instances_data=$(gcloud compute instances list \ + --project="$PROJECT_ID" \ + --filter="labels.do-not-delete:*" \ + --format="value(name,zone.basename(),labels.map(),disks[].source.list(separator=';'),networkInterfaces[].network.list(separator=';'),networkInterfaces[].subnetwork.list(separator=';'),networkInterfaces[].networkIP.list(separator=';'),networkInterfaces[].accessConfigs[].natIP.list(separator=';'))"); then + log "ERROR" "Failed to list instances with do-not-delete label." + ((ERROR_COUNT++)) || true + return + fi + + if [[ -z "$instances_data" ]]; then + log "INFO" "No instances found with do-not-delete label." + return + fi + + while IFS=$'\t' read -r inst_name zone labels_str disks_list nets_list subs_list ips_list nat_ips_list; do + if is_excluded "$inst_name" "$labels_str"; then # Returns 0 if excluded + log "INFO" "Instance ${inst_name} in ${zone} is PROTECTED. Adding associated resources to exclusions." + EXCLUSION_MAP["${inst_name}"]=1 + + # Protect Attached Disks + IFS=';' read -ra disk_urls <<< "$disks_list" + for disk_url in "${disk_urls[@]}"; do + [[ -z "$disk_url" ]] && continue + local disk_name + disk_name=$(basename "${disk_url}") + if [[ -n "${disk_name}" && -z "${EXCLUSION_MAP[${disk_name}]:-}" ]]; then + log "INFO" " > Excluding Disk: ${disk_name}" + EXCLUSION_MAP["${disk_name}"]=1 + fi + done + + # Protect Network + IFS=';' read -ra net_urls <<< "$nets_list" + for net_url in "${net_urls[@]}"; do + [[ -z "$net_url" ]] && continue + local net_name + net_name=$(basename "${net_url}") + if [[ -n "${net_name}" && -z "${EXCLUSION_MAP[${net_name}]:-}" ]]; then + log "INFO" " > Excluding Network: ${net_name}" + EXCLUSION_MAP["${net_name}"]=1 + fi + done + + # Protect Subnetwork + IFS=';' read -ra sub_urls <<< "$subs_list" + for sub_url in "${sub_urls[@]}"; do + [[ -z "$sub_url" ]] && continue + local sub_name + sub_name=$(basename "${sub_url}") + if [[ -n "${sub_name}" && -z "${EXCLUSION_MAP[${sub_name}]:-}" ]]; then + log "INFO" " > Excluding Subnetwork: ${sub_name}" + EXCLUSION_MAP["${sub_name}"]=1 + fi + done + + # Protect Network IPs + IFS=';' read -ra network_ips <<< "$ips_list" + for ip in "${network_ips[@]}"; do + [[ -n "$ip" ]] && PROTECTED_IPS["${ip}"]=1 + done + + # Protect External (NAT) IPs + IFS=';' read -ra nat_ips <<< "$nat_ips_list" + for ip in "${nat_ips[@]}"; do + [[ -n "$ip" ]] && PROTECTED_IPS["${ip}"]=1 + done + fi + done <<< "$instances_data" + + # Find Address resource names for the PROTECTED_IPS + if ((${#PROTECTED_IPS[@]} > 0)); then + log "INFO" "Finding Address resource names for protected IPs..." + local addresses_data + if ! addresses_data=$(gcloud compute addresses list --project="$PROJECT_ID" --format="value(name,address)"); then + log "WARNING" "Failed to list addresses to protect by IP." + else + while IFS=$'\t' read -r addr_name addr_ip; do + if [[ -n "${addr_ip}" && -n "${PROTECTED_IPS[${addr_ip}]:-}" ]]; then + if [[ -n "${addr_name}" && -z "${EXCLUSION_MAP[${addr_name}]:-}" ]]; then + log "INFO" " > Excluding Address: ${addr_name} (${addr_ip})" + EXCLUSION_MAP["${addr_name}"]=1 + fi + fi + done <<< "$addresses_data" + fi + fi +} + +log_exclusion_map() { + log "INFO" "--- Current Exclusion Map Contents ---" + if [ ${#EXCLUSION_MAP[@]} -eq 0 ]; then + log "INFO" "Exclusion map is empty." + return + fi + for key in "${!EXCLUSION_MAP[@]}"; do + log "INFO" "EXCLUDED: $key" + done + log "INFO" "--- End of Exclusion Map ---" +} + # STANDARD PROCESSOR -# ============================================================================== process_resources() { local label="$1" @@ -139,21 +263,19 @@ process_resources() { while IFS=$'\t' read -r name scope labels_str; do [[ -z "$name" ]] && continue - if is_excluded "$name" "${labels_str:-}"; then continue; fi + if ! is_excluded "$name" "${labels_str:-}"; then + local final_cmd="$delete_command_base \"$name\" --quiet" + if [[ "$scope_type" != "none" && -n "$scope" ]]; then + final_cmd="$final_cmd --$scope_type=\"$scope\"" + fi - local final_cmd="$delete_command_base \"$name\" --quiet" - if [[ "$scope_type" != "none" && -n "$scope" ]]; then - final_cmd="$final_cmd --$scope_type=\"$scope\"" + execute_delete "$label" "$name" "$final_cmd" "${scope:-(Global)}" + ((count++)) || true fi - - execute_delete "$label" "$name" "$final_cmd" "${scope:-(Global)}" - ((count++)) || true done <<< "$resources" } -# ============================================================================== # SPECIFIC HANDLERS -# ============================================================================== process_instance_templates() { log "INFO" "--- Processing: Instance Templates ---" @@ -161,7 +283,7 @@ process_instance_templates() { if ! templates=$(gcloud compute instance-templates list \ --project="$PROJECT_ID" \ --filter="creationTimestamp < '$CUTOFF_TIME'" \ - --format="value(name, labels)" | sort); then + --format="value(name, labels.map())" | sort); then log "ERROR" "Failed to list instance templates." ((ERROR_COUNT++)) || true return 0 @@ -171,191 +293,215 @@ process_instance_templates() { local count=0 while IFS=$'\t' read -r name labels_str; do if [[ -z "$name" ]]; then continue; fi - if is_excluded "$name" "${labels_str:-}"; then continue; fi - execute_delete "Instance Template" "$name" \ - "gcloud compute instance-templates delete \"$name\" --project=\"$PROJECT_ID\" --quiet" \ - "(Global)" - ((count++)) || true + if ! is_excluded "$name" "${labels_str:-}"; then + execute_delete "Instance Template" "$name" \ + "gcloud compute instance-templates delete \"$name\" --project=\"$PROJECT_ID\" --quiet" \ + "(Global)" + ((count++)) || true + fi done <<< "$templates" } process_addresses() { log "INFO" "--- Processing: Compute Addresses ---" - process_resources "Regional Address" \ - "gcloud compute addresses list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND region:*\" --format=\"value(name,region,labels)\" | sort" \ - "gcloud compute addresses delete --project=\"$PROJECT_ID\"" \ - "region" - process_resources "Global Address" \ - "gcloud compute addresses list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND NOT region:*\" --format=\"value[separator='t'](name, labels)\" | awk 'BEGIN{OFS=\"\t\"} {if (NF==1) print \$1, \"Global\", \"\"; else print \$1, \"Global\", \$2}' | sort" \ - "gcloud compute addresses delete --project=\"$PROJECT_ID\" --global" \ - "none" + # Regional Addresses + local regional_addresses + if ! regional_addresses=$(gcloud compute addresses list --project="$PROJECT_ID" \ + --filter="creationTimestamp < '$CUTOFF_TIME' AND region:*" \ + --format="value(name,region.basename(),labels.map(),status)" | sort); then + log "ERROR" "Failed to list Regional Addresses." + ((ERROR_COUNT++)) || true + else + while IFS=$'\t' read -r name region labels_str status; do + [[ -z "$name" ]] && continue + if ! is_excluded "$name" "${labels_str:-}"; then + if [[ "$status" == "IN_USE" ]]; then + log "WARNING" "Skipping IN_USE Regional Address $name ($region) NOT explicitly excluded." + continue + fi + execute_delete "Regional Address" "$name" \ + "gcloud compute addresses delete \"$name\" --project=\"$PROJECT_ID\" --region=\"$region\" --quiet" \ + "($region)" + fi + done <<< "$regional_addresses" + fi + + # Global Addresses + local global_addresses + if ! global_addresses=$(gcloud compute addresses list --project="$PROJECT_ID" \ + --filter="creationTimestamp < '$CUTOFF_TIME' AND NOT region:*" \ + --format="value(name, labels.map(), status)" | sort); then + log "ERROR" "Failed to list Global Addresses." + ((ERROR_COUNT++)) || true + else + while IFS=$'\t' read -r name labels_str status; do + [[ -z "$name" ]] && continue + if ! is_excluded "$name" "${labels_str:-}"; then + if [[ "$status" == "IN_USE" ]]; then + log "WARNING" "Skipping IN_USE Global Address $name NOT explicitly excluded." + continue + fi + execute_delete "Global Address" "$name" \ + "gcloud compute addresses delete \"$name\" --project=\"$PROJECT_ID\" --global --quiet" \ + "(Global)" + fi + done <<< "$global_addresses" + fi } + process_vpc_peerings() { log "INFO" "--- Processing: VPC Peerings---" - local networks_json - if ! networks_json=$(gcloud compute networks list --project="$PROJECT_ID" --format="json"); then + local networks_data + if ! networks_data=$(gcloud compute networks list --project="$PROJECT_ID" --format="value(name)"); then log "ERROR" "Failed to list networks." ((ERROR_COUNT++)) || true return 0 fi - if [[ -z "$networks_json" || "$networks_json" == "[]" ]]; then log "INFO" "No networks found in project."; return 0; fi + if [[ -z "$networks_data" ]]; then log "INFO" "No networks found in project."; return 0; fi local count=0 - while IFS= read -r net_obj; do - local net_name - net_name=$(echo "$net_obj" | jq -r '.name') - if [[ -z "$net_name" || "$net_name" == "null" ]]; then continue; fi - - local peerings_json - peerings_json=$(echo "$net_obj" | jq -c '.peerings // []') - if [[ "$peerings_json" == "[]" || "$peerings_json" == "null" ]]; then continue; fi - - while IFS= read -r peering_obj; do - local peering_name - peering_name=$(echo "$peering_obj" | jq -r '.name') - if [[ -z "$peering_name" || "$peering_name" == "null" ]]; then continue; fi - - if is_excluded "$peering_name" || is_excluded "$net_name"; then continue; fi - - local peer_network - peer_network=$(echo "$peering_obj" | jq -r '.network // ""') - local state - state=$(echo "$peering_obj" | jq -r '.state // ""') - - if [[ "$peering_name" == "servicenetworking-googleapis-com" ]]; then - execute_delete "Service Peering" "$peering_name" \ - "gcloud services vpc-peerings delete --service=servicenetworking.googleapis.com --network=\"$net_name\" --project=\"$PROJECT_ID\" --quiet" \ - "(Network: $net_name)" - ((count++)) || true - elif [[ "$peering_name" == filestore-peer-* ]]; then continue; - elif [[ "$peer_network" == *"/global/networks/servicenetworking" ]]; then continue; - else - execute_delete "VPC Peering" "$peering_name" \ - "gcloud compute networks peerings delete \"$peering_name\" --network=\"$net_name\" --project=\"$PROJECT_ID\" --quiet" \ - "(Network: $net_name, State: $state)" - ((count++)) || true + while IFS=$'\t' read -r net_name; do + if [[ -z "$net_name" ]]; then continue; fi + + if [[ -n "${EXCLUSION_MAP[${net_name}]:-}" ]]; then + log "SKIP" "VPC Peerings for Network $net_name (Protected)" + continue + fi + + local peerings_data + if ! peerings_data=$(gcloud compute networks peerings list --network="$net_name" --project="$PROJECT_ID" --format="value(name,network,state)"); then + log "WARNING" "Failed to list peerings for network $net_name" + continue + fi + + if [[ -z "$peerings_data" ]]; then continue; fi + + while IFS=$'\t' read -r peering_name peer_network state; do + if [[ -z "$peering_name" ]]; then continue; fi + + if ! is_excluded "$peering_name"; then + if [[ "$peering_name" == "servicenetworking-googleapis-com" ]]; then + execute_delete "Service Peering" "$peering_name" \ + "gcloud services vpc-peerings delete --service=servicenetworking.googleapis.com --network=\"$net_name\" --project=\"$PROJECT_ID\" --quiet" \ + "(Network: $net_name)" + ((count++)) || true + elif [[ "$peering_name" == filestore-peer-* ]]; then continue; + elif [[ "$peer_network" == *"/global/networks/servicenetworking" ]]; then continue; + else + execute_delete "VPC Peering" "$peering_name" \ + "gcloud compute networks peerings delete \"$peering_name\" --network=\"$net_name\" --project=\"$PROJECT_ID\" --quiet" \ + "(Network: $net_name, State: $state)" + ((count++)) || true + fi fi - done < <(echo "$peerings_json" | jq -c '.[]') - done < <(echo "$networks_json" | jq -c '.[]') + done <<< "$peerings_data" + done <<< "$networks_data" log "INFO" "Finished processing VPC Peerings. $count peerings actioned." } process_iam_deleted_members() { log "INFO" "--- Processing: IAM Role Bindings for Deleted SAs ---" - local policy_json - if ! policy_json=$(gcloud projects get-iam-policy "$PROJECT_ID" --format=json); then + local policy_data + if ! policy_data=$(gcloud projects get-iam-policy "$PROJECT_ID" --format="value(bindings[].role,bindings[].members)"); then log "ERROR" "Failed to get IAM policy." ((ERROR_COUNT++)) || true return 0 fi - local deleted_bindings - deleted_bindings=$(echo "$policy_json" | jq -r '.bindings[] | .role as $r | .members[] | select(startswith("deleted:serviceAccount:")) | "\($r)\t\(.)"') - if [[ -z "$deleted_bindings" ]]; then log "INFO" "No 'deleted:serviceAccount' bindings found."; return 0; fi + if [[ -z "$policy_data" ]]; then log "INFO" "No IAM bindings found."; return 0; fi local count=0 - while IFS=$'\t' read -r role member; do - if [[ -z "$role" || -z "$member" ]]; then continue; fi - - local cmd="gcloud projects remove-iam-policy-binding \"$PROJECT_ID\" --member=\"$member\" --role=\"$role\" --condition=None --quiet" - - if [[ "$DRY_RUN" == "true" ]]; then - log "DRY-RUN" "Would remove IAM binding: $member from role $role" - else - log "EXECUTE" "Removing IAM binding: $member from role $role" - if ! eval "$cmd" >/dev/null; then - log "ERROR" "Failed to remove binding" - ((ERROR_COUNT++)) || true + while IFS=$'\t' read -r role members_str; do + if [[ -z "$role" || -z "$members_str" ]]; then continue; fi + + IFS=';' read -ra members <<< "$members_str" + for member in "${members[@]}"; do + if [[ "$member" == deleted:serviceAccount:* ]]; then + local cmd="gcloud projects remove-iam-policy-binding \"$PROJECT_ID\" --member=\"$member\" --role=\"$role\" --condition=None --quiet" + + if [[ "$DRY_RUN" == "true" ]]; then + log "DRY-RUN" "Would remove IAM binding: $member from role $role" + else + log "EXECUTE" "Removing IAM binding: $member from role $role" + if ! eval "$cmd" >/dev/null; then + log "ERROR" "Failed to remove binding for $member in $role" + ((ERROR_COUNT++)) || true + fi + fi + ((count++)) || true fi - fi - ((count++)) || true - done <<< "$deleted_bindings" + done + done <<< "$policy_data" + log "INFO" "Finished processing IAM deleted members. $count bindings actioned." } process_vm_images() { log "INFO" "--- Processing: VM Images ---" local images if ! images=$(gcloud compute images list --project="$PROJECT_ID" --no-standard-images \ - --format="value(name,creationTimestamp,labels)"); then + --format="value(name,creationTimestamp,labels.map())"); then log "ERROR" "Failed to list VM images" ((ERROR_COUNT++)) || true return 0 fi - if [[ -z "$images" ]]; then log "INFO" "No custom VM images found."; return 0; fi - local cutoff_seconds - # Using date check directly; if date fails, we handle it inside the loop if ! cutoff_seconds=$(date -d "$CUTOFF_TIME_IMAGES" +%s); then log "ERROR" "Failed to calculate cutoff time for images" ((ERROR_COUNT++)) || true return 0 fi - local count=0 while IFS=$'\t' read -r name timestamp labels_str; do [[ -z "$name" ]] && continue - if is_excluded "$name" "${labels_str:-}"; then continue; fi - - local ts_seconds - if ! ts_seconds=$(date -d "$timestamp" +%s 2>/dev/null); then - log "WARNING" "Could not parse timestamp '$timestamp' for image $name. Skipping." - continue - fi - if [[ $ts_seconds -lt $cutoff_seconds ]]; then - execute_delete "VM Image" "$name" \ - "gcloud compute images delete \"$name\" --project=\"$PROJECT_ID\" --quiet" - ((count++)) || true + if ! is_excluded "$name" "${labels_str:-}"; then + local ts_seconds + if ! ts_seconds=$(date -d "$timestamp" +%s 2>/dev/null); then + log "WARNING" "Could not parse timestamp '$timestamp' for image $name. Skipping." + continue + fi + if [[ $ts_seconds -lt $cutoff_seconds ]]; then + execute_delete "VM Image" "$name" \ + "gcloud compute images delete \"$name\" --project=\"$PROJECT_ID\" --quiet" + ((count++)) || true + fi fi done <<< "$images" } process_docker_images() { log "INFO" "--- Processing: Docker Images for 'test-runner' (Artifact Registry) ---" - local cutoff_date - cutoff_date=$(date -u -d "14 days ago" '+%Y-%m-%dT%H:%M:%SZ') + local cutoff_date=$(date -u -d "14 days ago" '+%Y-%m-%dT%H:%M:%SZ') local cutoff_seconds - if ! cutoff_seconds=$(date -u -d "$cutoff_date" +%s); then + if ! cutoff_seconds=$(date -u -d "$cutoff_date" +%s); then log "ERROR" "Failed to calculate cutoff_seconds." ((ERROR_COUNT++)) || true return 0 fi - log "INFO" "Policy: Delete 'test-runner' images updated before $cutoff_date (Unix: $cutoff_seconds)" - local location="us-central1" local repo_name="hpc-toolkit-repo" local package_name="test-runner" local full_package_url="${location}-docker.pkg.dev/${PROJECT_ID}/${repo_name}/${package_name}" - local images_output - # Use if ! to catch failure without exiting if ! images_output=$(gcloud artifacts docker images list "$full_package_url" --format="csv[no-heading](uri,updateTime)" --sort-by="updateTime" 2>/dev/null); then log "WARNING" "Failed to list images for $full_package_url (Repo might not exist or empty)" return 0 fi if [[ -z "$images_output" ]]; then log "INFO" " > No image versions found for $package_name."; return 0; fi - local count=0 while IFS=, read -r full_image_ref update_time; do - if [[ -z "$full_image_ref" ]]; then continue; fi - if [[ -z "$update_time" ]]; then continue; fi - if [[ "$full_image_ref" != *"@sha256:"* ]]; then continue; fi - + if [[ -z "$full_image_ref" || -z "$update_time" || "$full_image_ref" != *"@sha256:"* ]]; then continue; fi local image_seconds if ! image_seconds=$(date -u -d "$update_time" +%s 2>/dev/null); then continue; fi - - if [[ $image_seconds -ge $cutoff_seconds ]]; then continue; - else - if is_excluded "$package_name"; then continue; fi - if is_excluded "$full_image_ref"; then continue; fi - - execute_delete "Docker Image Version" "$full_image_ref" \ - "gcloud artifacts docker images delete \"$full_image_ref\" --project=\"$PROJECT_ID\" --delete-tags --quiet" \ - "(Updated: $update_time)" - ((count++)) || true + if [[ $image_seconds -lt $cutoff_seconds ]]; then + if ! is_excluded "$package_name" && ! is_excluded "$full_image_ref"; then + execute_delete "Docker Image Version" "$full_image_ref" \ + "gcloud artifacts docker images delete \"$full_image_ref\" --project=\"$PROJECT_ID\" --delete-tags --quiet" \ + "(Updated: $update_time)" + ((count++)) || true + fi fi done <<< "$images_output" - log "INFO" "Finished Docker Image processing for $package_name. $count images marked for deletion." } process_firewalls() { @@ -363,7 +509,7 @@ process_firewalls() { local fws if ! fws=$(gcloud compute firewall-rules list --project="$PROJECT_ID" \ --filter="creationTimestamp < '$CUTOFF_TIME'" \ - --format="value(name,network,labels)" | sort); then + --format="value(name,network,labels.map())" | sort); then log "ERROR" "Failed to list firewall rules" ((ERROR_COUNT++)) || true return 0 @@ -376,99 +522,166 @@ process_firewalls() { local network_name network_name=$(basename "$network_uri") if [[ "$network_name" == "default" ]]; then continue; fi - if is_excluded "$name" "${labels_str:-}"; then continue; fi - execute_delete "Firewall Rule" "$name" \ - "gcloud compute firewall-rules delete \"$name\" --project=\"$PROJECT_ID\" --quiet" - ((count++)) || true + + if [[ -n "${EXCLUSION_MAP[${network_name}]:-}" ]]; then + log "SKIP" "Firewall Rule $name - Network $network_name is protected." + continue + fi + + if ! is_excluded "$name" "${labels_str:-}"; then + execute_delete "Firewall Rule" "$name" \ + "gcloud compute firewall-rules delete \"$name\" --project=\"$PROJECT_ID\" --quiet" + ((count++)) || true + fi done <<< "$fws" } process_filestore() { log "INFO" "--- Processing: Filestore Instances ---" - local fs_json - if ! fs_json=$(gcloud filestore instances list --project="$PROJECT_ID" --filter="createTime < '$CUTOFF_TIME'" --format="json"); then + local fs_data + # Get instance name, location, and labels using segment projections + if ! fs_data=$(gcloud filestore instances list --project="$PROJECT_ID" --filter="createTime < '$CUTOFF_TIME'" \ + --format="value(name.segment(5), name.segment(3), labels.map())"); then log "ERROR" "Failed to list Filestore instances." ((ERROR_COUNT++)) || true return 0 fi - if [[ -z "$fs_json" || "$fs_json" == "[]" ]]; then log "INFO" "No Filestore instances found matching criteria."; return 0; fi - - local fs_list - if ! fs_list=$(echo "$fs_json" | jq -r '.[] | select(.name) | "\(.name | split("/")[3])\t\(.name | split("/")[-1])\t\(.labels | to_entries | map("\(.key)=\(.value)") | join(";"))"'); then - log "ERROR" "Failed to parse Filestore JSON with jq." - ((ERROR_COUNT++)) || true - return 0 - fi - if [[ -z "$fs_list" ]]; then log "INFO" "No instances found after jq parsing."; return 0; fi + if [[ -z "$fs_data" ]]; then log "INFO" "No Filestore instances found matching criteria."; return 0; fi local count=0 - while IFS=$'\t' read -r location name labels_str; do - location=$(echo "$location" | awk '{$1=$1};1'); name=$(echo "$name" | awk '{$1=$1};1') - if [[ -z "$location" || -z "$name" ]]; then continue; fi - if is_excluded "$name" "${labels_str:-}"; then continue; fi - local delete_cmd="gcloud filestore instances delete \"$name\" --project=\"$PROJECT_ID\" --location=\"$location\" --quiet --force" - execute_delete "Filestore" "$name" "$delete_cmd" "($location)" - ((count++)) || true - done <<< "$fs_list" - log "INFO" "Finished processing Filestore instances. Attempted to delete $count." + while IFS=$'\t' read -r name location labels_str; do + # Trim potential whitespace + name=$(echo "$name" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + location=$(echo "$location" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + + if [[ -z "$name" || "$name" == "None" || -z "$location" || "$location" == "None" ]]; then + log "WARNING" "Could not extract valid name or location for a Filestore instance from line: $name $location $labels_str" + continue + fi + + if ! is_excluded "$name" "${labels_str:-}"; then + log "INFO" "Processing Filestore instance: $name in $location" + + if [[ "$DRY_RUN" == "true" ]]; then + log "DRY-RUN" "Would disable deletion protection on Filestore: $name ($location)" + log "DRY-RUN" "Would delete Filestore: $name ($location)" + ((count++)) || true + else + log "EXECUTE" "Attempting to disable deletion protection on Filestore: $name ($location)" + local disable_cmd="gcloud filestore instances update \"$name\" --location=\"$location\" --project=\"$PROJECT_ID\" --no-deletion-protection --quiet" + if ! eval "$disable_cmd"; then + log "WARNING" "Failed to disable deletion protection for $name. This may be OK if it was already disabled or the instance is not in a state to be updated. Continuing with delete attempt." + else + log "INFO" "Deletion protection update command executed successfully for $name" + fi + + log "EXECUTE" "Deleting Filestore: $name ($location)" + local delete_cmd="gcloud filestore instances delete \"$name\" --project=\"$PROJECT_ID\" --location=\"$location\" --quiet --force" + if eval "$delete_cmd"; then + log "SUCCESS" "Deleted Filestore $name" + ((count++)) || true + else + log "ERROR" "Failed to delete Filestore $name" + ((ERROR_COUNT++)) || true + fi + fi + fi + done <<< "$fs_data" } process_subnetworks() { log "INFO" "--- Processing: Subnetworks ---" local subnets - if ! subnets=$(gcloud compute networks subnets list --project="$PROJECT_ID" --filter="creationTimestamp < '$CUTOFF_TIME'" --format="value(name,region,network,selfLink)"); then + if ! subnets=$(gcloud compute networks subnets list --project="$PROJECT_ID" --filter="creationTimestamp < '$CUTOFF_TIME'" --format="value(name,region.basename(),network)"); then log "ERROR" "Failed to list subnets" ((ERROR_COUNT++)) || true return 0 fi - + local count=0 - while IFS=$'\t' read -r name region network_uri self_link; do + while IFS=$'\t' read -r name region network_uri; do [[ -z "$name" ]] && continue - local network_name=$(basename "$network_uri") + local network_name + network_name=$(basename "$network_uri") if [[ "$network_name" == "default" ]]; then continue; fi - if is_excluded "$name"; then continue; fi - - # Note: listing dependents might fail, wrapping in error check not strictly necessary for deletion loop but good practice - local dependents - dependents=$(gcloud compute addresses list --project="$PROJECT_ID" --filter="purpose=GCE_ENDPOINT AND region=(\"$region\") AND subnetwork=(\"$self_link\")" --format="value(name)" 2>/dev/null || true) - - for addr in $dependents; do - execute_delete "Dependent Address" "$addr" "gcloud compute addresses delete \"$addr\" --project=\"$PROJECT_ID\" --region=\"$region\" --quiet" - done - execute_delete "Subnetwork" "$name" "gcloud compute networks subnets delete \"$name\" --project=\"$PROJECT_ID\" --region=\"$region\" --quiet" - ((count++)) || true + + if [[ -n "${EXCLUSION_MAP[${network_name}]:-}" ]]; then + log "SKIP" "Subnetwork $name - Network $network_name is protected." + continue + fi + if [[ -n "${EXCLUSION_MAP[${name}]:-}" ]]; then + log "SKIP" "Subnetwork $name - Explicitly protected." + continue + fi + + if ! is_excluded "$name"; then + # Dependent address cleanup can be added here if needed + execute_delete "Subnetwork" "$name" "gcloud compute networks subnets delete \"$name\" --project=\"$PROJECT_ID\" --region=\"$region\" --quiet" + ((count++)) || true + fi done <<< "$subnets" } process_networks() { log "INFO" "--- Processing: VPC Networks ---" - local networks + local networks if ! networks=$(gcloud compute networks list --project="$PROJECT_ID" --filter="creationTimestamp < '$CUTOFF_TIME'" --format="value(name,selfLink)"); then log "ERROR" "Failed to list networks" ((ERROR_COUNT++)) || true return 0 fi - + local count=0 while IFS=$'\t' read -r name self_link; do [[ -z "$name" ]] && continue if [[ "$name" == "default" ]]; then continue; fi - if is_excluded "$name"; then continue; fi - - local routes - routes=$(gcloud compute routes list --project="$PROJECT_ID" --filter="network=\"$self_link\"" --format="value(name)" 2>/dev/null || true) - for r in $routes; do if ! is_excluded "$r"; then execute_delete "Dep. Route" "$r" "gcloud compute routes delete \"$r\" --project=\"$PROJECT_ID\" --quiet"; fi; done - - local fws - fws=$(gcloud compute firewall-rules list --project="$PROJECT_ID" --filter="network=\"$self_link\"" --format="value(name)" 2>/dev/null || true) - for fw in $fws; do if ! is_excluded "$fw"; then execute_delete "Dep. FW" "$fw" "gcloud compute firewall-rules delete \"$fw\" --project=\"$PROJECT_ID\" --quiet"; fi; done - - execute_delete "Network" "$name" "gcloud compute networks delete \"$name\" --project=\"$PROJECT_ID\" --quiet" - ((count++)) || true + if ! is_excluded "$name"; then + local routes + routes=$(gcloud compute routes list --project="$PROJECT_ID" --filter="network=\"$self_link\"" --format="value(name)" 2>/dev/null || true) + for r in $routes; do if ! is_excluded "$r"; then execute_delete "Dep. Route" "$r" "gcloud compute routes delete \"$r\" --project=\"$PROJECT_ID\" --quiet"; fi; done + + local fws + fws=$(gcloud compute firewall-rules list --project="$PROJECT_ID" --filter="network=\"$self_link\"" --format="value(name)" 2>/dev/null || true) + for fw in $fws; do if ! is_excluded "$fw"; then execute_delete "Dep. FW" "$fw" "gcloud compute firewall-rules delete \"$fw\" --project=\"$PROJECT_ID\" --quiet"; fi; done + + execute_delete "Network" "$name" "gcloud compute networks delete \"$name\" --project=\"$PROJECT_ID\" --quiet" + ((count++)) || true + fi done <<< "$networks" } +process_routers() { + log "INFO" "--- Processing: Cloud Routers ---" + local routers + if ! routers=$(gcloud compute routers list --project="$PROJECT_ID" \ + --filter="creationTimestamp < '$CUTOFF_TIME'" \ + --format="value(name,region.basename(),network,labels.map())" | sort); then + log "ERROR" "Failed to list Cloud Routers" + ((ERROR_COUNT++)) || true + return 0 + fi + if [[ -z "$routers" ]]; then log "INFO" "No Cloud Routers found matching criteria."; return 0; fi + + local count=0 + while IFS=$'\t' read -r name region network_uri labels_str; do + [[ -z "$name" ]] && continue + + local network_name + network_name=$(basename "$network_uri") + if [[ -n "${EXCLUSION_MAP[${network_name}]:-}" ]]; then + log "SKIP" "Cloud Router $name - Network $network_name is protected." + continue + fi + + if ! is_excluded "$name" "${labels_str:-}"; then + execute_delete "Cloud Router" "$name" \ + "gcloud compute routers delete \"$name\" --project=\"$PROJECT_ID\" --region=\"$region\" --quiet" \ + "($region)" + ((count++)) || true + fi + done <<< "$routers" +} + # ============================================================================== # MAIN EXECUTION # ============================================================================== @@ -482,14 +695,18 @@ main() { check_dependencies load_exclusions + populate_protected_resources + log_exclusion_map # Log the map contents # --- Phase 1: High Level Resources --- process_resources "GKE Cluster" \ - "gcloud container clusters list --project=\"$PROJECT_ID\" --filter=\"createTime < '$CUTOFF_TIME'\" --format=\"value(name,location,resourceLabels)\" | sort" \ + "gcloud container clusters list --project=\"$PROJECT_ID\" --filter=\"createTime < '$CUTOFF_TIME'\" --format=\"value(name,location,resourceLabels.map())\" | sort" \ "gcloud container clusters delete --project=\"$PROJECT_ID\"" "location" - process_instance_templates + process_resources "Instance Template" \ + "gcloud compute instance-templates list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name, 'Global', labels.map())\" | sort" \ + "gcloud compute instance-templates delete --project=\"$PROJECT_ID\"" process_resources "Compute Instance" \ - "gcloud compute instances list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,zone,labels)\" | sort" \ + "gcloud compute instances list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,zone.basename(),labels.map())\" | sort" \ "gcloud compute instances delete --project=\"$PROJECT_ID\" --delete-disks=all" "zone" process_filestore @@ -498,25 +715,22 @@ main() { process_docker_images # --- Phase 3: Network Infrastructure --- - process_resources "Cloud Router" \ - "gcloud compute routers list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,region,labels)\" | sort" \ - "gcloud compute routers delete --project=\"$PROJECT_ID\"" "region" - process_firewalls + process_routers + # process_firewalls # Called in process_networks for dependencies process_addresses - process_vpc_peerings process_resources "Zonal Disk" \ - "gcloud compute disks list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND zone:*\" --format=\"value(name,zone,labels)\" | sort" \ + "gcloud compute disks list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND zone:*\" --format=\"value(name,zone.basename(),labels.map())\" | sort" \ "gcloud compute disks delete --project=\"$PROJECT_ID\"" "zone" - # --- Phase 4: Networking Hierarchies --- + # # --- Phase 4: Networking Hierarchies --- process_subnetworks - process_networks + process_networks # This now handles dependent firewalls and routes - # --- Phase 5: IAM Cleanup --- + # # --- Phase 5: IAM Cleanup --- process_iam_deleted_members - + log "INFO" "CLEANUP RUN FINISHED" - + if [[ $ERROR_COUNT -gt 0 ]]; then log "WARNING" "Finished with $ERROR_COUNT errors during execution." exit 1 @@ -526,4 +740,4 @@ main() { fi } -main \ No newline at end of file +main diff --git a/tools/cloud-build/project-cleanup.yaml b/tools/cloud-build/project-cleanup.yaml index 268b4b1353..0643843107 100644 --- a/tools/cloud-build/project-cleanup.yaml +++ b/tools/cloud-build/project-cleanup.yaml @@ -21,26 +21,21 @@ steps: - "BUILD_ID=${BUILD_ID}" - "PROJECT_ID=${PROJECT_ID}" - "DRY_RUN=true" - - "EXCLUSION_FILE=tools/exclusions.txt" + - "EXCLUSION_FILE=gs://hpc-ctk1357/cleanup/exclusions.txt" args: - -c - | set -euo pipefail - # Install dependencies - echo "Installing jq..." - apt-get update -y && apt-get install -y jq - # Set time variables - export CUTOFF_TIME=$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%S%z) + export CUTOFF_TIME=$(date -d '2 days ago' -u +%Y-%m-%dT%H:%M:%S%z) export CUTOFF_TIME_IMAGES=$(date -d "60 days ago" -u +%Y-%m-%dT%H:%M:%S%z) attempt=1 - max_retries=10 - + max_retries=5 + while [ "$attempt" -le "$max_retries" ]; do echo "--- Execution Attempt ${attempt} of ${max_retries} ---" - if /workspace/tools/cleanup.sh; then echo "Cleanup completed successfully." exit 0 diff --git a/tools/exclusions.txt b/tools/exclusions.txt index cbfd136504..deb3ac9d5d 100644 --- a/tools/exclusions.txt +++ b/tools/exclusions.txt @@ -1,2 +1,47 @@ -a3mega-a3meganodeset-0 -a3mega-a3meganodeset-1 \ No newline at end of file +vertexui-do-not-kill +hpc-ctk1357 +hpc-toolkit-dev@appspot.gserviceaccount.com +build-notifier@hpc-toolkit-dev.iam.gserviceaccount.com +cloud-build-trigger-scheduler@hpc-toolkit-dev.iam.gserviceaccount.com +cloud-run-pubsub-invoker@hpc-toolkit-dev.iam.gserviceaccount.com +cloud-build-integration-tester@hpc-toolkit-dev.iam.gserviceaccount.com +508417052821-compute@developer.gserviceaccount.com +hpc-toolkit-sa@hpc-toolkit-dev.iam.gserviceaccount.com +hpc-toolkit-build-notification@hpc-toolkit-dev.iam.gserviceaccount.com +htcondor-173362-access@hpc-toolkit-dev.iam.gserviceaccount.com +htcondor-173362-cm@hpc-toolkit-dev.iam.gserviceaccount.com +htcondor-173362-execute@hpc-toolkit-dev.iam.gserviceaccount.com +pcb-740@hpc-toolkit-dev.iam.gserviceaccount.com +test-script-runner-sa@hpc-toolkit-dev.iam.gserviceaccount.com +telemetry@hpc-toolkit-dev.iam.gserviceaccount.com +telemetry-ingestor@hpc-toolkit-dev.iam.gserviceaccount.com +telemetry-sa@hpc-toolkit-dev.iam.gserviceaccount.com +vertexui-do-not-kill-boot +vertexui-do-not-kill-data +default-router-us-west1 +default-router-us-west4 +default-net-router +default-router-australia-southeast1 +default-router-us-east4 +image-inspector-550 +image-inspector +gke-managed-lustre-basic-net-fw-allow-iap-ingress +gke-managed-lustre-basic-net-fw-allow-internal-traffic +a3-slurm-sa@hpc-toolkit-dev.iam.gserviceaccount.com +hpc-vpc +allow-internal +allow-ssh +a4high-image-builder-20250214t220935z +chs-dcgmi-metric-u22-20250925t121709z +common-slurm-image-20250725t234825z +pbspro0 +a3u-image-u22-20250325t162635z +harsh-a4-image +rocka4hf-rocky9-20250910t040750z +rocka4h-rocky9-20250908t175724z +slurm-gcp-next-hpc-rocky-linux-8-1739990978 +slurm-gcp-next-hpc-rocky-linux-8-1740100297 +welp-insta-temp +testing2 +global-psconnect-ip-7130e09e +global-psconnect-ip-e5427150 \ No newline at end of file From c4ecaea54c6dbdd6648db042657bd1bc785b4f5f Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Mon, 22 Dec 2025 11:15:07 +0000 Subject: [PATCH 09/19] Cluster connected instances --- tools/cleanup.sh | 199 ++++++++++++++++++++++++++--------------------- 1 file changed, 111 insertions(+), 88 deletions(-) diff --git a/tools/cleanup.sh b/tools/cleanup.sh index ee3755398b..daba13b542 100755 --- a/tools/cleanup.sh +++ b/tools/cleanup.sh @@ -135,35 +135,117 @@ execute_delete() { } populate_protected_resources() { - log "INFO" "Identifying protected instances and their associated resources..." - local instances_data - if ! instances_data=$(gcloud compute instances list \ + log "INFO" "Identifying protected resources..." + declare -A INSTANCES_TO_PROTECT # Map instance_name -> zone + + # Part 1: Instances from EXCLUDED GKE clusters + log "INFO" "Checking for instances in EXCLUDED GKE clusters..." + local clusters_data + if ! clusters_data=$(gcloud container clusters list --project="$PROJECT_ID" --format="value(name,location,resourceLabels.map())"); then + log "ERROR" "Failed to list GKE clusters." + ((ERROR_COUNT++)) || true + else + while IFS=$'\t' read -r cluster_name location labels_str; do + if ! is_excluded "$cluster_name" "$labels_str"; then # Returns 1 if NOT excluded + continue + fi + + log "INFO" "GKE Cluster ${cluster_name} in ${location} is PROTECTED." + EXCLUSION_MAP["${cluster_name}"]=1 # Add cluster itself to exclusion map + + local node_pools_data + if ! node_pools_data=$(gcloud container node-pools list --cluster="${cluster_name}" --location="${location}" --project="${PROJECT_ID}" --format="value(name)"); then + log "WARNING" "Failed to list node pools for protected cluster ${cluster_name}." + continue + fi + + while IFS=$'\t' read -r np_name; do + local ig_urls + if ! ig_urls=$(gcloud container node-pools describe "${np_name}" --cluster="${cluster_name}" --location="${location}" --project="${PROJECT_ID}" --format="value(instanceGroupUrls)"); then + log "WARNING" "Failed to describe node pool ${np_name} in cluster ${cluster_name}." + continue + fi + + IFS=';' read -ra ig_url_list <<< "$ig_urls" + for ig_url in "${ig_url_list[@]}"; do + local ig_name=$(basename "${ig_url}") + local ig_scope_type=$(echo "${ig_url}" | awk -F'/' '{print $(NF-2)}') + local ig_scope_name=$(echo "${ig_url}" | awk -F'/' '{print $(NF-3)}') + local scope_flag="" + if [[ "$ig_scope_type" == "zones" ]]; then + scope_flag="--zone=${ig_scope_name}" + elif [[ "$ig_scope_type" == "regions" ]]; then + scope_flag="--region=${ig_scope_name}" + else + log "WARNING" "Unknown scope type for instance group: ${ig_url}" + continue + fi + + local instances_in_mig + if ! instances_in_mig=$(gcloud compute instance-groups managed list-instances "${ig_name}" --project="${PROJECT_ID}" "${scope_flag}" --format="value(NAME,ZONE)"); then + log "WARNING" "Failed to list instances for MIG ${ig_name}." + continue + fi + + while IFS=$'\t' read -r inst_name inst_zone_url; do + if [[ -n "$inst_name" ]]; then + local inst_zone=$(basename "$inst_zone_url") + if [[ -z "${EXCLUSION_MAP[${inst_name}]:-}" ]]; then + log "INFO" " > Protecting Instance (from GKE ${cluster_name}): ${inst_name} in ${inst_zone}" + EXCLUSION_MAP["${inst_name}"]=1 + fi + INSTANCES_TO_PROTECT["${inst_name}"]="${inst_zone}" + fi + done <<< "$instances_in_mig" + done + done <<< "$node_pools_data" + done <<< "$clusters_data" + fi + + # Part 2: Instances protected via direct labels + log "INFO" "Checking for instances with do-not-delete label..." + local labeled_instances_data + if ! labeled_instances_data=$(gcloud compute instances list \ --project="$PROJECT_ID" \ --filter="labels.do-not-delete:*" \ - --format="value(name,zone.basename(),labels.map(),disks[].source.list(separator=';'),networkInterfaces[].network.list(separator=';'),networkInterfaces[].subnetwork.list(separator=';'),networkInterfaces[].networkIP.list(separator=';'),networkInterfaces[].accessConfigs[].natIP.list(separator=';'))"); then + --format="value(name,zone.basename(),labels.map())"); then log "ERROR" "Failed to list instances with do-not-delete label." ((ERROR_COUNT++)) || true - return + else + while IFS=$'\t' read -r inst_name zone labels_str; do + if is_excluded "$inst_name" "$labels_str"; then # Returns 0 if excluded + if [[ -z "${EXCLUSION_MAP[${inst_name}]:-}" ]]; then + log "INFO" " > Protecting Instance (from Label): ${inst_name} in ${zone}" + EXCLUSION_MAP["${inst_name}"]=1 + fi + INSTANCES_TO_PROTECT["${inst_name}"]="${zone}" + fi + done <<< "$labeled_instances_data" fi - if [[ -z "$instances_data" ]]; then - log "INFO" "No instances found with do-not-delete label." - return - fi + # Part 3: Protect resources associated with the collected instances + if ((${#INSTANCES_TO_PROTECT[@]} > 0)); then + log "INFO" "Protecting sub-resources of ${#INSTANCES_TO_PROTECT[@]} instances..." + for inst_name in "${!INSTANCES_TO_PROTECT[@]}"; do + local zone="${INSTANCES_TO_PROTECT[$inst_name]}" + log "DEBUG" "Fetching details for protected instance: ${inst_name} in ${zone}" + local inst_details + if ! inst_details=$(gcloud compute instances describe "${inst_name}" --zone="${zone}" --project="${PROJECT_ID}" \ + --format="value(disks[].source.list(separator=';'),networkInterfaces[].network.list(separator=';'),networkInterfaces[].subnetwork.list(separator=';'),networkInterfaces[].networkIP.list(separator=';'),networkInterfaces[].accessConfigs[].natIP.list(separator=';'))"); then + log "WARNING" "Failed to describe protected instance ${inst_name} in ${zone}. Sub-resources might not be protected." + continue + fi - while IFS=$'\t' read -r inst_name zone labels_str disks_list nets_list subs_list ips_list nat_ips_list; do - if is_excluded "$inst_name" "$labels_str"; then # Returns 0 if excluded - log "INFO" "Instance ${inst_name} in ${zone} is PROTECTED. Adding associated resources to exclusions." - EXCLUSION_MAP["${inst_name}"]=1 + local disks_list nets_list subs_list ips_list nat_ips_list + IFS=$'\t' read -r disks_list nets_list subs_list ips_list nat_ips_list <<< "$inst_details" # Protect Attached Disks IFS=';' read -ra disk_urls <<< "$disks_list" for disk_url in "${disk_urls[@]}"; do [[ -z "$disk_url" ]] && continue - local disk_name - disk_name=$(basename "${disk_url}") + local disk_name=$(basename "${disk_url}") if [[ -n "${disk_name}" && -z "${EXCLUSION_MAP[${disk_name}]:-}" ]]; then - log "INFO" " > Excluding Disk: ${disk_name}" + log "INFO" " > Excluding Disk (for ${inst_name}): ${disk_name}" EXCLUSION_MAP["${disk_name}"]=1 fi done @@ -172,10 +254,9 @@ populate_protected_resources() { IFS=';' read -ra net_urls <<< "$nets_list" for net_url in "${net_urls[@]}"; do [[ -z "$net_url" ]] && continue - local net_name - net_name=$(basename "${net_url}") + local net_name=$(basename "${net_url}") if [[ -n "${net_name}" && -z "${EXCLUSION_MAP[${net_name}]:-}" ]]; then - log "INFO" " > Excluding Network: ${net_name}" + log "INFO" " > Excluding Network (for ${inst_name}): ${net_name}" EXCLUSION_MAP["${net_name}"]=1 fi done @@ -184,27 +265,18 @@ populate_protected_resources() { IFS=';' read -ra sub_urls <<< "$subs_list" for sub_url in "${sub_urls[@]}"; do [[ -z "$sub_url" ]] && continue - local sub_name - sub_name=$(basename "${sub_url}") + local sub_name=$(basename "${sub_url}") if [[ -n "${sub_name}" && -z "${EXCLUSION_MAP[${sub_name}]:-}" ]]; then - log "INFO" " > Excluding Subnetwork: ${sub_name}" + log "INFO" " > Excluding Subnetwork (for ${inst_name}): ${sub_name}" EXCLUSION_MAP["${sub_name}"]=1 fi done - # Protect Network IPs - IFS=';' read -ra network_ips <<< "$ips_list" - for ip in "${network_ips[@]}"; do - [[ -n "$ip" ]] && PROTECTED_IPS["${ip}"]=1 - done - - # Protect External (NAT) IPs - IFS=';' read -ra nat_ips <<< "$nat_ips_list" - for ip in "${nat_ips[@]}"; do - [[ -n "$ip" ]] && PROTECTED_IPS["${ip}"]=1 - done - fi - done <<< "$instances_data" + # Collect IPs to protect Addresses later + IFS=';' read -ra network_ips <<< "$ips_list"; for ip in "${network_ips[@]}"; do [[ -n "$ip" ]] && PROTECTED_IPS["${ip}"]=1; done + IFS=';' read -ra nat_ips <<< "$nat_ips_list"; for ip in "${nat_ips[@]}"; do [[ -n "$ip" ]] && PROTECTED_IPS["${ip}"]=1; done + done + fi # Find Address resource names for the PROTECTED_IPS if ((${#PROTECTED_IPS[@]} > 0)); then @@ -273,6 +345,7 @@ process_resources() { ((count++)) || true fi done <<< "$resources" + log "INFO" "Finished processing $label. $count resources actioned." } # SPECIFIC HANDLERS @@ -338,7 +411,7 @@ process_addresses() { [[ -z "$name" ]] && continue if ! is_excluded "$name" "${labels_str:-}"; then if [[ "$status" == "IN_USE" ]]; then - log "WARNING" "Skipping IN_USE Global Address $name NOT explicitly excluded." + log "DEBUG" "Skipping IN_USE Global Address $name NOT explicitly excluded." continue fi execute_delete "Global Address" "$name" \ @@ -349,57 +422,6 @@ process_addresses() { fi } - -process_vpc_peerings() { - log "INFO" "--- Processing: VPC Peerings---" - local networks_data - if ! networks_data=$(gcloud compute networks list --project="$PROJECT_ID" --format="value(name)"); then - log "ERROR" "Failed to list networks." - ((ERROR_COUNT++)) || true - return 0 - fi - if [[ -z "$networks_data" ]]; then log "INFO" "No networks found in project."; return 0; fi - - local count=0 - while IFS=$'\t' read -r net_name; do - if [[ -z "$net_name" ]]; then continue; fi - - if [[ -n "${EXCLUSION_MAP[${net_name}]:-}" ]]; then - log "SKIP" "VPC Peerings for Network $net_name (Protected)" - continue - fi - - local peerings_data - if ! peerings_data=$(gcloud compute networks peerings list --network="$net_name" --project="$PROJECT_ID" --format="value(name,network,state)"); then - log "WARNING" "Failed to list peerings for network $net_name" - continue - fi - - if [[ -z "$peerings_data" ]]; then continue; fi - - while IFS=$'\t' read -r peering_name peer_network state; do - if [[ -z "$peering_name" ]]; then continue; fi - - if ! is_excluded "$peering_name"; then - if [[ "$peering_name" == "servicenetworking-googleapis-com" ]]; then - execute_delete "Service Peering" "$peering_name" \ - "gcloud services vpc-peerings delete --service=servicenetworking.googleapis.com --network=\"$net_name\" --project=\"$PROJECT_ID\" --quiet" \ - "(Network: $net_name)" - ((count++)) || true - elif [[ "$peering_name" == filestore-peer-* ]]; then continue; - elif [[ "$peer_network" == *"/global/networks/servicenetworking" ]]; then continue; - else - execute_delete "VPC Peering" "$peering_name" \ - "gcloud compute networks peerings delete \"$peering_name\" --network=\"$net_name\" --project=\"$PROJECT_ID\" --quiet" \ - "(Network: $net_name, State: $state)" - ((count++)) || true - fi - fi - done <<< "$peerings_data" - done <<< "$networks_data" - log "INFO" "Finished processing VPC Peerings. $count peerings actioned." -} - process_iam_deleted_members() { log "INFO" "--- Processing: IAM Role Bindings for Deleted SAs ---" local policy_data @@ -713,10 +735,11 @@ main() { # --- Phase 2: Images & Artifacts --- process_vm_images process_docker_images + # process_instance_templates # --- Phase 3: Network Infrastructure --- process_routers - # process_firewalls # Called in process_networks for dependencies + process_firewalls process_addresses process_resources "Zonal Disk" \ "gcloud compute disks list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND zone:*\" --format=\"value(name,zone.basename(),labels.map())\" | sort" \ From 2d5d9807524c68f3efdeda00ccf69d0a98ec4165 Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Mon, 22 Dec 2025 11:28:24 +0000 Subject: [PATCH 10/19] Instance templates and filestore --- tools/cleanup.sh | 61 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 7 deletions(-) diff --git a/tools/cleanup.sh b/tools/cleanup.sh index daba13b542..c2c101ba67 100755 --- a/tools/cleanup.sh +++ b/tools/cleanup.sh @@ -138,7 +138,7 @@ populate_protected_resources() { log "INFO" "Identifying protected resources..." declare -A INSTANCES_TO_PROTECT # Map instance_name -> zone - # Part 1: Instances from EXCLUDED GKE clusters + # Part 1: Instances and Templates from EXCLUDED GKE clusters log "INFO" "Checking for instances in EXCLUDED GKE clusters..." local clusters_data if ! clusters_data=$(gcloud container clusters list --project="$PROJECT_ID" --format="value(name,location,resourceLabels.map())"); then @@ -181,6 +181,20 @@ populate_protected_resources() { continue fi + # Get the instance template from the MIG + local template_url_from_mig + if ! template_url_from_mig=$(gcloud compute instance-groups managed describe "${ig_name}" --project="${PROJECT_ID}" "${scope_flag}" --format="value(instanceTemplate)"); then + log "WARNING" "Failed to get instance template for MIG ${ig_name}." + else + if [[ -n "$template_url_from_mig" ]]; then + local template_name=$(basename "$template_url_from_mig") + if [[ -n "$template_name" && -z "${EXCLUSION_MAP[${template_name}]:-}" ]]; then + log "INFO" " > Excluding Instance Template (from GKE MIG ${ig_name}): ${template_name}" + EXCLUSION_MAP["${template_name}"]=1 + fi + fi + fi + local instances_in_mig if ! instances_in_mig=$(gcloud compute instance-groups managed list-instances "${ig_name}" --project="${PROJECT_ID}" "${scope_flag}" --format="value(NAME,ZONE)"); then log "WARNING" "Failed to list instances for MIG ${ig_name}." @@ -231,13 +245,22 @@ populate_protected_resources() { log "DEBUG" "Fetching details for protected instance: ${inst_name} in ${zone}" local inst_details if ! inst_details=$(gcloud compute instances describe "${inst_name}" --zone="${zone}" --project="${PROJECT_ID}" \ - --format="value(disks[].source.list(separator=';'),networkInterfaces[].network.list(separator=';'),networkInterfaces[].subnetwork.list(separator=';'),networkInterfaces[].networkIP.list(separator=';'),networkInterfaces[].accessConfigs[].natIP.list(separator=';'))"); then + --format="value(disks[].source.list(separator=';'),networkInterfaces[].network.list(separator=';'),networkInterfaces[].subnetwork.list(separator=';'),networkInterfaces[].networkIP.list(separator=';'),networkInterfaces[].accessConfigs[].natIP.list(separator=';'),sourceInstanceTemplate)"); then log "WARNING" "Failed to describe protected instance ${inst_name} in ${zone}. Sub-resources might not be protected." continue fi - local disks_list nets_list subs_list ips_list nat_ips_list - IFS=$'\t' read -r disks_list nets_list subs_list ips_list nat_ips_list <<< "$inst_details" + local disks_list nets_list subs_list ips_list nat_ips_list template_url + IFS=$'\t' read -r disks_list nets_list subs_list ips_list nat_ips_list template_url <<< "$inst_details" + + # Protect Instance Template (for non-MIG instances) + if [[ -n "$template_url" ]]; then + local template_name=$(basename "$template_url") + if [[ -n "$template_name" && -z "${EXCLUSION_MAP[${template_name}]:-}" ]]; then + log "INFO" " > Excluding Instance Template (for ${inst_name}): ${template_name}" + EXCLUSION_MAP["${template_name}"]=1 + fi + fi # Protect Attached Disks IFS=';' read -ra disk_urls <<< "$disks_list" @@ -250,7 +273,7 @@ populate_protected_resources() { fi done - # Protect Network + # Protect Network & Subnetwork IFS=';' read -ra net_urls <<< "$nets_list" for net_url in "${net_urls[@]}"; do [[ -z "$net_url" ]] && continue @@ -259,9 +282,9 @@ populate_protected_resources() { log "INFO" " > Excluding Network (for ${inst_name}): ${net_name}" EXCLUSION_MAP["${net_name}"]=1 fi + PROTECTED_NETWORKS["${net_url}"]=1 # Store full network URL done - # Protect Subnetwork IFS=';' read -ra sub_urls <<< "$subs_list" for sub_url in "${sub_urls[@]}"; do [[ -z "$sub_url" ]] && continue @@ -278,7 +301,7 @@ populate_protected_resources() { done fi - # Find Address resource names for the PROTECTED_IPS + # Part 4: Find Address resource names for the PROTECTED_IPS if ((${#PROTECTED_IPS[@]} > 0)); then log "INFO" "Finding Address resource names for protected IPs..." local addresses_data @@ -295,6 +318,30 @@ populate_protected_resources() { done <<< "$addresses_data" fi fi + + # Part 5: Protect Filestore instances on the same network + if ((${#PROTECTED_NETWORKS[@]} > 0)); then + log "INFO" "Checking for Filestore instances on protected networks..." + local fs_data + if ! fs_data=$(gcloud filestore instances list --project="$PROJECT_ID" --format="value(name.segment(5), networks[0].network)"); then + log "ERROR" "Failed to list Filestore instances for network check." + ((ERROR_COUNT++)) || true + else + while IFS=$'\t' read -r fs_name fs_network; do + [[ -z "$fs_name" || -z "$fs_network" ]] && continue + + # Normalize fs_network URL to match the format from instance description + local full_fs_network="https://www.googleapis.com/compute/v1/projects/${PROJECT_ID}/global/networks/${fs_network}" + + if [[ -n "${PROTECTED_NETWORKS[${full_fs_network}]:-}" ]]; then + if [[ -z "${EXCLUSION_MAP[${fs_name}]:-}" ]]; then + log "INFO" " > Excluding Filestore (on network ${fs_network}): ${fs_name}" + EXCLUSION_MAP["${fs_name}"]=1 + fi + fi + done <<< "$fs_data" + fi + fi } log_exclusion_map() { From 8ec2e09415f869f0f9e63c57af2e84a8df6cc778 Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Mon, 22 Dec 2025 13:30:25 +0000 Subject: [PATCH 11/19] Instance templates --- tools/cleanup.sh | 75 ++++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/tools/cleanup.sh b/tools/cleanup.sh index c2c101ba67..c487ebb508 100755 --- a/tools/cleanup.sh +++ b/tools/cleanup.sh @@ -169,29 +169,43 @@ populate_protected_resources() { IFS=';' read -ra ig_url_list <<< "$ig_urls" for ig_url in "${ig_url_list[@]}"; do local ig_name=$(basename "${ig_url}") - local ig_scope_type=$(echo "${ig_url}" | awk -F'/' '{print $(NF-2)}') - local ig_scope_name=$(echo "${ig_url}" | awk -F'/' '{print $(NF-3)}') + # Corrected AWK indices + local ig_scope_type=$(echo "${ig_url}" | awk -F'/' '{print $(NF-3)}') + local ig_scope_name=$(echo "${ig_url}" | awk -F'/' '{print $(NF-2)}') local scope_flag="" + if [[ "$ig_scope_type" == "zones" ]]; then scope_flag="--zone=${ig_scope_name}" elif [[ "$ig_scope_type" == "regions" ]]; then scope_flag="--region=${ig_scope_name}" else - log "WARNING" "Unknown scope type for instance group: ${ig_url}" + log "WARNING" "Unknown scope type ('${ig_scope_type}') for instance group: ${ig_url}" continue fi + log "DEBUG" "Processing MIG: ${ig_name} (${scope_flag}) for cluster ${cluster_name}" + # Get the instance template from the MIG local template_url_from_mig if ! template_url_from_mig=$(gcloud compute instance-groups managed describe "${ig_name}" --project="${PROJECT_ID}" "${scope_flag}" --format="value(instanceTemplate)"); then log "WARNING" "Failed to get instance template for MIG ${ig_name}." else - if [[ -n "$template_url_from_mig" ]]; then + log "DEBUG" "MIG ${ig_name}: instanceTemplate URL is '${template_url_from_mig}'" + if [[ -n "$template_url_from_mig" && "$template_url_from_mig" != "None" ]]; then local template_name=$(basename "$template_url_from_mig") - if [[ -n "$template_name" && -z "${EXCLUSION_MAP[${template_name}]:-}" ]]; then - log "INFO" " > Excluding Instance Template (from GKE MIG ${ig_name}): ${template_name}" - EXCLUSION_MAP["${template_name}"]=1 + log "DEBUG" "MIG ${ig_name}: Extracted template name is '${template_name}'" + if [[ -n "$template_name" && "$template_name" != "None" ]]; then + if ! [[ -v EXCLUSION_MAP["$template_name"] ]]; then + log "INFO" " > Excluding Instance Template (from GKE MIG ${ig_name}): ${template_name}" + EXCLUSION_MAP["${template_name}"]=1 + else + log "DEBUG" " > Instance Template ${template_name} (from GKE MIG ${ig_name}) is already excluded." + fi + else + log "WARNING" " > Could not extract a valid template name from URL '${template_url_from_mig}' for MIG ${ig_name}" fi + else + log "DEBUG" "MIG ${ig_name}: No instanceTemplate URL found or value is 'None'." fi fi @@ -204,7 +218,7 @@ populate_protected_resources() { while IFS=$'\t' read -r inst_name inst_zone_url; do if [[ -n "$inst_name" ]]; then local inst_zone=$(basename "$inst_zone_url") - if [[ -z "${EXCLUSION_MAP[${inst_name}]:-}" ]]; then + if ! [[ -v EXCLUSION_MAP["$inst_name"] ]]; then log "INFO" " > Protecting Instance (from GKE ${cluster_name}): ${inst_name} in ${inst_zone}" EXCLUSION_MAP["${inst_name}"]=1 fi @@ -228,7 +242,7 @@ populate_protected_resources() { else while IFS=$'\t' read -r inst_name zone labels_str; do if is_excluded "$inst_name" "$labels_str"; then # Returns 0 if excluded - if [[ -z "${EXCLUSION_MAP[${inst_name}]:-}" ]]; then + if ! [[ -v EXCLUSION_MAP["$inst_name"] ]]; then log "INFO" " > Protecting Instance (from Label): ${inst_name} in ${zone}" EXCLUSION_MAP["${inst_name}"]=1 fi @@ -253,13 +267,23 @@ populate_protected_resources() { local disks_list nets_list subs_list ips_list nat_ips_list template_url IFS=$'\t' read -r disks_list nets_list subs_list ips_list nat_ips_list template_url <<< "$inst_details" + log "DEBUG" "Instance ${inst_name}: sourceInstanceTemplate value: '${template_url}'" + # Protect Instance Template (for non-MIG instances) if [[ -n "$template_url" ]]; then local template_name=$(basename "$template_url") - if [[ -n "$template_name" && -z "${EXCLUSION_MAP[${template_name}]:-}" ]]; then - log "INFO" " > Excluding Instance Template (for ${inst_name}): ${template_name}" - EXCLUSION_MAP["${template_name}"]=1 + if [[ -n "$template_name" && "$template_name" != "None" ]]; then + if ! [[ -v EXCLUSION_MAP["$template_name"] ]]; then + log "INFO" " > Excluding Instance Template (for ${inst_name}): ${template_name}" + EXCLUSION_MAP["${template_name}"]=1 + else + log "DEBUG" " > Instance Template ${template_name} (for ${inst_name}) is already excluded." + fi + else + log "DEBUG" "Instance ${inst_name}: No valid template name found from URL '${template_url}'" fi + else + log "DEBUG" "Instance ${inst_name}: sourceInstanceTemplate is empty or not set." fi # Protect Attached Disks @@ -282,7 +306,8 @@ populate_protected_resources() { log "INFO" " > Excluding Network (for ${inst_name}): ${net_name}" EXCLUSION_MAP["${net_name}"]=1 fi - PROTECTED_NETWORKS["${net_url}"]=1 # Store full network URL + # Use network name as key + [[ -n "${net_name}" ]] && PROTECTED_NETWORK_NAMES["${net_name}"]=1 done IFS=';' read -ra sub_urls <<< "$subs_list" @@ -318,30 +343,6 @@ populate_protected_resources() { done <<< "$addresses_data" fi fi - - # Part 5: Protect Filestore instances on the same network - if ((${#PROTECTED_NETWORKS[@]} > 0)); then - log "INFO" "Checking for Filestore instances on protected networks..." - local fs_data - if ! fs_data=$(gcloud filestore instances list --project="$PROJECT_ID" --format="value(name.segment(5), networks[0].network)"); then - log "ERROR" "Failed to list Filestore instances for network check." - ((ERROR_COUNT++)) || true - else - while IFS=$'\t' read -r fs_name fs_network; do - [[ -z "$fs_name" || -z "$fs_network" ]] && continue - - # Normalize fs_network URL to match the format from instance description - local full_fs_network="https://www.googleapis.com/compute/v1/projects/${PROJECT_ID}/global/networks/${fs_network}" - - if [[ -n "${PROTECTED_NETWORKS[${full_fs_network}]:-}" ]]; then - if [[ -z "${EXCLUSION_MAP[${fs_name}]:-}" ]]; then - log "INFO" " > Excluding Filestore (on network ${fs_network}): ${fs_name}" - EXCLUSION_MAP["${fs_name}"]=1 - fi - fi - done <<< "$fs_data" - fi - fi } log_exclusion_map() { From d11294e2bc33dac5f00e34c90e7b7c5bd2d57677 Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Mon, 22 Dec 2025 13:34:18 +0000 Subject: [PATCH 12/19] dry-run false --- tools/cloud-build/project-cleanup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cloud-build/project-cleanup.yaml b/tools/cloud-build/project-cleanup.yaml index 0643843107..399dfd0db3 100644 --- a/tools/cloud-build/project-cleanup.yaml +++ b/tools/cloud-build/project-cleanup.yaml @@ -20,7 +20,7 @@ steps: env: - "BUILD_ID=${BUILD_ID}" - "PROJECT_ID=${PROJECT_ID}" - - "DRY_RUN=true" + - "DRY_RUN=false" - "EXCLUSION_FILE=gs://hpc-ctk1357/cleanup/exclusions.txt" args: - -c From 21a0a839d46f47b8f2a8d6215f24955146bb119d Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Mon, 22 Dec 2025 16:49:30 +0000 Subject: [PATCH 13/19] Change label name --- tools/cleanup.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/cleanup.sh b/tools/cleanup.sh index c487ebb508..0dda9623f0 100755 --- a/tools/cleanup.sh +++ b/tools/cleanup.sh @@ -87,25 +87,25 @@ is_excluded() { local KEY VAL KEY="${PAIR%%=*}" VAL="${PAIR#*=}" - if [[ "$KEY" == "do-not-delete" ]]; then + if [[ "$KEY" == "cleanup-exemption-date" ]]; then if [[ "$VAL" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then local exp_seconds if ! exp_seconds=$(date -d "$VAL + 1 day" -u +%s 2>/dev/null); then - log "WARNING" "$resource_name (Label: do-not-delete invalid date value: $VAL)" + log "WARNING" "$resource_name (Label: cleanup-exemption-date invalid date value: $VAL)" return 1 # Not excluded else local current_seconds current_seconds=$(date -u +%s) if [[ "$exp_seconds" -gt "$current_seconds" ]]; then - log "SKIP" "$resource_name (Label: do-not-delete=$VAL, valid)" + log "SKIP" "$resource_name (Label: cleanup-exemption-date=$VAL, valid)" return 0 # Excluded else - log "INFO" "$resource_name (Label: do-not-delete=$VAL expired)" + log "INFO" "$resource_name (Label: cleanup-exemption-date=$VAL expired)" return 1 # Not excluded fi fi else - log "WARNING" "$resource_name (Label: do-not-delete invalid date format: $VAL, expected YYYY-MM-DD)" + log "WARNING" "$resource_name (Label: cleanup-exemption-date invalid date format: $VAL, expected YYYY-MM-DD)" return 1 # Not excluded fi break @@ -231,13 +231,13 @@ populate_protected_resources() { fi # Part 2: Instances protected via direct labels - log "INFO" "Checking for instances with do-not-delete label..." + log "INFO" "Checking for instances with cleanup-exemption-date label..." local labeled_instances_data if ! labeled_instances_data=$(gcloud compute instances list \ --project="$PROJECT_ID" \ - --filter="labels.do-not-delete:*" \ + --filter="labels.cleanup-exemption-date:*" \ --format="value(name,zone.basename(),labels.map())"); then - log "ERROR" "Failed to list instances with do-not-delete label." + log "ERROR" "Failed to list instances with cleanup-exemption-date label." ((ERROR_COUNT++)) || true else while IFS=$'\t' read -r inst_name zone labels_str; do From 2adb0043fb52352706f49645d2884ae0f30403ba Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Mon, 22 Dec 2025 17:33:04 +0000 Subject: [PATCH 14/19] Filestore protection --- tools/cleanup.sh | 61 +++++++++++++++++++++----- tools/cloud-build/project-cleanup.yaml | 2 +- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/tools/cleanup.sh b/tools/cleanup.sh index 0dda9623f0..8c20842993 100755 --- a/tools/cleanup.sh +++ b/tools/cleanup.sh @@ -6,6 +6,11 @@ declare -A EXCLUSION_MAP ERROR_COUNT=0 +# To store IPs of protected instances, to find matching Address resources +declare -A PROTECTED_IPS +# To store Network URIs used by protected instances +declare -A PROTECTED_NETWORK_URIS + # Environment Variables expected from Cloud Build PROJECT_ID="${PROJECT_ID:-hpc-toolkit-dev}" DRY_RUN="${DRY_RUN:-true}" @@ -13,9 +18,6 @@ EXCLUSION_FILE="${EXCLUSION_FILE:-gs://hpc-ctk1357/cleanup/exclusions.txt}" CUTOFF_TIME="${CUTOFF_TIME:-$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%S%z)}" CUTOFF_TIME_IMAGES="${CUTOFF_TIME_IMAGES:-$(date -d "60 days ago" -u +%Y-%m-%dT%H:%M:%S%z)}" -# To store IPs of protected instances, to find matching Address resources -declare -A PROTECTED_IPS - # HELPER FUNCTIONS log() { @@ -306,8 +308,11 @@ populate_protected_resources() { log "INFO" " > Excluding Network (for ${inst_name}): ${net_name}" EXCLUSION_MAP["${net_name}"]=1 fi - # Use network name as key - [[ -n "${net_name}" ]] && PROTECTED_NETWORK_NAMES["${net_name}"]=1 + # Store the full network URI for Filestore matching + if [[ -n "${net_url}" ]]; then + log "DEBUG" "Adding protected network URI: ${net_url}" + PROTECTED_NETWORK_URIS["${net_url}"]=1 + fi done IFS=';' read -ra sub_urls <<< "$subs_list" @@ -357,6 +362,18 @@ log_exclusion_map() { log "INFO" "--- End of Exclusion Map ---" } +log_protected_network_uris() { + log "INFO" "--- Protected Network URIs ---" + if [ ${#PROTECTED_NETWORK_URIS[@]} -eq 0 ]; then + log "INFO" "PROTECTED_NETWORK_URIS map is empty." + return + fi + for key in "${!PROTECTED_NETWORK_URIS[@]}"; do + log "INFO" "PROTECTED NET URI: $key" + done + log "INFO" "--- End of Protected Network URIs ---" +} + # STANDARD PROCESSOR process_resources() { @@ -609,9 +626,9 @@ process_firewalls() { process_filestore() { log "INFO" "--- Processing: Filestore Instances ---" local fs_data - # Get instance name, location, and labels using segment projections + # Get instance name, location, labels, and network URI if ! fs_data=$(gcloud filestore instances list --project="$PROJECT_ID" --filter="createTime < '$CUTOFF_TIME'" \ - --format="value(name.segment(5), name.segment(3), labels.map())"); then + --format="value(name.segment(5), name.segment(3), labels.map(), networks[0].network)"); then log "ERROR" "Failed to list Filestore instances." ((ERROR_COUNT++)) || true return 0 @@ -619,18 +636,39 @@ process_filestore() { if [[ -z "$fs_data" ]]; then log "INFO" "No Filestore instances found matching criteria."; return 0; fi local count=0 - while IFS=$'\t' read -r name location labels_str; do + while IFS=$'\t' read -r name location labels_str network_uri; do # Trim potential whitespace name=$(echo "$name" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') location=$(echo "$location" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + network_uri=$(echo "$network_uri" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') if [[ -z "$name" || "$name" == "None" || -z "$location" || "$location" == "None" ]]; then - log "WARNING" "Could not extract valid name or location for a Filestore instance from line: $name $location $labels_str" + log "WARNING" "Could not extract valid name or location for a Filestore instance." + continue + fi + + log "DEBUG" "Filestore $name: Checking network URI: '$network_uri'" + # Protect Filestore if its network is used by any protected VM + local network_basename=$(basename "$network_uri") + # Construct the expected full URI format similar to compute instances + local full_network_uri="https://www.googleapis.com/compute/v1/projects/${PROJECT_ID}/global/networks/${network_basename}" + + if [[ -n "${network_uri}" && -n "${PROTECTED_NETWORK_URIS[${full_network_uri}]:-}" ]]; then + if [[ -z "${EXCLUSION_MAP[${name}]:-}" ]]; then + log "INFO" "SKIP : Filestore $name ($location) on PROTECTED network $network_basename (URI: $full_network_uri)" + EXCLUSION_MAP["${name}"]=1 + fi continue + else + if [[ -n "${network_uri}" ]]; then + log "DEBUG" "Filestore $name: Network URI '$full_network_uri' not found in PROTECTED_NETWORK_URIS." + else + log "DEBUG" "Filestore $name: Network URI is empty." + fi fi if ! is_excluded "$name" "${labels_str:-}"; then - log "INFO" "Processing Filestore instance: $name in $location" + log "INFO" "Processing Filestore instance for potential deletion: $name in $location" if [[ "$DRY_RUN" == "true" ]]; then log "DRY-RUN" "Would disable deletion protection on Filestore: $name ($location)" @@ -766,6 +804,7 @@ main() { check_dependencies load_exclusions populate_protected_resources + log_protected_network_uris # Log the protected network URIs log_exclusion_map # Log the map contents # --- Phase 1: High Level Resources --- @@ -774,7 +813,7 @@ main() { "gcloud container clusters delete --project=\"$PROJECT_ID\"" "location" process_resources "Instance Template" \ "gcloud compute instance-templates list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name, 'Global', labels.map())\" | sort" \ - "gcloud compute instance-templates delete --project=\"$PROJECT_ID\"" + "gcloud compute instance-templates delete --project=\"$PROJECT_ID\"" process_resources "Compute Instance" \ "gcloud compute instances list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,zone.basename(),labels.map())\" | sort" \ "gcloud compute instances delete --project=\"$PROJECT_ID\" --delete-disks=all" "zone" diff --git a/tools/cloud-build/project-cleanup.yaml b/tools/cloud-build/project-cleanup.yaml index 399dfd0db3..2f04ac746d 100644 --- a/tools/cloud-build/project-cleanup.yaml +++ b/tools/cloud-build/project-cleanup.yaml @@ -28,7 +28,7 @@ steps: set -euo pipefail # Set time variables - export CUTOFF_TIME=$(date -d '2 days ago' -u +%Y-%m-%dT%H:%M:%S%z) + export CUTOFF_TIME=$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%S%z) export CUTOFF_TIME_IMAGES=$(date -d "60 days ago" -u +%Y-%m-%dT%H:%M:%S%z) attempt=1 From 5f0bd3ca5c1e05d4659dedbdacb2e1013297afb7 Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Tue, 23 Dec 2025 18:18:06 +0000 Subject: [PATCH 15/19] Adding TPUs --- tools/cleanup.sh | 147 ++++++++++++++++--------- tools/cloud-build/project-cleanup.yaml | 2 +- 2 files changed, 95 insertions(+), 54 deletions(-) diff --git a/tools/cleanup.sh b/tools/cleanup.sh index 8c20842993..7fa4f912f3 100755 --- a/tools/cleanup.sh +++ b/tools/cleanup.sh @@ -136,9 +136,39 @@ execute_delete() { fi } +# Helper function to add network/subnetwork to exclusion lists +_protect_network_resources() { + local source_resource_type="$1" + local source_resource_name="$2" + local net_url="$3" + local sub_url="$4" + + # Protect Network + if [[ -n "$net_url" && "$net_url" != "None" ]]; then + local net_name=$(basename "${net_url}") + if [[ -n "${net_name}" && -z "${EXCLUSION_MAP[${net_name}]:-}" ]]; then + log "INFO" " > Excluding Network (for ${source_resource_type} ${source_resource_name}): ${net_name}" + EXCLUSION_MAP["${net_name}"]=1 + fi + # Store the full network URI for Filestore matching + log "DEBUG" "Adding protected network URI (for ${source_resource_type} ${source_resource_name}): ${net_url}" + PROTECTED_NETWORK_URIS["${net_url}"]=1 + fi + + # Protect Subnetwork + if [[ -n "$sub_url" && "$sub_url" != "None" ]]; then + local sub_name=$(basename "${sub_url}") + if [[ -n "${sub_name}" && -z "${EXCLUSION_MAP[${sub_name}]:-}" ]]; then + log "INFO" " > Excluding Subnetwork (for ${source_resource_type} ${source_resource_name}): ${sub_name}" + EXCLUSION_MAP["${sub_name}"]=1 + fi + fi +} + populate_protected_resources() { log "INFO" "Identifying protected resources..." declare -A INSTANCES_TO_PROTECT # Map instance_name -> zone + declare -A TPUS_TO_PROTECT # Map tpu_name -> zone # Part 1: Instances and Templates from EXCLUDED GKE clusters log "INFO" "Checking for instances in EXCLUDED GKE clusters..." @@ -232,25 +262,24 @@ populate_protected_resources() { done <<< "$clusters_data" fi - # Part 2: Instances protected via direct labels - log "INFO" "Checking for instances with cleanup-exemption-date label..." - local labeled_instances_data - if ! labeled_instances_data=$(gcloud compute instances list \ + # Part 2: Instances protected via direct labels or name + log "INFO" "Checking for instances to protect..." + local instances_data + if ! instances_data=$(gcloud compute instances list \ --project="$PROJECT_ID" \ - --filter="labels.cleanup-exemption-date:*" \ --format="value(name,zone.basename(),labels.map())"); then - log "ERROR" "Failed to list instances with cleanup-exemption-date label." + log "ERROR" "Failed to list instances." ((ERROR_COUNT++)) || true else while IFS=$'\t' read -r inst_name zone labels_str; do if is_excluded "$inst_name" "$labels_str"; then # Returns 0 if excluded if ! [[ -v EXCLUSION_MAP["$inst_name"] ]]; then - log "INFO" " > Protecting Instance (from Label): ${inst_name} in ${zone}" + log "INFO" " > Protecting Instance: ${inst_name} in ${zone}" EXCLUSION_MAP["${inst_name}"]=1 fi INSTANCES_TO_PROTECT["${inst_name}"]="${zone}" fi - done <<< "$labeled_instances_data" + done <<< "$instances_data" fi # Part 3: Protect resources associated with the collected instances @@ -299,30 +328,12 @@ populate_protected_resources() { fi done - # Protect Network & Subnetwork + # Refactored Network & Subnetwork Protection IFS=';' read -ra net_urls <<< "$nets_list" - for net_url in "${net_urls[@]}"; do - [[ -z "$net_url" ]] && continue - local net_name=$(basename "${net_url}") - if [[ -n "${net_name}" && -z "${EXCLUSION_MAP[${net_name}]:-}" ]]; then - log "INFO" " > Excluding Network (for ${inst_name}): ${net_name}" - EXCLUSION_MAP["${net_name}"]=1 - fi - # Store the full network URI for Filestore matching - if [[ -n "${net_url}" ]]; then - log "DEBUG" "Adding protected network URI: ${net_url}" - PROTECTED_NETWORK_URIS["${net_url}"]=1 - fi - done - IFS=';' read -ra sub_urls <<< "$subs_list" - for sub_url in "${sub_urls[@]}"; do - [[ -z "$sub_url" ]] && continue - local sub_name=$(basename "${sub_url}") - if [[ -n "${sub_name}" && -z "${EXCLUSION_MAP[${sub_name}]:-}" ]]; then - log "INFO" " > Excluding Subnetwork (for ${inst_name}): ${sub_name}" - EXCLUSION_MAP["${sub_name}"]=1 - fi + # Assuming the number of networks and subnetworks in the arrays match order + for i in "${!net_urls[@]}"; do + _protect_network_resources "Instance" "$inst_name" "${net_urls[$i]:-}" "${sub_urls[$i]:-}" done # Collect IPs to protect Addresses later @@ -331,7 +342,46 @@ populate_protected_resources() { done fi - # Part 4: Find Address resource names for the PROTECTED_IPS + # Part 4: TPU VMs protected via direct labels or name + log "INFO" "Checking for TPU VMs to protect..." + local tpus_data + if ! tpus_data=$(gcloud compute tpus tpu-vm list \ + --project="$PROJECT_ID" \ + --format="value(name,zone.basename(),labels.map())"); then + log "ERROR" "Failed to list TPU VMs." + ((ERROR_COUNT++)) || true + else + while IFS=$'\t' read -r tpu_name zone labels_str; do + if is_excluded "$tpu_name" "$labels_str"; then # Returns 0 if excluded + if ! [[ -v EXCLUSION_MAP["$tpu_name"] ]]; then + log "INFO" " > Protecting TPU VM: ${tpu_name} in ${zone}" + EXCLUSION_MAP["${tpu_name}"]=1 + fi + TPUS_TO_PROTECT["${tpu_name}"]="${zone}" + fi + done <<< "$tpus_data" + fi + + # Part 5: Protect resources associated with the collected TPU VMs + if ((${#TPUS_TO_PROTECT[@]} > 0)); then + log "INFO" "Protecting sub-resources of ${#TPUS_TO_PROTECT[@]} TPU VMs..." + for tpu_name in "${!TPUS_TO_PROTECT[@]}"; do + local zone="${TPUS_TO_PROTECT[$tpu_name]}" + log "DEBUG" "Fetching details for protected TPU VM: ${tpu_name} in ${zone}" + local tpu_details + if ! tpu_details=$(gcloud compute tpus tpu-vm describe "${tpu_name}" --zone="${zone}" --project="${PROJECT_ID}" \ + --format="value(networkEndpoints[0].network,networkEndpoints[0].subnetwork)"); then + log "WARNING" "Failed to describe protected TPU VM ${tpu_name} in ${zone}. Sub-resources might not be protected." + continue + fi + + local net_url sub_url + IFS=$'\t' read -r net_url sub_url <<< "$tpu_details" + _protect_network_resources "TPU" "$tpu_name" "$net_url" "$sub_url" + done + fi + + # Part 6: Find Address resource names for the PROTECTED_IPS if ((${#PROTECTED_IPS[@]} > 0)); then log "INFO" "Finding Address resource names for protected IPs..." local addresses_data @@ -526,32 +576,21 @@ process_vm_images() { log "INFO" "--- Processing: VM Images ---" local images if ! images=$(gcloud compute images list --project="$PROJECT_ID" --no-standard-images \ + --filter="creationTimestamp < '$CUTOFF_TIME_IMAGES'" \ --format="value(name,creationTimestamp,labels.map())"); then log "ERROR" "Failed to list VM images" ((ERROR_COUNT++)) || true return 0 fi - if [[ -z "$images" ]]; then log "INFO" "No custom VM images found."; return 0; fi - local cutoff_seconds - if ! cutoff_seconds=$(date -d "$CUTOFF_TIME_IMAGES" +%s); then - log "ERROR" "Failed to calculate cutoff time for images" - ((ERROR_COUNT++)) || true - return 0 - fi + if [[ -z "$images" ]]; then log "INFO" "No custom VM images found matching criteria."; return 0; fi + local count=0 while IFS=$'\t' read -r name timestamp labels_str; do [[ -z "$name" ]] && continue if ! is_excluded "$name" "${labels_str:-}"; then - local ts_seconds - if ! ts_seconds=$(date -d "$timestamp" +%s 2>/dev/null); then - log "WARNING" "Could not parse timestamp '$timestamp' for image $name. Skipping." - continue - fi - if [[ $ts_seconds -lt $cutoff_seconds ]]; then - execute_delete "VM Image" "$name" \ - "gcloud compute images delete \"$name\" --project=\"$PROJECT_ID\" --quiet" - ((count++)) || true - fi + execute_delete "VM Image" "$name" \ + "gcloud compute images delete \"$name\" --project=\"$PROJECT_ID\" --quiet" + ((count++)) || true fi done <<< "$images" } @@ -807,13 +846,15 @@ main() { log_protected_network_uris # Log the protected network URIs log_exclusion_map # Log the map contents - # --- Phase 1: High Level Resources --- + # --- Phase 1: High Level Compute Resources --- process_resources "GKE Cluster" \ "gcloud container clusters list --project=\"$PROJECT_ID\" --filter=\"createTime < '$CUTOFF_TIME'\" --format=\"value(name,location,resourceLabels.map())\" | sort" \ "gcloud container clusters delete --project=\"$PROJECT_ID\"" "location" - process_resources "Instance Template" \ - "gcloud compute instance-templates list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name, 'Global', labels.map())\" | sort" \ - "gcloud compute instance-templates delete --project=\"$PROJECT_ID\"" + + process_resources "TPU VM" \ + "gcloud compute tpus tpu-vm list --project=\"$PROJECT_ID\" --filter=\"createTime < '$CUTOFF_TIME'\" --format=\"value(name,zone.basename(),labels.map())\" | sort" \ + "gcloud compute tpus tpu-vm delete --project=\"$PROJECT_ID\"" "zone" + process_resources "Compute Instance" \ "gcloud compute instances list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,zone.basename(),labels.map())\" | sort" \ "gcloud compute instances delete --project=\"$PROJECT_ID\" --delete-disks=all" "zone" @@ -822,7 +863,7 @@ main() { # --- Phase 2: Images & Artifacts --- process_vm_images process_docker_images - # process_instance_templates + process_instance_templates # --- Phase 3: Network Infrastructure --- process_routers diff --git a/tools/cloud-build/project-cleanup.yaml b/tools/cloud-build/project-cleanup.yaml index 2f04ac746d..cbf859c4fa 100644 --- a/tools/cloud-build/project-cleanup.yaml +++ b/tools/cloud-build/project-cleanup.yaml @@ -20,7 +20,7 @@ steps: env: - "BUILD_ID=${BUILD_ID}" - "PROJECT_ID=${PROJECT_ID}" - - "DRY_RUN=false" + - "DRY_RUN=true" - "EXCLUSION_FILE=gs://hpc-ctk1357/cleanup/exclusions.txt" args: - -c From d674acc125d343d42442233d5b1d2cb933a6714d Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Wed, 24 Dec 2025 07:04:02 +0000 Subject: [PATCH 16/19] Enhanced TPU deletion --- tools/cleanup.sh | 136 +++++++++++++++++++++---- tools/cloud-build/project-cleanup.yaml | 5 +- 2 files changed, 119 insertions(+), 22 deletions(-) diff --git a/tools/cleanup.sh b/tools/cleanup.sh index 7fa4f912f3..dc485b6729 100755 --- a/tools/cleanup.sh +++ b/tools/cleanup.sh @@ -344,22 +344,53 @@ populate_protected_resources() { # Part 4: TPU VMs protected via direct labels or name log "INFO" "Checking for TPU VMs to protect..." - local tpus_data - if ! tpus_data=$(gcloud compute tpus tpu-vm list \ - --project="$PROJECT_ID" \ - --format="value(name,zone.basename(),labels.map())"); then + local tpus_raw + # Get the raw list with all details + if ! tpus_raw=$(gcloud compute tpus tpu-vm list --project="$PROJECT_ID" --zone - --format="json"); then log "ERROR" "Failed to list TPU VMs." ((ERROR_COUNT++)) || true else - while IFS=$'\t' read -r tpu_name zone labels_str; do - if is_excluded "$tpu_name" "$labels_str"; then # Returns 0 if excluded + # Extract 'name' and 'labels' from JSON blocks manually using grep/sed + # We look for lines containing "name": and "labels": + # This assumes standard JSON formatting from gcloud + + # We'll use a temporary file to process blocks safely + local tmp_file=$(mktemp) + echo "$tpus_raw" > "$tmp_file" + + # Find the line numbers where a new TPU object starts + local start_lines=$(grep -n "{" "$tmp_file" | cut -d: -f1) + + # Instead of complex block parsing, we'll use a simpler 'grep' approach + # that pairs the Name and Labels together. + while read -r line; do + # Extract the full resource path from the "name": line + local full_path=$(echo "$line" | grep '"name":' | sed 's/.*"name": "\(.*\)",/\1/') + [[ -z "$full_path" ]] && continue + + # Extract the short name + local tpu_name="${full_path##*/}" + + # Extract the zone from the resource path + local tpu_zone=$(echo "$full_path" | sed -n 's/.*\/locations\/\([^\/]*\)\/.*/\1/p') + + # Extract labels for this specific TPU + # Since we can't easily parse JSON labels with sed, we'll fetch them specifically for this TPU + local tpu_labels=$(gcloud compute tpus tpu-vm describe "$tpu_name" --zone="$tpu_zone" --project="$PROJECT_ID" --format="value(labels.map())" 2>/dev/null) + + log "DEBUG" "Parsed TPU: name='${tpu_name}', zone='${tpu_zone}'" + + if is_excluded "$tpu_name" "$tpu_labels"; then if ! [[ -v EXCLUSION_MAP["$tpu_name"] ]]; then - log "INFO" " > Protecting TPU VM: ${tpu_name} in ${zone}" + log "INFO" " > Protecting TPU VM: ${tpu_name} in ${tpu_zone}" EXCLUSION_MAP["${tpu_name}"]=1 fi - TPUS_TO_PROTECT["${tpu_name}"]="${zone}" + if [[ -n "$tpu_zone" ]]; then + TPUS_TO_PROTECT["${tpu_name}"]="${tpu_zone}" + fi fi - done <<< "$tpus_data" + done < <(grep '"name":' "$tmp_file") + rm -f "$tmp_file" fi # Part 5: Protect resources associated with the collected TPU VMs @@ -367,10 +398,16 @@ populate_protected_resources() { log "INFO" "Protecting sub-resources of ${#TPUS_TO_PROTECT[@]} TPU VMs..." for tpu_name in "${!TPUS_TO_PROTECT[@]}"; do local zone="${TPUS_TO_PROTECT[$tpu_name]}" + if [[ -z "$zone" ]]; then + log "WARNING" "Skipping sub-resource protection for TPU ${tpu_name} due to missing zone." + continue + fi + log "DEBUG" "Fetching details for protected TPU VM: ${tpu_name} in ${zone}" local tpu_details - if ! tpu_details=$(gcloud compute tpus tpu-vm describe "${tpu_name}" --zone="${zone}" --project="${PROJECT_ID}" \ - --format="value(networkEndpoints[0].network,networkEndpoints[0].subnetwork)"); then + if ! tpu_details=$(gcloud compute tpus tpu-vm describe "${tpu_name}" \ + --zone="${zone}" --project="${PROJECT_ID}" \ + --format="value(networkConfig.network, networkConfig.subnetwork)"); then log "WARNING" "Failed to describe protected TPU VM ${tpu_name} in ${zone}. Sub-resources might not be protected." continue fi @@ -398,6 +435,62 @@ populate_protected_resources() { done <<< "$addresses_data" fi fi + + # Part 7: Protect Instance Templates based on Network Configuration (No jq) + log "INFO" "Checking Instance Templates for usage of protected networks..." + local templates_data + if ! templates_data=$(gcloud compute instance-templates list --project="$PROJECT_ID" \ + --format="value(name,properties.networkInterfaces.network.list(separator=';'),properties.networkInterfaces.subnetwork.list(separator=';'))"); then + log "WARNING" "Failed to list instance templates for network-based protection." + else + while IFS=$'\t' read -r template_name nets_str subs_str; do + if [[ -z "$template_name" ]]; then + continue + fi + + # Skip if template is already excluded + if [[ -v EXCLUSION_MAP["$template_name"] ]]; then + continue + fi + + local found_protected=false + + # Check Network URIs + if [[ -n "$nets_str" ]]; then + IFS=';' read -ra net_uris <<< "$nets_str" + for net_uri in "${net_uris[@]}"; do + if [[ -n "$net_uri" ]] && [[ -v PROTECTED_NETWORK_URIS["$net_uri"] ]]; then + local net_name=$(basename "$net_uri") + log "INFO" " > Excluding Instance Template (uses protected network '${net_name}'): ${template_name}" + EXCLUSION_MAP["${template_name}"]=1 + found_protected=true + break + fi + done + fi + + if [[ "$found_protected" = true ]]; then + continue # Move to the next template + fi + + # Check Subnetwork URIs + if [[ -n "$subs_str" ]]; then + IFS=';' read -ra sub_uris <<< "$subs_str" + for sub_uri in "${sub_uris[@]}"; do + if [[ -n "$sub_uri" ]]; then + local sub_name=$(basename "$sub_uri") + if [[ -n "$sub_name" ]] && [[ -v EXCLUSION_MAP["$sub_name"] ]]; then + log "INFO" " > Excluding Instance Template (uses protected subnetwork '${sub_name}'): ${template_name}" + EXCLUSION_MAP["${template_name}"]=1 + # found_protected=true # Not strictly needed as we don't have further checks for this template + break + fi + fi + done + fi + + done <<< "$templates_data" + fi } log_exclusion_map() { @@ -430,7 +523,7 @@ process_resources() { local label="$1" local list_command="$2" local delete_command_base="$3" - local scope_type="$4" + local scope_type="$4" # Should be 'zone' or 'location' or 'region' log "INFO" "--- Processing: $label ---" @@ -447,7 +540,12 @@ process_resources() { fi local count=0 - while IFS=$'\t' read -r name scope labels_str; do + while IFS=',' read -r name scope labels_str; do + # Trim potential whitespace and quotes from CSV + name=$(echo "$name" | sed -e 's/^[[:space:]"]*//' -e 's/[[:space:]"]*$//') + scope=$(echo "$scope" | sed -e 's/^[[:space:]"]*//' -e 's/[[:space:]"]*$//') + labels_str=$(echo "$labels_str" | sed -e 's/^[[:space:]"]*//' -e 's/[[:space:]"]*$//') + [[ -z "$name" ]] && continue if ! is_excluded "$name" "${labels_str:-}"; then @@ -456,14 +554,14 @@ process_resources() { final_cmd="$final_cmd --$scope_type=\"$scope\"" fi - execute_delete "$label" "$name" "$final_cmd" "${scope:-(Global)}" + execute_delete "$label" "$name" "$final_cmd" "($scope)" ((count++)) || true fi done <<< "$resources" log "INFO" "Finished processing $label. $count resources actioned." } -# SPECIFIC HANDLERS +# SPECIFIC HANDLERS (Unchanged from previous version) process_instance_templates() { log "INFO" "--- Processing: Instance Templates ---" @@ -848,15 +946,15 @@ main() { # --- Phase 1: High Level Compute Resources --- process_resources "GKE Cluster" \ - "gcloud container clusters list --project=\"$PROJECT_ID\" --filter=\"createTime < '$CUTOFF_TIME'\" --format=\"value(name,location,resourceLabels.map())\" | sort" \ + "gcloud container clusters list --project=\"$PROJECT_ID\" --filter=\"createTime < '$CUTOFF_TIME'\" --format=\"csv[no-heading](name,location,resourceLabels.map())\" | sort" \ "gcloud container clusters delete --project=\"$PROJECT_ID\"" "location" process_resources "TPU VM" \ - "gcloud compute tpus tpu-vm list --project=\"$PROJECT_ID\" --filter=\"createTime < '$CUTOFF_TIME'\" --format=\"value(name,zone.basename(),labels.map())\" | sort" \ + "gcloud compute tpus tpu-vm list --project=\"$PROJECT_ID\" --filter=\"createTime < '$CUTOFF_TIME'\" --zone - --format=\"csv[no-heading](name,location,labels.map())\" | sort" \ "gcloud compute tpus tpu-vm delete --project=\"$PROJECT_ID\"" "zone" process_resources "Compute Instance" \ - "gcloud compute instances list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"value(name,zone.basename(),labels.map())\" | sort" \ + "gcloud compute instances list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME'\" --format=\"csv[no-heading](name,zone.basename(),labels.map())\" | sort" \ "gcloud compute instances delete --project=\"$PROJECT_ID\" --delete-disks=all" "zone" process_filestore @@ -870,7 +968,7 @@ main() { process_firewalls process_addresses process_resources "Zonal Disk" \ - "gcloud compute disks list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND zone:*\" --format=\"value(name,zone.basename(),labels.map())\" | sort" \ + "gcloud compute disks list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND zone:*\" --format=\"csv[no-heading](name,zone.basename(),labels.map())\" | sort" \ "gcloud compute disks delete --project=\"$PROJECT_ID\"" "zone" # # --- Phase 4: Networking Hierarchies --- diff --git a/tools/cloud-build/project-cleanup.yaml b/tools/cloud-build/project-cleanup.yaml index cbf859c4fa..9983431026 100644 --- a/tools/cloud-build/project-cleanup.yaml +++ b/tools/cloud-build/project-cleanup.yaml @@ -20,19 +20,18 @@ steps: env: - "BUILD_ID=${BUILD_ID}" - "PROJECT_ID=${PROJECT_ID}" - - "DRY_RUN=true" + - "DRY_RUN=false" - "EXCLUSION_FILE=gs://hpc-ctk1357/cleanup/exclusions.txt" args: - -c - | set -euo pipefail - # Set time variables export CUTOFF_TIME=$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%S%z) export CUTOFF_TIME_IMAGES=$(date -d "60 days ago" -u +%Y-%m-%dT%H:%M:%S%z) attempt=1 - max_retries=5 + max_retries=3 while [ "$attempt" -le "$max_retries" ]; do echo "--- Execution Attempt ${attempt} of ${max_retries} ---" From 1aa1499ce339186a978bfa3c6876800724ce2c60 Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Sat, 27 Dec 2025 09:17:15 +0000 Subject: [PATCH 17/19] Cleaning up code --- tools/cleanup.sh | 13 +++---------- tools/cloud-build/project-cleanup.yaml | 2 +- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/tools/cleanup.sh b/tools/cleanup.sh index dc485b6729..8aae0b7227 100755 --- a/tools/cleanup.sh +++ b/tools/cleanup.sh @@ -11,13 +11,6 @@ declare -A PROTECTED_IPS # To store Network URIs used by protected instances declare -A PROTECTED_NETWORK_URIS -# Environment Variables expected from Cloud Build -PROJECT_ID="${PROJECT_ID:-hpc-toolkit-dev}" -DRY_RUN="${DRY_RUN:-true}" -EXCLUSION_FILE="${EXCLUSION_FILE:-gs://hpc-ctk1357/cleanup/exclusions.txt}" -CUTOFF_TIME="${CUTOFF_TIME:-$(date -d '2 hours ago' -u +%Y-%m-%dT%H:%M:%S%z)}" -CUTOFF_TIME_IMAGES="${CUTOFF_TIME_IMAGES:-$(date -d "60 days ago" -u +%Y-%m-%dT%H:%M:%S%z)}" - # HELPER FUNCTIONS log() { @@ -72,7 +65,7 @@ load_exclusions() { } -# Returns 0 if EXCLUDED (do NOT delete) +# Returns 0 if EXCLUDED (DO NOT delete) # Returns 1 if NOT excluded (OK to delete) is_excluded() { local resource_name="$1" @@ -179,7 +172,7 @@ populate_protected_resources() { else while IFS=$'\t' read -r cluster_name location labels_str; do if ! is_excluded "$cluster_name" "$labels_str"; then # Returns 1 if NOT excluded - continue + continue fi log "INFO" "GKE Cluster ${cluster_name} in ${location} is PROTECTED." @@ -965,7 +958,7 @@ main() { # --- Phase 3: Network Infrastructure --- process_routers - process_firewalls + # process_firewalls process_addresses process_resources "Zonal Disk" \ "gcloud compute disks list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND zone:*\" --format=\"csv[no-heading](name,zone.basename(),labels.map())\" | sort" \ diff --git a/tools/cloud-build/project-cleanup.yaml b/tools/cloud-build/project-cleanup.yaml index 9983431026..bfa65cc7d8 100644 --- a/tools/cloud-build/project-cleanup.yaml +++ b/tools/cloud-build/project-cleanup.yaml @@ -13,7 +13,7 @@ # limitations under the License. --- - +timeout: 28800s steps: - name: gcr.io/cloud-builders/gcloud entrypoint: /bin/bash From 60b5b3f26bae5c63a01ec85aecfd37f1c8ec3f86 Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Sat, 27 Dec 2025 16:04:43 +0000 Subject: [PATCH 18/19] Firewall remove --- tools/cleanup.sh | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/tools/cleanup.sh b/tools/cleanup.sh index 8aae0b7227..b2fd07d354 100755 --- a/tools/cleanup.sh +++ b/tools/cleanup.sh @@ -721,38 +721,6 @@ process_docker_images() { done <<< "$images_output" } -process_firewalls() { - log "INFO" "--- Processing: Firewall Rules ---" - local fws - if ! fws=$(gcloud compute firewall-rules list --project="$PROJECT_ID" \ - --filter="creationTimestamp < '$CUTOFF_TIME'" \ - --format="value(name,network,labels.map())" | sort); then - log "ERROR" "Failed to list firewall rules" - ((ERROR_COUNT++)) || true - return 0 - fi - if [[ -z "$fws" ]]; then log "INFO" "No Firewall Rules found matching criteria."; return 0; fi - - local count=0 - while IFS=$'\t' read -r name network_uri labels_str; do - [[ -z "$name" ]] && continue - local network_name - network_name=$(basename "$network_uri") - if [[ "$network_name" == "default" ]]; then continue; fi - - if [[ -n "${EXCLUSION_MAP[${network_name}]:-}" ]]; then - log "SKIP" "Firewall Rule $name - Network $network_name is protected." - continue - fi - - if ! is_excluded "$name" "${labels_str:-}"; then - execute_delete "Firewall Rule" "$name" \ - "gcloud compute firewall-rules delete \"$name\" --project=\"$PROJECT_ID\" --quiet" - ((count++)) || true - fi - done <<< "$fws" -} - process_filestore() { log "INFO" "--- Processing: Filestore Instances ---" local fs_data @@ -958,7 +926,6 @@ main() { # --- Phase 3: Network Infrastructure --- process_routers - # process_firewalls process_addresses process_resources "Zonal Disk" \ "gcloud compute disks list --project=\"$PROJECT_ID\" --filter=\"creationTimestamp < '$CUTOFF_TIME' AND zone:*\" --format=\"csv[no-heading](name,zone.basename(),labels.map())\" | sort" \ From bc07c80752aff50d9386b005f3674afd9a3c53ff Mon Sep 17 00:00:00 2001 From: simrankaurb Date: Mon, 29 Dec 2025 06:58:30 +0000 Subject: [PATCH 19/19] Exclusion file separate --- tools/cloud-build/project-cleanup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cloud-build/project-cleanup.yaml b/tools/cloud-build/project-cleanup.yaml index bfa65cc7d8..64a47bd5e3 100644 --- a/tools/cloud-build/project-cleanup.yaml +++ b/tools/cloud-build/project-cleanup.yaml @@ -21,7 +21,7 @@ steps: - "BUILD_ID=${BUILD_ID}" - "PROJECT_ID=${PROJECT_ID}" - "DRY_RUN=false" - - "EXCLUSION_FILE=gs://hpc-ctk1357/cleanup/exclusions.txt" + - "EXCLUSION_FILE=gs://hpc-ctk1357/cleanup/${PROJECT_ID}-exclusions.txt" args: - -c - |